@netlify/axis 0.3.1 → 1.1.2

package/dist/scoring/goal-achievement.js

@@ -1,8 +1,6 @@
-import * as fs from "node:fs";
-import * as os from "node:os";
-import * as path from "node:path";
-import { getAdapter } from "../adapters/registry.js";
+import { callJudge } from "./judge.js";
 import { parseJsonFromText } from "./parse-json.js";
+import { getPromptTemplates, interpolate } from "./prompt-templates.js";
 export async function scoreGoalAchievement(result, normalizedEntries) {
     const { rubric } = result;
     const { result: finalResult } = result.output;
@@ -16,7 +14,10 @@ export async function scoreGoalAchievement(result, normalizedEntries) {
 }
 async function scoreStringRubric(runResult, rubric, entries, finalResult) {
     const prompt = buildStringRubricPrompt(runResult, entries, finalResult, rubric);
-    const responseText = await callJudge(runResult, prompt);
+    const responseText = await callJudge(runResult, prompt, {
+        scenarioKey: "__judge__",
+        scenarioName: "AXIS Judge",
+    });
     const parsed = parseJsonFromText(responseText);
     if (!parsed || typeof parsed.score !== "number") {
         return {
@@ -46,115 +47,38 @@ async function scoreStringRubric(runResult, rubric, entries, finalResult) {
 }
 async function scoreArrayRubric(runResult, rubric, entries, finalResult) {
     const prompt = buildArrayRubricPrompt(runResult, entries, finalResult, rubric);
-    const responseText = await callJudge(runResult, prompt);
+    const responseText = await callJudge(runResult, prompt, {
+        scenarioKey: "__judge__",
+        scenarioName: "AXIS Judge",
+    });
     const criteria = parseArrayJudgeResponse(responseText, rubric);
     const score = computeWeightedScore(criteria);
     return { score, criteria };
 }
-async function callJudge(runResult, prompt) {
-    const adapter = getAdapter(runResult.agentConfig.adapter);
-    const workspace = fs.mkdtempSync(path.join(os.tmpdir(), "axis-judge-"));
-    try {
-        const output = await adapter.run({
-            prompt,
-            config: runResult.agentConfig,
-            scenario: {
-                key: "__judge__",
-                name: "AXIS Judge",
-                prompt,
-                rubric: [],
-            },
-            workingDirectory: workspace,
-        });
-        return output.result ?? "";
-    }
-    finally {
-        try {
-            fs.rmSync(workspace, { recursive: true, force: true });
-        }
-        catch {
-            /* ignore */
-        }
-    }
-}
 /** Max characters for the condensed transcript section. */
 const MAX_TRANSCRIPT_CHARS = 50_000;
 /** Max characters per individual transcript entry. */
 const MAX_ENTRY_CHARS = 2_000;
 function buildStringRubricPrompt(result, entries, finalResult, rubric) {
-    return `You are an expert evaluator for an AI agent testing framework called AXIS.
-
-An AI agent was given a task. You must evaluate how well it performed by reviewing its transcript AND by independently verifying the results yourself.
-
-SCENARIO: ${result.scenarioName}
-
-TASK GIVEN TO AGENT:
-${getOriginalPrompt(result)}
-
----
-
-AGENT TRANSCRIPT (condensed):
-${formatTranscriptForJudge(entries)}
-
----
-
-AGENT'S FINAL RESULT:
-${finalResult ?? "(no final result)"}
-
----
-
-RUBRIC:
-${rubric}
-
----
-
-INSTRUCTIONS:
-1. Review the transcript to understand what the agent did.
-2. Where possible, independently verify the results — visit URLs, check endpoints, confirm that the claimed outcomes actually exist. Do not trust the transcript alone.
-3. Score based on what you can verify, not just what the agent claims.
-
-When done, respond with ONLY valid JSON on its own line:
-{"score": <0-10>, "rationale": "<1-2 sentence explanation>"}
-
-Score guide: 0 = not met at all, 5 = partially met, 10 = fully met.`;
+    const { goal_string_rubric } = getPromptTemplates();
+    return interpolate(goal_string_rubric.template, {
+        scenarioName: result.scenarioName,
+        prompt: getOriginalPrompt(result),
+        transcript: formatTranscriptForJudge(entries),
+        finalResult: finalResult ?? "(no final result)",
+        rubric,
+    });
 }
 function buildArrayRubricPrompt(result, entries, finalResult, rubric) {
     const rubricText = rubric.map((r, i) => `${i}. "${r.check}" (weight: ${r.weight})`).join("\n");
-    return `You are an expert evaluator for an AI agent testing framework called AXIS.
-
-An AI agent was given a task. You must evaluate how well it performed by reviewing its transcript AND by independently verifying the results yourself.
-
-SCENARIO: ${result.scenarioName}
-
-TASK GIVEN TO AGENT:
-${getOriginalPrompt(result)}
-
----
-
-AGENT TRANSCRIPT (condensed):
-${formatTranscriptForJudge(entries)}
-
----
-
-AGENT'S FINAL RESULT:
-${finalResult ?? "(no final result)"}
-
----
-
-RUBRIC CRITERIA:
-${rubricText}
-
----
-
-INSTRUCTIONS:
-1. Review the transcript to understand what the agent did.
-2. Where possible, independently verify the results — visit URLs, check endpoints, confirm that the claimed outcomes actually exist. Do not trust the transcript alone.
-3. For each criterion, provide a score from 0 to 10 and a brief rationale.
-
-Score guide: 0 = not met at all, 5 = partially met, 10 = fully met.
-
-When done, respond with ONLY valid JSON on its own line:
-{"grades": [{"criterion_index": 0, "score": <0-10>, "rationale": "<string>"}, ...]}`;
+    const { goal_array_rubric } = getPromptTemplates();
+    return interpolate(goal_array_rubric.template, {
+        scenarioName: result.scenarioName,
+        prompt: getOriginalPrompt(result),
+        transcript: formatTranscriptForJudge(entries),
+        finalResult: finalResult ?? "(no final result)",
+        rubricText,
+    });
 }
 function getOriginalPrompt(result) {
     return result.prompt;

package/dist/scoring/goal-achievement.js.map

@@ -1 +1 @@
-{"version":3,"file":"goal-achievement.js","sourceRoot":"","sources":["../../src/scoring/goal-achievement.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAKrD,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAEpD,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,MAAiB,EACjB,iBAAoC;IAEpC,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,CAAC;IAC1B,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC;IAE9C,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC/B,OAAO,iBAAiB,CAAC,MAAM,EAAE,MAAM,EAAE,iBAAiB,EAAE,WAAW,CAAC,CAAC;IAC3E,CAAC;IAED,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACnC,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;IACpC,CAAC;IAED,OAAO,gBAAgB,CAAC,MAAM,EAAE,MAAM,EAAE,iBAAiB,EAAE,WAAW,CAAC,CAAC;AAC1E,CAAC;AAED,KAAK,UAAU,iBAAiB,CAC9B,SAAoB,EACpB,MAAc,EACd,OAA0B,EAC1B,WAA0B;IAE1B,MAAM,MAAM,GAAG,uBAAuB,CAAC,SAAS,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,CAAC,CAAC;IAChF,MAAM,YAAY,GAAG,MAAM,SAAS,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAExD,MAAM,MAAM,GAAG,iBAAiB,CAAC,YAAY,CAAC,CAAC;IAC/C,IAAI,CAAC,MAAM,IAAI,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;QAChD,OAAO;YACL,KAAK,EAAE,CAAC;YACR,QAAQ,EAAE;gBACR;oBACE,KAAK,EAAE,MAAM;oBACb,MAAM,EAAE,GAAG;oBACX,KAAK,EAAE,CAAC;oBACR,SAAS,EAAE,gCAAgC;iBAC5C;aACF;SACF,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAClE,OAAO;QACL,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC,GAAG,GAAG,CAAC;QACrC,QAAQ,EAAE;YACR;gBACE,KAAK,EAAE,MAAM;gBACb,MAAM,EAAE,GAAG;gBACX,KAAK;gBACL,SAAS,EAAG,MAAM,CAAC,SAAoB,IAAI,EAAE;aAC9C;SACF;KACF,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,gBAAgB,CAC7B,SAAoB,EACpB,MAAyB,EACzB,OAA0B,EAC1B,WAA0B;IAE1B,MAAM,MAAM,GAAG,sBAAsB,CAAC,SAAS,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,CAAC,CAAC;IAC/E,MAAM,YAAY,GAAG,MAAM,SAAS,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAExD,MAAM,QAAQ,GAAG,uBAAuB,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;IAC/D,MAAM,KAAK,GAAG,oBAAoB,CAAC,QAAQ,CAAC,CAAC;IAE7C,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;AAC7B,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,SAAoB,EAAE,MAAc;IAC3D,MAAM,OAAO,GAAG,UAAU,CAAC,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE1D,MAAM,SAAS,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,aAAa,CAAC,CAAC,CAAC;IACxE,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAC/B,MAAM;YACN,MAAM,EAAE,SAAS,CAAC,WAAW;YAC7B,QAAQ,EAAE;gBACR,GAAG,EAAE,WAAW;gBAChB,IAAI,EAAE,YAAY;gBAClB,MAAM;gBACN,MAAM,EAAE,EAAE;aACX;YACD,gBAAgB,EAAE,SAAS;SAC5B,CAAC,CAAC;QACH,OAAO,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC;IAC7B,CAAC;YAAS,CAAC;QACT,IAAI,CAAC;YACH,EAAE,CAAC,MAAM,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACzD,CAAC;QAAC,MAAM,CAAC;YACP,YAAY;QACd,CAAC;IACH,CAAC;AACH,CAAC;AAED,2DAA2D;AAC3D,MAAM,oBAAoB,GAAG,MAAM,CAAC;AAEpC,sDAAsD;AACtD,MAAM,eAAe,GAAG,KAAK,CAAC;AAE9B,SAAS,uBAAuB,CAC9B,MAAiB,EACjB,OAA0B,EAC1B,WAA0B,EAC1B,MAAc;IAEd,OAAO;;;;YAIG,MAAM,CAAC,YAAY;;;EAG7B,iBAAiB,CAAC,MAAM,CAAC;;;;;EAKzB,wBAAwB,CAAC,OAAO,CAAC;;;;;EAKjC,WAAW,IAAI,mBAAmB;;;;;EAKlC,MAAM;;;;;;;;;;;;oEAY4D,CAAC;AACrE,CAAC;AAED,SAAS,sBAAsB,CAC7B,MAAiB,EACjB,OAA0B,EAC1B,WAA0B,EAC1B,MAAyB;IAEzB,MAAM,UAAU,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,KAAK,cAAc,CAAC,CAAC,MAAO,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAEhG,OAAO;;;;YAIG,MAAM,CAAC,YAAY;;;EAG7B,iBAAiB,CAAC,MAAM,CAAC;;;;;EAKzB,wBAAwB,CAAC,OAAO,CAAC;;;;;EAKjC,WAAW,IAAI,mBAAmB;;;;;EAKlC,UAAU;;;;;;;;;;;;oFAYwE,CAAC;AACrF,CAAC;AAED,SAAS,iBAAiB,CAAC,MAAiB;IAC1C,OAAO,MAAM,CAAC,MAAM,CAAC;AACvB,CAAC;AAED;;GAEG;AACH,SAAS,wBAAwB,CAAC,OAA0B;IAC1D,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,oBAAoB,CAAC;IAEtD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,MAAM,SAAS,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;QAEnD,IAAI,UAAU,GAAG,SAAS,CAAC,MAAM,GAAG,oBAAoB,EAAE,CAAC;YACzD,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;YACrC,KAAK,CAAC,IAAI,CAAC,UAAU,SAAS,sCAAsC,CAAC,CAAC;YACtE,MAAM;QACR,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACtB,UAAU,IAAI,SAAS,CAAC,MAAM,CAAC;IACjC,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED;;GAEG;AACH,SAAS,aAAa,CAAC,KAAsB,EAAE,KAAa;IAC1D,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;QACnB,KAAK,WAAW;YACd,OAAO,IAAI,KAAK,gBAAgB,QAAQ,CAAC,KAAK,CAAC,IAAI,IAAI,WAAW,EAAE,eAAe,CAAC,EAAE,CAAC;QACzF,KAAK,UAAU,CAAC,CAAC,CAAC;YAChB,MAAM,IAAI,GAAG,KAAK,CAAC,QAAQ,IAAI,SAAS,CAAC;YACzC,MAAM,KAAK,GAAG,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAC,KAAK,CAAC,gBAAgB,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;YACzF,OAAO,IAAI,KAAK,eAAe,IAAI,GAAG,KAAK,EAAE,CAAC;QAChD,CAAC;QACD,KAAK,aAAa;YAChB,OAAO,IAAI,KAAK,kBAAkB,QAAQ,CAAC,KAAK,CAAC,cAAc,IAAI,aAAa,EAAE,eAAe,CAAC,EAAE,CAAC;QACvG,KAAK,OAAO;YACV,OAAO,IAAI,KAAK,YAAY,QAAQ,CAAC,KAAK,CAAC,YAAY,IAAI,KAAK,CAAC,IAAI,IAAI,iBAAiB,EAAE,eAAe,CAAC,EAAE,CAAC;QACjH,KAAK,QAAQ;YACX,OAAO,IAAI,KAAK,aAAa,QAAQ,CAAC,KAAK,CAAC,IAAI,IAAI,cAAc,EAAE,GAAG,CAAC,EAAE,CAAC;QAC7E,KAAK,MAAM;YACT,OAAO,IAAI,KAAK,WAAW,QAAQ,CAAC,KAAK,CAAC,IAAI,IAAI,cAAc,EAAE,eAAe,CAAC,EAAE,CAAC;QACvF;YACE,OAAO,IAAI,KAAK,KAAK,KAAK,CAAC,IAAI,KAAK,QAAQ,CAAC,KAAK,CAAC,IAAI,IAAI,cAAc,EAAE,GAAG,CAAC,EAAE,CAAC;IACtF,CAAC;AACH,CAAC;AAED,SAAS,QAAQ,CAAC,IAAY,EAAE,MAAc;IAC5C,IAAI,IAAI,CAAC,MAAM,IAAI,MAAM;QAAE,OAAO,IAAI,CAAC;IACvC,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,GAAG,KAAK,CAAC;AACvC,CAAC;AAED,SAAS,uBAAuB,CAAC,YAAoB,EAAE,MAAyB;IAC9E,MAAM,MAAM,GAAG,iBAAiB,CAAC,YAAY,CAAC,CAAC;IAC/C,IAAI,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;QAC7C,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACxB,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,MAAM,EAAE,CAAC,CAAC,MAAO;YACjB,KAAK,EAAE,CAAC;YACR,SAAS,EAAE,gCAAgC;SAC5C,CAAC,CAAC,CAAC;IACN,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,CAAC,MAIpB,CAAC;IAEH,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACzB,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,eAAe,KAAK,CAAC,CAAC,CAAC;QAC1D,OAAO;YACL,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,MAAM,EAAE,CAAC,CAAC,MAAO;YACjB,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACrE,SAAS,EAAE,KAAK,EAAE,SAAS,IAAI,mBAAmB;SACnD,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,oBAAoB,CAAC,QAA0B;IACtD,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAEpC,MAAM,WAAW,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACnE,IAAI,WAAW,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAEhC,MAAM,WAAW,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAEpF,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,WAAW,GAAG,WAAW,CAAC,GAAG,GAAG,CAAC,CAAC;AACvD,CAAC"}
+{"version":3,"file":"goal-achievement.js","sourceRoot":"","sources":["../../src/scoring/goal-achievement.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AACvC,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AACpD,OAAO,EAAE,kBAAkB,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAExE,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,MAAiB,EACjB,iBAAoC;IAEpC,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,CAAC;IAC1B,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC;IAE9C,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC/B,OAAO,iBAAiB,CAAC,MAAM,EAAE,MAAM,EAAE,iBAAiB,EAAE,WAAW,CAAC,CAAC;IAC3E,CAAC;IAED,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACnC,OAAO,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;IACpC,CAAC;IAED,OAAO,gBAAgB,CAAC,MAAM,EAAE,MAAM,EAAE,iBAAiB,EAAE,WAAW,CAAC,CAAC;AAC1E,CAAC;AAED,KAAK,UAAU,iBAAiB,CAC9B,SAAoB,EACpB,MAAc,EACd,OAA0B,EAC1B,WAA0B;IAE1B,MAAM,MAAM,GAAG,uBAAuB,CAAC,SAAS,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,CAAC,CAAC;IAChF,MAAM,YAAY,GAAG,MAAM,SAAS,CAAC,SAAS,EAAE,MAAM,EAAE;QACtD,WAAW,EAAE,WAAW;QACxB,YAAY,EAAE,YAAY;KAC3B,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,iBAAiB,CAAC,YAAY,CAAC,CAAC;IAC/C,IAAI,CAAC,MAAM,IAAI,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;QAChD,OAAO;YACL,KAAK,EAAE,CAAC;YACR,QAAQ,EAAE;gBACR;oBACE,KAAK,EAAE,MAAM;oBACb,MAAM,EAAE,GAAG;oBACX,KAAK,EAAE,CAAC;oBACR,SAAS,EAAE,gCAAgC;iBAC5C;aACF;SACF,CAAC;IACJ,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;IAClE,OAAO;QACL,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC,GAAG,GAAG,CAAC;QACrC,QAAQ,EAAE;YACR;gBACE,KAAK,EAAE,MAAM;gBACb,MAAM,EAAE,GAAG;gBACX,KAAK;gBACL,SAAS,EAAG,MAAM,CAAC,SAAoB,IAAI,EAAE;aAC9C;SACF;KACF,CAAC;AACJ,CAAC;AAED,KAAK,UAAU,gBAAgB,CAC7B,SAAoB,EACpB,MAAyB,EACzB,OAA0B,EAC1B,WAA0B;IAE1B,MAAM,MAAM,GAAG,sBAAsB,CAAC,SAAS,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,CAAC,CAAC;IAC/E,MAAM,YAAY,GAAG,MAAM,SAAS,CAAC,SAAS,EAAE,MAAM,EAAE;QACtD,WAAW,EAAE,WAAW;QACxB,YAAY,EAAE,YAAY;KAC3B,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAG,uBAAuB,CAAC,YAAY,EAAE,MAAM,CAAC,CAAC;IAC/D,MAAM,KAAK,GAAG,oBAAoB,CAAC,QAAQ,CAAC,CAAC;IAE7C,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,CAAC;AAC7B,CAAC;AAGD,2DAA2D;AAC3D,MAAM,oBAAoB,GAAG,MAAM,CAAC;AAEpC,sDAAsD;AACtD,MAAM,eAAe,GAAG,KAAK,CAAC;AAE9B,SAAS,uBAAuB,CAC9B,MAAiB,EACjB,OAA0B,EAC1B,WAA0B,EAC1B,MAAc;IAEd,MAAM,EAAE,kBAAkB,EAAE,GAAG,kBAAkB,EAAE,CAAC;IAEpD,OAAO,WAAW,CAAC,kBAAkB,CAAC,QAAQ,EAAE;QAC9C,YAAY,EAAE,MAAM,CAAC,YAAY;QACjC,MAAM,EAAE,iBAAiB,CAAC,MAAM,CAAC;QACjC,UAAU,EAAE,wBAAwB,CAAC,OAAO,CAAC;QAC7C,WAAW,EAAE,WAAW,IAAI,mBAAmB;QAC/C,MAAM;KACP,CAAC,CAAC;AACL,CAAC;AAED,SAAS,sBAAsB,CAC7B,MAAiB,EACjB,OAA0B,EAC1B,WAA0B,EAC1B,MAAyB;IAEzB,MAAM,UAAU,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,KAAK,cAAc,CAAC,CAAC,MAAO,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAChG,MAAM,EAAE,iBAAiB,EAAE,GAAG,kBAAkB,EAAE,CAAC;IAEnD,OAAO,WAAW,CAAC,iBAAiB,CAAC,QAAQ,EAAE;QAC7C,YAAY,EAAE,MAAM,CAAC,YAAY;QACjC,MAAM,EAAE,iBAAiB,CAAC,MAAM,CAAC;QACjC,UAAU,EAAE,wBAAwB,CAAC,OAAO,CAAC;QAC7C,WAAW,EAAE,WAAW,IAAI,mBAAmB;QAC/C,UAAU;KACX,CAAC,CAAC;AACL,CAAC;AAED,SAAS,iBAAiB,CAAC,MAAiB;IAC1C,OAAO,MAAM,CAAC,MAAM,CAAC;AACvB,CAAC;AAED;;GAEG;AACH,SAAS,wBAAwB,CAAC,OAA0B;IAC1D,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,oBAAoB,CAAC;IAEtD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,MAAM,SAAS,GAAG,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;QAEnD,IAAI,UAAU,GAAG,SAAS,CAAC,MAAM,GAAG,oBAAoB,EAAE,CAAC;YACzD,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;YACrC,KAAK,CAAC,IAAI,CAAC,UAAU,SAAS,sCAAsC,CAAC,CAAC;YACtE,MAAM;QACR,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACtB,UAAU,IAAI,SAAS,CAAC,MAAM,CAAC;IACjC,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED;;GAEG;AACH,SAAS,aAAa,CAAC,KAAsB,EAAE,KAAa;IAC1D,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;QACnB,KAAK,WAAW;YACd,OAAO,IAAI,KAAK,gBAAgB,QAAQ,CAAC,KAAK,CAAC,IAAI,IAAI,WAAW,EAAE,eAAe,CAAC,EAAE,CAAC;QACzF,KAAK,UAAU,CAAC,CAAC,CAAC;YAChB,MAAM,IAAI,GAAG,KAAK,CAAC,QAAQ,IAAI,SAAS,CAAC;YACzC,MAAM,KAAK,GAAG,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAC,IAAI,QAAQ,CAAC,KAAK,CAAC,gBAAgB,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;YACzF,OAAO,IAAI,KAAK,eAAe,IAAI,GAAG,KAAK,EAAE,CAAC;QAChD,CAAC;QACD,KAAK,aAAa;YAChB,OAAO,IAAI,KAAK,kBAAkB,QAAQ,CAAC,KAAK,CAAC,cAAc,IAAI,aAAa,EAAE,eAAe,CAAC,EAAE,CAAC;QACvG,KAAK,OAAO;YACV,OAAO,IAAI,KAAK,YAAY,QAAQ,CAAC,KAAK,CAAC,YAAY,IAAI,KAAK,CAAC,IAAI,IAAI,iBAAiB,EAAE,eAAe,CAAC,EAAE,CAAC;QACjH,KAAK,QAAQ;YACX,OAAO,IAAI,KAAK,aAAa,QAAQ,CAAC,KAAK,CAAC,IAAI,IAAI,cAAc,EAAE,GAAG,CAAC,EAAE,CAAC;QAC7E,KAAK,MAAM;YACT,OAAO,IAAI,KAAK,WAAW,QAAQ,CAAC,KAAK,CAAC,IAAI,IAAI,cAAc,EAAE,eAAe,CAAC,EAAE,CAAC;QACvF;YACE,OAAO,IAAI,KAAK,KAAK,KAAK,CAAC,IAAI,KAAK,QAAQ,CAAC,KAAK,CAAC,IAAI,IAAI,cAAc,EAAE,GAAG,CAAC,EAAE,CAAC;IACtF,CAAC;AACH,CAAC;AAED,SAAS,QAAQ,CAAC,IAAY,EAAE,MAAc;IAC5C,IAAI,IAAI,CAAC,MAAM,IAAI,MAAM;QAAE,OAAO,IAAI,CAAC;IACvC,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,MAAM,CAAC,GAAG,KAAK,CAAC;AACvC,CAAC;AAED,SAAS,uBAAuB,CAAC,YAAoB,EAAE,MAAyB;IAC9E,MAAM,MAAM,GAAG,iBAAiB,CAAC,YAAY,CAAC,CAAC;IAC/C,IAAI,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;QAC7C,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YACxB,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,MAAM,EAAE,CAAC,CAAC,MAAO;YACjB,KAAK,EAAE,CAAC;YACR,SAAS,EAAE,gCAAgC;SAC5C,CAAC,CAAC,CAAC;IACN,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,CAAC,MAIpB,CAAC;IAEH,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACzB,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,eAAe,KAAK,CAAC,CAAC,CAAC;QAC1D,OAAO;YACL,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,MAAM,EAAE,CAAC,CAAC,MAAO;YACjB,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACrE,SAAS,EAAE,KAAK,EAAE,SAAS,IAAI,mBAAmB;SACnD,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,oBAAoB,CAAC,QAA0B;IACtD,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAEpC,MAAM,WAAW,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACnE,IAAI,WAAW,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAEhC,MAAM,WAAW,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAEpF,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,WAAW,GAAG,WAAW,CAAC,GAAG,GAAG,CAAC,CAAC;AACvD,CAAC"}

package/dist/scoring/index.d.ts

@@ -2,7 +2,7 @@ import type { RunOutput, RunResult } from "../types/output.js";
 import type { ScoredOutput, ScoredRunResult, ScoringOptions } from "../types/scoring.js";
 /**
  * Score a single run result using the interaction-based evaluation pipeline:
- * normalize → sparse index → triage → deep eval → category score → goal achievement → composite
+ * normalize → sparse index → write raw data → (deep eval || goal achievement) → category score → composite
  */
 export declare function scoreRunResult(result: RunResult, options?: ScoringOptions): Promise<ScoredRunResult>;
 /**

package/dist/scoring/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/scoring/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAE/D,OAAO,KAAK,EAAE,YAAY,EAAE,eAAe,EAAe,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAgBtG;;;GAGG;AACH,wBAAsB,cAAc,CAAC,MAAM,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,eAAe,CAAC,CAkF1G;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,SAAS,EAAE,aAAa,EAAE,eAAe,EAAE,GAAG,YAAY,CAmBtG;AAED;;GAEG;AACH,wBAAsB,YAAY,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,YAAY,CAAC,CAIxG"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/scoring/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAE/D,OAAO,KAAK,EAAE,YAAY,EAAE,eAAe,EAAe,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAgBtG;;;GAGG;AACH,wBAAsB,cAAc,CAAC,MAAM,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,eAAe,CAAC,CAuF1G;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,SAAS,EAAE,SAAS,EAAE,aAAa,EAAE,eAAe,EAAE,GAAG,YAAY,CAmBtG;AAED;;GAEG;AACH,wBAAsB,YAAY,CAAC,SAAS,EAAE,SAAS,EAAE,OAAO,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,YAAY,CAAC,CAIxG"}

package/dist/scoring/index.js

@@ -1,7 +1,7 @@
 import { normalizeTranscript, toTranscriptAnalysis } from "../transcript/normalize.js";
+import { writeScenarioRawData } from "../reports/writer.js";
 import { scoreGoalAchievement } from "./goal-achievement.js";
 import { buildSparseIndex, populateInteractionContent } from "./sparse-index.js";
-import { runTriage } from "./triage.js";
 import { runDeepEval } from "./deep-eval.js";
 import { computeCategoryScore } from "./category-score.js";
 import { computeComposite } from "./composite.js";
@@ -13,7 +13,7 @@ const DEFAULT_WEIGHTS = {
 };
 /**
  * Score a single run result using the interaction-based evaluation pipeline:
- * normalize → sparse index → triage → deep eval → category score → goal achievement → composite
+ * normalize → sparse index → write raw data → (deep eval || goal achievement) → category score → composite
  */
 export async function scoreRunResult(result, options) {
     const weights = options?.weights ?? DEFAULT_WEIGHTS;
@@ -26,13 +26,18 @@ export async function scoreRunResult(result, options) {
     // Step 2: Build sparse index (deterministic) and populate content for reports
     const sparseIndex = buildSparseIndex(normalized);
     populateInteractionContent(sparseIndex, normalized);
-    // Step 3 + 4: Triage → Deep eval (LLM calls), run in parallel with goal achievement
-    const [triageResult, goalAchievement] = await Promise.all([
-        runTriage(result, sparseIndex),
+    // Step 3: Write raw data to report dir so LLM judges can read it
+    if (options?.reportDir) {
+        writeScenarioRawData(options.reportDir, result, sparseIndex);
+    }
+    // Step 4: Deep eval + goal achievement in parallel
+    const [deepEvalResult, goalAchievement] = await Promise.all([
+        runDeepEval(result, sparseIndex, normalized, {
+            weights,
+            reportDir: options?.reportDir,
+        }),
         scoreGoalAchievement(result, normalized.entries),
     ]);
-    // Step 4 continued: Deep eval needs triage results
-    const deepEvalResult = await runDeepEval(result, sparseIndex, triageResult, normalized);
     // Step 5: Compute category scores
     const necessityMap = new Map(deepEvalResult.necessity.map((n) => [n.category, n]));
     const defaultNecessity = (category) => ({

package/dist/scoring/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/scoring/index.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,4BAA4B,CAAC;AACvF,OAAO,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAC7D,OAAO,EAAE,gBAAgB,EAAE,0BAA0B,EAAE,MAAM,mBAAmB,CAAC;AACjF,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,OAAO,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAC3D,OAAO,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAElD,MAAM,eAAe,GAAmB;IACtC,gBAAgB,EAAE,GAAG;IACrB,WAAW,EAAE,GAAG;IAChB,OAAO,EAAE,GAAG;IACZ,KAAK,EAAE,GAAG;CACX,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,MAAiB,EAAE,OAAwB;IAC9E,MAAM,OAAO,GAAG,OAAO,EAAE,OAAO,IAAI,eAAe,CAAC;IACpD,MAAM,MAAM,GAAG,OAAO,EAAE,MAAM,CAAC;IAC/B,MAAM,KAAK,GAAG,GAAG,MAAM,CAAC,WAAW,KAAK,MAAM,CAAC,SAAS,GAAG,CAAC;IAE5D,MAAM,EAAE,OAAO,EAAE,CAAC,WAAW,KAAK,KAAK,CAAC,CAAC;IACzC,OAAO,EAAE,UAAU,EAAE,CAAC,MAAM,CAAC,WAAW,EAAE,MAAM,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;IAErE,qDAAqD;IACrD,MAAM,UAAU,GAAG,mBAAmB,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IAEjE,8EAA8E;IAC9E,MAAM,WAAW,GAAG,gBAAgB,CAAC,UAAU,CAAC,CAAC;IACjD,0BAA0B,CAAC,WAAW,EAAE,UAAU,CAAC,CAAC;IAEpD,oFAAoF;IACpF,MAAM,CAAC,YAAY,EAAE,eAAe,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QACxD,SAAS,CAAC,MAAM,EAAE,WAAW,CAAC;QAC9B,oBAAoB,CAAC,MAAM,EAAE,UAAU,CAAC,OAAO,CAAC;KACjD,CAAC,CAAC;IAEH,mDAAmD;IACnD,MAAM,cAAc,GAAG,MAAM,WAAW,CAAC,MAAM,EAAE,WAAW,EAAE,YAAY,EAAE,UAAU,CAAC,CAAC;IAExF,kCAAkC;IAClC,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,cAAc,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IACnF,MAAM,gBAAgB,GAAG,CAAC,QAA6C,EAAE,EAAE,CAAC,CAAC;QAC3E,QAAQ;QACR,KAAK,EAAE,GAAG;QACV,cAAc,EAAE,EAAc;QAC9B,SAAS,EAAE,SAAS;KACrB,CAAC,CAAC;IAEH,MAAM,WAAW,GAAG,oBAAoB,CACtC,aAAa,EACb,cAAc,CAAC,MAAM,EACrB,YAAY,CAAC,GAAG,CAAC,aAAa,CAAC,IAAI,gBAAgB,CAAC,aAAa,CAAC,EAClE,WAAW,CAAC,YAAY,CACzB,CAAC;IAEF,MAAM,OAAO,GAAG,oBAAoB,CAClC,SAAS,EACT,cAAc,CAAC,MAAM,EACrB,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,gBAAgB,CAAC,SAAS,CAAC,EAC1D,WAAW,CAAC,YAAY,CACzB,CAAC;IAEF,MAAM,KAAK,GAAG,oBAAoB,CAChC,OAAO,EACP,cAAc,CAAC,MAAM,EACrB,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,gBAAgB,CAAC,OAAO,CAAC,EACtD,WAAW,CAAC,YAAY,CACzB,CAAC;IAEF,uCAAuC;IACvC,MAAM,SAAS,GAAG,gBAAgB,CAAC,eAAe,CAAC,KAAK,EAAE,WAAW,CAAC,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IAElH,MAAM,KAAK,GAAgB;QACzB,SAAS;QACT,eAAe;QACf,WAAW;QACX,OAAO;QACP,KAAK;QACL,OAAO;QACP,WAAW;KACZ,CAAC;IAEF,OAAO,EAAE,UAAU,EAAE,CAAC,MAAM,CAAC,WAAW,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAEpE,sEAAsE;IACtE,MAAM,CAAC,MAAM,CAAC,kBAAkB,GAAG,oBAAoB,CAAC,UAAU,CAAC,CAAC;IAEpE,OAAO;QACL,WAAW,EAAE,MAAM,CAAC,WAAW;QAC/B,YAAY,EAAE,MAAM,CAAC,YAAY;QACjC,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,WAAW,EAAE,MAAM,CAAC,WAAW;QAC/B,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,KAAK;KACN,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAAC,SAAoB,EAAE,aAAgC;IACtF,MAAM,gBAAgB,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,KAAK,CAAC,CAAC,CAAC;IACvF,MAAM,gBAAgB,GACpB,gBAAgB,CAAC,MAAM,GAAG,CAAC;QACzB,CAAC,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC,GAAG,gBAAgB,CAAC,MAAM;QAC3F,CAAC,CAAC,CAAC,CAAC;IAER,OAAO;QACL,OAAO,EAAE,SAAS,CAAC,OAAO;QAC1B,SAAS,EAAE,SAAS,CAAC,SAAS;QAC9B,UAAU,EAAE,SAAS,CAAC,UAAU;QAChC,OAAO,EAAE,aAAa;QACtB,OAAO,EAAE;YACP,KAAK,EAAE,SAAS,CAAC,OAAO,CAAC,KAAK;YAC9B,SAAS,EAAE,SAAS,CAAC,OAAO,CAAC,SAAS;YACtC,MAAM,EAAE,SAAS,CAAC,OAAO,CAAC,MAAM;YAChC,gBAAgB,EAAE,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC;SAC/C;KACF,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,SAAoB,EAAE,OAAwB;IAC/E,MAAM,aAAa,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,cAAc,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;IAElG,OAAO,iBAAiB,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC;AACrD,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/scoring/index.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,4BAA4B,CAAC;AACvF,OAAO,EAAE,oBAAoB,EAAE,MAAM,sBAAsB,CAAC;AAC5D,OAAO,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAC7D,OAAO,EAAE,gBAAgB,EAAE,0BAA0B,EAAE,MAAM,mBAAmB,CAAC;AACjF,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,OAAO,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAC3D,OAAO,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAElD,MAAM,eAAe,GAAmB;IACtC,gBAAgB,EAAE,GAAG;IACrB,WAAW,EAAE,GAAG;IAChB,OAAO,EAAE,GAAG;IACZ,KAAK,EAAE,GAAG;CACX,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,MAAiB,EAAE,OAAwB;IAC9E,MAAM,OAAO,GAAG,OAAO,EAAE,OAAO,IAAI,eAAe,CAAC;IACpD,MAAM,MAAM,GAAG,OAAO,EAAE,MAAM,CAAC;IAC/B,MAAM,KAAK,GAAG,GAAG,MAAM,CAAC,WAAW,KAAK,MAAM,CAAC,SAAS,GAAG,CAAC;IAE5D,MAAM,EAAE,OAAO,EAAE,CAAC,WAAW,KAAK,KAAK,CAAC,CAAC;IACzC,OAAO,EAAE,UAAU,EAAE,CAAC,MAAM,CAAC,WAAW,EAAE,MAAM,CAAC,SAAS,EAAE,OAAO,CAAC,CAAC;IAErE,qDAAqD;IACrD,MAAM,UAAU,GAAG,mBAAmB,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IAEjE,8EAA8E;IAC9E,MAAM,WAAW,GAAG,gBAAgB,CAAC,UAAU,CAAC,CAAC;IACjD,0BAA0B,CAAC,WAAW,EAAE,UAAU,CAAC,CAAC;IAEpD,iEAAiE;IACjE,IAAI,OAAO,EAAE,SAAS,EAAE,CAAC;QACvB,oBAAoB,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,EAAE,WAAW,CAAC,CAAC;IAC/D,CAAC;IAED,mDAAmD;IACnD,MAAM,CAAC,cAAc,EAAE,eAAe,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QAC1D,WAAW,CAAC,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE;YAC3C,OAAO;YACP,SAAS,EAAE,OAAO,EAAE,SAAS;SAC9B,CAAC;QACF,oBAAoB,CAAC,MAAM,EAAE,UAAU,CAAC,OAAO,CAAC;KACjD,CAAC,CAAC;IAEH,kCAAkC;IAClC,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,cAAc,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IACnF,MAAM,gBAAgB,GAAG,CAAC,QAA6C,EAAE,EAAE,CAAC,CAAC;QAC3E,QAAQ;QACR,KAAK,EAAE,GAAG;QACV,cAAc,EAAE,EAAc;QAC9B,SAAS,EAAE,SAAS;KACrB,CAAC,CAAC;IAEH,MAAM,WAAW,GAAG,oBAAoB,CACtC,aAAa,EACb,cAAc,CAAC,MAAM,EACrB,YAAY,CAAC,GAAG,CAAC,aAAa,CAAC,IAAI,gBAAgB,CAAC,aAAa,CAAC,EAClE,WAAW,CAAC,YAAY,CACzB,CAAC;IAEF,MAAM,OAAO,GAAG,oBAAoB,CAClC,SAAS,EACT,cAAc,CAAC,MAAM,EACrB,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,gBAAgB,CAAC,SAAS,CAAC,EAC1D,WAAW,CAAC,YAAY,CACzB,CAAC;IAEF,MAAM,KAAK,GAAG,oBAAoB,CAChC,OAAO,EACP,cAAc,CAAC,MAAM,EACrB,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,gBAAgB,CAAC,OAAO,CAAC,EACtD,WAAW,CAAC,YAAY,CACzB,CAAC;IAEF,uCAAuC;IACvC,MAAM,SAAS,GAAG,gBAAgB,CAAC,eAAe,CAAC,KAAK,EAAE,WAAW,CAAC,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IAElH,MAAM,KAAK,GAAgB;QACzB,SAAS;QACT,eAAe;QACf,WAAW;QACX,OAAO;QACP,KAAK;QACL,OAAO;QACP,WAAW;KACZ,CAAC;IAEF,OAAO,EAAE,UAAU,EAAE,CAAC,MAAM,CAAC,WAAW,EAAE,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAEpE,sEAAsE;IACtE,MAAM,CAAC,MAAM,CAAC,kBAAkB,GAAG,oBAAoB,CAAC,UAAU,CAAC,CAAC;IAEpE,OAAO;QACL,WAAW,EAAE,MAAM,CAAC,WAAW;QAC/B,YAAY,EAAE,MAAM,CAAC,YAAY;QACjC,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,WAAW,EAAE,MAAM,CAAC,WAAW;QAC/B,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,KAAK;KACN,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,iBAAiB,CAAC,SAAoB,EAAE,aAAgC;IACtF,MAAM,gBAAgB,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,KAAK,CAAC,CAAC,CAAC;IACvF,MAAM,gBAAgB,GACpB,gBAAgB,CAAC,MAAM,GAAG,CAAC;QACzB,CAAC,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC,GAAG,gBAAgB,CAAC,MAAM;QAC3F,CAAC,CAAC,CAAC,CAAC;IAER,OAAO;QACL,OAAO,EAAE,SAAS,CAAC,OAAO;QAC1B,SAAS,EAAE,SAAS,CAAC,SAAS;QAC9B,UAAU,EAAE,SAAS,CAAC,UAAU;QAChC,OAAO,EAAE,aAAa;QACtB,OAAO,EAAE;YACP,KAAK,EAAE,SAAS,CAAC,OAAO,CAAC,KAAK;YAC9B,SAAS,EAAE,SAAS,CAAC,OAAO,CAAC,SAAS;YACtC,MAAM,EAAE,SAAS,CAAC,OAAO,CAAC,MAAM;YAChC,gBAAgB,EAAE,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC;SAC/C;KACF,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,SAAoB,EAAE,OAAwB;IAC/E,MAAM,aAAa,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,cAAc,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC;IAElG,OAAO,iBAAiB,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC;AACrD,CAAC"}

package/dist/scoring/judge.d.ts

@@ -0,0 +1,17 @@
+import type { RunResult } from "../types/output.js";
+export interface JudgeCallOptions {
+    /** Scenario key for the judge run (e.g., "__env_eval__"). */
+    scenarioKey: string;
+    /** Human-readable name for the judge run. */
+    scenarioName: string;
+}
+/**
+ * Call an LLM judge using the same adapter as the test run.
+ *
+ * Uses the agent's original workspace when available so the judge can
+ * independently verify the agent's actual work (files created, endpoints
+ * deployed, etc.). Falls back to a disposable temp directory only when
+ * no workspace is set (e.g. programmatic API usage without the runner).
+ */
+export declare function callJudge(runResult: RunResult, prompt: string, options: JudgeCallOptions): Promise<string>;
+//# sourceMappingURL=judge.d.ts.map

package/dist/scoring/judge.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"judge.d.ts","sourceRoot":"","sources":["../../src/scoring/judge.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAEpD,MAAM,WAAW,gBAAgB;IAC/B,6DAA6D;IAC7D,WAAW,EAAE,MAAM,CAAC;IACpB,6CAA6C;IAC7C,YAAY,EAAE,MAAM,CAAC;CACtB;AAED;;;;;;;GAOG;AACH,wBAAsB,SAAS,CAC7B,SAAS,EAAE,SAAS,EACpB,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,gBAAgB,GACxB,OAAO,CAAC,MAAM,CAAC,CA6BjB"}

package/dist/scoring/judge.js

@@ -0,0 +1,43 @@
+import * as fs from "node:fs";
+import * as os from "node:os";
+import * as path from "node:path";
+import { getAdapter } from "../adapters/registry.js";
+/**
+ * Call an LLM judge using the same adapter as the test run.
+ *
+ * Uses the agent's original workspace when available so the judge can
+ * independently verify the agent's actual work (files created, endpoints
+ * deployed, etc.). Falls back to a disposable temp directory only when
+ * no workspace is set (e.g. programmatic API usage without the runner).
+ */
+export async function callJudge(runResult, prompt, options) {
+    const adapter = getAdapter(runResult.agentConfig.adapter);
+    const originalWorkspace = runResult.workingDirectory;
+    const workspace = originalWorkspace ?? fs.mkdtempSync(path.join(os.tmpdir(), `axis-${options.scenarioKey}-`));
+    const shouldCleanup = !originalWorkspace;
+    try {
+        const output = await adapter.run({
+            prompt,
+            config: runResult.agentConfig,
+            scenario: {
+                key: options.scenarioKey,
+                name: options.scenarioName,
+                prompt,
+                rubric: [],
+            },
+            workingDirectory: workspace,
+        });
+        return output.result ?? "";
+    }
+    finally {
+        if (shouldCleanup) {
+            try {
+                fs.rmSync(workspace, { recursive: true, force: true });
+            }
+            catch {
+                /* ignore */
+            }
+        }
+    }
+}
+//# sourceMappingURL=judge.js.map

package/dist/scoring/judge.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"judge.js","sourceRoot":"","sources":["../../src/scoring/judge.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAUrD;;;;;;;GAOG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,SAAoB,EACpB,MAAc,EACd,OAAyB;IAEzB,MAAM,OAAO,GAAG,UAAU,CAAC,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAC1D,MAAM,iBAAiB,GAAG,SAAS,CAAC,gBAAgB,CAAC;IAErD,MAAM,SAAS,GAAG,iBAAiB,IAAI,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,QAAQ,OAAO,CAAC,WAAW,GAAG,CAAC,CAAC,CAAC;IAC9G,MAAM,aAAa,GAAG,CAAC,iBAAiB,CAAC;IAEzC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;YAC/B,MAAM;YACN,MAAM,EAAE,SAAS,CAAC,WAAW;YAC7B,QAAQ,EAAE;gBACR,GAAG,EAAE,OAAO,CAAC,WAAW;gBACxB,IAAI,EAAE,OAAO,CAAC,YAAY;gBAC1B,MAAM;gBACN,MAAM,EAAE,EAAE;aACX;YACD,gBAAgB,EAAE,SAAS;SAC5B,CAAC,CAAC;QACH,OAAO,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC;IAC7B,CAAC;YAAS,CAAC;QACT,IAAI,aAAa,EAAE,CAAC;YAClB,IAAI,CAAC;gBACH,EAAE,CAAC,MAAM,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YACzD,CAAC;YAAC,MAAM,CAAC;gBACP,YAAY;YACd,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC"}

package/dist/scoring/prompt-templates.d.ts

@@ -0,0 +1,50 @@
+/**
+ * Declarative prompt templates for the AXIS scoring pipeline.
+ *
+ * Each template uses `{{variable}}` placeholders that are substituted at
+ * runtime via `interpolate()`. The raw templates (with placeholders intact)
+ * are exposed via `getPromptTemplates()` so documentation UIs can display
+ * the exact prompts used during scoring.
+ */
+/** Describes a single placeholder variable within a prompt template. */
+export interface PromptVariable {
+    /** The placeholder name as it appears in `{{name}}`. */
+    name: string;
+    /** Human-readable description of what this variable contains. */
+    description: string;
+    /** Type hint for documentation purposes — not enforced at runtime. */
+    type: "string" | "number" | "text";
+    /** Whether the variable may legitimately resolve to an empty string. */
+    optional?: boolean;
+}
+/** A self-describing prompt template for one stage of the scoring pipeline. */
+export interface PromptTemplate {
+    /** Unique identifier (also the record key returned by `getPromptTemplates`). */
+    name: string;
+    /** Human-readable description of what the prompt does. */
+    description: string;
+    /** Which scoring pipeline stage uses this prompt. */
+    stage: "deep_eval" | "goal_achievement";
+    /** The template string with `{{variable}}` placeholders. */
+    template: string;
+    /** Metadata about each placeholder the template accepts. */
+    variables: PromptVariable[];
+}
+/**
+ * Replace `{{key}}` placeholders in `template` with values from `vars`.
+ *
+ * Throws if any placeholder in the template has no corresponding key in
+ * `vars`. Numbers are coerced to strings via `String()`.
+ */
+export declare function interpolate(template: string, vars: Record<string, string | number>): string;
+/** Per-category evaluation guidance for focused judge prompts. */
+export declare const CATEGORY_GUIDANCE: Record<string, string>;
+/**
+ * Return all scoring prompt templates keyed by name.
+ *
+ * The templates contain `{{variable}}` placeholders — use `interpolate()`
+ * to substitute runtime values, or display the raw template text in
+ * documentation UIs.
+ */
+export declare function getPromptTemplates(): Record<string, PromptTemplate>;
+//# sourceMappingURL=prompt-templates.d.ts.map

package/dist/scoring/prompt-templates.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"prompt-templates.d.ts","sourceRoot":"","sources":["../../src/scoring/prompt-templates.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAMH,wEAAwE;AACxE,MAAM,WAAW,cAAc;IAC7B,wDAAwD;IACxD,IAAI,EAAE,MAAM,CAAC;IACb,iEAAiE;IACjE,WAAW,EAAE,MAAM,CAAC;IACpB,sEAAsE;IACtE,IAAI,EAAE,QAAQ,GAAG,QAAQ,GAAG,MAAM,CAAC;IACnC,wEAAwE;IACxE,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB;AAED,+EAA+E;AAC/E,MAAM,WAAW,cAAc;IAC7B,gFAAgF;IAChF,IAAI,EAAE,MAAM,CAAC;IACb,0DAA0D;IAC1D,WAAW,EAAE,MAAM,CAAC;IACpB,qDAAqD;IACrD,KAAK,EAAE,WAAW,GAAG,kBAAkB,CAAC;IACxC,4DAA4D;IAC5D,QAAQ,EAAE,MAAM,CAAC;IACjB,4DAA4D;IAC5D,SAAS,EAAE,cAAc,EAAE,CAAC;CAC7B;AAMD;;;;;GAKG;AACH,wBAAgB,WAAW,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,GAAG,MAAM,CAO3F;AAMD,kEAAkE;AAClE,eAAO,MAAM,iBAAiB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CA2BpD,CAAC;AA4LF;;;;;;GAMG;AACH,wBAAgB,kBAAkB,IAAI,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAMnE"}

package/dist/scoring/prompt-templates.js

@@ -0,0 +1,251 @@
+/**
+ * Declarative prompt templates for the AXIS scoring pipeline.
+ *
+ * Each template uses `{{variable}}` placeholders that are substituted at
+ * runtime via `interpolate()`. The raw templates (with placeholders intact)
+ * are exposed via `getPromptTemplates()` so documentation UIs can display
+ * the exact prompts used during scoring.
+ */
+// ---------------------------------------------------------------------------
+// Interpolation
+// ---------------------------------------------------------------------------
+/**
+ * Replace `{{key}}` placeholders in `template` with values from `vars`.
+ *
+ * Throws if any placeholder in the template has no corresponding key in
+ * `vars`. Numbers are coerced to strings via `String()`.
+ */
+export function interpolate(template, vars) {
+    return template.replace(/\{\{(\w+)\}\}/g, (_match, key) => {
+        if (!(key in vars)) {
+            throw new Error(`Missing template variable: {{${key}}}`);
+        }
+        return String(vars[key]);
+    });
+}
+// ---------------------------------------------------------------------------
+// Template definitions
+// ---------------------------------------------------------------------------
+/** Per-category evaluation guidance for focused judge prompts. */
+export const CATEGORY_GUIDANCE = {
+    environment: `You are evaluating ENVIRONMENT interactions — file system operations, shell commands, code edits, and local workspace manipulation.
+
+Key considerations:
+- File reads/writes: Was the operation necessary? Was it right-sized (not reading entire files when a section would suffice)?
+- Shell commands: Did they succeed? Were they idempotent or did they cause side effects?
+- Code edits: Were they precise (targeted edits vs. rewriting entire files)?
+- Error recovery: Did the agent handle file-not-found, permission errors, or failed commands well?
+- Workspace hygiene: Did the agent clean up temp files, avoid polluting the workspace?`,
+    service: `You are evaluating SERVICE interactions — API calls, web fetches, external service requests, and network operations.
+
+Key considerations:
+- API calls: Were they well-formed? Did the agent handle rate limits, auth errors, and timeouts?
+- Web fetches: Did the agent fetch relevant pages? Were redundant fetches avoided?
+- Data handling: Was response data used effectively or was it fetched and ignored?
+- Error recovery: Did the agent retry appropriately on transient failures?
+- Efficiency: Were batch operations used when available instead of multiple individual calls?`,
+    agent: `You are evaluating AGENT interactions — the agent's own reasoning, planning, tool discovery, and communication.
+
+Key considerations:
+- Planning: Did the agent form a clear plan before acting, or did it thrash?
+- Tool discovery: Was tool/capability lookup necessary, or was it redundant exploration?
+- Reasoning quality: Was the agent's reasoning focused and productive?
+- Human interaction: Were questions to the user clear and necessary, or could the agent have proceeded independently?
+- Self-correction: When the agent detected errors, did it adjust its approach efficiently?`,
+};
+const CATEGORY_EVAL_TEMPLATE = {
+    name: "category_eval",
+    description: "Per-category evaluation of interactions for success, weight, contextRelevance, necessity, and patterns. Run once per category (environment, service, agent) in parallel.",
+    stage: "deep_eval",
+    template: `You are an expert evaluator for AXIS, an AI agent testing framework.
+
+You are evaluating the {{categoryName}} dimension of an agent execution. Focus ONLY on {{categoryName}} interactions, but use the full transcript context to understand the agent's overall behavior.
+
+SCENARIO: {{scenarioName}}
+
+TASK GIVEN TO AGENT:
+{{prompt}}
+
+COMPLETE SPARSE INDEX ({{totalInteractions}} total interactions, {{categoryInteractionCount}} are {{categoryName}}):
+{{sparseLines}}
+
+{{categoryName}} CATEGORY GUIDANCE:
+{{categoryGuidance}}
+
+{{categoryName}} INTERACTION DETAILS ({{categoryInteractionCount}} interactions):
+{{interactionContent}}
+
+RAW DATA FILES (for additional detail if needed):
+{{dataDir}}
+
+NOTE: Content shown above may be truncated for evaluation purposes. This does NOT mean the agent's actual tool results were truncated — evaluate based on the quality and structure of what is shown, not on apparent truncation boundaries.
+
+EVALUATION DIMENSIONS (score each 0.0 to 1.0):
+- success: Did the interaction complete without errors? Were the results correct and usable? Evaluate based on the actual content returned, not assumptions about what a "complete" result should look like.
+- weight: Was the tool invocation right-sized for the operation? Evaluate whether the agent sent an appropriate amount of data to the tool and received a proportionate response. (1.0 = right-sized, 0.3 = bloated/wasteful)
+- contextRelevance: Was the tool's output relevant and usable for the task? If the tool succeeded and the agent used the output to make progress, score 1.0. Only reduce this score if the output was genuinely irrelevant noise that the agent could not use. (1.0 = all useful/necessary, 0.0 = all noise)
+
+Also evaluate NECESSITY for the {{categoryName}} category as a whole:
+- necessity (0.0 to 1.0): Were the {{categoryName}} interactions necessary for the task? Evaluate only what the agent actually did — do not penalize for hypothetical steps it could have taken. 1.0 = all interactions were necessary, 0.0 = all were unnecessary.
+- List any interaction IDs that were unnecessary.
+
+Identify any patterns within {{categoryName}} interactions:
+- Repeated failures or retries
+- Redundant operations (same action performed multiple times)
+- Excessive operations for simple tasks
+- Wasted effort that didn't lead to progress
+
+CONTEXT FOR EVALUATION:
+- Tool discovery (e.g., ToolSearch, ListTools) and agent configuration reads are required infrastructure — do not flag as unnecessary unless genuinely redundant (same query repeated).
+- Byte counts in sparse lines show total I/O transferred, not file content size. Small results are normal for write/edit confirmations.
+
+Respond with ONLY valid JSON:
+{
+  "audits": [
+    {"id": 1, "success": 0.9, "weight": 0.8, "contextRelevance": 0.6, "rationale": "brief explanation"},
+    ...
+  ],
+  "necessity": {"score": 0.85, "unnecessaryIds": [4], "rationale": "brief explanation"},
+  "patterns": [
+    {"description": "pattern description", "interactionIds": [1, 2, 3], "severity": "high"},
+    ...
+  ]
+}
+
+Include an audit for EVERY {{categoryName}} interaction listed in the details above.`,
+    variables: [
+        { name: "scenarioName", description: "Name of the test scenario", type: "string" },
+        { name: "prompt", description: "The original task prompt given to the agent", type: "text" },
+        { name: "categoryName", description: "The category being evaluated (environment, service, or agent)", type: "string" },
+        { name: "totalInteractions", description: "Total number of interactions across all categories", type: "number" },
+        {
+            name: "categoryInteractionCount",
+            description: "Number of interactions in this specific category",
+            type: "number",
+        },
+        { name: "sparseLines", description: "Complete sparse index content (all categories)", type: "text" },
+        { name: "categoryGuidance", description: "Category-specific evaluation guidance", type: "text" },
+        {
+            name: "interactionContent",
+            description: "Full formatted content for this category's interactions only",
+            type: "text",
+        },
+        {
+            name: "dataDir",
+            description: "Path to the report directory containing raw data files",
+            type: "string",
+            optional: true,
+        },
+    ],
+};
+const GOAL_STRING_RUBRIC_TEMPLATE = {
+    name: "goal_string_rubric",
+    description: "Evaluates goal achievement when the rubric is a single string criterion.",
+    stage: "goal_achievement",
+    template: `You are an expert evaluator for an AI agent testing framework called AXIS.
+
+An AI agent was given a task. You must evaluate how well it performed based on the evidence in its transcript.
+
+SCENARIO: {{scenarioName}}
+
+TASK GIVEN TO AGENT:
+{{prompt}}
+
+---
+
+AGENT TRANSCRIPT (condensed):
+{{transcript}}
+
+---
+
+AGENT'S FINAL RESULT:
+{{finalResult}}
+
+---
+
+RUBRIC:
+{{rubric}}
+
+---
+
+INSTRUCTIONS:
+1. Review the transcript to understand what the agent did.
+2. Where possible, independently verify the results — check the filesystem for created/modified files, visit URLs, confirm that the claimed outcomes actually exist. Do not trust the transcript alone.
+
+When done, respond with ONLY valid JSON on its own line:
+{"score": <0-10>, "rationale": "<1-2 sentence explanation>"}
+
+Score guide: 0 = not met at all, 5 = partially met, 10 = fully met.`,
+    variables: [
+        { name: "scenarioName", description: "Name of the test scenario", type: "string" },
+        { name: "prompt", description: "The original task prompt given to the agent", type: "text" },
+        { name: "transcript", description: "Condensed agent transcript", type: "text" },
+        { name: "finalResult", description: "The agent's final result text", type: "text" },
+        { name: "rubric", description: "The evaluation criterion as a single string", type: "text" },
+    ],
+};
+const GOAL_ARRAY_RUBRIC_TEMPLATE = {
+    name: "goal_array_rubric",
+    description: "Evaluates goal achievement when the rubric has multiple weighted criteria.",
+    stage: "goal_achievement",
+    template: `You are an expert evaluator for an AI agent testing framework called AXIS.
+
+An AI agent was given a task. You must evaluate how well it performed by reviewing its transcript AND by independently verifying the results yourself.
+
+SCENARIO: {{scenarioName}}
+
+TASK GIVEN TO AGENT:
+{{prompt}}
+
+---
+
+AGENT TRANSCRIPT (condensed):
+{{transcript}}
+
+---
+
+AGENT'S FINAL RESULT:
+{{finalResult}}
+
+---
+
+RUBRIC CRITERIA:
+{{rubricText}}
+
+---
+
+INSTRUCTIONS:
+1. Review the transcript to understand what the agent did.
+2. Where possible, independently verify the results — visit URLs, check endpoints, confirm that the claimed outcomes actually exist. Do not trust the transcript alone.
+3. For each criterion, provide a score from 0 to 10 and a brief rationale.
+
+Score guide: 0 = not met at all, 5 = partially met, 10 = fully met.
+
+When done, respond with ONLY valid JSON on its own line:
+{"grades": [{"criterion_index": 0, "score": <0-10>, "rationale": "<string>"}, ...]}`,
+    variables: [
+        { name: "scenarioName", description: "Name of the test scenario", type: "string" },
+        { name: "prompt", description: "The original task prompt given to the agent", type: "text" },
+        { name: "transcript", description: "Condensed agent transcript", type: "text" },
+        { name: "finalResult", description: "The agent's final result text", type: "text" },
+        { name: "rubricText", description: "Formatted rubric criteria with weights", type: "text" },
+    ],
+};
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+/**
+ * Return all scoring prompt templates keyed by name.
+ *
+ * The templates contain `{{variable}}` placeholders — use `interpolate()`
+ * to substitute runtime values, or display the raw template text in
+ * documentation UIs.
+ */
+export function getPromptTemplates() {
+    return {
+        category_eval: CATEGORY_EVAL_TEMPLATE,
+        goal_string_rubric: GOAL_STRING_RUBRIC_TEMPLATE,
+        goal_array_rubric: GOAL_ARRAY_RUBRIC_TEMPLATE,
+    };
+}
+//# sourceMappingURL=prompt-templates.js.map