npm - @agent-native/core - Versions diffs - 0.83.0 → 0.84.1 - Mend

@agent-native/core 0.83.0 → 0.84.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (508) hide show

package/dist/eval/define-eval.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"define-eval.d.ts","sourceRoot":"","sources":["../../src/eval/define-eval.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAEvC,0EAA0E;AAC1E,eAAO,MAAM,sBAAsB,MAAM,CAAC;AAE1C,wBAAgB,UAAU,CAAC,IAAI,EAAE,IAAI,GAAG,IAAI,~~CAqB3C~~"}
1	+ {"version":3,"file":"define-eval.d.ts","sourceRoot":"","sources":["../../src/eval/define-eval.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAEvC,0EAA0E;AAC1E,eAAO,MAAM,sBAAsB,MAAM,CAAC;AAE1C,wBAAgB,UAAU,CAAC,IAAI,EAAE,IAAI,GAAG,IAAI,CAwB3C"}

package/dist/eval/define-eval.js CHANGED Viewed

@@ -31,7 +31,8 @@ export function defineEval(spec) {
     if (!spec.input || typeof spec.input.prompt !== "string") {
         throw new Error(`defineEval("${spec.name}"): \`input.prompt\` is required`);
     }
-    if (!Array.isArray(spec.scorers) || spec.scorers.length === 0) {
+    if ((!Array.isArray(spec.scorers) || spec.scorers.length === 0) &&
+        !spec.skipReason) {
         throw new Error(`defineEval("${spec.name}"): at least one scorer is required`);
     }
     if (spec.threshold !== undefined &&

package/dist/eval/define-eval.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"define-eval.js","sourceRoot":"","sources":["../../src/eval/define-eval.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAIH,0EAA0E;AAC1E,MAAM,CAAC,MAAM,sBAAsB,GAAG,GAAG,CAAC;AAE1C,MAAM,UAAU,UAAU,CAAC,IAAU;IACnC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QAChD,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;IACpD,CAAC;IACD,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;QACzD,MAAM,IAAI,KAAK,CAAC,eAAe,IAAI,CAAC,IAAI,kCAAkC,CAAC,CAAC;IAC9E,CAAC;IACD,~~IAAI~~,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,~~EAAE~~,CAAC;~~QAC9D~~,MAAM,IAAI,KAAK,CACb,eAAe,IAAI,CAAC,IAAI,qCAAqC,CAC9D,CAAC;IACJ,CAAC;IACD,IACE,IAAI,CAAC,SAAS,KAAK,SAAS;QAC5B,CAAC,IAAI,CAAC,SAAS,GAAG,CAAC,IAAI,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC,EAC1C,CAAC;QACD,MAAM,IAAI,KAAK,CACb,eAAe,IAAI,CAAC,IAAI,qCAAqC,CAC9D,CAAC;IACJ,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC","sourcesContent":["/*\n `defineEval` — declare a named eval test case.\n \n An eval pairs an input (prompt + optional history/setup) with a list of\n * scorers and a pass threshold. The runner (see `runner.ts`) actually runs the\n * agent for the input, scores the output with each scorer, and gates on the\n * threshold. Authors write `.eval.ts` files that `export default defineEval(...)`\n or export an array of them.\n \n Example (`evals/greeting.eval.ts`):\n * ```ts\n * import { defineEval, contains, llmJudge } from \"@agent-native/core/eval\";\n \n export default defineEval({\n * name: \"greets the user by name\",\n * input: { prompt: \"Say hi to Ada.\" },\n * threshold: 0.7,\n * scorers: [\n * contains(\"Ada\"),\n * llmJudge({ criteria: \"friendliness\", rubric: \"1.0 = warm greeting\" }),\n * ],\n * });\n * ```\n /\n\nimport type { Eval } from \"./types.js\";\n\n/* Default per-scorer pass threshold when an eval doesn't specify one. */\nexport const DEFAULT_EVAL_THRESHOLD = 0.5;\n\nexport function defineEval(spec: Eval): Eval {\n if (!spec.name \|\| typeof spec.name !== \"string\") {\n throw new Error(\"defineEval: `name` is required\");\n }\n if (!spec.input \|\| typeof spec.input.prompt !== \"string\") {\n throw new Error(`defineEval(\"${spec.name}\"): \\`input.prompt\\` is required`);\n }\n if (!Array.isArray(spec.scorers) \|\| spec.scorers.length === 0) {\n throw new Error(\n `defineEval(\"${spec.name}\"): at least one scorer is required`,\n );\n }\n if (\n spec.threshold !== undefined &&\n (spec.threshold < 0 \|\| spec.threshold > 1)\n ) {\n throw new Error(\n `defineEval(\"${spec.name}\"): \\`threshold\\` must be in [0, 1]`,\n );\n }\n return spec;\n}\n"]}
1	+ {"version":3,"file":"define-eval.js","sourceRoot":"","sources":["../../src/eval/define-eval.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAIH,0EAA0E;AAC1E,MAAM,CAAC,MAAM,sBAAsB,GAAG,GAAG,CAAC;AAE1C,MAAM,UAAU,UAAU,CAAC,IAAU;IACnC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QAChD,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;IACpD,CAAC;IACD,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;QACzD,MAAM,IAAI,KAAK,CAAC,eAAe,IAAI,CAAC,IAAI,kCAAkC,CAAC,CAAC;IAC9E,CAAC;IACD,IACE,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC;QAC3D,CAAC,IAAI,CAAC,UAAU,EAChB,CAAC;QACD,MAAM,IAAI,KAAK,CACb,eAAe,IAAI,CAAC,IAAI,qCAAqC,CAC9D,CAAC;IACJ,CAAC;IACD,IACE,IAAI,CAAC,SAAS,KAAK,SAAS;QAC5B,CAAC,IAAI,CAAC,SAAS,GAAG,CAAC,IAAI,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC,EAC1C,CAAC;QACD,MAAM,IAAI,KAAK,CACb,eAAe,IAAI,CAAC,IAAI,qCAAqC,CAC9D,CAAC;IACJ,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC","sourcesContent":["/*\n `defineEval` — declare a named eval test case.\n \n An eval pairs an input (prompt + optional history/setup) with a list of\n * scorers and a pass threshold. The runner (see `runner.ts`) actually runs the\n * agent for the input, scores the output with each scorer, and gates on the\n * threshold. Authors write `.eval.ts` files that `export default defineEval(...)`\n or export an array of them.\n \n Example (`evals/greeting.eval.ts`):\n * ```ts\n * import { defineEval, contains, llmJudge } from \"@agent-native/core/eval\";\n \n export default defineEval({\n * name: \"greets the user by name\",\n * input: { prompt: \"Say hi to Ada.\" },\n * threshold: 0.7,\n * scorers: [\n * contains(\"Ada\"),\n * llmJudge({ criteria: \"friendliness\", rubric: \"1.0 = warm greeting\" }),\n * ],\n * });\n * ```\n /\n\nimport type { Eval } from \"./types.js\";\n\n/* Default per-scorer pass threshold when an eval doesn't specify one. */\nexport const DEFAULT_EVAL_THRESHOLD = 0.5;\n\nexport function defineEval(spec: Eval): Eval {\n if (!spec.name \|\| typeof spec.name !== \"string\") {\n throw new Error(\"defineEval: `name` is required\");\n }\n if (!spec.input \|\| typeof spec.input.prompt !== \"string\") {\n throw new Error(`defineEval(\"${spec.name}\"): \\`input.prompt\\` is required`);\n }\n if (\n (!Array.isArray(spec.scorers) \|\| spec.scorers.length === 0) &&\n !spec.skipReason\n ) {\n throw new Error(\n `defineEval(\"${spec.name}\"): at least one scorer is required`,\n );\n }\n if (\n spec.threshold !== undefined &&\n (spec.threshold < 0 \|\| spec.threshold > 1)\n ) {\n throw new Error(\n `defineEval(\"${spec.name}\"): \\`threshold\\` must be in [0, 1]`,\n );\n }\n return spec;\n}\n"]}

package/dist/eval/report.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"report.d.ts","sourceRoot":"","sources":["../../src/eval/report.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAgBhD,8CAA8C;AAC9C,wBAAgB,YAAY,CAAC,MAAM,EAAE,aAAa,GAAG,MAAM,~~CAqC1D~~"}
1	+ {"version":3,"file":"report.d.ts","sourceRoot":"","sources":["../../src/eval/report.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAgBhD,8CAA8C;AAC9C,wBAAgB,YAAY,CAAC,MAAM,EAAE,aAAa,GAAG,MAAM,CAoD1D"}

package/dist/eval/report.js CHANGED Viewed

@@ -21,9 +21,16 @@ export function formatReport(report) {
     lines.push("  Evals");
     lines.push("  ─────");
     for (const row of report.results) {
-        const mark = row.passed ? "✓" : "✗";
+        const mark = row.status === "skipped" ? "-" : row.passed ? "✓" : "✗";
         lines.push("");
-        lines.push(`  ${mark} ${row.eval}  (avg ${pct(row.avgScore)}, threshold ${pct(row.threshold)})`);
+        if (row.status === "skipped") {
+            lines.push(`  ${mark} ${row.eval}  (skipped)`);
+            lines.push(`      reason: ${row.skipReason ?? "No reason provided"}`);
+            continue;
+        }
+        else {
+            lines.push(`  ${mark} ${row.eval}  (avg ${pct(row.avgScore)}, threshold ${pct(row.threshold)})`);
+        }
         if (row.error) {
             lines.push(`      ⚠ run error: ${row.error}`);
         }
@@ -35,8 +42,16 @@ export function formatReport(report) {
     }
     lines.push("");
     lines.push("  ─────");
-    const verdict = report.failed === 0 ? "PASS" : "FAIL";
-    lines.push(`  ${verdict}: ${report.passed}/${report.total} evals passed` +
+    const skipped = report.skipped ?? 0;
+    const executedTotal = report.total - skipped;
+    const executedPassed = report.passed - skipped;
+    const verdict = executedTotal === 0 && skipped > 0
+        ? "SKIPPED"
+        : report.failed === 0
+            ? "PASS"
+            : "FAIL";
+    lines.push(`  ${verdict}: ${executedPassed}/${executedTotal} evals passed` +
+        (skipped > 0 ? `, ${skipped} skipped` : "") +
         (report.failed > 0 ? `, ${report.failed} below threshold` : ""));
     lines.push("");
     return lines.join("\n");

package/dist/eval/report.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"report.js","sourceRoot":"","sources":["../../src/eval/report.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAIH,SAAS,GAAG,CAAC,KAAa,EAAE,KAAK,GAAG,EAAE;IACpC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC,CAAC;IAClD,OAAO,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC;AACzD,CAAC;AAED,SAAS,OAAO,CAAC,CAAS;IACxB,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;QAAE,OAAO,CAAC,CAAC;IAClC,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;AACrC,CAAC;AAED,SAAS,GAAG,CAAC,KAAa;IACxB,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;AAC5D,CAAC;AAED,8CAA8C;AAC9C,MAAM,UAAU,YAAY,CAAC,MAAqB;IAChD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACtB,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAEtB,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QACjC,MAAM,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;~~QACpC~~,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CACR,KAAK,IAAI,IAAI,GAAG,CAAC,IAAI,UAAU,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,eAAe,GAAG,CAChE,GAAG,CAAC,SAAS,CACd,GAAG,CACL,CAAC;~~QACF~~,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;YACd,KAAK,CAAC,IAAI,CAAC,sBAAsB,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;QAChD,CAAC;QACD,KAAK,MAAM,CAAC,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC;YAC3B,MAAM,KAAK,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;YACnC,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACjD,KAAK,CAAC,IAAI,CACR,SAAS,KAAK,IAAI,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,GAAG,CAC1D,CAAC,CAAC,KAAK,CACR,GAAG,MAAM,EAAE,CACb,CAAC;QACJ,CAAC;IACH,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACtB,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;~~IACtD~~,KAAK,CAAC,IAAI,CACR,KAAK,OAAO,KAAK,~~MAAM~~,CAAC,~~MAAM~~,~~IAAI~~,~~MAAM~~,CAAC,KAAK,~~eAAe~~;~~QAC3D~~,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,MAAM,kBAAkB,CAAC,CAAC,CAAC,EAAE,CAAC,CAClE,CAAC;IACF,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC","sourcesContent":["/*\n Human-readable formatting for an eval run report. Kept separate from the\n * runner so the CLI can print a table while CI consumes the JSON shape.\n /\n\nimport type { EvalRunReport } from \"./types.js\";\n\nfunction bar(score: number, width = 10): string {\n const filled = Math.round(clamp01(score) width);\n return \"█\".repeat(filled) + \"░\".repeat(width - filled);\n}\n\nfunction clamp01(n: number): number {\n if (!Number.isFinite(n)) return 0;\n return Math.max(0, Math.min(1, n));\n}\n\nfunction pct(score: number): string {\n return `${Math.round(clamp01(score) * 100)}%`.padStart(4);\n}\n\n/** Render a scored table for the terminal. */\nexport function formatReport(report: EvalRunReport): string {\n const lines: string[] = [];\n lines.push(\"\");\n lines.push(\" Evals\");\n lines.push(\" ─────\");\n\n for (const row of report.results) {\n const mark = row.passed ? \"✓\" : \"✗\";\n lines.push(\"\");\n lines.push(\n ` ${mark} ${row.eval} (avg ${pct(row.avgScore)}, threshold ${pct(\n row.threshold,\n )})`,\n );\n if (row.error) {\n lines.push(` ⚠ run error: ${row.error}`);\n }\n for (const s of row.scores) {\n const smark = s.passed ? \"✓\" : \"✗\";\n const reason = s.reason ? ` — ${s.reason}` : \"\";\n lines.push(\n ` ${smark} ${s.scorer.padEnd(20)} ${bar(s.score)} ${pct(\n s.score,\n )}${reason}`,\n );\n }\n }\n\n lines.push(\"\");\n lines.push(\" ─────\");\n const ~~verdict~~ = report.~~failed~~ === 0 ? \"PASS\" : \"FAIL\";\n lines.push(\n ` ${verdict}: ${~~report.passed~~}/${~~report.total~~} evals passed` +\n (report.failed > 0 ? `, ${report.failed} below threshold` : \"\"),\n );\n lines.push(\"\");\n return lines.join(\"\\n\");\n}\n"]}
1	+ {"version":3,"file":"report.js","sourceRoot":"","sources":["../../src/eval/report.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAIH,SAAS,GAAG,CAAC,KAAa,EAAE,KAAK,GAAG,EAAE;IACpC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC,CAAC;IAClD,OAAO,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC;AACzD,CAAC;AAED,SAAS,OAAO,CAAC,CAAS;IACxB,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;QAAE,OAAO,CAAC,CAAC;IAClC,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;AACrC,CAAC;AAED,SAAS,GAAG,CAAC,KAAa;IACxB,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;AAC5D,CAAC;AAED,8CAA8C;AAC9C,MAAM,UAAU,YAAY,CAAC,MAAqB;IAChD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACtB,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAEtB,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QACjC,MAAM,IAAI,GAAG,GAAG,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;QACrE,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YAC7B,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,IAAI,GAAG,CAAC,IAAI,aAAa,CAAC,CAAC;YAC/C,KAAK,CAAC,IAAI,CAAC,iBAAiB,GAAG,CAAC,UAAU,IAAI,oBAAoB,EAAE,CAAC,CAAC;YACtE,SAAS;QACX,CAAC;aAAM,CAAC;YACN,KAAK,CAAC,IAAI,CACR,KAAK,IAAI,IAAI,GAAG,CAAC,IAAI,UAAU,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,eAAe,GAAG,CAChE,GAAG,CAAC,SAAS,CACd,GAAG,CACL,CAAC;QACJ,CAAC;QACD,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;YACd,KAAK,CAAC,IAAI,CAAC,sBAAsB,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;QAChD,CAAC;QACD,KAAK,MAAM,CAAC,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC;YAC3B,MAAM,KAAK,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;YACnC,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACjD,KAAK,CAAC,IAAI,CACR,SAAS,KAAK,IAAI,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,GAAG,CAC1D,CAAC,CAAC,KAAK,CACR,GAAG,MAAM,EAAE,CACb,CAAC;QACJ,CAAC;IACH,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACtB,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,CAAC,CAAC;IACpC,MAAM,aAAa,GAAG,MAAM,CAAC,KAAK,GAAG,OAAO,CAAC;IAC7C,MAAM,cAAc,GAAG,MAAM,CAAC,MAAM,GAAG,OAAO,CAAC;IAC/C,MAAM,OAAO,GACX,aAAa,KAAK,CAAC,IAAI,OAAO,GAAG,CAAC;QAChC,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC;YACnB,CAAC,CAAC,MAAM;YACR,CAAC,CAAC,MAAM,CAAC;IACf,KAAK,CAAC,IAAI,CACR,KAAK,OAAO,KAAK,cAAc,IAAI,aAAa,eAAe;QAC7D,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,OAAO,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3C,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,MAAM,kBAAkB,CAAC,CAAC,CAAC,EAAE,CAAC,CAClE,CAAC;IACF,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC","sourcesContent":["/*\n Human-readable formatting for an eval run report. Kept separate from the\n * runner so the CLI can print a table while CI consumes the JSON shape.\n /\n\nimport type { EvalRunReport } from \"./types.js\";\n\nfunction bar(score: number, width = 10): string {\n const filled = Math.round(clamp01(score) width);\n return \"█\".repeat(filled) + \"░\".repeat(width - filled);\n}\n\nfunction clamp01(n: number): number {\n if (!Number.isFinite(n)) return 0;\n return Math.max(0, Math.min(1, n));\n}\n\nfunction pct(score: number): string {\n return `${Math.round(clamp01(score) * 100)}%`.padStart(4);\n}\n\n/** Render a scored table for the terminal. */\nexport function formatReport(report: EvalRunReport): string {\n const lines: string[] = [];\n lines.push(\"\");\n lines.push(\" Evals\");\n lines.push(\" ─────\");\n\n for (const row of report.results) {\n const mark = row.status === \"skipped\" ? \"-\" : row.passed ? \"✓\" : \"✗\";\n lines.push(\"\");\n if (row.status === \"skipped\") {\n lines.push(` ${mark} ${row.eval} (skipped)`);\n lines.push(` reason: ${row.skipReason ?? \"No reason provided\"}`);\n continue;\n } else {\n lines.push(\n ` ${mark} ${row.eval} (avg ${pct(row.avgScore)}, threshold ${pct(\n row.threshold,\n )})`,\n );\n }\n if (row.error) {\n lines.push(` ⚠ run error: ${row.error}`);\n }\n for (const s of row.scores) {\n const smark = s.passed ? \"✓\" : \"✗\";\n const reason = s.reason ? ` — ${s.reason}` : \"\";\n lines.push(\n ` ${smark} ${s.scorer.padEnd(20)} ${bar(s.score)} ${pct(\n s.score,\n )}${reason}`,\n );\n }\n }\n\n lines.push(\"\");\n lines.push(\" ─────\");\n const skipped = report.skipped ?? 0;\n const executedTotal = report.total - skipped;\n const executedPassed = report.passed - skipped;\n const verdict =\n executedTotal === 0 && skipped > 0\n ? \"SKIPPED\"\n : report.failed === 0\n ? \"PASS\"\n : \"FAIL\";\n lines.push(\n ` ${verdict}: ${executedPassed}/${executedTotal} evals passed` +\n (skipped > 0 ? `, ${skipped} skipped` : \"\") +\n (report.failed > 0 ? `, ${report.failed} below threshold` : \"\"),\n );\n lines.push(\"\");\n return lines.join(\"\\n\");\n}\n"]}

package/dist/eval/runner.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/eval/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAKH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,8BAA8B,CAAC;AAGhE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAIrD,OAAO,KAAK,EAEV,IAAI,EACJ,aAAa,EACb,aAAa,EAEd,MAAM,YAAY,CAAC;AAsCpB,wEAAwE;AACxE,wBAAsB,SAAS,CAC7B,QAAQ,EAAE,IAAI,EACd,MAAM,EAAE,WAAW,EACnB,IAAI,GAAE;IAAE,iBAAiB,CAAC,EAAE,MAAM,CAAA;CAAO,GACxC,OAAO,CAAC,aAAa,CAAC,~~CAkCxB~~;AAED,sEAAsE;AACtE,wBAAsB,QAAQ,CAC5B,KAAK,EAAE,IAAI,EAAE,EACb,MAAM,EAAE,WAAW,EACnB,IAAI,GAAE;IAAE,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,OAAO,CAAA;CAAO,GAC3D,OAAO,CAAC,aAAa,CAAC,~~CAexB~~;AA4CD;;;;;GAKG;AACH,wBAAsB,iBAAiB,CACrC,IAAI,EAAE,MAAM,EACZ,OAAO,CAAC,EAAE,MAAM,GACf,OAAO,CAAC,MAAM,EAAE,CAAC,CAqCnB;AA0BD,8EAA8E;AAC9E,wBAAsB,SAAS,CAC7B,IAAI,EAAE,MAAM,EACZ,OAAO,CAAC,EAAE,MAAM,GACf,OAAO,CAAC;IAAE,KAAK,EAAE,MAAM,EAAE,CAAC;IAAC,KAAK,EAAE,IAAI,EAAE,CAAA;CAAE,CAAC,CAW7C;AAID,MAAM,WAAW,mBAAmB;IAClC,wEAAwE;IACxE,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,8CAA8C;IAC9C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iEAAiE;IACjE,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,wEAAwE;IACxE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;IACtC,8BAA8B;IAC9B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,oEAAoE;IACpE,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,0EAA0E;IAC1E,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,yEAAyE;IACzE,KAAK,CAAC,EAAE,IAAI,EAAE,CAAC;CAChB;AAED;;;GAGG;AACH,wBAAsB,YAAY,CAChC,IAAI,GAAE,mBAAwB,GAC7B,OAAO,CAAC;IAAE,MAAM,EAAE,aAAa,CAAC;IAAC,KAAK,EAAE,MAAM,EAAE,CAAA;CAAE,CAAC,~~CAuBrD~~"}
1	+ {"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/eval/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAKH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,8BAA8B,CAAC;AAGhE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAIrD,OAAO,KAAK,EAEV,IAAI,EACJ,aAAa,EACb,aAAa,EAEd,MAAM,YAAY,CAAC;AAsCpB,wEAAwE;AACxE,wBAAsB,SAAS,CAC7B,QAAQ,EAAE,IAAI,EACd,MAAM,EAAE,WAAW,EACnB,IAAI,GAAE;IAAE,iBAAiB,CAAC,EAAE,MAAM,CAAA;CAAO,GACxC,OAAO,CAAC,aAAa,CAAC,CAiDxB;AAED,sEAAsE;AACtE,wBAAsB,QAAQ,CAC5B,KAAK,EAAE,IAAI,EAAE,EACb,MAAM,EAAE,WAAW,EACnB,IAAI,GAAE;IAAE,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,OAAO,CAAA;CAAO,GAC3D,OAAO,CAAC,aAAa,CAAC,CAmBxB;AA4CD;;;;;GAKG;AACH,wBAAsB,iBAAiB,CACrC,IAAI,EAAE,MAAM,EACZ,OAAO,CAAC,EAAE,MAAM,GACf,OAAO,CAAC,MAAM,EAAE,CAAC,CAqCnB;AA0BD,8EAA8E;AAC9E,wBAAsB,SAAS,CAC7B,IAAI,EAAE,MAAM,EACZ,OAAO,CAAC,EAAE,MAAM,GACf,OAAO,CAAC;IAAE,KAAK,EAAE,MAAM,EAAE,CAAC;IAAC,KAAK,EAAE,IAAI,EAAE,CAAA;CAAE,CAAC,CAW7C;AAID,MAAM,WAAW,mBAAmB;IAClC,wEAAwE;IACxE,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,8CAA8C;IAC9C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iEAAiE;IACjE,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,wEAAwE;IACxE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;IACtC,8BAA8B;IAC9B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,oEAAoE;IACpE,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,0EAA0E;IAC1E,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,yEAAyE;IACzE,KAAK,CAAC,EAAE,IAAI,EAAE,CAAC;CAChB;AAED;;;GAGG;AACH,wBAAsB,YAAY,CAChC,IAAI,GAAE,mBAAwB,GAC7B,OAAO,CAAC;IAAE,MAAM,EAAE,aAAa,CAAC;IAAC,KAAK,EAAE,MAAM,EAAE,CAAA;CAAE,CAAC,CA0BrD"}

package/dist/eval/runner.js CHANGED Viewed

@@ -53,6 +53,19 @@ async function runScorer(scorer, run, runner, threshold) {
 /** Run a single eval: invoke the agent, then score with each scorer. */
 export async function scoreEval(evalCase, runner, opts = {}) {
     const threshold = opts.thresholdOverride ?? evalCase.threshold ?? DEFAULT_EVAL_THRESHOLD;
+    if (evalCase.skipReason) {
+        return {
+            eval: evalCase.name,
+            threshold,
+            scores: [],
+            status: "skipped",
+            skipReason: evalCase.skipReason,
+            // Keep the legacy boolean gate friendly: skipped rows do not fail CI.
+            passed: true,
+            avgScore: 0,
+            durationMs: 0,
+        };
+    }
     let run;
     if (evalCase.run) {
         run = await evalCase.run({
@@ -76,6 +89,7 @@ export async function scoreEval(evalCase, runner, opts = {}) {
         scores,
         // A run that errored, or any sub-threshold scorer, fails the case.
         passed: run.ok && scores.every((s) => s.passed),
+        status: run.ok && scores.every((s) => s.passed) ? "passed" : "failed",
         avgScore,
         durationMs: run.durationMs,
         error: run.ok ? undefined : run.error,
@@ -87,14 +101,17 @@ export async function runEvals(evals, runner, opts = {}) {
     for (const evalCase of evals) {
         const row = await scoreEval(evalCase, runner, opts);
         results.push(row);
-        if (opts.persist)
+        if (opts.persist && row.status !== "skipped") {
             await persistEvalRow(row).catch(() => { });
+        }
     }
     const passed = results.filter((r) => r.passed).length;
+    const skipped = results.filter((r) => r.status === "skipped").length;
     return {
         total: results.length,
         passed,
-        failed: results.length - passed,
+        failed: results.filter((r) => r.status !== "skipped" && !r.passed).length,
+        skipped,
         results,
     };
 }
@@ -227,11 +244,14 @@ export async function runEvalSuite(opts = {}) {
         files = loaded.files;
         evals = loaded.evals;
     }
+    const needsRunner = evals.some((evalCase) => !evalCase.skipReason);
     const runner = opts.runner ??
-        (await createAgentRunner({
-            actions: opts.actions ?? (await discoverActions(cwd)),
-            systemPrompt: opts.systemPrompt,
-        }));
+        (needsRunner
+            ? await createAgentRunner({
+                actions: opts.actions ?? (await discoverActions(cwd)),
+                systemPrompt: opts.systemPrompt,
+            })
+            : createInertRunner());
     const report = await runEvals(evals, runner, {
         thresholdOverride: opts.thresholdOverride,
         persist: opts.persist ?? true,
@@ -253,4 +273,16 @@ async function discoverActions(cwd) {
         return {};
     }
 }
+function createInertRunner() {
+    return {
+        engine: {},
+        model: "inert",
+        async runAgent() {
+            throw new Error("Eval unexpectedly requested the agent runner");
+        },
+        analyzeContext() {
+            throw new Error("Eval unexpectedly requested analyze context");
+        },
+    };
+}
 //# sourceMappingURL=runner.js.map

package/dist/eval/runner.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"runner.js","sourceRoot":"","sources":["../../src/eval/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,QAAQ,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAGzC,OAAO,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAC;AAG7D,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,EAAE,sBAAsB,EAAE,MAAM,kBAAkB,CAAC;AAC1D,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAStC,yEAAyE;AAEzE,yEAAyE;AACzE,KAAK,UAAU,SAAS,CACtB,MAA+B,EAC/B,GAAmB,EACnB,MAAmB,EACnB,SAAiB;IAEjB,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;QACnE,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO;YAC7B,CAAC,CAAC,MAAM,MAAM,CAAC,OAAO,CAAC,GAAY,EAAE,MAAM,CAAC,cAAc,EAAE,CAAC;YAC7D,CAAC,CAAC,GAAG,CAAC;QACR,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,QAAiB,CAAC,CAAC;QAC/D,MAAM,KAAK,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;QAChC,MAAM,MAAM,GAAG,MAAM,CAAC,cAAc;YAClC,CAAC,CAAC,MAAM,MAAM,CAAC,cAAc,CAAC;gBAC1B,GAAG;gBACH,QAAQ,EAAE,QAAiB;gBAC3B,KAAK;aACN,CAAC;YACJ,CAAC,CAAC,SAAS,CAAC;QACd,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,IAAI,SAAS,EAAE,CAAC;IAC5E,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,uEAAuE;QACvE,kEAAkE;QAClE,OAAO;YACL,MAAM,EAAE,MAAM,CAAC,IAAI;YACnB,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,mBAAmB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;YAC7E,MAAM,EAAE,KAAK;SACd,CAAC;IACJ,CAAC;AACH,CAAC;AAED,wEAAwE;AACxE,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,QAAc,EACd,MAAmB,EACnB,IAAI,GAAmC,EAAE;IAEzC,MAAM,SAAS,GACb,IAAI,CAAC,iBAAiB,IAAI,QAAQ,CAAC,SAAS,IAAI,sBAAsB,CAAC;IAEzE,IAAI,GAAmB,CAAC;IACxB,IAAI,QAAQ,CAAC,GAAG,EAAE,CAAC;QACjB,GAAG,GAAG,MAAM,QAAQ,CAAC,GAAG,CAAC;YACvB,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,QAAQ,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;SAC5C,CAAC,CAAC;IACL,CAAC;SAAM,CAAC;QACN,GAAG,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC9C,CAAC;IAED,MAAM,MAAM,GAAmB,EAAE,CAAC;IAClC,KAAK,MAAM,MAAM,IAAI,QAAQ,CAAC,OAAO,EAAE,CAAC;QACtC,MAAM,CAAC,IAAI,CAAC,MAAM,SAAS,CAAC,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC;IAC/D,CAAC;IAED,MAAM,QAAQ,GACZ,MAAM,CAAC,MAAM,GAAG,CAAC;QACf,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM;QACzD,CAAC,CAAC,CAAC,CAAC;IAER,OAAO;QACL,IAAI,EAAE,QAAQ,CAAC,IAAI;QACnB,SAAS;QACT,MAAM;QACN,mEAAmE;QACnE,MAAM,EAAE,GAAG,CAAC,EAAE,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;QAC/C,QAAQ;QACR,UAAU,EAAE,GAAG,CAAC,UAAU;QAC1B,KAAK,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK;KACtC,CAAC;AACJ,CAAC;AAED,sEAAsE;AACtE,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,KAAa,EACb,MAAmB,EACnB,IAAI,GAAsD,EAAE;IAE5D,MAAM,OAAO,GAAoB,EAAE,CAAC;IACpC,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE,CAAC;QAC7B,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;QACpD,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAClB,IAAI,IAAI,CAAC,OAAO;YAAE,MAAM,cAAc,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IAC9D,CAAC;IAED,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IACtD,OAAO;QACL,KAAK,EAAE,OAAO,CAAC,MAAM;QACrB,MAAM;QACN,MAAM,EAAE,OAAO,CAAC,MAAM,GAAG,MAAM;QAC/B,OAAO;KACR,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,KAAK,UAAU,cAAc,CAAC,GAAkB;IAC9C,MAAM,KAAK,GAAG,QAAQ,GAAG,CAAC,IAAI,IAAI,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;IAC/C,KAAK,MAAM,CAAC,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC;QAC3B,MAAM,MAAM,GAA4B;YACtC,EAAE,EAAE,MAAM,CAAC,UAAU,EAAE;YACvB,KAAK;YACL,QAAQ,EAAE,IAAI;YACd,MAAM,EAAE,IAAI;YACZ,QAAQ,EAAE,WAAW;YACrB,QAAQ,EAAE,QAAQ,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,MAAM,EAAE;YACxC,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,SAAS,EAAE,CAAC,CAAC,MAAM,IAAI,IAAI;YAC3B,QAAQ,EAAE;gBACR,MAAM,EAAE,UAAU;gBAClB,SAAS,EAAE,GAAG,CAAC,SAAS;gBACxB,MAAM,EAAE,CAAC,CAAC,MAAM;aACjB;YACD,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;SACtB,CAAC;QACF,MAAM,gBAAgB,CAAC,MAAM,CAAC,CAAC;IACjC,CAAC;AACH,CAAC;AAED,yEAAyE;AAEzE,MAAM,YAAY,GAAG,sBAAsB,CAAC;AAC5C,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,CAAC,cAAc,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC,CAAC;AAEhF;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,IAAY,EACZ,OAAgB;IAEhB,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC;IACnC,MAAM,GAAG,GAAa,EAAE,CAAC;IAEzB,SAAS,UAAU,CAAC,IAAY,EAAE,UAAkB;QAClD,MAAM,IAAI,GAAG,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACrC,IAAI,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,OAAO,IAAI,CAAC;QACzC,IAAI,UAAU,KAAK,OAAO,IAAI,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC1D,4CAA4C;YAC5C,OAAO,CAAC,+BAA+B,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACrD,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,SAAS,IAAI,CAAC,GAAW,EAAE,UAAkB;QAC3C,IAAI,OAAmC,CAAC;QACxC,IAAI,CAAC;YACH,OAAO,GAAG,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;QACzD,CAAC;QAAC,MAAM,CAAC;YACP,OAAO;QACT,CAAC;QACD,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;YAC5C,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;gBACxB,IAAI,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;oBAAE,SAAS;gBACtE,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;YACzB,CAAC;iBAAM,IAAI,KAAK,CAAC,MAAM,EAAE,IAAI,UAAU,CAAC,IAAI,EAAE,UAAU,CAAC,EAAE,CAAC;gBAC1D,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACjB,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;IACpC,GAAG,CAAC,IAAI,EAAE,CAAC;IAEX,IAAI,CAAC,OAAO;QAAE,OAAO,GAAG,CAAC;IACzB,OAAO,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;AACzE,CAAC;AAED,yEAAyE;AACzE,SAAS,YAAY,CAAC,GAA4B;IAChD,MAAM,UAAU,GAAc,EAAE,CAAC;IACjC,IAAI,GAAG,CAAC,OAAO,KAAK,SAAS;QAAE,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAC5D,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;QAC/C,IAAI,GAAG,KAAK,SAAS;YAAE,SAAS;QAChC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACzB,CAAC;IAED,MAAM,KAAK,GAAW,EAAE,CAAC;IACzB,KAAK,MAAM,CAAC,IAAI,UAAU,CAAC,IAAI,EAAE,EAAE,CAAC;QAClC,IACE,CAAC;YACD,OAAO,CAAC,KAAK,QAAQ;YACrB,OAAQ,CAAU,CAAC,IAAI,KAAK,QAAQ;YACpC,KAAK,CAAC,OAAO,CAAE,CAAU,CAAC,OAAO,CAAC;YACjC,CAAU,CAAC,KAAK,EACjB,CAAC;YACD,KAAK,CAAC,IAAI,CAAC,CAAS,CAAC,CAAC;QACxB,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,8EAA8E;AAC9E,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,IAAY,EACZ,OAAgB;IAEhB,MAAM,KAAK,GAAG,MAAM,iBAAiB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACrD,MAAM,KAAK,GAAW,EAAE,CAAC;IACzB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,GAAG,GAAG,CAAC,MAAM,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAGlD,CAAC;QACF,KAAK,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC;IACnC,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;AAC1B,CAAC;AAuBD;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,IAAI,GAAwB,EAAE;IAE9B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;IAEtC,IAAI,KAAK,GAAa,EAAE,CAAC;IACzB,IAAI,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;IACvB,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAClD,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;QACrB,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;IACvB,CAAC;IAED,MAAM,MAAM,GACV,IAAI,CAAC,MAAM;QACX,CAAC,MAAM,iBAAiB,CAAC;YACvB,OAAO,EAAE,IAAI,CAAC,OAAO,IAAI,CAAC,MAAM,eAAe,CAAC,GAAG,CAAC,CAAC;YACrD,YAAY,EAAE,IAAI,CAAC,YAAY;SAChC,CAAC,CAAC,CAAC;IAEN,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE;QAC3C,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;QACzC,OAAO,EAAE,IAAI,CAAC,OAAO,IAAI,IAAI;KAC9B,CAAC,CAAC;IACH,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;AAC3B,CAAC;AAED;;;;GAIG;AACH,KAAK,UAAU,eAAe,CAC5B,GAAW;IAEX,IAAI,CAAC;QACH,MAAM,EAAE,mBAAmB,EAAE,GAC3B,MAAM,MAAM,CAAC,+BAA+B,CAAC,CAAC;QAChD,MAAM,UAAU,GAAG,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;QACjD,OAAO,MAAM,mBAAmB,CAAC,aAAa,CAAC,UAAU,GAAG,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;IACzE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC","sourcesContent":["/*\n The evals runner: discover `.eval.ts` / `evals/.ts` files, run each eval\n * through its scorer pipeline against the real agent loop, score, and report.\n \n It is the engine behind `agent-native eval` — when used as a CI deploy gate\n * the CLI exits non-zero if any eval scores below its threshold.\n \n Two layers:\n * - `scoreEval` / `runEvals` — pure orchestration over an `AgentRunner` and\n * a list of evals. Fully unit-testable with an injected runner (no model).\n * - `discoverEvalFiles` / `loadEvals` — filesystem discovery + dynamic import\n * of author-written eval modules.\n \n Results are also (best-effort) written to the observability eval store so a\n * dashboard can surface CI eval history next to production run evals.\n /\n\nimport nodePath from \"node:path\";\nimport { pathToFileURL } from \"node:url\";\n\nimport type { ActionEntry } from \"../agent/production-agent.js\";\nimport { insertEvalResult } from \"../observability/store.js\";\nimport type { EvalResult as ObservabilityEvalResult } from \"../observability/types.js\";\nimport type { AgentRunner } from \"./agent-runner.js\";\nimport { createAgentRunner } from \"./agent-runner.js\";\nimport { DEFAULT_EVAL_THRESHOLD } from \"./define-eval.js\";\nimport { clamp01 } from \"./scorer.js\";\nimport type {\n AgentRunOutput,\n Eval,\n EvalResultRow,\n EvalRunReport,\n ScorerResult,\n} from \"./types.js\";\n\n// ─── Scoring orchestration ────────────────────────────────────────────\n\n/* Run one scorer's pipeline (preprocess → analyze → score → reason). /\nasync function runScorer(\n scorer: Eval[\"scorers\"][number],\n run: AgentRunOutput,\n runner: AgentRunner,\n threshold: number,\n): Promise<ScorerResult> {\n try {\n const pre = scorer.preprocess ? await scorer.preprocess(run) : run;\n const analysis = scorer.analyze\n ? await scorer.analyze(pre as never, runner.analyzeContext())\n : pre;\n const rawScore = await scorer.generateScore(analysis as never);\n const score = clamp01(rawScore);\n const reason = scorer.generateReason\n ? await scorer.generateReason({\n run,\n analysis: analysis as never,\n score,\n })\n : undefined;\n return { scorer: scorer.name, score, reason, passed: score >= threshold };\n } catch (err) {\n // A scorer that throws is a failed scorer, not a crashed run — degrade\n // gracefully so one bad scorer can't take down the whole CI gate.\n return {\n scorer: scorer.name,\n score: 0,\n reason: `Scorer errored: ${err instanceof Error ? err.message : String(err)}`,\n passed: false,\n };\n }\n}\n\n/* Run a single eval: invoke the agent, then score with each scorer. /\nexport async function scoreEval(\n evalCase: Eval,\n runner: AgentRunner,\n opts: { thresholdOverride?: number } = {},\n): Promise<EvalResultRow> {\n const threshold =\n opts.thresholdOverride ?? evalCase.threshold ?? DEFAULT_EVAL_THRESHOLD;\n\n let run: AgentRunOutput;\n if (evalCase.run) {\n run = await evalCase.run({\n input: evalCase.input,\n runAgent: (input) => runner.runAgent(input),\n });\n } else {\n run = await runner.runAgent(evalCase.input);\n }\n\n const scores: ScorerResult[] = [];\n for (const scorer of evalCase.scorers) {\n scores.push(await runScorer(scorer, run, runner, threshold));\n }\n\n const avgScore =\n scores.length > 0\n ? scores.reduce((s, r) => s + r.score, 0) / scores.length\n : 0;\n\n return {\n eval: evalCase.name,\n threshold,\n scores,\n // A run that errored, or any sub-threshold scorer, fails the case.\n passed: run.ok && scores.every((s) => s.passed),\n avgScore,\n durationMs: run.durationMs,\n error: run.ok ? undefined : run.error,\n };\n}\n\n/* Run a batch of evals against one runner and aggregate a report. /\nexport async function runEvals(\n evals: Eval[],\n runner: AgentRunner,\n opts: { thresholdOverride?: number; persist?: boolean } = {},\n): Promise<EvalRunReport> {\n const results: EvalResultRow[] = [];\n for (const evalCase of evals) {\n const row = await scoreEval(evalCase, runner, opts);\n results.push(row);\n if (opts.persist) await persistEvalRow(row).catch(() => {});\n }\n\n const passed = results.filter((r) => r.passed).length;\n return {\n total: results.length,\n passed,\n failed: results.length - passed,\n results,\n };\n}\n\n/\n Best-effort write of one eval result to the observability eval store so a\n * dashboard can show CI eval history alongside production run evals. We write\n * one row per (eval × scorer), tagged `evalType: \"automated\"` with a synthetic\n * `eval:` run id.\n \n TODO(live-sampling): the same scorer list should also run on a sampled\n * fraction of real production runs. That hook belongs in the agent loop's\n * (not-yet-added) post-run processor seam: when a run finishes, roll the\n * configured sample rate and, if it hits, replay the run output through these\n * scorers and write the rows here. Wiring it now would require the in-loop\n * processor seam another wave is adding — so this is the single intended\n * attachment point, intentionally left as a note.\n /\nasync function persistEvalRow(row: EvalResultRow): Promise<void> {\n const runId = `eval:${row.eval}:${Date.now()}`;\n for (const s of row.scores) {\n const result: ObservabilityEvalResult = {\n id: crypto.randomUUID(),\n runId,\n threadId: null,\n userId: null,\n evalType: \"automated\",\n criteria: `eval:${row.eval}:${s.scorer}`,\n score: s.score,\n reasoning: s.reason ?? null,\n metadata: {\n source: \"cli-eval\",\n threshold: row.threshold,\n passed: s.passed,\n },\n createdAt: Date.now(),\n };\n await insertEvalResult(result);\n }\n}\n\n// ─── Discovery + loading ──────────────────────────────────────────────\n\nconst EVAL_FILE_RE = /\\.eval\\.(ts\|js\|mjs)$/;\nconst SKIP_DIRS = new Set([\"node_modules\", \"dist\", \".git\", \".output\", \"build\"]);\n\n/\n Walk `root` for eval files. Matches two conventions:\n * - any `*\\/.eval.ts` (co-located with code), and\n * - any `.ts` directly inside an `evals/` directory.\n `pattern` further filters by substring of the relative path.\n /\nexport async function discoverEvalFiles(\n root: string,\n pattern?: string,\n): Promise<string[]> {\n const fs = await import(\"node:fs\");\n const out: string[] = [];\n\n function isEvalFile(full: string, parentName: string): boolean {\n const base = nodePath.basename(full);\n if (EVAL_FILE_RE.test(base)) return true;\n if (parentName === \"evals\" && /\\.(ts\|js\|mjs)$/.test(base)) {\n // Skip obvious support files inside evals/.\n return !/\\.(spec\|test\|d)\\.(ts\|js\|mjs)$/.test(base);\n }\n return false;\n }\n\n function walk(dir: string, parentName: string): void {\n let entries: import(\"node:fs\").Dirent[];\n try {\n entries = fs.readdirSync(dir, { withFileTypes: true });\n } catch {\n return;\n }\n for (const entry of entries) {\n const full = nodePath.join(dir, entry.name);\n if (entry.isDirectory()) {\n if (SKIP_DIRS.has(entry.name) \|\| entry.name.startsWith(\".\")) continue;\n walk(full, entry.name);\n } else if (entry.isFile() && isEvalFile(full, parentName)) {\n out.push(full);\n }\n }\n }\n\n walk(root, nodePath.basename(root));\n out.sort();\n\n if (!pattern) return out;\n return out.filter((f) => nodePath.relative(root, f).includes(pattern));\n}\n\n/* Pull `Eval` definitions out of a dynamically-imported eval module. /\nfunction extractEvals(mod: Record<string, unknown>): Eval[] {\n const candidates: unknown[] = [];\n if (mod.default !== undefined) candidates.push(mod.default);\n for (const [key, value] of Object.entries(mod)) {\n if (key === \"default\") continue;\n candidates.push(value);\n }\n\n const evals: Eval[] = [];\n for (const c of candidates.flat()) {\n if (\n c &&\n typeof c === \"object\" &&\n typeof (c as Eval).name === \"string\" &&\n Array.isArray((c as Eval).scorers) &&\n (c as Eval).input\n ) {\n evals.push(c as Eval);\n }\n }\n return evals;\n}\n\n/* Discover and import all eval files under `root`, returning their evals. /\nexport async function loadEvals(\n root: string,\n pattern?: string,\n): Promise<{ files: string[]; evals: Eval[] }> {\n const files = await discoverEvalFiles(root, pattern);\n const evals: Eval[] = [];\n for (const file of files) {\n const mod = (await import(pathToFileURL(file).href)) as Record<\n string,\n unknown\n >;\n evals.push(...extractEvals(mod));\n }\n return { files, evals };\n}\n\n// ─── High-level entry used by the CLI ─────────────────────────────────\n\nexport interface RunEvalSuiteOptions {\n /* App root to discover eval files + actions under. Defaults to cwd. /\n cwd?: string;\n /* Substring filter on the eval file path. /\n pattern?: string;\n /* Global threshold override (wins over per-eval thresholds). /\n thresholdOverride?: number;\n /* App actions to expose to the agent. Auto-discovered when omitted. /\n actions?: Record<string, ActionEntry>;\n /* System prompt for runs. /\n systemPrompt?: string;\n /* Write results to the observability eval store (default true). /\n persist?: boolean;\n /* Pre-built runner (tests inject this to avoid touching engine/loop). /\n runner?: AgentRunner;\n /* Pre-loaded evals (tests inject this to skip filesystem discovery). /\n evals?: Eval[];\n}\n\n/\n End-to-end: load evals, build a runner, score, report. The CLI wraps this\n * and maps `report.failed > 0` to a non-zero exit code (the CI gate).\n /\nexport async function runEvalSuite(\n opts: RunEvalSuiteOptions = {},\n): Promise<{ report: EvalRunReport; files: string[] }> {\n const cwd = opts.cwd ?? process.cwd();\n\n let files: string[] = [];\n let evals = opts.evals;\n if (!evals) {\n const loaded = await loadEvals(cwd, opts.pattern);\n files = loaded.files;\n evals = loaded.evals;\n }\n\n const runner =\n opts.runner ??\n (await createAgentRunner({\n actions: opts.actions ?? (await discoverActions(cwd)),\n systemPrompt: opts.systemPrompt,\n }));\n\n const report = await runEvals(evals, runner, {\n thresholdOverride: opts.thresholdOverride,\n persist: opts.persist ?? true,\n });\n return { report, files };\n}\n\n/\n Discover the app's actions so the agent under test has the real tool\n * surface. Lazy-imports `autoDiscoverActions` to keep server-only deps out of\n * any browser bundle that might touch this module's types.\n */\nasync function discoverActions(\n cwd: string,\n): Promise<Record<string, ActionEntry>> {\n try {\n const { autoDiscoverActions } =\n await import(\"../server/action-discovery.js\");\n const actionsDir = nodePath.join(cwd, \"actions\");\n return await autoDiscoverActions(pathToFileURL(actionsDir + \"/\").href);\n } catch {\n return {};\n }\n}\n"]}
1	+ {"version":3,"file":"runner.js","sourceRoot":"","sources":["../../src/eval/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,QAAQ,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAGzC,OAAO,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAC;AAG7D,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,EAAE,sBAAsB,EAAE,MAAM,kBAAkB,CAAC;AAC1D,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAStC,yEAAyE;AAEzE,yEAAyE;AACzE,KAAK,UAAU,SAAS,CACtB,MAA+B,EAC/B,GAAmB,EACnB,MAAmB,EACnB,SAAiB;IAEjB,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;QACnE,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO;YAC7B,CAAC,CAAC,MAAM,MAAM,CAAC,OAAO,CAAC,GAAY,EAAE,MAAM,CAAC,cAAc,EAAE,CAAC;YAC7D,CAAC,CAAC,GAAG,CAAC;QACR,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,QAAiB,CAAC,CAAC;QAC/D,MAAM,KAAK,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;QAChC,MAAM,MAAM,GAAG,MAAM,CAAC,cAAc;YAClC,CAAC,CAAC,MAAM,MAAM,CAAC,cAAc,CAAC;gBAC1B,GAAG;gBACH,QAAQ,EAAE,QAAiB;gBAC3B,KAAK;aACN,CAAC;YACJ,CAAC,CAAC,SAAS,CAAC;QACd,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,IAAI,SAAS,EAAE,CAAC;IAC5E,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,uEAAuE;QACvE,kEAAkE;QAClE,OAAO;YACL,MAAM,EAAE,MAAM,CAAC,IAAI;YACnB,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,mBAAmB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;YAC7E,MAAM,EAAE,KAAK;SACd,CAAC;IACJ,CAAC;AACH,CAAC;AAED,wEAAwE;AACxE,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,QAAc,EACd,MAAmB,EACnB,IAAI,GAAmC,EAAE;IAEzC,MAAM,SAAS,GACb,IAAI,CAAC,iBAAiB,IAAI,QAAQ,CAAC,SAAS,IAAI,sBAAsB,CAAC;IAEzE,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC;QACxB,OAAO;YACL,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,SAAS;YACT,MAAM,EAAE,EAAE;YACV,MAAM,EAAE,SAAS;YACjB,UAAU,EAAE,QAAQ,CAAC,UAAU;YAC/B,sEAAsE;YACtE,MAAM,EAAE,IAAI;YACZ,QAAQ,EAAE,CAAC;YACX,UAAU,EAAE,CAAC;SACd,CAAC;IACJ,CAAC;IAED,IAAI,GAAmB,CAAC;IACxB,IAAI,QAAQ,CAAC,GAAG,EAAE,CAAC;QACjB,GAAG,GAAG,MAAM,QAAQ,CAAC,GAAG,CAAC;YACvB,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,QAAQ,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;SAC5C,CAAC,CAAC;IACL,CAAC;SAAM,CAAC;QACN,GAAG,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC9C,CAAC;IAED,MAAM,MAAM,GAAmB,EAAE,CAAC;IAClC,KAAK,MAAM,MAAM,IAAI,QAAQ,CAAC,OAAO,EAAE,CAAC;QACtC,MAAM,CAAC,IAAI,CAAC,MAAM,SAAS,CAAC,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC;IAC/D,CAAC;IAED,MAAM,QAAQ,GACZ,MAAM,CAAC,MAAM,GAAG,CAAC;QACf,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM;QACzD,CAAC,CAAC,CAAC,CAAC;IAER,OAAO;QACL,IAAI,EAAE,QAAQ,CAAC,IAAI;QACnB,SAAS;QACT,MAAM;QACN,mEAAmE;QACnE,MAAM,EAAE,GAAG,CAAC,EAAE,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;QAC/C,MAAM,EAAE,GAAG,CAAC,EAAE,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ;QACrE,QAAQ;QACR,UAAU,EAAE,GAAG,CAAC,UAAU;QAC1B,KAAK,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK;KACtC,CAAC;AACJ,CAAC;AAED,sEAAsE;AACtE,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,KAAa,EACb,MAAmB,EACnB,IAAI,GAAsD,EAAE;IAE5D,MAAM,OAAO,GAAoB,EAAE,CAAC;IACpC,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE,CAAC;QAC7B,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;QACpD,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAClB,IAAI,IAAI,CAAC,OAAO,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YAC7C,MAAM,cAAc,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAC5C,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IACtD,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,MAAM,CAAC;IACrE,OAAO;QACL,KAAK,EAAE,OAAO,CAAC,MAAM;QACrB,MAAM;QACN,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,SAAS,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM;QACzE,OAAO;QACP,OAAO;KACR,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,KAAK,UAAU,cAAc,CAAC,GAAkB;IAC9C,MAAM,KAAK,GAAG,QAAQ,GAAG,CAAC,IAAI,IAAI,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;IAC/C,KAAK,MAAM,CAAC,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC;QAC3B,MAAM,MAAM,GAA4B;YACtC,EAAE,EAAE,MAAM,CAAC,UAAU,EAAE;YACvB,KAAK;YACL,QAAQ,EAAE,IAAI;YACd,MAAM,EAAE,IAAI;YACZ,QAAQ,EAAE,WAAW;YACrB,QAAQ,EAAE,QAAQ,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,MAAM,EAAE;YACxC,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,SAAS,EAAE,CAAC,CAAC,MAAM,IAAI,IAAI;YAC3B,QAAQ,EAAE;gBACR,MAAM,EAAE,UAAU;gBAClB,SAAS,EAAE,GAAG,CAAC,SAAS;gBACxB,MAAM,EAAE,CAAC,CAAC,MAAM;aACjB;YACD,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;SACtB,CAAC;QACF,MAAM,gBAAgB,CAAC,MAAM,CAAC,CAAC;IACjC,CAAC;AACH,CAAC;AAED,yEAAyE;AAEzE,MAAM,YAAY,GAAG,sBAAsB,CAAC;AAC5C,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,CAAC,cAAc,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC,CAAC;AAEhF;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,IAAY,EACZ,OAAgB;IAEhB,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC;IACnC,MAAM,GAAG,GAAa,EAAE,CAAC;IAEzB,SAAS,UAAU,CAAC,IAAY,EAAE,UAAkB;QAClD,MAAM,IAAI,GAAG,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACrC,IAAI,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,OAAO,IAAI,CAAC;QACzC,IAAI,UAAU,KAAK,OAAO,IAAI,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC1D,4CAA4C;YAC5C,OAAO,CAAC,+BAA+B,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACrD,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,SAAS,IAAI,CAAC,GAAW,EAAE,UAAkB;QAC3C,IAAI,OAAmC,CAAC;QACxC,IAAI,CAAC;YACH,OAAO,GAAG,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;QACzD,CAAC;QAAC,MAAM,CAAC;YACP,OAAO;QACT,CAAC;QACD,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;YAC5C,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;gBACxB,IAAI,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;oBAAE,SAAS;gBACtE,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;YACzB,CAAC;iBAAM,IAAI,KAAK,CAAC,MAAM,EAAE,IAAI,UAAU,CAAC,IAAI,EAAE,UAAU,CAAC,EAAE,CAAC;gBAC1D,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACjB,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;IACpC,GAAG,CAAC,IAAI,EAAE,CAAC;IAEX,IAAI,CAAC,OAAO;QAAE,OAAO,GAAG,CAAC;IACzB,OAAO,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;AACzE,CAAC;AAED,yEAAyE;AACzE,SAAS,YAAY,CAAC,GAA4B;IAChD,MAAM,UAAU,GAAc,EAAE,CAAC;IACjC,IAAI,GAAG,CAAC,OAAO,KAAK,SAAS;QAAE,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAC5D,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;QAC/C,IAAI,GAAG,KAAK,SAAS;YAAE,SAAS;QAChC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACzB,CAAC;IAED,MAAM,KAAK,GAAW,EAAE,CAAC;IACzB,KAAK,MAAM,CAAC,IAAI,UAAU,CAAC,IAAI,EAAE,EAAE,CAAC;QAClC,IACE,CAAC;YACD,OAAO,CAAC,KAAK,QAAQ;YACrB,OAAQ,CAAU,CAAC,IAAI,KAAK,QAAQ;YACpC,KAAK,CAAC,OAAO,CAAE,CAAU,CAAC,OAAO,CAAC;YACjC,CAAU,CAAC,KAAK,EACjB,CAAC;YACD,KAAK,CAAC,IAAI,CAAC,CAAS,CAAC,CAAC;QACxB,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,8EAA8E;AAC9E,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,IAAY,EACZ,OAAgB;IAEhB,MAAM,KAAK,GAAG,MAAM,iBAAiB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACrD,MAAM,KAAK,GAAW,EAAE,CAAC;IACzB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,GAAG,GAAG,CAAC,MAAM,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAGlD,CAAC;QACF,KAAK,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC;IACnC,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;AAC1B,CAAC;AAuBD;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,IAAI,GAAwB,EAAE;IAE9B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;IAEtC,IAAI,KAAK,GAAa,EAAE,CAAC;IACzB,IAAI,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;IACvB,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAClD,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;QACrB,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;IACvB,CAAC;IAED,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;IACnE,MAAM,MAAM,GACV,IAAI,CAAC,MAAM;QACX,CAAC,WAAW;YACV,CAAC,CAAC,MAAM,iBAAiB,CAAC;gBACtB,OAAO,EAAE,IAAI,CAAC,OAAO,IAAI,CAAC,MAAM,eAAe,CAAC,GAAG,CAAC,CAAC;gBACrD,YAAY,EAAE,IAAI,CAAC,YAAY;aAChC,CAAC;YACJ,CAAC,CAAC,iBAAiB,EAAE,CAAC,CAAC;IAE3B,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE;QAC3C,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;QACzC,OAAO,EAAE,IAAI,CAAC,OAAO,IAAI,IAAI;KAC9B,CAAC,CAAC;IACH,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;AAC3B,CAAC;AAED;;;;GAIG;AACH,KAAK,UAAU,eAAe,CAC5B,GAAW;IAEX,IAAI,CAAC;QACH,MAAM,EAAE,mBAAmB,EAAE,GAC3B,MAAM,MAAM,CAAC,+BAA+B,CAAC,CAAC;QAChD,MAAM,UAAU,GAAG,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;QACjD,OAAO,MAAM,mBAAmB,CAAC,aAAa,CAAC,UAAU,GAAG,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;IACzE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,SAAS,iBAAiB;IACxB,OAAO;QACL,MAAM,EAAE,EAA2B;QACnC,KAAK,EAAE,OAAO;QACd,KAAK,CAAC,QAAQ;YACZ,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;QAClE,CAAC;QACD,cAAc;YACZ,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;QACjE,CAAC;KACF,CAAC;AACJ,CAAC","sourcesContent":["/*\n The evals runner: discover `.eval.ts` / `evals/.ts` files, run each eval\n * through its scorer pipeline against the real agent loop, score, and report.\n \n It is the engine behind `agent-native eval` — when used as a CI deploy gate\n * the CLI exits non-zero if any eval scores below its threshold.\n \n Two layers:\n * - `scoreEval` / `runEvals` — pure orchestration over an `AgentRunner` and\n * a list of evals. Fully unit-testable with an injected runner (no model).\n * - `discoverEvalFiles` / `loadEvals` — filesystem discovery + dynamic import\n * of author-written eval modules.\n \n Results are also (best-effort) written to the observability eval store so a\n * dashboard can surface CI eval history next to production run evals.\n /\n\nimport nodePath from \"node:path\";\nimport { pathToFileURL } from \"node:url\";\n\nimport type { ActionEntry } from \"../agent/production-agent.js\";\nimport { insertEvalResult } from \"../observability/store.js\";\nimport type { EvalResult as ObservabilityEvalResult } from \"../observability/types.js\";\nimport type { AgentRunner } from \"./agent-runner.js\";\nimport { createAgentRunner } from \"./agent-runner.js\";\nimport { DEFAULT_EVAL_THRESHOLD } from \"./define-eval.js\";\nimport { clamp01 } from \"./scorer.js\";\nimport type {\n AgentRunOutput,\n Eval,\n EvalResultRow,\n EvalRunReport,\n ScorerResult,\n} from \"./types.js\";\n\n// ─── Scoring orchestration ────────────────────────────────────────────\n\n/* Run one scorer's pipeline (preprocess → analyze → score → reason). /\nasync function runScorer(\n scorer: Eval[\"scorers\"][number],\n run: AgentRunOutput,\n runner: AgentRunner,\n threshold: number,\n): Promise<ScorerResult> {\n try {\n const pre = scorer.preprocess ? await scorer.preprocess(run) : run;\n const analysis = scorer.analyze\n ? await scorer.analyze(pre as never, runner.analyzeContext())\n : pre;\n const rawScore = await scorer.generateScore(analysis as never);\n const score = clamp01(rawScore);\n const reason = scorer.generateReason\n ? await scorer.generateReason({\n run,\n analysis: analysis as never,\n score,\n })\n : undefined;\n return { scorer: scorer.name, score, reason, passed: score >= threshold };\n } catch (err) {\n // A scorer that throws is a failed scorer, not a crashed run — degrade\n // gracefully so one bad scorer can't take down the whole CI gate.\n return {\n scorer: scorer.name,\n score: 0,\n reason: `Scorer errored: ${err instanceof Error ? err.message : String(err)}`,\n passed: false,\n };\n }\n}\n\n/* Run a single eval: invoke the agent, then score with each scorer. /\nexport async function scoreEval(\n evalCase: Eval,\n runner: AgentRunner,\n opts: { thresholdOverride?: number } = {},\n): Promise<EvalResultRow> {\n const threshold =\n opts.thresholdOverride ?? evalCase.threshold ?? DEFAULT_EVAL_THRESHOLD;\n\n if (evalCase.skipReason) {\n return {\n eval: evalCase.name,\n threshold,\n scores: [],\n status: \"skipped\",\n skipReason: evalCase.skipReason,\n // Keep the legacy boolean gate friendly: skipped rows do not fail CI.\n passed: true,\n avgScore: 0,\n durationMs: 0,\n };\n }\n\n let run: AgentRunOutput;\n if (evalCase.run) {\n run = await evalCase.run({\n input: evalCase.input,\n runAgent: (input) => runner.runAgent(input),\n });\n } else {\n run = await runner.runAgent(evalCase.input);\n }\n\n const scores: ScorerResult[] = [];\n for (const scorer of evalCase.scorers) {\n scores.push(await runScorer(scorer, run, runner, threshold));\n }\n\n const avgScore =\n scores.length > 0\n ? scores.reduce((s, r) => s + r.score, 0) / scores.length\n : 0;\n\n return {\n eval: evalCase.name,\n threshold,\n scores,\n // A run that errored, or any sub-threshold scorer, fails the case.\n passed: run.ok && scores.every((s) => s.passed),\n status: run.ok && scores.every((s) => s.passed) ? \"passed\" : \"failed\",\n avgScore,\n durationMs: run.durationMs,\n error: run.ok ? undefined : run.error,\n };\n}\n\n/* Run a batch of evals against one runner and aggregate a report. /\nexport async function runEvals(\n evals: Eval[],\n runner: AgentRunner,\n opts: { thresholdOverride?: number; persist?: boolean } = {},\n): Promise<EvalRunReport> {\n const results: EvalResultRow[] = [];\n for (const evalCase of evals) {\n const row = await scoreEval(evalCase, runner, opts);\n results.push(row);\n if (opts.persist && row.status !== \"skipped\") {\n await persistEvalRow(row).catch(() => {});\n }\n }\n\n const passed = results.filter((r) => r.passed).length;\n const skipped = results.filter((r) => r.status === \"skipped\").length;\n return {\n total: results.length,\n passed,\n failed: results.filter((r) => r.status !== \"skipped\" && !r.passed).length,\n skipped,\n results,\n };\n}\n\n/\n Best-effort write of one eval result to the observability eval store so a\n * dashboard can show CI eval history alongside production run evals. We write\n * one row per (eval × scorer), tagged `evalType: \"automated\"` with a synthetic\n * `eval:` run id.\n \n TODO(live-sampling): the same scorer list should also run on a sampled\n * fraction of real production runs. That hook belongs in the agent loop's\n * (not-yet-added) post-run processor seam: when a run finishes, roll the\n * configured sample rate and, if it hits, replay the run output through these\n * scorers and write the rows here. Wiring it now would require the in-loop\n * processor seam another wave is adding — so this is the single intended\n * attachment point, intentionally left as a note.\n /\nasync function persistEvalRow(row: EvalResultRow): Promise<void> {\n const runId = `eval:${row.eval}:${Date.now()}`;\n for (const s of row.scores) {\n const result: ObservabilityEvalResult = {\n id: crypto.randomUUID(),\n runId,\n threadId: null,\n userId: null,\n evalType: \"automated\",\n criteria: `eval:${row.eval}:${s.scorer}`,\n score: s.score,\n reasoning: s.reason ?? null,\n metadata: {\n source: \"cli-eval\",\n threshold: row.threshold,\n passed: s.passed,\n },\n createdAt: Date.now(),\n };\n await insertEvalResult(result);\n }\n}\n\n// ─── Discovery + loading ──────────────────────────────────────────────\n\nconst EVAL_FILE_RE = /\\.eval\\.(ts\|js\|mjs)$/;\nconst SKIP_DIRS = new Set([\"node_modules\", \"dist\", \".git\", \".output\", \"build\"]);\n\n/\n Walk `root` for eval files. Matches two conventions:\n * - any `*\\/.eval.ts` (co-located with code), and\n * - any `.ts` directly inside an `evals/` directory.\n `pattern` further filters by substring of the relative path.\n /\nexport async function discoverEvalFiles(\n root: string,\n pattern?: string,\n): Promise<string[]> {\n const fs = await import(\"node:fs\");\n const out: string[] = [];\n\n function isEvalFile(full: string, parentName: string): boolean {\n const base = nodePath.basename(full);\n if (EVAL_FILE_RE.test(base)) return true;\n if (parentName === \"evals\" && /\\.(ts\|js\|mjs)$/.test(base)) {\n // Skip obvious support files inside evals/.\n return !/\\.(spec\|test\|d)\\.(ts\|js\|mjs)$/.test(base);\n }\n return false;\n }\n\n function walk(dir: string, parentName: string): void {\n let entries: import(\"node:fs\").Dirent[];\n try {\n entries = fs.readdirSync(dir, { withFileTypes: true });\n } catch {\n return;\n }\n for (const entry of entries) {\n const full = nodePath.join(dir, entry.name);\n if (entry.isDirectory()) {\n if (SKIP_DIRS.has(entry.name) \|\| entry.name.startsWith(\".\")) continue;\n walk(full, entry.name);\n } else if (entry.isFile() && isEvalFile(full, parentName)) {\n out.push(full);\n }\n }\n }\n\n walk(root, nodePath.basename(root));\n out.sort();\n\n if (!pattern) return out;\n return out.filter((f) => nodePath.relative(root, f).includes(pattern));\n}\n\n/* Pull `Eval` definitions out of a dynamically-imported eval module. /\nfunction extractEvals(mod: Record<string, unknown>): Eval[] {\n const candidates: unknown[] = [];\n if (mod.default !== undefined) candidates.push(mod.default);\n for (const [key, value] of Object.entries(mod)) {\n if (key === \"default\") continue;\n candidates.push(value);\n }\n\n const evals: Eval[] = [];\n for (const c of candidates.flat()) {\n if (\n c &&\n typeof c === \"object\" &&\n typeof (c as Eval).name === \"string\" &&\n Array.isArray((c as Eval).scorers) &&\n (c as Eval).input\n ) {\n evals.push(c as Eval);\n }\n }\n return evals;\n}\n\n/* Discover and import all eval files under `root`, returning their evals. /\nexport async function loadEvals(\n root: string,\n pattern?: string,\n): Promise<{ files: string[]; evals: Eval[] }> {\n const files = await discoverEvalFiles(root, pattern);\n const evals: Eval[] = [];\n for (const file of files) {\n const mod = (await import(pathToFileURL(file).href)) as Record<\n string,\n unknown\n >;\n evals.push(...extractEvals(mod));\n }\n return { files, evals };\n}\n\n// ─── High-level entry used by the CLI ─────────────────────────────────\n\nexport interface RunEvalSuiteOptions {\n /* App root to discover eval files + actions under. Defaults to cwd. /\n cwd?: string;\n /* Substring filter on the eval file path. /\n pattern?: string;\n /* Global threshold override (wins over per-eval thresholds). /\n thresholdOverride?: number;\n /* App actions to expose to the agent. Auto-discovered when omitted. /\n actions?: Record<string, ActionEntry>;\n /* System prompt for runs. /\n systemPrompt?: string;\n /* Write results to the observability eval store (default true). /\n persist?: boolean;\n /* Pre-built runner (tests inject this to avoid touching engine/loop). /\n runner?: AgentRunner;\n /* Pre-loaded evals (tests inject this to skip filesystem discovery). /\n evals?: Eval[];\n}\n\n/\n End-to-end: load evals, build a runner, score, report. The CLI wraps this\n * and maps `report.failed > 0` to a non-zero exit code (the CI gate).\n /\nexport async function runEvalSuite(\n opts: RunEvalSuiteOptions = {},\n): Promise<{ report: EvalRunReport; files: string[] }> {\n const cwd = opts.cwd ?? process.cwd();\n\n let files: string[] = [];\n let evals = opts.evals;\n if (!evals) {\n const loaded = await loadEvals(cwd, opts.pattern);\n files = loaded.files;\n evals = loaded.evals;\n }\n\n const needsRunner = evals.some((evalCase) => !evalCase.skipReason);\n const runner =\n opts.runner ??\n (needsRunner\n ? await createAgentRunner({\n actions: opts.actions ?? (await discoverActions(cwd)),\n systemPrompt: opts.systemPrompt,\n })\n : createInertRunner());\n\n const report = await runEvals(evals, runner, {\n thresholdOverride: opts.thresholdOverride,\n persist: opts.persist ?? true,\n });\n return { report, files };\n}\n\n/\n Discover the app's actions so the agent under test has the real tool\n * surface. Lazy-imports `autoDiscoverActions` to keep server-only deps out of\n * any browser bundle that might touch this module's types.\n */\nasync function discoverActions(\n cwd: string,\n): Promise<Record<string, ActionEntry>> {\n try {\n const { autoDiscoverActions } =\n await import(\"../server/action-discovery.js\");\n const actionsDir = nodePath.join(cwd, \"actions\");\n return await autoDiscoverActions(pathToFileURL(actionsDir + \"/\").href);\n } catch {\n return {};\n }\n}\n\nfunction createInertRunner(): AgentRunner {\n return {\n engine: {} as AgentRunner[\"engine\"],\n model: \"inert\",\n async runAgent() {\n throw new Error(\"Eval unexpectedly requested the agent runner\");\n },\n analyzeContext() {\n throw new Error(\"Eval unexpectedly requested analyze context\");\n },\n };\n}\n"]}

package/dist/eval/types.d.ts CHANGED Viewed

@@ -122,6 +122,12 @@ export interface EvalRunContext {
 export interface Eval {
     name: string;
     input: EvalInput;
+    /**
+     * Skip this case without running the agent or scorers. Use for opt-in
+     * suites that need secrets, live provider credentials, or manual/nightly
+     * gates. Skipped rows are reported distinctly and do not fail the suite.
+     */
+    skipReason?: string;
     /**
      * Optional override for how the agent is run for this case. Defaults to the
      * runner's headless `runAgent`. Use this to do custom setup (seed data,
@@ -144,6 +150,10 @@ export interface EvalResultRow {
     eval: string;
     threshold: number;
     scores: ScorerResult[];
+    /** Explicit row status for reports. Older consumers can still read `passed`. */
+    status?: "passed" | "failed" | "skipped";
+    /** Human-readable reason when `status` is `skipped`. */
+    skipReason?: string;
     /** True only when every scorer passed. */
     passed: boolean;
     /** Mean of the scorer scores, for at-a-glance ranking. */
@@ -157,6 +167,7 @@ export interface EvalRunReport {
     total: number;
     passed: number;
     failed: number;
+    skipped?: number;
     results: EvalResultRow[];
 }
 //# sourceMappingURL=types.d.ts.map

package/dist/eval/types.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/eval/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAI5D;;;;GAIG;AACH,MAAM,WAAW,cAAc;IAC7B,0DAA0D;IAC1D,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,+DAA+D;IAC/D,QAAQ,CAAC,SAAS,EAAE,SAAS,MAAM,EAAE,CAAC;IACtC,gEAAgE;IAChE,QAAQ,CAAC,EAAE,EAAE,OAAO,CAAC;IACrB,kDAAkD;IAClD,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,iFAAiF;IACjF,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,sDAAsD;IACtD,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC7B;AAID;;;;;;GAMG;AACH,MAAM,WAAW,oBAAoB;IACnC,oEAAoE;IACpE,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC;IAC7B,gDAAgD;IAChD,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB;;;;OAIG;IACH,KAAK,CAAC,IAAI,EAAE;QACV,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,MAAM,EAAE,MAAM,CAAC;QACf,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,MAAM,CAAC,EAAE,WAAW,CAAC;KACtB,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CACrB;AAED;;;;;;;;;;GAUG;AACH,MAAM,WAAW,MAAM,CAAC,GAAG,GAAG,cAAc,EAAE,GAAG,GAAG,GAAG;IACrD,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,CAAC,GAAG,EAAE,cAAc,GAAG,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;IACrD,OAAO,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,GAAG,EAAE,oBAAoB,GAAG,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;IACpE,sDAAsD;IACtD,aAAa,CAAC,QAAQ,EAAE,GAAG,GAAG,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACvD,cAAc,CAAC,CAAC,IAAI,EAAE;QACpB,GAAG,EAAE,cAAc,CAAC;QACpB,QAAQ,EAAE,GAAG,CAAC;QACd,KAAK,EAAE,MAAM,CAAC;KACf,GAAG,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CAC9B;AAED,kDAAkD;AAClD,MAAM,WAAW,gBAAgB,CAAC,GAAG,GAAG,cAAc,EAAE,GAAG,GAAG,GAAG;IAC/D,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,CAAC,CAAC,GAAG,EAAE,cAAc,GAAG,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;IACrD,OAAO,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,GAAG,EAAE,oBAAoB,GAAG,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;IACpE,aAAa,CAAC,QAAQ,EAAE,GAAG,GAAG,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACvD,cAAc,CAAC,CAAC,IAAI,EAAE;QACpB,GAAG,EAAE,cAAc,CAAC;QACpB,QAAQ,EAAE,GAAG,CAAC;QACd,KAAK,EAAE,MAAM,CAAC;KACf,GAAG,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CAC9B;AAID,6DAA6D;AAC7D,MAAM,WAAW,SAAS;IACxB,mDAAmD;IACnD,MAAM,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,OAAO,CAAC,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,GAAG,WAAW,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CAC/D;AAED,2DAA2D;AAC3D,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,KAAK,EAAE,SAAS,CAAC;IAC1B,8EAA8E;IAC9E,QAAQ,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;CACrD;AAED;;;;GAIG;AACH,MAAM,WAAW,IAAI;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,SAAS,CAAC;IACjB;;;;OAIG;IACH,GAAG,CAAC,CAAC,GAAG,EAAE,cAAc,GAAG,cAAc,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;IACpE,OAAO,EAAE,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,CAAC;IAC5B,oEAAoE;IACpE,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAID,iFAAiF;AACjF,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,OAAO,CAAC;CACjB;AAED,gEAAgE;AAChE,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,YAAY,EAAE,CAAC;IACvB,0CAA0C;IAC1C,MAAM,EAAE,OAAO,CAAC;IAChB,0DAA0D;IAC1D,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,qDAAqD;IACrD,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,8BAA8B;AAC9B,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,aAAa,EAAE,CAAC;CAC1B"}
1	+ {"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/eval/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAI5D;;;;GAIG;AACH,MAAM,WAAW,cAAc;IAC7B,0DAA0D;IAC1D,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,+DAA+D;IAC/D,QAAQ,CAAC,SAAS,EAAE,SAAS,MAAM,EAAE,CAAC;IACtC,gEAAgE;IAChE,QAAQ,CAAC,EAAE,EAAE,OAAO,CAAC;IACrB,kDAAkD;IAClD,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,iFAAiF;IACjF,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,sDAAsD;IACtD,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC7B;AAID;;;;;;GAMG;AACH,MAAM,WAAW,oBAAoB;IACnC,oEAAoE;IACpE,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC;IAC7B,gDAAgD;IAChD,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB;;;;OAIG;IACH,KAAK,CAAC,IAAI,EAAE;QACV,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,MAAM,EAAE,MAAM,CAAC;QACf,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,MAAM,CAAC,EAAE,WAAW,CAAC;KACtB,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CACrB;AAED;;;;;;;;;;GAUG;AACH,MAAM,WAAW,MAAM,CAAC,GAAG,GAAG,cAAc,EAAE,GAAG,GAAG,GAAG;IACrD,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,CAAC,GAAG,EAAE,cAAc,GAAG,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;IACrD,OAAO,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,GAAG,EAAE,oBAAoB,GAAG,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;IACpE,sDAAsD;IACtD,aAAa,CAAC,QAAQ,EAAE,GAAG,GAAG,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACvD,cAAc,CAAC,CAAC,IAAI,EAAE;QACpB,GAAG,EAAE,cAAc,CAAC;QACpB,QAAQ,EAAE,GAAG,CAAC;QACd,KAAK,EAAE,MAAM,CAAC;KACf,GAAG,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CAC9B;AAED,kDAAkD;AAClD,MAAM,WAAW,gBAAgB,CAAC,GAAG,GAAG,cAAc,EAAE,GAAG,GAAG,GAAG;IAC/D,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,CAAC,CAAC,GAAG,EAAE,cAAc,GAAG,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;IACrD,OAAO,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,GAAG,EAAE,oBAAoB,GAAG,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;IACpE,aAAa,CAAC,QAAQ,EAAE,GAAG,GAAG,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACvD,cAAc,CAAC,CAAC,IAAI,EAAE;QACpB,GAAG,EAAE,cAAc,CAAC;QACpB,QAAQ,EAAE,GAAG,CAAC;QACd,KAAK,EAAE,MAAM,CAAC;KACf,GAAG,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CAC9B;AAID,6DAA6D;AAC7D,MAAM,WAAW,SAAS;IACxB,mDAAmD;IACnD,MAAM,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,OAAO,CAAC,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,GAAG,WAAW,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CAC/D;AAED,2DAA2D;AAC3D,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,KAAK,EAAE,SAAS,CAAC;IAC1B,8EAA8E;IAC9E,QAAQ,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;CACrD;AAED;;;;GAIG;AACH,MAAM,WAAW,IAAI;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,SAAS,CAAC;IACjB;;;;OAIG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;;OAIG;IACH,GAAG,CAAC,CAAC,GAAG,EAAE,cAAc,GAAG,cAAc,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;IACpE,OAAO,EAAE,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,CAAC;IAC5B,oEAAoE;IACpE,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAID,iFAAiF;AACjF,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,OAAO,CAAC;CACjB;AAED,gEAAgE;AAChE,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,YAAY,EAAE,CAAC;IACvB,gFAAgF;IAChF,MAAM,CAAC,EAAE,QAAQ,GAAG,QAAQ,GAAG,SAAS,CAAC;IACzC,wDAAwD;IACxD,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,0CAA0C;IAC1C,MAAM,EAAE,OAAO,CAAC;IAChB,0DAA0D;IAC1D,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,qDAAqD;IACrD,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,8BAA8B;AAC9B,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,aAAa,EAAE,CAAC;CAC1B"}

package/dist/eval/types.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/eval/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG","sourcesContent":["/*\n Types for the first-class evals primitive.\n \n This is a test-case eval system (define a prompt + expected behavior,\n * actually run the agent, score the output) — distinct from the post-hoc\n * run-scoring engine in `../observability/evals.ts`, which scores already-\n * completed production runs. The two are complementary:\n \n - `observability/evals.ts` — \"how did this real run do?\" (passive,\n * sampled, lives next to traces).\n * - `eval/` (this module) — \"does the agent do the right thing on this\n fixed input?\" (active, deterministic CI gate, run via the CLI).\n \n The pipeline shape (preprocess → analyze → generateScore → generateReason)\n * is borrowed from Mastra's scorer design: each scorer is a small, composable\n * 4-step pipeline so a single scorer can mix plain-JS checks with an LLM\n * judge while still producing one normalized 0..1 number plus a reason.\n /\n\nimport type { AgentEngine } from \"../agent/engine/types.js\";\n\n// ─── Agent run output ─────────────────────────────────────────────────\n\n/\n The result of actually running the agent loop for one eval input. Scorers\n * receive this as the thing under test. It is intentionally small and\n * transport-agnostic so a scorer never reaches into framework internals.\n /\nexport interface AgentRunOutput {\n /* Concatenated assistant text emitted across the run. /\n readonly text: string;\n /* Names of tools/actions the agent invoked, in call order. /\n readonly toolCalls: readonly string[];\n /* Whether the run completed without a terminal error event. /\n readonly ok: boolean;\n /* Terminal error message, if the run errored. /\n readonly error?: string;\n /* Synthetic run id, useful for writing eval rows to the observability store. /\n readonly runId: string;\n /* Wall-clock duration of the run in milliseconds. /\n readonly durationMs: number;\n}\n\n// ─── Scorer pipeline ──────────────────────────────────────────────────\n\n/\n Context handed to a scorer's analyze step when it needs an LLM judge. The\n * engine/model are resolved by the runner from the existing engine registry —\n * a scorer NEVER hardcodes a provider or model, keeping evals provider-\n * agnostic. `judge()` is a convenience that streams a single judging turn and\n * returns the raw model text.\n /\nexport interface ScorerAnalyzeContext {\n /* The resolved, provider-agnostic engine for LLM-judge scorers. /\n readonly engine: AgentEngine;\n /* The resolved model string for the engine. /\n readonly model: string;\n /\n Run a single LLM judging turn. Returns the model's raw text output. Used\n * by `llmJudge` and any custom LLM-backed analyze step. Provider-agnostic —\n * the engine is whatever the app/CLI resolved.\n /\n judge(opts: {\n systemPrompt?: string;\n prompt: string;\n maxOutputTokens?: number;\n signal?: AbortSignal;\n }): Promise<string>;\n}\n\n/\n A 4-step scoring pipeline (Mastra-style):\n \n preprocess(run) → x (transform the run/output; optional)\n * analyze(x, ctx) → analysis (plain JS OR an LLM judge; optional)\n * generateScore(a) → 0..1 (REQUIRED, normalized)\n * generateReason(...) → string (human-readable why; optional)\n \n Generics flow `Pre` (preprocess output) → `Ana` (analyze output) so a\n * single scorer is fully typed end-to-end.\n /\nexport interface Scorer<Pre = AgentRunOutput, Ana = Pre> {\n readonly name: string;\n preprocess?(run: AgentRunOutput): Pre \| Promise<Pre>;\n analyze?(input: Pre, ctx: ScorerAnalyzeContext): Ana \| Promise<Ana>;\n /* REQUIRED. Returns a normalized score in [0, 1]. /\n generateScore(analysis: Ana): number \| Promise<number>;\n generateReason?(args: {\n run: AgentRunOutput;\n analysis: Ana;\n score: number;\n }): string \| Promise<string>;\n}\n\n/* Definition object passed to `createScorer`. /\nexport interface ScorerDefinition<Pre = AgentRunOutput, Ana = Pre> {\n name: string;\n preprocess?(run: AgentRunOutput): Pre \| Promise<Pre>;\n analyze?(input: Pre, ctx: ScorerAnalyzeContext): Ana \| Promise<Ana>;\n generateScore(analysis: Ana): number \| Promise<number>;\n generateReason?(args: {\n run: AgentRunOutput;\n analysis: Ana;\n score: number;\n }): string \| Promise<string>;\n}\n\n// ─── Eval definition ──────────────────────────────────────────────────\n\n/* The prompt + optional setup that drives one eval case. /\nexport interface EvalInput {\n /* The user prompt / message sent to the agent. /\n prompt: string;\n /\n Optional prior conversation turns to seed before `prompt`. Each is a\n * plain { role, text } pair; the runner converts them to engine messages.\n /\n history?: Array<{ role: \"user\" \| \"assistant\"; text: string }>;\n}\n\n/* Context passed to an eval's optional `run` override. /\nexport interface EvalRunContext {\n readonly input: EvalInput;\n /* The default agent runner — invoke it to run the agent loop as a caller. /\n runAgent(input: EvalInput): Promise<AgentRunOutput>;\n}\n\n/\n A named eval = one test case. `scorers` produce per-scorer rows; the case\n * passes when EVERY scorer meets `threshold` (default 0.5, overridable per\n * eval and globally from the CLI).\n /\nexport interface Eval {\n name: string;\n input: EvalInput;\n /\n Optional override for how the agent is run for this case. Defaults to the\n * runner's headless `runAgent`. Use this to do custom setup (seed data,\n * multi-turn) before/after the agent call.\n /\n run?(ctx: EvalRunContext): AgentRunOutput \| Promise<AgentRunOutput>;\n scorers: Scorer<any, any>[];\n /* Minimum acceptable score (per scorer) in [0, 1]. Default 0.5. /\n threshold?: number;\n}\n\n// ─── Results ──────────────────────────────────────────────────────────\n\n/* One result row per (eval × scorer). Stores both the number AND the reason. /\nexport interface ScorerResult {\n scorer: string;\n score: number;\n reason?: string;\n passed: boolean;\n}\n\n/* Aggregated result for a single eval (all of its scorers). /\nexport interface EvalResultRow {\n eval: string;\n threshold: number;\n scores: ScorerResult[];\n /* True only when every scorer passed. /\n passed: boolean;\n /* Mean of the scorer scores, for at-a-glance ranking. /\n avgScore: number;\n durationMs: number;\n /* Terminal error if the agent run itself failed. /\n error?: string;\n}\n\n/* The full runner report. */\nexport interface EvalRunReport {\n total: number;\n passed: number;\n failed: number;\n results: EvalResultRow[];\n}\n"]}
1	+ {"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/eval/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG","sourcesContent":["/*\n Types for the first-class evals primitive.\n \n This is a test-case eval system (define a prompt + expected behavior,\n * actually run the agent, score the output) — distinct from the post-hoc\n * run-scoring engine in `../observability/evals.ts`, which scores already-\n * completed production runs. The two are complementary:\n \n - `observability/evals.ts` — \"how did this real run do?\" (passive,\n * sampled, lives next to traces).\n * - `eval/` (this module) — \"does the agent do the right thing on this\n fixed input?\" (active, deterministic CI gate, run via the CLI).\n \n The pipeline shape (preprocess → analyze → generateScore → generateReason)\n * is borrowed from Mastra's scorer design: each scorer is a small, composable\n * 4-step pipeline so a single scorer can mix plain-JS checks with an LLM\n * judge while still producing one normalized 0..1 number plus a reason.\n /\n\nimport type { AgentEngine } from \"../agent/engine/types.js\";\n\n// ─── Agent run output ─────────────────────────────────────────────────\n\n/\n The result of actually running the agent loop for one eval input. Scorers\n * receive this as the thing under test. It is intentionally small and\n * transport-agnostic so a scorer never reaches into framework internals.\n /\nexport interface AgentRunOutput {\n /* Concatenated assistant text emitted across the run. /\n readonly text: string;\n /* Names of tools/actions the agent invoked, in call order. /\n readonly toolCalls: readonly string[];\n /* Whether the run completed without a terminal error event. /\n readonly ok: boolean;\n /* Terminal error message, if the run errored. /\n readonly error?: string;\n /* Synthetic run id, useful for writing eval rows to the observability store. /\n readonly runId: string;\n /* Wall-clock duration of the run in milliseconds. /\n readonly durationMs: number;\n}\n\n// ─── Scorer pipeline ──────────────────────────────────────────────────\n\n/\n Context handed to a scorer's analyze step when it needs an LLM judge. The\n * engine/model are resolved by the runner from the existing engine registry —\n * a scorer NEVER hardcodes a provider or model, keeping evals provider-\n * agnostic. `judge()` is a convenience that streams a single judging turn and\n * returns the raw model text.\n /\nexport interface ScorerAnalyzeContext {\n /* The resolved, provider-agnostic engine for LLM-judge scorers. /\n readonly engine: AgentEngine;\n /* The resolved model string for the engine. /\n readonly model: string;\n /\n Run a single LLM judging turn. Returns the model's raw text output. Used\n * by `llmJudge` and any custom LLM-backed analyze step. Provider-agnostic —\n * the engine is whatever the app/CLI resolved.\n /\n judge(opts: {\n systemPrompt?: string;\n prompt: string;\n maxOutputTokens?: number;\n signal?: AbortSignal;\n }): Promise<string>;\n}\n\n/\n A 4-step scoring pipeline (Mastra-style):\n \n preprocess(run) → x (transform the run/output; optional)\n * analyze(x, ctx) → analysis (plain JS OR an LLM judge; optional)\n * generateScore(a) → 0..1 (REQUIRED, normalized)\n * generateReason(...) → string (human-readable why; optional)\n \n Generics flow `Pre` (preprocess output) → `Ana` (analyze output) so a\n * single scorer is fully typed end-to-end.\n /\nexport interface Scorer<Pre = AgentRunOutput, Ana = Pre> {\n readonly name: string;\n preprocess?(run: AgentRunOutput): Pre \| Promise<Pre>;\n analyze?(input: Pre, ctx: ScorerAnalyzeContext): Ana \| Promise<Ana>;\n /* REQUIRED. Returns a normalized score in [0, 1]. /\n generateScore(analysis: Ana): number \| Promise<number>;\n generateReason?(args: {\n run: AgentRunOutput;\n analysis: Ana;\n score: number;\n }): string \| Promise<string>;\n}\n\n/* Definition object passed to `createScorer`. /\nexport interface ScorerDefinition<Pre = AgentRunOutput, Ana = Pre> {\n name: string;\n preprocess?(run: AgentRunOutput): Pre \| Promise<Pre>;\n analyze?(input: Pre, ctx: ScorerAnalyzeContext): Ana \| Promise<Ana>;\n generateScore(analysis: Ana): number \| Promise<number>;\n generateReason?(args: {\n run: AgentRunOutput;\n analysis: Ana;\n score: number;\n }): string \| Promise<string>;\n}\n\n// ─── Eval definition ──────────────────────────────────────────────────\n\n/* The prompt + optional setup that drives one eval case. /\nexport interface EvalInput {\n /* The user prompt / message sent to the agent. /\n prompt: string;\n /\n Optional prior conversation turns to seed before `prompt`. Each is a\n * plain { role, text } pair; the runner converts them to engine messages.\n /\n history?: Array<{ role: \"user\" \| \"assistant\"; text: string }>;\n}\n\n/* Context passed to an eval's optional `run` override. /\nexport interface EvalRunContext {\n readonly input: EvalInput;\n /* The default agent runner — invoke it to run the agent loop as a caller. /\n runAgent(input: EvalInput): Promise<AgentRunOutput>;\n}\n\n/\n A named eval = one test case. `scorers` produce per-scorer rows; the case\n * passes when EVERY scorer meets `threshold` (default 0.5, overridable per\n * eval and globally from the CLI).\n /\nexport interface Eval {\n name: string;\n input: EvalInput;\n /\n Skip this case without running the agent or scorers. Use for opt-in\n * suites that need secrets, live provider credentials, or manual/nightly\n * gates. Skipped rows are reported distinctly and do not fail the suite.\n /\n skipReason?: string;\n /\n Optional override for how the agent is run for this case. Defaults to the\n * runner's headless `runAgent`. Use this to do custom setup (seed data,\n * multi-turn) before/after the agent call.\n /\n run?(ctx: EvalRunContext): AgentRunOutput \| Promise<AgentRunOutput>;\n scorers: Scorer<any, any>[];\n /* Minimum acceptable score (per scorer) in [0, 1]. Default 0.5. /\n threshold?: number;\n}\n\n// ─── Results ──────────────────────────────────────────────────────────\n\n/* One result row per (eval × scorer). Stores both the number AND the reason. /\nexport interface ScorerResult {\n scorer: string;\n score: number;\n reason?: string;\n passed: boolean;\n}\n\n/* Aggregated result for a single eval (all of its scorers). /\nexport interface EvalResultRow {\n eval: string;\n threshold: number;\n scores: ScorerResult[];\n /* Explicit row status for reports. Older consumers can still read `passed`. /\n status?: \"passed\" \| \"failed\" \| \"skipped\";\n /* Human-readable reason when `status` is `skipped`. /\n skipReason?: string;\n /* True only when every scorer passed. /\n passed: boolean;\n /* Mean of the scorer scores, for at-a-glance ranking. /\n avgScore: number;\n durationMs: number;\n /* Terminal error if the agent run itself failed. /\n error?: string;\n}\n\n/* The full runner report. */\nexport interface EvalRunReport {\n total: number;\n passed: number;\n failed: number;\n skipped?: number;\n results: EvalResultRow[];\n}\n"]}

package/dist/file-upload/actions/upload-image.d.ts CHANGED Viewed

@@ -3,18 +3,18 @@ declare const _default: import("../../action.js").ActionDefinition<{
     url?: string;
     filename?: string;
 }, {
-    id?: undefined;
     error: string;
     configured?: undefined;
     connectPath?: undefined;
     url?: undefined;
+    id?: undefined;
     provider?: undefined;
 } | {
-    id?: undefined;
     error: string;
     configured: boolean;
     connectPath: string;
     url?: undefined;
+    id?: undefined;
     provider?: undefined;
 } | {
     error?: undefined;

package/dist/notifications/routes.d.ts CHANGED Viewed

@@ -11,14 +11,14 @@
  *   DELETE /_agent-native/notifications/:id         — delete
  */
 export declare function createNotificationsHandler(): import("h3").EventHandlerWithFetch<import("h3").EventHandlerRequest, Promise<"" | import("./types.js").Notification[] | {
+    error?: undefined;
     count: number;
     updated?: undefined;
-    error?: undefined;
     ok?: undefined;
 } | {
+    error?: undefined;
     count?: undefined;
     updated: number;
-    error?: undefined;
     ok?: undefined;
 } | {
     count?: undefined;
@@ -26,9 +26,9 @@ export declare function createNotificationsHandler(): import("h3").EventHandlerW
     error: string;
     ok?: undefined;
 } | {
+    error?: undefined;
     count?: undefined;
     updated?: undefined;
     ok: boolean;
-    error?: undefined;
 }>>;
 //# sourceMappingURL=routes.d.ts.map

package/dist/observability/routes.d.ts CHANGED Viewed

@@ -42,27 +42,27 @@ export declare function createObservabilityHandler(): import("h3").EventHandlerW
     avgEvalScore: number;
 } | {
     error?: undefined;
+    ok?: undefined;
     summary: import("./types.js").TraceSummary;
     spans: import("./types.js").TraceSpan[];
     id?: undefined;
-    ok?: undefined;
 } | {
     error?: undefined;
+    ok?: undefined;
     summary?: undefined;
     spans?: undefined;
     id: string;
-    ok?: undefined;
 } | {
+    ok?: undefined;
     summary?: undefined;
     spans?: undefined;
-    id?: undefined;
     error: any;
-    ok?: undefined;
+    id?: undefined;
 } | {
     error?: undefined;
     summary?: undefined;
     spans?: undefined;
-    id?: undefined;
     ok: boolean;
+    id?: undefined;
 }>>;
 //# sourceMappingURL=routes.d.ts.map

package/dist/observability/traces.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"traces.d.ts","sourceRoot":"","sources":["../../src/observability/traces.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AACnE,OAAO,KAAK,EAAE,cAAc,~~EAAE~~,MAAM,mBAAmB,CAAC;~~AAExD~~,OAAO,KAAK,EAA2B,mBAAmB,EAAE,MAAM,YAAY,CAAC;AAiB/E;;;wEAGwE;AACxE,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAE7D;AAoBD,wBAAsB,sBAAsB,IAAI,OAAO,CAAC,mBAAmB,CAAC,CAY3E;AAED,wBAAsB,mBAAmB,CAAC,IAAI,EAAE;IAC9C,YAAY,EAAE,CAAC,QAAQ,EAAE;QACvB,MAAM,EAAE,GAAG,CAAC;QACZ,KAAK,EAAE,MAAM,CAAC;QACd,YAAY,EAAE,MAAM,CAAC;QACrB,KAAK,EAAE,GAAG,EAAE,CAAC;QACb,QAAQ,EAAE,GAAG,EAAE,CAAC;QAChB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAC7B,IAAI,EAAE,CAAC,KAAK,EAAE,cAAc,KAAK,IAAI,CAAC;QACtC,MAAM,EAAE,WAAW,CAAC;QACpB,eAAe,CAAC,EAAE,GAAG,CAAC;KACvB,KAAK,OAAO,CAAC,cAAc,CAAC,CAAC;IAC9B,QAAQ,EAAE;QACR,MAAM,EAAE,GAAG,CAAC;QACZ,KAAK,EAAE,MAAM,CAAC;QACd,YAAY,EAAE,MAAM,CAAC;QACrB,KAAK,EAAE,GAAG,EAAE,CAAC;QACb,QAAQ,EAAE,GAAG,EAAE,CAAC;QAChB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAC7B,IAAI,EAAE,CAAC,KAAK,EAAE,cAAc,KAAK,IAAI,CAAC;QACtC,MAAM,EAAE,WAAW,CAAC;QACpB,eAAe,CAAC,EAAE,GAAG,CAAC;KACvB,CAAC;IACF,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB;;;iBAGa;IACb,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,MAAM,EAAE,mBAAmB,CAAC;CAC7B,GAAG,OAAO,CAAC,cAAc,CAAC,CAiT1B"}
1	+ {"version":3,"file":"traces.d.ts","sourceRoot":"","sources":["../../src/observability/traces.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,8BAA8B,CAAC;AACnE,OAAO,KAAK,EAAE,cAAc,EAAkB,MAAM,mBAAmB,CAAC;AAExE,OAAO,KAAK,EAA2B,mBAAmB,EAAE,MAAM,YAAY,CAAC;AAiB/E;;;wEAGwE;AACxE,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAE7D;AAoBD,wBAAsB,sBAAsB,IAAI,OAAO,CAAC,mBAAmB,CAAC,CAY3E;AAED,wBAAsB,mBAAmB,CAAC,IAAI,EAAE;IAC9C,YAAY,EAAE,CAAC,QAAQ,EAAE;QACvB,MAAM,EAAE,GAAG,CAAC;QACZ,KAAK,EAAE,MAAM,CAAC;QACd,YAAY,EAAE,MAAM,CAAC;QACrB,KAAK,EAAE,GAAG,EAAE,CAAC;QACb,QAAQ,EAAE,GAAG,EAAE,CAAC;QAChB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAC7B,IAAI,EAAE,CAAC,KAAK,EAAE,cAAc,KAAK,IAAI,CAAC;QACtC,MAAM,EAAE,WAAW,CAAC;QACpB,eAAe,CAAC,EAAE,GAAG,CAAC;KACvB,KAAK,OAAO,CAAC,cAAc,CAAC,CAAC;IAC9B,QAAQ,EAAE;QACR,MAAM,EAAE,GAAG,CAAC;QACZ,KAAK,EAAE,MAAM,CAAC;QACd,YAAY,EAAE,MAAM,CAAC;QACrB,KAAK,EAAE,GAAG,EAAE,CAAC;QACb,QAAQ,EAAE,GAAG,EAAE,CAAC;QAChB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAC7B,IAAI,EAAE,CAAC,KAAK,EAAE,cAAc,KAAK,IAAI,CAAC;QACtC,MAAM,EAAE,WAAW,CAAC;QACpB,eAAe,CAAC,EAAE,GAAG,CAAC;KACvB,CAAC;IACF,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB;;;iBAGa;IACb,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IACtB,MAAM,EAAE,mBAAmB,CAAC;CAC7B,GAAG,OAAO,CAAC,cAAc,CAAC,CAiT1B"}

package/dist/observability/traces.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"traces.js","sourceRoot":"","sources":["../../src/observability/traces.ts"],"names":[],"mappings":"AAEA,OAAO,EAAkB,YAAY,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAE5E,OAAO,EAAE,4BAA4B,EAAE,MAAM,YAAY,CAAC;AAE1D,SAAS,MAAM;IACb,OAAO,QAAQ,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;AACxE,CAAC;AAED;;;;;;2BAM2B;AAC3B,MAAM,uBAAuB,GAC3B,uGAAuG,CAAC;AAE1G;;;wEAGwE;AACxE,MAAM,UAAU,qBAAqB,CAAC,KAAc;IAClD,OAAO,UAAU,CAAC,KAAK,EAAE,IAAI,OAAO,EAAU,CAAC,CAAC;AAClD,CAAC;AAED,SAAS,UAAU,CAAC,KAAc,EAAE,IAAqB;IACvD,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IAC9D,IAAI,IAAI,CAAC,GAAG,CAAC,KAAe,CAAC;QAAE,OAAO,YAAY,CAAC;IACnD,IAAI,CAAC,GAAG,CAAC,KAAe,CAAC,CAAC;IAC1B,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;IAC/C,CAAC;IACD,MAAM,GAAG,GAA4B,EAAE,CAAC;IACxC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAgC,CAAC,EAAE,CAAC;QACtE,IAAI,uBAAuB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YACpC,GAAG,CAAC,CAAC,CAAC,GAAG,YAAY,CAAC;QACxB,CAAC;aAAM,CAAC;YACN,GAAG,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,sBAAsB;IAC1C,IAAI,CAAC;QACH,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,MAAM,CAAC,sBAAsB,CAAC,CAAC;QAC5D,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,sBAAsB,CAAC,CAAC;QACxD,IAAI,MAAM,EAAE,CAAC;YACX,OAAO;gBACL,GAAG,4BAA4B;gBAC/B,GAAG,MAAM;aACa,CAAC;QAC3B,CAAC;IACH,CAAC;IAAC,MAAM,CAAC,CAAA,CAAC;IACV,OAAO,4BAA4B,CAAC;AACtC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,IA+BzC;IACC,MAAM,EAAE,YAAY,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;IACzE,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC5B,MAAM,YAAY,GAAG,MAAM,EAAE,CAAC;IAE9B,0EAA0E;IAC1E,2EAA2E;IAC3E,2EAA2E;IAC3E,yEAAyE;IACzE,oDAAoD;IACpD,MAAM,kBAAkB,GAAG,cAAc,CAAC,WAAW,EAAE;QACrD,cAAc,EAAE,KAAK;QACrB,iBAAiB,EAAE,QAAQ,IAAI,SAAS;QACxC,eAAe,EAAE,MAAM,IAAI,SAAS;QACpC,aAAa,EAAE,QAAQ,CAAC,KAAK;KAC9B,CAAC,CAAC;IAEH,MAAM,KAAK,GAAgB,EAAE,CAAC;IAC9B,IAAI,qBAAqB,GAAG,CAAC,CAAC;IAC9B,oEAAoE;IACpE,MAAM,YAAY,GAAG,IAAI,GAAG,EAUzB,CAAC;IACJ,0EAA0E;IAC1E,2EAA2E;IAC3E,8EAA8E;IAC9E,+EAA+E;IAC/E,wEAAwE;IACxE,MAAM,kBAAkB,GAAG,IAAI,GAAG,EAAoB,CAAC;IAEvD,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,eAAe,GAAG,CAAC,CAAC;IACxB,IAAI,WAAW,GAAG,CAAC,CAAC;IAEpB,wEAAwE;IACxE,gDAAgD;IAChD,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAa,CAAC;IAE/C,MAAM,gBAAgB,GAAG,CAAC,KAAqB,EAAQ,EAAE;QACvD,IAAI,CAAC;YACH,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;gBAChC,MAAM,OAAO,GAAG,qBAAqB,EAAE,CAAC;gBACxC,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC;gBACrB,iEAAiE;gBACjE,oEAAoE;gBACpE,qEAAqE;gBACrE,uEAAuE;gBACvE,MAAM,KAAK,GAYP;oBACF,MAAM,EAAE,GAAG;oBACX,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE;oBACnB,QAAQ,EAAE,KAAK,CAAC,IAAI;oBACpB,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,QAAQ,EAAE,IAAI;iBACf,CAAC;gBACF,YAAY,CAAC,GAAG,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;gBACjC,KAAK,cAAc,CAAC,WAAW,EAAE;oBAC/B,WAAW,EAAE,KAAK,CAAC,IAAI;iBACxB,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE;oBACf,IAAI,CAAC,IAAI;wBAAE,OAAO;oBAClB,sEAAsE;oBACtE,+DAA+D;oBAC/D,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;wBACpB,YAAY,CAAC,IAAI,EAAE;4BACjB,MAAM,EAAE,KAAK,CAAC,SAAS,CAAC,MAAM;4BAC9B,YAAY,EAAE,KAAK,CAAC,SAAS,CAAC,YAAY;yBAC3C,CAAC,CAAC;oBACL,CAAC;yBAAM,CAAC;wBACN,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC;wBACtB,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;oBAC9B,CAAC;gBACH,CAAC,CAAC,CAAC;gBACH,MAAM,KAAK,GAAG,kBAAkB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBACjD,IAAI,KAAK;oBAAE,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;;oBAC1B,kBAAkB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;YACrD,CAAC;iBAAM,IAAI,KAAK,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;gBACtC,MAAM,KAAK,GAAG,kBAAkB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBACjD,MAAM,OAAO,GAAG,KAAK,EAAE,KAAK,EAAE,CAAC;gBAC/B,MAAM,OAAO,GACX,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;gBAChE,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;oBAC1B,YAAY,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;oBAC7B,IAAI,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;wBAC7B,kBAAkB,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBAC1C,CAAC;gBACD,aAAa,EAAE,CAAC;gBAEhB,MAAM,OAAO,GACX,OAAO,KAAK,CAAC,MAAM,KAAK,QAAQ;oBAChC,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC;wBAC/B,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC,CAAC;gBAC/C,IAAI,OAAO;oBAAE,WAAW,EAAE,CAAC;;oBACtB,eAAe,EAAE,CAAC;gBAEvB,uEAAuE;gBACvE,oEAAoE;gBACpE,MAAM,aAAa,GAAG;oBACpB,MAAM,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAwB;oBAC9D,YAAY,EAAE,OAAO,CAAC,CAAC,CAAE,KAAK,CAAC,MAAiB,CAAC,CAAC,CAAC,IAAI;iBACxD,CAAC;gBACF,IAAI,OAAO,EAAE,QAAQ,EAAE,CAAC;oBACtB,iBAAiB,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;oBAC3C,YAAY,CAAC,OAAO,CAAC,QAAQ,EAAE;wBAC7B,MAAM,EAAE,aAAa,CAAC,MAAM;wBAC5B,YAAY,EAAE,aAAa,CAAC,YAAY;wBACxC,UAAU,EAAE,EAAE,WAAW,EAAE,KAAK,CAAC,IAAI,EAAE;qBACxC,CAAC,CAAC;gBACL,CAAC;qBAAM,IAAI,OAAO,EAAE,CAAC;oBACnB,OAAO,CAAC,SAAS,GAAG,aAAa,CAAC;gBACpC,CAAC;gBAED,MAAM,IAAI,GAAc;oBACtB,EAAE,EAAE,OAAO,EAAE,MAAM,IAAI,MAAM,EAAE;oBAC/B,KAAK;oBACL,QAAQ;oBACR,MAAM;oBACN,YAAY;oBACZ,QAAQ,EAAE,WAAW;oBACrB,IAAI,EAAE,KAAK,CAAC,IAAI;oBAChB,WAAW,EAAE,CAAC;oBACd,YAAY,EAAE,CAAC;oBACf,eAAe,EAAE,CAAC;oBAClB,gBAAgB,EAAE,CAAC;oBACnB,aAAa,EAAE,CAAC;oBAChB,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;oBACtD,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS;oBACrC,YAAY,EAAE,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI;oBAC3C,QAAQ,EACN,MAAM,CAAC,eAAe,IAAI,OAAO;wBAC/B,CAAC,CAAC,yDAAyD;4BACzD,sDAAsD;4BACtD,uDAAuD;4BACvD,oCAAoC;4BACpC;gCACE,KAAK,EAAE,qBAAqB,CAAC,OAAO,CAAC,KAAK,CAGzC;6BACF;wBACH,CAAC,CAAC,IAAI;oBACV,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;iBACtB,CAAC;gBACF,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACnB,CAAC;QACH,CAAC;QAAC,MAAM,CAAC,CAAA,CAAC;QAEV,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACvB,CAAC,CAAC;IAEF,IAAI,KAAiC,CAAC;IACtC,IAAI,SAAS,GAAwB,SAAS,CAAC;IAC/C,IAAI,YAAY,GAAkB,IAAI,CAAC;IACvC,IAAI,CAAC;QACH,KAAK,GAAG,MAAM,YAAY,CAAC,EAAE,GAAG,QAAQ,EAAE,IAAI,EAAE,gBAAgB,EAAE,CAAC,CAAC;IACtE,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAClB,SAAS,GAAG,OAAO,CAAC;QACpB,YAAY,GAAG,GAAG,EAAE,OAAO,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC;QAC3C,MAAM,GAAG,CAAC;IACZ,CAAC;YAAS,CAAC;QACT,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC1B,MAAM,eAAe,GAAG,MAAM,GAAG,QAAQ,CAAC;QAE1C,IAAI,aAAa,GAAG,CAAC,CAAC;QACtB,IAAI,CAAC;YACH,MAAM,EAAE,aAAa,EAAE,GAAG,MAAM,MAAM,CAAC,mBAAmB,CAAC,CAAC;YAC5D,IAAI,KAAK,EAAE,CAAC;gBACV,aAAa,GAAG,aAAa,CAC3B,KAAK,CAAC,WAAW,EACjB,KAAK,CAAC,YAAY,EAClB,KAAK,CAAC,KAAK,EACX,KAAK,CAAC,eAAe,EACrB,KAAK,CAAC,gBAAgB,CACvB,CAAC;YACJ,CAAC;QACH,CAAC;QAAC,MAAM,CAAC,CAAA,CAAC;QAEV,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,IAAI,KAAK,EAAE,CAAC;YACV,YAAY,GAAG,CAAC,CAAC;YACjB,MAAM,OAAO,GAAc;gBACzB,EAAE,EAAE,MAAM,EAAE;gBACZ,KAAK;gBACL,QAAQ;gBACR,MAAM;gBACN,YAAY;gBACZ,QAAQ,EAAE,UAAU;gBACpB,IAAI,EAAE,KAAK,CAAC,KAAK;gBACjB,WAAW,EAAE,KAAK,CAAC,WAAW;gBAC9B,YAAY,EAAE,KAAK,CAAC,YAAY;gBAChC,eAAe,EAAE,KAAK,CAAC,eAAe;gBACtC,gBAAgB,EAAE,KAAK,CAAC,gBAAgB;gBACxC,aAAa;gBACb,UAAU,EAAE,eAAe;gBAC3B,MAAM,EAAE,SAAS;gBACjB,YAAY;gBACZ,QAAQ,EAAE,IAAI;gBACd,SAAS,EAAE,QAAQ;aACpB,CAAC;YACF,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACtB,CAAC;QAED,MAAM,UAAU,GAAc;YAC5B,EAAE,EAAE,YAAY;YAChB,KAAK;YACL,QAAQ;YACR,MAAM;YACN,YAAY,EAAE,IAAI;YAClB,QAAQ,EAAE,WAAW;YACrB,IAAI,EAAE,WAAW;YACjB,WAAW,EAAE,KAAK,EAAE,WAAW,IAAI,CAAC;YACpC,YAAY,EAAE,KAAK,EAAE,YAAY,IAAI,CAAC;YACtC,eAAe,EAAE,KAAK,EAAE,eAAe,IAAI,CAAC;YAC5C,gBAAgB,EAAE,KAAK,EAAE,gBAAgB,IAAI,CAAC;YAC9C,aAAa;YACb,UAAU,EAAE,eAAe;YAC3B,MAAM,EAAE,SAAS;YACjB,YAAY;YACZ,QAAQ,EAAE,IAAI;YACd,SAAS,EAAE,QAAQ;SACpB,CAAC;QACF,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAEvB,MAAM,OAAO,GAAiB;YAC5B,KAAK;YACL,QAAQ;YACR,MAAM;YACN,UAAU,EAAE,KAAK,CAAC,MAAM;YACxB,QAAQ,EAAE,YAAY;YACtB,SAAS,EAAE,aAAa;YACxB,eAAe;YACf,WAAW;YACX,eAAe;YACf,kBAAkB,EAAE,aAAa;YACjC,gBAAgB,EAAE,KAAK,EAAE,WAAW,IAAI,CAAC;YACzC,iBAAiB,EAAE,KAAK,EAAE,YAAY,IAAI,CAAC;YAC3C,KAAK,EAAE,KAAK,EAAE,KAAK,IAAI,QAAQ,CAAC,KAAK;YACrC,SAAS,EAAE,QAAQ;SACpB,CAAC;QAEF,cAAc,CAAC,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAE9D,uEAAuE;QACvE,uEAAuE;QACvE,6EAA6E;QAC7E,0EAA0E;QAC1E,IAAI,CAAC;YACH,IAAI,KAAK,EAAE,CAAC;gBACV,YAAY,CAAC,MAAM,cAAc,CAAC,UAAU,EAAE,EAAE,CAAC,EAAE;oBACjD,MAAM,EAAE,SAAS;oBACjB,YAAY;oBACZ,UAAU,EAAE;wBACV,WAAW,EAAE,KAAK,CAAC,KAAK;wBACxB,kBAAkB,EAAE,KAAK,CAAC,WAAW;wBACrC,mBAAmB,EAAE,KAAK,CAAC,YAAY;wBACvC,uBAAuB,EAAE,KAAK,CAAC,eAAe;wBAC9C,wBAAwB,EAAE,KAAK,CAAC,gBAAgB;wBAChD,qBAAqB,EAAE,aAAa;qBACrC;iBACF,CAAC,CAAC;YACL,CAAC;YACD,KAAK,MAAM,QAAQ,IAAI,iBAAiB,EAAE,CAAC;gBACzC,YAAY,CAAC,QAAQ,EAAE;oBACrB,MAAM,EAAE,OAAO;oBACf,YAAY,EAAE,mCAAmC;iBAClD,CAAC,CAAC;YACL,CAAC;YACD,iBAAiB,CAAC,KAAK,EAAE,CAAC;YAC1B,YAAY,CAAC,MAAM,kBAAkB,EAAE;gBACrC,MAAM,EAAE,SAAS;gBACjB,YAAY;gBACZ,UAAU,EAAE;oBACV,kBAAkB,EAAE,aAAa;oBACjC,wBAAwB,EAAE,eAAe;oBACzC,oBAAoB,EAAE,WAAW;oBACjC,mBAAmB,EAAE,eAAe;oBACpC,oBAAoB,EAAE,KAAK,EAAE,WAAW,IAAI,CAAC;oBAC7C,qBAAqB,EAAE,KAAK,EAAE,YAAY,IAAI,CAAC;oBAC/C,uBAAuB,EAAE,aAAa;iBACvC;aACF,CAAC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACP,wCAAwC;QAC1C,CAAC;IACH,CAAC;IAED,OAAO,KAAM,CAAC;AAChB,CAAC;AAED,KAAK,UAAU,cAAc,CAC3B,KAAkB,EAClB,OAAqB,EACrB,KAAa,EACb,MAA2B;IAE3B,MAAM,EAAE,eAAe,EAAE,kBAAkB,EAAE,GAAG,MAAM,MAAM,CAAC,YAAY,CAAC,CAAC;IAC3E,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IACxE,MAAM,kBAAkB,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IAElD,qDAAqD;IACrD,IAAI,CAAC;QACH,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,MAAM,CAAC,YAAY,CAAC,CAAC;QACnD,MAAM,WAAW,CAAC,KAAK,EAAE,EAAE,UAAU,EAAE,MAAM,CAAC,cAAc,EAAE,CAAC,CAAC;IAClE,CAAC;IAAC,MAAM,CAAC,CAAA,CAAC;AACZ,CAAC","sourcesContent":["import type { AgentLoopUsage } from \"../agent/production-agent.js\";\nimport type { AgentChatEvent } from \"../agent/types.js\";\nimport { type AgentSpan, endAgentSpan, startAgentSpan } from \"./tracing.js\";\nimport type { TraceSpan, TraceSummary, ObservabilityConfig } from \"./types.js\";\nimport { DEFAULT_OBSERVABILITY_CONFIG } from \"./types.js\";\n\nfunction spanId(): string {\n return `span-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;\n}\n\n/** Keys whose values are stripped from persisted tool inputs when\n * `captureToolArgs` is enabled. Matched case-insensitively and tolerant\n * of `_` / `-` separators. M14 in the MCP/A2A audit: tool calls\n * routinely receive credentials verbatim (db-exec INSERTs, fetchTool\n * Authorization headers, ad-hoc bearer tokens) — keeping those values\n * out of agent_trace_spans.metadata avoids long-term storage of\n * short-lived secrets. /\nconst SENSITIVE_FIELD_PATTERN =\n /^(authorization\|cookie\|api[_-]?key\|password\|secret\|token\|access[_-]?token\|refresh[_-]?token\|bearer)$/i;\n\n/* Recursively walk a structured value and replace sensitive field\n * values with the literal string \"[REDACTED]\". Pure (returns a copy);\n * the original input is never mutated. Cycles are tolerated via a\n * small WeakSet seen-tracker that returns \"[Circular]\" for repeats. /\nexport function redactSensitiveFields(value: unknown): unknown {\n return redactWalk(value, new WeakSet<object>());\n}\n\nfunction redactWalk(value: unknown, seen: WeakSet<object>): unknown {\n if (value === null \|\| typeof value !== \"object\") return value;\n if (seen.has(value as object)) return \"[Circular]\";\n seen.add(value as object);\n if (Array.isArray(value)) {\n return value.map((v) => redactWalk(v, seen));\n }\n const out: Record<string, unknown> = {};\n for (const [k, v] of Object.entries(value as Record<string, unknown>)) {\n if (SENSITIVE_FIELD_PATTERN.test(k)) {\n out[k] = \"[REDACTED]\";\n } else {\n out[k] = redactWalk(v, seen);\n }\n }\n return out;\n}\n\nexport async function getObservabilityConfig(): Promise<ObservabilityConfig> {\n try {\n const { getSetting } = await import(\"../settings/store.js\");\n const stored = await getSetting(\"observability-config\");\n if (stored) {\n return {\n ...DEFAULT_OBSERVABILITY_CONFIG,\n ...stored,\n } as ObservabilityConfig;\n }\n } catch {}\n return DEFAULT_OBSERVABILITY_CONFIG;\n}\n\nexport async function instrumentAgentLoop(opts: {\n runAgentLoop: (loopOpts: {\n engine: any;\n model: string;\n systemPrompt: string;\n tools: any[];\n messages: any[];\n actions: Record<string, any>;\n send: (event: AgentChatEvent) => void;\n signal: AbortSignal;\n providerOptions?: any;\n }) => Promise<AgentLoopUsage>;\n loopOpts: {\n engine: any;\n model: string;\n systemPrompt: string;\n tools: any[];\n messages: any[];\n actions: Record<string, any>;\n send: (event: AgentChatEvent) => void;\n signal: AbortSignal;\n providerOptions?: any;\n };\n runId: string;\n threadId: string \| null;\n /* Owner of this run; persisted on every span + summary so dashboard\n * reads can filter to a single user. Null for unauthenticated callers\n * (background tasks, etc.) — those rows aren't returned by per-user\n * reads. */\n userId: string \| null;\n config: ObservabilityConfig;\n}): Promise<AgentLoopUsage> {\n const { runAgentLoop, loopOpts, runId, threadId, userId, config } = opts;\n const runStart = Date.now();\n const parentSpanId = spanId();\n\n // Optional OpenTelemetry root span for this run. No-ops unless a host has\n // installed `@opentelemetry/api` and registered a provider. The promise is\n // resolved before the loop runs so child tool/model spans can parent under\n // it conceptually (we keep them flat in the same tracer, which is enough\n // for the dashboards an embedding app would build).\n const otelRunSpanPromise = startAgentSpan(\"agent.run\", {\n \"agent.run_id\": runId,\n \"agent.thread_id\": threadId ?? undefined,\n \"agent.user_id\": userId ?? undefined,\n \"agent.model\": loopOpts.model,\n });\n\n const spans: TraceSpan[] = [];\n let toolInvocationCounter = 0;\n // Keyed by counter to handle concurrent calls to the same tool name\n const pendingTools = new Map<\n number,\n {\n spanId: string;\n startMs: number;\n toolName: string;\n input: Record<string, string>;\n otelSpan: AgentSpan \| null;\n endResult?: { status: \"success\" \| \"error\"; errorMessage: string \| null };\n }\n >();\n // Secondary index: tool name → FIFO queue of pending invocation counters.\n // tool_start/tool_done events carry only the tool name (no call id), so to\n // pair starts and dones correctly when the agent runs concurrent calls to the\n // same tool name (read-only / parallelSafe batches via Promise.all), we keep a\n // queue per name and match each done to the OLDEST still-pending start.\n const toolNameToCounters = new Map<string, number[]>();\n\n let toolCallCount = 0;\n let successfulTools = 0;\n let failedTools = 0;\n\n // Track in-flight OTel tool spans so they're all ended even if the loop\n // throws before a matching `tool_done` arrives.\n const openOtelToolSpans = new Set<AgentSpan>();\n\n const instrumentedSend = (event: AgentChatEvent): void => {\n try {\n if (event.type === \"tool_start\") {\n const counter = toolInvocationCounter++;\n const sid = spanId();\n // Start the OTel tool span synchronously-ish: kick off the async\n // resolution and stash the span once it lands. Tool spans are short\n // and the api tracer is synchronous in practice, but we tolerate the\n // microtask gap by recording the span on the pending entry when ready.\n const entry: {\n spanId: string;\n startMs: number;\n toolName: string;\n input: Record<string, string>;\n otelSpan: AgentSpan \| null;\n // Set by the done handler if it fires before the span promise\n // resolves, so the resolved span is ended with the correct status.\n endResult?: {\n status: \"success\" \| \"error\";\n errorMessage: string \| null;\n };\n } = {\n spanId: sid,\n startMs: Date.now(),\n toolName: event.tool,\n input: event.input,\n otelSpan: null,\n };\n pendingTools.set(counter, entry);\n void startAgentSpan(\"tool.call\", {\n \"tool.name\": event.tool,\n }).then((span) => {\n if (!span) return;\n // If `tool_done` already ran for this call, end the span now with the\n // status it recorded; otherwise stash it for the done handler.\n if (entry.endResult) {\n endAgentSpan(span, {\n status: entry.endResult.status,\n errorMessage: entry.endResult.errorMessage,\n });\n } else {\n entry.otelSpan = span;\n openOtelToolSpans.add(span);\n }\n });\n const queue = toolNameToCounters.get(event.tool);\n if (queue) queue.push(counter);\n else toolNameToCounters.set(event.tool, [counter]);\n } else if (event.type === \"tool_done\") {\n const queue = toolNameToCounters.get(event.tool);\n const counter = queue?.shift();\n const pending =\n counter !== undefined ? pendingTools.get(counter) : undefined;\n if (counter !== undefined) {\n pendingTools.delete(counter);\n if (queue && queue.length === 0)\n toolNameToCounters.delete(event.tool);\n }\n toolCallCount++;\n\n const isError =\n typeof event.result === \"string\" &&\n (event.result.startsWith(\"Error\") \|\|\n event.result.startsWith(\"Error running \"));\n if (isError) failedTools++;\n else successfulTools++;\n\n // Finalize the OTel tool span. If the span promise hasn't resolved yet\n // we record the result on the entry so its `.then` handler ends it.\n const otelEndResult = {\n status: (isError ? \"error\" : \"success\") as \"success\" \| \"error\",\n errorMessage: isError ? (event.result as string) : null,\n };\n if (pending?.otelSpan) {\n openOtelToolSpans.delete(pending.otelSpan);\n endAgentSpan(pending.otelSpan, {\n status: otelEndResult.status,\n errorMessage: otelEndResult.errorMessage,\n attributes: { \"tool.name\": event.tool },\n });\n } else if (pending) {\n pending.endResult = otelEndResult;\n }\n\n const span: TraceSpan = {\n id: pending?.spanId ?? spanId(),\n runId,\n threadId,\n userId,\n parentSpanId,\n spanType: \"tool_call\",\n name: event.tool,\n inputTokens: 0,\n outputTokens: 0,\n cacheReadTokens: 0,\n cacheWriteTokens: 0,\n costCentsX100: 0,\n durationMs: pending ? Date.now() - pending.startMs : 0,\n status: isError ? \"error\" : \"success\",\n errorMessage: isError ? event.result : null,\n metadata:\n config.captureToolArgs && pending\n ? // Strip Authorization/api-key/token-shaped values before\n // persisting (M14 in the MCP/A2A audit). Tool-runtime\n // execution still sees the unredacted input — only the\n // long-lived span row is sanitized.\n {\n input: redactSensitiveFields(pending.input) as Record<\n string,\n string\n >,\n }\n : null,\n createdAt: Date.now(),\n };\n spans.push(span);\n }\n } catch {}\n\n loopOpts.send(event);\n };\n\n let usage: AgentLoopUsage \| undefined;\n let runStatus: \"success\" \| \"error\" = \"success\";\n let errorMessage: string \| null = null;\n try {\n usage = await runAgentLoop({ ...loopOpts, send: instrumentedSend });\n } catch (err: any) {\n runStatus = \"error\";\n errorMessage = err?.message ?? String(err);\n throw err;\n } finally {\n const runEnd = Date.now();\n const totalDurationMs = runEnd - runStart;\n\n let costCentsX100 = 0;\n try {\n const { calculateCost } = await import(\"../usage/store.js\");\n if (usage) {\n costCentsX100 = calculateCost(\n usage.inputTokens,\n usage.outputTokens,\n usage.model,\n usage.cacheReadTokens,\n usage.cacheWriteTokens,\n );\n }\n } catch {}\n\n let llmCallCount = 0;\n if (usage) {\n llmCallCount = 1;\n const llmSpan: TraceSpan = {\n id: spanId(),\n runId,\n threadId,\n userId,\n parentSpanId,\n spanType: \"llm_call\",\n name: usage.model,\n inputTokens: usage.inputTokens,\n outputTokens: usage.outputTokens,\n cacheReadTokens: usage.cacheReadTokens,\n cacheWriteTokens: usage.cacheWriteTokens,\n costCentsX100,\n durationMs: totalDurationMs,\n status: runStatus,\n errorMessage,\n metadata: null,\n createdAt: runStart,\n };\n spans.push(llmSpan);\n }\n\n const parentSpan: TraceSpan = {\n id: parentSpanId,\n runId,\n threadId,\n userId,\n parentSpanId: null,\n spanType: \"agent_run\",\n name: \"agent_run\",\n inputTokens: usage?.inputTokens ?? 0,\n outputTokens: usage?.outputTokens ?? 0,\n cacheReadTokens: usage?.cacheReadTokens ?? 0,\n cacheWriteTokens: usage?.cacheWriteTokens ?? 0,\n costCentsX100,\n durationMs: totalDurationMs,\n status: runStatus,\n errorMessage,\n metadata: null,\n createdAt: runStart,\n };\n spans.push(parentSpan);\n\n const summary: TraceSummary = {\n runId,\n threadId,\n userId,\n totalSpans: spans.length,\n llmCalls: llmCallCount,\n toolCalls: toolCallCount,\n successfulTools,\n failedTools,\n totalDurationMs,\n totalCostCentsX100: costCentsX100,\n totalInputTokens: usage?.inputTokens ?? 0,\n totalOutputTokens: usage?.outputTokens ?? 0,\n model: usage?.model ?? loopOpts.model,\n createdAt: runStart,\n };\n\n writeTraceData(spans, summary, runId, config).catch(() => {});\n\n // OpenTelemetry export (no-op unless a provider is registered). Emit a\n // self-contained `llm.call` span carrying model + token usage, end any\n // tool spans still open (loop threw mid-tool), and end the run span. Awaited\n // so the spans are emitted before the function returns; cheap when no-op.\n try {\n if (usage) {\n endAgentSpan(await startAgentSpan(\"llm.call\", {}), {\n status: runStatus,\n errorMessage,\n attributes: {\n \"llm.model\": usage.model,\n \"llm.input_tokens\": usage.inputTokens,\n \"llm.output_tokens\": usage.outputTokens,\n \"llm.cache_read_tokens\": usage.cacheReadTokens,\n \"llm.cache_write_tokens\": usage.cacheWriteTokens,\n \"llm.cost_cents_x100\": costCentsX100,\n },\n });\n }\n for (const toolSpan of openOtelToolSpans) {\n endAgentSpan(toolSpan, {\n status: \"error\",\n errorMessage: \"Agent run ended before tool_done.\",\n });\n }\n openOtelToolSpans.clear();\n endAgentSpan(await otelRunSpanPromise, {\n status: runStatus,\n errorMessage,\n attributes: {\n \"agent.tool_calls\": toolCallCount,\n \"agent.successful_tools\": successfulTools,\n \"agent.failed_tools\": failedTools,\n \"agent.duration_ms\": totalDurationMs,\n \"agent.input_tokens\": usage?.inputTokens ?? 0,\n \"agent.output_tokens\": usage?.outputTokens ?? 0,\n \"agent.cost_cents_x100\": costCentsX100,\n },\n });\n } catch {\n // OTel export must never break the run.\n }\n }\n\n return usage!;\n}\n\nasync function writeTraceData(\n spans: TraceSpan[],\n summary: TraceSummary,\n runId: string,\n config: ObservabilityConfig,\n): Promise<void> {\n const { insertTraceSpan, upsertTraceSummary } = await import(\"./store.js\");\n await Promise.all(spans.map((s) => insertTraceSpan(s).catch(() => {})));\n await upsertTraceSummary(summary).catch(() => {});\n\n // Fire automated evals after trace data is persisted\n try {\n const { evaluateRun } = await import(\"./evals.js\");\n await evaluateRun(runId, { sampleRate: config.evalSampleRate });\n } catch {}\n}\n"]}
1	+ {"version":3,"file":"traces.js","sourceRoot":"","sources":["../../src/observability/traces.ts"],"names":[],"mappings":"AAEA,OAAO,EAAkB,YAAY,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAE5E,OAAO,EAAE,4BAA4B,EAAE,MAAM,YAAY,CAAC;AAE1D,SAAS,MAAM;IACb,OAAO,QAAQ,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;AACxE,CAAC;AAED;;;;;;2BAM2B;AAC3B,MAAM,uBAAuB,GAC3B,uGAAuG,CAAC;AAE1G;;;wEAGwE;AACxE,MAAM,UAAU,qBAAqB,CAAC,KAAc;IAClD,OAAO,UAAU,CAAC,KAAK,EAAE,IAAI,OAAO,EAAU,CAAC,CAAC;AAClD,CAAC;AAED,SAAS,UAAU,CAAC,KAAc,EAAE,IAAqB;IACvD,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IAC9D,IAAI,IAAI,CAAC,GAAG,CAAC,KAAe,CAAC;QAAE,OAAO,YAAY,CAAC;IACnD,IAAI,CAAC,GAAG,CAAC,KAAe,CAAC,CAAC;IAC1B,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC;IAC/C,CAAC;IACD,MAAM,GAAG,GAA4B,EAAE,CAAC;IACxC,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAgC,CAAC,EAAE,CAAC;QACtE,IAAI,uBAAuB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YACpC,GAAG,CAAC,CAAC,CAAC,GAAG,YAAY,CAAC;QACxB,CAAC;aAAM,CAAC;YACN,GAAG,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,sBAAsB;IAC1C,IAAI,CAAC;QACH,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,MAAM,CAAC,sBAAsB,CAAC,CAAC;QAC5D,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,sBAAsB,CAAC,CAAC;QACxD,IAAI,MAAM,EAAE,CAAC;YACX,OAAO;gBACL,GAAG,4BAA4B;gBAC/B,GAAG,MAAM;aACa,CAAC;QAC3B,CAAC;IACH,CAAC;IAAC,MAAM,CAAC,CAAA,CAAC;IACV,OAAO,4BAA4B,CAAC;AACtC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,IA+BzC;IACC,MAAM,EAAE,YAAY,EAAE,QAAQ,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC;IACzE,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC5B,MAAM,YAAY,GAAG,MAAM,EAAE,CAAC;IAE9B,0EAA0E;IAC1E,2EAA2E;IAC3E,2EAA2E;IAC3E,yEAAyE;IACzE,oDAAoD;IACpD,MAAM,kBAAkB,GAAG,cAAc,CAAC,WAAW,EAAE;QACrD,cAAc,EAAE,KAAK;QACrB,iBAAiB,EAAE,QAAQ,IAAI,SAAS;QACxC,eAAe,EAAE,MAAM,IAAI,SAAS;QACpC,aAAa,EAAE,QAAQ,CAAC,KAAK;KAC9B,CAAC,CAAC;IAEH,MAAM,KAAK,GAAgB,EAAE,CAAC;IAC9B,IAAI,qBAAqB,GAAG,CAAC,CAAC;IAC9B,oEAAoE;IACpE,MAAM,YAAY,GAAG,IAAI,GAAG,EAUzB,CAAC;IACJ,0EAA0E;IAC1E,2EAA2E;IAC3E,8EAA8E;IAC9E,+EAA+E;IAC/E,wEAAwE;IACxE,MAAM,kBAAkB,GAAG,IAAI,GAAG,EAAoB,CAAC;IAEvD,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,eAAe,GAAG,CAAC,CAAC;IACxB,IAAI,WAAW,GAAG,CAAC,CAAC;IAEpB,wEAAwE;IACxE,gDAAgD;IAChD,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAa,CAAC;IAE/C,MAAM,gBAAgB,GAAG,CAAC,KAAqB,EAAQ,EAAE;QACvD,IAAI,CAAC;YACH,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;gBAChC,MAAM,OAAO,GAAG,qBAAqB,EAAE,CAAC;gBACxC,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC;gBACrB,iEAAiE;gBACjE,oEAAoE;gBACpE,qEAAqE;gBACrE,uEAAuE;gBACvE,MAAM,KAAK,GAYP;oBACF,MAAM,EAAE,GAAG;oBACX,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE;oBACnB,QAAQ,EAAE,KAAK,CAAC,IAAI;oBACpB,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,QAAQ,EAAE,IAAI;iBACf,CAAC;gBACF,YAAY,CAAC,GAAG,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;gBACjC,KAAK,cAAc,CAAC,WAAW,EAAE;oBAC/B,WAAW,EAAE,KAAK,CAAC,IAAI;iBACxB,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE;oBACf,IAAI,CAAC,IAAI;wBAAE,OAAO;oBAClB,sEAAsE;oBACtE,+DAA+D;oBAC/D,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;wBACpB,YAAY,CAAC,IAAI,EAAE;4BACjB,MAAM,EAAE,KAAK,CAAC,SAAS,CAAC,MAAM;4BAC9B,YAAY,EAAE,KAAK,CAAC,SAAS,CAAC,YAAY;yBAC3C,CAAC,CAAC;oBACL,CAAC;yBAAM,CAAC;wBACN,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAC;wBACtB,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;oBAC9B,CAAC;gBACH,CAAC,CAAC,CAAC;gBACH,MAAM,KAAK,GAAG,kBAAkB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBACjD,IAAI,KAAK;oBAAE,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;;oBAC1B,kBAAkB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;YACrD,CAAC;iBAAM,IAAI,KAAK,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;gBACtC,MAAM,KAAK,GAAG,kBAAkB,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBACjD,MAAM,OAAO,GAAG,KAAK,EAAE,KAAK,EAAE,CAAC;gBAC/B,MAAM,OAAO,GACX,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;gBAChE,IAAI,OAAO,KAAK,SAAS,EAAE,CAAC;oBAC1B,YAAY,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;oBAC7B,IAAI,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;wBAC7B,kBAAkB,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBAC1C,CAAC;gBACD,aAAa,EAAE,CAAC;gBAEhB,MAAM,OAAO,GACX,OAAO,KAAK,CAAC,MAAM,KAAK,QAAQ;oBAChC,CAAC,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC;wBAC/B,KAAK,CAAC,MAAM,CAAC,UAAU,CAAC,gBAAgB,CAAC,CAAC,CAAC;gBAC/C,IAAI,OAAO;oBAAE,WAAW,EAAE,CAAC;;oBACtB,eAAe,EAAE,CAAC;gBAEvB,uEAAuE;gBACvE,oEAAoE;gBACpE,MAAM,aAAa,GAAG;oBACpB,MAAM,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAwB;oBAC9D,YAAY,EAAE,OAAO,CAAC,CAAC,CAAE,KAAK,CAAC,MAAiB,CAAC,CAAC,CAAC,IAAI;iBACxD,CAAC;gBACF,IAAI,OAAO,EAAE,QAAQ,EAAE,CAAC;oBACtB,iBAAiB,CAAC,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;oBAC3C,YAAY,CAAC,OAAO,CAAC,QAAQ,EAAE;wBAC7B,MAAM,EAAE,aAAa,CAAC,MAAM;wBAC5B,YAAY,EAAE,aAAa,CAAC,YAAY;wBACxC,UAAU,EAAE,EAAE,WAAW,EAAE,KAAK,CAAC,IAAI,EAAE;qBACxC,CAAC,CAAC;gBACL,CAAC;qBAAM,IAAI,OAAO,EAAE,CAAC;oBACnB,OAAO,CAAC,SAAS,GAAG,aAAa,CAAC;gBACpC,CAAC;gBAED,MAAM,IAAI,GAAc;oBACtB,EAAE,EAAE,OAAO,EAAE,MAAM,IAAI,MAAM,EAAE;oBAC/B,KAAK;oBACL,QAAQ;oBACR,MAAM;oBACN,YAAY;oBACZ,QAAQ,EAAE,WAAW;oBACrB,IAAI,EAAE,KAAK,CAAC,IAAI;oBAChB,WAAW,EAAE,CAAC;oBACd,YAAY,EAAE,CAAC;oBACf,eAAe,EAAE,CAAC;oBAClB,gBAAgB,EAAE,CAAC;oBACnB,aAAa,EAAE,CAAC;oBAChB,UAAU,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;oBACtD,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS;oBACrC,YAAY,EAAE,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI;oBAC3C,QAAQ,EACN,MAAM,CAAC,eAAe,IAAI,OAAO;wBAC/B,CAAC,CAAC,yDAAyD;4BACzD,sDAAsD;4BACtD,uDAAuD;4BACvD,oCAAoC;4BACpC;gCACE,KAAK,EAAE,qBAAqB,CAAC,OAAO,CAAC,KAAK,CAGzC;6BACF;wBACH,CAAC,CAAC,IAAI;oBACV,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;iBACtB,CAAC;gBACF,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACnB,CAAC;QACH,CAAC;QAAC,MAAM,CAAC,CAAA,CAAC;QAEV,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACvB,CAAC,CAAC;IAEF,IAAI,KAAiC,CAAC;IACtC,IAAI,SAAS,GAAwB,SAAS,CAAC;IAC/C,IAAI,YAAY,GAAkB,IAAI,CAAC;IACvC,IAAI,CAAC;QACH,KAAK,GAAG,MAAM,YAAY,CAAC,EAAE,GAAG,QAAQ,EAAE,IAAI,EAAE,gBAAgB,EAAE,CAAC,CAAC;IACtE,CAAC;IAAC,OAAO,GAAQ,EAAE,CAAC;QAClB,SAAS,GAAG,OAAO,CAAC;QACpB,YAAY,GAAG,GAAG,EAAE,OAAO,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC;QAC3C,MAAM,GAAG,CAAC;IACZ,CAAC;YAAS,CAAC;QACT,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC1B,MAAM,eAAe,GAAG,MAAM,GAAG,QAAQ,CAAC;QAE1C,IAAI,aAAa,GAAG,CAAC,CAAC;QACtB,IAAI,CAAC;YACH,MAAM,EAAE,aAAa,EAAE,GAAG,MAAM,MAAM,CAAC,mBAAmB,CAAC,CAAC;YAC5D,IAAI,KAAK,EAAE,CAAC;gBACV,aAAa,GAAG,aAAa,CAC3B,KAAK,CAAC,WAAW,EACjB,KAAK,CAAC,YAAY,EAClB,KAAK,CAAC,KAAK,EACX,KAAK,CAAC,eAAe,EACrB,KAAK,CAAC,gBAAgB,CACvB,CAAC;YACJ,CAAC;QACH,CAAC;QAAC,MAAM,CAAC,CAAA,CAAC;QAEV,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,IAAI,KAAK,EAAE,CAAC;YACV,YAAY,GAAG,CAAC,CAAC;YACjB,MAAM,OAAO,GAAc;gBACzB,EAAE,EAAE,MAAM,EAAE;gBACZ,KAAK;gBACL,QAAQ;gBACR,MAAM;gBACN,YAAY;gBACZ,QAAQ,EAAE,UAAU;gBACpB,IAAI,EAAE,KAAK,CAAC,KAAK;gBACjB,WAAW,EAAE,KAAK,CAAC,WAAW;gBAC9B,YAAY,EAAE,KAAK,CAAC,YAAY;gBAChC,eAAe,EAAE,KAAK,CAAC,eAAe;gBACtC,gBAAgB,EAAE,KAAK,CAAC,gBAAgB;gBACxC,aAAa;gBACb,UAAU,EAAE,eAAe;gBAC3B,MAAM,EAAE,SAAS;gBACjB,YAAY;gBACZ,QAAQ,EAAE,IAAI;gBACd,SAAS,EAAE,QAAQ;aACpB,CAAC;YACF,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACtB,CAAC;QAED,MAAM,UAAU,GAAc;YAC5B,EAAE,EAAE,YAAY;YAChB,KAAK;YACL,QAAQ;YACR,MAAM;YACN,YAAY,EAAE,IAAI;YAClB,QAAQ,EAAE,WAAW;YACrB,IAAI,EAAE,WAAW;YACjB,WAAW,EAAE,KAAK,EAAE,WAAW,IAAI,CAAC;YACpC,YAAY,EAAE,KAAK,EAAE,YAAY,IAAI,CAAC;YACtC,eAAe,EAAE,KAAK,EAAE,eAAe,IAAI,CAAC;YAC5C,gBAAgB,EAAE,KAAK,EAAE,gBAAgB,IAAI,CAAC;YAC9C,aAAa;YACb,UAAU,EAAE,eAAe;YAC3B,MAAM,EAAE,SAAS;YACjB,YAAY;YACZ,QAAQ,EAAE,IAAI;YACd,SAAS,EAAE,QAAQ;SACpB,CAAC;QACF,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAEvB,MAAM,OAAO,GAAiB;YAC5B,KAAK;YACL,QAAQ;YACR,MAAM;YACN,UAAU,EAAE,KAAK,CAAC,MAAM;YACxB,QAAQ,EAAE,YAAY;YACtB,SAAS,EAAE,aAAa;YACxB,eAAe;YACf,WAAW;YACX,eAAe;YACf,kBAAkB,EAAE,aAAa;YACjC,gBAAgB,EAAE,KAAK,EAAE,WAAW,IAAI,CAAC;YACzC,iBAAiB,EAAE,KAAK,EAAE,YAAY,IAAI,CAAC;YAC3C,KAAK,EAAE,KAAK,EAAE,KAAK,IAAI,QAAQ,CAAC,KAAK;YACrC,SAAS,EAAE,QAAQ;SACpB,CAAC;QAEF,cAAc,CAAC,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAE9D,uEAAuE;QACvE,uEAAuE;QACvE,6EAA6E;QAC7E,0EAA0E;QAC1E,IAAI,CAAC;YACH,IAAI,KAAK,EAAE,CAAC;gBACV,YAAY,CAAC,MAAM,cAAc,CAAC,UAAU,EAAE,EAAE,CAAC,EAAE;oBACjD,MAAM,EAAE,SAAS;oBACjB,YAAY;oBACZ,UAAU,EAAE;wBACV,WAAW,EAAE,KAAK,CAAC,KAAK;wBACxB,kBAAkB,EAAE,KAAK,CAAC,WAAW;wBACrC,mBAAmB,EAAE,KAAK,CAAC,YAAY;wBACvC,uBAAuB,EAAE,KAAK,CAAC,eAAe;wBAC9C,wBAAwB,EAAE,KAAK,CAAC,gBAAgB;wBAChD,qBAAqB,EAAE,aAAa;qBACrC;iBACF,CAAC,CAAC;YACL,CAAC;YACD,KAAK,MAAM,QAAQ,IAAI,iBAAiB,EAAE,CAAC;gBACzC,YAAY,CAAC,QAAQ,EAAE;oBACrB,MAAM,EAAE,OAAO;oBACf,YAAY,EAAE,mCAAmC;iBAClD,CAAC,CAAC;YACL,CAAC;YACD,iBAAiB,CAAC,KAAK,EAAE,CAAC;YAC1B,YAAY,CAAC,MAAM,kBAAkB,EAAE;gBACrC,MAAM,EAAE,SAAS;gBACjB,YAAY;gBACZ,UAAU,EAAE;oBACV,kBAAkB,EAAE,aAAa;oBACjC,wBAAwB,EAAE,eAAe;oBACzC,oBAAoB,EAAE,WAAW;oBACjC,mBAAmB,EAAE,eAAe;oBACpC,oBAAoB,EAAE,KAAK,EAAE,WAAW,IAAI,CAAC;oBAC7C,qBAAqB,EAAE,KAAK,EAAE,YAAY,IAAI,CAAC;oBAC/C,uBAAuB,EAAE,aAAa;iBACvC;aACF,CAAC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACP,wCAAwC;QAC1C,CAAC;IACH,CAAC;IAED,OAAO,KAAM,CAAC;AAChB,CAAC;AAED,KAAK,UAAU,cAAc,CAC3B,KAAkB,EAClB,OAAqB,EACrB,KAAa,EACb,MAA2B;IAE3B,MAAM,EAAE,eAAe,EAAE,kBAAkB,EAAE,GAAG,MAAM,MAAM,CAAC,YAAY,CAAC,CAAC;IAC3E,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IACxE,MAAM,kBAAkB,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IAElD,qDAAqD;IACrD,IAAI,CAAC;QACH,MAAM,EAAE,WAAW,EAAE,GAAG,MAAM,MAAM,CAAC,YAAY,CAAC,CAAC;QACnD,MAAM,WAAW,CAAC,KAAK,EAAE,EAAE,UAAU,EAAE,MAAM,CAAC,cAAc,EAAE,CAAC,CAAC;IAClE,CAAC;IAAC,MAAM,CAAC,CAAA,CAAC;AACZ,CAAC","sourcesContent":["import type { AgentLoopUsage } from \"../agent/production-agent.js\";\nimport type { AgentChatEvent, AgentToolInput } from \"../agent/types.js\";\nimport { type AgentSpan, endAgentSpan, startAgentSpan } from \"./tracing.js\";\nimport type { TraceSpan, TraceSummary, ObservabilityConfig } from \"./types.js\";\nimport { DEFAULT_OBSERVABILITY_CONFIG } from \"./types.js\";\n\nfunction spanId(): string {\n return `span-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;\n}\n\n/** Keys whose values are stripped from persisted tool inputs when\n * `captureToolArgs` is enabled. Matched case-insensitively and tolerant\n * of `_` / `-` separators. M14 in the MCP/A2A audit: tool calls\n * routinely receive credentials verbatim (db-exec INSERTs, fetchTool\n * Authorization headers, ad-hoc bearer tokens) — keeping those values\n * out of agent_trace_spans.metadata avoids long-term storage of\n * short-lived secrets. /\nconst SENSITIVE_FIELD_PATTERN =\n /^(authorization\|cookie\|api[_-]?key\|password\|secret\|token\|access[_-]?token\|refresh[_-]?token\|bearer)$/i;\n\n/* Recursively walk a structured value and replace sensitive field\n * values with the literal string \"[REDACTED]\". Pure (returns a copy);\n * the original input is never mutated. Cycles are tolerated via a\n * small WeakSet seen-tracker that returns \"[Circular]\" for repeats. /\nexport function redactSensitiveFields(value: unknown): unknown {\n return redactWalk(value, new WeakSet<object>());\n}\n\nfunction redactWalk(value: unknown, seen: WeakSet<object>): unknown {\n if (value === null \|\| typeof value !== \"object\") return value;\n if (seen.has(value as object)) return \"[Circular]\";\n seen.add(value as object);\n if (Array.isArray(value)) {\n return value.map((v) => redactWalk(v, seen));\n }\n const out: Record<string, unknown> = {};\n for (const [k, v] of Object.entries(value as Record<string, unknown>)) {\n if (SENSITIVE_FIELD_PATTERN.test(k)) {\n out[k] = \"[REDACTED]\";\n } else {\n out[k] = redactWalk(v, seen);\n }\n }\n return out;\n}\n\nexport async function getObservabilityConfig(): Promise<ObservabilityConfig> {\n try {\n const { getSetting } = await import(\"../settings/store.js\");\n const stored = await getSetting(\"observability-config\");\n if (stored) {\n return {\n ...DEFAULT_OBSERVABILITY_CONFIG,\n ...stored,\n } as ObservabilityConfig;\n }\n } catch {}\n return DEFAULT_OBSERVABILITY_CONFIG;\n}\n\nexport async function instrumentAgentLoop(opts: {\n runAgentLoop: (loopOpts: {\n engine: any;\n model: string;\n systemPrompt: string;\n tools: any[];\n messages: any[];\n actions: Record<string, any>;\n send: (event: AgentChatEvent) => void;\n signal: AbortSignal;\n providerOptions?: any;\n }) => Promise<AgentLoopUsage>;\n loopOpts: {\n engine: any;\n model: string;\n systemPrompt: string;\n tools: any[];\n messages: any[];\n actions: Record<string, any>;\n send: (event: AgentChatEvent) => void;\n signal: AbortSignal;\n providerOptions?: any;\n };\n runId: string;\n threadId: string \| null;\n /* Owner of this run; persisted on every span + summary so dashboard\n * reads can filter to a single user. Null for unauthenticated callers\n * (background tasks, etc.) — those rows aren't returned by per-user\n * reads. */\n userId: string \| null;\n config: ObservabilityConfig;\n}): Promise<AgentLoopUsage> {\n const { runAgentLoop, loopOpts, runId, threadId, userId, config } = opts;\n const runStart = Date.now();\n const parentSpanId = spanId();\n\n // Optional OpenTelemetry root span for this run. No-ops unless a host has\n // installed `@opentelemetry/api` and registered a provider. The promise is\n // resolved before the loop runs so child tool/model spans can parent under\n // it conceptually (we keep them flat in the same tracer, which is enough\n // for the dashboards an embedding app would build).\n const otelRunSpanPromise = startAgentSpan(\"agent.run\", {\n \"agent.run_id\": runId,\n \"agent.thread_id\": threadId ?? undefined,\n \"agent.user_id\": userId ?? undefined,\n \"agent.model\": loopOpts.model,\n });\n\n const spans: TraceSpan[] = [];\n let toolInvocationCounter = 0;\n // Keyed by counter to handle concurrent calls to the same tool name\n const pendingTools = new Map<\n number,\n {\n spanId: string;\n startMs: number;\n toolName: string;\n input: AgentToolInput;\n otelSpan: AgentSpan \| null;\n endResult?: { status: \"success\" \| \"error\"; errorMessage: string \| null };\n }\n >();\n // Secondary index: tool name → FIFO queue of pending invocation counters.\n // tool_start/tool_done events carry only the tool name (no call id), so to\n // pair starts and dones correctly when the agent runs concurrent calls to the\n // same tool name (read-only / parallelSafe batches via Promise.all), we keep a\n // queue per name and match each done to the OLDEST still-pending start.\n const toolNameToCounters = new Map<string, number[]>();\n\n let toolCallCount = 0;\n let successfulTools = 0;\n let failedTools = 0;\n\n // Track in-flight OTel tool spans so they're all ended even if the loop\n // throws before a matching `tool_done` arrives.\n const openOtelToolSpans = new Set<AgentSpan>();\n\n const instrumentedSend = (event: AgentChatEvent): void => {\n try {\n if (event.type === \"tool_start\") {\n const counter = toolInvocationCounter++;\n const sid = spanId();\n // Start the OTel tool span synchronously-ish: kick off the async\n // resolution and stash the span once it lands. Tool spans are short\n // and the api tracer is synchronous in practice, but we tolerate the\n // microtask gap by recording the span on the pending entry when ready.\n const entry: {\n spanId: string;\n startMs: number;\n toolName: string;\n input: AgentToolInput;\n otelSpan: AgentSpan \| null;\n // Set by the done handler if it fires before the span promise\n // resolves, so the resolved span is ended with the correct status.\n endResult?: {\n status: \"success\" \| \"error\";\n errorMessage: string \| null;\n };\n } = {\n spanId: sid,\n startMs: Date.now(),\n toolName: event.tool,\n input: event.input,\n otelSpan: null,\n };\n pendingTools.set(counter, entry);\n void startAgentSpan(\"tool.call\", {\n \"tool.name\": event.tool,\n }).then((span) => {\n if (!span) return;\n // If `tool_done` already ran for this call, end the span now with the\n // status it recorded; otherwise stash it for the done handler.\n if (entry.endResult) {\n endAgentSpan(span, {\n status: entry.endResult.status,\n errorMessage: entry.endResult.errorMessage,\n });\n } else {\n entry.otelSpan = span;\n openOtelToolSpans.add(span);\n }\n });\n const queue = toolNameToCounters.get(event.tool);\n if (queue) queue.push(counter);\n else toolNameToCounters.set(event.tool, [counter]);\n } else if (event.type === \"tool_done\") {\n const queue = toolNameToCounters.get(event.tool);\n const counter = queue?.shift();\n const pending =\n counter !== undefined ? pendingTools.get(counter) : undefined;\n if (counter !== undefined) {\n pendingTools.delete(counter);\n if (queue && queue.length === 0)\n toolNameToCounters.delete(event.tool);\n }\n toolCallCount++;\n\n const isError =\n typeof event.result === \"string\" &&\n (event.result.startsWith(\"Error\") \|\|\n event.result.startsWith(\"Error running \"));\n if (isError) failedTools++;\n else successfulTools++;\n\n // Finalize the OTel tool span. If the span promise hasn't resolved yet\n // we record the result on the entry so its `.then` handler ends it.\n const otelEndResult = {\n status: (isError ? \"error\" : \"success\") as \"success\" \| \"error\",\n errorMessage: isError ? (event.result as string) : null,\n };\n if (pending?.otelSpan) {\n openOtelToolSpans.delete(pending.otelSpan);\n endAgentSpan(pending.otelSpan, {\n status: otelEndResult.status,\n errorMessage: otelEndResult.errorMessage,\n attributes: { \"tool.name\": event.tool },\n });\n } else if (pending) {\n pending.endResult = otelEndResult;\n }\n\n const span: TraceSpan = {\n id: pending?.spanId ?? spanId(),\n runId,\n threadId,\n userId,\n parentSpanId,\n spanType: \"tool_call\",\n name: event.tool,\n inputTokens: 0,\n outputTokens: 0,\n cacheReadTokens: 0,\n cacheWriteTokens: 0,\n costCentsX100: 0,\n durationMs: pending ? Date.now() - pending.startMs : 0,\n status: isError ? \"error\" : \"success\",\n errorMessage: isError ? event.result : null,\n metadata:\n config.captureToolArgs && pending\n ? // Strip Authorization/api-key/token-shaped values before\n // persisting (M14 in the MCP/A2A audit). Tool-runtime\n // execution still sees the unredacted input — only the\n // long-lived span row is sanitized.\n {\n input: redactSensitiveFields(pending.input) as Record<\n string,\n string\n >,\n }\n : null,\n createdAt: Date.now(),\n };\n spans.push(span);\n }\n } catch {}\n\n loopOpts.send(event);\n };\n\n let usage: AgentLoopUsage \| undefined;\n let runStatus: \"success\" \| \"error\" = \"success\";\n let errorMessage: string \| null = null;\n try {\n usage = await runAgentLoop({ ...loopOpts, send: instrumentedSend });\n } catch (err: any) {\n runStatus = \"error\";\n errorMessage = err?.message ?? String(err);\n throw err;\n } finally {\n const runEnd = Date.now();\n const totalDurationMs = runEnd - runStart;\n\n let costCentsX100 = 0;\n try {\n const { calculateCost } = await import(\"../usage/store.js\");\n if (usage) {\n costCentsX100 = calculateCost(\n usage.inputTokens,\n usage.outputTokens,\n usage.model,\n usage.cacheReadTokens,\n usage.cacheWriteTokens,\n );\n }\n } catch {}\n\n let llmCallCount = 0;\n if (usage) {\n llmCallCount = 1;\n const llmSpan: TraceSpan = {\n id: spanId(),\n runId,\n threadId,\n userId,\n parentSpanId,\n spanType: \"llm_call\",\n name: usage.model,\n inputTokens: usage.inputTokens,\n outputTokens: usage.outputTokens,\n cacheReadTokens: usage.cacheReadTokens,\n cacheWriteTokens: usage.cacheWriteTokens,\n costCentsX100,\n durationMs: totalDurationMs,\n status: runStatus,\n errorMessage,\n metadata: null,\n createdAt: runStart,\n };\n spans.push(llmSpan);\n }\n\n const parentSpan: TraceSpan = {\n id: parentSpanId,\n runId,\n threadId,\n userId,\n parentSpanId: null,\n spanType: \"agent_run\",\n name: \"agent_run\",\n inputTokens: usage?.inputTokens ?? 0,\n outputTokens: usage?.outputTokens ?? 0,\n cacheReadTokens: usage?.cacheReadTokens ?? 0,\n cacheWriteTokens: usage?.cacheWriteTokens ?? 0,\n costCentsX100,\n durationMs: totalDurationMs,\n status: runStatus,\n errorMessage,\n metadata: null,\n createdAt: runStart,\n };\n spans.push(parentSpan);\n\n const summary: TraceSummary = {\n runId,\n threadId,\n userId,\n totalSpans: spans.length,\n llmCalls: llmCallCount,\n toolCalls: toolCallCount,\n successfulTools,\n failedTools,\n totalDurationMs,\n totalCostCentsX100: costCentsX100,\n totalInputTokens: usage?.inputTokens ?? 0,\n totalOutputTokens: usage?.outputTokens ?? 0,\n model: usage?.model ?? loopOpts.model,\n createdAt: runStart,\n };\n\n writeTraceData(spans, summary, runId, config).catch(() => {});\n\n // OpenTelemetry export (no-op unless a provider is registered). Emit a\n // self-contained `llm.call` span carrying model + token usage, end any\n // tool spans still open (loop threw mid-tool), and end the run span. Awaited\n // so the spans are emitted before the function returns; cheap when no-op.\n try {\n if (usage) {\n endAgentSpan(await startAgentSpan(\"llm.call\", {}), {\n status: runStatus,\n errorMessage,\n attributes: {\n \"llm.model\": usage.model,\n \"llm.input_tokens\": usage.inputTokens,\n \"llm.output_tokens\": usage.outputTokens,\n \"llm.cache_read_tokens\": usage.cacheReadTokens,\n \"llm.cache_write_tokens\": usage.cacheWriteTokens,\n \"llm.cost_cents_x100\": costCentsX100,\n },\n });\n }\n for (const toolSpan of openOtelToolSpans) {\n endAgentSpan(toolSpan, {\n status: \"error\",\n errorMessage: \"Agent run ended before tool_done.\",\n });\n }\n openOtelToolSpans.clear();\n endAgentSpan(await otelRunSpanPromise, {\n status: runStatus,\n errorMessage,\n attributes: {\n \"agent.tool_calls\": toolCallCount,\n \"agent.successful_tools\": successfulTools,\n \"agent.failed_tools\": failedTools,\n \"agent.duration_ms\": totalDurationMs,\n \"agent.input_tokens\": usage?.inputTokens ?? 0,\n \"agent.output_tokens\": usage?.outputTokens ?? 0,\n \"agent.cost_cents_x100\": costCentsX100,\n },\n });\n } catch {\n // OTel export must never break the run.\n }\n }\n\n return usage!;\n}\n\nasync function writeTraceData(\n spans: TraceSpan[],\n summary: TraceSummary,\n runId: string,\n config: ObservabilityConfig,\n): Promise<void> {\n const { insertTraceSpan, upsertTraceSummary } = await import(\"./store.js\");\n await Promise.all(spans.map((s) => insertTraceSpan(s).catch(() => {})));\n await upsertTraceSummary(summary).catch(() => {});\n\n // Fire automated evals after trace data is persisted\n try {\n const { evaluateRun } = await import(\"./evals.js\");\n await evaluateRun(runId, { sampleRate: config.evalSampleRate });\n } catch {}\n}\n"]}

package/dist/onboarding/default-steps.js CHANGED Viewed

@@ -33,7 +33,7 @@ const LLM_KEY_METHODS = [
         provider: "openrouter",
         id: "openrouter-key",
         label: "OpenRouter",
-        description: "OpenRouter models with your own OpenRouter key.",
+        description: "OpenRouter models, including GLM 5.2, with your own key.",
     },
     {
         provider: "groq",