@agent-native/core 0.84.0 → 0.84.1

package/corpus/templates/content/parity/render-matrix.ts

@@ -0,0 +1,90 @@
+import { writeFileSync } from "node:fs";
+import { fileURLToPath } from "node:url";
+
+import { parityMatrix } from "./matrix";
+import type { ParityRow } from "./matrix.types";
+
+function escapeCell(value: string) {
+  return value.replace(/\|/g, "\\|").replace(/\n/g, "<br>");
+}
+
+function cell(value: string | string[] | null | undefined) {
+  if (Array.isArray(value))
+    return escapeCell(value.length ? value.join(", ") : "-");
+  if (value === null || value === undefined || value === "") return "-";
+  return escapeCell(value);
+}
+
+const headers = [
+  "ID",
+  "Surface",
+  "User-visible action",
+  "Status",
+  "Actions",
+  "UI entrypoints",
+  "Durable effect",
+  "Exception / gap",
+  "Reliability risk",
+  "Spine priority",
+  "Test coverage",
+  "Coverage refs",
+  "Eval scenarios",
+  "Follow-up",
+];
+
+function renderRow(row: ParityRow) {
+  return [
+    row.id,
+    row.surface,
+    row.label,
+    row.status,
+    row.actions.length
+      ? row.actions.map((action) => `\`${action}\``).join(", ")
+      : "-",
+    row.uiEntrypoints.map((entry) => `\`${entry}\``).join(", "),
+    row.durableEffect,
+    row.exception,
+    row.reliabilityRisk === "none" ? "-" : row.reliabilityRisk,
+    row.spinePriority,
+    row.testCoverage,
+    row.coverageRefs?.map((ref) => `\`${ref}\``).join(", "),
+    row.evalScenarioIds?.map((id) => `\`${id}\``).join(", "),
+    row.followUpPR,
+  ].map(cell);
+}
+
+function renderTable(rows: string[][]) {
+  const widths = headers.map((header, index) =>
+    Math.max(header.length, ...rows.map((row) => row[index]?.length ?? 0)),
+  );
+  const renderCells = (cells: string[]) =>
+    `| ${cells.map((value, index) => value.padEnd(widths[index])).join(" | ")} |`;
+  return [
+    renderCells(headers),
+    renderCells(widths.map((width) => "-".repeat(width))),
+    ...rows.map(renderCells),
+  ];
+}
+
+export function renderParityMatrixMarkdown(rows: ParityRow[] = parityMatrix) {
+  const sortedRows = [...rows].sort((a, b) => a.id.localeCompare(b.id));
+  return [
+    "# Content Agent/UI Action Parity Matrix",
+    "",
+    "This generated matrix tracks whether high-value Content UI operations use the same action surface agents can call, or have an explicit exception. Edit `matrix.ts`, then regenerate this file.",
+    "",
+    ...renderTable(sortedRows.map(renderRow)),
+    "",
+  ].join("\n");
+}
+
+const isCli = process.argv[1] === fileURLToPath(import.meta.url);
+
+if (isCli) {
+  const markdown = renderParityMatrixMarkdown();
+  if (process.argv.includes("--write")) {
+    writeFileSync(new URL("./matrix.md", import.meta.url), markdown);
+  } else {
+    process.stdout.write(markdown);
+  }
+}

package/corpus/templates/content/parity/scenario-to-eval.ts

@@ -0,0 +1,57 @@
+import {
+  contains,
+  createScorer,
+  defineEval,
+  type AgentRunOutput,
+  type Eval,
+} from "@agent-native/core/eval";
+
+import type { ParityEvalScenario } from "./eval-scenarios.ts";
+
+function expectedToolScorer(expectedTools: string[]) {
+  return createScorer<AgentRunOutput, { used: string[]; missing: string[] }>({
+    name: "expected_tools",
+    analyze(run) {
+      const usedTools = new Set(run.toolCalls);
+      return {
+        used: expectedTools.filter((tool) => usedTools.has(tool)),
+        missing: expectedTools.filter((tool) => !usedTools.has(tool)),
+      };
+    },
+    generateScore({ missing }) {
+      return expectedTools.length === 0 || missing.length === 0 ? 1 : 0;
+    },
+    generateReason({ analysis: { used, missing } }) {
+      if (missing.length === 0) {
+        return `Agent called all expected tool(s): ${used.join(", ")}`;
+      }
+      return `Called expected tool(s): ${used.join(", ") || "none"}; missing: ${missing.join(", ")}`;
+    },
+  });
+}
+
+export function scenarioToEval(scenario: ParityEvalScenario): Eval {
+  const name = `content-parity:${scenario.id}`;
+
+  if (!process.env[scenario.gateEnv]) {
+    return defineEval({
+      name,
+      input: { prompt: scenario.prompt },
+      threshold: 1,
+      skipReason: `Skipped because ${scenario.gateEnv} is unset`,
+      scorers: [],
+    });
+  }
+
+  return defineEval({
+    name,
+    input: { prompt: scenario.prompt },
+    threshold: 0.6,
+    scorers: [
+      contains(scenario.successSignals),
+      ...(scenario.expectedTools?.length
+        ? [expectedToolScorer(scenario.expectedTools)]
+        : []),
+    ],
+  });
+}

package/corpus/templates/content/shared/api.ts

@@ -646,6 +646,10 @@ export interface ContentDatabaseResponse {
   createdDocumentId?: string;
   duplicatedItemId?: string;
   duplicatedDocumentId?: string;
+  duplicatedItemIds?: string[];
+  duplicatedDocumentIds?: string[];
+  deletedItemIds?: string[];
+  deletedDocumentIds?: string[];
 }
 
 export interface ContentDatabaseUnavailableResponse {
@@ -701,6 +705,13 @@ export interface DuplicateDatabaseItemRequest {
   title?: string;
 }
 
+export interface DatabaseItemsBatchRequest {
+  databaseId?: string;
+  documentId?: string;
+  itemIds?: string[];
+  documentIds?: string[];
+}
+
 export interface MoveDatabaseItemRequest {
   itemId?: string;
   documentId?: string;

package/dist/collab/awareness.d.ts

@@ -49,11 +49,11 @@ export declare const postAwareness: import("h3").EventHandlerWithFetch<import("h
     error: string;
     states?: undefined;
 } | {
+    error?: undefined;
     states: {
         clientId: number;
         state: string;
     }[];
-    error?: undefined;
 }>>;
 /**
  * GET /_agent-native/collab/:docId/users
@@ -64,10 +64,10 @@ export declare const getActiveUsers: import("h3").EventHandlerWithFetch<import("
     error: string;
     users?: undefined;
 } | {
+    error?: undefined;
     users: {
         clientId: number;
         lastSeen: number;
     }[];
-    error?: undefined;
 }>>;
 //# sourceMappingURL=awareness.d.ts.map

package/dist/collab/awareness.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"awareness.d.ts","sourceRoot":"","sources":["../../src/collab/awareness.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAS3C,MAAM,WAAW,cAAc;IAC7B,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;CAClB;AAOD,eAAO,MAAM,sBAAsB,EAAG,kBAA2B,CAAC;AAElE,MAAM,WAAW,oBAAoB;IACnC,MAAM,EAAE,WAAW,CAAC;IACpB,IAAI,EAAE,kBAAkB,CAAC;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,2EAA2E;IAC3E,MAAM,EAAE,KAAK,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACnD,gFAAgF;IAChF,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,sCAAsC;IACtC,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAKD,wBAAgB,mBAAmB,IAAI,YAAY,CAElD;AAED,wBAAgB,mBAAmB,CACjC,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,KAAK,CAAC;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,EAClD,KAAK,CAAC,EAAE,MAAM,EACd,KAAK,CAAC,EAAE,MAAM,GACb,IAAI,CAUN;AAKD,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC,CAO1E;AAED,wBAAgB,YAAY,CAAC,GAAG,EAAE,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC,GAAG,IAAI,CAOnE;AAUD;;;;;;;GAOG;AACH,eAAO,MAAM,aAAa;;;;;kBAuCa,MAAM;eAAS,MAAM;;;GAa1D,CAAC;AAEH;;;;GAIG;AACH,eAAO,MAAM,cAAc;;;;;kBAWM,MAAM;kBAAY,MAAM;;;GAMvD,CAAC"}
+{"version":3,"file":"awareness.d.ts","sourceRoot":"","sources":["../../src/collab/awareness.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAS3C,MAAM,WAAW,cAAc;IAC7B,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;CAClB;AAOD,eAAO,MAAM,sBAAsB,EAAG,kBAA2B,CAAC;AAElE,MAAM,WAAW,oBAAoB;IACnC,MAAM,EAAE,WAAW,CAAC;IACpB,IAAI,EAAE,kBAAkB,CAAC;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,2EAA2E;IAC3E,MAAM,EAAE,KAAK,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACnD,gFAAgF;IAChF,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,sCAAsC;IACtC,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAKD,wBAAgB,mBAAmB,IAAI,YAAY,CAElD;AAED,wBAAgB,mBAAmB,CACjC,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,KAAK,CAAC;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,EAClD,KAAK,CAAC,EAAE,MAAM,EACd,KAAK,CAAC,EAAE,MAAM,GACb,IAAI,CAUN;AAKD,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC,CAO1E;AAED,wBAAgB,YAAY,CAAC,GAAG,EAAE,GAAG,CAAC,MAAM,EAAE,cAAc,CAAC,GAAG,IAAI,CAOnE;AAUD;;;;;;;GAOG;AACH,eAAO,MAAM,aAAa;;;;;;kBAuCa,MAAM;eAAS,MAAM;;GAa1D,CAAC;AAEH;;;;GAIG;AACH,eAAO,MAAM,cAAc;;;;;;kBAWM,MAAM;kBAAY,MAAM;;GAMvD,CAAC"}

package/dist/collab/routes.d.ts

@@ -41,9 +41,9 @@ export declare const postCollabUpdate: import("h3").EventHandlerWithFetch<import
  * Body: { text: string, fieldName?: string, requestSource?: string }
  */
 export declare const postCollabText: import("h3").EventHandlerWithFetch<import("h3").EventHandlerRequest, Promise<{
+    text?: undefined;
     ok?: undefined;
     error: string;
-    text?: undefined;
 } | {
     error?: undefined;
     ok: boolean;

package/dist/eval/define-eval.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"define-eval.d.ts","sourceRoot":"","sources":["../../src/eval/define-eval.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAEvC,0EAA0E;AAC1E,eAAO,MAAM,sBAAsB,MAAM,CAAC;AAE1C,wBAAgB,UAAU,CAAC,IAAI,EAAE,IAAI,GAAG,IAAI,CAqB3C"}
+{"version":3,"file":"define-eval.d.ts","sourceRoot":"","sources":["../../src/eval/define-eval.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAEvC,0EAA0E;AAC1E,eAAO,MAAM,sBAAsB,MAAM,CAAC;AAE1C,wBAAgB,UAAU,CAAC,IAAI,EAAE,IAAI,GAAG,IAAI,CAwB3C"}

package/dist/eval/define-eval.js

@@ -31,7 +31,8 @@ export function defineEval(spec) {
     if (!spec.input || typeof spec.input.prompt !== "string") {
         throw new Error(`defineEval("${spec.name}"): \`input.prompt\` is required`);
     }
-    if (!Array.isArray(spec.scorers) || spec.scorers.length === 0) {
+    if ((!Array.isArray(spec.scorers) || spec.scorers.length === 0) &&
+        !spec.skipReason) {
         throw new Error(`defineEval("${spec.name}"): at least one scorer is required`);
     }
     if (spec.threshold !== undefined &&

package/dist/eval/define-eval.js.map

@@ -1 +1 @@
-{"version":3,"file":"define-eval.js","sourceRoot":"","sources":["../../src/eval/define-eval.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAIH,0EAA0E;AAC1E,MAAM,CAAC,MAAM,sBAAsB,GAAG,GAAG,CAAC;AAE1C,MAAM,UAAU,UAAU,CAAC,IAAU;IACnC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QAChD,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;IACpD,CAAC;IACD,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;QACzD,MAAM,IAAI,KAAK,CAAC,eAAe,IAAI,CAAC,IAAI,kCAAkC,CAAC,CAAC;IAC9E,CAAC;IACD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC9D,MAAM,IAAI,KAAK,CACb,eAAe,IAAI,CAAC,IAAI,qCAAqC,CAC9D,CAAC;IACJ,CAAC;IACD,IACE,IAAI,CAAC,SAAS,KAAK,SAAS;QAC5B,CAAC,IAAI,CAAC,SAAS,GAAG,CAAC,IAAI,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC,EAC1C,CAAC;QACD,MAAM,IAAI,KAAK,CACb,eAAe,IAAI,CAAC,IAAI,qCAAqC,CAC9D,CAAC;IACJ,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC","sourcesContent":["/**\n * `defineEval` — declare a named eval test case.\n *\n * An eval pairs an input (prompt + optional history/setup) with a list of\n * scorers and a pass threshold. The runner (see `runner.ts`) actually runs the\n * agent for the input, scores the output with each scorer, and gates on the\n * threshold. Authors write `*.eval.ts` files that `export default defineEval(...)`\n * or export an array of them.\n *\n * Example (`evals/greeting.eval.ts`):\n * ```ts\n * import { defineEval, contains, llmJudge } from \"@agent-native/core/eval\";\n *\n * export default defineEval({\n *   name: \"greets the user by name\",\n *   input: { prompt: \"Say hi to Ada.\" },\n *   threshold: 0.7,\n *   scorers: [\n *     contains(\"Ada\"),\n *     llmJudge({ criteria: \"friendliness\", rubric: \"1.0 = warm greeting\" }),\n *   ],\n * });\n * ```\n */\n\nimport type { Eval } from \"./types.js\";\n\n/** Default per-scorer pass threshold when an eval doesn't specify one. */\nexport const DEFAULT_EVAL_THRESHOLD = 0.5;\n\nexport function defineEval(spec: Eval): Eval {\n  if (!spec.name || typeof spec.name !== \"string\") {\n    throw new Error(\"defineEval: `name` is required\");\n  }\n  if (!spec.input || typeof spec.input.prompt !== \"string\") {\n    throw new Error(`defineEval(\"${spec.name}\"): \\`input.prompt\\` is required`);\n  }\n  if (!Array.isArray(spec.scorers) || spec.scorers.length === 0) {\n    throw new Error(\n      `defineEval(\"${spec.name}\"): at least one scorer is required`,\n    );\n  }\n  if (\n    spec.threshold !== undefined &&\n    (spec.threshold < 0 || spec.threshold > 1)\n  ) {\n    throw new Error(\n      `defineEval(\"${spec.name}\"): \\`threshold\\` must be in [0, 1]`,\n    );\n  }\n  return spec;\n}\n"]}
+{"version":3,"file":"define-eval.js","sourceRoot":"","sources":["../../src/eval/define-eval.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAIH,0EAA0E;AAC1E,MAAM,CAAC,MAAM,sBAAsB,GAAG,GAAG,CAAC;AAE1C,MAAM,UAAU,UAAU,CAAC,IAAU;IACnC,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QAChD,MAAM,IAAI,KAAK,CAAC,gCAAgC,CAAC,CAAC;IACpD,CAAC;IACD,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC;QACzD,MAAM,IAAI,KAAK,CAAC,eAAe,IAAI,CAAC,IAAI,kCAAkC,CAAC,CAAC;IAC9E,CAAC;IACD,IACE,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC;QAC3D,CAAC,IAAI,CAAC,UAAU,EAChB,CAAC;QACD,MAAM,IAAI,KAAK,CACb,eAAe,IAAI,CAAC,IAAI,qCAAqC,CAC9D,CAAC;IACJ,CAAC;IACD,IACE,IAAI,CAAC,SAAS,KAAK,SAAS;QAC5B,CAAC,IAAI,CAAC,SAAS,GAAG,CAAC,IAAI,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC,EAC1C,CAAC;QACD,MAAM,IAAI,KAAK,CACb,eAAe,IAAI,CAAC,IAAI,qCAAqC,CAC9D,CAAC;IACJ,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC","sourcesContent":["/**\n * `defineEval` — declare a named eval test case.\n *\n * An eval pairs an input (prompt + optional history/setup) with a list of\n * scorers and a pass threshold. The runner (see `runner.ts`) actually runs the\n * agent for the input, scores the output with each scorer, and gates on the\n * threshold. Authors write `*.eval.ts` files that `export default defineEval(...)`\n * or export an array of them.\n *\n * Example (`evals/greeting.eval.ts`):\n * ```ts\n * import { defineEval, contains, llmJudge } from \"@agent-native/core/eval\";\n *\n * export default defineEval({\n *   name: \"greets the user by name\",\n *   input: { prompt: \"Say hi to Ada.\" },\n *   threshold: 0.7,\n *   scorers: [\n *     contains(\"Ada\"),\n *     llmJudge({ criteria: \"friendliness\", rubric: \"1.0 = warm greeting\" }),\n *   ],\n * });\n * ```\n */\n\nimport type { Eval } from \"./types.js\";\n\n/** Default per-scorer pass threshold when an eval doesn't specify one. */\nexport const DEFAULT_EVAL_THRESHOLD = 0.5;\n\nexport function defineEval(spec: Eval): Eval {\n  if (!spec.name || typeof spec.name !== \"string\") {\n    throw new Error(\"defineEval: `name` is required\");\n  }\n  if (!spec.input || typeof spec.input.prompt !== \"string\") {\n    throw new Error(`defineEval(\"${spec.name}\"): \\`input.prompt\\` is required`);\n  }\n  if (\n    (!Array.isArray(spec.scorers) || spec.scorers.length === 0) &&\n    !spec.skipReason\n  ) {\n    throw new Error(\n      `defineEval(\"${spec.name}\"): at least one scorer is required`,\n    );\n  }\n  if (\n    spec.threshold !== undefined &&\n    (spec.threshold < 0 || spec.threshold > 1)\n  ) {\n    throw new Error(\n      `defineEval(\"${spec.name}\"): \\`threshold\\` must be in [0, 1]`,\n    );\n  }\n  return spec;\n}\n"]}

package/dist/eval/report.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"report.d.ts","sourceRoot":"","sources":["../../src/eval/report.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAgBhD,8CAA8C;AAC9C,wBAAgB,YAAY,CAAC,MAAM,EAAE,aAAa,GAAG,MAAM,CAqC1D"}
+{"version":3,"file":"report.d.ts","sourceRoot":"","sources":["../../src/eval/report.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAgBhD,8CAA8C;AAC9C,wBAAgB,YAAY,CAAC,MAAM,EAAE,aAAa,GAAG,MAAM,CAoD1D"}

package/dist/eval/report.js

@@ -21,9 +21,16 @@ export function formatReport(report) {
     lines.push("  Evals");
     lines.push("  ─────");
     for (const row of report.results) {
-        const mark = row.passed ? "✓" : "✗";
+        const mark = row.status === "skipped" ? "-" : row.passed ? "✓" : "✗";
         lines.push("");
-        lines.push(`  ${mark} ${row.eval}  (avg ${pct(row.avgScore)}, threshold ${pct(row.threshold)})`);
+        if (row.status === "skipped") {
+            lines.push(`  ${mark} ${row.eval}  (skipped)`);
+            lines.push(`      reason: ${row.skipReason ?? "No reason provided"}`);
+            continue;
+        }
+        else {
+            lines.push(`  ${mark} ${row.eval}  (avg ${pct(row.avgScore)}, threshold ${pct(row.threshold)})`);
+        }
         if (row.error) {
             lines.push(`      ⚠ run error: ${row.error}`);
         }
@@ -35,8 +42,16 @@ export function formatReport(report) {
     }
     lines.push("");
     lines.push("  ─────");
-    const verdict = report.failed === 0 ? "PASS" : "FAIL";
-    lines.push(`  ${verdict}: ${report.passed}/${report.total} evals passed` +
+    const skipped = report.skipped ?? 0;
+    const executedTotal = report.total - skipped;
+    const executedPassed = report.passed - skipped;
+    const verdict = executedTotal === 0 && skipped > 0
+        ? "SKIPPED"
+        : report.failed === 0
+            ? "PASS"
+            : "FAIL";
+    lines.push(`  ${verdict}: ${executedPassed}/${executedTotal} evals passed` +
+        (skipped > 0 ? `, ${skipped} skipped` : "") +
         (report.failed > 0 ? `, ${report.failed} below threshold` : ""));
     lines.push("");
     return lines.join("\n");

package/dist/eval/report.js.map

@@ -1 +1 @@
-{"version":3,"file":"report.js","sourceRoot":"","sources":["../../src/eval/report.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAIH,SAAS,GAAG,CAAC,KAAa,EAAE,KAAK,GAAG,EAAE;IACpC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC,CAAC;IAClD,OAAO,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC;AACzD,CAAC;AAED,SAAS,OAAO,CAAC,CAAS;IACxB,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;QAAE,OAAO,CAAC,CAAC;IAClC,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;AACrC,CAAC;AAED,SAAS,GAAG,CAAC,KAAa;IACxB,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;AAC5D,CAAC;AAED,8CAA8C;AAC9C,MAAM,UAAU,YAAY,CAAC,MAAqB;IAChD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACtB,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAEtB,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QACjC,MAAM,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;QACpC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CACR,KAAK,IAAI,IAAI,GAAG,CAAC,IAAI,UAAU,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,eAAe,GAAG,CAChE,GAAG,CAAC,SAAS,CACd,GAAG,CACL,CAAC;QACF,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;YACd,KAAK,CAAC,IAAI,CAAC,sBAAsB,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;QAChD,CAAC;QACD,KAAK,MAAM,CAAC,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC;YAC3B,MAAM,KAAK,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;YACnC,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACjD,KAAK,CAAC,IAAI,CACR,SAAS,KAAK,IAAI,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,GAAG,CAC1D,CAAC,CAAC,KAAK,CACR,GAAG,MAAM,EAAE,CACb,CAAC;QACJ,CAAC;IACH,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACtB,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;IACtD,KAAK,CAAC,IAAI,CACR,KAAK,OAAO,KAAK,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,KAAK,eAAe;QAC3D,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,MAAM,kBAAkB,CAAC,CAAC,CAAC,EAAE,CAAC,CAClE,CAAC;IACF,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC","sourcesContent":["/**\n * Human-readable formatting for an eval run report. Kept separate from the\n * runner so the CLI can print a table while CI consumes the JSON shape.\n */\n\nimport type { EvalRunReport } from \"./types.js\";\n\nfunction bar(score: number, width = 10): string {\n  const filled = Math.round(clamp01(score) * width);\n  return \"█\".repeat(filled) + \"░\".repeat(width - filled);\n}\n\nfunction clamp01(n: number): number {\n  if (!Number.isFinite(n)) return 0;\n  return Math.max(0, Math.min(1, n));\n}\n\nfunction pct(score: number): string {\n  return `${Math.round(clamp01(score) * 100)}%`.padStart(4);\n}\n\n/** Render a scored table for the terminal. */\nexport function formatReport(report: EvalRunReport): string {\n  const lines: string[] = [];\n  lines.push(\"\");\n  lines.push(\"  Evals\");\n  lines.push(\"  ─────\");\n\n  for (const row of report.results) {\n    const mark = row.passed ? \"✓\" : \"✗\";\n    lines.push(\"\");\n    lines.push(\n      `  ${mark} ${row.eval}  (avg ${pct(row.avgScore)}, threshold ${pct(\n        row.threshold,\n      )})`,\n    );\n    if (row.error) {\n      lines.push(`      ⚠ run error: ${row.error}`);\n    }\n    for (const s of row.scores) {\n      const smark = s.passed ? \"✓\" : \"✗\";\n      const reason = s.reason ? `  — ${s.reason}` : \"\";\n      lines.push(\n        `      ${smark} ${s.scorer.padEnd(20)} ${bar(s.score)} ${pct(\n          s.score,\n        )}${reason}`,\n      );\n    }\n  }\n\n  lines.push(\"\");\n  lines.push(\"  ─────\");\n  const verdict = report.failed === 0 ? \"PASS\" : \"FAIL\";\n  lines.push(\n    `  ${verdict}: ${report.passed}/${report.total} evals passed` +\n      (report.failed > 0 ? `, ${report.failed} below threshold` : \"\"),\n  );\n  lines.push(\"\");\n  return lines.join(\"\\n\");\n}\n"]}
+{"version":3,"file":"report.js","sourceRoot":"","sources":["../../src/eval/report.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAIH,SAAS,GAAG,CAAC,KAAa,EAAE,KAAK,GAAG,EAAE;IACpC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,KAAK,CAAC,CAAC;IAClD,OAAO,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC;AACzD,CAAC;AAED,SAAS,OAAO,CAAC,CAAS;IACxB,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC;QAAE,OAAO,CAAC,CAAC;IAClC,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;AACrC,CAAC;AAED,SAAS,GAAG,CAAC,KAAa;IACxB,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;AAC5D,CAAC;AAED,8CAA8C;AAC9C,MAAM,UAAU,YAAY,CAAC,MAAqB;IAChD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACtB,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAEtB,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QACjC,MAAM,IAAI,GAAG,GAAG,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;QACrE,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YAC7B,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,IAAI,GAAG,CAAC,IAAI,aAAa,CAAC,CAAC;YAC/C,KAAK,CAAC,IAAI,CAAC,iBAAiB,GAAG,CAAC,UAAU,IAAI,oBAAoB,EAAE,CAAC,CAAC;YACtE,SAAS;QACX,CAAC;aAAM,CAAC;YACN,KAAK,CAAC,IAAI,CACR,KAAK,IAAI,IAAI,GAAG,CAAC,IAAI,UAAU,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,eAAe,GAAG,CAChE,GAAG,CAAC,SAAS,CACd,GAAG,CACL,CAAC;QACJ,CAAC;QACD,IAAI,GAAG,CAAC,KAAK,EAAE,CAAC;YACd,KAAK,CAAC,IAAI,CAAC,sBAAsB,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;QAChD,CAAC;QACD,KAAK,MAAM,CAAC,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC;YAC3B,MAAM,KAAK,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC;YACnC,MAAM,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACjD,KAAK,CAAC,IAAI,CACR,SAAS,KAAK,IAAI,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,GAAG,CAC1D,CAAC,CAAC,KAAK,CACR,GAAG,MAAM,EAAE,CACb,CAAC;QACJ,CAAC;IACH,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACtB,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,IAAI,CAAC,CAAC;IACpC,MAAM,aAAa,GAAG,MAAM,CAAC,KAAK,GAAG,OAAO,CAAC;IAC7C,MAAM,cAAc,GAAG,MAAM,CAAC,MAAM,GAAG,OAAO,CAAC;IAC/C,MAAM,OAAO,GACX,aAAa,KAAK,CAAC,IAAI,OAAO,GAAG,CAAC;QAChC,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC;YACnB,CAAC,CAAC,MAAM;YACR,CAAC,CAAC,MAAM,CAAC;IACf,KAAK,CAAC,IAAI,CACR,KAAK,OAAO,KAAK,cAAc,IAAI,aAAa,eAAe;QAC7D,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,OAAO,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC;QAC3C,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,MAAM,CAAC,MAAM,kBAAkB,CAAC,CAAC,CAAC,EAAE,CAAC,CAClE,CAAC;IACF,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC","sourcesContent":["/**\n * Human-readable formatting for an eval run report. Kept separate from the\n * runner so the CLI can print a table while CI consumes the JSON shape.\n */\n\nimport type { EvalRunReport } from \"./types.js\";\n\nfunction bar(score: number, width = 10): string {\n  const filled = Math.round(clamp01(score) * width);\n  return \"█\".repeat(filled) + \"░\".repeat(width - filled);\n}\n\nfunction clamp01(n: number): number {\n  if (!Number.isFinite(n)) return 0;\n  return Math.max(0, Math.min(1, n));\n}\n\nfunction pct(score: number): string {\n  return `${Math.round(clamp01(score) * 100)}%`.padStart(4);\n}\n\n/** Render a scored table for the terminal. */\nexport function formatReport(report: EvalRunReport): string {\n  const lines: string[] = [];\n  lines.push(\"\");\n  lines.push(\"  Evals\");\n  lines.push(\"  ─────\");\n\n  for (const row of report.results) {\n    const mark = row.status === \"skipped\" ? \"-\" : row.passed ? \"✓\" : \"✗\";\n    lines.push(\"\");\n    if (row.status === \"skipped\") {\n      lines.push(`  ${mark} ${row.eval}  (skipped)`);\n      lines.push(`      reason: ${row.skipReason ?? \"No reason provided\"}`);\n      continue;\n    } else {\n      lines.push(\n        `  ${mark} ${row.eval}  (avg ${pct(row.avgScore)}, threshold ${pct(\n          row.threshold,\n        )})`,\n      );\n    }\n    if (row.error) {\n      lines.push(`      ⚠ run error: ${row.error}`);\n    }\n    for (const s of row.scores) {\n      const smark = s.passed ? \"✓\" : \"✗\";\n      const reason = s.reason ? `  — ${s.reason}` : \"\";\n      lines.push(\n        `      ${smark} ${s.scorer.padEnd(20)} ${bar(s.score)} ${pct(\n          s.score,\n        )}${reason}`,\n      );\n    }\n  }\n\n  lines.push(\"\");\n  lines.push(\"  ─────\");\n  const skipped = report.skipped ?? 0;\n  const executedTotal = report.total - skipped;\n  const executedPassed = report.passed - skipped;\n  const verdict =\n    executedTotal === 0 && skipped > 0\n      ? \"SKIPPED\"\n      : report.failed === 0\n        ? \"PASS\"\n        : \"FAIL\";\n  lines.push(\n    `  ${verdict}: ${executedPassed}/${executedTotal} evals passed` +\n      (skipped > 0 ? `, ${skipped} skipped` : \"\") +\n      (report.failed > 0 ? `, ${report.failed} below threshold` : \"\"),\n  );\n  lines.push(\"\");\n  return lines.join(\"\\n\");\n}\n"]}

package/dist/eval/runner.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/eval/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAKH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,8BAA8B,CAAC;AAGhE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAIrD,OAAO,KAAK,EAEV,IAAI,EACJ,aAAa,EACb,aAAa,EAEd,MAAM,YAAY,CAAC;AAsCpB,wEAAwE;AACxE,wBAAsB,SAAS,CAC7B,QAAQ,EAAE,IAAI,EACd,MAAM,EAAE,WAAW,EACnB,IAAI,GAAE;IAAE,iBAAiB,CAAC,EAAE,MAAM,CAAA;CAAO,GACxC,OAAO,CAAC,aAAa,CAAC,CAkCxB;AAED,sEAAsE;AACtE,wBAAsB,QAAQ,CAC5B,KAAK,EAAE,IAAI,EAAE,EACb,MAAM,EAAE,WAAW,EACnB,IAAI,GAAE;IAAE,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,OAAO,CAAA;CAAO,GAC3D,OAAO,CAAC,aAAa,CAAC,CAexB;AA4CD;;;;;GAKG;AACH,wBAAsB,iBAAiB,CACrC,IAAI,EAAE,MAAM,EACZ,OAAO,CAAC,EAAE,MAAM,GACf,OAAO,CAAC,MAAM,EAAE,CAAC,CAqCnB;AA0BD,8EAA8E;AAC9E,wBAAsB,SAAS,CAC7B,IAAI,EAAE,MAAM,EACZ,OAAO,CAAC,EAAE,MAAM,GACf,OAAO,CAAC;IAAE,KAAK,EAAE,MAAM,EAAE,CAAC;IAAC,KAAK,EAAE,IAAI,EAAE,CAAA;CAAE,CAAC,CAW7C;AAID,MAAM,WAAW,mBAAmB;IAClC,wEAAwE;IACxE,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,8CAA8C;IAC9C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iEAAiE;IACjE,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,wEAAwE;IACxE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;IACtC,8BAA8B;IAC9B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,oEAAoE;IACpE,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,0EAA0E;IAC1E,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,yEAAyE;IACzE,KAAK,CAAC,EAAE,IAAI,EAAE,CAAC;CAChB;AAED;;;GAGG;AACH,wBAAsB,YAAY,CAChC,IAAI,GAAE,mBAAwB,GAC7B,OAAO,CAAC;IAAE,MAAM,EAAE,aAAa,CAAC;IAAC,KAAK,EAAE,MAAM,EAAE,CAAA;CAAE,CAAC,CAuBrD"}
+{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/eval/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAKH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,8BAA8B,CAAC;AAGhE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAIrD,OAAO,KAAK,EAEV,IAAI,EACJ,aAAa,EACb,aAAa,EAEd,MAAM,YAAY,CAAC;AAsCpB,wEAAwE;AACxE,wBAAsB,SAAS,CAC7B,QAAQ,EAAE,IAAI,EACd,MAAM,EAAE,WAAW,EACnB,IAAI,GAAE;IAAE,iBAAiB,CAAC,EAAE,MAAM,CAAA;CAAO,GACxC,OAAO,CAAC,aAAa,CAAC,CAiDxB;AAED,sEAAsE;AACtE,wBAAsB,QAAQ,CAC5B,KAAK,EAAE,IAAI,EAAE,EACb,MAAM,EAAE,WAAW,EACnB,IAAI,GAAE;IAAE,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,OAAO,CAAA;CAAO,GAC3D,OAAO,CAAC,aAAa,CAAC,CAmBxB;AA4CD;;;;;GAKG;AACH,wBAAsB,iBAAiB,CACrC,IAAI,EAAE,MAAM,EACZ,OAAO,CAAC,EAAE,MAAM,GACf,OAAO,CAAC,MAAM,EAAE,CAAC,CAqCnB;AA0BD,8EAA8E;AAC9E,wBAAsB,SAAS,CAC7B,IAAI,EAAE,MAAM,EACZ,OAAO,CAAC,EAAE,MAAM,GACf,OAAO,CAAC;IAAE,KAAK,EAAE,MAAM,EAAE,CAAC;IAAC,KAAK,EAAE,IAAI,EAAE,CAAA;CAAE,CAAC,CAW7C;AAID,MAAM,WAAW,mBAAmB;IAClC,wEAAwE;IACxE,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,8CAA8C;IAC9C,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,iEAAiE;IACjE,iBAAiB,CAAC,EAAE,MAAM,CAAC;IAC3B,wEAAwE;IACxE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;IACtC,8BAA8B;IAC9B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,oEAAoE;IACpE,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,0EAA0E;IAC1E,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,yEAAyE;IACzE,KAAK,CAAC,EAAE,IAAI,EAAE,CAAC;CAChB;AAED;;;GAGG;AACH,wBAAsB,YAAY,CAChC,IAAI,GAAE,mBAAwB,GAC7B,OAAO,CAAC;IAAE,MAAM,EAAE,aAAa,CAAC;IAAC,KAAK,EAAE,MAAM,EAAE,CAAA;CAAE,CAAC,CA0BrD"}

package/dist/eval/runner.js

@@ -53,6 +53,19 @@ async function runScorer(scorer, run, runner, threshold) {
 /** Run a single eval: invoke the agent, then score with each scorer. */
 export async function scoreEval(evalCase, runner, opts = {}) {
     const threshold = opts.thresholdOverride ?? evalCase.threshold ?? DEFAULT_EVAL_THRESHOLD;
+    if (evalCase.skipReason) {
+        return {
+            eval: evalCase.name,
+            threshold,
+            scores: [],
+            status: "skipped",
+            skipReason: evalCase.skipReason,
+            // Keep the legacy boolean gate friendly: skipped rows do not fail CI.
+            passed: true,
+            avgScore: 0,
+            durationMs: 0,
+        };
+    }
     let run;
     if (evalCase.run) {
         run = await evalCase.run({
@@ -76,6 +89,7 @@ export async function scoreEval(evalCase, runner, opts = {}) {
         scores,
         // A run that errored, or any sub-threshold scorer, fails the case.
         passed: run.ok && scores.every((s) => s.passed),
+        status: run.ok && scores.every((s) => s.passed) ? "passed" : "failed",
         avgScore,
         durationMs: run.durationMs,
         error: run.ok ? undefined : run.error,
@@ -87,14 +101,17 @@ export async function runEvals(evals, runner, opts = {}) {
     for (const evalCase of evals) {
         const row = await scoreEval(evalCase, runner, opts);
         results.push(row);
-        if (opts.persist)
+        if (opts.persist && row.status !== "skipped") {
             await persistEvalRow(row).catch(() => { });
+        }
     }
     const passed = results.filter((r) => r.passed).length;
+    const skipped = results.filter((r) => r.status === "skipped").length;
     return {
         total: results.length,
         passed,
-        failed: results.length - passed,
+        failed: results.filter((r) => r.status !== "skipped" && !r.passed).length,
+        skipped,
         results,
     };
 }
@@ -227,11 +244,14 @@ export async function runEvalSuite(opts = {}) {
         files = loaded.files;
         evals = loaded.evals;
     }
+    const needsRunner = evals.some((evalCase) => !evalCase.skipReason);
     const runner = opts.runner ??
-        (await createAgentRunner({
-            actions: opts.actions ?? (await discoverActions(cwd)),
-            systemPrompt: opts.systemPrompt,
-        }));
+        (needsRunner
+            ? await createAgentRunner({
+                actions: opts.actions ?? (await discoverActions(cwd)),
+                systemPrompt: opts.systemPrompt,
+            })
+            : createInertRunner());
     const report = await runEvals(evals, runner, {
         thresholdOverride: opts.thresholdOverride,
         persist: opts.persist ?? true,
@@ -253,4 +273,16 @@ async function discoverActions(cwd) {
         return {};
     }
 }
+function createInertRunner() {
+    return {
+        engine: {},
+        model: "inert",
+        async runAgent() {
+            throw new Error("Eval unexpectedly requested the agent runner");
+        },
+        analyzeContext() {
+            throw new Error("Eval unexpectedly requested analyze context");
+        },
+    };
+}
 //# sourceMappingURL=runner.js.map

package/dist/eval/runner.js.map

@@ -1 +1 @@
-{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../src/eval/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,QAAQ,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAGzC,OAAO,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAC;AAG7D,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,EAAE,sBAAsB,EAAE,MAAM,kBAAkB,CAAC;AAC1D,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAStC,yEAAyE;AAEzE,yEAAyE;AACzE,KAAK,UAAU,SAAS,CACtB,MAA+B,EAC/B,GAAmB,EACnB,MAAmB,EACnB,SAAiB;IAEjB,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;QACnE,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO;YAC7B,CAAC,CAAC,MAAM,MAAM,CAAC,OAAO,CAAC,GAAY,EAAE,MAAM,CAAC,cAAc,EAAE,CAAC;YAC7D,CAAC,CAAC,GAAG,CAAC;QACR,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,QAAiB,CAAC,CAAC;QAC/D,MAAM,KAAK,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;QAChC,MAAM,MAAM,GAAG,MAAM,CAAC,cAAc;YAClC,CAAC,CAAC,MAAM,MAAM,CAAC,cAAc,CAAC;gBAC1B,GAAG;gBACH,QAAQ,EAAE,QAAiB;gBAC3B,KAAK;aACN,CAAC;YACJ,CAAC,CAAC,SAAS,CAAC;QACd,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,IAAI,SAAS,EAAE,CAAC;IAC5E,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,uEAAuE;QACvE,kEAAkE;QAClE,OAAO;YACL,MAAM,EAAE,MAAM,CAAC,IAAI;YACnB,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,mBAAmB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;YAC7E,MAAM,EAAE,KAAK;SACd,CAAC;IACJ,CAAC;AACH,CAAC;AAED,wEAAwE;AACxE,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,QAAc,EACd,MAAmB,EACnB,IAAI,GAAmC,EAAE;IAEzC,MAAM,SAAS,GACb,IAAI,CAAC,iBAAiB,IAAI,QAAQ,CAAC,SAAS,IAAI,sBAAsB,CAAC;IAEzE,IAAI,GAAmB,CAAC;IACxB,IAAI,QAAQ,CAAC,GAAG,EAAE,CAAC;QACjB,GAAG,GAAG,MAAM,QAAQ,CAAC,GAAG,CAAC;YACvB,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,QAAQ,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;SAC5C,CAAC,CAAC;IACL,CAAC;SAAM,CAAC;QACN,GAAG,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC9C,CAAC;IAED,MAAM,MAAM,GAAmB,EAAE,CAAC;IAClC,KAAK,MAAM,MAAM,IAAI,QAAQ,CAAC,OAAO,EAAE,CAAC;QACtC,MAAM,CAAC,IAAI,CAAC,MAAM,SAAS,CAAC,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC;IAC/D,CAAC;IAED,MAAM,QAAQ,GACZ,MAAM,CAAC,MAAM,GAAG,CAAC;QACf,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM;QACzD,CAAC,CAAC,CAAC,CAAC;IAER,OAAO;QACL,IAAI,EAAE,QAAQ,CAAC,IAAI;QACnB,SAAS;QACT,MAAM;QACN,mEAAmE;QACnE,MAAM,EAAE,GAAG,CAAC,EAAE,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;QAC/C,QAAQ;QACR,UAAU,EAAE,GAAG,CAAC,UAAU;QAC1B,KAAK,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK;KACtC,CAAC;AACJ,CAAC;AAED,sEAAsE;AACtE,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,KAAa,EACb,MAAmB,EACnB,IAAI,GAAsD,EAAE;IAE5D,MAAM,OAAO,GAAoB,EAAE,CAAC;IACpC,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE,CAAC;QAC7B,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;QACpD,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAClB,IAAI,IAAI,CAAC,OAAO;YAAE,MAAM,cAAc,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IAC9D,CAAC;IAED,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IACtD,OAAO;QACL,KAAK,EAAE,OAAO,CAAC,MAAM;QACrB,MAAM;QACN,MAAM,EAAE,OAAO,CAAC,MAAM,GAAG,MAAM;QAC/B,OAAO;KACR,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,KAAK,UAAU,cAAc,CAAC,GAAkB;IAC9C,MAAM,KAAK,GAAG,QAAQ,GAAG,CAAC,IAAI,IAAI,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;IAC/C,KAAK,MAAM,CAAC,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC;QAC3B,MAAM,MAAM,GAA4B;YACtC,EAAE,EAAE,MAAM,CAAC,UAAU,EAAE;YACvB,KAAK;YACL,QAAQ,EAAE,IAAI;YACd,MAAM,EAAE,IAAI;YACZ,QAAQ,EAAE,WAAW;YACrB,QAAQ,EAAE,QAAQ,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,MAAM,EAAE;YACxC,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,SAAS,EAAE,CAAC,CAAC,MAAM,IAAI,IAAI;YAC3B,QAAQ,EAAE;gBACR,MAAM,EAAE,UAAU;gBAClB,SAAS,EAAE,GAAG,CAAC,SAAS;gBACxB,MAAM,EAAE,CAAC,CAAC,MAAM;aACjB;YACD,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;SACtB,CAAC;QACF,MAAM,gBAAgB,CAAC,MAAM,CAAC,CAAC;IACjC,CAAC;AACH,CAAC;AAED,yEAAyE;AAEzE,MAAM,YAAY,GAAG,sBAAsB,CAAC;AAC5C,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,CAAC,cAAc,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC,CAAC;AAEhF;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,IAAY,EACZ,OAAgB;IAEhB,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC;IACnC,MAAM,GAAG,GAAa,EAAE,CAAC;IAEzB,SAAS,UAAU,CAAC,IAAY,EAAE,UAAkB;QAClD,MAAM,IAAI,GAAG,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACrC,IAAI,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,OAAO,IAAI,CAAC;QACzC,IAAI,UAAU,KAAK,OAAO,IAAI,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC1D,4CAA4C;YAC5C,OAAO,CAAC,+BAA+B,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACrD,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,SAAS,IAAI,CAAC,GAAW,EAAE,UAAkB;QAC3C,IAAI,OAAmC,CAAC;QACxC,IAAI,CAAC;YACH,OAAO,GAAG,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;QACzD,CAAC;QAAC,MAAM,CAAC;YACP,OAAO;QACT,CAAC;QACD,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;YAC5C,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;gBACxB,IAAI,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;oBAAE,SAAS;gBACtE,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;YACzB,CAAC;iBAAM,IAAI,KAAK,CAAC,MAAM,EAAE,IAAI,UAAU,CAAC,IAAI,EAAE,UAAU,CAAC,EAAE,CAAC;gBAC1D,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACjB,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;IACpC,GAAG,CAAC,IAAI,EAAE,CAAC;IAEX,IAAI,CAAC,OAAO;QAAE,OAAO,GAAG,CAAC;IACzB,OAAO,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;AACzE,CAAC;AAED,yEAAyE;AACzE,SAAS,YAAY,CAAC,GAA4B;IAChD,MAAM,UAAU,GAAc,EAAE,CAAC;IACjC,IAAI,GAAG,CAAC,OAAO,KAAK,SAAS;QAAE,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAC5D,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;QAC/C,IAAI,GAAG,KAAK,SAAS;YAAE,SAAS;QAChC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACzB,CAAC;IAED,MAAM,KAAK,GAAW,EAAE,CAAC;IACzB,KAAK,MAAM,CAAC,IAAI,UAAU,CAAC,IAAI,EAAE,EAAE,CAAC;QAClC,IACE,CAAC;YACD,OAAO,CAAC,KAAK,QAAQ;YACrB,OAAQ,CAAU,CAAC,IAAI,KAAK,QAAQ;YACpC,KAAK,CAAC,OAAO,CAAE,CAAU,CAAC,OAAO,CAAC;YACjC,CAAU,CAAC,KAAK,EACjB,CAAC;YACD,KAAK,CAAC,IAAI,CAAC,CAAS,CAAC,CAAC;QACxB,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,8EAA8E;AAC9E,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,IAAY,EACZ,OAAgB;IAEhB,MAAM,KAAK,GAAG,MAAM,iBAAiB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACrD,MAAM,KAAK,GAAW,EAAE,CAAC;IACzB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,GAAG,GAAG,CAAC,MAAM,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAGlD,CAAC;QACF,KAAK,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC;IACnC,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;AAC1B,CAAC;AAuBD;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,IAAI,GAAwB,EAAE;IAE9B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;IAEtC,IAAI,KAAK,GAAa,EAAE,CAAC;IACzB,IAAI,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;IACvB,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAClD,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;QACrB,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;IACvB,CAAC;IAED,MAAM,MAAM,GACV,IAAI,CAAC,MAAM;QACX,CAAC,MAAM,iBAAiB,CAAC;YACvB,OAAO,EAAE,IAAI,CAAC,OAAO,IAAI,CAAC,MAAM,eAAe,CAAC,GAAG,CAAC,CAAC;YACrD,YAAY,EAAE,IAAI,CAAC,YAAY;SAChC,CAAC,CAAC,CAAC;IAEN,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE;QAC3C,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;QACzC,OAAO,EAAE,IAAI,CAAC,OAAO,IAAI,IAAI;KAC9B,CAAC,CAAC;IACH,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;AAC3B,CAAC;AAED;;;;GAIG;AACH,KAAK,UAAU,eAAe,CAC5B,GAAW;IAEX,IAAI,CAAC;QACH,MAAM,EAAE,mBAAmB,EAAE,GAC3B,MAAM,MAAM,CAAC,+BAA+B,CAAC,CAAC;QAChD,MAAM,UAAU,GAAG,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;QACjD,OAAO,MAAM,mBAAmB,CAAC,aAAa,CAAC,UAAU,GAAG,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;IACzE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC","sourcesContent":["/**\n * The evals runner: discover `*.eval.ts` / `evals/*.ts` files, run each eval\n * through its scorer pipeline against the *real* agent loop, score, and report.\n *\n * It is the engine behind `agent-native eval` — when used as a CI deploy gate\n * the CLI exits non-zero if any eval scores below its threshold.\n *\n * Two layers:\n *   - `scoreEval` / `runEvals` — pure orchestration over an `AgentRunner` and\n *     a list of evals. Fully unit-testable with an injected runner (no model).\n *   - `discoverEvalFiles` / `loadEvals` — filesystem discovery + dynamic import\n *     of author-written eval modules.\n *\n * Results are also (best-effort) written to the observability eval store so a\n * dashboard can surface CI eval history next to production run evals.\n */\n\nimport nodePath from \"node:path\";\nimport { pathToFileURL } from \"node:url\";\n\nimport type { ActionEntry } from \"../agent/production-agent.js\";\nimport { insertEvalResult } from \"../observability/store.js\";\nimport type { EvalResult as ObservabilityEvalResult } from \"../observability/types.js\";\nimport type { AgentRunner } from \"./agent-runner.js\";\nimport { createAgentRunner } from \"./agent-runner.js\";\nimport { DEFAULT_EVAL_THRESHOLD } from \"./define-eval.js\";\nimport { clamp01 } from \"./scorer.js\";\nimport type {\n  AgentRunOutput,\n  Eval,\n  EvalResultRow,\n  EvalRunReport,\n  ScorerResult,\n} from \"./types.js\";\n\n// ─── Scoring orchestration ────────────────────────────────────────────\n\n/** Run one scorer's pipeline (preprocess → analyze → score → reason). */\nasync function runScorer(\n  scorer: Eval[\"scorers\"][number],\n  run: AgentRunOutput,\n  runner: AgentRunner,\n  threshold: number,\n): Promise<ScorerResult> {\n  try {\n    const pre = scorer.preprocess ? await scorer.preprocess(run) : run;\n    const analysis = scorer.analyze\n      ? await scorer.analyze(pre as never, runner.analyzeContext())\n      : pre;\n    const rawScore = await scorer.generateScore(analysis as never);\n    const score = clamp01(rawScore);\n    const reason = scorer.generateReason\n      ? await scorer.generateReason({\n          run,\n          analysis: analysis as never,\n          score,\n        })\n      : undefined;\n    return { scorer: scorer.name, score, reason, passed: score >= threshold };\n  } catch (err) {\n    // A scorer that throws is a failed scorer, not a crashed run — degrade\n    // gracefully so one bad scorer can't take down the whole CI gate.\n    return {\n      scorer: scorer.name,\n      score: 0,\n      reason: `Scorer errored: ${err instanceof Error ? err.message : String(err)}`,\n      passed: false,\n    };\n  }\n}\n\n/** Run a single eval: invoke the agent, then score with each scorer. */\nexport async function scoreEval(\n  evalCase: Eval,\n  runner: AgentRunner,\n  opts: { thresholdOverride?: number } = {},\n): Promise<EvalResultRow> {\n  const threshold =\n    opts.thresholdOverride ?? evalCase.threshold ?? DEFAULT_EVAL_THRESHOLD;\n\n  let run: AgentRunOutput;\n  if (evalCase.run) {\n    run = await evalCase.run({\n      input: evalCase.input,\n      runAgent: (input) => runner.runAgent(input),\n    });\n  } else {\n    run = await runner.runAgent(evalCase.input);\n  }\n\n  const scores: ScorerResult[] = [];\n  for (const scorer of evalCase.scorers) {\n    scores.push(await runScorer(scorer, run, runner, threshold));\n  }\n\n  const avgScore =\n    scores.length > 0\n      ? scores.reduce((s, r) => s + r.score, 0) / scores.length\n      : 0;\n\n  return {\n    eval: evalCase.name,\n    threshold,\n    scores,\n    // A run that errored, or any sub-threshold scorer, fails the case.\n    passed: run.ok && scores.every((s) => s.passed),\n    avgScore,\n    durationMs: run.durationMs,\n    error: run.ok ? undefined : run.error,\n  };\n}\n\n/** Run a batch of evals against one runner and aggregate a report. */\nexport async function runEvals(\n  evals: Eval[],\n  runner: AgentRunner,\n  opts: { thresholdOverride?: number; persist?: boolean } = {},\n): Promise<EvalRunReport> {\n  const results: EvalResultRow[] = [];\n  for (const evalCase of evals) {\n    const row = await scoreEval(evalCase, runner, opts);\n    results.push(row);\n    if (opts.persist) await persistEvalRow(row).catch(() => {});\n  }\n\n  const passed = results.filter((r) => r.passed).length;\n  return {\n    total: results.length,\n    passed,\n    failed: results.length - passed,\n    results,\n  };\n}\n\n/**\n * Best-effort write of one eval result to the observability eval store so a\n * dashboard can show CI eval history alongside production run evals. We write\n * one row per (eval × scorer), tagged `evalType: \"automated\"` with a synthetic\n * `eval:` run id.\n *\n * TODO(live-sampling): the same scorer list should also run on a sampled\n * fraction of *real* production runs. That hook belongs in the agent loop's\n * (not-yet-added) post-run processor seam: when a run finishes, roll the\n * configured sample rate and, if it hits, replay the run output through these\n * scorers and write the rows here. Wiring it now would require the in-loop\n * processor seam another wave is adding — so this is the single intended\n * attachment point, intentionally left as a note.\n */\nasync function persistEvalRow(row: EvalResultRow): Promise<void> {\n  const runId = `eval:${row.eval}:${Date.now()}`;\n  for (const s of row.scores) {\n    const result: ObservabilityEvalResult = {\n      id: crypto.randomUUID(),\n      runId,\n      threadId: null,\n      userId: null,\n      evalType: \"automated\",\n      criteria: `eval:${row.eval}:${s.scorer}`,\n      score: s.score,\n      reasoning: s.reason ?? null,\n      metadata: {\n        source: \"cli-eval\",\n        threshold: row.threshold,\n        passed: s.passed,\n      },\n      createdAt: Date.now(),\n    };\n    await insertEvalResult(result);\n  }\n}\n\n// ─── Discovery + loading ──────────────────────────────────────────────\n\nconst EVAL_FILE_RE = /\\.eval\\.(ts|js|mjs)$/;\nconst SKIP_DIRS = new Set([\"node_modules\", \"dist\", \".git\", \".output\", \"build\"]);\n\n/**\n * Walk `root` for eval files. Matches two conventions:\n *   - any `**\\/*.eval.ts` (co-located with code), and\n *   - any `*.ts` directly inside an `evals/` directory.\n * `pattern` further filters by substring of the relative path.\n */\nexport async function discoverEvalFiles(\n  root: string,\n  pattern?: string,\n): Promise<string[]> {\n  const fs = await import(\"node:fs\");\n  const out: string[] = [];\n\n  function isEvalFile(full: string, parentName: string): boolean {\n    const base = nodePath.basename(full);\n    if (EVAL_FILE_RE.test(base)) return true;\n    if (parentName === \"evals\" && /\\.(ts|js|mjs)$/.test(base)) {\n      // Skip obvious support files inside evals/.\n      return !/\\.(spec|test|d)\\.(ts|js|mjs)$/.test(base);\n    }\n    return false;\n  }\n\n  function walk(dir: string, parentName: string): void {\n    let entries: import(\"node:fs\").Dirent[];\n    try {\n      entries = fs.readdirSync(dir, { withFileTypes: true });\n    } catch {\n      return;\n    }\n    for (const entry of entries) {\n      const full = nodePath.join(dir, entry.name);\n      if (entry.isDirectory()) {\n        if (SKIP_DIRS.has(entry.name) || entry.name.startsWith(\".\")) continue;\n        walk(full, entry.name);\n      } else if (entry.isFile() && isEvalFile(full, parentName)) {\n        out.push(full);\n      }\n    }\n  }\n\n  walk(root, nodePath.basename(root));\n  out.sort();\n\n  if (!pattern) return out;\n  return out.filter((f) => nodePath.relative(root, f).includes(pattern));\n}\n\n/** Pull `Eval` definitions out of a dynamically-imported eval module. */\nfunction extractEvals(mod: Record<string, unknown>): Eval[] {\n  const candidates: unknown[] = [];\n  if (mod.default !== undefined) candidates.push(mod.default);\n  for (const [key, value] of Object.entries(mod)) {\n    if (key === \"default\") continue;\n    candidates.push(value);\n  }\n\n  const evals: Eval[] = [];\n  for (const c of candidates.flat()) {\n    if (\n      c &&\n      typeof c === \"object\" &&\n      typeof (c as Eval).name === \"string\" &&\n      Array.isArray((c as Eval).scorers) &&\n      (c as Eval).input\n    ) {\n      evals.push(c as Eval);\n    }\n  }\n  return evals;\n}\n\n/** Discover and import all eval files under `root`, returning their evals. */\nexport async function loadEvals(\n  root: string,\n  pattern?: string,\n): Promise<{ files: string[]; evals: Eval[] }> {\n  const files = await discoverEvalFiles(root, pattern);\n  const evals: Eval[] = [];\n  for (const file of files) {\n    const mod = (await import(pathToFileURL(file).href)) as Record<\n      string,\n      unknown\n    >;\n    evals.push(...extractEvals(mod));\n  }\n  return { files, evals };\n}\n\n// ─── High-level entry used by the CLI ─────────────────────────────────\n\nexport interface RunEvalSuiteOptions {\n  /** App root to discover eval files + actions under. Defaults to cwd. */\n  cwd?: string;\n  /** Substring filter on the eval file path. */\n  pattern?: string;\n  /** Global threshold override (wins over per-eval thresholds). */\n  thresholdOverride?: number;\n  /** App actions to expose to the agent. Auto-discovered when omitted. */\n  actions?: Record<string, ActionEntry>;\n  /** System prompt for runs. */\n  systemPrompt?: string;\n  /** Write results to the observability eval store (default true). */\n  persist?: boolean;\n  /** Pre-built runner (tests inject this to avoid touching engine/loop). */\n  runner?: AgentRunner;\n  /** Pre-loaded evals (tests inject this to skip filesystem discovery). */\n  evals?: Eval[];\n}\n\n/**\n * End-to-end: load evals, build a runner, score, report. The CLI wraps this\n * and maps `report.failed > 0` to a non-zero exit code (the CI gate).\n */\nexport async function runEvalSuite(\n  opts: RunEvalSuiteOptions = {},\n): Promise<{ report: EvalRunReport; files: string[] }> {\n  const cwd = opts.cwd ?? process.cwd();\n\n  let files: string[] = [];\n  let evals = opts.evals;\n  if (!evals) {\n    const loaded = await loadEvals(cwd, opts.pattern);\n    files = loaded.files;\n    evals = loaded.evals;\n  }\n\n  const runner =\n    opts.runner ??\n    (await createAgentRunner({\n      actions: opts.actions ?? (await discoverActions(cwd)),\n      systemPrompt: opts.systemPrompt,\n    }));\n\n  const report = await runEvals(evals, runner, {\n    thresholdOverride: opts.thresholdOverride,\n    persist: opts.persist ?? true,\n  });\n  return { report, files };\n}\n\n/**\n * Discover the app's actions so the agent under test has the real tool\n * surface. Lazy-imports `autoDiscoverActions` to keep server-only deps out of\n * any browser bundle that might touch this module's types.\n */\nasync function discoverActions(\n  cwd: string,\n): Promise<Record<string, ActionEntry>> {\n  try {\n    const { autoDiscoverActions } =\n      await import(\"../server/action-discovery.js\");\n    const actionsDir = nodePath.join(cwd, \"actions\");\n    return await autoDiscoverActions(pathToFileURL(actionsDir + \"/\").href);\n  } catch {\n    return {};\n  }\n}\n"]}
+{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../src/eval/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,QAAQ,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAGzC,OAAO,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAC;AAG7D,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,EAAE,sBAAsB,EAAE,MAAM,kBAAkB,CAAC;AAC1D,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAStC,yEAAyE;AAEzE,yEAAyE;AACzE,KAAK,UAAU,SAAS,CACtB,MAA+B,EAC/B,GAAmB,EACnB,MAAmB,EACnB,SAAiB;IAEjB,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;QACnE,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO;YAC7B,CAAC,CAAC,MAAM,MAAM,CAAC,OAAO,CAAC,GAAY,EAAE,MAAM,CAAC,cAAc,EAAE,CAAC;YAC7D,CAAC,CAAC,GAAG,CAAC;QACR,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,QAAiB,CAAC,CAAC;QAC/D,MAAM,KAAK,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;QAChC,MAAM,MAAM,GAAG,MAAM,CAAC,cAAc;YAClC,CAAC,CAAC,MAAM,MAAM,CAAC,cAAc,CAAC;gBAC1B,GAAG;gBACH,QAAQ,EAAE,QAAiB;gBAC3B,KAAK;aACN,CAAC;YACJ,CAAC,CAAC,SAAS,CAAC;QACd,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,IAAI,SAAS,EAAE,CAAC;IAC5E,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,uEAAuE;QACvE,kEAAkE;QAClE,OAAO;YACL,MAAM,EAAE,MAAM,CAAC,IAAI;YACnB,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,mBAAmB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;YAC7E,MAAM,EAAE,KAAK;SACd,CAAC;IACJ,CAAC;AACH,CAAC;AAED,wEAAwE;AACxE,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,QAAc,EACd,MAAmB,EACnB,IAAI,GAAmC,EAAE;IAEzC,MAAM,SAAS,GACb,IAAI,CAAC,iBAAiB,IAAI,QAAQ,CAAC,SAAS,IAAI,sBAAsB,CAAC;IAEzE,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC;QACxB,OAAO;YACL,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,SAAS;YACT,MAAM,EAAE,EAAE;YACV,MAAM,EAAE,SAAS;YACjB,UAAU,EAAE,QAAQ,CAAC,UAAU;YAC/B,sEAAsE;YACtE,MAAM,EAAE,IAAI;YACZ,QAAQ,EAAE,CAAC;YACX,UAAU,EAAE,CAAC;SACd,CAAC;IACJ,CAAC;IAED,IAAI,GAAmB,CAAC;IACxB,IAAI,QAAQ,CAAC,GAAG,EAAE,CAAC;QACjB,GAAG,GAAG,MAAM,QAAQ,CAAC,GAAG,CAAC;YACvB,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,QAAQ,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;SAC5C,CAAC,CAAC;IACL,CAAC;SAAM,CAAC;QACN,GAAG,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC9C,CAAC;IAED,MAAM,MAAM,GAAmB,EAAE,CAAC;IAClC,KAAK,MAAM,MAAM,IAAI,QAAQ,CAAC,OAAO,EAAE,CAAC;QACtC,MAAM,CAAC,IAAI,CAAC,MAAM,SAAS,CAAC,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC;IAC/D,CAAC;IAED,MAAM,QAAQ,GACZ,MAAM,CAAC,MAAM,GAAG,CAAC;QACf,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM;QACzD,CAAC,CAAC,CAAC,CAAC;IAER,OAAO;QACL,IAAI,EAAE,QAAQ,CAAC,IAAI;QACnB,SAAS;QACT,MAAM;QACN,mEAAmE;QACnE,MAAM,EAAE,GAAG,CAAC,EAAE,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;QAC/C,MAAM,EAAE,GAAG,CAAC,EAAE,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ;QACrE,QAAQ;QACR,UAAU,EAAE,GAAG,CAAC,UAAU;QAC1B,KAAK,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK;KACtC,CAAC;AACJ,CAAC;AAED,sEAAsE;AACtE,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,KAAa,EACb,MAAmB,EACnB,IAAI,GAAsD,EAAE;IAE5D,MAAM,OAAO,GAAoB,EAAE,CAAC;IACpC,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE,CAAC;QAC7B,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;QACpD,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAClB,IAAI,IAAI,CAAC,OAAO,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YAC7C,MAAM,cAAc,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAC5C,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IACtD,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,MAAM,CAAC;IACrE,OAAO;QACL,KAAK,EAAE,OAAO,CAAC,MAAM;QACrB,MAAM;QACN,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,SAAS,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM;QACzE,OAAO;QACP,OAAO;KACR,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,KAAK,UAAU,cAAc,CAAC,GAAkB;IAC9C,MAAM,KAAK,GAAG,QAAQ,GAAG,CAAC,IAAI,IAAI,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;IAC/C,KAAK,MAAM,CAAC,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC;QAC3B,MAAM,MAAM,GAA4B;YACtC,EAAE,EAAE,MAAM,CAAC,UAAU,EAAE;YACvB,KAAK;YACL,QAAQ,EAAE,IAAI;YACd,MAAM,EAAE,IAAI;YACZ,QAAQ,EAAE,WAAW;YACrB,QAAQ,EAAE,QAAQ,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,MAAM,EAAE;YACxC,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,SAAS,EAAE,CAAC,CAAC,MAAM,IAAI,IAAI;YAC3B,QAAQ,EAAE;gBACR,MAAM,EAAE,UAAU;gBAClB,SAAS,EAAE,GAAG,CAAC,SAAS;gBACxB,MAAM,EAAE,CAAC,CAAC,MAAM;aACjB;YACD,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;SACtB,CAAC;QACF,MAAM,gBAAgB,CAAC,MAAM,CAAC,CAAC;IACjC,CAAC;AACH,CAAC;AAED,yEAAyE;AAEzE,MAAM,YAAY,GAAG,sBAAsB,CAAC;AAC5C,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,CAAC,cAAc,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC,CAAC;AAEhF;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,IAAY,EACZ,OAAgB;IAEhB,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC;IACnC,MAAM,GAAG,GAAa,EAAE,CAAC;IAEzB,SAAS,UAAU,CAAC,IAAY,EAAE,UAAkB;QAClD,MAAM,IAAI,GAAG,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACrC,IAAI,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,OAAO,IAAI,CAAC;QACzC,IAAI,UAAU,KAAK,OAAO,IAAI,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC1D,4CAA4C;YAC5C,OAAO,CAAC,+BAA+B,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACrD,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,SAAS,IAAI,CAAC,GAAW,EAAE,UAAkB;QAC3C,IAAI,OAAmC,CAAC;QACxC,IAAI,CAAC;YACH,OAAO,GAAG,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;QACzD,CAAC;QAAC,MAAM,CAAC;YACP,OAAO;QACT,CAAC;QACD,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;YAC5C,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;gBACxB,IAAI,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;oBAAE,SAAS;gBACtE,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;YACzB,CAAC;iBAAM,IAAI,KAAK,CAAC,MAAM,EAAE,IAAI,UAAU,CAAC,IAAI,EAAE,UAAU,CAAC,EAAE,CAAC;gBAC1D,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACjB,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;IACpC,GAAG,CAAC,IAAI,EAAE,CAAC;IAEX,IAAI,CAAC,OAAO;QAAE,OAAO,GAAG,CAAC;IACzB,OAAO,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;AACzE,CAAC;AAED,yEAAyE;AACzE,SAAS,YAAY,CAAC,GAA4B;IAChD,MAAM,UAAU,GAAc,EAAE,CAAC;IACjC,IAAI,GAAG,CAAC,OAAO,KAAK,SAAS;QAAE,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAC5D,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;QAC/C,IAAI,GAAG,KAAK,SAAS;YAAE,SAAS;QAChC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACzB,CAAC;IAED,MAAM,KAAK,GAAW,EAAE,CAAC;IACzB,KAAK,MAAM,CAAC,IAAI,UAAU,CAAC,IAAI,EAAE,EAAE,CAAC;QAClC,IACE,CAAC;YACD,OAAO,CAAC,KAAK,QAAQ;YACrB,OAAQ,CAAU,CAAC,IAAI,KAAK,QAAQ;YACpC,KAAK,CAAC,OAAO,CAAE,CAAU,CAAC,OAAO,CAAC;YACjC,CAAU,CAAC,KAAK,EACjB,CAAC;YACD,KAAK,CAAC,IAAI,CAAC,CAAS,CAAC,CAAC;QACxB,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,8EAA8E;AAC9E,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,IAAY,EACZ,OAAgB;IAEhB,MAAM,KAAK,GAAG,MAAM,iBAAiB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACrD,MAAM,KAAK,GAAW,EAAE,CAAC;IACzB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,GAAG,GAAG,CAAC,MAAM,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAGlD,CAAC;QACF,KAAK,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC;IACnC,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;AAC1B,CAAC;AAuBD;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,IAAI,GAAwB,EAAE;IAE9B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;IAEtC,IAAI,KAAK,GAAa,EAAE,CAAC;IACzB,IAAI,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;IACvB,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAClD,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;QACrB,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;IACvB,CAAC;IAED,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;IACnE,MAAM,MAAM,GACV,IAAI,CAAC,MAAM;QACX,CAAC,WAAW;YACV,CAAC,CAAC,MAAM,iBAAiB,CAAC;gBACtB,OAAO,EAAE,IAAI,CAAC,OAAO,IAAI,CAAC,MAAM,eAAe,CAAC,GAAG,CAAC,CAAC;gBACrD,YAAY,EAAE,IAAI,CAAC,YAAY;aAChC,CAAC;YACJ,CAAC,CAAC,iBAAiB,EAAE,CAAC,CAAC;IAE3B,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE;QAC3C,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;QACzC,OAAO,EAAE,IAAI,CAAC,OAAO,IAAI,IAAI;KAC9B,CAAC,CAAC;IACH,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;AAC3B,CAAC;AAED;;;;GAIG;AACH,KAAK,UAAU,eAAe,CAC5B,GAAW;IAEX,IAAI,CAAC;QACH,MAAM,EAAE,mBAAmB,EAAE,GAC3B,MAAM,MAAM,CAAC,+BAA+B,CAAC,CAAC;QAChD,MAAM,UAAU,GAAG,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;QACjD,OAAO,MAAM,mBAAmB,CAAC,aAAa,CAAC,UAAU,GAAG,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;IACzE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,SAAS,iBAAiB;IACxB,OAAO;QACL,MAAM,EAAE,EAA2B;QACnC,KAAK,EAAE,OAAO;QACd,KAAK,CAAC,QAAQ;YACZ,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;QAClE,CAAC;QACD,cAAc;YACZ,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;QACjE,CAAC;KACF,CAAC;AACJ,CAAC","sourcesContent":["/**\n * The evals runner: discover `*.eval.ts` / `evals/*.ts` files, run each eval\n * through its scorer pipeline against the *real* agent loop, score, and report.\n *\n * It is the engine behind `agent-native eval` — when used as a CI deploy gate\n * the CLI exits non-zero if any eval scores below its threshold.\n *\n * Two layers:\n *   - `scoreEval` / `runEvals` — pure orchestration over an `AgentRunner` and\n *     a list of evals. Fully unit-testable with an injected runner (no model).\n *   - `discoverEvalFiles` / `loadEvals` — filesystem discovery + dynamic import\n *     of author-written eval modules.\n *\n * Results are also (best-effort) written to the observability eval store so a\n * dashboard can surface CI eval history next to production run evals.\n */\n\nimport nodePath from \"node:path\";\nimport { pathToFileURL } from \"node:url\";\n\nimport type { ActionEntry } from \"../agent/production-agent.js\";\nimport { insertEvalResult } from \"../observability/store.js\";\nimport type { EvalResult as ObservabilityEvalResult } from \"../observability/types.js\";\nimport type { AgentRunner } from \"./agent-runner.js\";\nimport { createAgentRunner } from \"./agent-runner.js\";\nimport { DEFAULT_EVAL_THRESHOLD } from \"./define-eval.js\";\nimport { clamp01 } from \"./scorer.js\";\nimport type {\n  AgentRunOutput,\n  Eval,\n  EvalResultRow,\n  EvalRunReport,\n  ScorerResult,\n} from \"./types.js\";\n\n// ─── Scoring orchestration ────────────────────────────────────────────\n\n/** Run one scorer's pipeline (preprocess → analyze → score → reason). */\nasync function runScorer(\n  scorer: Eval[\"scorers\"][number],\n  run: AgentRunOutput,\n  runner: AgentRunner,\n  threshold: number,\n): Promise<ScorerResult> {\n  try {\n    const pre = scorer.preprocess ? await scorer.preprocess(run) : run;\n    const analysis = scorer.analyze\n      ? await scorer.analyze(pre as never, runner.analyzeContext())\n      : pre;\n    const rawScore = await scorer.generateScore(analysis as never);\n    const score = clamp01(rawScore);\n    const reason = scorer.generateReason\n      ? await scorer.generateReason({\n          run,\n          analysis: analysis as never,\n          score,\n        })\n      : undefined;\n    return { scorer: scorer.name, score, reason, passed: score >= threshold };\n  } catch (err) {\n    // A scorer that throws is a failed scorer, not a crashed run — degrade\n    // gracefully so one bad scorer can't take down the whole CI gate.\n    return {\n      scorer: scorer.name,\n      score: 0,\n      reason: `Scorer errored: ${err instanceof Error ? err.message : String(err)}`,\n      passed: false,\n    };\n  }\n}\n\n/** Run a single eval: invoke the agent, then score with each scorer. */\nexport async function scoreEval(\n  evalCase: Eval,\n  runner: AgentRunner,\n  opts: { thresholdOverride?: number } = {},\n): Promise<EvalResultRow> {\n  const threshold =\n    opts.thresholdOverride ?? evalCase.threshold ?? DEFAULT_EVAL_THRESHOLD;\n\n  if (evalCase.skipReason) {\n    return {\n      eval: evalCase.name,\n      threshold,\n      scores: [],\n      status: \"skipped\",\n      skipReason: evalCase.skipReason,\n      // Keep the legacy boolean gate friendly: skipped rows do not fail CI.\n      passed: true,\n      avgScore: 0,\n      durationMs: 0,\n    };\n  }\n\n  let run: AgentRunOutput;\n  if (evalCase.run) {\n    run = await evalCase.run({\n      input: evalCase.input,\n      runAgent: (input) => runner.runAgent(input),\n    });\n  } else {\n    run = await runner.runAgent(evalCase.input);\n  }\n\n  const scores: ScorerResult[] = [];\n  for (const scorer of evalCase.scorers) {\n    scores.push(await runScorer(scorer, run, runner, threshold));\n  }\n\n  const avgScore =\n    scores.length > 0\n      ? scores.reduce((s, r) => s + r.score, 0) / scores.length\n      : 0;\n\n  return {\n    eval: evalCase.name,\n    threshold,\n    scores,\n    // A run that errored, or any sub-threshold scorer, fails the case.\n    passed: run.ok && scores.every((s) => s.passed),\n    status: run.ok && scores.every((s) => s.passed) ? \"passed\" : \"failed\",\n    avgScore,\n    durationMs: run.durationMs,\n    error: run.ok ? undefined : run.error,\n  };\n}\n\n/** Run a batch of evals against one runner and aggregate a report. */\nexport async function runEvals(\n  evals: Eval[],\n  runner: AgentRunner,\n  opts: { thresholdOverride?: number; persist?: boolean } = {},\n): Promise<EvalRunReport> {\n  const results: EvalResultRow[] = [];\n  for (const evalCase of evals) {\n    const row = await scoreEval(evalCase, runner, opts);\n    results.push(row);\n    if (opts.persist && row.status !== \"skipped\") {\n      await persistEvalRow(row).catch(() => {});\n    }\n  }\n\n  const passed = results.filter((r) => r.passed).length;\n  const skipped = results.filter((r) => r.status === \"skipped\").length;\n  return {\n    total: results.length,\n    passed,\n    failed: results.filter((r) => r.status !== \"skipped\" && !r.passed).length,\n    skipped,\n    results,\n  };\n}\n\n/**\n * Best-effort write of one eval result to the observability eval store so a\n * dashboard can show CI eval history alongside production run evals. We write\n * one row per (eval × scorer), tagged `evalType: \"automated\"` with a synthetic\n * `eval:` run id.\n *\n * TODO(live-sampling): the same scorer list should also run on a sampled\n * fraction of *real* production runs. That hook belongs in the agent loop's\n * (not-yet-added) post-run processor seam: when a run finishes, roll the\n * configured sample rate and, if it hits, replay the run output through these\n * scorers and write the rows here. Wiring it now would require the in-loop\n * processor seam another wave is adding — so this is the single intended\n * attachment point, intentionally left as a note.\n */\nasync function persistEvalRow(row: EvalResultRow): Promise<void> {\n  const runId = `eval:${row.eval}:${Date.now()}`;\n  for (const s of row.scores) {\n    const result: ObservabilityEvalResult = {\n      id: crypto.randomUUID(),\n      runId,\n      threadId: null,\n      userId: null,\n      evalType: \"automated\",\n      criteria: `eval:${row.eval}:${s.scorer}`,\n      score: s.score,\n      reasoning: s.reason ?? null,\n      metadata: {\n        source: \"cli-eval\",\n        threshold: row.threshold,\n        passed: s.passed,\n      },\n      createdAt: Date.now(),\n    };\n    await insertEvalResult(result);\n  }\n}\n\n// ─── Discovery + loading ──────────────────────────────────────────────\n\nconst EVAL_FILE_RE = /\\.eval\\.(ts|js|mjs)$/;\nconst SKIP_DIRS = new Set([\"node_modules\", \"dist\", \".git\", \".output\", \"build\"]);\n\n/**\n * Walk `root` for eval files. Matches two conventions:\n *   - any `**\\/*.eval.ts` (co-located with code), and\n *   - any `*.ts` directly inside an `evals/` directory.\n * `pattern` further filters by substring of the relative path.\n */\nexport async function discoverEvalFiles(\n  root: string,\n  pattern?: string,\n): Promise<string[]> {\n  const fs = await import(\"node:fs\");\n  const out: string[] = [];\n\n  function isEvalFile(full: string, parentName: string): boolean {\n    const base = nodePath.basename(full);\n    if (EVAL_FILE_RE.test(base)) return true;\n    if (parentName === \"evals\" && /\\.(ts|js|mjs)$/.test(base)) {\n      // Skip obvious support files inside evals/.\n      return !/\\.(spec|test|d)\\.(ts|js|mjs)$/.test(base);\n    }\n    return false;\n  }\n\n  function walk(dir: string, parentName: string): void {\n    let entries: import(\"node:fs\").Dirent[];\n    try {\n      entries = fs.readdirSync(dir, { withFileTypes: true });\n    } catch {\n      return;\n    }\n    for (const entry of entries) {\n      const full = nodePath.join(dir, entry.name);\n      if (entry.isDirectory()) {\n        if (SKIP_DIRS.has(entry.name) || entry.name.startsWith(\".\")) continue;\n        walk(full, entry.name);\n      } else if (entry.isFile() && isEvalFile(full, parentName)) {\n        out.push(full);\n      }\n    }\n  }\n\n  walk(root, nodePath.basename(root));\n  out.sort();\n\n  if (!pattern) return out;\n  return out.filter((f) => nodePath.relative(root, f).includes(pattern));\n}\n\n/** Pull `Eval` definitions out of a dynamically-imported eval module. */\nfunction extractEvals(mod: Record<string, unknown>): Eval[] {\n  const candidates: unknown[] = [];\n  if (mod.default !== undefined) candidates.push(mod.default);\n  for (const [key, value] of Object.entries(mod)) {\n    if (key === \"default\") continue;\n    candidates.push(value);\n  }\n\n  const evals: Eval[] = [];\n  for (const c of candidates.flat()) {\n    if (\n      c &&\n      typeof c === \"object\" &&\n      typeof (c as Eval).name === \"string\" &&\n      Array.isArray((c as Eval).scorers) &&\n      (c as Eval).input\n    ) {\n      evals.push(c as Eval);\n    }\n  }\n  return evals;\n}\n\n/** Discover and import all eval files under `root`, returning their evals. */\nexport async function loadEvals(\n  root: string,\n  pattern?: string,\n): Promise<{ files: string[]; evals: Eval[] }> {\n  const files = await discoverEvalFiles(root, pattern);\n  const evals: Eval[] = [];\n  for (const file of files) {\n    const mod = (await import(pathToFileURL(file).href)) as Record<\n      string,\n      unknown\n    >;\n    evals.push(...extractEvals(mod));\n  }\n  return { files, evals };\n}\n\n// ─── High-level entry used by the CLI ─────────────────────────────────\n\nexport interface RunEvalSuiteOptions {\n  /** App root to discover eval files + actions under. Defaults to cwd. */\n  cwd?: string;\n  /** Substring filter on the eval file path. */\n  pattern?: string;\n  /** Global threshold override (wins over per-eval thresholds). */\n  thresholdOverride?: number;\n  /** App actions to expose to the agent. Auto-discovered when omitted. */\n  actions?: Record<string, ActionEntry>;\n  /** System prompt for runs. */\n  systemPrompt?: string;\n  /** Write results to the observability eval store (default true). */\n  persist?: boolean;\n  /** Pre-built runner (tests inject this to avoid touching engine/loop). */\n  runner?: AgentRunner;\n  /** Pre-loaded evals (tests inject this to skip filesystem discovery). */\n  evals?: Eval[];\n}\n\n/**\n * End-to-end: load evals, build a runner, score, report. The CLI wraps this\n * and maps `report.failed > 0` to a non-zero exit code (the CI gate).\n */\nexport async function runEvalSuite(\n  opts: RunEvalSuiteOptions = {},\n): Promise<{ report: EvalRunReport; files: string[] }> {\n  const cwd = opts.cwd ?? process.cwd();\n\n  let files: string[] = [];\n  let evals = opts.evals;\n  if (!evals) {\n    const loaded = await loadEvals(cwd, opts.pattern);\n    files = loaded.files;\n    evals = loaded.evals;\n  }\n\n  const needsRunner = evals.some((evalCase) => !evalCase.skipReason);\n  const runner =\n    opts.runner ??\n    (needsRunner\n      ? await createAgentRunner({\n          actions: opts.actions ?? (await discoverActions(cwd)),\n          systemPrompt: opts.systemPrompt,\n        })\n      : createInertRunner());\n\n  const report = await runEvals(evals, runner, {\n    thresholdOverride: opts.thresholdOverride,\n    persist: opts.persist ?? true,\n  });\n  return { report, files };\n}\n\n/**\n * Discover the app's actions so the agent under test has the real tool\n * surface. Lazy-imports `autoDiscoverActions` to keep server-only deps out of\n * any browser bundle that might touch this module's types.\n */\nasync function discoverActions(\n  cwd: string,\n): Promise<Record<string, ActionEntry>> {\n  try {\n    const { autoDiscoverActions } =\n      await import(\"../server/action-discovery.js\");\n    const actionsDir = nodePath.join(cwd, \"actions\");\n    return await autoDiscoverActions(pathToFileURL(actionsDir + \"/\").href);\n  } catch {\n    return {};\n  }\n}\n\nfunction createInertRunner(): AgentRunner {\n  return {\n    engine: {} as AgentRunner[\"engine\"],\n    model: \"inert\",\n    async runAgent() {\n      throw new Error(\"Eval unexpectedly requested the agent runner\");\n    },\n    analyzeContext() {\n      throw new Error(\"Eval unexpectedly requested analyze context\");\n    },\n  };\n}\n"]}

package/dist/eval/types.d.ts

@@ -122,6 +122,12 @@ export interface EvalRunContext {
 export interface Eval {
     name: string;
     input: EvalInput;
+    /**
+     * Skip this case without running the agent or scorers. Use for opt-in
+     * suites that need secrets, live provider credentials, or manual/nightly
+     * gates. Skipped rows are reported distinctly and do not fail the suite.
+     */
+    skipReason?: string;
     /**
      * Optional override for how the agent is run for this case. Defaults to the
      * runner's headless `runAgent`. Use this to do custom setup (seed data,
@@ -144,6 +150,10 @@ export interface EvalResultRow {
     eval: string;
     threshold: number;
     scores: ScorerResult[];
+    /** Explicit row status for reports. Older consumers can still read `passed`. */
+    status?: "passed" | "failed" | "skipped";
+    /** Human-readable reason when `status` is `skipped`. */
+    skipReason?: string;
     /** True only when every scorer passed. */
     passed: boolean;
     /** Mean of the scorer scores, for at-a-glance ranking. */
@@ -157,6 +167,7 @@ export interface EvalRunReport {
     total: number;
     passed: number;
     failed: number;
+    skipped?: number;
     results: EvalResultRow[];
 }
 //# sourceMappingURL=types.d.ts.map

package/dist/eval/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/eval/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAI5D;;;;GAIG;AACH,MAAM,WAAW,cAAc;IAC7B,0DAA0D;IAC1D,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,+DAA+D;IAC/D,QAAQ,CAAC,SAAS,EAAE,SAAS,MAAM,EAAE,CAAC;IACtC,gEAAgE;IAChE,QAAQ,CAAC,EAAE,EAAE,OAAO,CAAC;IACrB,kDAAkD;IAClD,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,iFAAiF;IACjF,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,sDAAsD;IACtD,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC7B;AAID;;;;;;GAMG;AACH,MAAM,WAAW,oBAAoB;IACnC,oEAAoE;IACpE,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC;IAC7B,gDAAgD;IAChD,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB;;;;OAIG;IACH,KAAK,CAAC,IAAI,EAAE;QACV,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,MAAM,EAAE,MAAM,CAAC;QACf,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,MAAM,CAAC,EAAE,WAAW,CAAC;KACtB,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CACrB;AAED;;;;;;;;;;GAUG;AACH,MAAM,WAAW,MAAM,CAAC,GAAG,GAAG,cAAc,EAAE,GAAG,GAAG,GAAG;IACrD,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,CAAC,GAAG,EAAE,cAAc,GAAG,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;IACrD,OAAO,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,GAAG,EAAE,oBAAoB,GAAG,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;IACpE,sDAAsD;IACtD,aAAa,CAAC,QAAQ,EAAE,GAAG,GAAG,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACvD,cAAc,CAAC,CAAC,IAAI,EAAE;QACpB,GAAG,EAAE,cAAc,CAAC;QACpB,QAAQ,EAAE,GAAG,CAAC;QACd,KAAK,EAAE,MAAM,CAAC;KACf,GAAG,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CAC9B;AAED,kDAAkD;AAClD,MAAM,WAAW,gBAAgB,CAAC,GAAG,GAAG,cAAc,EAAE,GAAG,GAAG,GAAG;IAC/D,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,CAAC,CAAC,GAAG,EAAE,cAAc,GAAG,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;IACrD,OAAO,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,GAAG,EAAE,oBAAoB,GAAG,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;IACpE,aAAa,CAAC,QAAQ,EAAE,GAAG,GAAG,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACvD,cAAc,CAAC,CAAC,IAAI,EAAE;QACpB,GAAG,EAAE,cAAc,CAAC;QACpB,QAAQ,EAAE,GAAG,CAAC;QACd,KAAK,EAAE,MAAM,CAAC;KACf,GAAG,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CAC9B;AAID,6DAA6D;AAC7D,MAAM,WAAW,SAAS;IACxB,mDAAmD;IACnD,MAAM,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,OAAO,CAAC,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,GAAG,WAAW,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CAC/D;AAED,2DAA2D;AAC3D,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,KAAK,EAAE,SAAS,CAAC;IAC1B,8EAA8E;IAC9E,QAAQ,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;CACrD;AAED;;;;GAIG;AACH,MAAM,WAAW,IAAI;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,SAAS,CAAC;IACjB;;;;OAIG;IACH,GAAG,CAAC,CAAC,GAAG,EAAE,cAAc,GAAG,cAAc,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;IACpE,OAAO,EAAE,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,CAAC;IAC5B,oEAAoE;IACpE,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAID,iFAAiF;AACjF,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,OAAO,CAAC;CACjB;AAED,gEAAgE;AAChE,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,YAAY,EAAE,CAAC;IACvB,0CAA0C;IAC1C,MAAM,EAAE,OAAO,CAAC;IAChB,0DAA0D;IAC1D,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,qDAAqD;IACrD,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,8BAA8B;AAC9B,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,aAAa,EAAE,CAAC;CAC1B"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/eval/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAI5D;;;;GAIG;AACH,MAAM,WAAW,cAAc;IAC7B,0DAA0D;IAC1D,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,+DAA+D;IAC/D,QAAQ,CAAC,SAAS,EAAE,SAAS,MAAM,EAAE,CAAC;IACtC,gEAAgE;IAChE,QAAQ,CAAC,EAAE,EAAE,OAAO,CAAC;IACrB,kDAAkD;IAClD,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,iFAAiF;IACjF,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,sDAAsD;IACtD,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC7B;AAID;;;;;;GAMG;AACH,MAAM,WAAW,oBAAoB;IACnC,oEAAoE;IACpE,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC;IAC7B,gDAAgD;IAChD,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB;;;;OAIG;IACH,KAAK,CAAC,IAAI,EAAE;QACV,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,MAAM,EAAE,MAAM,CAAC;QACf,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,MAAM,CAAC,EAAE,WAAW,CAAC;KACtB,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CACrB;AAED;;;;;;;;;;GAUG;AACH,MAAM,WAAW,MAAM,CAAC,GAAG,GAAG,cAAc,EAAE,GAAG,GAAG,GAAG;IACrD,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,CAAC,GAAG,EAAE,cAAc,GAAG,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;IACrD,OAAO,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,GAAG,EAAE,oBAAoB,GAAG,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;IACpE,sDAAsD;IACtD,aAAa,CAAC,QAAQ,EAAE,GAAG,GAAG,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACvD,cAAc,CAAC,CAAC,IAAI,EAAE;QACpB,GAAG,EAAE,cAAc,CAAC;QACpB,QAAQ,EAAE,GAAG,CAAC;QACd,KAAK,EAAE,MAAM,CAAC;KACf,GAAG,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CAC9B;AAED,kDAAkD;AAClD,MAAM,WAAW,gBAAgB,CAAC,GAAG,GAAG,cAAc,EAAE,GAAG,GAAG,GAAG;IAC/D,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,CAAC,CAAC,GAAG,EAAE,cAAc,GAAG,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;IACrD,OAAO,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,GAAG,EAAE,oBAAoB,GAAG,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;IACpE,aAAa,CAAC,QAAQ,EAAE,GAAG,GAAG,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACvD,cAAc,CAAC,CAAC,IAAI,EAAE;QACpB,GAAG,EAAE,cAAc,CAAC;QACpB,QAAQ,EAAE,GAAG,CAAC;QACd,KAAK,EAAE,MAAM,CAAC;KACf,GAAG,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CAC9B;AAID,6DAA6D;AAC7D,MAAM,WAAW,SAAS;IACxB,mDAAmD;IACnD,MAAM,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,OAAO,CAAC,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,GAAG,WAAW,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CAC/D;AAED,2DAA2D;AAC3D,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,KAAK,EAAE,SAAS,CAAC;IAC1B,8EAA8E;IAC9E,QAAQ,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;CACrD;AAED;;;;GAIG;AACH,MAAM,WAAW,IAAI;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,SAAS,CAAC;IACjB;;;;OAIG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;;OAIG;IACH,GAAG,CAAC,CAAC,GAAG,EAAE,cAAc,GAAG,cAAc,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;IACpE,OAAO,EAAE,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,CAAC;IAC5B,oEAAoE;IACpE,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAID,iFAAiF;AACjF,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,OAAO,CAAC;CACjB;AAED,gEAAgE;AAChE,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,YAAY,EAAE,CAAC;IACvB,gFAAgF;IAChF,MAAM,CAAC,EAAE,QAAQ,GAAG,QAAQ,GAAG,SAAS,CAAC;IACzC,wDAAwD;IACxD,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,0CAA0C;IAC1C,MAAM,EAAE,OAAO,CAAC;IAChB,0DAA0D;IAC1D,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,qDAAqD;IACrD,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,8BAA8B;AAC9B,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,aAAa,EAAE,CAAC;CAC1B"}

package/dist/eval/types.js.map

@@ -1 +1 @@
-{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/eval/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG","sourcesContent":["/**\n * Types for the first-class evals primitive.\n *\n * This is a *test-case* eval system (define a prompt + expected behavior,\n * actually run the agent, score the output) — distinct from the post-hoc\n * run-scoring engine in `../observability/evals.ts`, which scores already-\n * completed production runs. The two are complementary:\n *\n *   - `observability/evals.ts` — \"how did this real run do?\" (passive,\n *     sampled, lives next to traces).\n *   - `eval/*` (this module) — \"does the agent do the right thing on this\n *     fixed input?\" (active, deterministic CI gate, run via the CLI).\n *\n * The pipeline shape (preprocess → analyze → generateScore → generateReason)\n * is borrowed from Mastra's scorer design: each scorer is a small, composable\n * 4-step pipeline so a single scorer can mix plain-JS checks with an LLM\n * judge while still producing one normalized 0..1 number plus a reason.\n */\n\nimport type { AgentEngine } from \"../agent/engine/types.js\";\n\n// ─── Agent run output ─────────────────────────────────────────────────\n\n/**\n * The result of actually running the agent loop for one eval input. Scorers\n * receive this as the thing under test. It is intentionally small and\n * transport-agnostic so a scorer never reaches into framework internals.\n */\nexport interface AgentRunOutput {\n  /** Concatenated assistant text emitted across the run. */\n  readonly text: string;\n  /** Names of tools/actions the agent invoked, in call order. */\n  readonly toolCalls: readonly string[];\n  /** Whether the run completed without a terminal error event. */\n  readonly ok: boolean;\n  /** Terminal error message, if the run errored. */\n  readonly error?: string;\n  /** Synthetic run id, useful for writing eval rows to the observability store. */\n  readonly runId: string;\n  /** Wall-clock duration of the run in milliseconds. */\n  readonly durationMs: number;\n}\n\n// ─── Scorer pipeline ──────────────────────────────────────────────────\n\n/**\n * Context handed to a scorer's analyze step when it needs an LLM judge. The\n * engine/model are resolved by the runner from the existing engine registry —\n * a scorer NEVER hardcodes a provider or model, keeping evals provider-\n * agnostic. `judge()` is a convenience that streams a single judging turn and\n * returns the raw model text.\n */\nexport interface ScorerAnalyzeContext {\n  /** The resolved, provider-agnostic engine for LLM-judge scorers. */\n  readonly engine: AgentEngine;\n  /** The resolved model string for the engine. */\n  readonly model: string;\n  /**\n   * Run a single LLM judging turn. Returns the model's raw text output. Used\n   * by `llmJudge` and any custom LLM-backed analyze step. Provider-agnostic —\n   * the engine is whatever the app/CLI resolved.\n   */\n  judge(opts: {\n    systemPrompt?: string;\n    prompt: string;\n    maxOutputTokens?: number;\n    signal?: AbortSignal;\n  }): Promise<string>;\n}\n\n/**\n * A 4-step scoring pipeline (Mastra-style):\n *\n *   preprocess(run)      → x        (transform the run/output; optional)\n *   analyze(x, ctx)      → analysis (plain JS OR an LLM judge; optional)\n *   generateScore(a)     → 0..1     (REQUIRED, normalized)\n *   generateReason(...)  → string   (human-readable why; optional)\n *\n * Generics flow `Pre` (preprocess output) → `Ana` (analyze output) so a\n * single scorer is fully typed end-to-end.\n */\nexport interface Scorer<Pre = AgentRunOutput, Ana = Pre> {\n  readonly name: string;\n  preprocess?(run: AgentRunOutput): Pre | Promise<Pre>;\n  analyze?(input: Pre, ctx: ScorerAnalyzeContext): Ana | Promise<Ana>;\n  /** REQUIRED. Returns a normalized score in [0, 1]. */\n  generateScore(analysis: Ana): number | Promise<number>;\n  generateReason?(args: {\n    run: AgentRunOutput;\n    analysis: Ana;\n    score: number;\n  }): string | Promise<string>;\n}\n\n/** Definition object passed to `createScorer`. */\nexport interface ScorerDefinition<Pre = AgentRunOutput, Ana = Pre> {\n  name: string;\n  preprocess?(run: AgentRunOutput): Pre | Promise<Pre>;\n  analyze?(input: Pre, ctx: ScorerAnalyzeContext): Ana | Promise<Ana>;\n  generateScore(analysis: Ana): number | Promise<number>;\n  generateReason?(args: {\n    run: AgentRunOutput;\n    analysis: Ana;\n    score: number;\n  }): string | Promise<string>;\n}\n\n// ─── Eval definition ──────────────────────────────────────────────────\n\n/** The prompt + optional setup that drives one eval case. */\nexport interface EvalInput {\n  /** The user prompt / message sent to the agent. */\n  prompt: string;\n  /**\n   * Optional prior conversation turns to seed before `prompt`. Each is a\n   * plain { role, text } pair; the runner converts them to engine messages.\n   */\n  history?: Array<{ role: \"user\" | \"assistant\"; text: string }>;\n}\n\n/** Context passed to an eval's optional `run` override. */\nexport interface EvalRunContext {\n  readonly input: EvalInput;\n  /** The default agent runner — invoke it to run the agent loop as a caller. */\n  runAgent(input: EvalInput): Promise<AgentRunOutput>;\n}\n\n/**\n * A named eval = one test case. `scorers` produce per-scorer rows; the case\n * passes when EVERY scorer meets `threshold` (default 0.5, overridable per\n * eval and globally from the CLI).\n */\nexport interface Eval {\n  name: string;\n  input: EvalInput;\n  /**\n   * Optional override for how the agent is run for this case. Defaults to the\n   * runner's headless `runAgent`. Use this to do custom setup (seed data,\n   * multi-turn) before/after the agent call.\n   */\n  run?(ctx: EvalRunContext): AgentRunOutput | Promise<AgentRunOutput>;\n  scorers: Scorer<any, any>[];\n  /** Minimum acceptable score (per scorer) in [0, 1]. Default 0.5. */\n  threshold?: number;\n}\n\n// ─── Results ──────────────────────────────────────────────────────────\n\n/** One result row per (eval × scorer). Stores both the number AND the reason. */\nexport interface ScorerResult {\n  scorer: string;\n  score: number;\n  reason?: string;\n  passed: boolean;\n}\n\n/** Aggregated result for a single eval (all of its scorers). */\nexport interface EvalResultRow {\n  eval: string;\n  threshold: number;\n  scores: ScorerResult[];\n  /** True only when every scorer passed. */\n  passed: boolean;\n  /** Mean of the scorer scores, for at-a-glance ranking. */\n  avgScore: number;\n  durationMs: number;\n  /** Terminal error if the agent run itself failed. */\n  error?: string;\n}\n\n/** The full runner report. */\nexport interface EvalRunReport {\n  total: number;\n  passed: number;\n  failed: number;\n  results: EvalResultRow[];\n}\n"]}
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/eval/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG","sourcesContent":["/**\n * Types for the first-class evals primitive.\n *\n * This is a *test-case* eval system (define a prompt + expected behavior,\n * actually run the agent, score the output) — distinct from the post-hoc\n * run-scoring engine in `../observability/evals.ts`, which scores already-\n * completed production runs. The two are complementary:\n *\n *   - `observability/evals.ts` — \"how did this real run do?\" (passive,\n *     sampled, lives next to traces).\n *   - `eval/*` (this module) — \"does the agent do the right thing on this\n *     fixed input?\" (active, deterministic CI gate, run via the CLI).\n *\n * The pipeline shape (preprocess → analyze → generateScore → generateReason)\n * is borrowed from Mastra's scorer design: each scorer is a small, composable\n * 4-step pipeline so a single scorer can mix plain-JS checks with an LLM\n * judge while still producing one normalized 0..1 number plus a reason.\n */\n\nimport type { AgentEngine } from \"../agent/engine/types.js\";\n\n// ─── Agent run output ─────────────────────────────────────────────────\n\n/**\n * The result of actually running the agent loop for one eval input. Scorers\n * receive this as the thing under test. It is intentionally small and\n * transport-agnostic so a scorer never reaches into framework internals.\n */\nexport interface AgentRunOutput {\n  /** Concatenated assistant text emitted across the run. */\n  readonly text: string;\n  /** Names of tools/actions the agent invoked, in call order. */\n  readonly toolCalls: readonly string[];\n  /** Whether the run completed without a terminal error event. */\n  readonly ok: boolean;\n  /** Terminal error message, if the run errored. */\n  readonly error?: string;\n  /** Synthetic run id, useful for writing eval rows to the observability store. */\n  readonly runId: string;\n  /** Wall-clock duration of the run in milliseconds. */\n  readonly durationMs: number;\n}\n\n// ─── Scorer pipeline ──────────────────────────────────────────────────\n\n/**\n * Context handed to a scorer's analyze step when it needs an LLM judge. The\n * engine/model are resolved by the runner from the existing engine registry —\n * a scorer NEVER hardcodes a provider or model, keeping evals provider-\n * agnostic. `judge()` is a convenience that streams a single judging turn and\n * returns the raw model text.\n */\nexport interface ScorerAnalyzeContext {\n  /** The resolved, provider-agnostic engine for LLM-judge scorers. */\n  readonly engine: AgentEngine;\n  /** The resolved model string for the engine. */\n  readonly model: string;\n  /**\n   * Run a single LLM judging turn. Returns the model's raw text output. Used\n   * by `llmJudge` and any custom LLM-backed analyze step. Provider-agnostic —\n   * the engine is whatever the app/CLI resolved.\n   */\n  judge(opts: {\n    systemPrompt?: string;\n    prompt: string;\n    maxOutputTokens?: number;\n    signal?: AbortSignal;\n  }): Promise<string>;\n}\n\n/**\n * A 4-step scoring pipeline (Mastra-style):\n *\n *   preprocess(run)      → x        (transform the run/output; optional)\n *   analyze(x, ctx)      → analysis (plain JS OR an LLM judge; optional)\n *   generateScore(a)     → 0..1     (REQUIRED, normalized)\n *   generateReason(...)  → string   (human-readable why; optional)\n *\n * Generics flow `Pre` (preprocess output) → `Ana` (analyze output) so a\n * single scorer is fully typed end-to-end.\n */\nexport interface Scorer<Pre = AgentRunOutput, Ana = Pre> {\n  readonly name: string;\n  preprocess?(run: AgentRunOutput): Pre | Promise<Pre>;\n  analyze?(input: Pre, ctx: ScorerAnalyzeContext): Ana | Promise<Ana>;\n  /** REQUIRED. Returns a normalized score in [0, 1]. */\n  generateScore(analysis: Ana): number | Promise<number>;\n  generateReason?(args: {\n    run: AgentRunOutput;\n    analysis: Ana;\n    score: number;\n  }): string | Promise<string>;\n}\n\n/** Definition object passed to `createScorer`. */\nexport interface ScorerDefinition<Pre = AgentRunOutput, Ana = Pre> {\n  name: string;\n  preprocess?(run: AgentRunOutput): Pre | Promise<Pre>;\n  analyze?(input: Pre, ctx: ScorerAnalyzeContext): Ana | Promise<Ana>;\n  generateScore(analysis: Ana): number | Promise<number>;\n  generateReason?(args: {\n    run: AgentRunOutput;\n    analysis: Ana;\n    score: number;\n  }): string | Promise<string>;\n}\n\n// ─── Eval definition ──────────────────────────────────────────────────\n\n/** The prompt + optional setup that drives one eval case. */\nexport interface EvalInput {\n  /** The user prompt / message sent to the agent. */\n  prompt: string;\n  /**\n   * Optional prior conversation turns to seed before `prompt`. Each is a\n   * plain { role, text } pair; the runner converts them to engine messages.\n   */\n  history?: Array<{ role: \"user\" | \"assistant\"; text: string }>;\n}\n\n/** Context passed to an eval's optional `run` override. */\nexport interface EvalRunContext {\n  readonly input: EvalInput;\n  /** The default agent runner — invoke it to run the agent loop as a caller. */\n  runAgent(input: EvalInput): Promise<AgentRunOutput>;\n}\n\n/**\n * A named eval = one test case. `scorers` produce per-scorer rows; the case\n * passes when EVERY scorer meets `threshold` (default 0.5, overridable per\n * eval and globally from the CLI).\n */\nexport interface Eval {\n  name: string;\n  input: EvalInput;\n  /**\n   * Skip this case without running the agent or scorers. Use for opt-in\n   * suites that need secrets, live provider credentials, or manual/nightly\n   * gates. Skipped rows are reported distinctly and do not fail the suite.\n   */\n  skipReason?: string;\n  /**\n   * Optional override for how the agent is run for this case. Defaults to the\n   * runner's headless `runAgent`. Use this to do custom setup (seed data,\n   * multi-turn) before/after the agent call.\n   */\n  run?(ctx: EvalRunContext): AgentRunOutput | Promise<AgentRunOutput>;\n  scorers: Scorer<any, any>[];\n  /** Minimum acceptable score (per scorer) in [0, 1]. Default 0.5. */\n  threshold?: number;\n}\n\n// ─── Results ──────────────────────────────────────────────────────────\n\n/** One result row per (eval × scorer). Stores both the number AND the reason. */\nexport interface ScorerResult {\n  scorer: string;\n  score: number;\n  reason?: string;\n  passed: boolean;\n}\n\n/** Aggregated result for a single eval (all of its scorers). */\nexport interface EvalResultRow {\n  eval: string;\n  threshold: number;\n  scores: ScorerResult[];\n  /** Explicit row status for reports. Older consumers can still read `passed`. */\n  status?: \"passed\" | \"failed\" | \"skipped\";\n  /** Human-readable reason when `status` is `skipped`. */\n  skipReason?: string;\n  /** True only when every scorer passed. */\n  passed: boolean;\n  /** Mean of the scorer scores, for at-a-glance ranking. */\n  avgScore: number;\n  durationMs: number;\n  /** Terminal error if the agent run itself failed. */\n  error?: string;\n}\n\n/** The full runner report. */\nexport interface EvalRunReport {\n  total: number;\n  passed: number;\n  failed: number;\n  skipped?: number;\n  results: EvalResultRow[];\n}\n"]}

package/dist/notifications/routes.d.ts

@@ -11,14 +11,14 @@
  *   DELETE /_agent-native/notifications/:id         — delete
  */
 export declare function createNotificationsHandler(): import("h3").EventHandlerWithFetch<import("h3").EventHandlerRequest, Promise<"" | import("./types.js").Notification[] | {
+    error?: undefined;
     count: number;
     updated?: undefined;
-    error?: undefined;
     ok?: undefined;
 } | {
+    error?: undefined;
     count?: undefined;
     updated: number;
-    error?: undefined;
     ok?: undefined;
 } | {
     count?: undefined;
@@ -26,9 +26,9 @@ export declare function createNotificationsHandler(): import("h3").EventHandlerW
     error: string;
     ok?: undefined;
 } | {
+    error?: undefined;
     count?: undefined;
     updated?: undefined;
-    error?: undefined;
     ok: boolean;
 }>>;
 //# sourceMappingURL=routes.d.ts.map