@agent-native/core 0.84.0 → 0.84.2

package/dist/eval/runner.js

@@ -53,6 +53,19 @@ async function runScorer(scorer, run, runner, threshold) {
 /** Run a single eval: invoke the agent, then score with each scorer. */
 export async function scoreEval(evalCase, runner, opts = {}) {
     const threshold = opts.thresholdOverride ?? evalCase.threshold ?? DEFAULT_EVAL_THRESHOLD;
+    if (evalCase.skipReason) {
+        return {
+            eval: evalCase.name,
+            threshold,
+            scores: [],
+            status: "skipped",
+            skipReason: evalCase.skipReason,
+            // Keep the legacy boolean gate friendly: skipped rows do not fail CI.
+            passed: true,
+            avgScore: 0,
+            durationMs: 0,
+        };
+    }
     let run;
     if (evalCase.run) {
         run = await evalCase.run({
@@ -76,6 +89,7 @@ export async function scoreEval(evalCase, runner, opts = {}) {
         scores,
         // A run that errored, or any sub-threshold scorer, fails the case.
         passed: run.ok && scores.every((s) => s.passed),
+        status: run.ok && scores.every((s) => s.passed) ? "passed" : "failed",
         avgScore,
         durationMs: run.durationMs,
         error: run.ok ? undefined : run.error,
@@ -87,14 +101,17 @@ export async function runEvals(evals, runner, opts = {}) {
     for (const evalCase of evals) {
         const row = await scoreEval(evalCase, runner, opts);
         results.push(row);
-        if (opts.persist)
+        if (opts.persist && row.status !== "skipped") {
             await persistEvalRow(row).catch(() => { });
+        }
     }
     const passed = results.filter((r) => r.passed).length;
+    const skipped = results.filter((r) => r.status === "skipped").length;
     return {
         total: results.length,
         passed,
-        failed: results.length - passed,
+        failed: results.filter((r) => r.status !== "skipped" && !r.passed).length,
+        skipped,
         results,
     };
 }
@@ -227,11 +244,14 @@ export async function runEvalSuite(opts = {}) {
         files = loaded.files;
         evals = loaded.evals;
     }
+    const needsRunner = evals.some((evalCase) => !evalCase.skipReason);
     const runner = opts.runner ??
-        (await createAgentRunner({
-            actions: opts.actions ?? (await discoverActions(cwd)),
-            systemPrompt: opts.systemPrompt,
-        }));
+        (needsRunner
+            ? await createAgentRunner({
+                actions: opts.actions ?? (await discoverActions(cwd)),
+                systemPrompt: opts.systemPrompt,
+            })
+            : createInertRunner());
     const report = await runEvals(evals, runner, {
         thresholdOverride: opts.thresholdOverride,
         persist: opts.persist ?? true,
@@ -253,4 +273,16 @@ async function discoverActions(cwd) {
         return {};
     }
 }
+function createInertRunner() {
+    return {
+        engine: {},
+        model: "inert",
+        async runAgent() {
+            throw new Error("Eval unexpectedly requested the agent runner");
+        },
+        analyzeContext() {
+            throw new Error("Eval unexpectedly requested analyze context");
+        },
+    };
+}
 //# sourceMappingURL=runner.js.map

package/dist/eval/runner.js.map

@@ -1 +1 @@
-{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../src/eval/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,QAAQ,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAGzC,OAAO,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAC;AAG7D,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,EAAE,sBAAsB,EAAE,MAAM,kBAAkB,CAAC;AAC1D,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAStC,yEAAyE;AAEzE,yEAAyE;AACzE,KAAK,UAAU,SAAS,CACtB,MAA+B,EAC/B,GAAmB,EACnB,MAAmB,EACnB,SAAiB;IAEjB,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;QACnE,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO;YAC7B,CAAC,CAAC,MAAM,MAAM,CAAC,OAAO,CAAC,GAAY,EAAE,MAAM,CAAC,cAAc,EAAE,CAAC;YAC7D,CAAC,CAAC,GAAG,CAAC;QACR,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,QAAiB,CAAC,CAAC;QAC/D,MAAM,KAAK,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;QAChC,MAAM,MAAM,GAAG,MAAM,CAAC,cAAc;YAClC,CAAC,CAAC,MAAM,MAAM,CAAC,cAAc,CAAC;gBAC1B,GAAG;gBACH,QAAQ,EAAE,QAAiB;gBAC3B,KAAK;aACN,CAAC;YACJ,CAAC,CAAC,SAAS,CAAC;QACd,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,IAAI,SAAS,EAAE,CAAC;IAC5E,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,uEAAuE;QACvE,kEAAkE;QAClE,OAAO;YACL,MAAM,EAAE,MAAM,CAAC,IAAI;YACnB,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,mBAAmB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;YAC7E,MAAM,EAAE,KAAK;SACd,CAAC;IACJ,CAAC;AACH,CAAC;AAED,wEAAwE;AACxE,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,QAAc,EACd,MAAmB,EACnB,IAAI,GAAmC,EAAE;IAEzC,MAAM,SAAS,GACb,IAAI,CAAC,iBAAiB,IAAI,QAAQ,CAAC,SAAS,IAAI,sBAAsB,CAAC;IAEzE,IAAI,GAAmB,CAAC;IACxB,IAAI,QAAQ,CAAC,GAAG,EAAE,CAAC;QACjB,GAAG,GAAG,MAAM,QAAQ,CAAC,GAAG,CAAC;YACvB,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,QAAQ,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;SAC5C,CAAC,CAAC;IACL,CAAC;SAAM,CAAC;QACN,GAAG,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC9C,CAAC;IAED,MAAM,MAAM,GAAmB,EAAE,CAAC;IAClC,KAAK,MAAM,MAAM,IAAI,QAAQ,CAAC,OAAO,EAAE,CAAC;QACtC,MAAM,CAAC,IAAI,CAAC,MAAM,SAAS,CAAC,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC;IAC/D,CAAC;IAED,MAAM,QAAQ,GACZ,MAAM,CAAC,MAAM,GAAG,CAAC;QACf,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM;QACzD,CAAC,CAAC,CAAC,CAAC;IAER,OAAO;QACL,IAAI,EAAE,QAAQ,CAAC,IAAI;QACnB,SAAS;QACT,MAAM;QACN,mEAAmE;QACnE,MAAM,EAAE,GAAG,CAAC,EAAE,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;QAC/C,QAAQ;QACR,UAAU,EAAE,GAAG,CAAC,UAAU;QAC1B,KAAK,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK;KACtC,CAAC;AACJ,CAAC;AAED,sEAAsE;AACtE,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,KAAa,EACb,MAAmB,EACnB,IAAI,GAAsD,EAAE;IAE5D,MAAM,OAAO,GAAoB,EAAE,CAAC;IACpC,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE,CAAC;QAC7B,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;QACpD,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAClB,IAAI,IAAI,CAAC,OAAO;YAAE,MAAM,cAAc,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IAC9D,CAAC;IAED,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IACtD,OAAO;QACL,KAAK,EAAE,OAAO,CAAC,MAAM;QACrB,MAAM;QACN,MAAM,EAAE,OAAO,CAAC,MAAM,GAAG,MAAM;QAC/B,OAAO;KACR,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,KAAK,UAAU,cAAc,CAAC,GAAkB;IAC9C,MAAM,KAAK,GAAG,QAAQ,GAAG,CAAC,IAAI,IAAI,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;IAC/C,KAAK,MAAM,CAAC,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC;QAC3B,MAAM,MAAM,GAA4B;YACtC,EAAE,EAAE,MAAM,CAAC,UAAU,EAAE;YACvB,KAAK;YACL,QAAQ,EAAE,IAAI;YACd,MAAM,EAAE,IAAI;YACZ,QAAQ,EAAE,WAAW;YACrB,QAAQ,EAAE,QAAQ,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,MAAM,EAAE;YACxC,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,SAAS,EAAE,CAAC,CAAC,MAAM,IAAI,IAAI;YAC3B,QAAQ,EAAE;gBACR,MAAM,EAAE,UAAU;gBAClB,SAAS,EAAE,GAAG,CAAC,SAAS;gBACxB,MAAM,EAAE,CAAC,CAAC,MAAM;aACjB;YACD,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;SACtB,CAAC;QACF,MAAM,gBAAgB,CAAC,MAAM,CAAC,CAAC;IACjC,CAAC;AACH,CAAC;AAED,yEAAyE;AAEzE,MAAM,YAAY,GAAG,sBAAsB,CAAC;AAC5C,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,CAAC,cAAc,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC,CAAC;AAEhF;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,IAAY,EACZ,OAAgB;IAEhB,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC;IACnC,MAAM,GAAG,GAAa,EAAE,CAAC;IAEzB,SAAS,UAAU,CAAC,IAAY,EAAE,UAAkB;QAClD,MAAM,IAAI,GAAG,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACrC,IAAI,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,OAAO,IAAI,CAAC;QACzC,IAAI,UAAU,KAAK,OAAO,IAAI,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC1D,4CAA4C;YAC5C,OAAO,CAAC,+BAA+B,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACrD,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,SAAS,IAAI,CAAC,GAAW,EAAE,UAAkB;QAC3C,IAAI,OAAmC,CAAC;QACxC,IAAI,CAAC;YACH,OAAO,GAAG,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;QACzD,CAAC;QAAC,MAAM,CAAC;YACP,OAAO;QACT,CAAC;QACD,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;YAC5C,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;gBACxB,IAAI,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;oBAAE,SAAS;gBACtE,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;YACzB,CAAC;iBAAM,IAAI,KAAK,CAAC,MAAM,EAAE,IAAI,UAAU,CAAC,IAAI,EAAE,UAAU,CAAC,EAAE,CAAC;gBAC1D,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACjB,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;IACpC,GAAG,CAAC,IAAI,EAAE,CAAC;IAEX,IAAI,CAAC,OAAO;QAAE,OAAO,GAAG,CAAC;IACzB,OAAO,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;AACzE,CAAC;AAED,yEAAyE;AACzE,SAAS,YAAY,CAAC,GAA4B;IAChD,MAAM,UAAU,GAAc,EAAE,CAAC;IACjC,IAAI,GAAG,CAAC,OAAO,KAAK,SAAS;QAAE,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAC5D,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;QAC/C,IAAI,GAAG,KAAK,SAAS;YAAE,SAAS;QAChC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACzB,CAAC;IAED,MAAM,KAAK,GAAW,EAAE,CAAC;IACzB,KAAK,MAAM,CAAC,IAAI,UAAU,CAAC,IAAI,EAAE,EAAE,CAAC;QAClC,IACE,CAAC;YACD,OAAO,CAAC,KAAK,QAAQ;YACrB,OAAQ,CAAU,CAAC,IAAI,KAAK,QAAQ;YACpC,KAAK,CAAC,OAAO,CAAE,CAAU,CAAC,OAAO,CAAC;YACjC,CAAU,CAAC,KAAK,EACjB,CAAC;YACD,KAAK,CAAC,IAAI,CAAC,CAAS,CAAC,CAAC;QACxB,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,8EAA8E;AAC9E,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,IAAY,EACZ,OAAgB;IAEhB,MAAM,KAAK,GAAG,MAAM,iBAAiB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACrD,MAAM,KAAK,GAAW,EAAE,CAAC;IACzB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,GAAG,GAAG,CAAC,MAAM,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAGlD,CAAC;QACF,KAAK,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC;IACnC,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;AAC1B,CAAC;AAuBD;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,IAAI,GAAwB,EAAE;IAE9B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;IAEtC,IAAI,KAAK,GAAa,EAAE,CAAC;IACzB,IAAI,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;IACvB,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAClD,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;QACrB,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;IACvB,CAAC;IAED,MAAM,MAAM,GACV,IAAI,CAAC,MAAM;QACX,CAAC,MAAM,iBAAiB,CAAC;YACvB,OAAO,EAAE,IAAI,CAAC,OAAO,IAAI,CAAC,MAAM,eAAe,CAAC,GAAG,CAAC,CAAC;YACrD,YAAY,EAAE,IAAI,CAAC,YAAY;SAChC,CAAC,CAAC,CAAC;IAEN,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE;QAC3C,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;QACzC,OAAO,EAAE,IAAI,CAAC,OAAO,IAAI,IAAI;KAC9B,CAAC,CAAC;IACH,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;AAC3B,CAAC;AAED;;;;GAIG;AACH,KAAK,UAAU,eAAe,CAC5B,GAAW;IAEX,IAAI,CAAC;QACH,MAAM,EAAE,mBAAmB,EAAE,GAC3B,MAAM,MAAM,CAAC,+BAA+B,CAAC,CAAC;QAChD,MAAM,UAAU,GAAG,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;QACjD,OAAO,MAAM,mBAAmB,CAAC,aAAa,CAAC,UAAU,GAAG,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;IACzE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC","sourcesContent":["/**\n * The evals runner: discover `*.eval.ts` / `evals/*.ts` files, run each eval\n * through its scorer pipeline against the *real* agent loop, score, and report.\n *\n * It is the engine behind `agent-native eval` — when used as a CI deploy gate\n * the CLI exits non-zero if any eval scores below its threshold.\n *\n * Two layers:\n *   - `scoreEval` / `runEvals` — pure orchestration over an `AgentRunner` and\n *     a list of evals. Fully unit-testable with an injected runner (no model).\n *   - `discoverEvalFiles` / `loadEvals` — filesystem discovery + dynamic import\n *     of author-written eval modules.\n *\n * Results are also (best-effort) written to the observability eval store so a\n * dashboard can surface CI eval history next to production run evals.\n */\n\nimport nodePath from \"node:path\";\nimport { pathToFileURL } from \"node:url\";\n\nimport type { ActionEntry } from \"../agent/production-agent.js\";\nimport { insertEvalResult } from \"../observability/store.js\";\nimport type { EvalResult as ObservabilityEvalResult } from \"../observability/types.js\";\nimport type { AgentRunner } from \"./agent-runner.js\";\nimport { createAgentRunner } from \"./agent-runner.js\";\nimport { DEFAULT_EVAL_THRESHOLD } from \"./define-eval.js\";\nimport { clamp01 } from \"./scorer.js\";\nimport type {\n  AgentRunOutput,\n  Eval,\n  EvalResultRow,\n  EvalRunReport,\n  ScorerResult,\n} from \"./types.js\";\n\n// ─── Scoring orchestration ────────────────────────────────────────────\n\n/** Run one scorer's pipeline (preprocess → analyze → score → reason). */\nasync function runScorer(\n  scorer: Eval[\"scorers\"][number],\n  run: AgentRunOutput,\n  runner: AgentRunner,\n  threshold: number,\n): Promise<ScorerResult> {\n  try {\n    const pre = scorer.preprocess ? await scorer.preprocess(run) : run;\n    const analysis = scorer.analyze\n      ? await scorer.analyze(pre as never, runner.analyzeContext())\n      : pre;\n    const rawScore = await scorer.generateScore(analysis as never);\n    const score = clamp01(rawScore);\n    const reason = scorer.generateReason\n      ? await scorer.generateReason({\n          run,\n          analysis: analysis as never,\n          score,\n        })\n      : undefined;\n    return { scorer: scorer.name, score, reason, passed: score >= threshold };\n  } catch (err) {\n    // A scorer that throws is a failed scorer, not a crashed run — degrade\n    // gracefully so one bad scorer can't take down the whole CI gate.\n    return {\n      scorer: scorer.name,\n      score: 0,\n      reason: `Scorer errored: ${err instanceof Error ? err.message : String(err)}`,\n      passed: false,\n    };\n  }\n}\n\n/** Run a single eval: invoke the agent, then score with each scorer. */\nexport async function scoreEval(\n  evalCase: Eval,\n  runner: AgentRunner,\n  opts: { thresholdOverride?: number } = {},\n): Promise<EvalResultRow> {\n  const threshold =\n    opts.thresholdOverride ?? evalCase.threshold ?? DEFAULT_EVAL_THRESHOLD;\n\n  let run: AgentRunOutput;\n  if (evalCase.run) {\n    run = await evalCase.run({\n      input: evalCase.input,\n      runAgent: (input) => runner.runAgent(input),\n    });\n  } else {\n    run = await runner.runAgent(evalCase.input);\n  }\n\n  const scores: ScorerResult[] = [];\n  for (const scorer of evalCase.scorers) {\n    scores.push(await runScorer(scorer, run, runner, threshold));\n  }\n\n  const avgScore =\n    scores.length > 0\n      ? scores.reduce((s, r) => s + r.score, 0) / scores.length\n      : 0;\n\n  return {\n    eval: evalCase.name,\n    threshold,\n    scores,\n    // A run that errored, or any sub-threshold scorer, fails the case.\n    passed: run.ok && scores.every((s) => s.passed),\n    avgScore,\n    durationMs: run.durationMs,\n    error: run.ok ? undefined : run.error,\n  };\n}\n\n/** Run a batch of evals against one runner and aggregate a report. */\nexport async function runEvals(\n  evals: Eval[],\n  runner: AgentRunner,\n  opts: { thresholdOverride?: number; persist?: boolean } = {},\n): Promise<EvalRunReport> {\n  const results: EvalResultRow[] = [];\n  for (const evalCase of evals) {\n    const row = await scoreEval(evalCase, runner, opts);\n    results.push(row);\n    if (opts.persist) await persistEvalRow(row).catch(() => {});\n  }\n\n  const passed = results.filter((r) => r.passed).length;\n  return {\n    total: results.length,\n    passed,\n    failed: results.length - passed,\n    results,\n  };\n}\n\n/**\n * Best-effort write of one eval result to the observability eval store so a\n * dashboard can show CI eval history alongside production run evals. We write\n * one row per (eval × scorer), tagged `evalType: \"automated\"` with a synthetic\n * `eval:` run id.\n *\n * TODO(live-sampling): the same scorer list should also run on a sampled\n * fraction of *real* production runs. That hook belongs in the agent loop's\n * (not-yet-added) post-run processor seam: when a run finishes, roll the\n * configured sample rate and, if it hits, replay the run output through these\n * scorers and write the rows here. Wiring it now would require the in-loop\n * processor seam another wave is adding — so this is the single intended\n * attachment point, intentionally left as a note.\n */\nasync function persistEvalRow(row: EvalResultRow): Promise<void> {\n  const runId = `eval:${row.eval}:${Date.now()}`;\n  for (const s of row.scores) {\n    const result: ObservabilityEvalResult = {\n      id: crypto.randomUUID(),\n      runId,\n      threadId: null,\n      userId: null,\n      evalType: \"automated\",\n      criteria: `eval:${row.eval}:${s.scorer}`,\n      score: s.score,\n      reasoning: s.reason ?? null,\n      metadata: {\n        source: \"cli-eval\",\n        threshold: row.threshold,\n        passed: s.passed,\n      },\n      createdAt: Date.now(),\n    };\n    await insertEvalResult(result);\n  }\n}\n\n// ─── Discovery + loading ──────────────────────────────────────────────\n\nconst EVAL_FILE_RE = /\\.eval\\.(ts|js|mjs)$/;\nconst SKIP_DIRS = new Set([\"node_modules\", \"dist\", \".git\", \".output\", \"build\"]);\n\n/**\n * Walk `root` for eval files. Matches two conventions:\n *   - any `**\\/*.eval.ts` (co-located with code), and\n *   - any `*.ts` directly inside an `evals/` directory.\n * `pattern` further filters by substring of the relative path.\n */\nexport async function discoverEvalFiles(\n  root: string,\n  pattern?: string,\n): Promise<string[]> {\n  const fs = await import(\"node:fs\");\n  const out: string[] = [];\n\n  function isEvalFile(full: string, parentName: string): boolean {\n    const base = nodePath.basename(full);\n    if (EVAL_FILE_RE.test(base)) return true;\n    if (parentName === \"evals\" && /\\.(ts|js|mjs)$/.test(base)) {\n      // Skip obvious support files inside evals/.\n      return !/\\.(spec|test|d)\\.(ts|js|mjs)$/.test(base);\n    }\n    return false;\n  }\n\n  function walk(dir: string, parentName: string): void {\n    let entries: import(\"node:fs\").Dirent[];\n    try {\n      entries = fs.readdirSync(dir, { withFileTypes: true });\n    } catch {\n      return;\n    }\n    for (const entry of entries) {\n      const full = nodePath.join(dir, entry.name);\n      if (entry.isDirectory()) {\n        if (SKIP_DIRS.has(entry.name) || entry.name.startsWith(\".\")) continue;\n        walk(full, entry.name);\n      } else if (entry.isFile() && isEvalFile(full, parentName)) {\n        out.push(full);\n      }\n    }\n  }\n\n  walk(root, nodePath.basename(root));\n  out.sort();\n\n  if (!pattern) return out;\n  return out.filter((f) => nodePath.relative(root, f).includes(pattern));\n}\n\n/** Pull `Eval` definitions out of a dynamically-imported eval module. */\nfunction extractEvals(mod: Record<string, unknown>): Eval[] {\n  const candidates: unknown[] = [];\n  if (mod.default !== undefined) candidates.push(mod.default);\n  for (const [key, value] of Object.entries(mod)) {\n    if (key === \"default\") continue;\n    candidates.push(value);\n  }\n\n  const evals: Eval[] = [];\n  for (const c of candidates.flat()) {\n    if (\n      c &&\n      typeof c === \"object\" &&\n      typeof (c as Eval).name === \"string\" &&\n      Array.isArray((c as Eval).scorers) &&\n      (c as Eval).input\n    ) {\n      evals.push(c as Eval);\n    }\n  }\n  return evals;\n}\n\n/** Discover and import all eval files under `root`, returning their evals. */\nexport async function loadEvals(\n  root: string,\n  pattern?: string,\n): Promise<{ files: string[]; evals: Eval[] }> {\n  const files = await discoverEvalFiles(root, pattern);\n  const evals: Eval[] = [];\n  for (const file of files) {\n    const mod = (await import(pathToFileURL(file).href)) as Record<\n      string,\n      unknown\n    >;\n    evals.push(...extractEvals(mod));\n  }\n  return { files, evals };\n}\n\n// ─── High-level entry used by the CLI ─────────────────────────────────\n\nexport interface RunEvalSuiteOptions {\n  /** App root to discover eval files + actions under. Defaults to cwd. */\n  cwd?: string;\n  /** Substring filter on the eval file path. */\n  pattern?: string;\n  /** Global threshold override (wins over per-eval thresholds). */\n  thresholdOverride?: number;\n  /** App actions to expose to the agent. Auto-discovered when omitted. */\n  actions?: Record<string, ActionEntry>;\n  /** System prompt for runs. */\n  systemPrompt?: string;\n  /** Write results to the observability eval store (default true). */\n  persist?: boolean;\n  /** Pre-built runner (tests inject this to avoid touching engine/loop). */\n  runner?: AgentRunner;\n  /** Pre-loaded evals (tests inject this to skip filesystem discovery). */\n  evals?: Eval[];\n}\n\n/**\n * End-to-end: load evals, build a runner, score, report. The CLI wraps this\n * and maps `report.failed > 0` to a non-zero exit code (the CI gate).\n */\nexport async function runEvalSuite(\n  opts: RunEvalSuiteOptions = {},\n): Promise<{ report: EvalRunReport; files: string[] }> {\n  const cwd = opts.cwd ?? process.cwd();\n\n  let files: string[] = [];\n  let evals = opts.evals;\n  if (!evals) {\n    const loaded = await loadEvals(cwd, opts.pattern);\n    files = loaded.files;\n    evals = loaded.evals;\n  }\n\n  const runner =\n    opts.runner ??\n    (await createAgentRunner({\n      actions: opts.actions ?? (await discoverActions(cwd)),\n      systemPrompt: opts.systemPrompt,\n    }));\n\n  const report = await runEvals(evals, runner, {\n    thresholdOverride: opts.thresholdOverride,\n    persist: opts.persist ?? true,\n  });\n  return { report, files };\n}\n\n/**\n * Discover the app's actions so the agent under test has the real tool\n * surface. Lazy-imports `autoDiscoverActions` to keep server-only deps out of\n * any browser bundle that might touch this module's types.\n */\nasync function discoverActions(\n  cwd: string,\n): Promise<Record<string, ActionEntry>> {\n  try {\n    const { autoDiscoverActions } =\n      await import(\"../server/action-discovery.js\");\n    const actionsDir = nodePath.join(cwd, \"actions\");\n    return await autoDiscoverActions(pathToFileURL(actionsDir + \"/\").href);\n  } catch {\n    return {};\n  }\n}\n"]}
+{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../src/eval/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,QAAQ,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAGzC,OAAO,EAAE,gBAAgB,EAAE,MAAM,2BAA2B,CAAC;AAG7D,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AACtD,OAAO,EAAE,sBAAsB,EAAE,MAAM,kBAAkB,CAAC;AAC1D,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AAStC,yEAAyE;AAEzE,yEAAyE;AACzE,KAAK,UAAU,SAAS,CACtB,MAA+B,EAC/B,GAAmB,EACnB,MAAmB,EACnB,SAAiB;IAEjB,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;QACnE,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO;YAC7B,CAAC,CAAC,MAAM,MAAM,CAAC,OAAO,CAAC,GAAY,EAAE,MAAM,CAAC,cAAc,EAAE,CAAC;YAC7D,CAAC,CAAC,GAAG,CAAC;QACR,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,QAAiB,CAAC,CAAC;QAC/D,MAAM,KAAK,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;QAChC,MAAM,MAAM,GAAG,MAAM,CAAC,cAAc;YAClC,CAAC,CAAC,MAAM,MAAM,CAAC,cAAc,CAAC;gBAC1B,GAAG;gBACH,QAAQ,EAAE,QAAiB;gBAC3B,KAAK;aACN,CAAC;YACJ,CAAC,CAAC,SAAS,CAAC;QACd,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,IAAI,SAAS,EAAE,CAAC;IAC5E,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,uEAAuE;QACvE,kEAAkE;QAClE,OAAO;YACL,MAAM,EAAE,MAAM,CAAC,IAAI;YACnB,KAAK,EAAE,CAAC;YACR,MAAM,EAAE,mBAAmB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;YAC7E,MAAM,EAAE,KAAK;SACd,CAAC;IACJ,CAAC;AACH,CAAC;AAED,wEAAwE;AACxE,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,QAAc,EACd,MAAmB,EACnB,IAAI,GAAmC,EAAE;IAEzC,MAAM,SAAS,GACb,IAAI,CAAC,iBAAiB,IAAI,QAAQ,CAAC,SAAS,IAAI,sBAAsB,CAAC;IAEzE,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC;QACxB,OAAO;YACL,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,SAAS;YACT,MAAM,EAAE,EAAE;YACV,MAAM,EAAE,SAAS;YACjB,UAAU,EAAE,QAAQ,CAAC,UAAU;YAC/B,sEAAsE;YACtE,MAAM,EAAE,IAAI;YACZ,QAAQ,EAAE,CAAC;YACX,UAAU,EAAE,CAAC;SACd,CAAC;IACJ,CAAC;IAED,IAAI,GAAmB,CAAC;IACxB,IAAI,QAAQ,CAAC,GAAG,EAAE,CAAC;QACjB,GAAG,GAAG,MAAM,QAAQ,CAAC,GAAG,CAAC;YACvB,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,QAAQ,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;SAC5C,CAAC,CAAC;IACL,CAAC;SAAM,CAAC;QACN,GAAG,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC9C,CAAC;IAED,MAAM,MAAM,GAAmB,EAAE,CAAC;IAClC,KAAK,MAAM,MAAM,IAAI,QAAQ,CAAC,OAAO,EAAE,CAAC;QACtC,MAAM,CAAC,IAAI,CAAC,MAAM,SAAS,CAAC,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC;IAC/D,CAAC;IAED,MAAM,QAAQ,GACZ,MAAM,CAAC,MAAM,GAAG,CAAC;QACf,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM;QACzD,CAAC,CAAC,CAAC,CAAC;IAER,OAAO;QACL,IAAI,EAAE,QAAQ,CAAC,IAAI;QACnB,SAAS;QACT,MAAM;QACN,mEAAmE;QACnE,MAAM,EAAE,GAAG,CAAC,EAAE,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC;QAC/C,MAAM,EAAE,GAAG,CAAC,EAAE,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ;QACrE,QAAQ;QACR,UAAU,EAAE,GAAG,CAAC,UAAU;QAC1B,KAAK,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK;KACtC,CAAC;AACJ,CAAC;AAED,sEAAsE;AACtE,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,KAAa,EACb,MAAmB,EACnB,IAAI,GAAsD,EAAE;IAE5D,MAAM,OAAO,GAAoB,EAAE,CAAC;IACpC,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE,CAAC;QAC7B,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;QACpD,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAClB,IAAI,IAAI,CAAC,OAAO,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YAC7C,MAAM,cAAc,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QAC5C,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IACtD,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,MAAM,CAAC;IACrE,OAAO;QACL,KAAK,EAAE,OAAO,CAAC,MAAM;QACrB,MAAM;QACN,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,SAAS,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM;QACzE,OAAO;QACP,OAAO;KACR,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,KAAK,UAAU,cAAc,CAAC,GAAkB;IAC9C,MAAM,KAAK,GAAG,QAAQ,GAAG,CAAC,IAAI,IAAI,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;IAC/C,KAAK,MAAM,CAAC,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC;QAC3B,MAAM,MAAM,GAA4B;YACtC,EAAE,EAAE,MAAM,CAAC,UAAU,EAAE;YACvB,KAAK;YACL,QAAQ,EAAE,IAAI;YACd,MAAM,EAAE,IAAI;YACZ,QAAQ,EAAE,WAAW;YACrB,QAAQ,EAAE,QAAQ,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,MAAM,EAAE;YACxC,KAAK,EAAE,CAAC,CAAC,KAAK;YACd,SAAS,EAAE,CAAC,CAAC,MAAM,IAAI,IAAI;YAC3B,QAAQ,EAAE;gBACR,MAAM,EAAE,UAAU;gBAClB,SAAS,EAAE,GAAG,CAAC,SAAS;gBACxB,MAAM,EAAE,CAAC,CAAC,MAAM;aACjB;YACD,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;SACtB,CAAC;QACF,MAAM,gBAAgB,CAAC,MAAM,CAAC,CAAC;IACjC,CAAC;AACH,CAAC;AAED,yEAAyE;AAEzE,MAAM,YAAY,GAAG,sBAAsB,CAAC;AAC5C,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,CAAC,cAAc,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC,CAAC;AAEhF;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,IAAY,EACZ,OAAgB;IAEhB,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC;IACnC,MAAM,GAAG,GAAa,EAAE,CAAC;IAEzB,SAAS,UAAU,CAAC,IAAY,EAAE,UAAkB;QAClD,MAAM,IAAI,GAAG,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACrC,IAAI,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,OAAO,IAAI,CAAC;QACzC,IAAI,UAAU,KAAK,OAAO,IAAI,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC1D,4CAA4C;YAC5C,OAAO,CAAC,+BAA+B,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACrD,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,SAAS,IAAI,CAAC,GAAW,EAAE,UAAkB;QAC3C,IAAI,OAAmC,CAAC;QACxC,IAAI,CAAC;YACH,OAAO,GAAG,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;QACzD,CAAC;QAAC,MAAM,CAAC;YACP,OAAO;QACT,CAAC;QACD,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;YAC5C,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;gBACxB,IAAI,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;oBAAE,SAAS;gBACtE,IAAI,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;YACzB,CAAC;iBAAM,IAAI,KAAK,CAAC,MAAM,EAAE,IAAI,UAAU,CAAC,IAAI,EAAE,UAAU,CAAC,EAAE,CAAC;gBAC1D,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACjB,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;IACpC,GAAG,CAAC,IAAI,EAAE,CAAC;IAEX,IAAI,CAAC,OAAO;QAAE,OAAO,GAAG,CAAC;IACzB,OAAO,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;AACzE,CAAC;AAED,yEAAyE;AACzE,SAAS,YAAY,CAAC,GAA4B;IAChD,MAAM,UAAU,GAAc,EAAE,CAAC;IACjC,IAAI,GAAG,CAAC,OAAO,KAAK,SAAS;QAAE,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAC5D,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;QAC/C,IAAI,GAAG,KAAK,SAAS;YAAE,SAAS;QAChC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACzB,CAAC;IAED,MAAM,KAAK,GAAW,EAAE,CAAC;IACzB,KAAK,MAAM,CAAC,IAAI,UAAU,CAAC,IAAI,EAAE,EAAE,CAAC;QAClC,IACE,CAAC;YACD,OAAO,CAAC,KAAK,QAAQ;YACrB,OAAQ,CAAU,CAAC,IAAI,KAAK,QAAQ;YACpC,KAAK,CAAC,OAAO,CAAE,CAAU,CAAC,OAAO,CAAC;YACjC,CAAU,CAAC,KAAK,EACjB,CAAC;YACD,KAAK,CAAC,IAAI,CAAC,CAAS,CAAC,CAAC;QACxB,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,8EAA8E;AAC9E,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,IAAY,EACZ,OAAgB;IAEhB,MAAM,KAAK,GAAG,MAAM,iBAAiB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACrD,MAAM,KAAK,GAAW,EAAE,CAAC;IACzB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,GAAG,GAAG,CAAC,MAAM,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAGlD,CAAC;QACF,KAAK,CAAC,IAAI,CAAC,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC;IACnC,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;AAC1B,CAAC;AAuBD;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,IAAI,GAAwB,EAAE;IAE9B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;IAEtC,IAAI,KAAK,GAAa,EAAE,CAAC;IACzB,IAAI,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;IACvB,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAClD,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;QACrB,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;IACvB,CAAC;IAED,MAAM,WAAW,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;IACnE,MAAM,MAAM,GACV,IAAI,CAAC,MAAM;QACX,CAAC,WAAW;YACV,CAAC,CAAC,MAAM,iBAAiB,CAAC;gBACtB,OAAO,EAAE,IAAI,CAAC,OAAO,IAAI,CAAC,MAAM,eAAe,CAAC,GAAG,CAAC,CAAC;gBACrD,YAAY,EAAE,IAAI,CAAC,YAAY;aAChC,CAAC;YACJ,CAAC,CAAC,iBAAiB,EAAE,CAAC,CAAC;IAE3B,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE;QAC3C,iBAAiB,EAAE,IAAI,CAAC,iBAAiB;QACzC,OAAO,EAAE,IAAI,CAAC,OAAO,IAAI,IAAI;KAC9B,CAAC,CAAC;IACH,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;AAC3B,CAAC;AAED;;;;GAIG;AACH,KAAK,UAAU,eAAe,CAC5B,GAAW;IAEX,IAAI,CAAC;QACH,MAAM,EAAE,mBAAmB,EAAE,GAC3B,MAAM,MAAM,CAAC,+BAA+B,CAAC,CAAC;QAChD,MAAM,UAAU,GAAG,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;QACjD,OAAO,MAAM,mBAAmB,CAAC,aAAa,CAAC,UAAU,GAAG,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;IACzE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,SAAS,iBAAiB;IACxB,OAAO;QACL,MAAM,EAAE,EAA2B;QACnC,KAAK,EAAE,OAAO;QACd,KAAK,CAAC,QAAQ;YACZ,MAAM,IAAI,KAAK,CAAC,8CAA8C,CAAC,CAAC;QAClE,CAAC;QACD,cAAc;YACZ,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;QACjE,CAAC;KACF,CAAC;AACJ,CAAC","sourcesContent":["/**\n * The evals runner: discover `*.eval.ts` / `evals/*.ts` files, run each eval\n * through its scorer pipeline against the *real* agent loop, score, and report.\n *\n * It is the engine behind `agent-native eval` — when used as a CI deploy gate\n * the CLI exits non-zero if any eval scores below its threshold.\n *\n * Two layers:\n *   - `scoreEval` / `runEvals` — pure orchestration over an `AgentRunner` and\n *     a list of evals. Fully unit-testable with an injected runner (no model).\n *   - `discoverEvalFiles` / `loadEvals` — filesystem discovery + dynamic import\n *     of author-written eval modules.\n *\n * Results are also (best-effort) written to the observability eval store so a\n * dashboard can surface CI eval history next to production run evals.\n */\n\nimport nodePath from \"node:path\";\nimport { pathToFileURL } from \"node:url\";\n\nimport type { ActionEntry } from \"../agent/production-agent.js\";\nimport { insertEvalResult } from \"../observability/store.js\";\nimport type { EvalResult as ObservabilityEvalResult } from \"../observability/types.js\";\nimport type { AgentRunner } from \"./agent-runner.js\";\nimport { createAgentRunner } from \"./agent-runner.js\";\nimport { DEFAULT_EVAL_THRESHOLD } from \"./define-eval.js\";\nimport { clamp01 } from \"./scorer.js\";\nimport type {\n  AgentRunOutput,\n  Eval,\n  EvalResultRow,\n  EvalRunReport,\n  ScorerResult,\n} from \"./types.js\";\n\n// ─── Scoring orchestration ────────────────────────────────────────────\n\n/** Run one scorer's pipeline (preprocess → analyze → score → reason). */\nasync function runScorer(\n  scorer: Eval[\"scorers\"][number],\n  run: AgentRunOutput,\n  runner: AgentRunner,\n  threshold: number,\n): Promise<ScorerResult> {\n  try {\n    const pre = scorer.preprocess ? await scorer.preprocess(run) : run;\n    const analysis = scorer.analyze\n      ? await scorer.analyze(pre as never, runner.analyzeContext())\n      : pre;\n    const rawScore = await scorer.generateScore(analysis as never);\n    const score = clamp01(rawScore);\n    const reason = scorer.generateReason\n      ? await scorer.generateReason({\n          run,\n          analysis: analysis as never,\n          score,\n        })\n      : undefined;\n    return { scorer: scorer.name, score, reason, passed: score >= threshold };\n  } catch (err) {\n    // A scorer that throws is a failed scorer, not a crashed run — degrade\n    // gracefully so one bad scorer can't take down the whole CI gate.\n    return {\n      scorer: scorer.name,\n      score: 0,\n      reason: `Scorer errored: ${err instanceof Error ? err.message : String(err)}`,\n      passed: false,\n    };\n  }\n}\n\n/** Run a single eval: invoke the agent, then score with each scorer. */\nexport async function scoreEval(\n  evalCase: Eval,\n  runner: AgentRunner,\n  opts: { thresholdOverride?: number } = {},\n): Promise<EvalResultRow> {\n  const threshold =\n    opts.thresholdOverride ?? evalCase.threshold ?? DEFAULT_EVAL_THRESHOLD;\n\n  if (evalCase.skipReason) {\n    return {\n      eval: evalCase.name,\n      threshold,\n      scores: [],\n      status: \"skipped\",\n      skipReason: evalCase.skipReason,\n      // Keep the legacy boolean gate friendly: skipped rows do not fail CI.\n      passed: true,\n      avgScore: 0,\n      durationMs: 0,\n    };\n  }\n\n  let run: AgentRunOutput;\n  if (evalCase.run) {\n    run = await evalCase.run({\n      input: evalCase.input,\n      runAgent: (input) => runner.runAgent(input),\n    });\n  } else {\n    run = await runner.runAgent(evalCase.input);\n  }\n\n  const scores: ScorerResult[] = [];\n  for (const scorer of evalCase.scorers) {\n    scores.push(await runScorer(scorer, run, runner, threshold));\n  }\n\n  const avgScore =\n    scores.length > 0\n      ? scores.reduce((s, r) => s + r.score, 0) / scores.length\n      : 0;\n\n  return {\n    eval: evalCase.name,\n    threshold,\n    scores,\n    // A run that errored, or any sub-threshold scorer, fails the case.\n    passed: run.ok && scores.every((s) => s.passed),\n    status: run.ok && scores.every((s) => s.passed) ? \"passed\" : \"failed\",\n    avgScore,\n    durationMs: run.durationMs,\n    error: run.ok ? undefined : run.error,\n  };\n}\n\n/** Run a batch of evals against one runner and aggregate a report. */\nexport async function runEvals(\n  evals: Eval[],\n  runner: AgentRunner,\n  opts: { thresholdOverride?: number; persist?: boolean } = {},\n): Promise<EvalRunReport> {\n  const results: EvalResultRow[] = [];\n  for (const evalCase of evals) {\n    const row = await scoreEval(evalCase, runner, opts);\n    results.push(row);\n    if (opts.persist && row.status !== \"skipped\") {\n      await persistEvalRow(row).catch(() => {});\n    }\n  }\n\n  const passed = results.filter((r) => r.passed).length;\n  const skipped = results.filter((r) => r.status === \"skipped\").length;\n  return {\n    total: results.length,\n    passed,\n    failed: results.filter((r) => r.status !== \"skipped\" && !r.passed).length,\n    skipped,\n    results,\n  };\n}\n\n/**\n * Best-effort write of one eval result to the observability eval store so a\n * dashboard can show CI eval history alongside production run evals. We write\n * one row per (eval × scorer), tagged `evalType: \"automated\"` with a synthetic\n * `eval:` run id.\n *\n * TODO(live-sampling): the same scorer list should also run on a sampled\n * fraction of *real* production runs. That hook belongs in the agent loop's\n * (not-yet-added) post-run processor seam: when a run finishes, roll the\n * configured sample rate and, if it hits, replay the run output through these\n * scorers and write the rows here. Wiring it now would require the in-loop\n * processor seam another wave is adding — so this is the single intended\n * attachment point, intentionally left as a note.\n */\nasync function persistEvalRow(row: EvalResultRow): Promise<void> {\n  const runId = `eval:${row.eval}:${Date.now()}`;\n  for (const s of row.scores) {\n    const result: ObservabilityEvalResult = {\n      id: crypto.randomUUID(),\n      runId,\n      threadId: null,\n      userId: null,\n      evalType: \"automated\",\n      criteria: `eval:${row.eval}:${s.scorer}`,\n      score: s.score,\n      reasoning: s.reason ?? null,\n      metadata: {\n        source: \"cli-eval\",\n        threshold: row.threshold,\n        passed: s.passed,\n      },\n      createdAt: Date.now(),\n    };\n    await insertEvalResult(result);\n  }\n}\n\n// ─── Discovery + loading ──────────────────────────────────────────────\n\nconst EVAL_FILE_RE = /\\.eval\\.(ts|js|mjs)$/;\nconst SKIP_DIRS = new Set([\"node_modules\", \"dist\", \".git\", \".output\", \"build\"]);\n\n/**\n * Walk `root` for eval files. Matches two conventions:\n *   - any `**\\/*.eval.ts` (co-located with code), and\n *   - any `*.ts` directly inside an `evals/` directory.\n * `pattern` further filters by substring of the relative path.\n */\nexport async function discoverEvalFiles(\n  root: string,\n  pattern?: string,\n): Promise<string[]> {\n  const fs = await import(\"node:fs\");\n  const out: string[] = [];\n\n  function isEvalFile(full: string, parentName: string): boolean {\n    const base = nodePath.basename(full);\n    if (EVAL_FILE_RE.test(base)) return true;\n    if (parentName === \"evals\" && /\\.(ts|js|mjs)$/.test(base)) {\n      // Skip obvious support files inside evals/.\n      return !/\\.(spec|test|d)\\.(ts|js|mjs)$/.test(base);\n    }\n    return false;\n  }\n\n  function walk(dir: string, parentName: string): void {\n    let entries: import(\"node:fs\").Dirent[];\n    try {\n      entries = fs.readdirSync(dir, { withFileTypes: true });\n    } catch {\n      return;\n    }\n    for (const entry of entries) {\n      const full = nodePath.join(dir, entry.name);\n      if (entry.isDirectory()) {\n        if (SKIP_DIRS.has(entry.name) || entry.name.startsWith(\".\")) continue;\n        walk(full, entry.name);\n      } else if (entry.isFile() && isEvalFile(full, parentName)) {\n        out.push(full);\n      }\n    }\n  }\n\n  walk(root, nodePath.basename(root));\n  out.sort();\n\n  if (!pattern) return out;\n  return out.filter((f) => nodePath.relative(root, f).includes(pattern));\n}\n\n/** Pull `Eval` definitions out of a dynamically-imported eval module. */\nfunction extractEvals(mod: Record<string, unknown>): Eval[] {\n  const candidates: unknown[] = [];\n  if (mod.default !== undefined) candidates.push(mod.default);\n  for (const [key, value] of Object.entries(mod)) {\n    if (key === \"default\") continue;\n    candidates.push(value);\n  }\n\n  const evals: Eval[] = [];\n  for (const c of candidates.flat()) {\n    if (\n      c &&\n      typeof c === \"object\" &&\n      typeof (c as Eval).name === \"string\" &&\n      Array.isArray((c as Eval).scorers) &&\n      (c as Eval).input\n    ) {\n      evals.push(c as Eval);\n    }\n  }\n  return evals;\n}\n\n/** Discover and import all eval files under `root`, returning their evals. */\nexport async function loadEvals(\n  root: string,\n  pattern?: string,\n): Promise<{ files: string[]; evals: Eval[] }> {\n  const files = await discoverEvalFiles(root, pattern);\n  const evals: Eval[] = [];\n  for (const file of files) {\n    const mod = (await import(pathToFileURL(file).href)) as Record<\n      string,\n      unknown\n    >;\n    evals.push(...extractEvals(mod));\n  }\n  return { files, evals };\n}\n\n// ─── High-level entry used by the CLI ─────────────────────────────────\n\nexport interface RunEvalSuiteOptions {\n  /** App root to discover eval files + actions under. Defaults to cwd. */\n  cwd?: string;\n  /** Substring filter on the eval file path. */\n  pattern?: string;\n  /** Global threshold override (wins over per-eval thresholds). */\n  thresholdOverride?: number;\n  /** App actions to expose to the agent. Auto-discovered when omitted. */\n  actions?: Record<string, ActionEntry>;\n  /** System prompt for runs. */\n  systemPrompt?: string;\n  /** Write results to the observability eval store (default true). */\n  persist?: boolean;\n  /** Pre-built runner (tests inject this to avoid touching engine/loop). */\n  runner?: AgentRunner;\n  /** Pre-loaded evals (tests inject this to skip filesystem discovery). */\n  evals?: Eval[];\n}\n\n/**\n * End-to-end: load evals, build a runner, score, report. The CLI wraps this\n * and maps `report.failed > 0` to a non-zero exit code (the CI gate).\n */\nexport async function runEvalSuite(\n  opts: RunEvalSuiteOptions = {},\n): Promise<{ report: EvalRunReport; files: string[] }> {\n  const cwd = opts.cwd ?? process.cwd();\n\n  let files: string[] = [];\n  let evals = opts.evals;\n  if (!evals) {\n    const loaded = await loadEvals(cwd, opts.pattern);\n    files = loaded.files;\n    evals = loaded.evals;\n  }\n\n  const needsRunner = evals.some((evalCase) => !evalCase.skipReason);\n  const runner =\n    opts.runner ??\n    (needsRunner\n      ? await createAgentRunner({\n          actions: opts.actions ?? (await discoverActions(cwd)),\n          systemPrompt: opts.systemPrompt,\n        })\n      : createInertRunner());\n\n  const report = await runEvals(evals, runner, {\n    thresholdOverride: opts.thresholdOverride,\n    persist: opts.persist ?? true,\n  });\n  return { report, files };\n}\n\n/**\n * Discover the app's actions so the agent under test has the real tool\n * surface. Lazy-imports `autoDiscoverActions` to keep server-only deps out of\n * any browser bundle that might touch this module's types.\n */\nasync function discoverActions(\n  cwd: string,\n): Promise<Record<string, ActionEntry>> {\n  try {\n    const { autoDiscoverActions } =\n      await import(\"../server/action-discovery.js\");\n    const actionsDir = nodePath.join(cwd, \"actions\");\n    return await autoDiscoverActions(pathToFileURL(actionsDir + \"/\").href);\n  } catch {\n    return {};\n  }\n}\n\nfunction createInertRunner(): AgentRunner {\n  return {\n    engine: {} as AgentRunner[\"engine\"],\n    model: \"inert\",\n    async runAgent() {\n      throw new Error(\"Eval unexpectedly requested the agent runner\");\n    },\n    analyzeContext() {\n      throw new Error(\"Eval unexpectedly requested analyze context\");\n    },\n  };\n}\n"]}

package/dist/eval/types.d.ts

@@ -122,6 +122,12 @@ export interface EvalRunContext {
 export interface Eval {
     name: string;
     input: EvalInput;
+    /**
+     * Skip this case without running the agent or scorers. Use for opt-in
+     * suites that need secrets, live provider credentials, or manual/nightly
+     * gates. Skipped rows are reported distinctly and do not fail the suite.
+     */
+    skipReason?: string;
     /**
      * Optional override for how the agent is run for this case. Defaults to the
      * runner's headless `runAgent`. Use this to do custom setup (seed data,
@@ -144,6 +150,10 @@ export interface EvalResultRow {
     eval: string;
     threshold: number;
     scores: ScorerResult[];
+    /** Explicit row status for reports. Older consumers can still read `passed`. */
+    status?: "passed" | "failed" | "skipped";
+    /** Human-readable reason when `status` is `skipped`. */
+    skipReason?: string;
     /** True only when every scorer passed. */
     passed: boolean;
     /** Mean of the scorer scores, for at-a-glance ranking. */
@@ -157,6 +167,7 @@ export interface EvalRunReport {
     total: number;
     passed: number;
     failed: number;
+    skipped?: number;
     results: EvalResultRow[];
 }
 //# sourceMappingURL=types.d.ts.map

package/dist/eval/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/eval/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAI5D;;;;GAIG;AACH,MAAM,WAAW,cAAc;IAC7B,0DAA0D;IAC1D,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,+DAA+D;IAC/D,QAAQ,CAAC,SAAS,EAAE,SAAS,MAAM,EAAE,CAAC;IACtC,gEAAgE;IAChE,QAAQ,CAAC,EAAE,EAAE,OAAO,CAAC;IACrB,kDAAkD;IAClD,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,iFAAiF;IACjF,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,sDAAsD;IACtD,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC7B;AAID;;;;;;GAMG;AACH,MAAM,WAAW,oBAAoB;IACnC,oEAAoE;IACpE,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC;IAC7B,gDAAgD;IAChD,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB;;;;OAIG;IACH,KAAK,CAAC,IAAI,EAAE;QACV,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,MAAM,EAAE,MAAM,CAAC;QACf,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,MAAM,CAAC,EAAE,WAAW,CAAC;KACtB,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CACrB;AAED;;;;;;;;;;GAUG;AACH,MAAM,WAAW,MAAM,CAAC,GAAG,GAAG,cAAc,EAAE,GAAG,GAAG,GAAG;IACrD,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,CAAC,GAAG,EAAE,cAAc,GAAG,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;IACrD,OAAO,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,GAAG,EAAE,oBAAoB,GAAG,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;IACpE,sDAAsD;IACtD,aAAa,CAAC,QAAQ,EAAE,GAAG,GAAG,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACvD,cAAc,CAAC,CAAC,IAAI,EAAE;QACpB,GAAG,EAAE,cAAc,CAAC;QACpB,QAAQ,EAAE,GAAG,CAAC;QACd,KAAK,EAAE,MAAM,CAAC;KACf,GAAG,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CAC9B;AAED,kDAAkD;AAClD,MAAM,WAAW,gBAAgB,CAAC,GAAG,GAAG,cAAc,EAAE,GAAG,GAAG,GAAG;IAC/D,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,CAAC,CAAC,GAAG,EAAE,cAAc,GAAG,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;IACrD,OAAO,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,GAAG,EAAE,oBAAoB,GAAG,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;IACpE,aAAa,CAAC,QAAQ,EAAE,GAAG,GAAG,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACvD,cAAc,CAAC,CAAC,IAAI,EAAE;QACpB,GAAG,EAAE,cAAc,CAAC;QACpB,QAAQ,EAAE,GAAG,CAAC;QACd,KAAK,EAAE,MAAM,CAAC;KACf,GAAG,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CAC9B;AAID,6DAA6D;AAC7D,MAAM,WAAW,SAAS;IACxB,mDAAmD;IACnD,MAAM,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,OAAO,CAAC,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,GAAG,WAAW,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CAC/D;AAED,2DAA2D;AAC3D,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,KAAK,EAAE,SAAS,CAAC;IAC1B,8EAA8E;IAC9E,QAAQ,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;CACrD;AAED;;;;GAIG;AACH,MAAM,WAAW,IAAI;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,SAAS,CAAC;IACjB;;;;OAIG;IACH,GAAG,CAAC,CAAC,GAAG,EAAE,cAAc,GAAG,cAAc,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;IACpE,OAAO,EAAE,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,CAAC;IAC5B,oEAAoE;IACpE,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAID,iFAAiF;AACjF,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,OAAO,CAAC;CACjB;AAED,gEAAgE;AAChE,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,YAAY,EAAE,CAAC;IACvB,0CAA0C;IAC1C,MAAM,EAAE,OAAO,CAAC;IAChB,0DAA0D;IAC1D,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,qDAAqD;IACrD,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,8BAA8B;AAC9B,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,aAAa,EAAE,CAAC;CAC1B"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/eval/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,0BAA0B,CAAC;AAI5D;;;;GAIG;AACH,MAAM,WAAW,cAAc;IAC7B,0DAA0D;IAC1D,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,+DAA+D;IAC/D,QAAQ,CAAC,SAAS,EAAE,SAAS,MAAM,EAAE,CAAC;IACtC,gEAAgE;IAChE,QAAQ,CAAC,EAAE,EAAE,OAAO,CAAC;IACrB,kDAAkD;IAClD,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,iFAAiF;IACjF,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,sDAAsD;IACtD,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC7B;AAID;;;;;;GAMG;AACH,MAAM,WAAW,oBAAoB;IACnC,oEAAoE;IACpE,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC;IAC7B,gDAAgD;IAChD,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB;;;;OAIG;IACH,KAAK,CAAC,IAAI,EAAE;QACV,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,MAAM,EAAE,MAAM,CAAC;QACf,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,MAAM,CAAC,EAAE,WAAW,CAAC;KACtB,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CACrB;AAED;;;;;;;;;;GAUG;AACH,MAAM,WAAW,MAAM,CAAC,GAAG,GAAG,cAAc,EAAE,GAAG,GAAG,GAAG;IACrD,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,CAAC,GAAG,EAAE,cAAc,GAAG,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;IACrD,OAAO,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,GAAG,EAAE,oBAAoB,GAAG,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;IACpE,sDAAsD;IACtD,aAAa,CAAC,QAAQ,EAAE,GAAG,GAAG,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACvD,cAAc,CAAC,CAAC,IAAI,EAAE;QACpB,GAAG,EAAE,cAAc,CAAC;QACpB,QAAQ,EAAE,GAAG,CAAC;QACd,KAAK,EAAE,MAAM,CAAC;KACf,GAAG,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CAC9B;AAED,kDAAkD;AAClD,MAAM,WAAW,gBAAgB,CAAC,GAAG,GAAG,cAAc,EAAE,GAAG,GAAG,GAAG;IAC/D,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,CAAC,CAAC,GAAG,EAAE,cAAc,GAAG,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;IACrD,OAAO,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,GAAG,EAAE,oBAAoB,GAAG,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;IACpE,aAAa,CAAC,QAAQ,EAAE,GAAG,GAAG,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IACvD,cAAc,CAAC,CAAC,IAAI,EAAE;QACpB,GAAG,EAAE,cAAc,CAAC;QACpB,QAAQ,EAAE,GAAG,CAAC;QACd,KAAK,EAAE,MAAM,CAAC;KACf,GAAG,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CAC9B;AAID,6DAA6D;AAC7D,MAAM,WAAW,SAAS;IACxB,mDAAmD;IACnD,MAAM,EAAE,MAAM,CAAC;IACf;;;OAGG;IACH,OAAO,CAAC,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,GAAG,WAAW,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;CAC/D;AAED,2DAA2D;AAC3D,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,KAAK,EAAE,SAAS,CAAC;IAC1B,8EAA8E;IAC9E,QAAQ,CAAC,KAAK,EAAE,SAAS,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;CACrD;AAED;;;;GAIG;AACH,MAAM,WAAW,IAAI;IACnB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,SAAS,CAAC;IACjB;;;;OAIG;IACH,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB;;;;OAIG;IACH,GAAG,CAAC,CAAC,GAAG,EAAE,cAAc,GAAG,cAAc,GAAG,OAAO,CAAC,cAAc,CAAC,CAAC;IACpE,OAAO,EAAE,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,CAAC;IAC5B,oEAAoE;IACpE,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAID,iFAAiF;AACjF,MAAM,WAAW,YAAY;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,OAAO,CAAC;CACjB;AAED,gEAAgE;AAChE,MAAM,WAAW,aAAa;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,YAAY,EAAE,CAAC;IACvB,gFAAgF;IAChF,MAAM,CAAC,EAAE,QAAQ,GAAG,QAAQ,GAAG,SAAS,CAAC;IACzC,wDAAwD;IACxD,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,0CAA0C;IAC1C,MAAM,EAAE,OAAO,CAAC;IAChB,0DAA0D;IAC1D,QAAQ,EAAE,MAAM,CAAC;IACjB,UAAU,EAAE,MAAM,CAAC;IACnB,qDAAqD;IACrD,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,8BAA8B;AAC9B,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE,aAAa,EAAE,CAAC;CAC1B"}

package/dist/eval/types.js.map

@@ -1 +1 @@
-{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/eval/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG","sourcesContent":["/**\n * Types for the first-class evals primitive.\n *\n * This is a *test-case* eval system (define a prompt + expected behavior,\n * actually run the agent, score the output) — distinct from the post-hoc\n * run-scoring engine in `../observability/evals.ts`, which scores already-\n * completed production runs. The two are complementary:\n *\n *   - `observability/evals.ts` — \"how did this real run do?\" (passive,\n *     sampled, lives next to traces).\n *   - `eval/*` (this module) — \"does the agent do the right thing on this\n *     fixed input?\" (active, deterministic CI gate, run via the CLI).\n *\n * The pipeline shape (preprocess → analyze → generateScore → generateReason)\n * is borrowed from Mastra's scorer design: each scorer is a small, composable\n * 4-step pipeline so a single scorer can mix plain-JS checks with an LLM\n * judge while still producing one normalized 0..1 number plus a reason.\n */\n\nimport type { AgentEngine } from \"../agent/engine/types.js\";\n\n// ─── Agent run output ─────────────────────────────────────────────────\n\n/**\n * The result of actually running the agent loop for one eval input. Scorers\n * receive this as the thing under test. It is intentionally small and\n * transport-agnostic so a scorer never reaches into framework internals.\n */\nexport interface AgentRunOutput {\n  /** Concatenated assistant text emitted across the run. */\n  readonly text: string;\n  /** Names of tools/actions the agent invoked, in call order. */\n  readonly toolCalls: readonly string[];\n  /** Whether the run completed without a terminal error event. */\n  readonly ok: boolean;\n  /** Terminal error message, if the run errored. */\n  readonly error?: string;\n  /** Synthetic run id, useful for writing eval rows to the observability store. */\n  readonly runId: string;\n  /** Wall-clock duration of the run in milliseconds. */\n  readonly durationMs: number;\n}\n\n// ─── Scorer pipeline ──────────────────────────────────────────────────\n\n/**\n * Context handed to a scorer's analyze step when it needs an LLM judge. The\n * engine/model are resolved by the runner from the existing engine registry —\n * a scorer NEVER hardcodes a provider or model, keeping evals provider-\n * agnostic. `judge()` is a convenience that streams a single judging turn and\n * returns the raw model text.\n */\nexport interface ScorerAnalyzeContext {\n  /** The resolved, provider-agnostic engine for LLM-judge scorers. */\n  readonly engine: AgentEngine;\n  /** The resolved model string for the engine. */\n  readonly model: string;\n  /**\n   * Run a single LLM judging turn. Returns the model's raw text output. Used\n   * by `llmJudge` and any custom LLM-backed analyze step. Provider-agnostic —\n   * the engine is whatever the app/CLI resolved.\n   */\n  judge(opts: {\n    systemPrompt?: string;\n    prompt: string;\n    maxOutputTokens?: number;\n    signal?: AbortSignal;\n  }): Promise<string>;\n}\n\n/**\n * A 4-step scoring pipeline (Mastra-style):\n *\n *   preprocess(run)      → x        (transform the run/output; optional)\n *   analyze(x, ctx)      → analysis (plain JS OR an LLM judge; optional)\n *   generateScore(a)     → 0..1     (REQUIRED, normalized)\n *   generateReason(...)  → string   (human-readable why; optional)\n *\n * Generics flow `Pre` (preprocess output) → `Ana` (analyze output) so a\n * single scorer is fully typed end-to-end.\n */\nexport interface Scorer<Pre = AgentRunOutput, Ana = Pre> {\n  readonly name: string;\n  preprocess?(run: AgentRunOutput): Pre | Promise<Pre>;\n  analyze?(input: Pre, ctx: ScorerAnalyzeContext): Ana | Promise<Ana>;\n  /** REQUIRED. Returns a normalized score in [0, 1]. */\n  generateScore(analysis: Ana): number | Promise<number>;\n  generateReason?(args: {\n    run: AgentRunOutput;\n    analysis: Ana;\n    score: number;\n  }): string | Promise<string>;\n}\n\n/** Definition object passed to `createScorer`. */\nexport interface ScorerDefinition<Pre = AgentRunOutput, Ana = Pre> {\n  name: string;\n  preprocess?(run: AgentRunOutput): Pre | Promise<Pre>;\n  analyze?(input: Pre, ctx: ScorerAnalyzeContext): Ana | Promise<Ana>;\n  generateScore(analysis: Ana): number | Promise<number>;\n  generateReason?(args: {\n    run: AgentRunOutput;\n    analysis: Ana;\n    score: number;\n  }): string | Promise<string>;\n}\n\n// ─── Eval definition ──────────────────────────────────────────────────\n\n/** The prompt + optional setup that drives one eval case. */\nexport interface EvalInput {\n  /** The user prompt / message sent to the agent. */\n  prompt: string;\n  /**\n   * Optional prior conversation turns to seed before `prompt`. Each is a\n   * plain { role, text } pair; the runner converts them to engine messages.\n   */\n  history?: Array<{ role: \"user\" | \"assistant\"; text: string }>;\n}\n\n/** Context passed to an eval's optional `run` override. */\nexport interface EvalRunContext {\n  readonly input: EvalInput;\n  /** The default agent runner — invoke it to run the agent loop as a caller. */\n  runAgent(input: EvalInput): Promise<AgentRunOutput>;\n}\n\n/**\n * A named eval = one test case. `scorers` produce per-scorer rows; the case\n * passes when EVERY scorer meets `threshold` (default 0.5, overridable per\n * eval and globally from the CLI).\n */\nexport interface Eval {\n  name: string;\n  input: EvalInput;\n  /**\n   * Optional override for how the agent is run for this case. Defaults to the\n   * runner's headless `runAgent`. Use this to do custom setup (seed data,\n   * multi-turn) before/after the agent call.\n   */\n  run?(ctx: EvalRunContext): AgentRunOutput | Promise<AgentRunOutput>;\n  scorers: Scorer<any, any>[];\n  /** Minimum acceptable score (per scorer) in [0, 1]. Default 0.5. */\n  threshold?: number;\n}\n\n// ─── Results ──────────────────────────────────────────────────────────\n\n/** One result row per (eval × scorer). Stores both the number AND the reason. */\nexport interface ScorerResult {\n  scorer: string;\n  score: number;\n  reason?: string;\n  passed: boolean;\n}\n\n/** Aggregated result for a single eval (all of its scorers). */\nexport interface EvalResultRow {\n  eval: string;\n  threshold: number;\n  scores: ScorerResult[];\n  /** True only when every scorer passed. */\n  passed: boolean;\n  /** Mean of the scorer scores, for at-a-glance ranking. */\n  avgScore: number;\n  durationMs: number;\n  /** Terminal error if the agent run itself failed. */\n  error?: string;\n}\n\n/** The full runner report. */\nexport interface EvalRunReport {\n  total: number;\n  passed: number;\n  failed: number;\n  results: EvalResultRow[];\n}\n"]}
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/eval/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG","sourcesContent":["/**\n * Types for the first-class evals primitive.\n *\n * This is a *test-case* eval system (define a prompt + expected behavior,\n * actually run the agent, score the output) — distinct from the post-hoc\n * run-scoring engine in `../observability/evals.ts`, which scores already-\n * completed production runs. The two are complementary:\n *\n *   - `observability/evals.ts` — \"how did this real run do?\" (passive,\n *     sampled, lives next to traces).\n *   - `eval/*` (this module) — \"does the agent do the right thing on this\n *     fixed input?\" (active, deterministic CI gate, run via the CLI).\n *\n * The pipeline shape (preprocess → analyze → generateScore → generateReason)\n * is borrowed from Mastra's scorer design: each scorer is a small, composable\n * 4-step pipeline so a single scorer can mix plain-JS checks with an LLM\n * judge while still producing one normalized 0..1 number plus a reason.\n */\n\nimport type { AgentEngine } from \"../agent/engine/types.js\";\n\n// ─── Agent run output ─────────────────────────────────────────────────\n\n/**\n * The result of actually running the agent loop for one eval input. Scorers\n * receive this as the thing under test. It is intentionally small and\n * transport-agnostic so a scorer never reaches into framework internals.\n */\nexport interface AgentRunOutput {\n  /** Concatenated assistant text emitted across the run. */\n  readonly text: string;\n  /** Names of tools/actions the agent invoked, in call order. */\n  readonly toolCalls: readonly string[];\n  /** Whether the run completed without a terminal error event. */\n  readonly ok: boolean;\n  /** Terminal error message, if the run errored. */\n  readonly error?: string;\n  /** Synthetic run id, useful for writing eval rows to the observability store. */\n  readonly runId: string;\n  /** Wall-clock duration of the run in milliseconds. */\n  readonly durationMs: number;\n}\n\n// ─── Scorer pipeline ──────────────────────────────────────────────────\n\n/**\n * Context handed to a scorer's analyze step when it needs an LLM judge. The\n * engine/model are resolved by the runner from the existing engine registry —\n * a scorer NEVER hardcodes a provider or model, keeping evals provider-\n * agnostic. `judge()` is a convenience that streams a single judging turn and\n * returns the raw model text.\n */\nexport interface ScorerAnalyzeContext {\n  /** The resolved, provider-agnostic engine for LLM-judge scorers. */\n  readonly engine: AgentEngine;\n  /** The resolved model string for the engine. */\n  readonly model: string;\n  /**\n   * Run a single LLM judging turn. Returns the model's raw text output. Used\n   * by `llmJudge` and any custom LLM-backed analyze step. Provider-agnostic —\n   * the engine is whatever the app/CLI resolved.\n   */\n  judge(opts: {\n    systemPrompt?: string;\n    prompt: string;\n    maxOutputTokens?: number;\n    signal?: AbortSignal;\n  }): Promise<string>;\n}\n\n/**\n * A 4-step scoring pipeline (Mastra-style):\n *\n *   preprocess(run)      → x        (transform the run/output; optional)\n *   analyze(x, ctx)      → analysis (plain JS OR an LLM judge; optional)\n *   generateScore(a)     → 0..1     (REQUIRED, normalized)\n *   generateReason(...)  → string   (human-readable why; optional)\n *\n * Generics flow `Pre` (preprocess output) → `Ana` (analyze output) so a\n * single scorer is fully typed end-to-end.\n */\nexport interface Scorer<Pre = AgentRunOutput, Ana = Pre> {\n  readonly name: string;\n  preprocess?(run: AgentRunOutput): Pre | Promise<Pre>;\n  analyze?(input: Pre, ctx: ScorerAnalyzeContext): Ana | Promise<Ana>;\n  /** REQUIRED. Returns a normalized score in [0, 1]. */\n  generateScore(analysis: Ana): number | Promise<number>;\n  generateReason?(args: {\n    run: AgentRunOutput;\n    analysis: Ana;\n    score: number;\n  }): string | Promise<string>;\n}\n\n/** Definition object passed to `createScorer`. */\nexport interface ScorerDefinition<Pre = AgentRunOutput, Ana = Pre> {\n  name: string;\n  preprocess?(run: AgentRunOutput): Pre | Promise<Pre>;\n  analyze?(input: Pre, ctx: ScorerAnalyzeContext): Ana | Promise<Ana>;\n  generateScore(analysis: Ana): number | Promise<number>;\n  generateReason?(args: {\n    run: AgentRunOutput;\n    analysis: Ana;\n    score: number;\n  }): string | Promise<string>;\n}\n\n// ─── Eval definition ──────────────────────────────────────────────────\n\n/** The prompt + optional setup that drives one eval case. */\nexport interface EvalInput {\n  /** The user prompt / message sent to the agent. */\n  prompt: string;\n  /**\n   * Optional prior conversation turns to seed before `prompt`. Each is a\n   * plain { role, text } pair; the runner converts them to engine messages.\n   */\n  history?: Array<{ role: \"user\" | \"assistant\"; text: string }>;\n}\n\n/** Context passed to an eval's optional `run` override. */\nexport interface EvalRunContext {\n  readonly input: EvalInput;\n  /** The default agent runner — invoke it to run the agent loop as a caller. */\n  runAgent(input: EvalInput): Promise<AgentRunOutput>;\n}\n\n/**\n * A named eval = one test case. `scorers` produce per-scorer rows; the case\n * passes when EVERY scorer meets `threshold` (default 0.5, overridable per\n * eval and globally from the CLI).\n */\nexport interface Eval {\n  name: string;\n  input: EvalInput;\n  /**\n   * Skip this case without running the agent or scorers. Use for opt-in\n   * suites that need secrets, live provider credentials, or manual/nightly\n   * gates. Skipped rows are reported distinctly and do not fail the suite.\n   */\n  skipReason?: string;\n  /**\n   * Optional override for how the agent is run for this case. Defaults to the\n   * runner's headless `runAgent`. Use this to do custom setup (seed data,\n   * multi-turn) before/after the agent call.\n   */\n  run?(ctx: EvalRunContext): AgentRunOutput | Promise<AgentRunOutput>;\n  scorers: Scorer<any, any>[];\n  /** Minimum acceptable score (per scorer) in [0, 1]. Default 0.5. */\n  threshold?: number;\n}\n\n// ─── Results ──────────────────────────────────────────────────────────\n\n/** One result row per (eval × scorer). Stores both the number AND the reason. */\nexport interface ScorerResult {\n  scorer: string;\n  score: number;\n  reason?: string;\n  passed: boolean;\n}\n\n/** Aggregated result for a single eval (all of its scorers). */\nexport interface EvalResultRow {\n  eval: string;\n  threshold: number;\n  scores: ScorerResult[];\n  /** Explicit row status for reports. Older consumers can still read `passed`. */\n  status?: \"passed\" | \"failed\" | \"skipped\";\n  /** Human-readable reason when `status` is `skipped`. */\n  skipReason?: string;\n  /** True only when every scorer passed. */\n  passed: boolean;\n  /** Mean of the scorer scores, for at-a-glance ranking. */\n  avgScore: number;\n  durationMs: number;\n  /** Terminal error if the agent run itself failed. */\n  error?: string;\n}\n\n/** The full runner report. */\nexport interface EvalRunReport {\n  total: number;\n  passed: number;\n  failed: number;\n  skipped?: number;\n  results: EvalResultRow[];\n}\n"]}

package/dist/file-upload/actions/upload-image.d.ts

@@ -3,18 +3,18 @@ declare const _default: import("../../action.js").ActionDefinition<{
     url?: string;
     filename?: string;
 }, {
+    id?: undefined;
     error: string;
     configured?: undefined;
     connectPath?: undefined;
     url?: undefined;
-    id?: undefined;
     provider?: undefined;
 } | {
+    id?: undefined;
     error: string;
     configured: boolean;
     connectPath: string;
     url?: undefined;
-    id?: undefined;
     provider?: undefined;
 } | {
     error?: undefined;

package/dist/observability/routes.d.ts

@@ -56,13 +56,13 @@ export declare function createObservabilityHandler(): import("h3").EventHandlerW
     ok?: undefined;
     summary?: undefined;
     spans?: undefined;
-    error: any;
     id?: undefined;
+    error: any;
 } | {
     error?: undefined;
     summary?: undefined;
     spans?: undefined;
-    ok: boolean;
     id?: undefined;
+    ok: boolean;
 }>>;
 //# sourceMappingURL=routes.d.ts.map

package/dist/progress/routes.d.ts

@@ -15,7 +15,7 @@ export declare function createProgressHandler(): import("h3").EventHandlerWithFe
     error: string;
     ok?: undefined;
 } | {
-    error?: undefined;
     ok: boolean;
+    error?: undefined;
 }>>;
 //# sourceMappingURL=routes.d.ts.map

package/dist/resources/handlers.d.ts

@@ -49,11 +49,11 @@ export declare function handleUpdateResource(event: any): Promise<import("./stor
 }>;
 /** DELETE /_agent-native/resources/:id — delete a resource */
 export declare function handleDeleteResource(event: any): Promise<{
-    ok?: undefined;
     error: string;
+    ok?: undefined;
 } | {
-    error?: undefined;
     ok: boolean;
+    error?: undefined;
 }>;
 /** POST /_agent-native/resources/upload — upload a file as a resource */
 export declare function handleUploadResource(event: any): Promise<import("./store.js").Resource | {
@@ -73,9 +73,9 @@ export declare function handleUploadResource(event: any): Promise<import("./stor
     runId: string | null;
     expiresAt: number | null;
     metadata: string | null;
-    error?: undefined;
     url: string;
     provider: string;
+    error?: undefined;
 }>;
 export {};
 //# sourceMappingURL=handlers.d.ts.map

package/dist/server/transcribe-voice.d.ts

@@ -20,7 +20,7 @@ export declare function createTranscribeVoiceHandler(): import("h3").EventHandle
     error: string;
     text?: undefined;
 } | {
-    error?: undefined;
     text: string;
+    error?: undefined;
 }>>;
 //# sourceMappingURL=transcribe-voice.d.ts.map

package/dist/templates/workspace-core/.agents/skills/reliable-mutations/SKILL.md

@@ -2,9 +2,9 @@
 name: reliable-mutations
 description: >-
   How the agent must perform writes so they actually persist under the hosted
-  ~40s run budget. Use whenever you create, update, delete, or batch-write app
-  data — especially "do this for many items" loops, or any task where the user
-  expects N things to end up saved.
+  foreground run budget and long-running background handoffs. Use whenever you
+  create, update, delete, or batch-write app data — especially "do this for many
+  items" loops, or any task where the user expects N things to end up saved.
 ---
 
 # Reliable Mutations
@@ -18,15 +18,12 @@ tool ✓ alone.
 
 ## Why
 
-Hosted agent runs have a ~40s soft budget (it exists to hand off cleanly under
-the upstream gateway/function walls — it is correct and is not raisable; see the
-durable-agent-runs design doc). When you loop many sequential writes inside one
-turn, the budget can cut you off mid-loop. The run resumes in a new chunk, but
-the resume often re-does earlier steps instead of advancing, so the loop churns
-and the *saved* result ends up partial — or empty — even though each individual
-tool call appeared to succeed. The user gets told "done" while the data says
-otherwise. One atomic call commits or fails as a unit; verification turns a
-hopeful ✓ into a fact.
+Hosted foreground agent runs have a short soft budget so they can hand off
+cleanly under synchronous serverless walls. Durable background runs get a much
+longer budget, but they still should not rely on loops of many small writes:
+continuations can retry the same intent and leave partial state if each item is
+committed separately. One atomic call commits or fails as a unit; verification
+turns a hopeful ✓ into a fact.
 
 ## How
 
@@ -34,11 +31,12 @@ hopeful ✓ into a fact.
    many, set all, bulk update), pass the full batch in one call so it commits
    atomically. Check the action surface for a batch/plural form before reaching
    for a loop.
-2. **Do not loop many small writes under the run budget.** A sequence of N
-   per-item writes in one turn will race the ~40s cutoff and can leave partial
-   or no state. If no batch action exists, that is a gap in the action layer —
-   add or extend an action that accepts the batch (see the `actions` skill)
-   rather than papering over it with a loop.
+2. **Do not loop many small writes under any run budget.** A sequence of N
+   per-item writes can still leave partial or no state when a foreground run
+   hands off, a background run continues, or an upstream provider fails. If no
+   batch action exists, that is a gap in the action layer — add or extend an
+   action that accepts the batch (see the `actions` skill) rather than papering
+   over it with a loop.
 3. **Verify the end state after writing.** Re-read the data (a list/read action,
    a count query) and confirm the result matches intent — the right number of
    rows, the expected ids/fields. Do this before you tell the user it worked.

package/docs/content/evals.mdx

@@ -90,13 +90,14 @@ export default defineEval({
 
 An eval passes only when **every** scorer meets the threshold. Key `defineEval` fields:
 
-| Field       | Type                  | Notes                                                         |
-| ----------- | --------------------- | ------------------------------------------------------------- |
-| `name`      | string                | Required. Shown in the report.                                |
-| `input`     | `{ prompt, history }` | Required `prompt`; optional prior `{ role, text }` turns.     |
-| `scorers`   | `Scorer[]`            | Required, at least one.                                       |
-| `threshold` | number `0..1`         | Per-scorer pass bar. Default `0.5`; overridable from the CLI. |
-| `run`       | function              | Optional override for custom setup (seed data, multi-turn).   |
+| Field        | Type                  | Notes                                                                                                           |
+| ------------ | --------------------- | --------------------------------------------------------------------------------------------------------------- |
+| `name`       | string                | Required. Shown in the report.                                                                                  |
+| `input`      | `{ prompt, history }` | Required `prompt`; optional prior `{ role, text }` turns.                                                       |
+| `scorers`    | `Scorer[]`            | Required, at least one, unless `skipReason` is set.                                                             |
+| `threshold`  | number `0..1`         | Per-scorer pass bar. Default `0.5`; overridable from the CLI.                                                   |
+| `run`        | function              | Optional override for custom setup (seed data, multi-turn).                                                     |
+| `skipReason` | string                | Optional reason to skip this eval without running the agent; useful for manual/nightly gates that need secrets. |
 
 The agent run handed to scorers is small and transport-agnostic:
 
@@ -225,11 +226,32 @@ The command discovers `**/*.eval.ts` and `evals/*.ts` under the current app, run
 
 Exit codes:
 
-| Code | Meaning                                                         |
-| ---- | --------------------------------------------------------------- |
-| `0`  | All evals passed — _or_ no eval files were found (CI-friendly). |
-| `1`  | At least one eval scored below threshold, or the suite errored. |
-| `2`  | Bad arguments (e.g. `--threshold` outside `[0, 1]`).            |
+| Code | Meaning                                                                          |
+| ---- | -------------------------------------------------------------------------------- |
+| `0`  | All evals passed, evals were skipped, or no eval files were found (CI-friendly). |
+| `1`  | At least one eval scored below threshold, or the suite errored.                  |
+| `2`  | Bad arguments (e.g. `--threshold` outside `[0, 1]`).                             |
+
+### Skipping gated evals {#skipping}
+
+Use `skipReason` when an eval suite is intentionally gated behind manual,
+nightly, or secret-backed execution:
+
+```ts
+export default defineEval({
+  name: "provider-backed source sync",
+  input: { prompt: "Refresh the source and summarize the changes." },
+  skipReason: process.env.SOURCE_SYNC_EVALS
+    ? undefined
+    : "Skipped because SOURCE_SYNC_EVALS is unset",
+  scorers: process.env.SOURCE_SYNC_EVALS ? [contains("updated")] : [],
+});
+```
+
+Skipped evals do not run the agent or scorers, do not fail the suite, and are
+reported distinctly with `status: "skipped"` and `skipReason` in JSON output.
+Use this for expensive or credential-dependent evals; keep cheap deterministic
+coverage in normal PR CI.
 
 ### As a CI deploy gate {#ci}
 

package/docs/content/locales/ar-SA/evals.mdx

@@ -231,6 +231,13 @@ agent-native eval --threshold 0.8    # override every eval's pass threshold (0..
 | `1`   | تم تسجيل تقييم واحد على الأقل أقل من الحد الأدنى، أو حدث خطأ في المجموعة.           |
 | `2`   | وسيطات غير صالحة (على سبيل المثال، `--threshold` خارج `[0, 1]`).                    |
 
+### تخطي التقييمات المحكومة {#skipping}
+
+استخدم `skipReason` عندما تكون مجموعة التقييمات مخصصة عمدا للتشغيل اليدوي أو
+الليلي أو المعتمد على الأسرار. لا تشغل التقييمات المتخطاة الوكيل أو أدوات
+التسجيل، ولا تفشل المجموعة، وتظهر في JSON مع `status: "skipped"` و
+`skipReason`. أبق التغطية الحتمية الرخيصة في CI العادي لطلبات السحب.
+
 ### كبوابة نشر CI {#ci}
 
 قم بإضافته إلى المسار الذي يتم تشغيله قبل النشر:

package/docs/content/locales/de-DE/evals.mdx

@@ -231,6 +231,14 @@ Exit-Codes:
 | `1`  | Mindestens eine Bewertung lag unter dem Schwellenwert oder die Suite ist fehlerhaft.                |
 | `2`  | Ungültige Argumente (z. B. `--threshold` außerhalb von `[0, 1]`).                                   |
 
+### Gated Evals überspringen {#skipping}
+
+Verwenden Sie `skipReason`, wenn eine Eval-Suite absichtlich nur manuell,
+nachts oder mit Secrets ausgeführt werden soll. Übersprungene Evals führen
+weder Agent noch Scorer aus, lassen die Suite nicht fehlschlagen und erscheinen
+in der JSON-Ausgabe mit `status: "skipped"` und `skipReason`. Günstige
+deterministische Abdeckung bleibt im normalen PR-CI.
+
 ### Als CI-Deploy-Gate {#ci}
 
 Fügen Sie es der Pipeline hinzu, die vor einer Bereitstellung ausgeführt wird:

package/docs/content/locales/es-ES/evals.mdx

@@ -231,6 +231,14 @@ Códigos de salida:
 | `1`    | Al menos una evaluación obtuvo una puntuación inferior al umbral o la suite tuvo un error.                |
 | `2`    | Malos argumentos (por ejemplo, `--threshold` fuera de `[0, 1]`).                                          |
 
+### Omitir evaluaciones gated {#skipping}
+
+Use `skipReason` cuando una suite de evaluaciones esté reservada
+intencionalmente para ejecución manual, nocturna o con secretos. Las
+evaluaciones omitidas no ejecutan el agente ni los scorers, no hacen fallar la
+suite y aparecen en JSON con `status: "skipped"` y `skipReason`. Mantenga la
+cobertura determinista barata en el CI normal de PR.
+
 ### Como puerta de implementación de CI {#ci}
 
 Agréguelo al proceso que se ejecuta antes de una implementación:

package/docs/content/locales/fr-FR/evals.mdx

@@ -231,6 +231,14 @@ Codes de sortie :
 | `1`  | Au moins une évaluation a obtenu un score inférieur au seuil, ou la suite a commis une erreur.      |
 | `2`  | Mauvais arguments (par exemple `--threshold` en dehors de `[0, 1]`).                                |
 
+### Ignorer les évaluations gated {#skipping}
+
+Utilisez `skipReason` lorsqu'une suite d'évaluations est volontairement réservée
+à une exécution manuelle, nocturne ou dépendante de secrets. Les évaluations
+ignorées n'exécutent ni l'agent ni les scorers, ne font pas échouer la suite, et
+apparaissent dans la sortie JSON avec `status: "skipped"` et `skipReason`.
+Gardez la couverture déterministe peu coûteuse dans le CI normal des PR.
+
 ### En tant que porte de déploiement CI {#ci}
 
 Ajoutez-le au pipeline qui s'exécute avant un déploiement :

package/docs/content/locales/hi-IN/evals.mdx

@@ -229,6 +229,14 @@ agent-native eval --threshold 0.8    # override every eval's pass threshold (0..
 | `1` | कम से कम एक ईवल ने सीमा से नीचे स्कोर किया, या सुइट में त्रुटि हुई। |
 | `2` | खराब तर्क (उदाहरण के लिए `[0, 1]` के बाहर `--threshold`)।           |
 
+### gated evals छोड़ना {#skipping}
+
+जब कोई eval suite केवल manual, nightly, या secret-backed execution के लिए हो,
+तो `skipReason` का उपयोग करें। छोड़े गए evals agent या scorers नहीं चलाते,
+suite को fail नहीं करते, और JSON output में `status: "skipped"` और
+`skipReason` के साथ दिखते हैं। सस्ती deterministic coverage को सामान्य PR CI
+में रखें।
+
 ### सीआई परिनियोजन गेट के रूप में {#ci}
 
 इसे उस पाइपलाइन में जोड़ें जो परिनियोजन से पहले चलती है:

package/docs/content/locales/ja-JP/evals.mdx

@@ -231,6 +231,13 @@ agent-native eval --threshold 0.8    # override every eval's pass threshold (0..
 | `1`    | 少なくとも 1 つの評価スコアがしきい値を下回ったか、スイートでエラーが発生しました。             |
 | `2`    | 不正な引数 (例: `[0, 1]` の外の `--threshold`)。                                                |
 
+### gated eval をスキップする {#skipping}
+
+手動、夜間、または secret が必要な eval スイートには `skipReason` を使い
+ます。スキップされた eval はエージェントや scorer を実行せず、スイート
+を失敗させず、JSON 出力では `status: "skipped"` と `skipReason` とし
+て表示されます。安価で決定的なカバレッジは通常の PR CI に残します。
+
 ### CI デプロイゲートとして {#ci}
 
 デプロイ前に実行されるパイプラインに追加します。

package/docs/content/locales/ko-KR/evals.mdx

@@ -231,6 +231,13 @@ agent-native eval --threshold 0.8    # override every eval's pass threshold (0..
 | `1`  | 최소 하나의 평가가 임계값 미만의 점수를 얻었거나 제품군에 오류가 발생했습니다.   |
 | `2`  | 잘못된 인수(예: `[0, 1]` 외부의 `--threshold`).                                  |
 
+### 게이트된 평가 건너뛰기 {#skipping}
+
+수동, 야간 또는 시크릿이 필요한 평가 스위트에는 `skipReason`를 사용하세요.
+건너뛴 평가는 에이전트나 scorer를 실행하지 않고, 스위트를 실패시키지
+않으며, JSON 출력에 `status: "skipped"` 및 `skipReason`로 표시됩니다.
+저렴한 결정적 커버리지는 일반 PR CI에 유지하세요.
+
 ### CI 배포 게이트로 {#ci}
 
 배포 전에 실행되는 파이프라인에 추가하세요:

package/docs/content/locales/pt-BR/evals.mdx

@@ -231,6 +231,14 @@ Códigos de saída:
 | `1`    | Pelo menos uma avaliação com pontuação abaixo do limite ou o conjunto apresentou erro.                     |
 | `2`    | Argumentos incorretos (por exemplo, `--threshold` fora de `[0, 1]`).                                       |
 
+### Ignorar evals gated {#skipping}
+
+Use `skipReason` quando uma suíte de evals for intencionalmente reservada para
+execução manual, noturna ou dependente de secrets. Evals ignorados não executam
+o agente nem os scorers, não fazem a suíte falhar e aparecem no JSON com
+`status: "skipped"` e `skipReason`. Mantenha cobertura determinística barata no
+CI normal de PR.
+
 ### Como porta de implantação de CI {#ci}
 
 Adicione-o ao pipeline executado antes da implantação:

package/docs/content/locales/zh-CN/evals.mdx

@@ -229,6 +229,13 @@ agent-native eval --threshold 0.8    # override every eval's pass threshold (0..
 | `1`  | 至少有一项评估得分低于阈值，或者套件出错。            |
 | `2`  | 错误的参数（例如 `--threshold` 位于 `[0, 1]` 之外）。 |
 
+### 跳过受控评估 {#skipping}
+
+当某个评估套件只应在手动、夜间或具备密钥的环境中运行时，请使用
+`skipReason`。被跳过的评估不会运行代理或 scorer，不会让套件失败，并会
+在 JSON 输出中显示 `status: "skipped"` 和 `skipReason`。廉价的确定性覆
+盖应保留在普通 PR CI 中。
+
 ### 作为 CI 部署门 {#ci}
 
 将其添加到部署之前运行的管道：