npm - @jean.gnc/harness-kit - Versions diffs - 0.11.2 → 0.12.1 - Mend

@jean.gnc/harness-kit 0.11.2 → 0.12.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (36) hide show

package/dist/agent/schema.d.ts CHANGED Viewed

@@ -7,13 +7,13 @@ export declare const AgentSchema: z.ZodObject<{
 }, "strict", z.ZodTypeAny, {
     name: string;
     description: string;
-    tools?: string | string[] | undefined;
     model?: string | undefined;
+    tools?: string | string[] | undefined;
 }, {
     name: string;
     description: string;
-    tools?: string | string[] | undefined;
     model?: string | undefined;
+    tools?: string | string[] | undefined;
 }>;
 export type Agent = z.infer<typeof AgentSchema>;
 export declare function defineAgent(agent: Agent): Agent;

package/dist/cli.js CHANGED Viewed

@@ -3,8 +3,12 @@ import { readFileSync } from "node:fs";
 import { fileURLToPath } from "node:url";
 import { defineCommand, runMain } from "citty";
 import { z } from "zod";
+import { mkdir, writeFile } from "node:fs/promises";
+import { dirname } from "node:path";
 import { compile } from "./compile.js";
 import { CHECK_MODES, check } from "./check/index.js";
+import { formatConsole, runEval, toJson } from "./eval/index.js";
+import { TIERS } from "./eval/schema.js";
 import { initHarness } from "./init/index.js";
 import { install, uninstall } from "./install/index.js";
 import { parseInstallMode } from "./install/mode.js";
@@ -166,6 +170,70 @@ const lintCmd = defineCommand({
             process.exit(1);
     },
 });
+function parseTier(value) {
+    if (TIERS.includes(value))
+        return value;
+    throw new Error(`Unknown tier "${value}". Valid: ${TIERS.join(", ")}`);
+}
+function parsePositiveInt(value, flag) {
+    const parsed = Number(value);
+    if (!Number.isInteger(parsed) || parsed < 1) {
+        throw new Error(`--${flag} must be a positive integer, got "${value}"`);
+    }
+    return parsed;
+}
+const evalCmd = defineCommand({
+    meta: {
+        name: "eval",
+        description: "Run skill-routing evals against the installed harness",
+    },
+    args: {
+        cases: {
+            type: "string",
+            default: "./evals/cases",
+            description: "directory of eval case files",
+        },
+        cwd: { type: "string", default: ".", description: "working directory for claude -p sessions" },
+        suite: { type: "string", description: "only run cases from this suite" },
+        case: { type: "string", description: "only run the case with this id" },
+        tier: { type: "string", description: `only run this tier: ${TIERS.join(" | ")}` },
+        runs: { type: "string", description: "runs per case (overrides per-case default)" },
+        concurrency: {
+            type: "string",
+            default: "1",
+            description: "max concurrent sessions (parallel sessions interfere with routing; default 1)",
+        },
+        model: {
+            type: "string",
+            description: "model for claude -p (default: user's configured model)",
+        },
+        json: { type: "string", description: "write machine-readable results to this path" },
+    },
+    run: async ({ args }) => {
+        const result = await runEval({
+            casesDir: args.cases,
+            cwd: args.cwd,
+            concurrency: parsePositiveInt(args.concurrency, "concurrency"),
+            ...(args.suite !== undefined && { suite: args.suite }),
+            ...(args.case !== undefined && { caseId: args.case }),
+            ...(args.tier !== undefined && { tier: parseTier(args.tier) }),
+            ...(args.runs !== undefined && { runs: parsePositiveInt(args.runs, "runs") }),
+            ...(args.model !== undefined && { model: args.model }),
+        });
+        if (!result.ok) {
+            for (const e of result.error)
+                console.error(`${e.file}: ${e.message}`);
+            process.exit(1);
+        }
+        console.log(formatConsole(result.value));
+        if (args.json) {
+            await mkdir(dirname(args.json), { recursive: true });
+            await writeFile(args.json, toJson(result.value) + "\n");
+        }
+        if (result.value.failed > 0)
+            process.exit(1);
+    },
+});
 const main = defineCommand({
     meta: {
         name: "harness",
@@ -175,6 +243,7 @@ const main = defineCommand({
     subCommands: {
         check: checkCmd,
         compile: compileCmd,
+        eval: evalCmd,
         init: initCmd,
         install: installCmd,
         lint: lintCmd,

package/dist/cli.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAEzC,OAAO,EAAE,aAAa,EAAE,OAAO,EAAE,MAAM,OAAO,CAAC;AAC/C,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,WAAW,EAAE,KAAK,EAA2C,MAAM,kBAAkB,CAAC;AAC/F,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAC9C,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AACxD,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,cAAc,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAE7E,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;AAEnE,MAAM,OAAO,GAAG,aAAa,CAAC,IAAI,GAAG,CAAC,iBAAiB,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAC3E,MAAM,GAAG,GAAG,iBAAiB,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC;AAE/E,SAAS,WAAW,CAAC,KAAa;IAChC,OAAQ,WAAiC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;AAC5D,CAAC;AAED,SAAS,cAAc,CAAC,KAAa;IACnC,IAAI,WAAW,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACrC,MAAM,IAAI,KAAK,CAAC,uBAAuB,KAAK,aAAa,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;AACrF,CAAC;AAED,MAAM,UAAU,GAAG,aAAa,CAAC;IAC/B,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,kCAAkC,EAAE;IAC1E,IAAI,EAAE;QACJ,GAAG,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,aAAa,EAAE;QACrE,GAAG,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,aAAa,EAAE;QACtE,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,sCAAsC,EAAE;QAC3F,MAAM,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,sBAAsB,EAAE;KACjF;IACD,GAAG,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE;QACtB,MAAM,OAAO,CAAC;YACZ,OAAO,EAAE,IAAI,CAAC,GAAG;YACjB,OAAO,EAAE,IAAI,CAAC,GAAG;YACjB,QAAQ,EAAE,IAAI,CAAC,IAAI;YACnB,MAAM,EAAE,IAAI,CAAC,MAAM;SACpB,CAAC,CAAC;IACL,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,WAAW,GAAG;IAClB,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,WAAW,EAAE;IACrE,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,sCAAsC,EAAE;IAC3F,MAAM,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,sBAAsB,EAAE;IAChF,SAAS,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,6BAA6B,EAAE;CAClF,CAAC;AAEX,MAAM,UAAU,GAAG,aAAa,CAAC;IAC/B,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,qDAAqD,EAAE;IAC7F,IAAI,EAAE;QACJ,GAAG,WAAW;QACd,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,gCAAgC,EAAE;KAC1F;IACD,GAAG,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE;QACtB,MAAM,IAAI,GAAG,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACzC,MAAM,OAAO,GAAG,MAAM,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvD,MAAM,OAAO,CAAC;YACZ,QAAQ,EAAE,IAAI,CAAC,IAAI;YACnB,QAAQ,EAAE,IAAI,CAAC,IAAI;YACnB,OAAO;YACP,IAAI;YACJ,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC;SACxB,CAAC,CAAC;IACL,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,YAAY,GAAG,aAAa,CAAC;IACjC,IAAI,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,WAAW,EAAE,8CAA8C,EAAE;IACxF,IAAI,EAAE,WAAW;IACjB,GAAG,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE;QACtB,MAAM,OAAO,GAAG,MAAM,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvD,MAAM,SAAS,CAAC;YACd,QAAQ,EAAE,IAAI,CAAC,IAAI;YACnB,QAAQ,EAAE,IAAI,CAAC,IAAI;YACnB,OAAO;YACP,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC;SACxB,CAAC,CAAC;IACL,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,OAAO,GAAG,aAAa,CAAC;IAC5B,IAAI,EAAE;QACJ,IAAI,EAAE,MAAM;QACZ,WAAW,EAAE,2EAA2E;KACzF;IACD,IAAI,EAAE;QACJ,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,4BAA4B,EAAE;QACjF,WAAW,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW,EAAE,kBAAkB,EAAE;QAChF,OAAO,EAAE;YACP,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,IAAI;YACd,WAAW,EAAE,mCAAmC,cAAc,EAAE;iBAC7D,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;iBAClB,IAAI,CAAC,IAAI,CAAC,GAAG;SACjB;QACD,MAAM,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,sBAAsB,EAAE;KACjF;IACD,GAAG,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE;QACtB,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO;aACzB,KAAK,CAAC,GAAG,CAAC;aACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;aACpB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC/B,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;QACrF,cAAc,CAAC,OAAO,EAAE,cAAc,EAAE,CAAC,CAAC;QAC1C,MAAM,WAAW,CAAC;YAChB,QAAQ,EAAE,IAAI,CAAC,IAAI;YACnB,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,OAAO;YACP,MAAM,EAAE,IAAI,CAAC,MAAM;SACpB,CAAC,CAAC;IACL,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,QAAQ,GAAG,aAAa,CAAC;IAC7B,IAAI,EAAE;QACJ,IAAI,EAAE,OAAO;QACb,WAAW,EAAE,8DAA8D;KAC5E;IACD,IAAI,EAAE;QACJ,GAAG,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,aAAa,EAAE;QACrE,IAAI,EAAE;YACJ,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,WAAW;YACpB,WAAW,EAAE,2CAA2C;SACzD;QACD,MAAM,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,2BAA2B,EAAE;KACtF;IACD,GAAG,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE;QACtB,MAAM,IAAI,GAAG,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvC,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC;QACxD,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,MAAM,SAAS,GAAG,MAAM,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC7F,MAAM,KAAK,GAAG,MAAM,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;YAC9E,OAAO,CAAC,GAAG,CACT,WAAW,KAAK,kBAAkB,MAAM,CAAC,cAAc,CAAC,MAAM,aAAa,SAAS,GAAG,CACxF,CAAC;YACF,OAAO,CAAC,GAAG,CAAC,WAAW,MAAM,CAAC,YAAY,eAAe,CAAC,CAAC;YAC3D,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC;gBAAE,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACpD,CAAC;QACD,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;YAClC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC;QAClC,CAAC;QACD,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACjC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;gBACjB,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBAChB,OAAO,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,aAAa,CAAC,CAAC;YACxD,CAAC;YACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;IACH,CAAC;CACF,CAAC,CAAC;AAEH,SAAS,eAAe,CAAC,CAAqB;IAC5C,OAAO,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,MAAM,OAAO,CAAC,CAAC,KAAK,QAAQ,CAAC,CAAC,OAAO,EAAE,CAAC;AAC1E,CAAC;AAED,MAAM,OAAO,GAAG,aAAa,CAAC;IAC5B,IAAI,EAAE;QACJ,IAAI,EAAE,MAAM;QACZ,WAAW,EAAE,qEAAqE;KACnF;IACD,IAAI,EAAE;QACJ,GAAG,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,aAAa,EAAE;QACtE,MAAM,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,2BAA2B,EAAE;KACtF;IACD,GAAG,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE;QACtB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QACtE,IAAI,MAAM,CAAC,UAAU,GAAG,CAAC;YAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC7C,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,IAAI,GAAG,aAAa,CAAC;IACzB,IAAI,EAAE;QACJ,IAAI,EAAE,SAAS;QACf,OAAO,EAAE,GAAG,CAAC,OAAO;QACpB,WAAW,EACT,yFAAyF;KAC5F;IACD,WAAW,EAAE;QACX,KAAK,EAAE,QAAQ;QACf,OAAO,EAAE,UAAU;QACnB,IAAI,EAAE,OAAO;QACb,OAAO,EAAE,UAAU;QACnB,IAAI,EAAE,OAAO;QACb,SAAS,EAAE,YAAY;KACxB;CACF,CAAC,CAAC;AAEH,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC"}
1	+ {"version":3,"file":"cli.js","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AACA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AAEzC,OAAO,EAAE,aAAa,EAAE,OAAO,EAAE,MAAM,OAAO,CAAC;AAC/C,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,OAAO,EAAE,WAAW,EAAE,KAAK,EAA2C,MAAM,kBAAkB,CAAC;AAC/F,OAAO,EAAE,aAAa,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AACjE,OAAO,EAAE,KAAK,EAAa,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAC9C,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AACxD,OAAO,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AACrD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,cAAc,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAE7E,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;AAEnE,MAAM,OAAO,GAAG,aAAa,CAAC,IAAI,GAAG,CAAC,iBAAiB,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAC3E,MAAM,GAAG,GAAG,iBAAiB,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC;AAE/E,SAAS,WAAW,CAAC,KAAa;IAChC,OAAQ,WAAiC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;AAC5D,CAAC;AAED,SAAS,cAAc,CAAC,KAAa;IACnC,IAAI,WAAW,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACrC,MAAM,IAAI,KAAK,CAAC,uBAAuB,KAAK,aAAa,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;AACrF,CAAC;AAED,MAAM,UAAU,GAAG,aAAa,CAAC;IAC/B,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,kCAAkC,EAAE;IAC1E,IAAI,EAAE;QACJ,GAAG,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,aAAa,EAAE;QACrE,GAAG,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,aAAa,EAAE;QACtE,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,sCAAsC,EAAE;QAC3F,MAAM,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,sBAAsB,EAAE;KACjF;IACD,GAAG,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE;QACtB,MAAM,OAAO,CAAC;YACZ,OAAO,EAAE,IAAI,CAAC,GAAG;YACjB,OAAO,EAAE,IAAI,CAAC,GAAG;YACjB,QAAQ,EAAE,IAAI,CAAC,IAAI;YACnB,MAAM,EAAE,IAAI,CAAC,MAAM;SACpB,CAAC,CAAC;IACL,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,WAAW,GAAG;IAClB,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,WAAW,EAAE;IACrE,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,sCAAsC,EAAE;IAC3F,MAAM,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,sBAAsB,EAAE;IAChF,SAAS,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,6BAA6B,EAAE;CAClF,CAAC;AAEX,MAAM,UAAU,GAAG,aAAa,CAAC;IAC/B,IAAI,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,qDAAqD,EAAE;IAC7F,IAAI,EAAE;QACJ,GAAG,WAAW;QACd,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,gCAAgC,EAAE;KAC1F;IACD,GAAG,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE;QACtB,MAAM,IAAI,GAAG,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACzC,MAAM,OAAO,GAAG,MAAM,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvD,MAAM,OAAO,CAAC;YACZ,QAAQ,EAAE,IAAI,CAAC,IAAI;YACnB,QAAQ,EAAE,IAAI,CAAC,IAAI;YACnB,OAAO;YACP,IAAI;YACJ,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC;SACxB,CAAC,CAAC;IACL,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,YAAY,GAAG,aAAa,CAAC;IACjC,IAAI,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,WAAW,EAAE,8CAA8C,EAAE;IACxF,IAAI,EAAE,WAAW;IACjB,GAAG,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE;QACtB,MAAM,OAAO,GAAG,MAAM,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvD,MAAM,SAAS,CAAC;YACd,QAAQ,EAAE,IAAI,CAAC,IAAI;YACnB,QAAQ,EAAE,IAAI,CAAC,IAAI;YACnB,OAAO;YACP,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC;SACxB,CAAC,CAAC;IACL,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,OAAO,GAAG,aAAa,CAAC;IAC5B,IAAI,EAAE;QACJ,IAAI,EAAE,MAAM;QACZ,WAAW,EAAE,2EAA2E;KACzF;IACD,IAAI,EAAE;QACJ,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,4BAA4B,EAAE;QACjF,WAAW,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,WAAW,EAAE,kBAAkB,EAAE;QAChF,OAAO,EAAE;YACP,IAAI,EAAE,QAAQ;YACd,QAAQ,EAAE,IAAI;YACd,WAAW,EAAE,mCAAmC,cAAc,EAAE;iBAC7D,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;iBAClB,IAAI,CAAC,IAAI,CAAC,GAAG;SACjB;QACD,MAAM,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,sBAAsB,EAAE;KACjF;IACD,GAAG,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE;QACtB,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO;aACzB,KAAK,CAAC,GAAG,CAAC;aACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;aACpB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC/B,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;QACrF,cAAc,CAAC,OAAO,EAAE,cAAc,EAAE,CAAC,CAAC;QAC1C,MAAM,WAAW,CAAC;YAChB,QAAQ,EAAE,IAAI,CAAC,IAAI;YACnB,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,OAAO;YACP,MAAM,EAAE,IAAI,CAAC,MAAM;SACpB,CAAC,CAAC;IACL,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,QAAQ,GAAG,aAAa,CAAC;IAC7B,IAAI,EAAE;QACJ,IAAI,EAAE,OAAO;QACb,WAAW,EAAE,8DAA8D;KAC5E;IACD,IAAI,EAAE;QACJ,GAAG,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,aAAa,EAAE;QACrE,IAAI,EAAE;YACJ,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,WAAW;YACpB,WAAW,EAAE,2CAA2C;SACzD;QACD,MAAM,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,2BAA2B,EAAE;KACtF;IACD,GAAG,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE;QACtB,MAAM,IAAI,GAAG,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvC,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC;QACxD,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,MAAM,SAAS,GAAG,MAAM,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAC7F,MAAM,KAAK,GAAG,MAAM,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;YAC9E,OAAO,CAAC,GAAG,CACT,WAAW,KAAK,kBAAkB,MAAM,CAAC,cAAc,CAAC,MAAM,aAAa,SAAS,GAAG,CACxF,CAAC;YACF,OAAO,CAAC,GAAG,CAAC,WAAW,MAAM,CAAC,YAAY,eAAe,CAAC,CAAC;YAC3D,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC;gBAAE,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QACpD,CAAC;QACD,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;YAClC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC;QAClC,CAAC;QACD,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACjC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;gBACjB,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBAChB,OAAO,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,UAAU,CAAC,MAAM,aAAa,CAAC,CAAC;YACxD,CAAC;YACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;IACH,CAAC;CACF,CAAC,CAAC;AAEH,SAAS,eAAe,CAAC,CAAqB;IAC5C,OAAO,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,MAAM,OAAO,CAAC,CAAC,KAAK,QAAQ,CAAC,CAAC,OAAO,EAAE,CAAC;AAC1E,CAAC;AAED,MAAM,OAAO,GAAG,aAAa,CAAC;IAC5B,IAAI,EAAE;QACJ,IAAI,EAAE,MAAM;QACZ,WAAW,EAAE,qEAAqE;KACnF;IACD,IAAI,EAAE;QACJ,GAAG,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,aAAa,EAAE;QACtE,MAAM,EAAE,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,2BAA2B,EAAE;KACtF;IACD,GAAG,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE;QACtB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,EAAE,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;QACtE,IAAI,MAAM,CAAC,UAAU,GAAG,CAAC;YAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC7C,CAAC;CACF,CAAC,CAAC;AAEH,SAAS,SAAS,CAAC,KAAa;IAC9B,IAAK,KAA2B,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,KAAa,CAAC;IACvE,MAAM,IAAI,KAAK,CAAC,iBAAiB,KAAK,aAAa,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;AACzE,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAa,EAAE,IAAY;IACnD,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC7B,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,KAAK,IAAI,qCAAqC,KAAK,GAAG,CAAC,CAAC;IAC1E,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,OAAO,GAAG,aAAa,CAAC;IAC5B,IAAI,EAAE;QACJ,IAAI,EAAE,MAAM;QACZ,WAAW,EAAE,uDAAuD;KACrE;IACD,IAAI,EAAE;QACJ,KAAK,EAAE;YACL,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,eAAe;YACxB,WAAW,EAAE,8BAA8B;SAC5C;QACD,GAAG,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,EAAE,WAAW,EAAE,0CAA0C,EAAE;QAC9F,KAAK,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,gCAAgC,EAAE;QACxE,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,gCAAgC,EAAE;QACvE,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,uBAAuB,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE;QACjF,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,4CAA4C,EAAE;QACnF,WAAW,EAAE;YACX,IAAI,EAAE,QAAQ;YACd,OAAO,EAAE,GAAG;YACZ,WAAW,EAAE,+EAA+E;SAC7F;QACD,KAAK,EAAE;YACL,IAAI,EAAE,QAAQ;YACd,WAAW,EAAE,wDAAwD;SACtE;QACD,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,WAAW,EAAE,6CAA6C,EAAE;KACrF;IACD,GAAG,EAAE,KAAK,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE;QACtB,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC;YAC3B,QAAQ,EAAE,IAAI,CAAC,KAAK;YACpB,GAAG,EAAE,IAAI,CAAC,GAAG;YACb,WAAW,EAAE,gBAAgB,CAAC,IAAI,CAAC,WAAW,EAAE,aAAa,CAAC;YAC9D,GAAG,CAAC,IAAI,CAAC,KAAK,KAAK,SAAS,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC;YACtD,GAAG,CAAC,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC;YACrD,GAAG,CAAC,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,EAAE,IAAI,EAAE,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YAC9D,GAAG,CAAC,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,EAAE,IAAI,EAAE,gBAAgB,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,EAAE,CAAC;YAC7E,GAAG,CAAC,IAAI,CAAC,KAAK,KAAK,SAAS,IAAI,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC;SACvD,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;YACf,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,KAAK;gBAAE,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC;YACvE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;QACzC,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACd,MAAM,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YACrD,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,CAAC;QAC1D,CAAC;QACD,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC;YAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC/C,CAAC;CACF,CAAC,CAAC;AAEH,MAAM,IAAI,GAAG,aAAa,CAAC;IACzB,IAAI,EAAE;QACJ,IAAI,EAAE,SAAS;QACf,OAAO,EAAE,GAAG,CAAC,OAAO;QACpB,WAAW,EACT,yFAAyF;KAC5F;IACD,WAAW,EAAE;QACX,KAAK,EAAE,QAAQ;QACf,OAAO,EAAE,UAAU;QACnB,IAAI,EAAE,OAAO;QACb,IAAI,EAAE,OAAO;QACb,OAAO,EAAE,UAAU;QACnB,IAAI,EAAE,OAAO;QACb,SAAS,EAAE,YAAY;KACxB;CACF,CAAC,CAAC;AAEH,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC"}

package/dist/eval/cases.d.ts ADDED Viewed

@@ -0,0 +1,14 @@
+import { type Result } from "../result.js";
+import { type EvalCase, type Tier } from "./schema.js";
+export interface LoadedCase extends EvalCase {
+    readonly suite: string;
+    readonly tier: Tier;
+    readonly file: string;
+}
+export interface CaseLoadError {
+    readonly file: string;
+    readonly message: string;
+}
+export declare function loadCases(casesDir: string): Promise<Result<LoadedCase[], CaseLoadError[]>>;
+export declare function unresolvedSkills(cases: readonly LoadedCase[], installedIds: ReadonlySet<string>): CaseLoadError[];
+//# sourceMappingURL=cases.d.ts.map

package/dist/eval/cases.d.ts.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"cases.d.ts","sourceRoot":"","sources":["../../src/eval/cases.ts"],"names":[],"mappings":"AAMA,OAAO,EAAW,KAAK,MAAM,EAAE,MAAM,cAAc,CAAC;AACpD,OAAO,EAIL,KAAK,QAAQ,EACb,KAAK,IAAI,EACV,MAAM,aAAa,CAAC;AAErB,MAAM,WAAW,UAAW,SAAQ,QAAQ;IAC1C,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC;IACpB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B;AAED,wBAAsB,SAAS,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,UAAU,EAAE,EAAE,aAAa,EAAE,CAAC,CAAC,CAuBhG;AAED,wBAAgB,gBAAgB,CAC9B,KAAK,EAAE,SAAS,UAAU,EAAE,EAC5B,YAAY,EAAE,WAAW,CAAC,MAAM,CAAC,GAChC,aAAa,EAAE,CAajB"}

package/dist/eval/cases.js ADDED Viewed

@@ -0,0 +1,84 @@
+import { readdir, readFile } from "node:fs/promises";
+import { join } from "node:path";
+import yaml from "js-yaml";
+import { formatZodIssues } from "../errors/zod.js";
+import { err, ok } from "../result.js";
+import { CaseFileSchema, expectedSkills, } from "./schema.js";
+export async function loadCases(casesDir) {
+    const files = await collectYamlFiles(casesDir);
+    const loaded = [];
+    const errors = [];
+    const seenIds = new Set();
+    for (const file of files) {
+        const parsed = await parseFile(file);
+        if (!parsed.ok) {
+            errors.push(parsed.error);
+            continue;
+        }
+        for (const evalCase of parsed.value.cases) {
+            if (seenIds.has(evalCase.id)) {
+                errors.push({ file, message: `duplicate case id "${evalCase.id}"` });
+                continue;
+            }
+            seenIds.add(evalCase.id);
+            loaded.push({ ...evalCase, suite: parsed.value.suite, tier: parsed.value.tier, file });
+        }
+    }
+    return errors.length > 0 ? err(errors) : ok(loaded);
+}
+export function unresolvedSkills(cases, installedIds) {
+    const errors = [];
+    for (const evalCase of cases) {
+        for (const id of expectedSkills(evalCase.expect)) {
+            if (!installedIds.has(id)) {
+                errors.push({
+                    file: evalCase.file,
+                    message: `case "${evalCase.id}" expects skill "${id}", which is not installed`,
+                });
+            }
+        }
+    }
+    return errors;
+}
+async function parseFile(file) {
+    let raw;
+    try {
+        raw = await readFile(file, "utf8");
+    }
+    catch (cause) {
+        return err({ file, message: `cannot read file: ${cause.message}` });
+    }
+    let doc;
+    try {
+        doc = yaml.load(raw);
+    }
+    catch (cause) {
+        return err({ file, message: `invalid YAML: ${cause.message}` });
+    }
+    const result = CaseFileSchema.safeParse(doc);
+    if (!result.success) {
+        return err({ file, message: formatZodIssues(result.error).join("; ") });
+    }
+    return ok(result.data);
+}
+async function collectYamlFiles(dir) {
+    const files = [];
+    let entries;
+    try {
+        entries = await readdir(dir, { withFileTypes: true });
+    }
+    catch {
+        return files;
+    }
+    for (const entry of entries) {
+        const full = join(dir, entry.name);
+        if (entry.isDirectory()) {
+            files.push(...(await collectYamlFiles(full)));
+        }
+        else if (entry.isFile() && (entry.name.endsWith(".yaml") || entry.name.endsWith(".yml"))) {
+            files.push(full);
+        }
+    }
+    return files.sort();
+}
+//# sourceMappingURL=cases.js.map

package/dist/eval/cases.js.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"cases.js","sourceRoot":"","sources":["../../src/eval/cases.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AACrD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,OAAO,IAAI,MAAM,SAAS,CAAC;AAE3B,OAAO,EAAE,eAAe,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,EAAE,GAAG,EAAE,EAAE,EAAe,MAAM,cAAc,CAAC;AACpD,OAAO,EACL,cAAc,EACd,cAAc,GAIf,MAAM,aAAa,CAAC;AAarB,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,QAAgB;IAC9C,MAAM,KAAK,GAAG,MAAM,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IAC/C,MAAM,MAAM,GAAiB,EAAE,CAAC;IAChC,MAAM,MAAM,GAAoB,EAAE,CAAC;IACnC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAElC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,CAAC;QACrC,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;YACf,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAC1B,SAAS;QACX,CAAC;QACD,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;YAC1C,IAAI,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC;gBAC7B,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,sBAAsB,QAAQ,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC;gBACrE,SAAS;YACX,CAAC;YACD,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;YACzB,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,QAAQ,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,IAAI,EAAE,MAAM,CAAC,KAAK,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QACzF,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC;AACtD,CAAC;AAED,MAAM,UAAU,gBAAgB,CAC9B,KAA4B,EAC5B,YAAiC;IAEjC,MAAM,MAAM,GAAoB,EAAE,CAAC;IACnC,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE,CAAC;QAC7B,KAAK,MAAM,EAAE,IAAI,cAAc,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;YACjD,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;gBAC1B,MAAM,CAAC,IAAI,CAAC;oBACV,IAAI,EAAE,QAAQ,CAAC,IAAI;oBACnB,OAAO,EAAE,SAAS,QAAQ,CAAC,EAAE,oBAAoB,EAAE,2BAA2B;iBAC/E,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,KAAK,UAAU,SAAS,CAAC,IAAY;IACnC,IAAI,GAAW,CAAC;IAChB,IAAI,CAAC;QACH,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IACrC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,GAAG,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,qBAAsB,KAAe,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;IACjF,CAAC;IACD,IAAI,GAAY,CAAC;IACjB,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACvB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,GAAG,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,iBAAkB,KAAe,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;IAC7E,CAAC;IACD,MAAM,MAAM,GAAG,cAAc,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IAC7C,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,OAAO,GAAG,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,eAAe,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC1E,CAAC;IACD,OAAO,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;AACzB,CAAC;AAED,KAAK,UAAU,gBAAgB,CAAC,GAAW;IACzC,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,OAAO,CAAC;IACZ,IAAI,CAAC;QACH,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IACxD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;IACD,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QACnC,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;YACxB,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,gBAAgB,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAChD,CAAC;aAAM,IAAI,KAAK,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC;YAC3F,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACnB,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,EAAE,CAAC;AACtB,CAAC"}

package/dist/eval/detect.d.ts ADDED Viewed

@@ -0,0 +1,14 @@
+export type ExitReason = "skill" | "no-skill" | "timeout" | "stream-end";
+export interface DetectionResult {
+    readonly observed: readonly string[];
+    readonly firstSkill: string | null;
+    readonly exitReason: ExitReason;
+}
+interface Detector {
+    readonly push: (line: string) => void;
+    readonly done: boolean;
+    readonly result: (reason: ExitReason) => DetectionResult;
+}
+export declare function createDetector(stopAfter?: number): Detector;
+export {};
+//# sourceMappingURL=detect.d.ts.map

package/dist/eval/detect.d.ts.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"detect.d.ts","sourceRoot":"","sources":["../../src/eval/detect.ts"],"names":[],"mappings":"AAEA,MAAM,MAAM,UAAU,GAAG,OAAO,GAAG,UAAU,GAAG,SAAS,GAAG,YAAY,CAAC;AAEzE,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,CAAC;IACrC,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,QAAQ,CAAC,UAAU,EAAE,UAAU,CAAC;CACjC;AAED,UAAU,QAAQ;IAChB,QAAQ,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;IACtC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,CAAC,MAAM,EAAE,UAAU,KAAK,eAAe,CAAC;CAC1D;AAwBD,wBAAgB,cAAc,CAAC,SAAS,SAAI,GAAG,QAAQ,CA6EtD"}

package/dist/eval/detect.js ADDED Viewed

@@ -0,0 +1,105 @@
+import { FQ_ID } from "../ids.js";
+function asRecord(value) {
+    return typeof value === "object" && value !== null ? value : {};
+}
+function isSkillToolUse(block) {
+    return block["type"] === "tool_use" && block["name"] === "Skill";
+}
+function skillIdOf(input) {
+    const record = asRecord(input);
+    const value = record["skill"] ?? record["command"];
+    return typeof value === "string" && FQ_ID.test(value) ? value : null;
+}
+function skillIdInJson(buffer) {
+    try {
+        return skillIdOf(JSON.parse(buffer));
+    }
+    catch {
+        return null;
+    }
+}
+export function createDetector(stopAfter = 1) {
+    const observed = [];
+    let pendingSkillJson = null;
+    let finished = false;
+    function record(id) {
+        if (id !== null)
+            observed.push(id);
+        if (observed.length >= stopAfter)
+            finished = true;
+        return finished;
+    }
+    function flushPending() {
+        if (pendingSkillJson === null)
+            return;
+        record(skillIdInJson(pendingSkillJson));
+        pendingSkillJson = null;
+    }
+    return {
+        get done() {
+            return finished;
+        },
+        push(line) {
+            if (finished)
+                return;
+            const trimmed = line.trim();
+            if (!trimmed)
+                return;
+            let parsed;
+            try {
+                parsed = JSON.parse(trimmed);
+            }
+            catch {
+                return;
+            }
+            const event = asRecord(parsed);
+            const type = event["type"];
+            if (type === "stream_event") {
+                const se = asRecord(event["event"]);
+                const seType = se["type"];
+                if (seType === "content_block_start") {
+                    if (isSkillToolUse(asRecord(se["content_block"])))
+                        pendingSkillJson = "";
+                }
+                else if (seType === "content_block_delta" && pendingSkillJson !== null) {
+                    const delta = asRecord(se["delta"]);
+                    if (delta["type"] === "input_json_delta") {
+                        const partial = delta["partial_json"];
+                        if (typeof partial === "string")
+                            pendingSkillJson += partial;
+                        if (skillIdInJson(pendingSkillJson) !== null)
+                            flushPending();
+                    }
+                }
+                else if (seType === "content_block_stop") {
+                    flushPending();
+                }
+                else if (seType === "message_stop") {
+                    flushPending();
+                    finished = true;
+                }
+                return;
+            }
+            if (type === "assistant") {
+                const content = asRecord(event["message"])["content"];
+                for (const item of Array.isArray(content) ? content : []) {
+                    const block = asRecord(item);
+                    if (isSkillToolUse(block) && record(skillIdOf(block["input"])))
+                        return;
+                }
+                return;
+            }
+            if (type === "result") {
+                finished = true;
+            }
+        },
+        result(reason) {
+            return {
+                observed,
+                firstSkill: observed[0] ?? null,
+                exitReason: observed.length > 0 ? "skill" : reason,
+            };
+        },
+    };
+}
+//# sourceMappingURL=detect.js.map

package/dist/eval/detect.js.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"detect.js","sourceRoot":"","sources":["../../src/eval/detect.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,WAAW,CAAC;AAgBlC,SAAS,QAAQ,CAAC,KAAc;IAC9B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,CAAC,CAAC,CAAE,KAAiC,CAAC,CAAC,CAAC,EAAE,CAAC;AAC/F,CAAC;AAED,SAAS,cAAc,CAAC,KAA8B;IACpD,OAAO,KAAK,CAAC,MAAM,CAAC,KAAK,UAAU,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,OAAO,CAAC;AACnE,CAAC;AAED,SAAS,SAAS,CAAC,KAAc;IAC/B,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;IAC/B,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,IAAI,MAAM,CAAC,SAAS,CAAC,CAAC;IACnD,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;AACvE,CAAC;AAED,SAAS,aAAa,CAAC,MAAc;IACnC,IAAI,CAAC;QACH,OAAO,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;IACvC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,SAAS,GAAG,CAAC;IAC1C,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,IAAI,gBAAgB,GAAkB,IAAI,CAAC;IAC3C,IAAI,QAAQ,GAAG,KAAK,CAAC;IAErB,SAAS,MAAM,CAAC,EAAiB;QAC/B,IAAI,EAAE,KAAK,IAAI;YAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACnC,IAAI,QAAQ,CAAC,MAAM,IAAI,SAAS;YAAE,QAAQ,GAAG,IAAI,CAAC;QAClD,OAAO,QAAQ,CAAC;IAClB,CAAC;IAED,SAAS,YAAY;QACnB,IAAI,gBAAgB,KAAK,IAAI;YAAE,OAAO;QACtC,MAAM,CAAC,aAAa,CAAC,gBAAgB,CAAC,CAAC,CAAC;QACxC,gBAAgB,GAAG,IAAI,CAAC;IAC1B,CAAC;IAED,OAAO;QACL,IAAI,IAAI;YACN,OAAO,QAAQ,CAAC;QAClB,CAAC;QACD,IAAI,CAAC,IAAY;YACf,IAAI,QAAQ;gBAAE,OAAO;YACrB,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;YAC5B,IAAI,CAAC,OAAO;gBAAE,OAAO;YACrB,IAAI,MAAe,CAAC;YACpB,IAAI,CAAC;gBACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAC/B,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO;YACT,CAAC;YACD,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC;YAC/B,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;YAE3B,IAAI,IAAI,KAAK,cAAc,EAAE,CAAC;gBAC5B,MAAM,EAAE,GAAG,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;gBACpC,MAAM,MAAM,GAAG,EAAE,CAAC,MAAM,CAAC,CAAC;gBAE1B,IAAI,MAAM,KAAK,qBAAqB,EAAE,CAAC;oBACrC,IAAI,cAAc,CAAC,QAAQ,CAAC,EAAE,CAAC,eAAe,CAAC,CAAC,CAAC;wBAAE,gBAAgB,GAAG,EAAE,CAAC;gBAC3E,CAAC;qBAAM,IAAI,MAAM,KAAK,qBAAqB,IAAI,gBAAgB,KAAK,IAAI,EAAE,CAAC;oBACzE,MAAM,KAAK,GAAG,QAAQ,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC;oBACpC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,kBAAkB,EAAE,CAAC;wBACzC,MAAM,OAAO,GAAG,KAAK,CAAC,cAAc,CAAC,CAAC;wBACtC,IAAI,OAAO,OAAO,KAAK,QAAQ;4BAAE,gBAAgB,IAAI,OAAO,CAAC;wBAC7D,IAAI,aAAa,CAAC,gBAAgB,CAAC,KAAK,IAAI;4BAAE,YAAY,EAAE,CAAC;oBAC/D,CAAC;gBACH,CAAC;qBAAM,IAAI,MAAM,KAAK,oBAAoB,EAAE,CAAC;oBAC3C,YAAY,EAAE,CAAC;gBACjB,CAAC;qBAAM,IAAI,MAAM,KAAK,cAAc,EAAE,CAAC;oBACrC,YAAY,EAAE,CAAC;oBACf,QAAQ,GAAG,IAAI,CAAC;gBAClB,CAAC;gBACD,OAAO;YACT,CAAC;YAED,IAAI,IAAI,KAAK,WAAW,EAAE,CAAC;gBACzB,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;gBACtD,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;oBACzD,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;oBAC7B,IAAI,cAAc,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;wBAAE,OAAO;gBACzE,CAAC;gBACD,OAAO;YACT,CAAC;YAED,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;gBACtB,QAAQ,GAAG,IAAI,CAAC;YAClB,CAAC;QACH,CAAC;QACD,MAAM,CAAC,MAAkB;YACvB,OAAO;gBACL,QAAQ;gBACR,UAAU,EAAE,QAAQ,CAAC,CAAC,CAAC,IAAI,IAAI;gBAC/B,UAAU,EAAE,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM;aACnD,CAAC;QACJ,CAAC;KACF,CAAC;AACJ,CAAC"}

package/dist/eval/index.d.ts ADDED Viewed

@@ -0,0 +1,20 @@
+import { type Result } from "../result.js";
+import { type CaseLoadError, type LoadedCase } from "./cases.js";
+import { type EvalReport } from "./report.js";
+import { type RunnerOptions } from "./runner.js";
+export interface EvalOptions {
+    readonly casesDir: string;
+    readonly cwd: string;
+    readonly suite?: string;
+    readonly caseId?: string;
+    readonly tier?: LoadedCase["tier"];
+    readonly runs?: number;
+    readonly concurrency?: number;
+    readonly model?: string;
+    readonly onRun?: RunnerOptions["onRun"];
+}
+export declare function runEval(options: EvalOptions): Promise<Result<EvalReport, CaseLoadError[]>>;
+export type { EvalReport, CaseReport } from "./report.js";
+export { formatConsole, toJson } from "./report.js";
+export type { LoadedCase, CaseLoadError } from "./cases.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/eval/index.d.ts.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/eval/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAW,KAAK,MAAM,EAAE,MAAM,cAAc,CAAC;AACpD,OAAO,EAA+B,KAAK,aAAa,EAAE,KAAK,UAAU,EAAE,MAAM,YAAY,CAAC;AAC9F,OAAO,EAAgC,KAAK,UAAU,EAAE,MAAM,aAAa,CAAC;AAC5E,OAAO,EAAY,KAAK,aAAa,EAAE,MAAM,aAAa,CAAC;AAG3D,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,IAAI,CAAC,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;IACnC,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,KAAK,CAAC,EAAE,aAAa,CAAC,OAAO,CAAC,CAAC;CACzC;AAED,wBAAsB,OAAO,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,UAAU,EAAE,aAAa,EAAE,CAAC,CAAC,CA6BhG;AAWD,YAAY,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAC1D,OAAO,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACpD,YAAY,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC"}

package/dist/eval/index.js ADDED Viewed

@@ -0,0 +1,46 @@
+import { defaultSources, discoverInstalled, indexInstalled } from "../installed.js";
+import { err, ok } from "../result.js";
+import { loadCases, unresolvedSkills } from "./cases.js";
+import { buildReport } from "./report.js";
+import { runCases } from "./runner.js";
+import { scoreCase } from "./score.js";
+export async function runEval(options) {
+    const loaded = await loadCases(options.casesDir);
+    if (!loaded.ok)
+        return err(loaded.error);
+    const selected = select(loaded.value, options);
+    if (selected.length === 0) {
+        return err([{ file: options.casesDir, message: "no cases matched the given filters" }]);
+    }
+    const installed = indexInstalled(await discoverInstalled(defaultSources()));
+    const installedIds = new Set(installed.skills.keys());
+    const unresolved = unresolvedSkills(selected, installedIds);
+    if (unresolved.length > 0)
+        return err(unresolved);
+    const runnerOptions = {
+        cwd: options.cwd,
+        ...(options.runs !== undefined && { runs: options.runs }),
+        ...(options.concurrency !== undefined && { concurrency: options.concurrency }),
+        ...(options.model !== undefined && { model: options.model }),
+        ...(options.onRun !== undefined && { onRun: options.onRun }),
+    };
+    const results = await runCases(selected, runnerOptions);
+    const reports = results.map(({ evalCase, runs }) => ({
+        evalCase,
+        score: scoreCase(evalCase.expect, runs, evalCase.threshold),
+    }));
+    return ok(buildReport(reports));
+}
+function select(cases, options) {
+    return cases.filter((evalCase) => {
+        if (options.caseId && evalCase.id !== options.caseId)
+            return false;
+        if (options.suite && evalCase.suite !== options.suite)
+            return false;
+        if (options.tier && evalCase.tier !== options.tier)
+            return false;
+        return true;
+    });
+}
+export { formatConsole, toJson } from "./report.js";
+//# sourceMappingURL=index.js.map

package/dist/eval/index.js.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/eval/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,iBAAiB,EAAE,cAAc,EAAE,MAAM,iBAAiB,CAAC;AACpF,OAAO,EAAE,GAAG,EAAE,EAAE,EAAe,MAAM,cAAc,CAAC;AACpD,OAAO,EAAE,SAAS,EAAE,gBAAgB,EAAuC,MAAM,YAAY,CAAC;AAC9F,OAAO,EAAE,WAAW,EAAoC,MAAM,aAAa,CAAC;AAC5E,OAAO,EAAE,QAAQ,EAAsB,MAAM,aAAa,CAAC;AAC3D,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAcvC,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,OAAoB;IAChD,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IACjD,IAAI,CAAC,MAAM,CAAC,EAAE;QAAE,OAAO,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAEzC,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IAC/C,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,OAAO,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,QAAQ,EAAE,OAAO,EAAE,oCAAoC,EAAE,CAAC,CAAC,CAAC;IAC1F,CAAC;IAED,MAAM,SAAS,GAAG,cAAc,CAAC,MAAM,iBAAiB,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC;IAC5E,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;IACtD,MAAM,UAAU,GAAG,gBAAgB,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;IAC5D,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,GAAG,CAAC,UAAU,CAAC,CAAC;IAElD,MAAM,aAAa,GAAkB;QACnC,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,GAAG,CAAC,OAAO,CAAC,IAAI,KAAK,SAAS,IAAI,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC;QACzD,GAAG,CAAC,OAAO,CAAC,WAAW,KAAK,SAAS,IAAI,EAAE,WAAW,EAAE,OAAO,CAAC,WAAW,EAAE,CAAC;QAC9E,GAAG,CAAC,OAAO,CAAC,KAAK,KAAK,SAAS,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC;QAC5D,GAAG,CAAC,OAAO,CAAC,KAAK,KAAK,SAAS,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAC;KAC7D,CAAC;IACF,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;IAExD,MAAM,OAAO,GAAiB,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;QACjE,QAAQ;QACR,KAAK,EAAE,SAAS,CAAC,QAAQ,CAAC,MAAM,EAAE,IAAI,EAAE,QAAQ,CAAC,SAAS,CAAC;KAC5D,CAAC,CAAC,CAAC;IAEJ,OAAO,EAAE,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC;AAClC,CAAC;AAED,SAAS,MAAM,CAAC,KAA4B,EAAE,OAAoB;IAChE,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE;QAC/B,IAAI,OAAO,CAAC,MAAM,IAAI,QAAQ,CAAC,EAAE,KAAK,OAAO,CAAC,MAAM;YAAE,OAAO,KAAK,CAAC;QACnE,IAAI,OAAO,CAAC,KAAK,IAAI,QAAQ,CAAC,KAAK,KAAK,OAAO,CAAC,KAAK;YAAE,OAAO,KAAK,CAAC;QACpE,IAAI,OAAO,CAAC,IAAI,IAAI,QAAQ,CAAC,IAAI,KAAK,OAAO,CAAC,IAAI;YAAE,OAAO,KAAK,CAAC;QACjE,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;AACL,CAAC;AAGD,OAAO,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC"}

package/dist/eval/report.d.ts ADDED Viewed

@@ -0,0 +1,15 @@
+import type { LoadedCase } from "./cases.js";
+import type { CaseScore } from "./score.js";
+export interface CaseReport {
+    readonly evalCase: LoadedCase;
+    readonly score: CaseScore;
+}
+export interface EvalReport {
+    readonly cases: readonly CaseReport[];
+    readonly passed: number;
+    readonly failed: number;
+}
+export declare function buildReport(cases: readonly CaseReport[]): EvalReport;
+export declare function formatConsole(report: EvalReport): string;
+export declare function toJson(report: EvalReport): string;
+//# sourceMappingURL=report.d.ts.map

package/dist/eval/report.d.ts.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"report.d.ts","sourceRoot":"","sources":["../../src/eval/report.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAG5C,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,QAAQ,EAAE,UAAU,CAAC;IAC9B,QAAQ,CAAC,KAAK,EAAE,SAAS,CAAC;CAC3B;AAED,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,KAAK,EAAE,SAAS,UAAU,EAAE,CAAC;IACtC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED,wBAAgB,WAAW,CAAC,KAAK,EAAE,SAAS,UAAU,EAAE,GAAG,UAAU,CAGpE;AAED,wBAAgB,aAAa,CAAC,MAAM,EAAE,UAAU,GAAG,MAAM,CAuBxD;AAED,wBAAgB,MAAM,CAAC,MAAM,EAAE,UAAU,GAAG,MAAM,CAyBjD"}

package/dist/eval/report.js ADDED Viewed

@@ -0,0 +1,81 @@
+export function buildReport(cases) {
+    const passed = cases.filter((c) => c.score.pass).length;
+    return { cases, passed, failed: cases.length - passed };
+}
+export function formatConsole(report) {
+    const lines = [];
+    const groups = groupBy(report.cases, (c) => `${c.evalCase.tier}/${c.evalCase.suite}`);
+    for (const [group, entries] of groups) {
+        lines.push(group);
+        for (const { evalCase, score } of entries) {
+            const tag = score.pass ? "PASS" : "FAIL";
+            const tally = `${score.matched}/${score.runs}`;
+            lines.push(`  ${tag}  ${evalCase.id}  ${tally}  → ${describeExpectation(evalCase.expect)}`);
+            if (!score.pass) {
+                lines.push(`          got: ${formatHistogram(score.histogram)}`);
+                lines.push(`          prompt: ${truncate(evalCase.prompt)}`);
+                if (evalCase.note)
+                    lines.push(`          note: ${evalCase.note}`);
+            }
+        }
+        lines.push("");
+    }
+    const total = report.passed + report.failed;
+    const pct = total === 0 ? 0 : Math.round((report.passed / total) * 100);
+    lines.push(`Summary: ${report.passed}/${total} cases passed (${pct}%).`);
+    return lines.join("\n");
+}
+export function toJson(report) {
+    return JSON.stringify({
+        summary: {
+            total: report.passed + report.failed,
+            passed: report.passed,
+            failed: report.failed,
+        },
+        cases: report.cases.map(({ evalCase, score }) => ({
+            id: evalCase.id,
+            suite: evalCase.suite,
+            tier: evalCase.tier,
+            prompt: evalCase.prompt,
+            expect: evalCase.expect,
+            pass: score.pass,
+            matched: score.matched,
+            runs: score.runs,
+            triggerRate: score.triggerRate,
+            threshold: score.threshold,
+            histogram: Object.fromEntries(score.histogram),
+        })),
+    }, null, 2);
+}
+function describeExpectation(expectation) {
+    if ("noSkill" in expectation)
+        return "(no skill)";
+    if ("first" in expectation)
+        return expectation.first;
+    if ("anyOf" in expectation)
+        return `one of [${expectation.anyOf.join(", ")}]`;
+    return expectation.path.join(" → ");
+}
+function formatHistogram(histogram) {
+    return [...histogram.entries()]
+        .sort((a, b) => b[1] - a[1])
+        .map(([id, count]) => `${id} ×${count}`)
+        .join(", ");
+}
+function truncate(text, max = 80) {
+    const single = text.replace(/\s+/g, " ").trim();
+    return single.length > max ? `${single.slice(0, max - 1)}…` : single;
+}
+function groupBy(items, key) {
+    const groups = new Map();
+    for (const item of items) {
+        const k = key(item);
+        const existing = groups.get(k);
+        if (existing)
+            existing.push(item);
+        else
+            groups.set(k, [item]);
+    }
+    return groups;
+}
+//# sourceMappingURL=report.js.map

package/dist/eval/report.js.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"report.js","sourceRoot":"","sources":["../../src/eval/report.ts"],"names":[],"mappings":"AAeA,MAAM,UAAU,WAAW,CAAC,KAA4B;IACtD,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;IACxD,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,GAAG,MAAM,EAAE,CAAC;AAC1D,CAAC;AAED,MAAM,UAAU,aAAa,CAAC,MAAkB;IAC9C,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC;IAEtF,KAAK,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,MAAM,EAAE,CAAC;QACtC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAClB,KAAK,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,IAAI,OAAO,EAAE,CAAC;YAC1C,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;YACzC,MAAM,KAAK,GAAG,GAAG,KAAK,CAAC,OAAO,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;YAC/C,KAAK,CAAC,IAAI,CAAC,KAAK,GAAG,KAAK,QAAQ,CAAC,EAAE,KAAK,KAAK,OAAO,mBAAmB,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YAC5F,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;gBAChB,KAAK,CAAC,IAAI,CAAC,kBAAkB,eAAe,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;gBACjE,KAAK,CAAC,IAAI,CAAC,qBAAqB,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;gBAC7D,IAAI,QAAQ,CAAC,IAAI;oBAAE,KAAK,CAAC,IAAI,CAAC,mBAAmB,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;YACpE,CAAC;QACH,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;IAC5C,MAAM,GAAG,GAAG,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,MAAM,GAAG,KAAK,CAAC,GAAG,GAAG,CAAC,CAAC;IACxE,KAAK,CAAC,IAAI,CAAC,YAAY,MAAM,CAAC,MAAM,IAAI,KAAK,kBAAkB,GAAG,KAAK,CAAC,CAAC;IACzE,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,MAAM,UAAU,MAAM,CAAC,MAAkB;IACvC,OAAO,IAAI,CAAC,SAAS,CACnB;QACE,OAAO,EAAE;YACP,KAAK,EAAE,MAAM,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM;YACpC,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,MAAM,EAAE,MAAM,CAAC,MAAM;SACtB;QACD,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;YAChD,EAAE,EAAE,QAAQ,CAAC,EAAE;YACf,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,SAAS,EAAE,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,SAAS,CAAC;SAC/C,CAAC,CAAC;KACJ,EACD,IAAI,EACJ,CAAC,CACF,CAAC;AACJ,CAAC;AAED,SAAS,mBAAmB,CAAC,WAAwB;IACnD,IAAI,SAAS,IAAI,WAAW;QAAE,OAAO,YAAY,CAAC;IAClD,IAAI,OAAO,IAAI,WAAW;QAAE,OAAO,WAAW,CAAC,KAAK,CAAC;IACrD,IAAI,OAAO,IAAI,WAAW;QAAE,OAAO,WAAW,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC;IAC9E,OAAO,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AACtC,CAAC;AAED,SAAS,eAAe,CAAC,SAAsC;IAC7D,OAAO,CAAC,GAAG,SAAS,CAAC,OAAO,EAAE,CAAC;SAC5B,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;SAC3B,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,GAAG,EAAE,KAAK,KAAK,EAAE,CAAC;SACvC,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED,SAAS,QAAQ,CAAC,IAAY,EAAE,GAAG,GAAG,EAAE;IACtC,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IAChD,OAAO,MAAM,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;AACvE,CAAC;AAED,SAAS,OAAO,CAAI,KAAmB,EAAE,GAAwB;IAC/D,MAAM,MAAM,GAAG,IAAI,GAAG,EAAe,CAAC;IACtC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC;QACpB,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAC/B,IAAI,QAAQ;YAAE,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;;YAC7B,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC;IAC7B,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/eval/runner.d.ts ADDED Viewed

@@ -0,0 +1,17 @@
+import { type DetectionResult } from "./detect.js";
+import type { LoadedCase } from "./cases.js";
+export interface RunnerOptions {
+    readonly cwd: string;
+    readonly runs?: number;
+    readonly concurrency?: number;
+    readonly timeoutMs?: number;
+    readonly model?: string;
+    readonly claudeBin?: string;
+    readonly onRun?: (caseId: string, result: DetectionResult) => void;
+}
+export interface CaseRuns {
+    readonly evalCase: LoadedCase;
+    readonly runs: readonly DetectionResult[];
+}
+export declare function runCases(cases: readonly LoadedCase[], options: RunnerOptions): Promise<CaseRuns[]>;
+//# sourceMappingURL=runner.d.ts.map

package/dist/eval/runner.d.ts.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/eval/runner.ts"],"names":[],"mappings":"AAIA,OAAO,EAAkB,KAAK,eAAe,EAAE,MAAM,aAAa,CAAC;AACnE,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAM7C,MAAM,WAAW,aAAa;IAC5B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,eAAe,KAAK,IAAI,CAAC;CACpE;AAED,MAAM,WAAW,QAAQ;IACvB,QAAQ,CAAC,QAAQ,EAAE,UAAU,CAAC;IAC9B,QAAQ,CAAC,IAAI,EAAE,SAAS,eAAe,EAAE,CAAC;CAC3C;AAED,wBAAsB,QAAQ,CAC5B,KAAK,EAAE,SAAS,UAAU,EAAE,EAC5B,OAAO,EAAE,aAAa,GACrB,OAAO,CAAC,QAAQ,EAAE,CAAC,CAgBrB"}

package/dist/eval/runner.js ADDED Viewed

@@ -0,0 +1,89 @@
+import { spawn } from "node:child_process";
+import { isAbsolute, resolve } from "node:path";
+import { createInterface } from "node:readline";
+import { createDetector } from "./detect.js";
+const DEFAULT_RUNS = 5;
+const DEFAULT_CONCURRENCY = 1;
+const DEFAULT_TIMEOUT_MS = 60_000;
+export async function runCases(cases, options) {
+    const byCase = new Map();
+    const jobs = cases.flatMap((evalCase) => {
+        byCase.set(evalCase, []);
+        const count = evalCase.runs ?? options.runs ?? DEFAULT_RUNS;
+        return Array.from({ length: count }, () => evalCase);
+    });
+    const concurrency = Math.max(1, options.concurrency ?? DEFAULT_CONCURRENCY);
+    await forEachLimit(jobs, concurrency, async (evalCase) => {
+        const result = await runOnce(evalCase, options);
+        byCase.get(evalCase)?.push(result);
+        options.onRun?.(evalCase.id, result);
+    });
+    return cases.map((evalCase) => ({ evalCase, runs: byCase.get(evalCase) ?? [] }));
+}
+function skillsToCollect(evalCase) {
+    return "path" in evalCase.expect ? evalCase.expect.path.length : 1;
+}
+async function runOnce(evalCase, options) {
+    const cwd = evalCase.cwd ? resolveCwd(options.cwd, evalCase.cwd) : options.cwd;
+    const args = [
+        "-p",
+        evalCase.prompt,
+        "--output-format",
+        "stream-json",
+        "--verbose",
+        "--include-partial-messages",
+    ];
+    if (options.model)
+        args.push("--model", options.model);
+    const env = { ...process.env };
+    delete env["CLAUDECODE"];
+    const detector = createDetector(skillsToCollect(evalCase));
+    const child = spawn(options.claudeBin ?? "claude", args, {
+        cwd,
+        env,
+        stdio: ["ignore", "pipe", "ignore"],
+    });
+    const spawnFailure = new Promise((_, reject) => {
+        child.on("error", (cause) => reject(new Error(`failed to spawn claude: ${cause.message}`)));
+    });
+    const deadline = { reached: false };
+    const timer = setTimeout(() => {
+        deadline.reached = true;
+        child.kill("SIGKILL");
+    }, options.timeoutMs ?? DEFAULT_TIMEOUT_MS);
+    try {
+        await Promise.race([drain(child.stdout, () => child.kill("SIGKILL"), detector), spawnFailure]);
+    }
+    finally {
+        clearTimeout(timer);
+        if (child.exitCode === null)
+            child.kill("SIGKILL");
+    }
+    return detector.result(deadline.reached ? "timeout" : "no-skill");
+}
+async function drain(stdout, stop, detector) {
+    const lines = createInterface({ input: stdout });
+    for await (const line of lines) {
+        detector.push(line);
+        if (detector.done) {
+            stop();
+            break;
+        }
+    }
+}
+function resolveCwd(base, caseCwd) {
+    return isAbsolute(caseCwd) ? caseCwd : resolve(base, caseCwd);
+}
+async function forEachLimit(items, limit, worker) {
+    let cursor = 0;
+    const runners = Array.from({ length: Math.min(limit, items.length) }, async () => {
+        while (cursor < items.length) {
+            const item = items[cursor];
+            cursor += 1;
+            if (item !== undefined)
+                await worker(item);
+        }
+    });
+    await Promise.all(runners);
+}
+//# sourceMappingURL=runner.js.map

package/dist/eval/runner.js.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"runner.js","sourceRoot":"","sources":["../../src/eval/runner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAChD,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAEhD,OAAO,EAAE,cAAc,EAAwB,MAAM,aAAa,CAAC;AAGnE,MAAM,YAAY,GAAG,CAAC,CAAC;AACvB,MAAM,mBAAmB,GAAG,CAAC,CAAC;AAC9B,MAAM,kBAAkB,GAAG,MAAM,CAAC;AAiBlC,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,KAA4B,EAC5B,OAAsB;IAEtB,MAAM,MAAM,GAAG,IAAI,GAAG,EAAiC,CAAC;IACxD,MAAM,IAAI,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE;QACtC,MAAM,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;QACzB,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,IAAI,OAAO,CAAC,IAAI,IAAI,YAAY,CAAC;QAC5D,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE,CAAC,QAAQ,CAAC,CAAC;IACvD,CAAC,CAAC,CAAC;IAEH,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,WAAW,IAAI,mBAAmB,CAAC,CAAC;IAC5E,MAAM,YAAY,CAAC,IAAI,EAAE,WAAW,EAAE,KAAK,EAAE,QAAQ,EAAE,EAAE;QACvD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAChD,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QACnC,OAAO,CAAC,KAAK,EAAE,CAAC,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;IACvC,CAAC,CAAC,CAAC;IAEH,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC;AACnF,CAAC;AAED,SAAS,eAAe,CAAC,QAAoB;IAC3C,OAAO,MAAM,IAAI,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;AACrE,CAAC;AAED,KAAK,UAAU,OAAO,CAAC,QAAoB,EAAE,OAAsB;IACjE,MAAM,GAAG,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,GAAG,EAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC;IAC/E,MAAM,IAAI,GAAG;QACX,IAAI;QACJ,QAAQ,CAAC,MAAM;QACf,iBAAiB;QACjB,aAAa;QACb,WAAW;QACX,4BAA4B;KAC7B,CAAC;IACF,IAAI,OAAO,CAAC,KAAK;QAAE,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC;IAEvD,MAAM,GAAG,GAAG,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;IAC/B,OAAO,GAAG,CAAC,YAAY,CAAC,CAAC;IAEzB,MAAM,QAAQ,GAAG,cAAc,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC3D,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,SAAS,IAAI,QAAQ,EAAE,IAAI,EAAE;QACvD,GAAG;QACH,GAAG;QACH,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC;KACpC,CAAC,CAAC;IAEH,MAAM,YAAY,GAAG,IAAI,OAAO,CAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE;QACpD,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,2BAA2B,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC;IAC9F,CAAC,CAAC,CAAC;IAEH,MAAM,QAAQ,GAAG,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;IACpC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;QAC5B,QAAQ,CAAC,OAAO,GAAG,IAAI,CAAC;QACxB,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACxB,CAAC,EAAE,OAAO,CAAC,SAAS,IAAI,kBAAkB,CAAC,CAAC;IAE5C,IAAI,CAAC;QACH,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,QAAQ,CAAC,EAAE,YAAY,CAAC,CAAC,CAAC;IACjG,CAAC;YAAS,CAAC;QACT,YAAY,CAAC,KAAK,CAAC,CAAC;QACpB,IAAI,KAAK,CAAC,QAAQ,KAAK,IAAI;YAAE,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACrD,CAAC;IAED,OAAO,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;AACpE,CAAC;AAED,KAAK,UAAU,KAAK,CAClB,MAA6B,EAC7B,IAAgB,EAChB,QAA2C;IAE3C,MAAM,KAAK,GAAG,eAAe,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;IACjD,IAAI,KAAK,EAAE,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QAC/B,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpB,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC;YAClB,IAAI,EAAE,CAAC;YACP,MAAM;QACR,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,UAAU,CAAC,IAAY,EAAE,OAAe;IAC/C,OAAO,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;AAChE,CAAC;AAED,KAAK,UAAU,YAAY,CACzB,KAAmB,EACnB,KAAa,EACb,MAAkC;IAElC,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,EAAE,KAAK,IAAI,EAAE;QAC/E,OAAO,MAAM,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;YAC7B,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC;YAC3B,MAAM,IAAI,CAAC,CAAC;YACZ,IAAI,IAAI,KAAK,SAAS;gBAAE,MAAM,MAAM,CAAC,IAAI,CAAC,CAAC;QAC7C,CAAC;IACH,CAAC,CAAC,CAAC;IACH,MAAM,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;AAC7B,CAAC"}

package/dist/eval/schema.d.ts ADDED Viewed

@@ -0,0 +1,253 @@
+import { z } from "zod";
+declare const Expectation: z.ZodUnion<[z.ZodObject<{
+    first: z.ZodString;
+    not: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+}, "strip", z.ZodTypeAny, {
+    first: string;
+    not?: string[] | undefined;
+}, {
+    first: string;
+    not?: string[] | undefined;
+}>, z.ZodObject<{
+    anyOf: z.ZodArray<z.ZodString, "many">;
+    not: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+}, "strip", z.ZodTypeAny, {
+    anyOf: string[];
+    not?: string[] | undefined;
+}, {
+    anyOf: string[];
+    not?: string[] | undefined;
+}>, z.ZodObject<{
+    path: z.ZodArray<z.ZodString, "many">;
+    not: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+}, "strip", z.ZodTypeAny, {
+    path: string[];
+    not?: string[] | undefined;
+}, {
+    path: string[];
+    not?: string[] | undefined;
+}>, z.ZodObject<{
+    noSkill: z.ZodLiteral<true>;
+}, "strip", z.ZodTypeAny, {
+    noSkill: true;
+}, {
+    noSkill: true;
+}>]>;
+declare const Case: z.ZodObject<{
+    id: z.ZodString;
+    prompt: z.ZodString;
+    expect: z.ZodUnion<[z.ZodObject<{
+        first: z.ZodString;
+        not: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    }, "strip", z.ZodTypeAny, {
+        first: string;
+        not?: string[] | undefined;
+    }, {
+        first: string;
+        not?: string[] | undefined;
+    }>, z.ZodObject<{
+        anyOf: z.ZodArray<z.ZodString, "many">;
+        not: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    }, "strip", z.ZodTypeAny, {
+        anyOf: string[];
+        not?: string[] | undefined;
+    }, {
+        anyOf: string[];
+        not?: string[] | undefined;
+    }>, z.ZodObject<{
+        path: z.ZodArray<z.ZodString, "many">;
+        not: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+    }, "strip", z.ZodTypeAny, {
+        path: string[];
+        not?: string[] | undefined;
+    }, {
+        path: string[];
+        not?: string[] | undefined;
+    }>, z.ZodObject<{
+        noSkill: z.ZodLiteral<true>;
+    }, "strip", z.ZodTypeAny, {
+        noSkill: true;
+    }, {
+        noSkill: true;
+    }>]>;
+    cwd: z.ZodOptional<z.ZodString>;
+    runs: z.ZodOptional<z.ZodNumber>;
+    threshold: z.ZodOptional<z.ZodNumber>;
+    note: z.ZodOptional<z.ZodString>;
+}, "strip", z.ZodTypeAny, {
+    id: string;
+    prompt: string;
+    expect: {
+        first: string;
+        not?: string[] | undefined;
+    } | {
+        anyOf: string[];
+        not?: string[] | undefined;
+    } | {
+        path: string[];
+        not?: string[] | undefined;
+    } | {
+        noSkill: true;
+    };
+    cwd?: string | undefined;
+    runs?: number | undefined;
+    threshold?: number | undefined;
+    note?: string | undefined;
+}, {
+    id: string;
+    prompt: string;
+    expect: {
+        first: string;
+        not?: string[] | undefined;
+    } | {
+        anyOf: string[];
+        not?: string[] | undefined;
+    } | {
+        path: string[];
+        not?: string[] | undefined;
+    } | {
+        noSkill: true;
+    };
+    cwd?: string | undefined;
+    runs?: number | undefined;
+    threshold?: number | undefined;
+    note?: string | undefined;
+}>;
+export declare const TIERS: readonly ["routing", "solving"];
+export declare const CaseFileSchema: z.ZodObject<{
+    suite: z.ZodString;
+    tier: z.ZodEnum<["routing", "solving"]>;
+    cases: z.ZodArray<z.ZodObject<{
+        id: z.ZodString;
+        prompt: z.ZodString;
+        expect: z.ZodUnion<[z.ZodObject<{
+            first: z.ZodString;
+            not: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        }, "strip", z.ZodTypeAny, {
+            first: string;
+            not?: string[] | undefined;
+        }, {
+            first: string;
+            not?: string[] | undefined;
+        }>, z.ZodObject<{
+            anyOf: z.ZodArray<z.ZodString, "many">;
+            not: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        }, "strip", z.ZodTypeAny, {
+            anyOf: string[];
+            not?: string[] | undefined;
+        }, {
+            anyOf: string[];
+            not?: string[] | undefined;
+        }>, z.ZodObject<{
+            path: z.ZodArray<z.ZodString, "many">;
+            not: z.ZodOptional<z.ZodArray<z.ZodString, "many">>;
+        }, "strip", z.ZodTypeAny, {
+            path: string[];
+            not?: string[] | undefined;
+        }, {
+            path: string[];
+            not?: string[] | undefined;
+        }>, z.ZodObject<{
+            noSkill: z.ZodLiteral<true>;
+        }, "strip", z.ZodTypeAny, {
+            noSkill: true;
+        }, {
+            noSkill: true;
+        }>]>;
+        cwd: z.ZodOptional<z.ZodString>;
+        runs: z.ZodOptional<z.ZodNumber>;
+        threshold: z.ZodOptional<z.ZodNumber>;
+        note: z.ZodOptional<z.ZodString>;
+    }, "strip", z.ZodTypeAny, {
+        id: string;
+        prompt: string;
+        expect: {
+            first: string;
+            not?: string[] | undefined;
+        } | {
+            anyOf: string[];
+            not?: string[] | undefined;
+        } | {
+            path: string[];
+            not?: string[] | undefined;
+        } | {
+            noSkill: true;
+        };
+        cwd?: string | undefined;
+        runs?: number | undefined;
+        threshold?: number | undefined;
+        note?: string | undefined;
+    }, {
+        id: string;
+        prompt: string;
+        expect: {
+            first: string;
+            not?: string[] | undefined;
+        } | {
+            anyOf: string[];
+            not?: string[] | undefined;
+        } | {
+            path: string[];
+            not?: string[] | undefined;
+        } | {
+            noSkill: true;
+        };
+        cwd?: string | undefined;
+        runs?: number | undefined;
+        threshold?: number | undefined;
+        note?: string | undefined;
+    }>, "many">;
+}, "strip", z.ZodTypeAny, {
+    suite: string;
+    tier: "routing" | "solving";
+    cases: {
+        id: string;
+        prompt: string;
+        expect: {
+            first: string;
+            not?: string[] | undefined;
+        } | {
+            anyOf: string[];
+            not?: string[] | undefined;
+        } | {
+            path: string[];
+            not?: string[] | undefined;
+        } | {
+            noSkill: true;
+        };
+        cwd?: string | undefined;
+        runs?: number | undefined;
+        threshold?: number | undefined;
+        note?: string | undefined;
+    }[];
+}, {
+    suite: string;
+    tier: "routing" | "solving";
+    cases: {
+        id: string;
+        prompt: string;
+        expect: {
+            first: string;
+            not?: string[] | undefined;
+        } | {
+            anyOf: string[];
+            not?: string[] | undefined;
+        } | {
+            path: string[];
+            not?: string[] | undefined;
+        } | {
+            noSkill: true;
+        };
+        cwd?: string | undefined;
+        runs?: number | undefined;
+        threshold?: number | undefined;
+        note?: string | undefined;
+    }[];
+}>;
+export type Tier = (typeof TIERS)[number];
+export type Expectation = z.infer<typeof Expectation>;
+export type EvalCase = z.infer<typeof Case>;
+export type CaseFile = z.infer<typeof CaseFileSchema>;
+export declare function expectedSkills(expectation: Expectation): readonly string[];
+export {};
+//# sourceMappingURL=schema.d.ts.map

package/dist/eval/schema.d.ts.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../src/eval/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAyBxB,QAAA,MAAM,WAAW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAKf,CAAC;AAEH,QAAA,MAAM,IAAI;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAQR,CAAC;AAEH,eAAO,MAAM,KAAK,iCAAkC,CAAC;AAErD,eAAO,MAAM,cAAc;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAIzB,CAAC;AAEH,MAAM,MAAM,IAAI,GAAG,CAAC,OAAO,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC;AAC1C,MAAM,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,WAAW,CAAC,CAAC;AACtD,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,IAAI,CAAC,CAAC;AAC5C,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AAEtD,wBAAgB,cAAc,CAAC,WAAW,EAAE,WAAW,GAAG,SAAS,MAAM,EAAE,CAM1E"}

package/dist/eval/schema.js ADDED Viewed

@@ -0,0 +1,50 @@
+import { z } from "zod";
+import { FQ_ID } from "../ids.js";
+const FqId = z.string().regex(FQ_ID, "must be a `plugin:name` id");
+const FirstExpectation = z.object({
+    first: FqId,
+    not: z.array(FqId).optional(),
+});
+const AnyOfExpectation = z.object({
+    anyOf: z.array(FqId).min(1),
+    not: z.array(FqId).optional(),
+});
+const PathExpectation = z.object({
+    path: z.array(FqId).min(2),
+    not: z.array(FqId).optional(),
+});
+const NoSkillExpectation = z.object({
+    noSkill: z.literal(true),
+});
+const Expectation = z.union([
+    FirstExpectation,
+    AnyOfExpectation,
+    PathExpectation,
+    NoSkillExpectation,
+]);
+const Case = z.object({
+    id: z.string().min(1),
+    prompt: z.string().min(1),
+    expect: Expectation,
+    cwd: z.string().optional(),
+    runs: z.number().int().positive().optional(),
+    threshold: z.number().min(0).max(1).optional(),
+    note: z.string().optional(),
+});
+export const TIERS = ["routing", "solving"];
+export const CaseFileSchema = z.object({
+    suite: z.string().min(1),
+    tier: z.enum(TIERS),
+    cases: z.array(Case).min(1),
+});
+export function expectedSkills(expectation) {
+    if ("noSkill" in expectation)
+        return [];
+    const forbidden = expectation.not ?? [];
+    if ("first" in expectation)
+        return [expectation.first, ...forbidden];
+    if ("anyOf" in expectation)
+        return [...expectation.anyOf, ...forbidden];
+    return [...expectation.path, ...forbidden];
+}
+//# sourceMappingURL=schema.js.map

package/dist/eval/schema.js.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"schema.js","sourceRoot":"","sources":["../../src/eval/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,KAAK,EAAE,MAAM,WAAW,CAAC;AAElC,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,KAAK,EAAE,4BAA4B,CAAC,CAAC;AAEnE,MAAM,gBAAgB,GAAG,CAAC,CAAC,MAAM,CAAC;IAChC,KAAK,EAAE,IAAI;IACX,GAAG,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE;CAC9B,CAAC,CAAC;AAEH,MAAM,gBAAgB,GAAG,CAAC,CAAC,MAAM,CAAC;IAChC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC3B,GAAG,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE;CAC9B,CAAC,CAAC;AAEH,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC;IAC/B,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1B,GAAG,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,QAAQ,EAAE;CAC9B,CAAC,CAAC;AAEH,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;IAClC,OAAO,EAAE,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC;CACzB,CAAC,CAAC;AAEH,MAAM,WAAW,GAAG,CAAC,CAAC,KAAK,CAAC;IAC1B,gBAAgB;IAChB,gBAAgB;IAChB,eAAe;IACf,kBAAkB;CACnB,CAAC,CAAC;AAEH,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC;IACpB,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACrB,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACzB,MAAM,EAAE,WAAW;IACnB,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC1B,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IAC5C,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE;IAC9C,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CAC5B,CAAC,CAAC;AAEH,MAAM,CAAC,MAAM,KAAK,GAAG,CAAC,SAAS,EAAE,SAAS,CAAU,CAAC;AAErD,MAAM,CAAC,MAAM,cAAc,GAAG,CAAC,CAAC,MAAM,CAAC;IACrC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACxB,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC;IACnB,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;CAC5B,CAAC,CAAC;AAOH,MAAM,UAAU,cAAc,CAAC,WAAwB;IACrD,IAAI,SAAS,IAAI,WAAW;QAAE,OAAO,EAAE,CAAC;IACxC,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,IAAI,EAAE,CAAC;IACxC,IAAI,OAAO,IAAI,WAAW;QAAE,OAAO,CAAC,WAAW,CAAC,KAAK,EAAE,GAAG,SAAS,CAAC,CAAC;IACrE,IAAI,OAAO,IAAI,WAAW;QAAE,OAAO,CAAC,GAAG,WAAW,CAAC,KAAK,EAAE,GAAG,SAAS,CAAC,CAAC;IACxE,OAAO,CAAC,GAAG,WAAW,CAAC,IAAI,EAAE,GAAG,SAAS,CAAC,CAAC;AAC7C,CAAC"}

package/dist/eval/score.d.ts ADDED Viewed

@@ -0,0 +1,13 @@
+import type { DetectionResult } from "./detect.js";
+import type { Expectation } from "./schema.js";
+export interface CaseScore {
+    readonly matched: number;
+    readonly runs: number;
+    readonly triggerRate: number;
+    readonly threshold: number;
+    readonly pass: boolean;
+    readonly histogram: ReadonlyMap<string, number>;
+}
+export declare function matchesExpectation(expectation: Expectation, run: DetectionResult): boolean;
+export declare function scoreCase(expectation: Expectation, runs: readonly DetectionResult[], threshold?: number): CaseScore;
+//# sourceMappingURL=score.d.ts.map

package/dist/eval/score.d.ts.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"score.d.ts","sourceRoot":"","sources":["../../src/eval/score.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AACnD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAI/C,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;IACvB,QAAQ,CAAC,SAAS,EAAE,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACjD;AAED,wBAAgB,kBAAkB,CAAC,WAAW,EAAE,WAAW,EAAE,GAAG,EAAE,eAAe,GAAG,OAAO,CAa1F;AAED,wBAAgB,SAAS,CACvB,WAAW,EAAE,WAAW,EACxB,IAAI,EAAE,SAAS,eAAe,EAAE,EAChC,SAAS,SAAoB,GAC5B,SAAS,CAWX"}

package/dist/eval/score.js ADDED Viewed

@@ -0,0 +1,52 @@
+const DEFAULT_THRESHOLD = 1.0;
+export function matchesExpectation(expectation, run) {
+    if (violatesNot(expectation, run))
+        return false;
+    if ("noSkill" in expectation) {
+        return run.firstSkill === null;
+    }
+    if ("first" in expectation) {
+        return run.firstSkill === expectation.first;
+    }
+    if ("anyOf" in expectation) {
+        return run.firstSkill !== null && expectation.anyOf.includes(run.firstSkill);
+    }
+    return isOrderedSubsequence(expectation.path, run.observed);
+}
+export function scoreCase(expectation, runs, threshold = DEFAULT_THRESHOLD) {
+    const matched = runs.filter((run) => matchesExpectation(expectation, run)).length;
+    const triggerRate = runs.length === 0 ? 0 : matched / runs.length;
+    return {
+        matched,
+        runs: runs.length,
+        triggerRate,
+        threshold,
+        pass: triggerRate >= threshold,
+        histogram: histogramOf(runs),
+    };
+}
+function violatesNot(expectation, run) {
+    const forbidden = "noSkill" in expectation ? undefined : expectation.not;
+    if (!forbidden || forbidden.length === 0)
+        return false;
+    return run.firstSkill !== null && forbidden.includes(run.firstSkill);
+}
+function isOrderedSubsequence(needle, haystack) {
+    let cursor = 0;
+    for (const id of haystack) {
+        if (id === needle[cursor])
+            cursor += 1;
+        if (cursor === needle.length)
+            return true;
+    }
+    return cursor === needle.length;
+}
+function histogramOf(runs) {
+    const counts = new Map();
+    for (const run of runs) {
+        const key = run.firstSkill ?? "(no skill)";
+        counts.set(key, (counts.get(key) ?? 0) + 1);
+    }
+    return counts;
+}
+//# sourceMappingURL=score.js.map

package/dist/eval/score.js.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"score.js","sourceRoot":"","sources":["../../src/eval/score.ts"],"names":[],"mappings":"AAGA,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAW9B,MAAM,UAAU,kBAAkB,CAAC,WAAwB,EAAE,GAAoB;IAC/E,IAAI,WAAW,CAAC,WAAW,EAAE,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IAEhD,IAAI,SAAS,IAAI,WAAW,EAAE,CAAC;QAC7B,OAAO,GAAG,CAAC,UAAU,KAAK,IAAI,CAAC;IACjC,CAAC;IACD,IAAI,OAAO,IAAI,WAAW,EAAE,CAAC;QAC3B,OAAO,GAAG,CAAC,UAAU,KAAK,WAAW,CAAC,KAAK,CAAC;IAC9C,CAAC;IACD,IAAI,OAAO,IAAI,WAAW,EAAE,CAAC;QAC3B,OAAO,GAAG,CAAC,UAAU,KAAK,IAAI,IAAI,WAAW,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IAC/E,CAAC;IACD,OAAO,oBAAoB,CAAC,WAAW,CAAC,IAAI,EAAE,GAAG,CAAC,QAAQ,CAAC,CAAC;AAC9D,CAAC;AAED,MAAM,UAAU,SAAS,CACvB,WAAwB,EACxB,IAAgC,EAChC,SAAS,GAAG,iBAAiB;IAE7B,MAAM,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,kBAAkB,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;IAClF,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC;IAClE,OAAO;QACL,OAAO;QACP,IAAI,EAAE,IAAI,CAAC,MAAM;QACjB,WAAW;QACX,SAAS;QACT,IAAI,EAAE,WAAW,IAAI,SAAS;QAC9B,SAAS,EAAE,WAAW,CAAC,IAAI,CAAC;KAC7B,CAAC;AACJ,CAAC;AAED,SAAS,WAAW,CAAC,WAAwB,EAAE,GAAoB;IACjE,MAAM,SAAS,GAAG,SAAS,IAAI,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC;IACzE,IAAI,CAAC,SAAS,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACvD,OAAO,GAAG,CAAC,UAAU,KAAK,IAAI,IAAI,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;AACvE,CAAC;AAED,SAAS,oBAAoB,CAAC,MAAyB,EAAE,QAA2B;IAClF,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,KAAK,MAAM,EAAE,IAAI,QAAQ,EAAE,CAAC;QAC1B,IAAI,EAAE,KAAK,MAAM,CAAC,MAAM,CAAC;YAAE,MAAM,IAAI,CAAC,CAAC;QACvC,IAAI,MAAM,KAAK,MAAM,CAAC,MAAM;YAAE,OAAO,IAAI,CAAC;IAC5C,CAAC;IACD,OAAO,MAAM,KAAK,MAAM,CAAC,MAAM,CAAC;AAClC,CAAC;AAED,SAAS,WAAW,CAAC,IAAgC;IACnD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,MAAM,GAAG,GAAG,GAAG,CAAC,UAAU,IAAI,YAAY,CAAC;QAC3C,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC9C,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/index.d.ts CHANGED Viewed

@@ -27,6 +27,10 @@ export { resolveVendors } from "./vendor/registry.js";
 export type { DiscoveredVendorPlugin, LinkedFile, Vendor, VendorEmitContext, VendorInstallContext, } from "./vendor/schema.js";
 export { check } from "./check/index.js";
 export type { CheckOptions, CheckResult, ReferenceViolation, ReferenceViolationKind, SourceSummary, } from "./check/index.js";
+export { runEval, formatConsole, toJson } from "./eval/index.js";
+export type { EvalOptions, EvalReport, CaseReport, LoadedCase, CaseLoadError, } from "./eval/index.js";
+export { CaseFileSchema, TIERS } from "./eval/schema.js";
+export type { CaseFile, EvalCase, Expectation, Tier } from "./eval/schema.js";
 export { defaultSources, discoverInstalled, indexInstalled } from "./installed.js";
 export type { InstalledAgent, InstalledArtifacts, InstalledCommand, InstalledIndex, InstalledSkill, PluginSource, } from "./installed.js";
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,eAAe,EACf,WAAW,EACX,mBAAmB,EACnB,WAAW,EACX,aAAa,EACb,oBAAoB,EACpB,2BAA2B,EAC3B,SAAS,GACV,MAAM,kBAAkB,CAAC;AAC1B,YAAY,EACV,SAAS,EACT,kBAAkB,EAClB,WAAW,EACX,cAAc,EACd,KAAK,EACL,SAAS,EACT,WAAW,GACZ,MAAM,kBAAkB,CAAC;AAE1B,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAC/D,YAAY,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAEhD,OAAO,EAAE,sBAAsB,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAC9E,YAAY,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAE9E,OAAO,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,aAAa,CAAC;AACtC,YAAY,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAE1C,OAAO,EAAE,iBAAiB,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AACxE,YAAY,EACV,WAAW,EACX,gBAAgB,EAChB,SAAS,EACT,iBAAiB,EACjB,eAAe,GAChB,MAAM,yBAAyB,CAAC;AAEjC,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,YAAY,EAAE,qBAAqB,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAE7E,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,YAAY,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAEnD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,YAAY,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAEzD,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AACxD,YAAY,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AACnF,YAAY,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAErD,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAC9C,YAAY,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAEnD,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AACxD,YAAY,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEzD,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,YAAY,EACV,sBAAsB,EACtB,UAAU,EACV,MAAM,EACN,iBAAiB,EACjB,oBAAoB,GACrB,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACzC,YAAY,EACV,YAAY,EACZ,WAAW,EACX,kBAAkB,EAClB,sBAAsB,EACtB,aAAa,GACd,MAAM,kBAAkB,CAAC;AAE1B,OAAO,EAAE,cAAc,EAAE,iBAAiB,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AACnF,YAAY,EACV,cAAc,EACd,kBAAkB,EAClB,gBAAgB,EAChB,cAAc,EACd,cAAc,EACd,YAAY,GACb,MAAM,gBAAgB,CAAC"}
1	+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,eAAe,EACf,WAAW,EACX,mBAAmB,EACnB,WAAW,EACX,aAAa,EACb,oBAAoB,EACpB,2BAA2B,EAC3B,SAAS,GACV,MAAM,kBAAkB,CAAC;AAC1B,YAAY,EACV,SAAS,EACT,kBAAkB,EAClB,WAAW,EACX,cAAc,EACd,KAAK,EACL,SAAS,EACT,WAAW,GACZ,MAAM,kBAAkB,CAAC;AAE1B,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAC/D,YAAY,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAEhD,OAAO,EAAE,sBAAsB,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAC9E,YAAY,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AAE9E,OAAO,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,aAAa,CAAC;AACtC,YAAY,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAE1C,OAAO,EAAE,iBAAiB,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AACxE,YAAY,EACV,WAAW,EACX,gBAAgB,EAChB,SAAS,EACT,iBAAiB,EACjB,eAAe,GAChB,MAAM,yBAAyB,CAAC;AAEjC,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,YAAY,EAAE,qBAAqB,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAE7E,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACvC,YAAY,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAEnD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,YAAY,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,WAAW,CAAC;AAEzD,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AACxD,YAAY,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACzD,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AACnF,YAAY,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAErD,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAC9C,YAAY,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAEnD,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AACxD,YAAY,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAEzD,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,YAAY,EACV,sBAAsB,EACtB,UAAU,EACV,MAAM,EACN,iBAAiB,EACjB,oBAAoB,GACrB,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACzC,YAAY,EACV,YAAY,EACZ,WAAW,EACX,kBAAkB,EAClB,sBAAsB,EACtB,aAAa,GACd,MAAM,kBAAkB,CAAC;AAE1B,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AACjE,YAAY,EACV,WAAW,EACX,UAAU,EACV,UAAU,EACV,UAAU,EACV,aAAa,GACd,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACzD,YAAY,EAAE,QAAQ,EAAE,QAAQ,EAAE,WAAW,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAE9E,OAAO,EAAE,cAAc,EAAE,iBAAiB,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AACnF,YAAY,EACV,cAAc,EACd,kBAAkB,EAClB,gBAAgB,EAChB,cAAc,EACd,cAAc,EACd,YAAY,GACb,MAAM,gBAAgB,CAAC"}

package/dist/index.js CHANGED Viewed

@@ -13,5 +13,7 @@ export { loadHarnessConfig } from "./config/harness.js";
 export { builtinVendors } from "./vendor/builtins.js";
 export { resolveVendors } from "./vendor/registry.js";
 export { check } from "./check/index.js";
+export { runEval, formatConsole, toJson } from "./eval/index.js";
+export { CaseFileSchema, TIERS } from "./eval/schema.js";
 export { defaultSources, discoverInstalled, indexInstalled } from "./installed.js";
 //# sourceMappingURL=index.js.map

package/dist/index.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,eAAe,EACf,WAAW,EACX,mBAAmB,EACnB,WAAW,EACX,aAAa,EACb,oBAAoB,EACpB,2BAA2B,EAC3B,SAAS,GACV,MAAM,kBAAkB,CAAC;AAW1B,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAG/D,OAAO,EAAE,sBAAsB,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAG9E,OAAO,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,aAAa,CAAC;AAGtC,OAAO,EAAE,iBAAiB,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AASxE,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAGpD,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAGvC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAGjC,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAExD,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAGnF,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAG9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAGxD,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAStD,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AASzC,OAAO,EAAE,cAAc,EAAE,iBAAiB,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC"}
1	+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,eAAe,EACf,WAAW,EACX,mBAAmB,EACnB,WAAW,EACX,aAAa,EACb,oBAAoB,EACpB,2BAA2B,EAC3B,SAAS,GACV,MAAM,kBAAkB,CAAC;AAW1B,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAG/D,OAAO,EAAE,sBAAsB,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAG9E,OAAO,EAAE,GAAG,EAAE,EAAE,EAAE,MAAM,aAAa,CAAC;AAGtC,OAAO,EAAE,iBAAiB,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AASxE,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAGpD,OAAO,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAGvC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAGjC,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAExD,OAAO,EAAE,aAAa,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,mBAAmB,CAAC;AAGnF,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAG9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,qBAAqB,CAAC;AAGxD,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAStD,OAAO,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AASzC,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAQjE,OAAO,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AAGzD,OAAO,EAAE,cAAc,EAAE,iBAAiB,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC"}

package/package.json CHANGED Viewed

@@ -1,6 +1,6 @@
 {
   "name": "@jean.gnc/harness-kit",
-  "version": "0.11.2",
+  "version": "0.12.1",
   "type": "module",
   "description": "Build your own multi-agent harness: typed toolkit for authoring plugins (skills, agents, commands, hooks) and shipping them to Claude Code and Codex from a single source tree.",
   "license": "MIT",