@ashlr/hub 3.4.0 → 3.5.1

package/dist/core/local-eval/report.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"report.js","sourceRoot":"","sources":["../../../src/core/local-eval/report.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAWH,SAAS,MAAM,CAAC,MAAyB;IACvC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAClC,MAAM,MAAM,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACjD,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC1C,OAAO,MAAM,CAAC,MAAM,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,GAAG,GAAG,CAAC,CAAE,GAAG,MAAM,CAAC,GAAG,CAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAE,CAAC;AACpG,CAAC;AAED,SAAS,IAAI,CAAC,MAAyB;IACrC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAClC,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;AACvE,CAAC;AAED,yCAAyC;AACzC,MAAM,UAAU,aAAa,CAAC,IAAc,EAAE,MAA8B;IAC1E,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IACrD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAuB,CAAC;IAC9C,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,KAAK,CAAC,MAAM;YAAE,SAAS;QAC3B,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC5D,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IACzC,OAAO;QACL,MAAM,EAAE,IAAI,CAAC,EAAE;QACf,GAAG,EAAE,IAAI,CAAC,GAAG;QACb,WAAW,EAAE,IAAI,CAAC,WAAW;QAC7B,MAAM;QACN,MAAM;QACN,KAAK,EAAE,MAAM,CAAC,MAAM;QACpB,QAAQ,EAAE,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM;QAC1D,YAAY,EAAE,MAAM,CAAC,IAAI,CAAC;QAC1B,UAAU,EAAE,IAAI,CAAC,IAAI,CAAC;QACtB,KAAK,EAAE,CAAC,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC;aACzB,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;aACzC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;KACrE,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,IAO3B;IACC,MAAM,WAAW,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;IACnE,MAAM,WAAW,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACpE,OAAO;QACL,aAAa,EAAE,IAAI,CAAC,aAAa;QACjC,aAAa,EAAE,IAAI,CAAC,aAAa;QACjC,WAAW,EAAE,IAAI,CAAC,WAAW;QAC7B,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,eAAe,EAAE,WAAW,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,GAAG,WAAW;QAClE,WAAW;QACX,WAAW;QACX,SAAS,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE;QACjD,UAAU,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,WAAW,EAAE;QACnD,MAAM,EAAE,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,SAAS;KACzC,CAAC;AACJ,CAAC;AAED,MAAM,GAAG,GAAG,CAAC,CAAS,EAAU,EAAE,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;AAC9D,MAAM,IAAI,GAAG,CAAC,EAAU,EAAU,EAAE,CAAC,GAAG,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;AAElE,kFAAkF;AAClF,MAAM,UAAU,YAAY,CAAC,MAAkB;IAC7C,MAAM,CAAC,GAAG,MAAM,CAAC,aAAa,CAAC;IAC/B,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;IAC5B,KAAK,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC;IAChE,KAAK,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC;IAC5C,KAAK,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC,cAAc,CAAC,cAAc,EAAE,WAAW,CAAC,CAAC,YAAY,CAAC,cAAc,EAAE,GAAG,CAAC,CAAC;IACjH,KAAK,CAAC,IAAI,CAAC,sBAAsB,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC;IACrE,6EAA6E;IAC7E,qEAAqE;IACrE,MAAM,OAAO,GAAW,CAAC,CAAC,OAAO,IAAI,2CAA2C,CAAC;IACjF,KAAK,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC,OAAO,YAAY,CAAC,CAAC,KAAK,aAAa,OAAO,GAAG,CAAC,CAAC;IACtF,KAAK,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC,mBAAmB,EAAE,CAAC,CAAC;IAC1D,KAAK,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC;IAC/C,KAAK,CAAC,IAAI,CAAC,sBAAsB,MAAM,CAAC,WAAW,qBAAqB,CAAC,CAAC;IAC1E,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,KAAK,CAAC,IAAI,CAAC,WAAW,MAAM,CAAC,WAAW,IAAI,MAAM,CAAC,WAAW,MAAM,GAAG,CAAC,MAAM,CAAC,eAAe,CAAC,YAAY;UACvG,MAAM,MAAM,CAAC,aAAa,iBAAiB,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IAC5E,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,MAAM,GAAG,GAAG,CAAC,CAAS,EAAE,CAAS,EAAU,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;IAC1D,KAAK,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,gBAAgB,CAAC,CAAC;IACzG,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;QAChC,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC;YAChC,CAAC,CAAC,GAAG;YACL,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC3D,KAAK,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,GAAG;cACpE,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,IAAI,KAAK,EAAE,CAAC,CAAC;IAChF,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,8DAA8D;IAC9D,6EAA6E;IAC7E,uEAAuE;IACvE,2EAA2E;IAC3E,4EAA4E;IAC5E,2BAA2B;IAC3B,MAAM,OAAO,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAC5C,CAAC,CAAC,MAAM;SACL,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,KAAK,mBAAmB,CAAC;SAClD,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC;IACvF,KAAK,CAAC,IAAI,CAAC,oBAAoB,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACrF,KAAK,CAAC,IAAI,CAAC,mGAAmG,CAAC,CAAC;IAChH,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,2EAA2E;IAC3E,8EAA8E;IAC9E,0EAA0E;IAC1E,6EAA6E;IAC7E,0DAA0D;IAC1D,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAC7C,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;IAC3F,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,KAAK,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;QAClC,KAAK,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,QAAQ,EAAE,CAAC;YACvC,MAAM,CAAC,GAAG,KAAK,CAAC,gBAAgB,CAAC;YACjC,KAAK,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,GAAG,IAAI,IAAI,KAAK,CAAC,KAAK,EAAE,EAAE,EAAE,CAAC,GAAG;kBAChD,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,6EAA6E,CAAC,CAAC,CAAC;YAClG,IAAI,CAAC;gBAAE,KAAK,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;YACrD,IAAI,KAAK,CAAC,KAAK,EAAE,CAAC;gBAChB,oEAAoE;gBACpE,qEAAqE;gBACrE,qEAAqE;gBACrE,uCAAuC;gBACvC,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CACtC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,eAAe,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,KAAK,KAAK,cAAc,CAAC,CAAC;gBAC7E,MAAM,YAAY,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,aAAa,IAAI,CAAC,CAAC,KAAK,KAAK,cAAc,CAAC,CAAC,MAAM,CAAC;gBAChG,KAAK,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,KAAK,CAAC,MAAM,sBAAsB;sBAChE,GAAG,YAAY,+CAA+C,KAAK,CAAC,KAAK,CAAC,UAAU,EAAE,CAAC,CAAC;gBAC5F,oEAAoE;gBACpE,uEAAuE;gBACvE,+CAA+C;gBAC/C,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACrB,MAAM,OAAO,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;oBAC/E,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;oBAC1D,KAAK,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,2BAA2B,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG;0BAChF,KAAK,OAAO,CAAC,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAC;gBACtE,CAAC;YACH,CAAC;QACH,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,gDAAgD,CAAC,CAAC;IAC7D,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;QAChC,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC;YACzB,KAAK,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,GAAG,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,EAAE,CAAC,GAAG;kBAChD,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,GAAG;kBACvE,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,EAAE,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,GAAG;kBAC5D,GAAG,CAAC,CAAC,YAAY,IAAI,GAAG,aAAa,CAAC,CAAC,KAAK,IAAI,GAAG,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAClG,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC"}

package/dist/core/local-eval/runner.d.ts

@@ -0,0 +1,55 @@
+/**
+ * Running one trial, and running a whole set of them.
+ *
+ * THREE RULES THIS FILE ENFORCES, each learned the hard way here:
+ *
+ *   1. NEVER PIPE A COMMAND WHOSE STATUS MATTERS. `vitest ... | tail` reports
+ *      tail's exit code, and has already made a failing run look green in this
+ *      repository. Every child process below is spawned with its output
+ *      captured to a buffer and a file, and the status read from the child
+ *      itself. Nothing in this harness is piped.
+ *   2. EVERY TRIAL IS ISOLATED. Fixtures are rebuilt from source into a fresh
+ *      directory per trial, so trial 3 cannot inherit trial 2's half-finished
+ *      edit and the order trials run in cannot change the result.
+ *   3. THE AGENT CANNOT REACH ITS OWN GRADER. The checker is written one level
+ *      ABOVE the directory the agent is given, which Claude Code confines its
+ *      file tools to.
+ */
+import type { TaskSpec, TrialResult, TrialTokens } from './types.js';
+export interface RunTrialOptions {
+    readonly task: TaskSpec;
+    readonly trial: number;
+    /** Fresh directory for this trial. Created; must not already hold state. */
+    readonly trialDir: string;
+    readonly baseUrl: string;
+    readonly model: string;
+    readonly agentCli: string;
+    /** Wall-clock budget for the agent turn. */
+    readonly timeoutMs: number;
+    /**
+     * Insert the tracing proxy in front of `baseUrl`. Default on.
+     *
+     * Off is for proving the instrument innocent: if a result differs with
+     * tracing on and off, the instrument is the finding, not the model.
+     */
+    readonly trace?: boolean;
+}
+/**
+ * How many files differ between two snapshots.
+ *
+ * Counts additions, deletions and modifications alike. Returning a count rather
+ * than a diff is deliberate: `turnIntegrity` wants exactly this number, and it
+ * is the one fact that decides whether a completion claim was supported.
+ */
+export declare function countChanges(before: Map<string, string>, after: Map<string, string>): number;
+/** Pull the fields we report out of the agent CLI's result JSON. */
+export declare function parseAgentResult(stdout: string): {
+    finalMessage: string;
+    tokens: TrialTokens;
+    turns: number | null;
+    stopReason: string | null;
+    terminalReason: string | null;
+    isError: boolean;
+};
+/** Run one task once, end to end, and return a fully classified result. */
+export declare function runTrial(opts: RunTrialOptions): Promise<TrialResult>;

package/dist/core/local-eval/runner.js

@@ -0,0 +1,290 @@
+/**
+ * Running one trial, and running a whole set of them.
+ *
+ * THREE RULES THIS FILE ENFORCES, each learned the hard way here:
+ *
+ *   1. NEVER PIPE A COMMAND WHOSE STATUS MATTERS. `vitest ... | tail` reports
+ *      tail's exit code, and has already made a failing run look green in this
+ *      repository. Every child process below is spawned with its output
+ *      captured to a buffer and a file, and the status read from the child
+ *      itself. Nothing in this harness is piped.
+ *   2. EVERY TRIAL IS ISOLATED. Fixtures are rebuilt from source into a fresh
+ *      directory per trial, so trial 3 cannot inherit trial 2's half-finished
+ *      edit and the order trials run in cannot change the result.
+ *   3. THE AGENT CANNOT REACH ITS OWN GRADER. The checker is written one level
+ *      ABOVE the directory the agent is given, which Claude Code confines its
+ *      file tools to.
+ */
+import { spawn } from 'node:child_process';
+import { createHash } from 'node:crypto';
+import { mkdir, readdir, readFile, rm, writeFile } from 'node:fs/promises';
+import { dirname, join, relative } from 'node:path';
+import { classifyTrial } from './classify.js';
+import { diagnoseTimeout, startTrace } from './trace.js';
+/** Directories that are never part of a fixture's observable state. */
+const SNAPSHOT_IGNORE = new Set(['.git', 'node_modules', '.claude']);
+/**
+ * Spawn a child, capture both streams in full, and return its REAL status.
+ *
+ * `timeoutMs` is enforced here rather than with the `timeout` binary, which
+ * does not exist on this machine — a fact discovered when an invocation exited
+ * 127 and would have been read as a model failure by a harness that only
+ * checked for "not zero".
+ */
+function runChild(cmd, args, opts) {
+    return new Promise((resolve) => {
+        const child = spawn(cmd, [...args], {
+            cwd: opts.cwd,
+            env: opts.env ?? process.env,
+            stdio: ['ignore', 'pipe', 'pipe'],
+            // Own process group, so a timeout kills the agent's children too rather
+            // than orphaning a model request that keeps a runtime slot busy.
+            detached: true,
+        });
+        let stdout = '';
+        let stderr = '';
+        let timedOut = false;
+        child.stdout?.on('data', (c) => { stdout += c.toString('utf8'); });
+        child.stderr?.on('data', (c) => { stderr += c.toString('utf8'); });
+        const timer = opts.timeoutMs
+            ? setTimeout(() => {
+                timedOut = true;
+                try {
+                    opts.onTimeout?.();
+                }
+                catch { /* forensics must never block the kill */ }
+                try {
+                    process.kill(-child.pid, 'SIGKILL');
+                }
+                catch {
+                    child.kill('SIGKILL');
+                }
+            }, opts.timeoutMs)
+            : null;
+        const finish = (status) => {
+            if (timer)
+                clearTimeout(timer);
+            resolve({ status, stdout, stderr, timedOut });
+        };
+        child.on('error', () => finish(null));
+        child.on('close', (code) => finish(code));
+    });
+}
+/** Content hash of every file under `dir`, keyed by relative path. */
+async function snapshot(dir) {
+    const out = new Map();
+    const walk = async (current) => {
+        let entries;
+        try {
+            entries = await readdir(current, { withFileTypes: true });
+        }
+        catch {
+            return;
+        }
+        for (const entry of entries) {
+            if (SNAPSHOT_IGNORE.has(entry.name))
+                continue;
+            const full = join(current, entry.name);
+            if (entry.isDirectory()) {
+                await walk(full);
+                continue;
+            }
+            try {
+                const buf = await readFile(full);
+                out.set(relative(dir, full), createHash('sha256').update(buf).digest('hex'));
+            }
+            catch {
+                /* unreadable files are not observable state */
+            }
+        }
+    };
+    await walk(dir);
+    return out;
+}
+/**
+ * How many files differ between two snapshots.
+ *
+ * Counts additions, deletions and modifications alike. Returning a count rather
+ * than a diff is deliberate: `turnIntegrity` wants exactly this number, and it
+ * is the one fact that decides whether a completion claim was supported.
+ */
+export function countChanges(before, after) {
+    let changed = 0;
+    for (const [path, hash] of before) {
+        if (after.get(path) !== hash)
+            changed += 1;
+    }
+    for (const path of after.keys()) {
+        if (!before.has(path))
+            changed += 1;
+    }
+    return changed;
+}
+/** Pull the fields we report out of the agent CLI's result JSON. */
+export function parseAgentResult(stdout) {
+    const empty = { input: 0, output: 0, cacheRead: 0, cacheCreation: 0 };
+    let parsed;
+    try {
+        parsed = JSON.parse(stdout);
+    }
+    catch {
+        return {
+            finalMessage: '', tokens: empty, turns: null,
+            stopReason: null, terminalReason: null, isError: true,
+        };
+    }
+    const usage = (parsed['usage'] ?? {});
+    const num = (v) => (typeof v === 'number' && Number.isFinite(v) ? v : 0);
+    const str = (v) => (typeof v === 'string' ? v : null);
+    return {
+        finalMessage: typeof parsed['result'] === 'string' ? parsed['result'] : '',
+        tokens: {
+            input: num(usage['input_tokens']),
+            output: num(usage['output_tokens']),
+            cacheRead: num(usage['cache_read_input_tokens']),
+            cacheCreation: num(usage['cache_creation_input_tokens']),
+        },
+        turns: typeof parsed['num_turns'] === 'number' ? parsed['num_turns'] : null,
+        stopReason: str(parsed['stop_reason']),
+        terminalReason: str(parsed['terminal_reason']),
+        isError: parsed['is_error'] === true,
+    };
+}
+/** Write a task's fixture into `trialDir`, with the checker out of reach. */
+async function materialise(task, trialDir) {
+    const work = join(trialDir, 'work');
+    await rm(trialDir, { recursive: true, force: true });
+    await mkdir(work, { recursive: true });
+    for (const [path, contents] of Object.entries(task.files)) {
+        const target = join(work, path);
+        await mkdir(dirname(target), { recursive: true });
+        await writeFile(target, contents, 'utf8');
+    }
+    // Fixtures are ESM. Without this, `import './total.js'` fails for reasons
+    // that have nothing to do with the agent under test.
+    await writeFile(join(work, 'package.json'), JSON.stringify({ type: 'module' }, null, 2), 'utf8');
+    await writeFile(join(trialDir, 'check.mjs'), task.check, 'utf8');
+    return work;
+}
+/** Run one task once, end to end, and return a fully classified result. */
+export async function runTrial(opts) {
+    const { task, trialDir } = opts;
+    const work = await materialise(task, trialDir);
+    const before = await snapshot(work);
+    const args = [
+        '-p', task.prompt,
+        // --bare keeps hooks, plugins, CLAUDE.md, auto-memory and keychain reads
+        // out of the measurement. Without it the harness would be scoring this
+        // machine's configuration as much as the model.
+        '--bare',
+        '--model', opts.model,
+        '--output-format', 'json',
+        '--permission-mode', 'bypassPermissions',
+        '--permission-prompts', 'none',
+        '--no-session-persistence',
+        '--strict-mcp-config', '--mcp-config', '{"mcpServers":{}}',
+    ];
+    // The agent talks to the tracer, and the tracer talks to whatever the run was
+    // pointed at. A trial that produces no result JSON is otherwise a black box:
+    // the CLI emits its usage and turn count once, at the end, so a kill at the
+    // budget records zero tokens and zero turns no matter how much work happened.
+    //
+    // The tracer is an OBSERVER, so failing to start one must not fail the trial
+    // it was only watching. A benchmark that dies because its instrument could
+    // not bind a port has turned the instrument into the experiment.
+    let trace = null;
+    if (opts.trace !== false) {
+        try {
+            trace = await startTrace({ upstream: opts.baseUrl, captureDir: join(trialDir, 'wire') });
+        }
+        catch {
+            trace = null;
+        }
+    }
+    const agentBaseUrl = trace?.baseUrl ?? opts.baseUrl;
+    // Read at the kill, not after it: see `onTimeout`.
+    let traceAtTimeout = null;
+    const started = Date.now();
+    let agent;
+    try {
+        agent = await runChild(opts.agentCli, args, {
+            cwd: work,
+            env: {
+                ...process.env,
+                ANTHROPIC_BASE_URL: agentBaseUrl,
+                ANTHROPIC_API_KEY: 'local-eval',
+                CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: '1',
+            },
+            timeoutMs: opts.timeoutMs,
+            onTimeout: () => { traceAtTimeout = trace?.snapshot() ?? null; },
+        });
+    }
+    finally {
+        await trace?.close().catch(() => undefined);
+    }
+    const wallMs = Date.now() - started;
+    const finalTrace = traceAtTimeout ?? trace?.snapshot() ?? null;
+    const timeoutDiagnosis = agent.timedOut && finalTrace ? diagnoseTimeout(finalTrace) : null;
+    if (finalTrace) {
+        await writeFile(join(trialDir, 'wire.summary.json'), JSON.stringify({ trace: finalTrace, timeoutDiagnosis }, null, 2), 'utf8');
+    }
+    // Kept for forensics: a pass rate you cannot drill into is not evidence.
+    await writeFile(join(trialDir, 'agent.stdout.json'), agent.stdout, 'utf8');
+    await writeFile(join(trialDir, 'agent.stderr.log'), agent.stderr, 'utf8');
+    const parsed = parseAgentResult(agent.stdout);
+    const after = await snapshot(work);
+    const changedFiles = countChanges(before, after);
+    // The check runs from the trial root so `check.mjs` can read ./work, and its
+    // status is read from the child — never through a pipe.
+    let verifyExit = null;
+    let verifyOutput = '';
+    if (!agent.timedOut) {
+        // `verify` is a full argv INCLUDING the binary, so the head is the command
+        // and the tail is its arguments. Passing the whole array as arguments ran
+        // `node node check.mjs`, which fails with MODULE_NOT_FOUND — and scored a
+        // flawless answer as `wrong-edit`. A harness bug that only ever moves the
+        // pass rate DOWN is the most expensive kind: it looks like a model result.
+        const [checkBin, ...checkArgs] = task.verify;
+        const check = await runChild(checkBin, checkArgs, { cwd: trialDir, timeoutMs: 60_000 });
+        verifyExit = check.status;
+        verifyOutput = check.stdout + check.stderr;
+        await writeFile(join(trialDir, 'verify.log'), verifyOutput, 'utf8');
+    }
+    const verdict = classifyTrial({
+        expectation: task.expectation,
+        timedOut: agent.timedOut,
+        agentExit: agent.status,
+        agentReportedError: parsed.isError,
+        stopReason: parsed.stopReason,
+        terminalReason: parsed.terminalReason,
+        finalMessage: parsed.finalMessage,
+        changedFiles,
+        verifyExit,
+        diagnostics: `${agent.stderr}\n${verifyOutput}`,
+    });
+    const firstFailLine = verifyOutput.split('\n').find((l) => l.startsWith('FAIL:')) ?? '';
+    return {
+        taskId: task.id,
+        trial: opts.trial,
+        mode: verdict.mode,
+        passed: verdict.passed,
+        wallMs,
+        tokens: parsed.tokens,
+        agentExit: agent.status,
+        verifyExit,
+        changedFiles,
+        claim: verdict.claim,
+        integrity: verdict.integrity,
+        turns: parsed.turns,
+        // A timeout's note is its diagnosis. "killed at the wall-clock budget"
+        // restates the classification and explains nothing; the whole reason the
+        // tracer exists is so this line names a thing to go and fix.
+        note: firstFailLine
+            || (agent.timedOut
+                ? (timeoutDiagnosis ? `${timeoutDiagnosis.kind}: ${timeoutDiagnosis.detail}` : 'killed at the wall-clock budget (untraced)')
+                : ''),
+        timeoutDiagnosis,
+        trace: finalTrace,
+    };
+}
+//# sourceMappingURL=runner.js.map

package/dist/core/local-eval/runner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../../src/core/local-eval/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC3E,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AACpD,OAAO,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC9C,OAAO,EAAE,eAAe,EAAE,UAAU,EAAoB,MAAM,YAAY,CAAC;AAG3E,uEAAuE;AACvE,MAAM,eAAe,GAAwB,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,cAAc,EAAE,SAAS,CAAC,CAAC,CAAC;AA4B1F;;;;;;;GAOG;AACH,SAAS,QAAQ,CACf,GAAW,EACX,IAAuB,EACvB,IAcC;IAED,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QAC7B,MAAM,KAAK,GAAG,KAAK,CAAC,GAAG,EAAE,CAAC,GAAG,IAAI,CAAC,EAAE;YAClC,GAAG,EAAE,IAAI,CAAC,GAAG;YACb,GAAG,EAAE,IAAI,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG;YAC5B,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC;YACjC,wEAAwE;YACxE,iEAAiE;YACjE,QAAQ,EAAE,IAAI;SACf,CAAC,CAAC;QAEH,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,IAAI,QAAQ,GAAG,KAAK,CAAC;QACrB,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,CAAS,EAAE,EAAE,GAAG,MAAM,IAAI,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAC3E,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,CAAS,EAAE,EAAE,GAAG,MAAM,IAAI,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAE3E,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS;YAC1B,CAAC,CAAC,UAAU,CAAC,GAAG,EAAE;gBACd,QAAQ,GAAG,IAAI,CAAC;gBAChB,IAAI,CAAC;oBAAC,IAAI,CAAC,SAAS,EAAE,EAAE,CAAC;gBAAC,CAAC;gBAAC,MAAM,CAAC,CAAC,yCAAyC,CAAC,CAAC;gBAC/E,IAAI,CAAC;oBAAC,OAAO,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,GAAI,EAAE,SAAS,CAAC,CAAC;gBAAC,CAAC;gBAAC,MAAM,CAAC;oBAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;gBAAC,CAAC;YAChF,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC;YACpB,CAAC,CAAC,IAAI,CAAC;QAET,MAAM,MAAM,GAAG,CAAC,MAAqB,EAAQ,EAAE;YAC7C,IAAI,KAAK;gBAAE,YAAY,CAAC,KAAK,CAAC,CAAC;YAC/B,OAAO,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC;QAChD,CAAC,CAAC;QACF,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;QACtC,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;IAC5C,CAAC,CAAC,CAAC;AACL,CAAC;AAED,sEAAsE;AACtE,KAAK,UAAU,QAAQ,CAAC,GAAW;IACjC,MAAM,GAAG,GAAG,IAAI,GAAG,EAAkB,CAAC;IACtC,MAAM,IAAI,GAAG,KAAK,EAAE,OAAe,EAAiB,EAAE;QACpD,IAAI,OAAO,CAAC;QACZ,IAAI,CAAC;YACH,OAAO,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;QAC5D,CAAC;QAAC,MAAM,CAAC;YACP,OAAO;QACT,CAAC;QACD,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,IAAI,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC;gBAAE,SAAS;YAC9C,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;YACvC,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;gBACxB,MAAM,IAAI,CAAC,IAAI,CAAC,CAAC;gBACjB,SAAS;YACX,CAAC;YACD,IAAI,CAAC;gBACH,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,CAAC;gBACjC,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;YAC/E,CAAC;YAAC,MAAM,CAAC;gBACP,+CAA+C;YACjD,CAAC;QACH,CAAC;IACH,CAAC,CAAC;IACF,MAAM,IAAI,CAAC,GAAG,CAAC,CAAC;IAChB,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,YAAY,CAAC,MAA2B,EAAE,KAA0B;IAClF,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,KAAK,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,MAAM,EAAE,CAAC;QAClC,IAAI,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC,CAAC;IAC7C,CAAC;IACD,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;QAChC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,OAAO,IAAI,CAAC,CAAC;IACtC,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,oEAAoE;AACpE,MAAM,UAAU,gBAAgB,CAAC,MAAc;IAQ7C,MAAM,KAAK,GAAgB,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,CAAC;IACnF,IAAI,MAA+B,CAAC;IACpC,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAA4B,CAAC;IACzD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO;YACL,YAAY,EAAE,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,IAAI;YAC5C,UAAU,EAAE,IAAI,EAAE,cAAc,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI;SACtD,CAAC;IACJ,CAAC;IACD,MAAM,KAAK,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,EAAE,CAA4B,CAAC;IACjE,MAAM,GAAG,GAAG,CAAC,CAAU,EAAU,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,QAAQ,IAAI,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC1F,MAAM,GAAG,GAAG,CAAC,CAAU,EAAiB,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAC9E,OAAO;QACL,YAAY,EAAE,OAAO,MAAM,CAAC,QAAQ,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE;QAC1E,MAAM,EAAE;YACN,KAAK,EAAE,GAAG,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;YACjC,MAAM,EAAE,GAAG,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;YACnC,SAAS,EAAE,GAAG,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC;YAChD,aAAa,EAAE,GAAG,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAC;SACzD;QACD,KAAK,EAAE,OAAO,MAAM,CAAC,WAAW,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,IAAI;QAC3E,UAAU,EAAE,GAAG,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;QACtC,cAAc,EAAE,GAAG,CAAC,MAAM,CAAC,iBAAiB,CAAC,CAAC;QAC9C,OAAO,EAAE,MAAM,CAAC,UAAU,CAAC,KAAK,IAAI;KACrC,CAAC;AACJ,CAAC;AAED,6EAA6E;AAC7E,KAAK,UAAU,WAAW,CAAC,IAAc,EAAE,QAAgB;IACzD,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;IACpC,MAAM,EAAE,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IACrD,MAAM,KAAK,CAAC,IAAI,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACvC,KAAK,MAAM,CAAC,IAAI,EAAE,QAAQ,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QAC1D,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAChC,MAAM,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAClD,MAAM,SAAS,CAAC,MAAM,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;IAC5C,CAAC;IACD,0EAA0E;IAC1E,qDAAqD;IACrD,MAAM,SAAS,CAAC,IAAI,CAAC,IAAI,EAAE,cAAc,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;IACjG,MAAM,SAAS,CAAC,IAAI,CAAC,QAAQ,EAAE,WAAW,CAAC,EAAE,IAAI,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IACjE,OAAO,IAAI,CAAC;AACd,CAAC;AAED,2EAA2E;AAC3E,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,IAAqB;IAClD,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC;IAChC,MAAM,IAAI,GAAG,MAAM,WAAW,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IAC/C,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,CAAC;IAEpC,MAAM,IAAI,GAAG;QACX,IAAI,EAAE,IAAI,CAAC,MAAM;QACjB,yEAAyE;QACzE,uEAAuE;QACvE,gDAAgD;QAChD,QAAQ;QACR,SAAS,EAAE,IAAI,CAAC,KAAK;QACrB,iBAAiB,EAAE,MAAM;QACzB,mBAAmB,EAAE,mBAAmB;QACxC,sBAAsB,EAAE,MAAM;QAC9B,0BAA0B;QAC1B,qBAAqB,EAAE,cAAc,EAAE,mBAAmB;KAC3D,CAAC;IAEF,8EAA8E;IAC9E,6EAA6E;IAC7E,4EAA4E;IAC5E,8EAA8E;IAC9E,EAAE;IACF,6EAA6E;IAC7E,2EAA2E;IAC3E,iEAAiE;IACjE,IAAI,KAAK,GAAuB,IAAI,CAAC;IACrC,IAAI,IAAI,CAAC,KAAK,KAAK,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC;YACH,KAAK,GAAG,MAAM,UAAU,CAAC,EAAE,QAAQ,EAAE,IAAI,CAAC,OAAO,EAAE,UAAU,EAAE,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC;QAC3F,CAAC;QAAC,MAAM,CAAC;YACP,KAAK,GAAG,IAAI,CAAC;QACf,CAAC;IACH,CAAC;IACD,MAAM,YAAY,GAAG,KAAK,EAAE,OAAO,IAAI,IAAI,CAAC,OAAO,CAAC;IAEpD,mDAAmD;IACnD,IAAI,cAAc,GAAsB,IAAI,CAAC;IAE7C,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC3B,IAAI,KAAmB,CAAC;IACxB,IAAI,CAAC;QACH,KAAK,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,EAAE;YAC1C,GAAG,EAAE,IAAI;YACT,GAAG,EAAE;gBACH,GAAG,OAAO,CAAC,GAAG;gBACd,kBAAkB,EAAE,YAAY;gBAChC,iBAAiB,EAAE,YAAY;gBAC/B,wCAAwC,EAAE,GAAG;aAC9C;YACD,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,SAAS,EAAE,GAAG,EAAE,GAAG,cAAc,GAAG,KAAK,EAAE,QAAQ,EAAE,IAAI,IAAI,CAAC,CAAC,CAAC;SACjE,CAAC,CAAC;IACL,CAAC;YAAS,CAAC;QACT,MAAM,KAAK,EAAE,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;IAC9C,CAAC;IACD,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC;IAEpC,MAAM,UAAU,GAAsB,cAAc,IAAI,KAAK,EAAE,QAAQ,EAAE,IAAI,IAAI,CAAC;IAClF,MAAM,gBAAgB,GACpB,KAAK,CAAC,QAAQ,IAAI,UAAU,CAAC,CAAC,CAAC,eAAe,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IACpE,IAAI,UAAU,EAAE,CAAC;QACf,MAAM,SAAS,CAAC,IAAI,CAAC,QAAQ,EAAE,mBAAmB,CAAC,EACjD,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,gBAAgB,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;IAC9E,CAAC;IAED,yEAAyE;IACzE,MAAM,SAAS,CAAC,IAAI,CAAC,QAAQ,EAAE,mBAAmB,CAAC,EAAE,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC3E,MAAM,SAAS,CAAC,IAAI,CAAC,QAAQ,EAAE,kBAAkB,CAAC,EAAE,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAE1E,MAAM,MAAM,GAAG,gBAAgB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAC9C,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,YAAY,GAAG,YAAY,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IAEjD,6EAA6E;IAC7E,wDAAwD;IACxD,IAAI,UAAU,GAAkB,IAAI,CAAC;IACrC,IAAI,YAAY,GAAG,EAAE,CAAC;IACtB,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC;QACpB,2EAA2E;QAC3E,0EAA0E;QAC1E,0EAA0E;QAC1E,0EAA0E;QAC1E,2EAA2E;QAC3E,MAAM,CAAC,QAAQ,EAAE,GAAG,SAAS,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC;QAC7C,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,QAAS,EAAE,SAAS,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC;QACzF,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC;QAC1B,YAAY,GAAG,KAAK,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;QAC3C,MAAM,SAAS,CAAC,IAAI,CAAC,QAAQ,EAAE,YAAY,CAAC,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;IACtE,CAAC;IAED,MAAM,OAAO,GAAG,aAAa,CAAC;QAC5B,WAAW,EAAE,IAAI,CAAC,WAAW;QAC7B,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,SAAS,EAAE,KAAK,CAAC,MAAM;QACvB,kBAAkB,EAAE,MAAM,CAAC,OAAO;QAClC,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,cAAc,EAAE,MAAM,CAAC,cAAc;QACrC,YAAY,EAAE,MAAM,CAAC,YAAY;QACjC,YAAY;QACZ,UAAU;QACV,WAAW,EAAE,GAAG,KAAK,CAAC,MAAM,KAAK,YAAY,EAAE;KAChD,CAAC,CAAC;IAEH,MAAM,aAAa,GAAG,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;IACxF,OAAO;QACL,MAAM,EAAE,IAAI,CAAC,EAAE;QACf,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,IAAI,EAAE,OAAO,CAAC,IAAI;QAClB,MAAM,EAAE,OAAO,CAAC,MAAM;QACtB,MAAM;QACN,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,SAAS,EAAE,KAAK,CAAC,MAAM;QACvB,UAAU;QACV,YAAY;QACZ,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,SAAS,EAAE,OAAO,CAAC,SAAS;QAC5B,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,uEAAuE;QACvE,yEAAyE;QACzE,6DAA6D;QAC7D,IAAI,EAAE,aAAa;eACd,CAAC,KAAK,CAAC,QAAQ;gBAChB,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,GAAG,gBAAgB,CAAC,IAAI,KAAK,gBAAgB,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,4CAA4C,CAAC;gBAC5H,CAAC,CAAC,EAAE,CAAC;QACT,gBAAgB;QAChB,KAAK,EAAE,UAAU;KAClB,CAAC;AACJ,CAAC"}

package/dist/core/local-eval/tasks.d.ts

@@ -0,0 +1,22 @@
+/**
+ * The task set.
+ *
+ * SELECTION RULE. Every task here is a shape of work the local seat is actually
+ * asked to do and that a one-line fix does not exercise. Toy fixes were the
+ * whole sample before this file existed, and they are the reason nobody could
+ * say what the local seat's success rate was: a single-line `a + b` → `a * b`
+ * is passed by any model that can use the Edit tool at all, so it discriminates
+ * nothing. Each task below fails for a DIFFERENT reason, which is what makes
+ * the failure-mode column worth reading.
+ *
+ * EVERY CHECK IS A COMMAND. No task is graded by asking a model whether the
+ * answer looks right. Each `check` is a Node script whose exit code is the
+ * verdict, and it lives above the agent's working directory so the agent cannot
+ * edit its own grader.
+ *
+ * FIXTURES ARE PURE ESM WITH NO DEPENDENCIES. A trial must not depend on
+ * `npm install` completing, because a network hiccup would then read as a model
+ * regression. `node --test` and `node:assert` ship with the runtime.
+ */
+import type { TaskSpec } from './types.js';
+export declare const TASKS: readonly TaskSpec[];