@theokit/sdk 4.63.3 → 5.0.0-next.1

package/dist/eval.cjs

@@ -1,61 +1,62 @@
 'use strict';
 
-var chunkJ24VJOH3_cjs = require('./chunk-J24VJOH3.cjs');
-var chunkJJ3OVRUM_cjs = require('./chunk-JJ3OVRUM.cjs');
-require('./chunk-3FFAEAUJ.cjs');
-require('./chunk-RMWTPV3M.cjs');
+var chunkUADQJBHR_cjs = require('./chunk-UADQJBHR.cjs');
+var chunkWMH2MSOP_cjs = require('./chunk-WMH2MSOP.cjs');
+require('./chunk-KVSAY6NZ.cjs');
+require('./chunk-Y2KYR2ED.cjs');
 var chunkBUUUWQMB_cjs = require('./chunk-BUUUWQMB.cjs');
-require('./chunk-VLVVJLDH.cjs');
-require('./chunk-ZCIRZE37.cjs');
+require('./chunk-2NE7DTOK.cjs');
+require('./chunk-HWSALKOR.cjs');
 require('./chunk-BV2MWEMV.cjs');
-require('./chunk-KQ7TS7WW.cjs');
+require('./chunk-D6POWE7E.cjs');
 require('./chunk-GHX4P3V2.cjs');
-require('./chunk-GKEFRLU7.cjs');
-require('./chunk-IAHZCGBE.cjs');
-require('./chunk-ZLQSWSJ2.cjs');
-require('./chunk-KYAJINUI.cjs');
-require('./chunk-O7DK5PBJ.cjs');
-require('./chunk-4OTIXDMU.cjs');
-require('./chunk-UGT7GK7I.cjs');
-require('./chunk-O7LZFG6O.cjs');
+require('./chunk-TMKTSYDS.cjs');
+require('./chunk-LHSWHYF3.cjs');
+require('./chunk-7FDKWDA2.cjs');
+require('./chunk-2HPNT6HH.cjs');
+require('./chunk-FUSDYC2B.cjs');
+require('./chunk-YJNYDJ2T.cjs');
+require('./chunk-24SYBZPL.cjs');
+require('./chunk-QOGNZMHD.cjs');
+require('./chunk-YIFFF23U.cjs');
 require('./chunk-2ZPEDVLM.cjs');
-require('./chunk-7BG5UH3T.cjs');
+require('./chunk-RHZ6HP23.cjs');
 require('./chunk-KRD3GQAA.cjs');
-require('./chunk-2HVF2WJY.cjs');
+require('./chunk-RQMCOFAI.cjs');
 require('./chunk-MUUQ2WFJ.cjs');
-require('./chunk-WJVHMTKB.cjs');
-require('./chunk-6JSIEDKL.cjs');
+require('./chunk-AA27GEYS.cjs');
+require('./chunk-YSTXFWQE.cjs');
 require('./chunk-FRITRLZQ.cjs');
-require('./chunk-4O5TGQBW.cjs');
+require('./chunk-QME6FDFG.cjs');
 var chunkNINIJCHU_cjs = require('./chunk-NINIJCHU.cjs');
-require('./chunk-OHLDPWTA.cjs');
+require('./chunk-UGRS7ZA7.cjs');
 require('./chunk-LKFET5A3.cjs');
-require('./chunk-VJ26EKGJ.cjs');
-require('./chunk-V2BOS4WF.cjs');
-require('./chunk-VKJ7V7EB.cjs');
-require('./chunk-ZSWPZ23K.cjs');
-require('./chunk-XWTLIGO3.cjs');
-require('./chunk-I6HGWHPU.cjs');
-require('./chunk-5HEG4CQH.cjs');
+require('./chunk-IJ7M6GOG.cjs');
+require('./chunk-7A6535RA.cjs');
+require('./chunk-SA6K24NB.cjs');
+require('./chunk-I2PWY7VM.cjs');
+require('./chunk-KTTHHYEW.cjs');
+require('./chunk-Z3NTRKIC.cjs');
+require('./chunk-GZV6AVRL.cjs');
 require('./chunk-A4RAL2ER.cjs');
-require('./chunk-5VB22SYX.cjs');
-var chunkUU36SHFY_cjs = require('./chunk-UU36SHFY.cjs');
+require('./chunk-DQZU7JK6.cjs');
+var chunkBJUJT5ED_cjs = require('./chunk-BJUJT5ED.cjs');
 require('./chunk-ZF2LDKQQ.cjs');
-require('./chunk-RFMBZPOW.cjs');
-require('./chunk-UALX2J2Q.cjs');
-require('./chunk-CQBXOJ6U.cjs');
-require('./chunk-AQO3NSRG.cjs');
-require('./chunk-4CLOXP6H.cjs');
-require('./chunk-DXGZ2NUG.cjs');
+require('./chunk-HCT4HPCL.cjs');
+require('./chunk-JLRLCBJ4.cjs');
+require('./chunk-G4ZGYKJG.cjs');
+require('./chunk-CEILSTWB.cjs');
+require('./chunk-HW7SEELD.cjs');
+require('./chunk-ATT276RD.cjs');
 require('./chunk-3EE6LVWT.cjs');
-require('./chunk-JPBKYL3B.cjs');
-require('./chunk-HCZD23VS.cjs');
-require('./chunk-6HKGLTJI.cjs');
-require('./chunk-XJCXEKMD.cjs');
-require('./chunk-52NKC5HT.cjs');
-require('./chunk-4ZHTIJNE.cjs');
-require('./chunk-XL7Q7HB2.cjs');
-var chunkEG4QM7ZJ_cjs = require('./chunk-EG4QM7ZJ.cjs');
+require('./chunk-ANMJJD6D.cjs');
+require('./chunk-QC55H6RJ.cjs');
+require('./chunk-NVXXWAU7.cjs');
+require('./chunk-4NLCWGH7.cjs');
+require('./chunk-EXTRW7GW.cjs');
+require('./chunk-HUFSABUZ.cjs');
+var chunkJ7J7J2GN_cjs = require('./chunk-J7J7J2GN.cjs');
+var chunk6LHQPOMI_cjs = require('./chunk-6LHQPOMI.cjs');
 var zod = require('zod');
 var crypto = require('crypto');
 
@@ -172,12 +173,13 @@ async function materializeDataset(d) {
 
 // src/internal/eval/single-flight.ts
 var RUNNING = /* @__PURE__ */ new Set();
-var EvalAlreadyRunningError = class extends Error {
+var EvalAlreadyRunningError = class extends chunkJ7J7J2GN_cjs.TheokitAgentError {
   name = "EvalAlreadyRunningError";
   evalName;
   constructor(evalName) {
     super(
-      `Eval "${evalName}" is already running in this process. Use a unique name per concurrent run (D213).`
+      `Eval "${evalName}" is already running in this process. Use a unique name per concurrent run (D213).`,
+      { code: "eval_already_running", isRetryable: false }
     );
     this.evalName = evalName;
   }
@@ -312,7 +314,7 @@ function safeHook(fn) {
   try {
     fn();
   } catch (err) {
-    chunkEG4QM7ZJ_cjs.diag(`[eval] hook threw (ignored): ${err instanceof Error ? err.message : String(err)}
+    chunk6LHQPOMI_cjs.diag(`[eval] hook threw (ignored): ${err instanceof Error ? err.message : String(err)}
 `);
   }
 }
@@ -341,16 +343,16 @@ function probeRow(entry, index) {
 }
 function computeDoneKeys(persist) {
   if (persist.resume !== true) return /* @__PURE__ */ new Set();
-  return chunkUU36SHFY_cjs.readJsonlIds(
+  return chunkBJUJT5ED_cjs.readJsonlIds(
     persist.path,
     (parsed) => parsed.error === void 0 ? persist.key(parsed) : void 0
   );
 }
 function appendRowSafely(path, row) {
   try {
-    chunkUU36SHFY_cjs.appendJsonl(path, row);
+    chunkBJUJT5ED_cjs.appendJsonl(path, row);
   } catch (err) {
-    chunkEG4QM7ZJ_cjs.diag(
+    chunk6LHQPOMI_cjs.diag(
       `[eval] persist append failed (ignored): ${err instanceof Error ? err.message : String(err)}
 `
     );
@@ -416,7 +418,8 @@ function makeAgentForBatch(spec, _entries) {
   }
   return spec;
 }
-async function runManualSlot(idx, entries, spec, scorers, sink, rows, onRow) {
+async function runManualSlot(ctx, spec, idx, rows) {
+  const { entries, scorers, sink, onRow } = ctx;
   const entry = entries[idx];
   if (entry === void 0) return;
   if (sink.isResumed(entry, idx)) return;
@@ -424,7 +427,8 @@ async function runManualSlot(idx, entries, spec, scorers, sink, rows, onRow) {
   rows[idx] = row;
   onRow(row, idx);
 }
-async function runRowsManually(entries, spec, scorers, concurrency, signal, onRow, sink) {
+async function runRowsManually(ctx, spec) {
+  const { entries, concurrency, signal } = ctx;
   const rows = new Array(entries.length);
   const state = { cursor: 0 };
   const worker = async () => {
@@ -432,7 +436,7 @@ async function runRowsManually(entries, spec, scorers, concurrency, signal, onRo
       if (signal?.aborted === true) return;
       const idx = state.cursor;
       state.cursor += 1;
-      await runManualSlot(idx, entries, spec, scorers, sink, rows, onRow);
+      await runManualSlot(ctx, spec, idx, rows);
     }
   };
   const workers = Array.from({ length: Math.min(concurrency, entries.length) }, () => worker());
@@ -482,7 +486,8 @@ async function scoreBatchOutput(br, expected, scorers) {
   }
   return scoreEntries;
 }
-async function runRowsViaBatch(entries, agentOptions, scorers, concurrency, signal, onRow, sink) {
+async function runRowsViaBatch(ctx, agentOptions) {
+  const { entries, scorers, concurrency, signal, onRow, sink } = ctx;
   const pending = [];
   for (let i = 0; i < entries.length; i += 1) {
     const entry = entries[i];
@@ -535,21 +540,11 @@ async function runEval(options, runOpts) {
       safeHook(() => hooks?.afterRow?.(row, i));
     };
     const sink = makeRowSink(runOpts?.persist, runOpts?.classify);
-    let rows;
-    if (isAgentInstance(options.agent) || typeof options.agent === "function") {
-      rows = await runRowsManually(
-        indexed,
-        options.agent,
-        scorers,
-        concurrency,
-        signal,
-        onRow,
-        sink
-      );
-    } else {
-      const batchOpts = makeAgentForBatch(options.agent, indexed);
-      rows = await runRowsViaBatch(indexed, batchOpts, scorers, concurrency, signal, onRow, sink);
-    }
+    const runCtx = { entries: indexed, scorers, concurrency, signal, onRow, sink };
+    const rows = await (isAgentInstance(options.agent) || typeof options.agent === "function" ? runRowsManually(
+      runCtx,
+      options.agent
+    ) : runRowsViaBatch(runCtx, makeAgentForBatch(options.agent, indexed)));
     const finalRows = trials > 1 ? collapseTrials(rows, trials) : rows;
     const aggregate = computeAggregate(finalRows);
     const endedAt = Date.now();
@@ -576,7 +571,7 @@ async function runEval(options, runOpts) {
 }
 
 // src/internal/eval/assert.ts
-var EvalThresholdError = class extends Error {
+var EvalThresholdError = class extends chunkJ7J7J2GN_cjs.TheokitAgentError {
   name = "EvalThresholdError";
   /** The eval's name (`EvalRun.name`). */
   evalName;
@@ -588,7 +583,8 @@ var EvalThresholdError = class extends Error {
     );
     super(
       `Eval "${evalName}" failed ${failures.length} threshold${failures.length === 1 ? "" : "s"}:
-${lines.join("\n")}`
+${lines.join("\n")}`,
+      { code: "eval_threshold_failed", isRetryable: false }
     );
     this.evalName = evalName;
     this.failures = failures;
@@ -631,13 +627,13 @@ function assertEval(run, thresholds) {
 // src/internal/eval/code-runner.ts
 var ARTIFACT_PATCH = ".theo-artifact.patch";
 async function captureArtifact(sandbox, repoDir) {
-  const dir = chunkJ24VJOH3_cjs.shellEscapePosix(repoDir);
+  const dir = chunkUADQJBHR_cjs.shellEscapePosix(repoDir);
   const diffRes = await sandbox.execute(`git -C ${dir} diff`);
   const diff = diffRes.stdout;
   if (diff.length === 0) return { diff: "", applies: false };
   await sandbox.uploadFile(`${repoDir}/${ARTIFACT_PATCH}`, diff);
   const check = await sandbox.execute(
-    `git -C ${dir} apply --check --reverse ${chunkJ24VJOH3_cjs.shellEscapePosix(ARTIFACT_PATCH)}`
+    `git -C ${dir} apply --check --reverse ${chunkUADQJBHR_cjs.shellEscapePosix(ARTIFACT_PATCH)}`
   );
   return { diff, applies: check.exitCode === 0 };
 }
@@ -784,7 +780,7 @@ function makeDefaultEmbedder(opts) {
   let runtime;
   return (texts) => {
     if (runtime === void 0) {
-      runtime = chunkJJ3OVRUM_cjs.openRouterMemoryEmbeddingProviderAdapter.create(embedderCreateOptions(opts));
+      runtime = chunkWMH2MSOP_cjs.openRouterMemoryEmbeddingProviderAdapter.create(embedderCreateOptions(opts));
     }
     return runtime.then((rt) => rt.embed(texts));
   };
@@ -930,7 +926,7 @@ var Scorers = {
    * that rejects shell metacharacters in `execute` is unsupported for this scorer.
    */
   verifyGate(opts) {
-    const { sandbox = new chunkJ24VJOH3_cjs.LocalSandbox(), repoDir, failToPass, passToPass, command } = opts;
+    const { sandbox = new chunkUADQJBHR_cjs.LocalSandbox(), repoDir, failToPass, passToPass, command } = opts;
     return {
       name: "verify-gate",
       score: async () => {
@@ -938,7 +934,7 @@ var Scorers = {
         if (cmd.length === 0) {
           return { score: 0, reason: "verify_gate_empty_command" };
         }
-        const r = await sandbox.execute(`cd ${chunkJ24VJOH3_cjs.shellEscapePosix(repoDir)} && ${cmd}`);
+        const r = await sandbox.execute(`cd ${chunkUADQJBHR_cjs.shellEscapePosix(repoDir)} && ${cmd}`);
         if (r.exitCode === 0) return { score: 1 };
         return { score: 0, reason: `exit=${r.exitCode} ${r.stderr.slice(0, 200)}`.trim() };
       }
@@ -1029,11 +1025,11 @@ var Eval = class _Eval {
 
 Object.defineProperty(exports, "JsonlParseError", {
   enumerable: true,
-  get: function () { return chunkUU36SHFY_cjs.JsonlParseError; }
+  get: function () { return chunkBJUJT5ED_cjs.JsonlParseError; }
 });
 Object.defineProperty(exports, "loadJsonl", {
   enumerable: true,
-  get: function () { return chunkUU36SHFY_cjs.loadJsonl; }
+  get: function () { return chunkBJUJT5ED_cjs.loadJsonl; }
 });
 exports.Eval = Eval;
 exports.EvalAlreadyRunningError = EvalAlreadyRunningError;