@theokit/sdk 4.63.4 → 5.0.0-next.2

package/dist/eval.js

@@ -1,60 +1,61 @@
-import { shellEscapePosix, LocalSandbox } from './chunk-2D34UTDC.js';
-import { openRouterMemoryEmbeddingProviderAdapter } from './chunk-LYTOXSRH.js';
-import './chunk-A46SWMIF.js';
-import './chunk-4FFBAUFS.js';
+import { shellEscapePosix, LocalSandbox } from './chunk-O6USDERH.js';
+import { openRouterMemoryEmbeddingProviderAdapter } from './chunk-KDR53EO7.js';
+import './chunk-K2VMFZQ5.js';
+import './chunk-AWINL3ZC.js';
 import { getTracer } from './chunk-K2BQQ445.js';
-import './chunk-L3MTL4NA.js';
-import './chunk-NDGGJQTZ.js';
+import './chunk-2ZLVT7H3.js';
+import './chunk-XJZEBGXD.js';
 import './chunk-2SFBB54R.js';
-import './chunk-UBDMVMBZ.js';
+import './chunk-55GB6JYQ.js';
 import './chunk-H73MEMQB.js';
-import './chunk-XO4SOZN6.js';
-import './chunk-L4BDVXIM.js';
-import './chunk-BP46RJ5X.js';
-import './chunk-DN2PACRI.js';
-import './chunk-TAZVKYWJ.js';
-import './chunk-WTMU7J4U.js';
-import './chunk-7YGCPMGB.js';
-import './chunk-N2U4AZGB.js';
+import './chunk-OR6XIWLB.js';
+import './chunk-UNK4HNJH.js';
+import './chunk-AAEZSPTC.js';
+import './chunk-K3NH62DG.js';
+import './chunk-UBR5PWD7.js';
+import './chunk-2XC6KDJM.js';
+import './chunk-XB4GI5RZ.js';
+import './chunk-24MGQDK3.js';
+import './chunk-2WXHA33T.js';
 import './chunk-HKCKOAIO.js';
-import './chunk-SAVIWMZB.js';
+import './chunk-ILGZ2ZZV.js';
 import './chunk-KVNWIAO4.js';
-import './chunk-PUAEFLYK.js';
+import './chunk-SVQIW5RJ.js';
 import './chunk-FD2UT76F.js';
-import './chunk-4VPXM6UU.js';
-import './chunk-U54XCYG3.js';
+import './chunk-7I5Z3UBJ.js';
+import './chunk-FJWQP6EE.js';
 import './chunk-NAF4L3T6.js';
-import './chunk-RNB4APBZ.js';
+import './chunk-6WAKKTBO.js';
 import { getAgentFacade } from './chunk-6X6ID4MO.js';
-import './chunk-O73J46PV.js';
+import './chunk-XDANGA2C.js';
 import './chunk-WFC26L6Y.js';
-import './chunk-7AD3ZTZL.js';
-import './chunk-HS7AVXTF.js';
-import './chunk-RIAM53CP.js';
-import './chunk-C72AZW7J.js';
-import './chunk-4QZSA2HJ.js';
-import './chunk-B3IMEBOA.js';
-import './chunk-UMP6OJIM.js';
+import './chunk-SAGRF4IB.js';
+import './chunk-EH6XD3FY.js';
+import './chunk-UKMBRMGT.js';
+import './chunk-4MMWFSCD.js';
+import './chunk-G7BLGBRN.js';
+import './chunk-VE6DFSKU.js';
+import './chunk-JOFVLOFY.js';
 import './chunk-WE22OXQA.js';
-import './chunk-LWGSI6M3.js';
-import { readJsonlIds, appendJsonl } from './chunk-3O42QOMJ.js';
-export { JsonlParseError, loadJsonl } from './chunk-3O42QOMJ.js';
+import './chunk-6PWOWXMC.js';
+import { readJsonlIds, appendJsonl } from './chunk-TA3K7SBK.js';
+export { JsonlParseError, loadJsonl } from './chunk-TA3K7SBK.js';
 import './chunk-Q5EWJPRY.js';
-import './chunk-KQRW2GPZ.js';
-import './chunk-2GUYQR5P.js';
-import './chunk-LKBEROHN.js';
-import './chunk-247ZTLNU.js';
-import './chunk-LXQ6HKPS.js';
-import './chunk-A4JVILQU.js';
+import './chunk-EIQFAOFD.js';
+import './chunk-3JHIFQ4I.js';
+import './chunk-4S5TWFIZ.js';
+import './chunk-67ADYSTW.js';
+import './chunk-JNAA4G4H.js';
+import './chunk-6M2OIS4Y.js';
 import './chunk-R7WIIPUR.js';
-import './chunk-UTSRIDKZ.js';
-import './chunk-WLQNMXQE.js';
-import './chunk-2QDYBS2A.js';
-import './chunk-C3NT3PVE.js';
-import './chunk-3OR54XG4.js';
-import './chunk-ZZMZWBR6.js';
-import './chunk-D4CUGAKZ.js';
-import { diag } from './chunk-XEQ3TM66.js';
+import './chunk-XYFGKVZ7.js';
+import './chunk-VWUXIVEI.js';
+import './chunk-AGACYRJ2.js';
+import './chunk-JMDDYA22.js';
+import './chunk-V22DZIXO.js';
+import './chunk-NJWYQWDL.js';
+import { TheokitAgentError } from './chunk-ALUN2B4W.js';
+import { diag } from './chunk-CZJ6Q7CW.js';
 import { z } from 'zod';
 import { randomUUID } from 'crypto';
 
@@ -171,12 +172,13 @@ async function materializeDataset(d) {
 
 // src/internal/eval/single-flight.ts
 var RUNNING = /* @__PURE__ */ new Set();
-var EvalAlreadyRunningError = class extends Error {
+var EvalAlreadyRunningError = class extends TheokitAgentError {
   name = "EvalAlreadyRunningError";
   evalName;
   constructor(evalName) {
     super(
-      `Eval "${evalName}" is already running in this process. Use a unique name per concurrent run (D213).`
+      `Eval "${evalName}" is already running in this process. Use a unique name per concurrent run (D213).`,
+      { code: "eval_already_running", isRetryable: false }
     );
     this.evalName = evalName;
   }
@@ -415,7 +417,8 @@ function makeAgentForBatch(spec, _entries) {
   }
   return spec;
 }
-async function runManualSlot(idx, entries, spec, scorers, sink, rows, onRow) {
+async function runManualSlot(ctx, spec, idx, rows) {
+  const { entries, scorers, sink, onRow } = ctx;
   const entry = entries[idx];
   if (entry === void 0) return;
   if (sink.isResumed(entry, idx)) return;
@@ -423,7 +426,8 @@ async function runManualSlot(idx, entries, spec, scorers, sink, rows, onRow) {
   rows[idx] = row;
   onRow(row, idx);
 }
-async function runRowsManually(entries, spec, scorers, concurrency, signal, onRow, sink) {
+async function runRowsManually(ctx, spec) {
+  const { entries, concurrency, signal } = ctx;
   const rows = new Array(entries.length);
   const state = { cursor: 0 };
   const worker = async () => {
@@ -431,7 +435,7 @@ async function runRowsManually(entries, spec, scorers, concurrency, signal, onRo
       if (signal?.aborted === true) return;
       const idx = state.cursor;
       state.cursor += 1;
-      await runManualSlot(idx, entries, spec, scorers, sink, rows, onRow);
+      await runManualSlot(ctx, spec, idx, rows);
     }
   };
   const workers = Array.from({ length: Math.min(concurrency, entries.length) }, () => worker());
@@ -481,7 +485,8 @@ async function scoreBatchOutput(br, expected, scorers) {
   }
   return scoreEntries;
 }
-async function runRowsViaBatch(entries, agentOptions, scorers, concurrency, signal, onRow, sink) {
+async function runRowsViaBatch(ctx, agentOptions) {
+  const { entries, scorers, concurrency, signal, onRow, sink } = ctx;
   const pending = [];
   for (let i = 0; i < entries.length; i += 1) {
     const entry = entries[i];
@@ -534,21 +539,11 @@ async function runEval(options, runOpts) {
       safeHook(() => hooks?.afterRow?.(row, i));
     };
     const sink = makeRowSink(runOpts?.persist, runOpts?.classify);
-    let rows;
-    if (isAgentInstance(options.agent) || typeof options.agent === "function") {
-      rows = await runRowsManually(
-        indexed,
-        options.agent,
-        scorers,
-        concurrency,
-        signal,
-        onRow,
-        sink
-      );
-    } else {
-      const batchOpts = makeAgentForBatch(options.agent, indexed);
-      rows = await runRowsViaBatch(indexed, batchOpts, scorers, concurrency, signal, onRow, sink);
-    }
+    const runCtx = { entries: indexed, scorers, concurrency, signal, onRow, sink };
+    const rows = await (isAgentInstance(options.agent) || typeof options.agent === "function" ? runRowsManually(
+      runCtx,
+      options.agent
+    ) : runRowsViaBatch(runCtx, makeAgentForBatch(options.agent, indexed)));
     const finalRows = trials > 1 ? collapseTrials(rows, trials) : rows;
     const aggregate = computeAggregate(finalRows);
     const endedAt = Date.now();
@@ -575,7 +570,7 @@ async function runEval(options, runOpts) {
 }
 
 // src/internal/eval/assert.ts
-var EvalThresholdError = class extends Error {
+var EvalThresholdError = class extends TheokitAgentError {
   name = "EvalThresholdError";
   /** The eval's name (`EvalRun.name`). */
   evalName;
@@ -587,7 +582,8 @@ var EvalThresholdError = class extends Error {
     );
     super(
       `Eval "${evalName}" failed ${failures.length} threshold${failures.length === 1 ? "" : "s"}:
-${lines.join("\n")}`
+${lines.join("\n")}`,
+      { code: "eval_threshold_failed", isRetryable: false }
     );
     this.evalName = evalName;
     this.failures = failures;