@theokit/sdk 2.4.0 → 2.5.0

package/dist/eval.d.cts

@@ -31,6 +31,8 @@ export declare class Eval {
      */
     run(runOpts?: EvalRunOptions): Promise<EvalRun>;
 }
+export { captureArtifact } from "./internal/eval/code-runner.js";
 export { EvalAlreadyRunningError } from "./internal/eval/single-flight.js";
+export { JsonlParseError, loadJsonl } from "./internal/persistence/jsonl.js";
 export { Scorers } from "./scorers.js";
 export type * from "./types/eval.js";

package/dist/eval.d.ts

@@ -31,6 +31,8 @@ export declare class Eval {
      */
     run(runOpts?: EvalRunOptions): Promise<EvalRun>;
 }
+export { captureArtifact } from "./internal/eval/code-runner.js";
 export { EvalAlreadyRunningError } from "./internal/eval/single-flight.js";
+export { JsonlParseError, loadJsonl } from "./internal/persistence/jsonl.js";
 export { Scorers } from "./scorers.js";
 export type * from "./types/eval.js";

package/dist/eval.js

@@ -2,7 +2,7 @@ import { randomUUID, randomBytes, createHash } from 'crypto';
 import { readFile, unlink, mkdir, open, rename, statfs, stat, rm, readdir, appendFile, access } from 'fs/promises';
 import { join, dirname, resolve, sep, relative, isAbsolute } from 'path';
 import { z, toJSONSchema } from 'zod';
-import { mkdirSync, readdirSync, existsSync, realpathSync, lstatSync, readlinkSync, readFileSync } from 'fs';
+import { readFileSync, mkdirSync, appendFileSync, readdirSync, existsSync, realpathSync, lstatSync, readlinkSync } from 'fs';
 import { AsyncLocalStorage } from 'async_hooks';
 import { createRequire } from 'module';
 import { homedir } from 'os';
@@ -15535,6 +15535,69 @@ setAgentFacade({
   resume: (agentId, options) => Agent.resume(agentId, options),
   batch: (prompts, options) => Agent.batch(prompts, options)
 });
+var JsonlParseError = class extends Error {
+  constructor(message, line) {
+    super(message);
+    this.line = line;
+    this.name = "JsonlParseError";
+  }
+  line;
+};
+function isPlainObject(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function tryParseObjectLine(line) {
+  if (line.length === 0) return void 0;
+  let parsed;
+  try {
+    parsed = JSON.parse(line);
+  } catch {
+    return void 0;
+  }
+  return isPlainObject(parsed) ? parsed : void 0;
+}
+function loadJsonl(path, opts = {}) {
+  const text = readFileSync(path, "utf8");
+  const out = [];
+  let lineNumber = 0;
+  for (const rawLine of text.split("\n")) {
+    lineNumber += 1;
+    const line = rawLine.trim();
+    if (line.length === 0) continue;
+    let parsed;
+    try {
+      parsed = JSON.parse(line);
+    } catch {
+      throw new JsonlParseError(`line ${lineNumber}: invalid JSON`, lineNumber);
+    }
+    if (!isPlainObject(parsed)) {
+      throw new JsonlParseError(`line ${lineNumber}: not a JSON object`, lineNumber);
+    }
+    out.push(opts.map ? opts.map(parsed, lineNumber) : parsed);
+  }
+  return out;
+}
+function appendJsonl(path, record) {
+  mkdirSync(dirname(path), { recursive: true });
+  appendFileSync(path, `${JSON.stringify(record)}
+`);
+}
+function readJsonlIds(path, keyFn) {
+  const done = /* @__PURE__ */ new Set();
+  let text;
+  try {
+    text = readFileSync(path, "utf8");
+  } catch {
+    return done;
+  }
+  for (const rawLine of text.split("\n")) {
+    const parsed = tryParseObjectLine(rawLine.trim());
+    if (parsed === void 0) continue;
+    const key = keyFn(parsed);
+    if (typeof key === "string" && key.length > 0) done.add(key);
+  }
+  return done;
+}
 
 // src/internal/eval/runner.ts
 init_agent_factory_registry();
@@ -15726,6 +15789,50 @@ function normalizeScorers(input) {
     return { name: s.name, score: s.score };
   });
 }
+function probeRow(entry, index) {
+  return {
+    index,
+    input: entry.input,
+    output: "",
+    ...entry.expected !== void 0 ? { expected: entry.expected } : {},
+    scores: [],
+    meanScore: 0,
+    durationMs: 0,
+    ...entry.metadata !== void 0 ? { metadata: entry.metadata } : {}
+  };
+}
+function computeDoneKeys(persist) {
+  if (persist.resume !== true) return /* @__PURE__ */ new Set();
+  return readJsonlIds(
+    persist.path,
+    (parsed) => parsed.error === void 0 ? persist.key(parsed) : void 0
+  );
+}
+function appendRowSafely(path, row) {
+  try {
+    appendJsonl(path, row);
+  } catch (err) {
+    console.warn(
+      "[eval] persist append failed (ignored):",
+      err instanceof Error ? err.message : err
+    );
+  }
+}
+function makeRowSink(persist, classify) {
+  const doneKeys = persist !== void 0 ? computeDoneKeys(persist) : /* @__PURE__ */ new Set();
+  return {
+    isResumed(entry, index) {
+      if (persist === void 0 || doneKeys.size === 0) return false;
+      return doneKeys.has(persist.key(probeRow(entry, index)));
+    },
+    finalize(row) {
+      const outcome = classify?.(row);
+      const finalRow = outcome !== void 0 ? { ...row, outcome } : row;
+      if (persist !== void 0) appendRowSafely(persist.path, finalRow);
+      return finalRow;
+    }
+  };
+}
 async function applyScorer(scorer, output, expected) {
   let raw;
   try {
@@ -15771,7 +15878,15 @@ function makeAgentForBatch(spec, _entries) {
   }
   return spec;
 }
-async function runRowsManually(entries, spec, scorers, concurrency, signal, onRow) {
+async function runManualSlot(idx, entries, spec, scorers, sink, rows, onRow) {
+  const entry = entries[idx];
+  if (entry === void 0) return;
+  if (sink.isResumed(entry, idx)) return;
+  const row = sink.finalize(await runOneEntry(spec, entry, idx, scorers));
+  rows[idx] = row;
+  onRow(row, idx);
+}
+async function runRowsManually(entries, spec, scorers, concurrency, signal, onRow, sink) {
   const rows = new Array(entries.length);
   const state2 = { cursor: 0 };
   const worker = async () => {
@@ -15779,11 +15894,7 @@ async function runRowsManually(entries, spec, scorers, concurrency, signal, onRo
       if (signal?.aborted === true) return;
       const idx = state2.cursor;
       state2.cursor += 1;
-      const entry = entries[idx];
-      if (entry === void 0) continue;
-      const row = await runOneEntry(spec, entry, idx, scorers);
-      rows[idx] = row;
-      onRow(row, idx);
+      await runManualSlot(idx, entries, spec, scorers, sink, rows, onRow);
     }
   };
   const workers = Array.from({ length: Math.min(concurrency, entries.length) }, () => worker());
@@ -15833,23 +15944,32 @@ async function scoreBatchOutput(br, expected, scorers) {
   }
   return scoreEntries;
 }
-async function runRowsViaBatch(entries, agentOptions, scorers, concurrency, signal, onRow) {
-  const prompts = entries.map((e) => e.input);
+async function runRowsViaBatch(entries, agentOptions, scorers, concurrency, signal, onRow, sink) {
+  const pending = [];
+  for (let i = 0; i < entries.length; i += 1) {
+    const entry = entries[i];
+    if (entry === void 0) continue;
+    if (sink.isResumed(entry, i)) continue;
+    pending.push({ entry, index: i });
+  }
   const batchOpts = {
     ...agentOptions,
     concurrency,
     ...signal !== void 0 ? { signal } : {}
   };
-  const batchResults = await getAgentFacade().batch(prompts, batchOpts);
+  const batchResults = await getAgentFacade().batch(
+    pending.map((p) => p.entry.input),
+    batchOpts
+  );
   const rows = [];
   for (let i = 0; i < batchResults.length; i += 1) {
-    const entry = entries[i];
+    const slot = pending[i];
     const br = batchResults[i];
-    if (entry === void 0 || br === void 0) continue;
-    const scoreEntries = await scoreBatchOutput(br, entry.expected, scorers);
-    const row = rowFromBatchResult(entry, br, scoreEntries, i);
+    if (slot === void 0 || br === void 0) continue;
+    const scoreEntries = await scoreBatchOutput(br, slot.entry.expected, scorers);
+    const row = sink.finalize(rowFromBatchResult(slot.entry, br, scoreEntries, slot.index));
     rows.push(row);
-    onRow(row, i);
+    onRow(row, slot.index);
   }
   return rows;
 }
@@ -15874,6 +15994,7 @@ async function runEval(options, runOpts) {
     const onRow = (row, i) => {
       safeHook(() => hooks?.afterRow?.(row, i));
     };
+    const sink = makeRowSink(runOpts?.persist, runOpts?.classify);
     let rows;
     if (isAgentInstance(options.agent) || typeof options.agent === "function") {
       rows = await runRowsManually(
@@ -15882,11 +16003,12 @@ async function runEval(options, runOpts) {
         scorers,
         concurrency,
         signal,
-        onRow
+        onRow,
+        sink
       );
     } else {
       const batchOpts = makeAgentForBatch(options.agent, indexed);
-      rows = await runRowsViaBatch(indexed, batchOpts, scorers, concurrency, signal, onRow);
+      rows = await runRowsViaBatch(indexed, batchOpts, scorers, concurrency, signal, onRow, sink);
     }
     const aggregate = computeAggregate(rows);
     const endedAt = Date.now();
@@ -15912,6 +16034,25 @@ async function runEval(options, runOpts) {
   }
 }
 
+// src/sandbox/shell-escape.ts
+function shellEscapePosix(arg) {
+  return `'${arg.replace(/'/g, "'\\''")}'`;
+}
+
+// src/internal/eval/code-runner.ts
+var ARTIFACT_PATCH = ".theo-artifact.patch";
+async function captureArtifact(sandbox, repoDir) {
+  const dir = shellEscapePosix(repoDir);
+  const diffRes = await sandbox.execute(`git -C ${dir} diff`);
+  const diff = diffRes.stdout;
+  if (diff.length === 0) return { diff: "", applies: false };
+  await sandbox.uploadFile(`${repoDir}/${ARTIFACT_PATCH}`, diff);
+  const check = await sandbox.execute(
+    `git -C ${dir} apply --check --reverse ${shellEscapePosix(ARTIFACT_PATCH)}`
+  );
+  return { diff, applies: check.exitCode === 0 };
+}
+
 // src/internal/scorers/llm-judge.ts
 init_agent_factory_registry();
 function buildPrompt(subject, criteria, rubric, expected) {
@@ -16054,6 +16195,38 @@ var Scorers = {
       }
     };
   },
+  /**
+   * Verify-gate scorer (M6-2): runs the project's tests in the provisioned
+   * repo via `SandboxBackend.execute` and scores `1` iff the command exits `0`,
+   * else `0` with the exit code + truncated stderr in `reason`. Grades the
+   * artifact captured by `captureArtifact` (D2 — rides `execute`, never a
+   * direct `child_process`).
+   *
+   * SECURITY: `command` is REQUIRED and the caller's builder owns shell-safety
+   * of the (potentially untrusted, dataset-derived) test identifiers. There is
+   * NO default that runs bare test names — that would interpolate untrusted
+   * `failToPass`/`passToPass` straight into a shell. `repoDir` is shell-escaped
+   * by the SDK; the test list is the builder's responsibility to render safely.
+   *
+   * PORTABILITY: the command is wrapped as `cd <repoDir> && <cmd>`, which
+   * assumes a shell-backed `SandboxBackend` (LocalSandbox/Docker). A backend
+   * that rejects shell metacharacters in `execute` is unsupported for this scorer.
+   */
+  verifyGate(opts) {
+    const { sandbox, repoDir, failToPass, passToPass, command } = opts;
+    return {
+      name: "verify-gate",
+      score: async () => {
+        const cmd = command([...failToPass, ...passToPass]).trim();
+        if (cmd.length === 0) {
+          return { score: 0, reason: "verify_gate_empty_command" };
+        }
+        const r = await sandbox.execute(`cd ${shellEscapePosix(repoDir)} && ${cmd}`);
+        if (r.exitCode === 0) return { score: 1 };
+        return { score: 0, reason: `exit=${r.exitCode} ${r.stderr.slice(0, 200)}`.trim() };
+      }
+    };
+  },
   jsonShape(schema, opts = {}) {
     return {
       name: "json-shape",
@@ -16126,6 +16299,6 @@ var Eval = class _Eval {
   }
 };
 
-export { Eval, EvalAlreadyRunningError, Scorers };
+export { Eval, EvalAlreadyRunningError, JsonlParseError, Scorers, captureArtifact, loadJsonl };
 //# sourceMappingURL=eval.js.map
 //# sourceMappingURL=eval.js.map