@theokit/sdk 0.0.0-compat-580-20260905204608

package/dist/eval.js

@@ -0,0 +1,1042 @@
+import { shellEscapePosix, LocalSandbox } from './chunk-O6USDERH.js';
+import { openRouterMemoryEmbeddingProviderAdapter } from './chunk-OYD3U3LY.js';
+import './chunk-K2VMFZQ5.js';
+import './chunk-AWINL3ZC.js';
+import { getTracer } from './chunk-K2BQQ445.js';
+import './chunk-7SZAV6QG.js';
+import './chunk-AW6F6HZR.js';
+import './chunk-2SFBB54R.js';
+import './chunk-55GB6JYQ.js';
+import './chunk-H73MEMQB.js';
+import './chunk-OR6XIWLB.js';
+import './chunk-SADXXGWU.js';
+import './chunk-AAEZSPTC.js';
+import './chunk-K3NH62DG.js';
+import './chunk-UBR5PWD7.js';
+import './chunk-2XC6KDJM.js';
+import './chunk-XB4GI5RZ.js';
+import './chunk-24MGQDK3.js';
+import './chunk-2WXHA33T.js';
+import './chunk-HKCKOAIO.js';
+import './chunk-N6OOOYFZ.js';
+import './chunk-KVNWIAO4.js';
+import './chunk-SVQIW5RJ.js';
+import './chunk-FD2UT76F.js';
+import './chunk-7I5Z3UBJ.js';
+import './chunk-FJWQP6EE.js';
+import './chunk-NAF4L3T6.js';
+import './chunk-CFE6QF2Q.js';
+import { getAgentFacade } from './chunk-6X6ID4MO.js';
+import './chunk-LX7SEXOQ.js';
+import './chunk-2XRAWOZZ.js';
+import './chunk-SAGRF4IB.js';
+import './chunk-EH6XD3FY.js';
+import './chunk-UKMBRMGT.js';
+import './chunk-4MMWFSCD.js';
+import './chunk-G7BLGBRN.js';
+import './chunk-VE6DFSKU.js';
+import './chunk-JOFVLOFY.js';
+import './chunk-WE22OXQA.js';
+import './chunk-6PWOWXMC.js';
+import { readJsonlIds, appendJsonl } from './chunk-TA3K7SBK.js';
+export { JsonlParseError, loadJsonl } from './chunk-TA3K7SBK.js';
+import './chunk-Q5EWJPRY.js';
+import './chunk-EIQFAOFD.js';
+import './chunk-3JHIFQ4I.js';
+import './chunk-OQRGVTQF.js';
+import './chunk-WMWEI3NS.js';
+import './chunk-JNAA4G4H.js';
+import './chunk-6M2OIS4Y.js';
+import './chunk-R7WIIPUR.js';
+import './chunk-XYFGKVZ7.js';
+import './chunk-VWUXIVEI.js';
+import './chunk-AGACYRJ2.js';
+import './chunk-JMDDYA22.js';
+import './chunk-V22DZIXO.js';
+import './chunk-NJWYQWDL.js';
+import { TheokitAgentError } from './chunk-ALUN2B4W.js';
+import { diag } from './chunk-CZJ6Q7CW.js';
+import { z } from 'zod';
+import { randomUUID } from 'crypto';
+
+// src/internal/eval/aggregate.ts
+function percentile(sorted, p) {
+  if (sorted.length === 0) return 0;
+  if (sorted.length === 1) return sorted[0] ?? 0;
+  const idx = Math.max(0, Math.min(sorted.length - 1, Math.ceil(p / 100 * sorted.length) - 1));
+  return sorted[idx] ?? 0;
+}
+function median(sorted) {
+  if (sorted.length === 0) return 0;
+  const mid = Math.floor(sorted.length / 2);
+  if (sorted.length % 2 === 0) {
+    return ((sorted[mid - 1] ?? 0) + (sorted[mid] ?? 0)) / 2;
+  }
+  return sorted[mid] ?? 0;
+}
+function computeAggregate(rows) {
+  if (rows.length === 0) {
+    return {
+      meanScore: 0,
+      medianScore: 0,
+      passRatio: 0,
+      perScorer: {},
+      totalRows: 0,
+      errorRows: 0,
+      durationMsP50: 0,
+      durationMsP95: 0,
+      tokensInTotal: 0,
+      tokensOutTotal: 0
+    };
+  }
+  const meanScores = rows.map((r) => r.meanScore);
+  const sortedMeans = [...meanScores].sort((a, b) => a - b);
+  const meanScore = meanScores.reduce((acc, s) => acc + s, 0) / rows.length;
+  const medianScore = median(sortedMeans);
+  const passRatio = rows.filter((r) => r.meanScore >= 0.5).length / rows.length;
+  const errorRows = rows.filter((r) => r.error !== void 0).length;
+  const perScorerBuckets = /* @__PURE__ */ new Map();
+  for (const row of rows) {
+    for (const s of row.scores) {
+      const bucket = perScorerBuckets.get(s.name) ?? [];
+      bucket.push(s.score);
+      perScorerBuckets.set(s.name, bucket);
+    }
+  }
+  const perScorer = {};
+  for (const [name, scores] of perScorerBuckets) {
+    const sorted = [...scores].sort((a, b) => a - b);
+    perScorer[name] = {
+      mean: scores.reduce((acc, s) => acc + s, 0) / scores.length,
+      median: median(sorted),
+      min: sorted[0] ?? 0,
+      max: sorted[sorted.length - 1] ?? 0
+    };
+  }
+  const durations = [...rows.map((r) => r.durationMs)].sort((a, b) => a - b);
+  const durationMsP50 = percentile(durations, 50);
+  const durationMsP95 = percentile(durations, 95);
+  const tokensInTotal = rows.reduce((acc, r) => acc + (r.tokensIn ?? 0), 0);
+  const tokensOutTotal = rows.reduce((acc, r) => acc + (r.tokensOut ?? 0), 0);
+  return {
+    meanScore,
+    medianScore,
+    passRatio,
+    perScorer,
+    totalRows: rows.length,
+    errorRows,
+    durationMsP50,
+    durationMsP95,
+    tokensInTotal,
+    tokensOutTotal
+  };
+}
+function clampScore(raw) {
+  let score = raw.score;
+  let reason = raw.reason;
+  if (!Number.isFinite(score)) {
+    score = 0;
+    reason = reason ?? "score_not_finite";
+  } else if (score < 0) {
+    score = 0;
+    reason = reason ?? "score_below_zero";
+  } else if (score > 1) {
+    score = 1;
+    reason = reason ?? "score_above_one";
+  }
+  return reason === void 0 ? { score } : { score, reason };
+}
+
+// src/internal/eval/dataset-iter.ts
+async function* normalizeDataset(d) {
+  if (Array.isArray(d)) {
+    for (const entry of d) yield entry;
+    return;
+  }
+  if (typeof d === "function") {
+    const iterable = d();
+    for await (const entry of iterable) {
+      yield entry;
+    }
+    return;
+  }
+  throw new Error(
+    `Invalid dataset shape: expected Array<DatasetEntry> OR (() => Iterable<DatasetEntry> | AsyncIterable<DatasetEntry>); got ${typeof d}`
+  );
+}
+async function materializeDataset(d) {
+  const out = [];
+  for await (const entry of normalizeDataset(d)) out.push(entry);
+  return out;
+}
+
+// src/internal/eval/single-flight.ts
+var RUNNING = /* @__PURE__ */ new Set();
+var EvalAlreadyRunningError = class extends TheokitAgentError {
+  name = "EvalAlreadyRunningError";
+  evalName;
+  constructor(evalName) {
+    super(
+      `Eval "${evalName}" is already running in this process. Use a unique name per concurrent run (D213).`,
+      { code: "eval_already_running", isRetryable: false }
+    );
+    this.evalName = evalName;
+  }
+};
+function acquireSingleFlight(name) {
+  if (RUNNING.has(name)) {
+    throw new EvalAlreadyRunningError(name);
+  }
+  RUNNING.add(name);
+}
+function releaseSingleFlight(name) {
+  RUNNING.delete(name);
+}
+
+// src/internal/eval/telemetry.ts
+var TRACER_NAME = "theokit-sdk-eval";
+var NOOP_HANDLE = {
+  setAttribute: () => void 0,
+  end: () => void 0
+};
+function safe(fn, fallback) {
+  try {
+    return fn();
+  } catch {
+    return fallback;
+  }
+}
+function startEvalRunSpan(attrs) {
+  const tracer = getTracer(TRACER_NAME);
+  if (tracer === void 0) return NOOP_HANDLE;
+  const span = safe(
+    () => tracer.startSpan("eval.run", {
+      attributes: {
+        "eval.name": attrs.name,
+        "eval.id": attrs.id,
+        "eval.rows.total": attrs.rows,
+        "eval.concurrency": attrs.concurrency
+      }
+    }),
+    void 0
+  );
+  if (span === void 0) return NOOP_HANDLE;
+  return {
+    setAttribute: (key, value) => safe(() => span.setAttribute(key, value), void 0),
+    end: () => safe(() => span.end(), void 0)
+  };
+}
+
+// src/internal/eval/trials.ts
+var TRIAL_INDEX_KEY = "__evalRowIndex";
+var TRIAL_NUM_KEY = "__evalTrial";
+function expandForTrials(entries, trials) {
+  const out = [];
+  for (let i = 0; i < entries.length; i += 1) {
+    const entry = entries[i];
+    if (entry === void 0) continue;
+    for (let t = 0; t < trials; t += 1) {
+      out.push({
+        ...entry,
+        metadata: { ...entry.metadata ?? {}, [TRIAL_INDEX_KEY]: i, [TRIAL_NUM_KEY]: t }
+      });
+    }
+  }
+  return out;
+}
+function stripReserved(metadata) {
+  if (metadata === void 0) return void 0;
+  const { [TRIAL_INDEX_KEY]: _idx, [TRIAL_NUM_KEY]: _trial, ...rest } = metadata;
+  return Object.keys(rest).length > 0 ? rest : void 0;
+}
+function sumDefined(values) {
+  const present = values.filter((v) => typeof v === "number");
+  return present.length > 0 ? present.reduce((acc, v) => acc + v, 0) : void 0;
+}
+function meanScoresByScorer(group, trials) {
+  const names = [];
+  const sums = /* @__PURE__ */ new Map();
+  for (const row of group) {
+    for (const s of row.scores) {
+      if (!sums.has(s.name)) {
+        names.push(s.name);
+        sums.set(s.name, 0);
+      }
+      sums.set(s.name, (sums.get(s.name) ?? 0) + s.score);
+    }
+  }
+  return names.map((name) => ({
+    name,
+    score: (sums.get(name) ?? 0) / trials,
+    reason: `mean of ${trials} trials`
+  }));
+}
+function collapseGroup(index, group, trials) {
+  const first = group[0];
+  const scores = meanScoresByScorer(group, trials);
+  const allErrored = group.every((r) => r.error !== void 0);
+  const meanScore = allErrored || scores.length === 0 ? 0 : scores.reduce((acc, s) => acc + s.score, 0) / scores.length;
+  const firstOk = group.find((r) => r.error === void 0);
+  const errorMsg = allErrored ? group.find((r) => r.error !== void 0)?.error : void 0;
+  const metadata = stripReserved(first.metadata);
+  const tokensIn = sumDefined(group.map((r) => r.tokensIn));
+  const tokensOut = sumDefined(group.map((r) => r.tokensOut));
+  return {
+    index,
+    input: first.input,
+    output: firstOk?.output ?? first.output,
+    ...first.expected !== void 0 ? { expected: first.expected } : {},
+    scores,
+    meanScore,
+    durationMs: group.reduce((acc, r) => acc + r.durationMs, 0),
+    ...tokensIn !== void 0 ? { tokensIn } : {},
+    ...tokensOut !== void 0 ? { tokensOut } : {},
+    ...errorMsg !== void 0 ? { error: errorMsg } : {},
+    ...metadata !== void 0 ? { metadata } : {},
+    trialCount: trials
+  };
+}
+function collapseTrials(rows, trials) {
+  const groups = /* @__PURE__ */ new Map();
+  for (const row of rows) {
+    const meta = row.metadata;
+    const origIdx = typeof meta?.[TRIAL_INDEX_KEY] === "number" ? meta[TRIAL_INDEX_KEY] : row.index;
+    const arr = groups.get(origIdx) ?? [];
+    arr.push(row);
+    groups.set(origIdx, arr);
+  }
+  return [...groups.entries()].sort((a, b) => a[0] - b[0]).map(([origIdx, group]) => collapseGroup(origIdx, group, trials));
+}
+
+// src/internal/eval/runner.ts
+function safeHook(fn) {
+  try {
+    fn();
+  } catch (err) {
+    diag(`[eval] hook threw (ignored): ${err instanceof Error ? err.message : String(err)}
+`);
+  }
+}
+function isAgentInstance(value) {
+  return typeof value === "object" && value !== null && typeof value.send === "function" && typeof value.agentId === "string";
+}
+function normalizeScorers(input) {
+  return input.map((s, i) => {
+    if (typeof s === "function") {
+      return { name: `scorer-${i}`, score: s };
+    }
+    return { name: s.name, score: s.score };
+  });
+}
+function probeRow(entry, index) {
+  return {
+    index,
+    input: entry.input,
+    output: "",
+    ...entry.expected !== void 0 ? { expected: entry.expected } : {},
+    scores: [],
+    meanScore: 0,
+    durationMs: 0,
+    ...entry.metadata !== void 0 ? { metadata: entry.metadata } : {}
+  };
+}
+function computeDoneKeys(persist) {
+  if (persist.resume !== true) return /* @__PURE__ */ new Set();
+  return readJsonlIds(
+    persist.path,
+    (parsed) => parsed.error === void 0 ? persist.key(parsed) : void 0
+  );
+}
+function appendRowSafely(path, row) {
+  try {
+    appendJsonl(path, row);
+  } catch (err) {
+    diag(
+      `[eval] persist append failed (ignored): ${err instanceof Error ? err.message : String(err)}
+`
+    );
+  }
+}
+function makeRowSink(persist, classify) {
+  const doneKeys = persist !== void 0 ? computeDoneKeys(persist) : /* @__PURE__ */ new Set();
+  return {
+    isResumed(entry, index) {
+      if (persist === void 0 || doneKeys.size === 0) return false;
+      return doneKeys.has(persist.key(probeRow(entry, index)));
+    },
+    finalize(row) {
+      const outcome = classify?.(row);
+      const finalRow = outcome !== void 0 ? { ...row, outcome } : row;
+      if (persist !== void 0) appendRowSafely(persist.path, finalRow);
+      return finalRow;
+    }
+  };
+}
+async function applyScorer(scorer, output, expected) {
+  let raw;
+  try {
+    const result = scorer.score(output, expected);
+    raw = result instanceof Promise ? await result : result;
+  } catch (err) {
+    return {
+      name: scorer.name,
+      score: 0,
+      reason: `scorer_threw: ${err instanceof Error ? err.message : String(err)}`
+    };
+  }
+  const clamped = clampScore(raw);
+  return clamped.reason === void 0 ? { name: scorer.name, score: clamped.score } : { name: scorer.name, score: clamped.score, reason: clamped.reason };
+}
+function rowFromBatchResult(entry, batchResult, scorerResults, index) {
+  const meanScore = scorerResults.length === 0 ? 0 : scorerResults.reduce((acc, s) => acc + s.score, 0) / scorerResults.length;
+  const baseFields = {
+    index,
+    input: entry.input,
+    expected: entry.expected,
+    durationMs: batchResult.durationMs ?? 0,
+    scores: scorerResults,
+    meanScore
+  };
+  if (batchResult.ok === true) {
+    return {
+      ...baseFields,
+      output: batchResult.result.result ?? "",
+      ...entry.metadata !== void 0 ? { metadata: entry.metadata } : {}
+    };
+  }
+  return {
+    ...baseFields,
+    output: "",
+    error: batchResult.error.message,
+    ...entry.metadata !== void 0 ? { metadata: entry.metadata } : {}
+  };
+}
+function makeAgentForBatch(spec, _entries) {
+  if (isAgentInstance(spec) || typeof spec === "function") {
+    throw new Error("internal: makeAgentForBatch only handles AgentOptions");
+  }
+  return spec;
+}
+async function runManualSlot(ctx, spec, idx, rows) {
+  const { entries, scorers, sink, onRow } = ctx;
+  const entry = entries[idx];
+  if (entry === void 0) return;
+  if (sink.isResumed(entry, idx)) return;
+  const row = sink.finalize(await runOneEntry(spec, entry, idx, scorers));
+  rows[idx] = row;
+  onRow(row, idx);
+}
+async function runRowsManually(ctx, spec) {
+  const { entries, concurrency, signal } = ctx;
+  const rows = new Array(entries.length);
+  const state = { cursor: 0 };
+  const worker = async () => {
+    while (state.cursor < entries.length) {
+      if (signal?.aborted === true) return;
+      const idx = state.cursor;
+      state.cursor += 1;
+      await runManualSlot(ctx, spec, idx, rows);
+    }
+  };
+  const workers = Array.from({ length: Math.min(concurrency, entries.length) }, () => worker());
+  await Promise.all(workers);
+  return rows.filter((r) => r !== void 0);
+}
+async function executeAgent(spec, entry) {
+  try {
+    const agent = isAgentInstance(spec) ? spec : await spec(entry);
+    const run = await agent.send(entry.input);
+    const result = await run.wait();
+    if (result.status === "finished") return { output: result.result ?? "" };
+    return { output: "", errorMsg: result.error?.message ?? `run ${result.status}` };
+  } catch (err) {
+    return { output: "", errorMsg: err instanceof Error ? err.message : String(err) };
+  }
+}
+async function runOneEntry(spec, entry, idx, scorers) {
+  const t0 = Date.now();
+  const { output, errorMsg } = await executeAgent(spec, entry);
+  const durationMs = Date.now() - t0;
+  const scoreEntries = [];
+  if (errorMsg === void 0) {
+    for (const scorer of scorers) {
+      scoreEntries.push(await applyScorer(scorer, output, entry.expected));
+    }
+  }
+  const meanScore = scoreEntries.length === 0 ? 0 : scoreEntries.reduce((acc, s) => acc + s.score, 0) / scoreEntries.length;
+  return {
+    index: idx,
+    input: entry.input,
+    output,
+    ...entry.expected !== void 0 ? { expected: entry.expected } : {},
+    scores: scoreEntries,
+    meanScore: errorMsg === void 0 ? meanScore : 0,
+    durationMs,
+    ...errorMsg !== void 0 ? { error: errorMsg } : {},
+    ...entry.metadata !== void 0 ? { metadata: entry.metadata } : {}
+  };
+}
+async function scoreBatchOutput(br, expected, scorers) {
+  const scoreEntries = [];
+  if (br?.ok !== true) return scoreEntries;
+  const output = br.result?.result ?? "";
+  for (const scorer of scorers) {
+    scoreEntries.push(await applyScorer(scorer, output, expected));
+  }
+  return scoreEntries;
+}
+async function runRowsViaBatch(ctx, agentOptions) {
+  const { entries, scorers, concurrency, signal, onRow, sink } = ctx;
+  const pending = [];
+  for (let i = 0; i < entries.length; i += 1) {
+    const entry = entries[i];
+    if (entry === void 0) continue;
+    if (sink.isResumed(entry, i)) continue;
+    pending.push({ entry, index: i });
+  }
+  const batchOpts = {
+    ...agentOptions,
+    concurrency,
+    ...signal !== void 0 ? { signal } : {}
+  };
+  const batchResults = await getAgentFacade().batch(
+    pending.map((p) => p.entry.input),
+    batchOpts
+  );
+  const rows = [];
+  for (let i = 0; i < batchResults.length; i += 1) {
+    const slot = pending[i];
+    const br = batchResults[i];
+    if (slot === void 0 || br === void 0) continue;
+    const scoreEntries = await scoreBatchOutput(br, slot.entry.expected, scorers);
+    const row = sink.finalize(rowFromBatchResult(slot.entry, br, scoreEntries, slot.index));
+    rows.push(row);
+    onRow(row, slot.index);
+  }
+  return rows;
+}
+async function runEval(options, runOpts) {
+  acquireSingleFlight(options.name);
+  const id = randomUUID();
+  const startedAt = Date.now();
+  const entries = await materializeDataset(options.dataset);
+  const materialized = entries.map((e) => ({ ...e }));
+  const trials = options.trials ?? 1;
+  const indexed = trials > 1 ? expandForTrials(materialized, trials) : materialized;
+  const scorers = normalizeScorers(options.scorers);
+  const concurrency = options.concurrency ?? 4;
+  const signal = runOpts?.signal;
+  const span = startEvalRunSpan({
+    name: options.name,
+    id,
+    rows: entries.length,
+    concurrency
+  });
+  try {
+    const hooks = options.hooks;
+    safeHook(() => hooks?.beforeRun?.({ name: options.name, totalEstimate: entries.length }));
+    const onRow = (row, i) => {
+      safeHook(() => hooks?.afterRow?.(row, i));
+    };
+    const sink = makeRowSink(runOpts?.persist, runOpts?.classify);
+    const runCtx = { entries: indexed, scorers, concurrency, signal, onRow, sink };
+    const rows = await (isAgentInstance(options.agent) || typeof options.agent === "function" ? runRowsManually(
+      runCtx,
+      options.agent
+    ) : runRowsViaBatch(runCtx, makeAgentForBatch(options.agent, indexed)));
+    const finalRows = trials > 1 ? collapseTrials(rows, trials) : rows;
+    const aggregate = computeAggregate(finalRows);
+    const endedAt = Date.now();
+    const run = {
+      id,
+      name: options.name,
+      startedAt,
+      endedAt,
+      durationMs: endedAt - startedAt,
+      aggregate,
+      rows: finalRows,
+      ...options.metadata !== void 0 ? { metadata: options.metadata } : {}
+    };
+    safeHook(() => hooks?.afterRun?.(run));
+    span.setAttribute("eval.aggregate.meanScore", run.aggregate.meanScore);
+    span.setAttribute("eval.aggregate.errorRows", run.aggregate.errorRows);
+    span.setAttribute("eval.aggregate.durationMsP95", run.aggregate.durationMsP95);
+    span.setAttribute("eval.aggregate.tokensInTotal", run.aggregate.tokensInTotal);
+    return run;
+  } finally {
+    span.end();
+    releaseSingleFlight(options.name);
+  }
+}
+
+// src/internal/eval/assert.ts
+var EvalThresholdError = class extends TheokitAgentError {
+  name = "EvalThresholdError";
+  /** The eval's name (`EvalRun.name`). */
+  evalName;
+  /** Every unmet threshold, in check order. */
+  failures;
+  constructor(evalName, failures) {
+    const lines = failures.map(
+      (f) => `  - ${f.metric}: required ${f.threshold}, got ${Number.isNaN(f.actual) ? "n/a (scorer absent)" : f.actual}`
+    );
+    super(
+      `Eval "${evalName}" failed ${failures.length} threshold${failures.length === 1 ? "" : "s"}:
+${lines.join("\n")}`,
+      { code: "eval_threshold_failed", isRetryable: false }
+    );
+    this.evalName = evalName;
+    this.failures = failures;
+  }
+};
+function floor(metric, actual, threshold) {
+  if (threshold === void 0 || actual >= threshold) return void 0;
+  return { metric, threshold, actual };
+}
+function perScorerFailures(perScorer, thresholds) {
+  if (thresholds === void 0) return [];
+  const out = [];
+  for (const [name, min] of Object.entries(thresholds)) {
+    const stats = perScorer[name];
+    if (stats === void 0 || stats.mean < min) {
+      out.push({
+        metric: `perScorer.${name}`,
+        threshold: min,
+        actual: stats === void 0 ? Number.NaN : stats.mean
+      });
+    }
+  }
+  return out;
+}
+function assertEval(run, thresholds) {
+  const a = run.aggregate;
+  const errorRatio = a.totalRows > 0 ? a.errorRows / a.totalRows : 0;
+  const ceiling = thresholds.maxErrorRatio !== void 0 && errorRatio > thresholds.maxErrorRatio ? { metric: "errorRatio", threshold: thresholds.maxErrorRatio, actual: errorRatio } : void 0;
+  const failures = [
+    floor("meanScore", a.meanScore, thresholds.minMeanScore),
+    floor("passRatio", a.passRatio, thresholds.minPassRatio),
+    ceiling,
+    ...perScorerFailures(a.perScorer, thresholds.perScorer)
+  ].filter((f) => f !== void 0);
+  if (failures.length > 0) {
+    throw new EvalThresholdError(run.name, failures);
+  }
+}
+
+// src/internal/eval/code-runner.ts
+var ARTIFACT_PATCH = ".theo-artifact.patch";
+async function captureArtifact(sandbox, repoDir) {
+  const dir = shellEscapePosix(repoDir);
+  const diffRes = await sandbox.execute(`git -C ${dir} diff`);
+  const diff = diffRes.stdout;
+  if (diff.length === 0) return { diff: "", applies: false };
+  await sandbox.uploadFile(`${repoDir}/${ARTIFACT_PATCH}`, diff);
+  const check = await sandbox.execute(
+    `git -C ${dir} apply --check --reverse ${shellEscapePosix(ARTIFACT_PATCH)}`
+  );
+  return { diff, applies: check.exitCode === 0 };
+}
+
+// src/internal/scorers/levenshtein.ts
+var LEVENSHTEIN_MAX_LEN = 4e3;
+function nextRow(prev, rowIndex, ai, b) {
+  const curr = [rowIndex];
+  for (let j = 1; j <= b.length; j += 1) {
+    const cost = ai === b.charCodeAt(j - 1) ? 0 : 1;
+    curr[j] = Math.min((prev[j] ?? 0) + 1, (curr[j - 1] ?? 0) + 1, (prev[j - 1] ?? 0) + cost);
+  }
+  return curr;
+}
+function levenshteinDistance(a, b) {
+  const m = a.length;
+  const n = b.length;
+  if (m === 0) return n;
+  if (n === 0) return m;
+  let prev = [];
+  for (let j = 0; j <= n; j += 1) prev[j] = j;
+  for (let i = 1; i <= m; i += 1) prev = nextRow(prev, i, a.charCodeAt(i - 1), b);
+  return prev[n] ?? 0;
+}
+
+// src/internal/scorers/llm-judge.ts
+function buildPrompt(subject, criteria, rubric, expected) {
+  const scoreHint = rubric === "discrete" ? "between 0 and 1 (use 0 for fail, 1 for pass)" : "between 0.0 and 1.0";
+  const expectedBlock = expected !== void 0 ? `EXPECTED (reference): ${JSON.stringify(expected)}
+
+` : "";
+  return [
+    "You are evaluating an AI assistant's output.",
+    "",
+    `CRITERIA: ${criteria}`,
+    "",
+    `${expectedBlock}ACTUAL OUTPUT:`,
+    '"""',
+    subject,
+    '"""',
+    "",
+    `Reply with EXACTLY this JSON object on a single line:`,
+    `{"score": <number ${scoreHint}>, "reason": "<one short sentence>"}`,
+    "",
+    "Reply with the JSON object ONLY. No preface, no explanation outside the JSON."
+  ].join("\n");
+}
+var SCORE_REGEX = /\{\s*"score"\s*:\s*([0-9]*\.?[0-9]+)\s*,\s*"reason"\s*:\s*"([^"]*)"\s*\}/;
+async function callJudge(message, options) {
+  try {
+    const result = await getAgentFacade().prompt(message, {
+      apiKey: options.apiKey,
+      model: options.model,
+      local: { cwd: process.cwd(), sandboxOptions: { enabled: false } },
+      // #581 — a `shell` tool is ALWAYS registered on a local agent, including when `tools: []` is
+      // passed, so this judge held one it never asked for. `sandboxOptions: { enabled: false }` is
+      // not a mitigation for that and reads like one: it does not restrict the shell, it removes the
+      // sandbox around it.
+      //
+      // A judge is the worst place for it. Its entire job is to read content produced by the thing
+      // under evaluation, so anything able to influence that content was talking to a process that
+      // held an unsandboxed shell in `process.cwd()`. `types/agent.ts` § LocalOptions records the
+      // case that already happened: a working directory holding the benchmark's answer key, and two
+      // transcripts showing the model citing it.
+      //
+      // Withholding rather than sandboxing, because this judge needs no shell at all — removing it
+      // from the catalog beats confining it. The line above is left as it was: with no shell there
+      // is nothing for it to govern, and changing it would be a second, unrelated decision.
+      withheldBuiltinTools: ["shell"],
+      ...options.providers !== void 0 ? { providers: options.providers } : {}
+    });
+    if (result.status === "finished") return { ok: true, text: result.result ?? "" };
+    return { ok: false, reason: `judge_run_failed: ${result.error?.message ?? result.status}` };
+  } catch (err) {
+    return {
+      ok: false,
+      reason: `judge_threw: ${err instanceof Error ? err.message : String(err)}`
+    };
+  }
+}
+function parseScore(text, rubric) {
+  const match = SCORE_REGEX.exec(text);
+  if (match === null) return { score: 0, reason: "judge_parse_failed" };
+  const raw = Number(match[1]);
+  if (!Number.isFinite(raw)) return { score: 0, reason: "judge_score_not_finite" };
+  const clamped = rubric === "discrete" ? raw >= 0.5 ? 1 : 0 : Math.max(0, Math.min(1, raw));
+  return { score: clamped, reason: match[2] };
+}
+async function llmJudgeScore(options) {
+  const rubric = options.rubric ?? "continuous";
+  const message = buildPrompt(options.output, options.criteria, rubric, options.expected);
+  const judgement = await callJudge(message, options);
+  if (!judgement.ok) return { score: 0, reason: judgement.reason };
+  return parseScore(judgement.text, rubric);
+}
+
+// src/scorers.ts
+var JSON_SHAPE_MAX_BYTES = 1e6;
+function cosineSimilarity(a, b) {
+  const n = Math.min(a.length, b.length);
+  let dot = 0;
+  let na = 0;
+  let nb = 0;
+  for (let i = 0; i < n; i += 1) {
+    const av = a[i] ?? 0;
+    const bv = b[i] ?? 0;
+    dot += av * bv;
+    na += av * av;
+    nb += bv * bv;
+  }
+  if (na === 0 || nb === 0) return 0;
+  return dot / (Math.sqrt(na) * Math.sqrt(nb));
+}
+function normalizeStringInputs(output, expected, caseSensitive) {
+  if (typeof expected !== "string") return { score: 0, reason: "expected_not_string" };
+  if (expected.length === 0) return { score: 0, reason: "expected_empty" };
+  return {
+    o: caseSensitive ? output : output.toLowerCase(),
+    e: caseSensitive ? expected : expected.toLowerCase()
+  };
+}
+function scoreLevenshtein(output, expected, caseSensitive, threshold) {
+  const norm = normalizeStringInputs(output, expected, caseSensitive);
+  if (!("o" in norm)) return norm;
+  const { o, e } = norm;
+  if (o.length > LEVENSHTEIN_MAX_LEN || e.length > LEVENSHTEIN_MAX_LEN) {
+    return { score: 0, reason: "input_too_large" };
+  }
+  const sim = 1 - levenshteinDistance(o, e) / Math.max(o.length, e.length, 1);
+  if (threshold === void 0) return { score: sim };
+  return sim >= threshold ? { score: 1 } : { score: 0, reason: `sim=${sim.toFixed(3)}` };
+}
+function toFiniteNumber(value) {
+  if (typeof value === "number") return Number.isFinite(value) ? value : void 0;
+  if (value === void 0 || value === null) return void 0;
+  const n = Number(String(value).trim());
+  return Number.isFinite(n) ? n : void 0;
+}
+function scoreNumericDiff(output, expected, tolerance) {
+  const o = toFiniteNumber(output);
+  if (o === void 0) return { score: 0, reason: "output_not_numeric" };
+  const e = toFiniteNumber(expected);
+  if (e === void 0) return { score: 0, reason: "expected_not_numeric" };
+  if (tolerance !== void 0) {
+    return Math.abs(o - e) <= tolerance ? { score: 1 } : { score: 0, reason: `abs_diff=${Math.abs(o - e)}` };
+  }
+  const denom = Math.max(Math.abs(o), Math.abs(e));
+  if (denom === 0) return { score: 1 };
+  return { score: Math.max(0, 1 - Math.abs(o - e) / denom) };
+}
+function embedderCreateOptions(opts) {
+  return {
+    ...opts.model !== void 0 ? { model: opts.model } : {},
+    ...opts.apiKey !== void 0 ? { apiKey: opts.apiKey } : {},
+    ...opts.baseUrl !== void 0 ? { baseUrl: opts.baseUrl } : {}
+  };
+}
+function makeDefaultEmbedder(opts) {
+  let runtime;
+  return (texts) => {
+    if (runtime === void 0) {
+      runtime = openRouterMemoryEmbeddingProviderAdapter.create(embedderCreateOptions(opts));
+    }
+    return runtime.then((rt) => rt.embed(texts));
+  };
+}
+function makeStringScorer(name, caseSensitive, compare) {
+  return {
+    name,
+    score: (output, expected) => {
+      const norm = normalizeStringInputs(output, expected, caseSensitive);
+      return "o" in norm ? compare(norm.o, norm.e) : norm;
+    }
+  };
+}
+var Scorers = {
+  /**
+   * `output.trim() === expected.trim()`. Refuses empty `expected` (EC-1
+   * — two empties would otherwise pass silently).
+   */
+  exactMatch(opts = {}) {
+    return makeStringScorer(
+      "exact-match",
+      opts.caseSensitive ?? true,
+      (o, e) => o.trim() === e.trim() ? { score: 1 } : { score: 0, reason: "mismatch" }
+    );
+  },
+  /**
+   * `output.includes(expected)`. Refuses empty `expected` (EC-1 —
+   * `"".includes("")` is always `true`, would inflate pass ratio).
+   */
+  containsExpected(opts = {}) {
+    return makeStringScorer(
+      "contains-expected",
+      opts.caseSensitive ?? false,
+      (o, e) => o.includes(e) ? { score: 1 } : { score: 0, reason: "not_found" }
+    );
+  },
+  /**
+   * `pattern.test(output)`.
+   *
+   * **ReDoS caveat (EC-10):** the regex is applied to LLM output, which
+   * may be adversarial. Patterns with catastrophic backtracking (e.g.
+   * `/(a+)+$/`) can hang the eval. Test your pattern against worst-case
+   * inputs before using in production.
+   */
+  regex(pattern) {
+    return {
+      name: `regex(${pattern.source})`,
+      score: (output) => {
+        const ok = pattern.test(output);
+        return ok ? { score: 1 } : { score: 0, reason: "regex_no_match" };
+      }
+    };
+  },
+  /**
+   * SE41 — normalized Levenshtein similarity: `1 - editDistance / max(len)`.
+   * Deterministic (no LLM), so it always runs in CI. `threshold` binarizes.
+   *
+   * Refuses empty/non-string `expected` (EC-1 parity) and caps input at
+   * {@link LEVENSHTEIN_MAX_LEN} chars to bound the O(n*m) cost on adversarial output.
+   */
+  levenshtein(opts = {}) {
+    const caseSensitive = opts.caseSensitive ?? false;
+    const { threshold } = opts;
+    return {
+      name: threshold !== void 0 ? `levenshtein(>=${threshold})` : "levenshtein",
+      score: (output, expected) => scoreLevenshtein(output, expected, caseSensitive, threshold)
+    };
+  },
+  /**
+   * SE41 — numeric closeness. Parses `output` and `expected` as numbers and
+   * scores continuous relative closeness `1 - |o-e| / max(|o|,|e|)` (both 0 ⇒ 1),
+   * or a binary pass when `tolerance` is set. Deterministic (no LLM).
+   */
+  numericDiff(opts = {}) {
+    const { tolerance } = opts;
+    return {
+      name: "numeric-diff",
+      score: (output, expected) => scoreNumericDiff(output, expected, tolerance)
+    };
+  },
+  /**
+   * SE41 — semantic similarity via embeddings: cosine of `embed(output)` vs
+   * `embed(expected)`, clamped to `[0, 1]` (negatives → 0). `threshold` binarizes.
+   *
+   * By default routes through OpenRouter's embeddings endpoint
+   * (`OPENROUTER_API_KEY`); inject `embed` to use another provider or to test
+   * deterministically. Each scored row costs one embeddings call.
+   */
+  embeddingSimilarity(opts) {
+    const { threshold } = opts;
+    const embed = opts.embed ?? makeDefaultEmbedder(opts);
+    return {
+      name: threshold !== void 0 ? `embedding-similarity(>=${threshold})` : "embedding-similarity",
+      score: async (output, expected) => {
+        if (typeof expected !== "string") return { score: 0, reason: "expected_not_string" };
+        const [vo, ve] = await embed([output, expected]);
+        if (vo === void 0 || ve === void 0) return { score: 0, reason: "embed_failed" };
+        const score = Math.max(0, cosineSimilarity(vo, ve));
+        if (threshold === void 0) return { score };
+        return score >= threshold ? { score: 1 } : { score: 0, reason: `cos=${score.toFixed(3)}` };
+      }
+    };
+  },
+  /**
+   * LLM-as-judge scorer (ADR D205).
+   *
+   * **Cost note (EC-12):** every row with this scorer adds ~1 LLM call
+   * for the judge. For 1000 rows × gpt-4o-mini, expect ~2x the baseline
+   * token spend. The eval `aggregate.tokensInTotal` only reflects the
+   * EVAL agent's tokens, not the judge's — forecast accordingly.
+   *
+   * **Bias note (D205):** `apiKey` is intentionally separate so callers
+   * cannot accidentally judge their own output. Pass a different key
+   * (can target the same provider, but explicit by design).
+   *
+   * **EC-8 (parser robustness):** the regex accepts JSON inside markdown
+   * code fences and prose-wrapped JSON — common LLM output shapes.
+   */
+  llmJudge(opts) {
+    return {
+      name: "llm-judge",
+      score: (output, expected) => {
+        const callOpts = expected !== void 0 ? { ...opts, output, expected } : { ...opts, output };
+        return llmJudgeScore(callOpts);
+      }
+    };
+  },
+  /**
+   * Verify-gate scorer (M6-2): runs the project's tests in the provisioned
+   * repo via `SandboxBackend.execute` and scores `1` iff the command exits `0`,
+   * else `0` with the exit code + truncated stderr in `reason`. Grades the
+   * artifact captured by `captureArtifact` (D2 — rides `execute`, never a
+   * direct `child_process`).
+   *
+   * SECURITY: `command` is REQUIRED and the caller's builder owns shell-safety
+   * of the (potentially untrusted, dataset-derived) test identifiers. There is
+   * NO default that runs bare test names — that would interpolate untrusted
+   * `failToPass`/`passToPass` straight into a shell. `repoDir` is shell-escaped
+   * by the SDK; the test list is the builder's responsibility to render safely.
+   *
+   * PORTABILITY: the command is wrapped as `cd <repoDir> && <cmd>`, which
+   * assumes a shell-backed `SandboxBackend` (LocalSandbox/Docker). A backend
+   * that rejects shell metacharacters in `execute` is unsupported for this scorer.
+   */
+  verifyGate(opts) {
+    const { sandbox = new LocalSandbox(), repoDir, failToPass, passToPass, command } = opts;
+    return {
+      name: "verify-gate",
+      score: async () => {
+        const cmd = command([...failToPass, ...passToPass]).trim();
+        if (cmd.length === 0) {
+          return { score: 0, reason: "verify_gate_empty_command" };
+        }
+        const r = await sandbox.execute(`cd ${shellEscapePosix(repoDir)} && ${cmd}`);
+        if (r.exitCode === 0) return { score: 1 };
+        return { score: 0, reason: `exit=${r.exitCode} ${r.stderr.slice(0, 200)}`.trim() };
+      }
+    };
+  },
+  /**
+   * Parse `output` as JSON and validate against a Zod schema.
+   *
+   * - EC-2 fix: output is bounded by `JSON_SHAPE_MAX_BYTES` (1e6) BEFORE
+   *   `JSON.parse` to avoid OOM on runaway LLM responses. The bound is applied to
+   *   `output.length` — UTF-16 code units, not bytes, despite the constant's name.
+   * - `strict: true` includes Zod's first error message in `reason`.
+   */
+  jsonShape(schema, opts = {}) {
+    return {
+      name: "json-shape",
+      score: (output) => {
+        if (output.length > JSON_SHAPE_MAX_BYTES) {
+          return { score: 0, reason: "output_too_large" };
+        }
+        let parsed;
+        try {
+          parsed = JSON.parse(output);
+        } catch {
+          return { score: 0, reason: "invalid_json" };
+        }
+        const result = schema.safeParse(parsed);
+        if (result.success) return { score: 1 };
+        if (opts.strict === true) {
+          const issue = result.error.issues?.[0];
+          return { score: 0, reason: `schema_invalid: ${issue?.message ?? "?"}` };
+        }
+        return { score: 0, reason: "schema_invalid" };
+      }
+    };
+  }
+};
+
+// src/eval.ts
+var EvalOptionsSchema = z.object({
+  name: z.string().min(1, "Eval.name must be non-empty"),
+  // dataset / scorers / agent are validated lightly here; deep validation is
+  // structural and lives in the runner (where it's cheap to do once).
+  dataset: z.unknown().refine((v) => Array.isArray(v) || typeof v === "function", {
+    message: "dataset must be an array of DatasetEntry OR a factory returning Iterable/AsyncIterable"
+  }),
+  scorers: z.unknown().refine((v) => Array.isArray(v) && v.length >= 1, {
+    message: "scorers must be an array with at least one scorer"
+  }),
+  agent: z.unknown().refine((v) => v !== void 0 && v !== null, {
+    message: "agent must be an SDKAgent, AgentOptions, or factory function"
+  }),
+  // EC-3: concurrency MUST be a finite integer in [1, 64]. Bare `z.number()`
+  // would let Infinity / 0 / negatives slip through.
+  concurrency: z.number().int("concurrency must be an integer").min(1, "concurrency must be >= 1").max(64, "concurrency must be <= 64").optional(),
+  // SE41: trials MUST be a finite integer in [1, 100]. Guards against 0
+  // (no-op run) and unbounded fanout that would DoS the provider.
+  trials: z.number().int("trials must be an integer").min(1, "trials must be >= 1").max(100, "trials must be <= 100").optional(),
+  metadata: z.record(z.string(), z.unknown()).optional(),
+  hooks: z.object({
+    beforeRun: z.function().optional(),
+    afterRow: z.function().optional(),
+    afterRun: z.function().optional()
+  }).partial().optional()
+});
+var Eval = class _Eval {
+  constructor(options) {
+    this.options = options;
+  }
+  options;
+  /**
+   * Construct an Eval. Throws on bad options (empty name, concurrency
+   * out of range, empty scorers, etc).
+   */
+  static create(options) {
+    EvalOptionsSchema.parse(options);
+    return new _Eval(options);
+  }
+  /**
+   * Run the eval. Resolves with a populated {@link EvalRun}. Per D208,
+   * row errors are isolated; per D213, two runs with the same name
+   * throw `EvalAlreadyRunningError`.
+   */
+  run(runOpts) {
+    return runEval(this.options, runOpts);
+  }
+};
+
+export { Eval, EvalAlreadyRunningError, EvalThresholdError, Scorers, assertEval, captureArtifact };
+//# sourceMappingURL=eval.js.map
+//# sourceMappingURL=eval.js.map