@theokit/sdk 4.4.2 → 4.5.1

package/dist/eval.d.cts

@@ -31,6 +31,7 @@ export declare class Eval {
      */
     run(runOpts?: EvalRunOptions): Promise<EvalRun>;
 }
+export { assertEval, EvalThresholdError } from "./internal/eval/assert.js";
 export { captureArtifact } from "./internal/eval/code-runner.js";
 export { EvalAlreadyRunningError } from "./internal/eval/single-flight.js";
 export { JsonlParseError, loadJsonl } from "./internal/persistence/jsonl.js";

package/dist/eval.d.ts

@@ -31,6 +31,7 @@ export declare class Eval {
      */
     run(runOpts?: EvalRunOptions): Promise<EvalRun>;
 }
+export { assertEval, EvalThresholdError } from "./internal/eval/assert.js";
 export { captureArtifact } from "./internal/eval/code-runner.js";
 export { EvalAlreadyRunningError } from "./internal/eval/single-flight.js";
 export { JsonlParseError, loadJsonl } from "./internal/persistence/jsonl.js";

package/dist/eval.js

@@ -7253,11 +7253,11 @@ function truncateToBudget(sources, maxTokens) {
   if (total <= maxTokens) {
     return sources.map((src) => ({ name: src.name, tokens: src.tokens }));
   }
-  const floor = Math.min(MIN_SOURCE_TOKENS, Math.floor(maxTokens / sources.length));
-  const remaining = Math.max(0, maxTokens - floor * sources.length);
+  const floor2 = Math.min(MIN_SOURCE_TOKENS, Math.floor(maxTokens / sources.length));
+  const remaining = Math.max(0, maxTokens - floor2 * sources.length);
   return sources.map((src) => {
     const proportional = total === 0 ? 0 : Math.floor(src.tokens.length / total * remaining);
-    const allotted = Math.min(src.tokens.length, floor + proportional);
+    const allotted = Math.min(src.tokens.length, floor2 + proportional);
     return { name: src.name, tokens: src.tokens.slice(0, allotted) };
   });
 }
@@ -18245,6 +18245,87 @@ function startEvalRunSpan(attrs) {
   };
 }
 
+// src/internal/eval/trials.ts
+var TRIAL_INDEX_KEY = "__evalRowIndex";
+var TRIAL_NUM_KEY = "__evalTrial";
+function expandForTrials(entries, trials) {
+  const out = [];
+  for (let i = 0; i < entries.length; i += 1) {
+    const entry = entries[i];
+    if (entry === void 0) continue;
+    for (let t = 0; t < trials; t += 1) {
+      out.push({
+        ...entry,
+        metadata: { ...entry.metadata ?? {}, [TRIAL_INDEX_KEY]: i, [TRIAL_NUM_KEY]: t }
+      });
+    }
+  }
+  return out;
+}
+function stripReserved(metadata) {
+  if (metadata === void 0) return void 0;
+  const { [TRIAL_INDEX_KEY]: _idx, [TRIAL_NUM_KEY]: _trial, ...rest } = metadata;
+  return Object.keys(rest).length > 0 ? rest : void 0;
+}
+function sumDefined(values) {
+  const present = values.filter((v) => typeof v === "number");
+  return present.length > 0 ? present.reduce((acc, v) => acc + v, 0) : void 0;
+}
+function meanScoresByScorer(group, trials) {
+  const names = [];
+  const sums = /* @__PURE__ */ new Map();
+  for (const row of group) {
+    for (const s of row.scores) {
+      if (!sums.has(s.name)) {
+        names.push(s.name);
+        sums.set(s.name, 0);
+      }
+      sums.set(s.name, (sums.get(s.name) ?? 0) + s.score);
+    }
+  }
+  return names.map((name) => ({
+    name,
+    score: (sums.get(name) ?? 0) / trials,
+    reason: `mean of ${trials} trials`
+  }));
+}
+function collapseGroup(index, group, trials) {
+  const first = group[0];
+  const scores = meanScoresByScorer(group, trials);
+  const allErrored = group.every((r) => r.error !== void 0);
+  const meanScore = allErrored || scores.length === 0 ? 0 : scores.reduce((acc, s) => acc + s.score, 0) / scores.length;
+  const firstOk = group.find((r) => r.error === void 0);
+  const errorMsg = allErrored ? group.find((r) => r.error !== void 0)?.error : void 0;
+  const metadata = stripReserved(first.metadata);
+  const tokensIn = sumDefined(group.map((r) => r.tokensIn));
+  const tokensOut = sumDefined(group.map((r) => r.tokensOut));
+  return {
+    index,
+    input: first.input,
+    output: firstOk?.output ?? first.output,
+    ...first.expected !== void 0 ? { expected: first.expected } : {},
+    scores,
+    meanScore,
+    durationMs: group.reduce((acc, r) => acc + r.durationMs, 0),
+    ...tokensIn !== void 0 ? { tokensIn } : {},
+    ...tokensOut !== void 0 ? { tokensOut } : {},
+    ...errorMsg !== void 0 ? { error: errorMsg } : {},
+    ...metadata !== void 0 ? { metadata } : {},
+    trialCount: trials
+  };
+}
+function collapseTrials(rows, trials) {
+  const groups = /* @__PURE__ */ new Map();
+  for (const row of rows) {
+    const meta = row.metadata;
+    const origIdx = typeof meta?.[TRIAL_INDEX_KEY] === "number" ? meta[TRIAL_INDEX_KEY] : row.index;
+    const arr = groups.get(origIdx) ?? [];
+    arr.push(row);
+    groups.set(origIdx, arr);
+  }
+  return [...groups.entries()].sort((a, b) => a[0] - b[0]).map(([origIdx, group]) => collapseGroup(origIdx, group, trials));
+}
+
 // src/internal/eval/runner.ts
 function safeHook(fn) {
   try {
@@ -18453,7 +18534,9 @@ async function runEval(options, runOpts) {
   const id = randomUUID();
   const startedAt = Date.now();
   const entries = await materializeDataset(options.dataset);
-  const indexed = entries.map((e) => ({ ...e }));
+  const materialized = entries.map((e) => ({ ...e }));
+  const trials = options.trials ?? 1;
+  const indexed = trials > 1 ? expandForTrials(materialized, trials) : materialized;
   const scorers = normalizeScorers(options.scorers);
   const concurrency = options.concurrency ?? 4;
   const signal = runOpts?.signal;
@@ -18485,7 +18568,8 @@ async function runEval(options, runOpts) {
       const batchOpts = makeAgentForBatch(options.agent, indexed);
       rows = await runRowsViaBatch(indexed, batchOpts, scorers, concurrency, signal, onRow, sink);
     }
-    const aggregate = computeAggregate(rows);
+    const finalRows = trials > 1 ? collapseTrials(rows, trials) : rows;
+    const aggregate = computeAggregate(finalRows);
     const endedAt = Date.now();
     const run = {
       id,
@@ -18494,7 +18578,7 @@ async function runEval(options, runOpts) {
       endedAt,
       durationMs: endedAt - startedAt,
       aggregate,
-      rows,
+      rows: finalRows,
       ...options.metadata !== void 0 ? { metadata: options.metadata } : {}
     };
     safeHook(() => hooks?.afterRun?.(run));
@@ -18509,6 +18593,59 @@ async function runEval(options, runOpts) {
   }
 }
 
+// src/internal/eval/assert.ts
+var EvalThresholdError = class extends Error {
+  name = "EvalThresholdError";
+  /** The eval's name (`EvalRun.name`). */
+  evalName;
+  /** Every unmet threshold, in check order. */
+  failures;
+  constructor(evalName, failures) {
+    const lines = failures.map(
+      (f) => `  - ${f.metric}: required ${f.threshold}, got ${Number.isNaN(f.actual) ? "n/a (scorer absent)" : f.actual}`
+    );
+    super(
+      `Eval "${evalName}" failed ${failures.length} threshold${failures.length === 1 ? "" : "s"}:
+${lines.join("\n")}`
+    );
+    this.evalName = evalName;
+    this.failures = failures;
+  }
+};
+function floor(metric, actual, threshold) {
+  if (threshold === void 0 || actual >= threshold) return void 0;
+  return { metric, threshold, actual };
+}
+function perScorerFailures(perScorer, thresholds) {
+  if (thresholds === void 0) return [];
+  const out = [];
+  for (const [name, min] of Object.entries(thresholds)) {
+    const stats = perScorer[name];
+    if (stats === void 0 || stats.mean < min) {
+      out.push({
+        metric: `perScorer.${name}`,
+        threshold: min,
+        actual: stats === void 0 ? Number.NaN : stats.mean
+      });
+    }
+  }
+  return out;
+}
+function assertEval(run, thresholds) {
+  const a = run.aggregate;
+  const errorRatio = a.totalRows > 0 ? a.errorRows / a.totalRows : 0;
+  const ceiling = thresholds.maxErrorRatio !== void 0 && errorRatio > thresholds.maxErrorRatio ? { metric: "errorRatio", threshold: thresholds.maxErrorRatio, actual: errorRatio } : void 0;
+  const failures = [
+    floor("meanScore", a.meanScore, thresholds.minMeanScore),
+    floor("passRatio", a.passRatio, thresholds.minPassRatio),
+    ceiling,
+    ...perScorerFailures(a.perScorer, thresholds.perScorer)
+  ].filter((f) => f !== void 0);
+  if (failures.length > 0) {
+    throw new EvalThresholdError(run.name, failures);
+  }
+}
+
 // src/sandbox/shell-escape.ts
 function shellEscapePosix(arg) {
   return `'${arg.replace(/'/g, "'\\''")}'`;
@@ -18528,6 +18665,27 @@ async function captureArtifact(sandbox, repoDir) {
   return { diff, applies: check.exitCode === 0 };
 }
 
+// src/internal/scorers/levenshtein.ts
+var LEVENSHTEIN_MAX_LEN = 4e3;
+function nextRow(prev, rowIndex, ai, b) {
+  const curr = [rowIndex];
+  for (let j = 1; j <= b.length; j += 1) {
+    const cost = ai === b.charCodeAt(j - 1) ? 0 : 1;
+    curr[j] = Math.min((prev[j] ?? 0) + 1, (curr[j - 1] ?? 0) + 1, (prev[j - 1] ?? 0) + cost);
+  }
+  return curr;
+}
+function levenshteinDistance(a, b) {
+  const m = a.length;
+  const n = b.length;
+  if (m === 0) return n;
+  if (n === 0) return m;
+  let prev = [];
+  for (let j = 0; j <= n; j += 1) prev[j] = j;
+  for (let i = 1; i <= m; i += 1) prev = nextRow(prev, i, a.charCodeAt(i - 1), b);
+  return prev[n] ?? 0;
+}
+
 // src/internal/scorers/llm-judge.ts
 init_agent_factory_registry();
 function buildPrompt(subject, criteria, rubric, expected) {
@@ -18688,15 +18846,80 @@ var LocalSandbox = class extends SandboxBackend {
 
 // src/scorers.ts
 var JSON_SHAPE_MAX_BYTES = 1e6;
+function cosineSimilarity(a, b) {
+  const n = Math.min(a.length, b.length);
+  let dot = 0;
+  let na = 0;
+  let nb = 0;
+  for (let i = 0; i < n; i += 1) {
+    const av = a[i] ?? 0;
+    const bv = b[i] ?? 0;
+    dot += av * bv;
+    na += av * av;
+    nb += bv * bv;
+  }
+  if (na === 0 || nb === 0) return 0;
+  return dot / (Math.sqrt(na) * Math.sqrt(nb));
+}
+function normalizeStringInputs(output, expected, caseSensitive) {
+  if (typeof expected !== "string") return { score: 0, reason: "expected_not_string" };
+  if (expected.length === 0) return { score: 0, reason: "expected_empty" };
+  return {
+    o: caseSensitive ? output : output.toLowerCase(),
+    e: caseSensitive ? expected : expected.toLowerCase()
+  };
+}
+function scoreLevenshtein(output, expected, caseSensitive, threshold) {
+  const norm = normalizeStringInputs(output, expected, caseSensitive);
+  if (!("o" in norm)) return norm;
+  const { o, e } = norm;
+  if (o.length > LEVENSHTEIN_MAX_LEN || e.length > LEVENSHTEIN_MAX_LEN) {
+    return { score: 0, reason: "input_too_large" };
+  }
+  const sim = 1 - levenshteinDistance(o, e) / Math.max(o.length, e.length, 1);
+  if (threshold === void 0) return { score: sim };
+  return sim >= threshold ? { score: 1 } : { score: 0, reason: `sim=${sim.toFixed(3)}` };
+}
+function toFiniteNumber2(value) {
+  if (typeof value === "number") return Number.isFinite(value) ? value : void 0;
+  if (value === void 0 || value === null) return void 0;
+  const n = Number(String(value).trim());
+  return Number.isFinite(n) ? n : void 0;
+}
+function scoreNumericDiff(output, expected, tolerance) {
+  const o = toFiniteNumber2(output);
+  if (o === void 0) return { score: 0, reason: "output_not_numeric" };
+  const e = toFiniteNumber2(expected);
+  if (e === void 0) return { score: 0, reason: "expected_not_numeric" };
+  if (tolerance !== void 0) {
+    return Math.abs(o - e) <= tolerance ? { score: 1 } : { score: 0, reason: `abs_diff=${Math.abs(o - e)}` };
+  }
+  const denom = Math.max(Math.abs(o), Math.abs(e));
+  if (denom === 0) return { score: 1 };
+  return { score: Math.max(0, 1 - Math.abs(o - e) / denom) };
+}
+function embedderCreateOptions(opts) {
+  return {
+    ...opts.model !== void 0 ? { model: opts.model } : {},
+    ...opts.apiKey !== void 0 ? { apiKey: opts.apiKey } : {},
+    ...opts.baseUrl !== void 0 ? { baseUrl: opts.baseUrl } : {}
+  };
+}
+function makeDefaultEmbedder(opts) {
+  let runtime;
+  return (texts) => {
+    if (runtime === void 0) {
+      runtime = openRouterMemoryEmbeddingProviderAdapter.create(embedderCreateOptions(opts));
+    }
+    return runtime.then((rt) => rt.embed(texts));
+  };
+}
 function makeStringScorer(name, caseSensitive, compare) {
   return {
     name,
     score: (output, expected) => {
-      if (typeof expected !== "string") return { score: 0, reason: "expected_not_string" };
-      if (expected.length === 0) return { score: 0, reason: "expected_empty" };
-      const o = caseSensitive ? output : output.toLowerCase();
-      const e = caseSensitive ? expected : expected.toLowerCase();
-      return compare(o, e);
+      const norm = normalizeStringInputs(output, expected, caseSensitive);
+      return "o" in norm ? compare(norm.o, norm.e) : norm;
     }
   };
 }
@@ -18740,6 +18963,56 @@ var Scorers = {
       }
     };
   },
+  /**
+   * SE41 — normalized Levenshtein similarity: `1 - editDistance / max(len)`.
+   * Deterministic (no LLM), so it always runs in CI. `threshold` binarizes.
+   *
+   * Refuses empty/non-string `expected` (EC-1 parity) and caps input at
+   * {@link LEVENSHTEIN_MAX_LEN} chars to bound the O(n*m) cost on adversarial output.
+   */
+  levenshtein(opts = {}) {
+    const caseSensitive = opts.caseSensitive ?? false;
+    const { threshold } = opts;
+    return {
+      name: threshold !== void 0 ? `levenshtein(>=${threshold})` : "levenshtein",
+      score: (output, expected) => scoreLevenshtein(output, expected, caseSensitive, threshold)
+    };
+  },
+  /**
+   * SE41 — numeric closeness. Parses `output` and `expected` as numbers and
+   * scores continuous relative closeness `1 - |o-e| / max(|o|,|e|)` (both 0 ⇒ 1),
+   * or a binary pass when `tolerance` is set. Deterministic (no LLM).
+   */
+  numericDiff(opts = {}) {
+    const { tolerance } = opts;
+    return {
+      name: "numeric-diff",
+      score: (output, expected) => scoreNumericDiff(output, expected, tolerance)
+    };
+  },
+  /**
+   * SE41 — semantic similarity via embeddings: cosine of `embed(output)` vs
+   * `embed(expected)`, clamped to `[0, 1]` (negatives → 0). `threshold` binarizes.
+   *
+   * By default routes through OpenRouter's embeddings endpoint
+   * (`OPENROUTER_API_KEY`); inject `embed` to use another provider or to test
+   * deterministically. Each scored row costs one embeddings call.
+   */
+  embeddingSimilarity(opts) {
+    const { threshold } = opts;
+    const embed = opts.embed ?? makeDefaultEmbedder(opts);
+    return {
+      name: threshold !== void 0 ? `embedding-similarity(>=${threshold})` : "embedding-similarity",
+      score: async (output, expected) => {
+        if (typeof expected !== "string") return { score: 0, reason: "expected_not_string" };
+        const [vo, ve] = await embed([output, expected]);
+        if (vo === void 0 || ve === void 0) return { score: 0, reason: "embed_failed" };
+        const score = Math.max(0, cosineSimilarity(vo, ve));
+        if (threshold === void 0) return { score };
+        return score >= threshold ? { score: 1 } : { score: 0, reason: `cos=${score.toFixed(3)}` };
+      }
+    };
+  },
   /**
    * Parse `output` as JSON and validate against a Zod schema.
    *
@@ -18845,6 +19118,9 @@ var EvalOptionsSchema = z.object({
   // EC-3: concurrency MUST be a finite integer in [1, 64]. Bare `z.number()`
   // would let Infinity / 0 / negatives slip through.
   concurrency: z.number().int("concurrency must be an integer").min(1, "concurrency must be >= 1").max(64, "concurrency must be <= 64").optional(),
+  // SE41: trials MUST be a finite integer in [1, 100]. Guards against 0
+  // (no-op run) and unbounded fanout that would DoS the provider.
+  trials: z.number().int("trials must be an integer").min(1, "trials must be >= 1").max(100, "trials must be <= 100").optional(),
   metadata: z.record(z.string(), z.unknown()).optional(),
   hooks: z.object({
     beforeRun: z.function().optional(),
@@ -18875,6 +19151,6 @@ var Eval = class _Eval {
   }
 };
 
-export { Eval, EvalAlreadyRunningError, JsonlParseError, Scorers, captureArtifact, loadJsonl };
+export { Eval, EvalAlreadyRunningError, EvalThresholdError, JsonlParseError, Scorers, assertEval, captureArtifact, loadJsonl };
 //# sourceMappingURL=eval.js.map
 //# sourceMappingURL=eval.js.map