blackriver-gateway 3.8.46

package/open-sse/services/compression/entrypointWrap.ts

@@ -0,0 +1,43 @@
+import type { CompressionConfig, CompressionResult } from "./types.ts";
+import type { CachingDetectionContext } from "./cachingAware.ts";
+import type { RiskGateConfig } from "./riskGate/riskGate.ts";
+import { resolveRiskGate, withRiskGate } from "./riskGate/strategyWrap.ts";
+import {
+  resolveQuantumLock,
+  quantumCachingContext,
+  withQuantumLock,
+  withQuantumLockAsync,
+} from "./quantumLock/index.ts";
+
+export interface CompressionEntrypointOptions {
+  config?: CompressionConfig;
+  riskGate?: RiskGateConfig;
+  cachingContext?: CachingDetectionContext;
+}
+
+export function withCompressionEntrypointGuards<T extends CompressionEntrypointOptions>(
+  body: Record<string, unknown>,
+  options: T | undefined,
+  run: (body: Record<string, unknown>) => CompressionResult
+): CompressionResult {
+  return withQuantumLock(
+    body,
+    resolveQuantumLock(options),
+    quantumCachingContext(body, options),
+    (quantumBody) =>
+      withRiskGate(quantumBody, resolveRiskGate(options), (riskBody) => run(riskBody))
+  );
+}
+
+export function withCompressionEntrypointGuardsAsync<T extends CompressionEntrypointOptions>(
+  body: Record<string, unknown>,
+  options: T | undefined,
+  run: (body: Record<string, unknown>) => Promise<CompressionResult>
+): Promise<CompressionResult> {
+  return withQuantumLockAsync(
+    body,
+    resolveQuantumLock(options),
+    quantumCachingContext(body, options),
+    run
+  );
+}

package/open-sse/services/compression/eval/aggregate.ts

@@ -0,0 +1,62 @@
+import type { ContentKind, EvalRecord, EvalReport, KindSummary, RunStamps } from "./types.ts";
+
+function pct(n: number, d: number): number {
+  return d > 0 ? Math.round((n / d) * 1000) / 10 : 0;
+}
+
+function goldDelta(scored: EvalRecord[]): number | null {
+  const gold = scored.filter((r) => r.goldFull !== null && r.goldCompressed !== null);
+  if (gold.length === 0) return null;
+  const fullCorrect = gold.filter((r) => r.goldFull === true).length;
+  const compCorrect = gold.filter((r) => r.goldCompressed === true).length;
+  return Math.round((pct(compCorrect, gold.length) - pct(fullCorrect, gold.length)) * 10) / 10;
+}
+
+function meanRatio(scored: EvalRecord[]): number {
+  if (scored.length === 0) return 1;
+  const sum = scored.reduce((s, r) => s + r.savings.ratio, 0);
+  return Math.round((sum / scored.length) * 10000) / 10000;
+}
+
+function summarizeKind(kind: ContentKind, scored: EvalRecord[]): KindSummary {
+  const same = scored.filter((r) => r.fidelity === "same").length;
+  return {
+    kind,
+    casesScored: scored.length,
+    fidelityPreservedPct: pct(same, scored.length),
+    goldAccuracyDeltaPct: goldDelta(scored),
+    meanRatio: meanRatio(scored),
+  };
+}
+
+/**
+ * Aggregate per content-kind + overall. Errored records (D1 §6) are counted but EXCLUDED
+ * from every rate so a failed model call can't masquerade as a fidelity loss. `run.partial`
+ * (cost-cap stop) and `run.totalCostUsd` flow through unchanged.
+ */
+export function aggregateRecords(
+  records: EvalRecord[],
+  stamps: RunStamps,
+  run: { partial: boolean; totalCostUsd: number }
+): EvalReport {
+  const scored = records.filter((r) => !r.errored);
+  const errored = records.length - scored.length;
+
+  const kinds = Array.from(new Set(scored.map((r) => r.kind)));
+  const perKind = kinds.map((k) => summarizeKind(k, scored.filter((r) => r.kind === k)));
+
+  const same = scored.filter((r) => r.fidelity === "same").length;
+  return {
+    stamps,
+    partial: run.partial,
+    totalCostUsd: run.totalCostUsd,
+    overall: {
+      casesScored: scored.length,
+      casesErrored: errored,
+      fidelityPreservedPct: pct(same, scored.length),
+      goldAccuracyDeltaPct: goldDelta(scored),
+      meanRatio: meanRatio(scored),
+    },
+    perKind,
+  };
+}

package/open-sse/services/compression/eval/corpus.ts

@@ -0,0 +1,45 @@
+import { createHash } from "node:crypto";
+import type { EvalCase, ContentKind } from "./types.ts";
+
+const KINDS: ContentKind[] = ["tool-output-json", "logs", "code", "prose", "multi-turn"];
+
+/** Best-effort PII markers; captured cases must be anonymized BEFORE ingestion. */
+const PII_PATTERNS: RegExp[] = [
+  /[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/, // email
+  /\b\d{3}-\d{2}-\d{4}\b/,                          // US SSN shape
+  /\b(?:\d[ -]*?){13,16}\b/,                        // card-number shape
+];
+
+function looksLikePII(text: string): boolean {
+  return PII_PATTERNS.some((re) => re.test(text));
+}
+
+/**
+ * Validate + sanity-check a corpus. Throws on a malformed case so a bad corpus
+ * fails loudly rather than silently skewing results. Captured cases (captured === true)
+ * are rejected on obvious PII markers (D-D5 anonymization bar); curated seed cases
+ * (captured falsy) are trusted as already-vetted.
+ */
+export function loadCorpus(rawCases: EvalCase[]): EvalCase[] {
+  return rawCases.map((c) => {
+    if (!c.id || !c.context || !c.question) {
+      throw new Error(`eval corpus: case "${c.id ?? "?"}" missing id/context/question`);
+    }
+    if (!KINDS.includes(c.kind)) {
+      throw new Error(`eval corpus: case "${c.id}" has unknown kind "${c.kind}"`);
+    }
+    if (c.captured === true && looksLikePII(c.context)) {
+      throw new Error(`eval corpus: captured case "${c.id}" contains an obvious PII marker — anonymize before ingestion`);
+    }
+    return c;
+  });
+}
+
+/** Stable, order-independent sha-256 hex over the canonical case payloads (for the report stamp). */
+export function hashCorpus(cases: EvalCase[]): string {
+  const canonical = cases
+    .map((c) => JSON.stringify([c.id, c.kind, c.context, c.question, c.gold ?? null]))
+    .sort()
+    .join("\n");
+  return createHash("sha256").update(canonical).digest("hex");
+}

package/open-sse/services/compression/eval/costMeter.ts

@@ -0,0 +1,31 @@
+export interface CostMeter {
+  add(usd: number): void;
+  /** True if adding `usd` would cross the cap (caller stops BEFORE the call that exceeds). */
+  wouldExceed(usd: number): boolean;
+  readonly spent: number;
+  readonly exceeded: boolean;
+}
+
+/**
+ * Pure cost accumulator driving the per-run cost cap (D1 §6: cap reached => stop,
+ * report partial, never silent truncation). `cap <= 0` (or NaN) means unbounded.
+ */
+export function createCostMeter(cap: number): CostMeter {
+  const bounded = typeof cap === "number" && cap > 0;
+  let spent = 0;
+  return {
+    add(usd: number) {
+      spent += Number.isFinite(usd) && usd > 0 ? usd : 0;
+    },
+    wouldExceed(usd: number) {
+      if (!bounded) return false;
+      return spent + (Number.isFinite(usd) && usd > 0 ? usd : 0) > cap;
+    },
+    get spent() {
+      return spent;
+    },
+    get exceeded() {
+      return bounded && spent > cap;
+    },
+  };
+}

package/open-sse/services/compression/eval/executorModelClient.ts

@@ -0,0 +1,44 @@
+import { getExecutor } from "../../../executors/index.ts";
+import type { ExecuteInput, ProviderCredentials } from "../../../executors/base.ts";
+import type { ChatTurn, ModelCallResult, ModelClient } from "./types.ts";
+
+/**
+ * Production ModelClient adapter (Hard Rule #18 — NOT unit-tested; validated on a real
+ * VPS/account). Wraps the server executor: builds a minimal non-stream chat body, calls
+ * `getExecutor(provider).execute(...)`, reads the response text + (best-effort) usage cost.
+ *
+ * The pure runner depends only on the `ModelClient` interface; this adapter is the single
+ * place that touches credentials, the executor, and Response parsing — so the eval stays
+ * faithful to production while the runner/scorers remain fully testable with a stub.
+ */
+export function createExecutorModelClient(
+  provider: string,
+  credentials: ProviderCredentials,
+  costPerKTokenOut?: number
+): ModelClient {
+  const executor = getExecutor(provider);
+  return {
+    async complete(model: string, messages: ChatTurn[]): Promise<ModelCallResult> {
+      const body = { model, messages, stream: false };
+      const input: ExecuteInput = {
+        model,
+        body,
+        stream: false,
+        credentials,
+      };
+      // BaseExecutor.execute resolves to { response, url, headers, transformedBody } — the
+      // upstream Response lives on `.response` (never a bare Response). Validated live on VPS.
+      const raw = (await executor.execute(input)) as { response: Response };
+      const response = raw.response;
+      const json = (await response.json()) as {
+        choices?: Array<{ message?: { content?: string } }>;
+        usage?: { completion_tokens?: number };
+      };
+      const text = json.choices?.[0]?.message?.content ?? "";
+      const outTokens = json.usage?.completion_tokens ?? 0;
+      const usdCost =
+        typeof costPerKTokenOut === "number" ? (outTokens / 1000) * costPerKTokenOut : undefined;
+      return usdCost === undefined ? { text } : { text, usdCost };
+    },
+  };
+}

package/open-sse/services/compression/eval/fidelityCheck.ts

@@ -0,0 +1,61 @@
+import { createCostMeter } from "./costMeter.ts";
+import { buildJudgePrompt, parseJudgeVerdict } from "./judge.ts";
+import type { JudgeVerdict, ModelClient } from "./types.ts";
+
+export interface FidelityItem {
+  id: string;
+  original: string;
+  compressed: string;
+}
+
+export interface FidelityVerdict {
+  id: string;
+  verdict: JudgeVerdict | null;
+  usdCost: number;
+  skippedCapped: boolean;
+}
+
+export interface FidelityBatchResult {
+  results: FidelityVerdict[];
+  totalUsd: number;
+  capped: boolean;
+}
+
+/**
+ * Run a USD-capped fidelity judge over a batch of (original, compressed) pairs.
+ *
+ * Iterates items in order; stops adding new LLM calls once the accumulated cost
+ * exceeds `costCapUsd`. Remaining items receive `skippedCapped: true` and
+ * `verdict: null` so callers can distinguish "not judged" from "unparseable".
+ *
+ * Hard Rule #18: NOT unit-tested for real LLM calls; VPS-validated via the
+ * /api/compression/compare/verify route.
+ */
+export async function judgeFidelityBatch(
+  client: ModelClient,
+  judgeModel: string,
+  items: FidelityItem[],
+  costCapUsd: number
+): Promise<FidelityBatchResult> {
+  const meter = createCostMeter(costCapUsd);
+  const results: FidelityVerdict[] = [];
+  let capped = false;
+
+  for (const item of items) {
+    if (capped || meter.exceeded) {
+      results.push({ id: item.id, verdict: null, usdCost: 0, skippedCapped: true });
+      continue;
+    }
+    try {
+      const prompt = buildJudgePrompt(item.original, item.compressed);
+      const { text, usdCost } = await client.complete(judgeModel, prompt);
+      meter.add(usdCost ?? 0);
+      results.push({ id: item.id, verdict: parseJudgeVerdict(text), usdCost: usdCost ?? 0, skippedCapped: false });
+    } catch {
+      results.push({ id: item.id, verdict: "unparseable", usdCost: 0, skippedCapped: false });
+    }
+    if (meter.exceeded) capped = true;
+  }
+
+  return { results, totalUsd: meter.spent, capped };
+}

package/open-sse/services/compression/eval/grader.ts

@@ -0,0 +1,27 @@
+import type { ChatTurn, GradeVerdict } from "./types.ts";
+
+/**
+ * Gold grader prompt (D-D2b). Grades a single answer against the gold answer — semantically,
+ * so a DIFFERENT-but-correct phrasing still grades CORRECT (catches "different but still right"),
+ * while a wrong fact grades INCORRECT. The grader must end with `VERDICT: CORRECT|INCORRECT`.
+ */
+export function buildGradePrompt(answer: string, gold: string): ChatTurn[] {
+  return [
+    {
+      role: "system",
+      content:
+        "You are a strict grader. Decide whether the candidate answer is CORRECT with respect " +
+        "to the gold answer — judge meaning, not wording (a correctly-phrased-differently answer " +
+        "is CORRECT). Reply with exactly one final line: `VERDICT: CORRECT` or `VERDICT: INCORRECT`.",
+    },
+    { role: "user", content: `Gold answer:\n${gold}\n\nCandidate answer:\n${answer}` },
+  ];
+}
+
+/** PURE grade parser. Conservative: anything not clearly CORRECT grades INCORRECT (no benefit of doubt). */
+export function parseGradeVerdict(raw: string): GradeVerdict {
+  const text = raw.toLowerCase();
+  if (/\bincorrect\b/.test(text)) return { correct: false, raw };
+  if (/\bcorrect\b/.test(text)) return { correct: true, raw };
+  return { correct: false, raw };
+}

package/open-sse/services/compression/eval/judge.ts

@@ -0,0 +1,68 @@
+import type { ChatTurn, JudgeVerdict, ModelClient } from "./types.ts";
+
+/**
+ * Fidelity judge prompt (D-D2a). Asks the judge to decide whether the compressed-context
+ * answer MATERIALLY differs from the full-context answer — wording differences that do not
+ * change the substance are "SAME". The judge must end with a single `VERDICT:` line.
+ */
+export function buildJudgePrompt(fullAnswer: string, compressedAnswer: string): ChatTurn[] {
+  return [
+    {
+      role: "system",
+      content:
+        "You are a strict evaluation judge. You are given two answers to the same question: " +
+        "answer A produced from the full context, and answer B produced from a compressed context. " +
+        "Decide whether B MATERIALLY differs from A (a difference that changes the substance, " +
+        "correctness, or key facts — NOT mere wording/format). Reply with exactly one final line: " +
+        "`VERDICT: SAME` or `VERDICT: MATERIALLY_DIFFERS`.",
+    },
+    {
+      role: "user",
+      content: `Answer A (full context):\n${fullAnswer}\n\nAnswer B (compressed context):\n${compressedAnswer}`,
+    },
+  ];
+}
+
+/** PURE verdict parser. Tolerant of case/format; unrecognized output => "unparseable" (never guessed). */
+export function parseJudgeVerdict(raw: string): JudgeVerdict {
+  const text = raw.toLowerCase();
+  const differs = /materially[_\s-]*differs|differs[_\s]+materially|\bdiffers\b/.test(text);
+  const same = /verdict:\s*same|\bsame\b/.test(text);
+  if (differs) return "materially-differs";
+  if (same) return "same";
+  return "unparseable";
+}
+
+/**
+ * Control pair for the self-test (D-D3 / ponytail discipline). The judge must rank the
+ * KNOWN-DEGRADED answer as MATERIALLY_DIFFERS and the KNOWN-GOOD answer as SAME, both relative
+ * to the same reference. A judge that mis-ranks either is untrusted and aborts the run.
+ */
+export const CONTROL_PAIR = {
+  reference: "The function returns 3 because the input is clamped to the upper bound.",
+  good: "It returns 3 since the value is clamped to the maximum allowed.",
+  degraded: "It returns 0 because the value is set to zero.",
+} as const;
+
+export interface SelfTestResult { passed: boolean; detail: string; }
+
+/**
+ * Run the control pair through the judge. PASS requires: degraded => materially-differs AND
+ * good => same. Any other outcome (including unparseable) FAILS, so the runner aborts before
+ * emitting untrusted scores.
+ */
+export async function runSelfTest(client: ModelClient, judgeModel: string): Promise<SelfTestResult> {
+  const goodVerdict = parseJudgeVerdict(
+    (await client.complete(judgeModel, buildJudgePrompt(CONTROL_PAIR.reference, CONTROL_PAIR.good))).text
+  );
+  const degradedVerdict = parseJudgeVerdict(
+    (await client.complete(judgeModel, buildJudgePrompt(CONTROL_PAIR.reference, CONTROL_PAIR.degraded))).text
+  );
+  if (degradedVerdict !== "materially-differs") {
+    return { passed: false, detail: `judge failed to flag the known-degraded control (got "${degradedVerdict}")` };
+  }
+  if (goodVerdict !== "same") {
+    return { passed: false, detail: `judge flagged the known-good control as "${goodVerdict}" (expected same)` };
+  }
+  return { passed: true, detail: "control pair ranked correctly" };
+}

package/open-sse/services/compression/eval/report.ts

@@ -0,0 +1,44 @@
+import type { EvalReport, KindSummary } from "./types.ts";
+
+function fmtDelta(d: number | null): string {
+  return d === null ? "n/a" : `${d > 0 ? "+" : ""}${d}%`;
+}
+
+function kindRow(k: KindSummary): string {
+  return `| ${k.kind} | ${k.casesScored} | ${k.fidelityPreservedPct}% | ${fmtDelta(k.goldAccuracyDeltaPct)} | ${k.meanRatio} |`;
+}
+
+/**
+ * Render the eval report as markdown with the pinned reproducibility stamps in the header
+ * (D1 §4.1) and a prominent PARTIAL banner when the cost cap stopped the run.
+ */
+export function formatReport(report: EvalReport): string {
+  const { stamps, overall } = report;
+  const lines: string[] = [];
+  lines.push("# Compression evaluation report (D1)");
+  lines.push("");
+  if (report.partial) {
+    lines.push("> ⚠️ **PARTIAL RUN** — the per-run cost cap was reached; results below cover only the cases scored before the stop.");
+    lines.push("");
+  }
+  lines.push(`- answer model: \`${stamps.answerModel}\``);
+  lines.push(`- judge model: \`${stamps.judgeModel}\``);
+  lines.push(`- corpus hash: \`${stamps.corpusHash}\``);
+  lines.push(`- sample size: ${stamps.sampleSize}`);
+  lines.push(`- total cost: $${Math.round(report.totalCostUsd * 1e4) / 1e4}`);
+  lines.push("");
+  lines.push("## Overall");
+  lines.push("");
+  lines.push(`- cases scored: ${overall.casesScored} (errored, excluded: ${overall.casesErrored})`);
+  lines.push(`- fidelity preserved: ${overall.fidelityPreservedPct}%`);
+  lines.push(`- gold-accuracy delta (compressed − full): ${fmtDelta(overall.goldAccuracyDeltaPct)}`);
+  lines.push(`- mean compression ratio: ${overall.meanRatio}`);
+  lines.push("");
+  lines.push("## Per content-kind");
+  lines.push("");
+  lines.push("| kind | scored | fidelity preserved | gold Δ | mean ratio |");
+  lines.push("|---|---|---|---|---|");
+  for (const k of report.perKind) lines.push(kindRow(k));
+  lines.push("");
+  return lines.join("\n");
+}

package/open-sse/services/compression/eval/runner.ts

@@ -0,0 +1,135 @@
+import { selectCompressionPlan, applyCompressionAsync } from "../strategySelector.ts";
+import { estimateCompressionTokens } from "../stats.ts";
+import type { CompressionConfig, CompressionMode } from "../types.ts";
+import { loadCorpus, hashCorpus } from "./corpus.ts";
+import { buildJudgePrompt, parseJudgeVerdict, runSelfTest } from "./judge.ts";
+import { buildGradePrompt, parseGradeVerdict } from "./grader.ts";
+import { computeSavings } from "./savings.ts";
+import { createCostMeter } from "./costMeter.ts";
+import { aggregateRecords } from "./aggregate.ts";
+import type { EvalCase, EvalRecord, EvalReport, ModelClient, RunStamps } from "./types.ts";
+
+export interface RunEvalOptions {
+  corpus: EvalCase[];
+  client: ModelClient;
+  config: CompressionConfig;
+  comboId: string | null;
+  combos: Record<string, CompressionConfig["stackedPipeline"]>;
+  answerModel: string;
+  judgeModel: string;
+  provider: string;
+  /** USD; <= 0 means unbounded. */
+  costCapUsd: number;
+  /** Score at most N cases (the seam for `--sample N`). */
+  sample?: number;
+  costPerKTokenIn?: number;
+}
+
+export interface RunEvalResult {
+  aborted: boolean;
+  abortReason?: string;
+  report: EvalReport | null;
+}
+
+/** Build a one-question chat body the pipeline + model both accept. */
+function buildBody(context: string, question: string): Record<string, unknown> {
+  return { messages: [{ role: "user", content: `${context}\n\nQuestion: ${question}` }] };
+}
+
+function answerText(body: Record<string, unknown>): string {
+  const messages = (body.messages ?? []) as Array<{ content?: unknown }>;
+  return messages.map((m) => (typeof m.content === "string" ? m.content : "")).join("\n");
+}
+
+/**
+ * Offline corpus eval (D1 §4.1). Self-test the judge FIRST (D-D3 abort), then for each case:
+ * model(full) baseline, compress via the REAL pipeline at the target config, model(compressed),
+ * judge fidelity, grade gold when present, compute mechanical savings. Errored cases are
+ * recorded + excluded; the cost cap stops the loop and flags partial (no silent truncation).
+ */
+export async function runEval(opts: RunEvalOptions): Promise<RunEvalResult> {
+  const corpus = loadCorpus(opts.corpus);
+
+  // D-D3 self-test gate — a broken judge aborts before any score is emitted.
+  const selfTest = await runSelfTest(opts.client, opts.judgeModel);
+  if (!selfTest.passed) {
+    return { aborted: true, abortReason: `judge self-test failed: ${selfTest.detail}`, report: null };
+  }
+
+  const limit = typeof opts.sample === "number" ? Math.max(0, opts.sample) : corpus.length;
+  const cases = corpus.slice(0, limit);
+  const meter = createCostMeter(opts.costCapUsd);
+  // Self-test calls are NOT metered against the run cap — the cap governs only the corpus loop.
+  const records: EvalRecord[] = [];
+  let partial = false;
+
+  for (const c of cases) {
+    // Stop BEFORE a case if we cannot afford its (~3) model calls; flag partial.
+    if (meter.exceeded) { partial = true; break; }
+
+    const fullBody = buildBody(c.context, c.question);
+    try {
+      const full = await opts.client.complete(opts.answerModel, [{ role: "user", content: answerText(fullBody) }]);
+      meter.add(full.usdCost ?? 0);
+
+      const estimatedTokens = estimateCompressionTokens(fullBody);
+      const plan = selectCompressionPlan(opts.config, opts.comboId, estimatedTokens, fullBody, undefined, opts.combos);
+      const compressedResult = await applyCompressionAsync(fullBody, plan.mode as CompressionMode, {
+        config: opts.config,
+        model: opts.answerModel,
+      });
+      const compressedBody = compressedResult.compressed ? (compressedResult.body as Record<string, unknown>) : fullBody;
+
+      const compressed = await opts.client.complete(opts.answerModel, [{ role: "user", content: answerText(compressedBody) }]);
+      meter.add(compressed.usdCost ?? 0);
+
+      const judge = await opts.client.complete(opts.judgeModel, buildJudgePrompt(full.text, compressed.text));
+      meter.add(judge.usdCost ?? 0);
+      const fidelity = parseJudgeVerdict(judge.text);
+
+      let goldFull: boolean | null = null;
+      let goldCompressed: boolean | null = null;
+      if (typeof c.gold === "string") {
+        const gf = await opts.client.complete(opts.judgeModel, buildGradePrompt(full.text, c.gold));
+        meter.add(gf.usdCost ?? 0);
+        const gc = await opts.client.complete(opts.judgeModel, buildGradePrompt(compressed.text, c.gold));
+        meter.add(gc.usdCost ?? 0);
+        goldFull = parseGradeVerdict(gf.text).correct;
+        goldCompressed = parseGradeVerdict(gc.text).correct;
+      }
+
+      records.push({
+        id: c.id,
+        kind: c.kind,
+        fidelity,
+        goldFull,
+        goldCompressed,
+        savings: computeSavings(fullBody, compressedBody, opts.costPerKTokenIn),
+        errored: false,
+      });
+    } catch (err) {
+      records.push({
+        id: c.id,
+        kind: c.kind,
+        fidelity: "unparseable",
+        goldFull: null,
+        goldCompressed: null,
+        savings: { tokensBefore: 0, tokensAfter: 0, ratio: 1 },
+        errored: true,
+        errorDetail: err instanceof Error ? err.message : String(err),
+      });
+    }
+
+    // After the case, if the cap is now crossed, stop the loop and flag partial.
+    if (meter.exceeded) { partial = true; break; }
+  }
+
+  const stamps: RunStamps = {
+    answerModel: opts.answerModel,
+    judgeModel: opts.judgeModel,
+    corpusHash: hashCorpus(corpus),
+    sampleSize: typeof opts.sample === "number" ? opts.sample : "all",
+  };
+  const report = aggregateRecords(records, stamps, { partial, totalCostUsd: meter.spent });
+  return { aborted: false, report };
+}

package/open-sse/services/compression/eval/savings.ts

@@ -0,0 +1,30 @@
+import { estimateCompressionTokens } from "../stats.ts";
+
+export interface SavingsResult {
+  tokensBefore: number;
+  tokensAfter: number;
+  ratio: number; // tokensAfter / tokensBefore (1 = no savings)
+  costDelta?: number; // USD saved on input tokens, when a price is supplied
+}
+
+/**
+ * Mechanical savings for a full-vs-compressed body pair. Reuses the production
+ * token estimator so the eval reports the same numbers the pipeline reports.
+ * `costPerKTokenIn` (USD per 1000 input tokens) is optional — when supplied, the
+ * positive cost saved on the input side is reported (the eval does not model output cost).
+ */
+export function computeSavings(
+  fullBody: Record<string, unknown>,
+  compressedBody: Record<string, unknown>,
+  costPerKTokenIn?: number
+): SavingsResult {
+  const tokensBefore = estimateCompressionTokens(fullBody);
+  const tokensAfter = estimateCompressionTokens(compressedBody);
+  const ratio = tokensBefore > 0 ? Math.round((tokensAfter / tokensBefore) * 10000) / 10000 : 1;
+  const result: SavingsResult = { tokensBefore, tokensAfter, ratio };
+  if (typeof costPerKTokenIn === "number" && costPerKTokenIn > 0) {
+    const saved = ((tokensBefore - tokensAfter) / 1000) * costPerKTokenIn;
+    result.costDelta = Math.round(saved * 1e6) / 1e6;
+  }
+  return result;
+}

package/open-sse/services/compression/eval/seedCorpus.ts

@@ -0,0 +1,60 @@
+import type { EvalCase } from "./types.ts";
+
+/**
+ * Curated seed corpus (D-D5). Small, diverse content kinds; pinned for reproducible runs.
+ * Anonymized captured cases append to this array with `captured: true` (loader vets PII).
+ * Keep every `context` synthetic / public so the corpus carries no real user data.
+ */
+export const SEED_CORPUS: EvalCase[] = [
+  {
+    id: "prose-1",
+    kind: "prose",
+    context:
+      "The deployment pipeline has three stages. First, the build stage compiles the " +
+      "TypeScript sources and bundles them. Second, the test stage runs the unit suite " +
+      "and the integration suite in parallel. Third, the deploy stage pushes the artifact " +
+      "to the staging environment and waits for a manual approval before promoting to prod.",
+    question: "How many stages does the deployment pipeline have, and what is the last stage?",
+    gold: "Three stages; the last is the deploy stage (push to staging, manual approval before prod).",
+  },
+  {
+    id: "code-1",
+    kind: "code",
+    context:
+      "export function clamp(n: number, lo: number, hi: number): number {\n" +
+      "  if (n < lo) return lo;\n  if (n > hi) return hi;\n  return n;\n}",
+    question: "What does clamp(5, 0, 3) return?",
+    gold: "3",
+  },
+  {
+    id: "tool-output-json-1",
+    kind: "tool-output-json",
+    context:
+      '{"status":"ok","results":[{"id":1,"name":"alpha","score":0.91},' +
+      '{"id":2,"name":"beta","score":0.42},{"id":3,"name":"gamma","score":0.77}]}',
+    question: "Which result has the highest score?",
+    gold: "alpha (score 0.91)",
+  },
+  {
+    id: "logs-1",
+    kind: "logs",
+    context:
+      "2026-06-22T10:00:01Z INFO  worker started pid=4821\n" +
+      "2026-06-22T10:00:02Z WARN  retrying upstream attempt=1\n" +
+      "2026-06-22T10:00:03Z ERROR upstream timeout after=30000ms\n" +
+      "2026-06-22T10:00:04Z INFO  fell back to secondary provider",
+    question: "What error occurred and what happened after it?",
+    gold: "An upstream timeout after 30000ms; the worker then fell back to the secondary provider.",
+  },
+  {
+    id: "multi-turn-1",
+    kind: "multi-turn",
+    context:
+      "User: I need to rename the column user_name to username in the users table.\n" +
+      "Assistant: You can run ALTER TABLE users RENAME COLUMN user_name TO username.\n" +
+      "User: Will that drop the data in the column?\n" +
+      "Assistant: No — RENAME COLUMN only changes the column name; the data is preserved.",
+    question: "Does renaming the column drop its data?",
+    gold: "No; RENAME COLUMN preserves the data.",
+  },
+];