@workos/quickstudy 0.1.1 → 0.2.0

package/README.md

@@ -41,7 +41,7 @@ modules), so it requires [Bun](https://bun.sh) ≥ 1.1 — there is no Node
 build. Add it to the repository that holds your evals and experiments:
 
 ```bash
-bun add quickstudy       # Bun >= 1.1 required; no Node build exists
+bun add @workos/quickstudy   # Bun >= 1.1 required; no Node build exists
 bunx quickstudy --version
 ```
 
@@ -261,7 +261,6 @@ bun run ui:e2e           # Playwright smoke: served routes + offline file:// loa
 - **[docs/getting-started.md](./docs/getting-started.md)** — install to first report.
 - **[docs/writing-an-eval.md](./docs/writing-an-eval.md)** — authoring evals and experiments.
 - **[docs/publishing-a-benchmark.md](./docs/publishing-a-benchmark.md)** — gates for publishing results.
-- **[docs/releasing.md](./docs/releasing.md)** — cutting prebuilt binaries.
 
 ## License
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@workos/quickstudy",
-  "version": "0.1.1",
+  "version": "0.2.0",
   "description": "An open-source, company-agnostic eval harness for measuring whether coding agents can complete real integration tasks, across an agent x feature x framework x surface matrix.",
   "license": "MIT",
   "type": "module",

package/src/adapters/parse.ts

@@ -115,3 +115,57 @@ export function parseCodexStream(raw: string): AttemptMetrics {
   return metrics;
 }
 
+/** Pricing needs complete usage, including the cache subset; partial telemetry cannot be priced. */
+export interface CodexCostUsage {
+  input_tokens: number;
+  cached_input_tokens: number;
+  output_tokens: number;
+}
+
+function costUsage(value: unknown): CodexCostUsage | null {
+  const usage = record(value);
+  const input = usage["input_tokens"], cached = usage["cached_input_tokens"], output = usage["output_tokens"];
+  const count = (v: unknown): v is number => typeof v === "number" && Number.isSafeInteger(v) && v >= 0;
+  if (!count(input) || !count(cached) || !count(output) || cached > input) return null;
+  const reasoning = usage["reasoning_output_tokens"];
+  if (reasoning !== undefined && (!count(reasoning) || reasoning > output)) return null;
+  // The three-rate model does not cover separately billed cache writes.
+  if (usage["cache_write_input_tokens"] !== undefined && usage["cache_write_input_tokens"] !== 0) return null;
+  return { input_tokens: input, cached_input_tokens: cached, output_tokens: output };
+}
+
+/** Strict counterpart to parseCodexStream: malformed/truncated usage stays unpriced. */
+export function parseCodexCostUsage(raw: string): CodexCostUsage | null {
+  let total: CodexCostUsage | null = null;
+  let complete = false;
+  let dialect: "thread" | "legacy" | undefined;
+  for (const line of raw.split("\n")) {
+    if (!line.trim().startsWith("{")) continue;
+    let obj: Record<string, unknown>;
+    try { obj = record(JSON.parse(line)); } catch { return null; }
+    const msg = record(obj["msg"]);
+    const type = obj["type"] ?? msg["type"];
+    if (type === "turn.failed" || type === "error") return null;
+    if (type === "turn.started" || type === "task_started") complete = false;
+    if (type === "task_complete" && dialect === "legacy") complete = true;
+    if (type !== "turn.completed" && type !== "token_count") continue;
+    const nextDialect = type === "turn.completed" ? "thread" : "legacy";
+    if (dialect !== undefined && dialect !== nextDialect) return null;
+    dialect = nextDialect;
+    const usage = costUsage(dialect === "thread" ? obj["usage"] : record(msg["info"])["total_token_usage"]);
+    if (usage === null) return null;
+    if (dialect === "legacy") {
+      total = usage; // Cumulative: never sum token_count snapshots.
+      complete = false;
+    } else {
+      total = total === null ? usage : {
+        input_tokens: total.input_tokens + usage.input_tokens,
+        cached_input_tokens: total.cached_input_tokens + usage.cached_input_tokens,
+        output_tokens: total.output_tokens + usage.output_tokens,
+      };
+      complete = true;
+    }
+  }
+  return complete && total !== null ? costUsage(total) : null;
+}
+

package/src/build-info.generated.ts

@@ -9,7 +9,7 @@ export const EMBEDDED_BUILD_INFO: {
   // (the `x-release-please-version` marker anchors the replacement). It is
   // restored as-is after a binary build (scripts/build-binary.ts), so the
   // committed source `--version` stays in lockstep with package.json.
-  version: "0.1.1", // x-release-please-version
+  version: "0.2.0", // x-release-please-version
   revision: null,
   dirty: null,
   sourceHash: null,

package/src/cli.ts

@@ -47,6 +47,7 @@ import {
   type ReportJson,
 } from "./report/report.ts";
 import { ExportError, exportSite } from "./export.ts";
+import { PricingError, readPricing } from "./report/costs.ts";
 import { renderDiagnosisText } from "./diagnose/render.ts";
 import { diagnosisJsonPath, runDiagnose } from "./diagnose/run.ts";
 import { DiagnoseError } from "./diagnose/types.ts";
@@ -85,6 +86,7 @@ Usage:
     [--latest]                            Report the most recent run instead of naming one
     [--json]                              Print report.json instead of the text report
     [--strict]                            Exit non-zero when incomplete or any attempt errored (CI/publication)
+    [--pricing <file>]                    Estimate missing Codex token costs from a runtime pricing JSON
     [--db <path>] [--results <dir>]       Stores (default: results.db, results/)
   quickstudy diagnose <run-id>            LLM-diagnose a run's failing pairs into diagnosis.json
     [--latest]                            Diagnose the most recent run instead of naming one
@@ -92,6 +94,7 @@ Usage:
   quickstudy export <run-id>...           Materialize a run as a self-contained static site
     [--runs <a,b|all>]                    More run ids (comma-separated), or "all" for every run in the DB
     [--out <dir>]                         Output directory (default: ./site)
+    [--pricing <file>]                    Recalculate token-cost estimates; otherwise retain saved pricing
     [--db <path>] [--results <dir>]       Stores (default: results.db, results/)
   quickstudy ui                           Export to a temp dir and serve the UI locally
     [--runs <a,b>]                        Runs to include (default: every run in the DB)
@@ -103,10 +106,10 @@ Usage:
 const COMMAND_HELP: Record<string, string> = {
   validate: `Usage: quickstudy validate [--evals-root <dir>] [--experiments-root <dir>]\n\nDiscover and validate the evals root (frontmatter, scorer default exports)\nand the experiments root (defineExperiment shapes, comparison groups).\nDefaults: evals, experiments.`,
   run: `Usage: quickstudy run [--eval <id>...] [--experiment <id>...] [options]\n\nExecute the selected eval × experiment pairs. Omit --eval to select every\ndiscovered eval; omit --experiment to select every discovered experiment.\n\nOptions:\n  --eval <id>... --experiment <id>...\n  --evals-root <dir> --experiments-root <dir>   (defaults: evals, experiments)\n  --suite <s>... --framework <f>...\n  --trials <n> --egress-proxy\n  --concurrency <n> --seed <value>             (default concurrency: 1)\n  --resume <run-id> --retry-unfinished        (explicit error/timeout retries)\n  --provision-ms <n> --agent-ms <n> --score-ms <n>\n  --export-ms <n> --cleanup-ms <n>\n  --db <path> --results <dir>`,
-  report: `Usage: quickstudy report <run-id> [options]\n\nBuild report.json into the results dir and print the text report\n(pass \`--latest\` to pick the most recent run; \`--json\` prints the JSON).\n\nOptions:\n  --latest --json --strict\n  --strict exits non-zero for incomplete runs or any attempt error\n  --db <path> --results <dir>`,
+  report: `Usage: quickstudy report <run-id> [options]\n\nBuild report.json into the results dir and print the text report\n(pass \`--latest\` to pick the most recent run; \`--json\` prints the JSON).\n\nOptions:\n  --latest --json --strict\n  --strict exits non-zero for incomplete runs or any attempt error\n  --pricing <file>   Estimate missing Codex token costs using runtime pricing JSON\n  --db <path> --results <dir>`,
   diagnose: `Usage: quickstudy diagnose <run-id> [options]\n\nSend each failing (eval × experiment) pair's harness-selected evidence to a\nstructured model call and write diagnosis.json beside report.json\n(pass \`--latest\` to pick the most recent run). Findings are hypotheses with\nevidence links — never verdicts. Requires ANTHROPIC_API_KEY; override the\nanalyst model with QUICKSTUDY_ANALYST_MODEL.\n\nOptions:\n  --latest\n  --db <path> --results <dir>`,
   clean: `Usage: quickstudy clean\n\nRemove quickstudy-labeled containers, networks, and sidecars.`,
-  export: `Usage: quickstudy export <run-id>... [options]\n\nBuild a self-contained static site.\n\nOptions:\n  --runs <a,b|all> --out <dir> --db <path> --results <dir>\n  --mode <offline|hosted>                    (default: offline)\n  --expected-trials <n>                      Publish fully sampled pairs; fail if none\n                                             Writes omissions and retains source diagnostics`,
+  export: `Usage: quickstudy export <run-id>... [options]\n\nBuild a self-contained static site.\n\nOptions:\n  --runs <a,b|all> --out <dir> --db <path> --results <dir>\n  --mode <offline|hosted>                    (default: offline)\n  --pricing <file>                           Recalculate costs using runtime pricing JSON\n  --expected-trials <n>                      Publish fully sampled pairs; fail if none\n                                             Writes omissions and retains source diagnostics`,
   ui: `Usage: quickstudy ui [options]\n\nExport stored runs to a temporary directory and serve the local UI.\n\nOptions:\n  --runs <a,b> --port <n> --watch --db <path> --results <dir>`,
 };
 
@@ -330,6 +333,7 @@ async function commandReport(args: string[]): Promise<void> {
       allowPositionals: true,
       options: {
         json: { type: "boolean", default: false },
+        pricing: { type: "string" },
         strict: { type: "boolean", default: false },
         latest: { type: "boolean", default: false },
         db: { type: "string", default: "results.db" },
@@ -342,6 +346,7 @@ async function commandReport(args: string[]): Promise<void> {
 
   const requestedRunId = positionals[0];
   if (!requestedRunId && !values.latest) fail(`report: missing <run-id> (or pass --latest)\n\n${USAGE}`);
+  const pricing = values.pricing === undefined ? undefined : readPricing(values.pricing);
 
   const store = new ResultsStore(values.db);
   try {
@@ -356,7 +361,7 @@ async function commandReport(args: string[]): Promise<void> {
       runId = latest.id;
     }
 
-    const report: ReportJson = buildReport({ store, runId: runId as string, resultsDir: values.results });
+    const report: ReportJson = buildReport({ store, runId: runId as string, resultsDir: values.results, pricing });
     const path = writeReportJson(report, values.results);
     console.log(values.json ? JSON.stringify(report, null, 2) : renderReportText(report));
     console.error(`report written: ${path}`);
@@ -532,7 +537,7 @@ function collectRunIds(positionals: string[], runsFlag: string | undefined): str
 }
 
 async function commandExport(args: string[]): Promise<void> {
-  let values: { runs?: string; out: string; db: string; results: string; mode: string; "expected-trials"?: string };
+  let values: { runs?: string; out: string; db: string; results: string; mode: string; "expected-trials"?: string; pricing?: string };
   let positionals: string[];
   try {
     ({ values, positionals } = parseArgs({
@@ -541,6 +546,7 @@ async function commandExport(args: string[]): Promise<void> {
       options: {
         runs: { type: "string" },
         out: { type: "string", default: "site" },
+        pricing: { type: "string" },
         mode: { type: "string", default: "offline" },
         "expected-trials": { type: "string" },
         db: { type: "string", default: "results.db" },
@@ -551,6 +557,7 @@ async function commandExport(args: string[]): Promise<void> {
     fail(`export: ${err instanceof Error ? err.message : String(err)}\n\n${USAGE}`);
   }
 
+  const pricing = values.pricing === undefined ? undefined : readPricing(values.pricing);
   const bundleDir = mustUiBundleDir();
   const store = new ResultsStore(values.db);
   try {
@@ -572,6 +579,7 @@ async function commandExport(args: string[]): Promise<void> {
       resultsDir: values.results,
       runIds,
       outDir: values.out,
+      pricing,
       mode: values.mode === "hosted" || values.mode === "offline" ? values.mode : fail("export --mode must be hosted or offline"),
       ...(values["expected-trials"] ? { expectedTrials: parsePositiveInt(values["expected-trials"], "--expected-trials") } : {}),
       bundleDir,
@@ -663,6 +671,7 @@ async function commandUi(args: string[]): Promise<void> {
 }
 
 interface ReportFlags {
+  pricing?: string;
   json: boolean;
   strict: boolean;
   latest: boolean;
@@ -774,6 +783,7 @@ async function main(): Promise<void> {
       err instanceof MissingProviderKeyError ||
       err instanceof InvalidProviderKeyError ||
       err instanceof ReportError ||
+      err instanceof PricingError ||
       err instanceof ExportError ||
       err instanceof DiagnoseError ||
       err instanceof ModelOutputError

package/src/export.ts

@@ -34,6 +34,7 @@ import {
 import { trialCoverage } from "./completeness.ts";
 import { hashString } from "./hash.ts";
 import { buildReport, renderReportText, type ReportJson } from "./report/report.ts";
+import { parsePricing, type Pricing } from "./report/costs.ts";
 import { activeAttempts, type AttemptRecord, type ResultsStore, type RunRecord } from "./store/db.ts";
 
 /** Loud, CLI-friendly failures (unknown run, missing bundle, ...). */
@@ -45,6 +46,8 @@ export class ExportError extends Error {
 }
 
 export interface ExportOptions {
+  /** Recalculate token-cost estimates from saved transcripts using this runtime snapshot. */
+  pricing?: Pricing;
   /** Offline embeds JSON; hosted fetches immutable per-run snapshots lazily. */
   mode?: "offline" | "hosted";
   /** Explicit publication gate; omits incompletely sampled pairs with reasons. */
@@ -175,7 +178,9 @@ function resolveReport(
   runId: string,
   log: (message: string) => void,
   generatedAt?: string,
+  pricing?: Pricing,
 ): ReportJson {
+  if (pricing !== undefined) return buildReport({ store, runId, resultsDir, generatedAt, pricing });
   const path = join(resultsDir, runId, "report.json");
   if (existsSync(path)) {
     try {
@@ -186,8 +191,9 @@ function resolveReport(
       const run = store.getRun(runId)!;
       // Historical reports retain their methodology. New reports are reused
       // only when the entire source record still matches (including retries).
-      if (run.manifest.manifestVersion === "v3") return stored;
-      const current = buildReport({ store, runId, resultsDir, generatedAt: generatedAt ?? iso(run.finishedAt) ?? iso(run.startedAt)! });
+      if (run.manifest.manifestVersion === "v3" && stored.cost_estimate === undefined) return stored;
+      const current = buildReport({ store, runId, resultsDir, generatedAt: generatedAt ?? iso(run.finishedAt) ?? iso(run.startedAt)!,
+        ...(stored.cost_estimate ? { pricing: parsePricing(stored.cost_estimate.pricing) } : {}) });
       return stored.source_fingerprint === current.source_fingerprint ? stored : current;
     } catch (err) {
       log(`stored report.json for ${runId} is unreadable (${err instanceof Error ? err.message : String(err)}) — rebuilding`);
@@ -256,6 +262,14 @@ export function inlineDataIntoHtml(html: string, data: InlineData): string {
   return html.replace(QS_DATA_SLOT, () => `<script id="qs-data">window.__QS_DATA__ = ${json};</script>`);
 }
 
+function publicationReport(options: ExportOptions, run: RunRecord, log: (message: string) => void): ReportJson | undefined {
+  if (options.expectedTrials === undefined) return undefined;
+  const pricing = options.pricing ?? resolveReport(options.store, options.resultsDir, run.id, log, options.generatedAt).cost_estimate?.pricing;
+  return buildReport({ store: options.store, runId: run.id, resultsDir: options.resultsDir,
+    expectedTrials: options.expectedTrials, pricing,
+    generatedAt: options.generatedAt ?? iso(run.finishedAt) ?? iso(run.startedAt)! });
+}
+
 /** Export one or more runs into a self-contained static site directory. */
 export async function exportSite(options: ExportOptions): Promise<ExportSummary> {
   const log = options.log ?? ((message: string) => console.warn(message));
@@ -269,7 +283,7 @@ export async function exportSite(options: ExportOptions): Promise<ExportSummary>
     );
   }
 
-  const publications = new Map(runs.map((run) => [run.id, options.expectedTrials === undefined ? undefined : buildReport({ store, runId: run.id, resultsDir, expectedTrials: options.expectedTrials, generatedAt: options.generatedAt ?? iso(run.finishedAt) ?? iso(run.startedAt)! })]));
+  const publications = new Map(runs.map((run) => [run.id, publicationReport(options, run, log)]));
   const dataDir = join(outDir, "data");
   mkdirSync(dataDir, { recursive: true });
 
@@ -321,11 +335,11 @@ export async function exportSite(options: ExportOptions): Promise<ExportSummary>
     };
     atomicWrite(join(runDir, "run.json"), `${JSON.stringify(dataRun, null, 2)}\n`, "utf8");
 
-    const report = publications.get(run.id) ?? resolveReport(store, resultsDir, run.id, log, options.generatedAt);
+    const report = publications.get(run.id) ?? resolveReport(store, resultsDir, run.id, log, options.generatedAt, options.pricing);
     if (publication) {
       atomicWrite(join(runDir, "omissions.json"), JSON.stringify(publication, null, 2));
       atomicWrite(join(runDir, "omissions.txt"), renderReportText(report));
-      atomicWrite(join(runDir, "source-report.json"), JSON.stringify(buildReport({ store, runId: run.id, resultsDir }), null, 2));
+      atomicWrite(join(runDir, "source-report.json"), JSON.stringify(buildReport({ store, runId: run.id, resultsDir, pricing: report.cost_estimate?.pricing }), null, 2));
       log(`${run.id}: complete-sample publication includes ${publication.eligible.length} pair(s), omits ${publication.omitted.length}; see data/runs/${run.id}/omissions.json and omissions.txt`);
     }
     atomicWrite(join(runDir, "report.json"), `${JSON.stringify(report, null, 2)}\n`, "utf8");

package/src/report/costs.ts

@@ -0,0 +1,116 @@
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+import { z } from "zod";
+import { parseCodexCostUsage, type CodexCostUsage } from "../adapters/parse.ts";
+import { hashString } from "../hash.ts";
+import type { AttemptRecord, RunRecord } from "../store/db.ts";
+
+const rateSchema = z.object({
+  provider: z.literal("openai"),
+  model: z.string().min(1),
+  effective_from: z.iso.datetime({ offset: true }),
+  effective_until: z.iso.datetime({ offset: true }).optional(),
+  source: z.url({ protocol: /^https?$/ }),
+  input_usd_per_million: z.number().finite().nonnegative(),
+  cached_input_usd_per_million: z.number().finite().nonnegative(),
+  output_usd_per_million: z.number().finite().nonnegative(),
+}).strict();
+const pricingSchema = z.object({
+  version: z.string().min(1), currency: z.literal("USD"), rates: z.array(rateSchema).min(1),
+}).strict();
+export type Pricing = z.infer<typeof pricingSchema>;
+type Rate = Pricing["rates"][number];
+
+export class PricingError extends Error {
+  constructor(message: string) { super(message); this.name = "PricingError"; }
+}
+
+/** Reject ambiguous schedules rather than silently selecting one matching price. */
+export function parsePricing(value: unknown): Pricing {
+  const result = pricingSchema.safeParse(value);
+  if (!result.success) throw new PricingError(`invalid pricing: ${result.error.message}`);
+  const pricing = result.data;
+  const ends = new Map<string, number>();
+  for (const rate of [...pricing.rates].sort((a, b) => Date.parse(a.effective_from) - Date.parse(b.effective_from))) {
+    const start = Date.parse(rate.effective_from);
+    const end = rate.effective_until === undefined ? Infinity : Date.parse(rate.effective_until);
+    if (end <= start) throw new PricingError(`invalid pricing: ${rate.model} effective_until must follow effective_from`);
+    const key = `${rate.provider}:${rate.model}`;
+    if ((ends.get(key) ?? -Infinity) > start) throw new PricingError(`invalid pricing: overlapping rates for ${rate.model}`);
+    ends.set(key, end);
+  }
+  return pricing;
+}
+
+export function readPricing(path: string): Pricing {
+  try { return parsePricing(JSON.parse(readFileSync(path, "utf8"))); }
+  catch (err) {
+    throw new PricingError(`pricing file ${path}: ${err instanceof Error ? err.message : String(err)}`);
+  }
+}
+
+export interface AttemptCostEstimate {
+  attempt_id: string;
+  cost_usd: number | null;
+  unpriced_reason?: "unsupported_adapter" | "pricing_unavailable" | "transcript_unreadable" | "usage_unavailable" | "invalid_cost";
+  usage?: CodexCostUsage;
+  transcript_sha256?: string;
+  rate_index?: number;
+}
+
+/** Estimates supplement, never replace, telemetry.cost_usd and its reported-attempt count. */
+export interface RunCostEstimate {
+  pricing: Pricing;
+  cost_usd: number | null;
+  cost_attempts: number;
+  unpriced_attempts: number;
+  /** Only attempts without a provider-reported cost. All values are agent token costs, not invoices. */
+  attempts: AttemptCostEstimate[];
+}
+
+function readTranscript(resultsDir: string, runId: string, attempt: AttemptRecord): string {
+  // Prefer the restored artifacts tree to the original machine's absolute reference.
+  const canonical = join(resultsDir, runId, attempt.id, "transcript.jsonl");
+  try { return readFileSync(canonical, "utf8"); }
+  catch (err) {
+    if ((err as NodeJS.ErrnoException).code !== "ENOENT" || !attempt.transcriptRef || attempt.transcriptRef === canonical) throw err;
+    return readFileSync(attempt.transcriptRef, "utf8");
+  }
+}
+
+function estimateAttempt(run: RunRecord, attempt: AttemptRecord, resultsDir: string, pricing: Pricing): AttemptCostEstimate {
+  const unavailable = (unpriced_reason: NonNullable<AttemptCostEstimate["unpriced_reason"]>): AttemptCostEstimate =>
+    ({ attempt_id: attempt.id, cost_usd: null, unpriced_reason });
+  const agent = run.manifest.experiments[attempt.experimentId]?.components.agent;
+  if (agent?.adapter !== "codex" || agent.provider !== "openai") return unavailable("unsupported_adapter");
+  const at = attempt.startedAt ?? run.startedAt;
+  const rateIndex = pricing.rates.findIndex((rate) => rate.provider === agent.provider && rate.model === agent.model &&
+    at >= Date.parse(rate.effective_from) && (rate.effective_until === undefined || at < Date.parse(rate.effective_until)));
+  const rate = pricing.rates[rateIndex];
+  if (rate === undefined) return unavailable("pricing_unavailable");
+  let raw: string;
+  try { raw = readTranscript(resultsDir, run.id, attempt); }
+  catch { return unavailable("transcript_unreadable"); }
+  const usage = parseCodexCostUsage(raw);
+  if (usage === null) return { ...unavailable("usage_unavailable"), transcript_sha256: hashString(raw) };
+  const cost = tokenCost(usage, rate);
+  if (!Number.isFinite(cost)) return unavailable("invalid_cost");
+  return { attempt_id: attempt.id, cost_usd: cost, usage, transcript_sha256: hashString(raw), rate_index: rateIndex };
+}
+
+function tokenCost(usage: CodexCostUsage, rate: Rate): number {
+  return ((usage.input_tokens - usage.cached_input_tokens) * rate.input_usd_per_million +
+    usage.cached_input_tokens * rate.cached_input_usd_per_million +
+    usage.output_tokens * rate.output_usd_per_million) / 1_000_000;
+}
+
+/** Uses the report's selected attempts (excluding superseded retries and publication omissions). */
+export function estimateRunCosts(run: RunRecord, attempts: AttemptRecord[], resultsDir: string, input: Pricing): RunCostEstimate {
+  const pricing = parsePricing(input);
+  const estimates = attempts.filter((a) => a.costUsd === null).map((a) => estimateAttempt(run, a, resultsDir, pricing));
+  const costs = estimates.flatMap((a) => a.cost_usd === null ? [] : [a.cost_usd]);
+  const total = costs.reduce((sum, cost) => sum + cost, 0);
+  if (!Number.isFinite(total)) throw new PricingError("estimated token cost exceeds the supported numeric range");
+  return { pricing, cost_usd: costs.length ? total : null, cost_attempts: costs.length,
+    unpriced_attempts: estimates.length - costs.length, attempts: estimates };
+}

package/src/report/report.ts

@@ -18,8 +18,9 @@
  *   - offered-versus-used surface telemetry per experiment;
  *   - egress denials, when the proxied run recorded any.
  *
- * There are deliberately no intervals, no estimates, no analyst layers, and
- * no language stronger than the counts themselves (see BANNED_LANGUAGE).
+ * There are deliberately no intervals, no statistical estimates, no analyst layers,
+ * and no language stronger than the counts themselves (see BANNED_LANGUAGE).
+ * Opt-in token-cost estimates are a separate layer, never provider-reported telemetry.
  */
 
 import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
@@ -30,6 +31,7 @@ import type { ComparisonGroup } from "../experiments/groups.ts";
 import { activeAttempts, type AttemptRecord, type ResultsStore, type RunRecord } from "../store/db.ts";
 import type { SurfaceUsage } from "../surface-usage.ts";
 import { collectPassAtK, displayPassAtK, type PassAtKEntry } from "./pass-at-k.ts";
+import { estimateRunCosts, type Pricing, type RunCostEstimate } from "./costs.ts";
 
 /** Loud, CLI-friendly failures (unknown run, missing report.json, ...). */
 export class ReportError extends Error {
@@ -217,6 +219,8 @@ export interface ReportJson {
   retry_history?: Array<{ attempt_id: string; status: string; error: string | null }>;
   totals: ReportTotals;
   telemetry: ReportTelemetry;
+  /** Optional runtime-priced supplement; absent on historical or unpriced reports. */
+  cost_estimate?: RunCostEstimate;
   judge_telemetry?: { provenance: "semantic-judge"; calls: number; tokens_in: number | null; tokens_out: number | null; cost_usd: number | null };
   /**
    * Per-experiment identity cards (model, reasoning, adapter, ...). Optional
@@ -486,6 +490,8 @@ function collectEgressDenials(resultsDir: string, runId: string, attempts: Attem
 
 export interface BuildReportOptions {
   expectedTrials?: number;
+  /** Explicit runtime rate snapshot; never changes stored provider-reported costs. */
+  pricing?: Pricing;
   store: ResultsStore;
   runId: string;
   /** Artifacts root (the harness `--results` dir) — egress denial logs live here. */
@@ -514,8 +520,10 @@ export function buildReport(options: BuildReportOptions): ReportJson {
   if (publication && publication.eligible.length === 0) throw new ReportError(`no eligible complete samples for run ${runId}: ${publication.omitted.map((p) => `${p.eval_id} × ${p.experiment_id}: ${p.reasons.join("; ")}`).join(" | ")}`);
   const attempts = publication ? current.filter((a) => publication.eligible.some((p) => p.eval_id === a.evalId && p.experiment_id === a.experimentId)) : current;
   const pairs = collectPairs(attempts);
+  const costEstimate = options.pricing === undefined ? undefined : estimateRunCosts(run, attempts, options.resultsDir, options.pricing);
 
-  const methodology = methodologyNote(run.config.trials);
+  const methodology = methodologyNote(run.config.trials) + (costEstimate === undefined ? "" :
+    " Optional token-cost estimates supplement reported cost using the supplied pricing snapshot. They cover only the selected attempts, not superseded retries, tool fees, diagnosis or infrastructure, and are not billing records.");
   const banned = findBannedLanguage(methodology);
   if (banned.length > 0) {
     throw new ReportError(`report methodology violates the language guard: ${banned.join(", ")}`);
@@ -534,12 +542,13 @@ export function buildReport(options: BuildReportOptions): ReportJson {
       manifest_hash: run.manifestHash,
     },
     incomplete: publication ? false : run.finishedAt === null || coverage.length === 0 || coverage.some((pair) => !pair.complete),
-    source_fingerprint: hashString(JSON.stringify({ manifest: run.manifestHash, finishedAt: run.finishedAt, trials: run.config.trials, evals: run.config.evalIds, experiments: run.config.experimentIds, history: history.map(({ transcriptRef, diffRef, ...row }) => ({ ...row, transcript: Boolean(transcriptRef), diff: Boolean(diffRef) })) })),
+    source_fingerprint: hashString(JSON.stringify({ manifest: run.manifestHash, finishedAt: run.finishedAt, trials: run.config.trials, evals: run.config.evalIds, experiments: run.config.experimentIds, history: history.map(({ transcriptRef, diffRef, ...row }) => ({ ...row, transcript: Boolean(transcriptRef), diff: Boolean(diffRef) })), costEstimate })),
     ...(publication ? { publication: { ...publication, source_totals: collectTotals(current), source_incomplete: run.finishedAt === null || coverage.some((p) => !p.complete) } } : {}),
     coverage,
     retry_history: history.filter((a) => !current.includes(a)).map((a) => ({ attempt_id: a.id, status: a.status, error: a.error })),
     totals: collectTotals(attempts),
     telemetry: collectTelemetry(attempts),
+    ...(costEstimate === undefined ? {} : { cost_estimate: costEstimate }),
     judge_telemetry: collectJudgeTelemetry(attempts),
     experiments: collectExperiments(run),
     pairs,
@@ -605,6 +614,16 @@ export function renderReportText(report: ReportJson): string {
   lines.push(
     `  cost:    ${tel.cost_usd === null ? "unavailable" : `$${tel.cost_usd.toFixed(4)} across ${tel.cost_attempts} attempt(s)`}`,
   );
+  if (report.cost_estimate) {
+    const estimate = report.cost_estimate;
+    const cost = estimate.cost_usd;
+    const displayed = cost === null ? "unavailable" : cost > 0 && cost < 0.0001 ? "<$0.0001" : `$${cost.toFixed(4)}`;
+    lines.push(`  estimated token cost: ${displayed} (in addition to reported cost)`);
+    lines.push(`    ${estimate.cost_attempts} estimated, ${estimate.unpriced_attempts} unpriced attempt(s); pricing ${estimate.pricing.version}`);
+    for (const attempt of estimate.attempts) {
+      if (attempt.unpriced_reason) lines.push(`    ${attempt.attempt_id}: ${attempt.unpriced_reason}`);
+    }
+  }
   lines.push(
     `  tokens:  ${
       tel.tokens_in === null && tel.tokens_out === null