@cleocode/core 2026.9.23 → 2026.9.24

package/dist/decide/bench/questions.d.ts

@@ -0,0 +1,40 @@
+/**
+ * Turn a benchmark row into the exact question its decision site asks, the
+ * site's heuristic answer, and the mapping from an answer back to a label
+ * (T12495).
+ *
+ * The request builders are the sites' own (`buildDuplicateDecisionRequest`,
+ * `buildObservationTypeRequest`, `buildContradictionRequest`), so the
+ * benchmark measures the production wording. The heuristic is the rule each
+ * site falls back to: Tier-1 lexical similarity for duplicates, whole-word
+ * keywords for observation types, the word-Jaccard collision check for
+ * decisions.
+ *
+ * @task T12495
+ * @epic T12486
+ */
+import type { DecisionAnswer, DecisionRequest } from '@cleocode/contracts';
+import type { BenchRow } from './types.js';
+/** One row made ready to ask. */
+export interface BenchQuestion {
+    /** The site's request, with `cache: false` (no provider answer cache). */
+    readonly req: DecisionRequest;
+    /** The site heuristic's answers, in the request's shape (the fallback). */
+    readonly heuristicAnswers: Readonly<Record<string, DecisionAnswer>>;
+    /** The label the heuristic predicts. */
+    readonly heuristicLabel: string;
+    /**
+     * Map answers back to a label.
+     *
+     * @returns The predicted label, or `null` when the answer is unusable.
+     */
+    readonly predict: (answers: Readonly<Record<string, DecisionAnswer>>) => string | null;
+}
+/**
+ * Build the question for `row`.
+ *
+ * @param row - A dataset row.
+ * @returns Request, heuristic answer and label mapping.
+ */
+export declare function benchQuestionFor(row: BenchRow): BenchQuestion;
+//# sourceMappingURL=questions.d.ts.map

package/dist/decide/bench/questions.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"questions.d.ts","sourceRoot":"","sources":["../../../src/decide/bench/questions.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAiB3E,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAE3C,iCAAiC;AACjC,MAAM,WAAW,aAAa;IAC5B,0EAA0E;IAC1E,QAAQ,CAAC,GAAG,EAAE,eAAe,CAAC;IAC9B,2EAA2E;IAC3E,QAAQ,CAAC,gBAAgB,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC,CAAC;IACpE,wCAAwC;IACxC,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC;;;;OAIG;IACH,QAAQ,CAAC,OAAO,EAAE,CAAC,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC,KAAK,MAAM,GAAG,IAAI,CAAC;CACxF;AAcD;;;;;GAKG;AACH,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,QAAQ,GAAG,aAAa,CA+E7D"}

package/dist/decide/bench/questions.js

@@ -0,0 +1,107 @@
+/**
+ * Turn a benchmark row into the exact question its decision site asks, the
+ * site's heuristic answer, and the mapping from an answer back to a label
+ * (T12495).
+ *
+ * The request builders are the sites' own (`buildDuplicateDecisionRequest`,
+ * `buildObservationTypeRequest`, `buildContradictionRequest`), so the
+ * benchmark measures the production wording. The heuristic is the rule each
+ * site falls back to: Tier-1 lexical similarity for duplicates, whole-word
+ * keywords for observation types, the word-Jaccard collision check for
+ * decisions.
+ *
+ * @task T12495
+ * @epic T12486
+ */
+import { buildContradictionRequest, CONTRADICTION_RELATIONS, } from '../../memory/decision-contradiction.js';
+import { DECISION_COLLISION_THRESHOLD, decisionWordJaccard } from '../../memory/decisions.js';
+import { buildObservationTypeRequest, classifyObservationTypeByKeywords, OBSERVATION_TYPE_OPTIONS, } from '../../memory/observation-type-decision.js';
+import { redactContent } from '../../memory/redaction.js';
+import { buildDuplicateDecisionRequest, computeLexicalSimilarity, DUPLICATE_WARN_THRESHOLD, } from '../../tasks/duplicate-detector.js';
+/** The redaction System One applies (idempotent on already-redacted text). */
+const redact = (text) => redactContent(text).content;
+/** Numeric part of a task id, for "which task is newer". */
+function taskNumber(id) {
+    const n = Number(/\d+/.exec(id)?.[0] ?? Number.NaN);
+    return Number.isFinite(n) ? n : 0;
+}
+/** Question name the site builders give the first (only) candidate. */
+const FIRST = 'c1';
+/**
+ * Build the question for `row`.
+ *
+ * @param row - A dataset row.
+ * @returns Request, heuristic answer and label mapping.
+ */
+export function benchQuestionFor(row) {
+    if (row.site === 'duplicateDetection') {
+        // The later-created task (higher id) plays the "new" task.
+        const [candidate, fresh] = taskNumber(row.input.a.id) <= taskNumber(row.input.b.id)
+            ? [row.input.a, row.input.b]
+            : [row.input.b, row.input.a];
+        const score = computeLexicalSimilarity(fresh.title, fresh.description, candidate.title, candidate.description);
+        const dup = score >= DUPLICATE_WARN_THRESHOLD;
+        const req = buildDuplicateDecisionRequest(fresh.title, fresh.description, [{ task: candidate }], redact);
+        return {
+            req: { ...req, cache: false },
+            heuristicAnswers: {
+                [FIRST]: {
+                    type: 'noul',
+                    value: dup,
+                    probability: Math.min(1, Math.max(0, score)),
+                    confidence: 0.5,
+                },
+            },
+            heuristicLabel: dup ? 'duplicate' : 'distinct',
+            predict: (answers) => {
+                const a = answers[FIRST];
+                if (a?.type !== 'noul')
+                    return null;
+                return a.value ? 'duplicate' : 'distinct';
+            },
+        };
+    }
+    if (row.site === 'observationType') {
+        const guess = classifyObservationTypeByKeywords(row.input.text);
+        const req = buildObservationTypeRequest(row.input.text, row.input.title || undefined, redact);
+        const probabilities = {};
+        for (const o of OBSERVATION_TYPE_OPTIONS)
+            probabilities[o] = o === guess ? 1 : 0;
+        return {
+            req: { ...req, cache: false },
+            heuristicAnswers: { type: { type: 'choice', value: guess, probabilities, confidence: 0.5 } },
+            heuristicLabel: guess,
+            predict: (answers) => {
+                const a = answers['type'];
+                if (a?.type !== 'choice')
+                    return null;
+                return OBSERVATION_TYPE_OPTIONS.some((o) => o === a.value) ? a.value : null;
+            },
+        };
+    }
+    const { newer, older } = row.input;
+    const collision = decisionWordJaccard(newer, older) >= DECISION_COLLISION_THRESHOLD;
+    const req = buildContradictionRequest(
+    // The declared `supersedes` is deliberately NOT sent: it is the label.
+    { type: newer.type ?? 'unknown', decision: newer.decision, rationale: newer.rationale }, [{ id: older.id, decision: older.decision, rationale: older.rationale, score: 0 }], redact);
+    const guess = collision ? 'supersedes' : 'unrelated';
+    const probabilities = {};
+    for (const r of CONTRADICTION_RELATIONS)
+        probabilities[r] = r === guess ? 1 : 0;
+    return {
+        req: { ...req, cache: false },
+        heuristicAnswers: { [FIRST]: { type: 'choice', value: guess, probabilities, confidence: 0.5 } },
+        heuristicLabel: collision ? 'conflict' : 'compatible',
+        predict: (answers) => {
+            const a = answers[FIRST];
+            if (a?.type !== 'choice')
+                return null;
+            if (a.value === 'contradicts' || a.value === 'supersedes')
+                return 'conflict';
+            if (a.value === 'refines' || a.value === 'unrelated')
+                return 'compatible';
+            return null;
+        },
+    };
+}
+//# sourceMappingURL=questions.js.map

package/dist/decide/bench/questions.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"questions.js","sourceRoot":"","sources":["../../../src/decide/bench/questions.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAGH,OAAO,EACL,yBAAyB,EACzB,uBAAuB,GACxB,MAAM,wCAAwC,CAAC;AAChD,OAAO,EAAE,4BAA4B,EAAE,mBAAmB,EAAE,MAAM,2BAA2B,CAAC;AAC9F,OAAO,EACL,2BAA2B,EAC3B,iCAAiC,EACjC,wBAAwB,GACzB,MAAM,2CAA2C,CAAC;AACnD,OAAO,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAC1D,OAAO,EACL,6BAA6B,EAC7B,wBAAwB,EACxB,wBAAwB,GACzB,MAAM,mCAAmC,CAAC;AAmB3C,8EAA8E;AAC9E,MAAM,MAAM,GAAG,CAAC,IAAY,EAAU,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC;AAErE,4DAA4D;AAC5D,SAAS,UAAU,CAAC,EAAU;IAC5B,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC,GAAG,CAAC,CAAC;IACpD,OAAO,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACpC,CAAC;AAED,uEAAuE;AACvE,MAAM,KAAK,GAAG,IAAI,CAAC;AAEnB;;;;;GAKG;AACH,MAAM,UAAU,gBAAgB,CAAC,GAAa;IAC5C,IAAI,GAAG,CAAC,IAAI,KAAK,oBAAoB,EAAE,CAAC;QACtC,2DAA2D;QAC3D,MAAM,CAAC,SAAS,EAAE,KAAK,CAAC,GACtB,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;YACtD,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;YAC5B,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACjC,MAAM,KAAK,GAAG,wBAAwB,CACpC,KAAK,CAAC,KAAK,EACX,KAAK,CAAC,WAAW,EACjB,SAAS,CAAC,KAAK,EACf,SAAS,CAAC,WAAW,CACtB,CAAC;QACF,MAAM,GAAG,GAAG,KAAK,IAAI,wBAAwB,CAAC;QAC9C,MAAM,GAAG,GAAG,6BAA6B,CACvC,KAAK,CAAC,KAAK,EACX,KAAK,CAAC,WAAW,EACjB,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,EACrB,MAAM,CACP,CAAC;QACF,OAAO;YACL,GAAG,EAAE,EAAE,GAAG,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE;YAC7B,gBAAgB,EAAE;gBAChB,CAAC,KAAK,CAAC,EAAE;oBACP,IAAI,EAAE,MAAM;oBACZ,KAAK,EAAE,GAAG;oBACV,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;oBAC5C,UAAU,EAAE,GAAG;iBAChB;aACF;YACD,cAAc,EAAE,GAAG,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,UAAU;YAC9C,OAAO,EAAE,CAAC,OAAO,EAAE,EAAE;gBACnB,MAAM,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;gBACzB,IAAI,CAAC,EAAE,IAAI,KAAK,MAAM;oBAAE,OAAO,IAAI,CAAC;gBACpC,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,UAAU,CAAC;YAC5C,CAAC;SACF,CAAC;IACJ,CAAC;IAED,IAAI,GAAG,CAAC,IAAI,KAAK,iBAAiB,EAAE,CAAC;QACnC,MAAM,KAAK,GAAG,iCAAiC,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAChE,MAAM,GAAG,GAAG,2BAA2B,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,GAAG,CAAC,KAAK,CAAC,KAAK,IAAI,SAAS,EAAE,MAAM,CAAC,CAAC;QAC9F,MAAM,aAAa,GAA2B,EAAE,CAAC;QACjD,KAAK,MAAM,CAAC,IAAI,wBAAwB;YAAE,aAAa,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACjF,OAAO;YACL,GAAG,EAAE,EAAE,GAAG,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE;YAC7B,gBAAgB,EAAE,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,EAAE,aAAa,EAAE,UAAU,EAAE,GAAG,EAAE,EAAE;YAC5F,cAAc,EAAE,KAAK;YACrB,OAAO,EAAE,CAAC,OAAO,EAAE,EAAE;gBACnB,MAAM,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;gBAC1B,IAAI,CAAC,EAAE,IAAI,KAAK,QAAQ;oBAAE,OAAO,IAAI,CAAC;gBACtC,OAAO,wBAAwB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;YAC9E,CAAC;SACF,CAAC;IACJ,CAAC;IAED,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,GAAG,CAAC,KAAK,CAAC;IACnC,MAAM,SAAS,GAAG,mBAAmB,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,4BAA4B,CAAC;IACpF,MAAM,GAAG,GAAG,yBAAyB;IACnC,uEAAuE;IACvE,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,IAAI,SAAS,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,EACvF,CAAC,EAAE,EAAE,EAAE,KAAK,CAAC,EAAE,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC,EAClF,MAAM,CACP,CAAC;IACF,MAAM,KAAK,GAAG,SAAS,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,WAAW,CAAC;IACrD,MAAM,aAAa,GAA2B,EAAE,CAAC;IACjD,KAAK,MAAM,CAAC,IAAI,uBAAuB;QAAE,aAAa,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAChF,OAAO;QACL,GAAG,EAAE,EAAE,GAAG,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE;QAC7B,gBAAgB,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,EAAE,aAAa,EAAE,UAAU,EAAE,GAAG,EAAE,EAAE;QAC/F,cAAc,EAAE,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,YAAY;QACrD,OAAO,EAAE,CAAC,OAAO,EAAE,EAAE;YACnB,MAAM,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC;YACzB,IAAI,CAAC,EAAE,IAAI,KAAK,QAAQ;gBAAE,OAAO,IAAI,CAAC;YACtC,IAAI,CAAC,CAAC,KAAK,KAAK,aAAa,IAAI,CAAC,CAAC,KAAK,KAAK,YAAY;gBAAE,OAAO,UAAU,CAAC;YAC7E,IAAI,CAAC,CAAC,KAAK,KAAK,SAAS,IAAI,CAAC,CAAC,KAAK,KAAK,WAAW;gBAAE,OAAO,YAAY,CAAC;YAC1E,OAAO,IAAI,CAAC;QACd,CAAC;KACF,CAAC;AACJ,CAAC"}

package/dist/decide/bench/report.d.ts

@@ -0,0 +1,21 @@
+/**
+ * Markdown report fragment for a System One benchmark (T12495): method,
+ * dataset sizes, a per-site table, latency, cost and re-run instructions.
+ * Meant to be pasted into the README, a docs page and the site; this module
+ * only renders it.
+ *
+ * @task T12495
+ * @epic T12486
+ */
+import type { BenchResults } from './runner.js';
+/**
+ * Render the Markdown fragment.
+ *
+ * @param results - Benchmark results.
+ * @param opts - The re-run command to print (default: built from the config).
+ * @returns Markdown (starts at a level-2 heading so it embeds anywhere).
+ */
+export declare function renderBenchReport(results: BenchResults, opts?: {
+    readonly rerunCommand?: string;
+}): string;
+//# sourceMappingURL=report.d.ts.map

package/dist/decide/bench/report.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"report.d.ts","sourceRoot":"","sources":["../../../src/decide/bench/report.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EAAqB,YAAY,EAAE,MAAM,aAAa,CAAC;AA4CnE;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAC/B,OAAO,EAAE,YAAY,EACrB,IAAI,GAAE;IAAE,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAA;CAAO,GAC5C,MAAM,CA2FR"}

package/dist/decide/bench/report.js

@@ -0,0 +1,120 @@
+/**
+ * Markdown report fragment for a System One benchmark (T12495): method,
+ * dataset sizes, a per-site table, latency, cost and re-run instructions.
+ * Meant to be pasted into the README, a docs page and the site; this module
+ * only renders it.
+ *
+ * @task T12495
+ * @epic T12486
+ */
+import { HEURISTIC_PROVIDER } from './runner.js';
+/** Human site names. */
+const SITE_TITLES = {
+    duplicateDetection: 'Duplicate task detection (`tasks.duplicate-detection`)',
+    observationType: 'Observation type (`memory.observation-type`)',
+    decisionContradiction: 'Decision contradiction (`memory.decision-contradiction`)',
+};
+/** Format a ratio spread as a percentage with its standard deviation. */
+function pct(s) {
+    if (s.mean === null)
+        return '—';
+    const mean = (s.mean * 100).toFixed(1);
+    return s.n > 1 && s.stddev !== null ? `${mean}% ± ${(s.stddev * 100).toFixed(1)}` : `${mean}%`;
+}
+/** Format a millisecond spread. */
+function ms(s) {
+    if (s.mean === null)
+        return '—';
+    return s.n > 1 && s.stddev !== null
+        ? `${s.mean.toFixed(0)} ± ${s.stddev.toFixed(0)}`
+        : s.mean.toFixed(s.mean < 10 ? 2 : 0);
+}
+/** Format micro-dollars as dollars. */
+function usd(micros) {
+    return `$${(micros / 1e6).toFixed(micros < 10_000 ? 6 : 4)}`;
+}
+/** Format fallbacks as `reason×n`. */
+function fallbacks(row) {
+    const parts = Object.entries(row.fallbacks).map(([k, v]) => `${k}×${v}`);
+    return parts.length > 0 ? parts.join(', ') : '0';
+}
+/** One table row. */
+function tableRow(row) {
+    const n = row.answered.mean === null ? '—' : row.answered.mean.toFixed(0);
+    const cost = row.provider === HEURISTIC_PROVIDER ? '$0' : usd(row.costMicros.mean ?? 0);
+    return `| ${row.provider} | ${n} | ${pct(row.accuracy)} | ${pct(row.precision)} | ${pct(row.recall)} | ${pct(row.f1)} | ${pct(row.falsePositiveRate)} | ${ms(row.p50Ms)} | ${ms(row.p95Ms)} | ${cost} | ${fallbacks(row)} |`;
+}
+/**
+ * Render the Markdown fragment.
+ *
+ * @param results - Benchmark results.
+ * @param opts - The re-run command to print (default: built from the config).
+ * @returns Markdown (starts at a level-2 heading so it embeds anywhere).
+ */
+export function renderBenchReport(results, opts = {}) {
+    const providers = results.config.providers.map((p) => p.name);
+    const rerun = opts.rerunCommand ??
+        `cleo decide bench --profiles ${providers.join(',') || '<a,b>'} --sites ${results.config.sites.join(',')} --runs ${results.config.runs} --max-usd ${results.config.maxMicros / 1e6}`;
+    const lines = [];
+    lines.push('## System One accuracy benchmark');
+    lines.push('');
+    lines.push(`Measured ${results.finishedAt.slice(0, 10)} with \`cleo decide bench\` (T12495): ${providers.join(' vs ')} against each site's heuristic, ${results.config.runs} run(s).`);
+    lines.push('');
+    lines.push('### Method');
+    lines.push('');
+    lines.push("- **Control group: CLEO's own history.** Duplicate pairs come from `duplicates` relations, tasks cancelled as a duplicate of a named task and duplicate notes; distinct pairs are same-parent tasks with no relation. Observation types are those a caller set explicitly (the stored type differs from both keyword defaults). Contradiction positives are supersedes edges; negatives are random pairs with no supersedes edge.");
+    lines.push('- Pair rows whose text gives the label away (one side names the other\'s id, or says "duplicate of T123" / "supersedes D12") are dropped.');
+    lines.push('- The dataset and the owner spot-check sample (`dataset.jsonl`, `spot-check.json`) stay local and are never published: they hold redacted project text. Only this aggregate report is shared.');
+    lines.push('- Every row is the exact question the site asks (the site request builders), redacted with the same patterns System One uses, sent through `decideBatch` with the provider cache off (`cache: false`).');
+    lines.push(`- Metrics are over rows the provider answered; fallbacks (timeouts, errors, malformed answers) are counted separately. Latency is the wall time of the batch call that carried a row (batch size ${results.config.batchSize}). Cost is the provider-reported \`meta.cost_micros\` (the estimate when not reported).`);
+    lines.push('- Two-label sites report precision, recall, F1 and false-positive rate for the positive class (duplicate, conflict); the observation-type site reports macro averages. Values are the mean ± standard deviation across runs.');
+    lines.push('');
+    lines.push('### Dataset');
+    lines.push('');
+    lines.push('| Site | Rows | Labels |');
+    lines.push('|---|---|---|');
+    for (const [site, labels] of Object.entries(results.dataset.bySite)) {
+        const total = Object.values(labels).reduce((s, n) => s + n, 0);
+        const detail = Object.entries(labels)
+            .map(([l, n]) => `${l} ${n}`)
+            .join(', ');
+        lines.push(`| ${site} | ${total} | ${detail} |`);
+    }
+    lines.push('');
+    lines.push(`${results.dataset.total} rows, ${results.dataset.ownerVerified} verified by the owner in the spot-check.`);
+    lines.push('');
+    lines.push('### Results');
+    for (const site of results.config.sites) {
+        lines.push('');
+        lines.push(`#### ${SITE_TITLES[site] ?? site}`);
+        lines.push('');
+        lines.push('| Provider | n | Accuracy | Precision | Recall | F1 | FP rate | p50 ms | p95 ms | Cost/run | Fallbacks |');
+        lines.push('|---|---|---|---|---|---|---|---|---|---|---|');
+        for (const row of results.aggregate.filter((r) => r.site === site))
+            lines.push(tableRow(row));
+        if (site === 'observationType') {
+            lines.push('');
+            lines.push('> The keyword heuristic scores near zero here by construction: a row is only used when its type differs from the keyword default. Compare the providers with each other on this site.');
+        }
+    }
+    lines.push('');
+    lines.push('### Cost');
+    lines.push('');
+    lines.push(`Spent ${usd(results.spend.spentMicros)} (${usd(results.spend.reportedMicros)} reported by the providers) of a ${usd(results.spend.capMicros)} cap, in ${results.spend.batchesSent} of ${results.spend.batchesPlanned} planned batch calls.`);
+    if (results.spend.capReached && results.spend.stoppedAt) {
+        const s = results.spend.stoppedAt;
+        lines.push('');
+        lines.push(`**Stopped at the cap** in run ${s.run}, site ${s.site}, batch ${s.batch} (${s.provider}): the next batch's estimate (${usd(s.estimateMicros)}) would have passed it. The tables cover what was completed.`);
+    }
+    lines.push('');
+    lines.push('### Re-run');
+    lines.push('');
+    lines.push('```bash');
+    lines.push('cleo decide bench --sample-only        # build the dataset and the spot-check sample, no spend');
+    lines.push('cleo decide bench --corrections <file> --sample-only   # apply owner corrections');
+    lines.push(rerun);
+    lines.push('```');
+    lines.push('');
+    return lines.join('\n');
+}
+//# sourceMappingURL=report.js.map

package/dist/decide/bench/report.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"report.js","sourceRoot":"","sources":["../../../src/decide/bench/report.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAGH,OAAO,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAGjD,wBAAwB;AACxB,MAAM,WAAW,GAAqC;IACpD,kBAAkB,EAAE,wDAAwD;IAC5E,eAAe,EAAE,8CAA8C;IAC/D,qBAAqB,EAAE,0DAA0D;CAClF,CAAC;AAEF,yEAAyE;AACzE,SAAS,GAAG,CAAC,CAAc;IACzB,IAAI,CAAC,CAAC,IAAI,KAAK,IAAI;QAAE,OAAO,GAAG,CAAC;IAChC,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IACvC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,OAAO,CAAC,CAAC,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,IAAI,GAAG,CAAC;AACjG,CAAC;AAED,mCAAmC;AACnC,SAAS,EAAE,CAAC,CAAc;IACxB,IAAI,CAAC,CAAC,IAAI,KAAK,IAAI;QAAE,OAAO,GAAG,CAAC;IAChC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,MAAM,KAAK,IAAI;QACjC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE;QACjD,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAC1C,CAAC;AAED,uCAAuC;AACvC,SAAS,GAAG,CAAC,MAAc;IACzB,OAAO,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;AAC/D,CAAC;AAED,sCAAsC;AACtC,SAAS,SAAS,CAAC,GAAsB;IACvC,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACzE,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC;AACnD,CAAC;AAED,qBAAqB;AACrB,SAAS,QAAQ,CAAC,GAAsB;IACtC,MAAM,CAAC,GAAG,GAAG,CAAC,QAAQ,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IAC1E,MAAM,IAAI,GAAG,GAAG,CAAC,QAAQ,KAAK,kBAAkB,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC;IACxF,OAAO,KAAK,GAAG,CAAC,QAAQ,MAAM,CAAC,MAAM,GAAG,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,GAAG,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,MAAM,GAAG,CAAC,GAAG,CAAC,iBAAiB,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,IAAI,MAAM,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC;AAC/N,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,iBAAiB,CAC/B,OAAqB,EACrB,OAA2C,EAAE;IAE7C,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAC9D,MAAM,KAAK,GACT,IAAI,CAAC,YAAY;QACjB,gCAAgC,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,OAAO,YAAY,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,WAAW,OAAO,CAAC,MAAM,CAAC,IAAI,cAAc,OAAO,CAAC,MAAM,CAAC,SAAS,GAAG,GAAG,EAAE,CAAC;IACvL,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;IAC/C,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CACR,YAAY,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,yCAAyC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,mCAAmC,OAAO,CAAC,MAAM,CAAC,IAAI,UAAU,CAC3K,CAAC;IACF,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IACzB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CACR,maAAma,CACpa,CAAC;IACF,KAAK,CAAC,IAAI,CACR,2IAA2I,CAC5I,CAAC;IACF,KAAK,CAAC,IAAI,CACR,+LAA+L,CAChM,CAAC;IACF,KAAK,CAAC,IAAI,CACR,wMAAwM,CACzM,CAAC;IACF,KAAK,CAAC,IAAI,CACR,oMAAoM,OAAO,CAAC,MAAM,CAAC,SAAS,yFAAyF,CACtT,CAAC;IACF,KAAK,CAAC,IAAI,CACR,8NAA8N,CAC/N,CAAC;IACF,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAC1B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;IACvC,KAAK,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;IAC5B,KAAK,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QACpE,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;QAC/D,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC;aAClC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;aAC5B,IAAI,CAAC,IAAI,CAAC,CAAC;QACd,KAAK,CAAC,IAAI,CAAC,KAAK,IAAI,MAAM,KAAK,MAAM,MAAM,IAAI,CAAC,CAAC;IACnD,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CACR,GAAG,OAAO,CAAC,OAAO,CAAC,KAAK,UAAU,OAAO,CAAC,OAAO,CAAC,aAAa,2CAA2C,CAC3G,CAAC;IACF,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAC1B,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QACxC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,QAAQ,WAAW,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;QAChD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CACR,0GAA0G,CAC3G,CAAC;QACF,KAAK,CAAC,IAAI,CAAC,+CAA+C,CAAC,CAAC;QAC5D,KAAK,MAAM,GAAG,IAAI,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC;YAAE,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC;QAC9F,IAAI,IAAI,KAAK,iBAAiB,EAAE,CAAC;YAC/B,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACf,KAAK,CAAC,IAAI,CACR,uLAAuL,CACxL,CAAC;QACJ,CAAC;IACH,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IACvB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CACR,SAAS,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,KAAK,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,cAAc,CAAC,oCAAoC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,YAAY,OAAO,CAAC,KAAK,CAAC,WAAW,OAAO,OAAO,CAAC,KAAK,CAAC,cAAc,uBAAuB,CAC7O,CAAC;IACF,IAAI,OAAO,CAAC,KAAK,CAAC,UAAU,IAAI,OAAO,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;QACxD,MAAM,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC;QAClC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CACR,iCAAiC,CAAC,CAAC,GAAG,UAAU,CAAC,CAAC,IAAI,WAAW,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,QAAQ,iCAAiC,GAAG,CAAC,CAAC,CAAC,cAAc,CAAC,8DAA8D,CAC5M,CAAC;IACJ,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IACzB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACtB,KAAK,CAAC,IAAI,CACR,gGAAgG,CACjG,CAAC;IACF,KAAK,CAAC,IAAI,CAAC,kFAAkF,CAAC,CAAC;IAC/F,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAClB,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAClB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC"}

package/dist/decide/bench/runner.d.ts

@@ -0,0 +1,203 @@
+/**
+ * The System One benchmark runner (T12495): replay every labelled row through
+ * every compared provider with `decideBatch`, next to each site's heuristic,
+ * and score the answers.
+ *
+ * - Provider answer cache off (`cache: false` on every request) and the
+ *   in-process outcome cache off, so every row is a real provider call.
+ * - One batch deadline of at least {@link MIN_BENCH_TIMEOUT_MS} (spec §1: the
+ *   batch endpoint answers its items serially).
+ * - A hard total spend cap across all providers and runs, SEPARATE from the
+ *   everyday monthly cap: before each batch the spend estimate (per
+ *   question, the larger of {@link DECISION_COST_ESTIMATE_MICROS_PER_QUESTION}
+ *   and the connection's observed mean reported cost) is added to what was
+ *   spent so far, and the run stops cleanly when that would pass the cap.
+ *   Billed answers the client rejected (`invalid_response`) count at their
+ *   reported cost (else the estimate), so a model that keeps answering in
+ *   the wrong shape is still capped. The monthly cap does not gate the benchmark
+ *   (`spend: null` on the calls), but every batch's spend is still recorded
+ *   in the monthly ledger so month-to-date spend stays true.
+ * - Providers take turns batch by batch, so a cap stop leaves them with the
+ *   same coverage.
+ *
+ * @task T12495
+ * @epic T12486
+ */
+import { type SpendLedger } from '../spend.js';
+import type { DecideFetch } from '../transport.js';
+import { type BenchDatasetSizes } from './dataset.js';
+import { type BenchClassificationMetrics, type BenchLatency } from './metrics.js';
+import { type BenchSpread } from './stats.js';
+import { type BenchConnection, type BenchRow, type BenchSite } from './types.js';
+/** Default hard spend cap across providers and runs: $5. */
+export declare const DEFAULT_BENCH_MAX_MICROS = 5000000;
+/** Minimum (and default) batch deadline, ms. */
+export declare const MIN_BENCH_TIMEOUT_MS = 30000;
+/** Default rows per batch call. */
+export declare const DEFAULT_BENCH_BATCH_SIZE = 16;
+/** Largest batch the layahost batch endpoint accepts (64 requests). */
+export declare const MAX_BENCH_BATCH_SIZE = 64;
+/** Name of the heuristic pseudo-provider in results. */
+export declare const HEURISTIC_PROVIDER = "heuristic";
+/** Options for {@link runDecideBench}. */
+export interface RunDecideBenchOptions {
+    /** Labelled rows (already filtered to the sites to run). */
+    readonly rows: readonly BenchRow[];
+    /** Providers to compare. */
+    readonly connections: readonly BenchConnection[];
+    /** Repeated runs. Default 1. */
+    readonly runs?: number;
+    /** Hard total spend cap in micro-dollars. Default {@link DEFAULT_BENCH_MAX_MICROS}. */
+    readonly maxMicros?: number;
+    /** Rows per batch call, 1–{@link MAX_BENCH_BATCH_SIZE}. Default {@link DEFAULT_BENCH_BATCH_SIZE}. */
+    readonly batchSize?: number;
+    /** Batch deadline, ms; raised to at least {@link MIN_BENCH_TIMEOUT_MS}. */
+    readonly timeoutMs?: number;
+    /** Monthly ledger the actual spend is recorded in; `null` skips it. Default: the file ledger. */
+    readonly spendLedger?: SpendLedger | null;
+    /** Transport (tests). Default: the decide transport. */
+    readonly fetch?: DecideFetch;
+    /**
+     * Directory for per-connection provider-state files (detected
+     * capabilities). Default `<cleoHome>/decide/bench/`. Each compared
+     * connection gets its own file: the everyday state file holds ONE identity,
+     * so two connections sharing it would overwrite each other's detection and
+     * one would fall back to the Jev minimum (no batch, no `cache: false`).
+     */
+    readonly providerStateDir?: string;
+    /** Clock (tests). */
+    readonly now?: () => Date;
+}
+/** One provider's (or the heuristic's) result on one site in one run. */
+export interface BenchSiteResult extends BenchClassificationMetrics {
+    /** Site. */
+    readonly site: BenchSite;
+    /** Rows sent (or scored, for the heuristic). */
+    readonly attempted: number;
+    /** Rows the provider answered usably (the metrics' `n`). */
+    readonly answered: number;
+    /** Latency of provider-answered rows: the wall time of the batch call that carried each row. */
+    readonly latency: BenchLatency;
+    /** Spend accounted to this site, micro-dollars (reported cost, else the estimate). */
+    readonly costMicros: number;
+    /** Whether the provider reported its cost (`meta.cost_micros` / `cost_usd`). */
+    readonly costReported: boolean;
+    /** Fallback reason → rows (timeouts, 4xx/5xx, malformed answers, …). */
+    readonly fallbacks: Readonly<Record<string, number>>;
+    /** Rows the provider answered with a value outside the site's labels. */
+    readonly unusable: number;
+}
+/** One provider in one run. */
+export interface BenchProviderRun {
+    /** Profile name. */
+    readonly provider: string;
+    /** Per-site results, in {@link BENCH_SITES} order. */
+    readonly sites: readonly BenchSiteResult[];
+}
+/** One repeated run. */
+export interface BenchRunRecord {
+    /** 1-based run number. */
+    readonly run: number;
+    /** Whether every batch of the run was sent (false when the cap stopped it). */
+    readonly complete: boolean;
+    /** Per provider. */
+    readonly providers: readonly BenchProviderRun[];
+}
+/** Mean and spread of one provider × site across runs. */
+export interface BenchAggregateRow {
+    /** Profile name (or {@link HEURISTIC_PROVIDER}). */
+    readonly provider: string;
+    /** Site. */
+    readonly site: BenchSite;
+    /** Rows answered, per run. */
+    readonly answered: BenchSpread;
+    /** Accuracy. */
+    readonly accuracy: BenchSpread;
+    /** Precision. */
+    readonly precision: BenchSpread;
+    /** Recall. */
+    readonly recall: BenchSpread;
+    /** F1. */
+    readonly f1: BenchSpread;
+    /** False-positive rate. */
+    readonly falsePositiveRate: BenchSpread;
+    /** p50 latency, ms. */
+    readonly p50Ms: BenchSpread;
+    /** p95 latency, ms. */
+    readonly p95Ms: BenchSpread;
+    /** Cost per run, micro-dollars. */
+    readonly costMicros: BenchSpread;
+    /** Fallbacks summed over runs, by reason. */
+    readonly fallbacks: Readonly<Record<string, number>>;
+}
+/** Spend accounting of a benchmark. */
+export interface BenchSpendSummary {
+    /** The hard cap, micro-dollars. */
+    readonly capMicros: number;
+    /** Spent (reported cost, else the estimate), micro-dollars. */
+    readonly spentMicros: number;
+    /** Of which the providers reported. */
+    readonly reportedMicros: number;
+    /** Whether the cap stopped the benchmark. */
+    readonly capReached: boolean;
+    /** Where it stopped, when it did. */
+    readonly stoppedAt?: {
+        readonly run: number;
+        readonly site: BenchSite;
+        readonly provider: string;
+        readonly batch: number;
+        readonly estimateMicros: number;
+    };
+    /** Batch calls made. */
+    readonly batchesSent: number;
+    /** Batch calls planned (runs × sites × batches × providers). */
+    readonly batchesPlanned: number;
+    /** Whether the monthly ledger accepted the spend record (false: it could not be written). */
+    readonly recordedInMonthlyLedger: boolean;
+}
+/** The machine-readable benchmark result. */
+export interface BenchResults {
+    /** Format version. */
+    readonly schemaVersion: 1;
+    /** Owning task. */
+    readonly task: 'T12495';
+    /** ISO start time. */
+    readonly startedAt: string;
+    /** ISO end time. */
+    readonly finishedAt: string;
+    /** Run configuration; never contains an API key. */
+    readonly config: {
+        readonly runs: number;
+        readonly batchSize: number;
+        readonly timeoutMs: number;
+        readonly maxMicros: number;
+        readonly sites: readonly BenchSite[];
+        readonly providers: readonly {
+            readonly name: string;
+            readonly provider: string;
+            readonly baseUrl: string;
+            readonly model?: string;
+        }[];
+    };
+    /** Dataset sizes. */
+    readonly dataset: BenchDatasetSizes;
+    /** The heuristic baseline (deterministic, scored once). */
+    readonly heuristic: readonly BenchSiteResult[];
+    /** Every run. */
+    readonly runs: readonly BenchRunRecord[];
+    /** Mean and spread per provider × site (heuristic included). */
+    readonly aggregate: readonly BenchAggregateRow[];
+    /** Spend. */
+    readonly spend: BenchSpendSummary;
+}
+/**
+ * Run the benchmark.
+ *
+ * Never throws because of a provider: failures become per-row fallbacks with
+ * their reason. Stops cleanly at the spend cap and reports what it completed.
+ *
+ * @param opts - Rows, connections, runs, cap and wiring.
+ * @returns The results.
+ */
+export declare function runDecideBench(opts: RunDecideBenchOptions): Promise<BenchResults>;
+//# sourceMappingURL=runner.d.ts.map

package/dist/decide/bench/runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../../src/decide/bench/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAcH,OAAO,EAAyB,KAAK,WAAW,EAAE,MAAM,aAAa,CAAC;AACtE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AACnD,OAAO,EAAE,KAAK,iBAAiB,EAAqB,MAAM,cAAc,CAAC;AACzE,OAAO,EACL,KAAK,0BAA0B,EAC/B,KAAK,YAAY,EAIlB,MAAM,cAAc,CAAC;AAEtB,OAAO,EAAE,KAAK,WAAW,EAAY,MAAM,YAAY,CAAC;AACxD,OAAO,EAEL,KAAK,eAAe,EACpB,KAAK,QAAQ,EACb,KAAK,SAAS,EAEf,MAAM,YAAY,CAAC;AAEpB,4DAA4D;AAC5D,eAAO,MAAM,wBAAwB,UAAY,CAAC;AAElD,gDAAgD;AAChD,eAAO,MAAM,oBAAoB,QAAS,CAAC;AAE3C,mCAAmC;AACnC,eAAO,MAAM,wBAAwB,KAAK,CAAC;AAE3C,uEAAuE;AACvE,eAAO,MAAM,oBAAoB,KAAK,CAAC;AAEvC,wDAAwD;AACxD,eAAO,MAAM,kBAAkB,cAAc,CAAC;AAE9C,0CAA0C;AAC1C,MAAM,WAAW,qBAAqB;IACpC,4DAA4D;IAC5D,QAAQ,CAAC,IAAI,EAAE,SAAS,QAAQ,EAAE,CAAC;IACnC,4BAA4B;IAC5B,QAAQ,CAAC,WAAW,EAAE,SAAS,eAAe,EAAE,CAAC;IACjD,gCAAgC;IAChC,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,uFAAuF;IACvF,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,qGAAqG;IACrG,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,2EAA2E;IAC3E,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,iGAAiG;IACjG,QAAQ,CAAC,WAAW,CAAC,EAAE,WAAW,GAAG,IAAI,CAAC;IAC1C,wDAAwD;IACxD,QAAQ,CAAC,KAAK,CAAC,EAAE,WAAW,CAAC;IAC7B;;;;;;OAMG;IACH,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IACnC,qBAAqB;IACrB,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;CAC3B;AAED,yEAAyE;AACzE,MAAM,WAAW,eAAgB,SAAQ,0BAA0B;IACjE,YAAY;IACZ,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;IACzB,gDAAgD;IAChD,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,4DAA4D;IAC5D,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,gGAAgG;IAChG,QAAQ,CAAC,OAAO,EAAE,YAAY,CAAC;IAC/B,sFAAsF;IACtF,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,gFAAgF;IAChF,QAAQ,CAAC,YAAY,EAAE,OAAO,CAAC;IAC/B,wEAAwE;IACxE,QAAQ,CAAC,SAAS,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IACrD,yEAAyE;IACzE,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;CAC3B;AAED,+BAA+B;AAC/B,MAAM,WAAW,gBAAgB;IAC/B,oBAAoB;IACpB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,sDAAsD;IACtD,QAAQ,CAAC,KAAK,EAAE,SAAS,eAAe,EAAE,CAAC;CAC5C;AAED,wBAAwB;AACxB,MAAM,WAAW,cAAc;IAC7B,0BAA0B;IAC1B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,+EAA+E;IAC/E,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC;IAC3B,oBAAoB;IACpB,QAAQ,CAAC,SAAS,EAAE,SAAS,gBAAgB,EAAE,CAAC;CACjD;AAED,0DAA0D;AAC1D,MAAM,WAAW,iBAAiB;IAChC,oDAAoD;IACpD,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,YAAY;IACZ,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;IACzB,8BAA8B;IAC9B,QAAQ,CAAC,QAAQ,EAAE,WAAW,CAAC;IAC/B,gBAAgB;IAChB,QAAQ,CAAC,QAAQ,EAAE,WAAW,CAAC;IAC/B,iBAAiB;IACjB,QAAQ,CAAC,SAAS,EAAE,WAAW,CAAC;IAChC,cAAc;IACd,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC;IAC7B,UAAU;IACV,QAAQ,CAAC,EAAE,EAAE,WAAW,CAAC;IACzB,2BAA2B;IAC3B,QAAQ,CAAC,iBAAiB,EAAE,WAAW,CAAC;IACxC,uBAAuB;IACvB,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC;IAC5B,uBAAuB;IACvB,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC;IAC5B,mCAAmC;IACnC,QAAQ,CAAC,UAAU,EAAE,WAAW,CAAC;IACjC,6CAA6C;IAC7C,QAAQ,CAAC,SAAS,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;CACtD;AAED,uCAAuC;AACvC,MAAM,WAAW,iBAAiB;IAChC,mCAAmC;IACnC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,+DAA+D;IAC/D,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,uCAAuC;IACvC,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,6CAA6C;IAC7C,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC;IAC7B,qCAAqC;IACrC,QAAQ,CAAC,SAAS,CAAC,EAAE;QACnB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;QACrB,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;QACzB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;QAC1B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;QACvB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;KACjC,CAAC;IACF,wBAAwB;IACxB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,gEAAgE;IAChE,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,6FAA6F;IAC7F,QAAQ,CAAC,uBAAuB,EAAE,OAAO,CAAC;CAC3C;AAED,6CAA6C;AAC7C,MAAM,WAAW,YAAY;IAC3B,sBAAsB;IACtB,QAAQ,CAAC,aAAa,EAAE,CAAC,CAAC;IAC1B,mBAAmB;IACnB,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IACxB,sBAAsB;IACtB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,oBAAoB;IACpB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,oDAAoD;IACpD,QAAQ,CAAC,MAAM,EAAE;QACf,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QACtB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;QAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;QAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;QAC3B,QAAQ,CAAC,KAAK,EAAE,SAAS,SAAS,EAAE,CAAC;QACrC,QAAQ,CAAC,SAAS,EAAE,SAAS;YAC3B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;YACtB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;YAC1B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;YACzB,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;SACzB,EAAE,CAAC;KACL,CAAC;IACF,qBAAqB;IACrB,QAAQ,CAAC,OAAO,EAAE,iBAAiB,CAAC;IACpC,2DAA2D;IAC3D,QAAQ,CAAC,SAAS,EAAE,SAAS,eAAe,EAAE,CAAC;IAC/C,iBAAiB;IACjB,QAAQ,CAAC,IAAI,EAAE,SAAS,cAAc,EAAE,CAAC;IACzC,gEAAgE;IAChE,QAAQ,CAAC,SAAS,EAAE,SAAS,iBAAiB,EAAE,CAAC;IACjD,aAAa;IACb,QAAQ,CAAC,KAAK,EAAE,iBAAiB,CAAC;CACnC;AAoKD;;;;;;;;GAQG;AACH,wBAAsB,cAAc,CAAC,IAAI,EAAE,qBAAqB,GAAG,OAAO,CAAC,YAAY,CAAC,CA8KvF"}