@cleocode/core 2026.9.23 → 2026.9.24

package/dist/decide/bench/runner.js

@@ -0,0 +1,335 @@
+/**
+ * The System One benchmark runner (T12495): replay every labelled row through
+ * every compared provider with `decideBatch`, next to each site's heuristic,
+ * and score the answers.
+ *
+ * - Provider answer cache off (`cache: false` on every request) and the
+ *   in-process outcome cache off, so every row is a real provider call.
+ * - One batch deadline of at least {@link MIN_BENCH_TIMEOUT_MS} (spec §1: the
+ *   batch endpoint answers its items serially).
+ * - A hard total spend cap across all providers and runs, SEPARATE from the
+ *   everyday monthly cap: before each batch the spend estimate (per
+ *   question, the larger of {@link DECISION_COST_ESTIMATE_MICROS_PER_QUESTION}
+ *   and the connection's observed mean reported cost) is added to what was
+ *   spent so far, and the run stops cleanly when that would pass the cap.
+ *   Billed answers the client rejected (`invalid_response`) count at their
+ *   reported cost (else the estimate), so a model that keeps answering in
+ *   the wrong shape is still capped. The monthly cap does not gate the benchmark
+ *   (`spend: null` on the calls), but every batch's spend is still recorded
+ *   in the monthly ledger so month-to-date spend stays true.
+ * - Providers take turns batch by batch, so a cap stop leaves them with the
+ *   same coverage.
+ *
+ * @task T12495
+ * @epic T12486
+ */
+import { join } from 'node:path';
+import { getCleoHome } from '@cleocode/paths';
+import { hashCanonical } from '../cache.js';
+import { DECISION_COST_ESTIMATE_MICROS_PER_QUESTION, decideBatch, redactDecisionState, } from '../client.js';
+import { DECIDE_BENCH_DECISION_SITE } from '../sites/registry.js';
+import { createFileSpendLedger } from '../spend.js';
+import { benchDatasetSizes } from './dataset.js';
+import { classificationMetrics, latencyOf, } from './metrics.js';
+import { benchQuestionFor } from './questions.js';
+import { spreadOf } from './stats.js';
+import { BENCH_SITES, positiveLabel, } from './types.js';
+/** Default hard spend cap across providers and runs: $5. */
+export const DEFAULT_BENCH_MAX_MICROS = 5_000_000;
+/** Minimum (and default) batch deadline, ms. */
+export const MIN_BENCH_TIMEOUT_MS = 30_000;
+/** Default rows per batch call. */
+export const DEFAULT_BENCH_BATCH_SIZE = 16;
+/** Largest batch the layahost batch endpoint accepts (64 requests). */
+export const MAX_BENCH_BATCH_SIZE = 64;
+/** Name of the heuristic pseudo-provider in results. */
+export const HEURISTIC_PROVIDER = 'heuristic';
+function newCollector() {
+    return {
+        attempted: 0,
+        predictions: [],
+        latencies: [],
+        costMicros: 0,
+        costReported: false,
+        fallbacks: {},
+        unusable: 0,
+    };
+}
+function finishCollector(site, c) {
+    return {
+        site,
+        ...classificationMetrics(c.predictions, positiveLabel(site)),
+        attempted: c.attempted,
+        answered: c.predictions.length,
+        latency: latencyOf(c.latencies),
+        costMicros: c.costMicros,
+        costReported: c.costReported,
+        fallbacks: c.fallbacks,
+        unusable: c.unusable,
+    };
+}
+/**
+ * Estimated cost per question for the pre-batch cap check: the client's
+ * reservation estimate, or the connection's observed mean when a pricier
+ * model has reported more, so one batch cannot overshoot the cap by much.
+ */
+function perQuestionEstimate(seen) {
+    const mean = seen && seen.questions > 0 ? Math.ceil(seen.micros / seen.questions) : 0;
+    return Math.max(DECISION_COST_ESTIMATE_MICROS_PER_QUESTION, mean);
+}
+/** Provider-reported cost of an outcome in micros (`costMicros`, else `costUsd`). */
+function reportedCostMicros(outcome) {
+    if (outcome.costMicros !== undefined)
+        return outcome.costMicros;
+    return outcome.costUsd !== undefined ? Math.round(outcome.costUsd * 1e6) : undefined;
+}
+/** A profile name made safe for a file name. */
+function safeFileName(name) {
+    return name.replace(/[^A-Za-z0-9._-]/g, '_').slice(0, 64) || 'profile';
+}
+/** Split `items` into chunks of `size`. */
+function chunks(items, size) {
+    const out = [];
+    for (let i = 0; i < items.length; i += size)
+        out.push(items.slice(i, i + size));
+    return out;
+}
+/** Audit key of a request, matching what `decide()` writes for it. */
+function auditKey(q) {
+    return `${hashCanonical(q.req.questions)}:${hashCanonical(redactDecisionState(q.req.state))}`;
+}
+/** An in-memory audit sink that keeps fallback reasons by request key. */
+function reasonSink() {
+    const reasons = new Map();
+    return {
+        sink: {
+            write(entry) {
+                if (entry.source !== 'fallback')
+                    return;
+                const key = `${entry.questionsHash}:${entry.stateHash}`;
+                reasons.set(key, [...(reasons.get(key) ?? []), entry.fallbackReason ?? 'unknown']);
+            },
+        },
+        take: (key) => {
+            const list = reasons.get(key);
+            return list?.shift();
+        },
+    };
+}
+/** Score the heuristic on every row (deterministic, no provider). */
+function scoreHeuristic(sites, questions) {
+    return sites.map((site) => {
+        const c = newCollector();
+        for (const { row, q } of questions.get(site) ?? []) {
+            const started = performance.now();
+            const label = q.predict(q.heuristicAnswers) ?? q.heuristicLabel;
+            c.latencies.push(Math.max(0, performance.now() - started));
+            c.attempted++;
+            c.predictions.push({ gold: row.label, predicted: label });
+        }
+        return finishCollector(site, c);
+    });
+}
+/** Mean and spread per provider × site. */
+function aggregate(heuristic, runs, providers, sites) {
+    const rows = [];
+    const build = (provider, site, results) => {
+        const fallbacks = {};
+        for (const r of results) {
+            for (const [k, v] of Object.entries(r.fallbacks))
+                fallbacks[k] = (fallbacks[k] ?? 0) + v;
+        }
+        const scored = results.filter((r) => r.answered > 0);
+        rows.push({
+            provider,
+            site,
+            answered: spreadOf(results.map((r) => r.answered)),
+            accuracy: spreadOf(scored.map((r) => r.accuracy)),
+            precision: spreadOf(scored.map((r) => r.precision)),
+            recall: spreadOf(scored.map((r) => r.recall)),
+            f1: spreadOf(scored.map((r) => r.f1)),
+            falsePositiveRate: spreadOf(scored.map((r) => r.falsePositiveRate)),
+            p50Ms: spreadOf(scored.map((r) => r.latency.p50Ms)),
+            p95Ms: spreadOf(scored.map((r) => r.latency.p95Ms)),
+            costMicros: spreadOf(results.map((r) => r.costMicros)),
+            fallbacks,
+        });
+    };
+    for (const site of sites) {
+        build(HEURISTIC_PROVIDER, site, heuristic.filter((h) => h.site === site));
+        for (const provider of providers) {
+            const results = runs.flatMap((run) => run.providers
+                .filter((p) => p.provider === provider)
+                .flatMap((p) => p.sites.filter((s) => s.site === site && s.attempted > 0)));
+            build(provider, site, results);
+        }
+    }
+    return rows;
+}
+/**
+ * Run the benchmark.
+ *
+ * Never throws because of a provider: failures become per-row fallbacks with
+ * their reason. Stops cleanly at the spend cap and reports what it completed.
+ *
+ * @param opts - Rows, connections, runs, cap and wiring.
+ * @returns The results.
+ */
+export async function runDecideBench(opts) {
+    const now = opts.now ?? (() => new Date());
+    const startedAt = now().toISOString();
+    const runs = Math.max(1, Math.floor(opts.runs ?? 1));
+    const capMicros = Math.max(0, opts.maxMicros ?? DEFAULT_BENCH_MAX_MICROS);
+    const batchSize = Math.min(MAX_BENCH_BATCH_SIZE, Math.max(1, Math.floor(opts.batchSize ?? DEFAULT_BENCH_BATCH_SIZE)));
+    const timeoutMs = Math.max(MIN_BENCH_TIMEOUT_MS, opts.timeoutMs ?? MIN_BENCH_TIMEOUT_MS);
+    const ledger = opts.spendLedger === undefined ? createFileSpendLedger() : opts.spendLedger;
+    const stateDir = opts.providerStateDir ?? join(getCleoHome(), 'decide', 'bench');
+    const sites = BENCH_SITES.filter((s) => opts.rows.some((r) => r.site === s));
+    const questions = new Map();
+    for (const row of opts.rows) {
+        questions.set(row.site, [
+            ...(questions.get(row.site) ?? []),
+            { row, q: benchQuestionFor(row) },
+        ]);
+    }
+    const heuristic = scoreHeuristic(sites, questions);
+    const batchesPerRun = sites.reduce((n, s) => n + Math.ceil((questions.get(s)?.length ?? 0) / batchSize), 0);
+    let spentMicros = 0;
+    let reportedMicros = 0;
+    let batchesSent = 0;
+    let recordedInMonthlyLedger = true;
+    let stoppedAt;
+    const runRecords = [];
+    /** Reported cost per connection so far, for the pre-batch estimate. */
+    const observed = new Map();
+    runLoop: for (let run = 1; run <= runs; run++) {
+        const collectors = new Map();
+        for (const c of opts.connections) {
+            collectors.set(c.name, new Map(sites.map((s) => [s, newCollector()])));
+        }
+        const record = (complete) => ({
+            run,
+            complete,
+            providers: opts.connections.map((c) => ({
+                provider: c.name,
+                sites: sites.map((s) => finishCollector(s, collectors.get(c.name)?.get(s) ?? newCollector())),
+            })),
+        });
+        for (const site of sites) {
+            for (const [b, batch] of chunks(questions.get(site) ?? [], batchSize).entries()) {
+                for (const conn of opts.connections) {
+                    const questionCount = batch.reduce((n, x) => n + Object.keys(x.q.req.questions).length, 0);
+                    const estimate = questionCount * perQuestionEstimate(observed.get(conn.name));
+                    if (spentMicros + estimate > capMicros) {
+                        stoppedAt = { run, site, provider: conn.name, batch: b + 1, estimateMicros: estimate };
+                        runRecords.push(record(false));
+                        break runLoop;
+                    }
+                    const reasons = reasonSink();
+                    const entries = batch.map(({ q }) => ({
+                        req: q.req,
+                        fallback: () => q.heuristicAnswers,
+                    }));
+                    const outcomes = await decideBatch(DECIDE_BENCH_DECISION_SITE.id, entries, {
+                        connection: {
+                            baseUrl: conn.baseUrl,
+                            apiKey: conn.apiKey,
+                            ...(conn.model ? { model: conn.model } : {}),
+                        },
+                        timeoutMs,
+                        cache: null,
+                        budget: null,
+                        spend: null,
+                        audit: reasons.sink,
+                        ...(opts.fetch ? { fetch: opts.fetch } : {}),
+                        providerStatePath: join(stateDir, `provider-state-${safeFileName(conn.name)}.json`),
+                    });
+                    batchesSent++;
+                    const col = collectors.get(conn.name)?.get(site) ?? newCollector();
+                    let batchSpend = 0;
+                    const perQuestion = perQuestionEstimate(observed.get(conn.name));
+                    const seen = observed.get(conn.name) ?? { micros: 0, questions: 0 };
+                    for (const [i, { row, q }] of batch.entries()) {
+                        const outcome = outcomes[i];
+                        col.attempted++;
+                        const questionsHere = Object.keys(q.req.questions).length;
+                        const reported = outcome ? reportedCostMicros(outcome) : undefined;
+                        if (reported !== undefined) {
+                            col.costReported = true;
+                            reportedMicros += reported;
+                            seen.micros += reported;
+                            seen.questions += questionsHere;
+                        }
+                        if (!outcome || outcome.source !== 'provider') {
+                            const reason = reasons.take(auditKey(q)) ?? 'unknown';
+                            col.fallbacks[reason] = (col.fallbacks[reason] ?? 0) + 1;
+                            // Billed without a usable answer: a 2xx the client rejected
+                            // (`invalid_response`, at its reported cost), or a call that
+                            // timed out after send. Other failures are error responses,
+                            // which the provider does not bill (layahost docs).
+                            if (reason === 'invalid_response' || reason === 'timeout') {
+                                batchSpend += reported ?? questionsHere * perQuestion;
+                            }
+                            continue;
+                        }
+                        batchSpend += reported ?? questionsHere * perQuestion;
+                        col.latencies.push(outcome.latencyMs);
+                        const predicted = q.predict(outcome.answers);
+                        if (predicted === null) {
+                            col.unusable++;
+                            continue;
+                        }
+                        col.predictions.push({ gold: row.label, predicted });
+                    }
+                    observed.set(conn.name, seen);
+                    col.costMicros += batchSpend;
+                    spentMicros += batchSpend;
+                    if (ledger && batchSpend > 0) {
+                        try {
+                            await ledger.record(batchSpend);
+                        }
+                        catch {
+                            recordedInMonthlyLedger = false;
+                        }
+                    }
+                }
+            }
+        }
+        runRecords.push(record(true));
+    }
+    const providerNames = opts.connections.map((c) => c.name);
+    return {
+        schemaVersion: 1,
+        task: 'T12495',
+        startedAt,
+        finishedAt: now().toISOString(),
+        config: {
+            runs,
+            batchSize,
+            timeoutMs,
+            maxMicros: capMicros,
+            sites,
+            providers: opts.connections.map((c) => ({
+                name: c.name,
+                provider: c.provider,
+                baseUrl: c.baseUrl,
+                ...(c.model ? { model: c.model } : {}),
+            })),
+        },
+        dataset: benchDatasetSizes(opts.rows),
+        heuristic,
+        runs: runRecords,
+        aggregate: aggregate(heuristic, runRecords, providerNames, sites),
+        spend: {
+            capMicros,
+            spentMicros,
+            reportedMicros,
+            capReached: stoppedAt !== undefined,
+            ...(stoppedAt ? { stoppedAt } : {}),
+            batchesSent,
+            batchesPlanned: batchesPerRun * runs * opts.connections.length,
+            recordedInMonthlyLedger,
+        },
+    };
+}
+//# sourceMappingURL=runner.js.map

package/dist/decide/bench/runner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../../src/decide/bench/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAE9C,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EACL,0CAA0C,EAE1C,WAAW,EACX,mBAAmB,GACpB,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,0BAA0B,EAAE,MAAM,sBAAsB,CAAC;AAClE,OAAO,EAAE,qBAAqB,EAAoB,MAAM,aAAa,CAAC;AAEtE,OAAO,EAA0B,iBAAiB,EAAE,MAAM,cAAc,CAAC;AACzE,OAAO,EAIL,qBAAqB,EACrB,SAAS,GACV,MAAM,cAAc,CAAC;AACtB,OAAO,EAAsB,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AACtE,OAAO,EAAoB,QAAQ,EAAE,MAAM,YAAY,CAAC;AACxD,OAAO,EACL,WAAW,EAIX,aAAa,GACd,MAAM,YAAY,CAAC;AAEpB,4DAA4D;AAC5D,MAAM,CAAC,MAAM,wBAAwB,GAAG,SAAS,CAAC;AAElD,gDAAgD;AAChD,MAAM,CAAC,MAAM,oBAAoB,GAAG,MAAM,CAAC;AAE3C,mCAAmC;AACnC,MAAM,CAAC,MAAM,wBAAwB,GAAG,EAAE,CAAC;AAE3C,uEAAuE;AACvE,MAAM,CAAC,MAAM,oBAAoB,GAAG,EAAE,CAAC;AAEvC,wDAAwD;AACxD,MAAM,CAAC,MAAM,kBAAkB,GAAG,WAAW,CAAC;AA2K9C,SAAS,YAAY;IACnB,OAAO;QACL,SAAS,EAAE,CAAC;QACZ,WAAW,EAAE,EAAE;QACf,SAAS,EAAE,EAAE;QACb,UAAU,EAAE,CAAC;QACb,YAAY,EAAE,KAAK;QACnB,SAAS,EAAE,EAAE;QACb,QAAQ,EAAE,CAAC;KACZ,CAAC;AACJ,CAAC;AAED,SAAS,eAAe,CAAC,IAAe,EAAE,CAAY;IACpD,OAAO;QACL,IAAI;QACJ,GAAG,qBAAqB,CAAC,CAAC,CAAC,WAAW,EAAE,aAAa,CAAC,IAAI,CAAC,CAAC;QAC5D,SAAS,EAAE,CAAC,CAAC,SAAS;QACtB,QAAQ,EAAE,CAAC,CAAC,WAAW,CAAC,MAAM;QAC9B,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC;QAC/B,UAAU,EAAE,CAAC,CAAC,UAAU;QACxB,YAAY,EAAE,CAAC,CAAC,YAAY;QAC5B,SAAS,EAAE,CAAC,CAAC,SAAS;QACtB,QAAQ,EAAE,CAAC,CAAC,QAAQ;KACrB,CAAC;AACJ,CAAC;AAQD;;;;GAIG;AACH,SAAS,mBAAmB,CAAC,IAA8B;IACzD,MAAM,IAAI,GAAG,IAAI,IAAI,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACtF,OAAO,IAAI,CAAC,GAAG,CAAC,0CAA0C,EAAE,IAAI,CAAC,CAAC;AACpE,CAAC;AAED,qFAAqF;AACrF,SAAS,kBAAkB,CAAC,OAAwB;IAClD,IAAI,OAAO,CAAC,UAAU,KAAK,SAAS;QAAE,OAAO,OAAO,CAAC,UAAU,CAAC;IAChE,OAAO,OAAO,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AACvF,CAAC;AAED,gDAAgD;AAChD,SAAS,YAAY,CAAC,IAAY;IAChC,OAAO,IAAI,CAAC,OAAO,CAAC,kBAAkB,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,SAAS,CAAC;AACzE,CAAC;AAED,2CAA2C;AAC3C,SAAS,MAAM,CAAI,KAAmB,EAAE,IAAY;IAClD,MAAM,GAAG,GAAU,EAAE,CAAC;IACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,IAAI;QAAE,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC;IAChF,OAAO,GAAG,CAAC;AACb,CAAC;AAED,sEAAsE;AACtE,SAAS,QAAQ,CAAC,CAAgB;IAChC,OAAO,GAAG,aAAa,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,aAAa,CAAC,mBAAmB,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;AAChG,CAAC;AAED,0EAA0E;AAC1E,SAAS,UAAU;IAIjB,MAAM,OAAO,GAAG,IAAI,GAAG,EAAoB,CAAC;IAC5C,OAAO;QACL,IAAI,EAAE;YACJ,KAAK,CAAC,KAAyB;gBAC7B,IAAI,KAAK,CAAC,MAAM,KAAK,UAAU;oBAAE,OAAO;gBACxC,MAAM,GAAG,GAAG,GAAG,KAAK,CAAC,aAAa,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;gBACxD,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,EAAE,KAAK,CAAC,cAAc,IAAI,SAAS,CAAC,CAAC,CAAC;YACrF,CAAC;SACF;QACD,IAAI,EAAE,CAAC,GAAG,EAAE,EAAE;YACZ,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC9B,OAAO,IAAI,EAAE,KAAK,EAAE,CAAC;QACvB,CAAC;KACF,CAAC;AACJ,CAAC;AAED,qEAAqE;AACrE,SAAS,cAAc,CACrB,KAA2B,EAC3B,SAAiF;IAEjF,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACxB,MAAM,CAAC,GAAG,YAAY,EAAE,CAAC;QACzB,KAAK,MAAM,EAAE,GAAG,EAAE,CAAC,EAAE,IAAI,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;YACnD,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;YAClC,MAAM,KAAK,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,cAAc,CAAC;YAChE,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC;YAC3D,CAAC,CAAC,SAAS,EAAE,CAAC;YACd,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,CAAC,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;QAC5D,CAAC;QACD,OAAO,eAAe,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAClC,CAAC,CAAC,CAAC;AACL,CAAC;AAED,2CAA2C;AAC3C,SAAS,SAAS,CAChB,SAAqC,EACrC,IAA+B,EAC/B,SAA4B,EAC5B,KAA2B;IAE3B,MAAM,IAAI,GAAwB,EAAE,CAAC;IACrC,MAAM,KAAK,GAAG,CAAC,QAAgB,EAAE,IAAe,EAAE,OAAmC,EAAE,EAAE;QACvF,MAAM,SAAS,GAA2B,EAAE,CAAC;QAC7C,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC;gBAAE,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;QAC3F,CAAC;QACD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC;QACrD,IAAI,CAAC,IAAI,CAAC;YACR,QAAQ;YACR,IAAI;YACJ,QAAQ,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;YAClD,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;YACjD,SAAS,EAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;YACnD,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YAC7C,EAAE,EAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACrC,iBAAiB,EAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC;YACnE,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YACnD,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YACnD,UAAU,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;YACtD,SAAS;SACV,CAAC,CAAC;IACL,CAAC,CAAC;IACF,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,KAAK,CACH,kBAAkB,EAClB,IAAI,EACJ,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,CACzC,CAAC;QACF,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YACjC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CACnC,GAAG,CAAC,SAAS;iBACV,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,QAAQ,CAAC;iBACtC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,IAAI,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAC7E,CAAC;YACF,KAAK,CAAC,QAAQ,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;QACjC,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,IAA2B;IAC9D,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;IAC3C,MAAM,SAAS,GAAG,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC;IACtC,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC;IACrD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,SAAS,IAAI,wBAAwB,CAAC,CAAC;IAC1E,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CACxB,oBAAoB,EACpB,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,IAAI,wBAAwB,CAAC,CAAC,CACpE,CAAC;IACF,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,oBAAoB,EAAE,IAAI,CAAC,SAAS,IAAI,oBAAoB,CAAC,CAAC;IACzF,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,qBAAqB,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC;IAC3F,MAAM,QAAQ,GAAG,IAAI,CAAC,gBAAgB,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IAEjF,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC;IAC7E,MAAM,SAAS,GAAG,IAAI,GAAG,EAAoD,CAAC;IAC9E,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;QAC5B,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE;YACtB,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;YAClC,EAAE,GAAG,EAAE,CAAC,EAAE,gBAAgB,CAAC,GAAG,CAAC,EAAE;SAClC,CAAC,CAAC;IACL,CAAC;IACD,MAAM,SAAS,GAAG,cAAc,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;IAEnD,MAAM,aAAa,GAAG,KAAK,CAAC,MAAM,CAChC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,MAAM,IAAI,CAAC,CAAC,GAAG,SAAS,CAAC,EACpE,CAAC,CACF,CAAC;IACF,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,cAAc,GAAG,CAAC,CAAC;IACvB,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,uBAAuB,GAAG,IAAI,CAAC;IACnC,IAAI,SAAyC,CAAC;IAC9C,MAAM,UAAU,GAAqB,EAAE,CAAC;IACxC,uEAAuE;IACvE,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAwB,CAAC;IAEjD,OAAO,EAAE,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,IAAI,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC;QAC9C,MAAM,UAAU,GAAG,IAAI,GAAG,EAAqC,CAAC;QAChE,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACjC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACzE,CAAC;QACD,MAAM,MAAM,GAAG,CAAC,QAAiB,EAAkB,EAAE,CAAC,CAAC;YACrD,GAAG;YACH,QAAQ;YACR,SAAS,EAAE,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBACtC,QAAQ,EAAE,CAAC,CAAC,IAAI;gBAChB,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACrB,eAAe,CAAC,CAAC,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,YAAY,EAAE,CAAC,CACrE;aACF,CAAC,CAAC;SACJ,CAAC,CAAC;QACH,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,KAAK,MAAM,CAAC,CAAC,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,SAAS,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC;gBAChF,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;oBACpC,MAAM,aAAa,GAAG,KAAK,CAAC,MAAM,CAChC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,MAAM,EACnD,CAAC,CACF,CAAC;oBACF,MAAM,QAAQ,GAAG,aAAa,GAAG,mBAAmB,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;oBAC9E,IAAI,WAAW,GAAG,QAAQ,GAAG,SAAS,EAAE,CAAC;wBACvC,SAAS,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,GAAG,CAAC,EAAE,cAAc,EAAE,QAAQ,EAAE,CAAC;wBACvF,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;wBAC/B,MAAM,OAAO,CAAC;oBAChB,CAAC;oBACD,MAAM,OAAO,GAAG,UAAU,EAAE,CAAC;oBAC7B,MAAM,OAAO,GAAyB,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;wBAC1D,GAAG,EAAE,CAAC,CAAC,GAAG;wBACV,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,gBAAgB;qBACnC,CAAC,CAAC,CAAC;oBACJ,MAAM,QAAQ,GAAsB,MAAM,WAAW,CACnD,0BAA0B,CAAC,EAAE,EAC7B,OAAO,EACP;wBACE,UAAU,EAAE;4BACV,OAAO,EAAE,IAAI,CAAC,OAAO;4BACrB,MAAM,EAAE,IAAI,CAAC,MAAM;4BACnB,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;yBAC7C;wBACD,SAAS;wBACT,KAAK,EAAE,IAAI;wBACX,MAAM,EAAE,IAAI;wBACZ,KAAK,EAAE,IAAI;wBACX,KAAK,EAAE,OAAO,CAAC,IAAI;wBACnB,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;wBAC5C,iBAAiB,EAAE,IAAI,CAAC,QAAQ,EAAE,kBAAkB,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC;qBACpF,CACF,CAAC;oBACF,WAAW,EAAE,CAAC;oBACd,MAAM,GAAG,GAAG,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,IAAI,YAAY,EAAE,CAAC;oBACnE,IAAI,UAAU,GAAG,CAAC,CAAC;oBACnB,MAAM,WAAW,GAAG,mBAAmB,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;oBACjE,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;oBACpE,KAAK,MAAM,CAAC,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;wBAC9C,MAAM,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;wBAC5B,GAAG,CAAC,SAAS,EAAE,CAAC;wBAChB,MAAM,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC;wBAC1D,MAAM,QAAQ,GAAG,OAAO,CAAC,CAAC,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;wBACnE,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;4BAC3B,GAAG,CAAC,YAAY,GAAG,IAAI,CAAC;4BACxB,cAAc,IAAI,QAAQ,CAAC;4BAC3B,IAAI,CAAC,MAAM,IAAI,QAAQ,CAAC;4BACxB,IAAI,CAAC,SAAS,IAAI,aAAa,CAAC;wBAClC,CAAC;wBACD,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;4BAC9C,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,SAAS,CAAC;4BACtD,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;4BACzD,4DAA4D;4BAC5D,6DAA6D;4BAC7D,4DAA4D;4BAC5D,oDAAoD;4BACpD,IAAI,MAAM,KAAK,kBAAkB,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;gCAC1D,UAAU,IAAI,QAAQ,IAAI,aAAa,GAAG,WAAW,CAAC;4BACxD,CAAC;4BACD,SAAS;wBACX,CAAC;wBACD,UAAU,IAAI,QAAQ,IAAI,aAAa,GAAG,WAAW,CAAC;wBACtD,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;wBACtC,MAAM,SAAS,GAAG,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;wBAC7C,IAAI,SAAS,KAAK,IAAI,EAAE,CAAC;4BACvB,GAAG,CAAC,QAAQ,EAAE,CAAC;4BACf,SAAS;wBACX,CAAC;wBACD,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,CAAC,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC;oBACvD,CAAC;oBACD,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;oBAC9B,GAAG,CAAC,UAAU,IAAI,UAAU,CAAC;oBAC7B,WAAW,IAAI,UAAU,CAAC;oBAC1B,IAAI,MAAM,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;wBAC7B,IAAI,CAAC;4BACH,MAAM,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;wBAClC,CAAC;wBAAC,MAAM,CAAC;4BACP,uBAAuB,GAAG,KAAK,CAAC;wBAClC,CAAC;oBACH,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QACD,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;IAChC,CAAC;IAED,MAAM,aAAa,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAC1D,OAAO;QACL,aAAa,EAAE,CAAC;QAChB,IAAI,EAAE,QAAQ;QACd,SAAS;QACT,UAAU,EAAE,GAAG,EAAE,CAAC,WAAW,EAAE;QAC/B,MAAM,EAAE;YACN,IAAI;YACJ,SAAS;YACT,SAAS;YACT,SAAS,EAAE,SAAS;YACpB,KAAK;YACL,SAAS,EAAE,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBACtC,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,QAAQ,EAAE,CAAC,CAAC,QAAQ;gBACpB,OAAO,EAAE,CAAC,CAAC,OAAO;gBAClB,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACvC,CAAC,CAAC;SACJ;QACD,OAAO,EAAE,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC;QACrC,SAAS;QACT,IAAI,EAAE,UAAU;QAChB,SAAS,EAAE,SAAS,CAAC,SAAS,EAAE,UAAU,EAAE,aAAa,EAAE,KAAK,CAAC;QACjE,KAAK,EAAE;YACL,SAAS;YACT,WAAW;YACX,cAAc;YACd,UAAU,EAAE,SAAS,KAAK,SAAS;YACnC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACnC,WAAW;YACX,cAAc,EAAE,aAAa,GAAG,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM;YAC9D,uBAAuB;SACxB;KACF,CAAC;AACJ,CAAC"}

package/dist/decide/bench/source.d.ts

@@ -0,0 +1,21 @@
+/**
+ * The store-backed {@link BenchSource}: reads tasks through the task data
+ * accessor and observations and decisions through the brain accessor — the
+ * core accessors only, never raw SQL on `.cleo/*.db` (T12495).
+ *
+ * @task T12495
+ * @epic T12486
+ */
+import type { BenchSource } from './types.js';
+/**
+ * A {@link BenchSource} over the project's stores.
+ *
+ * Tasks include every status (cancelled and archived too); observations and
+ * decisions include history (invalidated and superseded rows), because the
+ * labels come from exactly those records.
+ *
+ * @param projectRoot - Project root whose stores are read.
+ * @returns The source; each method opens its accessor lazily.
+ */
+export declare function createStoreBenchSource(projectRoot: string): BenchSource;
+//# sourceMappingURL=source.d.ts.map

package/dist/decide/bench/source.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"source.d.ts","sourceRoot":"","sources":["../../../src/decide/bench/source.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAGH,OAAO,KAAK,EAGV,WAAW,EAEZ,MAAM,YAAY,CAAC;AAEpB;;;;;;;;;GASG;AACH,wBAAgB,sBAAsB,CAAC,WAAW,EAAE,MAAM,GAAG,WAAW,CA8CvE"}

package/dist/decide/bench/source.js

@@ -0,0 +1,67 @@
+/**
+ * The store-backed {@link BenchSource}: reads tasks through the task data
+ * accessor and observations and decisions through the brain accessor — the
+ * core accessors only, never raw SQL on `.cleo/*.db` (T12495).
+ *
+ * @task T12495
+ * @epic T12486
+ */
+import { TASK_STATUSES } from '@cleocode/contracts';
+/**
+ * A {@link BenchSource} over the project's stores.
+ *
+ * Tasks include every status (cancelled and archived too); observations and
+ * decisions include history (invalidated and superseded rows), because the
+ * labels come from exactly those records.
+ *
+ * @param projectRoot - Project root whose stores are read.
+ * @returns The source; each method opens its accessor lazily.
+ */
+export function createStoreBenchSource(projectRoot) {
+    return {
+        async tasks() {
+            const { getTaskAccessor } = await import('../../store/data-accessor.js');
+            const accessor = await getTaskAccessor(projectRoot);
+            const { tasks } = await accessor.queryTasks({ status: [...TASK_STATUSES] });
+            return tasks.map((t) => ({
+                id: t.id,
+                title: t.title ?? '',
+                description: t.description ?? '',
+                status: t.status,
+                parentId: t.parentId ?? null,
+                ...(t.cancellationReason ? { cancellationReason: t.cancellationReason } : {}),
+                notes: t.notes ?? [],
+                relates: (t.relates ?? []).map((r) => ({
+                    taskId: r.taskId,
+                    type: r.type,
+                    ...(r.reason ? { reason: r.reason } : {}),
+                })),
+            }));
+        },
+        async observations() {
+            const { getBrainAccessor } = await import('../../store/memory-accessor.js');
+            const accessor = await getBrainAccessor(projectRoot);
+            const rows = await accessor.findObservations({ includeHistory: true });
+            return rows.map((o) => ({
+                id: o.id,
+                type: o.type,
+                title: o.title,
+                narrative: o.narrative ?? '',
+            }));
+        },
+        async decisions() {
+            const { getBrainAccessor } = await import('../../store/memory-accessor.js');
+            const accessor = await getBrainAccessor(projectRoot);
+            const rows = await accessor.findDecisions({ includeHistory: true });
+            return rows.map((d) => ({
+                id: d.id,
+                type: d.type,
+                decision: d.decision,
+                rationale: d.rationale,
+                supersedes: d.supersedes ?? null,
+                supersededBy: d.supersededBy ?? null,
+            }));
+        },
+    };
+}
+//# sourceMappingURL=source.js.map

package/dist/decide/bench/source.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"source.js","sourceRoot":"","sources":["../../../src/decide/bench/source.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAQpD;;;;;;;;;GASG;AACH,MAAM,UAAU,sBAAsB,CAAC,WAAmB;IACxD,OAAO;QACL,KAAK,CAAC,KAAK;YACT,MAAM,EAAE,eAAe,EAAE,GAAG,MAAM,MAAM,CAAC,8BAA8B,CAAC,CAAC;YACzE,MAAM,QAAQ,GAAG,MAAM,eAAe,CAAC,WAAW,CAAC,CAAC;YACpD,MAAM,EAAE,KAAK,EAAE,GAAG,MAAM,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC,GAAG,aAAa,CAAC,EAAE,CAAC,CAAC;YAC5E,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBACvB,EAAE,EAAE,CAAC,CAAC,EAAE;gBACR,KAAK,EAAE,CAAC,CAAC,KAAK,IAAI,EAAE;gBACpB,WAAW,EAAE,CAAC,CAAC,WAAW,IAAI,EAAE;gBAChC,MAAM,EAAE,CAAC,CAAC,MAAM;gBAChB,QAAQ,EAAE,CAAC,CAAC,QAAQ,IAAI,IAAI;gBAC5B,GAAG,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,EAAE,kBAAkB,EAAE,CAAC,CAAC,kBAAkB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC7E,KAAK,EAAE,CAAC,CAAC,KAAK,IAAI,EAAE;gBACpB,OAAO,EAAE,CAAC,CAAC,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;oBACrC,MAAM,EAAE,CAAC,CAAC,MAAM;oBAChB,IAAI,EAAE,CAAC,CAAC,IAAI;oBACZ,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;iBAC1C,CAAC,CAAC;aACJ,CAAC,CAAC,CAAC;QACN,CAAC;QACD,KAAK,CAAC,YAAY;YAChB,MAAM,EAAE,gBAAgB,EAAE,GAAG,MAAM,MAAM,CAAC,gCAAgC,CAAC,CAAC;YAC5E,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAAC,WAAW,CAAC,CAAC;YACrD,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,gBAAgB,CAAC,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC;YACvE,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBACtB,EAAE,EAAE,CAAC,CAAC,EAAE;gBACR,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,KAAK,EAAE,CAAC,CAAC,KAAK;gBACd,SAAS,EAAE,CAAC,CAAC,SAAS,IAAI,EAAE;aAC7B,CAAC,CAAC,CAAC;QACN,CAAC;QACD,KAAK,CAAC,SAAS;YACb,MAAM,EAAE,gBAAgB,EAAE,GAAG,MAAM,MAAM,CAAC,gCAAgC,CAAC,CAAC;YAC5E,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAAC,WAAW,CAAC,CAAC;YACrD,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,aAAa,CAAC,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC,CAAC;YACpE,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBACtB,EAAE,EAAE,CAAC,CAAC,EAAE;gBACR,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,QAAQ,EAAE,CAAC,CAAC,QAAQ;gBACpB,SAAS,EAAE,CAAC,CAAC,SAAS;gBACtB,UAAU,EAAE,CAAC,CAAC,UAAU,IAAI,IAAI;gBAChC,YAAY,EAAE,CAAC,CAAC,YAAY,IAAI,IAAI;aACrC,CAAC,CAAC,CAAC;QACN,CAAC;KACF,CAAC;AACJ,CAAC"}

package/dist/decide/bench/spot-check.d.ts

@@ -0,0 +1,117 @@
+/**
+ * Owner spot-check for the System One benchmark dataset (T12495).
+ *
+ * Before any spend, a small random sample stratified by site and label is
+ * written to a file. The orchestrator shows each item to the owner through
+ * the ask tool (the item carries the proposed label and the allowed labels
+ * as ready-made options) and writes the answers to a corrections file.
+ * {@link applyBenchCorrections} then relabels (or drops) those rows and marks
+ * them owner-verified. Both steps are offline: no provider is contacted.
+ *
+ * @task T12495
+ * @epic T12486
+ */
+import { z } from 'zod';
+import { type BenchRow } from './types.js';
+/** Default spot-check sample size. */
+export declare const DEFAULT_SPOT_CHECK_SIZE = 30;
+/** One item the owner reviews. */
+export interface BenchSpotCheckItem {
+    /** Dataset row id (the key a correction names). */
+    readonly id: string;
+    /** Site. */
+    readonly site: BenchRow['site'];
+    /** The label the builder proposed. */
+    readonly proposedLabel: string;
+    /** Labels the owner may choose from (the proposed one first). */
+    readonly options: readonly string[];
+    /** Short, redacted summary of what is being labelled. */
+    readonly summary: string;
+    /** Why the builder proposed the label. */
+    readonly provenance: BenchRow['provenance'];
+    /** Already verified by the owner in an earlier round. */
+    readonly ownerVerified: boolean;
+}
+/** The spot-check file. */
+export interface BenchSpotCheckFile {
+    /** Format version. */
+    readonly schemaVersion: 1;
+    /** ISO time the sample was drawn. */
+    readonly generatedAt: string;
+    /** Seed the sample was drawn with. */
+    readonly seed: number;
+    /** How to answer. */
+    readonly instructions: string;
+    /** Sampled items. */
+    readonly items: readonly BenchSpotCheckItem[];
+}
+/**
+ * Draw a random sample stratified by site and label.
+ *
+ * @param rows - Dataset rows.
+ * @param opts - Sample size (default {@link DEFAULT_SPOT_CHECK_SIZE}) and seed.
+ * @returns The spot-check file contents.
+ */
+export declare function sampleBenchSpotCheck(rows: readonly BenchRow[], opts?: {
+    readonly size?: number;
+    readonly seed?: number;
+    readonly now?: Date;
+}): BenchSpotCheckFile;
+/** Zod schema of one correction. */
+declare const correctionSchema: z.ZodUnion<readonly [z.ZodObject<{
+    id: z.ZodString;
+    label: z.ZodString;
+    note: z.ZodOptional<z.ZodString>;
+}, z.core.$strip>, z.ZodObject<{
+    id: z.ZodString;
+    drop: z.ZodLiteral<true>;
+    note: z.ZodOptional<z.ZodString>;
+}, z.core.$strip>]>;
+/** One owner answer: a label for a row, or a drop. */
+export type BenchCorrection = z.infer<typeof correctionSchema>;
+/** Outcome of {@link applyBenchCorrections}. */
+export interface BenchCorrectionsReceipt {
+    /** Rows confirmed with the proposed label. */
+    readonly confirmed: number;
+    /** Rows relabelled. */
+    readonly relabelled: number;
+    /** Rows removed. */
+    readonly dropped: number;
+    /** Correction ids that matched no row. */
+    readonly unmatched: readonly string[];
+    /** Corrections whose label is not allowed for the row's site (not applied). */
+    readonly rejected: readonly {
+        readonly id: string;
+        readonly label: string;
+    }[];
+}
+/**
+ * Parse a corrections file.
+ *
+ * @param text - File contents (JSON).
+ * @returns The corrections.
+ * @throws Error when the JSON or its shape is invalid.
+ */
+export declare function parseBenchCorrections(text: string): BenchCorrection[];
+/**
+ * Read and parse a corrections file.
+ *
+ * @param path - Corrections file.
+ * @returns The corrections.
+ */
+export declare function readBenchCorrections(path: string): Promise<BenchCorrection[]>;
+/**
+ * Apply owner corrections: relabel or drop the named rows and mark every
+ * corrected (or confirmed) row owner-verified. A relabelled row keeps its
+ * original label in `correctedFrom`.
+ *
+ * @param rows - Dataset rows.
+ * @param corrections - Owner answers.
+ * @returns The new rows and a receipt.
+ */
+export declare function applyBenchCorrections(rows: readonly BenchRow[], corrections: readonly BenchCorrection[]): {
+    rows: BenchRow[];
+    receipt: BenchCorrectionsReceipt;
+};
+export {};
+//# sourceMappingURL=spot-check.d.ts.map

package/dist/decide/bench/spot-check.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"spot-check.d.ts","sourceRoot":"","sources":["../../../src/decide/bench/spot-check.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAGH,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,KAAK,QAAQ,EAAiB,MAAM,YAAY,CAAC;AAE1D,sCAAsC;AACtC,eAAO,MAAM,uBAAuB,KAAK,CAAC;AAK1C,kCAAkC;AAClC,MAAM,WAAW,kBAAkB;IACjC,mDAAmD;IACnD,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,YAAY;IACZ,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;IAChC,sCAAsC;IACtC,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,iEAAiE;IACjE,QAAQ,CAAC,OAAO,EAAE,SAAS,MAAM,EAAE,CAAC;IACpC,yDAAyD;IACzD,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,0CAA0C;IAC1C,QAAQ,CAAC,UAAU,EAAE,QAAQ,CAAC,YAAY,CAAC,CAAC;IAC5C,yDAAyD;IACzD,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC;CACjC;AAED,2BAA2B;AAC3B,MAAM,WAAW,kBAAkB;IACjC,sBAAsB;IACtB,QAAQ,CAAC,aAAa,EAAE,CAAC,CAAC;IAC1B,qCAAqC;IACrC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,sCAAsC;IACtC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,qBAAqB;IACrB,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,qBAAqB;IACrB,QAAQ,CAAC,KAAK,EAAE,SAAS,kBAAkB,EAAE,CAAC;CAC/C;AAyDD;;;;;;GAMG;AACH,wBAAgB,oBAAoB,CAClC,IAAI,EAAE,SAAS,QAAQ,EAAE,EACzB,IAAI,GAAE;IAAE,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,IAAI,CAAA;CAAO,GACjF,kBAAkB,CAoCpB;AAED,oCAAoC;AACpC,QAAA,MAAM,gBAAgB;;;;;;;;mBAOpB,CAAC;AAQH,sDAAsD;AACtD,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAE/D,gDAAgD;AAChD,MAAM,WAAW,uBAAuB;IACtC,8CAA8C;IAC9C,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,uBAAuB;IACvB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,oBAAoB;IACpB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,0CAA0C;IAC1C,QAAQ,CAAC,SAAS,EAAE,SAAS,MAAM,EAAE,CAAC;IACtC,+EAA+E;IAC/E,QAAQ,CAAC,QAAQ,EAAE,SAAS;QAAE,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CAC/E;AAED;;;;;;GAMG;AACH,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,MAAM,GAAG,eAAe,EAAE,CAYrE;AAED;;;;;GAKG;AACH,wBAAsB,oBAAoB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAEnF;AAED;;;;;;;;GAQG;AACH,wBAAgB,qBAAqB,CACnC,IAAI,EAAE,SAAS,QAAQ,EAAE,EACzB,WAAW,EAAE,SAAS,eAAe,EAAE,GACtC;IAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;IAAC,OAAO,EAAE,uBAAuB,CAAA;CAAE,CAiDxD"}