@cleocode/core 2026.9.23 → 2026.9.25

package/dist/decide/bench/runner.d.ts

@@ -0,0 +1,208 @@
+/**
+ * The System One benchmark runner (T12495): replay every labelled row through
+ * every compared provider with `decideBatch`, next to each site's heuristic,
+ * and score the answers.
+ *
+ * - Provider answer cache off (`cache: false` on every request) and the
+ *   in-process outcome cache off, so every row is a real provider call.
+ * - One batch deadline of at least {@link MIN_BENCH_TIMEOUT_MS} (spec §1: the
+ *   batch endpoint answers its items serially).
+ * - A hard total spend cap across all providers and runs, SEPARATE from the
+ *   everyday monthly cap: before each batch the spend estimate (per
+ *   question, the larger of {@link DECISION_COST_ESTIMATE_MICROS_PER_QUESTION}
+ *   and the connection's observed mean reported cost) is added to what was
+ *   spent so far, and the run stops cleanly when that would pass the cap.
+ *   Billed answers the client rejected (`invalid_response`) count at their
+ *   reported cost (else the estimate), so a model that keeps answering in
+ *   the wrong shape is still capped. The monthly cap does not gate the benchmark
+ *   (`spend: null` on the calls), but every batch's spend is still recorded
+ *   in the monthly ledger so month-to-date spend stays true.
+ * - Providers take turns batch by batch, so a cap stop leaves them with the
+ *   same coverage.
+ *
+ * @task T12495
+ * @epic T12486
+ */
+import { type SpendLedger } from '../spend.js';
+import type { DecideFetch } from '../transport.js';
+import { type BenchDatasetSizes } from './dataset.js';
+import { type BenchClassificationMetrics, type BenchLatency } from './metrics.js';
+import { type BenchSpread } from './stats.js';
+import { type BenchConnection, type BenchRow, type BenchSite } from './types.js';
+/** Default hard spend cap across providers and runs: $5. */
+export declare const DEFAULT_BENCH_MAX_MICROS = 5000000;
+/** Minimum (and default) batch deadline, ms. */
+export declare const MIN_BENCH_TIMEOUT_MS = 30000;
+/** Default rows per batch call. */
+export declare const DEFAULT_BENCH_BATCH_SIZE = 16;
+/** Largest batch the layahost batch endpoint accepts (64 requests). */
+export declare const MAX_BENCH_BATCH_SIZE = 64;
+/** Name of the heuristic pseudo-provider in results. */
+export declare const HEURISTIC_PROVIDER = "heuristic";
+/** Options for {@link runDecideBench}. */
+export interface RunDecideBenchOptions {
+    /** Labelled rows (already filtered to the sites to run). */
+    readonly rows: readonly BenchRow[];
+    /** Providers to compare. */
+    readonly connections: readonly BenchConnection[];
+    /** Repeated runs. Default 1. */
+    readonly runs?: number;
+    /** Hard total spend cap in micro-dollars. Default {@link DEFAULT_BENCH_MAX_MICROS}. */
+    readonly maxMicros?: number;
+    /** Rows per batch call, 1–{@link MAX_BENCH_BATCH_SIZE}. Default {@link DEFAULT_BENCH_BATCH_SIZE}. */
+    readonly batchSize?: number;
+    /** Batch deadline, ms; raised to at least {@link MIN_BENCH_TIMEOUT_MS}. */
+    readonly timeoutMs?: number;
+    /** Monthly ledger the actual spend is recorded in; `null` skips it. Default: the file ledger. */
+    readonly spendLedger?: SpendLedger | null;
+    /** Transport (tests). Default: the decide transport. */
+    readonly fetch?: DecideFetch;
+    /**
+     * Directory for per-connection provider-state files (detected
+     * capabilities). Default `<cleoHome>/decide/bench/`. Each compared
+     * connection gets its own file: the everyday state file holds ONE identity,
+     * so two connections sharing it would overwrite each other's detection and
+     * one would fall back to the Jev minimum (no batch, no `cache: false`).
+     */
+    readonly providerStateDir?: string;
+    /** Clock (tests). */
+    readonly now?: () => Date;
+}
+/** One provider's (or the heuristic's) result on one site in one run. */
+export interface BenchSiteResult extends BenchClassificationMetrics {
+    /** Site. */
+    readonly site: BenchSite;
+    /** Rows sent (or scored, for the heuristic). */
+    readonly attempted: number;
+    /** Rows the provider answered usably (the metrics' `n`). */
+    readonly answered: number;
+    /** Latency of provider-answered rows: the wall time of the batch call that carried each row. */
+    readonly latency: BenchLatency;
+    /** Spend accounted to this site, micro-dollars (reported cost, else the estimate). */
+    readonly costMicros: number;
+    /** Whether the provider reported its cost (`meta.cost_micros` / `cost_usd`). */
+    readonly costReported: boolean;
+    /** Fallback reason → rows (timeouts, 4xx/5xx, malformed answers, …). */
+    readonly fallbacks: Readonly<Record<string, number>>;
+    /** Rows the provider answered with a value outside the site's labels. */
+    readonly unusable: number;
+}
+/** One provider in one run. */
+export interface BenchProviderRun {
+    /** Profile name. */
+    readonly provider: string;
+    /** Per-site results, in {@link BENCH_SITES} order. */
+    readonly sites: readonly BenchSiteResult[];
+}
+/** One repeated run. */
+export interface BenchRunRecord {
+    /** 1-based run number. */
+    readonly run: number;
+    /** Whether every batch of the run was sent (false when the cap stopped it). */
+    readonly complete: boolean;
+    /** Per provider. */
+    readonly providers: readonly BenchProviderRun[];
+}
+/** Mean and spread of one provider × site across runs. */
+export interface BenchAggregateRow {
+    /** Profile name (or {@link HEURISTIC_PROVIDER}). */
+    readonly provider: string;
+    /** Site. */
+    readonly site: BenchSite;
+    /** Rows answered, per run. */
+    readonly answered: BenchSpread;
+    /** Accuracy. */
+    readonly accuracy: BenchSpread;
+    /** Precision. */
+    readonly precision: BenchSpread;
+    /** Recall. */
+    readonly recall: BenchSpread;
+    /** F1. */
+    readonly f1: BenchSpread;
+    /** False-positive rate. */
+    readonly falsePositiveRate: BenchSpread;
+    /** p50 latency, ms. */
+    readonly p50Ms: BenchSpread;
+    /** p95 latency, ms. */
+    readonly p95Ms: BenchSpread;
+    /** Cost per run, micro-dollars (reported cost, else the estimate). */
+    readonly costMicros: BenchSpread;
+    /**
+     * Whether any run's provider reported its cost. False for a plain Jev host
+     * (no `meta.cost_*`): `costMicros` is then only the reservation estimate.
+     */
+    readonly costReported: boolean;
+    /** Fallbacks summed over runs, by reason. */
+    readonly fallbacks: Readonly<Record<string, number>>;
+}
+/** Spend accounting of a benchmark. */
+export interface BenchSpendSummary {
+    /** The hard cap, micro-dollars. */
+    readonly capMicros: number;
+    /** Spent (reported cost, else the estimate), micro-dollars. */
+    readonly spentMicros: number;
+    /** Of which the providers reported. */
+    readonly reportedMicros: number;
+    /** Whether the cap stopped the benchmark. */
+    readonly capReached: boolean;
+    /** Where it stopped, when it did. */
+    readonly stoppedAt?: {
+        readonly run: number;
+        readonly site: BenchSite;
+        readonly provider: string;
+        readonly batch: number;
+        readonly estimateMicros: number;
+    };
+    /** Batch calls made. */
+    readonly batchesSent: number;
+    /** Batch calls planned (runs × sites × batches × providers). */
+    readonly batchesPlanned: number;
+    /** Whether the monthly ledger accepted the spend record (false: it could not be written). */
+    readonly recordedInMonthlyLedger: boolean;
+}
+/** The machine-readable benchmark result. */
+export interface BenchResults {
+    /** Format version. */
+    readonly schemaVersion: 1;
+    /** Owning task. */
+    readonly task: 'T12495';
+    /** ISO start time. */
+    readonly startedAt: string;
+    /** ISO end time. */
+    readonly finishedAt: string;
+    /** Run configuration; never contains an API key. */
+    readonly config: {
+        readonly runs: number;
+        readonly batchSize: number;
+        readonly timeoutMs: number;
+        readonly maxMicros: number;
+        readonly sites: readonly BenchSite[];
+        readonly providers: readonly {
+            readonly name: string;
+            readonly provider: string;
+            readonly baseUrl: string;
+            readonly model?: string;
+        }[];
+    };
+    /** Dataset sizes. */
+    readonly dataset: BenchDatasetSizes;
+    /** The heuristic baseline (deterministic, scored once). */
+    readonly heuristic: readonly BenchSiteResult[];
+    /** Every run. */
+    readonly runs: readonly BenchRunRecord[];
+    /** Mean and spread per provider × site (heuristic included). */
+    readonly aggregate: readonly BenchAggregateRow[];
+    /** Spend. */
+    readonly spend: BenchSpendSummary;
+}
+/**
+ * Run the benchmark.
+ *
+ * Never throws because of a provider: failures become per-row fallbacks with
+ * their reason. Stops cleanly at the spend cap and reports what it completed.
+ *
+ * @param opts - Rows, connections, runs, cap and wiring.
+ * @returns The results.
+ */
+export declare function runDecideBench(opts: RunDecideBenchOptions): Promise<BenchResults>;
+//# sourceMappingURL=runner.d.ts.map

package/dist/decide/bench/runner.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../../src/decide/bench/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAcH,OAAO,EAAyB,KAAK,WAAW,EAAE,MAAM,aAAa,CAAC;AACtE,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AACnD,OAAO,EAAE,KAAK,iBAAiB,EAAqB,MAAM,cAAc,CAAC;AACzE,OAAO,EACL,KAAK,0BAA0B,EAC/B,KAAK,YAAY,EAIlB,MAAM,cAAc,CAAC;AAEtB,OAAO,EAAE,KAAK,WAAW,EAAY,MAAM,YAAY,CAAC;AACxD,OAAO,EAEL,KAAK,eAAe,EACpB,KAAK,QAAQ,EACb,KAAK,SAAS,EAEf,MAAM,YAAY,CAAC;AAEpB,4DAA4D;AAC5D,eAAO,MAAM,wBAAwB,UAAY,CAAC;AAElD,gDAAgD;AAChD,eAAO,MAAM,oBAAoB,QAAS,CAAC;AAE3C,mCAAmC;AACnC,eAAO,MAAM,wBAAwB,KAAK,CAAC;AAE3C,uEAAuE;AACvE,eAAO,MAAM,oBAAoB,KAAK,CAAC;AAEvC,wDAAwD;AACxD,eAAO,MAAM,kBAAkB,cAAc,CAAC;AAE9C,0CAA0C;AAC1C,MAAM,WAAW,qBAAqB;IACpC,4DAA4D;IAC5D,QAAQ,CAAC,IAAI,EAAE,SAAS,QAAQ,EAAE,CAAC;IACnC,4BAA4B;IAC5B,QAAQ,CAAC,WAAW,EAAE,SAAS,eAAe,EAAE,CAAC;IACjD,gCAAgC;IAChC,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,uFAAuF;IACvF,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,qGAAqG;IACrG,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,2EAA2E;IAC3E,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,iGAAiG;IACjG,QAAQ,CAAC,WAAW,CAAC,EAAE,WAAW,GAAG,IAAI,CAAC;IAC1C,wDAAwD;IACxD,QAAQ,CAAC,KAAK,CAAC,EAAE,WAAW,CAAC;IAC7B;;;;;;OAMG;IACH,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IACnC,qBAAqB;IACrB,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;CAC3B;AAED,yEAAyE;AACzE,MAAM,WAAW,eAAgB,SAAQ,0BAA0B;IACjE,YAAY;IACZ,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;IACzB,gDAAgD;IAChD,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,4DAA4D;IAC5D,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,gGAAgG;IAChG,QAAQ,CAAC,OAAO,EAAE,YAAY,CAAC;IAC/B,sFAAsF;IACtF,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,gFAAgF;IAChF,QAAQ,CAAC,YAAY,EAAE,OAAO,CAAC;IAC/B,wEAAwE;IACxE,QAAQ,CAAC,SAAS,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IACrD,yEAAyE;IACzE,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;CAC3B;AAED,+BAA+B;AAC/B,MAAM,WAAW,gBAAgB;IAC/B,oBAAoB;IACpB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,sDAAsD;IACtD,QAAQ,CAAC,KAAK,EAAE,SAAS,eAAe,EAAE,CAAC;CAC5C;AAED,wBAAwB;AACxB,MAAM,WAAW,cAAc;IAC7B,0BAA0B;IAC1B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,+EAA+E;IAC/E,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC;IAC3B,oBAAoB;IACpB,QAAQ,CAAC,SAAS,EAAE,SAAS,gBAAgB,EAAE,CAAC;CACjD;AAED,0DAA0D;AAC1D,MAAM,WAAW,iBAAiB;IAChC,oDAAoD;IACpD,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,YAAY;IACZ,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;IACzB,8BAA8B;IAC9B,QAAQ,CAAC,QAAQ,EAAE,WAAW,CAAC;IAC/B,gBAAgB;IAChB,QAAQ,CAAC,QAAQ,EAAE,WAAW,CAAC;IAC/B,iBAAiB;IACjB,QAAQ,CAAC,SAAS,EAAE,WAAW,CAAC;IAChC,cAAc;IACd,QAAQ,CAAC,MAAM,EAAE,WAAW,CAAC;IAC7B,UAAU;IACV,QAAQ,CAAC,EAAE,EAAE,WAAW,CAAC;IACzB,2BAA2B;IAC3B,QAAQ,CAAC,iBAAiB,EAAE,WAAW,CAAC;IACxC,uBAAuB;IACvB,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC;IAC5B,uBAAuB;IACvB,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC;IAC5B,sEAAsE;IACtE,QAAQ,CAAC,UAAU,EAAE,WAAW,CAAC;IACjC;;;OAGG;IACH,QAAQ,CAAC,YAAY,EAAE,OAAO,CAAC;IAC/B,6CAA6C;IAC7C,QAAQ,CAAC,SAAS,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;CACtD;AAED,uCAAuC;AACvC,MAAM,WAAW,iBAAiB;IAChC,mCAAmC;IACnC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,+DAA+D;IAC/D,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,uCAAuC;IACvC,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,6CAA6C;IAC7C,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC;IAC7B,qCAAqC;IACrC,QAAQ,CAAC,SAAS,CAAC,EAAE;QACnB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;QACrB,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;QACzB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;QAC1B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;QACvB,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;KACjC,CAAC;IACF,wBAAwB;IACxB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,gEAAgE;IAChE,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,6FAA6F;IAC7F,QAAQ,CAAC,uBAAuB,EAAE,OAAO,CAAC;CAC3C;AAED,6CAA6C;AAC7C,MAAM,WAAW,YAAY;IAC3B,sBAAsB;IACtB,QAAQ,CAAC,aAAa,EAAE,CAAC,CAAC;IAC1B,mBAAmB;IACnB,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IACxB,sBAAsB;IACtB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,oBAAoB;IACpB,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,oDAAoD;IACpD,QAAQ,CAAC,MAAM,EAAE;QACf,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QACtB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;QAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;QAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;QAC3B,QAAQ,CAAC,KAAK,EAAE,SAAS,SAAS,EAAE,CAAC;QACrC,QAAQ,CAAC,SAAS,EAAE,SAAS;YAC3B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;YACtB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;YAC1B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;YACzB,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;SACzB,EAAE,CAAC;KACL,CAAC;IACF,qBAAqB;IACrB,QAAQ,CAAC,OAAO,EAAE,iBAAiB,CAAC;IACpC,2DAA2D;IAC3D,QAAQ,CAAC,SAAS,EAAE,SAAS,eAAe,EAAE,CAAC;IAC/C,iBAAiB;IACjB,QAAQ,CAAC,IAAI,EAAE,SAAS,cAAc,EAAE,CAAC;IACzC,gEAAgE;IAChE,QAAQ,CAAC,SAAS,EAAE,SAAS,iBAAiB,EAAE,CAAC;IACjD,aAAa;IACb,QAAQ,CAAC,KAAK,EAAE,iBAAiB,CAAC;CACnC;AAqKD;;;;;;;;GAQG;AACH,wBAAsB,cAAc,CAAC,IAAI,EAAE,qBAAqB,GAAG,OAAO,CAAC,YAAY,CAAC,CA8KvF"}

package/dist/decide/bench/runner.js

@@ -0,0 +1,336 @@
+/**
+ * The System One benchmark runner (T12495): replay every labelled row through
+ * every compared provider with `decideBatch`, next to each site's heuristic,
+ * and score the answers.
+ *
+ * - Provider answer cache off (`cache: false` on every request) and the
+ *   in-process outcome cache off, so every row is a real provider call.
+ * - One batch deadline of at least {@link MIN_BENCH_TIMEOUT_MS} (spec §1: the
+ *   batch endpoint answers its items serially).
+ * - A hard total spend cap across all providers and runs, SEPARATE from the
+ *   everyday monthly cap: before each batch the spend estimate (per
+ *   question, the larger of {@link DECISION_COST_ESTIMATE_MICROS_PER_QUESTION}
+ *   and the connection's observed mean reported cost) is added to what was
+ *   spent so far, and the run stops cleanly when that would pass the cap.
+ *   Billed answers the client rejected (`invalid_response`) count at their
+ *   reported cost (else the estimate), so a model that keeps answering in
+ *   the wrong shape is still capped. The monthly cap does not gate the benchmark
+ *   (`spend: null` on the calls), but every batch's spend is still recorded
+ *   in the monthly ledger so month-to-date spend stays true.
+ * - Providers take turns batch by batch, so a cap stop leaves them with the
+ *   same coverage.
+ *
+ * @task T12495
+ * @epic T12486
+ */
+import { join } from 'node:path';
+import { getCleoHome } from '@cleocode/paths';
+import { hashCanonical } from '../cache.js';
+import { DECISION_COST_ESTIMATE_MICROS_PER_QUESTION, decideBatch, redactDecisionState, } from '../client.js';
+import { DECIDE_BENCH_DECISION_SITE } from '../sites/registry.js';
+import { createFileSpendLedger } from '../spend.js';
+import { benchDatasetSizes } from './dataset.js';
+import { classificationMetrics, latencyOf, } from './metrics.js';
+import { benchQuestionFor } from './questions.js';
+import { spreadOf } from './stats.js';
+import { BENCH_SITES, positiveLabel, } from './types.js';
+/** Default hard spend cap across providers and runs: $5. */
+export const DEFAULT_BENCH_MAX_MICROS = 5_000_000;
+/** Minimum (and default) batch deadline, ms. */
+export const MIN_BENCH_TIMEOUT_MS = 30_000;
+/** Default rows per batch call. */
+export const DEFAULT_BENCH_BATCH_SIZE = 16;
+/** Largest batch the layahost batch endpoint accepts (64 requests). */
+export const MAX_BENCH_BATCH_SIZE = 64;
+/** Name of the heuristic pseudo-provider in results. */
+export const HEURISTIC_PROVIDER = 'heuristic';
+function newCollector() {
+    return {
+        attempted: 0,
+        predictions: [],
+        latencies: [],
+        costMicros: 0,
+        costReported: false,
+        fallbacks: {},
+        unusable: 0,
+    };
+}
+function finishCollector(site, c) {
+    return {
+        site,
+        ...classificationMetrics(c.predictions, positiveLabel(site)),
+        attempted: c.attempted,
+        answered: c.predictions.length,
+        latency: latencyOf(c.latencies),
+        costMicros: c.costMicros,
+        costReported: c.costReported,
+        fallbacks: c.fallbacks,
+        unusable: c.unusable,
+    };
+}
+/**
+ * Estimated cost per question for the pre-batch cap check: the client's
+ * reservation estimate, or the connection's observed mean when a pricier
+ * model has reported more, so one batch cannot overshoot the cap by much.
+ */
+function perQuestionEstimate(seen) {
+    const mean = seen && seen.questions > 0 ? Math.ceil(seen.micros / seen.questions) : 0;
+    return Math.max(DECISION_COST_ESTIMATE_MICROS_PER_QUESTION, mean);
+}
+/** Provider-reported cost of an outcome in micros (`costMicros`, else `costUsd`). */
+function reportedCostMicros(outcome) {
+    if (outcome.costMicros !== undefined)
+        return outcome.costMicros;
+    return outcome.costUsd !== undefined ? Math.round(outcome.costUsd * 1e6) : undefined;
+}
+/** A profile name made safe for a file name. */
+function safeFileName(name) {
+    return name.replace(/[^A-Za-z0-9._-]/g, '_').slice(0, 64) || 'profile';
+}
+/** Split `items` into chunks of `size`. */
+function chunks(items, size) {
+    const out = [];
+    for (let i = 0; i < items.length; i += size)
+        out.push(items.slice(i, i + size));
+    return out;
+}
+/** Audit key of a request, matching what `decide()` writes for it. */
+function auditKey(q) {
+    return `${hashCanonical(q.req.questions)}:${hashCanonical(redactDecisionState(q.req.state))}`;
+}
+/** An in-memory audit sink that keeps fallback reasons by request key. */
+function reasonSink() {
+    const reasons = new Map();
+    return {
+        sink: {
+            write(entry) {
+                if (entry.source !== 'fallback')
+                    return;
+                const key = `${entry.questionsHash}:${entry.stateHash}`;
+                reasons.set(key, [...(reasons.get(key) ?? []), entry.fallbackReason ?? 'unknown']);
+            },
+        },
+        take: (key) => {
+            const list = reasons.get(key);
+            return list?.shift();
+        },
+    };
+}
+/** Score the heuristic on every row (deterministic, no provider). */
+function scoreHeuristic(sites, questions) {
+    return sites.map((site) => {
+        const c = newCollector();
+        for (const { row, q } of questions.get(site) ?? []) {
+            const started = performance.now();
+            const label = q.predict(q.heuristicAnswers) ?? q.heuristicLabel;
+            c.latencies.push(Math.max(0, performance.now() - started));
+            c.attempted++;
+            c.predictions.push({ gold: row.label, predicted: label });
+        }
+        return finishCollector(site, c);
+    });
+}
+/** Mean and spread per provider × site. */
+function aggregate(heuristic, runs, providers, sites) {
+    const rows = [];
+    const build = (provider, site, results) => {
+        const fallbacks = {};
+        for (const r of results) {
+            for (const [k, v] of Object.entries(r.fallbacks))
+                fallbacks[k] = (fallbacks[k] ?? 0) + v;
+        }
+        const scored = results.filter((r) => r.answered > 0);
+        rows.push({
+            provider,
+            site,
+            answered: spreadOf(results.map((r) => r.answered)),
+            accuracy: spreadOf(scored.map((r) => r.accuracy)),
+            precision: spreadOf(scored.map((r) => r.precision)),
+            recall: spreadOf(scored.map((r) => r.recall)),
+            f1: spreadOf(scored.map((r) => r.f1)),
+            falsePositiveRate: spreadOf(scored.map((r) => r.falsePositiveRate)),
+            p50Ms: spreadOf(scored.map((r) => r.latency.p50Ms)),
+            p95Ms: spreadOf(scored.map((r) => r.latency.p95Ms)),
+            costMicros: spreadOf(results.map((r) => r.costMicros)),
+            costReported: results.some((r) => r.costReported),
+            fallbacks,
+        });
+    };
+    for (const site of sites) {
+        build(HEURISTIC_PROVIDER, site, heuristic.filter((h) => h.site === site));
+        for (const provider of providers) {
+            const results = runs.flatMap((run) => run.providers
+                .filter((p) => p.provider === provider)
+                .flatMap((p) => p.sites.filter((s) => s.site === site && s.attempted > 0)));
+            build(provider, site, results);
+        }
+    }
+    return rows;
+}
+/**
+ * Run the benchmark.
+ *
+ * Never throws because of a provider: failures become per-row fallbacks with
+ * their reason. Stops cleanly at the spend cap and reports what it completed.
+ *
+ * @param opts - Rows, connections, runs, cap and wiring.
+ * @returns The results.
+ */
+export async function runDecideBench(opts) {
+    const now = opts.now ?? (() => new Date());
+    const startedAt = now().toISOString();
+    const runs = Math.max(1, Math.floor(opts.runs ?? 1));
+    const capMicros = Math.max(0, opts.maxMicros ?? DEFAULT_BENCH_MAX_MICROS);
+    const batchSize = Math.min(MAX_BENCH_BATCH_SIZE, Math.max(1, Math.floor(opts.batchSize ?? DEFAULT_BENCH_BATCH_SIZE)));
+    const timeoutMs = Math.max(MIN_BENCH_TIMEOUT_MS, opts.timeoutMs ?? MIN_BENCH_TIMEOUT_MS);
+    const ledger = opts.spendLedger === undefined ? createFileSpendLedger() : opts.spendLedger;
+    const stateDir = opts.providerStateDir ?? join(getCleoHome(), 'decide', 'bench');
+    const sites = BENCH_SITES.filter((s) => opts.rows.some((r) => r.site === s));
+    const questions = new Map();
+    for (const row of opts.rows) {
+        questions.set(row.site, [
+            ...(questions.get(row.site) ?? []),
+            { row, q: benchQuestionFor(row) },
+        ]);
+    }
+    const heuristic = scoreHeuristic(sites, questions);
+    const batchesPerRun = sites.reduce((n, s) => n + Math.ceil((questions.get(s)?.length ?? 0) / batchSize), 0);
+    let spentMicros = 0;
+    let reportedMicros = 0;
+    let batchesSent = 0;
+    let recordedInMonthlyLedger = true;
+    let stoppedAt;
+    const runRecords = [];
+    /** Reported cost per connection so far, for the pre-batch estimate. */
+    const observed = new Map();
+    runLoop: for (let run = 1; run <= runs; run++) {
+        const collectors = new Map();
+        for (const c of opts.connections) {
+            collectors.set(c.name, new Map(sites.map((s) => [s, newCollector()])));
+        }
+        const record = (complete) => ({
+            run,
+            complete,
+            providers: opts.connections.map((c) => ({
+                provider: c.name,
+                sites: sites.map((s) => finishCollector(s, collectors.get(c.name)?.get(s) ?? newCollector())),
+            })),
+        });
+        for (const site of sites) {
+            for (const [b, batch] of chunks(questions.get(site) ?? [], batchSize).entries()) {
+                for (const conn of opts.connections) {
+                    const questionCount = batch.reduce((n, x) => n + Object.keys(x.q.req.questions).length, 0);
+                    const estimate = questionCount * perQuestionEstimate(observed.get(conn.name));
+                    if (spentMicros + estimate > capMicros) {
+                        stoppedAt = { run, site, provider: conn.name, batch: b + 1, estimateMicros: estimate };
+                        runRecords.push(record(false));
+                        break runLoop;
+                    }
+                    const reasons = reasonSink();
+                    const entries = batch.map(({ q }) => ({
+                        req: q.req,
+                        fallback: () => q.heuristicAnswers,
+                    }));
+                    const outcomes = await decideBatch(DECIDE_BENCH_DECISION_SITE.id, entries, {
+                        connection: {
+                            baseUrl: conn.baseUrl,
+                            apiKey: conn.apiKey,
+                            ...(conn.model ? { model: conn.model } : {}),
+                        },
+                        timeoutMs,
+                        cache: null,
+                        budget: null,
+                        spend: null,
+                        audit: reasons.sink,
+                        ...(opts.fetch ? { fetch: opts.fetch } : {}),
+                        providerStatePath: join(stateDir, `provider-state-${safeFileName(conn.name)}.json`),
+                    });
+                    batchesSent++;
+                    const col = collectors.get(conn.name)?.get(site) ?? newCollector();
+                    let batchSpend = 0;
+                    const perQuestion = perQuestionEstimate(observed.get(conn.name));
+                    const seen = observed.get(conn.name) ?? { micros: 0, questions: 0 };
+                    for (const [i, { row, q }] of batch.entries()) {
+                        const outcome = outcomes[i];
+                        col.attempted++;
+                        const questionsHere = Object.keys(q.req.questions).length;
+                        const reported = outcome ? reportedCostMicros(outcome) : undefined;
+                        if (reported !== undefined) {
+                            col.costReported = true;
+                            reportedMicros += reported;
+                            seen.micros += reported;
+                            seen.questions += questionsHere;
+                        }
+                        if (!outcome || outcome.source !== 'provider') {
+                            const reason = reasons.take(auditKey(q)) ?? 'unknown';
+                            col.fallbacks[reason] = (col.fallbacks[reason] ?? 0) + 1;
+                            // Billed without a usable answer: a 2xx the client rejected
+                            // (`invalid_response`, at its reported cost), or a call that
+                            // timed out after send. Other failures are error responses,
+                            // which the provider does not bill (layahost docs).
+                            if (reason === 'invalid_response' || reason === 'timeout') {
+                                batchSpend += reported ?? questionsHere * perQuestion;
+                            }
+                            continue;
+                        }
+                        batchSpend += reported ?? questionsHere * perQuestion;
+                        col.latencies.push(outcome.latencyMs);
+                        const predicted = q.predict(outcome.answers);
+                        if (predicted === null) {
+                            col.unusable++;
+                            continue;
+                        }
+                        col.predictions.push({ gold: row.label, predicted });
+                    }
+                    observed.set(conn.name, seen);
+                    col.costMicros += batchSpend;
+                    spentMicros += batchSpend;
+                    if (ledger && batchSpend > 0) {
+                        try {
+                            await ledger.record(batchSpend);
+                        }
+                        catch {
+                            recordedInMonthlyLedger = false;
+                        }
+                    }
+                }
+            }
+        }
+        runRecords.push(record(true));
+    }
+    const providerNames = opts.connections.map((c) => c.name);
+    return {
+        schemaVersion: 1,
+        task: 'T12495',
+        startedAt,
+        finishedAt: now().toISOString(),
+        config: {
+            runs,
+            batchSize,
+            timeoutMs,
+            maxMicros: capMicros,
+            sites,
+            providers: opts.connections.map((c) => ({
+                name: c.name,
+                provider: c.provider,
+                baseUrl: c.baseUrl,
+                ...(c.model ? { model: c.model } : {}),
+            })),
+        },
+        dataset: benchDatasetSizes(opts.rows),
+        heuristic,
+        runs: runRecords,
+        aggregate: aggregate(heuristic, runRecords, providerNames, sites),
+        spend: {
+            capMicros,
+            spentMicros,
+            reportedMicros,
+            capReached: stoppedAt !== undefined,
+            ...(stoppedAt ? { stoppedAt } : {}),
+            batchesSent,
+            batchesPlanned: batchesPerRun * runs * opts.connections.length,
+            recordedInMonthlyLedger,
+        },
+    };
+}
+//# sourceMappingURL=runner.js.map

package/dist/decide/bench/runner.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../../src/decide/bench/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;GAwBG;AAEH,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAE9C,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EACL,0CAA0C,EAE1C,WAAW,EACX,mBAAmB,GACpB,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,0BAA0B,EAAE,MAAM,sBAAsB,CAAC;AAClE,OAAO,EAAE,qBAAqB,EAAoB,MAAM,aAAa,CAAC;AAEtE,OAAO,EAA0B,iBAAiB,EAAE,MAAM,cAAc,CAAC;AACzE,OAAO,EAIL,qBAAqB,EACrB,SAAS,GACV,MAAM,cAAc,CAAC;AACtB,OAAO,EAAsB,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AACtE,OAAO,EAAoB,QAAQ,EAAE,MAAM,YAAY,CAAC;AACxD,OAAO,EACL,WAAW,EAIX,aAAa,GACd,MAAM,YAAY,CAAC;AAEpB,4DAA4D;AAC5D,MAAM,CAAC,MAAM,wBAAwB,GAAG,SAAS,CAAC;AAElD,gDAAgD;AAChD,MAAM,CAAC,MAAM,oBAAoB,GAAG,MAAM,CAAC;AAE3C,mCAAmC;AACnC,MAAM,CAAC,MAAM,wBAAwB,GAAG,EAAE,CAAC;AAE3C,uEAAuE;AACvE,MAAM,CAAC,MAAM,oBAAoB,GAAG,EAAE,CAAC;AAEvC,wDAAwD;AACxD,MAAM,CAAC,MAAM,kBAAkB,GAAG,WAAW,CAAC;AAgL9C,SAAS,YAAY;IACnB,OAAO;QACL,SAAS,EAAE,CAAC;QACZ,WAAW,EAAE,EAAE;QACf,SAAS,EAAE,EAAE;QACb,UAAU,EAAE,CAAC;QACb,YAAY,EAAE,KAAK;QACnB,SAAS,EAAE,EAAE;QACb,QAAQ,EAAE,CAAC;KACZ,CAAC;AACJ,CAAC;AAED,SAAS,eAAe,CAAC,IAAe,EAAE,CAAY;IACpD,OAAO;QACL,IAAI;QACJ,GAAG,qBAAqB,CAAC,CAAC,CAAC,WAAW,EAAE,aAAa,CAAC,IAAI,CAAC,CAAC;QAC5D,SAAS,EAAE,CAAC,CAAC,SAAS;QACtB,QAAQ,EAAE,CAAC,CAAC,WAAW,CAAC,MAAM;QAC9B,OAAO,EAAE,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC;QAC/B,UAAU,EAAE,CAAC,CAAC,UAAU;QACxB,YAAY,EAAE,CAAC,CAAC,YAAY;QAC5B,SAAS,EAAE,CAAC,CAAC,SAAS;QACtB,QAAQ,EAAE,CAAC,CAAC,QAAQ;KACrB,CAAC;AACJ,CAAC;AAQD;;;;GAIG;AACH,SAAS,mBAAmB,CAAC,IAA8B;IACzD,MAAM,IAAI,GAAG,IAAI,IAAI,IAAI,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACtF,OAAO,IAAI,CAAC,GAAG,CAAC,0CAA0C,EAAE,IAAI,CAAC,CAAC;AACpE,CAAC;AAED,qFAAqF;AACrF,SAAS,kBAAkB,CAAC,OAAwB;IAClD,IAAI,OAAO,CAAC,UAAU,KAAK,SAAS;QAAE,OAAO,OAAO,CAAC,UAAU,CAAC;IAChE,OAAO,OAAO,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;AACvF,CAAC;AAED,gDAAgD;AAChD,SAAS,YAAY,CAAC,IAAY;IAChC,OAAO,IAAI,CAAC,OAAO,CAAC,kBAAkB,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,SAAS,CAAC;AACzE,CAAC;AAED,2CAA2C;AAC3C,SAAS,MAAM,CAAI,KAAmB,EAAE,IAAY;IAClD,MAAM,GAAG,GAAU,EAAE,CAAC;IACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,IAAI;QAAE,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC;IAChF,OAAO,GAAG,CAAC;AACb,CAAC;AAED,sEAAsE;AACtE,SAAS,QAAQ,CAAC,CAAgB;IAChC,OAAO,GAAG,aAAa,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,aAAa,CAAC,mBAAmB,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;AAChG,CAAC;AAED,0EAA0E;AAC1E,SAAS,UAAU;IAIjB,MAAM,OAAO,GAAG,IAAI,GAAG,EAAoB,CAAC;IAC5C,OAAO;QACL,IAAI,EAAE;YACJ,KAAK,CAAC,KAAyB;gBAC7B,IAAI,KAAK,CAAC,MAAM,KAAK,UAAU;oBAAE,OAAO;gBACxC,MAAM,GAAG,GAAG,GAAG,KAAK,CAAC,aAAa,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;gBACxD,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,EAAE,KAAK,CAAC,cAAc,IAAI,SAAS,CAAC,CAAC,CAAC;YACrF,CAAC;SACF;QACD,IAAI,EAAE,CAAC,GAAG,EAAE,EAAE;YACZ,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC9B,OAAO,IAAI,EAAE,KAAK,EAAE,CAAC;QACvB,CAAC;KACF,CAAC;AACJ,CAAC;AAED,qEAAqE;AACrE,SAAS,cAAc,CACrB,KAA2B,EAC3B,SAAiF;IAEjF,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACxB,MAAM,CAAC,GAAG,YAAY,EAAE,CAAC;QACzB,KAAK,MAAM,EAAE,GAAG,EAAE,CAAC,EAAE,IAAI,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;YACnD,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;YAClC,MAAM,KAAK,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,gBAAgB,CAAC,IAAI,CAAC,CAAC,cAAc,CAAC;YAChE,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC;YAC3D,CAAC,CAAC,SAAS,EAAE,CAAC;YACd,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,CAAC,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;QAC5D,CAAC;QACD,OAAO,eAAe,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAClC,CAAC,CAAC,CAAC;AACL,CAAC;AAED,2CAA2C;AAC3C,SAAS,SAAS,CAChB,SAAqC,EACrC,IAA+B,EAC/B,SAA4B,EAC5B,KAA2B;IAE3B,MAAM,IAAI,GAAwB,EAAE,CAAC;IACrC,MAAM,KAAK,GAAG,CAAC,QAAgB,EAAE,IAAe,EAAE,OAAmC,EAAE,EAAE;QACvF,MAAM,SAAS,GAA2B,EAAE,CAAC;QAC7C,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;YACxB,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC;gBAAE,SAAS,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;QAC3F,CAAC;QACD,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC;QACrD,IAAI,CAAC,IAAI,CAAC;YACR,QAAQ;YACR,IAAI;YACJ,QAAQ,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;YAClD,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;YACjD,SAAS,EAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;YACnD,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YAC7C,EAAE,EAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YACrC,iBAAiB,EAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC;YACnE,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YACnD,KAAK,EAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YACnD,UAAU,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC;YACtD,YAAY,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,YAAY,CAAC;YACjD,SAAS;SACV,CAAC,CAAC;IACL,CAAC,CAAC;IACF,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,KAAK,CACH,kBAAkB,EAClB,IAAI,EACJ,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,CACzC,CAAC;QACF,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YACjC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CACnC,GAAG,CAAC,SAAS;iBACV,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,QAAQ,CAAC;iBACtC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,IAAI,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,CAAC,CAC7E,CAAC;YACF,KAAK,CAAC,QAAQ,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;QACjC,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,IAA2B;IAC9D,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;IAC3C,MAAM,SAAS,GAAG,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC;IACtC,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC;IACrD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,SAAS,IAAI,wBAAwB,CAAC,CAAC;IAC1E,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CACxB,oBAAoB,EACpB,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,IAAI,wBAAwB,CAAC,CAAC,CACpE,CAAC;IACF,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,oBAAoB,EAAE,IAAI,CAAC,SAAS,IAAI,oBAAoB,CAAC,CAAC;IACzF,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,qBAAqB,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC;IAC3F,MAAM,QAAQ,GAAG,IAAI,CAAC,gBAAgB,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;IAEjF,MAAM,KAAK,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC;IAC7E,MAAM,SAAS,GAAG,IAAI,GAAG,EAAoD,CAAC;IAC9E,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;QAC5B,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE;YACtB,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;YAClC,EAAE,GAAG,EAAE,CAAC,EAAE,gBAAgB,CAAC,GAAG,CAAC,EAAE;SAClC,CAAC,CAAC;IACL,CAAC;IACD,MAAM,SAAS,GAAG,cAAc,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;IAEnD,MAAM,aAAa,GAAG,KAAK,CAAC,MAAM,CAChC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,MAAM,IAAI,CAAC,CAAC,GAAG,SAAS,CAAC,EACpE,CAAC,CACF,CAAC;IACF,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,cAAc,GAAG,CAAC,CAAC;IACvB,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,uBAAuB,GAAG,IAAI,CAAC;IACnC,IAAI,SAAyC,CAAC;IAC9C,MAAM,UAAU,GAAqB,EAAE,CAAC;IACxC,uEAAuE;IACvE,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAwB,CAAC;IAEjD,OAAO,EAAE,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,IAAI,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC;QAC9C,MAAM,UAAU,GAAG,IAAI,GAAG,EAAqC,CAAC;QAChE,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACjC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACzE,CAAC;QACD,MAAM,MAAM,GAAG,CAAC,QAAiB,EAAkB,EAAE,CAAC,CAAC;YACrD,GAAG;YACH,QAAQ;YACR,SAAS,EAAE,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBACtC,QAAQ,EAAE,CAAC,CAAC,IAAI;gBAChB,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACrB,eAAe,CAAC,CAAC,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,YAAY,EAAE,CAAC,CACrE;aACF,CAAC,CAAC;SACJ,CAAC,CAAC;QACH,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,KAAK,MAAM,CAAC,CAAC,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,SAAS,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC;gBAChF,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;oBACpC,MAAM,aAAa,GAAG,KAAK,CAAC,MAAM,CAChC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,MAAM,EACnD,CAAC,CACF,CAAC;oBACF,MAAM,QAAQ,GAAG,aAAa,GAAG,mBAAmB,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;oBAC9E,IAAI,WAAW,GAAG,QAAQ,GAAG,SAAS,EAAE,CAAC;wBACvC,SAAS,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,GAAG,CAAC,EAAE,cAAc,EAAE,QAAQ,EAAE,CAAC;wBACvF,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;wBAC/B,MAAM,OAAO,CAAC;oBAChB,CAAC;oBACD,MAAM,OAAO,GAAG,UAAU,EAAE,CAAC;oBAC7B,MAAM,OAAO,GAAyB,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC;wBAC1D,GAAG,EAAE,CAAC,CAAC,GAAG;wBACV,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,gBAAgB;qBACnC,CAAC,CAAC,CAAC;oBACJ,MAAM,QAAQ,GAAsB,MAAM,WAAW,CACnD,0BAA0B,CAAC,EAAE,EAC7B,OAAO,EACP;wBACE,UAAU,EAAE;4BACV,OAAO,EAAE,IAAI,CAAC,OAAO;4BACrB,MAAM,EAAE,IAAI,CAAC,MAAM;4BACnB,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;yBAC7C;wBACD,SAAS;wBACT,KAAK,EAAE,IAAI;wBACX,MAAM,EAAE,IAAI;wBACZ,KAAK,EAAE,IAAI;wBACX,KAAK,EAAE,OAAO,CAAC,IAAI;wBACnB,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;wBAC5C,iBAAiB,EAAE,IAAI,CAAC,QAAQ,EAAE,kBAAkB,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC;qBACpF,CACF,CAAC;oBACF,WAAW,EAAE,CAAC;oBACd,MAAM,GAAG,GAAG,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,IAAI,YAAY,EAAE,CAAC;oBACnE,IAAI,UAAU,GAAG,CAAC,CAAC;oBACnB,MAAM,WAAW,GAAG,mBAAmB,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;oBACjE,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;oBACpE,KAAK,MAAM,CAAC,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,EAAE,CAAC;wBAC9C,MAAM,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;wBAC5B,GAAG,CAAC,SAAS,EAAE,CAAC;wBAChB,MAAM,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC;wBAC1D,MAAM,QAAQ,GAAG,OAAO,CAAC,CAAC,CAAC,kBAAkB,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;wBACnE,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;4BAC3B,GAAG,CAAC,YAAY,GAAG,IAAI,CAAC;4BACxB,cAAc,IAAI,QAAQ,CAAC;4BAC3B,IAAI,CAAC,MAAM,IAAI,QAAQ,CAAC;4BACxB,IAAI,CAAC,SAAS,IAAI,aAAa,CAAC;wBAClC,CAAC;wBACD,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;4BAC9C,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,SAAS,CAAC;4BACtD,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;4BACzD,4DAA4D;4BAC5D,6DAA6D;4BAC7D,4DAA4D;4BAC5D,oDAAoD;4BACpD,IAAI,MAAM,KAAK,kBAAkB,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;gCAC1D,UAAU,IAAI,QAAQ,IAAI,aAAa,GAAG,WAAW,CAAC;4BACxD,CAAC;4BACD,SAAS;wBACX,CAAC;wBACD,UAAU,IAAI,QAAQ,IAAI,aAAa,GAAG,WAAW,CAAC;wBACtD,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;wBACtC,MAAM,SAAS,GAAG,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;wBAC7C,IAAI,SAAS,KAAK,IAAI,EAAE,CAAC;4BACvB,GAAG,CAAC,QAAQ,EAAE,CAAC;4BACf,SAAS;wBACX,CAAC;wBACD,GAAG,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,CAAC,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC;oBACvD,CAAC;oBACD,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;oBAC9B,GAAG,CAAC,UAAU,IAAI,UAAU,CAAC;oBAC7B,WAAW,IAAI,UAAU,CAAC;oBAC1B,IAAI,MAAM,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;wBAC7B,IAAI,CAAC;4BACH,MAAM,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;wBAClC,CAAC;wBAAC,MAAM,CAAC;4BACP,uBAAuB,GAAG,KAAK,CAAC;wBAClC,CAAC;oBACH,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;QACD,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC;IAChC,CAAC;IAED,MAAM,aAAa,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAC1D,OAAO;QACL,aAAa,EAAE,CAAC;QAChB,IAAI,EAAE,QAAQ;QACd,SAAS;QACT,UAAU,EAAE,GAAG,EAAE,CAAC,WAAW,EAAE;QAC/B,MAAM,EAAE;YACN,IAAI;YACJ,SAAS;YACT,SAAS;YACT,SAAS,EAAE,SAAS;YACpB,KAAK;YACL,SAAS,EAAE,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBACtC,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,QAAQ,EAAE,CAAC,CAAC,QAAQ;gBACpB,OAAO,EAAE,CAAC,CAAC,OAAO;gBAClB,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACvC,CAAC,CAAC;SACJ;QACD,OAAO,EAAE,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC;QACrC,SAAS;QACT,IAAI,EAAE,UAAU;QAChB,SAAS,EAAE,SAAS,CAAC,SAAS,EAAE,UAAU,EAAE,aAAa,EAAE,KAAK,CAAC;QACjE,KAAK,EAAE;YACL,SAAS;YACT,WAAW;YACX,cAAc;YACd,UAAU,EAAE,SAAS,KAAK,SAAS;YACnC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACnC,WAAW;YACX,cAAc,EAAE,aAAa,GAAG,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,MAAM;YAC9D,uBAAuB;SACxB;KACF,CAAC;AACJ,CAAC"}

package/dist/decide/bench/source.d.ts

@@ -0,0 +1,21 @@
+/**
+ * The store-backed {@link BenchSource}: reads tasks through the task data
+ * accessor and observations and decisions through the brain accessor — the
+ * core accessors only, never raw SQL on `.cleo/*.db` (T12495).
+ *
+ * @task T12495
+ * @epic T12486
+ */
+import type { BenchSource } from './types.js';
+/**
+ * A {@link BenchSource} over the project's stores.
+ *
+ * Tasks include every status (cancelled and archived too); observations and
+ * decisions include history (invalidated and superseded rows), because the
+ * labels come from exactly those records.
+ *
+ * @param projectRoot - Project root whose stores are read.
+ * @returns The source; each method opens its accessor lazily.
+ */
+export declare function createStoreBenchSource(projectRoot: string): BenchSource;
+//# sourceMappingURL=source.d.ts.map

package/dist/decide/bench/source.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"source.d.ts","sourceRoot":"","sources":["../../../src/decide/bench/source.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAGH,OAAO,KAAK,EAGV,WAAW,EAEZ,MAAM,YAAY,CAAC;AAEpB;;;;;;;;;GASG;AACH,wBAAgB,sBAAsB,CAAC,WAAW,EAAE,MAAM,GAAG,WAAW,CA8CvE"}