@cleocode/core 2026.9.23 → 2026.9.24

package/dist/decide/bench/metrics.js

@@ -0,0 +1,100 @@
+/**
+ * Classification metrics for the System One benchmark (T12495).
+ *
+ * Two-label sites report precision, recall, F1 and false-positive rate for
+ * their positive class. The multi-class observation-type site reports macro
+ * averages over the labels present in the gold set (one-vs-rest per label).
+ *
+ * @task T12495
+ * @epic T12486
+ */
+import { percentile } from './stats.js';
+/** Ratio, or `null` when the denominator is zero. */
+function ratio(num, den) {
+    return den > 0 ? num / den : null;
+}
+/** Harmonic mean of precision and recall. */
+function f1Of(p, r) {
+    if (p === null || r === null)
+        return null;
+    return p + r > 0 ? (2 * p * r) / (p + r) : 0;
+}
+/** One-vs-rest counts for `label`. */
+function counts(preds, label) {
+    let tp = 0;
+    let fp = 0;
+    let fn = 0;
+    let tn = 0;
+    for (const p of preds) {
+        const g = p.gold === label;
+        const q = p.predicted === label;
+        if (g && q)
+            tp++;
+        else if (!g && q)
+            fp++;
+        else if (g && !q)
+            fn++;
+        else
+            tn++;
+    }
+    return { tp, fp, fn, tn };
+}
+/** Mean of the non-null values, or null. */
+function meanOf(values) {
+    const xs = values.filter((v) => v !== null);
+    return xs.length > 0 ? xs.reduce((s, x) => s + x, 0) / xs.length : null;
+}
+/**
+ * Score predictions.
+ *
+ * @param preds - Gold and predicted labels.
+ * @param positive - The positive class of a two-label site; `null` → macro averages.
+ * @returns The metrics (`null` where undefined, e.g. no positives predicted).
+ */
+export function classificationMetrics(preds, positive) {
+    const n = preds.length;
+    const accuracy = ratio(preds.filter((p) => p.gold === p.predicted).length, n);
+    if (positive !== null) {
+        const c = counts(preds, positive);
+        const precision = ratio(c.tp, c.tp + c.fp);
+        const recall = ratio(c.tp, c.tp + c.fn);
+        return {
+            n,
+            accuracy,
+            precision,
+            recall,
+            f1: f1Of(precision, recall),
+            falsePositiveRate: ratio(c.fp, c.fp + c.tn),
+        };
+    }
+    const labels = [...new Set(preds.map((p) => p.gold))].sort();
+    const per = labels.map((label) => {
+        const c = counts(preds, label);
+        const precision = ratio(c.tp, c.tp + c.fp) ?? 0;
+        const recall = ratio(c.tp, c.tp + c.fn);
+        return {
+            precision,
+            recall,
+            f1: f1Of(precision, recall),
+            fpr: ratio(c.fp, c.fp + c.tn),
+        };
+    });
+    return {
+        n,
+        accuracy,
+        precision: meanOf(per.map((p) => p.precision)),
+        recall: meanOf(per.map((p) => p.recall)),
+        f1: meanOf(per.map((p) => p.f1)),
+        falsePositiveRate: meanOf(per.map((p) => p.fpr)),
+    };
+}
+/**
+ * Latency percentiles.
+ *
+ * @param samples - Latencies in ms.
+ * @returns p50 and p95 (nearest rank).
+ */
+export function latencyOf(samples) {
+    return { p50Ms: percentile(samples, 50), p95Ms: percentile(samples, 95) };
+}
+//# sourceMappingURL=metrics.js.map

package/dist/decide/bench/metrics.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"metrics.js","sourceRoot":"","sources":["../../../src/decide/bench/metrics.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AA0BxC,qDAAqD;AACrD,SAAS,KAAK,CAAC,GAAW,EAAE,GAAW;IACrC,OAAO,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC;AACpC,CAAC;AAED,6CAA6C;AAC7C,SAAS,IAAI,CAAC,CAAgB,EAAE,CAAgB;IAC9C,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,IAAI;QAAE,OAAO,IAAI,CAAC;IAC1C,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAC/C,CAAC;AAED,sCAAsC;AACtC,SAAS,MAAM,CAAC,KAAiC,EAAE,KAAa;IAC9D,IAAI,EAAE,GAAG,CAAC,CAAC;IACX,IAAI,EAAE,GAAG,CAAC,CAAC;IACX,IAAI,EAAE,GAAG,CAAC,CAAC;IACX,IAAI,EAAE,GAAG,CAAC,CAAC;IACX,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC;QAC3B,MAAM,CAAC,GAAG,CAAC,CAAC,SAAS,KAAK,KAAK,CAAC;QAChC,IAAI,CAAC,IAAI,CAAC;YAAE,EAAE,EAAE,CAAC;aACZ,IAAI,CAAC,CAAC,IAAI,CAAC;YAAE,EAAE,EAAE,CAAC;aAClB,IAAI,CAAC,IAAI,CAAC,CAAC;YAAE,EAAE,EAAE,CAAC;;YAClB,EAAE,EAAE,CAAC;IACZ,CAAC;IACD,OAAO,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC;AAC5B,CAAC;AAED,4CAA4C;AAC5C,SAAS,MAAM,CAAC,MAAkC;IAChD,MAAM,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;IACzD,OAAO,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;AAC1E,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,qBAAqB,CACnC,KAAiC,EACjC,QAAuB;IAEvB,MAAM,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC;IACvB,MAAM,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,SAAS,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAC9E,IAAI,QAAQ,KAAK,IAAI,EAAE,CAAC;QACtB,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QAClC,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;QAC3C,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;QACxC,OAAO;YACL,CAAC;YACD,QAAQ;YACR,SAAS;YACT,MAAM;YACN,EAAE,EAAE,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC;YAC3B,iBAAiB,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC;SAC5C,CAAC;IACJ,CAAC;IACD,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAC7D,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QAC/B,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QAC/B,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;QAChD,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;QACxC,OAAO;YACL,SAAS;YACT,MAAM;YACN,EAAE,EAAE,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC;YAC3B,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC;SAC9B,CAAC;IACJ,CAAC,CAAC,CAAC;IACH,OAAO;QACL,CAAC;QACD,QAAQ;QACR,SAAS,EAAE,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QAC9C,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QACxC,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAChC,iBAAiB,EAAE,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;KACjD,CAAC;AACJ,CAAC;AAUD;;;;;GAKG;AACH,MAAM,UAAU,SAAS,CAAC,OAA0B;IAClD,OAAO,EAAE,KAAK,EAAE,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,EAAE,KAAK,EAAE,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,EAAE,CAAC;AAC5E,CAAC"}

package/dist/decide/bench/operation.d.ts

@@ -0,0 +1,119 @@
+/**
+ * `cleo decide bench` as one core operation (T12495): build or reuse the
+ * dataset, apply owner corrections, draw the spot-check sample and — unless
+ * `sampleOnly` — run the providers and write the results and the report.
+ *
+ * Files, all in `outDir` (default `<projectRoot>/.cleo/decide-bench/`, which
+ * git ignores):
+ *
+ * - `dataset.jsonl` — the labelled, redacted rows. Reused on later calls so
+ *   the owner's corrections stick; `rebuild` builds it again from the stores.
+ * - `spot-check.json` — the stratified sample for the owner.
+ * - `results.json` / `report.md` — written by a run.
+ *
+ * Dataset building, corrections and sampling never contact a provider.
+ *
+ * @task T12495
+ * @epic T12486
+ */
+import type { SpendLedger } from '../spend.js';
+import type { DecideFetch } from '../transport.js';
+import { type BenchDatasetSizes } from './dataset.js';
+import { type BenchProfileResolver } from './profiles.js';
+import { type BenchAggregateRow, type BenchSpendSummary } from './runner.js';
+import { type BenchCorrectionsReceipt } from './spot-check.js';
+import { type BenchConnection, type BenchSite, type BenchSource } from './types.js';
+/** Dataset file name inside the output directory. */
+export declare const BENCH_DATASET_FILE = "dataset.jsonl";
+/** Spot-check file name inside the output directory. */
+export declare const BENCH_SPOT_CHECK_FILE = "spot-check.json";
+/** Results file name inside the output directory. */
+export declare const BENCH_RESULTS_FILE = "results.json";
+/** Report fragment file name inside the output directory. */
+export declare const BENCH_REPORT_FILE = "report.md";
+/** Invalid benchmark input (maps to a validation error in the CLI). */
+export declare class DecideBenchInputError extends Error {
+    /** How to fix it. */
+    readonly fix: string;
+    /**
+     * @param message - What is wrong.
+     * @param fix - How to fix it.
+     */
+    constructor(message: string, fix: string);
+}
+/** Input of {@link runDecideBenchOperation}. */
+export interface DecideBenchInput {
+    /** Project whose stores are read. Default: the resolved project root. */
+    readonly projectRoot?: string;
+    /** Output directory. Default `<projectRoot>/.cleo/decide-bench`. */
+    readonly outDir?: string;
+    /** Profile names to compare (required unless `sampleOnly`). */
+    readonly profiles?: readonly string[];
+    /** Sites to sample and run. Default: all. */
+    readonly sites?: readonly string[];
+    /** Build the dataset and the sample only; no provider is contacted. */
+    readonly sampleOnly?: boolean;
+    /** Owner corrections file to apply before sampling. */
+    readonly correctionsPath?: string;
+    /** Hard total spend cap in US dollars. Default $5. */
+    readonly maxUsd?: number;
+    /** Repeated runs. Default 1. */
+    readonly runs?: number;
+    /** Rows per batch call. */
+    readonly batchSize?: number;
+    /** Rebuild the dataset from the stores even when one exists. */
+    readonly rebuild?: boolean;
+    /** PRNG seed for negatives and the sample. */
+    readonly seed?: number;
+    /** Spot-check sample size. Default 30. */
+    readonly sampleSize?: number;
+    /** Store port (tests). Default: the core accessors. */
+    readonly source?: BenchSource;
+    /** Profile resolver (tests; T12733 profiles later). Default: the interim adapter. */
+    readonly resolver?: BenchProfileResolver;
+    /** Explicit connections; skip profile resolution. */
+    readonly connections?: readonly BenchConnection[];
+    /** Transport (tests). */
+    readonly fetch?: DecideFetch;
+    /** Directory for per-connection provider-state files (tests). */
+    readonly providerStateDir?: string;
+    /** Monthly ledger to record spend in; `null` skips. Default: the file ledger. */
+    readonly spendLedger?: SpendLedger | null;
+}
+/** Output of {@link runDecideBenchOperation}. */
+export interface DecideBenchSummary {
+    /** Output directory. */
+    readonly outDir: string;
+    /** Files written or reused. */
+    readonly files: {
+        readonly dataset: string;
+        readonly spotCheck: string;
+        readonly results?: string;
+        readonly report?: string;
+    };
+    /** Whether an existing dataset was reused. */
+    readonly datasetReused: boolean;
+    /** Dataset sizes (all sites). */
+    readonly dataset: BenchDatasetSizes;
+    /** Sites sampled and run. */
+    readonly sites: readonly BenchSite[];
+    /** Items in the spot-check sample. */
+    readonly spotCheckItems: number;
+    /** Applied corrections, when a file was given. */
+    readonly corrections?: BenchCorrectionsReceipt;
+    /** Whether providers were run. */
+    readonly ran: boolean;
+    /** Spend, when run. */
+    readonly spend?: BenchSpendSummary;
+    /** Mean and spread per provider × site, when run. */
+    readonly aggregate?: readonly BenchAggregateRow[];
+}
+/**
+ * Run `cleo decide bench`.
+ *
+ * @param input - Flags and wiring.
+ * @returns What was written and, after a run, the spend and the aggregate.
+ * @throws DecideBenchInputError on invalid input (unknown site or profile, no profiles).
+ */
+export declare function runDecideBenchOperation(input?: DecideBenchInput): Promise<DecideBenchSummary>;
+//# sourceMappingURL=operation.d.ts.map

package/dist/decide/bench/operation.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"operation.d.ts","sourceRoot":"","sources":["../../../src/decide/bench/operation.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAMH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC/C,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AACnD,OAAO,EACL,KAAK,iBAAiB,EAKvB,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,KAAK,oBAAoB,EAAwB,MAAM,eAAe,CAAC;AAEhF,OAAO,EACL,KAAK,iBAAiB,EACtB,KAAK,iBAAiB,EAGvB,MAAM,aAAa,CAAC;AAErB,OAAO,EAEL,KAAK,uBAAuB,EAG7B,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EAAe,KAAK,eAAe,EAAE,KAAK,SAAS,EAAE,KAAK,WAAW,EAAE,MAAM,YAAY,CAAC;AAEjG,qDAAqD;AACrD,eAAO,MAAM,kBAAkB,kBAAkB,CAAC;AAElD,wDAAwD;AACxD,eAAO,MAAM,qBAAqB,oBAAoB,CAAC;AAEvD,qDAAqD;AACrD,eAAO,MAAM,kBAAkB,iBAAiB,CAAC;AAEjD,6DAA6D;AAC7D,eAAO,MAAM,iBAAiB,cAAc,CAAC;AAE7C,uEAAuE;AACvE,qBAAa,qBAAsB,SAAQ,KAAK;IAC9C,qBAAqB;IACrB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IAErB;;;OAGG;gBACS,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM;CAKzC;AAED,gDAAgD;AAChD,MAAM,WAAW,gBAAgB;IAC/B,yEAAyE;IACzE,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,oEAAoE;IACpE,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB,+DAA+D;IAC/D,QAAQ,CAAC,QAAQ,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACtC,6CAA6C;IAC7C,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,uEAAuE;IACvE,QAAQ,CAAC,UAAU,CAAC,EAAE,OAAO,CAAC;IAC9B,uDAAuD;IACvD,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,CAAC;IAClC,sDAAsD;IACtD,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB,gCAAgC;IAChC,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,2BAA2B;IAC3B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,gEAAgE;IAChE,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC;IAC3B,8CAA8C;IAC9C,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,0CAA0C;IAC1C,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B,uDAAuD;IACvD,QAAQ,CAAC,MAAM,CAAC,EAAE,WAAW,CAAC;IAC9B,qFAAqF;IACrF,QAAQ,CAAC,QAAQ,CAAC,EAAE,oBAAoB,CAAC;IACzC,qDAAqD;IACrD,QAAQ,CAAC,WAAW,CAAC,EAAE,SAAS,eAAe,EAAE,CAAC;IAClD,yBAAyB;IACzB,QAAQ,CAAC,KAAK,CAAC,EAAE,WAAW,CAAC;IAC7B,iEAAiE;IACjE,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IACnC,iFAAiF;IACjF,QAAQ,CAAC,WAAW,CAAC,EAAE,WAAW,GAAG,IAAI,CAAC;CAC3C;AAED,iDAAiD;AACjD,MAAM,WAAW,kBAAkB;IACjC,wBAAwB;IACxB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,+BAA+B;IAC/B,QAAQ,CAAC,KAAK,EAAE;QACd,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;QACzB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;QAC3B,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;QAC1B,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;KAC1B,CAAC;IACF,8CAA8C;IAC9C,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC;IAChC,iCAAiC;IACjC,QAAQ,CAAC,OAAO,EAAE,iBAAiB,CAAC;IACpC,6BAA6B;IAC7B,QAAQ,CAAC,KAAK,EAAE,SAAS,SAAS,EAAE,CAAC;IACrC,sCAAsC;IACtC,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,kDAAkD;IAClD,QAAQ,CAAC,WAAW,CAAC,EAAE,uBAAuB,CAAC;IAC/C,kCAAkC;IAClC,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAC;IACtB,uBAAuB;IACvB,QAAQ,CAAC,KAAK,CAAC,EAAE,iBAAiB,CAAC;IACnC,qDAAqD;IACrD,QAAQ,CAAC,SAAS,CAAC,EAAE,SAAS,iBAAiB,EAAE,CAAC;CACnD;AAyBD;;;;;;GAMG;AACH,wBAAsB,uBAAuB,CAC3C,KAAK,GAAE,gBAAqB,GAC3B,OAAO,CAAC,kBAAkB,CAAC,CAgG7B"}

package/dist/decide/bench/operation.js

@@ -0,0 +1,171 @@
+/**
+ * `cleo decide bench` as one core operation (T12495): build or reuse the
+ * dataset, apply owner corrections, draw the spot-check sample and — unless
+ * `sampleOnly` — run the providers and write the results and the report.
+ *
+ * Files, all in `outDir` (default `<projectRoot>/.cleo/decide-bench/`, which
+ * git ignores):
+ *
+ * - `dataset.jsonl` — the labelled, redacted rows. Reused on later calls so
+ *   the owner's corrections stick; `rebuild` builds it again from the stores.
+ * - `spot-check.json` — the stratified sample for the owner.
+ * - `results.json` / `report.md` — written by a run.
+ *
+ * Dataset building, corrections and sampling never contact a provider.
+ *
+ * @task T12495
+ * @epic T12486
+ */
+import { existsSync } from 'node:fs';
+import { mkdir, writeFile } from 'node:fs/promises';
+import { join, resolve } from 'node:path';
+import { getProjectRoot } from '../../paths.js';
+import { benchDatasetSizes, buildBenchDataset, readBenchDataset, writeBenchDataset, } from './dataset.js';
+import { resolveBenchProfiles } from './profiles.js';
+import { renderBenchReport } from './report.js';
+import { DEFAULT_BENCH_MAX_MICROS, runDecideBench, } from './runner.js';
+import { createStoreBenchSource } from './source.js';
+import { applyBenchCorrections, readBenchCorrections, sampleBenchSpotCheck, } from './spot-check.js';
+import { DEFAULT_BENCH_SEED } from './stats.js';
+import { BENCH_SITES } from './types.js';
+/** Dataset file name inside the output directory. */
+export const BENCH_DATASET_FILE = 'dataset.jsonl';
+/** Spot-check file name inside the output directory. */
+export const BENCH_SPOT_CHECK_FILE = 'spot-check.json';
+/** Results file name inside the output directory. */
+export const BENCH_RESULTS_FILE = 'results.json';
+/** Report fragment file name inside the output directory. */
+export const BENCH_REPORT_FILE = 'report.md';
+/** Invalid benchmark input (maps to a validation error in the CLI). */
+export class DecideBenchInputError extends Error {
+    /** How to fix it. */
+    fix;
+    /**
+     * @param message - What is wrong.
+     * @param fix - How to fix it.
+     */
+    constructor(message, fix) {
+        super(message);
+        this.name = 'DecideBenchInputError';
+        this.fix = fix;
+    }
+}
+/** Narrow the requested sites. */
+function parseSites(sites) {
+    if (!sites || sites.length === 0)
+        return [...BENCH_SITES];
+    const out = [];
+    for (const s of sites) {
+        const site = BENCH_SITES.find((b) => b === s.trim());
+        if (!site) {
+            throw new DecideBenchInputError(`unknown site '${s}'`, `--sites ${BENCH_SITES.join(',')}`);
+        }
+        if (!out.includes(site))
+            out.push(site);
+    }
+    return out;
+}
+/** Validate a positive number flag. */
+function positive(value, name, fallback) {
+    if (value === undefined)
+        return fallback;
+    if (!Number.isFinite(value) || value <= 0) {
+        throw new DecideBenchInputError(`${name} must be a positive number`, `${name} <n>`);
+    }
+    return value;
+}
+/**
+ * Run `cleo decide bench`.
+ *
+ * @param input - Flags and wiring.
+ * @returns What was written and, after a run, the spend and the aggregate.
+ * @throws DecideBenchInputError on invalid input (unknown site or profile, no profiles).
+ */
+export async function runDecideBenchOperation(input = {}) {
+    const sites = parseSites(input.sites);
+    const maxUsd = positive(input.maxUsd, '--max-usd', DEFAULT_BENCH_MAX_MICROS / 1e6);
+    const runs = Math.floor(positive(input.runs, '--runs', 1));
+    const batchSize = input.batchSize === undefined
+        ? undefined
+        : Math.floor(positive(input.batchSize, '--batch-size', 1));
+    const seed = input.seed ?? DEFAULT_BENCH_SEED;
+    if (!Number.isSafeInteger(seed)) {
+        throw new DecideBenchInputError('--seed must be an integer', '--seed 12495');
+    }
+    const wantsRun = input.sampleOnly !== true;
+    const profiles = input.profiles ?? [];
+    if (wantsRun && profiles.length === 0 && !input.connections?.length) {
+        throw new DecideBenchInputError('no providers to compare', 'cleo decide bench --profiles layahost,jev (or --sample-only to build the dataset and sample without spend)');
+    }
+    // Resolve profiles before any file is written, so a typo fails fast.
+    let connections = input.connections ?? [];
+    if (wantsRun && connections.length === 0) {
+        try {
+            connections = resolveBenchProfiles(profiles, input.resolver);
+        }
+        catch (err) {
+            throw new DecideBenchInputError(err instanceof Error ? err.message : 'profile resolution failed', 'CLEO_DECIDE_PROFILE_<NAME>_KEY=… (and _URL to override the preset URL)');
+        }
+    }
+    const projectRoot = input.projectRoot ?? getProjectRoot();
+    const outDir = resolve(input.outDir ?? join(projectRoot, '.cleo', 'decide-bench'));
+    await mkdir(outDir, { recursive: true, mode: 0o700 });
+    const datasetPath = join(outDir, BENCH_DATASET_FILE);
+    const spotCheckPath = join(outDir, BENCH_SPOT_CHECK_FILE);
+    const reuse = input.rebuild !== true && existsSync(datasetPath);
+    let rows = reuse
+        ? await readBenchDataset(datasetPath)
+        : await buildBenchDataset(input.source ?? createStoreBenchSource(projectRoot), { seed });
+    let corrections;
+    if (input.correctionsPath) {
+        const applied = applyBenchCorrections(rows, await readBenchCorrections(input.correctionsPath));
+        rows = applied.rows;
+        corrections = applied.receipt;
+    }
+    if (!reuse || corrections)
+        await writeBenchDataset(datasetPath, rows);
+    const selected = rows.filter((r) => sites.includes(r.site));
+    const spotCheck = sampleBenchSpotCheck(selected, {
+        seed,
+        ...(input.sampleSize !== undefined ? { size: input.sampleSize } : {}),
+    });
+    await writeFile(spotCheckPath, `${JSON.stringify(spotCheck, null, 2)}\n`, { mode: 0o600 });
+    const base = {
+        outDir,
+        datasetReused: reuse,
+        dataset: benchDatasetSizes(rows),
+        sites,
+        spotCheckItems: spotCheck.items.length,
+        ...(corrections ? { corrections } : {}),
+    };
+    if (!wantsRun) {
+        return { ...base, files: { dataset: datasetPath, spotCheck: spotCheckPath }, ran: false };
+    }
+    const results = await runDecideBench({
+        rows: selected,
+        connections,
+        runs,
+        maxMicros: Math.round(maxUsd * 1e6),
+        ...(batchSize !== undefined ? { batchSize } : {}),
+        ...(input.fetch ? { fetch: input.fetch } : {}),
+        ...(input.providerStateDir ? { providerStateDir: input.providerStateDir } : {}),
+        ...(input.spendLedger !== undefined ? { spendLedger: input.spendLedger } : {}),
+    });
+    const resultsPath = join(outDir, BENCH_RESULTS_FILE);
+    const reportPath = join(outDir, BENCH_REPORT_FILE);
+    await writeFile(resultsPath, `${JSON.stringify(results, null, 2)}\n`, { mode: 0o600 });
+    await writeFile(reportPath, renderBenchReport(results), { mode: 0o600 });
+    return {
+        ...base,
+        files: {
+            dataset: datasetPath,
+            spotCheck: spotCheckPath,
+            results: resultsPath,
+            report: reportPath,
+        },
+        ran: true,
+        spend: results.spend,
+        aggregate: results.aggregate,
+    };
+}
+//# sourceMappingURL=operation.js.map

package/dist/decide/bench/operation.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"operation.js","sourceRoot":"","sources":["../../../src/decide/bench/operation.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAGhD,OAAO,EAEL,iBAAiB,EACjB,iBAAiB,EACjB,gBAAgB,EAChB,iBAAiB,GAClB,MAAM,cAAc,CAAC;AACtB,OAAO,EAA6B,oBAAoB,EAAE,MAAM,eAAe,CAAC;AAChF,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAChD,OAAO,EAGL,wBAAwB,EACxB,cAAc,GACf,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,sBAAsB,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,EACL,qBAAqB,EAErB,oBAAoB,EACpB,oBAAoB,GACrB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAChD,OAAO,EAAE,WAAW,EAA0D,MAAM,YAAY,CAAC;AAEjG,qDAAqD;AACrD,MAAM,CAAC,MAAM,kBAAkB,GAAG,eAAe,CAAC;AAElD,wDAAwD;AACxD,MAAM,CAAC,MAAM,qBAAqB,GAAG,iBAAiB,CAAC;AAEvD,qDAAqD;AACrD,MAAM,CAAC,MAAM,kBAAkB,GAAG,cAAc,CAAC;AAEjD,6DAA6D;AAC7D,MAAM,CAAC,MAAM,iBAAiB,GAAG,WAAW,CAAC;AAE7C,uEAAuE;AACvE,MAAM,OAAO,qBAAsB,SAAQ,KAAK;IAC9C,qBAAqB;IACZ,GAAG,CAAS;IAErB;;;OAGG;IACH,YAAY,OAAe,EAAE,GAAW;QACtC,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,uBAAuB,CAAC;QACpC,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;IACjB,CAAC;CACF;AAuED,kCAAkC;AAClC,SAAS,UAAU,CAAC,KAAoC;IACtD,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,GAAG,WAAW,CAAC,CAAC;IAC1D,MAAM,GAAG,GAAgB,EAAE,CAAC;IAC5B,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QACrD,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,MAAM,IAAI,qBAAqB,CAAC,iBAAiB,CAAC,GAAG,EAAE,WAAW,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC7F,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC;YAAE,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1C,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,uCAAuC;AACvC,SAAS,QAAQ,CAAC,KAAyB,EAAE,IAAY,EAAE,QAAgB;IACzE,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,QAAQ,CAAC;IACzC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;QAC1C,MAAM,IAAI,qBAAqB,CAAC,GAAG,IAAI,4BAA4B,EAAE,GAAG,IAAI,MAAM,CAAC,CAAC;IACtF,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,QAA0B,EAAE;IAE5B,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACtC,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,MAAM,EAAE,WAAW,EAAE,wBAAwB,GAAG,GAAG,CAAC,CAAC;IACnF,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC;IAC3D,MAAM,SAAS,GACb,KAAK,CAAC,SAAS,KAAK,SAAS;QAC3B,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,SAAS,EAAE,cAAc,EAAE,CAAC,CAAC,CAAC,CAAC;IAC/D,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,IAAI,kBAAkB,CAAC;IAC9C,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC;QAChC,MAAM,IAAI,qBAAqB,CAAC,2BAA2B,EAAE,cAAc,CAAC,CAAC;IAC/E,CAAC;IACD,MAAM,QAAQ,GAAG,KAAK,CAAC,UAAU,KAAK,IAAI,CAAC;IAC3C,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,IAAI,EAAE,CAAC;IACtC,IAAI,QAAQ,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,MAAM,EAAE,CAAC;QACpE,MAAM,IAAI,qBAAqB,CAC7B,yBAAyB,EACzB,4GAA4G,CAC7G,CAAC;IACJ,CAAC;IACD,qEAAqE;IACrE,IAAI,WAAW,GAA+B,KAAK,CAAC,WAAW,IAAI,EAAE,CAAC;IACtE,IAAI,QAAQ,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzC,IAAI,CAAC;YACH,WAAW,GAAG,oBAAoB,CAAC,QAAQ,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC/D,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,IAAI,qBAAqB,CAC7B,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,2BAA2B,EAChE,wEAAwE,CACzE,CAAC;QACJ,CAAC;IACH,CAAC;IAED,MAAM,WAAW,GAAG,KAAK,CAAC,WAAW,IAAI,cAAc,EAAE,CAAC;IAC1D,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,IAAI,IAAI,CAAC,WAAW,EAAE,OAAO,EAAE,cAAc,CAAC,CAAC,CAAC;IACnF,MAAM,KAAK,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IACtD,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAC;IACrD,MAAM,aAAa,GAAG,IAAI,CAAC,MAAM,EAAE,qBAAqB,CAAC,CAAC;IAE1D,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,KAAK,IAAI,IAAI,UAAU,CAAC,WAAW,CAAC,CAAC;IAChE,IAAI,IAAI,GAAG,KAAK;QACd,CAAC,CAAC,MAAM,gBAAgB,CAAC,WAAW,CAAC;QACrC,CAAC,CAAC,MAAM,iBAAiB,CAAC,KAAK,CAAC,MAAM,IAAI,sBAAsB,CAAC,WAAW,CAAC,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;IAC3F,IAAI,WAAgD,CAAC;IACrD,IAAI,KAAK,CAAC,eAAe,EAAE,CAAC;QAC1B,MAAM,OAAO,GAAG,qBAAqB,CAAC,IAAI,EAAE,MAAM,oBAAoB,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC;QAC/F,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;QACpB,WAAW,GAAG,OAAO,CAAC,OAAO,CAAC;IAChC,CAAC;IACD,IAAI,CAAC,KAAK,IAAI,WAAW;QAAE,MAAM,iBAAiB,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;IAEtE,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAC5D,MAAM,SAAS,GAAG,oBAAoB,CAAC,QAAQ,EAAE;QAC/C,IAAI;QACJ,GAAG,CAAC,KAAK,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACtE,CAAC,CAAC;IACH,MAAM,SAAS,CAAC,aAAa,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IAE3F,MAAM,IAAI,GAAG;QACX,MAAM;QACN,aAAa,EAAE,KAAK;QACpB,OAAO,EAAE,iBAAiB,CAAC,IAAI,CAAC;QAChC,KAAK;QACL,cAAc,EAAE,SAAS,CAAC,KAAK,CAAC,MAAM;QACtC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACxC,CAAC;IACF,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,OAAO,EAAE,GAAG,IAAI,EAAE,KAAK,EAAE,EAAE,OAAO,EAAE,WAAW,EAAE,SAAS,EAAE,aAAa,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC;IAC5F,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC;QACnC,IAAI,EAAE,QAAQ;QACd,WAAW;QACX,IAAI;QACJ,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,GAAG,CAAC;QACnC,GAAG,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACjD,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9C,GAAG,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,KAAK,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/E,GAAG,CAAC,KAAK,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC/E,CAAC,CAAC;IACH,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAC;IACrD,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,EAAE,iBAAiB,CAAC,CAAC;IACnD,MAAM,SAAS,CAAC,WAAW,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IACvF,MAAM,SAAS,CAAC,UAAU,EAAE,iBAAiB,CAAC,OAAO,CAAC,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IACzE,OAAO;QACL,GAAG,IAAI;QACP,KAAK,EAAE;YACL,OAAO,EAAE,WAAW;YACpB,SAAS,EAAE,aAAa;YACxB,OAAO,EAAE,WAAW;YACpB,MAAM,EAAE,UAAU;SACnB;QACD,GAAG,EAAE,IAAI;QACT,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,SAAS,EAAE,OAAO,CAAC,SAAS;KAC7B,CAAC;AACJ,CAAC"}

package/dist/decide/bench/profiles.d.ts

@@ -0,0 +1,74 @@
+/**
+ * Resolve `--profiles a,b` to benchmark connections (T12495).
+ *
+ * Named decide profiles are being built in core by T12733, which will export
+ * `resolveDecideProfile(name)` and `listDecideProfiles()`. Until it lands,
+ * the benchmark resolves names through this small adapter, behind the
+ * {@link BenchProfileResolver} interface so the switch is one function:
+ *
+ * 1. `CLEO_DECIDE_PROFILE_<NAME>_KEY` (plus `_URL`, `_MODEL`, `_PROVIDER`) in
+ *    the environment — `<NAME>` upper-cased, non-alphanumerics as `_`. A
+ *    profile whose provider has a preset URL (layahost, jev) needs only the
+ *    key; `_URL` overrides the preset.
+ * 2. Otherwise, a name equal to the stored connection's provider kind
+ *    (`cleo decide config`), or `default`, resolves to the stored connection.
+ *
+ * @task T12495
+ * @epic T12486
+ */
+import { loadDecideConnection } from '../credentials.js';
+import type { BenchConnection } from './types.js';
+/** Resolves one profile name to a connection. */
+export interface BenchProfileResolver {
+    /**
+     * @param name - Profile name.
+     * @returns The connection, or `null` when no profile has that name.
+     * @throws BenchProfileInvalidError when the profile exists but is misconfigured.
+     */
+    resolve(name: string): BenchConnection | null;
+}
+/** A profile name that resolved to nothing. */
+export declare class BenchProfileError extends Error {
+    /** The names that did not resolve. */
+    readonly missing: readonly string[];
+    /**
+     * @param missing - Unresolved names.
+     */
+    constructor(missing: readonly string[]);
+}
+/** A profile that exists but is misconfigured (bad URL or model). */
+export declare class BenchProfileInvalidError extends Error {
+    /** The profile name. */
+    readonly profile: string;
+    /**
+     * @param profile - Profile name.
+     * @param problem - What is wrong, e.g. `invalid URL "…"`.
+     */
+    constructor(profile: string, problem: string);
+}
+/**
+ * Environment variable prefix for one profile.
+ *
+ * @param name - Profile name.
+ * @returns `CLEO_DECIDE_PROFILE_<NAME>` (upper-cased, non-alphanumerics as `_`).
+ */
+export declare function benchProfileEnvPrefix(name: string): string;
+/**
+ * The interim resolver (environment, then the stored connection).
+ *
+ * @param env - Environment to read. Default `process.env`.
+ * @param stored - Stored-connection loader. Default `loadDecideConnection`.
+ * @returns A resolver.
+ */
+export declare function createInterimProfileResolver(env?: NodeJS.ProcessEnv, stored?: typeof loadDecideConnection): BenchProfileResolver;
+/**
+ * Resolve every name, or throw naming all that did not resolve.
+ *
+ * @param names - Profile names (duplicates removed, order kept).
+ * @param resolver - Resolver. Default: {@link createInterimProfileResolver}.
+ * @returns One connection per name.
+ * @throws BenchProfileError when any name is unknown.
+ * @throws BenchProfileInvalidError when a profile has a bad URL or model.
+ */
+export declare function resolveBenchProfiles(names: readonly string[], resolver?: BenchProfileResolver): BenchConnection[];
+//# sourceMappingURL=profiles.d.ts.map

package/dist/decide/bench/profiles.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"profiles.d.ts","sourceRoot":"","sources":["../../../src/decide/bench/profiles.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAGH,OAAO,EAA0B,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AAGjF,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAElD,iDAAiD;AACjD,MAAM,WAAW,oBAAoB;IACnC;;;;OAIG;IACH,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,eAAe,GAAG,IAAI,CAAC;CAC/C;AAED,+CAA+C;AAC/C,qBAAa,iBAAkB,SAAQ,KAAK;IAC1C,sCAAsC;IACtC,QAAQ,CAAC,OAAO,EAAE,SAAS,MAAM,EAAE,CAAC;IAEpC;;OAEG;gBACS,OAAO,EAAE,SAAS,MAAM,EAAE;CAOvC;AAED,qEAAqE;AACrE,qBAAa,wBAAyB,SAAQ,KAAK;IACjD,wBAAwB;IACxB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IAEzB;;;OAGG;gBACS,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM;CAK7C;AAED;;;;;GAKG;AACH,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAE1D;AAED;;;;;;GAMG;AACH,wBAAgB,4BAA4B,CAC1C,GAAG,GAAE,MAAM,CAAC,UAAwB,EACpC,MAAM,GAAE,OAAO,oBAA2C,GACzD,oBAAoB,CA2CtB;AAED;;;;;;;;GAQG;AACH,wBAAgB,oBAAoB,CAClC,KAAK,EAAE,SAAS,MAAM,EAAE,EACxB,QAAQ,GAAE,oBAAqD,GAC9D,eAAe,EAAE,CAWnB"}

package/dist/decide/bench/profiles.js

@@ -0,0 +1,131 @@
+/**
+ * Resolve `--profiles a,b` to benchmark connections (T12495).
+ *
+ * Named decide profiles are being built in core by T12733, which will export
+ * `resolveDecideProfile(name)` and `listDecideProfiles()`. Until it lands,
+ * the benchmark resolves names through this small adapter, behind the
+ * {@link BenchProfileResolver} interface so the switch is one function:
+ *
+ * 1. `CLEO_DECIDE_PROFILE_<NAME>_KEY` (plus `_URL`, `_MODEL`, `_PROVIDER`) in
+ *    the environment — `<NAME>` upper-cased, non-alphanumerics as `_`. A
+ *    profile whose provider has a preset URL (layahost, jev) needs only the
+ *    key; `_URL` overrides the preset.
+ * 2. Otherwise, a name equal to the stored connection's provider kind
+ *    (`cleo decide config`), or `default`, resolves to the stored connection.
+ *
+ * @task T12495
+ * @epic T12486
+ */
+import { DECISION_PROVIDER_KINDS } from '@cleocode/contracts';
+import { isAllowedDecideBaseUrl, loadDecideConnection } from '../credentials.js';
+import { isValidDecisionModelName } from '../jev-wire.js';
+import { DECISION_PROVIDER_PRESETS, parseDecisionProviderKind } from '../providers.js';
+/** A profile name that resolved to nothing. */
+export class BenchProfileError extends Error {
+    /** The names that did not resolve. */
+    missing;
+    /**
+     * @param missing - Unresolved names.
+     */
+    constructor(missing) {
+        super(`unknown decide profile(s): ${missing.join(', ')}. Set CLEO_DECIDE_PROFILE_<NAME>_KEY (and _URL to override the provider's preset URL), or store one with cleo decide config and name its provider kind.`);
+        this.name = 'BenchProfileError';
+        this.missing = missing;
+    }
+}
+/** A profile that exists but is misconfigured (bad URL or model). */
+export class BenchProfileInvalidError extends Error {
+    /** The profile name. */
+    profile;
+    /**
+     * @param profile - Profile name.
+     * @param problem - What is wrong, e.g. `invalid URL "…"`.
+     */
+    constructor(profile, problem) {
+        super(`decide profile '${profile}': ${problem}`);
+        this.name = 'BenchProfileInvalidError';
+        this.profile = profile;
+    }
+}
+/**
+ * Environment variable prefix for one profile.
+ *
+ * @param name - Profile name.
+ * @returns `CLEO_DECIDE_PROFILE_<NAME>` (upper-cased, non-alphanumerics as `_`).
+ */
+export function benchProfileEnvPrefix(name) {
+    return `CLEO_DECIDE_PROFILE_${name.toUpperCase().replace(/[^A-Z0-9]/g, '_')}`;
+}
+/**
+ * The interim resolver (environment, then the stored connection).
+ *
+ * @param env - Environment to read. Default `process.env`.
+ * @param stored - Stored-connection loader. Default `loadDecideConnection`.
+ * @returns A resolver.
+ */
+export function createInterimProfileResolver(env = process.env, stored = loadDecideConnection) {
+    return {
+        resolve(name) {
+            const prefix = benchProfileEnvPrefix(name);
+            const key = env[`${prefix}_KEY`]?.trim();
+            if (key) {
+                const kind = parseDecisionProviderKind(env[`${prefix}_PROVIDER`]) ??
+                    parseDecisionProviderKind(name) ??
+                    'jev';
+                const preset = DECISION_PROVIDER_PRESETS[kind];
+                const baseUrl = env[`${prefix}_URL`]?.trim() || preset.defaultBaseUrl;
+                if (!baseUrl) {
+                    throw new BenchProfileInvalidError(name, `no URL (set ${prefix}_URL)`);
+                }
+                if (!isAllowedDecideBaseUrl(baseUrl)) {
+                    throw new BenchProfileInvalidError(name, `invalid URL in ${prefix}_URL (https://, or http:// to a loopback host)`);
+                }
+                const model = env[`${prefix}_MODEL`]?.trim() || preset.defaultModel;
+                if (model !== undefined && !isValidDecisionModelName(model)) {
+                    throw new BenchProfileInvalidError(name, `invalid model name in ${prefix}_MODEL`);
+                }
+                return { name, provider: kind, baseUrl, apiKey: key, ...(model ? { model } : {}) };
+            }
+            const sealed = stored();
+            if (!sealed)
+                return null;
+            const matches = name === 'default' ||
+                (DECISION_PROVIDER_KINDS.some((k) => k === name) && sealed.provider === name);
+            if (!matches)
+                return null;
+            const c = sealed.connection();
+            return {
+                name,
+                provider: sealed.provider,
+                baseUrl: c.baseUrl,
+                apiKey: c.apiKey,
+                ...(c.model ? { model: c.model } : {}),
+            };
+        },
+    };
+}
+/**
+ * Resolve every name, or throw naming all that did not resolve.
+ *
+ * @param names - Profile names (duplicates removed, order kept).
+ * @param resolver - Resolver. Default: {@link createInterimProfileResolver}.
+ * @returns One connection per name.
+ * @throws BenchProfileError when any name is unknown.
+ * @throws BenchProfileInvalidError when a profile has a bad URL or model.
+ */
+export function resolveBenchProfiles(names, resolver = createInterimProfileResolver()) {
+    const unique = [...new Set(names.map((n) => n.trim()).filter((n) => n !== ''))];
+    const out = [];
+    const missing = [];
+    for (const name of unique) {
+        const c = resolver.resolve(name);
+        if (c)
+            out.push(c);
+        else
+            missing.push(name);
+    }
+    if (missing.length > 0)
+        throw new BenchProfileError(missing);
+    return out;
+}
+//# sourceMappingURL=profiles.js.map

package/dist/decide/bench/profiles.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"profiles.js","sourceRoot":"","sources":["../../../src/decide/bench/profiles.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAC;AAC9D,OAAO,EAAE,sBAAsB,EAAE,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AACjF,OAAO,EAAE,wBAAwB,EAAE,MAAM,gBAAgB,CAAC;AAC1D,OAAO,EAAE,yBAAyB,EAAE,yBAAyB,EAAE,MAAM,iBAAiB,CAAC;AAavF,+CAA+C;AAC/C,MAAM,OAAO,iBAAkB,SAAQ,KAAK;IAC1C,sCAAsC;IAC7B,OAAO,CAAoB;IAEpC;;OAEG;IACH,YAAY,OAA0B;QACpC,KAAK,CACH,8BAA8B,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,yJAAyJ,CAC1M,CAAC;QACF,IAAI,CAAC,IAAI,GAAG,mBAAmB,CAAC;QAChC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IACzB,CAAC;CACF;AAED,qEAAqE;AACrE,MAAM,OAAO,wBAAyB,SAAQ,KAAK;IACjD,wBAAwB;IACf,OAAO,CAAS;IAEzB;;;OAGG;IACH,YAAY,OAAe,EAAE,OAAe;QAC1C,KAAK,CAAC,mBAAmB,OAAO,MAAM,OAAO,EAAE,CAAC,CAAC;QACjD,IAAI,CAAC,IAAI,GAAG,0BAA0B,CAAC;QACvC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;IACzB,CAAC;CACF;AAED;;;;;GAKG;AACH,MAAM,UAAU,qBAAqB,CAAC,IAAY;IAChD,OAAO,uBAAuB,IAAI,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,YAAY,EAAE,GAAG,CAAC,EAAE,CAAC;AAChF,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,4BAA4B,CAC1C,MAAyB,OAAO,CAAC,GAAG,EACpC,SAAsC,oBAAoB;IAE1D,OAAO;QACL,OAAO,CAAC,IAAY;YAClB,MAAM,MAAM,GAAG,qBAAqB,CAAC,IAAI,CAAC,CAAC;YAC3C,MAAM,GAAG,GAAG,GAAG,CAAC,GAAG,MAAM,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC;YACzC,IAAI,GAAG,EAAE,CAAC;gBACR,MAAM,IAAI,GACR,yBAAyB,CAAC,GAAG,CAAC,GAAG,MAAM,WAAW,CAAC,CAAC;oBACpD,yBAAyB,CAAC,IAAI,CAAC;oBAC/B,KAAK,CAAC;gBACR,MAAM,MAAM,GAAG,yBAAyB,CAAC,IAAI,CAAC,CAAC;gBAC/C,MAAM,OAAO,GAAG,GAAG,CAAC,GAAG,MAAM,MAAM,CAAC,EAAE,IAAI,EAAE,IAAI,MAAM,CAAC,cAAc,CAAC;gBACtE,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,MAAM,IAAI,wBAAwB,CAAC,IAAI,EAAE,eAAe,MAAM,OAAO,CAAC,CAAC;gBACzE,CAAC;gBACD,IAAI,CAAC,sBAAsB,CAAC,OAAO,CAAC,EAAE,CAAC;oBACrC,MAAM,IAAI,wBAAwB,CAChC,IAAI,EACJ,kBAAkB,MAAM,gDAAgD,CACzE,CAAC;gBACJ,CAAC;gBACD,MAAM,KAAK,GAAG,GAAG,CAAC,GAAG,MAAM,QAAQ,CAAC,EAAE,IAAI,EAAE,IAAI,MAAM,CAAC,YAAY,CAAC;gBACpE,IAAI,KAAK,KAAK,SAAS,IAAI,CAAC,wBAAwB,CAAC,KAAK,CAAC,EAAE,CAAC;oBAC5D,MAAM,IAAI,wBAAwB,CAAC,IAAI,EAAE,yBAAyB,MAAM,QAAQ,CAAC,CAAC;gBACpF,CAAC;gBACD,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC;YACrF,CAAC;YACD,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC;YACxB,IAAI,CAAC,MAAM;gBAAE,OAAO,IAAI,CAAC;YACzB,MAAM,OAAO,GACX,IAAI,KAAK,SAAS;gBAClB,CAAC,uBAAuB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,IAAI,MAAM,CAAC,QAAQ,KAAK,IAAI,CAAC,CAAC;YAChF,IAAI,CAAC,OAAO;gBAAE,OAAO,IAAI,CAAC;YAC1B,MAAM,CAAC,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC;YAC9B,OAAO;gBACL,IAAI;gBACJ,QAAQ,EAAE,MAAM,CAAC,QAAQ;gBACzB,OAAO,EAAE,CAAC,CAAC,OAAO;gBAClB,MAAM,EAAE,CAAC,CAAC,MAAM;gBAChB,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACvC,CAAC;QACJ,CAAC;KACF,CAAC;AACJ,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,oBAAoB,CAClC,KAAwB,EACxB,WAAiC,4BAA4B,EAAE;IAE/D,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC;IAChF,MAAM,GAAG,GAAsB,EAAE,CAAC;IAClC,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;QAC1B,MAAM,CAAC,GAAG,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACjC,IAAI,CAAC;YAAE,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;;YACd,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC;IACD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;QAAE,MAAM,IAAI,iBAAiB,CAAC,OAAO,CAAC,CAAC;IAC7D,OAAO,GAAG,CAAC;AACb,CAAC"}