@cleocode/core 2026.9.23 → 2026.9.25

package/dist/decide/bench/index.d.ts

@@ -0,0 +1,23 @@
+/**
+ * System One accuracy benchmark (`cleo decide bench`): a labelled dataset
+ * from CLEO's own history, an owner spot-check, and a capped, repeated
+ * comparison of providers against each site's heuristic.
+ *
+ * Loaded on demand (`@cleocode/core/decide/bench/index.js`); the decide
+ * barrel does not re-export it, so the CLI's startup never pays for it.
+ *
+ * @task T12495
+ * @epic T12486
+ */
+export { BENCH_TEXT_MAX_CHARS, type BenchDatasetSizes, type BuildBenchDatasetOptions, benchDatasetSizes, buildBenchDataset, DEFAULT_BENCH_MAX_ROWS_PER_SITE, DEFAULT_BENCH_NEGATIVES_PER_POSITIVE, leaksBenchLabel, legacyKeywordObservationType, MIN_BENCH_NEGATIVES, parseBenchDataset, readBenchDataset, serializeBenchDataset, writeBenchDataset, } from './dataset.js';
+export { type BenchClassificationMetrics, type BenchLatency, type BenchPrediction, classificationMetrics, latencyOf, } from './metrics.js';
+export { BENCH_DATASET_FILE, BENCH_REPORT_FILE, BENCH_RESULTS_FILE, BENCH_SPOT_CHECK_FILE, type DecideBenchInput, DecideBenchInputError, type DecideBenchSummary, runDecideBenchOperation, } from './operation.js';
+export { BenchProfileError, BenchProfileInvalidError, type BenchProfileResolver, type BenchProfileResolverOptions, type BenchProfileStore, benchProfileEnvPrefix, createProfileResolver, resolveBenchProfiles, } from './profiles.js';
+export { type BenchQuestion, benchQuestionFor } from './questions.js';
+export { renderBenchReport } from './report.js';
+export { type BenchAggregateRow, type BenchProviderRun, type BenchResults, type BenchRunRecord, type BenchSiteResult, type BenchSpendSummary, DEFAULT_BENCH_BATCH_SIZE, DEFAULT_BENCH_MAX_MICROS, HEURISTIC_PROVIDER, MAX_BENCH_BATCH_SIZE, MIN_BENCH_TIMEOUT_MS, type RunDecideBenchOptions, runDecideBench, } from './runner.js';
+export { createStoreBenchSource } from './source.js';
+export { applyBenchCorrections, type BenchCorrection, type BenchCorrectionsReceipt, type BenchSpotCheckFile, type BenchSpotCheckItem, DEFAULT_SPOT_CHECK_SIZE, parseBenchCorrections, readBenchCorrections, sampleBenchSpotCheck, } from './spot-check.js';
+export { type BenchRandom, type BenchSpread, createSeededRandom, DEFAULT_BENCH_SEED, percentile, shuffled, spreadOf, } from './stats.js';
+export { BENCH_LABEL_RULES, BENCH_SITES, type BenchConnection, type BenchDecisionRecord, type BenchLabelRule, type BenchObservationRecord, type BenchProvenance, type BenchRow, type BenchSite, type BenchSource, type BenchTaskRecord, benchRowSchema, CONTRADICTION_LABELS, DUPLICATE_LABELS, labelsForSite, positiveLabel, } from './types.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/decide/bench/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/decide/bench/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EACL,oBAAoB,EACpB,KAAK,iBAAiB,EACtB,KAAK,wBAAwB,EAC7B,iBAAiB,EACjB,iBAAiB,EACjB,+BAA+B,EAC/B,oCAAoC,EACpC,eAAe,EACf,4BAA4B,EAC5B,mBAAmB,EACnB,iBAAiB,EACjB,gBAAgB,EAChB,qBAAqB,EACrB,iBAAiB,GAClB,MAAM,cAAc,CAAC;AACtB,OAAO,EACL,KAAK,0BAA0B,EAC/B,KAAK,YAAY,EACjB,KAAK,eAAe,EACpB,qBAAqB,EACrB,SAAS,GACV,MAAM,cAAc,CAAC;AACtB,OAAO,EACL,kBAAkB,EAClB,iBAAiB,EACjB,kBAAkB,EAClB,qBAAqB,EACrB,KAAK,gBAAgB,EACrB,qBAAqB,EACrB,KAAK,kBAAkB,EACvB,uBAAuB,GACxB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EACL,iBAAiB,EACjB,wBAAwB,EACxB,KAAK,oBAAoB,EACzB,KAAK,2BAA2B,EAChC,KAAK,iBAAiB,EACtB,qBAAqB,EACrB,qBAAqB,EACrB,oBAAoB,GACrB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,KAAK,aAAa,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AACtE,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAChD,OAAO,EACL,KAAK,iBAAiB,EACtB,KAAK,gBAAgB,EACrB,KAAK,YAAY,EACjB,KAAK,cAAc,EACnB,KAAK,eAAe,EACpB,KAAK,iBAAiB,EACtB,wBAAwB,EACxB,wBAAwB,EACxB,kBAAkB,EAClB,oBAAoB,EACpB,oBAAoB,EACpB,KAAK,qBAAqB,EAC1B,cAAc,GACf,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,sBAAsB,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,EACL,qBAAqB,EACrB,KAAK,eAAe,EACpB,KAAK,uBAAuB,EAC5B,KAAK,kBAAkB,EACvB,KAAK,kBAAkB,EACvB,uBAAuB,EACvB,qBAAqB,EACrB,oBAAoB,EACpB,oBAAoB,GACrB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,KAAK,WAAW,EAChB,KAAK,WAAW,EAChB,kBAAkB,EAClB,kBAAkB,EAClB,UAAU,EACV,QAAQ,EACR,QAAQ,GACT,MAAM,YAAY,CAAC;AACpB,OAAO,EACL,iBAAiB,EACjB,WAAW,EACX,KAAK,eAAe,EACpB,KAAK,mBAAmB,EACxB,KAAK,cAAc,EACnB,KAAK,sBAAsB,EAC3B,KAAK,eAAe,EACpB,KAAK,QAAQ,EACb,KAAK,SAAS,EACd,KAAK,WAAW,EAChB,KAAK,eAAe,EACpB,cAAc,EACd,oBAAoB,EACpB,gBAAgB,EAChB,aAAa,EACb,aAAa,GACd,MAAM,YAAY,CAAC"}

package/dist/decide/bench/index.js

@@ -0,0 +1,23 @@
+/**
+ * System One accuracy benchmark (`cleo decide bench`): a labelled dataset
+ * from CLEO's own history, an owner spot-check, and a capped, repeated
+ * comparison of providers against each site's heuristic.
+ *
+ * Loaded on demand (`@cleocode/core/decide/bench/index.js`); the decide
+ * barrel does not re-export it, so the CLI's startup never pays for it.
+ *
+ * @task T12495
+ * @epic T12486
+ */
+export { BENCH_TEXT_MAX_CHARS, benchDatasetSizes, buildBenchDataset, DEFAULT_BENCH_MAX_ROWS_PER_SITE, DEFAULT_BENCH_NEGATIVES_PER_POSITIVE, leaksBenchLabel, legacyKeywordObservationType, MIN_BENCH_NEGATIVES, parseBenchDataset, readBenchDataset, serializeBenchDataset, writeBenchDataset, } from './dataset.js';
+export { classificationMetrics, latencyOf, } from './metrics.js';
+export { BENCH_DATASET_FILE, BENCH_REPORT_FILE, BENCH_RESULTS_FILE, BENCH_SPOT_CHECK_FILE, DecideBenchInputError, runDecideBenchOperation, } from './operation.js';
+export { BenchProfileError, BenchProfileInvalidError, benchProfileEnvPrefix, createProfileResolver, resolveBenchProfiles, } from './profiles.js';
+export { benchQuestionFor } from './questions.js';
+export { renderBenchReport } from './report.js';
+export { DEFAULT_BENCH_BATCH_SIZE, DEFAULT_BENCH_MAX_MICROS, HEURISTIC_PROVIDER, MAX_BENCH_BATCH_SIZE, MIN_BENCH_TIMEOUT_MS, runDecideBench, } from './runner.js';
+export { createStoreBenchSource } from './source.js';
+export { applyBenchCorrections, DEFAULT_SPOT_CHECK_SIZE, parseBenchCorrections, readBenchCorrections, sampleBenchSpotCheck, } from './spot-check.js';
+export { createSeededRandom, DEFAULT_BENCH_SEED, percentile, shuffled, spreadOf, } from './stats.js';
+export { BENCH_LABEL_RULES, BENCH_SITES, benchRowSchema, CONTRADICTION_LABELS, DUPLICATE_LABELS, labelsForSite, positiveLabel, } from './types.js';
+//# sourceMappingURL=index.js.map

package/dist/decide/bench/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/decide/bench/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EACL,oBAAoB,EAGpB,iBAAiB,EACjB,iBAAiB,EACjB,+BAA+B,EAC/B,oCAAoC,EACpC,eAAe,EACf,4BAA4B,EAC5B,mBAAmB,EACnB,iBAAiB,EACjB,gBAAgB,EAChB,qBAAqB,EACrB,iBAAiB,GAClB,MAAM,cAAc,CAAC;AACtB,OAAO,EAIL,qBAAqB,EACrB,SAAS,GACV,MAAM,cAAc,CAAC;AACtB,OAAO,EACL,kBAAkB,EAClB,iBAAiB,EACjB,kBAAkB,EAClB,qBAAqB,EAErB,qBAAqB,EAErB,uBAAuB,GACxB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EACL,iBAAiB,EACjB,wBAAwB,EAIxB,qBAAqB,EACrB,qBAAqB,EACrB,oBAAoB,GACrB,MAAM,eAAe,CAAC;AACvB,OAAO,EAAsB,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AACtE,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAChD,OAAO,EAOL,wBAAwB,EACxB,wBAAwB,EACxB,kBAAkB,EAClB,oBAAoB,EACpB,oBAAoB,EAEpB,cAAc,GACf,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,sBAAsB,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,EACL,qBAAqB,EAKrB,uBAAuB,EACvB,qBAAqB,EACrB,oBAAoB,EACpB,oBAAoB,GACrB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAGL,kBAAkB,EAClB,kBAAkB,EAClB,UAAU,EACV,QAAQ,EACR,QAAQ,GACT,MAAM,YAAY,CAAC;AACpB,OAAO,EACL,iBAAiB,EACjB,WAAW,EAUX,cAAc,EACd,oBAAoB,EACpB,gBAAgB,EAChB,aAAa,EACb,aAAa,GACd,MAAM,YAAY,CAAC"}

package/dist/decide/bench/metrics.d.ts

@@ -0,0 +1,55 @@
+/**
+ * Classification metrics for the System One benchmark (T12495).
+ *
+ * Two-label sites report precision, recall, F1 and false-positive rate for
+ * their positive class. The multi-class observation-type site reports macro
+ * averages over the labels present in the gold set (one-vs-rest per label).
+ *
+ * @task T12495
+ * @epic T12486
+ */
+/** One scored prediction. */
+export interface BenchPrediction {
+    /** Gold label. */
+    readonly gold: string;
+    /** Predicted label. */
+    readonly predicted: string;
+}
+/** Classification quality over a set of predictions. */
+export interface BenchClassificationMetrics {
+    /** Scored predictions. */
+    readonly n: number;
+    /** Share predicted correctly. */
+    readonly accuracy: number | null;
+    /** Precision (positive class, or macro average). */
+    readonly precision: number | null;
+    /** Recall (positive class, or macro average). */
+    readonly recall: number | null;
+    /** F1 (positive class, or macro average). */
+    readonly f1: number | null;
+    /** False-positive rate (positive class, or macro average one-vs-rest). */
+    readonly falsePositiveRate: number | null;
+}
+/**
+ * Score predictions.
+ *
+ * @param preds - Gold and predicted labels.
+ * @param positive - The positive class of a two-label site; `null` → macro averages.
+ * @returns The metrics (`null` where undefined, e.g. no positives predicted).
+ */
+export declare function classificationMetrics(preds: readonly BenchPrediction[], positive: string | null): BenchClassificationMetrics;
+/** p50 and p95 of latencies, in ms. */
+export interface BenchLatency {
+    /** Median. */
+    readonly p50Ms: number | null;
+    /** 95th percentile. */
+    readonly p95Ms: number | null;
+}
+/**
+ * Latency percentiles.
+ *
+ * @param samples - Latencies in ms.
+ * @returns p50 and p95 (nearest rank).
+ */
+export declare function latencyOf(samples: readonly number[]): BenchLatency;
+//# sourceMappingURL=metrics.d.ts.map

package/dist/decide/bench/metrics.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../../../src/decide/bench/metrics.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAIH,6BAA6B;AAC7B,MAAM,WAAW,eAAe;IAC9B,kBAAkB;IAClB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,uBAAuB;IACvB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B;AAED,wDAAwD;AACxD,MAAM,WAAW,0BAA0B;IACzC,0BAA0B;IAC1B,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC;IACnB,iCAAiC;IACjC,QAAQ,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,oDAAoD;IACpD,QAAQ,CAAC,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IAClC,iDAAiD;IACjD,QAAQ,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,6CAA6C;IAC7C,QAAQ,CAAC,EAAE,EAAE,MAAM,GAAG,IAAI,CAAC;IAC3B,0EAA0E;IAC1E,QAAQ,CAAC,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;CAC3C;AAoCD;;;;;;GAMG;AACH,wBAAgB,qBAAqB,CACnC,KAAK,EAAE,SAAS,eAAe,EAAE,EACjC,QAAQ,EAAE,MAAM,GAAG,IAAI,GACtB,0BAA0B,CAoC5B;AAED,uCAAuC;AACvC,MAAM,WAAW,YAAY;IAC3B,cAAc;IACd,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,uBAAuB;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;CAC/B;AAED;;;;;GAKG;AACH,wBAAgB,SAAS,CAAC,OAAO,EAAE,SAAS,MAAM,EAAE,GAAG,YAAY,CAElE"}

package/dist/decide/bench/metrics.js

@@ -0,0 +1,100 @@
+/**
+ * Classification metrics for the System One benchmark (T12495).
+ *
+ * Two-label sites report precision, recall, F1 and false-positive rate for
+ * their positive class. The multi-class observation-type site reports macro
+ * averages over the labels present in the gold set (one-vs-rest per label).
+ *
+ * @task T12495
+ * @epic T12486
+ */
+import { percentile } from './stats.js';
+/** Ratio, or `null` when the denominator is zero. */
+function ratio(num, den) {
+    return den > 0 ? num / den : null;
+}
+/** Harmonic mean of precision and recall. */
+function f1Of(p, r) {
+    if (p === null || r === null)
+        return null;
+    return p + r > 0 ? (2 * p * r) / (p + r) : 0;
+}
+/** One-vs-rest counts for `label`. */
+function counts(preds, label) {
+    let tp = 0;
+    let fp = 0;
+    let fn = 0;
+    let tn = 0;
+    for (const p of preds) {
+        const g = p.gold === label;
+        const q = p.predicted === label;
+        if (g && q)
+            tp++;
+        else if (!g && q)
+            fp++;
+        else if (g && !q)
+            fn++;
+        else
+            tn++;
+    }
+    return { tp, fp, fn, tn };
+}
+/** Mean of the non-null values, or null. */
+function meanOf(values) {
+    const xs = values.filter((v) => v !== null);
+    return xs.length > 0 ? xs.reduce((s, x) => s + x, 0) / xs.length : null;
+}
+/**
+ * Score predictions.
+ *
+ * @param preds - Gold and predicted labels.
+ * @param positive - The positive class of a two-label site; `null` → macro averages.
+ * @returns The metrics (`null` where undefined, e.g. no positives predicted).
+ */
+export function classificationMetrics(preds, positive) {
+    const n = preds.length;
+    const accuracy = ratio(preds.filter((p) => p.gold === p.predicted).length, n);
+    if (positive !== null) {
+        const c = counts(preds, positive);
+        const precision = ratio(c.tp, c.tp + c.fp);
+        const recall = ratio(c.tp, c.tp + c.fn);
+        return {
+            n,
+            accuracy,
+            precision,
+            recall,
+            f1: f1Of(precision, recall),
+            falsePositiveRate: ratio(c.fp, c.fp + c.tn),
+        };
+    }
+    const labels = [...new Set(preds.map((p) => p.gold))].sort();
+    const per = labels.map((label) => {
+        const c = counts(preds, label);
+        const precision = ratio(c.tp, c.tp + c.fp) ?? 0;
+        const recall = ratio(c.tp, c.tp + c.fn);
+        return {
+            precision,
+            recall,
+            f1: f1Of(precision, recall),
+            fpr: ratio(c.fp, c.fp + c.tn),
+        };
+    });
+    return {
+        n,
+        accuracy,
+        precision: meanOf(per.map((p) => p.precision)),
+        recall: meanOf(per.map((p) => p.recall)),
+        f1: meanOf(per.map((p) => p.f1)),
+        falsePositiveRate: meanOf(per.map((p) => p.fpr)),
+    };
+}
+/**
+ * Latency percentiles.
+ *
+ * @param samples - Latencies in ms.
+ * @returns p50 and p95 (nearest rank).
+ */
+export function latencyOf(samples) {
+    return { p50Ms: percentile(samples, 50), p95Ms: percentile(samples, 95) };
+}
+//# sourceMappingURL=metrics.js.map

package/dist/decide/bench/metrics.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"metrics.js","sourceRoot":"","sources":["../../../src/decide/bench/metrics.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AA0BxC,qDAAqD;AACrD,SAAS,KAAK,CAAC,GAAW,EAAE,GAAW;IACrC,OAAO,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC;AACpC,CAAC;AAED,6CAA6C;AAC7C,SAAS,IAAI,CAAC,CAAgB,EAAE,CAAgB;IAC9C,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,IAAI;QAAE,OAAO,IAAI,CAAC;IAC1C,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAC/C,CAAC;AAED,sCAAsC;AACtC,SAAS,MAAM,CAAC,KAAiC,EAAE,KAAa;IAC9D,IAAI,EAAE,GAAG,CAAC,CAAC;IACX,IAAI,EAAE,GAAG,CAAC,CAAC;IACX,IAAI,EAAE,GAAG,CAAC,CAAC;IACX,IAAI,EAAE,GAAG,CAAC,CAAC;IACX,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,KAAK,KAAK,CAAC;QAC3B,MAAM,CAAC,GAAG,CAAC,CAAC,SAAS,KAAK,KAAK,CAAC;QAChC,IAAI,CAAC,IAAI,CAAC;YAAE,EAAE,EAAE,CAAC;aACZ,IAAI,CAAC,CAAC,IAAI,CAAC;YAAE,EAAE,EAAE,CAAC;aAClB,IAAI,CAAC,IAAI,CAAC,CAAC;YAAE,EAAE,EAAE,CAAC;;YAClB,EAAE,EAAE,CAAC;IACZ,CAAC;IACD,OAAO,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC;AAC5B,CAAC;AAED,4CAA4C;AAC5C,SAAS,MAAM,CAAC,MAAkC;IAChD,MAAM,EAAE,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;IACzD,OAAO,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;AAC1E,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,qBAAqB,CACnC,KAAiC,EACjC,QAAuB;IAEvB,MAAM,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC;IACvB,MAAM,QAAQ,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,SAAS,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAC9E,IAAI,QAAQ,KAAK,IAAI,EAAE,CAAC;QACtB,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;QAClC,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;QAC3C,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;QACxC,OAAO;YACL,CAAC;YACD,QAAQ;YACR,SAAS;YACT,MAAM;YACN,EAAE,EAAE,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC;YAC3B,iBAAiB,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC;SAC5C,CAAC;IACJ,CAAC;IACD,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAC7D,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QAC/B,MAAM,CAAC,GAAG,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;QAC/B,MAAM,SAAS,GAAG,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;QAChD,MAAM,MAAM,GAAG,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;QACxC,OAAO;YACL,SAAS;YACT,MAAM;YACN,EAAE,EAAE,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC;YAC3B,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,EAAE,CAAC;SAC9B,CAAC;IACJ,CAAC,CAAC,CAAC;IACH,OAAO;QACL,CAAC;QACD,QAAQ;QACR,SAAS,EAAE,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QAC9C,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QACxC,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAChC,iBAAiB,EAAE,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;KACjD,CAAC;AACJ,CAAC;AAUD;;;;;GAKG;AACH,MAAM,UAAU,SAAS,CAAC,OAA0B;IAClD,OAAO,EAAE,KAAK,EAAE,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,EAAE,KAAK,EAAE,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,EAAE,CAAC;AAC5E,CAAC"}

package/dist/decide/bench/operation.d.ts

@@ -0,0 +1,119 @@
+/**
+ * `cleo decide bench` as one core operation (T12495): build or reuse the
+ * dataset, apply owner corrections, draw the spot-check sample and — unless
+ * `sampleOnly` — run the providers and write the results and the report.
+ *
+ * Files, all in `outDir` (default `<projectRoot>/.cleo/decide-bench/`, which
+ * git ignores):
+ *
+ * - `dataset.jsonl` — the labelled, redacted rows. Reused on later calls so
+ *   the owner's corrections stick; `rebuild` builds it again from the stores.
+ * - `spot-check.json` — the stratified sample for the owner.
+ * - `results.json` / `report.md` — written by a run.
+ *
+ * Dataset building, corrections and sampling never contact a provider.
+ *
+ * @task T12495
+ * @epic T12486
+ */
+import type { SpendLedger } from '../spend.js';
+import type { DecideFetch } from '../transport.js';
+import { type BenchDatasetSizes } from './dataset.js';
+import { type BenchProfileResolver } from './profiles.js';
+import { type BenchAggregateRow, type BenchSpendSummary } from './runner.js';
+import { type BenchCorrectionsReceipt } from './spot-check.js';
+import { type BenchConnection, type BenchSite, type BenchSource } from './types.js';
+/** Dataset file name inside the output directory. */
+export declare const BENCH_DATASET_FILE = "dataset.jsonl";
+/** Spot-check file name inside the output directory. */
+export declare const BENCH_SPOT_CHECK_FILE = "spot-check.json";
+/** Results file name inside the output directory. */
+export declare const BENCH_RESULTS_FILE = "results.json";
+/** Report fragment file name inside the output directory. */
+export declare const BENCH_REPORT_FILE = "report.md";
+/** Invalid benchmark input (maps to a validation error in the CLI). */
+export declare class DecideBenchInputError extends Error {
+    /** How to fix it. */
+    readonly fix: string;
+    /**
+     * @param message - What is wrong.
+     * @param fix - How to fix it.
+     */
+    constructor(message: string, fix: string);
+}
+/** Input of {@link runDecideBenchOperation}. */
+export interface DecideBenchInput {
+    /** Project whose stores are read. Default: the resolved project root. */
+    readonly projectRoot?: string;
+    /** Output directory. Default `<projectRoot>/.cleo/decide-bench`. */
+    readonly outDir?: string;
+    /** Profile names to compare (required unless `sampleOnly`). */
+    readonly profiles?: readonly string[];
+    /** Sites to sample and run. Default: all. */
+    readonly sites?: readonly string[];
+    /** Build the dataset and the sample only; no provider is contacted. */
+    readonly sampleOnly?: boolean;
+    /** Owner corrections file to apply before sampling. */
+    readonly correctionsPath?: string;
+    /** Hard total spend cap in US dollars. Default $5. */
+    readonly maxUsd?: number;
+    /** Repeated runs. Default 1. */
+    readonly runs?: number;
+    /** Rows per batch call. */
+    readonly batchSize?: number;
+    /** Rebuild the dataset from the stores even when one exists. */
+    readonly rebuild?: boolean;
+    /** PRNG seed for negatives and the sample. */
+    readonly seed?: number;
+    /** Spot-check sample size. Default 30. */
+    readonly sampleSize?: number;
+    /** Store port (tests). Default: the core accessors. */
+    readonly source?: BenchSource;
+    /** Profile resolver (tests). Default: the T12733 profile store, then the env override layer. */
+    readonly resolver?: BenchProfileResolver;
+    /** Explicit connections; skip profile resolution. */
+    readonly connections?: readonly BenchConnection[];
+    /** Transport (tests). */
+    readonly fetch?: DecideFetch;
+    /** Directory for per-connection provider-state files (tests). */
+    readonly providerStateDir?: string;
+    /** Monthly ledger to record spend in; `null` skips. Default: the file ledger. */
+    readonly spendLedger?: SpendLedger | null;
+}
+/** Output of {@link runDecideBenchOperation}. */
+export interface DecideBenchSummary {
+    /** Output directory. */
+    readonly outDir: string;
+    /** Files written or reused. */
+    readonly files: {
+        readonly dataset: string;
+        readonly spotCheck: string;
+        readonly results?: string;
+        readonly report?: string;
+    };
+    /** Whether an existing dataset was reused. */
+    readonly datasetReused: boolean;
+    /** Dataset sizes (all sites). */
+    readonly dataset: BenchDatasetSizes;
+    /** Sites sampled and run. */
+    readonly sites: readonly BenchSite[];
+    /** Items in the spot-check sample. */
+    readonly spotCheckItems: number;
+    /** Applied corrections, when a file was given. */
+    readonly corrections?: BenchCorrectionsReceipt;
+    /** Whether providers were run. */
+    readonly ran: boolean;
+    /** Spend, when run. */
+    readonly spend?: BenchSpendSummary;
+    /** Mean and spread per provider × site, when run. */
+    readonly aggregate?: readonly BenchAggregateRow[];
+}
+/**
+ * Run `cleo decide bench`.
+ *
+ * @param input - Flags and wiring.
+ * @returns What was written and, after a run, the spend and the aggregate.
+ * @throws DecideBenchInputError on invalid input (unknown site or profile, no profiles).
+ */
+export declare function runDecideBenchOperation(input?: DecideBenchInput): Promise<DecideBenchSummary>;
+//# sourceMappingURL=operation.d.ts.map

package/dist/decide/bench/operation.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"operation.d.ts","sourceRoot":"","sources":["../../../src/decide/bench/operation.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAMH,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC/C,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AACnD,OAAO,EACL,KAAK,iBAAiB,EAKvB,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,KAAK,oBAAoB,EAAwB,MAAM,eAAe,CAAC;AAEhF,OAAO,EACL,KAAK,iBAAiB,EACtB,KAAK,iBAAiB,EAGvB,MAAM,aAAa,CAAC;AAErB,OAAO,EAEL,KAAK,uBAAuB,EAG7B,MAAM,iBAAiB,CAAC;AAEzB,OAAO,EAAe,KAAK,eAAe,EAAE,KAAK,SAAS,EAAE,KAAK,WAAW,EAAE,MAAM,YAAY,CAAC;AAEjG,qDAAqD;AACrD,eAAO,MAAM,kBAAkB,kBAAkB,CAAC;AAElD,wDAAwD;AACxD,eAAO,MAAM,qBAAqB,oBAAoB,CAAC;AAEvD,qDAAqD;AACrD,eAAO,MAAM,kBAAkB,iBAAiB,CAAC;AAEjD,6DAA6D;AAC7D,eAAO,MAAM,iBAAiB,cAAc,CAAC;AAE7C,uEAAuE;AACvE,qBAAa,qBAAsB,SAAQ,KAAK;IAC9C,qBAAqB;IACrB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IAErB;;;OAGG;gBACS,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM;CAKzC;AAED,gDAAgD;AAChD,MAAM,WAAW,gBAAgB;IAC/B,yEAAyE;IACzE,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,oEAAoE;IACpE,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB,+DAA+D;IAC/D,QAAQ,CAAC,QAAQ,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACtC,6CAA6C;IAC7C,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,uEAAuE;IACvE,QAAQ,CAAC,UAAU,CAAC,EAAE,OAAO,CAAC;IAC9B,uDAAuD;IACvD,QAAQ,CAAC,eAAe,CAAC,EAAE,MAAM,CAAC;IAClC,sDAAsD;IACtD,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB,gCAAgC;IAChC,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,2BAA2B;IAC3B,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAC5B,gEAAgE;IAChE,QAAQ,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC;IAC3B,8CAA8C;IAC9C,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC;IACvB,0CAA0C;IAC1C,QAAQ,CAAC,UAAU,CAAC,EAAE,MAAM,CAAC;IAC7B,uDAAuD;IACvD,QAAQ,CAAC,MAAM,CAAC,EAAE,WAAW,CAAC;IAC9B,gGAAgG;IAChG,QAAQ,CAAC,QAAQ,CAAC,EAAE,oBAAoB,CAAC;IACzC,qDAAqD;IACrD,QAAQ,CAAC,WAAW,CAAC,EAAE,SAAS,eAAe,EAAE,CAAC;IAClD,yBAAyB;IACzB,QAAQ,CAAC,KAAK,CAAC,EAAE,WAAW,CAAC;IAC7B,iEAAiE;IACjE,QAAQ,CAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;IACnC,iFAAiF;IACjF,QAAQ,CAAC,WAAW,CAAC,EAAE,WAAW,GAAG,IAAI,CAAC;CAC3C;AAED,iDAAiD;AACjD,MAAM,WAAW,kBAAkB;IACjC,wBAAwB;IACxB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,+BAA+B;IAC/B,QAAQ,CAAC,KAAK,EAAE;QACd,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;QACzB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;QAC3B,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAC;QAC1B,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;KAC1B,CAAC;IACF,8CAA8C;IAC9C,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAC;IAChC,iCAAiC;IACjC,QAAQ,CAAC,OAAO,EAAE,iBAAiB,CAAC;IACpC,6BAA6B;IAC7B,QAAQ,CAAC,KAAK,EAAE,SAAS,SAAS,EAAE,CAAC;IACrC,sCAAsC;IACtC,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,kDAAkD;IAClD,QAAQ,CAAC,WAAW,CAAC,EAAE,uBAAuB,CAAC;IAC/C,kCAAkC;IAClC,QAAQ,CAAC,GAAG,EAAE,OAAO,CAAC;IACtB,uBAAuB;IACvB,QAAQ,CAAC,KAAK,CAAC,EAAE,iBAAiB,CAAC;IACnC,qDAAqD;IACrD,QAAQ,CAAC,SAAS,CAAC,EAAE,SAAS,iBAAiB,EAAE,CAAC;CACnD;AAyBD;;;;;;GAMG;AACH,wBAAsB,uBAAuB,CAC3C,KAAK,GAAE,gBAAqB,GAC3B,OAAO,CAAC,kBAAkB,CAAC,CAgG7B"}

package/dist/decide/bench/operation.js

@@ -0,0 +1,171 @@
+/**
+ * `cleo decide bench` as one core operation (T12495): build or reuse the
+ * dataset, apply owner corrections, draw the spot-check sample and — unless
+ * `sampleOnly` — run the providers and write the results and the report.
+ *
+ * Files, all in `outDir` (default `<projectRoot>/.cleo/decide-bench/`, which
+ * git ignores):
+ *
+ * - `dataset.jsonl` — the labelled, redacted rows. Reused on later calls so
+ *   the owner's corrections stick; `rebuild` builds it again from the stores.
+ * - `spot-check.json` — the stratified sample for the owner.
+ * - `results.json` / `report.md` — written by a run.
+ *
+ * Dataset building, corrections and sampling never contact a provider.
+ *
+ * @task T12495
+ * @epic T12486
+ */
+import { existsSync } from 'node:fs';
+import { mkdir, writeFile } from 'node:fs/promises';
+import { join, resolve } from 'node:path';
+import { getProjectRoot } from '../../paths.js';
+import { benchDatasetSizes, buildBenchDataset, readBenchDataset, writeBenchDataset, } from './dataset.js';
+import { resolveBenchProfiles } from './profiles.js';
+import { renderBenchReport } from './report.js';
+import { DEFAULT_BENCH_MAX_MICROS, runDecideBench, } from './runner.js';
+import { createStoreBenchSource } from './source.js';
+import { applyBenchCorrections, readBenchCorrections, sampleBenchSpotCheck, } from './spot-check.js';
+import { DEFAULT_BENCH_SEED } from './stats.js';
+import { BENCH_SITES } from './types.js';
+/** Dataset file name inside the output directory. */
+export const BENCH_DATASET_FILE = 'dataset.jsonl';
+/** Spot-check file name inside the output directory. */
+export const BENCH_SPOT_CHECK_FILE = 'spot-check.json';
+/** Results file name inside the output directory. */
+export const BENCH_RESULTS_FILE = 'results.json';
+/** Report fragment file name inside the output directory. */
+export const BENCH_REPORT_FILE = 'report.md';
+/** Invalid benchmark input (maps to a validation error in the CLI). */
+export class DecideBenchInputError extends Error {
+    /** How to fix it. */
+    fix;
+    /**
+     * @param message - What is wrong.
+     * @param fix - How to fix it.
+     */
+    constructor(message, fix) {
+        super(message);
+        this.name = 'DecideBenchInputError';
+        this.fix = fix;
+    }
+}
+/** Narrow the requested sites. */
+function parseSites(sites) {
+    if (!sites || sites.length === 0)
+        return [...BENCH_SITES];
+    const out = [];
+    for (const s of sites) {
+        const site = BENCH_SITES.find((b) => b === s.trim());
+        if (!site) {
+            throw new DecideBenchInputError(`unknown site '${s}'`, `--sites ${BENCH_SITES.join(',')}`);
+        }
+        if (!out.includes(site))
+            out.push(site);
+    }
+    return out;
+}
+/** Validate a positive number flag. */
+function positive(value, name, fallback) {
+    if (value === undefined)
+        return fallback;
+    if (!Number.isFinite(value) || value <= 0) {
+        throw new DecideBenchInputError(`${name} must be a positive number`, `${name} <n>`);
+    }
+    return value;
+}
+/**
+ * Run `cleo decide bench`.
+ *
+ * @param input - Flags and wiring.
+ * @returns What was written and, after a run, the spend and the aggregate.
+ * @throws DecideBenchInputError on invalid input (unknown site or profile, no profiles).
+ */
+export async function runDecideBenchOperation(input = {}) {
+    const sites = parseSites(input.sites);
+    const maxUsd = positive(input.maxUsd, '--max-usd', DEFAULT_BENCH_MAX_MICROS / 1e6);
+    const runs = Math.floor(positive(input.runs, '--runs', 1));
+    const batchSize = input.batchSize === undefined
+        ? undefined
+        : Math.floor(positive(input.batchSize, '--batch-size', 1));
+    const seed = input.seed ?? DEFAULT_BENCH_SEED;
+    if (!Number.isSafeInteger(seed)) {
+        throw new DecideBenchInputError('--seed must be an integer', '--seed 12495');
+    }
+    const wantsRun = input.sampleOnly !== true;
+    const profiles = input.profiles ?? [];
+    if (wantsRun && profiles.length === 0 && !input.connections?.length) {
+        throw new DecideBenchInputError('no providers to compare', 'cleo decide bench --profiles layahost,jev (or --sample-only to build the dataset and sample without spend)');
+    }
+    // Resolve profiles before any file is written, so a typo fails fast.
+    let connections = input.connections ?? [];
+    if (wantsRun && connections.length === 0) {
+        try {
+            connections = resolveBenchProfiles(profiles, input.resolver);
+        }
+        catch (err) {
+            throw new DecideBenchInputError(err instanceof Error ? err.message : 'profile resolution failed', 'cleo decide profiles (to list them), or cleo decide config --profile <provider>/<name>');
+        }
+    }
+    const projectRoot = input.projectRoot ?? getProjectRoot();
+    const outDir = resolve(input.outDir ?? join(projectRoot, '.cleo', 'decide-bench'));
+    await mkdir(outDir, { recursive: true, mode: 0o700 });
+    const datasetPath = join(outDir, BENCH_DATASET_FILE);
+    const spotCheckPath = join(outDir, BENCH_SPOT_CHECK_FILE);
+    const reuse = input.rebuild !== true && existsSync(datasetPath);
+    let rows = reuse
+        ? await readBenchDataset(datasetPath)
+        : await buildBenchDataset(input.source ?? createStoreBenchSource(projectRoot), { seed });
+    let corrections;
+    if (input.correctionsPath) {
+        const applied = applyBenchCorrections(rows, await readBenchCorrections(input.correctionsPath));
+        rows = applied.rows;
+        corrections = applied.receipt;
+    }
+    if (!reuse || corrections)
+        await writeBenchDataset(datasetPath, rows);
+    const selected = rows.filter((r) => sites.includes(r.site));
+    const spotCheck = sampleBenchSpotCheck(selected, {
+        seed,
+        ...(input.sampleSize !== undefined ? { size: input.sampleSize } : {}),
+    });
+    await writeFile(spotCheckPath, `${JSON.stringify(spotCheck, null, 2)}\n`, { mode: 0o600 });
+    const base = {
+        outDir,
+        datasetReused: reuse,
+        dataset: benchDatasetSizes(rows),
+        sites,
+        spotCheckItems: spotCheck.items.length,
+        ...(corrections ? { corrections } : {}),
+    };
+    if (!wantsRun) {
+        return { ...base, files: { dataset: datasetPath, spotCheck: spotCheckPath }, ran: false };
+    }
+    const results = await runDecideBench({
+        rows: selected,
+        connections,
+        runs,
+        maxMicros: Math.round(maxUsd * 1e6),
+        ...(batchSize !== undefined ? { batchSize } : {}),
+        ...(input.fetch ? { fetch: input.fetch } : {}),
+        ...(input.providerStateDir ? { providerStateDir: input.providerStateDir } : {}),
+        ...(input.spendLedger !== undefined ? { spendLedger: input.spendLedger } : {}),
+    });
+    const resultsPath = join(outDir, BENCH_RESULTS_FILE);
+    const reportPath = join(outDir, BENCH_REPORT_FILE);
+    await writeFile(resultsPath, `${JSON.stringify(results, null, 2)}\n`, { mode: 0o600 });
+    await writeFile(reportPath, renderBenchReport(results), { mode: 0o600 });
+    return {
+        ...base,
+        files: {
+            dataset: datasetPath,
+            spotCheck: spotCheckPath,
+            results: resultsPath,
+            report: reportPath,
+        },
+        ran: true,
+        spend: results.spend,
+        aggregate: results.aggregate,
+    };
+}
+//# sourceMappingURL=operation.js.map

package/dist/decide/bench/operation.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"operation.js","sourceRoot":"","sources":["../../../src/decide/bench/operation.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AACrC,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAGhD,OAAO,EAEL,iBAAiB,EACjB,iBAAiB,EACjB,gBAAgB,EAChB,iBAAiB,GAClB,MAAM,cAAc,CAAC;AACtB,OAAO,EAA6B,oBAAoB,EAAE,MAAM,eAAe,CAAC;AAChF,OAAO,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAChD,OAAO,EAGL,wBAAwB,EACxB,cAAc,GACf,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,sBAAsB,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,EACL,qBAAqB,EAErB,oBAAoB,EACpB,oBAAoB,GACrB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAChD,OAAO,EAAE,WAAW,EAA0D,MAAM,YAAY,CAAC;AAEjG,qDAAqD;AACrD,MAAM,CAAC,MAAM,kBAAkB,GAAG,eAAe,CAAC;AAElD,wDAAwD;AACxD,MAAM,CAAC,MAAM,qBAAqB,GAAG,iBAAiB,CAAC;AAEvD,qDAAqD;AACrD,MAAM,CAAC,MAAM,kBAAkB,GAAG,cAAc,CAAC;AAEjD,6DAA6D;AAC7D,MAAM,CAAC,MAAM,iBAAiB,GAAG,WAAW,CAAC;AAE7C,uEAAuE;AACvE,MAAM,OAAO,qBAAsB,SAAQ,KAAK;IAC9C,qBAAqB;IACZ,GAAG,CAAS;IAErB;;;OAGG;IACH,YAAY,OAAe,EAAE,GAAW;QACtC,KAAK,CAAC,OAAO,CAAC,CAAC;QACf,IAAI,CAAC,IAAI,GAAG,uBAAuB,CAAC;QACpC,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;IACjB,CAAC;CACF;AAuED,kCAAkC;AAClC,SAAS,UAAU,CAAC,KAAoC;IACtD,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,GAAG,WAAW,CAAC,CAAC;IAC1D,MAAM,GAAG,GAAgB,EAAE,CAAC;IAC5B,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;QACrD,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,MAAM,IAAI,qBAAqB,CAAC,iBAAiB,CAAC,GAAG,EAAE,WAAW,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC7F,CAAC;QACD,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC;YAAE,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1C,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,uCAAuC;AACvC,SAAS,QAAQ,CAAC,KAAyB,EAAE,IAAY,EAAE,QAAgB;IACzE,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,QAAQ,CAAC;IACzC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;QAC1C,MAAM,IAAI,qBAAqB,CAAC,GAAG,IAAI,4BAA4B,EAAE,GAAG,IAAI,MAAM,CAAC,CAAC;IACtF,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,QAA0B,EAAE;IAE5B,MAAM,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACtC,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,MAAM,EAAE,WAAW,EAAE,wBAAwB,GAAG,GAAG,CAAC,CAAC;IACnF,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC;IAC3D,MAAM,SAAS,GACb,KAAK,CAAC,SAAS,KAAK,SAAS;QAC3B,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,SAAS,EAAE,cAAc,EAAE,CAAC,CAAC,CAAC,CAAC;IAC/D,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,IAAI,kBAAkB,CAAC;IAC9C,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC;QAChC,MAAM,IAAI,qBAAqB,CAAC,2BAA2B,EAAE,cAAc,CAAC,CAAC;IAC/E,CAAC;IACD,MAAM,QAAQ,GAAG,KAAK,CAAC,UAAU,KAAK,IAAI,CAAC;IAC3C,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,IAAI,EAAE,CAAC;IACtC,IAAI,QAAQ,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,WAAW,EAAE,MAAM,EAAE,CAAC;QACpE,MAAM,IAAI,qBAAqB,CAC7B,yBAAyB,EACzB,4GAA4G,CAC7G,CAAC;IACJ,CAAC;IACD,qEAAqE;IACrE,IAAI,WAAW,GAA+B,KAAK,CAAC,WAAW,IAAI,EAAE,CAAC;IACtE,IAAI,QAAQ,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzC,IAAI,CAAC;YACH,WAAW,GAAG,oBAAoB,CAAC,QAAQ,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC/D,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,IAAI,qBAAqB,CAC7B,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,2BAA2B,EAChE,wFAAwF,CACzF,CAAC;QACJ,CAAC;IACH,CAAC;IAED,MAAM,WAAW,GAAG,KAAK,CAAC,WAAW,IAAI,cAAc,EAAE,CAAC;IAC1D,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,IAAI,IAAI,CAAC,WAAW,EAAE,OAAO,EAAE,cAAc,CAAC,CAAC,CAAC;IACnF,MAAM,KAAK,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IACtD,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAC;IACrD,MAAM,aAAa,GAAG,IAAI,CAAC,MAAM,EAAE,qBAAqB,CAAC,CAAC;IAE1D,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,KAAK,IAAI,IAAI,UAAU,CAAC,WAAW,CAAC,CAAC;IAChE,IAAI,IAAI,GAAG,KAAK;QACd,CAAC,CAAC,MAAM,gBAAgB,CAAC,WAAW,CAAC;QACrC,CAAC,CAAC,MAAM,iBAAiB,CAAC,KAAK,CAAC,MAAM,IAAI,sBAAsB,CAAC,WAAW,CAAC,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC;IAC3F,IAAI,WAAgD,CAAC;IACrD,IAAI,KAAK,CAAC,eAAe,EAAE,CAAC;QAC1B,MAAM,OAAO,GAAG,qBAAqB,CAAC,IAAI,EAAE,MAAM,oBAAoB,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,CAAC;QAC/F,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;QACpB,WAAW,GAAG,OAAO,CAAC,OAAO,CAAC;IAChC,CAAC;IACD,IAAI,CAAC,KAAK,IAAI,WAAW;QAAE,MAAM,iBAAiB,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;IAEtE,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;IAC5D,MAAM,SAAS,GAAG,oBAAoB,CAAC,QAAQ,EAAE;QAC/C,IAAI;QACJ,GAAG,CAAC,KAAK,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACtE,CAAC,CAAC;IACH,MAAM,SAAS,CAAC,aAAa,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IAE3F,MAAM,IAAI,GAAG;QACX,MAAM;QACN,aAAa,EAAE,KAAK;QACpB,OAAO,EAAE,iBAAiB,CAAC,IAAI,CAAC;QAChC,KAAK;QACL,cAAc,EAAE,SAAS,CAAC,KAAK,CAAC,MAAM;QACtC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACxC,CAAC;IACF,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,OAAO,EAAE,GAAG,IAAI,EAAE,KAAK,EAAE,EAAE,OAAO,EAAE,WAAW,EAAE,SAAS,EAAE,aAAa,EAAE,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC;IAC5F,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,cAAc,CAAC;QACnC,IAAI,EAAE,QAAQ;QACd,WAAW;QACX,IAAI;QACJ,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,GAAG,CAAC;QACnC,GAAG,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACjD,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9C,GAAG,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,gBAAgB,EAAE,KAAK,CAAC,gBAAgB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/E,GAAG,CAAC,KAAK,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,KAAK,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC/E,CAAC,CAAC;IACH,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,EAAE,kBAAkB,CAAC,CAAC;IACrD,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,EAAE,iBAAiB,CAAC,CAAC;IACnD,MAAM,SAAS,CAAC,WAAW,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IACvF,MAAM,SAAS,CAAC,UAAU,EAAE,iBAAiB,CAAC,OAAO,CAAC,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IACzE,OAAO;QACL,GAAG,IAAI;QACP,KAAK,EAAE;YACL,OAAO,EAAE,WAAW;YACpB,SAAS,EAAE,aAAa;YACxB,OAAO,EAAE,WAAW;YACpB,MAAM,EAAE,UAAU;SACnB;QACD,GAAG,EAAE,IAAI;QACT,KAAK,EAAE,OAAO,CAAC,KAAK;QACpB,SAAS,EAAE,OAAO,CAAC,SAAS;KAC7B,CAAC;AACJ,CAAC"}

package/dist/decide/bench/profiles.d.ts

@@ -0,0 +1,102 @@
+/**
+ * Resolve `--profiles a,b` to benchmark connections (T12495, T12735).
+ *
+ * Names resolve through the System One profile store (T12733,
+ * `decide-credentials.json`), in this order:
+ *
+ * 1. An exact profile id, `<provider>/<name>` (e.g. `layahost/work`).
+ * 2. A bare provider kind (`layahost`, `jev`): that provider's ACTIVE
+ *    profile when the active profile belongs to it, else `<provider>/default`.
+ * 3. The environment override layer, for a name the store does not hold:
+ *    `CLEO_DECIDE_PROFILE_<NAME>_KEY` (plus `_URL`, `_MODEL`, `_PROVIDER`),
+ *    `<NAME>` upper-cased with non-alphanumerics as `_`
+ *    (`layahost/lab` → `CLEO_DECIDE_PROFILE_LAYAHOST_LAB_KEY`). Useful for a
+ *    throwaway key that should never be written to disk.
+ *
+ * A stored profile's `default` URL, and an environment profile without
+ * `_URL`, both resolve through {@link presetBaseUrl} — the same preset table
+ * everyday decisions use. An unknown name fails with {@link BenchProfileError},
+ * which lists the stored profile ids with masked keys.
+ *
+ * @task T12735
+ * @epic T12486
+ */
+import { listDecideProfiles, resolveDecideProfile } from '../credentials.js';
+import type { BenchConnection } from './types.js';
+/** Resolves one profile name to a connection. */
+export interface BenchProfileResolver {
+    /**
+     * @param name - Profile name.
+     * @returns The connection, or `null` when no profile has that name.
+     * @throws BenchProfileInvalidError when the profile exists but is misconfigured.
+     */
+    resolve(name: string): BenchConnection | null;
+    /**
+     * The profiles that could be named, for the unknown-profile error.
+     *
+     * @returns Secret-free labels, e.g. `layahost/work (key …1111)`.
+     */
+    available?(): readonly string[];
+}
+/** A profile name that resolved to nothing. */
+export declare class BenchProfileError extends Error {
+    /** The names that did not resolve. */
+    readonly missing: readonly string[];
+    /** Secret-free labels of the stored profiles (id and masked key). */
+    readonly available: readonly string[];
+    /**
+     * @param missing - Unresolved names.
+     * @param available - Secret-free labels of the profiles that do exist.
+     */
+    constructor(missing: readonly string[], available?: readonly string[]);
+}
+/** A profile that exists but is misconfigured (bad URL, key or model). */
+export declare class BenchProfileInvalidError extends Error {
+    /** The profile name. */
+    readonly profile: string;
+    /**
+     * @param profile - Profile name.
+     * @param problem - What is wrong, e.g. `invalid URL "…"`.
+     */
+    constructor(profile: string, problem: string);
+}
+/**
+ * Environment variable prefix for one profile.
+ *
+ * @param name - Profile name.
+ * @returns `CLEO_DECIDE_PROFILE_<NAME>` (upper-cased, non-alphanumerics as `_`).
+ */
+export declare function benchProfileEnvPrefix(name: string): string;
+/** Profile-store access, injectable for tests. Defaults to the real store. */
+export interface BenchProfileStore {
+    /** List the stored profiles (secret-free). */
+    readonly list: typeof listDecideProfiles;
+    /** Resolve one stored profile (contains the key). */
+    readonly resolve: typeof resolveDecideProfile;
+}
+/** Options for {@link createProfileResolver}. */
+export interface BenchProfileResolverOptions {
+    /** Environment for the override layer. Default `process.env`. */
+    readonly env?: NodeJS.ProcessEnv;
+    /** Profile store. Default: `decide-credentials.json` in the CLEO home. */
+    readonly store?: BenchProfileStore;
+}
+/**
+ * The benchmark's profile resolver: the T12733 profile store (exact id, then
+ * bare provider), then the environment override layer. See the module doc.
+ *
+ * @param opts - Environment and store. Defaults: `process.env` and the real store.
+ * @returns A resolver.
+ */
+export declare function createProfileResolver(opts?: BenchProfileResolverOptions): BenchProfileResolver;
+/**
+ * Resolve every name, or throw naming all that did not resolve.
+ *
+ * @param names - Profile names (duplicates removed, order kept).
+ * @param resolver - Resolver. Default: {@link createProfileResolver}.
+ * @returns One connection per name.
+ * @throws BenchProfileError when any name is unknown (lists the stored profiles, keys masked).
+ * @throws BenchProfileInvalidError when a profile has a bad URL, key or model.
+ */
+export declare function resolveBenchProfiles(names: readonly string[], resolver?: BenchProfileResolver): BenchConnection[];
+//# sourceMappingURL=profiles.d.ts.map

package/dist/decide/bench/profiles.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"profiles.d.ts","sourceRoot":"","sources":["../../../src/decide/bench/profiles.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAGH,OAAO,EAGL,kBAAkB,EAElB,oBAAoB,EACrB,MAAM,mBAAmB,CAAC;AAO3B,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAElD,iDAAiD;AACjD,MAAM,WAAW,oBAAoB;IACnC;;;;OAIG;IACH,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,eAAe,GAAG,IAAI,CAAC;IAC9C;;;;OAIG;IACH,SAAS,CAAC,IAAI,SAAS,MAAM,EAAE,CAAC;CACjC;AAED,+CAA+C;AAC/C,qBAAa,iBAAkB,SAAQ,KAAK;IAC1C,sCAAsC;IACtC,QAAQ,CAAC,OAAO,EAAE,SAAS,MAAM,EAAE,CAAC;IACpC,qEAAqE;IACrE,QAAQ,CAAC,SAAS,EAAE,SAAS,MAAM,EAAE,CAAC;IAEtC;;;OAGG;gBACS,OAAO,EAAE,SAAS,MAAM,EAAE,EAAE,SAAS,GAAE,SAAS,MAAM,EAAO;CAW1E;AAED,0EAA0E;AAC1E,qBAAa,wBAAyB,SAAQ,KAAK;IACjD,wBAAwB;IACxB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IAEzB;;;OAGG;gBACS,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM;CAK7C;AAED;;;;;GAKG;AACH,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAE1D;AAED,8EAA8E;AAC9E,MAAM,WAAW,iBAAiB;IAChC,8CAA8C;IAC9C,QAAQ,CAAC,IAAI,EAAE,OAAO,kBAAkB,CAAC;IACzC,qDAAqD;IACrD,QAAQ,CAAC,OAAO,EAAE,OAAO,oBAAoB,CAAC;CAC/C;AAED,iDAAiD;AACjD,MAAM,WAAW,2BAA2B;IAC1C,iEAAiE;IACjE,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,UAAU,CAAC;IACjC,0EAA0E;IAC1E,QAAQ,CAAC,KAAK,CAAC,EAAE,iBAAiB,CAAC;CACpC;AAyCD;;;;;;GAMG;AACH,wBAAgB,qBAAqB,CACnC,IAAI,GAAE,2BAAgC,GACrC,oBAAoB,CAmCtB;AAED;;;;;;;;GAQG;AACH,wBAAgB,oBAAoB,CAClC,KAAK,EAAE,SAAS,MAAM,EAAE,EACxB,QAAQ,GAAE,oBAA8C,GACvD,eAAe,EAAE,CAWnB"}