@tangle-network/agent-eval 0.144.11 → 0.144.13

package/dist/index-YE4KdKbO2.d.ts

@@ -1,335 +0,0 @@
-import { s as RunSplitTag } from "./run-record-CF4Dwpxr.js";
-import { R as Scenario, a as CampaignResult, d as DispatchContext } from "./types-BnjdJ70P.js";
-import { Zt as CampaignStorage } from "./skillopt-optimization-method-USDKhxSA.js";
-import "./index-DPPGNJ_R.js";
-//#region src/benchmarks/types.d.ts
-type BenchmarkTaskKind = 'retrieval' | 'rag-answer' | 'hallucination' | 'kb-improvement' | 'routing' | 'custom';
-type BenchmarkFamily = 'beir' | 'mteb-retrieval' | 'msmarco' | 'trec-dl' | 'miracl' | 'lotte' | 'bright' | 'crag' | 'hotpotqa' | 'kilt' | 'ragtruth' | 'faithbench' | 'first-party' | 'custom';
-interface BenchmarkDatasetItem<TPayload = unknown> {
-  /** Stable dataset-local item id (used for split assignment + paper
-   *  references). Unique within a benchmark. */
-  id: string;
-  /** Free-form payload. Each benchmark defines its own shape. */
-  payload: TPayload;
-  /** Optional precomputed split. When absent, adapters use assignSplit(id). */
-  split?: RunSplitTag;
-  /** Benchmark family this row came from, e.g. `beir` or `crag`. */
-  family?: BenchmarkFamily | string;
-  /** Benchmark-local task kind, e.g. retrieval vs answer quality. */
-  taskKind?: BenchmarkTaskKind | string;
-  /** Slice labels such as language, domain, freshness, multihop, long-tail. */
-  tags?: string[];
-  /** Dataset provenance, version, URL, or license notes. */
-  source?: BenchmarkSource;
-  metadata?: Record<string, unknown>;
-}
-interface BenchmarkEvaluation {
-  /** [0, 1] score for the response on this item. Exact-match
-   *  benchmarks use 0/1; partial-credit benchmarks may return
-   *  fractional values. */
-  score: number;
-  /** Optional pass/fail projection. Defaults to `score > 0` when absent. */
-  passed?: boolean;
-  /** Numeric sub-metrics. These become report dimensions. */
-  dimensions?: Record<string, number>;
-  /** Optional bag of raw scoring signals — e.g. parsed numeric
-   *  answer, regex match, judge sub-scores. */
-  raw: Record<string, unknown>;
-  notes?: string;
-}
-interface BenchmarkSource {
-  name?: string;
-  url?: string;
-  version?: string;
-  license?: string;
-  citation?: string;
-}
-/** Common signature implemented by every adapter under `src/benchmarks/*`. */
-interface BenchmarkAdapter<_TItem = unknown, TPayload = unknown, TArtifact = string> {
-  /** Stable benchmark id such as `beir/nfcorpus` or `crag/smoke`. */
-  id?: string;
-  family?: BenchmarkFamily | string;
-  taskKind?: BenchmarkTaskKind | string;
-  description?: string;
-  source?: BenchmarkSource;
-  defaultMetric?: string;
-  /** Load the dataset for the given split. May hit the network on
-   *  first call but should be cache-friendly. Adapters that don't
-   *  ship the dataset itself MUST throw a clearly-marked error
-   *  pointing the caller at the loader script. */
-  loadDataset(split: RunSplitTag): Promise<BenchmarkDatasetItem<TPayload>[]>;
-  /** Score a single response. Pure with respect to the inputs. */
-  evaluate(item: BenchmarkDatasetItem<TPayload>, artifact: TArtifact): Promise<BenchmarkEvaluation>;
-  /** Deterministic split assignment via item id hashing. The
-   *  fraction of items in each split is implementation-defined but
-   *  MUST be stable across processes and platforms. */
-  assignSplit(itemId: string): RunSplitTag;
-}
-interface BenchmarkScenario<TPayload = unknown> extends Scenario {
-  kind: 'benchmark';
-  benchmarkId: string;
-  family: BenchmarkFamily | string;
-  taskKind: BenchmarkTaskKind | string;
-  splitTag: RunSplitTag;
-  item: BenchmarkDatasetItem<TPayload>;
-}
-type BenchmarkResponder<TPayload = unknown, TArtifact = string> = (input: {
-  scenario: BenchmarkScenario<TPayload>;
-  item: BenchmarkDatasetItem<TPayload>;
-  context: DispatchContext;
-}) => Promise<TArtifact> | TArtifact;
-/** Split-assignment seed shared across all benchmarks. Bumping this
- *  value reshuffles every split — do NOT do that lightly. */
-declare const BENCHMARK_SPLIT_SEED = "agent-eval-v1";
-/**
- * Assign an item id to one of `'search' | 'dev' | 'holdout'` using a
- * stable 32-bit hash of `${seed}::${id}`. Default proportions:
- *
- *   search: 60%   (optimization-readable)
- *   dev:    20%   (held-out for tuning, leak-on-purpose during dev)
- *   holdout:20%   (paper-grade held-out, gated reads)
- */
-declare function deterministicSplit(itemId: string, seed?: string): RunSplitTag;
-//#endregion
-//#region src/benchmarks/calibration.d.ts
-interface BenchmarkMetricCalibrationOptions<TPayload = unknown, TArtifact = string> {
-  adapter: BenchmarkAdapter<BenchmarkDatasetItem<TPayload>, TPayload, TArtifact>;
-  item: BenchmarkDatasetItem<TPayload>;
-  weakArtifact: TArtifact;
-  strongArtifact: TArtifact;
-  maxWeakScore?: number;
-  minStrongScore?: number;
-  minGap?: number;
-}
-interface BenchmarkMetricCalibrationResult {
-  passed: boolean;
-  weak: BenchmarkEvaluation;
-  strong: BenchmarkEvaluation;
-  weakScore: number;
-  strongScore: number;
-  gap: number;
-  reasons: string[];
-}
-declare function calibrateBenchmarkMetric<TPayload = unknown, TArtifact = string>(options: BenchmarkMetricCalibrationOptions<TPayload, TArtifact>): Promise<BenchmarkMetricCalibrationResult>;
-//#endregion
-//#region src/benchmarks/routing/dataset.d.ts
-/**
- * Synthetic routing dataset. 16 tasks across 4 categories. Used as a
- * deterministic, dependency-free benchmark for any router that maps a
- * natural-language request to one of a fixed set of route labels.
- *
- * Format (see `routing/README.md` for prose):
- *
- *   {
- *     id:           stable per-task ID (matches across processes).
- *     category:     one of the four route labels.
- *     prompt:       the user-facing request the router must classify.
- *     route:        the ground-truth route the router should pick.
- *     synonyms:     other strings that count as a correct answer.
- *     hardNegatives:close-but-wrong route labels — used to detect the
- *                   "always picks the popular route" failure mode.
- *   }
- *
- * The four categories are intentionally cross-domain (file ops,
- * math, search, conversation) so a router that collapses to one
- * category is easy to spot.
- */
-interface RoutingItem {
-  id: string;
-  category: 'file' | 'math' | 'search' | 'chat';
-  prompt: string;
-  /** Canonical correct route label. */
-  route: string;
-  /** Alternate route labels that also count as correct. */
-  synonyms: string[];
-  /** Wrong-but-tempting route labels (for analysis, not grading). */
-  hardNegatives: string[];
-}
-declare const ROUTING_DATASET: RoutingItem[];
-declare namespace index_d_exports$1 {
-  export { ROUTING_DATASET, RoutingAdapter, RoutingDatasetItem, RoutingItem, RoutingPayload, assignSplit, evaluate, extractRouteTokens, loadDataset };
-}
-type RoutingPayload = RoutingItem;
-type RoutingDatasetItem = BenchmarkDatasetItem<RoutingPayload>;
-declare class RoutingAdapter implements BenchmarkAdapter<RoutingDatasetItem, RoutingPayload> {
-  readonly id = "first-party/routing";
-  readonly family = "first-party";
-  readonly taskKind = "routing";
-  readonly description = "Synthetic fixed-route classification smoke benchmark";
-  readonly defaultMetric = "route_exact_match";
-  loadDataset(split: RunSplitTag): Promise<RoutingDatasetItem[]>;
-  evaluate(item: RoutingDatasetItem, response: string): Promise<BenchmarkEvaluation>;
-  assignSplit(itemId: string): RunSplitTag;
-}
-/**
- * Pull route-shaped tokens out of a model response. Routes look like
- * `category.action` (`fs.write`, `chat.reply`). Bare alphanumerics
- * are not routes, but `category.action` patterns are robust to most
- * model wrappers (JSON output, prose explanations, code fences).
- */
-declare function extractRouteTokens(response: string): string[];
-declare const loadDataset: (split: RunSplitTag) => Promise<RoutingDatasetItem[]>;
-declare const evaluate: (item: RoutingDatasetItem, response: string) => Promise<BenchmarkEvaluation>;
-declare const assignSplit: (itemId: string) => RunSplitTag;
-//#endregion
-//#region src/benchmarks/runner.d.ts
-interface BenchmarkRunOptions<TPayload = unknown, TArtifact = string> {
-  adapter: BenchmarkAdapter<BenchmarkDatasetItem<TPayload>, TPayload, TArtifact>;
-  respond: BenchmarkResponder<TPayload, TArtifact>;
-  splits?: readonly RunSplitTag[];
-  runDir: string;
-  repo?: string;
-  seed?: number;
-  reps?: number;
-  resumable?: boolean;
-  costCeiling?: number;
-  maxConcurrency?: number;
-  dispatchTimeoutMs?: number;
-  expectUsage?: 'assert' | 'warn' | 'off';
-  storage?: CampaignStorage;
-  now?: () => Date;
-}
-interface BenchmarkReport {
-  benchmarkId: string;
-  family: BenchmarkFamily | string;
-  taskKind: BenchmarkTaskKind | string;
-  source?: BenchmarkAdapter['source'];
-  runDir: string;
-  manifestHash: string;
-  seed: number;
-  startedAt: string;
-  endedAt: string;
-  durationMs: number;
-  totalItems: number;
-  totalCells: number;
-  cellsFailed: number;
-  cellsCached: number;
-  totalCostUsd: number;
-  splits: Record<string, BenchmarkSliceSummary>;
-  tags: Record<string, BenchmarkSliceSummary>;
-  dimensions: Record<string, BenchmarkDistribution>;
-  score: BenchmarkDistribution;
-  costUsd: BenchmarkDistribution;
-  latencyMs: BenchmarkDistribution;
-}
-interface BenchmarkSliceSummary {
-  n: number;
-  meanScore: number;
-  passRate: number;
-  score: BenchmarkDistribution;
-  costUsd: BenchmarkDistribution;
-  latencyMs: BenchmarkDistribution;
-}
-interface BenchmarkDistribution {
-  n: number;
-  min: number;
-  mean: number;
-  median: number;
-  p90: number;
-  max: number;
-}
-interface BenchmarkRunResult<TPayload = unknown, TArtifact = string> {
-  scenarios: Array<BenchmarkScenario<TPayload>>;
-  campaign: CampaignResult<TArtifact, BenchmarkScenario<TPayload>>;
-  report: BenchmarkReport;
-  reportJsonPath: string;
-  reportMarkdownPath: string;
-}
-declare function runBenchmarkAdapter<TPayload = unknown, TArtifact = string>(options: BenchmarkRunOptions<TPayload, TArtifact>): Promise<BenchmarkRunResult<TPayload, TArtifact>>;
-declare function summarizeBenchmarkCampaign<TPayload, TArtifact>(input: {
-  adapter: BenchmarkAdapter<BenchmarkDatasetItem<TPayload>, TPayload, TArtifact>;
-  scenarios: Array<BenchmarkScenario<TPayload>>;
-  campaign: CampaignResult<TArtifact, BenchmarkScenario<TPayload>>;
-}): BenchmarkReport;
-declare function renderBenchmarkReportMarkdown(report: BenchmarkReport): string;
-//#endregion
-//#region src/benchmarks/standard-formats.d.ts
-interface StandardRetrievalDocument {
-  id: string;
-  title?: string;
-  text: string;
-  metadata?: Record<string, unknown>;
-}
-interface StandardRetrievalQuery {
-  id: string;
-  text: string;
-  metadata?: Record<string, unknown>;
-}
-interface StandardRetrievalQrel {
-  queryId: string;
-  documentId: string;
-  score: number;
-}
-interface StandardRetrievalPayload {
-  queryId: string;
-  query: string;
-  expectedDocumentIds: string[];
-  expectedScores: Record<string, number>;
-  corpus?: Record<string, StandardRetrievalDocument>;
-  metadata?: Record<string, unknown>;
-}
-interface BuildStandardRetrievalItemsOptions {
-  benchmarkId: string;
-  family: BenchmarkFamily | string;
-  queries: readonly StandardRetrievalQuery[];
-  qrels: readonly StandardRetrievalQrel[];
-  corpus?: readonly StandardRetrievalDocument[];
-  includeCorpusInPayload?: boolean;
-  source?: BenchmarkSource;
-  tags?: readonly string[];
-  splitOf?: (queryId: string) => RunSplitTag;
-}
-interface RetrievalIdAdapterOptions extends BuildStandardRetrievalItemsOptions {
-  responseIdPattern?: RegExp;
-  cutoffs?: readonly number[];
-  primaryMetric?: string;
-  passMetric?: string;
-  passThreshold?: number;
-}
-interface StandardRetrievalResult {
-  id?: string;
-  documentId?: string;
-  docId?: string;
-  score?: number;
-}
-type StandardRetrievalArtifact = string | readonly string[] | readonly StandardRetrievalResult[] | {
-  ids?: readonly string[];
-  documentIds?: readonly string[];
-  results?: readonly StandardRetrievalResult[];
-};
-interface StandardRetrievalEvaluationOptions {
-  responseIdPattern?: RegExp;
-  cutoffs?: readonly number[];
-  primaryMetric?: string;
-  passMetric?: string;
-  passThreshold?: number;
-}
-declare function parseJsonlRows<T = unknown>(text: string): T[];
-declare function parseTsvRows(text: string): string[][];
-declare function parseQrels(text: string): StandardRetrievalQrel[];
-declare function parseBeirCorpusJsonl(text: string): StandardRetrievalDocument[];
-declare function parseBeirQueriesJsonl(text: string): StandardRetrievalQuery[];
-declare function buildStandardRetrievalItems(options: BuildStandardRetrievalItemsOptions): Array<BenchmarkDatasetItem<StandardRetrievalPayload>>;
-declare function createRetrievalIdBenchmarkAdapter(options: RetrievalIdAdapterOptions): BenchmarkAdapter<BenchmarkDatasetItem<StandardRetrievalPayload>, StandardRetrievalPayload, StandardRetrievalArtifact>;
-declare function evaluateStandardRetrieval(payload: StandardRetrievalPayload, artifact: StandardRetrievalArtifact, options?: StandardRetrievalEvaluationOptions): {
-  score: number;
-  passed: boolean;
-  dimensions: Record<string, number>;
-  raw: {
-    rankedDocumentIds: string[];
-    expectedDocumentIds: string[];
-    expectedScores: Record<string, number>;
-  };
-};
-declare function normalizeRetrievedDocumentIds(artifact: StandardRetrievalArtifact, responseIdPattern?: RegExp): string[];
-declare function retrievalMetricsAtCutoff(input: {
-  rankedDocumentIds: readonly string[];
-  expectedScores: Record<string, number>;
-  cutoff: number;
-}): Record<string, number>;
-declare namespace index_d_exports {
-  export { BENCHMARK_SPLIT_SEED, BenchmarkAdapter, BenchmarkDatasetItem, BenchmarkDistribution, BenchmarkEvaluation, BenchmarkFamily, BenchmarkMetricCalibrationOptions, BenchmarkMetricCalibrationResult, BenchmarkReport, BenchmarkResponder, BenchmarkRunOptions, BenchmarkRunResult, BenchmarkScenario, BenchmarkSliceSummary, BenchmarkSource, BenchmarkTaskKind, BuildStandardRetrievalItemsOptions, RetrievalIdAdapterOptions, StandardRetrievalArtifact, StandardRetrievalDocument, StandardRetrievalEvaluationOptions, StandardRetrievalPayload, StandardRetrievalQrel, StandardRetrievalQuery, StandardRetrievalResult, buildStandardRetrievalItems, calibrateBenchmarkMetric, createRetrievalIdBenchmarkAdapter, deterministicSplit, evaluateStandardRetrieval, normalizeRetrievedDocumentIds, parseBeirCorpusJsonl, parseBeirQueriesJsonl, parseJsonlRows, parseQrels, parseTsvRows, renderBenchmarkReportMarkdown, retrievalMetricsAtCutoff, index_d_exports$1 as routing, runBenchmarkAdapter, summarizeBenchmarkCampaign };
-}
-//#endregion
-export { BenchmarkMetricCalibrationOptions as A, BenchmarkSource as B, BenchmarkRunOptions as C, runBenchmarkAdapter as D, renderBenchmarkReportMarkdown as E, BenchmarkDatasetItem as F, deterministicSplit as H, BenchmarkEvaluation as I, BenchmarkFamily as L, calibrateBenchmarkMetric as M, BENCHMARK_SPLIT_SEED as N, summarizeBenchmarkCampaign as O, BenchmarkAdapter as P, BenchmarkResponder as R, BenchmarkReport as S, BenchmarkSliceSummary as T, BenchmarkTaskKind as V, parseJsonlRows as _, StandardRetrievalDocument as a, retrievalMetricsAtCutoff as b, StandardRetrievalQrel as c, buildStandardRetrievalItems as d, createRetrievalIdBenchmarkAdapter as f, parseBeirQueriesJsonl as g, parseBeirCorpusJsonl as h, StandardRetrievalArtifact as i, BenchmarkMetricCalibrationResult as j, index_d_exports$1 as k, StandardRetrievalQuery as l, normalizeRetrievedDocumentIds as m, BuildStandardRetrievalItemsOptions as n, StandardRetrievalEvaluationOptions as o, evaluateStandardRetrieval as p, RetrievalIdAdapterOptions as r, StandardRetrievalPayload as s, index_d_exports as t, StandardRetrievalResult as u, parseQrels as v, BenchmarkRunResult as w, BenchmarkDistribution as x, parseTsvRows as y, BenchmarkScenario as z };
-//# sourceMappingURL=index-YE4KdKbO2.d.ts.map

package/dist/index-YE4KdKbO2.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"index-YE4KdKbO2.d.ts","names":[],"sources":["../src/benchmarks/types.ts","../src/benchmarks/calibration.ts","../src/benchmarks/routing/dataset.ts","../src/benchmarks/routing/index.ts","../src/benchmarks/runner.ts","../src/benchmarks/standard-formats.ts","../src/benchmarks/index.ts"],"mappings":";;;;;KASY;KAQA;UAgBK,qBAAqB;;;EAGpC;;EAEA,SAAS;;EAET,QAAQ;;EAER,SAAS;;EAET,WAAW;;EAEX;;EAEA,SAAS;EACT,WAAW;;UAGI;;;;EAIf;;EAEA;;EAEA,aAAa;;;EAGb,KAAK;EACL;;UAGe;EACf;EACA;EACA;EACA;EACA;;;UAQe,iBAAiB,kBAAkB,oBAAoB;;EAEtE;EACA,SAAS;EACT,WAAW;EACX;EACA,SAAS;EACT;;;;;EAKA,YAAY,OAAO,cAAc,QAAQ,qBAAqB;;EAE9D,SAAS,MAAM,qBAAqB,WAAW,UAAU,YAAY,QAAQ;;;;EAI7E,YAAY,iBAAiB;;UAGd,kBAAkB,4BAA4B;EAC7D;EACA;EACA,QAAQ;EACR,UAAU;EACV,UAAU;EACV,MAAM,qBAAqB;;KAGjB,mBAAmB,oBAAoB,uBAAuB;EACxE,UAAU,kBAAkB;EAC5B,MAAM,qBAAqB;EAC3B,SAAS;MACL,QAAQ,aAAa;;;cAoBd;;;;;;;;;iBAUG,mBACd,gBACA,gBACC;;;UCjJc,kCAAkC,oBAAoB;EACrE,SAAS,iBAAiB,qBAAqB,WAAW,UAAU;EACpE,MAAM,qBAAqB;EAC3B,cAAc;EACd,gBAAgB;EAChB;EACA;EACA;;UAGe;EACf;EACA,MAAM;EACN,QAAQ;EACR;EACA;EACA;EACA;;iBAGoB,yBAAyB,oBAAoB,oBACjE,SAAS,kCAAkC,UAAU,aACpD,QAAQ;;;;;;;;;;;;;;;;;;;;;;;;UCFM;EACf;EACA;EACA;;EAEA;;EAEA;;EAEA;;cAGW,iBAAiB;;;;KChBlB,iBAAiB;KACjB,qBAAqB,qBAAqB;cAEhD,0BAA0B,iBAAiB,oBAAoB;WAC1D;WACA;WACA;WACA;WACA;EAEH,YAAY,OAAO,cAAc,QAAQ;EAMzC,SAAS,MAAM,oBAAoB,mBAAmB,QAAQ;EAqBpE,YAAY,iBAAiB;;;;;;;;iBAef,mBAAmB;cAOtB,cAAW,OAjDG,gBAAc,QAAQ;cAkDpC,WAAQ,MA5CE,oBAAkB,qBAAqB,QAAQ;cA6CzD,cAAW,mBAxBO;;;UClCd,oBAAoB,oBAAoB;EACvD,SAAS,iBAAiB,qBAAqB,WAAW,UAAU;EACpE,SAAS,mBAAmB,UAAU;EACtC,kBAAkB;EAClB;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA,UAAU;EACV,YAAY;;UAGG;EACf;EACA,QAAQ;EACR,UAAU;EACV,SAAS;EACT;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA,QAAQ,eAAe;EACvB,MAAM,eAAe;EACrB,YAAY,eAAe;EAC3B,OAAO;EACP,SAAS;EACT,WAAW;;UAGI;EACf;EACA;EACA;EACA,OAAO;EACP,SAAS;EACT,WAAW;;UAGI;EACf;EACA;EACA;EACA;EACA;EACA;;UAGe,mBAAmB,oBAAoB;EACtD,WAAW,MAAM,kBAAkB;EACnC,UAAU,eAAe,WAAW,kBAAkB;EACtD,QAAQ;EACR;EACA;;iBAGoB,oBAAoB,oBAAoB,oBAC5D,SAAS,oBAAoB,UAAU,aACtC,QAAQ,mBAAmB,UAAU;iBAmGxB,2BAA2B,UAAU,WAAW;EAC9D,SAAS,iBAAiB,qBAAqB,WAAW,UAAU;EACpE,WAAW,MAAM,kBAAkB;EACnC,UAAU,eAAe,WAAW,kBAAkB;IACpD;iBA6CY,8BAA8B,QAAQ;;;UCpOrC;EACf;EACA;EACA;EACA,WAAW;;UAGI;EACf;EACA;EACA,WAAW;;UAGI;EACf;EACA;EACA;;UAGe;EACf;EACA;EACA;EACA,gBAAgB;EAChB,SAAS,eAAe;EACxB,WAAW;;UAGI;EACf;EACA,QAAQ;EACR,kBAAkB;EAClB,gBAAgB;EAChB,kBAAkB;EAClB;EACA,SAAS;EACT;EACA,WAAW,oBAAoB;;UAGhB,kCAAkC;EACjD,oBAAoB;EACpB;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA;EACA;;KAGU,kEAGC;EAEP;EACA;EACA,mBAAmB;;UAGR;EACf,oBAAoB;EACpB;EACA;EACA;EACA;;iBAGc,eAAe,aAAa,eAAe;iBAc3C,aAAa;iBASb,WAAW,eAAe;iBAgB1B,qBAAqB,eAAe;iBAgBpC,sBAAsB,eAAe;iBAerC,4BACd,SAAS,qCACR,MAAM,qBAAqB;iBAuCd,kCACd,SAAS,4BACR,iBACD,qBAAqB,2BACrB,0BACA;iBAqBc,0BACd,SAAS,0BACT,UAAU,2BACV,UAAS;;;;;;;;;;iBAsCK,8BACd,UAAU,2BACV,oBAAmB;iBA0BL,yBAAyB;EACvC;EACA,gBAAgB;EAChB;IACE"}

package/dist/paired-arms-iZ08VFMN.js

@@ -1,260 +0,0 @@
-import { c as ValidationError } from "./errors-D-LKuDhb.js";
-import { A as pairedRiskDifference, E as pairedBootstrap, W as wilcoxonSignedRank, b as mcnemar } from "./statistics-ByxzSiOM.js";
-//#region src/paired-arms.ts
-/**
-* Matched-pair arm comparison — "did the treatment arm beat the baseline arm
-* on the SAME work items?"
-*
-* An arm A/B over run records is only trustworthy when it is PAIRED: the same
-* task/scenario/seed evaluated under both arms, compared item-by-item, so
-* inter-item difficulty variance cancels instead of masquerading as an arm
-* effect. This module owns the two error-prone steps every consumer otherwise
-* hand-rolls:
-*
-*   1. Pairing — matching rows across arms by `pairKey` (and by `repKey`
-*      within multi-rep items), with leftovers REPORTED rather than silently
-*      dropped (a silently unbalanced pairing biases every paired statistic
-*      downstream). Pairing never keys on outcome content: matching reps by
-*      their outcomes deflates discordant-pair counts and makes McNemar
-*      anti-conservative, so reps pair only by (`pairKey`, `repKey`) identity.
-*   2. Composition — feeding the matched pairs to the correct paired
-*      estimators that already live in `statistics`: `mcnemar` +
-*      `pairedRiskDifference` for pass/fail, `pairedBootstrap` +
-*      `wilcoxonSignedRank` for continuous metrics. No statistic is
-*      re-implemented here.
-*
-* The row shape is deliberately structural — callers project a `RunRecord`
-* (or any record) into `{ pairKey, arm, pass?, metrics? }`. Arm names are
-* caller-supplied parameters; the module ships no domain literal.
-*/
-/**
-* Match rows across two arms into (baseline, treatment) pairs by `pairKey`.
-*
-* A `pairKey` with at most one row per arm pairs directly, no `repKey`
-* needed. A `pairKey` with multiple reps in either arm requires `repKey` on
-* every one of its rows, and reps pair only on exact (`pairKey`, `repKey`)
-* match — pairing is keyed purely on row identity, never on outcome content
-* (outcome-keyed matching deflates discordant counts and biases McNemar), and
-* is therefore independent of input order. Reps whose `repKey` has no
-* counterpart in the other arm, and items present in only one arm, land in
-* the unpaired lists — reported, never truncated.
-*
-* Fail-loud: throws when either named arm has zero rows (an unknown arm
-* name would otherwise read as "everything unpaired"), when the two arm
-* names are equal, when a multi-rep `pairKey` has a row without `repKey`, or
-* when a (`pairKey`, arm) group repeats a `repKey` (the match would be
-* ambiguous).
-*/
-function pairArms(rows, opts) {
-	const { baselineArm, treatmentArm } = opts;
-	if (baselineArm === treatmentArm) throw new ValidationError(`pairArms: baselineArm and treatmentArm are both '${baselineArm}' — an arm cannot be compared to itself`);
-	const byArm = /* @__PURE__ */ new Map();
-	const armsSeen = /* @__PURE__ */ new Set();
-	for (const row of rows) {
-		armsSeen.add(row.arm);
-		if (row.arm !== baselineArm && row.arm !== treatmentArm) continue;
-		const byKey = byArm.get(row.arm) ?? /* @__PURE__ */ new Map();
-		const group = byKey.get(row.pairKey) ?? [];
-		group.push(row);
-		byKey.set(row.pairKey, group);
-		byArm.set(row.arm, byKey);
-	}
-	for (const arm of [baselineArm, treatmentArm]) if (!byArm.has(arm)) throw new ValidationError(`pairArms: no rows for arm '${arm}' (arms present: ${[...armsSeen].sort().join(", ") || "<none>"})`);
-	const baselineByKey = byArm.get(baselineArm);
-	const treatmentByKey = byArm.get(treatmentArm);
-	const allKeys = [.../* @__PURE__ */ new Set([...baselineByKey.keys(), ...treatmentByKey.keys()])].sort();
-	const pairs = [];
-	const unpairedBaseline = [];
-	const unpairedTreatment = [];
-	for (const pairKey of allKeys) {
-		const b = baselineByKey.get(pairKey) ?? [];
-		const t = treatmentByKey.get(pairKey) ?? [];
-		if (b.length <= 1 && t.length <= 1) {
-			if (b.length === 1 && t.length === 1) {
-				const baseline = b[0];
-				const treatment = t[0];
-				if (baseline.repKey !== void 0 || treatment.repKey !== void 0) {
-					if (baseline.repKey === void 0 || treatment.repKey === void 0 || baseline.repKey !== treatment.repKey) {
-						unpairedBaseline.push(baseline);
-						unpairedTreatment.push(treatment);
-						continue;
-					}
-				}
-				pairs.push({
-					pairKey,
-					repIndex: 0,
-					baseline,
-					treatment
-				});
-			} else {
-				unpairedBaseline.push(...b);
-				unpairedTreatment.push(...t);
-			}
-			continue;
-		}
-		const bByRep = indexByRepKey(b, pairKey, baselineArm);
-		const tByRep = indexByRepKey(t, pairKey, treatmentArm);
-		const repKeys = [.../* @__PURE__ */ new Set([...bByRep.keys(), ...tByRep.keys()])].sort();
-		let repIndex = 0;
-		for (const repKey of repKeys) {
-			const baseline = bByRep.get(repKey);
-			const treatment = tByRep.get(repKey);
-			if (baseline !== void 0 && treatment !== void 0) pairs.push({
-				pairKey,
-				repIndex: repIndex++,
-				baseline,
-				treatment
-			});
-			else if (baseline !== void 0) unpairedBaseline.push(baseline);
-			else if (treatment !== void 0) unpairedTreatment.push(treatment);
-		}
-	}
-	return {
-		pairs,
-		unpairedBaseline,
-		unpairedTreatment
-	};
-}
-/** Index a multi-rep (pairKey, arm) group by `repKey`, enforcing that every
-*  row carries one and that no repKey repeats within the group. */
-function indexByRepKey(group, pairKey, arm) {
-	const byRep = /* @__PURE__ */ new Map();
-	for (const row of group) {
-		if (row.repKey === void 0) throw new ValidationError(`pairArms: pairKey '${pairKey}' has multiple reps in an arm, but a row in arm '${arm}' is missing repKey — multi-rep items require an explicit repKey on every row so reps pair by identity (pairing reps by outcome or by index would bias the paired statistics)`);
-		if (byRep.has(row.repKey)) throw new ValidationError(`pairArms: duplicate repKey '${row.repKey}' for pairKey '${pairKey}' in arm '${arm}' — (pairKey, repKey) must uniquely identify a rep within an arm`);
-		byRep.set(row.repKey, row);
-	}
-	return byRep;
-}
-/**
-* Full matched-pair arm comparison: pair via {@link pairArms}, then compose
-* the paired estimators from `statistics` over the matched pairs.
-*
-* Correctness uses only the pairs where both sides carry `pass` (`mcnemar.n`
-* is that subset's size); each metric uses only the pairs where both sides
-* carry a finite value for it, with the remainder counted in `nMissing`.
-* Deltas are treatment − baseline throughout.
-*
-* Fail-loud: inherits {@link pairArms}'s unknown-arm throw, and throws on a
-* non-finite metric value — silently treating corrupt telemetry as "metric
-* absent" would misreport it as missing coverage.
-*/
-function comparePairedArms(rows, opts) {
-	const { pairs, unpairedBaseline, unpairedTreatment } = pairArms(rows, opts);
-	let correctness = null;
-	const baselinePass = [];
-	const treatmentPass = [];
-	for (const pair of pairs) {
-		if (pair.baseline.pass === void 0 || pair.treatment.pass === void 0) continue;
-		baselinePass.push(pair.baseline.pass ? 1 : 0);
-		treatmentPass.push(pair.treatment.pass ? 1 : 0);
-	}
-	if (baselinePass.length > 0) {
-		const mc = mcnemar(baselinePass, treatmentPass);
-		correctness = {
-			b10: mc.b,
-			b01: mc.c,
-			mcnemar: mc,
-			riskDifference: pairedRiskDifference(baselinePass, treatmentPass)
-		};
-	}
-	const metricDeltas = (opts.metricNames ?? [...new Set(pairs.flatMap((p) => [...Object.keys(p.baseline.metrics ?? {}), ...Object.keys(p.treatment.metrics ?? {})]))].sort()).map((name) => {
-		const before = [];
-		const after = [];
-		let nMissing = 0;
-		for (const pair of pairs) {
-			const b = metricValue(pair.baseline, name);
-			const t = metricValue(pair.treatment, name);
-			if (b === void 0 || t === void 0) {
-				nMissing++;
-				continue;
-			}
-			before.push(b);
-			after.push(t);
-		}
-		const bootstrapCi = before.length === 0 ? null : pairedBootstrap(before, after, opts.bootstrap);
-		return {
-			name,
-			n: before.length,
-			nMissing,
-			medianDelta: bootstrapCi?.median ?? null,
-			meanDelta: bootstrapCi?.mean ?? null,
-			bootstrapCi,
-			wilcoxon: before.length === 0 ? null : wilcoxonSignedRank(before, after)
-		};
-	});
-	return {
-		nPairs: pairs.length,
-		nUnpairedBaseline: unpairedBaseline.length,
-		nUnpairedTreatment: unpairedTreatment.length,
-		correctness,
-		metricDeltas
-	};
-}
-/**
-* Pair two RunRecord arms by the identity of the evaluated work:
-* `(experimentId, scenarioId, seed)`.
-*
-* Falling back to array order, candidate id, or experiment id can compare
-* different tasks and fabricate lift. Duplicate identities throw.
-*/
-function pairRunRecords(baselineRuns, treatmentRuns) {
-	const baselineRows = runRecordArmRows(baselineRuns, "baseline");
-	const treatmentRows = runRecordArmRows(treatmentRuns, "treatment");
-	validateRunRecordArmRows(baselineRows, "baseline");
-	validateRunRecordArmRows(treatmentRows, "treatment");
-	if (baselineRows.length === 0 || treatmentRows.length === 0) return {
-		pairs: [],
-		unpairedBaseline: baselineRows.map((row) => row.run),
-		unpairedTreatment: treatmentRows.map((row) => row.run)
-	};
-	const result = pairArms([...baselineRows, ...treatmentRows], {
-		baselineArm: "baseline",
-		treatmentArm: "treatment"
-	});
-	return {
-		pairs: result.pairs.map((pair) => {
-			const baseline = pair.baseline;
-			const treatment = pair.treatment;
-			return {
-				pairKey: pair.pairKey,
-				repKey: baseline.repKey,
-				baseline: baseline.run,
-				treatment: treatment.run
-			};
-		}),
-		unpairedBaseline: result.unpairedBaseline.map((row) => row.run),
-		unpairedTreatment: result.unpairedTreatment.map((row) => row.run)
-	};
-}
-function runRecordArmRows(runs, arm) {
-	return runs.map((run) => {
-		const scenarioId = run.scenarioId.trim();
-		if (!scenarioId) throw new ValidationError(`pairRunRecords: run '${run.runId}' is missing scenarioId; paired comparisons require explicit scenario identity`);
-		return {
-			pairKey: JSON.stringify([run.experimentId, scenarioId]),
-			repKey: String(run.seed),
-			arm,
-			run
-		};
-	});
-}
-function validateRunRecordArmRows(rows, arm) {
-	const byPairKey = /* @__PURE__ */ new Map();
-	for (const row of rows) {
-		const group = byPairKey.get(row.pairKey) ?? [];
-		group.push(row);
-		byPairKey.set(row.pairKey, group);
-	}
-	for (const [pairKey, group] of byPairKey) if (group.length > 1) indexByRepKey(group, pairKey, arm);
-}
-function metricValue(row, name) {
-	const v = row.metrics?.[name];
-	if (v === void 0) return void 0;
-	if (!Number.isFinite(v)) throw new ValidationError(`comparePairedArms: non-finite value for metric '${name}' on pairKey '${row.pairKey}' (arm '${row.arm}'): ${v}`);
-	return v;
-}
-//#endregion
-export { pairArms as n, pairRunRecords as r, comparePairedArms as t };
-
-//# sourceMappingURL=paired-arms-iZ08VFMN.js.map

package/dist/paired-arms-iZ08VFMN.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"paired-arms-iZ08VFMN.js","names":[],"sources":["../src/paired-arms.ts"],"sourcesContent":["/**\n * Matched-pair arm comparison — \"did the treatment arm beat the baseline arm\n * on the SAME work items?\"\n *\n * An arm A/B over run records is only trustworthy when it is PAIRED: the same\n * task/scenario/seed evaluated under both arms, compared item-by-item, so\n * inter-item difficulty variance cancels instead of masquerading as an arm\n * effect. This module owns the two error-prone steps every consumer otherwise\n * hand-rolls:\n *\n *   1. Pairing — matching rows across arms by `pairKey` (and by `repKey`\n *      within multi-rep items), with leftovers REPORTED rather than silently\n *      dropped (a silently unbalanced pairing biases every paired statistic\n *      downstream). Pairing never keys on outcome content: matching reps by\n *      their outcomes deflates discordant-pair counts and makes McNemar\n *      anti-conservative, so reps pair only by (`pairKey`, `repKey`) identity.\n *   2. Composition — feeding the matched pairs to the correct paired\n *      estimators that already live in `statistics`: `mcnemar` +\n *      `pairedRiskDifference` for pass/fail, `pairedBootstrap` +\n *      `wilcoxonSignedRank` for continuous metrics. No statistic is\n *      re-implemented here.\n *\n * The row shape is deliberately structural — callers project a `RunRecord`\n * (or any record) into `{ pairKey, arm, pass?, metrics? }`. Arm names are\n * caller-supplied parameters; the module ships no domain literal.\n */\n\nimport { ValidationError } from './errors'\nimport type { RunRecord } from './run-record'\nimport type { McNemarResult, PairedBootstrapOptions, PairedBootstrapResult } from './statistics'\nimport {\n  mcnemar,\n  pairedBootstrap,\n  pairedRiskDifference,\n  type RiskDifferenceResult,\n  wilcoxonSignedRank,\n} from './statistics'\n\n/** One arm observation of one work item. Structural on purpose: callers\n *  project their own record type (e.g. a `RunRecord`) into this shape. */\nexport interface PairedArmRow {\n  /** Matching key — rows sharing a `pairKey` across both arms form pairs\n   *  (typically the task/scenario/seed identity). */\n  pairKey: string\n  /** Rep identity within a `pairKey` (e.g. a seed or rep number). Required on\n   *  every row of a `pairKey` that has more than one rep in either arm; reps\n   *  then pair only on exact (`pairKey`, `repKey`) match, never on outcome\n   *  content. Optional when each arm has at most one rep of the item. */\n  repKey?: string\n  /** Arm label this row was produced under. */\n  arm: string\n  /** Binary outcome; omit when the comparison has no pass/fail notion. */\n  pass?: boolean\n  /** Named numeric measurements (score, cost, latency, …). */\n  metrics?: Record<string, number>\n}\n\nexport interface PairArmsOptions {\n  /** Arm treated as the control side of every pair. */\n  baselineArm: string\n  /** Arm treated as the treatment side of every pair. */\n  treatmentArm: string\n}\n\n/** One matched (baseline, treatment) observation of the same work item. */\nexport interface MatchedPair {\n  pairKey: string\n  /** 0-based position of this pair within its `pairKey`, ordered by sorted\n   *  `repKey` (always 0 for a single-rep item). The rep identity itself is on\n   *  the rows (`baseline.repKey` / `treatment.repKey`). */\n  repIndex: number\n  baseline: PairedArmRow\n  treatment: PairedArmRow\n}\n\nexport interface PairArmsResult {\n  /** Matched pairs, ordered by (`pairKey`, `repIndex`). */\n  pairs: MatchedPair[]\n  /** Baseline rows left without a treatment counterpart — reported, never\n   *  silently dropped. */\n  unpairedBaseline: PairedArmRow[]\n  /** Treatment rows left without a baseline counterpart. */\n  unpairedTreatment: PairedArmRow[]\n}\n\n/**\n * Match rows across two arms into (baseline, treatment) pairs by `pairKey`.\n *\n * A `pairKey` with at most one row per arm pairs directly, no `repKey`\n * needed. A `pairKey` with multiple reps in either arm requires `repKey` on\n * every one of its rows, and reps pair only on exact (`pairKey`, `repKey`)\n * match — pairing is keyed purely on row identity, never on outcome content\n * (outcome-keyed matching deflates discordant counts and biases McNemar), and\n * is therefore independent of input order. Reps whose `repKey` has no\n * counterpart in the other arm, and items present in only one arm, land in\n * the unpaired lists — reported, never truncated.\n *\n * Fail-loud: throws when either named arm has zero rows (an unknown arm\n * name would otherwise read as \"everything unpaired\"), when the two arm\n * names are equal, when a multi-rep `pairKey` has a row without `repKey`, or\n * when a (`pairKey`, arm) group repeats a `repKey` (the match would be\n * ambiguous).\n */\nexport function pairArms(rows: readonly PairedArmRow[], opts: PairArmsOptions): PairArmsResult {\n  const { baselineArm, treatmentArm } = opts\n  if (baselineArm === treatmentArm) {\n    throw new ValidationError(\n      `pairArms: baselineArm and treatmentArm are both '${baselineArm}' — an arm cannot be compared to itself`,\n    )\n  }\n\n  // arm → pairKey → rows\n  const byArm = new Map<string, Map<string, PairedArmRow[]>>()\n  const armsSeen = new Set<string>()\n  for (const row of rows) {\n    armsSeen.add(row.arm)\n    if (row.arm !== baselineArm && row.arm !== treatmentArm) continue\n    const byKey = byArm.get(row.arm) ?? new Map<string, PairedArmRow[]>()\n    const group = byKey.get(row.pairKey) ?? []\n    group.push(row)\n    byKey.set(row.pairKey, group)\n    byArm.set(row.arm, byKey)\n  }\n\n  for (const arm of [baselineArm, treatmentArm]) {\n    if (!byArm.has(arm)) {\n      const seen = [...armsSeen].sort().join(', ') || '<none>'\n      throw new ValidationError(`pairArms: no rows for arm '${arm}' (arms present: ${seen})`)\n    }\n  }\n\n  const baselineByKey = byArm.get(baselineArm)!\n  const treatmentByKey = byArm.get(treatmentArm)!\n\n  const allKeys = [...new Set([...baselineByKey.keys(), ...treatmentByKey.keys()])].sort()\n  const pairs: MatchedPair[] = []\n  const unpairedBaseline: PairedArmRow[] = []\n  const unpairedTreatment: PairedArmRow[] = []\n  for (const pairKey of allKeys) {\n    const b = baselineByKey.get(pairKey) ?? []\n    const t = treatmentByKey.get(pairKey) ?? []\n\n    if (b.length <= 1 && t.length <= 1) {\n      if (b.length === 1 && t.length === 1) {\n        const baseline = b[0]!\n        const treatment = t[0]!\n        if (baseline.repKey !== undefined || treatment.repKey !== undefined) {\n          if (\n            baseline.repKey === undefined ||\n            treatment.repKey === undefined ||\n            baseline.repKey !== treatment.repKey\n          ) {\n            unpairedBaseline.push(baseline)\n            unpairedTreatment.push(treatment)\n            continue\n          }\n        }\n        pairs.push({ pairKey, repIndex: 0, baseline, treatment })\n      } else {\n        unpairedBaseline.push(...b)\n        unpairedTreatment.push(...t)\n      }\n      continue\n    }\n\n    const bByRep = indexByRepKey(b, pairKey, baselineArm)\n    const tByRep = indexByRepKey(t, pairKey, treatmentArm)\n    const repKeys = [...new Set([...bByRep.keys(), ...tByRep.keys()])].sort()\n    let repIndex = 0\n    for (const repKey of repKeys) {\n      const baseline = bByRep.get(repKey)\n      const treatment = tByRep.get(repKey)\n      if (baseline !== undefined && treatment !== undefined) {\n        pairs.push({ pairKey, repIndex: repIndex++, baseline, treatment })\n      } else if (baseline !== undefined) {\n        unpairedBaseline.push(baseline)\n      } else if (treatment !== undefined) {\n        unpairedTreatment.push(treatment)\n      }\n    }\n  }\n\n  return { pairs, unpairedBaseline, unpairedTreatment }\n}\n\n/** Index a multi-rep (pairKey, arm) group by `repKey`, enforcing that every\n *  row carries one and that no repKey repeats within the group. */\nfunction indexByRepKey(\n  group: readonly PairedArmRow[],\n  pairKey: string,\n  arm: string,\n): Map<string, PairedArmRow> {\n  const byRep = new Map<string, PairedArmRow>()\n  for (const row of group) {\n    if (row.repKey === undefined) {\n      throw new ValidationError(\n        `pairArms: pairKey '${pairKey}' has multiple reps in an arm, but a row in arm '${arm}' ` +\n          `is missing repKey — multi-rep items require an explicit repKey on every row so reps ` +\n          `pair by identity (pairing reps by outcome or by index would bias the paired statistics)`,\n      )\n    }\n    if (byRep.has(row.repKey)) {\n      throw new ValidationError(\n        `pairArms: duplicate repKey '${row.repKey}' for pairKey '${pairKey}' in arm '${arm}' — ` +\n          `(pairKey, repKey) must uniquely identify a rep within an arm`,\n      )\n    }\n    byRep.set(row.repKey, row)\n  }\n  return byRep\n}\n\n/** Paired pass/fail comparison over the pairs where BOTH sides carry `pass`. */\nexport interface PairedCorrectness {\n  /** Discordant pairs where the treatment passed and the baseline failed. */\n  b10: number\n  /** Discordant pairs where the baseline passed and the treatment failed. */\n  b01: number\n  /** Exact McNemar significance over the paired outcomes (`b === b10`, `c === b01`). */\n  mcnemar: McNemarResult\n  /** Paired effect size: p(treatment) − p(baseline) with a paired-variance CI. */\n  riskDifference: RiskDifferenceResult\n}\n\n/** Paired delta summary for one named metric (delta = treatment − baseline). */\nexport interface PairedMetricDelta {\n  name: string\n  /** Pairs where BOTH sides carry a finite value for this metric. */\n  n: number\n  /** Pairs where at least one side does not carry the metric. */\n  nMissing: number\n  /** Median paired delta, or null when `n === 0`. */\n  medianDelta: number | null\n  /** Mean paired delta, or null when `n === 0`. */\n  meanDelta: number | null\n  /** Bootstrap CI on the paired delta (`pairedBootstrap`); null when\n   *  `n === 0` — a zero-width [0, 0] interval on no data would read as a\n   *  measured tight null. */\n  bootstrapCi: PairedBootstrapResult | null\n  /** Wilcoxon signed-rank test on the paired deltas; null when `n === 0`. */\n  wilcoxon: { w: number; p: number } | null\n}\n\nexport interface ComparePairedArmsOptions extends PairArmsOptions {\n  /** Metrics to compare. Default: every metric name observed on any matched\n   *  pair, sorted. A name that appears on no pair is still reported (with\n   *  `n = 0`) so a misspelled metric is visible instead of vanishing. */\n  metricNames?: string[]\n  /** Passed through to `pairedBootstrap` — set `seed` for reproducible CIs. */\n  bootstrap?: PairedBootstrapOptions\n}\n\nexport interface PairedArmsComparison {\n  nPairs: number\n  nUnpairedBaseline: number\n  nUnpairedTreatment: number\n  /** null when no matched pair carries `pass` on both sides — a pass/fail\n   *  verdict over rows that never measured pass/fail would be fabricated. */\n  correctness: PairedCorrectness | null\n  metricDeltas: PairedMetricDelta[]\n}\n\n/**\n * Full matched-pair arm comparison: pair via {@link pairArms}, then compose\n * the paired estimators from `statistics` over the matched pairs.\n *\n * Correctness uses only the pairs where both sides carry `pass` (`mcnemar.n`\n * is that subset's size); each metric uses only the pairs where both sides\n * carry a finite value for it, with the remainder counted in `nMissing`.\n * Deltas are treatment − baseline throughout.\n *\n * Fail-loud: inherits {@link pairArms}'s unknown-arm throw, and throws on a\n * non-finite metric value — silently treating corrupt telemetry as \"metric\n * absent\" would misreport it as missing coverage.\n */\nexport function comparePairedArms(\n  rows: readonly PairedArmRow[],\n  opts: ComparePairedArmsOptions,\n): PairedArmsComparison {\n  const { pairs, unpairedBaseline, unpairedTreatment } = pairArms(rows, opts)\n\n  let correctness: PairedCorrectness | null = null\n  const baselinePass: number[] = []\n  const treatmentPass: number[] = []\n  for (const pair of pairs) {\n    if (pair.baseline.pass === undefined || pair.treatment.pass === undefined) continue\n    baselinePass.push(pair.baseline.pass ? 1 : 0)\n    treatmentPass.push(pair.treatment.pass ? 1 : 0)\n  }\n  if (baselinePass.length > 0) {\n    const mc = mcnemar(baselinePass, treatmentPass)\n    correctness = {\n      b10: mc.b,\n      b01: mc.c,\n      mcnemar: mc,\n      riskDifference: pairedRiskDifference(baselinePass, treatmentPass),\n    }\n  }\n\n  const metricNames =\n    opts.metricNames ??\n    [\n      ...new Set(\n        pairs.flatMap((p) => [\n          ...Object.keys(p.baseline.metrics ?? {}),\n          ...Object.keys(p.treatment.metrics ?? {}),\n        ]),\n      ),\n    ].sort()\n\n  const metricDeltas: PairedMetricDelta[] = metricNames.map((name) => {\n    const before: number[] = []\n    const after: number[] = []\n    let nMissing = 0\n    for (const pair of pairs) {\n      const b = metricValue(pair.baseline, name)\n      const t = metricValue(pair.treatment, name)\n      if (b === undefined || t === undefined) {\n        nMissing++\n        continue\n      }\n      before.push(b)\n      after.push(t)\n    }\n    const bootstrapCi = before.length === 0 ? null : pairedBootstrap(before, after, opts.bootstrap)\n    return {\n      name,\n      n: before.length,\n      nMissing,\n      medianDelta: bootstrapCi?.median ?? null,\n      meanDelta: bootstrapCi?.mean ?? null,\n      bootstrapCi,\n      wilcoxon: before.length === 0 ? null : wilcoxonSignedRank(before, after),\n    }\n  })\n\n  return {\n    nPairs: pairs.length,\n    nUnpairedBaseline: unpairedBaseline.length,\n    nUnpairedTreatment: unpairedTreatment.length,\n    correctness,\n    metricDeltas,\n  }\n}\n\nexport interface MatchedRunRecordPair {\n  pairKey: string\n  repKey: string\n  baseline: RunRecord\n  treatment: RunRecord\n}\n\nexport interface PairRunRecordsResult {\n  pairs: MatchedRunRecordPair[]\n  unpairedBaseline: RunRecord[]\n  unpairedTreatment: RunRecord[]\n}\n\ninterface RunRecordArmRow extends PairedArmRow {\n  run: RunRecord\n  repKey: string\n}\n\n/**\n * Pair two RunRecord arms by the identity of the evaluated work:\n * `(experimentId, scenarioId, seed)`.\n *\n * Falling back to array order, candidate id, or experiment id can compare\n * different tasks and fabricate lift. Duplicate identities throw.\n */\nexport function pairRunRecords(\n  baselineRuns: readonly RunRecord[],\n  treatmentRuns: readonly RunRecord[],\n): PairRunRecordsResult {\n  const baselineRows = runRecordArmRows(baselineRuns, 'baseline')\n  const treatmentRows = runRecordArmRows(treatmentRuns, 'treatment')\n  validateRunRecordArmRows(baselineRows, 'baseline')\n  validateRunRecordArmRows(treatmentRows, 'treatment')\n  if (baselineRows.length === 0 || treatmentRows.length === 0) {\n    return {\n      pairs: [],\n      unpairedBaseline: baselineRows.map((row) => row.run),\n      unpairedTreatment: treatmentRows.map((row) => row.run),\n    }\n  }\n\n  const result = pairArms([...baselineRows, ...treatmentRows], {\n    baselineArm: 'baseline',\n    treatmentArm: 'treatment',\n  })\n  return {\n    pairs: result.pairs.map((pair) => {\n      const baseline = pair.baseline as RunRecordArmRow\n      const treatment = pair.treatment as RunRecordArmRow\n      return {\n        pairKey: pair.pairKey,\n        repKey: baseline.repKey,\n        baseline: baseline.run,\n        treatment: treatment.run,\n      }\n    }),\n    unpairedBaseline: result.unpairedBaseline.map((row) => (row as RunRecordArmRow).run),\n    unpairedTreatment: result.unpairedTreatment.map((row) => (row as RunRecordArmRow).run),\n  }\n}\n\nfunction runRecordArmRows(runs: readonly RunRecord[], arm: string): RunRecordArmRow[] {\n  return runs.map((run) => {\n    const scenarioId = run.scenarioId.trim()\n    if (!scenarioId) {\n      throw new ValidationError(\n        `pairRunRecords: run '${run.runId}' is missing scenarioId; paired comparisons require explicit scenario identity`,\n      )\n    }\n    return {\n      pairKey: JSON.stringify([run.experimentId, scenarioId]),\n      repKey: String(run.seed),\n      arm,\n      run,\n    }\n  })\n}\n\nfunction validateRunRecordArmRows(rows: readonly RunRecordArmRow[], arm: string): void {\n  const byPairKey = new Map<string, RunRecordArmRow[]>()\n  for (const row of rows) {\n    const group = byPairKey.get(row.pairKey) ?? []\n    group.push(row)\n    byPairKey.set(row.pairKey, group)\n  }\n  for (const [pairKey, group] of byPairKey) {\n    if (group.length > 1) indexByRepKey(group, pairKey, arm)\n  }\n}\n\nfunction metricValue(row: PairedArmRow, name: string): number | undefined {\n  const v = row.metrics?.[name]\n  if (v === undefined) return undefined\n  if (!Number.isFinite(v)) {\n    throw new ValidationError(\n      `comparePairedArms: non-finite value for metric '${name}' on pairKey '${row.pairKey}' (arm '${row.arm}'): ${v}`,\n    )\n  }\n  return v\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAuGA,SAAgB,SAAS,MAA+B,MAAuC;CAC7F,MAAM,EAAE,aAAa,iBAAiB;CACtC,IAAI,gBAAgB,cAClB,MAAM,IAAI,gBACR,oDAAoD,YAAY,wCAClE;CAIF,MAAM,wBAAQ,IAAI,IAAyC;CAC3D,MAAM,2BAAW,IAAI,IAAY;CACjC,KAAK,MAAM,OAAO,MAAM;EACtB,SAAS,IAAI,IAAI,GAAG;EACpB,IAAI,IAAI,QAAQ,eAAe,IAAI,QAAQ,cAAc;EACzD,MAAM,QAAQ,MAAM,IAAI,IAAI,GAAG,qBAAK,IAAI,IAA4B;EACpE,MAAM,QAAQ,MAAM,IAAI,IAAI,OAAO,KAAK,CAAC;EACzC,MAAM,KAAK,GAAG;EACd,MAAM,IAAI,IAAI,SAAS,KAAK;EAC5B,MAAM,IAAI,IAAI,KAAK,KAAK;CAC1B;CAEA,KAAK,MAAM,OAAO,CAAC,aAAa,YAAY,GAC1C,IAAI,CAAC,MAAM,IAAI,GAAG,GAEhB,MAAM,IAAI,gBAAgB,8BAA8B,IAAI,mBAD/C,CAAC,GAAG,QAAQ,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,IAAI,KAAK,SACoC,EAAE;CAI1F,MAAM,gBAAgB,MAAM,IAAI,WAAW;CAC3C,MAAM,iBAAiB,MAAM,IAAI,YAAY;CAE7C,MAAM,UAAU,CAAC,mBAAG,IAAI,IAAI,CAAC,GAAG,cAAc,KAAK,GAAG,GAAG,eAAe,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK;CACvF,MAAM,QAAuB,CAAC;CAC9B,MAAM,mBAAmC,CAAC;CAC1C,MAAM,oBAAoC,CAAC;CAC3C,KAAK,MAAM,WAAW,SAAS;EAC7B,MAAM,IAAI,cAAc,IAAI,OAAO,KAAK,CAAC;EACzC,MAAM,IAAI,eAAe,IAAI,OAAO,KAAK,CAAC;EAE1C,IAAI,EAAE,UAAU,KAAK,EAAE,UAAU,GAAG;GAClC,IAAI,EAAE,WAAW,KAAK,EAAE,WAAW,GAAG;IACpC,MAAM,WAAW,EAAE;IACnB,MAAM,YAAY,EAAE;IACpB,IAAI,SAAS,WAAW,KAAA,KAAa,UAAU,WAAW,KAAA,GAEtD;SAAA,SAAS,WAAW,KAAA,KACpB,UAAU,WAAW,KAAA,KACrB,SAAS,WAAW,UAAU,QAC9B;MACA,iBAAiB,KAAK,QAAQ;MAC9B,kBAAkB,KAAK,SAAS;MAChC;KACF;;IAEF,MAAM,KAAK;KAAE;KAAS,UAAU;KAAG;KAAU;IAAU,CAAC;GAC1D,OAAO;IACL,iBAAiB,KAAK,GAAG,CAAC;IAC1B,kBAAkB,KAAK,GAAG,CAAC;GAC7B;GACA;EACF;EAEA,MAAM,SAAS,cAAc,GAAG,SAAS,WAAW;EACpD,MAAM,SAAS,cAAc,GAAG,SAAS,YAAY;EACrD,MAAM,UAAU,CAAC,mBAAG,IAAI,IAAI,CAAC,GAAG,OAAO,KAAK,GAAG,GAAG,OAAO,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK;EACxE,IAAI,WAAW;EACf,KAAK,MAAM,UAAU,SAAS;GAC5B,MAAM,WAAW,OAAO,IAAI,MAAM;GAClC,MAAM,YAAY,OAAO,IAAI,MAAM;GACnC,IAAI,aAAa,KAAA,KAAa,cAAc,KAAA,GAC1C,MAAM,KAAK;IAAE;IAAS,UAAU;IAAY;IAAU;GAAU,CAAC;QAC5D,IAAI,aAAa,KAAA,GACtB,iBAAiB,KAAK,QAAQ;QACzB,IAAI,cAAc,KAAA,GACvB,kBAAkB,KAAK,SAAS;EAEpC;CACF;CAEA,OAAO;EAAE;EAAO;EAAkB;CAAkB;AACtD;;;AAIA,SAAS,cACP,OACA,SACA,KAC2B;CAC3B,MAAM,wBAAQ,IAAI,IAA0B;CAC5C,KAAK,MAAM,OAAO,OAAO;EACvB,IAAI,IAAI,WAAW,KAAA,GACjB,MAAM,IAAI,gBACR,sBAAsB,QAAQ,mDAAmD,IAAI,8KAGvF;EAEF,IAAI,MAAM,IAAI,IAAI,MAAM,GACtB,MAAM,IAAI,gBACR,+BAA+B,IAAI,OAAO,iBAAiB,QAAQ,YAAY,IAAI,iEAErF;EAEF,MAAM,IAAI,IAAI,QAAQ,GAAG;CAC3B;CACA,OAAO;AACT;;;;;;;;;;;;;;AAiEA,SAAgB,kBACd,MACA,MACsB;CACtB,MAAM,EAAE,OAAO,kBAAkB,sBAAsB,SAAS,MAAM,IAAI;CAE1E,IAAI,cAAwC;CAC5C,MAAM,eAAyB,CAAC;CAChC,MAAM,gBAA0B,CAAC;CACjC,KAAK,MAAM,QAAQ,OAAO;EACxB,IAAI,KAAK,SAAS,SAAS,KAAA,KAAa,KAAK,UAAU,SAAS,KAAA,GAAW;EAC3E,aAAa,KAAK,KAAK,SAAS,OAAO,IAAI,CAAC;EAC5C,cAAc,KAAK,KAAK,UAAU,OAAO,IAAI,CAAC;CAChD;CACA,IAAI,aAAa,SAAS,GAAG;EAC3B,MAAM,KAAK,QAAQ,cAAc,aAAa;EAC9C,cAAc;GACZ,KAAK,GAAG;GACR,KAAK,GAAG;GACR,SAAS;GACT,gBAAgB,qBAAqB,cAAc,aAAa;EAClE;CACF;CAaA,MAAM,gBAVJ,KAAK,eACL,CACE,GAAG,IAAI,IACL,MAAM,SAAS,MAAM,CACnB,GAAG,OAAO,KAAK,EAAE,SAAS,WAAW,CAAC,CAAC,GACvC,GAAG,OAAO,KAAK,EAAE,UAAU,WAAW,CAAC,CAAC,CAC1C,CAAC,CACH,CACF,CAAC,CAAC,KAAK,EAAA,CAE6C,KAAK,SAAS;EAClE,MAAM,SAAmB,CAAC;EAC1B,MAAM,QAAkB,CAAC;EACzB,IAAI,WAAW;EACf,KAAK,MAAM,QAAQ,OAAO;GACxB,MAAM,IAAI,YAAY,KAAK,UAAU,IAAI;GACzC,MAAM,IAAI,YAAY,KAAK,WAAW,IAAI;GAC1C,IAAI,MAAM,KAAA,KAAa,MAAM,KAAA,GAAW;IACtC;IACA;GACF;GACA,OAAO,KAAK,CAAC;GACb,MAAM,KAAK,CAAC;EACd;EACA,MAAM,cAAc,OAAO,WAAW,IAAI,OAAO,gBAAgB,QAAQ,OAAO,KAAK,SAAS;EAC9F,OAAO;GACL;GACA,GAAG,OAAO;GACV;GACA,aAAa,aAAa,UAAU;GACpC,WAAW,aAAa,QAAQ;GAChC;GACA,UAAU,OAAO,WAAW,IAAI,OAAO,mBAAmB,QAAQ,KAAK;EACzE;CACF,CAAC;CAED,OAAO;EACL,QAAQ,MAAM;EACd,mBAAmB,iBAAiB;EACpC,oBAAoB,kBAAkB;EACtC;EACA;CACF;AACF;;;;;;;;AA2BA,SAAgB,eACd,cACA,eACsB;CACtB,MAAM,eAAe,iBAAiB,cAAc,UAAU;CAC9D,MAAM,gBAAgB,iBAAiB,eAAe,WAAW;CACjE,yBAAyB,cAAc,UAAU;CACjD,yBAAyB,eAAe,WAAW;CACnD,IAAI,aAAa,WAAW,KAAK,cAAc,WAAW,GACxD,OAAO;EACL,OAAO,CAAC;EACR,kBAAkB,aAAa,KAAK,QAAQ,IAAI,GAAG;EACnD,mBAAmB,cAAc,KAAK,QAAQ,IAAI,GAAG;CACvD;CAGF,MAAM,SAAS,SAAS,CAAC,GAAG,cAAc,GAAG,aAAa,GAAG;EAC3D,aAAa;EACb,cAAc;CAChB,CAAC;CACD,OAAO;EACL,OAAO,OAAO,MAAM,KAAK,SAAS;GAChC,MAAM,WAAW,KAAK;GACtB,MAAM,YAAY,KAAK;GACvB,OAAO;IACL,SAAS,KAAK;IACd,QAAQ,SAAS;IACjB,UAAU,SAAS;IACnB,WAAW,UAAU;GACvB;EACF,CAAC;EACD,kBAAkB,OAAO,iBAAiB,KAAK,QAAS,IAAwB,GAAG;EACnF,mBAAmB,OAAO,kBAAkB,KAAK,QAAS,IAAwB,GAAG;CACvF;AACF;AAEA,SAAS,iBAAiB,MAA4B,KAAgC;CACpF,OAAO,KAAK,KAAK,QAAQ;EACvB,MAAM,aAAa,IAAI,WAAW,KAAK;EACvC,IAAI,CAAC,YACH,MAAM,IAAI,gBACR,wBAAwB,IAAI,MAAM,+EACpC;EAEF,OAAO;GACL,SAAS,KAAK,UAAU,CAAC,IAAI,cAAc,UAAU,CAAC;GACtD,QAAQ,OAAO,IAAI,IAAI;GACvB;GACA;EACF;CACF,CAAC;AACH;AAEA,SAAS,yBAAyB,MAAkC,KAAmB;CACrF,MAAM,4BAAY,IAAI,IAA+B;CACrD,KAAK,MAAM,OAAO,MAAM;EACtB,MAAM,QAAQ,UAAU,IAAI,IAAI,OAAO,KAAK,CAAC;EAC7C,MAAM,KAAK,GAAG;EACd,UAAU,IAAI,IAAI,SAAS,KAAK;CAClC;CACA,KAAK,MAAM,CAAC,SAAS,UAAU,WAC7B,IAAI,MAAM,SAAS,GAAG,cAAc,OAAO,SAAS,GAAG;AAE3D;AAEA,SAAS,YAAY,KAAmB,MAAkC;CACxE,MAAM,IAAI,IAAI,UAAU;CACxB,IAAI,MAAM,KAAA,GAAW,OAAO,KAAA;CAC5B,IAAI,CAAC,OAAO,SAAS,CAAC,GACpB,MAAM,IAAI,gBACR,mDAAmD,KAAK,gBAAgB,IAAI,QAAQ,UAAU,IAAI,IAAI,MAAM,GAC9G;CAEF,OAAO;AACT"}