@tangle-network/agent-eval 0.128.1 → 0.129.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +271 -0
- package/README.md +18 -0
- package/dist/analyst/index.d.ts +107 -165
- package/dist/analyst/index.js +5 -9
- package/dist/analyst/index.js.map +1 -1
- package/dist/belief-state/index.d.ts +2 -19
- package/dist/belief-state/index.js +30 -31
- package/dist/belief-state/index.js.map +1 -1
- package/dist/benchmarks/index.d.ts +5 -8
- package/dist/benchmarks/index.js +12 -11
- package/dist/builder-eval/index.js +1 -1
- package/dist/campaign/index.d.ts +30 -39
- package/dist/campaign/index.js +11 -10
- package/dist/{chunk-NKAGIDE2.js → chunk-2QU3YOPR.js} +15 -274
- package/dist/chunk-2QU3YOPR.js.map +1 -0
- package/dist/{chunk-EJGRPCO3.js → chunk-3OCR4R5I.js} +245 -134
- package/dist/chunk-3OCR4R5I.js.map +1 -0
- package/dist/{chunk-2JX3CFMB.js → chunk-56TAVBOK.js} +5 -2
- package/dist/chunk-56TAVBOK.js.map +1 -0
- package/dist/{chunk-DPUHNQLN.js → chunk-7FO3TNPI.js} +2 -2
- package/dist/{chunk-DJKY2TSY.js → chunk-BSO5JDQH.js} +27 -120
- package/dist/chunk-BSO5JDQH.js.map +1 -0
- package/dist/{chunk-EZJEIH2R.js → chunk-C6LXANRU.js} +11 -20
- package/dist/chunk-C6LXANRU.js.map +1 -0
- package/dist/{chunk-ZUUWPZCV.js → chunk-DODXQREJ.js} +4 -4
- package/dist/{chunk-2MKQIFS4.js → chunk-E7QXT7SX.js} +2 -2
- package/dist/{chunk-NYLOYM6N.js → chunk-EG66UGL4.js} +37 -28
- package/dist/chunk-EG66UGL4.js.map +1 -0
- package/dist/{chunk-P5W7RQKK.js → chunk-FXTVJPYD.js} +2 -2
- package/dist/{chunk-VZSRQ272.js → chunk-G7MGMCZD.js} +6 -2
- package/dist/chunk-G7MGMCZD.js.map +1 -0
- package/dist/{chunk-IHQDPH7D.js → chunk-H23X7XKK.js} +85 -75
- package/dist/chunk-H23X7XKK.js.map +1 -0
- package/dist/{chunk-VBQ3CRKH.js → chunk-HPWUNB47.js} +4 -6
- package/dist/chunk-HPWUNB47.js.map +1 -0
- package/dist/{chunk-XPRT64IE.js → chunk-IYCLP2N2.js} +3 -3
- package/dist/chunk-IYCLP2N2.js.map +1 -0
- package/dist/{chunk-XDWDC2MP.js → chunk-JQSF5DQT.js} +11 -5
- package/dist/chunk-JQSF5DQT.js.map +1 -0
- package/dist/{chunk-NACAGYSY.js → chunk-M4YBQKIJ.js} +11 -11
- package/dist/chunk-M4YBQKIJ.js.map +1 -0
- package/dist/{chunk-YJBNWCAA.js → chunk-NY44NC4A.js} +3 -3
- package/dist/chunk-OIUOT4QD.js +44 -0
- package/dist/chunk-OIUOT4QD.js.map +1 -0
- package/dist/chunk-OWN5NPMC.js +152 -0
- package/dist/chunk-OWN5NPMC.js.map +1 -0
- package/dist/chunk-PC5DOSM7.js +579 -0
- package/dist/chunk-PC5DOSM7.js.map +1 -0
- package/dist/{chunk-UB2LOJ6Q.js → chunk-QB6BDBP2.js} +23 -20
- package/dist/chunk-QB6BDBP2.js.map +1 -0
- package/dist/chunk-RXHCETDZ.js +536 -0
- package/dist/chunk-RXHCETDZ.js.map +1 -0
- package/dist/{chunk-PBE2LOSS.js → chunk-SFLLL76A.js} +7 -7
- package/dist/chunk-SFLLL76A.js.map +1 -0
- package/dist/{chunk-VGRCHJON.js → chunk-T6RLYGAD.js} +3 -8
- package/dist/chunk-T6RLYGAD.js.map +1 -0
- package/dist/{chunk-VLOATJQ2.js → chunk-TJVT4QFF.js} +21 -18
- package/dist/chunk-TJVT4QFF.js.map +1 -0
- package/dist/{chunk-S5YLIBFX.js → chunk-TQ7LNKZ3.js} +2 -2
- package/dist/{chunk-EOSZT7PL.js → chunk-U4L7JRPZ.js} +2 -297
- package/dist/chunk-U4L7JRPZ.js.map +1 -0
- package/dist/chunk-U4PHLT2N.js +419 -0
- package/dist/chunk-U4PHLT2N.js.map +1 -0
- package/dist/{chunk-WS3NZZQQ.js → chunk-VCZ5FQYW.js} +3 -4
- package/dist/chunk-VCZ5FQYW.js.map +1 -0
- package/dist/{chunk-BYT7ELPS.js → chunk-WVATSFCP.js} +2 -2
- package/dist/{chunk-TSN7JT6D.js → chunk-X4YIBDER.js} +21 -5
- package/dist/{chunk-TSN7JT6D.js.map → chunk-X4YIBDER.js.map} +1 -1
- package/dist/{chunk-TBL77AUT.js → chunk-YQN4ICPP.js} +5 -5
- package/dist/{chunk-MHELPNRP.js → chunk-ZHTZ4EYI.js} +1 -1
- package/dist/chunk-ZHTZ4EYI.js.map +1 -0
- package/dist/cli.js +6 -5
- package/dist/cli.js.map +1 -1
- package/dist/contract/index.d.ts +47 -87
- package/dist/contract/index.js +14 -13
- package/dist/contract/index.js.map +1 -1
- package/dist/control.js +3 -2
- package/dist/fuzz.js +3 -2
- package/dist/fuzz.js.map +1 -1
- package/dist/index.d.ts +659 -203
- package/dist/index.js +145 -117
- package/dist/index.js.map +1 -1
- package/dist/meta-eval/index.js +2 -2
- package/dist/multishot/index.d.ts +3 -4
- package/dist/multishot/index.js.map +1 -1
- package/dist/openapi.json +1 -1
- package/dist/pipelines/index.js +5 -5
- package/dist/reporting.d.ts +14 -0
- package/dist/reporting.js +7 -6
- package/dist/rl.d.ts +652 -82
- package/dist/rl.js +415 -171
- package/dist/rl.js.map +1 -1
- package/dist/rollout/index.d.ts +1071 -32
- package/dist/rollout/index.js +68 -10
- package/dist/run-campaign-OJJ7CZF4.js +18 -0
- package/dist/supervisor-run/index.d.ts +114 -4
- package/dist/supervisor-run/index.js +4 -3
- package/dist/traces.d.ts +1 -1
- package/dist/traces.js +6 -5
- package/dist/wire/index.d.ts +10 -11
- package/dist/wire/index.js +3 -3
- package/docs/feature-guide.md +1 -1
- package/docs/rollout.md +116 -2
- package/package.json +4 -4
- package/dist/chunk-2JX3CFMB.js.map +0 -1
- package/dist/chunk-DJKY2TSY.js.map +0 -1
- package/dist/chunk-EJGRPCO3.js.map +0 -1
- package/dist/chunk-EOSZT7PL.js.map +0 -1
- package/dist/chunk-EZJEIH2R.js.map +0 -1
- package/dist/chunk-IHQDPH7D.js.map +0 -1
- package/dist/chunk-MHELPNRP.js.map +0 -1
- package/dist/chunk-NACAGYSY.js.map +0 -1
- package/dist/chunk-NKAGIDE2.js.map +0 -1
- package/dist/chunk-NYLOYM6N.js.map +0 -1
- package/dist/chunk-PBE2LOSS.js.map +0 -1
- package/dist/chunk-TT4KNT67.js +0 -124
- package/dist/chunk-TT4KNT67.js.map +0 -1
- package/dist/chunk-UB2LOJ6Q.js.map +0 -1
- package/dist/chunk-UWZZKKU7.js +0 -237
- package/dist/chunk-UWZZKKU7.js.map +0 -1
- package/dist/chunk-VBQ3CRKH.js.map +0 -1
- package/dist/chunk-VGRCHJON.js.map +0 -1
- package/dist/chunk-VLOATJQ2.js.map +0 -1
- package/dist/chunk-VZSRQ272.js.map +0 -1
- package/dist/chunk-WS3NZZQQ.js.map +0 -1
- package/dist/chunk-XDWDC2MP.js.map +0 -1
- package/dist/chunk-XPRT64IE.js.map +0 -1
- package/dist/run-campaign-ISHFZ7FJ.js +0 -17
- /package/dist/{chunk-DPUHNQLN.js.map → chunk-7FO3TNPI.js.map} +0 -0
- /package/dist/{chunk-ZUUWPZCV.js.map → chunk-DODXQREJ.js.map} +0 -0
- /package/dist/{chunk-2MKQIFS4.js.map → chunk-E7QXT7SX.js.map} +0 -0
- /package/dist/{chunk-P5W7RQKK.js.map → chunk-FXTVJPYD.js.map} +0 -0
- /package/dist/{chunk-YJBNWCAA.js.map → chunk-NY44NC4A.js.map} +0 -0
- /package/dist/{chunk-S5YLIBFX.js.map → chunk-TQ7LNKZ3.js.map} +0 -0
- /package/dist/{chunk-BYT7ELPS.js.map → chunk-WVATSFCP.js.map} +0 -0
- /package/dist/{chunk-TBL77AUT.js.map → chunk-YQN4ICPP.js.map} +0 -0
- /package/dist/{run-campaign-ISHFZ7FJ.js.map → run-campaign-OJJ7CZF4.js.map} +0 -0
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/rl/active-curriculum.ts"],"sourcesContent":["/**\n * Adaptive curriculum / active scenario selection.\n *\n * Fixed scenario sets waste sample budget on cells the policy already\n * passes (no information left) and cells the policy never passes (no\n * gradient available either). Active learning over scenarios fixes this\n * by allocating the next sample budget to cells where the policy's\n * outcome is *uncertain* — those carry the most decision-relevant signal.\n *\n * This module ships two complementary strategies:\n *\n * 1. **Variance-based** — score each (variant, scenario) cell by the\n * empirical variance of past observations. Allocate next-round budget\n * proportional to variance. Standard active-learning-by-uncertainty\n * heuristic; works well when the policy is non-deterministic and\n * cells differ in observation noise.\n *\n * 2. **Bandit-based (Thompson sampling)** — model each (variant,\n * scenario) cell as a Beta-Bernoulli arm; sample a posterior; pick\n * cells whose posterior mean is closest to the per-scenario decision\n * threshold. The right primitive when scenarios are\n * \"pass/fail\" rather than continuous, and when promotion gates fire\n * at a known threshold (e.g., 0.5).\n *\n * The output is a *next-round budget allocation* — a list of (variant,\n * scenario, count) triples. The consumer's matrix runner consumes the\n * allocation, runs those cells, feeds the new observations back. Loop.\n *\n * Out of scope (deliberate): scenario *generation* — that's the\n * adversarial primitive's job. This module allocates over an existing\n * scenario pool.\n */\n\nimport type { RunRecord } from '../run-record'\n\nexport interface CellObservation {\n variantId: string\n scenarioId: string\n /** Observed score in [0, 1]. */\n score: number\n /** For Bernoulli arms — derive from the score with a threshold if needed. */\n pass?: boolean\n}\n\nexport interface CurriculumAllocation {\n variantId: string\n scenarioId: string\n /** How many additional reps to run on this cell. */\n count: number\n /** Strategy-specific reason for the allocation. */\n reason: string\n}\n\nexport interface VarianceCurriculumOptions {\n /** Total reps to allocate across all cells. */\n budget: number\n /**\n * Smoothing prior on variance — keeps the allocator from concentrating\n * on a cell with one observation just because its 1-sample variance is\n * 0. Default 0.05.\n */\n variancePrior?: number\n /**\n * Minimum reps per cell — even when the variance estimate is low, give\n * every cell at least this many. Default 1.\n */\n floorPerCell?: number\n}\n\n/**\n * Variance-proportional allocation. For each cell, estimate variance from\n * past observations + a prior, then allocate the budget proportional to\n * (sqrt(variance) + 1/sqrt(n)) — a classical optimal-allocation rule\n * (Neyman 1934) that balances \"explore noisy cells\" with \"explore\n * under-sampled cells.\"\n */\nexport function varianceBasedCurriculum(\n observations: CellObservation[],\n candidateCells: Array<{ variantId: string; scenarioId: string }>,\n opts: VarianceCurriculumOptions,\n): CurriculumAllocation[] {\n const variancePrior = opts.variancePrior ?? 0.05\n const floor = opts.floorPerCell ?? 1\n const budget = opts.budget\n\n const grouped = new Map<string, number[]>()\n for (const o of observations) {\n const k = `${o.variantId}::${o.scenarioId}`\n const arr = grouped.get(k) ?? []\n arr.push(o.score)\n grouped.set(k, arr)\n }\n\n const cellStats = candidateCells.map((c) => {\n const k = `${c.variantId}::${c.scenarioId}`\n const samples = grouped.get(k) ?? []\n const n = samples.length\n const mean = n === 0 ? 0.5 : samples.reduce((s, v) => s + v, 0) / n\n const variance =\n n < 2\n ? variancePrior\n : samples.reduce((s, v) => s + (v - mean) ** 2, 0) / (n - 1) + variancePrior\n // Neyman optimal allocation: weight ∝ √variance; add √(1/n) to break\n // ties toward under-sampled cells.\n const weight = Math.sqrt(variance) + 1 / Math.sqrt(Math.max(1, n))\n return { variantId: c.variantId, scenarioId: c.scenarioId, n, mean, variance, weight }\n })\n\n // Reserve floor*N for the floor; allocate the rest proportional to weight.\n const floorTotal = floor * cellStats.length\n if (floorTotal >= budget) {\n const each = Math.max(1, Math.floor(budget / Math.max(1, cellStats.length)))\n return cellStats.map((c) => ({\n variantId: c.variantId,\n scenarioId: c.scenarioId,\n count: each,\n reason: `floor allocation (budget tight; n=${c.n})`,\n }))\n }\n const remaining = budget - floorTotal\n const totalWeight = cellStats.reduce((s, c) => s + c.weight, 0)\n return cellStats.map((c) => {\n const proportional = totalWeight === 0 ? 0 : Math.round((c.weight / totalWeight) * remaining)\n return {\n variantId: c.variantId,\n scenarioId: c.scenarioId,\n count: floor + proportional,\n reason: `variance ${c.variance.toFixed(3)} (n=${c.n}, mean=${c.mean.toFixed(3)})`,\n }\n })\n}\n\nexport interface ThompsonCurriculumOptions {\n budget: number\n /**\n * The per-scenario decision threshold. Cells whose posterior mean is\n * closest to this get the most budget — that's where the next observation\n * has the highest information value for the gate decision. Default 0.5.\n */\n decisionThreshold?: number\n /** Beta prior parameters. Default α=β=1 (uniform). */\n priorAlpha?: number\n priorBeta?: number\n /** Seed the Thompson sampler. Default unset (Math.random). */\n seed?: number\n}\n\n/**\n * Thompson-sampling-style allocation for pass/fail cells. For each cell:\n *\n * - Maintain Beta(α + passes, β + failures) posterior on pass-rate\n * - Allocation weight ∝ exp(-((sampledMean - threshold) / σ)^2):\n * cells whose sampled posterior straddles the decision boundary get\n * the most weight; cells already clearly above or below get less.\n *\n * This is the right primitive when promotion gates fire at a known\n * threshold and you want to sharpen the posterior near the boundary.\n */\nexport function thompsonCurriculum(\n observations: CellObservation[],\n candidateCells: Array<{ variantId: string; scenarioId: string }>,\n opts: ThompsonCurriculumOptions,\n): CurriculumAllocation[] {\n const threshold = opts.decisionThreshold ?? 0.5\n const alpha0 = opts.priorAlpha ?? 1\n const beta0 = opts.priorBeta ?? 1\n const rng = makeRng(opts.seed)\n\n const grouped = new Map<string, { passes: number; failures: number }>()\n for (const o of observations) {\n const k = `${o.variantId}::${o.scenarioId}`\n const cur = grouped.get(k) ?? { passes: 0, failures: 0 }\n const pass = o.pass ?? o.score >= threshold\n if (pass) cur.passes += 1\n else cur.failures += 1\n grouped.set(k, cur)\n }\n\n const stats = candidateCells.map((c) => {\n const k = `${c.variantId}::${c.scenarioId}`\n const cur = grouped.get(k) ?? { passes: 0, failures: 0 }\n const a = alpha0 + cur.passes\n const b = beta0 + cur.failures\n // Sample a single Beta draw — the Thompson signal.\n const sampled = sampleBeta(a, b, rng)\n const distance = Math.abs(sampled - threshold)\n // Information-near-threshold weight: closer = higher.\n // Use Gaussian-shaped kernel with σ tuned to posterior std.\n const variance = (a * b) / ((a + b) ** 2 * (a + b + 1))\n const sigma = Math.max(0.05, Math.sqrt(variance))\n const weight = Math.exp(-((distance / sigma) ** 2))\n return {\n variantId: c.variantId,\n scenarioId: c.scenarioId,\n n: cur.passes + cur.failures,\n sampled,\n sigma,\n weight,\n a,\n b,\n }\n })\n\n const totalWeight = stats.reduce((s, c) => s + c.weight, 0)\n return stats.map((c) => {\n const proportional = totalWeight === 0 ? 0 : Math.round((c.weight / totalWeight) * opts.budget)\n return {\n variantId: c.variantId,\n scenarioId: c.scenarioId,\n count: Math.max(0, proportional),\n reason: `Beta(${c.a.toFixed(1)},${c.b.toFixed(1)}) sample=${c.sampled.toFixed(3)} (target ${threshold})`,\n }\n })\n}\n\n/** Convenience: extract `CellObservation[]` directly from `RunRecord[]`. */\nexport function observationsFromRunRecords(\n runs: RunRecord[],\n opts: { passThreshold?: number; useHoldout?: boolean } = {},\n): CellObservation[] {\n const threshold = opts.passThreshold ?? 0.5\n const useHoldout = opts.useHoldout ?? true\n const out: CellObservation[] = []\n for (const r of runs) {\n if (!r.scenarioId) continue\n const score = useHoldout\n ? (r.outcome.holdoutScore ?? r.outcome.searchScore)\n : (r.outcome.searchScore ?? r.outcome.holdoutScore)\n if (typeof score !== 'number' || !Number.isFinite(score)) continue\n out.push({\n variantId: r.candidateId,\n scenarioId: r.scenarioId,\n score,\n pass: score >= threshold,\n })\n }\n return out\n}\n\n// ── Helpers ──────────────────────────────────────────────────────────────\n\nfunction makeRng(seed?: number): () => number {\n if (seed === undefined) return Math.random\n let s = seed >>> 0\n return () => {\n s = (s + 0x6d2b79f5) >>> 0\n let t = s\n t = Math.imul(t ^ (t >>> 15), t | 1)\n t ^= t + Math.imul(t ^ (t >>> 7), t | 61)\n return ((t ^ (t >>> 14)) >>> 0) / 4294967296\n }\n}\n\n/**\n * Sample from Beta(α, β) via the Marsaglia–Tsang method using two Gamma\n * variates. Accuracy is good for α, β > 1; we floor the parameters at 1\n * to avoid degenerate cases.\n */\nfunction sampleBeta(alpha: number, beta: number, rng: () => number): number {\n const a = Math.max(1, alpha)\n const b = Math.max(1, beta)\n const x = sampleGamma(a, rng)\n const y = sampleGamma(b, rng)\n return x / (x + y)\n}\n\nfunction sampleGamma(shape: number, rng: () => number): number {\n // Marsaglia–Tsang for shape ≥ 1.\n const d = shape - 1 / 3\n const c = 1 / Math.sqrt(9 * d)\n while (true) {\n let x: number\n let v: number\n do {\n const u1 = rng() || 1e-12\n const u2 = rng() || 1e-12\n // Box-Muller for a normal sample.\n x = Math.sqrt(-2 * Math.log(u1)) * Math.cos(2 * Math.PI * u2)\n v = 1 + c * x\n } while (v <= 0)\n v = v * v * v\n const u = rng()\n if (u < 1 - 0.0331 * x ** 4) return d * v\n if (Math.log(u) < 0.5 * x * x + d * (1 - v + Math.log(v))) return d * v\n }\n}\n"],"mappings":";AA4EO,SAAS,wBACd,cACA,gBACA,MACwB;AACxB,QAAM,gBAAgB,KAAK,iBAAiB;AAC5C,QAAM,QAAQ,KAAK,gBAAgB;AACnC,QAAM,SAAS,KAAK;AAEpB,QAAM,UAAU,oBAAI,IAAsB;AAC1C,aAAW,KAAK,cAAc;AAC5B,UAAM,IAAI,GAAG,EAAE,SAAS,KAAK,EAAE,UAAU;AACzC,UAAM,MAAM,QAAQ,IAAI,CAAC,KAAK,CAAC;AAC/B,QAAI,KAAK,EAAE,KAAK;AAChB,YAAQ,IAAI,GAAG,GAAG;AAAA,EACpB;AAEA,QAAM,YAAY,eAAe,IAAI,CAAC,MAAM;AAC1C,UAAM,IAAI,GAAG,EAAE,SAAS,KAAK,EAAE,UAAU;AACzC,UAAM,UAAU,QAAQ,IAAI,CAAC,KAAK,CAAC;AACnC,UAAM,IAAI,QAAQ;AAClB,UAAM,OAAO,MAAM,IAAI,MAAM,QAAQ,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI;AAClE,UAAM,WACJ,IAAI,IACA,gBACA,QAAQ,OAAO,CAAC,GAAG,MAAM,KAAK,IAAI,SAAS,GAAG,CAAC,KAAK,IAAI,KAAK;AAGnE,UAAM,SAAS,KAAK,KAAK,QAAQ,IAAI,IAAI,KAAK,KAAK,KAAK,IAAI,GAAG,CAAC,CAAC;AACjE,WAAO,EAAE,WAAW,EAAE,WAAW,YAAY,EAAE,YAAY,GAAG,MAAM,UAAU,OAAO;AAAA,EACvF,CAAC;AAGD,QAAM,aAAa,QAAQ,UAAU;AACrC,MAAI,cAAc,QAAQ;AACxB,UAAM,OAAO,KAAK,IAAI,GAAG,KAAK,MAAM,SAAS,KAAK,IAAI,GAAG,UAAU,MAAM,CAAC,CAAC;AAC3E,WAAO,UAAU,IAAI,CAAC,OAAO;AAAA,MAC3B,WAAW,EAAE;AAAA,MACb,YAAY,EAAE;AAAA,MACd,OAAO;AAAA,MACP,QAAQ,qCAAqC,EAAE,CAAC;AAAA,IAClD,EAAE;AAAA,EACJ;AACA,QAAM,YAAY,SAAS;AAC3B,QAAM,cAAc,UAAU,OAAO,CAAC,GAAG,MAAM,IAAI,EAAE,QAAQ,CAAC;AAC9D,SAAO,UAAU,IAAI,CAAC,MAAM;AAC1B,UAAM,eAAe,gBAAgB,IAAI,IAAI,KAAK,MAAO,EAAE,SAAS,cAAe,SAAS;AAC5F,WAAO;AAAA,MACL,WAAW,EAAE;AAAA,MACb,YAAY,EAAE;AAAA,MACd,OAAO,QAAQ;AAAA,MACf,QAAQ,YAAY,EAAE,SAAS,QAAQ,CAAC,CAAC,OAAO,EAAE,CAAC,UAAU,EAAE,KAAK,QAAQ,CAAC,CAAC;AAAA,IAChF;AAAA,EACF,CAAC;AACH;AA4BO,SAAS,mBACd,cACA,gBACA,MACwB;AACxB,QAAM,YAAY,KAAK,qBAAqB;AAC5C,QAAM,SAAS,KAAK,cAAc;AAClC,QAAM,QAAQ,KAAK,aAAa;AAChC,QAAM,MAAM,QAAQ,KAAK,IAAI;AAE7B,QAAM,UAAU,oBAAI,IAAkD;AACtE,aAAW,KAAK,cAAc;AAC5B,UAAM,IAAI,GAAG,EAAE,SAAS,KAAK,EAAE,UAAU;AACzC,UAAM,MAAM,QAAQ,IAAI,CAAC,KAAK,EAAE,QAAQ,GAAG,UAAU,EAAE;AACvD,UAAM,OAAO,EAAE,QAAQ,EAAE,SAAS;AAClC,QAAI,KAAM,KAAI,UAAU;AAAA,QACnB,KAAI,YAAY;AACrB,YAAQ,IAAI,GAAG,GAAG;AAAA,EACpB;AAEA,QAAM,QAAQ,eAAe,IAAI,CAAC,MAAM;AACtC,UAAM,IAAI,GAAG,EAAE,SAAS,KAAK,EAAE,UAAU;AACzC,UAAM,MAAM,QAAQ,IAAI,CAAC,KAAK,EAAE,QAAQ,GAAG,UAAU,EAAE;AACvD,UAAM,IAAI,SAAS,IAAI;AACvB,UAAM,IAAI,QAAQ,IAAI;AAEtB,UAAM,UAAU,WAAW,GAAG,GAAG,GAAG;AACpC,UAAM,WAAW,KAAK,IAAI,UAAU,SAAS;AAG7C,UAAM,WAAY,IAAI,MAAO,IAAI,MAAM,KAAK,IAAI,IAAI;AACpD,UAAM,QAAQ,KAAK,IAAI,MAAM,KAAK,KAAK,QAAQ,CAAC;AAChD,UAAM,SAAS,KAAK,IAAI,GAAG,WAAW,UAAU,EAAE;AAClD,WAAO;AAAA,MACL,WAAW,EAAE;AAAA,MACb,YAAY,EAAE;AAAA,MACd,GAAG,IAAI,SAAS,IAAI;AAAA,MACpB;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF,CAAC;AAED,QAAM,cAAc,MAAM,OAAO,CAAC,GAAG,MAAM,IAAI,EAAE,QAAQ,CAAC;AAC1D,SAAO,MAAM,IAAI,CAAC,MAAM;AACtB,UAAM,eAAe,gBAAgB,IAAI,IAAI,KAAK,MAAO,EAAE,SAAS,cAAe,KAAK,MAAM;AAC9F,WAAO;AAAA,MACL,WAAW,EAAE;AAAA,MACb,YAAY,EAAE;AAAA,MACd,OAAO,KAAK,IAAI,GAAG,YAAY;AAAA,MAC/B,QAAQ,QAAQ,EAAE,EAAE,QAAQ,CAAC,CAAC,IAAI,EAAE,EAAE,QAAQ,CAAC,CAAC,YAAY,EAAE,QAAQ,QAAQ,CAAC,CAAC,YAAY,SAAS;AAAA,IACvG;AAAA,EACF,CAAC;AACH;AAGO,SAAS,2BACd,MACA,OAAyD,CAAC,GACvC;AACnB,QAAM,YAAY,KAAK,iBAAiB;AACxC,QAAM,aAAa,KAAK,cAAc;AACtC,QAAM,MAAyB,CAAC;AAChC,aAAW,KAAK,MAAM;AACpB,QAAI,CAAC,EAAE,WAAY;AACnB,UAAM,QAAQ,aACT,EAAE,QAAQ,gBAAgB,EAAE,QAAQ,cACpC,EAAE,QAAQ,eAAe,EAAE,QAAQ;AACxC,QAAI,OAAO,UAAU,YAAY,CAAC,OAAO,SAAS,KAAK,EAAG;AAC1D,QAAI,KAAK;AAAA,MACP,WAAW,EAAE;AAAA,MACb,YAAY,EAAE;AAAA,MACd;AAAA,MACA,MAAM,SAAS;AAAA,IACjB,CAAC;AAAA,EACH;AACA,SAAO;AACT;AAIA,SAAS,QAAQ,MAA6B;AAC5C,MAAI,SAAS,OAAW,QAAO,KAAK;AACpC,MAAI,IAAI,SAAS;AACjB,SAAO,MAAM;AACX,QAAK,IAAI,eAAgB;AACzB,QAAI,IAAI;AACR,QAAI,KAAK,KAAK,IAAK,MAAM,IAAK,IAAI,CAAC;AACnC,SAAK,IAAI,KAAK,KAAK,IAAK,MAAM,GAAI,IAAI,EAAE;AACxC,aAAS,IAAK,MAAM,QAAS,KAAK;AAAA,EACpC;AACF;AAOA,SAAS,WAAW,OAAe,MAAc,KAA2B;AAC1E,QAAM,IAAI,KAAK,IAAI,GAAG,KAAK;AAC3B,QAAM,IAAI,KAAK,IAAI,GAAG,IAAI;AAC1B,QAAM,IAAI,YAAY,GAAG,GAAG;AAC5B,QAAM,IAAI,YAAY,GAAG,GAAG;AAC5B,SAAO,KAAK,IAAI;AAClB;AAEA,SAAS,YAAY,OAAe,KAA2B;AAE7D,QAAM,IAAI,QAAQ,IAAI;AACtB,QAAM,IAAI,IAAI,KAAK,KAAK,IAAI,CAAC;AAC7B,SAAO,MAAM;AACX,QAAI;AACJ,QAAI;AACJ,OAAG;AACD,YAAM,KAAK,IAAI,KAAK;AACpB,YAAM,KAAK,IAAI,KAAK;AAEpB,UAAI,KAAK,KAAK,KAAK,KAAK,IAAI,EAAE,CAAC,IAAI,KAAK,IAAI,IAAI,KAAK,KAAK,EAAE;AAC5D,UAAI,IAAI,IAAI;AAAA,IACd,SAAS,KAAK;AACd,QAAI,IAAI,IAAI;AACZ,UAAM,IAAI,IAAI;AACd,QAAI,IAAI,IAAI,SAAS,KAAK,EAAG,QAAO,IAAI;AACxC,QAAI,KAAK,IAAI,CAAC,IAAI,MAAM,IAAI,IAAI,KAAK,IAAI,IAAI,KAAK,IAAI,CAAC,GAAI,QAAO,IAAI;AAAA,EACxE;AACF;","names":[]}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/cost-ledger.ts"],"sourcesContent":["import { z } from 'zod'\nimport { ValidationError } from './errors'\nimport { estimateCost, isModelPriced, resolveModelPricing } from './metrics'\n\nexport type CostChannel = 'agent' | 'judge' | 'verifier' | 'analyst' | 'driver' | (string & {})\n\nexport interface CostUsage {\n inputTokens: number\n /** Includes reasoning tokens when the provider bills them as output. */\n outputTokens: number\n /** Reasoning-token subset of outputTokens, when reported. */\n reasoningTokens?: number\n /** Prompt tokens served from a provider cache. */\n cachedTokens?: number\n /** Prompt tokens written into a provider cache. */\n cacheWriteTokens?: number\n}\n\ninterface CostCallBase {\n callId: string\n channel: CostChannel\n phase: string\n actor: string\n model: string\n maximumCostUsd?: number\n tags?: Record<string, string>\n timestamp: number\n}\n\nexport interface PendingCostCall extends CostCallBase {\n status: 'pending'\n}\n\nexport interface PendingCostCallView extends PendingCostCall {\n state: 'active' | 'late' | 'interrupted'\n}\n\nexport interface CostReceipt extends CostCallBase, CostUsage {\n status: 'settled'\n costUsd: number\n costUnknown: boolean\n usageUnknown?: boolean\n /** Rates used to estimate cost locally. Absent when cost is provider-reported or unknown. */\n pricing?: {\n inputUsdPerThousand: number\n cachedInputUsdPerThousand?: number\n cacheWriteUsdPerThousand?: number\n outputUsdPerThousand: number\n }\n /** Cost reported by the provider, not a local token-price calculation. */\n actualCostUsd?: number\n error?: string\n}\n\nexport type CostLedgerRecord = PendingCostCall | CostReceipt\n\n/** @deprecated Read-only compatibility shape. New paid work uses `runPaidCall`. */\nexport type CostLedgerEntry = Omit<\n CostReceipt,\n 'status' | 'callId' | 'phase' | 'actor' | 'maximumCostUsd' | 'usageUnknown' | 'pricing' | 'error'\n>\n\nexport interface CostReceiptInput extends CostUsage {\n model: string\n /** Caller-supplied rates for a local estimate when the provider does not report billed cost. */\n customTokenPricing?: CustomTokenPricing\n actualCostUsd?: number\n costUnknown?: boolean\n usageUnknown?: boolean\n}\n\n/** Per-million token rates for a model or endpoint not covered by package pricing. */\nexport interface CustomTokenPricing {\n /** Non-cached input tokens. */\n inputUsdPerMillion: number\n /** Cache-read tokens. Falls back to the normal input rate when omitted. */\n cachedInputUsdPerMillion?: number\n /** Cache-creation or cache-write tokens. Falls back to the normal input rate when omitted. */\n cacheWriteUsdPerMillion?: number\n outputUsdPerMillion: number\n}\n\nexport type MaximumCharge =\n | { externallyEnforcedMaximumUsd: number }\n | ({\n customTokenPricing: CustomTokenPricing\n } & Pick<CostUsage, 'inputTokens' | 'outputTokens' | 'cachedTokens' | 'cacheWriteTokens'>)\n | ({ model: string } & CostUsage)\n\nexport interface RunPaidCallInput<T> {\n callId?: string\n channel: CostChannel\n phase: string\n actor: string\n /** Used before a provider receipt exists and on failures without one. */\n model?: string\n tags?: Record<string, string>\n signal?: AbortSignal\n /** Provider-enforced dollar maximum, or maximum token usage with known pricing. Required when capped. */\n maximumCharge?: MaximumCharge\n /** `callId` can be forwarded as the provider's idempotency key. */\n execute(signal: AbortSignal, callId: string): Promise<T>\n receipt(value: T): CostReceiptInput\n receiptFromError?(error: Error): CostReceiptInput | undefined\n}\n\nexport type PaidCallResult<T> =\n | { succeeded: true; callId: string; value: T; receipt: CostReceipt }\n | { succeeded: false; callId?: string; error: Error; receipt?: CostReceipt }\n\nexport interface ChannelRollup {\n channel: CostChannel\n calls: number\n inputTokens: number\n outputTokens: number\n reasoningTokens?: number\n cachedTokens: number\n cacheWriteTokens?: number\n costUsd: number\n unpricedCalls: number\n unknownUsageCalls: number\n}\n\nexport interface CostLedgerSummary {\n totalCalls: number\n pendingCalls: number\n unresolvedCalls: number\n reservedCostUsd: number\n inputTokens: number\n outputTokens: number\n reasoningTokens?: number\n cachedTokens: number\n cacheWriteTokens?: number\n totalCostUsd: number\n byChannel: ChannelRollup[]\n unpricedModels: string[]\n fullyPriced: boolean\n usageComplete: boolean\n accountingComplete: boolean\n incompleteReasons: string[]\n}\n\nexport interface CostLedgerFilter {\n channel?: CostChannel\n phase?: string\n tags?: Record<string, string>\n}\n\nexport interface CostLedgerWaitOptions {\n /** Maximum time to wait for active provider calls. Default 5 seconds. */\n timeoutMs?: number\n /** Wait only for calls matching this attribution filter. */\n filter?: CostLedgerFilter\n}\n\n/** Append-only storage. `append` must atomically reject stale revisions. */\nexport interface CostLedgerPersistence {\n read(): { revision: string; events: string }\n append(expectedRevision: string, event: string): string | undefined\n}\n\nexport interface CostLedgerOptions {\n costCeilingUsd?: number\n persistence?: CostLedgerPersistence\n /** Import already-settled receipts without admitting new paid work. */\n receipts?: readonly CostReceipt[]\n}\n\nexport class CostCeilingReachedError extends ValidationError {\n constructor(\n ceilingUsd: number,\n committedAndReservedUsd: number,\n requestedUsd: number,\n phase: string,\n actor: string,\n ) {\n super(\n `CostLedger: reserving ${requestedUsd} for '${actor}' during '${phase}' would exceed ceiling ${ceilingUsd} with ${committedAndReservedUsd} already committed or reserved`,\n )\n }\n}\n\nexport class CostAccountingIncompleteError extends ValidationError {}\n\nexport class CostReservationExceededError extends ValidationError {\n constructor(actor: string, actualUsd: number, maximumUsd: number) {\n super(\n `CostLedger: '${actor}' charged ${actualUsd}, exceeding its enforced maximum ${maximumUsd}`,\n )\n }\n}\n\nexport class CostCallConflictError extends ValidationError {\n readonly callId?: string\n readonly receipt?: CostReceipt\n\n constructor(\n message: string,\n options: { callId?: string; receipt?: CostReceipt; cause?: unknown } = {},\n ) {\n super(message, options.cause === undefined ? undefined : { cause: options.cause })\n this.callId = options.callId\n this.receipt = options.receipt ? cloneReceipt(options.receipt) : undefined\n }\n}\n\nexport class CostLedgerPersistenceError extends ValidationError {\n readonly callId?: string\n readonly receipt?: CostReceipt\n\n constructor(cause: unknown, callId?: string, receipt?: CostReceipt) {\n super(\n `CostLedger: failed to persist${callId ? ` call '${callId}'` : ''}: ${toError(cause).message}`,\n { cause },\n )\n this.callId = callId\n this.receipt = receipt ? cloneReceipt(receipt) : undefined\n }\n}\n\nexport class CostReceiptCaptureError extends ValidationError {\n readonly callId: string\n readonly receipt?: CostReceipt\n readonly receiptError: Error\n\n constructor(callId: string, cause: unknown, receiptError: unknown, receipt?: CostReceipt) {\n super(`CostLedger: could not capture the provider receipt for call '${callId}'`, { cause })\n this.callId = callId\n this.receipt = receipt ? cloneReceipt(receipt) : undefined\n this.receiptError = toError(receiptError)\n }\n}\n\ninterface CostCallEventV1 {\n version: 1\n record: CostLedgerRecord\n}\n\ninterface CostCallEventV2 {\n version: 2\n record: CostReceipt\n}\n\ninterface CompletedTasksEvent {\n version: 1\n completedTasks: number\n}\n\ninterface CostLimitEvent {\n version: 1\n costCeilingUsd: number\n}\n\ntype CostCallEvent = CostCallEventV1 | CostCallEventV2\ntype CostLedgerEvent = CostCallEvent | CompletedTasksEvent | CostLimitEvent\n\n/** Run-wide paid-call admission, durable call state, receipts, and summaries. */\nexport class CostLedger {\n private readonly records = new Map<string, CostLedgerRecord>()\n private readonly activeCallIds = new Set<string>()\n private readonly lateCallIds = new Set<string>()\n private readonly idleWaiters = new Set<() => void>()\n private completedTasks = 0\n private revision = 'memory'\n private costLimitPersisted = false\n readonly costCeilingUsd?: number\n private readonly persistence?: CostLedgerPersistence\n\n constructor(input?: number | CostLedgerOptions) {\n const options = typeof input === 'number' ? { costCeilingUsd: input } : (input ?? {})\n this.persistence = options.persistence\n if (options.costCeilingUsd !== undefined) {\n assertNonNegative(options.costCeilingUsd, 'costCeilingUsd')\n }\n\n let persistedCostCeilingUsd: number | undefined\n if (this.persistence) {\n let stored: ReturnType<CostLedgerPersistence['read']>\n try {\n stored = this.persistence.read()\n } catch (cause) {\n throw new CostLedgerPersistenceError(cause)\n }\n assertString(stored.revision, 'persistence revision')\n this.revision = stored.revision\n const restored = parseEvents(stored.events)\n for (const record of restored.records) this.records.set(record.callId, record)\n this.completedTasks = restored.completedTasks\n persistedCostCeilingUsd = restored.costCeilingUsd\n }\n\n if (\n options.costCeilingUsd !== undefined &&\n persistedCostCeilingUsd !== undefined &&\n options.costCeilingUsd !== persistedCostCeilingUsd\n ) {\n throw new ValidationError(\n `CostLedger: requested cost ceiling ${options.costCeilingUsd} does not match persisted ceiling ${persistedCostCeilingUsd}`,\n )\n }\n this.costCeilingUsd = persistedCostCeilingUsd ?? options.costCeilingUsd\n this.costLimitPersisted =\n !this.persistence ||\n this.costCeilingUsd === undefined ||\n persistedCostCeilingUsd !== undefined\n\n if (\n this.costCeilingUsd !== undefined &&\n [...this.records.values()].some(\n (record) => record.status === 'pending' && record.maximumCostUsd === undefined,\n )\n ) {\n throw new ValidationError('CostLedger: capped event log contains an unbounded pending call')\n }\n\n if (options.receipts?.length) {\n const imported = options.receipts.map((receipt, index) =>\n parseImportedReceipt(receipt, `imported receipt ${index + 1}`),\n )\n this.ensureCostLimitPersisted()\n for (const receipt of imported) this.appendRecord(receipt)\n }\n }\n\n async runPaidCall<T>(input: RunPaidCallInput<T>): Promise<PaidCallResult<T>> {\n let callId: string | undefined\n let pending: PendingCostCall | undefined\n try {\n callId = resolveCallId(input.callId)\n validateAttribution(input)\n if (input.signal?.aborted) {\n return { succeeded: false, callId, error: abortError(input.signal) }\n }\n if (this.records.has(callId)) {\n return {\n succeeded: false,\n callId,\n error: new CostCallConflictError(`CostLedger: callId '${callId}' already exists`),\n }\n }\n this.ensureCostLimitPersisted(callId)\n\n const summary = this.summary()\n if (summary.unresolvedCalls > 0) {\n return {\n succeeded: false,\n callId,\n error: new CostAccountingIncompleteError(\n `CostLedger: ${summary.unresolvedCalls} unresolved call(s) must be reconciled before new paid work`,\n ),\n }\n }\n\n const maximumCostUsd = this.resolveMaximum(input.maximumCharge)\n if (this.costCeilingUsd !== undefined && this.hasIncompleteSettledCall()) {\n return {\n succeeded: false,\n callId,\n error: new CostAccountingIncompleteError(\n `CostLedger: accounting is incomplete; refusing paid call '${input.actor}' during '${input.phase}'`,\n ),\n }\n }\n if (this.costCeilingUsd !== undefined) {\n const committedAndReserved = summary.totalCostUsd + summary.reservedCostUsd\n if (committedAndReserved + maximumCostUsd! > this.costCeilingUsd) {\n return {\n succeeded: false,\n callId,\n error: new CostCeilingReachedError(\n this.costCeilingUsd,\n committedAndReserved,\n maximumCostUsd!,\n input.phase,\n input.actor,\n ),\n }\n }\n }\n\n pending = {\n status: 'pending',\n callId,\n channel: input.channel,\n phase: input.phase,\n actor: input.actor,\n model: pendingModel(input),\n ...(maximumCostUsd === undefined ? {} : { maximumCostUsd }),\n ...(input.tags ? { tags: { ...input.tags } } : {}),\n timestamp: Date.now(),\n }\n this.appendRecord(pending)\n this.activeCallIds.add(callId)\n } catch (error) {\n return { succeeded: false, ...(callId ? { callId } : {}), error: toError(error) }\n }\n\n try {\n return await this.execute(input, pending)\n } catch (error) {\n return { succeeded: false, callId: pending.callId, error: toError(error) }\n } finally {\n if (!this.lateCallIds.has(pending.callId)) this.releaseActiveCall(pending.callId)\n }\n }\n\n /** Wait until every call started by this ledger has produced a durable outcome. */\n async waitForIdle(options: CostLedgerWaitOptions = {}): Promise<boolean> {\n const hasActiveMatch = (): boolean =>\n [...this.activeCallIds].some((callId) => {\n const record = this.records.get(callId)\n return record !== undefined && matches(record, options.filter)\n })\n if (!hasActiveMatch()) return true\n const timeoutMs = options.timeoutMs ?? 5_000\n assertTimeout(timeoutMs, 'waitForIdle.timeoutMs')\n return new Promise<boolean>((resolve) => {\n let timer: ReturnType<typeof setTimeout> | undefined\n const finish = (settled: boolean): void => {\n this.idleWaiters.delete(onIdle)\n if (timer !== undefined) clearTimeout(timer)\n resolve(settled)\n }\n const onIdle = (): void => {\n if (!hasActiveMatch()) finish(true)\n }\n this.idleWaiters.add(onIdle)\n timer = setTimeout(() => finish(false), timeoutMs)\n onIdle()\n })\n }\n\n /** Settle a call left pending by a crashed process after reconciling with the provider. */\n reconcile(\n callId: string,\n observed: CostReceiptInput,\n options: { error?: string } = {},\n ): CostReceipt {\n const pending = [...this.records.values()].find(\n (record): record is PendingCostCall =>\n record.callId === callId && record.status === 'pending',\n )\n if (!pending) throw new CostCallConflictError(`CostLedger: no pending call '${callId}'`)\n if (this.activeCallIds.has(callId)) {\n throw new CostCallConflictError(`CostLedger: call '${callId}' is still active`)\n }\n this.ensureCostLimitPersisted(callId)\n return this.commitReceipt(pending, observed, options.error)\n }\n\n list(filter?: CostLedgerFilter): CostReceipt[] {\n return [...this.records.values()]\n .filter((record): record is CostReceipt => record.status === 'settled')\n .filter((receipt) => matches(receipt, filter))\n .map(cloneReceipt)\n }\n\n /** Read pending calls without exposing mutable ledger state. */\n listPending(filter?: CostLedgerFilter): PendingCostCallView[] {\n return [...this.records.values()]\n .filter((record): record is PendingCostCall => record.status === 'pending')\n .filter((record) => matches(record, filter))\n .map((record) => ({\n ...clonePendingCall(record),\n state: this.lateCallIds.has(record.callId)\n ? 'late'\n : this.activeCallIds.has(record.callId)\n ? 'active'\n : 'interrupted',\n }))\n }\n\n summary(filter?: CostLedgerFilter): CostLedgerSummary {\n const records = [...this.records.values()].filter((record) => matches(record, filter))\n const pending = records.filter(\n (record): record is PendingCostCall => record.status === 'pending',\n )\n const receipts = records.filter((record): record is CostReceipt => record.status === 'settled')\n const byChannel = new Map<string, ChannelRollup>()\n const unpriced = new Set<string>()\n const incompleteReasons: string[] = pending.map(\n (record) => `call '${record.callId}' for '${record.actor}' is pending`,\n )\n let inputTokens = 0\n let outputTokens = 0\n let reasoningTokens = 0\n let cachedTokens = 0\n let cacheWriteTokens = 0\n let totalCostUsd = 0\n\n for (const receipt of receipts) {\n inputTokens += receipt.inputTokens\n outputTokens += receipt.outputTokens\n reasoningTokens += receipt.reasoningTokens ?? 0\n cachedTokens += receipt.cachedTokens ?? 0\n cacheWriteTokens += receipt.cacheWriteTokens ?? 0\n totalCostUsd += receipt.costUsd\n if (receipt.costUnknown) {\n unpriced.add(receipt.model)\n incompleteReasons.push(\n receipt.error ?? `cost unknown for '${receipt.actor}' using '${receipt.model}'`,\n )\n }\n if (receipt.usageUnknown) {\n incompleteReasons.push(\n `token usage unknown for '${receipt.actor}' using '${receipt.model}'`,\n )\n }\n if (receipt.maximumCostUsd !== undefined && receipt.costUsd > receipt.maximumCostUsd) {\n incompleteReasons.push(\n `'${receipt.actor}' charged ${receipt.costUsd}, exceeding its enforced maximum ${receipt.maximumCostUsd}`,\n )\n }\n const rollup = byChannel.get(receipt.channel) ?? emptyRollup(receipt.channel)\n rollup.calls += 1\n rollup.inputTokens += receipt.inputTokens\n rollup.outputTokens += receipt.outputTokens\n rollup.reasoningTokens = (rollup.reasoningTokens ?? 0) + (receipt.reasoningTokens ?? 0)\n rollup.cachedTokens += receipt.cachedTokens ?? 0\n rollup.cacheWriteTokens = (rollup.cacheWriteTokens ?? 0) + (receipt.cacheWriteTokens ?? 0)\n rollup.costUsd += receipt.costUsd\n if (receipt.costUnknown) rollup.unpricedCalls += 1\n if (receipt.usageUnknown) rollup.unknownUsageCalls += 1\n byChannel.set(receipt.channel, rollup)\n }\n\n return {\n totalCalls: receipts.length,\n pendingCalls: pending.length,\n unresolvedCalls: pending.filter(\n (record) => !this.activeCallIds.has(record.callId) || this.lateCallIds.has(record.callId),\n ).length,\n reservedCostUsd: pending.reduce((sum, record) => sum + (record.maximumCostUsd ?? 0), 0),\n inputTokens,\n outputTokens,\n reasoningTokens,\n cachedTokens,\n cacheWriteTokens,\n totalCostUsd,\n byChannel: [...byChannel.values()].sort((a, b) => a.channel.localeCompare(b.channel)),\n unpricedModels: [...unpriced].sort(),\n fullyPriced: unpriced.size === 0,\n usageComplete: pending.length === 0 && receipts.every((receipt) => !receipt.usageUnknown),\n accountingComplete: incompleteReasons.length === 0,\n incompleteReasons: [...new Set(incompleteReasons)],\n }\n }\n\n markCompleted(count = 1): void {\n if (!Number.isInteger(count) || count < 0) {\n throw new ValidationError(\n `CostLedger.markCompleted: count must be a non-negative integer, got ${count}`,\n )\n }\n if (count === 0) return\n this.ensureCostLimitPersisted()\n this.appendEvent({ version: 1, completedTasks: count })\n this.completedTasks += count\n }\n\n costPerCompletedTask(): number | null {\n return this.completedTasks === 0 ? null : this.summary().totalCostUsd / this.completedTasks\n }\n\n private async execute<T>(\n input: RunPaidCallInput<T>,\n pending: PendingCostCall,\n ): Promise<PaidCallResult<T>> {\n const signal = input.signal ?? new AbortController().signal\n if (signal.aborted) {\n return this.commitOutcome(pending, abortError(signal), {\n model: pending.model,\n inputTokens: 0,\n outputTokens: 0,\n actualCostUsd: 0,\n })\n }\n\n const operation = Promise.resolve().then(() => input.execute(signal, pending.callId))\n const settled = await settle(operation, signal)\n if (settled.kind === 'aborted') {\n this.lateCallIds.add(pending.callId)\n this.captureLateOutcome(input, pending, operation)\n return paidFailure(pending.callId, abortError(signal))\n }\n if (settled.kind === 'error') {\n let observed: CostReceiptInput | undefined\n try {\n observed = input.receiptFromError?.(settled.error)\n } catch (receiptError) {\n return this.captureFailure(pending, settled.error, receiptError)\n }\n return this.commitOutcome(pending, settled.error, observed ?? unknownReceipt(pending.model))\n }\n\n try {\n const receipt = this.commitReceipt(pending, input.receipt(settled.value))\n if (receipt.maximumCostUsd !== undefined && receipt.costUsd > receipt.maximumCostUsd) {\n return paidFailure(\n pending.callId,\n new CostReservationExceededError(pending.actor, receipt.costUsd, receipt.maximumCostUsd),\n receipt,\n )\n }\n return { succeeded: true, callId: pending.callId, value: settled.value, receipt }\n } catch (receiptError) {\n if (\n receiptError instanceof CostLedgerPersistenceError ||\n receiptError instanceof CostCallConflictError\n ) {\n return paidFailure(pending.callId, receiptError)\n }\n return this.captureFailure(pending, receiptError, receiptError)\n }\n }\n\n private captureLateOutcome<T>(\n input: RunPaidCallInput<T>,\n pending: PendingCostCall,\n operation: Promise<T>,\n ): void {\n void operation\n .then(\n (value) => {\n if (this.records.get(pending.callId)?.status !== 'pending') return\n try {\n this.commitReceipt(pending, input.receipt(value))\n } catch {\n // A failed durable settlement leaves the reservation pending and blocks new paid work.\n }\n },\n (cause) => {\n if (this.records.get(pending.callId)?.status !== 'pending') return\n const error = toError(cause)\n try {\n const observed = input.receiptFromError?.(error)\n this.commitReceipt(pending, observed ?? unknownReceipt(pending.model), error.message)\n } catch (receiptError) {\n if (\n receiptError instanceof CostLedgerPersistenceError ||\n receiptError instanceof CostCallConflictError\n ) {\n return\n }\n this.captureFailure(pending, error, receiptError)\n }\n },\n )\n .finally(() => {\n this.lateCallIds.delete(pending.callId)\n this.releaseActiveCall(pending.callId)\n })\n }\n\n private releaseActiveCall(callId: string): void {\n this.activeCallIds.delete(callId)\n for (const resolve of [...this.idleWaiters]) resolve()\n }\n\n private commitOutcome<T>(\n pending: PendingCostCall,\n error: Error,\n observed: CostReceiptInput,\n ): PaidCallResult<T> {\n try {\n const receipt = this.commitReceipt(pending, observed, error.message)\n return paidFailure(pending.callId, error, receipt)\n } catch (receiptError) {\n if (\n receiptError instanceof CostLedgerPersistenceError ||\n receiptError instanceof CostCallConflictError\n ) {\n return paidFailure(pending.callId, receiptError)\n }\n return this.captureFailure(pending, error, receiptError)\n }\n }\n\n private captureFailure<T>(\n pending: PendingCostCall,\n cause: unknown,\n receiptError: unknown,\n ): PaidCallResult<T> {\n try {\n const receipt = this.commitReceipt(\n pending,\n unknownReceipt(pending.model),\n toError(receiptError).message,\n )\n return paidFailure(\n pending.callId,\n new CostReceiptCaptureError(pending.callId, cause, receiptError, receipt),\n receipt,\n )\n } catch (error) {\n const typed = error instanceof Error ? error : toError(error)\n return paidFailure(pending.callId, typed)\n }\n }\n\n private commitReceipt(\n pending: PendingCostCall,\n observed: CostReceiptInput,\n error?: string,\n ): CostReceipt {\n const receipt = buildReceipt(pending, observed, error)\n if (this.records.get(pending.callId)?.status !== 'pending') {\n throw new CostCallConflictError(`CostLedger: call '${pending.callId}' is not pending`)\n }\n this.appendRecord(receipt)\n return cloneReceipt(receipt)\n }\n\n private resolveMaximum(maximum: MaximumCharge | undefined): number | undefined {\n if (!maximum) {\n if (this.costCeilingUsd !== undefined) {\n throw new CostAccountingIncompleteError(\n 'CostLedger: capped paid calls require a hard maximumCharge before execution',\n )\n }\n return undefined\n }\n if ('externallyEnforcedMaximumUsd' in maximum) {\n assertNonNegative(\n maximum.externallyEnforcedMaximumUsd,\n 'maximumCharge.externallyEnforcedMaximumUsd',\n )\n return maximum.externallyEnforcedMaximumUsd\n }\n if ('customTokenPricing' in maximum) {\n return costForTokenPricing(maximum.customTokenPricing, maximum)\n }\n const priced = costForUsage(maximum.model, maximum)\n if (priced.costUnknown) {\n if (this.costCeilingUsd !== undefined) {\n throw new CostAccountingIncompleteError(\n `CostLedger: cannot reserve unpriced model '${maximum.model}' in a capped run`,\n )\n }\n return undefined\n }\n return priced.costUsd\n }\n\n private hasIncompleteSettledCall(): boolean {\n return [...this.records.values()].some(\n (record) =>\n record.status === 'settled' &&\n (record.costUnknown ||\n record.usageUnknown ||\n (record.maximumCostUsd !== undefined && record.costUsd > record.maximumCostUsd)),\n )\n }\n\n private appendRecord(record: CostLedgerRecord): void {\n const callId = record.callId\n const receipt = record.status === 'settled' ? record : undefined\n validateTransition(this.records, record)\n const event: CostCallEvent =\n record.status === 'settled' &&\n (record.reasoningTokens !== undefined || record.cacheWriteTokens !== undefined)\n ? { version: 2, record: cloneReceipt(record) }\n : { version: 1, record: cloneRecord(record) }\n this.appendEvent(event, callId, receipt)\n this.records.set(callId, cloneRecord(record))\n }\n\n private ensureCostLimitPersisted(callId?: string): void {\n if (this.costLimitPersisted || this.costCeilingUsd === undefined) return\n this.appendEvent({ version: 1, costCeilingUsd: this.costCeilingUsd }, callId)\n this.costLimitPersisted = true\n }\n\n private appendEvent(event: CostLedgerEvent, callId?: string, receipt?: CostReceipt): void {\n try {\n if (this.persistence) {\n const nextRevision = this.persistence.append(this.revision, `${JSON.stringify(event)}\\n`)\n if (nextRevision === undefined) {\n throw new CostCallConflictError(\n `CostLedger: persisted revision changed while writing call '${callId}'`,\n { callId, receipt },\n )\n }\n assertString(nextRevision, 'persistence revision')\n this.revision = nextRevision\n }\n } catch (cause) {\n if (cause instanceof CostCallConflictError) throw cause\n throw new CostLedgerPersistenceError(cause, callId, receipt)\n }\n }\n}\n\n/** Public callback surface for a shared cost ledger.\n *\n * Declaration bundles may expose this type through multiple package subpaths.\n * Keeping callback contracts structural lets those subpaths compose while the\n * concrete {@link CostLedger} retains its private durable state.\n */\nexport type CostLedgerHandle = Pick<\n CostLedger,\n Exclude<keyof CostLedger, 'listPending' | 'waitForIdle'>\n> &\n Partial<Pick<CostLedger, 'listPending' | 'waitForIdle'>>\n\n/** Return the canonical pricing-table key, or null when the model is unpriced. */\nexport function modelPriceKey(model: string): string | null {\n return isModelPriced(model) ? model : null\n}\n\nexport interface CostResult {\n costUsd: number\n costUnknown: boolean\n}\n\nexport function costForUsage(model: string, usage: CostUsage): CostResult {\n assertUsage(usage)\n if (!resolveModelPricing(model)) return { costUsd: 0, costUnknown: true }\n return {\n costUsd: estimateCost(\n usage.inputTokens + (usage.cachedTokens ?? 0) + (usage.cacheWriteTokens ?? 0),\n usage.outputTokens,\n model,\n ),\n costUnknown: false,\n }\n}\n\n/** Price token counts with caller-supplied per-million rates. */\nexport function costForTokenPricing(\n pricing: CustomTokenPricing,\n usage: Pick<CostUsage, 'inputTokens' | 'outputTokens' | 'cachedTokens' | 'cacheWriteTokens'>,\n): number {\n assertTokenCount(usage.inputTokens, 'usage.inputTokens')\n assertTokenCount(usage.outputTokens, 'usage.outputTokens')\n if (usage.cachedTokens !== undefined) {\n assertTokenCount(usage.cachedTokens, 'usage.cachedTokens')\n }\n if (usage.cacheWriteTokens !== undefined) {\n assertTokenCount(usage.cacheWriteTokens, 'usage.cacheWriteTokens')\n }\n assertNonNegative(pricing.inputUsdPerMillion, 'pricing.inputUsdPerMillion')\n if (pricing.cachedInputUsdPerMillion !== undefined) {\n assertNonNegative(pricing.cachedInputUsdPerMillion, 'pricing.cachedInputUsdPerMillion')\n }\n if (pricing.cacheWriteUsdPerMillion !== undefined) {\n assertNonNegative(pricing.cacheWriteUsdPerMillion, 'pricing.cacheWriteUsdPerMillion')\n }\n assertNonNegative(pricing.outputUsdPerMillion, 'pricing.outputUsdPerMillion')\n const cachedInputRate = pricing.cachedInputUsdPerMillion ?? pricing.inputUsdPerMillion\n const cacheWriteRate = pricing.cacheWriteUsdPerMillion ?? pricing.inputUsdPerMillion\n return (\n (usage.inputTokens / 1_000_000) * pricing.inputUsdPerMillion +\n ((usage.cachedTokens ?? 0) / 1_000_000) * cachedInputRate +\n ((usage.cacheWriteTokens ?? 0) / 1_000_000) * cacheWriteRate +\n (usage.outputTokens / 1_000_000) * pricing.outputUsdPerMillion\n )\n}\n\ntype Settled<T> =\n | { kind: 'value'; value: T }\n | { kind: 'error'; error: Error }\n | { kind: 'aborted' }\n\nasync function settle<T>(promise: Promise<T>, signal: AbortSignal): Promise<Settled<T>> {\n return await new Promise((resolve) => {\n let done = false\n const finish = (value: Settled<T>): void => {\n if (done) return\n done = true\n signal.removeEventListener('abort', onAbort)\n resolve(value)\n }\n const onAbort = () => finish({ kind: 'aborted' })\n signal.addEventListener('abort', onAbort, { once: true })\n if (signal.aborted) onAbort()\n promise.then(\n (value) => finish({ kind: 'value', value }),\n (error) => finish({ kind: 'error', error: toError(error) }),\n )\n })\n}\n\nfunction buildReceipt(\n pending: PendingCostCall,\n observed: CostReceiptInput,\n error?: string,\n): CostReceipt {\n assertUsage(observed)\n assertString(observed.model, 'receipt.model')\n if (observed.actualCostUsd !== undefined && observed.customTokenPricing !== undefined) {\n throw new ValidationError(\n 'CostLedger: a receipt cannot have both actualCostUsd and customTokenPricing',\n )\n }\n const customPricingSnapshot =\n observed.customTokenPricing === undefined\n ? undefined\n : pricingSnapshot(observed.customTokenPricing)\n const estimated = customPricingSnapshot\n ? {\n costUsd: costFromPricing(observed, customPricingSnapshot),\n costUnknown: false,\n }\n : costForUsage(observed.model, observed)\n const hasActual = observed.actualCostUsd !== undefined\n if (hasActual && observed.costUnknown === true) {\n throw new ValidationError(\n 'CostLedger: a receipt cannot have both actualCostUsd and costUnknown=true',\n )\n }\n if (hasActual) assertNonNegative(observed.actualCostUsd!, 'actualCostUsd')\n const usageUnknown = observed.usageUnknown === true\n const costUnknown =\n observed.costUnknown === true || (!hasActual && (usageUnknown || estimated.costUnknown))\n const resolvedModelPricing =\n !customPricingSnapshot && !hasActual && !costUnknown\n ? resolveModelPricing(observed.model)\n : null\n const receiptPricing =\n !hasActual && !costUnknown\n ? (customPricingSnapshot ??\n (resolvedModelPricing\n ? {\n inputUsdPerThousand: resolvedModelPricing.input,\n outputUsdPerThousand: resolvedModelPricing.output,\n }\n : undefined))\n : undefined\n return parseReceipt(\n {\n status: 'settled',\n callId: pending.callId,\n channel: pending.channel,\n phase: pending.phase,\n actor: pending.actor,\n model: observed.model,\n inputTokens: observed.inputTokens,\n outputTokens: observed.outputTokens,\n ...(observed.reasoningTokens === undefined\n ? {}\n : { reasoningTokens: observed.reasoningTokens }),\n ...(observed.cachedTokens === undefined ? {} : { cachedTokens: observed.cachedTokens }),\n ...(observed.cacheWriteTokens === undefined\n ? {}\n : { cacheWriteTokens: observed.cacheWriteTokens }),\n costUsd: costUnknown ? 0 : hasActual ? observed.actualCostUsd! : estimated.costUsd,\n costUnknown,\n usageUnknown,\n ...(receiptPricing ? { pricing: receiptPricing } : {}),\n ...(hasActual ? { actualCostUsd: observed.actualCostUsd } : {}),\n ...(pending.maximumCostUsd === undefined ? {} : { maximumCostUsd: pending.maximumCostUsd }),\n ...(error ? { error } : {}),\n ...(pending.tags ? { tags: { ...pending.tags } } : {}),\n timestamp: pending.timestamp,\n },\n 'provider receipt',\n )\n}\n\nconst NonEmptyString = z.string().refine((value) => value.trim().length > 0, 'must be non-empty')\nconst TokenCount = z.number().int().nonnegative().finite()\nconst NonNegative = z.number().nonnegative().finite()\nconst Tags = z.record(NonEmptyString, z.string())\nconst CostPricingSchema = z.strictObject({\n inputUsdPerThousand: NonNegative,\n cachedInputUsdPerThousand: NonNegative.optional(),\n cacheWriteUsdPerThousand: NonNegative.optional(),\n outputUsdPerThousand: NonNegative,\n})\nconst CostCallBaseShape = {\n callId: NonEmptyString,\n channel: NonEmptyString,\n phase: NonEmptyString,\n actor: NonEmptyString,\n model: NonEmptyString,\n maximumCostUsd: NonNegative.optional(),\n tags: Tags.optional(),\n timestamp: NonNegative,\n}\nconst PendingCostCallSchema = z.strictObject({\n status: z.literal('pending'),\n ...CostCallBaseShape,\n})\nconst CostReceiptBaseShape = {\n status: z.literal('settled'),\n ...CostCallBaseShape,\n inputTokens: TokenCount,\n outputTokens: TokenCount,\n cachedTokens: TokenCount.optional(),\n costUsd: NonNegative,\n costUnknown: z.boolean(),\n usageUnknown: z.boolean().default(false),\n pricing: CostPricingSchema.optional(),\n actualCostUsd: NonNegative.optional(),\n error: z.string().optional(),\n}\nconst LegacyCostReceiptSchema = z\n .strictObject(CostReceiptBaseShape)\n .superRefine((receipt, ctx) => validateCostReceipt(receipt, ctx))\nconst CostReceiptSchema = z\n .strictObject({\n ...CostReceiptBaseShape,\n reasoningTokens: TokenCount.optional(),\n cacheWriteTokens: TokenCount.optional(),\n })\n .superRefine((receipt, ctx) => validateCostReceipt(receipt, ctx))\n\nfunction validateCostReceipt(\n receipt: CostUsage & {\n costUsd: number\n costUnknown: boolean\n usageUnknown: boolean\n pricing?: NonNullable<CostReceipt['pricing']>\n actualCostUsd?: number\n },\n ctx: z.RefinementCtx,\n): void {\n if (receipt.reasoningTokens !== undefined && receipt.reasoningTokens > receipt.outputTokens) {\n ctx.addIssue({ code: 'custom', message: 'reasoningTokens must not exceed outputTokens' })\n }\n\n if (receipt.actualCostUsd !== undefined) {\n if (receipt.costUnknown || receipt.costUsd !== receipt.actualCostUsd) {\n ctx.addIssue({ code: 'custom', message: 'actual cost must be known and equal costUsd' })\n }\n if (receipt.pricing !== undefined) {\n ctx.addIssue({ code: 'custom', message: 'actual cost must not include estimated pricing' })\n }\n return\n }\n\n if (receipt.costUnknown) {\n if (receipt.costUsd !== 0) {\n ctx.addIssue({ code: 'custom', message: 'unknown cost must have costUsd 0' })\n }\n if (receipt.pricing !== undefined) {\n ctx.addIssue({ code: 'custom', message: 'unknown cost must not include estimated pricing' })\n }\n return\n }\n\n if (receipt.usageUnknown) {\n ctx.addIssue({ code: 'custom', message: 'known estimated cost requires known usage' })\n }\n if (!receipt.pricing) {\n ctx.addIssue({ code: 'custom', message: 'known estimated cost requires a pricing snapshot' })\n return\n }\n\n const expected = costFromPricing(receipt, receipt.pricing)\n if (receipt.costUsd !== expected) {\n ctx.addIssue({\n code: 'custom',\n message: `estimated cost ${receipt.costUsd} does not match pricing snapshot ${expected}`,\n })\n }\n}\n\nconst CostLedgerEventSchema = z.union([\n z.strictObject({\n version: z.literal(1),\n record: z.union([PendingCostCallSchema, LegacyCostReceiptSchema]),\n }),\n z.strictObject({\n version: z.literal(2),\n record: CostReceiptSchema,\n }),\n z.strictObject({\n version: z.literal(1),\n completedTasks: TokenCount,\n }),\n z.strictObject({\n version: z.literal(1),\n costCeilingUsd: NonNegative,\n }),\n])\n\nfunction parseEvents(serialized: string): {\n records: CostLedgerRecord[]\n completedTasks: number\n costCeilingUsd?: number\n} {\n const records = new Map<string, CostLedgerRecord>()\n let completedTasks = 0\n let costCeilingUsd: number | undefined\n let lineNumber = 0\n try {\n for (const line of serialized.split('\\n')) {\n if (!line.trim()) continue\n lineNumber += 1\n const event = CostLedgerEventSchema.parse(JSON.parse(line)) as CostLedgerEvent\n if ('record' in event) {\n const record = event.record\n validateTransition(records, record)\n records.set(record.callId, cloneRecord(record))\n } else if ('completedTasks' in event) {\n completedTasks += event.completedTasks\n if (!Number.isSafeInteger(completedTasks)) {\n throw new ValidationError('CostLedger: completed task count exceeds safe integer range')\n }\n } else {\n if (costCeilingUsd !== undefined) {\n throw new ValidationError('CostLedger: duplicate persisted cost ceiling')\n }\n costCeilingUsd = event.costCeilingUsd\n }\n }\n return {\n records: [...records.values()],\n completedTasks,\n ...(costCeilingUsd === undefined ? {} : { costCeilingUsd }),\n }\n } catch (cause) {\n throw new ValidationError(\n `CostLedger: invalid persisted event ${lineNumber || 1}: ${validationMessage(cause)}`,\n { cause },\n )\n }\n}\n\nfunction validateTransition(\n records: ReadonlyMap<string, CostLedgerRecord>,\n record: CostLedgerRecord,\n): void {\n const current = records.get(record.callId)\n if (!current) return\n if (record.status === 'pending' || current.status === 'settled') {\n throw new CostCallConflictError(`CostLedger: duplicate callId '${record.callId}'`)\n }\n if (!sameAttribution(current, record)) {\n throw new ValidationError(`CostLedger: receipt attribution changed for call '${record.callId}'`)\n }\n}\n\nfunction sameAttribution(before: CostCallBase, after: CostCallBase): boolean {\n return (\n before.channel === after.channel &&\n before.phase === after.phase &&\n before.actor === after.actor &&\n before.maximumCostUsd === after.maximumCostUsd &&\n before.timestamp === after.timestamp &&\n JSON.stringify(before.tags ?? {}) === JSON.stringify(after.tags ?? {})\n )\n}\n\nfunction parseReceipt(value: unknown, path: string): CostReceipt {\n try {\n return CostReceiptSchema.parse(value) as CostReceipt\n } catch (cause) {\n throw new ValidationError(`CostLedger: invalid ${path}: ${validationMessage(cause)}`, { cause })\n }\n}\n\nfunction parseImportedReceipt(value: unknown, path: string): CostReceipt {\n if (typeof value !== 'object' || value === null) return parseReceipt(value, path)\n const candidate = { ...value } as Record<string, unknown>\n if (\n candidate.status === 'settled' &&\n candidate.actualCostUsd === undefined &&\n candidate.costUnknown === false &&\n candidate.pricing === undefined &&\n typeof candidate.model === 'string'\n ) {\n const pricing = resolveModelPricing(candidate.model)\n if (pricing) {\n candidate.pricing = {\n inputUsdPerThousand: pricing.input,\n outputUsdPerThousand: pricing.output,\n }\n }\n }\n return parseReceipt(candidate, path)\n}\n\nfunction validateAttribution(\n input: Pick<RunPaidCallInput<unknown>, 'channel' | 'phase' | 'actor' | 'model' | 'tags'>,\n): void {\n assertString(input.channel, 'channel')\n assertString(input.phase, 'phase')\n assertString(input.actor, 'actor')\n if (input.model !== undefined) assertString(input.model, 'model')\n if (input.tags !== undefined) {\n const parsed = Tags.safeParse(input.tags)\n if (!parsed.success)\n throw new ValidationError(`CostLedger: invalid tags: ${parsed.error.message}`)\n }\n}\n\nfunction matches(record: CostCallBase, filter: CostLedgerFilter | undefined): boolean {\n if (!filter) return true\n if (filter.channel !== undefined && record.channel !== filter.channel) return false\n if (filter.phase !== undefined && record.phase !== filter.phase) return false\n return Object.entries(filter.tags ?? {}).every(([key, value]) => record.tags?.[key] === value)\n}\n\nfunction cloneRecord(record: CostLedgerRecord): CostLedgerRecord {\n if (record.status === 'settled') return cloneReceipt(record)\n return clonePendingCall(record)\n}\n\nfunction clonePendingCall(record: PendingCostCall): PendingCostCall {\n const { tags, ...rest } = record\n return { ...rest, ...(tags ? { tags: { ...tags } } : {}) }\n}\n\nfunction cloneReceipt(receipt: CostReceipt): CostReceipt {\n const { pricing, tags, ...rest } = receipt\n return {\n ...rest,\n ...(tags ? { tags: { ...tags } } : {}),\n ...(pricing ? { pricing: { ...pricing } } : {}),\n }\n}\n\nfunction costFromPricing(usage: CostUsage, pricing: NonNullable<CostReceipt['pricing']>): number {\n const cachedInputRate = pricing.cachedInputUsdPerThousand ?? pricing.inputUsdPerThousand\n const cacheWriteRate = pricing.cacheWriteUsdPerThousand ?? pricing.inputUsdPerThousand\n return (\n (usage.inputTokens / 1000) * pricing.inputUsdPerThousand +\n ((usage.cachedTokens ?? 0) / 1000) * cachedInputRate +\n ((usage.cacheWriteTokens ?? 0) / 1000) * cacheWriteRate +\n (usage.outputTokens / 1000) * pricing.outputUsdPerThousand\n )\n}\n\nfunction pricingSnapshot(pricing: CustomTokenPricing): NonNullable<CostReceipt['pricing']> {\n costForTokenPricing(pricing, { inputTokens: 0, outputTokens: 0 })\n return {\n inputUsdPerThousand: pricing.inputUsdPerMillion / 1000,\n ...(pricing.cachedInputUsdPerMillion === undefined\n ? {}\n : { cachedInputUsdPerThousand: pricing.cachedInputUsdPerMillion / 1000 }),\n ...(pricing.cacheWriteUsdPerMillion === undefined\n ? {}\n : { cacheWriteUsdPerThousand: pricing.cacheWriteUsdPerMillion / 1000 }),\n outputUsdPerThousand: pricing.outputUsdPerMillion / 1000,\n }\n}\n\nfunction emptyRollup(channel: CostChannel): ChannelRollup {\n return {\n channel,\n calls: 0,\n inputTokens: 0,\n outputTokens: 0,\n reasoningTokens: 0,\n cachedTokens: 0,\n cacheWriteTokens: 0,\n costUsd: 0,\n unpricedCalls: 0,\n unknownUsageCalls: 0,\n }\n}\n\nfunction pendingModel(input: RunPaidCallInput<unknown>): string {\n if (input.model) return input.model\n if (input.maximumCharge && 'model' in input.maximumCharge) return input.maximumCharge.model\n return 'unknown'\n}\n\nfunction unknownReceipt(model: string): CostReceiptInput {\n return { model, inputTokens: 0, outputTokens: 0, costUnknown: true, usageUnknown: true }\n}\n\nfunction resolveCallId(input: string | undefined): string {\n if (input !== undefined) {\n assertString(input, 'callId')\n return input\n }\n if (typeof globalThis.crypto?.randomUUID !== 'function') {\n throw new ValidationError('CostLedger: crypto.randomUUID is required when callId is omitted')\n }\n return globalThis.crypto.randomUUID()\n}\n\nfunction abortError(signal: AbortSignal): Error {\n const reason = (signal as { reason?: unknown }).reason\n if (reason instanceof Error) return reason\n const error = new Error('CostLedger: paid call aborted')\n error.name = 'AbortError'\n return error\n}\n\nfunction toError(value: unknown): Error {\n return value instanceof Error ? value : new Error(String(value))\n}\n\nfunction paidFailure<T>(callId: string, error: Error, receipt?: CostReceipt): PaidCallResult<T> {\n return {\n succeeded: false,\n callId,\n error,\n ...(receipt ? { receipt: cloneReceipt(receipt) } : {}),\n }\n}\n\nfunction validationMessage(cause: unknown): string {\n return cause instanceof z.ZodError ? z.prettifyError(cause) : toError(cause).message\n}\n\nfunction assertUsage(usage: CostUsage): void {\n assertTokenCount(usage.inputTokens, 'inputTokens')\n assertTokenCount(usage.outputTokens, 'outputTokens')\n if (usage.reasoningTokens !== undefined) {\n assertTokenCount(usage.reasoningTokens, 'reasoningTokens')\n if (usage.reasoningTokens > usage.outputTokens) {\n throw new ValidationError('CostLedger: reasoningTokens must not exceed outputTokens')\n }\n }\n if (usage.cachedTokens !== undefined) assertTokenCount(usage.cachedTokens, 'cachedTokens')\n if (usage.cacheWriteTokens !== undefined) {\n assertTokenCount(usage.cacheWriteTokens, 'cacheWriteTokens')\n }\n}\n\nfunction assertTokenCount(value: unknown, name: string): asserts value is number {\n if (typeof value !== 'number' || !Number.isInteger(value) || value < 0) {\n throw new ValidationError(\n `CostLedger: ${name} must be a non-negative integer, got ${String(value)}`,\n )\n }\n}\n\nfunction assertNonNegative(value: unknown, name: string): asserts value is number {\n if (typeof value !== 'number' || !Number.isFinite(value) || value < 0) {\n throw new ValidationError(\n `CostLedger: ${name} must be a non-negative finite number, got ${String(value)}`,\n )\n }\n}\n\nfunction assertTimeout(value: unknown, name: string): asserts value is number {\n if (\n typeof value !== 'number' ||\n !Number.isSafeInteger(value) ||\n value < 0 ||\n value > 2_147_483_647\n ) {\n throw new ValidationError(\n `CostLedger: ${name} must be a non-negative safe integer no greater than 2147483647, got ${String(value)}`,\n )\n }\n}\n\nfunction assertString(value: unknown, name: string): asserts value is string {\n if (typeof value !== 'string' || value.trim().length === 0) {\n throw new ValidationError(`CostLedger: ${name} must be a non-empty string`)\n }\n}\n"],"mappings":";;;;;;;;;;AAAA,SAAS,SAAS;AAwKX,IAAM,0BAAN,cAAsC,gBAAgB;AAAA,EAC3D,YACE,YACA,yBACA,cACA,OACA,OACA;AACA;AAAA,MACE,yBAAyB,YAAY,SAAS,KAAK,aAAa,KAAK,0BAA0B,UAAU,SAAS,uBAAuB;AAAA,IAC3I;AAAA,EACF;AACF;AAEO,IAAM,gCAAN,cAA4C,gBAAgB;AAAC;AAE7D,IAAM,+BAAN,cAA2C,gBAAgB;AAAA,EAChE,YAAY,OAAe,WAAmB,YAAoB;AAChE;AAAA,MACE,gBAAgB,KAAK,aAAa,SAAS,oCAAoC,UAAU;AAAA,IAC3F;AAAA,EACF;AACF;AAEO,IAAM,wBAAN,cAAoC,gBAAgB;AAAA,EAChD;AAAA,EACA;AAAA,EAET,YACE,SACA,UAAuE,CAAC,GACxE;AACA,UAAM,SAAS,QAAQ,UAAU,SAAY,SAAY,EAAE,OAAO,QAAQ,MAAM,CAAC;AACjF,SAAK,SAAS,QAAQ;AACtB,SAAK,UAAU,QAAQ,UAAU,aAAa,QAAQ,OAAO,IAAI;AAAA,EACnE;AACF;AAEO,IAAM,6BAAN,cAAyC,gBAAgB;AAAA,EACrD;AAAA,EACA;AAAA,EAET,YAAY,OAAgB,QAAiB,SAAuB;AAClE;AAAA,MACE,gCAAgC,SAAS,UAAU,MAAM,MAAM,EAAE,KAAK,QAAQ,KAAK,EAAE,OAAO;AAAA,MAC5F,EAAE,MAAM;AAAA,IACV;AACA,SAAK,SAAS;AACd,SAAK,UAAU,UAAU,aAAa,OAAO,IAAI;AAAA,EACnD;AACF;AAEO,IAAM,0BAAN,cAAsC,gBAAgB;AAAA,EAClD;AAAA,EACA;AAAA,EACA;AAAA,EAET,YAAY,QAAgB,OAAgB,cAAuB,SAAuB;AACxF,UAAM,gEAAgE,MAAM,KAAK,EAAE,MAAM,CAAC;AAC1F,SAAK,SAAS;AACd,SAAK,UAAU,UAAU,aAAa,OAAO,IAAI;AACjD,SAAK,eAAe,QAAQ,YAAY;AAAA,EAC1C;AACF;AA0BO,IAAM,aAAN,MAAiB;AAAA,EACL,UAAU,oBAAI,IAA8B;AAAA,EAC5C,gBAAgB,oBAAI,IAAY;AAAA,EAChC,cAAc,oBAAI,IAAY;AAAA,EAC9B,cAAc,oBAAI,IAAgB;AAAA,EAC3C,iBAAiB;AAAA,EACjB,WAAW;AAAA,EACX,qBAAqB;AAAA,EACpB;AAAA,EACQ;AAAA,EAEjB,YAAY,OAAoC;AAC9C,UAAM,UAAU,OAAO,UAAU,WAAW,EAAE,gBAAgB,MAAM,IAAK,SAAS,CAAC;AACnF,SAAK,cAAc,QAAQ;AAC3B,QAAI,QAAQ,mBAAmB,QAAW;AACxC,wBAAkB,QAAQ,gBAAgB,gBAAgB;AAAA,IAC5D;AAEA,QAAI;AACJ,QAAI,KAAK,aAAa;AACpB,UAAI;AACJ,UAAI;AACF,iBAAS,KAAK,YAAY,KAAK;AAAA,MACjC,SAAS,OAAO;AACd,cAAM,IAAI,2BAA2B,KAAK;AAAA,MAC5C;AACA,mBAAa,OAAO,UAAU,sBAAsB;AACpD,WAAK,WAAW,OAAO;AACvB,YAAM,WAAW,YAAY,OAAO,MAAM;AAC1C,iBAAW,UAAU,SAAS,QAAS,MAAK,QAAQ,IAAI,OAAO,QAAQ,MAAM;AAC7E,WAAK,iBAAiB,SAAS;AAC/B,gCAA0B,SAAS;AAAA,IACrC;AAEA,QACE,QAAQ,mBAAmB,UAC3B,4BAA4B,UAC5B,QAAQ,mBAAmB,yBAC3B;AACA,YAAM,IAAI;AAAA,QACR,sCAAsC,QAAQ,cAAc,qCAAqC,uBAAuB;AAAA,MAC1H;AAAA,IACF;AACA,SAAK,iBAAiB,2BAA2B,QAAQ;AACzD,SAAK,qBACH,CAAC,KAAK,eACN,KAAK,mBAAmB,UACxB,4BAA4B;AAE9B,QACE,KAAK,mBAAmB,UACxB,CAAC,GAAG,KAAK,QAAQ,OAAO,CAAC,EAAE;AAAA,MACzB,CAAC,WAAW,OAAO,WAAW,aAAa,OAAO,mBAAmB;AAAA,IACvE,GACA;AACA,YAAM,IAAI,gBAAgB,iEAAiE;AAAA,IAC7F;AAEA,QAAI,QAAQ,UAAU,QAAQ;AAC5B,YAAM,WAAW,QAAQ,SAAS;AAAA,QAAI,CAAC,SAAS,UAC9C,qBAAqB,SAAS,oBAAoB,QAAQ,CAAC,EAAE;AAAA,MAC/D;AACA,WAAK,yBAAyB;AAC9B,iBAAW,WAAW,SAAU,MAAK,aAAa,OAAO;AAAA,IAC3D;AAAA,EACF;AAAA,EAEA,MAAM,YAAe,OAAwD;AAC3E,QAAI;AACJ,QAAI;AACJ,QAAI;AACF,eAAS,cAAc,MAAM,MAAM;AACnC,0BAAoB,KAAK;AACzB,UAAI,MAAM,QAAQ,SAAS;AACzB,eAAO,EAAE,WAAW,OAAO,QAAQ,OAAO,WAAW,MAAM,MAAM,EAAE;AAAA,MACrE;AACA,UAAI,KAAK,QAAQ,IAAI,MAAM,GAAG;AAC5B,eAAO;AAAA,UACL,WAAW;AAAA,UACX;AAAA,UACA,OAAO,IAAI,sBAAsB,uBAAuB,MAAM,kBAAkB;AAAA,QAClF;AAAA,MACF;AACA,WAAK,yBAAyB,MAAM;AAEpC,YAAM,UAAU,KAAK,QAAQ;AAC7B,UAAI,QAAQ,kBAAkB,GAAG;AAC/B,eAAO;AAAA,UACL,WAAW;AAAA,UACX;AAAA,UACA,OAAO,IAAI;AAAA,YACT,eAAe,QAAQ,eAAe;AAAA,UACxC;AAAA,QACF;AAAA,MACF;AAEA,YAAM,iBAAiB,KAAK,eAAe,MAAM,aAAa;AAC9D,UAAI,KAAK,mBAAmB,UAAa,KAAK,yBAAyB,GAAG;AACxE,eAAO;AAAA,UACL,WAAW;AAAA,UACX;AAAA,UACA,OAAO,IAAI;AAAA,YACT,6DAA6D,MAAM,KAAK,aAAa,MAAM,KAAK;AAAA,UAClG;AAAA,QACF;AAAA,MACF;AACA,UAAI,KAAK,mBAAmB,QAAW;AACrC,cAAM,uBAAuB,QAAQ,eAAe,QAAQ;AAC5D,YAAI,uBAAuB,iBAAkB,KAAK,gBAAgB;AAChE,iBAAO;AAAA,YACL,WAAW;AAAA,YACX;AAAA,YACA,OAAO,IAAI;AAAA,cACT,KAAK;AAAA,cACL;AAAA,cACA;AAAA,cACA,MAAM;AAAA,cACN,MAAM;AAAA,YACR;AAAA,UACF;AAAA,QACF;AAAA,MACF;AAEA,gBAAU;AAAA,QACR,QAAQ;AAAA,QACR;AAAA,QACA,SAAS,MAAM;AAAA,QACf,OAAO,MAAM;AAAA,QACb,OAAO,MAAM;AAAA,QACb,OAAO,aAAa,KAAK;AAAA,QACzB,GAAI,mBAAmB,SAAY,CAAC,IAAI,EAAE,eAAe;AAAA,QACzD,GAAI,MAAM,OAAO,EAAE,MAAM,EAAE,GAAG,MAAM,KAAK,EAAE,IAAI,CAAC;AAAA,QAChD,WAAW,KAAK,IAAI;AAAA,MACtB;AACA,WAAK,aAAa,OAAO;AACzB,WAAK,cAAc,IAAI,MAAM;AAAA,IAC/B,SAAS,OAAO;AACd,aAAO,EAAE,WAAW,OAAO,GAAI,SAAS,EAAE,OAAO,IAAI,CAAC,GAAI,OAAO,QAAQ,KAAK,EAAE;AAAA,IAClF;AAEA,QAAI;AACF,aAAO,MAAM,KAAK,QAAQ,OAAO,OAAO;AAAA,IAC1C,SAAS,OAAO;AACd,aAAO,EAAE,WAAW,OAAO,QAAQ,QAAQ,QAAQ,OAAO,QAAQ,KAAK,EAAE;AAAA,IAC3E,UAAE;AACA,UAAI,CAAC,KAAK,YAAY,IAAI,QAAQ,MAAM,EAAG,MAAK,kBAAkB,QAAQ,MAAM;AAAA,IAClF;AAAA,EACF;AAAA;AAAA,EAGA,MAAM,YAAY,UAAiC,CAAC,GAAqB;AACvE,UAAM,iBAAiB,MACrB,CAAC,GAAG,KAAK,aAAa,EAAE,KAAK,CAAC,WAAW;AACvC,YAAM,SAAS,KAAK,QAAQ,IAAI,MAAM;AACtC,aAAO,WAAW,UAAa,QAAQ,QAAQ,QAAQ,MAAM;AAAA,IAC/D,CAAC;AACH,QAAI,CAAC,eAAe,EAAG,QAAO;AAC9B,UAAM,YAAY,QAAQ,aAAa;AACvC,kBAAc,WAAW,uBAAuB;AAChD,WAAO,IAAI,QAAiB,CAAC,YAAY;AACvC,UAAI;AACJ,YAAM,SAAS,CAAC,YAA2B;AACzC,aAAK,YAAY,OAAO,MAAM;AAC9B,YAAI,UAAU,OAAW,cAAa,KAAK;AAC3C,gBAAQ,OAAO;AAAA,MACjB;AACA,YAAM,SAAS,MAAY;AACzB,YAAI,CAAC,eAAe,EAAG,QAAO,IAAI;AAAA,MACpC;AACA,WAAK,YAAY,IAAI,MAAM;AAC3B,cAAQ,WAAW,MAAM,OAAO,KAAK,GAAG,SAAS;AACjD,aAAO;AAAA,IACT,CAAC;AAAA,EACH;AAAA;AAAA,EAGA,UACE,QACA,UACA,UAA8B,CAAC,GAClB;AACb,UAAM,UAAU,CAAC,GAAG,KAAK,QAAQ,OAAO,CAAC,EAAE;AAAA,MACzC,CAAC,WACC,OAAO,WAAW,UAAU,OAAO,WAAW;AAAA,IAClD;AACA,QAAI,CAAC,QAAS,OAAM,IAAI,sBAAsB,gCAAgC,MAAM,GAAG;AACvF,QAAI,KAAK,cAAc,IAAI,MAAM,GAAG;AAClC,YAAM,IAAI,sBAAsB,qBAAqB,MAAM,mBAAmB;AAAA,IAChF;AACA,SAAK,yBAAyB,MAAM;AACpC,WAAO,KAAK,cAAc,SAAS,UAAU,QAAQ,KAAK;AAAA,EAC5D;AAAA,EAEA,KAAK,QAA0C;AAC7C,WAAO,CAAC,GAAG,KAAK,QAAQ,OAAO,CAAC,EAC7B,OAAO,CAAC,WAAkC,OAAO,WAAW,SAAS,EACrE,OAAO,CAAC,YAAY,QAAQ,SAAS,MAAM,CAAC,EAC5C,IAAI,YAAY;AAAA,EACrB;AAAA;AAAA,EAGA,YAAY,QAAkD;AAC5D,WAAO,CAAC,GAAG,KAAK,QAAQ,OAAO,CAAC,EAC7B,OAAO,CAAC,WAAsC,OAAO,WAAW,SAAS,EACzE,OAAO,CAAC,WAAW,QAAQ,QAAQ,MAAM,CAAC,EAC1C,IAAI,CAAC,YAAY;AAAA,MAChB,GAAG,iBAAiB,MAAM;AAAA,MAC1B,OAAO,KAAK,YAAY,IAAI,OAAO,MAAM,IACrC,SACA,KAAK,cAAc,IAAI,OAAO,MAAM,IAClC,WACA;AAAA,IACR,EAAE;AAAA,EACN;AAAA,EAEA,QAAQ,QAA8C;AACpD,UAAM,UAAU,CAAC,GAAG,KAAK,QAAQ,OAAO,CAAC,EAAE,OAAO,CAAC,WAAW,QAAQ,QAAQ,MAAM,CAAC;AACrF,UAAM,UAAU,QAAQ;AAAA,MACtB,CAAC,WAAsC,OAAO,WAAW;AAAA,IAC3D;AACA,UAAM,WAAW,QAAQ,OAAO,CAAC,WAAkC,OAAO,WAAW,SAAS;AAC9F,UAAM,YAAY,oBAAI,IAA2B;AACjD,UAAM,WAAW,oBAAI,IAAY;AACjC,UAAM,oBAA8B,QAAQ;AAAA,MAC1C,CAAC,WAAW,SAAS,OAAO,MAAM,UAAU,OAAO,KAAK;AAAA,IAC1D;AACA,QAAI,cAAc;AAClB,QAAI,eAAe;AACnB,QAAI,kBAAkB;AACtB,QAAI,eAAe;AACnB,QAAI,mBAAmB;AACvB,QAAI,eAAe;AAEnB,eAAW,WAAW,UAAU;AAC9B,qBAAe,QAAQ;AACvB,sBAAgB,QAAQ;AACxB,yBAAmB,QAAQ,mBAAmB;AAC9C,sBAAgB,QAAQ,gBAAgB;AACxC,0BAAoB,QAAQ,oBAAoB;AAChD,sBAAgB,QAAQ;AACxB,UAAI,QAAQ,aAAa;AACvB,iBAAS,IAAI,QAAQ,KAAK;AAC1B,0BAAkB;AAAA,UAChB,QAAQ,SAAS,qBAAqB,QAAQ,KAAK,YAAY,QAAQ,KAAK;AAAA,QAC9E;AAAA,MACF;AACA,UAAI,QAAQ,cAAc;AACxB,0BAAkB;AAAA,UAChB,4BAA4B,QAAQ,KAAK,YAAY,QAAQ,KAAK;AAAA,QACpE;AAAA,MACF;AACA,UAAI,QAAQ,mBAAmB,UAAa,QAAQ,UAAU,QAAQ,gBAAgB;AACpF,0BAAkB;AAAA,UAChB,IAAI,QAAQ,KAAK,aAAa,QAAQ,OAAO,oCAAoC,QAAQ,cAAc;AAAA,QACzG;AAAA,MACF;AACA,YAAM,SAAS,UAAU,IAAI,QAAQ,OAAO,KAAK,YAAY,QAAQ,OAAO;AAC5E,aAAO,SAAS;AAChB,aAAO,eAAe,QAAQ;AAC9B,aAAO,gBAAgB,QAAQ;AAC/B,aAAO,mBAAmB,OAAO,mBAAmB,MAAM,QAAQ,mBAAmB;AACrF,aAAO,gBAAgB,QAAQ,gBAAgB;AAC/C,aAAO,oBAAoB,OAAO,oBAAoB,MAAM,QAAQ,oBAAoB;AACxF,aAAO,WAAW,QAAQ;AAC1B,UAAI,QAAQ,YAAa,QAAO,iBAAiB;AACjD,UAAI,QAAQ,aAAc,QAAO,qBAAqB;AACtD,gBAAU,IAAI,QAAQ,SAAS,MAAM;AAAA,IACvC;AAEA,WAAO;AAAA,MACL,YAAY,SAAS;AAAA,MACrB,cAAc,QAAQ;AAAA,MACtB,iBAAiB,QAAQ;AAAA,QACvB,CAAC,WAAW,CAAC,KAAK,cAAc,IAAI,OAAO,MAAM,KAAK,KAAK,YAAY,IAAI,OAAO,MAAM;AAAA,MAC1F,EAAE;AAAA,MACF,iBAAiB,QAAQ,OAAO,CAAC,KAAK,WAAW,OAAO,OAAO,kBAAkB,IAAI,CAAC;AAAA,MACtF;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,WAAW,CAAC,GAAG,UAAU,OAAO,CAAC,EAAE,KAAK,CAAC,GAAG,MAAM,EAAE,QAAQ,cAAc,EAAE,OAAO,CAAC;AAAA,MACpF,gBAAgB,CAAC,GAAG,QAAQ,EAAE,KAAK;AAAA,MACnC,aAAa,SAAS,SAAS;AAAA,MAC/B,eAAe,QAAQ,WAAW,KAAK,SAAS,MAAM,CAAC,YAAY,CAAC,QAAQ,YAAY;AAAA,MACxF,oBAAoB,kBAAkB,WAAW;AAAA,MACjD,mBAAmB,CAAC,GAAG,IAAI,IAAI,iBAAiB,CAAC;AAAA,IACnD;AAAA,EACF;AAAA,EAEA,cAAc,QAAQ,GAAS;AAC7B,QAAI,CAAC,OAAO,UAAU,KAAK,KAAK,QAAQ,GAAG;AACzC,YAAM,IAAI;AAAA,QACR,uEAAuE,KAAK;AAAA,MAC9E;AAAA,IACF;AACA,QAAI,UAAU,EAAG;AACjB,SAAK,yBAAyB;AAC9B,SAAK,YAAY,EAAE,SAAS,GAAG,gBAAgB,MAAM,CAAC;AACtD,SAAK,kBAAkB;AAAA,EACzB;AAAA,EAEA,uBAAsC;AACpC,WAAO,KAAK,mBAAmB,IAAI,OAAO,KAAK,QAAQ,EAAE,eAAe,KAAK;AAAA,EAC/E;AAAA,EAEA,MAAc,QACZ,OACA,SAC4B;AAC5B,UAAM,SAAS,MAAM,UAAU,IAAI,gBAAgB,EAAE;AACrD,QAAI,OAAO,SAAS;AAClB,aAAO,KAAK,cAAc,SAAS,WAAW,MAAM,GAAG;AAAA,QACrD,OAAO,QAAQ;AAAA,QACf,aAAa;AAAA,QACb,cAAc;AAAA,QACd,eAAe;AAAA,MACjB,CAAC;AAAA,IACH;AAEA,UAAM,YAAY,QAAQ,QAAQ,EAAE,KAAK,MAAM,MAAM,QAAQ,QAAQ,QAAQ,MAAM,CAAC;AACpF,UAAM,UAAU,MAAM,OAAO,WAAW,MAAM;AAC9C,QAAI,QAAQ,SAAS,WAAW;AAC9B,WAAK,YAAY,IAAI,QAAQ,MAAM;AACnC,WAAK,mBAAmB,OAAO,SAAS,SAAS;AACjD,aAAO,YAAY,QAAQ,QAAQ,WAAW,MAAM,CAAC;AAAA,IACvD;AACA,QAAI,QAAQ,SAAS,SAAS;AAC5B,UAAI;AACJ,UAAI;AACF,mBAAW,MAAM,mBAAmB,QAAQ,KAAK;AAAA,MACnD,SAAS,cAAc;AACrB,eAAO,KAAK,eAAe,SAAS,QAAQ,OAAO,YAAY;AAAA,MACjE;AACA,aAAO,KAAK,cAAc,SAAS,QAAQ,OAAO,YAAY,eAAe,QAAQ,KAAK,CAAC;AAAA,IAC7F;AAEA,QAAI;AACF,YAAM,UAAU,KAAK,cAAc,SAAS,MAAM,QAAQ,QAAQ,KAAK,CAAC;AACxE,UAAI,QAAQ,mBAAmB,UAAa,QAAQ,UAAU,QAAQ,gBAAgB;AACpF,eAAO;AAAA,UACL,QAAQ;AAAA,UACR,IAAI,6BAA6B,QAAQ,OAAO,QAAQ,SAAS,QAAQ,cAAc;AAAA,UACvF;AAAA,QACF;AAAA,MACF;AACA,aAAO,EAAE,WAAW,MAAM,QAAQ,QAAQ,QAAQ,OAAO,QAAQ,OAAO,QAAQ;AAAA,IAClF,SAAS,cAAc;AACrB,UACE,wBAAwB,8BACxB,wBAAwB,uBACxB;AACA,eAAO,YAAY,QAAQ,QAAQ,YAAY;AAAA,MACjD;AACA,aAAO,KAAK,eAAe,SAAS,cAAc,YAAY;AAAA,IAChE;AAAA,EACF;AAAA,EAEQ,mBACN,OACA,SACA,WACM;AACN,SAAK,UACF;AAAA,MACC,CAAC,UAAU;AACT,YAAI,KAAK,QAAQ,IAAI,QAAQ,MAAM,GAAG,WAAW,UAAW;AAC5D,YAAI;AACF,eAAK,cAAc,SAAS,MAAM,QAAQ,KAAK,CAAC;AAAA,QAClD,QAAQ;AAAA,QAER;AAAA,MACF;AAAA,MACA,CAAC,UAAU;AACT,YAAI,KAAK,QAAQ,IAAI,QAAQ,MAAM,GAAG,WAAW,UAAW;AAC5D,cAAM,QAAQ,QAAQ,KAAK;AAC3B,YAAI;AACF,gBAAM,WAAW,MAAM,mBAAmB,KAAK;AAC/C,eAAK,cAAc,SAAS,YAAY,eAAe,QAAQ,KAAK,GAAG,MAAM,OAAO;AAAA,QACtF,SAAS,cAAc;AACrB,cACE,wBAAwB,8BACxB,wBAAwB,uBACxB;AACA;AAAA,UACF;AACA,eAAK,eAAe,SAAS,OAAO,YAAY;AAAA,QAClD;AAAA,MACF;AAAA,IACF,EACC,QAAQ,MAAM;AACb,WAAK,YAAY,OAAO,QAAQ,MAAM;AACtC,WAAK,kBAAkB,QAAQ,MAAM;AAAA,IACvC,CAAC;AAAA,EACL;AAAA,EAEQ,kBAAkB,QAAsB;AAC9C,SAAK,cAAc,OAAO,MAAM;AAChC,eAAW,WAAW,CAAC,GAAG,KAAK,WAAW,EAAG,SAAQ;AAAA,EACvD;AAAA,EAEQ,cACN,SACA,OACA,UACmB;AACnB,QAAI;AACF,YAAM,UAAU,KAAK,cAAc,SAAS,UAAU,MAAM,OAAO;AACnE,aAAO,YAAY,QAAQ,QAAQ,OAAO,OAAO;AAAA,IACnD,SAAS,cAAc;AACrB,UACE,wBAAwB,8BACxB,wBAAwB,uBACxB;AACA,eAAO,YAAY,QAAQ,QAAQ,YAAY;AAAA,MACjD;AACA,aAAO,KAAK,eAAe,SAAS,OAAO,YAAY;AAAA,IACzD;AAAA,EACF;AAAA,EAEQ,eACN,SACA,OACA,cACmB;AACnB,QAAI;AACF,YAAM,UAAU,KAAK;AAAA,QACnB;AAAA,QACA,eAAe,QAAQ,KAAK;AAAA,QAC5B,QAAQ,YAAY,EAAE;AAAA,MACxB;AACA,aAAO;AAAA,QACL,QAAQ;AAAA,QACR,IAAI,wBAAwB,QAAQ,QAAQ,OAAO,cAAc,OAAO;AAAA,QACxE;AAAA,MACF;AAAA,IACF,SAAS,OAAO;AACd,YAAM,QAAQ,iBAAiB,QAAQ,QAAQ,QAAQ,KAAK;AAC5D,aAAO,YAAY,QAAQ,QAAQ,KAAK;AAAA,IAC1C;AAAA,EACF;AAAA,EAEQ,cACN,SACA,UACA,OACa;AACb,UAAM,UAAU,aAAa,SAAS,UAAU,KAAK;AACrD,QAAI,KAAK,QAAQ,IAAI,QAAQ,MAAM,GAAG,WAAW,WAAW;AAC1D,YAAM,IAAI,sBAAsB,qBAAqB,QAAQ,MAAM,kBAAkB;AAAA,IACvF;AACA,SAAK,aAAa,OAAO;AACzB,WAAO,aAAa,OAAO;AAAA,EAC7B;AAAA,EAEQ,eAAe,SAAwD;AAC7E,QAAI,CAAC,SAAS;AACZ,UAAI,KAAK,mBAAmB,QAAW;AACrC,cAAM,IAAI;AAAA,UACR;AAAA,QACF;AAAA,MACF;AACA,aAAO;AAAA,IACT;AACA,QAAI,kCAAkC,SAAS;AAC7C;AAAA,QACE,QAAQ;AAAA,QACR;AAAA,MACF;AACA,aAAO,QAAQ;AAAA,IACjB;AACA,QAAI,wBAAwB,SAAS;AACnC,aAAO,oBAAoB,QAAQ,oBAAoB,OAAO;AAAA,IAChE;AACA,UAAM,SAAS,aAAa,QAAQ,OAAO,OAAO;AAClD,QAAI,OAAO,aAAa;AACtB,UAAI,KAAK,mBAAmB,QAAW;AACrC,cAAM,IAAI;AAAA,UACR,8CAA8C,QAAQ,KAAK;AAAA,QAC7D;AAAA,MACF;AACA,aAAO;AAAA,IACT;AACA,WAAO,OAAO;AAAA,EAChB;AAAA,EAEQ,2BAAoC;AAC1C,WAAO,CAAC,GAAG,KAAK,QAAQ,OAAO,CAAC,EAAE;AAAA,MAChC,CAAC,WACC,OAAO,WAAW,cACjB,OAAO,eACN,OAAO,gBACN,OAAO,mBAAmB,UAAa,OAAO,UAAU,OAAO;AAAA,IACtE;AAAA,EACF;AAAA,EAEQ,aAAa,QAAgC;AACnD,UAAM,SAAS,OAAO;AACtB,UAAM,UAAU,OAAO,WAAW,YAAY,SAAS;AACvD,uBAAmB,KAAK,SAAS,MAAM;AACvC,UAAM,QACJ,OAAO,WAAW,cACjB,OAAO,oBAAoB,UAAa,OAAO,qBAAqB,UACjE,EAAE,SAAS,GAAG,QAAQ,aAAa,MAAM,EAAE,IAC3C,EAAE,SAAS,GAAG,QAAQ,YAAY,MAAM,EAAE;AAChD,SAAK,YAAY,OAAO,QAAQ,OAAO;AACvC,SAAK,QAAQ,IAAI,QAAQ,YAAY,MAAM,CAAC;AAAA,EAC9C;AAAA,EAEQ,yBAAyB,QAAuB;AACtD,QAAI,KAAK,sBAAsB,KAAK,mBAAmB,OAAW;AAClE,SAAK,YAAY,EAAE,SAAS,GAAG,gBAAgB,KAAK,eAAe,GAAG,MAAM;AAC5E,SAAK,qBAAqB;AAAA,EAC5B;AAAA,EAEQ,YAAY,OAAwB,QAAiB,SAA6B;AACxF,QAAI;AACF,UAAI,KAAK,aAAa;AACpB,cAAM,eAAe,KAAK,YAAY,OAAO,KAAK,UAAU,GAAG,KAAK,UAAU,KAAK,CAAC;AAAA,CAAI;AACxF,YAAI,iBAAiB,QAAW;AAC9B,gBAAM,IAAI;AAAA,YACR,8DAA8D,MAAM;AAAA,YACpE,EAAE,QAAQ,QAAQ;AAAA,UACpB;AAAA,QACF;AACA,qBAAa,cAAc,sBAAsB;AACjD,aAAK,WAAW;AAAA,MAClB;AAAA,IACF,SAAS,OAAO;AACd,UAAI,iBAAiB,sBAAuB,OAAM;AAClD,YAAM,IAAI,2BAA2B,OAAO,QAAQ,OAAO;AAAA,IAC7D;AAAA,EACF;AACF;AAeO,SAAS,cAAc,OAA8B;AAC1D,SAAO,cAAc,KAAK,IAAI,QAAQ;AACxC;AAOO,SAAS,aAAa,OAAe,OAA8B;AACxE,cAAY,KAAK;AACjB,MAAI,CAAC,oBAAoB,KAAK,EAAG,QAAO,EAAE,SAAS,GAAG,aAAa,KAAK;AACxE,SAAO;AAAA,IACL,SAAS;AAAA,MACP,MAAM,eAAe,MAAM,gBAAgB,MAAM,MAAM,oBAAoB;AAAA,MAC3E,MAAM;AAAA,MACN;AAAA,IACF;AAAA,IACA,aAAa;AAAA,EACf;AACF;AAGO,SAAS,oBACd,SACA,OACQ;AACR,mBAAiB,MAAM,aAAa,mBAAmB;AACvD,mBAAiB,MAAM,cAAc,oBAAoB;AACzD,MAAI,MAAM,iBAAiB,QAAW;AACpC,qBAAiB,MAAM,cAAc,oBAAoB;AAAA,EAC3D;AACA,MAAI,MAAM,qBAAqB,QAAW;AACxC,qBAAiB,MAAM,kBAAkB,wBAAwB;AAAA,EACnE;AACA,oBAAkB,QAAQ,oBAAoB,4BAA4B;AAC1E,MAAI,QAAQ,6BAA6B,QAAW;AAClD,sBAAkB,QAAQ,0BAA0B,kCAAkC;AAAA,EACxF;AACA,MAAI,QAAQ,4BAA4B,QAAW;AACjD,sBAAkB,QAAQ,yBAAyB,iCAAiC;AAAA,EACtF;AACA,oBAAkB,QAAQ,qBAAqB,6BAA6B;AAC5E,QAAM,kBAAkB,QAAQ,4BAA4B,QAAQ;AACpE,QAAM,iBAAiB,QAAQ,2BAA2B,QAAQ;AAClE,SACG,MAAM,cAAc,MAAa,QAAQ,sBACxC,MAAM,gBAAgB,KAAK,MAAa,mBACxC,MAAM,oBAAoB,KAAK,MAAa,iBAC7C,MAAM,eAAe,MAAa,QAAQ;AAE/C;AAOA,eAAe,OAAU,SAAqB,QAA0C;AACtF,SAAO,MAAM,IAAI,QAAQ,CAAC,YAAY;AACpC,QAAI,OAAO;AACX,UAAM,SAAS,CAAC,UAA4B;AAC1C,UAAI,KAAM;AACV,aAAO;AACP,aAAO,oBAAoB,SAAS,OAAO;AAC3C,cAAQ,KAAK;AAAA,IACf;AACA,UAAM,UAAU,MAAM,OAAO,EAAE,MAAM,UAAU,CAAC;AAChD,WAAO,iBAAiB,SAAS,SAAS,EAAE,MAAM,KAAK,CAAC;AACxD,QAAI,OAAO,QAAS,SAAQ;AAC5B,YAAQ;AAAA,MACN,CAAC,UAAU,OAAO,EAAE,MAAM,SAAS,MAAM,CAAC;AAAA,MAC1C,CAAC,UAAU,OAAO,EAAE,MAAM,SAAS,OAAO,QAAQ,KAAK,EAAE,CAAC;AAAA,IAC5D;AAAA,EACF,CAAC;AACH;AAEA,SAAS,aACP,SACA,UACA,OACa;AACb,cAAY,QAAQ;AACpB,eAAa,SAAS,OAAO,eAAe;AAC5C,MAAI,SAAS,kBAAkB,UAAa,SAAS,uBAAuB,QAAW;AACrF,UAAM,IAAI;AAAA,MACR;AAAA,IACF;AAAA,EACF;AACA,QAAM,wBACJ,SAAS,uBAAuB,SAC5B,SACA,gBAAgB,SAAS,kBAAkB;AACjD,QAAM,YAAY,wBACd;AAAA,IACE,SAAS,gBAAgB,UAAU,qBAAqB;AAAA,IACxD,aAAa;AAAA,EACf,IACA,aAAa,SAAS,OAAO,QAAQ;AACzC,QAAM,YAAY,SAAS,kBAAkB;AAC7C,MAAI,aAAa,SAAS,gBAAgB,MAAM;AAC9C,UAAM,IAAI;AAAA,MACR;AAAA,IACF;AAAA,EACF;AACA,MAAI,UAAW,mBAAkB,SAAS,eAAgB,eAAe;AACzE,QAAM,eAAe,SAAS,iBAAiB;AAC/C,QAAM,cACJ,SAAS,gBAAgB,QAAS,CAAC,cAAc,gBAAgB,UAAU;AAC7E,QAAM,uBACJ,CAAC,yBAAyB,CAAC,aAAa,CAAC,cACrC,oBAAoB,SAAS,KAAK,IAClC;AACN,QAAM,iBACJ,CAAC,aAAa,CAAC,cACV,0BACA,uBACG;AAAA,IACE,qBAAqB,qBAAqB;AAAA,IAC1C,sBAAsB,qBAAqB;AAAA,EAC7C,IACA,UACJ;AACN,SAAO;AAAA,IACL;AAAA,MACE,QAAQ;AAAA,MACR,QAAQ,QAAQ;AAAA,MAChB,SAAS,QAAQ;AAAA,MACjB,OAAO,QAAQ;AAAA,MACf,OAAO,QAAQ;AAAA,MACf,OAAO,SAAS;AAAA,MAChB,aAAa,SAAS;AAAA,MACtB,cAAc,SAAS;AAAA,MACvB,GAAI,SAAS,oBAAoB,SAC7B,CAAC,IACD,EAAE,iBAAiB,SAAS,gBAAgB;AAAA,MAChD,GAAI,SAAS,iBAAiB,SAAY,CAAC,IAAI,EAAE,cAAc,SAAS,aAAa;AAAA,MACrF,GAAI,SAAS,qBAAqB,SAC9B,CAAC,IACD,EAAE,kBAAkB,SAAS,iBAAiB;AAAA,MAClD,SAAS,cAAc,IAAI,YAAY,SAAS,gBAAiB,UAAU;AAAA,MAC3E;AAAA,MACA;AAAA,MACA,GAAI,iBAAiB,EAAE,SAAS,eAAe,IAAI,CAAC;AAAA,MACpD,GAAI,YAAY,EAAE,eAAe,SAAS,cAAc,IAAI,CAAC;AAAA,MAC7D,GAAI,QAAQ,mBAAmB,SAAY,CAAC,IAAI,EAAE,gBAAgB,QAAQ,eAAe;AAAA,MACzF,GAAI,QAAQ,EAAE,MAAM,IAAI,CAAC;AAAA,MACzB,GAAI,QAAQ,OAAO,EAAE,MAAM,EAAE,GAAG,QAAQ,KAAK,EAAE,IAAI,CAAC;AAAA,MACpD,WAAW,QAAQ;AAAA,IACrB;AAAA,IACA;AAAA,EACF;AACF;AAEA,IAAM,iBAAiB,EAAE,OAAO,EAAE,OAAO,CAAC,UAAU,MAAM,KAAK,EAAE,SAAS,GAAG,mBAAmB;AAChG,IAAM,aAAa,EAAE,OAAO,EAAE,IAAI,EAAE,YAAY,EAAE,OAAO;AACzD,IAAM,cAAc,EAAE,OAAO,EAAE,YAAY,EAAE,OAAO;AACpD,IAAM,OAAO,EAAE,OAAO,gBAAgB,EAAE,OAAO,CAAC;AAChD,IAAM,oBAAoB,EAAE,aAAa;AAAA,EACvC,qBAAqB;AAAA,EACrB,2BAA2B,YAAY,SAAS;AAAA,EAChD,0BAA0B,YAAY,SAAS;AAAA,EAC/C,sBAAsB;AACxB,CAAC;AACD,IAAM,oBAAoB;AAAA,EACxB,QAAQ;AAAA,EACR,SAAS;AAAA,EACT,OAAO;AAAA,EACP,OAAO;AAAA,EACP,OAAO;AAAA,EACP,gBAAgB,YAAY,SAAS;AAAA,EACrC,MAAM,KAAK,SAAS;AAAA,EACpB,WAAW;AACb;AACA,IAAM,wBAAwB,EAAE,aAAa;AAAA,EAC3C,QAAQ,EAAE,QAAQ,SAAS;AAAA,EAC3B,GAAG;AACL,CAAC;AACD,IAAM,uBAAuB;AAAA,EAC3B,QAAQ,EAAE,QAAQ,SAAS;AAAA,EAC3B,GAAG;AAAA,EACH,aAAa;AAAA,EACb,cAAc;AAAA,EACd,cAAc,WAAW,SAAS;AAAA,EAClC,SAAS;AAAA,EACT,aAAa,EAAE,QAAQ;AAAA,EACvB,cAAc,EAAE,QAAQ,EAAE,QAAQ,KAAK;AAAA,EACvC,SAAS,kBAAkB,SAAS;AAAA,EACpC,eAAe,YAAY,SAAS;AAAA,EACpC,OAAO,EAAE,OAAO,EAAE,SAAS;AAC7B;AACA,IAAM,0BAA0B,EAC7B,aAAa,oBAAoB,EACjC,YAAY,CAAC,SAAS,QAAQ,oBAAoB,SAAS,GAAG,CAAC;AAClE,IAAM,oBAAoB,EACvB,aAAa;AAAA,EACZ,GAAG;AAAA,EACH,iBAAiB,WAAW,SAAS;AAAA,EACrC,kBAAkB,WAAW,SAAS;AACxC,CAAC,EACA,YAAY,CAAC,SAAS,QAAQ,oBAAoB,SAAS,GAAG,CAAC;AAElE,SAAS,oBACP,SAOA,KACM;AACN,MAAI,QAAQ,oBAAoB,UAAa,QAAQ,kBAAkB,QAAQ,cAAc;AAC3F,QAAI,SAAS,EAAE,MAAM,UAAU,SAAS,+CAA+C,CAAC;AAAA,EAC1F;AAEA,MAAI,QAAQ,kBAAkB,QAAW;AACvC,QAAI,QAAQ,eAAe,QAAQ,YAAY,QAAQ,eAAe;AACpE,UAAI,SAAS,EAAE,MAAM,UAAU,SAAS,8CAA8C,CAAC;AAAA,IACzF;AACA,QAAI,QAAQ,YAAY,QAAW;AACjC,UAAI,SAAS,EAAE,MAAM,UAAU,SAAS,iDAAiD,CAAC;AAAA,IAC5F;AACA;AAAA,EACF;AAEA,MAAI,QAAQ,aAAa;AACvB,QAAI,QAAQ,YAAY,GAAG;AACzB,UAAI,SAAS,EAAE,MAAM,UAAU,SAAS,mCAAmC,CAAC;AAAA,IAC9E;AACA,QAAI,QAAQ,YAAY,QAAW;AACjC,UAAI,SAAS,EAAE,MAAM,UAAU,SAAS,kDAAkD,CAAC;AAAA,IAC7F;AACA;AAAA,EACF;AAEA,MAAI,QAAQ,cAAc;AACxB,QAAI,SAAS,EAAE,MAAM,UAAU,SAAS,4CAA4C,CAAC;AAAA,EACvF;AACA,MAAI,CAAC,QAAQ,SAAS;AACpB,QAAI,SAAS,EAAE,MAAM,UAAU,SAAS,mDAAmD,CAAC;AAC5F;AAAA,EACF;AAEA,QAAM,WAAW,gBAAgB,SAAS,QAAQ,OAAO;AACzD,MAAI,QAAQ,YAAY,UAAU;AAChC,QAAI,SAAS;AAAA,MACX,MAAM;AAAA,MACN,SAAS,kBAAkB,QAAQ,OAAO,oCAAoC,QAAQ;AAAA,IACxF,CAAC;AAAA,EACH;AACF;AAEA,IAAM,wBAAwB,EAAE,MAAM;AAAA,EACpC,EAAE,aAAa;AAAA,IACb,SAAS,EAAE,QAAQ,CAAC;AAAA,IACpB,QAAQ,EAAE,MAAM,CAAC,uBAAuB,uBAAuB,CAAC;AAAA,EAClE,CAAC;AAAA,EACD,EAAE,aAAa;AAAA,IACb,SAAS,EAAE,QAAQ,CAAC;AAAA,IACpB,QAAQ;AAAA,EACV,CAAC;AAAA,EACD,EAAE,aAAa;AAAA,IACb,SAAS,EAAE,QAAQ,CAAC;AAAA,IACpB,gBAAgB;AAAA,EAClB,CAAC;AAAA,EACD,EAAE,aAAa;AAAA,IACb,SAAS,EAAE,QAAQ,CAAC;AAAA,IACpB,gBAAgB;AAAA,EAClB,CAAC;AACH,CAAC;AAED,SAAS,YAAY,YAInB;AACA,QAAM,UAAU,oBAAI,IAA8B;AAClD,MAAI,iBAAiB;AACrB,MAAI;AACJ,MAAI,aAAa;AACjB,MAAI;AACF,eAAW,QAAQ,WAAW,MAAM,IAAI,GAAG;AACzC,UAAI,CAAC,KAAK,KAAK,EAAG;AAClB,oBAAc;AACd,YAAM,QAAQ,sBAAsB,MAAM,KAAK,MAAM,IAAI,CAAC;AAC1D,UAAI,YAAY,OAAO;AACrB,cAAM,SAAS,MAAM;AACrB,2BAAmB,SAAS,MAAM;AAClC,gBAAQ,IAAI,OAAO,QAAQ,YAAY,MAAM,CAAC;AAAA,MAChD,WAAW,oBAAoB,OAAO;AACpC,0BAAkB,MAAM;AACxB,YAAI,CAAC,OAAO,cAAc,cAAc,GAAG;AACzC,gBAAM,IAAI,gBAAgB,6DAA6D;AAAA,QACzF;AAAA,MACF,OAAO;AACL,YAAI,mBAAmB,QAAW;AAChC,gBAAM,IAAI,gBAAgB,8CAA8C;AAAA,QAC1E;AACA,yBAAiB,MAAM;AAAA,MACzB;AAAA,IACF;AACA,WAAO;AAAA,MACL,SAAS,CAAC,GAAG,QAAQ,OAAO,CAAC;AAAA,MAC7B;AAAA,MACA,GAAI,mBAAmB,SAAY,CAAC,IAAI,EAAE,eAAe;AAAA,IAC3D;AAAA,EACF,SAAS,OAAO;AACd,UAAM,IAAI;AAAA,MACR,uCAAuC,cAAc,CAAC,KAAK,kBAAkB,KAAK,CAAC;AAAA,MACnF,EAAE,MAAM;AAAA,IACV;AAAA,EACF;AACF;AAEA,SAAS,mBACP,SACA,QACM;AACN,QAAM,UAAU,QAAQ,IAAI,OAAO,MAAM;AACzC,MAAI,CAAC,QAAS;AACd,MAAI,OAAO,WAAW,aAAa,QAAQ,WAAW,WAAW;AAC/D,UAAM,IAAI,sBAAsB,iCAAiC,OAAO,MAAM,GAAG;AAAA,EACnF;AACA,MAAI,CAAC,gBAAgB,SAAS,MAAM,GAAG;AACrC,UAAM,IAAI,gBAAgB,qDAAqD,OAAO,MAAM,GAAG;AAAA,EACjG;AACF;AAEA,SAAS,gBAAgB,QAAsB,OAA8B;AAC3E,SACE,OAAO,YAAY,MAAM,WACzB,OAAO,UAAU,MAAM,SACvB,OAAO,UAAU,MAAM,SACvB,OAAO,mBAAmB,MAAM,kBAChC,OAAO,cAAc,MAAM,aAC3B,KAAK,UAAU,OAAO,QAAQ,CAAC,CAAC,MAAM,KAAK,UAAU,MAAM,QAAQ,CAAC,CAAC;AAEzE;AAEA,SAAS,aAAa,OAAgB,MAA2B;AAC/D,MAAI;AACF,WAAO,kBAAkB,MAAM,KAAK;AAAA,EACtC,SAAS,OAAO;AACd,UAAM,IAAI,gBAAgB,uBAAuB,IAAI,KAAK,kBAAkB,KAAK,CAAC,IAAI,EAAE,MAAM,CAAC;AAAA,EACjG;AACF;AAEA,SAAS,qBAAqB,OAAgB,MAA2B;AACvE,MAAI,OAAO,UAAU,YAAY,UAAU,KAAM,QAAO,aAAa,OAAO,IAAI;AAChF,QAAM,YAAY,EAAE,GAAG,MAAM;AAC7B,MACE,UAAU,WAAW,aACrB,UAAU,kBAAkB,UAC5B,UAAU,gBAAgB,SAC1B,UAAU,YAAY,UACtB,OAAO,UAAU,UAAU,UAC3B;AACA,UAAM,UAAU,oBAAoB,UAAU,KAAK;AACnD,QAAI,SAAS;AACX,gBAAU,UAAU;AAAA,QAClB,qBAAqB,QAAQ;AAAA,QAC7B,sBAAsB,QAAQ;AAAA,MAChC;AAAA,IACF;AAAA,EACF;AACA,SAAO,aAAa,WAAW,IAAI;AACrC;AAEA,SAAS,oBACP,OACM;AACN,eAAa,MAAM,SAAS,SAAS;AACrC,eAAa,MAAM,OAAO,OAAO;AACjC,eAAa,MAAM,OAAO,OAAO;AACjC,MAAI,MAAM,UAAU,OAAW,cAAa,MAAM,OAAO,OAAO;AAChE,MAAI,MAAM,SAAS,QAAW;AAC5B,UAAM,SAAS,KAAK,UAAU,MAAM,IAAI;AACxC,QAAI,CAAC,OAAO;AACV,YAAM,IAAI,gBAAgB,6BAA6B,OAAO,MAAM,OAAO,EAAE;AAAA,EACjF;AACF;AAEA,SAAS,QAAQ,QAAsB,QAA+C;AACpF,MAAI,CAAC,OAAQ,QAAO;AACpB,MAAI,OAAO,YAAY,UAAa,OAAO,YAAY,OAAO,QAAS,QAAO;AAC9E,MAAI,OAAO,UAAU,UAAa,OAAO,UAAU,OAAO,MAAO,QAAO;AACxE,SAAO,OAAO,QAAQ,OAAO,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,KAAK,KAAK,MAAM,OAAO,OAAO,GAAG,MAAM,KAAK;AAC/F;AAEA,SAAS,YAAY,QAA4C;AAC/D,MAAI,OAAO,WAAW,UAAW,QAAO,aAAa,MAAM;AAC3D,SAAO,iBAAiB,MAAM;AAChC;AAEA,SAAS,iBAAiB,QAA0C;AAClE,QAAM,EAAE,MAAM,GAAG,KAAK,IAAI;AAC1B,SAAO,EAAE,GAAG,MAAM,GAAI,OAAO,EAAE,MAAM,EAAE,GAAG,KAAK,EAAE,IAAI,CAAC,EAAG;AAC3D;AAEA,SAAS,aAAa,SAAmC;AACvD,QAAM,EAAE,SAAS,MAAM,GAAG,KAAK,IAAI;AACnC,SAAO;AAAA,IACL,GAAG;AAAA,IACH,GAAI,OAAO,EAAE,MAAM,EAAE,GAAG,KAAK,EAAE,IAAI,CAAC;AAAA,IACpC,GAAI,UAAU,EAAE,SAAS,EAAE,GAAG,QAAQ,EAAE,IAAI,CAAC;AAAA,EAC/C;AACF;AAEA,SAAS,gBAAgB,OAAkB,SAAsD;AAC/F,QAAM,kBAAkB,QAAQ,6BAA6B,QAAQ;AACrE,QAAM,iBAAiB,QAAQ,4BAA4B,QAAQ;AACnE,SACG,MAAM,cAAc,MAAQ,QAAQ,uBACnC,MAAM,gBAAgB,KAAK,MAAQ,mBACnC,MAAM,oBAAoB,KAAK,MAAQ,iBACxC,MAAM,eAAe,MAAQ,QAAQ;AAE1C;AAEA,SAAS,gBAAgB,SAAkE;AACzF,sBAAoB,SAAS,EAAE,aAAa,GAAG,cAAc,EAAE,CAAC;AAChE,SAAO;AAAA,IACL,qBAAqB,QAAQ,qBAAqB;AAAA,IAClD,GAAI,QAAQ,6BAA6B,SACrC,CAAC,IACD,EAAE,2BAA2B,QAAQ,2BAA2B,IAAK;AAAA,IACzE,GAAI,QAAQ,4BAA4B,SACpC,CAAC,IACD,EAAE,0BAA0B,QAAQ,0BAA0B,IAAK;AAAA,IACvE,sBAAsB,QAAQ,sBAAsB;AAAA,EACtD;AACF;AAEA,SAAS,YAAY,SAAqC;AACxD,SAAO;AAAA,IACL;AAAA,IACA,OAAO;AAAA,IACP,aAAa;AAAA,IACb,cAAc;AAAA,IACd,iBAAiB;AAAA,IACjB,cAAc;AAAA,IACd,kBAAkB;AAAA,IAClB,SAAS;AAAA,IACT,eAAe;AAAA,IACf,mBAAmB;AAAA,EACrB;AACF;AAEA,SAAS,aAAa,OAA0C;AAC9D,MAAI,MAAM,MAAO,QAAO,MAAM;AAC9B,MAAI,MAAM,iBAAiB,WAAW,MAAM,cAAe,QAAO,MAAM,cAAc;AACtF,SAAO;AACT;AAEA,SAAS,eAAe,OAAiC;AACvD,SAAO,EAAE,OAAO,aAAa,GAAG,cAAc,GAAG,aAAa,MAAM,cAAc,KAAK;AACzF;AAEA,SAAS,cAAc,OAAmC;AACxD,MAAI,UAAU,QAAW;AACvB,iBAAa,OAAO,QAAQ;AAC5B,WAAO;AAAA,EACT;AACA,MAAI,OAAO,WAAW,QAAQ,eAAe,YAAY;AACvD,UAAM,IAAI,gBAAgB,kEAAkE;AAAA,EAC9F;AACA,SAAO,WAAW,OAAO,WAAW;AACtC;AAEA,SAAS,WAAW,QAA4B;AAC9C,QAAM,SAAU,OAAgC;AAChD,MAAI,kBAAkB,MAAO,QAAO;AACpC,QAAM,QAAQ,IAAI,MAAM,+BAA+B;AACvD,QAAM,OAAO;AACb,SAAO;AACT;AAEA,SAAS,QAAQ,OAAuB;AACtC,SAAO,iBAAiB,QAAQ,QAAQ,IAAI,MAAM,OAAO,KAAK,CAAC;AACjE;AAEA,SAAS,YAAe,QAAgB,OAAc,SAA0C;AAC9F,SAAO;AAAA,IACL,WAAW;AAAA,IACX;AAAA,IACA;AAAA,IACA,GAAI,UAAU,EAAE,SAAS,aAAa,OAAO,EAAE,IAAI,CAAC;AAAA,EACtD;AACF;AAEA,SAAS,kBAAkB,OAAwB;AACjD,SAAO,iBAAiB,EAAE,WAAW,EAAE,cAAc,KAAK,IAAI,QAAQ,KAAK,EAAE;AAC/E;AAEA,SAAS,YAAY,OAAwB;AAC3C,mBAAiB,MAAM,aAAa,aAAa;AACjD,mBAAiB,MAAM,cAAc,cAAc;AACnD,MAAI,MAAM,oBAAoB,QAAW;AACvC,qBAAiB,MAAM,iBAAiB,iBAAiB;AACzD,QAAI,MAAM,kBAAkB,MAAM,cAAc;AAC9C,YAAM,IAAI,gBAAgB,0DAA0D;AAAA,IACtF;AAAA,EACF;AACA,MAAI,MAAM,iBAAiB,OAAW,kBAAiB,MAAM,cAAc,cAAc;AACzF,MAAI,MAAM,qBAAqB,QAAW;AACxC,qBAAiB,MAAM,kBAAkB,kBAAkB;AAAA,EAC7D;AACF;AAEA,SAAS,iBAAiB,OAAgB,MAAuC;AAC/E,MAAI,OAAO,UAAU,YAAY,CAAC,OAAO,UAAU,KAAK,KAAK,QAAQ,GAAG;AACtE,UAAM,IAAI;AAAA,MACR,eAAe,IAAI,wCAAwC,OAAO,KAAK,CAAC;AAAA,IAC1E;AAAA,EACF;AACF;AAEA,SAAS,kBAAkB,OAAgB,MAAuC;AAChF,MAAI,OAAO,UAAU,YAAY,CAAC,OAAO,SAAS,KAAK,KAAK,QAAQ,GAAG;AACrE,UAAM,IAAI;AAAA,MACR,eAAe,IAAI,8CAA8C,OAAO,KAAK,CAAC;AAAA,IAChF;AAAA,EACF;AACF;AAEA,SAAS,cAAc,OAAgB,MAAuC;AAC5E,MACE,OAAO,UAAU,YACjB,CAAC,OAAO,cAAc,KAAK,KAC3B,QAAQ,KACR,QAAQ,YACR;AACA,UAAM,IAAI;AAAA,MACR,eAAe,IAAI,wEAAwE,OAAO,KAAK,CAAC;AAAA,IAC1G;AAAA,EACF;AACF;AAEA,SAAS,aAAa,OAAgB,MAAuC;AAC3E,MAAI,OAAO,UAAU,YAAY,MAAM,KAAK,EAAE,WAAW,GAAG;AAC1D,UAAM,IAAI,gBAAgB,eAAe,IAAI,6BAA6B;AAAA,EAC5E;AACF;","names":[]}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/release-confidence.ts","../src/promotion-gate.ts","../src/release-report.ts"],"sourcesContent":["/**\n * Release confidence gate.\n *\n * This is the production-facing composition layer over the lower-level\n * primitives:\n * - Dataset manifests prove corpus/version coverage.\n * - RunRecord rows prove reproducible search/holdout outcomes.\n * - Multi-shot trace evidence carries turn counts and ASI diagnostics.\n * - HeldOutGate decisions remain the paired promotion authority.\n *\n * The gate is intentionally pure and conservative. Missing declared evidence\n * fails closed instead of being treated as a neutral zero.\n */\n\nimport type { DatasetManifest, DatasetScenario, DatasetSplit } from './dataset'\nimport { ValidationError, VerificationError } from './errors'\nimport type { GateDecision } from './held-out-gate'\nimport { type RunRecord, type RunSplitTag, validateRunRecord } from './run-record'\nimport { FAILURE_CLASSES, type FailureClass } from './trace/schema'\n\n/** Severity of an actionable finding attached to a run/trace. */\nexport type AsiSeverity = 'info' | 'warning' | 'error' | 'critical'\n\n/** Actionable side-info — a diagnosed finding the loop can act on. */\nexport interface ActionableSideInfo {\n /** Stable expectation/check id when available. */\n expectationId?: string\n /** Human-readable diagnosis of what happened. */\n message: string\n severity?: AsiSeverity\n /** Concrete trace excerpt, file path, tool call, screenshot id, etc. */\n evidence?: string\n /** Prompt/tool/context surface likely responsible. */\n responsibleSurface?: string\n /** Suggested fix in natural language. */\n suggestion?: string\n /** Whether this expectation was satisfied. Defaults to false for ASI rows. */\n matched?: boolean\n metadata?: Record<string, unknown>\n}\n\nexport type ReleaseConfidenceStatus = 'pass' | 'warn' | 'fail'\nexport type ReleaseConfidenceAxisName =\n | 'corpus'\n | 'quality'\n | 'reliability'\n | 'generalization'\n | 'diagnostics'\n | 'efficiency'\n\nexport interface ReleaseTraceEvidence {\n scenarioId: string\n candidateId?: string\n split?: RunSplitTag\n score?: number\n ok?: boolean\n turnCount?: number\n costUsd?: number\n durationMs?: number\n /** Canonical task-failure class. Free-form detail belongs in ASI. */\n failureClass?: FailureClass\n asi?: ActionableSideInfo[]\n metadata?: Record<string, unknown>\n}\n\nexport interface ReleaseConfidenceThresholds {\n /** Require a Dataset manifest or explicit scenarios. Default true. */\n requireCorpus?: boolean\n minScenarioCount?: number\n minSearchRuns?: number\n minHoldoutRuns?: number\n /** Require at least one holdout scenario/run. Default true. */\n requireHoldout?: boolean\n minPassRate?: number\n minMeanScore?: number\n /** Search mean may exceed holdout mean by at most this much. */\n maxOverfitGap?: number\n maxMeanCostUsd?: number\n maxP95WallMs?: number\n /** Low-score/failed rows must carry ASI. Default true. */\n requireAsiForFailures?: boolean\n /** Score below this is considered a failure for ASI coverage. Default 0.5. */\n failureScoreThreshold?: number\n}\n\nexport interface ReleaseConfidenceInput {\n target: string\n candidateId?: string\n baselineId?: string\n dataset?: DatasetManifest\n scenarios?: readonly DatasetScenario[]\n runs?: readonly RunRecord[]\n traces?: readonly ReleaseTraceEvidence[]\n gateDecision?: GateDecision | null\n thresholds?: ReleaseConfidenceThresholds\n}\n\nexport interface ReleaseConfidenceAxis {\n name: ReleaseConfidenceAxisName\n status: ReleaseConfidenceStatus\n score: number | null\n detail: string\n}\n\nexport interface ReleaseConfidenceIssue {\n axis: ReleaseConfidenceAxisName\n severity: 'critical' | 'warning'\n code: string\n detail: string\n}\n\nexport interface ReleaseConfidenceMetrics {\n scenarioCount: number\n /** Search rows with a finite search score. */\n searchRuns: number\n /** Holdout rows with a finite holdout score. */\n holdoutRuns: number\n /** Runs with neither a split-matched score nor an explicit task failure. */\n unscoredRuns: number\n /** Run rows, or trace rows when no runs exist, with no classified terminal result. */\n unclassifiedTerminalRuns: number\n /** Run rows, or trace rows when no runs exist, that ended unsuccessfully. */\n terminalFailureRuns: number\n /** Success fraction when every run or fallback trace row has a classified result. */\n reliabilityRate: number | null\n passRate: number | null\n meanScore: number | null\n searchMeanScore: number | null\n holdoutMeanScore: number | null\n overfitGap: number | null\n meanCostUsd: number | null\n p95WallMs: number | null\n failedRows: number\n failuresWithAsi: number\n singleShotTraces: number\n multiShotTraces: number\n splitCounts: Record<DatasetSplit, number>\n domainCounts: Record<string, number>\n failureClassCounts: Partial<Record<FailureClass, number>>\n responsibleSurfaceCounts: Record<string, number>\n}\n\nexport interface ReleaseConfidenceScorecard {\n target: string\n candidateId: string | null\n baselineId: string | null\n status: ReleaseConfidenceStatus\n promote: boolean\n axes: ReleaseConfidenceAxis[]\n issues: ReleaseConfidenceIssue[]\n metrics: ReleaseConfidenceMetrics\n dataset: DatasetManifest | null\n gateDecision: GateDecision | null\n summary: string\n}\n\nconst DEFAULT_THRESHOLDS: Required<ReleaseConfidenceThresholds> = {\n requireCorpus: true,\n minScenarioCount: 1,\n minSearchRuns: 1,\n minHoldoutRuns: 1,\n requireHoldout: true,\n minPassRate: 0.8,\n minMeanScore: 0.7,\n maxOverfitGap: 0.15,\n maxMeanCostUsd: Number.POSITIVE_INFINITY,\n maxP95WallMs: Number.POSITIVE_INFINITY,\n requireAsiForFailures: true,\n failureScoreThreshold: 0.5,\n}\n\nexport function evaluateReleaseConfidence(\n input: ReleaseConfidenceInput,\n): ReleaseConfidenceScorecard {\n const thresholds = { ...DEFAULT_THRESHOLDS, ...input.thresholds }\n const candidateId = input.candidateId ?? null\n const runs = filterCandidate(\n (input.runs ?? []).map(validateRunRecord),\n candidateId,\n input.baselineId,\n )\n const traces = filterTraceCandidate(\n (input.traces ?? []).map(validateReleaseTraceEvidence),\n candidateId,\n input.baselineId,\n )\n const scenarios = input.scenarios ?? []\n const scenarioCount = input.dataset?.scenarioCount ?? scenarios.length\n const splitCounts = input.dataset?.splitCounts ?? countScenarioSplits(scenarios)\n const searchScores = scoresFor(runs, 'search')\n const holdoutScores = scoresFor(runs, 'holdout')\n const runScores = runs.map(runSplitScore).filter(isFiniteNumber)\n const traceScores = traces.map((t) => t.score).filter(isFiniteNumber)\n const scoreUniverse = runs.length > 0 ? runScores : traceScores\n const qualityRuns = runs.filter((run) => runSplitScore(run) !== undefined)\n const unscoredRuns = runs.filter(\n (run) =>\n runSplitScore(run) === undefined &&\n !hasExplicitTaskFailure(run) &&\n !isFailedTerminalOutcome(run.terminalOutcome),\n ).length\n const passOutcomes =\n runs.length > 0\n ? runs.map((run) => runPassOutcome(run, thresholds.failureScoreThreshold))\n : traces.map((trace) => tracePassOutcome(trace, thresholds.failureScoreThreshold))\n const reliabilityRows =\n runs.length > 0\n ? runs.map((run) => terminalSuccess(run.terminalOutcome))\n : traces.map((trace) => trace.ok)\n const unclassifiedTerminalRuns = reliabilityRows.filter((outcome) => outcome === undefined).length\n const terminalFailureRuns = reliabilityRows.filter((outcome) => outcome === false).length\n const reliabilityRate =\n reliabilityRows.length === 0 || unclassifiedTerminalRuns > 0\n ? null\n : (reliabilityRows.length - terminalFailureRuns) / reliabilityRows.length\n const searchRuns = qualityRuns.filter((r) => r.splitTag === 'search').length\n const holdoutRuns = qualityRuns.filter((r) => r.splitTag === 'holdout').length\n const failed = failedRows(runs, traces, thresholds.failureScoreThreshold)\n const searchMeanScore = meanOrNull(searchScores)\n const holdoutMeanScore = meanOrNull(holdoutScores)\n const runCosts = runs.flatMap((run) =>\n run.costProvenance.kind === 'uncaptured' ? [] : [run.costProvenance.usd],\n )\n const traceCosts = traces.map((trace) => trace.costUsd).filter(isFiniteNumber)\n const meanCostUsd =\n runs.length > 0\n ? runCosts.length === runs.length\n ? meanOrNull(runCosts)\n : null\n : meanOrNull(traceCosts)\n const wallTimes =\n runs.length > 0\n ? runs.map((run) => run.wallMs)\n : traces.map((trace) => trace.durationMs).filter(isFiniteNumber)\n const metrics: ReleaseConfidenceMetrics = {\n scenarioCount,\n searchRuns,\n holdoutRuns,\n unscoredRuns,\n unclassifiedTerminalRuns,\n terminalFailureRuns,\n reliabilityRate,\n passRate: passOutcomeRate(passOutcomes),\n meanScore: meanOrNull(scoreUniverse),\n searchMeanScore,\n holdoutMeanScore,\n overfitGap: diffOrNull(searchMeanScore, holdoutMeanScore),\n meanCostUsd,\n p95WallMs: percentileOrNull(wallTimes, 0.95),\n failedRows: failed.length,\n failuresWithAsi: failed.filter((row) => row.hasAsi).length,\n singleShotTraces: traces.filter((t) => t.turnCount === 1).length,\n multiShotTraces: traces.filter((t) => (t.turnCount ?? 0) > 1).length,\n splitCounts,\n domainCounts: countDomains(scenarios),\n failureClassCounts: countFailureClasses(runs, traces, thresholds.failureScoreThreshold),\n responsibleSurfaceCounts: countResponsibleSurfaces(traces),\n }\n\n const issues: ReleaseConfidenceIssue[] = []\n checkCorpus(input, thresholds, metrics, issues)\n checkQuality(thresholds, metrics, issues)\n checkReliability(metrics, issues)\n checkGeneralization(input.gateDecision ?? null, thresholds, metrics, issues)\n checkDiagnostics(thresholds, metrics, issues)\n checkEfficiency(thresholds, metrics, issues)\n\n const axes = buildAxes(metrics, thresholds, issues)\n const status = issues.some((i) => i.severity === 'critical')\n ? 'fail'\n : issues.length > 0\n ? 'warn'\n : 'pass'\n\n return {\n target: input.target,\n candidateId,\n baselineId: input.baselineId ?? null,\n status,\n promote: status === 'pass' && (input.gateDecision ? input.gateDecision.promote : true),\n axes,\n issues,\n metrics,\n dataset: input.dataset ?? null,\n gateDecision: input.gateDecision ?? null,\n summary: renderSummary(input.target, status, metrics, issues),\n }\n}\n\nexport function assertReleaseConfidence(input: ReleaseConfidenceInput): ReleaseConfidenceScorecard {\n const scorecard = evaluateReleaseConfidence(input)\n if (scorecard.status === 'fail') {\n throw new VerificationError(scorecard.summary)\n }\n return scorecard\n}\n\nfunction filterCandidate(\n runs: readonly RunRecord[],\n candidateId: string | null,\n baselineId?: string,\n): RunRecord[] {\n if (candidateId) return runs.filter((r) => r.candidateId === candidateId)\n if (baselineId) return runs.filter((r) => r.candidateId !== baselineId)\n return [...runs]\n}\n\nfunction filterTraceCandidate(\n traces: readonly ReleaseTraceEvidence[],\n candidateId: string | null,\n baselineId?: string,\n): ReleaseTraceEvidence[] {\n if (candidateId)\n return traces.filter((t) => t.candidateId === undefined || t.candidateId === candidateId)\n if (baselineId)\n return traces.filter((t) => t.candidateId === undefined || t.candidateId !== baselineId)\n return [...traces]\n}\n\nfunction validateReleaseTraceEvidence(\n trace: ReleaseTraceEvidence,\n index: number,\n): ReleaseTraceEvidence {\n const value = trace as unknown as Record<string, unknown>\n if (Object.hasOwn(value, 'failureMode')) {\n throw new ValidationError(\n `traces[${index}].failureMode is not supported; use canonical failureClass`,\n )\n }\n if (trace.failureClass !== undefined && !FAILURE_CLASSES.includes(trace.failureClass)) {\n throw new ValidationError(\n `traces[${index}].failureClass must be one of ${FAILURE_CLASSES.join(', ')}`,\n )\n }\n return trace\n}\n\nfunction checkCorpus(\n input: ReleaseConfidenceInput,\n thresholds: Required<ReleaseConfidenceThresholds>,\n metrics: ReleaseConfidenceMetrics,\n issues: ReleaseConfidenceIssue[],\n): void {\n if (thresholds.requireCorpus && !input.dataset && (input.scenarios?.length ?? 0) === 0) {\n issues.push({\n axis: 'corpus',\n severity: 'critical',\n code: 'missing_corpus',\n detail: 'No Dataset manifest or scenarios supplied.',\n })\n }\n if (metrics.scenarioCount < thresholds.minScenarioCount) {\n issues.push({\n axis: 'corpus',\n severity: 'critical',\n code: 'few_scenarios',\n detail: `${metrics.scenarioCount} scenario(s) < min ${thresholds.minScenarioCount}.`,\n })\n }\n if (thresholds.requireHoldout && metrics.splitCounts.holdout === 0) {\n issues.push({\n axis: 'corpus',\n severity: 'critical',\n code: 'missing_holdout_split',\n detail: 'Corpus has no holdout scenarios.',\n })\n }\n}\n\nfunction checkQuality(\n thresholds: Required<ReleaseConfidenceThresholds>,\n metrics: ReleaseConfidenceMetrics,\n issues: ReleaseConfidenceIssue[],\n): void {\n if (metrics.searchRuns < thresholds.minSearchRuns) {\n issues.push({\n axis: 'quality',\n severity: 'critical',\n code: 'few_search_runs',\n detail: `${metrics.searchRuns} search run(s) < min ${thresholds.minSearchRuns}.`,\n })\n }\n if (metrics.unscoredRuns > 0) {\n issues.push({\n axis: 'quality',\n severity: 'critical',\n code: 'unscored_runs',\n detail: `${metrics.unscoredRuns} supplied run(s) have no task result.`,\n })\n }\n if (metrics.passRate === null || metrics.meanScore === null) {\n issues.push({\n axis: 'quality',\n severity: 'critical',\n code: 'missing_quality_scores',\n detail: 'No task-quality scores are available for pass-rate and mean-score checks.',\n })\n }\n if (metrics.passRate !== null && metrics.passRate < thresholds.minPassRate) {\n issues.push({\n axis: 'quality',\n severity: 'critical',\n code: 'low_pass_rate',\n detail: `passRate ${fmt(metrics.passRate)} < ${fmt(thresholds.minPassRate)}.`,\n })\n }\n if (metrics.meanScore !== null && metrics.meanScore < thresholds.minMeanScore) {\n issues.push({\n axis: 'quality',\n severity: 'critical',\n code: 'low_mean_score',\n detail: `meanScore ${fmt(metrics.meanScore)} < ${fmt(thresholds.minMeanScore)}.`,\n })\n }\n}\n\nfunction checkReliability(\n metrics: ReleaseConfidenceMetrics,\n issues: ReleaseConfidenceIssue[],\n): void {\n if (metrics.reliabilityRate === null) {\n issues.push({\n axis: 'reliability',\n severity: 'critical',\n code: 'missing_reliability_evidence',\n detail:\n metrics.unclassifiedTerminalRuns > 0\n ? `${metrics.unclassifiedTerminalRuns} supplied run(s) have no classified terminal result.`\n : 'No classified terminal results are available.',\n })\n }\n if (metrics.terminalFailureRuns > 0) {\n issues.push({\n axis: 'reliability',\n severity: 'critical',\n code: 'terminal_run_failures',\n detail: `${metrics.terminalFailureRuns} run(s) ended failed, cancelled, or incomplete.`,\n })\n }\n}\n\nfunction checkGeneralization(\n gateDecision: GateDecision | null,\n thresholds: Required<ReleaseConfidenceThresholds>,\n metrics: ReleaseConfidenceMetrics,\n issues: ReleaseConfidenceIssue[],\n): void {\n if (thresholds.requireHoldout && metrics.holdoutRuns < thresholds.minHoldoutRuns) {\n issues.push({\n axis: 'generalization',\n severity: 'critical',\n code: 'few_holdout_runs',\n detail: `${metrics.holdoutRuns} holdout run(s) < min ${thresholds.minHoldoutRuns}.`,\n })\n }\n if (metrics.overfitGap !== null && metrics.overfitGap > thresholds.maxOverfitGap) {\n issues.push({\n axis: 'generalization',\n severity: 'critical',\n code: 'overfit_gap',\n detail: `search-holdout gap ${fmt(metrics.overfitGap)} > ${fmt(thresholds.maxOverfitGap)}.`,\n })\n }\n if (gateDecision && !gateDecision.promote) {\n issues.push({\n axis: 'generalization',\n severity: 'critical',\n code: `gate_${gateDecision.rejectionCode ?? 'reject'}`,\n detail: gateDecision.reason,\n })\n }\n}\n\nfunction checkDiagnostics(\n thresholds: Required<ReleaseConfidenceThresholds>,\n metrics: ReleaseConfidenceMetrics,\n issues: ReleaseConfidenceIssue[],\n): void {\n if (!thresholds.requireAsiForFailures) return\n if (metrics.failedRows > metrics.failuresWithAsi) {\n issues.push({\n axis: 'diagnostics',\n severity: 'critical',\n code: 'missing_failure_asi',\n detail: `${metrics.failedRows - metrics.failuresWithAsi} failed row(s) have no actionable side information.`,\n })\n }\n}\n\nfunction checkEfficiency(\n thresholds: Required<ReleaseConfidenceThresholds>,\n metrics: ReleaseConfidenceMetrics,\n issues: ReleaseConfidenceIssue[],\n): void {\n if (Number.isFinite(thresholds.maxMeanCostUsd) && metrics.meanCostUsd === null) {\n issues.push({\n axis: 'efficiency',\n severity: 'critical',\n code: 'missing_cost',\n detail: 'A finite cost limit was configured but no cost evidence is available.',\n })\n } else if (metrics.meanCostUsd !== null && metrics.meanCostUsd > thresholds.maxMeanCostUsd) {\n issues.push({\n axis: 'efficiency',\n severity: 'critical',\n code: 'cost_budget',\n detail: `meanCostUsd ${fmt(metrics.meanCostUsd)} > ${fmt(thresholds.maxMeanCostUsd)}.`,\n })\n }\n if (Number.isFinite(thresholds.maxP95WallMs) && metrics.p95WallMs === null) {\n issues.push({\n axis: 'efficiency',\n severity: 'critical',\n code: 'missing_latency',\n detail: 'A finite latency limit was configured but no latency evidence is available.',\n })\n } else if (metrics.p95WallMs !== null && metrics.p95WallMs > thresholds.maxP95WallMs) {\n issues.push({\n axis: 'efficiency',\n severity: 'critical',\n code: 'latency_budget',\n detail: `p95WallMs ${fmt(metrics.p95WallMs)} > ${fmt(thresholds.maxP95WallMs)}.`,\n })\n }\n}\n\nfunction buildAxes(\n metrics: ReleaseConfidenceMetrics,\n thresholds: Required<ReleaseConfidenceThresholds>,\n issues: ReleaseConfidenceIssue[],\n): ReleaseConfidenceAxis[] {\n return [\n axis(\n 'corpus',\n issues,\n bounded(metrics.scenarioCount / Math.max(1, thresholds.minScenarioCount)),\n `${metrics.scenarioCount} scenarios; holdout=${metrics.splitCounts.holdout}`,\n ),\n axis(\n 'quality',\n issues,\n metrics.passRate === null || metrics.meanScore === null\n ? null\n : Math.min(metrics.passRate, metrics.meanScore),\n `passRate=${fmt(metrics.passRate)} meanScore=${fmt(metrics.meanScore)}`,\n ),\n axis(\n 'reliability',\n issues,\n metrics.reliabilityRate,\n `successRate=${fmt(metrics.reliabilityRate)} terminalFailures=${metrics.terminalFailureRuns} unclassified=${metrics.unclassifiedTerminalRuns}`,\n ),\n axis(\n 'generalization',\n issues,\n gapScore(metrics.overfitGap, thresholds.maxOverfitGap),\n `holdoutRuns=${metrics.holdoutRuns} overfitGap=${fmt(metrics.overfitGap)}`,\n ),\n axis(\n 'diagnostics',\n issues,\n metrics.failedRows === 0 ? 1 : metrics.failuresWithAsi / metrics.failedRows,\n `failuresWithAsi=${metrics.failuresWithAsi}/${metrics.failedRows}`,\n ),\n axis(\n 'efficiency',\n issues,\n efficiencyScore(metrics, thresholds),\n `meanCostUsd=${fmt(metrics.meanCostUsd)} p95WallMs=${fmt(metrics.p95WallMs)}`,\n ),\n ]\n}\n\nfunction axis(\n name: ReleaseConfidenceAxisName,\n issues: ReleaseConfidenceIssue[],\n score: number | null,\n detail: string,\n): ReleaseConfidenceAxis {\n const own = issues.filter((i) => i.axis === name)\n const status = own.some((i) => i.severity === 'critical')\n ? 'fail'\n : own.length > 0\n ? 'warn'\n : 'pass'\n return { name, status, score: score === null ? null : bounded(score), detail }\n}\n\nfunction countScenarioSplits(scenarios: readonly DatasetScenario[]): Record<DatasetSplit, number> {\n const counts: Record<DatasetSplit, number> = { train: 0, dev: 0, test: 0, holdout: 0 }\n for (const scenario of scenarios) counts[scenario.split ?? 'train']++\n return counts\n}\n\nfunction countDomains(scenarios: readonly DatasetScenario[]): Record<string, number> {\n const out: Record<string, number> = {}\n for (const scenario of scenarios) {\n const domain = scenario.tags?.domain ?? scenario.tags?.category ?? 'uncategorized'\n out[domain] = (out[domain] ?? 0) + 1\n }\n return out\n}\n\nfunction countFailureClasses(\n runs: readonly RunRecord[],\n traces: readonly ReleaseTraceEvidence[],\n threshold: number,\n): Partial<Record<FailureClass, number>> {\n const out: Partial<Record<FailureClass, number>> = {}\n for (const run of runs) {\n if (runPassOutcome(run, threshold) === false) {\n const failureClass =\n run.failureClass !== undefined && run.failureClass !== 'success'\n ? run.failureClass\n : 'unknown'\n out[failureClass] = (out[failureClass] ?? 0) + 1\n }\n }\n for (const trace of traces) {\n if (tracePassOutcome(trace, threshold) === false) {\n const failureClass =\n trace.failureClass !== undefined && trace.failureClass !== 'success'\n ? trace.failureClass\n : 'unknown'\n out[failureClass] = (out[failureClass] ?? 0) + 1\n }\n }\n return out\n}\n\nfunction countResponsibleSurfaces(traces: readonly ReleaseTraceEvidence[]): Record<string, number> {\n const out: Record<string, number> = {}\n for (const trace of traces) {\n for (const asi of trace.asi ?? []) {\n const surface = asi.responsibleSurface ?? 'unknown'\n out[surface] = (out[surface] ?? 0) + 1\n }\n }\n return out\n}\n\nfunction failedRows(\n runs: readonly RunRecord[],\n traces: readonly ReleaseTraceEvidence[],\n threshold: number,\n): Array<{ hasAsi: boolean }> {\n const out: Array<{ hasAsi: boolean }> = []\n for (const run of runs) {\n if (runPassOutcome(run, threshold) === false) {\n const asiMetric = run.outcome.raw.asi\n out.push({ hasAsi: typeof asiMetric === 'number' && asiMetric > 0 })\n }\n }\n for (const trace of traces) {\n if (tracePassOutcome(trace, threshold) === false) {\n out.push({ hasAsi: (trace.asi?.length ?? 0) > 0 })\n }\n }\n return out\n}\n\nfunction passOutcomeRate(outcomes: readonly (boolean | null)[]): number | null {\n const classified = outcomes.filter((outcome): outcome is boolean => outcome !== null)\n if (classified.length === 0) return null\n return classified.filter(Boolean).length / classified.length\n}\n\nfunction runPassOutcome(run: RunRecord, threshold: number): boolean | null {\n if (hasExplicitTaskFailure(run)) return false\n const score = runSplitScore(run)\n return score === undefined ? null : score >= threshold\n}\n\nfunction hasExplicitTaskFailure(run: RunRecord): boolean {\n return run.failureClass !== undefined && run.failureClass !== 'success'\n}\n\nfunction tracePassOutcome(trace: ReleaseTraceEvidence, threshold: number): boolean | null {\n if (trace.failureClass !== undefined && trace.failureClass !== 'success') return false\n if (trace.ok === false) return false\n if (isFiniteNumber(trace.score)) return trace.score >= threshold\n return trace.ok === true ? true : null\n}\n\nfunction isFailedTerminalOutcome(\n outcome: RunRecord['terminalOutcome'],\n): outcome is 'failed' | 'cancelled' | 'incomplete' {\n return outcome === 'failed' || outcome === 'cancelled' || outcome === 'incomplete'\n}\n\nfunction terminalSuccess(outcome: RunRecord['terminalOutcome']): boolean | undefined {\n if (outcome === 'succeeded') return true\n if (isFailedTerminalOutcome(outcome)) return false\n return undefined\n}\n\nfunction scoresFor(runs: readonly RunRecord[], split: RunSplitTag): number[] {\n return runs\n .filter((run) => run.splitTag === split)\n .map(runSplitScore)\n .filter(isFiniteNumber)\n}\n\nfunction runSplitScore(run: RunRecord): number | undefined {\n const score = run.splitTag === 'holdout' ? run.outcome.holdoutScore : run.outcome.searchScore\n return isFiniteNumber(score) ? score : undefined\n}\n\nfunction meanOrNull(xs: readonly number[]): number | null {\n if (xs.length === 0) return null\n return xs.reduce((sum, x) => sum + x, 0) / xs.length\n}\n\nfunction percentileOrNull(xs: readonly number[], p: number): number | null {\n if (xs.length === 0) return null\n const sorted = [...xs].sort((a, b) => a - b)\n return sorted[Math.min(sorted.length - 1, Math.max(0, Math.ceil(p * sorted.length) - 1))]!\n}\n\nfunction isFiniteNumber(value: unknown): value is number {\n return typeof value === 'number' && Number.isFinite(value)\n}\n\nfunction diffOrNull(a: number | null, b: number | null): number | null {\n if (a === null || b === null) return null\n return a - b\n}\n\nfunction gapScore(gap: number | null, maxGap: number): number | null {\n if (gap === null) return null\n if (maxGap <= 0) return gap <= 0 ? 1 : 0\n return bounded(1 - Math.max(0, gap) / maxGap)\n}\n\nfunction efficiencyScore(\n metrics: ReleaseConfidenceMetrics,\n thresholds: Required<ReleaseConfidenceThresholds>,\n): number | null {\n const cost = Number.isFinite(thresholds.maxMeanCostUsd)\n ? metrics.meanCostUsd === null\n ? null\n : bounded(thresholds.maxMeanCostUsd / Math.max(metrics.meanCostUsd, 1e-12))\n : 1\n const latency = Number.isFinite(thresholds.maxP95WallMs)\n ? metrics.p95WallMs === null\n ? null\n : bounded(thresholds.maxP95WallMs / Math.max(metrics.p95WallMs, 1e-12))\n : 1\n if (cost === null || latency === null) return null\n return Math.min(cost, latency)\n}\n\nfunction bounded(x: number): number {\n if (!Number.isFinite(x)) return 0\n return Math.max(0, Math.min(1, x))\n}\n\nfunction renderSummary(\n target: string,\n status: ReleaseConfidenceStatus,\n metrics: ReleaseConfidenceMetrics,\n issues: ReleaseConfidenceIssue[],\n): string {\n const prefix = `release confidence ${status}: ${target}`\n const metricText = `scenarios=${metrics.scenarioCount} searchRuns=${metrics.searchRuns} holdoutRuns=${metrics.holdoutRuns} passRate=${fmt(metrics.passRate)} meanScore=${fmt(metrics.meanScore)}`\n if (issues.length === 0) return `${prefix}; ${metricText}`\n return `${prefix}; ${metricText}; issues=${issues.map((i) => i.code).join(',')}`\n}\n\nfunction fmt(x: number | null): string {\n if (x === null) return 'n/a'\n return x.toFixed(4)\n}\n","/**\n * Bootstrap-CI promotion gate.\n *\n * In any iterative-improvement loop (GEPA, prompt evolution, dataset\n * curation), the question is \"did this generation actually improve, or are\n * we celebrating noise?\". With small N and noisy outcomes, point-estimate\n * deltas lie. Bootstrap confidence intervals tell the operator whether the\n * delta is real before code or prompts get promoted.\n *\n * This module is pure functions — no I/O, no model calls. Easy to unit-test\n * and to compose into any verdict gate.\n *\n * Default gate:\n * - Bootstrap mean baseline vs candidate (1k resamples).\n * - Compute the delta distribution; pass if the lower CI bound > 0.\n * - Tunable confidence (default 95%) and resample count.\n *\n * Verdict semantics intentionally match the existing `experiments.jsonl`\n * vocabulary:\n * - ADVANCE: candidate's CI lower bound > baseline mean (real win)\n * - KEEP: overlap, but candidate point estimate >= baseline (neutral)\n * - REVERT: candidate's CI upper bound < baseline mean (real regression)\n * - INCONCLUSIVE: not enough samples or CI straddles zero with no signal\n */\n\nexport type Verdict = 'ADVANCE' | 'KEEP' | 'REVERT' | 'INCONCLUSIVE'\n\nexport interface BootstrapResult {\n baselineMean: number\n candidateMean: number\n /** candidateMean - baselineMean, point estimate. */\n delta: number\n /** Lower bound of the (1 - alpha) CI on the delta. */\n ciLower: number\n /** Upper bound of the (1 - alpha) CI on the delta. */\n ciUpper: number\n /** Number of bootstrap resamples used. */\n iterations: number\n alpha: number\n verdict: Verdict\n}\n\nexport interface BootstrapOptions {\n /** Confidence level alpha (default 0.05 → 95% CI). */\n alpha?: number\n /** Number of resamples (default 1000). */\n iterations?: number\n /**\n * Minimum total samples (baseline + candidate) below which we always\n * return INCONCLUSIVE — bootstrap with too few samples is meaningless.\n * Default 6 (combined).\n */\n minTotalSamples?: number\n /** RNG seed for reproducibility. Default: Math.random. */\n seed?: number\n}\n\n/**\n * Compute the bootstrap CI on (candidateMean - baselineMean) and a verdict.\n *\n * Uses simple percentile bootstrap on the difference of resampled means.\n * That's the standard non-parametric primitive — no distributional\n * assumptions, robust to skew, easy to reason about.\n */\nexport function bootstrapCi(\n baseline: number[],\n candidate: number[],\n options: BootstrapOptions = {},\n): BootstrapResult {\n const alpha = options.alpha ?? 0.05\n const iterations = options.iterations ?? 1000\n const minTotal = options.minTotalSamples ?? 6\n const rng = mulberry32(options.seed ?? hashSeed(baseline, candidate))\n\n const baselineMean = mean(baseline)\n const candidateMean = mean(candidate)\n const delta = candidateMean - baselineMean\n\n if (\n baseline.length + candidate.length < minTotal ||\n baseline.length === 0 ||\n candidate.length === 0\n ) {\n return {\n baselineMean,\n candidateMean,\n delta,\n ciLower: -Infinity,\n ciUpper: Infinity,\n iterations: 0,\n alpha,\n verdict: 'INCONCLUSIVE',\n }\n }\n\n const deltas: number[] = new Array(iterations)\n for (let i = 0; i < iterations; i++) {\n const bResample = resample(baseline, rng)\n const cResample = resample(candidate, rng)\n deltas[i] = mean(cResample) - mean(bResample)\n }\n deltas.sort((a, b) => a - b)\n const lowerIdx = Math.floor((alpha / 2) * iterations)\n const upperIdx = Math.floor((1 - alpha / 2) * iterations) - 1\n const ciLower = deltas[Math.max(0, lowerIdx)]!\n const ciUpper = deltas[Math.min(iterations - 1, upperIdx)]!\n\n let verdict: Verdict\n if (ciLower > 0) verdict = 'ADVANCE'\n else if (ciUpper < 0) verdict = 'REVERT'\n else if (delta >= 0) verdict = 'KEEP'\n else verdict = 'INCONCLUSIVE'\n\n return {\n baselineMean,\n candidateMean,\n delta,\n ciLower,\n ciUpper,\n iterations,\n alpha,\n verdict,\n }\n}\n\nfunction mean(xs: number[]): number {\n if (xs.length === 0) return 0\n let s = 0\n for (const x of xs) s += x\n return s / xs.length\n}\n\nfunction resample(xs: number[], rng: () => number): number[] {\n const out = new Array(xs.length)\n for (let i = 0; i < xs.length; i++) out[i] = xs[Math.floor(rng() * xs.length)]\n return out\n}\n\n/** Mulberry32 — fast deterministic PRNG. Stable across runs given the same seed. */\nfunction mulberry32(seed: number): () => number {\n let t = seed >>> 0\n return () => {\n t += 0x6d2b79f5\n let r = t\n r = Math.imul(r ^ (r >>> 15), r | 1)\n r ^= r + Math.imul(r ^ (r >>> 7), r | 61)\n return ((r ^ (r >>> 14)) >>> 0) / 4294967296\n }\n}\n\n/** Stable seed derived from the inputs — same data → same CI bounds. */\nfunction hashSeed(a: number[], b: number[]): number {\n let h = 2166136261\n for (const x of [...a, ...b]) {\n const view = new Float64Array([x])\n const bytes = new Uint8Array(view.buffer)\n for (const byte of bytes) {\n h ^= byte\n h = Math.imul(h, 16777619)\n }\n }\n return h >>> 0\n}\n\n/**\n * Judge-replay promotion gate.\n *\n * The cheap inner-loop judge that drives an evolution run is by definition\n * fast and noisy. When you're about to promote a winning variant to the\n * canonical default, you want a STRONGER judge (a more expensive model, a\n * human grader, a separately-trained reward model) to confirm the win\n * generalises beyond the inner loop.\n *\n * This helper takes raw winner + baseline outputs, scores both through the\n * stronger judge, and applies `bootstrapCi`. ADVANCE means the stronger\n * judge agrees the winner is real with the configured confidence. Doesn't\n * matter what shape your \"output\" is — pass a string, an object, anything\n * the judge can read.\n */\nexport interface JudgeReplayGateArgs<TOutput> {\n baselineOutputs: TOutput[]\n candidateOutputs: TOutput[]\n /** Stronger judge — async to allow LLM calls. Return a 0..N scalar score. */\n judge: (output: TOutput) => Promise<number> | number\n alpha?: number\n iterations?: number\n /** RNG seed for reproducibility. */\n seed?: number\n /** Maximum concurrent judge calls. Default 4. */\n judgeConcurrency?: number\n}\n\n/**\n * Confirm a candidate's win with a stronger judge: score baseline and candidate outputs independently, then bootstrap a CI to verify the lift generalises beyond the inner loop.\n */\nexport async function judgeReplayGate<TOutput>(\n args: JudgeReplayGateArgs<TOutput>,\n): Promise<BootstrapResult & { baselineSamples: number; candidateSamples: number }> {\n const concurrency = args.judgeConcurrency ?? 4\n const baselineScores = await scoreAll(args.baselineOutputs, args.judge, concurrency)\n const candidateScores = await scoreAll(args.candidateOutputs, args.judge, concurrency)\n const ci = bootstrapCi(baselineScores, candidateScores, {\n ...(args.alpha !== undefined ? { alpha: args.alpha } : {}),\n ...(args.iterations !== undefined ? { iterations: args.iterations } : {}),\n ...(args.seed !== undefined ? { seed: args.seed } : {}),\n })\n return {\n ...ci,\n baselineSamples: baselineScores.length,\n candidateSamples: candidateScores.length,\n }\n}\n\nasync function scoreAll<TOutput>(\n outputs: TOutput[],\n judge: (output: TOutput) => Promise<number> | number,\n concurrency: number,\n): Promise<number[]> {\n const results: number[] = new Array(outputs.length)\n let next = 0\n async function worker(): Promise<void> {\n while (true) {\n const i = next++\n if (i >= outputs.length) return\n const v = await judge(outputs[i]!)\n results[i] = Number.isFinite(v) ? v : 0\n }\n }\n await Promise.all(Array.from({ length: Math.max(1, concurrency) }, () => worker()))\n return results\n}\n","import type { ReleaseConfidenceScorecard } from './release-confidence'\nimport type { RunRecord } from './run-record'\nimport { summaryTable } from './summary-report'\n\nexport interface RenderReleaseReportOptions {\n title?: string\n runs?: readonly RunRecord[]\n comparator?: string\n traceAnalystFindings?: readonly string[]\n nextActions?: readonly string[]\n}\n\nexport function renderReleaseReport(\n scorecard: ReleaseConfidenceScorecard,\n options: RenderReleaseReportOptions = {},\n): string {\n const title = options.title ?? `Release Report: ${scorecard.target}`\n const lines: string[] = []\n lines.push(`# ${title}`)\n lines.push('')\n lines.push(`Status: **${scorecard.status.toUpperCase()}**`)\n lines.push(`Promote: **${scorecard.promote ? 'yes' : 'no'}**`)\n if (scorecard.candidateId) lines.push(`Candidate: \\`${scorecard.candidateId}\\``)\n if (scorecard.baselineId) lines.push(`Baseline: \\`${scorecard.baselineId}\\``)\n lines.push('')\n lines.push(scorecard.summary)\n lines.push('')\n\n lines.push('## Metrics')\n lines.push('')\n lines.push('| Metric | Value |')\n lines.push('|---|---:|')\n lines.push(`| Scenarios | ${scorecard.metrics.scenarioCount} |`)\n lines.push(`| Search runs | ${scorecard.metrics.searchRuns} |`)\n lines.push(`| Holdout runs | ${scorecard.metrics.holdoutRuns} |`)\n lines.push(`| Unscored runs | ${scorecard.metrics.unscoredRuns} |`)\n lines.push(`| Terminal failures | ${scorecard.metrics.terminalFailureRuns} |`)\n lines.push(`| Pass rate | ${pct(scorecard.metrics.passRate)} |`)\n lines.push(`| Mean score | ${num(scorecard.metrics.meanScore)} |`)\n lines.push(`| Search mean | ${num(scorecard.metrics.searchMeanScore)} |`)\n lines.push(`| Holdout mean | ${num(scorecard.metrics.holdoutMeanScore)} |`)\n lines.push(`| Overfit gap | ${num(scorecard.metrics.overfitGap)} |`)\n lines.push(`| Mean cost | $${num(scorecard.metrics.meanCostUsd)} |`)\n lines.push(`| p95 wall time | ${duration(scorecard.metrics.p95WallMs)} |`)\n lines.push('')\n\n if (scorecard.issues.length > 0) {\n lines.push('## Issues')\n lines.push('')\n for (const issue of scorecard.issues) {\n lines.push(`- **${issue.severity}** \\`${issue.code}\\` (${issue.axis}): ${issue.detail}`)\n }\n lines.push('')\n }\n\n const surfaces = entries(scorecard.metrics.responsibleSurfaceCounts)\n if (surfaces.length > 0) {\n lines.push('## Responsible Surfaces')\n lines.push('')\n for (const [surface, count] of surfaces) lines.push(`- ${surface}: ${count}`)\n lines.push('')\n }\n\n const failures = entries(scorecard.metrics.failureClassCounts)\n if (failures.length > 0) {\n lines.push('## Failure Classes')\n lines.push('')\n for (const [mode, count] of failures) lines.push(`- ${mode}: ${count}`)\n lines.push('')\n }\n\n if (options.runs && options.runs.length > 0) {\n lines.push('## Run Summary')\n lines.push('')\n lines.push(\n summaryTable([...options.runs], {\n comparator: options.comparator ?? scorecard.baselineId ?? undefined,\n split: 'holdout',\n }).markdown,\n )\n lines.push('')\n }\n\n if (options.traceAnalystFindings && options.traceAnalystFindings.length > 0) {\n lines.push('## TraceAnalyst Findings')\n lines.push('')\n for (const finding of options.traceAnalystFindings) lines.push(`- ${finding}`)\n lines.push('')\n }\n\n const nextActions = options.nextActions ?? defaultNextActions(scorecard)\n if (nextActions.length > 0) {\n lines.push('## Next Actions')\n lines.push('')\n for (const action of nextActions) lines.push(`- ${action}`)\n lines.push('')\n }\n\n return `${lines.join('\\n').trimEnd()}\\n`\n}\n\nfunction defaultNextActions(scorecard: ReleaseConfidenceScorecard): string[] {\n if (scorecard.promote) return ['Promote the candidate and keep canaries enabled.']\n return scorecard.issues\n .filter((issue) => issue.severity === 'critical')\n .map((issue) => `Resolve ${issue.code}: ${issue.detail}`)\n}\n\nfunction entries(values: Record<string, number>): Array<[string, number]> {\n return Object.entries(values)\n .filter(([, count]) => count > 0)\n .sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))\n}\n\nfunction pct(value: number | null): string {\n return value === null ? 'n/a' : `${(value * 100).toFixed(1)}%`\n}\n\nfunction num(value: number | null): string {\n return value === null ? 'n/a' : value.toFixed(3)\n}\n\nfunction duration(value: number | null): string {\n return value === null ? 'n/a' : `${Math.round(value)} ms`\n}\n"],"mappings":";;;;;;;;;;;;;;;AA4JA,IAAM,qBAA4D;AAAA,EAChE,eAAe;AAAA,EACf,kBAAkB;AAAA,EAClB,eAAe;AAAA,EACf,gBAAgB;AAAA,EAChB,gBAAgB;AAAA,EAChB,aAAa;AAAA,EACb,cAAc;AAAA,EACd,eAAe;AAAA,EACf,gBAAgB,OAAO;AAAA,EACvB,cAAc,OAAO;AAAA,EACrB,uBAAuB;AAAA,EACvB,uBAAuB;AACzB;AAEO,SAAS,0BACd,OAC4B;AAC5B,QAAM,aAAa,EAAE,GAAG,oBAAoB,GAAG,MAAM,WAAW;AAChE,QAAM,cAAc,MAAM,eAAe;AACzC,QAAM,OAAO;AAAA,KACV,MAAM,QAAQ,CAAC,GAAG,IAAI,iBAAiB;AAAA,IACxC;AAAA,IACA,MAAM;AAAA,EACR;AACA,QAAM,SAAS;AAAA,KACZ,MAAM,UAAU,CAAC,GAAG,IAAI,4BAA4B;AAAA,IACrD;AAAA,IACA,MAAM;AAAA,EACR;AACA,QAAM,YAAY,MAAM,aAAa,CAAC;AACtC,QAAM,gBAAgB,MAAM,SAAS,iBAAiB,UAAU;AAChE,QAAM,cAAc,MAAM,SAAS,eAAe,oBAAoB,SAAS;AAC/E,QAAM,eAAe,UAAU,MAAM,QAAQ;AAC7C,QAAM,gBAAgB,UAAU,MAAM,SAAS;AAC/C,QAAM,YAAY,KAAK,IAAI,aAAa,EAAE,OAAO,cAAc;AAC/D,QAAM,cAAc,OAAO,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE,OAAO,cAAc;AACpE,QAAM,gBAAgB,KAAK,SAAS,IAAI,YAAY;AACpD,QAAM,cAAc,KAAK,OAAO,CAAC,QAAQ,cAAc,GAAG,MAAM,MAAS;AACzE,QAAM,eAAe,KAAK;AAAA,IACxB,CAAC,QACC,cAAc,GAAG,MAAM,UACvB,CAAC,uBAAuB,GAAG,KAC3B,CAAC,wBAAwB,IAAI,eAAe;AAAA,EAChD,EAAE;AACF,QAAM,eACJ,KAAK,SAAS,IACV,KAAK,IAAI,CAAC,QAAQ,eAAe,KAAK,WAAW,qBAAqB,CAAC,IACvE,OAAO,IAAI,CAAC,UAAU,iBAAiB,OAAO,WAAW,qBAAqB,CAAC;AACrF,QAAM,kBACJ,KAAK,SAAS,IACV,KAAK,IAAI,CAAC,QAAQ,gBAAgB,IAAI,eAAe,CAAC,IACtD,OAAO,IAAI,CAAC,UAAU,MAAM,EAAE;AACpC,QAAM,2BAA2B,gBAAgB,OAAO,CAAC,YAAY,YAAY,MAAS,EAAE;AAC5F,QAAM,sBAAsB,gBAAgB,OAAO,CAAC,YAAY,YAAY,KAAK,EAAE;AACnF,QAAM,kBACJ,gBAAgB,WAAW,KAAK,2BAA2B,IACvD,QACC,gBAAgB,SAAS,uBAAuB,gBAAgB;AACvE,QAAM,aAAa,YAAY,OAAO,CAAC,MAAM,EAAE,aAAa,QAAQ,EAAE;AACtE,QAAM,cAAc,YAAY,OAAO,CAAC,MAAM,EAAE,aAAa,SAAS,EAAE;AACxE,QAAM,SAAS,WAAW,MAAM,QAAQ,WAAW,qBAAqB;AACxE,QAAM,kBAAkB,WAAW,YAAY;AAC/C,QAAM,mBAAmB,WAAW,aAAa;AACjD,QAAM,WAAW,KAAK;AAAA,IAAQ,CAAC,QAC7B,IAAI,eAAe,SAAS,eAAe,CAAC,IAAI,CAAC,IAAI,eAAe,GAAG;AAAA,EACzE;AACA,QAAM,aAAa,OAAO,IAAI,CAAC,UAAU,MAAM,OAAO,EAAE,OAAO,cAAc;AAC7E,QAAM,cACJ,KAAK,SAAS,IACV,SAAS,WAAW,KAAK,SACvB,WAAW,QAAQ,IACnB,OACF,WAAW,UAAU;AAC3B,QAAM,YACJ,KAAK,SAAS,IACV,KAAK,IAAI,CAAC,QAAQ,IAAI,MAAM,IAC5B,OAAO,IAAI,CAAC,UAAU,MAAM,UAAU,EAAE,OAAO,cAAc;AACnE,QAAM,UAAoC;AAAA,IACxC;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,UAAU,gBAAgB,YAAY;AAAA,IACtC,WAAW,WAAW,aAAa;AAAA,IACnC;AAAA,IACA;AAAA,IACA,YAAY,WAAW,iBAAiB,gBAAgB;AAAA,IACxD;AAAA,IACA,WAAW,iBAAiB,WAAW,IAAI;AAAA,IAC3C,YAAY,OAAO;AAAA,IACnB,iBAAiB,OAAO,OAAO,CAAC,QAAQ,IAAI,MAAM,EAAE;AAAA,IACpD,kBAAkB,OAAO,OAAO,CAAC,MAAM,EAAE,cAAc,CAAC,EAAE;AAAA,IAC1D,iBAAiB,OAAO,OAAO,CAAC,OAAO,EAAE,aAAa,KAAK,CAAC,EAAE;AAAA,IAC9D;AAAA,IACA,cAAc,aAAa,SAAS;AAAA,IACpC,oBAAoB,oBAAoB,MAAM,QAAQ,WAAW,qBAAqB;AAAA,IACtF,0BAA0B,yBAAyB,MAAM;AAAA,EAC3D;AAEA,QAAM,SAAmC,CAAC;AAC1C,cAAY,OAAO,YAAY,SAAS,MAAM;AAC9C,eAAa,YAAY,SAAS,MAAM;AACxC,mBAAiB,SAAS,MAAM;AAChC,sBAAoB,MAAM,gBAAgB,MAAM,YAAY,SAAS,MAAM;AAC3E,mBAAiB,YAAY,SAAS,MAAM;AAC5C,kBAAgB,YAAY,SAAS,MAAM;AAE3C,QAAM,OAAO,UAAU,SAAS,YAAY,MAAM;AAClD,QAAM,SAAS,OAAO,KAAK,CAAC,MAAM,EAAE,aAAa,UAAU,IACvD,SACA,OAAO,SAAS,IACd,SACA;AAEN,SAAO;AAAA,IACL,QAAQ,MAAM;AAAA,IACd;AAAA,IACA,YAAY,MAAM,cAAc;AAAA,IAChC;AAAA,IACA,SAAS,WAAW,WAAW,MAAM,eAAe,MAAM,aAAa,UAAU;AAAA,IACjF;AAAA,IACA;AAAA,IACA;AAAA,IACA,SAAS,MAAM,WAAW;AAAA,IAC1B,cAAc,MAAM,gBAAgB;AAAA,IACpC,SAAS,cAAc,MAAM,QAAQ,QAAQ,SAAS,MAAM;AAAA,EAC9D;AACF;AAEO,SAAS,wBAAwB,OAA2D;AACjG,QAAM,YAAY,0BAA0B,KAAK;AACjD,MAAI,UAAU,WAAW,QAAQ;AAC/B,UAAM,IAAI,kBAAkB,UAAU,OAAO;AAAA,EAC/C;AACA,SAAO;AACT;AAEA,SAAS,gBACP,MACA,aACA,YACa;AACb,MAAI,YAAa,QAAO,KAAK,OAAO,CAAC,MAAM,EAAE,gBAAgB,WAAW;AACxE,MAAI,WAAY,QAAO,KAAK,OAAO,CAAC,MAAM,EAAE,gBAAgB,UAAU;AACtE,SAAO,CAAC,GAAG,IAAI;AACjB;AAEA,SAAS,qBACP,QACA,aACA,YACwB;AACxB,MAAI;AACF,WAAO,OAAO,OAAO,CAAC,MAAM,EAAE,gBAAgB,UAAa,EAAE,gBAAgB,WAAW;AAC1F,MAAI;AACF,WAAO,OAAO,OAAO,CAAC,MAAM,EAAE,gBAAgB,UAAa,EAAE,gBAAgB,UAAU;AACzF,SAAO,CAAC,GAAG,MAAM;AACnB;AAEA,SAAS,6BACP,OACA,OACsB;AACtB,QAAM,QAAQ;AACd,MAAI,OAAO,OAAO,OAAO,aAAa,GAAG;AACvC,UAAM,IAAI;AAAA,MACR,UAAU,KAAK;AAAA,IACjB;AAAA,EACF;AACA,MAAI,MAAM,iBAAiB,UAAa,CAAC,gBAAgB,SAAS,MAAM,YAAY,GAAG;AACrF,UAAM,IAAI;AAAA,MACR,UAAU,KAAK,iCAAiC,gBAAgB,KAAK,IAAI,CAAC;AAAA,IAC5E;AAAA,EACF;AACA,SAAO;AACT;AAEA,SAAS,YACP,OACA,YACA,SACA,QACM;AACN,MAAI,WAAW,iBAAiB,CAAC,MAAM,YAAY,MAAM,WAAW,UAAU,OAAO,GAAG;AACtF,WAAO,KAAK;AAAA,MACV,MAAM;AAAA,MACN,UAAU;AAAA,MACV,MAAM;AAAA,MACN,QAAQ;AAAA,IACV,CAAC;AAAA,EACH;AACA,MAAI,QAAQ,gBAAgB,WAAW,kBAAkB;AACvD,WAAO,KAAK;AAAA,MACV,MAAM;AAAA,MACN,UAAU;AAAA,MACV,MAAM;AAAA,MACN,QAAQ,GAAG,QAAQ,aAAa,sBAAsB,WAAW,gBAAgB;AAAA,IACnF,CAAC;AAAA,EACH;AACA,MAAI,WAAW,kBAAkB,QAAQ,YAAY,YAAY,GAAG;AAClE,WAAO,KAAK;AAAA,MACV,MAAM;AAAA,MACN,UAAU;AAAA,MACV,MAAM;AAAA,MACN,QAAQ;AAAA,IACV,CAAC;AAAA,EACH;AACF;AAEA,SAAS,aACP,YACA,SACA,QACM;AACN,MAAI,QAAQ,aAAa,WAAW,eAAe;AACjD,WAAO,KAAK;AAAA,MACV,MAAM;AAAA,MACN,UAAU;AAAA,MACV,MAAM;AAAA,MACN,QAAQ,GAAG,QAAQ,UAAU,wBAAwB,WAAW,aAAa;AAAA,IAC/E,CAAC;AAAA,EACH;AACA,MAAI,QAAQ,eAAe,GAAG;AAC5B,WAAO,KAAK;AAAA,MACV,MAAM;AAAA,MACN,UAAU;AAAA,MACV,MAAM;AAAA,MACN,QAAQ,GAAG,QAAQ,YAAY;AAAA,IACjC,CAAC;AAAA,EACH;AACA,MAAI,QAAQ,aAAa,QAAQ,QAAQ,cAAc,MAAM;AAC3D,WAAO,KAAK;AAAA,MACV,MAAM;AAAA,MACN,UAAU;AAAA,MACV,MAAM;AAAA,MACN,QAAQ;AAAA,IACV,CAAC;AAAA,EACH;AACA,MAAI,QAAQ,aAAa,QAAQ,QAAQ,WAAW,WAAW,aAAa;AAC1E,WAAO,KAAK;AAAA,MACV,MAAM;AAAA,MACN,UAAU;AAAA,MACV,MAAM;AAAA,MACN,QAAQ,YAAY,IAAI,QAAQ,QAAQ,CAAC,MAAM,IAAI,WAAW,WAAW,CAAC;AAAA,IAC5E,CAAC;AAAA,EACH;AACA,MAAI,QAAQ,cAAc,QAAQ,QAAQ,YAAY,WAAW,cAAc;AAC7E,WAAO,KAAK;AAAA,MACV,MAAM;AAAA,MACN,UAAU;AAAA,MACV,MAAM;AAAA,MACN,QAAQ,aAAa,IAAI,QAAQ,SAAS,CAAC,MAAM,IAAI,WAAW,YAAY,CAAC;AAAA,IAC/E,CAAC;AAAA,EACH;AACF;AAEA,SAAS,iBACP,SACA,QACM;AACN,MAAI,QAAQ,oBAAoB,MAAM;AACpC,WAAO,KAAK;AAAA,MACV,MAAM;AAAA,MACN,UAAU;AAAA,MACV,MAAM;AAAA,MACN,QACE,QAAQ,2BAA2B,IAC/B,GAAG,QAAQ,wBAAwB,yDACnC;AAAA,IACR,CAAC;AAAA,EACH;AACA,MAAI,QAAQ,sBAAsB,GAAG;AACnC,WAAO,KAAK;AAAA,MACV,MAAM;AAAA,MACN,UAAU;AAAA,MACV,MAAM;AAAA,MACN,QAAQ,GAAG,QAAQ,mBAAmB;AAAA,IACxC,CAAC;AAAA,EACH;AACF;AAEA,SAAS,oBACP,cACA,YACA,SACA,QACM;AACN,MAAI,WAAW,kBAAkB,QAAQ,cAAc,WAAW,gBAAgB;AAChF,WAAO,KAAK;AAAA,MACV,MAAM;AAAA,MACN,UAAU;AAAA,MACV,MAAM;AAAA,MACN,QAAQ,GAAG,QAAQ,WAAW,yBAAyB,WAAW,cAAc;AAAA,IAClF,CAAC;AAAA,EACH;AACA,MAAI,QAAQ,eAAe,QAAQ,QAAQ,aAAa,WAAW,eAAe;AAChF,WAAO,KAAK;AAAA,MACV,MAAM;AAAA,MACN,UAAU;AAAA,MACV,MAAM;AAAA,MACN,QAAQ,sBAAsB,IAAI,QAAQ,UAAU,CAAC,MAAM,IAAI,WAAW,aAAa,CAAC;AAAA,IAC1F,CAAC;AAAA,EACH;AACA,MAAI,gBAAgB,CAAC,aAAa,SAAS;AACzC,WAAO,KAAK;AAAA,MACV,MAAM;AAAA,MACN,UAAU;AAAA,MACV,MAAM,QAAQ,aAAa,iBAAiB,QAAQ;AAAA,MACpD,QAAQ,aAAa;AAAA,IACvB,CAAC;AAAA,EACH;AACF;AAEA,SAAS,iBACP,YACA,SACA,QACM;AACN,MAAI,CAAC,WAAW,sBAAuB;AACvC,MAAI,QAAQ,aAAa,QAAQ,iBAAiB;AAChD,WAAO,KAAK;AAAA,MACV,MAAM;AAAA,MACN,UAAU;AAAA,MACV,MAAM;AAAA,MACN,QAAQ,GAAG,QAAQ,aAAa,QAAQ,eAAe;AAAA,IACzD,CAAC;AAAA,EACH;AACF;AAEA,SAAS,gBACP,YACA,SACA,QACM;AACN,MAAI,OAAO,SAAS,WAAW,cAAc,KAAK,QAAQ,gBAAgB,MAAM;AAC9E,WAAO,KAAK;AAAA,MACV,MAAM;AAAA,MACN,UAAU;AAAA,MACV,MAAM;AAAA,MACN,QAAQ;AAAA,IACV,CAAC;AAAA,EACH,WAAW,QAAQ,gBAAgB,QAAQ,QAAQ,cAAc,WAAW,gBAAgB;AAC1F,WAAO,KAAK;AAAA,MACV,MAAM;AAAA,MACN,UAAU;AAAA,MACV,MAAM;AAAA,MACN,QAAQ,eAAe,IAAI,QAAQ,WAAW,CAAC,MAAM,IAAI,WAAW,cAAc,CAAC;AAAA,IACrF,CAAC;AAAA,EACH;AACA,MAAI,OAAO,SAAS,WAAW,YAAY,KAAK,QAAQ,cAAc,MAAM;AAC1E,WAAO,KAAK;AAAA,MACV,MAAM;AAAA,MACN,UAAU;AAAA,MACV,MAAM;AAAA,MACN,QAAQ;AAAA,IACV,CAAC;AAAA,EACH,WAAW,QAAQ,cAAc,QAAQ,QAAQ,YAAY,WAAW,cAAc;AACpF,WAAO,KAAK;AAAA,MACV,MAAM;AAAA,MACN,UAAU;AAAA,MACV,MAAM;AAAA,MACN,QAAQ,aAAa,IAAI,QAAQ,SAAS,CAAC,MAAM,IAAI,WAAW,YAAY,CAAC;AAAA,IAC/E,CAAC;AAAA,EACH;AACF;AAEA,SAAS,UACP,SACA,YACA,QACyB;AACzB,SAAO;AAAA,IACL;AAAA,MACE;AAAA,MACA;AAAA,MACA,QAAQ,QAAQ,gBAAgB,KAAK,IAAI,GAAG,WAAW,gBAAgB,CAAC;AAAA,MACxE,GAAG,QAAQ,aAAa,uBAAuB,QAAQ,YAAY,OAAO;AAAA,IAC5E;AAAA,IACA;AAAA,MACE;AAAA,MACA;AAAA,MACA,QAAQ,aAAa,QAAQ,QAAQ,cAAc,OAC/C,OACA,KAAK,IAAI,QAAQ,UAAU,QAAQ,SAAS;AAAA,MAChD,YAAY,IAAI,QAAQ,QAAQ,CAAC,cAAc,IAAI,QAAQ,SAAS,CAAC;AAAA,IACvE;AAAA,IACA;AAAA,MACE;AAAA,MACA;AAAA,MACA,QAAQ;AAAA,MACR,eAAe,IAAI,QAAQ,eAAe,CAAC,qBAAqB,QAAQ,mBAAmB,iBAAiB,QAAQ,wBAAwB;AAAA,IAC9I;AAAA,IACA;AAAA,MACE;AAAA,MACA;AAAA,MACA,SAAS,QAAQ,YAAY,WAAW,aAAa;AAAA,MACrD,eAAe,QAAQ,WAAW,eAAe,IAAI,QAAQ,UAAU,CAAC;AAAA,IAC1E;AAAA,IACA;AAAA,MACE;AAAA,MACA;AAAA,MACA,QAAQ,eAAe,IAAI,IAAI,QAAQ,kBAAkB,QAAQ;AAAA,MACjE,mBAAmB,QAAQ,eAAe,IAAI,QAAQ,UAAU;AAAA,IAClE;AAAA,IACA;AAAA,MACE;AAAA,MACA;AAAA,MACA,gBAAgB,SAAS,UAAU;AAAA,MACnC,eAAe,IAAI,QAAQ,WAAW,CAAC,cAAc,IAAI,QAAQ,SAAS,CAAC;AAAA,IAC7E;AAAA,EACF;AACF;AAEA,SAAS,KACP,MACA,QACA,OACA,QACuB;AACvB,QAAM,MAAM,OAAO,OAAO,CAAC,MAAM,EAAE,SAAS,IAAI;AAChD,QAAM,SAAS,IAAI,KAAK,CAAC,MAAM,EAAE,aAAa,UAAU,IACpD,SACA,IAAI,SAAS,IACX,SACA;AACN,SAAO,EAAE,MAAM,QAAQ,OAAO,UAAU,OAAO,OAAO,QAAQ,KAAK,GAAG,OAAO;AAC/E;AAEA,SAAS,oBAAoB,WAAqE;AAChG,QAAM,SAAuC,EAAE,OAAO,GAAG,KAAK,GAAG,MAAM,GAAG,SAAS,EAAE;AACrF,aAAW,YAAY,UAAW,QAAO,SAAS,SAAS,OAAO;AAClE,SAAO;AACT;AAEA,SAAS,aAAa,WAA+D;AACnF,QAAM,MAA8B,CAAC;AACrC,aAAW,YAAY,WAAW;AAChC,UAAM,SAAS,SAAS,MAAM,UAAU,SAAS,MAAM,YAAY;AACnE,QAAI,MAAM,KAAK,IAAI,MAAM,KAAK,KAAK;AAAA,EACrC;AACA,SAAO;AACT;AAEA,SAAS,oBACP,MACA,QACA,WACuC;AACvC,QAAM,MAA6C,CAAC;AACpD,aAAW,OAAO,MAAM;AACtB,QAAI,eAAe,KAAK,SAAS,MAAM,OAAO;AAC5C,YAAM,eACJ,IAAI,iBAAiB,UAAa,IAAI,iBAAiB,YACnD,IAAI,eACJ;AACN,UAAI,YAAY,KAAK,IAAI,YAAY,KAAK,KAAK;AAAA,IACjD;AAAA,EACF;AACA,aAAW,SAAS,QAAQ;AAC1B,QAAI,iBAAiB,OAAO,SAAS,MAAM,OAAO;AAChD,YAAM,eACJ,MAAM,iBAAiB,UAAa,MAAM,iBAAiB,YACvD,MAAM,eACN;AACN,UAAI,YAAY,KAAK,IAAI,YAAY,KAAK,KAAK;AAAA,IACjD;AAAA,EACF;AACA,SAAO;AACT;AAEA,SAAS,yBAAyB,QAAiE;AACjG,QAAM,MAA8B,CAAC;AACrC,aAAW,SAAS,QAAQ;AAC1B,eAAW,OAAO,MAAM,OAAO,CAAC,GAAG;AACjC,YAAM,UAAU,IAAI,sBAAsB;AAC1C,UAAI,OAAO,KAAK,IAAI,OAAO,KAAK,KAAK;AAAA,IACvC;AAAA,EACF;AACA,SAAO;AACT;AAEA,SAAS,WACP,MACA,QACA,WAC4B;AAC5B,QAAM,MAAkC,CAAC;AACzC,aAAW,OAAO,MAAM;AACtB,QAAI,eAAe,KAAK,SAAS,MAAM,OAAO;AAC5C,YAAM,YAAY,IAAI,QAAQ,IAAI;AAClC,UAAI,KAAK,EAAE,QAAQ,OAAO,cAAc,YAAY,YAAY,EAAE,CAAC;AAAA,IACrE;AAAA,EACF;AACA,aAAW,SAAS,QAAQ;AAC1B,QAAI,iBAAiB,OAAO,SAAS,MAAM,OAAO;AAChD,UAAI,KAAK,EAAE,SAAS,MAAM,KAAK,UAAU,KAAK,EAAE,CAAC;AAAA,IACnD;AAAA,EACF;AACA,SAAO;AACT;AAEA,SAAS,gBAAgB,UAAsD;AAC7E,QAAM,aAAa,SAAS,OAAO,CAAC,YAAgC,YAAY,IAAI;AACpF,MAAI,WAAW,WAAW,EAAG,QAAO;AACpC,SAAO,WAAW,OAAO,OAAO,EAAE,SAAS,WAAW;AACxD;AAEA,SAAS,eAAe,KAAgB,WAAmC;AACzE,MAAI,uBAAuB,GAAG,EAAG,QAAO;AACxC,QAAM,QAAQ,cAAc,GAAG;AAC/B,SAAO,UAAU,SAAY,OAAO,SAAS;AAC/C;AAEA,SAAS,uBAAuB,KAAyB;AACvD,SAAO,IAAI,iBAAiB,UAAa,IAAI,iBAAiB;AAChE;AAEA,SAAS,iBAAiB,OAA6B,WAAmC;AACxF,MAAI,MAAM,iBAAiB,UAAa,MAAM,iBAAiB,UAAW,QAAO;AACjF,MAAI,MAAM,OAAO,MAAO,QAAO;AAC/B,MAAI,eAAe,MAAM,KAAK,EAAG,QAAO,MAAM,SAAS;AACvD,SAAO,MAAM,OAAO,OAAO,OAAO;AACpC;AAEA,SAAS,wBACP,SACkD;AAClD,SAAO,YAAY,YAAY,YAAY,eAAe,YAAY;AACxE;AAEA,SAAS,gBAAgB,SAA4D;AACnF,MAAI,YAAY,YAAa,QAAO;AACpC,MAAI,wBAAwB,OAAO,EAAG,QAAO;AAC7C,SAAO;AACT;AAEA,SAAS,UAAU,MAA4B,OAA8B;AAC3E,SAAO,KACJ,OAAO,CAAC,QAAQ,IAAI,aAAa,KAAK,EACtC,IAAI,aAAa,EACjB,OAAO,cAAc;AAC1B;AAEA,SAAS,cAAc,KAAoC;AACzD,QAAM,QAAQ,IAAI,aAAa,YAAY,IAAI,QAAQ,eAAe,IAAI,QAAQ;AAClF,SAAO,eAAe,KAAK,IAAI,QAAQ;AACzC;AAEA,SAAS,WAAW,IAAsC;AACxD,MAAI,GAAG,WAAW,EAAG,QAAO;AAC5B,SAAO,GAAG,OAAO,CAAC,KAAK,MAAM,MAAM,GAAG,CAAC,IAAI,GAAG;AAChD;AAEA,SAAS,iBAAiB,IAAuB,GAA0B;AACzE,MAAI,GAAG,WAAW,EAAG,QAAO;AAC5B,QAAM,SAAS,CAAC,GAAG,EAAE,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AAC3C,SAAO,OAAO,KAAK,IAAI,OAAO,SAAS,GAAG,KAAK,IAAI,GAAG,KAAK,KAAK,IAAI,OAAO,MAAM,IAAI,CAAC,CAAC,CAAC;AAC1F;AAEA,SAAS,eAAe,OAAiC;AACvD,SAAO,OAAO,UAAU,YAAY,OAAO,SAAS,KAAK;AAC3D;AAEA,SAAS,WAAW,GAAkB,GAAiC;AACrE,MAAI,MAAM,QAAQ,MAAM,KAAM,QAAO;AACrC,SAAO,IAAI;AACb;AAEA,SAAS,SAAS,KAAoB,QAA+B;AACnE,MAAI,QAAQ,KAAM,QAAO;AACzB,MAAI,UAAU,EAAG,QAAO,OAAO,IAAI,IAAI;AACvC,SAAO,QAAQ,IAAI,KAAK,IAAI,GAAG,GAAG,IAAI,MAAM;AAC9C;AAEA,SAAS,gBACP,SACA,YACe;AACf,QAAM,OAAO,OAAO,SAAS,WAAW,cAAc,IAClD,QAAQ,gBAAgB,OACtB,OACA,QAAQ,WAAW,iBAAiB,KAAK,IAAI,QAAQ,aAAa,KAAK,CAAC,IAC1E;AACJ,QAAM,UAAU,OAAO,SAAS,WAAW,YAAY,IACnD,QAAQ,cAAc,OACpB,OACA,QAAQ,WAAW,eAAe,KAAK,IAAI,QAAQ,WAAW,KAAK,CAAC,IACtE;AACJ,MAAI,SAAS,QAAQ,YAAY,KAAM,QAAO;AAC9C,SAAO,KAAK,IAAI,MAAM,OAAO;AAC/B;AAEA,SAAS,QAAQ,GAAmB;AAClC,MAAI,CAAC,OAAO,SAAS,CAAC,EAAG,QAAO;AAChC,SAAO,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,CAAC,CAAC;AACnC;AAEA,SAAS,cACP,QACA,QACA,SACA,QACQ;AACR,QAAM,SAAS,sBAAsB,MAAM,KAAK,MAAM;AACtD,QAAM,aAAa,aAAa,QAAQ,aAAa,eAAe,QAAQ,UAAU,gBAAgB,QAAQ,WAAW,aAAa,IAAI,QAAQ,QAAQ,CAAC,cAAc,IAAI,QAAQ,SAAS,CAAC;AAC/L,MAAI,OAAO,WAAW,EAAG,QAAO,GAAG,MAAM,KAAK,UAAU;AACxD,SAAO,GAAG,MAAM,KAAK,UAAU,YAAY,OAAO,IAAI,CAAC,MAAM,EAAE,IAAI,EAAE,KAAK,GAAG,CAAC;AAChF;AAEA,SAAS,IAAI,GAA0B;AACrC,MAAI,MAAM,KAAM,QAAO;AACvB,SAAO,EAAE,QAAQ,CAAC;AACpB;;;ACpsBO,SAAS,YACd,UACA,WACA,UAA4B,CAAC,GACZ;AACjB,QAAM,QAAQ,QAAQ,SAAS;AAC/B,QAAM,aAAa,QAAQ,cAAc;AACzC,QAAM,WAAW,QAAQ,mBAAmB;AAC5C,QAAM,MAAM,WAAW,QAAQ,QAAQ,SAAS,UAAU,SAAS,CAAC;AAEpE,QAAM,eAAe,KAAK,QAAQ;AAClC,QAAM,gBAAgB,KAAK,SAAS;AACpC,QAAM,QAAQ,gBAAgB;AAE9B,MACE,SAAS,SAAS,UAAU,SAAS,YACrC,SAAS,WAAW,KACpB,UAAU,WAAW,GACrB;AACA,WAAO;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,MACA,SAAS;AAAA,MACT,SAAS;AAAA,MACT,YAAY;AAAA,MACZ;AAAA,MACA,SAAS;AAAA,IACX;AAAA,EACF;AAEA,QAAM,SAAmB,IAAI,MAAM,UAAU;AAC7C,WAAS,IAAI,GAAG,IAAI,YAAY,KAAK;AACnC,UAAM,YAAY,SAAS,UAAU,GAAG;AACxC,UAAM,YAAY,SAAS,WAAW,GAAG;AACzC,WAAO,CAAC,IAAI,KAAK,SAAS,IAAI,KAAK,SAAS;AAAA,EAC9C;AACA,SAAO,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AAC3B,QAAM,WAAW,KAAK,MAAO,QAAQ,IAAK,UAAU;AACpD,QAAM,WAAW,KAAK,OAAO,IAAI,QAAQ,KAAK,UAAU,IAAI;AAC5D,QAAM,UAAU,OAAO,KAAK,IAAI,GAAG,QAAQ,CAAC;AAC5C,QAAM,UAAU,OAAO,KAAK,IAAI,aAAa,GAAG,QAAQ,CAAC;AAEzD,MAAI;AACJ,MAAI,UAAU,EAAG,WAAU;AAAA,WAClB,UAAU,EAAG,WAAU;AAAA,WACvB,SAAS,EAAG,WAAU;AAAA,MAC1B,WAAU;AAEf,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;AAEA,SAAS,KAAK,IAAsB;AAClC,MAAI,GAAG,WAAW,EAAG,QAAO;AAC5B,MAAI,IAAI;AACR,aAAW,KAAK,GAAI,MAAK;AACzB,SAAO,IAAI,GAAG;AAChB;AAEA,SAAS,SAAS,IAAc,KAA6B;AAC3D,QAAM,MAAM,IAAI,MAAM,GAAG,MAAM;AAC/B,WAAS,IAAI,GAAG,IAAI,GAAG,QAAQ,IAAK,KAAI,CAAC,IAAI,GAAG,KAAK,MAAM,IAAI,IAAI,GAAG,MAAM,CAAC;AAC7E,SAAO;AACT;AAGA,SAAS,WAAW,MAA4B;AAC9C,MAAI,IAAI,SAAS;AACjB,SAAO,MAAM;AACX,SAAK;AACL,QAAI,IAAI;AACR,QAAI,KAAK,KAAK,IAAK,MAAM,IAAK,IAAI,CAAC;AACnC,SAAK,IAAI,KAAK,KAAK,IAAK,MAAM,GAAI,IAAI,EAAE;AACxC,aAAS,IAAK,MAAM,QAAS,KAAK;AAAA,EACpC;AACF;AAGA,SAAS,SAAS,GAAa,GAAqB;AAClD,MAAI,IAAI;AACR,aAAW,KAAK,CAAC,GAAG,GAAG,GAAG,CAAC,GAAG;AAC5B,UAAM,OAAO,IAAI,aAAa,CAAC,CAAC,CAAC;AACjC,UAAM,QAAQ,IAAI,WAAW,KAAK,MAAM;AACxC,eAAW,QAAQ,OAAO;AACxB,WAAK;AACL,UAAI,KAAK,KAAK,GAAG,QAAQ;AAAA,IAC3B;AAAA,EACF;AACA,SAAO,MAAM;AACf;AAiCA,eAAsB,gBACpB,MACkF;AAClF,QAAM,cAAc,KAAK,oBAAoB;AAC7C,QAAM,iBAAiB,MAAM,SAAS,KAAK,iBAAiB,KAAK,OAAO,WAAW;AACnF,QAAM,kBAAkB,MAAM,SAAS,KAAK,kBAAkB,KAAK,OAAO,WAAW;AACrF,QAAM,KAAK,YAAY,gBAAgB,iBAAiB;AAAA,IACtD,GAAI,KAAK,UAAU,SAAY,EAAE,OAAO,KAAK,MAAM,IAAI,CAAC;AAAA,IACxD,GAAI,KAAK,eAAe,SAAY,EAAE,YAAY,KAAK,WAAW,IAAI,CAAC;AAAA,IACvE,GAAI,KAAK,SAAS,SAAY,EAAE,MAAM,KAAK,KAAK,IAAI,CAAC;AAAA,EACvD,CAAC;AACD,SAAO;AAAA,IACL,GAAG;AAAA,IACH,iBAAiB,eAAe;AAAA,IAChC,kBAAkB,gBAAgB;AAAA,EACpC;AACF;AAEA,eAAe,SACb,SACA,OACA,aACmB;AACnB,QAAM,UAAoB,IAAI,MAAM,QAAQ,MAAM;AAClD,MAAI,OAAO;AACX,iBAAe,SAAwB;AACrC,WAAO,MAAM;AACX,YAAM,IAAI;AACV,UAAI,KAAK,QAAQ,OAAQ;AACzB,YAAM,IAAI,MAAM,MAAM,QAAQ,CAAC,CAAE;AACjC,cAAQ,CAAC,IAAI,OAAO,SAAS,CAAC,IAAI,IAAI;AAAA,IACxC;AAAA,EACF;AACA,QAAM,QAAQ,IAAI,MAAM,KAAK,EAAE,QAAQ,KAAK,IAAI,GAAG,WAAW,EAAE,GAAG,MAAM,OAAO,CAAC,CAAC;AAClF,SAAO;AACT;;;AC1NO,SAAS,oBACd,WACA,UAAsC,CAAC,GAC/B;AACR,QAAM,QAAQ,QAAQ,SAAS,mBAAmB,UAAU,MAAM;AAClE,QAAM,QAAkB,CAAC;AACzB,QAAM,KAAK,KAAK,KAAK,EAAE;AACvB,QAAM,KAAK,EAAE;AACb,QAAM,KAAK,aAAa,UAAU,OAAO,YAAY,CAAC,IAAI;AAC1D,QAAM,KAAK,cAAc,UAAU,UAAU,QAAQ,IAAI,IAAI;AAC7D,MAAI,UAAU,YAAa,OAAM,KAAK,gBAAgB,UAAU,WAAW,IAAI;AAC/E,MAAI,UAAU,WAAY,OAAM,KAAK,eAAe,UAAU,UAAU,IAAI;AAC5E,QAAM,KAAK,EAAE;AACb,QAAM,KAAK,UAAU,OAAO;AAC5B,QAAM,KAAK,EAAE;AAEb,QAAM,KAAK,YAAY;AACvB,QAAM,KAAK,EAAE;AACb,QAAM,KAAK,oBAAoB;AAC/B,QAAM,KAAK,YAAY;AACvB,QAAM,KAAK,iBAAiB,UAAU,QAAQ,aAAa,IAAI;AAC/D,QAAM,KAAK,mBAAmB,UAAU,QAAQ,UAAU,IAAI;AAC9D,QAAM,KAAK,oBAAoB,UAAU,QAAQ,WAAW,IAAI;AAChE,QAAM,KAAK,qBAAqB,UAAU,QAAQ,YAAY,IAAI;AAClE,QAAM,KAAK,yBAAyB,UAAU,QAAQ,mBAAmB,IAAI;AAC7E,QAAM,KAAK,iBAAiB,IAAI,UAAU,QAAQ,QAAQ,CAAC,IAAI;AAC/D,QAAM,KAAK,kBAAkB,IAAI,UAAU,QAAQ,SAAS,CAAC,IAAI;AACjE,QAAM,KAAK,mBAAmB,IAAI,UAAU,QAAQ,eAAe,CAAC,IAAI;AACxE,QAAM,KAAK,oBAAoB,IAAI,UAAU,QAAQ,gBAAgB,CAAC,IAAI;AAC1E,QAAM,KAAK,mBAAmB,IAAI,UAAU,QAAQ,UAAU,CAAC,IAAI;AACnE,QAAM,KAAK,kBAAkB,IAAI,UAAU,QAAQ,WAAW,CAAC,IAAI;AACnE,QAAM,KAAK,qBAAqB,SAAS,UAAU,QAAQ,SAAS,CAAC,IAAI;AACzE,QAAM,KAAK,EAAE;AAEb,MAAI,UAAU,OAAO,SAAS,GAAG;AAC/B,UAAM,KAAK,WAAW;AACtB,UAAM,KAAK,EAAE;AACb,eAAW,SAAS,UAAU,QAAQ;AACpC,YAAM,KAAK,OAAO,MAAM,QAAQ,QAAQ,MAAM,IAAI,OAAO,MAAM,IAAI,MAAM,MAAM,MAAM,EAAE;AAAA,IACzF;AACA,UAAM,KAAK,EAAE;AAAA,EACf;AAEA,QAAM,WAAW,QAAQ,UAAU,QAAQ,wBAAwB;AACnE,MAAI,SAAS,SAAS,GAAG;AACvB,UAAM,KAAK,yBAAyB;AACpC,UAAM,KAAK,EAAE;AACb,eAAW,CAAC,SAAS,KAAK,KAAK,SAAU,OAAM,KAAK,KAAK,OAAO,KAAK,KAAK,EAAE;AAC5E,UAAM,KAAK,EAAE;AAAA,EACf;AAEA,QAAM,WAAW,QAAQ,UAAU,QAAQ,kBAAkB;AAC7D,MAAI,SAAS,SAAS,GAAG;AACvB,UAAM,KAAK,oBAAoB;AAC/B,UAAM,KAAK,EAAE;AACb,eAAW,CAAC,MAAM,KAAK,KAAK,SAAU,OAAM,KAAK,KAAK,IAAI,KAAK,KAAK,EAAE;AACtE,UAAM,KAAK,EAAE;AAAA,EACf;AAEA,MAAI,QAAQ,QAAQ,QAAQ,KAAK,SAAS,GAAG;AAC3C,UAAM,KAAK,gBAAgB;AAC3B,UAAM,KAAK,EAAE;AACb,UAAM;AAAA,MACJ,aAAa,CAAC,GAAG,QAAQ,IAAI,GAAG;AAAA,QAC9B,YAAY,QAAQ,cAAc,UAAU,cAAc;AAAA,QAC1D,OAAO;AAAA,MACT,CAAC,EAAE;AAAA,IACL;AACA,UAAM,KAAK,EAAE;AAAA,EACf;AAEA,MAAI,QAAQ,wBAAwB,QAAQ,qBAAqB,SAAS,GAAG;AAC3E,UAAM,KAAK,0BAA0B;AACrC,UAAM,KAAK,EAAE;AACb,eAAW,WAAW,QAAQ,qBAAsB,OAAM,KAAK,KAAK,OAAO,EAAE;AAC7E,UAAM,KAAK,EAAE;AAAA,EACf;AAEA,QAAM,cAAc,QAAQ,eAAe,mBAAmB,SAAS;AACvE,MAAI,YAAY,SAAS,GAAG;AAC1B,UAAM,KAAK,iBAAiB;AAC5B,UAAM,KAAK,EAAE;AACb,eAAW,UAAU,YAAa,OAAM,KAAK,KAAK,MAAM,EAAE;AAC1D,UAAM,KAAK,EAAE;AAAA,EACf;AAEA,SAAO,GAAG,MAAM,KAAK,IAAI,EAAE,QAAQ,CAAC;AAAA;AACtC;AAEA,SAAS,mBAAmB,WAAiD;AAC3E,MAAI,UAAU,QAAS,QAAO,CAAC,kDAAkD;AACjF,SAAO,UAAU,OACd,OAAO,CAAC,UAAU,MAAM,aAAa,UAAU,EAC/C,IAAI,CAAC,UAAU,WAAW,MAAM,IAAI,KAAK,MAAM,MAAM,EAAE;AAC5D;AAEA,SAAS,QAAQ,QAAyD;AACxE,SAAO,OAAO,QAAQ,MAAM,EACzB,OAAO,CAAC,CAAC,EAAE,KAAK,MAAM,QAAQ,CAAC,EAC/B,KAAK,CAAC,GAAG,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,CAAC,EAAE,cAAc,EAAE,CAAC,CAAC,CAAC;AAC3D;AAEA,SAAS,IAAI,OAA8B;AACzC,SAAO,UAAU,OAAO,QAAQ,IAAI,QAAQ,KAAK,QAAQ,CAAC,CAAC;AAC7D;AAEA,SAAS,IAAI,OAA8B;AACzC,SAAO,UAAU,OAAO,QAAQ,MAAM,QAAQ,CAAC;AACjD;AAEA,SAAS,SAAS,OAA8B;AAC9C,SAAO,UAAU,OAAO,QAAQ,GAAG,KAAK,MAAM,KAAK,CAAC;AACtD;","names":[]}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/benchmarks/index.ts","../src/benchmarks/calibration.ts","../src/benchmarks/routing/index.ts","../src/benchmarks/types.ts","../src/benchmarks/routing/dataset.ts","../src/benchmarks/runner.ts","../src/benchmarks/standard-formats.ts"],"sourcesContent":["/**\n * Reference benchmark wrappers — entry point.\n *\n * Core surface (exported here):\n * - The `BenchmarkAdapter` contract.\n * - `runBenchmarkAdapter` for campaign-backed benchmark execution.\n * - `calibrateBenchmarkMetric` for weak/strong metric checks.\n * - Standard retrieval parsers for BEIR/MTEB/MS MARCO/TREC/MIRACL-style files.\n * - `deterministicSplit` + `BENCHMARK_SPLIT_SEED` for split assignment.\n * - `routing` — synthetic 16-task router benchmark. The only novel\n * benchmark we built; ships in the package.\n *\n * Example wrappers (under `examples/benchmarks/`, NOT in the bundle):\n * - `gsm8k` — exact-match math reasoning (HF mirror, dataset\n * not bundled).\n * - `swebench-lite` — 30-instance SWE-Bench subset via an external\n * grader command.\n *\n * The example wrappers are reference implementations of `BenchmarkAdapter`.\n * Read them, copy them, adapt them. They're intentionally not in the main\n * entry — every team will configure them differently.\n */\n\nexport {\n type BenchmarkMetricCalibrationOptions,\n type BenchmarkMetricCalibrationResult,\n calibrateBenchmarkMetric,\n} from './calibration'\nexport * as routing from './routing/index'\nexport {\n type BenchmarkDistribution,\n type BenchmarkReport,\n type BenchmarkRunOptions,\n type BenchmarkRunResult,\n type BenchmarkSliceSummary,\n renderBenchmarkReportMarkdown,\n runBenchmarkAdapter,\n summarizeBenchmarkCampaign,\n} from './runner'\nexport {\n type BuildStandardRetrievalItemsOptions,\n buildStandardRetrievalItems,\n createRetrievalIdBenchmarkAdapter,\n evaluateStandardRetrieval,\n normalizeRetrievedDocumentIds,\n parseBeirCorpusJsonl,\n parseBeirQueriesJsonl,\n parseJsonlRows,\n parseQrels,\n parseTsvRows,\n type RetrievalIdAdapterOptions,\n retrievalMetricsAtCutoff,\n type StandardRetrievalArtifact,\n type StandardRetrievalDocument,\n type StandardRetrievalEvaluationOptions,\n type StandardRetrievalPayload,\n type StandardRetrievalQrel,\n type StandardRetrievalQuery,\n type StandardRetrievalResult,\n} from './standard-formats'\nexport type {\n BenchmarkAdapter,\n BenchmarkDatasetItem,\n BenchmarkEvaluation,\n BenchmarkFamily,\n BenchmarkResponder,\n BenchmarkScenario,\n BenchmarkSource,\n BenchmarkTaskKind,\n} from './types'\nexport { BENCHMARK_SPLIT_SEED, deterministicSplit } from './types'\n","import type { BenchmarkAdapter, BenchmarkDatasetItem, BenchmarkEvaluation } from './types'\n\nexport interface BenchmarkMetricCalibrationOptions<TPayload = unknown, TArtifact = string> {\n adapter: BenchmarkAdapter<BenchmarkDatasetItem<TPayload>, TPayload, TArtifact>\n item: BenchmarkDatasetItem<TPayload>\n weakArtifact: TArtifact\n strongArtifact: TArtifact\n maxWeakScore?: number\n minStrongScore?: number\n minGap?: number\n}\n\nexport interface BenchmarkMetricCalibrationResult {\n passed: boolean\n weak: BenchmarkEvaluation\n strong: BenchmarkEvaluation\n weakScore: number\n strongScore: number\n gap: number\n reasons: string[]\n}\n\nexport async function calibrateBenchmarkMetric<TPayload = unknown, TArtifact = string>(\n options: BenchmarkMetricCalibrationOptions<TPayload, TArtifact>,\n): Promise<BenchmarkMetricCalibrationResult> {\n const weak = await options.adapter.evaluate(options.item, options.weakArtifact)\n const strong = await options.adapter.evaluate(options.item, options.strongArtifact)\n const weakScore = clamp01(weak.score)\n const strongScore = clamp01(strong.score)\n const maxWeakScore = options.maxWeakScore ?? 0.3\n const minStrongScore = options.minStrongScore ?? 0.7\n const minGap = options.minGap ?? 0.4\n const gap = strongScore - weakScore\n const reasons = [\n weakScore <= maxWeakScore\n ? undefined\n : `weak score ${weakScore.toFixed(3)} exceeds max ${maxWeakScore.toFixed(3)}`,\n strongScore >= minStrongScore\n ? undefined\n : `strong score ${strongScore.toFixed(3)} below min ${minStrongScore.toFixed(3)}`,\n gap >= minGap ? undefined : `gap ${gap.toFixed(3)} below min ${minGap.toFixed(3)}`,\n ].filter((reason): reason is string => Boolean(reason))\n\n return {\n passed: reasons.length === 0,\n weak,\n strong,\n weakScore,\n strongScore,\n gap,\n reasons,\n }\n}\n\nfunction clamp01(value: number): number {\n if (!Number.isFinite(value)) return 0\n if (value < 0) return 0\n if (value > 1) return 1\n return value\n}\n","/**\n * Routing benchmark — synthetic, dependency-free, ships in the\n * package. 16 cross-category items in `dataset.ts`. See\n * `routing/README.md` for the format.\n *\n * `evaluate` does case-insensitive exact match against the canonical\n * route plus declared synonyms. The first valid route token in the\n * response wins; everything else is ignored. Wrong answers also\n * report whether they hit a hard negative — useful when triaging\n * \"always picks the popular route\" failure modes.\n */\n\nimport type { RunSplitTag } from '../../run-record'\nimport type { BenchmarkAdapter, BenchmarkDatasetItem, BenchmarkEvaluation } from '../types'\nimport { deterministicSplit } from '../types'\nimport { ROUTING_DATASET, type RoutingItem } from './dataset'\n\nexport type { RoutingItem }\nexport type RoutingPayload = RoutingItem\nexport type RoutingDatasetItem = BenchmarkDatasetItem<RoutingPayload>\n\nclass RoutingAdapter implements BenchmarkAdapter<RoutingDatasetItem, RoutingPayload> {\n readonly id = 'first-party/routing'\n readonly family = 'first-party'\n readonly taskKind = 'routing'\n readonly description = 'Synthetic fixed-route classification smoke benchmark'\n readonly defaultMetric = 'route_exact_match'\n\n async loadDataset(split: RunSplitTag): Promise<RoutingDatasetItem[]> {\n return ROUTING_DATASET.map((item) => ({ id: item.id, payload: item })).filter(\n (it) => assignSplitImpl(it.id) === split,\n )\n }\n\n async evaluate(item: RoutingDatasetItem, response: string): Promise<BenchmarkEvaluation> {\n const tokens = extractRouteTokens(response)\n const correct = new Set<string>(\n [item.payload.route, ...item.payload.synonyms].map((s) => s.toLowerCase()),\n )\n const hardNeg = new Set<string>(item.payload.hardNegatives.map((s) => s.toLowerCase()))\n const firstMatch = tokens.find((t) => correct.has(t.toLowerCase())) ?? null\n const firstHardNeg = tokens.find((t) => hardNeg.has(t.toLowerCase())) ?? null\n const score = firstMatch ? 1 : 0\n return {\n score,\n raw: {\n firstToken: tokens[0] ?? null,\n matchedRoute: firstMatch,\n hitHardNegative: Boolean(firstHardNeg),\n hardNegativeRoute: firstHardNeg,\n category: item.payload.category,\n },\n }\n }\n\n assignSplit(itemId: string): RunSplitTag {\n return assignSplitImpl(itemId)\n }\n}\n\nfunction assignSplitImpl(itemId: string): RunSplitTag {\n return deterministicSplit(`routing::${itemId}`)\n}\n\n/**\n * Pull route-shaped tokens out of a model response. Routes look like\n * `category.action` (`fs.write`, `chat.reply`). Bare alphanumerics\n * are not routes, but `category.action` patterns are robust to most\n * model wrappers (JSON output, prose explanations, code fences).\n */\nexport function extractRouteTokens(response: string): string[] {\n const matches = response.match(/[a-z][a-z0-9_]*\\.[a-z][a-z0-9_]*/gi)\n return matches ?? []\n}\n\nconst adapter = new RoutingAdapter()\n\nexport const loadDataset = adapter.loadDataset.bind(adapter)\nexport const evaluate = adapter.evaluate.bind(adapter)\nexport const assignSplit = adapter.assignSplit.bind(adapter)\nexport { ROUTING_DATASET, RoutingAdapter }\n","/**\n * Shared types for the reference benchmark wrappers under\n * `src/benchmarks/`. Each wrapper exports the three functions in\n * `BenchmarkAdapter` plus its own typed `DatasetItem` shape.\n */\n\nimport type { DispatchContext, Scenario } from '../campaign/types'\nimport type { RunSplitTag } from '../run-record'\n\nexport type BenchmarkTaskKind =\n | 'retrieval'\n | 'rag-answer'\n | 'hallucination'\n | 'kb-improvement'\n | 'routing'\n | 'custom'\n\nexport type BenchmarkFamily =\n | 'beir'\n | 'mteb-retrieval'\n | 'msmarco'\n | 'trec-dl'\n | 'miracl'\n | 'lotte'\n | 'bright'\n | 'crag'\n | 'hotpotqa'\n | 'kilt'\n | 'ragtruth'\n | 'faithbench'\n | 'first-party'\n | 'custom'\n\nexport interface BenchmarkDatasetItem<TPayload = unknown> {\n /** Stable dataset-local item id (used for split assignment + paper\n * references). Unique within a benchmark. */\n id: string\n /** Free-form payload. Each benchmark defines its own shape. */\n payload: TPayload\n /** Optional precomputed split. When absent, adapters use assignSplit(id). */\n split?: RunSplitTag\n /** Benchmark family this row came from, e.g. `beir` or `crag`. */\n family?: BenchmarkFamily | string\n /** Benchmark-local task kind, e.g. retrieval vs answer quality. */\n taskKind?: BenchmarkTaskKind | string\n /** Slice labels such as language, domain, freshness, multihop, long-tail. */\n tags?: string[]\n /** Dataset provenance, version, URL, or license notes. */\n source?: BenchmarkSource\n metadata?: Record<string, unknown>\n}\n\nexport interface BenchmarkEvaluation {\n /** [0, 1] score for the response on this item. Exact-match\n * benchmarks use 0/1; partial-credit benchmarks may return\n * fractional values. */\n score: number\n /** Optional pass/fail projection. Defaults to `score > 0` when absent. */\n passed?: boolean\n /** Numeric sub-metrics. These become report dimensions. */\n dimensions?: Record<string, number>\n /** Optional bag of raw scoring signals — e.g. parsed numeric\n * answer, regex match, judge sub-scores. */\n raw: Record<string, unknown>\n notes?: string\n}\n\nexport interface BenchmarkSource {\n name?: string\n url?: string\n version?: string\n license?: string\n citation?: string\n}\n\n/** Common signature implemented by every adapter under `src/benchmarks/*`. */\n// `TPayload` is the per-item payload type; `_TItem` is preserved for\n// downstream type-narrowing extensions (a richer `BenchmarkDatasetItem`\n// subclass that adds e.g. provenance metadata) but is intentionally\n// unused here. `noUnusedLocals` requires the leading underscore.\nexport interface BenchmarkAdapter<_TItem = unknown, TPayload = unknown, TArtifact = string> {\n /** Stable benchmark id such as `beir/nfcorpus` or `crag/smoke`. */\n id?: string\n family?: BenchmarkFamily | string\n taskKind?: BenchmarkTaskKind | string\n description?: string\n source?: BenchmarkSource\n defaultMetric?: string\n /** Load the dataset for the given split. May hit the network on\n * first call but should be cache-friendly. Adapters that don't\n * ship the dataset itself MUST throw a clearly-marked error\n * pointing the caller at the loader script. */\n loadDataset(split: RunSplitTag): Promise<BenchmarkDatasetItem<TPayload>[]>\n /** Score a single response. Pure with respect to the inputs. */\n evaluate(item: BenchmarkDatasetItem<TPayload>, artifact: TArtifact): Promise<BenchmarkEvaluation>\n /** Deterministic split assignment via item id hashing. The\n * fraction of items in each split is implementation-defined but\n * MUST be stable across processes and platforms. */\n assignSplit(itemId: string): RunSplitTag\n}\n\nexport interface BenchmarkScenario<TPayload = unknown> extends Scenario {\n kind: 'benchmark'\n benchmarkId: string\n family: BenchmarkFamily | string\n taskKind: BenchmarkTaskKind | string\n splitTag: RunSplitTag\n item: BenchmarkDatasetItem<TPayload>\n}\n\nexport type BenchmarkResponder<TPayload = unknown, TArtifact = string> = (input: {\n scenario: BenchmarkScenario<TPayload>\n item: BenchmarkDatasetItem<TPayload>\n context: DispatchContext\n}) => Promise<TArtifact> | TArtifact\n\n// ── Deterministic split assignment ───────────────────────────────────\n\n/**\n * 32-bit FNV-1a hash. Stable, allocation-free, deterministic across\n * runtimes. We use it to assign items to splits rather than depending\n * on a polyfilled crypto.subtle path.\n */\nfunction fnv1a32(input: string): number {\n let h = 0x811c9dc5\n for (let i = 0; i < input.length; i++) {\n h ^= input.charCodeAt(i) & 0xff\n h = (h + ((h << 1) + (h << 4) + (h << 7) + (h << 8) + (h << 24))) >>> 0\n }\n return h >>> 0\n}\n\n/** Split-assignment seed shared across all benchmarks. Bumping this\n * value reshuffles every split — do NOT do that lightly. */\nexport const BENCHMARK_SPLIT_SEED = 'agent-eval-v1'\n\n/**\n * Assign an item id to one of `'search' | 'dev' | 'holdout'` using a\n * stable 32-bit hash of `${seed}::${id}`. Default proportions:\n *\n * search: 60% (optimization-readable)\n * dev: 20% (held-out for tuning, leak-on-purpose during dev)\n * holdout:20% (paper-grade held-out, gated reads)\n */\nexport function deterministicSplit(\n itemId: string,\n seed: string = BENCHMARK_SPLIT_SEED,\n): RunSplitTag {\n const h = fnv1a32(`${seed}::${itemId}`)\n const pos = h / 0x100000000\n if (pos < 0.6) return 'search'\n if (pos < 0.8) return 'dev'\n return 'holdout'\n}\n","/**\n * Synthetic routing dataset. 16 tasks across 4 categories. Used as a\n * deterministic, dependency-free benchmark for any router that maps a\n * natural-language request to one of a fixed set of route labels.\n *\n * Format (see `routing/README.md` for prose):\n *\n * {\n * id: stable per-task ID (matches across processes).\n * category: one of the four route labels.\n * prompt: the user-facing request the router must classify.\n * route: the ground-truth route the router should pick.\n * synonyms: other strings that count as a correct answer.\n * hardNegatives:close-but-wrong route labels — used to detect the\n * \"always picks the popular route\" failure mode.\n * }\n *\n * The four categories are intentionally cross-domain (file ops,\n * math, search, conversation) so a router that collapses to one\n * category is easy to spot.\n */\n\nexport interface RoutingItem {\n id: string\n category: 'file' | 'math' | 'search' | 'chat'\n prompt: string\n /** Canonical correct route label. */\n route: string\n /** Alternate route labels that also count as correct. */\n synonyms: string[]\n /** Wrong-but-tempting route labels (for analysis, not grading). */\n hardNegatives: string[]\n}\n\nexport const ROUTING_DATASET: RoutingItem[] = [\n {\n id: 'file_001',\n category: 'file',\n prompt: 'Save the meeting notes to /tmp/notes-2025-04.md as markdown.',\n route: 'fs.write',\n synonyms: ['filesystem.write', 'write_file'],\n hardNegatives: ['fs.read', 'chat.reply'],\n },\n {\n id: 'file_002',\n category: 'file',\n prompt: 'Read the contents of /etc/hosts and summarize the entries.',\n route: 'fs.read',\n synonyms: ['filesystem.read', 'read_file'],\n hardNegatives: ['fs.write', 'search.web'],\n },\n {\n id: 'file_003',\n category: 'file',\n prompt: 'List every Python file under src/ recursively.',\n route: 'fs.list',\n synonyms: ['filesystem.list', 'list_files'],\n hardNegatives: ['fs.read', 'search.code'],\n },\n {\n id: 'file_004',\n category: 'file',\n prompt: 'Delete the cached build at .turbo/cache.',\n route: 'fs.delete',\n synonyms: ['filesystem.delete', 'remove_file'],\n hardNegatives: ['fs.write', 'fs.list'],\n },\n {\n id: 'math_001',\n category: 'math',\n prompt: 'What is the integral of 3x^2 + 2x from 0 to 5?',\n route: 'math.integral',\n synonyms: ['calculator.integral', 'math.solve'],\n hardNegatives: ['math.derivative', 'chat.reply'],\n },\n {\n id: 'math_002',\n category: 'math',\n prompt: 'Compute the derivative of sin(x) * cos(x).',\n route: 'math.derivative',\n synonyms: ['calculator.derivative', 'math.solve'],\n hardNegatives: ['math.integral', 'math.algebra'],\n },\n {\n id: 'math_003',\n category: 'math',\n prompt: 'Solve 2x + 7 = 19 for x.',\n route: 'math.algebra',\n synonyms: ['calculator.algebra', 'math.solve'],\n hardNegatives: ['math.derivative', 'math.integral'],\n },\n {\n id: 'math_004',\n category: 'math',\n prompt: 'What is the prime factorization of 360?',\n route: 'math.numbertheory',\n synonyms: ['calculator.factor', 'math.solve'],\n hardNegatives: ['math.algebra', 'search.web'],\n },\n {\n id: 'search_001',\n category: 'search',\n prompt: 'Find recent papers on agent prompt optimization with held-out promotion gates.',\n route: 'search.web',\n synonyms: ['web.search', 'search.papers'],\n hardNegatives: ['search.code', 'chat.reply'],\n },\n {\n id: 'search_002',\n category: 'search',\n prompt: 'Search the codebase for every call site of `runProposeReview`.',\n route: 'search.code',\n synonyms: ['code.search', 'grep'],\n hardNegatives: ['search.web', 'fs.read'],\n },\n {\n id: 'search_003',\n category: 'search',\n prompt: 'What is the latest release of the Tangle network on GitHub?',\n route: 'search.web',\n synonyms: ['web.search', 'github.releases'],\n hardNegatives: ['search.code', 'chat.reply'],\n },\n {\n id: 'search_004',\n category: 'search',\n prompt: 'Find all TODO comments in the agent-eval src tree.',\n route: 'search.code',\n synonyms: ['code.search', 'grep'],\n hardNegatives: ['search.web', 'fs.list'],\n },\n {\n id: 'chat_001',\n category: 'chat',\n prompt: 'Hi there, how are you doing today?',\n route: 'chat.reply',\n synonyms: ['conversation.reply'],\n hardNegatives: ['search.web', 'fs.read'],\n },\n {\n id: 'chat_002',\n category: 'chat',\n prompt: 'Please explain the difference between an LLM and a foundation model.',\n route: 'chat.reply',\n synonyms: ['conversation.reply', 'qa.answer'],\n hardNegatives: ['search.web', 'math.algebra'],\n },\n {\n id: 'chat_003',\n category: 'chat',\n prompt: 'Tell me a short joke about distributed systems.',\n route: 'chat.reply',\n synonyms: ['conversation.reply'],\n hardNegatives: ['search.web', 'fs.read'],\n },\n {\n id: 'chat_004',\n category: 'chat',\n prompt: 'Acknowledge my last message with a thumbs up.',\n route: 'chat.reply',\n synonyms: ['conversation.reply', 'react'],\n hardNegatives: ['fs.write', 'search.web'],\n },\n]\n","import { join } from 'node:path'\nimport {\n type CampaignResult,\n type DispatchFn,\n type JudgeConfig,\n type JudgeScore,\n runCampaign,\n type Scenario,\n} from '../campaign'\nimport { type CampaignStorage, fsCampaignStorage } from '../campaign/storage'\nimport type { RunSplitTag } from '../run-record'\nimport type {\n BenchmarkAdapter,\n BenchmarkDatasetItem,\n BenchmarkEvaluation,\n BenchmarkFamily,\n BenchmarkResponder,\n BenchmarkScenario,\n BenchmarkTaskKind,\n} from './types'\n\nexport interface BenchmarkRunOptions<TPayload = unknown, TArtifact = string> {\n adapter: BenchmarkAdapter<BenchmarkDatasetItem<TPayload>, TPayload, TArtifact>\n respond: BenchmarkResponder<TPayload, TArtifact>\n splits?: readonly RunSplitTag[]\n runDir: string\n repo?: string\n seed?: number\n reps?: number\n resumable?: boolean\n costCeiling?: number\n maxConcurrency?: number\n dispatchTimeoutMs?: number\n expectUsage?: 'assert' | 'warn' | 'off'\n storage?: CampaignStorage\n now?: () => Date\n}\n\nexport interface BenchmarkReport {\n benchmarkId: string\n family: BenchmarkFamily | string\n taskKind: BenchmarkTaskKind | string\n source?: BenchmarkAdapter['source']\n runDir: string\n manifestHash: string\n seed: number\n startedAt: string\n endedAt: string\n durationMs: number\n totalItems: number\n totalCells: number\n cellsFailed: number\n cellsCached: number\n totalCostUsd: number\n splits: Record<string, BenchmarkSliceSummary>\n tags: Record<string, BenchmarkSliceSummary>\n dimensions: Record<string, BenchmarkDistribution>\n score: BenchmarkDistribution\n costUsd: BenchmarkDistribution\n latencyMs: BenchmarkDistribution\n}\n\nexport interface BenchmarkSliceSummary {\n n: number\n meanScore: number\n passRate: number\n score: BenchmarkDistribution\n costUsd: BenchmarkDistribution\n latencyMs: BenchmarkDistribution\n}\n\nexport interface BenchmarkDistribution {\n n: number\n min: number\n mean: number\n median: number\n p90: number\n max: number\n}\n\nexport interface BenchmarkRunResult<TPayload = unknown, TArtifact = string> {\n scenarios: Array<BenchmarkScenario<TPayload>>\n campaign: CampaignResult<TArtifact, BenchmarkScenario<TPayload>>\n report: BenchmarkReport\n reportJsonPath: string\n reportMarkdownPath: string\n}\n\nexport async function runBenchmarkAdapter<TPayload = unknown, TArtifact = string>(\n options: BenchmarkRunOptions<TPayload, TArtifact>,\n): Promise<BenchmarkRunResult<TPayload, TArtifact>> {\n const storage = options.storage ?? fsCampaignStorage()\n const benchmarkId = benchmarkIdFor(options.adapter)\n const scenarios = await loadBenchmarkScenarios(options.adapter, options.splits)\n const judge = benchmarkAdapterJudge(options.adapter)\n const dispatch: DispatchFn<BenchmarkScenario<TPayload>, TArtifact> = async (\n scenario,\n context,\n ) => {\n return options.respond({ scenario, item: scenario.item, context })\n }\n\n const campaign = await runCampaign<BenchmarkScenario<TPayload>, TArtifact>({\n scenarios,\n dispatch,\n dispatchRef: `benchmark:${benchmarkId}`,\n judges: [judge],\n seed: options.seed,\n reps: options.reps,\n resumable: options.resumable,\n costCeiling: options.costCeiling,\n maxConcurrency: options.maxConcurrency,\n dispatchTimeoutMs: options.dispatchTimeoutMs,\n expectUsage: options.expectUsage ?? 'off',\n runDir: options.runDir,\n repo: options.repo,\n storage,\n now: options.now,\n })\n const report = summarizeBenchmarkCampaign({\n adapter: options.adapter,\n scenarios,\n campaign,\n })\n storage.ensureDir(campaign.runDir)\n const reportJsonPath = join(campaign.runDir, 'benchmark-report.json')\n const reportMarkdownPath = join(campaign.runDir, 'benchmark-report.md')\n storage.write(reportJsonPath, `${JSON.stringify(report, null, 2)}\\n`)\n storage.write(reportMarkdownPath, renderBenchmarkReportMarkdown(report))\n return { scenarios, campaign, report, reportJsonPath, reportMarkdownPath }\n}\n\nexport async function loadBenchmarkScenarios<TPayload>(\n adapter: BenchmarkAdapter<BenchmarkDatasetItem<TPayload>, TPayload, unknown>,\n splits: readonly RunSplitTag[] = ['search', 'dev', 'holdout'],\n): Promise<Array<BenchmarkScenario<TPayload>>> {\n const benchmarkId = benchmarkIdFor(adapter)\n const family = adapter.family ?? 'custom'\n const taskKind = adapter.taskKind ?? 'custom'\n const out: Array<BenchmarkScenario<TPayload>> = []\n const seen = new Set<string>()\n for (const split of splits) {\n const items = await adapter.loadDataset(split)\n for (const item of items) {\n const splitTag = item.split ?? adapter.assignSplit(item.id)\n if (splitTag !== split) continue\n const key = `${benchmarkId}:${item.id}`\n if (seen.has(key)) continue\n seen.add(key)\n out.push({\n id: key,\n kind: 'benchmark',\n benchmarkId,\n family: item.family ?? family,\n taskKind: item.taskKind ?? taskKind,\n splitTag,\n tags: [...new Set([splitTag, ...(item.tags ?? [])])],\n item,\n })\n }\n }\n return out\n}\n\nexport function benchmarkAdapterJudge<TPayload, TArtifact>(\n adapter: BenchmarkAdapter<BenchmarkDatasetItem<TPayload>, TPayload, TArtifact>,\n): JudgeConfig<TArtifact, BenchmarkScenario<TPayload>> {\n const benchmarkId = benchmarkIdFor(adapter)\n return {\n name: `${benchmarkId}:score`,\n dimensions: [\n { key: 'score', description: `Primary ${benchmarkId} benchmark score` },\n { key: 'passed', description: 'Binary pass projection for aggregate reporting' },\n ],\n appliesTo: (scenario: Scenario): scenario is BenchmarkScenario<TPayload> => {\n return scenario.kind === 'benchmark' && scenario.id.startsWith(`${benchmarkId}:`)\n },\n async score({ artifact, scenario }) {\n const evaluation = await adapter.evaluate(scenario.item, artifact)\n const dimensions = normalizeEvaluationDimensions(evaluation)\n return {\n dimensions,\n composite: clamp01(evaluation.score),\n notes: evaluation.notes ?? '',\n }\n },\n }\n}\n\nexport function summarizeBenchmarkCampaign<TPayload, TArtifact>(input: {\n adapter: BenchmarkAdapter<BenchmarkDatasetItem<TPayload>, TPayload, TArtifact>\n scenarios: Array<BenchmarkScenario<TPayload>>\n campaign: CampaignResult<TArtifact, BenchmarkScenario<TPayload>>\n}): BenchmarkReport {\n const scenarioById = new Map(input.scenarios.map((scenario) => [scenario.id, scenario]))\n const rows = input.campaign.cells.map((cell) => {\n const scenario = scenarioById.get(cell.scenarioId)\n const judge = firstJudgeScore(cell.judgeScores)\n const score = judge?.composite ?? 0\n return {\n cell,\n scenario,\n score,\n passed: (judge?.dimensions.passed ?? (score > 0 ? 1 : 0)) >= 1,\n dimensions: judge?.dimensions ?? {},\n }\n })\n const successful = rows.filter((row) => !row.cell.error)\n const scoreValues = successful.map((row) => row.score)\n return {\n benchmarkId: benchmarkIdFor(input.adapter),\n family: input.adapter.family ?? 'custom',\n taskKind: input.adapter.taskKind ?? 'custom',\n ...(input.adapter.source ? { source: input.adapter.source } : {}),\n runDir: input.campaign.runDir,\n manifestHash: input.campaign.manifestHash,\n seed: input.campaign.seed,\n startedAt: input.campaign.startedAt,\n endedAt: input.campaign.endedAt,\n durationMs: input.campaign.durationMs,\n totalItems: input.scenarios.length,\n totalCells: input.campaign.cells.length,\n cellsFailed: input.campaign.aggregates.cellsFailed,\n cellsCached: input.campaign.aggregates.cellsCached,\n totalCostUsd: input.campaign.aggregates.totalCostUsd,\n splits: summarizeSlices(successful, (row) => row.scenario?.splitTag ?? 'unknown', [\n 'search',\n 'dev',\n 'holdout',\n ]),\n tags: summarizeSlices(successful, (row) => row.scenario?.tags ?? []),\n dimensions: summarizeDimensions(successful.map((row) => row.dimensions)),\n score: distribution(scoreValues),\n costUsd: distribution(successful.map((row) => row.cell.costUsd)),\n latencyMs: distribution(successful.map((row) => row.cell.durationMs)),\n }\n}\n\nexport function renderBenchmarkReportMarkdown(report: BenchmarkReport): string {\n const splitRows = Object.entries(report.splits)\n .map(([split, summary]) => {\n return `| ${split} | ${summary.n} | ${fmt(summary.meanScore)} | ${fmt(summary.passRate)} | ${fmt(summary.score.p90)} | ${fmt(summary.costUsd.mean)} | ${fmt(summary.latencyMs.p90)} |`\n })\n .join('\\n')\n const dimRows = Object.entries(report.dimensions)\n .sort(([a], [b]) => a.localeCompare(b))\n .map(([key, dist]) => `| ${key} | ${dist.n} | ${fmt(dist.mean)} | ${fmt(dist.p90)} |`)\n .join('\\n')\n return [\n `# Benchmark Report: ${report.benchmarkId}`,\n '',\n `- family: ${report.family}`,\n `- task kind: ${report.taskKind}`,\n `- run dir: ${report.runDir}`,\n `- manifest: ${report.manifestHash}`,\n `- cells: ${report.totalCells} total, ${report.cellsFailed} failed, ${report.cellsCached} cached`,\n `- cost: $${fmt(report.totalCostUsd)}`,\n `- score: mean ${fmt(report.score.mean)}, median ${fmt(report.score.median)}, p90 ${fmt(report.score.p90)}, n=${report.score.n}`,\n '',\n '## Splits',\n '',\n '| split | n | mean score | pass rate | score p90 | mean cost | latency p90 ms |',\n '| --- | ---: | ---: | ---: | ---: | ---: | ---: |',\n splitRows || '| none | 0 | 0 | 0 | 0 | 0 | 0 |',\n '',\n '## Dimensions',\n '',\n '| dimension | n | mean | p90 |',\n '| --- | ---: | ---: | ---: |',\n dimRows || '| none | 0 | 0 | 0 |',\n '',\n ].join('\\n')\n}\n\nfunction benchmarkIdFor(adapter: Pick<BenchmarkAdapter, 'id' | 'family' | 'taskKind'>): string {\n return adapter.id ?? `${adapter.family ?? 'custom'}/${adapter.taskKind ?? 'custom'}`\n}\n\nfunction normalizeEvaluationDimensions(evaluation: BenchmarkEvaluation): Record<string, number> {\n const dimensions: Record<string, number> = { score: clamp01(evaluation.score) }\n for (const [key, value] of Object.entries(evaluation.dimensions ?? {})) {\n if (Number.isFinite(value)) dimensions[key] = value\n }\n dimensions.passed = (evaluation.passed ?? evaluation.score > 0) ? 1 : 0\n return dimensions\n}\n\nfunction summarizeDimensions(\n rows: Array<Record<string, number>>,\n): Record<string, BenchmarkDistribution> {\n const values = new Map<string, number[]>()\n for (const row of rows) {\n for (const [key, value] of Object.entries(row)) {\n if (!Number.isFinite(value)) continue\n const list = values.get(key) ?? []\n list.push(value)\n values.set(key, list)\n }\n }\n return Object.fromEntries([...values.entries()].map(([key, vals]) => [key, distribution(vals)]))\n}\n\nfunction summarizeSlices<T>(\n rows: T[],\n keyOf: (row: T) => string | string[],\n knownKeys: readonly string[] = [],\n): Record<string, BenchmarkSliceSummary> {\n const grouped = new Map<string, T[]>()\n for (const key of knownKeys) grouped.set(key, [])\n for (const row of rows) {\n const keys = keyOf(row)\n for (const key of Array.isArray(keys) ? keys : [keys]) {\n const list = grouped.get(key) ?? []\n list.push(row)\n grouped.set(key, list)\n }\n }\n const out: Record<string, BenchmarkSliceSummary> = {}\n for (const [key, list] of grouped) {\n const withShape = list as Array<{\n score: number\n passed: boolean\n cell: { costUsd: number; durationMs: number }\n }>\n out[key] = {\n n: list.length,\n meanScore: mean(withShape.map((row) => row.score)),\n passRate: mean(withShape.map((row) => (row.passed ? 1 : 0))),\n score: distribution(withShape.map((row) => row.score)),\n costUsd: distribution(withShape.map((row) => row.cell.costUsd)),\n latencyMs: distribution(withShape.map((row) => row.cell.durationMs)),\n }\n }\n return out\n}\n\nfunction firstJudgeScore(\n judgeScores: CampaignResult<unknown>['cells'][number]['judgeScores'],\n): JudgeScore | undefined {\n return Object.values(judgeScores)[0]\n}\n\nfunction distribution(values: readonly number[]): BenchmarkDistribution {\n const finite = [...values].filter(Number.isFinite).sort((a, b) => a - b)\n if (finite.length === 0) return { n: 0, min: 0, mean: 0, median: 0, p90: 0, max: 0 }\n return {\n n: finite.length,\n min: finite[0]!,\n mean: mean(finite),\n median: percentile(finite, 0.5),\n p90: percentile(finite, 0.9),\n max: finite[finite.length - 1]!,\n }\n}\n\nfunction percentile(sortedValues: readonly number[], p: number): number {\n if (sortedValues.length === 0) return 0\n const index = Math.min(\n sortedValues.length - 1,\n Math.max(0, Math.ceil(p * sortedValues.length) - 1),\n )\n return sortedValues[index]!\n}\n\nfunction mean(values: readonly number[]): number {\n const finite = values.filter(Number.isFinite)\n if (finite.length === 0) return 0\n return finite.reduce((sum, value) => sum + value, 0) / finite.length\n}\n\nfunction clamp01(value: number): number {\n if (!Number.isFinite(value)) return 0\n if (value < 0) return 0\n if (value > 1) return 1\n return value\n}\n\nfunction fmt(value: number): string {\n if (!Number.isFinite(value)) return '0'\n return value.toFixed(value === 0 || Math.abs(value) >= 10 ? 0 : 3)\n}\n","import type { RunSplitTag } from '../run-record'\nimport type {\n BenchmarkAdapter,\n BenchmarkDatasetItem,\n BenchmarkFamily,\n BenchmarkSource,\n BenchmarkTaskKind,\n} from './types'\nimport { deterministicSplit } from './types'\n\nexport interface StandardRetrievalDocument {\n id: string\n title?: string\n text: string\n metadata?: Record<string, unknown>\n}\n\nexport interface StandardRetrievalQuery {\n id: string\n text: string\n metadata?: Record<string, unknown>\n}\n\nexport interface StandardRetrievalQrel {\n queryId: string\n documentId: string\n score: number\n}\n\nexport interface StandardRetrievalPayload {\n queryId: string\n query: string\n expectedDocumentIds: string[]\n expectedScores: Record<string, number>\n corpus?: Record<string, StandardRetrievalDocument>\n metadata?: Record<string, unknown>\n}\n\nexport interface BuildStandardRetrievalItemsOptions {\n benchmarkId: string\n family: BenchmarkFamily | string\n queries: readonly StandardRetrievalQuery[]\n qrels: readonly StandardRetrievalQrel[]\n corpus?: readonly StandardRetrievalDocument[]\n includeCorpusInPayload?: boolean\n source?: BenchmarkSource\n tags?: readonly string[]\n splitOf?: (queryId: string) => RunSplitTag\n}\n\nexport interface RetrievalIdAdapterOptions extends BuildStandardRetrievalItemsOptions {\n responseIdPattern?: RegExp\n cutoffs?: readonly number[]\n primaryMetric?: string\n passMetric?: string\n passThreshold?: number\n}\n\nexport interface StandardRetrievalResult {\n id?: string\n documentId?: string\n docId?: string\n score?: number\n}\n\nexport type StandardRetrievalArtifact =\n | string\n | readonly string[]\n | readonly StandardRetrievalResult[]\n | {\n ids?: readonly string[]\n documentIds?: readonly string[]\n results?: readonly StandardRetrievalResult[]\n }\n\nexport interface StandardRetrievalEvaluationOptions {\n responseIdPattern?: RegExp\n cutoffs?: readonly number[]\n primaryMetric?: string\n passMetric?: string\n passThreshold?: number\n}\n\nexport function parseJsonlRows<T = unknown>(text: string): T[] {\n return text\n .split(/\\r?\\n/)\n .map((line) => line.trim())\n .filter(Boolean)\n .map((line, index) => {\n try {\n return JSON.parse(line) as T\n } catch (error) {\n throw new Error(`invalid JSONL row ${index + 1}: ${(error as Error).message}`)\n }\n })\n}\n\nexport function parseTsvRows(text: string): string[][] {\n return text\n .split(/\\r?\\n/)\n .map((line) => line.trim())\n .filter(Boolean)\n .filter((line) => !line.startsWith('#'))\n .map((line) => line.split(/\\t|\\s+/))\n}\n\nexport function parseQrels(text: string): StandardRetrievalQrel[] {\n return parseTsvRows(text).flatMap((parts, index) => {\n if (parts.length < 3) return []\n const [queryId, maybeZeroOrDocId, maybeDocIdOrScore, maybeScore] = parts\n if (!queryId || !maybeZeroOrDocId || !maybeDocIdOrScore) return []\n if (queryId.toLowerCase() === 'query-id' || queryId.toLowerCase() === 'qid') return []\n const documentId = maybeScore === undefined ? maybeZeroOrDocId : maybeDocIdOrScore\n const scoreText = maybeScore === undefined ? maybeDocIdOrScore : maybeScore\n const score = Number(scoreText)\n if (!documentId || !Number.isFinite(score)) {\n throw new Error(`invalid qrels row ${index + 1}: expected query id, doc id, score`)\n }\n return [{ queryId, documentId, score }]\n })\n}\n\nexport function parseBeirCorpusJsonl(text: string): StandardRetrievalDocument[] {\n return parseJsonlRows<Record<string, unknown>>(text).map((row, index) => {\n const id = stringField(row, '_id') ?? stringField(row, 'id')\n const body = stringField(row, 'text') ?? stringField(row, 'contents')\n if (!id || body === undefined) {\n throw new Error(`invalid BEIR corpus row ${index + 1}: expected _id/id and text/contents`)\n }\n return {\n id,\n title: stringField(row, 'title'),\n text: body,\n metadata: stripKnown(row, ['_id', 'id', 'title', 'text', 'contents']),\n }\n })\n}\n\nexport function parseBeirQueriesJsonl(text: string): StandardRetrievalQuery[] {\n return parseJsonlRows<Record<string, unknown>>(text).map((row, index) => {\n const id = stringField(row, '_id') ?? stringField(row, 'id') ?? stringField(row, 'query_id')\n const query = stringField(row, 'text') ?? stringField(row, 'query')\n if (!id || !query) {\n throw new Error(`invalid BEIR query row ${index + 1}: expected _id/id and text/query`)\n }\n return {\n id,\n text: query,\n metadata: stripKnown(row, ['_id', 'id', 'query_id', 'text', 'query']),\n }\n })\n}\n\nexport function buildStandardRetrievalItems(\n options: BuildStandardRetrievalItemsOptions,\n): Array<BenchmarkDatasetItem<StandardRetrievalPayload>> {\n const qrelsByQuery = new Map<string, StandardRetrievalQrel[]>()\n for (const qrel of options.qrels) {\n if (qrel.score <= 0) continue\n const list = qrelsByQuery.get(qrel.queryId) ?? []\n list.push(qrel)\n qrelsByQuery.set(qrel.queryId, list)\n }\n const corpus =\n options.includeCorpusInPayload && options.corpus\n ? Object.fromEntries(options.corpus.map((document) => [document.id, document]))\n : undefined\n return options.queries.flatMap((query) => {\n const qrels = qrelsByQuery.get(query.id) ?? []\n if (qrels.length === 0) return []\n const split =\n options.splitOf?.(query.id) ?? deterministicSplit(`${options.benchmarkId}:${query.id}`)\n return [\n {\n id: query.id,\n split,\n family: options.family,\n taskKind: 'retrieval',\n tags: [...new Set([...(options.tags ?? []), split])],\n ...(options.source ? { source: options.source } : {}),\n ...(query.metadata ? { metadata: query.metadata } : {}),\n payload: {\n queryId: query.id,\n query: query.text,\n expectedDocumentIds: qrels.map((qrel) => qrel.documentId),\n expectedScores: Object.fromEntries(qrels.map((qrel) => [qrel.documentId, qrel.score])),\n ...(corpus ? { corpus } : {}),\n ...(query.metadata ? { metadata: query.metadata } : {}),\n },\n },\n ]\n })\n}\n\nexport function createRetrievalIdBenchmarkAdapter(\n options: RetrievalIdAdapterOptions,\n): BenchmarkAdapter<\n BenchmarkDatasetItem<StandardRetrievalPayload>,\n StandardRetrievalPayload,\n StandardRetrievalArtifact\n> {\n const items = buildStandardRetrievalItems(options)\n return {\n id: options.benchmarkId,\n family: options.family,\n taskKind: 'retrieval' satisfies BenchmarkTaskKind,\n source: options.source,\n defaultMetric: options.primaryMetric ?? 'ndcg@10',\n async loadDataset(split) {\n return items.filter((item) => item.split === split)\n },\n async evaluate(item, artifact) {\n return evaluateStandardRetrieval(item.payload, artifact, options)\n },\n assignSplit(itemId) {\n return options.splitOf?.(itemId) ?? deterministicSplit(`${options.benchmarkId}:${itemId}`)\n },\n }\n}\n\nexport function evaluateStandardRetrieval(\n payload: StandardRetrievalPayload,\n artifact: StandardRetrievalArtifact,\n options: StandardRetrievalEvaluationOptions = {},\n) {\n const rankedDocumentIds = normalizeRetrievedDocumentIds(\n artifact,\n options.responseIdPattern ?? /[A-Za-z0-9_.:/-]+/g,\n )\n const cutoffs = normalizeCutoffs(options.cutoffs ?? [1, 3, 5, 10])\n const dimensions: Record<string, number> = {\n expected_count: payload.expectedDocumentIds.length,\n returned_count: rankedDocumentIds.length,\n }\n for (const cutoff of cutoffs) {\n Object.assign(\n dimensions,\n retrievalMetricsAtCutoff({\n rankedDocumentIds,\n expectedScores: payload.expectedScores,\n cutoff,\n }),\n )\n }\n const primaryMetric = options.primaryMetric ?? 'ndcg@10'\n const passMetric = options.passMetric ?? 'hit@10'\n const score = dimensions[primaryMetric] ?? dimensions[`ndcg@${cutoffs[cutoffs.length - 1]}`] ?? 0\n const passScore =\n dimensions[passMetric] ?? dimensions[`hit@${cutoffs[cutoffs.length - 1]}`] ?? score\n return {\n score,\n passed: passScore >= (options.passThreshold ?? 1),\n dimensions,\n raw: {\n rankedDocumentIds,\n expectedDocumentIds: payload.expectedDocumentIds,\n expectedScores: payload.expectedScores,\n },\n }\n}\n\nexport function normalizeRetrievedDocumentIds(\n artifact: StandardRetrievalArtifact,\n responseIdPattern: RegExp = /[A-Za-z0-9_.:/-]+/g,\n): string[] {\n if (typeof artifact === 'string') {\n return uniqueOrdered((artifact.match(responseIdPattern) ?? []).map((value) => value.trim()))\n }\n if (Array.isArray(artifact)) {\n const entries = artifact as readonly (string | StandardRetrievalResult)[]\n return uniqueOrdered(\n entries.flatMap((entry) => {\n if (typeof entry === 'string') return [entry]\n return [entry.documentId ?? entry.docId ?? entry.id ?? '']\n }),\n )\n }\n const objectArtifact = artifact as Exclude<StandardRetrievalArtifact, string | readonly unknown[]>\n return uniqueOrdered(\n [\n ...(objectArtifact.documentIds ?? []),\n ...(objectArtifact.ids ?? []),\n ...(objectArtifact.results ?? []).map(\n (entry) => entry.documentId ?? entry.docId ?? entry.id ?? '',\n ),\n ].map((value) => value.trim()),\n )\n}\n\nexport function retrievalMetricsAtCutoff(input: {\n rankedDocumentIds: readonly string[]\n expectedScores: Record<string, number>\n cutoff: number\n}): Record<string, number> {\n const top = input.rankedDocumentIds.slice(0, input.cutoff)\n const relevantIds = Object.keys(input.expectedScores).filter(\n (id) => input.expectedScores[id]! > 0,\n )\n const relevant = new Set(relevantIds)\n const hits = top.filter((id) => relevant.has(id)).length\n const firstRelevantRank = top.findIndex((id) => relevant.has(id))\n const prefix = `@${input.cutoff}`\n return {\n [`hit${prefix}`]: hits > 0 ? 1 : 0,\n [`recall${prefix}`]: relevant.size === 0 ? 1 : hits / relevant.size,\n [`precision${prefix}`]: hits / input.cutoff,\n [`mrr${prefix}`]: firstRelevantRank === -1 ? 0 : 1 / (firstRelevantRank + 1),\n [`ndcg${prefix}`]: ndcgAt(input.rankedDocumentIds, input.expectedScores, input.cutoff),\n }\n}\n\nfunction stringField(row: Record<string, unknown>, key: string): string | undefined {\n const value = row[key]\n return typeof value === 'string' ? value : undefined\n}\n\nfunction stripKnown(\n row: Record<string, unknown>,\n keys: readonly string[],\n): Record<string, unknown> {\n const known = new Set(keys)\n return Object.fromEntries(Object.entries(row).filter(([key]) => !known.has(key)))\n}\n\nfunction normalizeCutoffs(cutoffs: readonly number[]): number[] {\n return [\n ...new Set(cutoffs.map((cutoff) => Math.trunc(cutoff)).filter((cutoff) => cutoff > 0)),\n ].sort((a, b) => a - b)\n}\n\nfunction uniqueOrdered(values: readonly string[]): string[] {\n const seen = new Set<string>()\n const out: string[] = []\n for (const value of values) {\n const trimmed = value.trim()\n if (!trimmed || seen.has(trimmed)) continue\n seen.add(trimmed)\n out.push(trimmed)\n }\n return out\n}\n\nfunction ndcgAt(\n rankedDocumentIds: readonly string[],\n expectedScores: Record<string, number>,\n cutoff: number,\n): number {\n const dcg = rankedDocumentIds\n .slice(0, cutoff)\n .reduce(\n (sum, documentId, index) => sum + discountedGain(expectedScores[documentId] ?? 0, index),\n 0,\n )\n const ideal = Object.values(expectedScores)\n .filter((score) => score > 0)\n .sort((a, b) => b - a)\n .slice(0, cutoff)\n .reduce((sum, score, index) => sum + discountedGain(score, index), 0)\n return ideal === 0 ? 1 : dcg / ideal\n}\n\nfunction discountedGain(relevance: number, zeroBasedRank: number): number {\n if (relevance <= 0) return 0\n return (2 ** relevance - 1) / Math.log2(zeroBasedRank + 2)\n}\n"],"mappings":";;;;;;;;;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;;ACsBA,eAAsB,yBACpB,SAC2C;AAC3C,QAAM,OAAO,MAAM,QAAQ,QAAQ,SAAS,QAAQ,MAAM,QAAQ,YAAY;AAC9E,QAAM,SAAS,MAAM,QAAQ,QAAQ,SAAS,QAAQ,MAAM,QAAQ,cAAc;AAClF,QAAM,YAAY,QAAQ,KAAK,KAAK;AACpC,QAAM,cAAc,QAAQ,OAAO,KAAK;AACxC,QAAM,eAAe,QAAQ,gBAAgB;AAC7C,QAAM,iBAAiB,QAAQ,kBAAkB;AACjD,QAAM,SAAS,QAAQ,UAAU;AACjC,QAAM,MAAM,cAAc;AAC1B,QAAM,UAAU;AAAA,IACd,aAAa,eACT,SACA,cAAc,UAAU,QAAQ,CAAC,CAAC,gBAAgB,aAAa,QAAQ,CAAC,CAAC;AAAA,IAC7E,eAAe,iBACX,SACA,gBAAgB,YAAY,QAAQ,CAAC,CAAC,cAAc,eAAe,QAAQ,CAAC,CAAC;AAAA,IACjF,OAAO,SAAS,SAAY,OAAO,IAAI,QAAQ,CAAC,CAAC,cAAc,OAAO,QAAQ,CAAC,CAAC;AAAA,EAClF,EAAE,OAAO,CAAC,WAA6B,QAAQ,MAAM,CAAC;AAEtD,SAAO;AAAA,IACL,QAAQ,QAAQ,WAAW;AAAA,IAC3B;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;AAEA,SAAS,QAAQ,OAAuB;AACtC,MAAI,CAAC,OAAO,SAAS,KAAK,EAAG,QAAO;AACpC,MAAI,QAAQ,EAAG,QAAO;AACtB,MAAI,QAAQ,EAAG,QAAO;AACtB,SAAO;AACT;;;AC3DA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;;;AC2HA,SAAS,QAAQ,OAAuB;AACtC,MAAI,IAAI;AACR,WAAS,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK;AACrC,SAAK,MAAM,WAAW,CAAC,IAAI;AAC3B,QAAK,MAAM,KAAK,MAAM,KAAK,MAAM,KAAK,MAAM,KAAK,MAAM,KAAK,SAAU;AAAA,EACxE;AACA,SAAO,MAAM;AACf;AAIO,IAAM,uBAAuB;AAU7B,SAAS,mBACd,QACA,OAAe,sBACF;AACb,QAAM,IAAI,QAAQ,GAAG,IAAI,KAAK,MAAM,EAAE;AACtC,QAAM,MAAM,IAAI;AAChB,MAAI,MAAM,IAAK,QAAO;AACtB,MAAI,MAAM,IAAK,QAAO;AACtB,SAAO;AACT;;;ACvHO,IAAM,kBAAiC;AAAA,EAC5C;AAAA,IACE,IAAI;AAAA,IACJ,UAAU;AAAA,IACV,QAAQ;AAAA,IACR,OAAO;AAAA,IACP,UAAU,CAAC,oBAAoB,YAAY;AAAA,IAC3C,eAAe,CAAC,WAAW,YAAY;AAAA,EACzC;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,UAAU;AAAA,IACV,QAAQ;AAAA,IACR,OAAO;AAAA,IACP,UAAU,CAAC,mBAAmB,WAAW;AAAA,IACzC,eAAe,CAAC,YAAY,YAAY;AAAA,EAC1C;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,UAAU;AAAA,IACV,QAAQ;AAAA,IACR,OAAO;AAAA,IACP,UAAU,CAAC,mBAAmB,YAAY;AAAA,IAC1C,eAAe,CAAC,WAAW,aAAa;AAAA,EAC1C;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,UAAU;AAAA,IACV,QAAQ;AAAA,IACR,OAAO;AAAA,IACP,UAAU,CAAC,qBAAqB,aAAa;AAAA,IAC7C,eAAe,CAAC,YAAY,SAAS;AAAA,EACvC;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,UAAU;AAAA,IACV,QAAQ;AAAA,IACR,OAAO;AAAA,IACP,UAAU,CAAC,uBAAuB,YAAY;AAAA,IAC9C,eAAe,CAAC,mBAAmB,YAAY;AAAA,EACjD;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,UAAU;AAAA,IACV,QAAQ;AAAA,IACR,OAAO;AAAA,IACP,UAAU,CAAC,yBAAyB,YAAY;AAAA,IAChD,eAAe,CAAC,iBAAiB,cAAc;AAAA,EACjD;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,UAAU;AAAA,IACV,QAAQ;AAAA,IACR,OAAO;AAAA,IACP,UAAU,CAAC,sBAAsB,YAAY;AAAA,IAC7C,eAAe,CAAC,mBAAmB,eAAe;AAAA,EACpD;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,UAAU;AAAA,IACV,QAAQ;AAAA,IACR,OAAO;AAAA,IACP,UAAU,CAAC,qBAAqB,YAAY;AAAA,IAC5C,eAAe,CAAC,gBAAgB,YAAY;AAAA,EAC9C;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,UAAU;AAAA,IACV,QAAQ;AAAA,IACR,OAAO;AAAA,IACP,UAAU,CAAC,cAAc,eAAe;AAAA,IACxC,eAAe,CAAC,eAAe,YAAY;AAAA,EAC7C;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,UAAU;AAAA,IACV,QAAQ;AAAA,IACR,OAAO;AAAA,IACP,UAAU,CAAC,eAAe,MAAM;AAAA,IAChC,eAAe,CAAC,cAAc,SAAS;AAAA,EACzC;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,UAAU;AAAA,IACV,QAAQ;AAAA,IACR,OAAO;AAAA,IACP,UAAU,CAAC,cAAc,iBAAiB;AAAA,IAC1C,eAAe,CAAC,eAAe,YAAY;AAAA,EAC7C;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,UAAU;AAAA,IACV,QAAQ;AAAA,IACR,OAAO;AAAA,IACP,UAAU,CAAC,eAAe,MAAM;AAAA,IAChC,eAAe,CAAC,cAAc,SAAS;AAAA,EACzC;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,UAAU;AAAA,IACV,QAAQ;AAAA,IACR,OAAO;AAAA,IACP,UAAU,CAAC,oBAAoB;AAAA,IAC/B,eAAe,CAAC,cAAc,SAAS;AAAA,EACzC;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,UAAU;AAAA,IACV,QAAQ;AAAA,IACR,OAAO;AAAA,IACP,UAAU,CAAC,sBAAsB,WAAW;AAAA,IAC5C,eAAe,CAAC,cAAc,cAAc;AAAA,EAC9C;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,UAAU;AAAA,IACV,QAAQ;AAAA,IACR,OAAO;AAAA,IACP,UAAU,CAAC,oBAAoB;AAAA,IAC/B,eAAe,CAAC,cAAc,SAAS;AAAA,EACzC;AAAA,EACA;AAAA,IACE,IAAI;AAAA,IACJ,UAAU;AAAA,IACV,QAAQ;AAAA,IACR,OAAO;AAAA,IACP,UAAU,CAAC,sBAAsB,OAAO;AAAA,IACxC,eAAe,CAAC,YAAY,YAAY;AAAA,EAC1C;AACF;;;AF9IA,IAAM,iBAAN,MAAqF;AAAA,EAC1E,KAAK;AAAA,EACL,SAAS;AAAA,EACT,WAAW;AAAA,EACX,cAAc;AAAA,EACd,gBAAgB;AAAA,EAEzB,MAAM,YAAY,OAAmD;AACnE,WAAO,gBAAgB,IAAI,CAAC,UAAU,EAAE,IAAI,KAAK,IAAI,SAAS,KAAK,EAAE,EAAE;AAAA,MACrE,CAAC,OAAO,gBAAgB,GAAG,EAAE,MAAM;AAAA,IACrC;AAAA,EACF;AAAA,EAEA,MAAM,SAAS,MAA0B,UAAgD;AACvF,UAAM,SAAS,mBAAmB,QAAQ;AAC1C,UAAM,UAAU,IAAI;AAAA,MAClB,CAAC,KAAK,QAAQ,OAAO,GAAG,KAAK,QAAQ,QAAQ,EAAE,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC;AAAA,IAC3E;AACA,UAAM,UAAU,IAAI,IAAY,KAAK,QAAQ,cAAc,IAAI,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;AACtF,UAAM,aAAa,OAAO,KAAK,CAAC,MAAM,QAAQ,IAAI,EAAE,YAAY,CAAC,CAAC,KAAK;AACvE,UAAM,eAAe,OAAO,KAAK,CAAC,MAAM,QAAQ,IAAI,EAAE,YAAY,CAAC,CAAC,KAAK;AACzE,UAAM,QAAQ,aAAa,IAAI;AAC/B,WAAO;AAAA,MACL;AAAA,MACA,KAAK;AAAA,QACH,YAAY,OAAO,CAAC,KAAK;AAAA,QACzB,cAAc;AAAA,QACd,iBAAiB,QAAQ,YAAY;AAAA,QACrC,mBAAmB;AAAA,QACnB,UAAU,KAAK,QAAQ;AAAA,MACzB;AAAA,IACF;AAAA,EACF;AAAA,EAEA,YAAY,QAA6B;AACvC,WAAO,gBAAgB,MAAM;AAAA,EAC/B;AACF;AAEA,SAAS,gBAAgB,QAA6B;AACpD,SAAO,mBAAmB,YAAY,MAAM,EAAE;AAChD;AAQO,SAAS,mBAAmB,UAA4B;AAC7D,QAAM,UAAU,SAAS,MAAM,oCAAoC;AACnE,SAAO,WAAW,CAAC;AACrB;AAEA,IAAM,UAAU,IAAI,eAAe;AAE5B,IAAM,cAAc,QAAQ,YAAY,KAAK,OAAO;AACpD,IAAM,WAAW,QAAQ,SAAS,KAAK,OAAO;AAC9C,IAAM,cAAc,QAAQ,YAAY,KAAK,OAAO;;;AG/E3D,SAAS,YAAY;AAwFrB,eAAsB,oBACpB,SACkD;AAClD,QAAM,UAAU,QAAQ,WAAW,kBAAkB;AACrD,QAAM,cAAc,eAAe,QAAQ,OAAO;AAClD,QAAM,YAAY,MAAM,uBAAuB,QAAQ,SAAS,QAAQ,MAAM;AAC9E,QAAM,QAAQ,sBAAsB,QAAQ,OAAO;AACnD,QAAM,WAA+D,OACnE,UACA,YACG;AACH,WAAO,QAAQ,QAAQ,EAAE,UAAU,MAAM,SAAS,MAAM,QAAQ,CAAC;AAAA,EACnE;AAEA,QAAM,WAAW,MAAM,YAAoD;AAAA,IACzE;AAAA,IACA;AAAA,IACA,aAAa,aAAa,WAAW;AAAA,IACrC,QAAQ,CAAC,KAAK;AAAA,IACd,MAAM,QAAQ;AAAA,IACd,MAAM,QAAQ;AAAA,IACd,WAAW,QAAQ;AAAA,IACnB,aAAa,QAAQ;AAAA,IACrB,gBAAgB,QAAQ;AAAA,IACxB,mBAAmB,QAAQ;AAAA,IAC3B,aAAa,QAAQ,eAAe;AAAA,IACpC,QAAQ,QAAQ;AAAA,IAChB,MAAM,QAAQ;AAAA,IACd;AAAA,IACA,KAAK,QAAQ;AAAA,EACf,CAAC;AACD,QAAM,SAAS,2BAA2B;AAAA,IACxC,SAAS,QAAQ;AAAA,IACjB;AAAA,IACA;AAAA,EACF,CAAC;AACD,UAAQ,UAAU,SAAS,MAAM;AACjC,QAAM,iBAAiB,KAAK,SAAS,QAAQ,uBAAuB;AACpE,QAAM,qBAAqB,KAAK,SAAS,QAAQ,qBAAqB;AACtE,UAAQ,MAAM,gBAAgB,GAAG,KAAK,UAAU,QAAQ,MAAM,CAAC,CAAC;AAAA,CAAI;AACpE,UAAQ,MAAM,oBAAoB,8BAA8B,MAAM,CAAC;AACvE,SAAO,EAAE,WAAW,UAAU,QAAQ,gBAAgB,mBAAmB;AAC3E;AAEA,eAAsB,uBACpBA,UACA,SAAiC,CAAC,UAAU,OAAO,SAAS,GACf;AAC7C,QAAM,cAAc,eAAeA,QAAO;AAC1C,QAAM,SAASA,SAAQ,UAAU;AACjC,QAAM,WAAWA,SAAQ,YAAY;AACrC,QAAM,MAA0C,CAAC;AACjD,QAAM,OAAO,oBAAI,IAAY;AAC7B,aAAW,SAAS,QAAQ;AAC1B,UAAM,QAAQ,MAAMA,SAAQ,YAAY,KAAK;AAC7C,eAAW,QAAQ,OAAO;AACxB,YAAM,WAAW,KAAK,SAASA,SAAQ,YAAY,KAAK,EAAE;AAC1D,UAAI,aAAa,MAAO;AACxB,YAAM,MAAM,GAAG,WAAW,IAAI,KAAK,EAAE;AACrC,UAAI,KAAK,IAAI,GAAG,EAAG;AACnB,WAAK,IAAI,GAAG;AACZ,UAAI,KAAK;AAAA,QACP,IAAI;AAAA,QACJ,MAAM;AAAA,QACN;AAAA,QACA,QAAQ,KAAK,UAAU;AAAA,QACvB,UAAU,KAAK,YAAY;AAAA,QAC3B;AAAA,QACA,MAAM,CAAC,GAAG,oBAAI,IAAI,CAAC,UAAU,GAAI,KAAK,QAAQ,CAAC,CAAE,CAAC,CAAC;AAAA,QACnD;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AACA,SAAO;AACT;AAEO,SAAS,sBACdA,UACqD;AACrD,QAAM,cAAc,eAAeA,QAAO;AAC1C,SAAO;AAAA,IACL,MAAM,GAAG,WAAW;AAAA,IACpB,YAAY;AAAA,MACV,EAAE,KAAK,SAAS,aAAa,WAAW,WAAW,mBAAmB;AAAA,MACtE,EAAE,KAAK,UAAU,aAAa,iDAAiD;AAAA,IACjF;AAAA,IACA,WAAW,CAAC,aAAgE;AAC1E,aAAO,SAAS,SAAS,eAAe,SAAS,GAAG,WAAW,GAAG,WAAW,GAAG;AAAA,IAClF;AAAA,IACA,MAAM,MAAM,EAAE,UAAU,SAAS,GAAG;AAClC,YAAM,aAAa,MAAMA,SAAQ,SAAS,SAAS,MAAM,QAAQ;AACjE,YAAM,aAAa,8BAA8B,UAAU;AAC3D,aAAO;AAAA,QACL;AAAA,QACA,WAAWC,SAAQ,WAAW,KAAK;AAAA,QACnC,OAAO,WAAW,SAAS;AAAA,MAC7B;AAAA,IACF;AAAA,EACF;AACF;AAEO,SAAS,2BAAgD,OAI5C;AAClB,QAAM,eAAe,IAAI,IAAI,MAAM,UAAU,IAAI,CAAC,aAAa,CAAC,SAAS,IAAI,QAAQ,CAAC,CAAC;AACvF,QAAM,OAAO,MAAM,SAAS,MAAM,IAAI,CAAC,SAAS;AAC9C,UAAM,WAAW,aAAa,IAAI,KAAK,UAAU;AACjD,UAAM,QAAQ,gBAAgB,KAAK,WAAW;AAC9C,UAAM,QAAQ,OAAO,aAAa;AAClC,WAAO;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,MACA,SAAS,OAAO,WAAW,WAAW,QAAQ,IAAI,IAAI,OAAO;AAAA,MAC7D,YAAY,OAAO,cAAc,CAAC;AAAA,IACpC;AAAA,EACF,CAAC;AACD,QAAM,aAAa,KAAK,OAAO,CAAC,QAAQ,CAAC,IAAI,KAAK,KAAK;AACvD,QAAM,cAAc,WAAW,IAAI,CAAC,QAAQ,IAAI,KAAK;AACrD,SAAO;AAAA,IACL,aAAa,eAAe,MAAM,OAAO;AAAA,IACzC,QAAQ,MAAM,QAAQ,UAAU;AAAA,IAChC,UAAU,MAAM,QAAQ,YAAY;AAAA,IACpC,GAAI,MAAM,QAAQ,SAAS,EAAE,QAAQ,MAAM,QAAQ,OAAO,IAAI,CAAC;AAAA,IAC/D,QAAQ,MAAM,SAAS;AAAA,IACvB,cAAc,MAAM,SAAS;AAAA,IAC7B,MAAM,MAAM,SAAS;AAAA,IACrB,WAAW,MAAM,SAAS;AAAA,IAC1B,SAAS,MAAM,SAAS;AAAA,IACxB,YAAY,MAAM,SAAS;AAAA,IAC3B,YAAY,MAAM,UAAU;AAAA,IAC5B,YAAY,MAAM,SAAS,MAAM;AAAA,IACjC,aAAa,MAAM,SAAS,WAAW;AAAA,IACvC,aAAa,MAAM,SAAS,WAAW;AAAA,IACvC,cAAc,MAAM,SAAS,WAAW;AAAA,IACxC,QAAQ,gBAAgB,YAAY,CAAC,QAAQ,IAAI,UAAU,YAAY,WAAW;AAAA,MAChF;AAAA,MACA;AAAA,MACA;AAAA,IACF,CAAC;AAAA,IACD,MAAM,gBAAgB,YAAY,CAAC,QAAQ,IAAI,UAAU,QAAQ,CAAC,CAAC;AAAA,IACnE,YAAY,oBAAoB,WAAW,IAAI,CAAC,QAAQ,IAAI,UAAU,CAAC;AAAA,IACvE,OAAO,aAAa,WAAW;AAAA,IAC/B,SAAS,aAAa,WAAW,IAAI,CAAC,QAAQ,IAAI,KAAK,OAAO,CAAC;AAAA,IAC/D,WAAW,aAAa,WAAW,IAAI,CAAC,QAAQ,IAAI,KAAK,UAAU,CAAC;AAAA,EACtE;AACF;AAEO,SAAS,8BAA8B,QAAiC;AAC7E,QAAM,YAAY,OAAO,QAAQ,OAAO,MAAM,EAC3C,IAAI,CAAC,CAAC,OAAO,OAAO,MAAM;AACzB,WAAO,KAAK,KAAK,MAAM,QAAQ,CAAC,MAAM,IAAI,QAAQ,SAAS,CAAC,MAAM,IAAI,QAAQ,QAAQ,CAAC,MAAM,IAAI,QAAQ,MAAM,GAAG,CAAC,MAAM,IAAI,QAAQ,QAAQ,IAAI,CAAC,MAAM,IAAI,QAAQ,UAAU,GAAG,CAAC;AAAA,EACpL,CAAC,EACA,KAAK,IAAI;AACZ,QAAM,UAAU,OAAO,QAAQ,OAAO,UAAU,EAC7C,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC,EACrC,IAAI,CAAC,CAAC,KAAK,IAAI,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,MAAM,IAAI,KAAK,IAAI,CAAC,MAAM,IAAI,KAAK,GAAG,CAAC,IAAI,EACpF,KAAK,IAAI;AACZ,SAAO;AAAA,IACL,uBAAuB,OAAO,WAAW;AAAA,IACzC;AAAA,IACA,aAAa,OAAO,MAAM;AAAA,IAC1B,gBAAgB,OAAO,QAAQ;AAAA,IAC/B,cAAc,OAAO,MAAM;AAAA,IAC3B,eAAe,OAAO,YAAY;AAAA,IAClC,YAAY,OAAO,UAAU,WAAW,OAAO,WAAW,YAAY,OAAO,WAAW;AAAA,IACxF,YAAY,IAAI,OAAO,YAAY,CAAC;AAAA,IACpC,iBAAiB,IAAI,OAAO,MAAM,IAAI,CAAC,YAAY,IAAI,OAAO,MAAM,MAAM,CAAC,SAAS,IAAI,OAAO,MAAM,GAAG,CAAC,OAAO,OAAO,MAAM,CAAC;AAAA,IAC9H;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,aAAa;AAAA,IACb;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA,WAAW;AAAA,IACX;AAAA,EACF,EAAE,KAAK,IAAI;AACb;AAEA,SAAS,eAAeD,UAAuE;AAC7F,SAAOA,SAAQ,MAAM,GAAGA,SAAQ,UAAU,QAAQ,IAAIA,SAAQ,YAAY,QAAQ;AACpF;AAEA,SAAS,8BAA8B,YAAyD;AAC9F,QAAM,aAAqC,EAAE,OAAOC,SAAQ,WAAW,KAAK,EAAE;AAC9E,aAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,WAAW,cAAc,CAAC,CAAC,GAAG;AACtE,QAAI,OAAO,SAAS,KAAK,EAAG,YAAW,GAAG,IAAI;AAAA,EAChD;AACA,aAAW,SAAU,WAAW,UAAU,WAAW,QAAQ,IAAK,IAAI;AACtE,SAAO;AACT;AAEA,SAAS,oBACP,MACuC;AACvC,QAAM,SAAS,oBAAI,IAAsB;AACzC,aAAW,OAAO,MAAM;AACtB,eAAW,CAAC,KAAK,KAAK,KAAK,OAAO,QAAQ,GAAG,GAAG;AAC9C,UAAI,CAAC,OAAO,SAAS,KAAK,EAAG;AAC7B,YAAM,OAAO,OAAO,IAAI,GAAG,KAAK,CAAC;AACjC,WAAK,KAAK,KAAK;AACf,aAAO,IAAI,KAAK,IAAI;AAAA,IACtB;AAAA,EACF;AACA,SAAO,OAAO,YAAY,CAAC,GAAG,OAAO,QAAQ,CAAC,EAAE,IAAI,CAAC,CAAC,KAAK,IAAI,MAAM,CAAC,KAAK,aAAa,IAAI,CAAC,CAAC,CAAC;AACjG;AAEA,SAAS,gBACP,MACA,OACA,YAA+B,CAAC,GACO;AACvC,QAAM,UAAU,oBAAI,IAAiB;AACrC,aAAW,OAAO,UAAW,SAAQ,IAAI,KAAK,CAAC,CAAC;AAChD,aAAW,OAAO,MAAM;AACtB,UAAM,OAAO,MAAM,GAAG;AACtB,eAAW,OAAO,MAAM,QAAQ,IAAI,IAAI,OAAO,CAAC,IAAI,GAAG;AACrD,YAAM,OAAO,QAAQ,IAAI,GAAG,KAAK,CAAC;AAClC,WAAK,KAAK,GAAG;AACb,cAAQ,IAAI,KAAK,IAAI;AAAA,IACvB;AAAA,EACF;AACA,QAAM,MAA6C,CAAC;AACpD,aAAW,CAAC,KAAK,IAAI,KAAK,SAAS;AACjC,UAAM,YAAY;AAKlB,QAAI,GAAG,IAAI;AAAA,MACT,GAAG,KAAK;AAAA,MACR,WAAW,KAAK,UAAU,IAAI,CAAC,QAAQ,IAAI,KAAK,CAAC;AAAA,MACjD,UAAU,KAAK,UAAU,IAAI,CAAC,QAAS,IAAI,SAAS,IAAI,CAAE,CAAC;AAAA,MAC3D,OAAO,aAAa,UAAU,IAAI,CAAC,QAAQ,IAAI,KAAK,CAAC;AAAA,MACrD,SAAS,aAAa,UAAU,IAAI,CAAC,QAAQ,IAAI,KAAK,OAAO,CAAC;AAAA,MAC9D,WAAW,aAAa,UAAU,IAAI,CAAC,QAAQ,IAAI,KAAK,UAAU,CAAC;AAAA,IACrE;AAAA,EACF;AACA,SAAO;AACT;AAEA,SAAS,gBACP,aACwB;AACxB,SAAO,OAAO,OAAO,WAAW,EAAE,CAAC;AACrC;AAEA,SAAS,aAAa,QAAkD;AACtE,QAAM,SAAS,CAAC,GAAG,MAAM,EAAE,OAAO,OAAO,QAAQ,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AACvE,MAAI,OAAO,WAAW,EAAG,QAAO,EAAE,GAAG,GAAG,KAAK,GAAG,MAAM,GAAG,QAAQ,GAAG,KAAK,GAAG,KAAK,EAAE;AACnF,SAAO;AAAA,IACL,GAAG,OAAO;AAAA,IACV,KAAK,OAAO,CAAC;AAAA,IACb,MAAM,KAAK,MAAM;AAAA,IACjB,QAAQ,WAAW,QAAQ,GAAG;AAAA,IAC9B,KAAK,WAAW,QAAQ,GAAG;AAAA,IAC3B,KAAK,OAAO,OAAO,SAAS,CAAC;AAAA,EAC/B;AACF;AAEA,SAAS,WAAW,cAAiC,GAAmB;AACtE,MAAI,aAAa,WAAW,EAAG,QAAO;AACtC,QAAM,QAAQ,KAAK;AAAA,IACjB,aAAa,SAAS;AAAA,IACtB,KAAK,IAAI,GAAG,KAAK,KAAK,IAAI,aAAa,MAAM,IAAI,CAAC;AAAA,EACpD;AACA,SAAO,aAAa,KAAK;AAC3B;AAEA,SAAS,KAAK,QAAmC;AAC/C,QAAM,SAAS,OAAO,OAAO,OAAO,QAAQ;AAC5C,MAAI,OAAO,WAAW,EAAG,QAAO;AAChC,SAAO,OAAO,OAAO,CAAC,KAAK,UAAU,MAAM,OAAO,CAAC,IAAI,OAAO;AAChE;AAEA,SAASA,SAAQ,OAAuB;AACtC,MAAI,CAAC,OAAO,SAAS,KAAK,EAAG,QAAO;AACpC,MAAI,QAAQ,EAAG,QAAO;AACtB,MAAI,QAAQ,EAAG,QAAO;AACtB,SAAO;AACT;AAEA,SAAS,IAAI,OAAuB;AAClC,MAAI,CAAC,OAAO,SAAS,KAAK,EAAG,QAAO;AACpC,SAAO,MAAM,QAAQ,UAAU,KAAK,KAAK,IAAI,KAAK,KAAK,KAAK,IAAI,CAAC;AACnE;;;ACzSO,SAAS,eAA4B,MAAmB;AAC7D,SAAO,KACJ,MAAM,OAAO,EACb,IAAI,CAAC,SAAS,KAAK,KAAK,CAAC,EACzB,OAAO,OAAO,EACd,IAAI,CAAC,MAAM,UAAU;AACpB,QAAI;AACF,aAAO,KAAK,MAAM,IAAI;AAAA,IACxB,SAAS,OAAO;AACd,YAAM,IAAI,MAAM,qBAAqB,QAAQ,CAAC,KAAM,MAAgB,OAAO,EAAE;AAAA,IAC/E;AAAA,EACF,CAAC;AACL;AAEO,SAAS,aAAa,MAA0B;AACrD,SAAO,KACJ,MAAM,OAAO,EACb,IAAI,CAAC,SAAS,KAAK,KAAK,CAAC,EACzB,OAAO,OAAO,EACd,OAAO,CAAC,SAAS,CAAC,KAAK,WAAW,GAAG,CAAC,EACtC,IAAI,CAAC,SAAS,KAAK,MAAM,QAAQ,CAAC;AACvC;AAEO,SAAS,WAAW,MAAuC;AAChE,SAAO,aAAa,IAAI,EAAE,QAAQ,CAAC,OAAO,UAAU;AAClD,QAAI,MAAM,SAAS,EAAG,QAAO,CAAC;AAC9B,UAAM,CAAC,SAAS,kBAAkB,mBAAmB,UAAU,IAAI;AACnE,QAAI,CAAC,WAAW,CAAC,oBAAoB,CAAC,kBAAmB,QAAO,CAAC;AACjE,QAAI,QAAQ,YAAY,MAAM,cAAc,QAAQ,YAAY,MAAM,MAAO,QAAO,CAAC;AACrF,UAAM,aAAa,eAAe,SAAY,mBAAmB;AACjE,UAAM,YAAY,eAAe,SAAY,oBAAoB;AACjE,UAAM,QAAQ,OAAO,SAAS;AAC9B,QAAI,CAAC,cAAc,CAAC,OAAO,SAAS,KAAK,GAAG;AAC1C,YAAM,IAAI,MAAM,qBAAqB,QAAQ,CAAC,oCAAoC;AAAA,IACpF;AACA,WAAO,CAAC,EAAE,SAAS,YAAY,MAAM,CAAC;AAAA,EACxC,CAAC;AACH;AAEO,SAAS,qBAAqB,MAA2C;AAC9E,SAAO,eAAwC,IAAI,EAAE,IAAI,CAAC,KAAK,UAAU;AACvE,UAAM,KAAK,YAAY,KAAK,KAAK,KAAK,YAAY,KAAK,IAAI;AAC3D,UAAM,OAAO,YAAY,KAAK,MAAM,KAAK,YAAY,KAAK,UAAU;AACpE,QAAI,CAAC,MAAM,SAAS,QAAW;AAC7B,YAAM,IAAI,MAAM,2BAA2B,QAAQ,CAAC,qCAAqC;AAAA,IAC3F;AACA,WAAO;AAAA,MACL;AAAA,MACA,OAAO,YAAY,KAAK,OAAO;AAAA,MAC/B,MAAM;AAAA,MACN,UAAU,WAAW,KAAK,CAAC,OAAO,MAAM,SAAS,QAAQ,UAAU,CAAC;AAAA,IACtE;AAAA,EACF,CAAC;AACH;AAEO,SAAS,sBAAsB,MAAwC;AAC5E,SAAO,eAAwC,IAAI,EAAE,IAAI,CAAC,KAAK,UAAU;AACvE,UAAM,KAAK,YAAY,KAAK,KAAK,KAAK,YAAY,KAAK,IAAI,KAAK,YAAY,KAAK,UAAU;AAC3F,UAAM,QAAQ,YAAY,KAAK,MAAM,KAAK,YAAY,KAAK,OAAO;AAClE,QAAI,CAAC,MAAM,CAAC,OAAO;AACjB,YAAM,IAAI,MAAM,0BAA0B,QAAQ,CAAC,kCAAkC;AAAA,IACvF;AACA,WAAO;AAAA,MACL;AAAA,MACA,MAAM;AAAA,MACN,UAAU,WAAW,KAAK,CAAC,OAAO,MAAM,YAAY,QAAQ,OAAO,CAAC;AAAA,IACtE;AAAA,EACF,CAAC;AACH;AAEO,SAAS,4BACd,SACuD;AACvD,QAAM,eAAe,oBAAI,IAAqC;AAC9D,aAAW,QAAQ,QAAQ,OAAO;AAChC,QAAI,KAAK,SAAS,EAAG;AACrB,UAAM,OAAO,aAAa,IAAI,KAAK,OAAO,KAAK,CAAC;AAChD,SAAK,KAAK,IAAI;AACd,iBAAa,IAAI,KAAK,SAAS,IAAI;AAAA,EACrC;AACA,QAAM,SACJ,QAAQ,0BAA0B,QAAQ,SACtC,OAAO,YAAY,QAAQ,OAAO,IAAI,CAAC,aAAa,CAAC,SAAS,IAAI,QAAQ,CAAC,CAAC,IAC5E;AACN,SAAO,QAAQ,QAAQ,QAAQ,CAAC,UAAU;AACxC,UAAM,QAAQ,aAAa,IAAI,MAAM,EAAE,KAAK,CAAC;AAC7C,QAAI,MAAM,WAAW,EAAG,QAAO,CAAC;AAChC,UAAM,QACJ,QAAQ,UAAU,MAAM,EAAE,KAAK,mBAAmB,GAAG,QAAQ,WAAW,IAAI,MAAM,EAAE,EAAE;AACxF,WAAO;AAAA,MACL;AAAA,QACE,IAAI,MAAM;AAAA,QACV;AAAA,QACA,QAAQ,QAAQ;AAAA,QAChB,UAAU;AAAA,QACV,MAAM,CAAC,GAAG,oBAAI,IAAI,CAAC,GAAI,QAAQ,QAAQ,CAAC,GAAI,KAAK,CAAC,CAAC;AAAA,QACnD,GAAI,QAAQ,SAAS,EAAE,QAAQ,QAAQ,OAAO,IAAI,CAAC;AAAA,QACnD,GAAI,MAAM,WAAW,EAAE,UAAU,MAAM,SAAS,IAAI,CAAC;AAAA,QACrD,SAAS;AAAA,UACP,SAAS,MAAM;AAAA,UACf,OAAO,MAAM;AAAA,UACb,qBAAqB,MAAM,IAAI,CAAC,SAAS,KAAK,UAAU;AAAA,UACxD,gBAAgB,OAAO,YAAY,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,YAAY,KAAK,KAAK,CAAC,CAAC;AAAA,UACrF,GAAI,SAAS,EAAE,OAAO,IAAI,CAAC;AAAA,UAC3B,GAAI,MAAM,WAAW,EAAE,UAAU,MAAM,SAAS,IAAI,CAAC;AAAA,QACvD;AAAA,MACF;AAAA,IACF;AAAA,EACF,CAAC;AACH;AAEO,SAAS,kCACd,SAKA;AACA,QAAM,QAAQ,4BAA4B,OAAO;AACjD,SAAO;AAAA,IACL,IAAI,QAAQ;AAAA,IACZ,QAAQ,QAAQ;AAAA,IAChB,UAAU;AAAA,IACV,QAAQ,QAAQ;AAAA,IAChB,eAAe,QAAQ,iBAAiB;AAAA,IACxC,MAAM,YAAY,OAAO;AACvB,aAAO,MAAM,OAAO,CAAC,SAAS,KAAK,UAAU,KAAK;AAAA,IACpD;AAAA,IACA,MAAM,SAAS,MAAM,UAAU;AAC7B,aAAO,0BAA0B,KAAK,SAAS,UAAU,OAAO;AAAA,IAClE;AAAA,IACA,YAAY,QAAQ;AAClB,aAAO,QAAQ,UAAU,MAAM,KAAK,mBAAmB,GAAG,QAAQ,WAAW,IAAI,MAAM,EAAE;AAAA,IAC3F;AAAA,EACF;AACF;AAEO,SAAS,0BACd,SACA,UACA,UAA8C,CAAC,GAC/C;AACA,QAAM,oBAAoB;AAAA,IACxB;AAAA,IACA,QAAQ,qBAAqB;AAAA,EAC/B;AACA,QAAM,UAAU,iBAAiB,QAAQ,WAAW,CAAC,GAAG,GAAG,GAAG,EAAE,CAAC;AACjE,QAAM,aAAqC;AAAA,IACzC,gBAAgB,QAAQ,oBAAoB;AAAA,IAC5C,gBAAgB,kBAAkB;AAAA,EACpC;AACA,aAAW,UAAU,SAAS;AAC5B,WAAO;AAAA,MACL;AAAA,MACA,yBAAyB;AAAA,QACvB;AAAA,QACA,gBAAgB,QAAQ;AAAA,QACxB;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AACA,QAAM,gBAAgB,QAAQ,iBAAiB;AAC/C,QAAM,aAAa,QAAQ,cAAc;AACzC,QAAM,QAAQ,WAAW,aAAa,KAAK,WAAW,QAAQ,QAAQ,QAAQ,SAAS,CAAC,CAAC,EAAE,KAAK;AAChG,QAAM,YACJ,WAAW,UAAU,KAAK,WAAW,OAAO,QAAQ,QAAQ,SAAS,CAAC,CAAC,EAAE,KAAK;AAChF,SAAO;AAAA,IACL;AAAA,IACA,QAAQ,cAAc,QAAQ,iBAAiB;AAAA,IAC/C;AAAA,IACA,KAAK;AAAA,MACH;AAAA,MACA,qBAAqB,QAAQ;AAAA,MAC7B,gBAAgB,QAAQ;AAAA,IAC1B;AAAA,EACF;AACF;AAEO,SAAS,8BACd,UACA,oBAA4B,sBAClB;AACV,MAAI,OAAO,aAAa,UAAU;AAChC,WAAO,eAAe,SAAS,MAAM,iBAAiB,KAAK,CAAC,GAAG,IAAI,CAAC,UAAU,MAAM,KAAK,CAAC,CAAC;AAAA,EAC7F;AACA,MAAI,MAAM,QAAQ,QAAQ,GAAG;AAC3B,UAAM,UAAU;AAChB,WAAO;AAAA,MACL,QAAQ,QAAQ,CAAC,UAAU;AACzB,YAAI,OAAO,UAAU,SAAU,QAAO,CAAC,KAAK;AAC5C,eAAO,CAAC,MAAM,cAAc,MAAM,SAAS,MAAM,MAAM,EAAE;AAAA,MAC3D,CAAC;AAAA,IACH;AAAA,EACF;AACA,QAAM,iBAAiB;AACvB,SAAO;AAAA,IACL;AAAA,MACE,GAAI,eAAe,eAAe,CAAC;AAAA,MACnC,GAAI,eAAe,OAAO,CAAC;AAAA,MAC3B,IAAI,eAAe,WAAW,CAAC,GAAG;AAAA,QAChC,CAAC,UAAU,MAAM,cAAc,MAAM,SAAS,MAAM,MAAM;AAAA,MAC5D;AAAA,IACF,EAAE,IAAI,CAAC,UAAU,MAAM,KAAK,CAAC;AAAA,EAC/B;AACF;AAEO,SAAS,yBAAyB,OAId;AACzB,QAAM,MAAM,MAAM,kBAAkB,MAAM,GAAG,MAAM,MAAM;AACzD,QAAM,cAAc,OAAO,KAAK,MAAM,cAAc,EAAE;AAAA,IACpD,CAAC,OAAO,MAAM,eAAe,EAAE,IAAK;AAAA,EACtC;AACA,QAAM,WAAW,IAAI,IAAI,WAAW;AACpC,QAAM,OAAO,IAAI,OAAO,CAAC,OAAO,SAAS,IAAI,EAAE,CAAC,EAAE;AAClD,QAAM,oBAAoB,IAAI,UAAU,CAAC,OAAO,SAAS,IAAI,EAAE,CAAC;AAChE,QAAM,SAAS,IAAI,MAAM,MAAM;AAC/B,SAAO;AAAA,IACL,CAAC,MAAM,MAAM,EAAE,GAAG,OAAO,IAAI,IAAI;AAAA,IACjC,CAAC,SAAS,MAAM,EAAE,GAAG,SAAS,SAAS,IAAI,IAAI,OAAO,SAAS;AAAA,IAC/D,CAAC,YAAY,MAAM,EAAE,GAAG,OAAO,MAAM;AAAA,IACrC,CAAC,MAAM,MAAM,EAAE,GAAG,sBAAsB,KAAK,IAAI,KAAK,oBAAoB;AAAA,IAC1E,CAAC,OAAO,MAAM,EAAE,GAAG,OAAO,MAAM,mBAAmB,MAAM,gBAAgB,MAAM,MAAM;AAAA,EACvF;AACF;AAEA,SAAS,YAAY,KAA8B,KAAiC;AAClF,QAAM,QAAQ,IAAI,GAAG;AACrB,SAAO,OAAO,UAAU,WAAW,QAAQ;AAC7C;AAEA,SAAS,WACP,KACA,MACyB;AACzB,QAAM,QAAQ,IAAI,IAAI,IAAI;AAC1B,SAAO,OAAO,YAAY,OAAO,QAAQ,GAAG,EAAE,OAAO,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,IAAI,GAAG,CAAC,CAAC;AAClF;AAEA,SAAS,iBAAiB,SAAsC;AAC9D,SAAO;AAAA,IACL,GAAG,IAAI,IAAI,QAAQ,IAAI,CAAC,WAAW,KAAK,MAAM,MAAM,CAAC,EAAE,OAAO,CAAC,WAAW,SAAS,CAAC,CAAC;AAAA,EACvF,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AACxB;AAEA,SAAS,cAAc,QAAqC;AAC1D,QAAM,OAAO,oBAAI,IAAY;AAC7B,QAAM,MAAgB,CAAC;AACvB,aAAW,SAAS,QAAQ;AAC1B,UAAM,UAAU,MAAM,KAAK;AAC3B,QAAI,CAAC,WAAW,KAAK,IAAI,OAAO,EAAG;AACnC,SAAK,IAAI,OAAO;AAChB,QAAI,KAAK,OAAO;AAAA,EAClB;AACA,SAAO;AACT;AAEA,SAAS,OACP,mBACA,gBACA,QACQ;AACR,QAAM,MAAM,kBACT,MAAM,GAAG,MAAM,EACf;AAAA,IACC,CAAC,KAAK,YAAY,UAAU,MAAM,eAAe,eAAe,UAAU,KAAK,GAAG,KAAK;AAAA,IACvF;AAAA,EACF;AACF,QAAM,QAAQ,OAAO,OAAO,cAAc,EACvC,OAAO,CAAC,UAAU,QAAQ,CAAC,EAC3B,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC,EACpB,MAAM,GAAG,MAAM,EACf,OAAO,CAAC,KAAK,OAAO,UAAU,MAAM,eAAe,OAAO,KAAK,GAAG,CAAC;AACtE,SAAO,UAAU,IAAI,IAAI,MAAM;AACjC;AAEA,SAAS,eAAe,WAAmB,eAA+B;AACxE,MAAI,aAAa,EAAG,QAAO;AAC3B,UAAQ,KAAK,YAAY,KAAK,KAAK,KAAK,gBAAgB,CAAC;AAC3D;","names":["adapter","clamp01"]}
|
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
import {
|
|
2
|
-
planCampaignRun,
|
|
3
|
-
runCampaign
|
|
4
|
-
} from "./chunk-EZJEIH2R.js";
|
|
5
|
-
import "./chunk-2MKQIFS4.js";
|
|
6
|
-
import "./chunk-MHELPNRP.js";
|
|
7
|
-
import "./chunk-WS3NZZQQ.js";
|
|
8
|
-
import "./chunk-VI2UW6B6.js";
|
|
9
|
-
import "./chunk-2JX3CFMB.js";
|
|
10
|
-
import "./chunk-MA6HLL3S.js";
|
|
11
|
-
import "./chunk-ONWEPEDO.js";
|
|
12
|
-
import "./chunk-PZ5AY32C.js";
|
|
13
|
-
export {
|
|
14
|
-
planCampaignRun,
|
|
15
|
-
runCampaign
|
|
16
|
-
};
|
|
17
|
-
//# sourceMappingURL=run-campaign-ISHFZ7FJ.js.map
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|