npm - @tangle-network/agent-eval - Versions diffs - 1.0.25 → 1.0.26 - Mend

@tangle-network/agent-eval 1.0.25 → 1.0.26

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (14) hide show

package/CHANGELOG.md +45 -0
package/dist/app-grade/index.d.ts +1 -1
package/dist/benchmarks/index.d.ts +28 -3
package/dist/benchmarks/index.d.ts.map +1 -1
package/dist/benchmarks/index.js +102 -14
package/dist/benchmarks/index.js.map +1 -1
package/dist/{index-JPI2spOx.d.ts → index-C8a9Dyan.d.ts} +2 -2
package/dist/{index-JPI2spOx.d.ts.map → index-C8a9Dyan.d.ts.map} +1 -1
package/dist/index.d.ts +2 -2
package/dist/openapi.json +1 -1
package/dist/{types-D7g2XFTc.d.ts → types-CEgIBy0W.d.ts} +4 -3
package/dist/{types-D7g2XFTc.d.ts.map → types-CEgIBy0W.d.ts.map} +1 -1
package/dist/wire/index.d.ts +4 -4
package/package.json +1 -1

package/CHANGELOG.md CHANGED Viewed

@@ -4,6 +4,51 @@ All notable changes to `@tangle-network/agent-eval` and its sibling `agent-eval-
 ---
+## [1.0.26] — 2026-11-08
+- feat(benchmarks): grade a code patch with a Harbor task's own verifier
+- chore(release): 1.0.25
+- fix(judges): default to zai/glm-5.3, which every restricted Router key may call
+- chore(release): 1.0.24
+- test(multishot): judge default model and its price in runJudge
+- test(multishot): price judge calls at the new default judge model
+- fix(judges): one judge default, gpt-6-luna, which the Router serves
+- chore(release): 1.0.23
+- fix(run-record): a Claude id from 4.6 on is a pinned snapshot
+- feat(adopt): expose scoped consumer in managed adoption
+- fix(adopt): stage selected peer cohort before install
+- feat(search): an objective key names the judge revision
+- feat(adopt): scope programmatic adoption to one consumer
+- feat: size a sequential pair cap under a registered law (#1223)
+- fix(test): bound shared release verification workers (#1225)
+- fix(test): qualify refreshed harness contracts with maintained pnpm (#1224)
+- chore(release): 1.0.22 (#1222)
+- chore(deps): refresh first-party library dependencies (#1221)
+- docs(release): describe independent interval API and drop duplicate bumps
+- chore(release): 1.0.21
+- docs(test): cite printed Newcombe table pages
+- feat(statistics): add independent risk differences from Wilson limits
+- fix(test): mark disposable identities for host Git hooks
+- fix(test): declare identity in disposable integration repositories
+- fix(release): authenticate public evidence reads in verification
+- fix(statistics): exclude steps through every selected cluster extent
+- test(statistics): retain overlapping-cluster sealed-step regression
+- docs(release): remove redundant release commit entries
+- chore(release): 1.0.20
+- docs(statistics): trim whitespace in interval guidance
+- fix(verification): retain valid replay baselines and current release evidence
+- chore(analyst): refresh manifest for shared statistics owner
+- fix(statistics): preserve degenerate bootstrap results and mean seed
+- feat(statistics): share stratified cluster resampling and calibrated placebo bands
+- fix(adoption): retain unselected pnpm resolutions
+- fix(exports): ship ./statistics subpath, gpt-5-mini pricing, sequentialCrossingHorizon
+- fix(adopt-stack): qualify the committed caller tree
+- fix(adopt-stack): read large tracked path inventories
+- fix(adoption): read a Runtime release's evidence from its Publish run, and raise pins below a dependency floor (#1211)
+- fix(adoption): hold a Runtime release whose tag evidence failed or never ran
+- fix(adoption): gate stable-major candidates against installed versions
+- feat(app-grade): probes take a method and JSON body (#1208)
 ## [1.0.25] — 2026-11-08
 - fix(judges): default to zai/glm-5.3, which every restricted Router key may call

package/dist/app-grade/index.d.ts CHANGED Viewed

@@ -1,5 +1,5 @@
 import { m as CustomTokenPricing, t as CostLedger } from "../cost-ledger-Czro9wmo.js";
-import "../index-JPI2spOx.js";
+import "../index-C8a9Dyan.js";
 import { O as ChatClient, j as ChatTransport } from "../cluster-CSd0LE_M.js";
 import { d as SystemOneRequest, g as SystemOneState, s as SystemOneQuestions } from "../systemone-protocol-C4t7JH4Y.js";
 import "../systemone-CSwi1hp-.js";

package/dist/benchmarks/index.d.ts CHANGED Viewed

@@ -1,8 +1,8 @@
 import { c as RunSplitTag } from "../run-record-pBlhZizT.js";
 import "../index-BI9ecONy.js";
-import { a as CampaignResult } from "../types-ByrbgALT.js";
+import { S as JudgeConfig, a as CampaignResult } from "../types-ByrbgALT.js";
 import { Wt as CampaignStorage } from "../search-kernel-C4XnLKIW.js";
-import { a as BenchmarkFamily, c as BenchmarkSource, i as BenchmarkEvaluation, l as BenchmarkTaskKind, n as BenchmarkAdapter, o as BenchmarkResponder, r as BenchmarkDatasetItem, s as BenchmarkScenario, t as BENCHMARK_SPLIT_SEED, u as deterministicSplit } from "../types-D7g2XFTc.js";
+import { a as BenchmarkFamily, c as BenchmarkSource, i as BenchmarkEvaluation, l as BenchmarkTaskKind, n as BenchmarkAdapter, o as BenchmarkResponder, r as BenchmarkDatasetItem, s as BenchmarkScenario, t as BENCHMARK_SPLIT_SEED, u as deterministicSplit } from "../types-CEgIBy0W.js";
 interface BenchmarkMetricCalibrationOptions<TPayload = unknown, TArtifact = string> {
   adapter: BenchmarkAdapter<BenchmarkDatasetItem<TPayload>, TPayload, TArtifact>;
   item: BenchmarkDatasetItem<TPayload>;
@@ -88,6 +88,29 @@ interface HarborBenchmarkAdapter<TArtifact> extends BenchmarkAdapter<HarborTaskI
 }
 /** Import a Harbor dataset checkout as a `BenchmarkAdapter` whose scorer is the dataset's own verifier. */
 export declare function createHarborBenchmarkAdapter<TArtifact = string>(options: HarborBenchmarkOptions<TArtifact>): HarborBenchmarkAdapter<TArtifact>;
+/**
+ * `retries` counts extra runs after one with no reward (default 2); `maxDetailChars` bounds
+ * `JSON.stringify(detail).length` (default 2000); `timeoutMs` bounds one `harbor run`; `keepImage`
+ * (default true) passes `--no-delete` so the next grade reuses the built task image.
+ */
+interface HarborPatchGradeOptions {
+  harborBin?: string;
+  jobsDir?: string;
+  retries?: number;
+  maxDetailChars?: number;
+  timeoutMs?: number;
+  env?: Record<string, string>;
+  keepImage?: boolean;
+}
+/**
+ * A `grade` callback that scores `{ patch }` with the task's own verifier: Harbor's oracle agent
+ * runs a `solution/solve.sh` that applies the patch (`git apply --binary`) in the environment's
+ * working directory, on a copy of the task. An empty patch changes nothing. A run with no reward
+ * is retried, then reported as `succeeded: false`, never as a reward of 0.
+ */
+export declare function gradeHarborPatch(options?: HarborPatchGradeOptions): (item: HarborTaskItem, artifact: {
+  patch: string;
+}) => Promise<HarborGradeOutcome>;
 /**
  * Synthetic routing dataset. 16 tasks across 4 categories. Used as a
  * deterministic, dependency-free benchmark for any router that maps a
@@ -209,6 +232,8 @@ interface BenchmarkRunResult<TPayload = unknown, TArtifact = string> {
   reportMarkdownPath: string;
 }
 export declare function runBenchmarkAdapter<TPayload = unknown, TArtifact = string>(options: BenchmarkRunOptions<TPayload, TArtifact>): Promise<BenchmarkRunResult<TPayload, TArtifact>>;
+export declare function benchmarkScenarios<TPayload>(adapter: BenchmarkAdapter<BenchmarkDatasetItem<TPayload>, TPayload, unknown>, splits?: readonly RunSplitTag[]): Promise<Array<BenchmarkScenario<TPayload>>>;
+export declare function benchmarkJudge<TPayload, TArtifact>(adapter: BenchmarkAdapter<BenchmarkDatasetItem<TPayload>, TPayload, TArtifact>): JudgeConfig<TArtifact, BenchmarkScenario<TPayload>>;
 interface StandardRetrievalDocument {
   id: string;
   title?: string;
@@ -284,5 +309,5 @@ export declare function evaluateStandardRetrieval(payload: StandardRetrievalPayl
     expectedScores: Record<string, number>;
   };
 };
-export { BENCHMARK_SPLIT_SEED, type BenchmarkAdapter, type BenchmarkDatasetItem, type BenchmarkDistribution, type BenchmarkEvaluation, type BenchmarkFamily, type BenchmarkMetricCalibrationOptions, type BenchmarkMetricCalibrationResult, type BenchmarkReport, type BenchmarkResponder, type BenchmarkRunOptions, type BenchmarkRunResult, type BenchmarkScenario, type BenchmarkSliceSummary, type BenchmarkSource, type BenchmarkTaskKind, type BuildStandardRetrievalItemsOptions, type HarborBenchmarkAdapter, type HarborBenchmarkOptions, type HarborGradeOutcome, type HarborTaskItem, type HarborTaskPayload, type RetrievalIdAdapterOptions, type StandardRetrievalArtifact, type StandardRetrievalDocument, type StandardRetrievalEvaluationOptions, type StandardRetrievalPayload, type StandardRetrievalQrel, type StandardRetrievalQuery, type StandardRetrievalResult, deterministicSplit, index_d_exports as routing };
+export { BENCHMARK_SPLIT_SEED, type BenchmarkAdapter, type BenchmarkDatasetItem, type BenchmarkDistribution, type BenchmarkEvaluation, type BenchmarkFamily, type BenchmarkMetricCalibrationOptions, type BenchmarkMetricCalibrationResult, type BenchmarkReport, type BenchmarkResponder, type BenchmarkRunOptions, type BenchmarkRunResult, type BenchmarkScenario, type BenchmarkSliceSummary, type BenchmarkSource, type BenchmarkTaskKind, type BuildStandardRetrievalItemsOptions, type HarborBenchmarkAdapter, type HarborBenchmarkOptions, type HarborGradeOutcome, type HarborPatchGradeOptions, type HarborTaskItem, type HarborTaskPayload, type RetrievalIdAdapterOptions, type StandardRetrievalArtifact, type StandardRetrievalDocument, type StandardRetrievalEvaluationOptions, type StandardRetrievalPayload, type StandardRetrievalQrel, type StandardRetrievalQuery, type StandardRetrievalResult, deterministicSplit, index_d_exports as routing };
 //# sourceMappingURL=index.d.ts.map

package/dist/benchmarks/index.d.ts.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"index.d.ts","names":[],"sources":["../../src/benchmarks/calibration.ts","../../src/benchmarks/harbor.ts","../../src/benchmarks/routing/dataset.ts","../../src/benchmarks/routing/index.ts","../../src/benchmarks/runner.ts","../../src/benchmarks/standard-formats.ts"],"mappings":";;;;;UAEiB,kCAAkC,oBAAoB;EACrE,SAAS,iBAAiB,qBAAqB,WAAW,UAAU;EACpE,MAAM,qBAAqB;EAC3B,cAAc;EACd,gBAAgB;EAChB;EACA;EACA;;UAGe;EACf;EACA,MAAM;EACN,QAAQ;EACR;EACA;EACA;EACA;;wBAGoB,yBAAyB,oBAAoB,oBACjE,SAAS,kCAAkC,UAAU,aACpD,QAAQ;;~~UCKM~~;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;IACE;IACA;IACA;IACA;;;EAGF;;EAEA;;EAEA;;EAEA;;KAGU,iBAAiB,qBAAqB;;KAGtC;EACN;EAAiB;IAAS;IAAgB;;;EAC1C;EAAkB;;UAEP,uBAAuB;;EAEtC;;EAEA;;EAEA,QAAQ;IAAoB;;;EAE5B;;EAEA,QAAQ,MAAM,gBAAgB,UAAU,cAAc,QAAQ;;EAE9D,eAAe,mBAAmB;;;UAInB,uBAAuB,mBAC9B,iBAAiB,gBAAgB,mBAAmB;EAC5D,WAAW,QAAQ;;;wBA0DL,6BAA6B,oBAC3C,SAAS,uBAAuB,aAC/B,uBAAuB;;;;;;;;;;;;;;;;;;;;;;~~UC5HT~~;EACf;EACA;EACA;;EAEA;;EAEA;;EAEA;;cAGW,iBAAiB;;;;KChBlB,iBAAiB;KACjB,qBAAqB,qBAAqB;cAEhD,0BAA0B,iBAAiB,oBAAoB;WAC1D;WACA;WACA;WACA;WACA;EAEH,YAAY,OAAO,cAAc,QAAQ;EAMzC,SAAS,MAAM,oBAAoB,mBAAmB,QAAQ;EAqBpE,YAAY,iBAAiB;;;;;;;;iBAef,mBAAmB;cAOtB,cAAW,OAjDG,gBAAc,QAAQ;cAkDpC,WAAQ,MA5CE,oBAAkB,qBAAqB,QAAQ;cA6CzD,cAAW,mBAxBO;UChCd,oBAAoB,oBAAoB;EACvD,SAAS,iBAAiB,qBAAqB,WAAW,UAAU;EACpE,SAAS,mBAAmB,UAAU;EACtC,kBAAkB;EAClB;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA,UAAU;EACV,YAAY;;UAGG;EACf;EACA,QAAQ;EACR,UAAU;EACV,SAAS;EACT;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA,QAAQ,eAAe;EACvB,MAAM,eAAe;EACrB,YAAY,eAAe;EAC3B,OAAO;EACP,SAAS;EACT,WAAW;;UAGI;EACf;EACA;EACA;EACA,OAAO;EACP,SAAS;EACT,WAAW;;UAGI;EACf;EACA;EACA;EACA;EACA;EACA;;UAGe,mBAAmB,oBAAoB;EACtD,WAAW,MAAM,kBAAkB;EACnC,UAAU,eAAe,WAAW,kBAAkB;EACtD,QAAQ;EACR;EACA;;wBAGoB,oBAAoB,oBAAoB,oBAC5D,SAAS,oBAAoB,UAAU,aACtC,QAAQ,mBAAmB,UAAU;~~UClFvB~~;EACf;EACA;EACA;EACA,WAAW;;UAGI;EACf;EACA;EACA,WAAW;;UAGI;EACf;EACA;EACA;;UAGe;EACf;EACA;EACA;EACA,gBAAgB;EAChB,SAAS,eAAe;EACxB,WAAW;;UAGI;EACf;EACA,QAAQ;EACR,kBAAkB;EAClB,gBAAgB;EAChB,kBAAkB;EAClB;EACA,SAAS;EACT;EACA,WAAW,oBAAoB;;UAGhB,kCAAkC;EACjD,oBAAoB;EACpB;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA;EACA;;KAGU,kEAGC;EAEP;EACA;EACA,mBAAmB;;UAGR;EACf,oBAAoB;EACpB;EACA;EACA;EACA;;wBA0Bc,WAAW,eAAe;wBAgB1B,qBAAqB,eAAe;wBAgBpC,sBAAsB,eAAe;wBAerC,4BACd,SAAS,qCACR,MAAM,qBAAqB;wBAuCd,kCACd,SAAS,4BACR,iBACD,qBAAqB,2BACrB,0BACA;wBAqBc,0BACd,SAAS,0BACT,UAAU,2BACV,UAAS"}
1	+ {"version":3,"file":"index.d.ts","names":[],"sources":["../../src/benchmarks/calibration.ts","../../src/benchmarks/harbor.ts","../../src/benchmarks/routing/dataset.ts","../../src/benchmarks/routing/index.ts","../../src/benchmarks/runner.ts","../../src/benchmarks/standard-formats.ts"],"mappings":";;;;;UAEiB,kCAAkC,oBAAoB;EACrE,SAAS,iBAAiB,qBAAqB,WAAW,UAAU;EACpE,MAAM,qBAAqB;EAC3B,cAAc;EACd,gBAAgB;EAChB;EACA;EACA;;UAGe;EACf;EACA,MAAM;EACN,QAAQ;EACR;EACA;EACA;EACA;;wBAGoB,yBAAyB,oBAAoB,oBACjE,SAAS,kCAAkC,UAAU,aACpD,QAAQ;;UCWM;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;IACE;IACA;IACA;IACA;;;EAGF;;EAEA;;EAEA;;EAEA;;KAGU,iBAAiB,qBAAqB;;KAGtC;EACN;EAAiB;IAAS;IAAgB;;;EAC1C;EAAkB;;UAEP,uBAAuB;;EAEtC;;EAEA;;EAEA,QAAQ;IAAoB;;;EAE5B;;EAEA,QAAQ,MAAM,gBAAgB,UAAU,cAAc,QAAQ;;EAE9D,eAAe,mBAAmB;;;UAInB,uBAAuB,mBAC9B,iBAAiB,gBAAgB,mBAAmB;EAC5D,WAAW,QAAQ;;;wBA0DL,6BAA6B,oBAC3C,SAAS,uBAAuB,aAC/B,uBAAuB;;;;;;UAgFT;EACf;EACA;EACA;EACA;EACA;EACA,MAAM;EACN;;;;;;;;wBAoBc,iBACd,UAAS,2BACP,MAAM,gBAAgB;EAAY;MAAoB,QAAQ;;;;;;;;;;;;;;;;;;;;;;UC/OjD;EACf;EACA;EACA;;EAEA;;EAEA;;EAEA;;cAGW,iBAAiB;;;;KChBlB,iBAAiB;KACjB,qBAAqB,qBAAqB;cAEhD,0BAA0B,iBAAiB,oBAAoB;WAC1D;WACA;WACA;WACA;WACA;EAEH,YAAY,OAAO,cAAc,QAAQ;EAMzC,SAAS,MAAM,oBAAoB,mBAAmB,QAAQ;EAqBpE,YAAY,iBAAiB;;;;;;;;iBAef,mBAAmB;cAOtB,cAAW,OAjDG,gBAAc,QAAQ;cAkDpC,WAAQ,MA5CE,oBAAkB,qBAAqB,QAAQ;cA6CzD,cAAW,mBAxBO;UChCd,oBAAoB,oBAAoB;EACvD,SAAS,iBAAiB,qBAAqB,WAAW,UAAU;EACpE,SAAS,mBAAmB,UAAU;EACtC,kBAAkB;EAClB;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA,UAAU;EACV,YAAY;;UAGG;EACf;EACA,QAAQ;EACR,UAAU;EACV,SAAS;EACT;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA,QAAQ,eAAe;EACvB,MAAM,eAAe;EACrB,YAAY,eAAe;EAC3B,OAAO;EACP,SAAS;EACT,WAAW;;UAGI;EACf;EACA;EACA;EACA,OAAO;EACP,SAAS;EACT,WAAW;;UAGI;EACf;EACA;EACA;EACA;EACA;EACA;;UAGe,mBAAmB,oBAAoB;EACtD,WAAW,MAAM,kBAAkB;EACnC,UAAU,eAAe,WAAW,kBAAkB;EACtD,QAAQ;EACR;EACA;;wBAGoB,oBAAoB,oBAAoB,oBAC5D,SAAS,oBAAoB,UAAU,aACtC,QAAQ,mBAAmB,UAAU;wBA0ClB,mBAAmB,UACvC,SAAS,iBAAiB,qBAAqB,WAAW,oBAC1D,kBAAiB,gBAChB,QAAQ,MAAM,kBAAkB;wBA6BnB,eAAe,UAAU,WACvC,SAAS,iBAAiB,qBAAqB,WAAW,UAAU,aACnE,YAAY,WAAW,kBAAkB;UC9J3B;EACf;EACA;EACA;EACA,WAAW;;UAGI;EACf;EACA;EACA,WAAW;;UAGI;EACf;EACA;EACA;;UAGe;EACf;EACA;EACA;EACA,gBAAgB;EAChB,SAAS,eAAe;EACxB,WAAW;;UAGI;EACf;EACA,QAAQ;EACR,kBAAkB;EAClB,gBAAgB;EAChB,kBAAkB;EAClB;EACA,SAAS;EACT;EACA,WAAW,oBAAoB;;UAGhB,kCAAkC;EACjD,oBAAoB;EACpB;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA;EACA;;KAGU,kEAGC;EAEP;EACA;EACA,mBAAmB;;UAGR;EACf,oBAAoB;EACpB;EACA;EACA;EACA;;wBA0Bc,WAAW,eAAe;wBAgB1B,qBAAqB,eAAe;wBAgBpC,sBAAsB,eAAe;wBAerC,4BACd,SAAS,qCACR,MAAM,qBAAqB;wBAuCd,kCACd,SAAS,4BACR,iBACD,qBAAqB,2BACrB,0BACA;wBAqBc,0BACd,SAAS,0BACT,UAAU,2BACV,UAAS"}

package/dist/benchmarks/index.js CHANGED Viewed

@@ -3,8 +3,12 @@ import { at as summarizeNumberSeries, ct as mean } from "../statistics-CvvC-MPv.
 import { Z as runCampaign } from "../llm-judge-DzAT7Wka.js";
 import { M as fsCampaignStorage } from "../external-optimizer-subprocess-GO6s84Ov.js";
 import "../campaign-Cqz2vk2p.js";
+import { randomUUID } from "node:crypto";
 import { join, resolve } from "node:path";
-import { readFile, readdir, stat } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { execFile } from "node:child_process";
+import { cp, mkdir, mkdtemp, readFile, readdir, rm, stat, writeFile } from "node:fs/promises";
+import { promisify, stripVTControlCharacters } from "node:util";
 import { parse } from "smol-toml";
 // src/benchmarks/calibration.ts
 async function calibrateBenchmarkMetric(options) {
@@ -56,16 +60,19 @@ function fnv1a32(input) {
 const BENCHMARK_SPLIT_SEED = "agent-eval-v1";
 /**
 * Assign an item id to one of `'search' | 'dev' | 'holdout'` using a
-* stable 32-bit hash of `${seed}::${id}`. Default proportions:
+* stable 32-bit hash of `${seed}::${id}`. `cuts` [a, b] with 0 < a < b < 1 put
+* positions below a in search, below b in dev, the rest in holdout. Default [0.6, 0.8]:
 *
 *   search: 60%   (optimization-readable)
 *   dev:    20%   (held-out for tuning, leak-on-purpose during dev)
 *   holdout:20%   (paper-grade held-out, gated reads)
 */
-function deterministicSplit(itemId, seed = BENCHMARK_SPLIT_SEED) {
+function deterministicSplit(itemId, seed = BENCHMARK_SPLIT_SEED, cuts = [.6, .8]) {
+	const [a, b] = cuts;
+	if (!(a > 0 && a < b && b < 1)) throw new RangeError(`split cuts need 0 < a < b < 1, got [${a}, ${b}]`);
 	const pos = fnv1a32(`${seed}::${itemId}`) / 4294967296;
-	if (pos < .6) return "search";
-	if (pos < .8) return "dev";
+	if (pos < a) return "search";
+	if (pos < b) return "dev";
 	return "holdout";
 }
 // src/benchmarks/harbor.ts
@@ -78,10 +85,11 @@ function deterministicSplit(itemId, seed = BENCHMARK_SPLIT_SEED) {
 * revision the caller pins in `source`, into `BenchmarkDatasetItem`s.
 *
 * Scoring stays Harbor's. `evaluate` hands the task and the agent's artifact to the caller's
-* `grade` callback, which runs the task's own verifier (for example `harbor run -p <taskDir>`
-* with an agent that installs the collected artifacts) and reports its reward. This package does
-* not start containers. A verifier that did not run is an error, never a score of zero.
+* `grade` callback, which runs the task's own verifier and reports its reward. `gradeHarborPatch`
+* is that callback for a code patch: it spawns the Harbor CLI, which starts the containers; this
+* package starts none itself. A verifier that did not run is an error, never a score of zero.
 */
+const execute = promisify(execFile);
 const table = (value) => value !== null && typeof value === "object" && !Array.isArray(value) ? value : {};
 const numberOrNull = (value) => typeof value === "number" && Number.isFinite(value) ? value : null;
 const textOrNull = (value) => typeof value === "string" && value.length > 0 ? value : null;
@@ -163,12 +171,92 @@ function createHarborBenchmarkAdapter(options) {
 				raw: {
 					reward,
 					...detail === void 0 ? {} : { detail }
-				}
+				},
+				...detail === void 0 ? {} : { notes: detail }
 			};
 		},
 		assignSplit: split
 	};
 }
+const SOLVE_SH = "#!/bin/bash\nset -euo pipefail\npatch=\"$(dirname \"$0\")/patch.diff\"\nif [ -s \"$patch\" ]; then git apply --binary \"$patch\"; fi\n";
+/** The last characters of `text` without ANSI codes, so that its JSON string fits in `max`. */
+function detailTail(text, max) {
+	let tail = stripVTControlCharacters(text).slice(-max);
+	while (JSON.stringify(tail).length > max) tail = tail.slice(JSON.stringify(tail).length - max);
+	return tail;
+}
+/**
+* A `grade` callback that scores `{ patch }` with the task's own verifier: Harbor's oracle agent
+* runs a `solution/solve.sh` that applies the patch (`git apply --binary`) in the environment's
+* working directory, on a copy of the task. An empty patch changes nothing. A run with no reward
+* is retried, then reported as `succeeded: false`, never as a reward of 0.
+*/
+function gradeHarborPatch(options = {}) {
+	const { harborBin = "harbor", retries = 2, maxDetailChars = 2e3, keepImage = true } = options;
+	const { timeoutMs: timeout, env: extra } = options;
+	const run = {
+		env: {
+			...process.env,
+			...extra
+		},
+		timeout,
+		maxBuffer: 1 << 26
+	};
+	return async (item, artifact) => {
+		const work = await mkdtemp(join(tmpdir(), "harbor-grade-"));
+		try {
+			const task = join(work, item.payload.dir);
+			await cp(item.payload.taskDir, task, { recursive: true });
+			await rm(join(task, "solution"), {
+				recursive: true,
+				force: true
+			});
+			await mkdir(join(task, "solution"));
+			await writeFile(join(task, "solution/patch.diff"), artifact.patch);
+			await writeFile(join(task, "solution/solve.sh"), SOLVE_SH, { mode: 493 });
+			const jobsDir = resolve(options.jobsDir ?? join(work, "jobs"));
+			const errors = [];
+			const args = [
+				"run",
+				"-p",
+				task,
+				"-a",
+				"oracle",
+				"-o",
+				jobsDir,
+				"-q",
+				keepImage ? "--no-delete" : "--delete",
+				"--job-name"
+			];
+			for (let attempt = 1; attempt <= retries + 1; attempt += 1) {
+				const job = `${item.payload.dir}-${randomUUID()}`;
+				const failure = await execute(harborBin, [...args, job], run).then(() => null, (error) => detailTail(error.message, 500));
+				const trial = (await readdir(join(jobsDir, job), { withFileTypes: true }).catch(() => [])).filter((entry) => entry.isDirectory()).map((entry) => join(jobsDir, job, entry.name, "/"))[0];
+				const read = (path) => readFile(`${trial}${path}`, "utf8").catch(() => "");
+				const result = table(trial === void 0 ? {} : JSON.parse(await read("result.json") || "{}"));
+				const reward = numberOrNull(table(table(result.verifier_result).rewards).reward);
+				if (reward !== null) return {
+					succeeded: true,
+					value: {
+						reward,
+						detail: detailTail(await read("verifier/test-stdout.txt"), maxDetailChars)
+					}
+				};
+				const exception = textOrNull(table(result.exception_info).exception_message);
+				errors.push(`attempt ${attempt}: ${failure ?? exception ?? "harbor recorded no reward"}`);
+			}
+			return {
+				succeeded: false,
+				error: errors.join("; ")
+			};
+		} finally {
+			await rm(work, {
+				recursive: true,
+				force: true
+			});
+		}
+	};
+}
 // src/benchmarks/routing/dataset.ts
 const ROUTING_DATASET = [
 	{
@@ -362,8 +450,8 @@ const assignSplit = adapter.assignSplit.bind(adapter);
 async function runBenchmarkAdapter(options) {
 	const storage = options.storage ?? fsCampaignStorage();
 	const benchmarkId = benchmarkIdFor(options.adapter);
-	const scenarios = await loadBenchmarkScenarios(options.adapter, options.splits);
-	const judge = benchmarkAdapterJudge(options.adapter);
+	const scenarios = await benchmarkScenarios(options.adapter, options.splits);
+	const judge = benchmarkJudge(options.adapter);
 	const dispatch = async (scenario, context) => {
 		return options.respond({
 			scenario,
@@ -406,7 +494,7 @@ async function runBenchmarkAdapter(options) {
 		reportMarkdownPath
 	};
 }
-async function loadBenchmarkScenarios(adapter, splits = [
+async function benchmarkScenarios(adapter, splits = [
 	"search",
 	"dev",
 	"holdout"
@@ -438,7 +526,7 @@ async function loadBenchmarkScenarios(adapter, splits = [
 	}
 	return out;
 }
-function benchmarkAdapterJudge(adapter) {
+function benchmarkJudge(adapter) {
 	const benchmarkId = benchmarkIdFor(adapter);
 	return {
 		name: `${benchmarkId}:score`,
@@ -821,6 +909,6 @@ function discountedGain(relevance, zeroBasedRank) {
 	if (relevance <= 0) return 0;
 	return (2 ** relevance - 1) / Math.log2(zeroBasedRank + 2);
 }
-export { BENCHMARK_SPLIT_SEED, buildStandardRetrievalItems, calibrateBenchmarkMetric, createHarborBenchmarkAdapter, createRetrievalIdBenchmarkAdapter, deterministicSplit, evaluateStandardRetrieval, parseBeirCorpusJsonl, parseBeirQueriesJsonl, parseQrels, routing_exports as routing, runBenchmarkAdapter };
+export { BENCHMARK_SPLIT_SEED, benchmarkJudge, benchmarkScenarios, buildStandardRetrievalItems, calibrateBenchmarkMetric, createHarborBenchmarkAdapter, createRetrievalIdBenchmarkAdapter, deterministicSplit, evaluateStandardRetrieval, gradeHarborPatch, parseBeirCorpusJsonl, parseBeirQueriesJsonl, parseQrels, routing_exports as routing, runBenchmarkAdapter };
 //# sourceMappingURL=index.js.map

package/dist/benchmarks/index.js.map CHANGED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"index.js","names":["clamp01","parseToml","entries"],"sources":["../../src/benchmarks/calibration.ts","../../src/benchmarks/types.ts","../../src/benchmarks/harbor.ts","../../src/benchmarks/routing/dataset.ts","../../src/benchmarks/routing/index.ts","../../src/benchmarks/runner.ts","../../src/benchmarks/standard-formats.ts"],"sourcesContent":["import type { BenchmarkAdapter, BenchmarkDatasetItem, BenchmarkEvaluation } from './types'\n\nexport interface BenchmarkMetricCalibrationOptions<TPayload = unknown, TArtifact = string> {\n adapter: BenchmarkAdapter<BenchmarkDatasetItem<TPayload>, TPayload, TArtifact>\n item: BenchmarkDatasetItem<TPayload>\n weakArtifact: TArtifact\n strongArtifact: TArtifact\n maxWeakScore?: number\n minStrongScore?: number\n minGap?: number\n}\n\nexport interface BenchmarkMetricCalibrationResult {\n passed: boolean\n weak: BenchmarkEvaluation\n strong: BenchmarkEvaluation\n weakScore: number\n strongScore: number\n gap: number\n reasons: string[]\n}\n\nexport async function calibrateBenchmarkMetric<TPayload = unknown, TArtifact = string>(\n options: BenchmarkMetricCalibrationOptions<TPayload, TArtifact>,\n): Promise<BenchmarkMetricCalibrationResult> {\n const weak = await options.adapter.evaluate(options.item, options.weakArtifact)\n const strong = await options.adapter.evaluate(options.item, options.strongArtifact)\n const weakScore = clamp01(weak.score)\n const strongScore = clamp01(strong.score)\n const maxWeakScore = options.maxWeakScore ?? 0.3\n const minStrongScore = options.minStrongScore ?? 0.7\n const minGap = options.minGap ?? 0.4\n const gap = strongScore - weakScore\n const reasons = [\n weakScore <= maxWeakScore\n ? undefined\n : `weak score ${weakScore.toFixed(3)} exceeds max ${maxWeakScore.toFixed(3)}`,\n strongScore >= minStrongScore\n ? undefined\n : `strong score ${strongScore.toFixed(3)} below min ${minStrongScore.toFixed(3)}`,\n gap >= minGap ? undefined : `gap ${gap.toFixed(3)} below min ${minGap.toFixed(3)}`,\n ].filter((reason): reason is string => Boolean(reason))\n\n return {\n passed: reasons.length === 0,\n weak,\n strong,\n weakScore,\n strongScore,\n gap,\n reasons,\n }\n}\n\nfunction clamp01(value: number): number {\n if (!Number.isFinite(value)) return 0\n if (value < 0) return 0\n if (value > 1) return 1\n return value\n}\n","/*\n Shared types for the reference benchmark wrappers under\n * `src/benchmarks/`. Each wrapper exports the three functions in\n * `BenchmarkAdapter` plus its own typed `DatasetItem` shape.\n /\n\nimport type { DispatchContext, Scenario } from '../campaign/types'\nimport type { RunSplitTag } from '../run-record'\n\nexport type BenchmarkTaskKind =\n \| 'retrieval'\n \| 'rag-answer'\n \| 'hallucination'\n \| 'kb-improvement'\n \| 'routing'\n \| 'custom'\n\nexport type BenchmarkFamily =\n \| 'beir'\n \| 'mteb-retrieval'\n \| 'msmarco'\n \| 'trec-dl'\n \| 'miracl'\n \| 'lotte'\n \| 'bright'\n \| 'crag'\n \| 'hotpotqa'\n \| 'kilt'\n \| 'ragtruth'\n \| 'faithbench'\n \| 'first-party'\n \| 'custom'\n\nexport interface BenchmarkDatasetItem<TPayload = unknown> {\n /* Stable dataset-local item id (used for split assignment + paper\n * references). Unique within a benchmark. /\n id: string\n /* Free-form payload. Each benchmark defines its own shape. /\n payload: TPayload\n /* Optional precomputed split. When absent, adapters use assignSplit(id). /\n split?: RunSplitTag\n /* Benchmark family this row came from, e.g. `beir` or `crag`. /\n family?: BenchmarkFamily \| string\n /* Benchmark-local task kind, e.g. retrieval vs answer quality. /\n taskKind?: BenchmarkTaskKind \| string\n /* Slice labels such as language, domain, freshness, multihop, long-tail. /\n tags?: string[]\n /* Dataset provenance, version, URL, or license notes. /\n source?: BenchmarkSource\n metadata?: Record<string, unknown>\n}\n\nexport interface BenchmarkEvaluation {\n /* [0, 1] score for the response on this item. Exact-match\n * benchmarks use 0/1; partial-credit benchmarks may return\n * fractional values. /\n score: number\n /* Optional pass/fail projection. Defaults to `score > 0` when absent. /\n passed?: boolean\n /* Numeric sub-metrics. These become report dimensions. /\n dimensions?: Record<string, number>\n /* Optional bag of raw scoring signals — e.g. parsed numeric\n * answer, regex match, judge sub-scores. /\n raw: Record<string, unknown>\n notes?: string\n}\n\nexport interface BenchmarkSource {\n name?: string\n url?: string\n version?: string\n /* The exact upstream revision, such as a git commit, when `version` names a release. /\n revision?: string\n license?: string\n citation?: string\n /* Where the upstream states how a result must be produced to count, such as leaderboard rules. /\n rules?: string\n /\n Whether the tasks or their solutions may be in a model's training data: `unknown` until\n * checked, `none-known`, `suspected`, or `confirmed` (then report results as historical).\n /\n contamination?: 'unknown' \| 'none-known' \| 'suspected' \| 'confirmed'\n}\n\n/* Common signature implemented by every adapter under `src/benchmarks/`. /\n// `TPayload` is the per-item payload type; `_TItem` is preserved for\n// downstream type-narrowing extensions (a richer `BenchmarkDatasetItem`\n// subclass that adds e.g. provenance metadata) but is intentionally\n// unused here. `noUnusedLocals` requires the leading underscore.\nexport interface BenchmarkAdapter<_TItem = unknown, TPayload = unknown, TArtifact = string> {\n /** Stable benchmark id such as `beir/nfcorpus` or `crag/smoke`. /\n id?: string\n family?: BenchmarkFamily \| string\n taskKind?: BenchmarkTaskKind \| string\n description?: string\n source?: BenchmarkSource\n defaultMetric?: string\n /* Load the dataset for the given split. May hit the network on\n * first call but should be cache-friendly. Adapters that don't\n * ship the dataset itself MUST throw a clearly-marked error\n * pointing the caller at the loader script. /\n loadDataset(split: RunSplitTag): Promise<BenchmarkDatasetItem<TPayload>[]>\n /* Score a single response. Pure with respect to the inputs. /\n evaluate(item: BenchmarkDatasetItem<TPayload>, artifact: TArtifact): Promise<BenchmarkEvaluation>\n /* Deterministic split assignment via item id hashing. The\n * fraction of items in each split is implementation-defined but\n * MUST be stable across processes and platforms. /\n assignSplit(itemId: string): RunSplitTag\n}\n\nexport interface BenchmarkScenario<TPayload = unknown> extends Scenario {\n kind: 'benchmark'\n benchmarkId: string\n family: BenchmarkFamily \| string\n taskKind: BenchmarkTaskKind \| string\n splitTag: RunSplitTag\n item: BenchmarkDatasetItem<TPayload>\n}\n\nexport type BenchmarkResponder<TPayload = unknown, TArtifact = string> = (input: {\n scenario: BenchmarkScenario<TPayload>\n item: BenchmarkDatasetItem<TPayload>\n context: DispatchContext\n}) => Promise<TArtifact> \| TArtifact\n\n// ── Deterministic split assignment ───────────────────────────────────\n\n/\n 32-bit FNV-1a hash. Stable, allocation-free, deterministic across\n * runtimes. We use it to assign items to splits rather than depending\n * on a polyfilled crypto.subtle path.\n /\nfunction fnv1a32(input: string): number {\n let h = 0x811c9dc5\n for (let i = 0; i < input.length; i++) {\n h ^= input.charCodeAt(i) & 0xff\n h = (h + ((h << 1) + (h << 4) + (h << 7) + (h << 8) + (h << 24))) >>> 0\n }\n return h >>> 0\n}\n\n/* Split-assignment seed shared across all benchmarks. Bumping this\n * value reshuffles every split — do NOT do that lightly. /\nexport const BENCHMARK_SPLIT_SEED = 'agent-eval-v1'\n\n/\n Assign an item id to one of `'search' \| 'dev' \| 'holdout'` using a\n * stable 32-bit hash of `${seed}::${id}`. Default proportions:\n \n search: 60% (optimization-readable)\n * dev: 20% (held-out for tuning, leak-on-purpose during dev)\n * holdout:20% (paper-grade held-out, gated reads)\n /\nexport function deterministicSplit(\n itemId: string,\n seed: string = BENCHMARK_SPLIT_SEED,\n): RunSplitTag {\n const h = fnv1a32(`${seed}::${itemId}`)\n const pos = h / 0x100000000\n if (pos < 0.6) return 'search'\n if (pos < 0.8) return 'dev'\n return 'holdout'\n}\n","/\n A Harbor dataset as a `BenchmarkAdapter`.\n \n Harbor (https://harborframework.com) lays out each task as a directory with `task.toml`,\n * `instruction.md`, the agent's `environment/`, and the `tests/` its verifier runs, usually in a\n * verifier image of their own. This importer reads a local checkout of such a dataset, at the\n * revision the caller pins in `source`, into `BenchmarkDatasetItem`s.\n \n Scoring stays Harbor's. `evaluate` hands the task and the agent's artifact to the caller's\n * `grade` callback, which runs the task's own verifier (for example `harbor run -p <taskDir>`\n * with an agent that installs the collected artifacts) and reports its reward. This package does\n * not start containers. A verifier that did not run is an error, never a score of zero.\n /\n\nimport { readdir, readFile, stat } from 'node:fs/promises'\nimport { join, resolve } from 'node:path'\n\nimport { parse as parseToml } from 'smol-toml'\n\nimport type { RunSplitTag } from '../run-record'\nimport type {\n BenchmarkAdapter,\n BenchmarkDatasetItem,\n BenchmarkEvaluation,\n BenchmarkSource,\n} from './types'\nimport { deterministicSplit } from './types'\n\n/* One Harbor task, as its `task.toml` and `instruction.md` state it. /\nexport interface HarborTaskPayload {\n /* `[task].name`, such as `terminal-bench/interleaved-vigenere`; the directory name when absent. /\n name: string\n /* The task's directory name inside the dataset's `tasks/`. /\n dir: string\n /* Absolute path of the task directory, for the grader. /\n taskDir: string\n /* `instruction.md`, verbatim. /\n instruction: string\n /* `[agent].timeout_sec`; null when the task sets none. /\n agentTimeoutSec: number \| null\n /* `[verifier].timeout_sec`; null when the task sets none. /\n verifierTimeoutSec: number \| null\n /* `[verifier].environment_mode`: `separate` runs the tests in their own image. /\n verifierMode: string \| null\n /* `[environment]` requests; null where the task states none. /\n resources: {\n cpus: number \| null\n memoryMb: number \| null\n storageMb: number \| null\n gpus: number\n }\n /* Paths the verifier reads from the agent's environment (`artifacts`, sources only). /\n artifacts: string[]\n /* The environment starts more than one container (a docker-compose file). /\n multiContainer: boolean\n /* `[metadata].category`. /\n category: string \| null\n /* `[metadata].expert_time_estimate_hours`. /\n expertTimeEstimateHours: number \| null\n}\n\nexport type HarborTaskItem = BenchmarkDatasetItem<HarborTaskPayload>\n\n/* What the verifier returned for one task, or why it did not return. /\nexport type HarborGradeOutcome =\n \| { succeeded: true; value: { reward: number; detail?: string } }\n \| { succeeded: false; error: string }\n\nexport interface HarborBenchmarkOptions<TArtifact> {\n /* Stable benchmark id, such as `terminal-bench@4.0.0`. /\n id: string\n /* A checkout of the dataset: the directory that holds `tasks/`. /\n datasetDir: string\n /* The upstream name, URL, version, revision, rules and contamination status the checkout is. /\n source: BenchmarkSource & { revision: string }\n /* Task directory names to load, in this order; every task under `tasks/` when absent. /\n tasks?: readonly string[]\n /* Run the task's own verifier on the agent's artifact. /\n grade: (item: HarborTaskItem, artifact: TArtifact) => Promise<HarborGradeOutcome>\n /* Split assignment; `deterministicSplit` over the benchmark id and task directory by default. /\n assignSplit?: (itemId: string) => RunSplitTag\n}\n\n/* Harbor adapter plus a loader that ignores splits, since a public benchmark reports every task. /\nexport interface HarborBenchmarkAdapter<TArtifact>\n extends BenchmarkAdapter<HarborTaskItem, HarborTaskPayload, TArtifact> {\n loadAll(): Promise<HarborTaskItem[]>\n}\n\nconst table = (value: unknown): Record<string, unknown> =>\n value !== null && typeof value === 'object' && !Array.isArray(value)\n ? (value as Record<string, unknown>)\n : {}\nconst numberOrNull = (value: unknown): number \| null =>\n typeof value === 'number' && Number.isFinite(value) ? value : null\nconst textOrNull = (value: unknown): string \| null =>\n typeof value === 'string' && value.length > 0 ? value : null\n\n/* Read one Harbor task directory. Throws when `task.toml` or `instruction.md` is missing or malformed. /\nasync function readHarborTask(taskDir: string): Promise<HarborTaskPayload> {\n const dir = resolve(taskDir)\n const config = parseToml(await readFile(join(dir, 'task.toml'), 'utf8')) as Record<\n string,\n unknown\n >\n const instruction = await readFile(join(dir, 'instruction.md'), 'utf8')\n const task = table(config.task)\n const agent = table(config.agent)\n const verifier = table(config.verifier)\n const environment = table(config.environment)\n const metadata = table(config.metadata)\n const artifacts = (Array.isArray(config.artifacts) ? config.artifacts : []).map(\n (entry, index) => {\n const source = typeof entry === 'string' ? entry : table(entry).source\n if (typeof source !== 'string' \|\| source.length === 0) {\n throw new Error(`${dir}/task.toml: artifacts[${index}] names no source path`)\n }\n return source\n },\n )\n const environmentFiles = await readdir(join(dir, 'environment')).catch(() => [] as string[])\n const name = dir.split('/').pop() ?? dir\n return {\n name: textOrNull(task.name) ?? name,\n dir: name,\n taskDir: dir,\n instruction,\n agentTimeoutSec: numberOrNull(agent.timeout_sec),\n verifierTimeoutSec: numberOrNull(verifier.timeout_sec),\n verifierMode: textOrNull(verifier.environment_mode),\n resources: {\n cpus: numberOrNull(environment.cpus),\n memoryMb: numberOrNull(environment.memory_mb),\n storageMb: numberOrNull(environment.storage_mb),\n gpus: numberOrNull(environment.gpus) ?? 0,\n },\n artifacts,\n multiContainer: environmentFiles.some((file) => /compose\\.ya?ml$/u.test(file)),\n category: textOrNull(metadata.category),\n expertTimeEstimateHours: numberOrNull(metadata.expert_time_estimate_hours),\n }\n}\n\n/* Import a Harbor dataset checkout as a `BenchmarkAdapter` whose scorer is the dataset's own verifier. /\nexport function createHarborBenchmarkAdapter<TArtifact = string>(\n options: HarborBenchmarkOptions<TArtifact>,\n): HarborBenchmarkAdapter<TArtifact> {\n const split =\n options.assignSplit ?? ((itemId: string) => deterministicSplit(`${options.id}::${itemId}`))\n const tasksDir = join(resolve(options.datasetDir), 'tasks')\n\n async function loadAll(): Promise<HarborTaskItem[]> {\n const names =\n options.tasks ??\n (\n await Promise.all(\n (\n await readdir(tasksDir)\n ).map(async (entry) =>\n (await stat(join(tasksDir, entry, 'task.toml')).catch(() => null)) === null\n ? null\n : entry,\n ),\n )\n )\n .filter((entry): entry is string => entry !== null)\n .sort()\n return Promise.all(\n names.map(async (dir) => {\n const payload = await readHarborTask(join(tasksDir, dir))\n return {\n id: dir,\n payload,\n split: split(dir),\n family: options.source.name ?? 'harbor',\n taskKind: 'harbor-task',\n tags: payload.category === null ? [] : [payload.category],\n source: options.source,\n }\n }),\n )\n }\n\n return {\n id: options.id,\n family: options.source.name ?? 'harbor',\n taskKind: 'harbor-task',\n description:\n `${options.source.name ?? 'Harbor dataset'} ${options.source.version ?? ''} at ${options.source.revision}, scored by its own verifier`.replace(\n /\\s+/gu,\n ' ',\n ),\n source: options.source,\n defaultMetric: 'reward',\n loadAll,\n async loadDataset(tag: RunSplitTag) {\n return (await loadAll()).filter((item) => item.split === tag)\n },\n async evaluate(item: HarborTaskItem, artifact: TArtifact): Promise<BenchmarkEvaluation> {\n const outcome = await options.grade(item, artifact)\n if (!outcome.succeeded)\n throw new Error(\n `${options.id} ${item.id}: the verifier did not score this artifact: ${outcome.error}`,\n )\n const { reward, detail } = outcome.value\n if (!(Number.isFinite(reward) && reward >= 0 && reward <= 1)) {\n throw new Error(\n `${options.id} ${item.id}: the verifier returned reward ${reward}, outside [0, 1]`,\n )\n }\n return {\n score: reward,\n passed: reward === 1,\n raw: { reward, ...(detail === undefined ? {} : { detail }) },\n }\n },\n assignSplit: split,\n }\n}\n","/\n Synthetic routing dataset. 16 tasks across 4 categories. Used as a\n * deterministic, dependency-free benchmark for any router that maps a\n * natural-language request to one of a fixed set of route labels.\n \n Format (see `routing/README.md` for prose):\n \n {\n * id: stable per-task ID (matches across processes).\n * category: one of the four route labels.\n * prompt: the user-facing request the router must classify.\n * route: the ground-truth route the router should pick.\n * synonyms: other strings that count as a correct answer.\n * hardNegatives:close-but-wrong route labels — used to detect the\n * \"always picks the popular route\" failure mode.\n * }\n \n The four categories are intentionally cross-domain (file ops,\n * math, search, conversation) so a router that collapses to one\n * category is easy to spot.\n /\n\nexport interface RoutingItem {\n id: string\n category: 'file' \| 'math' \| 'search' \| 'chat'\n prompt: string\n /* Canonical correct route label. /\n route: string\n /* Alternate route labels that also count as correct. /\n synonyms: string[]\n /* Wrong-but-tempting route labels (for analysis, not grading). /\n hardNegatives: string[]\n}\n\nexport const ROUTING_DATASET: RoutingItem[] = [\n {\n id: 'file_001',\n category: 'file',\n prompt: 'Save the meeting notes to /tmp/notes-2025-04.md as markdown.',\n route: 'fs.write',\n synonyms: ['filesystem.write', 'write_file'],\n hardNegatives: ['fs.read', 'chat.reply'],\n },\n {\n id: 'file_002',\n category: 'file',\n prompt: 'Read the contents of /etc/hosts and summarize the entries.',\n route: 'fs.read',\n synonyms: ['filesystem.read', 'read_file'],\n hardNegatives: ['fs.write', 'search.web'],\n },\n {\n id: 'file_003',\n category: 'file',\n prompt: 'List every Python file under src/ recursively.',\n route: 'fs.list',\n synonyms: ['filesystem.list', 'list_files'],\n hardNegatives: ['fs.read', 'search.code'],\n },\n {\n id: 'file_004',\n category: 'file',\n prompt: 'Delete the cached build at .turbo/cache.',\n route: 'fs.delete',\n synonyms: ['filesystem.delete', 'remove_file'],\n hardNegatives: ['fs.write', 'fs.list'],\n },\n {\n id: 'math_001',\n category: 'math',\n prompt: 'What is the integral of 3x^2 + 2x from 0 to 5?',\n route: 'math.integral',\n synonyms: ['calculator.integral', 'math.solve'],\n hardNegatives: ['math.derivative', 'chat.reply'],\n },\n {\n id: 'math_002',\n category: 'math',\n prompt: 'Compute the derivative of sin(x) cos(x).',\n route: 'math.derivative',\n synonyms: ['calculator.derivative', 'math.solve'],\n hardNegatives: ['math.integral', 'math.algebra'],\n },\n {\n id: 'math_003',\n category: 'math',\n prompt: 'Solve 2x + 7 = 19 for x.',\n route: 'math.algebra',\n synonyms: ['calculator.algebra', 'math.solve'],\n hardNegatives: ['math.derivative', 'math.integral'],\n },\n {\n id: 'math_004',\n category: 'math',\n prompt: 'What is the prime factorization of 360?',\n route: 'math.numbertheory',\n synonyms: ['calculator.factor', 'math.solve'],\n hardNegatives: ['math.algebra', 'search.web'],\n },\n {\n id: 'search_001',\n category: 'search',\n prompt: 'Find recent papers on agent prompt optimization with held-out promotion gates.',\n route: 'search.web',\n synonyms: ['web.search', 'search.papers'],\n hardNegatives: ['search.code', 'chat.reply'],\n },\n {\n id: 'search_002',\n category: 'search',\n prompt: 'Search the codebase for every call site of `runProposeReview`.',\n route: 'search.code',\n synonyms: ['code.search', 'grep'],\n hardNegatives: ['search.web', 'fs.read'],\n },\n {\n id: 'search_003',\n category: 'search',\n prompt: 'What is the latest release of the Tangle network on GitHub?',\n route: 'search.web',\n synonyms: ['web.search', 'github.releases'],\n hardNegatives: ['search.code', 'chat.reply'],\n },\n {\n id: 'search_004',\n category: 'search',\n prompt: 'Find all TODO comments in the agent-eval src tree.',\n route: 'search.code',\n synonyms: ['code.search', 'grep'],\n hardNegatives: ['search.web', 'fs.list'],\n },\n {\n id: 'chat_001',\n category: 'chat',\n prompt: 'Hi there, how are you doing today?',\n route: 'chat.reply',\n synonyms: ['conversation.reply'],\n hardNegatives: ['search.web', 'fs.read'],\n },\n {\n id: 'chat_002',\n category: 'chat',\n prompt: 'Please explain the difference between an LLM and a foundation model.',\n route: 'chat.reply',\n synonyms: ['conversation.reply', 'qa.answer'],\n hardNegatives: ['search.web', 'math.algebra'],\n },\n {\n id: 'chat_003',\n category: 'chat',\n prompt: 'Tell me a short joke about distributed systems.',\n route: 'chat.reply',\n synonyms: ['conversation.reply'],\n hardNegatives: ['search.web', 'fs.read'],\n },\n {\n id: 'chat_004',\n category: 'chat',\n prompt: 'Acknowledge my last message with a thumbs up.',\n route: 'chat.reply',\n synonyms: ['conversation.reply', 'react'],\n hardNegatives: ['fs.write', 'search.web'],\n },\n]\n","/*\n Routing benchmark — synthetic, dependency-free, ships in the\n * package. 16 cross-category items in `dataset.ts`. See\n * `routing/README.md` for the format.\n \n `evaluate` does case-insensitive exact match against the canonical\n * route plus declared synonyms. The first valid route token in the\n * response wins; everything else is ignored. Wrong answers also\n * report whether they hit a hard negative — useful when triaging\n * \"always picks the popular route\" failure modes.\n /\n\nimport type { RunSplitTag } from '../../run-record'\nimport type { BenchmarkAdapter, BenchmarkDatasetItem, BenchmarkEvaluation } from '../types'\nimport { deterministicSplit } from '../types'\nimport { ROUTING_DATASET, type RoutingItem } from './dataset'\n\nexport type { RoutingItem }\nexport type RoutingPayload = RoutingItem\nexport type RoutingDatasetItem = BenchmarkDatasetItem<RoutingPayload>\n\nclass RoutingAdapter implements BenchmarkAdapter<RoutingDatasetItem, RoutingPayload> {\n readonly id = 'first-party/routing'\n readonly family = 'first-party'\n readonly taskKind = 'routing'\n readonly description = 'Synthetic fixed-route classification smoke benchmark'\n readonly defaultMetric = 'route_exact_match'\n\n async loadDataset(split: RunSplitTag): Promise<RoutingDatasetItem[]> {\n return ROUTING_DATASET.map((item) => ({ id: item.id, payload: item })).filter(\n (it) => assignSplitImpl(it.id) === split,\n )\n }\n\n async evaluate(item: RoutingDatasetItem, response: string): Promise<BenchmarkEvaluation> {\n const tokens = extractRouteTokens(response)\n const correct = new Set<string>(\n [item.payload.route, ...item.payload.synonyms].map((s) => s.toLowerCase()),\n )\n const hardNeg = new Set<string>(item.payload.hardNegatives.map((s) => s.toLowerCase()))\n const firstMatch = tokens.find((t) => correct.has(t.toLowerCase())) ?? null\n const firstHardNeg = tokens.find((t) => hardNeg.has(t.toLowerCase())) ?? null\n const score = firstMatch ? 1 : 0\n return {\n score,\n raw: {\n firstToken: tokens[0] ?? null,\n matchedRoute: firstMatch,\n hitHardNegative: Boolean(firstHardNeg),\n hardNegativeRoute: firstHardNeg,\n category: item.payload.category,\n },\n }\n }\n\n assignSplit(itemId: string): RunSplitTag {\n return assignSplitImpl(itemId)\n }\n}\n\nfunction assignSplitImpl(itemId: string): RunSplitTag {\n return deterministicSplit(`routing::${itemId}`)\n}\n\n/\n Pull route-shaped tokens out of a model response. Routes look like\n * `category.action` (`fs.write`, `chat.reply`). Bare alphanumerics\n * are not routes, but `category.action` patterns are robust to most\n * model wrappers (JSON output, prose explanations, code fences).\n /\nexport function extractRouteTokens(response: string): string[] {\n const matches = response.match(/[a-z][a-z0-9_]\\.[a-z][a-z0-9_]/gi)\n return matches ?? []\n}\n\nconst adapter = new RoutingAdapter()\n\nexport const loadDataset = adapter.loadDataset.bind(adapter)\nexport const evaluate = adapter.evaluate.bind(adapter)\nexport const assignSplit = adapter.assignSplit.bind(adapter)\nexport { ROUTING_DATASET, RoutingAdapter }\n","import { join } from 'node:path'\nimport {\n type CampaignResult,\n type DispatchFn,\n type JudgeConfig,\n type JudgeScore,\n runCampaign,\n type Scenario,\n} from '../campaign'\nimport { type CampaignStorage, fsCampaignStorage } from '../campaign/storage'\nimport type { RunSplitTag } from '../run-record'\nimport { summarizeNumberSeries } from '../statistics'\nimport { mean } from '../statistics/mean'\nimport type {\n BenchmarkAdapter,\n BenchmarkDatasetItem,\n BenchmarkEvaluation,\n BenchmarkFamily,\n BenchmarkResponder,\n BenchmarkScenario,\n BenchmarkTaskKind,\n} from './types'\n\nexport interface BenchmarkRunOptions<TPayload = unknown, TArtifact = string> {\n adapter: BenchmarkAdapter<BenchmarkDatasetItem<TPayload>, TPayload, TArtifact>\n respond: BenchmarkResponder<TPayload, TArtifact>\n splits?: readonly RunSplitTag[]\n runDir: string\n repo?: string\n seed?: number\n reps?: number\n resumable?: boolean\n costCeiling?: number\n maxConcurrency?: number\n dispatchTimeoutMs?: number\n expectUsage?: 'assert' \| 'warn' \| 'off'\n storage?: CampaignStorage\n now?: () => Date\n}\n\nexport interface BenchmarkReport {\n benchmarkId: string\n family: BenchmarkFamily \| string\n taskKind: BenchmarkTaskKind \| string\n source?: BenchmarkAdapter['source']\n runDir: string\n manifestHash: string\n seed: number\n startedAt: string\n endedAt: string\n durationMs: number\n totalItems: number\n totalCells: number\n cellsFailed: number\n cellsCached: number\n totalCostUsd: number\n splits: Record<string, BenchmarkSliceSummary>\n tags: Record<string, BenchmarkSliceSummary>\n dimensions: Record<string, BenchmarkDistribution>\n score: BenchmarkDistribution\n costUsd: BenchmarkDistribution\n latencyMs: BenchmarkDistribution\n}\n\nexport interface BenchmarkSliceSummary {\n n: number\n meanScore: number\n passRate: number\n score: BenchmarkDistribution\n costUsd: BenchmarkDistribution\n latencyMs: BenchmarkDistribution\n}\n\nexport interface BenchmarkDistribution {\n n: number\n min: number\n mean: number\n median: number\n p90: number\n max: number\n}\n\nexport interface BenchmarkRunResult<TPayload = unknown, TArtifact = string> {\n scenarios: Array<BenchmarkScenario<TPayload>>\n campaign: CampaignResult<TArtifact, BenchmarkScenario<TPayload>>\n report: BenchmarkReport\n reportJsonPath: string\n reportMarkdownPath: string\n}\n\nexport async function runBenchmarkAdapter<TPayload = unknown, TArtifact = string>(\n options: BenchmarkRunOptions<TPayload, TArtifact>,\n): Promise<BenchmarkRunResult<TPayload, TArtifact>> {\n const storage = options.storage ?? fsCampaignStorage()\n const benchmarkId = benchmarkIdFor(options.adapter)\n const scenarios = await loadBenchmarkScenarios(options.adapter, options.splits)\n const judge = benchmarkAdapterJudge(options.adapter)\n const dispatch: DispatchFn<BenchmarkScenario<TPayload>, TArtifact> = async (\n scenario,\n context,\n ) => {\n return options.respond({ scenario, item: scenario.item, context })\n }\n\n const campaign = await runCampaign<BenchmarkScenario<TPayload>, TArtifact>({\n scenarios,\n dispatch,\n dispatchRef: `benchmark:${benchmarkId}`,\n judges: [judge],\n seed: options.seed,\n reps: options.reps,\n resumable: options.resumable,\n costCeiling: options.costCeiling,\n maxConcurrency: options.maxConcurrency,\n dispatchTimeoutMs: options.dispatchTimeoutMs,\n expectUsage: options.expectUsage ?? 'off',\n runDir: options.runDir,\n repo: options.repo,\n storage,\n now: options.now,\n })\n const report = summarizeBenchmarkCampaign({\n adapter: options.adapter,\n scenarios,\n campaign,\n })\n storage.ensureDir(campaign.runDir)\n const reportJsonPath = join(campaign.runDir, 'benchmark-report.json')\n const reportMarkdownPath = join(campaign.runDir, 'benchmark-report.md')\n storage.write(reportJsonPath, `${JSON.stringify(report, null, 2)}\\n`)\n storage.write(reportMarkdownPath, renderBenchmarkReportMarkdown(report))\n return { scenarios, campaign, report, reportJsonPath, reportMarkdownPath }\n}\n\nasync function loadBenchmarkScenarios<TPayload>(\n adapter: BenchmarkAdapter<BenchmarkDatasetItem<TPayload>, TPayload, unknown>,\n splits: readonly RunSplitTag[] = ['search', 'dev', 'holdout'],\n): Promise<Array<BenchmarkScenario<TPayload>>> {\n const benchmarkId = benchmarkIdFor(adapter)\n const family = adapter.family ?? 'custom'\n const taskKind = adapter.taskKind ?? 'custom'\n const out: Array<BenchmarkScenario<TPayload>> = []\n const seen = new Set<string>()\n for (const split of splits) {\n const items = await adapter.loadDataset(split)\n for (const item of items) {\n const splitTag = item.split ?? adapter.assignSplit(item.id)\n if (splitTag !== split) continue\n const key = `${benchmarkId}:${item.id}`\n if (seen.has(key)) continue\n seen.add(key)\n out.push({\n id: key,\n kind: 'benchmark',\n benchmarkId,\n family: item.family ?? family,\n taskKind: item.taskKind ?? taskKind,\n splitTag,\n tags: [...new Set([splitTag, ...(item.tags ?? [])])],\n item,\n })\n }\n }\n return out\n}\n\nfunction benchmarkAdapterJudge<TPayload, TArtifact>(\n adapter: BenchmarkAdapter<BenchmarkDatasetItem<TPayload>, TPayload, TArtifact>,\n): JudgeConfig<TArtifact, BenchmarkScenario<TPayload>> {\n const benchmarkId = benchmarkIdFor(adapter)\n return {\n name: `${benchmarkId}:score`,\n dimensions: [\n { key: 'score', description: `Primary ${benchmarkId} benchmark score` },\n { key: 'passed', description: 'Binary pass projection for aggregate reporting' },\n ],\n appliesTo: (scenario: Scenario): scenario is BenchmarkScenario<TPayload> => {\n return scenario.kind === 'benchmark' && scenario.id.startsWith(`${benchmarkId}:`)\n },\n async score({ artifact, scenario }) {\n const evaluation = await adapter.evaluate(scenario.item, artifact)\n const dimensions = normalizeEvaluationDimensions(evaluation)\n return {\n dimensions,\n composite: clamp01(evaluation.score),\n notes: evaluation.notes ?? '',\n }\n },\n }\n}\n\nfunction summarizeBenchmarkCampaign<TPayload, TArtifact>(input: {\n adapter: BenchmarkAdapter<BenchmarkDatasetItem<TPayload>, TPayload, TArtifact>\n scenarios: Array<BenchmarkScenario<TPayload>>\n campaign: CampaignResult<TArtifact, BenchmarkScenario<TPayload>>\n}): BenchmarkReport {\n const scenarioById = new Map(input.scenarios.map((scenario) => [scenario.id, scenario]))\n const rows = input.campaign.cells.map((cell) => {\n const scenario = scenarioById.get(cell.scenarioId)\n const judge = firstJudgeScore(cell.judgeScores)\n const score = judge?.composite ?? 0\n return {\n cell,\n scenario,\n score,\n passed: (judge?.dimensions.passed ?? (score > 0 ? 1 : 0)) >= 1,\n dimensions: judge?.dimensions ?? {},\n }\n })\n const successful = rows.filter((row) => !row.cell.error)\n const scoreValues = successful.map((row) => row.score)\n return {\n benchmarkId: benchmarkIdFor(input.adapter),\n family: input.adapter.family ?? 'custom',\n taskKind: input.adapter.taskKind ?? 'custom',\n ...(input.adapter.source ? { source: input.adapter.source } : {}),\n runDir: input.campaign.runDir,\n manifestHash: input.campaign.manifestHash,\n seed: input.campaign.seed,\n startedAt: input.campaign.startedAt,\n endedAt: input.campaign.endedAt,\n durationMs: input.campaign.durationMs,\n totalItems: input.scenarios.length,\n totalCells: input.campaign.cells.length,\n cellsFailed: input.campaign.aggregates.cellsFailed,\n cellsCached: input.campaign.aggregates.cellsCached,\n totalCostUsd: input.campaign.aggregates.cost.totalCostUsd,\n splits: summarizeSlices(successful, (row) => row.scenario?.splitTag ?? 'unknown', [\n 'search',\n 'dev',\n 'holdout',\n ]),\n tags: summarizeSlices(successful, (row) => row.scenario?.tags ?? []),\n dimensions: summarizeDimensions(successful.map((row) => row.dimensions)),\n score: distribution(scoreValues),\n costUsd: distribution(successful.map((row) => row.cell.costUsd)),\n latencyMs: distribution(successful.map((row) => row.cell.durationMs)),\n }\n}\n\nfunction renderBenchmarkReportMarkdown(report: BenchmarkReport): string {\n const splitRows = Object.entries(report.splits)\n .map(([split, summary]) => {\n return `\| ${split} \| ${summary.n} \| ${fmt(summary.meanScore)} \| ${fmt(summary.passRate)} \| ${fmt(summary.score.p90)} \| ${fmt(summary.costUsd.mean)} \| ${fmt(summary.latencyMs.p90)} \|`\n })\n .join('\\n')\n const dimRows = Object.entries(report.dimensions)\n .sort(([a], [b]) => a.localeCompare(b))\n .map(([key, dist]) => `\| ${key} \| ${dist.n} \| ${fmt(dist.mean)} \| ${fmt(dist.p90)} \|`)\n .join('\\n')\n return [\n `# Benchmark Report: ${report.benchmarkId}`,\n '',\n `- family: ${report.family}`,\n `- task kind: ${report.taskKind}`,\n `- run dir: ${report.runDir}`,\n `- manifest: ${report.manifestHash}`,\n `- cells: ${report.totalCells} total, ${report.cellsFailed} failed, ${report.cellsCached} cached`,\n `- cost: $${fmt(report.totalCostUsd)}`,\n `- score: mean ${fmt(report.score.mean)}, median ${fmt(report.score.median)}, p90 ${fmt(report.score.p90)}, n=${report.score.n}`,\n '',\n '## Splits',\n '',\n '\| split \| n \| mean score \| pass rate \| score p90 \| mean cost \| latency p90 ms \|',\n '\| --- \| ---: \| ---: \| ---: \| ---: \| ---: \| ---: \|',\n splitRows \|\| '\| none \| 0 \| 0 \| 0 \| 0 \| 0 \| 0 \|',\n '',\n '## Dimensions',\n '',\n '\| dimension \| n \| mean \| p90 \|',\n '\| --- \| ---: \| ---: \| ---: \|',\n dimRows \|\| '\| none \| 0 \| 0 \| 0 \|',\n '',\n ].join('\\n')\n}\n\nfunction benchmarkIdFor(adapter: Pick<BenchmarkAdapter, 'id' \| 'family' \| 'taskKind'>): string {\n return adapter.id ?? `${adapter.family ?? 'custom'}/${adapter.taskKind ?? 'custom'}`\n}\n\nfunction normalizeEvaluationDimensions(evaluation: BenchmarkEvaluation): Record<string, number> {\n const dimensions: Record<string, number> = { score: clamp01(evaluation.score) }\n for (const [key, value] of Object.entries(evaluation.dimensions ?? {})) {\n if (Number.isFinite(value)) dimensions[key] = value\n }\n dimensions.passed = (evaluation.passed ?? evaluation.score > 0) ? 1 : 0\n return dimensions\n}\n\nfunction summarizeDimensions(\n rows: Array<Record<string, number>>,\n): Record<string, BenchmarkDistribution> {\n const values = new Map<string, number[]>()\n for (const row of rows) {\n for (const [key, value] of Object.entries(row)) {\n if (!Number.isFinite(value)) continue\n const list = values.get(key) ?? []\n list.push(value)\n values.set(key, list)\n }\n }\n return Object.fromEntries([...values.entries()].map(([key, vals]) => [key, distribution(vals)]))\n}\n\nfunction summarizeSlices<T>(\n rows: T[],\n keyOf: (row: T) => string \| string[],\n knownKeys: readonly string[] = [],\n): Record<string, BenchmarkSliceSummary> {\n const grouped = new Map<string, T[]>()\n for (const key of knownKeys) grouped.set(key, [])\n for (const row of rows) {\n const keys = keyOf(row)\n for (const key of Array.isArray(keys) ? keys : [keys]) {\n const list = grouped.get(key) ?? []\n list.push(row)\n grouped.set(key, list)\n }\n }\n const out: Record<string, BenchmarkSliceSummary> = {}\n for (const [key, list] of grouped) {\n const withShape = list as Array<{\n score: number\n passed: boolean\n cell: { costUsd: number; durationMs: number }\n }>\n const scores = withShape.map((row) => row.score).filter(Number.isFinite)\n const passed = withShape.filter((row) => row.passed).length\n out[key] = {\n n: list.length,\n meanScore: scores.length ? mean(scores) : 0,\n passRate: list.length ? passed / list.length : 0,\n score: distribution(scores),\n costUsd: distribution(withShape.map((row) => row.cell.costUsd)),\n latencyMs: distribution(withShape.map((row) => row.cell.durationMs)),\n }\n }\n return out\n}\n\nfunction firstJudgeScore(\n judgeScores: CampaignResult<unknown>['cells'][number]['judgeScores'],\n): JudgeScore \| undefined {\n return Object.values(judgeScores)[0]\n}\n\nfunction distribution(values: readonly number[]): BenchmarkDistribution {\n const summary = summarizeNumberSeries(values.filter(Number.isFinite))\n if (!summary) return { n: 0, min: 0, mean: 0, median: 0, p90: 0, max: 0 }\n const { n, min, p50: median, p90, max, sum } = summary\n return { n, min, mean: sum / n, median, p90, max }\n}\n\nfunction clamp01(value: number): number {\n if (!Number.isFinite(value)) return 0\n if (value < 0) return 0\n if (value > 1) return 1\n return value\n}\n\nfunction fmt(value: number): string {\n if (!Number.isFinite(value)) return '0'\n return value.toFixed(value === 0 \|\| Math.abs(value) >= 10 ? 0 : 3)\n}\n","import type { RunSplitTag } from '../run-record'\nimport type {\n BenchmarkAdapter,\n BenchmarkDatasetItem,\n BenchmarkFamily,\n BenchmarkSource,\n BenchmarkTaskKind,\n} from './types'\nimport { deterministicSplit } from './types'\n\nexport interface StandardRetrievalDocument {\n id: string\n title?: string\n text: string\n metadata?: Record<string, unknown>\n}\n\nexport interface StandardRetrievalQuery {\n id: string\n text: string\n metadata?: Record<string, unknown>\n}\n\nexport interface StandardRetrievalQrel {\n queryId: string\n documentId: string\n score: number\n}\n\nexport interface StandardRetrievalPayload {\n queryId: string\n query: string\n expectedDocumentIds: string[]\n expectedScores: Record<string, number>\n corpus?: Record<string, StandardRetrievalDocument>\n metadata?: Record<string, unknown>\n}\n\nexport interface BuildStandardRetrievalItemsOptions {\n benchmarkId: string\n family: BenchmarkFamily \| string\n queries: readonly StandardRetrievalQuery[]\n qrels: readonly StandardRetrievalQrel[]\n corpus?: readonly StandardRetrievalDocument[]\n includeCorpusInPayload?: boolean\n source?: BenchmarkSource\n tags?: readonly string[]\n splitOf?: (queryId: string) => RunSplitTag\n}\n\nexport interface RetrievalIdAdapterOptions extends BuildStandardRetrievalItemsOptions {\n responseIdPattern?: RegExp\n cutoffs?: readonly number[]\n primaryMetric?: string\n passMetric?: string\n passThreshold?: number\n}\n\nexport interface StandardRetrievalResult {\n id?: string\n documentId?: string\n docId?: string\n score?: number\n}\n\nexport type StandardRetrievalArtifact =\n \| string\n \| readonly string[]\n \| readonly StandardRetrievalResult[]\n \| {\n ids?: readonly string[]\n documentIds?: readonly string[]\n results?: readonly StandardRetrievalResult[]\n }\n\nexport interface StandardRetrievalEvaluationOptions {\n responseIdPattern?: RegExp\n cutoffs?: readonly number[]\n primaryMetric?: string\n passMetric?: string\n passThreshold?: number\n}\n\nfunction parseJsonlRows<T = unknown>(text: string): T[] {\n return text\n .split(/\\r?\\n/)\n .map((line) => line.trim())\n .filter(Boolean)\n .map((line, index) => {\n try {\n return JSON.parse(line) as T\n } catch (error) {\n throw new Error(`invalid JSONL row ${index + 1}: ${(error as Error).message}`)\n }\n })\n}\n\nfunction parseTsvRows(text: string): string[][] {\n return text\n .split(/\\r?\\n/)\n .map((line) => line.trim())\n .filter(Boolean)\n .filter((line) => !line.startsWith('#'))\n .map((line) => line.split(/\\t\|\\s+/))\n}\n\nexport function parseQrels(text: string): StandardRetrievalQrel[] {\n return parseTsvRows(text).flatMap((parts, index) => {\n if (parts.length < 3) return []\n const [queryId, maybeZeroOrDocId, maybeDocIdOrScore, maybeScore] = parts\n if (!queryId \|\| !maybeZeroOrDocId \|\| !maybeDocIdOrScore) return []\n if (queryId.toLowerCase() === 'query-id' \|\| queryId.toLowerCase() === 'qid') return []\n const documentId = maybeScore === undefined ? maybeZeroOrDocId : maybeDocIdOrScore\n const scoreText = maybeScore === undefined ? maybeDocIdOrScore : maybeScore\n const score = Number(scoreText)\n if (!documentId \|\| !Number.isFinite(score)) {\n throw new Error(`invalid qrels row ${index + 1}: expected query id, doc id, score`)\n }\n return [{ queryId, documentId, score }]\n })\n}\n\nexport function parseBeirCorpusJsonl(text: string): StandardRetrievalDocument[] {\n return parseJsonlRows<Record<string, unknown>>(text).map((row, index) => {\n const id = stringField(row, '_id') ?? stringField(row, 'id')\n const body = stringField(row, 'text') ?? stringField(row, 'contents')\n if (!id \|\| body === undefined) {\n throw new Error(`invalid BEIR corpus row ${index + 1}: expected _id/id and text/contents`)\n }\n return {\n id,\n title: stringField(row, 'title'),\n text: body,\n metadata: stripKnown(row, ['_id', 'id', 'title', 'text', 'contents']),\n }\n })\n}\n\nexport function parseBeirQueriesJsonl(text: string): StandardRetrievalQuery[] {\n return parseJsonlRows<Record<string, unknown>>(text).map((row, index) => {\n const id = stringField(row, '_id') ?? stringField(row, 'id') ?? stringField(row, 'query_id')\n const query = stringField(row, 'text') ?? stringField(row, 'query')\n if (!id \|\| !query) {\n throw new Error(`invalid BEIR query row ${index + 1}: expected _id/id and text/query`)\n }\n return {\n id,\n text: query,\n metadata: stripKnown(row, ['_id', 'id', 'query_id', 'text', 'query']),\n }\n })\n}\n\nexport function buildStandardRetrievalItems(\n options: BuildStandardRetrievalItemsOptions,\n): Array<BenchmarkDatasetItem<StandardRetrievalPayload>> {\n const qrelsByQuery = new Map<string, StandardRetrievalQrel[]>()\n for (const qrel of options.qrels) {\n if (qrel.score <= 0) continue\n const list = qrelsByQuery.get(qrel.queryId) ?? []\n list.push(qrel)\n qrelsByQuery.set(qrel.queryId, list)\n }\n const corpus =\n options.includeCorpusInPayload && options.corpus\n ? Object.fromEntries(options.corpus.map((document) => [document.id, document]))\n : undefined\n return options.queries.flatMap((query) => {\n const qrels = qrelsByQuery.get(query.id) ?? []\n if (qrels.length === 0) return []\n const split =\n options.splitOf?.(query.id) ?? deterministicSplit(`${options.benchmarkId}:${query.id}`)\n return [\n {\n id: query.id,\n split,\n family: options.family,\n taskKind: 'retrieval',\n tags: [...new Set([...(options.tags ?? []), split])],\n ...(options.source ? { source: options.source } : {}),\n ...(query.metadata ? { metadata: query.metadata } : {}),\n payload: {\n queryId: query.id,\n query: query.text,\n expectedDocumentIds: qrels.map((qrel) => qrel.documentId),\n expectedScores: Object.fromEntries(qrels.map((qrel) => [qrel.documentId, qrel.score])),\n ...(corpus ? { corpus } : {}),\n ...(query.metadata ? { metadata: query.metadata } : {}),\n },\n },\n ]\n })\n}\n\nexport function createRetrievalIdBenchmarkAdapter(\n options: RetrievalIdAdapterOptions,\n): BenchmarkAdapter<\n BenchmarkDatasetItem<StandardRetrievalPayload>,\n StandardRetrievalPayload,\n StandardRetrievalArtifact\n> {\n const items = buildStandardRetrievalItems(options)\n return {\n id: options.benchmarkId,\n family: options.family,\n taskKind: 'retrieval' satisfies BenchmarkTaskKind,\n source: options.source,\n defaultMetric: options.primaryMetric ?? 'ndcg@10',\n async loadDataset(split) {\n return items.filter((item) => item.split === split)\n },\n async evaluate(item, artifact) {\n return evaluateStandardRetrieval(item.payload, artifact, options)\n },\n assignSplit(itemId) {\n return options.splitOf?.(itemId) ?? deterministicSplit(`${options.benchmarkId}:${itemId}`)\n },\n }\n}\n\nexport function evaluateStandardRetrieval(\n payload: StandardRetrievalPayload,\n artifact: StandardRetrievalArtifact,\n options: StandardRetrievalEvaluationOptions = {},\n) {\n const rankedDocumentIds = normalizeRetrievedDocumentIds(\n artifact,\n options.responseIdPattern ?? /[A-Za-z0-9_.:/-]+/g,\n )\n const cutoffs = normalizeCutoffs(options.cutoffs ?? [1, 3, 5, 10])\n const dimensions: Record<string, number> = {\n expected_count: payload.expectedDocumentIds.length,\n returned_count: rankedDocumentIds.length,\n }\n for (const cutoff of cutoffs) {\n Object.assign(\n dimensions,\n retrievalMetricsAtCutoff({\n rankedDocumentIds,\n expectedScores: payload.expectedScores,\n cutoff,\n }),\n )\n }\n const primaryMetric = options.primaryMetric ?? 'ndcg@10'\n const passMetric = options.passMetric ?? 'hit@10'\n const score = dimensions[primaryMetric] ?? dimensions[`ndcg@${cutoffs[cutoffs.length - 1]}`] ?? 0\n const passScore =\n dimensions[passMetric] ?? dimensions[`hit@${cutoffs[cutoffs.length - 1]}`] ?? score\n return {\n score,\n passed: passScore >= (options.passThreshold ?? 1),\n dimensions,\n raw: {\n rankedDocumentIds,\n expectedDocumentIds: payload.expectedDocumentIds,\n expectedScores: payload.expectedScores,\n },\n }\n}\n\nfunction normalizeRetrievedDocumentIds(\n artifact: StandardRetrievalArtifact,\n responseIdPattern: RegExp = /[A-Za-z0-9_.:/-]+/g,\n): string[] {\n if (typeof artifact === 'string') {\n return uniqueOrdered((artifact.match(responseIdPattern) ?? []).map((value) => value.trim()))\n }\n if (Array.isArray(artifact)) {\n const entries = artifact as readonly (string \| StandardRetrievalResult)[]\n return uniqueOrdered(\n entries.flatMap((entry) => {\n if (typeof entry === 'string') return [entry]\n return [entry.documentId ?? entry.docId ?? entry.id ?? '']\n }),\n )\n }\n const objectArtifact = artifact as Exclude<StandardRetrievalArtifact, string \| readonly unknown[]>\n return uniqueOrdered(\n [\n ...(objectArtifact.documentIds ?? []),\n ...(objectArtifact.ids ?? []),\n ...(objectArtifact.results ?? []).map(\n (entry) => entry.documentId ?? entry.docId ?? entry.id ?? '',\n ),\n ].map((value) => value.trim()),\n )\n}\n\nfunction retrievalMetricsAtCutoff(input: {\n rankedDocumentIds: readonly string[]\n expectedScores: Record<string, number>\n cutoff: number\n}): Record<string, number> {\n const top = input.rankedDocumentIds.slice(0, input.cutoff)\n const relevantIds = Object.keys(input.expectedScores).filter(\n (id) => input.expectedScores[id]! > 0,\n )\n const relevant = new Set(relevantIds)\n const hits = top.filter((id) => relevant.has(id)).length\n const firstRelevantRank = top.findIndex((id) => relevant.has(id))\n const prefix = `@${input.cutoff}`\n return {\n [`hit${prefix}`]: hits > 0 ? 1 : 0,\n [`recall${prefix}`]: relevant.size === 0 ? 1 : hits / relevant.size,\n [`precision${prefix}`]: hits / input.cutoff,\n [`mrr${prefix}`]: firstRelevantRank === -1 ? 0 : 1 / (firstRelevantRank + 1),\n [`ndcg${prefix}`]: ndcgAt(input.rankedDocumentIds, input.expectedScores, input.cutoff),\n }\n}\n\nfunction stringField(row: Record<string, unknown>, key: string): string \| undefined {\n const value = row[key]\n return typeof value === 'string' ? value : undefined\n}\n\nfunction stripKnown(\n row: Record<string, unknown>,\n keys: readonly string[],\n): Record<string, unknown> {\n const known = new Set(keys)\n return Object.fromEntries(Object.entries(row).filter(([key]) => !known.has(key)))\n}\n\nfunction normalizeCutoffs(cutoffs: readonly number[]): number[] {\n return [\n ...new Set(cutoffs.map((cutoff) => Math.trunc(cutoff)).filter((cutoff) => cutoff > 0)),\n ].sort((a, b) => a - b)\n}\n\nfunction uniqueOrdered(values: readonly string[]): string[] {\n const seen = new Set<string>()\n const out: string[] = []\n for (const value of values) {\n const trimmed = value.trim()\n if (!trimmed \|\| seen.has(trimmed)) continue\n seen.add(trimmed)\n out.push(trimmed)\n }\n return out\n}\n\nfunction ndcgAt(\n rankedDocumentIds: readonly string[],\n expectedScores: Record<string, number>,\n cutoff: number,\n): number {\n const dcg = rankedDocumentIds\n .slice(0, cutoff)\n .reduce(\n (sum, documentId, index) => sum + discountedGain(expectedScores[documentId] ?? 0, index),\n 0,\n )\n const ideal = Object.values(expectedScores)\n .filter((score) => score > 0)\n .sort((a, b) => b - a)\n .slice(0, cutoff)\n .reduce((sum, score, index) => sum + discountedGain(score, index), 0)\n return ideal === 0 ? 1 : dcg / ideal\n}\n\nfunction discountedGain(relevance: number, zeroBasedRank: number): number {\n if (relevance <= 0) return 0\n return (2 * relevance - 1) / Math.log2(zeroBasedRank + 2)\n}\n"],"mappings":";;;;;;;;;AAsBA,eAAsB,yBACpB,SAC2C;CAC3C,MAAM,OAAO,MAAM,QAAQ,QAAQ,SAAS,QAAQ,MAAM,QAAQ,YAAY;CAC9E,MAAM,SAAS,MAAM,QAAQ,QAAQ,SAAS,QAAQ,MAAM,QAAQ,cAAc;CAClF,MAAM,YAAYA,UAAQ,KAAK,KAAK;CACpC,MAAM,cAAcA,UAAQ,OAAO,KAAK;CACxC,MAAM,eAAe,QAAQ,gBAAgB;CAC7C,MAAM,iBAAiB,QAAQ,kBAAkB;CACjD,MAAM,SAAS,QAAQ,UAAU;CACjC,MAAM,MAAM,cAAc;CAC1B,MAAM,UAAU;EACd,aAAa,eACT,KAAA,IACA,cAAc,UAAU,QAAQ,CAAC,EAAE,eAAe,aAAa,QAAQ,CAAC;EAC5E,eAAe,iBACX,KAAA,IACA,gBAAgB,YAAY,QAAQ,CAAC,EAAE,aAAa,eAAe,QAAQ,CAAC;EAChF,OAAO,SAAS,KAAA,IAAY,OAAO,IAAI,QAAQ,CAAC,EAAE,aAAa,OAAO,QAAQ,CAAC;CACjF,CAAC,CAAC,QAAQ,WAA6B,QAAQ,MAAM,CAAC;CAEtD,OAAO;EACL,QAAQ,QAAQ,WAAW;EAC3B;EACA;EACA;EACA;EACA;EACA;CACF;AACF;AAEA,SAASA,UAAQ,OAAuB;CACtC,IAAI,CAAC,OAAO,SAAS,KAAK,GAAG,OAAO;CACpC,IAAI,QAAQ,GAAG,OAAO;CACtB,IAAI,QAAQ,GAAG,OAAO;CACtB,OAAO;AACT;;;;;;;ACyEA,SAAS,QAAQ,OAAuB;CACtC,IAAI,IAAI;CACR,KAAK,IAAI,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK;EACrC,KAAK,MAAM,WAAW,CAAC,IAAI;EAC3B,IAAK,MAAM,KAAK,MAAM,KAAK,MAAM,KAAK,MAAM,KAAK,MAAM,KAAK,SAAU;CACxE;CACA,OAAO,MAAM;AACf;;;AAIA,MAAa,uBAAuB;;;;;;;;;AAUpC,SAAgB,mBACd,QACA,OAAe,sBACF;CAEb,MAAM,MADI,QAAQ,GAAG,KAAK,IAAI,QAClB,IAAI;CAChB,IAAI,MAAM,IAAK,OAAO;CACtB,IAAI,MAAM,IAAK,OAAO;CACtB,OAAO;AACT;;;;;;;;;;;;;;;ACzEA,MAAM,SAAS,UACb,UAAU,QAAQ,OAAO,UAAU,YAAY,CAAC,MAAM,QAAQ,KAAK,IAC9D,QACD,CAAC;AACP,MAAM,gBAAgB,UACpB,OAAO,UAAU,YAAY,OAAO,SAAS,KAAK,IAAI,QAAQ;AAChE,MAAM,cAAc,UAClB,OAAO,UAAU,YAAY,MAAM,SAAS,IAAI,QAAQ;;AAG1D,eAAe,eAAe,SAA6C;CACzE,MAAM,MAAM,QAAQ,OAAO;CAC3B,MAAM,SAASC,MAAU,MAAM,SAAS,KAAK,KAAK,WAAW,GAAG,MAAM,CAAC;CAIvE,MAAM,cAAc,MAAM,SAAS,KAAK,KAAK,gBAAgB,GAAG,MAAM;CACtE,MAAM,OAAO,MAAM,OAAO,IAAI;CAC9B,MAAM,QAAQ,MAAM,OAAO,KAAK;CAChC,MAAM,WAAW,MAAM,OAAO,QAAQ;CACtC,MAAM,cAAc,MAAM,OAAO,WAAW;CAC5C,MAAM,WAAW,MAAM,OAAO,QAAQ;CACtC,MAAM,aAAa,MAAM,QAAQ,OAAO,SAAS,IAAI,OAAO,YAAY,CAAC,EAAA,CAAG,KACzE,OAAO,UAAU;EAChB,MAAM,SAAS,OAAO,UAAU,WAAW,QAAQ,MAAM,KAAK,CAAC,CAAC;EAChE,IAAI,OAAO,WAAW,YAAY,OAAO,WAAW,GAClD,MAAM,IAAI,MAAM,GAAG,IAAI,wBAAwB,MAAM,uBAAuB;EAE9E,OAAO;CACT,CACF;CACA,MAAM,mBAAmB,MAAM,QAAQ,KAAK,KAAK,aAAa,CAAC,CAAC,CAAC,YAAY,CAAC,CAAa;CAC3F,MAAM,OAAO,IAAI,MAAM,GAAG,CAAC,CAAC,IAAI,KAAK;CACrC,OAAO;EACL,MAAM,WAAW,KAAK,IAAI,KAAK;EAC/B,KAAK;EACL,SAAS;EACT;EACA,iBAAiB,aAAa,MAAM,WAAW;EAC/C,oBAAoB,aAAa,SAAS,WAAW;EACrD,cAAc,WAAW,SAAS,gBAAgB;EAClD,WAAW;GACT,MAAM,aAAa,YAAY,IAAI;GACnC,UAAU,aAAa,YAAY,SAAS;GAC5C,WAAW,aAAa,YAAY,UAAU;GAC9C,MAAM,aAAa,YAAY,IAAI,KAAK;EAC1C;EACA;EACA,gBAAgB,iBAAiB,MAAM,SAAS,mBAAmB,KAAK,IAAI,CAAC;EAC7E,UAAU,WAAW,SAAS,QAAQ;EACtC,yBAAyB,aAAa,SAAS,0BAA0B;CAC3E;AACF;;AAGA,SAAgB,6BACd,SACmC;CACnC,MAAM,QACJ,QAAQ,iBAAiB,WAAmB,mBAAmB,GAAG,QAAQ,GAAG,IAAI,QAAQ;CAC3F,MAAM,WAAW,KAAK,QAAQ,QAAQ,UAAU,GAAG,OAAO;CAE1D,eAAe,UAAqC;EAClD,MAAM,QACJ,QAAQ,UAEN,MAAM,QAAQ,KAEV,MAAM,QAAQ,QAAQ,EAAA,CACtB,IAAI,OAAO,UACV,MAAM,KAAK,KAAK,UAAU,OAAO,WAAW,CAAC,CAAC,CAAC,YAAY,IAAI,MAAO,OACnE,OACA,KACN,CACF,EAAA,CAEC,QAAQ,UAA2B,UAAU,IAAI,CAAC,CAClD,KAAK;EACV,OAAO,QAAQ,IACb,MAAM,IAAI,OAAO,QAAQ;GACvB,MAAM,UAAU,MAAM,eAAe,KAAK,UAAU,GAAG,CAAC;GACxD,OAAO;IACL,IAAI;IACJ;IACA,OAAO,MAAM,GAAG;IAChB,QAAQ,QAAQ,OAAO,QAAQ;IAC/B,UAAU;IACV,MAAM,QAAQ,aAAa,OAAO,CAAC,IAAI,CAAC,QAAQ,QAAQ;IACxD,QAAQ,QAAQ;GAClB;EACF,CAAC,CACH;CACF;CAEA,OAAO;EACL,IAAI,QAAQ;EACZ,QAAQ,QAAQ,OAAO,QAAQ;EAC/B,UAAU;EACV,aACE,GAAG,QAAQ,OAAO,QAAQ,iBAAiB,GAAG,QAAQ,OAAO,WAAW,GAAG,MAAM,QAAQ,OAAO,SAAS,8BAA8B,QACrI,SACA,GACF;EACF,QAAQ,QAAQ;EAChB,eAAe;EACf;EACA,MAAM,YAAY,KAAkB;GAClC,QAAQ,MAAM,QAAQ,EAAA,CAAG,QAAQ,SAAS,KAAK,UAAU,GAAG;EAC9D;EACA,MAAM,SAAS,MAAsB,UAAmD;GACtF,MAAM,UAAU,MAAM,QAAQ,MAAM,MAAM,QAAQ;GAClD,IAAI,CAAC,QAAQ,WACX,MAAM,IAAI,MACR,GAAG,QAAQ,GAAG,GAAG,KAAK,GAAG,8CAA8C,QAAQ,OACjF;GACF,MAAM,EAAE,QAAQ,WAAW,QAAQ;GACnC,IAAI,EAAE,OAAO,SAAS,MAAM,KAAK,UAAU,KAAK,UAAU,IACxD,MAAM,IAAI,MACR,GAAG,QAAQ,GAAG,GAAG,KAAK,GAAG,iCAAiC,OAAO,iBACnE;GAEF,OAAO;IACL,OAAO;IACP,QAAQ,WAAW;IACnB,KAAK;KAAE;KAAQ,GAAI,WAAW,KAAA,IAAY,CAAC,IAAI,EAAE,OAAO;IAAG;GAC7D;EACF;EACA,aAAa;CACf;AACF;;ACxLA,MAAa,kBAAiC;CAC5C;EACE,IAAI;EACJ,UAAU;EACV,QAAQ;EACR,OAAO;EACP,UAAU,CAAC,oBAAoB,YAAY;EAC3C,eAAe,CAAC,WAAW,YAAY;CACzC;CACA;EACE,IAAI;EACJ,UAAU;EACV,QAAQ;EACR,OAAO;EACP,UAAU,CAAC,mBAAmB,WAAW;EACzC,eAAe,CAAC,YAAY,YAAY;CAC1C;CACA;EACE,IAAI;EACJ,UAAU;EACV,QAAQ;EACR,OAAO;EACP,UAAU,CAAC,mBAAmB,YAAY;EAC1C,eAAe,CAAC,WAAW,aAAa;CAC1C;CACA;EACE,IAAI;EACJ,UAAU;EACV,QAAQ;EACR,OAAO;EACP,UAAU,CAAC,qBAAqB,aAAa;EAC7C,eAAe,CAAC,YAAY,SAAS;CACvC;CACA;EACE,IAAI;EACJ,UAAU;EACV,QAAQ;EACR,OAAO;EACP,UAAU,CAAC,uBAAuB,YAAY;EAC9C,eAAe,CAAC,mBAAmB,YAAY;CACjD;CACA;EACE,IAAI;EACJ,UAAU;EACV,QAAQ;EACR,OAAO;EACP,UAAU,CAAC,yBAAyB,YAAY;EAChD,eAAe,CAAC,iBAAiB,cAAc;CACjD;CACA;EACE,IAAI;EACJ,UAAU;EACV,QAAQ;EACR,OAAO;EACP,UAAU,CAAC,sBAAsB,YAAY;EAC7C,eAAe,CAAC,mBAAmB,eAAe;CACpD;CACA;EACE,IAAI;EACJ,UAAU;EACV,QAAQ;EACR,OAAO;EACP,UAAU,CAAC,qBAAqB,YAAY;EAC5C,eAAe,CAAC,gBAAgB,YAAY;CAC9C;CACA;EACE,IAAI;EACJ,UAAU;EACV,QAAQ;EACR,OAAO;EACP,UAAU,CAAC,cAAc,eAAe;EACxC,eAAe,CAAC,eAAe,YAAY;CAC7C;CACA;EACE,IAAI;EACJ,UAAU;EACV,QAAQ;EACR,OAAO;EACP,UAAU,CAAC,eAAe,MAAM;EAChC,eAAe,CAAC,cAAc,SAAS;CACzC;CACA;EACE,IAAI;EACJ,UAAU;EACV,QAAQ;EACR,OAAO;EACP,UAAU,CAAC,cAAc,iBAAiB;EAC1C,eAAe,CAAC,eAAe,YAAY;CAC7C;CACA;EACE,IAAI;EACJ,UAAU;EACV,QAAQ;EACR,OAAO;EACP,UAAU,CAAC,eAAe,MAAM;EAChC,eAAe,CAAC,cAAc,SAAS;CACzC;CACA;EACE,IAAI;EACJ,UAAU;EACV,QAAQ;EACR,OAAO;EACP,UAAU,CAAC,oBAAoB;EAC/B,eAAe,CAAC,cAAc,SAAS;CACzC;CACA;EACE,IAAI;EACJ,UAAU;EACV,QAAQ;EACR,OAAO;EACP,UAAU,CAAC,sBAAsB,WAAW;EAC5C,eAAe,CAAC,cAAc,cAAc;CAC9C;CACA;EACE,IAAI;EACJ,UAAU;EACV,QAAQ;EACR,OAAO;EACP,UAAU,CAAC,oBAAoB;EAC/B,eAAe,CAAC,cAAc,SAAS;CACzC;CACA;EACE,IAAI;EACJ,UAAU;EACV,QAAQ;EACR,OAAO;EACP,UAAU,CAAC,sBAAsB,OAAO;EACxC,eAAe,CAAC,YAAY,YAAY;CAC1C;AACF;;;;;;;;;;AC9IA,IAAM,iBAAN,MAAqF;CACnF,KAAc;CACd,SAAkB;CAClB,WAAoB;CACpB,cAAuB;CACvB,gBAAyB;CAEzB,MAAM,YAAY,OAAmD;EACnE,OAAO,gBAAgB,KAAK,UAAU;GAAE,IAAI,KAAK;GAAI,SAAS;EAAK,EAAE,CAAC,CAAC,QACpE,OAAO,gBAAgB,GAAG,EAAE,MAAM,KACrC;CACF;CAEA,MAAM,SAAS,MAA0B,UAAgD;EACvF,MAAM,SAAS,mBAAmB,QAAQ;EAC1C,MAAM,UAAU,IAAI,IAClB,CAAC,KAAK,QAAQ,OAAO,GAAG,KAAK,QAAQ,QAAQ,CAAC,CAAC,KAAK,MAAM,EAAE,YAAY,CAAC,CAC3E;EACA,MAAM,UAAU,IAAI,IAAY,KAAK,QAAQ,cAAc,KAAK,MAAM,EAAE,YAAY,CAAC,CAAC;EACtF,MAAM,aAAa,OAAO,MAAM,MAAM,QAAQ,IAAI,EAAE,YAAY,CAAC,CAAC,KAAK;EACvE,MAAM,eAAe,OAAO,MAAM,MAAM,QAAQ,IAAI,EAAE,YAAY,CAAC,CAAC,KAAK;EAEzE,OAAO;GACL,OAFY,aAAa,IAAI;GAG7B,KAAK;IACH,YAAY,OAAO,MAAM;IACzB,cAAc;IACd,iBAAiB,QAAQ,YAAY;IACrC,mBAAmB;IACnB,UAAU,KAAK,QAAQ;GACzB;EACF;CACF;CAEA,YAAY,QAA6B;EACvC,OAAO,gBAAgB,MAAM;CAC/B;AACF;AAEA,SAAS,gBAAgB,QAA6B;CACpD,OAAO,mBAAmB,YAAY,QAAQ;AAChD;;;;;;;AAQA,SAAgB,mBAAmB,UAA4B;CAE7D,OADgB,SAAS,MAAM,oCAClB,KAAK,CAAC;AACrB;AAEA,MAAM,UAAU,IAAI,eAAe;AAEnC,MAAa,cAAc,QAAQ,YAAY,KAAK,OAAO;AAC3D,MAAa,WAAW,QAAQ,SAAS,KAAK,OAAO;AACrD,MAAa,cAAc,QAAQ,YAAY,KAAK,OAAO;;ACW3D,eAAsB,oBACpB,SACkD;CAClD,MAAM,UAAU,QAAQ,WAAW,kBAAkB;CACrD,MAAM,cAAc,eAAe,QAAQ,OAAO;CAClD,MAAM,YAAY,MAAM,uBAAuB,QAAQ,SAAS,QAAQ,MAAM;CAC9E,MAAM,QAAQ,sBAAsB,QAAQ,OAAO;CACnD,MAAM,WAA+D,OACnE,UACA,YACG;EACH,OAAO,QAAQ,QAAQ;GAAE;GAAU,MAAM,SAAS;GAAM;EAAQ,CAAC;CACnE;CAEA,MAAM,WAAW,MAAM,YAAoD;EACzE;EACA;EACA,aAAa,aAAa;EAC1B,QAAQ,CAAC,KAAK;EACd,MAAM,QAAQ;EACd,MAAM,QAAQ;EACd,WAAW,QAAQ;EACnB,aAAa,QAAQ;EACrB,gBAAgB,QAAQ;EACxB,mBAAmB,QAAQ;EAC3B,aAAa,QAAQ,eAAe;EACpC,QAAQ,QAAQ;EAChB,MAAM,QAAQ;EACd;EACA,KAAK,QAAQ;CACf,CAAC;CACD,MAAM,SAAS,2BAA2B;EACxC,SAAS,QAAQ;EACjB;EACA;CACF,CAAC;CACD,QAAQ,UAAU,SAAS,MAAM;CACjC,MAAM,iBAAiB,KAAK,SAAS,QAAQ,uBAAuB;CACpE,MAAM,qBAAqB,KAAK,SAAS,QAAQ,qBAAqB;CACtE,QAAQ,MAAM,gBAAgB,GAAG,KAAK,UAAU,QAAQ,MAAM,CAAC,EAAE,GAAG;CACpE,QAAQ,MAAM,oBAAoB,8BAA8B,MAAM,CAAC;CACvE,OAAO;EAAE;EAAW;EAAU;EAAQ;EAAgB;CAAmB;AAC3E;AAEA,eAAe,uBACb,SACA,SAAiC;CAAC;CAAU;CAAO;AAAS,GACf;CAC7C,MAAM,cAAc,eAAe,OAAO;CAC1C,MAAM,SAAS,QAAQ,UAAU;CACjC,MAAM,WAAW,QAAQ,YAAY;CACrC,MAAM,MAA0C,CAAC;CACjD,MAAM,uBAAO,IAAI,IAAY;CAC7B,KAAK,MAAM,SAAS,QAAQ;EAC1B,MAAM,QAAQ,MAAM,QAAQ,YAAY,KAAK;EAC7C,KAAK,MAAM,QAAQ,OAAO;GACxB,MAAM,WAAW,KAAK,SAAS,QAAQ,YAAY,KAAK,EAAE;GAC1D,IAAI,aAAa,OAAO;GACxB,MAAM,MAAM,GAAG,YAAY,GAAG,KAAK;GACnC,IAAI,KAAK,IAAI,GAAG,GAAG;GACnB,KAAK,IAAI,GAAG;GACZ,IAAI,KAAK;IACP,IAAI;IACJ,MAAM;IACN;IACA,QAAQ,KAAK,UAAU;IACvB,UAAU,KAAK,YAAY;IAC3B;IACA,MAAM,CAAC,mBAAG,IAAI,IAAI,CAAC,UAAU,GAAI,KAAK,QAAQ,CAAC,CAAE,CAAC,CAAC;IACnD;GACF,CAAC;EACH;CACF;CACA,OAAO;AACT;AAEA,SAAS,sBACP,SACqD;CACrD,MAAM,cAAc,eAAe,OAAO;CAC1C,OAAO;EACL,MAAM,GAAG,YAAY;EACrB,YAAY,CACV;GAAE,KAAK;GAAS,aAAa,WAAW,YAAY;EAAkB,GACtE;GAAE,KAAK;GAAU,aAAa;EAAiD,CACjF;EACA,YAAY,aAAgE;GAC1E,OAAO,SAAS,SAAS,eAAe,SAAS,GAAG,WAAW,GAAG,YAAY,EAAE;EAClF;EACA,MAAM,MAAM,EAAE,UAAU,YAAY;GAClC,MAAM,aAAa,MAAM,QAAQ,SAAS,SAAS,MAAM,QAAQ;GAEjE,OAAO;IACL,YAFiB,8BAA8B,UAEtC;IACT,WAAW,QAAQ,WAAW,KAAK;IACnC,OAAO,WAAW,SAAS;GAC7B;EACF;CACF;AACF;AAEA,SAAS,2BAAgD,OAIrC;CAClB,MAAM,eAAe,IAAI,IAAI,MAAM,UAAU,KAAK,aAAa,CAAC,SAAS,IAAI,QAAQ,CAAC,CAAC;CAavF,MAAM,aAZO,MAAM,SAAS,MAAM,KAAK,SAAS;EAC9C,MAAM,WAAW,aAAa,IAAI,KAAK,UAAU;EACjD,MAAM,QAAQ,gBAAgB,KAAK,WAAW;EAC9C,MAAM,QAAQ,OAAO,aAAa;EAClC,OAAO;GACL;GACA;GACA;GACA,SAAS,OAAO,WAAW,WAAW,QAAQ,IAAI,IAAI,OAAO;GAC7D,YAAY,OAAO,cAAc,CAAC;EACpC;CACF,CACsB,CAAC,CAAC,QAAQ,QAAQ,CAAC,IAAI,KAAK,KAAK;CACvD,MAAM,cAAc,WAAW,KAAK,QAAQ,IAAI,KAAK;CACrD,OAAO;EACL,aAAa,eAAe,MAAM,OAAO;EACzC,QAAQ,MAAM,QAAQ,UAAU;EAChC,UAAU,MAAM,QAAQ,YAAY;EACpC,GAAI,MAAM,QAAQ,SAAS,EAAE,QAAQ,MAAM,QAAQ,OAAO,IAAI,CAAC;EAC/D,QAAQ,MAAM,SAAS;EACvB,cAAc,MAAM,SAAS;EAC7B,MAAM,MAAM,SAAS;EACrB,WAAW,MAAM,SAAS;EAC1B,SAAS,MAAM,SAAS;EACxB,YAAY,MAAM,SAAS;EAC3B,YAAY,MAAM,UAAU;EAC5B,YAAY,MAAM,SAAS,MAAM;EACjC,aAAa,MAAM,SAAS,WAAW;EACvC,aAAa,MAAM,SAAS,WAAW;EACvC,cAAc,MAAM,SAAS,WAAW,KAAK;EAC7C,QAAQ,gBAAgB,aAAa,QAAQ,IAAI,UAAU,YAAY,WAAW;GAChF;GACA;GACA;EACF,CAAC;EACD,MAAM,gBAAgB,aAAa,QAAQ,IAAI,UAAU,QAAQ,CAAC,CAAC;EACnE,YAAY,oBAAoB,WAAW,KAAK,QAAQ,IAAI,UAAU,CAAC;EACvE,OAAO,aAAa,WAAW;EAC/B,SAAS,aAAa,WAAW,KAAK,QAAQ,IAAI,KAAK,OAAO,CAAC;EAC/D,WAAW,aAAa,WAAW,KAAK,QAAQ,IAAI,KAAK,UAAU,CAAC;CACtE;AACF;AAEA,SAAS,8BAA8B,QAAiC;CACtE,MAAM,YAAY,OAAO,QAAQ,OAAO,MAAM,CAAC,CAC5C,KAAK,CAAC,OAAO,aAAa;EACzB,OAAO,KAAK,MAAM,KAAK,QAAQ,EAAE,KAAK,IAAI,QAAQ,SAAS,EAAE,KAAK,IAAI,QAAQ,QAAQ,EAAE,KAAK,IAAI,QAAQ,MAAM,GAAG,EAAE,KAAK,IAAI,QAAQ,QAAQ,IAAI,EAAE,KAAK,IAAI,QAAQ,UAAU,GAAG,EAAE;CACrL,CAAC,CAAC,CACD,KAAK,IAAI;CACZ,MAAM,UAAU,OAAO,QAAQ,OAAO,UAAU,CAAC,CAC9C,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC,CAAC,CACtC,KAAK,CAAC,KAAK,UAAU,KAAK,IAAI,KAAK,KAAK,EAAE,KAAK,IAAI,KAAK,IAAI,EAAE,KAAK,IAAI,KAAK,GAAG,EAAE,GAAG,CAAC,CACrF,KAAK,IAAI;CACZ,OAAO;EACL,uBAAuB,OAAO;EAC9B;EACA,aAAa,OAAO;EACpB,gBAAgB,OAAO;EACvB,cAAc,OAAO;EACrB,eAAe,OAAO;EACtB,YAAY,OAAO,WAAW,UAAU,OAAO,YAAY,WAAW,OAAO,YAAY;EACzF,YAAY,IAAI,OAAO,YAAY;EACnC,iBAAiB,IAAI,OAAO,MAAM,IAAI,EAAE,WAAW,IAAI,OAAO,MAAM,MAAM,EAAE,QAAQ,IAAI,OAAO,MAAM,GAAG,EAAE,MAAM,OAAO,MAAM;EAC7H;EACA;EACA;EACA;EACA;EACA,aAAa;EACb;EACA;EACA;EACA;EACA;EACA,WAAW;EACX;CACF,CAAC,CAAC,KAAK,IAAI;AACb;AAEA,SAAS,eAAe,SAAuE;CAC7F,OAAO,QAAQ,MAAM,GAAG,QAAQ,UAAU,SAAS,GAAG,QAAQ,YAAY;AAC5E;AAEA,SAAS,8BAA8B,YAAyD;CAC9F,MAAM,aAAqC,EAAE,OAAO,QAAQ,WAAW,KAAK,EAAE;CAC9E,KAAK,MAAM,CAAC,KAAK,UAAU,OAAO,QAAQ,WAAW,cAAc,CAAC,CAAC,GACnE,IAAI,OAAO,SAAS,KAAK,GAAG,WAAW,OAAO;CAEhD,WAAW,SAAU,WAAW,UAAU,WAAW,QAAQ,IAAK,IAAI;CACtE,OAAO;AACT;AAEA,SAAS,oBACP,MACuC;CACvC,MAAM,yBAAS,IAAI,IAAsB;CACzC,KAAK,MAAM,OAAO,MAChB,KAAK,MAAM,CAAC,KAAK,UAAU,OAAO,QAAQ,GAAG,GAAG;EAC9C,IAAI,CAAC,OAAO,SAAS,KAAK,GAAG;EAC7B,MAAM,OAAO,OAAO,IAAI,GAAG,KAAK,CAAC;EACjC,KAAK,KAAK,KAAK;EACf,OAAO,IAAI,KAAK,IAAI;CACtB;CAEF,OAAO,OAAO,YAAY,CAAC,GAAG,OAAO,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,UAAU,CAAC,KAAK,aAAa,IAAI,CAAC,CAAC,CAAC;AACjG;AAEA,SAAS,gBACP,MACA,OACA,YAA+B,CAAC,GACO;CACvC,MAAM,0BAAU,IAAI,IAAiB;CACrC,KAAK,MAAM,OAAO,WAAW,QAAQ,IAAI,KAAK,CAAC,CAAC;CAChD,KAAK,MAAM,OAAO,MAAM;EACtB,MAAM,OAAO,MAAM,GAAG;EACtB,KAAK,MAAM,OAAO,MAAM,QAAQ,IAAI,IAAI,OAAO,CAAC,IAAI,GAAG;GACrD,MAAM,OAAO,QAAQ,IAAI,GAAG,KAAK,CAAC;GAClC,KAAK,KAAK,GAAG;GACb,QAAQ,IAAI,KAAK,IAAI;EACvB;CACF;CACA,MAAM,MAA6C,CAAC;CACpD,KAAK,MAAM,CAAC,KAAK,SAAS,SAAS;EACjC,MAAM,YAAY;EAKlB,MAAM,SAAS,UAAU,KAAK,QAAQ,IAAI,KAAK,CAAC,CAAC,OAAO,OAAO,QAAQ;EACvE,MAAM,SAAS,UAAU,QAAQ,QAAQ,IAAI,MAAM,CAAC,CAAC;EACrD,IAAI,OAAO;GACT,GAAG,KAAK;GACR,WAAW,OAAO,SAAS,KAAK,MAAM,IAAI;GAC1C,UAAU,KAAK,SAAS,SAAS,KAAK,SAAS;GAC/C,OAAO,aAAa,MAAM;GAC1B,SAAS,aAAa,UAAU,KAAK,QAAQ,IAAI,KAAK,OAAO,CAAC;GAC9D,WAAW,aAAa,UAAU,KAAK,QAAQ,IAAI,KAAK,UAAU,CAAC;EACrE;CACF;CACA,OAAO;AACT;AAEA,SAAS,gBACP,aACwB;CACxB,OAAO,OAAO,OAAO,WAAW,CAAC,CAAC;AACpC;AAEA,SAAS,aAAa,QAAkD;CACtE,MAAM,UAAU,sBAAsB,OAAO,OAAO,OAAO,QAAQ,CAAC;CACpE,IAAI,CAAC,SAAS,OAAO;EAAE,GAAG;EAAG,KAAK;EAAG,MAAM;EAAG,QAAQ;EAAG,KAAK;EAAG,KAAK;CAAE;CACxE,MAAM,EAAE,GAAG,KAAK,KAAK,QAAQ,KAAK,KAAK,QAAQ;CAC/C,OAAO;EAAE;EAAG;EAAK,MAAM,MAAM;EAAG;EAAQ;EAAK;CAAI;AACnD;AAEA,SAAS,QAAQ,OAAuB;CACtC,IAAI,CAAC,OAAO,SAAS,KAAK,GAAG,OAAO;CACpC,IAAI,QAAQ,GAAG,OAAO;CACtB,IAAI,QAAQ,GAAG,OAAO;CACtB,OAAO;AACT;AAEA,SAAS,IAAI,OAAuB;CAClC,IAAI,CAAC,OAAO,SAAS,KAAK,GAAG,OAAO;CACpC,OAAO,MAAM,QAAQ,UAAU,KAAK,KAAK,IAAI,KAAK,KAAK,KAAK,IAAI,CAAC;AACnE;;ACxRA,SAAS,eAA4B,MAAmB;CACtD,OAAO,KACJ,MAAM,OAAO,CAAC,CACd,KAAK,SAAS,KAAK,KAAK,CAAC,CAAC,CAC1B,OAAO,OAAO,CAAC,CACf,KAAK,MAAM,UAAU;EACpB,IAAI;GACF,OAAO,KAAK,MAAM,IAAI;EACxB,SAAS,OAAO;GACd,MAAM,IAAI,MAAM,qBAAqB,QAAQ,EAAE,IAAK,MAAgB,SAAS;EAC/E;CACF,CAAC;AACL;AAEA,SAAS,aAAa,MAA0B;CAC9C,OAAO,KACJ,MAAM,OAAO,CAAC,CACd,KAAK,SAAS,KAAK,KAAK,CAAC,CAAC,CAC1B,OAAO,OAAO,CAAC,CACf,QAAQ,SAAS,CAAC,KAAK,WAAW,GAAG,CAAC,CAAC,CACvC,KAAK,SAAS,KAAK,MAAM,QAAQ,CAAC;AACvC;AAEA,SAAgB,WAAW,MAAuC;CAChE,OAAO,aAAa,IAAI,CAAC,CAAC,SAAS,OAAO,UAAU;EAClD,IAAI,MAAM,SAAS,GAAG,OAAO,CAAC;EAC9B,MAAM,CAAC,SAAS,kBAAkB,mBAAmB,cAAc;EACnE,IAAI,CAAC,WAAW,CAAC,oBAAoB,CAAC,mBAAmB,OAAO,CAAC;EACjE,IAAI,QAAQ,YAAY,MAAM,cAAc,QAAQ,YAAY,MAAM,OAAO,OAAO,CAAC;EACrF,MAAM,aAAa,eAAe,KAAA,IAAY,mBAAmB;EAEjE,MAAM,QAAQ,OADI,eAAe,KAAA,IAAY,oBAAoB,UACnC;EAC9B,IAAI,CAAC,cAAc,CAAC,OAAO,SAAS,KAAK,GACvC,MAAM,IAAI,MAAM,qBAAqB,QAAQ,EAAE,mCAAmC;EAEpF,OAAO,CAAC;GAAE;GAAS;GAAY;EAAM,CAAC;CACxC,CAAC;AACH;AAEA,SAAgB,qBAAqB,MAA2C;CAC9E,OAAO,eAAwC,IAAI,CAAC,CAAC,KAAK,KAAK,UAAU;EACvE,MAAM,KAAK,YAAY,KAAK,KAAK,KAAK,YAAY,KAAK,IAAI;EAC3D,MAAM,OAAO,YAAY,KAAK,MAAM,KAAK,YAAY,KAAK,UAAU;EACpE,IAAI,CAAC,MAAM,SAAS,KAAA,GAClB,MAAM,IAAI,MAAM,2BAA2B,QAAQ,EAAE,oCAAoC;EAE3F,OAAO;GACL;GACA,OAAO,YAAY,KAAK,OAAO;GAC/B,MAAM;GACN,UAAU,WAAW,KAAK;IAAC;IAAO;IAAM;IAAS;IAAQ;GAAU,CAAC;EACtE;CACF,CAAC;AACH;AAEA,SAAgB,sBAAsB,MAAwC;CAC5E,OAAO,eAAwC,IAAI,CAAC,CAAC,KAAK,KAAK,UAAU;EACvE,MAAM,KAAK,YAAY,KAAK,KAAK,KAAK,YAAY,KAAK,IAAI,KAAK,YAAY,KAAK,UAAU;EAC3F,MAAM,QAAQ,YAAY,KAAK,MAAM,KAAK,YAAY,KAAK,OAAO;EAClE,IAAI,CAAC,MAAM,CAAC,OACV,MAAM,IAAI,MAAM,0BAA0B,QAAQ,EAAE,iCAAiC;EAEvF,OAAO;GACL;GACA,MAAM;GACN,UAAU,WAAW,KAAK;IAAC;IAAO;IAAM;IAAY;IAAQ;GAAO,CAAC;EACtE;CACF,CAAC;AACH;AAEA,SAAgB,4BACd,SACuD;CACvD,MAAM,+BAAe,IAAI,IAAqC;CAC9D,KAAK,MAAM,QAAQ,QAAQ,OAAO;EAChC,IAAI,KAAK,SAAS,GAAG;EACrB,MAAM,OAAO,aAAa,IAAI,KAAK,OAAO,KAAK,CAAC;EAChD,KAAK,KAAK,IAAI;EACd,aAAa,IAAI,KAAK,SAAS,IAAI;CACrC;CACA,MAAM,SACJ,QAAQ,0BAA0B,QAAQ,SACtC,OAAO,YAAY,QAAQ,OAAO,KAAK,aAAa,CAAC,SAAS,IAAI,QAAQ,CAAC,CAAC,IAC5E,KAAA;CACN,OAAO,QAAQ,QAAQ,SAAS,UAAU;EACxC,MAAM,QAAQ,aAAa,IAAI,MAAM,EAAE,KAAK,CAAC;EAC7C,IAAI,MAAM,WAAW,GAAG,OAAO,CAAC;EAChC,MAAM,QACJ,QAAQ,UAAU,MAAM,EAAE,KAAK,mBAAmB,GAAG,QAAQ,YAAY,GAAG,MAAM,IAAI;EACxF,OAAO,CACL;GACE,IAAI,MAAM;GACV;GACA,QAAQ,QAAQ;GAChB,UAAU;GACV,MAAM,CAAC,mBAAG,IAAI,IAAI,CAAC,GAAI,QAAQ,QAAQ,CAAC,GAAI,KAAK,CAAC,CAAC;GACnD,GAAI,QAAQ,SAAS,EAAE,QAAQ,QAAQ,OAAO,IAAI,CAAC;GACnD,GAAI,MAAM,WAAW,EAAE,UAAU,MAAM,SAAS,IAAI,CAAC;GACrD,SAAS;IACP,SAAS,MAAM;IACf,OAAO,MAAM;IACb,qBAAqB,MAAM,KAAK,SAAS,KAAK,UAAU;IACxD,gBAAgB,OAAO,YAAY,MAAM,KAAK,SAAS,CAAC,KAAK,YAAY,KAAK,KAAK,CAAC,CAAC;IACrF,GAAI,SAAS,EAAE,OAAO,IAAI,CAAC;IAC3B,GAAI,MAAM,WAAW,EAAE,UAAU,MAAM,SAAS,IAAI,CAAC;GACvD;EACF,CACF;CACF,CAAC;AACH;AAEA,SAAgB,kCACd,SAKA;CACA,MAAM,QAAQ,4BAA4B,OAAO;CACjD,OAAO;EACL,IAAI,QAAQ;EACZ,QAAQ,QAAQ;EAChB,UAAU;EACV,QAAQ,QAAQ;EAChB,eAAe,QAAQ,iBAAiB;EACxC,MAAM,YAAY,OAAO;GACvB,OAAO,MAAM,QAAQ,SAAS,KAAK,UAAU,KAAK;EACpD;EACA,MAAM,SAAS,MAAM,UAAU;GAC7B,OAAO,0BAA0B,KAAK,SAAS,UAAU,OAAO;EAClE;EACA,YAAY,QAAQ;GAClB,OAAO,QAAQ,UAAU,MAAM,KAAK,mBAAmB,GAAG,QAAQ,YAAY,GAAG,QAAQ;EAC3F;CACF;AACF;AAEA,SAAgB,0BACd,SACA,UACA,UAA8C,CAAC,GAC/C;CACA,MAAM,oBAAoB,8BACxB,UACA,QAAQ,qBAAqB,oBAC/B;CACA,MAAM,UAAU,iBAAiB,QAAQ,WAAW;EAAC;EAAG;EAAG;EAAG;CAAE,CAAC;CACjE,MAAM,aAAqC;EACzC,gBAAgB,QAAQ,oBAAoB;EAC5C,gBAAgB,kBAAkB;CACpC;CACA,KAAK,MAAM,UAAU,SACnB,OAAO,OACL,YACA,yBAAyB;EACvB;EACA,gBAAgB,QAAQ;EACxB;CACF,CAAC,CACH;CAEF,MAAM,gBAAgB,QAAQ,iBAAiB;CAC/C,MAAM,aAAa,QAAQ,cAAc;CACzC,MAAM,QAAQ,WAAW,kBAAkB,WAAW,QAAQ,QAAQ,QAAQ,SAAS,SAAS;CAGhG,OAAO;EACL;EACA,SAHA,WAAW,eAAe,WAAW,OAAO,QAAQ,QAAQ,SAAS,SAAS,WAGxD,QAAQ,iBAAiB;EAC/C;EACA,KAAK;GACH;GACA,qBAAqB,QAAQ;GAC7B,gBAAgB,QAAQ;EAC1B;CACF;AACF;AAEA,SAAS,8BACP,UACA,oBAA4B,sBAClB;CACV,IAAI,OAAO,aAAa,UACtB,OAAO,eAAe,SAAS,MAAM,iBAAiB,KAAK,CAAC,EAAA,CAAG,KAAK,UAAU,MAAM,KAAK,CAAC,CAAC;CAE7F,IAAI,MAAM,QAAQ,QAAQ,GAExB,OAAO,cACLC,SAAQ,SAAS,UAAU;EACzB,IAAI,OAAO,UAAU,UAAU,OAAO,CAAC,KAAK;EAC5C,OAAO,CAAC,MAAM,cAAc,MAAM,SAAS,MAAM,MAAM,EAAE;CAC3D,CAAC,CACH;CAEF,MAAM,iBAAiB;CACvB,OAAO,cACL;EACE,GAAI,eAAe,eAAe,CAAC;EACnC,GAAI,eAAe,OAAO,CAAC;EAC3B,IAAI,eAAe,WAAW,CAAC,EAAA,CAAG,KAC/B,UAAU,MAAM,cAAc,MAAM,SAAS,MAAM,MAAM,EAC5D;CACF,CAAC,CAAC,KAAK,UAAU,MAAM,KAAK,CAAC,CAC/B;AACF;AAEA,SAAS,yBAAyB,OAIP;CACzB,MAAM,MAAM,MAAM,kBAAkB,MAAM,GAAG,MAAM,MAAM;CACzD,MAAM,cAAc,OAAO,KAAK,MAAM,cAAc,CAAC,CAAC,QACnD,OAAO,MAAM,eAAe,MAAO,CACtC;CACA,MAAM,WAAW,IAAI,IAAI,WAAW;CACpC,MAAM,OAAO,IAAI,QAAQ,OAAO,SAAS,IAAI,EAAE,CAAC,CAAC,CAAC;CAClD,MAAM,oBAAoB,IAAI,WAAW,OAAO,SAAS,IAAI,EAAE,CAAC;CAChE,MAAM,SAAS,IAAI,MAAM;CACzB,OAAO;GACJ,MAAM,WAAW,OAAO,IAAI,IAAI;GAChC,SAAS,WAAW,SAAS,SAAS,IAAI,IAAI,OAAO,SAAS;GAC9D,YAAY,WAAW,OAAO,MAAM;GACpC,MAAM,WAAW,sBAAsB,KAAK,IAAI,KAAK,oBAAoB;GACzE,OAAO,WAAW,OAAO,MAAM,mBAAmB,MAAM,gBAAgB,MAAM,MAAM;CACvF;AACF;AAEA,SAAS,YAAY,KAA8B,KAAiC;CAClF,MAAM,QAAQ,IAAI;CAClB,OAAO,OAAO,UAAU,WAAW,QAAQ,KAAA;AAC7C;AAEA,SAAS,WACP,KACA,MACyB;CACzB,MAAM,QAAQ,IAAI,IAAI,IAAI;CAC1B,OAAO,OAAO,YAAY,OAAO,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,MAAM,IAAI,GAAG,CAAC,CAAC;AAClF;AAEA,SAAS,iBAAiB,SAAsC;CAC9D,OAAO,CACL,GAAG,IAAI,IAAI,QAAQ,KAAK,WAAW,KAAK,MAAM,MAAM,CAAC,CAAC,CAAC,QAAQ,WAAW,SAAS,CAAC,CAAC,CACvF,CAAC,CAAC,MAAM,GAAG,MAAM,IAAI,CAAC;AACxB;AAEA,SAAS,cAAc,QAAqC;CAC1D,MAAM,uBAAO,IAAI,IAAY;CAC7B,MAAM,MAAgB,CAAC;CACvB,KAAK,MAAM,SAAS,QAAQ;EAC1B,MAAM,UAAU,MAAM,KAAK;EAC3B,IAAI,CAAC,WAAW,KAAK,IAAI,OAAO,GAAG;EACnC,KAAK,IAAI,OAAO;EAChB,IAAI,KAAK,OAAO;CAClB;CACA,OAAO;AACT;AAEA,SAAS,OACP,mBACA,gBACA,QACQ;CACR,MAAM,MAAM,kBACT,MAAM,GAAG,MAAM,CAAC,CAChB,QACE,KAAK,YAAY,UAAU,MAAM,eAAe,eAAe,eAAe,GAAG,KAAK,GACvF,CACF;CACF,MAAM,QAAQ,OAAO,OAAO,cAAc,CAAC,CACxC,QAAQ,UAAU,QAAQ,CAAC,CAAC,CAC5B,MAAM,GAAG,MAAM,IAAI,CAAC,CAAC,CACrB,MAAM,GAAG,MAAM,CAAC,CAChB,QAAQ,KAAK,OAAO,UAAU,MAAM,eAAe,OAAO,KAAK,GAAG,CAAC;CACtE,OAAO,UAAU,IAAI,IAAI,MAAM;AACjC;AAEA,SAAS,eAAe,WAAmB,eAA+B;CACxE,IAAI,aAAa,GAAG,OAAO;CAC3B,QAAQ,KAAK,YAAY,KAAK,KAAK,KAAK,gBAAgB,CAAC;AAC3D"}
1	+ {"version":3,"file":"index.js","names":["clamp01","parseToml","entries"],"sources":["../../src/benchmarks/calibration.ts","../../src/benchmarks/types.ts","../../src/benchmarks/harbor.ts","../../src/benchmarks/routing/dataset.ts","../../src/benchmarks/routing/index.ts","../../src/benchmarks/runner.ts","../../src/benchmarks/standard-formats.ts"],"sourcesContent":["import type { BenchmarkAdapter, BenchmarkDatasetItem, BenchmarkEvaluation } from './types'\n\nexport interface BenchmarkMetricCalibrationOptions<TPayload = unknown, TArtifact = string> {\n adapter: BenchmarkAdapter<BenchmarkDatasetItem<TPayload>, TPayload, TArtifact>\n item: BenchmarkDatasetItem<TPayload>\n weakArtifact: TArtifact\n strongArtifact: TArtifact\n maxWeakScore?: number\n minStrongScore?: number\n minGap?: number\n}\n\nexport interface BenchmarkMetricCalibrationResult {\n passed: boolean\n weak: BenchmarkEvaluation\n strong: BenchmarkEvaluation\n weakScore: number\n strongScore: number\n gap: number\n reasons: string[]\n}\n\nexport async function calibrateBenchmarkMetric<TPayload = unknown, TArtifact = string>(\n options: BenchmarkMetricCalibrationOptions<TPayload, TArtifact>,\n): Promise<BenchmarkMetricCalibrationResult> {\n const weak = await options.adapter.evaluate(options.item, options.weakArtifact)\n const strong = await options.adapter.evaluate(options.item, options.strongArtifact)\n const weakScore = clamp01(weak.score)\n const strongScore = clamp01(strong.score)\n const maxWeakScore = options.maxWeakScore ?? 0.3\n const minStrongScore = options.minStrongScore ?? 0.7\n const minGap = options.minGap ?? 0.4\n const gap = strongScore - weakScore\n const reasons = [\n weakScore <= maxWeakScore\n ? undefined\n : `weak score ${weakScore.toFixed(3)} exceeds max ${maxWeakScore.toFixed(3)}`,\n strongScore >= minStrongScore\n ? undefined\n : `strong score ${strongScore.toFixed(3)} below min ${minStrongScore.toFixed(3)}`,\n gap >= minGap ? undefined : `gap ${gap.toFixed(3)} below min ${minGap.toFixed(3)}`,\n ].filter((reason): reason is string => Boolean(reason))\n\n return {\n passed: reasons.length === 0,\n weak,\n strong,\n weakScore,\n strongScore,\n gap,\n reasons,\n }\n}\n\nfunction clamp01(value: number): number {\n if (!Number.isFinite(value)) return 0\n if (value < 0) return 0\n if (value > 1) return 1\n return value\n}\n","/*\n Shared types for the reference benchmark wrappers under\n * `src/benchmarks/`. Each wrapper exports the three functions in\n * `BenchmarkAdapter` plus its own typed `DatasetItem` shape.\n /\n\nimport type { DispatchContext, Scenario } from '../campaign/types'\nimport type { RunSplitTag } from '../run-record'\n\nexport type BenchmarkTaskKind =\n \| 'retrieval'\n \| 'rag-answer'\n \| 'hallucination'\n \| 'kb-improvement'\n \| 'routing'\n \| 'custom'\n\nexport type BenchmarkFamily =\n \| 'beir'\n \| 'mteb-retrieval'\n \| 'msmarco'\n \| 'trec-dl'\n \| 'miracl'\n \| 'lotte'\n \| 'bright'\n \| 'crag'\n \| 'hotpotqa'\n \| 'kilt'\n \| 'ragtruth'\n \| 'faithbench'\n \| 'first-party'\n \| 'custom'\n\nexport interface BenchmarkDatasetItem<TPayload = unknown> {\n /* Stable dataset-local item id (used for split assignment + paper\n * references). Unique within a benchmark. /\n id: string\n /* Free-form payload. Each benchmark defines its own shape. /\n payload: TPayload\n /* Optional precomputed split. When absent, adapters use assignSplit(id). /\n split?: RunSplitTag\n /* Benchmark family this row came from, e.g. `beir` or `crag`. /\n family?: BenchmarkFamily \| string\n /* Benchmark-local task kind, e.g. retrieval vs answer quality. /\n taskKind?: BenchmarkTaskKind \| string\n /* Slice labels such as language, domain, freshness, multihop, long-tail. /\n tags?: string[]\n /* Dataset provenance, version, URL, or license notes. /\n source?: BenchmarkSource\n metadata?: Record<string, unknown>\n}\n\nexport interface BenchmarkEvaluation {\n /* [0, 1] score for the response on this item. Exact-match\n * benchmarks use 0/1; partial-credit benchmarks may return\n * fractional values. /\n score: number\n /* Optional pass/fail projection. Defaults to `score > 0` when absent. /\n passed?: boolean\n /* Numeric sub-metrics. These become report dimensions. /\n dimensions?: Record<string, number>\n /* Optional bag of raw scoring signals — e.g. parsed numeric\n * answer, regex match, judge sub-scores. /\n raw: Record<string, unknown>\n notes?: string\n}\n\nexport interface BenchmarkSource {\n name?: string\n url?: string\n version?: string\n /* The exact upstream revision, such as a git commit, when `version` names a release. /\n revision?: string\n license?: string\n citation?: string\n /* Where the upstream states how a result must be produced to count, such as leaderboard rules. /\n rules?: string\n /\n Whether the tasks or their solutions may be in a model's training data: `unknown` until\n * checked, `none-known`, `suspected`, or `confirmed` (then report results as historical).\n /\n contamination?: 'unknown' \| 'none-known' \| 'suspected' \| 'confirmed'\n}\n\n/* Common signature implemented by every adapter under `src/benchmarks/`. /\n// `TPayload` is the per-item payload type; `_TItem` is preserved for\n// downstream type-narrowing extensions (a richer `BenchmarkDatasetItem`\n// subclass that adds e.g. provenance metadata) but is intentionally\n// unused here. `noUnusedLocals` requires the leading underscore.\nexport interface BenchmarkAdapter<_TItem = unknown, TPayload = unknown, TArtifact = string> {\n /** Stable benchmark id such as `beir/nfcorpus` or `crag/smoke`. /\n id?: string\n family?: BenchmarkFamily \| string\n taskKind?: BenchmarkTaskKind \| string\n description?: string\n source?: BenchmarkSource\n defaultMetric?: string\n /* Load the dataset for the given split. May hit the network on\n * first call but should be cache-friendly. Adapters that don't\n * ship the dataset itself MUST throw a clearly-marked error\n * pointing the caller at the loader script. /\n loadDataset(split: RunSplitTag): Promise<BenchmarkDatasetItem<TPayload>[]>\n /* Score a single response. Pure with respect to the inputs. /\n evaluate(item: BenchmarkDatasetItem<TPayload>, artifact: TArtifact): Promise<BenchmarkEvaluation>\n /* Deterministic split assignment via item id hashing. The\n * fraction of items in each split is implementation-defined but\n * MUST be stable across processes and platforms. /\n assignSplit(itemId: string): RunSplitTag\n}\n\nexport interface BenchmarkScenario<TPayload = unknown> extends Scenario {\n kind: 'benchmark'\n benchmarkId: string\n family: BenchmarkFamily \| string\n taskKind: BenchmarkTaskKind \| string\n splitTag: RunSplitTag\n item: BenchmarkDatasetItem<TPayload>\n}\n\nexport type BenchmarkResponder<TPayload = unknown, TArtifact = string> = (input: {\n scenario: BenchmarkScenario<TPayload>\n item: BenchmarkDatasetItem<TPayload>\n context: DispatchContext\n}) => Promise<TArtifact> \| TArtifact\n\n// ── Deterministic split assignment ───────────────────────────────────\n\n/\n 32-bit FNV-1a hash. Stable, allocation-free, deterministic across\n * runtimes. We use it to assign items to splits rather than depending\n * on a polyfilled crypto.subtle path.\n /\nfunction fnv1a32(input: string): number {\n let h = 0x811c9dc5\n for (let i = 0; i < input.length; i++) {\n h ^= input.charCodeAt(i) & 0xff\n h = (h + ((h << 1) + (h << 4) + (h << 7) + (h << 8) + (h << 24))) >>> 0\n }\n return h >>> 0\n}\n\n/* Split-assignment seed shared across all benchmarks. Bumping this\n * value reshuffles every split — do NOT do that lightly. /\nexport const BENCHMARK_SPLIT_SEED = 'agent-eval-v1'\n\n/\n Assign an item id to one of `'search' \| 'dev' \| 'holdout'` using a\n * stable 32-bit hash of `${seed}::${id}`. `cuts` [a, b] with 0 < a < b < 1 put\n * positions below a in search, below b in dev, the rest in holdout. Default [0.6, 0.8]:\n \n search: 60% (optimization-readable)\n * dev: 20% (held-out for tuning, leak-on-purpose during dev)\n * holdout:20% (paper-grade held-out, gated reads)\n /\nexport function deterministicSplit(\n itemId: string,\n seed: string = BENCHMARK_SPLIT_SEED,\n cuts: readonly [number, number] = [0.6, 0.8],\n): RunSplitTag {\n const [a, b] = cuts\n if (!(a > 0 && a < b && b < 1))\n throw new RangeError(`split cuts need 0 < a < b < 1, got [${a}, ${b}]`)\n const pos = fnv1a32(`${seed}::${itemId}`) / 0x100000000\n if (pos < a) return 'search'\n if (pos < b) return 'dev'\n return 'holdout'\n}\n","/\n A Harbor dataset as a `BenchmarkAdapter`.\n \n Harbor (https://harborframework.com) lays out each task as a directory with `task.toml`,\n * `instruction.md`, the agent's `environment/`, and the `tests/` its verifier runs, usually in a\n * verifier image of their own. This importer reads a local checkout of such a dataset, at the\n * revision the caller pins in `source`, into `BenchmarkDatasetItem`s.\n \n Scoring stays Harbor's. `evaluate` hands the task and the agent's artifact to the caller's\n * `grade` callback, which runs the task's own verifier and reports its reward. `gradeHarborPatch`\n * is that callback for a code patch: it spawns the Harbor CLI, which starts the containers; this\n * package starts none itself. A verifier that did not run is an error, never a score of zero.\n /\n\nimport { execFile } from 'node:child_process'\nimport { randomUUID } from 'node:crypto'\nimport { cp, mkdir, mkdtemp, readdir, readFile, rm, stat, writeFile } from 'node:fs/promises'\nimport { tmpdir } from 'node:os'\nimport { join, resolve } from 'node:path'\nimport { promisify, stripVTControlCharacters } from 'node:util'\n\nimport { parse as parseToml } from 'smol-toml'\n\nimport type { RunSplitTag } from '../run-record'\nimport type {\n BenchmarkAdapter,\n BenchmarkDatasetItem,\n BenchmarkEvaluation,\n BenchmarkSource,\n} from './types'\nimport { deterministicSplit } from './types'\n\nconst execute = promisify(execFile)\n\n/* One Harbor task, as its `task.toml` and `instruction.md` state it. /\nexport interface HarborTaskPayload {\n /* `[task].name`, such as `terminal-bench/interleaved-vigenere`; the directory name when absent. /\n name: string\n /* The task's directory name inside the dataset's `tasks/`. /\n dir: string\n /* Absolute path of the task directory, for the grader. /\n taskDir: string\n /* `instruction.md`, verbatim. /\n instruction: string\n /* `[agent].timeout_sec`; null when the task sets none. /\n agentTimeoutSec: number \| null\n /* `[verifier].timeout_sec`; null when the task sets none. /\n verifierTimeoutSec: number \| null\n /* `[verifier].environment_mode`: `separate` runs the tests in their own image. /\n verifierMode: string \| null\n /* `[environment]` requests; null where the task states none. /\n resources: {\n cpus: number \| null\n memoryMb: number \| null\n storageMb: number \| null\n gpus: number\n }\n /* Paths the verifier reads from the agent's environment (`artifacts`, sources only). /\n artifacts: string[]\n /* The environment starts more than one container (a docker-compose file). /\n multiContainer: boolean\n /* `[metadata].category`. /\n category: string \| null\n /* `[metadata].expert_time_estimate_hours`. /\n expertTimeEstimateHours: number \| null\n}\n\nexport type HarborTaskItem = BenchmarkDatasetItem<HarborTaskPayload>\n\n/* What the verifier returned for one task, or why it did not return. /\nexport type HarborGradeOutcome =\n \| { succeeded: true; value: { reward: number; detail?: string } }\n \| { succeeded: false; error: string }\n\nexport interface HarborBenchmarkOptions<TArtifact> {\n /* Stable benchmark id, such as `terminal-bench@4.0.0`. /\n id: string\n /* A checkout of the dataset: the directory that holds `tasks/`. /\n datasetDir: string\n /* The upstream name, URL, version, revision, rules and contamination status the checkout is. /\n source: BenchmarkSource & { revision: string }\n /* Task directory names to load, in this order; every task under `tasks/` when absent. /\n tasks?: readonly string[]\n /* Run the task's own verifier on the agent's artifact. /\n grade: (item: HarborTaskItem, artifact: TArtifact) => Promise<HarborGradeOutcome>\n /* Split assignment; `deterministicSplit` over the benchmark id and task directory by default. /\n assignSplit?: (itemId: string) => RunSplitTag\n}\n\n/* Harbor adapter plus a loader that ignores splits, since a public benchmark reports every task. /\nexport interface HarborBenchmarkAdapter<TArtifact>\n extends BenchmarkAdapter<HarborTaskItem, HarborTaskPayload, TArtifact> {\n loadAll(): Promise<HarborTaskItem[]>\n}\n\nconst table = (value: unknown): Record<string, unknown> =>\n value !== null && typeof value === 'object' && !Array.isArray(value)\n ? (value as Record<string, unknown>)\n : {}\nconst numberOrNull = (value: unknown): number \| null =>\n typeof value === 'number' && Number.isFinite(value) ? value : null\nconst textOrNull = (value: unknown): string \| null =>\n typeof value === 'string' && value.length > 0 ? value : null\n\n/* Read one Harbor task directory. Throws when `task.toml` or `instruction.md` is missing or malformed. /\nasync function readHarborTask(taskDir: string): Promise<HarborTaskPayload> {\n const dir = resolve(taskDir)\n const config = parseToml(await readFile(join(dir, 'task.toml'), 'utf8')) as Record<\n string,\n unknown\n >\n const instruction = await readFile(join(dir, 'instruction.md'), 'utf8')\n const task = table(config.task)\n const agent = table(config.agent)\n const verifier = table(config.verifier)\n const environment = table(config.environment)\n const metadata = table(config.metadata)\n const artifacts = (Array.isArray(config.artifacts) ? config.artifacts : []).map(\n (entry, index) => {\n const source = typeof entry === 'string' ? entry : table(entry).source\n if (typeof source !== 'string' \|\| source.length === 0) {\n throw new Error(`${dir}/task.toml: artifacts[${index}] names no source path`)\n }\n return source\n },\n )\n const environmentFiles = await readdir(join(dir, 'environment')).catch(() => [] as string[])\n const name = dir.split('/').pop() ?? dir\n return {\n name: textOrNull(task.name) ?? name,\n dir: name,\n taskDir: dir,\n instruction,\n agentTimeoutSec: numberOrNull(agent.timeout_sec),\n verifierTimeoutSec: numberOrNull(verifier.timeout_sec),\n verifierMode: textOrNull(verifier.environment_mode),\n resources: {\n cpus: numberOrNull(environment.cpus),\n memoryMb: numberOrNull(environment.memory_mb),\n storageMb: numberOrNull(environment.storage_mb),\n gpus: numberOrNull(environment.gpus) ?? 0,\n },\n artifacts,\n multiContainer: environmentFiles.some((file) => /compose\\.ya?ml$/u.test(file)),\n category: textOrNull(metadata.category),\n expertTimeEstimateHours: numberOrNull(metadata.expert_time_estimate_hours),\n }\n}\n\n/* Import a Harbor dataset checkout as a `BenchmarkAdapter` whose scorer is the dataset's own verifier. /\nexport function createHarborBenchmarkAdapter<TArtifact = string>(\n options: HarborBenchmarkOptions<TArtifact>,\n): HarborBenchmarkAdapter<TArtifact> {\n const split =\n options.assignSplit ?? ((itemId: string) => deterministicSplit(`${options.id}::${itemId}`))\n const tasksDir = join(resolve(options.datasetDir), 'tasks')\n\n async function loadAll(): Promise<HarborTaskItem[]> {\n const names =\n options.tasks ??\n (\n await Promise.all(\n (\n await readdir(tasksDir)\n ).map(async (entry) =>\n (await stat(join(tasksDir, entry, 'task.toml')).catch(() => null)) === null\n ? null\n : entry,\n ),\n )\n )\n .filter((entry): entry is string => entry !== null)\n .sort()\n return Promise.all(\n names.map(async (dir) => {\n const payload = await readHarborTask(join(tasksDir, dir))\n return {\n id: dir,\n payload,\n split: split(dir),\n family: options.source.name ?? 'harbor',\n taskKind: 'harbor-task',\n tags: payload.category === null ? [] : [payload.category],\n source: options.source,\n }\n }),\n )\n }\n\n return {\n id: options.id,\n family: options.source.name ?? 'harbor',\n taskKind: 'harbor-task',\n description:\n `${options.source.name ?? 'Harbor dataset'} ${options.source.version ?? ''} at ${options.source.revision}, scored by its own verifier`.replace(\n /\\s+/gu,\n ' ',\n ),\n source: options.source,\n defaultMetric: 'reward',\n loadAll,\n async loadDataset(tag: RunSplitTag) {\n return (await loadAll()).filter((item) => item.split === tag)\n },\n async evaluate(item: HarborTaskItem, artifact: TArtifact): Promise<BenchmarkEvaluation> {\n const outcome = await options.grade(item, artifact)\n if (!outcome.succeeded)\n throw new Error(\n `${options.id} ${item.id}: the verifier did not score this artifact: ${outcome.error}`,\n )\n const { reward, detail } = outcome.value\n if (!(Number.isFinite(reward) && reward >= 0 && reward <= 1)) {\n throw new Error(\n `${options.id} ${item.id}: the verifier returned reward ${reward}, outside [0, 1]`,\n )\n }\n return {\n score: reward,\n passed: reward === 1,\n raw: { reward, ...(detail === undefined ? {} : { detail }) },\n ...(detail === undefined ? {} : { notes: detail }),\n }\n },\n assignSplit: split,\n }\n}\n\n/\n `retries` counts extra runs after one with no reward (default 2); `maxDetailChars` bounds\n * `JSON.stringify(detail).length` (default 2000); `timeoutMs` bounds one `harbor run`; `keepImage`\n * (default true) passes `--no-delete` so the next grade reuses the built task image.\n /\nexport interface HarborPatchGradeOptions {\n harborBin?: string\n jobsDir?: string\n retries?: number\n maxDetailChars?: number\n timeoutMs?: number\n env?: Record<string, string>\n keepImage?: boolean\n}\n\nconst SOLVE_SH =\n '#!/bin/bash\\nset -euo pipefail\\npatch=\"$(dirname \"$0\")/patch.diff\"\\n' +\n 'if [ -s \"$patch\" ]; then git apply --binary \"$patch\"; fi\\n'\n\n/* The last characters of `text` without ANSI codes, so that its JSON string fits in `max`. /\nfunction detailTail(text: string, max: number): string {\n let tail = stripVTControlCharacters(text).slice(-max)\n while (JSON.stringify(tail).length > max) tail = tail.slice(JSON.stringify(tail).length - max)\n return tail\n}\n\n/\n A `grade` callback that scores `{ patch }` with the task's own verifier: Harbor's oracle agent\n * runs a `solution/solve.sh` that applies the patch (`git apply --binary`) in the environment's\n * working directory, on a copy of the task. An empty patch changes nothing. A run with no reward\n * is retried, then reported as `succeeded: false`, never as a reward of 0.\n /\nexport function gradeHarborPatch(\n options: HarborPatchGradeOptions = {},\n): (item: HarborTaskItem, artifact: { patch: string }) => Promise<HarborGradeOutcome> {\n const { harborBin = 'harbor', retries = 2, maxDetailChars = 2000, keepImage = true } = options\n const { timeoutMs: timeout, env: extra } = options\n const run = { env: { ...process.env, ...extra }, timeout, maxBuffer: 1 << 26 }\n return async (item, artifact) => {\n const work = await mkdtemp(join(tmpdir(), 'harbor-grade-'))\n try {\n const task = join(work, item.payload.dir)\n await cp(item.payload.taskDir, task, { recursive: true })\n await rm(join(task, 'solution'), { recursive: true, force: true })\n await mkdir(join(task, 'solution'))\n await writeFile(join(task, 'solution/patch.diff'), artifact.patch)\n await writeFile(join(task, 'solution/solve.sh'), SOLVE_SH, { mode: 0o755 })\n const jobsDir = resolve(options.jobsDir ?? join(work, 'jobs'))\n const errors: string[] = []\n const keep = keepImage ? '--no-delete' : '--delete'\n const args = ['run', '-p', task, '-a', 'oracle', '-o', jobsDir, '-q', keep, '--job-name']\n for (let attempt = 1; attempt <= retries + 1; attempt += 1) {\n const job = `${item.payload.dir}-${randomUUID()}`\n const failure = await execute(harborBin, [...args, job], run).then(\n () => null,\n (error: Error) => detailTail(error.message, 500),\n )\n const trial = (await readdir(join(jobsDir, job), { withFileTypes: true }).catch(() => []))\n .filter((entry) => entry.isDirectory())\n .map((entry) => join(jobsDir, job, entry.name, '/'))[0]\n const read = (path: string) => readFile(`${trial}${path}`, 'utf8').catch(() => '')\n const result = table(\n trial === undefined ? {} : JSON.parse((await read('result.json')) \|\| '{}'),\n )\n const reward = numberOrNull(table(table(result.verifier_result).rewards).reward)\n if (reward !== null) {\n const detail = detailTail(await read('verifier/test-stdout.txt'), maxDetailChars)\n return { succeeded: true, value: { reward, detail } }\n }\n const exception = textOrNull(table(result.exception_info).exception_message)\n errors.push(`attempt ${attempt}: ${failure ?? exception ?? 'harbor recorded no reward'}`)\n }\n return { succeeded: false, error: errors.join('; ') }\n } finally {\n await rm(work, { recursive: true, force: true })\n }\n }\n}\n","/\n Synthetic routing dataset. 16 tasks across 4 categories. Used as a\n * deterministic, dependency-free benchmark for any router that maps a\n * natural-language request to one of a fixed set of route labels.\n \n Format (see `routing/README.md` for prose):\n \n {\n * id: stable per-task ID (matches across processes).\n * category: one of the four route labels.\n * prompt: the user-facing request the router must classify.\n * route: the ground-truth route the router should pick.\n * synonyms: other strings that count as a correct answer.\n * hardNegatives:close-but-wrong route labels — used to detect the\n * \"always picks the popular route\" failure mode.\n * }\n \n The four categories are intentionally cross-domain (file ops,\n * math, search, conversation) so a router that collapses to one\n * category is easy to spot.\n /\n\nexport interface RoutingItem {\n id: string\n category: 'file' \| 'math' \| 'search' \| 'chat'\n prompt: string\n /* Canonical correct route label. /\n route: string\n /* Alternate route labels that also count as correct. /\n synonyms: string[]\n /* Wrong-but-tempting route labels (for analysis, not grading). /\n hardNegatives: string[]\n}\n\nexport const ROUTING_DATASET: RoutingItem[] = [\n {\n id: 'file_001',\n category: 'file',\n prompt: 'Save the meeting notes to /tmp/notes-2025-04.md as markdown.',\n route: 'fs.write',\n synonyms: ['filesystem.write', 'write_file'],\n hardNegatives: ['fs.read', 'chat.reply'],\n },\n {\n id: 'file_002',\n category: 'file',\n prompt: 'Read the contents of /etc/hosts and summarize the entries.',\n route: 'fs.read',\n synonyms: ['filesystem.read', 'read_file'],\n hardNegatives: ['fs.write', 'search.web'],\n },\n {\n id: 'file_003',\n category: 'file',\n prompt: 'List every Python file under src/ recursively.',\n route: 'fs.list',\n synonyms: ['filesystem.list', 'list_files'],\n hardNegatives: ['fs.read', 'search.code'],\n },\n {\n id: 'file_004',\n category: 'file',\n prompt: 'Delete the cached build at .turbo/cache.',\n route: 'fs.delete',\n synonyms: ['filesystem.delete', 'remove_file'],\n hardNegatives: ['fs.write', 'fs.list'],\n },\n {\n id: 'math_001',\n category: 'math',\n prompt: 'What is the integral of 3x^2 + 2x from 0 to 5?',\n route: 'math.integral',\n synonyms: ['calculator.integral', 'math.solve'],\n hardNegatives: ['math.derivative', 'chat.reply'],\n },\n {\n id: 'math_002',\n category: 'math',\n prompt: 'Compute the derivative of sin(x) cos(x).',\n route: 'math.derivative',\n synonyms: ['calculator.derivative', 'math.solve'],\n hardNegatives: ['math.integral', 'math.algebra'],\n },\n {\n id: 'math_003',\n category: 'math',\n prompt: 'Solve 2x + 7 = 19 for x.',\n route: 'math.algebra',\n synonyms: ['calculator.algebra', 'math.solve'],\n hardNegatives: ['math.derivative', 'math.integral'],\n },\n {\n id: 'math_004',\n category: 'math',\n prompt: 'What is the prime factorization of 360?',\n route: 'math.numbertheory',\n synonyms: ['calculator.factor', 'math.solve'],\n hardNegatives: ['math.algebra', 'search.web'],\n },\n {\n id: 'search_001',\n category: 'search',\n prompt: 'Find recent papers on agent prompt optimization with held-out promotion gates.',\n route: 'search.web',\n synonyms: ['web.search', 'search.papers'],\n hardNegatives: ['search.code', 'chat.reply'],\n },\n {\n id: 'search_002',\n category: 'search',\n prompt: 'Search the codebase for every call site of `runProposeReview`.',\n route: 'search.code',\n synonyms: ['code.search', 'grep'],\n hardNegatives: ['search.web', 'fs.read'],\n },\n {\n id: 'search_003',\n category: 'search',\n prompt: 'What is the latest release of the Tangle network on GitHub?',\n route: 'search.web',\n synonyms: ['web.search', 'github.releases'],\n hardNegatives: ['search.code', 'chat.reply'],\n },\n {\n id: 'search_004',\n category: 'search',\n prompt: 'Find all TODO comments in the agent-eval src tree.',\n route: 'search.code',\n synonyms: ['code.search', 'grep'],\n hardNegatives: ['search.web', 'fs.list'],\n },\n {\n id: 'chat_001',\n category: 'chat',\n prompt: 'Hi there, how are you doing today?',\n route: 'chat.reply',\n synonyms: ['conversation.reply'],\n hardNegatives: ['search.web', 'fs.read'],\n },\n {\n id: 'chat_002',\n category: 'chat',\n prompt: 'Please explain the difference between an LLM and a foundation model.',\n route: 'chat.reply',\n synonyms: ['conversation.reply', 'qa.answer'],\n hardNegatives: ['search.web', 'math.algebra'],\n },\n {\n id: 'chat_003',\n category: 'chat',\n prompt: 'Tell me a short joke about distributed systems.',\n route: 'chat.reply',\n synonyms: ['conversation.reply'],\n hardNegatives: ['search.web', 'fs.read'],\n },\n {\n id: 'chat_004',\n category: 'chat',\n prompt: 'Acknowledge my last message with a thumbs up.',\n route: 'chat.reply',\n synonyms: ['conversation.reply', 'react'],\n hardNegatives: ['fs.write', 'search.web'],\n },\n]\n","/*\n Routing benchmark — synthetic, dependency-free, ships in the\n * package. 16 cross-category items in `dataset.ts`. See\n * `routing/README.md` for the format.\n \n `evaluate` does case-insensitive exact match against the canonical\n * route plus declared synonyms. The first valid route token in the\n * response wins; everything else is ignored. Wrong answers also\n * report whether they hit a hard negative — useful when triaging\n * \"always picks the popular route\" failure modes.\n /\n\nimport type { RunSplitTag } from '../../run-record'\nimport type { BenchmarkAdapter, BenchmarkDatasetItem, BenchmarkEvaluation } from '../types'\nimport { deterministicSplit } from '../types'\nimport { ROUTING_DATASET, type RoutingItem } from './dataset'\n\nexport type { RoutingItem }\nexport type RoutingPayload = RoutingItem\nexport type RoutingDatasetItem = BenchmarkDatasetItem<RoutingPayload>\n\nclass RoutingAdapter implements BenchmarkAdapter<RoutingDatasetItem, RoutingPayload> {\n readonly id = 'first-party/routing'\n readonly family = 'first-party'\n readonly taskKind = 'routing'\n readonly description = 'Synthetic fixed-route classification smoke benchmark'\n readonly defaultMetric = 'route_exact_match'\n\n async loadDataset(split: RunSplitTag): Promise<RoutingDatasetItem[]> {\n return ROUTING_DATASET.map((item) => ({ id: item.id, payload: item })).filter(\n (it) => assignSplitImpl(it.id) === split,\n )\n }\n\n async evaluate(item: RoutingDatasetItem, response: string): Promise<BenchmarkEvaluation> {\n const tokens = extractRouteTokens(response)\n const correct = new Set<string>(\n [item.payload.route, ...item.payload.synonyms].map((s) => s.toLowerCase()),\n )\n const hardNeg = new Set<string>(item.payload.hardNegatives.map((s) => s.toLowerCase()))\n const firstMatch = tokens.find((t) => correct.has(t.toLowerCase())) ?? null\n const firstHardNeg = tokens.find((t) => hardNeg.has(t.toLowerCase())) ?? null\n const score = firstMatch ? 1 : 0\n return {\n score,\n raw: {\n firstToken: tokens[0] ?? null,\n matchedRoute: firstMatch,\n hitHardNegative: Boolean(firstHardNeg),\n hardNegativeRoute: firstHardNeg,\n category: item.payload.category,\n },\n }\n }\n\n assignSplit(itemId: string): RunSplitTag {\n return assignSplitImpl(itemId)\n }\n}\n\nfunction assignSplitImpl(itemId: string): RunSplitTag {\n return deterministicSplit(`routing::${itemId}`)\n}\n\n/\n Pull route-shaped tokens out of a model response. Routes look like\n * `category.action` (`fs.write`, `chat.reply`). Bare alphanumerics\n * are not routes, but `category.action` patterns are robust to most\n * model wrappers (JSON output, prose explanations, code fences).\n /\nexport function extractRouteTokens(response: string): string[] {\n const matches = response.match(/[a-z][a-z0-9_]\\.[a-z][a-z0-9_]/gi)\n return matches ?? []\n}\n\nconst adapter = new RoutingAdapter()\n\nexport const loadDataset = adapter.loadDataset.bind(adapter)\nexport const evaluate = adapter.evaluate.bind(adapter)\nexport const assignSplit = adapter.assignSplit.bind(adapter)\nexport { ROUTING_DATASET, RoutingAdapter }\n","import { join } from 'node:path'\nimport {\n type CampaignResult,\n type DispatchFn,\n type JudgeConfig,\n type JudgeScore,\n runCampaign,\n type Scenario,\n} from '../campaign'\nimport { type CampaignStorage, fsCampaignStorage } from '../campaign/storage'\nimport type { RunSplitTag } from '../run-record'\nimport { summarizeNumberSeries } from '../statistics'\nimport { mean } from '../statistics/mean'\nimport type {\n BenchmarkAdapter,\n BenchmarkDatasetItem,\n BenchmarkEvaluation,\n BenchmarkFamily,\n BenchmarkResponder,\n BenchmarkScenario,\n BenchmarkTaskKind,\n} from './types'\n\nexport interface BenchmarkRunOptions<TPayload = unknown, TArtifact = string> {\n adapter: BenchmarkAdapter<BenchmarkDatasetItem<TPayload>, TPayload, TArtifact>\n respond: BenchmarkResponder<TPayload, TArtifact>\n splits?: readonly RunSplitTag[]\n runDir: string\n repo?: string\n seed?: number\n reps?: number\n resumable?: boolean\n costCeiling?: number\n maxConcurrency?: number\n dispatchTimeoutMs?: number\n expectUsage?: 'assert' \| 'warn' \| 'off'\n storage?: CampaignStorage\n now?: () => Date\n}\n\nexport interface BenchmarkReport {\n benchmarkId: string\n family: BenchmarkFamily \| string\n taskKind: BenchmarkTaskKind \| string\n source?: BenchmarkAdapter['source']\n runDir: string\n manifestHash: string\n seed: number\n startedAt: string\n endedAt: string\n durationMs: number\n totalItems: number\n totalCells: number\n cellsFailed: number\n cellsCached: number\n totalCostUsd: number\n splits: Record<string, BenchmarkSliceSummary>\n tags: Record<string, BenchmarkSliceSummary>\n dimensions: Record<string, BenchmarkDistribution>\n score: BenchmarkDistribution\n costUsd: BenchmarkDistribution\n latencyMs: BenchmarkDistribution\n}\n\nexport interface BenchmarkSliceSummary {\n n: number\n meanScore: number\n passRate: number\n score: BenchmarkDistribution\n costUsd: BenchmarkDistribution\n latencyMs: BenchmarkDistribution\n}\n\nexport interface BenchmarkDistribution {\n n: number\n min: number\n mean: number\n median: number\n p90: number\n max: number\n}\n\nexport interface BenchmarkRunResult<TPayload = unknown, TArtifact = string> {\n scenarios: Array<BenchmarkScenario<TPayload>>\n campaign: CampaignResult<TArtifact, BenchmarkScenario<TPayload>>\n report: BenchmarkReport\n reportJsonPath: string\n reportMarkdownPath: string\n}\n\nexport async function runBenchmarkAdapter<TPayload = unknown, TArtifact = string>(\n options: BenchmarkRunOptions<TPayload, TArtifact>,\n): Promise<BenchmarkRunResult<TPayload, TArtifact>> {\n const storage = options.storage ?? fsCampaignStorage()\n const benchmarkId = benchmarkIdFor(options.adapter)\n const scenarios = await benchmarkScenarios(options.adapter, options.splits)\n const judge = benchmarkJudge(options.adapter)\n const dispatch: DispatchFn<BenchmarkScenario<TPayload>, TArtifact> = async (\n scenario,\n context,\n ) => {\n return options.respond({ scenario, item: scenario.item, context })\n }\n\n const campaign = await runCampaign<BenchmarkScenario<TPayload>, TArtifact>({\n scenarios,\n dispatch,\n dispatchRef: `benchmark:${benchmarkId}`,\n judges: [judge],\n seed: options.seed,\n reps: options.reps,\n resumable: options.resumable,\n costCeiling: options.costCeiling,\n maxConcurrency: options.maxConcurrency,\n dispatchTimeoutMs: options.dispatchTimeoutMs,\n expectUsage: options.expectUsage ?? 'off',\n runDir: options.runDir,\n repo: options.repo,\n storage,\n now: options.now,\n })\n const report = summarizeBenchmarkCampaign({\n adapter: options.adapter,\n scenarios,\n campaign,\n })\n storage.ensureDir(campaign.runDir)\n const reportJsonPath = join(campaign.runDir, 'benchmark-report.json')\n const reportMarkdownPath = join(campaign.runDir, 'benchmark-report.md')\n storage.write(reportJsonPath, `${JSON.stringify(report, null, 2)}\\n`)\n storage.write(reportMarkdownPath, renderBenchmarkReportMarkdown(report))\n return { scenarios, campaign, report, reportJsonPath, reportMarkdownPath }\n}\n\nexport async function benchmarkScenarios<TPayload>(\n adapter: BenchmarkAdapter<BenchmarkDatasetItem<TPayload>, TPayload, unknown>,\n splits: readonly RunSplitTag[] = ['search', 'dev', 'holdout'],\n): Promise<Array<BenchmarkScenario<TPayload>>> {\n const benchmarkId = benchmarkIdFor(adapter)\n const family = adapter.family ?? 'custom'\n const taskKind = adapter.taskKind ?? 'custom'\n const out: Array<BenchmarkScenario<TPayload>> = []\n const seen = new Set<string>()\n for (const split of splits) {\n const items = await adapter.loadDataset(split)\n for (const item of items) {\n const splitTag = item.split ?? adapter.assignSplit(item.id)\n if (splitTag !== split) continue\n const key = `${benchmarkId}:${item.id}`\n if (seen.has(key)) continue\n seen.add(key)\n out.push({\n id: key,\n kind: 'benchmark',\n benchmarkId,\n family: item.family ?? family,\n taskKind: item.taskKind ?? taskKind,\n splitTag,\n tags: [...new Set([splitTag, ...(item.tags ?? [])])],\n item,\n })\n }\n }\n return out\n}\n\nexport function benchmarkJudge<TPayload, TArtifact>(\n adapter: BenchmarkAdapter<BenchmarkDatasetItem<TPayload>, TPayload, TArtifact>,\n): JudgeConfig<TArtifact, BenchmarkScenario<TPayload>> {\n const benchmarkId = benchmarkIdFor(adapter)\n return {\n name: `${benchmarkId}:score`,\n dimensions: [\n { key: 'score', description: `Primary ${benchmarkId} benchmark score` },\n { key: 'passed', description: 'Binary pass projection for aggregate reporting' },\n ],\n appliesTo: (scenario: Scenario): scenario is BenchmarkScenario<TPayload> => {\n return scenario.kind === 'benchmark' && scenario.id.startsWith(`${benchmarkId}:`)\n },\n async score({ artifact, scenario }) {\n const evaluation = await adapter.evaluate(scenario.item, artifact)\n const dimensions = normalizeEvaluationDimensions(evaluation)\n return {\n dimensions,\n composite: clamp01(evaluation.score),\n notes: evaluation.notes ?? '',\n }\n },\n }\n}\n\nfunction summarizeBenchmarkCampaign<TPayload, TArtifact>(input: {\n adapter: BenchmarkAdapter<BenchmarkDatasetItem<TPayload>, TPayload, TArtifact>\n scenarios: Array<BenchmarkScenario<TPayload>>\n campaign: CampaignResult<TArtifact, BenchmarkScenario<TPayload>>\n}): BenchmarkReport {\n const scenarioById = new Map(input.scenarios.map((scenario) => [scenario.id, scenario]))\n const rows = input.campaign.cells.map((cell) => {\n const scenario = scenarioById.get(cell.scenarioId)\n const judge = firstJudgeScore(cell.judgeScores)\n const score = judge?.composite ?? 0\n return {\n cell,\n scenario,\n score,\n passed: (judge?.dimensions.passed ?? (score > 0 ? 1 : 0)) >= 1,\n dimensions: judge?.dimensions ?? {},\n }\n })\n const successful = rows.filter((row) => !row.cell.error)\n const scoreValues = successful.map((row) => row.score)\n return {\n benchmarkId: benchmarkIdFor(input.adapter),\n family: input.adapter.family ?? 'custom',\n taskKind: input.adapter.taskKind ?? 'custom',\n ...(input.adapter.source ? { source: input.adapter.source } : {}),\n runDir: input.campaign.runDir,\n manifestHash: input.campaign.manifestHash,\n seed: input.campaign.seed,\n startedAt: input.campaign.startedAt,\n endedAt: input.campaign.endedAt,\n durationMs: input.campaign.durationMs,\n totalItems: input.scenarios.length,\n totalCells: input.campaign.cells.length,\n cellsFailed: input.campaign.aggregates.cellsFailed,\n cellsCached: input.campaign.aggregates.cellsCached,\n totalCostUsd: input.campaign.aggregates.cost.totalCostUsd,\n splits: summarizeSlices(successful, (row) => row.scenario?.splitTag ?? 'unknown', [\n 'search',\n 'dev',\n 'holdout',\n ]),\n tags: summarizeSlices(successful, (row) => row.scenario?.tags ?? []),\n dimensions: summarizeDimensions(successful.map((row) => row.dimensions)),\n score: distribution(scoreValues),\n costUsd: distribution(successful.map((row) => row.cell.costUsd)),\n latencyMs: distribution(successful.map((row) => row.cell.durationMs)),\n }\n}\n\nfunction renderBenchmarkReportMarkdown(report: BenchmarkReport): string {\n const splitRows = Object.entries(report.splits)\n .map(([split, summary]) => {\n return `\| ${split} \| ${summary.n} \| ${fmt(summary.meanScore)} \| ${fmt(summary.passRate)} \| ${fmt(summary.score.p90)} \| ${fmt(summary.costUsd.mean)} \| ${fmt(summary.latencyMs.p90)} \|`\n })\n .join('\\n')\n const dimRows = Object.entries(report.dimensions)\n .sort(([a], [b]) => a.localeCompare(b))\n .map(([key, dist]) => `\| ${key} \| ${dist.n} \| ${fmt(dist.mean)} \| ${fmt(dist.p90)} \|`)\n .join('\\n')\n return [\n `# Benchmark Report: ${report.benchmarkId}`,\n '',\n `- family: ${report.family}`,\n `- task kind: ${report.taskKind}`,\n `- run dir: ${report.runDir}`,\n `- manifest: ${report.manifestHash}`,\n `- cells: ${report.totalCells} total, ${report.cellsFailed} failed, ${report.cellsCached} cached`,\n `- cost: $${fmt(report.totalCostUsd)}`,\n `- score: mean ${fmt(report.score.mean)}, median ${fmt(report.score.median)}, p90 ${fmt(report.score.p90)}, n=${report.score.n}`,\n '',\n '## Splits',\n '',\n '\| split \| n \| mean score \| pass rate \| score p90 \| mean cost \| latency p90 ms \|',\n '\| --- \| ---: \| ---: \| ---: \| ---: \| ---: \| ---: \|',\n splitRows \|\| '\| none \| 0 \| 0 \| 0 \| 0 \| 0 \| 0 \|',\n '',\n '## Dimensions',\n '',\n '\| dimension \| n \| mean \| p90 \|',\n '\| --- \| ---: \| ---: \| ---: \|',\n dimRows \|\| '\| none \| 0 \| 0 \| 0 \|',\n '',\n ].join('\\n')\n}\n\nfunction benchmarkIdFor(adapter: Pick<BenchmarkAdapter, 'id' \| 'family' \| 'taskKind'>): string {\n return adapter.id ?? `${adapter.family ?? 'custom'}/${adapter.taskKind ?? 'custom'}`\n}\n\nfunction normalizeEvaluationDimensions(evaluation: BenchmarkEvaluation): Record<string, number> {\n const dimensions: Record<string, number> = { score: clamp01(evaluation.score) }\n for (const [key, value] of Object.entries(evaluation.dimensions ?? {})) {\n if (Number.isFinite(value)) dimensions[key] = value\n }\n dimensions.passed = (evaluation.passed ?? evaluation.score > 0) ? 1 : 0\n return dimensions\n}\n\nfunction summarizeDimensions(\n rows: Array<Record<string, number>>,\n): Record<string, BenchmarkDistribution> {\n const values = new Map<string, number[]>()\n for (const row of rows) {\n for (const [key, value] of Object.entries(row)) {\n if (!Number.isFinite(value)) continue\n const list = values.get(key) ?? []\n list.push(value)\n values.set(key, list)\n }\n }\n return Object.fromEntries([...values.entries()].map(([key, vals]) => [key, distribution(vals)]))\n}\n\nfunction summarizeSlices<T>(\n rows: T[],\n keyOf: (row: T) => string \| string[],\n knownKeys: readonly string[] = [],\n): Record<string, BenchmarkSliceSummary> {\n const grouped = new Map<string, T[]>()\n for (const key of knownKeys) grouped.set(key, [])\n for (const row of rows) {\n const keys = keyOf(row)\n for (const key of Array.isArray(keys) ? keys : [keys]) {\n const list = grouped.get(key) ?? []\n list.push(row)\n grouped.set(key, list)\n }\n }\n const out: Record<string, BenchmarkSliceSummary> = {}\n for (const [key, list] of grouped) {\n const withShape = list as Array<{\n score: number\n passed: boolean\n cell: { costUsd: number; durationMs: number }\n }>\n const scores = withShape.map((row) => row.score).filter(Number.isFinite)\n const passed = withShape.filter((row) => row.passed).length\n out[key] = {\n n: list.length,\n meanScore: scores.length ? mean(scores) : 0,\n passRate: list.length ? passed / list.length : 0,\n score: distribution(scores),\n costUsd: distribution(withShape.map((row) => row.cell.costUsd)),\n latencyMs: distribution(withShape.map((row) => row.cell.durationMs)),\n }\n }\n return out\n}\n\nfunction firstJudgeScore(\n judgeScores: CampaignResult<unknown>['cells'][number]['judgeScores'],\n): JudgeScore \| undefined {\n return Object.values(judgeScores)[0]\n}\n\nfunction distribution(values: readonly number[]): BenchmarkDistribution {\n const summary = summarizeNumberSeries(values.filter(Number.isFinite))\n if (!summary) return { n: 0, min: 0, mean: 0, median: 0, p90: 0, max: 0 }\n const { n, min, p50: median, p90, max, sum } = summary\n return { n, min, mean: sum / n, median, p90, max }\n}\n\nfunction clamp01(value: number): number {\n if (!Number.isFinite(value)) return 0\n if (value < 0) return 0\n if (value > 1) return 1\n return value\n}\n\nfunction fmt(value: number): string {\n if (!Number.isFinite(value)) return '0'\n return value.toFixed(value === 0 \|\| Math.abs(value) >= 10 ? 0 : 3)\n}\n","import type { RunSplitTag } from '../run-record'\nimport type {\n BenchmarkAdapter,\n BenchmarkDatasetItem,\n BenchmarkFamily,\n BenchmarkSource,\n BenchmarkTaskKind,\n} from './types'\nimport { deterministicSplit } from './types'\n\nexport interface StandardRetrievalDocument {\n id: string\n title?: string\n text: string\n metadata?: Record<string, unknown>\n}\n\nexport interface StandardRetrievalQuery {\n id: string\n text: string\n metadata?: Record<string, unknown>\n}\n\nexport interface StandardRetrievalQrel {\n queryId: string\n documentId: string\n score: number\n}\n\nexport interface StandardRetrievalPayload {\n queryId: string\n query: string\n expectedDocumentIds: string[]\n expectedScores: Record<string, number>\n corpus?: Record<string, StandardRetrievalDocument>\n metadata?: Record<string, unknown>\n}\n\nexport interface BuildStandardRetrievalItemsOptions {\n benchmarkId: string\n family: BenchmarkFamily \| string\n queries: readonly StandardRetrievalQuery[]\n qrels: readonly StandardRetrievalQrel[]\n corpus?: readonly StandardRetrievalDocument[]\n includeCorpusInPayload?: boolean\n source?: BenchmarkSource\n tags?: readonly string[]\n splitOf?: (queryId: string) => RunSplitTag\n}\n\nexport interface RetrievalIdAdapterOptions extends BuildStandardRetrievalItemsOptions {\n responseIdPattern?: RegExp\n cutoffs?: readonly number[]\n primaryMetric?: string\n passMetric?: string\n passThreshold?: number\n}\n\nexport interface StandardRetrievalResult {\n id?: string\n documentId?: string\n docId?: string\n score?: number\n}\n\nexport type StandardRetrievalArtifact =\n \| string\n \| readonly string[]\n \| readonly StandardRetrievalResult[]\n \| {\n ids?: readonly string[]\n documentIds?: readonly string[]\n results?: readonly StandardRetrievalResult[]\n }\n\nexport interface StandardRetrievalEvaluationOptions {\n responseIdPattern?: RegExp\n cutoffs?: readonly number[]\n primaryMetric?: string\n passMetric?: string\n passThreshold?: number\n}\n\nfunction parseJsonlRows<T = unknown>(text: string): T[] {\n return text\n .split(/\\r?\\n/)\n .map((line) => line.trim())\n .filter(Boolean)\n .map((line, index) => {\n try {\n return JSON.parse(line) as T\n } catch (error) {\n throw new Error(`invalid JSONL row ${index + 1}: ${(error as Error).message}`)\n }\n })\n}\n\nfunction parseTsvRows(text: string): string[][] {\n return text\n .split(/\\r?\\n/)\n .map((line) => line.trim())\n .filter(Boolean)\n .filter((line) => !line.startsWith('#'))\n .map((line) => line.split(/\\t\|\\s+/))\n}\n\nexport function parseQrels(text: string): StandardRetrievalQrel[] {\n return parseTsvRows(text).flatMap((parts, index) => {\n if (parts.length < 3) return []\n const [queryId, maybeZeroOrDocId, maybeDocIdOrScore, maybeScore] = parts\n if (!queryId \|\| !maybeZeroOrDocId \|\| !maybeDocIdOrScore) return []\n if (queryId.toLowerCase() === 'query-id' \|\| queryId.toLowerCase() === 'qid') return []\n const documentId = maybeScore === undefined ? maybeZeroOrDocId : maybeDocIdOrScore\n const scoreText = maybeScore === undefined ? maybeDocIdOrScore : maybeScore\n const score = Number(scoreText)\n if (!documentId \|\| !Number.isFinite(score)) {\n throw new Error(`invalid qrels row ${index + 1}: expected query id, doc id, score`)\n }\n return [{ queryId, documentId, score }]\n })\n}\n\nexport function parseBeirCorpusJsonl(text: string): StandardRetrievalDocument[] {\n return parseJsonlRows<Record<string, unknown>>(text).map((row, index) => {\n const id = stringField(row, '_id') ?? stringField(row, 'id')\n const body = stringField(row, 'text') ?? stringField(row, 'contents')\n if (!id \|\| body === undefined) {\n throw new Error(`invalid BEIR corpus row ${index + 1}: expected _id/id and text/contents`)\n }\n return {\n id,\n title: stringField(row, 'title'),\n text: body,\n metadata: stripKnown(row, ['_id', 'id', 'title', 'text', 'contents']),\n }\n })\n}\n\nexport function parseBeirQueriesJsonl(text: string): StandardRetrievalQuery[] {\n return parseJsonlRows<Record<string, unknown>>(text).map((row, index) => {\n const id = stringField(row, '_id') ?? stringField(row, 'id') ?? stringField(row, 'query_id')\n const query = stringField(row, 'text') ?? stringField(row, 'query')\n if (!id \|\| !query) {\n throw new Error(`invalid BEIR query row ${index + 1}: expected _id/id and text/query`)\n }\n return {\n id,\n text: query,\n metadata: stripKnown(row, ['_id', 'id', 'query_id', 'text', 'query']),\n }\n })\n}\n\nexport function buildStandardRetrievalItems(\n options: BuildStandardRetrievalItemsOptions,\n): Array<BenchmarkDatasetItem<StandardRetrievalPayload>> {\n const qrelsByQuery = new Map<string, StandardRetrievalQrel[]>()\n for (const qrel of options.qrels) {\n if (qrel.score <= 0) continue\n const list = qrelsByQuery.get(qrel.queryId) ?? []\n list.push(qrel)\n qrelsByQuery.set(qrel.queryId, list)\n }\n const corpus =\n options.includeCorpusInPayload && options.corpus\n ? Object.fromEntries(options.corpus.map((document) => [document.id, document]))\n : undefined\n return options.queries.flatMap((query) => {\n const qrels = qrelsByQuery.get(query.id) ?? []\n if (qrels.length === 0) return []\n const split =\n options.splitOf?.(query.id) ?? deterministicSplit(`${options.benchmarkId}:${query.id}`)\n return [\n {\n id: query.id,\n split,\n family: options.family,\n taskKind: 'retrieval',\n tags: [...new Set([...(options.tags ?? []), split])],\n ...(options.source ? { source: options.source } : {}),\n ...(query.metadata ? { metadata: query.metadata } : {}),\n payload: {\n queryId: query.id,\n query: query.text,\n expectedDocumentIds: qrels.map((qrel) => qrel.documentId),\n expectedScores: Object.fromEntries(qrels.map((qrel) => [qrel.documentId, qrel.score])),\n ...(corpus ? { corpus } : {}),\n ...(query.metadata ? { metadata: query.metadata } : {}),\n },\n },\n ]\n })\n}\n\nexport function createRetrievalIdBenchmarkAdapter(\n options: RetrievalIdAdapterOptions,\n): BenchmarkAdapter<\n BenchmarkDatasetItem<StandardRetrievalPayload>,\n StandardRetrievalPayload,\n StandardRetrievalArtifact\n> {\n const items = buildStandardRetrievalItems(options)\n return {\n id: options.benchmarkId,\n family: options.family,\n taskKind: 'retrieval' satisfies BenchmarkTaskKind,\n source: options.source,\n defaultMetric: options.primaryMetric ?? 'ndcg@10',\n async loadDataset(split) {\n return items.filter((item) => item.split === split)\n },\n async evaluate(item, artifact) {\n return evaluateStandardRetrieval(item.payload, artifact, options)\n },\n assignSplit(itemId) {\n return options.splitOf?.(itemId) ?? deterministicSplit(`${options.benchmarkId}:${itemId}`)\n },\n }\n}\n\nexport function evaluateStandardRetrieval(\n payload: StandardRetrievalPayload,\n artifact: StandardRetrievalArtifact,\n options: StandardRetrievalEvaluationOptions = {},\n) {\n const rankedDocumentIds = normalizeRetrievedDocumentIds(\n artifact,\n options.responseIdPattern ?? /[A-Za-z0-9_.:/-]+/g,\n )\n const cutoffs = normalizeCutoffs(options.cutoffs ?? [1, 3, 5, 10])\n const dimensions: Record<string, number> = {\n expected_count: payload.expectedDocumentIds.length,\n returned_count: rankedDocumentIds.length,\n }\n for (const cutoff of cutoffs) {\n Object.assign(\n dimensions,\n retrievalMetricsAtCutoff({\n rankedDocumentIds,\n expectedScores: payload.expectedScores,\n cutoff,\n }),\n )\n }\n const primaryMetric = options.primaryMetric ?? 'ndcg@10'\n const passMetric = options.passMetric ?? 'hit@10'\n const score = dimensions[primaryMetric] ?? dimensions[`ndcg@${cutoffs[cutoffs.length - 1]}`] ?? 0\n const passScore =\n dimensions[passMetric] ?? dimensions[`hit@${cutoffs[cutoffs.length - 1]}`] ?? score\n return {\n score,\n passed: passScore >= (options.passThreshold ?? 1),\n dimensions,\n raw: {\n rankedDocumentIds,\n expectedDocumentIds: payload.expectedDocumentIds,\n expectedScores: payload.expectedScores,\n },\n }\n}\n\nfunction normalizeRetrievedDocumentIds(\n artifact: StandardRetrievalArtifact,\n responseIdPattern: RegExp = /[A-Za-z0-9_.:/-]+/g,\n): string[] {\n if (typeof artifact === 'string') {\n return uniqueOrdered((artifact.match(responseIdPattern) ?? []).map((value) => value.trim()))\n }\n if (Array.isArray(artifact)) {\n const entries = artifact as readonly (string \| StandardRetrievalResult)[]\n return uniqueOrdered(\n entries.flatMap((entry) => {\n if (typeof entry === 'string') return [entry]\n return [entry.documentId ?? entry.docId ?? entry.id ?? '']\n }),\n )\n }\n const objectArtifact = artifact as Exclude<StandardRetrievalArtifact, string \| readonly unknown[]>\n return uniqueOrdered(\n [\n ...(objectArtifact.documentIds ?? []),\n ...(objectArtifact.ids ?? []),\n ...(objectArtifact.results ?? []).map(\n (entry) => entry.documentId ?? entry.docId ?? entry.id ?? '',\n ),\n ].map((value) => value.trim()),\n )\n}\n\nfunction retrievalMetricsAtCutoff(input: {\n rankedDocumentIds: readonly string[]\n expectedScores: Record<string, number>\n cutoff: number\n}): Record<string, number> {\n const top = input.rankedDocumentIds.slice(0, input.cutoff)\n const relevantIds = Object.keys(input.expectedScores).filter(\n (id) => input.expectedScores[id]! > 0,\n )\n const relevant = new Set(relevantIds)\n const hits = top.filter((id) => relevant.has(id)).length\n const firstRelevantRank = top.findIndex((id) => relevant.has(id))\n const prefix = `@${input.cutoff}`\n return {\n [`hit${prefix}`]: hits > 0 ? 1 : 0,\n [`recall${prefix}`]: relevant.size === 0 ? 1 : hits / relevant.size,\n [`precision${prefix}`]: hits / input.cutoff,\n [`mrr${prefix}`]: firstRelevantRank === -1 ? 0 : 1 / (firstRelevantRank + 1),\n [`ndcg${prefix}`]: ndcgAt(input.rankedDocumentIds, input.expectedScores, input.cutoff),\n }\n}\n\nfunction stringField(row: Record<string, unknown>, key: string): string \| undefined {\n const value = row[key]\n return typeof value === 'string' ? value : undefined\n}\n\nfunction stripKnown(\n row: Record<string, unknown>,\n keys: readonly string[],\n): Record<string, unknown> {\n const known = new Set(keys)\n return Object.fromEntries(Object.entries(row).filter(([key]) => !known.has(key)))\n}\n\nfunction normalizeCutoffs(cutoffs: readonly number[]): number[] {\n return [\n ...new Set(cutoffs.map((cutoff) => Math.trunc(cutoff)).filter((cutoff) => cutoff > 0)),\n ].sort((a, b) => a - b)\n}\n\nfunction uniqueOrdered(values: readonly string[]): string[] {\n const seen = new Set<string>()\n const out: string[] = []\n for (const value of values) {\n const trimmed = value.trim()\n if (!trimmed \|\| seen.has(trimmed)) continue\n seen.add(trimmed)\n out.push(trimmed)\n }\n return out\n}\n\nfunction ndcgAt(\n rankedDocumentIds: readonly string[],\n expectedScores: Record<string, number>,\n cutoff: number,\n): number {\n const dcg = rankedDocumentIds\n .slice(0, cutoff)\n .reduce(\n (sum, documentId, index) => sum + discountedGain(expectedScores[documentId] ?? 0, index),\n 0,\n )\n const ideal = Object.values(expectedScores)\n .filter((score) => score > 0)\n .sort((a, b) => b - a)\n .slice(0, cutoff)\n .reduce((sum, score, index) => sum + discountedGain(score, index), 0)\n return ideal === 0 ? 1 : dcg / ideal\n}\n\nfunction discountedGain(relevance: number, zeroBasedRank: number): number {\n if (relevance <= 0) return 0\n return (2 * relevance - 1) / Math.log2(zeroBasedRank + 2)\n}\n"],"mappings":";;;;;;;;;;;;;AAsBA,eAAsB,yBACpB,SAC2C;CAC3C,MAAM,OAAO,MAAM,QAAQ,QAAQ,SAAS,QAAQ,MAAM,QAAQ,YAAY;CAC9E,MAAM,SAAS,MAAM,QAAQ,QAAQ,SAAS,QAAQ,MAAM,QAAQ,cAAc;CAClF,MAAM,YAAYA,UAAQ,KAAK,KAAK;CACpC,MAAM,cAAcA,UAAQ,OAAO,KAAK;CACxC,MAAM,eAAe,QAAQ,gBAAgB;CAC7C,MAAM,iBAAiB,QAAQ,kBAAkB;CACjD,MAAM,SAAS,QAAQ,UAAU;CACjC,MAAM,MAAM,cAAc;CAC1B,MAAM,UAAU;EACd,aAAa,eACT,KAAA,IACA,cAAc,UAAU,QAAQ,CAAC,EAAE,eAAe,aAAa,QAAQ,CAAC;EAC5E,eAAe,iBACX,KAAA,IACA,gBAAgB,YAAY,QAAQ,CAAC,EAAE,aAAa,eAAe,QAAQ,CAAC;EAChF,OAAO,SAAS,KAAA,IAAY,OAAO,IAAI,QAAQ,CAAC,EAAE,aAAa,OAAO,QAAQ,CAAC;CACjF,CAAC,CAAC,QAAQ,WAA6B,QAAQ,MAAM,CAAC;CAEtD,OAAO;EACL,QAAQ,QAAQ,WAAW;EAC3B;EACA;EACA;EACA;EACA;EACA;CACF;AACF;AAEA,SAASA,UAAQ,OAAuB;CACtC,IAAI,CAAC,OAAO,SAAS,KAAK,GAAG,OAAO;CACpC,IAAI,QAAQ,GAAG,OAAO;CACtB,IAAI,QAAQ,GAAG,OAAO;CACtB,OAAO;AACT;;;;;;;ACyEA,SAAS,QAAQ,OAAuB;CACtC,IAAI,IAAI;CACR,KAAK,IAAI,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK;EACrC,KAAK,MAAM,WAAW,CAAC,IAAI;EAC3B,IAAK,MAAM,KAAK,MAAM,KAAK,MAAM,KAAK,MAAM,KAAK,MAAM,KAAK,SAAU;CACxE;CACA,OAAO,MAAM;AACf;;;AAIA,MAAa,uBAAuB;;;;;;;;;;AAWpC,SAAgB,mBACd,QACA,OAAe,sBACf,OAAkC,CAAC,IAAK,EAAG,GAC9B;CACb,MAAM,CAAC,GAAG,KAAK;CACf,IAAI,EAAE,IAAI,KAAK,IAAI,KAAK,IAAI,IAC1B,MAAM,IAAI,WAAW,uCAAuC,EAAE,IAAI,EAAE,EAAE;CACxE,MAAM,MAAM,QAAQ,GAAG,KAAK,IAAI,QAAQ,IAAI;CAC5C,IAAI,MAAM,GAAG,OAAO;CACpB,IAAI,MAAM,GAAG,OAAO;CACpB,OAAO;AACT;;;;;;;;;;;;;;;ACtIA,MAAM,UAAU,UAAU,QAAQ;AA+DlC,MAAM,SAAS,UACb,UAAU,QAAQ,OAAO,UAAU,YAAY,CAAC,MAAM,QAAQ,KAAK,IAC9D,QACD,CAAC;AACP,MAAM,gBAAgB,UACpB,OAAO,UAAU,YAAY,OAAO,SAAS,KAAK,IAAI,QAAQ;AAChE,MAAM,cAAc,UAClB,OAAO,UAAU,YAAY,MAAM,SAAS,IAAI,QAAQ;;AAG1D,eAAe,eAAe,SAA6C;CACzE,MAAM,MAAM,QAAQ,OAAO;CAC3B,MAAM,SAASC,MAAU,MAAM,SAAS,KAAK,KAAK,WAAW,GAAG,MAAM,CAAC;CAIvE,MAAM,cAAc,MAAM,SAAS,KAAK,KAAK,gBAAgB,GAAG,MAAM;CACtE,MAAM,OAAO,MAAM,OAAO,IAAI;CAC9B,MAAM,QAAQ,MAAM,OAAO,KAAK;CAChC,MAAM,WAAW,MAAM,OAAO,QAAQ;CACtC,MAAM,cAAc,MAAM,OAAO,WAAW;CAC5C,MAAM,WAAW,MAAM,OAAO,QAAQ;CACtC,MAAM,aAAa,MAAM,QAAQ,OAAO,SAAS,IAAI,OAAO,YAAY,CAAC,EAAA,CAAG,KACzE,OAAO,UAAU;EAChB,MAAM,SAAS,OAAO,UAAU,WAAW,QAAQ,MAAM,KAAK,CAAC,CAAC;EAChE,IAAI,OAAO,WAAW,YAAY,OAAO,WAAW,GAClD,MAAM,IAAI,MAAM,GAAG,IAAI,wBAAwB,MAAM,uBAAuB;EAE9E,OAAO;CACT,CACF;CACA,MAAM,mBAAmB,MAAM,QAAQ,KAAK,KAAK,aAAa,CAAC,CAAC,CAAC,YAAY,CAAC,CAAa;CAC3F,MAAM,OAAO,IAAI,MAAM,GAAG,CAAC,CAAC,IAAI,KAAK;CACrC,OAAO;EACL,MAAM,WAAW,KAAK,IAAI,KAAK;EAC/B,KAAK;EACL,SAAS;EACT;EACA,iBAAiB,aAAa,MAAM,WAAW;EAC/C,oBAAoB,aAAa,SAAS,WAAW;EACrD,cAAc,WAAW,SAAS,gBAAgB;EAClD,WAAW;GACT,MAAM,aAAa,YAAY,IAAI;GACnC,UAAU,aAAa,YAAY,SAAS;GAC5C,WAAW,aAAa,YAAY,UAAU;GAC9C,MAAM,aAAa,YAAY,IAAI,KAAK;EAC1C;EACA;EACA,gBAAgB,iBAAiB,MAAM,SAAS,mBAAmB,KAAK,IAAI,CAAC;EAC7E,UAAU,WAAW,SAAS,QAAQ;EACtC,yBAAyB,aAAa,SAAS,0BAA0B;CAC3E;AACF;;AAGA,SAAgB,6BACd,SACmC;CACnC,MAAM,QACJ,QAAQ,iBAAiB,WAAmB,mBAAmB,GAAG,QAAQ,GAAG,IAAI,QAAQ;CAC3F,MAAM,WAAW,KAAK,QAAQ,QAAQ,UAAU,GAAG,OAAO;CAE1D,eAAe,UAAqC;EAClD,MAAM,QACJ,QAAQ,UAEN,MAAM,QAAQ,KAEV,MAAM,QAAQ,QAAQ,EAAA,CACtB,IAAI,OAAO,UACV,MAAM,KAAK,KAAK,UAAU,OAAO,WAAW,CAAC,CAAC,CAAC,YAAY,IAAI,MAAO,OACnE,OACA,KACN,CACF,EAAA,CAEC,QAAQ,UAA2B,UAAU,IAAI,CAAC,CAClD,KAAK;EACV,OAAO,QAAQ,IACb,MAAM,IAAI,OAAO,QAAQ;GACvB,MAAM,UAAU,MAAM,eAAe,KAAK,UAAU,GAAG,CAAC;GACxD,OAAO;IACL,IAAI;IACJ;IACA,OAAO,MAAM,GAAG;IAChB,QAAQ,QAAQ,OAAO,QAAQ;IAC/B,UAAU;IACV,MAAM,QAAQ,aAAa,OAAO,CAAC,IAAI,CAAC,QAAQ,QAAQ;IACxD,QAAQ,QAAQ;GAClB;EACF,CAAC,CACH;CACF;CAEA,OAAO;EACL,IAAI,QAAQ;EACZ,QAAQ,QAAQ,OAAO,QAAQ;EAC/B,UAAU;EACV,aACE,GAAG,QAAQ,OAAO,QAAQ,iBAAiB,GAAG,QAAQ,OAAO,WAAW,GAAG,MAAM,QAAQ,OAAO,SAAS,8BAA8B,QACrI,SACA,GACF;EACF,QAAQ,QAAQ;EAChB,eAAe;EACf;EACA,MAAM,YAAY,KAAkB;GAClC,QAAQ,MAAM,QAAQ,EAAA,CAAG,QAAQ,SAAS,KAAK,UAAU,GAAG;EAC9D;EACA,MAAM,SAAS,MAAsB,UAAmD;GACtF,MAAM,UAAU,MAAM,QAAQ,MAAM,MAAM,QAAQ;GAClD,IAAI,CAAC,QAAQ,WACX,MAAM,IAAI,MACR,GAAG,QAAQ,GAAG,GAAG,KAAK,GAAG,8CAA8C,QAAQ,OACjF;GACF,MAAM,EAAE,QAAQ,WAAW,QAAQ;GACnC,IAAI,EAAE,OAAO,SAAS,MAAM,KAAK,UAAU,KAAK,UAAU,IACxD,MAAM,IAAI,MACR,GAAG,QAAQ,GAAG,GAAG,KAAK,GAAG,iCAAiC,OAAO,iBACnE;GAEF,OAAO;IACL,OAAO;IACP,QAAQ,WAAW;IACnB,KAAK;KAAE;KAAQ,GAAI,WAAW,KAAA,IAAY,CAAC,IAAI,EAAE,OAAO;IAAG;IAC3D,GAAI,WAAW,KAAA,IAAY,CAAC,IAAI,EAAE,OAAO,OAAO;GAClD;EACF;EACA,aAAa;CACf;AACF;AAiBA,MAAM,WACJ;;AAIF,SAAS,WAAW,MAAc,KAAqB;CACrD,IAAI,OAAO,yBAAyB,IAAI,CAAC,CAAC,MAAM,CAAC,GAAG;CACpD,OAAO,KAAK,UAAU,IAAI,CAAC,CAAC,SAAS,KAAK,OAAO,KAAK,MAAM,KAAK,UAAU,IAAI,CAAC,CAAC,SAAS,GAAG;CAC7F,OAAO;AACT;;;;;;;AAQA,SAAgB,iBACd,UAAmC,CAAC,GACgD;CACpF,MAAM,EAAE,YAAY,UAAU,UAAU,GAAG,iBAAiB,KAAM,YAAY,SAAS;CACvF,MAAM,EAAE,WAAW,SAAS,KAAK,UAAU;CAC3C,MAAM,MAAM;EAAE,KAAK;GAAE,GAAG,QAAQ;GAAK,GAAG;EAAM;EAAG;EAAS,WAAW,KAAK;CAAG;CAC7E,OAAO,OAAO,MAAM,aAAa;EAC/B,MAAM,OAAO,MAAM,QAAQ,KAAK,OAAO,GAAG,eAAe,CAAC;EAC1D,IAAI;GACF,MAAM,OAAO,KAAK,MAAM,KAAK,QAAQ,GAAG;GACxC,MAAM,GAAG,KAAK,QAAQ,SAAS,MAAM,EAAE,WAAW,KAAK,CAAC;GACxD,MAAM,GAAG,KAAK,MAAM,UAAU,GAAG;IAAE,WAAW;IAAM,OAAO;GAAK,CAAC;GACjE,MAAM,MAAM,KAAK,MAAM,UAAU,CAAC;GAClC,MAAM,UAAU,KAAK,MAAM,qBAAqB,GAAG,SAAS,KAAK;GACjE,MAAM,UAAU,KAAK,MAAM,mBAAmB,GAAG,UAAU,EAAE,MAAM,IAAM,CAAC;GAC1E,MAAM,UAAU,QAAQ,QAAQ,WAAW,KAAK,MAAM,MAAM,CAAC;GAC7D,MAAM,SAAmB,CAAC;GAE1B,MAAM,OAAO;IAAC;IAAO;IAAM;IAAM;IAAM;IAAU;IAAM;IAAS;IADnD,YAAY,gBAAgB;IACmC;GAAY;GACxF,KAAK,IAAI,UAAU,GAAG,WAAW,UAAU,GAAG,WAAW,GAAG;IAC1D,MAAM,MAAM,GAAG,KAAK,QAAQ,IAAI,GAAG,WAAW;IAC9C,MAAM,UAAU,MAAM,QAAQ,WAAW,CAAC,GAAG,MAAM,GAAG,GAAG,GAAG,CAAC,CAAC,WACtD,OACL,UAAiB,WAAW,MAAM,SAAS,GAAG,CACjD;IACA,MAAM,SAAS,MAAM,QAAQ,KAAK,SAAS,GAAG,GAAG,EAAE,eAAe,KAAK,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,EAAA,CACrF,QAAQ,UAAU,MAAM,YAAY,CAAC,CAAC,CACtC,KAAK,UAAU,KAAK,SAAS,KAAK,MAAM,MAAM,GAAG,CAAC,CAAC,CAAC;IACvD,MAAM,QAAQ,SAAiB,SAAS,GAAG,QAAQ,QAAQ,MAAM,CAAC,CAAC,YAAY,EAAE;IACjF,MAAM,SAAS,MACb,UAAU,KAAA,IAAY,CAAC,IAAI,KAAK,MAAO,MAAM,KAAK,aAAa,KAAM,IAAI,CAC3E;IACA,MAAM,SAAS,aAAa,MAAM,MAAM,OAAO,eAAe,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM;IAC/E,IAAI,WAAW,MAEb,OAAO;KAAE,WAAW;KAAM,OAAO;MAAE;MAAQ,QAD5B,WAAW,MAAM,KAAK,0BAA0B,GAAG,cAClB;KAAE;IAAE;IAEtD,MAAM,YAAY,WAAW,MAAM,OAAO,cAAc,CAAC,CAAC,iBAAiB;IAC3E,OAAO,KAAK,WAAW,QAAQ,IAAI,WAAW,aAAa,6BAA6B;GAC1F;GACA,OAAO;IAAE,WAAW;IAAO,OAAO,OAAO,KAAK,IAAI;GAAE;EACtD,UAAU;GACR,MAAM,GAAG,MAAM;IAAE,WAAW;IAAM,OAAO;GAAK,CAAC;EACjD;CACF;AACF;;AC9QA,MAAa,kBAAiC;CAC5C;EACE,IAAI;EACJ,UAAU;EACV,QAAQ;EACR,OAAO;EACP,UAAU,CAAC,oBAAoB,YAAY;EAC3C,eAAe,CAAC,WAAW,YAAY;CACzC;CACA;EACE,IAAI;EACJ,UAAU;EACV,QAAQ;EACR,OAAO;EACP,UAAU,CAAC,mBAAmB,WAAW;EACzC,eAAe,CAAC,YAAY,YAAY;CAC1C;CACA;EACE,IAAI;EACJ,UAAU;EACV,QAAQ;EACR,OAAO;EACP,UAAU,CAAC,mBAAmB,YAAY;EAC1C,eAAe,CAAC,WAAW,aAAa;CAC1C;CACA;EACE,IAAI;EACJ,UAAU;EACV,QAAQ;EACR,OAAO;EACP,UAAU,CAAC,qBAAqB,aAAa;EAC7C,eAAe,CAAC,YAAY,SAAS;CACvC;CACA;EACE,IAAI;EACJ,UAAU;EACV,QAAQ;EACR,OAAO;EACP,UAAU,CAAC,uBAAuB,YAAY;EAC9C,eAAe,CAAC,mBAAmB,YAAY;CACjD;CACA;EACE,IAAI;EACJ,UAAU;EACV,QAAQ;EACR,OAAO;EACP,UAAU,CAAC,yBAAyB,YAAY;EAChD,eAAe,CAAC,iBAAiB,cAAc;CACjD;CACA;EACE,IAAI;EACJ,UAAU;EACV,QAAQ;EACR,OAAO;EACP,UAAU,CAAC,sBAAsB,YAAY;EAC7C,eAAe,CAAC,mBAAmB,eAAe;CACpD;CACA;EACE,IAAI;EACJ,UAAU;EACV,QAAQ;EACR,OAAO;EACP,UAAU,CAAC,qBAAqB,YAAY;EAC5C,eAAe,CAAC,gBAAgB,YAAY;CAC9C;CACA;EACE,IAAI;EACJ,UAAU;EACV,QAAQ;EACR,OAAO;EACP,UAAU,CAAC,cAAc,eAAe;EACxC,eAAe,CAAC,eAAe,YAAY;CAC7C;CACA;EACE,IAAI;EACJ,UAAU;EACV,QAAQ;EACR,OAAO;EACP,UAAU,CAAC,eAAe,MAAM;EAChC,eAAe,CAAC,cAAc,SAAS;CACzC;CACA;EACE,IAAI;EACJ,UAAU;EACV,QAAQ;EACR,OAAO;EACP,UAAU,CAAC,cAAc,iBAAiB;EAC1C,eAAe,CAAC,eAAe,YAAY;CAC7C;CACA;EACE,IAAI;EACJ,UAAU;EACV,QAAQ;EACR,OAAO;EACP,UAAU,CAAC,eAAe,MAAM;EAChC,eAAe,CAAC,cAAc,SAAS;CACzC;CACA;EACE,IAAI;EACJ,UAAU;EACV,QAAQ;EACR,OAAO;EACP,UAAU,CAAC,oBAAoB;EAC/B,eAAe,CAAC,cAAc,SAAS;CACzC;CACA;EACE,IAAI;EACJ,UAAU;EACV,QAAQ;EACR,OAAO;EACP,UAAU,CAAC,sBAAsB,WAAW;EAC5C,eAAe,CAAC,cAAc,cAAc;CAC9C;CACA;EACE,IAAI;EACJ,UAAU;EACV,QAAQ;EACR,OAAO;EACP,UAAU,CAAC,oBAAoB;EAC/B,eAAe,CAAC,cAAc,SAAS;CACzC;CACA;EACE,IAAI;EACJ,UAAU;EACV,QAAQ;EACR,OAAO;EACP,UAAU,CAAC,sBAAsB,OAAO;EACxC,eAAe,CAAC,YAAY,YAAY;CAC1C;AACF;;;;;;;;;;AC9IA,IAAM,iBAAN,MAAqF;CACnF,KAAc;CACd,SAAkB;CAClB,WAAoB;CACpB,cAAuB;CACvB,gBAAyB;CAEzB,MAAM,YAAY,OAAmD;EACnE,OAAO,gBAAgB,KAAK,UAAU;GAAE,IAAI,KAAK;GAAI,SAAS;EAAK,EAAE,CAAC,CAAC,QACpE,OAAO,gBAAgB,GAAG,EAAE,MAAM,KACrC;CACF;CAEA,MAAM,SAAS,MAA0B,UAAgD;EACvF,MAAM,SAAS,mBAAmB,QAAQ;EAC1C,MAAM,UAAU,IAAI,IAClB,CAAC,KAAK,QAAQ,OAAO,GAAG,KAAK,QAAQ,QAAQ,CAAC,CAAC,KAAK,MAAM,EAAE,YAAY,CAAC,CAC3E;EACA,MAAM,UAAU,IAAI,IAAY,KAAK,QAAQ,cAAc,KAAK,MAAM,EAAE,YAAY,CAAC,CAAC;EACtF,MAAM,aAAa,OAAO,MAAM,MAAM,QAAQ,IAAI,EAAE,YAAY,CAAC,CAAC,KAAK;EACvE,MAAM,eAAe,OAAO,MAAM,MAAM,QAAQ,IAAI,EAAE,YAAY,CAAC,CAAC,KAAK;EAEzE,OAAO;GACL,OAFY,aAAa,IAAI;GAG7B,KAAK;IACH,YAAY,OAAO,MAAM;IACzB,cAAc;IACd,iBAAiB,QAAQ,YAAY;IACrC,mBAAmB;IACnB,UAAU,KAAK,QAAQ;GACzB;EACF;CACF;CAEA,YAAY,QAA6B;EACvC,OAAO,gBAAgB,MAAM;CAC/B;AACF;AAEA,SAAS,gBAAgB,QAA6B;CACpD,OAAO,mBAAmB,YAAY,QAAQ;AAChD;;;;;;;AAQA,SAAgB,mBAAmB,UAA4B;CAE7D,OADgB,SAAS,MAAM,oCAClB,KAAK,CAAC;AACrB;AAEA,MAAM,UAAU,IAAI,eAAe;AAEnC,MAAa,cAAc,QAAQ,YAAY,KAAK,OAAO;AAC3D,MAAa,WAAW,QAAQ,SAAS,KAAK,OAAO;AACrD,MAAa,cAAc,QAAQ,YAAY,KAAK,OAAO;;ACW3D,eAAsB,oBACpB,SACkD;CAClD,MAAM,UAAU,QAAQ,WAAW,kBAAkB;CACrD,MAAM,cAAc,eAAe,QAAQ,OAAO;CAClD,MAAM,YAAY,MAAM,mBAAmB,QAAQ,SAAS,QAAQ,MAAM;CAC1E,MAAM,QAAQ,eAAe,QAAQ,OAAO;CAC5C,MAAM,WAA+D,OACnE,UACA,YACG;EACH,OAAO,QAAQ,QAAQ;GAAE;GAAU,MAAM,SAAS;GAAM;EAAQ,CAAC;CACnE;CAEA,MAAM,WAAW,MAAM,YAAoD;EACzE;EACA;EACA,aAAa,aAAa;EAC1B,QAAQ,CAAC,KAAK;EACd,MAAM,QAAQ;EACd,MAAM,QAAQ;EACd,WAAW,QAAQ;EACnB,aAAa,QAAQ;EACrB,gBAAgB,QAAQ;EACxB,mBAAmB,QAAQ;EAC3B,aAAa,QAAQ,eAAe;EACpC,QAAQ,QAAQ;EAChB,MAAM,QAAQ;EACd;EACA,KAAK,QAAQ;CACf,CAAC;CACD,MAAM,SAAS,2BAA2B;EACxC,SAAS,QAAQ;EACjB;EACA;CACF,CAAC;CACD,QAAQ,UAAU,SAAS,MAAM;CACjC,MAAM,iBAAiB,KAAK,SAAS,QAAQ,uBAAuB;CACpE,MAAM,qBAAqB,KAAK,SAAS,QAAQ,qBAAqB;CACtE,QAAQ,MAAM,gBAAgB,GAAG,KAAK,UAAU,QAAQ,MAAM,CAAC,EAAE,GAAG;CACpE,QAAQ,MAAM,oBAAoB,8BAA8B,MAAM,CAAC;CACvE,OAAO;EAAE;EAAW;EAAU;EAAQ;EAAgB;CAAmB;AAC3E;AAEA,eAAsB,mBACpB,SACA,SAAiC;CAAC;CAAU;CAAO;AAAS,GACf;CAC7C,MAAM,cAAc,eAAe,OAAO;CAC1C,MAAM,SAAS,QAAQ,UAAU;CACjC,MAAM,WAAW,QAAQ,YAAY;CACrC,MAAM,MAA0C,CAAC;CACjD,MAAM,uBAAO,IAAI,IAAY;CAC7B,KAAK,MAAM,SAAS,QAAQ;EAC1B,MAAM,QAAQ,MAAM,QAAQ,YAAY,KAAK;EAC7C,KAAK,MAAM,QAAQ,OAAO;GACxB,MAAM,WAAW,KAAK,SAAS,QAAQ,YAAY,KAAK,EAAE;GAC1D,IAAI,aAAa,OAAO;GACxB,MAAM,MAAM,GAAG,YAAY,GAAG,KAAK;GACnC,IAAI,KAAK,IAAI,GAAG,GAAG;GACnB,KAAK,IAAI,GAAG;GACZ,IAAI,KAAK;IACP,IAAI;IACJ,MAAM;IACN;IACA,QAAQ,KAAK,UAAU;IACvB,UAAU,KAAK,YAAY;IAC3B;IACA,MAAM,CAAC,mBAAG,IAAI,IAAI,CAAC,UAAU,GAAI,KAAK,QAAQ,CAAC,CAAE,CAAC,CAAC;IACnD;GACF,CAAC;EACH;CACF;CACA,OAAO;AACT;AAEA,SAAgB,eACd,SACqD;CACrD,MAAM,cAAc,eAAe,OAAO;CAC1C,OAAO;EACL,MAAM,GAAG,YAAY;EACrB,YAAY,CACV;GAAE,KAAK;GAAS,aAAa,WAAW,YAAY;EAAkB,GACtE;GAAE,KAAK;GAAU,aAAa;EAAiD,CACjF;EACA,YAAY,aAAgE;GAC1E,OAAO,SAAS,SAAS,eAAe,SAAS,GAAG,WAAW,GAAG,YAAY,EAAE;EAClF;EACA,MAAM,MAAM,EAAE,UAAU,YAAY;GAClC,MAAM,aAAa,MAAM,QAAQ,SAAS,SAAS,MAAM,QAAQ;GAEjE,OAAO;IACL,YAFiB,8BAA8B,UAEtC;IACT,WAAW,QAAQ,WAAW,KAAK;IACnC,OAAO,WAAW,SAAS;GAC7B;EACF;CACF;AACF;AAEA,SAAS,2BAAgD,OAIrC;CAClB,MAAM,eAAe,IAAI,IAAI,MAAM,UAAU,KAAK,aAAa,CAAC,SAAS,IAAI,QAAQ,CAAC,CAAC;CAavF,MAAM,aAZO,MAAM,SAAS,MAAM,KAAK,SAAS;EAC9C,MAAM,WAAW,aAAa,IAAI,KAAK,UAAU;EACjD,MAAM,QAAQ,gBAAgB,KAAK,WAAW;EAC9C,MAAM,QAAQ,OAAO,aAAa;EAClC,OAAO;GACL;GACA;GACA;GACA,SAAS,OAAO,WAAW,WAAW,QAAQ,IAAI,IAAI,OAAO;GAC7D,YAAY,OAAO,cAAc,CAAC;EACpC;CACF,CACsB,CAAC,CAAC,QAAQ,QAAQ,CAAC,IAAI,KAAK,KAAK;CACvD,MAAM,cAAc,WAAW,KAAK,QAAQ,IAAI,KAAK;CACrD,OAAO;EACL,aAAa,eAAe,MAAM,OAAO;EACzC,QAAQ,MAAM,QAAQ,UAAU;EAChC,UAAU,MAAM,QAAQ,YAAY;EACpC,GAAI,MAAM,QAAQ,SAAS,EAAE,QAAQ,MAAM,QAAQ,OAAO,IAAI,CAAC;EAC/D,QAAQ,MAAM,SAAS;EACvB,cAAc,MAAM,SAAS;EAC7B,MAAM,MAAM,SAAS;EACrB,WAAW,MAAM,SAAS;EAC1B,SAAS,MAAM,SAAS;EACxB,YAAY,MAAM,SAAS;EAC3B,YAAY,MAAM,UAAU;EAC5B,YAAY,MAAM,SAAS,MAAM;EACjC,aAAa,MAAM,SAAS,WAAW;EACvC,aAAa,MAAM,SAAS,WAAW;EACvC,cAAc,MAAM,SAAS,WAAW,KAAK;EAC7C,QAAQ,gBAAgB,aAAa,QAAQ,IAAI,UAAU,YAAY,WAAW;GAChF;GACA;GACA;EACF,CAAC;EACD,MAAM,gBAAgB,aAAa,QAAQ,IAAI,UAAU,QAAQ,CAAC,CAAC;EACnE,YAAY,oBAAoB,WAAW,KAAK,QAAQ,IAAI,UAAU,CAAC;EACvE,OAAO,aAAa,WAAW;EAC/B,SAAS,aAAa,WAAW,KAAK,QAAQ,IAAI,KAAK,OAAO,CAAC;EAC/D,WAAW,aAAa,WAAW,KAAK,QAAQ,IAAI,KAAK,UAAU,CAAC;CACtE;AACF;AAEA,SAAS,8BAA8B,QAAiC;CACtE,MAAM,YAAY,OAAO,QAAQ,OAAO,MAAM,CAAC,CAC5C,KAAK,CAAC,OAAO,aAAa;EACzB,OAAO,KAAK,MAAM,KAAK,QAAQ,EAAE,KAAK,IAAI,QAAQ,SAAS,EAAE,KAAK,IAAI,QAAQ,QAAQ,EAAE,KAAK,IAAI,QAAQ,MAAM,GAAG,EAAE,KAAK,IAAI,QAAQ,QAAQ,IAAI,EAAE,KAAK,IAAI,QAAQ,UAAU,GAAG,EAAE;CACrL,CAAC,CAAC,CACD,KAAK,IAAI;CACZ,MAAM,UAAU,OAAO,QAAQ,OAAO,UAAU,CAAC,CAC9C,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC,CAAC,CACtC,KAAK,CAAC,KAAK,UAAU,KAAK,IAAI,KAAK,KAAK,EAAE,KAAK,IAAI,KAAK,IAAI,EAAE,KAAK,IAAI,KAAK,GAAG,EAAE,GAAG,CAAC,CACrF,KAAK,IAAI;CACZ,OAAO;EACL,uBAAuB,OAAO;EAC9B;EACA,aAAa,OAAO;EACpB,gBAAgB,OAAO;EACvB,cAAc,OAAO;EACrB,eAAe,OAAO;EACtB,YAAY,OAAO,WAAW,UAAU,OAAO,YAAY,WAAW,OAAO,YAAY;EACzF,YAAY,IAAI,OAAO,YAAY;EACnC,iBAAiB,IAAI,OAAO,MAAM,IAAI,EAAE,WAAW,IAAI,OAAO,MAAM,MAAM,EAAE,QAAQ,IAAI,OAAO,MAAM,GAAG,EAAE,MAAM,OAAO,MAAM;EAC7H;EACA;EACA;EACA;EACA;EACA,aAAa;EACb;EACA;EACA;EACA;EACA;EACA,WAAW;EACX;CACF,CAAC,CAAC,KAAK,IAAI;AACb;AAEA,SAAS,eAAe,SAAuE;CAC7F,OAAO,QAAQ,MAAM,GAAG,QAAQ,UAAU,SAAS,GAAG,QAAQ,YAAY;AAC5E;AAEA,SAAS,8BAA8B,YAAyD;CAC9F,MAAM,aAAqC,EAAE,OAAO,QAAQ,WAAW,KAAK,EAAE;CAC9E,KAAK,MAAM,CAAC,KAAK,UAAU,OAAO,QAAQ,WAAW,cAAc,CAAC,CAAC,GACnE,IAAI,OAAO,SAAS,KAAK,GAAG,WAAW,OAAO;CAEhD,WAAW,SAAU,WAAW,UAAU,WAAW,QAAQ,IAAK,IAAI;CACtE,OAAO;AACT;AAEA,SAAS,oBACP,MACuC;CACvC,MAAM,yBAAS,IAAI,IAAsB;CACzC,KAAK,MAAM,OAAO,MAChB,KAAK,MAAM,CAAC,KAAK,UAAU,OAAO,QAAQ,GAAG,GAAG;EAC9C,IAAI,CAAC,OAAO,SAAS,KAAK,GAAG;EAC7B,MAAM,OAAO,OAAO,IAAI,GAAG,KAAK,CAAC;EACjC,KAAK,KAAK,KAAK;EACf,OAAO,IAAI,KAAK,IAAI;CACtB;CAEF,OAAO,OAAO,YAAY,CAAC,GAAG,OAAO,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,KAAK,UAAU,CAAC,KAAK,aAAa,IAAI,CAAC,CAAC,CAAC;AACjG;AAEA,SAAS,gBACP,MACA,OACA,YAA+B,CAAC,GACO;CACvC,MAAM,0BAAU,IAAI,IAAiB;CACrC,KAAK,MAAM,OAAO,WAAW,QAAQ,IAAI,KAAK,CAAC,CAAC;CAChD,KAAK,MAAM,OAAO,MAAM;EACtB,MAAM,OAAO,MAAM,GAAG;EACtB,KAAK,MAAM,OAAO,MAAM,QAAQ,IAAI,IAAI,OAAO,CAAC,IAAI,GAAG;GACrD,MAAM,OAAO,QAAQ,IAAI,GAAG,KAAK,CAAC;GAClC,KAAK,KAAK,GAAG;GACb,QAAQ,IAAI,KAAK,IAAI;EACvB;CACF;CACA,MAAM,MAA6C,CAAC;CACpD,KAAK,MAAM,CAAC,KAAK,SAAS,SAAS;EACjC,MAAM,YAAY;EAKlB,MAAM,SAAS,UAAU,KAAK,QAAQ,IAAI,KAAK,CAAC,CAAC,OAAO,OAAO,QAAQ;EACvE,MAAM,SAAS,UAAU,QAAQ,QAAQ,IAAI,MAAM,CAAC,CAAC;EACrD,IAAI,OAAO;GACT,GAAG,KAAK;GACR,WAAW,OAAO,SAAS,KAAK,MAAM,IAAI;GAC1C,UAAU,KAAK,SAAS,SAAS,KAAK,SAAS;GAC/C,OAAO,aAAa,MAAM;GAC1B,SAAS,aAAa,UAAU,KAAK,QAAQ,IAAI,KAAK,OAAO,CAAC;GAC9D,WAAW,aAAa,UAAU,KAAK,QAAQ,IAAI,KAAK,UAAU,CAAC;EACrE;CACF;CACA,OAAO;AACT;AAEA,SAAS,gBACP,aACwB;CACxB,OAAO,OAAO,OAAO,WAAW,CAAC,CAAC;AACpC;AAEA,SAAS,aAAa,QAAkD;CACtE,MAAM,UAAU,sBAAsB,OAAO,OAAO,OAAO,QAAQ,CAAC;CACpE,IAAI,CAAC,SAAS,OAAO;EAAE,GAAG;EAAG,KAAK;EAAG,MAAM;EAAG,QAAQ;EAAG,KAAK;EAAG,KAAK;CAAE;CACxE,MAAM,EAAE,GAAG,KAAK,KAAK,QAAQ,KAAK,KAAK,QAAQ;CAC/C,OAAO;EAAE;EAAG;EAAK,MAAM,MAAM;EAAG;EAAQ;EAAK;CAAI;AACnD;AAEA,SAAS,QAAQ,OAAuB;CACtC,IAAI,CAAC,OAAO,SAAS,KAAK,GAAG,OAAO;CACpC,IAAI,QAAQ,GAAG,OAAO;CACtB,IAAI,QAAQ,GAAG,OAAO;CACtB,OAAO;AACT;AAEA,SAAS,IAAI,OAAuB;CAClC,IAAI,CAAC,OAAO,SAAS,KAAK,GAAG,OAAO;CACpC,OAAO,MAAM,QAAQ,UAAU,KAAK,KAAK,IAAI,KAAK,KAAK,KAAK,IAAI,CAAC;AACnE;;ACxRA,SAAS,eAA4B,MAAmB;CACtD,OAAO,KACJ,MAAM,OAAO,CAAC,CACd,KAAK,SAAS,KAAK,KAAK,CAAC,CAAC,CAC1B,OAAO,OAAO,CAAC,CACf,KAAK,MAAM,UAAU;EACpB,IAAI;GACF,OAAO,KAAK,MAAM,IAAI;EACxB,SAAS,OAAO;GACd,MAAM,IAAI,MAAM,qBAAqB,QAAQ,EAAE,IAAK,MAAgB,SAAS;EAC/E;CACF,CAAC;AACL;AAEA,SAAS,aAAa,MAA0B;CAC9C,OAAO,KACJ,MAAM,OAAO,CAAC,CACd,KAAK,SAAS,KAAK,KAAK,CAAC,CAAC,CAC1B,OAAO,OAAO,CAAC,CACf,QAAQ,SAAS,CAAC,KAAK,WAAW,GAAG,CAAC,CAAC,CACvC,KAAK,SAAS,KAAK,MAAM,QAAQ,CAAC;AACvC;AAEA,SAAgB,WAAW,MAAuC;CAChE,OAAO,aAAa,IAAI,CAAC,CAAC,SAAS,OAAO,UAAU;EAClD,IAAI,MAAM,SAAS,GAAG,OAAO,CAAC;EAC9B,MAAM,CAAC,SAAS,kBAAkB,mBAAmB,cAAc;EACnE,IAAI,CAAC,WAAW,CAAC,oBAAoB,CAAC,mBAAmB,OAAO,CAAC;EACjE,IAAI,QAAQ,YAAY,MAAM,cAAc,QAAQ,YAAY,MAAM,OAAO,OAAO,CAAC;EACrF,MAAM,aAAa,eAAe,KAAA,IAAY,mBAAmB;EAEjE,MAAM,QAAQ,OADI,eAAe,KAAA,IAAY,oBAAoB,UACnC;EAC9B,IAAI,CAAC,cAAc,CAAC,OAAO,SAAS,KAAK,GACvC,MAAM,IAAI,MAAM,qBAAqB,QAAQ,EAAE,mCAAmC;EAEpF,OAAO,CAAC;GAAE;GAAS;GAAY;EAAM,CAAC;CACxC,CAAC;AACH;AAEA,SAAgB,qBAAqB,MAA2C;CAC9E,OAAO,eAAwC,IAAI,CAAC,CAAC,KAAK,KAAK,UAAU;EACvE,MAAM,KAAK,YAAY,KAAK,KAAK,KAAK,YAAY,KAAK,IAAI;EAC3D,MAAM,OAAO,YAAY,KAAK,MAAM,KAAK,YAAY,KAAK,UAAU;EACpE,IAAI,CAAC,MAAM,SAAS,KAAA,GAClB,MAAM,IAAI,MAAM,2BAA2B,QAAQ,EAAE,oCAAoC;EAE3F,OAAO;GACL;GACA,OAAO,YAAY,KAAK,OAAO;GAC/B,MAAM;GACN,UAAU,WAAW,KAAK;IAAC;IAAO;IAAM;IAAS;IAAQ;GAAU,CAAC;EACtE;CACF,CAAC;AACH;AAEA,SAAgB,sBAAsB,MAAwC;CAC5E,OAAO,eAAwC,IAAI,CAAC,CAAC,KAAK,KAAK,UAAU;EACvE,MAAM,KAAK,YAAY,KAAK,KAAK,KAAK,YAAY,KAAK,IAAI,KAAK,YAAY,KAAK,UAAU;EAC3F,MAAM,QAAQ,YAAY,KAAK,MAAM,KAAK,YAAY,KAAK,OAAO;EAClE,IAAI,CAAC,MAAM,CAAC,OACV,MAAM,IAAI,MAAM,0BAA0B,QAAQ,EAAE,iCAAiC;EAEvF,OAAO;GACL;GACA,MAAM;GACN,UAAU,WAAW,KAAK;IAAC;IAAO;IAAM;IAAY;IAAQ;GAAO,CAAC;EACtE;CACF,CAAC;AACH;AAEA,SAAgB,4BACd,SACuD;CACvD,MAAM,+BAAe,IAAI,IAAqC;CAC9D,KAAK,MAAM,QAAQ,QAAQ,OAAO;EAChC,IAAI,KAAK,SAAS,GAAG;EACrB,MAAM,OAAO,aAAa,IAAI,KAAK,OAAO,KAAK,CAAC;EAChD,KAAK,KAAK,IAAI;EACd,aAAa,IAAI,KAAK,SAAS,IAAI;CACrC;CACA,MAAM,SACJ,QAAQ,0BAA0B,QAAQ,SACtC,OAAO,YAAY,QAAQ,OAAO,KAAK,aAAa,CAAC,SAAS,IAAI,QAAQ,CAAC,CAAC,IAC5E,KAAA;CACN,OAAO,QAAQ,QAAQ,SAAS,UAAU;EACxC,MAAM,QAAQ,aAAa,IAAI,MAAM,EAAE,KAAK,CAAC;EAC7C,IAAI,MAAM,WAAW,GAAG,OAAO,CAAC;EAChC,MAAM,QACJ,QAAQ,UAAU,MAAM,EAAE,KAAK,mBAAmB,GAAG,QAAQ,YAAY,GAAG,MAAM,IAAI;EACxF,OAAO,CACL;GACE,IAAI,MAAM;GACV;GACA,QAAQ,QAAQ;GAChB,UAAU;GACV,MAAM,CAAC,mBAAG,IAAI,IAAI,CAAC,GAAI,QAAQ,QAAQ,CAAC,GAAI,KAAK,CAAC,CAAC;GACnD,GAAI,QAAQ,SAAS,EAAE,QAAQ,QAAQ,OAAO,IAAI,CAAC;GACnD,GAAI,MAAM,WAAW,EAAE,UAAU,MAAM,SAAS,IAAI,CAAC;GACrD,SAAS;IACP,SAAS,MAAM;IACf,OAAO,MAAM;IACb,qBAAqB,MAAM,KAAK,SAAS,KAAK,UAAU;IACxD,gBAAgB,OAAO,YAAY,MAAM,KAAK,SAAS,CAAC,KAAK,YAAY,KAAK,KAAK,CAAC,CAAC;IACrF,GAAI,SAAS,EAAE,OAAO,IAAI,CAAC;IAC3B,GAAI,MAAM,WAAW,EAAE,UAAU,MAAM,SAAS,IAAI,CAAC;GACvD;EACF,CACF;CACF,CAAC;AACH;AAEA,SAAgB,kCACd,SAKA;CACA,MAAM,QAAQ,4BAA4B,OAAO;CACjD,OAAO;EACL,IAAI,QAAQ;EACZ,QAAQ,QAAQ;EAChB,UAAU;EACV,QAAQ,QAAQ;EAChB,eAAe,QAAQ,iBAAiB;EACxC,MAAM,YAAY,OAAO;GACvB,OAAO,MAAM,QAAQ,SAAS,KAAK,UAAU,KAAK;EACpD;EACA,MAAM,SAAS,MAAM,UAAU;GAC7B,OAAO,0BAA0B,KAAK,SAAS,UAAU,OAAO;EAClE;EACA,YAAY,QAAQ;GAClB,OAAO,QAAQ,UAAU,MAAM,KAAK,mBAAmB,GAAG,QAAQ,YAAY,GAAG,QAAQ;EAC3F;CACF;AACF;AAEA,SAAgB,0BACd,SACA,UACA,UAA8C,CAAC,GAC/C;CACA,MAAM,oBAAoB,8BACxB,UACA,QAAQ,qBAAqB,oBAC/B;CACA,MAAM,UAAU,iBAAiB,QAAQ,WAAW;EAAC;EAAG;EAAG;EAAG;CAAE,CAAC;CACjE,MAAM,aAAqC;EACzC,gBAAgB,QAAQ,oBAAoB;EAC5C,gBAAgB,kBAAkB;CACpC;CACA,KAAK,MAAM,UAAU,SACnB,OAAO,OACL,YACA,yBAAyB;EACvB;EACA,gBAAgB,QAAQ;EACxB;CACF,CAAC,CACH;CAEF,MAAM,gBAAgB,QAAQ,iBAAiB;CAC/C,MAAM,aAAa,QAAQ,cAAc;CACzC,MAAM,QAAQ,WAAW,kBAAkB,WAAW,QAAQ,QAAQ,QAAQ,SAAS,SAAS;CAGhG,OAAO;EACL;EACA,SAHA,WAAW,eAAe,WAAW,OAAO,QAAQ,QAAQ,SAAS,SAAS,WAGxD,QAAQ,iBAAiB;EAC/C;EACA,KAAK;GACH;GACA,qBAAqB,QAAQ;GAC7B,gBAAgB,QAAQ;EAC1B;CACF;AACF;AAEA,SAAS,8BACP,UACA,oBAA4B,sBAClB;CACV,IAAI,OAAO,aAAa,UACtB,OAAO,eAAe,SAAS,MAAM,iBAAiB,KAAK,CAAC,EAAA,CAAG,KAAK,UAAU,MAAM,KAAK,CAAC,CAAC;CAE7F,IAAI,MAAM,QAAQ,QAAQ,GAExB,OAAO,cACLC,SAAQ,SAAS,UAAU;EACzB,IAAI,OAAO,UAAU,UAAU,OAAO,CAAC,KAAK;EAC5C,OAAO,CAAC,MAAM,cAAc,MAAM,SAAS,MAAM,MAAM,EAAE;CAC3D,CAAC,CACH;CAEF,MAAM,iBAAiB;CACvB,OAAO,cACL;EACE,GAAI,eAAe,eAAe,CAAC;EACnC,GAAI,eAAe,OAAO,CAAC;EAC3B,IAAI,eAAe,WAAW,CAAC,EAAA,CAAG,KAC/B,UAAU,MAAM,cAAc,MAAM,SAAS,MAAM,MAAM,EAC5D;CACF,CAAC,CAAC,KAAK,UAAU,MAAM,KAAK,CAAC,CAC/B;AACF;AAEA,SAAS,yBAAyB,OAIP;CACzB,MAAM,MAAM,MAAM,kBAAkB,MAAM,GAAG,MAAM,MAAM;CACzD,MAAM,cAAc,OAAO,KAAK,MAAM,cAAc,CAAC,CAAC,QACnD,OAAO,MAAM,eAAe,MAAO,CACtC;CACA,MAAM,WAAW,IAAI,IAAI,WAAW;CACpC,MAAM,OAAO,IAAI,QAAQ,OAAO,SAAS,IAAI,EAAE,CAAC,CAAC,CAAC;CAClD,MAAM,oBAAoB,IAAI,WAAW,OAAO,SAAS,IAAI,EAAE,CAAC;CAChE,MAAM,SAAS,IAAI,MAAM;CACzB,OAAO;GACJ,MAAM,WAAW,OAAO,IAAI,IAAI;GAChC,SAAS,WAAW,SAAS,SAAS,IAAI,IAAI,OAAO,SAAS;GAC9D,YAAY,WAAW,OAAO,MAAM;GACpC,MAAM,WAAW,sBAAsB,KAAK,IAAI,KAAK,oBAAoB;GACzE,OAAO,WAAW,OAAO,MAAM,mBAAmB,MAAM,gBAAgB,MAAM,MAAM;CACvF;AACF;AAEA,SAAS,YAAY,KAA8B,KAAiC;CAClF,MAAM,QAAQ,IAAI;CAClB,OAAO,OAAO,UAAU,WAAW,QAAQ,KAAA;AAC7C;AAEA,SAAS,WACP,KACA,MACyB;CACzB,MAAM,QAAQ,IAAI,IAAI,IAAI;CAC1B,OAAO,OAAO,YAAY,OAAO,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC,MAAM,IAAI,GAAG,CAAC,CAAC;AAClF;AAEA,SAAS,iBAAiB,SAAsC;CAC9D,OAAO,CACL,GAAG,IAAI,IAAI,QAAQ,KAAK,WAAW,KAAK,MAAM,MAAM,CAAC,CAAC,CAAC,QAAQ,WAAW,SAAS,CAAC,CAAC,CACvF,CAAC,CAAC,MAAM,GAAG,MAAM,IAAI,CAAC;AACxB;AAEA,SAAS,cAAc,QAAqC;CAC1D,MAAM,uBAAO,IAAI,IAAY;CAC7B,MAAM,MAAgB,CAAC;CACvB,KAAK,MAAM,SAAS,QAAQ;EAC1B,MAAM,UAAU,MAAM,KAAK;EAC3B,IAAI,CAAC,WAAW,KAAK,IAAI,OAAO,GAAG;EACnC,KAAK,IAAI,OAAO;EAChB,IAAI,KAAK,OAAO;CAClB;CACA,OAAO;AACT;AAEA,SAAS,OACP,mBACA,gBACA,QACQ;CACR,MAAM,MAAM,kBACT,MAAM,GAAG,MAAM,CAAC,CAChB,QACE,KAAK,YAAY,UAAU,MAAM,eAAe,eAAe,eAAe,GAAG,KAAK,GACvF,CACF;CACF,MAAM,QAAQ,OAAO,OAAO,cAAc,CAAC,CACxC,QAAQ,UAAU,QAAQ,CAAC,CAAC,CAC5B,MAAM,GAAG,MAAM,IAAI,CAAC,CAAC,CACrB,MAAM,GAAG,MAAM,CAAC,CAChB,QAAQ,KAAK,OAAO,UAAU,MAAM,eAAe,OAAO,KAAK,GAAG,CAAC;CACtE,OAAO,UAAU,IAAI,IAAI,MAAM;AACjC;AAEA,SAAS,eAAe,WAAmB,eAA+B;CACxE,IAAI,aAAa,GAAG,OAAO;CAC3B,QAAQ,KAAK,YAAY,KAAK,KAAK,KAAK,gBAAgB,CAAC;AAC3D"}

package/dist/{index-JPI2spOx.d.ts → index-C8a9Dyan.d.ts} RENAMED Viewed

@@ -26,7 +26,7 @@ import "./engine-CPBhCLJR.js";
 import "./registry-BqoX13FD.js";
 import "./index-AjfEHHA8.js";
 import { d as Trajectory, f as TrajectoryStep } from "./tool-waste-CUASoTtk.js";
-import "./types-D7g2XFTc.js";
+import "./types-CEgIBy0W.js";
 import { B as ControlRuntimeConfig, H as ControlSeverity, R as ControlEvalResult, a as FeedbackLabel, v as ProposedSideEffect, z as ControlRunResult } from "./feedback-trajectory-DxkG5EPi.js";
 import "./store-tool-spans-kCDPEwjL.js";
 import "./researcher-BuojS14L.js";
@@ -3390,4 +3390,4 @@ declare function assertNotContains(name: string, actual: string, forbidden: stri
 declare function assertEventCount(name: string, count: number, min: number, max?: number): HarnessAssertion;
 declare function assertHasField(name: string, obj: Record<string, unknown>, field: string): HarnessAssertion;
 export { EnsembleAggregate as $, makePolicyEditCandidateRecord as $i, ReviewInput as $n, pairedDeltaTest as $r, TraceContract as $t, contentHash as A, AssertCapabilityHeadroomOptions as Ai, PromptRegistry as An, RuntimeTrajectoryHookEvent as Ar, checkCanaries as At, evaluateOracles as B, PolicyEditAdmissionOptions as Bi, ReflectionContext as Bn, ProductBenchmarkExportResult as Br, evaluateTraceContract as Bt, runAgentControlLoop as C, errorStreakDetector as Ci, aggregateRunScore as Cn, runBoundedProcess as Cr, DeployRunner as Ct, index_d_exports as D, CostSummary as Di, ReferenceReplayRun as Dn, ProjectRuntimeTrajectoryEvidenceOptions as Dr, viteDeployRunner as Dt, BudgetGuard as E, CostEntry as Ei, ReferenceReplayCaseRun as En, proposeSynthesisTargets as Er, deployGateLayer as Et, gateTreatmentApplied as F, assertCapabilityHeadroom as Fi, ModelEndpointCheck as Fn, ProductBenchmarkSplit as Fr, TraceContractSpec as Ft, KeywordCoverageResult as G, PolicyEditGainDirection as Gi, LlmReviewerConfig as Gn, HeldOutPartition as Gr, ContractRuleExecution as Gt, KeywordConceptSpec as H, PolicyEditCandidateRecord as Hi, TrialTrace as Hn, exportProductBenchmark as Hr, ArgumentCheck as Ht, Oracle as I, capabilityHeadroom as Ii, ModelEndpointRequest as In, ProductBenchmarkValidationReport as Ir, compileTraceContractSpec as It, EnsembleJudgeOptions as J, PolicyEditRisk as Ji, ProposeOutput as Jn, TrainSelectionTestPartition as Jr, ContractStatus as Jt, runKeywordCoverageJudge as K, PolicyEditGainUnit as Ki, ProposeFn as Kn, PartitionHeldOutOptions as Kr, ContractRun as Kt, OracleObservation as L, FindingToPolicyEditOptions as Li, preflightModels as Ln, assertProductBenchmarkRun as Lr, lintTraceContractSpec as Lt, TreatmentGate as M, CapabilityHeadroomResult as Mi, ModelSeats as Mn, RuntimeTrajectoryRunRecord as Mr, ActionPolicyDecision as Mt, TreatmentGateInput as N, HeadroomInput as Ni, resolveSeat as Nn, projectRuntimeTrajectoryEvidence as Nr, evaluateActionPolicy as Nt, VerdictCacheStore as O, CostTracker as Oi, ReferenceReplaySplit as On, RuntimeTrajectoryEvidenceProjection as Or, wranglerDeployRunner as Ot, TreatmentGateOptions as P, TaskHeadroom as Pi, seatPresets as Pn, ProductBenchmarkManifest as Pr, ContractLintFinding as Pt, withJudgeRetry as Q, PolicyEditTargetSurface as Qi, ReviewFn as Qn, minimumPairsForPairedDeltaTest as Qr, SpanPredicate as Qt, OracleReport as R, PolicyEdit as Ri, RunRecordBackend as Rn, readProductBenchmarkManifest as Rr, explainTraceContract as Rt, subjectiveEval as S, StreamingDetector as Si, RunScoreWeights as Sn, DEFAULT_BOUNDED_PROCESS_TIMEOUT_MS as Sr, DeployRunResult as St, BudgetBreachError as T, repeatedActionDetector as Ti, SteeringBundle as Tn, SynthesisTarget as Tr, WranglerDeployRunnerInput as Tt, KeywordCoverageFinding as U, PolicyEditChange as Ui, buildReflectionPrompt as Un, exportProductBenchmarkRuns as Ur, ContractCheckResult as Ut, jsonShape as V, PolicyEditAxis as Vi, ReflectionProposal as Vn, ProductBenchmarkSingleRunExportOptions as Vr, traceContract as Vt, KeywordCoverageOptions as W, PolicyEditExpectedGain as Wi, parseReflectionResponse as Wn, productBenchmarkRepoIdentity as Wr, ContractRule as Wt, JudgeRetryOutcome as X, PolicyEditSource as Xi, ProposeReviewReport as Xn, PairedDeltaTestOptions as Xr, OrderMode as Xt, ensembleJudge as Y, PolicyEditSchemaVersion as Yi, ProposeReviewConfig as Yn, partitionHeldOut as Yr, ContractVerdict as Yt, JudgeRetryPolicy as Z, PolicyEditTarget as Zi, Review as Zn, PairedDeltaTestResult as Zr, RetryWrite as Zt, blockingKnowledgeEval as _, extractErrorCount as _i, CounterfactualRunner as _n, RunCommandInput as _r, NoLeakOptions as _t, assertHttpOk as a, costReceiptFromLlmError as aa, KnowledgeBundle as ai, ScorecardCellDiff as an, inMemoryReviewStore as ar, runIntentMatchJudge as at, userQuestionsForKnowledgeGaps as b, ErrorStreakOptions as bi, SteeringOptimizationRow as bn, BoundedProcessInput as br, blendHeldout as bt, assertStatus as c, costForTokenPricing as ca, KnowledgeReadinessReport as ci, ScorecardLogLine as cn, ExperimentRep as cr, SingleBackendDivergence as ct, MODEL_PRICING as d, CostCeilingReachedError as da, KnowledgeSensitivity as di, loadScorecard as dn, ImprovementVerdictResult as dr, BlendWeights as dt, isTransientLlmError as ea, LeaderboardOptions as ei, CellVerdict as en, ReviewMemoryEntry as er, JudgeVerdict as et, estimateCost as f, UserQuestion as fi, recordRuns as fn, DiscoverPersonasOptions as fr, FieldDestination as ft, acquisitionPlansForKnowledgeGaps as g, ExtractResult as gi, CounterfactualResult as gn, DirEntry as gr, JudgeScoreInput as gt, resolveModelPricing as h, ExtractOptions as hi, CounterfactualMutation as hn, CommandRunner as hr, HiddenLeak as ht, assertHasField as i, costReceiptFromLlm as ia, KnowledgeAcquisitionMode as ii, ScorecardCell as in, createLlmReviewer as ir, IntentMatchResult as it, fileVerdictCache as j, CapabilityHeadroomOptions as ji, hashContent as jn, RuntimeTrajectoryRecord as jr, ActionExecutionPolicy as jt, canonicalJson as k, ScenarioCost as ki, PromptHandle as kn, RuntimeTrajectoryEvidenceSummary as kr, CanaryLeak as kt, analyzeAntiSlop as l, costForUsage as la, KnowledgeRequirement as li, diffScorecard as ln, ExperimentStats as lr, SingleBackendReport as lt, isModelPriced as m, ErrorCountPattern as mi, CounterfactualContext as mn, discoverPersonas as mr, HiddenGradeResult as mt, assertEventCount as n, LlmCallError as na, leaderboard as ni, RecordRunsOptions as nn, Verification as nr, IntentMatchInput as nt, assertLatencyUnder as o, maximumChargeForLlmRequest as oa, KnowledgeFreshness as oi, ScorecardDiff as on, jsonlReviewStore as or, AssertSingleBackendOptions as ot, estimateTokens as p, ERROR_COUNT_PATTERNS as pi, recordRunsToScorecard as pn, DiscoveredPersona as pr, HiddenCriteriaGrader as pt, runKeywordCoverageJudgeUrl as q, PolicyEditInit as qi, ProposeInput as qn, TrainSelectionTestOptions as qr, ContractSpan as qt, assertExists as r, LlmResponseError as ra, DataAcquisitionPlan as ri, Scorecard as rn, VerifyFn as rr, IntentMatchOptions as rt, assertNotContains as s, CostResult as sa, KnowledgeImportance as si, ScorecardEntry as sn, runProposeReview as sr, BackendDescriptor as st, assertContains as t, stripFencedJson as ta, LeaderboardRow as ti, DiffScorecardOptions as tn, ReviewMemoryStore as tr, aggregateJudgeVerdicts as tt, createAntiSlopJudge as u, CostAccountingIncompleteError as ua, KnowledgeRequirementCategory as ui, formatScorecardDiff as un, ImprovementThresholds as ur, assertSingleBackend as ut, knowledgeReadinessTracePayload as v, DetectorEvent as vi, PairwiseSteeringOptimizer as vn, RunCommandResult as vr, RoutedField as vt, ProductClient as w, observeAll as wi, clamp01 as wn, ActiveLearningOptions as wr, ViteDeployRunnerInput as wt, objectiveEval as x, RepeatedActionOptions as xi, RunScore as xn, BoundedProcessResult as xr, DeployGateLayerInput as xt, scoreKnowledgeReadiness as y, DetectorSignal as yi, SteeringOptimizationResult as yn, localCommandRunner as yr, assertNoHiddenLeak as yt, OracleResult as z, PolicyEditAdmission as zi, jsonlRunRecordBackend as zn, ProductBenchmarkExportOptions as zr, checkTraceContracts as zt };
-//# sourceMappingURL=index-JPI2spOx.d.ts.map
+//# sourceMappingURL=index-C8a9Dyan.d.ts.map

package/dist/{index-JPI2spOx.d.ts.map → index-C8a9Dyan.d.ts.map} RENAMED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"index-JPI2spOx.d.ts","names":[],"sources":["../src/cost-ledger/errors.ts","../src/cost-ledger/pricing.ts","../src/llm-client/accounting.ts","../src/llm-client/errors.ts","../src/llm-client/json.ts","../src/llm-client/resilience.ts","../src/analyst/policy-edit.ts","../src/capability-headroom.ts","../src/cost-tracker.ts","../src/detectors/index.ts","../src/error-count-extractor.ts","../src/knowledge/types.ts","../src/leaderboard.ts","../src/paired-delta-test.ts","../src/partition-held-out.ts","../src/product-benchmark/export.ts","../src/product-benchmark/index.ts","../src/runtime-trajectory.ts","../src/active-learning.ts","../src/bounded-process.ts","../src/command-runner.ts","../src/discover-personas.ts","../src/improvement-verdict.ts","../src/propose-review.ts","../src/reflective-mutation.ts","../src/eval-trace-store.ts","../src/integrity/preflight.ts","../src/model-seats.ts","../src/prompt-registry.ts","../src/reference-replay.ts","../src/steering.ts","../src/run-score.ts","../src/steering-optimizer.ts","../src/counterfactual.ts","../src/scorecard.ts","../src/trace-contracts/types.ts","../src/trace-contracts/builder.ts","../src/trace-contracts/evaluate.ts","../src/trace-contracts/explain.ts","../src/trace-contracts/spec.ts","../src/action-policy.ts","../src/contamination-guard.ts","../src/deploy-gate-layer.ts","../src/hidden-criteria-grading.ts","../src/integrity/single-backend.ts","../src/intent-match-judge.ts","../src/judge-ensemble.ts","../src/judge-retry.ts","../src/judge-panel.ts","../src/keyword-coverage-judge.ts","../src/oracle.ts","../src/treatment-gate.ts","../src/verdict-cache.ts","../src/profile/baselines.ts","../src/profile/index.ts","../src/budget-guard.ts","../src/client.ts","../src/control-runtime.ts","../src/control-runtime/policy.ts","../src/knowledge/readiness.ts","../src/metrics.ts","../src/anti-slop.ts","../src/sandbox-assertions.ts"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;cAMa,gCAAgC;EAEzC,YAAA,oBACA,sBACA,sBACA,eACA;;cAQS,sCAAsC;UCblC;EACf;EACA;;iBAGc,aAAa,eAAe,OAAO,YAAY;;iBAc/C,oBACd,SAAS,oBACT,OAAO,KAAK;iBCtBE,2BACd,SAAS,KACP,0GAGF,UAAS,kBACR;;;;;;iBAoCa,mBACd,QAAQ,eACR,qBAAqB,qBACpB;;iBA6Ba,wBACd,OAAO,OACP,qBAAqB,qBACpB;cChFU,qBAAqB;WAGd;WACA;WACA;EAHhB,YAAA,iBACgB,gBACA,cACA;;;;;cASP,yBAAyB;WAGlB,QAAQ;EADxB,YAAA,iBACgB,QAAQ,eACxB;IAAY;;;;;;;;iBCXA,gBAAgB;;;;;;;;;;;;iBCgFhB,oBAAoB;KClFxB;cAEN;KAaM,yBAAyB;cAE/B;KAUM,kCAAkC;KAClC;KACA;KACA;UAEK;EACf,SAAS;;;EAGT;;;EAGA,mBAAmB;;EAEnB;;KAGU;EAEN;EACA;EACA;;EAEA;;EAGA;EACA;EACA;EACA,QAAQ;;UAGG;;EAEf;EACA,WAAW;;EAEX;EACA,OAAO;EACP;;UAGe;EACf;EACA;EACA,cAAc;;EAEd;;UAGe;EACf,eAAe;EACf;EACA,MAAM;EACN,QAAQ;EACR,QAAQ;EACR;EACA,cAAc;EACd;EACA,MAAM;EACN,QAAQ;EACR;EACA;EACA,WAAW;;cAGP;;UAGW;EACf,eAAe;EACf,YAAY;;KAGF,iBAAiB,KAAK;EAChC,gBAAgB;EAChB;;UAWe;EACf,eACI,2BACE,SAAS,mBAAmB;EAClC,OAAO,mBAAmB,SAAS,mBAAmB;EACtD,cAAc;EACd,uBAAuB;;UAGR;EACf;EACA;EACA;EACA;;UAGe;EACf,MAAM;EACN;EACA;EACA;;iBAgDc,8BAA8B,MAAM,aAAa;;;;;;;;;;;;;;;;;;;;;;;UCrKhD;;EAEf;;;EAGA;;;;;;KAOG;UAEY;EACf;;EAEA;;;;EAIA;;;EAGA;EACA,UAAU;;UAGK;;;;EAIf;;UAGe;;EAEf,OAAO;EACP;IACE;IACA;IACA;;;;IAIA;;;;;;;;;;;;;;;;iBAiBY,mBACd,eAAe,iBACf,OAAM,4BACL;UAiEc;;EAEf;;;;;;;;iBASc,yBACd,QAAQ,0BACR,OAAM;;;;;;;;;;;UC3JE;EACR;EACA;EACA;EACA;;UAGe,kBAAkB;EACjC;EACA;;EAEA;EACA;;EAEA,OAAO;;UAGQ;EACf;EACA,SAAS;EACT;EACA;EACA;EACA;;EAEA;;cAGW;UACH;EAER,OAAO,OAAO,KAAK;IAA4B;MAAuB;EAwBtE,YAAY,oBAAoB;;;;;;;;;;;EAgBhC,cACE;IACE;MACE;MACA;MACA;MACA;MACA;;IAEF;KAEF,oBACA,OAAO,yBACN;EAeH,IAAI,qBAAqB;EAIzB,QAAQ;EAIR,WAAW;;UAmBI;EACf;EACA;EACA;EACA;EACA;EACA;;EAEA;;KCpIG;;;;UAKY;;WAEN;;WAEA;;WAEA;;WAEA;;WAEA;;UAGM;WACN;WACA,UAAU;WACV,eAAe;WACf;;WAEA;WACA,WAAW;;UAGL;WACN;;WAEA;;EAET,QAAQ,OAAO,gBAAgB;;EAE/B;;UAGe;;;WAGN;WACA,WAAW;;WAEX,eAAe;;;iBAIV,uBAAuB,OAAM,wBAA6B;UAkFzD;;WAEN;WACA,WAAW;WACX,eAAe;;;iBAIV,oBAAoB,OAAM,qBAA0B;;;iBA+BpD,WACd,WAAW,cAAc,oBACzB,OAAO,gBACN;;;;;;;;;;;;;;;;UC3Kc;;EAEf;;EAEA,OAAO;;EAEP,aAAa,OAAO;;cAGT,sBAAsB;UAoClB;;EAEf;;EAEA,QAAQ;;UAGO;;EAEf;;EAEA;;EAEA;;;;;;;;;iBAUc,kBAAkB,cAAc,OAAM,iBAAsB;KClFhE;KAaA;KASA;KACA;KACA;KACA;UAEK;EACf;EACA;EACA;EACA,UAAU;EACV,iBAAiB;EACjB,YAAY;EACZ,WAAW;EACX,aAAa;EACb;EACA;EACA;EACA,gBAAgB;;;;;EAKhB;;EAEA;EACA,WAAW;;UAGI;EACf;EACA,cAAc;EACd;EACA;EACA;EACA,aAAa;EACb,SAAS;EACT;EACA,WAAW;;KAGD;UAUK;EACf;EACA;EACA,6BAA6B;EAC7B,iBAAiB;EACjB,mBAAmB;EACnB,QAAQ;EACR,UAAU;EACV;;UAGe;EACf;EACA;EACA;EACA;EACA,YAAY;EACZ;EACA;EACA;EACA;EACA,WAAW;;UAGI;EACf;EACA;EACA,MACI,QAAQ;EAEZ;EACA,UAAU;EACV;EACA,YAAY;EACZ,WAAW;;UC9FI;;EAEf;;EAEA;EACA;EACA;EACA;;EAEA;EACA;EACA;;EAEA;EACA;EACA;EACA;;UAGe;;;EAGf,SAAS,QAAQ;;;EAGjB,WAAW,QAAQ;;;;;;;iBAYL,YAAY,SAAS,aAAa,MAAM,qBAAqB;UC5C5D,+BAA+B;;EAE9C;;EAEA;;UAGe;EACf,WAAW;EACX;;EAEA;;EAEA;EACA;;;;;;EAMA;EACA;;;iBAIc,+BAA+B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBA6C/B,gBACd,kBACA,iBACA,UAAS,yBACR;;;;;;;;;;;;;;;;;;UCrCc;;;;EAIf;;EAEA;;;;;;;EAOA;;EAEA;;UAGe;;EAEf;;EAEA;EACA;EACA;;;;;;;;;;;;iBAac,iBACd,eACA,UAAS,0BACR;UAmDc,0BAA0B;;EAEzC;;EAEA;;EAEA;;;;;;EAMA,UAAU,MAAM;;EAEhB;EACA;EACA;;UAGe,4BAA4B;;EAE3C,OAAO;;EAEP,WAAW;;EAEX,MAAM;EACN;EACA;EACA;;UCxHe;;WAEN;;WAEA;WACA;WACA;;WAEA;;WAEA;;;;;WAKA;;WAEA;;WAEA;;;;;;WAMA;;;;;WAKA,iBAAiB,QAAQ,cAAc;;WAEvC,oBAAoB,QAAQ,WAAW;;WAEvC;;;;;;;WAOA,YAAY,QAAQ;;UAGd,+CACP,KAAK;WACJ;;UAGM;WACN;WACA;WACA;;;iBA6DK,gCAAgC;;iBA4ehC,uBACd,SAAS,yCACR;;;;;;iBAUa,2BACd,SAAS,gCACR;cCjpBG;KAEM,gCAAgC;UAE3B;WACN;WACA;WACA;;UAGM;WACN;WACA;WACA;WACA;WACA;;UAGM;WACN;WACA;WACA;;UAGM;WACN;WACA;WACA;WACA,YAAY;;UAGN;WACN;WACA,OAAO;WACP;WACA;;UAGM;WACN;WACA;WACA;;UAGM;WACN;WACA;WACA;WACA,MAAM;WACN,WAAW;WACX,mBAAmB;WACnB,eAAe;WACf,oBAAoB;WACpB,SAAS;WACT;;UA0EM;WACN;WACA;WACA;;WAEA;;;WAGA;WACA;WACA;WACA;WACA;WACA;WACA;WACA;WACA;WACA;WACA;WACA;WACA;;iBA0TK,6BAA6B,eAAe;;;;;;iBAkH5C,0BAA0B,iBAAiB;UClkB1C;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA,WAAW;;UAGI;EACf;EACA;EACA,WAAW;EACX;GACC;;UAGc;EACf;EACA;EACA,UAAU;;UAGK;EACf;EACA;EACA;EACA;EACA;;UAGe;EACf,MAAM;EACN,QAAQ;EACR,SAAS;EACT;;UAGe,wCACf,gBAAgB,0BAA0B;EAE1C,SAAS;EACT,kBAAkB;EAClB,cAAc,QAAQ,SAAS;EAC/B,gBAAgB,QAAQ,SAAS;;iBAKnB,iCAAiC,gBAAgB,yBAC/D,SAAS,wCAAwC,WAChD;KClCE;UAEY;EACf,QAAQ;EACR;;;EAGA,WAAW;;EAEX;;EAEA;;UAGe;;EAEf;;EAEA;;EAEA;;iBAGoB,wBACpB,SAAS,SACT,YAAY,YACZ,UAAS,wBACR,QAAQ;;;;;;;;;;;;;;;;;;;;;;;;;;;;cCpBE;UAwBI;;;;;;EAMf;;;;;;;;;;;;;;;;;;;;;;;EAuBA;;;;;;;;;;;;;;;;;;;;;;;;;;;EA2BA;;;;;;;;;EASA;;EAEA;;;;;EAKA,MAAM,OAAO;;;;;;;;;;;;;;;;;;;EAmBb;;EAEA;;;;;;EAMA;;;;;;EAMA,SAAS;;UAGM;;;;;;;;;;;EAWf;;;;;;;;;EASA;;;;;;EAMA;;EAEA;;;;;;EAMA;;EAEA;;EAEA;;EAEA;;EAEA;;;;;;;;;;;;;;;;EAgBA;;iBAGoB,kBAAkB,OAAO,sBAAsB,QAAQ;;;;;;;;;;;;;;;;;;UCjM5D;;EAEf;;EAEA;;EAEA;;;;;;;;;;;EAWA;;EAEA,MAAM;;EAEN;;UAGe;;EAEf;EACA;EACA;EACA;EACA;;EAEA;;UAGe;EACf;EACA;EACA;;EAEA;;UAGe;;WAEN;;EAET,IAAI,OAAO,kBAAkB,QAAQ;;EAErC,OAAO,eAAe;;EAEtB,WAAW,eAAe;;EAE1B,SAAS,eAAe;;EAExB,QAAQ,eAAe,QAAQ;;cAkBpB,oBAAoB;;;;;;;UC9FhB;;;;;EAKf,UAAU;;;;;EAKV;;;;;EAKA;;EAEA;;UAGe;;EAEf;;EAEA;;EAEA;;iBAKoB,iBACpB,aACA,OAAM,0BACL,QAAQ;;;KClBN;;;UAIY;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA,WAAW;;EAEX;;EAEA,UAAU;;UAGK;EACf;EACA;EACA;EACA;;EAEA;;EAEA;;;EAGA;;EAEA;;EAEA;;UAGe;;EAEf;;EAEA;;EAEA;;EAEA;;;EAGA;;UAGe;EACf,SAAS;;EAET;;EAEA;;UCtCe;EACf;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA;EACA;EACA;;UAGe,0BAA0B;EACzC;EACA;EACA;IACE;IACA;IACA;;;UAIa,aAAa;EAC5B;EACA;EACA,OAAO;EACP,aAAa;EACb,aAAa;EACb,UAAU;;UAGK,cAAc,OAAO;EACpC,OAAO;EACP,eAAe;;UAGA,YAAY,OAAO;EAClC;EACA;EACA,OAAO;EACP,cAAc;EACd,cAAc;EACd,QAAQ;;KAGE,UAAU,OAAO,sBAC3B,OAAO,aAAa,WACjB,QAAQ,cAAc,OAAO;KAEtB,SAAS,UAAU,OAAO,UAAU,QAAQ;KAE5C,SAAS,OAAO,sBAC1B,OAAO,YAAY,OAAO,aACvB,QAAQ;UAEI;EACf,QAAQ,QAAQ;EAChB,OAAO,OAAO,oBAAoB;;UAGnB,oBAAoB,OAAO;EAC1C;EACA,cAAc;EACd,SAAS,UAAU,OAAO;EAC1B,QAAQ,SAAS;EACjB,QAAQ,SAAS,OAAO;;EAExB;;EAEA;;;;;;EAMA;EACA;;EAEA,SAAS;;EAET,QAAQ;EACR;EACA;EACA;;;;;EAKA;;UAGQ,kBAAkB,OAAO;EACjC;EACA,OAAO;EACP,cAAc;EACd,cAAc;EACd,QAAQ;EACR;EACA;EACA;;UAGe,oBAAoB,OAAO;EAC1C;EACA;EACA,OAAO,kBAAkB,OAAO;EAChC,YAAY;EACZ,mBAAmB;EACnB,eAAe;EACf;EACA;;iBAKc,oBAAoB,UAAS,sBAA2B;iBAYxD,iBAAiB,eAAe;iBA8B1B,iBAAiB,OAAO,mBAC5C,QAAQ,oBAAoB,OAAO,WAClC,QAAQ,oBAAoB,OAAO;KAiNjC,eAAe;EAAO;EAAgB;MAAmB;UAE7C,kBAAkB,OAAO;EACxC,UAAU;EACV,eAAe,OAAO;EACtB,sBAAsB,SAAS;;EAE/B;;iBAQc,kBAAkB,OAAO,mBACvC,KAAK,kBAAkB,OAAO,WAC7B,SAAS,OAAO;UCxZF;;EAEf;;EAEA;;EAEA;;;;;EAKA,eAAe;IAAQ;IAAY;IAAgB;;;EAEnD;;;;;EAKA;;EAEA,UAAU;;UAGK;;EAEf;;EAEA;;EAEA,WAAW;;EAEX,cAAc;;EAEd;;EAEA;;;;;;;iBAiBc,sBAAsB,KAAK;UAmG1B;EACf;EACA;EACA;;;;;;iBAOc,wBAAwB,aAAa,wBAAwB;;;;;;UCxJ5D;EACf,OAAO,QAAQ,YAAY;EAC3B,QAAQ,QAAQ;;;;;;;iBAUF,sBACd,cACA;EAAQ;IACP;UCJO;;EAER;;EAEA;;;;;;;EAOA;;EAEA;;EAEA;;;;;EAKA;;;;;EAKA,cAAc;;;KAIJ;WACG;;WAEA;WACA;;WAEA;;;;;;;;KASH,wBAAwB,OAAO,uBAAuB,QAAQ;UAEhE;;EAER,SAAS;;EAET;;EAEA;;;;;;EAMA;;UAGQ;EACR;EACA,OAAO;EACP;;;;;;;;;;;;iBAmCoB,gBAAgB,MAAM,yBAAyB,QAAQ;;;;;;;;;;;;;;;;;;;;;;;;;;UCrH5D;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;KAGG,iBAAiB;;;;;;;;;;;;;;;;;;;;;;;;cAyBT,aAAa,+BAA+B;;;;;;;;iBA4BzC,YAAY,OAAO,YAAY,gBAAgB;iBAC/C,YACd,OAAO,YACP,MAAM,QAAQ,qBACd;iBAEc,YAAY,OAAO,YAAY,MAAM,UAAU;;;;;;;;;;;;UCxF9C;;EAEf;;EAEA;;EAEA;;EAEA;;cAGW;mBACM;;;;;;EAOX,SAAS,YAAY,iBAAiB,kBAAkB,QAAQ;;EAqBtE,IAAI,YAAY,kBAAkB;;EAQlC,aAAa,aAAa;;EAO1B,QAAQ;;EAKR,WAAW,YAAY,iBAAiB;;;iBAQpB,YAAY,kBAAkB;;;;;;;;;KCvExC;UAGF;EACR;EACA;EACA;EACA;EACA;EACA;;UAGQ;EACR;EACA;EACA;EACA;EACA;EACA,WAAW;;UAiBH;EACR;EACA;EACA;EACA;EACA;EACA;EACA;;UAGQ;EACR;EACA,OAAO;EACP;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA,SAAS;;UAGD;EACR;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;UAGQ;EACR,WAAW;EACX,WAAW;EACX,SAAS,QAAQ,OAAO,sBAAsB;;UAe/B,uBAAuB;EACtC;EACA,OAAO;EACP,OAAO;EACP,WAAW;EACX,YAAY;EACZ,YAAY;EACZ,OAAO;EACP;EACA;;UAGe,mBAAmB;EAClC;EACA;EACA;EACA;EACA;EACA,OAAO,uBAAuB;EAC9B,OAAO;EACP,WAAW;;UCtHI;EACf;EACA;;UAGe;EACf;EACA;EACA;EACA,kBAAkB;EAClB;EACA,cAAc,eAAe;EAC7B,WAAW;;UCZI;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;KAGU,kBAAkB,KAAK;iBAgBnB,kBAAkB,OAAO,UAAU,UAAS,QAAQ;iBAapD,QAAQ;KCzCnB;UAEY;EACf;EACA;EACA,QAAQ;EACR,OAAO;EACP,WAAW;;UAGH;EACR,SAAS;EACT;EACA;EACA;;UAGe;EACf,SAAS;EACT;EACA;EACA,UAAU;IAAQ;IAAmB;IAAc;;EACnD,WAAW;EACX;;UAGQ;EACR,UAAU,QAAQ;;;;;;;;cASP;EACX,SACE,MAAM,2BACN,SAAQ,0BACP;;KCvBO;EACN;EAAoB;EAAY;;EAChC;EAA0B;EAAY;;EACtC;EAAwB;;EACxB;EAA+B;EAAY;;EAE3C;EACA;EACA;EACA,QAAQ,MAAM,mBAAmB;;UAGtB;EACf;EACA,oBAAoB;;;EAGpB,QAAQ;EACR,UAAU;;;;EAIV,aAAa;;UAGE;EACf;EACA;EACA,UAAU;;EAEV;IACE;IACA;IACA;;;UAIa;;;;;;EAMf,cAAc,KAAK,uBAAuB,SAAS,iBAAiB;;;UClCrD;EACf;;EAEA;;EAEA;;EAEA;;EAEA,eAAe;;EAEf;;;UAIe;EACf;EACA;;EAEA;EACA,UAAU;;;UAIK;EACf,OAAO;;EAEP,UAAU,eAAe;;;UAIV;EACf;EACA;EACA;EACA,SAAS;EACT,OAAO;;UAKQ;;EAEf,SAAS;EACT;;EAEA;;;;;;;;iBAiCc,WAAW,MAAM,aAAa,MAAM,oBAAoB;;iBAmDxD,sBACd,iBACA,MAAM,aACN,MAAM,oBACL;;;;;;iBAWa,cAAc,kBAAkB;KAsCpC;UAEK;EACf;EACA;EACA;EACA,SAAS;;EAET;;EAEA;;EAEA;;EAEA;;EAEA;EACA;EACA;;UAGe;EACf,OAAO;EACP;IAAW;IAAkB;IAAmB;IAAc;;;UAG/C;;EAEf;;EAEA;;EAEA;;;;;EAKA;;;;;;;;iBASc,cACd,WAAW,WACX,OAAM,uBACL;;;;;;iBAiFa,oBAAoB,MAAM;;;;;UC5UzB;EACf;EACA;EACA;;EAEA;EACA;EACA;EACA;EACA;;;EAGA;;EAEA;EACA;EACA;;EAEA;;EAEA;EACA,aAAa;;;;;;;UAQE;EACf;EACA;EACA;;;UAIe;EACf;EACA;;;UAIe;EACf;;KAGU,uBAAuB,SAAS,kBAAkB;;;;;;;;;;;;;;;;;UAkB7C;EACf,OAAO;EACP,OAAO;EACP,QAAQ;EACR,OAAO;EACP,OAAO;EACP,MAAM;EACN,UAAU,MAAM;;;;EAIhB;;cAKW;KACD,oBAAoB;cAInB;KACD,6BAA6B;cAE5B;KACD,uBAAuB;KAEvB;EACN;;EACA;EAAc;;EACd;EAAa;;EACb;EAAY,MAAM;;;UAGP;EACf;;;EAGA;;UAGe;;;EAGf;;EAEA,kBAAkB;;EAElB;;KAGU;EACN;EAAyC;EAAe,GAAG;;EAC3D;EAAkB;EAAe,GAAG;EAAe,GAAG;EAAe,QAAQ;;EAE7E;EACA;EACA,GAAG;EACH,OAAO;EACP,QAAQ;;EAER;EAAgB;EAAe,GAAG;EAAe;;EACjD;EAAsB;EAAe,GAAG;EAAe;;EACtD;EAAa;IAAkB;EAEhC;EACA;EACA,GAAG;;EAEH;EACA,OAAO;;EAEP,aAAa;;;;;;;;;EAUb;EACA;EACA;;;;;;;;EASA;EACA;EACA;EACA,SAAS;;;;UAKE;EACf;EACA,OAAO;;;UAIQ;EACf;;EAEA;;;EAGA;IAAU,MAAM;;EAChB,OAAO;EACP,eAAe;;KAGL;UAEK;EACf;EACA;EACA;;;UAIe;EACf;;EAEA;EACA,QAAQ;EACR;EACA;;UAGe,wBAAwB;;EAEvC;EACA,QAAQ;;EAER;;EAEA;;;EAGA,QAAQ;EACR,YAAY;EACZ,gBAAgB;;EAEhB;;UAGe;EACf,UAAU;;EAEV,QAAQ;EACR;;UAGe;;EAEf,MAAM;;UC1NE;EACR,QAAQ;;UAGA;EACR;EACA,OAAO;EACP,aAAa;;cAGT;mBAKyB;mBAJZ;mBACA;UACT;EAEqB,YAAA;;EAG7B,OAAO,GAAG,eAAe;;EAMzB,MAAM,GAAG,eAAe;;EAMxB,WAAW,GAAG,eAAe;;;;EAY7B,SAAS,GAAG,eAAe,GAAG,eAAe,gBAAgB,UAAS;;EActE,YACE,GAAG,eACH,OAAO,eACP,gBACA,UAAS;;EAeX,OAAO,GAAG,eAAe,aAAa;;EAYtC,aAAa,GAAG,eAAe,aAAa;;EAY5C,IAAI,QAAQ,WAAW;;EAOvB,SAAS,GAAG,eAAe,SAAS,qBAAqB;;;EAkBzD,aAAa,qBAAqB;;EAWlC,UAAU;IAAW;IAAkB,SAAS;KAAgB;;EAYhE,MAAM,MAAM;;EAMZ,YAAY,YAAY,SAAS,QAAQ,yBAAyB;EAWlE,SAAS;UAWD;UAIA;;iBAcM,cAAc,eAAe;;;;;;;;iBCxH7B,sBACd,UAAU,eACV,gBAAgB,gBAChB,UAAS,+BACR;;;;;iBAiJa,oBACd,gBAAgB,gBAChB,oBAAoB,iBACpB,UAAS,+BACR;;iBCvJa,qBAAqB,UAAU;UCtCrC;EACR;;EAEA;;EAEA,aAAa;;EAEb;EACA;EACA;EACA,OAAO;;UAGC;;EAER;;EAEA;;EAEA;;EAEA;;EAEA,kBAAkB;;EAElB;;EAEA,YAAY;EACZ,YAAY;;;;EAIZ;;UAGQ;;EAER;;;EAGA,SAAS;;UAGD;EACR;;EAEA;;EAEA;;;UAIQ;EACR,MAAM;EACN,QAAQ;EACR,MAAM;EACN,UAAU;;EAEV,QAAQ;;UAGA,wBAAwB;EAChC;;UAGe,0BAA0B;EACzC;EACA;;;;EAIA;IAAU,MAAM;;;EAEhB;IAAiB,OAAO;;;;;;;;iBAoDV,yBAAyB,iBAAiB;UAkYzC;;;EAGf;EACA;EACA;EACA;;;;;;iBAOc,sBAAsB,iBAAiB;UCjkBtC;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA;EACA;EACA,QAAQ;;iBAGM,qBACd,QAAQ,oBACR,SAAQ,uBACR;EAAW;IACV;UCNc;EACf;EACA;EACA;EACA;;iBAGc,cAAc,gBAAgB,WAAW,oBAAoB;KCLxE;UAEY;EACf;;EAEA;;EAEA;;EAEA;;EAEA;;UAGe;;EAEf,OAAO,QAAQ;;UAGA;;EAEf,cAAc,eAAe,QAAQ;;EAErC,SAAS;;EAET;;EAEA;;EAEA;;EAEA;;EAEA;;;;;;;;;;iBAac,gBAAgB,eAAe,OAAO,uBAAuB,MAAM;UA4FlE;;EAEf;;;;;EAKA,OACE,aACA;IAAS;IAAc;QACpB;IAAU;IAAgB;IAAgB;;;;;;EAK/C,SAAS,mCAAmC;;EAE5C;;EAEA;;EAEA;;EAEA;;;;;;;iBAQc,iBAAiB,OAAO,wBAAwB;UA+B/C;EACf;EACA,OACE,aACA;IAAS;IAAc;QACpB;IAAU;IAAgB;IAAgB;;EAC/C,SAAS,mCAAmC;;EAE5C;;EAEA;;EAEA;;;;;;;;;;;;iBAac,qBAAqB,OAAO,4BAA4B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;KC7L5D;;;;;;;UAmBK;;EAEf;;;;EAIA;EACA,aAAa;;;UAME;EACf;EACA,aAAa;;UAGE;;;;EAIf;;;;;;;;;;;;;iBAcc,mBACd,iBAAiB,eACjB,sBACA,OAAM,yBACI;;;;UAyBK;;EAEf;;EAEA;;EAEA;;EAEA;;;;;;;;;;;;;KAcU,qBAAqB,WAAW,YAC1C,UAAU,WACV,gBAAgB,SAChB,SAAS,gBACN,QAAQ,qBAAqB;;;UAUjB;;EAEf;;EAEA;;;;;UAUe,gBAAgB;EAC/B,UAAU;;GAET;;;;;;;;;;iBAuBa,aACd,yBACA,oBACA,UAAS;;;;;UCxMM;;;EAGf;;EAEA;;EAEA;;;EAGA;;;EAGA;;UAGe;;;;;EAKf;EACA;EACA;;KAGG;UAEY;EACf,OAAO;EACP;EACA;;UAGe;;EAEf;;EAEA,aAAa,cAAc;;;;;;;iBAsBb,oBACd,OAAO,mBACP,OAAO,mBACP,OAAM,6BACL;UCjDc;;EAEf;;EAEA,aAAa;IAAQ;IAAc;;;EAEnC;;EAEA;EACA;;UAGe;EACf;EACA;;EAEA;;EAEA;EACA;EACA;EACA;EACA;;UAGe;;EAEf,MAAM;EACN;EACA;EACA;EACA;EACA;EACA;;EAEA,UAAU;EACV,aAAa;EACb;EACA,WAAW;EACX,SAAS;;;;;iBA0EW,oBACpB,OAAO,kBACP,SAAS,qBACR,QAAQ;;;UCpIM,aAAa;;EAE5B;;EAEA,cAAc,OAAO;;EAErB;;EAEA;;EAEA,QAAQ;;;EAGR,SAAS,QAAQ,OAAO;IAAK;IAAoB;;;;UAIlC,kBAAkB;;EAEjC,cAAc,OAAO;;EAErB;;EAEA,UAAU,eAAe,OAAO;;EAEhC;;EAEA;;EAEA;;;EAGA,UAAU,aAAa;;;;;;;;;;;;;iBAcT,uBAAuB,kBACrC,mBAAmB,aAAa,MAChC,wBAAwB,KACxB,UAAU,QAAQ,OAAO,cACxB,kBAAkB;;;;;;;;;;;;;UCxDJ;;EAEf;;EAEA;;;;;;;;;EASA;;EAEA,aAAa;;;;;;;;;EASb,eAAe;;;UAIA,kBAAkB;;EAEjC,OAAO;;EAEP;;EAEA;;EAEA;;EAEA,QAAQ;;EAER,eAAe;IAAQ;IAAiB;IAAe;;;;;;;;;;;;;;iBAqBnC,eAAe,GACnC,UAAU,eAAe,QAAQ,gBAAgB,QAAQ,IACzD,SAAQ,mBACP,QAAQ,kBAAkB;UC/CZ,qBAAqB;;EAEpC;;EAEA,YAAY;;;EAGZ;;EAEA;;;;;;EAMA,YACE,eACA;IAAS;IAAmB;IAAoB,QAAQ;QACrD,QAAQ,aAAa;;EAE1B,oBAAoB,OAAO,OAAO,kBAAkB;;EAEpD,aAAa;;EAEb,gBAAgB,kBAAkB,kBAAkB;;;;;;EAMpD,QAAQ;;;;;;;;;EASR;;;EAGA,UAAU,QAAQ,OAAO;;;;;;;;;;;;iBAaX,cAAc,kBAC5B,MAAM,qBAAqB,KAC1B;;;;;;;;;;;;;;;;;;;;;;UCjEc;EACf;EACA;;;;;;;;;;EAUA;;UAGe;EACf;EACA;EACA;;EAEA;;UAGe;;EAEf;EACA;EACA;EACA,UAAU;EACV;;EAEA;;EAEA;;UAGe;;EAEf,eAAe;;EAEf;;EAEA;;;;;;;;iBAqEc,wBACd,cACA,kBAAkB,cAAc,qBAChC,SAAQ,wBACP;;;;;;iBAyCmB,2BACpB,oBACA,kBAAkB,cAAc,qBAChC,UAAS,yBACR,QAAQ;UCrKM;;EAEf;;EAEA;;EAEA;;EAEA,UAAU;;UAGK;EACf;EACA;EACA;EACA;;UAGe;EACf;EACA,MAAM,KAAK,oBAAoB;;iBAOjB,UAAU,UAAU,0BAA0B;;;;UAqC7C,qBAAqB;EACpC,SAAS;EACT;EACA;;;iBAIc,gBAAgB,KAAK,mBAAmB,SAAS,WAAW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;KC7CvE,eAAe;UAEH;;;EAGf,eAAe,SAAS;;EAExB,SAAS;;UAGM;;;;;EAKf;;UAGe;;;EAGf;;;;EAIA;EACA;;EAEA;;EAEA;;;;;;iBAOc,qBACd,OAAO,oBACP,OAAM,uBACL;;;;;;;iBC5Ca,cAAc;;;;iBAOd,YAAY;;;UAQX;EACf,IAAI,cAAc,QAAQ,0BAA0B;EACpD,IAAI,aAAa,OAAO,aAAa;;;;;;;;;iBA2CvB,iBAAiB,eAAe;;;;;;;;;;;;;;;;;;;;;;;;cC7EnC;;;cAaA;;cAYA;;cAYA;;;;;KAMD,+BAA+B;;;;;;;UClC1B;EACf;EACA;EACA;EACA;;;;;UAMe;EACf;EACA;EACA;EACA;;;;UAKe;EACf;EACA;EACA;EACA,QAAQ;EACR,QAAQ;;;;;;;;;iBAqBM,cAAc,GAAG;;;iBAmBjB,iBAAiB,GAAG;;;;;;iBAuCpB,gBAAgB;EAC9B;EACA;EACA;EACA,SAAS;IACP;;;;;;;;iBAiBY,wBACd,MAAM,iBACN;EAAQ;EAAsB;EAA0B,SAAS;IAChE;;;;;iBAQa,YACd,UAAU,eACV,SAAS,wBACR;;;;;;;;iBAWa,iBACd,GAAG,eACH,mBACA,kBACC;;;;;;;;;;iBA8Ba,YAAY,SAAS;cC3MxB,0BAA0B;EAE5B,iBAAiB;EACjB;EACA;EAFA,YAAA,iBAAiB,YACjB,eACA;;cAME;UACH;UACA;UACA;UACA;EAEI,YAAA,SAAS,cAAc,QAAQ,YAAY;;EAOjD,OAAO,OAAO,QAAQ,aAAa,sBAAsB,kBAAkB;;EA2B3E,SAAS,eAAe,kBAAkB;MAO5C,SAAS,aAAa;;;;;;;;;;;cCzDf;UACH;UACA;UACA;UACA;EAEI,YAAA,QAAQ;UAUZ;;;;;;;;UAaM;;;;;;UAkCN;EAUF,OAAO,cAAc,eAAe,mBAAmB;IAAU;;EAOjE,MAAM,eAAe,mBAAmB;EAexC,gBAAgB,cAAc,gBAAmB;EAOjD,aAAa,sBAAsB;EAOnC,KACJ,qBACA,kBACA,iBACA;IAAU,gBAAgB;MACzB;IAAU;IAAc;MAAU;MAAc;;;EAuD7C,SACJ,sBACC;IAAU;IAAY;IAAe;IAAgB;;EAQlD,UAAU,sBAAsB;IAAU;IAAY;IAAe;;EAKrE,aACJ,sBACC;IAAU;IAAY;IAAe;IAAgB;;EAQlD,aAAa,sBAAsB;EAcnC,cAAc,qBAAqB,aAAa;EAIhD,aAAa,qBAAqB,YAAY,iBAAiB;EAI/D,eACJ,sBACC;IAAU;IAAY;IAAc;;;EAMjC,IAAI,eAAe,QAAQ;;EAK3B,KAAK,cAAc,MAAM,0BAA0B,QAAQ;;EAK3D,MAAM,cAAc,MAAM,0BAA0B,QAAQ;;iBC1K9C,oBACpB,QACA,SACA,eACA,cAAc,oBAAoB,mBAElC,QAAQ,qBAAqB,QAAQ,SAAS,eAAe,SAC5D,QAAQ,iBAAiB,QAAQ,SAAS,eAAe;iBC7E5C,cAAc,OAAO,KAAK,kCAAkC;iBAI5D,eAAe,OAAO,KAAK,kCAAkC;UCEnE;EACR;EACA,cAAc;EACd;EACA;EACA;EACA,cAAc;EACd,WAAW;EACX,MAAM;;iBAGQ,wBACd,SAAS,iCACR;iBA+Ca,sBACd,QAAQ,0BACR;EAAW;EAAa;EAAuB,UAAU;IACxD;iBAuBa,+BACd,QAAQ,0BACR;EAAW;EAAkB;IAC5B;iBAea,8BAA8B,MAAM,yBAAyB;iBAc7D,iCACd,MAAM,yBACL;UCtIO;EACR;EACA;;;cAIW,eAAe,eAAe;;;;iBA0D3B,oBAAoB,gBAAgB;;iBAapC,cAAc;;iBAOd,eAAe;;;iBAMf,aAAa,qBAAqB,sBAAsB;UCxE9D;;EAER;;EAGA;;EAGA,iBAAiB;;EAGjB,kBAAkB;;EAGlB,kBAAkB;;EAGlB;;EAGA;;EAGA;;EAGA,iBAAiB,QAAQ,OAAO;;KAG7B;;iBAsBW,oBAAoB,SAAQ,iBAAsB;UA4CxD;EACR,UAAU;EACV;EACA;;UAGQ;;EAER;EACA,QAAQ;;EAER,QAAQ,OAAO;;;;;;iBAOD,gBACd,mBACA,QAAQ,KAAK,SAAS;EACpB,gBAAgB,OAAO;IAExB;iBC5Ha,aAAa,cAAc,KAAK,WAAW;iBAS3C,aACd,cACA,gBACA,8BACC;iBAUa,aAAa,cAAc,iBAAiB;iBAa5C,mBACd,cACA,kBACA,mBACC;iBAaa,eAAe,cAAc,gBAAgB,mBAAmB;iBAShE,kBACd,cACA,gBACA,oBACC;iBAaa,iBACd,cACA,eACA,aACA,eACC;iBAca,eACd,cACA,KAAK,yBACL,gBACC"}
1	+ {"version":3,"file":"index-C8a9Dyan.d.ts","names":[],"sources":["../src/cost-ledger/errors.ts","../src/cost-ledger/pricing.ts","../src/llm-client/accounting.ts","../src/llm-client/errors.ts","../src/llm-client/json.ts","../src/llm-client/resilience.ts","../src/analyst/policy-edit.ts","../src/capability-headroom.ts","../src/cost-tracker.ts","../src/detectors/index.ts","../src/error-count-extractor.ts","../src/knowledge/types.ts","../src/leaderboard.ts","../src/paired-delta-test.ts","../src/partition-held-out.ts","../src/product-benchmark/export.ts","../src/product-benchmark/index.ts","../src/runtime-trajectory.ts","../src/active-learning.ts","../src/bounded-process.ts","../src/command-runner.ts","../src/discover-personas.ts","../src/improvement-verdict.ts","../src/propose-review.ts","../src/reflective-mutation.ts","../src/eval-trace-store.ts","../src/integrity/preflight.ts","../src/model-seats.ts","../src/prompt-registry.ts","../src/reference-replay.ts","../src/steering.ts","../src/run-score.ts","../src/steering-optimizer.ts","../src/counterfactual.ts","../src/scorecard.ts","../src/trace-contracts/types.ts","../src/trace-contracts/builder.ts","../src/trace-contracts/evaluate.ts","../src/trace-contracts/explain.ts","../src/trace-contracts/spec.ts","../src/action-policy.ts","../src/contamination-guard.ts","../src/deploy-gate-layer.ts","../src/hidden-criteria-grading.ts","../src/integrity/single-backend.ts","../src/intent-match-judge.ts","../src/judge-ensemble.ts","../src/judge-retry.ts","../src/judge-panel.ts","../src/keyword-coverage-judge.ts","../src/oracle.ts","../src/treatment-gate.ts","../src/verdict-cache.ts","../src/profile/baselines.ts","../src/profile/index.ts","../src/budget-guard.ts","../src/client.ts","../src/control-runtime.ts","../src/control-runtime/policy.ts","../src/knowledge/readiness.ts","../src/metrics.ts","../src/anti-slop.ts","../src/sandbox-assertions.ts"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;cAMa,gCAAgC;EAEzC,YAAA,oBACA,sBACA,sBACA,eACA;;cAQS,sCAAsC;UCblC;EACf;EACA;;iBAGc,aAAa,eAAe,OAAO,YAAY;;iBAc/C,oBACd,SAAS,oBACT,OAAO,KAAK;iBCtBE,2BACd,SAAS,KACP,0GAGF,UAAS,kBACR;;;;;;iBAoCa,mBACd,QAAQ,eACR,qBAAqB,qBACpB;;iBA6Ba,wBACd,OAAO,OACP,qBAAqB,qBACpB;cChFU,qBAAqB;WAGd;WACA;WACA;EAHhB,YAAA,iBACgB,gBACA,cACA;;;;;cASP,yBAAyB;WAGlB,QAAQ;EADxB,YAAA,iBACgB,QAAQ,eACxB;IAAY;;;;;;;;iBCXA,gBAAgB;;;;;;;;;;;;iBCgFhB,oBAAoB;KClFxB;cAEN;KAaM,yBAAyB;cAE/B;KAUM,kCAAkC;KAClC;KACA;KACA;UAEK;EACf,SAAS;;;EAGT;;;EAGA,mBAAmB;;EAEnB;;KAGU;EAEN;EACA;EACA;;EAEA;;EAGA;EACA;EACA;EACA,QAAQ;;UAGG;;EAEf;EACA,WAAW;;EAEX;EACA,OAAO;EACP;;UAGe;EACf;EACA;EACA,cAAc;;EAEd;;UAGe;EACf,eAAe;EACf;EACA,MAAM;EACN,QAAQ;EACR,QAAQ;EACR;EACA,cAAc;EACd;EACA,MAAM;EACN,QAAQ;EACR;EACA;EACA,WAAW;;cAGP;;UAGW;EACf,eAAe;EACf,YAAY;;KAGF,iBAAiB,KAAK;EAChC,gBAAgB;EAChB;;UAWe;EACf,eACI,2BACE,SAAS,mBAAmB;EAClC,OAAO,mBAAmB,SAAS,mBAAmB;EACtD,cAAc;EACd,uBAAuB;;UAGR;EACf;EACA;EACA;EACA;;UAGe;EACf,MAAM;EACN;EACA;EACA;;iBAgDc,8BAA8B,MAAM,aAAa;;;;;;;;;;;;;;;;;;;;;;;UCrKhD;;EAEf;;;EAGA;;;;;;KAOG;UAEY;EACf;;EAEA;;;;EAIA;;;EAGA;EACA,UAAU;;UAGK;;;;EAIf;;UAGe;;EAEf,OAAO;EACP;IACE;IACA;IACA;;;;IAIA;;;;;;;;;;;;;;;;iBAiBY,mBACd,eAAe,iBACf,OAAM,4BACL;UAiEc;;EAEf;;;;;;;;iBASc,yBACd,QAAQ,0BACR,OAAM;;;;;;;;;;;UC3JE;EACR;EACA;EACA;EACA;;UAGe,kBAAkB;EACjC;EACA;;EAEA;EACA;;EAEA,OAAO;;UAGQ;EACf;EACA,SAAS;EACT;EACA;EACA;EACA;;EAEA;;cAGW;UACH;EAER,OAAO,OAAO,KAAK;IAA4B;MAAuB;EAwBtE,YAAY,oBAAoB;;;;;;;;;;;EAgBhC,cACE;IACE;MACE;MACA;MACA;MACA;MACA;;IAEF;KAEF,oBACA,OAAO,yBACN;EAeH,IAAI,qBAAqB;EAIzB,QAAQ;EAIR,WAAW;;UAmBI;EACf;EACA;EACA;EACA;EACA;EACA;;EAEA;;KCpIG;;;;UAKY;;WAEN;;WAEA;;WAEA;;WAEA;;WAEA;;UAGM;WACN;WACA,UAAU;WACV,eAAe;WACf;;WAEA;WACA,WAAW;;UAGL;WACN;;WAEA;;EAET,QAAQ,OAAO,gBAAgB;;EAE/B;;UAGe;;;WAGN;WACA,WAAW;;WAEX,eAAe;;;iBAIV,uBAAuB,OAAM,wBAA6B;UAkFzD;;WAEN;WACA,WAAW;WACX,eAAe;;;iBAIV,oBAAoB,OAAM,qBAA0B;;;iBA+BpD,WACd,WAAW,cAAc,oBACzB,OAAO,gBACN;;;;;;;;;;;;;;;;UC3Kc;;EAEf;;EAEA,OAAO;;EAEP,aAAa,OAAO;;cAGT,sBAAsB;UAoClB;;EAEf;;EAEA,QAAQ;;UAGO;;EAEf;;EAEA;;EAEA;;;;;;;;;iBAUc,kBAAkB,cAAc,OAAM,iBAAsB;KClFhE;KAaA;KASA;KACA;KACA;KACA;UAEK;EACf;EACA;EACA;EACA,UAAU;EACV,iBAAiB;EACjB,YAAY;EACZ,WAAW;EACX,aAAa;EACb;EACA;EACA;EACA,gBAAgB;;;;;EAKhB;;EAEA;EACA,WAAW;;UAGI;EACf;EACA,cAAc;EACd;EACA;EACA;EACA,aAAa;EACb,SAAS;EACT;EACA,WAAW;;KAGD;UAUK;EACf;EACA;EACA,6BAA6B;EAC7B,iBAAiB;EACjB,mBAAmB;EACnB,QAAQ;EACR,UAAU;EACV;;UAGe;EACf;EACA;EACA;EACA;EACA,YAAY;EACZ;EACA;EACA;EACA;EACA,WAAW;;UAGI;EACf;EACA;EACA,MACI,QAAQ;EAEZ;EACA,UAAU;EACV;EACA,YAAY;EACZ,WAAW;;UC9FI;;EAEf;;EAEA;EACA;EACA;EACA;;EAEA;EACA;EACA;;EAEA;EACA;EACA;EACA;;UAGe;;;EAGf,SAAS,QAAQ;;;EAGjB,WAAW,QAAQ;;;;;;;iBAYL,YAAY,SAAS,aAAa,MAAM,qBAAqB;UC5C5D,+BAA+B;;EAE9C;;EAEA;;UAGe;EACf,WAAW;EACX;;EAEA;;EAEA;EACA;;;;;;EAMA;EACA;;;iBAIc,+BAA+B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;iBA6C/B,gBACd,kBACA,iBACA,UAAS,yBACR;;;;;;;;;;;;;;;;;;UCrCc;;;;EAIf;;EAEA;;;;;;;EAOA;;EAEA;;UAGe;;EAEf;;EAEA;EACA;EACA;;;;;;;;;;;;iBAac,iBACd,eACA,UAAS,0BACR;UAmDc,0BAA0B;;EAEzC;;EAEA;;EAEA;;;;;;EAMA,UAAU,MAAM;;EAEhB;EACA;EACA;;UAGe,4BAA4B;;EAE3C,OAAO;;EAEP,WAAW;;EAEX,MAAM;EACN;EACA;EACA;;UCxHe;;WAEN;;WAEA;WACA;WACA;;WAEA;;WAEA;;;;;WAKA;;WAEA;;WAEA;;;;;;WAMA;;;;;WAKA,iBAAiB,QAAQ,cAAc;;WAEvC,oBAAoB,QAAQ,WAAW;;WAEvC;;;;;;;WAOA,YAAY,QAAQ;;UAGd,+CACP,KAAK;WACJ;;UAGM;WACN;WACA;WACA;;;iBA6DK,gCAAgC;;iBA4ehC,uBACd,SAAS,yCACR;;;;;;iBAUa,2BACd,SAAS,gCACR;cCjpBG;KAEM,gCAAgC;UAE3B;WACN;WACA;WACA;;UAGM;WACN;WACA;WACA;WACA;WACA;;UAGM;WACN;WACA;WACA;;UAGM;WACN;WACA;WACA;WACA,YAAY;;UAGN;WACN;WACA,OAAO;WACP;WACA;;UAGM;WACN;WACA;WACA;;UAGM;WACN;WACA;WACA;WACA,MAAM;WACN,WAAW;WACX,mBAAmB;WACnB,eAAe;WACf,oBAAoB;WACpB,SAAS;WACT;;UA0EM;WACN;WACA;WACA;;WAEA;;;WAGA;WACA;WACA;WACA;WACA;WACA;WACA;WACA;WACA;WACA;WACA;WACA;WACA;;iBA0TK,6BAA6B,eAAe;;;;;;iBAkH5C,0BAA0B,iBAAiB;UClkB1C;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA,WAAW;;UAGI;EACf;EACA;EACA,WAAW;EACX;GACC;;UAGc;EACf;EACA;EACA,UAAU;;UAGK;EACf;EACA;EACA;EACA;EACA;;UAGe;EACf,MAAM;EACN,QAAQ;EACR,SAAS;EACT;;UAGe,wCACf,gBAAgB,0BAA0B;EAE1C,SAAS;EACT,kBAAkB;EAClB,cAAc,QAAQ,SAAS;EAC/B,gBAAgB,QAAQ,SAAS;;iBAKnB,iCAAiC,gBAAgB,yBAC/D,SAAS,wCAAwC,WAChD;KClCE;UAEY;EACf,QAAQ;EACR;;;EAGA,WAAW;;EAEX;;EAEA;;UAGe;;EAEf;;EAEA;;EAEA;;iBAGoB,wBACpB,SAAS,SACT,YAAY,YACZ,UAAS,wBACR,QAAQ;;;;;;;;;;;;;;;;;;;;;;;;;;;;cCpBE;UAwBI;;;;;;EAMf;;;;;;;;;;;;;;;;;;;;;;;EAuBA;;;;;;;;;;;;;;;;;;;;;;;;;;;EA2BA;;;;;;;;;EASA;;EAEA;;;;;EAKA,MAAM,OAAO;;;;;;;;;;;;;;;;;;;EAmBb;;EAEA;;;;;;EAMA;;;;;;EAMA,SAAS;;UAGM;;;;;;;;;;;EAWf;;;;;;;;;EASA;;;;;;EAMA;;EAEA;;;;;;EAMA;;EAEA;;EAEA;;EAEA;;EAEA;;;;;;;;;;;;;;;;EAgBA;;iBAGoB,kBAAkB,OAAO,sBAAsB,QAAQ;;;;;;;;;;;;;;;;;;UCjM5D;;EAEf;;EAEA;;EAEA;;;;;;;;;;;EAWA;;EAEA,MAAM;;EAEN;;UAGe;;EAEf;EACA;EACA;EACA;EACA;;EAEA;;UAGe;EACf;EACA;EACA;;EAEA;;UAGe;;WAEN;;EAET,IAAI,OAAO,kBAAkB,QAAQ;;EAErC,OAAO,eAAe;;EAEtB,WAAW,eAAe;;EAE1B,SAAS,eAAe;;EAExB,QAAQ,eAAe,QAAQ;;cAkBpB,oBAAoB;;;;;;;UC9FhB;;;;;EAKf,UAAU;;;;;EAKV;;;;;EAKA;;EAEA;;UAGe;;EAEf;;EAEA;;EAEA;;iBAKoB,iBACpB,aACA,OAAM,0BACL,QAAQ;;;KClBN;;;UAIY;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA,WAAW;;EAEX;;EAEA,UAAU;;UAGK;EACf;EACA;EACA;EACA;;EAEA;;EAEA;;;EAGA;;EAEA;;EAEA;;UAGe;;EAEf;;EAEA;;EAEA;;EAEA;;;EAGA;;UAGe;EACf,SAAS;;EAET;;EAEA;;UCtCe;EACf;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA;EACA;EACA;;UAGe,0BAA0B;EACzC;EACA;EACA;IACE;IACA;IACA;;;UAIa,aAAa;EAC5B;EACA;EACA,OAAO;EACP,aAAa;EACb,aAAa;EACb,UAAU;;UAGK,cAAc,OAAO;EACpC,OAAO;EACP,eAAe;;UAGA,YAAY,OAAO;EAClC;EACA;EACA,OAAO;EACP,cAAc;EACd,cAAc;EACd,QAAQ;;KAGE,UAAU,OAAO,sBAC3B,OAAO,aAAa,WACjB,QAAQ,cAAc,OAAO;KAEtB,SAAS,UAAU,OAAO,UAAU,QAAQ;KAE5C,SAAS,OAAO,sBAC1B,OAAO,YAAY,OAAO,aACvB,QAAQ;UAEI;EACf,QAAQ,QAAQ;EAChB,OAAO,OAAO,oBAAoB;;UAGnB,oBAAoB,OAAO;EAC1C;EACA,cAAc;EACd,SAAS,UAAU,OAAO;EAC1B,QAAQ,SAAS;EACjB,QAAQ,SAAS,OAAO;;EAExB;;EAEA;;;;;;EAMA;EACA;;EAEA,SAAS;;EAET,QAAQ;EACR;EACA;EACA;;;;;EAKA;;UAGQ,kBAAkB,OAAO;EACjC;EACA,OAAO;EACP,cAAc;EACd,cAAc;EACd,QAAQ;EACR;EACA;EACA;;UAGe,oBAAoB,OAAO;EAC1C;EACA;EACA,OAAO,kBAAkB,OAAO;EAChC,YAAY;EACZ,mBAAmB;EACnB,eAAe;EACf;EACA;;iBAKc,oBAAoB,UAAS,sBAA2B;iBAYxD,iBAAiB,eAAe;iBA8B1B,iBAAiB,OAAO,mBAC5C,QAAQ,oBAAoB,OAAO,WAClC,QAAQ,oBAAoB,OAAO;KAiNjC,eAAe;EAAO;EAAgB;MAAmB;UAE7C,kBAAkB,OAAO;EACxC,UAAU;EACV,eAAe,OAAO;EACtB,sBAAsB,SAAS;;EAE/B;;iBAQc,kBAAkB,OAAO,mBACvC,KAAK,kBAAkB,OAAO,WAC7B,SAAS,OAAO;UCxZF;;EAEf;;EAEA;;EAEA;;;;;EAKA,eAAe;IAAQ;IAAY;IAAgB;;;EAEnD;;;;;EAKA;;EAEA,UAAU;;UAGK;;EAEf;;EAEA;;EAEA,WAAW;;EAEX,cAAc;;EAEd;;EAEA;;;;;;;iBAiBc,sBAAsB,KAAK;UAmG1B;EACf;EACA;EACA;;;;;;iBAOc,wBAAwB,aAAa,wBAAwB;;;;;;UCxJ5D;EACf,OAAO,QAAQ,YAAY;EAC3B,QAAQ,QAAQ;;;;;;;iBAUF,sBACd,cACA;EAAQ;IACP;UCJO;;EAER;;EAEA;;;;;;;EAOA;;EAEA;;EAEA;;;;;EAKA;;;;;EAKA,cAAc;;;KAIJ;WACG;;WAEA;WACA;;WAEA;;;;;;;;KASH,wBAAwB,OAAO,uBAAuB,QAAQ;UAEhE;;EAER,SAAS;;EAET;;EAEA;;;;;;EAMA;;UAGQ;EACR;EACA,OAAO;EACP;;;;;;;;;;;;iBAmCoB,gBAAgB,MAAM,yBAAyB,QAAQ;;;;;;;;;;;;;;;;;;;;;;;;;;UCrH5D;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;KAGG,iBAAiB;;;;;;;;;;;;;;;;;;;;;;;;cAyBT,aAAa,+BAA+B;;;;;;;;iBA4BzC,YAAY,OAAO,YAAY,gBAAgB;iBAC/C,YACd,OAAO,YACP,MAAM,QAAQ,qBACd;iBAEc,YAAY,OAAO,YAAY,MAAM,UAAU;;;;;;;;;;;;UCxF9C;;EAEf;;EAEA;;EAEA;;EAEA;;cAGW;mBACM;;;;;;EAOX,SAAS,YAAY,iBAAiB,kBAAkB,QAAQ;;EAqBtE,IAAI,YAAY,kBAAkB;;EAQlC,aAAa,aAAa;;EAO1B,QAAQ;;EAKR,WAAW,YAAY,iBAAiB;;;iBAQpB,YAAY,kBAAkB;;;;;;;;;KCvExC;UAGF;EACR;EACA;EACA;EACA;EACA;EACA;;UAGQ;EACR;EACA;EACA;EACA;EACA;EACA,WAAW;;UAiBH;EACR;EACA;EACA;EACA;EACA;EACA;EACA;;UAGQ;EACR;EACA,OAAO;EACP;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA,SAAS;;UAGD;EACR;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;UAGQ;EACR,WAAW;EACX,WAAW;EACX,SAAS,QAAQ,OAAO,sBAAsB;;UAe/B,uBAAuB;EACtC;EACA,OAAO;EACP,OAAO;EACP,WAAW;EACX,YAAY;EACZ,YAAY;EACZ,OAAO;EACP;EACA;;UAGe,mBAAmB;EAClC;EACA;EACA;EACA;EACA;EACA,OAAO,uBAAuB;EAC9B,OAAO;EACP,WAAW;;UCtHI;EACf;EACA;;UAGe;EACf;EACA;EACA;EACA,kBAAkB;EAClB;EACA,cAAc,eAAe;EAC7B,WAAW;;UCZI;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;KAGU,kBAAkB,KAAK;iBAgBnB,kBAAkB,OAAO,UAAU,UAAS,QAAQ;iBAapD,QAAQ;KCzCnB;UAEY;EACf;EACA;EACA,QAAQ;EACR,OAAO;EACP,WAAW;;UAGH;EACR,SAAS;EACT;EACA;EACA;;UAGe;EACf,SAAS;EACT;EACA;EACA,UAAU;IAAQ;IAAmB;IAAc;;EACnD,WAAW;EACX;;UAGQ;EACR,UAAU,QAAQ;;;;;;;;cASP;EACX,SACE,MAAM,2BACN,SAAQ,0BACP;;KCvBO;EACN;EAAoB;EAAY;;EAChC;EAA0B;EAAY;;EACtC;EAAwB;;EACxB;EAA+B;EAAY;;EAE3C;EACA;EACA;EACA,QAAQ,MAAM,mBAAmB;;UAGtB;EACf;EACA,oBAAoB;;;EAGpB,QAAQ;EACR,UAAU;;;;EAIV,aAAa;;UAGE;EACf;EACA;EACA,UAAU;;EAEV;IACE;IACA;IACA;;;UAIa;;;;;;EAMf,cAAc,KAAK,uBAAuB,SAAS,iBAAiB;;;UClCrD;EACf;;EAEA;;EAEA;;EAEA;;EAEA,eAAe;;EAEf;;;UAIe;EACf;EACA;;EAEA;EACA,UAAU;;;UAIK;EACf,OAAO;;EAEP,UAAU,eAAe;;;UAIV;EACf;EACA;EACA;EACA,SAAS;EACT,OAAO;;UAKQ;;EAEf,SAAS;EACT;;EAEA;;;;;;;;iBAiCc,WAAW,MAAM,aAAa,MAAM,oBAAoB;;iBAmDxD,sBACd,iBACA,MAAM,aACN,MAAM,oBACL;;;;;;iBAWa,cAAc,kBAAkB;KAsCpC;UAEK;EACf;EACA;EACA;EACA,SAAS;;EAET;;EAEA;;EAEA;;EAEA;;EAEA;EACA;EACA;;UAGe;EACf,OAAO;EACP;IAAW;IAAkB;IAAmB;IAAc;;;UAG/C;;EAEf;;EAEA;;EAEA;;;;;EAKA;;;;;;;;iBASc,cACd,WAAW,WACX,OAAM,uBACL;;;;;;iBAiFa,oBAAoB,MAAM;;;;;UC5UzB;EACf;EACA;EACA;;EAEA;EACA;EACA;EACA;EACA;;;EAGA;;EAEA;EACA;EACA;;EAEA;;EAEA;EACA,aAAa;;;;;;;UAQE;EACf;EACA;EACA;;;UAIe;EACf;EACA;;;UAIe;EACf;;KAGU,uBAAuB,SAAS,kBAAkB;;;;;;;;;;;;;;;;;UAkB7C;EACf,OAAO;EACP,OAAO;EACP,QAAQ;EACR,OAAO;EACP,OAAO;EACP,MAAM;EACN,UAAU,MAAM;;;;EAIhB;;cAKW;KACD,oBAAoB;cAInB;KACD,6BAA6B;cAE5B;KACD,uBAAuB;KAEvB;EACN;;EACA;EAAc;;EACd;EAAa;;EACb;EAAY,MAAM;;;UAGP;EACf;;;EAGA;;UAGe;;;EAGf;;EAEA,kBAAkB;;EAElB;;KAGU;EACN;EAAyC;EAAe,GAAG;;EAC3D;EAAkB;EAAe,GAAG;EAAe,GAAG;EAAe,QAAQ;;EAE7E;EACA;EACA,GAAG;EACH,OAAO;EACP,QAAQ;;EAER;EAAgB;EAAe,GAAG;EAAe;;EACjD;EAAsB;EAAe,GAAG;EAAe;;EACtD;EAAa;IAAkB;EAEhC;EACA;EACA,GAAG;;EAEH;EACA,OAAO;;EAEP,aAAa;;;;;;;;;EAUb;EACA;EACA;;;;;;;;EASA;EACA;EACA;EACA,SAAS;;;;UAKE;EACf;EACA,OAAO;;;UAIQ;EACf;;EAEA;;;EAGA;IAAU,MAAM;;EAChB,OAAO;EACP,eAAe;;KAGL;UAEK;EACf;EACA;EACA;;;UAIe;EACf;;EAEA;EACA,QAAQ;EACR;EACA;;UAGe,wBAAwB;;EAEvC;EACA,QAAQ;;EAER;;EAEA;;;EAGA,QAAQ;EACR,YAAY;EACZ,gBAAgB;;EAEhB;;UAGe;EACf,UAAU;;EAEV,QAAQ;EACR;;UAGe;;EAEf,MAAM;;UC1NE;EACR,QAAQ;;UAGA;EACR;EACA,OAAO;EACP,aAAa;;cAGT;mBAKyB;mBAJZ;mBACA;UACT;EAEqB,YAAA;;EAG7B,OAAO,GAAG,eAAe;;EAMzB,MAAM,GAAG,eAAe;;EAMxB,WAAW,GAAG,eAAe;;;;EAY7B,SAAS,GAAG,eAAe,GAAG,eAAe,gBAAgB,UAAS;;EActE,YACE,GAAG,eACH,OAAO,eACP,gBACA,UAAS;;EAeX,OAAO,GAAG,eAAe,aAAa;;EAYtC,aAAa,GAAG,eAAe,aAAa;;EAY5C,IAAI,QAAQ,WAAW;;EAOvB,SAAS,GAAG,eAAe,SAAS,qBAAqB;;;EAkBzD,aAAa,qBAAqB;;EAWlC,UAAU;IAAW;IAAkB,SAAS;KAAgB;;EAYhE,MAAM,MAAM;;EAMZ,YAAY,YAAY,SAAS,QAAQ,yBAAyB;EAWlE,SAAS;UAWD;UAIA;;iBAcM,cAAc,eAAe;;;;;;;;iBCxH7B,sBACd,UAAU,eACV,gBAAgB,gBAChB,UAAS,+BACR;;;;;iBAiJa,oBACd,gBAAgB,gBAChB,oBAAoB,iBACpB,UAAS,+BACR;;iBCvJa,qBAAqB,UAAU;UCtCrC;EACR;;EAEA;;EAEA,aAAa;;EAEb;EACA;EACA;EACA,OAAO;;UAGC;;EAER;;EAEA;;EAEA;;EAEA;;EAEA,kBAAkB;;EAElB;;EAEA,YAAY;EACZ,YAAY;;;;EAIZ;;UAGQ;;EAER;;;EAGA,SAAS;;UAGD;EACR;;EAEA;;EAEA;;;UAIQ;EACR,MAAM;EACN,QAAQ;EACR,MAAM;EACN,UAAU;;EAEV,QAAQ;;UAGA,wBAAwB;EAChC;;UAGe,0BAA0B;EACzC;EACA;;;;EAIA;IAAU,MAAM;;;EAEhB;IAAiB,OAAO;;;;;;;;iBAoDV,yBAAyB,iBAAiB;UAkYzC;;;EAGf;EACA;EACA;EACA;;;;;;iBAOc,sBAAsB,iBAAiB;UCjkBtC;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA;EACA;EACA,QAAQ;;iBAGM,qBACd,QAAQ,oBACR,SAAQ,uBACR;EAAW;IACV;UCNc;EACf;EACA;EACA;EACA;;iBAGc,cAAc,gBAAgB,WAAW,oBAAoB;KCLxE;UAEY;EACf;;EAEA;;EAEA;;EAEA;;EAEA;;UAGe;;EAEf,OAAO,QAAQ;;UAGA;;EAEf,cAAc,eAAe,QAAQ;;EAErC,SAAS;;EAET;;EAEA;;EAEA;;EAEA;;EAEA;;;;;;;;;;iBAac,gBAAgB,eAAe,OAAO,uBAAuB,MAAM;UA4FlE;;EAEf;;;;;EAKA,OACE,aACA;IAAS;IAAc;QACpB;IAAU;IAAgB;IAAgB;;;;;;EAK/C,SAAS,mCAAmC;;EAE5C;;EAEA;;EAEA;;EAEA;;;;;;;iBAQc,iBAAiB,OAAO,wBAAwB;UA+B/C;EACf;EACA,OACE,aACA;IAAS;IAAc;QACpB;IAAU;IAAgB;IAAgB;;EAC/C,SAAS,mCAAmC;;EAE5C;;EAEA;;EAEA;;;;;;;;;;;;iBAac,qBAAqB,OAAO,4BAA4B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;KC7L5D;;;;;;;UAmBK;;EAEf;;;;EAIA;EACA,aAAa;;;UAME;EACf;EACA,aAAa;;UAGE;;;;EAIf;;;;;;;;;;;;;iBAcc,mBACd,iBAAiB,eACjB,sBACA,OAAM,yBACI;;;;UAyBK;;EAEf;;EAEA;;EAEA;;EAEA;;;;;;;;;;;;;KAcU,qBAAqB,WAAW,YAC1C,UAAU,WACV,gBAAgB,SAChB,SAAS,gBACN,QAAQ,qBAAqB;;;UAUjB;;EAEf;;EAEA;;;;;UAUe,gBAAgB;EAC/B,UAAU;;GAET;;;;;;;;;;iBAuBa,aACd,yBACA,oBACA,UAAS;;;;;UCxMM;;;EAGf;;EAEA;;EAEA;;;EAGA;;;EAGA;;UAGe;;;;;EAKf;EACA;EACA;;KAGG;UAEY;EACf,OAAO;EACP;EACA;;UAGe;;EAEf;;EAEA,aAAa,cAAc;;;;;;;iBAsBb,oBACd,OAAO,mBACP,OAAO,mBACP,OAAM,6BACL;UCjDc;;EAEf;;EAEA,aAAa;IAAQ;IAAc;;;EAEnC;;EAEA;EACA;;UAGe;EACf;EACA;;EAEA;;EAEA;EACA;EACA;EACA;EACA;;UAGe;;EAEf,MAAM;EACN;EACA;EACA;EACA;EACA;EACA;;EAEA,UAAU;EACV,aAAa;EACb;EACA,WAAW;EACX,SAAS;;;;;iBA0EW,oBACpB,OAAO,kBACP,SAAS,qBACR,QAAQ;;;UCpIM,aAAa;;EAE5B;;EAEA,cAAc,OAAO;;EAErB;;EAEA;;EAEA,QAAQ;;;EAGR,SAAS,QAAQ,OAAO;IAAK;IAAoB;;;;UAIlC,kBAAkB;;EAEjC,cAAc,OAAO;;EAErB;;EAEA,UAAU,eAAe,OAAO;;EAEhC;;EAEA;;EAEA;;;EAGA,UAAU,aAAa;;;;;;;;;;;;;iBAcT,uBAAuB,kBACrC,mBAAmB,aAAa,MAChC,wBAAwB,KACxB,UAAU,QAAQ,OAAO,cACxB,kBAAkB;;;;;;;;;;;;;UCxDJ;;EAEf;;EAEA;;;;;;;;;EASA;;EAEA,aAAa;;;;;;;;;EASb,eAAe;;;UAIA,kBAAkB;;EAEjC,OAAO;;EAEP;;EAEA;;EAEA;;EAEA,QAAQ;;EAER,eAAe;IAAQ;IAAiB;IAAe;;;;;;;;;;;;;;iBAqBnC,eAAe,GACnC,UAAU,eAAe,QAAQ,gBAAgB,QAAQ,IACzD,SAAQ,mBACP,QAAQ,kBAAkB;UC/CZ,qBAAqB;;EAEpC;;EAEA,YAAY;;;EAGZ;;EAEA;;;;;;EAMA,YACE,eACA;IAAS;IAAmB;IAAoB,QAAQ;QACrD,QAAQ,aAAa;;EAE1B,oBAAoB,OAAO,OAAO,kBAAkB;;EAEpD,aAAa;;EAEb,gBAAgB,kBAAkB,kBAAkB;;;;;;EAMpD,QAAQ;;;;;;;;;EASR;;;EAGA,UAAU,QAAQ,OAAO;;;;;;;;;;;;iBAaX,cAAc,kBAC5B,MAAM,qBAAqB,KAC1B;;;;;;;;;;;;;;;;;;;;;;UCjEc;EACf;EACA;;;;;;;;;;EAUA;;UAGe;EACf;EACA;EACA;;EAEA;;UAGe;;EAEf;EACA;EACA;EACA,UAAU;EACV;;EAEA;;EAEA;;UAGe;;EAEf,eAAe;;EAEf;;EAEA;;;;;;;;iBAqEc,wBACd,cACA,kBAAkB,cAAc,qBAChC,SAAQ,wBACP;;;;;;iBAyCmB,2BACpB,oBACA,kBAAkB,cAAc,qBAChC,UAAS,yBACR,QAAQ;UCrKM;;EAEf;;EAEA;;EAEA;;EAEA,UAAU;;UAGK;EACf;EACA;EACA;EACA;;UAGe;EACf;EACA,MAAM,KAAK,oBAAoB;;iBAOjB,UAAU,UAAU,0BAA0B;;;;UAqC7C,qBAAqB;EACpC,SAAS;EACT;EACA;;;iBAIc,gBAAgB,KAAK,mBAAmB,SAAS,WAAW;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;KC7CvE,eAAe;UAEH;;;EAGf,eAAe,SAAS;;EAExB,SAAS;;UAGM;;;;;EAKf;;UAGe;;;EAGf;;;;EAIA;EACA;;EAEA;;EAEA;;;;;;iBAOc,qBACd,OAAO,oBACP,OAAM,uBACL;;;;;;;iBC5Ca,cAAc;;;;iBAOd,YAAY;;;UAQX;EACf,IAAI,cAAc,QAAQ,0BAA0B;EACpD,IAAI,aAAa,OAAO,aAAa;;;;;;;;;iBA2CvB,iBAAiB,eAAe;;;;;;;;;;;;;;;;;;;;;;;;cC7EnC;;;cAaA;;cAYA;;cAYA;;;;;KAMD,+BAA+B;;;;;;;UClC1B;EACf;EACA;EACA;EACA;;;;;UAMe;EACf;EACA;EACA;EACA;;;;UAKe;EACf;EACA;EACA;EACA,QAAQ;EACR,QAAQ;;;;;;;;;iBAqBM,cAAc,GAAG;;;iBAmBjB,iBAAiB,GAAG;;;;;;iBAuCpB,gBAAgB;EAC9B;EACA;EACA;EACA,SAAS;IACP;;;;;;;;iBAiBY,wBACd,MAAM,iBACN;EAAQ;EAAsB;EAA0B,SAAS;IAChE;;;;;iBAQa,YACd,UAAU,eACV,SAAS,wBACR;;;;;;;;iBAWa,iBACd,GAAG,eACH,mBACA,kBACC;;;;;;;;;;iBA8Ba,YAAY,SAAS;cC3MxB,0BAA0B;EAE5B,iBAAiB;EACjB;EACA;EAFA,YAAA,iBAAiB,YACjB,eACA;;cAME;UACH;UACA;UACA;UACA;EAEI,YAAA,SAAS,cAAc,QAAQ,YAAY;;EAOjD,OAAO,OAAO,QAAQ,aAAa,sBAAsB,kBAAkB;;EA2B3E,SAAS,eAAe,kBAAkB;MAO5C,SAAS,aAAa;;;;;;;;;;;cCzDf;UACH;UACA;UACA;UACA;EAEI,YAAA,QAAQ;UAUZ;;;;;;;;UAaM;;;;;;UAkCN;EAUF,OAAO,cAAc,eAAe,mBAAmB;IAAU;;EAOjE,MAAM,eAAe,mBAAmB;EAexC,gBAAgB,cAAc,gBAAmB;EAOjD,aAAa,sBAAsB;EAOnC,KACJ,qBACA,kBACA,iBACA;IAAU,gBAAgB;MACzB;IAAU;IAAc;MAAU;MAAc;;;EAuD7C,SACJ,sBACC;IAAU;IAAY;IAAe;IAAgB;;EAQlD,UAAU,sBAAsB;IAAU;IAAY;IAAe;;EAKrE,aACJ,sBACC;IAAU;IAAY;IAAe;IAAgB;;EAQlD,aAAa,sBAAsB;EAcnC,cAAc,qBAAqB,aAAa;EAIhD,aAAa,qBAAqB,YAAY,iBAAiB;EAI/D,eACJ,sBACC;IAAU;IAAY;IAAc;;;EAMjC,IAAI,eAAe,QAAQ;;EAK3B,KAAK,cAAc,MAAM,0BAA0B,QAAQ;;EAK3D,MAAM,cAAc,MAAM,0BAA0B,QAAQ;;iBC1K9C,oBACpB,QACA,SACA,eACA,cAAc,oBAAoB,mBAElC,QAAQ,qBAAqB,QAAQ,SAAS,eAAe,SAC5D,QAAQ,iBAAiB,QAAQ,SAAS,eAAe;iBC7E5C,cAAc,OAAO,KAAK,kCAAkC;iBAI5D,eAAe,OAAO,KAAK,kCAAkC;UCEnE;EACR;EACA,cAAc;EACd;EACA;EACA;EACA,cAAc;EACd,WAAW;EACX,MAAM;;iBAGQ,wBACd,SAAS,iCACR;iBA+Ca,sBACd,QAAQ,0BACR;EAAW;EAAa;EAAuB,UAAU;IACxD;iBAuBa,+BACd,QAAQ,0BACR;EAAW;EAAkB;IAC5B;iBAea,8BAA8B,MAAM,yBAAyB;iBAc7D,iCACd,MAAM,yBACL;UCtIO;EACR;EACA;;;cAIW,eAAe,eAAe;;;;iBA0D3B,oBAAoB,gBAAgB;;iBAapC,cAAc;;iBAOd,eAAe;;;iBAMf,aAAa,qBAAqB,sBAAsB;UCxE9D;;EAER;;EAGA;;EAGA,iBAAiB;;EAGjB,kBAAkB;;EAGlB,kBAAkB;;EAGlB;;EAGA;;EAGA;;EAGA,iBAAiB,QAAQ,OAAO;;KAG7B;;iBAsBW,oBAAoB,SAAQ,iBAAsB;UA4CxD;EACR,UAAU;EACV;EACA;;UAGQ;;EAER;EACA,QAAQ;;EAER,QAAQ,OAAO;;;;;;iBAOD,gBACd,mBACA,QAAQ,KAAK,SAAS;EACpB,gBAAgB,OAAO;IAExB;iBC5Ha,aAAa,cAAc,KAAK,WAAW;iBAS3C,aACd,cACA,gBACA,8BACC;iBAUa,aAAa,cAAc,iBAAiB;iBAa5C,mBACd,cACA,kBACA,mBACC;iBAaa,eAAe,cAAc,gBAAgB,mBAAmB;iBAShE,kBACd,cACA,gBACA,oBACC;iBAaa,iBACd,cACA,eACA,aACA,eACC;iBAca,eACd,cACA,KAAK,yBACL,gBACC"}

package/dist/index.d.ts CHANGED Viewed

@@ -1,6 +1,6 @@
 import { _ as PendingCostCall, a as CostLedgerFilter, c as CostLedgerPersistence, d as CostReceipt, f as CostReceiptInput, g as PaidCallResult, h as MaximumCharge, i as CostChannel, l as CostLedgerSummary, m as CustomTokenPricing, n as ChannelRollup, o as CostLedgerHandle, p as CostUsage, s as CostLedgerOptions, t as CostLedger, u as CostProvenance, v as PendingCostCallView, y as RunPaidCallInput } from "./cost-ledger-Czro9wmo.js";
 import { a as JudgeError, c as ValidationError, i as ConfigError, n as AgentEvalErrorCode, s as NotFoundError, t as AgentEvalError } from "./errors-E9I2hldP.js";
-import { $ as EnsembleAggregate, $i as makePolicyEditCandidateRecord, $n as ReviewInput, $r as pairedDeltaTest, $t as TraceContract, A as contentHash, Ai as AssertCapabilityHeadroomOptions, An as PromptRegistry, Ar as RuntimeTrajectoryHookEvent, At as checkCanaries, B as evaluateOracles, Bi as PolicyEditAdmissionOptions, Bn as ReflectionContext, Br as ProductBenchmarkExportResult, Bt as evaluateTraceContract, C as runAgentControlLoop, Ci as errorStreakDetector, Cn as aggregateRunScore, Cr as runBoundedProcess, Ct as DeployRunner, D as index_d_exports, Di as CostSummary, Dn as ReferenceReplayRun, Dr as ProjectRuntimeTrajectoryEvidenceOptions, Dt as viteDeployRunner, E as BudgetGuard, Ei as CostEntry, En as ReferenceReplayCaseRun, Er as proposeSynthesisTargets, Et as deployGateLayer, F as gateTreatmentApplied, Fi as assertCapabilityHeadroom, Fn as ModelEndpointCheck, Fr as ProductBenchmarkSplit, Ft as TraceContractSpec, G as KeywordCoverageResult, Gi as PolicyEditGainDirection, Gn as LlmReviewerConfig, Gr as HeldOutPartition, Gt as ContractRuleExecution, H as KeywordConceptSpec, Hi as PolicyEditCandidateRecord, Hn as TrialTrace, Hr as exportProductBenchmark, Ht as ArgumentCheck, I as Oracle, Ii as capabilityHeadroom, In as ModelEndpointRequest, Ir as ProductBenchmarkValidationReport, It as compileTraceContractSpec, J as EnsembleJudgeOptions, Ji as PolicyEditRisk, Jn as ProposeOutput, Jr as TrainSelectionTestPartition, Jt as ContractStatus, K as runKeywordCoverageJudge, Ki as PolicyEditGainUnit, Kn as ProposeFn, Kr as PartitionHeldOutOptions, Kt as ContractRun, L as OracleObservation, Li as FindingToPolicyEditOptions, Ln as preflightModels, Lr as assertProductBenchmarkRun, Lt as lintTraceContractSpec, M as TreatmentGate, Mi as CapabilityHeadroomResult, Mn as ModelSeats, Mr as RuntimeTrajectoryRunRecord, Mt as ActionPolicyDecision, N as TreatmentGateInput, Ni as HeadroomInput, Nn as resolveSeat, Nr as projectRuntimeTrajectoryEvidence, Nt as evaluateActionPolicy, O as VerdictCacheStore, Oi as CostTracker, On as ReferenceReplaySplit, Or as RuntimeTrajectoryEvidenceProjection, Ot as wranglerDeployRunner, P as TreatmentGateOptions, Pi as TaskHeadroom, Pn as seatPresets, Pr as ProductBenchmarkManifest, Pt as ContractLintFinding, Q as withJudgeRetry, Qi as PolicyEditTargetSurface, Qn as ReviewFn, Qr as minimumPairsForPairedDeltaTest, Qt as SpanPredicate, R as OracleReport, Ri as PolicyEdit, Rn as RunRecordBackend, Rr as readProductBenchmarkManifest, Rt as explainTraceContract, S as subjectiveEval, Si as StreamingDetector, Sn as RunScoreWeights, Sr as DEFAULT_BOUNDED_PROCESS_TIMEOUT_MS, St as DeployRunResult, T as BudgetBreachError, Ti as repeatedActionDetector, Tn as SteeringBundle, Tr as SynthesisTarget, Tt as WranglerDeployRunnerInput, U as KeywordCoverageFinding, Ui as PolicyEditChange, Un as buildReflectionPrompt, Ur as exportProductBenchmarkRuns, Ut as ContractCheckResult, V as jsonShape, Vi as PolicyEditAxis, Vn as ReflectionProposal, Vr as ProductBenchmarkSingleRunExportOptions, Vt as traceContract, W as KeywordCoverageOptions, Wi as PolicyEditExpectedGain, Wn as parseReflectionResponse, Wr as productBenchmarkRepoIdentity, Wt as ContractRule, X as JudgeRetryOutcome, Xi as PolicyEditSource, Xn as ProposeReviewReport, Xr as PairedDeltaTestOptions, Xt as OrderMode, Y as ensembleJudge, Yi as PolicyEditSchemaVersion, Yn as ProposeReviewConfig, Yr as partitionHeldOut, Yt as ContractVerdict, Z as JudgeRetryPolicy, Zi as PolicyEditTarget, Zn as Review, Zr as PairedDeltaTestResult, Zt as RetryWrite, _ as blockingKnowledgeEval, _i as extractErrorCount, _n as CounterfactualRunner, _r as RunCommandInput, _t as NoLeakOptions, a as assertHttpOk, aa as costReceiptFromLlmError, ai as KnowledgeBundle, an as ScorecardCellDiff, ar as inMemoryReviewStore, at as runIntentMatchJudge, b as userQuestionsForKnowledgeGaps, bi as ErrorStreakOptions, bn as SteeringOptimizationRow, br as BoundedProcessInput, bt as blendHeldout, c as assertStatus, ca as costForTokenPricing, ci as KnowledgeReadinessReport, cn as ScorecardLogLine, cr as ExperimentRep, ct as SingleBackendDivergence, d as MODEL_PRICING, da as CostCeilingReachedError, di as KnowledgeSensitivity, dn as loadScorecard, dr as ImprovementVerdictResult, dt as BlendWeights, ea as isTransientLlmError, ei as LeaderboardOptions, en as CellVerdict, er as ReviewMemoryEntry, et as JudgeVerdict, f as estimateCost, fi as UserQuestion, fn as recordRuns, fr as DiscoverPersonasOptions, ft as FieldDestination, g as acquisitionPlansForKnowledgeGaps, gi as ExtractResult, gn as CounterfactualResult, gr as DirEntry, gt as JudgeScoreInput, h as resolveModelPricing, hi as ExtractOptions, hn as CounterfactualMutation, hr as CommandRunner, ht as HiddenLeak, i as assertHasField, ia as costReceiptFromLlm, ii as KnowledgeAcquisitionMode, in as ScorecardCell, ir as createLlmReviewer, it as IntentMatchResult, j as fileVerdictCache, ji as CapabilityHeadroomOptions, jn as hashContent, jr as RuntimeTrajectoryRecord, jt as ActionExecutionPolicy, k as canonicalJson, ki as ScenarioCost, kn as PromptHandle, kr as RuntimeTrajectoryEvidenceSummary, kt as CanaryLeak, l as analyzeAntiSlop, la as costForUsage, li as KnowledgeRequirement, ln as diffScorecard, lr as ExperimentStats, lt as SingleBackendReport, m as isModelPriced, mi as ErrorCountPattern, mn as CounterfactualContext, mr as discoverPersonas, mt as HiddenGradeResult, n as assertEventCount, na as LlmCallError, ni as leaderboard, nn as RecordRunsOptions, nr as Verification, nt as IntentMatchInput, o as assertLatencyUnder, oa as maximumChargeForLlmRequest, oi as KnowledgeFreshness, on as ScorecardDiff, or as jsonlReviewStore, ot as AssertSingleBackendOptions, p as estimateTokens, pi as ERROR_COUNT_PATTERNS, pn as recordRunsToScorecard, pr as DiscoveredPersona, pt as HiddenCriteriaGrader, q as runKeywordCoverageJudgeUrl, qi as PolicyEditInit, qn as ProposeInput, qr as TrainSelectionTestOptions, qt as ContractSpan, r as assertExists, ra as LlmResponseError, ri as DataAcquisitionPlan, rn as Scorecard, rr as VerifyFn, rt as IntentMatchOptions, s as assertNotContains, sa as CostResult, si as KnowledgeImportance, sn as ScorecardEntry, sr as runProposeReview, st as BackendDescriptor, t as assertContains, ta as stripFencedJson, ti as LeaderboardRow, tn as DiffScorecardOptions, tr as ReviewMemoryStore, tt as aggregateJudgeVerdicts, u as createAntiSlopJudge, ua as CostAccountingIncompleteError, ui as KnowledgeRequirementCategory, un as formatScorecardDiff, ur as ImprovementThresholds, ut as assertSingleBackend, v as knowledgeReadinessTracePayload, vi as DetectorEvent, vn as PairwiseSteeringOptimizer, vr as RunCommandResult, vt as RoutedField, w as ProductClient, wi as observeAll, wn as clamp01, wr as ActiveLearningOptions, wt as ViteDeployRunnerInput, x as objectiveEval, xi as RepeatedActionOptions, xn as RunScore, xr as BoundedProcessResult, xt as DeployGateLayerInput, y as scoreKnowledgeReadiness, yi as DetectorSignal, yn as SteeringOptimizationResult, yr as localCommandRunner, yt as assertNoHiddenLeak, z as OracleResult, zi as PolicyEditAdmission, zn as jsonlRunRecordBackend, zr as ProductBenchmarkExportOptions, zt as checkTraceContracts } from "./index-JPI2spOx.js";
+import { $ as EnsembleAggregate, $i as makePolicyEditCandidateRecord, $n as ReviewInput, $r as pairedDeltaTest, $t as TraceContract, A as contentHash, Ai as AssertCapabilityHeadroomOptions, An as PromptRegistry, Ar as RuntimeTrajectoryHookEvent, At as checkCanaries, B as evaluateOracles, Bi as PolicyEditAdmissionOptions, Bn as ReflectionContext, Br as ProductBenchmarkExportResult, Bt as evaluateTraceContract, C as runAgentControlLoop, Ci as errorStreakDetector, Cn as aggregateRunScore, Cr as runBoundedProcess, Ct as DeployRunner, D as index_d_exports, Di as CostSummary, Dn as ReferenceReplayRun, Dr as ProjectRuntimeTrajectoryEvidenceOptions, Dt as viteDeployRunner, E as BudgetGuard, Ei as CostEntry, En as ReferenceReplayCaseRun, Er as proposeSynthesisTargets, Et as deployGateLayer, F as gateTreatmentApplied, Fi as assertCapabilityHeadroom, Fn as ModelEndpointCheck, Fr as ProductBenchmarkSplit, Ft as TraceContractSpec, G as KeywordCoverageResult, Gi as PolicyEditGainDirection, Gn as LlmReviewerConfig, Gr as HeldOutPartition, Gt as ContractRuleExecution, H as KeywordConceptSpec, Hi as PolicyEditCandidateRecord, Hn as TrialTrace, Hr as exportProductBenchmark, Ht as ArgumentCheck, I as Oracle, Ii as capabilityHeadroom, In as ModelEndpointRequest, Ir as ProductBenchmarkValidationReport, It as compileTraceContractSpec, J as EnsembleJudgeOptions, Ji as PolicyEditRisk, Jn as ProposeOutput, Jr as TrainSelectionTestPartition, Jt as ContractStatus, K as runKeywordCoverageJudge, Ki as PolicyEditGainUnit, Kn as ProposeFn, Kr as PartitionHeldOutOptions, Kt as ContractRun, L as OracleObservation, Li as FindingToPolicyEditOptions, Ln as preflightModels, Lr as assertProductBenchmarkRun, Lt as lintTraceContractSpec, M as TreatmentGate, Mi as CapabilityHeadroomResult, Mn as ModelSeats, Mr as RuntimeTrajectoryRunRecord, Mt as ActionPolicyDecision, N as TreatmentGateInput, Ni as HeadroomInput, Nn as resolveSeat, Nr as projectRuntimeTrajectoryEvidence, Nt as evaluateActionPolicy, O as VerdictCacheStore, Oi as CostTracker, On as ReferenceReplaySplit, Or as RuntimeTrajectoryEvidenceProjection, Ot as wranglerDeployRunner, P as TreatmentGateOptions, Pi as TaskHeadroom, Pn as seatPresets, Pr as ProductBenchmarkManifest, Pt as ContractLintFinding, Q as withJudgeRetry, Qi as PolicyEditTargetSurface, Qn as ReviewFn, Qr as minimumPairsForPairedDeltaTest, Qt as SpanPredicate, R as OracleReport, Ri as PolicyEdit, Rn as RunRecordBackend, Rr as readProductBenchmarkManifest, Rt as explainTraceContract, S as subjectiveEval, Si as StreamingDetector, Sn as RunScoreWeights, Sr as DEFAULT_BOUNDED_PROCESS_TIMEOUT_MS, St as DeployRunResult, T as BudgetBreachError, Ti as repeatedActionDetector, Tn as SteeringBundle, Tr as SynthesisTarget, Tt as WranglerDeployRunnerInput, U as KeywordCoverageFinding, Ui as PolicyEditChange, Un as buildReflectionPrompt, Ur as exportProductBenchmarkRuns, Ut as ContractCheckResult, V as jsonShape, Vi as PolicyEditAxis, Vn as ReflectionProposal, Vr as ProductBenchmarkSingleRunExportOptions, Vt as traceContract, W as KeywordCoverageOptions, Wi as PolicyEditExpectedGain, Wn as parseReflectionResponse, Wr as productBenchmarkRepoIdentity, Wt as ContractRule, X as JudgeRetryOutcome, Xi as PolicyEditSource, Xn as ProposeReviewReport, Xr as PairedDeltaTestOptions, Xt as OrderMode, Y as ensembleJudge, Yi as PolicyEditSchemaVersion, Yn as ProposeReviewConfig, Yr as partitionHeldOut, Yt as ContractVerdict, Z as JudgeRetryPolicy, Zi as PolicyEditTarget, Zn as Review, Zr as PairedDeltaTestResult, Zt as RetryWrite, _ as blockingKnowledgeEval, _i as extractErrorCount, _n as CounterfactualRunner, _r as RunCommandInput, _t as NoLeakOptions, a as assertHttpOk, aa as costReceiptFromLlmError, ai as KnowledgeBundle, an as ScorecardCellDiff, ar as inMemoryReviewStore, at as runIntentMatchJudge, b as userQuestionsForKnowledgeGaps, bi as ErrorStreakOptions, bn as SteeringOptimizationRow, br as BoundedProcessInput, bt as blendHeldout, c as assertStatus, ca as costForTokenPricing, ci as KnowledgeReadinessReport, cn as ScorecardLogLine, cr as ExperimentRep, ct as SingleBackendDivergence, d as MODEL_PRICING, da as CostCeilingReachedError, di as KnowledgeSensitivity, dn as loadScorecard, dr as ImprovementVerdictResult, dt as BlendWeights, ea as isTransientLlmError, ei as LeaderboardOptions, en as CellVerdict, er as ReviewMemoryEntry, et as JudgeVerdict, f as estimateCost, fi as UserQuestion, fn as recordRuns, fr as DiscoverPersonasOptions, ft as FieldDestination, g as acquisitionPlansForKnowledgeGaps, gi as ExtractResult, gn as CounterfactualResult, gr as DirEntry, gt as JudgeScoreInput, h as resolveModelPricing, hi as ExtractOptions, hn as CounterfactualMutation, hr as CommandRunner, ht as HiddenLeak, i as assertHasField, ia as costReceiptFromLlm, ii as KnowledgeAcquisitionMode, in as ScorecardCell, ir as createLlmReviewer, it as IntentMatchResult, j as fileVerdictCache, ji as CapabilityHeadroomOptions, jn as hashContent, jr as RuntimeTrajectoryRecord, jt as ActionExecutionPolicy, k as canonicalJson, ki as ScenarioCost, kn as PromptHandle, kr as RuntimeTrajectoryEvidenceSummary, kt as CanaryLeak, l as analyzeAntiSlop, la as costForUsage, li as KnowledgeRequirement, ln as diffScorecard, lr as ExperimentStats, lt as SingleBackendReport, m as isModelPriced, mi as ErrorCountPattern, mn as CounterfactualContext, mr as discoverPersonas, mt as HiddenGradeResult, n as assertEventCount, na as LlmCallError, ni as leaderboard, nn as RecordRunsOptions, nr as Verification, nt as IntentMatchInput, o as assertLatencyUnder, oa as maximumChargeForLlmRequest, oi as KnowledgeFreshness, on as ScorecardDiff, or as jsonlReviewStore, ot as AssertSingleBackendOptions, p as estimateTokens, pi as ERROR_COUNT_PATTERNS, pn as recordRunsToScorecard, pr as DiscoveredPersona, pt as HiddenCriteriaGrader, q as runKeywordCoverageJudgeUrl, qi as PolicyEditInit, qn as ProposeInput, qr as TrainSelectionTestOptions, qt as ContractSpan, r as assertExists, ra as LlmResponseError, ri as DataAcquisitionPlan, rn as Scorecard, rr as VerifyFn, rt as IntentMatchOptions, s as assertNotContains, sa as CostResult, si as KnowledgeImportance, sn as ScorecardEntry, sr as runProposeReview, st as BackendDescriptor, t as assertContains, ta as stripFencedJson, ti as LeaderboardRow, tn as DiffScorecardOptions, tr as ReviewMemoryStore, tt as aggregateJudgeVerdicts, u as createAntiSlopJudge, ua as CostAccountingIncompleteError, ui as KnowledgeRequirementCategory, un as formatScorecardDiff, ur as ImprovementThresholds, ut as assertSingleBackend, v as knowledgeReadinessTracePayload, vi as DetectorEvent, vn as PairwiseSteeringOptimizer, vr as RunCommandResult, vt as RoutedField, w as ProductClient, wi as observeAll, wn as clamp01, wr as ActiveLearningOptions, wt as ViteDeployRunnerInput, x as objectiveEval, xi as RepeatedActionOptions, xn as RunScore, xr as BoundedProcessResult, xt as DeployGateLayerInput, y as scoreKnowledgeReadiness, yi as DetectorSignal, yn as SteeringOptimizationResult, yr as localCommandRunner, yt as assertNoHiddenLeak, z as OracleResult, zi as PolicyEditAdmission, zn as jsonlRunRecordBackend, zr as ProductBenchmarkExportOptions, zt as checkTraceContracts } from "./index-C8a9Dyan.js";
 import { _ as toAgentProfileJson, g as groupRunsByAgentProfileCell, h as buildAgentProfileCell, i as AgentProfileCellInput, l as AgentProfileJson, m as agentProfileCellKey, p as AgentProfileSourceInput, r as AgentProfileCell, s as AgentProfileDimensionValue, t as AGENT_PROFILE_KINDS, y as verifyAgentProfileCell } from "./agent-profile-cell-3kYxUsK2.js";
 import { E as isToolSpan, S as TraceEvent, T as isLlmSpan, _ as Span, a as FAILURE_CLASSES, c as JudgeSpan, f as Run, h as RunStatus, l as LlmSpan, n as BudgetLedgerEntry, o as FailureClass, p as RunCaptureReport, r as BudgetSpec, s as GenericSpan, t as Artifact, w as isJudgeSpan, x as ToolSpan } from "./schema-CkIvnmLU.js";
 import { _ as runTaskScore, a as RunRecord, c as RunSplitTag, d as RunTokenUsage, f as RunTraceRef, g as parseRunRecordSafe, h as modelHasSnapshot, i as RunOutcome, l as RunTaskFailure, m as isRunRecord, n as RunCostProvenance, o as RunRecordValidationError, p as UNKNOWN_MODEL, r as RunJudgeMetadata, s as RunSearchCoordinates, t as JudgeScoresRecord, u as RunTerminalOutcome, v as validateRunRecord } from "./run-record-pBlhZizT.js";
@@ -33,7 +33,7 @@ import { r as TraceAnalysisEngineResult, s as RawAnalystFinding, t as TraceAnaly
 import { a as BudgetPolicy, c as RegistryRunOpts, i as AnalystRegistryOptions, s as ExactRegistryRunOpts, t as AnalystRegistry } from "./registry-BqoX13FD.js";
 import { C as SEMANTIC_CONCEPT_JUDGE_VERSION, D as runSemanticConceptJudge, E as SemanticConceptJudgeResult, S as ConceptSpec, T as SemanticConceptJudgeOptions, _ as DspyRlmTraceEngineOptions, a as FAILURE_MODE_KIND_SPEC, b as createChatTraceEngine, d as FindingSubject, f as FindingSubjectKind, t as DEFAULT_TRACE_ANALYST_KINDS, v as createDspyRlmTraceEngine, w as SemanticConceptJudgeInput, x as ConceptFinding, y as ChatTraceEngineOptions } from "./index-AjfEHHA8.js";
 import { S as welchsTTest, _ as computeToolUseMetrics, b as WelchTestStatus, d as Trajectory, f as TrajectoryStep, g as ToolUseOptions, h as ToolUseMetrics, i as toolWasteView, m as ToolStats, p as buildTrajectory, s as judgeAgreementView, u as budgetBreachView, x as iqr, y as WelchTestResult } from "./tool-waste-CUASoTtk.js";
-import { i as BenchmarkEvaluation } from "./types-D7g2XFTc.js";
+import { i as BenchmarkEvaluation } from "./types-CEgIBy0W.js";
 import { A as AnalystFindingDigest, B as ControlRuntimeConfig, C as createFeedbackTrajectory, D as renderPreferenceMemoryMarkdown, E as feedbackTrajectoryToOptimizerRow, F as ControlBudget, I as ControlContext, L as ControlDecision, M as AnalystRunDigest, N as analystRunDigest, O as summarizePreferenceMemory, P as ControlActionOutcome, R as ControlEvalResult, S as controlRunToFeedbackTrajectory, T as feedbackTrajectoriesToOptimizerRows, U as ControlStep, V as ControlRuntimeError, W as StopDecision, _ as PreferenceMemoryEntry, a as FeedbackLabel, b as analystRunToReviewRequests, c as FeedbackOptimizerRow, d as FeedbackTask, f as FeedbackTrajectory, g as InMemoryFeedbackTrajectoryStore, h as FileSystemFeedbackTrajectoryStore, i as FeedbackAttempt, j as AnalystReviewDecision, k as withAssignedFeedbackSplit, l as FeedbackOutcome, m as FeedbackTrajectoryStore, n as AnalystReviewRequest, o as FeedbackLabelKind, p as FeedbackTrajectoryFilter, r as FeedbackArtifactType, s as FeedbackLabelSource, t as AnalystFeedbackTrajectoryOptions, u as FeedbackSplitPolicy, v as ProposedSideEffect, w as feedbackTrajectoriesToDatasetScenarios, x as assignFeedbackSplit, y as analystRunToFeedbackTrajectory, z as ControlRunResult } from "./feedback-trajectory-DxkG5EPi.js";
 import { A as TraceInsightReadiness, F as describeTraceInsightScope, G as AnalyzeTracesResult, H as tokenizeDomainWords, I as domainEvidencePattern, K as analyzeTraces, L as inferDomainKeywords, N as buildTraceInsightContext, P as buildTraceInsightPrompt, V as scoreTraceInsightReadiness, a as captureFetchToRawSink, b as OtlpSpan, c as ExtractedUsage, d as extractUsageFromSse, f as OtlpFileTraceStore, g as spanRecordsToTraceAnalysisStore, h as otlpTextToTraceAnalysisStore, j as TraceInsightSuite, m as SpanRecordsToTraceAnalysisStoreOptions, n as toolSpansToTraceAnalysisStore, o as OtlpFlatLine, u as extractUsage, v as OtlpExport, x as exportRunAsOtlp } from "./store-tool-spans-kCDPEwjL.js";
 import { a as RunIntegrityReport, o as assertRunCaptured, t as RunIntegrityError } from "./integrity-BMyCpnZ8.js";

package/dist/openapi.json CHANGED Viewed

@@ -2,7 +2,7 @@
   "openapi": "3.1.0",
   "info": {
     "title": "@tangle-network/agent-eval — wire protocol",
-    "version": "1.0.25",
+    "version": "1.0.26",
     "description": "HTTP and stdio RPC interface to agent-eval. The TypeScript runtime is the source of truth; this spec is the contract that cross-language clients (Python, Rust, Go) generate from.\n\nWire-protocol version: 1.1.0. Bumps on breaking changes to request/response schemas.",
     "contact": {
       "name": "Tangle Network",

package/dist/{types-D7g2XFTc.d.ts → types-CEgIBy0W.d.ts} RENAMED Viewed

@@ -89,12 +89,13 @@ type BenchmarkResponder<TPayload = unknown, TArtifact = string> = (input: {
 declare const BENCHMARK_SPLIT_SEED = "agent-eval-v1";
 /**
  * Assign an item id to one of `'search' | 'dev' | 'holdout'` using a
- * stable 32-bit hash of `${seed}::${id}`. Default proportions:
+ * stable 32-bit hash of `${seed}::${id}`. `cuts` [a, b] with 0 < a < b < 1 put
+ * positions below a in search, below b in dev, the rest in holdout. Default [0.6, 0.8]:
  *
  *   search: 60%   (optimization-readable)
  *   dev:    20%   (held-out for tuning, leak-on-purpose during dev)
  *   holdout:20%   (paper-grade held-out, gated reads)
  */
-declare function deterministicSplit(itemId: string, seed?: string): RunSplitTag;
+declare function deterministicSplit(itemId: string, seed?: string, cuts?: readonly [number, number]): RunSplitTag;
 export { BenchmarkFamily as a, BenchmarkSource as c, BenchmarkEvaluation as i, BenchmarkTaskKind as l, BenchmarkAdapter as n, BenchmarkResponder as o, BenchmarkDatasetItem as r, BenchmarkScenario as s, BENCHMARK_SPLIT_SEED as t, deterministicSplit as u };
-//# sourceMappingURL=types-D7g2XFTc.d.ts.map
+//# sourceMappingURL=types-CEgIBy0W.d.ts.map

package/dist/{types-D7g2XFTc.d.ts.map → types-CEgIBy0W.d.ts.map} RENAMED Viewed

	@@ -1 +1 @@
1	- {"version":3,"file":"types-~~D7g2XFTc~~.d.ts","names":[],"sources":["../src/benchmarks/types.ts"],"mappings":";;KASY;KAQA;UAgBK,qBAAqB;;;EAGpC;;EAEA,SAAS;;EAET,QAAQ;;EAER,SAAS;;EAET,WAAW;;EAEX;;EAEA,SAAS;EACT,WAAW;;UAGI;;;;EAIf;;EAEA;;EAEA,aAAa;;;EAGb,KAAK;EACL;;UAGe;EACf;EACA;EACA;;EAEA;EACA;EACA;;EAEA;;;;;EAKA;;;UAQe,iBAAiB,kBAAkB,oBAAoB;;EAEtE;EACA,SAAS;EACT,WAAW;EACX;EACA,SAAS;EACT;;;;;EAKA,YAAY,OAAO,cAAc,QAAQ,qBAAqB;;EAE9D,SAAS,MAAM,qBAAqB,WAAW,UAAU,YAAY,QAAQ;;;;EAI7E,YAAY,iBAAiB;;UAGd,kBAAkB,4BAA4B;EAC7D;EACA;EACA,QAAQ;EACR,UAAU;EACV,UAAU;EACV,MAAM,qBAAqB;;KAGjB,mBAAmB,oBAAoB,uBAAuB;EACxE,UAAU,kBAAkB;EAC5B,MAAM,qBAAqB;EAC3B,SAAS;MACL,QAAQ,aAAa;;;cAoBd~~;;;;;;;;;iBAUG~~,mBACd,gBACA,~~gBACC~~"}
1	+ {"version":3,"file":"types-CEgIBy0W.d.ts","names":[],"sources":["../src/benchmarks/types.ts"],"mappings":";;KASY;KAQA;UAgBK,qBAAqB;;;EAGpC;;EAEA,SAAS;;EAET,QAAQ;;EAER,SAAS;;EAET,WAAW;;EAEX;;EAEA,SAAS;EACT,WAAW;;UAGI;;;;EAIf;;EAEA;;EAEA,aAAa;;;EAGb,KAAK;EACL;;UAGe;EACf;EACA;EACA;;EAEA;EACA;EACA;;EAEA;;;;;EAKA;;;UAQe,iBAAiB,kBAAkB,oBAAoB;;EAEtE;EACA,SAAS;EACT,WAAW;EACX;EACA,SAAS;EACT;;;;;EAKA,YAAY,OAAO,cAAc,QAAQ,qBAAqB;;EAE9D,SAAS,MAAM,qBAAqB,WAAW,UAAU,YAAY,QAAQ;;;;EAI7E,YAAY,iBAAiB;;UAGd,kBAAkB,4BAA4B;EAC7D;EACA;EACA,QAAQ;EACR,UAAU;EACV,UAAU;EACV,MAAM,qBAAqB;;KAGjB,mBAAmB,oBAAoB,uBAAuB;EACxE,UAAU,kBAAkB;EAC5B,MAAM,qBAAqB;EAC3B,SAAS;MACL,QAAQ,aAAa;;;cAoBd;;;;;;;;;;iBAWG,mBACd,gBACA,eACA,mCACC"}

package/dist/wire/index.d.ts CHANGED Viewed

@@ -213,9 +213,9 @@ export declare const FeedbackTrajectorySchema: z.ZodObject<{
       value: z.ZodUnknown;
       reason: z.ZodOptional<z.ZodString>;
       severity: z.ZodOptional<z.ZodEnum<{
+        error: "error";
         critical: "critical";
         info: "info";
-        error: "error";
         warning: "warning";
       }>>;
       createdAt: z.ZodString;
@@ -249,9 +249,9 @@ export declare const FeedbackTrajectorySchema: z.ZodObject<{
     value: z.ZodUnknown;
     reason: z.ZodOptional<z.ZodString>;
     severity: z.ZodOptional<z.ZodEnum<{
+      error: "error";
       critical: "critical";
       info: "info";
-      error: "error";
       warning: "warning";
     }>>;
     createdAt: z.ZodString;
@@ -267,10 +267,10 @@ export declare const FeedbackTrajectorySchema: z.ZodObject<{
     metadata: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
   }, z.core.$strip>>;
   split: z.ZodOptional<z.ZodEnum<{
-    dev: "dev";
-    holdout: "holdout";
     train: "train";
+    dev: "dev";
     test: "test";
+    holdout: "holdout";
   }>>;
   tags: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
   createdAt: z.ZodString;

package/package.json CHANGED Viewed

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-eval",
-  "version": "1.0.25",
+  "version": "1.0.26",
   "description": "Evaluate and improve AI agents from runs, traces, judges, and feedback. Compare candidates, cluster failures, measure lift, and gate releases.",
   "homepage": "https://github.com/tangle-network/agent-eval#readme",
   "repository": {