@tangle-network/agent-eval 1.0.25 → 1.0.27

package/CHANGELOG.md

@@ -4,6 +4,99 @@ All notable changes to `@tangle-network/agent-eval` and its sibling `agent-eval-
 
 ---
 
+## [1.0.27] — 2026-11-08
+
+- fix(models): default to the newest served model in each tier
+- chore(release): 1.0.26
+- fix(benchmarks): detailTail ends for a bound of 0 or 1, and keepImage says what it does
+- feat(benchmarks): grade a code patch with a Harbor task's own verifier
+- chore(release): 1.0.25
+- fix(judges): default to zai/glm-5.3, which every restricted Router key may call
+- chore(release): 1.0.24
+- test(multishot): judge default model and its price in runJudge
+- test(multishot): price judge calls at the new default judge model
+- fix(judges): one judge default, gpt-6-luna, which the Router serves
+- chore(release): 1.0.23
+- fix(run-record): a Claude id from 4.6 on is a pinned snapshot
+- feat(adopt): expose scoped consumer in managed adoption
+- fix(adopt): stage selected peer cohort before install
+- feat(search): an objective key names the judge revision
+- feat(adopt): scope programmatic adoption to one consumer
+- feat: size a sequential pair cap under a registered law (#1223)
+- fix(test): bound shared release verification workers (#1225)
+- fix(test): qualify refreshed harness contracts with maintained pnpm (#1224)
+- chore(release): 1.0.22 (#1222)
+- chore(deps): refresh first-party library dependencies (#1221)
+- docs(release): describe independent interval API and drop duplicate bumps
+- chore(release): 1.0.21
+- docs(test): cite printed Newcombe table pages
+- feat(statistics): add independent risk differences from Wilson limits
+- fix(test): mark disposable identities for host Git hooks
+- fix(test): declare identity in disposable integration repositories
+- fix(release): authenticate public evidence reads in verification
+- fix(statistics): exclude steps through every selected cluster extent
+- test(statistics): retain overlapping-cluster sealed-step regression
+- docs(release): remove redundant release commit entries
+- chore(release): 1.0.20
+- docs(statistics): trim whitespace in interval guidance
+- fix(verification): retain valid replay baselines and current release evidence
+- chore(analyst): refresh manifest for shared statistics owner
+- fix(statistics): preserve degenerate bootstrap results and mean seed
+- feat(statistics): share stratified cluster resampling and calibrated placebo bands
+- fix(adoption): retain unselected pnpm resolutions
+- fix(exports): ship ./statistics subpath, gpt-5-mini pricing, sequentialCrossingHorizon
+- fix(adopt-stack): qualify the committed caller tree
+- fix(adopt-stack): read large tracked path inventories
+- fix(adoption): read a Runtime release's evidence from its Publish run, and raise pins below a dependency floor (#1211)
+- fix(adoption): hold a Runtime release whose tag evidence failed or never ran
+- fix(adoption): gate stable-major candidates against installed versions
+- feat(app-grade): probes take a method and JSON body (#1208)
+
+## [1.0.26] — 2026-11-08
+
+- feat(benchmarks): grade a code patch with a Harbor task's own verifier
+- chore(release): 1.0.25
+- fix(judges): default to zai/glm-5.3, which every restricted Router key may call
+- chore(release): 1.0.24
+- test(multishot): judge default model and its price in runJudge
+- test(multishot): price judge calls at the new default judge model
+- fix(judges): one judge default, gpt-6-luna, which the Router serves
+- chore(release): 1.0.23
+- fix(run-record): a Claude id from 4.6 on is a pinned snapshot
+- feat(adopt): expose scoped consumer in managed adoption
+- fix(adopt): stage selected peer cohort before install
+- feat(search): an objective key names the judge revision
+- feat(adopt): scope programmatic adoption to one consumer
+- feat: size a sequential pair cap under a registered law (#1223)
+- fix(test): bound shared release verification workers (#1225)
+- fix(test): qualify refreshed harness contracts with maintained pnpm (#1224)
+- chore(release): 1.0.22 (#1222)
+- chore(deps): refresh first-party library dependencies (#1221)
+- docs(release): describe independent interval API and drop duplicate bumps
+- chore(release): 1.0.21
+- docs(test): cite printed Newcombe table pages
+- feat(statistics): add independent risk differences from Wilson limits
+- fix(test): mark disposable identities for host Git hooks
+- fix(test): declare identity in disposable integration repositories
+- fix(release): authenticate public evidence reads in verification
+- fix(statistics): exclude steps through every selected cluster extent
+- test(statistics): retain overlapping-cluster sealed-step regression
+- docs(release): remove redundant release commit entries
+- chore(release): 1.0.20
+- docs(statistics): trim whitespace in interval guidance
+- fix(verification): retain valid replay baselines and current release evidence
+- chore(analyst): refresh manifest for shared statistics owner
+- fix(statistics): preserve degenerate bootstrap results and mean seed
+- feat(statistics): share stratified cluster resampling and calibrated placebo bands
+- fix(adoption): retain unselected pnpm resolutions
+- fix(exports): ship ./statistics subpath, gpt-5-mini pricing, sequentialCrossingHorizon
+- fix(adopt-stack): qualify the committed caller tree
+- fix(adopt-stack): read large tracked path inventories
+- fix(adoption): read a Runtime release's evidence from its Publish run, and raise pins below a dependency floor (#1211)
+- fix(adoption): hold a Runtime release whose tag evidence failed or never ran
+- fix(adoption): gate stable-major candidates against installed versions
+- feat(app-grade): probes take a method and JSON body (#1208)
+
 ## [1.0.25] — 2026-11-08
 
 - fix(judges): default to zai/glm-5.3, which every restricted Router key may call

package/dist/app-grade/index.d.ts

@@ -1,5 +1,5 @@
 import { m as CustomTokenPricing, t as CostLedger } from "../cost-ledger-Czro9wmo.js";
-import "../index-JPI2spOx.js";
+import "../index-C8a9Dyan.js";
 import { O as ChatClient, j as ChatTransport } from "../cluster-CSd0LE_M.js";
 import { d as SystemOneRequest, g as SystemOneState, s as SystemOneQuestions } from "../systemone-protocol-C4t7JH4Y.js";
 import "../systemone-CSwi1hp-.js";

package/dist/benchmarks/index.d.ts

@@ -1,8 +1,8 @@
 import { c as RunSplitTag } from "../run-record-pBlhZizT.js";
 import "../index-BI9ecONy.js";
-import { a as CampaignResult } from "../types-ByrbgALT.js";
+import { S as JudgeConfig, a as CampaignResult } from "../types-ByrbgALT.js";
 import { Wt as CampaignStorage } from "../search-kernel-C4XnLKIW.js";
-import { a as BenchmarkFamily, c as BenchmarkSource, i as BenchmarkEvaluation, l as BenchmarkTaskKind, n as BenchmarkAdapter, o as BenchmarkResponder, r as BenchmarkDatasetItem, s as BenchmarkScenario, t as BENCHMARK_SPLIT_SEED, u as deterministicSplit } from "../types-D7g2XFTc.js";
+import { a as BenchmarkFamily, c as BenchmarkSource, i as BenchmarkEvaluation, l as BenchmarkTaskKind, n as BenchmarkAdapter, o as BenchmarkResponder, r as BenchmarkDatasetItem, s as BenchmarkScenario, t as BENCHMARK_SPLIT_SEED, u as deterministicSplit } from "../types-CEgIBy0W.js";
 interface BenchmarkMetricCalibrationOptions<TPayload = unknown, TArtifact = string> {
   adapter: BenchmarkAdapter<BenchmarkDatasetItem<TPayload>, TPayload, TArtifact>;
   item: BenchmarkDatasetItem<TPayload>;
@@ -88,6 +88,29 @@ interface HarborBenchmarkAdapter<TArtifact> extends BenchmarkAdapter<HarborTaskI
 }
 /** Import a Harbor dataset checkout as a `BenchmarkAdapter` whose scorer is the dataset's own verifier. */
 export declare function createHarborBenchmarkAdapter<TArtifact = string>(options: HarborBenchmarkOptions<TArtifact>): HarborBenchmarkAdapter<TArtifact>;
+/**
+ * Defaults: `retries` 2 (extra runs after one with no reward), `maxDetailChars` 2000 (bounds
+ * `JSON.stringify(detail).length`), `keepImage` true: `--no-delete` keeps the image after a run,
+ * yet a later run builds it again unless `task.toml` names a prebuilt `docker_image`.
+ */
+interface HarborPatchGradeOptions {
+  harborBin?: string;
+  jobsDir?: string;
+  retries?: number;
+  maxDetailChars?: number;
+  timeoutMs?: number;
+  env?: Record<string, string>;
+  keepImage?: boolean;
+}
+/**
+ * A `grade` callback that scores `{ patch }` with the task's own verifier: Harbor's oracle agent
+ * runs a `solution/solve.sh` that applies the patch (`git apply --binary`) in the environment's
+ * working directory, on a copy of the task. An empty patch changes nothing. A run with no reward
+ * is retried, then reported as `succeeded: false`, never as a reward of 0.
+ */
+export declare function gradeHarborPatch(options?: HarborPatchGradeOptions): (item: HarborTaskItem, artifact: {
+  patch: string;
+}) => Promise<HarborGradeOutcome>;
 /**
  * Synthetic routing dataset. 16 tasks across 4 categories. Used as a
  * deterministic, dependency-free benchmark for any router that maps a
@@ -209,6 +232,8 @@ interface BenchmarkRunResult<TPayload = unknown, TArtifact = string> {
   reportMarkdownPath: string;
 }
 export declare function runBenchmarkAdapter<TPayload = unknown, TArtifact = string>(options: BenchmarkRunOptions<TPayload, TArtifact>): Promise<BenchmarkRunResult<TPayload, TArtifact>>;
+export declare function benchmarkScenarios<TPayload>(adapter: BenchmarkAdapter<BenchmarkDatasetItem<TPayload>, TPayload, unknown>, splits?: readonly RunSplitTag[]): Promise<Array<BenchmarkScenario<TPayload>>>;
+export declare function benchmarkJudge<TPayload, TArtifact>(adapter: BenchmarkAdapter<BenchmarkDatasetItem<TPayload>, TPayload, TArtifact>): JudgeConfig<TArtifact, BenchmarkScenario<TPayload>>;
 interface StandardRetrievalDocument {
   id: string;
   title?: string;
@@ -284,5 +309,5 @@ export declare function evaluateStandardRetrieval(payload: StandardRetrievalPayl
     expectedScores: Record<string, number>;
   };
 };
-export { BENCHMARK_SPLIT_SEED, type BenchmarkAdapter, type BenchmarkDatasetItem, type BenchmarkDistribution, type BenchmarkEvaluation, type BenchmarkFamily, type BenchmarkMetricCalibrationOptions, type BenchmarkMetricCalibrationResult, type BenchmarkReport, type BenchmarkResponder, type BenchmarkRunOptions, type BenchmarkRunResult, type BenchmarkScenario, type BenchmarkSliceSummary, type BenchmarkSource, type BenchmarkTaskKind, type BuildStandardRetrievalItemsOptions, type HarborBenchmarkAdapter, type HarborBenchmarkOptions, type HarborGradeOutcome, type HarborTaskItem, type HarborTaskPayload, type RetrievalIdAdapterOptions, type StandardRetrievalArtifact, type StandardRetrievalDocument, type StandardRetrievalEvaluationOptions, type StandardRetrievalPayload, type StandardRetrievalQrel, type StandardRetrievalQuery, type StandardRetrievalResult, deterministicSplit, index_d_exports as routing };
+export { BENCHMARK_SPLIT_SEED, type BenchmarkAdapter, type BenchmarkDatasetItem, type BenchmarkDistribution, type BenchmarkEvaluation, type BenchmarkFamily, type BenchmarkMetricCalibrationOptions, type BenchmarkMetricCalibrationResult, type BenchmarkReport, type BenchmarkResponder, type BenchmarkRunOptions, type BenchmarkRunResult, type BenchmarkScenario, type BenchmarkSliceSummary, type BenchmarkSource, type BenchmarkTaskKind, type BuildStandardRetrievalItemsOptions, type HarborBenchmarkAdapter, type HarborBenchmarkOptions, type HarborGradeOutcome, type HarborPatchGradeOptions, type HarborTaskItem, type HarborTaskPayload, type RetrievalIdAdapterOptions, type StandardRetrievalArtifact, type StandardRetrievalDocument, type StandardRetrievalEvaluationOptions, type StandardRetrievalPayload, type StandardRetrievalQrel, type StandardRetrievalQuery, type StandardRetrievalResult, deterministicSplit, index_d_exports as routing };
 //# sourceMappingURL=index.d.ts.map

package/dist/benchmarks/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","names":[],"sources":["../../src/benchmarks/calibration.ts","../../src/benchmarks/harbor.ts","../../src/benchmarks/routing/dataset.ts","../../src/benchmarks/routing/index.ts","../../src/benchmarks/runner.ts","../../src/benchmarks/standard-formats.ts"],"mappings":";;;;;UAEiB,kCAAkC,oBAAoB;EACrE,SAAS,iBAAiB,qBAAqB,WAAW,UAAU;EACpE,MAAM,qBAAqB;EAC3B,cAAc;EACd,gBAAgB;EAChB;EACA;EACA;;UAGe;EACf;EACA,MAAM;EACN,QAAQ;EACR;EACA;EACA;EACA;;wBAGoB,yBAAyB,oBAAoB,oBACjE,SAAS,kCAAkC,UAAU,aACpD,QAAQ;;UCKM;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;IACE;IACA;IACA;IACA;;;EAGF;;EAEA;;EAEA;;EAEA;;KAGU,iBAAiB,qBAAqB;;KAGtC;EACN;EAAiB;IAAS;IAAgB;;;EAC1C;EAAkB;;UAEP,uBAAuB;;EAEtC;;EAEA;;EAEA,QAAQ;IAAoB;;;EAE5B;;EAEA,QAAQ,MAAM,gBAAgB,UAAU,cAAc,QAAQ;;EAE9D,eAAe,mBAAmB;;;UAInB,uBAAuB,mBAC9B,iBAAiB,gBAAgB,mBAAmB;EAC5D,WAAW,QAAQ;;;wBA0DL,6BAA6B,oBAC3C,SAAS,uBAAuB,aAC/B,uBAAuB;;;;;;;;;;;;;;;;;;;;;;UC5HT;EACf;EACA;EACA;;EAEA;;EAEA;;EAEA;;cAGW,iBAAiB;;;;KChBlB,iBAAiB;KACjB,qBAAqB,qBAAqB;cAEhD,0BAA0B,iBAAiB,oBAAoB;WAC1D;WACA;WACA;WACA;WACA;EAEH,YAAY,OAAO,cAAc,QAAQ;EAMzC,SAAS,MAAM,oBAAoB,mBAAmB,QAAQ;EAqBpE,YAAY,iBAAiB;;;;;;;;iBAef,mBAAmB;cAOtB,cAAW,OAjDG,gBAAc,QAAQ;cAkDpC,WAAQ,MA5CE,oBAAkB,qBAAqB,QAAQ;cA6CzD,cAAW,mBAxBO;UChCd,oBAAoB,oBAAoB;EACvD,SAAS,iBAAiB,qBAAqB,WAAW,UAAU;EACpE,SAAS,mBAAmB,UAAU;EACtC,kBAAkB;EAClB;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA,UAAU;EACV,YAAY;;UAGG;EACf;EACA,QAAQ;EACR,UAAU;EACV,SAAS;EACT;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA,QAAQ,eAAe;EACvB,MAAM,eAAe;EACrB,YAAY,eAAe;EAC3B,OAAO;EACP,SAAS;EACT,WAAW;;UAGI;EACf;EACA;EACA;EACA,OAAO;EACP,SAAS;EACT,WAAW;;UAGI;EACf;EACA;EACA;EACA;EACA;EACA;;UAGe,mBAAmB,oBAAoB;EACtD,WAAW,MAAM,kBAAkB;EACnC,UAAU,eAAe,WAAW,kBAAkB;EACtD,QAAQ;EACR;EACA;;wBAGoB,oBAAoB,oBAAoB,oBAC5D,SAAS,oBAAoB,UAAU,aACtC,QAAQ,mBAAmB,UAAU;UClFvB;EACf;EACA;EACA;EACA,WAAW;;UAGI;EACf;EACA;EACA,WAAW;;UAGI;EACf;EACA;EACA;;UAGe;EACf;EACA;EACA;EACA,gBAAgB;EAChB,SAAS,eAAe;EACxB,WAAW;;UAGI;EACf;EACA,QAAQ;EACR,kBAAkB;EAClB,gBAAgB;EAChB,kBAAkB;EAClB;EACA,SAAS;EACT;EACA,WAAW,oBAAoB;;UAGhB,kCAAkC;EACjD,oBAAoB;EACpB;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA;EACA;;KAGU,kEAGC;EAEP;EACA;EACA,mBAAmB;;UAGR;EACf,oBAAoB;EACpB;EACA;EACA;EACA;;wBA0Bc,WAAW,eAAe;wBAgB1B,qBAAqB,eAAe;wBAgBpC,sBAAsB,eAAe;wBAerC,4BACd,SAAS,qCACR,MAAM,qBAAqB;wBAuCd,kCACd,SAAS,4BACR,iBACD,qBAAqB,2BACrB,0BACA;wBAqBc,0BACd,SAAS,0BACT,UAAU,2BACV,UAAS"}
+{"version":3,"file":"index.d.ts","names":[],"sources":["../../src/benchmarks/calibration.ts","../../src/benchmarks/harbor.ts","../../src/benchmarks/routing/dataset.ts","../../src/benchmarks/routing/index.ts","../../src/benchmarks/runner.ts","../../src/benchmarks/standard-formats.ts"],"mappings":";;;;;UAEiB,kCAAkC,oBAAoB;EACrE,SAAS,iBAAiB,qBAAqB,WAAW,UAAU;EACpE,MAAM,qBAAqB;EAC3B,cAAc;EACd,gBAAgB;EAChB;EACA;EACA;;UAGe;EACf;EACA,MAAM;EACN,QAAQ;EACR;EACA;EACA;EACA;;wBAGoB,yBAAyB,oBAAoB,oBACjE,SAAS,kCAAkC,UAAU,aACpD,QAAQ;;UCWM;;EAEf;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;;EAEA;IACE;IACA;IACA;IACA;;;EAGF;;EAEA;;EAEA;;EAEA;;KAGU,iBAAiB,qBAAqB;;KAGtC;EACN;EAAiB;IAAS;IAAgB;;;EAC1C;EAAkB;;UAEP,uBAAuB;;EAEtC;;EAEA;;EAEA,QAAQ;IAAoB;;;EAE5B;;EAEA,QAAQ,MAAM,gBAAgB,UAAU,cAAc,QAAQ;;EAE9D,eAAe,mBAAmB;;;UAInB,uBAAuB,mBAC9B,iBAAiB,gBAAgB,mBAAmB;EAC5D,WAAW,QAAQ;;;wBA0DL,6BAA6B,oBAC3C,SAAS,uBAAuB,aAC/B,uBAAuB;;;;;;UAgFT;EACf;EACA;EACA;EACA;EACA;EACA,MAAM;EACN;;;;;;;;wBAoBc,iBACd,UAAS,2BACP,MAAM,gBAAgB;EAAY;MAAoB,QAAQ;;;;;;;;;;;;;;;;;;;;;;UC/OjD;EACf;EACA;EACA;;EAEA;;EAEA;;EAEA;;cAGW,iBAAiB;;;;KChBlB,iBAAiB;KACjB,qBAAqB,qBAAqB;cAEhD,0BAA0B,iBAAiB,oBAAoB;WAC1D;WACA;WACA;WACA;WACA;EAEH,YAAY,OAAO,cAAc,QAAQ;EAMzC,SAAS,MAAM,oBAAoB,mBAAmB,QAAQ;EAqBpE,YAAY,iBAAiB;;;;;;;;iBAef,mBAAmB;cAOtB,cAAW,OAjDG,gBAAc,QAAQ;cAkDpC,WAAQ,MA5CE,oBAAkB,qBAAqB,QAAQ;cA6CzD,cAAW,mBAxBO;UChCd,oBAAoB,oBAAoB;EACvD,SAAS,iBAAiB,qBAAqB,WAAW,UAAU;EACpE,SAAS,mBAAmB,UAAU;EACtC,kBAAkB;EAClB;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA,UAAU;EACV,YAAY;;UAGG;EACf;EACA,QAAQ;EACR,UAAU;EACV,SAAS;EACT;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA,QAAQ,eAAe;EACvB,MAAM,eAAe;EACrB,YAAY,eAAe;EAC3B,OAAO;EACP,SAAS;EACT,WAAW;;UAGI;EACf;EACA;EACA;EACA,OAAO;EACP,SAAS;EACT,WAAW;;UAGI;EACf;EACA;EACA;EACA;EACA;EACA;;UAGe,mBAAmB,oBAAoB;EACtD,WAAW,MAAM,kBAAkB;EACnC,UAAU,eAAe,WAAW,kBAAkB;EACtD,QAAQ;EACR;EACA;;wBAGoB,oBAAoB,oBAAoB,oBAC5D,SAAS,oBAAoB,UAAU,aACtC,QAAQ,mBAAmB,UAAU;wBA0ClB,mBAAmB,UACvC,SAAS,iBAAiB,qBAAqB,WAAW,oBAC1D,kBAAiB,gBAChB,QAAQ,MAAM,kBAAkB;wBA6BnB,eAAe,UAAU,WACvC,SAAS,iBAAiB,qBAAqB,WAAW,UAAU,aACnE,YAAY,WAAW,kBAAkB;UC9J3B;EACf;EACA;EACA;EACA,WAAW;;UAGI;EACf;EACA;EACA,WAAW;;UAGI;EACf;EACA;EACA;;UAGe;EACf;EACA;EACA;EACA,gBAAgB;EAChB,SAAS,eAAe;EACxB,WAAW;;UAGI;EACf;EACA,QAAQ;EACR,kBAAkB;EAClB,gBAAgB;EAChB,kBAAkB;EAClB;EACA,SAAS;EACT;EACA,WAAW,oBAAoB;;UAGhB,kCAAkC;EACjD,oBAAoB;EACpB;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA;EACA;;KAGU,kEAGC;EAEP;EACA;EACA,mBAAmB;;UAGR;EACf,oBAAoB;EACpB;EACA;EACA;EACA;;wBA0Bc,WAAW,eAAe;wBAgB1B,qBAAqB,eAAe;wBAgBpC,sBAAsB,eAAe;wBAerC,4BACd,SAAS,qCACR,MAAM,qBAAqB;wBAuCd,kCACd,SAAS,4BACR,iBACD,qBAAqB,2BACrB,0BACA;wBAqBc,0BACd,SAAS,0BACT,UAAU,2BACV,UAAS"}

package/dist/benchmarks/index.js

@@ -3,8 +3,12 @@ import { at as summarizeNumberSeries, ct as mean } from "../statistics-CvvC-MPv.
 import { Z as runCampaign } from "../llm-judge-DzAT7Wka.js";
 import { M as fsCampaignStorage } from "../external-optimizer-subprocess-GO6s84Ov.js";
 import "../campaign-Cqz2vk2p.js";
+import { randomUUID } from "node:crypto";
 import { join, resolve } from "node:path";
-import { readFile, readdir, stat } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { execFile } from "node:child_process";
+import { cp, mkdir, mkdtemp, readFile, readdir, rm, stat, writeFile } from "node:fs/promises";
+import { promisify, stripVTControlCharacters } from "node:util";
 import { parse } from "smol-toml";
 // src/benchmarks/calibration.ts
 async function calibrateBenchmarkMetric(options) {
@@ -56,16 +60,19 @@ function fnv1a32(input) {
 const BENCHMARK_SPLIT_SEED = "agent-eval-v1";
 /**
 * Assign an item id to one of `'search' | 'dev' | 'holdout'` using a
-* stable 32-bit hash of `${seed}::${id}`. Default proportions:
+* stable 32-bit hash of `${seed}::${id}`. `cuts` [a, b] with 0 < a < b < 1 put
+* positions below a in search, below b in dev, the rest in holdout. Default [0.6, 0.8]:
 *
 *   search: 60%   (optimization-readable)
 *   dev:    20%   (held-out for tuning, leak-on-purpose during dev)
 *   holdout:20%   (paper-grade held-out, gated reads)
 */
-function deterministicSplit(itemId, seed = BENCHMARK_SPLIT_SEED) {
+function deterministicSplit(itemId, seed = BENCHMARK_SPLIT_SEED, cuts = [.6, .8]) {
+	const [a, b] = cuts;
+	if (!(a > 0 && a < b && b < 1)) throw new RangeError(`split cuts need 0 < a < b < 1, got [${a}, ${b}]`);
 	const pos = fnv1a32(`${seed}::${itemId}`) / 4294967296;
-	if (pos < .6) return "search";
-	if (pos < .8) return "dev";
+	if (pos < a) return "search";
+	if (pos < b) return "dev";
 	return "holdout";
 }
 // src/benchmarks/harbor.ts
@@ -78,10 +85,11 @@ function deterministicSplit(itemId, seed = BENCHMARK_SPLIT_SEED) {
 * revision the caller pins in `source`, into `BenchmarkDatasetItem`s.
 *
 * Scoring stays Harbor's. `evaluate` hands the task and the agent's artifact to the caller's
-* `grade` callback, which runs the task's own verifier (for example `harbor run -p <taskDir>`
-* with an agent that installs the collected artifacts) and reports its reward. This package does
-* not start containers. A verifier that did not run is an error, never a score of zero.
+* `grade` callback, which runs the task's own verifier and reports its reward. `gradeHarborPatch`
+* is that callback for a code patch: it spawns the Harbor CLI, which starts the containers; this
+* package starts none itself. A verifier that did not run is an error, never a score of zero.
 */
+const execute = promisify(execFile);
 const table = (value) => value !== null && typeof value === "object" && !Array.isArray(value) ? value : {};
 const numberOrNull = (value) => typeof value === "number" && Number.isFinite(value) ? value : null;
 const textOrNull = (value) => typeof value === "string" && value.length > 0 ? value : null;
@@ -163,12 +171,92 @@ function createHarborBenchmarkAdapter(options) {
 				raw: {
 					reward,
 					...detail === void 0 ? {} : { detail }
-				}
+				},
+				...detail === void 0 ? {} : { notes: detail }
 			};
 		},
 		assignSplit: split
 	};
 }
+const SOLVE_SH = "#!/bin/bash\nset -euo pipefail\npatch=\"$(dirname \"$0\")/patch.diff\"\nif [ -s \"$patch\" ]; then git apply --binary \"$patch\"; fi\n";
+/** The last characters of `text` without ANSI codes, so that its JSON string fits in `max`. */
+function detailTail(text, max) {
+	let tail = stripVTControlCharacters(text).slice(-max);
+	while (tail && JSON.stringify(tail).length > max) tail = tail.slice(1);
+	return tail;
+}
+/**
+* A `grade` callback that scores `{ patch }` with the task's own verifier: Harbor's oracle agent
+* runs a `solution/solve.sh` that applies the patch (`git apply --binary`) in the environment's
+* working directory, on a copy of the task. An empty patch changes nothing. A run with no reward
+* is retried, then reported as `succeeded: false`, never as a reward of 0.
+*/
+function gradeHarborPatch(options = {}) {
+	const { harborBin = "harbor", retries = 2, maxDetailChars = 2e3, keepImage = true } = options;
+	const { timeoutMs: timeout, env: extra } = options;
+	const run = {
+		env: {
+			...process.env,
+			...extra
+		},
+		timeout,
+		maxBuffer: 1 << 26
+	};
+	return async (item, artifact) => {
+		const work = await mkdtemp(join(tmpdir(), "harbor-grade-"));
+		try {
+			const task = join(work, item.payload.dir);
+			await cp(item.payload.taskDir, task, { recursive: true });
+			await rm(join(task, "solution"), {
+				recursive: true,
+				force: true
+			});
+			await mkdir(join(task, "solution"));
+			await writeFile(join(task, "solution/patch.diff"), artifact.patch);
+			await writeFile(join(task, "solution/solve.sh"), SOLVE_SH, { mode: 493 });
+			const jobsDir = resolve(options.jobsDir ?? join(work, "jobs"));
+			const errors = [];
+			const args = [
+				"run",
+				"-p",
+				task,
+				"-a",
+				"oracle",
+				"-o",
+				jobsDir,
+				"-q",
+				keepImage ? "--no-delete" : "--delete",
+				"--job-name"
+			];
+			for (let attempt = 1; attempt <= retries + 1; attempt += 1) {
+				const job = `${item.payload.dir}-${randomUUID()}`;
+				const failure = await execute(harborBin, [...args, job], run).then(() => null, (error) => detailTail(error.message, 500));
+				const trial = (await readdir(join(jobsDir, job), { withFileTypes: true }).catch(() => [])).filter((entry) => entry.isDirectory()).map((entry) => join(jobsDir, job, entry.name, "/"))[0];
+				const read = (path) => readFile(`${trial}${path}`, "utf8").catch(() => "");
+				const result = table(trial === void 0 ? {} : JSON.parse(await read("result.json") || "{}"));
+				const reward = numberOrNull(table(table(result.verifier_result).rewards).reward);
+				if (reward !== null) return {
+					succeeded: true,
+					value: {
+						reward,
+						detail: detailTail(await read("verifier/test-stdout.txt"), maxDetailChars)
+					}
+				};
+				const exception = textOrNull(table(result.exception_info).exception_message);
+				errors.push(`attempt ${attempt}: ${failure ?? exception ?? "harbor recorded no reward"}`);
+			}
+			return {
+				succeeded: false,
+				error: errors.join("; ")
+			};
+		} finally {
+			await rm(work, {
+				recursive: true,
+				force: true
+			});
+		}
+	};
+}
 // src/benchmarks/routing/dataset.ts
 const ROUTING_DATASET = [
 	{
@@ -362,8 +450,8 @@ const assignSplit = adapter.assignSplit.bind(adapter);
 async function runBenchmarkAdapter(options) {
 	const storage = options.storage ?? fsCampaignStorage();
 	const benchmarkId = benchmarkIdFor(options.adapter);
-	const scenarios = await loadBenchmarkScenarios(options.adapter, options.splits);
-	const judge = benchmarkAdapterJudge(options.adapter);
+	const scenarios = await benchmarkScenarios(options.adapter, options.splits);
+	const judge = benchmarkJudge(options.adapter);
 	const dispatch = async (scenario, context) => {
 		return options.respond({
 			scenario,
@@ -406,7 +494,7 @@ async function runBenchmarkAdapter(options) {
 		reportMarkdownPath
 	};
 }
-async function loadBenchmarkScenarios(adapter, splits = [
+async function benchmarkScenarios(adapter, splits = [
 	"search",
 	"dev",
 	"holdout"
@@ -438,7 +526,7 @@ async function loadBenchmarkScenarios(adapter, splits = [
 	}
 	return out;
 }
-function benchmarkAdapterJudge(adapter) {
+function benchmarkJudge(adapter) {
 	const benchmarkId = benchmarkIdFor(adapter);
 	return {
 		name: `${benchmarkId}:score`,
@@ -821,6 +909,6 @@ function discountedGain(relevance, zeroBasedRank) {
 	if (relevance <= 0) return 0;
 	return (2 ** relevance - 1) / Math.log2(zeroBasedRank + 2);
 }
-export { BENCHMARK_SPLIT_SEED, buildStandardRetrievalItems, calibrateBenchmarkMetric, createHarborBenchmarkAdapter, createRetrievalIdBenchmarkAdapter, deterministicSplit, evaluateStandardRetrieval, parseBeirCorpusJsonl, parseBeirQueriesJsonl, parseQrels, routing_exports as routing, runBenchmarkAdapter };
+export { BENCHMARK_SPLIT_SEED, benchmarkJudge, benchmarkScenarios, buildStandardRetrievalItems, calibrateBenchmarkMetric, createHarborBenchmarkAdapter, createRetrievalIdBenchmarkAdapter, deterministicSplit, evaluateStandardRetrieval, gradeHarborPatch, parseBeirCorpusJsonl, parseBeirQueriesJsonl, parseQrels, routing_exports as routing, runBenchmarkAdapter };
 
 //# sourceMappingURL=index.js.map