@tangle-network/agent-eval 0.145.3 → 0.145.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/CHANGELOG.md +13 -0
  2. package/dist/analyst/index.d.ts +1 -1
  3. package/dist/analyst/index.js +1 -1
  4. package/dist/{benchmark-command-Iaqlp3DU.js → benchmark-command-BANo4dTm.js} +2 -2
  5. package/dist/{benchmark-command-Iaqlp3DU.js.map → benchmark-command-BANo4dTm.js.map} +1 -1
  6. package/dist/benchmarks/index.d.ts +2 -2
  7. package/dist/benchmarks/index.js +2 -2
  8. package/dist/campaign/index.d.ts +4 -4
  9. package/dist/campaign/index.js +4 -4
  10. package/dist/{campaign-C5kujyqv.js → campaign-FY23WYzZ.js} +503 -24
  11. package/dist/campaign-FY23WYzZ.js.map +1 -0
  12. package/dist/cli.js +1 -1
  13. package/dist/contract/index.d.ts +3 -3
  14. package/dist/contract/index.js +3 -3
  15. package/dist/{define-agent-eval-sH24zBfM.d.ts → define-agent-eval-DcAcC1yQ.d.ts} +2 -2
  16. package/dist/{define-agent-eval-sH24zBfM.d.ts.map → define-agent-eval-DcAcC1yQ.d.ts.map} +1 -1
  17. package/dist/{define-agent-eval-CAL5HOz9.js → define-agent-eval-xaceIpPS.js} +2 -2
  18. package/dist/{define-agent-eval-CAL5HOz9.js.map → define-agent-eval-xaceIpPS.js.map} +1 -1
  19. package/dist/{index-CUczrQ7a.d.ts → index-Dytg99g1.d.ts} +101 -4
  20. package/dist/index-Dytg99g1.d.ts.map +1 -0
  21. package/dist/index.d.ts +2 -2
  22. package/dist/index.js +3 -3
  23. package/dist/{llm-judge-C2lQBv7h.js → llm-judge-BPKBdQTp.js} +7 -4
  24. package/dist/llm-judge-BPKBdQTp.js.map +1 -0
  25. package/dist/openapi.json +1 -1
  26. package/dist/{produced-state-Bgph_Qj-.js → produced-state-B_-TW9o0.js} +2 -2
  27. package/dist/{produced-state-Bgph_Qj-.js.map → produced-state-B_-TW9o0.js.map} +1 -1
  28. package/dist/{provenance-CCdxgLDT.d.ts → provenance-C_ln89b0.d.ts} +15 -1
  29. package/dist/provenance-C_ln89b0.d.ts.map +1 -0
  30. package/dist/{skillopt-optimization-method-C5cotF4E.d.ts → skillopt-optimization-method-CkySQ0ca.d.ts} +2 -2
  31. package/dist/{skillopt-optimization-method-C5cotF4E.d.ts.map → skillopt-optimization-method-CkySQ0ca.d.ts.map} +1 -1
  32. package/dist/{skillopt-optimization-method-B__Jnto4.js → skillopt-optimization-method-CxrOjp1S.js} +2 -2
  33. package/dist/{skillopt-optimization-method-B__Jnto4.js.map → skillopt-optimization-method-CxrOjp1S.js.map} +1 -1
  34. package/docs/eval-surface-map.md +11 -0
  35. package/package.json +1 -1
  36. package/dist/campaign-C5kujyqv.js.map +0 -1
  37. package/dist/index-CUczrQ7a.d.ts.map +0 -1
  38. package/dist/llm-judge-C2lQBv7h.js.map +0 -1
  39. package/dist/provenance-CCdxgLDT.d.ts.map +0 -1
@@ -1,20 +1,20 @@
1
1
  import { s as ValidationError, t as AgentEvalError } from "./errors-Dngq5h35.js";
2
- import { c as agentProfileId, d as harnessAxisOf, i as verifyCompletion, l as agentProfileModelId, s as agentProfileHash, t as extractProducedState } from "./produced-state-Bgph_Qj-.js";
2
+ import { c as agentProfileId, d as harnessAxisOf, i as verifyCompletion, l as agentProfileModelId, s as agentProfileHash, t as extractProducedState } from "./produced-state-B_-TW9o0.js";
3
3
  import { buildAgentProfileCell } from "./profile-cell.js";
4
4
  import { t as comparePairedArms } from "./paired-arms-D-XRF_fy.js";
5
5
  import { r as pairedBootstrap } from "./paired-tests-BHIhYVdu.js";
6
6
  import { o as runTaskScore, r as modelHasSnapshot, s as validateRunRecord } from "./run-record-BvHPVS-i.js";
7
- import { n as contentHash } from "./verdict-cache-mZf5FEiY.js";
8
- import { K as runCampaign, Q as summarizeBackendIntegrity, Z as assertRealBackend, h as labelTrustRank, k as surfaceContentHash, q as planCampaignRun, w as assertCodeSurfaceIdentity } from "./llm-judge-C2lQBv7h.js";
9
- import { l as campaignCellToRunRecord } from "./reward-hacking-DNgjilrV.js";
7
+ import { n as contentHash, t as canonicalJson } from "./verdict-cache-mZf5FEiY.js";
8
+ import { $ as assertCampaignDesign, J as resolveRunDir, K as runCampaign, Q as summarizeBackendIntegrity, Z as assertRealBackend, at as computeManifestHash, h as labelTrustRank, it as cellCachePath, k as surfaceContentHash, q as planCampaignRun, tt as campaignScenarioIdentity, w as assertCodeSurfaceIdentity } from "./llm-judge-BPKBdQTp.js";
9
+ import { a as campaignCellCostProvenance, l as campaignCellToRunRecord, u as projectCampaignCellQuality } from "./reward-hacking-DNgjilrV.js";
10
10
  import { t as CostAccountingIncompleteError } from "./cost-ledger-BSe92yAV.js";
11
11
  import { t as FileLedgerJournal, u as mapConcurrent } from "./ledger-core-BmZt19oQ.js";
12
12
  import { r as canonicalString } from "./canonical-D-XsTQ6_.js";
13
- import { C as SEARCH_LEDGER_FILE_CONTEXT, E as SearchLedgerIntegrityError, T as SearchLedgerError, w as SearchLedgerConflictError } from "./external-optimizer-subprocess-BhKYK0Jv.js";
13
+ import { C as SEARCH_LEDGER_FILE_CONTEXT, E as SearchLedgerIntegrityError, T as SearchLedgerError, w as SearchLedgerConflictError, x as fsCampaignStorage } from "./external-optimizer-subprocess-BhKYK0Jv.js";
14
14
  import { o as pairHoldout } from "./power-preflight-DEw-uC7q.js";
15
15
  import { a as scoreAnalystFindings } from "./benchmark-BhT16ep9.js";
16
16
  import "./external-optimizer-process-BRE56woM.js";
17
- import "./skillopt-optimization-method-B__Jnto4.js";
17
+ import "./skillopt-optimization-method-CxrOjp1S.js";
18
18
  import { createHash } from "node:crypto";
19
19
  import { closeSync, constants, existsSync, fstatSync, lstatSync, mkdirSync, mkdtempSync, openSync, readFileSync, readSync, readdirSync, readlinkSync, realpathSync, rmSync, statSync, writeFileSync } from "node:fs";
20
20
  import { z } from "zod";
@@ -1490,6 +1490,45 @@ var ProfileMatrixError = class extends AgentEvalError {
1490
1490
  super("profile_matrix", message);
1491
1491
  }
1492
1492
  };
1493
+ /**
1494
+ * Return the exact per-profile campaign identity used by `runProfileMatrix`.
1495
+ * Segmented profile matrices use this to validate durable cell caches before
1496
+ * finalization without dispatching a probe request.
1497
+ */
1498
+ function profileMatrixCampaignIdentity(opts) {
1499
+ const seed = opts.seed ?? 42;
1500
+ const splitTag = opts.splitTag ?? "search";
1501
+ const reps = opts.reps ?? 1;
1502
+ const profileHash = agentProfileHash(opts.profile);
1503
+ const profileId = agentProfileId(opts.profile);
1504
+ const configHash = sha({
1505
+ profile: profileHash,
1506
+ judges: (opts.judges ?? []).map((judge) => judge.name),
1507
+ seed,
1508
+ splitTag,
1509
+ dispatchRef: opts.dispatchRef ?? null
1510
+ });
1511
+ const dispatchRef = `profile-matrix:${sha({
1512
+ commitSha: opts.commitSha,
1513
+ caller: opts.dispatchRef ?? null,
1514
+ profileId,
1515
+ profileHash,
1516
+ configHash
1517
+ })}`;
1518
+ return {
1519
+ profileHash,
1520
+ profileId,
1521
+ configHash,
1522
+ dispatchRef,
1523
+ manifestHash: computeManifestHash({
1524
+ scenarios: opts.scenarios,
1525
+ judges: opts.judges ?? [],
1526
+ dispatchRef,
1527
+ seed,
1528
+ reps
1529
+ })
1530
+ };
1531
+ }
1493
1532
  function sanitize(id) {
1494
1533
  return id.replace(/[^a-zA-Z0-9_-]/g, "_");
1495
1534
  }
@@ -1565,7 +1604,7 @@ async function runProfileMatrix(opts) {
1565
1604
  profileIds,
1566
1605
  scenarios: opts.scenarios.map((s) => s.id)
1567
1606
  }).slice(0, 16)}`;
1568
- const matrixId = `mtx_${sha({
1607
+ const derivedMatrixId = `mtx_${sha({
1569
1608
  experimentId,
1570
1609
  profileIds,
1571
1610
  seed,
@@ -1573,6 +1612,8 @@ async function runProfileMatrix(opts) {
1573
1612
  commitSha: opts.commitSha,
1574
1613
  dispatchRef: opts.dispatchRef ?? null
1575
1614
  }).slice(0, 16)}`;
1615
+ const matrixId = opts.matrixId ?? derivedMatrixId;
1616
+ if (typeof matrixId !== "string" || matrixId.trim().length === 0) throw new ProfileMatrixError("matrixId must be a non-empty string when provided");
1576
1617
  const scenarioById = new Map(opts.scenarios.map((s) => [s.id, s]));
1577
1618
  for (const profile of opts.profiles) {
1578
1619
  const profileHash = agentProfileHash(profile);
@@ -1612,23 +1653,17 @@ async function runProfileMatrix(opts) {
1612
1653
  const campaigns = {};
1613
1654
  const byProfile = {};
1614
1655
  const columns = await mapConcurrent(opts.profiles, maxProfileConcurrency, async (profile, _index, signal) => {
1615
- const profileHash = agentProfileHash(profile);
1616
- const profileId = agentProfileId(profile);
1617
- const declaredModel = agentProfileModelId(profile);
1618
- const configHash = sha({
1619
- profile: profileHash,
1620
- judges: (opts.judges ?? []).map((j) => j.name),
1621
- seed,
1656
+ const { profileHash, profileId, configHash, dispatchRef } = profileMatrixCampaignIdentity({
1657
+ profile,
1658
+ scenarios: opts.scenarios,
1659
+ judges: opts.judges,
1660
+ commitSha: opts.commitSha,
1661
+ dispatchRef: opts.dispatchRef,
1622
1662
  splitTag,
1623
- dispatchRef: opts.dispatchRef ?? null
1663
+ seed,
1664
+ reps: opts.reps
1624
1665
  });
1625
- const dispatchRef = `profile-matrix:${sha({
1626
- commitSha: opts.commitSha,
1627
- caller: opts.dispatchRef ?? null,
1628
- profileId,
1629
- profileHash,
1630
- configHash
1631
- })}`;
1666
+ const declaredModel = agentProfileModelId(profile);
1632
1667
  const dispatch = (scenario, ctx) => opts.dispatch(profile, scenario, ctx);
1633
1668
  Object.defineProperty(dispatch, "name", { value: `profile_${sanitize(profileId)}` });
1634
1669
  const campaign = await runCampaign({
@@ -1641,6 +1676,12 @@ async function runProfileMatrix(opts) {
1641
1676
  reps: opts.reps,
1642
1677
  maxConcurrency: opts.maxConcurrency,
1643
1678
  costCeiling: opts.costCeiling,
1679
+ cellFilter: opts.rowFilter ? ({ scenario, rep }) => opts.rowFilter?.({
1680
+ profile,
1681
+ scenario,
1682
+ rep
1683
+ }) ?? false : void 0,
1684
+ reuseFailedCells: opts.reuseFailedCells,
1644
1685
  labeledStore: opts.labeledStore,
1645
1686
  captureSource: opts.captureSource,
1646
1687
  storage: opts.storage,
@@ -1749,6 +1790,444 @@ function rollupByPersona(records, scenarios, personaOf) {
1749
1790
  return rollup(records, (r) => r.scenarioId ? personaByScenarioId.get(r.scenarioId) : void 0);
1750
1791
  }
1751
1792
  //#endregion
1793
+ //#region src/campaign/presets/segmented-profile-matrix.ts
1794
+ /**
1795
+ * Eval-owned execution for profile matrices whose external authorization is
1796
+ * shorter than the complete matrix.
1797
+ *
1798
+ * A plan freezes the complete profile × scenario × replicate design. A
1799
+ * segment claims an explicit, disjoint set of rows and uses the campaign cell
1800
+ * cache as the durable per-cell record. Finalization first checks that every
1801
+ * planned row has one claim and one valid record, then replays the ordinary
1802
+ * `runProfileMatrix` projection from those records without dispatching work.
1803
+ */
1804
+ const PLAN_FILE = "profile-matrix-plan.json";
1805
+ const SEGMENTS_FILE = "profile-matrix-segments.jsonl";
1806
+ var SegmentedProfileMatrixError = class extends ProfileMatrixError {
1807
+ constructor(message) {
1808
+ super(`segmented matrix: ${message}`);
1809
+ }
1810
+ };
1811
+ function createProfileMatrixPlan(opts) {
1812
+ if (!Array.isArray(opts.profiles) || opts.profiles.length === 0) throw new SegmentedProfileMatrixError("profiles must not be empty");
1813
+ if (!Array.isArray(opts.scenarios) || opts.scenarios.length === 0) throw new SegmentedProfileMatrixError("scenarios must not be empty");
1814
+ const reps = opts.reps ?? 1;
1815
+ const seed = opts.seed ?? 42;
1816
+ const splitTag = opts.splitTag ?? "search";
1817
+ const integrity = opts.integrity ?? "assert";
1818
+ const allowMixed = opts.allowMixed ?? true;
1819
+ const validate = opts.validate ?? true;
1820
+ if (typeof opts.commitSha !== "string" || opts.commitSha.trim().length === 0) throw new SegmentedProfileMatrixError("commitSha must be a non-empty string");
1821
+ if (typeof opts.dispatchRef !== "string" || opts.dispatchRef.trim().length === 0) throw new SegmentedProfileMatrixError("dispatchRef must be a non-empty stable identity for segmented execution");
1822
+ assertCampaignDesign(opts.scenarios, reps);
1823
+ const profiles = [...opts.profiles];
1824
+ const scenarios = [...opts.scenarios];
1825
+ const judges = [...opts.judges ?? []];
1826
+ const profileIds = profiles.map(agentProfileId);
1827
+ const duplicateProfiles = profileIds.filter((id, index) => profileIds.indexOf(id) !== index);
1828
+ if (duplicateProfiles.length > 0) throw new SegmentedProfileMatrixError(`duplicate profile ids: ${[...new Set(duplicateProfiles)].join(", ")}`);
1829
+ const experimentId = opts.experimentId ?? `pm_${contentHash({
1830
+ profileIds,
1831
+ scenarios: scenarios.map((scenario) => scenario.id)
1832
+ }).slice(0, 16)}`;
1833
+ if (experimentId.trim().length === 0) throw new SegmentedProfileMatrixError("experimentId must be non-empty");
1834
+ const planDigest = `sha256:${contentHash({
1835
+ schema: "tangle.profile-matrix-plan.v1",
1836
+ profiles: profiles.map((profile, index) => ({
1837
+ ordinal: index,
1838
+ id: profileIds[index],
1839
+ hash: `sha256:${agentProfileHash(profile)}`,
1840
+ profile
1841
+ })),
1842
+ scenarios: scenarios.map((scenario, index) => ({
1843
+ ordinal: index,
1844
+ identity: campaignScenarioIdentity(scenario),
1845
+ scenario
1846
+ })),
1847
+ judges: judges.map((judge) => ({
1848
+ name: judge.name,
1849
+ dimensions: judge.dimensions,
1850
+ judgeVersion: judge.judgeVersion ?? null,
1851
+ score: judge.score.toString(),
1852
+ appliesTo: judge.appliesTo?.toString() ?? null
1853
+ })),
1854
+ reps,
1855
+ seed,
1856
+ splitTag,
1857
+ commitSha: opts.commitSha,
1858
+ dispatchRef: opts.dispatchRef,
1859
+ experimentId,
1860
+ integrity,
1861
+ allowMixed,
1862
+ validate,
1863
+ personaOf: opts.personaOf?.toString() ?? null,
1864
+ corpusText: opts.corpusText?.toString() ?? null
1865
+ })}`;
1866
+ const matrixId = `pmx_${planDigest.slice(7, 31)}`;
1867
+ const rows = [];
1868
+ const seenCellIds = /* @__PURE__ */ new Set();
1869
+ let ordinal = 0;
1870
+ for (const profileId of profileIds) for (const scenario of scenarios) for (let rep = 0; rep < reps; rep += 1) {
1871
+ const cellId = `${scenario.id}:${rep}`;
1872
+ if (seenCellIds.has(`${profileId}:${cellId}`)) throw new SegmentedProfileMatrixError(`profile/scenario/rep cell id collision for '${profileId}:${cellId}'`);
1873
+ seenCellIds.add(`${profileId}:${cellId}`);
1874
+ rows.push({
1875
+ rowId: `row_${contentHash({
1876
+ matrixId,
1877
+ profileId,
1878
+ scenarioId: scenario.id,
1879
+ rep
1880
+ }).slice(0, 24)}`,
1881
+ ordinal,
1882
+ profileId,
1883
+ scenarioId: scenario.id,
1884
+ rep
1885
+ });
1886
+ ordinal += 1;
1887
+ }
1888
+ return Object.freeze({
1889
+ schemaVersion: 1,
1890
+ matrixId,
1891
+ experimentId,
1892
+ planDigest,
1893
+ profiles: Object.freeze(profiles),
1894
+ scenarios: Object.freeze(scenarios),
1895
+ judges: Object.freeze(judges),
1896
+ reps,
1897
+ seed,
1898
+ splitTag,
1899
+ commitSha: opts.commitSha,
1900
+ dispatchRef: opts.dispatchRef,
1901
+ integrity,
1902
+ allowMixed,
1903
+ validate,
1904
+ ...opts.personaOf === void 0 ? {} : { personaOf: opts.personaOf },
1905
+ ...opts.corpusText === void 0 ? {} : { corpusText: opts.corpusText },
1906
+ rows: Object.freeze(rows)
1907
+ });
1908
+ }
1909
+ async function runProfileMatrixSegment(opts) {
1910
+ const segmentId = requireIdentifier(opts.segmentId, "segmentId");
1911
+ const storage = opts.storage ?? fsCampaignStorage();
1912
+ const runDir = resolveRunDir(opts.runDir);
1913
+ assertPlan(opts.plan);
1914
+ const selected = normalizeRows(opts.plan, opts.rows);
1915
+ ensureManifest(storage, runDir, manifestFor(opts.plan));
1916
+ claimSegment(storage, runDir, opts.plan, segmentId, selected);
1917
+ const selectedIds = new Set(selected.map((row) => row.rowId));
1918
+ const selectedProfileIds = new Set(selected.map((row) => row.profileId));
1919
+ const matrix = await runProfileMatrix({
1920
+ profiles: opts.plan.profiles.filter((profile) => selectedProfileIds.has(agentProfileId(profile))),
1921
+ scenarios: [...opts.plan.scenarios],
1922
+ dispatch: opts.dispatch,
1923
+ judges: [...opts.plan.judges],
1924
+ runDir,
1925
+ commitSha: opts.plan.commitSha,
1926
+ dispatchRef: opts.plan.dispatchRef,
1927
+ experimentId: opts.plan.experimentId,
1928
+ matrixId: opts.plan.matrixId,
1929
+ splitTag: opts.plan.splitTag,
1930
+ reps: opts.plan.reps,
1931
+ seed: opts.plan.seed,
1932
+ integrity: "off",
1933
+ allowMixed: opts.plan.allowMixed,
1934
+ maxConcurrency: opts.maxConcurrency,
1935
+ maxProfileConcurrency: opts.maxProfileConcurrency,
1936
+ costCeiling: opts.costCeiling,
1937
+ labeledStore: opts.labeledStore,
1938
+ captureSource: opts.captureSource,
1939
+ storage,
1940
+ now: opts.now,
1941
+ personaOf: opts.plan.personaOf,
1942
+ validate: opts.plan.validate,
1943
+ corpusText: opts.plan.corpusText,
1944
+ reuseFailedCells: false,
1945
+ rowFilter: ({ profile, scenario, rep }) => selectedIds.has(rowIdFor(opts.plan, agentProfileId(profile), scenario.id, rep))
1946
+ });
1947
+ persistSegmentCells(storage, runDir, matrix);
1948
+ const coverage = readCoverage(storage, runDir, opts.plan);
1949
+ return {
1950
+ segmentId,
1951
+ rowIds: selected.map((row) => row.rowId),
1952
+ matrix,
1953
+ coverage
1954
+ };
1955
+ }
1956
+ async function finalizeProfileMatrix(opts) {
1957
+ const storage = opts.storage ?? fsCampaignStorage();
1958
+ const runDir = resolveRunDir(opts.runDir);
1959
+ assertPlan(opts.plan);
1960
+ ensureManifest(storage, runDir, manifestFor(opts.plan));
1961
+ const claims = readSegmentClaims(storage, runDir);
1962
+ assertCompleteClaims(opts.plan, claims);
1963
+ const coverage = readCoverage(storage, runDir, opts.plan);
1964
+ if (coverage.missing.length > 0) throw new SegmentedProfileMatrixError(`cannot finalize with missing rows: ${coverage.missing.join(", ")}`);
1965
+ const matrix = await runProfileMatrix({
1966
+ profiles: [...opts.plan.profiles],
1967
+ scenarios: [...opts.plan.scenarios],
1968
+ dispatch: async () => {
1969
+ throw new SegmentedProfileMatrixError("finalization attempted to dispatch an uncached row; the persisted coverage changed during finalization");
1970
+ },
1971
+ judges: [...opts.plan.judges],
1972
+ runDir,
1973
+ commitSha: opts.plan.commitSha,
1974
+ dispatchRef: opts.plan.dispatchRef,
1975
+ experimentId: opts.plan.experimentId,
1976
+ matrixId: opts.plan.matrixId,
1977
+ splitTag: opts.plan.splitTag,
1978
+ reps: opts.plan.reps,
1979
+ seed: opts.plan.seed,
1980
+ integrity: opts.plan.integrity,
1981
+ allowMixed: opts.plan.allowMixed,
1982
+ maxConcurrency: opts.maxConcurrency,
1983
+ maxProfileConcurrency: opts.maxProfileConcurrency,
1984
+ storage,
1985
+ now: opts.now,
1986
+ personaOf: opts.plan.personaOf,
1987
+ validate: opts.plan.validate,
1988
+ corpusText: opts.plan.corpusText,
1989
+ reuseFailedCells: true
1990
+ });
1991
+ assertFinalRows(opts.plan, matrix);
1992
+ return {
1993
+ ...matrix,
1994
+ coverage
1995
+ };
1996
+ }
1997
+ function manifestFor(plan) {
1998
+ return {
1999
+ schemaVersion: 1,
2000
+ matrixId: plan.matrixId,
2001
+ experimentId: plan.experimentId,
2002
+ planDigest: plan.planDigest,
2003
+ profiles: plan.profiles.map((profile) => ({
2004
+ id: agentProfileId(profile),
2005
+ hash: `sha256:${agentProfileHash(profile)}`
2006
+ })),
2007
+ scenarios: plan.scenarios.map((scenario) => campaignScenarioIdentity(scenario)),
2008
+ reps: plan.reps,
2009
+ seed: plan.seed,
2010
+ splitTag: plan.splitTag,
2011
+ commitSha: plan.commitSha,
2012
+ dispatchRef: plan.dispatchRef,
2013
+ rows: [...plan.rows]
2014
+ };
2015
+ }
2016
+ function assertPlan(plan) {
2017
+ const rebuilt = createProfileMatrixPlan({
2018
+ profiles: [...plan.profiles],
2019
+ scenarios: [...plan.scenarios],
2020
+ judges: [...plan.judges],
2021
+ commitSha: plan.commitSha,
2022
+ dispatchRef: plan.dispatchRef,
2023
+ experimentId: plan.experimentId,
2024
+ splitTag: plan.splitTag,
2025
+ reps: plan.reps,
2026
+ seed: plan.seed,
2027
+ integrity: plan.integrity,
2028
+ allowMixed: plan.allowMixed,
2029
+ validate: plan.validate,
2030
+ personaOf: plan.personaOf,
2031
+ corpusText: plan.corpusText
2032
+ });
2033
+ if (rebuilt.planDigest !== plan.planDigest || rebuilt.matrixId !== plan.matrixId || canonicalJson(rebuilt.rows) !== canonicalJson(plan.rows)) throw new SegmentedProfileMatrixError("plan inputs changed after identity was created");
2034
+ }
2035
+ function ensureManifest(storage, runDir, expected) {
2036
+ storage.ensureDir(runDir);
2037
+ const path = join(runDir, PLAN_FILE);
2038
+ if (storage.read(path) === void 0) {
2039
+ const content = `${canonicalJson(expected)}\n`;
2040
+ if (storage.append(path, content, 0) !== void 0) return;
2041
+ }
2042
+ const persisted = storage.read(path);
2043
+ if (persisted === void 0) throw new SegmentedProfileMatrixError("persisted plan is unreadable");
2044
+ let parsed;
2045
+ try {
2046
+ parsed = JSON.parse(persisted);
2047
+ } catch {
2048
+ throw new SegmentedProfileMatrixError("persisted plan is corrupt JSON");
2049
+ }
2050
+ if (canonicalJson(parsed) !== canonicalJson(expected)) throw new SegmentedProfileMatrixError("persisted plan identity does not match the supplied inputs");
2051
+ }
2052
+ function normalizeRows(plan, rows) {
2053
+ if (!Array.isArray(rows) || rows.length === 0) throw new SegmentedProfileMatrixError("a segment must select at least one row");
2054
+ const byId = new Map(plan.rows.map((row) => [row.rowId, row]));
2055
+ const selected = [];
2056
+ const seen = /* @__PURE__ */ new Set();
2057
+ for (const input of rows) {
2058
+ const rowId = typeof input === "string" ? input : input?.rowId;
2059
+ if (typeof rowId !== "string" || rowId.length === 0) throw new SegmentedProfileMatrixError("segment rows must contain row ids");
2060
+ if (seen.has(rowId)) throw new SegmentedProfileMatrixError(`duplicate row '${rowId}' in segment`);
2061
+ const row = byId.get(rowId);
2062
+ if (!row) throw new SegmentedProfileMatrixError(`row '${rowId}' is not declared by the plan`);
2063
+ if (typeof input !== "string" && canonicalJson(input) !== canonicalJson(row)) throw new SegmentedProfileMatrixError(`row '${rowId}' does not match the declared plan row`);
2064
+ seen.add(rowId);
2065
+ selected.push(row);
2066
+ }
2067
+ return selected.sort((left, right) => left.ordinal - right.ordinal);
2068
+ }
2069
+ function claimSegment(storage, runDir, plan, segmentId, selected) {
2070
+ const path = join(runDir, SEGMENTS_FILE);
2071
+ const expected = canonicalJson({
2072
+ schemaVersion: 1,
2073
+ segmentId,
2074
+ planDigest: plan.planDigest,
2075
+ rowIds: selected.map((row) => row.rowId).sort()
2076
+ });
2077
+ for (let attempt = 0; attempt < 8; attempt += 1) {
2078
+ const raw = storage.read(path) ?? "";
2079
+ const claims = parseSegmentClaims(raw, path);
2080
+ const existing = claims.find((claim) => claim.segmentId === segmentId);
2081
+ if (existing) {
2082
+ if (canonicalJson(existing) !== expected) throw new SegmentedProfileMatrixError(`segment '${segmentId}' was resumed with different rows or plan identity`);
2083
+ return;
2084
+ }
2085
+ const selectedIds = new Set(selected.map((row) => row.rowId));
2086
+ const overlap = claims.flatMap((claim) => claim.rowIds.filter((rowId) => selectedIds.has(rowId)).map((rowId) => `${rowId} (${claim.segmentId})`));
2087
+ if (overlap.length > 0) throw new SegmentedProfileMatrixError(`segment rows overlap existing claims: ${overlap.join(", ")}`);
2088
+ const event = `${expected}\n`;
2089
+ if (storage.append(path, event, byteLength(raw)) !== void 0) return;
2090
+ }
2091
+ throw new SegmentedProfileMatrixError(`could not claim segment '${segmentId}' after concurrent updates`);
2092
+ }
2093
+ function readSegmentClaims(storage, runDir) {
2094
+ const path = join(runDir, SEGMENTS_FILE);
2095
+ const raw = storage.read(path);
2096
+ if (raw === void 0) return [];
2097
+ return parseSegmentClaims(raw, path);
2098
+ }
2099
+ function parseSegmentClaims(raw, path) {
2100
+ const claims = [];
2101
+ const seenSegments = /* @__PURE__ */ new Set();
2102
+ const seenRows = /* @__PURE__ */ new Map();
2103
+ for (const [index, line] of raw.split("\n").entries()) {
2104
+ if (line.trim() === "") continue;
2105
+ let parsed;
2106
+ try {
2107
+ parsed = JSON.parse(line);
2108
+ } catch {
2109
+ throw new SegmentedProfileMatrixError(`segment registry is corrupt at ${path}:${index + 1}`);
2110
+ }
2111
+ if (canonicalJson(parsed) !== line) throw new SegmentedProfileMatrixError(`segment registry is not canonical at ${path}:${index + 1}`);
2112
+ const claim = parsed;
2113
+ if (claim.schemaVersion !== 1 || typeof claim.segmentId !== "string" || typeof claim.planDigest !== "string" || !Array.isArray(claim.rowIds) || claim.rowIds.some((rowId) => typeof rowId !== "string") || new Set(claim.rowIds).size !== claim.rowIds.length) throw new SegmentedProfileMatrixError(`segment registry has an invalid claim at ${path}:${index + 1}`);
2114
+ if (seenSegments.has(claim.segmentId)) throw new SegmentedProfileMatrixError(`segment registry repeats segment '${claim.segmentId}'`);
2115
+ seenSegments.add(claim.segmentId);
2116
+ const normalized = {
2117
+ schemaVersion: 1,
2118
+ segmentId: claim.segmentId,
2119
+ planDigest: claim.planDigest,
2120
+ rowIds: [...claim.rowIds].sort()
2121
+ };
2122
+ for (const rowId of normalized.rowIds) {
2123
+ const owner = seenRows.get(rowId);
2124
+ if (owner) throw new SegmentedProfileMatrixError(`segment registry overlaps row '${rowId}' between '${owner}' and '${normalized.segmentId}'`);
2125
+ seenRows.set(rowId, normalized.segmentId);
2126
+ }
2127
+ claims.push(normalized);
2128
+ }
2129
+ return claims;
2130
+ }
2131
+ function assertCompleteClaims(plan, claims) {
2132
+ const expected = new Set(plan.rows.map((row) => row.rowId));
2133
+ const claimed = /* @__PURE__ */ new Set();
2134
+ for (const claim of claims) {
2135
+ if (claim.planDigest !== plan.planDigest) throw new SegmentedProfileMatrixError(`segment '${claim.segmentId}' belongs to a different plan identity`);
2136
+ for (const rowId of claim.rowIds) {
2137
+ if (!expected.has(rowId)) throw new SegmentedProfileMatrixError(`segment '${claim.segmentId}' claims an undeclared row '${rowId}'`);
2138
+ claimed.add(rowId);
2139
+ }
2140
+ }
2141
+ const missingClaims = plan.rows.filter((row) => !claimed.has(row.rowId)).map((row) => row.rowId);
2142
+ if (missingClaims.length > 0) throw new SegmentedProfileMatrixError(`cannot finalize until every row is claimed: ${missingClaims.join(", ")}`);
2143
+ }
2144
+ function persistSegmentCells(storage, runDir, matrix) {
2145
+ for (const [profileId, campaign] of Object.entries(matrix.campaigns)) for (const cell of campaign.cells) {
2146
+ const path = cellCachePath(join(runDir, profileDirectory(profileId)), cell.cellId);
2147
+ storage.ensureDir(dirname(path));
2148
+ storage.write(path, `${JSON.stringify(cell)}\n`);
2149
+ }
2150
+ }
2151
+ function readCoverage(storage, runDir, plan) {
2152
+ const expectedManifestHashes = new Map(plan.profiles.map((profile) => {
2153
+ const identity = profileMatrixCampaignIdentity({
2154
+ profile,
2155
+ scenarios: [...plan.scenarios],
2156
+ judges: [...plan.judges],
2157
+ commitSha: plan.commitSha,
2158
+ dispatchRef: plan.dispatchRef,
2159
+ splitTag: plan.splitTag,
2160
+ seed: plan.seed,
2161
+ reps: plan.reps
2162
+ });
2163
+ return [identity.profileId, identity.manifestHash];
2164
+ }));
2165
+ const missing = [];
2166
+ const failed = [];
2167
+ const zeroScore = [];
2168
+ let present = 0;
2169
+ for (const row of plan.rows) {
2170
+ const path = cellCachePath(join(runDir, profileDirectory(row.profileId)), `${row.scenarioId}:${row.rep}`);
2171
+ const raw = storage.read(path);
2172
+ if (raw === void 0) {
2173
+ missing.push(row.rowId);
2174
+ continue;
2175
+ }
2176
+ let parsed;
2177
+ try {
2178
+ parsed = JSON.parse(raw);
2179
+ } catch {
2180
+ throw new SegmentedProfileMatrixError(`cell '${row.rowId}' has corrupt JSON`);
2181
+ }
2182
+ if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) throw new SegmentedProfileMatrixError(`cell '${row.rowId}' has an invalid record shape`);
2183
+ const cell = parsed;
2184
+ if (cell.cellId !== `${row.scenarioId}:${row.rep}` || cell.scenarioId !== row.scenarioId || cell.rep !== row.rep || cell.manifestHash !== expectedManifestHashes.get(row.profileId)) throw new SegmentedProfileMatrixError(`cell '${row.rowId}' has stale identity fields (manifest or row identity)`);
2185
+ try {
2186
+ campaignCellCostProvenance(cell);
2187
+ } catch (error) {
2188
+ throw new SegmentedProfileMatrixError(`cell '${row.rowId}' failed cost-record validation: ${error instanceof Error ? error.message : String(error)}`);
2189
+ }
2190
+ present += 1;
2191
+ if (cell.error !== void 0) {
2192
+ failed.push(row.rowId);
2193
+ continue;
2194
+ }
2195
+ if (projectCampaignCellQuality(cell).score === 0) zeroScore.push(row.rowId);
2196
+ }
2197
+ return {
2198
+ expected: plan.rows.length,
2199
+ present,
2200
+ missing,
2201
+ failed,
2202
+ zeroScore
2203
+ };
2204
+ }
2205
+ function assertFinalRows(plan, matrix) {
2206
+ const expected = new Set(plan.rows.map((row) => `${row.profileId}:${row.scenarioId}:${row.rep}`));
2207
+ const seen = /* @__PURE__ */ new Set();
2208
+ for (const record of matrix.records) {
2209
+ const key = `${record.candidateId}:${record.scenarioId}:${record.outcome.raw.rep}`;
2210
+ if (!expected.has(key) || seen.has(key)) throw new SegmentedProfileMatrixError(`final matrix has an unexpected or duplicate row '${key}'`);
2211
+ seen.add(key);
2212
+ }
2213
+ if (seen.size !== expected.size) throw new SegmentedProfileMatrixError(`final matrix returned ${seen.size}/${expected.size} declared rows`);
2214
+ }
2215
+ function rowIdFor(plan, profileId, scenarioId, rep) {
2216
+ const row = plan.rows.find((candidate) => candidate.profileId === profileId && candidate.scenarioId === scenarioId && candidate.rep === rep);
2217
+ if (!row) throw new SegmentedProfileMatrixError(`row is not declared: ${profileId}/${scenarioId}/${rep}`);
2218
+ return row.rowId;
2219
+ }
2220
+ function profileDirectory(profileId) {
2221
+ return profileId.replace(/[^a-zA-Z0-9_-]/g, "_");
2222
+ }
2223
+ function requireIdentifier(value, name) {
2224
+ if (typeof value !== "string" || value.trim().length === 0) throw new SegmentedProfileMatrixError(`${name} must be a non-empty string`);
2225
+ return value;
2226
+ }
2227
+ function byteLength(value) {
2228
+ return new TextEncoder().encode(value).byteLength;
2229
+ }
2230
+ //#endregion
1752
2231
  //#region src/campaign/scenario-selection.ts
1753
2232
  const DEFAULT_SATURATION_CEILING = .999;
1754
2233
  /** Variance below this is treated as "every candidate scored the same". */
@@ -3177,6 +3656,6 @@ function resolveWorktreePath(surface, worktreeDir) {
3177
3656
  return verifyCodeSurface(surface, worktreeDir).path;
3178
3657
  }
3179
3658
  //#endregion
3180
- export { planEvalFixtureRun as A, LabeledScenarioStoreError as C, discoverEvalFixtures as D, neutralizationGate as E, buildTraceAnalystSurfaceDispatch as M, traceAnalystQualityJudge as N, loadEvalFixture as O, FsLabeledScenarioStore as S, rolloutArgumentDiff as T, renderScoreboardMarkdown as _, autoevalsScorerJudge as a, userStoryScoreboard as b, FileSearchLedger as c, validateSearchLedgerEvent as d, scoreDiscrimination as f, makePlaybackDispatch as g, runProfileMatrix as h, verifyCodeSurface as i, analyzeCrossSurfaceInteractions as j, loadEvalFixtureScenarios as k, SEARCH_LEDGER_SCHEMA as l, ProfileMatrixError as m, gitWorktreeAdapter as n, phoenixEvaluatorJudge as o, selectDiscriminative as p, resolveWorktreePath as r, isTransientTransportFailure as s, WorktreeAdapterError as t, openSearchLedger as u, scoreUserStory as v, classifyUngroundedLiterals as w, neutralizeText as x, scoreboardSummary as y };
3659
+ export { neutralizationGate as A, scoreboardSummary as C, LabeledScenarioStoreError as D, FsLabeledScenarioStore as E, analyzeCrossSurfaceInteractions as F, buildTraceAnalystSurfaceDispatch as I, traceAnalystQualityJudge as L, loadEvalFixture as M, loadEvalFixtureScenarios as N, classifyUngroundedLiterals as O, planEvalFixtureRun as P, scoreUserStory as S, neutralizeText as T, runProfileMatrixSegment as _, autoevalsScorerJudge as a, makePlaybackDispatch as b, FileSearchLedger as c, validateSearchLedgerEvent as d, scoreDiscrimination as f, finalizeProfileMatrix as g, createProfileMatrixPlan as h, verifyCodeSurface as i, discoverEvalFixtures as j, rolloutArgumentDiff as k, SEARCH_LEDGER_SCHEMA as l, SegmentedProfileMatrixError as m, gitWorktreeAdapter as n, phoenixEvaluatorJudge as o, selectDiscriminative as p, resolveWorktreePath as r, isTransientTransportFailure as s, WorktreeAdapterError as t, openSearchLedger as u, ProfileMatrixError as v, userStoryScoreboard as w, renderScoreboardMarkdown as x, runProfileMatrix as y };
3181
3660
 
3182
- //# sourceMappingURL=campaign-C5kujyqv.js.map
3661
+ //# sourceMappingURL=campaign-FY23WYzZ.js.map