@tangle-network/agent-eval 0.145.3 → 0.145.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +22 -0
- package/README.md +5 -0
- package/dist/analyst/index.d.ts +1 -1
- package/dist/analyst/index.js +1 -1
- package/dist/{benchmark-command-Iaqlp3DU.js → benchmark-command-C44VE5rA.js} +2 -2
- package/dist/{benchmark-command-Iaqlp3DU.js.map → benchmark-command-C44VE5rA.js.map} +1 -1
- package/dist/benchmarks/index.d.ts +2 -2
- package/dist/benchmarks/index.js +2 -2
- package/dist/campaign/index.d.ts +4 -4
- package/dist/campaign/index.js +4 -4
- package/dist/{campaign-C5kujyqv.js → campaign-DGWSftLe.js} +503 -24
- package/dist/campaign-DGWSftLe.js.map +1 -0
- package/dist/cli.js +1 -1
- package/dist/contract/index.d.ts +3 -3
- package/dist/contract/index.js +3 -3
- package/dist/{define-agent-eval-sH24zBfM.d.ts → define-agent-eval-DcAcC1yQ.d.ts} +2 -2
- package/dist/{define-agent-eval-sH24zBfM.d.ts.map → define-agent-eval-DcAcC1yQ.d.ts.map} +1 -1
- package/dist/{define-agent-eval-CAL5HOz9.js → define-agent-eval-xaceIpPS.js} +2 -2
- package/dist/{define-agent-eval-CAL5HOz9.js.map → define-agent-eval-xaceIpPS.js.map} +1 -1
- package/dist/{index-CUczrQ7a.d.ts → index-DuKK7tJR.d.ts} +101 -4
- package/dist/index-DuKK7tJR.d.ts.map +1 -0
- package/dist/index.d.ts +2 -2
- package/dist/index.js +3 -3
- package/dist/{llm-judge-C2lQBv7h.js → llm-judge-BPKBdQTp.js} +7 -4
- package/dist/llm-judge-BPKBdQTp.js.map +1 -0
- package/dist/openapi.json +1 -1
- package/dist/{produced-state-Bgph_Qj-.js → produced-state-B_-TW9o0.js} +2 -2
- package/dist/{produced-state-Bgph_Qj-.js.map → produced-state-B_-TW9o0.js.map} +1 -1
- package/dist/{provenance-CCdxgLDT.d.ts → provenance-C_ln89b0.d.ts} +15 -1
- package/dist/provenance-C_ln89b0.d.ts.map +1 -0
- package/dist/{skillopt-optimization-method-C5cotF4E.d.ts → skillopt-optimization-method-Bb1OTHZW.d.ts} +9 -2
- package/dist/skillopt-optimization-method-Bb1OTHZW.d.ts.map +1 -0
- package/dist/{skillopt-optimization-method-B__Jnto4.js → skillopt-optimization-method-S7sdjzvh.js} +26 -3
- package/dist/skillopt-optimization-method-S7sdjzvh.js.map +1 -0
- package/docs/eval-surface-map.md +11 -0
- package/package.json +1 -1
- package/dist/campaign-C5kujyqv.js.map +0 -1
- package/dist/index-CUczrQ7a.d.ts.map +0 -1
- package/dist/llm-judge-C2lQBv7h.js.map +0 -1
- package/dist/provenance-CCdxgLDT.d.ts.map +0 -1
- package/dist/skillopt-optimization-method-B__Jnto4.js.map +0 -1
- package/dist/skillopt-optimization-method-C5cotF4E.d.ts.map +0 -1
|
@@ -1,20 +1,20 @@
|
|
|
1
1
|
import { s as ValidationError, t as AgentEvalError } from "./errors-Dngq5h35.js";
|
|
2
|
-
import { c as agentProfileId, d as harnessAxisOf, i as verifyCompletion, l as agentProfileModelId, s as agentProfileHash, t as extractProducedState } from "./produced-state-
|
|
2
|
+
import { c as agentProfileId, d as harnessAxisOf, i as verifyCompletion, l as agentProfileModelId, s as agentProfileHash, t as extractProducedState } from "./produced-state-B_-TW9o0.js";
|
|
3
3
|
import { buildAgentProfileCell } from "./profile-cell.js";
|
|
4
4
|
import { t as comparePairedArms } from "./paired-arms-D-XRF_fy.js";
|
|
5
5
|
import { r as pairedBootstrap } from "./paired-tests-BHIhYVdu.js";
|
|
6
6
|
import { o as runTaskScore, r as modelHasSnapshot, s as validateRunRecord } from "./run-record-BvHPVS-i.js";
|
|
7
|
-
import { n as contentHash } from "./verdict-cache-mZf5FEiY.js";
|
|
8
|
-
import { K as runCampaign, Q as summarizeBackendIntegrity, Z as assertRealBackend, h as labelTrustRank, k as surfaceContentHash, q as planCampaignRun, w as assertCodeSurfaceIdentity } from "./llm-judge-
|
|
9
|
-
import { l as campaignCellToRunRecord } from "./reward-hacking-DNgjilrV.js";
|
|
7
|
+
import { n as contentHash, t as canonicalJson } from "./verdict-cache-mZf5FEiY.js";
|
|
8
|
+
import { $ as assertCampaignDesign, J as resolveRunDir, K as runCampaign, Q as summarizeBackendIntegrity, Z as assertRealBackend, at as computeManifestHash, h as labelTrustRank, it as cellCachePath, k as surfaceContentHash, q as planCampaignRun, tt as campaignScenarioIdentity, w as assertCodeSurfaceIdentity } from "./llm-judge-BPKBdQTp.js";
|
|
9
|
+
import { a as campaignCellCostProvenance, l as campaignCellToRunRecord, u as projectCampaignCellQuality } from "./reward-hacking-DNgjilrV.js";
|
|
10
10
|
import { t as CostAccountingIncompleteError } from "./cost-ledger-BSe92yAV.js";
|
|
11
11
|
import { t as FileLedgerJournal, u as mapConcurrent } from "./ledger-core-BmZt19oQ.js";
|
|
12
12
|
import { r as canonicalString } from "./canonical-D-XsTQ6_.js";
|
|
13
|
-
import { C as SEARCH_LEDGER_FILE_CONTEXT, E as SearchLedgerIntegrityError, T as SearchLedgerError, w as SearchLedgerConflictError } from "./external-optimizer-subprocess-BhKYK0Jv.js";
|
|
13
|
+
import { C as SEARCH_LEDGER_FILE_CONTEXT, E as SearchLedgerIntegrityError, T as SearchLedgerError, w as SearchLedgerConflictError, x as fsCampaignStorage } from "./external-optimizer-subprocess-BhKYK0Jv.js";
|
|
14
14
|
import { o as pairHoldout } from "./power-preflight-DEw-uC7q.js";
|
|
15
15
|
import { a as scoreAnalystFindings } from "./benchmark-BhT16ep9.js";
|
|
16
16
|
import "./external-optimizer-process-BRE56woM.js";
|
|
17
|
-
import "./skillopt-optimization-method-
|
|
17
|
+
import "./skillopt-optimization-method-S7sdjzvh.js";
|
|
18
18
|
import { createHash } from "node:crypto";
|
|
19
19
|
import { closeSync, constants, existsSync, fstatSync, lstatSync, mkdirSync, mkdtempSync, openSync, readFileSync, readSync, readdirSync, readlinkSync, realpathSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
20
20
|
import { z } from "zod";
|
|
@@ -1490,6 +1490,45 @@ var ProfileMatrixError = class extends AgentEvalError {
|
|
|
1490
1490
|
super("profile_matrix", message);
|
|
1491
1491
|
}
|
|
1492
1492
|
};
|
|
1493
|
+
/**
|
|
1494
|
+
* Return the exact per-profile campaign identity used by `runProfileMatrix`.
|
|
1495
|
+
* Segmented profile matrices use this to validate durable cell caches before
|
|
1496
|
+
* finalization without dispatching a probe request.
|
|
1497
|
+
*/
|
|
1498
|
+
function profileMatrixCampaignIdentity(opts) {
|
|
1499
|
+
const seed = opts.seed ?? 42;
|
|
1500
|
+
const splitTag = opts.splitTag ?? "search";
|
|
1501
|
+
const reps = opts.reps ?? 1;
|
|
1502
|
+
const profileHash = agentProfileHash(opts.profile);
|
|
1503
|
+
const profileId = agentProfileId(opts.profile);
|
|
1504
|
+
const configHash = sha({
|
|
1505
|
+
profile: profileHash,
|
|
1506
|
+
judges: (opts.judges ?? []).map((judge) => judge.name),
|
|
1507
|
+
seed,
|
|
1508
|
+
splitTag,
|
|
1509
|
+
dispatchRef: opts.dispatchRef ?? null
|
|
1510
|
+
});
|
|
1511
|
+
const dispatchRef = `profile-matrix:${sha({
|
|
1512
|
+
commitSha: opts.commitSha,
|
|
1513
|
+
caller: opts.dispatchRef ?? null,
|
|
1514
|
+
profileId,
|
|
1515
|
+
profileHash,
|
|
1516
|
+
configHash
|
|
1517
|
+
})}`;
|
|
1518
|
+
return {
|
|
1519
|
+
profileHash,
|
|
1520
|
+
profileId,
|
|
1521
|
+
configHash,
|
|
1522
|
+
dispatchRef,
|
|
1523
|
+
manifestHash: computeManifestHash({
|
|
1524
|
+
scenarios: opts.scenarios,
|
|
1525
|
+
judges: opts.judges ?? [],
|
|
1526
|
+
dispatchRef,
|
|
1527
|
+
seed,
|
|
1528
|
+
reps
|
|
1529
|
+
})
|
|
1530
|
+
};
|
|
1531
|
+
}
|
|
1493
1532
|
function sanitize(id) {
|
|
1494
1533
|
return id.replace(/[^a-zA-Z0-9_-]/g, "_");
|
|
1495
1534
|
}
|
|
@@ -1565,7 +1604,7 @@ async function runProfileMatrix(opts) {
|
|
|
1565
1604
|
profileIds,
|
|
1566
1605
|
scenarios: opts.scenarios.map((s) => s.id)
|
|
1567
1606
|
}).slice(0, 16)}`;
|
|
1568
|
-
const
|
|
1607
|
+
const derivedMatrixId = `mtx_${sha({
|
|
1569
1608
|
experimentId,
|
|
1570
1609
|
profileIds,
|
|
1571
1610
|
seed,
|
|
@@ -1573,6 +1612,8 @@ async function runProfileMatrix(opts) {
|
|
|
1573
1612
|
commitSha: opts.commitSha,
|
|
1574
1613
|
dispatchRef: opts.dispatchRef ?? null
|
|
1575
1614
|
}).slice(0, 16)}`;
|
|
1615
|
+
const matrixId = opts.matrixId ?? derivedMatrixId;
|
|
1616
|
+
if (typeof matrixId !== "string" || matrixId.trim().length === 0) throw new ProfileMatrixError("matrixId must be a non-empty string when provided");
|
|
1576
1617
|
const scenarioById = new Map(opts.scenarios.map((s) => [s.id, s]));
|
|
1577
1618
|
for (const profile of opts.profiles) {
|
|
1578
1619
|
const profileHash = agentProfileHash(profile);
|
|
@@ -1612,23 +1653,17 @@ async function runProfileMatrix(opts) {
|
|
|
1612
1653
|
const campaigns = {};
|
|
1613
1654
|
const byProfile = {};
|
|
1614
1655
|
const columns = await mapConcurrent(opts.profiles, maxProfileConcurrency, async (profile, _index, signal) => {
|
|
1615
|
-
const profileHash =
|
|
1616
|
-
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
|
|
1620
|
-
|
|
1621
|
-
seed,
|
|
1656
|
+
const { profileHash, profileId, configHash, dispatchRef } = profileMatrixCampaignIdentity({
|
|
1657
|
+
profile,
|
|
1658
|
+
scenarios: opts.scenarios,
|
|
1659
|
+
judges: opts.judges,
|
|
1660
|
+
commitSha: opts.commitSha,
|
|
1661
|
+
dispatchRef: opts.dispatchRef,
|
|
1622
1662
|
splitTag,
|
|
1623
|
-
|
|
1663
|
+
seed,
|
|
1664
|
+
reps: opts.reps
|
|
1624
1665
|
});
|
|
1625
|
-
const
|
|
1626
|
-
commitSha: opts.commitSha,
|
|
1627
|
-
caller: opts.dispatchRef ?? null,
|
|
1628
|
-
profileId,
|
|
1629
|
-
profileHash,
|
|
1630
|
-
configHash
|
|
1631
|
-
})}`;
|
|
1666
|
+
const declaredModel = agentProfileModelId(profile);
|
|
1632
1667
|
const dispatch = (scenario, ctx) => opts.dispatch(profile, scenario, ctx);
|
|
1633
1668
|
Object.defineProperty(dispatch, "name", { value: `profile_${sanitize(profileId)}` });
|
|
1634
1669
|
const campaign = await runCampaign({
|
|
@@ -1641,6 +1676,12 @@ async function runProfileMatrix(opts) {
|
|
|
1641
1676
|
reps: opts.reps,
|
|
1642
1677
|
maxConcurrency: opts.maxConcurrency,
|
|
1643
1678
|
costCeiling: opts.costCeiling,
|
|
1679
|
+
cellFilter: opts.rowFilter ? ({ scenario, rep }) => opts.rowFilter?.({
|
|
1680
|
+
profile,
|
|
1681
|
+
scenario,
|
|
1682
|
+
rep
|
|
1683
|
+
}) ?? false : void 0,
|
|
1684
|
+
reuseFailedCells: opts.reuseFailedCells,
|
|
1644
1685
|
labeledStore: opts.labeledStore,
|
|
1645
1686
|
captureSource: opts.captureSource,
|
|
1646
1687
|
storage: opts.storage,
|
|
@@ -1749,6 +1790,444 @@ function rollupByPersona(records, scenarios, personaOf) {
|
|
|
1749
1790
|
return rollup(records, (r) => r.scenarioId ? personaByScenarioId.get(r.scenarioId) : void 0);
|
|
1750
1791
|
}
|
|
1751
1792
|
//#endregion
|
|
1793
|
+
//#region src/campaign/presets/segmented-profile-matrix.ts
|
|
1794
|
+
/**
|
|
1795
|
+
* Eval-owned execution for profile matrices whose external authorization is
|
|
1796
|
+
* shorter than the complete matrix.
|
|
1797
|
+
*
|
|
1798
|
+
* A plan freezes the complete profile × scenario × replicate design. A
|
|
1799
|
+
* segment claims an explicit, disjoint set of rows and uses the campaign cell
|
|
1800
|
+
* cache as the durable per-cell record. Finalization first checks that every
|
|
1801
|
+
* planned row has one claim and one valid record, then replays the ordinary
|
|
1802
|
+
* `runProfileMatrix` projection from those records without dispatching work.
|
|
1803
|
+
*/
|
|
1804
|
+
const PLAN_FILE = "profile-matrix-plan.json";
|
|
1805
|
+
const SEGMENTS_FILE = "profile-matrix-segments.jsonl";
|
|
1806
|
+
var SegmentedProfileMatrixError = class extends ProfileMatrixError {
|
|
1807
|
+
constructor(message) {
|
|
1808
|
+
super(`segmented matrix: ${message}`);
|
|
1809
|
+
}
|
|
1810
|
+
};
|
|
1811
|
+
function createProfileMatrixPlan(opts) {
|
|
1812
|
+
if (!Array.isArray(opts.profiles) || opts.profiles.length === 0) throw new SegmentedProfileMatrixError("profiles must not be empty");
|
|
1813
|
+
if (!Array.isArray(opts.scenarios) || opts.scenarios.length === 0) throw new SegmentedProfileMatrixError("scenarios must not be empty");
|
|
1814
|
+
const reps = opts.reps ?? 1;
|
|
1815
|
+
const seed = opts.seed ?? 42;
|
|
1816
|
+
const splitTag = opts.splitTag ?? "search";
|
|
1817
|
+
const integrity = opts.integrity ?? "assert";
|
|
1818
|
+
const allowMixed = opts.allowMixed ?? true;
|
|
1819
|
+
const validate = opts.validate ?? true;
|
|
1820
|
+
if (typeof opts.commitSha !== "string" || opts.commitSha.trim().length === 0) throw new SegmentedProfileMatrixError("commitSha must be a non-empty string");
|
|
1821
|
+
if (typeof opts.dispatchRef !== "string" || opts.dispatchRef.trim().length === 0) throw new SegmentedProfileMatrixError("dispatchRef must be a non-empty stable identity for segmented execution");
|
|
1822
|
+
assertCampaignDesign(opts.scenarios, reps);
|
|
1823
|
+
const profiles = [...opts.profiles];
|
|
1824
|
+
const scenarios = [...opts.scenarios];
|
|
1825
|
+
const judges = [...opts.judges ?? []];
|
|
1826
|
+
const profileIds = profiles.map(agentProfileId);
|
|
1827
|
+
const duplicateProfiles = profileIds.filter((id, index) => profileIds.indexOf(id) !== index);
|
|
1828
|
+
if (duplicateProfiles.length > 0) throw new SegmentedProfileMatrixError(`duplicate profile ids: ${[...new Set(duplicateProfiles)].join(", ")}`);
|
|
1829
|
+
const experimentId = opts.experimentId ?? `pm_${contentHash({
|
|
1830
|
+
profileIds,
|
|
1831
|
+
scenarios: scenarios.map((scenario) => scenario.id)
|
|
1832
|
+
}).slice(0, 16)}`;
|
|
1833
|
+
if (experimentId.trim().length === 0) throw new SegmentedProfileMatrixError("experimentId must be non-empty");
|
|
1834
|
+
const planDigest = `sha256:${contentHash({
|
|
1835
|
+
schema: "tangle.profile-matrix-plan.v1",
|
|
1836
|
+
profiles: profiles.map((profile, index) => ({
|
|
1837
|
+
ordinal: index,
|
|
1838
|
+
id: profileIds[index],
|
|
1839
|
+
hash: `sha256:${agentProfileHash(profile)}`,
|
|
1840
|
+
profile
|
|
1841
|
+
})),
|
|
1842
|
+
scenarios: scenarios.map((scenario, index) => ({
|
|
1843
|
+
ordinal: index,
|
|
1844
|
+
identity: campaignScenarioIdentity(scenario),
|
|
1845
|
+
scenario
|
|
1846
|
+
})),
|
|
1847
|
+
judges: judges.map((judge) => ({
|
|
1848
|
+
name: judge.name,
|
|
1849
|
+
dimensions: judge.dimensions,
|
|
1850
|
+
judgeVersion: judge.judgeVersion ?? null,
|
|
1851
|
+
score: judge.score.toString(),
|
|
1852
|
+
appliesTo: judge.appliesTo?.toString() ?? null
|
|
1853
|
+
})),
|
|
1854
|
+
reps,
|
|
1855
|
+
seed,
|
|
1856
|
+
splitTag,
|
|
1857
|
+
commitSha: opts.commitSha,
|
|
1858
|
+
dispatchRef: opts.dispatchRef,
|
|
1859
|
+
experimentId,
|
|
1860
|
+
integrity,
|
|
1861
|
+
allowMixed,
|
|
1862
|
+
validate,
|
|
1863
|
+
personaOf: opts.personaOf?.toString() ?? null,
|
|
1864
|
+
corpusText: opts.corpusText?.toString() ?? null
|
|
1865
|
+
})}`;
|
|
1866
|
+
const matrixId = `pmx_${planDigest.slice(7, 31)}`;
|
|
1867
|
+
const rows = [];
|
|
1868
|
+
const seenCellIds = /* @__PURE__ */ new Set();
|
|
1869
|
+
let ordinal = 0;
|
|
1870
|
+
for (const profileId of profileIds) for (const scenario of scenarios) for (let rep = 0; rep < reps; rep += 1) {
|
|
1871
|
+
const cellId = `${scenario.id}:${rep}`;
|
|
1872
|
+
if (seenCellIds.has(`${profileId}:${cellId}`)) throw new SegmentedProfileMatrixError(`profile/scenario/rep cell id collision for '${profileId}:${cellId}'`);
|
|
1873
|
+
seenCellIds.add(`${profileId}:${cellId}`);
|
|
1874
|
+
rows.push({
|
|
1875
|
+
rowId: `row_${contentHash({
|
|
1876
|
+
matrixId,
|
|
1877
|
+
profileId,
|
|
1878
|
+
scenarioId: scenario.id,
|
|
1879
|
+
rep
|
|
1880
|
+
}).slice(0, 24)}`,
|
|
1881
|
+
ordinal,
|
|
1882
|
+
profileId,
|
|
1883
|
+
scenarioId: scenario.id,
|
|
1884
|
+
rep
|
|
1885
|
+
});
|
|
1886
|
+
ordinal += 1;
|
|
1887
|
+
}
|
|
1888
|
+
return Object.freeze({
|
|
1889
|
+
schemaVersion: 1,
|
|
1890
|
+
matrixId,
|
|
1891
|
+
experimentId,
|
|
1892
|
+
planDigest,
|
|
1893
|
+
profiles: Object.freeze(profiles),
|
|
1894
|
+
scenarios: Object.freeze(scenarios),
|
|
1895
|
+
judges: Object.freeze(judges),
|
|
1896
|
+
reps,
|
|
1897
|
+
seed,
|
|
1898
|
+
splitTag,
|
|
1899
|
+
commitSha: opts.commitSha,
|
|
1900
|
+
dispatchRef: opts.dispatchRef,
|
|
1901
|
+
integrity,
|
|
1902
|
+
allowMixed,
|
|
1903
|
+
validate,
|
|
1904
|
+
...opts.personaOf === void 0 ? {} : { personaOf: opts.personaOf },
|
|
1905
|
+
...opts.corpusText === void 0 ? {} : { corpusText: opts.corpusText },
|
|
1906
|
+
rows: Object.freeze(rows)
|
|
1907
|
+
});
|
|
1908
|
+
}
|
|
1909
|
+
async function runProfileMatrixSegment(opts) {
|
|
1910
|
+
const segmentId = requireIdentifier(opts.segmentId, "segmentId");
|
|
1911
|
+
const storage = opts.storage ?? fsCampaignStorage();
|
|
1912
|
+
const runDir = resolveRunDir(opts.runDir);
|
|
1913
|
+
assertPlan(opts.plan);
|
|
1914
|
+
const selected = normalizeRows(opts.plan, opts.rows);
|
|
1915
|
+
ensureManifest(storage, runDir, manifestFor(opts.plan));
|
|
1916
|
+
claimSegment(storage, runDir, opts.plan, segmentId, selected);
|
|
1917
|
+
const selectedIds = new Set(selected.map((row) => row.rowId));
|
|
1918
|
+
const selectedProfileIds = new Set(selected.map((row) => row.profileId));
|
|
1919
|
+
const matrix = await runProfileMatrix({
|
|
1920
|
+
profiles: opts.plan.profiles.filter((profile) => selectedProfileIds.has(agentProfileId(profile))),
|
|
1921
|
+
scenarios: [...opts.plan.scenarios],
|
|
1922
|
+
dispatch: opts.dispatch,
|
|
1923
|
+
judges: [...opts.plan.judges],
|
|
1924
|
+
runDir,
|
|
1925
|
+
commitSha: opts.plan.commitSha,
|
|
1926
|
+
dispatchRef: opts.plan.dispatchRef,
|
|
1927
|
+
experimentId: opts.plan.experimentId,
|
|
1928
|
+
matrixId: opts.plan.matrixId,
|
|
1929
|
+
splitTag: opts.plan.splitTag,
|
|
1930
|
+
reps: opts.plan.reps,
|
|
1931
|
+
seed: opts.plan.seed,
|
|
1932
|
+
integrity: "off",
|
|
1933
|
+
allowMixed: opts.plan.allowMixed,
|
|
1934
|
+
maxConcurrency: opts.maxConcurrency,
|
|
1935
|
+
maxProfileConcurrency: opts.maxProfileConcurrency,
|
|
1936
|
+
costCeiling: opts.costCeiling,
|
|
1937
|
+
labeledStore: opts.labeledStore,
|
|
1938
|
+
captureSource: opts.captureSource,
|
|
1939
|
+
storage,
|
|
1940
|
+
now: opts.now,
|
|
1941
|
+
personaOf: opts.plan.personaOf,
|
|
1942
|
+
validate: opts.plan.validate,
|
|
1943
|
+
corpusText: opts.plan.corpusText,
|
|
1944
|
+
reuseFailedCells: false,
|
|
1945
|
+
rowFilter: ({ profile, scenario, rep }) => selectedIds.has(rowIdFor(opts.plan, agentProfileId(profile), scenario.id, rep))
|
|
1946
|
+
});
|
|
1947
|
+
persistSegmentCells(storage, runDir, matrix);
|
|
1948
|
+
const coverage = readCoverage(storage, runDir, opts.plan);
|
|
1949
|
+
return {
|
|
1950
|
+
segmentId,
|
|
1951
|
+
rowIds: selected.map((row) => row.rowId),
|
|
1952
|
+
matrix,
|
|
1953
|
+
coverage
|
|
1954
|
+
};
|
|
1955
|
+
}
|
|
1956
|
+
async function finalizeProfileMatrix(opts) {
|
|
1957
|
+
const storage = opts.storage ?? fsCampaignStorage();
|
|
1958
|
+
const runDir = resolveRunDir(opts.runDir);
|
|
1959
|
+
assertPlan(opts.plan);
|
|
1960
|
+
ensureManifest(storage, runDir, manifestFor(opts.plan));
|
|
1961
|
+
const claims = readSegmentClaims(storage, runDir);
|
|
1962
|
+
assertCompleteClaims(opts.plan, claims);
|
|
1963
|
+
const coverage = readCoverage(storage, runDir, opts.plan);
|
|
1964
|
+
if (coverage.missing.length > 0) throw new SegmentedProfileMatrixError(`cannot finalize with missing rows: ${coverage.missing.join(", ")}`);
|
|
1965
|
+
const matrix = await runProfileMatrix({
|
|
1966
|
+
profiles: [...opts.plan.profiles],
|
|
1967
|
+
scenarios: [...opts.plan.scenarios],
|
|
1968
|
+
dispatch: async () => {
|
|
1969
|
+
throw new SegmentedProfileMatrixError("finalization attempted to dispatch an uncached row; the persisted coverage changed during finalization");
|
|
1970
|
+
},
|
|
1971
|
+
judges: [...opts.plan.judges],
|
|
1972
|
+
runDir,
|
|
1973
|
+
commitSha: opts.plan.commitSha,
|
|
1974
|
+
dispatchRef: opts.plan.dispatchRef,
|
|
1975
|
+
experimentId: opts.plan.experimentId,
|
|
1976
|
+
matrixId: opts.plan.matrixId,
|
|
1977
|
+
splitTag: opts.plan.splitTag,
|
|
1978
|
+
reps: opts.plan.reps,
|
|
1979
|
+
seed: opts.plan.seed,
|
|
1980
|
+
integrity: opts.plan.integrity,
|
|
1981
|
+
allowMixed: opts.plan.allowMixed,
|
|
1982
|
+
maxConcurrency: opts.maxConcurrency,
|
|
1983
|
+
maxProfileConcurrency: opts.maxProfileConcurrency,
|
|
1984
|
+
storage,
|
|
1985
|
+
now: opts.now,
|
|
1986
|
+
personaOf: opts.plan.personaOf,
|
|
1987
|
+
validate: opts.plan.validate,
|
|
1988
|
+
corpusText: opts.plan.corpusText,
|
|
1989
|
+
reuseFailedCells: true
|
|
1990
|
+
});
|
|
1991
|
+
assertFinalRows(opts.plan, matrix);
|
|
1992
|
+
return {
|
|
1993
|
+
...matrix,
|
|
1994
|
+
coverage
|
|
1995
|
+
};
|
|
1996
|
+
}
|
|
1997
|
+
function manifestFor(plan) {
|
|
1998
|
+
return {
|
|
1999
|
+
schemaVersion: 1,
|
|
2000
|
+
matrixId: plan.matrixId,
|
|
2001
|
+
experimentId: plan.experimentId,
|
|
2002
|
+
planDigest: plan.planDigest,
|
|
2003
|
+
profiles: plan.profiles.map((profile) => ({
|
|
2004
|
+
id: agentProfileId(profile),
|
|
2005
|
+
hash: `sha256:${agentProfileHash(profile)}`
|
|
2006
|
+
})),
|
|
2007
|
+
scenarios: plan.scenarios.map((scenario) => campaignScenarioIdentity(scenario)),
|
|
2008
|
+
reps: plan.reps,
|
|
2009
|
+
seed: plan.seed,
|
|
2010
|
+
splitTag: plan.splitTag,
|
|
2011
|
+
commitSha: plan.commitSha,
|
|
2012
|
+
dispatchRef: plan.dispatchRef,
|
|
2013
|
+
rows: [...plan.rows]
|
|
2014
|
+
};
|
|
2015
|
+
}
|
|
2016
|
+
function assertPlan(plan) {
|
|
2017
|
+
const rebuilt = createProfileMatrixPlan({
|
|
2018
|
+
profiles: [...plan.profiles],
|
|
2019
|
+
scenarios: [...plan.scenarios],
|
|
2020
|
+
judges: [...plan.judges],
|
|
2021
|
+
commitSha: plan.commitSha,
|
|
2022
|
+
dispatchRef: plan.dispatchRef,
|
|
2023
|
+
experimentId: plan.experimentId,
|
|
2024
|
+
splitTag: plan.splitTag,
|
|
2025
|
+
reps: plan.reps,
|
|
2026
|
+
seed: plan.seed,
|
|
2027
|
+
integrity: plan.integrity,
|
|
2028
|
+
allowMixed: plan.allowMixed,
|
|
2029
|
+
validate: plan.validate,
|
|
2030
|
+
personaOf: plan.personaOf,
|
|
2031
|
+
corpusText: plan.corpusText
|
|
2032
|
+
});
|
|
2033
|
+
if (rebuilt.planDigest !== plan.planDigest || rebuilt.matrixId !== plan.matrixId || canonicalJson(rebuilt.rows) !== canonicalJson(plan.rows)) throw new SegmentedProfileMatrixError("plan inputs changed after identity was created");
|
|
2034
|
+
}
|
|
2035
|
+
function ensureManifest(storage, runDir, expected) {
|
|
2036
|
+
storage.ensureDir(runDir);
|
|
2037
|
+
const path = join(runDir, PLAN_FILE);
|
|
2038
|
+
if (storage.read(path) === void 0) {
|
|
2039
|
+
const content = `${canonicalJson(expected)}\n`;
|
|
2040
|
+
if (storage.append(path, content, 0) !== void 0) return;
|
|
2041
|
+
}
|
|
2042
|
+
const persisted = storage.read(path);
|
|
2043
|
+
if (persisted === void 0) throw new SegmentedProfileMatrixError("persisted plan is unreadable");
|
|
2044
|
+
let parsed;
|
|
2045
|
+
try {
|
|
2046
|
+
parsed = JSON.parse(persisted);
|
|
2047
|
+
} catch {
|
|
2048
|
+
throw new SegmentedProfileMatrixError("persisted plan is corrupt JSON");
|
|
2049
|
+
}
|
|
2050
|
+
if (canonicalJson(parsed) !== canonicalJson(expected)) throw new SegmentedProfileMatrixError("persisted plan identity does not match the supplied inputs");
|
|
2051
|
+
}
|
|
2052
|
+
function normalizeRows(plan, rows) {
|
|
2053
|
+
if (!Array.isArray(rows) || rows.length === 0) throw new SegmentedProfileMatrixError("a segment must select at least one row");
|
|
2054
|
+
const byId = new Map(plan.rows.map((row) => [row.rowId, row]));
|
|
2055
|
+
const selected = [];
|
|
2056
|
+
const seen = /* @__PURE__ */ new Set();
|
|
2057
|
+
for (const input of rows) {
|
|
2058
|
+
const rowId = typeof input === "string" ? input : input?.rowId;
|
|
2059
|
+
if (typeof rowId !== "string" || rowId.length === 0) throw new SegmentedProfileMatrixError("segment rows must contain row ids");
|
|
2060
|
+
if (seen.has(rowId)) throw new SegmentedProfileMatrixError(`duplicate row '${rowId}' in segment`);
|
|
2061
|
+
const row = byId.get(rowId);
|
|
2062
|
+
if (!row) throw new SegmentedProfileMatrixError(`row '${rowId}' is not declared by the plan`);
|
|
2063
|
+
if (typeof input !== "string" && canonicalJson(input) !== canonicalJson(row)) throw new SegmentedProfileMatrixError(`row '${rowId}' does not match the declared plan row`);
|
|
2064
|
+
seen.add(rowId);
|
|
2065
|
+
selected.push(row);
|
|
2066
|
+
}
|
|
2067
|
+
return selected.sort((left, right) => left.ordinal - right.ordinal);
|
|
2068
|
+
}
|
|
2069
|
+
function claimSegment(storage, runDir, plan, segmentId, selected) {
|
|
2070
|
+
const path = join(runDir, SEGMENTS_FILE);
|
|
2071
|
+
const expected = canonicalJson({
|
|
2072
|
+
schemaVersion: 1,
|
|
2073
|
+
segmentId,
|
|
2074
|
+
planDigest: plan.planDigest,
|
|
2075
|
+
rowIds: selected.map((row) => row.rowId).sort()
|
|
2076
|
+
});
|
|
2077
|
+
for (let attempt = 0; attempt < 8; attempt += 1) {
|
|
2078
|
+
const raw = storage.read(path) ?? "";
|
|
2079
|
+
const claims = parseSegmentClaims(raw, path);
|
|
2080
|
+
const existing = claims.find((claim) => claim.segmentId === segmentId);
|
|
2081
|
+
if (existing) {
|
|
2082
|
+
if (canonicalJson(existing) !== expected) throw new SegmentedProfileMatrixError(`segment '${segmentId}' was resumed with different rows or plan identity`);
|
|
2083
|
+
return;
|
|
2084
|
+
}
|
|
2085
|
+
const selectedIds = new Set(selected.map((row) => row.rowId));
|
|
2086
|
+
const overlap = claims.flatMap((claim) => claim.rowIds.filter((rowId) => selectedIds.has(rowId)).map((rowId) => `${rowId} (${claim.segmentId})`));
|
|
2087
|
+
if (overlap.length > 0) throw new SegmentedProfileMatrixError(`segment rows overlap existing claims: ${overlap.join(", ")}`);
|
|
2088
|
+
const event = `${expected}\n`;
|
|
2089
|
+
if (storage.append(path, event, byteLength(raw)) !== void 0) return;
|
|
2090
|
+
}
|
|
2091
|
+
throw new SegmentedProfileMatrixError(`could not claim segment '${segmentId}' after concurrent updates`);
|
|
2092
|
+
}
|
|
2093
|
+
function readSegmentClaims(storage, runDir) {
|
|
2094
|
+
const path = join(runDir, SEGMENTS_FILE);
|
|
2095
|
+
const raw = storage.read(path);
|
|
2096
|
+
if (raw === void 0) return [];
|
|
2097
|
+
return parseSegmentClaims(raw, path);
|
|
2098
|
+
}
|
|
2099
|
+
function parseSegmentClaims(raw, path) {
|
|
2100
|
+
const claims = [];
|
|
2101
|
+
const seenSegments = /* @__PURE__ */ new Set();
|
|
2102
|
+
const seenRows = /* @__PURE__ */ new Map();
|
|
2103
|
+
for (const [index, line] of raw.split("\n").entries()) {
|
|
2104
|
+
if (line.trim() === "") continue;
|
|
2105
|
+
let parsed;
|
|
2106
|
+
try {
|
|
2107
|
+
parsed = JSON.parse(line);
|
|
2108
|
+
} catch {
|
|
2109
|
+
throw new SegmentedProfileMatrixError(`segment registry is corrupt at ${path}:${index + 1}`);
|
|
2110
|
+
}
|
|
2111
|
+
if (canonicalJson(parsed) !== line) throw new SegmentedProfileMatrixError(`segment registry is not canonical at ${path}:${index + 1}`);
|
|
2112
|
+
const claim = parsed;
|
|
2113
|
+
if (claim.schemaVersion !== 1 || typeof claim.segmentId !== "string" || typeof claim.planDigest !== "string" || !Array.isArray(claim.rowIds) || claim.rowIds.some((rowId) => typeof rowId !== "string") || new Set(claim.rowIds).size !== claim.rowIds.length) throw new SegmentedProfileMatrixError(`segment registry has an invalid claim at ${path}:${index + 1}`);
|
|
2114
|
+
if (seenSegments.has(claim.segmentId)) throw new SegmentedProfileMatrixError(`segment registry repeats segment '${claim.segmentId}'`);
|
|
2115
|
+
seenSegments.add(claim.segmentId);
|
|
2116
|
+
const normalized = {
|
|
2117
|
+
schemaVersion: 1,
|
|
2118
|
+
segmentId: claim.segmentId,
|
|
2119
|
+
planDigest: claim.planDigest,
|
|
2120
|
+
rowIds: [...claim.rowIds].sort()
|
|
2121
|
+
};
|
|
2122
|
+
for (const rowId of normalized.rowIds) {
|
|
2123
|
+
const owner = seenRows.get(rowId);
|
|
2124
|
+
if (owner) throw new SegmentedProfileMatrixError(`segment registry overlaps row '${rowId}' between '${owner}' and '${normalized.segmentId}'`);
|
|
2125
|
+
seenRows.set(rowId, normalized.segmentId);
|
|
2126
|
+
}
|
|
2127
|
+
claims.push(normalized);
|
|
2128
|
+
}
|
|
2129
|
+
return claims;
|
|
2130
|
+
}
|
|
2131
|
+
function assertCompleteClaims(plan, claims) {
|
|
2132
|
+
const expected = new Set(plan.rows.map((row) => row.rowId));
|
|
2133
|
+
const claimed = /* @__PURE__ */ new Set();
|
|
2134
|
+
for (const claim of claims) {
|
|
2135
|
+
if (claim.planDigest !== plan.planDigest) throw new SegmentedProfileMatrixError(`segment '${claim.segmentId}' belongs to a different plan identity`);
|
|
2136
|
+
for (const rowId of claim.rowIds) {
|
|
2137
|
+
if (!expected.has(rowId)) throw new SegmentedProfileMatrixError(`segment '${claim.segmentId}' claims an undeclared row '${rowId}'`);
|
|
2138
|
+
claimed.add(rowId);
|
|
2139
|
+
}
|
|
2140
|
+
}
|
|
2141
|
+
const missingClaims = plan.rows.filter((row) => !claimed.has(row.rowId)).map((row) => row.rowId);
|
|
2142
|
+
if (missingClaims.length > 0) throw new SegmentedProfileMatrixError(`cannot finalize until every row is claimed: ${missingClaims.join(", ")}`);
|
|
2143
|
+
}
|
|
2144
|
+
function persistSegmentCells(storage, runDir, matrix) {
|
|
2145
|
+
for (const [profileId, campaign] of Object.entries(matrix.campaigns)) for (const cell of campaign.cells) {
|
|
2146
|
+
const path = cellCachePath(join(runDir, profileDirectory(profileId)), cell.cellId);
|
|
2147
|
+
storage.ensureDir(dirname(path));
|
|
2148
|
+
storage.write(path, `${JSON.stringify(cell)}\n`);
|
|
2149
|
+
}
|
|
2150
|
+
}
|
|
2151
|
+
function readCoverage(storage, runDir, plan) {
|
|
2152
|
+
const expectedManifestHashes = new Map(plan.profiles.map((profile) => {
|
|
2153
|
+
const identity = profileMatrixCampaignIdentity({
|
|
2154
|
+
profile,
|
|
2155
|
+
scenarios: [...plan.scenarios],
|
|
2156
|
+
judges: [...plan.judges],
|
|
2157
|
+
commitSha: plan.commitSha,
|
|
2158
|
+
dispatchRef: plan.dispatchRef,
|
|
2159
|
+
splitTag: plan.splitTag,
|
|
2160
|
+
seed: plan.seed,
|
|
2161
|
+
reps: plan.reps
|
|
2162
|
+
});
|
|
2163
|
+
return [identity.profileId, identity.manifestHash];
|
|
2164
|
+
}));
|
|
2165
|
+
const missing = [];
|
|
2166
|
+
const failed = [];
|
|
2167
|
+
const zeroScore = [];
|
|
2168
|
+
let present = 0;
|
|
2169
|
+
for (const row of plan.rows) {
|
|
2170
|
+
const path = cellCachePath(join(runDir, profileDirectory(row.profileId)), `${row.scenarioId}:${row.rep}`);
|
|
2171
|
+
const raw = storage.read(path);
|
|
2172
|
+
if (raw === void 0) {
|
|
2173
|
+
missing.push(row.rowId);
|
|
2174
|
+
continue;
|
|
2175
|
+
}
|
|
2176
|
+
let parsed;
|
|
2177
|
+
try {
|
|
2178
|
+
parsed = JSON.parse(raw);
|
|
2179
|
+
} catch {
|
|
2180
|
+
throw new SegmentedProfileMatrixError(`cell '${row.rowId}' has corrupt JSON`);
|
|
2181
|
+
}
|
|
2182
|
+
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) throw new SegmentedProfileMatrixError(`cell '${row.rowId}' has an invalid record shape`);
|
|
2183
|
+
const cell = parsed;
|
|
2184
|
+
if (cell.cellId !== `${row.scenarioId}:${row.rep}` || cell.scenarioId !== row.scenarioId || cell.rep !== row.rep || cell.manifestHash !== expectedManifestHashes.get(row.profileId)) throw new SegmentedProfileMatrixError(`cell '${row.rowId}' has stale identity fields (manifest or row identity)`);
|
|
2185
|
+
try {
|
|
2186
|
+
campaignCellCostProvenance(cell);
|
|
2187
|
+
} catch (error) {
|
|
2188
|
+
throw new SegmentedProfileMatrixError(`cell '${row.rowId}' failed cost-record validation: ${error instanceof Error ? error.message : String(error)}`);
|
|
2189
|
+
}
|
|
2190
|
+
present += 1;
|
|
2191
|
+
if (cell.error !== void 0) {
|
|
2192
|
+
failed.push(row.rowId);
|
|
2193
|
+
continue;
|
|
2194
|
+
}
|
|
2195
|
+
if (projectCampaignCellQuality(cell).score === 0) zeroScore.push(row.rowId);
|
|
2196
|
+
}
|
|
2197
|
+
return {
|
|
2198
|
+
expected: plan.rows.length,
|
|
2199
|
+
present,
|
|
2200
|
+
missing,
|
|
2201
|
+
failed,
|
|
2202
|
+
zeroScore
|
|
2203
|
+
};
|
|
2204
|
+
}
|
|
2205
|
+
function assertFinalRows(plan, matrix) {
|
|
2206
|
+
const expected = new Set(plan.rows.map((row) => `${row.profileId}:${row.scenarioId}:${row.rep}`));
|
|
2207
|
+
const seen = /* @__PURE__ */ new Set();
|
|
2208
|
+
for (const record of matrix.records) {
|
|
2209
|
+
const key = `${record.candidateId}:${record.scenarioId}:${record.outcome.raw.rep}`;
|
|
2210
|
+
if (!expected.has(key) || seen.has(key)) throw new SegmentedProfileMatrixError(`final matrix has an unexpected or duplicate row '${key}'`);
|
|
2211
|
+
seen.add(key);
|
|
2212
|
+
}
|
|
2213
|
+
if (seen.size !== expected.size) throw new SegmentedProfileMatrixError(`final matrix returned ${seen.size}/${expected.size} declared rows`);
|
|
2214
|
+
}
|
|
2215
|
+
function rowIdFor(plan, profileId, scenarioId, rep) {
|
|
2216
|
+
const row = plan.rows.find((candidate) => candidate.profileId === profileId && candidate.scenarioId === scenarioId && candidate.rep === rep);
|
|
2217
|
+
if (!row) throw new SegmentedProfileMatrixError(`row is not declared: ${profileId}/${scenarioId}/${rep}`);
|
|
2218
|
+
return row.rowId;
|
|
2219
|
+
}
|
|
2220
|
+
function profileDirectory(profileId) {
|
|
2221
|
+
return profileId.replace(/[^a-zA-Z0-9_-]/g, "_");
|
|
2222
|
+
}
|
|
2223
|
+
function requireIdentifier(value, name) {
|
|
2224
|
+
if (typeof value !== "string" || value.trim().length === 0) throw new SegmentedProfileMatrixError(`${name} must be a non-empty string`);
|
|
2225
|
+
return value;
|
|
2226
|
+
}
|
|
2227
|
+
function byteLength(value) {
|
|
2228
|
+
return new TextEncoder().encode(value).byteLength;
|
|
2229
|
+
}
|
|
2230
|
+
//#endregion
|
|
1752
2231
|
//#region src/campaign/scenario-selection.ts
|
|
1753
2232
|
const DEFAULT_SATURATION_CEILING = .999;
|
|
1754
2233
|
/** Variance below this is treated as "every candidate scored the same". */
|
|
@@ -3177,6 +3656,6 @@ function resolveWorktreePath(surface, worktreeDir) {
|
|
|
3177
3656
|
return verifyCodeSurface(surface, worktreeDir).path;
|
|
3178
3657
|
}
|
|
3179
3658
|
//#endregion
|
|
3180
|
-
export {
|
|
3659
|
+
export { neutralizationGate as A, scoreboardSummary as C, LabeledScenarioStoreError as D, FsLabeledScenarioStore as E, analyzeCrossSurfaceInteractions as F, buildTraceAnalystSurfaceDispatch as I, traceAnalystQualityJudge as L, loadEvalFixture as M, loadEvalFixtureScenarios as N, classifyUngroundedLiterals as O, planEvalFixtureRun as P, scoreUserStory as S, neutralizeText as T, runProfileMatrixSegment as _, autoevalsScorerJudge as a, makePlaybackDispatch as b, FileSearchLedger as c, validateSearchLedgerEvent as d, scoreDiscrimination as f, finalizeProfileMatrix as g, createProfileMatrixPlan as h, verifyCodeSurface as i, discoverEvalFixtures as j, rolloutArgumentDiff as k, SEARCH_LEDGER_SCHEMA as l, SegmentedProfileMatrixError as m, gitWorktreeAdapter as n, phoenixEvaluatorJudge as o, selectDiscriminative as p, resolveWorktreePath as r, isTransientTransportFailure as s, WorktreeAdapterError as t, openSearchLedger as u, ProfileMatrixError as v, userStoryScoreboard as w, renderScoreboardMarkdown as x, runProfileMatrix as y };
|
|
3181
3660
|
|
|
3182
|
-
//# sourceMappingURL=campaign-
|
|
3661
|
+
//# sourceMappingURL=campaign-DGWSftLe.js.map
|