@tangle-network/agent-eval 0.122.3 → 0.122.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/benchmarks/index.js +4 -4
- package/dist/campaign/index.d.ts +267 -252
- package/dist/campaign/index.js +7 -3
- package/dist/{chunk-N3QPYIVG.js → chunk-4R4GTZBZ.js} +10 -10
- package/dist/{chunk-IDC74VDL.js → chunk-75OCWDXJ.js} +111 -24
- package/dist/chunk-75OCWDXJ.js.map +1 -0
- package/dist/{chunk-BD2OK6ZW.js → chunk-7GLJX32M.js} +2 -2
- package/dist/{chunk-OZFTLMGS.js → chunk-GTSVLG2M.js} +2 -2
- package/dist/{chunk-DHJJACEX.js → chunk-MWN2X7C5.js} +3 -3
- package/dist/contract/index.js +3 -3
- package/dist/index.d.ts +29 -29
- package/dist/index.js +5 -5
- package/dist/openapi.json +1 -1
- package/dist/{run-campaign-2YJ3Z6JS.js → run-campaign-J6RHX2VO.js} +2 -2
- package/package.json +1 -1
- package/dist/chunk-IDC74VDL.js.map +0 -1
- /package/dist/{chunk-N3QPYIVG.js.map → chunk-4R4GTZBZ.js.map} +0 -0
- /package/dist/{chunk-BD2OK6ZW.js.map → chunk-7GLJX32M.js.map} +0 -0
- /package/dist/{chunk-OZFTLMGS.js.map → chunk-GTSVLG2M.js.map} +0 -0
- /package/dist/{chunk-DHJJACEX.js.map → chunk-MWN2X7C5.js.map} +0 -0
- /package/dist/{run-campaign-2YJ3Z6JS.js.map → run-campaign-J6RHX2VO.js.map} +0 -0
package/dist/campaign/index.js
CHANGED
|
@@ -4,6 +4,7 @@ import {
|
|
|
4
4
|
FsLabeledScenarioStore,
|
|
5
5
|
LabeledScenarioStoreError,
|
|
6
6
|
Lineage,
|
|
7
|
+
LineageStoreConflictError,
|
|
7
8
|
ProfileMatrixError,
|
|
8
9
|
SEARCH_LEDGER_SCHEMA,
|
|
9
10
|
SkillPatchParseError,
|
|
@@ -15,6 +16,7 @@ import {
|
|
|
15
16
|
assertPolicyEditAuthorContextBudget,
|
|
16
17
|
buildAnalystSurfaceDispatch,
|
|
17
18
|
callbackGovernor,
|
|
19
|
+
campaignLineageStore,
|
|
18
20
|
classifyUngroundedLiterals,
|
|
19
21
|
compareProposers,
|
|
20
22
|
compositeProposer,
|
|
@@ -66,7 +68,7 @@ import {
|
|
|
66
68
|
userStoryScoreboard,
|
|
67
69
|
validateSearchLedgerEvent,
|
|
68
70
|
verifyCodeSurface
|
|
69
|
-
} from "../chunk-
|
|
71
|
+
} from "../chunk-75OCWDXJ.js";
|
|
70
72
|
import {
|
|
71
73
|
assertCodeSurfaceIdentity,
|
|
72
74
|
buildEvidenceVector,
|
|
@@ -107,7 +109,7 @@ import {
|
|
|
107
109
|
surfaceContentHash,
|
|
108
110
|
surfaceHash,
|
|
109
111
|
verifyLoopProvenanceRecord
|
|
110
|
-
} from "../chunk-
|
|
112
|
+
} from "../chunk-MWN2X7C5.js";
|
|
111
113
|
import {
|
|
112
114
|
SearchLedgerConflictError,
|
|
113
115
|
SearchLedgerError,
|
|
@@ -124,7 +126,7 @@ import {
|
|
|
124
126
|
resolveRunDir,
|
|
125
127
|
runCampaign,
|
|
126
128
|
tangleTracesRoot
|
|
127
|
-
} from "../chunk-
|
|
129
|
+
} from "../chunk-4R4GTZBZ.js";
|
|
128
130
|
import {
|
|
129
131
|
POLICY_EDIT_CANDIDATE_RECORD_SCHEMA,
|
|
130
132
|
validatePolicyEditCandidateRecord
|
|
@@ -151,6 +153,7 @@ export {
|
|
|
151
153
|
FsLabeledScenarioStore,
|
|
152
154
|
LabeledScenarioStoreError,
|
|
153
155
|
Lineage,
|
|
156
|
+
LineageStoreConflictError,
|
|
154
157
|
POLICY_EDIT_CANDIDATE_RECORD_SCHEMA,
|
|
155
158
|
ProfileMatrixError,
|
|
156
159
|
SEARCH_LEDGER_SCHEMA,
|
|
@@ -172,6 +175,7 @@ export {
|
|
|
172
175
|
buildLoopProvenanceRecord,
|
|
173
176
|
callbackGovernor,
|
|
174
177
|
campaignBreakdown,
|
|
178
|
+
campaignLineageStore,
|
|
175
179
|
campaignMeanComposite,
|
|
176
180
|
campaignMeasurementDigest,
|
|
177
181
|
campaignScenarioIdentity,
|
|
@@ -1339,15 +1339,6 @@ export {
|
|
|
1339
1339
|
inMemoryVerdictCache,
|
|
1340
1340
|
fileVerdictCache,
|
|
1341
1341
|
cachedJudge,
|
|
1342
|
-
assertCampaignDesign,
|
|
1343
|
-
campaignScenarioIdentity,
|
|
1344
|
-
campaignSplitDigestFromIdentities,
|
|
1345
|
-
campaignSplitDigest,
|
|
1346
|
-
assertCampaignSplitIdentity,
|
|
1347
|
-
campaignCoverage,
|
|
1348
|
-
formatCoverageFailures,
|
|
1349
|
-
tangleTracesRoot,
|
|
1350
|
-
resolveRunDir,
|
|
1351
1342
|
probeAtomicFileLock,
|
|
1352
1343
|
tryAcquireAtomicFileLock,
|
|
1353
1344
|
SearchLedgerError,
|
|
@@ -1358,7 +1349,16 @@ export {
|
|
|
1358
1349
|
fsCampaignStorage,
|
|
1359
1350
|
inMemoryCampaignStorage,
|
|
1360
1351
|
createRunCostLedger,
|
|
1352
|
+
assertCampaignDesign,
|
|
1353
|
+
campaignScenarioIdentity,
|
|
1354
|
+
campaignSplitDigestFromIdentities,
|
|
1355
|
+
campaignSplitDigest,
|
|
1356
|
+
assertCampaignSplitIdentity,
|
|
1357
|
+
campaignCoverage,
|
|
1358
|
+
formatCoverageFailures,
|
|
1359
|
+
tangleTracesRoot,
|
|
1360
|
+
resolveRunDir,
|
|
1361
1361
|
runCampaign,
|
|
1362
1362
|
planCampaignRun
|
|
1363
1363
|
};
|
|
1364
|
-
//# sourceMappingURL=chunk-
|
|
1364
|
+
//# sourceMappingURL=chunk-4R4GTZBZ.js.map
|
|
@@ -15,13 +15,14 @@ import {
|
|
|
15
15
|
runImprovementLoop,
|
|
16
16
|
surfaceContentHash,
|
|
17
17
|
surfaceHash
|
|
18
|
-
} from "./chunk-
|
|
18
|
+
} from "./chunk-MWN2X7C5.js";
|
|
19
19
|
import {
|
|
20
20
|
SearchLedgerConflictError,
|
|
21
21
|
SearchLedgerError,
|
|
22
22
|
SearchLedgerIntegrityError,
|
|
23
23
|
appendSearchLedgerLine,
|
|
24
24
|
assertRealBackend,
|
|
25
|
+
canonicalJson,
|
|
25
26
|
contentHash,
|
|
26
27
|
createRunCostLedger,
|
|
27
28
|
fsCampaignStorage,
|
|
@@ -32,7 +33,7 @@ import {
|
|
|
32
33
|
summarizeBackendIntegrity,
|
|
33
34
|
tryAcquireAtomicFileLock,
|
|
34
35
|
withSearchLedgerFileLock
|
|
35
|
-
} from "./chunk-
|
|
36
|
+
} from "./chunk-4R4GTZBZ.js";
|
|
36
37
|
import {
|
|
37
38
|
AnalystRegistry,
|
|
38
39
|
DEFAULT_TRACE_ANALYST_KINDS,
|
|
@@ -93,7 +94,6 @@ import {
|
|
|
93
94
|
|
|
94
95
|
// src/campaign/lineage.ts
|
|
95
96
|
import { createHash } from "crypto";
|
|
96
|
-
import { appendFile, mkdir, readFile, writeFile } from "fs/promises";
|
|
97
97
|
import { dirname } from "path";
|
|
98
98
|
function lineageNodeId(input) {
|
|
99
99
|
const parents = [...input.parentIds].sort().join(",");
|
|
@@ -295,30 +295,71 @@ function dominates(a, b) {
|
|
|
295
295
|
}
|
|
296
296
|
return strictlyBetter;
|
|
297
297
|
}
|
|
298
|
-
|
|
299
|
-
|
|
298
|
+
var LineageStoreConflictError = class extends Error {
|
|
299
|
+
name = "LineageStoreConflictError";
|
|
300
|
+
};
|
|
301
|
+
function campaignLineageStore(storage, path, options = {}) {
|
|
302
|
+
const maxAppendAttempts = options.maxAppendAttempts ?? 100;
|
|
303
|
+
if (!Number.isInteger(maxAppendAttempts) || maxAppendAttempts < 1) {
|
|
304
|
+
throw new Error("campaignLineageStore: maxAppendAttempts must be a positive integer");
|
|
305
|
+
}
|
|
306
|
+
storage.ensureDir(dirname(path));
|
|
307
|
+
const read = () => {
|
|
308
|
+
const stored = storage.read(path);
|
|
309
|
+
if (stored === void 0 && storage.exists(path)) {
|
|
310
|
+
throw new Error(`campaignLineageStore: cannot read existing lineage '${path}'`);
|
|
311
|
+
}
|
|
312
|
+
const text = stored ?? "";
|
|
313
|
+
return { text, lineage: Lineage.fromJSONL(text) };
|
|
314
|
+
};
|
|
300
315
|
return {
|
|
301
316
|
async load() {
|
|
302
|
-
|
|
303
|
-
return Lineage.fromJSONL(await readFile(path, "utf8"));
|
|
304
|
-
} catch (err) {
|
|
305
|
-
if (err.code === "ENOENT") return new Lineage();
|
|
306
|
-
throw err;
|
|
307
|
-
}
|
|
317
|
+
return read().lineage;
|
|
308
318
|
},
|
|
309
319
|
async append(node) {
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
320
|
+
if (!storage.append) {
|
|
321
|
+
throw new Error("campaignLineageStore: CampaignStorage.append is required");
|
|
322
|
+
}
|
|
323
|
+
for (let attempt = 0; attempt < maxAppendAttempts; attempt += 1) {
|
|
324
|
+
const { text, lineage } = read();
|
|
325
|
+
const existing = lineage.get(node.id);
|
|
326
|
+
if (existing) {
|
|
327
|
+
assertSamePersistedNode(existing, node, "campaignLineageStore");
|
|
328
|
+
return;
|
|
329
|
+
}
|
|
330
|
+
for (const parentId of node.parentIds) {
|
|
331
|
+
if (!lineage.has(parentId)) {
|
|
332
|
+
throw new LineageStoreConflictError(
|
|
333
|
+
`campaignLineageStore: node '${node.id}' has unknown persisted parent '${parentId}'`
|
|
334
|
+
);
|
|
335
|
+
}
|
|
336
|
+
}
|
|
337
|
+
const persisted = lineage.all();
|
|
338
|
+
const expectedSeq = persisted.length === 0 ? 0 : Math.max(...persisted.map((entry) => entry.seq)) + 1;
|
|
339
|
+
if (node.seq !== expectedSeq) {
|
|
340
|
+
throw new LineageStoreConflictError(
|
|
341
|
+
`campaignLineageStore: stale controller tried sequence ${node.seq}; expected ${expectedSeq}`
|
|
342
|
+
);
|
|
343
|
+
}
|
|
344
|
+
const line = `${JSON.stringify(node)}
|
|
345
|
+
`;
|
|
346
|
+
const expectedBytes = new TextEncoder().encode(text).byteLength;
|
|
347
|
+
if (storage.append(path, line, expectedBytes) !== void 0) return;
|
|
348
|
+
}
|
|
349
|
+
throw new LineageStoreConflictError(
|
|
350
|
+
`campaignLineageStore: could not append after ${maxAppendAttempts} attempts`
|
|
351
|
+
);
|
|
313
352
|
},
|
|
314
353
|
async save(lineage) {
|
|
315
|
-
await ensureDir();
|
|
316
354
|
const jsonl = lineage.toJSONL();
|
|
317
|
-
|
|
318
|
-
` : ""
|
|
355
|
+
storage.write(path, jsonl.length > 0 ? `${jsonl}
|
|
356
|
+
` : "");
|
|
319
357
|
}
|
|
320
358
|
};
|
|
321
359
|
}
|
|
360
|
+
function fsLineageStore(path) {
|
|
361
|
+
return campaignLineageStore(fsCampaignStorage(), path);
|
|
362
|
+
}
|
|
322
363
|
function memLineageStore() {
|
|
323
364
|
const nodes = [];
|
|
324
365
|
return {
|
|
@@ -326,6 +367,11 @@ function memLineageStore() {
|
|
|
326
367
|
return new Lineage(nodes);
|
|
327
368
|
},
|
|
328
369
|
async append(node) {
|
|
370
|
+
const existing = nodes.find((entry) => entry.id === node.id);
|
|
371
|
+
if (existing) {
|
|
372
|
+
assertSamePersistedNode(existing, node, "memLineageStore");
|
|
373
|
+
return;
|
|
374
|
+
}
|
|
329
375
|
nodes.push(node);
|
|
330
376
|
},
|
|
331
377
|
async save(lineage) {
|
|
@@ -334,6 +380,13 @@ function memLineageStore() {
|
|
|
334
380
|
}
|
|
335
381
|
};
|
|
336
382
|
}
|
|
383
|
+
function assertSamePersistedNode(existing, candidate, store) {
|
|
384
|
+
if (canonicalJson(existing) !== canonicalJson(candidate)) {
|
|
385
|
+
throw new LineageStoreConflictError(
|
|
386
|
+
`${store}: node '${candidate.id}' already exists with different content`
|
|
387
|
+
);
|
|
388
|
+
}
|
|
389
|
+
}
|
|
337
390
|
function heuristicGovernor(opts = {}) {
|
|
338
391
|
const maxTracks = opts.maxTracks ?? 3;
|
|
339
392
|
const plateauSteps = opts.plateauSteps ?? 2;
|
|
@@ -389,11 +442,35 @@ function isPlateaued(trackNodes, window) {
|
|
|
389
442
|
return bestRecent <= bestBefore;
|
|
390
443
|
}
|
|
391
444
|
async function runLineage(opts) {
|
|
445
|
+
if (!Number.isInteger(opts.budget.maxSteps) || opts.budget.maxSteps < 0) {
|
|
446
|
+
throw new Error("runLineage: budget.maxSteps must be a non-negative integer");
|
|
447
|
+
}
|
|
448
|
+
if (opts.budget.maxNodes !== void 0 && (!Number.isInteger(opts.budget.maxNodes) || opts.budget.maxNodes < 0)) {
|
|
449
|
+
throw new Error("runLineage: budget.maxNodes must be a non-negative integer");
|
|
450
|
+
}
|
|
392
451
|
const store = opts.store ?? memLineageStore();
|
|
393
452
|
const lineage = await store.load();
|
|
394
453
|
const log = opts.log ?? (() => {
|
|
395
454
|
});
|
|
396
455
|
const pruned = /* @__PURE__ */ new Set();
|
|
456
|
+
if (opts.budget.maxNodes !== void 0) {
|
|
457
|
+
const missingSeedIds = new Set(
|
|
458
|
+
opts.seeds.map(
|
|
459
|
+
(seed) => lineageNodeId({
|
|
460
|
+
parentIds: [],
|
|
461
|
+
track: seed.track,
|
|
462
|
+
surface: seed.surface,
|
|
463
|
+
proposer: seed.proposer
|
|
464
|
+
})
|
|
465
|
+
).filter((id) => !lineage.has(id))
|
|
466
|
+
);
|
|
467
|
+
const available = Math.max(0, opts.budget.maxNodes - lineage.all().length);
|
|
468
|
+
if (missingSeedIds.size > available) {
|
|
469
|
+
throw new Error(
|
|
470
|
+
`runLineage: seed set requires ${missingSeedIds.size} new nodes but budget.maxNodes has ${available} slots remaining`
|
|
471
|
+
);
|
|
472
|
+
}
|
|
473
|
+
}
|
|
397
474
|
const trackProposer = /* @__PURE__ */ new Map();
|
|
398
475
|
const persist = async (node) => {
|
|
399
476
|
await store.append(node);
|
|
@@ -413,10 +490,18 @@ async function runLineage(opts) {
|
|
|
413
490
|
}
|
|
414
491
|
let steps = 0;
|
|
415
492
|
while (steps < opts.budget.maxSteps) {
|
|
493
|
+
if (opts.budget.maxNodes !== void 0 && lineage.all().length >= opts.budget.maxNodes) {
|
|
494
|
+
log("lineage: persisted node limit reached", {
|
|
495
|
+
maxNodes: opts.budget.maxNodes,
|
|
496
|
+
nodes: lineage.all().length
|
|
497
|
+
});
|
|
498
|
+
break;
|
|
499
|
+
}
|
|
500
|
+
const nodeBudgetRemaining = opts.budget.maxNodes === void 0 ? Number.POSITIVE_INFINITY : opts.budget.maxNodes - lineage.all().length;
|
|
416
501
|
const op = await opts.governor.decide({
|
|
417
502
|
lineage,
|
|
418
503
|
step: steps,
|
|
419
|
-
budgetRemaining: opts.budget.maxSteps - steps,
|
|
504
|
+
budgetRemaining: Math.min(opts.budget.maxSteps - steps, nodeBudgetRemaining),
|
|
420
505
|
prunedTracks: [...pruned]
|
|
421
506
|
});
|
|
422
507
|
if (op.op === "stop") {
|
|
@@ -2939,14 +3024,14 @@ function parameterSweepProposer(opts) {
|
|
|
2939
3024
|
const parse = opts.parse ?? parseJsonObject;
|
|
2940
3025
|
const stringify = opts.stringify ?? ((config) => JSON.stringify(config, null, 2));
|
|
2941
3026
|
const current = parse(ctx.currentSurface);
|
|
2942
|
-
const currentCanonical =
|
|
3027
|
+
const currentCanonical = canonicalJson2(current);
|
|
2943
3028
|
const tried = triedLabels(ctx.history);
|
|
2944
3029
|
const out = [];
|
|
2945
3030
|
for (const candidate of opts.candidates) {
|
|
2946
3031
|
if (tried.has(candidate.label)) continue;
|
|
2947
3032
|
const next = applyParameterCandidate(current, candidate);
|
|
2948
3033
|
const surface = stringify(next);
|
|
2949
|
-
if (surface === ctx.currentSurface ||
|
|
3034
|
+
if (surface === ctx.currentSurface || canonicalJson2(next) === currentCanonical) continue;
|
|
2950
3035
|
out.push({ surface, label: candidate.label, rationale: candidate.rationale });
|
|
2951
3036
|
if (out.length >= ctx.populationSize) break;
|
|
2952
3037
|
}
|
|
@@ -3023,10 +3108,10 @@ function assertSafeJsonKey(key) {
|
|
|
3023
3108
|
throw new Error(`parameterSweepProposer: unsafe JSON key "${key}" is not allowed`);
|
|
3024
3109
|
}
|
|
3025
3110
|
}
|
|
3026
|
-
function
|
|
3027
|
-
if (Array.isArray(value)) return `[${value.map((item) =>
|
|
3111
|
+
function canonicalJson2(value) {
|
|
3112
|
+
if (Array.isArray(value)) return `[${value.map((item) => canonicalJson2(item)).join(",")}]`;
|
|
3028
3113
|
if (isPlainObject(value)) {
|
|
3029
|
-
return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${
|
|
3114
|
+
return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${canonicalJson2(value[key])}`).join(",")}}`;
|
|
3030
3115
|
}
|
|
3031
3116
|
return JSON.stringify(value);
|
|
3032
3117
|
}
|
|
@@ -8057,6 +8142,8 @@ export {
|
|
|
8057
8142
|
agentProfileHash,
|
|
8058
8143
|
lineageNodeId,
|
|
8059
8144
|
Lineage,
|
|
8145
|
+
LineageStoreConflictError,
|
|
8146
|
+
campaignLineageStore,
|
|
8060
8147
|
fsLineageStore,
|
|
8061
8148
|
memLineageStore,
|
|
8062
8149
|
heuristicGovernor,
|
|
@@ -8123,4 +8210,4 @@ export {
|
|
|
8123
8210
|
verifyCodeSurface,
|
|
8124
8211
|
resolveWorktreePath
|
|
8125
8212
|
};
|
|
8126
|
-
//# sourceMappingURL=chunk-
|
|
8213
|
+
//# sourceMappingURL=chunk-75OCWDXJ.js.map
|