@tangle-network/agent-eval 0.122.2 → 0.122.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. package/dist/analyst/index.js +3 -4
  2. package/dist/analyst/index.js.map +1 -1
  3. package/dist/benchmarks/index.js +5 -6
  4. package/dist/campaign/index.d.ts +273 -252
  5. package/dist/campaign/index.js +8 -5
  6. package/dist/{chunk-N3QPYIVG.js → chunk-4R4GTZBZ.js} +10 -10
  7. package/dist/{chunk-EXMKNOP5.js → chunk-75OCWDXJ.js} +137 -43
  8. package/dist/chunk-75OCWDXJ.js.map +1 -0
  9. package/dist/{chunk-BD2OK6ZW.js → chunk-7GLJX32M.js} +2 -2
  10. package/dist/{chunk-5CVUPHJ4.js → chunk-EAWKAVID.js} +74 -1
  11. package/dist/chunk-EAWKAVID.js.map +1 -0
  12. package/dist/{chunk-OZFTLMGS.js → chunk-GTSVLG2M.js} +2 -2
  13. package/dist/{chunk-U4R2AGY5.js → chunk-MWN2X7C5.js} +42 -32
  14. package/dist/{chunk-U4R2AGY5.js.map → chunk-MWN2X7C5.js.map} +1 -1
  15. package/dist/{chunk-5BYTIDZ7.js → chunk-SFXDMFGV.js} +2 -2
  16. package/dist/{chunk-XKSDSGSC.js → chunk-ULW7AATT.js} +3 -5
  17. package/dist/{chunk-XKSDSGSC.js.map → chunk-ULW7AATT.js.map} +1 -1
  18. package/dist/contract/index.d.ts +3 -0
  19. package/dist/contract/index.js +5 -5
  20. package/dist/index.d.ts +36 -30
  21. package/dist/index.js +11 -11
  22. package/dist/index.js.map +1 -1
  23. package/dist/openapi.json +1 -1
  24. package/dist/{run-campaign-2YJ3Z6JS.js → run-campaign-J6RHX2VO.js} +2 -2
  25. package/package.json +1 -1
  26. package/dist/chunk-3YYRZDON.js +0 -45
  27. package/dist/chunk-3YYRZDON.js.map +0 -1
  28. package/dist/chunk-5CVUPHJ4.js.map +0 -1
  29. package/dist/chunk-EXMKNOP5.js.map +0 -1
  30. /package/dist/{chunk-N3QPYIVG.js.map → chunk-4R4GTZBZ.js.map} +0 -0
  31. /package/dist/{chunk-BD2OK6ZW.js.map → chunk-7GLJX32M.js.map} +0 -0
  32. /package/dist/{chunk-OZFTLMGS.js.map → chunk-GTSVLG2M.js.map} +0 -0
  33. /package/dist/{chunk-5BYTIDZ7.js.map → chunk-SFXDMFGV.js.map} +0 -0
  34. /package/dist/{run-campaign-2YJ3Z6JS.js.map → run-campaign-J6RHX2VO.js.map} +0 -0
@@ -4,6 +4,7 @@ import {
4
4
  FsLabeledScenarioStore,
5
5
  LabeledScenarioStoreError,
6
6
  Lineage,
7
+ LineageStoreConflictError,
7
8
  ProfileMatrixError,
8
9
  SEARCH_LEDGER_SCHEMA,
9
10
  SkillPatchParseError,
@@ -15,6 +16,7 @@ import {
15
16
  assertPolicyEditAuthorContextBudget,
16
17
  buildAnalystSurfaceDispatch,
17
18
  callbackGovernor,
19
+ campaignLineageStore,
18
20
  classifyUngroundedLiterals,
19
21
  compareProposers,
20
22
  compositeProposer,
@@ -66,7 +68,7 @@ import {
66
68
  userStoryScoreboard,
67
69
  validateSearchLedgerEvent,
68
70
  verifyCodeSurface
69
- } from "../chunk-EXMKNOP5.js";
71
+ } from "../chunk-75OCWDXJ.js";
70
72
  import {
71
73
  assertCodeSurfaceIdentity,
72
74
  buildEvidenceVector,
@@ -107,7 +109,7 @@ import {
107
109
  surfaceContentHash,
108
110
  surfaceHash,
109
111
  verifyLoopProvenanceRecord
110
- } from "../chunk-U4R2AGY5.js";
112
+ } from "../chunk-MWN2X7C5.js";
111
113
  import {
112
114
  SearchLedgerConflictError,
113
115
  SearchLedgerError,
@@ -124,12 +126,11 @@ import {
124
126
  resolveRunDir,
125
127
  runCampaign,
126
128
  tangleTracesRoot
127
- } from "../chunk-N3QPYIVG.js";
128
- import "../chunk-3YYRZDON.js";
129
+ } from "../chunk-4R4GTZBZ.js";
129
130
  import {
130
131
  POLICY_EDIT_CANDIDATE_RECORD_SCHEMA,
131
132
  validatePolicyEditCandidateRecord
132
- } from "../chunk-5CVUPHJ4.js";
133
+ } from "../chunk-EAWKAVID.js";
133
134
  import "../chunk-ARU2PZFM.js";
134
135
  import "../chunk-NJC7U437.js";
135
136
  import "../chunk-PJQFMIOX.js";
@@ -152,6 +153,7 @@ export {
152
153
  FsLabeledScenarioStore,
153
154
  LabeledScenarioStoreError,
154
155
  Lineage,
156
+ LineageStoreConflictError,
155
157
  POLICY_EDIT_CANDIDATE_RECORD_SCHEMA,
156
158
  ProfileMatrixError,
157
159
  SEARCH_LEDGER_SCHEMA,
@@ -173,6 +175,7 @@ export {
173
175
  buildLoopProvenanceRecord,
174
176
  callbackGovernor,
175
177
  campaignBreakdown,
178
+ campaignLineageStore,
176
179
  campaignMeanComposite,
177
180
  campaignMeasurementDigest,
178
181
  campaignScenarioIdentity,
@@ -1339,15 +1339,6 @@ export {
1339
1339
  inMemoryVerdictCache,
1340
1340
  fileVerdictCache,
1341
1341
  cachedJudge,
1342
- assertCampaignDesign,
1343
- campaignScenarioIdentity,
1344
- campaignSplitDigestFromIdentities,
1345
- campaignSplitDigest,
1346
- assertCampaignSplitIdentity,
1347
- campaignCoverage,
1348
- formatCoverageFailures,
1349
- tangleTracesRoot,
1350
- resolveRunDir,
1351
1342
  probeAtomicFileLock,
1352
1343
  tryAcquireAtomicFileLock,
1353
1344
  SearchLedgerError,
@@ -1358,7 +1349,16 @@ export {
1358
1349
  fsCampaignStorage,
1359
1350
  inMemoryCampaignStorage,
1360
1351
  createRunCostLedger,
1352
+ assertCampaignDesign,
1353
+ campaignScenarioIdentity,
1354
+ campaignSplitDigestFromIdentities,
1355
+ campaignSplitDigest,
1356
+ assertCampaignSplitIdentity,
1357
+ campaignCoverage,
1358
+ formatCoverageFailures,
1359
+ tangleTracesRoot,
1360
+ resolveRunDir,
1361
1361
  runCampaign,
1362
1362
  planCampaignRun
1363
1363
  };
1364
- //# sourceMappingURL=chunk-N3QPYIVG.js.map
1364
+ //# sourceMappingURL=chunk-4R4GTZBZ.js.map
@@ -15,13 +15,14 @@ import {
15
15
  runImprovementLoop,
16
16
  surfaceContentHash,
17
17
  surfaceHash
18
- } from "./chunk-U4R2AGY5.js";
18
+ } from "./chunk-MWN2X7C5.js";
19
19
  import {
20
20
  SearchLedgerConflictError,
21
21
  SearchLedgerError,
22
22
  SearchLedgerIntegrityError,
23
23
  appendSearchLedgerLine,
24
24
  assertRealBackend,
25
+ canonicalJson,
25
26
  contentHash,
26
27
  createRunCostLedger,
27
28
  fsCampaignStorage,
@@ -32,13 +33,11 @@ import {
32
33
  summarizeBackendIntegrity,
33
34
  tryAcquireAtomicFileLock,
34
35
  withSearchLedgerFileLock
35
- } from "./chunk-N3QPYIVG.js";
36
- import {
37
- Mutex
38
- } from "./chunk-3YYRZDON.js";
36
+ } from "./chunk-4R4GTZBZ.js";
39
37
  import {
40
38
  AnalystRegistry,
41
39
  DEFAULT_TRACE_ANALYST_KINDS,
40
+ Mutex,
42
41
  POLICY_EDIT_AXES,
43
42
  POLICY_EDIT_TARGET_SURFACES,
44
43
  admitPolicyEdit,
@@ -49,9 +48,10 @@ import {
49
48
  isPolicyEdit,
50
49
  makePolicyEdit,
51
50
  makePolicyEditCandidateRecord,
51
+ mapConcurrent,
52
52
  policyEditsFromFindings,
53
53
  validatePolicyEditCandidateRecord
54
- } from "./chunk-5CVUPHJ4.js";
54
+ } from "./chunk-EAWKAVID.js";
55
55
  import {
56
56
  callLlm,
57
57
  callLlmJson,
@@ -94,7 +94,6 @@ import {
94
94
 
95
95
  // src/campaign/lineage.ts
96
96
  import { createHash } from "crypto";
97
- import { appendFile, mkdir, readFile, writeFile } from "fs/promises";
98
97
  import { dirname } from "path";
99
98
  function lineageNodeId(input) {
100
99
  const parents = [...input.parentIds].sort().join(",");
@@ -296,30 +295,71 @@ function dominates(a, b) {
296
295
  }
297
296
  return strictlyBetter;
298
297
  }
299
- function fsLineageStore(path) {
300
- const ensureDir = () => mkdir(dirname(path), { recursive: true });
298
+ var LineageStoreConflictError = class extends Error {
299
+ name = "LineageStoreConflictError";
300
+ };
301
+ function campaignLineageStore(storage, path, options = {}) {
302
+ const maxAppendAttempts = options.maxAppendAttempts ?? 100;
303
+ if (!Number.isInteger(maxAppendAttempts) || maxAppendAttempts < 1) {
304
+ throw new Error("campaignLineageStore: maxAppendAttempts must be a positive integer");
305
+ }
306
+ storage.ensureDir(dirname(path));
307
+ const read = () => {
308
+ const stored = storage.read(path);
309
+ if (stored === void 0 && storage.exists(path)) {
310
+ throw new Error(`campaignLineageStore: cannot read existing lineage '${path}'`);
311
+ }
312
+ const text = stored ?? "";
313
+ return { text, lineage: Lineage.fromJSONL(text) };
314
+ };
301
315
  return {
302
316
  async load() {
303
- try {
304
- return Lineage.fromJSONL(await readFile(path, "utf8"));
305
- } catch (err) {
306
- if (err.code === "ENOENT") return new Lineage();
307
- throw err;
308
- }
317
+ return read().lineage;
309
318
  },
310
319
  async append(node) {
311
- await ensureDir();
312
- await appendFile(path, `${JSON.stringify(node)}
313
- `, "utf8");
320
+ if (!storage.append) {
321
+ throw new Error("campaignLineageStore: CampaignStorage.append is required");
322
+ }
323
+ for (let attempt = 0; attempt < maxAppendAttempts; attempt += 1) {
324
+ const { text, lineage } = read();
325
+ const existing = lineage.get(node.id);
326
+ if (existing) {
327
+ assertSamePersistedNode(existing, node, "campaignLineageStore");
328
+ return;
329
+ }
330
+ for (const parentId of node.parentIds) {
331
+ if (!lineage.has(parentId)) {
332
+ throw new LineageStoreConflictError(
333
+ `campaignLineageStore: node '${node.id}' has unknown persisted parent '${parentId}'`
334
+ );
335
+ }
336
+ }
337
+ const persisted = lineage.all();
338
+ const expectedSeq = persisted.length === 0 ? 0 : Math.max(...persisted.map((entry) => entry.seq)) + 1;
339
+ if (node.seq !== expectedSeq) {
340
+ throw new LineageStoreConflictError(
341
+ `campaignLineageStore: stale controller tried sequence ${node.seq}; expected ${expectedSeq}`
342
+ );
343
+ }
344
+ const line = `${JSON.stringify(node)}
345
+ `;
346
+ const expectedBytes = new TextEncoder().encode(text).byteLength;
347
+ if (storage.append(path, line, expectedBytes) !== void 0) return;
348
+ }
349
+ throw new LineageStoreConflictError(
350
+ `campaignLineageStore: could not append after ${maxAppendAttempts} attempts`
351
+ );
314
352
  },
315
353
  async save(lineage) {
316
- await ensureDir();
317
354
  const jsonl = lineage.toJSONL();
318
- await writeFile(path, jsonl.length > 0 ? `${jsonl}
319
- ` : "", "utf8");
355
+ storage.write(path, jsonl.length > 0 ? `${jsonl}
356
+ ` : "");
320
357
  }
321
358
  };
322
359
  }
360
+ function fsLineageStore(path) {
361
+ return campaignLineageStore(fsCampaignStorage(), path);
362
+ }
323
363
  function memLineageStore() {
324
364
  const nodes = [];
325
365
  return {
@@ -327,6 +367,11 @@ function memLineageStore() {
327
367
  return new Lineage(nodes);
328
368
  },
329
369
  async append(node) {
370
+ const existing = nodes.find((entry) => entry.id === node.id);
371
+ if (existing) {
372
+ assertSamePersistedNode(existing, node, "memLineageStore");
373
+ return;
374
+ }
330
375
  nodes.push(node);
331
376
  },
332
377
  async save(lineage) {
@@ -335,6 +380,13 @@ function memLineageStore() {
335
380
  }
336
381
  };
337
382
  }
383
+ function assertSamePersistedNode(existing, candidate, store) {
384
+ if (canonicalJson(existing) !== canonicalJson(candidate)) {
385
+ throw new LineageStoreConflictError(
386
+ `${store}: node '${candidate.id}' already exists with different content`
387
+ );
388
+ }
389
+ }
338
390
  function heuristicGovernor(opts = {}) {
339
391
  const maxTracks = opts.maxTracks ?? 3;
340
392
  const plateauSteps = opts.plateauSteps ?? 2;
@@ -390,11 +442,35 @@ function isPlateaued(trackNodes, window) {
390
442
  return bestRecent <= bestBefore;
391
443
  }
392
444
  async function runLineage(opts) {
445
+ if (!Number.isInteger(opts.budget.maxSteps) || opts.budget.maxSteps < 0) {
446
+ throw new Error("runLineage: budget.maxSteps must be a non-negative integer");
447
+ }
448
+ if (opts.budget.maxNodes !== void 0 && (!Number.isInteger(opts.budget.maxNodes) || opts.budget.maxNodes < 0)) {
449
+ throw new Error("runLineage: budget.maxNodes must be a non-negative integer");
450
+ }
393
451
  const store = opts.store ?? memLineageStore();
394
452
  const lineage = await store.load();
395
453
  const log = opts.log ?? (() => {
396
454
  });
397
455
  const pruned = /* @__PURE__ */ new Set();
456
+ if (opts.budget.maxNodes !== void 0) {
457
+ const missingSeedIds = new Set(
458
+ opts.seeds.map(
459
+ (seed) => lineageNodeId({
460
+ parentIds: [],
461
+ track: seed.track,
462
+ surface: seed.surface,
463
+ proposer: seed.proposer
464
+ })
465
+ ).filter((id) => !lineage.has(id))
466
+ );
467
+ const available = Math.max(0, opts.budget.maxNodes - lineage.all().length);
468
+ if (missingSeedIds.size > available) {
469
+ throw new Error(
470
+ `runLineage: seed set requires ${missingSeedIds.size} new nodes but budget.maxNodes has ${available} slots remaining`
471
+ );
472
+ }
473
+ }
398
474
  const trackProposer = /* @__PURE__ */ new Map();
399
475
  const persist = async (node) => {
400
476
  await store.append(node);
@@ -414,10 +490,18 @@ async function runLineage(opts) {
414
490
  }
415
491
  let steps = 0;
416
492
  while (steps < opts.budget.maxSteps) {
493
+ if (opts.budget.maxNodes !== void 0 && lineage.all().length >= opts.budget.maxNodes) {
494
+ log("lineage: persisted node limit reached", {
495
+ maxNodes: opts.budget.maxNodes,
496
+ nodes: lineage.all().length
497
+ });
498
+ break;
499
+ }
500
+ const nodeBudgetRemaining = opts.budget.maxNodes === void 0 ? Number.POSITIVE_INFINITY : opts.budget.maxNodes - lineage.all().length;
417
501
  const op = await opts.governor.decide({
418
502
  lineage,
419
503
  step: steps,
420
- budgetRemaining: opts.budget.maxSteps - steps,
504
+ budgetRemaining: Math.min(opts.budget.maxSteps - steps, nodeBudgetRemaining),
421
505
  prunedTracks: [...pruned]
422
506
  });
423
507
  if (op.op === "stop") {
@@ -2940,14 +3024,14 @@ function parameterSweepProposer(opts) {
2940
3024
  const parse = opts.parse ?? parseJsonObject;
2941
3025
  const stringify = opts.stringify ?? ((config) => JSON.stringify(config, null, 2));
2942
3026
  const current = parse(ctx.currentSurface);
2943
- const currentCanonical = canonicalJson(current);
3027
+ const currentCanonical = canonicalJson2(current);
2944
3028
  const tried = triedLabels(ctx.history);
2945
3029
  const out = [];
2946
3030
  for (const candidate of opts.candidates) {
2947
3031
  if (tried.has(candidate.label)) continue;
2948
3032
  const next = applyParameterCandidate(current, candidate);
2949
3033
  const surface = stringify(next);
2950
- if (surface === ctx.currentSurface || canonicalJson(next) === currentCanonical) continue;
3034
+ if (surface === ctx.currentSurface || canonicalJson2(next) === currentCanonical) continue;
2951
3035
  out.push({ surface, label: candidate.label, rationale: candidate.rationale });
2952
3036
  if (out.length >= ctx.populationSize) break;
2953
3037
  }
@@ -3024,10 +3108,10 @@ function assertSafeJsonKey(key) {
3024
3108
  throw new Error(`parameterSweepProposer: unsafe JSON key "${key}" is not allowed`);
3025
3109
  }
3026
3110
  }
3027
- function canonicalJson(value) {
3028
- if (Array.isArray(value)) return `[${value.map((item) => canonicalJson(item)).join(",")}]`;
3111
+ function canonicalJson2(value) {
3112
+ if (Array.isArray(value)) return `[${value.map((item) => canonicalJson2(item)).join(",")}]`;
3029
3113
  if (isPlainObject(value)) {
3030
- return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${canonicalJson(value[key])}`).join(",")}}`;
3114
+ return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${canonicalJson2(value[key])}`).join(",")}}`;
3031
3115
  }
3032
3116
  return JSON.stringify(value);
3033
3117
  }
@@ -4214,9 +4298,13 @@ function toCandidate(p) {
4214
4298
  async function runLineageLoop(opts) {
4215
4299
  const governor = opts.governor ?? heuristicGovernor();
4216
4300
  const populationSize = opts.populationSize ?? 4;
4301
+ const candidateConcurrency = opts.candidateConcurrency ?? 1;
4217
4302
  if (populationSize < 1) {
4218
4303
  throw new Error("runLineageLoop: populationSize must be >= 1");
4219
4304
  }
4305
+ if (!Number.isInteger(candidateConcurrency) || candidateConcurrency < 1) {
4306
+ throw new Error("runLineageLoop: candidateConcurrency must be a positive integer");
4307
+ }
4220
4308
  let proposer = opts.proposer;
4221
4309
  if (!proposer) {
4222
4310
  if (!opts.llm || !opts.model) {
@@ -4282,18 +4370,17 @@ async function runLineageLoop(opts) {
4282
4370
  objectives.composite = node.score;
4283
4371
  return objectives;
4284
4372
  };
4285
- const scoredSeeds = [];
4286
- for (const seed of opts.seeds) {
4373
+ const scoredSeeds = await mapConcurrent(opts.seeds, candidateConcurrency, async (seed) => {
4287
4374
  const measured = await scoreSurface(seed.surface);
4288
- scoredSeeds.push({
4375
+ return {
4289
4376
  surface: seed.surface,
4290
4377
  track: seed.track,
4291
4378
  proposer: seed.proposer,
4292
4379
  score: measured.score,
4293
4380
  ...seed.vision !== void 0 ? { vision: seed.vision } : {},
4294
4381
  ...measured.scoreVector !== void 0 ? { scoreVector: measured.scoreVector } : {}
4295
- });
4296
- }
4382
+ };
4383
+ });
4297
4384
  const step = async (args) => {
4298
4385
  const proposed = await proposer.propose({
4299
4386
  currentSurface: args.tip.surface,
@@ -4312,16 +4399,21 @@ async function runLineageLoop(opts) {
4312
4399
  ...args.tip.rationale !== void 0 ? { rationale: args.tip.rationale } : {}
4313
4400
  }
4314
4401
  ];
4315
- for (const p of proposed) {
4316
- const { surface, rationale } = toCandidate(p);
4317
- const measured = await scoreSurface(surface);
4318
- pool.push({
4319
- surface,
4320
- score: measured.score,
4321
- ...measured.scoreVector !== void 0 ? { scoreVector: measured.scoreVector } : {},
4322
- ...rationale !== void 0 ? { rationale } : {}
4323
- });
4324
- }
4402
+ const measuredCandidates = await mapConcurrent(
4403
+ proposed,
4404
+ candidateConcurrency,
4405
+ async (p) => {
4406
+ const { surface, rationale } = toCandidate(p);
4407
+ const measured = await scoreSurface(surface);
4408
+ return {
4409
+ surface,
4410
+ score: measured.score,
4411
+ ...measured.scoreVector !== void 0 ? { scoreVector: measured.scoreVector } : {},
4412
+ ...rationale !== void 0 ? { rationale } : {}
4413
+ };
4414
+ }
4415
+ );
4416
+ pool.push(...measuredCandidates);
4325
4417
  let best = pool[0];
4326
4418
  for (const entry of pool) {
4327
4419
  if (entry.score > best.score) best = entry;
@@ -8050,6 +8142,8 @@ export {
8050
8142
  agentProfileHash,
8051
8143
  lineageNodeId,
8052
8144
  Lineage,
8145
+ LineageStoreConflictError,
8146
+ campaignLineageStore,
8053
8147
  fsLineageStore,
8054
8148
  memLineageStore,
8055
8149
  heuristicGovernor,
@@ -8116,4 +8210,4 @@ export {
8116
8210
  verifyCodeSurface,
8117
8211
  resolveWorktreePath
8118
8212
  };
8119
- //# sourceMappingURL=chunk-EXMKNOP5.js.map
8213
+ //# sourceMappingURL=chunk-75OCWDXJ.js.map