@tangle-network/agent-knowledge 7.0.0 → 7.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,10 +1,10 @@
1
1
  import { r as stableId, t as sha256 } from "./ids-Bevz_pXV.js";
2
- import { B as memoryWriteResultToSourceRecord, C as appendDurableJournalEvent, D as reconcileInterruptedMemoryPaidCalls, E as readActiveAttemptJournal, O as reserveRecoveryAttempts, S as appendAttemptJournalEvent, T as hasSettledPaidCall, _ as memoryRecoveryDelayMs, b as runBoundedMemoryLifecycle, f as acquireAgentMemoryRunLease, g as createMemoryExecutionPool, u as scoreMemoryBenchmarkArtifact, v as releaseMemoryAdapterCreatedAfterAbort, w as assertNoInterruptedPaidCalls, x as sleepForMemoryRecovery, y as resolveMemoryCleanupTimeoutMs, z as memoryHitToSourceRecord } from "./benchmarks-C8L7HJb4.js";
2
+ import { A as reserveRecoveryAttempts, C as sleepForMemoryRecovery, D as hasSettledPaidCall, E as assertNoInterruptedPaidCalls, H as memoryWriteResultToSourceRecord, O as readActiveAttemptJournal, S as runBoundedMemoryLifecycle, T as appendDurableJournalEvent, V as memoryHitToSourceRecord, _ as MEMORY_OPERATION_CANCELLATION_TIMEOUT_MS, b as memoryRecoveryDelayMs, f as acquireAgentMemoryRunLease, g as MEMORY_CAMPAIGN_DISPATCH_SHUTDOWN_TIMEOUT_MS, k as reconcileInterruptedMemoryPaidCalls, u as scoreMemoryBenchmarkArtifact, v as createBoundedMemoryAdapter, w as appendAttemptJournalEvent, x as resolveMemoryCleanupTimeoutMs, y as createMemoryExecutionPool } from "./benchmarks-CPl4AO1k.js";
3
3
  import { randomUUID } from "node:crypto";
4
4
  import { dirname, join } from "node:path";
5
- import { canonicalJson, mulberry32 } from "@tangle-network/agent-eval";
5
+ import { canonicalJson, comparePairedArms, mulberry32, pairArms } from "@tangle-network/agent-eval";
6
6
  import { z } from "zod";
7
- import { compareOptimizationMethods, createRunCostLedger, fsCampaignStorage, heldoutSignificance, resolveRunDir, runCampaign, surfaceHash } from "@tangle-network/agent-eval/campaign";
7
+ import { assertCampaignSplitIdentity, canonicalDigest, compareOptimizationMethods, createRunCostLedger, fsCampaignStorage, heldoutSignificance, resolveRunDir, runCampaign, surfaceHash } from "@tangle-network/agent-eval/campaign";
8
8
  //#region src/immutable-ref.ts
9
9
  const SHA256_REF = /^sha256:[a-f0-9]{64}$/;
10
10
  const GIT_REF = /^git:[a-f0-9]{40}$/;
@@ -1035,7 +1035,8 @@ function assertMemorySequences(sequences) {
1035
1035
  if (sequence.steps.length === 0) throw new Error(`memory experiment sequence ${sequence.id} has no steps`);
1036
1036
  assertUnique(sequence.steps.map((step) => step.id), `step in sequence ${sequence.id}`);
1037
1037
  let probeCount = 0;
1038
- for (const step of sequence.steps) {
1038
+ const retentionMeasurements = /* @__PURE__ */ new Map();
1039
+ for (const [stepOrdinal, step] of sequence.steps.entries()) {
1039
1040
  for (const write of step.writes ?? []) {
1040
1041
  assertNonEmptyString(write.text, `memory experiment write in ${sequence.id}/${step.id}`);
1041
1042
  if (write.id !== void 0) assertNonEmptyString(write.id, `memory experiment write id in ${sequence.id}/${step.id}`);
@@ -1050,12 +1051,69 @@ function assertMemorySequences(sequences) {
1050
1051
  assertStringList(probe.expectedEventIds, `${sequence.id}/${step.id}/${probe.id} event ids`);
1051
1052
  assertStringList(probe.expectedActorIds, `${sequence.id}/${step.id}/${probe.id} actor ids`);
1052
1053
  if (probe.referenceAnswer !== void 0) assertNonEmptyString(probe.referenceAnswer, `memory experiment reference answer ${sequence.id}/${step.id}/${probe.id}`);
1054
+ if (probe.retentionKey !== void 0) {
1055
+ assertNonEmptyString(probe.retentionKey, `memory experiment retention key ${sequence.id}/${step.id}/${probe.id}`);
1056
+ if (probe.retentionKey !== probe.retentionKey.trim()) throw new Error(`memory experiment retention key ${sequence.id}/${step.id}/${probe.id} must not have surrounding whitespace`);
1057
+ const definition = retentionMeasurementDefinition(step.scope, probe);
1058
+ const prior = retentionMeasurements.get(probe.retentionKey);
1059
+ if (prior?.stepOrdinals.has(stepOrdinal)) throw new Error(`memory experiment retention key ${sequence.id}/${probe.retentionKey} may appear only once per step`);
1060
+ if (prior && prior.definition !== definition) throw new Error(`memory experiment retention key ${sequence.id}/${probe.retentionKey} must repeat the exact same measurement`);
1061
+ const measurement = prior ?? {
1062
+ definition,
1063
+ stepOrdinals: /* @__PURE__ */ new Set()
1064
+ };
1065
+ measurement.stepOrdinals.add(stepOrdinal);
1066
+ retentionMeasurements.set(probe.retentionKey, measurement);
1067
+ }
1068
+ if (probe.transferKey !== void 0) {
1069
+ assertNonEmptyString(probe.transferKey, `memory experiment transfer key ${sequence.id}/${step.id}/${probe.id}`);
1070
+ if (probe.transferKey !== probe.transferKey.trim()) throw new Error(`memory experiment transfer key ${sequence.id}/${step.id}/${probe.id} must not have surrounding whitespace`);
1071
+ if (probe.retentionKey !== void 0) throw new Error(`memory experiment probe ${sequence.id}/${step.id}/${probe.id} cannot be both transfer and retention`);
1072
+ if (stepOrdinal === 0) throw new Error(`memory experiment transfer probe ${sequence.id}/${step.id}/${probe.id} must run after the first step`);
1073
+ }
1053
1074
  if (!((probe.requiredFacts?.length ?? 0) > 0 || (probe.forbiddenFacts?.length ?? 0) > 0 || (probe.expectedEventIds?.length ?? 0) > 0 || (probe.expectedActorIds?.length ?? 0) > 0 || Boolean(probe.referenceAnswer?.trim()))) throw new Error(`memory experiment probe ${sequence.id}/${step.id}/${probe.id} has no measurable target`);
1054
1075
  }
1055
1076
  }
1056
1077
  if (probeCount === 0) throw new Error(`memory experiment sequence ${sequence.id} has no probes`);
1078
+ for (const [retentionKey, measurement] of retentionMeasurements) if (measurement.stepOrdinals.size < 2) throw new Error(`memory experiment retention key ${sequence.id}/${retentionKey} must appear in at least two distinct steps`);
1079
+ }
1080
+ }
1081
+ function assertMemoryLearningSequences(sequences) {
1082
+ for (const sequence of sequences) {
1083
+ if (sequence.steps.length < 2) throw new Error(`memory learning sequence ${sequence.id} requires at least two ordered steps`);
1084
+ if (!sequence.steps.slice(1).some((step) => (step.probes?.length ?? 0) > 0)) throw new Error(`memory learning sequence ${sequence.id} requires a probe after the first step`);
1057
1085
  }
1058
1086
  }
1087
+ function retentionMeasurementDefinition(stepScope, probe) {
1088
+ return canonicalJson({
1089
+ query: probe.query,
1090
+ scope: compactScope({
1091
+ ...stepScope ?? {},
1092
+ ...probe.scope ?? {},
1093
+ tags: {
1094
+ ...stepScope?.tags ?? {},
1095
+ ...probe.scope?.tags ?? {}
1096
+ }
1097
+ }),
1098
+ limit: probe.limit ?? null,
1099
+ taskKind: probe.taskKind ?? "memory-recall",
1100
+ requiredFacts: normalizeFactMatchers(probe.requiredFacts ?? []),
1101
+ forbiddenFacts: normalizeFactMatchers(probe.forbiddenFacts ?? []),
1102
+ expectedEventIds: [...probe.expectedEventIds ?? []],
1103
+ expectedActorIds: [...probe.expectedActorIds ?? []],
1104
+ referenceAnswer: probe.referenceAnswer ?? null
1105
+ });
1106
+ }
1107
+ function normalizeFactMatchers(matchers) {
1108
+ return matchers.map((matcher) => ({
1109
+ id: matcher.id,
1110
+ anyOf: [...matcher.anyOf],
1111
+ weight: matcher.weight ?? 1,
1112
+ sourceEventIds: [...matcher.sourceEventIds ?? []],
1113
+ validAt: matcher.validAt ?? null,
1114
+ obsolete: matcher.obsolete ?? false
1115
+ }));
1116
+ }
1059
1117
  function normalizeCleanupScope(scope) {
1060
1118
  const normalized = compactScope(scope);
1061
1119
  if (normalized.tags && Object.keys(normalized.tags).length === 0) delete normalized.tags;
@@ -1119,7 +1177,7 @@ function buildAgentMemorySequencesFromBenchmarkCases(cases, options = {}) {
1119
1177
  eventIndex
1120
1178
  }) ?? {
1121
1179
  agentId: memoryAgentId,
1122
- sessionId: testCase.id
1180
+ sessionId: "benchmark-session"
1123
1181
  }),
1124
1182
  writes: [{
1125
1183
  id: event.id,
@@ -1138,7 +1196,7 @@ function buildAgentMemorySequencesFromBenchmarkCases(cases, options = {}) {
1138
1196
  id: "probe",
1139
1197
  scope: compactScope(options.probeScope?.(testCase) ?? {
1140
1198
  agentId: memoryAgentId,
1141
- sessionId: testCase.id
1199
+ sessionId: "benchmark-session"
1142
1200
  }),
1143
1201
  probes: [{
1144
1202
  id: "answer",
@@ -1212,6 +1270,759 @@ function agentMemorySequenceJudge() {
1212
1270
  };
1213
1271
  }
1214
1272
  //#endregion
1273
+ //#region src/memory/experiment/metrics.ts
1274
+ function rankAgentMemoryExperiment(candidates, scenarios, campaign, costByCandidate) {
1275
+ const scenarioById = new Map(scenarios.map((scenario) => [scenario.id, scenario]));
1276
+ return candidates.map((candidate) => {
1277
+ const candidateScenarios = scenarios.filter((scenario) => scenario.candidateId === candidate.id);
1278
+ const cells = campaign.cells.filter((cell) => scenarioById.get(cell.scenarioId)?.candidateId === candidate.id);
1279
+ const successful = cells.filter((cell) => !cell.error && cell.artifact);
1280
+ const dimensionRows = successful.map((cell) => cell.artifact.dimensions);
1281
+ return {
1282
+ rank: 0,
1283
+ candidateId: candidate.id,
1284
+ label: candidate.label ?? candidate.id,
1285
+ scoreMean: mean$1(cells.map((cell) => !cell.error && cell.artifact ? cell.artifact.score : 0)),
1286
+ passRate: mean$1(cells.map((cell) => !cell.error && cell.artifact?.passed ? 1 : 0)),
1287
+ totalSequences: candidateScenarios.length,
1288
+ totalCells: cells.length,
1289
+ totalProbes: successful.reduce((sum, cell) => sum + cell.artifact.probes.length, 0),
1290
+ cellsFailed: cells.filter((cell) => Boolean(cell.error)).length,
1291
+ totalCostUsd: normalizeUsd(costByCandidate.get(candidate.id) ?? 0),
1292
+ durationMs: cells.reduce((sum, cell) => sum + cell.durationMs, 0),
1293
+ dimensions: meanDimensions(dimensionRows)
1294
+ };
1295
+ }).sort((a, b) => Number(a.cellsFailed > 0) - Number(b.cellsFailed > 0) || b.scoreMean - a.scoreMean || b.passRate - a.passRate || a.totalCostUsd - b.totalCostUsd || a.candidateId.localeCompare(b.candidateId)).map((row, index) => ({
1296
+ ...row,
1297
+ rank: index + 1
1298
+ }));
1299
+ }
1300
+ function memoryExperimentCostByCandidate(costLedger, runDir, scenarios, candidateIdsInput) {
1301
+ const candidateByScenario = new Map(scenarios.map((scenario) => [scenario.id, scenario.candidateId]));
1302
+ const candidateIds = new Set(candidateIdsInput);
1303
+ const sequenceIds = new Set(scenarios.map((scenario) => scenario.sequenceId));
1304
+ for (const candidateId of candidateIds) for (const sequenceId of sequenceIds) candidateByScenario.set(`${stableId("candidate", candidateId)}:${sequenceId}`, candidateId);
1305
+ const totals = /* @__PURE__ */ new Map();
1306
+ for (const receipt of costLedger.list()) {
1307
+ if (receipt.tags?.runDir !== runDir) continue;
1308
+ const scenarioCandidate = receipt.tags.scenarioId ? candidateByScenario.get(receipt.tags.scenarioId) : void 0;
1309
+ const recoveryCandidate = receipt.tags.memoryRecovery === "attempt" && receipt.tags.candidateId ? receipt.tags.candidateId : void 0;
1310
+ const candidateId = scenarioCandidate ?? recoveryCandidate;
1311
+ if (!candidateId || !candidateIds.has(candidateId)) continue;
1312
+ totals.set(candidateId, (totals.get(candidateId) ?? 0) + receipt.costUsd);
1313
+ }
1314
+ return totals;
1315
+ }
1316
+ function renderAgentMemoryExperimentRanking(rows, totalCostUsd, unrankedRecoveryCostUsd) {
1317
+ return [
1318
+ "# Agent Memory Experiment",
1319
+ "",
1320
+ `- total cost: $${format(totalCostUsd)}`,
1321
+ `- retired-candidate cost: $${format(unrankedRecoveryCostUsd)}`,
1322
+ "",
1323
+ "| rank | candidate | sequences | cells | probes | failed | score | pass rate | cost | duration ms |",
1324
+ "| ---: | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |",
1325
+ ...rows.map((row) => `| ${row.rank} | ${row.label} | ${row.totalSequences} | ${row.totalCells} | ${row.totalProbes} | ${row.cellsFailed} | ${format(row.scoreMean)} | ${format(row.passRate)} | $${format(row.totalCostUsd)} | ${format(row.durationMs)} |`),
1326
+ ""
1327
+ ].join("\n");
1328
+ }
1329
+ function meanDimensions(rows) {
1330
+ const values = /* @__PURE__ */ new Map();
1331
+ for (const row of rows) for (const [key, value] of Object.entries(row)) {
1332
+ if (!Number.isFinite(value)) continue;
1333
+ const bucket = values.get(key) ?? [];
1334
+ bucket.push(value);
1335
+ values.set(key, bucket);
1336
+ }
1337
+ return Object.fromEntries([...values].map(([key, bucket]) => [key, mean$1(bucket)]));
1338
+ }
1339
+ function countDimensions(rows) {
1340
+ const counts = /* @__PURE__ */ new Map();
1341
+ for (const row of rows) for (const key of new Set(row)) counts.set(key, (counts.get(key) ?? 0) + 1);
1342
+ return Object.fromEntries(counts);
1343
+ }
1344
+ function mean$1(values) {
1345
+ return values.length === 0 ? 0 : values.reduce((sum, value) => sum + value, 0) / values.length;
1346
+ }
1347
+ function format(value) {
1348
+ return Number.isFinite(value) ? value.toFixed(4) : "0.0000";
1349
+ }
1350
+ function normalizeUsd(value) {
1351
+ return Number(value.toFixed(12));
1352
+ }
1353
+ //#endregion
1354
+ //#region src/memory/experiment/probe-evaluation.ts
1355
+ function scoreAgentMemoryProbe(sequence, step, probe, evidence) {
1356
+ return scoreMemoryBenchmarkArtifact(memoryProbeBenchmarkCase(sequence, step, probe), evidence);
1357
+ }
1358
+ function memoryProbeBenchmarkCase(sequence, step, probe) {
1359
+ return {
1360
+ id: `${sequence.id}:${step.id}:${probe.id}`,
1361
+ family: sequence.family,
1362
+ taskKind: probe.taskKind ?? "memory-recall",
1363
+ split: sequence.split,
1364
+ events: [],
1365
+ prompt: probe.query,
1366
+ requiredFacts: probe.requiredFacts && probe.requiredFacts.length > 0 ? probe.requiredFacts : probe.referenceAnswer ? [{
1367
+ id: `${probe.id}:reference`,
1368
+ anyOf: [probe.referenceAnswer]
1369
+ }] : void 0,
1370
+ forbiddenFacts: probe.forbiddenFacts,
1371
+ expectedEventIds: probe.expectedEventIds,
1372
+ expectedActorIds: probe.expectedActorIds,
1373
+ referenceAnswer: probe.referenceAnswer
1374
+ };
1375
+ }
1376
+ //#endregion
1377
+ //#region src/memory/experiment/learning-evidence.ts
1378
+ function assertStoredAgentMemoryEvidence(storage, result, sequences) {
1379
+ const sequenceById = new Map(sequences.map((sequence) => [sequence.id, sequence]));
1380
+ for (const cell of result.campaign.cells) {
1381
+ if (cell.error || !cell.artifact) throw new Error(`memory learning cell '${cell.cellId}' has no valid artifact${cell.error ? `: ${cell.error}` : ""}`);
1382
+ const sequence = sequenceById.get(cell.artifact.sequenceId);
1383
+ if (!sequence) throw new Error(`memory learning cell '${cell.cellId}' references unknown sequence '${cell.artifact.sequenceId}'`);
1384
+ assertStoredCellEvidence(storage, cell.cellId, cell.artifact, sequence);
1385
+ }
1386
+ }
1387
+ function assertStoredCellEvidence(storage, cellId, artifact, sequence) {
1388
+ const definitions = indexProbeDefinitions(sequence);
1389
+ if (artifact.probes.length !== definitions.size) throw new Error(`memory learning cell '${cellId}' has ${artifact.probes.length} probes; expected ${definitions.size}`);
1390
+ const seen = /* @__PURE__ */ new Set();
1391
+ for (const probe of artifact.probes) {
1392
+ const key = probeKey(probe.stepOrdinal, probe.stepId, probe.id);
1393
+ const definition = definitions.get(key);
1394
+ if (!definition || seen.has(key)) throw new Error(`memory learning cell '${cellId}' has unexpected probe '${key}'`);
1395
+ seen.add(key);
1396
+ assertStoredProbeEvidence(storage, cellId, probe, definition, sequence);
1397
+ }
1398
+ const expectedScore = mean$1(artifact.probes.map((probe) => probe.score));
1399
+ const expectedPassed = artifact.probes.length > 0 && artifact.probes.every((probe) => probe.passed);
1400
+ const expectedDimensions = meanDimensions(artifact.probes.map((probe) => probe.dimensions));
1401
+ const expectedCounts = countDimensions(artifact.probes.map((probe) => probe.applicableDimensions));
1402
+ if (artifact.score !== expectedScore || artifact.passed !== expectedPassed || canonicalJson(artifact.dimensions) !== canonicalJson(expectedDimensions) || canonicalJson(artifact.dimensionSampleCounts) !== canonicalJson(expectedCounts)) throw new Error(`memory learning cell '${cellId}' aggregate does not match its probes`);
1403
+ }
1404
+ function assertStoredProbeEvidence(storage, cellId, observed, definition, sequence) {
1405
+ const { probe, step } = definition;
1406
+ if (observed.query !== probe.query || observed.retentionKey !== probe.retentionKey || observed.transferKey !== probe.transferKey) throw new Error(`memory learning cell '${cellId}' probe '${observed.id}' changed definition`);
1407
+ const raw = storage.read(observed.evidencePath);
1408
+ if (raw === void 0) throw new Error(`memory learning probe evidence is missing at '${observed.evidencePath}'`);
1409
+ let parsed;
1410
+ try {
1411
+ parsed = JSON.parse(raw);
1412
+ } catch (error) {
1413
+ throw new Error(`memory learning probe evidence is malformed at '${observed.evidencePath}'`, { cause: error });
1414
+ }
1415
+ if (canonicalDigest(parsed) !== observed.evidenceRef) throw new Error(`memory learning probe evidence hash does not match at '${observed.evidencePath}'`);
1416
+ const evidence = parseProbeEvidence(parsed, observed.evidencePath);
1417
+ const expected = scoreAgentMemoryProbe(sequence, step, probe, evidence);
1418
+ const observedEvaluation = {
1419
+ score: observed.score,
1420
+ passed: observed.passed,
1421
+ dimensions: observed.dimensions,
1422
+ applicableDimensions: observed.applicableDimensions,
1423
+ notes: observed.notes
1424
+ };
1425
+ const expectedEvaluation = {
1426
+ score: expected.score,
1427
+ passed: expected.passed,
1428
+ dimensions: expected.dimensions,
1429
+ applicableDimensions: expected.applicableDimensions ?? Object.keys(expected.dimensions),
1430
+ notes: expected.notes
1431
+ };
1432
+ if (canonicalJson(observedEvaluation) !== canonicalJson(expectedEvaluation)) throw new Error(`memory learning probe score does not match saved evidence at '${observed.evidencePath}'`);
1433
+ if (canonicalJson(observed.hitIds) !== canonicalJson(evidence.usedMemoryIds)) throw new Error(`memory learning probe hits do not match saved evidence at '${observed.evidencePath}'`);
1434
+ }
1435
+ function indexProbeDefinitions(sequence) {
1436
+ const definitions = /* @__PURE__ */ new Map();
1437
+ for (const [stepOrdinal, step] of sequence.steps.entries()) for (const probe of step.probes ?? []) definitions.set(probeKey(stepOrdinal, step.id, probe.id), {
1438
+ step,
1439
+ stepOrdinal,
1440
+ probe
1441
+ });
1442
+ return definitions;
1443
+ }
1444
+ function probeKey(stepOrdinal, stepId, probeId) {
1445
+ return JSON.stringify([
1446
+ stepOrdinal,
1447
+ stepId,
1448
+ probeId
1449
+ ]);
1450
+ }
1451
+ function parseProbeEvidence(value, path) {
1452
+ if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error(`memory learning probe evidence has invalid shape at '${path}'`);
1453
+ const record = value;
1454
+ if (typeof record.answer !== "string" || !isStringArray(record.rememberedFacts) || !isStringArray(record.citedEventIds) || !isStringArray(record.usedMemoryIds) || !isStringArray(record.actorIds)) throw new Error(`memory learning probe evidence has invalid shape at '${path}'`);
1455
+ return {
1456
+ answer: record.answer,
1457
+ rememberedFacts: record.rememberedFacts,
1458
+ citedEventIds: record.citedEventIds,
1459
+ usedMemoryIds: record.usedMemoryIds,
1460
+ actorIds: record.actorIds
1461
+ };
1462
+ }
1463
+ function isStringArray(value) {
1464
+ return Array.isArray(value) && value.every((item) => typeof item === "string");
1465
+ }
1466
+ //#endregion
1467
+ //#region src/memory/experiment/learning-metrics.ts
1468
+ function measureAgentMemoryLearning(paired, stateful, stateless) {
1469
+ const cells = paired.map(toCellComparison);
1470
+ const transferCells = paired.flatMap(toTransferCells);
1471
+ const forgettingProbes = paired.flatMap(toForgettingComparisons);
1472
+ const gainUnits = collapseLearningCells(cells);
1473
+ const forgetting = summarizeForgetting(forgettingProbes, stateful.campaign.seed);
1474
+ const preTreatment = measurePreTreatment(paired, stateful.campaign.seed);
1475
+ if (!paired[0]) throw new Error("cannot compare memory learning: no paired cells");
1476
+ return {
1477
+ comparisonRef: stateful.comparisonRef,
1478
+ evidence: {
1479
+ splitRef: evidenceRef(stateful.campaign.splitDigest, "task split"),
1480
+ statefulManifestRef: evidenceRef(stateful.campaign.manifestHash, "stateful campaign manifest"),
1481
+ statelessManifestRef: evidenceRef(stateless.campaign.manifestHash, "stateless campaign manifest"),
1482
+ statefulRunDir: stateful.campaign.runDir,
1483
+ statelessRunDir: stateless.campaign.runDir,
1484
+ candidateRefs: stateful.candidateRefs.map((candidate) => ({ ...candidate })),
1485
+ executionRef: stateful.executionRef
1486
+ },
1487
+ cells,
1488
+ preTreatment,
1489
+ gain: comparePairedRewards(gainUnits, stateful.campaign.seed, "overall gain"),
1490
+ gainByCandidate: summarizeCandidateGain(cells, stateful.campaign.seed),
1491
+ transfer: {
1492
+ definition: "explicit-transfer-probes",
1493
+ cells: transferCells,
1494
+ byStep: summarizeTransfer(transferCells, stateful.campaign.seed)
1495
+ },
1496
+ forgetting: {
1497
+ definition: "prior-peak-minus-final",
1498
+ probes: forgettingProbes,
1499
+ ...forgetting
1500
+ }
1501
+ };
1502
+ }
1503
+ function measurePreTreatment(paired, seed) {
1504
+ const cells = paired.flatMap((pair) => {
1505
+ const probes = pair.probes.filter((probe) => probe.stateful.stepOrdinal === 0);
1506
+ if (probes.length === 0) return [];
1507
+ return [{
1508
+ sequenceId: pair.stateful.artifact.sequenceId,
1509
+ statefulReward: mean$1(probes.map((probe) => probe.stateful.score)),
1510
+ statelessReward: mean$1(probes.map((probe) => probe.stateless.score)),
1511
+ exact: probes.every((probe) => probe.stateful.score === probe.stateless.score)
1512
+ }];
1513
+ });
1514
+ const bySequence = /* @__PURE__ */ new Map();
1515
+ for (const cell of cells) {
1516
+ const unit = bySequence.get(cell.sequenceId) ?? {
1517
+ statefulRewards: [],
1518
+ statelessRewards: [],
1519
+ exact: true
1520
+ };
1521
+ unit.statefulRewards.push(cell.statefulReward);
1522
+ unit.statelessRewards.push(cell.statelessReward);
1523
+ unit.exact &&= cell.exact;
1524
+ bySequence.set(cell.sequenceId, unit);
1525
+ }
1526
+ const units = [...bySequence.entries()].map(([sequenceId, unit]) => ({
1527
+ pairKey: sequenceId,
1528
+ statefulReward: mean$1(unit.statefulRewards),
1529
+ statelessReward: mean$1(unit.statelessRewards),
1530
+ exact: unit.exact
1531
+ }));
1532
+ return {
1533
+ definition: "first-step-probes",
1534
+ cells: cells.length,
1535
+ n: units.length,
1536
+ exactMatchRate: units.length === 0 ? null : mean$1(units.map((unit) => unit.exact ? 1 : 0)),
1537
+ difference: units.length === 0 ? null : comparePairedRewards(units, seed, "pre-treatment difference")
1538
+ };
1539
+ }
1540
+ function toCellComparison(pair) {
1541
+ const probes = pair.probes.filter((probe) => probe.stateful.stepOrdinal > 0);
1542
+ if (probes.length === 0) throw new Error(`cannot compare memory learning: cell ${pair.stateful.cell.cellId} has no probe after the first step`);
1543
+ const statefulReward = mean$1(probes.map((probe) => probe.stateful.score));
1544
+ const statelessReward = mean$1(probes.map((probe) => probe.stateless.score));
1545
+ return {
1546
+ cellId: pair.stateful.cell.cellId,
1547
+ candidateId: pair.stateful.artifact.candidateId,
1548
+ sequenceId: pair.stateful.artifact.sequenceId,
1549
+ rep: pair.stateful.cell.rep,
1550
+ seed: pair.stateful.cell.seed,
1551
+ statefulArtifactRef: pair.stateful.artifactRef,
1552
+ statelessArtifactRef: pair.stateless.artifactRef,
1553
+ probeCount: probes.length,
1554
+ statefulReward,
1555
+ statelessReward,
1556
+ gain: difference(statefulReward, statelessReward)
1557
+ };
1558
+ }
1559
+ function toTransferCells(pair) {
1560
+ const groups = /* @__PURE__ */ new Map();
1561
+ for (const probe of pair.probes) {
1562
+ const transferKey = probe.stateful.transferKey;
1563
+ if (transferKey === void 0) continue;
1564
+ const key = JSON.stringify([probe.stateful.stepOrdinal, transferKey]);
1565
+ const group = groups.get(key) ?? {
1566
+ stepOrdinal: probe.stateful.stepOrdinal,
1567
+ transferKey,
1568
+ probes: []
1569
+ };
1570
+ group.probes.push(probe);
1571
+ groups.set(key, group);
1572
+ }
1573
+ return [...groups.values()].map(({ stepOrdinal, transferKey, probes }) => {
1574
+ const stepIds = sortedUnique$1(probes.map((probe) => probe.stateful.stepId));
1575
+ if (stepIds.length !== 1) throw new Error(`cannot compare memory learning: step ordinal ${stepOrdinal} maps to multiple step ids in ${pair.stateful.cell.cellId}`);
1576
+ const statefulReward = mean$1(probes.map((probe) => probe.stateful.score));
1577
+ const statelessReward = mean$1(probes.map((probe) => probe.stateless.score));
1578
+ return {
1579
+ cellId: pair.stateful.cell.cellId,
1580
+ candidateId: pair.stateful.artifact.candidateId,
1581
+ sequenceId: pair.stateful.artifact.sequenceId,
1582
+ rep: pair.stateful.cell.rep,
1583
+ seed: pair.stateful.cell.seed,
1584
+ stepId: stepIds[0],
1585
+ stepOrdinal,
1586
+ transferKey,
1587
+ probeCount: probes.length,
1588
+ statefulArtifactRef: pair.stateful.artifactRef,
1589
+ statelessArtifactRef: pair.stateless.artifactRef,
1590
+ statefulReward,
1591
+ statelessReward,
1592
+ gain: difference(statefulReward, statelessReward)
1593
+ };
1594
+ }).sort((left, right) => left.stepOrdinal - right.stepOrdinal || left.transferKey.localeCompare(right.transferKey));
1595
+ }
1596
+ function toForgettingComparisons(pair) {
1597
+ const byRetentionKey = /* @__PURE__ */ new Map();
1598
+ for (const probe of pair.probes) {
1599
+ const retentionKey = probe.stateful.retentionKey;
1600
+ if (retentionKey === void 0) continue;
1601
+ const probes = byRetentionKey.get(retentionKey) ?? [];
1602
+ probes.push(probe);
1603
+ byRetentionKey.set(retentionKey, probes);
1604
+ }
1605
+ return [...byRetentionKey.entries()].map(([retentionKey, observations]) => {
1606
+ observations.sort((left, right) => left.stateful.stepOrdinal - right.stateful.stepOrdinal);
1607
+ if (observations.length < 2) throw new Error(`cannot compare memory learning: retention key ${retentionKey} in ${pair.stateful.cell.cellId} has fewer than two observations`);
1608
+ const ordinals = observations.map((probe) => probe.stateful.stepOrdinal);
1609
+ if (new Set(ordinals).size !== ordinals.length) throw new Error(`cannot compare memory learning: retention key ${retentionKey} in ${pair.stateful.cell.cellId} repeats within one step`);
1610
+ const final = observations[observations.length - 1];
1611
+ const prior = observations.slice(0, -1);
1612
+ const statefulPeak = maxProbe(prior, (probe) => probe.stateful.score);
1613
+ const statelessPeak = maxProbe(prior, (probe) => probe.stateless.score);
1614
+ const statefulForgetting = difference(statefulPeak.reward, final.stateful.score);
1615
+ const statelessForgetting = difference(statelessPeak.reward, final.stateless.score);
1616
+ return {
1617
+ cellId: pair.stateful.cell.cellId,
1618
+ candidateId: pair.stateful.artifact.candidateId,
1619
+ sequenceId: pair.stateful.artifact.sequenceId,
1620
+ rep: pair.stateful.cell.rep,
1621
+ seed: pair.stateful.cell.seed,
1622
+ retentionKey,
1623
+ observations: observations.length,
1624
+ firstStepOrdinal: observations[0].stateful.stepOrdinal,
1625
+ finalStepOrdinal: final.stateful.stepOrdinal,
1626
+ statefulArtifactRef: pair.stateful.artifactRef,
1627
+ statelessArtifactRef: pair.stateless.artifactRef,
1628
+ statefulPriorPeakReward: statefulPeak.reward,
1629
+ statefulPriorPeakStepOrdinal: statefulPeak.stepOrdinal,
1630
+ statefulFinalReward: final.stateful.score,
1631
+ statefulForgetting,
1632
+ statelessPriorPeakReward: statelessPeak.reward,
1633
+ statelessPriorPeakStepOrdinal: statelessPeak.stepOrdinal,
1634
+ statelessFinalReward: final.stateless.score,
1635
+ statelessForgetting,
1636
+ excessForgetting: difference(statefulForgetting, statelessForgetting)
1637
+ };
1638
+ }).sort((left, right) => left.retentionKey.localeCompare(right.retentionKey) || left.sequenceId.localeCompare(right.sequenceId) || left.rep - right.rep || left.seed - right.seed);
1639
+ }
1640
+ function summarizeCandidateGain(cells, seed) {
1641
+ return sortedUnique$1(cells.map((cell) => cell.candidateId)).map((candidateId) => {
1642
+ const units = collapseLearningCells(cells.filter((cell) => cell.candidateId === candidateId));
1643
+ return {
1644
+ candidateId,
1645
+ cells: cells.filter((cell) => cell.candidateId === candidateId).length,
1646
+ gain: comparePairedRewards(units, seed, `candidate ${candidateId} gain`)
1647
+ };
1648
+ });
1649
+ }
1650
+ function summarizeTransfer(cells, seed) {
1651
+ const groups = /* @__PURE__ */ new Map();
1652
+ for (const cell of cells) {
1653
+ const key = JSON.stringify([
1654
+ cell.candidateId,
1655
+ cell.transferKey,
1656
+ cell.stepOrdinal
1657
+ ]);
1658
+ const group = groups.get(key) ?? {
1659
+ candidateId: cell.candidateId,
1660
+ transferKey: cell.transferKey,
1661
+ stepOrdinal: cell.stepOrdinal,
1662
+ rewardsBySequence: /* @__PURE__ */ new Map()
1663
+ };
1664
+ const rewards = group.rewardsBySequence.get(cell.sequenceId) ?? {
1665
+ stateful: [],
1666
+ stateless: []
1667
+ };
1668
+ rewards.stateful.push(cell.statefulReward);
1669
+ rewards.stateless.push(cell.statelessReward);
1670
+ group.rewardsBySequence.set(cell.sequenceId, rewards);
1671
+ groups.set(key, group);
1672
+ }
1673
+ return [...groups.values()].sort((left, right) => left.candidateId.localeCompare(right.candidateId) || left.transferKey.localeCompare(right.transferKey) || left.stepOrdinal - right.stepOrdinal).map(({ candidateId, transferKey, stepOrdinal, rewardsBySequence }) => {
1674
+ return {
1675
+ candidateId,
1676
+ transferKey,
1677
+ stepOrdinal,
1678
+ gain: comparePairedRewards([...rewardsBySequence.entries()].sort(([left], [right]) => left.localeCompare(right)).map(([sequenceId, rewards]) => ({
1679
+ pairKey: sequenceId,
1680
+ statefulReward: mean$1(rewards.stateful),
1681
+ statelessReward: mean$1(rewards.stateless)
1682
+ })), seed, `transfer ${candidateId}:${transferKey}:${stepOrdinal}`)
1683
+ };
1684
+ });
1685
+ }
1686
+ function collapseLearningCells(cells) {
1687
+ const bySequence = /* @__PURE__ */ new Map();
1688
+ for (const cell of cells) {
1689
+ const unit = bySequence.get(cell.sequenceId) ?? {
1690
+ statefulRewards: [],
1691
+ statelessRewards: []
1692
+ };
1693
+ unit.statefulRewards.push(cell.statefulReward);
1694
+ unit.statelessRewards.push(cell.statelessReward);
1695
+ bySequence.set(cell.sequenceId, unit);
1696
+ }
1697
+ return [...bySequence.entries()].sort(([left], [right]) => left.localeCompare(right)).map(([sequenceId, unit]) => ({
1698
+ pairKey: sequenceId,
1699
+ statefulReward: mean$1(unit.statefulRewards),
1700
+ statelessReward: mean$1(unit.statelessRewards)
1701
+ }));
1702
+ }
1703
+ function summarizeForgetting(probes, seed) {
1704
+ const byTarget = /* @__PURE__ */ new Map();
1705
+ for (const probe of probes) {
1706
+ const key = JSON.stringify([probe.sequenceId, probe.retentionKey]);
1707
+ const unit = byTarget.get(key) ?? {
1708
+ stateful: [],
1709
+ stateless: [],
1710
+ excess: []
1711
+ };
1712
+ unit.stateful.push(probe.statefulForgetting);
1713
+ unit.stateless.push(probe.statelessForgetting);
1714
+ unit.excess.push(probe.excessForgetting);
1715
+ byTarget.set(key, unit);
1716
+ }
1717
+ const units = [...byTarget.entries()].map(([pairKey, unit]) => ({
1718
+ pairKey,
1719
+ statefulReward: mean$1(unit.stateful),
1720
+ statelessReward: mean$1(unit.stateless),
1721
+ excess: mean$1(unit.excess)
1722
+ }));
1723
+ return {
1724
+ n: units.length,
1725
+ meanStatefulForgetting: optionalMean(units.map((unit) => unit.statefulReward)),
1726
+ meanStatelessForgetting: optionalMean(units.map((unit) => unit.statelessReward)),
1727
+ meanExcessForgetting: optionalMean(units.map((unit) => unit.excess)),
1728
+ excess: units.length === 0 ? null : comparePairedRewards(units, seed, "excess forgetting")
1729
+ };
1730
+ }
1731
+ function comparePairedRewards(units, seed, label) {
1732
+ const comparison = comparePairedArms(units.flatMap((unit) => [{
1733
+ pairKey: unit.pairKey,
1734
+ arm: "stateless",
1735
+ metrics: { reward: unit.statelessReward }
1736
+ }, {
1737
+ pairKey: unit.pairKey,
1738
+ arm: "stateful",
1739
+ metrics: { reward: unit.statefulReward }
1740
+ }]), {
1741
+ baselineArm: "stateless",
1742
+ treatmentArm: "stateful",
1743
+ metricNames: ["reward"],
1744
+ bootstrap: {
1745
+ statistic: "mean",
1746
+ seed
1747
+ }
1748
+ });
1749
+ const metric = comparison.metricDeltas[0];
1750
+ if (comparison.nPairs !== units.length || comparison.nUnpairedBaseline !== 0 || comparison.nUnpairedTreatment !== 0 || !metric || metric.n !== units.length || metric.nMissing !== 0 || !metric.bootstrapCi) throw new Error(`cannot compare memory learning: incomplete paired ${label}`);
1751
+ return metric.bootstrapCi;
1752
+ }
1753
+ function maxProbe(probes, reward) {
1754
+ let best = probes[0];
1755
+ for (const probe of probes.slice(1)) if (reward(probe) > reward(best)) best = probe;
1756
+ return {
1757
+ reward: reward(best),
1758
+ stepOrdinal: best.stateful.stepOrdinal
1759
+ };
1760
+ }
1761
+ function optionalMean(values) {
1762
+ return values.length === 0 ? null : mean$1(values);
1763
+ }
1764
+ function difference(left, right) {
1765
+ const value = left - right;
1766
+ return Object.is(value, -0) ? 0 : value;
1767
+ }
1768
+ function sortedUnique$1(values) {
1769
+ return [...new Set(values)].sort();
1770
+ }
1771
+ function evidenceRef(value, label) {
1772
+ if (typeof value === "string" && /^sha256:[a-f0-9]{64}$/.test(value)) return value;
1773
+ if (typeof value !== "string" || !/^[a-f0-9]{64}$/.test(value)) throw new Error(`cannot compare memory learning: ${label} must be a full lowercase sha256 digest; received ${String(value)}`);
1774
+ return `sha256:${value}`;
1775
+ }
1776
+ //#endregion
1777
+ //#region src/memory/experiment/comparison-ref.ts
1778
+ const MEMORY_EXPERIMENT_IMPLEMENTATION_REF = `sha256:${sha256(canonicalJson({
1779
+ stateful: "retain declared sequence scopes across ordered steps",
1780
+ stateless: "clear every declared sequence scope before each step after the first",
1781
+ isolation: "provider scopes receive an opaque branch id but no experiment or sequence labels",
1782
+ recovery: "unfinished attempts bind cleanup to the exact sequence sha256",
1783
+ gain: "post-first-step stateful minus stateless reward",
1784
+ inference: "average candidates and repetitions within each sequence before paired bootstrap",
1785
+ preTreatment: "report first-step score balance separately from gain",
1786
+ transfer: "only probes with explicit transferKey after the first step",
1787
+ forgetting: "signed prior peak minus final score for exact repeated retentionKey probes",
1788
+ evidence: "full sha256 design, manifest, split, artifact, and probe scoring-input references"
1789
+ }))}`;
1790
+ const COMPARISON_REF_PATTERN = /^sha256:[a-f0-9]{64}$/;
1791
+ function resolveAgentMemoryMode(value) {
1792
+ if (value === void 0) return "stateful";
1793
+ if (value !== "stateful" && value !== "stateless") throw new Error("memory experiment memoryMode must be 'stateful' or 'stateless'");
1794
+ return value;
1795
+ }
1796
+ function memoryExperimentComparisonRef(options, resolvedRunDir) {
1797
+ return `sha256:${sha256(canonicalJson({
1798
+ implementationRef: MEMORY_EXPERIMENT_IMPLEMENTATION_REF,
1799
+ experimentId: options.experimentId,
1800
+ experimentRunId: options.experimentRunId ?? resolvedRunDir,
1801
+ sequences: options.sequences,
1802
+ candidates: options.candidates.map(candidateConditions),
1803
+ recoveryCandidates: (options.recoveryCandidates ?? []).map(candidateConditions),
1804
+ executeStepRef: options.executeStepRef ?? "fixtures",
1805
+ cleanupBranches: options.cleanupBranches ?? true,
1806
+ seed: options.seed ?? 42,
1807
+ reps: options.reps ?? 1,
1808
+ resumable: options.resumable ?? true,
1809
+ costCeilingUsd: options.costCeiling ?? options.costLedger?.costCeilingUsd ?? 0,
1810
+ costPhase: options.costPhase ?? null,
1811
+ maxConcurrency: options.maxConcurrency ?? 2,
1812
+ dispatchTimeoutMs: options.dispatchTimeoutMs || null,
1813
+ cleanupTimeoutMs: options.cleanupTimeoutMs ?? 18e4,
1814
+ maxRecoveryAttempts: options.maxRecoveryAttempts ?? 1e3,
1815
+ maxRecoveryRetriesPerAttempt: options.maxRecoveryRetriesPerAttempt ?? 3
1816
+ }))}`;
1817
+ }
1818
+ function assertAgentMemoryExperimentComparisonRef(value, label) {
1819
+ if (typeof value !== "string" || !COMPARISON_REF_PATTERN.test(value)) throw new Error(`${label} must be a full lowercase sha256 reference`);
1820
+ }
1821
+ function candidateConditions(candidate) {
1822
+ return {
1823
+ id: candidate.id,
1824
+ label: candidate.label ?? null,
1825
+ ref: candidate.ref,
1826
+ policy: candidate.policy ?? null,
1827
+ baseScope: candidate.baseScope ?? null,
1828
+ externalCostUsdPerSequence: candidate.externalCostUsdPerSequence ?? 0,
1829
+ externalRecoveryCostUsdPerAttempt: candidate.externalRecoveryCostUsdPerAttempt ?? 0,
1830
+ externalCostAccounting: candidate.externalCostAccounting ?? "exact"
1831
+ };
1832
+ }
1833
+ //#endregion
1834
+ //#region src/memory/experiment/learning-pairs.ts
1835
+ function pairAgentMemoryLearningRuns(options) {
1836
+ assertComparableDesign(options.stateful, options.stateless);
1837
+ const statefulCells = indexCompleteCells("stateful", options.stateful);
1838
+ const statelessCells = indexCompleteCells("stateless", options.stateless);
1839
+ const matched = pairArms([...memoryLearningArmRows("stateful", statefulCells.values()), ...memoryLearningArmRows("stateless", statelessCells.values())], {
1840
+ baselineArm: "stateless",
1841
+ treatmentArm: "stateful"
1842
+ });
1843
+ if (matched.unpairedBaseline.length > 0 || matched.unpairedTreatment.length > 0) throw new Error(`cannot compare memory learning: unmatched experiment cell identities; stateful-only=${unpairedCellIds(matched.unpairedTreatment).join(", ") || "none"}; stateless-only=${unpairedCellIds(matched.unpairedBaseline).join(", ") || "none"}`);
1844
+ return matched.pairs.map((pair) => {
1845
+ const stateful = pair.treatment.indexed;
1846
+ const stateless = pair.baseline.indexed;
1847
+ if (stateful.cell.cellId !== stateless.cell.cellId) throw new Error(`cannot compare memory learning: cell identity differs for ${pair.pairKey} (${stateful.cell.cellId} vs ${stateless.cell.cellId})`);
1848
+ return {
1849
+ stateful,
1850
+ stateless,
1851
+ probes: pairProbes(stateful, stateless)
1852
+ };
1853
+ }).sort(comparePairedCells);
1854
+ }
1855
+ function memoryLearningArmRows(arm, cells) {
1856
+ return [...cells].map((indexed) => ({
1857
+ arm,
1858
+ pairKey: JSON.stringify([indexed.artifact.candidateId, indexed.artifact.sequenceId]),
1859
+ repKey: JSON.stringify([indexed.cell.rep, indexed.cell.seed]),
1860
+ indexed
1861
+ }));
1862
+ }
1863
+ function unpairedCellIds(rows) {
1864
+ return rows.map((row) => row.indexed).map((indexed) => cellIdentity(indexed.artifact, indexed.cell.rep, indexed.cell.seed)).sort();
1865
+ }
1866
+ function assertComparableDesign(stateful, stateless) {
1867
+ if (stateful.memoryMode !== "stateful") throw new Error(`cannot compare memory learning: stateful arm is labeled ${String(stateful.memoryMode)}`);
1868
+ if (stateless.memoryMode !== "stateless") throw new Error(`cannot compare memory learning: stateless arm is labeled ${String(stateless.memoryMode)}`);
1869
+ assertAgentMemoryExperimentComparisonRef(stateful.comparisonRef, "cannot compare memory learning: stateful comparisonRef");
1870
+ assertAgentMemoryExperimentComparisonRef(stateless.comparisonRef, "cannot compare memory learning: stateless comparisonRef");
1871
+ if (stateful.comparisonRef !== stateless.comparisonRef) throw new Error(`cannot compare memory learning: experimental conditions differ (${stateful.comparisonRef} vs ${stateless.comparisonRef})`);
1872
+ if (stateful.campaign.manifestHash === stateless.campaign.manifestHash) throw new Error("cannot compare memory learning: campaign cache identity does not distinguish memoryMode");
1873
+ assertRunSplitIdentity("stateful", stateful);
1874
+ assertRunSplitIdentity("stateless", stateless);
1875
+ if (stateful.campaign.splitDigest !== stateless.campaign.splitDigest) throw new Error("cannot compare memory learning: stateful and stateless task identities differ");
1876
+ if (stateful.campaign.reps !== stateless.campaign.reps) throw new Error(`cannot compare memory learning: repetitions differ (${stateful.campaign.reps} vs ${stateless.campaign.reps})`);
1877
+ if (stateful.campaign.seed !== stateless.campaign.seed) throw new Error(`cannot compare memory learning: campaign seeds differ (${stateful.campaign.seed} vs ${stateless.campaign.seed})`);
1878
+ if (JSON.stringify(stateful.candidateRefs) !== JSON.stringify(stateless.candidateRefs) || stateful.candidateRefs.length === 0) throw new Error("cannot compare memory learning: candidate references differ between arms");
1879
+ const candidateIds = /* @__PURE__ */ new Set();
1880
+ for (const candidate of stateful.candidateRefs) {
1881
+ assertNonEmpty(candidate.id, "candidate reference id");
1882
+ if (candidateIds.has(candidate.id)) throw new Error(`cannot compare memory learning: duplicate candidate reference ${candidate.id}`);
1883
+ candidateIds.add(candidate.id);
1884
+ try {
1885
+ assertImmutableRef(candidate.ref, `${candidate.id}: candidate reference`);
1886
+ } catch (error) {
1887
+ throw new Error(`cannot compare memory learning: ${error instanceof Error ? error.message : String(error)}`);
1888
+ }
1889
+ }
1890
+ if (stateful.executionRef !== stateless.executionRef) throw new Error("cannot compare memory learning: executor references differ between arms");
1891
+ if (stateful.executionRef !== "fixtures") try {
1892
+ assertImmutableRef(stateful.executionRef, "memory learning executor reference");
1893
+ } catch (error) {
1894
+ throw new Error(`cannot compare memory learning: ${error instanceof Error ? error.message : String(error)}`);
1895
+ }
1896
+ }
1897
+ function assertRunSplitIdentity(arm, result) {
1898
+ try {
1899
+ assertCampaignSplitIdentity(result.campaign.scenarios, result.campaign.reps, result.campaign.splitDigest);
1900
+ } catch (error) {
1901
+ throw new Error(`cannot compare memory learning: ${arm} run has invalid task identity: ${error instanceof Error ? error.message : String(error)}`);
1902
+ }
1903
+ }
1904
+ function indexCompleteCells(arm, result) {
1905
+ const expectedCellIds = new Set(result.campaign.scenarios.flatMap((scenario) => Array.from({ length: result.campaign.reps }, (_, rep) => `${scenario.id}:${rep}`)));
1906
+ const byIdentity = /* @__PURE__ */ new Map();
1907
+ const seenCellIds = /* @__PURE__ */ new Set();
1908
+ for (const cell of result.campaign.cells) {
1909
+ if (!expectedCellIds.has(cell.cellId)) throw new Error(`cannot compare memory learning: ${arm} run has unexpected cell ${cell.cellId}`);
1910
+ if (seenCellIds.has(cell.cellId)) throw new Error(`cannot compare memory learning: ${arm} run duplicates cell ${cell.cellId}`);
1911
+ seenCellIds.add(cell.cellId);
1912
+ if (cell.cellId !== `${cell.scenarioId}:${cell.rep}`) throw new Error(`cannot compare memory learning: ${arm} cell ${cell.cellId} does not match its scenario and repetition`);
1913
+ if (cell.manifestHash !== result.campaign.manifestHash) throw new Error(`cannot compare memory learning: ${arm} cell ${cell.cellId} manifest differs from its campaign`);
1914
+ if (cell.error) throw new Error(`cannot compare memory learning: ${arm} cell ${cell.cellId} failed: ${cell.error}`);
1915
+ const artifact = cell.artifact;
1916
+ if (!artifact) throw new Error(`cannot compare memory learning: ${arm} cell ${cell.cellId} has no artifact`);
1917
+ assertArtifactIdentity(arm, result, cell, artifact);
1918
+ if (!Number.isSafeInteger(cell.rep) || cell.rep < 0 || cell.rep >= result.campaign.reps) throw new Error(`cannot compare memory learning: ${arm} cell ${cell.cellId} has invalid rep`);
1919
+ if (!Number.isSafeInteger(cell.seed)) throw new Error(`cannot compare memory learning: ${arm} cell ${cell.cellId} has invalid seed`);
1920
+ assertReward(artifact.score, `${arm} cell ${cell.cellId} reward`);
1921
+ if (artifact.probes.length === 0) throw new Error(`cannot compare memory learning: ${arm} cell ${cell.cellId} has no probes`);
1922
+ for (const probe of artifact.probes) assertProbe(arm, cell.cellId, probe);
1923
+ const probeMean = mean$1(artifact.probes.map((probe) => probe.score));
1924
+ if (Math.abs(probeMean - artifact.score) > 1e-12) throw new Error(`cannot compare memory learning: ${arm} cell ${cell.cellId} reward does not equal its probe mean`);
1925
+ const identity = cellIdentity(artifact, cell.rep, cell.seed);
1926
+ if (byIdentity.has(identity)) throw new Error(`cannot compare memory learning: ${arm} run duplicates identity ${identity}`);
1927
+ byIdentity.set(identity, {
1928
+ cell,
1929
+ artifact,
1930
+ artifactRef: canonicalDigest(artifact)
1931
+ });
1932
+ }
1933
+ const missing = [...expectedCellIds].filter((cellId) => !seenCellIds.has(cellId));
1934
+ if (missing.length > 0) throw new Error(`cannot compare memory learning: ${arm} run is missing cells ${missing.sort().join(", ")}`);
1935
+ return byIdentity;
1936
+ }
1937
+ function assertArtifactIdentity(arm, result, cell, artifact) {
1938
+ assertNonEmpty(artifact.candidateId, `${arm} cell ${cell.cellId} candidateId`);
1939
+ assertNonEmpty(artifact.sequenceId, `${arm} cell ${cell.cellId} sequenceId`);
1940
+ if (!result.candidateRefs.some((candidate) => candidate.id === artifact.candidateId)) throw new Error(`cannot compare memory learning: ${arm} cell ${cell.cellId} uses an undeclared candidate`);
1941
+ if (artifact.memoryMode !== arm) throw new Error(`cannot compare memory learning: ${arm} cell ${cell.cellId} artifact is labeled ${String(artifact.memoryMode)}`);
1942
+ if (artifact.comparisonRef !== result.comparisonRef) throw new Error(`cannot compare memory learning: ${arm} cell ${cell.cellId} comparisonRef differs from its run`);
1943
+ if (!/^sha256:[a-f0-9]{64}$/.test(artifact.branchDigest)) throw new Error(`cannot compare memory learning: ${arm} cell ${cell.cellId} has invalid branch evidence`);
1944
+ const expectedScenarioId = `${stableId("candidate", artifact.candidateId)}:${artifact.sequenceId}`;
1945
+ if (cell.scenarioId !== expectedScenarioId) throw new Error(`cannot compare memory learning: ${arm} cell ${cell.cellId} artifact identity does not match its scenario`);
1946
+ }
1947
+ function pairProbes(stateful, stateless) {
1948
+ const statefulProbes = indexProbes("stateful", stateful);
1949
+ const statelessProbes = indexProbes("stateless", stateless);
1950
+ assertSameKeys(statefulProbes, statelessProbes, `probe in cell ${stateful.cell.cellId}`);
1951
+ return [...statefulProbes.entries()].map(([key, statefulProbe]) => {
1952
+ const statelessProbe = statelessProbes.get(key);
1953
+ if (!statelessProbe) throw new Error(`cannot compare memory learning: missing stateless probe ${key} in ${stateful.cell.cellId}`);
1954
+ if (statefulProbe.query !== statelessProbe.query) throw new Error(`cannot compare memory learning: probe query differs for ${key} in ${stateful.cell.cellId}`);
1955
+ if (statefulProbe.retentionKey !== statelessProbe.retentionKey) throw new Error(`cannot compare memory learning: retention identity differs for ${key} in ${stateful.cell.cellId}`);
1956
+ if (statefulProbe.transferKey !== statelessProbe.transferKey) throw new Error(`cannot compare memory learning: transfer identity differs for ${key} in ${stateful.cell.cellId}`);
1957
+ if (sortedUnique(statefulProbe.applicableDimensions).join("\0") !== sortedUnique(statelessProbe.applicableDimensions).join("\0")) throw new Error(`cannot compare memory learning: measured dimensions differ for ${key} in ${stateful.cell.cellId}`);
1958
+ return {
1959
+ stateful: statefulProbe,
1960
+ stateless: statelessProbe
1961
+ };
1962
+ }).sort((left, right) => left.stateful.stepOrdinal !== right.stateful.stepOrdinal ? left.stateful.stepOrdinal - right.stateful.stepOrdinal : left.stateful.id.localeCompare(right.stateful.id));
1963
+ }
1964
+ function indexProbes(arm, indexed) {
1965
+ const probes = /* @__PURE__ */ new Map();
1966
+ for (const probe of indexed.artifact.probes) {
1967
+ const key = probeIdentity(probe);
1968
+ if (probes.has(key)) throw new Error(`cannot compare memory learning: ${arm} cell ${indexed.cell.cellId} duplicates probe ${key}`);
1969
+ probes.set(key, probe);
1970
+ }
1971
+ return probes;
1972
+ }
1973
+ function assertProbe(arm, cellId, probe) {
1974
+ assertNonEmpty(probe.id, `${arm} cell ${cellId} probe id`);
1975
+ assertNonEmpty(probe.stepId, `${arm} cell ${cellId} probe ${probe.id} stepId`);
1976
+ assertNonEmpty(probe.query, `${arm} cell ${cellId} probe ${probe.id} query`);
1977
+ if (!Number.isSafeInteger(probe.stepOrdinal) || probe.stepOrdinal < 0) throw new Error(`cannot compare memory learning: ${arm} cell ${cellId} probe ${probe.id} has invalid stepOrdinal`);
1978
+ if (probe.retentionKey !== void 0) {
1979
+ assertNonEmpty(probe.retentionKey, `${arm} cell ${cellId} probe ${probe.id} retentionKey`);
1980
+ if (probe.retentionKey !== probe.retentionKey.trim()) throw new Error(`cannot compare memory learning: ${arm} cell ${cellId} probe ${probe.id} retentionKey has surrounding whitespace`);
1981
+ }
1982
+ if (probe.transferKey !== void 0) {
1983
+ assertNonEmpty(probe.transferKey, `${arm} cell ${cellId} probe ${probe.id} transferKey`);
1984
+ if (probe.transferKey !== probe.transferKey.trim()) throw new Error(`cannot compare memory learning: ${arm} cell ${cellId} probe ${probe.id} transferKey has surrounding whitespace`);
1985
+ if (probe.retentionKey !== void 0 || probe.stepOrdinal === 0) throw new Error(`cannot compare memory learning: ${arm} cell ${cellId} probe ${probe.id} has invalid transfer placement`);
1986
+ }
1987
+ if (!/^sha256:[a-f0-9]{64}$/.test(probe.evidenceRef)) throw new Error(`cannot compare memory learning: ${arm} cell ${cellId} probe ${probe.id} has invalid evidence`);
1988
+ const expectedEvidenceSuffix = `/memory-evidence/${probe.evidenceRef.slice(7)}.json`;
1989
+ if (!probe.evidencePath.endsWith(expectedEvidenceSuffix)) throw new Error(`cannot compare memory learning: ${arm} cell ${cellId} probe ${probe.id} evidence is missing`);
1990
+ assertReward(probe.score, `${arm} cell ${cellId} probe ${probe.id} reward`);
1991
+ }
1992
+ function assertReward(value, label) {
1993
+ if (!Number.isFinite(value) || value < 0 || value > 1) throw new Error(`cannot compare memory learning: ${label} must be finite and between 0 and 1`);
1994
+ }
1995
+ function assertNonEmpty(value, label) {
1996
+ if (typeof value !== "string" || value.trim().length === 0) throw new Error(`cannot compare memory learning: ${label} must be a non-empty string`);
1997
+ }
1998
+ function assertSameKeys(left, right, label) {
1999
+ const missingRight = [...left.keys()].filter((key) => !right.has(key));
2000
+ const missingLeft = [...right.keys()].filter((key) => !left.has(key));
2001
+ if (missingRight.length === 0 && missingLeft.length === 0) return;
2002
+ throw new Error(`cannot compare memory learning: unmatched ${label} identities; stateful-only=${missingRight.sort().join(", ") || "none"}; stateless-only=${missingLeft.sort().join(", ") || "none"}`);
2003
+ }
2004
+ function cellIdentity(artifact, rep, seed) {
2005
+ return JSON.stringify([
2006
+ artifact.candidateId,
2007
+ artifact.sequenceId,
2008
+ rep,
2009
+ seed
2010
+ ]);
2011
+ }
2012
+ function probeIdentity(probe) {
2013
+ return JSON.stringify([
2014
+ probe.stepOrdinal,
2015
+ probe.stepId,
2016
+ probe.id
2017
+ ]);
2018
+ }
2019
+ function comparePairedCells(left, right) {
2020
+ return left.stateful.artifact.candidateId.localeCompare(right.stateful.artifact.candidateId) || left.stateful.artifact.sequenceId.localeCompare(right.stateful.artifact.sequenceId) || left.stateful.cell.rep - right.stateful.cell.rep || left.stateful.cell.seed - right.stateful.cell.seed;
2021
+ }
2022
+ function sortedUnique(values) {
2023
+ return [...new Set(values)].sort();
2024
+ }
2025
+ //#endregion
1215
2026
  //#region src/memory/experiment/cost.ts
1216
2027
  function createAgentMemoryCostRecorder(input) {
1217
2028
  let observedCostUsd = 0;
@@ -1265,6 +2076,7 @@ function createAgentMemoryExecutionContext(dispatch, sequence) {
1265
2076
  signal: signal.signal,
1266
2077
  cost
1267
2078
  }),
2079
+ abort: signal.abort,
1268
2080
  dispose: signal.dispose
1269
2081
  };
1270
2082
  }
@@ -1309,6 +2121,7 @@ function relayAbortWithoutReason(source) {
1309
2121
  else source.addEventListener("abort", abort, { once: true });
1310
2122
  return {
1311
2123
  signal: controller.signal,
2124
+ abort,
1312
2125
  dispose() {
1313
2126
  source.removeEventListener("abort", abort);
1314
2127
  }
@@ -1361,85 +2174,6 @@ function redactPrivateValues(value, privateValues) {
1361
2174
  return redacted;
1362
2175
  }
1363
2176
  //#endregion
1364
- //#region src/memory/experiment/metrics.ts
1365
- function rankAgentMemoryExperiment(candidates, scenarios, campaign, costByCandidate) {
1366
- const scenarioById = new Map(scenarios.map((scenario) => [scenario.id, scenario]));
1367
- return candidates.map((candidate) => {
1368
- const candidateScenarios = scenarios.filter((scenario) => scenario.candidateId === candidate.id);
1369
- const cells = campaign.cells.filter((cell) => scenarioById.get(cell.scenarioId)?.candidateId === candidate.id);
1370
- const successful = cells.filter((cell) => !cell.error && cell.artifact);
1371
- const dimensionRows = successful.map((cell) => cell.artifact.dimensions);
1372
- return {
1373
- rank: 0,
1374
- candidateId: candidate.id,
1375
- label: candidate.label ?? candidate.id,
1376
- scoreMean: mean$1(cells.map((cell) => !cell.error && cell.artifact ? cell.artifact.score : 0)),
1377
- passRate: mean$1(cells.map((cell) => !cell.error && cell.artifact?.passed ? 1 : 0)),
1378
- totalSequences: candidateScenarios.length,
1379
- totalCells: cells.length,
1380
- totalProbes: successful.reduce((sum, cell) => sum + cell.artifact.probes.length, 0),
1381
- cellsFailed: cells.filter((cell) => Boolean(cell.error)).length,
1382
- totalCostUsd: normalizeUsd(costByCandidate.get(candidate.id) ?? 0),
1383
- durationMs: cells.reduce((sum, cell) => sum + cell.durationMs, 0),
1384
- dimensions: meanDimensions(dimensionRows)
1385
- };
1386
- }).sort((a, b) => Number(a.cellsFailed > 0) - Number(b.cellsFailed > 0) || b.scoreMean - a.scoreMean || b.passRate - a.passRate || a.totalCostUsd - b.totalCostUsd || a.candidateId.localeCompare(b.candidateId)).map((row, index) => ({
1387
- ...row,
1388
- rank: index + 1
1389
- }));
1390
- }
1391
- function memoryExperimentCostByCandidate(costLedger, runDir, scenarios, candidateIdsInput) {
1392
- const candidateByScenario = new Map(scenarios.map((scenario) => [scenario.id, scenario.candidateId]));
1393
- const candidateIds = new Set(candidateIdsInput);
1394
- const totals = /* @__PURE__ */ new Map();
1395
- for (const receipt of costLedger.list()) {
1396
- if (receipt.tags?.runDir !== runDir) continue;
1397
- const scenarioCandidate = receipt.tags.scenarioId ? candidateByScenario.get(receipt.tags.scenarioId) : void 0;
1398
- const recoveryCandidate = receipt.tags.memoryRecovery === "attempt" && receipt.tags.candidateId ? receipt.tags.candidateId : void 0;
1399
- const candidateId = scenarioCandidate ?? recoveryCandidate;
1400
- if (!candidateId || !candidateIds.has(candidateId)) continue;
1401
- totals.set(candidateId, (totals.get(candidateId) ?? 0) + receipt.costUsd);
1402
- }
1403
- return totals;
1404
- }
1405
- function renderAgentMemoryExperimentRanking(rows, totalCostUsd, unrankedRecoveryCostUsd) {
1406
- return [
1407
- "# Agent Memory Experiment",
1408
- "",
1409
- `- total cost: $${format(totalCostUsd)}`,
1410
- `- retired-candidate recovery cost: $${format(unrankedRecoveryCostUsd)}`,
1411
- "",
1412
- "| rank | candidate | sequences | cells | probes | failed | score | pass rate | cost | duration ms |",
1413
- "| ---: | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |",
1414
- ...rows.map((row) => `| ${row.rank} | ${row.label} | ${row.totalSequences} | ${row.totalCells} | ${row.totalProbes} | ${row.cellsFailed} | ${format(row.scoreMean)} | ${format(row.passRate)} | $${format(row.totalCostUsd)} | ${format(row.durationMs)} |`),
1415
- ""
1416
- ].join("\n");
1417
- }
1418
- function meanDimensions(rows) {
1419
- const values = /* @__PURE__ */ new Map();
1420
- for (const row of rows) for (const [key, value] of Object.entries(row)) {
1421
- if (!Number.isFinite(value)) continue;
1422
- const bucket = values.get(key) ?? [];
1423
- bucket.push(value);
1424
- values.set(key, bucket);
1425
- }
1426
- return Object.fromEntries([...values].map(([key, bucket]) => [key, mean$1(bucket)]));
1427
- }
1428
- function countDimensions(rows) {
1429
- const counts = /* @__PURE__ */ new Map();
1430
- for (const row of rows) for (const key of new Set(row)) counts.set(key, (counts.get(key) ?? 0) + 1);
1431
- return Object.fromEntries(counts);
1432
- }
1433
- function mean$1(values) {
1434
- return values.length === 0 ? 0 : values.reduce((sum, value) => sum + value, 0) / values.length;
1435
- }
1436
- function format(value) {
1437
- return Number.isFinite(value) ? value.toFixed(4) : "0.0000";
1438
- }
1439
- function normalizeUsd(value) {
1440
- return Number(value.toFixed(12));
1441
- }
1442
- //#endregion
1443
2177
  //#region src/memory/experiment/runtime.ts
1444
2178
  var AgentMemoryCleanupError = class extends AggregateError {
1445
2179
  constructor(errors, message) {
@@ -1487,12 +2221,8 @@ function mergeScopes$2(base, extra) {
1487
2221
  }
1488
2222
  };
1489
2223
  }
1490
- function memoryExperimentBaseScope(options, candidate, sequenceId) {
1491
- return mergeScopes$2(candidate.baseScope, { tags: {
1492
- memoryExperimentId: options.experimentId,
1493
- memoryCandidateId: candidate.id,
1494
- memorySequenceId: sequenceId
1495
- } });
2224
+ function memoryExperimentBaseScope(candidate) {
2225
+ return mergeScopes$2(candidate.baseScope);
1496
2226
  }
1497
2227
  function sequenceCleanupScopes(sequence) {
1498
2228
  const scopes = /* @__PURE__ */ new Map();
@@ -1519,40 +2249,53 @@ async function clearSequenceScopes(memory, sequence) {
1519
2249
  //#endregion
1520
2250
  //#region src/memory/experiment/recovery.ts
1521
2251
  async function recoverAbandonedMemoryAttempts(input) {
2252
+ input.options.signal?.throwIfAborted();
1522
2253
  let attempts = readActiveMemoryAttempts(input.storage, input.attemptLogPath);
1523
2254
  if (attempts.length > input.maxRecoveryAttempts) throw new Error(`memory experiment has ${attempts.length} unfinished attempts; maxRecoveryAttempts is ${input.maxRecoveryAttempts}`);
1524
2255
  for (const attempt of attempts) {
1525
2256
  const candidate = input.candidateById.get(attempt.candidateId);
1526
2257
  if (!candidate) throw new Error(`cannot recover memory branch '${attempt.branchId}': candidate '${attempt.candidateId}' is missing; pass it in recoveryCandidates`);
1527
2258
  assertMemoryAttemptCandidateMatches(attempt, candidate);
1528
- if (!input.sequenceById.has(attempt.sequenceId)) throw new Error(`cannot recover memory branch '${attempt.branchId}': sequence '${attempt.sequenceId}' is missing`);
2259
+ const sequence = input.sequenceById.get(attempt.sequenceId);
2260
+ if (!sequence) throw new Error(`cannot recover memory branch '${attempt.branchId}': sequence '${attempt.sequenceId}' is missing`);
2261
+ assertMemoryAttemptSequenceMatches(attempt, sequence);
1529
2262
  if ((input.options.cleanupBranches ?? true) !== attempt.cleanupBranches) throw new Error(`cannot recover memory branch '${attempt.branchId}': cleanupBranches changed`);
1530
2263
  }
1531
2264
  reconcileInterruptedMemoryPaidCalls(input.costLedger);
1532
2265
  assertNoInterruptedPaidCalls(input.costLedger, "memory experiment recovery");
1533
- for (const attempt of attempts) if ((input.candidateById.get(attempt.candidateId).externalCostUsdPerSequence ?? 0) > 0 && !hasSettledPaidCall(input.costLedger, memoryAttemptCostCallId(attempt, "execute", 0))) appendMemoryAttemptEvent(input.storage, input.attemptLogPath, {
1534
- ...attempt,
1535
- status: "cleaned",
1536
- recovery: true,
1537
- recordedAt: (input.options.now ?? (() => /* @__PURE__ */ new Date()))().toISOString()
1538
- });
2266
+ for (const attempt of attempts) if ((input.candidateById.get(attempt.candidateId).externalCostUsdPerSequence ?? 0) > 0 && !hasSettledPaidCall(input.costLedger, memoryAttemptCostCallId(attempt, "execute", 0))) {
2267
+ await input.lease.assertOwned();
2268
+ appendMemoryAttemptEvent(input.storage, input.attemptLogPath, {
2269
+ ...attempt,
2270
+ status: "cleaned",
2271
+ recovery: true,
2272
+ recordedAt: (input.options.now ?? (() => /* @__PURE__ */ new Date()))().toISOString()
2273
+ });
2274
+ }
1539
2275
  attempts = readActiveMemoryAttempts(input.storage, input.attemptLogPath);
1540
- const recoveryGenerations = reserveRecoveryAttempts({
1541
- storage: input.storage,
1542
- path: input.recoveryLogPath,
1543
- attemptIds: attempts.map((attempt) => attempt.branchId),
1544
- maxRetriesPerAttempt: input.maxRecoveryRetriesPerAttempt,
1545
- label: "memory recovery attempt log",
1546
- now: input.options.now
1547
- });
2276
+ const recoveryGenerations = /* @__PURE__ */ new Map();
2277
+ for (const attempt of attempts.sort((left, right) => left.branchId.localeCompare(right.branchId))) {
2278
+ await input.lease.assertOwned();
2279
+ const reserved = reserveRecoveryAttempts({
2280
+ storage: input.storage,
2281
+ path: input.recoveryLogPath,
2282
+ attemptIds: [attempt.branchId],
2283
+ maxRetriesPerAttempt: input.maxRecoveryRetriesPerAttempt,
2284
+ label: "memory recovery attempt log",
2285
+ now: input.options.now
2286
+ });
2287
+ recoveryGenerations.set(attempt.branchId, reserved.get(attempt.branchId));
2288
+ }
1548
2289
  const pool = createMemoryExecutionPool(input.maxConcurrency);
1549
- const failures = (await Promise.allSettled(attempts.sort((left, right) => left.branchId.localeCompare(right.branchId)).map((attempt) => pool.run(async () => {
2290
+ const failures = (await Promise.allSettled(attempts.map((attempt) => pool.run(async () => {
2291
+ input.options.signal?.throwIfAborted();
1550
2292
  await input.lease.assertOwned();
1551
2293
  const candidate = input.candidateById.get(attempt.candidateId);
1552
2294
  if (!candidate) throw new Error(`cannot recover memory branch '${attempt.branchId}': candidate '${attempt.candidateId}' is missing; pass it in recoveryCandidates`);
1553
2295
  assertMemoryAttemptCandidateMatches(attempt, candidate);
1554
2296
  const sequence = input.sequenceById.get(attempt.sequenceId);
1555
2297
  if (!sequence) throw new Error(`cannot recover memory branch '${attempt.branchId}': sequence '${attempt.sequenceId}' is missing`);
2298
+ assertMemoryAttemptSequenceMatches(attempt, sequence);
1556
2299
  if ((input.options.cleanupBranches ?? true) !== attempt.cleanupBranches) throw new Error(`cannot recover memory branch '${attempt.branchId}': cleanupBranches changed`);
1557
2300
  const recoveryCostUsd = candidate.externalRecoveryCostUsdPerAttempt ?? 0;
1558
2301
  const recoveryGeneration = recoveryGenerations.get(attempt.branchId);
@@ -1578,6 +2321,7 @@ async function recoverAbandonedMemoryAttempts(input) {
1578
2321
  costRecorder.record(actualCostUsd);
1579
2322
  }
1580
2323
  });
2324
+ await input.lease.assertOwned();
1581
2325
  appendMemoryAttemptEvent(input.storage, input.attemptLogPath, {
1582
2326
  ...attempt,
1583
2327
  status: "cleaned",
@@ -1615,34 +2359,30 @@ async function recoverMemoryAttempt(input) {
1615
2359
  let memory;
1616
2360
  let primaryError;
1617
2361
  try {
1618
- const abortController = new AbortController();
1619
- const creation = Promise.resolve().then(() => candidate.createAdapter({
1620
- branchId: attempt.branchId,
1621
- purpose: "recovery",
1622
- signal: abortController.signal,
1623
- maximumCostUsd: candidate.externalRecoveryCostUsdPerAttempt ?? 0,
1624
- markExternalCall: onExternalCall,
1625
- recordExternalCost
1626
- }));
1627
- releaseMemoryAdapterCreatedAfterAbort({
1628
- creation,
1629
- signal: abortController.signal,
2362
+ options.signal?.throwIfAborted();
2363
+ const recovered = await createBoundedMemoryAdapter({
2364
+ operation: `${candidate.id}: recovery adapter creation`,
2365
+ timeoutMs: cleanupTimeoutMs,
2366
+ signal: options.signal,
2367
+ create: (signal) => candidate.createAdapter({
2368
+ branchId: attempt.branchId,
2369
+ purpose: "recovery",
2370
+ signal,
2371
+ maximumCostUsd: candidate.externalRecoveryCostUsdPerAttempt ?? 0,
2372
+ markExternalCall: onExternalCall,
2373
+ recordExternalCost
2374
+ }),
1630
2375
  dispose: candidate.disposeAdapter ? async (created) => {
1631
2376
  onExternalCall();
1632
2377
  await candidate.disposeAdapter?.(created);
1633
2378
  } : void 0
1634
2379
  });
1635
- const recovered = await runBoundedMemoryLifecycle({
1636
- operation: `${candidate.id}: recovery adapter creation`,
1637
- timeoutMs: cleanupTimeoutMs,
1638
- abortController,
1639
- run: () => creation
1640
- });
2380
+ options.signal?.throwIfAborted();
1641
2381
  await lease.assertOwned();
1642
2382
  if (recovered === null) return;
1643
2383
  rawAdapter = recovered;
1644
2384
  adapter = trackExternalMemoryCalls(recovered, onExternalCall);
1645
- await sleepForMemoryRecovery(memoryRecoveryDelayMs(adapter), () => lease.assertOwned(), cleanupTimeoutMs, `${candidate.id}: abandoned branch recovery visibility wait`);
2385
+ await sleepForMemoryRecovery(memoryRecoveryDelayMs(adapter), () => lease.assertOwned(), cleanupTimeoutMs, `${candidate.id}: abandoned branch recovery visibility wait`, options.signal);
1646
2386
  if (cleanupBranches) {
1647
2387
  if (!adapter.clear) throw new Error(`${candidate.id}: abandoned branch recovery requires an adapter with scoped clear support`);
1648
2388
  memory = createAgentMemoryBranch({
@@ -1651,7 +2391,7 @@ async function recoverMemoryAttempt(input) {
1651
2391
  lifetime: "attempt",
1652
2392
  policy: candidate.policy,
1653
2393
  allowedWriteScopes: sequenceCleanupScopes(sequence),
1654
- baseScope: memoryExperimentBaseScope(options, candidate, sequence.id)
2394
+ baseScope: memoryExperimentBaseScope(candidate)
1655
2395
  });
1656
2396
  await runBoundedMemoryLifecycle({
1657
2397
  operation: `${candidate.id}: abandoned branch cleanup`,
@@ -1713,6 +2453,7 @@ function memoryAttemptCostCallId(attempt, purpose, generation) {
1713
2453
  externalCostUsdPerSequence: attempt.externalCostUsdPerSequence,
1714
2454
  externalRecoveryCostUsdPerAttempt: attempt.externalRecoveryCostUsdPerAttempt,
1715
2455
  sequenceId: attempt.sequenceId,
2456
+ sequenceRef: attempt.sequenceRef,
1716
2457
  rep: attempt.rep,
1717
2458
  seed: attempt.seed
1718
2459
  }));
@@ -1732,6 +2473,7 @@ function memoryAttemptEvent(input) {
1732
2473
  candidateId: input.candidate.id,
1733
2474
  candidateRef: input.candidate.ref,
1734
2475
  sequenceId: input.sequence.id,
2476
+ sequenceRef: canonicalDigest(input.sequence),
1735
2477
  rep: input.rep,
1736
2478
  seed: input.seed,
1737
2479
  cleanupBranches: input.cleanupBranches ?? true,
@@ -1761,11 +2503,14 @@ function readActiveMemoryAttempts(storage, path) {
1761
2503
  }
1762
2504
  function parseMemoryAttemptEvent(value, path, line) {
1763
2505
  const event = value;
1764
- if (!(typeof event === "object" && event !== null && (event.status === "started" || event.status === "cleaned") && typeof event.branchId === "string" && event.branchId.length > 0 && typeof event.candidateId === "string" && event.candidateId.length > 0 && typeof event.candidateRef === "string" && event.candidateRef.length > 0 && typeof event.sequenceId === "string" && event.sequenceId.length > 0 && Number.isSafeInteger(event.rep) && Number.isSafeInteger(event.seed) && typeof event.cleanupBranches === "boolean" && typeof event.externalCostUsdPerSequence === "number" && Number.isFinite(event.externalCostUsdPerSequence) && event.externalCostUsdPerSequence >= 0 && typeof event.externalRecoveryCostUsdPerAttempt === "number" && Number.isFinite(event.externalRecoveryCostUsdPerAttempt) && event.externalRecoveryCostUsdPerAttempt >= 0 && typeof event.recordedAt === "string" && !Number.isNaN(Date.parse(event.recordedAt)) && typeof event.recovery === "boolean")) throw new Error(`invalid memory attempt event in '${path}' line ${line}`);
2506
+ if (!(typeof event === "object" && event !== null && (event.status === "started" || event.status === "cleaned") && typeof event.branchId === "string" && event.branchId.length > 0 && typeof event.candidateId === "string" && event.candidateId.length > 0 && typeof event.candidateRef === "string" && event.candidateRef.length > 0 && typeof event.sequenceId === "string" && event.sequenceId.length > 0 && typeof event.sequenceRef === "string" && /^sha256:[a-f0-9]{64}$/.test(event.sequenceRef) && Number.isSafeInteger(event.rep) && Number.isSafeInteger(event.seed) && typeof event.cleanupBranches === "boolean" && typeof event.externalCostUsdPerSequence === "number" && Number.isFinite(event.externalCostUsdPerSequence) && event.externalCostUsdPerSequence >= 0 && typeof event.externalRecoveryCostUsdPerAttempt === "number" && Number.isFinite(event.externalRecoveryCostUsdPerAttempt) && event.externalRecoveryCostUsdPerAttempt >= 0 && typeof event.recordedAt === "string" && !Number.isNaN(Date.parse(event.recordedAt)) && typeof event.recovery === "boolean")) throw new Error(`invalid memory attempt event in '${path}' line ${line}`);
1765
2507
  return value;
1766
2508
  }
1767
2509
  function sameMemoryAttempt(left, right) {
1768
- return left.branchId === right.branchId && left.candidateId === right.candidateId && left.candidateRef === right.candidateRef && left.sequenceId === right.sequenceId && left.rep === right.rep && left.seed === right.seed && left.cleanupBranches === right.cleanupBranches && left.externalCostUsdPerSequence === right.externalCostUsdPerSequence && left.externalRecoveryCostUsdPerAttempt === right.externalRecoveryCostUsdPerAttempt;
2510
+ return left.branchId === right.branchId && left.candidateId === right.candidateId && left.candidateRef === right.candidateRef && left.sequenceId === right.sequenceId && left.sequenceRef === right.sequenceRef && left.rep === right.rep && left.seed === right.seed && left.cleanupBranches === right.cleanupBranches && left.externalCostUsdPerSequence === right.externalCostUsdPerSequence && left.externalRecoveryCostUsdPerAttempt === right.externalRecoveryCostUsdPerAttempt;
2511
+ }
2512
+ function assertMemoryAttemptSequenceMatches(attempt, sequence) {
2513
+ if (canonicalDigest(sequence) !== attempt.sequenceRef) throw new Error(`cannot recover memory branch '${attempt.branchId}': sequence '${attempt.sequenceId}' changed; restore the recorded sequence before cleanup`);
1769
2514
  }
1770
2515
  function assertMemoryAttemptCandidateMatches(attempt, candidate) {
1771
2516
  if (candidate.ref !== attempt.candidateRef) throw new Error(`cannot recover memory branch '${attempt.branchId}': candidate ref changed from '${attempt.candidateRef}' to '${candidate.ref}'`);
@@ -1775,8 +2520,11 @@ function assertMemoryAttemptCandidateMatches(attempt, candidate) {
1775
2520
  }
1776
2521
  //#endregion
1777
2522
  //#region src/memory/experiment/cell.ts
2523
+ var AgentMemoryAdapterCapabilityError = class extends Error {
2524
+ name = "AgentMemoryAdapterCapabilityError";
2525
+ };
1778
2526
  async function runSequenceCell(input) {
1779
- const { options, candidate, scenario, context, runIdentity, storage, attemptLogPath, lease } = input;
2527
+ const { options, candidate, scenario, context, runIdentity, storage, attemptLogPath, lease, memoryMode, comparisonRef } = input;
1780
2528
  const cleanupBranches = options.cleanupBranches ?? true;
1781
2529
  const costUsd = candidate.externalCostUsdPerSequence ?? 0;
1782
2530
  if (!Number.isFinite(costUsd) || costUsd < 0) throw new Error(`${candidate.id}: externalCostUsdPerSequence must be a non-negative finite number`);
@@ -1797,6 +2545,7 @@ async function runSequenceCell(input) {
1797
2545
  cleanupBranches,
1798
2546
  now: options.now
1799
2547
  });
2548
+ await lease.assertOwned();
1800
2549
  appendMemoryAttemptEvent(storage, attemptLogPath, attempt);
1801
2550
  let externalCallAttempted = false;
1802
2551
  const costRecorder = createAgentMemoryCostRecorder({
@@ -1826,18 +2575,27 @@ async function runSequenceCell(input) {
1826
2575
  let finalClearStarted = false;
1827
2576
  let finalClearCompleted = false;
1828
2577
  try {
1829
- const created = await candidate.createAdapter({
1830
- branchId,
1831
- purpose: "execute",
2578
+ const created = await createBoundedMemoryAdapter({
2579
+ operation: `${candidate.id}: execution adapter creation`,
2580
+ timeoutMs: cleanupTimeoutMs,
1832
2581
  signal: context.signal,
1833
- maximumCostUsd: costUsd,
1834
- markExternalCall: () => {
1835
- externalCallAttempted = true;
1836
- },
1837
- recordExternalCost: (actualCostUsd) => {
2582
+ create: (signal) => candidate.createAdapter({
2583
+ branchId,
2584
+ purpose: "execute",
2585
+ signal,
2586
+ maximumCostUsd: costUsd,
2587
+ markExternalCall: () => {
2588
+ externalCallAttempted = true;
2589
+ },
2590
+ recordExternalCost: (actualCostUsd) => {
2591
+ externalCallAttempted = true;
2592
+ costRecorder.record(actualCostUsd);
2593
+ }
2594
+ }),
2595
+ dispose: candidate.disposeAdapter ? async (lateAdapter) => {
1838
2596
  externalCallAttempted = true;
1839
- costRecorder.record(actualCostUsd);
1840
- }
2597
+ await candidate.disposeAdapter?.(lateAdapter);
2598
+ } : void 0
1841
2599
  });
1842
2600
  if (!created) throw new Error(`${candidate.id}: createAdapter returned no execution adapter`);
1843
2601
  rawAdapter = created;
@@ -1846,6 +2604,7 @@ async function runSequenceCell(input) {
1846
2604
  });
1847
2605
  await lease.assertOwned();
1848
2606
  context.signal.throwIfAborted();
2607
+ if (memoryMode === "stateless" && !adapter.clear) throw new AgentMemoryAdapterCapabilityError(`${candidate.id}: stateless memoryMode requires scoped clear support`);
1849
2608
  if (cleanupBranches && !adapter.clear) throw new Error(`${candidate.id}: cleanupBranches requires an adapter with scoped clear support`);
1850
2609
  memory = createAgentMemoryBranch({
1851
2610
  adapter,
@@ -1853,40 +2612,67 @@ async function runSequenceCell(input) {
1853
2612
  lifetime: "attempt",
1854
2613
  policy: candidate.policy,
1855
2614
  allowedWriteScopes: sequenceCleanupScopes(scenario.sequence),
1856
- baseScope: memoryExperimentBaseScope(options, candidate, scenario.sequenceId)
2615
+ baseScope: memoryExperimentBaseScope(candidate)
2616
+ });
2617
+ const runProviderOperation = (operation, run) => runBoundedMemoryLifecycle({
2618
+ operation,
2619
+ timeoutMs: cleanupTimeoutMs,
2620
+ resource: adapter,
2621
+ signal: context.signal,
2622
+ cancellationTimeoutMs: MEMORY_OPERATION_CANCELLATION_TIMEOUT_MS,
2623
+ run
1857
2624
  });
1858
2625
  const probes = [];
1859
2626
  for (const [ordinal, step] of scenario.sequence.steps.entries()) {
1860
2627
  context.signal.throwIfAborted();
1861
2628
  await lease.assertOwned();
1862
- await writeStep(memory, step);
2629
+ if (memoryMode === "stateless" && ordinal > 0) {
2630
+ await runBoundedMemoryLifecycle({
2631
+ operation: `${candidate.id}: stateless reset before step ${step.id}`,
2632
+ timeoutMs: cleanupTimeoutMs,
2633
+ resource: adapter,
2634
+ run: () => clearSequenceScopes(memory, scenario.sequence)
2635
+ });
2636
+ context.signal.throwIfAborted();
2637
+ await lease.assertOwned();
2638
+ }
2639
+ await runProviderOperation(`${candidate.id}: writes for step ${step.id}`, () => writeStep(memory, step));
1863
2640
  await lease.assertOwned();
1864
2641
  if (options.executeStep) {
1865
2642
  const execution = createAgentMemoryExecutionContext(context, scenario.sequence);
1866
2643
  try {
1867
- await options.executeStep({
2644
+ await runProviderOperation(`${candidate.id}: execution for step ${step.id}`, () => options.executeStep({
1868
2645
  memory,
1869
2646
  candidateId: candidate.id,
1870
2647
  step: executionStep(step, ordinal),
1871
2648
  context: execution.context
1872
- });
2649
+ }));
2650
+ } catch (error) {
2651
+ execution.abort();
2652
+ throw error;
1873
2653
  } finally {
1874
2654
  execution.dispose();
1875
2655
  }
1876
2656
  }
1877
2657
  context.signal.throwIfAborted();
1878
2658
  await lease.assertOwned();
1879
- const stepProbes = await probeStep(memory, scenario.sequence, step);
2659
+ const stepProbes = await runProviderOperation(`${candidate.id}: probes for step ${step.id}`, () => probeStep(memory, scenario.sequence, step, ordinal, context));
1880
2660
  await lease.assertOwned();
1881
2661
  probes.push(...stepProbes);
1882
2662
  }
1883
- const snapshot = await memory.snapshot();
2663
+ const snapshot = await runProviderOperation(`${candidate.id}: branch snapshot`, () => memory.snapshot());
1884
2664
  await lease.assertOwned();
1885
- await options.onBranchSnapshot?.({
1886
- candidateId: candidate.id,
1887
- sequenceId: scenario.sequenceId,
1888
- cellId: context.cellId,
1889
- snapshot
2665
+ if (options.onBranchSnapshot) await runBoundedMemoryLifecycle({
2666
+ operation: `${candidate.id}: branch snapshot callback`,
2667
+ timeoutMs: cleanupTimeoutMs,
2668
+ signal: context.signal,
2669
+ cancellationTimeoutMs: MEMORY_OPERATION_CANCELLATION_TIMEOUT_MS,
2670
+ run: () => options.onBranchSnapshot({
2671
+ candidateId: candidate.id,
2672
+ sequenceId: scenario.sequenceId,
2673
+ cellId: context.cellId,
2674
+ snapshot
2675
+ })
1890
2676
  });
1891
2677
  await lease.assertOwned();
1892
2678
  const dimensions = meanDimensions(probes.map((probe) => probe.dimensions));
@@ -1894,6 +2680,8 @@ async function runSequenceCell(input) {
1894
2680
  const artifact = {
1895
2681
  candidateId: candidate.id,
1896
2682
  sequenceId: scenario.sequenceId,
2683
+ memoryMode,
2684
+ comparisonRef,
1897
2685
  score: mean$1(probes.map((probe) => probe.score)),
1898
2686
  passed: probes.length > 0 && probes.every((probe) => probe.passed),
1899
2687
  dimensions,
@@ -1969,6 +2757,12 @@ async function runSequenceCell(input) {
1969
2757
  cleanupErrors.push(error);
1970
2758
  }
1971
2759
  } else cleanupErrors.push(/* @__PURE__ */ new Error(`${candidate.id}: adapter creation failed before cleanup could be confirmed`));
2760
+ if (cleanupOwned && cleanupErrors.length === 0) try {
2761
+ await lease.assertOwned();
2762
+ } catch (error) {
2763
+ cleanupOwned = false;
2764
+ ownershipError = error;
2765
+ }
1972
2766
  if (cleanupOwned && cleanupErrors.length === 0) appendCleanedAttempt(primaryError);
1973
2767
  if (!cleanupOwned && cleanupErrors.length === 0) {
1974
2768
  if (primaryError) throw primaryError;
@@ -2028,46 +2822,38 @@ async function writeStep(memory, step) {
2028
2822
  }
2029
2823
  for (const write of writes) await memory.write(write);
2030
2824
  }
2031
- async function probeStep(memory, sequence, step) {
2825
+ async function probeStep(memory, sequence, step, stepOrdinal, dispatch) {
2032
2826
  const run = async (probe) => {
2033
2827
  const scope = mergeScopes$2(step.scope, probe.scope);
2034
2828
  const context = await memory.getContext(probe.query, {
2035
2829
  scope,
2036
2830
  limit: probe.limit
2037
2831
  });
2038
- const artifact = {
2832
+ const scoringEvidence = {
2039
2833
  answer: context.text,
2040
2834
  rememberedFacts: context.hits.map((hit) => hit.text),
2041
2835
  citedEventIds: uniqueStrings$1([...context.hits.map((hit) => hit.metadata?.eventId), ...context.hits.flatMap((hit) => stringArray$1(hit.metadata?.eventIds))]),
2042
2836
  usedMemoryIds: context.hits.map((hit) => hit.id),
2043
2837
  actorIds: uniqueStrings$1([...context.hits.map((hit) => hit.metadata?.actorId), ...context.hits.flatMap((hit) => stringArray$1(hit.metadata?.actorIds))])
2044
2838
  };
2045
- const evaluation = scoreMemoryBenchmarkArtifact({
2046
- id: `${sequence.id}:${step.id}:${probe.id}`,
2047
- family: sequence.family,
2048
- taskKind: probe.taskKind ?? "memory-recall",
2049
- split: sequence.split,
2050
- events: [],
2051
- prompt: probe.query,
2052
- requiredFacts: probe.requiredFacts && probe.requiredFacts.length > 0 ? probe.requiredFacts : probe.referenceAnswer ? [{
2053
- id: `${probe.id}:reference`,
2054
- anyOf: [probe.referenceAnswer]
2055
- }] : void 0,
2056
- forbiddenFacts: probe.forbiddenFacts,
2057
- expectedEventIds: probe.expectedEventIds,
2058
- expectedActorIds: probe.expectedActorIds,
2059
- referenceAnswer: probe.referenceAnswer
2060
- }, artifact);
2839
+ const evidenceRef = canonicalDigest(scoringEvidence);
2840
+ const evidencePath = await dispatch.artifacts.writeJson(`memory-evidence/${evidenceRef.slice(7)}.json`, scoringEvidence);
2841
+ const evaluation = scoreAgentMemoryProbe(sequence, step, probe, scoringEvidence);
2061
2842
  return {
2062
2843
  id: probe.id,
2063
2844
  stepId: step.id,
2845
+ stepOrdinal,
2846
+ ...probe.retentionKey !== void 0 ? { retentionKey: probe.retentionKey } : {},
2847
+ ...probe.transferKey !== void 0 ? { transferKey: probe.transferKey } : {},
2064
2848
  query: probe.query,
2065
2849
  score: evaluation.score,
2066
2850
  passed: evaluation.passed,
2067
2851
  dimensions: evaluation.dimensions,
2068
2852
  applicableDimensions: evaluation.applicableDimensions ?? Object.keys(evaluation.dimensions),
2069
2853
  notes: evaluation.notes,
2070
- hitIds: context.hits.map((hit) => hit.id)
2854
+ hitIds: context.hits.map((hit) => hit.id),
2855
+ evidenceRef,
2856
+ evidencePath
2071
2857
  };
2072
2858
  };
2073
2859
  if (step.parallelProbes === false) {
@@ -2079,12 +2865,13 @@ async function probeStep(memory, sequence, step) {
2079
2865
  }
2080
2866
  //#endregion
2081
2867
  //#region src/memory/experiment/run.ts
2082
- const MEMORY_EXPERIMENT_IMPLEMENTATION_REF = "agent-knowledge:memory-experiment:v6";
2083
2868
  /** Runs ordered, branch-isolated memory histories across candidate systems. */
2084
2869
  async function runAgentMemoryExperiment(options) {
2870
+ options.signal?.throwIfAborted();
2085
2871
  assertNonEmptyString(options.experimentId, "memory experiment experimentId");
2086
2872
  assertNonEmptyString(options.runDir, "memory experiment runDir");
2087
2873
  if (options.experimentRunId !== void 0) assertNonEmptyString(options.experimentRunId, "memory experiment experimentRunId");
2874
+ resolveAgentMemoryMode(options.memoryMode);
2088
2875
  if (options.sequences.length === 0) throw new Error("memory experiment requires sequences");
2089
2876
  if (options.candidates.length === 0) throw new Error("memory experiment requires candidates");
2090
2877
  if (options.executeStep && !options.executeStepRef) throw new Error("memory experiment executeStepRef is required when executeStep is configured");
@@ -2139,9 +2926,12 @@ async function runAgentMemoryExperiment(options) {
2139
2926
  return result;
2140
2927
  }
2141
2928
  async function runOwnedAgentMemoryExperiment(options, storage, runDir, lease) {
2929
+ const memoryMode = resolveAgentMemoryMode(options.memoryMode);
2930
+ const comparisonRef = memoryExperimentComparisonRef(options, runDir);
2142
2931
  const runIdentity = stableId("memory_run", canonicalJson({
2143
2932
  experimentId: options.experimentId,
2144
- experimentRunId: options.experimentRunId ?? runDir
2933
+ experimentRunId: options.experimentRunId ?? runDir,
2934
+ memoryMode
2145
2935
  }));
2146
2936
  const maxConcurrency = options.maxConcurrency ?? 2;
2147
2937
  if (!Number.isSafeInteger(maxConcurrency) || maxConcurrency <= 0) throw new Error("memory experiment maxConcurrency must be a positive safe integer");
@@ -2175,6 +2965,7 @@ async function runOwnedAgentMemoryExperiment(options, storage, runDir, lease) {
2175
2965
  recoveryLogPath,
2176
2966
  maxRecoveryRetriesPerAttempt: options.maxRecoveryRetriesPerAttempt ?? 3
2177
2967
  });
2968
+ options.signal?.throwIfAborted();
2178
2969
  await lease.assertOwned();
2179
2970
  let campaign;
2180
2971
  let campaignError;
@@ -2182,6 +2973,7 @@ async function runOwnedAgentMemoryExperiment(options, storage, runDir, lease) {
2182
2973
  try {
2183
2974
  campaign = await runCampaign({
2184
2975
  scenarios,
2976
+ signal: options.signal,
2185
2977
  dispatch: (scenario, context) => {
2186
2978
  const candidate = candidateById.get(scenario.candidateId);
2187
2979
  if (!candidate) throw new Error(`unknown memory candidate ${scenario.candidateId}`);
@@ -2193,12 +2985,14 @@ async function runOwnedAgentMemoryExperiment(options, storage, runDir, lease) {
2193
2985
  runIdentity,
2194
2986
  storage,
2195
2987
  attemptLogPath,
2196
- lease
2988
+ lease,
2989
+ memoryMode,
2990
+ comparisonRef
2197
2991
  }));
2198
2992
  dispatchedExecutions.push(operation);
2199
2993
  return operation;
2200
2994
  },
2201
- dispatchRef: memoryExperimentDispatchRef(options),
2995
+ dispatchRef: memoryExperimentDispatchRef(memoryMode, comparisonRef),
2202
2996
  judges: [agentMemorySequenceJudge()],
2203
2997
  runDir,
2204
2998
  storage,
@@ -2210,6 +3004,7 @@ async function runOwnedAgentMemoryExperiment(options, storage, runDir, lease) {
2210
3004
  costPhase: options.costPhase,
2211
3005
  maxConcurrency,
2212
3006
  dispatchTimeoutMs: options.dispatchTimeoutMs,
3007
+ dispatchShutdownTimeoutMs: MEMORY_CAMPAIGN_DISPATCH_SHUTDOWN_TIMEOUT_MS,
2213
3008
  expectUsage: "off",
2214
3009
  now: options.now
2215
3010
  });
@@ -2219,9 +3014,14 @@ async function runOwnedAgentMemoryExperiment(options, storage, runDir, lease) {
2219
3014
  settledExecutions = await Promise.allSettled(dispatchedExecutions);
2220
3015
  }
2221
3016
  const cleanupFailures = settledExecutions.flatMap((settled) => settled.status === "rejected" && settled.reason instanceof AgentMemoryCleanupError ? [settled.reason] : []);
2222
- if (campaignError && cleanupFailures.length > 0) throw new AggregateError([campaignError, ...cleanupFailures], "memory experiment failed and provider cleanup also failed");
3017
+ const capabilityFailures = settledExecutions.flatMap((settled) => settled.status === "rejected" && settled.reason instanceof AgentMemoryAdapterCapabilityError ? [settled.reason] : []);
3018
+ const terminalFailures = [...cleanupFailures, ...capabilityFailures];
3019
+ if (campaignError && terminalFailures.length > 0) throw new AggregateError([campaignError, ...terminalFailures], "memory experiment failed with terminal dispatch errors");
2223
3020
  if (campaignError) throw campaignError;
3021
+ if (cleanupFailures.length > 0 && capabilityFailures.length > 0) throw new AggregateError(terminalFailures, "memory experiment adapter requirements and provider cleanup failed");
2224
3022
  if (cleanupFailures.length > 0) throw new AggregateError(cleanupFailures, "memory experiment cleanup failed after dispatch");
3023
+ if (capabilityFailures.length === 1) throw capabilityFailures[0];
3024
+ if (capabilityFailures.length > 1) throw new AggregateError(capabilityFailures, "memory experiment adapter requirements failed");
2225
3025
  if (!campaign) throw new Error("memory experiment produced no campaign result");
2226
3026
  await lease.assertOwned();
2227
3027
  const costByCandidate = memoryExperimentCostByCandidate(costLedger, campaign.runDir, scenarios, [...options.candidates, ...options.recoveryCandidates ?? []].map((candidate) => candidate.id));
@@ -2232,12 +3032,26 @@ async function runOwnedAgentMemoryExperiment(options, storage, runDir, lease) {
2232
3032
  const rankingMarkdownPath = join(campaign.runDir, "memory-experiment-ranking.md");
2233
3033
  storage.write(rankingJsonPath, `${JSON.stringify({
2234
3034
  experimentId: options.experimentId,
3035
+ memoryMode,
3036
+ comparisonRef,
3037
+ candidateRefs: options.candidates.map(({ id, ref }) => ({
3038
+ id,
3039
+ ref
3040
+ })),
3041
+ executionRef: options.executeStepRef ?? "fixtures",
2235
3042
  totalCostUsd,
2236
3043
  unrankedRecoveryCostUsd,
2237
3044
  rows
2238
3045
  }, null, 2)}\n`);
2239
3046
  storage.write(rankingMarkdownPath, renderAgentMemoryExperimentRanking(rows, totalCostUsd, unrankedRecoveryCostUsd));
2240
3047
  return {
3048
+ memoryMode,
3049
+ comparisonRef,
3050
+ candidateRefs: options.candidates.map(({ id, ref }) => ({
3051
+ id,
3052
+ ref
3053
+ })),
3054
+ executionRef: options.executeStepRef ?? "fixtures",
2241
3055
  campaign,
2242
3056
  rows,
2243
3057
  totalCostUsd,
@@ -2249,25 +3063,98 @@ async function runOwnedAgentMemoryExperiment(options, storage, runDir, lease) {
2249
3063
  recoveryLogPath
2250
3064
  };
2251
3065
  }
2252
- function memoryExperimentDispatchRef(options) {
3066
+ function memoryExperimentDispatchRef(memoryMode, comparisonRef) {
2253
3067
  return stableId("memory_experiment", canonicalJson({
2254
3068
  implementationRef: MEMORY_EXPERIMENT_IMPLEMENTATION_REF,
2255
- experimentId: options.experimentId,
2256
- experimentRunId: options.experimentRunId ?? null,
2257
- executeStepRef: options.executeStepRef ?? "fixtures",
2258
- cleanupBranches: options.cleanupBranches ?? true,
2259
- candidates: options.candidates.map((candidate) => ({
2260
- id: candidate.id,
2261
- ref: candidate.ref,
2262
- policy: candidate.policy ?? null,
2263
- baseScope: candidate.baseScope ?? null,
2264
- externalCostUsdPerSequence: candidate.externalCostUsdPerSequence ?? 0,
2265
- externalRecoveryCostUsdPerAttempt: candidate.externalRecoveryCostUsdPerAttempt ?? 0,
2266
- externalCostAccounting: candidate.externalCostAccounting ?? "exact"
2267
- })).sort((a, b) => a.id.localeCompare(b.id))
3069
+ memoryMode,
3070
+ comparisonRef
2268
3071
  }));
2269
3072
  }
2270
3073
  //#endregion
3074
+ //#region src/memory/experiment/learning.ts
3075
+ /** Compare exact matched stateful and stateless memory runs. */
3076
+ function compareAgentMemoryLearning(options) {
3077
+ return measureAgentMemoryLearning(pairAgentMemoryLearningRuns(options), options.stateful, options.stateless);
3078
+ }
3079
+ /** Run a matched stateful-versus-stateless memory experiment under one cost limit. */
3080
+ async function runAgentMemoryLearningExperiment(options) {
3081
+ assertNonEmptyString(options.runDir, "memory learning experiment runDir");
3082
+ options.signal?.throwIfAborted();
3083
+ assertMemoryLearningSequences(options.sequences);
3084
+ for (const candidate of [...options.candidates, ...options.recoveryCandidates ?? []]) assertImmutableRef(candidate.ref, `${candidate.id}: memory learning candidate ref`);
3085
+ if (options.executeStep) assertImmutableRef(options.executeStepRef, "memory learning experiment executeStepRef");
3086
+ const storage = options.storage ?? fsCampaignStorage();
3087
+ const runDir = resolveRunDir(options.runDir, options.repo);
3088
+ storage.ensureDir(runDir);
3089
+ const costCeiling = options.costCeiling ?? options.costLedger?.costCeilingUsd ?? 0;
3090
+ const costLedger = options.costLedger ?? createRunCostLedger({
3091
+ storage,
3092
+ runDir,
3093
+ costCeilingUsd: costCeiling
3094
+ });
3095
+ if (costLedger.costCeilingUsd !== costCeiling) throw new Error("memory learning experiment costCeiling must match the shared cost ledger ceiling");
3096
+ const armOrder = options.armOrder ?? "stateful-first";
3097
+ if (armOrder !== "stateful-first" && armOrder !== "stateless-first") throw new Error("memory learning experiment armOrder must be 'stateful-first' or 'stateless-first'");
3098
+ const { armOrder: _armOrder, ...experimentOptions } = options;
3099
+ const shared = {
3100
+ ...experimentOptions,
3101
+ experimentRunId: options.experimentRunId ?? runDir,
3102
+ costCeiling,
3103
+ costLedger
3104
+ };
3105
+ const runArm = async (memoryMode) => {
3106
+ const result = await runAgentMemoryExperiment({
3107
+ ...shared,
3108
+ memoryMode,
3109
+ runDir: armRunDir(runDir, memoryMode)
3110
+ });
3111
+ assertStoredAgentMemoryEvidence(storage, result, options.sequences);
3112
+ return result;
3113
+ };
3114
+ const firstMode = armOrder === "stateful-first" ? "stateful" : "stateless";
3115
+ const secondMode = firstMode === "stateful" ? "stateless" : "stateful";
3116
+ const first = await runArm(firstMode);
3117
+ const second = await runArm(secondMode);
3118
+ const stateful = firstMode === "stateful" ? first : second;
3119
+ const stateless = firstMode === "stateless" ? first : second;
3120
+ const comparison = compareAgentMemoryLearning({
3121
+ stateful,
3122
+ stateless
3123
+ });
3124
+ const ledgerSummary = costLedger.summary();
3125
+ const cost = {
3126
+ experimentUsd: normalizeUsd(stateful.totalCostUsd + stateless.totalCostUsd),
3127
+ ledgerUsd: normalizeUsd(ledgerSummary.totalCostUsd),
3128
+ ceilingUsd: costLedger.costCeilingUsd,
3129
+ accountingComplete: ledgerSummary.accountingComplete
3130
+ };
3131
+ const report = {
3132
+ armOrder,
3133
+ comparison,
3134
+ cost
3135
+ };
3136
+ const evidenceRef = canonicalDigest(report);
3137
+ const comparisonDir = armRunDir(runDir, "comparisons");
3138
+ storage.ensureDir(comparisonDir);
3139
+ const comparisonPath = armRunDir(comparisonDir, `${evidenceRef.slice(7)}.json`);
3140
+ storage.write(comparisonPath, `${JSON.stringify({
3141
+ evidenceRef,
3142
+ ...report
3143
+ }, null, 2)}\n`);
3144
+ return {
3145
+ armOrder,
3146
+ stateful,
3147
+ stateless,
3148
+ comparison,
3149
+ evidenceRef,
3150
+ comparisonPath,
3151
+ cost
3152
+ };
3153
+ }
3154
+ function armRunDir(runDir, child) {
3155
+ return runDir.startsWith("mem://") ? `${runDir.replace(/\/+$/, "")}/${child}` : join(runDir, child);
3156
+ }
3157
+ //#endregion
2271
3158
  //#region src/memory/graphiti.ts
2272
3159
  const DEFAULT_GRAPHITI_INGESTION_TIMEOUT_MS = 12e4;
2273
3160
  /** Connects the official Graphiti MCP server without coupling to its Python internals. */
@@ -2949,7 +3836,8 @@ async function evaluateMemoryCandidate(input) {
2949
3836
  reps: 1,
2950
3837
  maxConcurrency: 1,
2951
3838
  costCeiling: evaluationCostLimit,
2952
- costPhase: `memory.config.${input.surfaceHash}`
3839
+ costPhase: `memory.config.${input.surfaceHash}`,
3840
+ signal: input.signal
2953
3841
  });
2954
3842
  const cell = experiment.campaign.cells[0];
2955
3843
  if (!cell || cell.error || cell.artifact.candidateId !== input.candidate.id) throw new Error(`${input.surfaceHash}/${input.scenario.sequenceId}: memory config evaluation did not complete`);
@@ -3083,9 +3971,20 @@ function storedMemoryArtifactRecord(input, artifact) {
3083
3971
  function parseMemoryArtifact(value, candidateId, sequenceId, path) {
3084
3972
  if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error(`memory config artifact '${path}' has no artifact`);
3085
3973
  const artifact = value;
3086
- if (artifact.candidateId !== candidateId || artifact.sequenceId !== sequenceId || typeof artifact.score !== "number" || !Number.isFinite(artifact.score) || artifact.score < 0 || artifact.score > 1 || typeof artifact.passed !== "boolean" || !isFiniteNumberRecord(artifact.dimensions) || !isNonnegativeIntegerRecord(artifact.dimensionSampleCounts) || !Array.isArray(artifact.probes) || typeof artifact.branchDigest !== "string" || !artifact.branchDigest || typeof artifact.journalEntries !== "number" || !Number.isSafeInteger(artifact.journalEntries) || artifact.journalEntries < 0 || typeof artifact.durationMs !== "number" || !Number.isFinite(artifact.durationMs) || artifact.durationMs < 0) throw new Error(`memory config artifact '${path}' is malformed`);
3974
+ try {
3975
+ assertAgentMemoryExperimentComparisonRef(artifact.comparisonRef, `memory config artifact '${path}' comparisonRef`);
3976
+ } catch {
3977
+ throw new Error(`memory config artifact '${path}' is malformed`);
3978
+ }
3979
+ if (artifact.candidateId !== candidateId || artifact.sequenceId !== sequenceId || artifact.memoryMode !== "stateful" || typeof artifact.score !== "number" || !Number.isFinite(artifact.score) || artifact.score < 0 || artifact.score > 1 || typeof artifact.passed !== "boolean" || !isFiniteNumberRecord(artifact.dimensions) || !isNonnegativeIntegerRecord(artifact.dimensionSampleCounts) || !Array.isArray(artifact.probes) || !artifact.probes.every(isMemoryProbeResult) || typeof artifact.branchDigest !== "string" || !/^sha256:[a-f0-9]{64}$/.test(artifact.branchDigest) || typeof artifact.journalEntries !== "number" || !Number.isSafeInteger(artifact.journalEntries) || artifact.journalEntries < 0 || typeof artifact.durationMs !== "number" || !Number.isFinite(artifact.durationMs) || artifact.durationMs < 0) throw new Error(`memory config artifact '${path}' is malformed`);
3087
3980
  return artifact;
3088
3981
  }
3982
+ function isMemoryProbeResult(value) {
3983
+ if (!value || typeof value !== "object" || Array.isArray(value)) return false;
3984
+ const probe = value;
3985
+ if (typeof probe.id !== "string" || !probe.id || typeof probe.stepId !== "string" || !probe.stepId || !Number.isSafeInteger(probe.stepOrdinal) || probe.stepOrdinal < 0 || typeof probe.query !== "string" || !probe.query || typeof probe.score !== "number" || !Number.isFinite(probe.score) || probe.score < 0 || probe.score > 1 || typeof probe.passed !== "boolean" || !isFiniteNumberRecord(probe.dimensions) || !Array.isArray(probe.applicableDimensions) || !probe.applicableDimensions.every((entry) => typeof entry === "string" && entry.length > 0) || typeof probe.notes !== "string" || !Array.isArray(probe.hitIds) || !probe.hitIds.every((entry) => typeof entry === "string") || typeof probe.evidenceRef !== "string" || !/^sha256:[a-f0-9]{64}$/.test(probe.evidenceRef) || typeof probe.evidencePath !== "string" || !probe.evidencePath.endsWith(`/memory-evidence/${probe.evidenceRef.slice(7)}.json`)) return false;
3986
+ return (probe.retentionKey === void 0 || typeof probe.retentionKey === "string" && probe.retentionKey.length > 0) && (probe.transferKey === void 0 || typeof probe.transferKey === "string" && probe.transferKey.length > 0);
3987
+ }
3089
3988
  function isFiniteNumberRecord(value) {
3090
3989
  if (!value || typeof value !== "object" || Array.isArray(value)) return false;
3091
3990
  return Object.values(value).every((entry) => typeof entry === "number" && Number.isFinite(entry));
@@ -4489,6 +5388,6 @@ const AgentMemoryWriteInputSchema = z.object({
4489
5388
  metadata: z.record(z.string(), z.unknown()).optional()
4490
5389
  });
4491
5390
  //#endregion
4492
- export { assertImmutableRef as A, resetRetrievalHoldoutRegistry as C, jsonObjectCandidateCodec as D, jsonCandidateCodec as E, runSerializedKnowledgeOptimization as O, emitRetrievalHoldoutBypass as S, toOffPolicyTrajectory as T, defaultGetMemoryContext as _, createNeo4jAgentMemoryAdapter as a, applySessionStickyRetrievalHoldout as b, runAgentMemoryImprovement as c, runAgentMemoryExperiment as d, agentMemorySequenceJudge as f, forkAgentMemoryBranchSnapshot as g, createAgentMemoryBranch as h, AgentMemoryWriteInputSchema as i, scenarioContentFingerprint as k, createGraphitiMemoryAdapter as l, buildAgentMemorySequencesFromBenchmarkCases as m, AgentMemoryKindSchema as n, createMem0MemoryAdapter as o, buildAgentMemorySequenceScenarios as p, AgentMemoryScopeSchema as r, mem0MemoryAdapterIdentity as s, AgentMemoryHitSchema as t, graphitiMemoryAdapterIdentity as u, renderMemoryContext as v, retrievalHoldoutConfigHash as w, deterministicRng as x, applyRetrievalHoldout as y };
5391
+ export { runSerializedKnowledgeOptimization as A, deterministicRng as C, toOffPolicyTrajectory as D, retrievalHoldoutConfigHash as E, assertImmutableRef as M, jsonCandidateCodec as O, applySessionStickyRetrievalHoldout as S, resetRetrievalHoldoutRegistry as T, createAgentMemoryBranch as _, createNeo4jAgentMemoryAdapter as a, renderMemoryContext as b, runAgentMemoryImprovement as c, compareAgentMemoryLearning as d, runAgentMemoryLearningExperiment as f, buildAgentMemorySequencesFromBenchmarkCases as g, buildAgentMemorySequenceScenarios as h, AgentMemoryWriteInputSchema as i, scenarioContentFingerprint as j, jsonObjectCandidateCodec as k, createGraphitiMemoryAdapter as l, agentMemorySequenceJudge as m, AgentMemoryKindSchema as n, createMem0MemoryAdapter as o, runAgentMemoryExperiment as p, AgentMemoryScopeSchema as r, mem0MemoryAdapterIdentity as s, AgentMemoryHitSchema as t, graphitiMemoryAdapterIdentity as u, forkAgentMemoryBranchSnapshot as v, emitRetrievalHoldoutBypass as w, applyRetrievalHoldout as x, defaultGetMemoryContext as y };
4493
5392
 
4494
- //# sourceMappingURL=memory-CIYRB_Q8.js.map
5393
+ //# sourceMappingURL=memory-DAFfGTJH.js.map