@tangle-network/agent-knowledge 4.0.0 → 4.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. package/AGENTS.md +32 -18
  2. package/CHANGELOG.md +32 -0
  3. package/README.md +31 -19
  4. package/dist/benchmarks/index.d.ts +51 -3
  5. package/dist/benchmarks/index.js +1 -1
  6. package/dist/{chunk-EUAXSBMH.js → chunk-AKYJG2MR.js} +224 -224
  7. package/dist/chunk-AKYJG2MR.js.map +1 -0
  8. package/dist/{chunk-DW6APRTX.js → chunk-BBCIB4UL.js} +2486 -2457
  9. package/dist/chunk-BBCIB4UL.js.map +1 -0
  10. package/dist/{chunk-UWOTQNBI.js → chunk-CPMLJYA3.js} +1886 -1842
  11. package/dist/chunk-CPMLJYA3.js.map +1 -0
  12. package/dist/{chunk-WCYW2GDA.js → chunk-MYFM6LKH.js} +2 -2
  13. package/dist/chunk-MYFM6LKH.js.map +1 -0
  14. package/dist/cli.js +1 -1
  15. package/dist/index.d.ts +48 -52
  16. package/dist/index.js +2906 -2874
  17. package/dist/index.js.map +1 -1
  18. package/dist/memory/index.d.ts +129 -6
  19. package/dist/memory/index.js +2 -2
  20. package/dist/sources/index.d.ts +17 -33
  21. package/dist/sources/index.js +1 -1
  22. package/dist/{index-Bqg1mBPt.d.ts → types-DP38encz.d.ts} +127 -284
  23. package/docs/eval/investment-material-facts.md +28 -29
  24. package/docs/eval/rag-eval-roadmap.md +6 -5
  25. package/docs/results/adaptive.md +13 -13
  26. package/docs/results/claim-grounding.md +11 -11
  27. package/docs/results/cost-quality.md +4 -4
  28. package/docs/results/investment-thesis.md +56 -56
  29. package/docs/results/research-driving.md +54 -54
  30. package/docs/two-agent-research-ab.md +94 -94
  31. package/package.json +4 -7
  32. package/dist/chunk-DW6APRTX.js.map +0 -1
  33. package/dist/chunk-EUAXSBMH.js.map +0 -1
  34. package/dist/chunk-UWOTQNBI.js.map +0 -1
  35. package/dist/chunk-WCYW2GDA.js.map +0 -1
@@ -18,7 +18,7 @@ import {
18
18
  runBoundedMemoryLifecycle,
19
19
  scoreMemoryBenchmarkArtifact,
20
20
  sleepForMemoryRecovery
21
- } from "./chunk-DW6APRTX.js";
21
+ } from "./chunk-BBCIB4UL.js";
22
22
  import {
23
23
  sha256,
24
24
  stableId
@@ -964,23 +964,127 @@ function cloneDurableValue(value) {
964
964
  );
965
965
  }
966
966
 
967
- // src/memory/experiment.ts
968
- import { randomUUID as randomUUID2 } from "crypto";
969
- import { join } from "path";
970
- import { canonicalJson as canonicalJson3 } from "@tangle-network/agent-eval";
971
- import {
972
- createRunCostLedger,
973
- fsCampaignStorage,
974
- resolveRunDir,
975
- runCampaign
976
- } from "@tangle-network/agent-eval/campaign";
977
- var MEMORY_EXPERIMENT_IMPLEMENTATION_REF = "agent-knowledge:memory-experiment:v6";
978
- var AgentMemoryCleanupError = class extends AggregateError {
979
- constructor(errors, message) {
980
- super(errors, message);
981
- this.name = "AgentMemoryCleanupError";
967
+ // src/memory/experiment/validation.ts
968
+ function assertMemorySequences(sequences) {
969
+ for (const sequence of sequences) {
970
+ assertNonEmptyString(sequence.family, `memory experiment sequence ${sequence.id} family`);
971
+ if (sequence.steps.length === 0) {
972
+ throw new Error(`memory experiment sequence ${sequence.id} has no steps`);
973
+ }
974
+ assertUnique(
975
+ sequence.steps.map((step) => step.id),
976
+ `step in sequence ${sequence.id}`
977
+ );
978
+ let probeCount = 0;
979
+ for (const step of sequence.steps) {
980
+ for (const write of step.writes ?? []) {
981
+ assertNonEmptyString(write.text, `memory experiment write in ${sequence.id}/${step.id}`);
982
+ if (write.id !== void 0) {
983
+ assertNonEmptyString(write.id, `memory experiment write id in ${sequence.id}/${step.id}`);
984
+ }
985
+ }
986
+ assertUnique(
987
+ (step.probes ?? []).map((probe) => probe.id),
988
+ `probe in sequence ${sequence.id} step ${step.id}`
989
+ );
990
+ for (const probe of step.probes ?? []) {
991
+ probeCount += 1;
992
+ assertNonEmptyString(
993
+ probe.query,
994
+ `memory experiment probe query ${sequence.id}/${step.id}/${probe.id}`
995
+ );
996
+ if (probe.limit !== void 0 && (!Number.isSafeInteger(probe.limit) || probe.limit <= 0)) {
997
+ throw new Error(
998
+ `memory experiment probe limit ${sequence.id}/${step.id}/${probe.id} must be a positive safe integer`
999
+ );
1000
+ }
1001
+ assertMemoryFactMatchers(
1002
+ probe.requiredFacts ?? [],
1003
+ `${sequence.id}/${step.id}/${probe.id} requiredFacts`
1004
+ );
1005
+ assertMemoryFactMatchers(
1006
+ probe.forbiddenFacts ?? [],
1007
+ `${sequence.id}/${step.id}/${probe.id} forbiddenFacts`
1008
+ );
1009
+ assertStringList(probe.expectedEventIds, `${sequence.id}/${step.id}/${probe.id} event ids`);
1010
+ assertStringList(probe.expectedActorIds, `${sequence.id}/${step.id}/${probe.id} actor ids`);
1011
+ if (probe.referenceAnswer !== void 0) {
1012
+ assertNonEmptyString(
1013
+ probe.referenceAnswer,
1014
+ `memory experiment reference answer ${sequence.id}/${step.id}/${probe.id}`
1015
+ );
1016
+ }
1017
+ const hasTarget = (probe.requiredFacts?.length ?? 0) > 0 || (probe.forbiddenFacts?.length ?? 0) > 0 || (probe.expectedEventIds?.length ?? 0) > 0 || (probe.expectedActorIds?.length ?? 0) > 0 || Boolean(probe.referenceAnswer?.trim());
1018
+ if (!hasTarget) {
1019
+ throw new Error(
1020
+ `memory experiment probe ${sequence.id}/${step.id}/${probe.id} has no measurable target`
1021
+ );
1022
+ }
1023
+ }
1024
+ }
1025
+ if (probeCount === 0) {
1026
+ throw new Error(`memory experiment sequence ${sequence.id} has no probes`);
1027
+ }
982
1028
  }
983
- };
1029
+ }
1030
+ function normalizeCleanupScope(scope) {
1031
+ const normalized = compactScope(scope);
1032
+ if (normalized.tags && Object.keys(normalized.tags).length === 0) {
1033
+ delete normalized.tags;
1034
+ }
1035
+ return normalized;
1036
+ }
1037
+ function compactScope(scope) {
1038
+ return Object.fromEntries(
1039
+ Object.entries(scope).filter(([, value]) => value !== void 0)
1040
+ );
1041
+ }
1042
+ function compactRecord(record2) {
1043
+ return Object.fromEntries(Object.entries(record2).filter(([, value]) => value !== void 0));
1044
+ }
1045
+ function uniqueStrings(values) {
1046
+ return [...new Set(values.filter((value) => typeof value === "string"))];
1047
+ }
1048
+ function stringArray(value) {
1049
+ return Array.isArray(value) ? value.filter((entry) => typeof entry === "string") : [];
1050
+ }
1051
+ function assertUnique(values, label) {
1052
+ const seen = /* @__PURE__ */ new Set();
1053
+ for (const value of values) {
1054
+ assertNonEmptyString(value, `memory experiment ${label} id`);
1055
+ if (seen.has(value)) throw new Error(`duplicate memory experiment ${label} id: ${value}`);
1056
+ seen.add(value);
1057
+ }
1058
+ }
1059
+ function assertMemoryFactMatchers(matchers, label) {
1060
+ assertUnique(
1061
+ matchers.map((matcher) => matcher.id),
1062
+ `${label} matcher`
1063
+ );
1064
+ for (const matcher of matchers) {
1065
+ if (matcher.anyOf.length === 0) {
1066
+ throw new Error(`memory experiment ${label} matcher ${matcher.id} requires anyOf`);
1067
+ }
1068
+ assertStringList(matcher.anyOf, `${label} matcher ${matcher.id} anyOf`);
1069
+ assertStringList(matcher.sourceEventIds, `${label} matcher ${matcher.id} source event ids`);
1070
+ if (matcher.weight !== void 0 && (!Number.isFinite(matcher.weight) || matcher.weight <= 0)) {
1071
+ throw new Error(
1072
+ `memory experiment ${label} matcher ${matcher.id} weight must be a positive finite number`
1073
+ );
1074
+ }
1075
+ }
1076
+ }
1077
+ function assertStringList(values, label) {
1078
+ if (values === void 0) return;
1079
+ assertUnique(values, label);
1080
+ }
1081
+ function assertNonEmptyString(value, label) {
1082
+ if (typeof value !== "string" || value.trim().length === 0) {
1083
+ throw new Error(`${label} must be a non-empty string`);
1084
+ }
1085
+ }
1086
+
1087
+ // src/memory/experiment/cases.ts
984
1088
  function buildAgentMemorySequencesFromBenchmarkCases(cases, options = {}) {
985
1089
  const memoryAgentId = options.memoryAgentId ?? "benchmark-agent";
986
1090
  return cases.map((testCase) => ({
@@ -1042,646 +1146,292 @@ function buildAgentMemorySequencesFromBenchmarkCases(cases, options = {}) {
1042
1146
  ]
1043
1147
  }));
1044
1148
  }
1045
- async function runAgentMemoryExperiment(options) {
1046
- assertNonEmptyString(options.experimentId, "memory experiment experimentId");
1047
- assertNonEmptyString(options.runDir, "memory experiment runDir");
1048
- if (options.experimentRunId !== void 0) {
1049
- assertNonEmptyString(options.experimentRunId, "memory experiment experimentRunId");
1050
- }
1051
- if (options.sequences.length === 0) throw new Error("memory experiment requires sequences");
1052
- if (options.candidates.length === 0) throw new Error("memory experiment requires candidates");
1053
- if (options.executeStep && !options.executeStepRef) {
1054
- throw new Error("memory experiment executeStepRef is required when executeStep is configured");
1055
- }
1056
- assertUnique(
1057
- options.sequences.map((sequence) => sequence.id),
1058
- "sequence"
1059
- );
1060
- assertUnique(
1061
- [...options.candidates, ...options.recoveryCandidates ?? []].map((candidate) => candidate.id),
1062
- "candidate"
1149
+ function buildAgentMemorySequenceScenarios(sequences, candidates) {
1150
+ return candidates.flatMap(
1151
+ (candidate) => sequences.map((sequence) => ({
1152
+ id: `${stableId("candidate", candidate.id)}:${sequence.id}`,
1153
+ kind: "agent-memory-sequence",
1154
+ candidateId: candidate.id,
1155
+ sequenceId: sequence.id,
1156
+ sequence,
1157
+ seedGroup: sequence.id,
1158
+ tags: [.../* @__PURE__ */ new Set([sequence.split ?? "dev", ...sequence.tags ?? [], candidate.id])]
1159
+ }))
1063
1160
  );
1064
- assertMemorySequences(options.sequences);
1065
- for (const candidate of [...options.candidates, ...options.recoveryCandidates ?? []]) {
1066
- if (options.cleanupBranches === false && !candidate.disposeAdapter) {
1067
- throw new Error(
1068
- `${candidate.id}: cleanupBranches=false requires disposeAdapter to delete isolated external state`
1069
- );
1070
- }
1071
- if (candidate.externalCostUsdPerSequence !== void 0 && (!Number.isFinite(candidate.externalCostUsdPerSequence) || candidate.externalCostUsdPerSequence < 0)) {
1072
- throw new Error(
1073
- `${candidate.id}: externalCostUsdPerSequence must be a non-negative finite number`
1074
- );
1075
- }
1076
- if (candidate.externalRecoveryCostUsdPerAttempt !== void 0 && (!Number.isFinite(candidate.externalRecoveryCostUsdPerAttempt) || candidate.externalRecoveryCostUsdPerAttempt < 0)) {
1077
- throw new Error(
1078
- `${candidate.id}: externalRecoveryCostUsdPerAttempt must be a non-negative finite number`
1079
- );
1161
+ }
1162
+ function agentMemorySequenceJudge() {
1163
+ return {
1164
+ name: "agent-memory-sequence",
1165
+ judgeVersion: "agent-knowledge:memory-sequence:v2",
1166
+ dimensions: [
1167
+ { key: "score", description: "mean memory probe score" },
1168
+ { key: "passed", description: "1 when every memory probe passes" },
1169
+ { key: "memory_fact_recall", description: "current memory fact coverage" },
1170
+ { key: "memory_event_recall", description: "memory source event coverage" },
1171
+ { key: "memory_actor_recall", description: "memory actor attribution coverage" },
1172
+ { key: "memory_stale_safe", description: "1 when obsolete memory is not reused" }
1173
+ ],
1174
+ score({ artifact }) {
1175
+ return {
1176
+ composite: artifact.score,
1177
+ dimensions: {
1178
+ score: artifact.score,
1179
+ passed: artifact.passed ? 1 : 0,
1180
+ ...artifact.dimensions
1181
+ },
1182
+ notes: `${artifact.probes.filter((probe) => probe.passed).length}/${artifact.probes.length} probes passed`
1183
+ };
1080
1184
  }
1081
- assertNonEmptyString(candidate.ref, `${candidate.id} ref`);
1082
- }
1083
- const storage = options.storage ?? fsCampaignStorage();
1084
- const runDir = resolveRunDir(options.runDir, options.repo);
1085
- if (!storage.append) {
1086
- throw new Error("memory experiment requires CampaignStorage.append for durable attempt state");
1087
- }
1088
- resolveMemoryCleanupTimeoutMs(options.cleanupTimeoutMs, "memory experiment");
1089
- const maxRecoveryAttempts = options.maxRecoveryAttempts ?? 1e3;
1090
- if (!Number.isSafeInteger(maxRecoveryAttempts) || maxRecoveryAttempts <= 0) {
1091
- throw new Error("memory experiment maxRecoveryAttempts must be a positive safe integer");
1092
- }
1093
- const maxRecoveryRetriesPerAttempt = options.maxRecoveryRetriesPerAttempt ?? DEFAULT_MEMORY_RECOVERY_RETRIES_PER_ATTEMPT;
1094
- if (!Number.isSafeInteger(maxRecoveryRetriesPerAttempt) || maxRecoveryRetriesPerAttempt <= 0) {
1095
- throw new Error(
1096
- "memory experiment maxRecoveryRetriesPerAttempt must be a positive safe integer"
1097
- );
1098
- }
1099
- storage.ensureDir(runDir);
1100
- const lease = await acquireAgentMemoryRunLease({
1101
- experimentId: options.experimentId,
1102
- runDir,
1103
- storage,
1104
- customStorage: options.storage !== void 0,
1105
- lockFileName: "memory-experiment.lock",
1106
- label: "memory experiment",
1107
- controllerMode: options.controllerMode,
1108
- acquireRunLease: options.acquireRunLease
1109
- });
1110
- let result;
1111
- let primaryError;
1112
- try {
1113
- await lease.assertOwned();
1114
- result = await runOwnedAgentMemoryExperiment(options, storage, runDir, lease);
1115
- } catch (error) {
1116
- primaryError = error;
1117
- }
1118
- let releaseError;
1119
- try {
1120
- await lease.release();
1121
- } catch (error) {
1122
- releaseError = error;
1123
- }
1124
- if (primaryError && releaseError) {
1125
- throw new AggregateError(
1126
- [primaryError, releaseError],
1127
- "memory experiment failed and its controller lease could not be released"
1128
- );
1129
- }
1130
- if (primaryError) throw primaryError;
1131
- if (releaseError) throw releaseError;
1132
- if (!result) throw new Error("memory experiment produced no result");
1133
- return result;
1185
+ };
1134
1186
  }
1135
- async function runOwnedAgentMemoryExperiment(options, storage, runDir, lease) {
1136
- const runIdentity = stableId(
1137
- "memory_run",
1138
- canonicalJson3({
1139
- experimentId: options.experimentId,
1140
- experimentRunId: options.experimentRunId ?? runDir
1141
- })
1142
- );
1143
- const maxConcurrency = options.maxConcurrency ?? 2;
1144
- if (!Number.isSafeInteger(maxConcurrency) || maxConcurrency <= 0) {
1145
- throw new Error("memory experiment maxConcurrency must be a positive safe integer");
1146
- }
1147
- const executionPool = createMemoryExecutionPool(maxConcurrency);
1148
- const dispatchedExecutions = [];
1149
- const candidateById = new Map(options.candidates.map((candidate) => [candidate.id, candidate]));
1150
- const recoveryCandidateById = new Map(
1151
- [...options.candidates, ...options.recoveryCandidates ?? []].map((candidate) => [
1152
- candidate.id,
1153
- candidate
1154
- ])
1155
- );
1156
- const sequenceById = new Map(options.sequences.map((sequence) => [sequence.id, sequence]));
1157
- const scenarios = buildAgentMemorySequenceScenarios(options.sequences, options.candidates);
1158
- const attemptLogPath = join(runDir, "memory-attempts.jsonl");
1159
- const recoveryLogPath = join(runDir, "memory-recovery-attempts.jsonl");
1160
- const costCeiling = options.costCeiling ?? options.costLedger?.costCeilingUsd ?? 0;
1161
- const costLedger = options.costLedger ?? createRunCostLedger({
1162
- storage,
1163
- runDir,
1164
- costCeilingUsd: costCeiling
1165
- });
1166
- if (costLedger.costCeilingUsd !== costCeiling) {
1167
- throw new Error("memory experiment costCeiling must match the shared cost ledger ceiling");
1168
- }
1169
- storage.ensureDir(runDir);
1170
- await recoverAbandonedMemoryAttempts({
1171
- options,
1172
- storage,
1173
- runDir,
1174
- attemptLogPath,
1175
- candidateById: recoveryCandidateById,
1176
- sequenceById,
1177
- lease,
1178
- maxConcurrency,
1179
- costLedger,
1180
- maxRecoveryAttempts: options.maxRecoveryAttempts ?? 1e3,
1181
- recoveryLogPath,
1182
- maxRecoveryRetriesPerAttempt: options.maxRecoveryRetriesPerAttempt ?? DEFAULT_MEMORY_RECOVERY_RETRIES_PER_ATTEMPT
1187
+
1188
+ // src/memory/experiment/run.ts
1189
+ import { join } from "path";
1190
+ import { canonicalJson as canonicalJson4 } from "@tangle-network/agent-eval";
1191
+ import {
1192
+ createRunCostLedger,
1193
+ fsCampaignStorage,
1194
+ resolveRunDir,
1195
+ runCampaign
1196
+ } from "@tangle-network/agent-eval/campaign";
1197
+
1198
+ // src/memory/experiment/cell.ts
1199
+ import { randomUUID as randomUUID2 } from "crypto";
1200
+
1201
+ // src/memory/experiment/metrics.ts
1202
+ function rankAgentMemoryExperiment(candidates, scenarios, campaign, costByCandidate) {
1203
+ const scenarioById = new Map(scenarios.map((scenario) => [scenario.id, scenario]));
1204
+ const rows = candidates.map((candidate) => {
1205
+ const candidateScenarios = scenarios.filter((scenario) => scenario.candidateId === candidate.id);
1206
+ const cells = campaign.cells.filter(
1207
+ (cell) => scenarioById.get(cell.scenarioId)?.candidateId === candidate.id
1208
+ );
1209
+ const successful = cells.filter((cell) => !cell.error && cell.artifact);
1210
+ const dimensionRows = successful.map((cell) => cell.artifact.dimensions);
1211
+ return {
1212
+ rank: 0,
1213
+ candidateId: candidate.id,
1214
+ label: candidate.label ?? candidate.id,
1215
+ scoreMean: mean(
1216
+ cells.map((cell) => !cell.error && cell.artifact ? cell.artifact.score : 0)
1217
+ ),
1218
+ passRate: mean(cells.map((cell) => !cell.error && cell.artifact?.passed ? 1 : 0)),
1219
+ totalSequences: candidateScenarios.length,
1220
+ totalCells: cells.length,
1221
+ totalProbes: successful.reduce((sum, cell) => sum + cell.artifact.probes.length, 0),
1222
+ cellsFailed: cells.filter((cell) => Boolean(cell.error)).length,
1223
+ totalCostUsd: normalizeUsd(costByCandidate.get(candidate.id) ?? 0),
1224
+ durationMs: cells.reduce((sum, cell) => sum + cell.durationMs, 0),
1225
+ dimensions: meanDimensions(dimensionRows)
1226
+ };
1183
1227
  });
1184
- await lease.assertOwned();
1185
- let campaign;
1186
- let campaignError;
1187
- let settledExecutions = [];
1188
- try {
1189
- campaign = await runCampaign({
1190
- scenarios,
1191
- dispatch: (scenario, context) => {
1192
- const candidate = candidateById.get(scenario.candidateId);
1193
- if (!candidate) throw new Error(`unknown memory candidate ${scenario.candidateId}`);
1194
- const operation = executionPool.run(
1195
- () => runSequenceCell({
1196
- options,
1197
- candidate,
1198
- scenario,
1199
- context,
1200
- runIdentity,
1201
- storage,
1202
- attemptLogPath,
1203
- lease
1204
- })
1205
- );
1206
- dispatchedExecutions.push(operation);
1207
- return operation;
1208
- },
1209
- dispatchRef: memoryExperimentDispatchRef(options),
1210
- judges: [agentMemorySequenceJudge()],
1211
- runDir,
1212
- storage,
1213
- seed: options.seed,
1214
- reps: options.reps,
1215
- resumable: options.resumable,
1216
- costCeiling,
1217
- costLedger,
1218
- costPhase: options.costPhase,
1219
- maxConcurrency,
1220
- dispatchTimeoutMs: options.dispatchTimeoutMs,
1221
- expectUsage: "off",
1222
- now: options.now
1223
- });
1224
- } catch (error) {
1225
- campaignError = error;
1226
- } finally {
1227
- settledExecutions = await Promise.allSettled(dispatchedExecutions);
1228
- }
1229
- const cleanupFailures = settledExecutions.flatMap(
1230
- (settled) => settled.status === "rejected" && settled.reason instanceof AgentMemoryCleanupError ? [settled.reason] : []
1228
+ return rows.sort(
1229
+ (a, b) => Number(a.cellsFailed > 0) - Number(b.cellsFailed > 0) || b.scoreMean - a.scoreMean || b.passRate - a.passRate || a.totalCostUsd - b.totalCostUsd || a.candidateId.localeCompare(b.candidateId)
1230
+ ).map((row, index) => ({ ...row, rank: index + 1 }));
1231
+ }
1232
+ function memoryExperimentCostByCandidate(costLedger, runDir, scenarios, candidateIdsInput) {
1233
+ const candidateByScenario = new Map(
1234
+ scenarios.map((scenario) => [scenario.id, scenario.candidateId])
1231
1235
  );
1232
- if (campaignError && cleanupFailures.length > 0) {
1233
- throw new AggregateError(
1234
- [campaignError, ...cleanupFailures],
1235
- "memory experiment failed and provider cleanup also failed"
1236
- );
1236
+ const candidateIds = new Set(candidateIdsInput);
1237
+ const totals = /* @__PURE__ */ new Map();
1238
+ for (const receipt of costLedger.list()) {
1239
+ if (receipt.tags?.runDir !== runDir) continue;
1240
+ const scenarioCandidate = receipt.tags.scenarioId ? candidateByScenario.get(receipt.tags.scenarioId) : void 0;
1241
+ const recoveryCandidate = receipt.tags.memoryRecovery === "attempt" && receipt.tags.candidateId ? receipt.tags.candidateId : void 0;
1242
+ const candidateId = scenarioCandidate ?? recoveryCandidate;
1243
+ if (!candidateId || !candidateIds.has(candidateId)) continue;
1244
+ totals.set(candidateId, (totals.get(candidateId) ?? 0) + receipt.costUsd);
1237
1245
  }
1238
- if (campaignError) throw campaignError;
1239
- if (cleanupFailures.length > 0) {
1240
- throw new AggregateError(cleanupFailures, "memory experiment cleanup failed after dispatch");
1246
+ return totals;
1247
+ }
1248
+ function renderAgentMemoryExperimentRanking(rows, totalCostUsd, unrankedRecoveryCostUsd) {
1249
+ return [
1250
+ "# Agent Memory Experiment",
1251
+ "",
1252
+ `- total cost: $${format(totalCostUsd)}`,
1253
+ `- retired-candidate recovery cost: $${format(unrankedRecoveryCostUsd)}`,
1254
+ "",
1255
+ "| rank | candidate | sequences | cells | probes | failed | score | pass rate | cost | duration ms |",
1256
+ "| ---: | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |",
1257
+ ...rows.map(
1258
+ (row) => `| ${row.rank} | ${row.label} | ${row.totalSequences} | ${row.totalCells} | ${row.totalProbes} | ${row.cellsFailed} | ${format(row.scoreMean)} | ${format(row.passRate)} | $${format(row.totalCostUsd)} | ${format(row.durationMs)} |`
1259
+ ),
1260
+ ""
1261
+ ].join("\n");
1262
+ }
1263
+ function meanDimensions(rows) {
1264
+ const values = /* @__PURE__ */ new Map();
1265
+ for (const row of rows) {
1266
+ for (const [key, value] of Object.entries(row)) {
1267
+ if (!Number.isFinite(value)) continue;
1268
+ const bucket = values.get(key) ?? [];
1269
+ bucket.push(value);
1270
+ values.set(key, bucket);
1271
+ }
1241
1272
  }
1242
- if (!campaign) throw new Error("memory experiment produced no campaign result");
1243
- await lease.assertOwned();
1244
- const costByCandidate = memoryExperimentCostByCandidate(
1245
- costLedger,
1246
- campaign.runDir,
1247
- scenarios,
1248
- [...options.candidates, ...options.recoveryCandidates ?? []].map((candidate) => candidate.id)
1249
- );
1250
- const unrankedRecoveryCostUsd = normalizeUsd(
1251
- (options.recoveryCandidates ?? []).reduce(
1252
- (sum, candidate) => sum + (costByCandidate.get(candidate.id) ?? 0),
1253
- 0
1254
- )
1255
- );
1256
- const totalCostUsd = normalizeUsd(
1257
- [...costByCandidate.values()].reduce((sum, cost) => sum + cost, 0)
1258
- );
1259
- const rows = rankAgentMemoryExperiment(options.candidates, scenarios, campaign, costByCandidate);
1260
- const rankingJsonPath = join(campaign.runDir, "memory-experiment-ranking.json");
1261
- const rankingMarkdownPath = join(campaign.runDir, "memory-experiment-ranking.md");
1262
- storage.write(
1263
- rankingJsonPath,
1264
- `${JSON.stringify(
1265
- { experimentId: options.experimentId, totalCostUsd, unrankedRecoveryCostUsd, rows },
1266
- null,
1267
- 2
1268
- )}
1269
- `
1270
- );
1271
- storage.write(
1272
- rankingMarkdownPath,
1273
- renderAgentMemoryExperimentRanking(rows, totalCostUsd, unrankedRecoveryCostUsd)
1274
- );
1275
- return {
1276
- campaign,
1277
- rows,
1278
- totalCostUsd,
1279
- unrankedRecoveryCostUsd,
1280
- leaderCandidateId: rows.find((row) => row.cellsFailed === 0)?.candidateId,
1281
- rankingJsonPath,
1282
- rankingMarkdownPath,
1283
- attemptLogPath,
1284
- recoveryLogPath
1285
- };
1273
+ return Object.fromEntries([...values].map(([key, bucket]) => [key, mean(bucket)]));
1286
1274
  }
1287
- function buildAgentMemorySequenceScenarios(sequences, candidates) {
1288
- return candidates.flatMap(
1289
- (candidate) => sequences.map((sequence) => ({
1290
- id: `${stableId("candidate", candidate.id)}:${sequence.id}`,
1291
- kind: "agent-memory-sequence",
1292
- candidateId: candidate.id,
1293
- sequenceId: sequence.id,
1294
- sequence,
1295
- seedGroup: sequence.id,
1296
- tags: [.../* @__PURE__ */ new Set([sequence.split ?? "dev", ...sequence.tags ?? [], candidate.id])]
1297
- }))
1298
- );
1275
+ function countDimensions(rows) {
1276
+ const counts = /* @__PURE__ */ new Map();
1277
+ for (const row of rows) {
1278
+ for (const key of new Set(row)) counts.set(key, (counts.get(key) ?? 0) + 1);
1279
+ }
1280
+ return Object.fromEntries(counts);
1299
1281
  }
1300
- function agentMemorySequenceJudge() {
1301
- return {
1302
- name: "agent-memory-sequence",
1303
- judgeVersion: "agent-knowledge:memory-sequence:v2",
1304
- dimensions: [
1305
- { key: "score", description: "mean memory probe score" },
1306
- { key: "passed", description: "1 when every memory probe passes" },
1307
- { key: "memory_fact_recall", description: "current memory fact coverage" },
1308
- { key: "memory_event_recall", description: "memory source event coverage" },
1309
- { key: "memory_actor_recall", description: "memory actor attribution coverage" },
1310
- { key: "memory_stale_safe", description: "1 when obsolete memory is not reused" }
1311
- ],
1312
- score({ artifact }) {
1313
- return {
1314
- composite: artifact.score,
1315
- dimensions: {
1316
- score: artifact.score,
1317
- passed: artifact.passed ? 1 : 0,
1318
- ...artifact.dimensions
1319
- },
1320
- notes: `${artifact.probes.filter((probe) => probe.passed).length}/${artifact.probes.length} probes passed`
1321
- };
1322
- }
1282
+ function mean(values) {
1283
+ return values.length === 0 ? 0 : values.reduce((sum, value) => sum + value, 0) / values.length;
1284
+ }
1285
+ function format(value) {
1286
+ return Number.isFinite(value) ? value.toFixed(4) : "0.0000";
1287
+ }
1288
+ function normalizeUsd(value) {
1289
+ return Number(value.toFixed(12));
1290
+ }
1291
+
1292
+ // src/memory/experiment/recovery.ts
1293
+ import { canonicalJson as canonicalJson3 } from "@tangle-network/agent-eval";
1294
+
1295
+ // src/memory/experiment/runtime.ts
1296
+ var AgentMemoryCleanupError = class extends AggregateError {
1297
+ constructor(errors, message) {
1298
+ super(errors, message);
1299
+ this.name = "AgentMemoryCleanupError";
1300
+ }
1301
+ };
1302
+ function trackExternalMemoryCalls(adapter, onExternalCall) {
1303
+ return {
1304
+ id: adapter.id,
1305
+ branchIsolation: adapter.branchIsolation,
1306
+ search(query, options) {
1307
+ onExternalCall();
1308
+ return adapter.search.call(adapter, query, options);
1309
+ },
1310
+ getContext(query, options) {
1311
+ onExternalCall();
1312
+ return adapter.getContext.call(adapter, query, options);
1313
+ },
1314
+ write(input) {
1315
+ onExternalCall();
1316
+ return adapter.write.call(adapter, input);
1317
+ },
1318
+ ...adapter.clear ? {
1319
+ clear(scope) {
1320
+ onExternalCall();
1321
+ return adapter.clear.call(adapter, scope);
1322
+ }
1323
+ } : {},
1324
+ ...adapter.flush ? {
1325
+ flush() {
1326
+ onExternalCall();
1327
+ return adapter.flush.call(adapter);
1328
+ }
1329
+ } : {},
1330
+ ...adapter.close ? {
1331
+ close() {
1332
+ onExternalCall();
1333
+ return adapter.close.call(adapter);
1334
+ }
1335
+ } : {}
1323
1336
  };
1324
1337
  }
1325
- async function runSequenceCell(input) {
1326
- const { options, candidate, scenario, context, runIdentity, storage, attemptLogPath, lease } = input;
1327
- const cleanupBranches = options.cleanupBranches ?? true;
1328
- const costUsd = candidate.externalCostUsdPerSequence ?? 0;
1329
- if (!Number.isFinite(costUsd) || costUsd < 0) {
1330
- throw new Error(
1331
- `${candidate.id}: externalCostUsdPerSequence must be a non-negative finite number`
1332
- );
1338
+ function mergeScopes2(base, extra) {
1339
+ return {
1340
+ ...base ?? {},
1341
+ ...extra ?? {},
1342
+ tags: { ...base?.tags ?? {}, ...extra?.tags ?? {} }
1343
+ };
1344
+ }
1345
+ function memoryExperimentBaseScope(options, candidate, sequenceId) {
1346
+ return mergeScopes2(candidate.baseScope, {
1347
+ tags: {
1348
+ memoryExperimentId: options.experimentId,
1349
+ memoryCandidateId: candidate.id,
1350
+ memorySequenceId: sequenceId
1351
+ }
1352
+ });
1353
+ }
1354
+ function sequenceCleanupScopes(sequence) {
1355
+ const scopes = /* @__PURE__ */ new Map();
1356
+ for (const scope of sequence.cleanupScopes ?? []) {
1357
+ const normalized = normalizeCleanupScope(scope);
1358
+ scopes.set(JSON.stringify(normalized), normalized);
1333
1359
  }
1334
- if (!cleanupBranches && !candidate.disposeAdapter) {
1360
+ for (const step of sequence.steps) {
1361
+ const candidates = [
1362
+ ...step.scope ? [step.scope] : [],
1363
+ ...(step.writes ?? []).map((write) => mergeScopes2(step.scope, write.scope)),
1364
+ ...(step.probes ?? []).map((probe) => mergeScopes2(step.scope, probe.scope))
1365
+ ];
1366
+ for (const scope of candidates) {
1367
+ const normalized = normalizeCleanupScope(scope);
1368
+ scopes.set(JSON.stringify(normalized), normalized);
1369
+ }
1370
+ }
1371
+ return [...scopes.values()];
1372
+ }
1373
+ async function clearSequenceScopes(memory, sequence) {
1374
+ for (const scope of sequenceCleanupScopes(sequence)) await memory.clear?.(scope);
1375
+ }
1376
+
1377
+ // src/memory/experiment/recovery.ts
1378
+ async function recoverAbandonedMemoryAttempts(input) {
1379
+ let attempts = readActiveMemoryAttempts(input.storage, input.attemptLogPath);
1380
+ if (attempts.length > input.maxRecoveryAttempts) {
1335
1381
  throw new Error(
1336
- `${candidate.id}: cleanupBranches=false requires disposeAdapter to delete isolated external state`
1382
+ `memory experiment has ${attempts.length} unfinished attempts; maxRecoveryAttempts is ${input.maxRecoveryAttempts}`
1337
1383
  );
1338
1384
  }
1339
- const cleanupTimeoutMs = resolveMemoryCleanupTimeoutMs(
1340
- options.cleanupTimeoutMs,
1341
- `${candidate.id}: memory sequence`
1342
- );
1343
- context.signal.throwIfAborted();
1344
- await lease.assertOwned();
1345
- const startedAt = Date.now();
1346
- const branchId = stableId(
1347
- "memory_branch",
1348
- `${runIdentity}:${context.cellId}:${context.seed}:${randomUUID2()}`
1349
- );
1350
- const attempt = memoryAttemptEvent({
1351
- status: "started",
1352
- branchId,
1353
- candidate,
1354
- sequence: scenario.sequence,
1355
- rep: context.rep,
1356
- seed: context.seed,
1357
- recovery: false,
1358
- cleanupBranches,
1359
- now: options.now
1360
- });
1361
- appendMemoryAttemptEvent(storage, attemptLogPath, attempt);
1362
- let externalCallAttempted = false;
1363
- const appendCleanedAttempt = (priorError) => {
1364
- try {
1365
- appendMemoryAttemptEvent(storage, attemptLogPath, {
1385
+ for (const attempt of attempts) {
1386
+ const candidate = input.candidateById.get(attempt.candidateId);
1387
+ if (!candidate) {
1388
+ throw new Error(
1389
+ `cannot recover memory branch '${attempt.branchId}': candidate '${attempt.candidateId}' is missing; pass it in recoveryCandidates`
1390
+ );
1391
+ }
1392
+ assertMemoryAttemptCandidateMatches(attempt, candidate);
1393
+ if (!input.sequenceById.has(attempt.sequenceId)) {
1394
+ throw new Error(
1395
+ `cannot recover memory branch '${attempt.branchId}': sequence '${attempt.sequenceId}' is missing`
1396
+ );
1397
+ }
1398
+ if ((input.options.cleanupBranches ?? true) !== attempt.cleanupBranches) {
1399
+ throw new Error(`cannot recover memory branch '${attempt.branchId}': cleanupBranches changed`);
1400
+ }
1401
+ }
1402
+ reconcileInterruptedMemoryPaidCalls(input.costLedger);
1403
+ assertNoInterruptedPaidCalls(input.costLedger, "memory experiment recovery");
1404
+ for (const attempt of attempts) {
1405
+ const candidate = input.candidateById.get(attempt.candidateId);
1406
+ const executionCostUsd = candidate.externalCostUsdPerSequence ?? 0;
1407
+ if (executionCostUsd > 0 && !hasSettledPaidCall(input.costLedger, memoryAttemptCostCallId(attempt, "execute", 0))) {
1408
+ appendMemoryAttemptEvent(input.storage, input.attemptLogPath, {
1366
1409
  ...attempt,
1367
1410
  status: "cleaned",
1368
- recordedAt: (options.now ?? (() => /* @__PURE__ */ new Date()))().toISOString()
1411
+ recovery: true,
1412
+ recordedAt: (input.options.now ?? (() => /* @__PURE__ */ new Date()))().toISOString()
1369
1413
  });
1370
- } catch (error) {
1371
- throw new AgentMemoryCleanupError(
1372
- [...priorError ? [priorError] : [], error],
1373
- `${candidate.id}: memory branch cleanup could not be recorded`
1374
- );
1375
1414
  }
1376
- };
1377
- const execute = async () => {
1378
- context.signal.throwIfAborted();
1379
- await lease.assertOwned();
1380
- let rawAdapter;
1381
- let adapter;
1382
- let memory;
1383
- let primaryError;
1384
- let completedArtifact;
1385
- let finalClearStarted = false;
1386
- let finalClearCompleted = false;
1387
- try {
1388
- const created = await candidate.createAdapter({
1389
- branchId,
1390
- sequence: scenario.sequence,
1391
- rep: context.rep,
1392
- seed: context.seed,
1393
- purpose: "execute",
1394
- signal: context.signal,
1395
- markExternalCall: () => {
1396
- externalCallAttempted = true;
1397
- }
1398
- });
1399
- if (!created) throw new Error(`${candidate.id}: createAdapter returned no execution adapter`);
1400
- rawAdapter = created;
1401
- adapter = trackExternalMemoryCalls(created, () => {
1402
- externalCallAttempted = true;
1403
- });
1404
- await lease.assertOwned();
1405
- context.signal.throwIfAborted();
1406
- if (cleanupBranches && !adapter.clear) {
1407
- throw new Error(
1408
- `${candidate.id}: cleanupBranches requires an adapter with scoped clear support`
1409
- );
1410
- }
1411
- memory = createAgentMemoryBranch({
1412
- adapter,
1413
- branchId,
1414
- lifetime: "attempt",
1415
- policy: candidate.policy,
1416
- allowedWriteScopes: sequenceCleanupScopes(scenario.sequence),
1417
- baseScope: memoryExperimentBaseScope(options, candidate, scenario.sequenceId)
1418
- });
1419
- const probes = [];
1420
- for (const step of scenario.sequence.steps) {
1421
- context.signal.throwIfAborted();
1422
- await lease.assertOwned();
1423
- await writeStep(memory, step);
1424
- await lease.assertOwned();
1425
- await options.executeStep?.({
1426
- memory,
1427
- candidateId: candidate.id,
1428
- sequence: scenario.sequence,
1429
- step,
1430
- context
1431
- });
1432
- context.signal.throwIfAborted();
1433
- await lease.assertOwned();
1434
- const stepProbes = await probeStep(memory, scenario.sequence, step);
1435
- await lease.assertOwned();
1436
- probes.push(...stepProbes);
1437
- }
1438
- const snapshot = await memory.snapshot();
1439
- await lease.assertOwned();
1440
- await options.onBranchSnapshot?.({
1441
- candidateId: candidate.id,
1442
- sequenceId: scenario.sequenceId,
1443
- cellId: context.cellId,
1444
- snapshot
1445
- });
1446
- await lease.assertOwned();
1447
- const dimensions = meanDimensions(probes.map((probe) => probe.dimensions));
1448
- const dimensionSampleCounts = countDimensions(
1449
- probes.map((probe) => probe.applicableDimensions)
1450
- );
1451
- const artifact = {
1452
- candidateId: candidate.id,
1453
- sequenceId: scenario.sequenceId,
1454
- score: mean(probes.map((probe) => probe.score)),
1455
- passed: probes.length > 0 && probes.every((probe) => probe.passed),
1456
- dimensions,
1457
- dimensionSampleCounts,
1458
- probes,
1459
- branchDigest: snapshot.digest,
1460
- journalEntries: snapshot.journal.length,
1461
- durationMs: Math.max(0, Date.now() - startedAt)
1462
- };
1463
- if (cleanupBranches) {
1464
- finalClearStarted = true;
1465
- await runBoundedMemoryLifecycle({
1466
- operation: `${candidate.id}: final branch cleanup`,
1467
- timeoutMs: cleanupTimeoutMs,
1468
- resource: adapter,
1469
- run: () => clearSequenceScopes(memory, scenario.sequence)
1470
- });
1471
- finalClearCompleted = true;
1472
- await lease.assertOwned();
1473
- }
1474
- completedArtifact = artifact;
1475
- } catch (error) {
1476
- primaryError = error;
1477
- }
1478
- const cleanupErrors = [];
1479
- let cleanupOwned = true;
1480
- let ownershipError;
1481
- try {
1482
- await lease.assertOwned();
1483
- } catch (error) {
1484
- cleanupOwned = false;
1485
- ownershipError = error;
1486
- }
1487
- if (finalClearStarted && !finalClearCompleted && primaryError) {
1488
- cleanupErrors.push(primaryError);
1489
- }
1490
- if (primaryError && cleanupOwned && !finalClearStarted && memory && cleanupBranches && adapter?.clear) {
1491
- try {
1492
- await runBoundedMemoryLifecycle({
1493
- operation: `${candidate.id}: failed branch cleanup`,
1494
- timeoutMs: cleanupTimeoutMs,
1495
- resource: adapter,
1496
- run: () => clearSequenceScopes(memory, scenario.sequence)
1497
- });
1498
- } catch (error) {
1499
- cleanupErrors.push(error);
1500
- }
1501
- }
1502
- if (adapter) {
1503
- try {
1504
- await runBoundedMemoryLifecycle({
1505
- operation: `${candidate.id}: adapter close`,
1506
- timeoutMs: cleanupTimeoutMs,
1507
- resource: adapter,
1508
- run: async () => {
1509
- if (memory && cleanupOwned) await memory.close?.();
1510
- else {
1511
- if (cleanupOwned) await adapter.flush?.();
1512
- await adapter.close?.();
1513
- }
1514
- }
1515
- });
1516
- } catch (error) {
1517
- cleanupErrors.push(error);
1518
- }
1519
- if (cleanupOwned) {
1520
- try {
1521
- await runBoundedMemoryLifecycle({
1522
- operation: `${candidate.id}: adapter disposal`,
1523
- timeoutMs: cleanupTimeoutMs,
1524
- resource: adapter,
1525
- run: () => {
1526
- if (candidate.disposeAdapter) externalCallAttempted = true;
1527
- return candidate.disposeAdapter?.(rawAdapter);
1528
- }
1529
- });
1530
- } catch (error) {
1531
- cleanupErrors.push(error);
1532
- }
1533
- }
1534
- } else {
1535
- cleanupErrors.push(
1536
- new Error(`${candidate.id}: adapter creation failed before cleanup could be confirmed`)
1537
- );
1538
- }
1539
- if (cleanupOwned && cleanupErrors.length === 0) appendCleanedAttempt(primaryError);
1540
- if (!cleanupOwned && cleanupErrors.length === 0) {
1541
- if (primaryError) throw primaryError;
1542
- throw ownershipError;
1543
- }
1544
- if (cleanupErrors.length > 0) {
1545
- const primaryMessage = primaryError instanceof Error ? primaryError.message : String(primaryError ?? "");
1546
- throw new AgentMemoryCleanupError(
1547
- [
1548
- ...primaryError && !cleanupErrors.includes(primaryError) ? [primaryError] : [],
1549
- ...ownershipError ? [ownershipError] : [],
1550
- ...cleanupErrors
1551
- ],
1552
- `${candidate.id}: memory branch cleanup failed${primaryMessage ? ` after: ${primaryMessage}` : ""}`
1553
- );
1554
- }
1555
- if (primaryError) throw primaryError;
1556
- if (!completedArtifact) throw new Error(`${candidate.id}: memory sequence produced no result`);
1557
- return completedArtifact;
1558
- };
1559
- if (costUsd === 0) {
1560
- let artifact;
1561
- let error;
1562
- try {
1563
- artifact = await execute();
1564
- } catch (caught) {
1565
- error = caught;
1566
- }
1567
- if (error) throw error;
1568
- if (!artifact) throw new Error(`${candidate.id}: memory sequence produced no result`);
1569
- return artifact;
1570
- }
1571
- const receipt = {
1572
- model: candidate.id,
1573
- inputTokens: 0,
1574
- outputTokens: 0,
1575
- actualCostUsd: costUsd
1576
- };
1577
- const paid = await context.cost.runPaidCall({
1578
- callId: memoryAttemptCostCallId(attempt, "execute", 0),
1579
- actor: `agent-knowledge:memory-experiment:${candidate.id}`,
1580
- model: candidate.id,
1581
- maximumCharge: { externallyEnforcedMaximumUsd: costUsd },
1582
- execute,
1583
- receipt: () => receipt,
1584
- receiptFromError: () => ({
1585
- ...receipt,
1586
- actualCostUsd: externalCallAttempted ? costUsd : 0
1587
- })
1588
- });
1589
- if (!paid.succeeded) throw paid.error;
1590
- return paid.value;
1591
- }
1592
- function trackExternalMemoryCalls(adapter, onExternalCall) {
1593
- return {
1594
- id: adapter.id,
1595
- branchIsolation: adapter.branchIsolation,
1596
- search(query, options) {
1597
- onExternalCall();
1598
- return adapter.search.call(adapter, query, options);
1599
- },
1600
- getContext(query, options) {
1601
- onExternalCall();
1602
- return adapter.getContext.call(adapter, query, options);
1603
- },
1604
- write(input) {
1605
- onExternalCall();
1606
- return adapter.write.call(adapter, input);
1607
- },
1608
- ...adapter.clear ? {
1609
- clear(scope) {
1610
- onExternalCall();
1611
- return adapter.clear.call(adapter, scope);
1612
- }
1613
- } : {},
1614
- ...adapter.flush ? {
1615
- flush() {
1616
- onExternalCall();
1617
- return adapter.flush.call(adapter);
1618
- }
1619
- } : {},
1620
- ...adapter.close ? {
1621
- close() {
1622
- onExternalCall();
1623
- return adapter.close.call(adapter);
1624
- }
1625
- } : {}
1626
- };
1627
- }
1628
- async function recoverAbandonedMemoryAttempts(input) {
1629
- let attempts = readActiveMemoryAttempts(input.storage, input.attemptLogPath);
1630
- if (attempts.length > input.maxRecoveryAttempts) {
1631
- throw new Error(
1632
- `memory experiment has ${attempts.length} unfinished attempts; maxRecoveryAttempts is ${input.maxRecoveryAttempts}`
1633
- );
1634
- }
1635
- for (const attempt of attempts) {
1636
- const candidate = input.candidateById.get(attempt.candidateId);
1637
- if (!candidate) {
1638
- throw new Error(
1639
- `cannot recover memory branch '${attempt.branchId}': candidate '${attempt.candidateId}' is missing; pass it in recoveryCandidates`
1640
- );
1641
- }
1642
- assertMemoryAttemptCandidateMatches(attempt, candidate);
1643
- if (!input.sequenceById.has(attempt.sequenceId)) {
1644
- throw new Error(
1645
- `cannot recover memory branch '${attempt.branchId}': sequence '${attempt.sequenceId}' is missing`
1646
- );
1647
- }
1648
- if ((input.options.cleanupBranches ?? true) !== attempt.cleanupBranches) {
1649
- throw new Error(`cannot recover memory branch '${attempt.branchId}': cleanupBranches changed`);
1650
- }
1651
- }
1652
- reconcileInterruptedMemoryPaidCalls(input.costLedger);
1653
- assertNoInterruptedPaidCalls(input.costLedger, "memory experiment recovery");
1654
- for (const attempt of attempts) {
1655
- const candidate = input.candidateById.get(attempt.candidateId);
1656
- const executionCostUsd = candidate.externalCostUsdPerSequence ?? 0;
1657
- if (executionCostUsd > 0 && !hasSettledPaidCall(input.costLedger, memoryAttemptCostCallId(attempt, "execute", 0))) {
1658
- appendMemoryAttemptEvent(input.storage, input.attemptLogPath, {
1659
- ...attempt,
1660
- status: "cleaned",
1661
- recovery: true,
1662
- recordedAt: (input.options.now ?? (() => /* @__PURE__ */ new Date()))().toISOString()
1663
- });
1664
- }
1665
- }
1666
- attempts = readActiveMemoryAttempts(input.storage, input.attemptLogPath);
1667
- const recoveryGenerations = reserveRecoveryAttempts({
1668
- storage: input.storage,
1669
- path: input.recoveryLogPath,
1670
- attemptIds: attempts.map((attempt) => attempt.branchId),
1671
- maxRetriesPerAttempt: input.maxRecoveryRetriesPerAttempt,
1672
- label: "memory recovery attempt log",
1673
- now: input.options.now
1674
- });
1675
- const pool = createMemoryExecutionPool(input.maxConcurrency);
1676
- const settled = await Promise.allSettled(
1677
- attempts.sort((left, right) => left.branchId.localeCompare(right.branchId)).map(
1678
- (attempt) => pool.run(async () => {
1679
- await input.lease.assertOwned();
1680
- const candidate = input.candidateById.get(attempt.candidateId);
1681
- if (!candidate) {
1682
- throw new Error(
1683
- `cannot recover memory branch '${attempt.branchId}': candidate '${attempt.candidateId}' is missing; pass it in recoveryCandidates`
1684
- );
1415
+ }
1416
+ attempts = readActiveMemoryAttempts(input.storage, input.attemptLogPath);
1417
+ const recoveryGenerations = reserveRecoveryAttempts({
1418
+ storage: input.storage,
1419
+ path: input.recoveryLogPath,
1420
+ attemptIds: attempts.map((attempt) => attempt.branchId),
1421
+ maxRetriesPerAttempt: input.maxRecoveryRetriesPerAttempt,
1422
+ label: "memory recovery attempt log",
1423
+ now: input.options.now
1424
+ });
1425
+ const pool = createMemoryExecutionPool(input.maxConcurrency);
1426
+ const settled = await Promise.allSettled(
1427
+ attempts.sort((left, right) => left.branchId.localeCompare(right.branchId)).map(
1428
+ (attempt) => pool.run(async () => {
1429
+ await input.lease.assertOwned();
1430
+ const candidate = input.candidateById.get(attempt.candidateId);
1431
+ if (!candidate) {
1432
+ throw new Error(
1433
+ `cannot recover memory branch '${attempt.branchId}': candidate '${attempt.candidateId}' is missing; pass it in recoveryCandidates`
1434
+ );
1685
1435
  }
1686
1436
  assertMemoryAttemptCandidateMatches(attempt, candidate);
1687
1437
  const sequence = input.sequenceById.get(attempt.sequenceId);
@@ -1972,418 +1722,687 @@ function assertMemoryAttemptCandidateMatches(attempt, candidate) {
1972
1722
  );
1973
1723
  }
1974
1724
  }
1975
- async function writeStep(memory, step) {
1976
- const writes = (step.writes ?? []).map((write) => ({
1977
- ...write,
1978
- scope: mergeScopes2(step.scope, write.scope)
1979
- }));
1980
- if (step.parallelWrites) {
1981
- await Promise.all(writes.map((write) => memory.write(write)));
1982
- return;
1983
- }
1984
- for (const write of writes) await memory.write(write);
1985
- }
1986
- async function probeStep(memory, sequence, step) {
1987
- const run = async (probe) => {
1988
- const scope = mergeScopes2(step.scope, probe.scope);
1989
- const context = await memory.getContext(probe.query, { scope, limit: probe.limit });
1990
- const artifact = {
1991
- answer: context.text,
1992
- rememberedFacts: context.hits.map((hit) => hit.text),
1993
- citedEventIds: uniqueStrings([
1994
- ...context.hits.map((hit) => hit.metadata?.eventId),
1995
- ...context.hits.flatMap((hit) => stringArray(hit.metadata?.eventIds))
1996
- ]),
1997
- usedMemoryIds: context.hits.map((hit) => hit.id),
1998
- actorIds: uniqueStrings([
1999
- ...context.hits.map((hit) => hit.metadata?.actorId),
2000
- ...context.hits.flatMap((hit) => stringArray(hit.metadata?.actorIds))
2001
- ])
2002
- };
2003
- const evaluation = scoreMemoryBenchmarkArtifact(
2004
- {
2005
- id: `${sequence.id}:${step.id}:${probe.id}`,
2006
- family: sequence.family,
2007
- taskKind: probe.taskKind ?? "memory-recall",
2008
- split: sequence.split,
2009
- events: [],
2010
- prompt: probe.query,
2011
- requiredFacts: probe.requiredFacts && probe.requiredFacts.length > 0 ? probe.requiredFacts : probe.referenceAnswer ? [{ id: `${probe.id}:reference`, anyOf: [probe.referenceAnswer] }] : void 0,
2012
- forbiddenFacts: probe.forbiddenFacts,
2013
- expectedEventIds: probe.expectedEventIds,
2014
- expectedActorIds: probe.expectedActorIds,
2015
- referenceAnswer: probe.referenceAnswer
2016
- },
2017
- artifact
1725
+
1726
+ // src/memory/experiment/cell.ts
1727
+ async function runSequenceCell(input) {
1728
+ const { options, candidate, scenario, context, runIdentity, storage, attemptLogPath, lease } = input;
1729
+ const cleanupBranches = options.cleanupBranches ?? true;
1730
+ const costUsd = candidate.externalCostUsdPerSequence ?? 0;
1731
+ if (!Number.isFinite(costUsd) || costUsd < 0) {
1732
+ throw new Error(
1733
+ `${candidate.id}: externalCostUsdPerSequence must be a non-negative finite number`
2018
1734
  );
2019
- return {
2020
- id: probe.id,
2021
- stepId: step.id,
2022
- query: probe.query,
2023
- score: evaluation.score,
2024
- passed: evaluation.passed,
2025
- dimensions: evaluation.dimensions,
2026
- applicableDimensions: evaluation.applicableDimensions ?? Object.keys(evaluation.dimensions),
2027
- notes: evaluation.notes,
2028
- hitIds: context.hits.map((hit) => hit.id)
2029
- };
2030
- };
2031
- if (step.parallelProbes === false) {
2032
- const results = [];
2033
- for (const probe of step.probes ?? []) results.push(await run(probe));
2034
- return results;
2035
1735
  }
2036
- return Promise.all((step.probes ?? []).map(run));
2037
- }
2038
- function rankAgentMemoryExperiment(candidates, scenarios, campaign, costByCandidate) {
2039
- const scenarioById = new Map(scenarios.map((scenario) => [scenario.id, scenario]));
2040
- const rows = candidates.map((candidate) => {
2041
- const candidateScenarios = scenarios.filter((scenario) => scenario.candidateId === candidate.id);
2042
- const cells = campaign.cells.filter(
2043
- (cell) => scenarioById.get(cell.scenarioId)?.candidateId === candidate.id
1736
+ if (!cleanupBranches && !candidate.disposeAdapter) {
1737
+ throw new Error(
1738
+ `${candidate.id}: cleanupBranches=false requires disposeAdapter to delete isolated external state`
2044
1739
  );
2045
- const successful = cells.filter((cell) => !cell.error && cell.artifact);
2046
- const dimensionRows = successful.map((cell) => cell.artifact.dimensions);
2047
- return {
2048
- rank: 0,
2049
- candidateId: candidate.id,
2050
- label: candidate.label ?? candidate.id,
2051
- scoreMean: mean(
2052
- cells.map((cell) => !cell.error && cell.artifact ? cell.artifact.score : 0)
2053
- ),
2054
- passRate: mean(cells.map((cell) => !cell.error && cell.artifact?.passed ? 1 : 0)),
2055
- totalSequences: candidateScenarios.length,
2056
- totalCells: cells.length,
2057
- totalProbes: successful.reduce((sum, cell) => sum + cell.artifact.probes.length, 0),
2058
- cellsFailed: cells.filter((cell) => Boolean(cell.error)).length,
2059
- totalCostUsd: normalizeUsd(costByCandidate.get(candidate.id) ?? 0),
2060
- durationMs: cells.reduce((sum, cell) => sum + cell.durationMs, 0),
2061
- dimensions: meanDimensions(dimensionRows)
2062
- };
2063
- });
2064
- return rows.sort(
2065
- (a, b) => Number(a.cellsFailed > 0) - Number(b.cellsFailed > 0) || b.scoreMean - a.scoreMean || b.passRate - a.passRate || a.totalCostUsd - b.totalCostUsd || a.candidateId.localeCompare(b.candidateId)
2066
- ).map((row, index) => ({ ...row, rank: index + 1 }));
2067
- }
2068
- function memoryExperimentCostByCandidate(costLedger, runDir, scenarios, candidateIdsInput) {
2069
- const candidateByScenario = new Map(
2070
- scenarios.map((scenario) => [scenario.id, scenario.candidateId])
2071
- );
2072
- const candidateIds = new Set(candidateIdsInput);
2073
- const totals = /* @__PURE__ */ new Map();
2074
- for (const receipt of costLedger.list()) {
2075
- if (receipt.tags?.runDir !== runDir) continue;
2076
- const scenarioCandidate = receipt.tags.scenarioId ? candidateByScenario.get(receipt.tags.scenarioId) : void 0;
2077
- const recoveryCandidate = receipt.tags.memoryRecovery === "attempt" && receipt.tags.candidateId ? receipt.tags.candidateId : void 0;
2078
- const candidateId = scenarioCandidate ?? recoveryCandidate;
2079
- if (!candidateId || !candidateIds.has(candidateId)) continue;
2080
- totals.set(candidateId, (totals.get(candidateId) ?? 0) + receipt.costUsd);
2081
1740
  }
2082
- return totals;
2083
- }
2084
- function renderAgentMemoryExperimentRanking(rows, totalCostUsd, unrankedRecoveryCostUsd) {
2085
- return [
2086
- "# Agent Memory Experiment",
2087
- "",
2088
- `- total cost: $${format(totalCostUsd)}`,
2089
- `- retired-candidate recovery cost: $${format(unrankedRecoveryCostUsd)}`,
2090
- "",
2091
- "| rank | candidate | sequences | cells | probes | failed | score | pass rate | cost | duration ms |",
2092
- "| ---: | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |",
2093
- ...rows.map(
2094
- (row) => `| ${row.rank} | ${row.label} | ${row.totalSequences} | ${row.totalCells} | ${row.totalProbes} | ${row.cellsFailed} | ${format(row.scoreMean)} | ${format(row.passRate)} | $${format(row.totalCostUsd)} | ${format(row.durationMs)} |`
2095
- ),
2096
- ""
2097
- ].join("\n");
2098
- }
2099
- function memoryExperimentDispatchRef(options) {
2100
- return stableId(
2101
- "memory_experiment",
2102
- canonicalJson3({
2103
- implementationRef: MEMORY_EXPERIMENT_IMPLEMENTATION_REF,
2104
- experimentId: options.experimentId,
2105
- experimentRunId: options.experimentRunId ?? null,
2106
- executeStepRef: options.executeStepRef ?? "fixtures",
2107
- cleanupBranches: options.cleanupBranches ?? true,
2108
- candidates: options.candidates.map((candidate) => ({
2109
- id: candidate.id,
2110
- ref: candidate.ref,
2111
- policy: candidate.policy ?? null,
2112
- baseScope: candidate.baseScope ?? null,
2113
- externalCostUsdPerSequence: candidate.externalCostUsdPerSequence ?? 0,
2114
- externalRecoveryCostUsdPerAttempt: candidate.externalRecoveryCostUsdPerAttempt ?? 0
2115
- })).sort((a, b) => a.id.localeCompare(b.id))
2116
- })
1741
+ const cleanupTimeoutMs = resolveMemoryCleanupTimeoutMs(
1742
+ options.cleanupTimeoutMs,
1743
+ `${candidate.id}: memory sequence`
2117
1744
  );
2118
- }
2119
- function meanDimensions(rows) {
2120
- const values = /* @__PURE__ */ new Map();
2121
- for (const row of rows) {
2122
- for (const [key, value] of Object.entries(row)) {
2123
- if (!Number.isFinite(value)) continue;
2124
- const bucket = values.get(key) ?? [];
2125
- bucket.push(value);
2126
- values.set(key, bucket);
2127
- }
2128
- }
2129
- return Object.fromEntries([...values].map(([key, bucket]) => [key, mean(bucket)]));
2130
- }
2131
- function countDimensions(rows) {
2132
- const counts = /* @__PURE__ */ new Map();
2133
- for (const row of rows) {
2134
- for (const key of new Set(row)) counts.set(key, (counts.get(key) ?? 0) + 1);
2135
- }
2136
- return Object.fromEntries(counts);
2137
- }
2138
- function mean(values) {
2139
- return values.length === 0 ? 0 : values.reduce((sum, value) => sum + value, 0) / values.length;
2140
- }
2141
- function mergeScopes2(base, extra) {
2142
- return {
2143
- ...base ?? {},
2144
- ...extra ?? {},
2145
- tags: { ...base?.tags ?? {}, ...extra?.tags ?? {} }
2146
- };
2147
- }
2148
- function memoryExperimentBaseScope(options, candidate, sequenceId) {
2149
- return mergeScopes2(candidate.baseScope, {
2150
- tags: {
2151
- memoryExperimentId: options.experimentId,
2152
- memoryCandidateId: candidate.id,
2153
- memorySequenceId: sequenceId
2154
- }
1745
+ context.signal.throwIfAborted();
1746
+ await lease.assertOwned();
1747
+ const startedAt = Date.now();
1748
+ const branchId = stableId(
1749
+ "memory_branch",
1750
+ `${runIdentity}:${context.cellId}:${context.seed}:${randomUUID2()}`
1751
+ );
1752
+ const attempt = memoryAttemptEvent({
1753
+ status: "started",
1754
+ branchId,
1755
+ candidate,
1756
+ sequence: scenario.sequence,
1757
+ rep: context.rep,
1758
+ seed: context.seed,
1759
+ recovery: false,
1760
+ cleanupBranches,
1761
+ now: options.now
2155
1762
  });
2156
- }
2157
- function sequenceCleanupScopes(sequence) {
2158
- const scopes = /* @__PURE__ */ new Map();
2159
- for (const scope of sequence.cleanupScopes ?? []) {
2160
- const normalized = normalizeCleanupScope(scope);
2161
- scopes.set(JSON.stringify(normalized), normalized);
2162
- }
2163
- for (const step of sequence.steps) {
2164
- const candidates = [
2165
- ...step.scope ? [step.scope] : [],
2166
- ...(step.writes ?? []).map((write) => mergeScopes2(step.scope, write.scope)),
2167
- ...(step.probes ?? []).map((probe) => mergeScopes2(step.scope, probe.scope))
2168
- ];
2169
- for (const scope of candidates) {
2170
- const normalized = normalizeCleanupScope(scope);
2171
- scopes.set(JSON.stringify(normalized), normalized);
2172
- }
2173
- }
2174
- return [...scopes.values()];
2175
- }
2176
- async function clearSequenceScopes(memory, sequence) {
2177
- for (const scope of sequenceCleanupScopes(sequence)) await memory.clear?.(scope);
2178
- }
2179
- function assertMemorySequences(sequences) {
2180
- for (const sequence of sequences) {
2181
- assertNonEmptyString(sequence.family, `memory experiment sequence ${sequence.id} family`);
2182
- if (sequence.steps.length === 0) {
2183
- throw new Error(`memory experiment sequence ${sequence.id} has no steps`);
2184
- }
2185
- assertUnique(
2186
- sequence.steps.map((step) => step.id),
2187
- `step in sequence ${sequence.id}`
2188
- );
2189
- let probeCount = 0;
2190
- for (const step of sequence.steps) {
2191
- for (const write of step.writes ?? []) {
2192
- assertNonEmptyString(write.text, `memory experiment write in ${sequence.id}/${step.id}`);
2193
- if (write.id !== void 0) {
2194
- assertNonEmptyString(write.id, `memory experiment write id in ${sequence.id}/${step.id}`);
2195
- }
2196
- }
2197
- assertUnique(
2198
- (step.probes ?? []).map((probe) => probe.id),
2199
- `probe in sequence ${sequence.id} step ${step.id}`
1763
+ appendMemoryAttemptEvent(storage, attemptLogPath, attempt);
1764
+ let externalCallAttempted = false;
1765
+ const appendCleanedAttempt = (priorError) => {
1766
+ try {
1767
+ appendMemoryAttemptEvent(storage, attemptLogPath, {
1768
+ ...attempt,
1769
+ status: "cleaned",
1770
+ recordedAt: (options.now ?? (() => /* @__PURE__ */ new Date()))().toISOString()
1771
+ });
1772
+ } catch (error) {
1773
+ throw new AgentMemoryCleanupError(
1774
+ [...priorError ? [priorError] : [], error],
1775
+ `${candidate.id}: memory branch cleanup could not be recorded`
2200
1776
  );
2201
- for (const probe of step.probes ?? []) {
2202
- probeCount += 1;
2203
- assertNonEmptyString(
2204
- probe.query,
2205
- `memory experiment probe query ${sequence.id}/${step.id}/${probe.id}`
2206
- );
2207
- if (probe.limit !== void 0 && (!Number.isSafeInteger(probe.limit) || probe.limit <= 0)) {
2208
- throw new Error(
2209
- `memory experiment probe limit ${sequence.id}/${step.id}/${probe.id} must be a positive safe integer`
2210
- );
1777
+ }
1778
+ };
1779
+ const execute = async () => {
1780
+ context.signal.throwIfAborted();
1781
+ await lease.assertOwned();
1782
+ let rawAdapter;
1783
+ let adapter;
1784
+ let memory;
1785
+ let primaryError;
1786
+ let completedArtifact;
1787
+ let finalClearStarted = false;
1788
+ let finalClearCompleted = false;
1789
+ try {
1790
+ const created = await candidate.createAdapter({
1791
+ branchId,
1792
+ sequence: scenario.sequence,
1793
+ rep: context.rep,
1794
+ seed: context.seed,
1795
+ purpose: "execute",
1796
+ signal: context.signal,
1797
+ markExternalCall: () => {
1798
+ externalCallAttempted = true;
2211
1799
  }
2212
- assertMemoryFactMatchers(
2213
- probe.requiredFacts ?? [],
2214
- `${sequence.id}/${step.id}/${probe.id} requiredFacts`
2215
- );
2216
- assertMemoryFactMatchers(
2217
- probe.forbiddenFacts ?? [],
2218
- `${sequence.id}/${step.id}/${probe.id} forbiddenFacts`
1800
+ });
1801
+ if (!created) throw new Error(`${candidate.id}: createAdapter returned no execution adapter`);
1802
+ rawAdapter = created;
1803
+ adapter = trackExternalMemoryCalls(created, () => {
1804
+ externalCallAttempted = true;
1805
+ });
1806
+ await lease.assertOwned();
1807
+ context.signal.throwIfAborted();
1808
+ if (cleanupBranches && !adapter.clear) {
1809
+ throw new Error(
1810
+ `${candidate.id}: cleanupBranches requires an adapter with scoped clear support`
2219
1811
  );
2220
- assertStringList(probe.expectedEventIds, `${sequence.id}/${step.id}/${probe.id} event ids`);
2221
- assertStringList(probe.expectedActorIds, `${sequence.id}/${step.id}/${probe.id} actor ids`);
2222
- if (probe.referenceAnswer !== void 0) {
2223
- assertNonEmptyString(
2224
- probe.referenceAnswer,
2225
- `memory experiment reference answer ${sequence.id}/${step.id}/${probe.id}`
2226
- );
2227
- }
2228
- const hasTarget = (probe.requiredFacts?.length ?? 0) > 0 || (probe.forbiddenFacts?.length ?? 0) > 0 || (probe.expectedEventIds?.length ?? 0) > 0 || (probe.expectedActorIds?.length ?? 0) > 0 || Boolean(probe.referenceAnswer?.trim());
2229
- if (!hasTarget) {
2230
- throw new Error(
2231
- `memory experiment probe ${sequence.id}/${step.id}/${probe.id} has no measurable target`
2232
- );
1812
+ }
1813
+ memory = createAgentMemoryBranch({
1814
+ adapter,
1815
+ branchId,
1816
+ lifetime: "attempt",
1817
+ policy: candidate.policy,
1818
+ allowedWriteScopes: sequenceCleanupScopes(scenario.sequence),
1819
+ baseScope: memoryExperimentBaseScope(options, candidate, scenario.sequenceId)
1820
+ });
1821
+ const probes = [];
1822
+ for (const step of scenario.sequence.steps) {
1823
+ context.signal.throwIfAborted();
1824
+ await lease.assertOwned();
1825
+ await writeStep(memory, step);
1826
+ await lease.assertOwned();
1827
+ await options.executeStep?.({
1828
+ memory,
1829
+ candidateId: candidate.id,
1830
+ sequence: scenario.sequence,
1831
+ step,
1832
+ context
1833
+ });
1834
+ context.signal.throwIfAborted();
1835
+ await lease.assertOwned();
1836
+ const stepProbes = await probeStep(memory, scenario.sequence, step);
1837
+ await lease.assertOwned();
1838
+ probes.push(...stepProbes);
1839
+ }
1840
+ const snapshot = await memory.snapshot();
1841
+ await lease.assertOwned();
1842
+ await options.onBranchSnapshot?.({
1843
+ candidateId: candidate.id,
1844
+ sequenceId: scenario.sequenceId,
1845
+ cellId: context.cellId,
1846
+ snapshot
1847
+ });
1848
+ await lease.assertOwned();
1849
+ const dimensions = meanDimensions(probes.map((probe) => probe.dimensions));
1850
+ const dimensionSampleCounts = countDimensions(
1851
+ probes.map((probe) => probe.applicableDimensions)
1852
+ );
1853
+ const artifact = {
1854
+ candidateId: candidate.id,
1855
+ sequenceId: scenario.sequenceId,
1856
+ score: mean(probes.map((probe) => probe.score)),
1857
+ passed: probes.length > 0 && probes.every((probe) => probe.passed),
1858
+ dimensions,
1859
+ dimensionSampleCounts,
1860
+ probes,
1861
+ branchDigest: snapshot.digest,
1862
+ journalEntries: snapshot.journal.length,
1863
+ durationMs: Math.max(0, Date.now() - startedAt)
1864
+ };
1865
+ if (cleanupBranches) {
1866
+ finalClearStarted = true;
1867
+ await runBoundedMemoryLifecycle({
1868
+ operation: `${candidate.id}: final branch cleanup`,
1869
+ timeoutMs: cleanupTimeoutMs,
1870
+ resource: adapter,
1871
+ run: () => clearSequenceScopes(memory, scenario.sequence)
1872
+ });
1873
+ finalClearCompleted = true;
1874
+ await lease.assertOwned();
1875
+ }
1876
+ completedArtifact = artifact;
1877
+ } catch (error) {
1878
+ primaryError = error;
1879
+ }
1880
+ const cleanupErrors = [];
1881
+ let cleanupOwned = true;
1882
+ let ownershipError;
1883
+ try {
1884
+ await lease.assertOwned();
1885
+ } catch (error) {
1886
+ cleanupOwned = false;
1887
+ ownershipError = error;
1888
+ }
1889
+ if (finalClearStarted && !finalClearCompleted && primaryError) {
1890
+ cleanupErrors.push(primaryError);
1891
+ }
1892
+ if (primaryError && cleanupOwned && !finalClearStarted && memory && cleanupBranches && adapter?.clear) {
1893
+ try {
1894
+ await runBoundedMemoryLifecycle({
1895
+ operation: `${candidate.id}: failed branch cleanup`,
1896
+ timeoutMs: cleanupTimeoutMs,
1897
+ resource: adapter,
1898
+ run: () => clearSequenceScopes(memory, scenario.sequence)
1899
+ });
1900
+ } catch (error) {
1901
+ cleanupErrors.push(error);
1902
+ }
1903
+ }
1904
+ if (adapter) {
1905
+ try {
1906
+ await runBoundedMemoryLifecycle({
1907
+ operation: `${candidate.id}: adapter close`,
1908
+ timeoutMs: cleanupTimeoutMs,
1909
+ resource: adapter,
1910
+ run: async () => {
1911
+ if (memory && cleanupOwned) await memory.close?.();
1912
+ else {
1913
+ if (cleanupOwned) await adapter.flush?.();
1914
+ await adapter.close?.();
1915
+ }
1916
+ }
1917
+ });
1918
+ } catch (error) {
1919
+ cleanupErrors.push(error);
1920
+ }
1921
+ if (cleanupOwned) {
1922
+ try {
1923
+ await runBoundedMemoryLifecycle({
1924
+ operation: `${candidate.id}: adapter disposal`,
1925
+ timeoutMs: cleanupTimeoutMs,
1926
+ resource: adapter,
1927
+ run: () => {
1928
+ if (candidate.disposeAdapter) externalCallAttempted = true;
1929
+ return candidate.disposeAdapter?.(rawAdapter);
1930
+ }
1931
+ });
1932
+ } catch (error) {
1933
+ cleanupErrors.push(error);
2233
1934
  }
2234
1935
  }
1936
+ } else {
1937
+ cleanupErrors.push(
1938
+ new Error(`${candidate.id}: adapter creation failed before cleanup could be confirmed`)
1939
+ );
2235
1940
  }
2236
- if (probeCount === 0) {
2237
- throw new Error(`memory experiment sequence ${sequence.id} has no probes`);
1941
+ if (cleanupOwned && cleanupErrors.length === 0) appendCleanedAttempt(primaryError);
1942
+ if (!cleanupOwned && cleanupErrors.length === 0) {
1943
+ if (primaryError) throw primaryError;
1944
+ throw ownershipError;
1945
+ }
1946
+ if (cleanupErrors.length > 0) {
1947
+ const primaryMessage = primaryError instanceof Error ? primaryError.message : String(primaryError ?? "");
1948
+ throw new AgentMemoryCleanupError(
1949
+ [
1950
+ ...primaryError && !cleanupErrors.includes(primaryError) ? [primaryError] : [],
1951
+ ...ownershipError ? [ownershipError] : [],
1952
+ ...cleanupErrors
1953
+ ],
1954
+ `${candidate.id}: memory branch cleanup failed${primaryMessage ? ` after: ${primaryMessage}` : ""}`
1955
+ );
1956
+ }
1957
+ if (primaryError) throw primaryError;
1958
+ if (!completedArtifact) throw new Error(`${candidate.id}: memory sequence produced no result`);
1959
+ return completedArtifact;
1960
+ };
1961
+ if (costUsd === 0) {
1962
+ let artifact;
1963
+ let error;
1964
+ try {
1965
+ artifact = await execute();
1966
+ } catch (caught) {
1967
+ error = caught;
2238
1968
  }
1969
+ if (error) throw error;
1970
+ if (!artifact) throw new Error(`${candidate.id}: memory sequence produced no result`);
1971
+ return artifact;
2239
1972
  }
1973
+ const receipt = {
1974
+ model: candidate.id,
1975
+ inputTokens: 0,
1976
+ outputTokens: 0,
1977
+ actualCostUsd: costUsd
1978
+ };
1979
+ const paid = await context.cost.runPaidCall({
1980
+ callId: memoryAttemptCostCallId(attempt, "execute", 0),
1981
+ actor: `agent-knowledge:memory-experiment:${candidate.id}`,
1982
+ model: candidate.id,
1983
+ maximumCharge: { externallyEnforcedMaximumUsd: costUsd },
1984
+ execute,
1985
+ receipt: () => receipt,
1986
+ receiptFromError: () => ({
1987
+ ...receipt,
1988
+ actualCostUsd: externalCallAttempted ? costUsd : 0
1989
+ })
1990
+ });
1991
+ if (!paid.succeeded) throw paid.error;
1992
+ return paid.value;
2240
1993
  }
2241
- function normalizeCleanupScope(scope) {
2242
- const normalized = compactScope(scope);
2243
- if (normalized.tags && Object.keys(normalized.tags).length === 0) {
2244
- delete normalized.tags;
1994
+ async function writeStep(memory, step) {
1995
+ const writes = (step.writes ?? []).map((write) => ({
1996
+ ...write,
1997
+ scope: mergeScopes2(step.scope, write.scope)
1998
+ }));
1999
+ if (step.parallelWrites) {
2000
+ await Promise.all(writes.map((write) => memory.write(write)));
2001
+ return;
2245
2002
  }
2246
- return normalized;
2003
+ for (const write of writes) await memory.write(write);
2247
2004
  }
2248
- function compactScope(scope) {
2249
- return Object.fromEntries(
2250
- Object.entries(scope).filter(([, value]) => value !== void 0)
2251
- );
2252
- }
2253
- function compactRecord(record2) {
2254
- return Object.fromEntries(Object.entries(record2).filter(([, value]) => value !== void 0));
2255
- }
2256
- function uniqueStrings(values) {
2257
- return [...new Set(values.filter((value) => typeof value === "string"))];
2258
- }
2259
- function stringArray(value) {
2260
- return Array.isArray(value) ? value.filter((entry) => typeof entry === "string") : [];
2261
- }
2262
- function assertUnique(values, label) {
2263
- const seen = /* @__PURE__ */ new Set();
2264
- for (const value of values) {
2265
- assertNonEmptyString(value, `memory experiment ${label} id`);
2266
- if (seen.has(value)) throw new Error(`duplicate memory experiment ${label} id: ${value}`);
2267
- seen.add(value);
2005
+ async function probeStep(memory, sequence, step) {
2006
+ const run = async (probe) => {
2007
+ const scope = mergeScopes2(step.scope, probe.scope);
2008
+ const context = await memory.getContext(probe.query, { scope, limit: probe.limit });
2009
+ const artifact = {
2010
+ answer: context.text,
2011
+ rememberedFacts: context.hits.map((hit) => hit.text),
2012
+ citedEventIds: uniqueStrings([
2013
+ ...context.hits.map((hit) => hit.metadata?.eventId),
2014
+ ...context.hits.flatMap((hit) => stringArray(hit.metadata?.eventIds))
2015
+ ]),
2016
+ usedMemoryIds: context.hits.map((hit) => hit.id),
2017
+ actorIds: uniqueStrings([
2018
+ ...context.hits.map((hit) => hit.metadata?.actorId),
2019
+ ...context.hits.flatMap((hit) => stringArray(hit.metadata?.actorIds))
2020
+ ])
2021
+ };
2022
+ const evaluation = scoreMemoryBenchmarkArtifact(
2023
+ {
2024
+ id: `${sequence.id}:${step.id}:${probe.id}`,
2025
+ family: sequence.family,
2026
+ taskKind: probe.taskKind ?? "memory-recall",
2027
+ split: sequence.split,
2028
+ events: [],
2029
+ prompt: probe.query,
2030
+ requiredFacts: probe.requiredFacts && probe.requiredFacts.length > 0 ? probe.requiredFacts : probe.referenceAnswer ? [{ id: `${probe.id}:reference`, anyOf: [probe.referenceAnswer] }] : void 0,
2031
+ forbiddenFacts: probe.forbiddenFacts,
2032
+ expectedEventIds: probe.expectedEventIds,
2033
+ expectedActorIds: probe.expectedActorIds,
2034
+ referenceAnswer: probe.referenceAnswer
2035
+ },
2036
+ artifact
2037
+ );
2038
+ return {
2039
+ id: probe.id,
2040
+ stepId: step.id,
2041
+ query: probe.query,
2042
+ score: evaluation.score,
2043
+ passed: evaluation.passed,
2044
+ dimensions: evaluation.dimensions,
2045
+ applicableDimensions: evaluation.applicableDimensions ?? Object.keys(evaluation.dimensions),
2046
+ notes: evaluation.notes,
2047
+ hitIds: context.hits.map((hit) => hit.id)
2048
+ };
2049
+ };
2050
+ if (step.parallelProbes === false) {
2051
+ const results = [];
2052
+ for (const probe of step.probes ?? []) results.push(await run(probe));
2053
+ return results;
2268
2054
  }
2055
+ return Promise.all((step.probes ?? []).map(run));
2269
2056
  }
2270
- function assertMemoryFactMatchers(matchers, label) {
2057
+
2058
+ // src/memory/experiment/run.ts
2059
+ var MEMORY_EXPERIMENT_IMPLEMENTATION_REF = "agent-knowledge:memory-experiment:v6";
2060
+ async function runAgentMemoryExperiment(options) {
2061
+ assertNonEmptyString(options.experimentId, "memory experiment experimentId");
2062
+ assertNonEmptyString(options.runDir, "memory experiment runDir");
2063
+ if (options.experimentRunId !== void 0) {
2064
+ assertNonEmptyString(options.experimentRunId, "memory experiment experimentRunId");
2065
+ }
2066
+ if (options.sequences.length === 0) throw new Error("memory experiment requires sequences");
2067
+ if (options.candidates.length === 0) throw new Error("memory experiment requires candidates");
2068
+ if (options.executeStep && !options.executeStepRef) {
2069
+ throw new Error("memory experiment executeStepRef is required when executeStep is configured");
2070
+ }
2271
2071
  assertUnique(
2272
- matchers.map((matcher) => matcher.id),
2273
- `${label} matcher`
2072
+ options.sequences.map((sequence) => sequence.id),
2073
+ "sequence"
2274
2074
  );
2275
- for (const matcher of matchers) {
2276
- if (matcher.anyOf.length === 0) {
2277
- throw new Error(`memory experiment ${label} matcher ${matcher.id} requires anyOf`);
2075
+ assertUnique(
2076
+ [...options.candidates, ...options.recoveryCandidates ?? []].map((candidate) => candidate.id),
2077
+ "candidate"
2078
+ );
2079
+ assertMemorySequences(options.sequences);
2080
+ for (const candidate of [...options.candidates, ...options.recoveryCandidates ?? []]) {
2081
+ if (options.cleanupBranches === false && !candidate.disposeAdapter) {
2082
+ throw new Error(
2083
+ `${candidate.id}: cleanupBranches=false requires disposeAdapter to delete isolated external state`
2084
+ );
2278
2085
  }
2279
- assertStringList(matcher.anyOf, `${label} matcher ${matcher.id} anyOf`);
2280
- assertStringList(matcher.sourceEventIds, `${label} matcher ${matcher.id} source event ids`);
2281
- if (matcher.weight !== void 0 && (!Number.isFinite(matcher.weight) || matcher.weight <= 0)) {
2086
+ if (candidate.externalCostUsdPerSequence !== void 0 && (!Number.isFinite(candidate.externalCostUsdPerSequence) || candidate.externalCostUsdPerSequence < 0)) {
2282
2087
  throw new Error(
2283
- `memory experiment ${label} matcher ${matcher.id} weight must be a positive finite number`
2088
+ `${candidate.id}: externalCostUsdPerSequence must be a non-negative finite number`
2284
2089
  );
2285
2090
  }
2091
+ if (candidate.externalRecoveryCostUsdPerAttempt !== void 0 && (!Number.isFinite(candidate.externalRecoveryCostUsdPerAttempt) || candidate.externalRecoveryCostUsdPerAttempt < 0)) {
2092
+ throw new Error(
2093
+ `${candidate.id}: externalRecoveryCostUsdPerAttempt must be a non-negative finite number`
2094
+ );
2095
+ }
2096
+ assertNonEmptyString(candidate.ref, `${candidate.id} ref`);
2286
2097
  }
2287
- }
2288
- function assertStringList(values, label) {
2289
- if (values === void 0) return;
2290
- assertUnique(values, label);
2291
- }
2292
- function assertNonEmptyString(value, label) {
2293
- if (typeof value !== "string" || value.trim().length === 0) {
2294
- throw new Error(`${label} must be a non-empty string`);
2098
+ const storage = options.storage ?? fsCampaignStorage();
2099
+ const runDir = resolveRunDir(options.runDir, options.repo);
2100
+ if (!storage.append) {
2101
+ throw new Error("memory experiment requires CampaignStorage.append for durable attempt state");
2295
2102
  }
2103
+ resolveMemoryCleanupTimeoutMs(options.cleanupTimeoutMs, "memory experiment");
2104
+ const maxRecoveryAttempts = options.maxRecoveryAttempts ?? 1e3;
2105
+ if (!Number.isSafeInteger(maxRecoveryAttempts) || maxRecoveryAttempts <= 0) {
2106
+ throw new Error("memory experiment maxRecoveryAttempts must be a positive safe integer");
2107
+ }
2108
+ const maxRecoveryRetriesPerAttempt = options.maxRecoveryRetriesPerAttempt ?? DEFAULT_MEMORY_RECOVERY_RETRIES_PER_ATTEMPT;
2109
+ if (!Number.isSafeInteger(maxRecoveryRetriesPerAttempt) || maxRecoveryRetriesPerAttempt <= 0) {
2110
+ throw new Error(
2111
+ "memory experiment maxRecoveryRetriesPerAttempt must be a positive safe integer"
2112
+ );
2113
+ }
2114
+ storage.ensureDir(runDir);
2115
+ const lease = await acquireAgentMemoryRunLease({
2116
+ experimentId: options.experimentId,
2117
+ runDir,
2118
+ storage,
2119
+ customStorage: options.storage !== void 0,
2120
+ lockFileName: "memory-experiment.lock",
2121
+ label: "memory experiment",
2122
+ controllerMode: options.controllerMode,
2123
+ acquireRunLease: options.acquireRunLease
2124
+ });
2125
+ let result;
2126
+ let primaryError;
2127
+ try {
2128
+ await lease.assertOwned();
2129
+ result = await runOwnedAgentMemoryExperiment(options, storage, runDir, lease);
2130
+ } catch (error) {
2131
+ primaryError = error;
2132
+ }
2133
+ let releaseError;
2134
+ try {
2135
+ await lease.release();
2136
+ } catch (error) {
2137
+ releaseError = error;
2138
+ }
2139
+ if (primaryError && releaseError) {
2140
+ throw new AggregateError(
2141
+ [primaryError, releaseError],
2142
+ "memory experiment failed and its controller lease could not be released"
2143
+ );
2144
+ }
2145
+ if (primaryError) throw primaryError;
2146
+ if (releaseError) throw releaseError;
2147
+ if (!result) throw new Error("memory experiment produced no result");
2148
+ return result;
2296
2149
  }
2297
- function format(value) {
2298
- return Number.isFinite(value) ? value.toFixed(4) : "0.0000";
2299
- }
2300
- function normalizeUsd(value) {
2301
- return Number(value.toFixed(12));
2302
- }
2303
-
2304
- // src/memory/graphiti.ts
2305
- import { randomUUID as randomUUID3 } from "crypto";
2306
- import { canonicalJson as canonicalJson4 } from "@tangle-network/agent-eval";
2307
- var DEFAULT_GRAPHITI_INGESTION_TIMEOUT_MS = 12e4;
2308
- function createGraphitiMemoryAdapter(options) {
2309
- assertGraphitiOptions(options);
2310
- const id = options.id ?? "graphiti";
2311
- const consistency = options.consistency ?? "visible";
2312
- const tools = graphitiToolNames(options);
2313
- const queued = /* @__PURE__ */ new Map();
2314
- const adapter = {
2315
- id,
2316
- branchIsolation: consistency === "visible" ? {
2317
- mode: "scoped",
2318
- processExitSafe: false,
2319
- recoveryDelayMs: options.ingestionTimeoutMs ?? DEFAULT_GRAPHITI_INGESTION_TIMEOUT_MS
2320
- } : {
2321
- mode: "unsupported",
2322
- reason: 'queued Graphiti writes can become visible after a process restart; use consistency="visible" for branch experiments'
2323
- },
2324
- async search(query, searchOptions = {}) {
2325
- if (searchOptions.minScore !== void 0 && !Number.isFinite(searchOptions.minScore)) {
2326
- throw new Error(`${id}: minScore must be finite`);
2327
- }
2328
- const scope = mergeScopes3(options.defaultScope, searchOptions.scope);
2329
- const groupId = graphitiGroupId(id, scope);
2330
- const modes = new Set(options.search ?? ["facts", "nodes"]);
2331
- const kinds = searchOptions.kinds && searchOptions.kinds.length > 0 ? new Set(searchOptions.kinds) : null;
2332
- const searchFacts = modes.has("facts") && (!kinds || kinds.has("fact"));
2333
- const searchNodes = modes.has("nodes") && (!kinds || kinds.has("entity"));
2334
- const limit = searchOptions.limit ?? 10;
2335
- if (!Number.isSafeInteger(limit) || limit < 0) {
2336
- throw new Error(`${id}: search limit must be a non-negative safe integer`);
2337
- }
2338
- if (limit === 0) return [];
2339
- const [factPayload, nodePayload] = await Promise.all([
2340
- searchFacts ? callGraphiti(options.client, tools.searchFacts, {
2341
- query,
2342
- group_ids: [groupId],
2343
- max_facts: limit
2344
- }) : void 0,
2345
- searchNodes ? callGraphiti(options.client, tools.searchNodes, {
2346
- query,
2347
- group_ids: [groupId],
2348
- max_nodes: limit,
2349
- ...searchOptions.metadata?.entityTypes ? { entity_types: searchOptions.metadata.entityTypes } : {}
2350
- }) : void 0
2351
- ]);
2352
- const hits = mergeRankedMemoryHits(
2353
- [normalizeGraphitiFacts(factPayload, id), normalizeGraphitiNodes(nodePayload, id)].map(
2354
- (group) => group.filter((hit) => {
2355
- if (searchOptions.minScore === void 0) return true;
2356
- const score = hit.normalizedScore ?? hit.score;
2357
- return score !== void 0 && score >= searchOptions.minScore;
2358
- })
2359
- ),
2360
- limit
2361
- );
2362
- await enrichGraphitiProvenance(options, groupId, hits);
2363
- return hits;
2364
- },
2365
- async getContext(query, searchOptions = {}) {
2366
- return defaultGetMemoryContext(adapter, query, searchOptions);
2367
- },
2368
- async write(input) {
2369
- const scope = mergeScopes3(options.defaultScope, input.scope);
2370
- const groupId = graphitiGroupId(id, scope);
2371
- const eventKey = canonicalJson4(
2372
- stripUndefined2({
2373
- id: input.id,
2374
- kind: input.kind,
2375
- text: input.text,
2376
- title: input.title,
2377
- role: input.role,
2378
- metadata: input.metadata
2379
- })
2380
- );
2381
- const episodeUuid = input.id === void 0 ? randomUUID3() : deterministicUuid(`${id}:${groupId}:${eventKey}`);
2382
- const name = input.title ?? `${input.kind}:${input.id ?? stableId("event", eventKey)}`;
2383
- const sourceDescription = canonicalJson4(
2384
- stripUndefined2({
2385
- adapter: "agent-knowledge",
2386
- memoryKind: input.kind,
2150
+ async function runOwnedAgentMemoryExperiment(options, storage, runDir, lease) {
2151
+ const runIdentity = stableId(
2152
+ "memory_run",
2153
+ canonicalJson4({
2154
+ experimentId: options.experimentId,
2155
+ experimentRunId: options.experimentRunId ?? runDir
2156
+ })
2157
+ );
2158
+ const maxConcurrency = options.maxConcurrency ?? 2;
2159
+ if (!Number.isSafeInteger(maxConcurrency) || maxConcurrency <= 0) {
2160
+ throw new Error("memory experiment maxConcurrency must be a positive safe integer");
2161
+ }
2162
+ const executionPool = createMemoryExecutionPool(maxConcurrency);
2163
+ const dispatchedExecutions = [];
2164
+ const candidateById = new Map(options.candidates.map((candidate) => [candidate.id, candidate]));
2165
+ const recoveryCandidateById = new Map(
2166
+ [...options.candidates, ...options.recoveryCandidates ?? []].map((candidate) => [
2167
+ candidate.id,
2168
+ candidate
2169
+ ])
2170
+ );
2171
+ const sequenceById = new Map(options.sequences.map((sequence) => [sequence.id, sequence]));
2172
+ const scenarios = buildAgentMemorySequenceScenarios(options.sequences, options.candidates);
2173
+ const attemptLogPath = join(runDir, "memory-attempts.jsonl");
2174
+ const recoveryLogPath = join(runDir, "memory-recovery-attempts.jsonl");
2175
+ const costCeiling = options.costCeiling ?? options.costLedger?.costCeilingUsd ?? 0;
2176
+ const costLedger = options.costLedger ?? createRunCostLedger({
2177
+ storage,
2178
+ runDir,
2179
+ costCeilingUsd: costCeiling
2180
+ });
2181
+ if (costLedger.costCeilingUsd !== costCeiling) {
2182
+ throw new Error("memory experiment costCeiling must match the shared cost ledger ceiling");
2183
+ }
2184
+ storage.ensureDir(runDir);
2185
+ await recoverAbandonedMemoryAttempts({
2186
+ options,
2187
+ storage,
2188
+ runDir,
2189
+ attemptLogPath,
2190
+ candidateById: recoveryCandidateById,
2191
+ sequenceById,
2192
+ lease,
2193
+ maxConcurrency,
2194
+ costLedger,
2195
+ maxRecoveryAttempts: options.maxRecoveryAttempts ?? 1e3,
2196
+ recoveryLogPath,
2197
+ maxRecoveryRetriesPerAttempt: options.maxRecoveryRetriesPerAttempt ?? DEFAULT_MEMORY_RECOVERY_RETRIES_PER_ATTEMPT
2198
+ });
2199
+ await lease.assertOwned();
2200
+ let campaign;
2201
+ let campaignError;
2202
+ let settledExecutions = [];
2203
+ try {
2204
+ campaign = await runCampaign({
2205
+ scenarios,
2206
+ dispatch: (scenario, context) => {
2207
+ const candidate = candidateById.get(scenario.candidateId);
2208
+ if (!candidate) throw new Error(`unknown memory candidate ${scenario.candidateId}`);
2209
+ const operation = executionPool.run(
2210
+ () => runSequenceCell({
2211
+ options,
2212
+ candidate,
2213
+ scenario,
2214
+ context,
2215
+ runIdentity,
2216
+ storage,
2217
+ attemptLogPath,
2218
+ lease
2219
+ })
2220
+ );
2221
+ dispatchedExecutions.push(operation);
2222
+ return operation;
2223
+ },
2224
+ dispatchRef: memoryExperimentDispatchRef(options),
2225
+ judges: [agentMemorySequenceJudge()],
2226
+ runDir,
2227
+ storage,
2228
+ seed: options.seed,
2229
+ reps: options.reps,
2230
+ resumable: options.resumable,
2231
+ costCeiling,
2232
+ costLedger,
2233
+ costPhase: options.costPhase,
2234
+ maxConcurrency,
2235
+ dispatchTimeoutMs: options.dispatchTimeoutMs,
2236
+ expectUsage: "off",
2237
+ now: options.now
2238
+ });
2239
+ } catch (error) {
2240
+ campaignError = error;
2241
+ } finally {
2242
+ settledExecutions = await Promise.allSettled(dispatchedExecutions);
2243
+ }
2244
+ const cleanupFailures = settledExecutions.flatMap(
2245
+ (settled) => settled.status === "rejected" && settled.reason instanceof AgentMemoryCleanupError ? [settled.reason] : []
2246
+ );
2247
+ if (campaignError && cleanupFailures.length > 0) {
2248
+ throw new AggregateError(
2249
+ [campaignError, ...cleanupFailures],
2250
+ "memory experiment failed and provider cleanup also failed"
2251
+ );
2252
+ }
2253
+ if (campaignError) throw campaignError;
2254
+ if (cleanupFailures.length > 0) {
2255
+ throw new AggregateError(cleanupFailures, "memory experiment cleanup failed after dispatch");
2256
+ }
2257
+ if (!campaign) throw new Error("memory experiment produced no campaign result");
2258
+ await lease.assertOwned();
2259
+ const costByCandidate = memoryExperimentCostByCandidate(
2260
+ costLedger,
2261
+ campaign.runDir,
2262
+ scenarios,
2263
+ [...options.candidates, ...options.recoveryCandidates ?? []].map((candidate) => candidate.id)
2264
+ );
2265
+ const unrankedRecoveryCostUsd = normalizeUsd(
2266
+ (options.recoveryCandidates ?? []).reduce(
2267
+ (sum, candidate) => sum + (costByCandidate.get(candidate.id) ?? 0),
2268
+ 0
2269
+ )
2270
+ );
2271
+ const totalCostUsd = normalizeUsd(
2272
+ [...costByCandidate.values()].reduce((sum, cost) => sum + cost, 0)
2273
+ );
2274
+ const rows = rankAgentMemoryExperiment(options.candidates, scenarios, campaign, costByCandidate);
2275
+ const rankingJsonPath = join(campaign.runDir, "memory-experiment-ranking.json");
2276
+ const rankingMarkdownPath = join(campaign.runDir, "memory-experiment-ranking.md");
2277
+ storage.write(
2278
+ rankingJsonPath,
2279
+ `${JSON.stringify(
2280
+ { experimentId: options.experimentId, totalCostUsd, unrankedRecoveryCostUsd, rows },
2281
+ null,
2282
+ 2
2283
+ )}
2284
+ `
2285
+ );
2286
+ storage.write(
2287
+ rankingMarkdownPath,
2288
+ renderAgentMemoryExperimentRanking(rows, totalCostUsd, unrankedRecoveryCostUsd)
2289
+ );
2290
+ return {
2291
+ campaign,
2292
+ rows,
2293
+ totalCostUsd,
2294
+ unrankedRecoveryCostUsd,
2295
+ leaderCandidateId: rows.find((row) => row.cellsFailed === 0)?.candidateId,
2296
+ rankingJsonPath,
2297
+ rankingMarkdownPath,
2298
+ attemptLogPath,
2299
+ recoveryLogPath
2300
+ };
2301
+ }
2302
+ function memoryExperimentDispatchRef(options) {
2303
+ return stableId(
2304
+ "memory_experiment",
2305
+ canonicalJson4({
2306
+ implementationRef: MEMORY_EXPERIMENT_IMPLEMENTATION_REF,
2307
+ experimentId: options.experimentId,
2308
+ experimentRunId: options.experimentRunId ?? null,
2309
+ executeStepRef: options.executeStepRef ?? "fixtures",
2310
+ cleanupBranches: options.cleanupBranches ?? true,
2311
+ candidates: options.candidates.map((candidate) => ({
2312
+ id: candidate.id,
2313
+ ref: candidate.ref,
2314
+ policy: candidate.policy ?? null,
2315
+ baseScope: candidate.baseScope ?? null,
2316
+ externalCostUsdPerSequence: candidate.externalCostUsdPerSequence ?? 0,
2317
+ externalRecoveryCostUsdPerAttempt: candidate.externalRecoveryCostUsdPerAttempt ?? 0
2318
+ })).sort((a, b) => a.id.localeCompare(b.id))
2319
+ })
2320
+ );
2321
+ }
2322
+
2323
+ // src/memory/graphiti.ts
2324
+ import { randomUUID as randomUUID3 } from "crypto";
2325
+ import { canonicalJson as canonicalJson5 } from "@tangle-network/agent-eval";
2326
+ var DEFAULT_GRAPHITI_INGESTION_TIMEOUT_MS = 12e4;
2327
+ function createGraphitiMemoryAdapter(options) {
2328
+ assertGraphitiOptions(options);
2329
+ const id = options.id ?? "graphiti";
2330
+ const consistency = options.consistency ?? "visible";
2331
+ const tools = graphitiToolNames(options);
2332
+ const queued = /* @__PURE__ */ new Map();
2333
+ const adapter = {
2334
+ id,
2335
+ branchIsolation: consistency === "visible" ? {
2336
+ mode: "scoped",
2337
+ processExitSafe: false,
2338
+ recoveryDelayMs: options.ingestionTimeoutMs ?? DEFAULT_GRAPHITI_INGESTION_TIMEOUT_MS
2339
+ } : {
2340
+ mode: "unsupported",
2341
+ reason: 'queued Graphiti writes can become visible after a process restart; use consistency="visible" for branch experiments'
2342
+ },
2343
+ async search(query, searchOptions = {}) {
2344
+ if (searchOptions.minScore !== void 0 && !Number.isFinite(searchOptions.minScore)) {
2345
+ throw new Error(`${id}: minScore must be finite`);
2346
+ }
2347
+ const scope = mergeScopes3(options.defaultScope, searchOptions.scope);
2348
+ const groupId = graphitiGroupId(id, scope);
2349
+ const modes = new Set(options.search ?? ["facts", "nodes"]);
2350
+ const kinds = searchOptions.kinds && searchOptions.kinds.length > 0 ? new Set(searchOptions.kinds) : null;
2351
+ const searchFacts = modes.has("facts") && (!kinds || kinds.has("fact"));
2352
+ const searchNodes = modes.has("nodes") && (!kinds || kinds.has("entity"));
2353
+ const limit = searchOptions.limit ?? 10;
2354
+ if (!Number.isSafeInteger(limit) || limit < 0) {
2355
+ throw new Error(`${id}: search limit must be a non-negative safe integer`);
2356
+ }
2357
+ if (limit === 0) return [];
2358
+ const [factPayload, nodePayload] = await Promise.all([
2359
+ searchFacts ? callGraphiti(options.client, tools.searchFacts, {
2360
+ query,
2361
+ group_ids: [groupId],
2362
+ max_facts: limit
2363
+ }) : void 0,
2364
+ searchNodes ? callGraphiti(options.client, tools.searchNodes, {
2365
+ query,
2366
+ group_ids: [groupId],
2367
+ max_nodes: limit,
2368
+ ...searchOptions.metadata?.entityTypes ? { entity_types: searchOptions.metadata.entityTypes } : {}
2369
+ }) : void 0
2370
+ ]);
2371
+ const hits = mergeRankedMemoryHits(
2372
+ [normalizeGraphitiFacts(factPayload, id), normalizeGraphitiNodes(nodePayload, id)].map(
2373
+ (group) => group.filter((hit) => {
2374
+ if (searchOptions.minScore === void 0) return true;
2375
+ const score = hit.normalizedScore ?? hit.score;
2376
+ return score !== void 0 && score >= searchOptions.minScore;
2377
+ })
2378
+ ),
2379
+ limit
2380
+ );
2381
+ await enrichGraphitiProvenance(options, groupId, hits);
2382
+ return hits;
2383
+ },
2384
+ async getContext(query, searchOptions = {}) {
2385
+ return defaultGetMemoryContext(adapter, query, searchOptions);
2386
+ },
2387
+ async write(input) {
2388
+ const scope = mergeScopes3(options.defaultScope, input.scope);
2389
+ const groupId = graphitiGroupId(id, scope);
2390
+ const eventKey = canonicalJson5(
2391
+ stripUndefined2({
2392
+ id: input.id,
2393
+ kind: input.kind,
2394
+ text: input.text,
2395
+ title: input.title,
2396
+ role: input.role,
2397
+ metadata: input.metadata
2398
+ })
2399
+ );
2400
+ const episodeUuid = input.id === void 0 ? randomUUID3() : deterministicUuid(`${id}:${groupId}:${eventKey}`);
2401
+ const name = input.title ?? `${input.kind}:${input.id ?? stableId("event", eventKey)}`;
2402
+ const sourceDescription = canonicalJson5(
2403
+ stripUndefined2({
2404
+ adapter: "agent-knowledge",
2405
+ memoryKind: input.kind,
2387
2406
  eventId: input.id,
2388
2407
  actorId: input.metadata?.actorId,
2389
2408
  sessionId: scope.sessionId,
@@ -2416,797 +2435,329 @@ function createGraphitiMemoryAdapter(options) {
2416
2435
  groupId,
2417
2436
  consistency,
2418
2437
  queued: consistency === "queued"
2419
- }
2420
- };
2421
- return {
2422
- ...result,
2423
- sourceRecord: memoryWriteResultToSourceRecord(result, input.text, { scope })
2424
- };
2425
- },
2426
- async clear(scope) {
2427
- const groupId = graphitiGroupId(id, mergeScopes3(options.defaultScope, scope));
2428
- for (const [episodeUuid, pending] of queued) {
2429
- if (pending.groupId !== groupId) continue;
2430
- await waitForGraphitiEpisode(options, groupId, episodeUuid);
2431
- queued.delete(episodeUuid);
2432
- }
2433
- await callGraphiti(options.client, tools.clearGraph, { group_ids: [groupId] });
2434
- },
2435
- async flush() {
2436
- for (const [episodeUuid, pending] of queued) {
2437
- await waitForGraphitiEpisode(options, pending.groupId, episodeUuid);
2438
- queued.delete(episodeUuid);
2439
- }
2440
- }
2441
- };
2442
- return adapter;
2443
- }
2444
- async function callGraphiti(client, name, args) {
2445
- const raw = await client.callTool({ name, arguments: args });
2446
- const payload = graphitiPayload(raw);
2447
- if (isRecord(payload) && typeof payload.error === "string") {
2448
- throw new Error(`Graphiti ${name}: ${payload.error}`);
2449
- }
2450
- return payload;
2451
- }
2452
- function graphitiPayload(raw) {
2453
- if (!isRecord(raw)) return raw;
2454
- if (raw.isError === true) throw new Error(`Graphiti MCP: ${toolText(raw) || "tool call failed"}`);
2455
- if (isRecord(raw.structuredContent)) return unwrapResult(raw.structuredContent);
2456
- const text = toolText(raw);
2457
- if (text) {
2458
- try {
2459
- return unwrapResult(JSON.parse(text));
2460
- } catch {
2461
- return { message: text };
2462
- }
2463
- }
2464
- return unwrapResult(raw);
2465
- }
2466
- function unwrapResult(value) {
2467
- if (isRecord(value) && Object.keys(value).length === 1 && "result" in value) {
2468
- return value.result;
2469
- }
2470
- return value;
2471
- }
2472
- function toolText(raw) {
2473
- if (!Array.isArray(raw.content)) return "";
2474
- return raw.content.filter(isRecord).map((part) => typeof part.text === "string" ? part.text : "").filter(Boolean).join("\n");
2475
- }
2476
- function normalizeGraphitiFacts(raw, adapterId) {
2477
- const facts = isRecord(raw) && Array.isArray(raw.facts) ? raw.facts.filter(isRecord) : [];
2478
- return facts.flatMap((fact) => {
2479
- const text = stringField(fact, ["fact", "text"]);
2480
- if (!text) return [];
2481
- const factId = stringField(fact, ["uuid", "id"]) ?? stableId("fact", canonicalJson4(stripUndefined2({ adapterId, text, fact })));
2482
- const episodeUuids = stringArray2(fact.episodes ?? fact.episode_uuids);
2483
- const score = numberField(fact, ["score", "relevance_score"]);
2484
- const normalizedScore = normalizedScoreOf(score);
2485
- return [
2486
- {
2487
- id: factId,
2488
- uri: `memory://${adapterId}/fact/${factId}`,
2489
- kind: "fact",
2490
- text,
2491
- ...stringField(fact, ["name"]) ? { title: stringField(fact, ["name"]) } : {},
2492
- ...stringField(fact, ["created_at"]) ? { createdAt: stringField(fact, ["created_at"]) } : {},
2493
- ...stringField(fact, ["invalid_at"]) ? { validUntil: stringField(fact, ["invalid_at"]) } : stringField(fact, ["expired_at"]) ? { validUntil: stringField(fact, ["expired_at"]) } : {},
2494
- ...score !== void 0 ? { score } : {},
2495
- ...normalizedScore !== void 0 ? { normalizedScore } : {},
2496
- metadata: {
2497
- provider: "graphiti",
2498
- groupId: fact.group_id,
2499
- sourceNodeUuid: fact.source_node_uuid,
2500
- targetNodeUuid: fact.target_node_uuid,
2501
- validAt: fact.valid_at,
2502
- invalidAt: fact.invalid_at,
2503
- episodeUuids,
2504
- attributes: fact.attributes
2505
- }
2506
- }
2507
- ];
2508
- });
2509
- }
2510
- function normalizeGraphitiNodes(raw, adapterId) {
2511
- const nodes = isRecord(raw) && Array.isArray(raw.nodes) ? raw.nodes.filter(isRecord) : [];
2512
- return nodes.flatMap((node) => {
2513
- const name = stringField(node, ["name"]);
2514
- const summary = stringField(node, ["summary"]);
2515
- const text = summary ?? name;
2516
- if (!text) return [];
2517
- const nodeId = stringField(node, ["uuid", "id"]) ?? stableId("node", canonicalJson4(stripUndefined2({ adapterId, text, node })));
2518
- const score = numberField(node, ["score", "relevance_score"]);
2519
- const normalizedScore = normalizedScoreOf(score);
2520
- return [
2521
- {
2522
- id: nodeId,
2523
- uri: `memory://${adapterId}/entity/${nodeId}`,
2524
- kind: "entity",
2525
- text,
2526
- ...name ? { title: name } : {},
2527
- ...stringField(node, ["created_at"]) ? { createdAt: stringField(node, ["created_at"]) } : {},
2528
- ...score !== void 0 ? { score } : {},
2529
- ...normalizedScore !== void 0 ? { normalizedScore } : {},
2530
- metadata: {
2531
- provider: "graphiti",
2532
- groupId: node.group_id,
2533
- labels: node.labels,
2534
- attributes: node.attributes
2535
- }
2536
- }
2537
- ];
2538
- });
2539
- }
2540
- async function enrichGraphitiProvenance(options, groupId, hits) {
2541
- const wanted = new Set(hits.flatMap((hit) => stringArray2(hit.metadata?.episodeUuids)));
2542
- if (wanted.size === 0) return;
2543
- const scan = await scanGraphitiEpisodes(
2544
- options,
2545
- groupId,
2546
- wanted,
2547
- options.provenanceEpisodeLimit ?? options.episodeScanLimit ?? 1e4
2548
- );
2549
- const episodes = scan.episodes;
2550
- const byId = new Map(episodes.map((episode) => [episode.uuid, episode]));
2551
- for (const hit of hits) {
2552
- const episodeUuids = stringArray2(hit.metadata?.episodeUuids);
2553
- const sourceEpisodes = episodeUuids.map((uuid) => byId.get(uuid)).filter((episode) => episode !== void 0);
2554
- const provenanceIncomplete = episodeUuids.some((uuid) => !byId.has(uuid));
2555
- if (sourceEpisodes.length === 0) {
2556
- if (provenanceIncomplete) {
2557
- hit.metadata = { ...hit.metadata, provenanceIncomplete: true };
2558
- }
2559
- continue;
2560
- }
2561
- const descriptions = sourceEpisodes.map((episode) => parseDescription(episode.source_description)).filter(isRecord);
2562
- const first = descriptions[0];
2563
- hit.metadata = {
2564
- ...hit.metadata,
2565
- sourceEpisodes,
2566
- ...provenanceIncomplete || !scan.complete ? { provenanceIncomplete: true } : {},
2567
- ...first ?? {},
2568
- eventIds: uniqueStrings2(descriptions.map((value) => value.eventId)),
2569
- actorIds: uniqueStrings2(descriptions.map((value) => value.actorId)),
2570
- ...typeof first?.eventId === "string" ? { eventId: first.eventId } : {},
2571
- ...typeof first?.actorId === "string" ? { actorId: first.actorId } : {}
2572
- };
2573
- }
2574
- }
2575
- async function getGraphitiEpisodes(options, groupId, limit) {
2576
- const raw = await callGraphiti(options.client, graphitiToolNames(options).getEpisodes, {
2577
- group_ids: [groupId],
2578
- max_episodes: limit
2579
- });
2580
- if (!isRecord(raw) || !Array.isArray(raw.episodes)) return [];
2581
- return raw.episodes.filter(isRecord).flatMap((episode) => {
2582
- const uuid = stringField(episode, ["uuid", "id"]);
2583
- return uuid ? [{ ...episode, uuid }] : [];
2584
- });
2585
- }
2586
- async function scanGraphitiEpisodes(options, groupId, wanted, maximum) {
2587
- let limit = Math.min(100, maximum);
2588
- for (; ; ) {
2589
- const episodes = await getGraphitiEpisodes(options, groupId, limit);
2590
- const foundAll = [...wanted].every((uuid) => episodes.some((episode) => episode.uuid === uuid));
2591
- if (foundAll || episodes.length < limit) return { episodes, complete: foundAll };
2592
- if (limit === maximum) return { episodes, complete: false };
2593
- limit = Math.min(maximum, limit * 2);
2594
- }
2595
- }
2596
- async function waitForGraphitiEpisode(options, groupId, episodeUuid) {
2597
- const timeoutMs = options.ingestionTimeoutMs ?? DEFAULT_GRAPHITI_INGESTION_TIMEOUT_MS;
2598
- const pollIntervalMs = options.pollIntervalMs ?? 500;
2599
- const maximum = options.episodeScanLimit ?? 1e4;
2600
- const deadline = Date.now() + timeoutMs;
2601
- let scanLimitReached = false;
2602
- for (; ; ) {
2603
- const scan = await scanGraphitiEpisodes(options, groupId, /* @__PURE__ */ new Set([episodeUuid]), maximum);
2604
- if (scan.complete) return;
2605
- scanLimitReached ||= scan.episodes.length >= maximum;
2606
- const remainingMs = deadline - Date.now();
2607
- if (remainingMs <= 0) break;
2608
- await new Promise((resolve) => setTimeout(resolve, Math.min(pollIntervalMs, remainingMs)));
2609
- }
2610
- if (scanLimitReached) {
2611
- throw new Error(
2612
- `Graphiti episode ${episodeUuid} is outside episodeScanLimit=${maximum}; raise the limit or use a smaller group`
2613
- );
2614
- }
2615
- throw new Error(`Graphiti episode ${episodeUuid} was not visible after ${timeoutMs}ms`);
2616
- }
2617
- function graphitiGroupId(adapterId, scope) {
2618
- return `ak_${sha256(
2619
- canonicalJson4(
2620
- stripUndefined2({
2621
- adapterId,
2622
- tenantId: scope.tenantId,
2623
- userId: scope.userId,
2624
- agentId: scope.agentId,
2625
- teamId: scope.teamId,
2626
- runId: scope.runId,
2627
- sessionId: scope.sessionId,
2628
- namespace: scope.namespace,
2629
- tags: scope.tags
2630
- })
2631
- )
2632
- ).slice(0, 32)}`;
2633
- }
2634
- function graphitiToolNames(options) {
2635
- return {
2636
- addMemory: options.toolNames?.addMemory ?? "add_memory",
2637
- searchFacts: options.toolNames?.searchFacts ?? "search_memory_facts",
2638
- searchNodes: options.toolNames?.searchNodes ?? "search_nodes",
2639
- getEpisodes: options.toolNames?.getEpisodes ?? "get_episodes",
2640
- clearGraph: options.toolNames?.clearGraph ?? "clear_graph"
2641
- };
2642
- }
2643
- function assertGraphitiOptions(options) {
2644
- if (options.id !== void 0 && (typeof options.id !== "string" || !options.id.trim())) {
2645
- throw new Error("Graphiti id must be a non-empty string");
2646
- }
2647
- if (options.backendRef !== void 0 && (typeof options.backendRef !== "string" || !options.backendRef.trim())) {
2648
- throw new Error("Graphiti backendRef must be a non-empty string");
2649
- }
2650
- if (options.consistency !== void 0 && !["queued", "visible"].includes(options.consistency)) {
2651
- throw new Error("Graphiti consistency must be queued or visible");
2652
- }
2653
- for (const [name, value] of [
2654
- ["ingestionTimeoutMs", options.ingestionTimeoutMs],
2655
- ["pollIntervalMs", options.pollIntervalMs],
2656
- ["episodeScanLimit", options.episodeScanLimit],
2657
- ["provenanceEpisodeLimit", options.provenanceEpisodeLimit]
2658
- ]) {
2659
- if (value !== void 0 && (!Number.isSafeInteger(value) || value <= 0)) {
2660
- throw new Error(`Graphiti ${name} must be a positive safe integer`);
2661
- }
2662
- }
2663
- for (const [name, value] of Object.entries(options.toolNames ?? {})) {
2664
- if (typeof value !== "string" || value.trim().length === 0) {
2665
- throw new Error(`Graphiti toolNames.${name} must be a non-empty string`);
2666
- }
2667
- }
2668
- }
2669
- function deterministicUuid(value) {
2670
- const hex = sha256(value);
2671
- return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-5${hex.slice(13, 16)}-a${hex.slice(17, 20)}-${hex.slice(20, 32)}`;
2672
- }
2673
- function parseDescription(value) {
2674
- if (!value) return void 0;
2675
- try {
2676
- return JSON.parse(value);
2677
- } catch {
2678
- return void 0;
2679
- }
2680
- }
2681
- function mergeScopes3(base, extra) {
2682
- return {
2683
- ...base ?? {},
2684
- ...extra ?? {},
2685
- tags: { ...base?.tags ?? {}, ...extra?.tags ?? {} }
2686
- };
2687
- }
2688
- function uniqueStrings2(values) {
2689
- return [...new Set(values.filter((value) => typeof value === "string"))];
2690
- }
2691
- function stringArray2(value) {
2692
- return Array.isArray(value) ? value.filter((entry) => typeof entry === "string") : [];
2693
- }
2694
- function stringField(value, keys) {
2695
- for (const key of keys) {
2696
- const field = value[key];
2697
- if (typeof field === "string" && field.length > 0) return field;
2698
- }
2699
- return void 0;
2700
- }
2701
- function numberField(value, keys) {
2702
- for (const key of keys) {
2703
- const field = value[key];
2704
- if (typeof field === "number" && Number.isFinite(field)) return field;
2705
- }
2706
- return void 0;
2707
- }
2708
- function normalizedScoreOf(score) {
2709
- return score !== void 0 && score >= 0 && score <= 1 ? score : void 0;
2710
- }
2711
- function isRecord(value) {
2712
- return typeof value === "object" && value !== null && !Array.isArray(value);
2713
- }
2714
- function stripUndefined2(value) {
2715
- if (Array.isArray(value)) return value.map(stripUndefined2);
2716
- if (!isRecord(value)) return value;
2717
- return Object.fromEntries(
2718
- Object.entries(value).filter(([, child]) => child !== void 0).map(([key, child]) => [key, stripUndefined2(child)])
2719
- );
2720
- }
2721
- function graphitiMemoryAdapterIdentity(options) {
2722
- if (typeof options.backendRef !== "string" || !options.backendRef.trim()) {
2723
- throw new Error("Graphiti backendRef must be a non-empty string");
2724
- }
2725
- return stableId("graphiti", canonicalJson4(stripUndefined2(options)));
2726
- }
2727
-
2728
- // src/memory/improvement.ts
2729
- import { join as join2 } from "path";
2730
- import { canonicalJson as canonicalJson5 } from "@tangle-network/agent-eval";
2731
- import {
2732
- campaignLineageStore,
2733
- createRunCostLedger as createRunCostLedger2,
2734
- fsCampaignStorage as fsCampaignStorage2,
2735
- heldoutSignificance,
2736
- memLineageStore,
2737
- resolveRunDir as resolveRunDir2,
2738
- runLineageLoop,
2739
- surfaceHash
2740
- } from "@tangle-network/agent-eval/campaign";
2741
- var DEFAULT_CRITICAL_DIMENSIONS = [
2742
- "memory_stale_safe",
2743
- "memory_actor_recall",
2744
- "memory_event_recall"
2745
- ];
2746
- var MEMORY_IMPROVEMENT_IMPLEMENTATION_REF = "agent-knowledge:memory-improvement:v2";
2747
- async function runAgentMemoryImprovement(options) {
2748
- if ("onPromote" in options) {
2749
- throw new Error(
2750
- "memory improvement onPromote was removed; use activation.readCurrent and activation.compareAndSet"
2751
- );
2752
- }
2753
- if (options.seeds.length === 0) throw new Error("memory improvement requires seed configs");
2754
- if (options.trainSequences.length === 0) {
2755
- throw new Error("memory improvement requires training sequences");
2756
- }
2757
- if (options.holdoutSequences.length === 0) {
2758
- throw new Error("memory improvement requires holdout sequences");
2759
- }
2760
- const trainIds = new Set(options.trainSequences.map((sequence) => sequence.id));
2761
- const overlap = options.holdoutSequences.map((sequence) => sequence.id).filter((id) => trainIds.has(id));
2762
- if (overlap.length > 0) {
2763
- throw new Error(`memory improvement train/holdout overlap: ${overlap.join(", ")}`);
2764
- }
2765
- const trainFingerprints = new Map(
2766
- options.trainSequences.map((sequence) => [memorySequenceFingerprint(sequence), sequence.id])
2767
- );
2768
- const duplicateHistories = options.holdoutSequences.flatMap((sequence) => {
2769
- const trainId = trainFingerprints.get(memorySequenceFingerprint(sequence));
2770
- return trainId ? [`${trainId}/${sequence.id}`] : [];
2771
- });
2772
- if (duplicateHistories.length > 0) {
2773
- throw new Error(
2774
- `memory improvement train/holdout histories duplicate content: ${duplicateHistories.join(", ")}`
2775
- );
2776
- }
2777
- if (typeof options.improvementRef !== "string" || !options.improvementRef.trim()) {
2778
- throw new Error("memory improvement improvementRef must be a non-empty string");
2779
- }
2780
- assertMemoryImprovementOptions(options);
2781
- const storage = options.storage ?? fsCampaignStorage2();
2782
- const runDir = resolveRunDir2(options.runDir, options.repo);
2783
- storage.ensureDir(runDir);
2784
- const lease = await acquireAgentMemoryRunLease({
2785
- experimentId: options.experimentId,
2786
- runDir,
2787
- storage,
2788
- customStorage: options.storage !== void 0,
2789
- lockFileName: "memory-improvement.lock",
2790
- label: "memory improvement",
2791
- controllerMode: options.controllerMode,
2792
- acquireRunLease: options.acquireRunLease
2793
- });
2794
- let result;
2795
- let primaryError;
2796
- try {
2797
- result = await runAgentMemoryImprovementOwned(options, storage, runDir, lease);
2798
- } catch (error) {
2799
- primaryError = error;
2800
- }
2801
- let releaseError;
2802
- try {
2803
- await lease.release();
2804
- } catch (error) {
2805
- releaseError = error;
2806
- }
2807
- if (primaryError && releaseError) {
2808
- throw new AggregateError(
2809
- [primaryError, releaseError],
2810
- "memory improvement failed and its controller lease could not be released"
2811
- );
2812
- }
2813
- if (primaryError) throw primaryError;
2814
- if (releaseError) throw releaseError;
2815
- if (!result) throw new Error("memory improvement produced no result");
2816
- return result;
2817
- }
2818
- async function runAgentMemoryImprovementOwned(options, storage, runDir, lease) {
2819
- await lease.assertOwned();
2820
- const serializeRaw = options.serializeConfig ?? ((config) => canonicalJson5(config));
2821
- const parseRaw = options.parseConfig ?? ((surface) => JSON.parse(surface));
2822
- const serialize = (config) => serializeMemoryConfig(serializeRaw, config);
2823
- const parse = (surface) => parseMemoryConfig(parseRaw, surface);
2824
- for (const seed of options.seeds) assertMemoryConfigRoundTrip(seed.config, serialize, parse);
2825
- if (options.activation && !storage.append) {
2826
- throw new Error("memory activation requires CampaignStorage.append");
2827
- }
2828
- if (options.resumable !== false && !options.lineageStore && !storage.append) {
2829
- throw new Error("resumable memory improvement requires CampaignStorage.append");
2830
- }
2831
- assertMemoryImprovementIdentity(options, storage, runDir, serialize);
2832
- const costLedger = createRunCostLedger2({
2833
- storage,
2834
- runDir,
2835
- costCeilingUsd: options.maxTotalCostUsd ?? 0
2836
- });
2837
- reconcileInterruptedRunPaidCalls(costLedger, "memory improvement run");
2838
- assertNoInterruptedPaidCalls(costLedger, "memory improvement recovery");
2839
- const trainScenarios = options.trainSequences.map((sequence) => ({
2840
- id: sequence.id,
2841
- kind: "agent-memory-config-search",
2842
- sequenceId: sequence.id
2843
- }));
2844
- const evaluations = /* @__PURE__ */ new Map();
2845
- const evaluateSurface = async (surface) => {
2846
- await lease.assertOwned();
2847
- const text = requireStringSurface(surface);
2848
- const config = parse(text);
2849
- const canonicalSurface = serialize(config);
2850
- const hash = surfaceHash(canonicalSurface);
2851
- let pending = evaluations.get(hash);
2852
- if (!pending) {
2853
- pending = (async () => {
2854
- const candidate = await buildCandidate(options, config, hash, `search-${hash}`);
2855
- await lease.assertOwned();
2856
- const experiment = await runAgentMemoryExperiment({
2857
- ...experimentOptions(options, costLedger, storage, lease),
2858
- experimentId: `${options.experimentId}:search:${hash}`,
2859
- sequences: options.trainSequences,
2860
- candidates: [candidate],
2861
- runDir: join2(runDir, "search", hash),
2862
- costPhase: `memory.search.${hash}`
2863
- });
2864
- await lease.assertOwned();
2865
- return experiment;
2866
- })();
2867
- evaluations.set(hash, pending);
2868
- void pending.catch(() => evaluations.delete(hash));
2869
- }
2870
- const result2 = await pending;
2871
- await lease.assertOwned();
2872
- const row = result2.rows[0];
2873
- if (!row || row.cellsFailed > 0) {
2874
- throw new Error(`${hash}: memory candidate did not complete every training cell`);
2875
- }
2876
- return {
2877
- score: row.scoreMean,
2878
- scoreVector: sequenceScores(result2, options.trainSequences, row.candidateId)
2879
- };
2880
- };
2881
- const lineageStore = options.lineageStore ?? (options.resumable === false ? memLineageStore() : campaignLineageStore(storage, join2(runDir, "lineage.jsonl")));
2882
- const lineageOptions = {
2883
- seeds: options.seeds.map((seed) => ({
2884
- surface: serialize(seed.config),
2885
- track: seed.track,
2886
- proposer: seed.proposer,
2887
- ...seed.vision !== void 0 ? { vision: seed.vision } : {}
2888
- })),
2889
- scenarios: trainScenarios,
2890
- proposer: withCostContext(options.proposer, costLedger, lease, "default"),
2891
- proposers: options.proposers ? Object.fromEntries(
2892
- Object.entries(options.proposers).map(([name, proposer]) => [
2893
- name,
2894
- withCostContext(proposer, costLedger, lease, name)
2895
- ])
2896
- ) : void 0,
2897
- scoreSurface: evaluateSurface,
2898
- governor: options.governor ? withGovernorCostContext(options.governor, costLedger, lease) : void 0,
2899
- budget: {
2900
- ...options.budget,
2901
- maxNodes: options.seeds.length + options.budget.maxSteps
2902
- },
2903
- store: lineageStore,
2904
- populationSize: options.populationSize,
2905
- candidateConcurrency: options.candidateConcurrency
2906
- };
2907
- const search = await runLineageLoop(lineageOptions);
2908
- await lease.assertOwned();
2909
- const best = search.best;
2910
- if (!best) throw new Error("memory improvement produced no measured config");
2911
- const baselineSurface = serialize(options.seeds[0].config);
2912
- const baselineHash = surfaceHash(baselineSurface);
2913
- const winnerConfig = parse(requireStringSurface(best.surface));
2914
- const winnerSurface = serialize(winnerConfig);
2915
- const winnerHash = surfaceHash(winnerSurface);
2916
- const baselineConfig = parse(baselineSurface);
2917
- let holdout;
2918
- let decision;
2919
- if (winnerHash === baselineHash) {
2920
- const baselineMeasurement = await evaluateSurface(baselineSurface);
2921
- decision = {
2922
- status: "no-change",
2923
- reasons: ["search did not find a config better than the baseline"],
2924
- baselineScore: baselineMeasurement.score,
2925
- winnerScore: baselineMeasurement.score,
2926
- lift: 0,
2927
- criticalDimensions: []
2928
- };
2929
- } else {
2930
- await lease.assertOwned();
2931
- const [baselineCandidate, winnerCandidate] = await Promise.all([
2932
- buildCandidate(options, baselineConfig, baselineHash, "baseline"),
2933
- buildCandidate(options, winnerConfig, winnerHash, "winner")
2934
- ]);
2935
- await lease.assertOwned();
2936
- holdout = await runAgentMemoryExperiment({
2937
- ...experimentOptions(options, costLedger, storage, lease),
2938
- experimentId: `${options.experimentId}:holdout`,
2939
- sequences: options.holdoutSequences,
2940
- candidates: [baselineCandidate, winnerCandidate],
2941
- runDir: join2(runDir, "holdout"),
2942
- costPhase: "memory.holdout"
2943
- });
2944
- decision = decidePromotion({
2945
- options,
2946
- result: holdout,
2947
- baselineId: baselineCandidate.id,
2948
- winnerId: winnerCandidate.id
2949
- });
2950
- }
2951
- const resultJsonPath = join2(runDir, "memory-improvement-result.json");
2952
- const activationRef = options.activation?.ref ?? "not-configured";
2953
- const activationId = `memory-activation-${surfaceHash(
2954
- canonicalJson5({
2955
- experimentId: options.experimentId,
2956
- improvementRef: options.improvementRef,
2957
- activationRef,
2958
- baselineSurfaceHash: baselineHash,
2959
- winnerSurfaceHash: winnerHash,
2960
- holdoutManifestHash: holdout?.campaign.manifestHash ?? null,
2961
- promotionPolicy: normalizedPromotionPolicy(options)
2962
- })
2963
- )}`;
2964
- const activationJournalDir = join2(runDir, "activations");
2965
- const activationJournalPath = join2(activationJournalDir, `${activationId}.jsonl`);
2966
- const activationEligible = decision.status === "promote" && holdout !== void 0;
2967
- const activationEventIdentity = holdout ? {
2968
- schema: 1,
2969
- activationId,
2970
- experimentId: options.experimentId,
2971
- activationRef,
2972
- baselineSurfaceHash: baselineHash,
2973
- winnerSurfaceHash: winnerHash,
2974
- holdoutManifestHash: holdout.campaign.manifestHash
2975
- } : void 0;
2976
- const activationJournal = activationEligible && activationEventIdentity ? readMemoryActivationJournal(storage, activationJournalPath, activationEventIdentity) : { prepared: false };
2977
- const activation = {
2978
- id: activationId,
2979
- status: !activationEligible ? "not-eligible" : activationJournal.activated ? "already-activated" : options.activation ? "pending" : "not-configured",
2980
- journalPath: activationJournalPath
2981
- };
2982
- const result = {
2983
- lineage: search.lineage,
2984
- baselineConfig,
2985
- winnerConfig,
2986
- baselineSurface,
2987
- winnerSurface,
2988
- baselineSurfaceHash: baselineHash,
2989
- winnerSurfaceHash: winnerHash,
2990
- decision,
2991
- activation,
2992
- ...holdout ? { holdout } : {},
2993
- totalCostUsd: costLedger.summary().totalCostUsd,
2994
- resultJsonPath
2995
- };
2996
- await lease.assertOwned();
2997
- writeMemoryImprovementResult(storage, options.experimentId, result);
2998
- if (activationEligible && !activationJournal.activated && activationEventIdentity && holdout && options.activation) {
2999
- const activationDriver = options.activation;
3000
- const activationTimeoutMs = options.activationTimeoutMs ?? 6e4;
3001
- const hadPreparedEvent = activationJournal.prepared;
3002
- if (!hadPreparedEvent) {
3003
- await lease.assertOwned();
3004
- storage.ensureDir(activationJournalDir);
3005
- appendMemoryActivationEvent(storage, activationJournalPath, {
3006
- ...activationEventIdentity,
3007
- status: "prepared",
3008
- recordedAt: (options.now ?? (() => /* @__PURE__ */ new Date()))().toISOString()
3009
- });
2438
+ }
2439
+ };
2440
+ return {
2441
+ ...result,
2442
+ sourceRecord: memoryWriteResultToSourceRecord(result, input.text, { scope })
2443
+ };
2444
+ },
2445
+ async clear(scope) {
2446
+ const groupId = graphitiGroupId(id, mergeScopes3(options.defaultScope, scope));
2447
+ for (const [episodeUuid, pending] of queued) {
2448
+ if (pending.groupId !== groupId) continue;
2449
+ await waitForGraphitiEpisode(options, groupId, episodeUuid);
2450
+ queued.delete(episodeUuid);
2451
+ }
2452
+ await callGraphiti(options.client, tools.clearGraph, { group_ids: [groupId] });
2453
+ },
2454
+ async flush() {
2455
+ for (const [episodeUuid, pending] of queued) {
2456
+ await waitForGraphitiEpisode(options, pending.groupId, episodeUuid);
2457
+ queued.delete(episodeUuid);
2458
+ }
3010
2459
  }
3011
- await lease.assertOwned();
3012
- const currentConfig = await runBoundedMemoryLifecycle({
3013
- operation: `${activationDriver.ref}: read current memory configuration`,
3014
- timeoutMs: activationTimeoutMs,
3015
- run: () => activationDriver.readCurrent()
3016
- });
3017
- await lease.assertOwned();
3018
- const currentHash = surfaceHash(serialize(currentConfig));
3019
- if (currentHash !== baselineHash && currentHash !== winnerHash) {
3020
- throw new Error(
3021
- `memory activation target '${activationDriver.ref}' changed concurrently; expected '${baselineHash}' or '${winnerHash}', found '${currentHash}'`
3022
- );
2460
+ };
2461
+ return adapter;
2462
+ }
2463
+ async function callGraphiti(client, name, args) {
2464
+ const raw = await client.callTool({ name, arguments: args });
2465
+ const payload = graphitiPayload(raw);
2466
+ if (isRecord(payload) && typeof payload.error === "string") {
2467
+ throw new Error(`Graphiti ${name}: ${payload.error}`);
2468
+ }
2469
+ return payload;
2470
+ }
2471
+ function graphitiPayload(raw) {
2472
+ if (!isRecord(raw)) return raw;
2473
+ if (raw.isError === true) throw new Error(`Graphiti MCP: ${toolText(raw) || "tool call failed"}`);
2474
+ if (isRecord(raw.structuredContent)) return unwrapResult(raw.structuredContent);
2475
+ const text = toolText(raw);
2476
+ if (text) {
2477
+ try {
2478
+ return unwrapResult(JSON.parse(text));
2479
+ } catch {
2480
+ return { message: text };
3023
2481
  }
3024
- let outcome;
3025
- if (currentHash === winnerHash) {
3026
- outcome = hadPreparedEvent ? "recovered" : "already-current";
3027
- activation.status = "recovered";
3028
- } else {
3029
- let compareError;
3030
- try {
3031
- await runBoundedMemoryLifecycle({
3032
- operation: `${activationDriver.ref}: activate memory configuration`,
3033
- timeoutMs: activationTimeoutMs,
3034
- run: () => activationDriver.compareAndSet({
3035
- activationId,
3036
- expectedConfig: baselineConfig,
3037
- expectedSurfaceHash: baselineHash,
3038
- config: winnerConfig,
3039
- surfaceHash: winnerHash,
3040
- decision,
3041
- lineage: search.lineage,
3042
- holdout
3043
- })
3044
- });
3045
- } catch (error) {
3046
- compareError = error;
3047
- }
3048
- await lease.assertOwned();
3049
- let observedConfig;
3050
- try {
3051
- observedConfig = await runBoundedMemoryLifecycle({
3052
- operation: `${activationDriver.ref}: confirm memory configuration`,
3053
- timeoutMs: activationTimeoutMs,
3054
- run: () => activationDriver.readCurrent()
3055
- });
3056
- } catch (error) {
3057
- if (compareError) {
3058
- throw new AggregateError(
3059
- [compareError, error],
3060
- `memory activation '${activationId}' failed and its live state could not be confirmed`
3061
- );
2482
+ }
2483
+ return unwrapResult(raw);
2484
+ }
2485
+ function unwrapResult(value) {
2486
+ if (isRecord(value) && Object.keys(value).length === 1 && "result" in value) {
2487
+ return value.result;
2488
+ }
2489
+ return value;
2490
+ }
2491
+ function toolText(raw) {
2492
+ if (!Array.isArray(raw.content)) return "";
2493
+ return raw.content.filter(isRecord).map((part) => typeof part.text === "string" ? part.text : "").filter(Boolean).join("\n");
2494
+ }
2495
+ function normalizeGraphitiFacts(raw, adapterId) {
2496
+ const facts = isRecord(raw) && Array.isArray(raw.facts) ? raw.facts.filter(isRecord) : [];
2497
+ return facts.flatMap((fact) => {
2498
+ const text = stringField(fact, ["fact", "text"]);
2499
+ if (!text) return [];
2500
+ const factId = stringField(fact, ["uuid", "id"]) ?? stableId("fact", canonicalJson5(stripUndefined2({ adapterId, text, fact })));
2501
+ const episodeUuids = stringArray2(fact.episodes ?? fact.episode_uuids);
2502
+ const score = numberField(fact, ["score", "relevance_score"]);
2503
+ const normalizedScore = normalizedScoreOf(score);
2504
+ return [
2505
+ {
2506
+ id: factId,
2507
+ uri: `memory://${adapterId}/fact/${factId}`,
2508
+ kind: "fact",
2509
+ text,
2510
+ ...stringField(fact, ["name"]) ? { title: stringField(fact, ["name"]) } : {},
2511
+ ...stringField(fact, ["created_at"]) ? { createdAt: stringField(fact, ["created_at"]) } : {},
2512
+ ...stringField(fact, ["invalid_at"]) ? { validUntil: stringField(fact, ["invalid_at"]) } : stringField(fact, ["expired_at"]) ? { validUntil: stringField(fact, ["expired_at"]) } : {},
2513
+ ...score !== void 0 ? { score } : {},
2514
+ ...normalizedScore !== void 0 ? { normalizedScore } : {},
2515
+ metadata: {
2516
+ provider: "graphiti",
2517
+ groupId: fact.group_id,
2518
+ sourceNodeUuid: fact.source_node_uuid,
2519
+ targetNodeUuid: fact.target_node_uuid,
2520
+ validAt: fact.valid_at,
2521
+ invalidAt: fact.invalid_at,
2522
+ episodeUuids,
2523
+ attributes: fact.attributes
3062
2524
  }
3063
- throw error;
3064
2525
  }
3065
- await lease.assertOwned();
3066
- const observedHash = surfaceHash(serialize(observedConfig));
3067
- if (observedHash !== winnerHash) {
3068
- const mismatch = new Error(
3069
- `memory activation '${activationId}' did not install the measured winner; found '${observedHash}'`
3070
- );
3071
- if (compareError) {
3072
- throw new AggregateError(
3073
- [compareError, mismatch],
3074
- `memory activation '${activationId}' failed without applying the measured winner`
3075
- );
2526
+ ];
2527
+ });
2528
+ }
2529
+ function normalizeGraphitiNodes(raw, adapterId) {
2530
+ const nodes = isRecord(raw) && Array.isArray(raw.nodes) ? raw.nodes.filter(isRecord) : [];
2531
+ return nodes.flatMap((node) => {
2532
+ const name = stringField(node, ["name"]);
2533
+ const summary = stringField(node, ["summary"]);
2534
+ const text = summary ?? name;
2535
+ if (!text) return [];
2536
+ const nodeId = stringField(node, ["uuid", "id"]) ?? stableId("node", canonicalJson5(stripUndefined2({ adapterId, text, node })));
2537
+ const score = numberField(node, ["score", "relevance_score"]);
2538
+ const normalizedScore = normalizedScoreOf(score);
2539
+ return [
2540
+ {
2541
+ id: nodeId,
2542
+ uri: `memory://${adapterId}/entity/${nodeId}`,
2543
+ kind: "entity",
2544
+ text,
2545
+ ...name ? { title: name } : {},
2546
+ ...stringField(node, ["created_at"]) ? { createdAt: stringField(node, ["created_at"]) } : {},
2547
+ ...score !== void 0 ? { score } : {},
2548
+ ...normalizedScore !== void 0 ? { normalizedScore } : {},
2549
+ metadata: {
2550
+ provider: "graphiti",
2551
+ groupId: node.group_id,
2552
+ labels: node.labels,
2553
+ attributes: node.attributes
3076
2554
  }
3077
- throw mismatch;
3078
2555
  }
3079
- outcome = compareError ? "recovered" : "applied";
3080
- activation.status = compareError ? "recovered" : "activated";
2556
+ ];
2557
+ });
2558
+ }
2559
+ async function enrichGraphitiProvenance(options, groupId, hits) {
2560
+ const wanted = new Set(hits.flatMap((hit) => stringArray2(hit.metadata?.episodeUuids)));
2561
+ if (wanted.size === 0) return;
2562
+ const scan = await scanGraphitiEpisodes(
2563
+ options,
2564
+ groupId,
2565
+ wanted,
2566
+ options.provenanceEpisodeLimit ?? options.episodeScanLimit ?? 1e4
2567
+ );
2568
+ const episodes = scan.episodes;
2569
+ const byId = new Map(episodes.map((episode) => [episode.uuid, episode]));
2570
+ for (const hit of hits) {
2571
+ const episodeUuids = stringArray2(hit.metadata?.episodeUuids);
2572
+ const sourceEpisodes = episodeUuids.map((uuid) => byId.get(uuid)).filter((episode) => episode !== void 0);
2573
+ const provenanceIncomplete = episodeUuids.some((uuid) => !byId.has(uuid));
2574
+ if (sourceEpisodes.length === 0) {
2575
+ if (provenanceIncomplete) {
2576
+ hit.metadata = { ...hit.metadata, provenanceIncomplete: true };
2577
+ }
2578
+ continue;
3081
2579
  }
3082
- await lease.assertOwned();
3083
- appendMemoryActivationEvent(storage, activationJournalPath, {
3084
- ...activationEventIdentity,
3085
- status: "activated",
3086
- outcome,
3087
- recordedAt: (options.now ?? (() => /* @__PURE__ */ new Date()))().toISOString()
3088
- });
3089
- writeMemoryImprovementResult(storage, options.experimentId, result);
2580
+ const descriptions = sourceEpisodes.map((episode) => parseDescription(episode.source_description)).filter(isRecord);
2581
+ const first = descriptions[0];
2582
+ hit.metadata = {
2583
+ ...hit.metadata,
2584
+ sourceEpisodes,
2585
+ ...provenanceIncomplete || !scan.complete ? { provenanceIncomplete: true } : {},
2586
+ ...first ?? {},
2587
+ eventIds: uniqueStrings2(descriptions.map((value) => value.eventId)),
2588
+ actorIds: uniqueStrings2(descriptions.map((value) => value.actorId)),
2589
+ ...typeof first?.eventId === "string" ? { eventId: first.eventId } : {},
2590
+ ...typeof first?.actorId === "string" ? { actorId: first.actorId } : {}
2591
+ };
2592
+ }
2593
+ }
2594
+ async function getGraphitiEpisodes(options, groupId, limit) {
2595
+ const raw = await callGraphiti(options.client, graphitiToolNames(options).getEpisodes, {
2596
+ group_ids: [groupId],
2597
+ max_episodes: limit
2598
+ });
2599
+ if (!isRecord(raw) || !Array.isArray(raw.episodes)) return [];
2600
+ return raw.episodes.filter(isRecord).flatMap((episode) => {
2601
+ const uuid = stringField(episode, ["uuid", "id"]);
2602
+ return uuid ? [{ ...episode, uuid }] : [];
2603
+ });
2604
+ }
2605
+ async function scanGraphitiEpisodes(options, groupId, wanted, maximum) {
2606
+ let limit = Math.min(100, maximum);
2607
+ for (; ; ) {
2608
+ const episodes = await getGraphitiEpisodes(options, groupId, limit);
2609
+ const foundAll = [...wanted].every((uuid) => episodes.some((episode) => episode.uuid === uuid));
2610
+ if (foundAll || episodes.length < limit) return { episodes, complete: foundAll };
2611
+ if (limit === maximum) return { episodes, complete: false };
2612
+ limit = Math.min(maximum, limit * 2);
2613
+ }
2614
+ }
2615
+ async function waitForGraphitiEpisode(options, groupId, episodeUuid) {
2616
+ const timeoutMs = options.ingestionTimeoutMs ?? DEFAULT_GRAPHITI_INGESTION_TIMEOUT_MS;
2617
+ const pollIntervalMs = options.pollIntervalMs ?? 500;
2618
+ const maximum = options.episodeScanLimit ?? 1e4;
2619
+ const deadline = Date.now() + timeoutMs;
2620
+ let scanLimitReached = false;
2621
+ for (; ; ) {
2622
+ const scan = await scanGraphitiEpisodes(options, groupId, /* @__PURE__ */ new Set([episodeUuid]), maximum);
2623
+ if (scan.complete) return;
2624
+ scanLimitReached ||= scan.episodes.length >= maximum;
2625
+ const remainingMs = deadline - Date.now();
2626
+ if (remainingMs <= 0) break;
2627
+ await new Promise((resolve) => setTimeout(resolve, Math.min(pollIntervalMs, remainingMs)));
2628
+ }
2629
+ if (scanLimitReached) {
2630
+ throw new Error(
2631
+ `Graphiti episode ${episodeUuid} is outside episodeScanLimit=${maximum}; raise the limit or use a smaller group`
2632
+ );
3090
2633
  }
3091
- return result;
2634
+ throw new Error(`Graphiti episode ${episodeUuid} was not visible after ${timeoutMs}ms`);
3092
2635
  }
3093
- function withCostContext(proposer, costLedger, lease, label) {
2636
+ function graphitiGroupId(adapterId, scope) {
2637
+ return `ak_${sha256(
2638
+ canonicalJson5(
2639
+ stripUndefined2({
2640
+ adapterId,
2641
+ tenantId: scope.tenantId,
2642
+ userId: scope.userId,
2643
+ agentId: scope.agentId,
2644
+ teamId: scope.teamId,
2645
+ runId: scope.runId,
2646
+ sessionId: scope.sessionId,
2647
+ namespace: scope.namespace,
2648
+ tags: scope.tags
2649
+ })
2650
+ )
2651
+ ).slice(0, 32)}`;
2652
+ }
2653
+ function graphitiToolNames(options) {
3094
2654
  return {
3095
- kind: proposer.kind,
3096
- async propose(context) {
3097
- await lease.assertOwned();
3098
- const proposal = await proposer.propose({
3099
- ...context,
3100
- costLedger,
3101
- costPhase: `memory.proposal.${context.track?.id ?? label}`
3102
- });
3103
- await lease.assertOwned();
3104
- return proposal;
3105
- },
3106
- ...proposer.decide ? { decide: (input) => proposer.decide(input) } : {}
2655
+ addMemory: options.toolNames?.addMemory ?? "add_memory",
2656
+ searchFacts: options.toolNames?.searchFacts ?? "search_memory_facts",
2657
+ searchNodes: options.toolNames?.searchNodes ?? "search_nodes",
2658
+ getEpisodes: options.toolNames?.getEpisodes ?? "get_episodes",
2659
+ clearGraph: options.toolNames?.clearGraph ?? "clear_graph"
3107
2660
  };
3108
2661
  }
3109
- function withGovernorCostContext(governor, costLedger, lease) {
3110
- return {
3111
- async decide(context) {
3112
- await lease.assertOwned();
3113
- const decision = await governor.decide({
3114
- ...context,
3115
- costLedger,
3116
- costPhase: "memory.governor"
3117
- });
3118
- await lease.assertOwned();
3119
- return decision;
2662
+ function assertGraphitiOptions(options) {
2663
+ if (options.id !== void 0 && (typeof options.id !== "string" || !options.id.trim())) {
2664
+ throw new Error("Graphiti id must be a non-empty string");
2665
+ }
2666
+ if (options.backendRef !== void 0 && (typeof options.backendRef !== "string" || !options.backendRef.trim())) {
2667
+ throw new Error("Graphiti backendRef must be a non-empty string");
2668
+ }
2669
+ if (options.consistency !== void 0 && !["queued", "visible"].includes(options.consistency)) {
2670
+ throw new Error("Graphiti consistency must be queued or visible");
2671
+ }
2672
+ for (const [name, value] of [
2673
+ ["ingestionTimeoutMs", options.ingestionTimeoutMs],
2674
+ ["pollIntervalMs", options.pollIntervalMs],
2675
+ ["episodeScanLimit", options.episodeScanLimit],
2676
+ ["provenanceEpisodeLimit", options.provenanceEpisodeLimit]
2677
+ ]) {
2678
+ if (value !== void 0 && (!Number.isSafeInteger(value) || value <= 0)) {
2679
+ throw new Error(`Graphiti ${name} must be a positive safe integer`);
3120
2680
  }
3121
- };
2681
+ }
2682
+ for (const [name, value] of Object.entries(options.toolNames ?? {})) {
2683
+ if (typeof value !== "string" || value.trim().length === 0) {
2684
+ throw new Error(`Graphiti toolNames.${name} must be a non-empty string`);
2685
+ }
2686
+ }
3122
2687
  }
3123
- async function buildCandidate(options, config, hash, role) {
3124
- const built = await options.createCandidate({
3125
- config,
3126
- candidateId: `${role}-${hash}`,
3127
- surfaceHash: hash
3128
- });
3129
- return {
3130
- ...built,
3131
- id: `${role}-${hash}`,
3132
- ref: built.ref
3133
- };
2688
+ function deterministicUuid(value) {
2689
+ const hex = sha256(value);
2690
+ return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-5${hex.slice(13, 16)}-a${hex.slice(17, 20)}-${hex.slice(20, 32)}`;
3134
2691
  }
3135
- function experimentOptions(options, costLedger, storage, lease) {
2692
+ function parseDescription(value) {
2693
+ if (!value) return void 0;
2694
+ try {
2695
+ return JSON.parse(value);
2696
+ } catch {
2697
+ return void 0;
2698
+ }
2699
+ }
2700
+ function mergeScopes3(base, extra) {
3136
2701
  return {
3137
- storage,
3138
- seed: options.seed,
3139
- reps: options.reps,
3140
- resumable: options.resumable,
3141
- maxConcurrency: options.sequenceConcurrency,
3142
- dispatchTimeoutMs: options.dispatchTimeoutMs,
3143
- cleanupTimeoutMs: options.cleanupTimeoutMs,
3144
- maxRecoveryAttempts: options.maxRecoveryAttempts,
3145
- maxRecoveryRetriesPerAttempt: options.maxRecoveryRetriesPerAttempt,
3146
- executeStep: options.executeStep,
3147
- executeStepRef: options.executeStepRef,
3148
- onBranchSnapshot: options.onBranchSnapshot,
3149
- cleanupBranches: options.cleanupBranches ?? true,
3150
- costLedger,
3151
- acquireRunLease: async () => ({
3152
- assertOwned: () => lease.assertOwned(),
3153
- release() {
3154
- }
3155
- }),
3156
- now: options.now
2702
+ ...base ?? {},
2703
+ ...extra ?? {},
2704
+ tags: { ...base?.tags ?? {}, ...extra?.tags ?? {} }
3157
2705
  };
3158
2706
  }
3159
- function assertMemoryImprovementIdentity(options, storage, runDir, serialize) {
3160
- const path = join2(runDir, "memory-improvement-manifest.json");
3161
- const identity = {
3162
- schema: 6,
3163
- implementationRef: MEMORY_IMPROVEMENT_IMPLEMENTATION_REF,
3164
- experimentId: options.experimentId,
3165
- improvementRef: options.improvementRef,
3166
- activationRef: options.activation?.ref ?? null,
3167
- proposerKind: options.proposer.kind,
3168
- proposerKinds: Object.fromEntries(
3169
- Object.entries(options.proposers ?? {}).sort(([a], [b]) => a.localeCompare(b)).map(([name, proposer]) => [name, proposer.kind])
3170
- ),
3171
- populationSize: options.populationSize ?? 4,
3172
- budget: { maxSteps: options.budget.maxSteps },
3173
- executeStepRef: options.executeStepRef ?? null,
3174
- cleanupBranches: options.cleanupBranches ?? true,
3175
- promotionPolicy: normalizedPromotionPolicy(options),
3176
- seed: options.seed ?? null,
3177
- reps: options.reps ?? 1,
3178
- seeds: options.seeds.map((entry) => ({
3179
- config: serialize(entry.config),
3180
- track: entry.track,
3181
- vision: entry.vision ?? null,
3182
- proposer: entry.proposer
3183
- })),
3184
- trainSequences: options.trainSequences,
3185
- holdoutSequences: options.holdoutSequences
3186
- };
3187
- const identityHash = surfaceHash(canonicalJson5(identity));
3188
- const stored = storage.read(path);
3189
- if (stored === void 0) {
3190
- if (storage.exists(path)) throw new Error(`cannot read memory improvement manifest '${path}'`);
3191
- if (storage.exists(join2(runDir, "lineage.jsonl")) || storage.exists(join2(runDir, "memory-improvement-result.json"))) {
3192
- throw new Error(`memory improvement run '${runDir}' has state without an identity manifest`);
3193
- }
3194
- storage.write(path, `${JSON.stringify({ identityHash, identity }, null, 2)}
3195
- `);
3196
- return;
2707
+ function uniqueStrings2(values) {
2708
+ return [...new Set(values.filter((value) => typeof value === "string"))];
2709
+ }
2710
+ function stringArray2(value) {
2711
+ return Array.isArray(value) ? value.filter((entry) => typeof entry === "string") : [];
2712
+ }
2713
+ function stringField(value, keys) {
2714
+ for (const key of keys) {
2715
+ const field = value[key];
2716
+ if (typeof field === "string" && field.length > 0) return field;
3197
2717
  }
3198
- let manifest;
3199
- try {
3200
- manifest = JSON.parse(stored);
3201
- } catch (error) {
3202
- throw new Error(`invalid memory improvement manifest '${path}'`, { cause: error });
2718
+ return void 0;
2719
+ }
2720
+ function numberField(value, keys) {
2721
+ for (const key of keys) {
2722
+ const field = value[key];
2723
+ if (typeof field === "number" && Number.isFinite(field)) return field;
3203
2724
  }
3204
- if (!manifest || typeof manifest !== "object" || manifest.identityHash !== identityHash || canonicalJson5(manifest.identity) !== canonicalJson5(identity)) {
3205
- throw new Error(
3206
- `memory improvement run '${runDir}' does not match its persisted inputs or implementationRef`
3207
- );
2725
+ return void 0;
2726
+ }
2727
+ function normalizedScoreOf(score) {
2728
+ return score !== void 0 && score >= 0 && score <= 1 ? score : void 0;
2729
+ }
2730
+ function isRecord(value) {
2731
+ return typeof value === "object" && value !== null && !Array.isArray(value);
2732
+ }
2733
+ function stripUndefined2(value) {
2734
+ if (Array.isArray(value)) return value.map(stripUndefined2);
2735
+ if (!isRecord(value)) return value;
2736
+ return Object.fromEntries(
2737
+ Object.entries(value).filter(([, child]) => child !== void 0).map(([key, child]) => [key, stripUndefined2(child)])
2738
+ );
2739
+ }
2740
+ function graphitiMemoryAdapterIdentity(options) {
2741
+ if (typeof options.backendRef !== "string" || !options.backendRef.trim()) {
2742
+ throw new Error("Graphiti backendRef must be a non-empty string");
3208
2743
  }
2744
+ return stableId("graphiti", canonicalJson5(stripUndefined2(options)));
3209
2745
  }
2746
+
2747
+ // src/memory/improvement/run.ts
2748
+ import { join as join3 } from "path";
2749
+ import { canonicalJson as canonicalJson7 } from "@tangle-network/agent-eval";
2750
+ import {
2751
+ campaignLineageStore,
2752
+ createRunCostLedger as createRunCostLedger2,
2753
+ fsCampaignStorage as fsCampaignStorage2,
2754
+ memLineageStore,
2755
+ resolveRunDir as resolveRunDir2,
2756
+ runLineageLoop,
2757
+ surfaceHash as surfaceHash2
2758
+ } from "@tangle-network/agent-eval/campaign";
2759
+
2760
+ // src/memory/improvement/activation.ts
3210
2761
  function appendMemoryActivationEvent(storage, path, event) {
3211
2762
  appendDurableJournalEvent({
3212
2763
  storage,
@@ -3274,27 +2825,94 @@ function parseMemoryActivationEvent(value, path, line, expected) {
3274
2825
  }
3275
2826
  return value;
3276
2827
  }
3277
- function writeMemoryImprovementResult(storage, experimentId, result) {
3278
- storage.write(
3279
- result.resultJsonPath,
3280
- `${JSON.stringify(
3281
- {
3282
- experimentId,
3283
- baselineSurfaceHash: result.baselineSurfaceHash,
3284
- winnerSurfaceHash: result.winnerSurfaceHash,
3285
- baselineSurface: result.baselineSurface,
3286
- winnerSurface: result.winnerSurface,
3287
- decision: result.decision,
3288
- activation: result.activation,
3289
- totalCostUsd: result.totalCostUsd,
3290
- lineage: result.lineage.toGraph()
3291
- },
3292
- null,
3293
- 2
3294
- )}
3295
- `
3296
- );
2828
+
2829
+ // src/memory/improvement/candidate.ts
2830
+ function withCostContext(proposer, costLedger, lease, label) {
2831
+ return {
2832
+ kind: proposer.kind,
2833
+ async propose(context) {
2834
+ await lease.assertOwned();
2835
+ const proposal = await proposer.propose({
2836
+ ...context,
2837
+ costLedger,
2838
+ costPhase: `memory.proposal.${context.track?.id ?? label}`
2839
+ });
2840
+ await lease.assertOwned();
2841
+ return proposal;
2842
+ },
2843
+ ...proposer.decide ? { decide: (input) => proposer.decide(input) } : {}
2844
+ };
2845
+ }
2846
+ function withGovernorCostContext(governor, costLedger, lease) {
2847
+ return {
2848
+ async decide(context) {
2849
+ await lease.assertOwned();
2850
+ const decision = await governor.decide({
2851
+ ...context,
2852
+ costLedger,
2853
+ costPhase: "memory.governor"
2854
+ });
2855
+ await lease.assertOwned();
2856
+ return decision;
2857
+ }
2858
+ };
2859
+ }
2860
+ async function buildCandidate(options, config, hash, role) {
2861
+ const built = await options.createCandidate({
2862
+ config,
2863
+ candidateId: `${role}-${hash}`,
2864
+ surfaceHash: hash
2865
+ });
2866
+ return {
2867
+ ...built,
2868
+ id: `${role}-${hash}`,
2869
+ ref: built.ref
2870
+ };
2871
+ }
2872
+ function experimentOptions(options, costLedger, storage, lease) {
2873
+ return {
2874
+ storage,
2875
+ seed: options.seed,
2876
+ reps: options.reps,
2877
+ resumable: options.resumable,
2878
+ maxConcurrency: options.sequenceConcurrency,
2879
+ dispatchTimeoutMs: options.dispatchTimeoutMs,
2880
+ cleanupTimeoutMs: options.cleanupTimeoutMs,
2881
+ maxRecoveryAttempts: options.maxRecoveryAttempts,
2882
+ maxRecoveryRetriesPerAttempt: options.maxRecoveryRetriesPerAttempt,
2883
+ executeStep: options.executeStep,
2884
+ executeStepRef: options.executeStepRef,
2885
+ onBranchSnapshot: options.onBranchSnapshot,
2886
+ cleanupBranches: options.cleanupBranches ?? true,
2887
+ costLedger,
2888
+ acquireRunLease: async () => ({
2889
+ assertOwned: () => lease.assertOwned(),
2890
+ release() {
2891
+ }
2892
+ }),
2893
+ now: options.now
2894
+ };
3297
2895
  }
2896
+
2897
+ // src/memory/improvement/identity.ts
2898
+ import { join as join2 } from "path";
2899
+ import { canonicalJson as canonicalJson6 } from "@tangle-network/agent-eval";
2900
+ import {
2901
+ surfaceHash
2902
+ } from "@tangle-network/agent-eval/campaign";
2903
+
2904
+ // src/memory/improvement/promotion.ts
2905
+ import { heldoutSignificance } from "@tangle-network/agent-eval/campaign";
2906
+
2907
+ // src/memory/improvement/types.ts
2908
+ var DEFAULT_CRITICAL_DIMENSIONS = [
2909
+ "memory_stale_safe",
2910
+ "memory_actor_recall",
2911
+ "memory_event_recall"
2912
+ ];
2913
+ var MEMORY_IMPROVEMENT_IMPLEMENTATION_REF = "agent-knowledge:memory-improvement:v2";
2914
+
2915
+ // src/memory/improvement/promotion.ts
3298
2916
  function decidePromotion(input) {
3299
2917
  const { options, result, baselineId, winnerId } = input;
3300
2918
  const baselineRow = result.rows.find((row) => row.candidateId === baselineId);
@@ -3433,6 +3051,62 @@ function sequenceScores(result, sequences, candidateId) {
3433
3051
  }
3434
3052
  return sequences.map((sequence) => mean2(bySequence.get(sequence.id) ?? []));
3435
3053
  }
3054
+ function mean2(values) {
3055
+ return values.length === 0 ? 0 : values.reduce((sum, value) => sum + value, 0) / values.length;
3056
+ }
3057
+
3058
+ // src/memory/improvement/identity.ts
3059
+ function assertMemoryImprovementIdentity(options, storage, runDir, serialize) {
3060
+ const path = join2(runDir, "memory-improvement-manifest.json");
3061
+ const identity = {
3062
+ schema: 6,
3063
+ implementationRef: MEMORY_IMPROVEMENT_IMPLEMENTATION_REF,
3064
+ experimentId: options.experimentId,
3065
+ improvementRef: options.improvementRef,
3066
+ activationRef: options.activation?.ref ?? null,
3067
+ proposerKind: options.proposer.kind,
3068
+ proposerKinds: Object.fromEntries(
3069
+ Object.entries(options.proposers ?? {}).sort(([a], [b]) => a.localeCompare(b)).map(([name, proposer]) => [name, proposer.kind])
3070
+ ),
3071
+ populationSize: options.populationSize ?? 4,
3072
+ budget: { maxSteps: options.budget.maxSteps },
3073
+ executeStepRef: options.executeStepRef ?? null,
3074
+ cleanupBranches: options.cleanupBranches ?? true,
3075
+ promotionPolicy: normalizedPromotionPolicy(options),
3076
+ seed: options.seed ?? null,
3077
+ reps: options.reps ?? 1,
3078
+ seeds: options.seeds.map((entry) => ({
3079
+ config: serialize(entry.config),
3080
+ track: entry.track,
3081
+ vision: entry.vision ?? null,
3082
+ proposer: entry.proposer
3083
+ })),
3084
+ trainSequences: options.trainSequences,
3085
+ holdoutSequences: options.holdoutSequences
3086
+ };
3087
+ const identityHash = surfaceHash(canonicalJson6(identity));
3088
+ const stored = storage.read(path);
3089
+ if (stored === void 0) {
3090
+ if (storage.exists(path)) throw new Error(`cannot read memory improvement manifest '${path}'`);
3091
+ if (storage.exists(join2(runDir, "lineage.jsonl")) || storage.exists(join2(runDir, "memory-improvement-result.json"))) {
3092
+ throw new Error(`memory improvement run '${runDir}' has state without an identity manifest`);
3093
+ }
3094
+ storage.write(path, `${JSON.stringify({ identityHash, identity }, null, 2)}
3095
+ `);
3096
+ return;
3097
+ }
3098
+ let manifest;
3099
+ try {
3100
+ manifest = JSON.parse(stored);
3101
+ } catch (error) {
3102
+ throw new Error(`invalid memory improvement manifest '${path}'`, { cause: error });
3103
+ }
3104
+ if (!manifest || typeof manifest !== "object" || manifest.identityHash !== identityHash || canonicalJson6(manifest.identity) !== canonicalJson6(identity)) {
3105
+ throw new Error(
3106
+ `memory improvement run '${runDir}' does not match its persisted inputs or implementationRef`
3107
+ );
3108
+ }
3109
+ }
3436
3110
  function requireStringSurface(surface) {
3437
3111
  if (typeof surface !== "string" || surface.trim().length === 0) {
3438
3112
  throw new Error("memory config proposer must return a JSON string surface");
@@ -3469,8 +3143,33 @@ function assertMemoryConfigRoundTrip(config, serialize, parse) {
3469
3143
  }
3470
3144
  function memorySequenceFingerprint(sequence) {
3471
3145
  const { id: _id, split: _split, tags: _tags, ...content } = sequence;
3472
- return surfaceHash(canonicalJson5(content));
3146
+ return surfaceHash(canonicalJson6(content));
3147
+ }
3148
+
3149
+ // src/memory/improvement/output.ts
3150
+ function writeMemoryImprovementResult(storage, experimentId, result) {
3151
+ storage.write(
3152
+ result.resultJsonPath,
3153
+ `${JSON.stringify(
3154
+ {
3155
+ experimentId,
3156
+ baselineSurfaceHash: result.baselineSurfaceHash,
3157
+ winnerSurfaceHash: result.winnerSurfaceHash,
3158
+ baselineSurface: result.baselineSurface,
3159
+ winnerSurface: result.winnerSurface,
3160
+ decision: result.decision,
3161
+ activation: result.activation,
3162
+ totalCostUsd: result.totalCostUsd,
3163
+ lineage: result.lineage.toGraph()
3164
+ },
3165
+ null,
3166
+ 2
3167
+ )}
3168
+ `
3169
+ );
3473
3170
  }
3171
+
3172
+ // src/memory/improvement/validation.ts
3474
3173
  function assertMemoryImprovementOptions(options) {
3475
3174
  for (const [name, value] of [
3476
3175
  ["experimentId", options.experimentId],
@@ -3545,40 +3244,385 @@ function assertMemoryImprovementOptions(options) {
3545
3244
  if (typeof seed.track !== "string" || !seed.track.trim() || typeof seed.proposer !== "string" || !seed.proposer.trim()) {
3546
3245
  throw new Error("memory improvement seeds require non-empty track and proposer values");
3547
3246
  }
3548
- if (seed.vision !== void 0 && (typeof seed.vision !== "string" || !seed.vision.trim())) {
3549
- throw new Error("memory improvement seed vision must be a non-empty string when provided");
3247
+ if (seed.vision !== void 0 && (typeof seed.vision !== "string" || !seed.vision.trim())) {
3248
+ throw new Error("memory improvement seed vision must be a non-empty string when provided");
3249
+ }
3250
+ }
3251
+ const dimensions = options.criticalDimensions ?? DEFAULT_CRITICAL_DIMENSIONS;
3252
+ if (dimensions.some((dimension) => typeof dimension !== "string" || !dimension.trim())) {
3253
+ throw new Error("memory improvement criticalDimensions must contain non-empty names");
3254
+ }
3255
+ if (new Set(dimensions).size !== dimensions.length) {
3256
+ throw new Error("memory improvement criticalDimensions must be unique");
3257
+ }
3258
+ assertDistinctSequenceContent(options.trainSequences, "train");
3259
+ assertDistinctSequenceContent(options.holdoutSequences, "holdout");
3260
+ }
3261
+ function assertDistinctSequenceContent(sequences, split) {
3262
+ const idsByFingerprint = /* @__PURE__ */ new Map();
3263
+ for (const sequence of sequences) {
3264
+ const fingerprint = memorySequenceFingerprint(sequence);
3265
+ const prior = idsByFingerprint.get(fingerprint);
3266
+ if (prior) {
3267
+ throw new Error(
3268
+ `memory improvement ${split} histories duplicate content: ${prior}/${sequence.id}`
3269
+ );
3270
+ }
3271
+ idsByFingerprint.set(fingerprint, sequence.id);
3272
+ }
3273
+ }
3274
+
3275
+ // src/memory/improvement/run.ts
3276
+ async function runAgentMemoryImprovement(options) {
3277
+ if ("onPromote" in options) {
3278
+ throw new Error(
3279
+ "memory improvement onPromote was removed; use activation.readCurrent and activation.compareAndSet"
3280
+ );
3281
+ }
3282
+ if (options.seeds.length === 0) throw new Error("memory improvement requires seed configs");
3283
+ if (options.trainSequences.length === 0) {
3284
+ throw new Error("memory improvement requires training sequences");
3285
+ }
3286
+ if (options.holdoutSequences.length === 0) {
3287
+ throw new Error("memory improvement requires holdout sequences");
3288
+ }
3289
+ const trainIds = new Set(options.trainSequences.map((sequence) => sequence.id));
3290
+ const overlap = options.holdoutSequences.map((sequence) => sequence.id).filter((id) => trainIds.has(id));
3291
+ if (overlap.length > 0) {
3292
+ throw new Error(`memory improvement train/holdout overlap: ${overlap.join(", ")}`);
3293
+ }
3294
+ const trainFingerprints = new Map(
3295
+ options.trainSequences.map((sequence) => [memorySequenceFingerprint(sequence), sequence.id])
3296
+ );
3297
+ const duplicateHistories = options.holdoutSequences.flatMap((sequence) => {
3298
+ const trainId = trainFingerprints.get(memorySequenceFingerprint(sequence));
3299
+ return trainId ? [`${trainId}/${sequence.id}`] : [];
3300
+ });
3301
+ if (duplicateHistories.length > 0) {
3302
+ throw new Error(
3303
+ `memory improvement train/holdout histories duplicate content: ${duplicateHistories.join(", ")}`
3304
+ );
3305
+ }
3306
+ if (typeof options.improvementRef !== "string" || !options.improvementRef.trim()) {
3307
+ throw new Error("memory improvement improvementRef must be a non-empty string");
3308
+ }
3309
+ assertMemoryImprovementOptions(options);
3310
+ const storage = options.storage ?? fsCampaignStorage2();
3311
+ const runDir = resolveRunDir2(options.runDir, options.repo);
3312
+ storage.ensureDir(runDir);
3313
+ const lease = await acquireAgentMemoryRunLease({
3314
+ experimentId: options.experimentId,
3315
+ runDir,
3316
+ storage,
3317
+ customStorage: options.storage !== void 0,
3318
+ lockFileName: "memory-improvement.lock",
3319
+ label: "memory improvement",
3320
+ controllerMode: options.controllerMode,
3321
+ acquireRunLease: options.acquireRunLease
3322
+ });
3323
+ let result;
3324
+ let primaryError;
3325
+ try {
3326
+ result = await runAgentMemoryImprovementOwned(options, storage, runDir, lease);
3327
+ } catch (error) {
3328
+ primaryError = error;
3329
+ }
3330
+ let releaseError;
3331
+ try {
3332
+ await lease.release();
3333
+ } catch (error) {
3334
+ releaseError = error;
3335
+ }
3336
+ if (primaryError && releaseError) {
3337
+ throw new AggregateError(
3338
+ [primaryError, releaseError],
3339
+ "memory improvement failed and its controller lease could not be released"
3340
+ );
3341
+ }
3342
+ if (primaryError) throw primaryError;
3343
+ if (releaseError) throw releaseError;
3344
+ if (!result) throw new Error("memory improvement produced no result");
3345
+ return result;
3346
+ }
3347
+ async function runAgentMemoryImprovementOwned(options, storage, runDir, lease) {
3348
+ await lease.assertOwned();
3349
+ const serializeRaw = options.serializeConfig ?? ((config) => canonicalJson7(config));
3350
+ const parseRaw = options.parseConfig ?? ((surface) => JSON.parse(surface));
3351
+ const serialize = (config) => serializeMemoryConfig(serializeRaw, config);
3352
+ const parse = (surface) => parseMemoryConfig(parseRaw, surface);
3353
+ for (const seed of options.seeds) assertMemoryConfigRoundTrip(seed.config, serialize, parse);
3354
+ if (options.activation && !storage.append) {
3355
+ throw new Error("memory activation requires CampaignStorage.append");
3356
+ }
3357
+ if (options.resumable !== false && !options.lineageStore && !storage.append) {
3358
+ throw new Error("resumable memory improvement requires CampaignStorage.append");
3359
+ }
3360
+ assertMemoryImprovementIdentity(options, storage, runDir, serialize);
3361
+ const costLedger = createRunCostLedger2({
3362
+ storage,
3363
+ runDir,
3364
+ costCeilingUsd: options.maxTotalCostUsd ?? 0
3365
+ });
3366
+ reconcileInterruptedRunPaidCalls(costLedger, "memory improvement run");
3367
+ assertNoInterruptedPaidCalls(costLedger, "memory improvement recovery");
3368
+ const trainScenarios = options.trainSequences.map((sequence) => ({
3369
+ id: sequence.id,
3370
+ kind: "agent-memory-config-search",
3371
+ sequenceId: sequence.id
3372
+ }));
3373
+ const evaluations = /* @__PURE__ */ new Map();
3374
+ const evaluateSurface = async (surface) => {
3375
+ await lease.assertOwned();
3376
+ const text = requireStringSurface(surface);
3377
+ const config = parse(text);
3378
+ const canonicalSurface = serialize(config);
3379
+ const hash = surfaceHash2(canonicalSurface);
3380
+ let pending = evaluations.get(hash);
3381
+ if (!pending) {
3382
+ pending = (async () => {
3383
+ const candidate = await buildCandidate(options, config, hash, `search-${hash}`);
3384
+ await lease.assertOwned();
3385
+ const experiment = await runAgentMemoryExperiment({
3386
+ ...experimentOptions(options, costLedger, storage, lease),
3387
+ experimentId: `${options.experimentId}:search:${hash}`,
3388
+ sequences: options.trainSequences,
3389
+ candidates: [candidate],
3390
+ runDir: join3(runDir, "search", hash),
3391
+ costPhase: `memory.search.${hash}`
3392
+ });
3393
+ await lease.assertOwned();
3394
+ return experiment;
3395
+ })();
3396
+ evaluations.set(hash, pending);
3397
+ void pending.catch(() => evaluations.delete(hash));
3398
+ }
3399
+ const result2 = await pending;
3400
+ await lease.assertOwned();
3401
+ const row = result2.rows[0];
3402
+ if (!row || row.cellsFailed > 0) {
3403
+ throw new Error(`${hash}: memory candidate did not complete every training cell`);
3550
3404
  }
3405
+ return {
3406
+ score: row.scoreMean,
3407
+ scoreVector: sequenceScores(result2, options.trainSequences, row.candidateId)
3408
+ };
3409
+ };
3410
+ const lineageStore = options.lineageStore ?? (options.resumable === false ? memLineageStore() : campaignLineageStore(storage, join3(runDir, "lineage.jsonl")));
3411
+ const lineageOptions = {
3412
+ seeds: options.seeds.map((seed) => ({
3413
+ surface: serialize(seed.config),
3414
+ track: seed.track,
3415
+ proposer: seed.proposer,
3416
+ ...seed.vision !== void 0 ? { vision: seed.vision } : {}
3417
+ })),
3418
+ scenarios: trainScenarios,
3419
+ proposer: withCostContext(options.proposer, costLedger, lease, "default"),
3420
+ proposers: options.proposers ? Object.fromEntries(
3421
+ Object.entries(options.proposers).map(([name, proposer]) => [
3422
+ name,
3423
+ withCostContext(proposer, costLedger, lease, name)
3424
+ ])
3425
+ ) : void 0,
3426
+ scoreSurface: evaluateSurface,
3427
+ governor: options.governor ? withGovernorCostContext(options.governor, costLedger, lease) : void 0,
3428
+ budget: {
3429
+ ...options.budget,
3430
+ maxNodes: options.seeds.length + options.budget.maxSteps
3431
+ },
3432
+ store: lineageStore,
3433
+ populationSize: options.populationSize,
3434
+ candidateConcurrency: options.candidateConcurrency
3435
+ };
3436
+ const search = await runLineageLoop(lineageOptions);
3437
+ await lease.assertOwned();
3438
+ const best = search.best;
3439
+ if (!best) throw new Error("memory improvement produced no measured config");
3440
+ const baselineSurface = serialize(options.seeds[0].config);
3441
+ const baselineHash = surfaceHash2(baselineSurface);
3442
+ const winnerConfig = parse(requireStringSurface(best.surface));
3443
+ const winnerSurface = serialize(winnerConfig);
3444
+ const winnerHash = surfaceHash2(winnerSurface);
3445
+ const baselineConfig = parse(baselineSurface);
3446
+ let holdout;
3447
+ let decision;
3448
+ if (winnerHash === baselineHash) {
3449
+ const baselineMeasurement = await evaluateSurface(baselineSurface);
3450
+ decision = {
3451
+ status: "no-change",
3452
+ reasons: ["search did not find a config better than the baseline"],
3453
+ baselineScore: baselineMeasurement.score,
3454
+ winnerScore: baselineMeasurement.score,
3455
+ lift: 0,
3456
+ criticalDimensions: []
3457
+ };
3458
+ } else {
3459
+ await lease.assertOwned();
3460
+ const [baselineCandidate, winnerCandidate] = await Promise.all([
3461
+ buildCandidate(options, baselineConfig, baselineHash, "baseline"),
3462
+ buildCandidate(options, winnerConfig, winnerHash, "winner")
3463
+ ]);
3464
+ await lease.assertOwned();
3465
+ holdout = await runAgentMemoryExperiment({
3466
+ ...experimentOptions(options, costLedger, storage, lease),
3467
+ experimentId: `${options.experimentId}:holdout`,
3468
+ sequences: options.holdoutSequences,
3469
+ candidates: [baselineCandidate, winnerCandidate],
3470
+ runDir: join3(runDir, "holdout"),
3471
+ costPhase: "memory.holdout"
3472
+ });
3473
+ decision = decidePromotion({
3474
+ options,
3475
+ result: holdout,
3476
+ baselineId: baselineCandidate.id,
3477
+ winnerId: winnerCandidate.id
3478
+ });
3551
3479
  }
3552
- const dimensions = options.criticalDimensions ?? DEFAULT_CRITICAL_DIMENSIONS;
3553
- if (dimensions.some((dimension) => typeof dimension !== "string" || !dimension.trim())) {
3554
- throw new Error("memory improvement criticalDimensions must contain non-empty names");
3555
- }
3556
- if (new Set(dimensions).size !== dimensions.length) {
3557
- throw new Error("memory improvement criticalDimensions must be unique");
3558
- }
3559
- assertDistinctSequenceContent(options.trainSequences, "train");
3560
- assertDistinctSequenceContent(options.holdoutSequences, "holdout");
3561
- }
3562
- function assertDistinctSequenceContent(sequences, split) {
3563
- const idsByFingerprint = /* @__PURE__ */ new Map();
3564
- for (const sequence of sequences) {
3565
- const fingerprint = memorySequenceFingerprint(sequence);
3566
- const prior = idsByFingerprint.get(fingerprint);
3567
- if (prior) {
3480
+ const resultJsonPath = join3(runDir, "memory-improvement-result.json");
3481
+ const activationRef = options.activation?.ref ?? "not-configured";
3482
+ const activationId = `memory-activation-${surfaceHash2(
3483
+ canonicalJson7({
3484
+ experimentId: options.experimentId,
3485
+ improvementRef: options.improvementRef,
3486
+ activationRef,
3487
+ baselineSurfaceHash: baselineHash,
3488
+ winnerSurfaceHash: winnerHash,
3489
+ holdoutManifestHash: holdout?.campaign.manifestHash ?? null,
3490
+ promotionPolicy: normalizedPromotionPolicy(options)
3491
+ })
3492
+ )}`;
3493
+ const activationJournalDir = join3(runDir, "activations");
3494
+ const activationJournalPath = join3(activationJournalDir, `${activationId}.jsonl`);
3495
+ const activationEligible = decision.status === "promote" && holdout !== void 0;
3496
+ const activationEventIdentity = holdout ? {
3497
+ schema: 1,
3498
+ activationId,
3499
+ experimentId: options.experimentId,
3500
+ activationRef,
3501
+ baselineSurfaceHash: baselineHash,
3502
+ winnerSurfaceHash: winnerHash,
3503
+ holdoutManifestHash: holdout.campaign.manifestHash
3504
+ } : void 0;
3505
+ const activationJournal = activationEligible && activationEventIdentity ? readMemoryActivationJournal(storage, activationJournalPath, activationEventIdentity) : { prepared: false };
3506
+ const activation = {
3507
+ id: activationId,
3508
+ status: !activationEligible ? "not-eligible" : activationJournal.activated ? "already-activated" : options.activation ? "pending" : "not-configured",
3509
+ journalPath: activationJournalPath
3510
+ };
3511
+ const result = {
3512
+ lineage: search.lineage,
3513
+ baselineConfig,
3514
+ winnerConfig,
3515
+ baselineSurface,
3516
+ winnerSurface,
3517
+ baselineSurfaceHash: baselineHash,
3518
+ winnerSurfaceHash: winnerHash,
3519
+ decision,
3520
+ activation,
3521
+ ...holdout ? { holdout } : {},
3522
+ totalCostUsd: costLedger.summary().totalCostUsd,
3523
+ resultJsonPath
3524
+ };
3525
+ await lease.assertOwned();
3526
+ writeMemoryImprovementResult(storage, options.experimentId, result);
3527
+ if (activationEligible && !activationJournal.activated && activationEventIdentity && holdout && options.activation) {
3528
+ const activationDriver = options.activation;
3529
+ const activationTimeoutMs = options.activationTimeoutMs ?? 6e4;
3530
+ const hadPreparedEvent = activationJournal.prepared;
3531
+ if (!hadPreparedEvent) {
3532
+ await lease.assertOwned();
3533
+ storage.ensureDir(activationJournalDir);
3534
+ appendMemoryActivationEvent(storage, activationJournalPath, {
3535
+ ...activationEventIdentity,
3536
+ status: "prepared",
3537
+ recordedAt: (options.now ?? (() => /* @__PURE__ */ new Date()))().toISOString()
3538
+ });
3539
+ }
3540
+ await lease.assertOwned();
3541
+ const currentConfig = await runBoundedMemoryLifecycle({
3542
+ operation: `${activationDriver.ref}: read current memory configuration`,
3543
+ timeoutMs: activationTimeoutMs,
3544
+ run: () => activationDriver.readCurrent()
3545
+ });
3546
+ await lease.assertOwned();
3547
+ const currentHash = surfaceHash2(serialize(currentConfig));
3548
+ if (currentHash !== baselineHash && currentHash !== winnerHash) {
3568
3549
  throw new Error(
3569
- `memory improvement ${split} histories duplicate content: ${prior}/${sequence.id}`
3550
+ `memory activation target '${activationDriver.ref}' changed concurrently; expected '${baselineHash}' or '${winnerHash}', found '${currentHash}'`
3570
3551
  );
3571
3552
  }
3572
- idsByFingerprint.set(fingerprint, sequence.id);
3553
+ let outcome;
3554
+ if (currentHash === winnerHash) {
3555
+ outcome = hadPreparedEvent ? "recovered" : "already-current";
3556
+ activation.status = "recovered";
3557
+ } else {
3558
+ let compareError;
3559
+ try {
3560
+ await runBoundedMemoryLifecycle({
3561
+ operation: `${activationDriver.ref}: activate memory configuration`,
3562
+ timeoutMs: activationTimeoutMs,
3563
+ run: () => activationDriver.compareAndSet({
3564
+ activationId,
3565
+ expectedConfig: baselineConfig,
3566
+ expectedSurfaceHash: baselineHash,
3567
+ config: winnerConfig,
3568
+ surfaceHash: winnerHash,
3569
+ decision,
3570
+ lineage: search.lineage,
3571
+ holdout
3572
+ })
3573
+ });
3574
+ } catch (error) {
3575
+ compareError = error;
3576
+ }
3577
+ await lease.assertOwned();
3578
+ let observedConfig;
3579
+ try {
3580
+ observedConfig = await runBoundedMemoryLifecycle({
3581
+ operation: `${activationDriver.ref}: confirm memory configuration`,
3582
+ timeoutMs: activationTimeoutMs,
3583
+ run: () => activationDriver.readCurrent()
3584
+ });
3585
+ } catch (error) {
3586
+ if (compareError) {
3587
+ throw new AggregateError(
3588
+ [compareError, error],
3589
+ `memory activation '${activationId}' failed and its live state could not be confirmed`
3590
+ );
3591
+ }
3592
+ throw error;
3593
+ }
3594
+ await lease.assertOwned();
3595
+ const observedHash = surfaceHash2(serialize(observedConfig));
3596
+ if (observedHash !== winnerHash) {
3597
+ const mismatch = new Error(
3598
+ `memory activation '${activationId}' did not install the measured winner; found '${observedHash}'`
3599
+ );
3600
+ if (compareError) {
3601
+ throw new AggregateError(
3602
+ [compareError, mismatch],
3603
+ `memory activation '${activationId}' failed without applying the measured winner`
3604
+ );
3605
+ }
3606
+ throw mismatch;
3607
+ }
3608
+ outcome = compareError ? "recovered" : "applied";
3609
+ activation.status = compareError ? "recovered" : "activated";
3610
+ }
3611
+ await lease.assertOwned();
3612
+ appendMemoryActivationEvent(storage, activationJournalPath, {
3613
+ ...activationEventIdentity,
3614
+ status: "activated",
3615
+ outcome,
3616
+ recordedAt: (options.now ?? (() => /* @__PURE__ */ new Date()))().toISOString()
3617
+ });
3618
+ writeMemoryImprovementResult(storage, options.experimentId, result);
3573
3619
  }
3574
- }
3575
- function mean2(values) {
3576
- return values.length === 0 ? 0 : values.reduce((sum, value) => sum + value, 0) / values.length;
3620
+ return result;
3577
3621
  }
3578
3622
 
3579
3623
  // src/memory/mem0.ts
3580
3624
  import { randomUUID as randomUUID4 } from "crypto";
3581
- import { canonicalJson as canonicalJson6 } from "@tangle-network/agent-eval";
3625
+ import { canonicalJson as canonicalJson8 } from "@tangle-network/agent-eval";
3582
3626
  function createMem0MemoryAdapter(options) {
3583
3627
  assertMem0Options(options);
3584
3628
  const id = options.id ?? `mem0-${options.mode}`;
@@ -3894,7 +3938,7 @@ function normalizeMem0Hits(raw, adapterId, options) {
3894
3938
  if (options.minScore !== void 0 && (score === void 0 || score < options.minScore)) {
3895
3939
  return void 0;
3896
3940
  }
3897
- const memoryId = stringField2(item, ["id"]) ?? stableId("mem", canonicalJson6(compactRecord2({ adapterId, text, metadata, item })));
3941
+ const memoryId = stringField2(item, ["id"]) ?? stableId("mem", canonicalJson8(compactRecord2({ adapterId, text, metadata, item })));
3898
3942
  const normalizedScore = score !== void 0 && score >= 0 && score <= 1 ? score : void 0;
3899
3943
  return {
3900
3944
  id: memoryId,
@@ -4093,11 +4137,11 @@ function mem0MemoryAdapterIdentity(options) {
4093
4137
  if (typeof options.backendRef !== "string" || !options.backendRef.trim()) {
4094
4138
  throw new Error("Mem0 backendRef must be a non-empty string");
4095
4139
  }
4096
- return stableId("mem0", canonicalJson6(compactRecord2(options)));
4140
+ return stableId("mem0", canonicalJson8(compactRecord2(options)));
4097
4141
  }
4098
4142
 
4099
4143
  // src/memory/neo4j.ts
4100
- import { canonicalJson as canonicalJson7 } from "@tangle-network/agent-eval";
4144
+ import { canonicalJson as canonicalJson9 } from "@tangle-network/agent-eval";
4101
4145
  function createNeo4jAgentMemoryAdapter(options) {
4102
4146
  assertNeo4jOptions(options);
4103
4147
  const client = options.client;
@@ -4362,7 +4406,7 @@ async function writeNeo4jMemory(client, input, adapterId, transport, isolatedIns
4362
4406
  assertNeo4jWriteSucceeded(result, adapterId, input.kind);
4363
4407
  const id = idFromResult(result) ?? input.id ?? stableId(
4364
4408
  "mem",
4365
- canonicalJson7({ adapterId, kind: input.kind, text: input.text, scope: input.scope ?? {} })
4409
+ canonicalJson9({ adapterId, kind: input.kind, text: input.text, scope: input.scope ?? {} })
4366
4410
  );
4367
4411
  return {
4368
4412
  accepted: true,
@@ -4723,9 +4767,9 @@ export {
4723
4767
  createAgentMemoryBranch,
4724
4768
  forkAgentMemoryBranchSnapshot,
4725
4769
  buildAgentMemorySequencesFromBenchmarkCases,
4726
- runAgentMemoryExperiment,
4727
4770
  buildAgentMemorySequenceScenarios,
4728
4771
  agentMemorySequenceJudge,
4772
+ runAgentMemoryExperiment,
4729
4773
  createGraphitiMemoryAdapter,
4730
4774
  graphitiMemoryAdapterIdentity,
4731
4775
  runAgentMemoryImprovement,
@@ -4737,4 +4781,4 @@ export {
4737
4781
  AgentMemoryHitSchema,
4738
4782
  AgentMemoryWriteInputSchema
4739
4783
  };
4740
- //# sourceMappingURL=chunk-UWOTQNBI.js.map
4784
+ //# sourceMappingURL=chunk-CPMLJYA3.js.map