@tangle-network/agent-knowledge 4.0.0 → 4.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/dist/benchmarks/index.d.ts +51 -3
- package/dist/benchmarks/index.js +1 -1
- package/dist/{chunk-UWOTQNBI.js → chunk-3CAAGJCE.js} +1886 -1842
- package/dist/chunk-3CAAGJCE.js.map +1 -0
- package/dist/{chunk-DW6APRTX.js → chunk-BBCIB4UL.js} +2486 -2457
- package/dist/chunk-BBCIB4UL.js.map +1 -0
- package/dist/index.d.ts +4 -3
- package/dist/index.js +2 -2
- package/dist/memory/index.d.ts +132 -6
- package/dist/memory/index.js +2 -2
- package/dist/{index-Bqg1mBPt.d.ts → types-ZzY_x0r7.d.ts} +115 -276
- package/package.json +1 -1
- package/dist/chunk-DW6APRTX.js.map +0 -1
- package/dist/chunk-UWOTQNBI.js.map +0 -1
|
@@ -18,7 +18,7 @@ import {
|
|
|
18
18
|
runBoundedMemoryLifecycle,
|
|
19
19
|
scoreMemoryBenchmarkArtifact,
|
|
20
20
|
sleepForMemoryRecovery
|
|
21
|
-
} from "./chunk-
|
|
21
|
+
} from "./chunk-BBCIB4UL.js";
|
|
22
22
|
import {
|
|
23
23
|
sha256,
|
|
24
24
|
stableId
|
|
@@ -964,23 +964,127 @@ function cloneDurableValue(value) {
|
|
|
964
964
|
);
|
|
965
965
|
}
|
|
966
966
|
|
|
967
|
-
// src/memory/experiment.ts
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
967
|
+
// src/memory/experiment/validation.ts
|
|
968
|
+
function assertMemorySequences(sequences) {
|
|
969
|
+
for (const sequence of sequences) {
|
|
970
|
+
assertNonEmptyString(sequence.family, `memory experiment sequence ${sequence.id} family`);
|
|
971
|
+
if (sequence.steps.length === 0) {
|
|
972
|
+
throw new Error(`memory experiment sequence ${sequence.id} has no steps`);
|
|
973
|
+
}
|
|
974
|
+
assertUnique(
|
|
975
|
+
sequence.steps.map((step) => step.id),
|
|
976
|
+
`step in sequence ${sequence.id}`
|
|
977
|
+
);
|
|
978
|
+
let probeCount = 0;
|
|
979
|
+
for (const step of sequence.steps) {
|
|
980
|
+
for (const write of step.writes ?? []) {
|
|
981
|
+
assertNonEmptyString(write.text, `memory experiment write in ${sequence.id}/${step.id}`);
|
|
982
|
+
if (write.id !== void 0) {
|
|
983
|
+
assertNonEmptyString(write.id, `memory experiment write id in ${sequence.id}/${step.id}`);
|
|
984
|
+
}
|
|
985
|
+
}
|
|
986
|
+
assertUnique(
|
|
987
|
+
(step.probes ?? []).map((probe) => probe.id),
|
|
988
|
+
`probe in sequence ${sequence.id} step ${step.id}`
|
|
989
|
+
);
|
|
990
|
+
for (const probe of step.probes ?? []) {
|
|
991
|
+
probeCount += 1;
|
|
992
|
+
assertNonEmptyString(
|
|
993
|
+
probe.query,
|
|
994
|
+
`memory experiment probe query ${sequence.id}/${step.id}/${probe.id}`
|
|
995
|
+
);
|
|
996
|
+
if (probe.limit !== void 0 && (!Number.isSafeInteger(probe.limit) || probe.limit <= 0)) {
|
|
997
|
+
throw new Error(
|
|
998
|
+
`memory experiment probe limit ${sequence.id}/${step.id}/${probe.id} must be a positive safe integer`
|
|
999
|
+
);
|
|
1000
|
+
}
|
|
1001
|
+
assertMemoryFactMatchers(
|
|
1002
|
+
probe.requiredFacts ?? [],
|
|
1003
|
+
`${sequence.id}/${step.id}/${probe.id} requiredFacts`
|
|
1004
|
+
);
|
|
1005
|
+
assertMemoryFactMatchers(
|
|
1006
|
+
probe.forbiddenFacts ?? [],
|
|
1007
|
+
`${sequence.id}/${step.id}/${probe.id} forbiddenFacts`
|
|
1008
|
+
);
|
|
1009
|
+
assertStringList(probe.expectedEventIds, `${sequence.id}/${step.id}/${probe.id} event ids`);
|
|
1010
|
+
assertStringList(probe.expectedActorIds, `${sequence.id}/${step.id}/${probe.id} actor ids`);
|
|
1011
|
+
if (probe.referenceAnswer !== void 0) {
|
|
1012
|
+
assertNonEmptyString(
|
|
1013
|
+
probe.referenceAnswer,
|
|
1014
|
+
`memory experiment reference answer ${sequence.id}/${step.id}/${probe.id}`
|
|
1015
|
+
);
|
|
1016
|
+
}
|
|
1017
|
+
const hasTarget = (probe.requiredFacts?.length ?? 0) > 0 || (probe.forbiddenFacts?.length ?? 0) > 0 || (probe.expectedEventIds?.length ?? 0) > 0 || (probe.expectedActorIds?.length ?? 0) > 0 || Boolean(probe.referenceAnswer?.trim());
|
|
1018
|
+
if (!hasTarget) {
|
|
1019
|
+
throw new Error(
|
|
1020
|
+
`memory experiment probe ${sequence.id}/${step.id}/${probe.id} has no measurable target`
|
|
1021
|
+
);
|
|
1022
|
+
}
|
|
1023
|
+
}
|
|
1024
|
+
}
|
|
1025
|
+
if (probeCount === 0) {
|
|
1026
|
+
throw new Error(`memory experiment sequence ${sequence.id} has no probes`);
|
|
1027
|
+
}
|
|
982
1028
|
}
|
|
983
|
-
}
|
|
1029
|
+
}
|
|
1030
|
+
function normalizeCleanupScope(scope) {
|
|
1031
|
+
const normalized = compactScope(scope);
|
|
1032
|
+
if (normalized.tags && Object.keys(normalized.tags).length === 0) {
|
|
1033
|
+
delete normalized.tags;
|
|
1034
|
+
}
|
|
1035
|
+
return normalized;
|
|
1036
|
+
}
|
|
1037
|
+
function compactScope(scope) {
|
|
1038
|
+
return Object.fromEntries(
|
|
1039
|
+
Object.entries(scope).filter(([, value]) => value !== void 0)
|
|
1040
|
+
);
|
|
1041
|
+
}
|
|
1042
|
+
function compactRecord(record2) {
|
|
1043
|
+
return Object.fromEntries(Object.entries(record2).filter(([, value]) => value !== void 0));
|
|
1044
|
+
}
|
|
1045
|
+
function uniqueStrings(values) {
|
|
1046
|
+
return [...new Set(values.filter((value) => typeof value === "string"))];
|
|
1047
|
+
}
|
|
1048
|
+
function stringArray(value) {
|
|
1049
|
+
return Array.isArray(value) ? value.filter((entry) => typeof entry === "string") : [];
|
|
1050
|
+
}
|
|
1051
|
+
function assertUnique(values, label) {
|
|
1052
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1053
|
+
for (const value of values) {
|
|
1054
|
+
assertNonEmptyString(value, `memory experiment ${label} id`);
|
|
1055
|
+
if (seen.has(value)) throw new Error(`duplicate memory experiment ${label} id: ${value}`);
|
|
1056
|
+
seen.add(value);
|
|
1057
|
+
}
|
|
1058
|
+
}
|
|
1059
|
+
function assertMemoryFactMatchers(matchers, label) {
|
|
1060
|
+
assertUnique(
|
|
1061
|
+
matchers.map((matcher) => matcher.id),
|
|
1062
|
+
`${label} matcher`
|
|
1063
|
+
);
|
|
1064
|
+
for (const matcher of matchers) {
|
|
1065
|
+
if (matcher.anyOf.length === 0) {
|
|
1066
|
+
throw new Error(`memory experiment ${label} matcher ${matcher.id} requires anyOf`);
|
|
1067
|
+
}
|
|
1068
|
+
assertStringList(matcher.anyOf, `${label} matcher ${matcher.id} anyOf`);
|
|
1069
|
+
assertStringList(matcher.sourceEventIds, `${label} matcher ${matcher.id} source event ids`);
|
|
1070
|
+
if (matcher.weight !== void 0 && (!Number.isFinite(matcher.weight) || matcher.weight <= 0)) {
|
|
1071
|
+
throw new Error(
|
|
1072
|
+
`memory experiment ${label} matcher ${matcher.id} weight must be a positive finite number`
|
|
1073
|
+
);
|
|
1074
|
+
}
|
|
1075
|
+
}
|
|
1076
|
+
}
|
|
1077
|
+
function assertStringList(values, label) {
|
|
1078
|
+
if (values === void 0) return;
|
|
1079
|
+
assertUnique(values, label);
|
|
1080
|
+
}
|
|
1081
|
+
function assertNonEmptyString(value, label) {
|
|
1082
|
+
if (typeof value !== "string" || value.trim().length === 0) {
|
|
1083
|
+
throw new Error(`${label} must be a non-empty string`);
|
|
1084
|
+
}
|
|
1085
|
+
}
|
|
1086
|
+
|
|
1087
|
+
// src/memory/experiment/cases.ts
|
|
984
1088
|
function buildAgentMemorySequencesFromBenchmarkCases(cases, options = {}) {
|
|
985
1089
|
const memoryAgentId = options.memoryAgentId ?? "benchmark-agent";
|
|
986
1090
|
return cases.map((testCase) => ({
|
|
@@ -1042,646 +1146,292 @@ function buildAgentMemorySequencesFromBenchmarkCases(cases, options = {}) {
|
|
|
1042
1146
|
]
|
|
1043
1147
|
}));
|
|
1044
1148
|
}
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
assertUnique(
|
|
1057
|
-
options.sequences.map((sequence) => sequence.id),
|
|
1058
|
-
"sequence"
|
|
1059
|
-
);
|
|
1060
|
-
assertUnique(
|
|
1061
|
-
[...options.candidates, ...options.recoveryCandidates ?? []].map((candidate) => candidate.id),
|
|
1062
|
-
"candidate"
|
|
1149
|
+
function buildAgentMemorySequenceScenarios(sequences, candidates) {
|
|
1150
|
+
return candidates.flatMap(
|
|
1151
|
+
(candidate) => sequences.map((sequence) => ({
|
|
1152
|
+
id: `${stableId("candidate", candidate.id)}:${sequence.id}`,
|
|
1153
|
+
kind: "agent-memory-sequence",
|
|
1154
|
+
candidateId: candidate.id,
|
|
1155
|
+
sequenceId: sequence.id,
|
|
1156
|
+
sequence,
|
|
1157
|
+
seedGroup: sequence.id,
|
|
1158
|
+
tags: [.../* @__PURE__ */ new Set([sequence.split ?? "dev", ...sequence.tags ?? [], candidate.id])]
|
|
1159
|
+
}))
|
|
1063
1160
|
);
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1161
|
+
}
|
|
1162
|
+
function agentMemorySequenceJudge() {
|
|
1163
|
+
return {
|
|
1164
|
+
name: "agent-memory-sequence",
|
|
1165
|
+
judgeVersion: "agent-knowledge:memory-sequence:v2",
|
|
1166
|
+
dimensions: [
|
|
1167
|
+
{ key: "score", description: "mean memory probe score" },
|
|
1168
|
+
{ key: "passed", description: "1 when every memory probe passes" },
|
|
1169
|
+
{ key: "memory_fact_recall", description: "current memory fact coverage" },
|
|
1170
|
+
{ key: "memory_event_recall", description: "memory source event coverage" },
|
|
1171
|
+
{ key: "memory_actor_recall", description: "memory actor attribution coverage" },
|
|
1172
|
+
{ key: "memory_stale_safe", description: "1 when obsolete memory is not reused" }
|
|
1173
|
+
],
|
|
1174
|
+
score({ artifact }) {
|
|
1175
|
+
return {
|
|
1176
|
+
composite: artifact.score,
|
|
1177
|
+
dimensions: {
|
|
1178
|
+
score: artifact.score,
|
|
1179
|
+
passed: artifact.passed ? 1 : 0,
|
|
1180
|
+
...artifact.dimensions
|
|
1181
|
+
},
|
|
1182
|
+
notes: `${artifact.probes.filter((probe) => probe.passed).length}/${artifact.probes.length} probes passed`
|
|
1183
|
+
};
|
|
1080
1184
|
}
|
|
1081
|
-
|
|
1082
|
-
}
|
|
1083
|
-
const storage = options.storage ?? fsCampaignStorage();
|
|
1084
|
-
const runDir = resolveRunDir(options.runDir, options.repo);
|
|
1085
|
-
if (!storage.append) {
|
|
1086
|
-
throw new Error("memory experiment requires CampaignStorage.append for durable attempt state");
|
|
1087
|
-
}
|
|
1088
|
-
resolveMemoryCleanupTimeoutMs(options.cleanupTimeoutMs, "memory experiment");
|
|
1089
|
-
const maxRecoveryAttempts = options.maxRecoveryAttempts ?? 1e3;
|
|
1090
|
-
if (!Number.isSafeInteger(maxRecoveryAttempts) || maxRecoveryAttempts <= 0) {
|
|
1091
|
-
throw new Error("memory experiment maxRecoveryAttempts must be a positive safe integer");
|
|
1092
|
-
}
|
|
1093
|
-
const maxRecoveryRetriesPerAttempt = options.maxRecoveryRetriesPerAttempt ?? DEFAULT_MEMORY_RECOVERY_RETRIES_PER_ATTEMPT;
|
|
1094
|
-
if (!Number.isSafeInteger(maxRecoveryRetriesPerAttempt) || maxRecoveryRetriesPerAttempt <= 0) {
|
|
1095
|
-
throw new Error(
|
|
1096
|
-
"memory experiment maxRecoveryRetriesPerAttempt must be a positive safe integer"
|
|
1097
|
-
);
|
|
1098
|
-
}
|
|
1099
|
-
storage.ensureDir(runDir);
|
|
1100
|
-
const lease = await acquireAgentMemoryRunLease({
|
|
1101
|
-
experimentId: options.experimentId,
|
|
1102
|
-
runDir,
|
|
1103
|
-
storage,
|
|
1104
|
-
customStorage: options.storage !== void 0,
|
|
1105
|
-
lockFileName: "memory-experiment.lock",
|
|
1106
|
-
label: "memory experiment",
|
|
1107
|
-
controllerMode: options.controllerMode,
|
|
1108
|
-
acquireRunLease: options.acquireRunLease
|
|
1109
|
-
});
|
|
1110
|
-
let result;
|
|
1111
|
-
let primaryError;
|
|
1112
|
-
try {
|
|
1113
|
-
await lease.assertOwned();
|
|
1114
|
-
result = await runOwnedAgentMemoryExperiment(options, storage, runDir, lease);
|
|
1115
|
-
} catch (error) {
|
|
1116
|
-
primaryError = error;
|
|
1117
|
-
}
|
|
1118
|
-
let releaseError;
|
|
1119
|
-
try {
|
|
1120
|
-
await lease.release();
|
|
1121
|
-
} catch (error) {
|
|
1122
|
-
releaseError = error;
|
|
1123
|
-
}
|
|
1124
|
-
if (primaryError && releaseError) {
|
|
1125
|
-
throw new AggregateError(
|
|
1126
|
-
[primaryError, releaseError],
|
|
1127
|
-
"memory experiment failed and its controller lease could not be released"
|
|
1128
|
-
);
|
|
1129
|
-
}
|
|
1130
|
-
if (primaryError) throw primaryError;
|
|
1131
|
-
if (releaseError) throw releaseError;
|
|
1132
|
-
if (!result) throw new Error("memory experiment produced no result");
|
|
1133
|
-
return result;
|
|
1185
|
+
};
|
|
1134
1186
|
}
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1174
|
-
|
|
1175
|
-
candidateById: recoveryCandidateById,
|
|
1176
|
-
sequenceById,
|
|
1177
|
-
lease,
|
|
1178
|
-
maxConcurrency,
|
|
1179
|
-
costLedger,
|
|
1180
|
-
maxRecoveryAttempts: options.maxRecoveryAttempts ?? 1e3,
|
|
1181
|
-
recoveryLogPath,
|
|
1182
|
-
maxRecoveryRetriesPerAttempt: options.maxRecoveryRetriesPerAttempt ?? DEFAULT_MEMORY_RECOVERY_RETRIES_PER_ATTEMPT
|
|
1187
|
+
|
|
1188
|
+
// src/memory/experiment/run.ts
|
|
1189
|
+
import { join } from "path";
|
|
1190
|
+
import { canonicalJson as canonicalJson4 } from "@tangle-network/agent-eval";
|
|
1191
|
+
import {
|
|
1192
|
+
createRunCostLedger,
|
|
1193
|
+
fsCampaignStorage,
|
|
1194
|
+
resolveRunDir,
|
|
1195
|
+
runCampaign
|
|
1196
|
+
} from "@tangle-network/agent-eval/campaign";
|
|
1197
|
+
|
|
1198
|
+
// src/memory/experiment/cell.ts
|
|
1199
|
+
import { randomUUID as randomUUID2 } from "crypto";
|
|
1200
|
+
|
|
1201
|
+
// src/memory/experiment/metrics.ts
|
|
1202
|
+
function rankAgentMemoryExperiment(candidates, scenarios, campaign, costByCandidate) {
|
|
1203
|
+
const scenarioById = new Map(scenarios.map((scenario) => [scenario.id, scenario]));
|
|
1204
|
+
const rows = candidates.map((candidate) => {
|
|
1205
|
+
const candidateScenarios = scenarios.filter((scenario) => scenario.candidateId === candidate.id);
|
|
1206
|
+
const cells = campaign.cells.filter(
|
|
1207
|
+
(cell) => scenarioById.get(cell.scenarioId)?.candidateId === candidate.id
|
|
1208
|
+
);
|
|
1209
|
+
const successful = cells.filter((cell) => !cell.error && cell.artifact);
|
|
1210
|
+
const dimensionRows = successful.map((cell) => cell.artifact.dimensions);
|
|
1211
|
+
return {
|
|
1212
|
+
rank: 0,
|
|
1213
|
+
candidateId: candidate.id,
|
|
1214
|
+
label: candidate.label ?? candidate.id,
|
|
1215
|
+
scoreMean: mean(
|
|
1216
|
+
cells.map((cell) => !cell.error && cell.artifact ? cell.artifact.score : 0)
|
|
1217
|
+
),
|
|
1218
|
+
passRate: mean(cells.map((cell) => !cell.error && cell.artifact?.passed ? 1 : 0)),
|
|
1219
|
+
totalSequences: candidateScenarios.length,
|
|
1220
|
+
totalCells: cells.length,
|
|
1221
|
+
totalProbes: successful.reduce((sum, cell) => sum + cell.artifact.probes.length, 0),
|
|
1222
|
+
cellsFailed: cells.filter((cell) => Boolean(cell.error)).length,
|
|
1223
|
+
totalCostUsd: normalizeUsd(costByCandidate.get(candidate.id) ?? 0),
|
|
1224
|
+
durationMs: cells.reduce((sum, cell) => sum + cell.durationMs, 0),
|
|
1225
|
+
dimensions: meanDimensions(dimensionRows)
|
|
1226
|
+
};
|
|
1183
1227
|
});
|
|
1184
|
-
|
|
1185
|
-
|
|
1186
|
-
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
dispatch: (scenario, context) => {
|
|
1192
|
-
const candidate = candidateById.get(scenario.candidateId);
|
|
1193
|
-
if (!candidate) throw new Error(`unknown memory candidate ${scenario.candidateId}`);
|
|
1194
|
-
const operation = executionPool.run(
|
|
1195
|
-
() => runSequenceCell({
|
|
1196
|
-
options,
|
|
1197
|
-
candidate,
|
|
1198
|
-
scenario,
|
|
1199
|
-
context,
|
|
1200
|
-
runIdentity,
|
|
1201
|
-
storage,
|
|
1202
|
-
attemptLogPath,
|
|
1203
|
-
lease
|
|
1204
|
-
})
|
|
1205
|
-
);
|
|
1206
|
-
dispatchedExecutions.push(operation);
|
|
1207
|
-
return operation;
|
|
1208
|
-
},
|
|
1209
|
-
dispatchRef: memoryExperimentDispatchRef(options),
|
|
1210
|
-
judges: [agentMemorySequenceJudge()],
|
|
1211
|
-
runDir,
|
|
1212
|
-
storage,
|
|
1213
|
-
seed: options.seed,
|
|
1214
|
-
reps: options.reps,
|
|
1215
|
-
resumable: options.resumable,
|
|
1216
|
-
costCeiling,
|
|
1217
|
-
costLedger,
|
|
1218
|
-
costPhase: options.costPhase,
|
|
1219
|
-
maxConcurrency,
|
|
1220
|
-
dispatchTimeoutMs: options.dispatchTimeoutMs,
|
|
1221
|
-
expectUsage: "off",
|
|
1222
|
-
now: options.now
|
|
1223
|
-
});
|
|
1224
|
-
} catch (error) {
|
|
1225
|
-
campaignError = error;
|
|
1226
|
-
} finally {
|
|
1227
|
-
settledExecutions = await Promise.allSettled(dispatchedExecutions);
|
|
1228
|
-
}
|
|
1229
|
-
const cleanupFailures = settledExecutions.flatMap(
|
|
1230
|
-
(settled) => settled.status === "rejected" && settled.reason instanceof AgentMemoryCleanupError ? [settled.reason] : []
|
|
1228
|
+
return rows.sort(
|
|
1229
|
+
(a, b) => Number(a.cellsFailed > 0) - Number(b.cellsFailed > 0) || b.scoreMean - a.scoreMean || b.passRate - a.passRate || a.totalCostUsd - b.totalCostUsd || a.candidateId.localeCompare(b.candidateId)
|
|
1230
|
+
).map((row, index) => ({ ...row, rank: index + 1 }));
|
|
1231
|
+
}
|
|
1232
|
+
function memoryExperimentCostByCandidate(costLedger, runDir, scenarios, candidateIdsInput) {
|
|
1233
|
+
const candidateByScenario = new Map(
|
|
1234
|
+
scenarios.map((scenario) => [scenario.id, scenario.candidateId])
|
|
1231
1235
|
);
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
);
|
|
1236
|
+
const candidateIds = new Set(candidateIdsInput);
|
|
1237
|
+
const totals = /* @__PURE__ */ new Map();
|
|
1238
|
+
for (const receipt of costLedger.list()) {
|
|
1239
|
+
if (receipt.tags?.runDir !== runDir) continue;
|
|
1240
|
+
const scenarioCandidate = receipt.tags.scenarioId ? candidateByScenario.get(receipt.tags.scenarioId) : void 0;
|
|
1241
|
+
const recoveryCandidate = receipt.tags.memoryRecovery === "attempt" && receipt.tags.candidateId ? receipt.tags.candidateId : void 0;
|
|
1242
|
+
const candidateId = scenarioCandidate ?? recoveryCandidate;
|
|
1243
|
+
if (!candidateId || !candidateIds.has(candidateId)) continue;
|
|
1244
|
+
totals.set(candidateId, (totals.get(candidateId) ?? 0) + receipt.costUsd);
|
|
1237
1245
|
}
|
|
1238
|
-
|
|
1239
|
-
|
|
1240
|
-
|
|
1246
|
+
return totals;
|
|
1247
|
+
}
|
|
1248
|
+
function renderAgentMemoryExperimentRanking(rows, totalCostUsd, unrankedRecoveryCostUsd) {
|
|
1249
|
+
return [
|
|
1250
|
+
"# Agent Memory Experiment",
|
|
1251
|
+
"",
|
|
1252
|
+
`- total cost: $${format(totalCostUsd)}`,
|
|
1253
|
+
`- retired-candidate recovery cost: $${format(unrankedRecoveryCostUsd)}`,
|
|
1254
|
+
"",
|
|
1255
|
+
"| rank | candidate | sequences | cells | probes | failed | score | pass rate | cost | duration ms |",
|
|
1256
|
+
"| ---: | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |",
|
|
1257
|
+
...rows.map(
|
|
1258
|
+
(row) => `| ${row.rank} | ${row.label} | ${row.totalSequences} | ${row.totalCells} | ${row.totalProbes} | ${row.cellsFailed} | ${format(row.scoreMean)} | ${format(row.passRate)} | $${format(row.totalCostUsd)} | ${format(row.durationMs)} |`
|
|
1259
|
+
),
|
|
1260
|
+
""
|
|
1261
|
+
].join("\n");
|
|
1262
|
+
}
|
|
1263
|
+
function meanDimensions(rows) {
|
|
1264
|
+
const values = /* @__PURE__ */ new Map();
|
|
1265
|
+
for (const row of rows) {
|
|
1266
|
+
for (const [key, value] of Object.entries(row)) {
|
|
1267
|
+
if (!Number.isFinite(value)) continue;
|
|
1268
|
+
const bucket = values.get(key) ?? [];
|
|
1269
|
+
bucket.push(value);
|
|
1270
|
+
values.set(key, bucket);
|
|
1271
|
+
}
|
|
1241
1272
|
}
|
|
1242
|
-
|
|
1243
|
-
await lease.assertOwned();
|
|
1244
|
-
const costByCandidate = memoryExperimentCostByCandidate(
|
|
1245
|
-
costLedger,
|
|
1246
|
-
campaign.runDir,
|
|
1247
|
-
scenarios,
|
|
1248
|
-
[...options.candidates, ...options.recoveryCandidates ?? []].map((candidate) => candidate.id)
|
|
1249
|
-
);
|
|
1250
|
-
const unrankedRecoveryCostUsd = normalizeUsd(
|
|
1251
|
-
(options.recoveryCandidates ?? []).reduce(
|
|
1252
|
-
(sum, candidate) => sum + (costByCandidate.get(candidate.id) ?? 0),
|
|
1253
|
-
0
|
|
1254
|
-
)
|
|
1255
|
-
);
|
|
1256
|
-
const totalCostUsd = normalizeUsd(
|
|
1257
|
-
[...costByCandidate.values()].reduce((sum, cost) => sum + cost, 0)
|
|
1258
|
-
);
|
|
1259
|
-
const rows = rankAgentMemoryExperiment(options.candidates, scenarios, campaign, costByCandidate);
|
|
1260
|
-
const rankingJsonPath = join(campaign.runDir, "memory-experiment-ranking.json");
|
|
1261
|
-
const rankingMarkdownPath = join(campaign.runDir, "memory-experiment-ranking.md");
|
|
1262
|
-
storage.write(
|
|
1263
|
-
rankingJsonPath,
|
|
1264
|
-
`${JSON.stringify(
|
|
1265
|
-
{ experimentId: options.experimentId, totalCostUsd, unrankedRecoveryCostUsd, rows },
|
|
1266
|
-
null,
|
|
1267
|
-
2
|
|
1268
|
-
)}
|
|
1269
|
-
`
|
|
1270
|
-
);
|
|
1271
|
-
storage.write(
|
|
1272
|
-
rankingMarkdownPath,
|
|
1273
|
-
renderAgentMemoryExperimentRanking(rows, totalCostUsd, unrankedRecoveryCostUsd)
|
|
1274
|
-
);
|
|
1275
|
-
return {
|
|
1276
|
-
campaign,
|
|
1277
|
-
rows,
|
|
1278
|
-
totalCostUsd,
|
|
1279
|
-
unrankedRecoveryCostUsd,
|
|
1280
|
-
leaderCandidateId: rows.find((row) => row.cellsFailed === 0)?.candidateId,
|
|
1281
|
-
rankingJsonPath,
|
|
1282
|
-
rankingMarkdownPath,
|
|
1283
|
-
attemptLogPath,
|
|
1284
|
-
recoveryLogPath
|
|
1285
|
-
};
|
|
1273
|
+
return Object.fromEntries([...values].map(([key, bucket]) => [key, mean(bucket)]));
|
|
1286
1274
|
}
|
|
1287
|
-
function
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
sequenceId: sequence.id,
|
|
1294
|
-
sequence,
|
|
1295
|
-
seedGroup: sequence.id,
|
|
1296
|
-
tags: [.../* @__PURE__ */ new Set([sequence.split ?? "dev", ...sequence.tags ?? [], candidate.id])]
|
|
1297
|
-
}))
|
|
1298
|
-
);
|
|
1275
|
+
function countDimensions(rows) {
|
|
1276
|
+
const counts = /* @__PURE__ */ new Map();
|
|
1277
|
+
for (const row of rows) {
|
|
1278
|
+
for (const key of new Set(row)) counts.set(key, (counts.get(key) ?? 0) + 1);
|
|
1279
|
+
}
|
|
1280
|
+
return Object.fromEntries(counts);
|
|
1299
1281
|
}
|
|
1300
|
-
function
|
|
1301
|
-
return
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1282
|
+
function mean(values) {
|
|
1283
|
+
return values.length === 0 ? 0 : values.reduce((sum, value) => sum + value, 0) / values.length;
|
|
1284
|
+
}
|
|
1285
|
+
function format(value) {
|
|
1286
|
+
return Number.isFinite(value) ? value.toFixed(4) : "0.0000";
|
|
1287
|
+
}
|
|
1288
|
+
function normalizeUsd(value) {
|
|
1289
|
+
return Number(value.toFixed(12));
|
|
1290
|
+
}
|
|
1291
|
+
|
|
1292
|
+
// src/memory/experiment/recovery.ts
|
|
1293
|
+
import { canonicalJson as canonicalJson3 } from "@tangle-network/agent-eval";
|
|
1294
|
+
|
|
1295
|
+
// src/memory/experiment/runtime.ts
|
|
1296
|
+
var AgentMemoryCleanupError = class extends AggregateError {
|
|
1297
|
+
constructor(errors, message) {
|
|
1298
|
+
super(errors, message);
|
|
1299
|
+
this.name = "AgentMemoryCleanupError";
|
|
1300
|
+
}
|
|
1301
|
+
};
|
|
1302
|
+
function trackExternalMemoryCalls(adapter, onExternalCall) {
|
|
1303
|
+
return {
|
|
1304
|
+
id: adapter.id,
|
|
1305
|
+
branchIsolation: adapter.branchIsolation,
|
|
1306
|
+
search(query, options) {
|
|
1307
|
+
onExternalCall();
|
|
1308
|
+
return adapter.search.call(adapter, query, options);
|
|
1309
|
+
},
|
|
1310
|
+
getContext(query, options) {
|
|
1311
|
+
onExternalCall();
|
|
1312
|
+
return adapter.getContext.call(adapter, query, options);
|
|
1313
|
+
},
|
|
1314
|
+
write(input) {
|
|
1315
|
+
onExternalCall();
|
|
1316
|
+
return adapter.write.call(adapter, input);
|
|
1317
|
+
},
|
|
1318
|
+
...adapter.clear ? {
|
|
1319
|
+
clear(scope) {
|
|
1320
|
+
onExternalCall();
|
|
1321
|
+
return adapter.clear.call(adapter, scope);
|
|
1322
|
+
}
|
|
1323
|
+
} : {},
|
|
1324
|
+
...adapter.flush ? {
|
|
1325
|
+
flush() {
|
|
1326
|
+
onExternalCall();
|
|
1327
|
+
return adapter.flush.call(adapter);
|
|
1328
|
+
}
|
|
1329
|
+
} : {},
|
|
1330
|
+
...adapter.close ? {
|
|
1331
|
+
close() {
|
|
1332
|
+
onExternalCall();
|
|
1333
|
+
return adapter.close.call(adapter);
|
|
1334
|
+
}
|
|
1335
|
+
} : {}
|
|
1323
1336
|
};
|
|
1324
1337
|
}
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1338
|
+
function mergeScopes2(base, extra) {
|
|
1339
|
+
return {
|
|
1340
|
+
...base ?? {},
|
|
1341
|
+
...extra ?? {},
|
|
1342
|
+
tags: { ...base?.tags ?? {}, ...extra?.tags ?? {} }
|
|
1343
|
+
};
|
|
1344
|
+
}
|
|
1345
|
+
function memoryExperimentBaseScope(options, candidate, sequenceId) {
|
|
1346
|
+
return mergeScopes2(candidate.baseScope, {
|
|
1347
|
+
tags: {
|
|
1348
|
+
memoryExperimentId: options.experimentId,
|
|
1349
|
+
memoryCandidateId: candidate.id,
|
|
1350
|
+
memorySequenceId: sequenceId
|
|
1351
|
+
}
|
|
1352
|
+
});
|
|
1353
|
+
}
|
|
1354
|
+
function sequenceCleanupScopes(sequence) {
|
|
1355
|
+
const scopes = /* @__PURE__ */ new Map();
|
|
1356
|
+
for (const scope of sequence.cleanupScopes ?? []) {
|
|
1357
|
+
const normalized = normalizeCleanupScope(scope);
|
|
1358
|
+
scopes.set(JSON.stringify(normalized), normalized);
|
|
1333
1359
|
}
|
|
1334
|
-
|
|
1360
|
+
for (const step of sequence.steps) {
|
|
1361
|
+
const candidates = [
|
|
1362
|
+
...step.scope ? [step.scope] : [],
|
|
1363
|
+
...(step.writes ?? []).map((write) => mergeScopes2(step.scope, write.scope)),
|
|
1364
|
+
...(step.probes ?? []).map((probe) => mergeScopes2(step.scope, probe.scope))
|
|
1365
|
+
];
|
|
1366
|
+
for (const scope of candidates) {
|
|
1367
|
+
const normalized = normalizeCleanupScope(scope);
|
|
1368
|
+
scopes.set(JSON.stringify(normalized), normalized);
|
|
1369
|
+
}
|
|
1370
|
+
}
|
|
1371
|
+
return [...scopes.values()];
|
|
1372
|
+
}
|
|
1373
|
+
async function clearSequenceScopes(memory, sequence) {
|
|
1374
|
+
for (const scope of sequenceCleanupScopes(sequence)) await memory.clear?.(scope);
|
|
1375
|
+
}
|
|
1376
|
+
|
|
1377
|
+
// src/memory/experiment/recovery.ts
|
|
1378
|
+
async function recoverAbandonedMemoryAttempts(input) {
|
|
1379
|
+
let attempts = readActiveMemoryAttempts(input.storage, input.attemptLogPath);
|
|
1380
|
+
if (attempts.length > input.maxRecoveryAttempts) {
|
|
1335
1381
|
throw new Error(
|
|
1336
|
-
|
|
1382
|
+
`memory experiment has ${attempts.length} unfinished attempts; maxRecoveryAttempts is ${input.maxRecoveryAttempts}`
|
|
1337
1383
|
);
|
|
1338
1384
|
}
|
|
1339
|
-
const
|
|
1340
|
-
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
|
|
1355
|
-
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
const appendCleanedAttempt = (priorError) => {
|
|
1364
|
-
try {
|
|
1365
|
-
appendMemoryAttemptEvent(storage, attemptLogPath, {
|
|
1385
|
+
for (const attempt of attempts) {
|
|
1386
|
+
const candidate = input.candidateById.get(attempt.candidateId);
|
|
1387
|
+
if (!candidate) {
|
|
1388
|
+
throw new Error(
|
|
1389
|
+
`cannot recover memory branch '${attempt.branchId}': candidate '${attempt.candidateId}' is missing; pass it in recoveryCandidates`
|
|
1390
|
+
);
|
|
1391
|
+
}
|
|
1392
|
+
assertMemoryAttemptCandidateMatches(attempt, candidate);
|
|
1393
|
+
if (!input.sequenceById.has(attempt.sequenceId)) {
|
|
1394
|
+
throw new Error(
|
|
1395
|
+
`cannot recover memory branch '${attempt.branchId}': sequence '${attempt.sequenceId}' is missing`
|
|
1396
|
+
);
|
|
1397
|
+
}
|
|
1398
|
+
if ((input.options.cleanupBranches ?? true) !== attempt.cleanupBranches) {
|
|
1399
|
+
throw new Error(`cannot recover memory branch '${attempt.branchId}': cleanupBranches changed`);
|
|
1400
|
+
}
|
|
1401
|
+
}
|
|
1402
|
+
reconcileInterruptedMemoryPaidCalls(input.costLedger);
|
|
1403
|
+
assertNoInterruptedPaidCalls(input.costLedger, "memory experiment recovery");
|
|
1404
|
+
for (const attempt of attempts) {
|
|
1405
|
+
const candidate = input.candidateById.get(attempt.candidateId);
|
|
1406
|
+
const executionCostUsd = candidate.externalCostUsdPerSequence ?? 0;
|
|
1407
|
+
if (executionCostUsd > 0 && !hasSettledPaidCall(input.costLedger, memoryAttemptCostCallId(attempt, "execute", 0))) {
|
|
1408
|
+
appendMemoryAttemptEvent(input.storage, input.attemptLogPath, {
|
|
1366
1409
|
...attempt,
|
|
1367
1410
|
status: "cleaned",
|
|
1368
|
-
|
|
1411
|
+
recovery: true,
|
|
1412
|
+
recordedAt: (input.options.now ?? (() => /* @__PURE__ */ new Date()))().toISOString()
|
|
1369
1413
|
});
|
|
1370
|
-
} catch (error) {
|
|
1371
|
-
throw new AgentMemoryCleanupError(
|
|
1372
|
-
[...priorError ? [priorError] : [], error],
|
|
1373
|
-
`${candidate.id}: memory branch cleanup could not be recorded`
|
|
1374
|
-
);
|
|
1375
1414
|
}
|
|
1376
|
-
}
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1382
|
-
|
|
1383
|
-
|
|
1384
|
-
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
|
|
1388
|
-
|
|
1389
|
-
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
externalCallAttempted = true;
|
|
1397
|
-
}
|
|
1398
|
-
});
|
|
1399
|
-
if (!created) throw new Error(`${candidate.id}: createAdapter returned no execution adapter`);
|
|
1400
|
-
rawAdapter = created;
|
|
1401
|
-
adapter = trackExternalMemoryCalls(created, () => {
|
|
1402
|
-
externalCallAttempted = true;
|
|
1403
|
-
});
|
|
1404
|
-
await lease.assertOwned();
|
|
1405
|
-
context.signal.throwIfAborted();
|
|
1406
|
-
if (cleanupBranches && !adapter.clear) {
|
|
1407
|
-
throw new Error(
|
|
1408
|
-
`${candidate.id}: cleanupBranches requires an adapter with scoped clear support`
|
|
1409
|
-
);
|
|
1410
|
-
}
|
|
1411
|
-
memory = createAgentMemoryBranch({
|
|
1412
|
-
adapter,
|
|
1413
|
-
branchId,
|
|
1414
|
-
lifetime: "attempt",
|
|
1415
|
-
policy: candidate.policy,
|
|
1416
|
-
allowedWriteScopes: sequenceCleanupScopes(scenario.sequence),
|
|
1417
|
-
baseScope: memoryExperimentBaseScope(options, candidate, scenario.sequenceId)
|
|
1418
|
-
});
|
|
1419
|
-
const probes = [];
|
|
1420
|
-
for (const step of scenario.sequence.steps) {
|
|
1421
|
-
context.signal.throwIfAborted();
|
|
1422
|
-
await lease.assertOwned();
|
|
1423
|
-
await writeStep(memory, step);
|
|
1424
|
-
await lease.assertOwned();
|
|
1425
|
-
await options.executeStep?.({
|
|
1426
|
-
memory,
|
|
1427
|
-
candidateId: candidate.id,
|
|
1428
|
-
sequence: scenario.sequence,
|
|
1429
|
-
step,
|
|
1430
|
-
context
|
|
1431
|
-
});
|
|
1432
|
-
context.signal.throwIfAborted();
|
|
1433
|
-
await lease.assertOwned();
|
|
1434
|
-
const stepProbes = await probeStep(memory, scenario.sequence, step);
|
|
1435
|
-
await lease.assertOwned();
|
|
1436
|
-
probes.push(...stepProbes);
|
|
1437
|
-
}
|
|
1438
|
-
const snapshot = await memory.snapshot();
|
|
1439
|
-
await lease.assertOwned();
|
|
1440
|
-
await options.onBranchSnapshot?.({
|
|
1441
|
-
candidateId: candidate.id,
|
|
1442
|
-
sequenceId: scenario.sequenceId,
|
|
1443
|
-
cellId: context.cellId,
|
|
1444
|
-
snapshot
|
|
1445
|
-
});
|
|
1446
|
-
await lease.assertOwned();
|
|
1447
|
-
const dimensions = meanDimensions(probes.map((probe) => probe.dimensions));
|
|
1448
|
-
const dimensionSampleCounts = countDimensions(
|
|
1449
|
-
probes.map((probe) => probe.applicableDimensions)
|
|
1450
|
-
);
|
|
1451
|
-
const artifact = {
|
|
1452
|
-
candidateId: candidate.id,
|
|
1453
|
-
sequenceId: scenario.sequenceId,
|
|
1454
|
-
score: mean(probes.map((probe) => probe.score)),
|
|
1455
|
-
passed: probes.length > 0 && probes.every((probe) => probe.passed),
|
|
1456
|
-
dimensions,
|
|
1457
|
-
dimensionSampleCounts,
|
|
1458
|
-
probes,
|
|
1459
|
-
branchDigest: snapshot.digest,
|
|
1460
|
-
journalEntries: snapshot.journal.length,
|
|
1461
|
-
durationMs: Math.max(0, Date.now() - startedAt)
|
|
1462
|
-
};
|
|
1463
|
-
if (cleanupBranches) {
|
|
1464
|
-
finalClearStarted = true;
|
|
1465
|
-
await runBoundedMemoryLifecycle({
|
|
1466
|
-
operation: `${candidate.id}: final branch cleanup`,
|
|
1467
|
-
timeoutMs: cleanupTimeoutMs,
|
|
1468
|
-
resource: adapter,
|
|
1469
|
-
run: () => clearSequenceScopes(memory, scenario.sequence)
|
|
1470
|
-
});
|
|
1471
|
-
finalClearCompleted = true;
|
|
1472
|
-
await lease.assertOwned();
|
|
1473
|
-
}
|
|
1474
|
-
completedArtifact = artifact;
|
|
1475
|
-
} catch (error) {
|
|
1476
|
-
primaryError = error;
|
|
1477
|
-
}
|
|
1478
|
-
const cleanupErrors = [];
|
|
1479
|
-
let cleanupOwned = true;
|
|
1480
|
-
let ownershipError;
|
|
1481
|
-
try {
|
|
1482
|
-
await lease.assertOwned();
|
|
1483
|
-
} catch (error) {
|
|
1484
|
-
cleanupOwned = false;
|
|
1485
|
-
ownershipError = error;
|
|
1486
|
-
}
|
|
1487
|
-
if (finalClearStarted && !finalClearCompleted && primaryError) {
|
|
1488
|
-
cleanupErrors.push(primaryError);
|
|
1489
|
-
}
|
|
1490
|
-
if (primaryError && cleanupOwned && !finalClearStarted && memory && cleanupBranches && adapter?.clear) {
|
|
1491
|
-
try {
|
|
1492
|
-
await runBoundedMemoryLifecycle({
|
|
1493
|
-
operation: `${candidate.id}: failed branch cleanup`,
|
|
1494
|
-
timeoutMs: cleanupTimeoutMs,
|
|
1495
|
-
resource: adapter,
|
|
1496
|
-
run: () => clearSequenceScopes(memory, scenario.sequence)
|
|
1497
|
-
});
|
|
1498
|
-
} catch (error) {
|
|
1499
|
-
cleanupErrors.push(error);
|
|
1500
|
-
}
|
|
1501
|
-
}
|
|
1502
|
-
if (adapter) {
|
|
1503
|
-
try {
|
|
1504
|
-
await runBoundedMemoryLifecycle({
|
|
1505
|
-
operation: `${candidate.id}: adapter close`,
|
|
1506
|
-
timeoutMs: cleanupTimeoutMs,
|
|
1507
|
-
resource: adapter,
|
|
1508
|
-
run: async () => {
|
|
1509
|
-
if (memory && cleanupOwned) await memory.close?.();
|
|
1510
|
-
else {
|
|
1511
|
-
if (cleanupOwned) await adapter.flush?.();
|
|
1512
|
-
await adapter.close?.();
|
|
1513
|
-
}
|
|
1514
|
-
}
|
|
1515
|
-
});
|
|
1516
|
-
} catch (error) {
|
|
1517
|
-
cleanupErrors.push(error);
|
|
1518
|
-
}
|
|
1519
|
-
if (cleanupOwned) {
|
|
1520
|
-
try {
|
|
1521
|
-
await runBoundedMemoryLifecycle({
|
|
1522
|
-
operation: `${candidate.id}: adapter disposal`,
|
|
1523
|
-
timeoutMs: cleanupTimeoutMs,
|
|
1524
|
-
resource: adapter,
|
|
1525
|
-
run: () => {
|
|
1526
|
-
if (candidate.disposeAdapter) externalCallAttempted = true;
|
|
1527
|
-
return candidate.disposeAdapter?.(rawAdapter);
|
|
1528
|
-
}
|
|
1529
|
-
});
|
|
1530
|
-
} catch (error) {
|
|
1531
|
-
cleanupErrors.push(error);
|
|
1532
|
-
}
|
|
1533
|
-
}
|
|
1534
|
-
} else {
|
|
1535
|
-
cleanupErrors.push(
|
|
1536
|
-
new Error(`${candidate.id}: adapter creation failed before cleanup could be confirmed`)
|
|
1537
|
-
);
|
|
1538
|
-
}
|
|
1539
|
-
if (cleanupOwned && cleanupErrors.length === 0) appendCleanedAttempt(primaryError);
|
|
1540
|
-
if (!cleanupOwned && cleanupErrors.length === 0) {
|
|
1541
|
-
if (primaryError) throw primaryError;
|
|
1542
|
-
throw ownershipError;
|
|
1543
|
-
}
|
|
1544
|
-
if (cleanupErrors.length > 0) {
|
|
1545
|
-
const primaryMessage = primaryError instanceof Error ? primaryError.message : String(primaryError ?? "");
|
|
1546
|
-
throw new AgentMemoryCleanupError(
|
|
1547
|
-
[
|
|
1548
|
-
...primaryError && !cleanupErrors.includes(primaryError) ? [primaryError] : [],
|
|
1549
|
-
...ownershipError ? [ownershipError] : [],
|
|
1550
|
-
...cleanupErrors
|
|
1551
|
-
],
|
|
1552
|
-
`${candidate.id}: memory branch cleanup failed${primaryMessage ? ` after: ${primaryMessage}` : ""}`
|
|
1553
|
-
);
|
|
1554
|
-
}
|
|
1555
|
-
if (primaryError) throw primaryError;
|
|
1556
|
-
if (!completedArtifact) throw new Error(`${candidate.id}: memory sequence produced no result`);
|
|
1557
|
-
return completedArtifact;
|
|
1558
|
-
};
|
|
1559
|
-
if (costUsd === 0) {
|
|
1560
|
-
let artifact;
|
|
1561
|
-
let error;
|
|
1562
|
-
try {
|
|
1563
|
-
artifact = await execute();
|
|
1564
|
-
} catch (caught) {
|
|
1565
|
-
error = caught;
|
|
1566
|
-
}
|
|
1567
|
-
if (error) throw error;
|
|
1568
|
-
if (!artifact) throw new Error(`${candidate.id}: memory sequence produced no result`);
|
|
1569
|
-
return artifact;
|
|
1570
|
-
}
|
|
1571
|
-
const receipt = {
|
|
1572
|
-
model: candidate.id,
|
|
1573
|
-
inputTokens: 0,
|
|
1574
|
-
outputTokens: 0,
|
|
1575
|
-
actualCostUsd: costUsd
|
|
1576
|
-
};
|
|
1577
|
-
const paid = await context.cost.runPaidCall({
|
|
1578
|
-
callId: memoryAttemptCostCallId(attempt, "execute", 0),
|
|
1579
|
-
actor: `agent-knowledge:memory-experiment:${candidate.id}`,
|
|
1580
|
-
model: candidate.id,
|
|
1581
|
-
maximumCharge: { externallyEnforcedMaximumUsd: costUsd },
|
|
1582
|
-
execute,
|
|
1583
|
-
receipt: () => receipt,
|
|
1584
|
-
receiptFromError: () => ({
|
|
1585
|
-
...receipt,
|
|
1586
|
-
actualCostUsd: externalCallAttempted ? costUsd : 0
|
|
1587
|
-
})
|
|
1588
|
-
});
|
|
1589
|
-
if (!paid.succeeded) throw paid.error;
|
|
1590
|
-
return paid.value;
|
|
1591
|
-
}
|
|
1592
|
-
function trackExternalMemoryCalls(adapter, onExternalCall) {
|
|
1593
|
-
return {
|
|
1594
|
-
id: adapter.id,
|
|
1595
|
-
branchIsolation: adapter.branchIsolation,
|
|
1596
|
-
search(query, options) {
|
|
1597
|
-
onExternalCall();
|
|
1598
|
-
return adapter.search.call(adapter, query, options);
|
|
1599
|
-
},
|
|
1600
|
-
getContext(query, options) {
|
|
1601
|
-
onExternalCall();
|
|
1602
|
-
return adapter.getContext.call(adapter, query, options);
|
|
1603
|
-
},
|
|
1604
|
-
write(input) {
|
|
1605
|
-
onExternalCall();
|
|
1606
|
-
return adapter.write.call(adapter, input);
|
|
1607
|
-
},
|
|
1608
|
-
...adapter.clear ? {
|
|
1609
|
-
clear(scope) {
|
|
1610
|
-
onExternalCall();
|
|
1611
|
-
return adapter.clear.call(adapter, scope);
|
|
1612
|
-
}
|
|
1613
|
-
} : {},
|
|
1614
|
-
...adapter.flush ? {
|
|
1615
|
-
flush() {
|
|
1616
|
-
onExternalCall();
|
|
1617
|
-
return adapter.flush.call(adapter);
|
|
1618
|
-
}
|
|
1619
|
-
} : {},
|
|
1620
|
-
...adapter.close ? {
|
|
1621
|
-
close() {
|
|
1622
|
-
onExternalCall();
|
|
1623
|
-
return adapter.close.call(adapter);
|
|
1624
|
-
}
|
|
1625
|
-
} : {}
|
|
1626
|
-
};
|
|
1627
|
-
}
|
|
1628
|
-
async function recoverAbandonedMemoryAttempts(input) {
|
|
1629
|
-
let attempts = readActiveMemoryAttempts(input.storage, input.attemptLogPath);
|
|
1630
|
-
if (attempts.length > input.maxRecoveryAttempts) {
|
|
1631
|
-
throw new Error(
|
|
1632
|
-
`memory experiment has ${attempts.length} unfinished attempts; maxRecoveryAttempts is ${input.maxRecoveryAttempts}`
|
|
1633
|
-
);
|
|
1634
|
-
}
|
|
1635
|
-
for (const attempt of attempts) {
|
|
1636
|
-
const candidate = input.candidateById.get(attempt.candidateId);
|
|
1637
|
-
if (!candidate) {
|
|
1638
|
-
throw new Error(
|
|
1639
|
-
`cannot recover memory branch '${attempt.branchId}': candidate '${attempt.candidateId}' is missing; pass it in recoveryCandidates`
|
|
1640
|
-
);
|
|
1641
|
-
}
|
|
1642
|
-
assertMemoryAttemptCandidateMatches(attempt, candidate);
|
|
1643
|
-
if (!input.sequenceById.has(attempt.sequenceId)) {
|
|
1644
|
-
throw new Error(
|
|
1645
|
-
`cannot recover memory branch '${attempt.branchId}': sequence '${attempt.sequenceId}' is missing`
|
|
1646
|
-
);
|
|
1647
|
-
}
|
|
1648
|
-
if ((input.options.cleanupBranches ?? true) !== attempt.cleanupBranches) {
|
|
1649
|
-
throw new Error(`cannot recover memory branch '${attempt.branchId}': cleanupBranches changed`);
|
|
1650
|
-
}
|
|
1651
|
-
}
|
|
1652
|
-
reconcileInterruptedMemoryPaidCalls(input.costLedger);
|
|
1653
|
-
assertNoInterruptedPaidCalls(input.costLedger, "memory experiment recovery");
|
|
1654
|
-
for (const attempt of attempts) {
|
|
1655
|
-
const candidate = input.candidateById.get(attempt.candidateId);
|
|
1656
|
-
const executionCostUsd = candidate.externalCostUsdPerSequence ?? 0;
|
|
1657
|
-
if (executionCostUsd > 0 && !hasSettledPaidCall(input.costLedger, memoryAttemptCostCallId(attempt, "execute", 0))) {
|
|
1658
|
-
appendMemoryAttemptEvent(input.storage, input.attemptLogPath, {
|
|
1659
|
-
...attempt,
|
|
1660
|
-
status: "cleaned",
|
|
1661
|
-
recovery: true,
|
|
1662
|
-
recordedAt: (input.options.now ?? (() => /* @__PURE__ */ new Date()))().toISOString()
|
|
1663
|
-
});
|
|
1664
|
-
}
|
|
1665
|
-
}
|
|
1666
|
-
attempts = readActiveMemoryAttempts(input.storage, input.attemptLogPath);
|
|
1667
|
-
const recoveryGenerations = reserveRecoveryAttempts({
|
|
1668
|
-
storage: input.storage,
|
|
1669
|
-
path: input.recoveryLogPath,
|
|
1670
|
-
attemptIds: attempts.map((attempt) => attempt.branchId),
|
|
1671
|
-
maxRetriesPerAttempt: input.maxRecoveryRetriesPerAttempt,
|
|
1672
|
-
label: "memory recovery attempt log",
|
|
1673
|
-
now: input.options.now
|
|
1674
|
-
});
|
|
1675
|
-
const pool = createMemoryExecutionPool(input.maxConcurrency);
|
|
1676
|
-
const settled = await Promise.allSettled(
|
|
1677
|
-
attempts.sort((left, right) => left.branchId.localeCompare(right.branchId)).map(
|
|
1678
|
-
(attempt) => pool.run(async () => {
|
|
1679
|
-
await input.lease.assertOwned();
|
|
1680
|
-
const candidate = input.candidateById.get(attempt.candidateId);
|
|
1681
|
-
if (!candidate) {
|
|
1682
|
-
throw new Error(
|
|
1683
|
-
`cannot recover memory branch '${attempt.branchId}': candidate '${attempt.candidateId}' is missing; pass it in recoveryCandidates`
|
|
1684
|
-
);
|
|
1415
|
+
}
|
|
1416
|
+
attempts = readActiveMemoryAttempts(input.storage, input.attemptLogPath);
|
|
1417
|
+
const recoveryGenerations = reserveRecoveryAttempts({
|
|
1418
|
+
storage: input.storage,
|
|
1419
|
+
path: input.recoveryLogPath,
|
|
1420
|
+
attemptIds: attempts.map((attempt) => attempt.branchId),
|
|
1421
|
+
maxRetriesPerAttempt: input.maxRecoveryRetriesPerAttempt,
|
|
1422
|
+
label: "memory recovery attempt log",
|
|
1423
|
+
now: input.options.now
|
|
1424
|
+
});
|
|
1425
|
+
const pool = createMemoryExecutionPool(input.maxConcurrency);
|
|
1426
|
+
const settled = await Promise.allSettled(
|
|
1427
|
+
attempts.sort((left, right) => left.branchId.localeCompare(right.branchId)).map(
|
|
1428
|
+
(attempt) => pool.run(async () => {
|
|
1429
|
+
await input.lease.assertOwned();
|
|
1430
|
+
const candidate = input.candidateById.get(attempt.candidateId);
|
|
1431
|
+
if (!candidate) {
|
|
1432
|
+
throw new Error(
|
|
1433
|
+
`cannot recover memory branch '${attempt.branchId}': candidate '${attempt.candidateId}' is missing; pass it in recoveryCandidates`
|
|
1434
|
+
);
|
|
1685
1435
|
}
|
|
1686
1436
|
assertMemoryAttemptCandidateMatches(attempt, candidate);
|
|
1687
1437
|
const sequence = input.sequenceById.get(attempt.sequenceId);
|
|
@@ -1972,418 +1722,687 @@ function assertMemoryAttemptCandidateMatches(attempt, candidate) {
|
|
|
1972
1722
|
);
|
|
1973
1723
|
}
|
|
1974
1724
|
}
|
|
1975
|
-
|
|
1976
|
-
|
|
1977
|
-
|
|
1978
|
-
|
|
1979
|
-
|
|
1980
|
-
|
|
1981
|
-
|
|
1982
|
-
|
|
1983
|
-
|
|
1984
|
-
for (const write of writes) await memory.write(write);
|
|
1985
|
-
}
|
|
1986
|
-
async function probeStep(memory, sequence, step) {
|
|
1987
|
-
const run = async (probe) => {
|
|
1988
|
-
const scope = mergeScopes2(step.scope, probe.scope);
|
|
1989
|
-
const context = await memory.getContext(probe.query, { scope, limit: probe.limit });
|
|
1990
|
-
const artifact = {
|
|
1991
|
-
answer: context.text,
|
|
1992
|
-
rememberedFacts: context.hits.map((hit) => hit.text),
|
|
1993
|
-
citedEventIds: uniqueStrings([
|
|
1994
|
-
...context.hits.map((hit) => hit.metadata?.eventId),
|
|
1995
|
-
...context.hits.flatMap((hit) => stringArray(hit.metadata?.eventIds))
|
|
1996
|
-
]),
|
|
1997
|
-
usedMemoryIds: context.hits.map((hit) => hit.id),
|
|
1998
|
-
actorIds: uniqueStrings([
|
|
1999
|
-
...context.hits.map((hit) => hit.metadata?.actorId),
|
|
2000
|
-
...context.hits.flatMap((hit) => stringArray(hit.metadata?.actorIds))
|
|
2001
|
-
])
|
|
2002
|
-
};
|
|
2003
|
-
const evaluation = scoreMemoryBenchmarkArtifact(
|
|
2004
|
-
{
|
|
2005
|
-
id: `${sequence.id}:${step.id}:${probe.id}`,
|
|
2006
|
-
family: sequence.family,
|
|
2007
|
-
taskKind: probe.taskKind ?? "memory-recall",
|
|
2008
|
-
split: sequence.split,
|
|
2009
|
-
events: [],
|
|
2010
|
-
prompt: probe.query,
|
|
2011
|
-
requiredFacts: probe.requiredFacts && probe.requiredFacts.length > 0 ? probe.requiredFacts : probe.referenceAnswer ? [{ id: `${probe.id}:reference`, anyOf: [probe.referenceAnswer] }] : void 0,
|
|
2012
|
-
forbiddenFacts: probe.forbiddenFacts,
|
|
2013
|
-
expectedEventIds: probe.expectedEventIds,
|
|
2014
|
-
expectedActorIds: probe.expectedActorIds,
|
|
2015
|
-
referenceAnswer: probe.referenceAnswer
|
|
2016
|
-
},
|
|
2017
|
-
artifact
|
|
1725
|
+
|
|
1726
|
+
// src/memory/experiment/cell.ts
|
|
1727
|
+
async function runSequenceCell(input) {
|
|
1728
|
+
const { options, candidate, scenario, context, runIdentity, storage, attemptLogPath, lease } = input;
|
|
1729
|
+
const cleanupBranches = options.cleanupBranches ?? true;
|
|
1730
|
+
const costUsd = candidate.externalCostUsdPerSequence ?? 0;
|
|
1731
|
+
if (!Number.isFinite(costUsd) || costUsd < 0) {
|
|
1732
|
+
throw new Error(
|
|
1733
|
+
`${candidate.id}: externalCostUsdPerSequence must be a non-negative finite number`
|
|
2018
1734
|
);
|
|
2019
|
-
return {
|
|
2020
|
-
id: probe.id,
|
|
2021
|
-
stepId: step.id,
|
|
2022
|
-
query: probe.query,
|
|
2023
|
-
score: evaluation.score,
|
|
2024
|
-
passed: evaluation.passed,
|
|
2025
|
-
dimensions: evaluation.dimensions,
|
|
2026
|
-
applicableDimensions: evaluation.applicableDimensions ?? Object.keys(evaluation.dimensions),
|
|
2027
|
-
notes: evaluation.notes,
|
|
2028
|
-
hitIds: context.hits.map((hit) => hit.id)
|
|
2029
|
-
};
|
|
2030
|
-
};
|
|
2031
|
-
if (step.parallelProbes === false) {
|
|
2032
|
-
const results = [];
|
|
2033
|
-
for (const probe of step.probes ?? []) results.push(await run(probe));
|
|
2034
|
-
return results;
|
|
2035
1735
|
}
|
|
2036
|
-
|
|
2037
|
-
|
|
2038
|
-
|
|
2039
|
-
const scenarioById = new Map(scenarios.map((scenario) => [scenario.id, scenario]));
|
|
2040
|
-
const rows = candidates.map((candidate) => {
|
|
2041
|
-
const candidateScenarios = scenarios.filter((scenario) => scenario.candidateId === candidate.id);
|
|
2042
|
-
const cells = campaign.cells.filter(
|
|
2043
|
-
(cell) => scenarioById.get(cell.scenarioId)?.candidateId === candidate.id
|
|
1736
|
+
if (!cleanupBranches && !candidate.disposeAdapter) {
|
|
1737
|
+
throw new Error(
|
|
1738
|
+
`${candidate.id}: cleanupBranches=false requires disposeAdapter to delete isolated external state`
|
|
2044
1739
|
);
|
|
2045
|
-
const successful = cells.filter((cell) => !cell.error && cell.artifact);
|
|
2046
|
-
const dimensionRows = successful.map((cell) => cell.artifact.dimensions);
|
|
2047
|
-
return {
|
|
2048
|
-
rank: 0,
|
|
2049
|
-
candidateId: candidate.id,
|
|
2050
|
-
label: candidate.label ?? candidate.id,
|
|
2051
|
-
scoreMean: mean(
|
|
2052
|
-
cells.map((cell) => !cell.error && cell.artifact ? cell.artifact.score : 0)
|
|
2053
|
-
),
|
|
2054
|
-
passRate: mean(cells.map((cell) => !cell.error && cell.artifact?.passed ? 1 : 0)),
|
|
2055
|
-
totalSequences: candidateScenarios.length,
|
|
2056
|
-
totalCells: cells.length,
|
|
2057
|
-
totalProbes: successful.reduce((sum, cell) => sum + cell.artifact.probes.length, 0),
|
|
2058
|
-
cellsFailed: cells.filter((cell) => Boolean(cell.error)).length,
|
|
2059
|
-
totalCostUsd: normalizeUsd(costByCandidate.get(candidate.id) ?? 0),
|
|
2060
|
-
durationMs: cells.reduce((sum, cell) => sum + cell.durationMs, 0),
|
|
2061
|
-
dimensions: meanDimensions(dimensionRows)
|
|
2062
|
-
};
|
|
2063
|
-
});
|
|
2064
|
-
return rows.sort(
|
|
2065
|
-
(a, b) => Number(a.cellsFailed > 0) - Number(b.cellsFailed > 0) || b.scoreMean - a.scoreMean || b.passRate - a.passRate || a.totalCostUsd - b.totalCostUsd || a.candidateId.localeCompare(b.candidateId)
|
|
2066
|
-
).map((row, index) => ({ ...row, rank: index + 1 }));
|
|
2067
|
-
}
|
|
2068
|
-
function memoryExperimentCostByCandidate(costLedger, runDir, scenarios, candidateIdsInput) {
|
|
2069
|
-
const candidateByScenario = new Map(
|
|
2070
|
-
scenarios.map((scenario) => [scenario.id, scenario.candidateId])
|
|
2071
|
-
);
|
|
2072
|
-
const candidateIds = new Set(candidateIdsInput);
|
|
2073
|
-
const totals = /* @__PURE__ */ new Map();
|
|
2074
|
-
for (const receipt of costLedger.list()) {
|
|
2075
|
-
if (receipt.tags?.runDir !== runDir) continue;
|
|
2076
|
-
const scenarioCandidate = receipt.tags.scenarioId ? candidateByScenario.get(receipt.tags.scenarioId) : void 0;
|
|
2077
|
-
const recoveryCandidate = receipt.tags.memoryRecovery === "attempt" && receipt.tags.candidateId ? receipt.tags.candidateId : void 0;
|
|
2078
|
-
const candidateId = scenarioCandidate ?? recoveryCandidate;
|
|
2079
|
-
if (!candidateId || !candidateIds.has(candidateId)) continue;
|
|
2080
|
-
totals.set(candidateId, (totals.get(candidateId) ?? 0) + receipt.costUsd);
|
|
2081
1740
|
}
|
|
2082
|
-
|
|
2083
|
-
|
|
2084
|
-
|
|
2085
|
-
return [
|
|
2086
|
-
"# Agent Memory Experiment",
|
|
2087
|
-
"",
|
|
2088
|
-
`- total cost: $${format(totalCostUsd)}`,
|
|
2089
|
-
`- retired-candidate recovery cost: $${format(unrankedRecoveryCostUsd)}`,
|
|
2090
|
-
"",
|
|
2091
|
-
"| rank | candidate | sequences | cells | probes | failed | score | pass rate | cost | duration ms |",
|
|
2092
|
-
"| ---: | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |",
|
|
2093
|
-
...rows.map(
|
|
2094
|
-
(row) => `| ${row.rank} | ${row.label} | ${row.totalSequences} | ${row.totalCells} | ${row.totalProbes} | ${row.cellsFailed} | ${format(row.scoreMean)} | ${format(row.passRate)} | $${format(row.totalCostUsd)} | ${format(row.durationMs)} |`
|
|
2095
|
-
),
|
|
2096
|
-
""
|
|
2097
|
-
].join("\n");
|
|
2098
|
-
}
|
|
2099
|
-
function memoryExperimentDispatchRef(options) {
|
|
2100
|
-
return stableId(
|
|
2101
|
-
"memory_experiment",
|
|
2102
|
-
canonicalJson3({
|
|
2103
|
-
implementationRef: MEMORY_EXPERIMENT_IMPLEMENTATION_REF,
|
|
2104
|
-
experimentId: options.experimentId,
|
|
2105
|
-
experimentRunId: options.experimentRunId ?? null,
|
|
2106
|
-
executeStepRef: options.executeStepRef ?? "fixtures",
|
|
2107
|
-
cleanupBranches: options.cleanupBranches ?? true,
|
|
2108
|
-
candidates: options.candidates.map((candidate) => ({
|
|
2109
|
-
id: candidate.id,
|
|
2110
|
-
ref: candidate.ref,
|
|
2111
|
-
policy: candidate.policy ?? null,
|
|
2112
|
-
baseScope: candidate.baseScope ?? null,
|
|
2113
|
-
externalCostUsdPerSequence: candidate.externalCostUsdPerSequence ?? 0,
|
|
2114
|
-
externalRecoveryCostUsdPerAttempt: candidate.externalRecoveryCostUsdPerAttempt ?? 0
|
|
2115
|
-
})).sort((a, b) => a.id.localeCompare(b.id))
|
|
2116
|
-
})
|
|
1741
|
+
const cleanupTimeoutMs = resolveMemoryCleanupTimeoutMs(
|
|
1742
|
+
options.cleanupTimeoutMs,
|
|
1743
|
+
`${candidate.id}: memory sequence`
|
|
2117
1744
|
);
|
|
2118
|
-
|
|
2119
|
-
|
|
2120
|
-
const
|
|
2121
|
-
|
|
2122
|
-
|
|
2123
|
-
|
|
2124
|
-
|
|
2125
|
-
|
|
2126
|
-
|
|
2127
|
-
|
|
2128
|
-
|
|
2129
|
-
|
|
2130
|
-
|
|
2131
|
-
|
|
2132
|
-
|
|
2133
|
-
|
|
2134
|
-
|
|
2135
|
-
}
|
|
2136
|
-
return Object.fromEntries(counts);
|
|
2137
|
-
}
|
|
2138
|
-
function mean(values) {
|
|
2139
|
-
return values.length === 0 ? 0 : values.reduce((sum, value) => sum + value, 0) / values.length;
|
|
2140
|
-
}
|
|
2141
|
-
function mergeScopes2(base, extra) {
|
|
2142
|
-
return {
|
|
2143
|
-
...base ?? {},
|
|
2144
|
-
...extra ?? {},
|
|
2145
|
-
tags: { ...base?.tags ?? {}, ...extra?.tags ?? {} }
|
|
2146
|
-
};
|
|
2147
|
-
}
|
|
2148
|
-
function memoryExperimentBaseScope(options, candidate, sequenceId) {
|
|
2149
|
-
return mergeScopes2(candidate.baseScope, {
|
|
2150
|
-
tags: {
|
|
2151
|
-
memoryExperimentId: options.experimentId,
|
|
2152
|
-
memoryCandidateId: candidate.id,
|
|
2153
|
-
memorySequenceId: sequenceId
|
|
2154
|
-
}
|
|
1745
|
+
context.signal.throwIfAborted();
|
|
1746
|
+
await lease.assertOwned();
|
|
1747
|
+
const startedAt = Date.now();
|
|
1748
|
+
const branchId = stableId(
|
|
1749
|
+
"memory_branch",
|
|
1750
|
+
`${runIdentity}:${context.cellId}:${context.seed}:${randomUUID2()}`
|
|
1751
|
+
);
|
|
1752
|
+
const attempt = memoryAttemptEvent({
|
|
1753
|
+
status: "started",
|
|
1754
|
+
branchId,
|
|
1755
|
+
candidate,
|
|
1756
|
+
sequence: scenario.sequence,
|
|
1757
|
+
rep: context.rep,
|
|
1758
|
+
seed: context.seed,
|
|
1759
|
+
recovery: false,
|
|
1760
|
+
cleanupBranches,
|
|
1761
|
+
now: options.now
|
|
2155
1762
|
});
|
|
2156
|
-
|
|
2157
|
-
|
|
2158
|
-
const
|
|
2159
|
-
|
|
2160
|
-
|
|
2161
|
-
|
|
2162
|
-
|
|
2163
|
-
|
|
2164
|
-
|
|
2165
|
-
|
|
2166
|
-
|
|
2167
|
-
|
|
2168
|
-
|
|
2169
|
-
for (const scope of candidates) {
|
|
2170
|
-
const normalized = normalizeCleanupScope(scope);
|
|
2171
|
-
scopes.set(JSON.stringify(normalized), normalized);
|
|
2172
|
-
}
|
|
2173
|
-
}
|
|
2174
|
-
return [...scopes.values()];
|
|
2175
|
-
}
|
|
2176
|
-
async function clearSequenceScopes(memory, sequence) {
|
|
2177
|
-
for (const scope of sequenceCleanupScopes(sequence)) await memory.clear?.(scope);
|
|
2178
|
-
}
|
|
2179
|
-
function assertMemorySequences(sequences) {
|
|
2180
|
-
for (const sequence of sequences) {
|
|
2181
|
-
assertNonEmptyString(sequence.family, `memory experiment sequence ${sequence.id} family`);
|
|
2182
|
-
if (sequence.steps.length === 0) {
|
|
2183
|
-
throw new Error(`memory experiment sequence ${sequence.id} has no steps`);
|
|
2184
|
-
}
|
|
2185
|
-
assertUnique(
|
|
2186
|
-
sequence.steps.map((step) => step.id),
|
|
2187
|
-
`step in sequence ${sequence.id}`
|
|
2188
|
-
);
|
|
2189
|
-
let probeCount = 0;
|
|
2190
|
-
for (const step of sequence.steps) {
|
|
2191
|
-
for (const write of step.writes ?? []) {
|
|
2192
|
-
assertNonEmptyString(write.text, `memory experiment write in ${sequence.id}/${step.id}`);
|
|
2193
|
-
if (write.id !== void 0) {
|
|
2194
|
-
assertNonEmptyString(write.id, `memory experiment write id in ${sequence.id}/${step.id}`);
|
|
2195
|
-
}
|
|
2196
|
-
}
|
|
2197
|
-
assertUnique(
|
|
2198
|
-
(step.probes ?? []).map((probe) => probe.id),
|
|
2199
|
-
`probe in sequence ${sequence.id} step ${step.id}`
|
|
1763
|
+
appendMemoryAttemptEvent(storage, attemptLogPath, attempt);
|
|
1764
|
+
let externalCallAttempted = false;
|
|
1765
|
+
const appendCleanedAttempt = (priorError) => {
|
|
1766
|
+
try {
|
|
1767
|
+
appendMemoryAttemptEvent(storage, attemptLogPath, {
|
|
1768
|
+
...attempt,
|
|
1769
|
+
status: "cleaned",
|
|
1770
|
+
recordedAt: (options.now ?? (() => /* @__PURE__ */ new Date()))().toISOString()
|
|
1771
|
+
});
|
|
1772
|
+
} catch (error) {
|
|
1773
|
+
throw new AgentMemoryCleanupError(
|
|
1774
|
+
[...priorError ? [priorError] : [], error],
|
|
1775
|
+
`${candidate.id}: memory branch cleanup could not be recorded`
|
|
2200
1776
|
);
|
|
2201
|
-
|
|
2202
|
-
|
|
2203
|
-
|
|
2204
|
-
|
|
2205
|
-
|
|
2206
|
-
|
|
2207
|
-
|
|
2208
|
-
|
|
2209
|
-
|
|
2210
|
-
|
|
1777
|
+
}
|
|
1778
|
+
};
|
|
1779
|
+
const execute = async () => {
|
|
1780
|
+
context.signal.throwIfAborted();
|
|
1781
|
+
await lease.assertOwned();
|
|
1782
|
+
let rawAdapter;
|
|
1783
|
+
let adapter;
|
|
1784
|
+
let memory;
|
|
1785
|
+
let primaryError;
|
|
1786
|
+
let completedArtifact;
|
|
1787
|
+
let finalClearStarted = false;
|
|
1788
|
+
let finalClearCompleted = false;
|
|
1789
|
+
try {
|
|
1790
|
+
const created = await candidate.createAdapter({
|
|
1791
|
+
branchId,
|
|
1792
|
+
sequence: scenario.sequence,
|
|
1793
|
+
rep: context.rep,
|
|
1794
|
+
seed: context.seed,
|
|
1795
|
+
purpose: "execute",
|
|
1796
|
+
signal: context.signal,
|
|
1797
|
+
markExternalCall: () => {
|
|
1798
|
+
externalCallAttempted = true;
|
|
2211
1799
|
}
|
|
2212
|
-
|
|
2213
|
-
|
|
2214
|
-
|
|
2215
|
-
|
|
2216
|
-
|
|
2217
|
-
|
|
2218
|
-
|
|
1800
|
+
});
|
|
1801
|
+
if (!created) throw new Error(`${candidate.id}: createAdapter returned no execution adapter`);
|
|
1802
|
+
rawAdapter = created;
|
|
1803
|
+
adapter = trackExternalMemoryCalls(created, () => {
|
|
1804
|
+
externalCallAttempted = true;
|
|
1805
|
+
});
|
|
1806
|
+
await lease.assertOwned();
|
|
1807
|
+
context.signal.throwIfAborted();
|
|
1808
|
+
if (cleanupBranches && !adapter.clear) {
|
|
1809
|
+
throw new Error(
|
|
1810
|
+
`${candidate.id}: cleanupBranches requires an adapter with scoped clear support`
|
|
2219
1811
|
);
|
|
2220
|
-
|
|
2221
|
-
|
|
2222
|
-
|
|
2223
|
-
|
|
2224
|
-
|
|
2225
|
-
|
|
2226
|
-
|
|
2227
|
-
|
|
2228
|
-
|
|
2229
|
-
|
|
2230
|
-
|
|
2231
|
-
|
|
2232
|
-
|
|
1812
|
+
}
|
|
1813
|
+
memory = createAgentMemoryBranch({
|
|
1814
|
+
adapter,
|
|
1815
|
+
branchId,
|
|
1816
|
+
lifetime: "attempt",
|
|
1817
|
+
policy: candidate.policy,
|
|
1818
|
+
allowedWriteScopes: sequenceCleanupScopes(scenario.sequence),
|
|
1819
|
+
baseScope: memoryExperimentBaseScope(options, candidate, scenario.sequenceId)
|
|
1820
|
+
});
|
|
1821
|
+
const probes = [];
|
|
1822
|
+
for (const step of scenario.sequence.steps) {
|
|
1823
|
+
context.signal.throwIfAborted();
|
|
1824
|
+
await lease.assertOwned();
|
|
1825
|
+
await writeStep(memory, step);
|
|
1826
|
+
await lease.assertOwned();
|
|
1827
|
+
await options.executeStep?.({
|
|
1828
|
+
memory,
|
|
1829
|
+
candidateId: candidate.id,
|
|
1830
|
+
sequence: scenario.sequence,
|
|
1831
|
+
step,
|
|
1832
|
+
context
|
|
1833
|
+
});
|
|
1834
|
+
context.signal.throwIfAborted();
|
|
1835
|
+
await lease.assertOwned();
|
|
1836
|
+
const stepProbes = await probeStep(memory, scenario.sequence, step);
|
|
1837
|
+
await lease.assertOwned();
|
|
1838
|
+
probes.push(...stepProbes);
|
|
1839
|
+
}
|
|
1840
|
+
const snapshot = await memory.snapshot();
|
|
1841
|
+
await lease.assertOwned();
|
|
1842
|
+
await options.onBranchSnapshot?.({
|
|
1843
|
+
candidateId: candidate.id,
|
|
1844
|
+
sequenceId: scenario.sequenceId,
|
|
1845
|
+
cellId: context.cellId,
|
|
1846
|
+
snapshot
|
|
1847
|
+
});
|
|
1848
|
+
await lease.assertOwned();
|
|
1849
|
+
const dimensions = meanDimensions(probes.map((probe) => probe.dimensions));
|
|
1850
|
+
const dimensionSampleCounts = countDimensions(
|
|
1851
|
+
probes.map((probe) => probe.applicableDimensions)
|
|
1852
|
+
);
|
|
1853
|
+
const artifact = {
|
|
1854
|
+
candidateId: candidate.id,
|
|
1855
|
+
sequenceId: scenario.sequenceId,
|
|
1856
|
+
score: mean(probes.map((probe) => probe.score)),
|
|
1857
|
+
passed: probes.length > 0 && probes.every((probe) => probe.passed),
|
|
1858
|
+
dimensions,
|
|
1859
|
+
dimensionSampleCounts,
|
|
1860
|
+
probes,
|
|
1861
|
+
branchDigest: snapshot.digest,
|
|
1862
|
+
journalEntries: snapshot.journal.length,
|
|
1863
|
+
durationMs: Math.max(0, Date.now() - startedAt)
|
|
1864
|
+
};
|
|
1865
|
+
if (cleanupBranches) {
|
|
1866
|
+
finalClearStarted = true;
|
|
1867
|
+
await runBoundedMemoryLifecycle({
|
|
1868
|
+
operation: `${candidate.id}: final branch cleanup`,
|
|
1869
|
+
timeoutMs: cleanupTimeoutMs,
|
|
1870
|
+
resource: adapter,
|
|
1871
|
+
run: () => clearSequenceScopes(memory, scenario.sequence)
|
|
1872
|
+
});
|
|
1873
|
+
finalClearCompleted = true;
|
|
1874
|
+
await lease.assertOwned();
|
|
1875
|
+
}
|
|
1876
|
+
completedArtifact = artifact;
|
|
1877
|
+
} catch (error) {
|
|
1878
|
+
primaryError = error;
|
|
1879
|
+
}
|
|
1880
|
+
const cleanupErrors = [];
|
|
1881
|
+
let cleanupOwned = true;
|
|
1882
|
+
let ownershipError;
|
|
1883
|
+
try {
|
|
1884
|
+
await lease.assertOwned();
|
|
1885
|
+
} catch (error) {
|
|
1886
|
+
cleanupOwned = false;
|
|
1887
|
+
ownershipError = error;
|
|
1888
|
+
}
|
|
1889
|
+
if (finalClearStarted && !finalClearCompleted && primaryError) {
|
|
1890
|
+
cleanupErrors.push(primaryError);
|
|
1891
|
+
}
|
|
1892
|
+
if (primaryError && cleanupOwned && !finalClearStarted && memory && cleanupBranches && adapter?.clear) {
|
|
1893
|
+
try {
|
|
1894
|
+
await runBoundedMemoryLifecycle({
|
|
1895
|
+
operation: `${candidate.id}: failed branch cleanup`,
|
|
1896
|
+
timeoutMs: cleanupTimeoutMs,
|
|
1897
|
+
resource: adapter,
|
|
1898
|
+
run: () => clearSequenceScopes(memory, scenario.sequence)
|
|
1899
|
+
});
|
|
1900
|
+
} catch (error) {
|
|
1901
|
+
cleanupErrors.push(error);
|
|
1902
|
+
}
|
|
1903
|
+
}
|
|
1904
|
+
if (adapter) {
|
|
1905
|
+
try {
|
|
1906
|
+
await runBoundedMemoryLifecycle({
|
|
1907
|
+
operation: `${candidate.id}: adapter close`,
|
|
1908
|
+
timeoutMs: cleanupTimeoutMs,
|
|
1909
|
+
resource: adapter,
|
|
1910
|
+
run: async () => {
|
|
1911
|
+
if (memory && cleanupOwned) await memory.close?.();
|
|
1912
|
+
else {
|
|
1913
|
+
if (cleanupOwned) await adapter.flush?.();
|
|
1914
|
+
await adapter.close?.();
|
|
1915
|
+
}
|
|
1916
|
+
}
|
|
1917
|
+
});
|
|
1918
|
+
} catch (error) {
|
|
1919
|
+
cleanupErrors.push(error);
|
|
1920
|
+
}
|
|
1921
|
+
if (cleanupOwned) {
|
|
1922
|
+
try {
|
|
1923
|
+
await runBoundedMemoryLifecycle({
|
|
1924
|
+
operation: `${candidate.id}: adapter disposal`,
|
|
1925
|
+
timeoutMs: cleanupTimeoutMs,
|
|
1926
|
+
resource: adapter,
|
|
1927
|
+
run: () => {
|
|
1928
|
+
if (candidate.disposeAdapter) externalCallAttempted = true;
|
|
1929
|
+
return candidate.disposeAdapter?.(rawAdapter);
|
|
1930
|
+
}
|
|
1931
|
+
});
|
|
1932
|
+
} catch (error) {
|
|
1933
|
+
cleanupErrors.push(error);
|
|
2233
1934
|
}
|
|
2234
1935
|
}
|
|
1936
|
+
} else {
|
|
1937
|
+
cleanupErrors.push(
|
|
1938
|
+
new Error(`${candidate.id}: adapter creation failed before cleanup could be confirmed`)
|
|
1939
|
+
);
|
|
2235
1940
|
}
|
|
2236
|
-
if (
|
|
2237
|
-
|
|
1941
|
+
if (cleanupOwned && cleanupErrors.length === 0) appendCleanedAttempt(primaryError);
|
|
1942
|
+
if (!cleanupOwned && cleanupErrors.length === 0) {
|
|
1943
|
+
if (primaryError) throw primaryError;
|
|
1944
|
+
throw ownershipError;
|
|
1945
|
+
}
|
|
1946
|
+
if (cleanupErrors.length > 0) {
|
|
1947
|
+
const primaryMessage = primaryError instanceof Error ? primaryError.message : String(primaryError ?? "");
|
|
1948
|
+
throw new AgentMemoryCleanupError(
|
|
1949
|
+
[
|
|
1950
|
+
...primaryError && !cleanupErrors.includes(primaryError) ? [primaryError] : [],
|
|
1951
|
+
...ownershipError ? [ownershipError] : [],
|
|
1952
|
+
...cleanupErrors
|
|
1953
|
+
],
|
|
1954
|
+
`${candidate.id}: memory branch cleanup failed${primaryMessage ? ` after: ${primaryMessage}` : ""}`
|
|
1955
|
+
);
|
|
1956
|
+
}
|
|
1957
|
+
if (primaryError) throw primaryError;
|
|
1958
|
+
if (!completedArtifact) throw new Error(`${candidate.id}: memory sequence produced no result`);
|
|
1959
|
+
return completedArtifact;
|
|
1960
|
+
};
|
|
1961
|
+
if (costUsd === 0) {
|
|
1962
|
+
let artifact;
|
|
1963
|
+
let error;
|
|
1964
|
+
try {
|
|
1965
|
+
artifact = await execute();
|
|
1966
|
+
} catch (caught) {
|
|
1967
|
+
error = caught;
|
|
2238
1968
|
}
|
|
1969
|
+
if (error) throw error;
|
|
1970
|
+
if (!artifact) throw new Error(`${candidate.id}: memory sequence produced no result`);
|
|
1971
|
+
return artifact;
|
|
2239
1972
|
}
|
|
1973
|
+
const receipt = {
|
|
1974
|
+
model: candidate.id,
|
|
1975
|
+
inputTokens: 0,
|
|
1976
|
+
outputTokens: 0,
|
|
1977
|
+
actualCostUsd: costUsd
|
|
1978
|
+
};
|
|
1979
|
+
const paid = await context.cost.runPaidCall({
|
|
1980
|
+
callId: memoryAttemptCostCallId(attempt, "execute", 0),
|
|
1981
|
+
actor: `agent-knowledge:memory-experiment:${candidate.id}`,
|
|
1982
|
+
model: candidate.id,
|
|
1983
|
+
maximumCharge: { externallyEnforcedMaximumUsd: costUsd },
|
|
1984
|
+
execute,
|
|
1985
|
+
receipt: () => receipt,
|
|
1986
|
+
receiptFromError: () => ({
|
|
1987
|
+
...receipt,
|
|
1988
|
+
actualCostUsd: externalCallAttempted ? costUsd : 0
|
|
1989
|
+
})
|
|
1990
|
+
});
|
|
1991
|
+
if (!paid.succeeded) throw paid.error;
|
|
1992
|
+
return paid.value;
|
|
2240
1993
|
}
|
|
2241
|
-
function
|
|
2242
|
-
const
|
|
2243
|
-
|
|
2244
|
-
|
|
1994
|
+
async function writeStep(memory, step) {
|
|
1995
|
+
const writes = (step.writes ?? []).map((write) => ({
|
|
1996
|
+
...write,
|
|
1997
|
+
scope: mergeScopes2(step.scope, write.scope)
|
|
1998
|
+
}));
|
|
1999
|
+
if (step.parallelWrites) {
|
|
2000
|
+
await Promise.all(writes.map((write) => memory.write(write)));
|
|
2001
|
+
return;
|
|
2245
2002
|
}
|
|
2246
|
-
|
|
2003
|
+
for (const write of writes) await memory.write(write);
|
|
2247
2004
|
}
|
|
2248
|
-
function
|
|
2249
|
-
|
|
2250
|
-
|
|
2251
|
-
|
|
2252
|
-
|
|
2253
|
-
|
|
2254
|
-
|
|
2255
|
-
|
|
2256
|
-
|
|
2257
|
-
|
|
2258
|
-
|
|
2259
|
-
|
|
2260
|
-
|
|
2261
|
-
|
|
2262
|
-
|
|
2263
|
-
|
|
2264
|
-
|
|
2265
|
-
|
|
2266
|
-
|
|
2267
|
-
|
|
2005
|
+
async function probeStep(memory, sequence, step) {
|
|
2006
|
+
const run = async (probe) => {
|
|
2007
|
+
const scope = mergeScopes2(step.scope, probe.scope);
|
|
2008
|
+
const context = await memory.getContext(probe.query, { scope, limit: probe.limit });
|
|
2009
|
+
const artifact = {
|
|
2010
|
+
answer: context.text,
|
|
2011
|
+
rememberedFacts: context.hits.map((hit) => hit.text),
|
|
2012
|
+
citedEventIds: uniqueStrings([
|
|
2013
|
+
...context.hits.map((hit) => hit.metadata?.eventId),
|
|
2014
|
+
...context.hits.flatMap((hit) => stringArray(hit.metadata?.eventIds))
|
|
2015
|
+
]),
|
|
2016
|
+
usedMemoryIds: context.hits.map((hit) => hit.id),
|
|
2017
|
+
actorIds: uniqueStrings([
|
|
2018
|
+
...context.hits.map((hit) => hit.metadata?.actorId),
|
|
2019
|
+
...context.hits.flatMap((hit) => stringArray(hit.metadata?.actorIds))
|
|
2020
|
+
])
|
|
2021
|
+
};
|
|
2022
|
+
const evaluation = scoreMemoryBenchmarkArtifact(
|
|
2023
|
+
{
|
|
2024
|
+
id: `${sequence.id}:${step.id}:${probe.id}`,
|
|
2025
|
+
family: sequence.family,
|
|
2026
|
+
taskKind: probe.taskKind ?? "memory-recall",
|
|
2027
|
+
split: sequence.split,
|
|
2028
|
+
events: [],
|
|
2029
|
+
prompt: probe.query,
|
|
2030
|
+
requiredFacts: probe.requiredFacts && probe.requiredFacts.length > 0 ? probe.requiredFacts : probe.referenceAnswer ? [{ id: `${probe.id}:reference`, anyOf: [probe.referenceAnswer] }] : void 0,
|
|
2031
|
+
forbiddenFacts: probe.forbiddenFacts,
|
|
2032
|
+
expectedEventIds: probe.expectedEventIds,
|
|
2033
|
+
expectedActorIds: probe.expectedActorIds,
|
|
2034
|
+
referenceAnswer: probe.referenceAnswer
|
|
2035
|
+
},
|
|
2036
|
+
artifact
|
|
2037
|
+
);
|
|
2038
|
+
return {
|
|
2039
|
+
id: probe.id,
|
|
2040
|
+
stepId: step.id,
|
|
2041
|
+
query: probe.query,
|
|
2042
|
+
score: evaluation.score,
|
|
2043
|
+
passed: evaluation.passed,
|
|
2044
|
+
dimensions: evaluation.dimensions,
|
|
2045
|
+
applicableDimensions: evaluation.applicableDimensions ?? Object.keys(evaluation.dimensions),
|
|
2046
|
+
notes: evaluation.notes,
|
|
2047
|
+
hitIds: context.hits.map((hit) => hit.id)
|
|
2048
|
+
};
|
|
2049
|
+
};
|
|
2050
|
+
if (step.parallelProbes === false) {
|
|
2051
|
+
const results = [];
|
|
2052
|
+
for (const probe of step.probes ?? []) results.push(await run(probe));
|
|
2053
|
+
return results;
|
|
2268
2054
|
}
|
|
2055
|
+
return Promise.all((step.probes ?? []).map(run));
|
|
2269
2056
|
}
|
|
2270
|
-
|
|
2057
|
+
|
|
2058
|
+
// src/memory/experiment/run.ts
|
|
2059
|
+
var MEMORY_EXPERIMENT_IMPLEMENTATION_REF = "agent-knowledge:memory-experiment:v6";
|
|
2060
|
+
async function runAgentMemoryExperiment(options) {
|
|
2061
|
+
assertNonEmptyString(options.experimentId, "memory experiment experimentId");
|
|
2062
|
+
assertNonEmptyString(options.runDir, "memory experiment runDir");
|
|
2063
|
+
if (options.experimentRunId !== void 0) {
|
|
2064
|
+
assertNonEmptyString(options.experimentRunId, "memory experiment experimentRunId");
|
|
2065
|
+
}
|
|
2066
|
+
if (options.sequences.length === 0) throw new Error("memory experiment requires sequences");
|
|
2067
|
+
if (options.candidates.length === 0) throw new Error("memory experiment requires candidates");
|
|
2068
|
+
if (options.executeStep && !options.executeStepRef) {
|
|
2069
|
+
throw new Error("memory experiment executeStepRef is required when executeStep is configured");
|
|
2070
|
+
}
|
|
2271
2071
|
assertUnique(
|
|
2272
|
-
|
|
2273
|
-
|
|
2072
|
+
options.sequences.map((sequence) => sequence.id),
|
|
2073
|
+
"sequence"
|
|
2274
2074
|
);
|
|
2275
|
-
|
|
2276
|
-
|
|
2277
|
-
|
|
2075
|
+
assertUnique(
|
|
2076
|
+
[...options.candidates, ...options.recoveryCandidates ?? []].map((candidate) => candidate.id),
|
|
2077
|
+
"candidate"
|
|
2078
|
+
);
|
|
2079
|
+
assertMemorySequences(options.sequences);
|
|
2080
|
+
for (const candidate of [...options.candidates, ...options.recoveryCandidates ?? []]) {
|
|
2081
|
+
if (options.cleanupBranches === false && !candidate.disposeAdapter) {
|
|
2082
|
+
throw new Error(
|
|
2083
|
+
`${candidate.id}: cleanupBranches=false requires disposeAdapter to delete isolated external state`
|
|
2084
|
+
);
|
|
2278
2085
|
}
|
|
2279
|
-
|
|
2280
|
-
assertStringList(matcher.sourceEventIds, `${label} matcher ${matcher.id} source event ids`);
|
|
2281
|
-
if (matcher.weight !== void 0 && (!Number.isFinite(matcher.weight) || matcher.weight <= 0)) {
|
|
2086
|
+
if (candidate.externalCostUsdPerSequence !== void 0 && (!Number.isFinite(candidate.externalCostUsdPerSequence) || candidate.externalCostUsdPerSequence < 0)) {
|
|
2282
2087
|
throw new Error(
|
|
2283
|
-
|
|
2088
|
+
`${candidate.id}: externalCostUsdPerSequence must be a non-negative finite number`
|
|
2284
2089
|
);
|
|
2285
2090
|
}
|
|
2091
|
+
if (candidate.externalRecoveryCostUsdPerAttempt !== void 0 && (!Number.isFinite(candidate.externalRecoveryCostUsdPerAttempt) || candidate.externalRecoveryCostUsdPerAttempt < 0)) {
|
|
2092
|
+
throw new Error(
|
|
2093
|
+
`${candidate.id}: externalRecoveryCostUsdPerAttempt must be a non-negative finite number`
|
|
2094
|
+
);
|
|
2095
|
+
}
|
|
2096
|
+
assertNonEmptyString(candidate.ref, `${candidate.id} ref`);
|
|
2286
2097
|
}
|
|
2287
|
-
|
|
2288
|
-
|
|
2289
|
-
if (
|
|
2290
|
-
|
|
2291
|
-
}
|
|
2292
|
-
function assertNonEmptyString(value, label) {
|
|
2293
|
-
if (typeof value !== "string" || value.trim().length === 0) {
|
|
2294
|
-
throw new Error(`${label} must be a non-empty string`);
|
|
2098
|
+
const storage = options.storage ?? fsCampaignStorage();
|
|
2099
|
+
const runDir = resolveRunDir(options.runDir, options.repo);
|
|
2100
|
+
if (!storage.append) {
|
|
2101
|
+
throw new Error("memory experiment requires CampaignStorage.append for durable attempt state");
|
|
2295
2102
|
}
|
|
2103
|
+
resolveMemoryCleanupTimeoutMs(options.cleanupTimeoutMs, "memory experiment");
|
|
2104
|
+
const maxRecoveryAttempts = options.maxRecoveryAttempts ?? 1e3;
|
|
2105
|
+
if (!Number.isSafeInteger(maxRecoveryAttempts) || maxRecoveryAttempts <= 0) {
|
|
2106
|
+
throw new Error("memory experiment maxRecoveryAttempts must be a positive safe integer");
|
|
2107
|
+
}
|
|
2108
|
+
const maxRecoveryRetriesPerAttempt = options.maxRecoveryRetriesPerAttempt ?? DEFAULT_MEMORY_RECOVERY_RETRIES_PER_ATTEMPT;
|
|
2109
|
+
if (!Number.isSafeInteger(maxRecoveryRetriesPerAttempt) || maxRecoveryRetriesPerAttempt <= 0) {
|
|
2110
|
+
throw new Error(
|
|
2111
|
+
"memory experiment maxRecoveryRetriesPerAttempt must be a positive safe integer"
|
|
2112
|
+
);
|
|
2113
|
+
}
|
|
2114
|
+
storage.ensureDir(runDir);
|
|
2115
|
+
const lease = await acquireAgentMemoryRunLease({
|
|
2116
|
+
experimentId: options.experimentId,
|
|
2117
|
+
runDir,
|
|
2118
|
+
storage,
|
|
2119
|
+
customStorage: options.storage !== void 0,
|
|
2120
|
+
lockFileName: "memory-experiment.lock",
|
|
2121
|
+
label: "memory experiment",
|
|
2122
|
+
controllerMode: options.controllerMode,
|
|
2123
|
+
acquireRunLease: options.acquireRunLease
|
|
2124
|
+
});
|
|
2125
|
+
let result;
|
|
2126
|
+
let primaryError;
|
|
2127
|
+
try {
|
|
2128
|
+
await lease.assertOwned();
|
|
2129
|
+
result = await runOwnedAgentMemoryExperiment(options, storage, runDir, lease);
|
|
2130
|
+
} catch (error) {
|
|
2131
|
+
primaryError = error;
|
|
2132
|
+
}
|
|
2133
|
+
let releaseError;
|
|
2134
|
+
try {
|
|
2135
|
+
await lease.release();
|
|
2136
|
+
} catch (error) {
|
|
2137
|
+
releaseError = error;
|
|
2138
|
+
}
|
|
2139
|
+
if (primaryError && releaseError) {
|
|
2140
|
+
throw new AggregateError(
|
|
2141
|
+
[primaryError, releaseError],
|
|
2142
|
+
"memory experiment failed and its controller lease could not be released"
|
|
2143
|
+
);
|
|
2144
|
+
}
|
|
2145
|
+
if (primaryError) throw primaryError;
|
|
2146
|
+
if (releaseError) throw releaseError;
|
|
2147
|
+
if (!result) throw new Error("memory experiment produced no result");
|
|
2148
|
+
return result;
|
|
2296
2149
|
}
|
|
2297
|
-
function
|
|
2298
|
-
|
|
2299
|
-
|
|
2300
|
-
|
|
2301
|
-
|
|
2302
|
-
|
|
2303
|
-
|
|
2304
|
-
|
|
2305
|
-
|
|
2306
|
-
|
|
2307
|
-
|
|
2308
|
-
|
|
2309
|
-
|
|
2310
|
-
const
|
|
2311
|
-
const
|
|
2312
|
-
const
|
|
2313
|
-
|
|
2314
|
-
|
|
2315
|
-
|
|
2316
|
-
|
|
2317
|
-
|
|
2318
|
-
|
|
2319
|
-
|
|
2320
|
-
|
|
2321
|
-
|
|
2322
|
-
|
|
2323
|
-
|
|
2324
|
-
|
|
2325
|
-
|
|
2326
|
-
|
|
2327
|
-
|
|
2328
|
-
|
|
2329
|
-
|
|
2330
|
-
|
|
2331
|
-
|
|
2332
|
-
|
|
2333
|
-
|
|
2334
|
-
|
|
2335
|
-
|
|
2336
|
-
|
|
2337
|
-
|
|
2338
|
-
|
|
2339
|
-
|
|
2340
|
-
|
|
2341
|
-
|
|
2342
|
-
|
|
2343
|
-
|
|
2344
|
-
|
|
2345
|
-
|
|
2346
|
-
|
|
2347
|
-
|
|
2348
|
-
|
|
2349
|
-
|
|
2350
|
-
|
|
2351
|
-
|
|
2352
|
-
|
|
2353
|
-
|
|
2354
|
-
|
|
2355
|
-
|
|
2356
|
-
|
|
2357
|
-
|
|
2358
|
-
|
|
2359
|
-
|
|
2360
|
-
|
|
2361
|
-
|
|
2362
|
-
|
|
2363
|
-
|
|
2364
|
-
|
|
2365
|
-
|
|
2366
|
-
|
|
2367
|
-
|
|
2368
|
-
|
|
2369
|
-
|
|
2370
|
-
|
|
2371
|
-
|
|
2372
|
-
|
|
2373
|
-
|
|
2374
|
-
|
|
2375
|
-
|
|
2376
|
-
|
|
2377
|
-
|
|
2378
|
-
|
|
2379
|
-
|
|
2380
|
-
|
|
2381
|
-
|
|
2382
|
-
|
|
2383
|
-
|
|
2384
|
-
|
|
2385
|
-
|
|
2386
|
-
|
|
2150
|
+
async function runOwnedAgentMemoryExperiment(options, storage, runDir, lease) {
|
|
2151
|
+
const runIdentity = stableId(
|
|
2152
|
+
"memory_run",
|
|
2153
|
+
canonicalJson4({
|
|
2154
|
+
experimentId: options.experimentId,
|
|
2155
|
+
experimentRunId: options.experimentRunId ?? runDir
|
|
2156
|
+
})
|
|
2157
|
+
);
|
|
2158
|
+
const maxConcurrency = options.maxConcurrency ?? 2;
|
|
2159
|
+
if (!Number.isSafeInteger(maxConcurrency) || maxConcurrency <= 0) {
|
|
2160
|
+
throw new Error("memory experiment maxConcurrency must be a positive safe integer");
|
|
2161
|
+
}
|
|
2162
|
+
const executionPool = createMemoryExecutionPool(maxConcurrency);
|
|
2163
|
+
const dispatchedExecutions = [];
|
|
2164
|
+
const candidateById = new Map(options.candidates.map((candidate) => [candidate.id, candidate]));
|
|
2165
|
+
const recoveryCandidateById = new Map(
|
|
2166
|
+
[...options.candidates, ...options.recoveryCandidates ?? []].map((candidate) => [
|
|
2167
|
+
candidate.id,
|
|
2168
|
+
candidate
|
|
2169
|
+
])
|
|
2170
|
+
);
|
|
2171
|
+
const sequenceById = new Map(options.sequences.map((sequence) => [sequence.id, sequence]));
|
|
2172
|
+
const scenarios = buildAgentMemorySequenceScenarios(options.sequences, options.candidates);
|
|
2173
|
+
const attemptLogPath = join(runDir, "memory-attempts.jsonl");
|
|
2174
|
+
const recoveryLogPath = join(runDir, "memory-recovery-attempts.jsonl");
|
|
2175
|
+
const costCeiling = options.costCeiling ?? options.costLedger?.costCeilingUsd ?? 0;
|
|
2176
|
+
const costLedger = options.costLedger ?? createRunCostLedger({
|
|
2177
|
+
storage,
|
|
2178
|
+
runDir,
|
|
2179
|
+
costCeilingUsd: costCeiling
|
|
2180
|
+
});
|
|
2181
|
+
if (costLedger.costCeilingUsd !== costCeiling) {
|
|
2182
|
+
throw new Error("memory experiment costCeiling must match the shared cost ledger ceiling");
|
|
2183
|
+
}
|
|
2184
|
+
storage.ensureDir(runDir);
|
|
2185
|
+
await recoverAbandonedMemoryAttempts({
|
|
2186
|
+
options,
|
|
2187
|
+
storage,
|
|
2188
|
+
runDir,
|
|
2189
|
+
attemptLogPath,
|
|
2190
|
+
candidateById: recoveryCandidateById,
|
|
2191
|
+
sequenceById,
|
|
2192
|
+
lease,
|
|
2193
|
+
maxConcurrency,
|
|
2194
|
+
costLedger,
|
|
2195
|
+
maxRecoveryAttempts: options.maxRecoveryAttempts ?? 1e3,
|
|
2196
|
+
recoveryLogPath,
|
|
2197
|
+
maxRecoveryRetriesPerAttempt: options.maxRecoveryRetriesPerAttempt ?? DEFAULT_MEMORY_RECOVERY_RETRIES_PER_ATTEMPT
|
|
2198
|
+
});
|
|
2199
|
+
await lease.assertOwned();
|
|
2200
|
+
let campaign;
|
|
2201
|
+
let campaignError;
|
|
2202
|
+
let settledExecutions = [];
|
|
2203
|
+
try {
|
|
2204
|
+
campaign = await runCampaign({
|
|
2205
|
+
scenarios,
|
|
2206
|
+
dispatch: (scenario, context) => {
|
|
2207
|
+
const candidate = candidateById.get(scenario.candidateId);
|
|
2208
|
+
if (!candidate) throw new Error(`unknown memory candidate ${scenario.candidateId}`);
|
|
2209
|
+
const operation = executionPool.run(
|
|
2210
|
+
() => runSequenceCell({
|
|
2211
|
+
options,
|
|
2212
|
+
candidate,
|
|
2213
|
+
scenario,
|
|
2214
|
+
context,
|
|
2215
|
+
runIdentity,
|
|
2216
|
+
storage,
|
|
2217
|
+
attemptLogPath,
|
|
2218
|
+
lease
|
|
2219
|
+
})
|
|
2220
|
+
);
|
|
2221
|
+
dispatchedExecutions.push(operation);
|
|
2222
|
+
return operation;
|
|
2223
|
+
},
|
|
2224
|
+
dispatchRef: memoryExperimentDispatchRef(options),
|
|
2225
|
+
judges: [agentMemorySequenceJudge()],
|
|
2226
|
+
runDir,
|
|
2227
|
+
storage,
|
|
2228
|
+
seed: options.seed,
|
|
2229
|
+
reps: options.reps,
|
|
2230
|
+
resumable: options.resumable,
|
|
2231
|
+
costCeiling,
|
|
2232
|
+
costLedger,
|
|
2233
|
+
costPhase: options.costPhase,
|
|
2234
|
+
maxConcurrency,
|
|
2235
|
+
dispatchTimeoutMs: options.dispatchTimeoutMs,
|
|
2236
|
+
expectUsage: "off",
|
|
2237
|
+
now: options.now
|
|
2238
|
+
});
|
|
2239
|
+
} catch (error) {
|
|
2240
|
+
campaignError = error;
|
|
2241
|
+
} finally {
|
|
2242
|
+
settledExecutions = await Promise.allSettled(dispatchedExecutions);
|
|
2243
|
+
}
|
|
2244
|
+
const cleanupFailures = settledExecutions.flatMap(
|
|
2245
|
+
(settled) => settled.status === "rejected" && settled.reason instanceof AgentMemoryCleanupError ? [settled.reason] : []
|
|
2246
|
+
);
|
|
2247
|
+
if (campaignError && cleanupFailures.length > 0) {
|
|
2248
|
+
throw new AggregateError(
|
|
2249
|
+
[campaignError, ...cleanupFailures],
|
|
2250
|
+
"memory experiment failed and provider cleanup also failed"
|
|
2251
|
+
);
|
|
2252
|
+
}
|
|
2253
|
+
if (campaignError) throw campaignError;
|
|
2254
|
+
if (cleanupFailures.length > 0) {
|
|
2255
|
+
throw new AggregateError(cleanupFailures, "memory experiment cleanup failed after dispatch");
|
|
2256
|
+
}
|
|
2257
|
+
if (!campaign) throw new Error("memory experiment produced no campaign result");
|
|
2258
|
+
await lease.assertOwned();
|
|
2259
|
+
const costByCandidate = memoryExperimentCostByCandidate(
|
|
2260
|
+
costLedger,
|
|
2261
|
+
campaign.runDir,
|
|
2262
|
+
scenarios,
|
|
2263
|
+
[...options.candidates, ...options.recoveryCandidates ?? []].map((candidate) => candidate.id)
|
|
2264
|
+
);
|
|
2265
|
+
const unrankedRecoveryCostUsd = normalizeUsd(
|
|
2266
|
+
(options.recoveryCandidates ?? []).reduce(
|
|
2267
|
+
(sum, candidate) => sum + (costByCandidate.get(candidate.id) ?? 0),
|
|
2268
|
+
0
|
|
2269
|
+
)
|
|
2270
|
+
);
|
|
2271
|
+
const totalCostUsd = normalizeUsd(
|
|
2272
|
+
[...costByCandidate.values()].reduce((sum, cost) => sum + cost, 0)
|
|
2273
|
+
);
|
|
2274
|
+
const rows = rankAgentMemoryExperiment(options.candidates, scenarios, campaign, costByCandidate);
|
|
2275
|
+
const rankingJsonPath = join(campaign.runDir, "memory-experiment-ranking.json");
|
|
2276
|
+
const rankingMarkdownPath = join(campaign.runDir, "memory-experiment-ranking.md");
|
|
2277
|
+
storage.write(
|
|
2278
|
+
rankingJsonPath,
|
|
2279
|
+
`${JSON.stringify(
|
|
2280
|
+
{ experimentId: options.experimentId, totalCostUsd, unrankedRecoveryCostUsd, rows },
|
|
2281
|
+
null,
|
|
2282
|
+
2
|
|
2283
|
+
)}
|
|
2284
|
+
`
|
|
2285
|
+
);
|
|
2286
|
+
storage.write(
|
|
2287
|
+
rankingMarkdownPath,
|
|
2288
|
+
renderAgentMemoryExperimentRanking(rows, totalCostUsd, unrankedRecoveryCostUsd)
|
|
2289
|
+
);
|
|
2290
|
+
return {
|
|
2291
|
+
campaign,
|
|
2292
|
+
rows,
|
|
2293
|
+
totalCostUsd,
|
|
2294
|
+
unrankedRecoveryCostUsd,
|
|
2295
|
+
leaderCandidateId: rows.find((row) => row.cellsFailed === 0)?.candidateId,
|
|
2296
|
+
rankingJsonPath,
|
|
2297
|
+
rankingMarkdownPath,
|
|
2298
|
+
attemptLogPath,
|
|
2299
|
+
recoveryLogPath
|
|
2300
|
+
};
|
|
2301
|
+
}
|
|
2302
|
+
function memoryExperimentDispatchRef(options) {
|
|
2303
|
+
return stableId(
|
|
2304
|
+
"memory_experiment",
|
|
2305
|
+
canonicalJson4({
|
|
2306
|
+
implementationRef: MEMORY_EXPERIMENT_IMPLEMENTATION_REF,
|
|
2307
|
+
experimentId: options.experimentId,
|
|
2308
|
+
experimentRunId: options.experimentRunId ?? null,
|
|
2309
|
+
executeStepRef: options.executeStepRef ?? "fixtures",
|
|
2310
|
+
cleanupBranches: options.cleanupBranches ?? true,
|
|
2311
|
+
candidates: options.candidates.map((candidate) => ({
|
|
2312
|
+
id: candidate.id,
|
|
2313
|
+
ref: candidate.ref,
|
|
2314
|
+
policy: candidate.policy ?? null,
|
|
2315
|
+
baseScope: candidate.baseScope ?? null,
|
|
2316
|
+
externalCostUsdPerSequence: candidate.externalCostUsdPerSequence ?? 0,
|
|
2317
|
+
externalRecoveryCostUsdPerAttempt: candidate.externalRecoveryCostUsdPerAttempt ?? 0
|
|
2318
|
+
})).sort((a, b) => a.id.localeCompare(b.id))
|
|
2319
|
+
})
|
|
2320
|
+
);
|
|
2321
|
+
}
|
|
2322
|
+
|
|
2323
|
+
// src/memory/graphiti.ts
|
|
2324
|
+
import { randomUUID as randomUUID3 } from "crypto";
|
|
2325
|
+
import { canonicalJson as canonicalJson5 } from "@tangle-network/agent-eval";
|
|
2326
|
+
var DEFAULT_GRAPHITI_INGESTION_TIMEOUT_MS = 12e4;
|
|
2327
|
+
function createGraphitiMemoryAdapter(options) {
|
|
2328
|
+
assertGraphitiOptions(options);
|
|
2329
|
+
const id = options.id ?? "graphiti";
|
|
2330
|
+
const consistency = options.consistency ?? "visible";
|
|
2331
|
+
const tools = graphitiToolNames(options);
|
|
2332
|
+
const queued = /* @__PURE__ */ new Map();
|
|
2333
|
+
const adapter = {
|
|
2334
|
+
id,
|
|
2335
|
+
branchIsolation: consistency === "visible" ? {
|
|
2336
|
+
mode: "scoped",
|
|
2337
|
+
processExitSafe: false,
|
|
2338
|
+
recoveryDelayMs: options.ingestionTimeoutMs ?? DEFAULT_GRAPHITI_INGESTION_TIMEOUT_MS
|
|
2339
|
+
} : {
|
|
2340
|
+
mode: "unsupported",
|
|
2341
|
+
reason: 'queued Graphiti writes can become visible after a process restart; use consistency="visible" for branch experiments'
|
|
2342
|
+
},
|
|
2343
|
+
async search(query, searchOptions = {}) {
|
|
2344
|
+
if (searchOptions.minScore !== void 0 && !Number.isFinite(searchOptions.minScore)) {
|
|
2345
|
+
throw new Error(`${id}: minScore must be finite`);
|
|
2346
|
+
}
|
|
2347
|
+
const scope = mergeScopes3(options.defaultScope, searchOptions.scope);
|
|
2348
|
+
const groupId = graphitiGroupId(id, scope);
|
|
2349
|
+
const modes = new Set(options.search ?? ["facts", "nodes"]);
|
|
2350
|
+
const kinds = searchOptions.kinds && searchOptions.kinds.length > 0 ? new Set(searchOptions.kinds) : null;
|
|
2351
|
+
const searchFacts = modes.has("facts") && (!kinds || kinds.has("fact"));
|
|
2352
|
+
const searchNodes = modes.has("nodes") && (!kinds || kinds.has("entity"));
|
|
2353
|
+
const limit = searchOptions.limit ?? 10;
|
|
2354
|
+
if (!Number.isSafeInteger(limit) || limit < 0) {
|
|
2355
|
+
throw new Error(`${id}: search limit must be a non-negative safe integer`);
|
|
2356
|
+
}
|
|
2357
|
+
if (limit === 0) return [];
|
|
2358
|
+
const [factPayload, nodePayload] = await Promise.all([
|
|
2359
|
+
searchFacts ? callGraphiti(options.client, tools.searchFacts, {
|
|
2360
|
+
query,
|
|
2361
|
+
group_ids: [groupId],
|
|
2362
|
+
max_facts: limit
|
|
2363
|
+
}) : void 0,
|
|
2364
|
+
searchNodes ? callGraphiti(options.client, tools.searchNodes, {
|
|
2365
|
+
query,
|
|
2366
|
+
group_ids: [groupId],
|
|
2367
|
+
max_nodes: limit,
|
|
2368
|
+
...searchOptions.metadata?.entityTypes ? { entity_types: searchOptions.metadata.entityTypes } : {}
|
|
2369
|
+
}) : void 0
|
|
2370
|
+
]);
|
|
2371
|
+
const hits = mergeRankedMemoryHits(
|
|
2372
|
+
[normalizeGraphitiFacts(factPayload, id), normalizeGraphitiNodes(nodePayload, id)].map(
|
|
2373
|
+
(group) => group.filter((hit) => {
|
|
2374
|
+
if (searchOptions.minScore === void 0) return true;
|
|
2375
|
+
const score = hit.normalizedScore ?? hit.score;
|
|
2376
|
+
return score !== void 0 && score >= searchOptions.minScore;
|
|
2377
|
+
})
|
|
2378
|
+
),
|
|
2379
|
+
limit
|
|
2380
|
+
);
|
|
2381
|
+
await enrichGraphitiProvenance(options, groupId, hits);
|
|
2382
|
+
return hits;
|
|
2383
|
+
},
|
|
2384
|
+
async getContext(query, searchOptions = {}) {
|
|
2385
|
+
return defaultGetMemoryContext(adapter, query, searchOptions);
|
|
2386
|
+
},
|
|
2387
|
+
async write(input) {
|
|
2388
|
+
const scope = mergeScopes3(options.defaultScope, input.scope);
|
|
2389
|
+
const groupId = graphitiGroupId(id, scope);
|
|
2390
|
+
const eventKey = canonicalJson5(
|
|
2391
|
+
stripUndefined2({
|
|
2392
|
+
id: input.id,
|
|
2393
|
+
kind: input.kind,
|
|
2394
|
+
text: input.text,
|
|
2395
|
+
title: input.title,
|
|
2396
|
+
role: input.role,
|
|
2397
|
+
metadata: input.metadata
|
|
2398
|
+
})
|
|
2399
|
+
);
|
|
2400
|
+
const episodeUuid = input.id === void 0 ? randomUUID3() : deterministicUuid(`${id}:${groupId}:${eventKey}`);
|
|
2401
|
+
const name = input.title ?? `${input.kind}:${input.id ?? stableId("event", eventKey)}`;
|
|
2402
|
+
const sourceDescription = canonicalJson5(
|
|
2403
|
+
stripUndefined2({
|
|
2404
|
+
adapter: "agent-knowledge",
|
|
2405
|
+
memoryKind: input.kind,
|
|
2387
2406
|
eventId: input.id,
|
|
2388
2407
|
actorId: input.metadata?.actorId,
|
|
2389
2408
|
sessionId: scope.sessionId,
|
|
@@ -2416,797 +2435,329 @@ function createGraphitiMemoryAdapter(options) {
|
|
|
2416
2435
|
groupId,
|
|
2417
2436
|
consistency,
|
|
2418
2437
|
queued: consistency === "queued"
|
|
2419
|
-
}
|
|
2420
|
-
};
|
|
2421
|
-
return {
|
|
2422
|
-
...result,
|
|
2423
|
-
sourceRecord: memoryWriteResultToSourceRecord(result, input.text, { scope })
|
|
2424
|
-
};
|
|
2425
|
-
},
|
|
2426
|
-
async clear(scope) {
|
|
2427
|
-
const groupId = graphitiGroupId(id, mergeScopes3(options.defaultScope, scope));
|
|
2428
|
-
for (const [episodeUuid, pending] of queued) {
|
|
2429
|
-
if (pending.groupId !== groupId) continue;
|
|
2430
|
-
await waitForGraphitiEpisode(options, groupId, episodeUuid);
|
|
2431
|
-
queued.delete(episodeUuid);
|
|
2432
|
-
}
|
|
2433
|
-
await callGraphiti(options.client, tools.clearGraph, { group_ids: [groupId] });
|
|
2434
|
-
},
|
|
2435
|
-
async flush() {
|
|
2436
|
-
for (const [episodeUuid, pending] of queued) {
|
|
2437
|
-
await waitForGraphitiEpisode(options, pending.groupId, episodeUuid);
|
|
2438
|
-
queued.delete(episodeUuid);
|
|
2439
|
-
}
|
|
2440
|
-
}
|
|
2441
|
-
};
|
|
2442
|
-
return adapter;
|
|
2443
|
-
}
|
|
2444
|
-
async function callGraphiti(client, name, args) {
|
|
2445
|
-
const raw = await client.callTool({ name, arguments: args });
|
|
2446
|
-
const payload = graphitiPayload(raw);
|
|
2447
|
-
if (isRecord(payload) && typeof payload.error === "string") {
|
|
2448
|
-
throw new Error(`Graphiti ${name}: ${payload.error}`);
|
|
2449
|
-
}
|
|
2450
|
-
return payload;
|
|
2451
|
-
}
|
|
2452
|
-
function graphitiPayload(raw) {
|
|
2453
|
-
if (!isRecord(raw)) return raw;
|
|
2454
|
-
if (raw.isError === true) throw new Error(`Graphiti MCP: ${toolText(raw) || "tool call failed"}`);
|
|
2455
|
-
if (isRecord(raw.structuredContent)) return unwrapResult(raw.structuredContent);
|
|
2456
|
-
const text = toolText(raw);
|
|
2457
|
-
if (text) {
|
|
2458
|
-
try {
|
|
2459
|
-
return unwrapResult(JSON.parse(text));
|
|
2460
|
-
} catch {
|
|
2461
|
-
return { message: text };
|
|
2462
|
-
}
|
|
2463
|
-
}
|
|
2464
|
-
return unwrapResult(raw);
|
|
2465
|
-
}
|
|
2466
|
-
function unwrapResult(value) {
|
|
2467
|
-
if (isRecord(value) && Object.keys(value).length === 1 && "result" in value) {
|
|
2468
|
-
return value.result;
|
|
2469
|
-
}
|
|
2470
|
-
return value;
|
|
2471
|
-
}
|
|
2472
|
-
function toolText(raw) {
|
|
2473
|
-
if (!Array.isArray(raw.content)) return "";
|
|
2474
|
-
return raw.content.filter(isRecord).map((part) => typeof part.text === "string" ? part.text : "").filter(Boolean).join("\n");
|
|
2475
|
-
}
|
|
2476
|
-
function normalizeGraphitiFacts(raw, adapterId) {
|
|
2477
|
-
const facts = isRecord(raw) && Array.isArray(raw.facts) ? raw.facts.filter(isRecord) : [];
|
|
2478
|
-
return facts.flatMap((fact) => {
|
|
2479
|
-
const text = stringField(fact, ["fact", "text"]);
|
|
2480
|
-
if (!text) return [];
|
|
2481
|
-
const factId = stringField(fact, ["uuid", "id"]) ?? stableId("fact", canonicalJson4(stripUndefined2({ adapterId, text, fact })));
|
|
2482
|
-
const episodeUuids = stringArray2(fact.episodes ?? fact.episode_uuids);
|
|
2483
|
-
const score = numberField(fact, ["score", "relevance_score"]);
|
|
2484
|
-
const normalizedScore = normalizedScoreOf(score);
|
|
2485
|
-
return [
|
|
2486
|
-
{
|
|
2487
|
-
id: factId,
|
|
2488
|
-
uri: `memory://${adapterId}/fact/${factId}`,
|
|
2489
|
-
kind: "fact",
|
|
2490
|
-
text,
|
|
2491
|
-
...stringField(fact, ["name"]) ? { title: stringField(fact, ["name"]) } : {},
|
|
2492
|
-
...stringField(fact, ["created_at"]) ? { createdAt: stringField(fact, ["created_at"]) } : {},
|
|
2493
|
-
...stringField(fact, ["invalid_at"]) ? { validUntil: stringField(fact, ["invalid_at"]) } : stringField(fact, ["expired_at"]) ? { validUntil: stringField(fact, ["expired_at"]) } : {},
|
|
2494
|
-
...score !== void 0 ? { score } : {},
|
|
2495
|
-
...normalizedScore !== void 0 ? { normalizedScore } : {},
|
|
2496
|
-
metadata: {
|
|
2497
|
-
provider: "graphiti",
|
|
2498
|
-
groupId: fact.group_id,
|
|
2499
|
-
sourceNodeUuid: fact.source_node_uuid,
|
|
2500
|
-
targetNodeUuid: fact.target_node_uuid,
|
|
2501
|
-
validAt: fact.valid_at,
|
|
2502
|
-
invalidAt: fact.invalid_at,
|
|
2503
|
-
episodeUuids,
|
|
2504
|
-
attributes: fact.attributes
|
|
2505
|
-
}
|
|
2506
|
-
}
|
|
2507
|
-
];
|
|
2508
|
-
});
|
|
2509
|
-
}
|
|
2510
|
-
function normalizeGraphitiNodes(raw, adapterId) {
|
|
2511
|
-
const nodes = isRecord(raw) && Array.isArray(raw.nodes) ? raw.nodes.filter(isRecord) : [];
|
|
2512
|
-
return nodes.flatMap((node) => {
|
|
2513
|
-
const name = stringField(node, ["name"]);
|
|
2514
|
-
const summary = stringField(node, ["summary"]);
|
|
2515
|
-
const text = summary ?? name;
|
|
2516
|
-
if (!text) return [];
|
|
2517
|
-
const nodeId = stringField(node, ["uuid", "id"]) ?? stableId("node", canonicalJson4(stripUndefined2({ adapterId, text, node })));
|
|
2518
|
-
const score = numberField(node, ["score", "relevance_score"]);
|
|
2519
|
-
const normalizedScore = normalizedScoreOf(score);
|
|
2520
|
-
return [
|
|
2521
|
-
{
|
|
2522
|
-
id: nodeId,
|
|
2523
|
-
uri: `memory://${adapterId}/entity/${nodeId}`,
|
|
2524
|
-
kind: "entity",
|
|
2525
|
-
text,
|
|
2526
|
-
...name ? { title: name } : {},
|
|
2527
|
-
...stringField(node, ["created_at"]) ? { createdAt: stringField(node, ["created_at"]) } : {},
|
|
2528
|
-
...score !== void 0 ? { score } : {},
|
|
2529
|
-
...normalizedScore !== void 0 ? { normalizedScore } : {},
|
|
2530
|
-
metadata: {
|
|
2531
|
-
provider: "graphiti",
|
|
2532
|
-
groupId: node.group_id,
|
|
2533
|
-
labels: node.labels,
|
|
2534
|
-
attributes: node.attributes
|
|
2535
|
-
}
|
|
2536
|
-
}
|
|
2537
|
-
];
|
|
2538
|
-
});
|
|
2539
|
-
}
|
|
2540
|
-
async function enrichGraphitiProvenance(options, groupId, hits) {
|
|
2541
|
-
const wanted = new Set(hits.flatMap((hit) => stringArray2(hit.metadata?.episodeUuids)));
|
|
2542
|
-
if (wanted.size === 0) return;
|
|
2543
|
-
const scan = await scanGraphitiEpisodes(
|
|
2544
|
-
options,
|
|
2545
|
-
groupId,
|
|
2546
|
-
wanted,
|
|
2547
|
-
options.provenanceEpisodeLimit ?? options.episodeScanLimit ?? 1e4
|
|
2548
|
-
);
|
|
2549
|
-
const episodes = scan.episodes;
|
|
2550
|
-
const byId = new Map(episodes.map((episode) => [episode.uuid, episode]));
|
|
2551
|
-
for (const hit of hits) {
|
|
2552
|
-
const episodeUuids = stringArray2(hit.metadata?.episodeUuids);
|
|
2553
|
-
const sourceEpisodes = episodeUuids.map((uuid) => byId.get(uuid)).filter((episode) => episode !== void 0);
|
|
2554
|
-
const provenanceIncomplete = episodeUuids.some((uuid) => !byId.has(uuid));
|
|
2555
|
-
if (sourceEpisodes.length === 0) {
|
|
2556
|
-
if (provenanceIncomplete) {
|
|
2557
|
-
hit.metadata = { ...hit.metadata, provenanceIncomplete: true };
|
|
2558
|
-
}
|
|
2559
|
-
continue;
|
|
2560
|
-
}
|
|
2561
|
-
const descriptions = sourceEpisodes.map((episode) => parseDescription(episode.source_description)).filter(isRecord);
|
|
2562
|
-
const first = descriptions[0];
|
|
2563
|
-
hit.metadata = {
|
|
2564
|
-
...hit.metadata,
|
|
2565
|
-
sourceEpisodes,
|
|
2566
|
-
...provenanceIncomplete || !scan.complete ? { provenanceIncomplete: true } : {},
|
|
2567
|
-
...first ?? {},
|
|
2568
|
-
eventIds: uniqueStrings2(descriptions.map((value) => value.eventId)),
|
|
2569
|
-
actorIds: uniqueStrings2(descriptions.map((value) => value.actorId)),
|
|
2570
|
-
...typeof first?.eventId === "string" ? { eventId: first.eventId } : {},
|
|
2571
|
-
...typeof first?.actorId === "string" ? { actorId: first.actorId } : {}
|
|
2572
|
-
};
|
|
2573
|
-
}
|
|
2574
|
-
}
|
|
2575
|
-
async function getGraphitiEpisodes(options, groupId, limit) {
|
|
2576
|
-
const raw = await callGraphiti(options.client, graphitiToolNames(options).getEpisodes, {
|
|
2577
|
-
group_ids: [groupId],
|
|
2578
|
-
max_episodes: limit
|
|
2579
|
-
});
|
|
2580
|
-
if (!isRecord(raw) || !Array.isArray(raw.episodes)) return [];
|
|
2581
|
-
return raw.episodes.filter(isRecord).flatMap((episode) => {
|
|
2582
|
-
const uuid = stringField(episode, ["uuid", "id"]);
|
|
2583
|
-
return uuid ? [{ ...episode, uuid }] : [];
|
|
2584
|
-
});
|
|
2585
|
-
}
|
|
2586
|
-
async function scanGraphitiEpisodes(options, groupId, wanted, maximum) {
|
|
2587
|
-
let limit = Math.min(100, maximum);
|
|
2588
|
-
for (; ; ) {
|
|
2589
|
-
const episodes = await getGraphitiEpisodes(options, groupId, limit);
|
|
2590
|
-
const foundAll = [...wanted].every((uuid) => episodes.some((episode) => episode.uuid === uuid));
|
|
2591
|
-
if (foundAll || episodes.length < limit) return { episodes, complete: foundAll };
|
|
2592
|
-
if (limit === maximum) return { episodes, complete: false };
|
|
2593
|
-
limit = Math.min(maximum, limit * 2);
|
|
2594
|
-
}
|
|
2595
|
-
}
|
|
2596
|
-
async function waitForGraphitiEpisode(options, groupId, episodeUuid) {
|
|
2597
|
-
const timeoutMs = options.ingestionTimeoutMs ?? DEFAULT_GRAPHITI_INGESTION_TIMEOUT_MS;
|
|
2598
|
-
const pollIntervalMs = options.pollIntervalMs ?? 500;
|
|
2599
|
-
const maximum = options.episodeScanLimit ?? 1e4;
|
|
2600
|
-
const deadline = Date.now() + timeoutMs;
|
|
2601
|
-
let scanLimitReached = false;
|
|
2602
|
-
for (; ; ) {
|
|
2603
|
-
const scan = await scanGraphitiEpisodes(options, groupId, /* @__PURE__ */ new Set([episodeUuid]), maximum);
|
|
2604
|
-
if (scan.complete) return;
|
|
2605
|
-
scanLimitReached ||= scan.episodes.length >= maximum;
|
|
2606
|
-
const remainingMs = deadline - Date.now();
|
|
2607
|
-
if (remainingMs <= 0) break;
|
|
2608
|
-
await new Promise((resolve) => setTimeout(resolve, Math.min(pollIntervalMs, remainingMs)));
|
|
2609
|
-
}
|
|
2610
|
-
if (scanLimitReached) {
|
|
2611
|
-
throw new Error(
|
|
2612
|
-
`Graphiti episode ${episodeUuid} is outside episodeScanLimit=${maximum}; raise the limit or use a smaller group`
|
|
2613
|
-
);
|
|
2614
|
-
}
|
|
2615
|
-
throw new Error(`Graphiti episode ${episodeUuid} was not visible after ${timeoutMs}ms`);
|
|
2616
|
-
}
|
|
2617
|
-
function graphitiGroupId(adapterId, scope) {
|
|
2618
|
-
return `ak_${sha256(
|
|
2619
|
-
canonicalJson4(
|
|
2620
|
-
stripUndefined2({
|
|
2621
|
-
adapterId,
|
|
2622
|
-
tenantId: scope.tenantId,
|
|
2623
|
-
userId: scope.userId,
|
|
2624
|
-
agentId: scope.agentId,
|
|
2625
|
-
teamId: scope.teamId,
|
|
2626
|
-
runId: scope.runId,
|
|
2627
|
-
sessionId: scope.sessionId,
|
|
2628
|
-
namespace: scope.namespace,
|
|
2629
|
-
tags: scope.tags
|
|
2630
|
-
})
|
|
2631
|
-
)
|
|
2632
|
-
).slice(0, 32)}`;
|
|
2633
|
-
}
|
|
2634
|
-
function graphitiToolNames(options) {
|
|
2635
|
-
return {
|
|
2636
|
-
addMemory: options.toolNames?.addMemory ?? "add_memory",
|
|
2637
|
-
searchFacts: options.toolNames?.searchFacts ?? "search_memory_facts",
|
|
2638
|
-
searchNodes: options.toolNames?.searchNodes ?? "search_nodes",
|
|
2639
|
-
getEpisodes: options.toolNames?.getEpisodes ?? "get_episodes",
|
|
2640
|
-
clearGraph: options.toolNames?.clearGraph ?? "clear_graph"
|
|
2641
|
-
};
|
|
2642
|
-
}
|
|
2643
|
-
function assertGraphitiOptions(options) {
|
|
2644
|
-
if (options.id !== void 0 && (typeof options.id !== "string" || !options.id.trim())) {
|
|
2645
|
-
throw new Error("Graphiti id must be a non-empty string");
|
|
2646
|
-
}
|
|
2647
|
-
if (options.backendRef !== void 0 && (typeof options.backendRef !== "string" || !options.backendRef.trim())) {
|
|
2648
|
-
throw new Error("Graphiti backendRef must be a non-empty string");
|
|
2649
|
-
}
|
|
2650
|
-
if (options.consistency !== void 0 && !["queued", "visible"].includes(options.consistency)) {
|
|
2651
|
-
throw new Error("Graphiti consistency must be queued or visible");
|
|
2652
|
-
}
|
|
2653
|
-
for (const [name, value] of [
|
|
2654
|
-
["ingestionTimeoutMs", options.ingestionTimeoutMs],
|
|
2655
|
-
["pollIntervalMs", options.pollIntervalMs],
|
|
2656
|
-
["episodeScanLimit", options.episodeScanLimit],
|
|
2657
|
-
["provenanceEpisodeLimit", options.provenanceEpisodeLimit]
|
|
2658
|
-
]) {
|
|
2659
|
-
if (value !== void 0 && (!Number.isSafeInteger(value) || value <= 0)) {
|
|
2660
|
-
throw new Error(`Graphiti ${name} must be a positive safe integer`);
|
|
2661
|
-
}
|
|
2662
|
-
}
|
|
2663
|
-
for (const [name, value] of Object.entries(options.toolNames ?? {})) {
|
|
2664
|
-
if (typeof value !== "string" || value.trim().length === 0) {
|
|
2665
|
-
throw new Error(`Graphiti toolNames.${name} must be a non-empty string`);
|
|
2666
|
-
}
|
|
2667
|
-
}
|
|
2668
|
-
}
|
|
2669
|
-
function deterministicUuid(value) {
|
|
2670
|
-
const hex = sha256(value);
|
|
2671
|
-
return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-5${hex.slice(13, 16)}-a${hex.slice(17, 20)}-${hex.slice(20, 32)}`;
|
|
2672
|
-
}
|
|
2673
|
-
function parseDescription(value) {
|
|
2674
|
-
if (!value) return void 0;
|
|
2675
|
-
try {
|
|
2676
|
-
return JSON.parse(value);
|
|
2677
|
-
} catch {
|
|
2678
|
-
return void 0;
|
|
2679
|
-
}
|
|
2680
|
-
}
|
|
2681
|
-
function mergeScopes3(base, extra) {
|
|
2682
|
-
return {
|
|
2683
|
-
...base ?? {},
|
|
2684
|
-
...extra ?? {},
|
|
2685
|
-
tags: { ...base?.tags ?? {}, ...extra?.tags ?? {} }
|
|
2686
|
-
};
|
|
2687
|
-
}
|
|
2688
|
-
function uniqueStrings2(values) {
|
|
2689
|
-
return [...new Set(values.filter((value) => typeof value === "string"))];
|
|
2690
|
-
}
|
|
2691
|
-
function stringArray2(value) {
|
|
2692
|
-
return Array.isArray(value) ? value.filter((entry) => typeof entry === "string") : [];
|
|
2693
|
-
}
|
|
2694
|
-
function stringField(value, keys) {
|
|
2695
|
-
for (const key of keys) {
|
|
2696
|
-
const field = value[key];
|
|
2697
|
-
if (typeof field === "string" && field.length > 0) return field;
|
|
2698
|
-
}
|
|
2699
|
-
return void 0;
|
|
2700
|
-
}
|
|
2701
|
-
function numberField(value, keys) {
|
|
2702
|
-
for (const key of keys) {
|
|
2703
|
-
const field = value[key];
|
|
2704
|
-
if (typeof field === "number" && Number.isFinite(field)) return field;
|
|
2705
|
-
}
|
|
2706
|
-
return void 0;
|
|
2707
|
-
}
|
|
2708
|
-
function normalizedScoreOf(score) {
|
|
2709
|
-
return score !== void 0 && score >= 0 && score <= 1 ? score : void 0;
|
|
2710
|
-
}
|
|
2711
|
-
function isRecord(value) {
|
|
2712
|
-
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
2713
|
-
}
|
|
2714
|
-
function stripUndefined2(value) {
|
|
2715
|
-
if (Array.isArray(value)) return value.map(stripUndefined2);
|
|
2716
|
-
if (!isRecord(value)) return value;
|
|
2717
|
-
return Object.fromEntries(
|
|
2718
|
-
Object.entries(value).filter(([, child]) => child !== void 0).map(([key, child]) => [key, stripUndefined2(child)])
|
|
2719
|
-
);
|
|
2720
|
-
}
|
|
2721
|
-
function graphitiMemoryAdapterIdentity(options) {
|
|
2722
|
-
if (typeof options.backendRef !== "string" || !options.backendRef.trim()) {
|
|
2723
|
-
throw new Error("Graphiti backendRef must be a non-empty string");
|
|
2724
|
-
}
|
|
2725
|
-
return stableId("graphiti", canonicalJson4(stripUndefined2(options)));
|
|
2726
|
-
}
|
|
2727
|
-
|
|
2728
|
-
// src/memory/improvement.ts
|
|
2729
|
-
import { join as join2 } from "path";
|
|
2730
|
-
import { canonicalJson as canonicalJson5 } from "@tangle-network/agent-eval";
|
|
2731
|
-
import {
|
|
2732
|
-
campaignLineageStore,
|
|
2733
|
-
createRunCostLedger as createRunCostLedger2,
|
|
2734
|
-
fsCampaignStorage as fsCampaignStorage2,
|
|
2735
|
-
heldoutSignificance,
|
|
2736
|
-
memLineageStore,
|
|
2737
|
-
resolveRunDir as resolveRunDir2,
|
|
2738
|
-
runLineageLoop,
|
|
2739
|
-
surfaceHash
|
|
2740
|
-
} from "@tangle-network/agent-eval/campaign";
|
|
2741
|
-
var DEFAULT_CRITICAL_DIMENSIONS = [
|
|
2742
|
-
"memory_stale_safe",
|
|
2743
|
-
"memory_actor_recall",
|
|
2744
|
-
"memory_event_recall"
|
|
2745
|
-
];
|
|
2746
|
-
var MEMORY_IMPROVEMENT_IMPLEMENTATION_REF = "agent-knowledge:memory-improvement:v2";
|
|
2747
|
-
async function runAgentMemoryImprovement(options) {
|
|
2748
|
-
if ("onPromote" in options) {
|
|
2749
|
-
throw new Error(
|
|
2750
|
-
"memory improvement onPromote was removed; use activation.readCurrent and activation.compareAndSet"
|
|
2751
|
-
);
|
|
2752
|
-
}
|
|
2753
|
-
if (options.seeds.length === 0) throw new Error("memory improvement requires seed configs");
|
|
2754
|
-
if (options.trainSequences.length === 0) {
|
|
2755
|
-
throw new Error("memory improvement requires training sequences");
|
|
2756
|
-
}
|
|
2757
|
-
if (options.holdoutSequences.length === 0) {
|
|
2758
|
-
throw new Error("memory improvement requires holdout sequences");
|
|
2759
|
-
}
|
|
2760
|
-
const trainIds = new Set(options.trainSequences.map((sequence) => sequence.id));
|
|
2761
|
-
const overlap = options.holdoutSequences.map((sequence) => sequence.id).filter((id) => trainIds.has(id));
|
|
2762
|
-
if (overlap.length > 0) {
|
|
2763
|
-
throw new Error(`memory improvement train/holdout overlap: ${overlap.join(", ")}`);
|
|
2764
|
-
}
|
|
2765
|
-
const trainFingerprints = new Map(
|
|
2766
|
-
options.trainSequences.map((sequence) => [memorySequenceFingerprint(sequence), sequence.id])
|
|
2767
|
-
);
|
|
2768
|
-
const duplicateHistories = options.holdoutSequences.flatMap((sequence) => {
|
|
2769
|
-
const trainId = trainFingerprints.get(memorySequenceFingerprint(sequence));
|
|
2770
|
-
return trainId ? [`${trainId}/${sequence.id}`] : [];
|
|
2771
|
-
});
|
|
2772
|
-
if (duplicateHistories.length > 0) {
|
|
2773
|
-
throw new Error(
|
|
2774
|
-
`memory improvement train/holdout histories duplicate content: ${duplicateHistories.join(", ")}`
|
|
2775
|
-
);
|
|
2776
|
-
}
|
|
2777
|
-
if (typeof options.improvementRef !== "string" || !options.improvementRef.trim()) {
|
|
2778
|
-
throw new Error("memory improvement improvementRef must be a non-empty string");
|
|
2779
|
-
}
|
|
2780
|
-
assertMemoryImprovementOptions(options);
|
|
2781
|
-
const storage = options.storage ?? fsCampaignStorage2();
|
|
2782
|
-
const runDir = resolveRunDir2(options.runDir, options.repo);
|
|
2783
|
-
storage.ensureDir(runDir);
|
|
2784
|
-
const lease = await acquireAgentMemoryRunLease({
|
|
2785
|
-
experimentId: options.experimentId,
|
|
2786
|
-
runDir,
|
|
2787
|
-
storage,
|
|
2788
|
-
customStorage: options.storage !== void 0,
|
|
2789
|
-
lockFileName: "memory-improvement.lock",
|
|
2790
|
-
label: "memory improvement",
|
|
2791
|
-
controllerMode: options.controllerMode,
|
|
2792
|
-
acquireRunLease: options.acquireRunLease
|
|
2793
|
-
});
|
|
2794
|
-
let result;
|
|
2795
|
-
let primaryError;
|
|
2796
|
-
try {
|
|
2797
|
-
result = await runAgentMemoryImprovementOwned(options, storage, runDir, lease);
|
|
2798
|
-
} catch (error) {
|
|
2799
|
-
primaryError = error;
|
|
2800
|
-
}
|
|
2801
|
-
let releaseError;
|
|
2802
|
-
try {
|
|
2803
|
-
await lease.release();
|
|
2804
|
-
} catch (error) {
|
|
2805
|
-
releaseError = error;
|
|
2806
|
-
}
|
|
2807
|
-
if (primaryError && releaseError) {
|
|
2808
|
-
throw new AggregateError(
|
|
2809
|
-
[primaryError, releaseError],
|
|
2810
|
-
"memory improvement failed and its controller lease could not be released"
|
|
2811
|
-
);
|
|
2812
|
-
}
|
|
2813
|
-
if (primaryError) throw primaryError;
|
|
2814
|
-
if (releaseError) throw releaseError;
|
|
2815
|
-
if (!result) throw new Error("memory improvement produced no result");
|
|
2816
|
-
return result;
|
|
2817
|
-
}
|
|
2818
|
-
async function runAgentMemoryImprovementOwned(options, storage, runDir, lease) {
|
|
2819
|
-
await lease.assertOwned();
|
|
2820
|
-
const serializeRaw = options.serializeConfig ?? ((config) => canonicalJson5(config));
|
|
2821
|
-
const parseRaw = options.parseConfig ?? ((surface) => JSON.parse(surface));
|
|
2822
|
-
const serialize = (config) => serializeMemoryConfig(serializeRaw, config);
|
|
2823
|
-
const parse = (surface) => parseMemoryConfig(parseRaw, surface);
|
|
2824
|
-
for (const seed of options.seeds) assertMemoryConfigRoundTrip(seed.config, serialize, parse);
|
|
2825
|
-
if (options.activation && !storage.append) {
|
|
2826
|
-
throw new Error("memory activation requires CampaignStorage.append");
|
|
2827
|
-
}
|
|
2828
|
-
if (options.resumable !== false && !options.lineageStore && !storage.append) {
|
|
2829
|
-
throw new Error("resumable memory improvement requires CampaignStorage.append");
|
|
2830
|
-
}
|
|
2831
|
-
assertMemoryImprovementIdentity(options, storage, runDir, serialize);
|
|
2832
|
-
const costLedger = createRunCostLedger2({
|
|
2833
|
-
storage,
|
|
2834
|
-
runDir,
|
|
2835
|
-
costCeilingUsd: options.maxTotalCostUsd ?? 0
|
|
2836
|
-
});
|
|
2837
|
-
reconcileInterruptedRunPaidCalls(costLedger, "memory improvement run");
|
|
2838
|
-
assertNoInterruptedPaidCalls(costLedger, "memory improvement recovery");
|
|
2839
|
-
const trainScenarios = options.trainSequences.map((sequence) => ({
|
|
2840
|
-
id: sequence.id,
|
|
2841
|
-
kind: "agent-memory-config-search",
|
|
2842
|
-
sequenceId: sequence.id
|
|
2843
|
-
}));
|
|
2844
|
-
const evaluations = /* @__PURE__ */ new Map();
|
|
2845
|
-
const evaluateSurface = async (surface) => {
|
|
2846
|
-
await lease.assertOwned();
|
|
2847
|
-
const text = requireStringSurface(surface);
|
|
2848
|
-
const config = parse(text);
|
|
2849
|
-
const canonicalSurface = serialize(config);
|
|
2850
|
-
const hash = surfaceHash(canonicalSurface);
|
|
2851
|
-
let pending = evaluations.get(hash);
|
|
2852
|
-
if (!pending) {
|
|
2853
|
-
pending = (async () => {
|
|
2854
|
-
const candidate = await buildCandidate(options, config, hash, `search-${hash}`);
|
|
2855
|
-
await lease.assertOwned();
|
|
2856
|
-
const experiment = await runAgentMemoryExperiment({
|
|
2857
|
-
...experimentOptions(options, costLedger, storage, lease),
|
|
2858
|
-
experimentId: `${options.experimentId}:search:${hash}`,
|
|
2859
|
-
sequences: options.trainSequences,
|
|
2860
|
-
candidates: [candidate],
|
|
2861
|
-
runDir: join2(runDir, "search", hash),
|
|
2862
|
-
costPhase: `memory.search.${hash}`
|
|
2863
|
-
});
|
|
2864
|
-
await lease.assertOwned();
|
|
2865
|
-
return experiment;
|
|
2866
|
-
})();
|
|
2867
|
-
evaluations.set(hash, pending);
|
|
2868
|
-
void pending.catch(() => evaluations.delete(hash));
|
|
2869
|
-
}
|
|
2870
|
-
const result2 = await pending;
|
|
2871
|
-
await lease.assertOwned();
|
|
2872
|
-
const row = result2.rows[0];
|
|
2873
|
-
if (!row || row.cellsFailed > 0) {
|
|
2874
|
-
throw new Error(`${hash}: memory candidate did not complete every training cell`);
|
|
2875
|
-
}
|
|
2876
|
-
return {
|
|
2877
|
-
score: row.scoreMean,
|
|
2878
|
-
scoreVector: sequenceScores(result2, options.trainSequences, row.candidateId)
|
|
2879
|
-
};
|
|
2880
|
-
};
|
|
2881
|
-
const lineageStore = options.lineageStore ?? (options.resumable === false ? memLineageStore() : campaignLineageStore(storage, join2(runDir, "lineage.jsonl")));
|
|
2882
|
-
const lineageOptions = {
|
|
2883
|
-
seeds: options.seeds.map((seed) => ({
|
|
2884
|
-
surface: serialize(seed.config),
|
|
2885
|
-
track: seed.track,
|
|
2886
|
-
proposer: seed.proposer,
|
|
2887
|
-
...seed.vision !== void 0 ? { vision: seed.vision } : {}
|
|
2888
|
-
})),
|
|
2889
|
-
scenarios: trainScenarios,
|
|
2890
|
-
proposer: withCostContext(options.proposer, costLedger, lease, "default"),
|
|
2891
|
-
proposers: options.proposers ? Object.fromEntries(
|
|
2892
|
-
Object.entries(options.proposers).map(([name, proposer]) => [
|
|
2893
|
-
name,
|
|
2894
|
-
withCostContext(proposer, costLedger, lease, name)
|
|
2895
|
-
])
|
|
2896
|
-
) : void 0,
|
|
2897
|
-
scoreSurface: evaluateSurface,
|
|
2898
|
-
governor: options.governor ? withGovernorCostContext(options.governor, costLedger, lease) : void 0,
|
|
2899
|
-
budget: {
|
|
2900
|
-
...options.budget,
|
|
2901
|
-
maxNodes: options.seeds.length + options.budget.maxSteps
|
|
2902
|
-
},
|
|
2903
|
-
store: lineageStore,
|
|
2904
|
-
populationSize: options.populationSize,
|
|
2905
|
-
candidateConcurrency: options.candidateConcurrency
|
|
2906
|
-
};
|
|
2907
|
-
const search = await runLineageLoop(lineageOptions);
|
|
2908
|
-
await lease.assertOwned();
|
|
2909
|
-
const best = search.best;
|
|
2910
|
-
if (!best) throw new Error("memory improvement produced no measured config");
|
|
2911
|
-
const baselineSurface = serialize(options.seeds[0].config);
|
|
2912
|
-
const baselineHash = surfaceHash(baselineSurface);
|
|
2913
|
-
const winnerConfig = parse(requireStringSurface(best.surface));
|
|
2914
|
-
const winnerSurface = serialize(winnerConfig);
|
|
2915
|
-
const winnerHash = surfaceHash(winnerSurface);
|
|
2916
|
-
const baselineConfig = parse(baselineSurface);
|
|
2917
|
-
let holdout;
|
|
2918
|
-
let decision;
|
|
2919
|
-
if (winnerHash === baselineHash) {
|
|
2920
|
-
const baselineMeasurement = await evaluateSurface(baselineSurface);
|
|
2921
|
-
decision = {
|
|
2922
|
-
status: "no-change",
|
|
2923
|
-
reasons: ["search did not find a config better than the baseline"],
|
|
2924
|
-
baselineScore: baselineMeasurement.score,
|
|
2925
|
-
winnerScore: baselineMeasurement.score,
|
|
2926
|
-
lift: 0,
|
|
2927
|
-
criticalDimensions: []
|
|
2928
|
-
};
|
|
2929
|
-
} else {
|
|
2930
|
-
await lease.assertOwned();
|
|
2931
|
-
const [baselineCandidate, winnerCandidate] = await Promise.all([
|
|
2932
|
-
buildCandidate(options, baselineConfig, baselineHash, "baseline"),
|
|
2933
|
-
buildCandidate(options, winnerConfig, winnerHash, "winner")
|
|
2934
|
-
]);
|
|
2935
|
-
await lease.assertOwned();
|
|
2936
|
-
holdout = await runAgentMemoryExperiment({
|
|
2937
|
-
...experimentOptions(options, costLedger, storage, lease),
|
|
2938
|
-
experimentId: `${options.experimentId}:holdout`,
|
|
2939
|
-
sequences: options.holdoutSequences,
|
|
2940
|
-
candidates: [baselineCandidate, winnerCandidate],
|
|
2941
|
-
runDir: join2(runDir, "holdout"),
|
|
2942
|
-
costPhase: "memory.holdout"
|
|
2943
|
-
});
|
|
2944
|
-
decision = decidePromotion({
|
|
2945
|
-
options,
|
|
2946
|
-
result: holdout,
|
|
2947
|
-
baselineId: baselineCandidate.id,
|
|
2948
|
-
winnerId: winnerCandidate.id
|
|
2949
|
-
});
|
|
2950
|
-
}
|
|
2951
|
-
const resultJsonPath = join2(runDir, "memory-improvement-result.json");
|
|
2952
|
-
const activationRef = options.activation?.ref ?? "not-configured";
|
|
2953
|
-
const activationId = `memory-activation-${surfaceHash(
|
|
2954
|
-
canonicalJson5({
|
|
2955
|
-
experimentId: options.experimentId,
|
|
2956
|
-
improvementRef: options.improvementRef,
|
|
2957
|
-
activationRef,
|
|
2958
|
-
baselineSurfaceHash: baselineHash,
|
|
2959
|
-
winnerSurfaceHash: winnerHash,
|
|
2960
|
-
holdoutManifestHash: holdout?.campaign.manifestHash ?? null,
|
|
2961
|
-
promotionPolicy: normalizedPromotionPolicy(options)
|
|
2962
|
-
})
|
|
2963
|
-
)}`;
|
|
2964
|
-
const activationJournalDir = join2(runDir, "activations");
|
|
2965
|
-
const activationJournalPath = join2(activationJournalDir, `${activationId}.jsonl`);
|
|
2966
|
-
const activationEligible = decision.status === "promote" && holdout !== void 0;
|
|
2967
|
-
const activationEventIdentity = holdout ? {
|
|
2968
|
-
schema: 1,
|
|
2969
|
-
activationId,
|
|
2970
|
-
experimentId: options.experimentId,
|
|
2971
|
-
activationRef,
|
|
2972
|
-
baselineSurfaceHash: baselineHash,
|
|
2973
|
-
winnerSurfaceHash: winnerHash,
|
|
2974
|
-
holdoutManifestHash: holdout.campaign.manifestHash
|
|
2975
|
-
} : void 0;
|
|
2976
|
-
const activationJournal = activationEligible && activationEventIdentity ? readMemoryActivationJournal(storage, activationJournalPath, activationEventIdentity) : { prepared: false };
|
|
2977
|
-
const activation = {
|
|
2978
|
-
id: activationId,
|
|
2979
|
-
status: !activationEligible ? "not-eligible" : activationJournal.activated ? "already-activated" : options.activation ? "pending" : "not-configured",
|
|
2980
|
-
journalPath: activationJournalPath
|
|
2981
|
-
};
|
|
2982
|
-
const result = {
|
|
2983
|
-
lineage: search.lineage,
|
|
2984
|
-
baselineConfig,
|
|
2985
|
-
winnerConfig,
|
|
2986
|
-
baselineSurface,
|
|
2987
|
-
winnerSurface,
|
|
2988
|
-
baselineSurfaceHash: baselineHash,
|
|
2989
|
-
winnerSurfaceHash: winnerHash,
|
|
2990
|
-
decision,
|
|
2991
|
-
activation,
|
|
2992
|
-
...holdout ? { holdout } : {},
|
|
2993
|
-
totalCostUsd: costLedger.summary().totalCostUsd,
|
|
2994
|
-
resultJsonPath
|
|
2995
|
-
};
|
|
2996
|
-
await lease.assertOwned();
|
|
2997
|
-
writeMemoryImprovementResult(storage, options.experimentId, result);
|
|
2998
|
-
if (activationEligible && !activationJournal.activated && activationEventIdentity && holdout && options.activation) {
|
|
2999
|
-
const activationDriver = options.activation;
|
|
3000
|
-
const activationTimeoutMs = options.activationTimeoutMs ?? 6e4;
|
|
3001
|
-
const hadPreparedEvent = activationJournal.prepared;
|
|
3002
|
-
if (!hadPreparedEvent) {
|
|
3003
|
-
await lease.assertOwned();
|
|
3004
|
-
storage.ensureDir(activationJournalDir);
|
|
3005
|
-
appendMemoryActivationEvent(storage, activationJournalPath, {
|
|
3006
|
-
...activationEventIdentity,
|
|
3007
|
-
status: "prepared",
|
|
3008
|
-
recordedAt: (options.now ?? (() => /* @__PURE__ */ new Date()))().toISOString()
|
|
3009
|
-
});
|
|
2438
|
+
}
|
|
2439
|
+
};
|
|
2440
|
+
return {
|
|
2441
|
+
...result,
|
|
2442
|
+
sourceRecord: memoryWriteResultToSourceRecord(result, input.text, { scope })
|
|
2443
|
+
};
|
|
2444
|
+
},
|
|
2445
|
+
async clear(scope) {
|
|
2446
|
+
const groupId = graphitiGroupId(id, mergeScopes3(options.defaultScope, scope));
|
|
2447
|
+
for (const [episodeUuid, pending] of queued) {
|
|
2448
|
+
if (pending.groupId !== groupId) continue;
|
|
2449
|
+
await waitForGraphitiEpisode(options, groupId, episodeUuid);
|
|
2450
|
+
queued.delete(episodeUuid);
|
|
2451
|
+
}
|
|
2452
|
+
await callGraphiti(options.client, tools.clearGraph, { group_ids: [groupId] });
|
|
2453
|
+
},
|
|
2454
|
+
async flush() {
|
|
2455
|
+
for (const [episodeUuid, pending] of queued) {
|
|
2456
|
+
await waitForGraphitiEpisode(options, pending.groupId, episodeUuid);
|
|
2457
|
+
queued.delete(episodeUuid);
|
|
2458
|
+
}
|
|
3010
2459
|
}
|
|
3011
|
-
|
|
3012
|
-
|
|
3013
|
-
|
|
3014
|
-
|
|
3015
|
-
|
|
3016
|
-
|
|
3017
|
-
|
|
3018
|
-
|
|
3019
|
-
|
|
3020
|
-
|
|
3021
|
-
|
|
3022
|
-
|
|
2460
|
+
};
|
|
2461
|
+
return adapter;
|
|
2462
|
+
}
|
|
2463
|
+
async function callGraphiti(client, name, args) {
|
|
2464
|
+
const raw = await client.callTool({ name, arguments: args });
|
|
2465
|
+
const payload = graphitiPayload(raw);
|
|
2466
|
+
if (isRecord(payload) && typeof payload.error === "string") {
|
|
2467
|
+
throw new Error(`Graphiti ${name}: ${payload.error}`);
|
|
2468
|
+
}
|
|
2469
|
+
return payload;
|
|
2470
|
+
}
|
|
2471
|
+
function graphitiPayload(raw) {
|
|
2472
|
+
if (!isRecord(raw)) return raw;
|
|
2473
|
+
if (raw.isError === true) throw new Error(`Graphiti MCP: ${toolText(raw) || "tool call failed"}`);
|
|
2474
|
+
if (isRecord(raw.structuredContent)) return unwrapResult(raw.structuredContent);
|
|
2475
|
+
const text = toolText(raw);
|
|
2476
|
+
if (text) {
|
|
2477
|
+
try {
|
|
2478
|
+
return unwrapResult(JSON.parse(text));
|
|
2479
|
+
} catch {
|
|
2480
|
+
return { message: text };
|
|
3023
2481
|
}
|
|
3024
|
-
|
|
3025
|
-
|
|
3026
|
-
|
|
3027
|
-
|
|
3028
|
-
|
|
3029
|
-
|
|
3030
|
-
|
|
3031
|
-
|
|
3032
|
-
|
|
3033
|
-
|
|
3034
|
-
|
|
3035
|
-
|
|
3036
|
-
|
|
3037
|
-
|
|
3038
|
-
|
|
3039
|
-
|
|
3040
|
-
|
|
3041
|
-
|
|
3042
|
-
|
|
3043
|
-
|
|
3044
|
-
|
|
3045
|
-
|
|
3046
|
-
|
|
3047
|
-
|
|
3048
|
-
|
|
3049
|
-
|
|
3050
|
-
|
|
3051
|
-
|
|
3052
|
-
|
|
3053
|
-
|
|
3054
|
-
|
|
3055
|
-
}
|
|
3056
|
-
|
|
3057
|
-
|
|
3058
|
-
|
|
3059
|
-
|
|
3060
|
-
|
|
3061
|
-
|
|
2482
|
+
}
|
|
2483
|
+
return unwrapResult(raw);
|
|
2484
|
+
}
|
|
2485
|
+
function unwrapResult(value) {
|
|
2486
|
+
if (isRecord(value) && Object.keys(value).length === 1 && "result" in value) {
|
|
2487
|
+
return value.result;
|
|
2488
|
+
}
|
|
2489
|
+
return value;
|
|
2490
|
+
}
|
|
2491
|
+
function toolText(raw) {
|
|
2492
|
+
if (!Array.isArray(raw.content)) return "";
|
|
2493
|
+
return raw.content.filter(isRecord).map((part) => typeof part.text === "string" ? part.text : "").filter(Boolean).join("\n");
|
|
2494
|
+
}
|
|
2495
|
+
function normalizeGraphitiFacts(raw, adapterId) {
|
|
2496
|
+
const facts = isRecord(raw) && Array.isArray(raw.facts) ? raw.facts.filter(isRecord) : [];
|
|
2497
|
+
return facts.flatMap((fact) => {
|
|
2498
|
+
const text = stringField(fact, ["fact", "text"]);
|
|
2499
|
+
if (!text) return [];
|
|
2500
|
+
const factId = stringField(fact, ["uuid", "id"]) ?? stableId("fact", canonicalJson5(stripUndefined2({ adapterId, text, fact })));
|
|
2501
|
+
const episodeUuids = stringArray2(fact.episodes ?? fact.episode_uuids);
|
|
2502
|
+
const score = numberField(fact, ["score", "relevance_score"]);
|
|
2503
|
+
const normalizedScore = normalizedScoreOf(score);
|
|
2504
|
+
return [
|
|
2505
|
+
{
|
|
2506
|
+
id: factId,
|
|
2507
|
+
uri: `memory://${adapterId}/fact/${factId}`,
|
|
2508
|
+
kind: "fact",
|
|
2509
|
+
text,
|
|
2510
|
+
...stringField(fact, ["name"]) ? { title: stringField(fact, ["name"]) } : {},
|
|
2511
|
+
...stringField(fact, ["created_at"]) ? { createdAt: stringField(fact, ["created_at"]) } : {},
|
|
2512
|
+
...stringField(fact, ["invalid_at"]) ? { validUntil: stringField(fact, ["invalid_at"]) } : stringField(fact, ["expired_at"]) ? { validUntil: stringField(fact, ["expired_at"]) } : {},
|
|
2513
|
+
...score !== void 0 ? { score } : {},
|
|
2514
|
+
...normalizedScore !== void 0 ? { normalizedScore } : {},
|
|
2515
|
+
metadata: {
|
|
2516
|
+
provider: "graphiti",
|
|
2517
|
+
groupId: fact.group_id,
|
|
2518
|
+
sourceNodeUuid: fact.source_node_uuid,
|
|
2519
|
+
targetNodeUuid: fact.target_node_uuid,
|
|
2520
|
+
validAt: fact.valid_at,
|
|
2521
|
+
invalidAt: fact.invalid_at,
|
|
2522
|
+
episodeUuids,
|
|
2523
|
+
attributes: fact.attributes
|
|
3062
2524
|
}
|
|
3063
|
-
throw error;
|
|
3064
2525
|
}
|
|
3065
|
-
|
|
3066
|
-
|
|
3067
|
-
|
|
3068
|
-
|
|
3069
|
-
|
|
3070
|
-
|
|
3071
|
-
|
|
3072
|
-
|
|
3073
|
-
|
|
3074
|
-
|
|
3075
|
-
|
|
2526
|
+
];
|
|
2527
|
+
});
|
|
2528
|
+
}
|
|
2529
|
+
function normalizeGraphitiNodes(raw, adapterId) {
|
|
2530
|
+
const nodes = isRecord(raw) && Array.isArray(raw.nodes) ? raw.nodes.filter(isRecord) : [];
|
|
2531
|
+
return nodes.flatMap((node) => {
|
|
2532
|
+
const name = stringField(node, ["name"]);
|
|
2533
|
+
const summary = stringField(node, ["summary"]);
|
|
2534
|
+
const text = summary ?? name;
|
|
2535
|
+
if (!text) return [];
|
|
2536
|
+
const nodeId = stringField(node, ["uuid", "id"]) ?? stableId("node", canonicalJson5(stripUndefined2({ adapterId, text, node })));
|
|
2537
|
+
const score = numberField(node, ["score", "relevance_score"]);
|
|
2538
|
+
const normalizedScore = normalizedScoreOf(score);
|
|
2539
|
+
return [
|
|
2540
|
+
{
|
|
2541
|
+
id: nodeId,
|
|
2542
|
+
uri: `memory://${adapterId}/entity/${nodeId}`,
|
|
2543
|
+
kind: "entity",
|
|
2544
|
+
text,
|
|
2545
|
+
...name ? { title: name } : {},
|
|
2546
|
+
...stringField(node, ["created_at"]) ? { createdAt: stringField(node, ["created_at"]) } : {},
|
|
2547
|
+
...score !== void 0 ? { score } : {},
|
|
2548
|
+
...normalizedScore !== void 0 ? { normalizedScore } : {},
|
|
2549
|
+
metadata: {
|
|
2550
|
+
provider: "graphiti",
|
|
2551
|
+
groupId: node.group_id,
|
|
2552
|
+
labels: node.labels,
|
|
2553
|
+
attributes: node.attributes
|
|
3076
2554
|
}
|
|
3077
|
-
throw mismatch;
|
|
3078
2555
|
}
|
|
3079
|
-
|
|
3080
|
-
|
|
2556
|
+
];
|
|
2557
|
+
});
|
|
2558
|
+
}
|
|
2559
|
+
async function enrichGraphitiProvenance(options, groupId, hits) {
|
|
2560
|
+
const wanted = new Set(hits.flatMap((hit) => stringArray2(hit.metadata?.episodeUuids)));
|
|
2561
|
+
if (wanted.size === 0) return;
|
|
2562
|
+
const scan = await scanGraphitiEpisodes(
|
|
2563
|
+
options,
|
|
2564
|
+
groupId,
|
|
2565
|
+
wanted,
|
|
2566
|
+
options.provenanceEpisodeLimit ?? options.episodeScanLimit ?? 1e4
|
|
2567
|
+
);
|
|
2568
|
+
const episodes = scan.episodes;
|
|
2569
|
+
const byId = new Map(episodes.map((episode) => [episode.uuid, episode]));
|
|
2570
|
+
for (const hit of hits) {
|
|
2571
|
+
const episodeUuids = stringArray2(hit.metadata?.episodeUuids);
|
|
2572
|
+
const sourceEpisodes = episodeUuids.map((uuid) => byId.get(uuid)).filter((episode) => episode !== void 0);
|
|
2573
|
+
const provenanceIncomplete = episodeUuids.some((uuid) => !byId.has(uuid));
|
|
2574
|
+
if (sourceEpisodes.length === 0) {
|
|
2575
|
+
if (provenanceIncomplete) {
|
|
2576
|
+
hit.metadata = { ...hit.metadata, provenanceIncomplete: true };
|
|
2577
|
+
}
|
|
2578
|
+
continue;
|
|
3081
2579
|
}
|
|
3082
|
-
|
|
3083
|
-
|
|
3084
|
-
|
|
3085
|
-
|
|
3086
|
-
|
|
3087
|
-
|
|
3088
|
-
|
|
3089
|
-
|
|
2580
|
+
const descriptions = sourceEpisodes.map((episode) => parseDescription(episode.source_description)).filter(isRecord);
|
|
2581
|
+
const first = descriptions[0];
|
|
2582
|
+
hit.metadata = {
|
|
2583
|
+
...hit.metadata,
|
|
2584
|
+
sourceEpisodes,
|
|
2585
|
+
...provenanceIncomplete || !scan.complete ? { provenanceIncomplete: true } : {},
|
|
2586
|
+
...first ?? {},
|
|
2587
|
+
eventIds: uniqueStrings2(descriptions.map((value) => value.eventId)),
|
|
2588
|
+
actorIds: uniqueStrings2(descriptions.map((value) => value.actorId)),
|
|
2589
|
+
...typeof first?.eventId === "string" ? { eventId: first.eventId } : {},
|
|
2590
|
+
...typeof first?.actorId === "string" ? { actorId: first.actorId } : {}
|
|
2591
|
+
};
|
|
2592
|
+
}
|
|
2593
|
+
}
|
|
2594
|
+
async function getGraphitiEpisodes(options, groupId, limit) {
|
|
2595
|
+
const raw = await callGraphiti(options.client, graphitiToolNames(options).getEpisodes, {
|
|
2596
|
+
group_ids: [groupId],
|
|
2597
|
+
max_episodes: limit
|
|
2598
|
+
});
|
|
2599
|
+
if (!isRecord(raw) || !Array.isArray(raw.episodes)) return [];
|
|
2600
|
+
return raw.episodes.filter(isRecord).flatMap((episode) => {
|
|
2601
|
+
const uuid = stringField(episode, ["uuid", "id"]);
|
|
2602
|
+
return uuid ? [{ ...episode, uuid }] : [];
|
|
2603
|
+
});
|
|
2604
|
+
}
|
|
2605
|
+
async function scanGraphitiEpisodes(options, groupId, wanted, maximum) {
|
|
2606
|
+
let limit = Math.min(100, maximum);
|
|
2607
|
+
for (; ; ) {
|
|
2608
|
+
const episodes = await getGraphitiEpisodes(options, groupId, limit);
|
|
2609
|
+
const foundAll = [...wanted].every((uuid) => episodes.some((episode) => episode.uuid === uuid));
|
|
2610
|
+
if (foundAll || episodes.length < limit) return { episodes, complete: foundAll };
|
|
2611
|
+
if (limit === maximum) return { episodes, complete: false };
|
|
2612
|
+
limit = Math.min(maximum, limit * 2);
|
|
2613
|
+
}
|
|
2614
|
+
}
|
|
2615
|
+
async function waitForGraphitiEpisode(options, groupId, episodeUuid) {
|
|
2616
|
+
const timeoutMs = options.ingestionTimeoutMs ?? DEFAULT_GRAPHITI_INGESTION_TIMEOUT_MS;
|
|
2617
|
+
const pollIntervalMs = options.pollIntervalMs ?? 500;
|
|
2618
|
+
const maximum = options.episodeScanLimit ?? 1e4;
|
|
2619
|
+
const deadline = Date.now() + timeoutMs;
|
|
2620
|
+
let scanLimitReached = false;
|
|
2621
|
+
for (; ; ) {
|
|
2622
|
+
const scan = await scanGraphitiEpisodes(options, groupId, /* @__PURE__ */ new Set([episodeUuid]), maximum);
|
|
2623
|
+
if (scan.complete) return;
|
|
2624
|
+
scanLimitReached ||= scan.episodes.length >= maximum;
|
|
2625
|
+
const remainingMs = deadline - Date.now();
|
|
2626
|
+
if (remainingMs <= 0) break;
|
|
2627
|
+
await new Promise((resolve) => setTimeout(resolve, Math.min(pollIntervalMs, remainingMs)));
|
|
2628
|
+
}
|
|
2629
|
+
if (scanLimitReached) {
|
|
2630
|
+
throw new Error(
|
|
2631
|
+
`Graphiti episode ${episodeUuid} is outside episodeScanLimit=${maximum}; raise the limit or use a smaller group`
|
|
2632
|
+
);
|
|
3090
2633
|
}
|
|
3091
|
-
|
|
2634
|
+
throw new Error(`Graphiti episode ${episodeUuid} was not visible after ${timeoutMs}ms`);
|
|
3092
2635
|
}
|
|
3093
|
-
function
|
|
2636
|
+
function graphitiGroupId(adapterId, scope) {
|
|
2637
|
+
return `ak_${sha256(
|
|
2638
|
+
canonicalJson5(
|
|
2639
|
+
stripUndefined2({
|
|
2640
|
+
adapterId,
|
|
2641
|
+
tenantId: scope.tenantId,
|
|
2642
|
+
userId: scope.userId,
|
|
2643
|
+
agentId: scope.agentId,
|
|
2644
|
+
teamId: scope.teamId,
|
|
2645
|
+
runId: scope.runId,
|
|
2646
|
+
sessionId: scope.sessionId,
|
|
2647
|
+
namespace: scope.namespace,
|
|
2648
|
+
tags: scope.tags
|
|
2649
|
+
})
|
|
2650
|
+
)
|
|
2651
|
+
).slice(0, 32)}`;
|
|
2652
|
+
}
|
|
2653
|
+
function graphitiToolNames(options) {
|
|
3094
2654
|
return {
|
|
3095
|
-
|
|
3096
|
-
|
|
3097
|
-
|
|
3098
|
-
|
|
3099
|
-
|
|
3100
|
-
costLedger,
|
|
3101
|
-
costPhase: `memory.proposal.${context.track?.id ?? label}`
|
|
3102
|
-
});
|
|
3103
|
-
await lease.assertOwned();
|
|
3104
|
-
return proposal;
|
|
3105
|
-
},
|
|
3106
|
-
...proposer.decide ? { decide: (input) => proposer.decide(input) } : {}
|
|
2655
|
+
addMemory: options.toolNames?.addMemory ?? "add_memory",
|
|
2656
|
+
searchFacts: options.toolNames?.searchFacts ?? "search_memory_facts",
|
|
2657
|
+
searchNodes: options.toolNames?.searchNodes ?? "search_nodes",
|
|
2658
|
+
getEpisodes: options.toolNames?.getEpisodes ?? "get_episodes",
|
|
2659
|
+
clearGraph: options.toolNames?.clearGraph ?? "clear_graph"
|
|
3107
2660
|
};
|
|
3108
2661
|
}
|
|
3109
|
-
function
|
|
3110
|
-
|
|
3111
|
-
|
|
3112
|
-
|
|
3113
|
-
|
|
3114
|
-
|
|
3115
|
-
|
|
3116
|
-
|
|
3117
|
-
|
|
3118
|
-
|
|
3119
|
-
|
|
2662
|
+
function assertGraphitiOptions(options) {
|
|
2663
|
+
if (options.id !== void 0 && (typeof options.id !== "string" || !options.id.trim())) {
|
|
2664
|
+
throw new Error("Graphiti id must be a non-empty string");
|
|
2665
|
+
}
|
|
2666
|
+
if (options.backendRef !== void 0 && (typeof options.backendRef !== "string" || !options.backendRef.trim())) {
|
|
2667
|
+
throw new Error("Graphiti backendRef must be a non-empty string");
|
|
2668
|
+
}
|
|
2669
|
+
if (options.consistency !== void 0 && !["queued", "visible"].includes(options.consistency)) {
|
|
2670
|
+
throw new Error("Graphiti consistency must be queued or visible");
|
|
2671
|
+
}
|
|
2672
|
+
for (const [name, value] of [
|
|
2673
|
+
["ingestionTimeoutMs", options.ingestionTimeoutMs],
|
|
2674
|
+
["pollIntervalMs", options.pollIntervalMs],
|
|
2675
|
+
["episodeScanLimit", options.episodeScanLimit],
|
|
2676
|
+
["provenanceEpisodeLimit", options.provenanceEpisodeLimit]
|
|
2677
|
+
]) {
|
|
2678
|
+
if (value !== void 0 && (!Number.isSafeInteger(value) || value <= 0)) {
|
|
2679
|
+
throw new Error(`Graphiti ${name} must be a positive safe integer`);
|
|
3120
2680
|
}
|
|
3121
|
-
}
|
|
2681
|
+
}
|
|
2682
|
+
for (const [name, value] of Object.entries(options.toolNames ?? {})) {
|
|
2683
|
+
if (typeof value !== "string" || value.trim().length === 0) {
|
|
2684
|
+
throw new Error(`Graphiti toolNames.${name} must be a non-empty string`);
|
|
2685
|
+
}
|
|
2686
|
+
}
|
|
3122
2687
|
}
|
|
3123
|
-
|
|
3124
|
-
const
|
|
3125
|
-
|
|
3126
|
-
candidateId: `${role}-${hash}`,
|
|
3127
|
-
surfaceHash: hash
|
|
3128
|
-
});
|
|
3129
|
-
return {
|
|
3130
|
-
...built,
|
|
3131
|
-
id: `${role}-${hash}`,
|
|
3132
|
-
ref: built.ref
|
|
3133
|
-
};
|
|
2688
|
+
function deterministicUuid(value) {
|
|
2689
|
+
const hex = sha256(value);
|
|
2690
|
+
return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-5${hex.slice(13, 16)}-a${hex.slice(17, 20)}-${hex.slice(20, 32)}`;
|
|
3134
2691
|
}
|
|
3135
|
-
function
|
|
2692
|
+
function parseDescription(value) {
|
|
2693
|
+
if (!value) return void 0;
|
|
2694
|
+
try {
|
|
2695
|
+
return JSON.parse(value);
|
|
2696
|
+
} catch {
|
|
2697
|
+
return void 0;
|
|
2698
|
+
}
|
|
2699
|
+
}
|
|
2700
|
+
function mergeScopes3(base, extra) {
|
|
3136
2701
|
return {
|
|
3137
|
-
|
|
3138
|
-
|
|
3139
|
-
|
|
3140
|
-
resumable: options.resumable,
|
|
3141
|
-
maxConcurrency: options.sequenceConcurrency,
|
|
3142
|
-
dispatchTimeoutMs: options.dispatchTimeoutMs,
|
|
3143
|
-
cleanupTimeoutMs: options.cleanupTimeoutMs,
|
|
3144
|
-
maxRecoveryAttempts: options.maxRecoveryAttempts,
|
|
3145
|
-
maxRecoveryRetriesPerAttempt: options.maxRecoveryRetriesPerAttempt,
|
|
3146
|
-
executeStep: options.executeStep,
|
|
3147
|
-
executeStepRef: options.executeStepRef,
|
|
3148
|
-
onBranchSnapshot: options.onBranchSnapshot,
|
|
3149
|
-
cleanupBranches: options.cleanupBranches ?? true,
|
|
3150
|
-
costLedger,
|
|
3151
|
-
acquireRunLease: async () => ({
|
|
3152
|
-
assertOwned: () => lease.assertOwned(),
|
|
3153
|
-
release() {
|
|
3154
|
-
}
|
|
3155
|
-
}),
|
|
3156
|
-
now: options.now
|
|
2702
|
+
...base ?? {},
|
|
2703
|
+
...extra ?? {},
|
|
2704
|
+
tags: { ...base?.tags ?? {}, ...extra?.tags ?? {} }
|
|
3157
2705
|
};
|
|
3158
2706
|
}
|
|
3159
|
-
function
|
|
3160
|
-
|
|
3161
|
-
|
|
3162
|
-
|
|
3163
|
-
|
|
3164
|
-
|
|
3165
|
-
|
|
3166
|
-
|
|
3167
|
-
|
|
3168
|
-
|
|
3169
|
-
Object.entries(options.proposers ?? {}).sort(([a], [b]) => a.localeCompare(b)).map(([name, proposer]) => [name, proposer.kind])
|
|
3170
|
-
),
|
|
3171
|
-
populationSize: options.populationSize ?? 4,
|
|
3172
|
-
budget: { maxSteps: options.budget.maxSteps },
|
|
3173
|
-
executeStepRef: options.executeStepRef ?? null,
|
|
3174
|
-
cleanupBranches: options.cleanupBranches ?? true,
|
|
3175
|
-
promotionPolicy: normalizedPromotionPolicy(options),
|
|
3176
|
-
seed: options.seed ?? null,
|
|
3177
|
-
reps: options.reps ?? 1,
|
|
3178
|
-
seeds: options.seeds.map((entry) => ({
|
|
3179
|
-
config: serialize(entry.config),
|
|
3180
|
-
track: entry.track,
|
|
3181
|
-
vision: entry.vision ?? null,
|
|
3182
|
-
proposer: entry.proposer
|
|
3183
|
-
})),
|
|
3184
|
-
trainSequences: options.trainSequences,
|
|
3185
|
-
holdoutSequences: options.holdoutSequences
|
|
3186
|
-
};
|
|
3187
|
-
const identityHash = surfaceHash(canonicalJson5(identity));
|
|
3188
|
-
const stored = storage.read(path);
|
|
3189
|
-
if (stored === void 0) {
|
|
3190
|
-
if (storage.exists(path)) throw new Error(`cannot read memory improvement manifest '${path}'`);
|
|
3191
|
-
if (storage.exists(join2(runDir, "lineage.jsonl")) || storage.exists(join2(runDir, "memory-improvement-result.json"))) {
|
|
3192
|
-
throw new Error(`memory improvement run '${runDir}' has state without an identity manifest`);
|
|
3193
|
-
}
|
|
3194
|
-
storage.write(path, `${JSON.stringify({ identityHash, identity }, null, 2)}
|
|
3195
|
-
`);
|
|
3196
|
-
return;
|
|
2707
|
+
function uniqueStrings2(values) {
|
|
2708
|
+
return [...new Set(values.filter((value) => typeof value === "string"))];
|
|
2709
|
+
}
|
|
2710
|
+
function stringArray2(value) {
|
|
2711
|
+
return Array.isArray(value) ? value.filter((entry) => typeof entry === "string") : [];
|
|
2712
|
+
}
|
|
2713
|
+
function stringField(value, keys) {
|
|
2714
|
+
for (const key of keys) {
|
|
2715
|
+
const field = value[key];
|
|
2716
|
+
if (typeof field === "string" && field.length > 0) return field;
|
|
3197
2717
|
}
|
|
3198
|
-
|
|
3199
|
-
|
|
3200
|
-
|
|
3201
|
-
|
|
3202
|
-
|
|
2718
|
+
return void 0;
|
|
2719
|
+
}
|
|
2720
|
+
function numberField(value, keys) {
|
|
2721
|
+
for (const key of keys) {
|
|
2722
|
+
const field = value[key];
|
|
2723
|
+
if (typeof field === "number" && Number.isFinite(field)) return field;
|
|
3203
2724
|
}
|
|
3204
|
-
|
|
3205
|
-
|
|
3206
|
-
|
|
3207
|
-
|
|
2725
|
+
return void 0;
|
|
2726
|
+
}
|
|
2727
|
+
function normalizedScoreOf(score) {
|
|
2728
|
+
return score !== void 0 && score >= 0 && score <= 1 ? score : void 0;
|
|
2729
|
+
}
|
|
2730
|
+
function isRecord(value) {
|
|
2731
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
2732
|
+
}
|
|
2733
|
+
function stripUndefined2(value) {
|
|
2734
|
+
if (Array.isArray(value)) return value.map(stripUndefined2);
|
|
2735
|
+
if (!isRecord(value)) return value;
|
|
2736
|
+
return Object.fromEntries(
|
|
2737
|
+
Object.entries(value).filter(([, child]) => child !== void 0).map(([key, child]) => [key, stripUndefined2(child)])
|
|
2738
|
+
);
|
|
2739
|
+
}
|
|
2740
|
+
function graphitiMemoryAdapterIdentity(options) {
|
|
2741
|
+
if (typeof options.backendRef !== "string" || !options.backendRef.trim()) {
|
|
2742
|
+
throw new Error("Graphiti backendRef must be a non-empty string");
|
|
3208
2743
|
}
|
|
2744
|
+
return stableId("graphiti", canonicalJson5(stripUndefined2(options)));
|
|
3209
2745
|
}
|
|
2746
|
+
|
|
2747
|
+
// src/memory/improvement/run.ts
|
|
2748
|
+
import { join as join3 } from "path";
|
|
2749
|
+
import { canonicalJson as canonicalJson7 } from "@tangle-network/agent-eval";
|
|
2750
|
+
import {
|
|
2751
|
+
campaignLineageStore,
|
|
2752
|
+
createRunCostLedger as createRunCostLedger2,
|
|
2753
|
+
fsCampaignStorage as fsCampaignStorage2,
|
|
2754
|
+
memLineageStore,
|
|
2755
|
+
resolveRunDir as resolveRunDir2,
|
|
2756
|
+
runLineageLoop,
|
|
2757
|
+
surfaceHash as surfaceHash2
|
|
2758
|
+
} from "@tangle-network/agent-eval/campaign";
|
|
2759
|
+
|
|
2760
|
+
// src/memory/improvement/activation.ts
|
|
3210
2761
|
function appendMemoryActivationEvent(storage, path, event) {
|
|
3211
2762
|
appendDurableJournalEvent({
|
|
3212
2763
|
storage,
|
|
@@ -3274,27 +2825,94 @@ function parseMemoryActivationEvent(value, path, line, expected) {
|
|
|
3274
2825
|
}
|
|
3275
2826
|
return value;
|
|
3276
2827
|
}
|
|
3277
|
-
|
|
3278
|
-
|
|
3279
|
-
|
|
3280
|
-
|
|
3281
|
-
|
|
3282
|
-
|
|
3283
|
-
|
|
3284
|
-
|
|
3285
|
-
|
|
3286
|
-
|
|
3287
|
-
|
|
3288
|
-
|
|
3289
|
-
|
|
3290
|
-
|
|
3291
|
-
|
|
3292
|
-
|
|
3293
|
-
|
|
3294
|
-
|
|
3295
|
-
|
|
3296
|
-
|
|
2828
|
+
|
|
2829
|
+
// src/memory/improvement/candidate.ts
|
|
2830
|
+
function withCostContext(proposer, costLedger, lease, label) {
|
|
2831
|
+
return {
|
|
2832
|
+
kind: proposer.kind,
|
|
2833
|
+
async propose(context) {
|
|
2834
|
+
await lease.assertOwned();
|
|
2835
|
+
const proposal = await proposer.propose({
|
|
2836
|
+
...context,
|
|
2837
|
+
costLedger,
|
|
2838
|
+
costPhase: `memory.proposal.${context.track?.id ?? label}`
|
|
2839
|
+
});
|
|
2840
|
+
await lease.assertOwned();
|
|
2841
|
+
return proposal;
|
|
2842
|
+
},
|
|
2843
|
+
...proposer.decide ? { decide: (input) => proposer.decide(input) } : {}
|
|
2844
|
+
};
|
|
2845
|
+
}
|
|
2846
|
+
function withGovernorCostContext(governor, costLedger, lease) {
|
|
2847
|
+
return {
|
|
2848
|
+
async decide(context) {
|
|
2849
|
+
await lease.assertOwned();
|
|
2850
|
+
const decision = await governor.decide({
|
|
2851
|
+
...context,
|
|
2852
|
+
costLedger,
|
|
2853
|
+
costPhase: "memory.governor"
|
|
2854
|
+
});
|
|
2855
|
+
await lease.assertOwned();
|
|
2856
|
+
return decision;
|
|
2857
|
+
}
|
|
2858
|
+
};
|
|
2859
|
+
}
|
|
2860
|
+
async function buildCandidate(options, config, hash, role) {
|
|
2861
|
+
const built = await options.createCandidate({
|
|
2862
|
+
config,
|
|
2863
|
+
candidateId: `${role}-${hash}`,
|
|
2864
|
+
surfaceHash: hash
|
|
2865
|
+
});
|
|
2866
|
+
return {
|
|
2867
|
+
...built,
|
|
2868
|
+
id: `${role}-${hash}`,
|
|
2869
|
+
ref: built.ref
|
|
2870
|
+
};
|
|
2871
|
+
}
|
|
2872
|
+
function experimentOptions(options, costLedger, storage, lease) {
|
|
2873
|
+
return {
|
|
2874
|
+
storage,
|
|
2875
|
+
seed: options.seed,
|
|
2876
|
+
reps: options.reps,
|
|
2877
|
+
resumable: options.resumable,
|
|
2878
|
+
maxConcurrency: options.sequenceConcurrency,
|
|
2879
|
+
dispatchTimeoutMs: options.dispatchTimeoutMs,
|
|
2880
|
+
cleanupTimeoutMs: options.cleanupTimeoutMs,
|
|
2881
|
+
maxRecoveryAttempts: options.maxRecoveryAttempts,
|
|
2882
|
+
maxRecoveryRetriesPerAttempt: options.maxRecoveryRetriesPerAttempt,
|
|
2883
|
+
executeStep: options.executeStep,
|
|
2884
|
+
executeStepRef: options.executeStepRef,
|
|
2885
|
+
onBranchSnapshot: options.onBranchSnapshot,
|
|
2886
|
+
cleanupBranches: options.cleanupBranches ?? true,
|
|
2887
|
+
costLedger,
|
|
2888
|
+
acquireRunLease: async () => ({
|
|
2889
|
+
assertOwned: () => lease.assertOwned(),
|
|
2890
|
+
release() {
|
|
2891
|
+
}
|
|
2892
|
+
}),
|
|
2893
|
+
now: options.now
|
|
2894
|
+
};
|
|
3297
2895
|
}
|
|
2896
|
+
|
|
2897
|
+
// src/memory/improvement/identity.ts
|
|
2898
|
+
import { join as join2 } from "path";
|
|
2899
|
+
import { canonicalJson as canonicalJson6 } from "@tangle-network/agent-eval";
|
|
2900
|
+
import {
|
|
2901
|
+
surfaceHash
|
|
2902
|
+
} from "@tangle-network/agent-eval/campaign";
|
|
2903
|
+
|
|
2904
|
+
// src/memory/improvement/promotion.ts
|
|
2905
|
+
import { heldoutSignificance } from "@tangle-network/agent-eval/campaign";
|
|
2906
|
+
|
|
2907
|
+
// src/memory/improvement/types.ts
|
|
2908
|
+
var DEFAULT_CRITICAL_DIMENSIONS = [
|
|
2909
|
+
"memory_stale_safe",
|
|
2910
|
+
"memory_actor_recall",
|
|
2911
|
+
"memory_event_recall"
|
|
2912
|
+
];
|
|
2913
|
+
var MEMORY_IMPROVEMENT_IMPLEMENTATION_REF = "agent-knowledge:memory-improvement:v2";
|
|
2914
|
+
|
|
2915
|
+
// src/memory/improvement/promotion.ts
|
|
3298
2916
|
function decidePromotion(input) {
|
|
3299
2917
|
const { options, result, baselineId, winnerId } = input;
|
|
3300
2918
|
const baselineRow = result.rows.find((row) => row.candidateId === baselineId);
|
|
@@ -3433,6 +3051,62 @@ function sequenceScores(result, sequences, candidateId) {
|
|
|
3433
3051
|
}
|
|
3434
3052
|
return sequences.map((sequence) => mean2(bySequence.get(sequence.id) ?? []));
|
|
3435
3053
|
}
|
|
3054
|
+
function mean2(values) {
|
|
3055
|
+
return values.length === 0 ? 0 : values.reduce((sum, value) => sum + value, 0) / values.length;
|
|
3056
|
+
}
|
|
3057
|
+
|
|
3058
|
+
// src/memory/improvement/identity.ts
|
|
3059
|
+
function assertMemoryImprovementIdentity(options, storage, runDir, serialize) {
|
|
3060
|
+
const path = join2(runDir, "memory-improvement-manifest.json");
|
|
3061
|
+
const identity = {
|
|
3062
|
+
schema: 6,
|
|
3063
|
+
implementationRef: MEMORY_IMPROVEMENT_IMPLEMENTATION_REF,
|
|
3064
|
+
experimentId: options.experimentId,
|
|
3065
|
+
improvementRef: options.improvementRef,
|
|
3066
|
+
activationRef: options.activation?.ref ?? null,
|
|
3067
|
+
proposerKind: options.proposer.kind,
|
|
3068
|
+
proposerKinds: Object.fromEntries(
|
|
3069
|
+
Object.entries(options.proposers ?? {}).sort(([a], [b]) => a.localeCompare(b)).map(([name, proposer]) => [name, proposer.kind])
|
|
3070
|
+
),
|
|
3071
|
+
populationSize: options.populationSize ?? 4,
|
|
3072
|
+
budget: { maxSteps: options.budget.maxSteps },
|
|
3073
|
+
executeStepRef: options.executeStepRef ?? null,
|
|
3074
|
+
cleanupBranches: options.cleanupBranches ?? true,
|
|
3075
|
+
promotionPolicy: normalizedPromotionPolicy(options),
|
|
3076
|
+
seed: options.seed ?? null,
|
|
3077
|
+
reps: options.reps ?? 1,
|
|
3078
|
+
seeds: options.seeds.map((entry) => ({
|
|
3079
|
+
config: serialize(entry.config),
|
|
3080
|
+
track: entry.track,
|
|
3081
|
+
vision: entry.vision ?? null,
|
|
3082
|
+
proposer: entry.proposer
|
|
3083
|
+
})),
|
|
3084
|
+
trainSequences: options.trainSequences,
|
|
3085
|
+
holdoutSequences: options.holdoutSequences
|
|
3086
|
+
};
|
|
3087
|
+
const identityHash = surfaceHash(canonicalJson6(identity));
|
|
3088
|
+
const stored = storage.read(path);
|
|
3089
|
+
if (stored === void 0) {
|
|
3090
|
+
if (storage.exists(path)) throw new Error(`cannot read memory improvement manifest '${path}'`);
|
|
3091
|
+
if (storage.exists(join2(runDir, "lineage.jsonl")) || storage.exists(join2(runDir, "memory-improvement-result.json"))) {
|
|
3092
|
+
throw new Error(`memory improvement run '${runDir}' has state without an identity manifest`);
|
|
3093
|
+
}
|
|
3094
|
+
storage.write(path, `${JSON.stringify({ identityHash, identity }, null, 2)}
|
|
3095
|
+
`);
|
|
3096
|
+
return;
|
|
3097
|
+
}
|
|
3098
|
+
let manifest;
|
|
3099
|
+
try {
|
|
3100
|
+
manifest = JSON.parse(stored);
|
|
3101
|
+
} catch (error) {
|
|
3102
|
+
throw new Error(`invalid memory improvement manifest '${path}'`, { cause: error });
|
|
3103
|
+
}
|
|
3104
|
+
if (!manifest || typeof manifest !== "object" || manifest.identityHash !== identityHash || canonicalJson6(manifest.identity) !== canonicalJson6(identity)) {
|
|
3105
|
+
throw new Error(
|
|
3106
|
+
`memory improvement run '${runDir}' does not match its persisted inputs or implementationRef`
|
|
3107
|
+
);
|
|
3108
|
+
}
|
|
3109
|
+
}
|
|
3436
3110
|
function requireStringSurface(surface) {
|
|
3437
3111
|
if (typeof surface !== "string" || surface.trim().length === 0) {
|
|
3438
3112
|
throw new Error("memory config proposer must return a JSON string surface");
|
|
@@ -3469,8 +3143,33 @@ function assertMemoryConfigRoundTrip(config, serialize, parse) {
|
|
|
3469
3143
|
}
|
|
3470
3144
|
function memorySequenceFingerprint(sequence) {
|
|
3471
3145
|
const { id: _id, split: _split, tags: _tags, ...content } = sequence;
|
|
3472
|
-
return surfaceHash(
|
|
3146
|
+
return surfaceHash(canonicalJson6(content));
|
|
3147
|
+
}
|
|
3148
|
+
|
|
3149
|
+
// src/memory/improvement/output.ts
|
|
3150
|
+
function writeMemoryImprovementResult(storage, experimentId, result) {
|
|
3151
|
+
storage.write(
|
|
3152
|
+
result.resultJsonPath,
|
|
3153
|
+
`${JSON.stringify(
|
|
3154
|
+
{
|
|
3155
|
+
experimentId,
|
|
3156
|
+
baselineSurfaceHash: result.baselineSurfaceHash,
|
|
3157
|
+
winnerSurfaceHash: result.winnerSurfaceHash,
|
|
3158
|
+
baselineSurface: result.baselineSurface,
|
|
3159
|
+
winnerSurface: result.winnerSurface,
|
|
3160
|
+
decision: result.decision,
|
|
3161
|
+
activation: result.activation,
|
|
3162
|
+
totalCostUsd: result.totalCostUsd,
|
|
3163
|
+
lineage: result.lineage.toGraph()
|
|
3164
|
+
},
|
|
3165
|
+
null,
|
|
3166
|
+
2
|
|
3167
|
+
)}
|
|
3168
|
+
`
|
|
3169
|
+
);
|
|
3473
3170
|
}
|
|
3171
|
+
|
|
3172
|
+
// src/memory/improvement/validation.ts
|
|
3474
3173
|
function assertMemoryImprovementOptions(options) {
|
|
3475
3174
|
for (const [name, value] of [
|
|
3476
3175
|
["experimentId", options.experimentId],
|
|
@@ -3545,40 +3244,385 @@ function assertMemoryImprovementOptions(options) {
|
|
|
3545
3244
|
if (typeof seed.track !== "string" || !seed.track.trim() || typeof seed.proposer !== "string" || !seed.proposer.trim()) {
|
|
3546
3245
|
throw new Error("memory improvement seeds require non-empty track and proposer values");
|
|
3547
3246
|
}
|
|
3548
|
-
if (seed.vision !== void 0 && (typeof seed.vision !== "string" || !seed.vision.trim())) {
|
|
3549
|
-
throw new Error("memory improvement seed vision must be a non-empty string when provided");
|
|
3247
|
+
if (seed.vision !== void 0 && (typeof seed.vision !== "string" || !seed.vision.trim())) {
|
|
3248
|
+
throw new Error("memory improvement seed vision must be a non-empty string when provided");
|
|
3249
|
+
}
|
|
3250
|
+
}
|
|
3251
|
+
const dimensions = options.criticalDimensions ?? DEFAULT_CRITICAL_DIMENSIONS;
|
|
3252
|
+
if (dimensions.some((dimension) => typeof dimension !== "string" || !dimension.trim())) {
|
|
3253
|
+
throw new Error("memory improvement criticalDimensions must contain non-empty names");
|
|
3254
|
+
}
|
|
3255
|
+
if (new Set(dimensions).size !== dimensions.length) {
|
|
3256
|
+
throw new Error("memory improvement criticalDimensions must be unique");
|
|
3257
|
+
}
|
|
3258
|
+
assertDistinctSequenceContent(options.trainSequences, "train");
|
|
3259
|
+
assertDistinctSequenceContent(options.holdoutSequences, "holdout");
|
|
3260
|
+
}
|
|
3261
|
+
function assertDistinctSequenceContent(sequences, split) {
|
|
3262
|
+
const idsByFingerprint = /* @__PURE__ */ new Map();
|
|
3263
|
+
for (const sequence of sequences) {
|
|
3264
|
+
const fingerprint = memorySequenceFingerprint(sequence);
|
|
3265
|
+
const prior = idsByFingerprint.get(fingerprint);
|
|
3266
|
+
if (prior) {
|
|
3267
|
+
throw new Error(
|
|
3268
|
+
`memory improvement ${split} histories duplicate content: ${prior}/${sequence.id}`
|
|
3269
|
+
);
|
|
3270
|
+
}
|
|
3271
|
+
idsByFingerprint.set(fingerprint, sequence.id);
|
|
3272
|
+
}
|
|
3273
|
+
}
|
|
3274
|
+
|
|
3275
|
+
// src/memory/improvement/run.ts
|
|
3276
|
+
async function runAgentMemoryImprovement(options) {
|
|
3277
|
+
if ("onPromote" in options) {
|
|
3278
|
+
throw new Error(
|
|
3279
|
+
"memory improvement onPromote was removed; use activation.readCurrent and activation.compareAndSet"
|
|
3280
|
+
);
|
|
3281
|
+
}
|
|
3282
|
+
if (options.seeds.length === 0) throw new Error("memory improvement requires seed configs");
|
|
3283
|
+
if (options.trainSequences.length === 0) {
|
|
3284
|
+
throw new Error("memory improvement requires training sequences");
|
|
3285
|
+
}
|
|
3286
|
+
if (options.holdoutSequences.length === 0) {
|
|
3287
|
+
throw new Error("memory improvement requires holdout sequences");
|
|
3288
|
+
}
|
|
3289
|
+
const trainIds = new Set(options.trainSequences.map((sequence) => sequence.id));
|
|
3290
|
+
const overlap = options.holdoutSequences.map((sequence) => sequence.id).filter((id) => trainIds.has(id));
|
|
3291
|
+
if (overlap.length > 0) {
|
|
3292
|
+
throw new Error(`memory improvement train/holdout overlap: ${overlap.join(", ")}`);
|
|
3293
|
+
}
|
|
3294
|
+
const trainFingerprints = new Map(
|
|
3295
|
+
options.trainSequences.map((sequence) => [memorySequenceFingerprint(sequence), sequence.id])
|
|
3296
|
+
);
|
|
3297
|
+
const duplicateHistories = options.holdoutSequences.flatMap((sequence) => {
|
|
3298
|
+
const trainId = trainFingerprints.get(memorySequenceFingerprint(sequence));
|
|
3299
|
+
return trainId ? [`${trainId}/${sequence.id}`] : [];
|
|
3300
|
+
});
|
|
3301
|
+
if (duplicateHistories.length > 0) {
|
|
3302
|
+
throw new Error(
|
|
3303
|
+
`memory improvement train/holdout histories duplicate content: ${duplicateHistories.join(", ")}`
|
|
3304
|
+
);
|
|
3305
|
+
}
|
|
3306
|
+
if (typeof options.improvementRef !== "string" || !options.improvementRef.trim()) {
|
|
3307
|
+
throw new Error("memory improvement improvementRef must be a non-empty string");
|
|
3308
|
+
}
|
|
3309
|
+
assertMemoryImprovementOptions(options);
|
|
3310
|
+
const storage = options.storage ?? fsCampaignStorage2();
|
|
3311
|
+
const runDir = resolveRunDir2(options.runDir, options.repo);
|
|
3312
|
+
storage.ensureDir(runDir);
|
|
3313
|
+
const lease = await acquireAgentMemoryRunLease({
|
|
3314
|
+
experimentId: options.experimentId,
|
|
3315
|
+
runDir,
|
|
3316
|
+
storage,
|
|
3317
|
+
customStorage: options.storage !== void 0,
|
|
3318
|
+
lockFileName: "memory-improvement.lock",
|
|
3319
|
+
label: "memory improvement",
|
|
3320
|
+
controllerMode: options.controllerMode,
|
|
3321
|
+
acquireRunLease: options.acquireRunLease
|
|
3322
|
+
});
|
|
3323
|
+
let result;
|
|
3324
|
+
let primaryError;
|
|
3325
|
+
try {
|
|
3326
|
+
result = await runAgentMemoryImprovementOwned(options, storage, runDir, lease);
|
|
3327
|
+
} catch (error) {
|
|
3328
|
+
primaryError = error;
|
|
3329
|
+
}
|
|
3330
|
+
let releaseError;
|
|
3331
|
+
try {
|
|
3332
|
+
await lease.release();
|
|
3333
|
+
} catch (error) {
|
|
3334
|
+
releaseError = error;
|
|
3335
|
+
}
|
|
3336
|
+
if (primaryError && releaseError) {
|
|
3337
|
+
throw new AggregateError(
|
|
3338
|
+
[primaryError, releaseError],
|
|
3339
|
+
"memory improvement failed and its controller lease could not be released"
|
|
3340
|
+
);
|
|
3341
|
+
}
|
|
3342
|
+
if (primaryError) throw primaryError;
|
|
3343
|
+
if (releaseError) throw releaseError;
|
|
3344
|
+
if (!result) throw new Error("memory improvement produced no result");
|
|
3345
|
+
return result;
|
|
3346
|
+
}
|
|
3347
|
+
async function runAgentMemoryImprovementOwned(options, storage, runDir, lease) {
|
|
3348
|
+
await lease.assertOwned();
|
|
3349
|
+
const serializeRaw = options.serializeConfig ?? ((config) => canonicalJson7(config));
|
|
3350
|
+
const parseRaw = options.parseConfig ?? ((surface) => JSON.parse(surface));
|
|
3351
|
+
const serialize = (config) => serializeMemoryConfig(serializeRaw, config);
|
|
3352
|
+
const parse = (surface) => parseMemoryConfig(parseRaw, surface);
|
|
3353
|
+
for (const seed of options.seeds) assertMemoryConfigRoundTrip(seed.config, serialize, parse);
|
|
3354
|
+
if (options.activation && !storage.append) {
|
|
3355
|
+
throw new Error("memory activation requires CampaignStorage.append");
|
|
3356
|
+
}
|
|
3357
|
+
if (options.resumable !== false && !options.lineageStore && !storage.append) {
|
|
3358
|
+
throw new Error("resumable memory improvement requires CampaignStorage.append");
|
|
3359
|
+
}
|
|
3360
|
+
assertMemoryImprovementIdentity(options, storage, runDir, serialize);
|
|
3361
|
+
const costLedger = createRunCostLedger2({
|
|
3362
|
+
storage,
|
|
3363
|
+
runDir,
|
|
3364
|
+
costCeilingUsd: options.maxTotalCostUsd ?? 0
|
|
3365
|
+
});
|
|
3366
|
+
reconcileInterruptedRunPaidCalls(costLedger, "memory improvement run");
|
|
3367
|
+
assertNoInterruptedPaidCalls(costLedger, "memory improvement recovery");
|
|
3368
|
+
const trainScenarios = options.trainSequences.map((sequence) => ({
|
|
3369
|
+
id: sequence.id,
|
|
3370
|
+
kind: "agent-memory-config-search",
|
|
3371
|
+
sequenceId: sequence.id
|
|
3372
|
+
}));
|
|
3373
|
+
const evaluations = /* @__PURE__ */ new Map();
|
|
3374
|
+
const evaluateSurface = async (surface) => {
|
|
3375
|
+
await lease.assertOwned();
|
|
3376
|
+
const text = requireStringSurface(surface);
|
|
3377
|
+
const config = parse(text);
|
|
3378
|
+
const canonicalSurface = serialize(config);
|
|
3379
|
+
const hash = surfaceHash2(canonicalSurface);
|
|
3380
|
+
let pending = evaluations.get(hash);
|
|
3381
|
+
if (!pending) {
|
|
3382
|
+
pending = (async () => {
|
|
3383
|
+
const candidate = await buildCandidate(options, config, hash, `search-${hash}`);
|
|
3384
|
+
await lease.assertOwned();
|
|
3385
|
+
const experiment = await runAgentMemoryExperiment({
|
|
3386
|
+
...experimentOptions(options, costLedger, storage, lease),
|
|
3387
|
+
experimentId: `${options.experimentId}:search:${hash}`,
|
|
3388
|
+
sequences: options.trainSequences,
|
|
3389
|
+
candidates: [candidate],
|
|
3390
|
+
runDir: join3(runDir, "search", hash),
|
|
3391
|
+
costPhase: `memory.search.${hash}`
|
|
3392
|
+
});
|
|
3393
|
+
await lease.assertOwned();
|
|
3394
|
+
return experiment;
|
|
3395
|
+
})();
|
|
3396
|
+
evaluations.set(hash, pending);
|
|
3397
|
+
void pending.catch(() => evaluations.delete(hash));
|
|
3398
|
+
}
|
|
3399
|
+
const result2 = await pending;
|
|
3400
|
+
await lease.assertOwned();
|
|
3401
|
+
const row = result2.rows[0];
|
|
3402
|
+
if (!row || row.cellsFailed > 0) {
|
|
3403
|
+
throw new Error(`${hash}: memory candidate did not complete every training cell`);
|
|
3550
3404
|
}
|
|
3405
|
+
return {
|
|
3406
|
+
score: row.scoreMean,
|
|
3407
|
+
scoreVector: sequenceScores(result2, options.trainSequences, row.candidateId)
|
|
3408
|
+
};
|
|
3409
|
+
};
|
|
3410
|
+
const lineageStore = options.lineageStore ?? (options.resumable === false ? memLineageStore() : campaignLineageStore(storage, join3(runDir, "lineage.jsonl")));
|
|
3411
|
+
const lineageOptions = {
|
|
3412
|
+
seeds: options.seeds.map((seed) => ({
|
|
3413
|
+
surface: serialize(seed.config),
|
|
3414
|
+
track: seed.track,
|
|
3415
|
+
proposer: seed.proposer,
|
|
3416
|
+
...seed.vision !== void 0 ? { vision: seed.vision } : {}
|
|
3417
|
+
})),
|
|
3418
|
+
scenarios: trainScenarios,
|
|
3419
|
+
proposer: withCostContext(options.proposer, costLedger, lease, "default"),
|
|
3420
|
+
proposers: options.proposers ? Object.fromEntries(
|
|
3421
|
+
Object.entries(options.proposers).map(([name, proposer]) => [
|
|
3422
|
+
name,
|
|
3423
|
+
withCostContext(proposer, costLedger, lease, name)
|
|
3424
|
+
])
|
|
3425
|
+
) : void 0,
|
|
3426
|
+
scoreSurface: evaluateSurface,
|
|
3427
|
+
governor: options.governor ? withGovernorCostContext(options.governor, costLedger, lease) : void 0,
|
|
3428
|
+
budget: {
|
|
3429
|
+
...options.budget,
|
|
3430
|
+
maxNodes: options.seeds.length + options.budget.maxSteps
|
|
3431
|
+
},
|
|
3432
|
+
store: lineageStore,
|
|
3433
|
+
populationSize: options.populationSize,
|
|
3434
|
+
candidateConcurrency: options.candidateConcurrency
|
|
3435
|
+
};
|
|
3436
|
+
const search = await runLineageLoop(lineageOptions);
|
|
3437
|
+
await lease.assertOwned();
|
|
3438
|
+
const best = search.best;
|
|
3439
|
+
if (!best) throw new Error("memory improvement produced no measured config");
|
|
3440
|
+
const baselineSurface = serialize(options.seeds[0].config);
|
|
3441
|
+
const baselineHash = surfaceHash2(baselineSurface);
|
|
3442
|
+
const winnerConfig = parse(requireStringSurface(best.surface));
|
|
3443
|
+
const winnerSurface = serialize(winnerConfig);
|
|
3444
|
+
const winnerHash = surfaceHash2(winnerSurface);
|
|
3445
|
+
const baselineConfig = parse(baselineSurface);
|
|
3446
|
+
let holdout;
|
|
3447
|
+
let decision;
|
|
3448
|
+
if (winnerHash === baselineHash) {
|
|
3449
|
+
const baselineMeasurement = await evaluateSurface(baselineSurface);
|
|
3450
|
+
decision = {
|
|
3451
|
+
status: "no-change",
|
|
3452
|
+
reasons: ["search did not find a config better than the baseline"],
|
|
3453
|
+
baselineScore: baselineMeasurement.score,
|
|
3454
|
+
winnerScore: baselineMeasurement.score,
|
|
3455
|
+
lift: 0,
|
|
3456
|
+
criticalDimensions: []
|
|
3457
|
+
};
|
|
3458
|
+
} else {
|
|
3459
|
+
await lease.assertOwned();
|
|
3460
|
+
const [baselineCandidate, winnerCandidate] = await Promise.all([
|
|
3461
|
+
buildCandidate(options, baselineConfig, baselineHash, "baseline"),
|
|
3462
|
+
buildCandidate(options, winnerConfig, winnerHash, "winner")
|
|
3463
|
+
]);
|
|
3464
|
+
await lease.assertOwned();
|
|
3465
|
+
holdout = await runAgentMemoryExperiment({
|
|
3466
|
+
...experimentOptions(options, costLedger, storage, lease),
|
|
3467
|
+
experimentId: `${options.experimentId}:holdout`,
|
|
3468
|
+
sequences: options.holdoutSequences,
|
|
3469
|
+
candidates: [baselineCandidate, winnerCandidate],
|
|
3470
|
+
runDir: join3(runDir, "holdout"),
|
|
3471
|
+
costPhase: "memory.holdout"
|
|
3472
|
+
});
|
|
3473
|
+
decision = decidePromotion({
|
|
3474
|
+
options,
|
|
3475
|
+
result: holdout,
|
|
3476
|
+
baselineId: baselineCandidate.id,
|
|
3477
|
+
winnerId: winnerCandidate.id
|
|
3478
|
+
});
|
|
3551
3479
|
}
|
|
3552
|
-
const
|
|
3553
|
-
|
|
3554
|
-
|
|
3555
|
-
|
|
3556
|
-
|
|
3557
|
-
|
|
3558
|
-
|
|
3559
|
-
|
|
3560
|
-
|
|
3561
|
-
|
|
3562
|
-
|
|
3563
|
-
|
|
3564
|
-
|
|
3565
|
-
|
|
3566
|
-
|
|
3567
|
-
|
|
3480
|
+
const resultJsonPath = join3(runDir, "memory-improvement-result.json");
|
|
3481
|
+
const activationRef = options.activation?.ref ?? "not-configured";
|
|
3482
|
+
const activationId = `memory-activation-${surfaceHash2(
|
|
3483
|
+
canonicalJson7({
|
|
3484
|
+
experimentId: options.experimentId,
|
|
3485
|
+
improvementRef: options.improvementRef,
|
|
3486
|
+
activationRef,
|
|
3487
|
+
baselineSurfaceHash: baselineHash,
|
|
3488
|
+
winnerSurfaceHash: winnerHash,
|
|
3489
|
+
holdoutManifestHash: holdout?.campaign.manifestHash ?? null,
|
|
3490
|
+
promotionPolicy: normalizedPromotionPolicy(options)
|
|
3491
|
+
})
|
|
3492
|
+
)}`;
|
|
3493
|
+
const activationJournalDir = join3(runDir, "activations");
|
|
3494
|
+
const activationJournalPath = join3(activationJournalDir, `${activationId}.jsonl`);
|
|
3495
|
+
const activationEligible = decision.status === "promote" && holdout !== void 0;
|
|
3496
|
+
const activationEventIdentity = holdout ? {
|
|
3497
|
+
schema: 1,
|
|
3498
|
+
activationId,
|
|
3499
|
+
experimentId: options.experimentId,
|
|
3500
|
+
activationRef,
|
|
3501
|
+
baselineSurfaceHash: baselineHash,
|
|
3502
|
+
winnerSurfaceHash: winnerHash,
|
|
3503
|
+
holdoutManifestHash: holdout.campaign.manifestHash
|
|
3504
|
+
} : void 0;
|
|
3505
|
+
const activationJournal = activationEligible && activationEventIdentity ? readMemoryActivationJournal(storage, activationJournalPath, activationEventIdentity) : { prepared: false };
|
|
3506
|
+
const activation = {
|
|
3507
|
+
id: activationId,
|
|
3508
|
+
status: !activationEligible ? "not-eligible" : activationJournal.activated ? "already-activated" : options.activation ? "pending" : "not-configured",
|
|
3509
|
+
journalPath: activationJournalPath
|
|
3510
|
+
};
|
|
3511
|
+
const result = {
|
|
3512
|
+
lineage: search.lineage,
|
|
3513
|
+
baselineConfig,
|
|
3514
|
+
winnerConfig,
|
|
3515
|
+
baselineSurface,
|
|
3516
|
+
winnerSurface,
|
|
3517
|
+
baselineSurfaceHash: baselineHash,
|
|
3518
|
+
winnerSurfaceHash: winnerHash,
|
|
3519
|
+
decision,
|
|
3520
|
+
activation,
|
|
3521
|
+
...holdout ? { holdout } : {},
|
|
3522
|
+
totalCostUsd: costLedger.summary().totalCostUsd,
|
|
3523
|
+
resultJsonPath
|
|
3524
|
+
};
|
|
3525
|
+
await lease.assertOwned();
|
|
3526
|
+
writeMemoryImprovementResult(storage, options.experimentId, result);
|
|
3527
|
+
if (activationEligible && !activationJournal.activated && activationEventIdentity && holdout && options.activation) {
|
|
3528
|
+
const activationDriver = options.activation;
|
|
3529
|
+
const activationTimeoutMs = options.activationTimeoutMs ?? 6e4;
|
|
3530
|
+
const hadPreparedEvent = activationJournal.prepared;
|
|
3531
|
+
if (!hadPreparedEvent) {
|
|
3532
|
+
await lease.assertOwned();
|
|
3533
|
+
storage.ensureDir(activationJournalDir);
|
|
3534
|
+
appendMemoryActivationEvent(storage, activationJournalPath, {
|
|
3535
|
+
...activationEventIdentity,
|
|
3536
|
+
status: "prepared",
|
|
3537
|
+
recordedAt: (options.now ?? (() => /* @__PURE__ */ new Date()))().toISOString()
|
|
3538
|
+
});
|
|
3539
|
+
}
|
|
3540
|
+
await lease.assertOwned();
|
|
3541
|
+
const currentConfig = await runBoundedMemoryLifecycle({
|
|
3542
|
+
operation: `${activationDriver.ref}: read current memory configuration`,
|
|
3543
|
+
timeoutMs: activationTimeoutMs,
|
|
3544
|
+
run: () => activationDriver.readCurrent()
|
|
3545
|
+
});
|
|
3546
|
+
await lease.assertOwned();
|
|
3547
|
+
const currentHash = surfaceHash2(serialize(currentConfig));
|
|
3548
|
+
if (currentHash !== baselineHash && currentHash !== winnerHash) {
|
|
3568
3549
|
throw new Error(
|
|
3569
|
-
`memory
|
|
3550
|
+
`memory activation target '${activationDriver.ref}' changed concurrently; expected '${baselineHash}' or '${winnerHash}', found '${currentHash}'`
|
|
3570
3551
|
);
|
|
3571
3552
|
}
|
|
3572
|
-
|
|
3553
|
+
let outcome;
|
|
3554
|
+
if (currentHash === winnerHash) {
|
|
3555
|
+
outcome = hadPreparedEvent ? "recovered" : "already-current";
|
|
3556
|
+
activation.status = "recovered";
|
|
3557
|
+
} else {
|
|
3558
|
+
let compareError;
|
|
3559
|
+
try {
|
|
3560
|
+
await runBoundedMemoryLifecycle({
|
|
3561
|
+
operation: `${activationDriver.ref}: activate memory configuration`,
|
|
3562
|
+
timeoutMs: activationTimeoutMs,
|
|
3563
|
+
run: () => activationDriver.compareAndSet({
|
|
3564
|
+
activationId,
|
|
3565
|
+
expectedConfig: baselineConfig,
|
|
3566
|
+
expectedSurfaceHash: baselineHash,
|
|
3567
|
+
config: winnerConfig,
|
|
3568
|
+
surfaceHash: winnerHash,
|
|
3569
|
+
decision,
|
|
3570
|
+
lineage: search.lineage,
|
|
3571
|
+
holdout
|
|
3572
|
+
})
|
|
3573
|
+
});
|
|
3574
|
+
} catch (error) {
|
|
3575
|
+
compareError = error;
|
|
3576
|
+
}
|
|
3577
|
+
await lease.assertOwned();
|
|
3578
|
+
let observedConfig;
|
|
3579
|
+
try {
|
|
3580
|
+
observedConfig = await runBoundedMemoryLifecycle({
|
|
3581
|
+
operation: `${activationDriver.ref}: confirm memory configuration`,
|
|
3582
|
+
timeoutMs: activationTimeoutMs,
|
|
3583
|
+
run: () => activationDriver.readCurrent()
|
|
3584
|
+
});
|
|
3585
|
+
} catch (error) {
|
|
3586
|
+
if (compareError) {
|
|
3587
|
+
throw new AggregateError(
|
|
3588
|
+
[compareError, error],
|
|
3589
|
+
`memory activation '${activationId}' failed and its live state could not be confirmed`
|
|
3590
|
+
);
|
|
3591
|
+
}
|
|
3592
|
+
throw error;
|
|
3593
|
+
}
|
|
3594
|
+
await lease.assertOwned();
|
|
3595
|
+
const observedHash = surfaceHash2(serialize(observedConfig));
|
|
3596
|
+
if (observedHash !== winnerHash) {
|
|
3597
|
+
const mismatch = new Error(
|
|
3598
|
+
`memory activation '${activationId}' did not install the measured winner; found '${observedHash}'`
|
|
3599
|
+
);
|
|
3600
|
+
if (compareError) {
|
|
3601
|
+
throw new AggregateError(
|
|
3602
|
+
[compareError, mismatch],
|
|
3603
|
+
`memory activation '${activationId}' failed without applying the measured winner`
|
|
3604
|
+
);
|
|
3605
|
+
}
|
|
3606
|
+
throw mismatch;
|
|
3607
|
+
}
|
|
3608
|
+
outcome = compareError ? "recovered" : "applied";
|
|
3609
|
+
activation.status = compareError ? "recovered" : "activated";
|
|
3610
|
+
}
|
|
3611
|
+
await lease.assertOwned();
|
|
3612
|
+
appendMemoryActivationEvent(storage, activationJournalPath, {
|
|
3613
|
+
...activationEventIdentity,
|
|
3614
|
+
status: "activated",
|
|
3615
|
+
outcome,
|
|
3616
|
+
recordedAt: (options.now ?? (() => /* @__PURE__ */ new Date()))().toISOString()
|
|
3617
|
+
});
|
|
3618
|
+
writeMemoryImprovementResult(storage, options.experimentId, result);
|
|
3573
3619
|
}
|
|
3574
|
-
|
|
3575
|
-
function mean2(values) {
|
|
3576
|
-
return values.length === 0 ? 0 : values.reduce((sum, value) => sum + value, 0) / values.length;
|
|
3620
|
+
return result;
|
|
3577
3621
|
}
|
|
3578
3622
|
|
|
3579
3623
|
// src/memory/mem0.ts
|
|
3580
3624
|
import { randomUUID as randomUUID4 } from "crypto";
|
|
3581
|
-
import { canonicalJson as
|
|
3625
|
+
import { canonicalJson as canonicalJson8 } from "@tangle-network/agent-eval";
|
|
3582
3626
|
function createMem0MemoryAdapter(options) {
|
|
3583
3627
|
assertMem0Options(options);
|
|
3584
3628
|
const id = options.id ?? `mem0-${options.mode}`;
|
|
@@ -3894,7 +3938,7 @@ function normalizeMem0Hits(raw, adapterId, options) {
|
|
|
3894
3938
|
if (options.minScore !== void 0 && (score === void 0 || score < options.minScore)) {
|
|
3895
3939
|
return void 0;
|
|
3896
3940
|
}
|
|
3897
|
-
const memoryId = stringField2(item, ["id"]) ?? stableId("mem",
|
|
3941
|
+
const memoryId = stringField2(item, ["id"]) ?? stableId("mem", canonicalJson8(compactRecord2({ adapterId, text, metadata, item })));
|
|
3898
3942
|
const normalizedScore = score !== void 0 && score >= 0 && score <= 1 ? score : void 0;
|
|
3899
3943
|
return {
|
|
3900
3944
|
id: memoryId,
|
|
@@ -4093,11 +4137,11 @@ function mem0MemoryAdapterIdentity(options) {
|
|
|
4093
4137
|
if (typeof options.backendRef !== "string" || !options.backendRef.trim()) {
|
|
4094
4138
|
throw new Error("Mem0 backendRef must be a non-empty string");
|
|
4095
4139
|
}
|
|
4096
|
-
return stableId("mem0",
|
|
4140
|
+
return stableId("mem0", canonicalJson8(compactRecord2(options)));
|
|
4097
4141
|
}
|
|
4098
4142
|
|
|
4099
4143
|
// src/memory/neo4j.ts
|
|
4100
|
-
import { canonicalJson as
|
|
4144
|
+
import { canonicalJson as canonicalJson9 } from "@tangle-network/agent-eval";
|
|
4101
4145
|
function createNeo4jAgentMemoryAdapter(options) {
|
|
4102
4146
|
assertNeo4jOptions(options);
|
|
4103
4147
|
const client = options.client;
|
|
@@ -4362,7 +4406,7 @@ async function writeNeo4jMemory(client, input, adapterId, transport, isolatedIns
|
|
|
4362
4406
|
assertNeo4jWriteSucceeded(result, adapterId, input.kind);
|
|
4363
4407
|
const id = idFromResult(result) ?? input.id ?? stableId(
|
|
4364
4408
|
"mem",
|
|
4365
|
-
|
|
4409
|
+
canonicalJson9({ adapterId, kind: input.kind, text: input.text, scope: input.scope ?? {} })
|
|
4366
4410
|
);
|
|
4367
4411
|
return {
|
|
4368
4412
|
accepted: true,
|
|
@@ -4723,9 +4767,9 @@ export {
|
|
|
4723
4767
|
createAgentMemoryBranch,
|
|
4724
4768
|
forkAgentMemoryBranchSnapshot,
|
|
4725
4769
|
buildAgentMemorySequencesFromBenchmarkCases,
|
|
4726
|
-
runAgentMemoryExperiment,
|
|
4727
4770
|
buildAgentMemorySequenceScenarios,
|
|
4728
4771
|
agentMemorySequenceJudge,
|
|
4772
|
+
runAgentMemoryExperiment,
|
|
4729
4773
|
createGraphitiMemoryAdapter,
|
|
4730
4774
|
graphitiMemoryAdapterIdentity,
|
|
4731
4775
|
runAgentMemoryImprovement,
|
|
@@ -4737,4 +4781,4 @@ export {
|
|
|
4737
4781
|
AgentMemoryHitSchema,
|
|
4738
4782
|
AgentMemoryWriteInputSchema
|
|
4739
4783
|
};
|
|
4740
|
-
//# sourceMappingURL=chunk-
|
|
4784
|
+
//# sourceMappingURL=chunk-3CAAGJCE.js.map
|