@tangle-network/agent-knowledge 7.0.0 → 7.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -1
- package/README.md +30 -0
- package/dist/benchmarks/index.js +1 -1
- package/dist/{benchmarks-C8L7HJb4.js → benchmarks-CPl4AO1k.js} +73 -5
- package/dist/benchmarks-CPl4AO1k.js.map +1 -0
- package/dist/{index-Dwp3Mx-w.d.ts → index-BEVmwlfJ.d.ts} +184 -5
- package/dist/index-BEVmwlfJ.d.ts.map +1 -0
- package/dist/index.d.ts +2 -2
- package/dist/index.js +3 -3
- package/dist/memory/index.d.ts +2 -2
- package/dist/memory/index.js +3 -3
- package/dist/{memory-CIYRB_Q8.js → memory-DAFfGTJH.js} +1092 -193
- package/dist/memory-DAFfGTJH.js.map +1 -0
- package/docs/architecture.md +6 -4
- package/package.json +3 -3
- package/dist/benchmarks-C8L7HJb4.js.map +0 -1
- package/dist/index-Dwp3Mx-w.d.ts.map +0 -1
- package/dist/memory-CIYRB_Q8.js.map +0 -1
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { r as stableId, t as sha256 } from "./ids-Bevz_pXV.js";
|
|
2
|
-
import {
|
|
2
|
+
import { A as reserveRecoveryAttempts, C as sleepForMemoryRecovery, D as hasSettledPaidCall, E as assertNoInterruptedPaidCalls, H as memoryWriteResultToSourceRecord, O as readActiveAttemptJournal, S as runBoundedMemoryLifecycle, T as appendDurableJournalEvent, V as memoryHitToSourceRecord, _ as MEMORY_OPERATION_CANCELLATION_TIMEOUT_MS, b as memoryRecoveryDelayMs, f as acquireAgentMemoryRunLease, g as MEMORY_CAMPAIGN_DISPATCH_SHUTDOWN_TIMEOUT_MS, k as reconcileInterruptedMemoryPaidCalls, u as scoreMemoryBenchmarkArtifact, v as createBoundedMemoryAdapter, w as appendAttemptJournalEvent, x as resolveMemoryCleanupTimeoutMs, y as createMemoryExecutionPool } from "./benchmarks-CPl4AO1k.js";
|
|
3
3
|
import { randomUUID } from "node:crypto";
|
|
4
4
|
import { dirname, join } from "node:path";
|
|
5
|
-
import { canonicalJson, mulberry32 } from "@tangle-network/agent-eval";
|
|
5
|
+
import { canonicalJson, comparePairedArms, mulberry32, pairArms } from "@tangle-network/agent-eval";
|
|
6
6
|
import { z } from "zod";
|
|
7
|
-
import { compareOptimizationMethods, createRunCostLedger, fsCampaignStorage, heldoutSignificance, resolveRunDir, runCampaign, surfaceHash } from "@tangle-network/agent-eval/campaign";
|
|
7
|
+
import { assertCampaignSplitIdentity, canonicalDigest, compareOptimizationMethods, createRunCostLedger, fsCampaignStorage, heldoutSignificance, resolveRunDir, runCampaign, surfaceHash } from "@tangle-network/agent-eval/campaign";
|
|
8
8
|
//#region src/immutable-ref.ts
|
|
9
9
|
const SHA256_REF = /^sha256:[a-f0-9]{64}$/;
|
|
10
10
|
const GIT_REF = /^git:[a-f0-9]{40}$/;
|
|
@@ -1035,7 +1035,8 @@ function assertMemorySequences(sequences) {
|
|
|
1035
1035
|
if (sequence.steps.length === 0) throw new Error(`memory experiment sequence ${sequence.id} has no steps`);
|
|
1036
1036
|
assertUnique(sequence.steps.map((step) => step.id), `step in sequence ${sequence.id}`);
|
|
1037
1037
|
let probeCount = 0;
|
|
1038
|
-
|
|
1038
|
+
const retentionMeasurements = /* @__PURE__ */ new Map();
|
|
1039
|
+
for (const [stepOrdinal, step] of sequence.steps.entries()) {
|
|
1039
1040
|
for (const write of step.writes ?? []) {
|
|
1040
1041
|
assertNonEmptyString(write.text, `memory experiment write in ${sequence.id}/${step.id}`);
|
|
1041
1042
|
if (write.id !== void 0) assertNonEmptyString(write.id, `memory experiment write id in ${sequence.id}/${step.id}`);
|
|
@@ -1050,12 +1051,69 @@ function assertMemorySequences(sequences) {
|
|
|
1050
1051
|
assertStringList(probe.expectedEventIds, `${sequence.id}/${step.id}/${probe.id} event ids`);
|
|
1051
1052
|
assertStringList(probe.expectedActorIds, `${sequence.id}/${step.id}/${probe.id} actor ids`);
|
|
1052
1053
|
if (probe.referenceAnswer !== void 0) assertNonEmptyString(probe.referenceAnswer, `memory experiment reference answer ${sequence.id}/${step.id}/${probe.id}`);
|
|
1054
|
+
if (probe.retentionKey !== void 0) {
|
|
1055
|
+
assertNonEmptyString(probe.retentionKey, `memory experiment retention key ${sequence.id}/${step.id}/${probe.id}`);
|
|
1056
|
+
if (probe.retentionKey !== probe.retentionKey.trim()) throw new Error(`memory experiment retention key ${sequence.id}/${step.id}/${probe.id} must not have surrounding whitespace`);
|
|
1057
|
+
const definition = retentionMeasurementDefinition(step.scope, probe);
|
|
1058
|
+
const prior = retentionMeasurements.get(probe.retentionKey);
|
|
1059
|
+
if (prior?.stepOrdinals.has(stepOrdinal)) throw new Error(`memory experiment retention key ${sequence.id}/${probe.retentionKey} may appear only once per step`);
|
|
1060
|
+
if (prior && prior.definition !== definition) throw new Error(`memory experiment retention key ${sequence.id}/${probe.retentionKey} must repeat the exact same measurement`);
|
|
1061
|
+
const measurement = prior ?? {
|
|
1062
|
+
definition,
|
|
1063
|
+
stepOrdinals: /* @__PURE__ */ new Set()
|
|
1064
|
+
};
|
|
1065
|
+
measurement.stepOrdinals.add(stepOrdinal);
|
|
1066
|
+
retentionMeasurements.set(probe.retentionKey, measurement);
|
|
1067
|
+
}
|
|
1068
|
+
if (probe.transferKey !== void 0) {
|
|
1069
|
+
assertNonEmptyString(probe.transferKey, `memory experiment transfer key ${sequence.id}/${step.id}/${probe.id}`);
|
|
1070
|
+
if (probe.transferKey !== probe.transferKey.trim()) throw new Error(`memory experiment transfer key ${sequence.id}/${step.id}/${probe.id} must not have surrounding whitespace`);
|
|
1071
|
+
if (probe.retentionKey !== void 0) throw new Error(`memory experiment probe ${sequence.id}/${step.id}/${probe.id} cannot be both transfer and retention`);
|
|
1072
|
+
if (stepOrdinal === 0) throw new Error(`memory experiment transfer probe ${sequence.id}/${step.id}/${probe.id} must run after the first step`);
|
|
1073
|
+
}
|
|
1053
1074
|
if (!((probe.requiredFacts?.length ?? 0) > 0 || (probe.forbiddenFacts?.length ?? 0) > 0 || (probe.expectedEventIds?.length ?? 0) > 0 || (probe.expectedActorIds?.length ?? 0) > 0 || Boolean(probe.referenceAnswer?.trim()))) throw new Error(`memory experiment probe ${sequence.id}/${step.id}/${probe.id} has no measurable target`);
|
|
1054
1075
|
}
|
|
1055
1076
|
}
|
|
1056
1077
|
if (probeCount === 0) throw new Error(`memory experiment sequence ${sequence.id} has no probes`);
|
|
1078
|
+
for (const [retentionKey, measurement] of retentionMeasurements) if (measurement.stepOrdinals.size < 2) throw new Error(`memory experiment retention key ${sequence.id}/${retentionKey} must appear in at least two distinct steps`);
|
|
1079
|
+
}
|
|
1080
|
+
}
|
|
1081
|
+
function assertMemoryLearningSequences(sequences) {
|
|
1082
|
+
for (const sequence of sequences) {
|
|
1083
|
+
if (sequence.steps.length < 2) throw new Error(`memory learning sequence ${sequence.id} requires at least two ordered steps`);
|
|
1084
|
+
if (!sequence.steps.slice(1).some((step) => (step.probes?.length ?? 0) > 0)) throw new Error(`memory learning sequence ${sequence.id} requires a probe after the first step`);
|
|
1057
1085
|
}
|
|
1058
1086
|
}
|
|
1087
|
+
function retentionMeasurementDefinition(stepScope, probe) {
|
|
1088
|
+
return canonicalJson({
|
|
1089
|
+
query: probe.query,
|
|
1090
|
+
scope: compactScope({
|
|
1091
|
+
...stepScope ?? {},
|
|
1092
|
+
...probe.scope ?? {},
|
|
1093
|
+
tags: {
|
|
1094
|
+
...stepScope?.tags ?? {},
|
|
1095
|
+
...probe.scope?.tags ?? {}
|
|
1096
|
+
}
|
|
1097
|
+
}),
|
|
1098
|
+
limit: probe.limit ?? null,
|
|
1099
|
+
taskKind: probe.taskKind ?? "memory-recall",
|
|
1100
|
+
requiredFacts: normalizeFactMatchers(probe.requiredFacts ?? []),
|
|
1101
|
+
forbiddenFacts: normalizeFactMatchers(probe.forbiddenFacts ?? []),
|
|
1102
|
+
expectedEventIds: [...probe.expectedEventIds ?? []],
|
|
1103
|
+
expectedActorIds: [...probe.expectedActorIds ?? []],
|
|
1104
|
+
referenceAnswer: probe.referenceAnswer ?? null
|
|
1105
|
+
});
|
|
1106
|
+
}
|
|
1107
|
+
function normalizeFactMatchers(matchers) {
|
|
1108
|
+
return matchers.map((matcher) => ({
|
|
1109
|
+
id: matcher.id,
|
|
1110
|
+
anyOf: [...matcher.anyOf],
|
|
1111
|
+
weight: matcher.weight ?? 1,
|
|
1112
|
+
sourceEventIds: [...matcher.sourceEventIds ?? []],
|
|
1113
|
+
validAt: matcher.validAt ?? null,
|
|
1114
|
+
obsolete: matcher.obsolete ?? false
|
|
1115
|
+
}));
|
|
1116
|
+
}
|
|
1059
1117
|
function normalizeCleanupScope(scope) {
|
|
1060
1118
|
const normalized = compactScope(scope);
|
|
1061
1119
|
if (normalized.tags && Object.keys(normalized.tags).length === 0) delete normalized.tags;
|
|
@@ -1119,7 +1177,7 @@ function buildAgentMemorySequencesFromBenchmarkCases(cases, options = {}) {
|
|
|
1119
1177
|
eventIndex
|
|
1120
1178
|
}) ?? {
|
|
1121
1179
|
agentId: memoryAgentId,
|
|
1122
|
-
sessionId:
|
|
1180
|
+
sessionId: "benchmark-session"
|
|
1123
1181
|
}),
|
|
1124
1182
|
writes: [{
|
|
1125
1183
|
id: event.id,
|
|
@@ -1138,7 +1196,7 @@ function buildAgentMemorySequencesFromBenchmarkCases(cases, options = {}) {
|
|
|
1138
1196
|
id: "probe",
|
|
1139
1197
|
scope: compactScope(options.probeScope?.(testCase) ?? {
|
|
1140
1198
|
agentId: memoryAgentId,
|
|
1141
|
-
sessionId:
|
|
1199
|
+
sessionId: "benchmark-session"
|
|
1142
1200
|
}),
|
|
1143
1201
|
probes: [{
|
|
1144
1202
|
id: "answer",
|
|
@@ -1212,6 +1270,759 @@ function agentMemorySequenceJudge() {
|
|
|
1212
1270
|
};
|
|
1213
1271
|
}
|
|
1214
1272
|
//#endregion
|
|
1273
|
+
//#region src/memory/experiment/metrics.ts
|
|
1274
|
+
function rankAgentMemoryExperiment(candidates, scenarios, campaign, costByCandidate) {
|
|
1275
|
+
const scenarioById = new Map(scenarios.map((scenario) => [scenario.id, scenario]));
|
|
1276
|
+
return candidates.map((candidate) => {
|
|
1277
|
+
const candidateScenarios = scenarios.filter((scenario) => scenario.candidateId === candidate.id);
|
|
1278
|
+
const cells = campaign.cells.filter((cell) => scenarioById.get(cell.scenarioId)?.candidateId === candidate.id);
|
|
1279
|
+
const successful = cells.filter((cell) => !cell.error && cell.artifact);
|
|
1280
|
+
const dimensionRows = successful.map((cell) => cell.artifact.dimensions);
|
|
1281
|
+
return {
|
|
1282
|
+
rank: 0,
|
|
1283
|
+
candidateId: candidate.id,
|
|
1284
|
+
label: candidate.label ?? candidate.id,
|
|
1285
|
+
scoreMean: mean$1(cells.map((cell) => !cell.error && cell.artifact ? cell.artifact.score : 0)),
|
|
1286
|
+
passRate: mean$1(cells.map((cell) => !cell.error && cell.artifact?.passed ? 1 : 0)),
|
|
1287
|
+
totalSequences: candidateScenarios.length,
|
|
1288
|
+
totalCells: cells.length,
|
|
1289
|
+
totalProbes: successful.reduce((sum, cell) => sum + cell.artifact.probes.length, 0),
|
|
1290
|
+
cellsFailed: cells.filter((cell) => Boolean(cell.error)).length,
|
|
1291
|
+
totalCostUsd: normalizeUsd(costByCandidate.get(candidate.id) ?? 0),
|
|
1292
|
+
durationMs: cells.reduce((sum, cell) => sum + cell.durationMs, 0),
|
|
1293
|
+
dimensions: meanDimensions(dimensionRows)
|
|
1294
|
+
};
|
|
1295
|
+
}).sort((a, b) => Number(a.cellsFailed > 0) - Number(b.cellsFailed > 0) || b.scoreMean - a.scoreMean || b.passRate - a.passRate || a.totalCostUsd - b.totalCostUsd || a.candidateId.localeCompare(b.candidateId)).map((row, index) => ({
|
|
1296
|
+
...row,
|
|
1297
|
+
rank: index + 1
|
|
1298
|
+
}));
|
|
1299
|
+
}
|
|
1300
|
+
function memoryExperimentCostByCandidate(costLedger, runDir, scenarios, candidateIdsInput) {
|
|
1301
|
+
const candidateByScenario = new Map(scenarios.map((scenario) => [scenario.id, scenario.candidateId]));
|
|
1302
|
+
const candidateIds = new Set(candidateIdsInput);
|
|
1303
|
+
const sequenceIds = new Set(scenarios.map((scenario) => scenario.sequenceId));
|
|
1304
|
+
for (const candidateId of candidateIds) for (const sequenceId of sequenceIds) candidateByScenario.set(`${stableId("candidate", candidateId)}:${sequenceId}`, candidateId);
|
|
1305
|
+
const totals = /* @__PURE__ */ new Map();
|
|
1306
|
+
for (const receipt of costLedger.list()) {
|
|
1307
|
+
if (receipt.tags?.runDir !== runDir) continue;
|
|
1308
|
+
const scenarioCandidate = receipt.tags.scenarioId ? candidateByScenario.get(receipt.tags.scenarioId) : void 0;
|
|
1309
|
+
const recoveryCandidate = receipt.tags.memoryRecovery === "attempt" && receipt.tags.candidateId ? receipt.tags.candidateId : void 0;
|
|
1310
|
+
const candidateId = scenarioCandidate ?? recoveryCandidate;
|
|
1311
|
+
if (!candidateId || !candidateIds.has(candidateId)) continue;
|
|
1312
|
+
totals.set(candidateId, (totals.get(candidateId) ?? 0) + receipt.costUsd);
|
|
1313
|
+
}
|
|
1314
|
+
return totals;
|
|
1315
|
+
}
|
|
1316
|
+
function renderAgentMemoryExperimentRanking(rows, totalCostUsd, unrankedRecoveryCostUsd) {
|
|
1317
|
+
return [
|
|
1318
|
+
"# Agent Memory Experiment",
|
|
1319
|
+
"",
|
|
1320
|
+
`- total cost: $${format(totalCostUsd)}`,
|
|
1321
|
+
`- retired-candidate cost: $${format(unrankedRecoveryCostUsd)}`,
|
|
1322
|
+
"",
|
|
1323
|
+
"| rank | candidate | sequences | cells | probes | failed | score | pass rate | cost | duration ms |",
|
|
1324
|
+
"| ---: | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |",
|
|
1325
|
+
...rows.map((row) => `| ${row.rank} | ${row.label} | ${row.totalSequences} | ${row.totalCells} | ${row.totalProbes} | ${row.cellsFailed} | ${format(row.scoreMean)} | ${format(row.passRate)} | $${format(row.totalCostUsd)} | ${format(row.durationMs)} |`),
|
|
1326
|
+
""
|
|
1327
|
+
].join("\n");
|
|
1328
|
+
}
|
|
1329
|
+
function meanDimensions(rows) {
|
|
1330
|
+
const values = /* @__PURE__ */ new Map();
|
|
1331
|
+
for (const row of rows) for (const [key, value] of Object.entries(row)) {
|
|
1332
|
+
if (!Number.isFinite(value)) continue;
|
|
1333
|
+
const bucket = values.get(key) ?? [];
|
|
1334
|
+
bucket.push(value);
|
|
1335
|
+
values.set(key, bucket);
|
|
1336
|
+
}
|
|
1337
|
+
return Object.fromEntries([...values].map(([key, bucket]) => [key, mean$1(bucket)]));
|
|
1338
|
+
}
|
|
1339
|
+
function countDimensions(rows) {
|
|
1340
|
+
const counts = /* @__PURE__ */ new Map();
|
|
1341
|
+
for (const row of rows) for (const key of new Set(row)) counts.set(key, (counts.get(key) ?? 0) + 1);
|
|
1342
|
+
return Object.fromEntries(counts);
|
|
1343
|
+
}
|
|
1344
|
+
function mean$1(values) {
|
|
1345
|
+
return values.length === 0 ? 0 : values.reduce((sum, value) => sum + value, 0) / values.length;
|
|
1346
|
+
}
|
|
1347
|
+
function format(value) {
|
|
1348
|
+
return Number.isFinite(value) ? value.toFixed(4) : "0.0000";
|
|
1349
|
+
}
|
|
1350
|
+
function normalizeUsd(value) {
|
|
1351
|
+
return Number(value.toFixed(12));
|
|
1352
|
+
}
|
|
1353
|
+
//#endregion
|
|
1354
|
+
//#region src/memory/experiment/probe-evaluation.ts
|
|
1355
|
+
function scoreAgentMemoryProbe(sequence, step, probe, evidence) {
|
|
1356
|
+
return scoreMemoryBenchmarkArtifact(memoryProbeBenchmarkCase(sequence, step, probe), evidence);
|
|
1357
|
+
}
|
|
1358
|
+
function memoryProbeBenchmarkCase(sequence, step, probe) {
|
|
1359
|
+
return {
|
|
1360
|
+
id: `${sequence.id}:${step.id}:${probe.id}`,
|
|
1361
|
+
family: sequence.family,
|
|
1362
|
+
taskKind: probe.taskKind ?? "memory-recall",
|
|
1363
|
+
split: sequence.split,
|
|
1364
|
+
events: [],
|
|
1365
|
+
prompt: probe.query,
|
|
1366
|
+
requiredFacts: probe.requiredFacts && probe.requiredFacts.length > 0 ? probe.requiredFacts : probe.referenceAnswer ? [{
|
|
1367
|
+
id: `${probe.id}:reference`,
|
|
1368
|
+
anyOf: [probe.referenceAnswer]
|
|
1369
|
+
}] : void 0,
|
|
1370
|
+
forbiddenFacts: probe.forbiddenFacts,
|
|
1371
|
+
expectedEventIds: probe.expectedEventIds,
|
|
1372
|
+
expectedActorIds: probe.expectedActorIds,
|
|
1373
|
+
referenceAnswer: probe.referenceAnswer
|
|
1374
|
+
};
|
|
1375
|
+
}
|
|
1376
|
+
//#endregion
|
|
1377
|
+
//#region src/memory/experiment/learning-evidence.ts
|
|
1378
|
+
function assertStoredAgentMemoryEvidence(storage, result, sequences) {
|
|
1379
|
+
const sequenceById = new Map(sequences.map((sequence) => [sequence.id, sequence]));
|
|
1380
|
+
for (const cell of result.campaign.cells) {
|
|
1381
|
+
if (cell.error || !cell.artifact) throw new Error(`memory learning cell '${cell.cellId}' has no valid artifact${cell.error ? `: ${cell.error}` : ""}`);
|
|
1382
|
+
const sequence = sequenceById.get(cell.artifact.sequenceId);
|
|
1383
|
+
if (!sequence) throw new Error(`memory learning cell '${cell.cellId}' references unknown sequence '${cell.artifact.sequenceId}'`);
|
|
1384
|
+
assertStoredCellEvidence(storage, cell.cellId, cell.artifact, sequence);
|
|
1385
|
+
}
|
|
1386
|
+
}
|
|
1387
|
+
function assertStoredCellEvidence(storage, cellId, artifact, sequence) {
|
|
1388
|
+
const definitions = indexProbeDefinitions(sequence);
|
|
1389
|
+
if (artifact.probes.length !== definitions.size) throw new Error(`memory learning cell '${cellId}' has ${artifact.probes.length} probes; expected ${definitions.size}`);
|
|
1390
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1391
|
+
for (const probe of artifact.probes) {
|
|
1392
|
+
const key = probeKey(probe.stepOrdinal, probe.stepId, probe.id);
|
|
1393
|
+
const definition = definitions.get(key);
|
|
1394
|
+
if (!definition || seen.has(key)) throw new Error(`memory learning cell '${cellId}' has unexpected probe '${key}'`);
|
|
1395
|
+
seen.add(key);
|
|
1396
|
+
assertStoredProbeEvidence(storage, cellId, probe, definition, sequence);
|
|
1397
|
+
}
|
|
1398
|
+
const expectedScore = mean$1(artifact.probes.map((probe) => probe.score));
|
|
1399
|
+
const expectedPassed = artifact.probes.length > 0 && artifact.probes.every((probe) => probe.passed);
|
|
1400
|
+
const expectedDimensions = meanDimensions(artifact.probes.map((probe) => probe.dimensions));
|
|
1401
|
+
const expectedCounts = countDimensions(artifact.probes.map((probe) => probe.applicableDimensions));
|
|
1402
|
+
if (artifact.score !== expectedScore || artifact.passed !== expectedPassed || canonicalJson(artifact.dimensions) !== canonicalJson(expectedDimensions) || canonicalJson(artifact.dimensionSampleCounts) !== canonicalJson(expectedCounts)) throw new Error(`memory learning cell '${cellId}' aggregate does not match its probes`);
|
|
1403
|
+
}
|
|
1404
|
+
function assertStoredProbeEvidence(storage, cellId, observed, definition, sequence) {
|
|
1405
|
+
const { probe, step } = definition;
|
|
1406
|
+
if (observed.query !== probe.query || observed.retentionKey !== probe.retentionKey || observed.transferKey !== probe.transferKey) throw new Error(`memory learning cell '${cellId}' probe '${observed.id}' changed definition`);
|
|
1407
|
+
const raw = storage.read(observed.evidencePath);
|
|
1408
|
+
if (raw === void 0) throw new Error(`memory learning probe evidence is missing at '${observed.evidencePath}'`);
|
|
1409
|
+
let parsed;
|
|
1410
|
+
try {
|
|
1411
|
+
parsed = JSON.parse(raw);
|
|
1412
|
+
} catch (error) {
|
|
1413
|
+
throw new Error(`memory learning probe evidence is malformed at '${observed.evidencePath}'`, { cause: error });
|
|
1414
|
+
}
|
|
1415
|
+
if (canonicalDigest(parsed) !== observed.evidenceRef) throw new Error(`memory learning probe evidence hash does not match at '${observed.evidencePath}'`);
|
|
1416
|
+
const evidence = parseProbeEvidence(parsed, observed.evidencePath);
|
|
1417
|
+
const expected = scoreAgentMemoryProbe(sequence, step, probe, evidence);
|
|
1418
|
+
const observedEvaluation = {
|
|
1419
|
+
score: observed.score,
|
|
1420
|
+
passed: observed.passed,
|
|
1421
|
+
dimensions: observed.dimensions,
|
|
1422
|
+
applicableDimensions: observed.applicableDimensions,
|
|
1423
|
+
notes: observed.notes
|
|
1424
|
+
};
|
|
1425
|
+
const expectedEvaluation = {
|
|
1426
|
+
score: expected.score,
|
|
1427
|
+
passed: expected.passed,
|
|
1428
|
+
dimensions: expected.dimensions,
|
|
1429
|
+
applicableDimensions: expected.applicableDimensions ?? Object.keys(expected.dimensions),
|
|
1430
|
+
notes: expected.notes
|
|
1431
|
+
};
|
|
1432
|
+
if (canonicalJson(observedEvaluation) !== canonicalJson(expectedEvaluation)) throw new Error(`memory learning probe score does not match saved evidence at '${observed.evidencePath}'`);
|
|
1433
|
+
if (canonicalJson(observed.hitIds) !== canonicalJson(evidence.usedMemoryIds)) throw new Error(`memory learning probe hits do not match saved evidence at '${observed.evidencePath}'`);
|
|
1434
|
+
}
|
|
1435
|
+
function indexProbeDefinitions(sequence) {
|
|
1436
|
+
const definitions = /* @__PURE__ */ new Map();
|
|
1437
|
+
for (const [stepOrdinal, step] of sequence.steps.entries()) for (const probe of step.probes ?? []) definitions.set(probeKey(stepOrdinal, step.id, probe.id), {
|
|
1438
|
+
step,
|
|
1439
|
+
stepOrdinal,
|
|
1440
|
+
probe
|
|
1441
|
+
});
|
|
1442
|
+
return definitions;
|
|
1443
|
+
}
|
|
1444
|
+
function probeKey(stepOrdinal, stepId, probeId) {
|
|
1445
|
+
return JSON.stringify([
|
|
1446
|
+
stepOrdinal,
|
|
1447
|
+
stepId,
|
|
1448
|
+
probeId
|
|
1449
|
+
]);
|
|
1450
|
+
}
|
|
1451
|
+
function parseProbeEvidence(value, path) {
|
|
1452
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error(`memory learning probe evidence has invalid shape at '${path}'`);
|
|
1453
|
+
const record = value;
|
|
1454
|
+
if (typeof record.answer !== "string" || !isStringArray(record.rememberedFacts) || !isStringArray(record.citedEventIds) || !isStringArray(record.usedMemoryIds) || !isStringArray(record.actorIds)) throw new Error(`memory learning probe evidence has invalid shape at '${path}'`);
|
|
1455
|
+
return {
|
|
1456
|
+
answer: record.answer,
|
|
1457
|
+
rememberedFacts: record.rememberedFacts,
|
|
1458
|
+
citedEventIds: record.citedEventIds,
|
|
1459
|
+
usedMemoryIds: record.usedMemoryIds,
|
|
1460
|
+
actorIds: record.actorIds
|
|
1461
|
+
};
|
|
1462
|
+
}
|
|
1463
|
+
function isStringArray(value) {
|
|
1464
|
+
return Array.isArray(value) && value.every((item) => typeof item === "string");
|
|
1465
|
+
}
|
|
1466
|
+
//#endregion
|
|
1467
|
+
//#region src/memory/experiment/learning-metrics.ts
|
|
1468
|
+
function measureAgentMemoryLearning(paired, stateful, stateless) {
|
|
1469
|
+
const cells = paired.map(toCellComparison);
|
|
1470
|
+
const transferCells = paired.flatMap(toTransferCells);
|
|
1471
|
+
const forgettingProbes = paired.flatMap(toForgettingComparisons);
|
|
1472
|
+
const gainUnits = collapseLearningCells(cells);
|
|
1473
|
+
const forgetting = summarizeForgetting(forgettingProbes, stateful.campaign.seed);
|
|
1474
|
+
const preTreatment = measurePreTreatment(paired, stateful.campaign.seed);
|
|
1475
|
+
if (!paired[0]) throw new Error("cannot compare memory learning: no paired cells");
|
|
1476
|
+
return {
|
|
1477
|
+
comparisonRef: stateful.comparisonRef,
|
|
1478
|
+
evidence: {
|
|
1479
|
+
splitRef: evidenceRef(stateful.campaign.splitDigest, "task split"),
|
|
1480
|
+
statefulManifestRef: evidenceRef(stateful.campaign.manifestHash, "stateful campaign manifest"),
|
|
1481
|
+
statelessManifestRef: evidenceRef(stateless.campaign.manifestHash, "stateless campaign manifest"),
|
|
1482
|
+
statefulRunDir: stateful.campaign.runDir,
|
|
1483
|
+
statelessRunDir: stateless.campaign.runDir,
|
|
1484
|
+
candidateRefs: stateful.candidateRefs.map((candidate) => ({ ...candidate })),
|
|
1485
|
+
executionRef: stateful.executionRef
|
|
1486
|
+
},
|
|
1487
|
+
cells,
|
|
1488
|
+
preTreatment,
|
|
1489
|
+
gain: comparePairedRewards(gainUnits, stateful.campaign.seed, "overall gain"),
|
|
1490
|
+
gainByCandidate: summarizeCandidateGain(cells, stateful.campaign.seed),
|
|
1491
|
+
transfer: {
|
|
1492
|
+
definition: "explicit-transfer-probes",
|
|
1493
|
+
cells: transferCells,
|
|
1494
|
+
byStep: summarizeTransfer(transferCells, stateful.campaign.seed)
|
|
1495
|
+
},
|
|
1496
|
+
forgetting: {
|
|
1497
|
+
definition: "prior-peak-minus-final",
|
|
1498
|
+
probes: forgettingProbes,
|
|
1499
|
+
...forgetting
|
|
1500
|
+
}
|
|
1501
|
+
};
|
|
1502
|
+
}
|
|
1503
|
+
function measurePreTreatment(paired, seed) {
|
|
1504
|
+
const cells = paired.flatMap((pair) => {
|
|
1505
|
+
const probes = pair.probes.filter((probe) => probe.stateful.stepOrdinal === 0);
|
|
1506
|
+
if (probes.length === 0) return [];
|
|
1507
|
+
return [{
|
|
1508
|
+
sequenceId: pair.stateful.artifact.sequenceId,
|
|
1509
|
+
statefulReward: mean$1(probes.map((probe) => probe.stateful.score)),
|
|
1510
|
+
statelessReward: mean$1(probes.map((probe) => probe.stateless.score)),
|
|
1511
|
+
exact: probes.every((probe) => probe.stateful.score === probe.stateless.score)
|
|
1512
|
+
}];
|
|
1513
|
+
});
|
|
1514
|
+
const bySequence = /* @__PURE__ */ new Map();
|
|
1515
|
+
for (const cell of cells) {
|
|
1516
|
+
const unit = bySequence.get(cell.sequenceId) ?? {
|
|
1517
|
+
statefulRewards: [],
|
|
1518
|
+
statelessRewards: [],
|
|
1519
|
+
exact: true
|
|
1520
|
+
};
|
|
1521
|
+
unit.statefulRewards.push(cell.statefulReward);
|
|
1522
|
+
unit.statelessRewards.push(cell.statelessReward);
|
|
1523
|
+
unit.exact &&= cell.exact;
|
|
1524
|
+
bySequence.set(cell.sequenceId, unit);
|
|
1525
|
+
}
|
|
1526
|
+
const units = [...bySequence.entries()].map(([sequenceId, unit]) => ({
|
|
1527
|
+
pairKey: sequenceId,
|
|
1528
|
+
statefulReward: mean$1(unit.statefulRewards),
|
|
1529
|
+
statelessReward: mean$1(unit.statelessRewards),
|
|
1530
|
+
exact: unit.exact
|
|
1531
|
+
}));
|
|
1532
|
+
return {
|
|
1533
|
+
definition: "first-step-probes",
|
|
1534
|
+
cells: cells.length,
|
|
1535
|
+
n: units.length,
|
|
1536
|
+
exactMatchRate: units.length === 0 ? null : mean$1(units.map((unit) => unit.exact ? 1 : 0)),
|
|
1537
|
+
difference: units.length === 0 ? null : comparePairedRewards(units, seed, "pre-treatment difference")
|
|
1538
|
+
};
|
|
1539
|
+
}
|
|
1540
|
+
function toCellComparison(pair) {
|
|
1541
|
+
const probes = pair.probes.filter((probe) => probe.stateful.stepOrdinal > 0);
|
|
1542
|
+
if (probes.length === 0) throw new Error(`cannot compare memory learning: cell ${pair.stateful.cell.cellId} has no probe after the first step`);
|
|
1543
|
+
const statefulReward = mean$1(probes.map((probe) => probe.stateful.score));
|
|
1544
|
+
const statelessReward = mean$1(probes.map((probe) => probe.stateless.score));
|
|
1545
|
+
return {
|
|
1546
|
+
cellId: pair.stateful.cell.cellId,
|
|
1547
|
+
candidateId: pair.stateful.artifact.candidateId,
|
|
1548
|
+
sequenceId: pair.stateful.artifact.sequenceId,
|
|
1549
|
+
rep: pair.stateful.cell.rep,
|
|
1550
|
+
seed: pair.stateful.cell.seed,
|
|
1551
|
+
statefulArtifactRef: pair.stateful.artifactRef,
|
|
1552
|
+
statelessArtifactRef: pair.stateless.artifactRef,
|
|
1553
|
+
probeCount: probes.length,
|
|
1554
|
+
statefulReward,
|
|
1555
|
+
statelessReward,
|
|
1556
|
+
gain: difference(statefulReward, statelessReward)
|
|
1557
|
+
};
|
|
1558
|
+
}
|
|
1559
|
+
function toTransferCells(pair) {
|
|
1560
|
+
const groups = /* @__PURE__ */ new Map();
|
|
1561
|
+
for (const probe of pair.probes) {
|
|
1562
|
+
const transferKey = probe.stateful.transferKey;
|
|
1563
|
+
if (transferKey === void 0) continue;
|
|
1564
|
+
const key = JSON.stringify([probe.stateful.stepOrdinal, transferKey]);
|
|
1565
|
+
const group = groups.get(key) ?? {
|
|
1566
|
+
stepOrdinal: probe.stateful.stepOrdinal,
|
|
1567
|
+
transferKey,
|
|
1568
|
+
probes: []
|
|
1569
|
+
};
|
|
1570
|
+
group.probes.push(probe);
|
|
1571
|
+
groups.set(key, group);
|
|
1572
|
+
}
|
|
1573
|
+
return [...groups.values()].map(({ stepOrdinal, transferKey, probes }) => {
|
|
1574
|
+
const stepIds = sortedUnique$1(probes.map((probe) => probe.stateful.stepId));
|
|
1575
|
+
if (stepIds.length !== 1) throw new Error(`cannot compare memory learning: step ordinal ${stepOrdinal} maps to multiple step ids in ${pair.stateful.cell.cellId}`);
|
|
1576
|
+
const statefulReward = mean$1(probes.map((probe) => probe.stateful.score));
|
|
1577
|
+
const statelessReward = mean$1(probes.map((probe) => probe.stateless.score));
|
|
1578
|
+
return {
|
|
1579
|
+
cellId: pair.stateful.cell.cellId,
|
|
1580
|
+
candidateId: pair.stateful.artifact.candidateId,
|
|
1581
|
+
sequenceId: pair.stateful.artifact.sequenceId,
|
|
1582
|
+
rep: pair.stateful.cell.rep,
|
|
1583
|
+
seed: pair.stateful.cell.seed,
|
|
1584
|
+
stepId: stepIds[0],
|
|
1585
|
+
stepOrdinal,
|
|
1586
|
+
transferKey,
|
|
1587
|
+
probeCount: probes.length,
|
|
1588
|
+
statefulArtifactRef: pair.stateful.artifactRef,
|
|
1589
|
+
statelessArtifactRef: pair.stateless.artifactRef,
|
|
1590
|
+
statefulReward,
|
|
1591
|
+
statelessReward,
|
|
1592
|
+
gain: difference(statefulReward, statelessReward)
|
|
1593
|
+
};
|
|
1594
|
+
}).sort((left, right) => left.stepOrdinal - right.stepOrdinal || left.transferKey.localeCompare(right.transferKey));
|
|
1595
|
+
}
|
|
1596
|
+
function toForgettingComparisons(pair) {
|
|
1597
|
+
const byRetentionKey = /* @__PURE__ */ new Map();
|
|
1598
|
+
for (const probe of pair.probes) {
|
|
1599
|
+
const retentionKey = probe.stateful.retentionKey;
|
|
1600
|
+
if (retentionKey === void 0) continue;
|
|
1601
|
+
const probes = byRetentionKey.get(retentionKey) ?? [];
|
|
1602
|
+
probes.push(probe);
|
|
1603
|
+
byRetentionKey.set(retentionKey, probes);
|
|
1604
|
+
}
|
|
1605
|
+
return [...byRetentionKey.entries()].map(([retentionKey, observations]) => {
|
|
1606
|
+
observations.sort((left, right) => left.stateful.stepOrdinal - right.stateful.stepOrdinal);
|
|
1607
|
+
if (observations.length < 2) throw new Error(`cannot compare memory learning: retention key ${retentionKey} in ${pair.stateful.cell.cellId} has fewer than two observations`);
|
|
1608
|
+
const ordinals = observations.map((probe) => probe.stateful.stepOrdinal);
|
|
1609
|
+
if (new Set(ordinals).size !== ordinals.length) throw new Error(`cannot compare memory learning: retention key ${retentionKey} in ${pair.stateful.cell.cellId} repeats within one step`);
|
|
1610
|
+
const final = observations[observations.length - 1];
|
|
1611
|
+
const prior = observations.slice(0, -1);
|
|
1612
|
+
const statefulPeak = maxProbe(prior, (probe) => probe.stateful.score);
|
|
1613
|
+
const statelessPeak = maxProbe(prior, (probe) => probe.stateless.score);
|
|
1614
|
+
const statefulForgetting = difference(statefulPeak.reward, final.stateful.score);
|
|
1615
|
+
const statelessForgetting = difference(statelessPeak.reward, final.stateless.score);
|
|
1616
|
+
return {
|
|
1617
|
+
cellId: pair.stateful.cell.cellId,
|
|
1618
|
+
candidateId: pair.stateful.artifact.candidateId,
|
|
1619
|
+
sequenceId: pair.stateful.artifact.sequenceId,
|
|
1620
|
+
rep: pair.stateful.cell.rep,
|
|
1621
|
+
seed: pair.stateful.cell.seed,
|
|
1622
|
+
retentionKey,
|
|
1623
|
+
observations: observations.length,
|
|
1624
|
+
firstStepOrdinal: observations[0].stateful.stepOrdinal,
|
|
1625
|
+
finalStepOrdinal: final.stateful.stepOrdinal,
|
|
1626
|
+
statefulArtifactRef: pair.stateful.artifactRef,
|
|
1627
|
+
statelessArtifactRef: pair.stateless.artifactRef,
|
|
1628
|
+
statefulPriorPeakReward: statefulPeak.reward,
|
|
1629
|
+
statefulPriorPeakStepOrdinal: statefulPeak.stepOrdinal,
|
|
1630
|
+
statefulFinalReward: final.stateful.score,
|
|
1631
|
+
statefulForgetting,
|
|
1632
|
+
statelessPriorPeakReward: statelessPeak.reward,
|
|
1633
|
+
statelessPriorPeakStepOrdinal: statelessPeak.stepOrdinal,
|
|
1634
|
+
statelessFinalReward: final.stateless.score,
|
|
1635
|
+
statelessForgetting,
|
|
1636
|
+
excessForgetting: difference(statefulForgetting, statelessForgetting)
|
|
1637
|
+
};
|
|
1638
|
+
}).sort((left, right) => left.retentionKey.localeCompare(right.retentionKey) || left.sequenceId.localeCompare(right.sequenceId) || left.rep - right.rep || left.seed - right.seed);
|
|
1639
|
+
}
|
|
1640
|
+
function summarizeCandidateGain(cells, seed) {
|
|
1641
|
+
return sortedUnique$1(cells.map((cell) => cell.candidateId)).map((candidateId) => {
|
|
1642
|
+
const units = collapseLearningCells(cells.filter((cell) => cell.candidateId === candidateId));
|
|
1643
|
+
return {
|
|
1644
|
+
candidateId,
|
|
1645
|
+
cells: cells.filter((cell) => cell.candidateId === candidateId).length,
|
|
1646
|
+
gain: comparePairedRewards(units, seed, `candidate ${candidateId} gain`)
|
|
1647
|
+
};
|
|
1648
|
+
});
|
|
1649
|
+
}
|
|
1650
|
+
function summarizeTransfer(cells, seed) {
|
|
1651
|
+
const groups = /* @__PURE__ */ new Map();
|
|
1652
|
+
for (const cell of cells) {
|
|
1653
|
+
const key = JSON.stringify([
|
|
1654
|
+
cell.candidateId,
|
|
1655
|
+
cell.transferKey,
|
|
1656
|
+
cell.stepOrdinal
|
|
1657
|
+
]);
|
|
1658
|
+
const group = groups.get(key) ?? {
|
|
1659
|
+
candidateId: cell.candidateId,
|
|
1660
|
+
transferKey: cell.transferKey,
|
|
1661
|
+
stepOrdinal: cell.stepOrdinal,
|
|
1662
|
+
rewardsBySequence: /* @__PURE__ */ new Map()
|
|
1663
|
+
};
|
|
1664
|
+
const rewards = group.rewardsBySequence.get(cell.sequenceId) ?? {
|
|
1665
|
+
stateful: [],
|
|
1666
|
+
stateless: []
|
|
1667
|
+
};
|
|
1668
|
+
rewards.stateful.push(cell.statefulReward);
|
|
1669
|
+
rewards.stateless.push(cell.statelessReward);
|
|
1670
|
+
group.rewardsBySequence.set(cell.sequenceId, rewards);
|
|
1671
|
+
groups.set(key, group);
|
|
1672
|
+
}
|
|
1673
|
+
return [...groups.values()].sort((left, right) => left.candidateId.localeCompare(right.candidateId) || left.transferKey.localeCompare(right.transferKey) || left.stepOrdinal - right.stepOrdinal).map(({ candidateId, transferKey, stepOrdinal, rewardsBySequence }) => {
|
|
1674
|
+
return {
|
|
1675
|
+
candidateId,
|
|
1676
|
+
transferKey,
|
|
1677
|
+
stepOrdinal,
|
|
1678
|
+
gain: comparePairedRewards([...rewardsBySequence.entries()].sort(([left], [right]) => left.localeCompare(right)).map(([sequenceId, rewards]) => ({
|
|
1679
|
+
pairKey: sequenceId,
|
|
1680
|
+
statefulReward: mean$1(rewards.stateful),
|
|
1681
|
+
statelessReward: mean$1(rewards.stateless)
|
|
1682
|
+
})), seed, `transfer ${candidateId}:${transferKey}:${stepOrdinal}`)
|
|
1683
|
+
};
|
|
1684
|
+
});
|
|
1685
|
+
}
|
|
1686
|
+
function collapseLearningCells(cells) {
|
|
1687
|
+
const bySequence = /* @__PURE__ */ new Map();
|
|
1688
|
+
for (const cell of cells) {
|
|
1689
|
+
const unit = bySequence.get(cell.sequenceId) ?? {
|
|
1690
|
+
statefulRewards: [],
|
|
1691
|
+
statelessRewards: []
|
|
1692
|
+
};
|
|
1693
|
+
unit.statefulRewards.push(cell.statefulReward);
|
|
1694
|
+
unit.statelessRewards.push(cell.statelessReward);
|
|
1695
|
+
bySequence.set(cell.sequenceId, unit);
|
|
1696
|
+
}
|
|
1697
|
+
return [...bySequence.entries()].sort(([left], [right]) => left.localeCompare(right)).map(([sequenceId, unit]) => ({
|
|
1698
|
+
pairKey: sequenceId,
|
|
1699
|
+
statefulReward: mean$1(unit.statefulRewards),
|
|
1700
|
+
statelessReward: mean$1(unit.statelessRewards)
|
|
1701
|
+
}));
|
|
1702
|
+
}
|
|
1703
|
+
function summarizeForgetting(probes, seed) {
|
|
1704
|
+
const byTarget = /* @__PURE__ */ new Map();
|
|
1705
|
+
for (const probe of probes) {
|
|
1706
|
+
const key = JSON.stringify([probe.sequenceId, probe.retentionKey]);
|
|
1707
|
+
const unit = byTarget.get(key) ?? {
|
|
1708
|
+
stateful: [],
|
|
1709
|
+
stateless: [],
|
|
1710
|
+
excess: []
|
|
1711
|
+
};
|
|
1712
|
+
unit.stateful.push(probe.statefulForgetting);
|
|
1713
|
+
unit.stateless.push(probe.statelessForgetting);
|
|
1714
|
+
unit.excess.push(probe.excessForgetting);
|
|
1715
|
+
byTarget.set(key, unit);
|
|
1716
|
+
}
|
|
1717
|
+
const units = [...byTarget.entries()].map(([pairKey, unit]) => ({
|
|
1718
|
+
pairKey,
|
|
1719
|
+
statefulReward: mean$1(unit.stateful),
|
|
1720
|
+
statelessReward: mean$1(unit.stateless),
|
|
1721
|
+
excess: mean$1(unit.excess)
|
|
1722
|
+
}));
|
|
1723
|
+
return {
|
|
1724
|
+
n: units.length,
|
|
1725
|
+
meanStatefulForgetting: optionalMean(units.map((unit) => unit.statefulReward)),
|
|
1726
|
+
meanStatelessForgetting: optionalMean(units.map((unit) => unit.statelessReward)),
|
|
1727
|
+
meanExcessForgetting: optionalMean(units.map((unit) => unit.excess)),
|
|
1728
|
+
excess: units.length === 0 ? null : comparePairedRewards(units, seed, "excess forgetting")
|
|
1729
|
+
};
|
|
1730
|
+
}
|
|
1731
|
+
function comparePairedRewards(units, seed, label) {
|
|
1732
|
+
const comparison = comparePairedArms(units.flatMap((unit) => [{
|
|
1733
|
+
pairKey: unit.pairKey,
|
|
1734
|
+
arm: "stateless",
|
|
1735
|
+
metrics: { reward: unit.statelessReward }
|
|
1736
|
+
}, {
|
|
1737
|
+
pairKey: unit.pairKey,
|
|
1738
|
+
arm: "stateful",
|
|
1739
|
+
metrics: { reward: unit.statefulReward }
|
|
1740
|
+
}]), {
|
|
1741
|
+
baselineArm: "stateless",
|
|
1742
|
+
treatmentArm: "stateful",
|
|
1743
|
+
metricNames: ["reward"],
|
|
1744
|
+
bootstrap: {
|
|
1745
|
+
statistic: "mean",
|
|
1746
|
+
seed
|
|
1747
|
+
}
|
|
1748
|
+
});
|
|
1749
|
+
const metric = comparison.metricDeltas[0];
|
|
1750
|
+
if (comparison.nPairs !== units.length || comparison.nUnpairedBaseline !== 0 || comparison.nUnpairedTreatment !== 0 || !metric || metric.n !== units.length || metric.nMissing !== 0 || !metric.bootstrapCi) throw new Error(`cannot compare memory learning: incomplete paired ${label}`);
|
|
1751
|
+
return metric.bootstrapCi;
|
|
1752
|
+
}
|
|
1753
|
+
function maxProbe(probes, reward) {
|
|
1754
|
+
let best = probes[0];
|
|
1755
|
+
for (const probe of probes.slice(1)) if (reward(probe) > reward(best)) best = probe;
|
|
1756
|
+
return {
|
|
1757
|
+
reward: reward(best),
|
|
1758
|
+
stepOrdinal: best.stateful.stepOrdinal
|
|
1759
|
+
};
|
|
1760
|
+
}
|
|
1761
|
+
function optionalMean(values) {
|
|
1762
|
+
return values.length === 0 ? null : mean$1(values);
|
|
1763
|
+
}
|
|
1764
|
+
function difference(left, right) {
|
|
1765
|
+
const value = left - right;
|
|
1766
|
+
return Object.is(value, -0) ? 0 : value;
|
|
1767
|
+
}
|
|
1768
|
+
function sortedUnique$1(values) {
|
|
1769
|
+
return [...new Set(values)].sort();
|
|
1770
|
+
}
|
|
1771
|
+
function evidenceRef(value, label) {
|
|
1772
|
+
if (typeof value === "string" && /^sha256:[a-f0-9]{64}$/.test(value)) return value;
|
|
1773
|
+
if (typeof value !== "string" || !/^[a-f0-9]{64}$/.test(value)) throw new Error(`cannot compare memory learning: ${label} must be a full lowercase sha256 digest; received ${String(value)}`);
|
|
1774
|
+
return `sha256:${value}`;
|
|
1775
|
+
}
|
|
1776
|
+
//#endregion
|
|
1777
|
+
//#region src/memory/experiment/comparison-ref.ts
|
|
1778
|
+
const MEMORY_EXPERIMENT_IMPLEMENTATION_REF = `sha256:${sha256(canonicalJson({
|
|
1779
|
+
stateful: "retain declared sequence scopes across ordered steps",
|
|
1780
|
+
stateless: "clear every declared sequence scope before each step after the first",
|
|
1781
|
+
isolation: "provider scopes receive an opaque branch id but no experiment or sequence labels",
|
|
1782
|
+
recovery: "unfinished attempts bind cleanup to the exact sequence sha256",
|
|
1783
|
+
gain: "post-first-step stateful minus stateless reward",
|
|
1784
|
+
inference: "average candidates and repetitions within each sequence before paired bootstrap",
|
|
1785
|
+
preTreatment: "report first-step score balance separately from gain",
|
|
1786
|
+
transfer: "only probes with explicit transferKey after the first step",
|
|
1787
|
+
forgetting: "signed prior peak minus final score for exact repeated retentionKey probes",
|
|
1788
|
+
evidence: "full sha256 design, manifest, split, artifact, and probe scoring-input references"
|
|
1789
|
+
}))}`;
|
|
1790
|
+
const COMPARISON_REF_PATTERN = /^sha256:[a-f0-9]{64}$/;
|
|
1791
|
+
function resolveAgentMemoryMode(value) {
|
|
1792
|
+
if (value === void 0) return "stateful";
|
|
1793
|
+
if (value !== "stateful" && value !== "stateless") throw new Error("memory experiment memoryMode must be 'stateful' or 'stateless'");
|
|
1794
|
+
return value;
|
|
1795
|
+
}
|
|
1796
|
+
function memoryExperimentComparisonRef(options, resolvedRunDir) {
|
|
1797
|
+
return `sha256:${sha256(canonicalJson({
|
|
1798
|
+
implementationRef: MEMORY_EXPERIMENT_IMPLEMENTATION_REF,
|
|
1799
|
+
experimentId: options.experimentId,
|
|
1800
|
+
experimentRunId: options.experimentRunId ?? resolvedRunDir,
|
|
1801
|
+
sequences: options.sequences,
|
|
1802
|
+
candidates: options.candidates.map(candidateConditions),
|
|
1803
|
+
recoveryCandidates: (options.recoveryCandidates ?? []).map(candidateConditions),
|
|
1804
|
+
executeStepRef: options.executeStepRef ?? "fixtures",
|
|
1805
|
+
cleanupBranches: options.cleanupBranches ?? true,
|
|
1806
|
+
seed: options.seed ?? 42,
|
|
1807
|
+
reps: options.reps ?? 1,
|
|
1808
|
+
resumable: options.resumable ?? true,
|
|
1809
|
+
costCeilingUsd: options.costCeiling ?? options.costLedger?.costCeilingUsd ?? 0,
|
|
1810
|
+
costPhase: options.costPhase ?? null,
|
|
1811
|
+
maxConcurrency: options.maxConcurrency ?? 2,
|
|
1812
|
+
dispatchTimeoutMs: options.dispatchTimeoutMs || null,
|
|
1813
|
+
cleanupTimeoutMs: options.cleanupTimeoutMs ?? 18e4,
|
|
1814
|
+
maxRecoveryAttempts: options.maxRecoveryAttempts ?? 1e3,
|
|
1815
|
+
maxRecoveryRetriesPerAttempt: options.maxRecoveryRetriesPerAttempt ?? 3
|
|
1816
|
+
}))}`;
|
|
1817
|
+
}
|
|
1818
|
+
function assertAgentMemoryExperimentComparisonRef(value, label) {
|
|
1819
|
+
if (typeof value !== "string" || !COMPARISON_REF_PATTERN.test(value)) throw new Error(`${label} must be a full lowercase sha256 reference`);
|
|
1820
|
+
}
|
|
1821
|
+
function candidateConditions(candidate) {
|
|
1822
|
+
return {
|
|
1823
|
+
id: candidate.id,
|
|
1824
|
+
label: candidate.label ?? null,
|
|
1825
|
+
ref: candidate.ref,
|
|
1826
|
+
policy: candidate.policy ?? null,
|
|
1827
|
+
baseScope: candidate.baseScope ?? null,
|
|
1828
|
+
externalCostUsdPerSequence: candidate.externalCostUsdPerSequence ?? 0,
|
|
1829
|
+
externalRecoveryCostUsdPerAttempt: candidate.externalRecoveryCostUsdPerAttempt ?? 0,
|
|
1830
|
+
externalCostAccounting: candidate.externalCostAccounting ?? "exact"
|
|
1831
|
+
};
|
|
1832
|
+
}
|
|
1833
|
+
//#endregion
|
|
1834
|
+
//#region src/memory/experiment/learning-pairs.ts
|
|
1835
|
+
function pairAgentMemoryLearningRuns(options) {
|
|
1836
|
+
assertComparableDesign(options.stateful, options.stateless);
|
|
1837
|
+
const statefulCells = indexCompleteCells("stateful", options.stateful);
|
|
1838
|
+
const statelessCells = indexCompleteCells("stateless", options.stateless);
|
|
1839
|
+
const matched = pairArms([...memoryLearningArmRows("stateful", statefulCells.values()), ...memoryLearningArmRows("stateless", statelessCells.values())], {
|
|
1840
|
+
baselineArm: "stateless",
|
|
1841
|
+
treatmentArm: "stateful"
|
|
1842
|
+
});
|
|
1843
|
+
if (matched.unpairedBaseline.length > 0 || matched.unpairedTreatment.length > 0) throw new Error(`cannot compare memory learning: unmatched experiment cell identities; stateful-only=${unpairedCellIds(matched.unpairedTreatment).join(", ") || "none"}; stateless-only=${unpairedCellIds(matched.unpairedBaseline).join(", ") || "none"}`);
|
|
1844
|
+
return matched.pairs.map((pair) => {
|
|
1845
|
+
const stateful = pair.treatment.indexed;
|
|
1846
|
+
const stateless = pair.baseline.indexed;
|
|
1847
|
+
if (stateful.cell.cellId !== stateless.cell.cellId) throw new Error(`cannot compare memory learning: cell identity differs for ${pair.pairKey} (${stateful.cell.cellId} vs ${stateless.cell.cellId})`);
|
|
1848
|
+
return {
|
|
1849
|
+
stateful,
|
|
1850
|
+
stateless,
|
|
1851
|
+
probes: pairProbes(stateful, stateless)
|
|
1852
|
+
};
|
|
1853
|
+
}).sort(comparePairedCells);
|
|
1854
|
+
}
|
|
1855
|
+
function memoryLearningArmRows(arm, cells) {
|
|
1856
|
+
return [...cells].map((indexed) => ({
|
|
1857
|
+
arm,
|
|
1858
|
+
pairKey: JSON.stringify([indexed.artifact.candidateId, indexed.artifact.sequenceId]),
|
|
1859
|
+
repKey: JSON.stringify([indexed.cell.rep, indexed.cell.seed]),
|
|
1860
|
+
indexed
|
|
1861
|
+
}));
|
|
1862
|
+
}
|
|
1863
|
+
function unpairedCellIds(rows) {
|
|
1864
|
+
return rows.map((row) => row.indexed).map((indexed) => cellIdentity(indexed.artifact, indexed.cell.rep, indexed.cell.seed)).sort();
|
|
1865
|
+
}
|
|
1866
|
+
function assertComparableDesign(stateful, stateless) {
|
|
1867
|
+
if (stateful.memoryMode !== "stateful") throw new Error(`cannot compare memory learning: stateful arm is labeled ${String(stateful.memoryMode)}`);
|
|
1868
|
+
if (stateless.memoryMode !== "stateless") throw new Error(`cannot compare memory learning: stateless arm is labeled ${String(stateless.memoryMode)}`);
|
|
1869
|
+
assertAgentMemoryExperimentComparisonRef(stateful.comparisonRef, "cannot compare memory learning: stateful comparisonRef");
|
|
1870
|
+
assertAgentMemoryExperimentComparisonRef(stateless.comparisonRef, "cannot compare memory learning: stateless comparisonRef");
|
|
1871
|
+
if (stateful.comparisonRef !== stateless.comparisonRef) throw new Error(`cannot compare memory learning: experimental conditions differ (${stateful.comparisonRef} vs ${stateless.comparisonRef})`);
|
|
1872
|
+
if (stateful.campaign.manifestHash === stateless.campaign.manifestHash) throw new Error("cannot compare memory learning: campaign cache identity does not distinguish memoryMode");
|
|
1873
|
+
assertRunSplitIdentity("stateful", stateful);
|
|
1874
|
+
assertRunSplitIdentity("stateless", stateless);
|
|
1875
|
+
if (stateful.campaign.splitDigest !== stateless.campaign.splitDigest) throw new Error("cannot compare memory learning: stateful and stateless task identities differ");
|
|
1876
|
+
if (stateful.campaign.reps !== stateless.campaign.reps) throw new Error(`cannot compare memory learning: repetitions differ (${stateful.campaign.reps} vs ${stateless.campaign.reps})`);
|
|
1877
|
+
if (stateful.campaign.seed !== stateless.campaign.seed) throw new Error(`cannot compare memory learning: campaign seeds differ (${stateful.campaign.seed} vs ${stateless.campaign.seed})`);
|
|
1878
|
+
if (JSON.stringify(stateful.candidateRefs) !== JSON.stringify(stateless.candidateRefs) || stateful.candidateRefs.length === 0) throw new Error("cannot compare memory learning: candidate references differ between arms");
|
|
1879
|
+
const candidateIds = /* @__PURE__ */ new Set();
|
|
1880
|
+
for (const candidate of stateful.candidateRefs) {
|
|
1881
|
+
assertNonEmpty(candidate.id, "candidate reference id");
|
|
1882
|
+
if (candidateIds.has(candidate.id)) throw new Error(`cannot compare memory learning: duplicate candidate reference ${candidate.id}`);
|
|
1883
|
+
candidateIds.add(candidate.id);
|
|
1884
|
+
try {
|
|
1885
|
+
assertImmutableRef(candidate.ref, `${candidate.id}: candidate reference`);
|
|
1886
|
+
} catch (error) {
|
|
1887
|
+
throw new Error(`cannot compare memory learning: ${error instanceof Error ? error.message : String(error)}`);
|
|
1888
|
+
}
|
|
1889
|
+
}
|
|
1890
|
+
if (stateful.executionRef !== stateless.executionRef) throw new Error("cannot compare memory learning: executor references differ between arms");
|
|
1891
|
+
if (stateful.executionRef !== "fixtures") try {
|
|
1892
|
+
assertImmutableRef(stateful.executionRef, "memory learning executor reference");
|
|
1893
|
+
} catch (error) {
|
|
1894
|
+
throw new Error(`cannot compare memory learning: ${error instanceof Error ? error.message : String(error)}`);
|
|
1895
|
+
}
|
|
1896
|
+
}
|
|
1897
|
+
function assertRunSplitIdentity(arm, result) {
|
|
1898
|
+
try {
|
|
1899
|
+
assertCampaignSplitIdentity(result.campaign.scenarios, result.campaign.reps, result.campaign.splitDigest);
|
|
1900
|
+
} catch (error) {
|
|
1901
|
+
throw new Error(`cannot compare memory learning: ${arm} run has invalid task identity: ${error instanceof Error ? error.message : String(error)}`);
|
|
1902
|
+
}
|
|
1903
|
+
}
|
|
1904
|
+
function indexCompleteCells(arm, result) {
|
|
1905
|
+
const expectedCellIds = new Set(result.campaign.scenarios.flatMap((scenario) => Array.from({ length: result.campaign.reps }, (_, rep) => `${scenario.id}:${rep}`)));
|
|
1906
|
+
const byIdentity = /* @__PURE__ */ new Map();
|
|
1907
|
+
const seenCellIds = /* @__PURE__ */ new Set();
|
|
1908
|
+
for (const cell of result.campaign.cells) {
|
|
1909
|
+
if (!expectedCellIds.has(cell.cellId)) throw new Error(`cannot compare memory learning: ${arm} run has unexpected cell ${cell.cellId}`);
|
|
1910
|
+
if (seenCellIds.has(cell.cellId)) throw new Error(`cannot compare memory learning: ${arm} run duplicates cell ${cell.cellId}`);
|
|
1911
|
+
seenCellIds.add(cell.cellId);
|
|
1912
|
+
if (cell.cellId !== `${cell.scenarioId}:${cell.rep}`) throw new Error(`cannot compare memory learning: ${arm} cell ${cell.cellId} does not match its scenario and repetition`);
|
|
1913
|
+
if (cell.manifestHash !== result.campaign.manifestHash) throw new Error(`cannot compare memory learning: ${arm} cell ${cell.cellId} manifest differs from its campaign`);
|
|
1914
|
+
if (cell.error) throw new Error(`cannot compare memory learning: ${arm} cell ${cell.cellId} failed: ${cell.error}`);
|
|
1915
|
+
const artifact = cell.artifact;
|
|
1916
|
+
if (!artifact) throw new Error(`cannot compare memory learning: ${arm} cell ${cell.cellId} has no artifact`);
|
|
1917
|
+
assertArtifactIdentity(arm, result, cell, artifact);
|
|
1918
|
+
if (!Number.isSafeInteger(cell.rep) || cell.rep < 0 || cell.rep >= result.campaign.reps) throw new Error(`cannot compare memory learning: ${arm} cell ${cell.cellId} has invalid rep`);
|
|
1919
|
+
if (!Number.isSafeInteger(cell.seed)) throw new Error(`cannot compare memory learning: ${arm} cell ${cell.cellId} has invalid seed`);
|
|
1920
|
+
assertReward(artifact.score, `${arm} cell ${cell.cellId} reward`);
|
|
1921
|
+
if (artifact.probes.length === 0) throw new Error(`cannot compare memory learning: ${arm} cell ${cell.cellId} has no probes`);
|
|
1922
|
+
for (const probe of artifact.probes) assertProbe(arm, cell.cellId, probe);
|
|
1923
|
+
const probeMean = mean$1(artifact.probes.map((probe) => probe.score));
|
|
1924
|
+
if (Math.abs(probeMean - artifact.score) > 1e-12) throw new Error(`cannot compare memory learning: ${arm} cell ${cell.cellId} reward does not equal its probe mean`);
|
|
1925
|
+
const identity = cellIdentity(artifact, cell.rep, cell.seed);
|
|
1926
|
+
if (byIdentity.has(identity)) throw new Error(`cannot compare memory learning: ${arm} run duplicates identity ${identity}`);
|
|
1927
|
+
byIdentity.set(identity, {
|
|
1928
|
+
cell,
|
|
1929
|
+
artifact,
|
|
1930
|
+
artifactRef: canonicalDigest(artifact)
|
|
1931
|
+
});
|
|
1932
|
+
}
|
|
1933
|
+
const missing = [...expectedCellIds].filter((cellId) => !seenCellIds.has(cellId));
|
|
1934
|
+
if (missing.length > 0) throw new Error(`cannot compare memory learning: ${arm} run is missing cells ${missing.sort().join(", ")}`);
|
|
1935
|
+
return byIdentity;
|
|
1936
|
+
}
|
|
1937
|
+
function assertArtifactIdentity(arm, result, cell, artifact) {
|
|
1938
|
+
assertNonEmpty(artifact.candidateId, `${arm} cell ${cell.cellId} candidateId`);
|
|
1939
|
+
assertNonEmpty(artifact.sequenceId, `${arm} cell ${cell.cellId} sequenceId`);
|
|
1940
|
+
if (!result.candidateRefs.some((candidate) => candidate.id === artifact.candidateId)) throw new Error(`cannot compare memory learning: ${arm} cell ${cell.cellId} uses an undeclared candidate`);
|
|
1941
|
+
if (artifact.memoryMode !== arm) throw new Error(`cannot compare memory learning: ${arm} cell ${cell.cellId} artifact is labeled ${String(artifact.memoryMode)}`);
|
|
1942
|
+
if (artifact.comparisonRef !== result.comparisonRef) throw new Error(`cannot compare memory learning: ${arm} cell ${cell.cellId} comparisonRef differs from its run`);
|
|
1943
|
+
if (!/^sha256:[a-f0-9]{64}$/.test(artifact.branchDigest)) throw new Error(`cannot compare memory learning: ${arm} cell ${cell.cellId} has invalid branch evidence`);
|
|
1944
|
+
const expectedScenarioId = `${stableId("candidate", artifact.candidateId)}:${artifact.sequenceId}`;
|
|
1945
|
+
if (cell.scenarioId !== expectedScenarioId) throw new Error(`cannot compare memory learning: ${arm} cell ${cell.cellId} artifact identity does not match its scenario`);
|
|
1946
|
+
}
|
|
1947
|
+
function pairProbes(stateful, stateless) {
|
|
1948
|
+
const statefulProbes = indexProbes("stateful", stateful);
|
|
1949
|
+
const statelessProbes = indexProbes("stateless", stateless);
|
|
1950
|
+
assertSameKeys(statefulProbes, statelessProbes, `probe in cell ${stateful.cell.cellId}`);
|
|
1951
|
+
return [...statefulProbes.entries()].map(([key, statefulProbe]) => {
|
|
1952
|
+
const statelessProbe = statelessProbes.get(key);
|
|
1953
|
+
if (!statelessProbe) throw new Error(`cannot compare memory learning: missing stateless probe ${key} in ${stateful.cell.cellId}`);
|
|
1954
|
+
if (statefulProbe.query !== statelessProbe.query) throw new Error(`cannot compare memory learning: probe query differs for ${key} in ${stateful.cell.cellId}`);
|
|
1955
|
+
if (statefulProbe.retentionKey !== statelessProbe.retentionKey) throw new Error(`cannot compare memory learning: retention identity differs for ${key} in ${stateful.cell.cellId}`);
|
|
1956
|
+
if (statefulProbe.transferKey !== statelessProbe.transferKey) throw new Error(`cannot compare memory learning: transfer identity differs for ${key} in ${stateful.cell.cellId}`);
|
|
1957
|
+
if (sortedUnique(statefulProbe.applicableDimensions).join("\0") !== sortedUnique(statelessProbe.applicableDimensions).join("\0")) throw new Error(`cannot compare memory learning: measured dimensions differ for ${key} in ${stateful.cell.cellId}`);
|
|
1958
|
+
return {
|
|
1959
|
+
stateful: statefulProbe,
|
|
1960
|
+
stateless: statelessProbe
|
|
1961
|
+
};
|
|
1962
|
+
}).sort((left, right) => left.stateful.stepOrdinal !== right.stateful.stepOrdinal ? left.stateful.stepOrdinal - right.stateful.stepOrdinal : left.stateful.id.localeCompare(right.stateful.id));
|
|
1963
|
+
}
|
|
1964
|
+
function indexProbes(arm, indexed) {
|
|
1965
|
+
const probes = /* @__PURE__ */ new Map();
|
|
1966
|
+
for (const probe of indexed.artifact.probes) {
|
|
1967
|
+
const key = probeIdentity(probe);
|
|
1968
|
+
if (probes.has(key)) throw new Error(`cannot compare memory learning: ${arm} cell ${indexed.cell.cellId} duplicates probe ${key}`);
|
|
1969
|
+
probes.set(key, probe);
|
|
1970
|
+
}
|
|
1971
|
+
return probes;
|
|
1972
|
+
}
|
|
1973
|
+
function assertProbe(arm, cellId, probe) {
|
|
1974
|
+
assertNonEmpty(probe.id, `${arm} cell ${cellId} probe id`);
|
|
1975
|
+
assertNonEmpty(probe.stepId, `${arm} cell ${cellId} probe ${probe.id} stepId`);
|
|
1976
|
+
assertNonEmpty(probe.query, `${arm} cell ${cellId} probe ${probe.id} query`);
|
|
1977
|
+
if (!Number.isSafeInteger(probe.stepOrdinal) || probe.stepOrdinal < 0) throw new Error(`cannot compare memory learning: ${arm} cell ${cellId} probe ${probe.id} has invalid stepOrdinal`);
|
|
1978
|
+
if (probe.retentionKey !== void 0) {
|
|
1979
|
+
assertNonEmpty(probe.retentionKey, `${arm} cell ${cellId} probe ${probe.id} retentionKey`);
|
|
1980
|
+
if (probe.retentionKey !== probe.retentionKey.trim()) throw new Error(`cannot compare memory learning: ${arm} cell ${cellId} probe ${probe.id} retentionKey has surrounding whitespace`);
|
|
1981
|
+
}
|
|
1982
|
+
if (probe.transferKey !== void 0) {
|
|
1983
|
+
assertNonEmpty(probe.transferKey, `${arm} cell ${cellId} probe ${probe.id} transferKey`);
|
|
1984
|
+
if (probe.transferKey !== probe.transferKey.trim()) throw new Error(`cannot compare memory learning: ${arm} cell ${cellId} probe ${probe.id} transferKey has surrounding whitespace`);
|
|
1985
|
+
if (probe.retentionKey !== void 0 || probe.stepOrdinal === 0) throw new Error(`cannot compare memory learning: ${arm} cell ${cellId} probe ${probe.id} has invalid transfer placement`);
|
|
1986
|
+
}
|
|
1987
|
+
if (!/^sha256:[a-f0-9]{64}$/.test(probe.evidenceRef)) throw new Error(`cannot compare memory learning: ${arm} cell ${cellId} probe ${probe.id} has invalid evidence`);
|
|
1988
|
+
const expectedEvidenceSuffix = `/memory-evidence/${probe.evidenceRef.slice(7)}.json`;
|
|
1989
|
+
if (!probe.evidencePath.endsWith(expectedEvidenceSuffix)) throw new Error(`cannot compare memory learning: ${arm} cell ${cellId} probe ${probe.id} evidence is missing`);
|
|
1990
|
+
assertReward(probe.score, `${arm} cell ${cellId} probe ${probe.id} reward`);
|
|
1991
|
+
}
|
|
1992
|
+
function assertReward(value, label) {
|
|
1993
|
+
if (!Number.isFinite(value) || value < 0 || value > 1) throw new Error(`cannot compare memory learning: ${label} must be finite and between 0 and 1`);
|
|
1994
|
+
}
|
|
1995
|
+
function assertNonEmpty(value, label) {
|
|
1996
|
+
if (typeof value !== "string" || value.trim().length === 0) throw new Error(`cannot compare memory learning: ${label} must be a non-empty string`);
|
|
1997
|
+
}
|
|
1998
|
+
function assertSameKeys(left, right, label) {
|
|
1999
|
+
const missingRight = [...left.keys()].filter((key) => !right.has(key));
|
|
2000
|
+
const missingLeft = [...right.keys()].filter((key) => !left.has(key));
|
|
2001
|
+
if (missingRight.length === 0 && missingLeft.length === 0) return;
|
|
2002
|
+
throw new Error(`cannot compare memory learning: unmatched ${label} identities; stateful-only=${missingRight.sort().join(", ") || "none"}; stateless-only=${missingLeft.sort().join(", ") || "none"}`);
|
|
2003
|
+
}
|
|
2004
|
+
function cellIdentity(artifact, rep, seed) {
|
|
2005
|
+
return JSON.stringify([
|
|
2006
|
+
artifact.candidateId,
|
|
2007
|
+
artifact.sequenceId,
|
|
2008
|
+
rep,
|
|
2009
|
+
seed
|
|
2010
|
+
]);
|
|
2011
|
+
}
|
|
2012
|
+
function probeIdentity(probe) {
|
|
2013
|
+
return JSON.stringify([
|
|
2014
|
+
probe.stepOrdinal,
|
|
2015
|
+
probe.stepId,
|
|
2016
|
+
probe.id
|
|
2017
|
+
]);
|
|
2018
|
+
}
|
|
2019
|
+
function comparePairedCells(left, right) {
|
|
2020
|
+
return left.stateful.artifact.candidateId.localeCompare(right.stateful.artifact.candidateId) || left.stateful.artifact.sequenceId.localeCompare(right.stateful.artifact.sequenceId) || left.stateful.cell.rep - right.stateful.cell.rep || left.stateful.cell.seed - right.stateful.cell.seed;
|
|
2021
|
+
}
|
|
2022
|
+
function sortedUnique(values) {
|
|
2023
|
+
return [...new Set(values)].sort();
|
|
2024
|
+
}
|
|
2025
|
+
//#endregion
|
|
1215
2026
|
//#region src/memory/experiment/cost.ts
|
|
1216
2027
|
function createAgentMemoryCostRecorder(input) {
|
|
1217
2028
|
let observedCostUsd = 0;
|
|
@@ -1265,6 +2076,7 @@ function createAgentMemoryExecutionContext(dispatch, sequence) {
|
|
|
1265
2076
|
signal: signal.signal,
|
|
1266
2077
|
cost
|
|
1267
2078
|
}),
|
|
2079
|
+
abort: signal.abort,
|
|
1268
2080
|
dispose: signal.dispose
|
|
1269
2081
|
};
|
|
1270
2082
|
}
|
|
@@ -1309,6 +2121,7 @@ function relayAbortWithoutReason(source) {
|
|
|
1309
2121
|
else source.addEventListener("abort", abort, { once: true });
|
|
1310
2122
|
return {
|
|
1311
2123
|
signal: controller.signal,
|
|
2124
|
+
abort,
|
|
1312
2125
|
dispose() {
|
|
1313
2126
|
source.removeEventListener("abort", abort);
|
|
1314
2127
|
}
|
|
@@ -1361,85 +2174,6 @@ function redactPrivateValues(value, privateValues) {
|
|
|
1361
2174
|
return redacted;
|
|
1362
2175
|
}
|
|
1363
2176
|
//#endregion
|
|
1364
|
-
//#region src/memory/experiment/metrics.ts
|
|
1365
|
-
function rankAgentMemoryExperiment(candidates, scenarios, campaign, costByCandidate) {
|
|
1366
|
-
const scenarioById = new Map(scenarios.map((scenario) => [scenario.id, scenario]));
|
|
1367
|
-
return candidates.map((candidate) => {
|
|
1368
|
-
const candidateScenarios = scenarios.filter((scenario) => scenario.candidateId === candidate.id);
|
|
1369
|
-
const cells = campaign.cells.filter((cell) => scenarioById.get(cell.scenarioId)?.candidateId === candidate.id);
|
|
1370
|
-
const successful = cells.filter((cell) => !cell.error && cell.artifact);
|
|
1371
|
-
const dimensionRows = successful.map((cell) => cell.artifact.dimensions);
|
|
1372
|
-
return {
|
|
1373
|
-
rank: 0,
|
|
1374
|
-
candidateId: candidate.id,
|
|
1375
|
-
label: candidate.label ?? candidate.id,
|
|
1376
|
-
scoreMean: mean$1(cells.map((cell) => !cell.error && cell.artifact ? cell.artifact.score : 0)),
|
|
1377
|
-
passRate: mean$1(cells.map((cell) => !cell.error && cell.artifact?.passed ? 1 : 0)),
|
|
1378
|
-
totalSequences: candidateScenarios.length,
|
|
1379
|
-
totalCells: cells.length,
|
|
1380
|
-
totalProbes: successful.reduce((sum, cell) => sum + cell.artifact.probes.length, 0),
|
|
1381
|
-
cellsFailed: cells.filter((cell) => Boolean(cell.error)).length,
|
|
1382
|
-
totalCostUsd: normalizeUsd(costByCandidate.get(candidate.id) ?? 0),
|
|
1383
|
-
durationMs: cells.reduce((sum, cell) => sum + cell.durationMs, 0),
|
|
1384
|
-
dimensions: meanDimensions(dimensionRows)
|
|
1385
|
-
};
|
|
1386
|
-
}).sort((a, b) => Number(a.cellsFailed > 0) - Number(b.cellsFailed > 0) || b.scoreMean - a.scoreMean || b.passRate - a.passRate || a.totalCostUsd - b.totalCostUsd || a.candidateId.localeCompare(b.candidateId)).map((row, index) => ({
|
|
1387
|
-
...row,
|
|
1388
|
-
rank: index + 1
|
|
1389
|
-
}));
|
|
1390
|
-
}
|
|
1391
|
-
function memoryExperimentCostByCandidate(costLedger, runDir, scenarios, candidateIdsInput) {
|
|
1392
|
-
const candidateByScenario = new Map(scenarios.map((scenario) => [scenario.id, scenario.candidateId]));
|
|
1393
|
-
const candidateIds = new Set(candidateIdsInput);
|
|
1394
|
-
const totals = /* @__PURE__ */ new Map();
|
|
1395
|
-
for (const receipt of costLedger.list()) {
|
|
1396
|
-
if (receipt.tags?.runDir !== runDir) continue;
|
|
1397
|
-
const scenarioCandidate = receipt.tags.scenarioId ? candidateByScenario.get(receipt.tags.scenarioId) : void 0;
|
|
1398
|
-
const recoveryCandidate = receipt.tags.memoryRecovery === "attempt" && receipt.tags.candidateId ? receipt.tags.candidateId : void 0;
|
|
1399
|
-
const candidateId = scenarioCandidate ?? recoveryCandidate;
|
|
1400
|
-
if (!candidateId || !candidateIds.has(candidateId)) continue;
|
|
1401
|
-
totals.set(candidateId, (totals.get(candidateId) ?? 0) + receipt.costUsd);
|
|
1402
|
-
}
|
|
1403
|
-
return totals;
|
|
1404
|
-
}
|
|
1405
|
-
function renderAgentMemoryExperimentRanking(rows, totalCostUsd, unrankedRecoveryCostUsd) {
|
|
1406
|
-
return [
|
|
1407
|
-
"# Agent Memory Experiment",
|
|
1408
|
-
"",
|
|
1409
|
-
`- total cost: $${format(totalCostUsd)}`,
|
|
1410
|
-
`- retired-candidate recovery cost: $${format(unrankedRecoveryCostUsd)}`,
|
|
1411
|
-
"",
|
|
1412
|
-
"| rank | candidate | sequences | cells | probes | failed | score | pass rate | cost | duration ms |",
|
|
1413
|
-
"| ---: | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |",
|
|
1414
|
-
...rows.map((row) => `| ${row.rank} | ${row.label} | ${row.totalSequences} | ${row.totalCells} | ${row.totalProbes} | ${row.cellsFailed} | ${format(row.scoreMean)} | ${format(row.passRate)} | $${format(row.totalCostUsd)} | ${format(row.durationMs)} |`),
|
|
1415
|
-
""
|
|
1416
|
-
].join("\n");
|
|
1417
|
-
}
|
|
1418
|
-
function meanDimensions(rows) {
|
|
1419
|
-
const values = /* @__PURE__ */ new Map();
|
|
1420
|
-
for (const row of rows) for (const [key, value] of Object.entries(row)) {
|
|
1421
|
-
if (!Number.isFinite(value)) continue;
|
|
1422
|
-
const bucket = values.get(key) ?? [];
|
|
1423
|
-
bucket.push(value);
|
|
1424
|
-
values.set(key, bucket);
|
|
1425
|
-
}
|
|
1426
|
-
return Object.fromEntries([...values].map(([key, bucket]) => [key, mean$1(bucket)]));
|
|
1427
|
-
}
|
|
1428
|
-
function countDimensions(rows) {
|
|
1429
|
-
const counts = /* @__PURE__ */ new Map();
|
|
1430
|
-
for (const row of rows) for (const key of new Set(row)) counts.set(key, (counts.get(key) ?? 0) + 1);
|
|
1431
|
-
return Object.fromEntries(counts);
|
|
1432
|
-
}
|
|
1433
|
-
function mean$1(values) {
|
|
1434
|
-
return values.length === 0 ? 0 : values.reduce((sum, value) => sum + value, 0) / values.length;
|
|
1435
|
-
}
|
|
1436
|
-
function format(value) {
|
|
1437
|
-
return Number.isFinite(value) ? value.toFixed(4) : "0.0000";
|
|
1438
|
-
}
|
|
1439
|
-
function normalizeUsd(value) {
|
|
1440
|
-
return Number(value.toFixed(12));
|
|
1441
|
-
}
|
|
1442
|
-
//#endregion
|
|
1443
2177
|
//#region src/memory/experiment/runtime.ts
|
|
1444
2178
|
var AgentMemoryCleanupError = class extends AggregateError {
|
|
1445
2179
|
constructor(errors, message) {
|
|
@@ -1487,12 +2221,8 @@ function mergeScopes$2(base, extra) {
|
|
|
1487
2221
|
}
|
|
1488
2222
|
};
|
|
1489
2223
|
}
|
|
1490
|
-
function memoryExperimentBaseScope(
|
|
1491
|
-
return mergeScopes$2(candidate.baseScope
|
|
1492
|
-
memoryExperimentId: options.experimentId,
|
|
1493
|
-
memoryCandidateId: candidate.id,
|
|
1494
|
-
memorySequenceId: sequenceId
|
|
1495
|
-
} });
|
|
2224
|
+
function memoryExperimentBaseScope(candidate) {
|
|
2225
|
+
return mergeScopes$2(candidate.baseScope);
|
|
1496
2226
|
}
|
|
1497
2227
|
function sequenceCleanupScopes(sequence) {
|
|
1498
2228
|
const scopes = /* @__PURE__ */ new Map();
|
|
@@ -1519,40 +2249,53 @@ async function clearSequenceScopes(memory, sequence) {
|
|
|
1519
2249
|
//#endregion
|
|
1520
2250
|
//#region src/memory/experiment/recovery.ts
|
|
1521
2251
|
async function recoverAbandonedMemoryAttempts(input) {
|
|
2252
|
+
input.options.signal?.throwIfAborted();
|
|
1522
2253
|
let attempts = readActiveMemoryAttempts(input.storage, input.attemptLogPath);
|
|
1523
2254
|
if (attempts.length > input.maxRecoveryAttempts) throw new Error(`memory experiment has ${attempts.length} unfinished attempts; maxRecoveryAttempts is ${input.maxRecoveryAttempts}`);
|
|
1524
2255
|
for (const attempt of attempts) {
|
|
1525
2256
|
const candidate = input.candidateById.get(attempt.candidateId);
|
|
1526
2257
|
if (!candidate) throw new Error(`cannot recover memory branch '${attempt.branchId}': candidate '${attempt.candidateId}' is missing; pass it in recoveryCandidates`);
|
|
1527
2258
|
assertMemoryAttemptCandidateMatches(attempt, candidate);
|
|
1528
|
-
|
|
2259
|
+
const sequence = input.sequenceById.get(attempt.sequenceId);
|
|
2260
|
+
if (!sequence) throw new Error(`cannot recover memory branch '${attempt.branchId}': sequence '${attempt.sequenceId}' is missing`);
|
|
2261
|
+
assertMemoryAttemptSequenceMatches(attempt, sequence);
|
|
1529
2262
|
if ((input.options.cleanupBranches ?? true) !== attempt.cleanupBranches) throw new Error(`cannot recover memory branch '${attempt.branchId}': cleanupBranches changed`);
|
|
1530
2263
|
}
|
|
1531
2264
|
reconcileInterruptedMemoryPaidCalls(input.costLedger);
|
|
1532
2265
|
assertNoInterruptedPaidCalls(input.costLedger, "memory experiment recovery");
|
|
1533
|
-
for (const attempt of attempts) if ((input.candidateById.get(attempt.candidateId).externalCostUsdPerSequence ?? 0) > 0 && !hasSettledPaidCall(input.costLedger, memoryAttemptCostCallId(attempt, "execute", 0)))
|
|
1534
|
-
|
|
1535
|
-
|
|
1536
|
-
|
|
1537
|
-
|
|
1538
|
-
|
|
2266
|
+
for (const attempt of attempts) if ((input.candidateById.get(attempt.candidateId).externalCostUsdPerSequence ?? 0) > 0 && !hasSettledPaidCall(input.costLedger, memoryAttemptCostCallId(attempt, "execute", 0))) {
|
|
2267
|
+
await input.lease.assertOwned();
|
|
2268
|
+
appendMemoryAttemptEvent(input.storage, input.attemptLogPath, {
|
|
2269
|
+
...attempt,
|
|
2270
|
+
status: "cleaned",
|
|
2271
|
+
recovery: true,
|
|
2272
|
+
recordedAt: (input.options.now ?? (() => /* @__PURE__ */ new Date()))().toISOString()
|
|
2273
|
+
});
|
|
2274
|
+
}
|
|
1539
2275
|
attempts = readActiveMemoryAttempts(input.storage, input.attemptLogPath);
|
|
1540
|
-
const recoveryGenerations =
|
|
1541
|
-
|
|
1542
|
-
|
|
1543
|
-
|
|
1544
|
-
|
|
1545
|
-
|
|
1546
|
-
|
|
1547
|
-
|
|
2276
|
+
const recoveryGenerations = /* @__PURE__ */ new Map();
|
|
2277
|
+
for (const attempt of attempts.sort((left, right) => left.branchId.localeCompare(right.branchId))) {
|
|
2278
|
+
await input.lease.assertOwned();
|
|
2279
|
+
const reserved = reserveRecoveryAttempts({
|
|
2280
|
+
storage: input.storage,
|
|
2281
|
+
path: input.recoveryLogPath,
|
|
2282
|
+
attemptIds: [attempt.branchId],
|
|
2283
|
+
maxRetriesPerAttempt: input.maxRecoveryRetriesPerAttempt,
|
|
2284
|
+
label: "memory recovery attempt log",
|
|
2285
|
+
now: input.options.now
|
|
2286
|
+
});
|
|
2287
|
+
recoveryGenerations.set(attempt.branchId, reserved.get(attempt.branchId));
|
|
2288
|
+
}
|
|
1548
2289
|
const pool = createMemoryExecutionPool(input.maxConcurrency);
|
|
1549
|
-
const failures = (await Promise.allSettled(attempts.
|
|
2290
|
+
const failures = (await Promise.allSettled(attempts.map((attempt) => pool.run(async () => {
|
|
2291
|
+
input.options.signal?.throwIfAborted();
|
|
1550
2292
|
await input.lease.assertOwned();
|
|
1551
2293
|
const candidate = input.candidateById.get(attempt.candidateId);
|
|
1552
2294
|
if (!candidate) throw new Error(`cannot recover memory branch '${attempt.branchId}': candidate '${attempt.candidateId}' is missing; pass it in recoveryCandidates`);
|
|
1553
2295
|
assertMemoryAttemptCandidateMatches(attempt, candidate);
|
|
1554
2296
|
const sequence = input.sequenceById.get(attempt.sequenceId);
|
|
1555
2297
|
if (!sequence) throw new Error(`cannot recover memory branch '${attempt.branchId}': sequence '${attempt.sequenceId}' is missing`);
|
|
2298
|
+
assertMemoryAttemptSequenceMatches(attempt, sequence);
|
|
1556
2299
|
if ((input.options.cleanupBranches ?? true) !== attempt.cleanupBranches) throw new Error(`cannot recover memory branch '${attempt.branchId}': cleanupBranches changed`);
|
|
1557
2300
|
const recoveryCostUsd = candidate.externalRecoveryCostUsdPerAttempt ?? 0;
|
|
1558
2301
|
const recoveryGeneration = recoveryGenerations.get(attempt.branchId);
|
|
@@ -1578,6 +2321,7 @@ async function recoverAbandonedMemoryAttempts(input) {
|
|
|
1578
2321
|
costRecorder.record(actualCostUsd);
|
|
1579
2322
|
}
|
|
1580
2323
|
});
|
|
2324
|
+
await input.lease.assertOwned();
|
|
1581
2325
|
appendMemoryAttemptEvent(input.storage, input.attemptLogPath, {
|
|
1582
2326
|
...attempt,
|
|
1583
2327
|
status: "cleaned",
|
|
@@ -1615,34 +2359,30 @@ async function recoverMemoryAttempt(input) {
|
|
|
1615
2359
|
let memory;
|
|
1616
2360
|
let primaryError;
|
|
1617
2361
|
try {
|
|
1618
|
-
|
|
1619
|
-
const
|
|
1620
|
-
|
|
1621
|
-
|
|
1622
|
-
signal:
|
|
1623
|
-
|
|
1624
|
-
|
|
1625
|
-
|
|
1626
|
-
|
|
1627
|
-
|
|
1628
|
-
|
|
1629
|
-
|
|
2362
|
+
options.signal?.throwIfAborted();
|
|
2363
|
+
const recovered = await createBoundedMemoryAdapter({
|
|
2364
|
+
operation: `${candidate.id}: recovery adapter creation`,
|
|
2365
|
+
timeoutMs: cleanupTimeoutMs,
|
|
2366
|
+
signal: options.signal,
|
|
2367
|
+
create: (signal) => candidate.createAdapter({
|
|
2368
|
+
branchId: attempt.branchId,
|
|
2369
|
+
purpose: "recovery",
|
|
2370
|
+
signal,
|
|
2371
|
+
maximumCostUsd: candidate.externalRecoveryCostUsdPerAttempt ?? 0,
|
|
2372
|
+
markExternalCall: onExternalCall,
|
|
2373
|
+
recordExternalCost
|
|
2374
|
+
}),
|
|
1630
2375
|
dispose: candidate.disposeAdapter ? async (created) => {
|
|
1631
2376
|
onExternalCall();
|
|
1632
2377
|
await candidate.disposeAdapter?.(created);
|
|
1633
2378
|
} : void 0
|
|
1634
2379
|
});
|
|
1635
|
-
|
|
1636
|
-
operation: `${candidate.id}: recovery adapter creation`,
|
|
1637
|
-
timeoutMs: cleanupTimeoutMs,
|
|
1638
|
-
abortController,
|
|
1639
|
-
run: () => creation
|
|
1640
|
-
});
|
|
2380
|
+
options.signal?.throwIfAborted();
|
|
1641
2381
|
await lease.assertOwned();
|
|
1642
2382
|
if (recovered === null) return;
|
|
1643
2383
|
rawAdapter = recovered;
|
|
1644
2384
|
adapter = trackExternalMemoryCalls(recovered, onExternalCall);
|
|
1645
|
-
await sleepForMemoryRecovery(memoryRecoveryDelayMs(adapter), () => lease.assertOwned(), cleanupTimeoutMs, `${candidate.id}: abandoned branch recovery visibility wait
|
|
2385
|
+
await sleepForMemoryRecovery(memoryRecoveryDelayMs(adapter), () => lease.assertOwned(), cleanupTimeoutMs, `${candidate.id}: abandoned branch recovery visibility wait`, options.signal);
|
|
1646
2386
|
if (cleanupBranches) {
|
|
1647
2387
|
if (!adapter.clear) throw new Error(`${candidate.id}: abandoned branch recovery requires an adapter with scoped clear support`);
|
|
1648
2388
|
memory = createAgentMemoryBranch({
|
|
@@ -1651,7 +2391,7 @@ async function recoverMemoryAttempt(input) {
|
|
|
1651
2391
|
lifetime: "attempt",
|
|
1652
2392
|
policy: candidate.policy,
|
|
1653
2393
|
allowedWriteScopes: sequenceCleanupScopes(sequence),
|
|
1654
|
-
baseScope: memoryExperimentBaseScope(
|
|
2394
|
+
baseScope: memoryExperimentBaseScope(candidate)
|
|
1655
2395
|
});
|
|
1656
2396
|
await runBoundedMemoryLifecycle({
|
|
1657
2397
|
operation: `${candidate.id}: abandoned branch cleanup`,
|
|
@@ -1713,6 +2453,7 @@ function memoryAttemptCostCallId(attempt, purpose, generation) {
|
|
|
1713
2453
|
externalCostUsdPerSequence: attempt.externalCostUsdPerSequence,
|
|
1714
2454
|
externalRecoveryCostUsdPerAttempt: attempt.externalRecoveryCostUsdPerAttempt,
|
|
1715
2455
|
sequenceId: attempt.sequenceId,
|
|
2456
|
+
sequenceRef: attempt.sequenceRef,
|
|
1716
2457
|
rep: attempt.rep,
|
|
1717
2458
|
seed: attempt.seed
|
|
1718
2459
|
}));
|
|
@@ -1732,6 +2473,7 @@ function memoryAttemptEvent(input) {
|
|
|
1732
2473
|
candidateId: input.candidate.id,
|
|
1733
2474
|
candidateRef: input.candidate.ref,
|
|
1734
2475
|
sequenceId: input.sequence.id,
|
|
2476
|
+
sequenceRef: canonicalDigest(input.sequence),
|
|
1735
2477
|
rep: input.rep,
|
|
1736
2478
|
seed: input.seed,
|
|
1737
2479
|
cleanupBranches: input.cleanupBranches ?? true,
|
|
@@ -1761,11 +2503,14 @@ function readActiveMemoryAttempts(storage, path) {
|
|
|
1761
2503
|
}
|
|
1762
2504
|
function parseMemoryAttemptEvent(value, path, line) {
|
|
1763
2505
|
const event = value;
|
|
1764
|
-
if (!(typeof event === "object" && event !== null && (event.status === "started" || event.status === "cleaned") && typeof event.branchId === "string" && event.branchId.length > 0 && typeof event.candidateId === "string" && event.candidateId.length > 0 && typeof event.candidateRef === "string" && event.candidateRef.length > 0 && typeof event.sequenceId === "string" && event.sequenceId.length > 0 && Number.isSafeInteger(event.rep) && Number.isSafeInteger(event.seed) && typeof event.cleanupBranches === "boolean" && typeof event.externalCostUsdPerSequence === "number" && Number.isFinite(event.externalCostUsdPerSequence) && event.externalCostUsdPerSequence >= 0 && typeof event.externalRecoveryCostUsdPerAttempt === "number" && Number.isFinite(event.externalRecoveryCostUsdPerAttempt) && event.externalRecoveryCostUsdPerAttempt >= 0 && typeof event.recordedAt === "string" && !Number.isNaN(Date.parse(event.recordedAt)) && typeof event.recovery === "boolean")) throw new Error(`invalid memory attempt event in '${path}' line ${line}`);
|
|
2506
|
+
if (!(typeof event === "object" && event !== null && (event.status === "started" || event.status === "cleaned") && typeof event.branchId === "string" && event.branchId.length > 0 && typeof event.candidateId === "string" && event.candidateId.length > 0 && typeof event.candidateRef === "string" && event.candidateRef.length > 0 && typeof event.sequenceId === "string" && event.sequenceId.length > 0 && typeof event.sequenceRef === "string" && /^sha256:[a-f0-9]{64}$/.test(event.sequenceRef) && Number.isSafeInteger(event.rep) && Number.isSafeInteger(event.seed) && typeof event.cleanupBranches === "boolean" && typeof event.externalCostUsdPerSequence === "number" && Number.isFinite(event.externalCostUsdPerSequence) && event.externalCostUsdPerSequence >= 0 && typeof event.externalRecoveryCostUsdPerAttempt === "number" && Number.isFinite(event.externalRecoveryCostUsdPerAttempt) && event.externalRecoveryCostUsdPerAttempt >= 0 && typeof event.recordedAt === "string" && !Number.isNaN(Date.parse(event.recordedAt)) && typeof event.recovery === "boolean")) throw new Error(`invalid memory attempt event in '${path}' line ${line}`);
|
|
1765
2507
|
return value;
|
|
1766
2508
|
}
|
|
1767
2509
|
function sameMemoryAttempt(left, right) {
|
|
1768
|
-
return left.branchId === right.branchId && left.candidateId === right.candidateId && left.candidateRef === right.candidateRef && left.sequenceId === right.sequenceId && left.rep === right.rep && left.seed === right.seed && left.cleanupBranches === right.cleanupBranches && left.externalCostUsdPerSequence === right.externalCostUsdPerSequence && left.externalRecoveryCostUsdPerAttempt === right.externalRecoveryCostUsdPerAttempt;
|
|
2510
|
+
return left.branchId === right.branchId && left.candidateId === right.candidateId && left.candidateRef === right.candidateRef && left.sequenceId === right.sequenceId && left.sequenceRef === right.sequenceRef && left.rep === right.rep && left.seed === right.seed && left.cleanupBranches === right.cleanupBranches && left.externalCostUsdPerSequence === right.externalCostUsdPerSequence && left.externalRecoveryCostUsdPerAttempt === right.externalRecoveryCostUsdPerAttempt;
|
|
2511
|
+
}
|
|
2512
|
+
function assertMemoryAttemptSequenceMatches(attempt, sequence) {
|
|
2513
|
+
if (canonicalDigest(sequence) !== attempt.sequenceRef) throw new Error(`cannot recover memory branch '${attempt.branchId}': sequence '${attempt.sequenceId}' changed; restore the recorded sequence before cleanup`);
|
|
1769
2514
|
}
|
|
1770
2515
|
function assertMemoryAttemptCandidateMatches(attempt, candidate) {
|
|
1771
2516
|
if (candidate.ref !== attempt.candidateRef) throw new Error(`cannot recover memory branch '${attempt.branchId}': candidate ref changed from '${attempt.candidateRef}' to '${candidate.ref}'`);
|
|
@@ -1775,8 +2520,11 @@ function assertMemoryAttemptCandidateMatches(attempt, candidate) {
|
|
|
1775
2520
|
}
|
|
1776
2521
|
//#endregion
|
|
1777
2522
|
//#region src/memory/experiment/cell.ts
|
|
2523
|
+
var AgentMemoryAdapterCapabilityError = class extends Error {
|
|
2524
|
+
name = "AgentMemoryAdapterCapabilityError";
|
|
2525
|
+
};
|
|
1778
2526
|
async function runSequenceCell(input) {
|
|
1779
|
-
const { options, candidate, scenario, context, runIdentity, storage, attemptLogPath, lease } = input;
|
|
2527
|
+
const { options, candidate, scenario, context, runIdentity, storage, attemptLogPath, lease, memoryMode, comparisonRef } = input;
|
|
1780
2528
|
const cleanupBranches = options.cleanupBranches ?? true;
|
|
1781
2529
|
const costUsd = candidate.externalCostUsdPerSequence ?? 0;
|
|
1782
2530
|
if (!Number.isFinite(costUsd) || costUsd < 0) throw new Error(`${candidate.id}: externalCostUsdPerSequence must be a non-negative finite number`);
|
|
@@ -1797,6 +2545,7 @@ async function runSequenceCell(input) {
|
|
|
1797
2545
|
cleanupBranches,
|
|
1798
2546
|
now: options.now
|
|
1799
2547
|
});
|
|
2548
|
+
await lease.assertOwned();
|
|
1800
2549
|
appendMemoryAttemptEvent(storage, attemptLogPath, attempt);
|
|
1801
2550
|
let externalCallAttempted = false;
|
|
1802
2551
|
const costRecorder = createAgentMemoryCostRecorder({
|
|
@@ -1826,18 +2575,27 @@ async function runSequenceCell(input) {
|
|
|
1826
2575
|
let finalClearStarted = false;
|
|
1827
2576
|
let finalClearCompleted = false;
|
|
1828
2577
|
try {
|
|
1829
|
-
const created = await
|
|
1830
|
-
|
|
1831
|
-
|
|
2578
|
+
const created = await createBoundedMemoryAdapter({
|
|
2579
|
+
operation: `${candidate.id}: execution adapter creation`,
|
|
2580
|
+
timeoutMs: cleanupTimeoutMs,
|
|
1832
2581
|
signal: context.signal,
|
|
1833
|
-
|
|
1834
|
-
|
|
1835
|
-
|
|
1836
|
-
|
|
1837
|
-
|
|
2582
|
+
create: (signal) => candidate.createAdapter({
|
|
2583
|
+
branchId,
|
|
2584
|
+
purpose: "execute",
|
|
2585
|
+
signal,
|
|
2586
|
+
maximumCostUsd: costUsd,
|
|
2587
|
+
markExternalCall: () => {
|
|
2588
|
+
externalCallAttempted = true;
|
|
2589
|
+
},
|
|
2590
|
+
recordExternalCost: (actualCostUsd) => {
|
|
2591
|
+
externalCallAttempted = true;
|
|
2592
|
+
costRecorder.record(actualCostUsd);
|
|
2593
|
+
}
|
|
2594
|
+
}),
|
|
2595
|
+
dispose: candidate.disposeAdapter ? async (lateAdapter) => {
|
|
1838
2596
|
externalCallAttempted = true;
|
|
1839
|
-
|
|
1840
|
-
}
|
|
2597
|
+
await candidate.disposeAdapter?.(lateAdapter);
|
|
2598
|
+
} : void 0
|
|
1841
2599
|
});
|
|
1842
2600
|
if (!created) throw new Error(`${candidate.id}: createAdapter returned no execution adapter`);
|
|
1843
2601
|
rawAdapter = created;
|
|
@@ -1846,6 +2604,7 @@ async function runSequenceCell(input) {
|
|
|
1846
2604
|
});
|
|
1847
2605
|
await lease.assertOwned();
|
|
1848
2606
|
context.signal.throwIfAborted();
|
|
2607
|
+
if (memoryMode === "stateless" && !adapter.clear) throw new AgentMemoryAdapterCapabilityError(`${candidate.id}: stateless memoryMode requires scoped clear support`);
|
|
1849
2608
|
if (cleanupBranches && !adapter.clear) throw new Error(`${candidate.id}: cleanupBranches requires an adapter with scoped clear support`);
|
|
1850
2609
|
memory = createAgentMemoryBranch({
|
|
1851
2610
|
adapter,
|
|
@@ -1853,40 +2612,67 @@ async function runSequenceCell(input) {
|
|
|
1853
2612
|
lifetime: "attempt",
|
|
1854
2613
|
policy: candidate.policy,
|
|
1855
2614
|
allowedWriteScopes: sequenceCleanupScopes(scenario.sequence),
|
|
1856
|
-
baseScope: memoryExperimentBaseScope(
|
|
2615
|
+
baseScope: memoryExperimentBaseScope(candidate)
|
|
2616
|
+
});
|
|
2617
|
+
const runProviderOperation = (operation, run) => runBoundedMemoryLifecycle({
|
|
2618
|
+
operation,
|
|
2619
|
+
timeoutMs: cleanupTimeoutMs,
|
|
2620
|
+
resource: adapter,
|
|
2621
|
+
signal: context.signal,
|
|
2622
|
+
cancellationTimeoutMs: MEMORY_OPERATION_CANCELLATION_TIMEOUT_MS,
|
|
2623
|
+
run
|
|
1857
2624
|
});
|
|
1858
2625
|
const probes = [];
|
|
1859
2626
|
for (const [ordinal, step] of scenario.sequence.steps.entries()) {
|
|
1860
2627
|
context.signal.throwIfAborted();
|
|
1861
2628
|
await lease.assertOwned();
|
|
1862
|
-
|
|
2629
|
+
if (memoryMode === "stateless" && ordinal > 0) {
|
|
2630
|
+
await runBoundedMemoryLifecycle({
|
|
2631
|
+
operation: `${candidate.id}: stateless reset before step ${step.id}`,
|
|
2632
|
+
timeoutMs: cleanupTimeoutMs,
|
|
2633
|
+
resource: adapter,
|
|
2634
|
+
run: () => clearSequenceScopes(memory, scenario.sequence)
|
|
2635
|
+
});
|
|
2636
|
+
context.signal.throwIfAborted();
|
|
2637
|
+
await lease.assertOwned();
|
|
2638
|
+
}
|
|
2639
|
+
await runProviderOperation(`${candidate.id}: writes for step ${step.id}`, () => writeStep(memory, step));
|
|
1863
2640
|
await lease.assertOwned();
|
|
1864
2641
|
if (options.executeStep) {
|
|
1865
2642
|
const execution = createAgentMemoryExecutionContext(context, scenario.sequence);
|
|
1866
2643
|
try {
|
|
1867
|
-
await options.executeStep({
|
|
2644
|
+
await runProviderOperation(`${candidate.id}: execution for step ${step.id}`, () => options.executeStep({
|
|
1868
2645
|
memory,
|
|
1869
2646
|
candidateId: candidate.id,
|
|
1870
2647
|
step: executionStep(step, ordinal),
|
|
1871
2648
|
context: execution.context
|
|
1872
|
-
});
|
|
2649
|
+
}));
|
|
2650
|
+
} catch (error) {
|
|
2651
|
+
execution.abort();
|
|
2652
|
+
throw error;
|
|
1873
2653
|
} finally {
|
|
1874
2654
|
execution.dispose();
|
|
1875
2655
|
}
|
|
1876
2656
|
}
|
|
1877
2657
|
context.signal.throwIfAborted();
|
|
1878
2658
|
await lease.assertOwned();
|
|
1879
|
-
const stepProbes = await probeStep(memory, scenario.sequence, step);
|
|
2659
|
+
const stepProbes = await runProviderOperation(`${candidate.id}: probes for step ${step.id}`, () => probeStep(memory, scenario.sequence, step, ordinal, context));
|
|
1880
2660
|
await lease.assertOwned();
|
|
1881
2661
|
probes.push(...stepProbes);
|
|
1882
2662
|
}
|
|
1883
|
-
const snapshot = await memory.snapshot();
|
|
2663
|
+
const snapshot = await runProviderOperation(`${candidate.id}: branch snapshot`, () => memory.snapshot());
|
|
1884
2664
|
await lease.assertOwned();
|
|
1885
|
-
|
|
1886
|
-
|
|
1887
|
-
|
|
1888
|
-
|
|
1889
|
-
|
|
2665
|
+
if (options.onBranchSnapshot) await runBoundedMemoryLifecycle({
|
|
2666
|
+
operation: `${candidate.id}: branch snapshot callback`,
|
|
2667
|
+
timeoutMs: cleanupTimeoutMs,
|
|
2668
|
+
signal: context.signal,
|
|
2669
|
+
cancellationTimeoutMs: MEMORY_OPERATION_CANCELLATION_TIMEOUT_MS,
|
|
2670
|
+
run: () => options.onBranchSnapshot({
|
|
2671
|
+
candidateId: candidate.id,
|
|
2672
|
+
sequenceId: scenario.sequenceId,
|
|
2673
|
+
cellId: context.cellId,
|
|
2674
|
+
snapshot
|
|
2675
|
+
})
|
|
1890
2676
|
});
|
|
1891
2677
|
await lease.assertOwned();
|
|
1892
2678
|
const dimensions = meanDimensions(probes.map((probe) => probe.dimensions));
|
|
@@ -1894,6 +2680,8 @@ async function runSequenceCell(input) {
|
|
|
1894
2680
|
const artifact = {
|
|
1895
2681
|
candidateId: candidate.id,
|
|
1896
2682
|
sequenceId: scenario.sequenceId,
|
|
2683
|
+
memoryMode,
|
|
2684
|
+
comparisonRef,
|
|
1897
2685
|
score: mean$1(probes.map((probe) => probe.score)),
|
|
1898
2686
|
passed: probes.length > 0 && probes.every((probe) => probe.passed),
|
|
1899
2687
|
dimensions,
|
|
@@ -1969,6 +2757,12 @@ async function runSequenceCell(input) {
|
|
|
1969
2757
|
cleanupErrors.push(error);
|
|
1970
2758
|
}
|
|
1971
2759
|
} else cleanupErrors.push(/* @__PURE__ */ new Error(`${candidate.id}: adapter creation failed before cleanup could be confirmed`));
|
|
2760
|
+
if (cleanupOwned && cleanupErrors.length === 0) try {
|
|
2761
|
+
await lease.assertOwned();
|
|
2762
|
+
} catch (error) {
|
|
2763
|
+
cleanupOwned = false;
|
|
2764
|
+
ownershipError = error;
|
|
2765
|
+
}
|
|
1972
2766
|
if (cleanupOwned && cleanupErrors.length === 0) appendCleanedAttempt(primaryError);
|
|
1973
2767
|
if (!cleanupOwned && cleanupErrors.length === 0) {
|
|
1974
2768
|
if (primaryError) throw primaryError;
|
|
@@ -2028,46 +2822,38 @@ async function writeStep(memory, step) {
|
|
|
2028
2822
|
}
|
|
2029
2823
|
for (const write of writes) await memory.write(write);
|
|
2030
2824
|
}
|
|
2031
|
-
async function probeStep(memory, sequence, step) {
|
|
2825
|
+
async function probeStep(memory, sequence, step, stepOrdinal, dispatch) {
|
|
2032
2826
|
const run = async (probe) => {
|
|
2033
2827
|
const scope = mergeScopes$2(step.scope, probe.scope);
|
|
2034
2828
|
const context = await memory.getContext(probe.query, {
|
|
2035
2829
|
scope,
|
|
2036
2830
|
limit: probe.limit
|
|
2037
2831
|
});
|
|
2038
|
-
const
|
|
2832
|
+
const scoringEvidence = {
|
|
2039
2833
|
answer: context.text,
|
|
2040
2834
|
rememberedFacts: context.hits.map((hit) => hit.text),
|
|
2041
2835
|
citedEventIds: uniqueStrings$1([...context.hits.map((hit) => hit.metadata?.eventId), ...context.hits.flatMap((hit) => stringArray$1(hit.metadata?.eventIds))]),
|
|
2042
2836
|
usedMemoryIds: context.hits.map((hit) => hit.id),
|
|
2043
2837
|
actorIds: uniqueStrings$1([...context.hits.map((hit) => hit.metadata?.actorId), ...context.hits.flatMap((hit) => stringArray$1(hit.metadata?.actorIds))])
|
|
2044
2838
|
};
|
|
2045
|
-
const
|
|
2046
|
-
|
|
2047
|
-
|
|
2048
|
-
taskKind: probe.taskKind ?? "memory-recall",
|
|
2049
|
-
split: sequence.split,
|
|
2050
|
-
events: [],
|
|
2051
|
-
prompt: probe.query,
|
|
2052
|
-
requiredFacts: probe.requiredFacts && probe.requiredFacts.length > 0 ? probe.requiredFacts : probe.referenceAnswer ? [{
|
|
2053
|
-
id: `${probe.id}:reference`,
|
|
2054
|
-
anyOf: [probe.referenceAnswer]
|
|
2055
|
-
}] : void 0,
|
|
2056
|
-
forbiddenFacts: probe.forbiddenFacts,
|
|
2057
|
-
expectedEventIds: probe.expectedEventIds,
|
|
2058
|
-
expectedActorIds: probe.expectedActorIds,
|
|
2059
|
-
referenceAnswer: probe.referenceAnswer
|
|
2060
|
-
}, artifact);
|
|
2839
|
+
const evidenceRef = canonicalDigest(scoringEvidence);
|
|
2840
|
+
const evidencePath = await dispatch.artifacts.writeJson(`memory-evidence/${evidenceRef.slice(7)}.json`, scoringEvidence);
|
|
2841
|
+
const evaluation = scoreAgentMemoryProbe(sequence, step, probe, scoringEvidence);
|
|
2061
2842
|
return {
|
|
2062
2843
|
id: probe.id,
|
|
2063
2844
|
stepId: step.id,
|
|
2845
|
+
stepOrdinal,
|
|
2846
|
+
...probe.retentionKey !== void 0 ? { retentionKey: probe.retentionKey } : {},
|
|
2847
|
+
...probe.transferKey !== void 0 ? { transferKey: probe.transferKey } : {},
|
|
2064
2848
|
query: probe.query,
|
|
2065
2849
|
score: evaluation.score,
|
|
2066
2850
|
passed: evaluation.passed,
|
|
2067
2851
|
dimensions: evaluation.dimensions,
|
|
2068
2852
|
applicableDimensions: evaluation.applicableDimensions ?? Object.keys(evaluation.dimensions),
|
|
2069
2853
|
notes: evaluation.notes,
|
|
2070
|
-
hitIds: context.hits.map((hit) => hit.id)
|
|
2854
|
+
hitIds: context.hits.map((hit) => hit.id),
|
|
2855
|
+
evidenceRef,
|
|
2856
|
+
evidencePath
|
|
2071
2857
|
};
|
|
2072
2858
|
};
|
|
2073
2859
|
if (step.parallelProbes === false) {
|
|
@@ -2079,12 +2865,13 @@ async function probeStep(memory, sequence, step) {
|
|
|
2079
2865
|
}
|
|
2080
2866
|
//#endregion
|
|
2081
2867
|
//#region src/memory/experiment/run.ts
|
|
2082
|
-
const MEMORY_EXPERIMENT_IMPLEMENTATION_REF = "agent-knowledge:memory-experiment:v6";
|
|
2083
2868
|
/** Runs ordered, branch-isolated memory histories across candidate systems. */
|
|
2084
2869
|
async function runAgentMemoryExperiment(options) {
|
|
2870
|
+
options.signal?.throwIfAborted();
|
|
2085
2871
|
assertNonEmptyString(options.experimentId, "memory experiment experimentId");
|
|
2086
2872
|
assertNonEmptyString(options.runDir, "memory experiment runDir");
|
|
2087
2873
|
if (options.experimentRunId !== void 0) assertNonEmptyString(options.experimentRunId, "memory experiment experimentRunId");
|
|
2874
|
+
resolveAgentMemoryMode(options.memoryMode);
|
|
2088
2875
|
if (options.sequences.length === 0) throw new Error("memory experiment requires sequences");
|
|
2089
2876
|
if (options.candidates.length === 0) throw new Error("memory experiment requires candidates");
|
|
2090
2877
|
if (options.executeStep && !options.executeStepRef) throw new Error("memory experiment executeStepRef is required when executeStep is configured");
|
|
@@ -2139,9 +2926,12 @@ async function runAgentMemoryExperiment(options) {
|
|
|
2139
2926
|
return result;
|
|
2140
2927
|
}
|
|
2141
2928
|
async function runOwnedAgentMemoryExperiment(options, storage, runDir, lease) {
|
|
2929
|
+
const memoryMode = resolveAgentMemoryMode(options.memoryMode);
|
|
2930
|
+
const comparisonRef = memoryExperimentComparisonRef(options, runDir);
|
|
2142
2931
|
const runIdentity = stableId("memory_run", canonicalJson({
|
|
2143
2932
|
experimentId: options.experimentId,
|
|
2144
|
-
experimentRunId: options.experimentRunId ?? runDir
|
|
2933
|
+
experimentRunId: options.experimentRunId ?? runDir,
|
|
2934
|
+
memoryMode
|
|
2145
2935
|
}));
|
|
2146
2936
|
const maxConcurrency = options.maxConcurrency ?? 2;
|
|
2147
2937
|
if (!Number.isSafeInteger(maxConcurrency) || maxConcurrency <= 0) throw new Error("memory experiment maxConcurrency must be a positive safe integer");
|
|
@@ -2175,6 +2965,7 @@ async function runOwnedAgentMemoryExperiment(options, storage, runDir, lease) {
|
|
|
2175
2965
|
recoveryLogPath,
|
|
2176
2966
|
maxRecoveryRetriesPerAttempt: options.maxRecoveryRetriesPerAttempt ?? 3
|
|
2177
2967
|
});
|
|
2968
|
+
options.signal?.throwIfAborted();
|
|
2178
2969
|
await lease.assertOwned();
|
|
2179
2970
|
let campaign;
|
|
2180
2971
|
let campaignError;
|
|
@@ -2182,6 +2973,7 @@ async function runOwnedAgentMemoryExperiment(options, storage, runDir, lease) {
|
|
|
2182
2973
|
try {
|
|
2183
2974
|
campaign = await runCampaign({
|
|
2184
2975
|
scenarios,
|
|
2976
|
+
signal: options.signal,
|
|
2185
2977
|
dispatch: (scenario, context) => {
|
|
2186
2978
|
const candidate = candidateById.get(scenario.candidateId);
|
|
2187
2979
|
if (!candidate) throw new Error(`unknown memory candidate ${scenario.candidateId}`);
|
|
@@ -2193,12 +2985,14 @@ async function runOwnedAgentMemoryExperiment(options, storage, runDir, lease) {
|
|
|
2193
2985
|
runIdentity,
|
|
2194
2986
|
storage,
|
|
2195
2987
|
attemptLogPath,
|
|
2196
|
-
lease
|
|
2988
|
+
lease,
|
|
2989
|
+
memoryMode,
|
|
2990
|
+
comparisonRef
|
|
2197
2991
|
}));
|
|
2198
2992
|
dispatchedExecutions.push(operation);
|
|
2199
2993
|
return operation;
|
|
2200
2994
|
},
|
|
2201
|
-
dispatchRef: memoryExperimentDispatchRef(
|
|
2995
|
+
dispatchRef: memoryExperimentDispatchRef(memoryMode, comparisonRef),
|
|
2202
2996
|
judges: [agentMemorySequenceJudge()],
|
|
2203
2997
|
runDir,
|
|
2204
2998
|
storage,
|
|
@@ -2210,6 +3004,7 @@ async function runOwnedAgentMemoryExperiment(options, storage, runDir, lease) {
|
|
|
2210
3004
|
costPhase: options.costPhase,
|
|
2211
3005
|
maxConcurrency,
|
|
2212
3006
|
dispatchTimeoutMs: options.dispatchTimeoutMs,
|
|
3007
|
+
dispatchShutdownTimeoutMs: MEMORY_CAMPAIGN_DISPATCH_SHUTDOWN_TIMEOUT_MS,
|
|
2213
3008
|
expectUsage: "off",
|
|
2214
3009
|
now: options.now
|
|
2215
3010
|
});
|
|
@@ -2219,9 +3014,14 @@ async function runOwnedAgentMemoryExperiment(options, storage, runDir, lease) {
|
|
|
2219
3014
|
settledExecutions = await Promise.allSettled(dispatchedExecutions);
|
|
2220
3015
|
}
|
|
2221
3016
|
const cleanupFailures = settledExecutions.flatMap((settled) => settled.status === "rejected" && settled.reason instanceof AgentMemoryCleanupError ? [settled.reason] : []);
|
|
2222
|
-
|
|
3017
|
+
const capabilityFailures = settledExecutions.flatMap((settled) => settled.status === "rejected" && settled.reason instanceof AgentMemoryAdapterCapabilityError ? [settled.reason] : []);
|
|
3018
|
+
const terminalFailures = [...cleanupFailures, ...capabilityFailures];
|
|
3019
|
+
if (campaignError && terminalFailures.length > 0) throw new AggregateError([campaignError, ...terminalFailures], "memory experiment failed with terminal dispatch errors");
|
|
2223
3020
|
if (campaignError) throw campaignError;
|
|
3021
|
+
if (cleanupFailures.length > 0 && capabilityFailures.length > 0) throw new AggregateError(terminalFailures, "memory experiment adapter requirements and provider cleanup failed");
|
|
2224
3022
|
if (cleanupFailures.length > 0) throw new AggregateError(cleanupFailures, "memory experiment cleanup failed after dispatch");
|
|
3023
|
+
if (capabilityFailures.length === 1) throw capabilityFailures[0];
|
|
3024
|
+
if (capabilityFailures.length > 1) throw new AggregateError(capabilityFailures, "memory experiment adapter requirements failed");
|
|
2225
3025
|
if (!campaign) throw new Error("memory experiment produced no campaign result");
|
|
2226
3026
|
await lease.assertOwned();
|
|
2227
3027
|
const costByCandidate = memoryExperimentCostByCandidate(costLedger, campaign.runDir, scenarios, [...options.candidates, ...options.recoveryCandidates ?? []].map((candidate) => candidate.id));
|
|
@@ -2232,12 +3032,26 @@ async function runOwnedAgentMemoryExperiment(options, storage, runDir, lease) {
|
|
|
2232
3032
|
const rankingMarkdownPath = join(campaign.runDir, "memory-experiment-ranking.md");
|
|
2233
3033
|
storage.write(rankingJsonPath, `${JSON.stringify({
|
|
2234
3034
|
experimentId: options.experimentId,
|
|
3035
|
+
memoryMode,
|
|
3036
|
+
comparisonRef,
|
|
3037
|
+
candidateRefs: options.candidates.map(({ id, ref }) => ({
|
|
3038
|
+
id,
|
|
3039
|
+
ref
|
|
3040
|
+
})),
|
|
3041
|
+
executionRef: options.executeStepRef ?? "fixtures",
|
|
2235
3042
|
totalCostUsd,
|
|
2236
3043
|
unrankedRecoveryCostUsd,
|
|
2237
3044
|
rows
|
|
2238
3045
|
}, null, 2)}\n`);
|
|
2239
3046
|
storage.write(rankingMarkdownPath, renderAgentMemoryExperimentRanking(rows, totalCostUsd, unrankedRecoveryCostUsd));
|
|
2240
3047
|
return {
|
|
3048
|
+
memoryMode,
|
|
3049
|
+
comparisonRef,
|
|
3050
|
+
candidateRefs: options.candidates.map(({ id, ref }) => ({
|
|
3051
|
+
id,
|
|
3052
|
+
ref
|
|
3053
|
+
})),
|
|
3054
|
+
executionRef: options.executeStepRef ?? "fixtures",
|
|
2241
3055
|
campaign,
|
|
2242
3056
|
rows,
|
|
2243
3057
|
totalCostUsd,
|
|
@@ -2249,25 +3063,98 @@ async function runOwnedAgentMemoryExperiment(options, storage, runDir, lease) {
|
|
|
2249
3063
|
recoveryLogPath
|
|
2250
3064
|
};
|
|
2251
3065
|
}
|
|
2252
|
-
function memoryExperimentDispatchRef(
|
|
3066
|
+
function memoryExperimentDispatchRef(memoryMode, comparisonRef) {
|
|
2253
3067
|
return stableId("memory_experiment", canonicalJson({
|
|
2254
3068
|
implementationRef: MEMORY_EXPERIMENT_IMPLEMENTATION_REF,
|
|
2255
|
-
|
|
2256
|
-
|
|
2257
|
-
executeStepRef: options.executeStepRef ?? "fixtures",
|
|
2258
|
-
cleanupBranches: options.cleanupBranches ?? true,
|
|
2259
|
-
candidates: options.candidates.map((candidate) => ({
|
|
2260
|
-
id: candidate.id,
|
|
2261
|
-
ref: candidate.ref,
|
|
2262
|
-
policy: candidate.policy ?? null,
|
|
2263
|
-
baseScope: candidate.baseScope ?? null,
|
|
2264
|
-
externalCostUsdPerSequence: candidate.externalCostUsdPerSequence ?? 0,
|
|
2265
|
-
externalRecoveryCostUsdPerAttempt: candidate.externalRecoveryCostUsdPerAttempt ?? 0,
|
|
2266
|
-
externalCostAccounting: candidate.externalCostAccounting ?? "exact"
|
|
2267
|
-
})).sort((a, b) => a.id.localeCompare(b.id))
|
|
3069
|
+
memoryMode,
|
|
3070
|
+
comparisonRef
|
|
2268
3071
|
}));
|
|
2269
3072
|
}
|
|
2270
3073
|
//#endregion
|
|
3074
|
+
//#region src/memory/experiment/learning.ts
|
|
3075
|
+
/** Compare exact matched stateful and stateless memory runs. */
|
|
3076
|
+
function compareAgentMemoryLearning(options) {
|
|
3077
|
+
return measureAgentMemoryLearning(pairAgentMemoryLearningRuns(options), options.stateful, options.stateless);
|
|
3078
|
+
}
|
|
3079
|
+
/** Run a matched stateful-versus-stateless memory experiment under one cost limit. */
|
|
3080
|
+
async function runAgentMemoryLearningExperiment(options) {
|
|
3081
|
+
assertNonEmptyString(options.runDir, "memory learning experiment runDir");
|
|
3082
|
+
options.signal?.throwIfAborted();
|
|
3083
|
+
assertMemoryLearningSequences(options.sequences);
|
|
3084
|
+
for (const candidate of [...options.candidates, ...options.recoveryCandidates ?? []]) assertImmutableRef(candidate.ref, `${candidate.id}: memory learning candidate ref`);
|
|
3085
|
+
if (options.executeStep) assertImmutableRef(options.executeStepRef, "memory learning experiment executeStepRef");
|
|
3086
|
+
const storage = options.storage ?? fsCampaignStorage();
|
|
3087
|
+
const runDir = resolveRunDir(options.runDir, options.repo);
|
|
3088
|
+
storage.ensureDir(runDir);
|
|
3089
|
+
const costCeiling = options.costCeiling ?? options.costLedger?.costCeilingUsd ?? 0;
|
|
3090
|
+
const costLedger = options.costLedger ?? createRunCostLedger({
|
|
3091
|
+
storage,
|
|
3092
|
+
runDir,
|
|
3093
|
+
costCeilingUsd: costCeiling
|
|
3094
|
+
});
|
|
3095
|
+
if (costLedger.costCeilingUsd !== costCeiling) throw new Error("memory learning experiment costCeiling must match the shared cost ledger ceiling");
|
|
3096
|
+
const armOrder = options.armOrder ?? "stateful-first";
|
|
3097
|
+
if (armOrder !== "stateful-first" && armOrder !== "stateless-first") throw new Error("memory learning experiment armOrder must be 'stateful-first' or 'stateless-first'");
|
|
3098
|
+
const { armOrder: _armOrder, ...experimentOptions } = options;
|
|
3099
|
+
const shared = {
|
|
3100
|
+
...experimentOptions,
|
|
3101
|
+
experimentRunId: options.experimentRunId ?? runDir,
|
|
3102
|
+
costCeiling,
|
|
3103
|
+
costLedger
|
|
3104
|
+
};
|
|
3105
|
+
const runArm = async (memoryMode) => {
|
|
3106
|
+
const result = await runAgentMemoryExperiment({
|
|
3107
|
+
...shared,
|
|
3108
|
+
memoryMode,
|
|
3109
|
+
runDir: armRunDir(runDir, memoryMode)
|
|
3110
|
+
});
|
|
3111
|
+
assertStoredAgentMemoryEvidence(storage, result, options.sequences);
|
|
3112
|
+
return result;
|
|
3113
|
+
};
|
|
3114
|
+
const firstMode = armOrder === "stateful-first" ? "stateful" : "stateless";
|
|
3115
|
+
const secondMode = firstMode === "stateful" ? "stateless" : "stateful";
|
|
3116
|
+
const first = await runArm(firstMode);
|
|
3117
|
+
const second = await runArm(secondMode);
|
|
3118
|
+
const stateful = firstMode === "stateful" ? first : second;
|
|
3119
|
+
const stateless = firstMode === "stateless" ? first : second;
|
|
3120
|
+
const comparison = compareAgentMemoryLearning({
|
|
3121
|
+
stateful,
|
|
3122
|
+
stateless
|
|
3123
|
+
});
|
|
3124
|
+
const ledgerSummary = costLedger.summary();
|
|
3125
|
+
const cost = {
|
|
3126
|
+
experimentUsd: normalizeUsd(stateful.totalCostUsd + stateless.totalCostUsd),
|
|
3127
|
+
ledgerUsd: normalizeUsd(ledgerSummary.totalCostUsd),
|
|
3128
|
+
ceilingUsd: costLedger.costCeilingUsd,
|
|
3129
|
+
accountingComplete: ledgerSummary.accountingComplete
|
|
3130
|
+
};
|
|
3131
|
+
const report = {
|
|
3132
|
+
armOrder,
|
|
3133
|
+
comparison,
|
|
3134
|
+
cost
|
|
3135
|
+
};
|
|
3136
|
+
const evidenceRef = canonicalDigest(report);
|
|
3137
|
+
const comparisonDir = armRunDir(runDir, "comparisons");
|
|
3138
|
+
storage.ensureDir(comparisonDir);
|
|
3139
|
+
const comparisonPath = armRunDir(comparisonDir, `${evidenceRef.slice(7)}.json`);
|
|
3140
|
+
storage.write(comparisonPath, `${JSON.stringify({
|
|
3141
|
+
evidenceRef,
|
|
3142
|
+
...report
|
|
3143
|
+
}, null, 2)}\n`);
|
|
3144
|
+
return {
|
|
3145
|
+
armOrder,
|
|
3146
|
+
stateful,
|
|
3147
|
+
stateless,
|
|
3148
|
+
comparison,
|
|
3149
|
+
evidenceRef,
|
|
3150
|
+
comparisonPath,
|
|
3151
|
+
cost
|
|
3152
|
+
};
|
|
3153
|
+
}
|
|
3154
|
+
function armRunDir(runDir, child) {
|
|
3155
|
+
return runDir.startsWith("mem://") ? `${runDir.replace(/\/+$/, "")}/${child}` : join(runDir, child);
|
|
3156
|
+
}
|
|
3157
|
+
//#endregion
|
|
2271
3158
|
//#region src/memory/graphiti.ts
|
|
2272
3159
|
const DEFAULT_GRAPHITI_INGESTION_TIMEOUT_MS = 12e4;
|
|
2273
3160
|
/** Connects the official Graphiti MCP server without coupling to its Python internals. */
|
|
@@ -2949,7 +3836,8 @@ async function evaluateMemoryCandidate(input) {
|
|
|
2949
3836
|
reps: 1,
|
|
2950
3837
|
maxConcurrency: 1,
|
|
2951
3838
|
costCeiling: evaluationCostLimit,
|
|
2952
|
-
costPhase: `memory.config.${input.surfaceHash}
|
|
3839
|
+
costPhase: `memory.config.${input.surfaceHash}`,
|
|
3840
|
+
signal: input.signal
|
|
2953
3841
|
});
|
|
2954
3842
|
const cell = experiment.campaign.cells[0];
|
|
2955
3843
|
if (!cell || cell.error || cell.artifact.candidateId !== input.candidate.id) throw new Error(`${input.surfaceHash}/${input.scenario.sequenceId}: memory config evaluation did not complete`);
|
|
@@ -3083,9 +3971,20 @@ function storedMemoryArtifactRecord(input, artifact) {
|
|
|
3083
3971
|
function parseMemoryArtifact(value, candidateId, sequenceId, path) {
|
|
3084
3972
|
if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error(`memory config artifact '${path}' has no artifact`);
|
|
3085
3973
|
const artifact = value;
|
|
3086
|
-
|
|
3974
|
+
try {
|
|
3975
|
+
assertAgentMemoryExperimentComparisonRef(artifact.comparisonRef, `memory config artifact '${path}' comparisonRef`);
|
|
3976
|
+
} catch {
|
|
3977
|
+
throw new Error(`memory config artifact '${path}' is malformed`);
|
|
3978
|
+
}
|
|
3979
|
+
if (artifact.candidateId !== candidateId || artifact.sequenceId !== sequenceId || artifact.memoryMode !== "stateful" || typeof artifact.score !== "number" || !Number.isFinite(artifact.score) || artifact.score < 0 || artifact.score > 1 || typeof artifact.passed !== "boolean" || !isFiniteNumberRecord(artifact.dimensions) || !isNonnegativeIntegerRecord(artifact.dimensionSampleCounts) || !Array.isArray(artifact.probes) || !artifact.probes.every(isMemoryProbeResult) || typeof artifact.branchDigest !== "string" || !/^sha256:[a-f0-9]{64}$/.test(artifact.branchDigest) || typeof artifact.journalEntries !== "number" || !Number.isSafeInteger(artifact.journalEntries) || artifact.journalEntries < 0 || typeof artifact.durationMs !== "number" || !Number.isFinite(artifact.durationMs) || artifact.durationMs < 0) throw new Error(`memory config artifact '${path}' is malformed`);
|
|
3087
3980
|
return artifact;
|
|
3088
3981
|
}
|
|
3982
|
+
function isMemoryProbeResult(value) {
|
|
3983
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return false;
|
|
3984
|
+
const probe = value;
|
|
3985
|
+
if (typeof probe.id !== "string" || !probe.id || typeof probe.stepId !== "string" || !probe.stepId || !Number.isSafeInteger(probe.stepOrdinal) || probe.stepOrdinal < 0 || typeof probe.query !== "string" || !probe.query || typeof probe.score !== "number" || !Number.isFinite(probe.score) || probe.score < 0 || probe.score > 1 || typeof probe.passed !== "boolean" || !isFiniteNumberRecord(probe.dimensions) || !Array.isArray(probe.applicableDimensions) || !probe.applicableDimensions.every((entry) => typeof entry === "string" && entry.length > 0) || typeof probe.notes !== "string" || !Array.isArray(probe.hitIds) || !probe.hitIds.every((entry) => typeof entry === "string") || typeof probe.evidenceRef !== "string" || !/^sha256:[a-f0-9]{64}$/.test(probe.evidenceRef) || typeof probe.evidencePath !== "string" || !probe.evidencePath.endsWith(`/memory-evidence/${probe.evidenceRef.slice(7)}.json`)) return false;
|
|
3986
|
+
return (probe.retentionKey === void 0 || typeof probe.retentionKey === "string" && probe.retentionKey.length > 0) && (probe.transferKey === void 0 || typeof probe.transferKey === "string" && probe.transferKey.length > 0);
|
|
3987
|
+
}
|
|
3089
3988
|
function isFiniteNumberRecord(value) {
|
|
3090
3989
|
if (!value || typeof value !== "object" || Array.isArray(value)) return false;
|
|
3091
3990
|
return Object.values(value).every((entry) => typeof entry === "number" && Number.isFinite(entry));
|
|
@@ -4489,6 +5388,6 @@ const AgentMemoryWriteInputSchema = z.object({
|
|
|
4489
5388
|
metadata: z.record(z.string(), z.unknown()).optional()
|
|
4490
5389
|
});
|
|
4491
5390
|
//#endregion
|
|
4492
|
-
export {
|
|
5391
|
+
export { runSerializedKnowledgeOptimization as A, deterministicRng as C, toOffPolicyTrajectory as D, retrievalHoldoutConfigHash as E, assertImmutableRef as M, jsonCandidateCodec as O, applySessionStickyRetrievalHoldout as S, resetRetrievalHoldoutRegistry as T, createAgentMemoryBranch as _, createNeo4jAgentMemoryAdapter as a, renderMemoryContext as b, runAgentMemoryImprovement as c, compareAgentMemoryLearning as d, runAgentMemoryLearningExperiment as f, buildAgentMemorySequencesFromBenchmarkCases as g, buildAgentMemorySequenceScenarios as h, AgentMemoryWriteInputSchema as i, scenarioContentFingerprint as j, jsonObjectCandidateCodec as k, createGraphitiMemoryAdapter as l, agentMemorySequenceJudge as m, AgentMemoryKindSchema as n, createMem0MemoryAdapter as o, runAgentMemoryExperiment as p, AgentMemoryScopeSchema as r, mem0MemoryAdapterIdentity as s, AgentMemoryHitSchema as t, graphitiMemoryAdapterIdentity as u, forkAgentMemoryBranchSnapshot as v, emitRetrievalHoldoutBypass as w, applyRetrievalHoldout as x, defaultGetMemoryContext as y };
|
|
4493
5392
|
|
|
4494
|
-
//# sourceMappingURL=memory-
|
|
5393
|
+
//# sourceMappingURL=memory-DAFfGTJH.js.map
|