@tangle-network/agent-knowledge 4.1.0 → 5.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +3 -1
- package/CHANGELOG.md +34 -0
- package/README.md +220 -955
- package/dist/benchmarks/index.d.ts +3 -2
- package/dist/benchmarks/index.js +1 -1
- package/dist/{chunk-BBCIB4UL.js → chunk-EYIA5PLQ.js} +27 -195
- package/dist/chunk-EYIA5PLQ.js.map +1 -0
- package/dist/{chunk-CPMLJYA3.js → chunk-LMR53POQ.js} +1227 -574
- package/dist/chunk-LMR53POQ.js.map +1 -0
- package/dist/index-Cf7txrYP.d.ts +790 -0
- package/dist/index.d.ts +208 -24
- package/dist/index.js +632 -80
- package/dist/index.js.map +1 -1
- package/dist/memory/index.d.ts +8 -706
- package/dist/memory/index.js +2 -2
- package/dist/{types-DP38encz.d.ts → types-BY-xLVw-.d.ts} +16 -59
- package/docs/eval/rag-eval-roadmap.md +15 -12
- package/package.json +8 -5
- package/skills/build-with-agent-knowledge/SKILL.md +97 -0
- package/dist/chunk-BBCIB4UL.js.map +0 -1
- package/dist/chunk-CPMLJYA3.js.map +0 -1
|
@@ -11,14 +11,13 @@ import {
|
|
|
11
11
|
memoryWriteResultToSourceRecord,
|
|
12
12
|
readActiveAttemptJournal,
|
|
13
13
|
reconcileInterruptedMemoryPaidCalls,
|
|
14
|
-
reconcileInterruptedRunPaidCalls,
|
|
15
14
|
releaseMemoryAdapterCreatedAfterAbort,
|
|
16
15
|
reserveRecoveryAttempts,
|
|
17
16
|
resolveMemoryCleanupTimeoutMs,
|
|
18
17
|
runBoundedMemoryLifecycle,
|
|
19
18
|
scoreMemoryBenchmarkArtifact,
|
|
20
19
|
sleepForMemoryRecovery
|
|
21
|
-
} from "./chunk-
|
|
20
|
+
} from "./chunk-EYIA5PLQ.js";
|
|
22
21
|
import {
|
|
23
22
|
sha256,
|
|
24
23
|
stableId
|
|
@@ -1198,6 +1197,165 @@ import {
|
|
|
1198
1197
|
// src/memory/experiment/cell.ts
|
|
1199
1198
|
import { randomUUID as randomUUID2 } from "crypto";
|
|
1200
1199
|
|
|
1200
|
+
// src/memory/experiment/cost.ts
|
|
1201
|
+
function createAgentMemoryCostRecorder(input) {
|
|
1202
|
+
let observedCostUsd = 0;
|
|
1203
|
+
let receiptCount = 0;
|
|
1204
|
+
return {
|
|
1205
|
+
record(actualCostUsd) {
|
|
1206
|
+
if (!Number.isFinite(actualCostUsd) || actualCostUsd < 0) {
|
|
1207
|
+
throw new Error(
|
|
1208
|
+
`${input.operation}: observed external cost must be non-negative and finite`
|
|
1209
|
+
);
|
|
1210
|
+
}
|
|
1211
|
+
const next = observedCostUsd + actualCostUsd;
|
|
1212
|
+
if (exceedsMaximum(next, input.maximumCostUsd)) {
|
|
1213
|
+
throw new Error(
|
|
1214
|
+
`${input.operation}: observed external cost ${next} exceeds declared maximum ${input.maximumCostUsd}`
|
|
1215
|
+
);
|
|
1216
|
+
}
|
|
1217
|
+
observedCostUsd = next;
|
|
1218
|
+
receiptCount += 1;
|
|
1219
|
+
},
|
|
1220
|
+
receipt(externalCallAttempted) {
|
|
1221
|
+
const base = {
|
|
1222
|
+
model: input.candidateRef,
|
|
1223
|
+
inputTokens: 0,
|
|
1224
|
+
outputTokens: 0
|
|
1225
|
+
};
|
|
1226
|
+
if (input.maximumCostUsd === 0 || !externalCallAttempted) {
|
|
1227
|
+
return { ...base, actualCostUsd: 0 };
|
|
1228
|
+
}
|
|
1229
|
+
if (receiptCount === 0) {
|
|
1230
|
+
return { ...base, costUnknown: true };
|
|
1231
|
+
}
|
|
1232
|
+
return { ...base, actualCostUsd: observedCostUsd };
|
|
1233
|
+
}
|
|
1234
|
+
};
|
|
1235
|
+
}
|
|
1236
|
+
function exceedsMaximum(actual, maximum) {
|
|
1237
|
+
const tolerance = Number.EPSILON * Math.max(1, Math.abs(actual), Math.abs(maximum)) * 8;
|
|
1238
|
+
return actual - maximum > tolerance;
|
|
1239
|
+
}
|
|
1240
|
+
|
|
1241
|
+
// src/memory/experiment/execution-context.ts
|
|
1242
|
+
function createAgentMemoryExecutionContext(dispatch, sequence) {
|
|
1243
|
+
const privateValues = memoryEvaluationPrivateValues(dispatch.cellId, sequence);
|
|
1244
|
+
const signal = relayAbortWithoutReason(dispatch.signal);
|
|
1245
|
+
const cost = Object.freeze({
|
|
1246
|
+
async runPaidCall(input) {
|
|
1247
|
+
const result = await dispatch.cost.runPaidCall({
|
|
1248
|
+
...input,
|
|
1249
|
+
execute: (sourceSignal, callId) => withRedactedAbortSignal(
|
|
1250
|
+
sourceSignal,
|
|
1251
|
+
(redactedSignal) => input.execute(redactedSignal, callId)
|
|
1252
|
+
)
|
|
1253
|
+
});
|
|
1254
|
+
return sanitizePaidCallResult(result, privateValues);
|
|
1255
|
+
}
|
|
1256
|
+
});
|
|
1257
|
+
return {
|
|
1258
|
+
context: Object.freeze({ signal: signal.signal, cost }),
|
|
1259
|
+
dispose: signal.dispose
|
|
1260
|
+
};
|
|
1261
|
+
}
|
|
1262
|
+
function sanitizePaidCallResult(result, privateValues) {
|
|
1263
|
+
if (result.succeeded) {
|
|
1264
|
+
return Object.freeze({
|
|
1265
|
+
succeeded: true,
|
|
1266
|
+
callId: result.callId,
|
|
1267
|
+
value: result.value,
|
|
1268
|
+
receipt: sanitizeCostReceipt(result.receipt, privateValues)
|
|
1269
|
+
});
|
|
1270
|
+
}
|
|
1271
|
+
return Object.freeze({
|
|
1272
|
+
succeeded: false,
|
|
1273
|
+
...result.callId === void 0 ? {} : { callId: result.callId },
|
|
1274
|
+
error: sanitizeError(result.error, privateValues),
|
|
1275
|
+
...result.receipt === void 0 ? {} : { receipt: sanitizeCostReceipt(result.receipt, privateValues) }
|
|
1276
|
+
});
|
|
1277
|
+
}
|
|
1278
|
+
function sanitizeCostReceipt(receipt, privateValues) {
|
|
1279
|
+
const { tags: _tags, phase: _phase, error, ...visible } = receipt;
|
|
1280
|
+
return Object.freeze({
|
|
1281
|
+
...visible,
|
|
1282
|
+
...error === void 0 ? {} : { error: redactPrivateValues(error, privateValues) }
|
|
1283
|
+
});
|
|
1284
|
+
}
|
|
1285
|
+
function sanitizeError(error, privateValues) {
|
|
1286
|
+
const sanitized = new Error(redactPrivateValues(error.message, privateValues));
|
|
1287
|
+
sanitized.name = redactPrivateValues(error.name, privateValues);
|
|
1288
|
+
return sanitized;
|
|
1289
|
+
}
|
|
1290
|
+
async function withRedactedAbortSignal(source, run) {
|
|
1291
|
+
const relay = relayAbortWithoutReason(source);
|
|
1292
|
+
try {
|
|
1293
|
+
return await run(relay.signal);
|
|
1294
|
+
} finally {
|
|
1295
|
+
relay.dispose();
|
|
1296
|
+
}
|
|
1297
|
+
}
|
|
1298
|
+
function relayAbortWithoutReason(source) {
|
|
1299
|
+
const controller = new AbortController();
|
|
1300
|
+
const abort = () => controller.abort(memoryExecutionAbortError());
|
|
1301
|
+
if (source.aborted) abort();
|
|
1302
|
+
else source.addEventListener("abort", abort, { once: true });
|
|
1303
|
+
return {
|
|
1304
|
+
signal: controller.signal,
|
|
1305
|
+
dispose() {
|
|
1306
|
+
source.removeEventListener("abort", abort);
|
|
1307
|
+
}
|
|
1308
|
+
};
|
|
1309
|
+
}
|
|
1310
|
+
function memoryExecutionAbortError() {
|
|
1311
|
+
const error = new Error("memory execution aborted");
|
|
1312
|
+
error.name = "AbortError";
|
|
1313
|
+
return error;
|
|
1314
|
+
}
|
|
1315
|
+
function memoryEvaluationPrivateValues(cellId, sequence) {
|
|
1316
|
+
const values = /* @__PURE__ */ new Set([cellId, sequence.id]);
|
|
1317
|
+
collectUnknownStrings(sequence.metadata, values);
|
|
1318
|
+
for (const step of sequence.steps) {
|
|
1319
|
+
values.add(step.id);
|
|
1320
|
+
collectUnknownStrings(step.metadata, values);
|
|
1321
|
+
for (const probe of step.probes ?? []) {
|
|
1322
|
+
values.add(probe.id);
|
|
1323
|
+
values.add(probe.query);
|
|
1324
|
+
if (probe.referenceAnswer !== void 0) values.add(probe.referenceAnswer);
|
|
1325
|
+
for (const matcher of [...probe.requiredFacts ?? [], ...probe.forbiddenFacts ?? []]) {
|
|
1326
|
+
values.add(matcher.id);
|
|
1327
|
+
for (const expected of matcher.anyOf) values.add(expected);
|
|
1328
|
+
}
|
|
1329
|
+
for (const expected of probe.expectedEventIds ?? []) values.add(expected);
|
|
1330
|
+
for (const expected of probe.expectedActorIds ?? []) values.add(expected);
|
|
1331
|
+
}
|
|
1332
|
+
}
|
|
1333
|
+
return [...values].filter(Boolean).sort((left, right) => right.length - left.length);
|
|
1334
|
+
}
|
|
1335
|
+
function collectUnknownStrings(value, output, seen = /* @__PURE__ */ new WeakSet()) {
|
|
1336
|
+
if (typeof value === "string") {
|
|
1337
|
+
output.add(value);
|
|
1338
|
+
return;
|
|
1339
|
+
}
|
|
1340
|
+
if (!value || typeof value !== "object" || seen.has(value)) return;
|
|
1341
|
+
seen.add(value);
|
|
1342
|
+
if (Array.isArray(value)) {
|
|
1343
|
+
for (const item of value) collectUnknownStrings(item, output, seen);
|
|
1344
|
+
return;
|
|
1345
|
+
}
|
|
1346
|
+
for (const [key, item] of Object.entries(value)) {
|
|
1347
|
+
output.add(key);
|
|
1348
|
+
collectUnknownStrings(item, output, seen);
|
|
1349
|
+
}
|
|
1350
|
+
}
|
|
1351
|
+
function redactPrivateValues(value, privateValues) {
|
|
1352
|
+
let redacted = value;
|
|
1353
|
+
for (const privateValue of privateValues) {
|
|
1354
|
+
redacted = redacted.split(privateValue).join("[redacted]");
|
|
1355
|
+
}
|
|
1356
|
+
return redacted;
|
|
1357
|
+
}
|
|
1358
|
+
|
|
1201
1359
|
// src/memory/experiment/metrics.ts
|
|
1202
1360
|
function rankAgentMemoryExperiment(candidates, scenarios, campaign, costByCandidate) {
|
|
1203
1361
|
const scenarioById = new Map(scenarios.map((scenario) => [scenario.id, scenario]));
|
|
@@ -1451,6 +1609,11 @@ async function recoverAbandonedMemoryAttempts(input) {
|
|
|
1451
1609
|
throw new Error(`missing recovery generation for memory branch '${attempt.branchId}'`);
|
|
1452
1610
|
}
|
|
1453
1611
|
let externalRecoveryAttempted = false;
|
|
1612
|
+
const costRecorder = createAgentMemoryCostRecorder({
|
|
1613
|
+
candidateRef: candidate.ref,
|
|
1614
|
+
maximumCostUsd: recoveryCostUsd,
|
|
1615
|
+
operation: `${candidate.id}: memory recovery`
|
|
1616
|
+
});
|
|
1454
1617
|
const recover = async () => {
|
|
1455
1618
|
await recoverMemoryAttempt({
|
|
1456
1619
|
options: input.options,
|
|
@@ -1460,6 +1623,10 @@ async function recoverAbandonedMemoryAttempts(input) {
|
|
|
1460
1623
|
lease: input.lease,
|
|
1461
1624
|
onExternalCall: () => {
|
|
1462
1625
|
externalRecoveryAttempted = true;
|
|
1626
|
+
},
|
|
1627
|
+
recordExternalCost: (actualCostUsd) => {
|
|
1628
|
+
externalRecoveryAttempted = true;
|
|
1629
|
+
costRecorder.record(actualCostUsd);
|
|
1463
1630
|
}
|
|
1464
1631
|
});
|
|
1465
1632
|
appendMemoryAttemptEvent(input.storage, input.attemptLogPath, {
|
|
@@ -1473,29 +1640,17 @@ async function recoverAbandonedMemoryAttempts(input) {
|
|
|
1473
1640
|
await recover();
|
|
1474
1641
|
} else {
|
|
1475
1642
|
const tags = memoryRecoveryCostTags(input.runDir, candidate.id, attempt.branchId);
|
|
1476
|
-
const receipt = {
|
|
1477
|
-
model: candidate.id,
|
|
1478
|
-
inputTokens: 0,
|
|
1479
|
-
outputTokens: 0,
|
|
1480
|
-
actualCostUsd: recoveryCostUsd
|
|
1481
|
-
};
|
|
1482
1643
|
const paid = await input.costLedger.runPaidCall({
|
|
1483
1644
|
callId: memoryAttemptCostCallId(attempt, "recovery", recoveryGeneration),
|
|
1484
1645
|
channel: "driver",
|
|
1485
1646
|
phase: `${input.options.costPhase ?? "memory.experiment"}.recovery`,
|
|
1486
1647
|
actor: `agent-knowledge:memory-recovery:${candidate.id}`,
|
|
1487
|
-
model: candidate.
|
|
1648
|
+
model: candidate.ref,
|
|
1488
1649
|
tags,
|
|
1489
1650
|
maximumCharge: { externallyEnforcedMaximumUsd: recoveryCostUsd },
|
|
1490
1651
|
execute: recover,
|
|
1491
|
-
receipt: () => (
|
|
1492
|
-
|
|
1493
|
-
actualCostUsd: externalRecoveryAttempted ? recoveryCostUsd : 0
|
|
1494
|
-
}),
|
|
1495
|
-
receiptFromError: () => ({
|
|
1496
|
-
...receipt,
|
|
1497
|
-
actualCostUsd: externalRecoveryAttempted ? recoveryCostUsd : 0
|
|
1498
|
-
})
|
|
1652
|
+
receipt: () => costRecorder.receipt(externalRecoveryAttempted),
|
|
1653
|
+
receiptFromError: () => costRecorder.receipt(externalRecoveryAttempted)
|
|
1499
1654
|
});
|
|
1500
1655
|
if (!paid.succeeded) throw paid.error;
|
|
1501
1656
|
}
|
|
@@ -1511,7 +1666,7 @@ async function recoverAbandonedMemoryAttempts(input) {
|
|
|
1511
1666
|
}
|
|
1512
1667
|
}
|
|
1513
1668
|
async function recoverMemoryAttempt(input) {
|
|
1514
|
-
const { options, candidate, sequence, attempt, lease, onExternalCall } = input;
|
|
1669
|
+
const { options, candidate, sequence, attempt, lease, onExternalCall, recordExternalCost } = input;
|
|
1515
1670
|
const cleanupBranches = attempt.cleanupBranches;
|
|
1516
1671
|
const cleanupTimeoutMs = resolveMemoryCleanupTimeoutMs(
|
|
1517
1672
|
options.cleanupTimeoutMs,
|
|
@@ -1526,12 +1681,11 @@ async function recoverMemoryAttempt(input) {
|
|
|
1526
1681
|
const creation = Promise.resolve().then(
|
|
1527
1682
|
() => candidate.createAdapter({
|
|
1528
1683
|
branchId: attempt.branchId,
|
|
1529
|
-
sequence,
|
|
1530
|
-
rep: attempt.rep,
|
|
1531
|
-
seed: attempt.seed,
|
|
1532
1684
|
purpose: "recovery",
|
|
1533
1685
|
signal: abortController.signal,
|
|
1534
|
-
|
|
1686
|
+
maximumCostUsd: candidate.externalRecoveryCostUsdPerAttempt ?? 0,
|
|
1687
|
+
markExternalCall: onExternalCall,
|
|
1688
|
+
recordExternalCost
|
|
1535
1689
|
})
|
|
1536
1690
|
);
|
|
1537
1691
|
releaseMemoryAdapterCreatedAfterAbort({
|
|
@@ -1666,7 +1820,6 @@ function memoryRecoveryCostTags(runDir, candidateId, branchId) {
|
|
|
1666
1820
|
}
|
|
1667
1821
|
function memoryAttemptEvent(input) {
|
|
1668
1822
|
return {
|
|
1669
|
-
schema: 2,
|
|
1670
1823
|
status: input.status,
|
|
1671
1824
|
branchId: input.branchId,
|
|
1672
1825
|
candidateId: input.candidate.id,
|
|
@@ -1701,7 +1854,7 @@ function readActiveMemoryAttempts(storage, path) {
|
|
|
1701
1854
|
}
|
|
1702
1855
|
function parseMemoryAttemptEvent(value, path, line) {
|
|
1703
1856
|
const event = value;
|
|
1704
|
-
const valid = typeof event === "object" && event !== null &&
|
|
1857
|
+
const valid = typeof event === "object" && event !== null && (event.status === "started" || event.status === "cleaned") && typeof event.branchId === "string" && event.branchId.length > 0 && typeof event.candidateId === "string" && event.candidateId.length > 0 && typeof event.candidateRef === "string" && event.candidateRef.length > 0 && typeof event.sequenceId === "string" && event.sequenceId.length > 0 && Number.isSafeInteger(event.rep) && Number.isSafeInteger(event.seed) && typeof event.cleanupBranches === "boolean" && typeof event.externalCostUsdPerSequence === "number" && Number.isFinite(event.externalCostUsdPerSequence) && event.externalCostUsdPerSequence >= 0 && typeof event.externalRecoveryCostUsdPerAttempt === "number" && Number.isFinite(event.externalRecoveryCostUsdPerAttempt) && event.externalRecoveryCostUsdPerAttempt >= 0 && typeof event.recordedAt === "string" && !Number.isNaN(Date.parse(event.recordedAt)) && typeof event.recovery === "boolean";
|
|
1705
1858
|
if (!valid) throw new Error(`invalid memory attempt event in '${path}' line ${line}`);
|
|
1706
1859
|
return value;
|
|
1707
1860
|
}
|
|
@@ -1762,6 +1915,11 @@ async function runSequenceCell(input) {
|
|
|
1762
1915
|
});
|
|
1763
1916
|
appendMemoryAttemptEvent(storage, attemptLogPath, attempt);
|
|
1764
1917
|
let externalCallAttempted = false;
|
|
1918
|
+
const costRecorder = createAgentMemoryCostRecorder({
|
|
1919
|
+
candidateRef: candidate.ref,
|
|
1920
|
+
maximumCostUsd: costUsd,
|
|
1921
|
+
operation: `${candidate.id}: memory sequence`
|
|
1922
|
+
});
|
|
1765
1923
|
const appendCleanedAttempt = (priorError) => {
|
|
1766
1924
|
try {
|
|
1767
1925
|
appendMemoryAttemptEvent(storage, attemptLogPath, {
|
|
@@ -1789,13 +1947,15 @@ async function runSequenceCell(input) {
|
|
|
1789
1947
|
try {
|
|
1790
1948
|
const created = await candidate.createAdapter({
|
|
1791
1949
|
branchId,
|
|
1792
|
-
sequence: scenario.sequence,
|
|
1793
|
-
rep: context.rep,
|
|
1794
|
-
seed: context.seed,
|
|
1795
1950
|
purpose: "execute",
|
|
1796
1951
|
signal: context.signal,
|
|
1952
|
+
maximumCostUsd: costUsd,
|
|
1797
1953
|
markExternalCall: () => {
|
|
1798
1954
|
externalCallAttempted = true;
|
|
1955
|
+
},
|
|
1956
|
+
recordExternalCost: (actualCostUsd) => {
|
|
1957
|
+
externalCallAttempted = true;
|
|
1958
|
+
costRecorder.record(actualCostUsd);
|
|
1799
1959
|
}
|
|
1800
1960
|
});
|
|
1801
1961
|
if (!created) throw new Error(`${candidate.id}: createAdapter returned no execution adapter`);
|
|
@@ -1819,18 +1979,24 @@ async function runSequenceCell(input) {
|
|
|
1819
1979
|
baseScope: memoryExperimentBaseScope(options, candidate, scenario.sequenceId)
|
|
1820
1980
|
});
|
|
1821
1981
|
const probes = [];
|
|
1822
|
-
for (const step of scenario.sequence.steps) {
|
|
1982
|
+
for (const [ordinal, step] of scenario.sequence.steps.entries()) {
|
|
1823
1983
|
context.signal.throwIfAborted();
|
|
1824
1984
|
await lease.assertOwned();
|
|
1825
1985
|
await writeStep(memory, step);
|
|
1826
1986
|
await lease.assertOwned();
|
|
1827
|
-
|
|
1828
|
-
|
|
1829
|
-
|
|
1830
|
-
|
|
1831
|
-
|
|
1832
|
-
|
|
1833
|
-
|
|
1987
|
+
if (options.executeStep) {
|
|
1988
|
+
const execution = createAgentMemoryExecutionContext(context, scenario.sequence);
|
|
1989
|
+
try {
|
|
1990
|
+
await options.executeStep({
|
|
1991
|
+
memory,
|
|
1992
|
+
candidateId: candidate.id,
|
|
1993
|
+
step: executionStep(step, ordinal),
|
|
1994
|
+
context: execution.context
|
|
1995
|
+
});
|
|
1996
|
+
} finally {
|
|
1997
|
+
execution.dispose();
|
|
1998
|
+
}
|
|
1999
|
+
}
|
|
1834
2000
|
context.signal.throwIfAborted();
|
|
1835
2001
|
await lease.assertOwned();
|
|
1836
2002
|
const stepProbes = await probeStep(memory, scenario.sequence, step);
|
|
@@ -1970,27 +2136,25 @@ async function runSequenceCell(input) {
|
|
|
1970
2136
|
if (!artifact) throw new Error(`${candidate.id}: memory sequence produced no result`);
|
|
1971
2137
|
return artifact;
|
|
1972
2138
|
}
|
|
1973
|
-
const receipt = {
|
|
1974
|
-
model: candidate.id,
|
|
1975
|
-
inputTokens: 0,
|
|
1976
|
-
outputTokens: 0,
|
|
1977
|
-
actualCostUsd: costUsd
|
|
1978
|
-
};
|
|
1979
2139
|
const paid = await context.cost.runPaidCall({
|
|
1980
2140
|
callId: memoryAttemptCostCallId(attempt, "execute", 0),
|
|
1981
2141
|
actor: `agent-knowledge:memory-experiment:${candidate.id}`,
|
|
1982
|
-
model: candidate.
|
|
2142
|
+
model: candidate.ref,
|
|
1983
2143
|
maximumCharge: { externallyEnforcedMaximumUsd: costUsd },
|
|
1984
2144
|
execute,
|
|
1985
|
-
receipt: () => receipt,
|
|
1986
|
-
receiptFromError: () => (
|
|
1987
|
-
...receipt,
|
|
1988
|
-
actualCostUsd: externalCallAttempted ? costUsd : 0
|
|
1989
|
-
})
|
|
2145
|
+
receipt: () => costRecorder.receipt(externalCallAttempted),
|
|
2146
|
+
receiptFromError: () => costRecorder.receipt(externalCallAttempted)
|
|
1990
2147
|
});
|
|
1991
2148
|
if (!paid.succeeded) throw paid.error;
|
|
1992
2149
|
return paid.value;
|
|
1993
2150
|
}
|
|
2151
|
+
function executionStep(step, ordinal) {
|
|
2152
|
+
return {
|
|
2153
|
+
ordinal,
|
|
2154
|
+
...step.instruction !== void 0 ? { instruction: step.instruction } : {},
|
|
2155
|
+
...step.scope !== void 0 ? { scope: structuredClone(step.scope) } : {}
|
|
2156
|
+
};
|
|
2157
|
+
}
|
|
1994
2158
|
async function writeStep(memory, step) {
|
|
1995
2159
|
const writes = (step.writes ?? []).map((write) => ({
|
|
1996
2160
|
...write,
|
|
@@ -2093,6 +2257,12 @@ async function runAgentMemoryExperiment(options) {
|
|
|
2093
2257
|
`${candidate.id}: externalRecoveryCostUsdPerAttempt must be a non-negative finite number`
|
|
2094
2258
|
);
|
|
2095
2259
|
}
|
|
2260
|
+
if (candidate.externalCostAccounting !== void 0 && candidate.externalCostAccounting !== "exact") {
|
|
2261
|
+
throw new Error(`${candidate.id}: externalCostAccounting must be 'exact'`);
|
|
2262
|
+
}
|
|
2263
|
+
if (((candidate.externalCostUsdPerSequence ?? 0) > 0 || (candidate.externalRecoveryCostUsdPerAttempt ?? 0) > 0) && candidate.externalCostAccounting !== "exact") {
|
|
2264
|
+
throw new Error(`${candidate.id}: positive external charges require exact cost accounting`);
|
|
2265
|
+
}
|
|
2096
2266
|
assertNonEmptyString(candidate.ref, `${candidate.id} ref`);
|
|
2097
2267
|
}
|
|
2098
2268
|
const storage = options.storage ?? fsCampaignStorage();
|
|
@@ -2314,7 +2484,8 @@ function memoryExperimentDispatchRef(options) {
|
|
|
2314
2484
|
policy: candidate.policy ?? null,
|
|
2315
2485
|
baseScope: candidate.baseScope ?? null,
|
|
2316
2486
|
externalCostUsdPerSequence: candidate.externalCostUsdPerSequence ?? 0,
|
|
2317
|
-
externalRecoveryCostUsdPerAttempt: candidate.externalRecoveryCostUsdPerAttempt ?? 0
|
|
2487
|
+
externalRecoveryCostUsdPerAttempt: candidate.externalRecoveryCostUsdPerAttempt ?? 0,
|
|
2488
|
+
externalCostAccounting: candidate.externalCostAccounting ?? "exact"
|
|
2318
2489
|
})).sort((a, b) => a.id.localeCompare(b.id))
|
|
2319
2490
|
})
|
|
2320
2491
|
);
|
|
@@ -2741,141 +2912,302 @@ function graphitiMemoryAdapterIdentity(options) {
|
|
|
2741
2912
|
if (typeof options.backendRef !== "string" || !options.backendRef.trim()) {
|
|
2742
2913
|
throw new Error("Graphiti backendRef must be a non-empty string");
|
|
2743
2914
|
}
|
|
2744
|
-
return
|
|
2915
|
+
return `sha256:${sha256(canonicalJson5(stripUndefined2(options)))}`;
|
|
2745
2916
|
}
|
|
2746
2917
|
|
|
2747
2918
|
// src/memory/improvement/run.ts
|
|
2748
|
-
import { join as
|
|
2749
|
-
import { canonicalJson as
|
|
2919
|
+
import { join as join5 } from "path";
|
|
2920
|
+
import { canonicalJson as canonicalJson10 } from "@tangle-network/agent-eval";
|
|
2750
2921
|
import {
|
|
2751
|
-
campaignLineageStore,
|
|
2752
|
-
createRunCostLedger as createRunCostLedger2,
|
|
2753
2922
|
fsCampaignStorage as fsCampaignStorage2,
|
|
2754
|
-
memLineageStore,
|
|
2755
2923
|
resolveRunDir as resolveRunDir2,
|
|
2756
|
-
|
|
2757
|
-
surfaceHash as surfaceHash2
|
|
2924
|
+
surfaceHash as surfaceHash5
|
|
2758
2925
|
} from "@tangle-network/agent-eval/campaign";
|
|
2759
2926
|
|
|
2760
|
-
// src/
|
|
2761
|
-
|
|
2762
|
-
|
|
2763
|
-
|
|
2764
|
-
|
|
2765
|
-
|
|
2766
|
-
|
|
2927
|
+
// src/optimization.ts
|
|
2928
|
+
import { canonicalJson as canonicalJson6 } from "@tangle-network/agent-eval";
|
|
2929
|
+
import {
|
|
2930
|
+
compareOptimizationMethods,
|
|
2931
|
+
surfaceHash
|
|
2932
|
+
} from "@tangle-network/agent-eval/campaign";
|
|
2933
|
+
|
|
2934
|
+
// src/immutable-ref.ts
|
|
2935
|
+
var SHA256_REF = /^sha256:[a-f0-9]{64}$/;
|
|
2936
|
+
var GIT_REF = /^git:[a-f0-9]{40}$/;
|
|
2937
|
+
function assertImmutableRef(value, label) {
|
|
2938
|
+
if (typeof value !== "string" || !SHA256_REF.test(value) && !GIT_REF.test(value)) {
|
|
2939
|
+
throw new Error(`${label} must be lowercase sha256:<64 hex> or git:<40 hex>`);
|
|
2940
|
+
}
|
|
2941
|
+
}
|
|
2942
|
+
|
|
2943
|
+
// src/optimization.ts
|
|
2944
|
+
async function runSerializedKnowledgeOptimization(options) {
|
|
2945
|
+
assertCompleteOptimizationMethod(options.method);
|
|
2946
|
+
assertImmutableRef(options.executionRef, "knowledge optimization executionRef");
|
|
2947
|
+
const codec = options.codec ?? jsonCandidateCodec();
|
|
2948
|
+
const baseline = normalizeCandidate(options.baseline, codec, "baseline");
|
|
2949
|
+
const dispatchRef = [
|
|
2950
|
+
"knowledge-optimization",
|
|
2951
|
+
options.executionRef,
|
|
2952
|
+
baseline.surfaceHash,
|
|
2953
|
+
surfaceHash(options.method.name)
|
|
2954
|
+
].join(":");
|
|
2955
|
+
assertPartitionContent(
|
|
2956
|
+
options.trainScenarios,
|
|
2957
|
+
options.selectionScenarios,
|
|
2958
|
+
options.finalScenarios,
|
|
2959
|
+
options.scenarioFingerprint ?? scenarioContentFingerprint
|
|
2960
|
+
);
|
|
2961
|
+
const method = canonicalCandidateMethod(options.method, codec);
|
|
2962
|
+
const {
|
|
2963
|
+
baseline: _baseline,
|
|
2964
|
+
method: _method,
|
|
2965
|
+
trainScenarios,
|
|
2966
|
+
selectionScenarios,
|
|
2967
|
+
finalScenarios,
|
|
2968
|
+
dispatchCandidate,
|
|
2969
|
+
executionRef: _executionRef,
|
|
2970
|
+
codec: _codec,
|
|
2971
|
+
scenarioFingerprint: _scenarioFingerprint,
|
|
2972
|
+
...comparisonOptions
|
|
2973
|
+
} = options;
|
|
2974
|
+
const optimizationRunOptions = {
|
|
2975
|
+
...comparisonOptions.reps !== void 0 ? { reps: comparisonOptions.reps } : {},
|
|
2976
|
+
...comparisonOptions.resumable !== void 0 ? { resumable: comparisonOptions.resumable } : {},
|
|
2977
|
+
...comparisonOptions.labeledStore !== void 0 ? { labeledStore: comparisonOptions.labeledStore } : {},
|
|
2978
|
+
...comparisonOptions.captureSource !== void 0 ? { captureSource: comparisonOptions.captureSource } : {},
|
|
2979
|
+
...comparisonOptions.captureSourceVersionHash !== void 0 ? { captureSourceVersionHash: comparisonOptions.captureSourceVersionHash } : {},
|
|
2980
|
+
...comparisonOptions.maxConcurrency !== void 0 ? { maxConcurrency: comparisonOptions.maxConcurrency } : {},
|
|
2981
|
+
...comparisonOptions.dispatchTimeoutMs !== void 0 ? { dispatchTimeoutMs: comparisonOptions.dispatchTimeoutMs } : {},
|
|
2982
|
+
...comparisonOptions.repo !== void 0 ? { repo: comparisonOptions.repo } : {},
|
|
2983
|
+
...comparisonOptions.tracing !== void 0 ? { tracing: comparisonOptions.tracing } : {},
|
|
2984
|
+
...comparisonOptions.expectUsage !== void 0 ? { expectUsage: comparisonOptions.expectUsage } : {},
|
|
2985
|
+
...comparisonOptions.now !== void 0 ? { now: comparisonOptions.now } : {},
|
|
2986
|
+
...comparisonOptions.buildTraceWriter !== void 0 ? { buildTraceWriter: comparisonOptions.buildTraceWriter } : {},
|
|
2987
|
+
...comparisonOptions.storage !== void 0 ? { storage: comparisonOptions.storage } : {},
|
|
2988
|
+
...comparisonOptions.cellPlacement !== void 0 ? { cellPlacement: comparisonOptions.cellPlacement } : {},
|
|
2989
|
+
...comparisonOptions.optimizationRunOptions ?? {},
|
|
2990
|
+
dispatchRef
|
|
2991
|
+
};
|
|
2992
|
+
const comparison = await compareOptimizationMethods({
|
|
2993
|
+
...comparisonOptions,
|
|
2994
|
+
dispatchRef,
|
|
2995
|
+
optimizationRunOptions,
|
|
2996
|
+
methods: [method],
|
|
2997
|
+
baselineSurface: baseline.surface,
|
|
2998
|
+
trainScenarios: trainScenarios.map((scenario) => structuredClone(scenario)),
|
|
2999
|
+
selectionScenarios: selectionScenarios.map((scenario) => structuredClone(scenario)),
|
|
3000
|
+
testScenarios: finalScenarios.map((scenario) => structuredClone(scenario)),
|
|
3001
|
+
dispatchWithSurface: async (surface, scenario, context) => {
|
|
3002
|
+
const candidate = normalizeSurface(surface, codec, "candidate");
|
|
3003
|
+
return dispatchCandidate({
|
|
3004
|
+
candidate: structuredClone(candidate.value),
|
|
3005
|
+
candidateSurface: candidate.surface,
|
|
3006
|
+
candidateSurfaceHash: candidate.surfaceHash,
|
|
3007
|
+
scenario,
|
|
3008
|
+
context
|
|
3009
|
+
});
|
|
3010
|
+
}
|
|
2767
3011
|
});
|
|
3012
|
+
const winner = normalizeSurface(comparison.best.winnerSurface, codec, "winner");
|
|
3013
|
+
return {
|
|
3014
|
+
methodName: options.method.name,
|
|
3015
|
+
baseline,
|
|
3016
|
+
winner,
|
|
3017
|
+
comparison
|
|
3018
|
+
};
|
|
2768
3019
|
}
|
|
2769
|
-
function
|
|
2770
|
-
const
|
|
2771
|
-
if (
|
|
2772
|
-
|
|
2773
|
-
|
|
3020
|
+
function assertCompleteOptimizationMethod(method) {
|
|
3021
|
+
const candidate = method;
|
|
3022
|
+
if (!candidate || typeof candidate.name !== "string" || candidate.name.trim().length === 0 || typeof candidate.optimize !== "function") {
|
|
3023
|
+
throw new Error(
|
|
3024
|
+
"knowledge optimization requires a complete OptimizationMethod with a non-empty name and optimize()"
|
|
3025
|
+
);
|
|
2774
3026
|
}
|
|
2775
|
-
|
|
2776
|
-
|
|
2777
|
-
|
|
2778
|
-
|
|
2779
|
-
|
|
2780
|
-
|
|
2781
|
-
|
|
2782
|
-
|
|
2783
|
-
|
|
2784
|
-
cause: error
|
|
2785
|
-
}
|
|
3027
|
+
}
|
|
3028
|
+
function jsonCandidateCodec() {
|
|
3029
|
+
return {
|
|
3030
|
+
serialize: (candidate) => canonicalJson6(candidate),
|
|
3031
|
+
parse(surface) {
|
|
3032
|
+
let parsed;
|
|
3033
|
+
try {
|
|
3034
|
+
parsed = JSON.parse(surface);
|
|
3035
|
+
} catch (error) {
|
|
3036
|
+
throw new Error("serialized knowledge candidate is not valid JSON", { cause: error });
|
|
3037
|
+
}
|
|
3038
|
+
return parsed;
|
|
2786
3039
|
}
|
|
2787
|
-
|
|
2788
|
-
|
|
2789
|
-
|
|
2790
|
-
|
|
3040
|
+
};
|
|
3041
|
+
}
|
|
3042
|
+
function jsonObjectCandidateCodec() {
|
|
3043
|
+
const codec = jsonCandidateCodec();
|
|
3044
|
+
return {
|
|
3045
|
+
serialize: codec.serialize,
|
|
3046
|
+
parse(surface) {
|
|
3047
|
+
const candidate = codec.parse(surface);
|
|
3048
|
+
if (!candidate || typeof candidate !== "object" || Array.isArray(candidate)) {
|
|
3049
|
+
throw new Error("serialized knowledge candidate must be a JSON object");
|
|
2791
3050
|
}
|
|
2792
|
-
|
|
2793
|
-
continue;
|
|
3051
|
+
return candidate;
|
|
2794
3052
|
}
|
|
2795
|
-
|
|
2796
|
-
|
|
3053
|
+
};
|
|
3054
|
+
}
|
|
3055
|
+
function scenarioContentFingerprint(scenario) {
|
|
3056
|
+
const {
|
|
3057
|
+
id: _id,
|
|
3058
|
+
tags: _tags,
|
|
3059
|
+
split: _split,
|
|
3060
|
+
splitTag: _splitTag,
|
|
3061
|
+
...content
|
|
3062
|
+
} = scenario;
|
|
3063
|
+
return surfaceHash(canonicalJson6(content));
|
|
3064
|
+
}
|
|
3065
|
+
function canonicalCandidateMethod(method, codec) {
|
|
3066
|
+
return {
|
|
3067
|
+
name: method.name,
|
|
3068
|
+
async optimize(input) {
|
|
3069
|
+
const result = await method.optimize(input);
|
|
3070
|
+
const winner = normalizeSurface(result.winnerSurface, codec, `${method.name} winner`);
|
|
3071
|
+
return {
|
|
3072
|
+
...result,
|
|
3073
|
+
winnerSurface: winner.surface
|
|
3074
|
+
};
|
|
2797
3075
|
}
|
|
2798
|
-
|
|
2799
|
-
}
|
|
2800
|
-
return { prepared, ...activated ? { activated } : {} };
|
|
3076
|
+
};
|
|
2801
3077
|
}
|
|
2802
|
-
function
|
|
2803
|
-
|
|
2804
|
-
|
|
3078
|
+
function normalizeCandidate(candidate, codec, label) {
|
|
3079
|
+
let surface;
|
|
3080
|
+
try {
|
|
3081
|
+
surface = codec.serialize(candidate);
|
|
3082
|
+
} catch (error) {
|
|
3083
|
+
throw new Error(`${label} serializer failed`, { cause: error });
|
|
2805
3084
|
}
|
|
2806
|
-
|
|
2807
|
-
|
|
2808
|
-
|
|
2809
|
-
|
|
2810
|
-
|
|
2811
|
-
);
|
|
2812
|
-
}
|
|
3085
|
+
return normalizeSurface(surface, codec, label);
|
|
3086
|
+
}
|
|
3087
|
+
function normalizeSurface(surface, codec, label) {
|
|
3088
|
+
if (typeof surface !== "string" || surface.trim().length === 0) {
|
|
3089
|
+
throw new Error(`${label} must be a non-empty serialized string`);
|
|
2813
3090
|
}
|
|
2814
|
-
|
|
2815
|
-
|
|
3091
|
+
let value;
|
|
3092
|
+
try {
|
|
3093
|
+
value = codec.parse(surface);
|
|
3094
|
+
} catch (error) {
|
|
3095
|
+
const detail = error instanceof Error ? `: ${error.message}` : "";
|
|
3096
|
+
throw new Error(`${label} parser failed${detail}`, { cause: error });
|
|
2816
3097
|
}
|
|
2817
|
-
|
|
2818
|
-
|
|
3098
|
+
let canonicalSurface;
|
|
3099
|
+
try {
|
|
3100
|
+
canonicalSurface = codec.serialize(value);
|
|
3101
|
+
} catch (error) {
|
|
3102
|
+
throw new Error(`${label} serializer failed after parsing`, { cause: error });
|
|
2819
3103
|
}
|
|
2820
|
-
if (
|
|
2821
|
-
throw new Error(
|
|
3104
|
+
if (typeof canonicalSurface !== "string" || canonicalSurface.trim().length === 0) {
|
|
3105
|
+
throw new Error(`${label} serializer must return a non-empty string`);
|
|
2822
3106
|
}
|
|
2823
|
-
|
|
2824
|
-
|
|
3107
|
+
const roundTrip = codec.serialize(codec.parse(canonicalSurface));
|
|
3108
|
+
if (roundTrip !== canonicalSurface) {
|
|
3109
|
+
throw new Error(`${label} codec must round-trip to one canonical surface`);
|
|
2825
3110
|
}
|
|
2826
|
-
return value;
|
|
2827
|
-
}
|
|
2828
|
-
|
|
2829
|
-
// src/memory/improvement/candidate.ts
|
|
2830
|
-
function withCostContext(proposer, costLedger, lease, label) {
|
|
2831
3111
|
return {
|
|
2832
|
-
|
|
2833
|
-
|
|
2834
|
-
|
|
2835
|
-
const proposal = await proposer.propose({
|
|
2836
|
-
...context,
|
|
2837
|
-
costLedger,
|
|
2838
|
-
costPhase: `memory.proposal.${context.track?.id ?? label}`
|
|
2839
|
-
});
|
|
2840
|
-
await lease.assertOwned();
|
|
2841
|
-
return proposal;
|
|
2842
|
-
},
|
|
2843
|
-
...proposer.decide ? { decide: (input) => proposer.decide(input) } : {}
|
|
3112
|
+
value: structuredClone(value),
|
|
3113
|
+
surface: canonicalSurface,
|
|
3114
|
+
surfaceHash: surfaceHash(canonicalSurface)
|
|
2844
3115
|
};
|
|
2845
3116
|
}
|
|
2846
|
-
function
|
|
2847
|
-
return
|
|
2848
|
-
|
|
2849
|
-
|
|
2850
|
-
|
|
2851
|
-
|
|
2852
|
-
|
|
2853
|
-
|
|
2854
|
-
|
|
2855
|
-
|
|
2856
|
-
|
|
3117
|
+
function assertPartitionContent(train, selection, final, fingerprint) {
|
|
3118
|
+
if (!fingerprint) return;
|
|
3119
|
+
const owner = /* @__PURE__ */ new Map();
|
|
3120
|
+
for (const [name, scenarios] of [
|
|
3121
|
+
["train", train],
|
|
3122
|
+
["selection", selection],
|
|
3123
|
+
["final", final]
|
|
3124
|
+
]) {
|
|
3125
|
+
for (const scenario of scenarios) {
|
|
3126
|
+
const identity = fingerprint(scenario);
|
|
3127
|
+
if (typeof identity !== "string" || identity.trim().length === 0) {
|
|
3128
|
+
throw new Error(`scenarioFingerprint returned no identity for '${scenario.id}'`);
|
|
3129
|
+
}
|
|
3130
|
+
const prior = owner.get(identity);
|
|
3131
|
+
if (prior) {
|
|
3132
|
+
const scope = prior.partition === name ? `${name} partition duplicates` : `${prior.partition}/${name} partitions duplicate`;
|
|
3133
|
+
throw new Error(
|
|
3134
|
+
`serialized knowledge optimization ${scope} scenario content at '${prior.scenarioId}'/'${scenario.id}'`
|
|
3135
|
+
);
|
|
3136
|
+
}
|
|
3137
|
+
owner.set(identity, { partition: name, scenarioId: scenario.id });
|
|
2857
3138
|
}
|
|
2858
|
-
}
|
|
3139
|
+
}
|
|
2859
3140
|
}
|
|
2860
|
-
|
|
3141
|
+
|
|
3142
|
+
// src/memory/improvement/activation.ts
|
|
3143
|
+
import { surfaceHash as surfaceHash4 } from "@tangle-network/agent-eval/campaign";
|
|
3144
|
+
|
|
3145
|
+
// src/memory/improvement/evaluation.ts
|
|
3146
|
+
import { dirname as dirname2, join as join4 } from "path";
|
|
3147
|
+
import { canonicalJson as canonicalJson9 } from "@tangle-network/agent-eval";
|
|
3148
|
+
import {
|
|
3149
|
+
surfaceHash as surfaceHash3
|
|
3150
|
+
} from "@tangle-network/agent-eval/campaign";
|
|
3151
|
+
|
|
3152
|
+
// src/memory/improvement/candidate.ts
|
|
3153
|
+
import { dirname, join as join2 } from "path";
|
|
3154
|
+
import { canonicalJson as canonicalJson7 } from "@tangle-network/agent-eval";
|
|
3155
|
+
async function buildCandidate(options, config, hash) {
|
|
3156
|
+
const id = `memory-config-${hash}`;
|
|
2861
3157
|
const built = await options.createCandidate({
|
|
2862
3158
|
config,
|
|
2863
|
-
candidateId:
|
|
3159
|
+
candidateId: id,
|
|
2864
3160
|
surfaceHash: hash
|
|
2865
3161
|
});
|
|
3162
|
+
assertImmutableRef(built.ref, `${id}: ref`);
|
|
3163
|
+
assertDeclaredCost(built.externalCostUsdPerSequence, `${id}: externalCostUsdPerSequence`);
|
|
3164
|
+
assertDeclaredCost(
|
|
3165
|
+
built.externalRecoveryCostUsdPerAttempt,
|
|
3166
|
+
`${id}: externalRecoveryCostUsdPerAttempt`
|
|
3167
|
+
);
|
|
3168
|
+
if (built.externalCostAccounting !== "exact") {
|
|
3169
|
+
throw new Error(`${id}: memory improvement requires exact external cost accounting`);
|
|
3170
|
+
}
|
|
2866
3171
|
return {
|
|
2867
3172
|
...built,
|
|
2868
|
-
id
|
|
3173
|
+
id,
|
|
2869
3174
|
ref: built.ref
|
|
2870
3175
|
};
|
|
2871
3176
|
}
|
|
2872
|
-
function
|
|
3177
|
+
async function buildRegisteredCandidate(options, storage, runDir, config, hash) {
|
|
3178
|
+
const candidate = await buildCandidate(options, config, hash);
|
|
3179
|
+
const path = join2(runDir, "memory-candidate-identities", `${hash}.json`);
|
|
3180
|
+
const record2 = { surfaceHash: hash, candidateRef: candidate.ref };
|
|
3181
|
+
const stored = storage.read(path);
|
|
3182
|
+
if (stored !== void 0) {
|
|
3183
|
+
let parsed;
|
|
3184
|
+
try {
|
|
3185
|
+
parsed = JSON.parse(stored);
|
|
3186
|
+
} catch (error) {
|
|
3187
|
+
throw new Error(`invalid memory candidate identity '${path}'`, { cause: error });
|
|
3188
|
+
}
|
|
3189
|
+
if (canonicalJson7(parsed) !== canonicalJson7(record2)) {
|
|
3190
|
+
throw new Error(
|
|
3191
|
+
`memory config '${hash}' changed candidate identity within the same improvement run`
|
|
3192
|
+
);
|
|
3193
|
+
}
|
|
3194
|
+
return candidate;
|
|
3195
|
+
}
|
|
3196
|
+
if (storage.exists(path)) throw new Error(`cannot read memory candidate identity '${path}'`);
|
|
3197
|
+
storage.ensureDir(dirname(path));
|
|
3198
|
+
storage.write(path, `${JSON.stringify(record2, null, 2)}
|
|
3199
|
+
`);
|
|
3200
|
+
return candidate;
|
|
3201
|
+
}
|
|
3202
|
+
function assertDeclaredCost(value, label) {
|
|
3203
|
+
if (typeof value !== "number" || !Number.isFinite(value) || value < 0) {
|
|
3204
|
+
throw new Error(`${label} must be a declared non-negative finite number`);
|
|
3205
|
+
}
|
|
3206
|
+
}
|
|
3207
|
+
function experimentOptions(options, storage, lease) {
|
|
2873
3208
|
return {
|
|
2874
3209
|
storage,
|
|
2875
|
-
seed: options.seed,
|
|
2876
|
-
reps: options.reps,
|
|
2877
3210
|
resumable: options.resumable,
|
|
2878
|
-
maxConcurrency: options.sequenceConcurrency,
|
|
2879
3211
|
dispatchTimeoutMs: options.dispatchTimeoutMs,
|
|
2880
3212
|
cleanupTimeoutMs: options.cleanupTimeoutMs,
|
|
2881
3213
|
maxRecoveryAttempts: options.maxRecoveryAttempts,
|
|
@@ -2884,7 +3216,6 @@ function experimentOptions(options, costLedger, storage, lease) {
|
|
|
2884
3216
|
executeStepRef: options.executeStepRef,
|
|
2885
3217
|
onBranchSnapshot: options.onBranchSnapshot,
|
|
2886
3218
|
cleanupBranches: options.cleanupBranches ?? true,
|
|
2887
|
-
costLedger,
|
|
2888
3219
|
acquireRunLease: async () => ({
|
|
2889
3220
|
assertOwned: () => lease.assertOwned(),
|
|
2890
3221
|
release() {
|
|
@@ -2895,11 +3226,9 @@ function experimentOptions(options, costLedger, storage, lease) {
|
|
|
2895
3226
|
}
|
|
2896
3227
|
|
|
2897
3228
|
// src/memory/improvement/identity.ts
|
|
2898
|
-
import { join as
|
|
2899
|
-
import { canonicalJson as
|
|
2900
|
-
import {
|
|
2901
|
-
surfaceHash
|
|
2902
|
-
} from "@tangle-network/agent-eval/campaign";
|
|
3229
|
+
import { join as join3 } from "path";
|
|
3230
|
+
import { canonicalJson as canonicalJson8 } from "@tangle-network/agent-eval";
|
|
3231
|
+
import { surfaceHash as surfaceHash2 } from "@tangle-network/agent-eval/campaign";
|
|
2903
3232
|
|
|
2904
3233
|
// src/memory/improvement/promotion.ts
|
|
2905
3234
|
import { heldoutSignificance } from "@tangle-network/agent-eval/campaign";
|
|
@@ -2910,27 +3239,38 @@ var DEFAULT_CRITICAL_DIMENSIONS = [
|
|
|
2910
3239
|
"memory_actor_recall",
|
|
2911
3240
|
"memory_event_recall"
|
|
2912
3241
|
];
|
|
2913
|
-
var MEMORY_IMPROVEMENT_IMPLEMENTATION_REF = "agent-knowledge:memory-improvement:v2";
|
|
2914
3242
|
|
|
2915
3243
|
// src/memory/improvement/promotion.ts
|
|
2916
3244
|
function decidePromotion(input) {
|
|
2917
|
-
const { options,
|
|
2918
|
-
const
|
|
2919
|
-
const
|
|
2920
|
-
|
|
2921
|
-
const
|
|
2922
|
-
|
|
3245
|
+
const { options, finalEvaluation } = input;
|
|
3246
|
+
const baselineScores = finalEvaluation.pairs.map((pair) => pair.baseline.score);
|
|
3247
|
+
const winnerScores = finalEvaluation.pairs.map((pair) => pair.winner.score);
|
|
3248
|
+
const baselineScore = mean2(baselineScores);
|
|
3249
|
+
const winnerScore = mean2(winnerScores);
|
|
3250
|
+
if (input.unchanged) {
|
|
3251
|
+
return {
|
|
3252
|
+
status: "no-change",
|
|
3253
|
+
reasons: ["optimization selected the baseline configuration"],
|
|
3254
|
+
baselineScore,
|
|
3255
|
+
winnerScore,
|
|
3256
|
+
lift: 0,
|
|
3257
|
+
criticalDimensions: []
|
|
3258
|
+
};
|
|
3259
|
+
}
|
|
3260
|
+
const significance = heldoutSignificance(
|
|
3261
|
+
{
|
|
3262
|
+
before: baselineScores,
|
|
3263
|
+
after: winnerScores,
|
|
3264
|
+
cellIds: finalEvaluation.pairs.map((pair) => `${pair.sequenceId}:${pair.rep}`)
|
|
3265
|
+
},
|
|
3266
|
+
options.significance
|
|
3267
|
+
);
|
|
2923
3268
|
const tolerance = options.criticalDimensionTolerance ?? 0.05;
|
|
2924
3269
|
const criticalDimensions = (options.criticalDimensions ?? DEFAULT_CRITICAL_DIMENSIONS).map(
|
|
2925
3270
|
(dimension) => {
|
|
2926
|
-
const expectedN = applicableSequenceCount(options.
|
|
2927
|
-
const
|
|
2928
|
-
|
|
2929
|
-
baselineId,
|
|
2930
|
-
winnerId,
|
|
2931
|
-
(artifact) => (artifact.dimensionSampleCounts?.[dimension] ?? 0) > 0 ? artifact.dimensions[dimension] : void 0
|
|
2932
|
-
);
|
|
2933
|
-
const comparison = heldoutSignificance(dimensionPairs, {
|
|
3271
|
+
const expectedN = applicableSequenceCount(options.finalSequences, dimension) * (options.reps ?? 1);
|
|
3272
|
+
const pairs = pairedDimension(finalEvaluation, dimension);
|
|
3273
|
+
const comparison = heldoutSignificance(pairs, {
|
|
2934
3274
|
...options.significance,
|
|
2935
3275
|
deltaThreshold: 0
|
|
2936
3276
|
});
|
|
@@ -2948,21 +3288,31 @@ function decidePromotion(input) {
|
|
|
2948
3288
|
}
|
|
2949
3289
|
);
|
|
2950
3290
|
const reasons = [];
|
|
2951
|
-
|
|
2952
|
-
|
|
3291
|
+
const totalCost = input.optimization.comparison.totalCost;
|
|
3292
|
+
if (!input.optimization.comparison.totalCost.accountingComplete && !options.allowIncompleteCostAccounting) {
|
|
3293
|
+
reasons.push("optimization or final cost accounting is incomplete");
|
|
3294
|
+
}
|
|
3295
|
+
const optimizerSource = input.optimization.comparison.best.provenance?.source;
|
|
3296
|
+
if (optimizerSource && optimizerSource.evidence !== "observed") {
|
|
3297
|
+
reasons.push("external optimizer package identity was not observed");
|
|
3298
|
+
}
|
|
3299
|
+
if (totalCost.totalCostUsd > (options.maxTotalCostUsd ?? 0)) {
|
|
3300
|
+
reasons.push(
|
|
3301
|
+
`total cost ${totalCost.totalCostUsd} exceeds the configured limit ${options.maxTotalCostUsd ?? 0}`
|
|
3302
|
+
);
|
|
2953
3303
|
}
|
|
2954
3304
|
if (!significance.significant) {
|
|
2955
3305
|
reasons.push(
|
|
2956
|
-
significance.fewRuns ? `only ${significance.n} paired
|
|
3306
|
+
significance.fewRuns ? `only ${significance.n} paired final cells; more are required` : "final lift is not confidently above the promotion threshold"
|
|
2957
3307
|
);
|
|
2958
3308
|
}
|
|
2959
|
-
if (
|
|
2960
|
-
reasons.push(`winner
|
|
3309
|
+
if (winnerScore < (options.minFinalScore ?? 0)) {
|
|
3310
|
+
reasons.push(`winner final score ${winnerScore} is below the required minimum`);
|
|
2961
3311
|
}
|
|
2962
3312
|
for (const dimension of criticalDimensions) {
|
|
2963
3313
|
if (!dimension.measured) {
|
|
2964
3314
|
reasons.push(
|
|
2965
|
-
dimension.expectedN === 0 ? `critical dimension ${dimension.dimension} has no applicable
|
|
3315
|
+
dimension.expectedN === 0 ? `critical dimension ${dimension.dimension} has no applicable final histories` : `critical dimension ${dimension.dimension} was measured on ${dimension.n}/${dimension.expectedN} applicable paired final cells`
|
|
2966
3316
|
);
|
|
2967
3317
|
} else if (dimension.regressed) {
|
|
2968
3318
|
reasons.push(`${dimension.dimension} may regress beyond ${tolerance}`);
|
|
@@ -2971,13 +3321,40 @@ function decidePromotion(input) {
|
|
|
2971
3321
|
return {
|
|
2972
3322
|
status: reasons.length === 0 ? "promote" : "hold",
|
|
2973
3323
|
reasons,
|
|
2974
|
-
baselineScore
|
|
2975
|
-
winnerScore
|
|
2976
|
-
lift:
|
|
3324
|
+
baselineScore,
|
|
3325
|
+
winnerScore,
|
|
3326
|
+
lift: winnerScore - baselineScore,
|
|
2977
3327
|
significance,
|
|
2978
3328
|
criticalDimensions
|
|
2979
3329
|
};
|
|
2980
3330
|
}
|
|
3331
|
+
function normalizedPromotionPolicy(options) {
|
|
3332
|
+
return {
|
|
3333
|
+
significance: {
|
|
3334
|
+
deltaThreshold: options.significance?.deltaThreshold ?? 0,
|
|
3335
|
+
minProductiveRuns: options.significance?.minProductiveRuns ?? 3,
|
|
3336
|
+
confidence: options.significance?.confidence ?? 0.95,
|
|
3337
|
+
resamples: options.significance?.resamples ?? 2e3,
|
|
3338
|
+
seed: options.significance?.seed ?? 1337,
|
|
3339
|
+
statistic: options.significance?.statistic ?? "mean"
|
|
3340
|
+
},
|
|
3341
|
+
criticalDimensions: [...options.criticalDimensions ?? DEFAULT_CRITICAL_DIMENSIONS],
|
|
3342
|
+
criticalDimensionTolerance: options.criticalDimensionTolerance ?? 0.05,
|
|
3343
|
+
minFinalScore: options.minFinalScore ?? 0,
|
|
3344
|
+
maxTotalCostUsd: options.maxTotalCostUsd ?? 0,
|
|
3345
|
+
allowIncompleteCostAccounting: options.allowIncompleteCostAccounting ?? false
|
|
3346
|
+
};
|
|
3347
|
+
}
|
|
3348
|
+
function pairedDimension(result, dimension) {
|
|
3349
|
+
const applicable = result.pairs.filter(
|
|
3350
|
+
(pair) => (pair.baseline.dimensionSampleCounts[dimension] ?? 0) > 0 && (pair.winner.dimensionSampleCounts[dimension] ?? 0) > 0
|
|
3351
|
+
);
|
|
3352
|
+
return {
|
|
3353
|
+
before: applicable.map((pair) => pair.baseline.dimensions[dimension]),
|
|
3354
|
+
after: applicable.map((pair) => pair.winner.dimensions[dimension]),
|
|
3355
|
+
cellIds: applicable.map((pair) => `${pair.sequenceId}:${pair.rep}`)
|
|
3356
|
+
};
|
|
3357
|
+
}
|
|
2981
3358
|
function applicableSequenceCount(sequences, dimension) {
|
|
2982
3359
|
return sequences.filter(
|
|
2983
3360
|
(sequence) => sequence.steps.some(
|
|
@@ -3006,89 +3383,36 @@ function probeAppliesToDimension(probe, dimension) {
|
|
|
3006
3383
|
return true;
|
|
3007
3384
|
}
|
|
3008
3385
|
}
|
|
3009
|
-
function normalizedPromotionPolicy(options) {
|
|
3010
|
-
return {
|
|
3011
|
-
significance: {
|
|
3012
|
-
deltaThreshold: options.significance?.deltaThreshold ?? 0,
|
|
3013
|
-
minProductiveRuns: options.significance?.minProductiveRuns ?? 3,
|
|
3014
|
-
confidence: options.significance?.confidence ?? 0.95,
|
|
3015
|
-
resamples: options.significance?.resamples ?? 2e3,
|
|
3016
|
-
seed: options.significance?.seed ?? 1337,
|
|
3017
|
-
statistic: options.significance?.statistic ?? "mean"
|
|
3018
|
-
},
|
|
3019
|
-
criticalDimensions: [...options.criticalDimensions ?? DEFAULT_CRITICAL_DIMENSIONS],
|
|
3020
|
-
criticalDimensionTolerance: options.criticalDimensionTolerance ?? 0.05,
|
|
3021
|
-
minHoldoutScore: options.minHoldoutScore ?? 0
|
|
3022
|
-
};
|
|
3023
|
-
}
|
|
3024
|
-
function pairedArtifacts(result, baselineId, winnerId, select) {
|
|
3025
|
-
const baseline = artifactValues(result, baselineId, select);
|
|
3026
|
-
const winner = artifactValues(result, winnerId, select);
|
|
3027
|
-
const keys = [...baseline.keys()].filter((key) => winner.has(key)).sort();
|
|
3028
|
-
return {
|
|
3029
|
-
before: keys.map((key) => baseline.get(key)),
|
|
3030
|
-
after: keys.map((key) => winner.get(key)),
|
|
3031
|
-
cellIds: keys
|
|
3032
|
-
};
|
|
3033
|
-
}
|
|
3034
|
-
function artifactValues(result, candidateId, select) {
|
|
3035
|
-
const values = /* @__PURE__ */ new Map();
|
|
3036
|
-
for (const cell of result.campaign.cells) {
|
|
3037
|
-
if (cell.error || cell.artifact.candidateId !== candidateId) continue;
|
|
3038
|
-
const value = select(cell.artifact);
|
|
3039
|
-
if (value === void 0 || !Number.isFinite(value)) continue;
|
|
3040
|
-
values.set(`${cell.artifact.sequenceId}:${cell.rep}`, value);
|
|
3041
|
-
}
|
|
3042
|
-
return values;
|
|
3043
|
-
}
|
|
3044
|
-
function sequenceScores(result, sequences, candidateId) {
|
|
3045
|
-
const bySequence = /* @__PURE__ */ new Map();
|
|
3046
|
-
for (const cell of result.campaign.cells) {
|
|
3047
|
-
if (cell.error || cell.artifact.candidateId !== candidateId) continue;
|
|
3048
|
-
const bucket = bySequence.get(cell.artifact.sequenceId) ?? [];
|
|
3049
|
-
bucket.push(cell.artifact.score);
|
|
3050
|
-
bySequence.set(cell.artifact.sequenceId, bucket);
|
|
3051
|
-
}
|
|
3052
|
-
return sequences.map((sequence) => mean2(bySequence.get(sequence.id) ?? []));
|
|
3053
|
-
}
|
|
3054
3386
|
function mean2(values) {
|
|
3055
|
-
return values.
|
|
3387
|
+
return values.reduce((sum, value) => sum + value, 0) / values.length;
|
|
3056
3388
|
}
|
|
3057
3389
|
|
|
3058
3390
|
// src/memory/improvement/identity.ts
|
|
3059
3391
|
function assertMemoryImprovementIdentity(options, storage, runDir, serialize) {
|
|
3060
|
-
const path =
|
|
3392
|
+
const path = join3(runDir, "memory-improvement-manifest.json");
|
|
3061
3393
|
const identity = {
|
|
3062
|
-
schema: 6,
|
|
3063
|
-
implementationRef: MEMORY_IMPROVEMENT_IMPLEMENTATION_REF,
|
|
3064
3394
|
experimentId: options.experimentId,
|
|
3065
|
-
|
|
3395
|
+
implementationRef: options.implementationRef,
|
|
3396
|
+
method: options.method.name,
|
|
3066
3397
|
activationRef: options.activation?.ref ?? null,
|
|
3067
|
-
|
|
3068
|
-
proposerKinds: Object.fromEntries(
|
|
3069
|
-
Object.entries(options.proposers ?? {}).sort(([a], [b]) => a.localeCompare(b)).map(([name, proposer]) => [name, proposer.kind])
|
|
3070
|
-
),
|
|
3071
|
-
populationSize: options.populationSize ?? 4,
|
|
3072
|
-
budget: { maxSteps: options.budget.maxSteps },
|
|
3398
|
+
baselineConfig: serialize(options.baselineConfig),
|
|
3073
3399
|
executeStepRef: options.executeStepRef ?? null,
|
|
3074
3400
|
cleanupBranches: options.cleanupBranches ?? true,
|
|
3075
3401
|
promotionPolicy: normalizedPromotionPolicy(options),
|
|
3076
|
-
seed: options.seed ??
|
|
3402
|
+
seed: options.seed ?? 42,
|
|
3077
3403
|
reps: options.reps ?? 1,
|
|
3078
|
-
|
|
3079
|
-
|
|
3080
|
-
|
|
3081
|
-
vision: entry.vision ?? null,
|
|
3082
|
-
proposer: entry.proposer
|
|
3083
|
-
})),
|
|
3404
|
+
maxTotalCostUsd: options.maxTotalCostUsd ?? null,
|
|
3405
|
+
maximumEvaluationCostUsd: options.maximumEvaluationCostUsd ?? null,
|
|
3406
|
+
allowIncompleteCostAccounting: options.allowIncompleteCostAccounting ?? false,
|
|
3084
3407
|
trainSequences: options.trainSequences,
|
|
3085
|
-
|
|
3408
|
+
selectionSequences: options.selectionSequences,
|
|
3409
|
+
finalSequences: options.finalSequences
|
|
3086
3410
|
};
|
|
3087
|
-
const identityHash =
|
|
3411
|
+
const identityHash = surfaceHash2(canonicalJson8(identity));
|
|
3088
3412
|
const stored = storage.read(path);
|
|
3089
3413
|
if (stored === void 0) {
|
|
3090
3414
|
if (storage.exists(path)) throw new Error(`cannot read memory improvement manifest '${path}'`);
|
|
3091
|
-
if (storage.exists(
|
|
3415
|
+
if (storage.exists(join3(runDir, "memory-improvement-result.json"))) {
|
|
3092
3416
|
throw new Error(`memory improvement run '${runDir}' has state without an identity manifest`);
|
|
3093
3417
|
}
|
|
3094
3418
|
storage.write(path, `${JSON.stringify({ identityHash, identity }, null, 2)}
|
|
@@ -3101,18 +3425,12 @@ function assertMemoryImprovementIdentity(options, storage, runDir, serialize) {
|
|
|
3101
3425
|
} catch (error) {
|
|
3102
3426
|
throw new Error(`invalid memory improvement manifest '${path}'`, { cause: error });
|
|
3103
3427
|
}
|
|
3104
|
-
if (!manifest || typeof manifest !== "object" || manifest.identityHash !== identityHash ||
|
|
3428
|
+
if (!manifest || typeof manifest !== "object" || manifest.identityHash !== identityHash || canonicalJson8(manifest.identity) !== canonicalJson8(identity)) {
|
|
3105
3429
|
throw new Error(
|
|
3106
|
-
`memory improvement run '${runDir}' does not match its persisted inputs or
|
|
3430
|
+
`memory improvement run '${runDir}' does not match its persisted inputs or implementation`
|
|
3107
3431
|
);
|
|
3108
3432
|
}
|
|
3109
3433
|
}
|
|
3110
|
-
function requireStringSurface(surface) {
|
|
3111
|
-
if (typeof surface !== "string" || surface.trim().length === 0) {
|
|
3112
|
-
throw new Error("memory config proposer must return a JSON string surface");
|
|
3113
|
-
}
|
|
3114
|
-
return surface;
|
|
3115
|
-
}
|
|
3116
3434
|
function serializeMemoryConfig(serialize, config) {
|
|
3117
3435
|
let surface;
|
|
3118
3436
|
try {
|
|
@@ -3138,12 +3456,465 @@ function assertMemoryConfigRoundTrip(config, serialize, parse) {
|
|
|
3138
3456
|
const first = serialize(config);
|
|
3139
3457
|
const second = serialize(parse(first));
|
|
3140
3458
|
if (first !== second) {
|
|
3141
|
-
throw new Error("memory config serializer and parser must round-trip
|
|
3459
|
+
throw new Error("memory config serializer and parser must round-trip exactly");
|
|
3142
3460
|
}
|
|
3143
3461
|
}
|
|
3144
3462
|
function memorySequenceFingerprint(sequence) {
|
|
3145
3463
|
const { id: _id, split: _split, tags: _tags, ...content } = sequence;
|
|
3146
|
-
return
|
|
3464
|
+
return surfaceHash2(canonicalJson8(content));
|
|
3465
|
+
}
|
|
3466
|
+
|
|
3467
|
+
// src/memory/improvement/evaluation.ts
|
|
3468
|
+
function memoryConfigCodec(options) {
|
|
3469
|
+
const serializeRaw = options.serializeConfig ?? ((config) => canonicalJson9(config));
|
|
3470
|
+
const parseRaw = options.parseConfig ?? ((surface) => JSON.parse(surface));
|
|
3471
|
+
return {
|
|
3472
|
+
serialize: (config) => serializeMemoryConfig(serializeRaw, config),
|
|
3473
|
+
parse: (surface) => parseMemoryConfig(parseRaw, surface)
|
|
3474
|
+
};
|
|
3475
|
+
}
|
|
3476
|
+
function memoryConfigScenarios(sequences) {
|
|
3477
|
+
return sequences.map(memoryConfigScenario);
|
|
3478
|
+
}
|
|
3479
|
+
async function evaluateMemoryCandidate(input) {
|
|
3480
|
+
await input.lease.assertOwned();
|
|
3481
|
+
const artifactPath = memoryArtifactPath(
|
|
3482
|
+
input.runDir,
|
|
3483
|
+
input.surfaceHash,
|
|
3484
|
+
input.scenario,
|
|
3485
|
+
input.rep,
|
|
3486
|
+
input.seed
|
|
3487
|
+
);
|
|
3488
|
+
const stored = readStoredMemoryArtifact(input.storage, artifactPath, {
|
|
3489
|
+
surfaceHash: input.surfaceHash,
|
|
3490
|
+
candidateRef: input.candidate.ref,
|
|
3491
|
+
scenario: input.scenario,
|
|
3492
|
+
rep: input.rep,
|
|
3493
|
+
seed: input.seed
|
|
3494
|
+
});
|
|
3495
|
+
if (stored) {
|
|
3496
|
+
if (input.final) {
|
|
3497
|
+
writeStoredMemoryArtifact(
|
|
3498
|
+
input.storage,
|
|
3499
|
+
memoryFinalArtifactPath(input.runDir, input.surfaceHash, input.scenario, input.rep),
|
|
3500
|
+
storedMemoryArtifactRecord(input, stored)
|
|
3501
|
+
);
|
|
3502
|
+
}
|
|
3503
|
+
return stored;
|
|
3504
|
+
}
|
|
3505
|
+
const evaluationCostLimit = input.options.maximumEvaluationCostUsd ?? 0;
|
|
3506
|
+
const evaluationId = stableId(
|
|
3507
|
+
"memory_eval",
|
|
3508
|
+
canonicalJson9({
|
|
3509
|
+
surfaceHash: input.surfaceHash,
|
|
3510
|
+
sequence: memorySequenceFingerprint(input.scenario.sequence),
|
|
3511
|
+
rep: input.rep,
|
|
3512
|
+
seed: input.seed
|
|
3513
|
+
})
|
|
3514
|
+
);
|
|
3515
|
+
const paid = await input.cost.runPaidCall({
|
|
3516
|
+
actor: "agent-knowledge:memory-config-evaluation",
|
|
3517
|
+
model: input.candidate.ref,
|
|
3518
|
+
signal: input.signal,
|
|
3519
|
+
maximumCharge: {
|
|
3520
|
+
externallyEnforcedMaximumUsd: evaluationCostLimit
|
|
3521
|
+
},
|
|
3522
|
+
execute: async () => {
|
|
3523
|
+
const experiment = await runAgentMemoryExperiment({
|
|
3524
|
+
...experimentOptions(input.options, input.storage, input.lease),
|
|
3525
|
+
experimentId: `${input.options.experimentId}:${evaluationId}`,
|
|
3526
|
+
experimentRunId: evaluationId,
|
|
3527
|
+
sequences: [input.scenario.sequence],
|
|
3528
|
+
candidates: [input.candidate],
|
|
3529
|
+
runDir: join4(input.runDir, "evaluations", evaluationId),
|
|
3530
|
+
seed: input.seed,
|
|
3531
|
+
reps: 1,
|
|
3532
|
+
maxConcurrency: 1,
|
|
3533
|
+
costCeiling: evaluationCostLimit,
|
|
3534
|
+
costPhase: `memory.config.${input.surfaceHash}`
|
|
3535
|
+
});
|
|
3536
|
+
const cell = experiment.campaign.cells[0];
|
|
3537
|
+
if (!cell || cell.error || cell.artifact.candidateId !== input.candidate.id) {
|
|
3538
|
+
throw new Error(
|
|
3539
|
+
`${input.surfaceHash}/${input.scenario.sequenceId}: memory config evaluation did not complete`
|
|
3540
|
+
);
|
|
3541
|
+
}
|
|
3542
|
+
const cost = experiment.campaign.aggregates.cost;
|
|
3543
|
+
return {
|
|
3544
|
+
artifact: cell.artifact,
|
|
3545
|
+
costUsd: cell.cached ? 0 : experiment.totalCostUsd,
|
|
3546
|
+
cost
|
|
3547
|
+
};
|
|
3548
|
+
},
|
|
3549
|
+
receipt: (value) => ({
|
|
3550
|
+
model: input.candidate.ref,
|
|
3551
|
+
inputTokens: value.cost.inputTokens,
|
|
3552
|
+
outputTokens: value.cost.outputTokens,
|
|
3553
|
+
...value.cost.reasoningTokens !== void 0 ? { reasoningTokens: value.cost.reasoningTokens } : {},
|
|
3554
|
+
cachedTokens: value.cost.cachedTokens,
|
|
3555
|
+
...value.cost.cacheWriteTokens !== void 0 ? { cacheWriteTokens: value.cost.cacheWriteTokens } : {},
|
|
3556
|
+
actualCostUsd: value.costUsd,
|
|
3557
|
+
costUnknown: !value.cost.accountingComplete,
|
|
3558
|
+
usageUnknown: !value.cost.usageComplete
|
|
3559
|
+
})
|
|
3560
|
+
});
|
|
3561
|
+
if (!paid.succeeded) throw paid.error;
|
|
3562
|
+
await input.lease.assertOwned();
|
|
3563
|
+
const record2 = storedMemoryArtifactRecord(input, paid.value.artifact);
|
|
3564
|
+
writeStoredMemoryArtifact(input.storage, artifactPath, record2);
|
|
3565
|
+
if (input.final) {
|
|
3566
|
+
writeStoredMemoryArtifact(
|
|
3567
|
+
input.storage,
|
|
3568
|
+
memoryFinalArtifactPath(input.runDir, input.surfaceHash, input.scenario, input.rep),
|
|
3569
|
+
record2
|
|
3570
|
+
);
|
|
3571
|
+
}
|
|
3572
|
+
return paid.value.artifact;
|
|
3573
|
+
}
|
|
3574
|
+
function loadFinalEvaluation(input) {
|
|
3575
|
+
const pairs = [];
|
|
3576
|
+
const reps = input.options.reps ?? 1;
|
|
3577
|
+
for (const sequence of input.options.finalSequences) {
|
|
3578
|
+
const scenario = memoryConfigScenario(sequence);
|
|
3579
|
+
for (let rep = 0; rep < reps; rep += 1) {
|
|
3580
|
+
const baseline = readStoredMemoryArtifact(
|
|
3581
|
+
input.storage,
|
|
3582
|
+
memoryFinalArtifactPath(input.runDir, input.baselineSurfaceHash, scenario, rep),
|
|
3583
|
+
{
|
|
3584
|
+
surfaceHash: input.baselineSurfaceHash,
|
|
3585
|
+
candidateRef: input.baselineCandidateRef,
|
|
3586
|
+
scenario,
|
|
3587
|
+
rep
|
|
3588
|
+
}
|
|
3589
|
+
);
|
|
3590
|
+
const winner = readStoredMemoryArtifact(
|
|
3591
|
+
input.storage,
|
|
3592
|
+
memoryFinalArtifactPath(input.runDir, input.winnerSurfaceHash, scenario, rep),
|
|
3593
|
+
{
|
|
3594
|
+
surfaceHash: input.winnerSurfaceHash,
|
|
3595
|
+
candidateRef: input.winnerCandidateRef,
|
|
3596
|
+
scenario,
|
|
3597
|
+
rep
|
|
3598
|
+
}
|
|
3599
|
+
);
|
|
3600
|
+
if (!baseline || !winner) {
|
|
3601
|
+
throw new Error(
|
|
3602
|
+
`memory final artifact is missing for sequence '${sequence.id}' repetition ${rep}`
|
|
3603
|
+
);
|
|
3604
|
+
}
|
|
3605
|
+
pairs.push({ sequenceId: sequence.id, rep, baseline, winner });
|
|
3606
|
+
}
|
|
3607
|
+
}
|
|
3608
|
+
return {
|
|
3609
|
+
manifestHash: surfaceHash3(
|
|
3610
|
+
canonicalJson9({
|
|
3611
|
+
baselineSurfaceHash: input.baselineSurfaceHash,
|
|
3612
|
+
baselineCandidateRef: input.baselineCandidateRef,
|
|
3613
|
+
winnerSurfaceHash: input.winnerSurfaceHash,
|
|
3614
|
+
winnerCandidateRef: input.winnerCandidateRef,
|
|
3615
|
+
pairs
|
|
3616
|
+
})
|
|
3617
|
+
),
|
|
3618
|
+
baselineCandidateRef: input.baselineCandidateRef,
|
|
3619
|
+
winnerCandidateRef: input.winnerCandidateRef,
|
|
3620
|
+
pairs
|
|
3621
|
+
};
|
|
3622
|
+
}
|
|
3623
|
+
function memoryArtifactPath(runDir, candidateSurfaceHash, scenario, rep, seed) {
|
|
3624
|
+
return join4(
|
|
3625
|
+
runDir,
|
|
3626
|
+
"memory-config-artifacts",
|
|
3627
|
+
candidateSurfaceHash,
|
|
3628
|
+
stableId("sequence", scenario.sequenceId),
|
|
3629
|
+
`rep-${rep}-${stableId("seed", String(seed))}.json`
|
|
3630
|
+
);
|
|
3631
|
+
}
|
|
3632
|
+
function memoryFinalArtifactPath(runDir, candidateSurfaceHash, scenario, rep) {
|
|
3633
|
+
return join4(
|
|
3634
|
+
runDir,
|
|
3635
|
+
"memory-final-artifacts",
|
|
3636
|
+
candidateSurfaceHash,
|
|
3637
|
+
stableId("sequence", scenario.sequenceId),
|
|
3638
|
+
`rep-${rep}.json`
|
|
3639
|
+
);
|
|
3640
|
+
}
|
|
3641
|
+
function memoryConfigScenario(sequence) {
|
|
3642
|
+
return {
|
|
3643
|
+
id: sequence.id,
|
|
3644
|
+
kind: "agent-memory-config-search",
|
|
3645
|
+
sequenceId: sequence.id,
|
|
3646
|
+
sequence
|
|
3647
|
+
};
|
|
3648
|
+
}
|
|
3649
|
+
function readStoredMemoryArtifact(storage, path, expected) {
|
|
3650
|
+
const stored = storage.read(path);
|
|
3651
|
+
if (stored === void 0) {
|
|
3652
|
+
if (storage.exists(path)) throw new Error(`cannot read memory config artifact '${path}'`);
|
|
3653
|
+
return void 0;
|
|
3654
|
+
}
|
|
3655
|
+
let record2;
|
|
3656
|
+
try {
|
|
3657
|
+
record2 = JSON.parse(stored);
|
|
3658
|
+
} catch (error) {
|
|
3659
|
+
throw new Error(`invalid memory config artifact '${path}'`, { cause: error });
|
|
3660
|
+
}
|
|
3661
|
+
if (!record2 || typeof record2 !== "object" || Array.isArray(record2)) {
|
|
3662
|
+
throw new Error(`invalid memory config artifact '${path}'`);
|
|
3663
|
+
}
|
|
3664
|
+
const value = record2;
|
|
3665
|
+
if (!hasExactKeys(record2, [
|
|
3666
|
+
"surfaceHash",
|
|
3667
|
+
"candidateRef",
|
|
3668
|
+
"sequenceFingerprint",
|
|
3669
|
+
"sequenceId",
|
|
3670
|
+
"rep",
|
|
3671
|
+
"seed",
|
|
3672
|
+
"artifact"
|
|
3673
|
+
]) || value.surfaceHash !== expected.surfaceHash || value.candidateRef !== expected.candidateRef || value.sequenceFingerprint !== memorySequenceFingerprint(expected.scenario.sequence) || value.sequenceId !== expected.scenario.sequenceId || value.rep !== expected.rep || !Number.isSafeInteger(value.seed) || expected.seed !== void 0 && value.seed !== expected.seed) {
|
|
3674
|
+
throw new Error(`memory config artifact '${path}' does not match its evaluation cell`);
|
|
3675
|
+
}
|
|
3676
|
+
return parseMemoryArtifact(
|
|
3677
|
+
value.artifact,
|
|
3678
|
+
`memory-config-${expected.surfaceHash}`,
|
|
3679
|
+
expected.scenario.sequenceId,
|
|
3680
|
+
path
|
|
3681
|
+
);
|
|
3682
|
+
}
|
|
3683
|
+
function writeStoredMemoryArtifact(storage, path, record2) {
|
|
3684
|
+
const serialized = `${JSON.stringify(record2, null, 2)}
|
|
3685
|
+
`;
|
|
3686
|
+
const existing = storage.read(path);
|
|
3687
|
+
if (existing !== void 0) {
|
|
3688
|
+
if (canonicalJson9(JSON.parse(existing)) !== canonicalJson9(record2)) {
|
|
3689
|
+
throw new Error(`memory config artifact '${path}' conflicts with durable content`);
|
|
3690
|
+
}
|
|
3691
|
+
return;
|
|
3692
|
+
}
|
|
3693
|
+
if (storage.exists(path)) throw new Error(`cannot read memory config artifact '${path}'`);
|
|
3694
|
+
storage.ensureDir(dirname2(path));
|
|
3695
|
+
storage.write(path, serialized);
|
|
3696
|
+
}
|
|
3697
|
+
function storedMemoryArtifactRecord(input, artifact) {
|
|
3698
|
+
return {
|
|
3699
|
+
surfaceHash: input.surfaceHash,
|
|
3700
|
+
candidateRef: input.candidate.ref,
|
|
3701
|
+
sequenceFingerprint: memorySequenceFingerprint(input.scenario.sequence),
|
|
3702
|
+
sequenceId: input.scenario.sequenceId,
|
|
3703
|
+
rep: input.rep,
|
|
3704
|
+
seed: input.seed,
|
|
3705
|
+
artifact
|
|
3706
|
+
};
|
|
3707
|
+
}
|
|
3708
|
+
function parseMemoryArtifact(value, candidateId, sequenceId, path) {
|
|
3709
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
3710
|
+
throw new Error(`memory config artifact '${path}' has no artifact`);
|
|
3711
|
+
}
|
|
3712
|
+
const artifact = value;
|
|
3713
|
+
if (artifact.candidateId !== candidateId || artifact.sequenceId !== sequenceId || typeof artifact.score !== "number" || !Number.isFinite(artifact.score) || artifact.score < 0 || artifact.score > 1 || typeof artifact.passed !== "boolean" || !isFiniteNumberRecord(artifact.dimensions) || !isNonnegativeIntegerRecord(artifact.dimensionSampleCounts) || !Array.isArray(artifact.probes) || typeof artifact.branchDigest !== "string" || !artifact.branchDigest || typeof artifact.journalEntries !== "number" || !Number.isSafeInteger(artifact.journalEntries) || artifact.journalEntries < 0 || typeof artifact.durationMs !== "number" || !Number.isFinite(artifact.durationMs) || artifact.durationMs < 0) {
|
|
3714
|
+
throw new Error(`memory config artifact '${path}' is malformed`);
|
|
3715
|
+
}
|
|
3716
|
+
return artifact;
|
|
3717
|
+
}
|
|
3718
|
+
function isFiniteNumberRecord(value) {
|
|
3719
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) return false;
|
|
3720
|
+
return Object.values(value).every((entry) => typeof entry === "number" && Number.isFinite(entry));
|
|
3721
|
+
}
|
|
3722
|
+
function isNonnegativeIntegerRecord(value) {
|
|
3723
|
+
return isFiniteNumberRecord(value) && Object.values(value).every((entry) => Number.isSafeInteger(entry) && entry >= 0);
|
|
3724
|
+
}
|
|
3725
|
+
function hasExactKeys(value, expected) {
|
|
3726
|
+
const keys = Object.keys(value);
|
|
3727
|
+
return keys.length === expected.length && keys.every((key) => expected.includes(key));
|
|
3728
|
+
}
|
|
3729
|
+
|
|
3730
|
+
// src/memory/improvement/activation.ts
|
|
3731
|
+
function appendMemoryActivationEvent(storage, path, event) {
|
|
3732
|
+
appendDurableJournalEvent({
|
|
3733
|
+
storage,
|
|
3734
|
+
path,
|
|
3735
|
+
event,
|
|
3736
|
+
label: "memory activation journal"
|
|
3737
|
+
});
|
|
3738
|
+
}
|
|
3739
|
+
function readMemoryActivationJournal(storage, path, expected) {
|
|
3740
|
+
const stored = storage.read(path);
|
|
3741
|
+
if (stored === void 0) {
|
|
3742
|
+
if (storage.exists(path)) throw new Error(`cannot read memory activation journal '${path}'`);
|
|
3743
|
+
return { prepared: false };
|
|
3744
|
+
}
|
|
3745
|
+
let prepared = false;
|
|
3746
|
+
let activated;
|
|
3747
|
+
for (const [index, line] of stored.split("\n").entries()) {
|
|
3748
|
+
if (!line) continue;
|
|
3749
|
+
let raw;
|
|
3750
|
+
try {
|
|
3751
|
+
raw = JSON.parse(line);
|
|
3752
|
+
} catch (error) {
|
|
3753
|
+
throw new Error(`invalid memory activation journal '${path}' line ${index + 1}`, {
|
|
3754
|
+
cause: error
|
|
3755
|
+
});
|
|
3756
|
+
}
|
|
3757
|
+
const event = parseMemoryActivationEvent(raw, path, index + 1, expected);
|
|
3758
|
+
if (event.status === "prepared") {
|
|
3759
|
+
if (prepared || activated) {
|
|
3760
|
+
throw new Error(`memory activation journal '${path}' repeats its prepared event`);
|
|
3761
|
+
}
|
|
3762
|
+
prepared = true;
|
|
3763
|
+
continue;
|
|
3764
|
+
}
|
|
3765
|
+
if (!prepared || activated) {
|
|
3766
|
+
throw new Error(`memory activation journal '${path}' has an out-of-order activated event`);
|
|
3767
|
+
}
|
|
3768
|
+
activated = event;
|
|
3769
|
+
}
|
|
3770
|
+
return { prepared, ...activated ? { activated } : {} };
|
|
3771
|
+
}
|
|
3772
|
+
async function assertActivatedMemoryWinner(input) {
|
|
3773
|
+
const activationDriver = input.options.activation;
|
|
3774
|
+
if (!activationDriver) {
|
|
3775
|
+
throw new Error("an activated memory journal requires its activation driver");
|
|
3776
|
+
}
|
|
3777
|
+
await input.lease.assertOwned();
|
|
3778
|
+
const currentConfig = await runBoundedMemoryLifecycle({
|
|
3779
|
+
operation: `${activationDriver.ref}: confirm active memory configuration`,
|
|
3780
|
+
timeoutMs: input.options.activationTimeoutMs ?? 6e4,
|
|
3781
|
+
run: () => activationDriver.readCurrent()
|
|
3782
|
+
});
|
|
3783
|
+
await input.lease.assertOwned();
|
|
3784
|
+
const currentHash = surfaceHash4(memoryConfigCodec(input.options).serialize(currentConfig));
|
|
3785
|
+
if (currentHash !== input.result.winnerSurfaceHash) {
|
|
3786
|
+
throw new Error(
|
|
3787
|
+
`memory activation target '${activationDriver.ref}' drifted from measured winner '${input.result.winnerSurfaceHash}' to '${currentHash}'`
|
|
3788
|
+
);
|
|
3789
|
+
}
|
|
3790
|
+
}
|
|
3791
|
+
async function activateMemoryWinner(input) {
|
|
3792
|
+
const activationDriver = input.options.activation;
|
|
3793
|
+
const activationTimeoutMs = input.options.activationTimeoutMs ?? 6e4;
|
|
3794
|
+
if (!input.hadPreparedEvent) {
|
|
3795
|
+
await input.lease.assertOwned();
|
|
3796
|
+
input.storage.ensureDir(input.activationJournalDir);
|
|
3797
|
+
appendMemoryActivationEvent(input.storage, input.activationJournalPath, {
|
|
3798
|
+
...input.activationEventIdentity,
|
|
3799
|
+
status: "prepared",
|
|
3800
|
+
recordedAt: (input.options.now ?? (() => /* @__PURE__ */ new Date()))().toISOString()
|
|
3801
|
+
});
|
|
3802
|
+
}
|
|
3803
|
+
await input.lease.assertOwned();
|
|
3804
|
+
const currentConfig = await runBoundedMemoryLifecycle({
|
|
3805
|
+
operation: `${activationDriver.ref}: read current memory configuration`,
|
|
3806
|
+
timeoutMs: activationTimeoutMs,
|
|
3807
|
+
run: () => activationDriver.readCurrent()
|
|
3808
|
+
});
|
|
3809
|
+
await input.lease.assertOwned();
|
|
3810
|
+
const codec = memoryConfigCodec(input.options);
|
|
3811
|
+
const currentHash = surfaceHash4(codec.serialize(currentConfig));
|
|
3812
|
+
if (currentHash !== input.result.baselineSurfaceHash && currentHash !== input.result.winnerSurfaceHash) {
|
|
3813
|
+
throw new Error(
|
|
3814
|
+
`memory activation target '${activationDriver.ref}' changed concurrently; expected '${input.result.baselineSurfaceHash}' or '${input.result.winnerSurfaceHash}', found '${currentHash}'`
|
|
3815
|
+
);
|
|
3816
|
+
}
|
|
3817
|
+
let outcome;
|
|
3818
|
+
if (currentHash === input.result.winnerSurfaceHash) {
|
|
3819
|
+
outcome = input.hadPreparedEvent ? "recovered" : "already-current";
|
|
3820
|
+
input.result.activation.status = "recovered";
|
|
3821
|
+
} else {
|
|
3822
|
+
let compareError;
|
|
3823
|
+
try {
|
|
3824
|
+
await runBoundedMemoryLifecycle({
|
|
3825
|
+
operation: `${activationDriver.ref}: activate memory configuration`,
|
|
3826
|
+
timeoutMs: activationTimeoutMs,
|
|
3827
|
+
run: () => activationDriver.compareAndSet({
|
|
3828
|
+
activationId: input.result.activation.id,
|
|
3829
|
+
expectedConfig: input.result.baselineConfig,
|
|
3830
|
+
expectedSurfaceHash: input.result.baselineSurfaceHash,
|
|
3831
|
+
config: input.result.winnerConfig,
|
|
3832
|
+
surfaceHash: input.result.winnerSurfaceHash,
|
|
3833
|
+
decision: input.result.decision,
|
|
3834
|
+
optimization: input.result.optimization,
|
|
3835
|
+
finalEvaluation: input.result.finalEvaluation
|
|
3836
|
+
})
|
|
3837
|
+
});
|
|
3838
|
+
} catch (error) {
|
|
3839
|
+
compareError = error;
|
|
3840
|
+
}
|
|
3841
|
+
await input.lease.assertOwned();
|
|
3842
|
+
let observedConfig;
|
|
3843
|
+
try {
|
|
3844
|
+
observedConfig = await runBoundedMemoryLifecycle({
|
|
3845
|
+
operation: `${activationDriver.ref}: confirm memory configuration`,
|
|
3846
|
+
timeoutMs: activationTimeoutMs,
|
|
3847
|
+
run: () => activationDriver.readCurrent()
|
|
3848
|
+
});
|
|
3849
|
+
} catch (error) {
|
|
3850
|
+
if (compareError) {
|
|
3851
|
+
throw new AggregateError(
|
|
3852
|
+
[compareError, error],
|
|
3853
|
+
`memory activation '${input.result.activation.id}' failed and its live state could not be confirmed`
|
|
3854
|
+
);
|
|
3855
|
+
}
|
|
3856
|
+
throw error;
|
|
3857
|
+
}
|
|
3858
|
+
await input.lease.assertOwned();
|
|
3859
|
+
const observedHash = surfaceHash4(codec.serialize(observedConfig));
|
|
3860
|
+
if (observedHash !== input.result.winnerSurfaceHash) {
|
|
3861
|
+
const mismatch = new Error(
|
|
3862
|
+
`memory activation '${input.result.activation.id}' did not install the measured winner; found '${observedHash}'`
|
|
3863
|
+
);
|
|
3864
|
+
if (compareError) {
|
|
3865
|
+
throw new AggregateError(
|
|
3866
|
+
[compareError, mismatch],
|
|
3867
|
+
`memory activation '${input.result.activation.id}' failed without applying the measured winner`
|
|
3868
|
+
);
|
|
3869
|
+
}
|
|
3870
|
+
throw mismatch;
|
|
3871
|
+
}
|
|
3872
|
+
outcome = compareError ? "recovered" : "applied";
|
|
3873
|
+
input.result.activation.status = compareError ? "recovered" : "activated";
|
|
3874
|
+
}
|
|
3875
|
+
await input.lease.assertOwned();
|
|
3876
|
+
appendMemoryActivationEvent(input.storage, input.activationJournalPath, {
|
|
3877
|
+
...input.activationEventIdentity,
|
|
3878
|
+
status: "activated",
|
|
3879
|
+
outcome,
|
|
3880
|
+
recordedAt: (input.options.now ?? (() => /* @__PURE__ */ new Date()))().toISOString()
|
|
3881
|
+
});
|
|
3882
|
+
}
|
|
3883
|
+
function parseMemoryActivationEvent(value, path, line, expected) {
|
|
3884
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
|
3885
|
+
throw new Error(`invalid memory activation journal '${path}' line ${line}`);
|
|
3886
|
+
}
|
|
3887
|
+
const event = value;
|
|
3888
|
+
for (const [key, expectedValue] of Object.entries(expected)) {
|
|
3889
|
+
if (event[key] !== expectedValue) {
|
|
3890
|
+
throw new Error(
|
|
3891
|
+
`memory activation journal '${path}' line ${line} does not match the measured winner`
|
|
3892
|
+
);
|
|
3893
|
+
}
|
|
3894
|
+
}
|
|
3895
|
+
if (event.status !== "prepared" && event.status !== "activated") {
|
|
3896
|
+
throw new Error(`invalid memory activation journal '${path}' line ${line} status`);
|
|
3897
|
+
}
|
|
3898
|
+
const expectedKeys = [
|
|
3899
|
+
...Object.keys(expected),
|
|
3900
|
+
"status",
|
|
3901
|
+
"recordedAt",
|
|
3902
|
+
...event.status === "activated" ? ["outcome"] : []
|
|
3903
|
+
];
|
|
3904
|
+
const actualKeys = Object.keys(event);
|
|
3905
|
+
if (actualKeys.length !== expectedKeys.length || actualKeys.some((key) => !expectedKeys.includes(key))) {
|
|
3906
|
+
throw new Error(`invalid memory activation journal '${path}' line ${line} fields`);
|
|
3907
|
+
}
|
|
3908
|
+
if (typeof event.recordedAt !== "string" || !Number.isFinite(Date.parse(event.recordedAt))) {
|
|
3909
|
+
throw new Error(`invalid memory activation journal '${path}' line ${line} recordedAt`);
|
|
3910
|
+
}
|
|
3911
|
+
if (event.status === "activated" && event.outcome !== "applied" && event.outcome !== "recovered" && event.outcome !== "already-current") {
|
|
3912
|
+
throw new Error(`invalid memory activation journal '${path}' line ${line} outcome`);
|
|
3913
|
+
}
|
|
3914
|
+
if (event.status === "prepared" && event.outcome !== void 0) {
|
|
3915
|
+
throw new Error(`invalid memory activation journal '${path}' line ${line} prepared outcome`);
|
|
3916
|
+
}
|
|
3917
|
+
return value;
|
|
3147
3918
|
}
|
|
3148
3919
|
|
|
3149
3920
|
// src/memory/improvement/output.ts
|
|
@@ -3158,9 +3929,11 @@ function writeMemoryImprovementResult(storage, experimentId, result) {
|
|
|
3158
3929
|
baselineSurface: result.baselineSurface,
|
|
3159
3930
|
winnerSurface: result.winnerSurface,
|
|
3160
3931
|
decision: result.decision,
|
|
3932
|
+
method: result.optimization.methodName,
|
|
3933
|
+
comparison: result.optimization.comparison,
|
|
3934
|
+
finalEvaluation: result.finalEvaluation,
|
|
3161
3935
|
activation: result.activation,
|
|
3162
|
-
totalCostUsd: result.totalCostUsd
|
|
3163
|
-
lineage: result.lineage.toGraph()
|
|
3936
|
+
totalCostUsd: result.totalCostUsd
|
|
3164
3937
|
},
|
|
3165
3938
|
null,
|
|
3166
3939
|
2
|
|
@@ -3179,19 +3952,15 @@ function assertMemoryImprovementOptions(options) {
|
|
|
3179
3952
|
throw new Error(`memory improvement ${name} must be a non-empty string`);
|
|
3180
3953
|
}
|
|
3181
3954
|
}
|
|
3182
|
-
|
|
3183
|
-
|
|
3184
|
-
|
|
3185
|
-
if (typeof options.proposer?.propose !== "function") {
|
|
3186
|
-
throw new Error("memory improvement proposer.propose must be a function");
|
|
3955
|
+
assertImmutableRef(options.implementationRef, "memory improvement implementationRef");
|
|
3956
|
+
if (!options.method || typeof options.method.name !== "string" || !options.method.name.trim() || typeof options.method.optimize !== "function") {
|
|
3957
|
+
throw new Error("memory improvement method must be a complete OptimizationMethod");
|
|
3187
3958
|
}
|
|
3188
|
-
if (
|
|
3189
|
-
throw new Error("memory improvement
|
|
3959
|
+
if (typeof options.createCandidate !== "function") {
|
|
3960
|
+
throw new Error("memory improvement createCandidate must be a function");
|
|
3190
3961
|
}
|
|
3191
3962
|
if (options.activation !== void 0) {
|
|
3192
|
-
|
|
3193
|
-
throw new Error("memory improvement activation.ref must be a non-empty string");
|
|
3194
|
-
}
|
|
3963
|
+
assertImmutableRef(options.activation.ref, "memory improvement activation.ref");
|
|
3195
3964
|
if (typeof options.activation.readCurrent !== "function") {
|
|
3196
3965
|
throw new Error("memory improvement activation.readCurrent must be a function");
|
|
3197
3966
|
}
|
|
@@ -3199,11 +3968,8 @@ function assertMemoryImprovementOptions(options) {
|
|
|
3199
3968
|
throw new Error("memory improvement activation.compareAndSet must be a function");
|
|
3200
3969
|
}
|
|
3201
3970
|
}
|
|
3202
|
-
|
|
3203
|
-
|
|
3204
|
-
if (!proposer || typeof proposer.kind !== "string" || !proposer.kind.trim() || typeof proposer.propose !== "function") {
|
|
3205
|
-
throw new Error(`memory improvement proposer '${name}' is invalid`);
|
|
3206
|
-
}
|
|
3971
|
+
if (options.executeStep) {
|
|
3972
|
+
assertImmutableRef(options.executeStepRef, "memory improvement executeStepRef");
|
|
3207
3973
|
}
|
|
3208
3974
|
if (options.serializeConfig !== void 0 && typeof options.serializeConfig !== "function") {
|
|
3209
3975
|
throw new Error("memory improvement serializeConfig must be a function");
|
|
@@ -3211,12 +3977,7 @@ function assertMemoryImprovementOptions(options) {
|
|
|
3211
3977
|
if (options.parseConfig !== void 0 && typeof options.parseConfig !== "function") {
|
|
3212
3978
|
throw new Error("memory improvement parseConfig must be a function");
|
|
3213
3979
|
}
|
|
3214
|
-
if (!Number.isSafeInteger(options.budget.maxSteps) || options.budget.maxSteps < 0) {
|
|
3215
|
-
throw new Error("memory improvement budget.maxSteps must be a non-negative safe integer");
|
|
3216
|
-
}
|
|
3217
3980
|
for (const [name, value] of [
|
|
3218
|
-
["populationSize", options.populationSize],
|
|
3219
|
-
["candidateConcurrency", options.candidateConcurrency],
|
|
3220
3981
|
["sequenceConcurrency", options.sequenceConcurrency],
|
|
3221
3982
|
["reps", options.reps],
|
|
3222
3983
|
["maxRecoveryAttempts", options.maxRecoveryAttempts],
|
|
@@ -3226,8 +3987,21 @@ function assertMemoryImprovementOptions(options) {
|
|
|
3226
3987
|
throw new Error(`memory improvement ${name} must be a positive safe integer`);
|
|
3227
3988
|
}
|
|
3228
3989
|
}
|
|
3229
|
-
|
|
3230
|
-
|
|
3990
|
+
for (const [name, value] of [["maxTotalCostUsd", options.maxTotalCostUsd]]) {
|
|
3991
|
+
if (value !== void 0 && (!Number.isFinite(value) || value < 0)) {
|
|
3992
|
+
throw new Error(`memory improvement ${name} must be a non-negative finite number`);
|
|
3993
|
+
}
|
|
3994
|
+
}
|
|
3995
|
+
if (options.maximumEvaluationCostUsd !== void 0 && (!Number.isFinite(options.maximumEvaluationCostUsd) || options.maximumEvaluationCostUsd <= 0)) {
|
|
3996
|
+
throw new Error("memory improvement maximumEvaluationCostUsd must be a positive finite number");
|
|
3997
|
+
}
|
|
3998
|
+
if ((options.maxTotalCostUsd ?? 0) > 0 && options.maximumEvaluationCostUsd === void 0) {
|
|
3999
|
+
throw new Error(
|
|
4000
|
+
"memory improvement maximumEvaluationCostUsd is required when a spend limit is configured"
|
|
4001
|
+
);
|
|
4002
|
+
}
|
|
4003
|
+
if (options.allowIncompleteCostAccounting !== void 0 && typeof options.allowIncompleteCostAccounting !== "boolean") {
|
|
4004
|
+
throw new Error("memory improvement allowIncompleteCostAccounting must be boolean");
|
|
3231
4005
|
}
|
|
3232
4006
|
if (options.activationTimeoutMs !== void 0 && (!Number.isSafeInteger(options.activationTimeoutMs) || options.activationTimeoutMs <= 0)) {
|
|
3233
4007
|
throw new Error("memory improvement activationTimeoutMs must be a positive safe integer");
|
|
@@ -3236,17 +4010,9 @@ function assertMemoryImprovementOptions(options) {
|
|
|
3236
4010
|
if (tolerance !== void 0 && (!Number.isFinite(tolerance) || tolerance < 0 || tolerance > 1)) {
|
|
3237
4011
|
throw new Error("memory improvement criticalDimensionTolerance must be between 0 and 1");
|
|
3238
4012
|
}
|
|
3239
|
-
const minimum = options.
|
|
4013
|
+
const minimum = options.minFinalScore;
|
|
3240
4014
|
if (minimum !== void 0 && (!Number.isFinite(minimum) || minimum < 0 || minimum > 1)) {
|
|
3241
|
-
throw new Error("memory improvement
|
|
3242
|
-
}
|
|
3243
|
-
for (const seed of options.seeds) {
|
|
3244
|
-
if (typeof seed.track !== "string" || !seed.track.trim() || typeof seed.proposer !== "string" || !seed.proposer.trim()) {
|
|
3245
|
-
throw new Error("memory improvement seeds require non-empty track and proposer values");
|
|
3246
|
-
}
|
|
3247
|
-
if (seed.vision !== void 0 && (typeof seed.vision !== "string" || !seed.vision.trim())) {
|
|
3248
|
-
throw new Error("memory improvement seed vision must be a non-empty string when provided");
|
|
3249
|
-
}
|
|
4015
|
+
throw new Error("memory improvement minFinalScore must be between 0 and 1");
|
|
3250
4016
|
}
|
|
3251
4017
|
const dimensions = options.criticalDimensions ?? DEFAULT_CRITICAL_DIMENSIONS;
|
|
3252
4018
|
if (dimensions.some((dimension) => typeof dimension !== "string" || !dimension.trim())) {
|
|
@@ -3255,57 +4021,47 @@ function assertMemoryImprovementOptions(options) {
|
|
|
3255
4021
|
if (new Set(dimensions).size !== dimensions.length) {
|
|
3256
4022
|
throw new Error("memory improvement criticalDimensions must be unique");
|
|
3257
4023
|
}
|
|
3258
|
-
|
|
3259
|
-
|
|
4024
|
+
assertIndependentSequences(
|
|
4025
|
+
options.trainSequences,
|
|
4026
|
+
options.selectionSequences,
|
|
4027
|
+
options.finalSequences
|
|
4028
|
+
);
|
|
3260
4029
|
}
|
|
3261
|
-
function
|
|
3262
|
-
|
|
3263
|
-
|
|
3264
|
-
|
|
3265
|
-
|
|
3266
|
-
|
|
3267
|
-
|
|
3268
|
-
|
|
3269
|
-
|
|
4030
|
+
function assertIndependentSequences(train, selection, final) {
|
|
4031
|
+
if (train.length === 0) throw new Error("memory improvement requires training sequences");
|
|
4032
|
+
if (selection.length === 0) throw new Error("memory improvement requires selection sequences");
|
|
4033
|
+
if (final.length < 2) {
|
|
4034
|
+
throw new Error("memory improvement requires at least 2 final sequences");
|
|
4035
|
+
}
|
|
4036
|
+
const ids = /* @__PURE__ */ new Map();
|
|
4037
|
+
const content = /* @__PURE__ */ new Map();
|
|
4038
|
+
for (const [split, sequences] of [
|
|
4039
|
+
["train", train],
|
|
4040
|
+
["selection", selection],
|
|
4041
|
+
["final", final]
|
|
4042
|
+
]) {
|
|
4043
|
+
for (const sequence of sequences) {
|
|
4044
|
+
const priorId = ids.get(sequence.id);
|
|
4045
|
+
if (priorId) {
|
|
4046
|
+
throw new Error(
|
|
4047
|
+
`memory improvement ${priorId}/${split} sequences share id '${sequence.id}'`
|
|
4048
|
+
);
|
|
4049
|
+
}
|
|
4050
|
+
ids.set(sequence.id, split);
|
|
4051
|
+
const fingerprint = memorySequenceFingerprint(sequence);
|
|
4052
|
+
const priorContent = content.get(fingerprint);
|
|
4053
|
+
if (priorContent) {
|
|
4054
|
+
throw new Error(
|
|
4055
|
+
`memory improvement ${priorContent.split}/${split} histories duplicate content at '${priorContent.sequenceId}'/'${sequence.id}'`
|
|
4056
|
+
);
|
|
4057
|
+
}
|
|
4058
|
+
content.set(fingerprint, { split, sequenceId: sequence.id });
|
|
3270
4059
|
}
|
|
3271
|
-
idsByFingerprint.set(fingerprint, sequence.id);
|
|
3272
4060
|
}
|
|
3273
4061
|
}
|
|
3274
4062
|
|
|
3275
4063
|
// src/memory/improvement/run.ts
|
|
3276
4064
|
async function runAgentMemoryImprovement(options) {
|
|
3277
|
-
if ("onPromote" in options) {
|
|
3278
|
-
throw new Error(
|
|
3279
|
-
"memory improvement onPromote was removed; use activation.readCurrent and activation.compareAndSet"
|
|
3280
|
-
);
|
|
3281
|
-
}
|
|
3282
|
-
if (options.seeds.length === 0) throw new Error("memory improvement requires seed configs");
|
|
3283
|
-
if (options.trainSequences.length === 0) {
|
|
3284
|
-
throw new Error("memory improvement requires training sequences");
|
|
3285
|
-
}
|
|
3286
|
-
if (options.holdoutSequences.length === 0) {
|
|
3287
|
-
throw new Error("memory improvement requires holdout sequences");
|
|
3288
|
-
}
|
|
3289
|
-
const trainIds = new Set(options.trainSequences.map((sequence) => sequence.id));
|
|
3290
|
-
const overlap = options.holdoutSequences.map((sequence) => sequence.id).filter((id) => trainIds.has(id));
|
|
3291
|
-
if (overlap.length > 0) {
|
|
3292
|
-
throw new Error(`memory improvement train/holdout overlap: ${overlap.join(", ")}`);
|
|
3293
|
-
}
|
|
3294
|
-
const trainFingerprints = new Map(
|
|
3295
|
-
options.trainSequences.map((sequence) => [memorySequenceFingerprint(sequence), sequence.id])
|
|
3296
|
-
);
|
|
3297
|
-
const duplicateHistories = options.holdoutSequences.flatMap((sequence) => {
|
|
3298
|
-
const trainId = trainFingerprints.get(memorySequenceFingerprint(sequence));
|
|
3299
|
-
return trainId ? [`${trainId}/${sequence.id}`] : [];
|
|
3300
|
-
});
|
|
3301
|
-
if (duplicateHistories.length > 0) {
|
|
3302
|
-
throw new Error(
|
|
3303
|
-
`memory improvement train/holdout histories duplicate content: ${duplicateHistories.join(", ")}`
|
|
3304
|
-
);
|
|
3305
|
-
}
|
|
3306
|
-
if (typeof options.improvementRef !== "string" || !options.improvementRef.trim()) {
|
|
3307
|
-
throw new Error("memory improvement improvementRef must be a non-empty string");
|
|
3308
|
-
}
|
|
3309
4065
|
assertMemoryImprovementOptions(options);
|
|
3310
4066
|
const storage = options.storage ?? fsCampaignStorage2();
|
|
3311
4067
|
const runDir = resolveRunDir2(options.runDir, options.repo);
|
|
@@ -3346,274 +4102,166 @@ async function runAgentMemoryImprovement(options) {
|
|
|
3346
4102
|
}
|
|
3347
4103
|
async function runAgentMemoryImprovementOwned(options, storage, runDir, lease) {
|
|
3348
4104
|
await lease.assertOwned();
|
|
3349
|
-
const
|
|
3350
|
-
|
|
3351
|
-
const serialize = (config) => serializeMemoryConfig(serializeRaw, config);
|
|
3352
|
-
const parse = (surface) => parseMemoryConfig(parseRaw, surface);
|
|
3353
|
-
for (const seed of options.seeds) assertMemoryConfigRoundTrip(seed.config, serialize, parse);
|
|
4105
|
+
const codec = memoryConfigCodec(options);
|
|
4106
|
+
assertMemoryConfigRoundTrip(options.baselineConfig, codec.serialize, codec.parse);
|
|
3354
4107
|
if (options.activation && !storage.append) {
|
|
3355
4108
|
throw new Error("memory activation requires CampaignStorage.append");
|
|
3356
4109
|
}
|
|
3357
|
-
|
|
3358
|
-
|
|
3359
|
-
|
|
3360
|
-
|
|
3361
|
-
const
|
|
3362
|
-
|
|
3363
|
-
|
|
3364
|
-
|
|
3365
|
-
|
|
3366
|
-
|
|
3367
|
-
|
|
3368
|
-
|
|
3369
|
-
|
|
3370
|
-
kind: "agent-memory-config-search",
|
|
3371
|
-
sequenceId: sequence.id
|
|
3372
|
-
}));
|
|
3373
|
-
const evaluations = /* @__PURE__ */ new Map();
|
|
3374
|
-
const evaluateSurface = async (surface) => {
|
|
3375
|
-
await lease.assertOwned();
|
|
3376
|
-
const text = requireStringSurface(surface);
|
|
3377
|
-
const config = parse(text);
|
|
3378
|
-
const canonicalSurface = serialize(config);
|
|
3379
|
-
const hash = surfaceHash2(canonicalSurface);
|
|
3380
|
-
let pending = evaluations.get(hash);
|
|
3381
|
-
if (!pending) {
|
|
3382
|
-
pending = (async () => {
|
|
3383
|
-
const candidate = await buildCandidate(options, config, hash, `search-${hash}`);
|
|
3384
|
-
await lease.assertOwned();
|
|
3385
|
-
const experiment = await runAgentMemoryExperiment({
|
|
3386
|
-
...experimentOptions(options, costLedger, storage, lease),
|
|
3387
|
-
experimentId: `${options.experimentId}:search:${hash}`,
|
|
3388
|
-
sequences: options.trainSequences,
|
|
3389
|
-
candidates: [candidate],
|
|
3390
|
-
runDir: join3(runDir, "search", hash),
|
|
3391
|
-
costPhase: `memory.search.${hash}`
|
|
3392
|
-
});
|
|
3393
|
-
await lease.assertOwned();
|
|
3394
|
-
return experiment;
|
|
3395
|
-
})();
|
|
3396
|
-
evaluations.set(hash, pending);
|
|
3397
|
-
void pending.catch(() => evaluations.delete(hash));
|
|
3398
|
-
}
|
|
3399
|
-
const result2 = await pending;
|
|
3400
|
-
await lease.assertOwned();
|
|
3401
|
-
const row = result2.rows[0];
|
|
3402
|
-
if (!row || row.cellsFailed > 0) {
|
|
3403
|
-
throw new Error(`${hash}: memory candidate did not complete every training cell`);
|
|
4110
|
+
assertMemoryImprovementIdentity(options, storage, runDir, codec.serialize);
|
|
4111
|
+
const trainScenarios = memoryConfigScenarios(options.trainSequences);
|
|
4112
|
+
const selectionScenarios = memoryConfigScenarios(options.selectionSequences);
|
|
4113
|
+
const finalScenarios = memoryConfigScenarios(options.finalSequences);
|
|
4114
|
+
const finalScenarioIds = new Set(finalScenarios.map((scenario) => scenario.id));
|
|
4115
|
+
const pending = /* @__PURE__ */ new Map();
|
|
4116
|
+
const candidates = /* @__PURE__ */ new Map();
|
|
4117
|
+
const candidateFor = (config, hash) => {
|
|
4118
|
+
let candidate = candidates.get(hash);
|
|
4119
|
+
if (!candidate) {
|
|
4120
|
+
candidate = buildRegisteredCandidate(options, storage, runDir, config, hash);
|
|
4121
|
+
candidates.set(hash, candidate);
|
|
4122
|
+
void candidate.catch(() => candidates.delete(hash));
|
|
3404
4123
|
}
|
|
3405
|
-
return
|
|
3406
|
-
score: row.scoreMean,
|
|
3407
|
-
scoreVector: sequenceScores(result2, options.trainSequences, row.candidateId)
|
|
3408
|
-
};
|
|
4124
|
+
return candidate;
|
|
3409
4125
|
};
|
|
3410
|
-
const
|
|
3411
|
-
|
|
3412
|
-
|
|
3413
|
-
|
|
3414
|
-
|
|
3415
|
-
|
|
3416
|
-
|
|
3417
|
-
|
|
3418
|
-
scenarios: trainScenarios,
|
|
3419
|
-
proposer: withCostContext(options.proposer, costLedger, lease, "default"),
|
|
3420
|
-
proposers: options.proposers ? Object.fromEntries(
|
|
3421
|
-
Object.entries(options.proposers).map(([name, proposer]) => [
|
|
3422
|
-
name,
|
|
3423
|
-
withCostContext(proposer, costLedger, lease, name)
|
|
3424
|
-
])
|
|
3425
|
-
) : void 0,
|
|
3426
|
-
scoreSurface: evaluateSurface,
|
|
3427
|
-
governor: options.governor ? withGovernorCostContext(options.governor, costLedger, lease) : void 0,
|
|
3428
|
-
budget: {
|
|
3429
|
-
...options.budget,
|
|
3430
|
-
maxNodes: options.seeds.length + options.budget.maxSteps
|
|
3431
|
-
},
|
|
3432
|
-
store: lineageStore,
|
|
3433
|
-
populationSize: options.populationSize,
|
|
3434
|
-
candidateConcurrency: options.candidateConcurrency
|
|
4126
|
+
const optimizationRunOptions = {
|
|
4127
|
+
...options.optimizationRunOptions ?? {},
|
|
4128
|
+
storage,
|
|
4129
|
+
...options.reps !== void 0 ? { reps: options.reps } : {},
|
|
4130
|
+
...options.resumable !== void 0 ? { resumable: options.resumable } : {},
|
|
4131
|
+
...options.sequenceConcurrency !== void 0 ? { maxConcurrency: options.sequenceConcurrency } : {},
|
|
4132
|
+
...options.dispatchTimeoutMs !== void 0 ? { dispatchTimeoutMs: options.dispatchTimeoutMs } : {},
|
|
4133
|
+
expectUsage: "off"
|
|
3435
4134
|
};
|
|
3436
|
-
const
|
|
4135
|
+
const optimization = await runSerializedKnowledgeOptimization({
|
|
4136
|
+
executionRef: options.implementationRef,
|
|
4137
|
+
baseline: options.baselineConfig,
|
|
4138
|
+
method: options.method,
|
|
4139
|
+
trainScenarios,
|
|
4140
|
+
selectionScenarios,
|
|
4141
|
+
finalScenarios,
|
|
4142
|
+
codec,
|
|
4143
|
+
scenarioFingerprint: (scenario) => memorySequenceFingerprint(scenario.sequence),
|
|
4144
|
+
dispatchCandidate: async ({ candidate, candidateSurfaceHash, scenario, context }) => {
|
|
4145
|
+
const key = memoryArtifactPath(
|
|
4146
|
+
runDir,
|
|
4147
|
+
candidateSurfaceHash,
|
|
4148
|
+
scenario,
|
|
4149
|
+
context.rep,
|
|
4150
|
+
context.seed
|
|
4151
|
+
);
|
|
4152
|
+
let operation = pending.get(key);
|
|
4153
|
+
if (!operation) {
|
|
4154
|
+
operation = candidateFor(candidate, candidateSurfaceHash).then(
|
|
4155
|
+
(builtCandidate) => evaluateMemoryCandidate({
|
|
4156
|
+
options,
|
|
4157
|
+
storage,
|
|
4158
|
+
runDir,
|
|
4159
|
+
lease,
|
|
4160
|
+
candidate: builtCandidate,
|
|
4161
|
+
surfaceHash: candidateSurfaceHash,
|
|
4162
|
+
scenario,
|
|
4163
|
+
rep: context.rep,
|
|
4164
|
+
seed: context.seed,
|
|
4165
|
+
final: finalScenarioIds.has(scenario.id),
|
|
4166
|
+
cost: context.cost,
|
|
4167
|
+
signal: context.signal
|
|
4168
|
+
})
|
|
4169
|
+
);
|
|
4170
|
+
pending.set(key, operation);
|
|
4171
|
+
void operation.catch(() => pending.delete(key));
|
|
4172
|
+
}
|
|
4173
|
+
return operation;
|
|
4174
|
+
},
|
|
4175
|
+
judges: [agentMemorySequenceJudge()],
|
|
4176
|
+
runDir,
|
|
4177
|
+
repo: options.repo,
|
|
4178
|
+
storage,
|
|
4179
|
+
seed: options.seed,
|
|
4180
|
+
reps: options.reps,
|
|
4181
|
+
resumable: options.resumable,
|
|
4182
|
+
costCeiling: options.maxTotalCostUsd ?? 0,
|
|
4183
|
+
maxConcurrency: options.sequenceConcurrency,
|
|
4184
|
+
dispatchTimeoutMs: options.dispatchTimeoutMs,
|
|
4185
|
+
expectUsage: "off",
|
|
4186
|
+
optimizationRunOptions,
|
|
4187
|
+
now: options.now
|
|
4188
|
+
});
|
|
3437
4189
|
await lease.assertOwned();
|
|
3438
|
-
const
|
|
3439
|
-
|
|
3440
|
-
|
|
3441
|
-
|
|
3442
|
-
const
|
|
3443
|
-
|
|
3444
|
-
|
|
3445
|
-
|
|
3446
|
-
|
|
3447
|
-
|
|
3448
|
-
|
|
3449
|
-
|
|
3450
|
-
|
|
3451
|
-
|
|
3452
|
-
|
|
3453
|
-
|
|
3454
|
-
winnerScore: baselineMeasurement.score,
|
|
3455
|
-
lift: 0,
|
|
3456
|
-
criticalDimensions: []
|
|
3457
|
-
};
|
|
3458
|
-
} else {
|
|
3459
|
-
await lease.assertOwned();
|
|
3460
|
-
const [baselineCandidate, winnerCandidate] = await Promise.all([
|
|
3461
|
-
buildCandidate(options, baselineConfig, baselineHash, "baseline"),
|
|
3462
|
-
buildCandidate(options, winnerConfig, winnerHash, "winner")
|
|
3463
|
-
]);
|
|
3464
|
-
await lease.assertOwned();
|
|
3465
|
-
holdout = await runAgentMemoryExperiment({
|
|
3466
|
-
...experimentOptions(options, costLedger, storage, lease),
|
|
3467
|
-
experimentId: `${options.experimentId}:holdout`,
|
|
3468
|
-
sequences: options.holdoutSequences,
|
|
3469
|
-
candidates: [baselineCandidate, winnerCandidate],
|
|
3470
|
-
runDir: join3(runDir, "holdout"),
|
|
3471
|
-
costPhase: "memory.holdout"
|
|
3472
|
-
});
|
|
3473
|
-
decision = decidePromotion({
|
|
3474
|
-
options,
|
|
3475
|
-
result: holdout,
|
|
3476
|
-
baselineId: baselineCandidate.id,
|
|
3477
|
-
winnerId: winnerCandidate.id
|
|
3478
|
-
});
|
|
3479
|
-
}
|
|
3480
|
-
const resultJsonPath = join3(runDir, "memory-improvement-result.json");
|
|
4190
|
+
const [baselineCandidate, winnerCandidate] = await Promise.all([
|
|
4191
|
+
candidateFor(optimization.baseline.value, optimization.baseline.surfaceHash),
|
|
4192
|
+
candidateFor(optimization.winner.value, optimization.winner.surfaceHash)
|
|
4193
|
+
]);
|
|
4194
|
+
const finalEvaluation = loadFinalEvaluation({
|
|
4195
|
+
options,
|
|
4196
|
+
storage,
|
|
4197
|
+
runDir,
|
|
4198
|
+
baselineSurfaceHash: optimization.baseline.surfaceHash,
|
|
4199
|
+
baselineCandidateRef: baselineCandidate.ref,
|
|
4200
|
+
winnerSurfaceHash: optimization.winner.surfaceHash,
|
|
4201
|
+
winnerCandidateRef: winnerCandidate.ref
|
|
4202
|
+
});
|
|
4203
|
+
const unchanged = optimization.baseline.surfaceHash === optimization.winner.surfaceHash;
|
|
4204
|
+
const decision = decidePromotion({ options, optimization, finalEvaluation, unchanged });
|
|
4205
|
+
const resultJsonPath = join5(runDir, "memory-improvement-result.json");
|
|
3481
4206
|
const activationRef = options.activation?.ref ?? "not-configured";
|
|
3482
|
-
const activationId = `memory-activation-${
|
|
3483
|
-
|
|
4207
|
+
const activationId = `memory-activation-${surfaceHash5(
|
|
4208
|
+
canonicalJson10({
|
|
3484
4209
|
experimentId: options.experimentId,
|
|
3485
|
-
|
|
4210
|
+
implementationRef: options.implementationRef,
|
|
4211
|
+
method: optimization.methodName,
|
|
3486
4212
|
activationRef,
|
|
3487
|
-
baselineSurfaceHash:
|
|
3488
|
-
winnerSurfaceHash:
|
|
3489
|
-
|
|
4213
|
+
baselineSurfaceHash: optimization.baseline.surfaceHash,
|
|
4214
|
+
winnerSurfaceHash: optimization.winner.surfaceHash,
|
|
4215
|
+
finalEvaluationHash: finalEvaluation.manifestHash,
|
|
3490
4216
|
promotionPolicy: normalizedPromotionPolicy(options)
|
|
3491
4217
|
})
|
|
3492
4218
|
)}`;
|
|
3493
|
-
const activationJournalDir =
|
|
3494
|
-
const activationJournalPath =
|
|
3495
|
-
const activationEligible = decision.status === "promote"
|
|
3496
|
-
const activationEventIdentity =
|
|
3497
|
-
schema: 1,
|
|
4219
|
+
const activationJournalDir = join5(runDir, "activations");
|
|
4220
|
+
const activationJournalPath = join5(activationJournalDir, `${activationId}.jsonl`);
|
|
4221
|
+
const activationEligible = decision.status === "promote";
|
|
4222
|
+
const activationEventIdentity = {
|
|
3498
4223
|
activationId,
|
|
3499
4224
|
experimentId: options.experimentId,
|
|
3500
4225
|
activationRef,
|
|
3501
|
-
baselineSurfaceHash:
|
|
3502
|
-
winnerSurfaceHash:
|
|
3503
|
-
|
|
3504
|
-
}
|
|
3505
|
-
const activationJournal = activationEligible
|
|
4226
|
+
baselineSurfaceHash: optimization.baseline.surfaceHash,
|
|
4227
|
+
winnerSurfaceHash: optimization.winner.surfaceHash,
|
|
4228
|
+
finalEvaluationHash: finalEvaluation.manifestHash
|
|
4229
|
+
};
|
|
4230
|
+
const activationJournal = activationEligible ? readMemoryActivationJournal(storage, activationJournalPath, activationEventIdentity) : { prepared: false };
|
|
3506
4231
|
const activation = {
|
|
3507
4232
|
id: activationId,
|
|
3508
4233
|
status: !activationEligible ? "not-eligible" : activationJournal.activated ? "already-activated" : options.activation ? "pending" : "not-configured",
|
|
3509
4234
|
journalPath: activationJournalPath
|
|
3510
4235
|
};
|
|
3511
4236
|
const result = {
|
|
3512
|
-
|
|
3513
|
-
baselineConfig,
|
|
3514
|
-
winnerConfig,
|
|
3515
|
-
baselineSurface,
|
|
3516
|
-
winnerSurface,
|
|
3517
|
-
baselineSurfaceHash:
|
|
3518
|
-
winnerSurfaceHash:
|
|
4237
|
+
optimization,
|
|
4238
|
+
baselineConfig: optimization.baseline.value,
|
|
4239
|
+
winnerConfig: optimization.winner.value,
|
|
4240
|
+
baselineSurface: optimization.baseline.surface,
|
|
4241
|
+
winnerSurface: optimization.winner.surface,
|
|
4242
|
+
baselineSurfaceHash: optimization.baseline.surfaceHash,
|
|
4243
|
+
winnerSurfaceHash: optimization.winner.surfaceHash,
|
|
3519
4244
|
decision,
|
|
4245
|
+
finalEvaluation,
|
|
3520
4246
|
activation,
|
|
3521
|
-
|
|
3522
|
-
totalCostUsd: costLedger.summary().totalCostUsd,
|
|
4247
|
+
totalCostUsd: optimization.comparison.totalCost.totalCostUsd,
|
|
3523
4248
|
resultJsonPath
|
|
3524
4249
|
};
|
|
3525
4250
|
await lease.assertOwned();
|
|
4251
|
+
if (activationJournal.activated) {
|
|
4252
|
+
await assertActivatedMemoryWinner({ options, lease, result });
|
|
4253
|
+
}
|
|
3526
4254
|
writeMemoryImprovementResult(storage, options.experimentId, result);
|
|
3527
|
-
if (activationEligible && !activationJournal.activated &&
|
|
3528
|
-
|
|
3529
|
-
|
|
3530
|
-
|
|
3531
|
-
|
|
3532
|
-
|
|
3533
|
-
|
|
3534
|
-
|
|
3535
|
-
|
|
3536
|
-
|
|
3537
|
-
recordedAt: (options.now ?? (() => /* @__PURE__ */ new Date()))().toISOString()
|
|
3538
|
-
});
|
|
3539
|
-
}
|
|
3540
|
-
await lease.assertOwned();
|
|
3541
|
-
const currentConfig = await runBoundedMemoryLifecycle({
|
|
3542
|
-
operation: `${activationDriver.ref}: read current memory configuration`,
|
|
3543
|
-
timeoutMs: activationTimeoutMs,
|
|
3544
|
-
run: () => activationDriver.readCurrent()
|
|
3545
|
-
});
|
|
3546
|
-
await lease.assertOwned();
|
|
3547
|
-
const currentHash = surfaceHash2(serialize(currentConfig));
|
|
3548
|
-
if (currentHash !== baselineHash && currentHash !== winnerHash) {
|
|
3549
|
-
throw new Error(
|
|
3550
|
-
`memory activation target '${activationDriver.ref}' changed concurrently; expected '${baselineHash}' or '${winnerHash}', found '${currentHash}'`
|
|
3551
|
-
);
|
|
3552
|
-
}
|
|
3553
|
-
let outcome;
|
|
3554
|
-
if (currentHash === winnerHash) {
|
|
3555
|
-
outcome = hadPreparedEvent ? "recovered" : "already-current";
|
|
3556
|
-
activation.status = "recovered";
|
|
3557
|
-
} else {
|
|
3558
|
-
let compareError;
|
|
3559
|
-
try {
|
|
3560
|
-
await runBoundedMemoryLifecycle({
|
|
3561
|
-
operation: `${activationDriver.ref}: activate memory configuration`,
|
|
3562
|
-
timeoutMs: activationTimeoutMs,
|
|
3563
|
-
run: () => activationDriver.compareAndSet({
|
|
3564
|
-
activationId,
|
|
3565
|
-
expectedConfig: baselineConfig,
|
|
3566
|
-
expectedSurfaceHash: baselineHash,
|
|
3567
|
-
config: winnerConfig,
|
|
3568
|
-
surfaceHash: winnerHash,
|
|
3569
|
-
decision,
|
|
3570
|
-
lineage: search.lineage,
|
|
3571
|
-
holdout
|
|
3572
|
-
})
|
|
3573
|
-
});
|
|
3574
|
-
} catch (error) {
|
|
3575
|
-
compareError = error;
|
|
3576
|
-
}
|
|
3577
|
-
await lease.assertOwned();
|
|
3578
|
-
let observedConfig;
|
|
3579
|
-
try {
|
|
3580
|
-
observedConfig = await runBoundedMemoryLifecycle({
|
|
3581
|
-
operation: `${activationDriver.ref}: confirm memory configuration`,
|
|
3582
|
-
timeoutMs: activationTimeoutMs,
|
|
3583
|
-
run: () => activationDriver.readCurrent()
|
|
3584
|
-
});
|
|
3585
|
-
} catch (error) {
|
|
3586
|
-
if (compareError) {
|
|
3587
|
-
throw new AggregateError(
|
|
3588
|
-
[compareError, error],
|
|
3589
|
-
`memory activation '${activationId}' failed and its live state could not be confirmed`
|
|
3590
|
-
);
|
|
3591
|
-
}
|
|
3592
|
-
throw error;
|
|
3593
|
-
}
|
|
3594
|
-
await lease.assertOwned();
|
|
3595
|
-
const observedHash = surfaceHash2(serialize(observedConfig));
|
|
3596
|
-
if (observedHash !== winnerHash) {
|
|
3597
|
-
const mismatch = new Error(
|
|
3598
|
-
`memory activation '${activationId}' did not install the measured winner; found '${observedHash}'`
|
|
3599
|
-
);
|
|
3600
|
-
if (compareError) {
|
|
3601
|
-
throw new AggregateError(
|
|
3602
|
-
[compareError, mismatch],
|
|
3603
|
-
`memory activation '${activationId}' failed without applying the measured winner`
|
|
3604
|
-
);
|
|
3605
|
-
}
|
|
3606
|
-
throw mismatch;
|
|
3607
|
-
}
|
|
3608
|
-
outcome = compareError ? "recovered" : "applied";
|
|
3609
|
-
activation.status = compareError ? "recovered" : "activated";
|
|
3610
|
-
}
|
|
3611
|
-
await lease.assertOwned();
|
|
3612
|
-
appendMemoryActivationEvent(storage, activationJournalPath, {
|
|
3613
|
-
...activationEventIdentity,
|
|
3614
|
-
status: "activated",
|
|
3615
|
-
outcome,
|
|
3616
|
-
recordedAt: (options.now ?? (() => /* @__PURE__ */ new Date()))().toISOString()
|
|
4255
|
+
if (activationEligible && !activationJournal.activated && options.activation) {
|
|
4256
|
+
await activateMemoryWinner({
|
|
4257
|
+
options,
|
|
4258
|
+
storage,
|
|
4259
|
+
lease,
|
|
4260
|
+
result,
|
|
4261
|
+
activationEventIdentity,
|
|
4262
|
+
activationJournalDir,
|
|
4263
|
+
activationJournalPath,
|
|
4264
|
+
hadPreparedEvent: activationJournal.prepared
|
|
3617
4265
|
});
|
|
3618
4266
|
writeMemoryImprovementResult(storage, options.experimentId, result);
|
|
3619
4267
|
}
|
|
@@ -3622,7 +4270,7 @@ async function runAgentMemoryImprovementOwned(options, storage, runDir, lease) {
|
|
|
3622
4270
|
|
|
3623
4271
|
// src/memory/mem0.ts
|
|
3624
4272
|
import { randomUUID as randomUUID4 } from "crypto";
|
|
3625
|
-
import { canonicalJson as
|
|
4273
|
+
import { canonicalJson as canonicalJson11 } from "@tangle-network/agent-eval";
|
|
3626
4274
|
function createMem0MemoryAdapter(options) {
|
|
3627
4275
|
assertMem0Options(options);
|
|
3628
4276
|
const id = options.id ?? `mem0-${options.mode}`;
|
|
@@ -3938,7 +4586,7 @@ function normalizeMem0Hits(raw, adapterId, options) {
|
|
|
3938
4586
|
if (options.minScore !== void 0 && (score === void 0 || score < options.minScore)) {
|
|
3939
4587
|
return void 0;
|
|
3940
4588
|
}
|
|
3941
|
-
const memoryId = stringField2(item, ["id"]) ?? stableId("mem",
|
|
4589
|
+
const memoryId = stringField2(item, ["id"]) ?? stableId("mem", canonicalJson11(compactRecord2({ adapterId, text, metadata, item })));
|
|
3942
4590
|
const normalizedScore = score !== void 0 && score >= 0 && score <= 1 ? score : void 0;
|
|
3943
4591
|
return {
|
|
3944
4592
|
id: memoryId,
|
|
@@ -4137,11 +4785,11 @@ function mem0MemoryAdapterIdentity(options) {
|
|
|
4137
4785
|
if (typeof options.backendRef !== "string" || !options.backendRef.trim()) {
|
|
4138
4786
|
throw new Error("Mem0 backendRef must be a non-empty string");
|
|
4139
4787
|
}
|
|
4140
|
-
return
|
|
4788
|
+
return `sha256:${sha256(canonicalJson11(compactRecord2(options)))}`;
|
|
4141
4789
|
}
|
|
4142
4790
|
|
|
4143
4791
|
// src/memory/neo4j.ts
|
|
4144
|
-
import { canonicalJson as
|
|
4792
|
+
import { canonicalJson as canonicalJson12 } from "@tangle-network/agent-eval";
|
|
4145
4793
|
function createNeo4jAgentMemoryAdapter(options) {
|
|
4146
4794
|
assertNeo4jOptions(options);
|
|
4147
4795
|
const client = options.client;
|
|
@@ -4406,7 +5054,7 @@ async function writeNeo4jMemory(client, input, adapterId, transport, isolatedIns
|
|
|
4406
5054
|
assertNeo4jWriteSucceeded(result, adapterId, input.kind);
|
|
4407
5055
|
const id = idFromResult(result) ?? input.id ?? stableId(
|
|
4408
5056
|
"mem",
|
|
4409
|
-
|
|
5057
|
+
canonicalJson12({ adapterId, kind: input.kind, text: input.text, scope: input.scope ?? {} })
|
|
4410
5058
|
);
|
|
4411
5059
|
return {
|
|
4412
5060
|
accepted: true,
|
|
@@ -4755,6 +5403,11 @@ var AgentMemoryWriteInputSchema = z.object({
|
|
|
4755
5403
|
});
|
|
4756
5404
|
|
|
4757
5405
|
export {
|
|
5406
|
+
assertImmutableRef,
|
|
5407
|
+
runSerializedKnowledgeOptimization,
|
|
5408
|
+
jsonCandidateCodec,
|
|
5409
|
+
jsonObjectCandidateCodec,
|
|
5410
|
+
scenarioContentFingerprint,
|
|
4758
5411
|
deterministicRng,
|
|
4759
5412
|
retrievalHoldoutConfigHash,
|
|
4760
5413
|
applyRetrievalHoldout,
|
|
@@ -4781,4 +5434,4 @@ export {
|
|
|
4781
5434
|
AgentMemoryHitSchema,
|
|
4782
5435
|
AgentMemoryWriteInputSchema
|
|
4783
5436
|
};
|
|
4784
|
-
//# sourceMappingURL=chunk-
|
|
5437
|
+
//# sourceMappingURL=chunk-LMR53POQ.js.map
|