@agent_memory/dsh 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/plugin.mjs +692 -107
- package/dist/plugin.mjs.map +4 -4
- package/package.json +1 -1
package/dist/plugin.mjs
CHANGED
|
@@ -104,6 +104,9 @@ function memoryPaths(home) {
|
|
|
104
104
|
manifestFile: join(home, ".index", "manifest.json"),
|
|
105
105
|
stateDir: join(home, ".state"),
|
|
106
106
|
journalDir: join(home, ".state", "journal"),
|
|
107
|
+
journalArchiveDir: join(home, ".state", "journal", "archive"),
|
|
108
|
+
checkpointFile: join(home, ".state", "checkpoint.json"),
|
|
109
|
+
checkpointKeyFile: join(home, ".state", "checkpoint.key"),
|
|
107
110
|
txnDir: join(home, ".state", "txn"),
|
|
108
111
|
snapshotsDir: join(home, ".state", "snapshots")
|
|
109
112
|
};
|
|
@@ -1145,17 +1148,17 @@ function loadPolicy(options) {
|
|
|
1145
1148
|
hint: "a human must review it and run `mem policy seal --accept`"
|
|
1146
1149
|
});
|
|
1147
1150
|
const seal2 = parseSeal(sealText, paths.sealFile);
|
|
1148
|
-
const
|
|
1149
|
-
if (seal2.sha256 !==
|
|
1151
|
+
const sha2562 = hashBytes(policyText);
|
|
1152
|
+
if (seal2.sha256 !== sha2562) {
|
|
1150
1153
|
throw new StartupError("POLICY_MISMATCH", "policy.json does not match its sealed hash", {
|
|
1151
1154
|
sourcePath: paths.policyFile,
|
|
1152
|
-
details: [`sealed sha256 ${seal2.sha256}`, `actual sha256 ${
|
|
1155
|
+
details: [`sealed sha256 ${seal2.sha256}`, `actual sha256 ${sha2562}`],
|
|
1153
1156
|
hint: "the policy is immutable at runtime; after a deliberate human edit run `mem policy seal --accept`"
|
|
1154
1157
|
});
|
|
1155
1158
|
}
|
|
1156
1159
|
const { policy, problems } = parsePolicyText(policyText);
|
|
1157
1160
|
if (problems.length > 0) throw policyProblems(paths.policyFile, problems);
|
|
1158
|
-
return { policy, sha256, sealedAt: seal2.sealed_at, policyPath: paths.policyFile, sealPath: paths.sealFile };
|
|
1161
|
+
return { policy, sha256: sha2562, sealedAt: seal2.sealed_at, policyPath: paths.policyFile, sealPath: paths.sealFile };
|
|
1159
1162
|
}
|
|
1160
1163
|
function sealPolicy(options) {
|
|
1161
1164
|
const paths = memoryPaths(options.home);
|
|
@@ -1182,9 +1185,9 @@ function sealPolicy(options) {
|
|
|
1182
1185
|
`;
|
|
1183
1186
|
writeFileAtomic(paths.policyFile, nextText);
|
|
1184
1187
|
const previous = tryReadSeal(paths.sealFile);
|
|
1185
|
-
const
|
|
1186
|
-
writeSeal(paths.sealFile, { policy: "policy.json", sha256, sealed_at: (options.now ?? /* @__PURE__ */ new Date()).toISOString() });
|
|
1187
|
-
return { sealPath: paths.sealFile, sha256, previousSha256: previous?.sha256 ?? null, sealedFiles };
|
|
1188
|
+
const sha2562 = hashBytes(nextText);
|
|
1189
|
+
writeSeal(paths.sealFile, { policy: "policy.json", sha256: sha2562, sealed_at: (options.now ?? /* @__PURE__ */ new Date()).toISOString() });
|
|
1190
|
+
return { sealPath: paths.sealFile, sha256: sha2562, previousSha256: previous?.sha256 ?? null, sealedFiles };
|
|
1188
1191
|
}
|
|
1189
1192
|
function writeSeal(sealPath, record) {
|
|
1190
1193
|
writeFileAtomic(sealPath, `${JSON.stringify(record, null, 2)}
|
|
@@ -1216,15 +1219,15 @@ function parseSeal(text, sealPath) {
|
|
|
1216
1219
|
details: unknown.map((key) => `unknown key "${key}"`)
|
|
1217
1220
|
});
|
|
1218
1221
|
}
|
|
1219
|
-
const
|
|
1220
|
-
if (typeof record.policy !== "string" || typeof
|
|
1222
|
+
const sha2562 = record.sha256;
|
|
1223
|
+
if (typeof record.policy !== "string" || typeof sha2562 !== "string" || !/^[0-9a-f]{64}$/.test(sha2562)) {
|
|
1221
1224
|
throw new StartupError("POLICY_INVALID", "the policy seal file is malformed", {
|
|
1222
1225
|
sourcePath: sealPath,
|
|
1223
1226
|
details: ['expected { "policy": string, "sha256": <64 hex chars>, "sealed_at": string }']
|
|
1224
1227
|
});
|
|
1225
1228
|
}
|
|
1226
1229
|
const sealedAt = typeof record.sealed_at === "string" ? record.sealed_at : "";
|
|
1227
|
-
return { policy: record.policy, sha256, sealed_at: sealedAt };
|
|
1230
|
+
return { policy: record.policy, sha256: sha2562, sealed_at: sealedAt };
|
|
1228
1231
|
}
|
|
1229
1232
|
function tryReadSeal(sealPath) {
|
|
1230
1233
|
let text;
|
|
@@ -1428,9 +1431,114 @@ function deleteProject(home, name2) {
|
|
|
1428
1431
|
return true;
|
|
1429
1432
|
}
|
|
1430
1433
|
|
|
1434
|
+
// packages/core/src/journal/checkpoint.ts
|
|
1435
|
+
import { createHash as createHash3, createHmac, randomBytes as randomBytes2, timingSafeEqual } from "node:crypto";
|
|
1436
|
+
import { existsSync as existsSync4, readFileSync as readFileSync4, writeFileSync as writeFileSync2 } from "node:fs";
|
|
1437
|
+
var CHECKPOINT_SCHEMA = 1;
|
|
1438
|
+
function sha256(text) {
|
|
1439
|
+
return createHash3("sha256").update(text, "utf8").digest("hex");
|
|
1440
|
+
}
|
|
1441
|
+
function merkleRoot(leaves) {
|
|
1442
|
+
if (leaves.length === 0) return sha256("");
|
|
1443
|
+
let level = leaves.map((leaf) => sha256(leaf));
|
|
1444
|
+
while (level.length > 1) {
|
|
1445
|
+
const next = [];
|
|
1446
|
+
for (let index = 0; index < level.length; index += 2) {
|
|
1447
|
+
const left = level[index];
|
|
1448
|
+
const right = level[index + 1];
|
|
1449
|
+
next.push(right === void 0 ? left : sha256(`${left}${right}`));
|
|
1450
|
+
}
|
|
1451
|
+
level = next;
|
|
1452
|
+
}
|
|
1453
|
+
return level[0];
|
|
1454
|
+
}
|
|
1455
|
+
function docsRoot(docs) {
|
|
1456
|
+
return merkleRoot(docs.map((doc) => `${doc.id}:${doc.sha256}:${doc.deleted ? "1" : "0"}`).sort());
|
|
1457
|
+
}
|
|
1458
|
+
function readKey(home) {
|
|
1459
|
+
const file = memoryPaths(home).checkpointKeyFile;
|
|
1460
|
+
if (!existsSync4(file)) return void 0;
|
|
1461
|
+
const key = readFileSync4(file);
|
|
1462
|
+
if (key.length !== 32) throw new Error("checkpoint key must be 32 bytes");
|
|
1463
|
+
return key;
|
|
1464
|
+
}
|
|
1465
|
+
function checkpointKey(home) {
|
|
1466
|
+
const existing = readKey(home);
|
|
1467
|
+
if (existing !== void 0) return existing;
|
|
1468
|
+
const file = memoryPaths(home).checkpointKeyFile;
|
|
1469
|
+
const key = randomBytes2(32);
|
|
1470
|
+
try {
|
|
1471
|
+
writeFileSync2(file, key, { flag: "wx", mode: 384 });
|
|
1472
|
+
return key;
|
|
1473
|
+
} catch {
|
|
1474
|
+
const raced = readKey(home);
|
|
1475
|
+
if (raced !== void 0) return raced;
|
|
1476
|
+
throw new Error("checkpoint key is unreadable");
|
|
1477
|
+
}
|
|
1478
|
+
}
|
|
1479
|
+
function canonical(checkpoint) {
|
|
1480
|
+
return JSON.stringify({
|
|
1481
|
+
schema: checkpoint.schema,
|
|
1482
|
+
created_at: checkpoint.created_at,
|
|
1483
|
+
journal: checkpoint.journal,
|
|
1484
|
+
docs: checkpoint.docs,
|
|
1485
|
+
journal_root: checkpoint.journal_root,
|
|
1486
|
+
docs_root: checkpoint.docs_root
|
|
1487
|
+
});
|
|
1488
|
+
}
|
|
1489
|
+
function writeCheckpoint(home, input) {
|
|
1490
|
+
const docs = input.docs.slice().sort((left, right) => left.id.localeCompare(right.id));
|
|
1491
|
+
const body = {
|
|
1492
|
+
schema: CHECKPOINT_SCHEMA,
|
|
1493
|
+
created_at: (input.now ?? /* @__PURE__ */ new Date()).toISOString(),
|
|
1494
|
+
journal: input.journal,
|
|
1495
|
+
docs,
|
|
1496
|
+
journal_root: input.journalRoot,
|
|
1497
|
+
docs_root: docsRoot(docs)
|
|
1498
|
+
};
|
|
1499
|
+
const hmac = createHmac("sha256", checkpointKey(home)).update(canonical(body), "utf8").digest("hex");
|
|
1500
|
+
const checkpoint = { ...body, hmac };
|
|
1501
|
+
writeFileAtomic(memoryPaths(home).checkpointFile, `${JSON.stringify(checkpoint, null, 2)}
|
|
1502
|
+
`);
|
|
1503
|
+
return checkpoint;
|
|
1504
|
+
}
|
|
1505
|
+
function readCheckpoint(home) {
|
|
1506
|
+
const file = memoryPaths(home).checkpointFile;
|
|
1507
|
+
if (!existsSync4(file)) return {};
|
|
1508
|
+
let parsed;
|
|
1509
|
+
try {
|
|
1510
|
+
parsed = JSON.parse(readFileSync4(file, "utf8"));
|
|
1511
|
+
} catch {
|
|
1512
|
+
return { error: "checkpoint is not valid JSON" };
|
|
1513
|
+
}
|
|
1514
|
+
if (parsed.schema !== CHECKPOINT_SCHEMA) return { error: `unsupported checkpoint schema ${String(parsed.schema)}` };
|
|
1515
|
+
if (typeof parsed.hmac !== "string" || parsed.hmac === "") return { error: "checkpoint has no HMAC" };
|
|
1516
|
+
if (docsRoot(parsed.docs) !== parsed.docs_root) return { error: "checkpoint docs root mismatch" };
|
|
1517
|
+
let key;
|
|
1518
|
+
try {
|
|
1519
|
+
const found = readKey(home);
|
|
1520
|
+
if (found === void 0) return { error: "checkpoint key is missing (restore .state/checkpoint.key or remove checkpoint)" };
|
|
1521
|
+
key = found;
|
|
1522
|
+
} catch (error) {
|
|
1523
|
+
return { error: error instanceof Error ? error.message : String(error) };
|
|
1524
|
+
}
|
|
1525
|
+
const expected = createHmac("sha256", key).update(canonical({
|
|
1526
|
+
schema: parsed.schema,
|
|
1527
|
+
created_at: parsed.created_at,
|
|
1528
|
+
journal: parsed.journal,
|
|
1529
|
+
docs: parsed.docs,
|
|
1530
|
+
journal_root: parsed.journal_root,
|
|
1531
|
+
docs_root: parsed.docs_root
|
|
1532
|
+
}), "utf8").digest("hex");
|
|
1533
|
+
const left = Buffer.from(expected, "hex");
|
|
1534
|
+
const right = Buffer.from(parsed.hmac, "hex");
|
|
1535
|
+
if (left.length !== right.length || !timingSafeEqual(left, right)) return { error: "checkpoint HMAC mismatch" };
|
|
1536
|
+
return { checkpoint: parsed };
|
|
1537
|
+
}
|
|
1538
|
+
|
|
1431
1539
|
// packages/core/src/fs/lock.ts
|
|
1432
|
-
import { randomBytes as
|
|
1433
|
-
import { closeSync as closeSync2, fsyncSync as fsyncSync2, openSync as openSync2, readFileSync as
|
|
1540
|
+
import { randomBytes as randomBytes3 } from "node:crypto";
|
|
1541
|
+
import { closeSync as closeSync2, fsyncSync as fsyncSync2, openSync as openSync2, readFileSync as readFileSync5, statSync as statSync2, unlinkSync, writeFileSync as writeFileSync3, writeSync } from "node:fs";
|
|
1434
1542
|
var LockError = class extends Error {
|
|
1435
1543
|
code;
|
|
1436
1544
|
constructor(code, message) {
|
|
@@ -1441,7 +1549,7 @@ var LockError = class extends Error {
|
|
|
1441
1549
|
};
|
|
1442
1550
|
function readLock(file) {
|
|
1443
1551
|
try {
|
|
1444
|
-
const raw = JSON.parse(
|
|
1552
|
+
const raw = JSON.parse(readFileSync5(file, "utf8"));
|
|
1445
1553
|
if (typeof raw.pid === "number" && typeof raw.token === "string" && typeof raw.acquired_at === "number" && typeof raw.heartbeat_at === "number") {
|
|
1446
1554
|
return { pid: raw.pid, token: raw.token, acquired_at: raw.acquired_at, heartbeat_at: raw.heartbeat_at };
|
|
1447
1555
|
}
|
|
@@ -1518,7 +1626,7 @@ var FileLock = class {
|
|
|
1518
1626
|
function acquireLock(options) {
|
|
1519
1627
|
const now = options.now ?? Date.now;
|
|
1520
1628
|
const deadline = now() + options.config.timeoutMs;
|
|
1521
|
-
const token =
|
|
1629
|
+
const token = randomBytes3(8).toString("hex");
|
|
1522
1630
|
for (; ; ) {
|
|
1523
1631
|
const info = {
|
|
1524
1632
|
pid: process.pid,
|
|
@@ -1558,7 +1666,7 @@ function acquireLock(options) {
|
|
|
1558
1666
|
}
|
|
1559
1667
|
|
|
1560
1668
|
// packages/core/src/model/ids.ts
|
|
1561
|
-
import { randomBytes as
|
|
1669
|
+
import { randomBytes as randomBytes4 } from "node:crypto";
|
|
1562
1670
|
var ID_PATTERN = /^mem-\d{8}-[0-9a-f]{8}$/;
|
|
1563
1671
|
function isMemoryId(value) {
|
|
1564
1672
|
return ID_PATTERN.test(value);
|
|
@@ -1569,7 +1677,7 @@ function formatDate(date) {
|
|
|
1569
1677
|
const day = String(date.getDate()).padStart(2, "0");
|
|
1570
1678
|
return `${year}-${month}-${day}`;
|
|
1571
1679
|
}
|
|
1572
|
-
function newMemoryId(now = /* @__PURE__ */ new Date(), bytes = () =>
|
|
1680
|
+
function newMemoryId(now = /* @__PURE__ */ new Date(), bytes = () => randomBytes4(4)) {
|
|
1573
1681
|
return `mem-${formatDate(now).replaceAll("-", "")}-${bytes().toString("hex")}`;
|
|
1574
1682
|
}
|
|
1575
1683
|
|
|
@@ -1661,13 +1769,26 @@ function supersedeProblems(candidate, docs) {
|
|
|
1661
1769
|
}
|
|
1662
1770
|
|
|
1663
1771
|
// packages/core/src/journal/journal.ts
|
|
1664
|
-
import { createHash as
|
|
1665
|
-
import { closeSync as closeSync3, fsyncSync as fsyncSync3, mkdirSync as mkdirSync2, openSync as openSync3, readFileSync as
|
|
1772
|
+
import { createHash as createHash4 } from "node:crypto";
|
|
1773
|
+
import { closeSync as closeSync3, existsSync as existsSync5, fsyncSync as fsyncSync3, mkdirSync as mkdirSync2, openSync as openSync3, readFileSync as readFileSync6, readdirSync, statSync as statSync3, writeSync as writeSync2 } from "node:fs";
|
|
1666
1774
|
import { join as join3 } from "node:path";
|
|
1667
1775
|
var SEGMENT_BYTES = 5 * 1024 * 1024;
|
|
1668
1776
|
var ZERO_HASH = "0".repeat(64);
|
|
1777
|
+
function readArchiveIndex(dir) {
|
|
1778
|
+
const file = join3(dir, "archive", "index.json");
|
|
1779
|
+
if (!existsSync5(file)) return void 0;
|
|
1780
|
+
try {
|
|
1781
|
+
const parsed = JSON.parse(readFileSync6(file, "utf8"));
|
|
1782
|
+
if (parsed.schema !== 1 || typeof parsed.through_seq !== "number" || typeof parsed.through_hash !== "string") {
|
|
1783
|
+
return void 0;
|
|
1784
|
+
}
|
|
1785
|
+
return parsed;
|
|
1786
|
+
} catch {
|
|
1787
|
+
return void 0;
|
|
1788
|
+
}
|
|
1789
|
+
}
|
|
1669
1790
|
function hashLine(line) {
|
|
1670
|
-
return
|
|
1791
|
+
return createHash4("sha256").update(line, "utf8").digest("hex");
|
|
1671
1792
|
}
|
|
1672
1793
|
var Journal = class {
|
|
1673
1794
|
dir;
|
|
@@ -1686,7 +1807,7 @@ var Journal = class {
|
|
|
1686
1807
|
const segments = this.segments();
|
|
1687
1808
|
for (let index = segments.length - 1; index >= 0; index -= 1) {
|
|
1688
1809
|
const file = join3(this.dir, segments[index]);
|
|
1689
|
-
const lines =
|
|
1810
|
+
const lines = readFileSync6(file, "utf8").split("\n").filter((line) => line !== "");
|
|
1690
1811
|
if (lines.length === 0) continue;
|
|
1691
1812
|
const last = lines[lines.length - 1];
|
|
1692
1813
|
try {
|
|
@@ -1696,20 +1817,22 @@ var Journal = class {
|
|
|
1696
1817
|
return null;
|
|
1697
1818
|
}
|
|
1698
1819
|
}
|
|
1699
|
-
|
|
1820
|
+
const archive = readArchiveIndex(this.dir);
|
|
1821
|
+
return archive === void 0 ? null : { seq: archive.through_seq, hash: archive.through_hash };
|
|
1700
1822
|
}
|
|
1701
1823
|
readAll() {
|
|
1702
1824
|
const entries = [];
|
|
1703
1825
|
const hashes = [];
|
|
1704
1826
|
const issues = [];
|
|
1705
1827
|
const opIds = /* @__PURE__ */ new Set();
|
|
1706
|
-
|
|
1707
|
-
let
|
|
1828
|
+
const archive = readArchiveIndex(this.dir);
|
|
1829
|
+
let expectedSeq = archive === void 0 ? 1 : archive.through_seq + 1;
|
|
1830
|
+
let prev = archive === void 0 ? ZERO_HASH : archive.through_hash;
|
|
1708
1831
|
for (const name2 of this.segments()) {
|
|
1709
1832
|
const file = join3(this.dir, name2);
|
|
1710
1833
|
let text;
|
|
1711
1834
|
try {
|
|
1712
|
-
text =
|
|
1835
|
+
text = readFileSync6(file, "utf8");
|
|
1713
1836
|
} catch (error) {
|
|
1714
1837
|
issues.push({ kind: "parse", message: `cannot read ${name2}: ${String(error)}` });
|
|
1715
1838
|
continue;
|
|
@@ -1726,6 +1849,7 @@ var Journal = class {
|
|
|
1726
1849
|
return { entries, hashes, issues, head: entries.length === 0 ? null : { seq: entries.length, hash: prev } };
|
|
1727
1850
|
}
|
|
1728
1851
|
const hash = hashLine(line);
|
|
1852
|
+
if (archive !== void 0 && entry.seq < expectedSeq) continue;
|
|
1729
1853
|
if (entry.seq !== expectedSeq) {
|
|
1730
1854
|
issues.push({
|
|
1731
1855
|
kind: "seq",
|
|
@@ -1750,7 +1874,92 @@ var Journal = class {
|
|
|
1750
1874
|
prev = hash;
|
|
1751
1875
|
}
|
|
1752
1876
|
}
|
|
1753
|
-
const head = entries.length === 0 ? null : { seq: entries[entries.length - 1]?.seq ?? 0, hash: prev };
|
|
1877
|
+
const head = entries.length === 0 ? archive === void 0 ? null : { seq: archive.through_seq, hash: archive.through_hash } : { seq: entries[entries.length - 1]?.seq ?? 0, hash: prev };
|
|
1878
|
+
return { entries, hashes, issues, head };
|
|
1879
|
+
}
|
|
1880
|
+
/** Cheap operational summary for `mem doctor` / metrics. */
|
|
1881
|
+
stats() {
|
|
1882
|
+
const segments = this.segments();
|
|
1883
|
+
let lines = 0;
|
|
1884
|
+
let bytes = 0;
|
|
1885
|
+
let largest = 0;
|
|
1886
|
+
let firstSeq = null;
|
|
1887
|
+
let lastSeq = null;
|
|
1888
|
+
for (const name2 of segments) {
|
|
1889
|
+
const text = readFileSync6(join3(this.dir, name2), "utf8");
|
|
1890
|
+
const size = Buffer.byteLength(text);
|
|
1891
|
+
bytes += size;
|
|
1892
|
+
largest = Math.max(largest, size);
|
|
1893
|
+
const rows = text.split("\n").filter((line) => line !== "");
|
|
1894
|
+
lines += rows.length;
|
|
1895
|
+
if (firstSeq === null && rows.length > 0) {
|
|
1896
|
+
try {
|
|
1897
|
+
firstSeq = JSON.parse(rows[0]).seq ?? null;
|
|
1898
|
+
} catch {
|
|
1899
|
+
firstSeq = null;
|
|
1900
|
+
}
|
|
1901
|
+
}
|
|
1902
|
+
if (rows.length > 0) {
|
|
1903
|
+
try {
|
|
1904
|
+
lastSeq = JSON.parse(rows[rows.length - 1]).seq ?? lastSeq;
|
|
1905
|
+
} catch {
|
|
1906
|
+
}
|
|
1907
|
+
}
|
|
1908
|
+
}
|
|
1909
|
+
return { segments: segments.length, lines, bytes, first_seq: firstSeq, last_seq: lastSeq, largest_segment_bytes: largest };
|
|
1910
|
+
}
|
|
1911
|
+
/**
|
|
1912
|
+
* Replay only entries after a checkpoint. Old lines are skipped without JSON
|
|
1913
|
+
* parsing; the checkpoint's last line hash is the expected `prev`.
|
|
1914
|
+
*/
|
|
1915
|
+
readAfter(afterSeq, prevHash, fromSegment) {
|
|
1916
|
+
if (afterSeq === 0) return this.readAll();
|
|
1917
|
+
const allSegments = this.segments();
|
|
1918
|
+
const start = fromSegment === void 0 ? 0 : allSegments.indexOf(fromSegment);
|
|
1919
|
+
const segments = start < 0 ? allSegments : allSegments.slice(start);
|
|
1920
|
+
const entries = [];
|
|
1921
|
+
const hashes = [];
|
|
1922
|
+
const issues = [];
|
|
1923
|
+
let expectedSeq = afterSeq + 1;
|
|
1924
|
+
let prev = prevHash;
|
|
1925
|
+
let seenTail = false;
|
|
1926
|
+
for (const name2 of segments) {
|
|
1927
|
+
const text = readFileSync6(join3(this.dir, name2), "utf8");
|
|
1928
|
+
for (const line of text.split("\n")) {
|
|
1929
|
+
if (line === "") continue;
|
|
1930
|
+
const seqMatch = /^\{"seq":(\d+)/.exec(line);
|
|
1931
|
+
if (seqMatch !== null && Number(seqMatch[1]) <= afterSeq) continue;
|
|
1932
|
+
let entry;
|
|
1933
|
+
try {
|
|
1934
|
+
entry = JSON.parse(line);
|
|
1935
|
+
} catch {
|
|
1936
|
+
issues.push({ kind: "parse", message: `${name2}: line is not valid JSON`, seq: expectedSeq });
|
|
1937
|
+
return { entries, hashes, issues, head: { seq: afterSeq, hash: prevHash } };
|
|
1938
|
+
}
|
|
1939
|
+
if (entry.seq !== expectedSeq) {
|
|
1940
|
+
issues.push({
|
|
1941
|
+
kind: "seq",
|
|
1942
|
+
message: `${name2}: expected seq ${expectedSeq}, found ${entry.seq}`,
|
|
1943
|
+
seq: entry.seq
|
|
1944
|
+
});
|
|
1945
|
+
}
|
|
1946
|
+
if (entry.prev !== prev) {
|
|
1947
|
+
issues.push({
|
|
1948
|
+
kind: "chain",
|
|
1949
|
+
message: `${name2}: seq ${entry.seq} prev does not match the checkpoint/previous line`,
|
|
1950
|
+
seq: entry.seq
|
|
1951
|
+
});
|
|
1952
|
+
}
|
|
1953
|
+
const hash = hashLine(line);
|
|
1954
|
+
if (entry.seq < expectedSeq) continue;
|
|
1955
|
+
entries.push(entry);
|
|
1956
|
+
hashes.push(hash);
|
|
1957
|
+
expectedSeq = entry.seq + 1;
|
|
1958
|
+
prev = hash;
|
|
1959
|
+
seenTail = true;
|
|
1960
|
+
}
|
|
1961
|
+
}
|
|
1962
|
+
const head = seenTail ? { seq: expectedSeq - 1, hash: prev } : { seq: afterSeq, hash: prevHash };
|
|
1754
1963
|
return { entries, hashes, issues, head };
|
|
1755
1964
|
}
|
|
1756
1965
|
findByOpId(opId) {
|
|
@@ -1802,10 +2011,64 @@ var Journal = class {
|
|
|
1802
2011
|
}
|
|
1803
2012
|
return entry;
|
|
1804
2013
|
}
|
|
2014
|
+
/**
|
|
2015
|
+
* Append a batch with one fsync. Bulk import writes files first, then one
|
|
2016
|
+
* journal batch makes the whole import durable. A torn final line is ignored
|
|
2017
|
+
* by recovery just like a single torn append.
|
|
2018
|
+
*/
|
|
2019
|
+
appendBatch(inputs) {
|
|
2020
|
+
if (inputs.length === 0) return [];
|
|
2021
|
+
mkdirSync2(this.dir, { recursive: true });
|
|
2022
|
+
const segments = this.segments();
|
|
2023
|
+
let segmentPath = join3(this.dir, segments[segments.length - 1] ?? "0001.jsonl");
|
|
2024
|
+
let size = 0;
|
|
2025
|
+
try {
|
|
2026
|
+
size = statSync3(segmentPath).size;
|
|
2027
|
+
} catch {
|
|
2028
|
+
size = 0;
|
|
2029
|
+
}
|
|
2030
|
+
let head = this.head();
|
|
2031
|
+
const entries = [];
|
|
2032
|
+
let fd = openSync3(segmentPath, "a");
|
|
2033
|
+
try {
|
|
2034
|
+
for (const input of inputs) {
|
|
2035
|
+
const seq = head === null ? 1 : head.seq + 1;
|
|
2036
|
+
const prev = head === null ? ZERO_HASH : head.hash;
|
|
2037
|
+
const entry = {
|
|
2038
|
+
seq,
|
|
2039
|
+
prev,
|
|
2040
|
+
at: input.at ?? (/* @__PURE__ */ new Date()).toISOString(),
|
|
2041
|
+
op: input.op,
|
|
2042
|
+
op_id: input.opId ?? `op-${seq}-${hashLine(`${seq}:${input.op}:${JSON.stringify(input.data)}`).slice(0, 8)}`,
|
|
2043
|
+
data: input.data
|
|
2044
|
+
};
|
|
2045
|
+
const line = JSON.stringify(entry);
|
|
2046
|
+
if (size > 0 && size + Buffer.byteLength(line) + 1 > SEGMENT_BYTES) {
|
|
2047
|
+
closeSync3(fd);
|
|
2048
|
+
const baseName = segmentPath.split(/[\\/]/).pop();
|
|
2049
|
+
const nextNumber = Number(baseName.slice(0, 4)) + 1;
|
|
2050
|
+
segmentPath = join3(this.dir, `${String(nextNumber).padStart(4, "0")}.jsonl`);
|
|
2051
|
+
fd = openSync3(segmentPath, "a");
|
|
2052
|
+
size = 0;
|
|
2053
|
+
}
|
|
2054
|
+
writeSync2(fd, `${line}
|
|
2055
|
+
`);
|
|
2056
|
+
size += Buffer.byteLength(line) + 1;
|
|
2057
|
+
head = { seq, hash: hashLine(line) };
|
|
2058
|
+
entries.push(entry);
|
|
2059
|
+
}
|
|
2060
|
+
fsyncSync3(fd);
|
|
2061
|
+
} finally {
|
|
2062
|
+
closeSync3(fd);
|
|
2063
|
+
}
|
|
2064
|
+
return entries;
|
|
2065
|
+
}
|
|
1805
2066
|
};
|
|
1806
2067
|
|
|
1807
2068
|
// packages/core/src/store/store.ts
|
|
1808
|
-
import { existsSync as
|
|
2069
|
+
import { existsSync as existsSync6, readdirSync as readdirSync2, readFileSync as readFileSync8, rmSync as rmSync3, statSync as statSync4, writeFileSync as writeFileSync5 } from "node:fs";
|
|
2070
|
+
import { gzipSync } from "node:zlib";
|
|
2071
|
+
import { createHash as createHash5 } from "node:crypto";
|
|
1809
2072
|
import { join as join5 } from "node:path";
|
|
1810
2073
|
|
|
1811
2074
|
// packages/core/src/index/tokenizer.ts
|
|
@@ -2011,7 +2274,7 @@ function fuseScores(lexical, vector, embeddingWeightPermille) {
|
|
|
2011
2274
|
}
|
|
2012
2275
|
|
|
2013
2276
|
// packages/core/src/embed/cache.ts
|
|
2014
|
-
import { mkdtempSync, readFileSync as
|
|
2277
|
+
import { mkdtempSync, readFileSync as readFileSync7, rmSync as rmSync2, writeFileSync as writeFileSync4 } from "node:fs";
|
|
2015
2278
|
import { tmpdir } from "node:os";
|
|
2016
2279
|
import { join as join4 } from "node:path";
|
|
2017
2280
|
import { spawn } from "node:child_process";
|
|
@@ -2021,7 +2284,7 @@ function embeddingsFile(home) {
|
|
|
2021
2284
|
}
|
|
2022
2285
|
function readEmbeddingCache(home) {
|
|
2023
2286
|
try {
|
|
2024
|
-
const parsed = JSON.parse(
|
|
2287
|
+
const parsed = JSON.parse(readFileSync7(embeddingsFile(home), "utf8"));
|
|
2025
2288
|
if (parsed.schema !== CACHE_SCHEMA) return void 0;
|
|
2026
2289
|
if (typeof parsed.provider !== "string" || typeof parsed.dimensions !== "number") return void 0;
|
|
2027
2290
|
if (typeof parsed.generated_at !== "string" || typeof parsed.docs !== "object" || parsed.docs === null) {
|
|
@@ -2041,7 +2304,7 @@ async function embedTexts(command, texts) {
|
|
|
2041
2304
|
const inputPath = join4(dir, "input.json");
|
|
2042
2305
|
const outputPath = join4(dir, "output.json");
|
|
2043
2306
|
try {
|
|
2044
|
-
|
|
2307
|
+
writeFileSync4(inputPath, JSON.stringify({ texts }));
|
|
2045
2308
|
await new Promise((resolve3, reject) => {
|
|
2046
2309
|
const child = spawn(command, [inputPath, outputPath], {
|
|
2047
2310
|
stdio: "ignore",
|
|
@@ -2054,7 +2317,7 @@ async function embedTexts(command, texts) {
|
|
|
2054
2317
|
else reject(new Error(`vectorizer exited with ${code ?? "no code"}`));
|
|
2055
2318
|
});
|
|
2056
2319
|
});
|
|
2057
|
-
const parsed = JSON.parse(
|
|
2320
|
+
const parsed = JSON.parse(readFileSync7(outputPath, "utf8"));
|
|
2058
2321
|
if (!Array.isArray(parsed.vectors) || parsed.vectors.length !== texts.length) {
|
|
2059
2322
|
throw new Error("vectorizer output must contain one vector per text");
|
|
2060
2323
|
}
|
|
@@ -2151,6 +2414,15 @@ var MemoryStore = class {
|
|
|
2151
2414
|
accounted = /* @__PURE__ */ new Map();
|
|
2152
2415
|
issues = [];
|
|
2153
2416
|
head = null;
|
|
2417
|
+
lastReloadMs = 0;
|
|
2418
|
+
lastCheckpointSeq = 0;
|
|
2419
|
+
checkpointState = null;
|
|
2420
|
+
cachedIndex = null;
|
|
2421
|
+
revision = 0;
|
|
2422
|
+
fileMeta = /* @__PURE__ */ new Map();
|
|
2423
|
+
dirtyDerived = 0;
|
|
2424
|
+
pendingAdds = [];
|
|
2425
|
+
addTimer;
|
|
2154
2426
|
constructor(options) {
|
|
2155
2427
|
this.paths = memoryPaths(options.home);
|
|
2156
2428
|
this.config = options.config;
|
|
@@ -2158,17 +2430,68 @@ var MemoryStore = class {
|
|
|
2158
2430
|
this.beforeStep = options.beforeStep;
|
|
2159
2431
|
this.journal = new Journal(this.paths.journalDir);
|
|
2160
2432
|
this.reload();
|
|
2433
|
+
if (this.journal.stats().lines > 5e4) {
|
|
2434
|
+
try {
|
|
2435
|
+
this.maintain({ force: true });
|
|
2436
|
+
} catch (error) {
|
|
2437
|
+
this.issues.push(`automatic journal maintenance failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
2438
|
+
}
|
|
2439
|
+
}
|
|
2161
2440
|
}
|
|
2162
|
-
/** Re-read the journal and the memory files; classify each file. */
|
|
2441
|
+
/** Re-read the journal checkpoint and the memory files; classify each file. */
|
|
2163
2442
|
reload() {
|
|
2164
|
-
const
|
|
2165
|
-
|
|
2166
|
-
|
|
2443
|
+
const started = performance.now();
|
|
2444
|
+
const previousDocs = this.docs;
|
|
2445
|
+
const previousMeta = this.fileMeta;
|
|
2446
|
+
this.issues = [];
|
|
2167
2447
|
this.docs = /* @__PURE__ */ new Map();
|
|
2168
2448
|
this.quarantined = /* @__PURE__ */ new Map();
|
|
2169
2449
|
this.deleted = /* @__PURE__ */ new Set();
|
|
2170
2450
|
this.accounted = /* @__PURE__ */ new Map();
|
|
2451
|
+
this.revision += 1;
|
|
2452
|
+
this.cachedIndex = null;
|
|
2453
|
+
this.fileMeta = /* @__PURE__ */ new Map();
|
|
2171
2454
|
const latest = /* @__PURE__ */ new Map();
|
|
2455
|
+
const checkpointRead = readCheckpoint(this.paths.home);
|
|
2456
|
+
let read;
|
|
2457
|
+
if (checkpointRead.checkpoint !== void 0) {
|
|
2458
|
+
const checkpoint = checkpointRead.checkpoint;
|
|
2459
|
+
this.lastCheckpointSeq = checkpoint.journal.seq;
|
|
2460
|
+
this.checkpointState = { seq: checkpoint.journal.seq, created_at: checkpoint.created_at };
|
|
2461
|
+
for (const doc of checkpoint.docs) {
|
|
2462
|
+
latest.set(doc.id, { hash: doc.sha256, deleted: doc.deleted });
|
|
2463
|
+
if (doc.deleted) this.deleted.add(doc.id);
|
|
2464
|
+
if (doc.sha256 !== "") this.accounted.set(doc.id, doc.sha256);
|
|
2465
|
+
}
|
|
2466
|
+
const tail = this.journal.readAfter(
|
|
2467
|
+
checkpoint.journal.seq,
|
|
2468
|
+
checkpoint.journal.hash,
|
|
2469
|
+
checkpoint.journal.segment
|
|
2470
|
+
);
|
|
2471
|
+
read = { entries: tail.entries, issues: tail.issues, head: tail.head ?? checkpoint.journal };
|
|
2472
|
+
const journalHead = this.journal.head();
|
|
2473
|
+
if (journalHead !== null && journalHead.seq < checkpoint.journal.seq) {
|
|
2474
|
+
read = {
|
|
2475
|
+
...read,
|
|
2476
|
+
issues: [...read.issues, {
|
|
2477
|
+
kind: "chain",
|
|
2478
|
+
message: `journal is behind checkpoint (#${checkpoint.journal.seq}): possible truncation`
|
|
2479
|
+
}]
|
|
2480
|
+
};
|
|
2481
|
+
}
|
|
2482
|
+
} else {
|
|
2483
|
+
this.lastCheckpointSeq = 0;
|
|
2484
|
+
this.checkpointState = null;
|
|
2485
|
+
read = this.journal.readAll();
|
|
2486
|
+
if (checkpointRead.error !== void 0) {
|
|
2487
|
+
read = {
|
|
2488
|
+
...read,
|
|
2489
|
+
issues: [...read.issues, { kind: "parse", message: `checkpoint: ${checkpointRead.error}` }]
|
|
2490
|
+
};
|
|
2491
|
+
}
|
|
2492
|
+
}
|
|
2493
|
+
this.issues = read.issues.map((issue) => issue.message);
|
|
2494
|
+
this.head = read.head;
|
|
2172
2495
|
for (const entry of read.entries) {
|
|
2173
2496
|
const id = typeof entry.data.id === "string" ? entry.data.id : void 0;
|
|
2174
2497
|
if (entry.op === "record" || entry.op === "adopt") {
|
|
@@ -2182,6 +2505,7 @@ var MemoryStore = class {
|
|
|
2182
2505
|
}
|
|
2183
2506
|
for (const [id, value] of latest) {
|
|
2184
2507
|
if (value.hash !== "") this.accounted.set(id, value.hash);
|
|
2508
|
+
if (value.deleted) this.deleted.add(id);
|
|
2185
2509
|
}
|
|
2186
2510
|
for (const kind of KINDS2) {
|
|
2187
2511
|
const dir = join5(this.paths.memoriesDir, kind);
|
|
@@ -2195,12 +2519,20 @@ var MemoryStore = class {
|
|
|
2195
2519
|
const file = join5(dir, name2);
|
|
2196
2520
|
const id = name2.slice(0, -3);
|
|
2197
2521
|
try {
|
|
2198
|
-
const
|
|
2199
|
-
const
|
|
2200
|
-
|
|
2201
|
-
|
|
2202
|
-
|
|
2203
|
-
|
|
2522
|
+
const stat = statSync4(file);
|
|
2523
|
+
const cachedDoc = previousDocs.get(id);
|
|
2524
|
+
const cachedMeta = previousMeta.get(id);
|
|
2525
|
+
let doc;
|
|
2526
|
+
if (cachedDoc !== void 0 && cachedMeta !== void 0 && cachedMeta.mtimeMs === stat.mtimeMs && cachedMeta.size === stat.size) {
|
|
2527
|
+
doc = cachedDoc;
|
|
2528
|
+
} else {
|
|
2529
|
+
const text = readFileSync8(file, "utf8");
|
|
2530
|
+
doc = parseMemoryFile(text, file, {
|
|
2531
|
+
abstractChars: this.config.limits.abstract_chars,
|
|
2532
|
+
bodyChars: this.config.limits.body_chars,
|
|
2533
|
+
maxRecordChars: this.config.storage.max_record_chars
|
|
2534
|
+
});
|
|
2535
|
+
}
|
|
2204
2536
|
if (!isMemoryId(id) || doc.id !== id) {
|
|
2205
2537
|
this.quarantined.set(doc.id || id, "id does not match filename");
|
|
2206
2538
|
continue;
|
|
@@ -2219,6 +2551,7 @@ var MemoryStore = class {
|
|
|
2219
2551
|
continue;
|
|
2220
2552
|
}
|
|
2221
2553
|
this.docs.set(doc.id, doc);
|
|
2554
|
+
this.fileMeta.set(doc.id, { mtimeMs: stat.mtimeMs, size: stat.size });
|
|
2222
2555
|
} catch (error) {
|
|
2223
2556
|
this.quarantined.set(id, `parse: ${error instanceof Error ? error.message : String(error)}`);
|
|
2224
2557
|
}
|
|
@@ -2231,7 +2564,7 @@ var MemoryStore = class {
|
|
|
2231
2564
|
}
|
|
2232
2565
|
}
|
|
2233
2566
|
try {
|
|
2234
|
-
const manifest = JSON.parse(
|
|
2567
|
+
const manifest = JSON.parse(readFileSync8(this.paths.manifestFile, "utf8"));
|
|
2235
2568
|
const expected = manifest.journal_head;
|
|
2236
2569
|
const expectedSeq = expected?.seq;
|
|
2237
2570
|
const expectedHash = expected?.hash;
|
|
@@ -2251,6 +2584,7 @@ var MemoryStore = class {
|
|
|
2251
2584
|
}
|
|
2252
2585
|
} catch {
|
|
2253
2586
|
}
|
|
2587
|
+
this.lastReloadMs = performance.now() - started;
|
|
2254
2588
|
}
|
|
2255
2589
|
active(date = today(this.now())) {
|
|
2256
2590
|
const live = activeDocs(this.docs.values(), date);
|
|
@@ -2271,7 +2605,7 @@ var MemoryStore = class {
|
|
|
2271
2605
|
recall(query, options = {}) {
|
|
2272
2606
|
const text = query.trim();
|
|
2273
2607
|
if (text === "") return { hits: [], warnings: ["empty query"] };
|
|
2274
|
-
this.
|
|
2608
|
+
this.reloadIfJournalChanged();
|
|
2275
2609
|
const date = options.asOf ?? today(this.now());
|
|
2276
2610
|
const limit = Math.min(
|
|
2277
2611
|
options.limit ?? this.config.recall.limit,
|
|
@@ -2280,7 +2614,7 @@ var MemoryStore = class {
|
|
|
2280
2614
|
const activeSet = activeDocs(this.docs.values(), date).filter(
|
|
2281
2615
|
(doc) => !this.deleted.has(doc.id) && !this.quarantined.has(doc.id)
|
|
2282
2616
|
);
|
|
2283
|
-
const index =
|
|
2617
|
+
const index = this.indexFor(activeSet, date);
|
|
2284
2618
|
const byId = new Map(activeSet.map((doc) => [doc.id, doc]));
|
|
2285
2619
|
const minScore = this.config.recall.min_score / 1e3;
|
|
2286
2620
|
let ranked;
|
|
@@ -2360,13 +2694,13 @@ ${doc.body}`), queryVector));
|
|
|
2360
2694
|
if (queryVector === void 0 || queryVector.length !== cache.dimensions) {
|
|
2361
2695
|
throw new Error(`query vector does not match cache dimensions (${cache.dimensions})`);
|
|
2362
2696
|
}
|
|
2363
|
-
this.
|
|
2697
|
+
this.reloadIfJournalChanged();
|
|
2364
2698
|
const date = options.asOf ?? today(this.now());
|
|
2365
2699
|
const limit = Math.min(options.limit ?? this.config.recall.limit, this.config.recall.max_limit);
|
|
2366
2700
|
const activeSet = activeDocs(this.docs.values(), date).filter(
|
|
2367
2701
|
(doc) => !this.deleted.has(doc.id) && !this.quarantined.has(doc.id)
|
|
2368
2702
|
);
|
|
2369
|
-
const index =
|
|
2703
|
+
const index = this.indexFor(activeSet, date);
|
|
2370
2704
|
const lexicalHits = search(index, text, {
|
|
2371
2705
|
limit: activeSet.length,
|
|
2372
2706
|
minScorePermille: 0,
|
|
@@ -2543,6 +2877,123 @@ ${doc.body}`), queryVector));
|
|
|
2543
2877
|
lock.release();
|
|
2544
2878
|
}
|
|
2545
2879
|
}
|
|
2880
|
+
/** Force checkpoint + archive old segments; returns what moved. */
|
|
2881
|
+
maintain(options = {}) {
|
|
2882
|
+
const lock = acquireLock({
|
|
2883
|
+
file: this.paths.lockFile,
|
|
2884
|
+
config: {
|
|
2885
|
+
heartbeatMs: this.config.lock.heartbeat_ms,
|
|
2886
|
+
staleMs: this.config.lock.stale_ms,
|
|
2887
|
+
timeoutMs: this.config.lock.timeout_ms
|
|
2888
|
+
},
|
|
2889
|
+
now: () => this.now().getTime()
|
|
2890
|
+
});
|
|
2891
|
+
try {
|
|
2892
|
+
this.reload();
|
|
2893
|
+
if (this.issues.length > 0) {
|
|
2894
|
+
throw new MemoryStoreError("CONFLICT", `journal is inconsistent, repair before maintain: ${this.issues[0]}`);
|
|
2895
|
+
}
|
|
2896
|
+
const checkpointed = this.checkpoint();
|
|
2897
|
+
const archived = this.archiveOldSegments(options.force ?? false);
|
|
2898
|
+
const pruned = this.pruneArchives(options.retentionDays ?? 365);
|
|
2899
|
+
this.reload();
|
|
2900
|
+
return { archived, pruned, checkpointed };
|
|
2901
|
+
} finally {
|
|
2902
|
+
lock.release();
|
|
2903
|
+
}
|
|
2904
|
+
}
|
|
2905
|
+
archiveOldSegments(force) {
|
|
2906
|
+
if (this.head === null) return [];
|
|
2907
|
+
const read = readCheckpoint(this.paths.home);
|
|
2908
|
+
if (read.checkpoint === void 0) return [];
|
|
2909
|
+
const checkpointSegment = read.checkpoint.journal.segment;
|
|
2910
|
+
if (checkpointSegment === void 0) return [];
|
|
2911
|
+
const segments = this.journal.segments();
|
|
2912
|
+
const at = segments.indexOf(checkpointSegment);
|
|
2913
|
+
if (at <= 0) return [];
|
|
2914
|
+
if (!force && this.journal.stats().lines <= 5e4) return [];
|
|
2915
|
+
const archiveDir = this.paths.journalArchiveDir;
|
|
2916
|
+
ensureDir(archiveDir);
|
|
2917
|
+
const archived = [];
|
|
2918
|
+
const toDelete = [];
|
|
2919
|
+
let throughSeq = 0;
|
|
2920
|
+
let throughHash = "";
|
|
2921
|
+
for (const name2 of segments.slice(0, at)) {
|
|
2922
|
+
const file = join5(this.paths.journalDir, name2);
|
|
2923
|
+
const text = readFileSync8(file, "utf8");
|
|
2924
|
+
const lines = text.split("\n").filter((line) => line !== "");
|
|
2925
|
+
if (lines.length === 0) continue;
|
|
2926
|
+
const hashes = lines.map((line) => hashLine(line));
|
|
2927
|
+
const first = JSON.parse(lines[0]);
|
|
2928
|
+
const last = JSON.parse(lines[lines.length - 1]);
|
|
2929
|
+
const lastHash = hashLine(lines[lines.length - 1]);
|
|
2930
|
+
const meta = {
|
|
2931
|
+
schema: 1,
|
|
2932
|
+
segment: name2,
|
|
2933
|
+
first_seq: first.seq,
|
|
2934
|
+
last_seq: last.seq,
|
|
2935
|
+
count: lines.length,
|
|
2936
|
+
bytes: Buffer.byteLength(text),
|
|
2937
|
+
sha256: createHash5("sha256").update(text, "utf8").digest("hex"),
|
|
2938
|
+
root: merkleRoot(hashes),
|
|
2939
|
+
archived_at: this.now().toISOString()
|
|
2940
|
+
};
|
|
2941
|
+
const base = name2.replace(/\.jsonl$/, "");
|
|
2942
|
+
writeFileSync5(join5(archiveDir, `${base}.jsonl.gz`), gzipSync(Buffer.from(text)));
|
|
2943
|
+
writeFileAtomic(join5(archiveDir, `${base}.meta.json`), `${JSON.stringify(meta, null, 2)}
|
|
2944
|
+
`);
|
|
2945
|
+
throughSeq = last.seq;
|
|
2946
|
+
throughHash = lastHash;
|
|
2947
|
+
toDelete.push(file);
|
|
2948
|
+
archived.push(name2);
|
|
2949
|
+
}
|
|
2950
|
+
if (archived.length > 0) {
|
|
2951
|
+
writeFileAtomic(
|
|
2952
|
+
join5(archiveDir, "index.json"),
|
|
2953
|
+
`${JSON.stringify({ schema: 1, through_seq: throughSeq, through_hash: throughHash, segments: archived }, null, 2)}
|
|
2954
|
+
`
|
|
2955
|
+
);
|
|
2956
|
+
for (const file of toDelete) rmSync3(file, { force: true });
|
|
2957
|
+
this.journal.append({ op: "audit", data: { event: "journal-archive", segments: archived, through_seq: throughSeq } });
|
|
2958
|
+
}
|
|
2959
|
+
return archived;
|
|
2960
|
+
}
|
|
2961
|
+
pruneArchives(retentionDays) {
|
|
2962
|
+
if (retentionDays <= 0) return [];
|
|
2963
|
+
const dir = this.paths.journalArchiveDir;
|
|
2964
|
+
if (!existsSync6(dir)) return [];
|
|
2965
|
+
const cutoff = Date.now() - retentionDays * 24 * 60 * 60 * 1e3;
|
|
2966
|
+
const pruned = [];
|
|
2967
|
+
for (const name2 of readdirSync2(dir)) {
|
|
2968
|
+
if (!name2.endsWith(".jsonl.gz")) continue;
|
|
2969
|
+
const file = join5(dir, name2);
|
|
2970
|
+
try {
|
|
2971
|
+
const stat = statSync4(file);
|
|
2972
|
+
if (stat.mtimeMs < cutoff) {
|
|
2973
|
+
rmSync3(file, { force: true });
|
|
2974
|
+
pruned.push(name2);
|
|
2975
|
+
}
|
|
2976
|
+
} catch {
|
|
2977
|
+
}
|
|
2978
|
+
}
|
|
2979
|
+
return pruned;
|
|
2980
|
+
}
|
|
2981
|
+
indexFor(activeSet, asOf) {
|
|
2982
|
+
if (this.cachedIndex !== null && this.cachedIndex.revision === this.revision && this.cachedIndex.asOf === asOf) {
|
|
2983
|
+
return this.cachedIndex.index;
|
|
2984
|
+
}
|
|
2985
|
+
const index = buildIndex(activeSet, TOKENIZER_VERSION);
|
|
2986
|
+
this.cachedIndex = { revision: this.revision, asOf, index };
|
|
2987
|
+
return index;
|
|
2988
|
+
}
|
|
2989
|
+
/** Recall fast path: skip a full reload when the journal head is unchanged. */
|
|
2990
|
+
reloadIfJournalChanged() {
|
|
2991
|
+
const head = this.journal.head();
|
|
2992
|
+
if (this.issues.length === 0 && head !== null && this.head !== null && head.seq === this.head.seq && head.hash === this.head.hash) {
|
|
2993
|
+
return;
|
|
2994
|
+
}
|
|
2995
|
+
this.reload();
|
|
2996
|
+
}
|
|
2546
2997
|
stats() {
|
|
2547
2998
|
return {
|
|
2548
2999
|
active: this.active().length,
|
|
@@ -2623,20 +3074,87 @@ ${doc.body}`), queryVector));
|
|
|
2623
3074
|
writeFileAtomic(filePath, text);
|
|
2624
3075
|
this.beforeStep?.("before-journal");
|
|
2625
3076
|
const hash = hashContent(text);
|
|
2626
|
-
this.journal.append({
|
|
3077
|
+
const entry = this.journal.append({
|
|
2627
3078
|
op: "record",
|
|
2628
3079
|
...options.opId === void 0 ? {} : { opId: options.opId },
|
|
2629
3080
|
data: { id: doc.id, sha256: hash }
|
|
2630
3081
|
});
|
|
2631
3082
|
this.beforeStep?.("before-derived");
|
|
2632
|
-
|
|
3083
|
+
const parsed = parseMemoryFile(text, filePath, {
|
|
3084
|
+
abstractChars: this.config.limits.abstract_chars,
|
|
3085
|
+
bodyChars: this.config.limits.body_chars,
|
|
3086
|
+
maxRecordChars: this.config.storage.max_record_chars
|
|
3087
|
+
});
|
|
3088
|
+
this.docs.set(parsed.id, parsed);
|
|
3089
|
+
this.accounted.set(parsed.id, hash);
|
|
3090
|
+
this.deleted.delete(parsed.id);
|
|
3091
|
+
this.head = { seq: entry.seq, hash: hashLine(JSON.stringify(entry)) };
|
|
3092
|
+
this.revision += 1;
|
|
3093
|
+
this.cachedIndex = null;
|
|
3094
|
+
try {
|
|
3095
|
+
const stat = statSync4(filePath);
|
|
3096
|
+
this.fileMeta.set(parsed.id, { mtimeMs: stat.mtimeMs, size: stat.size });
|
|
3097
|
+
} catch {
|
|
3098
|
+
}
|
|
2633
3099
|
this.writeDerived();
|
|
2634
|
-
this.reload();
|
|
2635
3100
|
return { id: doc.id, content_hash: hash, deduplicated: false, warnings: [] };
|
|
2636
3101
|
} finally {
|
|
2637
3102
|
lock.release();
|
|
2638
3103
|
}
|
|
2639
3104
|
}
|
|
3105
|
+
/**
|
|
3106
|
+
* Group-commit variant of `add`: concurrent callers are coalesced for a short
|
|
3107
|
+
* window, then written with one journal fsync. Sequential awaited calls still
|
|
3108
|
+
* pay one commit each; use it when several producers write at once.
|
|
3109
|
+
*/
|
|
3110
|
+
addAsync(input, options) {
|
|
3111
|
+
return new Promise((resolve3, reject) => {
|
|
3112
|
+
this.pendingAdds.push({ input, options, resolve: resolve3, reject });
|
|
3113
|
+
if (this.addTimer === void 0) {
|
|
3114
|
+
this.addTimer = setTimeout(() => {
|
|
3115
|
+
this.addTimer = void 0;
|
|
3116
|
+
this.flushAddGroup();
|
|
3117
|
+
}, 2);
|
|
3118
|
+
}
|
|
3119
|
+
});
|
|
3120
|
+
}
|
|
3121
|
+
flushAddGroup() {
|
|
3122
|
+
const pending = this.pendingAdds.splice(0);
|
|
3123
|
+
if (pending.length === 0) return;
|
|
3124
|
+
try {
|
|
3125
|
+
this.reload();
|
|
3126
|
+
if (this.issues.length > 0) {
|
|
3127
|
+
throw new MemoryStoreError("CONFLICT", `journal is inconsistent, writes are blocked: ${this.issues[0]}`);
|
|
3128
|
+
}
|
|
3129
|
+
const unique = [];
|
|
3130
|
+
for (const job of pending) {
|
|
3131
|
+
const duplicate = this.findDuplicate(job.input, job.options.author);
|
|
3132
|
+
if (duplicate === void 0) {
|
|
3133
|
+
unique.push(job);
|
|
3134
|
+
} else {
|
|
3135
|
+
this.appendAudit({ event: "dedupe", id: duplicate.id });
|
|
3136
|
+
job.resolve({ id: duplicate.id, content_hash: duplicate.content_hash, deduplicated: true, warnings: [] });
|
|
3137
|
+
}
|
|
3138
|
+
}
|
|
3139
|
+
for (const author of ["human", "agent"]) {
|
|
3140
|
+
const group = unique.filter((job) => job.options.author === author);
|
|
3141
|
+
if (group.length === 0) continue;
|
|
3142
|
+
const commonOpId = group.length === 1 ? group[0]?.options.opId : void 0;
|
|
3143
|
+
const results = this.addMany(
|
|
3144
|
+
group.map((job) => ({
|
|
3145
|
+
...job.input,
|
|
3146
|
+
...job.options.provenance === void 0 ? {} : { provenance: job.options.provenance }
|
|
3147
|
+
})),
|
|
3148
|
+
{ author, ...commonOpId === void 0 ? {} : { opId: commonOpId } }
|
|
3149
|
+
);
|
|
3150
|
+
for (let index = 0; index < group.length; index += 1) {
|
|
3151
|
+
group[index].resolve(results[index]);
|
|
3152
|
+
}
|
|
3153
|
+
}
|
|
3154
|
+
} catch (error) {
|
|
3155
|
+
for (const job of pending) job.reject(error);
|
|
3156
|
+
}
|
|
3157
|
+
}
|
|
2640
3158
|
/**
|
|
2641
3159
|
* Accept a hand-placed or hand-edited file after a human reviewed it: append
|
|
2642
3160
|
* an `adopt` journal line with the file's current hash. The file itself is
|
|
@@ -2667,7 +3185,7 @@ ${doc.body}`), queryVector));
|
|
|
2667
3185
|
if (file === void 0) {
|
|
2668
3186
|
throw new MemoryStoreError("NOT_FOUND", `no memory file for ${id}`);
|
|
2669
3187
|
}
|
|
2670
|
-
const text =
|
|
3188
|
+
const text = readFileSync8(file, "utf8");
|
|
2671
3189
|
let doc;
|
|
2672
3190
|
try {
|
|
2673
3191
|
doc = parseMemoryFile(text, file, {
|
|
@@ -2828,19 +3346,37 @@ ${doc.body}`), queryVector));
|
|
|
2828
3346
|
throw error;
|
|
2829
3347
|
}
|
|
2830
3348
|
this.beforeStep?.("before-journal");
|
|
2831
|
-
|
|
2832
|
-
|
|
2833
|
-
this.journal.append({
|
|
3349
|
+
const journalEntries = this.journal.appendBatch(
|
|
3350
|
+
prepared.map((entry) => ({
|
|
2834
3351
|
op: "record",
|
|
2835
3352
|
...options.opId === void 0 ? {} : { opId: `${options.opId}-${entry.id}` },
|
|
2836
|
-
data: { id: entry.id, sha256:
|
|
3353
|
+
data: { id: entry.id, sha256: hashContent(entry.text) }
|
|
3354
|
+
}))
|
|
3355
|
+
);
|
|
3356
|
+
for (let index = 0; index < prepared.length; index += 1) {
|
|
3357
|
+
const entry = prepared[index];
|
|
3358
|
+
entry.hash = hashContent(entry.text);
|
|
3359
|
+
const file = join5(this.paths.memoriesDir, entry.kind, `${entry.id}.md`);
|
|
3360
|
+
const parsed = parseMemoryFile(entry.text, file, {
|
|
3361
|
+
abstractChars: this.config.limits.abstract_chars,
|
|
3362
|
+
bodyChars: this.config.limits.body_chars,
|
|
3363
|
+
maxRecordChars: this.config.storage.max_record_chars
|
|
2837
3364
|
});
|
|
2838
|
-
|
|
3365
|
+
this.docs.set(parsed.id, parsed);
|
|
3366
|
+
this.accounted.set(parsed.id, entry.hash);
|
|
3367
|
+
this.deleted.delete(parsed.id);
|
|
3368
|
+
try {
|
|
3369
|
+
const stat = statSync4(file);
|
|
3370
|
+
this.fileMeta.set(parsed.id, { mtimeMs: stat.mtimeMs, size: stat.size });
|
|
3371
|
+
} catch {
|
|
3372
|
+
}
|
|
2839
3373
|
}
|
|
3374
|
+
const lastJournalEntry = journalEntries[journalEntries.length - 1];
|
|
3375
|
+
this.head = { seq: lastJournalEntry.seq, hash: hashLine(JSON.stringify(lastJournalEntry)) };
|
|
2840
3376
|
this.beforeStep?.("before-derived");
|
|
2841
|
-
this.
|
|
3377
|
+
this.revision += 1;
|
|
3378
|
+
this.cachedIndex = null;
|
|
2842
3379
|
this.writeDerived();
|
|
2843
|
-
this.reload();
|
|
2844
3380
|
return prepared.map((entry) => ({
|
|
2845
3381
|
id: entry.id,
|
|
2846
3382
|
content_hash: entry.hash,
|
|
@@ -2911,7 +3447,7 @@ ${doc.body}`), queryVector));
|
|
|
2911
3447
|
for (const name2 of names) {
|
|
2912
3448
|
const path = join5(this.paths.pendingDir, name2);
|
|
2913
3449
|
try {
|
|
2914
|
-
const proposal = JSON.parse(
|
|
3450
|
+
const proposal = JSON.parse(readFileSync8(path, "utf8"));
|
|
2915
3451
|
if (Array.isArray(proposal.targets) && proposal.targets.includes(id)) {
|
|
2916
3452
|
rmSync3(path, { force: true });
|
|
2917
3453
|
this.journal.append({
|
|
@@ -2949,7 +3485,8 @@ ${doc.body}`), queryVector));
|
|
|
2949
3485
|
`journal is inconsistent, repair it before rebuild: ${this.issues[0]}`
|
|
2950
3486
|
);
|
|
2951
3487
|
}
|
|
2952
|
-
this.writeDerived();
|
|
3488
|
+
this.writeDerived(true);
|
|
3489
|
+
this.checkpoint();
|
|
2953
3490
|
return {
|
|
2954
3491
|
records: this.accounted.size,
|
|
2955
3492
|
files: this.docs.size,
|
|
@@ -2965,9 +3502,25 @@ ${doc.body}`), queryVector));
|
|
|
2965
3502
|
}
|
|
2966
3503
|
/** Quarantine and journal diagnostics, used by `mem doctor`. */
|
|
2967
3504
|
diagnostics() {
|
|
2968
|
-
return {
|
|
3505
|
+
return {
|
|
3506
|
+
quarantined: this.quarantined,
|
|
3507
|
+
issues: this.issues,
|
|
3508
|
+
head: this.head,
|
|
3509
|
+
journal: this.journal.stats(),
|
|
3510
|
+
reload_ms: Number(this.lastReloadMs.toFixed(3)),
|
|
3511
|
+
checkpoint: this.checkpointState
|
|
3512
|
+
};
|
|
2969
3513
|
}
|
|
2970
|
-
writeDerived() {
|
|
3514
|
+
writeDerived(force = false) {
|
|
3515
|
+
if (!force && this.docs.size > 1e3) {
|
|
3516
|
+
this.dirtyDerived += 1;
|
|
3517
|
+
if (this.dirtyDerived < 50) {
|
|
3518
|
+
this.maybeCheckpoint();
|
|
3519
|
+
if (this.head !== null && this.head.seq % 500 === 0) this.maybeArchive();
|
|
3520
|
+
return;
|
|
3521
|
+
}
|
|
3522
|
+
}
|
|
3523
|
+
this.dirtyDerived = 0;
|
|
2971
3524
|
const active = this.active();
|
|
2972
3525
|
const manifest = {
|
|
2973
3526
|
schema: 1,
|
|
@@ -2999,6 +3552,38 @@ ${doc.body}`), queryVector));
|
|
|
2999
3552
|
for (const doc of sorted) lines.push(`- [${doc.kind}] ${doc.abstract} \u2014 ${doc.id}`);
|
|
3000
3553
|
writeFileAtomic(this.paths.memoryIndexFile, `${lines.join("\n")}
|
|
3001
3554
|
`);
|
|
3555
|
+
this.maybeCheckpoint();
|
|
3556
|
+
this.maybeArchive();
|
|
3557
|
+
}
|
|
3558
|
+
/** Force a checkpoint (maintenance/tests); returns false when journal is empty. */
|
|
3559
|
+
checkpoint() {
|
|
3560
|
+
if (this.head === null) return false;
|
|
3561
|
+
const full = this.journal.readAll();
|
|
3562
|
+
const docs = [...this.accounted.entries()].map(([id, sha2562]) => ({
|
|
3563
|
+
id,
|
|
3564
|
+
sha256: sha2562,
|
|
3565
|
+
deleted: this.deleted.has(id)
|
|
3566
|
+
}));
|
|
3567
|
+
writeCheckpoint(this.paths.home, {
|
|
3568
|
+
journal: {
|
|
3569
|
+
...this.head,
|
|
3570
|
+
segment: this.journal.segments().at(-1)
|
|
3571
|
+
},
|
|
3572
|
+
docs,
|
|
3573
|
+
journalRoot: merkleRoot(full.hashes)
|
|
3574
|
+
});
|
|
3575
|
+
this.lastCheckpointSeq = this.head.seq;
|
|
3576
|
+
return true;
|
|
3577
|
+
}
|
|
3578
|
+
maybeCheckpoint(force = false) {
|
|
3579
|
+
if (this.head === null) return;
|
|
3580
|
+
if (!force && this.head.seq - this.lastCheckpointSeq < 1e3) return;
|
|
3581
|
+
this.checkpoint();
|
|
3582
|
+
}
|
|
3583
|
+
maybeArchive() {
|
|
3584
|
+
if (this.journal.stats().lines <= 5e4) return;
|
|
3585
|
+
if (readCheckpoint(this.paths.home).checkpoint === void 0) this.checkpoint();
|
|
3586
|
+
this.archiveOldSegments(true);
|
|
3002
3587
|
}
|
|
3003
3588
|
validateInput(input) {
|
|
3004
3589
|
if (!KINDS2.includes(input.kind)) {
|
|
@@ -3056,7 +3641,7 @@ ${doc.body}`), queryVector));
|
|
|
3056
3641
|
findMemoryFile(id) {
|
|
3057
3642
|
for (const kind of KINDS2) {
|
|
3058
3643
|
const file = join5(this.paths.memoriesDir, kind, `${id}.md`);
|
|
3059
|
-
if (
|
|
3644
|
+
if (existsSync6(file)) return file;
|
|
3060
3645
|
}
|
|
3061
3646
|
return void 0;
|
|
3062
3647
|
}
|
|
@@ -3075,7 +3660,7 @@ ${doc.body}`), queryVector));
|
|
|
3075
3660
|
allocateId(now) {
|
|
3076
3661
|
for (let attempt = 0; attempt < 100; attempt += 1) {
|
|
3077
3662
|
const id = newMemoryId(now);
|
|
3078
|
-
const exists = this.docs.has(id) || this.accounted.has(id) ||
|
|
3663
|
+
const exists = this.docs.has(id) || this.accounted.has(id) || existsSync6(join5(this.paths.memoriesDir, "fact", `${id}.md`)) || existsSync6(join5(this.paths.memoriesDir, "pitfall", `${id}.md`)) || existsSync6(join5(this.paths.memoriesDir, "exp", `${id}.md`));
|
|
3079
3664
|
if (!exists) return id;
|
|
3080
3665
|
}
|
|
3081
3666
|
throw new MemoryStoreError("CONFLICT", "could not allocate a unique memory id");
|
|
@@ -3090,7 +3675,7 @@ function countPending(dir) {
|
|
|
3090
3675
|
}
|
|
3091
3676
|
|
|
3092
3677
|
// packages/core/src/rules/rules.ts
|
|
3093
|
-
import { existsSync as
|
|
3678
|
+
import { existsSync as existsSync7, readFileSync as readFileSync9 } from "node:fs";
|
|
3094
3679
|
import { join as join6 } from "node:path";
|
|
3095
3680
|
var HEADER = /^(\d+)\.\s*\[(机器门禁|模型指引|废止)\]\s*(.*)$/;
|
|
3096
3681
|
var EXTRA = /^\s+(check|test):\s*(.+)$/;
|
|
@@ -3147,8 +3732,8 @@ function checkSymbolExists(check, rootDir) {
|
|
|
3147
3732
|
const [path, symbol] = check.split("#");
|
|
3148
3733
|
if (path === void 0 || symbol === void 0 || path === "" || symbol === "") return false;
|
|
3149
3734
|
const file = join6(rootDir, path);
|
|
3150
|
-
if (!
|
|
3151
|
-
return
|
|
3735
|
+
if (!existsSync7(file)) return false;
|
|
3736
|
+
return readFileSync9(file, "utf8").includes(symbol);
|
|
3152
3737
|
}
|
|
3153
3738
|
function validateRules(rules, options) {
|
|
3154
3739
|
const errors = [];
|
|
@@ -3159,11 +3744,11 @@ function validateRules(rules, options) {
|
|
|
3159
3744
|
}
|
|
3160
3745
|
if (rule.test === void 0) continue;
|
|
3161
3746
|
const testFile = join6(options.rootDir, rule.test);
|
|
3162
|
-
if (!
|
|
3747
|
+
if (!existsSync7(testFile)) {
|
|
3163
3748
|
errors.push(`rule ${rule.number}: test file "${rule.test}" does not exist`);
|
|
3164
3749
|
continue;
|
|
3165
3750
|
}
|
|
3166
|
-
if (!/\[neg\]/.test(
|
|
3751
|
+
if (!/\[neg\]/.test(readFileSync9(testFile, "utf8"))) {
|
|
3167
3752
|
errors.push(`rule ${rule.number}: test file "${rule.test}" has no [neg] case`);
|
|
3168
3753
|
}
|
|
3169
3754
|
}
|
|
@@ -3175,8 +3760,8 @@ function parseAndValidateRules(text, options) {
|
|
|
3175
3760
|
}
|
|
3176
3761
|
function loadRules(home) {
|
|
3177
3762
|
const file = memoryPaths(home).rulesFile;
|
|
3178
|
-
if (!
|
|
3179
|
-
return parseRules(
|
|
3763
|
+
if (!existsSync7(file)) return [];
|
|
3764
|
+
return parseRules(readFileSync9(file, "utf8")).rules;
|
|
3180
3765
|
}
|
|
3181
3766
|
function renderRules(rules, options = {}) {
|
|
3182
3767
|
const lines = [];
|
|
@@ -3189,7 +3774,7 @@ function renderRules(rules, options = {}) {
|
|
|
3189
3774
|
}
|
|
3190
3775
|
function readRulesFile(home) {
|
|
3191
3776
|
const file = memoryPaths(home).rulesFile;
|
|
3192
|
-
return
|
|
3777
|
+
return existsSync7(file) ? readFileSync9(file, "utf8") : "";
|
|
3193
3778
|
}
|
|
3194
3779
|
function saveRulesFile(home, text) {
|
|
3195
3780
|
const parsed = parseAndValidateRules(text, { rootDir: home });
|
|
@@ -3200,20 +3785,20 @@ function saveRulesFile(home, text) {
|
|
|
3200
3785
|
}
|
|
3201
3786
|
|
|
3202
3787
|
// packages/core/src/inject/snapshot.ts
|
|
3203
|
-
import { createHash as
|
|
3204
|
-
import { existsSync as
|
|
3788
|
+
import { createHash as createHash6 } from "node:crypto";
|
|
3789
|
+
import { existsSync as existsSync8, readFileSync as readFileSync10, readdirSync as readdirSync3, rmSync as rmSync4, statSync as statSync5 } from "node:fs";
|
|
3205
3790
|
import { join as join7 } from "node:path";
|
|
3206
3791
|
var PROMPT_SCHEMA_VERSION = 4;
|
|
3207
3792
|
var SNAPSHOT_KEEP = 20;
|
|
3208
3793
|
function snapshotPath(home, sessionId) {
|
|
3209
|
-
const name2 =
|
|
3794
|
+
const name2 = createHash6("sha256").update(sessionId, "utf8").digest("hex").slice(0, 32);
|
|
3210
3795
|
return join7(memoryPaths(home).snapshotsDir, `${name2}.json`);
|
|
3211
3796
|
}
|
|
3212
3797
|
function readSnapshot(home, sessionId) {
|
|
3213
3798
|
const file = snapshotPath(home, sessionId);
|
|
3214
|
-
if (!
|
|
3799
|
+
if (!existsSync8(file)) return null;
|
|
3215
3800
|
try {
|
|
3216
|
-
const record = JSON.parse(
|
|
3801
|
+
const record = JSON.parse(readFileSync10(file, "utf8"));
|
|
3217
3802
|
if (record.session_id !== sessionId || typeof record.text !== "string" || typeof record.memory_revision !== "string" || typeof record.prompt_schema_version !== "number") {
|
|
3218
3803
|
return null;
|
|
3219
3804
|
}
|
|
@@ -3236,7 +3821,7 @@ function pruneSnapshots(home, keep = SNAPSHOT_KEEP) {
|
|
|
3236
3821
|
return;
|
|
3237
3822
|
}
|
|
3238
3823
|
if (names.length <= keep) return;
|
|
3239
|
-
const byAge = names.map((name2) => ({ name: name2, mtime:
|
|
3824
|
+
const byAge = names.map((name2) => ({ name: name2, mtime: statSync5(join7(dir, name2)).mtimeMs })).sort((left, right) => right.mtime - left.mtime);
|
|
3240
3825
|
for (const entry of byAge.slice(keep)) {
|
|
3241
3826
|
try {
|
|
3242
3827
|
rmSync4(join7(dir, entry.name), { force: true });
|
|
@@ -3246,10 +3831,10 @@ function pruneSnapshots(home, keep = SNAPSHOT_KEEP) {
|
|
|
3246
3831
|
}
|
|
3247
3832
|
|
|
3248
3833
|
// packages/core/src/inject/assemble.ts
|
|
3249
|
-
import { createHash as
|
|
3834
|
+
import { createHash as createHash7, randomBytes as randomBytes5 } from "node:crypto";
|
|
3250
3835
|
|
|
3251
3836
|
// packages/core/src/inject/sections.ts
|
|
3252
|
-
import { existsSync as
|
|
3837
|
+
import { existsSync as existsSync9, readFileSync as readFileSync11 } from "node:fs";
|
|
3253
3838
|
|
|
3254
3839
|
// packages/core/src/personalization/guidance.ts
|
|
3255
3840
|
function renderGuidance(mode) {
|
|
@@ -3276,7 +3861,7 @@ function truncate(text, budget) {
|
|
|
3276
3861
|
return `${points.slice(0, budget - 1).join("")}\u2026`;
|
|
3277
3862
|
}
|
|
3278
3863
|
function readIfExists(file) {
|
|
3279
|
-
return
|
|
3864
|
+
return existsSync9(file) ? readFileSync11(file, "utf8").replace(COMMENT, "").trim() : "";
|
|
3280
3865
|
}
|
|
3281
3866
|
var SECTION_ORDERS = { SOUL: 100, GUIDANCE: 110, RULES: 150, USER: 200, MEMORY: 300 };
|
|
3282
3867
|
function builtinSectionSources(context) {
|
|
@@ -3364,7 +3949,7 @@ function charCount2(text) {
|
|
|
3364
3949
|
}
|
|
3365
3950
|
function memoryRevision(docs) {
|
|
3366
3951
|
const lines = [...docs].map((doc) => `${doc.id}:${doc.content_hash}`).sort((left, right) => left.localeCompare(right));
|
|
3367
|
-
return
|
|
3952
|
+
return createHash7("sha256").update(lines.join("\n"), "utf8").digest("hex");
|
|
3368
3953
|
}
|
|
3369
3954
|
function catalogLines(docs) {
|
|
3370
3955
|
const sorted = [...docs].sort((left, right) => {
|
|
@@ -3447,7 +4032,7 @@ ${section.text}`).join("\n\n");
|
|
|
3447
4032
|
throw new InjectionError("OVER_BUDGET", `injection is over inject.total_max_chars (${input.config.inject.total_max_chars})`);
|
|
3448
4033
|
}
|
|
3449
4034
|
}
|
|
3450
|
-
const delimiter = input.delimiter ??
|
|
4035
|
+
const delimiter = input.delimiter ?? randomBytes5(8).toString("hex");
|
|
3451
4036
|
const escaped = body.split(delimiter).join("<escaped>");
|
|
3452
4037
|
const text = `<<AM-${delimiter}>>
|
|
3453
4038
|
${escaped}
|
|
@@ -3497,7 +4082,7 @@ ${escaped}
|
|
|
3497
4082
|
}
|
|
3498
4083
|
|
|
3499
4084
|
// packages/core/src/proposals/proposals.ts
|
|
3500
|
-
import { existsSync as
|
|
4085
|
+
import { existsSync as existsSync10, readdirSync as readdirSync4, readFileSync as readFileSync12, rmSync as rmSync5 } from "node:fs";
|
|
3501
4086
|
import { join as join8 } from "node:path";
|
|
3502
4087
|
var ProposalError = class extends Error {
|
|
3503
4088
|
code;
|
|
@@ -3523,7 +4108,7 @@ function listProposals(store) {
|
|
|
3523
4108
|
const proposals = [];
|
|
3524
4109
|
for (const name2 of names) {
|
|
3525
4110
|
try {
|
|
3526
|
-
proposals.push(JSON.parse(
|
|
4111
|
+
proposals.push(JSON.parse(readFileSync12(join8(pendingDir(store), name2), "utf8")));
|
|
3527
4112
|
} catch {
|
|
3528
4113
|
}
|
|
3529
4114
|
}
|
|
@@ -3531,8 +4116,8 @@ function listProposals(store) {
|
|
|
3531
4116
|
}
|
|
3532
4117
|
function readProposal(store, id) {
|
|
3533
4118
|
const file = proposalPath(store, id);
|
|
3534
|
-
if (!
|
|
3535
|
-
return JSON.parse(
|
|
4119
|
+
if (!existsSync10(file)) throw new ProposalError("NOT_FOUND", `no pending proposal ${id}`);
|
|
4120
|
+
return JSON.parse(readFileSync12(file, "utf8"));
|
|
3536
4121
|
}
|
|
3537
4122
|
function currentHash(store, id) {
|
|
3538
4123
|
const doc = store.read(id);
|
|
@@ -3632,7 +4217,7 @@ function decideProposal(deps, input) {
|
|
|
3632
4217
|
}
|
|
3633
4218
|
|
|
3634
4219
|
// packages/core/src/actor.ts
|
|
3635
|
-
import { createHash as
|
|
4220
|
+
import { createHash as createHash8 } from "node:crypto";
|
|
3636
4221
|
var EvidenceError = class extends Error {
|
|
3637
4222
|
constructor(message) {
|
|
3638
4223
|
super(message);
|
|
@@ -3652,8 +4237,8 @@ function bindEvidence(input) {
|
|
|
3652
4237
|
if (!event.text.includes(input.quote)) {
|
|
3653
4238
|
throw new EvidenceError("quote is not a substring of the referenced event");
|
|
3654
4239
|
}
|
|
3655
|
-
const
|
|
3656
|
-
if (input.expectedSha256 !== void 0 && input.expectedSha256 !==
|
|
4240
|
+
const sha2562 = createHash8("sha256").update(event.text, "utf8").digest("hex");
|
|
4241
|
+
if (input.expectedSha256 !== void 0 && input.expectedSha256 !== sha2562) {
|
|
3657
4242
|
throw new EvidenceError("event hash does not match the resolved event");
|
|
3658
4243
|
}
|
|
3659
4244
|
return {
|
|
@@ -3661,7 +4246,7 @@ function bindEvidence(input) {
|
|
|
3661
4246
|
event_seq: input.eventSeq,
|
|
3662
4247
|
speaker: event.speaker,
|
|
3663
4248
|
quote: input.quote,
|
|
3664
|
-
event_sha256:
|
|
4249
|
+
event_sha256: sha2562
|
|
3665
4250
|
};
|
|
3666
4251
|
}
|
|
3667
4252
|
|
|
@@ -3685,14 +4270,14 @@ var SessionQuota = class {
|
|
|
3685
4270
|
};
|
|
3686
4271
|
|
|
3687
4272
|
// packages/core/src/distill/closeout.ts
|
|
3688
|
-
import { createHash as
|
|
3689
|
-
import { existsSync as
|
|
4273
|
+
import { createHash as createHash9 } from "node:crypto";
|
|
4274
|
+
import { existsSync as existsSync11, readFileSync as readFileSync13 } from "node:fs";
|
|
3690
4275
|
import { join as join9 } from "node:path";
|
|
3691
4276
|
function loadRejected(home) {
|
|
3692
4277
|
const file = join9(memoryPaths(home).stateDir, "distill-rejected.json");
|
|
3693
|
-
if (!
|
|
4278
|
+
if (!existsSync11(file)) return /* @__PURE__ */ new Set();
|
|
3694
4279
|
try {
|
|
3695
|
-
const parsed = JSON.parse(
|
|
4280
|
+
const parsed = JSON.parse(readFileSync13(file, "utf8"));
|
|
3696
4281
|
return new Set(Array.isArray(parsed.hashes) ? parsed.hashes.filter((value) => typeof value === "string") : []);
|
|
3697
4282
|
} catch {
|
|
3698
4283
|
return /* @__PURE__ */ new Set();
|
|
@@ -3776,7 +4361,7 @@ function summaryHash(candidate) {
|
|
|
3776
4361
|
[...candidate.targets].sort().join(","),
|
|
3777
4362
|
...candidate.replacements.map((replacement) => normalizedAbstract(replacement.abstract))
|
|
3778
4363
|
].join("|");
|
|
3779
|
-
return
|
|
4364
|
+
return createHash9("sha256").update(material, "utf8").digest("hex");
|
|
3780
4365
|
}
|
|
3781
4366
|
function findJson(text) {
|
|
3782
4367
|
const start = text.indexOf("{");
|
|
@@ -3945,8 +4530,8 @@ var CloseoutRound = class {
|
|
|
3945
4530
|
};
|
|
3946
4531
|
|
|
3947
4532
|
// packages/core/src/ops/ops.ts
|
|
3948
|
-
import { createHash as
|
|
3949
|
-
import { cpSync, existsSync as
|
|
4533
|
+
import { createHash as createHash10 } from "node:crypto";
|
|
4534
|
+
import { cpSync, existsSync as existsSync12, mkdirSync as mkdirSync3, readFileSync as readFileSync14, readdirSync as readdirSync5, statSync as statSync6, writeFileSync as writeFileSync6 } from "node:fs";
|
|
3950
4535
|
import { join as join10, relative } from "node:path";
|
|
3951
4536
|
var INCLUDE = [
|
|
3952
4537
|
"config.toml",
|
|
@@ -3965,7 +4550,7 @@ function exportStore(options) {
|
|
|
3965
4550
|
mkdirSync3(options.out, { recursive: true });
|
|
3966
4551
|
for (const entry of INCLUDE) {
|
|
3967
4552
|
const source = join10(paths.home, entry);
|
|
3968
|
-
if (!
|
|
4553
|
+
if (!existsSync12(source)) continue;
|
|
3969
4554
|
const target = join10(options.out, entry);
|
|
3970
4555
|
cpSync(source, target, { recursive: true });
|
|
3971
4556
|
}
|
|
@@ -3975,13 +4560,13 @@ function exportStore(options) {
|
|
|
3975
4560
|
const walk2 = (dir, base) => {
|
|
3976
4561
|
for (const name2 of readdirSync5(dir)) {
|
|
3977
4562
|
const full = join10(dir, name2);
|
|
3978
|
-
if (
|
|
4563
|
+
if (statSync6(full).isDirectory()) {
|
|
3979
4564
|
walk2(full, base);
|
|
3980
4565
|
continue;
|
|
3981
4566
|
}
|
|
3982
4567
|
const rel = relative(base, full).replaceAll("\\", "/");
|
|
3983
|
-
const data =
|
|
3984
|
-
lines.push(`${
|
|
4568
|
+
const data = readFileSync14(full);
|
|
4569
|
+
lines.push(`${createHash10("sha256").update(data).digest("hex")} ${rel}`);
|
|
3985
4570
|
files += 1;
|
|
3986
4571
|
bytes += data.length;
|
|
3987
4572
|
}
|
|
@@ -3989,7 +4574,7 @@ function exportStore(options) {
|
|
|
3989
4574
|
walk2(options.out, options.out);
|
|
3990
4575
|
lines.sort();
|
|
3991
4576
|
const sums = join10(options.out, "SHA256SUMS");
|
|
3992
|
-
|
|
4577
|
+
writeFileSync6(sums, `${lines.join("\n")}
|
|
3993
4578
|
`, "utf8");
|
|
3994
4579
|
return { out: options.out, files, bytes, sums };
|
|
3995
4580
|
}
|
|
@@ -3997,8 +4582,8 @@ function personalizationChars(home) {
|
|
|
3997
4582
|
const paths = memoryPaths(home);
|
|
3998
4583
|
const charsOf = (file) => {
|
|
3999
4584
|
try {
|
|
4000
|
-
if (!
|
|
4001
|
-
return [...
|
|
4585
|
+
if (!existsSync12(file)) return 0;
|
|
4586
|
+
return [...readFileSync14(file, "utf8").replace(/<!--[\s\S]*?-->/g, "")].length;
|
|
4002
4587
|
} catch {
|
|
4003
4588
|
return 0;
|
|
4004
4589
|
}
|
|
@@ -4039,7 +4624,7 @@ function inspectStore(options) {
|
|
|
4039
4624
|
}
|
|
4040
4625
|
|
|
4041
4626
|
// packages/core/src/profile/profile.ts
|
|
4042
|
-
import { existsSync as
|
|
4627
|
+
import { existsSync as existsSync13, readFileSync as readFileSync15 } from "node:fs";
|
|
4043
4628
|
var PROFILE_SEPARATOR = "\xA7";
|
|
4044
4629
|
var PROFILE_MAX_CHARS = 1e4;
|
|
4045
4630
|
var ProfileError = class extends Error {
|
|
@@ -4099,7 +4684,7 @@ function validateProfileEntries(entries) {
|
|
|
4099
4684
|
}
|
|
4100
4685
|
function loadProfile(home) {
|
|
4101
4686
|
const file = memoryPaths(home).userFile;
|
|
4102
|
-
const text =
|
|
4687
|
+
const text = existsSync13(file) ? readFileSync15(file, "utf8") : "";
|
|
4103
4688
|
const entries = parseProfileEntries(text);
|
|
4104
4689
|
return { entries, chars: profileChars(entries) };
|
|
4105
4690
|
}
|
|
@@ -4128,7 +4713,7 @@ function findProfileEntry(entries, oldText) {
|
|
|
4128
4713
|
}
|
|
4129
4714
|
|
|
4130
4715
|
// packages/core/src/personalization/scaffold.ts
|
|
4131
|
-
import { existsSync as
|
|
4716
|
+
import { existsSync as existsSync14 } from "node:fs";
|
|
4132
4717
|
var SOUL_TEMPLATE = `<!--
|
|
4133
4718
|
\u4EBA\u683C\u6458\u8981\uFF1A\u6CE8\u5165\u65F6\u53EA\u53D6\u524D 200 \u5B57\u3002\u7528\u6563\u6587\u5199\u300C\u4F60\u662F\u8C01\u3001\u600E\u4E48\u8BF4\u8BDD\u3001\u8FB9\u754C\u5728\u54EA\u300D\uFF0C\u4E0D\u8981\u5199\u4E8B\u5B9E\u6761\u76EE\u3002
|
|
4134
4719
|
\u793A\u4F8B\uFF1A\u4F60\u7684\u89D2\u8272\u662F\u2026\u2026\uFF0C\u4EE5\u2026\u2026\u7684\u8EAB\u4EFD\u4E0E\u7528\u6237\u4EA4\u6D41\uFF1B\u8BED\u6C14\u2026\u2026\uFF1B\u4E0D\u786E\u5B9A\u65F6\u5148\u95EE\uFF0C\u4E0D\u66FF\u7528\u6237\u505A\u51B3\u5B9A\u3002
|
|
@@ -4160,7 +4745,7 @@ function scaffoldPersonalization(options) {
|
|
|
4160
4745
|
const created = [];
|
|
4161
4746
|
const kept = [];
|
|
4162
4747
|
for (const [file, template] of targets) {
|
|
4163
|
-
if (
|
|
4748
|
+
if (existsSync14(file)) {
|
|
4164
4749
|
kept.push(file);
|
|
4165
4750
|
continue;
|
|
4166
4751
|
}
|