@nxuss/lemma 1.22.1 → 1.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +33 -7
- package/dist/cjs/cli/lemma-proxy.d.ts.map +1 -1
- package/dist/cjs/cli/lemma-proxy.js +15 -3
- package/dist/cjs/cli/lemma-proxy.js.map +1 -1
- package/dist/cjs/cloud/store/FileBrainStore.d.ts.map +1 -1
- package/dist/cjs/cloud/store/FileBrainStore.js +5 -2
- package/dist/cjs/cloud/store/FileBrainStore.js.map +1 -1
- package/dist/cjs/infra/errorCause.d.ts +36 -0
- package/dist/cjs/infra/errorCause.d.ts.map +1 -0
- package/dist/cjs/infra/errorCause.js +143 -0
- package/dist/cjs/infra/errorCause.js.map +1 -0
- package/dist/cjs/infra/providers/JenkinsProvider.d.ts.map +1 -1
- package/dist/cjs/infra/providers/JenkinsProvider.js +5 -0
- package/dist/cjs/infra/providers/JenkinsProvider.js.map +1 -1
- package/dist/cjs/infra/providers/OKDProvider.d.ts.map +1 -1
- package/dist/cjs/infra/providers/OKDProvider.js +5 -1
- package/dist/cjs/infra/providers/OKDProvider.js.map +1 -1
- package/dist/cjs/infra/providers/TerraformProvider.d.ts.map +1 -1
- package/dist/cjs/infra/providers/TerraformProvider.js +3 -1
- package/dist/cjs/infra/providers/TerraformProvider.js.map +1 -1
- package/dist/cjs/mcp/index.js +21 -0
- package/dist/cjs/mcp/index.js.map +1 -1
- package/dist/cjs/mcp/tool-profiles.d.ts +6 -4
- package/dist/cjs/mcp/tool-profiles.d.ts.map +1 -1
- package/dist/cjs/mcp/tool-profiles.js +3 -0
- package/dist/cjs/mcp/tool-profiles.js.map +1 -1
- package/dist/cjs/mcp/tools/context-tools.d.ts.map +1 -1
- package/dist/cjs/mcp/tools/context-tools.js +3 -1
- package/dist/cjs/mcp/tools/context-tools.js.map +1 -1
- package/dist/cjs/mcp/tools/memory.d.ts.map +1 -1
- package/dist/cjs/mcp/tools/memory.js +255 -15
- package/dist/cjs/mcp/tools/memory.js.map +1 -1
- package/dist/cjs/mcp/tools/misc.d.ts.map +1 -1
- package/dist/cjs/mcp/tools/misc.js +21 -9
- package/dist/cjs/mcp/tools/misc.js.map +1 -1
- package/dist/cjs/mcp/tools/squeeze-cache.d.ts +6 -0
- package/dist/cjs/mcp/tools/squeeze-cache.d.ts.map +1 -1
- package/dist/cjs/mcp/tools/squeeze-cache.js +57 -17
- package/dist/cjs/mcp/tools/squeeze-cache.js.map +1 -1
- package/dist/cjs/mcp/tools/workspace.d.ts.map +1 -1
- package/dist/cjs/mcp/tools/workspace.js +7 -2
- package/dist/cjs/mcp/tools/workspace.js.map +1 -1
- package/dist/cjs/mcp/tools.d.ts +18 -0
- package/dist/cjs/mcp/tools.d.ts.map +1 -1
- package/dist/cjs/mcp/tools.js +97 -16
- package/dist/cjs/mcp/tools.js.map +1 -1
- package/dist/cjs/mcp/utils.d.ts.map +1 -1
- package/dist/cjs/mcp/utils.js +18 -4
- package/dist/cjs/mcp/utils.js.map +1 -1
- package/dist/cjs/mcp/workspaceShared.d.ts +1 -0
- package/dist/cjs/mcp/workspaceShared.d.ts.map +1 -1
- package/dist/cjs/mcp/workspaceShared.js +27 -0
- package/dist/cjs/mcp/workspaceShared.js.map +1 -1
- package/dist/cjs/security/scrubForStorage.d.ts +16 -0
- package/dist/cjs/security/scrubForStorage.d.ts.map +1 -0
- package/dist/cjs/security/scrubForStorage.js +35 -0
- package/dist/cjs/security/scrubForStorage.js.map +1 -0
- package/dist/cjs/subconscious/BrainEncryption.d.ts +38 -0
- package/dist/cjs/subconscious/BrainEncryption.d.ts.map +1 -0
- package/dist/cjs/subconscious/BrainEncryption.js +137 -0
- package/dist/cjs/subconscious/BrainEncryption.js.map +1 -0
- package/dist/cjs/subconscious/TheBrainV2.d.ts +337 -0
- package/dist/cjs/subconscious/TheBrainV2.d.ts.map +1 -1
- package/dist/cjs/subconscious/TheBrainV2.js +1149 -118
- package/dist/cjs/subconscious/TheBrainV2.js.map +1 -1
- package/dist/cjs/utils/AnomalyInventory.d.ts +29 -0
- package/dist/cjs/utils/AnomalyInventory.d.ts.map +1 -0
- package/dist/cjs/utils/AnomalyInventory.js +58 -0
- package/dist/cjs/utils/AnomalyInventory.js.map +1 -0
- package/dist/cjs/utils/ExecutiveReport.d.ts +10 -0
- package/dist/cjs/utils/ExecutiveReport.d.ts.map +1 -1
- package/dist/cjs/utils/ExecutiveReport.js +17 -1
- package/dist/cjs/utils/ExecutiveReport.js.map +1 -1
- package/dist/cjs/utils/SavingsLedger.d.ts +45 -2
- package/dist/cjs/utils/SavingsLedger.d.ts.map +1 -1
- package/dist/cjs/utils/SavingsLedger.js +89 -5
- package/dist/cjs/utils/SavingsLedger.js.map +1 -1
- package/dist/cjs/utils/SessionEvidence.d.ts +28 -0
- package/dist/cjs/utils/SessionEvidence.d.ts.map +1 -0
- package/dist/cjs/utils/SessionEvidence.js +55 -0
- package/dist/cjs/utils/SessionEvidence.js.map +1 -0
- package/dist/cjs/utils/StateHashCache.d.ts +10 -0
- package/dist/cjs/utils/StateHashCache.d.ts.map +1 -1
- package/dist/cjs/utils/StateHashCache.js +15 -0
- package/dist/cjs/utils/StateHashCache.js.map +1 -1
- package/dist/cjs/utils/reportSavings.d.ts +4 -0
- package/dist/cjs/utils/reportSavings.d.ts.map +1 -1
- package/dist/cjs/utils/reportSavings.js.map +1 -1
- package/dist/esm/cli/lemma-proxy.d.ts.map +1 -1
- package/dist/esm/cli/lemma-proxy.js +15 -3
- package/dist/esm/cli/lemma-proxy.js.map +1 -1
- package/dist/esm/cloud/store/FileBrainStore.d.ts.map +1 -1
- package/dist/esm/cloud/store/FileBrainStore.js +5 -2
- package/dist/esm/cloud/store/FileBrainStore.js.map +1 -1
- package/dist/esm/infra/errorCause.d.ts +36 -0
- package/dist/esm/infra/errorCause.d.ts.map +1 -0
- package/dist/esm/infra/errorCause.js +138 -0
- package/dist/esm/infra/errorCause.js.map +1 -0
- package/dist/esm/infra/providers/JenkinsProvider.d.ts.map +1 -1
- package/dist/esm/infra/providers/JenkinsProvider.js +5 -0
- package/dist/esm/infra/providers/JenkinsProvider.js.map +1 -1
- package/dist/esm/infra/providers/OKDProvider.d.ts.map +1 -1
- package/dist/esm/infra/providers/OKDProvider.js +5 -1
- package/dist/esm/infra/providers/OKDProvider.js.map +1 -1
- package/dist/esm/infra/providers/TerraformProvider.d.ts.map +1 -1
- package/dist/esm/infra/providers/TerraformProvider.js +3 -1
- package/dist/esm/infra/providers/TerraformProvider.js.map +1 -1
- package/dist/esm/mcp/index.js +21 -0
- package/dist/esm/mcp/index.js.map +1 -1
- package/dist/esm/mcp/tool-profiles.d.ts +6 -4
- package/dist/esm/mcp/tool-profiles.d.ts.map +1 -1
- package/dist/esm/mcp/tool-profiles.js +3 -0
- package/dist/esm/mcp/tool-profiles.js.map +1 -1
- package/dist/esm/mcp/tools/context-tools.d.ts.map +1 -1
- package/dist/esm/mcp/tools/context-tools.js +3 -1
- package/dist/esm/mcp/tools/context-tools.js.map +1 -1
- package/dist/esm/mcp/tools/memory.d.ts.map +1 -1
- package/dist/esm/mcp/tools/memory.js +255 -15
- package/dist/esm/mcp/tools/memory.js.map +1 -1
- package/dist/esm/mcp/tools/misc.d.ts.map +1 -1
- package/dist/esm/mcp/tools/misc.js +21 -9
- package/dist/esm/mcp/tools/misc.js.map +1 -1
- package/dist/esm/mcp/tools/squeeze-cache.d.ts +6 -0
- package/dist/esm/mcp/tools/squeeze-cache.d.ts.map +1 -1
- package/dist/esm/mcp/tools/squeeze-cache.js +56 -17
- package/dist/esm/mcp/tools/squeeze-cache.js.map +1 -1
- package/dist/esm/mcp/tools/workspace.d.ts.map +1 -1
- package/dist/esm/mcp/tools/workspace.js +8 -3
- package/dist/esm/mcp/tools/workspace.js.map +1 -1
- package/dist/esm/mcp/tools.d.ts +18 -0
- package/dist/esm/mcp/tools.d.ts.map +1 -1
- package/dist/esm/mcp/tools.js +96 -17
- package/dist/esm/mcp/tools.js.map +1 -1
- package/dist/esm/mcp/utils.d.ts.map +1 -1
- package/dist/esm/mcp/utils.js +18 -4
- package/dist/esm/mcp/utils.js.map +1 -1
- package/dist/esm/mcp/workspaceShared.d.ts +1 -0
- package/dist/esm/mcp/workspaceShared.d.ts.map +1 -1
- package/dist/esm/mcp/workspaceShared.js +26 -0
- package/dist/esm/mcp/workspaceShared.js.map +1 -1
- package/dist/esm/security/scrubForStorage.d.ts +16 -0
- package/dist/esm/security/scrubForStorage.d.ts.map +1 -0
- package/dist/esm/security/scrubForStorage.js +32 -0
- package/dist/esm/security/scrubForStorage.js.map +1 -0
- package/dist/esm/subconscious/BrainEncryption.d.ts +38 -0
- package/dist/esm/subconscious/BrainEncryption.d.ts.map +1 -0
- package/dist/esm/subconscious/BrainEncryption.js +126 -0
- package/dist/esm/subconscious/BrainEncryption.js.map +1 -0
- package/dist/esm/subconscious/TheBrainV2.d.ts +337 -0
- package/dist/esm/subconscious/TheBrainV2.d.ts.map +1 -1
- package/dist/esm/subconscious/TheBrainV2.js +1147 -118
- package/dist/esm/subconscious/TheBrainV2.js.map +1 -1
- package/dist/esm/utils/AnomalyInventory.d.ts +29 -0
- package/dist/esm/utils/AnomalyInventory.d.ts.map +1 -0
- package/dist/esm/utils/AnomalyInventory.js +49 -0
- package/dist/esm/utils/AnomalyInventory.js.map +1 -0
- package/dist/esm/utils/ExecutiveReport.d.ts +10 -0
- package/dist/esm/utils/ExecutiveReport.d.ts.map +1 -1
- package/dist/esm/utils/ExecutiveReport.js +17 -1
- package/dist/esm/utils/ExecutiveReport.js.map +1 -1
- package/dist/esm/utils/SavingsLedger.d.ts +45 -2
- package/dist/esm/utils/SavingsLedger.d.ts.map +1 -1
- package/dist/esm/utils/SavingsLedger.js +89 -5
- package/dist/esm/utils/SavingsLedger.js.map +1 -1
- package/dist/esm/utils/SessionEvidence.d.ts +28 -0
- package/dist/esm/utils/SessionEvidence.d.ts.map +1 -0
- package/dist/esm/utils/SessionEvidence.js +49 -0
- package/dist/esm/utils/SessionEvidence.js.map +1 -0
- package/dist/esm/utils/StateHashCache.d.ts +10 -0
- package/dist/esm/utils/StateHashCache.d.ts.map +1 -1
- package/dist/esm/utils/StateHashCache.js +14 -0
- package/dist/esm/utils/StateHashCache.js.map +1 -1
- package/dist/esm/utils/reportSavings.d.ts +4 -0
- package/dist/esm/utils/reportSavings.d.ts.map +1 -1
- package/dist/esm/utils/reportSavings.js.map +1 -1
- package/package.json +1 -1
|
@@ -15,6 +15,8 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
15
15
|
};
|
|
16
16
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
17
17
|
exports.TheBrainV2 = exports.MAX_DERIVED_DEPTH = exports.BloomFilter = void 0;
|
|
18
|
+
exports.resetBrainKeyCache = resetBrainKeyCache;
|
|
19
|
+
exports.listSnapshotNames = listSnapshotNames;
|
|
18
20
|
exports.mergeEntryMaps = mergeEntryMaps;
|
|
19
21
|
exports.currentGitContext = currentGitContext;
|
|
20
22
|
exports.resetGitContextCache = resetGitContextCache;
|
|
@@ -40,6 +42,8 @@ const crypto_1 = __importDefault(require("crypto"));
|
|
|
40
42
|
const child_process_1 = require("child_process");
|
|
41
43
|
const LocalFsResolver_1 = require("./freshness/LocalFsResolver");
|
|
42
44
|
const BrainEmbeddings_1 = require("./BrainEmbeddings");
|
|
45
|
+
const scrubForStorage_1 = require("../security/scrubForStorage");
|
|
46
|
+
const BrainEncryption_1 = require("./BrainEncryption");
|
|
43
47
|
/**
|
|
44
48
|
* Default resolver for the local MCP path. Behaviour is byte-for-byte identical to
|
|
45
49
|
* the pre-refactor inline `fs.readFileSync` hashing — it never returns `null`, so
|
|
@@ -99,13 +103,20 @@ const LOCK_FILE = path_1.default.join(BRAIN_DIR, '.write.lock');
|
|
|
99
103
|
/** Sidecar vector cache for the optional semantic re-rank. See BrainEmbeddings.ts. */
|
|
100
104
|
const EMBEDDINGS_FILE = path_1.default.join(BRAIN_DIR, 'embeddings.ndjson');
|
|
101
105
|
/**
|
|
102
|
-
*
|
|
103
|
-
*
|
|
104
|
-
*
|
|
105
|
-
*
|
|
106
|
-
* so this file is an archive to read, not a bundle to re-import.
|
|
106
|
+
* The trash: an append-only archive of everything forget() removed, and the restore path
|
|
107
|
+
* for it (see restoreForgotten/purgeForgotten). The id stays tombstoned (a deliberate
|
|
108
|
+
* deletion must not come back through a merge), so this file is read as history and
|
|
109
|
+
* restored from by id — never re-imported as a bundle.
|
|
107
110
|
*/
|
|
108
111
|
const FORGOTTEN_FILE = path_1.default.join(BRAIN_DIR, 'forgotten.ndjson');
|
|
112
|
+
/** Whole-file sha256 of entries.ndjson, written alongside it on every save. */
|
|
113
|
+
const ENTRIES_SHA_FILE = `${ENTRIES_FILE}.sha256`;
|
|
114
|
+
/** Last known-good copy of entries.ndjson, rotated on every clean save. */
|
|
115
|
+
const ENTRIES_PREV_FILE = `${ENTRIES_FILE}.prev`;
|
|
116
|
+
/** Timestamped full-corpus copies taken before destructive operations. See snapshotCorpus(). */
|
|
117
|
+
const SNAP_DIR = path_1.default.join(BRAIN_DIR, 'snapshots');
|
|
118
|
+
/** How many snapshots to keep — a rollback history, not an archive. */
|
|
119
|
+
const MAX_SNAPSHOTS = 10;
|
|
109
120
|
// ─── Cross-process durability ─────────────────────────────────────────────────
|
|
110
121
|
//
|
|
111
122
|
// Every MCP client session starts its own server process, and each one holds the whole
|
|
@@ -200,6 +211,95 @@ function writeFileAtomic(target, data) {
|
|
|
200
211
|
throw err;
|
|
201
212
|
}
|
|
202
213
|
}
|
|
214
|
+
/**
|
|
215
|
+
* Disk protection: every byte that reaches the corpus files passes through protectForDisk
|
|
216
|
+
* (encrypted when a key resolves, plaintext otherwise), and every byte read back passes
|
|
217
|
+
* through unprotectFromDisk. An envelope without a resolvable key — or one that fails
|
|
218
|
+
* authentication — returns null, which the checked loader treats exactly like corruption:
|
|
219
|
+
* fall back, never serve half a corpus.
|
|
220
|
+
*/
|
|
221
|
+
function protectForDisk(plain) {
|
|
222
|
+
const key = brainDiskKey();
|
|
223
|
+
return key ? (0, BrainEncryption_1.encryptEnvelope)(key, plain) : plain;
|
|
224
|
+
}
|
|
225
|
+
function unprotectFromDisk(stored) {
|
|
226
|
+
if (!stored.startsWith(BrainEncryption_1.BRAIN_ENC_MAGIC))
|
|
227
|
+
return stored;
|
|
228
|
+
const key = brainDiskKey();
|
|
229
|
+
if (!key)
|
|
230
|
+
return null;
|
|
231
|
+
return (0, BrainEncryption_1.decryptEnvelope)(key, stored);
|
|
232
|
+
}
|
|
233
|
+
/**
|
|
234
|
+
* Process-wide data key, resolved once from env/key-file (see BrainEncryption) and
|
|
235
|
+
* validated eagerly by the constructor — a malformed key configuration fails loud at
|
|
236
|
+
* startup, never as silent plaintext mid-session.
|
|
237
|
+
*/
|
|
238
|
+
let cachedDiskKey = undefined;
|
|
239
|
+
function brainDiskKey() {
|
|
240
|
+
if (cachedDiskKey === undefined)
|
|
241
|
+
cachedDiskKey = (0, BrainEncryption_1.resolveBrainKey)(BRAIN_DIR);
|
|
242
|
+
return cachedDiskKey;
|
|
243
|
+
}
|
|
244
|
+
/** Test seam: drop the cached key so a test can rotate env/key-file mid-process. */
|
|
245
|
+
function resetBrainKeyCache() {
|
|
246
|
+
cachedDiskKey = undefined;
|
|
247
|
+
}
|
|
248
|
+
function sha256hex(text) {
|
|
249
|
+
return crypto_1.default.createHash('sha256').update(text, 'utf8').digest('hex');
|
|
250
|
+
}
|
|
251
|
+
/**
|
|
252
|
+
* Read a corpus file with whole-file verification. No sidecar hash (files written before
|
|
253
|
+
* this existed) means legacy trust: parse it as before. A sidecar mismatch means the file
|
|
254
|
+
* changed outside a save() — bitrot, a killed process mid-rename, a foreign write — and
|
|
255
|
+
* the raw bytes still come back (verified:false) so the caller can salvage intact lines;
|
|
256
|
+
* only an unreadable or absent file comes back with text:null.
|
|
257
|
+
*/
|
|
258
|
+
function readStoredText(file) {
|
|
259
|
+
let raw;
|
|
260
|
+
try {
|
|
261
|
+
raw = fs_1.default.readFileSync(file, 'utf8');
|
|
262
|
+
}
|
|
263
|
+
catch {
|
|
264
|
+
return { text: null, verified: true, present: false }; // absent file: nothing to distrust
|
|
265
|
+
}
|
|
266
|
+
let expected = null;
|
|
267
|
+
try {
|
|
268
|
+
expected = fs_1.default.readFileSync(`${file}.sha256`, 'utf8').trim() || null;
|
|
269
|
+
}
|
|
270
|
+
catch { /* legacy file without a sidecar */ }
|
|
271
|
+
if (!expected)
|
|
272
|
+
return { text: raw, verified: true, present: true };
|
|
273
|
+
return sha256hex(raw) === expected
|
|
274
|
+
? { text: raw, verified: true, present: true }
|
|
275
|
+
: { text: raw, verified: false, present: true };
|
|
276
|
+
}
|
|
277
|
+
/** Move an untrustworthy corpus file aside instead of deleting it — it may hold recoverable text. */
|
|
278
|
+
function quarantineFile(file) {
|
|
279
|
+
try {
|
|
280
|
+
if (!fs_1.default.existsSync(file))
|
|
281
|
+
return null;
|
|
282
|
+
const dest = path_1.default.join(path_1.default.dirname(file), `corrupt-${new Date().toISOString().replace(/[:.]/g, '-')}-${process.pid}.ndjson`);
|
|
283
|
+
fs_1.default.renameSync(file, dest);
|
|
284
|
+
try {
|
|
285
|
+
fs_1.default.unlinkSync(`${file}.sha256`);
|
|
286
|
+
}
|
|
287
|
+
catch { /* sidecar already describes garbage */ }
|
|
288
|
+
return dest;
|
|
289
|
+
}
|
|
290
|
+
catch {
|
|
291
|
+
return null;
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
/** Names of rollback snapshots on disk, newest last. */
|
|
295
|
+
function listSnapshotNames() {
|
|
296
|
+
try {
|
|
297
|
+
return fs_1.default.readdirSync(SNAP_DIR).filter((n) => n.endsWith('.ndjson')).sort();
|
|
298
|
+
}
|
|
299
|
+
catch {
|
|
300
|
+
return [];
|
|
301
|
+
}
|
|
302
|
+
}
|
|
203
303
|
function stampOf(file) {
|
|
204
304
|
try {
|
|
205
305
|
const s = fs_1.default.statSync(file);
|
|
@@ -214,32 +314,131 @@ function sameStamp(a, b) {
|
|
|
214
314
|
return a === b;
|
|
215
315
|
return a.mtimeMs === b.mtimeMs && a.size === b.size;
|
|
216
316
|
}
|
|
217
|
-
/**
|
|
218
|
-
function
|
|
317
|
+
/** Parse NDJSON corpus text into entries, counting skipped lines. Pure: no filesystem. */
|
|
318
|
+
function parseEntriesWithStats(raw) {
|
|
219
319
|
const out = new Map();
|
|
220
|
-
let
|
|
221
|
-
try {
|
|
222
|
-
raw = fs_1.default.readFileSync(ENTRIES_FILE, 'utf8');
|
|
223
|
-
}
|
|
224
|
-
catch {
|
|
225
|
-
return out;
|
|
226
|
-
}
|
|
320
|
+
let skipped = 0;
|
|
227
321
|
for (const line of raw.split('\n')) {
|
|
228
322
|
if (!line)
|
|
229
323
|
continue;
|
|
230
324
|
try {
|
|
231
325
|
const entry = JSON.parse(line);
|
|
232
|
-
if (!entry || typeof entry.id !== 'string')
|
|
326
|
+
if (!entry || typeof entry.id !== 'string') {
|
|
327
|
+
skipped++;
|
|
233
328
|
continue;
|
|
329
|
+
}
|
|
234
330
|
// Entries written before the format was compacted still carry `terms`; newer ones
|
|
235
331
|
// don't. Rebuilding from termFreq covers both without a migration step.
|
|
236
332
|
if (!Array.isArray(entry.terms))
|
|
237
333
|
entry.terms = Object.keys(entry.termFreq || {});
|
|
334
|
+
// Entries written before clear-path persistence keep only hashes: backfill the
|
|
335
|
+
// labels from the hash keys so old memories become navigable on first read.
|
|
336
|
+
// Next save() persists the backfill — no separate migration step, no rewrite.
|
|
337
|
+
syncClearEvidencePaths(entry);
|
|
338
|
+
for (const claim of entry.claims || [])
|
|
339
|
+
syncClearEvidencePaths(claim);
|
|
238
340
|
out.set(entry.id, entry);
|
|
239
341
|
}
|
|
240
|
-
catch {
|
|
342
|
+
catch {
|
|
343
|
+
skipped++;
|
|
344
|
+
}
|
|
241
345
|
}
|
|
242
|
-
return out;
|
|
346
|
+
return { entries: out, skipped };
|
|
347
|
+
}
|
|
348
|
+
/** Parse NDJSON corpus text into entries, skipping corrupt lines. Pure: no filesystem. */
|
|
349
|
+
function parseEntriesText(raw) {
|
|
350
|
+
return parseEntriesWithStats(raw).entries;
|
|
351
|
+
}
|
|
352
|
+
/** Parses entries.ndjson exactly the way load() does, without touching instance state. */
|
|
353
|
+
function readEntriesFile() {
|
|
354
|
+
try {
|
|
355
|
+
return parseEntriesText(fs_1.default.readFileSync(ENTRIES_FILE, 'utf8'));
|
|
356
|
+
}
|
|
357
|
+
catch {
|
|
358
|
+
return new Map();
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
/**
|
|
362
|
+
* Load the corpus with verification and graceful degradation. Policy, in order:
|
|
363
|
+
*
|
|
364
|
+
* 1. Verified file (or legacy file without a sidecar): serve everything, no incident.
|
|
365
|
+
* 2. Changed out-of-band but with intact lines: serve the survivors and record a
|
|
366
|
+
* `partial-corpus` incident — discarding good memories because one line rotted would
|
|
367
|
+
* be worse than the rot. The next save() rewrites the file clean and heals this.
|
|
368
|
+
* 3. Total loss (nothing parses, or the envelope won't decrypt): fall back to the .prev
|
|
369
|
+
* backup, or start empty if there is none — quarantining the bad bytes either way
|
|
370
|
+
* when asked, never deleting them.
|
|
371
|
+
* 4. No main file at all: a fresh canvas. The .prev backup is deliberately NOT consulted
|
|
372
|
+
* here — absence of the corpus is a state (fresh install, clear()), not damage, and
|
|
373
|
+
* resurrecting an old backup over it would undo exactly that.
|
|
374
|
+
*
|
|
375
|
+
* Only load() — the startup path — quarantines: a long-lived process that quarantined on
|
|
376
|
+
* every racing read could destroy the corpus a concurrent writer is mid-merge on. save()
|
|
377
|
+
* and syncIfChanged() record the issue and heal on their own terms instead.
|
|
378
|
+
*/
|
|
379
|
+
function loadCorpusChecked(opts) {
|
|
380
|
+
const at = new Date().toISOString();
|
|
381
|
+
const main = readStoredText(ENTRIES_FILE);
|
|
382
|
+
if (!main.present || main.text === null) {
|
|
383
|
+
return { entries: new Map(), issue: null, diskUsable: true };
|
|
384
|
+
}
|
|
385
|
+
const plain = unprotectFromDisk(main.text);
|
|
386
|
+
const parsed = plain !== null ? parseEntriesWithStats(plain) : { entries: new Map(), skipped: Number.MAX_SAFE_INTEGER };
|
|
387
|
+
if (main.verified && plain !== null && parsed.skipped === 0) {
|
|
388
|
+
return { entries: parsed.entries, issue: null, diskUsable: true };
|
|
389
|
+
}
|
|
390
|
+
if (parsed.entries.size > 0) {
|
|
391
|
+
return {
|
|
392
|
+
entries: parsed.entries,
|
|
393
|
+
issue: {
|
|
394
|
+
at,
|
|
395
|
+
file: ENTRIES_FILE,
|
|
396
|
+
action: 'partial-corpus',
|
|
397
|
+
detail: plain === null
|
|
398
|
+
? 'entries.ndjson could not be decrypted with the available key; serving intact lines only.'
|
|
399
|
+
: `entries.ndjson changed outside a save() (${parsed.skipped} unreadable line(s) skipped); serving the intact entries. The next save rewrites the file clean.`,
|
|
400
|
+
},
|
|
401
|
+
diskUsable: true,
|
|
402
|
+
};
|
|
403
|
+
}
|
|
404
|
+
// Total loss: fall back to the backup, which has no sidecar of its own — acceptance is
|
|
405
|
+
// parse-ability, and a backup that parses to entries is strictly better than bytes that
|
|
406
|
+
// verify to nothing.
|
|
407
|
+
let prevEntries = null;
|
|
408
|
+
try {
|
|
409
|
+
const prevRaw = fs_1.default.readFileSync(ENTRIES_PREV_FILE, 'utf8');
|
|
410
|
+
const prevPlain = unprotectFromDisk(prevRaw);
|
|
411
|
+
if (prevPlain !== null) {
|
|
412
|
+
const prevParsed = parseEntriesText(prevPlain);
|
|
413
|
+
if (prevParsed.size > 0)
|
|
414
|
+
prevEntries = prevParsed;
|
|
415
|
+
}
|
|
416
|
+
}
|
|
417
|
+
catch { /* no usable backup */ }
|
|
418
|
+
if (opts.quarantine)
|
|
419
|
+
quarantineFile(ENTRIES_FILE);
|
|
420
|
+
if (prevEntries) {
|
|
421
|
+
return {
|
|
422
|
+
entries: prevEntries,
|
|
423
|
+
issue: {
|
|
424
|
+
at,
|
|
425
|
+
file: ENTRIES_FILE,
|
|
426
|
+
action: 'fallback-to-backup',
|
|
427
|
+
detail: 'entries.ndjson is entirely unreadable; serving the last known-good backup instead. The bad file was quarantined, not deleted.',
|
|
428
|
+
},
|
|
429
|
+
diskUsable: true,
|
|
430
|
+
};
|
|
431
|
+
}
|
|
432
|
+
return {
|
|
433
|
+
entries: new Map(),
|
|
434
|
+
issue: {
|
|
435
|
+
at,
|
|
436
|
+
file: ENTRIES_FILE,
|
|
437
|
+
action: 'fresh-start',
|
|
438
|
+
detail: 'entries.ndjson is entirely unreadable and no usable backup exists; starting empty. The bad file was quarantined, not deleted.',
|
|
439
|
+
},
|
|
440
|
+
diskUsable: false,
|
|
441
|
+
};
|
|
243
442
|
}
|
|
244
443
|
/** Reads the tombstone map straight off disk, for absorbing a peer process's deletions. */
|
|
245
444
|
function readTombstonesFile() {
|
|
@@ -288,11 +487,19 @@ function mergeEntryMaps(mine, theirs, tombstones = new Set()) {
|
|
|
288
487
|
return;
|
|
289
488
|
}
|
|
290
489
|
const winner = effectiveTime(entry) > effectiveTime(existing) ? entry : existing;
|
|
291
|
-
|
|
490
|
+
const mergedEntry = {
|
|
292
491
|
...winner,
|
|
293
492
|
hits: Math.max(entry.hits || 0, existing.hits || 0),
|
|
294
493
|
demerits: Math.max(entry.demerits || 0, existing.demerits || 0),
|
|
295
|
-
}
|
|
494
|
+
};
|
|
495
|
+
// ROI is a monotonic tally like hits: keep the best either side collected, and keep
|
|
496
|
+
// the field absent (not zero) when neither side ever saved anything.
|
|
497
|
+
const bestSavings = Math.max(entry.tokensSaved || 0, existing.tokensSaved || 0);
|
|
498
|
+
if (bestSavings > 0)
|
|
499
|
+
mergedEntry.tokensSaved = bestSavings;
|
|
500
|
+
else
|
|
501
|
+
delete mergedEntry.tokensSaved;
|
|
502
|
+
merged.set(entry.id, mergedEntry);
|
|
296
503
|
};
|
|
297
504
|
for (const entry of mine.values())
|
|
298
505
|
put(entry);
|
|
@@ -913,16 +1120,38 @@ function trackedClaimArtifactCount(claim) {
|
|
|
913
1120
|
return Object.keys(claim.fileHashes || {}).length + Object.keys(claim.symbolHashes || {}).length;
|
|
914
1121
|
}
|
|
915
1122
|
/** Builds the persisted `Claim[]` for an entry from the caller-supplied claim inputs. */
|
|
1123
|
+
/**
|
|
1124
|
+
* Re-derive the clear-path labels from the hash keys they describe. One-way only:
|
|
1125
|
+
* hashes are the source of truth, these fields are the navigable label. Called on
|
|
1126
|
+
* every path that (re)writes evidence — store(), reanchorEvidence(), and the load
|
|
1127
|
+
* backfill — so the three can never disagree about which file a memory means.
|
|
1128
|
+
*/
|
|
1129
|
+
function syncClearEvidencePaths(target) {
|
|
1130
|
+
const files = target.fileHashes ? Object.keys(target.fileHashes).sort() : [];
|
|
1131
|
+
if (files.length > 0)
|
|
1132
|
+
target.filePaths = files;
|
|
1133
|
+
else
|
|
1134
|
+
delete target.filePaths;
|
|
1135
|
+
const syms = target.symbolHashes ? Object.keys(target.symbolHashes).sort() : [];
|
|
1136
|
+
if (syms.length > 0)
|
|
1137
|
+
target.symbols = syms;
|
|
1138
|
+
else
|
|
1139
|
+
delete target.symbols;
|
|
1140
|
+
}
|
|
916
1141
|
function buildClaims(inputs) {
|
|
917
1142
|
return inputs
|
|
918
1143
|
.filter((c) => c?.text)
|
|
919
|
-
.map((c) =>
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
1144
|
+
.map((c) => {
|
|
1145
|
+
const claim = {
|
|
1146
|
+
id: crypto_1.default.createHash('sha1').update(c.text.trim().toLowerCase()).digest('hex').substring(0, 10),
|
|
1147
|
+
text: c.text.trim(),
|
|
1148
|
+
fileHashes: c.filePaths && c.filePaths.length > 0 ? hashFilesForFreshness(c.filePaths) : undefined,
|
|
1149
|
+
symbolHashes: c.symbols && c.symbols.length > 0 ? hashSymbolsForFreshness(c.symbols) : undefined,
|
|
1150
|
+
symbolNormalizedHashes: c.symbols && c.symbols.length > 0 ? hashSymbolsNormalizedForFreshness(c.symbols) : undefined,
|
|
1151
|
+
};
|
|
1152
|
+
syncClearEvidencePaths(claim);
|
|
1153
|
+
return claim;
|
|
1154
|
+
});
|
|
926
1155
|
}
|
|
927
1156
|
/**
|
|
928
1157
|
* Same hash-compare as checkEntryFreshness, scoped to one claim's own evidence — a claim
|
|
@@ -1000,6 +1229,13 @@ class TheBrainV2 {
|
|
|
1000
1229
|
this.diskStamp = null;
|
|
1001
1230
|
/** id -> ISO deletion time, for ids that must not come back through a merge. */
|
|
1002
1231
|
this.tombstones = new Map();
|
|
1232
|
+
/**
|
|
1233
|
+
* Last integrity incident on the corpus files (corrupt main file, undecryptable
|
|
1234
|
+
* envelope, skipped sync). Set by load()/save()/syncIfChanged(), never cleared
|
|
1235
|
+
* except by a clean verified read — health() and brain_stats surface it so a silent
|
|
1236
|
+
* fallback never looks like a healthy Brain.
|
|
1237
|
+
*/
|
|
1238
|
+
this.integrityIssue = null;
|
|
1003
1239
|
/**
|
|
1004
1240
|
* id -> terms of that entry's own `query` field. Derived state, never persisted: `terms`
|
|
1005
1241
|
* covers query+response together, so scoring the query field on its own (see the
|
|
@@ -1011,6 +1247,10 @@ class TheBrainV2 {
|
|
|
1011
1247
|
this.queryTermCache = new Map();
|
|
1012
1248
|
this.sidecar = null;
|
|
1013
1249
|
this.ensureDir();
|
|
1250
|
+
// Validate key configuration before touching the corpus: a malformed LEMMA_BRAIN_KEY
|
|
1251
|
+
// or corrupt key file throws here, loud, instead of degrading into silent plaintext
|
|
1252
|
+
// (or an unreadable corpus) mid-session.
|
|
1253
|
+
brainDiskKey();
|
|
1014
1254
|
this.load();
|
|
1015
1255
|
}
|
|
1016
1256
|
ensureDir() {
|
|
@@ -1028,7 +1268,12 @@ class TheBrainV2 {
|
|
|
1028
1268
|
this.avgDocLength = meta.avgDocLength || 0;
|
|
1029
1269
|
this.tombstones = readTombstones(meta);
|
|
1030
1270
|
}
|
|
1031
|
-
|
|
1271
|
+
// Checked: a corrupt main file falls back to the .prev backup (quarantining the
|
|
1272
|
+
// bad bytes) instead of silently starting empty and letting the next save()
|
|
1273
|
+
// legitimize the data loss with a fresh sidecar hash.
|
|
1274
|
+
const checked = loadCorpusChecked({ quarantine: true });
|
|
1275
|
+
this.integrityIssue = checked.issue;
|
|
1276
|
+
this.entries = checked.entries;
|
|
1032
1277
|
for (const id of this.tombstones.keys())
|
|
1033
1278
|
this.entries.delete(id);
|
|
1034
1279
|
this.diskStamp = stampOf(ENTRIES_FILE);
|
|
@@ -1089,10 +1334,23 @@ class TheBrainV2 {
|
|
|
1089
1334
|
return;
|
|
1090
1335
|
try {
|
|
1091
1336
|
this.absorbPeerTombstones();
|
|
1092
|
-
|
|
1337
|
+
// No quarantine here (see loadCorpusChecked): a mismatch on a live read is most
|
|
1338
|
+
// likely a racing writer, so record it, keep serving memory, and let save()/load()
|
|
1339
|
+
// do the healing. The stamp still advances — re-reading the same bad bytes on
|
|
1340
|
+
// every search would turn one incident into a permanent slowdown.
|
|
1341
|
+
const checked = loadCorpusChecked({ quarantine: false });
|
|
1342
|
+
if (checked.issue) {
|
|
1343
|
+
this.integrityIssue = { ...checked.issue, action: 'sync-skipped' };
|
|
1344
|
+
this.diskStamp = stamp;
|
|
1345
|
+
return;
|
|
1346
|
+
}
|
|
1347
|
+
const onDisk = checked.entries;
|
|
1093
1348
|
const before = this.entries.size;
|
|
1094
1349
|
this.entries = mergeEntryMaps(this.entries, onDisk, new Set(this.tombstones.keys()));
|
|
1095
1350
|
this.diskStamp = stamp;
|
|
1351
|
+
// A clean verified read heals a previously recorded incident — the next save()
|
|
1352
|
+
// rewrites the sidecar, so whatever was wrong is gone, not just unnoticed.
|
|
1353
|
+
this.integrityIssue = null;
|
|
1096
1354
|
if (this.entries.size !== before || onDisk.size > 0) {
|
|
1097
1355
|
this.rebuildIndex();
|
|
1098
1356
|
this.recalcAvgDocLength();
|
|
@@ -1122,8 +1380,25 @@ class TheBrainV2 {
|
|
|
1122
1380
|
this.save();
|
|
1123
1381
|
}, 500);
|
|
1124
1382
|
}
|
|
1383
|
+
/**
|
|
1384
|
+
* Synchronously persist pending writes, if any. The debounced scheduleSave() above
|
|
1385
|
+
* is correct for a long-lived server but loses data when the process is about to
|
|
1386
|
+
* die: one-shot clients (lemma-call, the Muse skill path) kill the MCP server right
|
|
1387
|
+
* after the tool response, so without this every one-shot store/update/forget is
|
|
1388
|
+
* silently dropped. Installed as a SIGTERM/SIGINT handler by the MCP bootstrap —
|
|
1389
|
+
* never called on the hot path, where the debounce still applies.
|
|
1390
|
+
*/
|
|
1391
|
+
flushSync() {
|
|
1392
|
+
if (this.flushTimer) {
|
|
1393
|
+
clearTimeout(this.flushTimer);
|
|
1394
|
+
this.flushTimer = null;
|
|
1395
|
+
}
|
|
1396
|
+
if (this.dirty)
|
|
1397
|
+
this.save();
|
|
1398
|
+
}
|
|
1125
1399
|
save() {
|
|
1126
1400
|
const fd = acquireLock();
|
|
1401
|
+
let wroteClean = false;
|
|
1127
1402
|
try {
|
|
1128
1403
|
this.ensureDir();
|
|
1129
1404
|
// Merge before writing. A full rewrite of what this process happens to hold would
|
|
@@ -1131,8 +1406,15 @@ class TheBrainV2 {
|
|
|
1131
1406
|
// this whole section exists to close. After the merge the file we write is a superset
|
|
1132
1407
|
// of both views, so a writer can only ever add.
|
|
1133
1408
|
this.absorbPeerTombstones();
|
|
1134
|
-
const
|
|
1135
|
-
if (
|
|
1409
|
+
const checked = loadCorpusChecked({ quarantine: false });
|
|
1410
|
+
if (checked.issue) {
|
|
1411
|
+
// The file on disk doesn't verify. Merge only what the fallback recovered (the
|
|
1412
|
+
// backup, never the corrupt bytes), surface the issue, and — critically — do NOT
|
|
1413
|
+
// rotate .prev below: the corrupt main file must not become the "last good" copy.
|
|
1414
|
+
this.integrityIssue = checked.issue;
|
|
1415
|
+
}
|
|
1416
|
+
const onDisk = checked.issue && checked.issue.action === 'fresh-start' ? new Map() : checked.entries;
|
|
1417
|
+
if (checked.diskUsable && onDisk.size > 0) {
|
|
1136
1418
|
this.entries = mergeEntryMaps(this.entries, onDisk, new Set(this.tombstones.keys()));
|
|
1137
1419
|
this.rebuildIndex();
|
|
1138
1420
|
this.recalcAvgDocLength();
|
|
@@ -1141,13 +1423,39 @@ class TheBrainV2 {
|
|
|
1141
1423
|
// Post-merge, so the file we write respects the cap even when the merge pulled in
|
|
1142
1424
|
// entries a peer had already evicted.
|
|
1143
1425
|
this.evictIfOverCapacity();
|
|
1426
|
+
// Total loss on disk (nothing parseable, or an envelope this key cannot open):
|
|
1427
|
+
// preserve those bytes before overwriting. They may be a corpus under a different
|
|
1428
|
+
// key, and an overwrite is forever while a quarantine is a rename. No-op when
|
|
1429
|
+
// load() already quarantined them at startup.
|
|
1430
|
+
if (checked.issue && (checked.issue.action === 'fresh-start' || checked.issue.action === 'fallback-to-backup')) {
|
|
1431
|
+
quarantineFile(ENTRIES_FILE);
|
|
1432
|
+
}
|
|
1433
|
+
// Rotate the backup BEFORE overwriting, but only when the file being replaced is
|
|
1434
|
+
// itself trustworthy — rotating a corrupt file would destroy the last good copy.
|
|
1435
|
+
if (!checked.issue) {
|
|
1436
|
+
try {
|
|
1437
|
+
if (fs_1.default.existsSync(ENTRIES_FILE))
|
|
1438
|
+
fs_1.default.copyFileSync(ENTRIES_FILE, ENTRIES_PREV_FILE);
|
|
1439
|
+
}
|
|
1440
|
+
catch { /* backup rotation is best-effort; the write below is what matters */ }
|
|
1441
|
+
}
|
|
1144
1442
|
// Write NDJSON entries. `terms` is dropped: it is exactly Object.keys(termFreq),
|
|
1145
1443
|
// and persisting both made the entry file 38% redundant bytes that every session
|
|
1146
1444
|
// re-read at startup. load() reconstructs it.
|
|
1147
1445
|
const ndjson = Array.from(this.entries.values())
|
|
1148
1446
|
.map(({ terms: _terms, ...persisted }) => JSON.stringify(persisted))
|
|
1149
1447
|
.join('\n');
|
|
1150
|
-
|
|
1448
|
+
const stored = protectForDisk(ndjson);
|
|
1449
|
+
writeFileAtomic(ENTRIES_FILE, stored);
|
|
1450
|
+
// Sidecar hash over the exact bytes on disk, so any out-of-band change fails the
|
|
1451
|
+
// next checked read instead of being served as corpus.
|
|
1452
|
+
try {
|
|
1453
|
+
writeFileAtomic(ENTRIES_SHA_FILE, sha256hex(stored));
|
|
1454
|
+
// Whatever was wrong before is gone: what is on disk now is exactly this
|
|
1455
|
+
// process's verified corpus, so a recorded incident must not linger past it.
|
|
1456
|
+
wroteClean = true;
|
|
1457
|
+
}
|
|
1458
|
+
catch { /* an unverified corpus still beats no corpus */ }
|
|
1151
1459
|
// Write inverted index. load() rebuilds this from the entries rather than reading it
|
|
1152
1460
|
// back, so it is now purely an inspection artifact for the dashboard and for anyone
|
|
1153
1461
|
// poking at the brain directory — kept because removing a file other tooling may read
|
|
@@ -1172,6 +1480,8 @@ class TheBrainV2 {
|
|
|
1172
1480
|
finally {
|
|
1173
1481
|
releaseLock(fd);
|
|
1174
1482
|
}
|
|
1483
|
+
if (wroteClean)
|
|
1484
|
+
this.integrityIssue = null;
|
|
1175
1485
|
}
|
|
1176
1486
|
/**
|
|
1177
1487
|
* Pull in deletions made by other processes.
|
|
@@ -1229,6 +1539,22 @@ class TheBrainV2 {
|
|
|
1229
1539
|
// Another session may have stored this exact thing since we loaded; without the sync
|
|
1230
1540
|
// the dedup check below would miss it and write a second copy.
|
|
1231
1541
|
this.syncIfChanged();
|
|
1542
|
+
// Secrets never reach the corpus: redact before dedup, indexing, and persistence
|
|
1543
|
+
// alike, so the inverted index and the NDJSON see only the redacted text. Two fixes
|
|
1544
|
+
// differing only in their pasted API keys correctly dedup to one entry afterwards.
|
|
1545
|
+
const scrubbedQuery = (0, scrubForStorage_1.scrubSecretsForStorage)(query);
|
|
1546
|
+
const scrubbedResponse = (0, scrubForStorage_1.scrubSecretsForStorage)(response);
|
|
1547
|
+
query = scrubbedQuery.text;
|
|
1548
|
+
response = scrubbedResponse.text;
|
|
1549
|
+
let redactions = scrubbedQuery.redactions + scrubbedResponse.redactions;
|
|
1550
|
+
let scrubbedClaims = claimInputs;
|
|
1551
|
+
if (claimInputs && claimInputs.length > 0) {
|
|
1552
|
+
scrubbedClaims = claimInputs.map((c) => {
|
|
1553
|
+
const s = (0, scrubForStorage_1.scrubSecretsForStorage)(c.text);
|
|
1554
|
+
redactions += s.redactions;
|
|
1555
|
+
return s.redactions > 0 ? { ...c, text: s.text } : c;
|
|
1556
|
+
});
|
|
1557
|
+
}
|
|
1232
1558
|
// Quick bloom check
|
|
1233
1559
|
const queryKey = query.trim().toLowerCase().substring(0, 200);
|
|
1234
1560
|
if (this.bloom.has(queryKey)) {
|
|
@@ -1239,6 +1565,7 @@ class TheBrainV2 {
|
|
|
1239
1565
|
stored: false,
|
|
1240
1566
|
reason: `Duplicate detected (${(existing[0].similarity * 100).toFixed(1)}% similar)`,
|
|
1241
1567
|
duplicate: existing[0],
|
|
1568
|
+
redactions,
|
|
1242
1569
|
};
|
|
1243
1570
|
}
|
|
1244
1571
|
}
|
|
@@ -1246,24 +1573,10 @@ class TheBrainV2 {
|
|
|
1246
1573
|
// outcome, is worth flagging before this one is even written — two memories about one
|
|
1247
1574
|
// function disagreeing on whether an approach works is a trap for whoever searches next.
|
|
1248
1575
|
// Not a block, just a warning: the newer entry may legitimately supersede the older one.
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
const found = [];
|
|
1254
|
-
for (const existingEntry of this.entries.values()) {
|
|
1255
|
-
if (!existingEntry.symbolHashes)
|
|
1256
|
-
continue;
|
|
1257
|
-
const existingOutcome = existingEntry.outcome === 'failed' ? 'failed' : 'confirmed';
|
|
1258
|
-
if (existingOutcome === newOutcome)
|
|
1259
|
-
continue;
|
|
1260
|
-
const sharesSymbol = Object.keys(existingEntry.symbolHashes).some((k) => newKeys.has(k));
|
|
1261
|
-
if (sharesSymbol)
|
|
1262
|
-
found.push({ id: existingEntry.id, query: existingEntry.query, outcome: existingOutcome });
|
|
1263
|
-
}
|
|
1264
|
-
if (found.length > 0)
|
|
1265
|
-
conflicts = found;
|
|
1266
|
-
}
|
|
1576
|
+
const foundConflicts = symbolRefs && symbolRefs.length > 0
|
|
1577
|
+
? this.findOutcomeConflicts(symbolRefs, outcome === 'failed' ? 'failed' : 'confirmed')
|
|
1578
|
+
: [];
|
|
1579
|
+
const conflicts = foundConflicts.length > 0 ? foundConflicts : undefined;
|
|
1267
1580
|
const terms = tokenize(query + ' ' + response);
|
|
1268
1581
|
const termFreq = termFrequencies(terms);
|
|
1269
1582
|
// Salted with random bytes, not just the clock: the id used to be sha1(queryKey + now),
|
|
@@ -1298,13 +1611,17 @@ class TheBrainV2 {
|
|
|
1298
1611
|
(symbolRefs && symbolRefs.length > 0 ? hashSymbolsForFreshness(symbolRefs) : undefined),
|
|
1299
1612
|
symbolNormalizedHashes: options.evidence?.symbolsNormalized ??
|
|
1300
1613
|
(symbolRefs && symbolRefs.length > 0 ? hashSymbolsNormalizedForFreshness(symbolRefs) : undefined),
|
|
1301
|
-
claims:
|
|
1614
|
+
claims: scrubbedClaims && scrubbedClaims.length > 0 ? buildClaims(scrubbedClaims) : undefined,
|
|
1615
|
+
...(redactions > 0 ? { redactions } : {}),
|
|
1302
1616
|
domain,
|
|
1303
1617
|
derivedFrom: derivedFrom && derivedFrom.length > 0 ? derivedFrom : undefined,
|
|
1304
1618
|
source: options.source || 'manual',
|
|
1305
1619
|
...(git.commit ? { gitCommit: git.commit } : {}),
|
|
1306
1620
|
...(git.branch ? { gitBranch: git.branch } : {}),
|
|
1307
1621
|
};
|
|
1622
|
+
// Persist the tracked paths in clear next to the hashes (AN-03) — the labels
|
|
1623
|
+
// display and navigation read from here; freshness keeps reading the hashes.
|
|
1624
|
+
syncClearEvidencePaths(entry);
|
|
1308
1625
|
this.entries.set(id, entry);
|
|
1309
1626
|
// Update inverted index
|
|
1310
1627
|
for (const term of entry.terms) {
|
|
@@ -1318,7 +1635,7 @@ class TheBrainV2 {
|
|
|
1318
1635
|
this.recalcAvgDocLength();
|
|
1319
1636
|
this.evictIfOverCapacity();
|
|
1320
1637
|
this.scheduleSave();
|
|
1321
|
-
return { stored: true, reason: 'Stored successfully', id, conflicts };
|
|
1638
|
+
return { stored: true, reason: 'Stored successfully', id, conflicts, redactions };
|
|
1322
1639
|
}
|
|
1323
1640
|
/**
|
|
1324
1641
|
* Evict the lowest-value entries once the Brain is over capacity. Value = hits (proven
|
|
@@ -1417,6 +1734,11 @@ class TheBrainV2 {
|
|
|
1417
1734
|
const entry = this.entries.get(id);
|
|
1418
1735
|
if (!entry)
|
|
1419
1736
|
continue;
|
|
1737
|
+
// A superseded entry is a known-wrong conclusion with a named successor (AN-22) — it
|
|
1738
|
+
// must never rank as if still live, unlike a downvote which only demotes. get_memory
|
|
1739
|
+
// still resolves it directly and points at the successor; search() just never offers it.
|
|
1740
|
+
if (entry.supersededBy)
|
|
1741
|
+
continue;
|
|
1420
1742
|
const bm25 = bm25Score(queryTerms, entry.termFreq, entry.terms.length, this.avgDocLength || 10, idf);
|
|
1421
1743
|
const docTermSet = new Set(entry.terms);
|
|
1422
1744
|
const jaccard = jaccardSimilarity(queryTermSet, docTermSet);
|
|
@@ -1450,25 +1772,41 @@ class TheBrainV2 {
|
|
|
1450
1772
|
let similarity = exactPrompt
|
|
1451
1773
|
? 1
|
|
1452
1774
|
: (s.bm25 / maxBm25) * BM25_WEIGHT + s.jaccard * JACCARD_WEIGHT + queryFieldMatch * QUERY_FIELD_WEIGHT;
|
|
1775
|
+
// whyShown: the human-readable half of the adjustments below. A memory the caller
|
|
1776
|
+
// can't audit is a memory the caller can't trust — and a downvote without a visible
|
|
1777
|
+
// reason teaches the ranker nothing. Only signals that actually fired are listed.
|
|
1778
|
+
const whyShown = [];
|
|
1779
|
+
if (exactPrompt)
|
|
1780
|
+
whyShown.push('exact-prompt-match');
|
|
1453
1781
|
// Popularity prior: entries other searches actually reused are more likely to be
|
|
1454
1782
|
// reused again. Log-scaled and capped at +0.08 so a handful of hits can't outrank a
|
|
1455
1783
|
// much better textual match.
|
|
1456
|
-
|
|
1784
|
+
const popularityBonus = Math.min(Math.log2(s.entry.hits + 1) * 0.02, 0.08);
|
|
1785
|
+
similarity += popularityBonus;
|
|
1786
|
+
if (s.entry.hits > 0)
|
|
1787
|
+
whyShown.push(`reused ${s.entry.hits}x (+${popularityBonus.toFixed(2)})`);
|
|
1457
1788
|
// A documented dead end ranking above a working fix is actively harmful — it reads as
|
|
1458
1789
|
// a suggestion even with the "STALE"/"FAILED" label attached downstream. Demote, don't
|
|
1459
1790
|
// hide: the warning is still worth surfacing, just not first.
|
|
1460
|
-
if (s.entry.outcome === 'failed')
|
|
1791
|
+
if (s.entry.outcome === 'failed') {
|
|
1461
1792
|
similarity -= 0.15;
|
|
1793
|
+
whyShown.push('failed-attempt (-0.15)');
|
|
1794
|
+
}
|
|
1462
1795
|
// Soft domain preference (see projectId scoping above for why this stays soft): an
|
|
1463
1796
|
// entry tagged with the requested domain is more likely relevant, but an untagged or
|
|
1464
1797
|
// cross-domain entry may still be the right answer.
|
|
1465
|
-
if (options.domain && s.entry.domain === options.domain)
|
|
1798
|
+
if (options.domain && s.entry.domain === options.domain) {
|
|
1466
1799
|
similarity += 0.05;
|
|
1800
|
+
whyShown.push(`domain '${options.domain}' match (+0.05)`);
|
|
1801
|
+
}
|
|
1467
1802
|
// Explicit negative feedback from downvote() — a caller saying "this was wrong" is a
|
|
1468
1803
|
// stronger, more deliberate signal than the absence of hits, so it outweighs the
|
|
1469
1804
|
// popularity prior above rather than just canceling it out.
|
|
1470
|
-
if (s.entry.demerits)
|
|
1471
|
-
|
|
1805
|
+
if (s.entry.demerits) {
|
|
1806
|
+
const demeritPenalty = Math.min(s.entry.demerits * 0.06, 0.25);
|
|
1807
|
+
similarity -= demeritPenalty;
|
|
1808
|
+
whyShown.push(`downvoted x${s.entry.demerits} (-${demeritPenalty.toFixed(2)})`);
|
|
1809
|
+
}
|
|
1472
1810
|
// Age decay. Every other signal here is about the memory's track record; none of them
|
|
1473
1811
|
// notice that the codebase it describes has been rewritten twice since. A confirmed
|
|
1474
1812
|
// memory from a year ago is not as likely to be current as yesterday's, and until now
|
|
@@ -1477,9 +1815,20 @@ class TheBrainV2 {
|
|
|
1477
1815
|
// which is the point of refresh(): a re-verified memory really is current again.
|
|
1478
1816
|
const ageDays = (Date.now() - effectiveTime(s.entry)) / 86400000;
|
|
1479
1817
|
if (Number.isFinite(ageDays) && ageDays > RECENCY_GRACE_DAYS) {
|
|
1480
|
-
|
|
1818
|
+
const agePenalty = Math.min((ageDays - RECENCY_GRACE_DAYS) / RECENCY_FULL_DECAY_DAYS, 1) * MAX_RECENCY_PENALTY;
|
|
1819
|
+
similarity -= agePenalty;
|
|
1820
|
+
whyShown.push(`age ${Math.floor(ageDays)}d (-${agePenalty.toFixed(2)})`);
|
|
1481
1821
|
}
|
|
1482
|
-
|
|
1822
|
+
if (s.entry.refreshedAt)
|
|
1823
|
+
whyShown.push(`re-verified ${s.entry.refreshedAt.slice(0, 10)}`);
|
|
1824
|
+
// Evidence the freshness verdict was judged against: absolute tracked paths and
|
|
1825
|
+
// path::symbol keys. Handlers relativize these for display; the Brain keeps them
|
|
1826
|
+
// absolute because cwd is a caller-side notion.
|
|
1827
|
+
const evidence = [
|
|
1828
|
+
...Object.keys(s.entry.fileHashes || {}),
|
|
1829
|
+
...Object.keys(s.entry.symbolHashes || {}),
|
|
1830
|
+
];
|
|
1831
|
+
return { entry: s.entry, similarity: Math.max(0, similarity), whyShown, evidence };
|
|
1483
1832
|
});
|
|
1484
1833
|
// 6. Sort and filter
|
|
1485
1834
|
combined.sort((a, b) => b.similarity - a.similarity);
|
|
@@ -1505,6 +1854,10 @@ class TheBrainV2 {
|
|
|
1505
1854
|
provider: r.entry.provider,
|
|
1506
1855
|
timestamp: r.entry.timestamp,
|
|
1507
1856
|
fresh,
|
|
1857
|
+
...(r.entry.hits ? { hits: r.entry.hits } : {}),
|
|
1858
|
+
...(r.entry.tokensSaved ? { tokensSaved: r.entry.tokensSaved } : {}),
|
|
1859
|
+
...(r.whyShown.length > 0 ? { whyShown: r.whyShown } : {}),
|
|
1860
|
+
...(r.evidence.length > 0 ? { evidence: r.evidence } : {}),
|
|
1508
1861
|
...(fresh ? {} : { staleFiles, ...(unverified ? {} : attributeStaleness(r.entry, git)) }),
|
|
1509
1862
|
...(unverifiedFiles.length > 0 ? { unverifiedFiles } : {}),
|
|
1510
1863
|
...(r.entry.outcome ? { outcome: r.entry.outcome } : {}),
|
|
@@ -1600,6 +1953,28 @@ class TheBrainV2 {
|
|
|
1600
1953
|
getEntriesForProject(projectId) {
|
|
1601
1954
|
return Array.from(this.entries.values()).filter((e) => e.projectId === undefined || e.projectId === projectId);
|
|
1602
1955
|
}
|
|
1956
|
+
/**
|
|
1957
|
+
* Opposite-outcome entries tracking any of these symbols. Shared by store() (warn before
|
|
1958
|
+
* writing) and previewImport() (warn before importing): two memories about one function
|
|
1959
|
+
* disagreeing on whether an approach works is a trap for whoever searches next.
|
|
1960
|
+
*/
|
|
1961
|
+
findOutcomeConflicts(symbolRefs, outcome, excludeIds = new Set()) {
|
|
1962
|
+
const newKeys = new Set(symbolRefs.map((r) => `${path_1.default.resolve(r.filePath)}::${r.symbolName}`));
|
|
1963
|
+
const found = [];
|
|
1964
|
+
for (const existingEntry of this.entries.values()) {
|
|
1965
|
+
if (excludeIds.has(existingEntry.id))
|
|
1966
|
+
continue;
|
|
1967
|
+
if (!existingEntry.symbolHashes)
|
|
1968
|
+
continue;
|
|
1969
|
+
const existingOutcome = existingEntry.outcome === 'failed' ? 'failed' : 'confirmed';
|
|
1970
|
+
if (existingOutcome === outcome)
|
|
1971
|
+
continue;
|
|
1972
|
+
const sharesSymbol = Object.keys(existingEntry.symbolHashes).some((k) => newKeys.has(k));
|
|
1973
|
+
if (sharesSymbol)
|
|
1974
|
+
found.push({ id: existingEntry.id, query: existingEntry.query, outcome: existingOutcome });
|
|
1975
|
+
}
|
|
1976
|
+
return found;
|
|
1977
|
+
}
|
|
1603
1978
|
// ─── Dedup Check ─────────────────────────────────────────────────────────
|
|
1604
1979
|
/**
|
|
1605
1980
|
* Check if a query is likely a duplicate before storing.
|
|
@@ -1677,7 +2052,44 @@ class TheBrainV2 {
|
|
|
1677
2052
|
this.scheduleSave();
|
|
1678
2053
|
return { ok: true, message: `Recorded negative feedback on entry "${id}" (demerits: ${entry.demerits}). It will rank lower and be evicted sooner.` };
|
|
1679
2054
|
}
|
|
2055
|
+
/**
|
|
2056
|
+
* Book tokens saved by reusing a memory, called by search_memory on a fresh hit — the
|
|
2057
|
+
* same event that books the savings ledger, so the two can never disagree about
|
|
2058
|
+
* whether a reuse happened. Unknown ids fail silently (a peer may have forgotten the
|
|
2059
|
+
* entry since the search ranked it); savings attribution must never break a search.
|
|
2060
|
+
*/
|
|
2061
|
+
creditSavings(id, tokens) {
|
|
2062
|
+
if (!Number.isFinite(tokens) || tokens <= 0)
|
|
2063
|
+
return;
|
|
2064
|
+
const entry = this.entries.get(id);
|
|
2065
|
+
if (!entry)
|
|
2066
|
+
return;
|
|
2067
|
+
entry.tokensSaved = (entry.tokensSaved || 0) + Math.floor(tokens);
|
|
2068
|
+
this.scheduleSave();
|
|
2069
|
+
}
|
|
1680
2070
|
// ─── Deletion ───────────────────────────────────────────────────────────────
|
|
2071
|
+
/**
|
|
2072
|
+
* Archive one entry to the trash, drop it from the corpus, and tombstone the id so no
|
|
2073
|
+
* merge resurrects it. Shared by forget() (one memory) and mergeEntries() (folded
|
|
2074
|
+
* fragments). Returns whether the archive write succeeded — a failure never blocks the
|
|
2075
|
+
* removal itself, it only narrows the restore paths.
|
|
2076
|
+
*/
|
|
2077
|
+
dropWithArchive(entry) {
|
|
2078
|
+
try {
|
|
2079
|
+
this.ensureDir();
|
|
2080
|
+
const { terms: _terms, ...persisted } = entry;
|
|
2081
|
+
fs_1.default.appendFileSync(FORGOTTEN_FILE, JSON.stringify({ ...persisted, forgottenAt: new Date().toISOString() }) + '\n', 'utf8');
|
|
2082
|
+
}
|
|
2083
|
+
catch {
|
|
2084
|
+
// An unwritable trash must not block a removal the caller asked for.
|
|
2085
|
+
this.dropEntry(entry);
|
|
2086
|
+
this.tombstone(entry.id);
|
|
2087
|
+
return false;
|
|
2088
|
+
}
|
|
2089
|
+
this.dropEntry(entry);
|
|
2090
|
+
this.tombstone(entry.id);
|
|
2091
|
+
return true;
|
|
2092
|
+
}
|
|
1681
2093
|
/**
|
|
1682
2094
|
* Permanently remove one memory.
|
|
1683
2095
|
*
|
|
@@ -1697,21 +2109,13 @@ class TheBrainV2 {
|
|
|
1697
2109
|
const entry = this.entries.get(id);
|
|
1698
2110
|
if (!entry)
|
|
1699
2111
|
return { ok: false, message: `No entry with id "${id}" in the Brain.` };
|
|
2112
|
+
// Snapshot BEFORE destroying: forget() with no restore point is exactly the data-loss
|
|
2113
|
+
// story snapshots exist to prevent.
|
|
2114
|
+
this.snapshotCorpus('forget');
|
|
1700
2115
|
// Archived before it is dropped: the body is the only copy of that text anywhere, and
|
|
1701
|
-
//
|
|
1702
|
-
|
|
1703
|
-
|
|
1704
|
-
this.ensureDir();
|
|
1705
|
-
const { terms: _terms, ...persisted } = entry;
|
|
1706
|
-
fs_1.default.appendFileSync(FORGOTTEN_FILE, JSON.stringify({ ...persisted, forgottenAt: new Date().toISOString() }) + '\n', 'utf8');
|
|
1707
|
-
archivePath = FORGOTTEN_FILE;
|
|
1708
|
-
}
|
|
1709
|
-
catch {
|
|
1710
|
-
// An unwritable archive must not block a deletion the user asked for — the body still
|
|
1711
|
-
// comes back in the result below, which is the copy that matters in the moment.
|
|
1712
|
-
}
|
|
1713
|
-
this.dropEntry(entry);
|
|
1714
|
-
this.tombstone(id);
|
|
2116
|
+
// the trash is what makes this deletion restorable.
|
|
2117
|
+
const archived = this.dropWithArchive(entry);
|
|
2118
|
+
const archivePath = archived ? FORGOTTEN_FILE : undefined;
|
|
1715
2119
|
this.recalcAvgDocLength();
|
|
1716
2120
|
this.scheduleSave();
|
|
1717
2121
|
return {
|
|
@@ -1725,6 +2129,320 @@ class TheBrainV2 {
|
|
|
1725
2129
|
...(archivePath ? { archivePath } : {}),
|
|
1726
2130
|
};
|
|
1727
2131
|
}
|
|
2132
|
+
// ─── Consolidation ──────────────────────────────────────────────────────────
|
|
2133
|
+
//
|
|
2134
|
+
// Long-lived Brains accumulate near-duplicate fragments about the same topic — each
|
|
2135
|
+
// stored in a different session, each partially overlapping. findMergeCandidates()
|
|
2136
|
+
// finds the pairs; this folds them: every folded fragment becomes a claim on the
|
|
2137
|
+
// keeper (carrying its own evidence, so per-claim freshness survives the merge), the
|
|
2138
|
+
// keeper inherits the counters, and the fragments leave through the same trash door
|
|
2139
|
+
// as forget(). The keeper's own text is untouched — merging rewrites nothing, it only
|
|
2140
|
+
// attaches.
|
|
2141
|
+
/**
|
|
2142
|
+
* Fold entries into a keeper. All-or-nothing on validation (unknown keeper, unknown or
|
|
2143
|
+
* repeated fragment, keeper listed as its own fragment all fail before anything
|
|
2144
|
+
* changes), then one snapshot covers the whole operation.
|
|
2145
|
+
*/
|
|
2146
|
+
mergeEntries(keepId, foldIds) {
|
|
2147
|
+
this.syncIfChanged();
|
|
2148
|
+
const keeper = this.entries.get(keepId);
|
|
2149
|
+
if (!keeper)
|
|
2150
|
+
return { ok: false, message: `No memory with id "${keepId}" in the Brain.` };
|
|
2151
|
+
const folds = [...new Set(foldIds || [])].filter((id) => id !== keepId);
|
|
2152
|
+
if (folds.length === 0) {
|
|
2153
|
+
return { ok: false, message: `Nothing to fold into "${keepId}" — name at least one other memory id.` };
|
|
2154
|
+
}
|
|
2155
|
+
const missing = folds.filter((id) => !this.entries.has(id));
|
|
2156
|
+
if (missing.length > 0) {
|
|
2157
|
+
return { ok: false, message: `Cannot merge: ${missing.map((id) => `"${id}"`).join(', ')} ${missing.length === 1 ? 'is' : 'are'} not in the Brain. Nothing was changed.` };
|
|
2158
|
+
}
|
|
2159
|
+
this.snapshotCorpus('merge');
|
|
2160
|
+
// Out of the index before mutating, same as update(): postings built from the old
|
|
2161
|
+
// text would keep serving the keeper under words the merge removed.
|
|
2162
|
+
this.dropEntry(keeper);
|
|
2163
|
+
const newClaims = [];
|
|
2164
|
+
for (const foldId of folds) {
|
|
2165
|
+
const folded = this.entries.get(foldId);
|
|
2166
|
+
newClaims.push({
|
|
2167
|
+
id: crypto_1.default.createHash('sha1').update(`merge:${keepId}:${foldId}:${Date.now()}:${crypto_1.default.randomBytes(4).toString('hex')}`).digest('hex').substring(0, 10),
|
|
2168
|
+
text: `[merged from ${foldId} — "${folded.query.slice(0, 120)}"] ${folded.response}`,
|
|
2169
|
+
...(folded.fileHashes ? { fileHashes: { ...folded.fileHashes } } : {}),
|
|
2170
|
+
...(folded.symbolHashes ? { symbolHashes: { ...folded.symbolHashes } } : {}),
|
|
2171
|
+
...(folded.symbolNormalizedHashes ? { symbolNormalizedHashes: { ...folded.symbolNormalizedHashes } } : {}),
|
|
2172
|
+
});
|
|
2173
|
+
keeper.hits += folded.hits || 0;
|
|
2174
|
+
keeper.demerits = Math.max(keeper.demerits || 0, folded.demerits || 0);
|
|
2175
|
+
keeper.tokensSaved = (keeper.tokensSaved || 0) + (folded.tokensSaved || 0);
|
|
2176
|
+
if (keeper.tokensSaved === 0)
|
|
2177
|
+
delete keeper.tokensSaved;
|
|
2178
|
+
if (folded.redactions)
|
|
2179
|
+
keeper.redactions = (keeper.redactions || 0) + folded.redactions;
|
|
2180
|
+
keeper.mergedFrom = [...(keeper.mergedFrom || []), foldId];
|
|
2181
|
+
// Folded fragments leave through the trash door: individually restorable, and the
|
|
2182
|
+
// merge snapshot restores the pre-merge corpus wholesale. No per-fragment snapshot
|
|
2183
|
+
// here — one snapshot already covers the operation.
|
|
2184
|
+
this.dropWithArchive(folded);
|
|
2185
|
+
}
|
|
2186
|
+
keeper.claims = [...(keeper.claims || []), ...newClaims];
|
|
2187
|
+
// The curator just reviewed this memory against the folded fragments: its age clock
|
|
2188
|
+
// restarts, exactly as refresh() does after a human re-verification.
|
|
2189
|
+
keeper.refreshedAt = new Date().toISOString();
|
|
2190
|
+
const terms = tokenize(keeper.query + ' ' + keeper.response);
|
|
2191
|
+
keeper.terms = [...new Set(terms)];
|
|
2192
|
+
keeper.termFreq = termFrequencies(terms);
|
|
2193
|
+
keeper.charCount = keeper.query.length + keeper.response.length;
|
|
2194
|
+
this.entries.set(keepId, keeper);
|
|
2195
|
+
for (const term of keeper.terms) {
|
|
2196
|
+
let ids = this.invertedIndex.get(term);
|
|
2197
|
+
if (!ids) {
|
|
2198
|
+
ids = new Set();
|
|
2199
|
+
this.invertedIndex.set(term, ids);
|
|
2200
|
+
}
|
|
2201
|
+
ids.add(keepId);
|
|
2202
|
+
}
|
|
2203
|
+
this.bloom.add(keeper.query.trim().toLowerCase().substring(0, 200));
|
|
2204
|
+
this.recalcAvgDocLength();
|
|
2205
|
+
this.scheduleSave();
|
|
2206
|
+
return {
|
|
2207
|
+
ok: true,
|
|
2208
|
+
kept: keepId,
|
|
2209
|
+
folded: folds,
|
|
2210
|
+
claimsAdded: newClaims.length,
|
|
2211
|
+
message: `Merged ${folds.length} ${folds.length === 1 ? 'fragment' : 'fragments'} into "${keepId}" as independently-verifiable claims (evidence carried over). The fragments are in the trash individually, and the pre-merge corpus is snapshotted — both undo paths work.`,
|
|
2212
|
+
};
|
|
2213
|
+
}
|
|
2214
|
+
// ─── Rollback: snapshots + trash ──────────────────────────────────────────
|
|
2215
|
+
//
|
|
2216
|
+
// forget() used to be a one-way door with the archive as its only safety net, and
|
|
2217
|
+
// import/clear had no net at all. Snapshots are automatic, timestamped, pruned full
|
|
2218
|
+
// copies taken before every destructive operation; the trash (forgotten.ndjson) is the
|
|
2219
|
+
// per-memory undo. Neither is consulted by freshness, ranking, or export — rollback
|
|
2220
|
+
// state must never leak into retrieval.
|
|
2221
|
+
/**
|
|
2222
|
+
* Persist the current in-memory corpus as a timestamped snapshot. Best-effort and
|
|
2223
|
+
* synchronous like save(): a failed snapshot must never block the operation it
|
|
2224
|
+
* protects. Returns the snapshot name, or null when nothing was written.
|
|
2225
|
+
*/
|
|
2226
|
+
snapshotCorpus(reason) {
|
|
2227
|
+
try {
|
|
2228
|
+
this.ensureDir();
|
|
2229
|
+
fs_1.default.mkdirSync(SNAP_DIR, { recursive: true });
|
|
2230
|
+
const safeReason = reason.replace(/[^a-z0-9-]+/gi, '-').slice(0, 24) || 'manual';
|
|
2231
|
+
const name = `${new Date().toISOString().replace(/[:.]/g, '-')}-${process.pid}-${crypto_1.default.randomBytes(3).toString('hex')}-${safeReason}.ndjson`;
|
|
2232
|
+
const ndjson = Array.from(this.entries.values())
|
|
2233
|
+
.map(({ terms: _terms, ...persisted }) => JSON.stringify(persisted))
|
|
2234
|
+
.join('\n');
|
|
2235
|
+
writeFileAtomic(path_1.default.join(SNAP_DIR, name), protectForDisk(ndjson));
|
|
2236
|
+
// Prune oldest, newest-last ordering: a rollback history, not an archive.
|
|
2237
|
+
const names = listSnapshotNames();
|
|
2238
|
+
for (const old of names.slice(0, Math.max(0, names.length - MAX_SNAPSHOTS))) {
|
|
2239
|
+
try {
|
|
2240
|
+
fs_1.default.unlinkSync(path_1.default.join(SNAP_DIR, old));
|
|
2241
|
+
}
|
|
2242
|
+
catch { /* keep the rest */ }
|
|
2243
|
+
}
|
|
2244
|
+
return name;
|
|
2245
|
+
}
|
|
2246
|
+
catch {
|
|
2247
|
+
return null;
|
|
2248
|
+
}
|
|
2249
|
+
}
|
|
2250
|
+
/** Rollback points on disk, newest last: name, size, and modification time. */
|
|
2251
|
+
listSnapshots() {
|
|
2252
|
+
return listSnapshotNames().map((name) => {
|
|
2253
|
+
try {
|
|
2254
|
+
const st = fs_1.default.statSync(path_1.default.join(SNAP_DIR, name));
|
|
2255
|
+
return { name, bytes: st.size, mtime: st.mtime.toISOString() };
|
|
2256
|
+
}
|
|
2257
|
+
catch {
|
|
2258
|
+
return { name, bytes: 0, mtime: '' };
|
|
2259
|
+
}
|
|
2260
|
+
});
|
|
2261
|
+
}
|
|
2262
|
+
/**
|
|
2263
|
+
* Replace the corpus with a snapshot. The current state is snapshotted first
|
|
2264
|
+
* (`pre-restore`), so a restore is itself undoable. Deliberately does NOT tombstone
|
|
2265
|
+
* the ids that vanish: a peer session holding unflushed memories would lose them on
|
|
2266
|
+
* its next read, and "restore never destroys" beats "restore is total" — whatever a
|
|
2267
|
+
* peer still holds comes back on its next flush, honestly, through the normal merge.
|
|
2268
|
+
*/
|
|
2269
|
+
restoreSnapshot(name) {
|
|
2270
|
+
if (typeof name !== 'string' || name.includes('/') || name.includes('\\') || !name.endsWith('.ndjson')) {
|
|
2271
|
+
return { ok: false, message: `Refusing to read "${name}": not a snapshot name.` };
|
|
2272
|
+
}
|
|
2273
|
+
if (!listSnapshotNames().includes(name)) {
|
|
2274
|
+
return { ok: false, message: `No snapshot named "${name}". Use brain_trash action=snapshots to list them.` };
|
|
2275
|
+
}
|
|
2276
|
+
let raw;
|
|
2277
|
+
try {
|
|
2278
|
+
raw = fs_1.default.readFileSync(path_1.default.join(SNAP_DIR, name), 'utf8');
|
|
2279
|
+
}
|
|
2280
|
+
catch {
|
|
2281
|
+
return { ok: false, message: `Could not read snapshot "${name}".` };
|
|
2282
|
+
}
|
|
2283
|
+
const plain = unprotectFromDisk(raw);
|
|
2284
|
+
if (plain === null) {
|
|
2285
|
+
return { ok: false, message: `Snapshot "${name}" cannot be decrypted with the available key — refusing to wipe the live corpus for bytes I cannot read.` };
|
|
2286
|
+
}
|
|
2287
|
+
const { entries } = parseEntriesWithStats(plain);
|
|
2288
|
+
if (entries.size === 0) {
|
|
2289
|
+
return { ok: false, message: `Snapshot "${name}" parses to zero entries — refusing to replace a live corpus with an empty one.` };
|
|
2290
|
+
}
|
|
2291
|
+
this.syncIfChanged();
|
|
2292
|
+
this.snapshotCorpus('pre-restore');
|
|
2293
|
+
this.entries = entries;
|
|
2294
|
+
this.rebuildIndex();
|
|
2295
|
+
this.queryTermCache.clear();
|
|
2296
|
+
this.recalcAvgDocLength();
|
|
2297
|
+
// Fresh bloom: the filter cannot un-add the deleted queries, and a restored corpus
|
|
2298
|
+
// with a stale filter would keep reporting ghosts as possible duplicates.
|
|
2299
|
+
this.bloom = new BloomFilter();
|
|
2300
|
+
for (const entry of this.entries.values()) {
|
|
2301
|
+
this.bloom.add(entry.query.trim().toLowerCase().substring(0, 200));
|
|
2302
|
+
}
|
|
2303
|
+
// A restored id was deliberately brought back — its old tombstone must not suppress it.
|
|
2304
|
+
for (const id of this.entries.keys())
|
|
2305
|
+
this.tombstones.delete(id);
|
|
2306
|
+
this.scheduleSave();
|
|
2307
|
+
return { ok: true, restored: entries.size, message: `Restored ${entries.size} memories from snapshot "${name}". The pre-restore state was snapshotted, so this is undoable too.` };
|
|
2308
|
+
}
|
|
2309
|
+
/** What the trash holds: one row per archived deletion, newest last. */
|
|
2310
|
+
listForgotten() {
|
|
2311
|
+
const out = [];
|
|
2312
|
+
let raw;
|
|
2313
|
+
try {
|
|
2314
|
+
raw = fs_1.default.readFileSync(FORGOTTEN_FILE, 'utf8');
|
|
2315
|
+
}
|
|
2316
|
+
catch {
|
|
2317
|
+
return out;
|
|
2318
|
+
}
|
|
2319
|
+
for (const line of raw.split('\n')) {
|
|
2320
|
+
if (!line)
|
|
2321
|
+
continue;
|
|
2322
|
+
try {
|
|
2323
|
+
const archived = JSON.parse(line);
|
|
2324
|
+
if (!archived || typeof archived.id !== 'string')
|
|
2325
|
+
continue;
|
|
2326
|
+
out.push({
|
|
2327
|
+
id: archived.id,
|
|
2328
|
+
query: typeof archived.query === 'string' ? archived.query.slice(0, 120) : '',
|
|
2329
|
+
timestamp: archived.timestamp || '',
|
|
2330
|
+
hits: typeof archived.hits === 'number' ? archived.hits : 0,
|
|
2331
|
+
forgottenAt: archived.forgottenAt || '',
|
|
2332
|
+
});
|
|
2333
|
+
}
|
|
2334
|
+
catch { /* a half-written archive line is not worth failing the listing */ }
|
|
2335
|
+
}
|
|
2336
|
+
return out;
|
|
2337
|
+
}
|
|
2338
|
+
/**
|
|
2339
|
+
* Bring a forgotten memory back. The archived copy becomes a live entry again under its
|
|
2340
|
+
* original id (so any external reference to the id keeps working), its old tombstone is
|
|
2341
|
+
* lifted, and freshness is re-judged on the next search like any other entry. The
|
|
2342
|
+
* archive line stays as history — the trash is append-only, restore doesn't rewrite it.
|
|
2343
|
+
*/
|
|
2344
|
+
restoreForgotten(id) {
|
|
2345
|
+
this.syncIfChanged();
|
|
2346
|
+
if (this.entries.has(id)) {
|
|
2347
|
+
return { ok: false, message: `Memory "${id}" is already in the Brain — nothing to restore.` };
|
|
2348
|
+
}
|
|
2349
|
+
let raw;
|
|
2350
|
+
try {
|
|
2351
|
+
raw = fs_1.default.readFileSync(FORGOTTEN_FILE, 'utf8');
|
|
2352
|
+
}
|
|
2353
|
+
catch {
|
|
2354
|
+
return { ok: false, message: `The trash is empty — nothing to restore.` };
|
|
2355
|
+
}
|
|
2356
|
+
let found = null;
|
|
2357
|
+
for (const line of raw.split('\n')) {
|
|
2358
|
+
if (!line)
|
|
2359
|
+
continue;
|
|
2360
|
+
try {
|
|
2361
|
+
const archived = JSON.parse(line);
|
|
2362
|
+
if (archived && archived.id === id && typeof archived.query === 'string')
|
|
2363
|
+
found = archived;
|
|
2364
|
+
}
|
|
2365
|
+
catch { /* skip */ }
|
|
2366
|
+
}
|
|
2367
|
+
if (!found)
|
|
2368
|
+
return { ok: false, message: `No forgotten memory with id "${id}" in the trash.` };
|
|
2369
|
+
const { forgottenAt: _forgottenAt, ...revived } = found;
|
|
2370
|
+
if (!Array.isArray(revived.terms))
|
|
2371
|
+
revived.terms = Object.keys(revived.termFreq || {});
|
|
2372
|
+
if (revived.terms.length === 0) {
|
|
2373
|
+
const terms = tokenize(revived.query + ' ' + (revived.response || ''));
|
|
2374
|
+
revived.terms = [...new Set(terms)];
|
|
2375
|
+
revived.termFreq = termFrequencies(terms);
|
|
2376
|
+
}
|
|
2377
|
+
this.entries.set(id, revived);
|
|
2378
|
+
for (const term of revived.terms) {
|
|
2379
|
+
let ids = this.invertedIndex.get(term);
|
|
2380
|
+
if (!ids) {
|
|
2381
|
+
ids = new Set();
|
|
2382
|
+
this.invertedIndex.set(term, ids);
|
|
2383
|
+
}
|
|
2384
|
+
ids.add(id);
|
|
2385
|
+
}
|
|
2386
|
+
this.bloom.add(revived.query.trim().toLowerCase().substring(0, 200));
|
|
2387
|
+
this.tombstones.delete(id);
|
|
2388
|
+
this.recalcAvgDocLength();
|
|
2389
|
+
this.scheduleSave();
|
|
2390
|
+
return { ok: true, message: `Restored memory "${id}" from the trash (${revived.hits || 0} prior reuse(s) kept). Its freshness will be re-judged on the next search.` };
|
|
2391
|
+
}
|
|
2392
|
+
/**
|
|
2393
|
+
* Permanently drop archived deletions. Without a filter this empties the whole trash;
|
|
2394
|
+
* with olderThanDays it keeps recent deletions restorable. Tombstones are untouched —
|
|
2395
|
+
* purging the archive removes the restore path, not the deletion itself, so purged ids
|
|
2396
|
+
* still cannot come back through a merge or an import.
|
|
2397
|
+
*/
|
|
2398
|
+
purgeForgotten(olderThanDays) {
|
|
2399
|
+
let raw = '';
|
|
2400
|
+
try {
|
|
2401
|
+
raw = fs_1.default.readFileSync(FORGOTTEN_FILE, 'utf8');
|
|
2402
|
+
}
|
|
2403
|
+
catch {
|
|
2404
|
+
return { purged: 0, remaining: 0 };
|
|
2405
|
+
}
|
|
2406
|
+
const cutoff = typeof olderThanDays === 'number' && Number.isFinite(olderThanDays) && olderThanDays > 0
|
|
2407
|
+
? Date.now() - olderThanDays * 86400000
|
|
2408
|
+
: null;
|
|
2409
|
+
const kept = [];
|
|
2410
|
+
let purged = 0;
|
|
2411
|
+
for (const line of raw.split('\n')) {
|
|
2412
|
+
if (!line)
|
|
2413
|
+
continue;
|
|
2414
|
+
let drop = false;
|
|
2415
|
+
try {
|
|
2416
|
+
const archived = JSON.parse(line);
|
|
2417
|
+
if (!archived || typeof archived.id !== 'string')
|
|
2418
|
+
drop = true; // useless line, not a memory
|
|
2419
|
+
else if (cutoff !== null) {
|
|
2420
|
+
const at = Date.parse(archived.forgottenAt || '');
|
|
2421
|
+
drop = Number.isFinite(at) && at < cutoff;
|
|
2422
|
+
}
|
|
2423
|
+
else
|
|
2424
|
+
drop = true;
|
|
2425
|
+
}
|
|
2426
|
+
catch {
|
|
2427
|
+
drop = true; // half-written line: un-restorable, so purging loses nothing
|
|
2428
|
+
}
|
|
2429
|
+
if (drop)
|
|
2430
|
+
purged++;
|
|
2431
|
+
else
|
|
2432
|
+
kept.push(line);
|
|
2433
|
+
}
|
|
2434
|
+
if (purged > 0) {
|
|
2435
|
+
const fd = acquireLock();
|
|
2436
|
+
try {
|
|
2437
|
+
writeFileAtomic(FORGOTTEN_FILE, kept.join('\n') + (kept.length > 0 ? '\n' : ''));
|
|
2438
|
+
}
|
|
2439
|
+
catch { /* best-effort; counts below still report honestly */ }
|
|
2440
|
+
finally {
|
|
2441
|
+
releaseLock(fd);
|
|
2442
|
+
}
|
|
2443
|
+
}
|
|
2444
|
+
return { purged, remaining: kept.length };
|
|
2445
|
+
}
|
|
1728
2446
|
// ─── Re-anchoring ───────────────────────────────────────────────────────────
|
|
1729
2447
|
/**
|
|
1730
2448
|
* Re-verify a memory against the code as it stands now, keeping its identity.
|
|
@@ -1804,6 +2522,11 @@ class TheBrainV2 {
|
|
|
1804
2522
|
}
|
|
1805
2523
|
}
|
|
1806
2524
|
}
|
|
2525
|
+
// Hashes were just rewritten — re-derive the clear labels so they track the same
|
|
2526
|
+
// files (covers both the re-point and the re-anchor branches above, and update()).
|
|
2527
|
+
syncClearEvidencePaths(entry);
|
|
2528
|
+
for (const claim of entry.claims || [])
|
|
2529
|
+
syncClearEvidencePaths(claim);
|
|
1807
2530
|
return retracked;
|
|
1808
2531
|
}
|
|
1809
2532
|
refresh(id, opts = {}) {
|
|
@@ -1855,6 +2578,38 @@ class TheBrainV2 {
|
|
|
1855
2578
|
const entry = this.entries.get(id);
|
|
1856
2579
|
if (!entry)
|
|
1857
2580
|
return { ok: false, message: `No entry with id "${id}" in the Brain.` };
|
|
2581
|
+
if (patch.supersededBy !== undefined) {
|
|
2582
|
+
const successorId = patch.supersededBy.trim();
|
|
2583
|
+
if (successorId && successorId === id) {
|
|
2584
|
+
return { ok: false, message: `"${id}" cannot supersede itself.` };
|
|
2585
|
+
}
|
|
2586
|
+
if (successorId && !this.entries.has(successorId)) {
|
|
2587
|
+
return { ok: false, message: `supersededBy target "${successorId}" is not an entry in the Brain — check the id with get_memory first.` };
|
|
2588
|
+
}
|
|
2589
|
+
entry.supersededBy = successorId || undefined;
|
|
2590
|
+
}
|
|
2591
|
+
// Same one-way redaction as store(): a corrected response pasted with a fresh secret
|
|
2592
|
+
// must not be the one write path that persists credentials.
|
|
2593
|
+
let updateRedactions = 0;
|
|
2594
|
+
if (typeof patch.query === 'string' && patch.query) {
|
|
2595
|
+
const s = (0, scrubForStorage_1.scrubSecretsForStorage)(patch.query);
|
|
2596
|
+
patch.query = s.text;
|
|
2597
|
+
updateRedactions += s.redactions;
|
|
2598
|
+
}
|
|
2599
|
+
if (typeof patch.response === 'string' && patch.response) {
|
|
2600
|
+
const s = (0, scrubForStorage_1.scrubSecretsForStorage)(patch.response);
|
|
2601
|
+
patch.response = s.text;
|
|
2602
|
+
updateRedactions += s.redactions;
|
|
2603
|
+
}
|
|
2604
|
+
if (patch.claims !== undefined) {
|
|
2605
|
+
patch.claims = patch.claims.map((c) => {
|
|
2606
|
+
const s = (0, scrubForStorage_1.scrubSecretsForStorage)(c.text);
|
|
2607
|
+
updateRedactions += s.redactions;
|
|
2608
|
+
return s.redactions > 0 ? { ...c, text: s.text } : c;
|
|
2609
|
+
});
|
|
2610
|
+
}
|
|
2611
|
+
if (updateRedactions > 0)
|
|
2612
|
+
entry.redactions = (entry.redactions || 0) + updateRedactions;
|
|
1858
2613
|
const beforeChars = entry.charCount;
|
|
1859
2614
|
// Out of the index before its terms change: a posting list built from the old text would
|
|
1860
2615
|
// keep serving this entry under words the new text no longer contains.
|
|
@@ -1980,6 +2735,126 @@ class TheBrainV2 {
|
|
|
1980
2735
|
this.syncIfChanged();
|
|
1981
2736
|
return findBlastRadius(this.entries, filePath, symbolName);
|
|
1982
2737
|
}
|
|
2738
|
+
// ─── Upkeep: one command to tend the corpus ───────────────────────────────
|
|
2739
|
+
//
|
|
2740
|
+
// Every maintenance primitive existed (verify/refresh/forget/downvote/stats) but each
|
|
2741
|
+
// one needed the agent to remember it, name an id, and interpret the result — so in
|
|
2742
|
+
// practice nobody maintained anything and the corpus rotted by default. upkeep() inverts
|
|
2743
|
+
// that: one call revalidates what gets reused most, surfaces duplicates and dead
|
|
2744
|
+
// weight, and reports the savings ledger. It never deletes, merges, or refreshes
|
|
2745
|
+
// anything itself — refresh() records a human assertion no automation can make, and a
|
|
2746
|
+
// janitor that destroys memories on its own schedule is a data-loss vector, not care.
|
|
2747
|
+
/**
|
|
2748
|
+
* Near-duplicate entry pairs worth consolidating with brain_merge (see Fase E). Pairs
|
|
2749
|
+
* at or above `hi` are excluded: those should have been refused at store time, so a
|
|
2750
|
+
* surviving one is a `forget one of them` case, not a merge — report those separately
|
|
2751
|
+
* via the same call with lo=hi. Bounded: at most `sample` entries each issue one
|
|
2752
|
+
* search, and at most 20 pairs come back.
|
|
2753
|
+
*/
|
|
2754
|
+
findMergeCandidates(opts = {}) {
|
|
2755
|
+
const lo = opts.lo ?? 0.6;
|
|
2756
|
+
const hi = opts.hi ?? 0.92;
|
|
2757
|
+
const sample = Math.max(0, opts.sample ?? 100);
|
|
2758
|
+
this.syncIfChanged();
|
|
2759
|
+
const pool = Array.from(this.entries.values())
|
|
2760
|
+
.sort((a, b) => (b.hits - a.hits) || (effectiveTime(b) - effectiveTime(a)))
|
|
2761
|
+
.slice(0, sample);
|
|
2762
|
+
const seen = new Set();
|
|
2763
|
+
const out = [];
|
|
2764
|
+
for (const entry of pool) {
|
|
2765
|
+
// Internal sweep, not a question: must not move hit counters or the ledger.
|
|
2766
|
+
const results = this.search(entry.query, 6, lo, { countStats: false });
|
|
2767
|
+
for (const r of results) {
|
|
2768
|
+
if (r.id === entry.id)
|
|
2769
|
+
continue;
|
|
2770
|
+
if (r.similarity >= hi)
|
|
2771
|
+
continue;
|
|
2772
|
+
const key = [entry.id, r.id].sort().join('|');
|
|
2773
|
+
if (seen.has(key))
|
|
2774
|
+
continue;
|
|
2775
|
+
seen.add(key);
|
|
2776
|
+
const other = this.entries.get(r.id);
|
|
2777
|
+
out.push({
|
|
2778
|
+
ids: [entry.id, r.id],
|
|
2779
|
+
similarity: Number(r.similarity.toFixed(3)),
|
|
2780
|
+
queries: [entry.query.slice(0, 100), (other?.query || '').slice(0, 100)],
|
|
2781
|
+
});
|
|
2782
|
+
if (out.length >= 20)
|
|
2783
|
+
return out;
|
|
2784
|
+
}
|
|
2785
|
+
}
|
|
2786
|
+
return out;
|
|
2787
|
+
}
|
|
2788
|
+
/**
|
|
2789
|
+
* Full upkeep pass. Read-only over the corpus: revalidates the top-N most-reused
|
|
2790
|
+
* memories by hash-compare, lists exact duplicates and merge candidates, dead weight,
|
|
2791
|
+
* and the savings ledger — with a concrete suggested action per finding.
|
|
2792
|
+
*/
|
|
2793
|
+
upkeep(opts = {}) {
|
|
2794
|
+
this.syncIfChanged();
|
|
2795
|
+
const topN = Math.max(0, opts.topN ?? 20);
|
|
2796
|
+
const top = Array.from(this.entries.values())
|
|
2797
|
+
.sort((a, b) => (b.hits - a.hits) || (effectiveTime(b) - effectiveTime(a)))
|
|
2798
|
+
.slice(0, topN);
|
|
2799
|
+
const verdicts = this.verifyByIds(top.map((e) => e.id));
|
|
2800
|
+
const queryOf = (id) => this.entries.get(id)?.query.slice(0, 100) || '';
|
|
2801
|
+
const revalidated = { checked: top.length, fresh: [], stale: [], untracked: [], unverified: [] };
|
|
2802
|
+
for (const v of verdicts) {
|
|
2803
|
+
if (v.status === 'fresh')
|
|
2804
|
+
revalidated.fresh.push(v.id);
|
|
2805
|
+
else if (v.status === 'stale') {
|
|
2806
|
+
revalidated.stale.push({ id: v.id, query: queryOf(v.id), staleFiles: v.staleFiles || [], action: 'refresh_memory after checking it is still true' });
|
|
2807
|
+
}
|
|
2808
|
+
else if (v.status === 'untracked') {
|
|
2809
|
+
revalidated.untracked.push({ id: v.id, query: queryOf(v.id), action: 'update_memory with filePaths/symbols so it becomes verifiable' });
|
|
2810
|
+
}
|
|
2811
|
+
else if (v.status === 'unverified') {
|
|
2812
|
+
revalidated.unverified.push({ id: v.id, query: queryOf(v.id), action: 'check the tracked paths exist on this machine, then refresh_memory' });
|
|
2813
|
+
}
|
|
2814
|
+
}
|
|
2815
|
+
// Exact duplicates (>= store threshold) slipped in through imports or forced stores:
|
|
2816
|
+
// merging paraphrases is brain_merge's job, but a 98%-identical pair just needs one
|
|
2817
|
+
// of them forgotten.
|
|
2818
|
+
const duplicatePairs = this.findMergeCandidates({ sample: opts.sample ?? 50, lo: 0.92, hi: 1.01 })
|
|
2819
|
+
.map((c) => ({ ...c, action: 'forget_memory one of the pair (they are the same memory twice)' }));
|
|
2820
|
+
const mergeCandidates = this.findMergeCandidates({ sample: opts.sample ?? 50, lo: 0.75, hi: 0.92 })
|
|
2821
|
+
.map((c) => ({ ...c, action: 'brain_merge the pair into one canonical memory' }));
|
|
2822
|
+
const neverHitIds = [];
|
|
2823
|
+
const downvoted = [];
|
|
2824
|
+
for (const entry of this.entries.values()) {
|
|
2825
|
+
if (!entry.hits && neverHitIds.length < 10)
|
|
2826
|
+
neverHitIds.push(entry.id);
|
|
2827
|
+
if (entry.demerits)
|
|
2828
|
+
downvoted.push({ id: entry.id, demerits: entry.demerits });
|
|
2829
|
+
}
|
|
2830
|
+
downvoted.sort((a, b) => b.demerits - a.demerits);
|
|
2831
|
+
const stats = this.getStats({ deep: opts.deep === true });
|
|
2832
|
+
const report = {
|
|
2833
|
+
checkedAt: new Date().toISOString(),
|
|
2834
|
+
totalEntries: this.entries.size,
|
|
2835
|
+
revalidated,
|
|
2836
|
+
duplicatePairs,
|
|
2837
|
+
mergeCandidates,
|
|
2838
|
+
deadWeight: {
|
|
2839
|
+
neverHit: stats.neverHit || 0,
|
|
2840
|
+
neverHitSample: neverHitIds,
|
|
2841
|
+
downvoted,
|
|
2842
|
+
action: 'forget_memory entries that are both never reused and downvoted; leave the rest — an unused memory costs bytes, a deleted one costs knowledge',
|
|
2843
|
+
},
|
|
2844
|
+
savings: {
|
|
2845
|
+
totalTokensSaved: stats.totalTokensSaved || 0,
|
|
2846
|
+
topSavers: (stats.topSavers || []).slice(0, 3),
|
|
2847
|
+
},
|
|
2848
|
+
};
|
|
2849
|
+
if (stats.staleEntries !== undefined) {
|
|
2850
|
+
report.deep = {
|
|
2851
|
+
staleEntries: stats.staleEntries,
|
|
2852
|
+
staleRatio: stats.staleRatio || 0,
|
|
2853
|
+
action: 'work the revalidated.stale list above oldest-first; refresh_memory what still holds, forget_memory what does not',
|
|
2854
|
+
};
|
|
2855
|
+
}
|
|
2856
|
+
return report;
|
|
2857
|
+
}
|
|
1983
2858
|
// ─── Portability ────────────────────────────────────────────────────────────
|
|
1984
2859
|
/**
|
|
1985
2860
|
* Serialize memories to a portable NDJSON bundle: a header line, then one entry per line.
|
|
@@ -2019,6 +2894,149 @@ class TheBrainV2 {
|
|
|
2019
2894
|
const body = selected.map(({ terms: _t, ...persisted }) => JSON.stringify(persisted));
|
|
2020
2895
|
return { text: [header, ...body].join('\n'), count: selected.length, skippedStale };
|
|
2021
2896
|
}
|
|
2897
|
+
/**
|
|
2898
|
+
* Classify one bundle line without touching the corpus. The single planning step behind
|
|
2899
|
+
* both importBundle() (which then applies the plan) and previewImport() (which only
|
|
2900
|
+
* reports it) — one classifier means a dry run can never disagree with the real thing
|
|
2901
|
+
* about what would happen. Scrubbing the freshly-parsed object is safe: it is not
|
|
2902
|
+
* corpus state yet.
|
|
2903
|
+
*/
|
|
2904
|
+
planImportLine(line, markSource) {
|
|
2905
|
+
let entry;
|
|
2906
|
+
try {
|
|
2907
|
+
entry = JSON.parse(line);
|
|
2908
|
+
}
|
|
2909
|
+
catch {
|
|
2910
|
+
return { disposition: 'skip', reason: 'unparseable line' };
|
|
2911
|
+
}
|
|
2912
|
+
if (!entry || typeof entry.id !== 'string' || typeof entry.query !== 'string') {
|
|
2913
|
+
return { disposition: 'skip', reason: 'missing id/query' };
|
|
2914
|
+
}
|
|
2915
|
+
if (this.tombstones.has(entry.id)) {
|
|
2916
|
+
return { disposition: 'skip', reason: 'id was deliberately forgotten here', entry, tombstoned: true };
|
|
2917
|
+
}
|
|
2918
|
+
// Bundles from pre-scrub Brains (or hand-written ones) can carry live secrets;
|
|
2919
|
+
// redact on the way in. Already-scrubbed `[NAME_n]` spans pass through untouched.
|
|
2920
|
+
if (typeof entry.query === 'string')
|
|
2921
|
+
entry.query = (0, scrubForStorage_1.scrubSecretsForStorage)(entry.query).text;
|
|
2922
|
+
if (typeof entry.response === 'string')
|
|
2923
|
+
entry.response = (0, scrubForStorage_1.scrubSecretsForStorage)(entry.response).text;
|
|
2924
|
+
if (Array.isArray(entry.claims)) {
|
|
2925
|
+
for (const claim of entry.claims) {
|
|
2926
|
+
if (claim && typeof claim.text === 'string')
|
|
2927
|
+
claim.text = (0, scrubForStorage_1.scrubSecretsForStorage)(claim.text).text;
|
|
2928
|
+
}
|
|
2929
|
+
}
|
|
2930
|
+
if (!Array.isArray(entry.terms))
|
|
2931
|
+
entry.terms = Object.keys(entry.termFreq || {});
|
|
2932
|
+
if (entry.terms.length === 0) {
|
|
2933
|
+
// A bundle from a version that persisted neither terms nor termFreq — re-tokenize
|
|
2934
|
+
// rather than admit an entry the inverted index could never retrieve.
|
|
2935
|
+
const terms = tokenize(entry.query + ' ' + (entry.response || ''));
|
|
2936
|
+
entry.terms = [...new Set(terms)];
|
|
2937
|
+
entry.termFreq = termFrequencies(terms);
|
|
2938
|
+
}
|
|
2939
|
+
if (markSource)
|
|
2940
|
+
entry.source = 'import';
|
|
2941
|
+
const existing = this.entries.get(entry.id);
|
|
2942
|
+
if (existing) {
|
|
2943
|
+
if (effectiveTime(entry) <= effectiveTime(existing)) {
|
|
2944
|
+
return { disposition: 'skip', reason: 'local copy is newer or same age', entry };
|
|
2945
|
+
}
|
|
2946
|
+
return { disposition: 'update', reason: 'bundle copy is newer', entry };
|
|
2947
|
+
}
|
|
2948
|
+
return { disposition: 'import', reason: 'new id', entry };
|
|
2949
|
+
}
|
|
2950
|
+
/** Fold the other side's counters into ours without losing feedback either side collected. */
|
|
2951
|
+
absorbImportCounters(existing, incoming) {
|
|
2952
|
+
existing.hits = Math.max(existing.hits || 0, incoming.hits || 0);
|
|
2953
|
+
existing.demerits = Math.max(existing.demerits || 0, incoming.demerits || 0);
|
|
2954
|
+
const bestSavings = Math.max(existing.tokensSaved || 0, incoming.tokensSaved || 0);
|
|
2955
|
+
if (bestSavings > 0)
|
|
2956
|
+
existing.tokensSaved = bestSavings;
|
|
2957
|
+
else
|
|
2958
|
+
delete existing.tokensSaved;
|
|
2959
|
+
}
|
|
2960
|
+
/** Split a bundle into candidate lines. No header means raw entries — see below. */
|
|
2961
|
+
static splitBundleLines(text) {
|
|
2962
|
+
const lines = text.split('\n').filter(Boolean);
|
|
2963
|
+
if (lines.length === 0)
|
|
2964
|
+
return { lines: [] };
|
|
2965
|
+
try {
|
|
2966
|
+
const header = JSON.parse(lines[0]);
|
|
2967
|
+
if (header && header.lemmaBrainExport)
|
|
2968
|
+
return { lines: lines.slice(1) };
|
|
2969
|
+
}
|
|
2970
|
+
catch {
|
|
2971
|
+
// No header — treat the whole file as entries. A raw entries.ndjson copied off another
|
|
2972
|
+
// machine is a perfectly reasonable thing to hand this, and rejecting it would be
|
|
2973
|
+
// pedantry rather than safety.
|
|
2974
|
+
}
|
|
2975
|
+
return { lines };
|
|
2976
|
+
}
|
|
2977
|
+
/**
|
|
2978
|
+
* What importBundle() WOULD do, without writing anything: counts, the first 50 items,
|
|
2979
|
+
* and opposite-outcome conflicts against the live corpus. Run this before importing a
|
|
2980
|
+
* bundle from a teammate or another machine — especially a large one — so a flood of
|
|
2981
|
+
* stale, wrong-project memories is a preview, not a surprise.
|
|
2982
|
+
*/
|
|
2983
|
+
previewImport(text, opts = {}) {
|
|
2984
|
+
this.syncIfChanged();
|
|
2985
|
+
const { lines } = TheBrainV2.splitBundleLines(text);
|
|
2986
|
+
const empty = { ok: false, wouldImport: 0, wouldUpdate: 0, wouldSkip: 0, items: [], truncated: false, conflicts: [], message: 'Bundle is empty.' };
|
|
2987
|
+
if (lines.length === 0)
|
|
2988
|
+
return empty;
|
|
2989
|
+
const markSource = opts.markSource !== false;
|
|
2990
|
+
let wouldImport = 0;
|
|
2991
|
+
let wouldUpdate = 0;
|
|
2992
|
+
let wouldSkip = 0;
|
|
2993
|
+
const items = [];
|
|
2994
|
+
const conflicts = [];
|
|
2995
|
+
const seenConflicts = new Set();
|
|
2996
|
+
for (const line of lines) {
|
|
2997
|
+
const plan = this.planImportLine(line, markSource);
|
|
2998
|
+
if (plan.disposition === 'import')
|
|
2999
|
+
wouldImport++;
|
|
3000
|
+
else if (plan.disposition === 'update')
|
|
3001
|
+
wouldUpdate++;
|
|
3002
|
+
else
|
|
3003
|
+
wouldSkip++;
|
|
3004
|
+
if (items.length < 50) {
|
|
3005
|
+
items.push({
|
|
3006
|
+
id: plan.entry?.id ?? '(unparseable)',
|
|
3007
|
+
query: (plan.entry?.query || '').slice(0, 100),
|
|
3008
|
+
action: plan.disposition,
|
|
3009
|
+
reason: plan.reason,
|
|
3010
|
+
});
|
|
3011
|
+
}
|
|
3012
|
+
if (plan.entry && plan.disposition !== 'skip' && plan.entry.symbolHashes) {
|
|
3013
|
+
const refs = Object.keys(plan.entry.symbolHashes).map((key) => {
|
|
3014
|
+
const sep = key.lastIndexOf('::');
|
|
3015
|
+
return { filePath: key.substring(0, sep), symbolName: key.substring(sep + 2) };
|
|
3016
|
+
});
|
|
3017
|
+
const outcome = plan.entry.outcome === 'failed' ? 'failed' : 'confirmed';
|
|
3018
|
+
for (const c of this.findOutcomeConflicts(refs, outcome, new Set([plan.entry.id]))) {
|
|
3019
|
+
const ck = `${c.id}|${plan.entry.id}`;
|
|
3020
|
+
if (seenConflicts.has(ck))
|
|
3021
|
+
continue;
|
|
3022
|
+
seenConflicts.add(ck);
|
|
3023
|
+
conflicts.push({ ...c, against: plan.entry.id });
|
|
3024
|
+
}
|
|
3025
|
+
}
|
|
3026
|
+
}
|
|
3027
|
+
return {
|
|
3028
|
+
ok: true,
|
|
3029
|
+
wouldImport,
|
|
3030
|
+
wouldUpdate,
|
|
3031
|
+
wouldSkip,
|
|
3032
|
+
items,
|
|
3033
|
+
truncated: lines.length > items.length,
|
|
3034
|
+
conflicts,
|
|
3035
|
+
message: `Dry run: would import ${wouldImport}, update ${wouldUpdate}, skip ${wouldSkip}.` +
|
|
3036
|
+
(conflicts.length > 0 ? ` ${conflicts.length} opposite-outcome conflict(s) with live memories — review before importing.` : '') +
|
|
3037
|
+
` Nothing was written.`,
|
|
3038
|
+
};
|
|
3039
|
+
}
|
|
2022
3040
|
/**
|
|
2023
3041
|
* Merge a bundle produced by exportBundle into this Brain.
|
|
2024
3042
|
*
|
|
@@ -2034,64 +3052,35 @@ class TheBrainV2 {
|
|
|
2034
3052
|
*/
|
|
2035
3053
|
importBundle(text, opts = {}) {
|
|
2036
3054
|
this.syncIfChanged();
|
|
2037
|
-
const lines =
|
|
3055
|
+
const { lines } = TheBrainV2.splitBundleLines(text);
|
|
2038
3056
|
if (lines.length === 0)
|
|
2039
3057
|
return { ok: false, imported: 0, updated: 0, skipped: 0, message: 'Bundle is empty.' };
|
|
2040
|
-
|
|
2041
|
-
|
|
2042
|
-
|
|
2043
|
-
if (header && header.lemmaBrainExport)
|
|
2044
|
-
start = 1;
|
|
2045
|
-
}
|
|
2046
|
-
catch {
|
|
2047
|
-
// No header — treat the whole file as entries. A raw entries.ndjson copied off another
|
|
2048
|
-
// machine is a perfectly reasonable thing to hand this, and rejecting it would be
|
|
2049
|
-
// pedantry rather than safety.
|
|
2050
|
-
}
|
|
3058
|
+
// Snapshot before merging foreign entries: an import is the easiest way to flood a
|
|
3059
|
+
// clean corpus with hundreds of stale, wrong-project memories.
|
|
3060
|
+
this.snapshotCorpus('import');
|
|
2051
3061
|
let imported = 0;
|
|
2052
3062
|
let updated = 0;
|
|
2053
3063
|
let skipped = 0;
|
|
2054
3064
|
const markSource = opts.markSource !== false;
|
|
2055
|
-
for (const line of lines
|
|
2056
|
-
|
|
2057
|
-
|
|
2058
|
-
|
|
2059
|
-
|
|
2060
|
-
|
|
2061
|
-
|
|
2062
|
-
|
|
2063
|
-
|
|
2064
|
-
|
|
2065
|
-
|
|
2066
|
-
continue;
|
|
2067
|
-
}
|
|
2068
|
-
if (this.tombstones.has(entry.id)) {
|
|
3065
|
+
for (const line of lines) {
|
|
3066
|
+
const plan = this.planImportLine(line, markSource);
|
|
3067
|
+
if (plan.disposition === 'skip') {
|
|
3068
|
+
// Tombstoned skips excepted: a deliberately forgotten id gets no counter updates,
|
|
3069
|
+
// exactly as before the planner existed — the deletion stands completely.
|
|
3070
|
+
if (plan.entry && !plan.tombstoned) {
|
|
3071
|
+
const existing = this.entries.get(plan.entry.id);
|
|
3072
|
+
// Older or same age: keep ours, but never lose feedback the other side collected.
|
|
3073
|
+
if (existing)
|
|
3074
|
+
this.absorbImportCounters(existing, plan.entry);
|
|
3075
|
+
}
|
|
2069
3076
|
skipped++;
|
|
2070
3077
|
continue;
|
|
2071
3078
|
}
|
|
2072
|
-
|
|
2073
|
-
entry.terms = Object.keys(entry.termFreq || {});
|
|
2074
|
-
if (entry.terms.length === 0) {
|
|
2075
|
-
// A bundle from a version that persisted neither terms nor termFreq — re-tokenize
|
|
2076
|
-
// rather than admit an entry the inverted index could never retrieve.
|
|
2077
|
-
const terms = tokenize(entry.query + ' ' + (entry.response || ''));
|
|
2078
|
-
entry.terms = [...new Set(terms)];
|
|
2079
|
-
entry.termFreq = termFrequencies(terms);
|
|
2080
|
-
}
|
|
2081
|
-
if (markSource)
|
|
2082
|
-
entry.source = 'import';
|
|
3079
|
+
const entry = plan.entry;
|
|
2083
3080
|
const existing = this.entries.get(entry.id);
|
|
2084
3081
|
if (existing) {
|
|
2085
|
-
if (effectiveTime(entry) <= effectiveTime(existing)) {
|
|
2086
|
-
// Older or same age: keep ours, but never lose feedback the other side collected.
|
|
2087
|
-
existing.hits = Math.max(existing.hits || 0, entry.hits || 0);
|
|
2088
|
-
existing.demerits = Math.max(existing.demerits || 0, entry.demerits || 0);
|
|
2089
|
-
skipped++;
|
|
2090
|
-
continue;
|
|
2091
|
-
}
|
|
2092
3082
|
this.dropEntry(existing);
|
|
2093
|
-
|
|
2094
|
-
entry.demerits = Math.max(existing.demerits || 0, entry.demerits || 0);
|
|
3083
|
+
this.absorbImportCounters(entry, existing);
|
|
2095
3084
|
updated++;
|
|
2096
3085
|
}
|
|
2097
3086
|
else {
|
|
@@ -2144,7 +3133,10 @@ class TheBrainV2 {
|
|
|
2144
3133
|
let downvoted = 0;
|
|
2145
3134
|
let unscoped = 0;
|
|
2146
3135
|
let untracked = 0;
|
|
3136
|
+
let withClearPaths = 0;
|
|
2147
3137
|
let oldest = '';
|
|
3138
|
+
let totalTokensSaved = 0;
|
|
3139
|
+
const savers = [];
|
|
2148
3140
|
const bySource = {};
|
|
2149
3141
|
const projects = new Set();
|
|
2150
3142
|
for (const entry of this.entries.values()) {
|
|
@@ -2158,17 +3150,30 @@ class TheBrainV2 {
|
|
|
2158
3150
|
projects.add(entry.projectId);
|
|
2159
3151
|
if (trackedArtifactCount(entry) === 0)
|
|
2160
3152
|
untracked++;
|
|
3153
|
+
else if (entry.filePaths !== undefined || entry.symbols !== undefined ||
|
|
3154
|
+
(entry.claims || []).some((c) => c.filePaths !== undefined || c.symbols !== undefined))
|
|
3155
|
+
withClearPaths++;
|
|
3156
|
+
if (entry.tokensSaved) {
|
|
3157
|
+
totalTokensSaved += entry.tokensSaved;
|
|
3158
|
+
savers.push({ id: entry.id, query: entry.query.slice(0, 80), tokensSaved: entry.tokensSaved, hits: entry.hits });
|
|
3159
|
+
}
|
|
2161
3160
|
const src = entrySource(entry);
|
|
2162
3161
|
bySource[src] = (bySource[src] || 0) + 1;
|
|
2163
3162
|
if (entry.timestamp && (!oldest || entry.timestamp < oldest))
|
|
2164
3163
|
oldest = entry.timestamp;
|
|
2165
3164
|
}
|
|
3165
|
+
savers.sort((a, b) => b.tokensSaved - a.tokensSaved);
|
|
3166
|
+
base.totalTokensSaved = totalTokensSaved;
|
|
3167
|
+
if (savers.length > 0)
|
|
3168
|
+
base.topSavers = savers.slice(0, 5);
|
|
3169
|
+
base.integrity = this.health();
|
|
2166
3170
|
base.neverHit = neverHit;
|
|
2167
3171
|
base.downvoted = downvoted;
|
|
2168
3172
|
base.bySource = bySource;
|
|
2169
3173
|
base.projects = projects.size;
|
|
2170
3174
|
base.unscopedEntries = unscoped;
|
|
2171
3175
|
base.untrackedEntries = untracked;
|
|
3176
|
+
base.withClearPaths = withClearPaths;
|
|
2172
3177
|
base.tombstones = this.tombstones.size;
|
|
2173
3178
|
if (oldest)
|
|
2174
3179
|
base.oldestEntry = oldest;
|
|
@@ -2191,7 +3196,26 @@ class TheBrainV2 {
|
|
|
2191
3196
|
}
|
|
2192
3197
|
return base;
|
|
2193
3198
|
}
|
|
3199
|
+
/**
|
|
3200
|
+
* Integrity + rollback posture in one cheap call (no hashing, no disk reads beyond a
|
|
3201
|
+
* directory listing). `ok` is false while an unverified read, a backup fallback, or an
|
|
3202
|
+
* undecryptable envelope is the reason this process is serving what it serves.
|
|
3203
|
+
*/
|
|
3204
|
+
health() {
|
|
3205
|
+
this.syncIfChanged();
|
|
3206
|
+
return {
|
|
3207
|
+
ok: this.integrityIssue === null,
|
|
3208
|
+
lastIssue: this.integrityIssue,
|
|
3209
|
+
snapshots: listSnapshotNames().length,
|
|
3210
|
+
backups: fs_1.default.existsSync(ENTRIES_PREV_FILE),
|
|
3211
|
+
encrypted: (0, BrainEncryption_1.encryptionActive)(BRAIN_DIR).enabled,
|
|
3212
|
+
};
|
|
3213
|
+
}
|
|
2194
3214
|
clear() {
|
|
3215
|
+
// Even a deliberate wipe gets a restore point: "I didn't mean that clear" is a
|
|
3216
|
+
// support ticket, "I didn't mean that clear and there is no snapshot" is data loss.
|
|
3217
|
+
if (this.entries.size > 0)
|
|
3218
|
+
this.snapshotCorpus('clear');
|
|
2195
3219
|
this.entries.clear();
|
|
2196
3220
|
this.invertedIndex.clear();
|
|
2197
3221
|
this.bloom = new BloomFilter();
|
|
@@ -2201,9 +3225,14 @@ class TheBrainV2 {
|
|
|
2201
3225
|
this.tombstones.clear();
|
|
2202
3226
|
this.diskStamp = null;
|
|
2203
3227
|
this.sidecar = null;
|
|
3228
|
+
this.integrityIssue = null;
|
|
2204
3229
|
try {
|
|
2205
3230
|
if (fs_1.default.existsSync(ENTRIES_FILE))
|
|
2206
3231
|
fs_1.default.unlinkSync(ENTRIES_FILE);
|
|
3232
|
+
if (fs_1.default.existsSync(ENTRIES_SHA_FILE))
|
|
3233
|
+
fs_1.default.unlinkSync(ENTRIES_SHA_FILE);
|
|
3234
|
+
if (fs_1.default.existsSync(ENTRIES_PREV_FILE))
|
|
3235
|
+
fs_1.default.unlinkSync(ENTRIES_PREV_FILE);
|
|
2207
3236
|
if (fs_1.default.existsSync(INDEX_FILE))
|
|
2208
3237
|
fs_1.default.unlinkSync(INDEX_FILE);
|
|
2209
3238
|
if (fs_1.default.existsSync(META_FILE))
|
|
@@ -2212,6 +3241,8 @@ class TheBrainV2 {
|
|
|
2212
3241
|
fs_1.default.unlinkSync(LOCK_FILE);
|
|
2213
3242
|
if (fs_1.default.existsSync(EMBEDDINGS_FILE))
|
|
2214
3243
|
fs_1.default.unlinkSync(EMBEDDINGS_FILE);
|
|
3244
|
+
// Snapshots deliberately survive clear(): they are the rollback history, and a wipe
|
|
3245
|
+
// that also destroys every restore point is not a wipe, it is data loss.
|
|
2215
3246
|
}
|
|
2216
3247
|
catch { /* ignore */ }
|
|
2217
3248
|
}
|