@remnic/plugin-openclaw 1.0.2 → 1.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +261 -0
- package/dist/{calibration-WZXRJMVP.js → calibration-3JHF25QT.js} +2 -2
- package/dist/{causal-chain-LA3IQNL6.js → causal-chain-OKDZSDEB.js} +3 -3
- package/dist/{causal-consolidation-YZLBOC7J.js → causal-consolidation-62IFBWHC.js} +22 -7
- package/dist/{causal-retrieval-ITNQBUQM.js → causal-retrieval-5UPIKZ4I.js} +4 -4
- package/dist/{causal-trajectory-graph-7Z5DD66L.js → causal-trajectory-graph-RQIT37DN.js} +1 -1
- package/dist/{chunk-BIBYVWVY.js → chunk-3SA5F4WT.js} +1 -1
- package/dist/{chunk-Y7JG2Q3V.js → chunk-5VTGFKKU.js} +2001 -275
- package/dist/{chunk-HSBPDYF4.js → chunk-BXTMZDRT.js} +2 -2
- package/dist/chunk-J2FCINY7.js +960 -0
- package/dist/{chunk-L46G4NGI.js → chunk-J7VGZNH4.js} +15 -4
- package/dist/chunk-NJG4HPAL.js +7099 -0
- package/dist/{chunk-BZ27H3BL.js → chunk-RMFPW4VK.js} +1 -1
- package/dist/{chunk-DMGIUDBO.js → chunk-UFU5GGGA.js} +8 -2
- package/dist/{chunk-JGEKL3WH.js → chunk-YHH3SXKD.js} +1 -1
- package/dist/{chunk-H3SKMYPU.js → chunk-Z7GRLVK3.js} +2 -2
- package/dist/{engine-GPDZXAXN.js → engine-BEV4BHEH.js} +3 -3
- package/dist/{fallback-llm-7UTWU27F.js → fallback-llm-HJRCHKSA.js} +2 -2
- package/dist/index.js +14018 -13908
- package/dist/{logger-NZE2OBVA.js → logger-TNOKCH7X.js} +1 -1
- package/dist/openai-53ZQ46RZ.js +44 -0
- package/dist/storage-HW6SRQCK.js +24 -0
- package/openclaw.plugin.json +807 -10
- package/package.json +8 -11
- package/dist/storage-AGYBIME4.js +0 -16
|
@@ -1,23 +1,30 @@
|
|
|
1
1
|
import {
|
|
2
2
|
log
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-UFU5GGGA.js";
|
|
4
4
|
|
|
5
5
|
// ../remnic-core/src/storage.ts
|
|
6
|
-
import { access, readdir, readFile as
|
|
6
|
+
import { access as access2, readdir, readFile as readFile3, stat as stat2, writeFile as writeFile3, mkdir as mkdir3, unlink as unlink2, rename, appendFile } from "fs/promises";
|
|
7
7
|
import { appendFileSync, mkdirSync, statSync } from "fs";
|
|
8
|
-
import { createHash } from "crypto";
|
|
9
|
-
import
|
|
8
|
+
import { createHash as createHash2 } from "crypto";
|
|
9
|
+
import path5 from "path";
|
|
10
10
|
|
|
11
11
|
// ../remnic-core/src/memory-cache.ts
|
|
12
12
|
var entityCacheByDir = /* @__PURE__ */ new Map();
|
|
13
|
-
function
|
|
13
|
+
function buildEntityCacheKey(baseDir, schemaKey = "") {
|
|
14
|
+
return `${baseDir}\0${schemaKey}`;
|
|
15
|
+
}
|
|
16
|
+
function getCachedEntities(baseDir, currentVersion, schemaKey = "") {
|
|
14
17
|
if (currentVersion === 0) return null;
|
|
15
|
-
const entry = entityCacheByDir.get(baseDir);
|
|
18
|
+
const entry = entityCacheByDir.get(buildEntityCacheKey(baseDir, schemaKey));
|
|
16
19
|
if (!entry || entry.version !== currentVersion) return null;
|
|
17
20
|
return entry.entities;
|
|
18
21
|
}
|
|
19
|
-
function setCachedEntities(baseDir, entities, version) {
|
|
20
|
-
entityCacheByDir.set(baseDir,
|
|
22
|
+
function setCachedEntities(baseDir, entities, version, schemaKey = "") {
|
|
23
|
+
entityCacheByDir.set(buildEntityCacheKey(baseDir, schemaKey), {
|
|
24
|
+
entities,
|
|
25
|
+
version,
|
|
26
|
+
loadedAt: Date.now()
|
|
27
|
+
});
|
|
21
28
|
}
|
|
22
29
|
var episodeMapByDir = /* @__PURE__ */ new Map();
|
|
23
30
|
var ruleMemoriesByDir = /* @__PURE__ */ new Map();
|
|
@@ -166,6 +173,474 @@ function sanitizeMemoryContent(text) {
|
|
|
166
173
|
};
|
|
167
174
|
}
|
|
168
175
|
|
|
176
|
+
// ../remnic-core/src/page-versioning.ts
|
|
177
|
+
import { createHash } from "crypto";
|
|
178
|
+
import path2 from "path";
|
|
179
|
+
import {
|
|
180
|
+
access,
|
|
181
|
+
mkdir as mkdir2,
|
|
182
|
+
readFile as readFile2,
|
|
183
|
+
writeFile as writeFile2,
|
|
184
|
+
unlink
|
|
185
|
+
} from "fs/promises";
|
|
186
|
+
var NOOP_LOGGER = {
|
|
187
|
+
debug: () => {
|
|
188
|
+
},
|
|
189
|
+
warn: () => {
|
|
190
|
+
}
|
|
191
|
+
};
|
|
192
|
+
var writeLocks = /* @__PURE__ */ new Map();
|
|
193
|
+
function withPageLock(pageKey, fn) {
|
|
194
|
+
const prev = writeLocks.get(pageKey) ?? Promise.resolve();
|
|
195
|
+
const next = prev.then(fn, fn);
|
|
196
|
+
writeLocks.set(pageKey, next.then(() => {
|
|
197
|
+
}, () => {
|
|
198
|
+
}));
|
|
199
|
+
return next;
|
|
200
|
+
}
|
|
201
|
+
function contentHash(content) {
|
|
202
|
+
return createHash("sha256").update(content, "utf-8").digest("hex");
|
|
203
|
+
}
|
|
204
|
+
function sidecarKey(pagePath) {
|
|
205
|
+
const withoutExt = pagePath.replace(/\.md$/i, "");
|
|
206
|
+
return withoutExt.replace(/[\\/]/g, "__");
|
|
207
|
+
}
|
|
208
|
+
function sidecarDir(memoryDir, sidecar, pagePath) {
|
|
209
|
+
return path2.join(memoryDir, sidecar, sidecarKey(pagePath));
|
|
210
|
+
}
|
|
211
|
+
function manifestPath(memoryDir, sidecar, pagePath) {
|
|
212
|
+
return path2.join(sidecarDir(memoryDir, sidecar, pagePath), "manifest.json");
|
|
213
|
+
}
|
|
214
|
+
async function readManifest(memoryDir, sidecar, pagePath) {
|
|
215
|
+
const mp = manifestPath(memoryDir, sidecar, pagePath);
|
|
216
|
+
try {
|
|
217
|
+
const raw = await readFile2(mp, "utf-8");
|
|
218
|
+
const parsed = JSON.parse(raw);
|
|
219
|
+
if (typeof parsed !== "object" || parsed === null) {
|
|
220
|
+
return { pagePath, versions: [], currentVersion: "0" };
|
|
221
|
+
}
|
|
222
|
+
const obj = parsed;
|
|
223
|
+
const versions = Array.isArray(obj.versions) ? obj.versions : [];
|
|
224
|
+
const currentVersion = typeof obj.currentVersion === "string" ? obj.currentVersion : "0";
|
|
225
|
+
return { pagePath: typeof obj.pagePath === "string" ? obj.pagePath : pagePath, versions, currentVersion };
|
|
226
|
+
} catch {
|
|
227
|
+
return { pagePath, versions: [], currentVersion: "0" };
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
async function writeManifest(memoryDir, sidecar, pagePath, history) {
|
|
231
|
+
const dir = sidecarDir(memoryDir, sidecar, pagePath);
|
|
232
|
+
await mkdir2(dir, { recursive: true });
|
|
233
|
+
const mp = manifestPath(memoryDir, sidecar, pagePath);
|
|
234
|
+
await writeFile2(mp, JSON.stringify(history, null, 2) + "\n", "utf-8");
|
|
235
|
+
}
|
|
236
|
+
async function createVersion(pagePath, content, trigger, config, log2 = NOOP_LOGGER, note, memoryDir) {
|
|
237
|
+
const { sidecarDir: sidecar, maxVersionsPerPage } = config;
|
|
238
|
+
const resolvedMemoryDir = memoryDir ?? resolveMemoryDir(pagePath);
|
|
239
|
+
const mPath = manifestPath(resolvedMemoryDir, sidecar, relPath(pagePath, resolvedMemoryDir));
|
|
240
|
+
return withPageLock(mPath, async () => {
|
|
241
|
+
const history = await readManifest(resolvedMemoryDir, sidecar, relPath(pagePath, resolvedMemoryDir));
|
|
242
|
+
const nextId = String(history.versions.length > 0 ? Math.max(...history.versions.map((v) => Number(v.versionId))) + 1 : 1);
|
|
243
|
+
const hash = contentHash(content);
|
|
244
|
+
const version = {
|
|
245
|
+
versionId: nextId,
|
|
246
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString(),
|
|
247
|
+
contentHash: hash,
|
|
248
|
+
sizeBytes: Buffer.byteLength(content, "utf-8"),
|
|
249
|
+
trigger,
|
|
250
|
+
...note !== void 0 ? { note } : {}
|
|
251
|
+
};
|
|
252
|
+
const dir = sidecarDir(resolvedMemoryDir, sidecar, relPath(pagePath, resolvedMemoryDir));
|
|
253
|
+
await mkdir2(dir, { recursive: true });
|
|
254
|
+
const ext = path2.extname(pagePath) || ".md";
|
|
255
|
+
const snapshotPath = path2.join(dir, `${nextId}${ext}`);
|
|
256
|
+
await writeFile2(snapshotPath, content, "utf-8");
|
|
257
|
+
history.versions.push(version);
|
|
258
|
+
history.currentVersion = nextId;
|
|
259
|
+
if (maxVersionsPerPage > 0 && history.versions.length > maxVersionsPerPage) {
|
|
260
|
+
const toRemove = history.versions.splice(0, history.versions.length - maxVersionsPerPage);
|
|
261
|
+
for (const old of toRemove) {
|
|
262
|
+
const oldPath = path2.join(dir, `${old.versionId}${ext}`);
|
|
263
|
+
try {
|
|
264
|
+
await unlink(oldPath);
|
|
265
|
+
} catch {
|
|
266
|
+
log2.debug(`page-versioning: could not remove old snapshot ${oldPath}`);
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
await writeManifest(resolvedMemoryDir, sidecar, relPath(pagePath, resolvedMemoryDir), history);
|
|
271
|
+
log2.debug(`page-versioning: created version ${nextId} for ${pagePath} (trigger=${trigger})`);
|
|
272
|
+
return version;
|
|
273
|
+
});
|
|
274
|
+
}
|
|
275
|
+
function resolveMemoryDir(pagePath) {
|
|
276
|
+
const knownSubdirs = /* @__PURE__ */ new Set([
|
|
277
|
+
"facts",
|
|
278
|
+
"corrections",
|
|
279
|
+
"entities",
|
|
280
|
+
"state",
|
|
281
|
+
"artifacts",
|
|
282
|
+
"questions",
|
|
283
|
+
"profiles"
|
|
284
|
+
]);
|
|
285
|
+
let dir = path2.dirname(pagePath);
|
|
286
|
+
for (let depth = 0; depth < 5; depth++) {
|
|
287
|
+
const base = path2.basename(dir);
|
|
288
|
+
if (knownSubdirs.has(base) || /^\d{4}-\d{2}-\d{2}$/.test(base)) {
|
|
289
|
+
dir = path2.dirname(dir);
|
|
290
|
+
} else {
|
|
291
|
+
break;
|
|
292
|
+
}
|
|
293
|
+
}
|
|
294
|
+
return dir;
|
|
295
|
+
}
|
|
296
|
+
function relPath(pagePath, memoryDir) {
|
|
297
|
+
return path2.relative(memoryDir, pagePath);
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
// ../remnic-core/src/entity-schema.ts
|
|
301
|
+
var DEFAULT_ENTITY_SCHEMAS = {
|
|
302
|
+
person: {
|
|
303
|
+
sections: [
|
|
304
|
+
{
|
|
305
|
+
key: "beliefs",
|
|
306
|
+
title: "Beliefs",
|
|
307
|
+
description: "",
|
|
308
|
+
aliases: ["belief", "beliefs", "believe", "believes"]
|
|
309
|
+
},
|
|
310
|
+
{
|
|
311
|
+
key: "communication_style",
|
|
312
|
+
title: "Communication Style",
|
|
313
|
+
description: "",
|
|
314
|
+
aliases: ["communication", "communication style", "communicate", "writes", "writing style"]
|
|
315
|
+
},
|
|
316
|
+
{
|
|
317
|
+
key: "building",
|
|
318
|
+
title: "Building / Working On",
|
|
319
|
+
description: "",
|
|
320
|
+
aliases: ["building", "working on", "work on", "projects"]
|
|
321
|
+
}
|
|
322
|
+
]
|
|
323
|
+
},
|
|
324
|
+
project: {
|
|
325
|
+
sections: [
|
|
326
|
+
{ key: "status", title: "Status", description: "" },
|
|
327
|
+
{
|
|
328
|
+
key: "building",
|
|
329
|
+
title: "Building / Working On",
|
|
330
|
+
description: "",
|
|
331
|
+
aliases: ["building", "working on", "work on"]
|
|
332
|
+
},
|
|
333
|
+
{ key: "risks", title: "Risks", description: "" },
|
|
334
|
+
{ key: "notes", title: "Notes", description: "" }
|
|
335
|
+
]
|
|
336
|
+
}
|
|
337
|
+
};
|
|
338
|
+
function normalizeEntityText(value) {
|
|
339
|
+
return value.toLowerCase().replace(/[^a-z0-9]+/g, " ").trim();
|
|
340
|
+
}
|
|
341
|
+
function toSnakeCase(value) {
|
|
342
|
+
return normalizeEntityText(value).replace(/\s+/g, "_");
|
|
343
|
+
}
|
|
344
|
+
function titleFromKey(key) {
|
|
345
|
+
return key.split("_").filter(Boolean).map((token) => token.charAt(0).toUpperCase() + token.slice(1)).join(" ");
|
|
346
|
+
}
|
|
347
|
+
function tokenizeNormalized(value) {
|
|
348
|
+
return normalizeEntityText(value).split(/\s+/).filter(Boolean);
|
|
349
|
+
}
|
|
350
|
+
function normalizeSectionDefinition(raw) {
|
|
351
|
+
if (!raw || typeof raw !== "object" || Array.isArray(raw)) return null;
|
|
352
|
+
const value = raw;
|
|
353
|
+
const keySource = typeof value.key === "string" ? value.key : typeof value.title === "string" ? value.title : "";
|
|
354
|
+
const titleSource = typeof value.title === "string" ? value.title : typeof value.key === "string" ? value.key : "";
|
|
355
|
+
const key = toSnakeCase(keySource);
|
|
356
|
+
const title = titleSource.trim() || titleFromKey(key);
|
|
357
|
+
if (!key || !title) return null;
|
|
358
|
+
const description = typeof value.description === "string" ? value.description : "";
|
|
359
|
+
const aliases = Array.isArray(value.aliases) ? value.aliases.filter((alias) => typeof alias === "string").map((alias) => alias.trim()).filter((alias) => alias.length > 0) : [];
|
|
360
|
+
return aliases.length > 0 ? { key, title, description, aliases } : { key, title, description };
|
|
361
|
+
}
|
|
362
|
+
function normalizeEntitySchemas(raw) {
|
|
363
|
+
if (!raw || typeof raw !== "object" || Array.isArray(raw)) return void 0;
|
|
364
|
+
const result = {};
|
|
365
|
+
for (const [entityType, schema] of Object.entries(raw)) {
|
|
366
|
+
if (!schema || typeof schema !== "object" || Array.isArray(schema)) continue;
|
|
367
|
+
const rawSections = schema.sections;
|
|
368
|
+
if (!Array.isArray(rawSections)) continue;
|
|
369
|
+
const sections = rawSections.map((section) => normalizeSectionDefinition(section)).filter((section) => section !== null);
|
|
370
|
+
if (sections.length === 0) continue;
|
|
371
|
+
result[toSnakeCase(entityType)] = { sections };
|
|
372
|
+
}
|
|
373
|
+
return Object.keys(result).length > 0 ? result : void 0;
|
|
374
|
+
}
|
|
375
|
+
function mergeEntitySchemaDefinitions(defaults, overrides) {
|
|
376
|
+
const overrideByKey = new Map(overrides.sections.map((section) => [section.key, section]));
|
|
377
|
+
const mergedSections = [];
|
|
378
|
+
const seen = /* @__PURE__ */ new Set();
|
|
379
|
+
for (const section of defaults.sections) {
|
|
380
|
+
const override = overrideByKey.get(section.key);
|
|
381
|
+
const mergedAliases = Array.from(/* @__PURE__ */ new Set([...section.aliases ?? [], ...override?.aliases ?? []]));
|
|
382
|
+
const nextSection = override ? {
|
|
383
|
+
...section,
|
|
384
|
+
...override,
|
|
385
|
+
...mergedAliases.length > 0 ? { aliases: mergedAliases } : {}
|
|
386
|
+
} : section;
|
|
387
|
+
mergedSections.push(nextSection);
|
|
388
|
+
seen.add(nextSection.key);
|
|
389
|
+
}
|
|
390
|
+
for (const section of overrides.sections) {
|
|
391
|
+
if (seen.has(section.key)) continue;
|
|
392
|
+
mergedSections.push(section);
|
|
393
|
+
seen.add(section.key);
|
|
394
|
+
}
|
|
395
|
+
return { sections: mergedSections };
|
|
396
|
+
}
|
|
397
|
+
function getEntitySchema(entityType, entitySchemas) {
|
|
398
|
+
const normalizedType = toSnakeCase(entityType);
|
|
399
|
+
const defaults = DEFAULT_ENTITY_SCHEMAS[normalizedType];
|
|
400
|
+
const overrides = entitySchemas?.[normalizedType];
|
|
401
|
+
if (!defaults) return overrides;
|
|
402
|
+
if (!overrides) return defaults;
|
|
403
|
+
return mergeEntitySchemaDefinitions(defaults, overrides);
|
|
404
|
+
}
|
|
405
|
+
function matchEntitySchemaSection(entityType, title, entitySchemas) {
|
|
406
|
+
const normalizedTitle = normalizeEntityText(title);
|
|
407
|
+
if (!normalizedTitle) return null;
|
|
408
|
+
const schema = getEntitySchema(entityType, entitySchemas);
|
|
409
|
+
if (!schema) return null;
|
|
410
|
+
for (const section of schema.sections) {
|
|
411
|
+
const aliases = [section.title, section.key, ...section.aliases ?? []];
|
|
412
|
+
if (aliases.some((alias) => normalizeEntityText(alias) === normalizedTitle)) {
|
|
413
|
+
return section;
|
|
414
|
+
}
|
|
415
|
+
}
|
|
416
|
+
return null;
|
|
417
|
+
}
|
|
418
|
+
function normalizeEntityStructuredSection(entityType, section, entitySchemas) {
|
|
419
|
+
const matchedSection = matchEntitySchemaSection(entityType, section.title, entitySchemas) ?? matchEntitySchemaSection(entityType, section.key, entitySchemas);
|
|
420
|
+
if (matchedSection) {
|
|
421
|
+
return {
|
|
422
|
+
key: matchedSection.key,
|
|
423
|
+
title: matchedSection.title
|
|
424
|
+
};
|
|
425
|
+
}
|
|
426
|
+
const key = toSnakeCase(section.key || section.title);
|
|
427
|
+
return {
|
|
428
|
+
key,
|
|
429
|
+
title: section.title.trim() || titleFromKey(key)
|
|
430
|
+
};
|
|
431
|
+
}
|
|
432
|
+
function queryMentionsAlias(query, alias) {
|
|
433
|
+
const queryTokens = tokenizeNormalized(query);
|
|
434
|
+
const aliasTokens = tokenizeNormalized(alias);
|
|
435
|
+
if (queryTokens.length === 0 || aliasTokens.length === 0) return false;
|
|
436
|
+
if (aliasTokens.length > queryTokens.length) return false;
|
|
437
|
+
for (let index = 0; index <= queryTokens.length - aliasTokens.length; index += 1) {
|
|
438
|
+
let matched = true;
|
|
439
|
+
for (let offset = 0; offset < aliasTokens.length; offset += 1) {
|
|
440
|
+
if (queryTokens[index + offset] !== aliasTokens[offset]) {
|
|
441
|
+
matched = false;
|
|
442
|
+
break;
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
if (matched) return true;
|
|
446
|
+
}
|
|
447
|
+
return false;
|
|
448
|
+
}
|
|
449
|
+
function resolveRequestedEntitySectionKeys(query, entityType, availableSections, entitySchemas) {
|
|
450
|
+
if (availableSections.length === 0) return [];
|
|
451
|
+
const availableKeys = new Set(availableSections.map((section) => toSnakeCase(section.key)));
|
|
452
|
+
const schema = getEntitySchema(entityType, entitySchemas);
|
|
453
|
+
if (!schema) return [];
|
|
454
|
+
const matches = [];
|
|
455
|
+
for (const section of schema.sections) {
|
|
456
|
+
const key = toSnakeCase(section.key);
|
|
457
|
+
if (!availableKeys.has(key)) continue;
|
|
458
|
+
const aliases = [section.title, section.key, ...section.aliases ?? []];
|
|
459
|
+
if (aliases.some((alias) => queryMentionsAlias(query, alias))) {
|
|
460
|
+
matches.push(key);
|
|
461
|
+
}
|
|
462
|
+
}
|
|
463
|
+
return matches;
|
|
464
|
+
}
|
|
465
|
+
function sortStructuredSectionsBySchema(entityType, sections, entitySchemas) {
|
|
466
|
+
const schema = getEntitySchema(entityType, entitySchemas);
|
|
467
|
+
if (!schema || sections.length <= 1) return sections;
|
|
468
|
+
const order = new Map(schema.sections.map((section, index) => [toSnakeCase(section.key), index]));
|
|
469
|
+
return [...sections].sort((left, right) => {
|
|
470
|
+
const leftRank = order.get(toSnakeCase(left.key)) ?? Number.MAX_SAFE_INTEGER;
|
|
471
|
+
const rightRank = order.get(toSnakeCase(right.key)) ?? Number.MAX_SAFE_INTEGER;
|
|
472
|
+
if (leftRank !== rightRank) return leftRank - rightRank;
|
|
473
|
+
return left.title.localeCompare(right.title);
|
|
474
|
+
});
|
|
475
|
+
}
|
|
476
|
+
|
|
477
|
+
// ../remnic-core/src/source-attribution.ts
|
|
478
|
+
var DEFAULT_CITATION_FORMAT = "[Source: agent={agent}, session={sessionId}, ts={ts}]";
|
|
479
|
+
var CITATION_UNKNOWN = "unknown";
|
|
480
|
+
function defaultCitationMatcher() {
|
|
481
|
+
return /\[Source:\s*([^\]\n]+?)\]/gi;
|
|
482
|
+
}
|
|
483
|
+
function deriveSessionId(session) {
|
|
484
|
+
if (!session) return void 0;
|
|
485
|
+
const trimmed = session.trim();
|
|
486
|
+
if (trimmed.length === 0) return void 0;
|
|
487
|
+
const parts = trimmed.split(":").filter((p) => p.length > 0);
|
|
488
|
+
if (parts.length === 0) return trimmed;
|
|
489
|
+
return parts[parts.length - 1];
|
|
490
|
+
}
|
|
491
|
+
function formatCitation(ctx, template = DEFAULT_CITATION_FORMAT) {
|
|
492
|
+
const session = ctx.session ?? "";
|
|
493
|
+
const sessionId = ctx.sessionId ?? deriveSessionId(session) ?? CITATION_UNKNOWN;
|
|
494
|
+
const ts = ctx.ts ?? CITATION_UNKNOWN;
|
|
495
|
+
const agent = ctx.agent && ctx.agent.trim().length > 0 ? ctx.agent : CITATION_UNKNOWN;
|
|
496
|
+
const date = ts && ts !== CITATION_UNKNOWN ? ts.slice(0, 10) : CITATION_UNKNOWN;
|
|
497
|
+
const sessionForTemplate = session.trim().length > 0 ? session : CITATION_UNKNOWN;
|
|
498
|
+
const values = {
|
|
499
|
+
agent,
|
|
500
|
+
session: sessionForTemplate,
|
|
501
|
+
sessionId,
|
|
502
|
+
ts,
|
|
503
|
+
date
|
|
504
|
+
};
|
|
505
|
+
return template.replace(/\{([a-zA-Z_][\w]*)\}/g, (match, name) => {
|
|
506
|
+
return Object.prototype.hasOwnProperty.call(values, name) ? values[name] : match;
|
|
507
|
+
});
|
|
508
|
+
}
|
|
509
|
+
function hasCitation(text) {
|
|
510
|
+
if (typeof text !== "string" || text.length === 0) return false;
|
|
511
|
+
return defaultCitationMatcher().test(text);
|
|
512
|
+
}
|
|
513
|
+
function escapeRegExp(s) {
|
|
514
|
+
return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
515
|
+
}
|
|
516
|
+
function escapeRegExpCharClass(ch) {
|
|
517
|
+
if (ch === "]") return "\\]";
|
|
518
|
+
if (ch === "\\") return "\\\\";
|
|
519
|
+
if (ch === "^") return "\\^";
|
|
520
|
+
if (ch === "-") return "\\-";
|
|
521
|
+
return escapeRegExp(ch);
|
|
522
|
+
}
|
|
523
|
+
function buildTokenPattern(nonWordSepChars) {
|
|
524
|
+
const base = "\\n\\s";
|
|
525
|
+
if (nonWordSepChars.size === 0) {
|
|
526
|
+
return `[^\\n]+?`;
|
|
527
|
+
}
|
|
528
|
+
const escaped = [...nonWordSepChars].map(escapeRegExpCharClass).join("");
|
|
529
|
+
return `[^${base}${escaped}]+?`;
|
|
530
|
+
}
|
|
531
|
+
var PLACEHOLDER_REGEX = /\{[a-zA-Z_][\w]*\}/g;
|
|
532
|
+
function templateMatcher(template) {
|
|
533
|
+
const parts = template.split(PLACEHOLDER_REGEX);
|
|
534
|
+
if (parts.length <= 1) return null;
|
|
535
|
+
const prefix = parts[0] ?? "";
|
|
536
|
+
const suffix = parts[parts.length - 1] ?? "";
|
|
537
|
+
if (prefix.length > 0 || suffix.length > 0) {
|
|
538
|
+
const escapedPrefix = escapeRegExp(prefix);
|
|
539
|
+
const escapedSuffix = escapeRegExp(suffix);
|
|
540
|
+
const innerParts = parts.slice(1, -1);
|
|
541
|
+
const nonWordSepChars = /* @__PURE__ */ new Set();
|
|
542
|
+
for (const sep of innerParts) {
|
|
543
|
+
for (const ch of sep) {
|
|
544
|
+
if (!/\w/.test(ch)) {
|
|
545
|
+
nonWordSepChars.add(ch);
|
|
546
|
+
}
|
|
547
|
+
}
|
|
548
|
+
}
|
|
549
|
+
const interToken = buildTokenPattern(nonWordSepChars);
|
|
550
|
+
const lastToken = buildTokenPattern(/* @__PURE__ */ new Set());
|
|
551
|
+
const middle = innerParts.length === 0 ? lastToken : interToken + innerParts.slice(0, -1).map((sep) => escapeRegExp(sep) + interToken).join("") + escapeRegExp(innerParts[innerParts.length - 1]) + lastToken;
|
|
552
|
+
const pattern = escapedPrefix + middle + escapedSuffix;
|
|
553
|
+
return new RegExp(pattern, "i");
|
|
554
|
+
}
|
|
555
|
+
const middleLiterals = parts.slice(1, -1);
|
|
556
|
+
const hasNonEmptyMiddle = middleLiterals.some((p) => p.length > 0);
|
|
557
|
+
if (!hasNonEmptyMiddle) {
|
|
558
|
+
return null;
|
|
559
|
+
}
|
|
560
|
+
const idToken = "[\\w.:-]+";
|
|
561
|
+
const body = idToken + middleLiterals.map((lit) => escapeRegExp(lit) + idToken).join("");
|
|
562
|
+
const separatorText = middleLiterals.join("");
|
|
563
|
+
if (/\s/.test(separatorText)) {
|
|
564
|
+
const opener = "[\\[\\(\\<]";
|
|
565
|
+
const closer = "[\\]\\)\\>]";
|
|
566
|
+
return new RegExp(opener + body + closer, "i");
|
|
567
|
+
}
|
|
568
|
+
const leadAnchor = "(?:(?<=[\\[\\(\\<])|(?<!\\S))";
|
|
569
|
+
const trailAnchor = "(?:(?=[\\]\\)\\>])|(?=\\s*$))";
|
|
570
|
+
return new RegExp(leadAnchor + body + trailAnchor, "i");
|
|
571
|
+
}
|
|
572
|
+
function hasCitationForTemplate(text, template) {
|
|
573
|
+
if (typeof text !== "string" || text.length === 0) return false;
|
|
574
|
+
if (hasCitation(text)) return true;
|
|
575
|
+
if (template === DEFAULT_CITATION_FORMAT) return false;
|
|
576
|
+
if (!PLACEHOLDER_REGEX.test(template)) {
|
|
577
|
+
PLACEHOLDER_REGEX.lastIndex = 0;
|
|
578
|
+
return text.includes(template);
|
|
579
|
+
}
|
|
580
|
+
PLACEHOLDER_REGEX.lastIndex = 0;
|
|
581
|
+
const matcher = templateMatcher(template);
|
|
582
|
+
if (!matcher) {
|
|
583
|
+
return false;
|
|
584
|
+
}
|
|
585
|
+
return matcher.test(text);
|
|
586
|
+
}
|
|
587
|
+
function attachCitation(text, ctx, template = DEFAULT_CITATION_FORMAT) {
|
|
588
|
+
if (typeof text !== "string") return text;
|
|
589
|
+
if (hasCitationForTemplate(text, template)) return text;
|
|
590
|
+
const trimmedEnd = text.replace(/\s+$/u, "");
|
|
591
|
+
if (trimmedEnd.length === 0) return text;
|
|
592
|
+
const citation = formatCitation(ctx, template);
|
|
593
|
+
const trailing = text.slice(trimmedEnd.length);
|
|
594
|
+
return `${trimmedEnd} ${citation}${trailing}`;
|
|
595
|
+
}
|
|
596
|
+
function stripCitation(text) {
|
|
597
|
+
if (typeof text !== "string" || text.length === 0) return text;
|
|
598
|
+
if (!hasCitation(text)) return text;
|
|
599
|
+
const matcher = defaultCitationMatcher();
|
|
600
|
+
let result = "";
|
|
601
|
+
let lastIndex = 0;
|
|
602
|
+
let match;
|
|
603
|
+
while ((match = matcher.exec(text)) !== null) {
|
|
604
|
+
const before = text.slice(lastIndex, match.index).replace(/[ \t]+$/, "");
|
|
605
|
+
result += before;
|
|
606
|
+
lastIndex = match.index + match[0].length;
|
|
607
|
+
}
|
|
608
|
+
const after = text.slice(lastIndex).replace(/^[ \t]+/, "");
|
|
609
|
+
if (after.length > 0) {
|
|
610
|
+
if (result.length > 0) result += " ";
|
|
611
|
+
result += after;
|
|
612
|
+
}
|
|
613
|
+
return result.trimEnd();
|
|
614
|
+
}
|
|
615
|
+
function stripCitationForTemplate(text, template) {
|
|
616
|
+
if (typeof text !== "string" || text.length === 0) return text;
|
|
617
|
+
if (hasCitation(text)) return stripCitation(text);
|
|
618
|
+
if (!hasCitationForTemplate(text, template)) return text;
|
|
619
|
+
const matcher = templateMatcher(template);
|
|
620
|
+
if (!matcher) return stripCitation(text);
|
|
621
|
+
const globalMatcher = new RegExp(
|
|
622
|
+
matcher.source,
|
|
623
|
+
matcher.flags.includes("g") ? matcher.flags : matcher.flags + "g"
|
|
624
|
+
);
|
|
625
|
+
let result = "";
|
|
626
|
+
let lastIndex = 0;
|
|
627
|
+
let match;
|
|
628
|
+
while ((match = globalMatcher.exec(text)) !== null) {
|
|
629
|
+
const before = text.slice(lastIndex, match.index).replace(/[ \t]+$/, "");
|
|
630
|
+
result += before;
|
|
631
|
+
lastIndex = match.index + match[0].length;
|
|
632
|
+
if (match[0].length === 0) {
|
|
633
|
+
globalMatcher.lastIndex++;
|
|
634
|
+
}
|
|
635
|
+
}
|
|
636
|
+
const after = text.slice(lastIndex).replace(/^[ \t]+/, "");
|
|
637
|
+
if (after.length > 0) {
|
|
638
|
+
if (result.length > 0) result += " ";
|
|
639
|
+
result += after;
|
|
640
|
+
}
|
|
641
|
+
return result.trimEnd();
|
|
642
|
+
}
|
|
643
|
+
|
|
169
644
|
// ../remnic-core/src/types.ts
|
|
170
645
|
function confidenceTier(score) {
|
|
171
646
|
if (score >= 0.95) return "explicit";
|
|
@@ -176,7 +651,7 @@ function confidenceTier(score) {
|
|
|
176
651
|
var SPECULATIVE_TTL_DAYS = 30;
|
|
177
652
|
|
|
178
653
|
// ../remnic-core/src/memory-projection-store.ts
|
|
179
|
-
import
|
|
654
|
+
import path3 from "path";
|
|
180
655
|
import { readFileSync } from "fs";
|
|
181
656
|
|
|
182
657
|
// ../remnic-core/src/runtime/better-sqlite.ts
|
|
@@ -209,7 +684,7 @@ function openBetterSqlite3(file, options) {
|
|
|
209
684
|
// ../remnic-core/src/memory-projection-store.ts
|
|
210
685
|
var MEMORY_PROJECTION_SCHEMA_VERSION = 2;
|
|
211
686
|
function getMemoryProjectionPath(memoryDir) {
|
|
212
|
-
return
|
|
687
|
+
return path3.join(memoryDir, "state", "memory-projection.sqlite");
|
|
213
688
|
}
|
|
214
689
|
function listTableColumns(db, tableName) {
|
|
215
690
|
try {
|
|
@@ -441,7 +916,7 @@ function parseCurrentRow(memoryDir, row) {
|
|
|
441
916
|
category: row.category,
|
|
442
917
|
status: row.status,
|
|
443
918
|
lifecycleState: typeof row.lifecycle_state === "string" ? row.lifecycle_state : void 0,
|
|
444
|
-
path:
|
|
919
|
+
path: path3.join(memoryDir, row.path_rel),
|
|
445
920
|
pathRel: row.path_rel,
|
|
446
921
|
created: row.created_at,
|
|
447
922
|
updated: row.updated_at,
|
|
@@ -602,7 +1077,7 @@ function readProjectedMemoryBrowse(memoryDir, options) {
|
|
|
602
1077
|
return true;
|
|
603
1078
|
}
|
|
604
1079
|
try {
|
|
605
|
-
const filePath =
|
|
1080
|
+
const filePath = path3.join(memoryDir, row.path_rel);
|
|
606
1081
|
const content = readFileSync(filePath, "utf-8").toLowerCase();
|
|
607
1082
|
return content.includes(normalizedQuery);
|
|
608
1083
|
} catch {
|
|
@@ -616,7 +1091,7 @@ function readProjectedMemoryBrowse(memoryDir, options) {
|
|
|
616
1091
|
(row) => typeof row.memory_id === "string" && typeof row.path_rel === "string" && typeof row.category === "string" && typeof row.status === "string"
|
|
617
1092
|
).map((row) => ({
|
|
618
1093
|
id: row.memory_id,
|
|
619
|
-
path:
|
|
1094
|
+
path: path3.join(memoryDir, row.path_rel),
|
|
620
1095
|
category: row.category,
|
|
621
1096
|
status: row.status,
|
|
622
1097
|
created: typeof row.created_at === "string" ? row.created_at : void 0,
|
|
@@ -650,7 +1125,7 @@ function readProjectedMemoryBrowse(memoryDir, options) {
|
|
|
650
1125
|
(row) => typeof row.memory_id === "string" && typeof row.path_rel === "string" && typeof row.category === "string" && typeof row.status === "string"
|
|
651
1126
|
).map((row) => ({
|
|
652
1127
|
id: row.memory_id,
|
|
653
|
-
path:
|
|
1128
|
+
path: path3.join(memoryDir, row.path_rel),
|
|
654
1129
|
category: row.category,
|
|
655
1130
|
status: row.status,
|
|
656
1131
|
created: typeof row.created_at === "string" ? row.created_at : void 0,
|
|
@@ -866,7 +1341,7 @@ function readProjectedGovernanceRecord(memoryDir) {
|
|
|
866
1341
|
}
|
|
867
1342
|
|
|
868
1343
|
// ../remnic-core/src/memory-lifecycle-ledger-utils.ts
|
|
869
|
-
import
|
|
1344
|
+
import path4 from "path";
|
|
870
1345
|
var MEMORY_LIFECYCLE_RULE_VERSION = "memory-lifecycle-ledger.v1";
|
|
871
1346
|
var MEMORY_LIFECYCLE_EVENT_SORT_ORDER = {
|
|
872
1347
|
created: 0,
|
|
@@ -882,8 +1357,8 @@ var MEMORY_LIFECYCLE_EVENT_SORT_ORDER = {
|
|
|
882
1357
|
archived: 10
|
|
883
1358
|
};
|
|
884
1359
|
function toMemoryPathRel(baseDir, filePath) {
|
|
885
|
-
if (!baseDir) return filePath.split(
|
|
886
|
-
return
|
|
1360
|
+
if (!baseDir) return filePath.split(path4.sep).join("/");
|
|
1361
|
+
return path4.relative(baseDir, filePath).split(path4.sep).join("/");
|
|
887
1362
|
}
|
|
888
1363
|
function isArchivedMemoryPath(pathRel) {
|
|
889
1364
|
return pathRel === "archive" || pathRel.startsWith("archive/");
|
|
@@ -1315,6 +1790,7 @@ function serializeFrontmatter(fm) {
|
|
|
1315
1790
|
if (fm.structuredAttributes && Object.keys(fm.structuredAttributes).length > 0) {
|
|
1316
1791
|
lines.push(`structuredAttributes: ${JSON.stringify(fm.structuredAttributes)}`);
|
|
1317
1792
|
}
|
|
1793
|
+
if (fm.contentHash) lines.push(`contentHash: ${fm.contentHash}`);
|
|
1318
1794
|
lines.push("---");
|
|
1319
1795
|
return lines.join("\n");
|
|
1320
1796
|
}
|
|
@@ -1449,7 +1925,9 @@ function parseFrontmatter2(raw) {
|
|
|
1449
1925
|
// v8.0 Phase 2B: HiMem episode/note classification
|
|
1450
1926
|
memoryKind: fm.memoryKind || void 0,
|
|
1451
1927
|
// Structured attributes (JSON on a single line)
|
|
1452
|
-
structuredAttributes: parseStructuredAttributes(fm.structuredAttributes)
|
|
1928
|
+
structuredAttributes: parseStructuredAttributes(fm.structuredAttributes),
|
|
1929
|
+
// Raw-content dedup hash (format-agnostic archive/consolidation cleanup)
|
|
1930
|
+
contentHash: fm.contentHash || void 0
|
|
1453
1931
|
},
|
|
1454
1932
|
content
|
|
1455
1933
|
};
|
|
@@ -1472,14 +1950,47 @@ function parseFrontmatter2(raw) {
|
|
|
1472
1950
|
}
|
|
1473
1951
|
return result;
|
|
1474
1952
|
}
|
|
1475
|
-
function
|
|
1476
|
-
|
|
1953
|
+
function inferEntityTypeFromContent(content) {
|
|
1954
|
+
const typeMatch = content.match(/^\*\*Type:\*\*\s*([^\n]+)/m)?.[1]?.trim().toLowerCase();
|
|
1955
|
+
return typeMatch || void 0;
|
|
1956
|
+
}
|
|
1957
|
+
var KNOWN_ENTITY_FILENAME_PREFIXES = /* @__PURE__ */ new Set([
|
|
1958
|
+
"company",
|
|
1959
|
+
"other",
|
|
1960
|
+
"person",
|
|
1961
|
+
"place",
|
|
1962
|
+
"project",
|
|
1963
|
+
"tool",
|
|
1964
|
+
"topic"
|
|
1965
|
+
]);
|
|
1966
|
+
function inferEntityTypeFromFilename(pathRel) {
|
|
1967
|
+
const basename = path5.basename(pathRel, ".md").toLowerCase();
|
|
1968
|
+
const separator = basename.indexOf("-");
|
|
1969
|
+
if (separator <= 0) return void 0;
|
|
1970
|
+
const candidate = basename.slice(0, separator);
|
|
1971
|
+
return KNOWN_ENTITY_FILENAME_PREFIXES.has(candidate) ? candidate : void 0;
|
|
1972
|
+
}
|
|
1973
|
+
function normalizeFrontmatterForPath(frontmatter, pathRel, content = "") {
|
|
1974
|
+
const normalizedPath = pathRel.split(path5.sep).join("/");
|
|
1975
|
+
let normalizedFrontmatter = frontmatter;
|
|
1976
|
+
if (normalizedPath === "entities" || normalizedPath.startsWith("entities/") || normalizedPath.includes("/entities/")) {
|
|
1977
|
+
const basename = path5.basename(pathRel, ".md");
|
|
1978
|
+
const inferredType = inferEntityTypeFromContent(content) || inferEntityTypeFromFilename(pathRel) || "entity";
|
|
1979
|
+
const existingTags = Array.isArray(frontmatter.tags) ? frontmatter.tags : [];
|
|
1980
|
+
normalizedFrontmatter = {
|
|
1981
|
+
...normalizedFrontmatter,
|
|
1982
|
+
id: typeof normalizedFrontmatter.id === "string" && normalizedFrontmatter.id.trim().length > 0 ? normalizedFrontmatter.id : basename,
|
|
1983
|
+
category: "entity",
|
|
1984
|
+
tags: existingTags.includes(inferredType) ? existingTags : [...existingTags, inferredType]
|
|
1985
|
+
};
|
|
1986
|
+
}
|
|
1987
|
+
if (isArchivedMemoryPath(pathRel) && (!normalizedFrontmatter.status || normalizedFrontmatter.status === "active")) {
|
|
1477
1988
|
return {
|
|
1478
|
-
...
|
|
1989
|
+
...normalizedFrontmatter,
|
|
1479
1990
|
status: "archived"
|
|
1480
1991
|
};
|
|
1481
1992
|
}
|
|
1482
|
-
return
|
|
1993
|
+
return normalizedFrontmatter;
|
|
1483
1994
|
}
|
|
1484
1995
|
function inferCurrentStateStatus(frontmatter, pathRel, fallbackStatus) {
|
|
1485
1996
|
return inferMemoryStatus(frontmatter, pathRel, fallbackStatus);
|
|
@@ -1528,12 +2039,12 @@ var ContentHashIndex = class _ContentHashIndex {
|
|
|
1528
2039
|
dirty = false;
|
|
1529
2040
|
filePath;
|
|
1530
2041
|
constructor(stateDir) {
|
|
1531
|
-
this.filePath =
|
|
2042
|
+
this.filePath = path5.join(stateDir, "fact-hashes.txt");
|
|
1532
2043
|
}
|
|
1533
2044
|
/** Load existing hashes from disk. Safe to call multiple times. */
|
|
1534
2045
|
async load() {
|
|
1535
2046
|
try {
|
|
1536
|
-
const raw = await
|
|
2047
|
+
const raw = await readFile3(this.filePath, "utf-8");
|
|
1537
2048
|
for (const line of raw.split("\n")) {
|
|
1538
2049
|
const trimmed = line.trim();
|
|
1539
2050
|
if (trimmed.length > 0) {
|
|
@@ -1563,8 +2074,8 @@ var ContentHashIndex = class _ContentHashIndex {
|
|
|
1563
2074
|
/** Persist index to disk if changed. */
|
|
1564
2075
|
async save() {
|
|
1565
2076
|
if (!this.dirty) return;
|
|
1566
|
-
await
|
|
1567
|
-
await
|
|
2077
|
+
await mkdir3(path5.dirname(this.filePath), { recursive: true });
|
|
2078
|
+
await writeFile3(this.filePath, [...this.hashes].join("\n") + "\n", "utf-8");
|
|
1568
2079
|
this.dirty = false;
|
|
1569
2080
|
log.debug(`content-hash index: saved ${this.hashes.size} hashes`);
|
|
1570
2081
|
}
|
|
@@ -1575,6 +2086,33 @@ var ContentHashIndex = class _ContentHashIndex {
|
|
|
1575
2086
|
this.dirty = true;
|
|
1576
2087
|
}
|
|
1577
2088
|
}
|
|
2089
|
+
/**
|
|
2090
|
+
* Remove a pre-computed SHA-256 hash directly from the index without
|
|
2091
|
+
* re-hashing. Use this when the caller already holds the stored hash
|
|
2092
|
+
* (e.g. `memory.frontmatter.contentHash`) to avoid the double-hash bug
|
|
2093
|
+
* where `remove(hash)` would compute `hash(hash)` and never match the
|
|
2094
|
+
* entry.
|
|
2095
|
+
*/
|
|
2096
|
+
removeByHash(hash) {
|
|
2097
|
+
if (this.hashes.delete(hash)) {
|
|
2098
|
+
this.dirty = true;
|
|
2099
|
+
}
|
|
2100
|
+
}
|
|
2101
|
+
/**
|
|
2102
|
+
* Add a pre-computed SHA-256 hash directly to the index without re-hashing.
|
|
2103
|
+
* Use this when the caller already holds the stored hash
|
|
2104
|
+
* (e.g. `memory.frontmatter.contentHash`) so that the index records the raw
|
|
2105
|
+
* content hash rather than re-hashing the citation-annotated body.
|
|
2106
|
+
*
|
|
2107
|
+
* @internal Only called from `StorageManager.ensureFactHashIndexAuthoritative`.
|
|
2108
|
+
* Not part of the public API — prefer `add(content)` for external callers.
|
|
2109
|
+
*/
|
|
2110
|
+
addByHash(hash) {
|
|
2111
|
+
if (!this.hashes.has(hash)) {
|
|
2112
|
+
this.hashes.add(hash);
|
|
2113
|
+
this.dirty = true;
|
|
2114
|
+
}
|
|
2115
|
+
}
|
|
1578
2116
|
/** Normalize content and compute SHA-256 hash. */
|
|
1579
2117
|
static normalizeContent(content) {
|
|
1580
2118
|
return content.toLowerCase().replace(/[^a-z0-9\s]/g, " ").replace(/\s+/g, " ").trim();
|
|
@@ -1582,39 +2120,601 @@ var ContentHashIndex = class _ContentHashIndex {
|
|
|
1582
2120
|
/** Normalize content and compute SHA-256 hash. */
|
|
1583
2121
|
static computeHash(content) {
|
|
1584
2122
|
const normalized = _ContentHashIndex.normalizeContent(content);
|
|
1585
|
-
return
|
|
2123
|
+
return createHash2("sha256").update(normalized).digest("hex");
|
|
1586
2124
|
}
|
|
1587
2125
|
};
|
|
1588
|
-
function
|
|
1589
|
-
|
|
2126
|
+
function normalizeAttributePairs(pairs) {
|
|
2127
|
+
return Object.entries(pairs).map(([k, v]) => [k.trim().toLowerCase(), v.trim()]).sort(([a], [b]) => a.localeCompare(b)).map(([k, v]) => `${k}: ${v}`).join("; ");
|
|
2128
|
+
}
|
|
2129
|
+
function parseEntityFrontmatter(raw) {
|
|
2130
|
+
const match = raw.match(/^---\r?\n([\s\S]*?)\r?\n---\r?\n?([\s\S]*)$/);
|
|
2131
|
+
if (!match) {
|
|
2132
|
+
return { frontmatter: {}, body: raw };
|
|
2133
|
+
}
|
|
2134
|
+
const values = {};
|
|
2135
|
+
const extraLines = [];
|
|
2136
|
+
const recognizedKeys = /* @__PURE__ */ new Set([
|
|
2137
|
+
"created",
|
|
2138
|
+
"updated",
|
|
2139
|
+
"synthesis_updated_at",
|
|
2140
|
+
"synthesis_timeline_count",
|
|
2141
|
+
"synthesis_structured_fact_count",
|
|
2142
|
+
"synthesis_structured_fact_digest",
|
|
2143
|
+
"synthesis_version"
|
|
2144
|
+
]);
|
|
2145
|
+
for (const line of match[1].split(/\r?\n/)) {
|
|
2146
|
+
if (/^\s/.test(line)) {
|
|
2147
|
+
extraLines.push(line);
|
|
2148
|
+
continue;
|
|
2149
|
+
}
|
|
2150
|
+
const colonIdx = line.indexOf(":");
|
|
2151
|
+
if (colonIdx === -1) {
|
|
2152
|
+
extraLines.push(line);
|
|
2153
|
+
continue;
|
|
2154
|
+
}
|
|
2155
|
+
const key = line.slice(0, colonIdx).trim();
|
|
2156
|
+
if (!recognizedKeys.has(key)) {
|
|
2157
|
+
extraLines.push(line);
|
|
2158
|
+
continue;
|
|
2159
|
+
}
|
|
2160
|
+
const value = parseManagedFrontmatterValue(line.slice(colonIdx + 1));
|
|
2161
|
+
values[key] = value;
|
|
2162
|
+
}
|
|
2163
|
+
const synthesisTimelineCount = Number.parseInt(values.synthesis_timeline_count ?? "", 10);
|
|
2164
|
+
const synthesisStructuredFactCount = Number.parseInt(values.synthesis_structured_fact_count ?? "", 10);
|
|
2165
|
+
const synthesisVersion = Number.parseInt(values.synthesis_version ?? "", 10);
|
|
2166
|
+
return {
|
|
2167
|
+
frontmatter: {
|
|
2168
|
+
created: values.created || void 0,
|
|
2169
|
+
updated: values.updated || void 0,
|
|
2170
|
+
synthesisUpdatedAt: values.synthesis_updated_at || void 0,
|
|
2171
|
+
synthesisTimelineCount: Number.isFinite(synthesisTimelineCount) ? synthesisTimelineCount : void 0,
|
|
2172
|
+
synthesisStructuredFactCount: Number.isFinite(synthesisStructuredFactCount) ? synthesisStructuredFactCount : void 0,
|
|
2173
|
+
synthesisStructuredFactDigest: values.synthesis_structured_fact_digest || void 0,
|
|
2174
|
+
synthesisVersion: Number.isFinite(synthesisVersion) ? synthesisVersion : void 0,
|
|
2175
|
+
extraLines
|
|
2176
|
+
},
|
|
2177
|
+
body: match[2]
|
|
2178
|
+
};
|
|
2179
|
+
}
|
|
2180
|
+
function parseManagedFrontmatterValue(rawValue) {
|
|
2181
|
+
const trimmed = rawValue.trim();
|
|
2182
|
+
if (!trimmed) return "";
|
|
2183
|
+
const openingQuote = trimmed[0];
|
|
2184
|
+
if (openingQuote === '"' || openingQuote === "'") {
|
|
2185
|
+
let escaped = false;
|
|
2186
|
+
for (let index = 1; index < trimmed.length; index += 1) {
|
|
2187
|
+
const char = trimmed[index];
|
|
2188
|
+
if (openingQuote === '"' && !escaped && char === "\\") {
|
|
2189
|
+
escaped = true;
|
|
2190
|
+
continue;
|
|
2191
|
+
}
|
|
2192
|
+
if (!escaped && char === openingQuote) {
|
|
2193
|
+
return trimmed.slice(1, index);
|
|
2194
|
+
}
|
|
2195
|
+
escaped = false;
|
|
2196
|
+
}
|
|
2197
|
+
return trimmed.slice(1).replace(new RegExp(`${openingQuote}$`), "");
|
|
2198
|
+
}
|
|
2199
|
+
for (let index = 0; index < trimmed.length; index += 1) {
|
|
2200
|
+
if (trimmed[index] === "#" && (index === 0 || /\s/.test(trimmed[index - 1] ?? ""))) {
|
|
2201
|
+
return trimmed.slice(0, index).trimEnd();
|
|
2202
|
+
}
|
|
2203
|
+
}
|
|
2204
|
+
return trimmed;
|
|
2205
|
+
}
|
|
2206
|
+
function readEntitySectionText(lines, sectionNames, options = {}) {
|
|
2207
|
+
const normalizedSections = new Set(sectionNames.map((name) => name.toLowerCase()));
|
|
2208
|
+
let section = "";
|
|
2209
|
+
const sectionLines = [];
|
|
2210
|
+
for (const line of lines) {
|
|
2211
|
+
if (line.startsWith("## ")) {
|
|
2212
|
+
const nextSection = line.slice(3).trim().toLowerCase();
|
|
2213
|
+
if (section && !normalizedSections.has(nextSection)) break;
|
|
2214
|
+
section = normalizedSections.has(nextSection) ? nextSection : "";
|
|
2215
|
+
continue;
|
|
2216
|
+
}
|
|
2217
|
+
if (!section) continue;
|
|
2218
|
+
const trimmed = line.trim();
|
|
2219
|
+
if (!trimmed) {
|
|
2220
|
+
if (options.preserveBullets === true && sectionLines.length > 0 && sectionLines[sectionLines.length - 1] !== "") {
|
|
2221
|
+
sectionLines.push("");
|
|
2222
|
+
}
|
|
2223
|
+
continue;
|
|
2224
|
+
}
|
|
2225
|
+
if (options.skipTimelineBullets === true && trimmed.startsWith("- ") && isEntitySynthesisTimelinePromotionBullet(trimmed.slice(2))) {
|
|
2226
|
+
continue;
|
|
2227
|
+
}
|
|
2228
|
+
if (trimmed.startsWith("- ") && options.preserveBullets !== true) continue;
|
|
2229
|
+
sectionLines.push(options.preserveBullets === true ? line.trimEnd() : trimmed);
|
|
2230
|
+
}
|
|
2231
|
+
while (sectionLines[sectionLines.length - 1] === "") {
|
|
2232
|
+
sectionLines.pop();
|
|
2233
|
+
}
|
|
2234
|
+
if (sectionLines.length === 0) return void 0;
|
|
2235
|
+
return sectionLines.join(options.preserveBullets === true ? "\n" : " ");
|
|
2236
|
+
}
|
|
2237
|
+
function parseEntityTimelineBullet(bullet, fallbackTimestamp) {
|
|
2238
|
+
const trimmed = bullet.trim();
|
|
2239
|
+
if (!trimmed) return null;
|
|
2240
|
+
let rest = trimmed;
|
|
2241
|
+
const entry = {
|
|
2242
|
+
timestamp: trimmed.startsWith("[") ? "" : fallbackTimestamp,
|
|
2243
|
+
text: ""
|
|
2244
|
+
};
|
|
2245
|
+
const consumedMetadataSegments = [];
|
|
2246
|
+
let literalSingleSourceSegment;
|
|
2247
|
+
if (!trimmed.startsWith("[")) {
|
|
2248
|
+
entry.text = trimmed;
|
|
2249
|
+
return entry.text ? entry : null;
|
|
2250
|
+
}
|
|
2251
|
+
const firstEnd = trimmed.indexOf("]");
|
|
2252
|
+
if (firstEnd === -1) {
|
|
2253
|
+
entry.text = trimmed;
|
|
2254
|
+
return entry.text ? entry : null;
|
|
2255
|
+
}
|
|
2256
|
+
const firstToken = trimmed.slice(1, firstEnd).trim();
|
|
2257
|
+
const parsedTimestamp = Date.parse(firstToken);
|
|
2258
|
+
if (Number.isFinite(parsedTimestamp)) {
|
|
2259
|
+
entry.timestamp = firstToken || fallbackTimestamp;
|
|
2260
|
+
rest = trimmed.slice(firstEnd + 1).trimStart();
|
|
2261
|
+
}
|
|
2262
|
+
while (rest.startsWith("[")) {
|
|
2263
|
+
const end = findEntityTimelineTokenEnd(rest);
|
|
2264
|
+
if (end === -1) break;
|
|
2265
|
+
const rawSegment = rest.slice(0, end + 1);
|
|
2266
|
+
const token = rest.slice(1, end).trim();
|
|
2267
|
+
const equalsIdx = token.indexOf("=");
|
|
2268
|
+
if (equalsIdx === -1) {
|
|
2269
|
+
if (rest === trimmed) {
|
|
2270
|
+
entry.text = trimmed;
|
|
2271
|
+
return entry.text ? entry : null;
|
|
2272
|
+
}
|
|
2273
|
+
break;
|
|
2274
|
+
}
|
|
2275
|
+
const key = token.slice(0, equalsIdx).trim().toLowerCase();
|
|
2276
|
+
const value = unescapeEntityTimelineMetadataValue(token.slice(equalsIdx + 1).trim());
|
|
2277
|
+
if (!value) break;
|
|
2278
|
+
const nextRest = rest.slice(end + 1).trimStart();
|
|
2279
|
+
switch (key) {
|
|
2280
|
+
case "source_meta":
|
|
2281
|
+
entry.source = value;
|
|
2282
|
+
break;
|
|
2283
|
+
case "source":
|
|
2284
|
+
if (consumedMetadataSegments.length === 0 && !nextRest.startsWith("[") && nextRest.length > 0 && !isManagedEntityTimelineSource(value)) {
|
|
2285
|
+
literalSingleSourceSegment = rawSegment;
|
|
2286
|
+
rest = nextRest;
|
|
2287
|
+
break;
|
|
2288
|
+
}
|
|
2289
|
+
entry.source = value;
|
|
2290
|
+
break;
|
|
2291
|
+
case "session":
|
|
2292
|
+
case "sessionkey":
|
|
2293
|
+
entry.sessionKey = value;
|
|
2294
|
+
break;
|
|
2295
|
+
case "principal":
|
|
2296
|
+
entry.principal = value;
|
|
2297
|
+
break;
|
|
2298
|
+
default:
|
|
2299
|
+
entry.text = rest.trim();
|
|
2300
|
+
return entry.text ? entry : null;
|
|
2301
|
+
}
|
|
2302
|
+
if (literalSingleSourceSegment) break;
|
|
2303
|
+
consumedMetadataSegments.push(rawSegment);
|
|
2304
|
+
rest = nextRest;
|
|
2305
|
+
}
|
|
2306
|
+
if (literalSingleSourceSegment) {
|
|
2307
|
+
return {
|
|
2308
|
+
timestamp: entry.timestamp,
|
|
2309
|
+
text: `${literalSingleSourceSegment} ${rest}`.trim()
|
|
2310
|
+
};
|
|
2311
|
+
}
|
|
2312
|
+
entry.text = rest.trim();
|
|
2313
|
+
if (!entry.text) return null;
|
|
2314
|
+
return entry;
|
|
2315
|
+
}
|
|
2316
|
+
function isEntitySynthesisTimelinePromotionBullet(bullet) {
|
|
2317
|
+
const trimmed = bullet.trim();
|
|
2318
|
+
if (!trimmed.startsWith("[")) return false;
|
|
2319
|
+
const firstEnd = findEntityTimelineTokenEnd(trimmed);
|
|
2320
|
+
if (firstEnd === -1) return false;
|
|
2321
|
+
const firstToken = trimmed.slice(1, firstEnd).trim();
|
|
2322
|
+
return looksLikeEntityTimelineTimestamp(firstToken);
|
|
2323
|
+
}
|
|
2324
|
+
function looksLikeEntityTimelineTimestamp(token) {
|
|
2325
|
+
if (!/^\d{4}-\d{2}-\d{2}(?:[T\s].*)?$/.test(token)) return false;
|
|
2326
|
+
return Number.isFinite(Date.parse(token));
|
|
2327
|
+
}
|
|
2328
|
+
function isManagedEntityTimelineSource(source) {
|
|
2329
|
+
switch (source.trim().toLowerCase()) {
|
|
2330
|
+
case "artifact":
|
|
2331
|
+
case "chunking":
|
|
2332
|
+
case "cli-migrate":
|
|
2333
|
+
case "compounding-promotion":
|
|
2334
|
+
case "consolidation":
|
|
2335
|
+
case "contradiction-detection":
|
|
2336
|
+
case "entity_extraction":
|
|
2337
|
+
case "explicit":
|
|
2338
|
+
case "explicit-inline":
|
|
2339
|
+
case "explicit-inline-review":
|
|
2340
|
+
case "explicit-review":
|
|
2341
|
+
case "extraction":
|
|
2342
|
+
case "extraction-shared-promotion":
|
|
2343
|
+
case "manual":
|
|
2344
|
+
case "migration":
|
|
2345
|
+
case "migration-rechunk":
|
|
2346
|
+
case "proactive":
|
|
2347
|
+
case "replay":
|
|
2348
|
+
case "semantic-consolidation":
|
|
2349
|
+
case "unknown":
|
|
2350
|
+
return true;
|
|
2351
|
+
default:
|
|
2352
|
+
return false;
|
|
2353
|
+
}
|
|
2354
|
+
}
|
|
2355
|
+
function findEntityTimelineTokenEnd(input) {
|
|
2356
|
+
let escaped = false;
|
|
2357
|
+
for (let index = 0; index < input.length; index += 1) {
|
|
2358
|
+
const char = input[index];
|
|
2359
|
+
if (escaped) {
|
|
2360
|
+
escaped = false;
|
|
2361
|
+
continue;
|
|
2362
|
+
}
|
|
2363
|
+
if (char === "\\") {
|
|
2364
|
+
escaped = true;
|
|
2365
|
+
continue;
|
|
2366
|
+
}
|
|
2367
|
+
if (char === "]") return index;
|
|
2368
|
+
}
|
|
2369
|
+
return -1;
|
|
2370
|
+
}
|
|
2371
|
+
function escapeEntityTimelineMetadataValue(value) {
|
|
2372
|
+
let escaped = "";
|
|
2373
|
+
for (const char of value) {
|
|
2374
|
+
switch (char) {
|
|
2375
|
+
case "\\":
|
|
2376
|
+
escaped += "\\\\";
|
|
2377
|
+
break;
|
|
2378
|
+
case "]":
|
|
2379
|
+
escaped += "\\]";
|
|
2380
|
+
break;
|
|
2381
|
+
case "\n":
|
|
2382
|
+
escaped += "\\n";
|
|
2383
|
+
break;
|
|
2384
|
+
case "\r":
|
|
2385
|
+
escaped += "\\r";
|
|
2386
|
+
break;
|
|
2387
|
+
case " ":
|
|
2388
|
+
escaped += "\\t";
|
|
2389
|
+
break;
|
|
2390
|
+
default: {
|
|
2391
|
+
const codePoint = char.codePointAt(0) ?? 0;
|
|
2392
|
+
if (codePoint < 32) {
|
|
2393
|
+
escaped += `\\u${codePoint.toString(16).padStart(4, "0")}`;
|
|
2394
|
+
} else {
|
|
2395
|
+
escaped += char;
|
|
2396
|
+
}
|
|
2397
|
+
}
|
|
2398
|
+
}
|
|
2399
|
+
}
|
|
2400
|
+
return escaped;
|
|
2401
|
+
}
|
|
2402
|
+
function unescapeEntityTimelineMetadataValue(value) {
|
|
2403
|
+
if (!value.includes("\\")) return value;
|
|
2404
|
+
let result = "";
|
|
2405
|
+
for (let index = 0; index < value.length; index += 1) {
|
|
2406
|
+
const char = value[index];
|
|
2407
|
+
if (char !== "\\") {
|
|
2408
|
+
result += char;
|
|
2409
|
+
continue;
|
|
2410
|
+
}
|
|
2411
|
+
const next = value[index + 1];
|
|
2412
|
+
if (!next) {
|
|
2413
|
+
result += "\\";
|
|
2414
|
+
break;
|
|
2415
|
+
}
|
|
2416
|
+
switch (next) {
|
|
2417
|
+
case "n":
|
|
2418
|
+
result += "\n";
|
|
2419
|
+
index += 1;
|
|
2420
|
+
break;
|
|
2421
|
+
case "r":
|
|
2422
|
+
result += "\r";
|
|
2423
|
+
index += 1;
|
|
2424
|
+
break;
|
|
2425
|
+
case "t":
|
|
2426
|
+
result += " ";
|
|
2427
|
+
index += 1;
|
|
2428
|
+
break;
|
|
2429
|
+
case "u": {
|
|
2430
|
+
const hex = value.slice(index + 2, index + 6);
|
|
2431
|
+
if (/^[0-9a-fA-F]{4}$/.test(hex)) {
|
|
2432
|
+
result += String.fromCharCode(parseInt(hex, 16));
|
|
2433
|
+
index += 5;
|
|
2434
|
+
break;
|
|
2435
|
+
}
|
|
2436
|
+
result += "u";
|
|
2437
|
+
index += 1;
|
|
2438
|
+
break;
|
|
2439
|
+
}
|
|
2440
|
+
default:
|
|
2441
|
+
result += next;
|
|
2442
|
+
index += 1;
|
|
2443
|
+
break;
|
|
2444
|
+
}
|
|
2445
|
+
}
|
|
2446
|
+
return result;
|
|
2447
|
+
}
|
|
2448
|
+
function serializeEntityTimelineEntry(entry) {
|
|
2449
|
+
const tokens = [];
|
|
2450
|
+
if (entry.timestamp.trim().length > 0) {
|
|
2451
|
+
tokens.push(`[${entry.timestamp}]`);
|
|
2452
|
+
}
|
|
2453
|
+
if (entry.source) {
|
|
2454
|
+
const sourceKey = isManagedEntityTimelineSource(entry.source) ? "source" : "source_meta";
|
|
2455
|
+
tokens.push(`[${sourceKey}=${escapeEntityTimelineMetadataValue(entry.source)}]`);
|
|
2456
|
+
}
|
|
2457
|
+
if (entry.sessionKey) {
|
|
2458
|
+
tokens.push(`[session=${escapeEntityTimelineMetadataValue(entry.sessionKey)}]`);
|
|
2459
|
+
}
|
|
2460
|
+
if (entry.principal) {
|
|
2461
|
+
tokens.push(`[principal=${escapeEntityTimelineMetadataValue(entry.principal)}]`);
|
|
2462
|
+
}
|
|
2463
|
+
const serializedMetadata = tokens.length > 0 ? `${tokens.join(" ")} ` : "";
|
|
2464
|
+
return `- ${serializedMetadata}${entry.text}`.trimEnd();
|
|
2465
|
+
}
|
|
2466
|
+
function dedupeEntityTimelineFacts(timeline) {
|
|
2467
|
+
return [...new Set(
|
|
2468
|
+
timeline.map((entry) => entry.text.trim()).filter((entry) => entry.length > 0)
|
|
2469
|
+
)];
|
|
2470
|
+
}
|
|
2471
|
+
function normalizeEntitySectionFact(value) {
|
|
2472
|
+
return value.replace(/\s+/g, " ").trim();
|
|
2473
|
+
}
|
|
2474
|
+
function normalizeStructuredSectionFacts(facts) {
|
|
2475
|
+
return [...new Set(
|
|
2476
|
+
facts.map((fact) => normalizeEntitySectionFact(fact)).filter((fact) => fact.length > 0)
|
|
2477
|
+
)];
|
|
2478
|
+
}
|
|
2479
|
+
function collectStructuredSectionFacts(structuredSections) {
|
|
2480
|
+
const facts = [];
|
|
2481
|
+
for (const section of structuredSections) {
|
|
2482
|
+
for (const fact of section.facts) {
|
|
2483
|
+
const normalized = normalizeEntitySectionFact(fact);
|
|
2484
|
+
if (!normalized) continue;
|
|
2485
|
+
facts.push(normalized);
|
|
2486
|
+
}
|
|
2487
|
+
}
|
|
2488
|
+
return [...new Set(facts)];
|
|
2489
|
+
}
|
|
2490
|
+
function compileEntityFacts(timeline, structuredSections) {
|
|
2491
|
+
const facts = [];
|
|
2492
|
+
const seen = /* @__PURE__ */ new Set();
|
|
2493
|
+
for (const fact of dedupeEntityTimelineFacts(timeline)) {
|
|
2494
|
+
if (seen.has(fact)) continue;
|
|
2495
|
+
seen.add(fact);
|
|
2496
|
+
facts.push(fact);
|
|
2497
|
+
}
|
|
2498
|
+
for (const fact of collectStructuredSectionFacts(structuredSections)) {
|
|
2499
|
+
if (seen.has(fact)) continue;
|
|
2500
|
+
seen.add(fact);
|
|
2501
|
+
facts.push(fact);
|
|
2502
|
+
}
|
|
2503
|
+
return facts;
|
|
2504
|
+
}
|
|
2505
|
+
function parseEntityStructuredSectionFacts(lines) {
|
|
2506
|
+
const facts = [];
|
|
2507
|
+
let currentBlock = [];
|
|
2508
|
+
const flushCurrentBlock = () => {
|
|
2509
|
+
const normalized = normalizeEntitySectionFact(currentBlock.join(" "));
|
|
2510
|
+
if (normalized.length > 0) facts.push(normalized);
|
|
2511
|
+
currentBlock = [];
|
|
2512
|
+
};
|
|
2513
|
+
for (const rawLine of lines) {
|
|
2514
|
+
const line = rawLine.trim();
|
|
2515
|
+
if (!line) {
|
|
2516
|
+
flushCurrentBlock();
|
|
2517
|
+
continue;
|
|
2518
|
+
}
|
|
2519
|
+
if (line.startsWith("- ")) {
|
|
2520
|
+
flushCurrentBlock();
|
|
2521
|
+
currentBlock = [line.slice(2).trim()];
|
|
2522
|
+
continue;
|
|
2523
|
+
}
|
|
2524
|
+
currentBlock.push(line);
|
|
2525
|
+
}
|
|
2526
|
+
flushCurrentBlock();
|
|
2527
|
+
return [...new Set(facts)];
|
|
2528
|
+
}
|
|
2529
|
+
function looksLikeStructuredSectionFactList(lines) {
|
|
2530
|
+
const firstNonBlank = lines.find((line) => line.trim().length > 0)?.trim() ?? "";
|
|
2531
|
+
return firstNonBlank.startsWith("- ");
|
|
2532
|
+
}
|
|
2533
|
+
function partitionEntityStructuredSections(entityType, extraSections, entitySchemas) {
|
|
2534
|
+
const structuredSections = [];
|
|
2535
|
+
const remainingExtraSections = [];
|
|
2536
|
+
const structuredSectionIndex = /* @__PURE__ */ new Map();
|
|
2537
|
+
for (const section of extraSections) {
|
|
2538
|
+
const matchedSection = matchEntitySchemaSection(entityType, section.title, entitySchemas);
|
|
2539
|
+
if (!matchedSection && !looksLikeStructuredSectionFactList(section.lines)) {
|
|
2540
|
+
remainingExtraSections.push(section);
|
|
2541
|
+
continue;
|
|
2542
|
+
}
|
|
2543
|
+
const facts = parseEntityStructuredSectionFacts(section.lines);
|
|
2544
|
+
if (!matchedSection && facts.length === 0) {
|
|
2545
|
+
remainingExtraSections.push(section);
|
|
2546
|
+
continue;
|
|
2547
|
+
}
|
|
2548
|
+
const normalizedSection = matchedSection ? { key: matchedSection.key, title: matchedSection.title } : normalizeEntityStructuredSection(
|
|
2549
|
+
entityType,
|
|
2550
|
+
{ key: section.title, title: section.title },
|
|
2551
|
+
entitySchemas
|
|
2552
|
+
);
|
|
2553
|
+
if (facts.length === 0) {
|
|
2554
|
+
remainingExtraSections.push(section);
|
|
2555
|
+
continue;
|
|
2556
|
+
}
|
|
2557
|
+
const existing = structuredSectionIndex.get(normalizedSection.key);
|
|
2558
|
+
if (existing) {
|
|
2559
|
+
existing.facts = normalizeStructuredSectionFacts([...existing.facts, ...facts]);
|
|
2560
|
+
continue;
|
|
2561
|
+
}
|
|
2562
|
+
const structuredSection = {
|
|
2563
|
+
key: normalizedSection.key,
|
|
2564
|
+
title: normalizedSection.title,
|
|
2565
|
+
facts: normalizeStructuredSectionFacts(facts)
|
|
2566
|
+
};
|
|
2567
|
+
structuredSections.push(structuredSection);
|
|
2568
|
+
structuredSectionIndex.set(normalizedSection.key, structuredSection);
|
|
2569
|
+
}
|
|
2570
|
+
return {
|
|
2571
|
+
structuredSections,
|
|
2572
|
+
remainingExtraSections
|
|
2573
|
+
};
|
|
2574
|
+
}
|
|
2575
|
+
function latestEntityTimelineTimestamp(entity) {
|
|
2576
|
+
let latestRaw;
|
|
2577
|
+
for (const entry of entity.timeline) {
|
|
2578
|
+
const timestamp = entry.timestamp.trim();
|
|
2579
|
+
if (!timestamp) continue;
|
|
2580
|
+
if (!latestRaw || compareEntityTimestamps(timestamp, latestRaw) > 0) {
|
|
2581
|
+
latestRaw = timestamp;
|
|
2582
|
+
}
|
|
2583
|
+
}
|
|
2584
|
+
return latestRaw;
|
|
2585
|
+
}
|
|
2586
|
+
function compareEntityTimestamps(left, right) {
|
|
2587
|
+
const leftValue = left?.trim() ?? "";
|
|
2588
|
+
const rightValue = right?.trim() ?? "";
|
|
2589
|
+
if (!leftValue && !rightValue) return 0;
|
|
2590
|
+
if (!leftValue) return -1;
|
|
2591
|
+
if (!rightValue) return 1;
|
|
2592
|
+
const leftMs = Date.parse(leftValue);
|
|
2593
|
+
const rightMs = Date.parse(rightValue);
|
|
2594
|
+
const leftParsed = Number.isFinite(leftMs);
|
|
2595
|
+
const rightParsed = Number.isFinite(rightMs);
|
|
2596
|
+
if (leftParsed && rightParsed) {
|
|
2597
|
+
if (leftMs === rightMs) return 0;
|
|
2598
|
+
return leftMs > rightMs ? 1 : -1;
|
|
2599
|
+
}
|
|
2600
|
+
if (leftParsed) return 1;
|
|
2601
|
+
if (rightParsed) return -1;
|
|
2602
|
+
return leftValue.localeCompare(rightValue);
|
|
2603
|
+
}
|
|
2604
|
+
function countEntityStructuredFacts(entity) {
|
|
2605
|
+
return (entity.structuredSections ?? []).reduce((count, section) => count + section.facts.length, 0);
|
|
2606
|
+
}
|
|
2607
|
+
function fingerprintEntityStructuredFacts(entity) {
|
|
2608
|
+
const normalizedSections = (entity.structuredSections ?? []).map((section) => ({
|
|
2609
|
+
key: section.key.trim().toLowerCase(),
|
|
2610
|
+
title: section.title.replace(/\s+/g, " ").trim(),
|
|
2611
|
+
facts: normalizeStructuredSectionFacts(section.facts).slice().sort((left, right) => left.localeCompare(right))
|
|
2612
|
+
})).filter((section) => section.facts.length > 0).sort((left, right) => left.key.localeCompare(right.key) || left.title.localeCompare(right.title) || left.facts.join("\n").localeCompare(right.facts.join("\n")));
|
|
2613
|
+
if (normalizedSections.length === 0) return void 0;
|
|
2614
|
+
return createHash2("sha256").update(JSON.stringify(normalizedSections)).digest("hex");
|
|
2615
|
+
}
|
|
2616
|
+
function isEntitySynthesisStale(entity) {
|
|
2617
|
+
const structuredFactCount = countEntityStructuredFacts(entity);
|
|
2618
|
+
const structuredFactDigest = fingerprintEntityStructuredFacts(entity);
|
|
2619
|
+
const storedStructuredFactDigest = entity.synthesisStructuredFactDigest?.trim() || void 0;
|
|
2620
|
+
if (entity.timeline.length === 0 && structuredFactCount === 0) return false;
|
|
2621
|
+
if (!entity.synthesis?.trim()) return true;
|
|
2622
|
+
if (entity.synthesisTimelineCount === void 0) return true;
|
|
2623
|
+
if (structuredFactCount > 0 && entity.synthesisStructuredFactCount === void 0) return true;
|
|
2624
|
+
if (structuredFactCount > 0 && !storedStructuredFactDigest) return true;
|
|
2625
|
+
const latestTimelineTimestamp = latestEntityTimelineTimestamp(entity);
|
|
2626
|
+
if (!latestTimelineTimestamp) {
|
|
2627
|
+
return entity.timeline.length > entity.synthesisTimelineCount || structuredFactCount > (entity.synthesisStructuredFactCount ?? 0) || structuredFactDigest !== storedStructuredFactDigest;
|
|
2628
|
+
}
|
|
2629
|
+
if (!entity.synthesisUpdatedAt?.trim()) return true;
|
|
2630
|
+
const timelineFreshness = compareEntityTimestamps(latestTimelineTimestamp, entity.synthesisUpdatedAt);
|
|
2631
|
+
if (timelineFreshness > 0) return true;
|
|
2632
|
+
return entity.timeline.length > entity.synthesisTimelineCount || structuredFactCount > (entity.synthesisStructuredFactCount ?? 0) || structuredFactDigest !== storedStructuredFactDigest;
|
|
2633
|
+
}
|
|
2634
|
+
function parseEntityFile(content, entitySchemas) {
|
|
2635
|
+
const { frontmatter, body } = parseEntityFrontmatter(content);
|
|
2636
|
+
const lines = body.split("\n");
|
|
2637
|
+
const recognizedSections = /* @__PURE__ */ new Set([
|
|
2638
|
+
"facts",
|
|
2639
|
+
"timeline",
|
|
2640
|
+
"summary",
|
|
2641
|
+
"synthesis",
|
|
2642
|
+
"connected to",
|
|
2643
|
+
"activity",
|
|
2644
|
+
"aliases"
|
|
2645
|
+
]);
|
|
1590
2646
|
let name = "";
|
|
1591
2647
|
let type = "other";
|
|
2648
|
+
let created = frontmatter.created ?? "";
|
|
1592
2649
|
let updated = "";
|
|
1593
|
-
|
|
1594
|
-
const facts = [];
|
|
2650
|
+
const legacyFacts = [];
|
|
1595
2651
|
const relationships = [];
|
|
1596
2652
|
const activity = [];
|
|
1597
2653
|
const aliases = [];
|
|
2654
|
+
const timeline = [];
|
|
2655
|
+
const extraSections = [];
|
|
1598
2656
|
const headingLine = lines.find((l) => l.startsWith("# "));
|
|
1599
2657
|
if (headingLine) name = headingLine.slice(2).trim();
|
|
1600
2658
|
const typeLine = lines.find((l) => l.startsWith("**Type:**"));
|
|
1601
2659
|
if (typeLine) type = typeLine.replace("**Type:**", "").trim();
|
|
1602
2660
|
const updatedLine = lines.find((l) => l.startsWith("**Updated:**"));
|
|
1603
2661
|
if (updatedLine) updated = updatedLine.replace("**Updated:**", "").trim();
|
|
2662
|
+
if (!updated) updated = frontmatter.updated ?? frontmatter.created ?? "";
|
|
2663
|
+
if (!created) created = updated;
|
|
2664
|
+
const headingLineIndex = lines.findIndex((l) => l.startsWith("# "));
|
|
2665
|
+
const firstSectionIndex = lines.findIndex((l) => l.startsWith("## "));
|
|
2666
|
+
const preSectionStartIndex = headingLineIndex > -1 ? headingLineIndex + 1 : 0;
|
|
2667
|
+
const preSectionCandidates = firstSectionIndex > -1 ? lines.slice(preSectionStartIndex, firstSectionIndex) : lines.slice(preSectionStartIndex);
|
|
2668
|
+
const preSectionLines = preSectionCandidates.filter(
|
|
2669
|
+
(line) => !line.startsWith("**Type:**") && !line.startsWith("**Updated:**")
|
|
2670
|
+
);
|
|
2671
|
+
const normalizedPreSectionLines = [...preSectionLines];
|
|
2672
|
+
while (normalizedPreSectionLines[0] === "") {
|
|
2673
|
+
normalizedPreSectionLines.shift();
|
|
2674
|
+
}
|
|
2675
|
+
const preservedPreSectionLines = normalizedPreSectionLines.some((line) => line.trim().length > 0) ? normalizedPreSectionLines : [];
|
|
2676
|
+
const fallbackTimestamp = updated || created || "";
|
|
1604
2677
|
let section = "";
|
|
2678
|
+
let currentExtraSection = null;
|
|
1605
2679
|
for (const line of lines) {
|
|
1606
2680
|
if (line.startsWith("## ")) {
|
|
1607
|
-
|
|
2681
|
+
const heading = line.slice(3).trim();
|
|
2682
|
+
section = heading.toLowerCase();
|
|
2683
|
+
if (recognizedSections.has(section)) {
|
|
2684
|
+
currentExtraSection = null;
|
|
2685
|
+
} else {
|
|
2686
|
+
currentExtraSection = { title: heading, lines: [] };
|
|
2687
|
+
extraSections.push(currentExtraSection);
|
|
2688
|
+
}
|
|
1608
2689
|
continue;
|
|
1609
2690
|
}
|
|
2691
|
+
if (currentExtraSection) {
|
|
2692
|
+
currentExtraSection.lines.push(line);
|
|
2693
|
+
}
|
|
1610
2694
|
if (!line.startsWith("- ")) continue;
|
|
1611
2695
|
const bullet = line.slice(2).trim();
|
|
1612
2696
|
if (!bullet) continue;
|
|
1613
2697
|
switch (section) {
|
|
1614
2698
|
case "facts":
|
|
1615
|
-
|
|
2699
|
+
legacyFacts.push(bullet);
|
|
1616
2700
|
break;
|
|
2701
|
+
case "timeline": {
|
|
2702
|
+
const parsed = parseEntityTimelineBullet(
|
|
2703
|
+
bullet,
|
|
2704
|
+
fallbackTimestamp
|
|
2705
|
+
);
|
|
2706
|
+
if (parsed) timeline.push(parsed);
|
|
2707
|
+
break;
|
|
2708
|
+
}
|
|
1617
2709
|
case "summary":
|
|
2710
|
+
case "synthesis":
|
|
2711
|
+
if (isEntitySynthesisTimelinePromotionBullet(bullet)) {
|
|
2712
|
+
const parsed = parseEntityTimelineBullet(
|
|
2713
|
+
bullet,
|
|
2714
|
+
fallbackTimestamp
|
|
2715
|
+
);
|
|
2716
|
+
if (parsed) timeline.push(parsed);
|
|
2717
|
+
}
|
|
1618
2718
|
break;
|
|
1619
2719
|
case "connected to": {
|
|
1620
2720
|
const relMatch = bullet.match(/^\[\[([^\]]+)\]\]\s*[—–-]\s*(.+)$/);
|
|
@@ -1635,34 +2735,128 @@ function parseEntityFile(content) {
|
|
|
1635
2735
|
break;
|
|
1636
2736
|
}
|
|
1637
2737
|
}
|
|
1638
|
-
const
|
|
1639
|
-
|
|
1640
|
-
|
|
1641
|
-
|
|
1642
|
-
|
|
1643
|
-
|
|
1644
|
-
|
|
1645
|
-
|
|
1646
|
-
|
|
1647
|
-
|
|
1648
|
-
|
|
2738
|
+
const legacyFactTimelineEntries = legacyFacts.map((fact) => ({
|
|
2739
|
+
timestamp: fallbackTimestamp,
|
|
2740
|
+
text: fact,
|
|
2741
|
+
source: "migration"
|
|
2742
|
+
}));
|
|
2743
|
+
if (legacyFactTimelineEntries.length > 0) {
|
|
2744
|
+
const existingTimelineFacts = new Set(
|
|
2745
|
+
timeline.map((entry) => entry.text.trim()).filter((entry) => entry.length > 0)
|
|
2746
|
+
);
|
|
2747
|
+
for (const fact of legacyFactTimelineEntries) {
|
|
2748
|
+
const normalizedFact = fact.text.trim();
|
|
2749
|
+
if (!normalizedFact || existingTimelineFacts.has(normalizedFact)) continue;
|
|
2750
|
+
timeline.push(fact);
|
|
2751
|
+
existingTimelineFacts.add(normalizedFact);
|
|
2752
|
+
}
|
|
2753
|
+
}
|
|
2754
|
+
const synthesis = readEntitySectionText(lines, ["Synthesis"], { preserveBullets: true, skipTimelineBullets: true }) ?? readEntitySectionText(lines, ["Summary"], { preserveBullets: true, skipTimelineBullets: true });
|
|
2755
|
+
const synthesisUpdatedAt = frontmatter.synthesisUpdatedAt || void 0;
|
|
2756
|
+
const synthesisTimelineCount = frontmatter.synthesisTimelineCount;
|
|
2757
|
+
const synthesisStructuredFactCount = frontmatter.synthesisStructuredFactCount;
|
|
2758
|
+
const synthesisStructuredFactDigest = frontmatter.synthesisStructuredFactDigest;
|
|
2759
|
+
const { structuredSections, remainingExtraSections } = partitionEntityStructuredSections(
|
|
2760
|
+
type,
|
|
2761
|
+
extraSections,
|
|
2762
|
+
entitySchemas
|
|
2763
|
+
);
|
|
2764
|
+
const facts = compileEntityFacts(timeline, structuredSections);
|
|
2765
|
+
return {
|
|
2766
|
+
name,
|
|
2767
|
+
type,
|
|
2768
|
+
created,
|
|
2769
|
+
updated,
|
|
2770
|
+
extraFrontmatterLines: frontmatter.extraLines ?? [],
|
|
2771
|
+
preSectionLines: preservedPreSectionLines,
|
|
2772
|
+
facts,
|
|
2773
|
+
summary: synthesis,
|
|
2774
|
+
synthesis,
|
|
2775
|
+
synthesisUpdatedAt,
|
|
2776
|
+
synthesisTimelineCount,
|
|
2777
|
+
synthesisStructuredFactCount,
|
|
2778
|
+
synthesisStructuredFactDigest,
|
|
2779
|
+
synthesisVersion: frontmatter.synthesisVersion,
|
|
2780
|
+
timeline,
|
|
2781
|
+
structuredSections,
|
|
2782
|
+
relationships,
|
|
2783
|
+
activity,
|
|
2784
|
+
aliases,
|
|
2785
|
+
extraSections: remainingExtraSections
|
|
2786
|
+
};
|
|
1649
2787
|
}
|
|
1650
|
-
function serializeEntityFile(entity) {
|
|
2788
|
+
function serializeEntityFile(entity, entitySchemas) {
|
|
2789
|
+
const synthesis = entity.synthesis || entity.summary || "";
|
|
2790
|
+
const created = entity.created?.trim() || entity.updated || (/* @__PURE__ */ new Date()).toISOString();
|
|
2791
|
+
const updated = entity.updated || created;
|
|
2792
|
+
const timeline = entity.timeline;
|
|
2793
|
+
const structuredSections = sortStructuredSectionsBySchema(
|
|
2794
|
+
entity.type,
|
|
2795
|
+
(entity.structuredSections ?? []).map((section) => ({
|
|
2796
|
+
...section,
|
|
2797
|
+
facts: normalizeStructuredSectionFacts(section.facts)
|
|
2798
|
+
})).filter((section) => section.facts.length > 0),
|
|
2799
|
+
entitySchemas
|
|
2800
|
+
);
|
|
2801
|
+
const sectionFacts = new Set(collectStructuredSectionFacts(structuredSections));
|
|
2802
|
+
const legacyFacts = timeline.length === 0 ? [...new Set(
|
|
2803
|
+
entity.facts.map((fact) => normalizeEntitySectionFact(fact)).filter((fact) => fact.length > 0 && !sectionFacts.has(fact))
|
|
2804
|
+
)] : [];
|
|
2805
|
+
const synthesisUpdatedAt = entity.synthesisUpdatedAt?.trim() || "";
|
|
2806
|
+
const synthesisTimelineCount = entity.synthesisTimelineCount;
|
|
2807
|
+
const synthesisStructuredFactCount = entity.synthesisStructuredFactCount;
|
|
2808
|
+
const synthesisStructuredFactDigest = entity.synthesisStructuredFactDigest?.trim() || "";
|
|
2809
|
+
const synthesisVersion = entity.synthesisVersion ?? (synthesis ? 1 : 0);
|
|
1651
2810
|
const lines = [
|
|
2811
|
+
"---",
|
|
2812
|
+
`created: ${created}`,
|
|
2813
|
+
`updated: ${updated}`,
|
|
2814
|
+
`synthesis_updated_at: "${synthesisUpdatedAt}"`,
|
|
2815
|
+
...synthesisTimelineCount === void 0 ? [] : [`synthesis_timeline_count: ${synthesisTimelineCount}`],
|
|
2816
|
+
...synthesisStructuredFactCount === void 0 ? [] : [`synthesis_structured_fact_count: ${synthesisStructuredFactCount}`],
|
|
2817
|
+
...synthesisStructuredFactDigest ? [`synthesis_structured_fact_digest: "${synthesisStructuredFactDigest}"`] : [],
|
|
2818
|
+
`synthesis_version: ${synthesisVersion}`,
|
|
2819
|
+
...entity.extraFrontmatterLines ?? [],
|
|
2820
|
+
"---",
|
|
2821
|
+
"",
|
|
1652
2822
|
`# ${entity.name}`,
|
|
1653
2823
|
"",
|
|
1654
2824
|
`**Type:** ${entity.type}`,
|
|
1655
|
-
`**Updated:** ${
|
|
2825
|
+
`**Updated:** ${updated}`,
|
|
1656
2826
|
""
|
|
1657
2827
|
];
|
|
1658
|
-
if (entity.
|
|
1659
|
-
lines.push(
|
|
2828
|
+
if ((entity.preSectionLines ?? []).length > 0) {
|
|
2829
|
+
lines.push(...entity.preSectionLines ?? []);
|
|
2830
|
+
if (entity.preSectionLines?.[entity.preSectionLines.length - 1] !== "") {
|
|
2831
|
+
lines.push("");
|
|
2832
|
+
}
|
|
1660
2833
|
}
|
|
1661
|
-
lines.push("##
|
|
1662
|
-
|
|
1663
|
-
lines.push(
|
|
2834
|
+
lines.push("## Synthesis", "");
|
|
2835
|
+
if (synthesis) {
|
|
2836
|
+
lines.push(synthesis);
|
|
1664
2837
|
}
|
|
1665
2838
|
lines.push("");
|
|
2839
|
+
if (timeline.length > 0 || legacyFacts.length === 0) {
|
|
2840
|
+
lines.push("## Timeline", "");
|
|
2841
|
+
for (const entry of timeline) {
|
|
2842
|
+
lines.push(serializeEntityTimelineEntry(entry));
|
|
2843
|
+
}
|
|
2844
|
+
lines.push("");
|
|
2845
|
+
}
|
|
2846
|
+
if (legacyFacts.length > 0) {
|
|
2847
|
+
lines.push("## Facts", "");
|
|
2848
|
+
for (const fact of legacyFacts) {
|
|
2849
|
+
lines.push(`- ${fact}`);
|
|
2850
|
+
}
|
|
2851
|
+
lines.push("");
|
|
2852
|
+
}
|
|
2853
|
+
for (const section of structuredSections) {
|
|
2854
|
+
lines.push(`## ${section.title}`, "");
|
|
2855
|
+
for (const fact of section.facts) {
|
|
2856
|
+
lines.push(`- ${fact}`);
|
|
2857
|
+
}
|
|
2858
|
+
lines.push("");
|
|
2859
|
+
}
|
|
1666
2860
|
if (entity.relationships.length > 0) {
|
|
1667
2861
|
lines.push("## Connected to", "");
|
|
1668
2862
|
for (const rel of entity.relationships) {
|
|
@@ -1684,13 +2878,37 @@ function serializeEntityFile(entity) {
|
|
|
1684
2878
|
}
|
|
1685
2879
|
lines.push("");
|
|
1686
2880
|
}
|
|
2881
|
+
for (const section of entity.extraSections ?? []) {
|
|
2882
|
+
lines.push(`## ${section.title}`);
|
|
2883
|
+
lines.push(...section.lines);
|
|
2884
|
+
if (section.lines.length > 0 && section.lines[section.lines.length - 1] !== "") {
|
|
2885
|
+
lines.push("");
|
|
2886
|
+
}
|
|
2887
|
+
}
|
|
1687
2888
|
return lines.join("\n");
|
|
1688
2889
|
}
|
|
2890
|
+
function buildEntitySchemaCacheKey(entitySchemas) {
|
|
2891
|
+
if (!entitySchemas) return "";
|
|
2892
|
+
const normalized = Object.entries(entitySchemas).sort(([left], [right]) => left.localeCompare(right)).map(([entityType, schema]) => [
|
|
2893
|
+
entityType,
|
|
2894
|
+
{
|
|
2895
|
+
sections: schema.sections.map((section) => ({
|
|
2896
|
+
key: section.key,
|
|
2897
|
+
title: section.title,
|
|
2898
|
+
description: section.description,
|
|
2899
|
+
aliases: section.aliases ? [...section.aliases] : void 0
|
|
2900
|
+
}))
|
|
2901
|
+
}
|
|
2902
|
+
]);
|
|
2903
|
+
return JSON.stringify(normalized);
|
|
2904
|
+
}
|
|
1689
2905
|
var StorageManager = class _StorageManager {
|
|
1690
|
-
constructor(baseDir) {
|
|
2906
|
+
constructor(baseDir, entitySchemas) {
|
|
1691
2907
|
this.baseDir = baseDir;
|
|
2908
|
+
this.entitySchemas = entitySchemas;
|
|
1692
2909
|
}
|
|
1693
2910
|
baseDir;
|
|
2911
|
+
entitySchemas;
|
|
1694
2912
|
knowledgeIndexCache = null;
|
|
1695
2913
|
static KNOWLEDGE_INDEX_CACHE_TTL_MS = 6e5;
|
|
1696
2914
|
// 10 minutes (entity mutations invalidate)
|
|
@@ -1699,6 +2917,9 @@ var StorageManager = class _StorageManager {
|
|
|
1699
2917
|
// 1 minute
|
|
1700
2918
|
static artifactWriteVersionByDir = /* @__PURE__ */ new Map();
|
|
1701
2919
|
static memoryStatusVersionByDir = /* @__PURE__ */ new Map();
|
|
2920
|
+
// In-process fallback for the cold-write sentinel (used when the disk file
|
|
2921
|
+
// is not accessible). The canonical source of truth is state/cold-write.log.
|
|
2922
|
+
static coldWriteVersionByDir = /* @__PURE__ */ new Map();
|
|
1702
2923
|
// Module-level cache for readAllMemories() keyed by base directory.
|
|
1703
2924
|
// Shared across all StorageManager instances to avoid duplicate I/O when
|
|
1704
2925
|
// multiple concurrent callers (e.g. verifiedRecall + verifiedRules) read the
|
|
@@ -1710,6 +2931,29 @@ var StorageManager = class _StorageManager {
|
|
|
1710
2931
|
// refresh. This eliminates the 13-60 s cold-scan penalty that would otherwise
|
|
1711
2932
|
// block recall requests every 5 minutes on large memory collections (80k+ files).
|
|
1712
2933
|
static allMemoriesInFlight = /* @__PURE__ */ new Map();
|
|
2934
|
+
// Cache for readAllColdMemories() — keyed by cold root directory path.
|
|
2935
|
+
// Prevents an uncached full-tree directory scan on every structured-attribute
|
|
2936
|
+
// write (Finding UOGi, PR #402 round-6). The cache is only invalidated when
|
|
2937
|
+
// cold-tier content actually changes (via invalidateColdMemoriesCache), NOT
|
|
2938
|
+
// on every hot-tier write. It also expires after COLD_SCAN_CACHE_TTL_MS as
|
|
2939
|
+
// a safety net.
|
|
2940
|
+
//
|
|
2941
|
+
// Finding UvUy (PR #402 round-11): cache entries now carry a `coldVersion`
|
|
2942
|
+
// sentinel that is bumped (via a file-size counter in state/cold-write.log)
|
|
2943
|
+
// on every write that modifies cold-tier content. Before serving a cached
|
|
2944
|
+
// result, readAllColdMemories() reads the sentinel from disk and compares.
|
|
2945
|
+
// If they differ the entry is dropped and the cold tree is re-scanned. This
|
|
2946
|
+
// makes the cache correct across process boundaries (gateway + CLI): a second
|
|
2947
|
+
// process that writes a new cold memory bumps the sentinel on disk, so the
|
|
2948
|
+
// first process's next readAllColdMemories() sees the change within one call
|
|
2949
|
+
// (rather than waiting up to 30s for TTL expiry).
|
|
2950
|
+
//
|
|
2951
|
+
// After Finding UTsP broadened readAllColdMemories to scan the entire cold/
|
|
2952
|
+
// subtree (not just facts/+corrections/), amortizing this I/O across
|
|
2953
|
+
// back-to-back writes in the same burst is even more important.
|
|
2954
|
+
static COLD_SCAN_CACHE_TTL_MS = 3e4;
|
|
2955
|
+
// 30 seconds
|
|
2956
|
+
static coldMemoriesCache = /* @__PURE__ */ new Map();
|
|
1713
2957
|
// Cache for readQuestions() — avoids serially re-reading tens of thousands of
|
|
1714
2958
|
// question files on every recall. 60-second TTL is intentionally short so that
|
|
1715
2959
|
// newly written questions surface quickly.
|
|
@@ -1720,19 +2964,39 @@ var StorageManager = class _StorageManager {
|
|
|
1720
2964
|
factHashIndexLoadPromise = null;
|
|
1721
2965
|
factHashIndexAuthoritative = null;
|
|
1722
2966
|
factHashIndexAuthoritativePromise = null;
|
|
2967
|
+
/** Optional: set by the orchestrator after construction to enable template-aware citation stripping during legacy hash rebuild. */
|
|
2968
|
+
citationTemplate = DEFAULT_CITATION_FORMAT;
|
|
2969
|
+
/** Page-versioning configuration. Set by the orchestrator after construction. */
|
|
2970
|
+
_versioningConfig = null;
|
|
2971
|
+
/** Set the page-versioning configuration. When `enabled` is false (default), all versioning calls are no-ops. */
|
|
2972
|
+
setVersioningConfig(config) {
|
|
2973
|
+
this._versioningConfig = config;
|
|
2974
|
+
}
|
|
2975
|
+
/**
|
|
2976
|
+
* Snapshot the current content of a page before overwriting.
|
|
2977
|
+
* No-op when versioning is disabled or the file does not yet exist.
|
|
2978
|
+
*/
|
|
2979
|
+
async snapshotBeforeWrite(filePath, trigger) {
|
|
2980
|
+
if (!this._versioningConfig || !this._versioningConfig.enabled) return;
|
|
2981
|
+
try {
|
|
2982
|
+
const existing = await readFile3(filePath, "utf-8");
|
|
2983
|
+
await createVersion(filePath, existing, trigger, this._versioningConfig, log, void 0, this.baseDir);
|
|
2984
|
+
} catch {
|
|
2985
|
+
}
|
|
2986
|
+
}
|
|
1723
2987
|
/** The root directory of this storage instance. */
|
|
1724
2988
|
get dir() {
|
|
1725
2989
|
return this.baseDir;
|
|
1726
2990
|
}
|
|
1727
2991
|
identityFilePath(workspaceDir, namespace) {
|
|
1728
2992
|
const rawNamespace = typeof namespace === "string" ? namespace.trim() : "";
|
|
1729
|
-
if (!rawNamespace) return
|
|
2993
|
+
if (!rawNamespace) return path5.join(workspaceDir, "IDENTITY.md");
|
|
1730
2994
|
const safeNamespace = rawNamespace.replace(/[^a-zA-Z0-9._-]/g, "-");
|
|
1731
|
-
return
|
|
2995
|
+
return path5.join(workspaceDir, `IDENTITY.${safeNamespace}.md`);
|
|
1732
2996
|
}
|
|
1733
2997
|
versionFilePath(kind) {
|
|
1734
|
-
const fileName = kind === "memory-status" ? ".memory-status-version.log" : ".artifact-write-version.log";
|
|
1735
|
-
return
|
|
2998
|
+
const fileName = kind === "memory-status" ? ".memory-status-version.log" : kind === "artifact-write" ? ".artifact-write-version.log" : ".cold-write-version.log";
|
|
2999
|
+
return path5.join(this.stateDir, fileName);
|
|
1736
3000
|
}
|
|
1737
3001
|
bumpSharedVersion(kind, fallbackMap) {
|
|
1738
3002
|
const filePath = this.versionFilePath(kind);
|
|
@@ -1769,19 +3033,22 @@ var StorageManager = class _StorageManager {
|
|
|
1769
3033
|
return this.readSharedVersion("artifact-write", _StorageManager.artifactWriteVersionByDir);
|
|
1770
3034
|
}
|
|
1771
3035
|
get factsDir() {
|
|
1772
|
-
return
|
|
3036
|
+
return path5.join(this.baseDir, "facts");
|
|
1773
3037
|
}
|
|
1774
3038
|
get correctionsDir() {
|
|
1775
|
-
return
|
|
3039
|
+
return path5.join(this.baseDir, "corrections");
|
|
1776
3040
|
}
|
|
1777
3041
|
get entitiesDir() {
|
|
1778
|
-
return
|
|
3042
|
+
return path5.join(this.baseDir, "entities");
|
|
1779
3043
|
}
|
|
1780
3044
|
get stateDir() {
|
|
1781
|
-
return
|
|
3045
|
+
return path5.join(this.baseDir, "state");
|
|
3046
|
+
}
|
|
3047
|
+
get entitySynthesisQueuePath() {
|
|
3048
|
+
return path5.join(this.stateDir, "entity-synthesis-queue.json");
|
|
1782
3049
|
}
|
|
1783
3050
|
get factHashIndexReadyPath() {
|
|
1784
|
-
return
|
|
3051
|
+
return path5.join(this.stateDir, "fact-hashes.ready");
|
|
1785
3052
|
}
|
|
1786
3053
|
async getFactHashIndex() {
|
|
1787
3054
|
if (this.factHashIndex) {
|
|
@@ -1809,21 +3076,46 @@ var StorageManager = class _StorageManager {
|
|
|
1809
3076
|
}
|
|
1810
3077
|
this.factHashIndexAuthoritativePromise = (async () => {
|
|
1811
3078
|
try {
|
|
1812
|
-
await
|
|
3079
|
+
await access2(this.factHashIndexReadyPath);
|
|
1813
3080
|
this.factHashIndexAuthoritative = true;
|
|
1814
3081
|
return;
|
|
1815
3082
|
} catch {
|
|
1816
3083
|
}
|
|
1817
3084
|
const factHashIndex = await this.getFactHashIndex();
|
|
1818
3085
|
const existing = await this.readAllMemories();
|
|
3086
|
+
let legacyRecovered = 0;
|
|
1819
3087
|
for (const memory of existing) {
|
|
1820
3088
|
if (memory.frontmatter.category !== "fact") continue;
|
|
1821
3089
|
if (inferMemoryStatus(memory.frontmatter, memory.path) !== "active") continue;
|
|
1822
|
-
|
|
3090
|
+
if (memory.frontmatter.contentHash) {
|
|
3091
|
+
factHashIndex.addByHash(memory.frontmatter.contentHash);
|
|
3092
|
+
continue;
|
|
3093
|
+
}
|
|
3094
|
+
const content = memory.content;
|
|
3095
|
+
const stripped = stripCitationForTemplate(content, this.citationTemplate);
|
|
3096
|
+
if (stripped !== content) {
|
|
3097
|
+
factHashIndex.addByHash(
|
|
3098
|
+
ContentHashIndex.computeHash(sanitizeMemoryContent(stripped).text)
|
|
3099
|
+
);
|
|
3100
|
+
continue;
|
|
3101
|
+
}
|
|
3102
|
+
if (!hasCitation(content)) {
|
|
3103
|
+
factHashIndex.addByHash(
|
|
3104
|
+
ContentHashIndex.computeHash(sanitizeMemoryContent(content).text)
|
|
3105
|
+
);
|
|
3106
|
+
continue;
|
|
3107
|
+
}
|
|
3108
|
+
legacyRecovered++;
|
|
3109
|
+
continue;
|
|
3110
|
+
}
|
|
3111
|
+
if (legacyRecovered > 0) {
|
|
3112
|
+
log.info(
|
|
3113
|
+
`ensureFactHashIndexAuthoritative: skipped ${legacyRecovered} legacy fact(s) with no contentHash in frontmatter`
|
|
3114
|
+
);
|
|
1823
3115
|
}
|
|
1824
3116
|
await factHashIndex.save();
|
|
1825
|
-
await
|
|
1826
|
-
await
|
|
3117
|
+
await mkdir3(path5.dirname(this.factHashIndexReadyPath), { recursive: true });
|
|
3118
|
+
await writeFile3(this.factHashIndexReadyPath, "v1\n", "utf-8");
|
|
1827
3119
|
this.factHashIndexAuthoritative = true;
|
|
1828
3120
|
})().finally(() => {
|
|
1829
3121
|
this.factHashIndexAuthoritativePromise = null;
|
|
@@ -1831,55 +3123,55 @@ var StorageManager = class _StorageManager {
|
|
|
1831
3123
|
await this.factHashIndexAuthoritativePromise;
|
|
1832
3124
|
}
|
|
1833
3125
|
get questionsDir() {
|
|
1834
|
-
return
|
|
3126
|
+
return path5.join(this.baseDir, "questions");
|
|
1835
3127
|
}
|
|
1836
3128
|
get artifactsDir() {
|
|
1837
|
-
return
|
|
3129
|
+
return path5.join(this.baseDir, "artifacts");
|
|
1838
3130
|
}
|
|
1839
3131
|
get identityDir() {
|
|
1840
|
-
return
|
|
3132
|
+
return path5.join(this.baseDir, "identity");
|
|
1841
3133
|
}
|
|
1842
3134
|
get identityAnchorPath() {
|
|
1843
|
-
return
|
|
3135
|
+
return path5.join(this.identityDir, "identity-anchor.md");
|
|
1844
3136
|
}
|
|
1845
3137
|
get identityIncidentsDir() {
|
|
1846
|
-
return
|
|
3138
|
+
return path5.join(this.identityDir, "incidents");
|
|
1847
3139
|
}
|
|
1848
3140
|
get identityAuditsWeeklyDir() {
|
|
1849
|
-
return
|
|
3141
|
+
return path5.join(this.identityDir, "audits", "weekly");
|
|
1850
3142
|
}
|
|
1851
3143
|
get identityAuditsMonthlyDir() {
|
|
1852
|
-
return
|
|
3144
|
+
return path5.join(this.identityDir, "audits", "monthly");
|
|
1853
3145
|
}
|
|
1854
3146
|
get identityImprovementLoopsPath() {
|
|
1855
|
-
return
|
|
3147
|
+
return path5.join(this.identityDir, "improvement-loops.md");
|
|
1856
3148
|
}
|
|
1857
3149
|
get identityReflectionsPath() {
|
|
1858
|
-
return
|
|
3150
|
+
return path5.join(this.identityDir, "reflections.md");
|
|
1859
3151
|
}
|
|
1860
3152
|
get profilePath() {
|
|
1861
|
-
return
|
|
3153
|
+
return path5.join(this.baseDir, "profile.md");
|
|
1862
3154
|
}
|
|
1863
3155
|
get memoryActionsPath() {
|
|
1864
|
-
return
|
|
3156
|
+
return path5.join(this.stateDir, "memory-actions.jsonl");
|
|
1865
3157
|
}
|
|
1866
3158
|
get memoryLifecycleLedgerPath() {
|
|
1867
|
-
return
|
|
3159
|
+
return path5.join(this.stateDir, "memory-lifecycle-ledger.jsonl");
|
|
1868
3160
|
}
|
|
1869
3161
|
get compressionGuidelinesPath() {
|
|
1870
|
-
return
|
|
3162
|
+
return path5.join(this.stateDir, "compression-guidelines.md");
|
|
1871
3163
|
}
|
|
1872
3164
|
get compressionGuidelineDraftPath() {
|
|
1873
|
-
return
|
|
3165
|
+
return path5.join(this.stateDir, "compression-guidelines.draft.md");
|
|
1874
3166
|
}
|
|
1875
3167
|
get compressionGuidelineStatePath() {
|
|
1876
|
-
return
|
|
3168
|
+
return path5.join(this.stateDir, "compression-guideline-state.json");
|
|
1877
3169
|
}
|
|
1878
3170
|
get compressionGuidelineDraftStatePath() {
|
|
1879
|
-
return
|
|
3171
|
+
return path5.join(this.stateDir, "compression-guideline-draft-state.json");
|
|
1880
3172
|
}
|
|
1881
3173
|
get behaviorSignalsPath() {
|
|
1882
|
-
return
|
|
3174
|
+
return path5.join(this.stateDir, "behavior-signals.jsonl");
|
|
1883
3175
|
}
|
|
1884
3176
|
/**
|
|
1885
3177
|
* Load user-defined entity aliases from config/aliases.json in the memory store.
|
|
@@ -1887,9 +3179,9 @@ var StorageManager = class _StorageManager {
|
|
|
1887
3179
|
* Call this once at startup (e.g. from orchestrator.initialize()).
|
|
1888
3180
|
*/
|
|
1889
3181
|
async loadAliases() {
|
|
1890
|
-
const aliasPath =
|
|
3182
|
+
const aliasPath = path5.join(this.baseDir, "config", "aliases.json");
|
|
1891
3183
|
try {
|
|
1892
|
-
const raw = await
|
|
3184
|
+
const raw = await readFile3(aliasPath, "utf-8");
|
|
1893
3185
|
const parsed = JSON.parse(raw);
|
|
1894
3186
|
if (typeof parsed === "object" && parsed !== null) {
|
|
1895
3187
|
userAliases = parsed;
|
|
@@ -1901,17 +3193,17 @@ var StorageManager = class _StorageManager {
|
|
|
1901
3193
|
}
|
|
1902
3194
|
async ensureDirectories() {
|
|
1903
3195
|
const today = (/* @__PURE__ */ new Date()).toISOString().slice(0, 10);
|
|
1904
|
-
await
|
|
1905
|
-
await
|
|
1906
|
-
await
|
|
1907
|
-
await
|
|
1908
|
-
await
|
|
1909
|
-
await
|
|
1910
|
-
await
|
|
1911
|
-
await
|
|
1912
|
-
await
|
|
1913
|
-
await
|
|
1914
|
-
await
|
|
3196
|
+
await mkdir3(path5.join(this.factsDir, today), { recursive: true });
|
|
3197
|
+
await mkdir3(this.correctionsDir, { recursive: true });
|
|
3198
|
+
await mkdir3(this.entitiesDir, { recursive: true });
|
|
3199
|
+
await mkdir3(this.stateDir, { recursive: true });
|
|
3200
|
+
await mkdir3(this.questionsDir, { recursive: true });
|
|
3201
|
+
await mkdir3(this.artifactsDir, { recursive: true });
|
|
3202
|
+
await mkdir3(this.identityDir, { recursive: true });
|
|
3203
|
+
await mkdir3(this.identityIncidentsDir, { recursive: true });
|
|
3204
|
+
await mkdir3(this.identityAuditsWeeklyDir, { recursive: true });
|
|
3205
|
+
await mkdir3(this.identityAuditsMonthlyDir, { recursive: true });
|
|
3206
|
+
await mkdir3(path5.join(this.baseDir, "config"), { recursive: true });
|
|
1915
3207
|
}
|
|
1916
3208
|
async writeMemory(category, content, options = {}) {
|
|
1917
3209
|
await this.ensureDirectories();
|
|
@@ -1953,25 +3245,29 @@ var StorageManager = class _StorageManager {
|
|
|
1953
3245
|
};
|
|
1954
3246
|
let enrichedContent = content;
|
|
1955
3247
|
if (options.structuredAttributes && Object.keys(options.structuredAttributes).length > 0) {
|
|
1956
|
-
const attrLines = Object.entries(options.structuredAttributes).map(([k, v]) => `${k}: ${v}`).join("; ");
|
|
1957
3248
|
enrichedContent = `${content}
|
|
1958
|
-
[Attributes: ${
|
|
3249
|
+
[Attributes: ${normalizeAttributePairs(options.structuredAttributes)}]`;
|
|
1959
3250
|
}
|
|
1960
3251
|
const sanitized = sanitizeMemoryContent(enrichedContent);
|
|
1961
3252
|
if (!sanitized.clean) {
|
|
1962
3253
|
log.warn(`memory content sanitized for ${id}; violations=${sanitized.violations.join(", ")}`);
|
|
1963
3254
|
}
|
|
3255
|
+
if (category === "fact") {
|
|
3256
|
+
const hashSource = options.contentHashSource !== void 0 && options.contentHashSource.length > 0 ? sanitizeMemoryContent(options.contentHashSource).text : sanitized.text;
|
|
3257
|
+
fm.contentHash = ContentHashIndex.computeHash(hashSource);
|
|
3258
|
+
}
|
|
1964
3259
|
const fileContent = `${serializeFrontmatter(fm)}
|
|
1965
3260
|
|
|
1966
3261
|
${sanitized.text}
|
|
1967
3262
|
`;
|
|
1968
3263
|
let filePath;
|
|
1969
3264
|
if (category === "correction") {
|
|
1970
|
-
filePath =
|
|
3265
|
+
filePath = path5.join(this.correctionsDir, `${id}.md`);
|
|
1971
3266
|
} else {
|
|
1972
|
-
filePath =
|
|
3267
|
+
filePath = path5.join(this.factsDir, today, `${id}.md`);
|
|
1973
3268
|
}
|
|
1974
|
-
await
|
|
3269
|
+
await this.snapshotBeforeWrite(filePath, "write");
|
|
3270
|
+
await writeFile3(filePath, fileContent, "utf-8");
|
|
1975
3271
|
this.invalidateAllMemoriesCache();
|
|
1976
3272
|
await this.appendGeneratedMemoryLifecycleEventFailOpen("storage.writeMemory", {
|
|
1977
3273
|
memoryId: id,
|
|
@@ -1987,7 +3283,12 @@ ${sanitized.text}
|
|
|
1987
3283
|
if (category === "fact") {
|
|
1988
3284
|
try {
|
|
1989
3285
|
const factHashIndex = await this.getFactHashIndex();
|
|
1990
|
-
|
|
3286
|
+
if (options.contentHashSource !== void 0 && options.contentHashSource.length > 0) {
|
|
3287
|
+
const hashSourceSanitized = sanitizeMemoryContent(options.contentHashSource);
|
|
3288
|
+
factHashIndex.add(hashSourceSanitized.text);
|
|
3289
|
+
} else {
|
|
3290
|
+
factHashIndex.add(sanitized.text);
|
|
3291
|
+
}
|
|
1991
3292
|
await factHashIndex.save();
|
|
1992
3293
|
} catch (err) {
|
|
1993
3294
|
log.warn(`storage.writeMemory completed but failed to update fact hash index: ${err}`);
|
|
@@ -2010,8 +3311,8 @@ ${sanitized.text}
|
|
|
2010
3311
|
await this.ensureDirectories();
|
|
2011
3312
|
const now = /* @__PURE__ */ new Date();
|
|
2012
3313
|
const day = now.toISOString().slice(0, 10);
|
|
2013
|
-
const dir =
|
|
2014
|
-
await
|
|
3314
|
+
const dir = path5.join(this.artifactsDir, day);
|
|
3315
|
+
await mkdir3(dir, { recursive: true });
|
|
2015
3316
|
const id = `artifact-${Date.now()}-${Math.random().toString(36).slice(2, 6)}`;
|
|
2016
3317
|
const fm = {
|
|
2017
3318
|
id,
|
|
@@ -2034,8 +3335,8 @@ ${sanitized.text}
|
|
|
2034
3335
|
log.warn(`artifact content rejected for ${id}; violations=${sanitized.violations.join(", ")}`);
|
|
2035
3336
|
return "";
|
|
2036
3337
|
}
|
|
2037
|
-
const filePath =
|
|
2038
|
-
await
|
|
3338
|
+
const filePath = path5.join(dir, `${id}.md`);
|
|
3339
|
+
await writeFile3(filePath, `${serializeFrontmatter(fm)}
|
|
2039
3340
|
|
|
2040
3341
|
${sanitized.text}
|
|
2041
3342
|
`, "utf-8");
|
|
@@ -2062,7 +3363,7 @@ ${sanitized.text}
|
|
|
2062
3363
|
try {
|
|
2063
3364
|
const entries = await readdir(dir, { withFileTypes: true });
|
|
2064
3365
|
for (const entry of entries) {
|
|
2065
|
-
const fullPath =
|
|
3366
|
+
const fullPath = path5.join(dir, entry.name);
|
|
2066
3367
|
if (entry.isDirectory()) {
|
|
2067
3368
|
await readDir(fullPath);
|
|
2068
3369
|
continue;
|
|
@@ -2110,7 +3411,7 @@ ${sanitized.text}
|
|
|
2110
3411
|
hits.sort((a, b) => b.score - a.score);
|
|
2111
3412
|
return hits.slice(0, maxResults).map((h) => h.memory);
|
|
2112
3413
|
}
|
|
2113
|
-
async writeEntity(name, type, facts) {
|
|
3414
|
+
async writeEntity(name, type, facts, options = {}) {
|
|
2114
3415
|
await this.ensureDirectories();
|
|
2115
3416
|
if (typeof name !== "string" || !name.trim() || typeof type !== "string" || !type.trim()) {
|
|
2116
3417
|
log.warn("writeEntity: invalid entity payload, skipping", {
|
|
@@ -2119,34 +3420,93 @@ ${sanitized.text}
|
|
|
2119
3420
|
});
|
|
2120
3421
|
return "";
|
|
2121
3422
|
}
|
|
2122
|
-
const safeFacts = Array.isArray(facts) ?
|
|
3423
|
+
const safeFacts = Array.isArray(facts) ? [...new Set(
|
|
3424
|
+
facts.filter((fact) => typeof fact === "string").map((fact) => fact.trim()).filter((fact) => fact.length > 0)
|
|
3425
|
+
)] : [];
|
|
2123
3426
|
let normalized = normalizeEntityName(name, type);
|
|
2124
3427
|
const match = await this.findMatchingEntity(name, type);
|
|
2125
3428
|
if (match && match !== normalized) {
|
|
2126
3429
|
log.debug(`fuzzy match: "${normalized}" \u2192 existing "${match}"`);
|
|
2127
3430
|
normalized = match;
|
|
2128
3431
|
}
|
|
2129
|
-
const filePath =
|
|
3432
|
+
const filePath = path5.join(this.entitiesDir, `${normalized}.md`);
|
|
2130
3433
|
let entity = {
|
|
2131
3434
|
name,
|
|
2132
3435
|
type,
|
|
3436
|
+
created: "",
|
|
2133
3437
|
updated: (/* @__PURE__ */ new Date()).toISOString(),
|
|
2134
3438
|
facts: [],
|
|
2135
3439
|
summary: void 0,
|
|
3440
|
+
synthesis: void 0,
|
|
3441
|
+
synthesisUpdatedAt: void 0,
|
|
3442
|
+
synthesisVersion: void 0,
|
|
3443
|
+
synthesisStructuredFactCount: void 0,
|
|
3444
|
+
synthesisStructuredFactDigest: void 0,
|
|
3445
|
+
timeline: [],
|
|
2136
3446
|
relationships: [],
|
|
2137
3447
|
activity: [],
|
|
2138
3448
|
aliases: []
|
|
2139
3449
|
};
|
|
2140
3450
|
try {
|
|
2141
|
-
const existing = await
|
|
2142
|
-
entity = parseEntityFile(existing);
|
|
3451
|
+
const existing = await readFile3(filePath, "utf-8");
|
|
3452
|
+
entity = parseEntityFile(existing, this.entitySchemas);
|
|
2143
3453
|
} catch {
|
|
2144
3454
|
}
|
|
2145
|
-
|
|
3455
|
+
const timestamp = options.timestamp?.trim() || (/* @__PURE__ */ new Date()).toISOString();
|
|
3456
|
+
const source = options.source?.trim() || void 0;
|
|
3457
|
+
const sessionKey = options.sessionKey?.trim() || void 0;
|
|
3458
|
+
const principal = options.principal?.trim() || void 0;
|
|
3459
|
+
const structuredSectionMap = new Map(
|
|
3460
|
+
(entity.structuredSections ?? []).map((section) => [section.key, {
|
|
3461
|
+
...section,
|
|
3462
|
+
facts: [...section.facts]
|
|
3463
|
+
}])
|
|
3464
|
+
);
|
|
3465
|
+
for (const section of options.structuredSections ?? []) {
|
|
3466
|
+
const normalizedSection = normalizeEntityStructuredSection(type, section, this.entitySchemas);
|
|
3467
|
+
const normalizedFacts = normalizeStructuredSectionFacts(section.facts);
|
|
3468
|
+
if (normalizedFacts.length === 0) continue;
|
|
3469
|
+
const existingSection = structuredSectionMap.get(normalizedSection.key);
|
|
3470
|
+
if (!existingSection) {
|
|
3471
|
+
structuredSectionMap.set(normalizedSection.key, {
|
|
3472
|
+
key: normalizedSection.key,
|
|
3473
|
+
title: normalizedSection.title,
|
|
3474
|
+
facts: normalizedFacts
|
|
3475
|
+
});
|
|
3476
|
+
continue;
|
|
3477
|
+
}
|
|
3478
|
+
existingSection.facts = normalizeStructuredSectionFacts([...existingSection.facts, ...normalizedFacts]);
|
|
3479
|
+
if (!existingSection.title.trim() && normalizedSection.title.trim()) {
|
|
3480
|
+
existingSection.title = normalizedSection.title;
|
|
3481
|
+
}
|
|
3482
|
+
}
|
|
3483
|
+
for (const fact of safeFacts) {
|
|
3484
|
+
const nextEntry = {
|
|
3485
|
+
timestamp,
|
|
3486
|
+
text: fact,
|
|
3487
|
+
...source ? { source } : {},
|
|
3488
|
+
...sessionKey ? { sessionKey } : {},
|
|
3489
|
+
...principal ? { principal } : {}
|
|
3490
|
+
};
|
|
3491
|
+
const alreadyPresent = entity.timeline.some(
|
|
3492
|
+
(entry) => entry.timestamp === nextEntry.timestamp && entry.text === nextEntry.text && entry.source === nextEntry.source && entry.sessionKey === nextEntry.sessionKey && entry.principal === nextEntry.principal
|
|
3493
|
+
);
|
|
3494
|
+
if (alreadyPresent) continue;
|
|
3495
|
+
entity.timeline.push(nextEntry);
|
|
3496
|
+
}
|
|
3497
|
+
entity.structuredSections = sortStructuredSectionsBySchema(
|
|
3498
|
+
type,
|
|
3499
|
+
Array.from(structuredSectionMap.values()).filter((section) => section.facts.length > 0),
|
|
3500
|
+
this.entitySchemas
|
|
3501
|
+
);
|
|
3502
|
+
entity.facts = compileEntityFacts(entity.timeline, entity.structuredSections);
|
|
3503
|
+
entity.summary = entity.synthesis || entity.summary;
|
|
2146
3504
|
entity.name = name;
|
|
2147
3505
|
entity.type = type;
|
|
3506
|
+
entity.created = entity.created || timestamp;
|
|
2148
3507
|
entity.updated = (/* @__PURE__ */ new Date()).toISOString();
|
|
2149
|
-
await
|
|
3508
|
+
await this.snapshotBeforeWrite(filePath, "write");
|
|
3509
|
+
await writeFile3(filePath, serializeEntityFile(entity, this.entitySchemas), "utf-8");
|
|
2150
3510
|
this.invalidateKnowledgeIndexCache();
|
|
2151
3511
|
this.bumpMemoryStatusVersion();
|
|
2152
3512
|
log.debug(`wrote entity ${normalized}`);
|
|
@@ -2154,14 +3514,15 @@ ${sanitized.text}
|
|
|
2154
3514
|
}
|
|
2155
3515
|
async readProfile() {
|
|
2156
3516
|
try {
|
|
2157
|
-
return await
|
|
3517
|
+
return await readFile3(this.profilePath, "utf-8");
|
|
2158
3518
|
} catch {
|
|
2159
3519
|
return "";
|
|
2160
3520
|
}
|
|
2161
3521
|
}
|
|
2162
3522
|
async writeProfile(content) {
|
|
2163
3523
|
await this.ensureDirectories();
|
|
2164
|
-
await
|
|
3524
|
+
await this.snapshotBeforeWrite(this.profilePath, "consolidation");
|
|
3525
|
+
await writeFile3(this.profilePath, content, "utf-8");
|
|
2165
3526
|
log.debug("updated profile.md");
|
|
2166
3527
|
}
|
|
2167
3528
|
/**
|
|
@@ -2249,12 +3610,51 @@ ${sanitized.text}
|
|
|
2249
3610
|
static clearAllStaticCaches() {
|
|
2250
3611
|
_StorageManager.allMemoriesInFlight.clear();
|
|
2251
3612
|
_StorageManager.questionsCache.clear();
|
|
3613
|
+
_StorageManager.coldMemoriesCache.clear();
|
|
2252
3614
|
}
|
|
2253
3615
|
/** Cancel any in-flight concurrent read so the next readAllMemories()
|
|
2254
|
-
* starts a fresh disk scan and sees the just-written data.
|
|
3616
|
+
* starts a fresh disk scan and sees the just-written data.
|
|
3617
|
+
*
|
|
3618
|
+
* Finding UvBq (PR #402 round-11): this method intentionally does NOT
|
|
3619
|
+
* invalidate the cold-scan cache. Ordinary hot-tier writes (writeMemory)
|
|
3620
|
+
* do not change cold-tier content, so evicting the cold cache on every hot
|
|
3621
|
+
* write was defeating the burst-dedup optimisation — the cold cache was
|
|
3622
|
+
* cleared before applyTemporalSupersession ran, causing a full cold-tree
|
|
3623
|
+
* disk scan on every write in a burst. Cold cache invalidation is handled
|
|
3624
|
+
* exclusively by invalidateColdMemoriesCache(), which is called only when
|
|
3625
|
+
* cold content actually changes (hot→cold demotions, writeMemoryFileAtomic
|
|
3626
|
+
* inside cold/, archiveMemory, etc.). */
|
|
2255
3627
|
invalidateAllMemoriesCache() {
|
|
2256
3628
|
_StorageManager.allMemoriesInFlight.delete(this.baseDir);
|
|
2257
3629
|
}
|
|
3630
|
+
/**
|
|
3631
|
+
* Invalidate the cold-scan cache for this storage root and bump the
|
|
3632
|
+
* on-disk cold-version sentinel so that other processes (gateway, CLI) see
|
|
3633
|
+
* the change immediately on their next readAllColdMemories() call.
|
|
3634
|
+
*
|
|
3635
|
+
* Must be called whenever a memory is written INTO the cold tier — hot→cold
|
|
3636
|
+
* demotion, atomic writes inside cold/, archiving a cold memory, etc.
|
|
3637
|
+
* NOT called on ordinary hot-tier writes (those don't change cold contents).
|
|
3638
|
+
*
|
|
3639
|
+
* Finding UvUy (PR #402 round-11): bumping the sentinel here makes the
|
|
3640
|
+
* per-process in-memory cache safe across process boundaries.
|
|
3641
|
+
*/
|
|
3642
|
+
invalidateColdMemoriesCache() {
|
|
3643
|
+
const coldRoot = path5.join(this.baseDir, "cold");
|
|
3644
|
+
_StorageManager.coldMemoriesCache.delete(coldRoot);
|
|
3645
|
+
this.bumpColdWriteVersion();
|
|
3646
|
+
}
|
|
3647
|
+
/** Return the current cold-write version counter for this storage root.
|
|
3648
|
+
* Reads the on-disk sentinel (state/cold-write.log) so it reflects writes
|
|
3649
|
+
* made by other processes. */
|
|
3650
|
+
readColdWriteVersion() {
|
|
3651
|
+
return this.readSharedVersion("cold-write", _StorageManager.coldWriteVersionByDir);
|
|
3652
|
+
}
|
|
3653
|
+
/** Bump the on-disk cold-write version sentinel and update the in-process
|
|
3654
|
+
* fallback map. Called by invalidateColdMemoriesCache(). */
|
|
3655
|
+
bumpColdWriteVersion() {
|
|
3656
|
+
this.bumpSharedVersion("cold-write", _StorageManager.coldWriteVersionByDir);
|
|
3657
|
+
}
|
|
2258
3658
|
normalizeMemoryReadBatchSize(batchSize) {
|
|
2259
3659
|
if (typeof batchSize !== "number" || !Number.isFinite(batchSize)) {
|
|
2260
3660
|
return 50;
|
|
@@ -2268,7 +3668,7 @@ ${sanitized.text}
|
|
|
2268
3668
|
const entries = await readdir(dir, { withFileTypes: true });
|
|
2269
3669
|
const subdirs = [];
|
|
2270
3670
|
for (const entry of entries) {
|
|
2271
|
-
const fullPath =
|
|
3671
|
+
const fullPath = path5.join(dir, entry.name);
|
|
2272
3672
|
if (entry.isDirectory()) {
|
|
2273
3673
|
subdirs.push(fullPath);
|
|
2274
3674
|
} else if (entry.name.endsWith(".md")) {
|
|
@@ -2294,14 +3694,15 @@ ${sanitized.text}
|
|
|
2294
3694
|
const results = await Promise.all(
|
|
2295
3695
|
batch.map(async (fullPath) => {
|
|
2296
3696
|
try {
|
|
2297
|
-
const raw = await
|
|
3697
|
+
const raw = await readFile3(fullPath, "utf-8");
|
|
2298
3698
|
const parsed = parseFrontmatter2(raw);
|
|
2299
3699
|
if (!parsed) return null;
|
|
2300
3700
|
return {
|
|
2301
3701
|
path: fullPath,
|
|
2302
3702
|
frontmatter: normalizeFrontmatterForPath(
|
|
2303
3703
|
parsed.frontmatter,
|
|
2304
|
-
toMemoryPathRel(this.baseDir, fullPath)
|
|
3704
|
+
toMemoryPathRel(this.baseDir, fullPath),
|
|
3705
|
+
parsed.content
|
|
2305
3706
|
),
|
|
2306
3707
|
content: parsed.content
|
|
2307
3708
|
};
|
|
@@ -2318,7 +3719,7 @@ ${sanitized.text}
|
|
|
2318
3719
|
}
|
|
2319
3720
|
async readWindowUpdatedMs(filePath) {
|
|
2320
3721
|
try {
|
|
2321
|
-
const raw = await
|
|
3722
|
+
const raw = await readFile3(filePath, "utf-8");
|
|
2322
3723
|
const match = raw.match(/^---\n([\s\S]*?)\n---\n?/);
|
|
2323
3724
|
if (!match) return null;
|
|
2324
3725
|
const frontmatterBlock = match[1];
|
|
@@ -2348,7 +3749,7 @@ ${sanitized.text}
|
|
|
2348
3749
|
const correctionPaths = [];
|
|
2349
3750
|
const factPaths = [];
|
|
2350
3751
|
for (const filePath of filePaths) {
|
|
2351
|
-
if (filePath === this.correctionsDir || filePath.startsWith(`${this.correctionsDir}${
|
|
3752
|
+
if (filePath === this.correctionsDir || filePath.startsWith(`${this.correctionsDir}${path5.sep}`)) {
|
|
2352
3753
|
correctionPaths.push(filePath);
|
|
2353
3754
|
} else {
|
|
2354
3755
|
factPaths.push(filePath);
|
|
@@ -2430,6 +3831,62 @@ ${sanitized.text}
|
|
|
2430
3831
|
const filePaths = await this.collectActiveMemoryPaths();
|
|
2431
3832
|
return this.readParsedMemoriesFromPaths(filePaths, 50);
|
|
2432
3833
|
}
|
|
3834
|
+
/**
|
|
3835
|
+
* Read all memories from the cold tier by scanning the entire cold/ root
|
|
3836
|
+
* tree. Previously this only scanned cold/facts/ and cold/corrections/, but
|
|
3837
|
+
* structuredAttributes can appear on any MemoryCategory (preference, decision,
|
|
3838
|
+
* entity, etc.). Although buildTierMemoryPath currently routes all
|
|
3839
|
+
* non-correction, non-artifact memories to cold/facts/, scanning the full
|
|
3840
|
+
* coldRoot ensures correctness if that routing ever changes and guards against
|
|
3841
|
+
* files placed in unexpected subdirectories during manual operations or future
|
|
3842
|
+
* refactors.
|
|
3843
|
+
*
|
|
3844
|
+
* Broadened in PR #402 round-6 (Finding UTsP): scanning only facts/ and
|
|
3845
|
+
* corrections/ was a narrower-than-necessary subset of the cold directory
|
|
3846
|
+
* tree. Correctness trumps the minor performance difference — cold scans
|
|
3847
|
+
* already happen at most once per supersession write.
|
|
3848
|
+
*
|
|
3849
|
+
* Used by applyTemporalSupersession so that memories already demoted to
|
|
3850
|
+
* cold/ can still be marked superseded when a newer hot fact arrives.
|
|
3851
|
+
*
|
|
3852
|
+
* Cached with a TTL (Finding UOGi, PR #402 round-6): back-to-back
|
|
3853
|
+
* structured-attribute writes in the same burst reuse the cached result
|
|
3854
|
+
* instead of re-scanning the cold tree on every call. The cache is
|
|
3855
|
+
* invalidated whenever a write calls invalidateAllMemoriesCache() (which
|
|
3856
|
+
* covers any hot→cold demotion that changes cold-tier contents) and
|
|
3857
|
+
* expires after COLD_SCAN_CACHE_TTL_MS as a safety net.
|
|
3858
|
+
*/
|
|
3859
|
+
async readAllColdMemories() {
|
|
3860
|
+
const coldRoot = this.resolveTierRootDir("cold");
|
|
3861
|
+
const currentColdVersion = this.readColdWriteVersion();
|
|
3862
|
+
const cached = _StorageManager.coldMemoriesCache.get(coldRoot);
|
|
3863
|
+
if (cached && Date.now() - cached.loadedAt < _StorageManager.COLD_SCAN_CACHE_TTL_MS && cached.coldVersion === currentColdVersion) {
|
|
3864
|
+
return cached.memories;
|
|
3865
|
+
}
|
|
3866
|
+
const filePaths = [];
|
|
3867
|
+
const collectPaths = async (dir) => {
|
|
3868
|
+
try {
|
|
3869
|
+
const entries = await readdir(dir, { withFileTypes: true });
|
|
3870
|
+
const subdirs = [];
|
|
3871
|
+
for (const entry of entries) {
|
|
3872
|
+
const fullPath = path5.join(dir, entry.name);
|
|
3873
|
+
if (entry.isDirectory()) {
|
|
3874
|
+
subdirs.push(fullPath);
|
|
3875
|
+
} else if (entry.name.endsWith(".md")) {
|
|
3876
|
+
filePaths.push(fullPath);
|
|
3877
|
+
}
|
|
3878
|
+
}
|
|
3879
|
+
for (const subdir of subdirs) {
|
|
3880
|
+
await collectPaths(subdir);
|
|
3881
|
+
}
|
|
3882
|
+
} catch {
|
|
3883
|
+
}
|
|
3884
|
+
};
|
|
3885
|
+
await collectPaths(coldRoot);
|
|
3886
|
+
const memories = await this.readParsedMemoriesFromPaths(filePaths, 50);
|
|
3887
|
+
_StorageManager.coldMemoriesCache.set(coldRoot, { memories, loadedAt: Date.now(), coldVersion: currentColdVersion });
|
|
3888
|
+
return memories;
|
|
3889
|
+
}
|
|
2433
3890
|
/**
|
|
2434
3891
|
* Read archived memory markdown files under archive/.
|
|
2435
3892
|
* Used by long-term recall fallback when hot recall has no hits.
|
|
@@ -2441,19 +3898,20 @@ ${sanitized.text}
|
|
|
2441
3898
|
try {
|
|
2442
3899
|
const entries = await readdir(dir, { withFileTypes: true });
|
|
2443
3900
|
for (const entry of entries) {
|
|
2444
|
-
const fullPath =
|
|
3901
|
+
const fullPath = path5.join(dir, entry.name);
|
|
2445
3902
|
if (entry.isDirectory()) {
|
|
2446
3903
|
await readDir(fullPath);
|
|
2447
3904
|
} else if (entry.name.endsWith(".md")) {
|
|
2448
3905
|
try {
|
|
2449
|
-
const raw = await
|
|
3906
|
+
const raw = await readFile3(fullPath, "utf-8");
|
|
2450
3907
|
const parsed = parseFrontmatter2(raw);
|
|
2451
3908
|
if (parsed) {
|
|
2452
3909
|
memories.push({
|
|
2453
3910
|
path: fullPath,
|
|
2454
3911
|
frontmatter: normalizeFrontmatterForPath(
|
|
2455
3912
|
parsed.frontmatter,
|
|
2456
|
-
toMemoryPathRel(this.baseDir, fullPath)
|
|
3913
|
+
toMemoryPathRel(this.baseDir, fullPath),
|
|
3914
|
+
parsed.content
|
|
2457
3915
|
),
|
|
2458
3916
|
content: parsed.content
|
|
2459
3917
|
});
|
|
@@ -2471,23 +3929,24 @@ ${sanitized.text}
|
|
|
2471
3929
|
/** Read a single memory file by its absolute path. Returns null if unreadable. */
|
|
2472
3930
|
async readMemoryByPath(filePath) {
|
|
2473
3931
|
try {
|
|
2474
|
-
const raw = await
|
|
3932
|
+
const raw = await readFile3(filePath, "utf-8");
|
|
2475
3933
|
const parsed = parseFrontmatter2(raw);
|
|
2476
3934
|
if (parsed) {
|
|
2477
3935
|
return {
|
|
2478
3936
|
path: filePath,
|
|
2479
3937
|
frontmatter: normalizeFrontmatterForPath(
|
|
2480
3938
|
parsed.frontmatter,
|
|
2481
|
-
toMemoryPathRel(this.baseDir, filePath)
|
|
3939
|
+
toMemoryPathRel(this.baseDir, filePath),
|
|
3940
|
+
parsed.content
|
|
2482
3941
|
),
|
|
2483
3942
|
content: parsed.content
|
|
2484
3943
|
};
|
|
2485
3944
|
}
|
|
2486
|
-
const normalizedPath = filePath.split(
|
|
3945
|
+
const normalizedPath = filePath.split(path5.sep).join("/");
|
|
2487
3946
|
if (normalizedPath.includes("/entities/") && filePath.endsWith(".md")) {
|
|
2488
|
-
const entity = parseEntityFile(raw);
|
|
3947
|
+
const entity = parseEntityFile(raw, this.entitySchemas);
|
|
2489
3948
|
if (!entity.name) return null;
|
|
2490
|
-
const nameWithoutExt =
|
|
3949
|
+
const nameWithoutExt = path5.basename(filePath, ".md");
|
|
2491
3950
|
const fileMtime = entity.updated || await stat2(filePath).then((s) => s.mtime.toISOString()).catch(() => (/* @__PURE__ */ new Date(0)).toISOString());
|
|
2492
3951
|
return {
|
|
2493
3952
|
path: filePath,
|
|
@@ -2510,7 +3969,7 @@ ${sanitized.text}
|
|
|
2510
3969
|
}
|
|
2511
3970
|
}
|
|
2512
3971
|
resolveTierRootDir(tier) {
|
|
2513
|
-
return tier === "cold" ?
|
|
3972
|
+
return tier === "cold" ? path5.join(this.baseDir, "cold") : this.baseDir;
|
|
2514
3973
|
}
|
|
2515
3974
|
resolveMemoryDateDir(memory) {
|
|
2516
3975
|
const preferred = memory.frontmatter.created || memory.frontmatter.updated;
|
|
@@ -2525,27 +3984,27 @@ ${sanitized.text}
|
|
|
2525
3984
|
buildTierMemoryPath(memory, tier) {
|
|
2526
3985
|
const root = this.resolveTierRootDir(tier);
|
|
2527
3986
|
if (this.isArtifactMemory(memory)) {
|
|
2528
|
-
return
|
|
3987
|
+
return path5.join(root, "artifacts", this.resolveMemoryDateDir(memory), `${memory.frontmatter.id}.md`);
|
|
2529
3988
|
}
|
|
2530
3989
|
if (memory.frontmatter.category === "correction") {
|
|
2531
|
-
return
|
|
3990
|
+
return path5.join(root, "corrections", `${memory.frontmatter.id}.md`);
|
|
2532
3991
|
}
|
|
2533
|
-
return
|
|
3992
|
+
return path5.join(root, "facts", this.resolveMemoryDateDir(memory), `${memory.frontmatter.id}.md`);
|
|
2534
3993
|
}
|
|
2535
3994
|
async writeMemoryFileAtomic(targetPath, memory) {
|
|
2536
3995
|
const fileContent = `${serializeFrontmatter(memory.frontmatter)}
|
|
2537
3996
|
|
|
2538
3997
|
${memory.content}
|
|
2539
3998
|
`;
|
|
2540
|
-
await
|
|
3999
|
+
await mkdir3(path5.dirname(targetPath), { recursive: true });
|
|
2541
4000
|
const tempPath = `${targetPath}.tmp-${process.pid}-${Date.now()}`;
|
|
2542
4001
|
try {
|
|
2543
|
-
await
|
|
4002
|
+
await writeFile3(tempPath, fileContent, "utf-8");
|
|
2544
4003
|
await rename(tempPath, targetPath);
|
|
2545
4004
|
this.invalidateAllMemoriesCache();
|
|
2546
4005
|
} catch (err) {
|
|
2547
4006
|
try {
|
|
2548
|
-
await
|
|
4007
|
+
await unlink2(tempPath);
|
|
2549
4008
|
} catch {
|
|
2550
4009
|
}
|
|
2551
4010
|
throw err;
|
|
@@ -2553,11 +4012,11 @@ ${memory.content}
|
|
|
2553
4012
|
}
|
|
2554
4013
|
async moveMemoryToPath(memory, targetPath) {
|
|
2555
4014
|
await this.writeMemoryFileAtomic(targetPath, memory);
|
|
2556
|
-
const sourcePath =
|
|
2557
|
-
const destPath =
|
|
4015
|
+
const sourcePath = path5.resolve(memory.path);
|
|
4016
|
+
const destPath = path5.resolve(targetPath);
|
|
2558
4017
|
if (sourcePath !== destPath) {
|
|
2559
4018
|
try {
|
|
2560
|
-
await
|
|
4019
|
+
await unlink2(memory.path);
|
|
2561
4020
|
} catch (err) {
|
|
2562
4021
|
const message = err instanceof Error ? err.message : String(err);
|
|
2563
4022
|
if (!message.includes("ENOENT")) {
|
|
@@ -2569,15 +4028,15 @@ ${memory.content}
|
|
|
2569
4028
|
}
|
|
2570
4029
|
async migrateMemoryToTier(memory, targetTier) {
|
|
2571
4030
|
const targetPath = this.buildTierMemoryPath(memory, targetTier);
|
|
2572
|
-
const sourcePath =
|
|
2573
|
-
const destPath =
|
|
4031
|
+
const sourcePath = path5.resolve(memory.path);
|
|
4032
|
+
const destPath = path5.resolve(targetPath);
|
|
2574
4033
|
if (sourcePath === destPath) {
|
|
2575
4034
|
return { changed: false, targetPath };
|
|
2576
4035
|
}
|
|
2577
4036
|
const existing = await this.readMemoryByPath(targetPath);
|
|
2578
4037
|
if (existing?.frontmatter.id === memory.frontmatter.id) {
|
|
2579
4038
|
try {
|
|
2580
|
-
await
|
|
4039
|
+
await unlink2(memory.path);
|
|
2581
4040
|
} catch (err) {
|
|
2582
4041
|
const message = err instanceof Error ? err.message : String(err);
|
|
2583
4042
|
if (!message.includes("ENOENT")) {
|
|
@@ -2589,11 +4048,14 @@ ${memory.content}
|
|
|
2589
4048
|
}
|
|
2590
4049
|
await this.moveMemoryToPath(memory, targetPath);
|
|
2591
4050
|
this.invalidateAllMemoriesCache();
|
|
4051
|
+
if (targetTier === "cold") {
|
|
4052
|
+
this.invalidateColdMemoriesCache();
|
|
4053
|
+
}
|
|
2592
4054
|
this.bumpMemoryStatusVersion();
|
|
2593
4055
|
return { changed: true, targetPath };
|
|
2594
4056
|
}
|
|
2595
4057
|
get archiveDir() {
|
|
2596
|
-
return
|
|
4058
|
+
return path5.join(this.baseDir, "archive");
|
|
2597
4059
|
}
|
|
2598
4060
|
/**
|
|
2599
4061
|
* Archive a memory by moving it from facts/ to archive/YYYY-MM-DD/.
|
|
@@ -2604,8 +4066,8 @@ ${memory.content}
|
|
|
2604
4066
|
try {
|
|
2605
4067
|
const now = lifecycle?.at ?? /* @__PURE__ */ new Date();
|
|
2606
4068
|
const today = now.toISOString().slice(0, 10);
|
|
2607
|
-
const destDir =
|
|
2608
|
-
await
|
|
4069
|
+
const destDir = path5.join(this.archiveDir, today);
|
|
4070
|
+
await mkdir3(destDir, { recursive: true });
|
|
2609
4071
|
const updatedFm = {
|
|
2610
4072
|
...memory.frontmatter,
|
|
2611
4073
|
status: "archived",
|
|
@@ -2616,9 +4078,9 @@ ${memory.content}
|
|
|
2616
4078
|
|
|
2617
4079
|
${memory.content}
|
|
2618
4080
|
`;
|
|
2619
|
-
const destPath =
|
|
2620
|
-
await
|
|
2621
|
-
await
|
|
4081
|
+
const destPath = path5.join(destDir, path5.basename(memory.path));
|
|
4082
|
+
await writeFile3(destPath, fileContent, "utf-8");
|
|
4083
|
+
await unlink2(memory.path);
|
|
2622
4084
|
this.invalidateAllMemoriesCache();
|
|
2623
4085
|
await this.appendGeneratedMemoryLifecycleEventFailOpen(
|
|
2624
4086
|
"storage.archiveMemory",
|
|
@@ -2653,7 +4115,7 @@ ${memory.content}
|
|
|
2653
4115
|
}
|
|
2654
4116
|
async readEntity(name) {
|
|
2655
4117
|
try {
|
|
2656
|
-
return await
|
|
4118
|
+
return await readFile3(path5.join(this.entitiesDir, `${name}.md`), "utf-8");
|
|
2657
4119
|
} catch {
|
|
2658
4120
|
return "";
|
|
2659
4121
|
}
|
|
@@ -2707,7 +4169,7 @@ ${memory.content}
|
|
|
2707
4169
|
const memory = memories.find((m) => m.frontmatter.id === id);
|
|
2708
4170
|
if (!memory) return false;
|
|
2709
4171
|
try {
|
|
2710
|
-
await
|
|
4172
|
+
await unlink2(memory.path);
|
|
2711
4173
|
this.invalidateAllMemoriesCache();
|
|
2712
4174
|
this.bumpMemoryStatusVersion();
|
|
2713
4175
|
log.debug(`invalidated memory ${id}`);
|
|
@@ -2738,7 +4200,7 @@ ${memory.content}
|
|
|
2738
4200
|
|
|
2739
4201
|
${sanitized.text}
|
|
2740
4202
|
`;
|
|
2741
|
-
await
|
|
4203
|
+
await writeFile3(memory.path, fileContent, "utf-8");
|
|
2742
4204
|
this.invalidateAllMemoriesCache();
|
|
2743
4205
|
await this.appendGeneratedMemoryLifecycleEventFailOpen("storage.updateMemory", {
|
|
2744
4206
|
memoryId: id,
|
|
@@ -2770,8 +4232,11 @@ ${sanitized.text}
|
|
|
2770
4232
|
|
|
2771
4233
|
${memory.content}
|
|
2772
4234
|
`;
|
|
2773
|
-
await
|
|
4235
|
+
await writeFile3(memory.path, fileContent, "utf-8");
|
|
2774
4236
|
this.invalidateAllMemoriesCache();
|
|
4237
|
+
if (memory.path.includes(`${path5.sep}cold${path5.sep}`)) {
|
|
4238
|
+
this.invalidateColdMemoriesCache();
|
|
4239
|
+
}
|
|
2775
4240
|
await this.appendGeneratedMemoryLifecycleEventFailOpen(
|
|
2776
4241
|
"storage.writeMemoryFrontmatter",
|
|
2777
4242
|
{
|
|
@@ -2816,7 +4281,7 @@ ${memory.content}
|
|
|
2816
4281
|
const expiresAt = new Date(m.frontmatter.expiresAt).getTime();
|
|
2817
4282
|
if (expiresAt < now) {
|
|
2818
4283
|
try {
|
|
2819
|
-
await
|
|
4284
|
+
await unlink2(m.path);
|
|
2820
4285
|
deleted.push(m);
|
|
2821
4286
|
log.debug(`cleaned expired memory ${m.frontmatter.id} (TTL expired)`);
|
|
2822
4287
|
} catch {
|
|
@@ -2830,9 +4295,9 @@ ${memory.content}
|
|
|
2830
4295
|
return deleted;
|
|
2831
4296
|
}
|
|
2832
4297
|
async loadBuffer() {
|
|
2833
|
-
const bufferPath =
|
|
4298
|
+
const bufferPath = path5.join(this.stateDir, "buffer.json");
|
|
2834
4299
|
try {
|
|
2835
|
-
const raw = await
|
|
4300
|
+
const raw = await readFile3(bufferPath, "utf-8");
|
|
2836
4301
|
return JSON.parse(raw);
|
|
2837
4302
|
} catch {
|
|
2838
4303
|
return { turns: [], lastExtractionAt: null, extractionCount: 0 };
|
|
@@ -2840,28 +4305,44 @@ ${memory.content}
|
|
|
2840
4305
|
}
|
|
2841
4306
|
async saveBuffer(state) {
|
|
2842
4307
|
await this.ensureDirectories();
|
|
2843
|
-
const bufferPath =
|
|
2844
|
-
await
|
|
4308
|
+
const bufferPath = path5.join(this.stateDir, "buffer.json");
|
|
4309
|
+
await writeFile3(bufferPath, JSON.stringify(state, null, 2), "utf-8");
|
|
2845
4310
|
}
|
|
2846
4311
|
async loadMeta() {
|
|
2847
|
-
const metaPath =
|
|
4312
|
+
const metaPath = path5.join(this.stateDir, "meta.json");
|
|
2848
4313
|
try {
|
|
2849
|
-
const raw = await
|
|
2850
|
-
|
|
4314
|
+
const raw = await readFile3(metaPath, "utf-8");
|
|
4315
|
+
const parsed = JSON.parse(raw);
|
|
4316
|
+
return {
|
|
4317
|
+
extractionCount: typeof parsed.extractionCount === "number" ? parsed.extractionCount : 0,
|
|
4318
|
+
lastExtractionAt: parsed.lastExtractionAt ?? null,
|
|
4319
|
+
lastConsolidationAt: parsed.lastConsolidationAt ?? null,
|
|
4320
|
+
totalMemories: typeof parsed.totalMemories === "number" ? parsed.totalMemories : 0,
|
|
4321
|
+
totalEntities: typeof parsed.totalEntities === "number" ? parsed.totalEntities : 0,
|
|
4322
|
+
processedExtractionFingerprints: Array.isArray(
|
|
4323
|
+
parsed.processedExtractionFingerprints
|
|
4324
|
+
) ? parsed.processedExtractionFingerprints.filter(
|
|
4325
|
+
(entry) => entry && typeof entry === "object" && typeof entry.fingerprint === "string" && typeof entry.observedAt === "string"
|
|
4326
|
+
).map((entry) => ({
|
|
4327
|
+
fingerprint: entry.fingerprint,
|
|
4328
|
+
observedAt: entry.observedAt
|
|
4329
|
+
})) : []
|
|
4330
|
+
};
|
|
2851
4331
|
} catch {
|
|
2852
4332
|
return {
|
|
2853
4333
|
extractionCount: 0,
|
|
2854
4334
|
lastExtractionAt: null,
|
|
2855
4335
|
lastConsolidationAt: null,
|
|
2856
4336
|
totalMemories: 0,
|
|
2857
|
-
totalEntities: 0
|
|
4337
|
+
totalEntities: 0,
|
|
4338
|
+
processedExtractionFingerprints: []
|
|
2858
4339
|
};
|
|
2859
4340
|
}
|
|
2860
4341
|
}
|
|
2861
4342
|
async saveMeta(state) {
|
|
2862
4343
|
await this.ensureDirectories();
|
|
2863
|
-
const metaPath =
|
|
2864
|
-
await
|
|
4344
|
+
const metaPath = path5.join(this.stateDir, "meta.json");
|
|
4345
|
+
await writeFile3(metaPath, JSON.stringify(state, null, 2), "utf-8");
|
|
2865
4346
|
}
|
|
2866
4347
|
async appendMemoryActionEvents(events) {
|
|
2867
4348
|
if (events.length === 0) return 0;
|
|
@@ -2898,7 +4379,7 @@ ${memory.content}
|
|
|
2898
4379
|
await this.ensureDirectories();
|
|
2899
4380
|
let existingKeys = /* @__PURE__ */ new Set();
|
|
2900
4381
|
try {
|
|
2901
|
-
const raw = await
|
|
4382
|
+
const raw = await readFile3(this.behaviorSignalsPath, "utf-8");
|
|
2902
4383
|
const lines = raw.split("\n");
|
|
2903
4384
|
for (const line of lines) {
|
|
2904
4385
|
const row = line.trim();
|
|
@@ -2934,7 +4415,7 @@ ${memory.content}
|
|
|
2934
4415
|
async appendReextractJobs(events) {
|
|
2935
4416
|
if (events.length === 0) return 0;
|
|
2936
4417
|
await this.ensureDirectories();
|
|
2937
|
-
const filePath =
|
|
4418
|
+
const filePath = path5.join(this.stateDir, "reextract-jobs.jsonl");
|
|
2938
4419
|
const lines = events.map((event) => JSON.stringify(event)).join("\n") + "\n";
|
|
2939
4420
|
try {
|
|
2940
4421
|
await appendFile(filePath, lines, "utf-8");
|
|
@@ -2945,9 +4426,9 @@ ${memory.content}
|
|
|
2945
4426
|
}
|
|
2946
4427
|
async readReextractJobs(limit = 200) {
|
|
2947
4428
|
const safeLimit = Number.isFinite(limit) ? Math.max(1, Math.min(1e3, Math.floor(limit))) : 200;
|
|
2948
|
-
const filePath =
|
|
4429
|
+
const filePath = path5.join(this.stateDir, "reextract-jobs.jsonl");
|
|
2949
4430
|
try {
|
|
2950
|
-
const raw = await
|
|
4431
|
+
const raw = await readFile3(filePath, "utf-8");
|
|
2951
4432
|
const lines = raw.split("\n").filter((line) => line.trim().length > 0);
|
|
2952
4433
|
const parsed = [];
|
|
2953
4434
|
for (const line of lines) {
|
|
@@ -2975,7 +4456,7 @@ ${memory.content}
|
|
|
2975
4456
|
const cappedLimit = Math.max(0, Math.floor(limit));
|
|
2976
4457
|
if (cappedLimit === 0) return [];
|
|
2977
4458
|
try {
|
|
2978
|
-
const raw = await
|
|
4459
|
+
const raw = await readFile3(this.behaviorSignalsPath, "utf-8");
|
|
2979
4460
|
const out = [];
|
|
2980
4461
|
const lines = raw.split("\n");
|
|
2981
4462
|
for (let i = lines.length - 1; i >= 0 && out.length < cappedLimit; i -= 1) {
|
|
@@ -2998,7 +4479,7 @@ ${memory.content}
|
|
|
2998
4479
|
const cappedLimit = Math.max(0, Math.floor(limit));
|
|
2999
4480
|
if (cappedLimit === 0) return [];
|
|
3000
4481
|
try {
|
|
3001
|
-
const raw = await
|
|
4482
|
+
const raw = await readFile3(this.memoryActionsPath, "utf-8");
|
|
3002
4483
|
const out = [];
|
|
3003
4484
|
const lines = raw.split("\n");
|
|
3004
4485
|
for (let i = lines.length - 1; i >= 0 && out.length < cappedLimit; i -= 1) {
|
|
@@ -3019,7 +4500,7 @@ ${memory.content}
|
|
|
3019
4500
|
}
|
|
3020
4501
|
async readAllMemoryLifecycleEvents() {
|
|
3021
4502
|
try {
|
|
3022
|
-
const raw = await
|
|
4503
|
+
const raw = await readFile3(this.memoryLifecycleLedgerPath, "utf-8");
|
|
3023
4504
|
const out = [];
|
|
3024
4505
|
const lines = raw.split("\n");
|
|
3025
4506
|
for (const line of lines) {
|
|
@@ -3046,34 +4527,34 @@ ${memory.content}
|
|
|
3046
4527
|
}
|
|
3047
4528
|
async writeCompressionGuidelines(content) {
|
|
3048
4529
|
await this.ensureDirectories();
|
|
3049
|
-
await
|
|
4530
|
+
await writeFile3(this.compressionGuidelinesPath, content, "utf-8");
|
|
3050
4531
|
}
|
|
3051
4532
|
async readCompressionGuidelines() {
|
|
3052
4533
|
try {
|
|
3053
|
-
return await
|
|
4534
|
+
return await readFile3(this.compressionGuidelinesPath, "utf-8");
|
|
3054
4535
|
} catch {
|
|
3055
4536
|
return null;
|
|
3056
4537
|
}
|
|
3057
4538
|
}
|
|
3058
4539
|
async writeCompressionGuidelineDraft(content) {
|
|
3059
4540
|
await this.ensureDirectories();
|
|
3060
|
-
await
|
|
4541
|
+
await writeFile3(this.compressionGuidelineDraftPath, content, "utf-8");
|
|
3061
4542
|
}
|
|
3062
4543
|
async readCompressionGuidelineDraft() {
|
|
3063
4544
|
try {
|
|
3064
|
-
return await
|
|
4545
|
+
return await readFile3(this.compressionGuidelineDraftPath, "utf-8");
|
|
3065
4546
|
} catch {
|
|
3066
4547
|
return null;
|
|
3067
4548
|
}
|
|
3068
4549
|
}
|
|
3069
4550
|
async writeCompressionGuidelineOptimizerState(state) {
|
|
3070
4551
|
await this.ensureDirectories();
|
|
3071
|
-
await
|
|
4552
|
+
await writeFile3(this.compressionGuidelineStatePath, `${JSON.stringify(state, null, 2)}
|
|
3072
4553
|
`, "utf-8");
|
|
3073
4554
|
}
|
|
3074
4555
|
async writeCompressionGuidelineDraftState(state) {
|
|
3075
4556
|
await this.ensureDirectories();
|
|
3076
|
-
await
|
|
4557
|
+
await writeFile3(this.compressionGuidelineDraftStatePath, `${JSON.stringify(state, null, 2)}
|
|
3077
4558
|
`, "utf-8");
|
|
3078
4559
|
}
|
|
3079
4560
|
async readCompressionGuidelineOptimizerState() {
|
|
@@ -3095,8 +4576,8 @@ ${memory.content}
|
|
|
3095
4576
|
return false;
|
|
3096
4577
|
}
|
|
3097
4578
|
if (draftState.contentHash) {
|
|
3098
|
-
const
|
|
3099
|
-
if (
|
|
4579
|
+
const contentHash2 = createHash2("sha256").update(draftContent).digest("hex");
|
|
4580
|
+
if (contentHash2 !== draftState.contentHash) return false;
|
|
3100
4581
|
}
|
|
3101
4582
|
await this.writeCompressionGuidelines(draftContent);
|
|
3102
4583
|
await this.writeCompressionGuidelineOptimizerState({
|
|
@@ -3104,8 +4585,8 @@ ${memory.content}
|
|
|
3104
4585
|
activationState: "active"
|
|
3105
4586
|
});
|
|
3106
4587
|
await Promise.all([
|
|
3107
|
-
|
|
3108
|
-
|
|
4588
|
+
unlink2(this.compressionGuidelineDraftPath).catch(() => void 0),
|
|
4589
|
+
unlink2(this.compressionGuidelineDraftStatePath).catch(() => void 0)
|
|
3109
4590
|
]);
|
|
3110
4591
|
return true;
|
|
3111
4592
|
}
|
|
@@ -3122,12 +4603,12 @@ ${memory.content}
|
|
|
3122
4603
|
return typeof rule.action === "string" && typeof rule.delta === "number" && Number.isFinite(rule.delta) && (rule.direction === "increase" || rule.direction === "decrease" || rule.direction === "hold") && (rule.confidence === "low" || rule.confidence === "medium" || rule.confidence === "high") && Array.isArray(rule.notes) && rule.notes.every((note) => typeof note === "string");
|
|
3123
4604
|
};
|
|
3124
4605
|
try {
|
|
3125
|
-
const raw = await
|
|
4606
|
+
const raw = await readFile3(filePath, "utf-8");
|
|
3126
4607
|
const parsed = JSON.parse(raw);
|
|
3127
4608
|
const sourceWindow = parsed?.sourceWindow;
|
|
3128
4609
|
const eventCounts = parsed?.eventCounts;
|
|
3129
4610
|
const activationState = parsed?.activationState === "draft" || parsed?.activationState === "active" ? parsed.activationState : void 0;
|
|
3130
|
-
const
|
|
4611
|
+
const contentHash2 = typeof parsed?.contentHash === "string" && parsed.contentHash.length > 0 ? parsed.contentHash : void 0;
|
|
3131
4612
|
const actionSummaries = Array.isArray(parsed?.actionSummaries) ? parsed.actionSummaries.filter(isValidActionSummary) : void 0;
|
|
3132
4613
|
const ruleUpdates = Array.isArray(parsed?.ruleUpdates) ? parsed.ruleUpdates.filter(isValidRuleUpdate) : void 0;
|
|
3133
4614
|
if (!isFiniteNonNegativeInteger(parsed?.version) || typeof parsed?.updatedAt !== "string" || parsed.updatedAt.length === 0 || !sourceWindow || typeof sourceWindow.from !== "string" || sourceWindow.from.length === 0 || typeof sourceWindow.to !== "string" || sourceWindow.to.length === 0 || !eventCounts || !isFiniteNonNegativeInteger(eventCounts.total) || !isFiniteNonNegativeInteger(eventCounts.applied) || !isFiniteNonNegativeInteger(eventCounts.skipped) || !isFiniteNonNegativeInteger(eventCounts.failed) || !isFiniteNonNegativeInteger(parsed?.guidelineVersion)) {
|
|
@@ -3147,7 +4628,7 @@ ${memory.content}
|
|
|
3147
4628
|
failed: eventCounts.failed
|
|
3148
4629
|
},
|
|
3149
4630
|
guidelineVersion: parsed.guidelineVersion,
|
|
3150
|
-
...
|
|
4631
|
+
...contentHash2 ? { contentHash: contentHash2 } : {},
|
|
3151
4632
|
...activationState ? { activationState } : {},
|
|
3152
4633
|
...actionSummaries ? { actionSummaries } : {},
|
|
3153
4634
|
...ruleUpdates ? { ruleUpdates } : {}
|
|
@@ -3158,11 +4639,11 @@ ${memory.content}
|
|
|
3158
4639
|
}
|
|
3159
4640
|
async writeIdentityAnchor(content) {
|
|
3160
4641
|
await this.ensureDirectories();
|
|
3161
|
-
await
|
|
4642
|
+
await writeFile3(this.identityAnchorPath, content, "utf-8");
|
|
3162
4643
|
}
|
|
3163
4644
|
async readIdentityAnchor() {
|
|
3164
4645
|
try {
|
|
3165
|
-
return await
|
|
4646
|
+
return await readFile3(this.identityAnchorPath, "utf-8");
|
|
3166
4647
|
} catch {
|
|
3167
4648
|
return null;
|
|
3168
4649
|
}
|
|
@@ -3174,8 +4655,8 @@ ${memory.content}
|
|
|
3174
4655
|
const date = nowIso.slice(0, 10);
|
|
3175
4656
|
const id = this.generateId("incident");
|
|
3176
4657
|
const incident = createContinuityIncidentRecord(id, input, nowIso);
|
|
3177
|
-
const filePath =
|
|
3178
|
-
await
|
|
4658
|
+
const filePath = path5.join(this.identityIncidentsDir, `${date}-${id}.md`);
|
|
4659
|
+
await writeFile3(filePath, serializeContinuityIncident(incident), "utf-8");
|
|
3179
4660
|
return { ...incident, filePath };
|
|
3180
4661
|
}
|
|
3181
4662
|
async readContinuityIncidents(limit = 200, state = "all") {
|
|
@@ -3187,9 +4668,9 @@ ${memory.content}
|
|
|
3187
4668
|
const incidents = [];
|
|
3188
4669
|
for (const file of candidates) {
|
|
3189
4670
|
if (incidents.length >= cappedLimit) break;
|
|
3190
|
-
const filePath =
|
|
4671
|
+
const filePath = path5.join(this.identityIncidentsDir, file);
|
|
3191
4672
|
try {
|
|
3192
|
-
const raw = await
|
|
4673
|
+
const raw = await readFile3(filePath, "utf-8");
|
|
3193
4674
|
const parsed = parseContinuityIncident(raw);
|
|
3194
4675
|
if (!parsed) continue;
|
|
3195
4676
|
if (state !== "all" && parsed.state !== state) continue;
|
|
@@ -3208,33 +4689,33 @@ ${memory.content}
|
|
|
3208
4689
|
if (!target || !directFilePath) return null;
|
|
3209
4690
|
if (target.state === "closed") return target;
|
|
3210
4691
|
const closed = closeContinuityIncidentRecord(target, closure, (/* @__PURE__ */ new Date()).toISOString());
|
|
3211
|
-
await
|
|
4692
|
+
await writeFile3(directFilePath, serializeContinuityIncident(closed), "utf-8");
|
|
3212
4693
|
return { ...closed, filePath: directFilePath };
|
|
3213
4694
|
}
|
|
3214
4695
|
async writeIdentityAudit(period, key, content) {
|
|
3215
4696
|
await this.ensureDirectories();
|
|
3216
4697
|
const safeKey = this.sanitizeIdentityAuditKey(key);
|
|
3217
4698
|
const dir = period === "weekly" ? this.identityAuditsWeeklyDir : this.identityAuditsMonthlyDir;
|
|
3218
|
-
const filePath =
|
|
3219
|
-
await
|
|
4699
|
+
const filePath = path5.join(dir, `${safeKey}.md`);
|
|
4700
|
+
await writeFile3(filePath, content, "utf-8");
|
|
3220
4701
|
return filePath;
|
|
3221
4702
|
}
|
|
3222
4703
|
async readIdentityAudit(period, key) {
|
|
3223
4704
|
try {
|
|
3224
4705
|
const safeKey = this.sanitizeIdentityAuditKey(key);
|
|
3225
4706
|
const dir = period === "weekly" ? this.identityAuditsWeeklyDir : this.identityAuditsMonthlyDir;
|
|
3226
|
-
return await
|
|
4707
|
+
return await readFile3(path5.join(dir, `${safeKey}.md`), "utf-8");
|
|
3227
4708
|
} catch {
|
|
3228
4709
|
return null;
|
|
3229
4710
|
}
|
|
3230
4711
|
}
|
|
3231
4712
|
async writeIdentityImprovementLoops(content) {
|
|
3232
4713
|
await this.ensureDirectories();
|
|
3233
|
-
await
|
|
4714
|
+
await writeFile3(this.identityImprovementLoopsPath, content, "utf-8");
|
|
3234
4715
|
}
|
|
3235
4716
|
async readIdentityImprovementLoops() {
|
|
3236
4717
|
try {
|
|
3237
|
-
return await
|
|
4718
|
+
return await readFile3(this.identityImprovementLoopsPath, "utf-8");
|
|
3238
4719
|
} catch {
|
|
3239
4720
|
return null;
|
|
3240
4721
|
}
|
|
@@ -3272,7 +4753,7 @@ ${memory.content}
|
|
|
3272
4753
|
}
|
|
3273
4754
|
async readContinuityIncidentFile(filePath) {
|
|
3274
4755
|
try {
|
|
3275
|
-
const raw = await
|
|
4756
|
+
const raw = await readFile3(filePath, "utf-8");
|
|
3276
4757
|
const parsed = parseContinuityIncident(raw);
|
|
3277
4758
|
return parsed ? { ...parsed, filePath } : null;
|
|
3278
4759
|
} catch {
|
|
@@ -3283,12 +4764,12 @@ ${memory.content}
|
|
|
3283
4764
|
const fileNames = await this.readContinuityIncidentFileNames();
|
|
3284
4765
|
const directMatch = fileNames.find((name) => name.endsWith(`-${id}.md`));
|
|
3285
4766
|
if (directMatch) {
|
|
3286
|
-
const directPath =
|
|
4767
|
+
const directPath = path5.join(this.identityIncidentsDir, directMatch);
|
|
3287
4768
|
const parsed = await this.readContinuityIncidentFile(directPath);
|
|
3288
4769
|
if (parsed?.id === id) return directPath;
|
|
3289
4770
|
}
|
|
3290
4771
|
for (const fileName of fileNames) {
|
|
3291
|
-
const filePath =
|
|
4772
|
+
const filePath = path5.join(this.identityIncidentsDir, fileName);
|
|
3292
4773
|
const parsed = await this.readContinuityIncidentFile(filePath);
|
|
3293
4774
|
if (parsed?.id === id) return filePath;
|
|
3294
4775
|
}
|
|
@@ -3302,7 +4783,7 @@ ${memory.content}
|
|
|
3302
4783
|
return trimmed;
|
|
3303
4784
|
}
|
|
3304
4785
|
async writeQuestion(question, context, priority) {
|
|
3305
|
-
await
|
|
4786
|
+
await mkdir3(this.questionsDir, { recursive: true });
|
|
3306
4787
|
const id = this.generateId("q");
|
|
3307
4788
|
const frontmatter = {
|
|
3308
4789
|
id,
|
|
@@ -3318,8 +4799,8 @@ ${question}
|
|
|
3318
4799
|
|
|
3319
4800
|
**Context:** ${context}
|
|
3320
4801
|
`;
|
|
3321
|
-
const filePath =
|
|
3322
|
-
await
|
|
4802
|
+
const filePath = path5.join(this.questionsDir, `${id}.md`);
|
|
4803
|
+
await writeFile3(filePath, content, "utf-8");
|
|
3323
4804
|
log.debug(`wrote question ${id} to ${filePath}`);
|
|
3324
4805
|
this.invalidateQuestionsCache();
|
|
3325
4806
|
return id;
|
|
@@ -3342,8 +4823,8 @@ ${question}
|
|
|
3342
4823
|
const questions = [];
|
|
3343
4824
|
for (const file of files) {
|
|
3344
4825
|
if (!file.endsWith(".md")) continue;
|
|
3345
|
-
const filePath =
|
|
3346
|
-
const raw = await
|
|
4826
|
+
const filePath = path5.join(this.questionsDir, file);
|
|
4827
|
+
const raw = await readFile3(filePath, "utf-8");
|
|
3347
4828
|
const parsed = this.parseQuestionFile(raw, filePath);
|
|
3348
4829
|
if (parsed) {
|
|
3349
4830
|
questions.push(parsed);
|
|
@@ -3365,7 +4846,7 @@ ${question}
|
|
|
3365
4846
|
if (!match) return null;
|
|
3366
4847
|
const frontmatterStr = match[1];
|
|
3367
4848
|
const body = match[2].trim();
|
|
3368
|
-
const id = this.extractFrontmatterValue(frontmatterStr, "id") ??
|
|
4849
|
+
const id = this.extractFrontmatterValue(frontmatterStr, "id") ?? path5.basename(filePath, ".md");
|
|
3369
4850
|
const created = this.extractFrontmatterValue(frontmatterStr, "created") ?? "";
|
|
3370
4851
|
const priority = parseFloat(
|
|
3371
4852
|
this.extractFrontmatterValue(frontmatterStr, "priority") ?? "0.5"
|
|
@@ -3386,7 +4867,7 @@ ${question}
|
|
|
3386
4867
|
const questions = await this.readQuestions();
|
|
3387
4868
|
const q = questions.find((q2) => q2.id === id);
|
|
3388
4869
|
if (!q) return false;
|
|
3389
|
-
let raw = await
|
|
4870
|
+
let raw = await readFile3(q.filePath, "utf-8");
|
|
3390
4871
|
raw = raw.replace(/resolved: false/, "resolved: true");
|
|
3391
4872
|
raw = raw.replace(
|
|
3392
4873
|
/---\n\n/,
|
|
@@ -3395,7 +4876,7 @@ ${question}
|
|
|
3395
4876
|
|
|
3396
4877
|
`
|
|
3397
4878
|
);
|
|
3398
|
-
await
|
|
4879
|
+
await writeFile3(q.filePath, raw, "utf-8");
|
|
3399
4880
|
log.debug(`resolved question ${id}`);
|
|
3400
4881
|
return true;
|
|
3401
4882
|
}
|
|
@@ -3405,14 +4886,14 @@ ${question}
|
|
|
3405
4886
|
async readIdentity(workspaceDir, namespace) {
|
|
3406
4887
|
const identityPath = this.identityFilePath(workspaceDir, namespace);
|
|
3407
4888
|
try {
|
|
3408
|
-
return await
|
|
4889
|
+
return await readFile3(identityPath, "utf-8");
|
|
3409
4890
|
} catch {
|
|
3410
4891
|
return "";
|
|
3411
4892
|
}
|
|
3412
4893
|
}
|
|
3413
4894
|
async writeIdentity(workspaceDir, content, namespace) {
|
|
3414
4895
|
const identityPath = this.identityFilePath(workspaceDir, namespace);
|
|
3415
|
-
await
|
|
4896
|
+
await writeFile3(identityPath, content, "utf-8");
|
|
3416
4897
|
log.debug(`wrote consolidated IDENTITY.md (${content.length} chars)`);
|
|
3417
4898
|
}
|
|
3418
4899
|
/** Max size for IDENTITY.md before we stop appending reflections (15KB leaves room under 20KB gateway limit) */
|
|
@@ -3423,22 +4904,22 @@ ${question}
|
|
|
3423
4904
|
const identityPath = this.identityFilePath(workspaceDir, opts?.namespace);
|
|
3424
4905
|
let existing = "";
|
|
3425
4906
|
try {
|
|
3426
|
-
existing = await
|
|
4907
|
+
existing = await readFile3(identityPath, "utf-8");
|
|
3427
4908
|
} catch {
|
|
3428
4909
|
}
|
|
3429
4910
|
const hygiene = opts?.hygiene;
|
|
3430
|
-
const rotateEnabled = hygiene?.enabled === true && hygiene.rotateEnabled === true && Array.isArray(hygiene.rotatePaths) && hygiene.rotatePaths.includes(
|
|
4911
|
+
const rotateEnabled = hygiene?.enabled === true && hygiene.rotateEnabled === true && Array.isArray(hygiene.rotatePaths) && hygiene.rotatePaths.includes(path5.basename(identityPath));
|
|
3431
4912
|
if (rotateEnabled) {
|
|
3432
4913
|
const maxBytes = hygiene.rotateMaxBytes;
|
|
3433
4914
|
if (existing.length > maxBytes) {
|
|
3434
|
-
const archiveDir =
|
|
4915
|
+
const archiveDir = path5.join(workspaceDir, hygiene.archiveDir);
|
|
3435
4916
|
const { newContent } = await rotateMarkdownFileToArchive({
|
|
3436
4917
|
filePath: identityPath,
|
|
3437
4918
|
archiveDir,
|
|
3438
4919
|
archivePrefix: "IDENTITY",
|
|
3439
4920
|
keepTailChars: hygiene.rotateKeepTailChars
|
|
3440
4921
|
});
|
|
3441
|
-
await
|
|
4922
|
+
await writeFile3(identityPath, newContent, "utf-8");
|
|
3442
4923
|
existing = newContent;
|
|
3443
4924
|
log.info(
|
|
3444
4925
|
`rotated IDENTITY.md to archive (size=${existing.length} chars, maxBytes=${maxBytes})`
|
|
@@ -3469,24 +4950,24 @@ ${question}
|
|
|
3469
4950
|
|
|
3470
4951
|
${reflection}
|
|
3471
4952
|
`;
|
|
3472
|
-
await
|
|
4953
|
+
await writeFile3(identityPath, existing + section, "utf-8");
|
|
3473
4954
|
log.debug(`appended reflection to ${identityPath}`);
|
|
3474
4955
|
}
|
|
3475
4956
|
async readIdentityReflections() {
|
|
3476
4957
|
try {
|
|
3477
|
-
return await
|
|
4958
|
+
return await readFile3(this.identityReflectionsPath, "utf-8");
|
|
3478
4959
|
} catch {
|
|
3479
4960
|
return null;
|
|
3480
4961
|
}
|
|
3481
4962
|
}
|
|
3482
4963
|
async writeIdentityReflections(content) {
|
|
3483
|
-
await
|
|
3484
|
-
await
|
|
4964
|
+
await mkdir3(this.identityDir, { recursive: true });
|
|
4965
|
+
await writeFile3(this.identityReflectionsPath, content, "utf-8");
|
|
3485
4966
|
}
|
|
3486
4967
|
async appendIdentityReflection(reflection) {
|
|
3487
4968
|
let existing = "";
|
|
3488
4969
|
try {
|
|
3489
|
-
existing = await
|
|
4970
|
+
existing = await readFile3(this.identityReflectionsPath, "utf-8");
|
|
3490
4971
|
} catch {
|
|
3491
4972
|
}
|
|
3492
4973
|
if (existing.length > _StorageManager.IDENTITY_MAX_BYTES) {
|
|
@@ -3511,8 +4992,8 @@ ${reflection}
|
|
|
3511
4992
|
|
|
3512
4993
|
${reflection}
|
|
3513
4994
|
`;
|
|
3514
|
-
await
|
|
3515
|
-
await
|
|
4995
|
+
await mkdir3(this.identityDir, { recursive: true });
|
|
4996
|
+
await writeFile3(this.identityReflectionsPath, `${existing.trimEnd()}${section}`, "utf-8");
|
|
3516
4997
|
log.debug(`appended namespace-local reflection to ${this.identityReflectionsPath}`);
|
|
3517
4998
|
}
|
|
3518
4999
|
// ---------------------------------------------------------------------------
|
|
@@ -3523,11 +5004,11 @@ ${reflection}
|
|
|
3523
5004
|
* Deduplicates by target+label.
|
|
3524
5005
|
*/
|
|
3525
5006
|
async addEntityRelationship(name, rel) {
|
|
3526
|
-
const filePath =
|
|
5007
|
+
const filePath = path5.join(this.entitiesDir, `${name}.md`);
|
|
3527
5008
|
let entity;
|
|
3528
5009
|
try {
|
|
3529
|
-
const content = await
|
|
3530
|
-
entity = parseEntityFile(content);
|
|
5010
|
+
const content = await readFile3(filePath, "utf-8");
|
|
5011
|
+
entity = parseEntityFile(content, this.entitySchemas);
|
|
3531
5012
|
} catch {
|
|
3532
5013
|
log.debug(`addEntityRelationship: entity file ${name}.md not found`);
|
|
3533
5014
|
return;
|
|
@@ -3538,7 +5019,7 @@ ${reflection}
|
|
|
3538
5019
|
if (exists) return;
|
|
3539
5020
|
entity.relationships.push(rel);
|
|
3540
5021
|
entity.updated = (/* @__PURE__ */ new Date()).toISOString();
|
|
3541
|
-
await
|
|
5022
|
+
await writeFile3(filePath, serializeEntityFile(entity, this.entitySchemas), "utf-8");
|
|
3542
5023
|
this.invalidateKnowledgeIndexCache();
|
|
3543
5024
|
}
|
|
3544
5025
|
/**
|
|
@@ -3546,11 +5027,11 @@ ${reflection}
|
|
|
3546
5027
|
* Prepends to the beginning, prunes oldest entries beyond maxEntries.
|
|
3547
5028
|
*/
|
|
3548
5029
|
async addEntityActivity(name, entry, maxEntries) {
|
|
3549
|
-
const filePath =
|
|
5030
|
+
const filePath = path5.join(this.entitiesDir, `${name}.md`);
|
|
3550
5031
|
let entity;
|
|
3551
5032
|
try {
|
|
3552
|
-
const content = await
|
|
3553
|
-
entity = parseEntityFile(content);
|
|
5033
|
+
const content = await readFile3(filePath, "utf-8");
|
|
5034
|
+
entity = parseEntityFile(content, this.entitySchemas);
|
|
3554
5035
|
} catch {
|
|
3555
5036
|
log.debug(`addEntityActivity: entity file ${name}.md not found`);
|
|
3556
5037
|
return;
|
|
@@ -3560,18 +5041,18 @@ ${reflection}
|
|
|
3560
5041
|
entity.activity = entity.activity.slice(0, maxEntries);
|
|
3561
5042
|
}
|
|
3562
5043
|
entity.updated = (/* @__PURE__ */ new Date()).toISOString();
|
|
3563
|
-
await
|
|
5044
|
+
await writeFile3(filePath, serializeEntityFile(entity, this.entitySchemas), "utf-8");
|
|
3564
5045
|
this.invalidateKnowledgeIndexCache();
|
|
3565
5046
|
}
|
|
3566
5047
|
/**
|
|
3567
5048
|
* Add an alias to an entity file. Deduplicates.
|
|
3568
5049
|
*/
|
|
3569
5050
|
async addEntityAlias(name, alias) {
|
|
3570
|
-
const filePath =
|
|
5051
|
+
const filePath = path5.join(this.entitiesDir, `${name}.md`);
|
|
3571
5052
|
let entity;
|
|
3572
5053
|
try {
|
|
3573
|
-
const content = await
|
|
3574
|
-
entity = parseEntityFile(content);
|
|
5054
|
+
const content = await readFile3(filePath, "utf-8");
|
|
5055
|
+
entity = parseEntityFile(content, this.entitySchemas);
|
|
3575
5056
|
} catch {
|
|
3576
5057
|
log.debug(`addEntityAlias: entity file ${name}.md not found`);
|
|
3577
5058
|
return;
|
|
@@ -3579,28 +5060,142 @@ ${reflection}
|
|
|
3579
5060
|
if (entity.aliases.includes(alias)) return;
|
|
3580
5061
|
entity.aliases.push(alias);
|
|
3581
5062
|
entity.updated = (/* @__PURE__ */ new Date()).toISOString();
|
|
3582
|
-
await
|
|
5063
|
+
await writeFile3(filePath, serializeEntityFile(entity, this.entitySchemas), "utf-8");
|
|
3583
5064
|
this.invalidateKnowledgeIndexCache();
|
|
3584
5065
|
}
|
|
3585
5066
|
/**
|
|
3586
|
-
* Set or
|
|
5067
|
+
* Set or rewrite the synthesis layer of an entity file.
|
|
3587
5068
|
*/
|
|
3588
|
-
async
|
|
3589
|
-
const filePath =
|
|
5069
|
+
async updateEntitySynthesis(name, synthesis, options = {}) {
|
|
5070
|
+
const filePath = path5.join(this.entitiesDir, `${name}.md`);
|
|
3590
5071
|
let entity;
|
|
3591
5072
|
try {
|
|
3592
|
-
const content = await
|
|
3593
|
-
entity = parseEntityFile(content);
|
|
5073
|
+
const content = await readFile3(filePath, "utf-8");
|
|
5074
|
+
entity = parseEntityFile(content, this.entitySchemas);
|
|
3594
5075
|
} catch {
|
|
3595
|
-
log.debug(`
|
|
5076
|
+
log.debug(`updateEntitySynthesis: entity file ${name}.md not found`);
|
|
3596
5077
|
return;
|
|
3597
5078
|
}
|
|
3598
|
-
entity.
|
|
3599
|
-
entity.updated
|
|
3600
|
-
|
|
5079
|
+
const updatedAt = options.updatedAt?.trim() || entity.synthesisUpdatedAt?.trim() || void 0;
|
|
5080
|
+
const entityUpdatedAt = options.entityUpdatedAt?.trim() || updatedAt || entity.updated || (/* @__PURE__ */ new Date()).toISOString();
|
|
5081
|
+
const synthesisTimelineCount = Number.isInteger(options.synthesisTimelineCount) && (options.synthesisTimelineCount ?? 0) >= 0 ? options.synthesisTimelineCount : void 0;
|
|
5082
|
+
const synthesisStructuredFactCount = Number.isInteger(options.synthesisStructuredFactCount) && (options.synthesisStructuredFactCount ?? 0) >= 0 ? options.synthesisStructuredFactCount : countEntityStructuredFacts(entity);
|
|
5083
|
+
const synthesisStructuredFactDigest = options.synthesisStructuredFactDigest?.trim() || fingerprintEntityStructuredFacts(entity);
|
|
5084
|
+
entity.synthesis = synthesis.trim();
|
|
5085
|
+
entity.summary = entity.synthesis;
|
|
5086
|
+
entity.synthesisUpdatedAt = updatedAt;
|
|
5087
|
+
entity.synthesisTimelineCount = synthesisTimelineCount;
|
|
5088
|
+
entity.synthesisStructuredFactCount = synthesisStructuredFactCount;
|
|
5089
|
+
entity.synthesisStructuredFactDigest = synthesisStructuredFactDigest;
|
|
5090
|
+
entity.synthesisVersion = Math.max(0, entity.synthesisVersion ?? 0) + (options.incrementVersion === false ? 0 : 1);
|
|
5091
|
+
entity.updated = entityUpdatedAt;
|
|
5092
|
+
await writeFile3(filePath, serializeEntityFile(entity, this.entitySchemas), "utf-8");
|
|
5093
|
+
await this.removeEntitySynthesisQueueEntries([
|
|
5094
|
+
.../* @__PURE__ */ new Set([name, normalizeEntityName(entity.name, entity.type)])
|
|
5095
|
+
]);
|
|
3601
5096
|
this.invalidateKnowledgeIndexCache();
|
|
3602
5097
|
this.bumpMemoryStatusVersion();
|
|
3603
5098
|
}
|
|
5099
|
+
/**
|
|
5100
|
+
* Backward-compatible alias for legacy callers/tests.
|
|
5101
|
+
*/
|
|
5102
|
+
async updateEntitySummary(name, summary) {
|
|
5103
|
+
const updatedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
5104
|
+
let synthesisTimelineCount;
|
|
5105
|
+
try {
|
|
5106
|
+
const filePath = path5.join(this.entitiesDir, `${name}.md`);
|
|
5107
|
+
const content = await readFile3(filePath, "utf-8");
|
|
5108
|
+
synthesisTimelineCount = parseEntityFile(content, this.entitySchemas).timeline.length;
|
|
5109
|
+
} catch {
|
|
5110
|
+
synthesisTimelineCount = void 0;
|
|
5111
|
+
}
|
|
5112
|
+
await this.updateEntitySynthesis(name, summary, {
|
|
5113
|
+
entityUpdatedAt: updatedAt,
|
|
5114
|
+
synthesisTimelineCount,
|
|
5115
|
+
updatedAt
|
|
5116
|
+
});
|
|
5117
|
+
}
|
|
5118
|
+
async readEntitySynthesisQueue() {
|
|
5119
|
+
try {
|
|
5120
|
+
const raw = await readFile3(this.entitySynthesisQueuePath, "utf-8");
|
|
5121
|
+
const parsed = JSON.parse(raw);
|
|
5122
|
+
return Array.isArray(parsed.entityNames) ? parsed.entityNames.filter((value) => typeof value === "string") : [];
|
|
5123
|
+
} catch {
|
|
5124
|
+
return [];
|
|
5125
|
+
}
|
|
5126
|
+
}
|
|
5127
|
+
async refreshEntitySynthesisQueue() {
|
|
5128
|
+
const entityNames = await this.listEntityNames();
|
|
5129
|
+
const entityQueueEntries = await Promise.all(
|
|
5130
|
+
entityNames.map(async (entityName) => {
|
|
5131
|
+
const raw = await this.readEntity(entityName);
|
|
5132
|
+
if (!raw) return null;
|
|
5133
|
+
return {
|
|
5134
|
+
entityName,
|
|
5135
|
+
entity: parseEntityFile(raw, this.entitySchemas)
|
|
5136
|
+
};
|
|
5137
|
+
})
|
|
5138
|
+
);
|
|
5139
|
+
const staleEntityNames = entityQueueEntries.filter((entry) => entry !== null).filter(({ entity }) => isEntitySynthesisStale(entity)).sort((left, right) => {
|
|
5140
|
+
const leftTs = latestEntityTimelineTimestamp(left.entity) ?? "";
|
|
5141
|
+
const rightTs = latestEntityTimelineTimestamp(right.entity) ?? "";
|
|
5142
|
+
return compareEntityTimestamps(rightTs, leftTs);
|
|
5143
|
+
}).map(({ entityName }) => entityName);
|
|
5144
|
+
await mkdir3(this.stateDir, { recursive: true });
|
|
5145
|
+
await writeFile3(
|
|
5146
|
+
this.entitySynthesisQueuePath,
|
|
5147
|
+
JSON.stringify(
|
|
5148
|
+
{
|
|
5149
|
+
updatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
5150
|
+
entityNames: staleEntityNames
|
|
5151
|
+
},
|
|
5152
|
+
null,
|
|
5153
|
+
2
|
|
5154
|
+
) + "\n",
|
|
5155
|
+
"utf-8"
|
|
5156
|
+
);
|
|
5157
|
+
return staleEntityNames;
|
|
5158
|
+
}
|
|
5159
|
+
async removeEntitySynthesisQueueEntries(entityNames) {
|
|
5160
|
+
if (entityNames.length === 0) return;
|
|
5161
|
+
const queue = await this.readEntitySynthesisQueue();
|
|
5162
|
+
if (queue.length === 0) return;
|
|
5163
|
+
const removals = new Set(entityNames);
|
|
5164
|
+
const nextQueue = queue.filter((name) => !removals.has(name));
|
|
5165
|
+
await mkdir3(this.stateDir, { recursive: true });
|
|
5166
|
+
await writeFile3(
|
|
5167
|
+
this.entitySynthesisQueuePath,
|
|
5168
|
+
JSON.stringify(
|
|
5169
|
+
{
|
|
5170
|
+
updatedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
5171
|
+
entityNames: nextQueue
|
|
5172
|
+
},
|
|
5173
|
+
null,
|
|
5174
|
+
2
|
|
5175
|
+
) + "\n",
|
|
5176
|
+
"utf-8"
|
|
5177
|
+
);
|
|
5178
|
+
}
|
|
5179
|
+
async migrateEntityFilesToCompiledTruthTimeline() {
|
|
5180
|
+
const entityNames = await this.listEntityNames();
|
|
5181
|
+
let migrated = 0;
|
|
5182
|
+
for (const entityName of entityNames) {
|
|
5183
|
+
const raw = await this.readEntity(entityName);
|
|
5184
|
+
if (!raw) continue;
|
|
5185
|
+
const serialized = serializeEntityFile(parseEntityFile(raw, this.entitySchemas), this.entitySchemas);
|
|
5186
|
+
if (raw.trimEnd() === serialized.trimEnd()) continue;
|
|
5187
|
+
await writeFile3(path5.join(this.entitiesDir, `${entityName}.md`), serialized, "utf-8");
|
|
5188
|
+
migrated += 1;
|
|
5189
|
+
}
|
|
5190
|
+
if (migrated > 0) {
|
|
5191
|
+
this.invalidateKnowledgeIndexCache();
|
|
5192
|
+
this.bumpMemoryStatusVersion();
|
|
5193
|
+
}
|
|
5194
|
+
return {
|
|
5195
|
+
total: entityNames.length,
|
|
5196
|
+
migrated
|
|
5197
|
+
};
|
|
5198
|
+
}
|
|
3604
5199
|
// ---------------------------------------------------------------------------
|
|
3605
5200
|
// Scoring + Knowledge Index (Knowledge Graph v7.0)
|
|
3606
5201
|
// ---------------------------------------------------------------------------
|
|
@@ -3610,7 +5205,8 @@ ${reflection}
|
|
|
3610
5205
|
*/
|
|
3611
5206
|
async readAllEntityFiles() {
|
|
3612
5207
|
const currentVersion = this.getMemoryStatusVersion();
|
|
3613
|
-
const
|
|
5208
|
+
const schemaCacheKey = buildEntitySchemaCacheKey(this.entitySchemas);
|
|
5209
|
+
const cached = getCachedEntities(this.baseDir, currentVersion, schemaCacheKey);
|
|
3614
5210
|
if (cached) return cached;
|
|
3615
5211
|
try {
|
|
3616
5212
|
const entries = await readdir(this.entitiesDir);
|
|
@@ -3622,14 +5218,14 @@ ${reflection}
|
|
|
3622
5218
|
const batch = mdFiles.slice(i, i + BATCH_SIZE);
|
|
3623
5219
|
const results = await Promise.all(
|
|
3624
5220
|
batch.map(
|
|
3625
|
-
(entry) =>
|
|
5221
|
+
(entry) => readFile3(path5.join(this.entitiesDir, entry), "utf-8").catch(() => null)
|
|
3626
5222
|
)
|
|
3627
5223
|
);
|
|
3628
5224
|
for (const content of results) {
|
|
3629
|
-
if (content !== null) entities.push(parseEntityFile(content));
|
|
5225
|
+
if (content !== null) entities.push(parseEntityFile(content, this.entitySchemas));
|
|
3630
5226
|
}
|
|
3631
5227
|
}
|
|
3632
|
-
setCachedEntities(this.baseDir, entities, currentVersion);
|
|
5228
|
+
setCachedEntities(this.baseDir, entities, currentVersion, schemaCacheKey);
|
|
3633
5229
|
return entities;
|
|
3634
5230
|
} catch {
|
|
3635
5231
|
return [];
|
|
@@ -3679,7 +5275,7 @@ ${reflection}
|
|
|
3679
5275
|
type: e.type,
|
|
3680
5276
|
score: _StorageManager.scoreEntity(e, now),
|
|
3681
5277
|
factCount: e.facts.length,
|
|
3682
|
-
summary: e.summary,
|
|
5278
|
+
summary: e.synthesis ?? e.summary,
|
|
3683
5279
|
topRelationships: e.relationships.slice(0, 3).map((r) => r.target)
|
|
3684
5280
|
}));
|
|
3685
5281
|
scored.sort((a, b) => b.score - a.score);
|
|
@@ -3742,38 +5338,121 @@ ${rows.join("\n")}
|
|
|
3742
5338
|
const mergedEntity = {
|
|
3743
5339
|
name: "",
|
|
3744
5340
|
type: "other",
|
|
5341
|
+
created: "",
|
|
3745
5342
|
updated: "",
|
|
5343
|
+
extraFrontmatterLines: [],
|
|
5344
|
+
preSectionLines: [],
|
|
3746
5345
|
facts: [],
|
|
3747
5346
|
summary: void 0,
|
|
5347
|
+
synthesis: void 0,
|
|
5348
|
+
synthesisUpdatedAt: void 0,
|
|
5349
|
+
synthesisTimelineCount: void 0,
|
|
5350
|
+
synthesisStructuredFactCount: void 0,
|
|
5351
|
+
synthesisStructuredFactDigest: void 0,
|
|
5352
|
+
synthesisVersion: void 0,
|
|
5353
|
+
timeline: [],
|
|
3748
5354
|
relationships: [],
|
|
3749
5355
|
activity: [],
|
|
3750
|
-
aliases: []
|
|
5356
|
+
aliases: [],
|
|
5357
|
+
structuredSections: [],
|
|
5358
|
+
extraSections: []
|
|
3751
5359
|
};
|
|
3752
5360
|
for (const file of files) {
|
|
3753
|
-
const filePath =
|
|
5361
|
+
const filePath = path5.join(this.entitiesDir, file);
|
|
3754
5362
|
try {
|
|
3755
|
-
const content = await
|
|
3756
|
-
const parsed = parseEntityFile(content);
|
|
5363
|
+
const content = await readFile3(filePath, "utf-8");
|
|
5364
|
+
const parsed = parseEntityFile(content, this.entitySchemas);
|
|
3757
5365
|
if (!mergedEntity.type || mergedEntity.type === "other") {
|
|
3758
5366
|
mergedEntity.type = parsed.type;
|
|
3759
5367
|
}
|
|
3760
|
-
if (!mergedEntity.updated || parsed.updated
|
|
5368
|
+
if (!mergedEntity.updated || compareEntityTimestamps(parsed.updated, mergedEntity.updated) > 0) {
|
|
3761
5369
|
mergedEntity.updated = parsed.updated;
|
|
3762
5370
|
}
|
|
5371
|
+
const parsedCreated = parsed.created || parsed.updated;
|
|
5372
|
+
const mergedCreated = mergedEntity.created?.trim() || "";
|
|
5373
|
+
const parsedCreatedMs = parsedCreated ? Date.parse(parsedCreated) : Number.NaN;
|
|
5374
|
+
const mergedCreatedMs = mergedCreated ? Date.parse(mergedCreated) : Number.NaN;
|
|
5375
|
+
const parsedCreatedIsValid = Number.isFinite(parsedCreatedMs);
|
|
5376
|
+
const mergedCreatedIsValid = Number.isFinite(mergedCreatedMs);
|
|
5377
|
+
if (parsedCreated && (!mergedCreated || parsedCreatedIsValid && !mergedCreatedIsValid || parsedCreatedIsValid && mergedCreatedIsValid && parsedCreatedMs < mergedCreatedMs || !parsedCreatedIsValid && !mergedCreatedIsValid && compareEntityTimestamps(parsedCreated, mergedCreated) < 0)) {
|
|
5378
|
+
mergedEntity.created = parsedCreated;
|
|
5379
|
+
}
|
|
3763
5380
|
if (parsed.name.length > mergedEntity.name.length) {
|
|
3764
5381
|
mergedEntity.name = parsed.name;
|
|
3765
5382
|
}
|
|
3766
|
-
|
|
5383
|
+
const parsedSynthesisUpdatedAt = parsed.synthesisUpdatedAt?.trim() || void 0;
|
|
5384
|
+
const mergedSynthesisUpdatedAt = mergedEntity.synthesisUpdatedAt?.trim() || void 0;
|
|
5385
|
+
if (parsed.synthesis && (!mergedEntity.synthesis || !mergedSynthesisUpdatedAt && Boolean(parsedSynthesisUpdatedAt) || Boolean(mergedSynthesisUpdatedAt) && Boolean(parsedSynthesisUpdatedAt) && compareEntityTimestamps(parsedSynthesisUpdatedAt, mergedSynthesisUpdatedAt) > 0)) {
|
|
5386
|
+
mergedEntity.synthesis = parsed.synthesis;
|
|
5387
|
+
mergedEntity.summary = parsed.synthesis;
|
|
5388
|
+
mergedEntity.synthesisUpdatedAt = parsedSynthesisUpdatedAt;
|
|
5389
|
+
mergedEntity.synthesisTimelineCount = parsed.synthesisTimelineCount;
|
|
5390
|
+
mergedEntity.synthesisStructuredFactCount = parsed.synthesisStructuredFactCount;
|
|
5391
|
+
mergedEntity.synthesisStructuredFactDigest = parsed.synthesisStructuredFactDigest;
|
|
5392
|
+
mergedEntity.synthesisVersion = parsed.synthesisVersion;
|
|
5393
|
+
} else if (!mergedEntity.summary && parsed.summary) {
|
|
3767
5394
|
mergedEntity.summary = parsed.summary;
|
|
5395
|
+
mergedEntity.synthesis = parsed.summary;
|
|
5396
|
+
mergedEntity.synthesisUpdatedAt = parsedSynthesisUpdatedAt;
|
|
5397
|
+
mergedEntity.synthesisTimelineCount = parsed.synthesisTimelineCount;
|
|
5398
|
+
mergedEntity.synthesisStructuredFactCount = parsed.synthesisStructuredFactCount;
|
|
5399
|
+
mergedEntity.synthesisStructuredFactDigest = parsed.synthesisStructuredFactDigest;
|
|
5400
|
+
mergedEntity.synthesisVersion = parsed.synthesisVersion;
|
|
3768
5401
|
}
|
|
3769
|
-
mergedEntity.
|
|
5402
|
+
mergedEntity.timeline.push(...parsed.timeline);
|
|
3770
5403
|
mergedEntity.relationships.push(...parsed.relationships);
|
|
3771
5404
|
mergedEntity.activity.push(...parsed.activity);
|
|
3772
5405
|
mergedEntity.aliases.push(...parsed.aliases);
|
|
5406
|
+
const mergedStructuredSectionMap = new Map(
|
|
5407
|
+
(mergedEntity.structuredSections ?? []).map((section) => [section.key, {
|
|
5408
|
+
...section,
|
|
5409
|
+
facts: [...section.facts]
|
|
5410
|
+
}])
|
|
5411
|
+
);
|
|
5412
|
+
for (const section of parsed.structuredSections ?? []) {
|
|
5413
|
+
const existingSection = mergedStructuredSectionMap.get(section.key);
|
|
5414
|
+
if (!existingSection) {
|
|
5415
|
+
mergedStructuredSectionMap.set(section.key, {
|
|
5416
|
+
key: section.key,
|
|
5417
|
+
title: section.title,
|
|
5418
|
+
facts: [...new Set(section.facts.map((fact) => fact.trim()).filter((fact) => fact.length > 0))]
|
|
5419
|
+
});
|
|
5420
|
+
continue;
|
|
5421
|
+
}
|
|
5422
|
+
const mergedFacts = new Set(existingSection.facts.map((fact) => fact.trim()));
|
|
5423
|
+
for (const fact of section.facts) {
|
|
5424
|
+
const trimmed = fact.trim();
|
|
5425
|
+
if (!trimmed) continue;
|
|
5426
|
+
mergedFacts.add(trimmed);
|
|
5427
|
+
}
|
|
5428
|
+
existingSection.facts = Array.from(mergedFacts);
|
|
5429
|
+
if (!existingSection.title.trim() && section.title.trim()) {
|
|
5430
|
+
existingSection.title = section.title;
|
|
5431
|
+
}
|
|
5432
|
+
}
|
|
5433
|
+
mergedEntity.structuredSections = Array.from(mergedStructuredSectionMap.values());
|
|
5434
|
+
mergedEntity.extraFrontmatterLines.push(...parsed.extraFrontmatterLines ?? []);
|
|
5435
|
+
mergedEntity.preSectionLines.push(...parsed.preSectionLines ?? []);
|
|
5436
|
+
mergedEntity.extraSections.push(...(parsed.extraSections ?? []).map((section) => ({
|
|
5437
|
+
title: section.title,
|
|
5438
|
+
lines: [...section.lines]
|
|
5439
|
+
})));
|
|
3773
5440
|
} catch {
|
|
3774
5441
|
}
|
|
3775
5442
|
}
|
|
3776
|
-
|
|
5443
|
+
const timelineKeys = /* @__PURE__ */ new Set();
|
|
5444
|
+
mergedEntity.timeline = mergedEntity.timeline.filter((entry) => {
|
|
5445
|
+
const key = JSON.stringify([
|
|
5446
|
+
entry.timestamp,
|
|
5447
|
+
entry.source ?? "",
|
|
5448
|
+
entry.sessionKey ?? "",
|
|
5449
|
+
entry.principal ?? "",
|
|
5450
|
+
entry.text
|
|
5451
|
+
]);
|
|
5452
|
+
if (timelineKeys.has(key)) return false;
|
|
5453
|
+
timelineKeys.add(key);
|
|
5454
|
+
return true;
|
|
5455
|
+
});
|
|
3777
5456
|
const relKeys = /* @__PURE__ */ new Set();
|
|
3778
5457
|
mergedEntity.relationships = mergedEntity.relationships.filter((r) => {
|
|
3779
5458
|
const key = `${r.target}::${r.label}`;
|
|
@@ -3789,18 +5468,32 @@ ${rows.join("\n")}
|
|
|
3789
5468
|
return true;
|
|
3790
5469
|
}).sort((a, b) => b.date.localeCompare(a.date));
|
|
3791
5470
|
mergedEntity.aliases = [...new Set(mergedEntity.aliases)];
|
|
5471
|
+
mergedEntity.structuredSections = sortStructuredSectionsBySchema(
|
|
5472
|
+
mergedEntity.type,
|
|
5473
|
+
mergedEntity.structuredSections ?? [],
|
|
5474
|
+
this.entitySchemas
|
|
5475
|
+
);
|
|
5476
|
+
mergedEntity.facts = compileEntityFacts(mergedEntity.timeline, mergedEntity.structuredSections);
|
|
5477
|
+
const extraSectionKeys = /* @__PURE__ */ new Set();
|
|
5478
|
+
mergedEntity.extraSections = (mergedEntity.extraSections ?? []).filter((section) => {
|
|
5479
|
+
const key = `${section.title}::${section.lines.join("\n")}`;
|
|
5480
|
+
if (extraSectionKeys.has(key)) return false;
|
|
5481
|
+
extraSectionKeys.add(key);
|
|
5482
|
+
return true;
|
|
5483
|
+
});
|
|
3792
5484
|
if (!mergedEntity.name) {
|
|
3793
5485
|
const dashIdx = canonical.indexOf("-");
|
|
3794
5486
|
mergedEntity.name = dashIdx !== -1 ? canonical.slice(dashIdx + 1) : canonical;
|
|
3795
5487
|
}
|
|
5488
|
+
mergedEntity.created = mergedEntity.created || mergedEntity.updated || (/* @__PURE__ */ new Date()).toISOString();
|
|
3796
5489
|
mergedEntity.updated = mergedEntity.updated || (/* @__PURE__ */ new Date()).toISOString();
|
|
3797
|
-
const canonicalPath =
|
|
3798
|
-
await
|
|
5490
|
+
const canonicalPath = path5.join(this.entitiesDir, `${canonical}.md`);
|
|
5491
|
+
await writeFile3(canonicalPath, serializeEntityFile(mergedEntity, this.entitySchemas), "utf-8");
|
|
3799
5492
|
for (const file of files) {
|
|
3800
|
-
const filePath =
|
|
5493
|
+
const filePath = path5.join(this.entitiesDir, file);
|
|
3801
5494
|
if (filePath !== canonicalPath) {
|
|
3802
5495
|
try {
|
|
3803
|
-
await
|
|
5496
|
+
await unlink2(filePath);
|
|
3804
5497
|
merged++;
|
|
3805
5498
|
log.debug(`merged entity ${file} \u2192 ${canonical}.md`);
|
|
3806
5499
|
} catch {
|
|
@@ -3825,7 +5518,7 @@ ${rows.join("\n")}
|
|
|
3825
5518
|
const updatedAt = new Date(m.frontmatter.updated).getTime();
|
|
3826
5519
|
if (updatedAt < cutoff) {
|
|
3827
5520
|
try {
|
|
3828
|
-
await
|
|
5521
|
+
await unlink2(m.path);
|
|
3829
5522
|
deleted.push(m);
|
|
3830
5523
|
log.debug(`cleaned expired commitment ${m.frontmatter.id}`);
|
|
3831
5524
|
} catch {
|
|
@@ -3862,7 +5555,7 @@ ${rows.join("\n")}
|
|
|
3862
5555
|
${memory.content}
|
|
3863
5556
|
`;
|
|
3864
5557
|
try {
|
|
3865
|
-
await
|
|
5558
|
+
await writeFile3(memory.path, fileContent, "utf-8");
|
|
3866
5559
|
updated++;
|
|
3867
5560
|
} catch (err) {
|
|
3868
5561
|
log.debug(`failed to update access tracking for ${entry.memoryId}: ${err}`);
|
|
@@ -3880,6 +5573,29 @@ ${memory.content}
|
|
|
3880
5573
|
const memories = await this.readAllMemories();
|
|
3881
5574
|
return memories.find((m) => m.frontmatter.id === id) ?? null;
|
|
3882
5575
|
}
|
|
5576
|
+
/**
|
|
5577
|
+
* Check which of the given memory IDs actually exist on disk.
|
|
5578
|
+
*
|
|
5579
|
+
* Uses a lightweight directory scan (collectActiveMemoryPaths) that reads
|
|
5580
|
+
* file names without parsing frontmatter — much cheaper than readAllMemories()
|
|
5581
|
+
* for simple existence checks like citation usage tracking.
|
|
5582
|
+
*
|
|
5583
|
+
* Returns the subset of `ids` that correspond to real memory files.
|
|
5584
|
+
*/
|
|
5585
|
+
async filterExistingMemoryIds(ids) {
|
|
5586
|
+
if (ids.length === 0) return /* @__PURE__ */ new Set();
|
|
5587
|
+
const wantedIds = new Set(ids);
|
|
5588
|
+
const filePaths = await this.collectActiveMemoryPaths();
|
|
5589
|
+
const foundIds = /* @__PURE__ */ new Set();
|
|
5590
|
+
for (const filePath of filePaths) {
|
|
5591
|
+
const basename = path5.basename(filePath, ".md");
|
|
5592
|
+
if (wantedIds.has(basename)) {
|
|
5593
|
+
foundIds.add(basename);
|
|
5594
|
+
if (foundIds.size === wantedIds.size) break;
|
|
5595
|
+
}
|
|
5596
|
+
}
|
|
5597
|
+
return foundIds;
|
|
5598
|
+
}
|
|
3883
5599
|
async getProjectedMemoryState(id) {
|
|
3884
5600
|
const projected = readProjectedMemoryState(this.baseDir, id);
|
|
3885
5601
|
if (projected) return projected;
|
|
@@ -3970,11 +5686,11 @@ ${sanitized.text}
|
|
|
3970
5686
|
`;
|
|
3971
5687
|
let filePath;
|
|
3972
5688
|
if (category === "correction") {
|
|
3973
|
-
filePath =
|
|
5689
|
+
filePath = path5.join(this.correctionsDir, `${id}.md`);
|
|
3974
5690
|
} else {
|
|
3975
|
-
filePath =
|
|
5691
|
+
filePath = path5.join(this.factsDir, today, `${id}.md`);
|
|
3976
5692
|
}
|
|
3977
|
-
await
|
|
5693
|
+
await writeFile3(filePath, fileContent, "utf-8");
|
|
3978
5694
|
log.debug(`wrote chunk ${id} (${chunkIndex + 1}/${chunkTotal}) to ${filePath}`);
|
|
3979
5695
|
return id;
|
|
3980
5696
|
}
|
|
@@ -4010,7 +5726,7 @@ ${sanitized.text}
|
|
|
4010
5726
|
${oldMemory.content}
|
|
4011
5727
|
`;
|
|
4012
5728
|
try {
|
|
4013
|
-
await
|
|
5729
|
+
await writeFile3(oldMemory.path, fileContent, "utf-8");
|
|
4014
5730
|
await this.appendGeneratedMemoryLifecycleEventFailOpen("storage.supersedeMemory", {
|
|
4015
5731
|
memoryId: oldMemoryId,
|
|
4016
5732
|
eventType: "superseded",
|
|
@@ -4041,15 +5757,15 @@ Reason: ${reason}`, {
|
|
|
4041
5757
|
// Memory Summarization (Phase 4A)
|
|
4042
5758
|
// ---------------------------------------------------------------------------
|
|
4043
5759
|
get summariesDir() {
|
|
4044
|
-
return
|
|
5760
|
+
return path5.join(this.baseDir, "summaries");
|
|
4045
5761
|
}
|
|
4046
5762
|
/**
|
|
4047
5763
|
* Write a memory summary.
|
|
4048
5764
|
*/
|
|
4049
5765
|
async writeSummary(summary) {
|
|
4050
|
-
await
|
|
4051
|
-
const filePath =
|
|
4052
|
-
await
|
|
5766
|
+
await mkdir3(this.summariesDir, { recursive: true });
|
|
5767
|
+
const filePath = path5.join(this.summariesDir, `${summary.id}.json`);
|
|
5768
|
+
await writeFile3(filePath, JSON.stringify(summary, null, 2), "utf-8");
|
|
4053
5769
|
log.debug(`wrote summary ${summary.id}`);
|
|
4054
5770
|
}
|
|
4055
5771
|
/**
|
|
@@ -4061,8 +5777,8 @@ Reason: ${reason}`, {
|
|
|
4061
5777
|
const summaries = [];
|
|
4062
5778
|
for (const file of files) {
|
|
4063
5779
|
if (!file.endsWith(".json")) continue;
|
|
4064
|
-
const filePath =
|
|
4065
|
-
const raw = await
|
|
5780
|
+
const filePath = path5.join(this.summariesDir, file);
|
|
5781
|
+
const raw = await readFile3(filePath, "utf-8");
|
|
4066
5782
|
summaries.push(JSON.parse(raw));
|
|
4067
5783
|
}
|
|
4068
5784
|
return summaries;
|
|
@@ -4092,7 +5808,7 @@ Reason: ${reason}`, {
|
|
|
4092
5808
|
${memory.content}
|
|
4093
5809
|
`;
|
|
4094
5810
|
try {
|
|
4095
|
-
await
|
|
5811
|
+
await writeFile3(memory.path, fileContent, "utf-8");
|
|
4096
5812
|
await this.appendGeneratedMemoryLifecycleEventFailOpen("storage.archiveMemories", {
|
|
4097
5813
|
memoryId: id,
|
|
4098
5814
|
eventType: "archived",
|
|
@@ -4120,18 +5836,18 @@ ${memory.content}
|
|
|
4120
5836
|
* Save topic scores to meta.json.
|
|
4121
5837
|
*/
|
|
4122
5838
|
async saveTopics(topics) {
|
|
4123
|
-
const metaPath =
|
|
4124
|
-
await
|
|
4125
|
-
await
|
|
5839
|
+
const metaPath = path5.join(this.stateDir, "topics.json");
|
|
5840
|
+
await mkdir3(this.stateDir, { recursive: true });
|
|
5841
|
+
await writeFile3(metaPath, JSON.stringify({ topics, updatedAt: (/* @__PURE__ */ new Date()).toISOString() }, null, 2), "utf-8");
|
|
4126
5842
|
log.debug(`saved ${topics.length} topic scores`);
|
|
4127
5843
|
}
|
|
4128
5844
|
/**
|
|
4129
5845
|
* Load topic scores from meta.json.
|
|
4130
5846
|
*/
|
|
4131
5847
|
async loadTopics() {
|
|
4132
|
-
const metaPath =
|
|
5848
|
+
const metaPath = path5.join(this.stateDir, "topics.json");
|
|
4133
5849
|
try {
|
|
4134
|
-
const raw = await
|
|
5850
|
+
const raw = await readFile3(metaPath, "utf-8");
|
|
4135
5851
|
return JSON.parse(raw);
|
|
4136
5852
|
} catch {
|
|
4137
5853
|
return { topics: [], updatedAt: null };
|
|
@@ -4205,7 +5921,13 @@ ${memory.content}
|
|
|
4205
5921
|
};
|
|
4206
5922
|
|
|
4207
5923
|
export {
|
|
5924
|
+
normalizeEntityText,
|
|
5925
|
+
normalizeEntitySchemas,
|
|
5926
|
+
resolveRequestedEntitySectionKeys,
|
|
4208
5927
|
sanitizeMemoryContent,
|
|
5928
|
+
hasCitationForTemplate,
|
|
5929
|
+
attachCitation,
|
|
5930
|
+
stripCitationForTemplate,
|
|
4209
5931
|
getCachedEpisodeMap,
|
|
4210
5932
|
setCachedEpisodeMap,
|
|
4211
5933
|
getCachedRuleMemories,
|
|
@@ -4236,6 +5958,10 @@ export {
|
|
|
4236
5958
|
parseContinuityImprovementLoops,
|
|
4237
5959
|
normalizeEntityName,
|
|
4238
5960
|
ContentHashIndex,
|
|
5961
|
+
normalizeAttributePairs,
|
|
5962
|
+
compareEntityTimestamps,
|
|
5963
|
+
fingerprintEntityStructuredFacts,
|
|
5964
|
+
isEntitySynthesisStale,
|
|
4239
5965
|
parseEntityFile,
|
|
4240
5966
|
serializeEntityFile,
|
|
4241
5967
|
StorageManager
|