@chiligpt/memory 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +30 -0
- package/dist/core/backfill.d.ts +78 -0
- package/dist/core/backfill.d.ts.map +1 -0
- package/dist/core/backfill.js +155 -0
- package/dist/core/context-graph.d.ts +73 -0
- package/dist/core/context-graph.d.ts.map +1 -0
- package/dist/core/context-graph.js +363 -0
- package/dist/core/entity-index.d.ts +33 -0
- package/dist/core/entity-index.d.ts.map +1 -0
- package/dist/core/entity-index.js +59 -0
- package/dist/core/entity.d.ts +56 -0
- package/dist/core/entity.d.ts.map +1 -0
- package/dist/core/entity.js +65 -0
- package/dist/core/narrative.d.ts +110 -0
- package/dist/core/narrative.d.ts.map +1 -0
- package/dist/core/narrative.js +765 -0
- package/dist/core/read-path.d.ts +53 -0
- package/dist/core/read-path.d.ts.map +1 -0
- package/dist/core/read-path.js +592 -0
- package/dist/core/session-tree-index.d.ts +69 -0
- package/dist/core/session-tree-index.d.ts.map +1 -0
- package/dist/core/session-tree-index.js +131 -0
- package/dist/index.d.ts +16 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +15 -0
- package/dist/service.d.ts +105 -0
- package/dist/service.d.ts.map +1 -0
- package/dist/service.js +465 -0
- package/dist/storage/adapter.d.ts +61 -0
- package/dist/storage/adapter.d.ts.map +1 -0
- package/dist/storage/adapter.js +14 -0
- package/dist/storage/file-utils.d.ts +7 -0
- package/dist/storage/file-utils.d.ts.map +1 -0
- package/dist/storage/file-utils.js +37 -0
- package/dist/storage/in-memory-entry.d.ts +5 -0
- package/dist/storage/in-memory-entry.d.ts.map +1 -0
- package/dist/storage/in-memory-entry.js +5 -0
- package/dist/storage/in-memory.d.ts +37 -0
- package/dist/storage/in-memory.d.ts.map +1 -0
- package/dist/storage/in-memory.js +54 -0
- package/dist/storage/jsonl-entry.d.ts +6 -0
- package/dist/storage/jsonl-entry.d.ts.map +1 -0
- package/dist/storage/jsonl-entry.js +6 -0
- package/dist/storage/jsonl.d.ts +32 -0
- package/dist/storage/jsonl.d.ts.map +1 -0
- package/dist/storage/jsonl.js +171 -0
- package/dist/types.d.ts +66 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +9 -0
- package/package.json +46 -0
package/README.md
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
# @chili/memory
|
|
2
|
+
|
|
3
|
+
Persistent agent memory system — storage-adapter-agnostic, LLM-provider-agnostic.
|
|
4
|
+
|
|
5
|
+
## Usage
|
|
6
|
+
|
|
7
|
+
```ts
|
|
8
|
+
import { MemoryService, JsonlMemoryStorage } from "@chili/memory";
|
|
9
|
+
|
|
10
|
+
const service = new MemoryService({
|
|
11
|
+
storage: new JsonlMemoryStorage("~/.my-app/memory"),
|
|
12
|
+
complete: async (messages, opts) => myLLMClient.complete(messages, opts),
|
|
13
|
+
cwd: process.cwd(),
|
|
14
|
+
sessionId: generateSessionId(),
|
|
15
|
+
});
|
|
16
|
+
|
|
17
|
+
const { systemPromptAppend } = await service.init();
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
## Storage Adapters
|
|
21
|
+
|
|
22
|
+
| Import | Description |
|
|
23
|
+
|---|---|
|
|
24
|
+
| `@chili/memory` | `JsonlMemoryStorage`, `InMemoryStorage` |
|
|
25
|
+
| `@chili/memory/storage/jsonl` | `JsonlMemoryStorage` directly |
|
|
26
|
+
| `@chili/memory/storage/in-memory` | `InMemoryStorage` directly |
|
|
27
|
+
|
|
28
|
+
## Requirements
|
|
29
|
+
|
|
30
|
+
Node.js ≥ 19 (for `globalThis.crypto.randomUUID()`).
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Retroactive session indexing — EPIC-31 (F-78, F-79, F-80).
|
|
3
|
+
*
|
|
4
|
+
* Detects sessions in Layer 0 (session tree index) that were never promoted
|
|
5
|
+
* into the memory graph (Layer 2/3) and re-runs the compaction indexing
|
|
6
|
+
* pipeline on their JSONL content.
|
|
7
|
+
*
|
|
8
|
+
* Covers two categories missed by the F-73 session-close flush:
|
|
9
|
+
* 1. Sessions created before F-73 was deployed.
|
|
10
|
+
* 2. Sessions that closed abnormally (process killed before session_shutdown).
|
|
11
|
+
*
|
|
12
|
+
* Adapted for @chili/memory: uses IMemoryStorage, LLMCompleteFn, and
|
|
13
|
+
* MemoryBranchEntry. readSessionForBackfill() uses local minimal types
|
|
14
|
+
* (RawFileRecord) instead of chili-internal SessionHeader/SessionEntry.
|
|
15
|
+
*/
|
|
16
|
+
import type { LLMCompleteFn, MemoryBranchEntry } from "../types.js";
|
|
17
|
+
import type { IMemoryStorage } from "../storage/adapter.js";
|
|
18
|
+
import type { EntityIndex } from "./entity-index.js";
|
|
19
|
+
import type { NarrativeCache } from "./narrative.js";
|
|
20
|
+
export interface UnindexedSession {
|
|
21
|
+
sessionId: string;
|
|
22
|
+
sessionFile: string;
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Compare the session tree index (Layer 0) against ContextNodes (Layer 2).
|
|
26
|
+
* Returns sessions that have no ContextNode, excluding the current session.
|
|
27
|
+
*
|
|
28
|
+
* Returns an empty array if the session tree index does not exist.
|
|
29
|
+
*/
|
|
30
|
+
export declare function findUnindexedSessions(storage: IMemoryStorage, currentSessionId: string): Promise<UnindexedSession[]>;
|
|
31
|
+
export interface BackfillSessionData {
|
|
32
|
+
sessionId: string;
|
|
33
|
+
sessionFile: string;
|
|
34
|
+
cwd: string;
|
|
35
|
+
entries: MemoryBranchEntry[];
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Read a session JSONL file and extract the branch entries for backfill indexing.
|
|
39
|
+
*
|
|
40
|
+
* Returns null if:
|
|
41
|
+
* - The file does not exist or cannot be read.
|
|
42
|
+
* - No session header record (type === "session") is found.
|
|
43
|
+
* - No message entries exist in the branch.
|
|
44
|
+
*
|
|
45
|
+
* Uses RawFileRecord (local minimal types) instead of chili-internal types.
|
|
46
|
+
* readJsonlFile reads via node:fs — this function is Node.js only.
|
|
47
|
+
*/
|
|
48
|
+
export declare function readSessionForBackfill(sessionFile: string): BackfillSessionData | null;
|
|
49
|
+
export interface BackfillResult {
|
|
50
|
+
/** Sessions that had content and were successfully written to Layer 2/3. */
|
|
51
|
+
processed: number;
|
|
52
|
+
/** Sessions skipped because the file was missing or had no message content. */
|
|
53
|
+
skipped: number;
|
|
54
|
+
/** Sessions where the indexing pipeline failed (LLM call error or extraction failure). */
|
|
55
|
+
errors: number;
|
|
56
|
+
/** Total unindexed sessions found before any processing. */
|
|
57
|
+
total: number;
|
|
58
|
+
}
|
|
59
|
+
export interface BackfillParams {
|
|
60
|
+
storage: IMemoryStorage;
|
|
61
|
+
entityIndex: EntityIndex;
|
|
62
|
+
narrativeCache: NarrativeCache;
|
|
63
|
+
complete: LLMCompleteFn;
|
|
64
|
+
currentSessionId: string;
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* Run the retroactive indexing pipeline for all unindexed sessions.
|
|
68
|
+
*
|
|
69
|
+
* Each session:
|
|
70
|
+
* 1. Reads the session JSONL via `readSessionForBackfill`.
|
|
71
|
+
* 2. Calls `writeContextNodeFromCompaction` to extract and write a ContextNode.
|
|
72
|
+
* 3. Calls `updateNarrativeLayer` to assign the node to a narrative hub.
|
|
73
|
+
*
|
|
74
|
+
* Per-session errors are caught and do not abort the loop.
|
|
75
|
+
* The current session is always excluded — F-80 handles it on the next session_start.
|
|
76
|
+
*/
|
|
77
|
+
export declare function runBackfillIndexing(params: BackfillParams): Promise<BackfillResult>;
|
|
78
|
+
//# sourceMappingURL=backfill.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"backfill.d.ts","sourceRoot":"","sources":["../../src/core/backfill.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,iBAAiB,EAAE,MAAM,aAAa,CAAC;AACpE,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAG5D,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAErD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAC;AAyBrD,MAAM,WAAW,gBAAgB;IAChC,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;CACpB;AAED;;;;;GAKG;AACH,wBAAsB,qBAAqB,CAC1C,OAAO,EAAE,cAAc,EACvB,gBAAgB,EAAE,MAAM,GACtB,OAAO,CAAC,gBAAgB,EAAE,CAAC,CAmB7B;AAMD,MAAM,WAAW,mBAAmB;IACnC,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,EAAE,MAAM,CAAC;IACpB,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,iBAAiB,EAAE,CAAC;CAC7B;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,sBAAsB,CAAC,WAAW,EAAE,MAAM,GAAG,mBAAmB,GAAG,IAAI,CAqCtF;AAMD,MAAM,WAAW,cAAc;IAC9B,4EAA4E;IAC5E,SAAS,EAAE,MAAM,CAAC;IAClB,+EAA+E;IAC/E,OAAO,EAAE,MAAM,CAAC;IAChB,0FAA0F;IAC1F,MAAM,EAAE,MAAM,CAAC;IACf,4DAA4D;IAC5D,KAAK,EAAE,MAAM,CAAC;CACd;AAED,MAAM,WAAW,cAAc;IAC9B,OAAO,EAAE,cAAc,CAAC;IACxB,WAAW,EAAE,WAAW,CAAC;IACzB,cAAc,EAAE,cAAc,CAAC;IAC/B,QAAQ,EAAE,aAAa,CAAC;IACxB,gBAAgB,EAAE,MAAM,CAAC;CACzB;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,mBAAmB,CAAC,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC,cAAc,CAAC,CAoEzF","sourcesContent":["/**\n * Retroactive session indexing — EPIC-31 (F-78, F-79, F-80).\n *\n * Detects sessions in Layer 0 (session tree index) that were never promoted\n * into the memory graph (Layer 2/3) and re-runs the compaction indexing\n * pipeline on their JSONL content.\n *\n * Covers two categories missed by the F-73 session-close flush:\n * 1. Sessions created before F-73 was deployed.\n * 2. Sessions that closed abnormally (process killed before session_shutdown).\n *\n * Adapted for @chili/memory: uses IMemoryStorage, LLMCompleteFn, and\n * MemoryBranchEntry. readSessionForBackfill() uses local minimal types\n * (RawFileRecord) instead of chili-internal SessionHeader/SessionEntry.\n */\n\nimport type { LLMCompleteFn, MemoryBranchEntry } from \"../types.js\";\nimport type { IMemoryStorage } from \"../storage/adapter.js\";\nimport { writeContextNodeFromCompaction } from \"./context-graph.js\";\nimport type { ContextNode } from \"./context-graph.js\";\nimport type { EntityIndex } from \"./entity-index.js\";\nimport { updateNarrativeLayer } from \"./narrative.js\";\nimport type { NarrativeCache } from \"./narrative.js\";\nimport { loadSessionTree } from \"./session-tree-index.js\";\nimport { readJsonlFile } from \"../storage/file-utils.js\";\n\n// ---------------------------------------------------------------------------\n// Local minimal types for reading raw session JSONL files\n// These replace chili-internal FileEntry/SessionEntry/SessionHeader.\n// The memory system only reads type, id, cwd, and message fields.\n// ---------------------------------------------------------------------------\n\ninterface RawFileRecord {\n\ttype: string;\n\tid?: string;\n\tcwd?: string;\n\tmessage?: {\n\t\trole: \"user\" | \"assistant\";\n\t\tcontent: string | Array<{ type: string; text?: string; [key: string]: unknown }>;\n\t};\n\t[key: string]: unknown;\n}\n\n// ---------------------------------------------------------------------------\n// F-78: Unindexed session scanner\n// ---------------------------------------------------------------------------\n\nexport interface UnindexedSession {\n\tsessionId: string;\n\tsessionFile: string;\n}\n\n/**\n * Compare the session tree index (Layer 0) against ContextNodes (Layer 2).\n * Returns sessions that have no ContextNode, excluding the current session.\n *\n * Returns an empty array if the session tree index does not exist.\n */\nexport async function findUnindexedSessions(\n\tstorage: IMemoryStorage,\n\tcurrentSessionId: string,\n): Promise<UnindexedSession[]> {\n\tconst treeEntries = await loadSessionTree(storage);\n\tif (treeEntries.length === 0) return [];\n\n\tconst contextNodes = await storage.readAll<ContextNode>(\"context\");\n\tconst indexedIds = new Set(contextNodes.map((n) => n.sessionId));\n\n\t// Deduplicate by sessionId — last-wins (handles re-recorded entries)\n\tconst latestBySessionId = new Map<string, UnindexedSession>();\n\tfor (const entry of treeEntries) {\n\t\tif (!indexedIds.has(entry.sessionId) && entry.sessionId !== currentSessionId) {\n\t\t\tlatestBySessionId.set(entry.sessionId, {\n\t\t\t\tsessionId: entry.sessionId,\n\t\t\t\tsessionFile: entry.sessionFile,\n\t\t\t});\n\t\t}\n\t}\n\n\treturn [...latestBySessionId.values()];\n}\n\n// ---------------------------------------------------------------------------\n// F-79: Session JSONL reader for backfill\n// ---------------------------------------------------------------------------\n\nexport interface BackfillSessionData {\n\tsessionId: string;\n\tsessionFile: string;\n\tcwd: string;\n\tentries: MemoryBranchEntry[];\n}\n\n/**\n * Read a session JSONL file and extract the branch entries for backfill indexing.\n *\n * Returns null if:\n * - The file does not exist or cannot be read.\n * - No session header record (type === \"session\") is found.\n * - No message entries exist in the branch.\n *\n * Uses RawFileRecord (local minimal types) instead of chili-internal types.\n * readJsonlFile reads via node:fs — this function is Node.js only.\n */\nexport function readSessionForBackfill(sessionFile: string): BackfillSessionData | null {\n\tlet records: RawFileRecord[];\n\ttry {\n\t\trecords = readJsonlFile<RawFileRecord>(sessionFile);\n\t} catch {\n\t\treturn null;\n\t}\n\n\tconst header = records.find((r) => r.type === \"session\");\n\tif (!header || typeof header.id !== \"string\") return null;\n\n\t// All non-header entries\n\tconst allEntries = records.filter((r) => r.type !== \"session\");\n\n\t// Find the last compaction entry\n\tlet lastCompactionIdx = -1;\n\tfor (let i = allEntries.length - 1; i >= 0; i--) {\n\t\tif (allEntries[i].type === \"compaction\") {\n\t\t\tlastCompactionIdx = i;\n\t\t\tbreak;\n\t\t}\n\t}\n\n\t// Branch: everything after the last compaction, or all entries if none\n\tconst branchEntries = lastCompactionIdx >= 0 ? allEntries.slice(lastCompactionIdx + 1) : allEntries;\n\n\t// Require at least one message entry — nothing to index otherwise\n\tconst hasMessages = branchEntries.some((e) => e.type === \"message\");\n\tif (!hasMessages) return null;\n\n\t// Cast to MemoryBranchEntry — structurally compatible (type + optional message)\n\treturn {\n\t\tsessionId: header.id,\n\t\tsessionFile,\n\t\tcwd: typeof header.cwd === \"string\" ? header.cwd : \"\",\n\t\tentries: branchEntries as MemoryBranchEntry[],\n\t};\n}\n\n// ---------------------------------------------------------------------------\n// F-80: Backfill runner\n// ---------------------------------------------------------------------------\n\nexport interface BackfillResult {\n\t/** Sessions that had content and were successfully written to Layer 2/3. */\n\tprocessed: number;\n\t/** Sessions skipped because the file was missing or had no message content. */\n\tskipped: number;\n\t/** Sessions where the indexing pipeline failed (LLM call error or extraction failure). */\n\terrors: number;\n\t/** Total unindexed sessions found before any processing. */\n\ttotal: number;\n}\n\nexport interface BackfillParams {\n\tstorage: IMemoryStorage;\n\tentityIndex: EntityIndex;\n\tnarrativeCache: NarrativeCache;\n\tcomplete: LLMCompleteFn;\n\tcurrentSessionId: string;\n}\n\n/**\n * Run the retroactive indexing pipeline for all unindexed sessions.\n *\n * Each session:\n * 1. Reads the session JSONL via `readSessionForBackfill`.\n * 2. Calls `writeContextNodeFromCompaction` to extract and write a ContextNode.\n * 3. Calls `updateNarrativeLayer` to assign the node to a narrative hub.\n *\n * Per-session errors are caught and do not abort the loop.\n * The current session is always excluded — F-80 handles it on the next session_start.\n */\nexport async function runBackfillIndexing(params: BackfillParams): Promise<BackfillResult> {\n\tconst { storage, entityIndex, narrativeCache, complete, currentSessionId } = params;\n\n\tconst unindexed = await findUnindexedSessions(storage, currentSessionId);\n\tconst total = unindexed.length;\n\n\tlet processed = 0;\n\tlet skipped = 0;\n\tlet errors = 0;\n\n\tfor (const { sessionId, sessionFile } of unindexed) {\n\t\ttry {\n\t\t\tconst data = readSessionForBackfill(sessionFile);\n\t\t\tif (!data) {\n\t\t\t\tskipped++;\n\t\t\t\tcontinue;\n\t\t\t}\n\n\t\t\t// Wrap complete() to log LLM-level failures with session context\n\t\t\tconst verboseComplete: LLMCompleteFn = async (messages, options) => {\n\t\t\t\ttry {\n\t\t\t\t\treturn await complete(messages, options);\n\t\t\t\t} catch (err) {\n\t\t\t\t\tconsole.error(\n\t\t\t\t\t\t`[memory:backfill] LLM call failed — session ${sessionId} (${sessionFile}):`,\n\t\t\t\t\t\terr instanceof Error ? err.message : String(err),\n\t\t\t\t\t);\n\t\t\t\t\tthrow err;\n\t\t\t\t}\n\t\t\t};\n\n\t\t\tconst node = await writeContextNodeFromCompaction({\n\t\t\t\tbranchEntries: data.entries,\n\t\t\t\tsessionId: data.sessionId,\n\t\t\t\tsessionFile: data.sessionFile,\n\t\t\t\tstorage,\n\t\t\t\tentityIndex,\n\t\t\t\tcomplete: verboseComplete,\n\t\t\t\tcwd: data.cwd,\n\t\t\t});\n\n\t\t\tif (!node) {\n\t\t\t\tconsole.error(\n\t\t\t\t\t`[memory:backfill] Indexing returned no node — session ${sessionId}. ` +\n\t\t\t\t\t`LLM extraction likely failed. Run /memory index to retry.`,\n\t\t\t\t);\n\t\t\t\terrors++;\n\t\t\t\tcontinue;\n\t\t\t}\n\n\t\t\tawait updateNarrativeLayer({\n\t\t\t\tnewNode: node,\n\t\t\t\tstorage,\n\t\t\t\tcache: narrativeCache,\n\t\t\t\tcomplete,\n\t\t\t});\n\n\t\t\tprocessed++;\n\t\t} catch (err) {\n\t\t\tconsole.error(\n\t\t\t\t`[memory] backfill error for session ${sessionId}:`,\n\t\t\t\terr instanceof Error ? err.message : String(err),\n\t\t\t);\n\t\t\terrors++;\n\t\t}\n\t}\n\n\treturn { processed, skipped, errors, total };\n}\n"]}
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Retroactive session indexing — EPIC-31 (F-78, F-79, F-80).
|
|
3
|
+
*
|
|
4
|
+
* Detects sessions in Layer 0 (session tree index) that were never promoted
|
|
5
|
+
* into the memory graph (Layer 2/3) and re-runs the compaction indexing
|
|
6
|
+
* pipeline on their JSONL content.
|
|
7
|
+
*
|
|
8
|
+
* Covers two categories missed by the F-73 session-close flush:
|
|
9
|
+
* 1. Sessions created before F-73 was deployed.
|
|
10
|
+
* 2. Sessions that closed abnormally (process killed before session_shutdown).
|
|
11
|
+
*
|
|
12
|
+
* Adapted for @chili/memory: uses IMemoryStorage, LLMCompleteFn, and
|
|
13
|
+
* MemoryBranchEntry. readSessionForBackfill() uses local minimal types
|
|
14
|
+
* (RawFileRecord) instead of chili-internal SessionHeader/SessionEntry.
|
|
15
|
+
*/
|
|
16
|
+
import { writeContextNodeFromCompaction } from "./context-graph.js";
|
|
17
|
+
import { updateNarrativeLayer } from "./narrative.js";
|
|
18
|
+
import { loadSessionTree } from "./session-tree-index.js";
|
|
19
|
+
import { readJsonlFile } from "../storage/file-utils.js";
|
|
20
|
+
/**
|
|
21
|
+
* Compare the session tree index (Layer 0) against ContextNodes (Layer 2).
|
|
22
|
+
* Returns sessions that have no ContextNode, excluding the current session.
|
|
23
|
+
*
|
|
24
|
+
* Returns an empty array if the session tree index does not exist.
|
|
25
|
+
*/
|
|
26
|
+
export async function findUnindexedSessions(storage, currentSessionId) {
|
|
27
|
+
const treeEntries = await loadSessionTree(storage);
|
|
28
|
+
if (treeEntries.length === 0)
|
|
29
|
+
return [];
|
|
30
|
+
const contextNodes = await storage.readAll("context");
|
|
31
|
+
const indexedIds = new Set(contextNodes.map((n) => n.sessionId));
|
|
32
|
+
// Deduplicate by sessionId — last-wins (handles re-recorded entries)
|
|
33
|
+
const latestBySessionId = new Map();
|
|
34
|
+
for (const entry of treeEntries) {
|
|
35
|
+
if (!indexedIds.has(entry.sessionId) && entry.sessionId !== currentSessionId) {
|
|
36
|
+
latestBySessionId.set(entry.sessionId, {
|
|
37
|
+
sessionId: entry.sessionId,
|
|
38
|
+
sessionFile: entry.sessionFile,
|
|
39
|
+
});
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
return [...latestBySessionId.values()];
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Read a session JSONL file and extract the branch entries for backfill indexing.
|
|
46
|
+
*
|
|
47
|
+
* Returns null if:
|
|
48
|
+
* - The file does not exist or cannot be read.
|
|
49
|
+
* - No session header record (type === "session") is found.
|
|
50
|
+
* - No message entries exist in the branch.
|
|
51
|
+
*
|
|
52
|
+
* Uses RawFileRecord (local minimal types) instead of chili-internal types.
|
|
53
|
+
* readJsonlFile reads via node:fs — this function is Node.js only.
|
|
54
|
+
*/
|
|
55
|
+
export function readSessionForBackfill(sessionFile) {
|
|
56
|
+
let records;
|
|
57
|
+
try {
|
|
58
|
+
records = readJsonlFile(sessionFile);
|
|
59
|
+
}
|
|
60
|
+
catch {
|
|
61
|
+
return null;
|
|
62
|
+
}
|
|
63
|
+
const header = records.find((r) => r.type === "session");
|
|
64
|
+
if (!header || typeof header.id !== "string")
|
|
65
|
+
return null;
|
|
66
|
+
// All non-header entries
|
|
67
|
+
const allEntries = records.filter((r) => r.type !== "session");
|
|
68
|
+
// Find the last compaction entry
|
|
69
|
+
let lastCompactionIdx = -1;
|
|
70
|
+
for (let i = allEntries.length - 1; i >= 0; i--) {
|
|
71
|
+
if (allEntries[i].type === "compaction") {
|
|
72
|
+
lastCompactionIdx = i;
|
|
73
|
+
break;
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
// Branch: everything after the last compaction, or all entries if none
|
|
77
|
+
const branchEntries = lastCompactionIdx >= 0 ? allEntries.slice(lastCompactionIdx + 1) : allEntries;
|
|
78
|
+
// Require at least one message entry — nothing to index otherwise
|
|
79
|
+
const hasMessages = branchEntries.some((e) => e.type === "message");
|
|
80
|
+
if (!hasMessages)
|
|
81
|
+
return null;
|
|
82
|
+
// Cast to MemoryBranchEntry — structurally compatible (type + optional message)
|
|
83
|
+
return {
|
|
84
|
+
sessionId: header.id,
|
|
85
|
+
sessionFile,
|
|
86
|
+
cwd: typeof header.cwd === "string" ? header.cwd : "",
|
|
87
|
+
entries: branchEntries,
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
/**
|
|
91
|
+
* Run the retroactive indexing pipeline for all unindexed sessions.
|
|
92
|
+
*
|
|
93
|
+
* Each session:
|
|
94
|
+
* 1. Reads the session JSONL via `readSessionForBackfill`.
|
|
95
|
+
* 2. Calls `writeContextNodeFromCompaction` to extract and write a ContextNode.
|
|
96
|
+
* 3. Calls `updateNarrativeLayer` to assign the node to a narrative hub.
|
|
97
|
+
*
|
|
98
|
+
* Per-session errors are caught and do not abort the loop.
|
|
99
|
+
* The current session is always excluded — F-80 handles it on the next session_start.
|
|
100
|
+
*/
|
|
101
|
+
export async function runBackfillIndexing(params) {
|
|
102
|
+
const { storage, entityIndex, narrativeCache, complete, currentSessionId } = params;
|
|
103
|
+
const unindexed = await findUnindexedSessions(storage, currentSessionId);
|
|
104
|
+
const total = unindexed.length;
|
|
105
|
+
let processed = 0;
|
|
106
|
+
let skipped = 0;
|
|
107
|
+
let errors = 0;
|
|
108
|
+
for (const { sessionId, sessionFile } of unindexed) {
|
|
109
|
+
try {
|
|
110
|
+
const data = readSessionForBackfill(sessionFile);
|
|
111
|
+
if (!data) {
|
|
112
|
+
skipped++;
|
|
113
|
+
continue;
|
|
114
|
+
}
|
|
115
|
+
// Wrap complete() to log LLM-level failures with session context
|
|
116
|
+
const verboseComplete = async (messages, options) => {
|
|
117
|
+
try {
|
|
118
|
+
return await complete(messages, options);
|
|
119
|
+
}
|
|
120
|
+
catch (err) {
|
|
121
|
+
console.error(`[memory:backfill] LLM call failed — session ${sessionId} (${sessionFile}):`, err instanceof Error ? err.message : String(err));
|
|
122
|
+
throw err;
|
|
123
|
+
}
|
|
124
|
+
};
|
|
125
|
+
const node = await writeContextNodeFromCompaction({
|
|
126
|
+
branchEntries: data.entries,
|
|
127
|
+
sessionId: data.sessionId,
|
|
128
|
+
sessionFile: data.sessionFile,
|
|
129
|
+
storage,
|
|
130
|
+
entityIndex,
|
|
131
|
+
complete: verboseComplete,
|
|
132
|
+
cwd: data.cwd,
|
|
133
|
+
});
|
|
134
|
+
if (!node) {
|
|
135
|
+
console.error(`[memory:backfill] Indexing returned no node — session ${sessionId}. ` +
|
|
136
|
+
`LLM extraction likely failed. Run /memory index to retry.`);
|
|
137
|
+
errors++;
|
|
138
|
+
continue;
|
|
139
|
+
}
|
|
140
|
+
await updateNarrativeLayer({
|
|
141
|
+
newNode: node,
|
|
142
|
+
storage,
|
|
143
|
+
cache: narrativeCache,
|
|
144
|
+
complete,
|
|
145
|
+
});
|
|
146
|
+
processed++;
|
|
147
|
+
}
|
|
148
|
+
catch (err) {
|
|
149
|
+
console.error(`[memory] backfill error for session ${sessionId}:`, err instanceof Error ? err.message : String(err));
|
|
150
|
+
errors++;
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
return { processed, skipped, errors, total };
|
|
154
|
+
}
|
|
155
|
+
//# sourceMappingURL=backfill.js.map
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Context graph — ContextNode write path (F-57) and entity coupling (F-58).
|
|
3
|
+
*
|
|
4
|
+
* Hooks into `session_before_compact` (blocking). An LLM call extracts
|
|
5
|
+
* userContext, agentContext, topics[], and entities[] from branch entries.
|
|
6
|
+
* Writes a ContextNode and coupled EntityNodes to storage.
|
|
7
|
+
*
|
|
8
|
+
* Adapted for @chili/memory: uses IMemoryStorage (collection names) instead of
|
|
9
|
+
* StorageLayout (file paths), LLMCompleteFn instead of @chili/ai types, and
|
|
10
|
+
* MemoryBranchEntry instead of chili-internal SessionEntry.
|
|
11
|
+
*/
|
|
12
|
+
import type { LLMCompleteFn, MemoryBranchEntry } from "../types.js";
|
|
13
|
+
import type { IMemoryStorage } from "../storage/adapter.js";
|
|
14
|
+
import type { EntityIndex } from "./entity-index.js";
|
|
15
|
+
export interface ContextNode {
|
|
16
|
+
id: string;
|
|
17
|
+
/** ID of the session this node was written in. */
|
|
18
|
+
sessionId: string;
|
|
19
|
+
/** IDs of all sessions involved (for multi-session compaction). */
|
|
20
|
+
sessionIds: string[];
|
|
21
|
+
/** Unix timestamp (ms) when this node was written. */
|
|
22
|
+
timestamp: number;
|
|
23
|
+
/** What the user was asking / working on (immutable after write). */
|
|
24
|
+
userContext: string;
|
|
25
|
+
/** What the agent did in response (immutable after write). */
|
|
26
|
+
agentContext: string;
|
|
27
|
+
/** Topic keywords for narrative hub matching. */
|
|
28
|
+
topics: string[];
|
|
29
|
+
/** UUIDs of EntityNodes referenced in this context window. */
|
|
30
|
+
entityPointerIds: string[];
|
|
31
|
+
/** ID of the newer node that supersedes this one. */
|
|
32
|
+
supersededBy?: string;
|
|
33
|
+
/** ID of the older node that this one supersedes. */
|
|
34
|
+
supersedes?: string;
|
|
35
|
+
/** Timestamp when supersession was recorded. */
|
|
36
|
+
supersededAt?: number;
|
|
37
|
+
/** True when all entityPointerIds have been archived (set by orphan detection). */
|
|
38
|
+
allEntitiesArchived?: boolean;
|
|
39
|
+
/** Working directory at compaction time (process.cwd()). */
|
|
40
|
+
workingDirectory?: string;
|
|
41
|
+
/** File paths extracted from agentContext by the compaction LLM. */
|
|
42
|
+
relevantFilePaths?: string[];
|
|
43
|
+
/** Package/module names referenced, extracted by the compaction LLM. */
|
|
44
|
+
relevantModules?: string[];
|
|
45
|
+
/** Natural-language queries a user would type to find this session — broadens BM25F recall. */
|
|
46
|
+
searchTerms?: string[];
|
|
47
|
+
/** Absolute path to the Layer 1 session JSONL file that produced this node. */
|
|
48
|
+
sessionFile?: string;
|
|
49
|
+
/** Set by orphan detection when all entity pointers are archived. */
|
|
50
|
+
_archived?: boolean;
|
|
51
|
+
}
|
|
52
|
+
export declare function loadAllContextNodes(storage: IMemoryStorage): Promise<ContextNode[]>;
|
|
53
|
+
export declare function updateContextNode(storage: IMemoryStorage, id: string, patch: Partial<ContextNode>): Promise<void>;
|
|
54
|
+
export declare function loadContextNodeWithSupersession(storage: IMemoryStorage, id: string): Promise<[ContextNode | null, ContextNode | null]>;
|
|
55
|
+
export declare function writeContextNodeFromCompaction(params: {
|
|
56
|
+
branchEntries: MemoryBranchEntry[];
|
|
57
|
+
sessionId: string;
|
|
58
|
+
/** Absolute path to the Layer 1 session JSONL file — persisted on the ContextNode (F-75). */
|
|
59
|
+
sessionFile?: string;
|
|
60
|
+
storage: IMemoryStorage;
|
|
61
|
+
entityIndex: EntityIndex;
|
|
62
|
+
complete: LLMCompleteFn;
|
|
63
|
+
sessionOrdinal?: number;
|
|
64
|
+
cwd?: string;
|
|
65
|
+
}): Promise<ContextNode | null>;
|
|
66
|
+
export interface ContextNodeHealth {
|
|
67
|
+
meanValidity: number;
|
|
68
|
+
minValidity: number;
|
|
69
|
+
degradedCount: number;
|
|
70
|
+
deprecatedCount: number;
|
|
71
|
+
}
|
|
72
|
+
export declare function projectHealth(node: ContextNode, entityIndex: EntityIndex, now: number): ContextNodeHealth;
|
|
73
|
+
//# sourceMappingURL=context-graph.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"context-graph.d.ts","sourceRoot":"","sources":["../../src/core/context-graph.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAe,iBAAiB,EAAE,MAAM,aAAa,CAAC;AAGjF,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,uBAAuB,CAAC;AAC5D,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAMrD,MAAM,WAAW,WAAW;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,kDAAkD;IAClD,SAAS,EAAE,MAAM,CAAC;IAClB,mEAAmE;IACnE,UAAU,EAAE,MAAM,EAAE,CAAC;IACrB,sDAAsD;IACtD,SAAS,EAAE,MAAM,CAAC;IAClB,qEAAqE;IACrE,WAAW,EAAE,MAAM,CAAC;IACpB,8DAA8D;IAC9D,YAAY,EAAE,MAAM,CAAC;IACrB,iDAAiD;IACjD,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,8DAA8D;IAC9D,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,qDAAqD;IACrD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,qDAAqD;IACrD,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,gDAAgD;IAChD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,mFAAmF;IACnF,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAE9B,4DAA4D;IAC5D,gBAAgB,CAAC,EAAE,MAAM,CAAC;IAC1B,oEAAoE;IACpE,iBAAiB,CAAC,EAAE,MAAM,EAAE,CAAC;IAC7B,wEAAwE;IACxE,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAE3B,iGAA+F;IAC/F,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IAEvB,+EAA+E;IAC/E,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,qEAAqE;IACrE,SAAS,CAAC,EAAE,OAAO,CAAC;CACpB;AAwMD,wBAAsB,mBAAmB,CAAC,OAAO,EAAE,cAAc,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,CAEzF;AAED,wBAAsB,iBAAiB,CACtC,OAAO,EAAE,cAAc,EACvB,EAAE,EAAE,MAAM,EACV,KAAK,EAAE,OAAO,CAAC,WAAW,CAAC,GACzB,OAAO,CAAC,IAAI,CAAC,CAEf;AAED,wBAAsB,+BAA+B,CACpD,OAAO,EAAE,cAAc,EACvB,EAAE,EAAE,MAAM,GACR,OAAO,CAAC,CAAC,WAAW,GAAG,IAAI,EAAE,WAAW,GAAG,IAAI,CAAC,CAAC,CASnD;AA6CD,wBAAsB,8BAA8B,CAAC,MAAM,EAAE;IAC5D,aAAa,EAAE,iBAAiB,EAAE,CAAC;IACnC,SAAS,EAAE,MAAM,CAAC;IAClB,+FAA6F;IAC7F,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,OAAO,EAAE,cAAc,CAAC;IACxB,WAAW,EAAE,WAAW,CAAC;IACzB,QAAQ,EAAE,aAAa,CAAC;IACxB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,GAAG,CAAC,EAAE,MAAM,CAAC;CACb,GAAG,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,CAiE9B;AA0FD,MAAM,WAAW,iBAAiB;IACjC,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,eAAe,EAAE,MAAM,CAAC;CACxB;AAED,wBAAgB,aAAa,CAC5B,IAAI,EAAE,WAAW,EACjB,WAAW,EAAE,WAAW,EACxB,GAAG,EAAE,MAAM,GACT,iBAAiB,CA4BnB","sourcesContent":["/**\n * Context graph — ContextNode write path (F-57) and entity coupling (F-58).\n *\n * Hooks into `session_before_compact` (blocking). An LLM call extracts\n * userContext, agentContext, topics[], and entities[] from branch entries.\n * Writes a ContextNode and coupled EntityNodes to storage.\n *\n * Adapted for @chili/memory: uses IMemoryStorage (collection names) instead of\n * StorageLayout (file paths), LLMCompleteFn instead of @chili/ai types, and\n * MemoryBranchEntry instead of chili-internal SessionEntry.\n */\n\nimport type { LLMCompleteFn, LLMResponse, MemoryBranchEntry } from \"../types.js\";\nimport type { EntityType, EntityNode } from \"./entity.js\";\nimport { createEntity, nameHash, updateEntity, validityScore } from \"./entity.js\";\nimport type { IMemoryStorage } from \"../storage/adapter.js\";\nimport type { EntityIndex } from \"./entity-index.js\";\n\n// ---------------------------------------------------------------------------\n// ContextNode type\n// ---------------------------------------------------------------------------\n\nexport interface ContextNode {\n\tid: string;\n\t/** ID of the session this node was written in. */\n\tsessionId: string;\n\t/** IDs of all sessions involved (for multi-session compaction). */\n\tsessionIds: string[];\n\t/** Unix timestamp (ms) when this node was written. */\n\ttimestamp: number;\n\t/** What the user was asking / working on (immutable after write). */\n\tuserContext: string;\n\t/** What the agent did in response (immutable after write). */\n\tagentContext: string;\n\t/** Topic keywords for narrative hub matching. */\n\ttopics: string[];\n\t/** UUIDs of EntityNodes referenced in this context window. */\n\tentityPointerIds: string[];\n\t/** ID of the newer node that supersedes this one. */\n\tsupersededBy?: string;\n\t/** ID of the older node that this one supersedes. */\n\tsupersedes?: string;\n\t/** Timestamp when supersession was recorded. */\n\tsupersededAt?: number;\n\t/** True when all entityPointerIds have been archived (set by orphan detection). */\n\tallEntitiesArchived?: boolean;\n\t// F-68: Extended staleness-evidence fields\n\t/** Working directory at compaction time (process.cwd()). */\n\tworkingDirectory?: string;\n\t/** File paths extracted from agentContext by the compaction LLM. */\n\trelevantFilePaths?: string[];\n\t/** Package/module names referenced, extracted by the compaction LLM. */\n\trelevantModules?: string[];\n\t// F-108: Search-oriented query terms generated at write time\n\t/** Natural-language queries a user would type to find this session — broadens BM25F recall. */\n\tsearchTerms?: string[];\n\t// F-75: Self-contained Layer 1 file path for cross-session retrieval\n\t/** Absolute path to the Layer 1 session JSONL file that produced this node. */\n\tsessionFile?: string;\n\t/** Set by orphan detection when all entity pointers are archived. */\n\t_archived?: boolean;\n}\n\n// ---------------------------------------------------------------------------\n// Prompt helpers\n// ---------------------------------------------------------------------------\n\ntype TextBlock = { type: string; text: string };\n\nfunction extractText(content: unknown): string {\n\tif (typeof content === \"string\") return content;\n\tif (Array.isArray(content)) {\n\t\treturn (content as TextBlock[])\n\t\t\t.filter((b) => b.type === \"text\")\n\t\t\t.map((b) => b.text)\n\t\t\t.join(\" \");\n\t}\n\treturn \"\";\n}\n\n/** Serialise branch entries to a readable transcript for the LLM. */\nfunction formatBranchEntries(entries: MemoryBranchEntry[]): string {\n\tconst lines: string[] = [];\n\tfor (const entry of entries) {\n\t\tif (entry.type === \"message\" && entry.message) {\n\t\t\tconst msg = entry.message;\n\t\t\tif (msg.role === \"user\") {\n\t\t\t\tconst text = extractText(msg.content).trim();\n\t\t\t\tif (text) lines.push(`User: ${text}`);\n\t\t\t} else if (msg.role === \"assistant\") {\n\t\t\t\tconst text = extractText(msg.content).trim();\n\t\t\t\tif (text) lines.push(`Assistant: ${text.slice(0, 500)}`);\n\t\t\t}\n\t\t}\n\t}\n\treturn lines.join(\"\\n\") || \"(no messages)\";\n}\n\nconst EXTRACTION_SYSTEM_PROMPT = `You are a memory extraction assistant for a coding agent session.\nAnalyze the provided conversation transcript and extract structured metadata.\nRespond ONLY with valid JSON matching the schema — no markdown, no explanation.`;\n\nfunction buildExtractionPrompt(transcript: string, cwd: string) {\n\treturn [\n\t\t{\n\t\t\trole: \"user\" as const,\n\t\t\ttimestamp: Date.now(),\n\t\t\tcontent: `Extract metadata from this coding session transcript.\nWorking directory: ${cwd}\n\nTRANSCRIPT:\n${transcript}\n\nRespond with JSON only:\n{\n \"userContext\": \"<1-3 sentences: what the user was asking or working on>\",\n \"agentContext\": \"<1-3 sentences: what the agent did in response>\",\n \"topics\": [\"<topic1>\", \"<topic2>\"],\n \"entities\": [\n { \"name\": \"<canonical name>\", \"type\": \"<file|person|concept|module|decision|url>\" }\n ],\n \"relevantFilePaths\": [\"<file path mentioned in agent work>\"],\n \"relevantModules\": [\"<package or module name referenced>\"],\n \"searchTerms\": [\"<query 1>\", \"<query 2>\"],\n \"entityEvents\": [\n { \"name\": \"<entity canonical name>\", \"type\": \"<entity type>\", \"event\": \"<renamed|deprecated|replaced>\", \"newName\": \"<new name if renamed>\" }\n ]\n}\n\nRules:\n- userContext: from the user's perspective — their goal/question\n- agentContext: from the agent's perspective — actions taken\n- topics: 2-5 short keyword phrases for categorization\n- entities: named things that appeared — files, people, modules, decisions, URLs, and concepts (including programming languages, frameworks, algorithms, data structures, and named techniques)\n- relevantFilePaths: actual file paths the agent read/edited/created (empty if none)\n- relevantModules: npm packages or internal module names mentioned (empty if none)\n- searchTerms: 3-5 natural-language queries someone would type to find this session later — use varied vocabulary, not just technical terms (e.g. \"how we fixed the memory search bug\", \"storage adapter interface design\")\n- entityEvents: observable state changes — renames, deprecations, replacements (empty if none)\n- Keep all strings concise (under 200 chars each)`,\n\t\t},\n\t];\n}\n\n// ---------------------------------------------------------------------------\n// LLM response parsing\n// ---------------------------------------------------------------------------\n\ninterface EntityEventRaw {\n\tname: string;\n\ttype: EntityType;\n\tevent: \"renamed\" | \"deprecated\" | \"replaced\";\n\tnewName?: string;\n}\n\ninterface ExtractionResult {\n\tuserContext: string;\n\tagentContext: string;\n\ttopics: string[];\n\tentities: Array<{ name: string; type: EntityType }>;\n\trelevantFilePaths: string[];\n\trelevantModules: string[];\n\tsearchTerms: string[];\n\tentityEvents: EntityEventRaw[];\n}\n\nfunction parseExtractionResponse(response: LLMResponse): ExtractionResult | null {\n\tconst textBlock = Array.isArray(response.content)\n\t\t? response.content.find((b) => b.type === \"text\")\n\t\t: null;\n\tconst raw = textBlock ? (textBlock as { text: string }).text : \"\";\n\n\t// Strip markdown code fences if present\n\tconst cleaned = raw.replace(/^```(?:json)?\\n?/m, \"\").replace(/\\n?```$/m, \"\").trim();\n\n\ttry {\n\t\tconst parsed = JSON.parse(cleaned) as Partial<ExtractionResult>;\n\t\tconst validTypes = [\"file\", \"person\", \"concept\", \"module\", \"decision\", \"url\"];\n\t\treturn {\n\t\t\tuserContext: typeof parsed.userContext === \"string\" ? parsed.userContext : \"(no context)\",\n\t\t\tagentContext: typeof parsed.agentContext === \"string\" ? parsed.agentContext : \"(no context)\",\n\t\t\ttopics: Array.isArray(parsed.topics) ? parsed.topics.filter((t) => typeof t === \"string\") : [],\n\t\t\tentities: Array.isArray(parsed.entities)\n\t\t\t\t? parsed.entities.filter(\n\t\t\t\t\t\t(e): e is { name: string; type: EntityType } =>\n\t\t\t\t\t\t\ttypeof e?.name === \"string\" && validTypes.includes(e?.type),\n\t\t\t\t\t)\n\t\t\t\t: [],\n\t\t\trelevantFilePaths: Array.isArray(parsed.relevantFilePaths)\n\t\t\t\t? parsed.relevantFilePaths.filter((p) => typeof p === \"string\")\n\t\t\t\t: [],\n\t\t\trelevantModules: Array.isArray(parsed.relevantModules)\n\t\t\t\t? parsed.relevantModules.filter((m) => typeof m === \"string\")\n\t\t\t\t: [],\n\t\t\tsearchTerms: Array.isArray(parsed.searchTerms)\n\t\t\t\t? parsed.searchTerms.filter((s) => typeof s === \"string\")\n\t\t\t\t: [],\n\t\t\tentityEvents: Array.isArray(parsed.entityEvents)\n\t\t\t\t? parsed.entityEvents.filter(\n\t\t\t\t\t\t(e): e is EntityEventRaw =>\n\t\t\t\t\t\t\ttypeof e?.name === \"string\" &&\n\t\t\t\t\t\t\tvalidTypes.includes(e?.type) &&\n\t\t\t\t\t\t\t[\"renamed\", \"deprecated\", \"replaced\"].includes(e?.event),\n\t\t\t\t\t)\n\t\t\t\t: [],\n\t\t};\n\t} catch {\n\t\treturn null;\n\t}\n}\n\n// ---------------------------------------------------------------------------\n// Entity coupling (F-58)\n// ---------------------------------------------------------------------------\n\nconst VALID_ENTITY_TYPES = new Set<string>([\"file\", \"person\", \"concept\", \"module\", \"decision\", \"url\"]);\n\nasync function coupleEntities(\n\tentities: Array<{ name: string; type: EntityType }>,\n\tentityIndex: EntityIndex,\n\tstorage: IMemoryStorage,\n\tsessionOrdinal: number,\n\tnow: number,\n): Promise<string[]> {\n\tconst entityPointerIds: string[] = [];\n\n\tfor (const { name, type } of entities) {\n\t\tif (!VALID_ENTITY_TYPES.has(type)) continue;\n\n\t\tconst existing = entityIndex.get(name, type);\n\n\t\tif (existing) {\n\t\t\t// Update lastReferencedAt in memory and on disk\n\t\t\tconst updated = { ...existing, lastReferencedAt: now };\n\t\t\tentityIndex.update(existing.id, { lastReferencedAt: now });\n\t\t\tawait updateEntity(storage, existing.id, { lastReferencedAt: now });\n\t\t\tentityPointerIds.push(updated.id);\n\t\t} else {\n\t\t\t// Create new EntityNode with confirmed event (eventType=1)\n\t\t\tconst hash = nameHash(name);\n\t\t\tconst node: EntityNode = {\n\t\t\t\tid: globalThis.crypto.randomUUID(),\n\t\t\t\ttype,\n\t\t\t\tcanonicalName: name,\n\t\t\t\tcurrentNameHash: hash,\n\t\t\t\teventMatrix: [[1, now, 0, hash, sessionOrdinal]],\n\t\t\t\tlastReferencedAt: now,\n\t\t\t\tedges: [],\n\t\t\t};\n\t\t\tawait createEntity(storage, node);\n\t\t\tentityIndex.set(node);\n\t\t\tentityPointerIds.push(node.id);\n\t\t}\n\t}\n\n\treturn entityPointerIds;\n}\n\n// ---------------------------------------------------------------------------\n// ContextNode CRUD\n// ---------------------------------------------------------------------------\n\nexport async function loadAllContextNodes(storage: IMemoryStorage): Promise<ContextNode[]> {\n\treturn storage.readAll<ContextNode>(\"context\");\n}\n\nexport async function updateContextNode(\n\tstorage: IMemoryStorage,\n\tid: string,\n\tpatch: Partial<ContextNode>,\n): Promise<void> {\n\tawait storage.update<ContextNode>(\"context\", id, patch);\n}\n\nexport async function loadContextNodeWithSupersession(\n\tstorage: IMemoryStorage,\n\tid: string,\n): Promise<[ContextNode | null, ContextNode | null]> {\n\tconst all = await loadAllContextNodes(storage);\n\tconst node = all.find((n) => n.id === id) ?? null;\n\tif (!node) return [null, null];\n\n\tconst neighbourId = node.supersededBy ?? node.supersedes;\n\tconst neighbour = neighbourId ? (all.find((n) => n.id === neighbourId) ?? null) : null;\n\n\treturn [node, neighbour];\n}\n\n// ---------------------------------------------------------------------------\n// F-67: Entity event updates from agent observations\n// ---------------------------------------------------------------------------\n\nconst EVENT_TYPE_MAP: Record<\"renamed\" | \"deprecated\" | \"replaced\", 2 | 3 | 4> = {\n\trenamed: 2,\n\tdeprecated: 3,\n\treplaced: 4,\n};\n\nasync function applyEntityEvents(\n\tevents: EntityEventRaw[],\n\tentityIndex: EntityIndex,\n\tstorage: IMemoryStorage,\n\tsessionOrdinal: number,\n\tnow: number,\n): Promise<void> {\n\tfor (const evt of events) {\n\t\t// AC-67-02: Skip entities not in index — no creation in this path\n\t\tconst existing = entityIndex.get(evt.name, evt.type);\n\t\tif (!existing) continue;\n\n\t\tconst eventType = EVENT_TYPE_MAP[evt.event];\n\t\tconst newHash = evt.newName ? nameHash(evt.newName) : existing.currentNameHash;\n\t\tconst vector: import(\"./entity.js\").EventVector = [eventType, now, existing.currentNameHash, newHash, sessionOrdinal];\n\n\t\tconst updatedMatrix = [...existing.eventMatrix, vector];\n\t\tconst patch: Partial<import(\"./entity.js\").EntityNode> = { eventMatrix: updatedMatrix };\n\n\t\t// Update currentNameHash on rename\n\t\tif (evt.event === \"renamed\" && evt.newName) {\n\t\t\tpatch.currentNameHash = newHash;\n\t\t}\n\n\t\tawait updateEntity(storage, existing.id, patch);\n\t\tentityIndex.update(existing.id, patch);\n\t}\n}\n\n// ---------------------------------------------------------------------------\n// Main: write ContextNode from compaction event (F-57 + F-58 + F-67 + F-68)\n// ---------------------------------------------------------------------------\n\nexport async function writeContextNodeFromCompaction(params: {\n\tbranchEntries: MemoryBranchEntry[];\n\tsessionId: string;\n\t/** Absolute path to the Layer 1 session JSONL file — persisted on the ContextNode (F-75). */\n\tsessionFile?: string;\n\tstorage: IMemoryStorage;\n\tentityIndex: EntityIndex;\n\tcomplete: LLMCompleteFn;\n\tsessionOrdinal?: number;\n\tcwd?: string;\n}): Promise<ContextNode | null> {\n\tconst { branchEntries, sessionId, sessionFile, storage, entityIndex, complete, sessionOrdinal = 0, cwd = \"\" } = params;\n\n\tif (branchEntries.length === 0) return null;\n\n\tconst transcript = formatBranchEntries(branchEntries);\n\tconst messages = buildExtractionPrompt(transcript, cwd);\n\n\tlet extraction: ExtractionResult | null = null;\n\ttry {\n\t\tconst response = await complete(messages, { systemPrompt: EXTRACTION_SYSTEM_PROMPT });\n\t\textraction = parseExtractionResponse(response);\n\t} catch (err) {\n\t\tconsole.error(\"[memory] LLM extraction failed:\", err instanceof Error ? err.message : String(err));\n\t\treturn null;\n\t}\n\n\tif (!extraction) return null;\n\n\tconst now = Date.now();\n\n\t// F-58: Entity coupling — write entity nodes and get their IDs\n\tconst entityPointerIds = await coupleEntities(\n\t\textraction.entities,\n\t\tentityIndex,\n\t\tstorage,\n\t\tsessionOrdinal,\n\t\tnow,\n\t);\n\n\t// F-67: Apply entity event updates (AC-67-01, AC-67-02)\n\tif (extraction.entityEvents.length > 0) {\n\t\tawait applyEntityEvents(extraction.entityEvents, entityIndex, storage, sessionOrdinal, now);\n\t}\n\n\t// F-57 + F-68 + F-75: Write ContextNode with extended staleness fields and session file path\n\tconst node: ContextNode = {\n\t\tid: globalThis.crypto.randomUUID(),\n\t\tsessionId,\n\t\tsessionIds: [sessionId],\n\t\ttimestamp: now,\n\t\tuserContext: extraction.userContext,\n\t\tagentContext: extraction.agentContext,\n\t\ttopics: extraction.topics,\n\t\tentityPointerIds,\n\t\t// F-68: Extended staleness-evidence fields (AC-68-01, AC-68-02)\n\t\tworkingDirectory: cwd || undefined,\n\t\trelevantFilePaths: extraction.relevantFilePaths,\n\t\trelevantModules: extraction.relevantModules,\n\t\t// F-108: Search-oriented query terms for BM25F recall widening\n\t\tsearchTerms: extraction.searchTerms,\n\t\t// F-75: Self-contained Layer 1 file path (AC-75-01)\n\t\tsessionFile: sessionFile || undefined,\n\t};\n\n\tawait storage.append<ContextNode>(\"context\", node);\n\n\t// F-59: Supersession detection (second LLM call — best-effort, non-blocking)\n\ttry {\n\t\tawait detectSupersession(node, storage, sessionId, complete);\n\t} catch {\n\t\t// Non-fatal\n\t}\n\n\treturn node;\n}\n\n// ---------------------------------------------------------------------------\n// F-59: Supersession detection (second LLM call after F-58)\n// ---------------------------------------------------------------------------\n\nconst SUPERSESSION_SYSTEM_PROMPT = `You are a memory deduplication assistant for a coding agent.\nYou will be shown a new context summary and a list of previous summaries from the same session.\nDetermine if the new summary substantially revisits or supersedes exactly one of the previous ones.\n\nReturn ONLY the ID of the superseded summary, or null if nothing is superseded.\nA summary is superseded if the new one covers the same ground with updated or corrected information.\nDo NOT mark as superseded if both summaries are simultaneously valid (different scope, different files, different problems).\n\nRespond ONLY with valid JSON — no markdown, no explanation.`;\n\nfunction buildSupersessionPrompt(newNode: ContextNode, candidates: ContextNode[]) {\n\tconst candidateText = candidates\n\t\t.map((c, i) => `[${i + 1}] ID: ${c.id}\\nUser: ${c.userContext}\\nAgent: ${c.agentContext}`)\n\t\t.join(\"\\n\\n\");\n\n\treturn [\n\t\t{\n\t\t\trole: \"user\" as const,\n\t\t\ttimestamp: Date.now(),\n\t\t\tcontent: `New summary:\nUser: ${newNode.userContext}\nAgent: ${newNode.agentContext}\n\nPrevious summaries from this session:\n${candidateText}\n\nDoes the new summary substantially revisit or supersede exactly one of the previous summaries?\nIf yes, return the ID of the superseded summary. If no, return null.\n\nRespond with JSON only:\n{ \"supersededId\": \"<id>\" | null }`,\n\t\t},\n\t];\n}\n\nasync function detectSupersession(\n\tnewNode: ContextNode,\n\tstorage: IMemoryStorage,\n\tsessionId: string,\n\tcomplete: LLMCompleteFn,\n): Promise<void> {\n\t// AC-59-03: Only consider nodes from the current session (parent-chain scoping)\n\tconst all = await loadAllContextNodes(storage);\n\tconst candidates = all.filter((n) => n.id !== newNode.id && n.sessionIds.includes(sessionId) && !n.supersededBy);\n\n\tif (candidates.length === 0) return;\n\n\tconst messages = buildSupersessionPrompt(newNode, candidates);\n\tlet raw: string;\n\ttry {\n\t\tconst response = await complete(messages, { systemPrompt: SUPERSESSION_SYSTEM_PROMPT });\n\t\tconst textBlock = Array.isArray(response.content)\n\t\t\t? response.content.find((b) => (b as { type: string }).type === \"text\")\n\t\t\t: null;\n\t\traw = textBlock ? (textBlock as { text: string }).text : \"\";\n\t} catch {\n\t\treturn; // Non-fatal — supersession is best-effort\n\t}\n\n\tconst cleaned = raw.replace(/^```(?:json)?\\n?/m, \"\").replace(/\\n?```$/m, \"\").trim();\n\tlet supersededId: string | null = null;\n\ttry {\n\t\tconst parsed = JSON.parse(cleaned) as { supersededId?: string | null };\n\t\tsupersededId = typeof parsed.supersededId === \"string\" ? parsed.supersededId : null;\n\t} catch {\n\t\treturn;\n\t}\n\n\tif (!supersededId) return;\n\n\t// Verify the ID is actually a candidate (guard against hallucinated IDs)\n\tconst superseded = candidates.find((n) => n.id === supersededId);\n\tif (!superseded) return;\n\n\t// AC-59-01: Update both sides of the supersession pair\n\tconst now = Date.now();\n\tawait updateContextNode(storage, superseded.id, { supersededBy: newNode.id, supersededAt: now });\n\tawait updateContextNode(storage, newNode.id, { supersedes: superseded.id });\n}\n\n// ---------------------------------------------------------------------------\n// F-60: projectHealth — pure validity aggregation (no LLM)\n// ---------------------------------------------------------------------------\n\nexport interface ContextNodeHealth {\n\tmeanValidity: number;\n\tminValidity: number;\n\tdegradedCount: number;\n\tdeprecatedCount: number;\n}\n\nexport function projectHealth(\n\tnode: ContextNode,\n\tentityIndex: EntityIndex,\n\tnow: number,\n): ContextNodeHealth {\n\tif (node.entityPointerIds.length === 0) {\n\t\treturn { meanValidity: 1, minValidity: 1, degradedCount: 0, deprecatedCount: 0 };\n\t}\n\n\tlet sum = 0;\n\tlet min = Number.POSITIVE_INFINITY;\n\tlet degradedCount = 0;\n\tlet deprecatedCount = 0;\n\n\tfor (const id of node.entityPointerIds) {\n\t\tconst entity = entityIndex.getById(id);\n\t\tif (!entity) continue;\n\n\t\tconst score = validityScore(entity.eventMatrix, entity.type, now);\n\t\tsum += score;\n\t\tif (score < min) min = score;\n\t\tif (score < 0.3) degradedCount++;\n\t\tif (score < 0) deprecatedCount++;\n\t}\n\n\tconst count = node.entityPointerIds.length;\n\treturn {\n\t\tmeanValidity: sum / count,\n\t\tminValidity: min === Number.POSITIVE_INFINITY ? 1 : min,\n\t\tdegradedCount,\n\t\tdeprecatedCount,\n\t};\n}\n"]}
|