opencode-memory-pro 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +24 -0
- package/README.md +409 -0
- package/dist/config.d.ts +3 -0
- package/dist/config.js +398 -0
- package/dist/embedder.d.ts +26 -0
- package/dist/embedder.js +260 -0
- package/dist/extract.d.ts +4 -0
- package/dist/extract.js +181 -0
- package/dist/graph.js +701 -0
- package/dist/index.d.ts +4 -0
- package/dist/index.js +953 -0
- package/dist/llm.d.ts +14 -0
- package/dist/llm.js +212 -0
- package/dist/logger.d.ts +9 -0
- package/dist/logger.js +126 -0
- package/dist/ports.d.ts +34 -0
- package/dist/ports.js +129 -0
- package/dist/preference.d.ts +10 -0
- package/dist/preference.js +125 -0
- package/dist/scope.d.ts +2 -0
- package/dist/scope.js +48 -0
- package/dist/store.d.ts +194 -0
- package/dist/store.js +2738 -0
- package/dist/summarize.d.ts +52 -0
- package/dist/summarize.js +350 -0
- package/dist/tools/episodic.d.ts +68 -0
- package/dist/tools/episodic.js +145 -0
- package/dist/tools/feedback.d.ts +51 -0
- package/dist/tools/feedback.js +112 -0
- package/dist/tools/index.d.ts +3 -0
- package/dist/tools/index.js +3 -0
- package/dist/tools/memory.d.ts +293 -0
- package/dist/tools/memory.js +1487 -0
- package/dist/types.d.ts +489 -0
- package/dist/types.js +54 -0
- package/dist/utils.d.ts +18 -0
- package/dist/utils.js +214 -0
- package/package.json +49 -0
package/dist/graph.js
ADDED
|
@@ -0,0 +1,701 @@
|
|
|
1
|
+
// GRAPH_STORE_PHASE1: offline entity graph for opencode-memory-pro.
|
|
2
|
+
// Pure-heuristic entity extraction (no LLM), sqlite-backed co-occurrence
|
|
3
|
+
// edges, and a multiplicative graphBoost factor applied to recall scores.
|
|
4
|
+
// Storage is global-only by design (single-user scope patch).
|
|
5
|
+
// GRAPH_STORE_PHASE2: typed-relation edges. On top of every co-occurrence
|
|
6
|
+
// edge, sentence-level verb-pattern heuristics emit directional relations
|
|
7
|
+
// (uses / depends_on / runs_on / configured_in / connects_to / part_of /
|
|
8
|
+
// managed_by / manages / imports / writes_to / reads_from). Only emitted
|
|
9
|
+
// when BOTH endpoints are real extracted entities — plain words are ignored.
|
|
10
|
+
// Config: graph.typedEdges (default true), env OPENCODE_MEMORY_PRO_GRAPH_TYPED_EDGES.
|
|
11
|
+
// GRAPH_STORE_PHASE2B: graph-expansion recall. BFS from the query's
|
|
12
|
+
// extracted entities over the edge table (up to graph.maxHops, preferring
|
|
13
|
+
// typed relations to generic co_occurs), collecting memory ids attached to
|
|
14
|
+
// the visited non-seed entities via memory_entities. Callers merge the
|
|
15
|
+
// returned candidates into hybrid-search results with a graph-origin score,
|
|
16
|
+
// so memories that DON'T text/vector-match can still surface when they are
|
|
17
|
+
// 1..maxHops away in the entity graph. Work is bounded (per-entity edge
|
|
18
|
+
// fanout + total visited-entity budget) so a recall never scans the whole
|
|
19
|
+
// graph. Config: graph.expansionEnabled / maxHops / expansionLimit /
|
|
20
|
+
// expansionLambda (env OPENCODE_MEMORY_PRO_GRAPH_EXPANSION_*).
|
|
21
|
+
// GRAPH_STORE_POLISH: phased scoring polish (0.9) — boostResults strength
|
|
22
|
+
// smoothed (single-entity match 0.75 instead of 0.5; >=2 entities capped at
|
|
23
|
+
// 1.0) and typed-edge preference in expandRecall raised 1.3x -> 1.5x.
|
|
24
|
+
import { mkdirSync } from "node:fs";
|
|
25
|
+
import { dirname } from "node:path";
|
|
26
|
+
import { GLOBAL_KEYWORDS } from "./extract.js";
|
|
27
|
+
import { log } from "./logger.js";
|
|
28
|
+
const FILE_EXTENSION_RE = /\b[\w@./-]+\.(?:js|jsx|ts|tsx|mjs|cjs|json|jsonc|sh|bash|py|md|markdown|toml|yaml|yml|css|scss|html|go|rs|c|h|cpp|hpp|java|kt|sql|lock|mod|sum|env|conf|ini|cfg|service|db|sqlite|png|jpg|jpeg|svg|webp|gif|pdf|zip|tar|gz|log|txt|xml|proto|graphql|prisma|d\.ts|tsbuildinfo)\b/gi;
|
|
29
|
+
const DOT_KEY_RE = /\b[a-zA-Z][\w-]*(?:\.[\w-]+){1,4}\b/g;
|
|
30
|
+
const SCOPED_PKG_RE = /(?<![\w@.-])@[\w-]+\/[\w@./-]+\b/g;
|
|
31
|
+
const CAMEL_CASE_RE = /\b[a-z][a-z0-9]{1,}[A-Z][a-zA-Z0-9]*\b/g;
|
|
32
|
+
const SNAKE_CASE_RE = /\b[a-z][a-z0-9]*(?:_[a-z0-9]+){1,}\b/g;
|
|
33
|
+
const STOPWORDS = new Set([
|
|
34
|
+
"the", "and", "for", "with", "this", "that", "from", "were", "have", "been", "when", "what", "which",
|
|
35
|
+
"your", "you", "our", "about", "into", "after", "before", "over", "under", "again", "then", "them",
|
|
36
|
+
"some", "such", "only", "other", "just", "than", "very", "will", "would", "there", "their", "these",
|
|
37
|
+
"those", "can", "could", "should", "shall", "may", "might", "must", "not", "are", "was", "out", "off",
|
|
38
|
+
"does", "did", "done", "also", "because", "until", "while", "using", "used", "use", "via", "per", "its",
|
|
39
|
+
"has", "had", "being", "both", "each", "few", "more", "most", "nor", "own", "same", "so", "too", "up",
|
|
40
|
+
"down", "in", "on", "at", "to", "of", "is", "as", "by", "be", "or", "an", "a", "it", "no", "yes",
|
|
41
|
+
]);
|
|
42
|
+
// ---------------------------------------------------------------------------
|
|
43
|
+
// GRAPH_STORE_PHASE2: typed-relation patterns.
|
|
44
|
+
// Each pattern matches a verbal phrase; the subject/object are resolved by
|
|
45
|
+
// finding the extracted entities closest to the phrase (rightmost on the
|
|
46
|
+
// left / leftmost on the right), so only real entity pairs become edges.
|
|
47
|
+
// ---------------------------------------------------------------------------
|
|
48
|
+
const RELATION_PATTERNS = [
|
|
49
|
+
{ relation: "uses", re: /\b(?:uses|use|using|consumes?|leverages?|utilizes?)\b/g },
|
|
50
|
+
{ relation: "depends_on", re: /\b(?:depends?\s+(?:on|upon)|requires?|needs?)\b/g },
|
|
51
|
+
{ relation: "runs_on", re: /\b(?:runs?\s+on|running\s+on|deployed\s+(?:on|to)|hosted\s+on|installed\s+on|executes?\s+on)\b/g },
|
|
52
|
+
{ relation: "configured_in", re: /\b(?:configured\s+(?:in|via|through|by|with)|configures?\s+(?:in|via|through)|set\s+in|defined\s+in)\b/g },
|
|
53
|
+
{ relation: "connects_to", re: /\b(?:connects?\s+to|connected\s+to|talks?\s+to|communicates?\s+with|listens?\s+on|bound\s+to|attached\s+to)\b/g },
|
|
54
|
+
{ relation: "part_of", re: /\b(?:part\s+of|portion\s+of|member\s+of|included\s+in|bundled\s+with|shipped\s+with)\b/g },
|
|
55
|
+
{ relation: "managed_by", re: /\b(?:managed\s+(?:by|with|via)|controlled\s+by|orchestrated\s+by|supervised\s+by|handled\s+by)\b/g },
|
|
56
|
+
{ relation: "manages", re: /\b(?:manages?|controls?|orchestrates?|supervises?)\b/g },
|
|
57
|
+
{ relation: "imports", re: /\b(?:imports?|importing|bundles?|embeds?|includes?)\b/g },
|
|
58
|
+
{ relation: "writes_to", re: /\b(?:writes?\s+to|writing\s+to|pushes?\s+to|saves?\s+to|persists?\s+to|logs?\s+to)\b/g },
|
|
59
|
+
{ relation: "reads_from", re: /\b(?:reads?\s+from|reading\s+from|pulls?\s+from|loads?\s+from)\b/g },
|
|
60
|
+
];
|
|
61
|
+
const MAX_TYPED_RELATIONS_PER_MEMORY = 40;
|
|
62
|
+
// GRAPH_STORE_PHASE2: conjunctive-continuation heuristic. When a verb phrase
|
|
63
|
+
// directly follows a conjunction ("A uses B and runs on C"), the clause has
|
|
64
|
+
// no explicit subject — inherit it from the preceding clause.
|
|
65
|
+
const CONJUNCTION_RE = /\b(?:and|but|then|so|while|because)\s*$/;
|
|
66
|
+
// ---------------------------------------------------------------------------
|
|
67
|
+
export function extractTypedRelations(text, entities) {
|
|
68
|
+
if (!text || !entities || entities.length < 2)
|
|
69
|
+
return [];
|
|
70
|
+
const names = Array.from(new Set(entities.map((e) => e.name).filter(Boolean)))
|
|
71
|
+
.sort((a, b) => b.length - a.length);
|
|
72
|
+
if (names.length < 2)
|
|
73
|
+
return [];
|
|
74
|
+
const found = new Map();
|
|
75
|
+
const sentences = text.toLowerCase().split(/[.!?;]+\s+|\n+/);
|
|
76
|
+
for (const sentence of sentences) {
|
|
77
|
+
if (sentence.trim().length === 0)
|
|
78
|
+
continue;
|
|
79
|
+
for (const pattern of RELATION_PATTERNS) {
|
|
80
|
+
pattern.re.lastIndex = 0;
|
|
81
|
+
let match;
|
|
82
|
+
while ((match = pattern.re.exec(sentence)) !== null) {
|
|
83
|
+
if (found.size >= MAX_TYPED_RELATIONS_PER_MEMORY)
|
|
84
|
+
return Array.from(found.values());
|
|
85
|
+
const left = sentence.slice(0, match.index);
|
|
86
|
+
// Ellipsis rule: "docker-compose uses postgres and runs on linux" —
|
|
87
|
+
// the verb directly follows a conjunction, so the clause has no
|
|
88
|
+
// subject of its own; reuse the preceding clause's subject
|
|
89
|
+
// (approximated as the leftmost entity before the conjunction).
|
|
90
|
+
const conj = CONJUNCTION_RE.exec(left);
|
|
91
|
+
const subject = conj ? pickClosestEntity(left.slice(0, conj.index), names, true) : pickClosestEntity(left, names, false);
|
|
92
|
+
const object = pickClosestEntity(sentence.slice(match.index + match[0].length), names, true);
|
|
93
|
+
if (subject && object && subject !== object) {
|
|
94
|
+
found.set(`${subject}|${pattern.relation}|${object}`, { src: subject, dst: object, relation: pattern.relation });
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
return Array.from(found.values());
|
|
100
|
+
}
|
|
101
|
+
function pickClosestEntity(windowText, names, fromStart) {
|
|
102
|
+
let best = null;
|
|
103
|
+
let bestIdx = fromStart ? Infinity : -1;
|
|
104
|
+
let bestLen = -1;
|
|
105
|
+
for (const name of names) {
|
|
106
|
+
const idx = windowText.indexOf(name);
|
|
107
|
+
if (idx === -1)
|
|
108
|
+
continue;
|
|
109
|
+
const better = fromStart ? idx < bestIdx : idx > bestIdx;
|
|
110
|
+
if (better || (idx === bestIdx && name.length > bestLen)) {
|
|
111
|
+
best = name;
|
|
112
|
+
bestIdx = idx;
|
|
113
|
+
bestLen = name.length;
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
return best;
|
|
117
|
+
}
|
|
118
|
+
let driverModulePromise = null;
|
|
119
|
+
function loadDriverModule() {
|
|
120
|
+
if (driverModulePromise)
|
|
121
|
+
return driverModulePromise;
|
|
122
|
+
driverModulePromise = (async () => {
|
|
123
|
+
try {
|
|
124
|
+
const mod = await import("bun:sqlite");
|
|
125
|
+
if (mod?.Database)
|
|
126
|
+
return { ctor: mod.Database, name: "bun:sqlite" };
|
|
127
|
+
}
|
|
128
|
+
catch { }
|
|
129
|
+
try {
|
|
130
|
+
const mod = await import("node:sqlite");
|
|
131
|
+
if (mod?.DatabaseSync)
|
|
132
|
+
return { ctor: mod.DatabaseSync, name: "node:sqlite" };
|
|
133
|
+
}
|
|
134
|
+
catch { }
|
|
135
|
+
return null;
|
|
136
|
+
})();
|
|
137
|
+
return driverModulePromise;
|
|
138
|
+
}
|
|
139
|
+
function normalizeEntityName(raw, type, skipDotted = false) {
|
|
140
|
+
let name = raw.trim().toLowerCase().replace(/\s+/g, " ");
|
|
141
|
+
name = name.replace(/^[`'"(\[{~]/, "");
|
|
142
|
+
name = name.replace(/[`'")\]},;:.]+$/, "");
|
|
143
|
+
name = name.replace(/^\.\//, "");
|
|
144
|
+
if (name.length < 3 || name.length > 96)
|
|
145
|
+
return null;
|
|
146
|
+
if (/^\d+(\.\d+)*$/.test(name))
|
|
147
|
+
return null;
|
|
148
|
+
if (skipDotted) {
|
|
149
|
+
const segments = name.split(".");
|
|
150
|
+
if (segments.length > 1 && segments.some((s) => s.length < 2 || /^\d+$/.test(s)))
|
|
151
|
+
return null;
|
|
152
|
+
}
|
|
153
|
+
if (STOPWORDS.has(name))
|
|
154
|
+
return null;
|
|
155
|
+
return { name, type };
|
|
156
|
+
}
|
|
157
|
+
export function extractEntities(text) {
|
|
158
|
+
if (!text || text.trim().length === 0)
|
|
159
|
+
return [];
|
|
160
|
+
const seen = new Map();
|
|
161
|
+
const add = (raw, type, skipDotted = false) => {
|
|
162
|
+
if (!raw)
|
|
163
|
+
return;
|
|
164
|
+
const normalized = normalizeEntityName(raw, type, skipDotted);
|
|
165
|
+
if (!normalized)
|
|
166
|
+
return;
|
|
167
|
+
if (!seen.has(normalized.name)) {
|
|
168
|
+
seen.set(normalized.name, normalized);
|
|
169
|
+
}
|
|
170
|
+
};
|
|
171
|
+
let m;
|
|
172
|
+
FILE_EXTENSION_RE.lastIndex = 0;
|
|
173
|
+
while ((m = FILE_EXTENSION_RE.exec(text)) !== null) {
|
|
174
|
+
add(m[0], "file");
|
|
175
|
+
}
|
|
176
|
+
DOT_KEY_RE.lastIndex = 0;
|
|
177
|
+
while ((m = DOT_KEY_RE.exec(text)) !== null) {
|
|
178
|
+
const candidate = m[0];
|
|
179
|
+
if (/\.(?:js|jsx|ts|tsx|json|jsonc|sh|py|md|toml|yaml|yml|css|html|go|rs|sql)$/i.test(candidate))
|
|
180
|
+
continue;
|
|
181
|
+
add(candidate, "config-key", true);
|
|
182
|
+
}
|
|
183
|
+
SCOPED_PKG_RE.lastIndex = 0;
|
|
184
|
+
while ((m = SCOPED_PKG_RE.exec(text)) !== null) {
|
|
185
|
+
add(m[0], "package");
|
|
186
|
+
}
|
|
187
|
+
CAMEL_CASE_RE.lastIndex = 0;
|
|
188
|
+
while ((m = CAMEL_CASE_RE.exec(text)) !== null) {
|
|
189
|
+
const candidate = m[0];
|
|
190
|
+
if (candidate.length < 6)
|
|
191
|
+
continue;
|
|
192
|
+
add(candidate, "identifier");
|
|
193
|
+
}
|
|
194
|
+
SNAKE_CASE_RE.lastIndex = 0;
|
|
195
|
+
while ((m = SNAKE_CASE_RE.exec(text)) !== null) {
|
|
196
|
+
const candidate = m[0];
|
|
197
|
+
if (candidate.length < 5)
|
|
198
|
+
continue;
|
|
199
|
+
add(candidate, "identifier");
|
|
200
|
+
}
|
|
201
|
+
for (const keyword of GLOBAL_KEYWORDS) {
|
|
202
|
+
const escaped = keyword.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
203
|
+
const re = new RegExp(`\\b${escaped}\\b`, "i");
|
|
204
|
+
if (re.test(text)) {
|
|
205
|
+
add(keyword, "infra");
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
return Array.from(seen.values());
|
|
209
|
+
}
|
|
210
|
+
export async function createGraphStore(config) {
|
|
211
|
+
try {
|
|
212
|
+
const driver = await loadDriverModule();
|
|
213
|
+
if (!driver) {
|
|
214
|
+
log("warn", "[graph] no sqlite driver available (bun:sqlite/node:sqlite) - graph disabled");
|
|
215
|
+
return new DisabledGraphStore();
|
|
216
|
+
}
|
|
217
|
+
return new GraphStore(config, driver);
|
|
218
|
+
}
|
|
219
|
+
catch (error) {
|
|
220
|
+
log("warn", `[graph] failed to initialize graph store: ${error instanceof Error ? error.message : String(error)}`);
|
|
221
|
+
return new DisabledGraphStore();
|
|
222
|
+
}
|
|
223
|
+
}
|
|
224
|
+
export class GraphStore {
|
|
225
|
+
db = null;
|
|
226
|
+
enabled = false;
|
|
227
|
+
maxEntitiesPerMemory;
|
|
228
|
+
maxEdgeProvenance;
|
|
229
|
+
typedEdges = true;
|
|
230
|
+
constructor(config, driver) {
|
|
231
|
+
this.maxEntitiesPerMemory = config.maxEntitiesPerMemory ?? 20;
|
|
232
|
+
this.maxEdgeProvenance = config.maxEdgeProvenance ?? 20;
|
|
233
|
+
this.typedEdges = config.typedEdges !== false;
|
|
234
|
+
const expanded = config.dbPath.replace(/^~(?=\/)/, process.env.HOME ?? "");
|
|
235
|
+
mkdirSync(dirname(expanded), { recursive: true });
|
|
236
|
+
this.db = new driver.ctor(expanded);
|
|
237
|
+
this.db.exec("PRAGMA journal_mode=WAL");
|
|
238
|
+
this.db.exec(`
|
|
239
|
+
CREATE TABLE IF NOT EXISTS entities (
|
|
240
|
+
name TEXT PRIMARY KEY,
|
|
241
|
+
type TEXT NOT NULL DEFAULT 'other',
|
|
242
|
+
first_seen INTEGER NOT NULL,
|
|
243
|
+
last_seen INTEGER NOT NULL,
|
|
244
|
+
mention_count INTEGER NOT NULL DEFAULT 1
|
|
245
|
+
);
|
|
246
|
+
CREATE TABLE IF NOT EXISTS memory_entities (
|
|
247
|
+
memory_id TEXT NOT NULL,
|
|
248
|
+
entity_name TEXT NOT NULL,
|
|
249
|
+
PRIMARY KEY (memory_id, entity_name)
|
|
250
|
+
);
|
|
251
|
+
CREATE INDEX IF NOT EXISTS idx_memory_entities_entity ON memory_entities(entity_name);
|
|
252
|
+
CREATE TABLE IF NOT EXISTS edges (
|
|
253
|
+
src TEXT NOT NULL,
|
|
254
|
+
dst TEXT NOT NULL,
|
|
255
|
+
relation TEXT NOT NULL DEFAULT 'co_occurs',
|
|
256
|
+
weight REAL NOT NULL DEFAULT 1,
|
|
257
|
+
first_seen INTEGER NOT NULL,
|
|
258
|
+
last_seen INTEGER NOT NULL,
|
|
259
|
+
provenance TEXT NOT NULL DEFAULT '[]',
|
|
260
|
+
PRIMARY KEY (src, dst, relation)
|
|
261
|
+
);
|
|
262
|
+
CREATE INDEX IF NOT EXISTS idx_edges_src ON edges(src);
|
|
263
|
+
CREATE INDEX IF NOT EXISTS idx_edges_dst ON edges(dst);
|
|
264
|
+
`);
|
|
265
|
+
this.enabled = true;
|
|
266
|
+
log("info", `[graph] enabled (sqlite: ${driver.name}, ${config.dbPath}, typedEdges=${this.typedEdges})`);
|
|
267
|
+
}
|
|
268
|
+
extract(text) {
|
|
269
|
+
if (!this.enabled)
|
|
270
|
+
return [];
|
|
271
|
+
return extractEntities(text).slice(0, this.maxEntitiesPerMemory);
|
|
272
|
+
}
|
|
273
|
+
indexMemory(memoryId, text, timestamp) {
|
|
274
|
+
if (!this.enabled || !memoryId || !text)
|
|
275
|
+
return;
|
|
276
|
+
const entities = extractEntities(text).slice(0, this.maxEntitiesPerMemory);
|
|
277
|
+
if (entities.length === 0)
|
|
278
|
+
return;
|
|
279
|
+
this.begin();
|
|
280
|
+
try {
|
|
281
|
+
for (const entity of entities) {
|
|
282
|
+
this.db.prepare(`
|
|
283
|
+
INSERT INTO entities (name, type, first_seen, last_seen, mention_count)
|
|
284
|
+
VALUES (?, ?, ?, ?, 1)
|
|
285
|
+
ON CONFLICT(name) DO UPDATE SET
|
|
286
|
+
last_seen = excluded.last_seen,
|
|
287
|
+
mention_count = mention_count + 1
|
|
288
|
+
`).run(entity.name, entity.type, timestamp, timestamp);
|
|
289
|
+
this.db.prepare(`
|
|
290
|
+
INSERT OR IGNORE INTO memory_entities (memory_id, entity_name) VALUES (?, ?)
|
|
291
|
+
`).run(memoryId, entity.name);
|
|
292
|
+
}
|
|
293
|
+
for (let i = 0; i < entities.length; i += 1) {
|
|
294
|
+
for (let j = i + 1; j < entities.length; j += 1) {
|
|
295
|
+
const a = entities[i].name;
|
|
296
|
+
const b = entities[j].name;
|
|
297
|
+
const pair = a < b ? [a, b] : [b, a];
|
|
298
|
+
this.upsertEdge(pair[0], pair[1], "co_occurs", memoryId, timestamp);
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
if (this.typedEdges) {
|
|
302
|
+
const typed = extractTypedRelations(text, entities);
|
|
303
|
+
for (const t of typed) {
|
|
304
|
+
this.upsertEdge(t.src, t.dst, t.relation, memoryId, timestamp);
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
this.commit();
|
|
308
|
+
}
|
|
309
|
+
catch (error) {
|
|
310
|
+
this.rollback();
|
|
311
|
+
log("warn", `[graph] indexMemory failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
upsertEdge(src, dst, relation, memoryId, timestamp) {
|
|
315
|
+
const row = this.db.prepare("SELECT weight, provenance, first_seen FROM edges WHERE src = ? AND dst = ? AND relation = ?").get(src, dst, relation);
|
|
316
|
+
let provenance = [];
|
|
317
|
+
let firstSeen = timestamp;
|
|
318
|
+
let weight = 0;
|
|
319
|
+
if (row) {
|
|
320
|
+
try {
|
|
321
|
+
provenance = JSON.parse(row.provenance ?? "[]");
|
|
322
|
+
}
|
|
323
|
+
catch {
|
|
324
|
+
provenance = [];
|
|
325
|
+
}
|
|
326
|
+
firstSeen = row.first_seen;
|
|
327
|
+
weight = typeof row.weight === "number" ? row.weight : 0;
|
|
328
|
+
}
|
|
329
|
+
if (!provenance.includes(memoryId)) {
|
|
330
|
+
provenance.push(memoryId);
|
|
331
|
+
if (provenance.length > this.maxEdgeProvenance) {
|
|
332
|
+
provenance = provenance.slice(-this.maxEdgeProvenance);
|
|
333
|
+
}
|
|
334
|
+
weight = provenance.length;
|
|
335
|
+
}
|
|
336
|
+
this.db.prepare(`
|
|
337
|
+
INSERT INTO edges (src, dst, relation, weight, first_seen, last_seen, provenance)
|
|
338
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
339
|
+
ON CONFLICT(src, dst, relation) DO UPDATE SET
|
|
340
|
+
weight = excluded.weight,
|
|
341
|
+
last_seen = excluded.last_seen,
|
|
342
|
+
provenance = excluded.provenance
|
|
343
|
+
`).run(src, dst, relation, weight, firstSeen, timestamp, JSON.stringify(provenance));
|
|
344
|
+
}
|
|
345
|
+
// GRAPH_STORE_PHASE2: one-time/manual pass that adds typed edges for
|
|
346
|
+
// memories indexed before typed relations existed. Does NOT touch
|
|
347
|
+
// entities/memory_entities (idempotent: upsertEdge only appends a
|
|
348
|
+
// memory_id to provenance if it isn't already there).
|
|
349
|
+
backfillTypedEdges(records) {
|
|
350
|
+
if (!this.enabled || !this.typedEdges || !records || records.length === 0)
|
|
351
|
+
return;
|
|
352
|
+
const now = Date.now();
|
|
353
|
+
let added = 0;
|
|
354
|
+
for (const record of records) {
|
|
355
|
+
if (!record?.id || !record?.text)
|
|
356
|
+
continue;
|
|
357
|
+
const entities = extractEntities(record.text).slice(0, this.maxEntitiesPerMemory);
|
|
358
|
+
const typed = extractTypedRelations(record.text, entities);
|
|
359
|
+
if (typed.length === 0)
|
|
360
|
+
continue;
|
|
361
|
+
this.begin();
|
|
362
|
+
try {
|
|
363
|
+
for (const t of typed) {
|
|
364
|
+
this.upsertEdge(t.src, t.dst, t.relation, record.id, record.timestamp ?? now);
|
|
365
|
+
}
|
|
366
|
+
added += typed.length;
|
|
367
|
+
this.commit();
|
|
368
|
+
}
|
|
369
|
+
catch (error) {
|
|
370
|
+
this.rollback();
|
|
371
|
+
log("warn", `[graph] backfillTypedEdges failed for ${record.id}: ${error instanceof Error ? error.message : String(error)}`);
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
log("info", `[graph] typed-edge backfill: scanned ${records.length} memories, +${added} typed edges`);
|
|
375
|
+
}
|
|
376
|
+
onMemoryRemoved(memoryId) {
|
|
377
|
+
if (!this.enabled || !memoryId)
|
|
378
|
+
return;
|
|
379
|
+
this.begin();
|
|
380
|
+
try {
|
|
381
|
+
const mapped = this.db.prepare("SELECT entity_name FROM memory_entities WHERE memory_id = ?").all(memoryId);
|
|
382
|
+
const entityNames = mapped.map((r) => r.entity_name);
|
|
383
|
+
this.db.prepare("DELETE FROM memory_entities WHERE memory_id = ?").run(memoryId);
|
|
384
|
+
const edges = this.db.prepare(`
|
|
385
|
+
SELECT src, dst, relation, weight, provenance FROM edges
|
|
386
|
+
WHERE instr(provenance, ?) > 0
|
|
387
|
+
`).all(JSON.stringify(memoryId));
|
|
388
|
+
for (const edge of edges) {
|
|
389
|
+
let provenance = [];
|
|
390
|
+
try {
|
|
391
|
+
provenance = JSON.parse(edge.provenance ?? "[]");
|
|
392
|
+
}
|
|
393
|
+
catch {
|
|
394
|
+
provenance = [];
|
|
395
|
+
}
|
|
396
|
+
const filtered = provenance.filter((id) => id !== memoryId);
|
|
397
|
+
if (filtered.length === 0) {
|
|
398
|
+
this.db.prepare("DELETE FROM edges WHERE src = ? AND dst = ? AND relation = ?").run(edge.src, edge.dst, edge.relation);
|
|
399
|
+
}
|
|
400
|
+
else {
|
|
401
|
+
this.db.prepare(`
|
|
402
|
+
UPDATE edges SET weight = ?, provenance = ? WHERE src = ? AND dst = ? AND relation = ?
|
|
403
|
+
`).run(filtered.length, JSON.stringify(filtered), edge.src, edge.dst, edge.relation);
|
|
404
|
+
}
|
|
405
|
+
}
|
|
406
|
+
for (const name of entityNames) {
|
|
407
|
+
this.db.prepare("UPDATE entities SET mention_count = mention_count - 1 WHERE name = ?").run(name);
|
|
408
|
+
const entity = this.db.prepare("SELECT mention_count FROM entities WHERE name = ?").get(name);
|
|
409
|
+
if (entity && entity.mention_count <= 0) {
|
|
410
|
+
this.db.prepare("DELETE FROM entities WHERE name = ?").run(name);
|
|
411
|
+
}
|
|
412
|
+
}
|
|
413
|
+
this.commit();
|
|
414
|
+
}
|
|
415
|
+
catch (error) {
|
|
416
|
+
this.rollback();
|
|
417
|
+
log("warn", `[graph] onMemoryRemoved failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
418
|
+
}
|
|
419
|
+
}
|
|
420
|
+
onMemoryMerged(olderId, newerId) {
|
|
421
|
+
if (!this.enabled || !olderId || !newerId || olderId === newerId)
|
|
422
|
+
return;
|
|
423
|
+
this.begin();
|
|
424
|
+
try {
|
|
425
|
+
this.db.prepare(`
|
|
426
|
+
INSERT OR IGNORE INTO memory_entities (memory_id, entity_name)
|
|
427
|
+
SELECT ?, entity_name FROM memory_entities WHERE memory_id = ?
|
|
428
|
+
`).run(newerId, olderId);
|
|
429
|
+
this.db.prepare("DELETE FROM memory_entities WHERE memory_id = ?").run(olderId);
|
|
430
|
+
const edges = this.db.prepare(`
|
|
431
|
+
SELECT src, dst, relation, weight, provenance FROM edges
|
|
432
|
+
WHERE instr(provenance, ?) > 0
|
|
433
|
+
`).all(JSON.stringify(olderId));
|
|
434
|
+
for (const edge of edges) {
|
|
435
|
+
let provenance = [];
|
|
436
|
+
try {
|
|
437
|
+
provenance = JSON.parse(edge.provenance ?? "[]");
|
|
438
|
+
}
|
|
439
|
+
catch {
|
|
440
|
+
provenance = [];
|
|
441
|
+
}
|
|
442
|
+
let filtered = provenance.filter((id) => id !== olderId);
|
|
443
|
+
if (!filtered.includes(newerId)) {
|
|
444
|
+
filtered.push(newerId);
|
|
445
|
+
}
|
|
446
|
+
if (filtered.length > this.maxEdgeProvenance) {
|
|
447
|
+
filtered = filtered.slice(-this.maxEdgeProvenance);
|
|
448
|
+
}
|
|
449
|
+
this.db.prepare(`
|
|
450
|
+
UPDATE edges SET weight = ?, provenance = ? WHERE src = ? AND dst = ? AND relation = ?
|
|
451
|
+
`).run(filtered.length, JSON.stringify(filtered), edge.src, edge.dst, edge.relation);
|
|
452
|
+
}
|
|
453
|
+
this.commit();
|
|
454
|
+
}
|
|
455
|
+
catch (error) {
|
|
456
|
+
this.rollback();
|
|
457
|
+
log("warn", `[graph] onMemoryMerged failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
458
|
+
}
|
|
459
|
+
}
|
|
460
|
+
getMemoryEntities(memoryIds) {
|
|
461
|
+
const out = new Map();
|
|
462
|
+
if (!this.enabled || !memoryIds || memoryIds.length === 0)
|
|
463
|
+
return out;
|
|
464
|
+
const validIds = memoryIds.filter((id) => typeof id === "string" && id.length > 0);
|
|
465
|
+
if (validIds.length === 0)
|
|
466
|
+
return out;
|
|
467
|
+
for (let i = 0; i < validIds.length; i += 200) {
|
|
468
|
+
const chunk = validIds.slice(i, i + 200);
|
|
469
|
+
const placeholders = chunk.map(() => "?").join(",");
|
|
470
|
+
const rows = this.db.prepare(`SELECT memory_id, entity_name FROM memory_entities WHERE memory_id IN (${placeholders})`).all(...chunk);
|
|
471
|
+
for (const row of rows) {
|
|
472
|
+
let set = out.get(row.memory_id);
|
|
473
|
+
if (!set) {
|
|
474
|
+
set = new Set();
|
|
475
|
+
out.set(row.memory_id, set);
|
|
476
|
+
}
|
|
477
|
+
set.add(row.entity_name);
|
|
478
|
+
}
|
|
479
|
+
}
|
|
480
|
+
return out;
|
|
481
|
+
}
|
|
482
|
+
getEntitiesForQuery(text) {
|
|
483
|
+
if (!this.enabled)
|
|
484
|
+
return [];
|
|
485
|
+
return extractEntities(text).slice(0, this.maxEntitiesPerMemory);
|
|
486
|
+
}
|
|
487
|
+
boostResults(query, results, lambda) {
|
|
488
|
+
if (!this.enabled || !results || results.length === 0)
|
|
489
|
+
return results;
|
|
490
|
+
const boostLambda = lambda && Number.isFinite(lambda) ? lambda : 0;
|
|
491
|
+
if (boostLambda <= 0)
|
|
492
|
+
return results;
|
|
493
|
+
const queryEntities = extractEntities(query).slice(0, this.maxEntitiesPerMemory);
|
|
494
|
+
if (queryEntities.length === 0)
|
|
495
|
+
return results;
|
|
496
|
+
const idToEntityMap = this.getMemoryEntities(results.map((r) => r.record?.id));
|
|
497
|
+
return results.map((r) => {
|
|
498
|
+
const record = r.record;
|
|
499
|
+
if (!record)
|
|
500
|
+
return r;
|
|
501
|
+
const memEntities = idToEntityMap.get(record.id);
|
|
502
|
+
if (!memEntities || memEntities.size === 0)
|
|
503
|
+
return r;
|
|
504
|
+
let overlap = 0;
|
|
505
|
+
for (const qe of queryEntities) {
|
|
506
|
+
if (memEntities.has(qe.name)) {
|
|
507
|
+
overlap += 1;
|
|
508
|
+
}
|
|
509
|
+
}
|
|
510
|
+
if (overlap === 0)
|
|
511
|
+
return r;
|
|
512
|
+
// GRAPH_STORE_POLISH: smoothed strength curve — 1 entity = 0.75,
|
|
513
|
+
// 2 or more = 1.0 (capped). Previously 1 entity only got 0.5,
|
|
514
|
+
// which under-rated sparse-but-relevant single-topic matches.
|
|
515
|
+
const strength = Math.min(1, 0.5 + overlap * 0.25);
|
|
516
|
+
const graphBoost = 1 + boostLambda * strength;
|
|
517
|
+
return { ...r, score: r.score * graphBoost, graphBoost, graphOverlap: overlap };
|
|
518
|
+
});
|
|
519
|
+
}
|
|
520
|
+
// GRAPH_STORE_PHASE2B: graph-expansion recall (BFS). See header comment.
|
|
521
|
+
// Returns candidates [{memoryId, hops, path, relation, typed, scoreFactor}]
|
|
522
|
+
// SORTED best-first, capped at opts.expansionLimit. Callers resolve the
|
|
523
|
+
// actual records (scope-filtered) and merge with a graph-origin score:
|
|
524
|
+
// entryScore = floorScore * scoreFactor, where floorScore is the weakest
|
|
525
|
+
// retrieved result (or the retrieve floor when nothing matched) — so
|
|
526
|
+
// expansions rank just below/around real matches, never above them.
|
|
527
|
+
expandRecall(query, opts = {}) {
|
|
528
|
+
if (!this.enabled || !query)
|
|
529
|
+
return [];
|
|
530
|
+
const queryEntities = extractEntities(query).slice(0, this.maxEntitiesPerMemory);
|
|
531
|
+
if (queryEntities.length === 0)
|
|
532
|
+
return [];
|
|
533
|
+
const seeds = new Set(queryEntities.map((e) => e.name));
|
|
534
|
+
const maxHops = Math.min(4, Math.max(1, Math.floor(opts.maxHops ?? 2)));
|
|
535
|
+
const expansionLimit = Math.max(1, Math.floor(opts.expansionLimit ?? 5));
|
|
536
|
+
const expansionLambda = opts.expansionLambda != null && Number.isFinite(opts.expansionLambda)
|
|
537
|
+
? Math.max(0, Math.min(1, opts.expansionLambda))
|
|
538
|
+
: 0.3;
|
|
539
|
+
const hopDecay = opts.hopDecay != null && Number.isFinite(opts.hopDecay)
|
|
540
|
+
? Math.max(0, Math.min(1, opts.hopDecay))
|
|
541
|
+
: 0.7;
|
|
542
|
+
const MAX_EDGES_PER_ENTITY = 100;
|
|
543
|
+
const MAX_VISITED_ENTITIES = 200;
|
|
544
|
+
// GRAPH_STORE_POLISH: typed-edge preference raised from 1.3x to
|
|
545
|
+
// 1.5x (the phase-2a weight nudge) — directional relations should
|
|
546
|
+
// outrank generic co-occurrence when both reach the same neighbor.
|
|
547
|
+
const TYPED_RELATION_STRENGTH = 1.5;
|
|
548
|
+
const visitedEntities = new Set(seeds);
|
|
549
|
+
const chainByEntity = new Map();
|
|
550
|
+
for (const seed of seeds) {
|
|
551
|
+
chainByEntity.set(seed, [seed]);
|
|
552
|
+
}
|
|
553
|
+
const memCache = new Map();
|
|
554
|
+
const best = new Map();
|
|
555
|
+
const now = Date.now();
|
|
556
|
+
let frontier = Array.from(seeds);
|
|
557
|
+
let hop = 1;
|
|
558
|
+
while (hop <= maxHops && frontier.length > 0 && visitedEntities.size <= MAX_VISITED_ENTITIES) {
|
|
559
|
+
const next = [];
|
|
560
|
+
for (const entity of frontier) {
|
|
561
|
+
if (visitedEntities.size > MAX_VISITED_ENTITIES)
|
|
562
|
+
break;
|
|
563
|
+
let edgeRows = [];
|
|
564
|
+
try {
|
|
565
|
+
// EDGE_ORDER (1.3.0): was LIMIT-without-ORDER-BY, i.e. an
|
|
566
|
+
// arbitrary fanout subset; now the strongest/most-recent
|
|
567
|
+
// edges win the per-entity budget so weak stale links no
|
|
568
|
+
// longer crowd out good ones.
|
|
569
|
+
edgeRows = this.db.prepare("SELECT src, dst, relation, weight, last_seen FROM edges WHERE src = ? OR dst = ? ORDER BY weight DESC, last_seen DESC LIMIT ?").all(entity, entity, MAX_EDGES_PER_ENTITY);
|
|
570
|
+
}
|
|
571
|
+
catch {
|
|
572
|
+
edgeRows = [];
|
|
573
|
+
}
|
|
574
|
+
const chainToEntity = chainByEntity.get(entity) ?? [entity];
|
|
575
|
+
for (const edge of edgeRows) {
|
|
576
|
+
const neighbor = edge.src === entity ? edge.dst : edge.src;
|
|
577
|
+
if (!neighbor)
|
|
578
|
+
continue;
|
|
579
|
+
const typed = edge.relation !== "co_occurs";
|
|
580
|
+
const edgeStrength = Math.min(1, (typeof edge.weight === "number" && edge.weight > 0 ? edge.weight : 1) / 3);
|
|
581
|
+
const relationStrength = typed ? TYPED_RELATION_STRENGTH : 1.0;
|
|
582
|
+
// EDGE_DECAY (1.3.0): stored weights are bounded by
|
|
583
|
+
// maxEdgeProvenance but never age — add a ranking-only
|
|
584
|
+
// recency factor so long-dormant pairs (>=1yr) fade to a
|
|
585
|
+
// 0.35 floor instead of holding their old strength forever.
|
|
586
|
+
const edgeAgeMs = now - (typeof edge.last_seen === "number" ? edge.last_seen : now);
|
|
587
|
+
const edgeAgeDays = Math.max(0, edgeAgeMs) / 86400000;
|
|
588
|
+
const recencyFactor = Math.max(0.35, 1 - edgeAgeDays / 365);
|
|
589
|
+
const scoreFactor = (1 + expansionLambda) * Math.pow(hopDecay, hop - 1) * relationStrength * edgeStrength * recencyFactor;
|
|
590
|
+
// Every edge is scored (a typed edge reaching the SAME
|
|
591
|
+
// neighbor as an earlier co_occurs edge upgrades the
|
|
592
|
+
// candidate); the discovered set only prevents duplicate
|
|
593
|
+
// next-hop frontier entries.
|
|
594
|
+
let memoryIds = memCache.get(neighbor);
|
|
595
|
+
if (memoryIds === undefined) {
|
|
596
|
+
try {
|
|
597
|
+
memoryIds = this.db.prepare("SELECT memory_id AS id FROM memory_entities WHERE entity_name = ?").all(neighbor).map((r) => r.id);
|
|
598
|
+
}
|
|
599
|
+
catch {
|
|
600
|
+
memoryIds = [];
|
|
601
|
+
}
|
|
602
|
+
memCache.set(neighbor, memoryIds);
|
|
603
|
+
}
|
|
604
|
+
const chainToNeighbor = [...chainToEntity, neighbor];
|
|
605
|
+
for (const memoryId of memoryIds) {
|
|
606
|
+
const existing = best.get(memoryId);
|
|
607
|
+
if (!existing || hop < existing.hops || (hop === existing.hops && scoreFactor > existing.scoreFactor)) {
|
|
608
|
+
best.set(memoryId, {
|
|
609
|
+
memoryId,
|
|
610
|
+
hops: hop,
|
|
611
|
+
path: chainToNeighbor,
|
|
612
|
+
relation: edge.relation,
|
|
613
|
+
typed,
|
|
614
|
+
scoreFactor,
|
|
615
|
+
});
|
|
616
|
+
}
|
|
617
|
+
}
|
|
618
|
+
if (!visitedEntities.has(neighbor)) {
|
|
619
|
+
visitedEntities.add(neighbor);
|
|
620
|
+
if (next.length < MAX_VISITED_ENTITIES) {
|
|
621
|
+
chainByEntity.set(neighbor, chainToNeighbor);
|
|
622
|
+
next.push(neighbor);
|
|
623
|
+
}
|
|
624
|
+
}
|
|
625
|
+
}
|
|
626
|
+
}
|
|
627
|
+
frontier = next;
|
|
628
|
+
hop += 1;
|
|
629
|
+
}
|
|
630
|
+
return Array.from(best.values())
|
|
631
|
+
.sort((a, b) => b.scoreFactor - a.scoreFactor || a.hops - b.hops)
|
|
632
|
+
.slice(0, expansionLimit);
|
|
633
|
+
}
|
|
634
|
+
reindexMemories(records) {
|
|
635
|
+
if (!this.enabled || !records || records.length === 0)
|
|
636
|
+
return;
|
|
637
|
+
const countRow = this.db.prepare("SELECT COUNT(*) AS c FROM memory_entities").get();
|
|
638
|
+
if (countRow.c > 0)
|
|
639
|
+
return;
|
|
640
|
+
log("info", `[graph] backfilling ${records.length} existing memories into entity graph`);
|
|
641
|
+
for (const record of records) {
|
|
642
|
+
if (!record?.id || !record?.text)
|
|
643
|
+
continue;
|
|
644
|
+
this.indexMemory(record.id, record.text, record.timestamp ?? Date.now());
|
|
645
|
+
}
|
|
646
|
+
}
|
|
647
|
+
stats() {
|
|
648
|
+
if (!this.enabled)
|
|
649
|
+
return { enabled: false, entities: 0, memoryMappings: 0, edges: 0, relations: {} };
|
|
650
|
+
const count = (sql) => {
|
|
651
|
+
const row = this.db.prepare(`SELECT COUNT(*) AS c FROM ${sql}`).get();
|
|
652
|
+
return row?.c ?? 0;
|
|
653
|
+
};
|
|
654
|
+
const relations = {};
|
|
655
|
+
for (const row of this.db.prepare("SELECT relation AS r, COUNT(*) AS c FROM edges GROUP BY relation ORDER BY c DESC").all()) {
|
|
656
|
+
relations[row.r] = row.c;
|
|
657
|
+
}
|
|
658
|
+
return {
|
|
659
|
+
enabled: true,
|
|
660
|
+
entities: count("entities"),
|
|
661
|
+
memoryMappings: count("memory_entities"),
|
|
662
|
+
edges: count("edges"),
|
|
663
|
+
relations,
|
|
664
|
+
};
|
|
665
|
+
}
|
|
666
|
+
begin() {
|
|
667
|
+
this.db.exec("BEGIN");
|
|
668
|
+
}
|
|
669
|
+
commit() {
|
|
670
|
+
this.db.exec("COMMIT");
|
|
671
|
+
}
|
|
672
|
+
rollback() {
|
|
673
|
+
this.db.exec("ROLLBACK");
|
|
674
|
+
}
|
|
675
|
+
}
|
|
676
|
+
export class DisabledGraphStore {
|
|
677
|
+
enabled = false;
|
|
678
|
+
extract() {
|
|
679
|
+
return [];
|
|
680
|
+
}
|
|
681
|
+
indexMemory() { }
|
|
682
|
+
backfillTypedEdges() { }
|
|
683
|
+
onMemoryRemoved() { }
|
|
684
|
+
onMemoryMerged() { }
|
|
685
|
+
getMemoryEntities() {
|
|
686
|
+
return new Map();
|
|
687
|
+
}
|
|
688
|
+
getEntitiesForQuery() {
|
|
689
|
+
return [];
|
|
690
|
+
}
|
|
691
|
+
boostResults(_query, results) {
|
|
692
|
+
return results;
|
|
693
|
+
}
|
|
694
|
+
expandRecall() {
|
|
695
|
+
return [];
|
|
696
|
+
}
|
|
697
|
+
reindexMemories() { }
|
|
698
|
+
stats() {
|
|
699
|
+
return { enabled: false, entities: 0, memoryMappings: 0, edges: 0, relations: {} };
|
|
700
|
+
}
|
|
701
|
+
}
|