@mem-weave/server 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +74 -0
- package/dist/cli-entry.js +49 -0
- package/dist/cli.js +53 -0
- package/dist/commands/backup.js +28 -0
- package/dist/commands/doctor.js +108 -0
- package/dist/commands/help.js +29 -0
- package/dist/commands/index.js +27 -0
- package/dist/commands/init.js +58 -0
- package/dist/commands/migrate.js +25 -0
- package/dist/commands/start.js +29 -0
- package/dist/commands/status.js +19 -0
- package/dist/commands/stop.js +46 -0
- package/dist/commands/version.js +21 -0
- package/dist/core/config.js +161 -0
- package/dist/core/decay.js +50 -0
- package/dist/core/types.js +72 -0
- package/dist/db/database.js +58 -0
- package/dist/db/repositories/access-log-repo.js +59 -0
- package/dist/db/repositories/consolidation-run-repo.js +86 -0
- package/dist/db/repositories/device-repo.js +66 -0
- package/dist/db/repositories/edge-repo.js +104 -0
- package/dist/db/repositories/memory-repo.js +294 -0
- package/dist/db/repositories/observation-repo.js +65 -0
- package/dist/db/repositories/session-repo.js +81 -0
- package/dist/db/repositories/stats-repo.js +92 -0
- package/dist/db/repositories/vector-repo.js +55 -0
- package/dist/db/schema.js +185 -0
- package/dist/injection/bundler.js +39 -0
- package/dist/injection/formatter.js +23 -0
- package/dist/prompts/compression.js +43 -0
- package/dist/prompts/edge-extract.js +21 -0
- package/dist/prompts/value-gate.js +27 -0
- package/dist/providers/embedding/index.js +36 -0
- package/dist/providers/embedding/local-xenova.js +166 -0
- package/dist/providers/embedding/noop.js +40 -0
- package/dist/providers/embedding/openai-compatible.js +46 -0
- package/dist/providers/llm/index.js +12 -0
- package/dist/providers/llm/noop.js +5 -0
- package/dist/providers/llm/openai.js +45 -0
- package/dist/rest/routes/consolidation.js +62 -0
- package/dist/rest/routes/devices.js +47 -0
- package/dist/rest/routes/injection.js +76 -0
- package/dist/rest/routes/memories.js +349 -0
- package/dist/rest/routes/observations.js +29 -0
- package/dist/rest/routes/sessions.js +37 -0
- package/dist/rest/routes/settings.js +25 -0
- package/dist/rest/routes/stats.js +15 -0
- package/dist/retrieval/bm25-search.js +91 -0
- package/dist/retrieval/causal-chain.js +197 -0
- package/dist/retrieval/fusion.js +48 -0
- package/dist/retrieval/graph-traversal.js +144 -0
- package/dist/retrieval/search-engine.js +150 -0
- package/dist/retrieval/vector-search.js +91 -0
- package/dist/server/auth.js +80 -0
- package/dist/server/bootstrap.js +28 -0
- package/dist/server/http.js +77 -0
- package/dist/server/logger.js +36 -0
- package/dist/server/rate-limiter.js +81 -0
- package/dist/server/scheduler.js +99 -0
- package/dist/workers/association.js +41 -0
- package/dist/workers/compressor.js +14 -0
- package/dist/workers/consolidator.js +201 -0
- package/dist/workers/embedder.js +102 -0
- package/dist/workers/graph-worker.js +166 -0
- package/dist/workers/value-gate.js +38 -0
- package/package.json +40 -0
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { openDatabase } from '../../db/database.js';
|
|
2
|
+
import { StatsRepo } from '../../db/repositories/stats-repo.js';
|
|
3
|
+
const TENANT_DEFAULT = 'tenant_default';
|
|
4
|
+
export function registerStatsRoute(app, dbPath) {
|
|
5
|
+
app.get('/api/v1/stats', async (_request, reply) => {
|
|
6
|
+
const db = openDatabase(dbPath);
|
|
7
|
+
try {
|
|
8
|
+
const stats = new StatsRepo(db).getStats(TENANT_DEFAULT);
|
|
9
|
+
return reply.code(200).send(stats);
|
|
10
|
+
}
|
|
11
|
+
finally {
|
|
12
|
+
db.close();
|
|
13
|
+
}
|
|
14
|
+
});
|
|
15
|
+
}
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
import { logger } from '../server/logger.js';
|
|
2
|
+
export function bm25Search(db, tenantId, query, limit, scopes) {
|
|
3
|
+
try {
|
|
4
|
+
if (!query.trim() || limit <= 0)
|
|
5
|
+
return [];
|
|
6
|
+
// FTS5 escaping: strip special chars, split into tokens, wrap each in double-quotes for literal matching
|
|
7
|
+
const tokens = query.replace(/[^\w\s\u4e00-\u9fff]/g, ' ')
|
|
8
|
+
.split(/\s+/)
|
|
9
|
+
.filter(w => w.length > 0)
|
|
10
|
+
.map(w => `"${w}"`);
|
|
11
|
+
if (tokens.length === 0)
|
|
12
|
+
return [];
|
|
13
|
+
const safe = tokens.join(' ');
|
|
14
|
+
// Build query dynamically — scope filter uses EXISTS to avoid row duplication
|
|
15
|
+
let sql = `
|
|
16
|
+
SELECT m.*, bm25(memory_fts) AS bm25_score
|
|
17
|
+
FROM memory_fts
|
|
18
|
+
JOIN memories m ON m.rowid = memory_fts.rowid
|
|
19
|
+
WHERE memory_fts MATCH ?
|
|
20
|
+
AND m.tenant_id = ?
|
|
21
|
+
AND m.deleted_at IS NULL
|
|
22
|
+
`;
|
|
23
|
+
const params = [safe, tenantId];
|
|
24
|
+
if (scopes && scopes.length > 0) {
|
|
25
|
+
for (const scope of scopes) {
|
|
26
|
+
sql += ` AND EXISTS (SELECT 1 FROM memory_scopes ms WHERE ms.memory_id = m.id AND ms.key = ? AND ms.value = ?)`;
|
|
27
|
+
params.push(scope.key, scope.value);
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
sql += ` ORDER BY bm25_score DESC LIMIT ?`;
|
|
31
|
+
params.push(limit);
|
|
32
|
+
const rows = db.prepare(sql).all(...params);
|
|
33
|
+
// Batch query scopes for all returned memory IDs (populate mapRow correctly)
|
|
34
|
+
const memoryIds = rows.map(r => r.id);
|
|
35
|
+
const scopeMap = new Map();
|
|
36
|
+
if (memoryIds.length > 0) {
|
|
37
|
+
const placeholders = memoryIds.map(() => '?').join(',');
|
|
38
|
+
const scopeRows = db.prepare(`
|
|
39
|
+
SELECT memory_id, key, value FROM memory_scopes
|
|
40
|
+
WHERE tenant_id = ? AND memory_id IN (${placeholders})
|
|
41
|
+
ORDER BY key, value
|
|
42
|
+
`).all(tenantId, ...memoryIds);
|
|
43
|
+
for (const sr of scopeRows) {
|
|
44
|
+
if (!scopeMap.has(sr.memory_id))
|
|
45
|
+
scopeMap.set(sr.memory_id, []);
|
|
46
|
+
scopeMap.get(sr.memory_id).push({ key: sr.key, value: sr.value });
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
return rows.map(row => ({
|
|
50
|
+
memory: mapRow(row, scopeMap.get(row.id) ?? []),
|
|
51
|
+
bm25Score: row.bm25_score
|
|
52
|
+
}));
|
|
53
|
+
}
|
|
54
|
+
catch (err) {
|
|
55
|
+
logger.error({ err }, 'bm25 search failed');
|
|
56
|
+
return [];
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
function mapRow(row, scopes) {
|
|
60
|
+
return {
|
|
61
|
+
id: row.id,
|
|
62
|
+
tenantId: row.tenant_id,
|
|
63
|
+
tier: row.tier,
|
|
64
|
+
type: row.type,
|
|
65
|
+
title: row.title,
|
|
66
|
+
content: row.content,
|
|
67
|
+
summary: row.summary,
|
|
68
|
+
concepts: JSON.parse(row.concepts_json),
|
|
69
|
+
files: JSON.parse(row.files_json),
|
|
70
|
+
importance: row.importance,
|
|
71
|
+
confidence: row.confidence,
|
|
72
|
+
strength: row.strength,
|
|
73
|
+
source: row.source,
|
|
74
|
+
scopeLevel: row.scope_level,
|
|
75
|
+
scopes,
|
|
76
|
+
sourceClient: row.source_client,
|
|
77
|
+
sourceDeviceId: row.source_device_id,
|
|
78
|
+
sourceSessionId: row.source_session_id,
|
|
79
|
+
tau: row.tau,
|
|
80
|
+
accessCount: row.access_count,
|
|
81
|
+
lastAccessedAt: row.last_accessed_at,
|
|
82
|
+
lastReinforcedAt: row.last_reinforced_at,
|
|
83
|
+
lastDecayAt: row.last_decay_at,
|
|
84
|
+
reinforcementScore: row.reinforcement_score,
|
|
85
|
+
promotedAt: row.promoted_at,
|
|
86
|
+
createdAt: row.created_at,
|
|
87
|
+
updatedAt: row.updated_at,
|
|
88
|
+
deletedAt: row.deleted_at,
|
|
89
|
+
evictionReason: row.eviction_reason
|
|
90
|
+
};
|
|
91
|
+
}
|
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
const CAUSAL_EDGE_TYPES = ['causes', 'before', 'after', 'refines', 'supersedes'];
|
|
2
|
+
const STRONG_CAUSAL = ['causes', 'refines'];
|
|
3
|
+
/**
|
|
4
|
+
* Causal-chain detection (design spec §4.6).
|
|
5
|
+
*
|
|
6
|
+
* Starting from a set of seed memories, walk along causal edges in both
|
|
7
|
+
* directions to extract linear chains. Each chain is scored by:
|
|
8
|
+
*
|
|
9
|
+
* chainScore = average(memory.strength)
|
|
10
|
+
* * average(edge.strength)
|
|
11
|
+
* * completeness
|
|
12
|
+
*
|
|
13
|
+
* where `completeness` is the fraction of edges in the chain that are
|
|
14
|
+
* strong causal relations ('causes' or 'refines') — chains dominated by
|
|
15
|
+
* 'before'/'after' temporal links score lower.
|
|
16
|
+
*
|
|
17
|
+
* The implementation is bounded: each chain explores at most `maxLength` edges
|
|
18
|
+
* and we return at most `maxChains` chains total (ranked by chainScore).
|
|
19
|
+
*/
|
|
20
|
+
export function detectCausalChains(db, options) {
|
|
21
|
+
const maxLength = Math.max(1, Math.min(10, options.maxLength ?? 5));
|
|
22
|
+
const maxChains = Math.max(1, options.maxChains ?? 10);
|
|
23
|
+
const edgeTypes = options.edgeTypes ?? CAUSAL_EDGE_TYPES;
|
|
24
|
+
const bidirectional = options.bidirectional ?? true;
|
|
25
|
+
const allChains = [];
|
|
26
|
+
for (const seedId of options.seedMemoryIds) {
|
|
27
|
+
if (allChains.length >= maxChains)
|
|
28
|
+
break;
|
|
29
|
+
// Forward chain: from seed outward via outgoing edges
|
|
30
|
+
const forward = walkChain(db, options.tenantId, seedId, 'out', edgeTypes, maxLength);
|
|
31
|
+
for (const c of forward) {
|
|
32
|
+
allChains.push(c);
|
|
33
|
+
if (allChains.length >= maxChains)
|
|
34
|
+
break;
|
|
35
|
+
}
|
|
36
|
+
// Backward chain: from seed via incoming edges
|
|
37
|
+
if (bidirectional) {
|
|
38
|
+
const backward = walkChain(db, options.tenantId, seedId, 'in', edgeTypes, maxLength);
|
|
39
|
+
for (const c of backward) {
|
|
40
|
+
allChains.push(c);
|
|
41
|
+
if (allChains.length >= maxChains)
|
|
42
|
+
break;
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
// Rank by chain score, descending
|
|
47
|
+
allChains.sort((a, b) => b.chainScore - a.chainScore);
|
|
48
|
+
return allChains.slice(0, maxChains);
|
|
49
|
+
}
|
|
50
|
+
function walkChain(db, tenantId, seedId, direction, edgeTypes, maxLength) {
|
|
51
|
+
// Greedy chain extension: at each step, pick the highest-strength edge
|
|
52
|
+
// to extend the chain. Returns one chain per "branch" up to maxLength.
|
|
53
|
+
const chains = [[]];
|
|
54
|
+
// The seed starts every chain
|
|
55
|
+
for (const c of chains)
|
|
56
|
+
c.push({ memoryId: seedId, edgeId: null });
|
|
57
|
+
for (let step = 1; step <= maxLength; step++) {
|
|
58
|
+
const newChains = [];
|
|
59
|
+
for (const chain of chains) {
|
|
60
|
+
const last = chain[chain.length - 1];
|
|
61
|
+
const nextEdges = pullNextCausalEdges(db, tenantId, last.memoryId, direction, edgeTypes);
|
|
62
|
+
if (nextEdges.length === 0) {
|
|
63
|
+
// Dead end — keep this chain as-is
|
|
64
|
+
newChains.push([...chain]);
|
|
65
|
+
continue;
|
|
66
|
+
}
|
|
67
|
+
for (const e of nextEdges) {
|
|
68
|
+
if (chain.some((s) => s.memoryId === e.toMemoryId))
|
|
69
|
+
continue; // avoid cycles
|
|
70
|
+
newChains.push([...chain, { memoryId: e.toMemoryId, edgeId: e.id }]);
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
chains.length = 0;
|
|
74
|
+
chains.push(...newChains);
|
|
75
|
+
}
|
|
76
|
+
// Hydrate and score each chain
|
|
77
|
+
const allIds = new Set();
|
|
78
|
+
for (const c of chains)
|
|
79
|
+
for (const s of c)
|
|
80
|
+
allIds.add(s.memoryId);
|
|
81
|
+
if (allIds.size === 0)
|
|
82
|
+
return [];
|
|
83
|
+
const placeholders = [...allIds].map(() => '?').join(',');
|
|
84
|
+
const rows = db.prepare(`
|
|
85
|
+
SELECT * FROM memories
|
|
86
|
+
WHERE tenant_id = ? AND id IN (${placeholders}) AND deleted_at IS NULL
|
|
87
|
+
`).all(tenantId, ...allIds);
|
|
88
|
+
const byId = new Map();
|
|
89
|
+
for (const row of rows)
|
|
90
|
+
byId.set(row.id, rowToMemory(row));
|
|
91
|
+
const edgeIdSet = new Set();
|
|
92
|
+
for (const c of chains)
|
|
93
|
+
for (const s of c)
|
|
94
|
+
if (s.edgeId)
|
|
95
|
+
edgeIdSet.add(s.edgeId);
|
|
96
|
+
const edgeStrengthById = new Map();
|
|
97
|
+
if (edgeIdSet.size > 0) {
|
|
98
|
+
const edgePlaceholders = [...edgeIdSet].map(() => '?').join(',');
|
|
99
|
+
const edgeRows = db.prepare(`
|
|
100
|
+
SELECT id, strength, type FROM edges
|
|
101
|
+
WHERE id IN (${edgePlaceholders})
|
|
102
|
+
`).all(...edgeIdSet);
|
|
103
|
+
for (const r of edgeRows)
|
|
104
|
+
edgeStrengthById.set(r.id, { strength: r.strength, type: r.type });
|
|
105
|
+
}
|
|
106
|
+
const candidates = [];
|
|
107
|
+
for (const chain of chains) {
|
|
108
|
+
if (chain.length < 2)
|
|
109
|
+
continue; // need at least one edge
|
|
110
|
+
const memories = [];
|
|
111
|
+
for (const s of chain) {
|
|
112
|
+
const m = byId.get(s.memoryId);
|
|
113
|
+
if (m)
|
|
114
|
+
memories.push(m);
|
|
115
|
+
}
|
|
116
|
+
if (memories.length !== chain.length)
|
|
117
|
+
continue; // hydration failed for some node
|
|
118
|
+
let edgeStrengthSum = 0;
|
|
119
|
+
let edgeCount = 0;
|
|
120
|
+
let strongCausalCount = 0;
|
|
121
|
+
const edgeIds = [];
|
|
122
|
+
for (const s of chain) {
|
|
123
|
+
if (s.edgeId) {
|
|
124
|
+
const e = edgeStrengthById.get(s.edgeId);
|
|
125
|
+
if (e) {
|
|
126
|
+
edgeStrengthSum += e.strength;
|
|
127
|
+
edgeCount++;
|
|
128
|
+
if (STRONG_CAUSAL.includes(e.type))
|
|
129
|
+
strongCausalCount++;
|
|
130
|
+
edgeIds.push(s.edgeId);
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
const avgEdgeStrength = edgeCount > 0 ? edgeStrengthSum / edgeCount : 1;
|
|
135
|
+
const avgMemStrength = memories.reduce((s, m) => s + m.strength, 0) / memories.length;
|
|
136
|
+
const completeness = edgeCount > 0 ? strongCausalCount / edgeCount : 0;
|
|
137
|
+
const chainScore = avgMemStrength * avgEdgeStrength * completeness;
|
|
138
|
+
candidates.push({
|
|
139
|
+
memoryIds: chain.map((s) => s.memoryId),
|
|
140
|
+
edgeIds,
|
|
141
|
+
memories,
|
|
142
|
+
chainScore,
|
|
143
|
+
completeness
|
|
144
|
+
});
|
|
145
|
+
}
|
|
146
|
+
return candidates;
|
|
147
|
+
}
|
|
148
|
+
function pullNextCausalEdges(db, tenantId, fromMemoryId, direction, edgeTypes) {
|
|
149
|
+
const placeholders = edgeTypes.map(() => '?').join(',');
|
|
150
|
+
const sql = direction === 'out'
|
|
151
|
+
? `SELECT id, to_memory_id, strength, type FROM edges
|
|
152
|
+
WHERE tenant_id = ? AND from_memory_id = ? AND type IN (${placeholders})
|
|
153
|
+
ORDER BY strength DESC LIMIT 5`
|
|
154
|
+
: `SELECT id, from_memory_id AS to_memory_id, strength, type FROM edges
|
|
155
|
+
WHERE tenant_id = ? AND to_memory_id = ? AND type IN (${placeholders})
|
|
156
|
+
ORDER BY strength DESC LIMIT 5`;
|
|
157
|
+
const rows = db.prepare(sql).all(tenantId, fromMemoryId, ...edgeTypes);
|
|
158
|
+
return rows.map((r) => ({
|
|
159
|
+
id: r.id,
|
|
160
|
+
toMemoryId: r.to_memory_id,
|
|
161
|
+
strength: r.strength,
|
|
162
|
+
type: r.type
|
|
163
|
+
}));
|
|
164
|
+
}
|
|
165
|
+
function rowToMemory(row) {
|
|
166
|
+
return {
|
|
167
|
+
id: row.id,
|
|
168
|
+
tenantId: row.tenant_id,
|
|
169
|
+
tier: row.tier,
|
|
170
|
+
type: row.type,
|
|
171
|
+
title: row.title,
|
|
172
|
+
content: row.content,
|
|
173
|
+
summary: row.summary,
|
|
174
|
+
concepts: JSON.parse(row.concepts_json),
|
|
175
|
+
files: JSON.parse(row.files_json),
|
|
176
|
+
importance: row.importance,
|
|
177
|
+
confidence: row.confidence,
|
|
178
|
+
strength: row.strength,
|
|
179
|
+
source: row.source,
|
|
180
|
+
scopeLevel: row.scope_level,
|
|
181
|
+
scopes: [],
|
|
182
|
+
sourceClient: row.source_client,
|
|
183
|
+
sourceDeviceId: row.source_device_id,
|
|
184
|
+
sourceSessionId: row.source_session_id,
|
|
185
|
+
tau: row.tau,
|
|
186
|
+
accessCount: row.access_count,
|
|
187
|
+
lastAccessedAt: row.last_accessed_at,
|
|
188
|
+
lastReinforcedAt: row.last_reinforced_at,
|
|
189
|
+
lastDecayAt: row.last_decay_at,
|
|
190
|
+
reinforcementScore: row.reinforcement_score,
|
|
191
|
+
promotedAt: row.promoted_at,
|
|
192
|
+
createdAt: row.created_at,
|
|
193
|
+
updatedAt: row.updated_at,
|
|
194
|
+
deletedAt: row.deleted_at,
|
|
195
|
+
evictionReason: row.eviction_reason
|
|
196
|
+
};
|
|
197
|
+
}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
const TIER_WEIGHTS = {
|
|
2
|
+
long: 1.15,
|
|
3
|
+
medium: 1.0,
|
|
4
|
+
short: 0.85
|
|
5
|
+
};
|
|
6
|
+
const DEFAULT_RRF_K = 60;
|
|
7
|
+
export function fuseResults(streams, rrfK = DEFAULT_RRF_K) {
|
|
8
|
+
const byMemoryId = new Map();
|
|
9
|
+
for (const stream of streams) {
|
|
10
|
+
for (const ranked of stream) {
|
|
11
|
+
if (ranked.rank < 0)
|
|
12
|
+
continue;
|
|
13
|
+
const id = ranked.candidate.memory.id;
|
|
14
|
+
const rrfContribution = 1 / (rrfK + ranked.rank);
|
|
15
|
+
const existing = byMemoryId.get(id);
|
|
16
|
+
if (existing) {
|
|
17
|
+
existing.rrfScore += rrfContribution;
|
|
18
|
+
existing.candidate.sources.add(ranked.source);
|
|
19
|
+
}
|
|
20
|
+
else {
|
|
21
|
+
byMemoryId.set(id, {
|
|
22
|
+
...ranked,
|
|
23
|
+
rrfScore: rrfContribution,
|
|
24
|
+
candidate: {
|
|
25
|
+
...ranked.candidate,
|
|
26
|
+
sources: new Set(ranked.candidate.sources)
|
|
27
|
+
}
|
|
28
|
+
});
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
const results = [];
|
|
33
|
+
for (const entry of byMemoryId.values()) {
|
|
34
|
+
const tierWeight = TIER_WEIGHTS[entry.candidate.memory.tier] ?? 1.0;
|
|
35
|
+
const safeStrength = Number.isFinite(entry.candidate.memory.strength) ? Math.max(0, Math.min(1, entry.candidate.memory.strength)) : 0;
|
|
36
|
+
const strengthWeight = 0.5 + safeStrength;
|
|
37
|
+
const finalScore = entry.rrfScore * tierWeight * strengthWeight;
|
|
38
|
+
results.push({
|
|
39
|
+
candidate: entry.candidate,
|
|
40
|
+
finalScore,
|
|
41
|
+
rrfScore: entry.rrfScore,
|
|
42
|
+
tierWeight,
|
|
43
|
+
strengthWeight
|
|
44
|
+
});
|
|
45
|
+
}
|
|
46
|
+
results.sort((a, b) => b.finalScore - a.finalScore);
|
|
47
|
+
return results;
|
|
48
|
+
}
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
const DEFAULT_EDGE_TYPES = [
|
|
2
|
+
'causes', 'enables', 'supersedes', 'references', 'refines', 'contradicts'
|
|
3
|
+
];
|
|
4
|
+
/**
|
|
5
|
+
* Graph expansion: starting from `startMemoryId`, BFS outward through edges
|
|
6
|
+
* of the specified types and return the discovered memory nodes ranked by
|
|
7
|
+
* their `distance` from the start. Implements design spec §4.4.
|
|
8
|
+
*
|
|
9
|
+
* Notes:
|
|
10
|
+
* - The starting memory itself is NOT returned (only its neighbors).
|
|
11
|
+
* - `related_to` and `duplicates` are NOT followed by default (too noisy).
|
|
12
|
+
* - `before`/`after` are deferred to the causal-chain layer.
|
|
13
|
+
*/
|
|
14
|
+
export function graphExpand(db, options) {
|
|
15
|
+
const depth = Math.max(1, Math.min(3, options.depth ?? 1));
|
|
16
|
+
const maxNodes = options.maxNodes ?? 30;
|
|
17
|
+
const edgeTypes = options.edgeTypes ?? DEFAULT_EDGE_TYPES;
|
|
18
|
+
const direction = options.direction ?? 'both';
|
|
19
|
+
const visited = new Set([options.startMemoryId]);
|
|
20
|
+
const allDiscovered = [];
|
|
21
|
+
let frontier = [{
|
|
22
|
+
memoryId: options.startMemoryId,
|
|
23
|
+
distance: 0,
|
|
24
|
+
edgePath: [],
|
|
25
|
+
memoryPath: [options.startMemoryId],
|
|
26
|
+
pathStrength: 1
|
|
27
|
+
}];
|
|
28
|
+
for (let d = 1; d <= depth; d++) {
|
|
29
|
+
const nextFrontier = [];
|
|
30
|
+
for (const node of frontier) {
|
|
31
|
+
// Pull outgoing edges
|
|
32
|
+
if (direction === 'out' || direction === 'both') {
|
|
33
|
+
const outRows = db.prepare(`
|
|
34
|
+
SELECT id, to_memory_id, strength
|
|
35
|
+
FROM edges
|
|
36
|
+
WHERE tenant_id = ? AND from_memory_id = ? AND type IN (${edgeTypes.map(() => '?').join(',')})
|
|
37
|
+
`).all(options.tenantId, node.memoryId, ...edgeTypes);
|
|
38
|
+
for (const r of outRows) {
|
|
39
|
+
if (visited.has(r.to_memory_id))
|
|
40
|
+
continue;
|
|
41
|
+
visited.add(r.to_memory_id);
|
|
42
|
+
nextFrontier.push({
|
|
43
|
+
memoryId: r.to_memory_id,
|
|
44
|
+
distance: d,
|
|
45
|
+
edgePath: [...node.edgePath, r.id],
|
|
46
|
+
memoryPath: [...node.memoryPath, r.to_memory_id],
|
|
47
|
+
pathStrength: Math.min(node.pathStrength, r.strength)
|
|
48
|
+
});
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
// Pull incoming edges
|
|
52
|
+
if (direction === 'in' || direction === 'both') {
|
|
53
|
+
const inRows = db.prepare(`
|
|
54
|
+
SELECT id, from_memory_id, strength
|
|
55
|
+
FROM edges
|
|
56
|
+
WHERE tenant_id = ? AND to_memory_id = ? AND type IN (${edgeTypes.map(() => '?').join(',')})
|
|
57
|
+
`).all(options.tenantId, node.memoryId, ...edgeTypes);
|
|
58
|
+
for (const r of inRows) {
|
|
59
|
+
if (visited.has(r.from_memory_id))
|
|
60
|
+
continue;
|
|
61
|
+
visited.add(r.from_memory_id);
|
|
62
|
+
nextFrontier.push({
|
|
63
|
+
memoryId: r.from_memory_id,
|
|
64
|
+
distance: d,
|
|
65
|
+
edgePath: [...node.edgePath, r.id],
|
|
66
|
+
memoryPath: [...node.memoryPath, r.from_memory_id],
|
|
67
|
+
pathStrength: Math.min(node.pathStrength, r.strength)
|
|
68
|
+
});
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
frontier = nextFrontier;
|
|
73
|
+
// Accumulate every discovered node, not just the final frontier.
|
|
74
|
+
for (const node of nextFrontier)
|
|
75
|
+
allDiscovered.push(node);
|
|
76
|
+
if (allDiscovered.length >= maxNodes)
|
|
77
|
+
break;
|
|
78
|
+
}
|
|
79
|
+
// Collect all discovered nodes
|
|
80
|
+
const visitedIds = [];
|
|
81
|
+
for (const node of allDiscovered)
|
|
82
|
+
visitedIds.push(node.memoryId);
|
|
83
|
+
if (visitedIds.length > maxNodes)
|
|
84
|
+
visitedIds.length = maxNodes;
|
|
85
|
+
// Hydrate memory records
|
|
86
|
+
if (visitedIds.length === 0)
|
|
87
|
+
return [];
|
|
88
|
+
const placeholders = visitedIds.map(() => '?').join(',');
|
|
89
|
+
const rows = db.prepare(`
|
|
90
|
+
SELECT * FROM memories
|
|
91
|
+
WHERE tenant_id = ? AND id IN (${placeholders}) AND deleted_at IS NULL
|
|
92
|
+
`).all(options.tenantId, ...visitedIds);
|
|
93
|
+
const byId = new Map();
|
|
94
|
+
for (const row of rows)
|
|
95
|
+
byId.set(row.id, rowToMemory(row));
|
|
96
|
+
return allDiscovered
|
|
97
|
+
.slice(0, maxNodes)
|
|
98
|
+
.map((node) => {
|
|
99
|
+
const memory = byId.get(node.memoryId);
|
|
100
|
+
return memory
|
|
101
|
+
? {
|
|
102
|
+
memory,
|
|
103
|
+
distance: node.distance,
|
|
104
|
+
edgePath: node.edgePath,
|
|
105
|
+
memoryPath: node.memoryPath,
|
|
106
|
+
pathStrength: node.pathStrength
|
|
107
|
+
}
|
|
108
|
+
: null;
|
|
109
|
+
})
|
|
110
|
+
.filter((c) => c !== null);
|
|
111
|
+
}
|
|
112
|
+
function rowToMemory(row) {
|
|
113
|
+
return {
|
|
114
|
+
id: row.id,
|
|
115
|
+
tenantId: row.tenant_id,
|
|
116
|
+
tier: row.tier,
|
|
117
|
+
type: row.type,
|
|
118
|
+
title: row.title,
|
|
119
|
+
content: row.content,
|
|
120
|
+
summary: row.summary,
|
|
121
|
+
concepts: JSON.parse(row.concepts_json),
|
|
122
|
+
files: JSON.parse(row.files_json),
|
|
123
|
+
importance: row.importance,
|
|
124
|
+
confidence: row.confidence,
|
|
125
|
+
strength: row.strength,
|
|
126
|
+
source: row.source,
|
|
127
|
+
scopeLevel: row.scope_level,
|
|
128
|
+
scopes: [],
|
|
129
|
+
sourceClient: row.source_client,
|
|
130
|
+
sourceDeviceId: row.source_device_id,
|
|
131
|
+
sourceSessionId: row.source_session_id,
|
|
132
|
+
tau: row.tau,
|
|
133
|
+
accessCount: row.access_count,
|
|
134
|
+
lastAccessedAt: row.last_accessed_at,
|
|
135
|
+
lastReinforcedAt: row.last_reinforced_at,
|
|
136
|
+
lastDecayAt: row.last_decay_at,
|
|
137
|
+
reinforcementScore: row.reinforcement_score,
|
|
138
|
+
promotedAt: row.promoted_at,
|
|
139
|
+
createdAt: row.created_at,
|
|
140
|
+
updatedAt: row.updated_at,
|
|
141
|
+
deletedAt: row.deleted_at,
|
|
142
|
+
evictionReason: row.eviction_reason
|
|
143
|
+
};
|
|
144
|
+
}
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
import { bm25Search } from './bm25-search.js';
|
|
2
|
+
import { vectorSearch } from './vector-search.js';
|
|
3
|
+
import { graphExpand } from './graph-traversal.js';
|
|
4
|
+
import { detectCausalChains } from './causal-chain.js';
|
|
5
|
+
import { fuseResults } from './fusion.js';
|
|
6
|
+
/**
|
|
7
|
+
* Multi-layer search (design spec §4.13):
|
|
8
|
+
*
|
|
9
|
+
* 1. BM25 keyword recall (FTS5)
|
|
10
|
+
* 2. Vector recall (sqlite-vec; skipped when `queryEmbedding` is not provided)
|
|
11
|
+
* 3. Graph expansion (BFS from BM25/vector seeds, depth 1)
|
|
12
|
+
* 4. Causal chain detection (from BM25/vector seeds, length ≤ 3)
|
|
13
|
+
* 5. RRF fusion of all 4 streams
|
|
14
|
+
* 6. Tier / strength / scope / freshness weighting
|
|
15
|
+
* 7. Return top-K
|
|
16
|
+
*
|
|
17
|
+
* When `bm25Only` is true, layers 2-4 are skipped.
|
|
18
|
+
*/
|
|
19
|
+
export async function searchMemories(db, tenantId, options) {
|
|
20
|
+
const limit = options.limit ?? 8;
|
|
21
|
+
const query = options.query.trim();
|
|
22
|
+
const bm25Only = options.bm25Only ?? false;
|
|
23
|
+
const rrfK = options.rrfK ?? 60;
|
|
24
|
+
const bm25Limit = options.bm25Limit ?? 50;
|
|
25
|
+
const vectorLimit = options.vectorLimit ?? 50;
|
|
26
|
+
const graphLimit = options.graphLimit ?? 30;
|
|
27
|
+
const causalLimit = options.causalLimit ?? 30;
|
|
28
|
+
const vectorMinSimilarity = options.vectorMinSimilarity ?? 0;
|
|
29
|
+
const vectorDimensions = options.vectorDimensions ?? 768;
|
|
30
|
+
const layerStats = { bm25: 0, vector: 0, graph: 0, causal: 0 };
|
|
31
|
+
if (!query && !options.queryEmbedding) {
|
|
32
|
+
return { query, results: [], totalCandidates: 0, layerStats };
|
|
33
|
+
}
|
|
34
|
+
// --- Layer 1: BM25 ---
|
|
35
|
+
let bm25Rows = query ? bm25Search(db, tenantId, query, bm25Limit) : [];
|
|
36
|
+
layerStats.bm25 = bm25Rows.length;
|
|
37
|
+
if (options.scope || options.types) {
|
|
38
|
+
bm25Rows = bm25Rows.filter((row) => {
|
|
39
|
+
if (options.scope && !matchesScope(row.memory, options.scope))
|
|
40
|
+
return false;
|
|
41
|
+
if (options.types && !options.types.includes(row.memory.type))
|
|
42
|
+
return false;
|
|
43
|
+
return true;
|
|
44
|
+
});
|
|
45
|
+
}
|
|
46
|
+
// --- Layer 2: Vector (optional) ---
|
|
47
|
+
let vectorRows = [];
|
|
48
|
+
if (!bm25Only && options.queryEmbedding && options.queryEmbedding.length === vectorDimensions) {
|
|
49
|
+
vectorRows = vectorSearch(db, tenantId, options.queryEmbedding, vectorLimit, vectorDimensions)
|
|
50
|
+
.filter((r) => r.similarity >= vectorMinSimilarity);
|
|
51
|
+
layerStats.vector = vectorRows.length;
|
|
52
|
+
if (options.scope || options.types) {
|
|
53
|
+
vectorRows = vectorRows.filter((row) => {
|
|
54
|
+
if (options.scope && !matchesScope(row.memory, options.scope))
|
|
55
|
+
return false;
|
|
56
|
+
if (options.types && !options.types.includes(row.memory.type))
|
|
57
|
+
return false;
|
|
58
|
+
return true;
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
// Build streams
|
|
63
|
+
const streams = [];
|
|
64
|
+
if (bm25Rows.length > 0) {
|
|
65
|
+
streams.push(bm25Rows.map((r, idx) => ({
|
|
66
|
+
candidate: { memory: r.memory, sources: new Set(['bm25']) },
|
|
67
|
+
rank: idx,
|
|
68
|
+
source: 'bm25'
|
|
69
|
+
})));
|
|
70
|
+
}
|
|
71
|
+
if (vectorRows.length > 0) {
|
|
72
|
+
streams.push(vectorRows.map((r, idx) => ({
|
|
73
|
+
candidate: { memory: r.memory, sources: new Set(['vector']) },
|
|
74
|
+
rank: idx,
|
|
75
|
+
source: 'vector'
|
|
76
|
+
})));
|
|
77
|
+
}
|
|
78
|
+
// --- Layer 3: Graph expansion ---
|
|
79
|
+
if (!bm25Only) {
|
|
80
|
+
const seeds = [...bm25Rows, ...vectorRows].slice(0, 5).map((r) => r.memory.id);
|
|
81
|
+
const allGraph = [];
|
|
82
|
+
for (const seed of seeds) {
|
|
83
|
+
const out = graphExpand(db, { startMemoryId: seed, tenantId, depth: 1, maxNodes: graphLimit });
|
|
84
|
+
for (const c of out) {
|
|
85
|
+
if (options.scope && !matchesScope(c.memory, options.scope))
|
|
86
|
+
continue;
|
|
87
|
+
if (options.types && !options.types.includes(c.memory.type))
|
|
88
|
+
continue;
|
|
89
|
+
allGraph.push(c);
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
layerStats.graph = allGraph.length;
|
|
93
|
+
if (allGraph.length > 0) {
|
|
94
|
+
streams.push(allGraph.map((c, idx) => ({
|
|
95
|
+
candidate: { memory: c.memory, sources: new Set(['graph']) },
|
|
96
|
+
rank: idx,
|
|
97
|
+
source: 'graph'
|
|
98
|
+
})));
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
// --- Layer 4: Causal chains ---
|
|
102
|
+
if (!bm25Only) {
|
|
103
|
+
const seeds = [...bm25Rows, ...vectorRows].slice(0, 5).map((r) => r.memory.id);
|
|
104
|
+
const allCausal = [];
|
|
105
|
+
for (const seed of seeds) {
|
|
106
|
+
const out = detectCausalChains(db, { seedMemoryIds: [seed], tenantId, maxLength: 3, maxChains: causalLimit });
|
|
107
|
+
for (const chain of out) {
|
|
108
|
+
for (const m of chain.memories) {
|
|
109
|
+
if (m.id === seed)
|
|
110
|
+
continue;
|
|
111
|
+
if (options.scope && !matchesScope(m, options.scope))
|
|
112
|
+
continue;
|
|
113
|
+
if (options.types && !options.types.includes(m.type))
|
|
114
|
+
continue;
|
|
115
|
+
allCausal.push(chain);
|
|
116
|
+
break;
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
layerStats.causal = allCausal.length;
|
|
121
|
+
if (allCausal.length > 0) {
|
|
122
|
+
streams.push(allCausal.map((c, idx) => {
|
|
123
|
+
const m = c.memories[0];
|
|
124
|
+
return {
|
|
125
|
+
candidate: { memory: m, sources: new Set(['causal']) },
|
|
126
|
+
rank: idx,
|
|
127
|
+
source: 'causal'
|
|
128
|
+
};
|
|
129
|
+
}));
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
// --- RRF fusion ---
|
|
133
|
+
const fused = fuseResults(streams, rrfK);
|
|
134
|
+
return {
|
|
135
|
+
query,
|
|
136
|
+
results: fused.slice(0, limit),
|
|
137
|
+
totalCandidates: streams.reduce((s, x) => s + x.length, 0),
|
|
138
|
+
layerStats
|
|
139
|
+
};
|
|
140
|
+
}
|
|
141
|
+
function matchesScope(memory, scope) {
|
|
142
|
+
for (const [key, value] of Object.entries(scope)) {
|
|
143
|
+
if (!value)
|
|
144
|
+
continue;
|
|
145
|
+
const found = memory.scopes.some((s) => s.key === key && s.value === value);
|
|
146
|
+
if (!found)
|
|
147
|
+
return false;
|
|
148
|
+
}
|
|
149
|
+
return true;
|
|
150
|
+
}
|