@ijfw/memory-server 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/ijfw +27 -0
- package/bin/ijfw-dashboard +180 -0
- package/bin/ijfw-dispatch-plan +41 -0
- package/bin/ijfw-memorize +273 -0
- package/bin/ijfw-memory +51 -0
- package/fixtures/demo-target.js +28 -0
- package/package.json +53 -0
- package/src/api-client.js +190 -0
- package/src/audit-roster.js +315 -0
- package/src/caps.js +37 -0
- package/src/cold-scan-runner.mjs +37 -0
- package/src/compute/edges.js +155 -0
- package/src/compute/extract.js +560 -0
- package/src/compute/fts5.js +420 -0
- package/src/compute/graph-auto-index.js +191 -0
- package/src/compute/graph-lock.js +114 -0
- package/src/compute/index.js +18 -0
- package/src/compute/migration-runner.js +116 -0
- package/src/compute/migrations/001-initial.js +23 -0
- package/src/compute/migrations/002-porter-stemming-source.js +139 -0
- package/src/compute/migrations/003-tier-semantic.js +69 -0
- package/src/compute/migrations/004-kg-tables.js +83 -0
- package/src/compute/migrations/005-stale-candidate.js +72 -0
- package/src/compute/python-resolver.js +106 -0
- package/src/compute/runner-vm.js +185 -0
- package/src/compute/runner.js +416 -0
- package/src/compute/sandbox-detect.js +122 -0
- package/src/compute/sandbox-linux.js +164 -0
- package/src/compute/sandbox-macos.js +167 -0
- package/src/compute/sandbox-windows.js +63 -0
- package/src/compute/schema.sql +118 -0
- package/src/compute/staleness.js +239 -0
- package/src/compute/synonyms.js +367 -0
- package/src/compute/traverse.js +180 -0
- package/src/cost/aggregator.js +229 -0
- package/src/cost/pricing.js +134 -0
- package/src/cost/readers/claude.js +179 -0
- package/src/cost/readers/codex.js +131 -0
- package/src/cost/readers/gemini.js +111 -0
- package/src/cost/savings.js +243 -0
- package/src/cross-dispatcher.js +437 -0
- package/src/cross-orchestrator-cli.js +1885 -0
- package/src/cross-orchestrator.js +598 -0
- package/src/cross-project-search.js +114 -0
- package/src/dashboard-client.html +1180 -0
- package/src/dashboard-server.js +895 -0
- package/src/design-companion.js +81 -0
- package/src/dispatch/colon-syntax.js +732 -0
- package/src/dispatch-planner.js +235 -0
- package/src/dream/cooldown.js +105 -0
- package/src/dream/runner.mjs +373 -0
- package/src/dream/staleness-wiring.js +195 -0
- package/src/feedback-detector.js +57 -0
- package/src/hero-line.js +115 -0
- package/src/importers/claude-mem.js +152 -0
- package/src/importers/cli.js +311 -0
- package/src/importers/common.js +84 -0
- package/src/importers/discover.js +235 -0
- package/src/importers/rtk.js +107 -0
- package/src/intent-router.js +221 -0
- package/src/lib/atomic-io.js +201 -0
- package/src/lib/cache.js +33 -0
- package/src/lib/npm-view.js +104 -0
- package/src/lib/status-card.js +95 -0
- package/src/lib/token.js +85 -0
- package/src/memory/fts5.js +349 -0
- package/src/memory/migration-runner.js +116 -0
- package/src/memory/migrations/001-fts5-init.js +26 -0
- package/src/memory/migrations/002-tier-semantic.js +60 -0
- package/src/memory/migrations/003-stale-candidate.js +60 -0
- package/src/memory/reader.js +300 -0
- package/src/memory/recall-counter.js +76 -0
- package/src/memory/schema.sql +79 -0
- package/src/memory/search.js +431 -0
- package/src/memory/staleness.js +237 -0
- package/src/memory/tier-promotion.js +377 -0
- package/src/memory/tokenize.js +63 -0
- package/src/project-type-detector.js +866 -0
- package/src/prompt-check.js +171 -0
- package/src/ralph-allowlist.js +88 -0
- package/src/receipts.js +129 -0
- package/src/redactor.js +107 -0
- package/src/sandbox.js +275 -0
- package/src/sanitizer.js +69 -0
- package/src/scan-resume.js +167 -0
- package/src/schema.js +82 -0
- package/src/search-bm25.js +108 -0
- package/src/server.js +1414 -0
- package/src/swarm-config.js +80 -0
- package/src/trident/dispatch.js +211 -0
- package/src/trident/lens-health.js +253 -0
- package/src/update-apply.js +79 -0
- package/src/update-check.js +136 -0
- package/src/vectors.js +178 -0
- package/templates/design/bento-grid.md +84 -0
- package/templates/design/brutalist-luxe.md +82 -0
- package/templates/design/cinematic-dark.md +82 -0
- package/templates/design/data-dense-dashboard.md +88 -0
- package/templates/design/editorial-warm.md +81 -0
- package/templates/design/glassmorphic.md +84 -0
- package/templates/design/magazine-editorial.md +84 -0
- package/templates/design/maximalist-vibrant.md +85 -0
- package/templates/design/neo-swiss-tech.md +85 -0
- package/templates/design/swiss-minimal.md +80 -0
- package/templates/design/terminal-native.md +83 -0
- package/templates/design/warm-organic.md +84 -0
|
@@ -0,0 +1,431 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Memory search -- D-Pillar / D0 (IJFW v1.3.0).
|
|
3
|
+
*
|
|
4
|
+
* Tiered pipeline:
|
|
5
|
+
* Hot -- linear regex over markdown files (always available; the existing
|
|
6
|
+
* v1.2 path). Used both as the source for FTS5 auto-index and as
|
|
7
|
+
* the graceful-degradation fallback when FTS5 is unavailable or
|
|
8
|
+
* returns no hits.
|
|
9
|
+
* Warm -- FTS5 over <repoRoot>/.ijfw/index/memory.db (porter unicode61).
|
|
10
|
+
* Auto-indexes from the file list when the index is empty.
|
|
11
|
+
* Synonym expansion via shared compute/synonyms.js so coding
|
|
12
|
+
* shorthand ("db" -> "database", "auth" -> "authentication") fires
|
|
13
|
+
* the same way as on the compute tier.
|
|
14
|
+
*
|
|
15
|
+
* Public surface preserved: searchMemory(q, files, limit) -> Array<result>
|
|
16
|
+
* (synchronous) so the dashboard /api/memory/search handler keeps working
|
|
17
|
+
* with no caller-side change. We achieve this by lazily resolving the
|
|
18
|
+
* better-sqlite3 driver via a top-level await at module load and using its
|
|
19
|
+
* synchronous open() inside searchMemory.
|
|
20
|
+
*
|
|
21
|
+
* Result-array decorations (non-enumerable):
|
|
22
|
+
* - synonym_matches -- { token: [expansions] } when expansion fired
|
|
23
|
+
* - tier -- 'warm-fts5' | 'hot-linear' | 'hot-linear-empty-fts5'
|
|
24
|
+
* Legacy callers see no shape change because Object.keys() on an array
|
|
25
|
+
* yields only the indices.
|
|
26
|
+
*
|
|
27
|
+
* Zero new deps. better-sqlite3 already ships from Phase 1.
|
|
28
|
+
*/
|
|
29
|
+
|
|
30
|
+
import { readFileSync, existsSync, mkdirSync } from 'node:fs';
|
|
31
|
+
import { dirname, join, resolve, normalize, isAbsolute } from 'node:path';
|
|
32
|
+
|
|
33
|
+
import { expandQuery } from '../compute/synonyms.js';
|
|
34
|
+
|
|
35
|
+
const MAX_RESULTS = 50;
|
|
36
|
+
const SNIPPET_HALF = 60;
|
|
37
|
+
const DB_FILENAME = 'memory.db';
|
|
38
|
+
const INDEX_DIR_NAME = 'index';
|
|
39
|
+
const IJFW_DIR_NAME = '.ijfw';
|
|
40
|
+
|
|
41
|
+
// --- Driver bootstrap (top-level await; resolves once at module load) -----
|
|
42
|
+
|
|
43
|
+
let DRIVER = null;
|
|
44
|
+
try {
|
|
45
|
+
const mod = await import('better-sqlite3');
|
|
46
|
+
const Database = mod.default || mod;
|
|
47
|
+
DRIVER = { kind: 'better-sqlite3', Database };
|
|
48
|
+
} catch {
|
|
49
|
+
DRIVER = null;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
// Resolve migration modules synchronously at module load via top-level
|
|
53
|
+
// await. Replayed inside searchMemory's sync path. Keep in lockstep with
|
|
54
|
+
// ./migrations/.
|
|
55
|
+
const MEMORY_MIGRATIONS = await loadMemoryMigrationsSync();
|
|
56
|
+
|
|
57
|
+
async function loadMemoryMigrationsSync() {
|
|
58
|
+
const v1 = await import('./migrations/001-fts5-init.js');
|
|
59
|
+
const v2 = await import('./migrations/002-tier-semantic.js');
|
|
60
|
+
const v3 = await import('./migrations/003-stale-candidate.js');
|
|
61
|
+
return [
|
|
62
|
+
{ version: v1.VERSION, description: v1.DESCRIPTION, up: v1.up },
|
|
63
|
+
{ version: v2.VERSION, description: v2.DESCRIPTION, up: v2.up },
|
|
64
|
+
{ version: v3.VERSION, description: v3.DESCRIPTION, up: v3.up },
|
|
65
|
+
].sort((a, b) => a.version - b.version);
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
function highestMigrationVersion() {
|
|
69
|
+
if (!MEMORY_MIGRATIONS.length) return 0;
|
|
70
|
+
return MEMORY_MIGRATIONS[MEMORY_MIGRATIONS.length - 1].version;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
// --- Snippet helper ---------------------------------------------------------
|
|
74
|
+
|
|
75
|
+
function snippet(body, pattern) {
|
|
76
|
+
const idx = body.search(pattern);
|
|
77
|
+
if (idx === -1) return body.slice(0, 120).trim();
|
|
78
|
+
const start = Math.max(0, idx - SNIPPET_HALF);
|
|
79
|
+
const end = Math.min(body.length, idx + SNIPPET_HALF + pattern.source.length);
|
|
80
|
+
let s = body.slice(start, end).replace(/\s+/g, ' ').trim();
|
|
81
|
+
if (start > 0) s = '...' + s;
|
|
82
|
+
if (end < body.length) s = s + '...';
|
|
83
|
+
return s;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
// --- Hot tier (legacy linear regex; preserved for fallback) -----------------
|
|
87
|
+
|
|
88
|
+
function searchLinear(q, files, limit) {
|
|
89
|
+
if (!q || !q.trim() || !files.length) return [];
|
|
90
|
+
|
|
91
|
+
const escaped = q.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
92
|
+
let pattern;
|
|
93
|
+
try {
|
|
94
|
+
pattern = new RegExp(escaped, 'gi');
|
|
95
|
+
} catch {
|
|
96
|
+
return [];
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
const results = [];
|
|
100
|
+
|
|
101
|
+
for (const f of files) {
|
|
102
|
+
let body = '';
|
|
103
|
+
try {
|
|
104
|
+
body = existsSync(f.path) ? readFileSync(f.path, 'utf8') : '';
|
|
105
|
+
} catch { /* skip */ }
|
|
106
|
+
|
|
107
|
+
const titleMatches = (f.title.match(pattern) || []).length;
|
|
108
|
+
const bodyMatches = (body.match(pattern) || []).length;
|
|
109
|
+
const total = titleMatches + bodyMatches;
|
|
110
|
+
if (total === 0) continue;
|
|
111
|
+
|
|
112
|
+
pattern.lastIndex = 0;
|
|
113
|
+
const score = titleMatches * 3 + bodyMatches;
|
|
114
|
+
|
|
115
|
+
results.push({
|
|
116
|
+
path: f.path,
|
|
117
|
+
relpath: f.relpath,
|
|
118
|
+
title: f.title,
|
|
119
|
+
snippet: snippet(body, pattern),
|
|
120
|
+
score,
|
|
121
|
+
});
|
|
122
|
+
|
|
123
|
+
pattern.lastIndex = 0;
|
|
124
|
+
if (results.length >= limit * 2) break;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
results.sort((a, b) => b.score - a.score);
|
|
128
|
+
return results.slice(0, limit);
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
// --- Warm tier (FTS5; synchronous via cached driver) ------------------------
|
|
132
|
+
|
|
133
|
+
function resolveProjectRoot(raw) {
|
|
134
|
+
const v = raw || process.env.IJFW_PROJECT_DIR || process.cwd();
|
|
135
|
+
if (typeof v !== 'string' || !v) return null;
|
|
136
|
+
const abs = resolve(v);
|
|
137
|
+
const norm = normalize(abs);
|
|
138
|
+
if (!isAbsolute(norm)) return null;
|
|
139
|
+
return norm;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
function dbPathFor(root) {
|
|
143
|
+
return join(root, IJFW_DIR_NAME, INDEX_DIR_NAME, DB_FILENAME);
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
function resolveIndexRoot(files) {
|
|
147
|
+
if (process.env.IJFW_PROJECT_DIR) return resolveProjectRoot(process.env.IJFW_PROJECT_DIR);
|
|
148
|
+
if (files && files.length > 0) {
|
|
149
|
+
for (const f of files) {
|
|
150
|
+
if (typeof f.path !== 'string') continue;
|
|
151
|
+
// Match both POSIX and Windows path separators around .ijfw segment.
|
|
152
|
+
const m = f.path.match(/[\\/]\.ijfw[\\/]/);
|
|
153
|
+
if (m) return f.path.slice(0, m.index);
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
return resolveProjectRoot(process.cwd());
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
function openMemoryDbSync(root) {
|
|
160
|
+
if (!DRIVER) return null;
|
|
161
|
+
if (!root) return null;
|
|
162
|
+
|
|
163
|
+
const filename = dbPathFor(root);
|
|
164
|
+
try {
|
|
165
|
+
const dir = dirname(filename);
|
|
166
|
+
if (dir && !existsSync(dir)) mkdirSync(dir, { recursive: true });
|
|
167
|
+
const db = new DRIVER.Database(filename);
|
|
168
|
+
db.__ijfw_filename = filename;
|
|
169
|
+
|
|
170
|
+
try { db.exec('PRAGMA journal_mode = WAL'); } catch { /* default fine */ }
|
|
171
|
+
try { db.exec('PRAGMA synchronous = NORMAL'); } catch { /* default fine */ }
|
|
172
|
+
try { db.exec('PRAGMA busy_timeout = 5000'); } catch { /* default fine */ }
|
|
173
|
+
|
|
174
|
+
return db;
|
|
175
|
+
} catch {
|
|
176
|
+
return null;
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
function readUserVersion(db) {
|
|
181
|
+
try {
|
|
182
|
+
const row = db.prepare('PRAGMA user_version').get();
|
|
183
|
+
if (!row) return 0;
|
|
184
|
+
return Number(row.user_version ?? row.USER_VERSION ?? 0);
|
|
185
|
+
} catch {
|
|
186
|
+
return 0;
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
function runMemoryMigrationsSync(db, currentVersion, targetVersion) {
|
|
191
|
+
if (currentVersion >= targetVersion) return currentVersion;
|
|
192
|
+
for (const m of MEMORY_MIGRATIONS) {
|
|
193
|
+
if (m.version <= currentVersion || m.version > targetVersion) continue;
|
|
194
|
+
db.exec('BEGIN IMMEDIATE');
|
|
195
|
+
try {
|
|
196
|
+
m.up(db);
|
|
197
|
+
db.exec(`PRAGMA user_version = ${m.version}`);
|
|
198
|
+
const stmt = db.prepare(
|
|
199
|
+
'INSERT OR IGNORE INTO schema_meta(version, applied_at, description) VALUES (?, ?, ?)'
|
|
200
|
+
);
|
|
201
|
+
stmt.run(m.version, Date.now(), m.description);
|
|
202
|
+
db.exec('COMMIT');
|
|
203
|
+
} catch (err) {
|
|
204
|
+
try { db.exec('ROLLBACK'); } catch { /* ignore */ }
|
|
205
|
+
throw err;
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
return targetVersion;
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
function autoIndex(db, files) {
|
|
212
|
+
let n = 0;
|
|
213
|
+
const txfn = db.transaction((batch) => {
|
|
214
|
+
const stmt = db.prepare(
|
|
215
|
+
'INSERT INTO memory_entries (body, source, session_id, created_at) VALUES (?, ?, ?, ?)'
|
|
216
|
+
);
|
|
217
|
+
for (const item of batch) {
|
|
218
|
+
stmt.run(item.body, item.source, null, item.created_at);
|
|
219
|
+
n++;
|
|
220
|
+
}
|
|
221
|
+
});
|
|
222
|
+
|
|
223
|
+
const batch = [];
|
|
224
|
+
const now = Date.now();
|
|
225
|
+
for (const f of files) {
|
|
226
|
+
if (typeof f.path !== 'string') continue;
|
|
227
|
+
if (!existsSync(f.path)) continue;
|
|
228
|
+
let body;
|
|
229
|
+
try { body = readFileSync(f.path, 'utf8'); } catch { continue; }
|
|
230
|
+
if (!body) continue;
|
|
231
|
+
batch.push({ body, source: f.relpath || f.path, created_at: now });
|
|
232
|
+
}
|
|
233
|
+
if (batch.length === 0) return 0;
|
|
234
|
+
try { txfn.immediate(batch); } catch { /* one bad batch should not abort the search */ }
|
|
235
|
+
return n;
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
// FTS5 search joined to the content table. When tier_semantic is provided,
|
|
239
|
+
// the join filters rows by the D1 axis (working/episodic/semantic/
|
|
240
|
+
// procedural/procedural_candidate). When undefined, all tiers return --
|
|
241
|
+
// preserves the pre-D1 default. tier_semantic is a defensive enum check
|
|
242
|
+
// inside this module; the SQL itself uses parameter binding so the value
|
|
243
|
+
// can never form an injection vector.
|
|
244
|
+
const VALID_TIER_SEMANTIC = new Set([
|
|
245
|
+
'working', 'episodic', 'semantic', 'procedural', 'procedural_candidate',
|
|
246
|
+
]);
|
|
247
|
+
|
|
248
|
+
function searchFts5(db, query, k, tier_semantic, include_stale) {
|
|
249
|
+
const limit = Math.min(Math.max(1, parseInt(k, 10) || 10), 1000);
|
|
250
|
+
const tierFilter = tier_semantic && VALID_TIER_SEMANTIC.has(tier_semantic);
|
|
251
|
+
// D4 GA-B2 retrieval guard: exclude rows with stale_candidate >= 1 by
|
|
252
|
+
// default; opt-in via include_stale=true (mirrors compute/fts5.js search).
|
|
253
|
+
// Pre-v3 dbs (no stale_candidate column) silently drop the WHERE filter
|
|
254
|
+
// so older callers and fixture dbs keep working.
|
|
255
|
+
const hasStaleCol = hasMemoryStaleColumn(db);
|
|
256
|
+
const staleClause = (!include_stale && hasStaleCol)
|
|
257
|
+
? ' AND COALESCE(t.stale_candidate, 0) = 0'
|
|
258
|
+
: '';
|
|
259
|
+
const sql = `
|
|
260
|
+
SELECT t.id, t.body, t.source, t.session_id, t.created_at, t.tier_semantic,
|
|
261
|
+
bm25(memory_entries_fts) AS rank
|
|
262
|
+
FROM memory_entries_fts f
|
|
263
|
+
JOIN memory_entries t ON t.id = f.rowid
|
|
264
|
+
WHERE memory_entries_fts MATCH ?
|
|
265
|
+
${tierFilter ? 'AND t.tier_semantic = ?' : ''}${staleClause}
|
|
266
|
+
ORDER BY rank ASC
|
|
267
|
+
LIMIT ?`;
|
|
268
|
+
return tierFilter
|
|
269
|
+
? db.prepare(sql).all(query, tier_semantic, limit)
|
|
270
|
+
: db.prepare(sql).all(query, limit);
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
// Cache PRAGMA table_info lookups per-db so repeated search() calls don't
|
|
274
|
+
// re-scan the schema. WeakMap keyed on the db handle gives us automatic
|
|
275
|
+
// cleanup when the db is closed.
|
|
276
|
+
const __memoryColumnCache = new WeakMap();
|
|
277
|
+
function hasMemoryStaleColumn(db) {
|
|
278
|
+
let perDb = __memoryColumnCache.get(db);
|
|
279
|
+
if (!perDb) {
|
|
280
|
+
perDb = new Map();
|
|
281
|
+
__memoryColumnCache.set(db, perDb);
|
|
282
|
+
}
|
|
283
|
+
if (perDb.has('stale_candidate')) return perDb.get('stale_candidate');
|
|
284
|
+
let present = false;
|
|
285
|
+
try {
|
|
286
|
+
const rows = db.prepare(`PRAGMA table_info(memory_entries)`).all();
|
|
287
|
+
present = rows.some(r => String(r.name) === 'stale_candidate');
|
|
288
|
+
} catch { /* missing table -> treat column as absent */ }
|
|
289
|
+
perDb.set('stale_candidate', present);
|
|
290
|
+
return present;
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
function ftsRowToResult(row, fileBySource) {
|
|
294
|
+
const src = row.source || '';
|
|
295
|
+
const meta = fileBySource.get(src) || null;
|
|
296
|
+
const path = (meta && meta.path) || src;
|
|
297
|
+
const relpath = (meta && meta.relpath) || src;
|
|
298
|
+
const title = (meta && meta.title) || src.split('/').pop() || src;
|
|
299
|
+
const score = 100 - Number(row.rank || 0);
|
|
300
|
+
const text = String(row.body || '');
|
|
301
|
+
const snip = text.slice(0, 200).replace(/\s+/g, ' ').trim();
|
|
302
|
+
// tier_semantic surfaced per result so callers building tier-aware
|
|
303
|
+
// dashboards / consolidation views can read it without a separate
|
|
304
|
+
// lookup. Pre-D1 rows return 'working' from the column DEFAULT.
|
|
305
|
+
const tier_semantic = row.tier_semantic || 'working';
|
|
306
|
+
return { path, relpath, title, snippet: snip, score, tier_semantic };
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
function rowCount(db) {
|
|
310
|
+
try {
|
|
311
|
+
const row = db.prepare('SELECT COUNT(*) AS n FROM memory_entries').get();
|
|
312
|
+
return row ? Number(row.n) : 0;
|
|
313
|
+
} catch {
|
|
314
|
+
return 0;
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
// --- Public API -------------------------------------------------------------
|
|
319
|
+
|
|
320
|
+
/**
|
|
321
|
+
* Search memory files for query string.
|
|
322
|
+
*
|
|
323
|
+
* Tries FTS5 (auto-indexes if empty) with synonym expansion, falls back to
|
|
324
|
+
* linear regex on empty/miss/error. Stays synchronous to preserve the
|
|
325
|
+
* existing public contract.
|
|
326
|
+
*
|
|
327
|
+
* D1 (tier_semantic) filter:
|
|
328
|
+
* The fourth argument may be either an `options` object or a tier string
|
|
329
|
+
* (legacy positional shorthand). Recognised:
|
|
330
|
+
* - { tier_semantic: 'working' | 'episodic' | 'semantic'
|
|
331
|
+
* | 'procedural' | 'procedural_candidate' | undefined }
|
|
332
|
+
* - undefined (default) returns all tiers -- pre-D1 behaviour preserved.
|
|
333
|
+
* When the tier filter is set the warm-tier search restricts results;
|
|
334
|
+
* the hot-linear fallback is unfiltered (D1 does not yet write tier
|
|
335
|
+
* metadata into the markdown surface).
|
|
336
|
+
*
|
|
337
|
+
* @param {string} q
|
|
338
|
+
* @param {Array<{path,relpath,title,preview}>} files
|
|
339
|
+
* @param {number} limit
|
|
340
|
+
* @param {object|undefined} options
|
|
341
|
+
* @returns {Array<{path,relpath,title,snippet,score,tier_semantic}>}
|
|
342
|
+
*/
|
|
343
|
+
export function searchMemory(q, files, limit = MAX_RESULTS, options) {
|
|
344
|
+
if (!q || !q.trim() || !files || files.length === 0) return [];
|
|
345
|
+
|
|
346
|
+
// Normalise options. Allow undefined / { tier_semantic, include_stale } /
|
|
347
|
+
// a bare string (treated as the tier_semantic value) for ergonomic call
|
|
348
|
+
// sites. include_stale defaults to false -- D4 GA-B2 retrieval guard.
|
|
349
|
+
let tier_semantic;
|
|
350
|
+
let include_stale = false;
|
|
351
|
+
if (typeof options === 'string') {
|
|
352
|
+
tier_semantic = options;
|
|
353
|
+
} else if (options && typeof options === 'object') {
|
|
354
|
+
tier_semantic = options.tier_semantic;
|
|
355
|
+
include_stale = options.include_stale === true;
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
const { expanded, synonym_matches, applied } = expandQuery(q);
|
|
359
|
+
|
|
360
|
+
let warmHits = null;
|
|
361
|
+
let warmEmpty = false;
|
|
362
|
+
let db = null;
|
|
363
|
+
|
|
364
|
+
try {
|
|
365
|
+
const root = resolveIndexRoot(files);
|
|
366
|
+
db = openMemoryDbSync(root);
|
|
367
|
+
|
|
368
|
+
if (db) {
|
|
369
|
+
const target = highestMigrationVersion();
|
|
370
|
+
const current = readUserVersion(db);
|
|
371
|
+
if (current < target) {
|
|
372
|
+
runMemoryMigrationsSync(db, current, target);
|
|
373
|
+
} else if (current > target) {
|
|
374
|
+
// Newer schema -- refuse rather than downgrade. Legacy fallback.
|
|
375
|
+
try { db.close(); } catch { /* ignore */ }
|
|
376
|
+
db = null;
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
if (db) {
|
|
380
|
+
if (rowCount(db) === 0 && files.length > 0) {
|
|
381
|
+
autoIndex(db, files);
|
|
382
|
+
}
|
|
383
|
+
const ftsQuery = applied ? expanded : q;
|
|
384
|
+
let rows;
|
|
385
|
+
try {
|
|
386
|
+
rows = searchFts5(db, ftsQuery, limit, tier_semantic, include_stale);
|
|
387
|
+
} catch {
|
|
388
|
+
try { rows = searchFts5(db, q, limit, tier_semantic, include_stale); } catch { rows = []; }
|
|
389
|
+
}
|
|
390
|
+
if (rows.length > 0) {
|
|
391
|
+
const fileBySource = new Map();
|
|
392
|
+
for (const f of files) {
|
|
393
|
+
if (f.relpath) fileBySource.set(f.relpath, f);
|
|
394
|
+
if (f.path) fileBySource.set(f.path, f);
|
|
395
|
+
}
|
|
396
|
+
warmHits = rows.map(r => ftsRowToResult(r, fileBySource));
|
|
397
|
+
} else {
|
|
398
|
+
warmEmpty = true;
|
|
399
|
+
}
|
|
400
|
+
}
|
|
401
|
+
}
|
|
402
|
+
} catch {
|
|
403
|
+
warmHits = null;
|
|
404
|
+
} finally {
|
|
405
|
+
if (db) { try { db.close(); } catch { /* ignore */ } }
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
let results;
|
|
409
|
+
if (warmHits && warmHits.length > 0) {
|
|
410
|
+
results = warmHits.slice(0, limit);
|
|
411
|
+
} else if (tier_semantic) {
|
|
412
|
+
// Tier filter active and warm tier has no matches -- the hot-linear
|
|
413
|
+
// tier doesn't carry tier metadata so it can't honour the filter.
|
|
414
|
+
// Returning [] here keeps the contract honest ("only matching tier").
|
|
415
|
+
results = [];
|
|
416
|
+
} else {
|
|
417
|
+
results = searchLinear(q, files, limit);
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
Object.defineProperty(results, 'synonym_matches', {
|
|
421
|
+
value: applied ? synonym_matches : {},
|
|
422
|
+
enumerable: false,
|
|
423
|
+
});
|
|
424
|
+
Object.defineProperty(results, 'tier', {
|
|
425
|
+
value: warmHits && warmHits.length > 0
|
|
426
|
+
? 'warm-fts5'
|
|
427
|
+
: (warmEmpty ? 'hot-linear-empty-fts5' : 'hot-linear'),
|
|
428
|
+
enumerable: false,
|
|
429
|
+
});
|
|
430
|
+
return results;
|
|
431
|
+
}
|
|
@@ -0,0 +1,237 @@
|
|
|
1
|
+
// IJFW v1.3.0 -- D4 memory-side cascading staleness propagation.
|
|
2
|
+
//
|
|
3
|
+
// Source authority: PRD-v2 section 9 Pillar D D4 + .planning/1.3.0/D-PILLAR-SPEC.md
|
|
4
|
+
// section 2 (cascading staleness propagation) + GA real fix-wave finding F2.
|
|
5
|
+
//
|
|
6
|
+
// PROBLEM (pre-F2): mcp-server/src/compute/staleness.js only mutates the
|
|
7
|
+
// COMPUTE store (`raw` + `compiled` tables in compute.db). The memory store
|
|
8
|
+
// has its own `memory_entries.stale_candidate` column (memory migration 003)
|
|
9
|
+
// and the memory search filter excludes flagged rows by default, but
|
|
10
|
+
// production never WRITES to that column. Memory rows that should be flagged
|
|
11
|
+
// stale by the dream cycle's BFS propagation never get flagged in practice;
|
|
12
|
+
// the column + filter are dead infrastructure.
|
|
13
|
+
//
|
|
14
|
+
// FIX: this module mirrors the compute-side `propagateStale` shape but
|
|
15
|
+
// operates on `memory_entries`. It is invoked by
|
|
16
|
+
// `mcp-server/src/dream/staleness-wiring.js` AFTER `propagateStale` runs
|
|
17
|
+
// against the compute db, so a single supersession event flags BOTH
|
|
18
|
+
// stores in the same dream cycle.
|
|
19
|
+
//
|
|
20
|
+
// The COMPUTE kg graph is the single source of truth for entity relations
|
|
21
|
+
// (per architecture: memory.indexEntry fires `autoIndexGraphFromMemoryBody`
|
|
22
|
+
// against the SAME compute kg). Memory propagation walks the same BFS
|
|
23
|
+
// surface but resolves matching rows by literal substring against
|
|
24
|
+
// `memory_entries.body`, mirroring `compute.staleness.js`'s name-pattern
|
|
25
|
+
// approach.
|
|
26
|
+
//
|
|
27
|
+
// Read-only access to compute kg: this module accepts a SECOND db handle
|
|
28
|
+
// (`computeDb`) opened by the caller. We never write the compute db here;
|
|
29
|
+
// we only walk kg_nodes + kg_edges to discover the BFS frontier. Writes
|
|
30
|
+
// land on `memory_entries.stale_candidate` only.
|
|
31
|
+
//
|
|
32
|
+
// Locking: caller holds .graph-write.lock for the dream cycle (per
|
|
33
|
+
// staleness-wiring.js header). This module performs reads against compute
|
|
34
|
+
// kg + writes against memory; the lock keeps concurrent kg writers out
|
|
35
|
+
// while we walk.
|
|
36
|
+
|
|
37
|
+
const DEFAULT_DEPTH_CAP = 2;
|
|
38
|
+
const DEFAULT_WEIGHT_THRESHOLD = 0.5;
|
|
39
|
+
const DEFAULT_EDGE_KINDS = ['co_occurs'];
|
|
40
|
+
const DEFAULT_STALE_VALUE = 1;
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* propagateStaleMemory(memDb, computeDb, supersededNodeId, options) -> envelope
|
|
44
|
+
*
|
|
45
|
+
* Walk the COMPUTE kg graph from `supersededNodeId` (BFS, weight + depth
|
|
46
|
+
* gated per D-PILLAR-SPEC §2). For every reachable node (excluding the
|
|
47
|
+
* start node by default), find `memory_entries` rows whose `body` mentions
|
|
48
|
+
* the node's name and flip their `stale_candidate` column to `staleValue`.
|
|
49
|
+
*
|
|
50
|
+
* options:
|
|
51
|
+
* weight_threshold (number, default 0.5)
|
|
52
|
+
* depth_cap (integer, default 2)
|
|
53
|
+
* edge_kinds (string[], default ['co_occurs'])
|
|
54
|
+
* stale_value (integer, default 1)
|
|
55
|
+
* include_start (boolean, default false)
|
|
56
|
+
*
|
|
57
|
+
* Returns:
|
|
58
|
+
* {
|
|
59
|
+
* flagged_count: number, // memory_entries rows flipped
|
|
60
|
+
* reached_nodes: [{id,kind,name,depth,redacted}],
|
|
61
|
+
* edge_weights_sampled: number[],
|
|
62
|
+
* depth_distribution: {1:n, 2:n},
|
|
63
|
+
* traversal_path: number[],
|
|
64
|
+
* weight_threshold: number,
|
|
65
|
+
* depth_cap: number,
|
|
66
|
+
* }
|
|
67
|
+
*/
|
|
68
|
+
export function propagateStaleMemory(memDb, computeDb, supersededNodeId, options = {}) {
|
|
69
|
+
if (!memDb || typeof memDb.prepare !== 'function') {
|
|
70
|
+
throw new Error('propagateStaleMemory: memDb handle is invalid.');
|
|
71
|
+
}
|
|
72
|
+
if (!computeDb || typeof computeDb.prepare !== 'function') {
|
|
73
|
+
throw new Error('propagateStaleMemory: computeDb handle is invalid.');
|
|
74
|
+
}
|
|
75
|
+
const startId = Number(supersededNodeId);
|
|
76
|
+
if (!Number.isFinite(startId) || startId <= 0) {
|
|
77
|
+
throw new Error(`propagateStaleMemory: supersededNodeId must be a positive number; got ${supersededNodeId}`);
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
const weightThreshold = Number.isFinite(options.weight_threshold)
|
|
81
|
+
? Number(options.weight_threshold)
|
|
82
|
+
: DEFAULT_WEIGHT_THRESHOLD;
|
|
83
|
+
const depthCap = Number.isInteger(options.depth_cap) && options.depth_cap >= 0
|
|
84
|
+
? options.depth_cap
|
|
85
|
+
: DEFAULT_DEPTH_CAP;
|
|
86
|
+
const edgeKinds = Array.isArray(options.edge_kinds) && options.edge_kinds.length > 0
|
|
87
|
+
? options.edge_kinds.map(String)
|
|
88
|
+
: DEFAULT_EDGE_KINDS;
|
|
89
|
+
const staleValue = Number.isInteger(options.stale_value) && options.stale_value >= 0
|
|
90
|
+
? options.stale_value
|
|
91
|
+
: DEFAULT_STALE_VALUE;
|
|
92
|
+
const includeStart = options.include_start === true;
|
|
93
|
+
|
|
94
|
+
// Verify start node exists in compute kg.
|
|
95
|
+
const startNode = computeDb.prepare(
|
|
96
|
+
`SELECT id, kind, name, redacted FROM kg_nodes WHERE id = ?`
|
|
97
|
+
).get(startId);
|
|
98
|
+
if (!startNode) {
|
|
99
|
+
return emptyEnvelope();
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
// BFS over the compute kg (read-only).
|
|
103
|
+
const visited = new Map();
|
|
104
|
+
visited.set(startNode.id, { node: startNode, depth: 0 });
|
|
105
|
+
const traversalPath = [startNode.id];
|
|
106
|
+
const edgeWeightsSampled = [];
|
|
107
|
+
|
|
108
|
+
const placeholders = edgeKinds.map(() => '?').join(', ');
|
|
109
|
+
const queryNeighbours = computeDb.prepare(
|
|
110
|
+
`SELECT src, dst, kind, weight, co_occurrence_count, ts FROM kg_edges ` +
|
|
111
|
+
`WHERE (src = ? OR dst = ?) AND kind IN (${placeholders}) AND weight >= ?`
|
|
112
|
+
);
|
|
113
|
+
const queryNode = computeDb.prepare(
|
|
114
|
+
`SELECT id, kind, name, redacted FROM kg_nodes WHERE id = ?`
|
|
115
|
+
);
|
|
116
|
+
|
|
117
|
+
let frontier = [startNode.id];
|
|
118
|
+
for (let hop = 0; hop < depthCap && frontier.length > 0; hop++) {
|
|
119
|
+
const next = [];
|
|
120
|
+
for (const nodeId of frontier) {
|
|
121
|
+
const rows = queryNeighbours.all(nodeId, nodeId, ...edgeKinds, weightThreshold);
|
|
122
|
+
for (const row of rows) {
|
|
123
|
+
const otherId = Number(row.src) === nodeId ? Number(row.dst) : Number(row.src);
|
|
124
|
+
if (visited.has(otherId)) continue;
|
|
125
|
+
const nrow = queryNode.get(otherId);
|
|
126
|
+
if (!nrow) continue;
|
|
127
|
+
const depth = hop + 1;
|
|
128
|
+
visited.set(otherId, { node: nrow, depth });
|
|
129
|
+
traversalPath.push(otherId);
|
|
130
|
+
edgeWeightsSampled.push(Number(row.weight));
|
|
131
|
+
if (!nrow.redacted) next.push(otherId);
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
frontier = next;
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
// Build the set of names to flag against memory_entries.
|
|
138
|
+
const namesToFlag = [];
|
|
139
|
+
const reachedNodes = [];
|
|
140
|
+
const depthDist = {};
|
|
141
|
+
for (const { node, depth } of visited.values()) {
|
|
142
|
+
if (depth === 0 && !includeStart) {
|
|
143
|
+
reachedNodes.push({ id: node.id, kind: node.kind, name: node.name, depth, redacted: !!node.redacted });
|
|
144
|
+
continue;
|
|
145
|
+
}
|
|
146
|
+
reachedNodes.push({ id: node.id, kind: node.kind, name: node.name, depth, redacted: !!node.redacted });
|
|
147
|
+
depthDist[depth] = (depthDist[depth] || 0) + 1;
|
|
148
|
+
if (!node.redacted && typeof node.name === 'string' && node.name.length >= 2) {
|
|
149
|
+
namesToFlag.push(node.name);
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
// Defence: if the memory schema predates migration 003, the column
|
|
154
|
+
// doesn't exist; degrade silently rather than throw. (Production callers
|
|
155
|
+
// run migrations before opening; tests that bypass openDb may not.)
|
|
156
|
+
if (!hasStaleCandidateColumn(memDb)) {
|
|
157
|
+
return {
|
|
158
|
+
flagged_count: 0,
|
|
159
|
+
reached_nodes: reachedNodes,
|
|
160
|
+
edge_weights_sampled: edgeWeightsSampled,
|
|
161
|
+
depth_distribution: depthDist,
|
|
162
|
+
traversal_path: traversalPath,
|
|
163
|
+
weight_threshold: weightThreshold,
|
|
164
|
+
depth_cap: depthCap,
|
|
165
|
+
};
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
let flagged = 0;
|
|
169
|
+
if (namesToFlag.length > 0) {
|
|
170
|
+
const updateMem = memDb.prepare(
|
|
171
|
+
`UPDATE memory_entries SET stale_candidate = ? ` +
|
|
172
|
+
`WHERE COALESCE(stale_candidate, 0) < ? AND body LIKE ?`
|
|
173
|
+
);
|
|
174
|
+
|
|
175
|
+
const txWrap = (typeof memDb.transaction === 'function')
|
|
176
|
+
? memDb.transaction(() => doFlag())
|
|
177
|
+
: (typeof memDb.txn === 'function' ? memDb.txn(() => doFlag()) : null);
|
|
178
|
+
|
|
179
|
+
function doFlag() {
|
|
180
|
+
for (const name of namesToFlag) {
|
|
181
|
+
const pattern = `%${escapeLike(name)}%`;
|
|
182
|
+
const info = updateMem.run(staleValue, staleValue, pattern);
|
|
183
|
+
flagged += Number(info.changes || 0);
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
if (typeof txWrap === 'function') txWrap();
|
|
188
|
+
else doFlag();
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
return {
|
|
192
|
+
flagged_count: flagged,
|
|
193
|
+
reached_nodes: reachedNodes,
|
|
194
|
+
edge_weights_sampled: edgeWeightsSampled,
|
|
195
|
+
depth_distribution: depthDist,
|
|
196
|
+
traversal_path: traversalPath,
|
|
197
|
+
weight_threshold: weightThreshold,
|
|
198
|
+
depth_cap: depthCap,
|
|
199
|
+
};
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
// LIKE escape -- mirrors compute.staleness.js.
|
|
203
|
+
function escapeLike(s) {
|
|
204
|
+
return String(s).replace(/[\\%_]/g, '\\$&');
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
function hasStaleCandidateColumn(db) {
|
|
208
|
+
try {
|
|
209
|
+
const rows = db.prepare(`PRAGMA table_info(memory_entries)`).all();
|
|
210
|
+
return rows.some(r => String(r.name) === 'stale_candidate');
|
|
211
|
+
} catch {
|
|
212
|
+
return false;
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
function emptyEnvelope() {
|
|
217
|
+
return {
|
|
218
|
+
flagged_count: 0,
|
|
219
|
+
reached_nodes: [],
|
|
220
|
+
edge_weights_sampled: [],
|
|
221
|
+
depth_distribution: {},
|
|
222
|
+
traversal_path: [],
|
|
223
|
+
weight_threshold: DEFAULT_WEIGHT_THRESHOLD,
|
|
224
|
+
depth_cap: DEFAULT_DEPTH_CAP,
|
|
225
|
+
};
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
export const __test = {
|
|
229
|
+
DEFAULT_DEPTH_CAP,
|
|
230
|
+
DEFAULT_WEIGHT_THRESHOLD,
|
|
231
|
+
DEFAULT_EDGE_KINDS,
|
|
232
|
+
DEFAULT_STALE_VALUE,
|
|
233
|
+
escapeLike,
|
|
234
|
+
hasStaleCandidateColumn,
|
|
235
|
+
};
|
|
236
|
+
|
|
237
|
+
export default { propagateStaleMemory };
|