claude-mem-lite 3.56.0 → 3.57.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +1 -1
- package/hook-context.mjs +4 -0
- package/hook-episode.mjs +5 -2
- package/hook-memory.mjs +4 -2
- package/hook-optimize.mjs +4 -0
- package/hook-shared.mjs +13 -4
- package/hook-update.mjs +108 -14
- package/hook.mjs +2 -2
- package/install.mjs +2 -2
- package/lib/recent-core.mjs +56 -0
- package/lib/save-observation.mjs +38 -29
- package/lib/search-core.mjs +17 -3
- package/mem-cli.mjs +32 -18
- package/package.json +6 -3
- package/project-utils.mjs +20 -1
- package/registry.mjs +96 -8
- package/scripts/hook-launcher.mjs +32 -0
- package/scripts/post-tool-use.sh +39 -0
- package/search-engine.mjs +6 -3
- package/search-scoring.mjs +35 -14
- package/server.mjs +122 -26
- package/source-files.mjs +4 -0
- package/tfidf.mjs +42 -6
- package/utils.mjs +16 -18
package/project-utils.mjs
CHANGED
|
@@ -1,10 +1,29 @@
|
|
|
1
1
|
// claude-mem-lite shared project resolution
|
|
2
2
|
// Extracted from server.mjs and mem-cli.mjs to eliminate duplication
|
|
3
3
|
|
|
4
|
-
import {
|
|
4
|
+
import { basename, dirname } from 'path';
|
|
5
|
+
|
|
6
|
+
// Leaf module: imports nothing from utils.mjs. utils.mjs re-exports this module's
|
|
7
|
+
// symbols as a backward-compat barrel, so importing the barrel back from here
|
|
8
|
+
// would close a cycle (v3.56 P2-9). Keep this file dependency-free of the barrel.
|
|
5
9
|
|
|
6
10
|
const _cache = new Map();
|
|
7
11
|
|
|
12
|
+
/**
|
|
13
|
+
* Infer a sanitized project name from CLAUDE_PROJECT_DIR, PWD, or cwd.
|
|
14
|
+
* Format: "parent--basename" with non-alphanumeric chars replaced by hyphens.
|
|
15
|
+
* @returns {string} Sanitized project identifier safe for use in filenames
|
|
16
|
+
*/
|
|
17
|
+
export function inferProject() {
|
|
18
|
+
const p = process.env.CLAUDE_PROJECT_DIR || process.env.PWD || process.cwd();
|
|
19
|
+
const base = basename(p);
|
|
20
|
+
const parent = basename(dirname(p));
|
|
21
|
+
const raw = parent && parent !== '.' && parent !== '/' ? `${parent}--${base}` : base;
|
|
22
|
+
// Sanitize to prevent path traversal when used in filenames (ep-<project>.json)
|
|
23
|
+
// Truncate to 100 chars to avoid exceeding filesystem name limits (255 bytes)
|
|
24
|
+
return raw.replace(/[^a-zA-Z0-9_.-]/g, '-').slice(0, 100);
|
|
25
|
+
}
|
|
26
|
+
|
|
8
27
|
/**
|
|
9
28
|
* Resolve short project name to canonical "parent--base" form.
|
|
10
29
|
* Uses DB suffix match with in-process cache.
|
package/registry.mjs
CHANGED
|
@@ -133,6 +133,13 @@ const INVOCATIONS_SCHEMA = `
|
|
|
133
133
|
ON invocations(created_at);
|
|
134
134
|
`;
|
|
135
135
|
|
|
136
|
+
// Canonical indexed-column list for resources_fts, in FTS5_SCHEMA order (see the BM25
|
|
137
|
+
// weight note above — order is load-bearing). Drift from this list triggers a rebuild.
|
|
138
|
+
const FTS5_COLUMNS = [
|
|
139
|
+
'trigger_patterns', 'keywords', 'capability_summary', 'intent_tags',
|
|
140
|
+
'use_cases', 'domain_tags', 'tech_stack', 'name',
|
|
141
|
+
];
|
|
142
|
+
|
|
136
143
|
const PREINSTALLED_SCHEMA = `
|
|
137
144
|
CREATE TABLE IF NOT EXISTS preinstalled (
|
|
138
145
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
@@ -152,6 +159,73 @@ const PREINSTALLED_SCHEMA = `
|
|
|
152
159
|
ON preinstalled(type, name);
|
|
153
160
|
`;
|
|
154
161
|
|
|
162
|
+
// ─── Schema version ──────────────────────────────────────────────────────────
|
|
163
|
+
|
|
164
|
+
// Registry DB schema version. Tracked independently of schema.mjs's
|
|
165
|
+
// CURRENT_SCHEMA_VERSION — resource-registry.db is a separate file with its own
|
|
166
|
+
// migration history. v1 = the shape at the time version tracking was introduced; every
|
|
167
|
+
// pre-existing (un-versioned) registry in the wild ADOPTS v1 on first open, keeping its
|
|
168
|
+
// data. Bump when a migration below changes a layout an older binary would mis-handle.
|
|
169
|
+
export const REGISTRY_SCHEMA_VERSION = 1;
|
|
170
|
+
|
|
171
|
+
/**
|
|
172
|
+
* Forward-incompatibility guard, mirroring initSchema's in schema.mjs. If a NEWER
|
|
173
|
+
* claude-mem-lite wrote this registry, this (older) build re-applying its own migrations
|
|
174
|
+
* over the newer layout corrupts it silently. Throw instead. MUST run before any DDL so
|
|
175
|
+
* a newer DB is left untouched.
|
|
176
|
+
* @param {Database} db
|
|
177
|
+
*/
|
|
178
|
+
function assertRegistryNotNewer(db) {
|
|
179
|
+
let row;
|
|
180
|
+
try {
|
|
181
|
+
row = db.prepare('SELECT version FROM schema_version LIMIT 1').get();
|
|
182
|
+
} catch {
|
|
183
|
+
return; // table absent = pre-version DB → adopted (stamped) at the end of init
|
|
184
|
+
}
|
|
185
|
+
if (row && typeof row.version === 'number' && row.version > REGISTRY_SCHEMA_VERSION) {
|
|
186
|
+
throw new Error(
|
|
187
|
+
`Registry DB schema is v${row.version} but this claude-mem-lite binary supports up to v${REGISTRY_SCHEMA_VERSION}. ` +
|
|
188
|
+
`A newer version wrote this DB; upgrade claude-mem-lite (npm i -g claude-mem-lite@latest) or point CLAUDE_MEM_DIR to a fresh directory.`
|
|
189
|
+
);
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
/**
|
|
194
|
+
* Column-drift self-heal for resources_fts. Deliberately NOT routed through the exported
|
|
195
|
+
* ensureFTS in schema.mjs: that helper creates the index without this one's `tokenize=`
|
|
196
|
+
* clause, and installs `<table>_ai/_ad/_au` triggers that would double-write alongside
|
|
197
|
+
* the res_fts_* set here. Its drift check also compares column SETS, while resources_fts
|
|
198
|
+
* is order-sensitive (registry-retriever's bm25(resources_fts, 3,3,3,2,2,1,1,1) weights
|
|
199
|
+
* by position). Mirrors ensureEventsFTS in schema.mjs, which exists for the same reason.
|
|
200
|
+
*
|
|
201
|
+
* Without this, widening FTS5_SCHEMA leaves existing DBs on the NARROW index while
|
|
202
|
+
* TRIGGERS_SCHEMA is re-exec'd from the wider list on every open — so every
|
|
203
|
+
* `INSERT INTO resources` throws "no column named X" and the write is silently lost.
|
|
204
|
+
* @param {Database} db
|
|
205
|
+
* @returns {boolean} true if the index was recreated (caller must repopulate it)
|
|
206
|
+
*/
|
|
207
|
+
function ensureResourcesFts(db) {
|
|
208
|
+
const ftsRow = db.prepare(`SELECT 1 FROM sqlite_master WHERE type='table' AND name='resources_fts'`).get();
|
|
209
|
+
let recreated = false;
|
|
210
|
+
if (ftsRow) {
|
|
211
|
+
let existingCols = [];
|
|
212
|
+
try { existingCols = db.prepare(`PRAGMA table_info(resources_fts)`).all().map(c => c.name); }
|
|
213
|
+
catch { /* unreadable → treat as drifted, recreate */ }
|
|
214
|
+
// Element-wise so additions, removals AND reordering all count as drift.
|
|
215
|
+
const drifted = existingCols.length !== FTS5_COLUMNS.length
|
|
216
|
+
|| FTS5_COLUMNS.some((c, i) => existingCols[i] !== c);
|
|
217
|
+
if (drifted) {
|
|
218
|
+
db.exec(`DROP TRIGGER IF EXISTS res_fts_insert`);
|
|
219
|
+
db.exec(`DROP TRIGGER IF EXISTS res_fts_update`);
|
|
220
|
+
db.exec(`DROP TRIGGER IF EXISTS res_fts_delete`);
|
|
221
|
+
db.exec(`DROP TABLE IF EXISTS resources_fts`);
|
|
222
|
+
recreated = true;
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
if (!ftsRow || recreated) db.exec(FTS5_SCHEMA);
|
|
226
|
+
return recreated;
|
|
227
|
+
}
|
|
228
|
+
|
|
155
229
|
// ─── Initialization ──────────────────────────────────────────────────────────
|
|
156
230
|
|
|
157
231
|
/**
|
|
@@ -173,6 +247,10 @@ export function ensureRegistryDb(dbPath) {
|
|
|
173
247
|
db.pragma('synchronous = NORMAL');
|
|
174
248
|
db.pragma('foreign_keys = ON');
|
|
175
249
|
|
|
250
|
+
// Before ANY DDL: refuse a registry written by a newer client (see assertRegistryNotNewer).
|
|
251
|
+
// Close the handle on the way out so the caller isn't left leaking an open file.
|
|
252
|
+
try { assertRegistryNotNewer(db); } catch (e) { db.close(); throw e; }
|
|
253
|
+
|
|
176
254
|
db.exec(RESOURCES_SCHEMA);
|
|
177
255
|
|
|
178
256
|
// Migrate: add missing columns to resources (single PRAGMA call for all)
|
|
@@ -243,20 +321,21 @@ export function ensureRegistryDb(dbPath) {
|
|
|
243
321
|
}
|
|
244
322
|
} catch (e) { debugCatch(e, 'resources-source-check-migration'); }
|
|
245
323
|
|
|
246
|
-
// FTS5: create if
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
// Triggers: always ensure (IF NOT EXISTS) — fixes DBs where FTS5 was created without triggers
|
|
324
|
+
// FTS5: create if absent, or drop+recreate if the indexed columns have drifted from
|
|
325
|
+
// FTS5_COLUMNS (a stale narrow index silently kills every resources write — audit P2-6).
|
|
326
|
+
const ftsRecreated = ensureResourcesFts(db);
|
|
327
|
+
// Triggers: always ensure (IF NOT EXISTS) — fixes DBs where FTS5 was created without
|
|
328
|
+
// triggers, and reinstates the set ensureResourcesFts drops on drift.
|
|
252
329
|
db.exec(TRIGGERS_SCHEMA);
|
|
253
330
|
|
|
254
331
|
// The source-CHECK migration replaced the `resources` content table out from under
|
|
255
332
|
// the external-content FTS index (content=resources), leaving resources_fts stale.
|
|
256
333
|
// Rebuild it so a later DELETE's res_fts_delete trigger doesn't throw "database disk
|
|
257
334
|
// image is malformed" against the mismatched index. Gated on the migration actually
|
|
258
|
-
// having run so we don't rebuild on every open.
|
|
259
|
-
|
|
335
|
+
// having run so we don't rebuild on every open. A drift-recreated index needs the same
|
|
336
|
+
// repopulation for the opposite reason: it starts EMPTY, so rows written before the
|
|
337
|
+
// drift would be unreachable through search forever.
|
|
338
|
+
if (resourcesRebuilt || ftsRecreated) {
|
|
260
339
|
try { db.exec("INSERT INTO resources_fts(resources_fts) VALUES('rebuild')"); }
|
|
261
340
|
catch (e) { debugCatch(e, 'resources-fts-rebuild-after-source-check'); }
|
|
262
341
|
}
|
|
@@ -352,6 +431,15 @@ export function ensureRegistryDb(dbPath) {
|
|
|
352
431
|
|
|
353
432
|
db.exec(PREINSTALLED_SCHEMA);
|
|
354
433
|
|
|
434
|
+
// Stamp the version LAST — only after every table + migration above succeeded. This is
|
|
435
|
+
// also the ADOPTION path for pre-version DBs: they are stamped in place, never wiped or
|
|
436
|
+
// refused. Rewritten each open so exactly one row exists and it reflects this build.
|
|
437
|
+
db.exec('CREATE TABLE IF NOT EXISTS schema_version (version INTEGER NOT NULL)');
|
|
438
|
+
db.transaction(() => {
|
|
439
|
+
db.exec('DELETE FROM schema_version');
|
|
440
|
+
db.prepare('INSERT INTO schema_version (version) VALUES (?)').run(REGISTRY_SCHEMA_VERSION);
|
|
441
|
+
})();
|
|
442
|
+
|
|
355
443
|
return db;
|
|
356
444
|
}
|
|
357
445
|
|
|
@@ -68,6 +68,38 @@ if (!entryArg) {
|
|
|
68
68
|
|
|
69
69
|
const entryAbs = entryArg.startsWith('/') ? entryArg : join(INSTALL_DIR, entryArg);
|
|
70
70
|
|
|
71
|
+
// Swap barrier. An auto-update / repair renames files into the install dir one at
|
|
72
|
+
// a time — atomic per file, not per file SET — so a hook process that starts
|
|
73
|
+
// mid-swap can resolve its entry from the old version and an import from the new
|
|
74
|
+
// one. hook-update.mjs marks that window; skip the fire instead of importing a
|
|
75
|
+
// mixed module graph. Hooks are best-effort and the swap lasts ~a second, so the
|
|
76
|
+
// next fire runs against a settled install.
|
|
77
|
+
//
|
|
78
|
+
// Stale-guarded on BOTH pid and ts: an updater killed mid-swap leaves the marker
|
|
79
|
+
// behind, and a marker that outlives its writer must never mute hooks permanently.
|
|
80
|
+
// Anything unreadable/torn/expired reads as "no swap" — fail-open, like the rest
|
|
81
|
+
// of this launcher. Inline (not lib/proc-lock.mjs) per the pure-node: charter.
|
|
82
|
+
const SWAP_MARKER = join(RUNTIME_DIR, 'swap-in-progress');
|
|
83
|
+
const SWAP_MAX_MS = 2 * 60 * 1000;
|
|
84
|
+
|
|
85
|
+
function swapInProgress() {
|
|
86
|
+
try {
|
|
87
|
+
const { pid, ts } = JSON.parse(readFileSync(SWAP_MARKER, 'utf8'));
|
|
88
|
+
if (typeof ts !== 'number' || Date.now() - ts > SWAP_MAX_MS) return false;
|
|
89
|
+
if (typeof pid !== 'number' || pid <= 0) return false;
|
|
90
|
+
try {
|
|
91
|
+
process.kill(pid, 0); // signal 0 = existence probe
|
|
92
|
+
return true;
|
|
93
|
+
} catch (e) {
|
|
94
|
+
return e.code === 'EPERM'; // alive, owned by another user
|
|
95
|
+
}
|
|
96
|
+
} catch {
|
|
97
|
+
return false; // no marker / unreadable / torn JSON → not a swap
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
if (swapInProgress()) process.exit(0);
|
|
102
|
+
|
|
71
103
|
async function runEntry({ bustCache = false } = {}) {
|
|
72
104
|
// Mirror direct invocation: process.argv[1] is the entry, [2..] are its args.
|
|
73
105
|
process.argv = [process.argv[0], entryAbs, ...rest];
|
package/scripts/post-tool-use.sh
CHANGED
|
@@ -7,6 +7,34 @@
|
|
|
7
7
|
# Prevent recursive hooks
|
|
8
8
|
[[ -n "$CLAUDE_MEM_HOOK_RUNNING" ]] && exit 0
|
|
9
9
|
|
|
10
|
+
# Claude Code plugin-disable guard (audit P3-4).
|
|
11
|
+
# install.mjs writes DIRECT hook entries into ~/.claude/settings.json, so disabling the
|
|
12
|
+
# plugin in the Claude UI leaves them firing. hook.mjs exits 0 for that case, but the
|
|
13
|
+
# Read fast-path below never reaches Node: it kept appending to reads-<project>.txt on
|
|
14
|
+
# EVERY Read, while the 24h sweep that reaps those files (sweepOrphanEpisodeFiles, via
|
|
15
|
+
# runSessionStartAutoMaintain) sits behind that same Node-side exit — unbounded growth
|
|
16
|
+
# in runtime/ for a plugin the user believes is off.
|
|
17
|
+
# MUST agree with hook.mjs isPluginExplicitlyDisabled(): same $HOME/.claude/settings.json
|
|
18
|
+
# (NOT CLAUDE_CONFIG_DIR — hook.mjs resolves it via homedir()), same plugin key, and the
|
|
19
|
+
# same fail-open-on-unreadable semantics (its try/catch returns false). Parity pinned by
|
|
20
|
+
# tests/post-tool-use-disabled.test.mjs.
|
|
21
|
+
# Cheap by construction: no `node`, no external command on this ~5ms per-tool-call path.
|
|
22
|
+
# `-r` covers missing/unreadable in one builtin test, and `$(<file)` slurps the whole file
|
|
23
|
+
# for ONE regex — measured +0.4ms/call vs +4.0ms for a `while read` loop over the same
|
|
24
|
+
# 239-line settings.json (bash pays a syscall + a regex per line there).
|
|
25
|
+
# The ERE (not a `==` glob) is what keeps a MINIFIED single-line settings.json from
|
|
26
|
+
# matching `"<key>": true, … "other": false` as a false positive: the pattern is anchored
|
|
27
|
+
# to the key, so only that key's own value can satisfy it.
|
|
28
|
+
# Deliberately NOT applied to the Node handoff at the tail: hook.mjs already self-guards
|
|
29
|
+
# there, so a bash false positive could only lose data, never save work.
|
|
30
|
+
_mem_settings_file="${HOME}/.claude/settings.json"
|
|
31
|
+
_mem_plugin_disabled() {
|
|
32
|
+
[[ -r "$_mem_settings_file" ]] || return 1
|
|
33
|
+
local _settings
|
|
34
|
+
_settings=$(<"$_mem_settings_file")
|
|
35
|
+
[[ "$_settings" =~ \"claude-mem-lite@sdsrss\"[[:space:]]*:[[:space:]]*false ]]
|
|
36
|
+
}
|
|
37
|
+
|
|
10
38
|
# Read stdin (tool hook JSON)
|
|
11
39
|
input=$(head -c 262144)
|
|
12
40
|
|
|
@@ -19,6 +47,10 @@ fi
|
|
|
19
47
|
|
|
20
48
|
# Read tool: track file path for episode context, then exit (no Node needed)
|
|
21
49
|
if [[ "$tool" == "Read" ]]; then
|
|
50
|
+
# Disabled plugin → write nothing; nothing would ever sweep the file (see guard above).
|
|
51
|
+
# 2>/dev/null: a settings.json unlinked between the -r test and the read must stay
|
|
52
|
+
# silent — Claude Code surfaces hook stderr.
|
|
53
|
+
_mem_plugin_disabled 2>/dev/null && exit 0
|
|
22
54
|
if [[ "$input" =~ \"file_path\"[[:space:]]*:[[:space:]]*\"([^\"]+)\" ]]; then
|
|
23
55
|
file_path="${BASH_REMATCH[1]}"
|
|
24
56
|
_dir="${CLAUDE_PROJECT_DIR:-$PWD}"
|
|
@@ -46,6 +78,13 @@ if [[ "$tool" == "Read" ]]; then
|
|
|
46
78
|
# bash fast-path wrote to $HOME unconditionally, a relocated install would drop all
|
|
47
79
|
# Read context from episodes AND grow an uncollected reads file in $HOME forever.
|
|
48
80
|
runtime_dir="${CLAUDE_MEM_DIR:-$HOME/.claude-mem-lite}/runtime"
|
|
81
|
+
# Owner-only (0700 dir / 0600 file): reads-<project>.txt lists captured file
|
|
82
|
+
# paths, so on a shared host the default umask leaked them to every local user.
|
|
83
|
+
# umask is a shell builtin — no extra process on this ~5ms per-tool-call path
|
|
84
|
+
# (a chmod would be a spawn). It only applies at creation; server.mjs
|
|
85
|
+
# hardenRuntimeFiles() remediates files that predate this fix. Scoped safely:
|
|
86
|
+
# this branch always exits before the node handoff below.
|
|
87
|
+
umask 077
|
|
49
88
|
mkdir -p "$runtime_dir" 2>/dev/null
|
|
50
89
|
# Use printf to avoid shell interpretation of special characters in file paths
|
|
51
90
|
printf '%s\n' "$file_path" >> "${runtime_dir}/reads-${project}.txt"
|
package/search-engine.mjs
CHANGED
|
@@ -478,12 +478,14 @@ export function searchObservationsHybrid(db, ctx) {
|
|
|
478
478
|
if (args.importance && (obs.importance ?? 1) < args.importance) continue;
|
|
479
479
|
if (args.branch && obs.branch !== args.branch) continue;
|
|
480
480
|
if (!includeNoise && obs.title && LOW_SIGNAL_TITLE.test(obs.title)) continue;
|
|
481
|
-
resultMap.set(vr.id, { source: 'obs', id: obs.id, type: obs.type, title: obs.title, subtitle: obs.subtitle, project: obs.project, date: obs.created_at, importance: obs.importance, files_modified: obs.files_modified, lesson_learned: obs.lesson_learned, snippet: '' });
|
|
481
|
+
resultMap.set(vr.id, { source: 'obs', id: obs.id, type: obs.type, title: obs.title, subtitle: obs.subtitle, project: obs.project, date: obs.created_at, created_at: obs.created_at, created_at_epoch: obs.created_at_epoch, importance: obs.importance, files_modified: obs.files_modified, lesson_learned: obs.lesson_learned, snippet: '' });
|
|
482
482
|
}
|
|
483
483
|
}
|
|
484
|
+
// The WHOLE obs leg is now on the RRF scale (score = -rrfScore ≈ 1/(60+rank)), not
|
|
485
|
+
// BM25 — tag every row so cross-source lone-hit banding treats it correctly (P2-12).
|
|
484
486
|
const reordered = rrfRanking
|
|
485
487
|
.filter(rr => resultMap.has(rr.id))
|
|
486
|
-
.map(rr => ({ ...resultMap.get(rr.id), score: -rr.rrfScore }));
|
|
488
|
+
.map(rr => ({ ...resultMap.get(rr.id), score: -rr.rrfScore, scoreScale: 'vector' }));
|
|
487
489
|
results.length = 0;
|
|
488
490
|
results.push(...reordered);
|
|
489
491
|
} else {
|
|
@@ -496,7 +498,8 @@ export function searchObservationsHybrid(db, ctx) {
|
|
|
496
498
|
if (args.importance && (obs.importance ?? 1) < args.importance) continue;
|
|
497
499
|
if (args.branch && obs.branch !== args.branch) continue;
|
|
498
500
|
if (!includeNoise && obs.title && LOW_SIGNAL_TITLE.test(obs.title)) continue;
|
|
499
|
-
|
|
501
|
+
// Raw cosine similarity scale (≈0.1-1), also not BM25-comparable → tag it (P2-12).
|
|
502
|
+
results.push({ source: 'obs', id: obs.id, type: obs.type, title: obs.title, subtitle: obs.subtitle, project: obs.project, date: obs.created_at, created_at: obs.created_at, created_at_epoch: obs.created_at_epoch, importance: obs.importance, files_modified: obs.files_modified, lesson_learned: obs.lesson_learned, score: -vr.similarity, snippet: '', scoreScale: 'vector' });
|
|
500
503
|
}
|
|
501
504
|
}
|
|
502
505
|
} catch (e) { debugCatch(e, 'searchObservationsHybrid-vector'); }
|
package/search-scoring.mjs
CHANGED
|
@@ -149,36 +149,57 @@ export const PRF_STOP_WORDS = new Set([
|
|
|
149
149
|
* @returns {string[]} Array of discriminative terms for query expansion
|
|
150
150
|
*/
|
|
151
151
|
export function extractPRFTerms(results, ftsQuery, limit = 3) {
|
|
152
|
-
//
|
|
152
|
+
// Query tokens to exclude from expansion — stemmed so morphological variants of a
|
|
153
|
+
// query term (e.g. "authenticate" when the user searched "authentication") are also
|
|
154
|
+
// excluded, not just the exact surface form.
|
|
153
155
|
const queryTokens = new Set(
|
|
154
156
|
ftsQuery.replace(/["()]/g, ' ').split(/\s+/)
|
|
155
157
|
.map(t => porterStem(t.toLowerCase()))
|
|
156
158
|
.filter(t => t.length > 1 && t !== 'or' && t !== 'and')
|
|
157
159
|
);
|
|
158
160
|
|
|
159
|
-
//
|
|
160
|
-
|
|
161
|
+
// Bucket morphological variants by porter STEM so "cache"/"caching"/"cached" jointly
|
|
162
|
+
// clear the ">= 2 docs" discriminativeness bar. But the SELECTED term must be a SURFACE
|
|
163
|
+
// form: the emitted terms are fed back into `observations_fts MATCH` (search-engine.mjs),
|
|
164
|
+
// and that index uses FTS5's DEFAULT unicode61 tokenizer — NO stemming (verified: MATCH
|
|
165
|
+
// "cach" returns zero rows, only "caching"/"cache" match). Emitting a bare stem would
|
|
166
|
+
// silently match nothing and kill expansion recall. Track each stem's surface forms with
|
|
167
|
+
// their occurrence counts and emit the most frequent (best-matchable) surface.
|
|
168
|
+
const stemDocCount = {}; // stem -> # of top docs it appears in (the >=2 bar)
|
|
169
|
+
const stemSurfaces = {}; // stem -> Map(surface -> total occurrences)
|
|
161
170
|
const docCount = Math.min(results.length, 8);
|
|
162
171
|
for (let i = 0; i < docCount; i++) {
|
|
163
172
|
const r = results[i];
|
|
164
173
|
const text = ((r.title || '') + ' ' + (r.narrative || '')).toLowerCase();
|
|
165
|
-
const
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
174
|
+
const surfaces = text.replace(/[^a-z0-9_-]/g, ' ').split(/\s+/).filter(t => t.length >= 3);
|
|
175
|
+
const docStems = new Set();
|
|
176
|
+
for (const surface of surfaces) {
|
|
177
|
+
// Stop-word filter at BOTH surface and stem level: PRF_STOP_WORDS lists surface
|
|
178
|
+
// inflections ("changed"/"updated"/"files") whose porter stem ("chang"/"updat") is
|
|
179
|
+
// not itself listed, so a stem-only check would let the surface through to emission.
|
|
180
|
+
if (PRF_STOP_WORDS.has(surface)) continue;
|
|
181
|
+
const stem = porterStem(surface);
|
|
182
|
+
if (stem.length < 3 || PRF_STOP_WORDS.has(stem) || queryTokens.has(stem)) continue;
|
|
183
|
+
const sm = (stemSurfaces[stem] ||= new Map());
|
|
184
|
+
sm.set(surface, (sm.get(surface) || 0) + 1);
|
|
185
|
+
docStems.add(stem);
|
|
173
186
|
}
|
|
187
|
+
for (const stem of docStems) stemDocCount[stem] = (stemDocCount[stem] || 0) + 1;
|
|
174
188
|
}
|
|
175
189
|
|
|
176
|
-
// Select
|
|
177
|
-
|
|
190
|
+
// Select stems appearing in >= 2 top docs (discriminative, not noise), emitting the
|
|
191
|
+
// most frequent surface form of each so the term actually matches the unstemmed index.
|
|
192
|
+
return Object.entries(stemDocCount)
|
|
178
193
|
.filter(([, count]) => count >= 2)
|
|
179
194
|
.sort((a, b) => b[1] - a[1])
|
|
180
195
|
.slice(0, limit)
|
|
181
|
-
.map(([
|
|
196
|
+
.map(([stem]) => {
|
|
197
|
+
let best = null, bestN = -1;
|
|
198
|
+
for (const [surface, n] of stemSurfaces[stem]) {
|
|
199
|
+
if (n > bestN) { best = surface; bestN = n; }
|
|
200
|
+
}
|
|
201
|
+
return best;
|
|
202
|
+
});
|
|
182
203
|
}
|
|
183
204
|
|
|
184
205
|
// ─── Concept Co-occurrence Query Expansion ─────────────────────────────────
|
package/server.mjs
CHANGED
|
@@ -48,6 +48,7 @@ import { saveObservation } from './lib/save-observation.mjs';
|
|
|
48
48
|
import { rebuildObservationDerived } from './lib/observation-write.mjs';
|
|
49
49
|
import { EXPORT_COLUMNS_SQL } from './lib/export-columns.mjs';
|
|
50
50
|
import { recallByFile } from './lib/recall-core.mjs';
|
|
51
|
+
import { fetchRecent } from './lib/recent-core.mjs';
|
|
51
52
|
import { AUTO_MERGE_THRESHOLD } from './lib/dedup-constants.mjs';
|
|
52
53
|
import {
|
|
53
54
|
insertDeferred, listOpenWithOrdinal, dropDeferred,
|
|
@@ -65,7 +66,7 @@ const { version: PKG_VERSION } = require('./package.json');
|
|
|
65
66
|
|
|
66
67
|
// ─── Database ───────────────────────────────────────────────────────────────
|
|
67
68
|
|
|
68
|
-
import { rmSync, existsSync, readFileSync, appendFileSync, mkdirSync } from 'fs';
|
|
69
|
+
import { rmSync, existsSync, readFileSync, writeFileSync, appendFileSync, mkdirSync, readdirSync, chmodSync } from 'fs';
|
|
69
70
|
|
|
70
71
|
let db;
|
|
71
72
|
try {
|
|
@@ -404,27 +405,19 @@ async function runRecent(db, args) {
|
|
|
404
405
|
const limit = args.limit ?? 10;
|
|
405
406
|
const project = args.project || inferProject();
|
|
406
407
|
|
|
407
|
-
const params = [];
|
|
408
|
-
const wheres = ['COALESCE(compressed_into, 0) = 0', 'superseded_at IS NULL'];
|
|
409
|
-
if (project) { wheres.push('project = ?'); params.push(project); }
|
|
410
|
-
// obs_type: observation-type filter (CLI `recent --type` parity). The zod enum
|
|
411
|
-
// already rejected an invalid value before the handler ran.
|
|
412
|
-
if (args.obs_type) { wheres.push('type = ?'); params.push(args.obs_type); }
|
|
413
408
|
// date_since: relative lower bound on created_at (CLI `recent --since` parity).
|
|
409
|
+
// Parsed here rather than in the core because the surfaces reject a bad duration
|
|
410
|
+
// in their own dialect — MCP throws, CLI fail()s.
|
|
411
|
+
let since = null;
|
|
414
412
|
if (args.date_since !== undefined) {
|
|
415
413
|
const d = parseDuration(args.date_since);
|
|
416
414
|
if (!d.ok) throw new Error(`Invalid date_since: "${args.date_since}" (use <N><unit>, e.g. 7d, 24h, 90m, 2w)`);
|
|
417
|
-
|
|
415
|
+
since = Date.now() - d.ms;
|
|
418
416
|
}
|
|
419
|
-
params.push(limit);
|
|
420
417
|
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
WHERE ${wheres.join(' AND ')}
|
|
425
|
-
ORDER BY created_at_epoch DESC
|
|
426
|
-
LIMIT ?
|
|
427
|
-
`).all(...params);
|
|
418
|
+
// Shared core with CLI `recent`: live-rows filter + ordering (lib/recent-core.mjs).
|
|
419
|
+
// obs_type is already enum-validated by zod before the handler runs.
|
|
420
|
+
const rows = fetchRecent(db, { project, type: args.obs_type || null, since, limit });
|
|
428
421
|
|
|
429
422
|
if (rows.length === 0) {
|
|
430
423
|
return { content: [{ type: 'text', text: `No recent observations${project ? ` (${project})` : ''}.` }] };
|
|
@@ -745,6 +738,34 @@ server.registerTool(
|
|
|
745
738
|
|
|
746
739
|
// ─── Tool: mem_save ─────────────────────────────────────────────────────────
|
|
747
740
|
|
|
741
|
+
// Sanity ceilings on the free-text fields that reach unbounded TEXT columns. zod
|
|
742
|
+
// validates TYPES; before this, `title` had no size bound at all
|
|
743
|
+
// (`z.string().optional()`), so one runaway summarizer write stored a multi-KB title
|
|
744
|
+
// verbatim and every later FTS rebuild, minhash and vector recompute paid for that row
|
|
745
|
+
// forever. Truncate rather than reject: rejection would be a breaking contract change
|
|
746
|
+
// on a published MCP tool, and a caller that over-writes still wants its memory saved.
|
|
747
|
+
//
|
|
748
|
+
// Numbers: `title` at 500 is ~5× the longest form any read path renders (titles are
|
|
749
|
+
// truncate()'d to 80–100 chars on display) and 5× saveObservation's own
|
|
750
|
+
// content.slice(0, 100) fallback — generous headroom, still a bound. `content` (50000)
|
|
751
|
+
// and `lesson_learned` (500) mirror the zod maxima already published in
|
|
752
|
+
// tool-schemas.mjs, so the advertised contract is not narrowed; applying them here
|
|
753
|
+
// converts a hard zod REJECT into graceful truncation for any caller that reaches this
|
|
754
|
+
// path without schema validation.
|
|
755
|
+
export const SAVE_TEXT_LIMITS = { content: 50000, title: 500, lesson_learned: 500 };
|
|
756
|
+
|
|
757
|
+
/**
|
|
758
|
+
* Cap one free-text field at `limit`, marking the loss in-band so the truncation is
|
|
759
|
+
* visible in the stored row (a silently shortened memory is worse than a marked one).
|
|
760
|
+
* The marker names the ORIGINAL length, and the result never exceeds `limit`.
|
|
761
|
+
* Non-strings (absent optional fields) pass through untouched.
|
|
762
|
+
*/
|
|
763
|
+
export function clampSaveText(value, limit) {
|
|
764
|
+
if (typeof value !== 'string' || value.length <= limit) return value;
|
|
765
|
+
const marker = ` … [truncated from ${value.length} chars]`;
|
|
766
|
+
return value.slice(0, Math.max(0, limit - marker.length)) + marker;
|
|
767
|
+
}
|
|
768
|
+
|
|
748
769
|
server.registerTool(
|
|
749
770
|
'mem_save',
|
|
750
771
|
{
|
|
@@ -760,13 +781,15 @@ server.registerTool(
|
|
|
760
781
|
try {
|
|
761
782
|
result = db.transaction(() => {
|
|
762
783
|
const r = saveObservation(db, {
|
|
763
|
-
|
|
764
|
-
|
|
784
|
+
// Size ceiling applied here rather than in lib/save-observation.mjs so the
|
|
785
|
+
// shared pipeline keeps its "caller validates" contract (see its header).
|
|
786
|
+
content: clampSaveText(args.content, SAVE_TEXT_LIMITS.content),
|
|
787
|
+
title: clampSaveText(args.title, SAVE_TEXT_LIMITS.title),
|
|
765
788
|
type: args.type || 'discovery',
|
|
766
789
|
importance: args.importance,
|
|
767
790
|
project,
|
|
768
791
|
files: args.files || [],
|
|
769
|
-
lesson_learned: args.lesson_learned,
|
|
792
|
+
lesson_learned: clampSaveText(args.lesson_learned, SAVE_TEXT_LIMITS.lesson_learned),
|
|
770
793
|
supersedes: args.supersedes,
|
|
771
794
|
});
|
|
772
795
|
if (r.kind === 'duplicate') return r; // dedup short-circuits BEFORE resolver — replay is idempotent
|
|
@@ -1890,6 +1913,69 @@ process.on('SIGTERM', () => shutdown(0));
|
|
|
1890
1913
|
process.on('uncaughtException', (err) => { debugCatch(err, 'uncaughtException'); shutdown(1); });
|
|
1891
1914
|
process.on('unhandledRejection', (err) => { debugCatch(err, 'unhandledRejection'); shutdown(1); });
|
|
1892
1915
|
|
|
1916
|
+
// ─── Runtime Dir Retention + Permissions ────────────────────────────────────
|
|
1917
|
+
|
|
1918
|
+
/** Spawn-log retention window — same 14 days as every sibling JSONL sink. */
|
|
1919
|
+
export const SPAWN_LOG_RETENTION_MS = 14 * 24 * 60 * 60 * 1000;
|
|
1920
|
+
/** Hard ceiling for pathological spawn churn inside the retention window. */
|
|
1921
|
+
export const SPAWN_LOG_MAX_LINES = 2000;
|
|
1922
|
+
|
|
1923
|
+
/**
|
|
1924
|
+
* Trim mcp-spawns.log to the retention window. The sibling sinks (lib/metrics,
|
|
1925
|
+
* lib/err-sampler, lib/hook-telemetry) shard per day and unlink old shards; this
|
|
1926
|
+
* sink is one flat file at a documented path, so the same 14-day window is applied
|
|
1927
|
+
* line-wise instead. Records without a parseable `ts` are dropped — the log is
|
|
1928
|
+
* append-only JSONL, so a malformed line is truncated telemetry, not data.
|
|
1929
|
+
* Only rewrites when something actually expires. Never throws.
|
|
1930
|
+
*
|
|
1931
|
+
* @param {string} path Absolute path to mcp-spawns.log
|
|
1932
|
+
* @param {number} [nowMs]
|
|
1933
|
+
* @returns {number} lines removed
|
|
1934
|
+
*/
|
|
1935
|
+
export function pruneSpawnLog(path, nowMs = Date.now()) {
|
|
1936
|
+
try {
|
|
1937
|
+
if (!existsSync(path)) return 0;
|
|
1938
|
+
const lines = readFileSync(path, 'utf8').split('\n').filter(Boolean);
|
|
1939
|
+
// ISO-8601 sorts lexically = chronologically, so no Date.parse per line
|
|
1940
|
+
// (same trick as hook-telemetry.countRecentHookErrors).
|
|
1941
|
+
const cutoffIso = new Date(nowMs - SPAWN_LOG_RETENTION_MS).toISOString();
|
|
1942
|
+
let kept = lines.filter((l) => {
|
|
1943
|
+
try {
|
|
1944
|
+
const { ts } = JSON.parse(l);
|
|
1945
|
+
return typeof ts === 'string' && ts >= cutoffIso;
|
|
1946
|
+
} catch { return false; }
|
|
1947
|
+
});
|
|
1948
|
+
if (kept.length > SPAWN_LOG_MAX_LINES) kept = kept.slice(-SPAWN_LOG_MAX_LINES);
|
|
1949
|
+
if (kept.length === lines.length) return 0;
|
|
1950
|
+
writeFileSync(path, kept.length ? kept.join('\n') + '\n' : '', { mode: 0o600 });
|
|
1951
|
+
return lines.length - kept.length;
|
|
1952
|
+
} catch { return 0; }
|
|
1953
|
+
}
|
|
1954
|
+
|
|
1955
|
+
/**
|
|
1956
|
+
* Restrict the runtime dir and its files to owner-only. Directory 0700 is the
|
|
1957
|
+
* load-bearing part (without +x another local user cannot traverse in); the
|
|
1958
|
+
* per-file 0600 is defense in depth and remediates files created before this fix.
|
|
1959
|
+
* Subdirectories are skipped — they are created with `{mode: 0o700}` by their own
|
|
1960
|
+
* sinks. Symlinks are skipped (Dirent.isFile() is false) so this never chmods
|
|
1961
|
+
* through to a target outside the runtime dir. Idempotent, never throws.
|
|
1962
|
+
*
|
|
1963
|
+
* @param {string} dir Absolute path to RUNTIME_DIR
|
|
1964
|
+
* @returns {number} files chmod'ed
|
|
1965
|
+
*/
|
|
1966
|
+
export function hardenRuntimeFiles(dir) {
|
|
1967
|
+
try {
|
|
1968
|
+
if (!existsSync(dir)) return 0;
|
|
1969
|
+
chmodSync(dir, 0o700);
|
|
1970
|
+
let touched = 0;
|
|
1971
|
+
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
|
1972
|
+
if (!entry.isFile()) continue;
|
|
1973
|
+
try { chmodSync(join(dir, entry.name), 0o600); touched++; } catch { /* per-entry, silent */ }
|
|
1974
|
+
}
|
|
1975
|
+
return touched;
|
|
1976
|
+
} catch { return 0; }
|
|
1977
|
+
}
|
|
1978
|
+
|
|
1893
1979
|
// ─── Start Server ───────────────────────────────────────────────────────────
|
|
1894
1980
|
|
|
1895
1981
|
// Spawn telemetry — appends one JSON line per process start so we can diagnose
|
|
@@ -1898,14 +1984,13 @@ process.on('unhandledRejection', (err) => { debugCatch(err, 'unhandledRejection'
|
|
|
1898
1984
|
// the same ppid is the smoking gun. Never throws — telemetry must not block
|
|
1899
1985
|
// startup. Disable with MEM_DISABLE_SPAWN_LOG=1.
|
|
1900
1986
|
//
|
|
1901
|
-
// File mode:
|
|
1902
|
-
//
|
|
1903
|
-
//
|
|
1904
|
-
//
|
|
1905
|
-
// comment > misleading code.
|
|
1987
|
+
// File mode: `{mode: 0o600}` on appendFileSync only applies on file creation
|
|
1988
|
+
// (umask-default after the first append) — pre-v2.79.1 relied on it alone, so
|
|
1989
|
+
// the "0600" claim was misleading. hardenRuntimeFiles below sets the mode for
|
|
1990
|
+
// real, on every start, including logs created before this fix.
|
|
1906
1991
|
if (process.env.MEM_DISABLE_SPAWN_LOG !== '1') {
|
|
1907
1992
|
try {
|
|
1908
|
-
if (!existsSync(RUNTIME_DIR)) mkdirSync(RUNTIME_DIR, { recursive: true });
|
|
1993
|
+
if (!existsSync(RUNTIME_DIR)) mkdirSync(RUNTIME_DIR, { recursive: true, mode: 0o700 });
|
|
1909
1994
|
const line = JSON.stringify({
|
|
1910
1995
|
ts: new Date().toISOString(),
|
|
1911
1996
|
pid: process.pid,
|
|
@@ -1913,9 +1998,20 @@ if (process.env.MEM_DISABLE_SPAWN_LOG !== '1') {
|
|
|
1913
1998
|
argv1: process.argv[1] || '',
|
|
1914
1999
|
version: PKG_VERSION,
|
|
1915
2000
|
}) + '\n';
|
|
1916
|
-
|
|
2001
|
+
const spawnLog = join(RUNTIME_DIR, 'mcp-spawns.log');
|
|
2002
|
+
appendFileSync(spawnLog, line, { mode: 0o600 });
|
|
2003
|
+
// Lazy GC on append, mirroring lib/hook-telemetry.pruneOldShards. Single-file
|
|
2004
|
+
// sink (not day-sharded) so retention is applied line-wise instead.
|
|
2005
|
+
pruneSpawnLog(spawnLog);
|
|
1917
2006
|
} catch { /* never block startup on telemetry failure */ }
|
|
1918
2007
|
}
|
|
1919
2008
|
|
|
2009
|
+
// Owner-only for the whole runtime dir. Sibling aux files carry captured file
|
|
2010
|
+
// paths (reads-<project>.txt) and scrubbed activity (ep-<project>.json), but were
|
|
2011
|
+
// written at the default umask (0644) while the DB itself is 0600 / DB_DIR 0700
|
|
2012
|
+
// (schema.mjs). The hooks that create them set the mode at creation; this sweep
|
|
2013
|
+
// is what remediates files already on disk from before the fix.
|
|
2014
|
+
hardenRuntimeFiles(RUNTIME_DIR);
|
|
2015
|
+
|
|
1920
2016
|
const transport = new StdioServerTransport();
|
|
1921
2017
|
await server.connect(transport);
|
package/source-files.mjs
CHANGED
|
@@ -136,6 +136,10 @@ export const SOURCE_FILES = [
|
|
|
136
136
|
// `timeline`/`search` and mem_timeline/mem_search on auto-update.
|
|
137
137
|
'lib/timeline-core.mjs',
|
|
138
138
|
'lib/search-core.mjs',
|
|
139
|
+
// Shared `recent` core (live-row filter + ORDER BY created_at DESC LIMIT).
|
|
140
|
+
// Statically imported by mem-cli.mjs AND server.mjs — same single-source pattern;
|
|
141
|
+
// missing it from the manifest would break `recent` and mem_recent on auto-update.
|
|
142
|
+
'lib/recent-core.mjs',
|
|
139
143
|
// Reciprocal Rank Fusion core (D#42 single source-of-truth); transitively
|
|
140
144
|
// reached via tfidf.mjs (rrfMerge) and deep-search.mjs (rrfFuseN).
|
|
141
145
|
'lib/rrf.mjs',
|