claude-mem-lite 3.16.3 → 3.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +1 -1
- package/deep-search.mjs +7 -22
- package/hook-context.mjs +4 -1
- package/hook-llm.mjs +8 -1
- package/hook-memory.mjs +6 -2
- package/hook-semaphore.mjs +9 -2
- package/lib/maintain-core.mjs +2 -1
- package/lib/observation-write.mjs +2 -1
- package/lib/rrf.mjs +42 -0
- package/mem-cli.mjs +34 -2
- package/package.json +2 -1
- package/search-engine.mjs +2 -1
- package/server.mjs +12 -1
- package/source-files.mjs +3 -0
- package/tfidf.mjs +29 -10
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
"plugins": [
|
|
11
11
|
{
|
|
12
12
|
"name": "claude-mem-lite",
|
|
13
|
-
"version": "3.
|
|
13
|
+
"version": "3.17.0",
|
|
14
14
|
"source": "./",
|
|
15
15
|
"description": "Persistent long-term memory for Claude Code via MCP — captures coding decisions, bugfixes, and context across sessions. Hybrid FTS5 + TF-IDF search with episode batching. Single SQLite DB, no external services. A lighter, lower-cost alternative to claude-mem (episode batching + a smaller model; cost savings are an internal estimate, not a measured benchmark)."
|
|
16
16
|
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "claude-mem-lite",
|
|
3
|
-
"version": "3.
|
|
3
|
+
"version": "3.17.0",
|
|
4
4
|
"description": "Persistent long-term memory for Claude Code via MCP — captures coding decisions, bugfixes, and context across sessions. Hybrid FTS5 + TF-IDF search with episode batching. Single SQLite DB, no external services. A lighter, lower-cost alternative to claude-mem (episode batching + a smaller model; cost savings are an internal estimate, not a measured benchmark).",
|
|
5
5
|
"author": {
|
|
6
6
|
"name": "sdsrss"
|
package/deep-search.mjs
CHANGED
|
@@ -34,6 +34,7 @@
|
|
|
34
34
|
import { searchObservationsHybrid } from './search-engine.mjs';
|
|
35
35
|
import { sanitizeFtsQuery } from './utils.mjs';
|
|
36
36
|
import { RRF_K } from './tfidf.mjs';
|
|
37
|
+
import { rrfAccumulate } from './lib/rrf.mjs';
|
|
37
38
|
import { llmRerankOrder, defaultRerankLLM } from './rerank.mjs';
|
|
38
39
|
|
|
39
40
|
// original + up to 3 rewrites (keyword / concept-expansion / HyDE).
|
|
@@ -350,28 +351,12 @@ export async function rewriteQuery(query, { llm = defaultLLM, retries = 1, cache
|
|
|
350
351
|
* match the hybrid path's convention) plus an rrfScore field.
|
|
351
352
|
*/
|
|
352
353
|
export function rrfFuseN(rankedLists, k = RRF_K) {
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
const prev = scores.get(r.id);
|
|
360
|
-
if (prev) {
|
|
361
|
-
prev.score += add;
|
|
362
|
-
// Keep the row from the variant that ranked this id HIGHEST (lowest
|
|
363
|
-
// index). searchObservationsHybrid emits query-dependent fields per
|
|
364
|
-
// variant (notably the FTS snippet), so first-seen would often show the
|
|
365
|
-
// weaker original/keyword variant's context; the best-ranked appearance
|
|
366
|
-
// carries the most relevant snippet/match context (F10).
|
|
367
|
-
if (i < prev.bestRank) { prev.row = r; prev.bestRank = i; }
|
|
368
|
-
} else {
|
|
369
|
-
scores.set(r.id, { row: r, score: add, bestRank: i });
|
|
370
|
-
}
|
|
371
|
-
});
|
|
372
|
-
}
|
|
373
|
-
return [...scores.values()]
|
|
374
|
-
.sort((a, b) => b.score - a.score)
|
|
354
|
+
// Thin N-list adapter over the shared RRF core (lib/rrf.mjs). Emits full source
|
|
355
|
+
// rows with score = -rrfScore (negative = better, matching the hybrid path's
|
|
356
|
+
// convention) plus an rrfScore field. rrfAccumulate already keeps each id's
|
|
357
|
+
// best-ranked row, so query-dependent fields (notably the FTS snippet) come from
|
|
358
|
+
// the strongest variant rather than first-seen (F10).
|
|
359
|
+
return rrfAccumulate(rankedLists, k)
|
|
375
360
|
.map(({ row, score }) => ({ ...row, score: -score, rrfScore: score }));
|
|
376
361
|
}
|
|
377
362
|
|
package/hook-context.mjs
CHANGED
|
@@ -244,7 +244,10 @@ export function cleanupClaudeMdLegacyBlock() {
|
|
|
244
244
|
|
|
245
245
|
if (normalized === content) { dropMarker(); return; }
|
|
246
246
|
|
|
247
|
-
|
|
247
|
+
// Per-pid temp suffix so two concurrent first-run SessionStarts in the same
|
|
248
|
+
// project (e.g. two terminals) can't rename each other's half-written temp
|
|
249
|
+
// onto the user's tracked CLAUDE.md. Matches the idiom in hook-shared.mjs.
|
|
250
|
+
const tmp = claudeMdPath + `.mem-tmp-${process.pid}`;
|
|
248
251
|
try {
|
|
249
252
|
writeFileSync(tmp, normalized);
|
|
250
253
|
renameSync(tmp, claudeMdPath);
|
package/hook-llm.mjs
CHANGED
|
@@ -740,9 +740,15 @@ ${actionList}`;
|
|
|
740
740
|
(parsed.type === 'bugfix' || parsed.type === 'decision') &&
|
|
741
741
|
!process.env.CLAUDE_MEM_NO_LESSON_RETRY) {
|
|
742
742
|
retryAttempted = true;
|
|
743
|
+
// The first callLLM released its slot in the finally above; this lesson
|
|
744
|
+
// retry is a SECOND LLM call and must re-acquire the semaphore or it
|
|
745
|
+
// bypasses LLM_SEM_MAX — a burst of bugfix/decision episodes would otherwise
|
|
746
|
+
// spawn unbounded concurrent Haiku calls. Under contention we skip the retry
|
|
747
|
+
// rather than exceed the limit (the lesson is an optional enhancement).
|
|
748
|
+
const retrySlot = await acquireLLMSlot();
|
|
743
749
|
try {
|
|
744
750
|
const retryPrompt = buildLessonRetryPrompt(episode, parsed);
|
|
745
|
-
const retryRaw = await callLLM(retryPrompt, 10000);
|
|
751
|
+
const retryRaw = retrySlot ? await callLLM(retryPrompt, 10000) : null;
|
|
746
752
|
if (retryRaw) {
|
|
747
753
|
const retry = parseJsonFromLLM(retryRaw);
|
|
748
754
|
const retryLesson = typeof retry?.lesson === 'string' ? retry.lesson.trim() : '';
|
|
@@ -754,6 +760,7 @@ ${actionList}`;
|
|
|
754
760
|
}
|
|
755
761
|
}
|
|
756
762
|
} catch (e) { debugCatch(e, 'lesson-retry'); }
|
|
763
|
+
finally { if (retrySlot) releaseLLMSlot(); }
|
|
757
764
|
}
|
|
758
765
|
// v2.57.x B2: persist retry outcome counters. The retry path costs
|
|
759
766
|
// 1 extra Haiku call per bugfix/decision episode; if recovered/attempts
|
package/hook-memory.mjs
CHANGED
|
@@ -222,7 +222,7 @@ export function searchRelevantMemories(db, userPrompt, project, excludeIds = [])
|
|
|
222
222
|
// Count original search terms (AND-separated groups), not expanded synonym tokens.
|
|
223
223
|
const queryTokenCount = ftsQuery.includes(' AND ')
|
|
224
224
|
? ftsQuery.split(' AND ').length
|
|
225
|
-
: ftsQuery.split(/\s+/).filter(t => t && !t.startsWith('(')
|
|
225
|
+
: ftsQuery.split(/\s+/).filter(t => t && !t.startsWith('(') && !t.endsWith(')')).length;
|
|
226
226
|
// CJK-dominant queries bypass the token-count gate: a single CJK word becomes
|
|
227
227
|
// 2-N overlapping bigrams (优化召回率 → 优化/召回/回率), inflating
|
|
228
228
|
// queryTokenCount past the gate, so the AND-too-strict query never gets the OR
|
|
@@ -385,7 +385,11 @@ export function recallForFile(db, filePath, project) {
|
|
|
385
385
|
`).all(project, cutoff, filePath, likePattern, MAX_FILE_RECALL);
|
|
386
386
|
const now = Date.now();
|
|
387
387
|
const updateStmt = db.prepare('UPDATE observations SET access_count = COALESCE(access_count, 0) + 1, last_accessed_at = ? WHERE id = ?');
|
|
388
|
-
for
|
|
388
|
+
// Per-row try/catch for FTS trigger safety — mirror the injection-bump loop
|
|
389
|
+
// (searchRelevantMemories) and project_non_obvious.md. Without it, one
|
|
390
|
+
// SQLITE_CORRUPT_VTAB on the access_count UPDATE trigger throws to the outer
|
|
391
|
+
// catch and discards the ENTIRE file-recall result set.
|
|
392
|
+
for (const r of rows) { try { updateStmt.run(now, r.id); } catch {} }
|
|
389
393
|
return rows;
|
|
390
394
|
} catch (e) {
|
|
391
395
|
debugCatch(e, 'recallForFile');
|
package/hook-semaphore.mjs
CHANGED
|
@@ -33,8 +33,15 @@ export async function acquireLLMSlot() {
|
|
|
33
33
|
if (fd !== undefined) closeSync(fd);
|
|
34
34
|
}
|
|
35
35
|
} catch {
|
|
36
|
-
//
|
|
37
|
-
//
|
|
36
|
+
// Our own pid-named slot file already exists: a leftover from a prior acquire
|
|
37
|
+
// that never released (crash between acquire and releaseLLMSlot, or PID reuse).
|
|
38
|
+
// We are inside acquire and therefore do NOT currently hold a slot, so it is
|
|
39
|
+
// always stale — remove it and retry. The await is essential: a bare `continue`
|
|
40
|
+
// here re-hits the same EEXIST every iteration, a synchronous tight loop that
|
|
41
|
+
// pins a core until the 30s deadline (the age-based cleanup below is unreachable
|
|
42
|
+
// on this path — it only runs after a successful create).
|
|
43
|
+
try { unlinkSync(slotFile); } catch {}
|
|
44
|
+
await sleepMs(50 + Math.random() * 100);
|
|
38
45
|
continue;
|
|
39
46
|
}
|
|
40
47
|
|
package/lib/maintain-core.mjs
CHANGED
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
// { projectFilter: 'AND project = ?' | '', baseParams: [project?] , staleAge, opCap }
|
|
14
14
|
|
|
15
15
|
import { COMPRESSED_PENDING_PURGE, computeMinHash, estimateJaccardFromMinHash, jaccardSimilarity } from '../utils.mjs';
|
|
16
|
-
import { rebuildVocabulary, computeVector, _resetVocabCache } from '../tfidf.mjs';
|
|
16
|
+
import { rebuildVocabulary, computeVector, _resetVocabCache, vectorsEnabled } from '../tfidf.mjs';
|
|
17
17
|
import { DEDUP_JACCARD_THRESHOLD, MINHASH_PRE_THRESHOLD as MINHASH_PRE_THRESHOLD_SRC, FUZZY_DEDUP_THRESHOLD, FUZZY_BODY_THRESHOLD, MINHASH_PREFILTER } from './dedup-constants.mjs';
|
|
18
18
|
|
|
19
19
|
export const STALE_AGE_MS = 30 * 86400000;
|
|
@@ -326,6 +326,7 @@ export function maintenanceStats(db, { projectFilter, baseParams, staleAge }) {
|
|
|
326
326
|
|
|
327
327
|
/** Rebuild the TF-IDF vocabulary + every active observation vector (own transaction). */
|
|
328
328
|
export function rebuildVectors(db) {
|
|
329
|
+
if (!vectorsEnabled()) return { ok: false, reason: 'vector arm disabled (set CLAUDE_MEM_VECTORS=1 to re-enable)', updated: 0, total: 0 };
|
|
329
330
|
_resetVocabCache();
|
|
330
331
|
const vocab = rebuildVocabulary(db);
|
|
331
332
|
if (!vocab) return { ok: false, reason: 'no observations to build vocabulary from' };
|
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
// Statement-only: callers own the transaction boundary (both wrap the row + files
|
|
9
9
|
// + vector writes in one db.transaction so a failure can't leave a partial row).
|
|
10
10
|
|
|
11
|
-
import { getVocabulary, computeVector } from '../tfidf.mjs';
|
|
11
|
+
import { getVocabulary, computeVector, vectorsEnabled } from '../tfidf.mjs';
|
|
12
12
|
import { debugCatch, cjkBigrams } from '../utils.mjs';
|
|
13
13
|
|
|
14
14
|
// Canonical column order — must mirror the observations schema (schema.mjs).
|
|
@@ -56,6 +56,7 @@ export function insertObservationFiles(db, obsId, files) {
|
|
|
56
56
|
* observation over a missing vector).
|
|
57
57
|
*/
|
|
58
58
|
export function insertObservationVector(db, obsId, vecText) {
|
|
59
|
+
if (!vectorsEnabled()) return; // Phase-1: vector arm disabled by default (audit 2026-06-27)
|
|
59
60
|
try {
|
|
60
61
|
const vocab = getVocabulary(db);
|
|
61
62
|
if (!vocab) return;
|
package/lib/rrf.mjs
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
// lib/rrf.mjs — Reciprocal Rank Fusion core (single source of truth, D#42).
|
|
2
|
+
//
|
|
3
|
+
// Both tfidf.rrfMerge (2-list, minimal { id, rrfScore } output) and
|
|
4
|
+
// deep-search.rrfFuseN (N-list, full-row output with best-rank row selection)
|
|
5
|
+
// are thin shape-adapters over rrfAccumulate below. Keeping the scoring math in
|
|
6
|
+
// one dependency-free place prevents the two from silently drifting (different k
|
|
7
|
+
// or formula — the prior state where each had its own hand-written loop). Callers
|
|
8
|
+
// pass their own RRF_K default; 60 is the canonical value.
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Core Reciprocal Rank Fusion accumulator. For each id, sums 1/(k + rank + 1)
|
|
12
|
+
* across every ranked list it appears in, and keeps the source row from the list
|
|
13
|
+
* where the id ranked highest (lowest index) — query-dependent fields like an FTS
|
|
14
|
+
* snippet then come from the best-ranked appearance. Callers that only need the id
|
|
15
|
+
* ignore the row.
|
|
16
|
+
*
|
|
17
|
+
* @param {Array<Array<{id:any}>>} rankedLists one entry per ranked list; each list
|
|
18
|
+
* is ordered best-first. Non-array entries and rows with null/undefined id are skipped.
|
|
19
|
+
* @param {number} [k=60] RRF constant.
|
|
20
|
+
* @returns {Array<{ id:any, row:object, score:number }>} descending by fused score;
|
|
21
|
+
* ties preserve first-list-first insertion order (stable sort).
|
|
22
|
+
*/
|
|
23
|
+
export function rrfAccumulate(rankedLists, k = 60) {
|
|
24
|
+
const scores = new Map();
|
|
25
|
+
for (const list of rankedLists) {
|
|
26
|
+
if (!Array.isArray(list)) continue;
|
|
27
|
+
list.forEach((r, i) => {
|
|
28
|
+
if (!r || r.id === undefined || r.id === null) return;
|
|
29
|
+
const add = 1 / (k + i + 1);
|
|
30
|
+
const prev = scores.get(r.id);
|
|
31
|
+
if (prev) {
|
|
32
|
+
prev.score += add;
|
|
33
|
+
if (i < prev.bestRank) { prev.row = r; prev.bestRank = i; }
|
|
34
|
+
} else {
|
|
35
|
+
scores.set(r.id, { row: r, score: add, bestRank: i });
|
|
36
|
+
}
|
|
37
|
+
});
|
|
38
|
+
}
|
|
39
|
+
return [...scores.values()]
|
|
40
|
+
.sort((a, b) => b.score - a.score)
|
|
41
|
+
.map(({ row, score }) => ({ id: row.id, row, score }));
|
|
42
|
+
}
|
package/mem-cli.mjs
CHANGED
|
@@ -15,6 +15,7 @@ import { ensureRegistryDb, upsertResource } from './registry.mjs';
|
|
|
15
15
|
import { searchResources } from './registry-retriever.mjs';
|
|
16
16
|
import { computeFunnel, formatFunnel, computeSweep, formatSweep, DEFAULT_SWEEP_FLOORS, DEFAULT_SWEEP_MARGINS } from './registry-recommend.mjs';
|
|
17
17
|
import { selectCompressionCandidates, groupByProjectWeek, compressGroup } from './lib/compress-core.mjs';
|
|
18
|
+
import { buildNotLowSignalSql } from './lib/low-signal-patterns.mjs';
|
|
18
19
|
import {
|
|
19
20
|
cleanupBroken, decayAndMarkIdle, boostAccessed, demotePinned, mergeDuplicates,
|
|
20
21
|
purgeStale, purgeStalePreview, findDuplicates, maintenanceStats, rebuildVectors, vacuum,
|
|
@@ -1098,6 +1099,18 @@ async function cmdStats(db, args) {
|
|
|
1098
1099
|
AND created_at_epoch < ? ${projectFilter}
|
|
1099
1100
|
`).get(thirtyDaysAgo, ...baseParams);
|
|
1100
1101
|
const noiseRatio = obsTotal.c > 0 ? lowVal.c / obsTotal.c : 0;
|
|
1102
|
+
// Low-signal-title population: template / tool-log titles (Modified, Worked on,
|
|
1103
|
+
// Error while working, Error:, node/npm/npx …) that the retrieval layer already
|
|
1104
|
+
// filters out by default. The imp=1 "Low-value" metric above structurally can't
|
|
1105
|
+
// see these — they often carry inflated importance and recent access — so the
|
|
1106
|
+
// health gauge under-reports real noise without this line. Same LOW_SIGNAL
|
|
1107
|
+
// pattern source as the read-side filter (lib/low-signal-patterns.mjs).
|
|
1108
|
+
const lowSignalTitle = db.prepare(`
|
|
1109
|
+
SELECT COUNT(*) as c FROM observations
|
|
1110
|
+
WHERE NOT ${buildNotLowSignalSql()}
|
|
1111
|
+
AND COALESCE(compressed_into, 0) = 0 ${projectFilter}
|
|
1112
|
+
`).get(...baseParams);
|
|
1113
|
+
const lowSignalRatio = obsTotal.c > 0 ? lowSignalTitle.c / obsTotal.c : 0;
|
|
1101
1114
|
const compressedCount = db.prepare(
|
|
1102
1115
|
`SELECT COUNT(*) as c FROM observations WHERE compressed_into IS NOT NULL ${projectFilter}`
|
|
1103
1116
|
).get(...baseParams);
|
|
@@ -1143,6 +1156,8 @@ async function cmdStats(db, args) {
|
|
|
1143
1156
|
avg_importance: Number((avgImp.v ?? 1).toFixed(2)),
|
|
1144
1157
|
low_value_count: lowVal.c,
|
|
1145
1158
|
noise_ratio: Number(noiseRatio.toFixed(4)),
|
|
1159
|
+
low_signal_titles: lowSignalTitle.c,
|
|
1160
|
+
low_signal_ratio: Number(lowSignalRatio.toFixed(4)),
|
|
1146
1161
|
compressed: compressedCount.c,
|
|
1147
1162
|
superseded_only: supersededOnlyCount.c,
|
|
1148
1163
|
hook_errors_24h: hookErrors24h,
|
|
@@ -1179,6 +1194,7 @@ async function cmdStats(db, args) {
|
|
|
1179
1194
|
out(` Est. tokens: ${tokenEst.t ?? 0}`);
|
|
1180
1195
|
out(` Avg importance: ${(avgImp.v ?? 1).toFixed(2)}`);
|
|
1181
1196
|
out(` Low-value (imp=1, never accessed, >30d): ${lowVal.c} (${(noiseRatio * 100).toFixed(1)}% noise)`);
|
|
1197
|
+
out(` Low-signal titles (Modified/Error/Worked on…): ${lowSignalTitle.c} (${(lowSignalRatio * 100).toFixed(1)}%)`);
|
|
1182
1198
|
out(` Compressed: ${compressedCount.c}`);
|
|
1183
1199
|
out(` Hook errors (last 24h): ${hookErrors24h}${hookErrors24h > 0 ? ` ← tail ${join(DB_DIR, 'runtime/hook-errors')}` : ''}`);
|
|
1184
1200
|
// Tier-1 firing counters for ① file-intel + ② reread-guard (recorded by
|
|
@@ -1188,7 +1204,7 @@ async function cmdStats(db, args) {
|
|
|
1188
1204
|
const rrN = featAgg.reread_warn?.count ?? 0;
|
|
1189
1205
|
const metricsOn = process.env.CLAUDE_MEM_METRICS === '1';
|
|
1190
1206
|
out(` Feature injections (7d): 📄 file-intel ${fiN} · 🔁 reread-warn ${rrN}${(!metricsOn && fiN + rrN === 0) ? ' (set CLAUDE_MEM_METRICS=1 to record)' : ''}`);
|
|
1191
|
-
if (noiseRatio > 0.6) out(' ⚠️ High noise ratio — consider running mem compress');
|
|
1207
|
+
if (noiseRatio > 0.6 || lowSignalRatio > 0.3) out(' ⚠️ High noise ratio — consider running mem maintain / compress');
|
|
1192
1208
|
out('');
|
|
1193
1209
|
// Tier counts only live (uncompressed, non-superseded) observations — surface the
|
|
1194
1210
|
// full decomposition so live + compressed + superseded = Total adds up cleanly.
|
|
@@ -1902,7 +1918,23 @@ function cmdMaintain(db, args) {
|
|
|
1902
1918
|
}
|
|
1903
1919
|
|
|
1904
1920
|
if (ops.includes('purge_stale')) {
|
|
1905
|
-
|
|
1921
|
+
// --retain-days: default 30 when absent; reject negative / 0 / NaN / out-of-range.
|
|
1922
|
+
// A negative value made retainCutoff a FUTURE timestamp → purged the entire
|
|
1923
|
+
// pending-purge backlog regardless of age; 0/garbage silently became 30 and
|
|
1924
|
+
// masked typos. Parity with the mem_maintain MCP zod bound [7, 365].
|
|
1925
|
+
let retainDays = 30;
|
|
1926
|
+
if (flags['retain-days'] !== undefined) {
|
|
1927
|
+
// Number + isInteger (not parseInt) so "7.5"/"30x" are rejected rather than
|
|
1928
|
+
// silently truncated to 7/30 — parity with the mem_maintain zod .int() bound.
|
|
1929
|
+
const parsed = Number(flags['retain-days']);
|
|
1930
|
+
if (!Number.isInteger(parsed) || parsed < 7 || parsed > 365) {
|
|
1931
|
+
// fail() only sets exitCode + writes stderr; it does NOT throw, so we
|
|
1932
|
+
// MUST return or execution falls through to the DELETE with the bad value.
|
|
1933
|
+
fail(`[mem] --retain-days must be an integer in [7, 365] (got "${flags['retain-days']}")`);
|
|
1934
|
+
return;
|
|
1935
|
+
}
|
|
1936
|
+
retainDays = parsed;
|
|
1937
|
+
}
|
|
1906
1938
|
const retainCutoff = Date.now() - retainDays * 86400000;
|
|
1907
1939
|
// T2-P0-A (CLI parity): purge_stale is the only DELETE in this code path — require
|
|
1908
1940
|
// --confirm so a mis-typed `maintain execute --ops purge_stale` can't wipe rows silently.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "claude-mem-lite",
|
|
3
|
-
"version": "3.
|
|
3
|
+
"version": "3.17.0",
|
|
4
4
|
"description": "Persistent long-term memory for Claude Code via MCP — captures coding decisions, bugfixes, and context across sessions. Hybrid FTS5 + TF-IDF search with episode batching. Single SQLite DB, no external services. A lighter, lower-cost alternative to claude-mem (episode batching + a smaller model; cost savings are an internal estimate, not a measured benchmark).",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"packageManager": "npm@10.9.2",
|
|
@@ -81,6 +81,7 @@
|
|
|
81
81
|
"lib/recall-core.mjs",
|
|
82
82
|
"lib/timeline-core.mjs",
|
|
83
83
|
"lib/search-core.mjs",
|
|
84
|
+
"lib/rrf.mjs",
|
|
84
85
|
"lib/compress-core.mjs",
|
|
85
86
|
"lib/maintain-core.mjs",
|
|
86
87
|
"lib/dedup-constants.mjs",
|
package/search-engine.mjs
CHANGED
|
@@ -11,7 +11,7 @@ import {
|
|
|
11
11
|
notLowSignalTitleClause, LOW_SIGNAL_TITLE,
|
|
12
12
|
relaxFtsQueryToOr, debugLog, debugCatch, estimateTokens,
|
|
13
13
|
} from './utils.mjs';
|
|
14
|
-
import { getVocabulary, computeVector, vectorSearch, rrfMerge } from './tfidf.mjs';
|
|
14
|
+
import { getVocabulary, computeVector, vectorSearch, rrfMerge, vectorsEnabled } from './tfidf.mjs';
|
|
15
15
|
import { extractPRFTerms, expandQueryByConcepts } from './search-scoring.mjs';
|
|
16
16
|
|
|
17
17
|
// Scoring expressions — full adds project boost + access bonus; simple is for
|
|
@@ -395,6 +395,7 @@ export function searchObservationsHybrid(db, ctx) {
|
|
|
395
395
|
|
|
396
396
|
// Vector search + RRF hybrid merge
|
|
397
397
|
try {
|
|
398
|
+
if (!vectorsEnabled()) return results; // Phase-1: vector arm disabled → BM25-only path (audit 2026-06-27)
|
|
398
399
|
const vocab = getVocabulary(db);
|
|
399
400
|
if (!vocab) return results;
|
|
400
401
|
const queryText = ftsQuery.replace(/['"()]/g, ' ');
|
package/server.mjs
CHANGED
|
@@ -12,6 +12,7 @@ import { reRankWithContext, markSuperseded, autoBoostIfNeeded, runIdleCleanup, b
|
|
|
12
12
|
import { searchObservationsHybrid } from './search-engine.mjs';
|
|
13
13
|
import { deepSearch, resolveDeepMode, shouldEscalateToDeep, autoDeepLlmReady } from './deep-search.mjs';
|
|
14
14
|
import { selectCompressionCandidates, groupByProjectWeek, compressGroup } from './lib/compress-core.mjs';
|
|
15
|
+
import { buildNotLowSignalSql } from './lib/low-signal-patterns.mjs';
|
|
15
16
|
import { resolveAnchorToken, formatAnchorError, resolveQueryAnchor, fetchRecentTimeline, fetchTimelineWindow } from './lib/timeline-core.mjs';
|
|
16
17
|
import { buildSearchFtsQuery, parseDateBounds, coreRunSearchPipeline } from './lib/search-core.mjs';
|
|
17
18
|
import {
|
|
@@ -864,6 +865,15 @@ server.registerTool(
|
|
|
864
865
|
`).get(thirtyDaysAgo, ...baseParams);
|
|
865
866
|
|
|
866
867
|
const noiseRatio = obsTotal.c > 0 ? lowVal.c / obsTotal.c : 0;
|
|
868
|
+
// Low-signal-title population (template / tool-log titles the read-side filter
|
|
869
|
+
// already excludes). The imp=1 "Low-value" metric can't see these, so the
|
|
870
|
+
// gauge under-reports real noise without it. See lib/low-signal-patterns.mjs.
|
|
871
|
+
const lowSignalTitle = db.prepare(`
|
|
872
|
+
SELECT COUNT(*) as c FROM observations
|
|
873
|
+
WHERE NOT ${buildNotLowSignalSql()}
|
|
874
|
+
AND COALESCE(compressed_into, 0) = 0 ${projectFilter}
|
|
875
|
+
`).get(...baseParams);
|
|
876
|
+
const lowSignalRatio = obsTotal.c > 0 ? lowSignalTitle.c / obsTotal.c : 0;
|
|
867
877
|
const compressedCount = db.prepare(`
|
|
868
878
|
SELECT COUNT(*) as c FROM observations WHERE compressed_into IS NOT NULL ${projectFilter}
|
|
869
879
|
`).get(...baseParams);
|
|
@@ -900,8 +910,9 @@ server.registerTool(
|
|
|
900
910
|
` Est. tokens: ${tokenEst.t ?? 0}`,
|
|
901
911
|
` Avg importance: ${(avgImp.v ?? 1).toFixed(2)}`,
|
|
902
912
|
` Low-value (imp=1, never accessed, >30d): ${lowVal.c} (${(noiseRatio * 100).toFixed(1)}% noise)`,
|
|
913
|
+
` Low-signal titles (Modified/Error/Worked on…): ${lowSignalTitle.c} (${(lowSignalRatio * 100).toFixed(1)}%)`,
|
|
903
914
|
` Compressed: ${compressedCount.c}`,
|
|
904
|
-
...(noiseRatio > 0.6 ? [' ⚠️ High noise ratio — consider running mem_compress'] : []),
|
|
915
|
+
...((noiseRatio > 0.6 || lowSignalRatio > 0.3) ? [' ⚠️ High noise ratio — consider running mem_compress / maintain'] : []),
|
|
905
916
|
'',
|
|
906
917
|
// Tier counts only live (uncompressed, non-superseded) observations — surface
|
|
907
918
|
// the full decomposition so live + compressed + superseded = Total adds up cleanly.
|
package/source-files.mjs
CHANGED
|
@@ -114,6 +114,9 @@ export const SOURCE_FILES = [
|
|
|
114
114
|
// `timeline`/`search` and mem_timeline/mem_search on auto-update.
|
|
115
115
|
'lib/timeline-core.mjs',
|
|
116
116
|
'lib/search-core.mjs',
|
|
117
|
+
// Reciprocal Rank Fusion core (D#42 single source-of-truth); transitively
|
|
118
|
+
// reached via tfidf.mjs (rrfMerge) and deep-search.mjs (rrfFuseN).
|
|
119
|
+
'lib/rrf.mjs',
|
|
117
120
|
// Shared "compress old low-value observations into weekly summaries" core.
|
|
118
121
|
// Statically imported by mem-cli.mjs (cmdCompress), server.mjs (mem_compress),
|
|
119
122
|
// and hook.mjs (handleAutoCompress) — same single-source-of-truth pattern as
|
package/tfidf.mjs
CHANGED
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
import { cjkBigrams } from './utils.mjs';
|
|
7
7
|
import { BASE_STOP_WORDS } from './stop-words.mjs';
|
|
8
8
|
import { createHash } from 'crypto';
|
|
9
|
+
import { rrfAccumulate } from './lib/rrf.mjs';
|
|
9
10
|
|
|
10
11
|
export const VOCAB_DIM = 512;
|
|
11
12
|
export const MIN_COSINE_SIMILARITY = 0.05;
|
|
@@ -15,6 +16,21 @@ export const VECTOR_SCAN_LIMIT = 500;
|
|
|
15
16
|
// dominate. 60 is the de-facto RRF default and balances the two retrievers here.
|
|
16
17
|
export const RRF_K = 60;
|
|
17
18
|
|
|
19
|
+
// Phase-1 disable of the TF-IDF vector arm (memory-quality audit 2026-06-27).
|
|
20
|
+
// The arm reads ~0 benchmark lift (ci-gate hybrid_over_bm25 = 0), pays a
|
|
21
|
+
// computeVector on every observation write, only scans the most-recent
|
|
22
|
+
// VECTOR_SCAN_LIMIT rows (misses older memories), and drifted unnoticed for
|
|
23
|
+
// months. Default OFF. To re-enable: set CLAUDE_MEM_VECTORS=1 AND run
|
|
24
|
+
// `claude-mem-lite maintain execute --ops rebuild_vectors` to repopulate vectors for
|
|
25
|
+
// observations created while disabled — re-enabling WITHOUT a rebuild leaves a
|
|
26
|
+
// partially-populated index (the primary + enrich/compress/optimize write paths all
|
|
27
|
+
// stopped writing while off) → silently degraded hybrid recall until the next
|
|
28
|
+
// rebuild. Tables + code are retained pending Phase-2 removal. Read at call time (not
|
|
29
|
+
// import) so tests + the benchmark A/B can toggle it via env.
|
|
30
|
+
export function vectorsEnabled() {
|
|
31
|
+
return process.env.CLAUDE_MEM_VECTORS === '1';
|
|
32
|
+
}
|
|
33
|
+
|
|
18
34
|
const VOCAB_STOP_WORDS = new Set([
|
|
19
35
|
...BASE_STOP_WORDS,
|
|
20
36
|
'now','only','still','here','there','up','out','am',
|
|
@@ -275,6 +291,14 @@ export function rebuildVocabulary(db, opts) {
|
|
|
275
291
|
* @returns {object|null} vocabulary
|
|
276
292
|
*/
|
|
277
293
|
export function getVocabulary(db) {
|
|
294
|
+
// Phase-1 choke point: when the vector arm is disabled there is no vocabulary to
|
|
295
|
+
// serve. Returning null here makes EVERY vector write/read path skip via its
|
|
296
|
+
// existing `if (vocab)` guard — including the enrich (hook-llm), compress
|
|
297
|
+
// (compress-core) and optimize (hook-optimize) write paths that don't go through
|
|
298
|
+
// insertObservationVector — so no path can be "missed" and observation_vectors
|
|
299
|
+
// stays uniformly stale (not half-populated) while disabled. computeVector is
|
|
300
|
+
// null-safe, so even a caller that skips its guard no-ops rather than crashes.
|
|
301
|
+
if (!vectorsEnabled()) return null;
|
|
278
302
|
if (_vocabCache) return _vocabCache;
|
|
279
303
|
|
|
280
304
|
// Try loading from persisted vocab_state
|
|
@@ -423,14 +447,9 @@ export function vectorSearch(db, queryVec, { project, type, vocabVersion, limit
|
|
|
423
447
|
* @returns {{ id: number, rrfScore: number }[]}
|
|
424
448
|
*/
|
|
425
449
|
export function rrfMerge(bm25Results, vectorResults, k = RRF_K) {
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
scores.set(r.id, (scores.get(r.id) ?? 0) + 1 / (k + i + 1));
|
|
432
|
-
});
|
|
433
|
-
return [...scores.entries()]
|
|
434
|
-
.sort((a, b) => b[1] - a[1])
|
|
435
|
-
.map(([id, score]) => ({ id, rrfScore: score }));
|
|
450
|
+
// Thin 2-list adapter over the shared RRF core (lib/rrf.mjs). Emits the minimal
|
|
451
|
+
// { id, rrfScore } shape this module's callers (search-engine.mjs) expect; the
|
|
452
|
+
// accumulator's best-rank row tracking is irrelevant here and ignored.
|
|
453
|
+
return rrfAccumulate([bm25Results, vectorResults], k)
|
|
454
|
+
.map(({ id, score }) => ({ id, rrfScore: score }));
|
|
436
455
|
}
|