claude-mem-lite 3.40.0 → 3.42.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/.claude-plugin/plugin.json +1 -1
- package/hook-context.mjs +1 -0
- package/hook-handoff.mjs +21 -6
- package/hook-llm.mjs +10 -7
- package/hook-optimize.mjs +40 -18
- package/hook-shared.mjs +3 -1
- package/hook.mjs +12 -1
- package/install.mjs +8 -2
- package/lib/citation-tracker.mjs +3 -2
- package/lib/import-jsonl.mjs +5 -1
- package/lib/maintain-core.mjs +3 -4
- package/lib/search-core.mjs +6 -0
- package/mem-cli.mjs +35 -11
- package/package.json +1 -1
- package/registry.mjs +7 -2
- package/scripts/setup.sh +9 -11
- package/search-engine.mjs +2 -2
- package/secret-scrub.mjs +28 -12
- package/tfidf.mjs +18 -2
- package/tool-schemas.mjs +5 -3
- package/utils.mjs +6 -0
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
"plugins": [
|
|
11
11
|
{
|
|
12
12
|
"name": "claude-mem-lite",
|
|
13
|
-
"version": "3.
|
|
13
|
+
"version": "3.42.0",
|
|
14
14
|
"source": "./",
|
|
15
15
|
"description": "Persistent long-term memory for Claude Code via MCP — captures coding decisions, bugfixes, and context across sessions. Hybrid FTS5 + TF-IDF search with episode batching. Single SQLite DB, no external services. A lighter, lower-cost alternative to claude-mem (episode batching + a smaller model; cost savings are an internal estimate, not a measured benchmark)."
|
|
16
16
|
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "claude-mem-lite",
|
|
3
|
-
"version": "3.
|
|
3
|
+
"version": "3.42.0",
|
|
4
4
|
"description": "Persistent long-term memory for Claude Code via MCP — captures coding decisions, bugfixes, and context across sessions. Hybrid FTS5 + TF-IDF search with episode batching. Single SQLite DB, no external services. A lighter, lower-cost alternative to claude-mem (episode batching + a smaller model; cost savings are an internal estimate, not a measured benchmark).",
|
|
5
5
|
"author": {
|
|
6
6
|
"name": "sdsrss"
|
package/hook-context.mjs
CHANGED
|
@@ -305,6 +305,7 @@ export function buildSessionContextLines(db, project, now = new Date(), currentC
|
|
|
305
305
|
FROM observations
|
|
306
306
|
WHERE COALESCE(compressed_into, 0) = 0
|
|
307
307
|
AND superseded_at IS NULL
|
|
308
|
+
AND ${notLowSignalTitleClause('')}
|
|
308
309
|
AND (
|
|
309
310
|
(created_at_epoch > ? AND importance >= 1)
|
|
310
311
|
OR (created_at_epoch > ? AND importance >= 2)
|
package/hook-handoff.mjs
CHANGED
|
@@ -271,13 +271,28 @@ export function detectContinuationIntent(db, promptText, project, currentCcSessi
|
|
|
271
271
|
try {
|
|
272
272
|
const currentSha = gitStateModule.readGitState({ cwd: process.cwd() }).headSha;
|
|
273
273
|
if (currentSha) {
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
274
|
+
// Scope like Stage 2: an 'exit' anchor is cross-session (resume after /exit), but a 'clear'
|
|
275
|
+
// anchor is same-session only — else a parallel same-project session at the same commit
|
|
276
|
+
// would hijack (and then delete) another session's clear handoff.
|
|
277
|
+
const anchor = currentCcSessionId
|
|
278
|
+
? db.prepare(`
|
|
279
|
+
SELECT created_at_epoch, match_keywords FROM session_handoffs
|
|
280
|
+
WHERE project = ? AND git_sha_at_handoff = ? AND (type = 'exit' OR session_id = ?)
|
|
281
|
+
ORDER BY created_at_epoch DESC LIMIT 1
|
|
282
|
+
`).get(project, currentSha, currentCcSessionId)
|
|
283
|
+
: db.prepare(`
|
|
284
|
+
SELECT created_at_epoch, match_keywords FROM session_handoffs
|
|
285
|
+
WHERE project = ? AND git_sha_at_handoff = ?
|
|
286
|
+
ORDER BY created_at_epoch DESC LIMIT 1
|
|
287
|
+
`).get(project, currentSha);
|
|
279
288
|
if (anchor && (Date.now() - anchor.created_at_epoch <= HANDOFF_ANCHOR_MAX_AGE)) {
|
|
280
|
-
|
|
289
|
+
// Unmoved HEAD is a strong resume signal, but must not hijack a NEW task typed at the
|
|
290
|
+
// same commit: gate long prompts on keyword overlap (mirror Stage 0). Short prompts
|
|
291
|
+
// (resume nudges) auto-continue.
|
|
292
|
+
if (promptText.length < 40) return true;
|
|
293
|
+
const hTokens = anchor.match_keywords ? new Set(tokenizeHandoff(anchor.match_keywords)) : null;
|
|
294
|
+
if (!hTokens || tokenizeHandoff(promptText).some(t => hTokens.has(t))) return true;
|
|
295
|
+
// long prompt with zero keyword overlap → fall through to Stage 0/1/2
|
|
281
296
|
}
|
|
282
297
|
}
|
|
283
298
|
} catch { /* git/DB failure must not break the rest of the pipeline */ }
|
package/hook-llm.mjs
CHANGED
|
@@ -11,7 +11,7 @@ import {
|
|
|
11
11
|
} from './utils.mjs';
|
|
12
12
|
import { acquireLLMSlot, releaseLLMSlot } from './hook-semaphore.mjs';
|
|
13
13
|
import { scrubRecord } from './lib/scrub-record.mjs';
|
|
14
|
-
import { getVocabulary, computeVector } from './tfidf.mjs';
|
|
14
|
+
import { getVocabulary, computeVector, vecTextForRow } from './tfidf.mjs';
|
|
15
15
|
import { insertObservationRow, insertObservationFiles, insertObservationVector } from './lib/observation-write.mjs';
|
|
16
16
|
import { DEDUP_JACCARD_THRESHOLD, AUTO_MERGE_THRESHOLD } from './lib/dedup-constants.mjs';
|
|
17
17
|
import {
|
|
@@ -124,10 +124,12 @@ function buildFtsTextField(obs) {
|
|
|
124
124
|
// weight) and search_aliases (finding #8: previously omitted, so even with vectors
|
|
125
125
|
// enabled the paraphrase-bridge alias terms were invisible to cosine similarity).
|
|
126
126
|
export function buildVecText(obs) {
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
return
|
|
130
|
-
.
|
|
127
|
+
// Single source (V-F1): map the camelCase obs onto vecTextForRow's row shape so save and
|
|
128
|
+
// every rebuild path encode the identical field set (title/narrative/concepts/lesson/aliases).
|
|
129
|
+
return vecTextForRow({
|
|
130
|
+
title: obs.title, narrative: obs.narrative, concepts: obs.concepts,
|
|
131
|
+
lesson_learned: obs.lessonLearned, search_aliases: obs.searchAliases,
|
|
132
|
+
});
|
|
131
133
|
}
|
|
132
134
|
|
|
133
135
|
/**
|
|
@@ -996,7 +998,8 @@ ${actionList}`;
|
|
|
996
998
|
search_aliases: obs.searchAliases || null,
|
|
997
999
|
});
|
|
998
1000
|
db.prepare(`
|
|
999
|
-
UPDATE observations SET type=?, title=?, subtitle=?,
|
|
1001
|
+
UPDATE observations SET type=?, title=?, subtitle=?,
|
|
1002
|
+
narrative=COALESCE(NULLIF(?, ''), narrative), concepts=?, facts=?,
|
|
1000
1003
|
text=?, importance=?, files_read=?, minhash_sig=?, lesson_learned=?, search_aliases=?
|
|
1001
1004
|
WHERE id = ?
|
|
1002
1005
|
`).run(
|
|
@@ -1018,7 +1021,7 @@ ${actionList}`;
|
|
|
1018
1021
|
try {
|
|
1019
1022
|
const vocab = getVocabulary(db);
|
|
1020
1023
|
if (vocab) {
|
|
1021
|
-
const vecText =
|
|
1024
|
+
const vecText = vecTextForRow({ title: obs.title, narrative: obs.narrative, concepts: conceptsText, lesson_learned: safe.lesson_learned, search_aliases: safe.search_aliases });
|
|
1022
1025
|
const vec = computeVector(vecText, vocab);
|
|
1023
1026
|
if (vec) {
|
|
1024
1027
|
db.prepare('INSERT OR REPLACE INTO observation_vectors (observation_id, vector, vocab_version, created_at_epoch) VALUES (?, ?, ?, ?)')
|
package/hook-optimize.mjs
CHANGED
|
@@ -12,7 +12,7 @@ import {
|
|
|
12
12
|
import { callModelJSON } from './haiku-client.mjs';
|
|
13
13
|
import { acquireLLMSlot, releaseLLMSlot } from './hook-semaphore.mjs';
|
|
14
14
|
import { scrubRecord } from './lib/scrub-record.mjs';
|
|
15
|
-
import { getVocabulary, computeVector, cosineSimilarity } from './tfidf.mjs';
|
|
15
|
+
import { getVocabulary, computeVector, cosineSimilarity, vecTextForRow } from './tfidf.mjs';
|
|
16
16
|
import { MERGE_JACCARD_LOW, AUTO_MERGE_THRESHOLD } from './lib/dedup-constants.mjs';
|
|
17
17
|
import { DB_DIR } from './schema.mjs';
|
|
18
18
|
|
|
@@ -25,9 +25,14 @@ export function distributeBudget(total = 15) {
|
|
|
25
25
|
const reenrich = Math.max(1, Math.floor(total * 0.4));
|
|
26
26
|
const clusterMerge = Math.max(1, Math.floor(total * 0.3));
|
|
27
27
|
const smartCompress = Math.max(1, total - reenrich - normalize - clusterMerge);
|
|
28
|
-
// Clamp: if total is too small for 4 tasks,
|
|
28
|
+
// Clamp: if total is too small for all 4 tasks, allocate 1 each by priority until `total`
|
|
29
|
+
// is exhausted so the returned sum is ≤ total (the old fallback returned {1,1,1,1}=4 for
|
|
30
|
+
// total≤3, over-running `optimize --max N`).
|
|
29
31
|
if (reenrich + normalize + clusterMerge + smartCompress > total) {
|
|
30
|
-
|
|
32
|
+
const alloc = { reenrich: 0, clusterMerge: 0, smartCompress: 0, normalize: 0 };
|
|
33
|
+
const order = ['reenrich', 'clusterMerge', 'smartCompress', 'normalize'];
|
|
34
|
+
for (let i = 0; i < total && i < order.length; i++) alloc[order[i]] = 1;
|
|
35
|
+
return alloc;
|
|
31
36
|
}
|
|
32
37
|
return { reenrich, normalize, clusterMerge, smartCompress };
|
|
33
38
|
}
|
|
@@ -39,11 +44,16 @@ export function distributeBudget(total = 15) {
|
|
|
39
44
|
* Exported for testing; also kept as the single source of vector-rebuild logic
|
|
40
45
|
* for the optimize / re-enrich path to avoid drift with the hook-llm write path.
|
|
41
46
|
*/
|
|
42
|
-
export function rebuildVector(db, obsId,
|
|
47
|
+
export function rebuildVector(db, obsId, textPartsOrRow) {
|
|
43
48
|
try {
|
|
44
49
|
const vocab = getVocabulary(db);
|
|
45
50
|
if (!vocab) return;
|
|
46
|
-
|
|
51
|
+
// Accept a legacy [parts] array OR an observation row (preferred — single-source field set
|
|
52
|
+
// incl. lesson_learned/search_aliases via vecTextForRow, so rebuilds match the save path).
|
|
53
|
+
const text = Array.isArray(textPartsOrRow)
|
|
54
|
+
? textPartsOrRow.filter(Boolean).join(' ')
|
|
55
|
+
: vecTextForRow(textPartsOrRow);
|
|
56
|
+
const vec = computeVector(text, vocab);
|
|
47
57
|
if (vec) {
|
|
48
58
|
// Bug #1 fix: column is `created_at_epoch`, not `computed_at`. Every other
|
|
49
59
|
// INSERT callsite (server.mjs, hook-llm.mjs, mem-cli.mjs) uses the correct
|
|
@@ -87,7 +97,7 @@ export function findReenrichCandidates(db, limit = 10, { scope = 'narrow', proje
|
|
|
87
97
|
// deliberately NOT gated on optimized_at, so a lesson-less row can still be
|
|
88
98
|
// picked up by wide scope for lesson enrichment afterward.
|
|
89
99
|
const stmt = db.prepare(`
|
|
90
|
-
SELECT id, title, narrative, type, subtitle, concepts, facts, text, search_aliases, project
|
|
100
|
+
SELECT id, title, narrative, type, subtitle, concepts, facts, text, search_aliases, importance, project
|
|
91
101
|
FROM observations
|
|
92
102
|
WHERE COALESCE(compressed_into, 0) = 0
|
|
93
103
|
AND superseded_at IS NULL
|
|
@@ -102,7 +112,7 @@ export function findReenrichCandidates(db, limit = 10, { scope = 'narrow', proje
|
|
|
102
112
|
}
|
|
103
113
|
if (scope === 'wide') {
|
|
104
114
|
const stmt = db.prepare(`
|
|
105
|
-
SELECT id, title, narrative, type, subtitle, concepts, facts, search_aliases, project
|
|
115
|
+
SELECT id, title, narrative, type, subtitle, concepts, facts, search_aliases, importance, project
|
|
106
116
|
FROM observations
|
|
107
117
|
WHERE COALESCE(compressed_into, 0) = 0
|
|
108
118
|
AND superseded_at IS NULL
|
|
@@ -120,7 +130,7 @@ export function findReenrichCandidates(db, limit = 10, { scope = 'narrow', proje
|
|
|
120
130
|
return project ? stmt.all(project, limit) : stmt.all(limit);
|
|
121
131
|
}
|
|
122
132
|
const stmt = db.prepare(`
|
|
123
|
-
SELECT id, title, narrative, type, subtitle, project
|
|
133
|
+
SELECT id, title, narrative, type, subtitle, importance, project
|
|
124
134
|
FROM observations
|
|
125
135
|
WHERE COALESCE(compressed_into, 0) = 0
|
|
126
136
|
AND (concepts IS NULL OR concepts = '')
|
|
@@ -205,7 +215,10 @@ search_aliases: 2-6 alternative search terms (include CJK if applicable).`;
|
|
|
205
215
|
continue;
|
|
206
216
|
}
|
|
207
217
|
|
|
208
|
-
|
|
218
|
+
// Enrichment ("add a lesson") must not reclassify a specific type down to the generic
|
|
219
|
+
// 'change' (lower TYPE_QUALITY + faster decay); keep the stored type on that downgrade.
|
|
220
|
+
let type = validTypes.has(parsed.type) ? parsed.type : (cand.type || 'change');
|
|
221
|
+
if (type === 'change' && cand.type && cand.type !== 'change') type = cand.type;
|
|
209
222
|
const concepts = Array.isArray(parsed.concepts) ? parsed.concepts.slice(0, 10) : [];
|
|
210
223
|
const facts = Array.isArray(parsed.facts) ? parsed.facts.slice(0, 10) : [];
|
|
211
224
|
// Preserve-on-empty: wide-scope candidates can already carry concepts/facts/aliases
|
|
@@ -227,7 +240,10 @@ search_aliases: 2-6 alternative search terms (include CJK if applicable).`;
|
|
|
227
240
|
? parsed.search_aliases.slice(0, 6).join(' ') : (cand.search_aliases || null);
|
|
228
241
|
const title = truncate(scrubSecrets(parsed.title || ''), 120);
|
|
229
242
|
const narrative = truncate(scrubSecrets(parsed.narrative || cand.narrative || ''), 500);
|
|
230
|
-
|
|
243
|
+
// Floor at the stored importance: re-enrich adds a lesson, it must never silently downgrade
|
|
244
|
+
// a user-set/promoted importance (the UPDATE also sets optimized_at → the loss is permanent).
|
|
245
|
+
// Upgrades are still honored.
|
|
246
|
+
const importance = Math.max(clampImportance(parsed.importance), cand.importance || 1);
|
|
231
247
|
|
|
232
248
|
const bigramText = cjkBigrams((title || '') + ' ' + (narrative || ''));
|
|
233
249
|
const textField = [conceptsText, factsText, searchAliases || '', bigramText].filter(Boolean).join(' ');
|
|
@@ -250,7 +266,7 @@ search_aliases: 2-6 alternative search terms (include CJK if applicable).`;
|
|
|
250
266
|
`).run(type, safe.title, safe.narrative, safe.concepts, safe.facts, safe.text,
|
|
251
267
|
importance, safe.lesson_learned, safe.search_aliases, minhashSig, Date.now(), cand.id);
|
|
252
268
|
|
|
253
|
-
rebuildVector(db, cand.id,
|
|
269
|
+
rebuildVector(db, cand.id, { title, narrative, concepts: conceptsText, lesson_learned: safe.lesson_learned, search_aliases: safe.search_aliases });
|
|
254
270
|
|
|
255
271
|
processed++;
|
|
256
272
|
} catch (e) {
|
|
@@ -362,7 +378,7 @@ export function applyNormalization(db, groups, { project = null } = {}) {
|
|
|
362
378
|
// contamination the --project flag was added to prevent. NULL → all projects (legacy
|
|
363
379
|
// unscoped run), matching the search-engine `(? IS NULL OR project = ?)` idiom.
|
|
364
380
|
const rows = db.prepare(`
|
|
365
|
-
SELECT id, concepts, search_aliases FROM observations
|
|
381
|
+
SELECT id, title, narrative, concepts, search_aliases, lesson_learned FROM observations
|
|
366
382
|
WHERE COALESCE(compressed_into, 0) = 0
|
|
367
383
|
AND concepts IS NOT NULL AND concepts != ''
|
|
368
384
|
AND (? IS NULL OR project = ?)
|
|
@@ -395,6 +411,9 @@ export function applyNormalization(db, groups, { project = null } = {}) {
|
|
|
395
411
|
search_aliases: newAliases,
|
|
396
412
|
});
|
|
397
413
|
updateStmt.run(safe.concepts, safe.search_aliases, Date.now(), row.id);
|
|
414
|
+
// V-F3: normalize mutated concepts + search_aliases (both vector fields) — rebuild the
|
|
415
|
+
// vector so it reflects the canonicalized terms (no-op when the vector arm is disabled).
|
|
416
|
+
rebuildVector(db, row.id, { title: row.title, narrative: row.narrative, concepts: safe.concepts, search_aliases: safe.search_aliases, lesson_learned: row.lesson_learned });
|
|
398
417
|
updated++;
|
|
399
418
|
}
|
|
400
419
|
}
|
|
@@ -433,7 +452,7 @@ export function findMergeCandidates(db, maxClusters = 5, { project } = {}) {
|
|
|
433
452
|
const cutoff = Date.now() - MERGE_TIME_WINDOW_MS;
|
|
434
453
|
const projectClause = project ? 'AND project = ?' : '';
|
|
435
454
|
const stmt = db.prepare(`
|
|
436
|
-
SELECT id, title, narrative, project, type, access_count, importance, created_at_epoch, minhash_sig, lesson_learned
|
|
455
|
+
SELECT id, title, narrative, project, type, access_count, importance, created_at_epoch, minhash_sig, lesson_learned, concepts, facts
|
|
437
456
|
FROM observations
|
|
438
457
|
WHERE COALESCE(compressed_into, 0) = 0
|
|
439
458
|
AND superseded_at IS NULL
|
|
@@ -520,12 +539,15 @@ Return ONLY valid JSON:
|
|
|
520
539
|
|
|
521
540
|
const concepts = Array.isArray(parsed.merged_concepts) ? parsed.merged_concepts.slice(0, 10) : [];
|
|
522
541
|
const facts = Array.isArray(parsed.merged_facts) ? parsed.merged_facts.slice(0, 10) : [];
|
|
523
|
-
|
|
524
|
-
|
|
542
|
+
// Preserve-on-empty (mirror the merged_lesson guard below + the re-enrich path): the merge
|
|
543
|
+
// overwrites the keeper in place, so a partial LLM response that omits these must fall back to
|
|
544
|
+
// the keeper's own values, not blank its live concepts/facts (findMergeCandidates now selects them).
|
|
545
|
+
const conceptsText = concepts.length ? concepts.join(' ') : (keeper.concepts || '');
|
|
546
|
+
const factsText = facts.length ? facts.join(' ') : (keeper.facts || '');
|
|
525
547
|
// Scrub BEFORE truncate (see re-enrich note): keep the boundary cut on
|
|
526
548
|
// already-scrubbed text so a straddling secret can't leak a sub-floor head.
|
|
527
549
|
const title = truncate(scrubSecrets(parsed.merged_title || ''), 120);
|
|
528
|
-
const narrative = truncate(scrubSecrets(parsed.merged_narrative || ''), 800);
|
|
550
|
+
const narrative = truncate(scrubSecrets(parsed.merged_narrative || keeper.narrative || ''), 800);
|
|
529
551
|
// Preserve-on-empty. The merge overwrites the keeper in place and hides every non-keeper
|
|
530
552
|
// member (compressed_into=keeper.id), so if the LLM returns merged_lesson:null (the prompt
|
|
531
553
|
// at :429 explicitly permits it) every cluster lesson would leave all live surfaces at once
|
|
@@ -589,7 +611,7 @@ Return ONLY valid JSON:
|
|
|
589
611
|
.run(keeper.id, ...otherIds);
|
|
590
612
|
})();
|
|
591
613
|
|
|
592
|
-
rebuildVector(db, keeper.id,
|
|
614
|
+
rebuildVector(db, keeper.id, { title, narrative, concepts: conceptsText, lesson_learned: lessonLearned, search_aliases: keeper.search_aliases });
|
|
593
615
|
|
|
594
616
|
debugLog('DEBUG', 'llm-optimize', `merged ${cluster.length} observations into #${keeper.id}`);
|
|
595
617
|
return { merged: true, keeperId: keeper.id, mergedCount: others.length };
|
|
@@ -793,7 +815,7 @@ JSON: {"title":"descriptive summary ≤120 chars","narrative":"comprehensive sum
|
|
|
793
815
|
return sId;
|
|
794
816
|
})();
|
|
795
817
|
|
|
796
|
-
rebuildVector(db, summaryId,
|
|
818
|
+
rebuildVector(db, summaryId, { title, narrative, concepts: conceptsText });
|
|
797
819
|
|
|
798
820
|
debugLog('DEBUG', 'llm-optimize', `smart-compressed ${observations.length} observations into #${summaryId}`);
|
|
799
821
|
return { compressed: true, summaryId, count: observations.length };
|
package/hook-shared.mjs
CHANGED
|
@@ -98,7 +98,9 @@ export function sweepOrphanEpisodeFiles(runtimeDir, { ageMs = ORPHAN_EPISODE_AGE
|
|
|
98
98
|
const readsCutoff = now - readsAgeMs;
|
|
99
99
|
let count = 0;
|
|
100
100
|
for (const f of entries) {
|
|
101
|
-
|
|
101
|
+
// `.claim-` = handleStop's lock-contended fallback claim file (ep-<project>.json.claim-<pid>-<ts>),
|
|
102
|
+
// which leaks if the process dies between rename and unlink; sweep it on the same 1h cutoff.
|
|
103
|
+
const isEpisode = f.startsWith('ep-flush-') || f.startsWith('pending-') || f.includes('.claim-');
|
|
102
104
|
const isReads = f.startsWith('reads-') && f.endsWith('.txt');
|
|
103
105
|
if (!isEpisode && !isReads) continue;
|
|
104
106
|
const full = join(runtimeDir, f);
|
package/hook.mjs
CHANGED
|
@@ -484,6 +484,10 @@ async function handleStop() {
|
|
|
484
484
|
for (const sub of planEpisodeFlush(episode)) {
|
|
485
485
|
if (!sub.sessionId) sub.sessionId = sessionId;
|
|
486
486
|
if (!sub.project) sub.project = project;
|
|
487
|
+
// Per-sub significance gate — parity with flushEpisodeGroup. The whole-episode check
|
|
488
|
+
// above can pass while an interleaved concurrent-session sub is pure noise (e.g. a lone
|
|
489
|
+
// Read); without this the fallback persists that noise sub AND spawns an LLM worker for it.
|
|
490
|
+
if (!episodeHasSignificantContent(sub)) continue;
|
|
487
491
|
try {
|
|
488
492
|
const obs = buildImmediateObservation(sub);
|
|
489
493
|
const id = saveObservation(obs, sub.project, sub.sessionId);
|
|
@@ -1412,7 +1416,14 @@ async function handleUserPrompt() {
|
|
|
1412
1416
|
);
|
|
1413
1417
|
|
|
1414
1418
|
// Cross-session handoff injection (first 3 prompts window, before semantic memory).
|
|
1415
|
-
|
|
1419
|
+
// prompt_counter is project-scoped (shared across concurrent same-project CC sessions), so a
|
|
1420
|
+
// parallel session would start past the window and never get its handoff injected. Count THIS
|
|
1421
|
+
// cc_session's own prompts instead (the current one is already inserted above); legacy null cc
|
|
1422
|
+
// id falls back to the shared counter.
|
|
1423
|
+
const windowPos = ccSessionId
|
|
1424
|
+
? (db.prepare('SELECT COUNT(*) c FROM user_prompts WHERE cc_session_id = ?').get(ccSessionId)?.c || promptNumber)
|
|
1425
|
+
: promptNumber;
|
|
1426
|
+
if (windowPos <= 3) {
|
|
1416
1427
|
try {
|
|
1417
1428
|
if (detectContinuationIntent(db, promptText, project, ccSessionId)) {
|
|
1418
1429
|
const picked = pickHandoffToInject(db, project, ccSessionId);
|
package/install.mjs
CHANGED
|
@@ -1688,12 +1688,18 @@ async function doctor() {
|
|
|
1688
1688
|
|
|
1689
1689
|
// ─── Settings helpers ───────────────────────────────────────────────────────
|
|
1690
1690
|
|
|
1691
|
-
|
|
1691
|
+
// Identify a settings hook as one of OURS (to replace on install / strip on uninstall).
|
|
1692
|
+
// Must be tight: the old `hook.mjs` + event-word test matched a user's OWN generic hook
|
|
1693
|
+
// (`node ~/.config/hook.mjs session-start`) and install/uninstall silently deleted it.
|
|
1694
|
+
// The launcher marker (`hook-launcher.mjs`, which every Node hook routes through since v2.84)
|
|
1695
|
+
// replaces that clause; the product-name substring (our install-dir / legacy-direct hooks)
|
|
1696
|
+
// and the bash prefilter round out the real markers.
|
|
1697
|
+
export function isMemHook(cfg) {
|
|
1692
1698
|
if (!cfg.hooks) return false;
|
|
1693
1699
|
return cfg.hooks.some(h => {
|
|
1694
1700
|
const cmd = h.command || '';
|
|
1695
1701
|
return cmd.includes('claude-mem-lite') ||
|
|
1696
|
-
|
|
1702
|
+
cmd.includes('hook-launcher.mjs') ||
|
|
1697
1703
|
cmd.includes('scripts/post-tool-use.sh');
|
|
1698
1704
|
});
|
|
1699
1705
|
}
|
package/lib/citation-tracker.mjs
CHANGED
|
@@ -626,9 +626,10 @@ export function applyCitationDecay(db, project, injectedIds, citedIds, sessionId
|
|
|
626
626
|
// cite-rate reads N/2 for a single injected-then-cited obs instead of N/1.
|
|
627
627
|
const updatePromote = db.prepare(`
|
|
628
628
|
UPDATE observations
|
|
629
|
-
SET importance = MIN(?, importance + 1),
|
|
629
|
+
SET importance = MIN(?, COALESCE(importance, 1) + 1),
|
|
630
630
|
cited_count = cited_count + 1,
|
|
631
631
|
uncited_streak = 0,
|
|
632
|
+
demoted_at = NULL,
|
|
632
633
|
last_decided_session_id = ?,
|
|
633
634
|
last_cited_session_id = ?,
|
|
634
635
|
decay_seen_count = decay_seen_count + ?
|
|
@@ -655,7 +656,7 @@ export function applyCitationDecay(db, project, injectedIds, citedIds, sessionId
|
|
|
655
656
|
`);
|
|
656
657
|
const updateDemote = db.prepare(`
|
|
657
658
|
UPDATE observations
|
|
658
|
-
SET importance = MAX(?, importance - 1),
|
|
659
|
+
SET importance = MAX(?, COALESCE(importance, 1) - 1),
|
|
659
660
|
uncited_streak = 0,
|
|
660
661
|
last_decided_session_id = ?,
|
|
661
662
|
demoted_at = ?,
|
package/lib/import-jsonl.mjs
CHANGED
|
@@ -147,7 +147,11 @@ export async function importJsonl(db, path, { project }) {
|
|
|
147
147
|
const mb = (n) => Math.round(n / (1024 * 1024));
|
|
148
148
|
throw new Error(`transcript too large (${mb(st.size)}MB > ${mb(MAX_IMPORT_BYTES)}MB cap); split it (e.g. split -l 50000 into parts) and import the parts`);
|
|
149
149
|
}
|
|
150
|
-
|
|
150
|
+
// Strip a leading UTF-8 BOM — Node's utf8 read leaves it on, so line 1 would become
|
|
151
|
+
// a U+FEFF-prefixed "{...}" and fail JSON.parse (silently dropped as "skipped"). Real CC
|
|
152
|
+
// are BOM-less, but an editor-touched or re-encoded file can carry one.
|
|
153
|
+
const rawText = readFileSync(path, 'utf8');
|
|
154
|
+
const lines = (rawText.charCodeAt(0) === 0xFEFF ? rawText.slice(1) : rawText).split('\n');
|
|
151
155
|
const seenPrompts = new Set();
|
|
152
156
|
const seenObs = new Set();
|
|
153
157
|
// Pre-seed dedup sets from existing rows so a second run on the same file
|
package/lib/maintain-core.mjs
CHANGED
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
// { projectFilter: 'AND project = ?' | '', baseParams: [project?] , staleAge, opCap }
|
|
14
14
|
|
|
15
15
|
import { COMPRESSED_PENDING_PURGE, computeMinHash, estimateJaccardFromMinHash, jaccardSimilarity } from '../utils.mjs';
|
|
16
|
-
import { rebuildVocabulary, computeVector, _resetVocabCache, vectorsEnabled } from '../tfidf.mjs';
|
|
16
|
+
import { rebuildVocabulary, computeVector, _resetVocabCache, vectorsEnabled, vecTextForRow } from '../tfidf.mjs';
|
|
17
17
|
import { DEDUP_JACCARD_THRESHOLD, MINHASH_PRE_THRESHOLD as MINHASH_PRE_THRESHOLD_SRC, FUZZY_DEDUP_THRESHOLD, FUZZY_BODY_THRESHOLD, MINHASH_PREFILTER } from './dedup-constants.mjs';
|
|
18
18
|
|
|
19
19
|
export const STALE_AGE_MS = 30 * 86400000;
|
|
@@ -428,7 +428,7 @@ export function rebuildVectors(db) {
|
|
|
428
428
|
const vocab = rebuildVocabulary(db);
|
|
429
429
|
if (!vocab) return { ok: false, reason: 'no observations to build vocabulary from' };
|
|
430
430
|
const allObs = db.prepare(`
|
|
431
|
-
SELECT id, title, narrative, concepts FROM observations
|
|
431
|
+
SELECT id, title, narrative, concepts, lesson_learned, search_aliases FROM observations
|
|
432
432
|
WHERE COALESCE(compressed_into, 0) = 0 AND superseded_at IS NULL
|
|
433
433
|
`).all();
|
|
434
434
|
let updated = 0;
|
|
@@ -437,8 +437,7 @@ export function rebuildVectors(db) {
|
|
|
437
437
|
db.transaction(() => {
|
|
438
438
|
db.prepare('DELETE FROM observation_vectors').run();
|
|
439
439
|
for (const obs of allObs) {
|
|
440
|
-
const
|
|
441
|
-
const vec = computeVector(text, vocab);
|
|
440
|
+
const vec = computeVector(vecTextForRow(obs), vocab);
|
|
442
441
|
if (vec) {
|
|
443
442
|
insertStmt.run(obs.id, Buffer.from(vec.buffer), vocab.version, now);
|
|
444
443
|
updated++;
|
package/lib/search-core.mjs
CHANGED
|
@@ -28,6 +28,7 @@ import { sanitizeFtsQuery, relaxFtsQueryToOr, SESS_BM25, DEFAULT_DECAY_HALF_LIFE
|
|
|
28
28
|
import { cjkPrecisionOk, extractCjkLikePatterns } from '../nlp.mjs';
|
|
29
29
|
import { computeTier } from '../tier.mjs';
|
|
30
30
|
import { countSearchTotal, attachBodyTokens } from '../search-engine.mjs';
|
|
31
|
+
import { notLowSignalTitleClause } from '../scoring-sql.mjs';
|
|
31
32
|
|
|
32
33
|
/** Sanitize a user query to FTS5 syntax; optionally force OR semantics. */
|
|
33
34
|
export function buildSearchFtsQuery(query, { or = false } = {}) {
|
|
@@ -459,6 +460,11 @@ export async function coreRunSearchPipeline(ctx, opts) {
|
|
|
459
460
|
// ── Type-list fallback (MCP): obs_type set + 0 matches → list recent of that type ──
|
|
460
461
|
if (obsTypeFallback && results.length === 0 && obsType) {
|
|
461
462
|
const typeWheres = ['COALESCE(compressed_into, 0) = 0', 'superseded_at IS NULL', 'type = ?'];
|
|
463
|
+
// Mirror the FTS path's low-signal filter (buildObsFtsQuery): this fallback is still a
|
|
464
|
+
// SEARCH surface, so degraded titles ("Modified X", "Error: …") must not lead it —
|
|
465
|
+
// acute for obs_type='change', the noise band. (The no-query recent-listing in
|
|
466
|
+
// searchObservationsHybrid deliberately does NOT filter — it mirrors mem_recent.)
|
|
467
|
+
if (!includeNoise) typeWheres.push(notLowSignalTitleClause(''));
|
|
462
468
|
const typeParams = [obsType];
|
|
463
469
|
if (project) { typeWheres.push('project = ?'); typeParams.push(project); }
|
|
464
470
|
if (epochFrom !== null) { typeWheres.push('created_at_epoch >= ?'); typeParams.push(epochFrom); }
|
package/mem-cli.mjs
CHANGED
|
@@ -1538,7 +1538,15 @@ function cmdUpdate(db, args) {
|
|
|
1538
1538
|
}
|
|
1539
1539
|
updates.push('title = ?'); params.push(scrubSecrets(flags.title));
|
|
1540
1540
|
}
|
|
1541
|
-
if (flags.narrative !== undefined) {
|
|
1541
|
+
if (flags.narrative !== undefined) {
|
|
1542
|
+
// Reject empty (mirror --title): an explicit '' would blank the narrative
|
|
1543
|
+
// irrecoverably (update takes no snapshot). Omit the flag to leave it unchanged.
|
|
1544
|
+
if (typeof flags.narrative === 'string' && flags.narrative.trim() === '') {
|
|
1545
|
+
fail('[mem] --narrative cannot be empty. Omit the flag to leave it unchanged.');
|
|
1546
|
+
return;
|
|
1547
|
+
}
|
|
1548
|
+
updates.push('narrative = ?'); params.push(scrubSecrets(flags.narrative));
|
|
1549
|
+
}
|
|
1542
1550
|
if (flags.type) {
|
|
1543
1551
|
const validTypes = new Set(['decision', 'bugfix', 'feature', 'refactor', 'discovery', 'change']);
|
|
1544
1552
|
if (!validTypes.has(flags.type)) {
|
|
@@ -1566,13 +1574,23 @@ function cmdUpdate(db, args) {
|
|
|
1566
1574
|
fail(`[mem] --lesson too long (${rawLesson.length} chars, max 500).`);
|
|
1567
1575
|
return;
|
|
1568
1576
|
}
|
|
1577
|
+
if (typeof rawLesson === 'string' && rawLesson.trim() === '') {
|
|
1578
|
+
fail('[mem] --lesson cannot be empty. Omit the flag to leave it unchanged.');
|
|
1579
|
+
return;
|
|
1580
|
+
}
|
|
1569
1581
|
updates.push('lesson_learned = ?');
|
|
1570
1582
|
params.push(scrubSecrets(rawLesson));
|
|
1571
1583
|
}
|
|
1572
1584
|
// Scrub like the sibling text fields above (title/narrative/lesson) and the MCP twin
|
|
1573
1585
|
// mem_update — concepts is a scrub-target + FTS-indexed column, so a raw secret here
|
|
1574
1586
|
// lands searchable + exportable (rebuildObservationDerived folds it into `text`).
|
|
1575
|
-
if (flags.concepts !== undefined) {
|
|
1587
|
+
if (flags.concepts !== undefined) {
|
|
1588
|
+
if (typeof flags.concepts === 'string' && flags.concepts.trim() === '') {
|
|
1589
|
+
fail('[mem] --concepts cannot be empty. Omit the flag to leave it unchanged.');
|
|
1590
|
+
return;
|
|
1591
|
+
}
|
|
1592
|
+
updates.push('concepts = ?'); params.push(scrubSecrets(flags.concepts));
|
|
1593
|
+
}
|
|
1576
1594
|
|
|
1577
1595
|
if (updates.length === 0) {
|
|
1578
1596
|
fail('[mem] No fields to update. Use --title, --type, --importance, --lesson/--lesson-learned, --narrative, --concepts');
|
|
@@ -1664,7 +1682,7 @@ function cmdExport(db, args) {
|
|
|
1664
1682
|
// id + memory_session_id are informational (restore remaps id and buckets under
|
|
1665
1683
|
// a restore session).
|
|
1666
1684
|
const rows = db.prepare(`
|
|
1667
|
-
SELECT id, memory_session_id, project, type, title, subtitle, narrative, concepts, facts,
|
|
1685
|
+
SELECT id, memory_session_id, project, type, title, subtitle, narrative, text, concepts, facts,
|
|
1668
1686
|
files_read, files_modified, lesson_learned, search_aliases, importance, branch,
|
|
1669
1687
|
access_count, cited_count, uncited_streak, injection_count, decay_seen_count,
|
|
1670
1688
|
last_accessed_at, created_at, created_at_epoch
|
|
@@ -1740,6 +1758,7 @@ function cmdRestore(db, argv) {
|
|
|
1740
1758
|
|
|
1741
1759
|
const dupCheck = db.prepare('SELECT id FROM observations WHERE project = ? AND title = ? AND created_at_epoch = ? LIMIT 1');
|
|
1742
1760
|
const signalUpdate = db.prepare(`UPDATE observations SET
|
|
1761
|
+
text = COALESCE(?, text),
|
|
1743
1762
|
subtitle = ?, concepts = ?, facts = ?, search_aliases = ?, files_read = ?, branch = COALESCE(?, branch),
|
|
1744
1763
|
access_count = ?, cited_count = ?, uncited_streak = ?, injection_count = ?,
|
|
1745
1764
|
decay_seen_count = ?, last_accessed_at = ?
|
|
@@ -1770,15 +1789,20 @@ function cmdRestore(db, argv) {
|
|
|
1770
1789
|
});
|
|
1771
1790
|
if (res.kind !== 'saved') { skipped++; continue; } // saveObservation Jaccard dedup
|
|
1772
1791
|
// Re-apply the fields saveObservation zeros/derives so the backup is faithful.
|
|
1773
|
-
//
|
|
1774
|
-
//
|
|
1775
|
-
//
|
|
1776
|
-
// (
|
|
1777
|
-
//
|
|
1778
|
-
//
|
|
1779
|
-
//
|
|
1780
|
-
// paths
|
|
1792
|
+
// `text` is the observation BODY and its own FTS5 column — import-jsonl / cold-start
|
|
1793
|
+
// rows keep the body there with an empty narrative, so saveObservation (content =
|
|
1794
|
+
// narrative || title) would collapse it to the bare title. COALESCE re-applies the
|
|
1795
|
+
// exported body (NULL when absent → keep saveObservation's derived text, so old
|
|
1796
|
+
// backups without the column degrade gracefully). search_aliases is its own FTS5
|
|
1797
|
+
// column too, so this UPDATE re-syncs the index (via the observations FTS triggers)
|
|
1798
|
+
// and restored body/aliases stay searchable. Scrub the FTS-indexed text fields on
|
|
1799
|
+
// the way in — the sibling ingest paths (import-jsonl, compress-core) scrub as
|
|
1800
|
+
// defense-in-depth, and restore is the only rewrite path that skipped it. A backup
|
|
1801
|
+
// made before a SECRET_PATTERNS entry existed would otherwise re-index an old secret
|
|
1802
|
+
// in text/facts/concepts. files_read/branch are paths/identifiers, not scrub-target
|
|
1803
|
+
// text — left as-is.
|
|
1781
1804
|
signalUpdate.run(
|
|
1805
|
+
r.text ? scrubSecrets(r.text) : null,
|
|
1782
1806
|
scrubSecrets(r.subtitle || ''), scrubSecrets(r.concepts || ''), scrubSecrets(r.facts || ''),
|
|
1783
1807
|
r.search_aliases === null || r.search_aliases === undefined ? null : scrubSecrets(r.search_aliases),
|
|
1784
1808
|
r.files_read || '[]', r.branch ?? null,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "claude-mem-lite",
|
|
3
|
-
"version": "3.
|
|
3
|
+
"version": "3.42.0",
|
|
4
4
|
"description": "Persistent long-term memory for Claude Code via MCP — captures coding decisions, bugfixes, and context across sessions. Hybrid FTS5 + TF-IDF search with episode batching. Single SQLite DB, no external services. A lighter, lower-cost alternative to claude-mem (episode batching + a smaller model; cost savings are an internal estimate, not a measured benchmark).",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"packageManager": "npm@10.9.2",
|
package/registry.mjs
CHANGED
|
@@ -390,11 +390,16 @@ const UPSERT_SQL = `
|
|
|
390
390
|
capability_summary=CASE WHEN excluded.capability_summary != '' THEN excluded.capability_summary ELSE capability_summary END,
|
|
391
391
|
input_type=CASE WHEN excluded.input_type != '' THEN excluded.input_type ELSE input_type END,
|
|
392
392
|
output_type=CASE WHEN excluded.output_type != '' THEN excluded.output_type ELSE output_type END,
|
|
393
|
-
|
|
393
|
+
-- Preserve-on-empty (same class as the FTS text columns above): a partial re-import
|
|
394
|
+
-- omits prerequisites/complexity, so upsertResource supplies their DEFAULTS ('{}' /
|
|
395
|
+
-- 'intermediate'). No CLI flag sets these (only a full re-index does), so treating the
|
|
396
|
+
-- default as the "absent" sentinel is safe: a real re-index sends non-default values,
|
|
397
|
+
-- which still overwrite; a metadata edit sends the default, which now preserves.
|
|
398
|
+
prerequisites=CASE WHEN excluded.prerequisites NOT IN ('', '{}') THEN excluded.prerequisites ELSE prerequisites END,
|
|
394
399
|
keywords=CASE WHEN excluded.keywords != '' THEN excluded.keywords ELSE keywords END,
|
|
395
400
|
tech_stack=CASE WHEN excluded.tech_stack != '' THEN excluded.tech_stack ELSE tech_stack END,
|
|
396
401
|
use_cases=CASE WHEN excluded.use_cases != '' THEN excluded.use_cases ELSE use_cases END,
|
|
397
|
-
complexity=excluded.complexity,
|
|
402
|
+
complexity=CASE WHEN excluded.complexity NOT IN ('', 'intermediate') THEN excluded.complexity ELSE complexity END,
|
|
398
403
|
indexed_at=excluded.indexed_at, updated_at=datetime('now')
|
|
399
404
|
`;
|
|
400
405
|
|
package/scripts/setup.sh
CHANGED
|
@@ -39,20 +39,18 @@ fi
|
|
|
39
39
|
mkdir -p "$DATA_DIR"
|
|
40
40
|
log_ok "Data directory: $DATA_DIR"
|
|
41
41
|
|
|
42
|
-
# 3.
|
|
42
|
+
# 3. Legacy ~/.claude-mem/ DB is schema-v16 (no memory_session_id) with no migration bridge to
|
|
43
|
+
# the current schema — activating it FATALs on first launch ("no such column: memory_session_id")
|
|
44
|
+
# and the "! -f claude-mem-lite.db" guard would re-copy it every time the user deletes the broken
|
|
45
|
+
# DB (recovery loop). Mirror install.mjs migrateLegacyClaudeMemData: back it up (don't activate)
|
|
46
|
+
# and let a fresh DB be created. Source ~/.claude-mem/ is left intact.
|
|
43
47
|
OLD_DIR="$HOME/.claude-mem"
|
|
44
48
|
if [[ -f "$OLD_DIR/claude-mem.db" && ! -f "$DATA_DIR/claude-mem-lite.db" && ! -f "$DATA_DIR/claude-mem.db" ]]; then
|
|
45
|
-
|
|
46
|
-
if cp "$OLD_DIR/claude-mem.db" "$
|
|
47
|
-
|
|
48
|
-
cp "$OLD_DIR/claude-mem.db-wal" "$DATA_DIR/claude-mem-lite.db-wal" 2>/dev/null || true
|
|
49
|
-
cp "$OLD_DIR/claude-mem.db-shm" "$DATA_DIR/claude-mem-lite.db-shm" 2>/dev/null || true
|
|
50
|
-
if [[ -d "$OLD_DIR/runtime" && ! -d "$DATA_DIR/runtime" ]]; then
|
|
51
|
-
cp -r "$OLD_DIR/runtime" "$DATA_DIR/runtime" 2>/dev/null || true
|
|
52
|
-
fi
|
|
53
|
-
log_ok "Data migrated (old ~/.claude-mem/ preserved)"
|
|
49
|
+
BACKUP="$DATA_DIR/claude-mem-lite.db.legacy-backup-$(date +%s)"
|
|
50
|
+
if cp "$OLD_DIR/claude-mem.db" "$BACKUP" 2>/dev/null; then
|
|
51
|
+
log_info "Legacy ~/.claude-mem/ DB is schema-incompatible; backed up to $(basename "$BACKUP") (a fresh DB will be created). Old ~/.claude-mem/ preserved."
|
|
54
52
|
else
|
|
55
|
-
log_warn "
|
|
53
|
+
log_warn "Legacy DB backup failed — a fresh database will be created"
|
|
56
54
|
fi
|
|
57
55
|
fi
|
|
58
56
|
|
package/search-engine.mjs
CHANGED
|
@@ -27,13 +27,13 @@ const FULL_SCORE = `${OBS_BM25}
|
|
|
27
27
|
* (CASE WHEN ? IS NOT NULL AND o.project = ? THEN 2.0 ELSE 1.0 END)
|
|
28
28
|
* (0.5 + 0.5 * COALESCE(o.importance, 1))
|
|
29
29
|
* (1.0 + 0.1 * LN(1 + COALESCE(o.access_count, 0)))
|
|
30
|
-
* (1.0 + 0.3 * (o.lesson_learned IS NOT NULL))`;
|
|
30
|
+
* (1.0 + 0.3 * (o.lesson_learned IS NOT NULL AND o.lesson_learned NOT IN ('', 'none')))`;
|
|
31
31
|
|
|
32
32
|
const SIMPLE_SCORE = `${OBS_BM25}
|
|
33
33
|
* (1.0 + EXP(-0.693 * MAX(0, ? - MAX(o.created_at_epoch, COALESCE(o.last_accessed_at, o.created_at_epoch))) / ${TYPE_DECAY_CASE}))
|
|
34
34
|
* ${TYPE_QUALITY_CASE}
|
|
35
35
|
* (0.5 + 0.5 * COALESCE(o.importance, 1))
|
|
36
|
-
* (1.0 + 0.3 * (o.lesson_learned IS NOT NULL))`;
|
|
36
|
+
* (1.0 + 0.3 * (o.lesson_learned IS NOT NULL AND o.lesson_learned NOT IN ('', 'none')))`;
|
|
37
37
|
|
|
38
38
|
export function buildObsFtsQuery(scoring, { multiplier, withSnippet, withOffset, includeNoise } = {}) {
|
|
39
39
|
const scoreExpr = scoring === 'full' ? FULL_SCORE : SIMPLE_SCORE;
|
package/secret-scrub.mjs
CHANGED
|
@@ -44,6 +44,11 @@ export const SECRET_PATTERNS = [
|
|
|
44
44
|
// low-FP decision that `topsecret=` / `access_token_count:` are non-credentials
|
|
45
45
|
// (#8283 + utils.test.mjs:1089-1100); bare `pwd` is omitted so `PWD=` (a path) survives.
|
|
46
46
|
[/((?:\b|_)(?:api[_-]?key|api[_-]?secret|secret[_-]?key|access[_-]?key|private[_-]?key|client[_-]?secret|auth[_-]?token|access[_-]?token|refresh[_-]?token|pgpassword|pgpass|mysql_pwd)\s*[=:]\s*)(?!process\.env\.)(?!new\s)(?!\w+\()(?!(?:null|undefined|true|false|None|nil|empty|""|''|0)\b)[^\s,;'"}\]]{6,}/gi, '$1***'],
|
|
47
|
+
// Space-separated credential CLI flag: `--password <value>` (long-form). The KV
|
|
48
|
+
// patterns above require `=`/`:`; the shell long-flag form uses a space. Long-form
|
|
49
|
+
// only — `-p`/`-u` short flags collide with unit/user/update flags (too FP-risky).
|
|
50
|
+
// `(?!-)` stops it eating a following `--flag` when --password has no value.
|
|
51
|
+
[/(--(?:password|passwd)[=\s]+)(?!-)[^\s'"]{6,}/gi, '$1***'],
|
|
47
52
|
// Bare-key QUOTED values — `api_key="..."`, `password: '...'`. The unquoted KV
|
|
48
53
|
// patterns above stop at `'`/`"` (excluded from their value class), so a quoted
|
|
49
54
|
// value matched 0 chars and slipped through. Consumes the opening quote, the value,
|
|
@@ -68,35 +73,46 @@ export const SECRET_PATTERNS = [
|
|
|
68
73
|
[/\bsk-(?:proj|ant|ant-api\d{2})-[a-zA-Z0-9_-]{8,}\b/g, '***'],
|
|
69
74
|
[/\bsk-[a-zA-Z0-9_-]{20,}\b/g, '***'],
|
|
70
75
|
// GitHub tokens (ghp_, gho_, github_pat_)
|
|
71
|
-
[/\b(?:ghp_|gho_|ghs_|ghr_)[a-zA-Z0-9_]{30,}\b/g, '***'],
|
|
76
|
+
[/\b(?:ghp_|gho_|ghs_|ghr_|ghu_)[a-zA-Z0-9_]{30,}\b/g, '***'],
|
|
72
77
|
[/\bgithub_pat_[a-zA-Z0-9_]{22,}\b/g, '***'],
|
|
73
78
|
// GitLab tokens (glpat-)
|
|
74
79
|
[/\bglpat-[a-zA-Z0-9_-]{20,}\b/g, '***'],
|
|
75
|
-
// Slack tokens (xox[
|
|
76
|
-
[/\
|
|
80
|
+
// Slack tokens (xox[bpasr]-, xapp-, xoxe-)
|
|
81
|
+
[/\b(?:xox[bpasr]|xapp|xoxe)-[a-zA-Z0-9-]{10,}\b/g, '***'],
|
|
82
|
+
// Slack incoming-webhook URL — the path after /services/ is the shared secret.
|
|
83
|
+
[/(https:\/\/hooks\.slack\.com\/services\/)[A-Za-z0-9/]+/g, '$1***'],
|
|
77
84
|
// JWT tokens (eyJ...eyJ...)
|
|
78
85
|
[/\beyJ[a-zA-Z0-9_-]{10,}\.eyJ[a-zA-Z0-9_-]{10,}\.[a-zA-Z0-9_-]+\b/g, '***'],
|
|
79
86
|
// PEM private key blocks. `[A-Z0-9 ]*` covers every armor label — RSA/EC/DSA/
|
|
80
87
|
// OPENSSH plus ENCRYPTED and PGP (… PRIVATE KEY BLOCK) — that the fixed
|
|
81
88
|
// alternation missed; the block delimiters make FP impossible.
|
|
82
89
|
[/-----BEGIN [A-Z0-9 ]*PRIVATE KEY(?: BLOCK)?-----[\s\S]*?-----END [A-Z0-9 ]*PRIVATE KEY(?: BLOCK)?-----/g, '***PEM_KEY***'],
|
|
83
|
-
// Long hex strings in assignments (e.g. SECRET_KEY=abc123def456...)
|
|
84
|
-
|
|
90
|
+
// Long hex strings in credential assignments (e.g. SECRET_KEY=abc123def456...).
|
|
91
|
+
// `hash` deliberately excluded: `hash: <40hex>` / `hash=<md5>` are git SHAs and
|
|
92
|
+
// checksums (real, preserved data in this hash-heavy repo), not credentials.
|
|
93
|
+
[/(\b(?:key|secret|token)\s*[=:]\s*)[0-9a-f]{32,}\b/gi, '$1***'],
|
|
85
94
|
// Google Cloud API keys (AIza...)
|
|
86
95
|
[/\bAIza[A-Za-z0-9_-]{35}\b/g, '***'],
|
|
87
|
-
//
|
|
88
|
-
|
|
96
|
+
// Authorization header credentials — Bearer (opaque), Basic (base64 user:pass),
|
|
97
|
+
// and GitHub's `token` scheme all carry secrets after the scheme word.
|
|
98
|
+
[/(Authorization:\s*(?:Bearer|Basic|token)\s+)[^\s,;'"}\]]+/gi, '$1***'],
|
|
89
99
|
// Supabase / generic long base64 keys (40+ chars, common in env vars)
|
|
90
100
|
[/(\b(?:SUPABASE_KEY|SUPABASE_ANON_KEY|SUPABASE_SERVICE_ROLE_KEY|DATABASE_URL|REDIS_URL)\s*[=:]\s*)[^\s,;'"}\]]+/gi, '$1***'],
|
|
91
101
|
// Basic auth in URLs (https://user:password@host). ftp/ftps added — file-drop
|
|
92
|
-
// creds are a common leak shape the https-only form missed.
|
|
93
|
-
|
|
94
|
-
//
|
|
95
|
-
|
|
102
|
+
// creds are a common leak shape the https-only form missed. The userinfo run
|
|
103
|
+
// EXCLUDES `:` (`[^@/\s:]+`) so the two runs can't overlap on a colon — the
|
|
104
|
+
// overlapping form caused O(n²) catastrophic backtracking on a colon-heavy
|
|
105
|
+
// non-terminating input (an availability DoS on the synchronous prompt path).
|
|
106
|
+
[/(https?|ftps?):\/\/[^@/\s:]+:[^@/\s]+@/gi, '$1://***:***@'],
|
|
107
|
+
// Database connection strings (postgres, mysql, mariadb, mssql, mongodb, redis,
|
|
108
|
+
// amqp) incl. their TLS/alias variants (rediss/amqps/mssql/sqlserver) — managed
|
|
109
|
+
// cloud DBs almost always use the TLS scheme, which the base-only list leaked.
|
|
110
|
+
[/\b(postgres(?:ql)?|mysql|mariadb|mssql|sqlserver|mongodb(?:\+srv)?|rediss?|amqps?):\/\/[^\s,;'"}\]]+/gi, '$1://***'],
|
|
96
111
|
// npm tokens (npm_...)
|
|
97
112
|
[/\bnpm_[a-zA-Z0-9]{36,}\b/g, '***'],
|
|
98
|
-
// Stripe keys (sk_live_, rk_live_, pk_live_, sk_test_, pk_test_)
|
|
113
|
+
// Stripe keys (sk_live_, rk_live_, pk_live_, sk_test_, pk_test_) + webhook signing secret (whsec_)
|
|
99
114
|
[/\b[srp]k_(?:live|test)_[a-zA-Z0-9]{20,}\b/g, '***'],
|
|
115
|
+
[/\bwhsec_[a-zA-Z0-9]{20,}\b/g, '***'],
|
|
100
116
|
// SendGrid API keys: SG.<22>.<43> — two dots at fixed offsets make this
|
|
101
117
|
// structurally unmistakable; near-zero false-positive risk.
|
|
102
118
|
[/\bSG\.[A-Za-z0-9_-]{22}\.[A-Za-z0-9_-]{43}\b/g, '***'],
|
package/tfidf.mjs
CHANGED
|
@@ -207,6 +207,19 @@ let _vocabCache = null;
|
|
|
207
207
|
/** Reset vocabulary cache (for testing). */
|
|
208
208
|
export function _resetVocabCache() { _vocabCache = null; }
|
|
209
209
|
|
|
210
|
+
/**
|
|
211
|
+
* Canonical TF-IDF vector text for an observation ROW (snake_case DB fields). Single source for
|
|
212
|
+
* EVERY (re)build path — save, enrich, rebuild_vectors, cluster-merge, compress, normalize — so
|
|
213
|
+
* they encode the identical field set and can't drift (V-F1). Mirrors the FTS-searchable columns
|
|
214
|
+
* incl. lesson_learned + search_aliases (finding #8). `concepts` may be an array or a string.
|
|
215
|
+
*/
|
|
216
|
+
export function vecTextForRow(row) {
|
|
217
|
+
if (!row) return '';
|
|
218
|
+
const concepts = Array.isArray(row.concepts) ? row.concepts.join(' ') : (row.concepts || '');
|
|
219
|
+
return [row.title || '', row.narrative || '', concepts, row.lesson_learned || '', row.search_aliases || '']
|
|
220
|
+
.filter(Boolean).join(' ');
|
|
221
|
+
}
|
|
222
|
+
|
|
210
223
|
/**
|
|
211
224
|
* Build global vocabulary (IDF table) from all active observations.
|
|
212
225
|
* @param {object} db - better-sqlite3 database
|
|
@@ -214,7 +227,7 @@ export function _resetVocabCache() { _vocabCache = null; }
|
|
|
214
227
|
*/
|
|
215
228
|
export function buildVocabulary(db, { dim = VOCAB_DIM } = {}) {
|
|
216
229
|
const rows = db.prepare(`
|
|
217
|
-
SELECT title, narrative, concepts FROM observations
|
|
230
|
+
SELECT title, narrative, concepts, lesson_learned, search_aliases FROM observations
|
|
218
231
|
WHERE COALESCE(compressed_into, 0) = 0 AND superseded_at IS NULL
|
|
219
232
|
`).all();
|
|
220
233
|
|
|
@@ -224,7 +237,10 @@ export function buildVocabulary(db, { dim = VOCAB_DIM } = {}) {
|
|
|
224
237
|
// Count document frequency for each term
|
|
225
238
|
const df = new Map();
|
|
226
239
|
for (const row of rows) {
|
|
227
|
-
|
|
240
|
+
// V-F2: include lesson_learned + search_aliases so terms living only there get a vocab
|
|
241
|
+
// dimension (else computeVector silently drops them — the exact paraphrase-bridge terms
|
|
242
|
+
// search_aliases exists to carry). Mirrors vecTextForRow's field set.
|
|
243
|
+
const text = [row.title || '', row.narrative || '', row.concepts || '', row.lesson_learned || '', row.search_aliases || ''].join(' ');
|
|
228
244
|
const docTerms = new Set(tokenize(text));
|
|
229
245
|
for (const term of docTerms) {
|
|
230
246
|
df.set(term, (df.get(term) || 0) + 1);
|
package/tool-schemas.mjs
CHANGED
|
@@ -237,13 +237,15 @@ export const memUpdateSchema = {
|
|
|
237
237
|
// CLI parity (cmdUpdate): empty/whitespace title would render as `(untitled)`
|
|
238
238
|
// in every listing — reject here like the CLI does, instead of persisting it.
|
|
239
239
|
title: z.string().refine(s => s.trim() !== '', 'title cannot be empty').optional().describe('New title'),
|
|
240
|
-
|
|
240
|
+
// Reject empty content fields (parity with `title` above + cmdUpdate): an explicit
|
|
241
|
+
// '' would blank narrative/lesson/concepts irrecoverably (mem_update takes no snapshot).
|
|
242
|
+
narrative: z.string().refine(s => s.trim() !== '', 'narrative cannot be empty').optional().describe('New narrative/content'),
|
|
241
243
|
type: OBS_TYPE_ENUM.optional().describe('New observation type'),
|
|
242
244
|
importance: coerceInt.pipe(z.number().int().min(1).max(3)).optional().describe('New importance (1-3)'),
|
|
243
245
|
// 500-char cap mirrors memSaveSchema + cmdUpdate — update was the one path
|
|
244
246
|
// that let overlong lessons leak into the DB via MCP.
|
|
245
|
-
lesson_learned: z.string().max(500).optional().describe('Add or update lesson learned'),
|
|
246
|
-
concepts: z.string().optional().describe('Space-separated concept tags'),
|
|
247
|
+
lesson_learned: z.string().max(500).refine(s => s.trim() !== '', 'lesson_learned cannot be empty').optional().describe('Add or update lesson learned'),
|
|
248
|
+
concepts: z.string().refine(s => s.trim() !== '', 'concepts cannot be empty').optional().describe('Space-separated concept tags'),
|
|
247
249
|
};
|
|
248
250
|
|
|
249
251
|
export const memExportSchema = {
|
package/utils.mjs
CHANGED
|
@@ -404,7 +404,13 @@ export function isMetaTriggerPrompt(text) {
|
|
|
404
404
|
.replace(/上次(到哪了|说到哪了)?/g, '')
|
|
405
405
|
.replace(/总结一下|复盘一下/g, '')
|
|
406
406
|
.replace(/前面(的)?(工作|话题|讨论|内容)/g, '')
|
|
407
|
+
// R3 H-M2: control phrases named in lesson #8287 / common continuations. Phrase-anchored
|
|
408
|
+
// (or bare suffix/prefix) so a real subject that merely contains one keeps its other tokens.
|
|
409
|
+
.replace(/停了|停不下来/g, '')
|
|
410
|
+
.replace(/怎么(回事|了|样了|停的)?/g, '')
|
|
411
|
+
.replace(/再来(一次|一遍|一下)?/g, '')
|
|
407
412
|
.replace(/\/?(clear|exit)\b/gi, '')
|
|
413
|
+
.replace(/\b(go on|go ahead|keep going|carry on|proceed|why(?:'?d| did) you stop)\b/gi, '')
|
|
408
414
|
.replace(/\b(commit|continue|resume|push|save|restart|exit|next)\b/gi, '')
|
|
409
415
|
.replace(/[,,。.!!??::;;()()【】[\]\s/\\-]+/g, '')
|
|
410
416
|
.trim();
|