claude-mem-lite 2.28.2 → 2.30.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -10,7 +10,7 @@
10
10
  "plugins": [
11
11
  {
12
12
  "name": "claude-mem-lite",
13
- "version": "2.28.2",
13
+ "version": "2.30.0",
14
14
  "source": "./",
15
15
  "description": "Lightweight persistent memory system for Claude Code — FTS5 search, episode batching, error-triggered recall"
16
16
  }
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "claude-mem-lite",
3
- "version": "2.28.2",
3
+ "version": "2.30.0",
4
4
  "description": "Lightweight persistent memory system for Claude Code — FTS5 search, episode batching, error-triggered recall",
5
5
  "author": {
6
6
  "name": "sdsrss"
package/cli.mjs CHANGED
@@ -1,5 +1,5 @@
1
1
  #!/usr/bin/env node
2
- const CLI_COMMANDS = new Set(['search', 'recent', 'recall', 'get', 'timeline', 'save', 'stats', 'context', 'browse', 'delete', 'update', 'export', 'compress', 'maintain', 'fts-check', 'registry', 'import', 'enrich', 'help']);
2
+ const CLI_COMMANDS = new Set(['search', 'recent', 'recall', 'get', 'timeline', 'save', 'stats', 'context', 'browse', 'delete', 'update', 'export', 'compress', 'maintain', 'optimize', 'fts-check', 'registry', 'import', 'enrich', 'help']);
3
3
  const INSTALL_COMMANDS = new Set(['install', 'uninstall', 'status', 'doctor', 'cleanup', 'cleanup-hooks', 'self-update', 'release']);
4
4
 
5
5
  const cmd = process.argv[2];
package/commands/mem.md CHANGED
@@ -1,5 +1,6 @@
1
1
  ---
2
- description: "Search and manage project memory (observations, sessions, prompts). Use when: user asks about past work, wants to find a previous bugfix, check project history, save a decision, or manage stored memories"
2
+ name: mem
3
+ description: "Use when: querying past work, managing memories, or checking project history"
3
4
  ---
4
5
 
5
6
  # Memory
@@ -1,5 +1,6 @@
1
1
  ---
2
- description: "Save content to memory — with explicit content, instructions, or auto-summarize current session. Use when: the user asks to remember something, after solving a non-obvious problem, or to capture key session findings"
2
+ name: memory
3
+ description: "Use when: user asks to remember something, after solving a non-obvious problem, or to capture key session findings"
3
4
  ---
4
5
 
5
6
  # Memory Save
package/commands/tools.md CHANGED
@@ -1,5 +1,6 @@
1
1
  ---
2
- description: "Import skills and agents from GitHub repositories into the tool resource registry. Use when: looking for a skill to solve a problem, importing tools from a repo, or managing installed tools"
2
+ name: tools
3
+ description: "Use when: importing skills/agents from GitHub, managing registry resources, or searching for tools to solve a problem"
3
4
  ---
4
5
 
5
6
  # Tool Import
@@ -1,5 +1,6 @@
1
1
  ---
2
- description: "Auto-maintain memory and resource registry — deduplicate, merge, decay, cleanup, reindex. Use when: search results seem noisy, after bulk imports, or during periodic maintenance"
2
+ name: update
3
+ description: "Use when: search results seem noisy, after bulk imports, or for periodic memory/registry maintenance"
3
4
  ---
4
5
 
5
6
  # Memory & Registry Maintenance
package/haiku-client.mjs CHANGED
@@ -100,6 +100,109 @@ export async function callHaikuJSON(prompt, opts) {
100
100
  return parseJsonFromLLM(result.text);
101
101
  }
102
102
 
103
+ // ─── Model-Selectable API ────────────────────────────────────────────────────
104
+
105
+ /**
106
+ * Call LLM with explicit model selection. Supports 'haiku' and 'sonnet'.
107
+ * Reuses existing API/CLI dual-mode infrastructure.
108
+ * Never throws — returns null on any error.
109
+ *
110
+ * @param {string} prompt The prompt text
111
+ * @param {'haiku'|'sonnet'} model Model to use (default: 'haiku')
112
+ * @param {object} [opts] Options
113
+ * @param {number} [opts.timeout=15000] Timeout in milliseconds
114
+ * @param {number} [opts.maxTokens=1000] Max tokens in response
115
+ * @returns {Promise<{text: string}|null>} Response or null on failure
116
+ */
117
+ export async function callLLMWithModel(prompt, model = 'haiku', { timeout = 15000, maxTokens = 1000 } = {}) {
118
+ if (!prompt) return null;
119
+ const resolvedModel = MODEL_MAP[model] ? model : 'haiku';
120
+ const mode = detectMode();
121
+
122
+ try {
123
+ if (mode === 'api') {
124
+ return await callModelAPI(prompt, resolvedModel, { timeout, maxTokens });
125
+ }
126
+ return callModelCLI(prompt, resolvedModel, { timeout });
127
+ } catch (e) {
128
+ debugCatch(e, `callLLMWithModel:${resolvedModel}`);
129
+ return null;
130
+ }
131
+ }
132
+
133
+ /**
134
+ * Call LLM with model selection and parse JSON response.
135
+ * @param {string} prompt
136
+ * @param {'haiku'|'sonnet'} model
137
+ * @param {object} [opts]
138
+ * @returns {Promise<object|null>}
139
+ */
140
+ export async function callModelJSON(prompt, model = 'haiku', opts) {
141
+ const result = await callLLMWithModel(prompt, model, opts);
142
+ if (!result?.text) return null;
143
+ return parseJsonFromLLM(result.text);
144
+ }
145
+
146
+ async function callModelAPI(prompt, model, { timeout, maxTokens }) {
147
+ const apiKey = process.env.ANTHROPIC_API_KEY;
148
+ if (!apiKey) return null;
149
+
150
+ const modelId = MODEL_MAP[model];
151
+ const controller = new AbortController();
152
+ const timer = setTimeout(() => controller.abort(), timeout);
153
+
154
+ try {
155
+ const res = await fetch('https://api.anthropic.com/v1/messages', {
156
+ method: 'POST',
157
+ headers: {
158
+ 'Content-Type': 'application/json',
159
+ 'x-api-key': apiKey,
160
+ 'anthropic-version': '2023-06-01',
161
+ },
162
+ body: JSON.stringify({
163
+ model: modelId,
164
+ max_tokens: maxTokens,
165
+ messages: [{ role: 'user', content: prompt }],
166
+ }),
167
+ signal: controller.signal,
168
+ });
169
+
170
+ if (!res.ok) {
171
+ debugLog('WARN', `${model}-api`, `HTTP ${res.status}`);
172
+ return null;
173
+ }
174
+
175
+ const data = await res.json();
176
+ const text = data.content?.[0]?.text;
177
+ return text ? { text } : null;
178
+ } finally {
179
+ clearTimeout(timer);
180
+ }
181
+ }
182
+
183
+ function callModelCLI(prompt, model, { timeout }) {
184
+ const modelName = MODEL_MAP[model] ? model : 'haiku';
185
+ try {
186
+ const result = execFileSync(getClaudePath(), ['-p', '--model', modelName], {
187
+ input: prompt,
188
+ timeout,
189
+ encoding: 'utf8',
190
+ env: { ...process.env, CLAUDE_MEM_HOOK_RUNNING: '1' },
191
+ stdio: ['pipe', 'pipe', 'pipe'],
192
+ cwd: '/tmp',
193
+ });
194
+ const text = result.trim();
195
+ return text ? { text } : null;
196
+ } catch (e) {
197
+ const out = e.stdout?.toString?.()?.trim() || e.output?.[1]?.toString?.()?.trim();
198
+ if (out && out.startsWith('{') && out.endsWith('}')) {
199
+ try { JSON.parse(out); return { text: out }; } catch {}
200
+ }
201
+ debugCatch(e, `${model}-cli`);
202
+ return null;
203
+ }
204
+ }
205
+
103
206
  // ─── API Mode ────────────────────────────────────────────────────────────────
104
207
 
105
208
  async function callHaikuAPI(prompt, { timeout, maxTokens }) {
package/hook-context.mjs CHANGED
@@ -1,9 +1,15 @@
1
1
  // claude-mem-lite CLAUDE.md context injection and token budgeting
2
- // Handles adaptive time windows, token-budgeted selection, and CLAUDE.md persistence
2
+ // Handles adaptive time windows, token-budgeted selection, and legacy CLAUDE.md cleanup.
3
3
 
4
- import { join } from 'path';
4
+ import { basename, join } from 'path';
5
5
  import { readFileSync, writeFileSync, renameSync, unlinkSync } from 'fs';
6
- import { estimateTokens, truncate, debugLog, debugCatch, DECAY_HALF_LIFE_BY_TYPE, DEFAULT_DECAY_HALF_LIFE_MS } from './utils.mjs';
6
+ import {
7
+ estimateTokens, truncate, typeIcon, fmtTime,
8
+ debugLog, debugCatch,
9
+ DECAY_HALF_LIFE_BY_TYPE, DEFAULT_DECAY_HALF_LIFE_MS, notLowSignalTitleClause,
10
+ } from './utils.mjs';
11
+ import { STALE_SESSION_MS, FALLBACK_OBS_WINDOW_MS } from './hook-shared.mjs';
12
+ import { extractUnfinishedSummary } from './hook-handoff.mjs';
7
13
 
8
14
  /**
9
15
  * Infer the project directory from environment variables or cwd.
@@ -56,11 +62,15 @@ export function selectWithTokenBudget(db, project, budget = 2000) {
56
62
  const tier2Ago = now_ms - windows.tier2;
57
63
  const tier3Ago = now_ms - windows.tier3;
58
64
 
59
- // Candidate pool: tiered time windows by importance (adaptive)
65
+ // Candidate pool: tiered time windows by importance (adaptive).
66
+ // R1/R3: exclude LOW_SIGNAL degraded titles ("Modified X", "Worked on X",
67
+ // "Reviewed N files:", raw error logs) from the Key Context table at
68
+ // session start — they pollute the visible "Recent" table with noise.
60
69
  const obsPool = db.prepare(`
61
70
  SELECT id, type, title, narrative, importance, created_at_epoch, files_modified, lesson_learned
62
71
  FROM observations
63
72
  WHERE project = ? AND COALESCE(compressed_into, 0) = 0
73
+ AND ${notLowSignalTitleClause('')}
64
74
  AND (
65
75
  (created_at_epoch > ? AND importance >= 1)
66
76
  OR (created_at_epoch > ? AND importance >= 2)
@@ -82,9 +92,11 @@ export function selectWithTokenBudget(db, project, budget = 2000) {
82
92
  const selectedSess = [];
83
93
  let totalTokens = 0;
84
94
 
85
- // Type quality multipliers — aligned with scoring-sql.mjs TYPE_QUALITY_CASE
86
- // Demotes bugfix (noisy error logs) and promotes high-signal types
87
- const TYPE_QUALITY = { decision: 1.5, discovery: 1.3, feature: 1.2, refactor: 1.0, change: 0.8, bugfix: 0.35 };
95
+ // Type quality multipliers — aligned with scoring-sql.mjs TYPE_QUALITY_CASE (R2).
96
+ // Weights calibrated from empirical avg access_count per type:
97
+ // decision 6.05, discovery 3.32, bugfix 2.24, feature 2.04, change 0.93, refactor 0.54.
98
+ // Pre-R2 had bugfix=0.35 (inverted vs reality — bugfixes are 2.4× more used than changes).
99
+ const TYPE_QUALITY = { decision: 1.5, discovery: 1.3, bugfix: 1.1, feature: 1.0, refactor: 0.6, change: 0.5 };
88
100
 
89
101
  // Score each candidate: value = recency * type_quality * importance, cost = tokens
90
102
  // Recency uses exponential half-life (consistent with server.mjs BM25 scoring)
@@ -154,48 +166,217 @@ export function selectWithTokenBudget(db, project, budget = 2000) {
154
166
  }
155
167
 
156
168
  /**
157
- * Update the project's CLAUDE.md file with a context block.
158
- * Replaces existing <claude-mem-context> section or appends a new one.
159
- * Uses atomic tmp+rename write to prevent partial writes.
160
- * @param {string} contextBlock Markdown content to inject
169
+ * One-time cleanup of the legacy <claude-mem-context> block from the project's
170
+ * CLAUDE.md file. Pre-v2.30 the hook wrote a slim context snapshot here on every
171
+ * session start, causing constant git noise and stale, one-session-behind content.
172
+ * Context is now delivered exclusively via SessionStart hook stdout.
173
+ *
174
+ * Idempotent: if no legacy block (or no CLAUDE.md) exists, it is a no-op. Also
175
+ * removes the paired hint comment if present, and normalizes residual whitespace
176
+ * at the seam. Uses atomic tmp+rename write.
161
177
  */
162
- export function updateClaudeMd(contextBlock) {
178
+ export function cleanupClaudeMdLegacyBlock() {
163
179
  const claudeMdPath = join(inferProjectDir(), 'CLAUDE.md');
164
- let content = '';
165
- try { content = readFileSync(claudeMdPath, 'utf8'); } catch {}
180
+ let content;
181
+ try { content = readFileSync(claudeMdPath, 'utf8'); } catch { return; }
166
182
 
167
183
  const startTag = '<claude-mem-context>';
168
184
  const endTag = '</claude-mem-context>';
169
- const hintComment = '<!-- claude-mem-lite: auto-updated context. To avoid git noise, add CLAUDE.md to .gitignore -->';
170
- const newSection = `${startTag}\n${contextBlock}\n${endTag}`;
171
185
 
172
- // Use lastIndexOf for both tags prevents matching documentation references
173
- // to <claude-mem-context> that appear in code/markdown before the actual context block
186
+ // Use lastIndexOf so documentation references to the tag earlier in the file
187
+ // (e.g. inside a code block in architecture notes) are not accidentally swept.
174
188
  const startIdx = content.lastIndexOf(startTag);
175
189
  const endIdx = content.lastIndexOf(endTag);
190
+ if (startIdx === -1 || endIdx === -1 || startIdx >= endIdx) return;
176
191
 
177
- if (startIdx !== -1 && endIdx !== -1 && startIdx < endIdx) {
178
- // Skip write if content is unchanged — reduces git noise
179
- const existingSection = content.slice(startIdx, endIdx + endTag.length);
180
- if (existingSection === newSection) return;
181
- // Replace from first start to last end collapses any duplicate sections into one
182
- content = content.slice(0, startIdx) + newSection + content.slice(endIdx + endTag.length);
183
- } else if (content.length > 0) {
184
- // Append to end never disturb existing CLAUDE.md structure
185
- const hint = content.includes(hintComment) ? '' : hintComment + '\n';
186
- content = content.trimEnd() + '\n\n' + hint + newSection + '\n';
187
- } else {
188
- content = hintComment + '\n' + newSection + '\n';
192
+ // Extend forward to swallow a trailing newline so we don't leave a stranded blank line.
193
+ let removeEnd = endIdx + endTag.length;
194
+ if (content[removeEnd] === '\n') removeEnd += 1;
195
+
196
+ // Extend backward if the paired hint comment sits on the line immediately before
197
+ // the start tag. The hint is the exact string the old updateClaudeMd emitted.
198
+ let removeStart = startIdx;
199
+ const hintPattern = '<!-- claude-mem-lite: auto-updated context';
200
+ const leadingSlice = content.slice(0, startIdx);
201
+ const hintIdx = leadingSlice.lastIndexOf(hintPattern);
202
+ if (hintIdx !== -1) {
203
+ const between = content.slice(hintIdx, startIdx);
204
+ if (/^<!-- claude-mem-lite: [^\n]*-->\s*$/.test(between)) {
205
+ removeStart = hintIdx;
206
+ }
189
207
  }
190
208
 
209
+ // Swallow a single preceding newline to avoid leaving a blank-line gap behind.
210
+ if (removeStart > 0 && content[removeStart - 1] === '\n') removeStart -= 1;
211
+
212
+ const cleaned = content.slice(0, removeStart) + content.slice(removeEnd);
213
+ // Collapse any ≥3 consecutive newlines to two, then ensure exactly one trailing newline.
214
+ const normalized = cleaned.replace(/\n{3,}/g, '\n\n').replace(/\s*$/, '\n');
215
+
216
+ if (normalized === content) return;
217
+
191
218
  const tmp = claudeMdPath + '.mem-tmp';
192
219
  try {
193
- writeFileSync(tmp, content);
220
+ writeFileSync(tmp, normalized);
194
221
  renameSync(tmp, claudeMdPath);
195
222
  } catch (e) {
196
223
  try { unlinkSync(tmp); } catch {}
197
- debugLog('ERROR', 'updateClaudeMd', `CLAUDE.md write failed: ${e.message}`);
224
+ debugLog('ERROR', 'cleanupClaudeMdLegacyBlock', `CLAUDE.md write failed: ${e.message}`);
225
+ }
226
+ }
227
+
228
+ /**
229
+ * Assemble the full markdown body that goes inside the <claude-mem-context>
230
+ * block emitted at session start. Same shape as the inline builder hook.mjs
231
+ * used to compose directly; extracted so both the SessionStart hook AND the
232
+ * `claude-mem-lite context` CLI can read live context from the DB.
233
+ *
234
+ * Sections (in order):
235
+ * 1. Last Session (from session_summaries.latest)
236
+ * 2. File Lessons / Key Context (top importance≥2 observations)
237
+ * 3. Recent Activity fallback (when no summary and no key obs)
238
+ * 4. Working State (from latest clear handoff)
239
+ * 5. Recent (N) table (observations via selectWithTokenBudget + fallback)
240
+ *
241
+ * @param {import('better-sqlite3').Database} db Opened main DB
242
+ * @param {string} project Canonical project name (from inferProject())
243
+ * @param {Date} [now=new Date()] Clock reference for time windows and table header
244
+ * @returns {string} Joined markdown lines (without <claude-mem-context> wrappers)
245
+ */
246
+ export function buildSessionContextLines(db, project, now = new Date()) {
247
+ // 1. Token-budgeted observation selection
248
+ const selected = selectWithTokenBudget(db, project, 2000);
249
+ const observations = selected.observations;
250
+
251
+ // 2. Fallback: recent across all projects with tiered windows (when local pool is thin)
252
+ let fallbackObs = [];
253
+ if (observations.length < 3) {
254
+ const fbOneDayAgo = now.getTime() - STALE_SESSION_MS;
255
+ const fbSevenDaysAgo = now.getTime() - FALLBACK_OBS_WINDOW_MS;
256
+ fallbackObs = db.prepare(`
257
+ SELECT id, type, title, project, created_at
258
+ FROM observations
259
+ WHERE COALESCE(compressed_into, 0) = 0
260
+ AND (
261
+ (created_at_epoch > ? AND importance >= 1)
262
+ OR (created_at_epoch > ? AND importance >= 2)
263
+ )
264
+ ORDER BY created_at_epoch DESC
265
+ LIMIT 5
266
+ `).all(fbOneDayAgo, fbSevenDaysAgo);
267
+ }
268
+
269
+ // 3. Latest session summary → base summaryLines
270
+ const latestSummary = db.prepare(`
271
+ SELECT request, completed, next_steps, remaining_items, lessons, key_decisions, created_at
272
+ FROM session_summaries
273
+ WHERE project = ?
274
+ ORDER BY created_at_epoch DESC
275
+ LIMIT 1
276
+ `).get(project);
277
+
278
+ const summaryLines = buildSummaryLines(latestSummary);
279
+
280
+ // 4. Key context: top high-importance observations split into File Lessons (actionable)
281
+ // and Key Context (informational). Pushed into summaryLines.
282
+ const keyObs = db.prepare(`
283
+ SELECT o.id, o.type, o.title, o.lesson_learned, o.files_modified FROM observations o
284
+ WHERE o.project = ? AND COALESCE(o.compressed_into, 0) = 0
285
+ AND o.superseded_at IS NULL
286
+ AND COALESCE(o.importance, 1) >= 2
287
+ ORDER BY o.created_at_epoch DESC LIMIT 10
288
+ `).all(project);
289
+
290
+ if (keyObs.length > 0) {
291
+ const fileLessons = [];
292
+ const keyContext = [];
293
+
294
+ for (const o of keyObs) {
295
+ const clean = (o.title || '(untitled)')
296
+ .replace(/ → (?:ERROR: )?\{".*$/, '')
297
+ .replace(/ → (?:ERROR: )?\{[^}]*\.{3}$/, '');
298
+ const hasLesson = o.lesson_learned && o.lesson_learned.trim();
299
+ const hasFiles = o.files_modified && o.files_modified !== '[]';
300
+
301
+ if (hasLesson && hasFiles) {
302
+ try {
303
+ const files = JSON.parse(o.files_modified);
304
+ const fname = basename(Array.isArray(files) && files.length > 0 ? files[0] : '');
305
+ if (fname) {
306
+ fileLessons.push(`- ${fname}: ${truncate(o.lesson_learned, 100)} (#${o.id})`);
307
+ continue;
308
+ }
309
+ } catch { /* fall through to keyContext */ }
310
+ }
311
+ const lesson = hasLesson ? ` — ${truncate(o.lesson_learned, 60)}` : '';
312
+ keyContext.push(`- [${o.type || 'discovery'}] ${truncate(clean, 80)} (#${o.id})${lesson}`);
313
+ }
314
+
315
+ if (fileLessons.length > 0) {
316
+ summaryLines.push('### File Lessons');
317
+ summaryLines.push(...fileLessons.slice(0, 5));
318
+ summaryLines.push('');
319
+ }
320
+ if (keyContext.length > 0) {
321
+ summaryLines.push('### Key Context');
322
+ summaryLines.push(...keyContext.slice(0, 5));
323
+ summaryLines.push('');
324
+ }
325
+ } else if (!latestSummary) {
326
+ // Fallback: no summary AND no key observations — show recent activity
327
+ const recentObs = (observations.length >= 3 ? observations : fallbackObs).slice(0, 3);
328
+ if (recentObs.length > 0) {
329
+ summaryLines.push('### Recent Activity');
330
+ for (const o of recentObs) {
331
+ summaryLines.push(`- ${truncate(o.title || '(untitled)', 80)}`);
332
+ }
333
+ summaryLines.push('');
334
+ }
198
335
  }
336
+
337
+ // 5. Working state from latest /clear handoff
338
+ const prevClearHandoff = db.prepare(`
339
+ SELECT working_on, unfinished, key_files
340
+ FROM session_handoffs
341
+ WHERE project = ? AND type = 'clear'
342
+ ORDER BY created_at_epoch DESC LIMIT 1
343
+ `).get(project);
344
+
345
+ const handoffLines = [];
346
+ if (prevClearHandoff) {
347
+ handoffLines.push('### Working State (from /clear)');
348
+ if (prevClearHandoff.working_on) {
349
+ handoffLines.push(`- Working on: ${truncate(prevClearHandoff.working_on, 200)}`);
350
+ }
351
+ if (prevClearHandoff.unfinished) {
352
+ const pendingSummary = extractUnfinishedSummary(prevClearHandoff.unfinished);
353
+ if (pendingSummary) handoffLines.push(`- Unfinished: ${truncate(pendingSummary, 200)}`);
354
+ }
355
+ if (prevClearHandoff.key_files) {
356
+ try {
357
+ const files = JSON.parse(prevClearHandoff.key_files);
358
+ if (files.length > 0) handoffLines.push(`- Key files: ${files.map(f => basename(f)).join(', ')}`);
359
+ } catch { /* malformed JSON — skip */ }
360
+ }
361
+ handoffLines.push('');
362
+ }
363
+
364
+ // 6. Recent observations table
365
+ const obsLines = [];
366
+ const obsToShow = observations.length >= 3 ? observations : fallbackObs;
367
+ if (obsToShow.length > 0) {
368
+ const today = now.toISOString().slice(0, 10);
369
+ obsLines.push(`### Recent (${today})`);
370
+ obsLines.push('');
371
+ obsLines.push('| ID | Time | T | Title |');
372
+ obsLines.push('|----|------|---|-------|');
373
+ for (const o of obsToShow) {
374
+ const proj = o.project && o.project !== project ? ` (${o.project})` : '';
375
+ obsLines.push(`| #${o.id} | ${fmtTime(o.created_at)} | ${typeIcon(o.type)} | ${truncate(o.title || '(untitled)', 60)}${proj} |`);
376
+ }
377
+ }
378
+
379
+ return [...summaryLines, ...handoffLines, ...obsLines].join('\n');
199
380
  }
200
381
 
201
382
  /**
package/hook-memory.mjs CHANGED
@@ -1,13 +1,20 @@
1
1
  // claude-mem-lite — Semantic Memory Injection
2
2
  // Search past observations for relevant memories to inject as context at user-prompt time.
3
3
 
4
- import { sanitizeFtsQuery, relaxFtsQueryToOr, debugCatch, OBS_BM25 } from './utils.mjs';
4
+ import { sanitizeFtsQuery, relaxFtsQueryToOr, debugCatch, OBS_BM25, notLowSignalTitleClause } from './utils.mjs';
5
5
 
6
6
  const MAX_MEMORY_INJECTIONS = 3;
7
7
  const MEMORY_LOOKBACK_MS = 60 * 86400000; // 60 days
8
- // Aligned with TYPE_QUALITY_CASE: high-signal types > noisy types
9
- // Bugfix lessons are still surfaced via the separate lesson_learned boost (1.5×)
10
- const MEMORY_TYPE_BOOST = { decision: 1.5, discovery: 1.3, feature: 1.2, refactor: 1.0, change: 0.8, bugfix: 0.5 };
8
+ // Aligned with TYPE_QUALITY_CASE in scoring-sql.mjs (R2 rebalance).
9
+ // Weights calibrated to empirical avg access_count:
10
+ // decision 6.05, discovery 3.32, bugfix 2.24, feature 2.04, change 0.93, refactor 0.54.
11
+ // lesson_learned boost (1.5×) stacks for entries with a real takeaway.
12
+ const MEMORY_TYPE_BOOST = { decision: 1.5, discovery: 1.3, bugfix: 1.1, feature: 1.0, refactor: 0.6, change: 0.5 };
13
+ // Adaptive BM25 thresholds — scale with corpus size to filter noise.
14
+ // Larger corpora produce more weak matches from common words.
15
+ const BM25_THRESHOLD = { TINY: 0, SMALL: 1.5, MEDIUM: 2.5, LARGE: 3.5 };
16
+ // OR fallback max token count — queries with 3+ tokens that fail AND are likely off-topic
17
+ const OR_FALLBACK_MAX_TOKENS = 2;
11
18
 
12
19
  const FILE_RECALL_LOOKBACK_MS = 60 * 86400000; // 60 days
13
20
  const MAX_FILE_RECALL = 2;
@@ -32,6 +39,9 @@ export function searchRelevantMemories(db, userPrompt, project, excludeIds = [])
32
39
  const excludeSet = new Set(excludeIds);
33
40
 
34
41
  // Phase 1: Same-project search (highest priority)
42
+ // R1: notLowSignalTitleClause() excludes hook-llm fallback titles
43
+ // ("Modified X", "Worked on X", "Reviewed N files:", raw error logs, etc.)
44
+ // that almost never get referenced (3.3% access rate) but compete for BM25 rank.
35
45
  const selectStmt = db.prepare(`
36
46
  SELECT o.id, o.type, o.title, o.importance, o.lesson_learned, o.project,
37
47
  ${OBS_BM25} as relevance
@@ -43,22 +53,30 @@ export function searchRelevantMemories(db, userPrompt, project, excludeIds = [])
43
53
  AND o.created_at_epoch > ?
44
54
  AND COALESCE(o.compressed_into, 0) = 0
45
55
  AND o.superseded_at IS NULL
56
+ AND ${notLowSignalTitleClause('o')}
46
57
  ORDER BY ${OBS_BM25}
47
58
  LIMIT 10
48
59
  `);
49
60
  let rows = selectStmt.all(ftsQuery, project, cutoff);
61
+ let usedOrFallback = false;
50
62
 
51
- // OR fallback when AND returns nothing
63
+ // OR fallback when AND returns nothing — only for short queries (specific enough).
64
+ // 3+ token queries that fail AND are likely off-topic; OR would match individual common words.
65
+ // Count original search terms (AND-separated groups), not expanded synonym tokens.
66
+ const queryTokenCount = ftsQuery.includes(' AND ')
67
+ ? ftsQuery.split(' AND ').length
68
+ : ftsQuery.split(/\s+/).filter(t => t && !t.startsWith('(') || !t.endsWith(')')).length;
52
69
  if (rows.length === 0) {
53
70
  const orQuery = relaxFtsQueryToOr(ftsQuery);
54
- if (orQuery) {
55
- try { rows = selectStmt.all(orQuery, project, cutoff); } catch {}
71
+ if (orQuery && queryTokenCount <= OR_FALLBACK_MAX_TOKENS) {
72
+ try { rows = selectStmt.all(orQuery, project, cutoff); usedOrFallback = true; } catch {}
56
73
  }
57
74
  }
58
75
 
59
76
  // Phase 2: Cross-project search for high-value decisions/discoveries
60
77
  // These are transferable insights (debugging patterns, architectural reasons, gotchas)
61
78
  let crossRows = [];
79
+ let crossUsedOr = false;
62
80
  try {
63
81
  const crossStmt = db.prepare(`
64
82
  SELECT o.id, o.type, o.title, o.importance, o.lesson_learned, o.project,
@@ -72,46 +90,51 @@ export function searchRelevantMemories(db, userPrompt, project, excludeIds = [])
72
90
  AND o.created_at_epoch > ?
73
91
  AND COALESCE(o.compressed_into, 0) = 0
74
92
  AND o.superseded_at IS NULL
93
+ AND ${notLowSignalTitleClause('o')}
75
94
  ORDER BY ${OBS_BM25}
76
95
  LIMIT 5
77
96
  `);
78
97
  crossRows = crossStmt.all(ftsQuery, project, cutoff);
79
98
  if (crossRows.length === 0) {
80
99
  const orQuery = relaxFtsQueryToOr(ftsQuery);
81
- if (orQuery) {
82
- try { crossRows = crossStmt.all(orQuery, project, cutoff); } catch {}
100
+ if (orQuery && queryTokenCount <= OR_FALLBACK_MAX_TOKENS) {
101
+ try { crossRows = crossStmt.all(orQuery, project, cutoff); crossUsedOr = true; } catch {}
83
102
  }
84
103
  }
85
104
  } catch (e) { debugCatch(e, 'crossProjectSearch'); }
86
105
 
87
106
  // Merge and score: same-project full weight, cross-project 0.7x
88
- const allRows = [...rows, ...crossRows];
107
+ // OR-fallback results get 0.4x penalty — they matched individual words, not the full intent
108
+ const allRows = [...rows.map(r => ({ ...r, _or: usedOrFallback })), ...crossRows.map(r => ({ ...r, _or: crossUsedOr }))];
89
109
  const scored = allRows
90
110
  .filter(r => !excludeSet.has(r.id))
91
111
  .map(r => {
92
112
  const crossProjectPenalty = r.project === project ? 1.0 : 0.7;
113
+ const orFallbackPenalty = r._or ? 0.4 : 1.0;
93
114
  return {
94
115
  ...r,
95
116
  score: Math.abs(r.relevance)
96
117
  * (MEMORY_TYPE_BOOST[r.type] || 1.0)
97
118
  * (r.lesson_learned ? 1.5 : 1.0)
98
119
  * (r.importance >= 2 ? 1.0 : 0.6)
99
- * crossProjectPenalty,
120
+ * crossProjectPenalty
121
+ * orFallbackPenalty,
100
122
  };
101
123
  })
102
124
  .sort((a, b) => b.score - a.score);
103
125
 
104
- // Adaptive threshold: BM25 IDF collapses when corpus has <5 observations,
105
- // producing scores ~0.00001 even for exact matches. At 5+ obs, IDF provides
106
- // meaningful discrimination and the calibrated 1.5 threshold works well.
126
+ // Adaptive threshold: scales with corpus size to filter noise.
127
+ // Each result must individually exceed the threshold (not just the top one).
107
128
  const obsCount = db.prepare(
108
129
  'SELECT COUNT(*) as c FROM observations WHERE project = ? AND COALESCE(compressed_into, 0) = 0',
109
130
  ).get(project)?.c || 0;
110
- const threshold = obsCount < 5 ? 0 : 1.5;
111
- if (scored.length === 0 || scored[0].score < threshold) return [];
131
+ const { TINY, SMALL, MEDIUM, LARGE } = BM25_THRESHOLD;
132
+ const threshold = obsCount < 5 ? TINY : obsCount < 100 ? SMALL : obsCount < 500 ? MEDIUM : LARGE;
133
+ const aboveThreshold = scored.filter(r => r.score >= threshold);
134
+ if (aboveThreshold.length === 0) return [];
112
135
 
113
136
  // Update access_count for injected memories
114
- const result = scored.slice(0, MAX_MEMORY_INJECTIONS);
137
+ const result = aboveThreshold.slice(0, MAX_MEMORY_INJECTIONS);
115
138
  const now = Date.now();
116
139
  const updateStmt = db.prepare('UPDATE observations SET access_count = COALESCE(access_count, 0) + 1, last_accessed_at = ? WHERE id = ?');
117
140
  for (const r of result) {