claude-mem-lite 2.49.1 → 2.50.0

package/.claude-plugin/marketplace.json

@@ -10,7 +10,7 @@
   "plugins": [
     {
       "name": "claude-mem-lite",
-      "version": "2.49.1",
+      "version": "2.50.0",
       "source": "./",
       "description": "Lightweight persistent memory system for Claude Code — FTS5 search, episode batching, error-triggered recall"
     }

package/.claude-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "claude-mem-lite",
-  "version": "2.49.1",
+  "version": "2.50.0",
   "description": "Lightweight persistent memory system for Claude Code — FTS5 search, episode batching, error-triggered recall",
   "author": {
     "name": "sdsrss"

package/mem-cli.mjs

@@ -5,6 +5,7 @@
 import { homedir } from 'os';
 import { ensureDb, DB_PATH, REGISTRY_DB_PATH } from './schema.mjs';
 import { sanitizeFtsQuery, relaxFtsQueryToOr, truncate, typeIcon, inferProject, jaccardSimilarity, computeMinHash, estimateJaccardFromMinHash, scrubSecrets, cjkBigrams, isoWeekKey, COMPRESSED_PENDING_PURGE, OBS_BM25, SESS_BM25, TYPE_DECAY_CASE, TYPE_QUALITY_CASE, DEFAULT_DECAY_HALF_LIFE_MS, getCurrentBranch, notLowSignalTitleClause, LOW_SIGNAL_TITLE } from './utils.mjs';
+import { cjkPrecisionOk } from './nlp.mjs';
 import { extractCjkLikePatterns } from './nlp.mjs';
 import { resolveProject } from './project-utils.mjs';
 import { computeTier, TIER_CASE_SQL, tierSqlParams } from './tier.mjs';
@@ -240,9 +241,14 @@ function cmdSearch(db, args) {
         ORDER BY score
         LIMIT ? OFFSET ?
       `).all(...promptParams);
-      for (const r of promptRows) results.push({ ...r, _source: 'prompt' });
+      // CJK precision filter (read-path parity with server.mjs): unicode61
+      // degrades bigram queries to single-char AND, letting common-char
+      // Chinese prose leak through. Drop rows that miss < 20% of query
+      // bigrams/keywords as contiguous substrings.
+      const keptPromptRows = promptRows.filter(r => cjkPrecisionOk(query, r.prompt_text));
+      for (const r of keptPromptRows) results.push({ ...r, _source: 'prompt' });
       // CJK LIKE fallback: FTS5 unicode61 can't tokenize CJK substrings in prompts
-      if (promptRows.length === 0) {
+      if (keptPromptRows.length === 0) {
         const cjkPatterns = extractCjkLikePatterns(query);
         if (cjkPatterns.length > 0) {
           const likeConds = cjkPatterns.map(() => 'p.prompt_text LIKE ?');
@@ -263,7 +269,12 @@ function cmdSearch(db, args) {
             ORDER BY p.created_at_epoch DESC
             LIMIT ? OFFSET ?
           `).all(...likeParams);
-          for (const r of fallbackRows) results.push({ ...r, _source: 'prompt', score: 0 });
+          // CJK precision filter applies here too: the LIKE fallback is just
+          // OR'd substring bigrams; without the precision gate it re-admits
+          // the same common-char noise the FTS path dropped (this was the
+          // actual leak source — FTS returned 0, fallback filled 20).
+          const keptFallback = fallbackRows.filter(r => cjkPrecisionOk(query, r.prompt_text));
+          for (const r of keptFallback) results.push({ ...r, _source: 'prompt', score: 0 });
         }
       }
     } catch { /* prompt FTS may not exist in older DBs */ }

package/nlp.mjs

@@ -124,6 +124,58 @@ export function extractCjkLikePatterns(query) {
   return [...new Set([...keywords, ...bigrams])];
 }
 
+/**
+ * Post-FTS precision filter for CJK queries.
+ *
+ * Background: FTS5 unicode61 tokenizer splits every CJK character into its
+ * own token. An application-layer bigram query like "我是" then reduces to
+ * (我 AND 是) at match time — matching any document that happens to contain
+ * both chars anywhere, which is extremely permissive in Chinese prose.
+ *
+ * Precision check: given the raw query and a candidate result's full text,
+ * require that at least `threshold` fraction of the query's CJK bigrams
+ * (or dictionary words, if any matched) appear as contiguous substrings in
+ * the result. Non-CJK queries bypass this filter entirely.
+ *
+ * Applied only to the prompts/user-prompt path — observations have richer
+ * rerank + low-signal filtering that already control noise there. Also,
+ * obs-side synonym expansion ("查询"→"(查询 OR query OR search)") is a
+ * legitimate recall mechanism that this filter would break.
+ *
+ * Threshold default 0.2 is tunable via `CLAUDE_MEM_CJK_PREC_MIN` env var.
+ * Explicit threshold arg still overrides the env value — tests and in-code
+ * callers with domain context stay authoritative.
+ *
+ * Default was tuned from 0.3 → 0.2 after a 20-query production-DB fixture
+ * showed 0.3 over-rejected legitimate multi-bigram queries whose dict-
+ * keyword coverage was incomplete (e.g. "同义词扩展" — neither compound
+ * is in CJK_COMPOUNDS → 4 bigrams required, single-keyword match only
+ * 25% < 30% rejected 19/20 real hits). At 0.2, pure-noise reduction stays
+ * ≥85% on noise fixture while SIG-6 recall recovered to 100%.
+ *
+ * @param {string} query Raw query text
+ * @param {string} text Candidate result text
+ * @param {number} [threshold] Fraction of patterns that must match. If
+ *   omitted, reads CLAUDE_MEM_CJK_PREC_MIN (default 0.2).
+ * @returns {boolean}
+ */
+export function cjkPrecisionOk(query, text, threshold) {
+  if (threshold === undefined) {
+    const envVal = process.env.CLAUDE_MEM_CJK_PREC_MIN;
+    const parsed = envVal ? parseFloat(envVal) : NaN;
+    threshold = Number.isFinite(parsed) && parsed >= 0 && parsed <= 1 ? parsed : 0.2;
+  }
+  if (!query || !text) return true;
+  if (!/[一-鿿㐀-䶿]{2,}/.test(query)) return true;
+  const keywords = extractCjkKeywords(query);
+  const required = keywords.length > 0
+    ? keywords
+    : cjkBigrams(query).split(' ').filter(b => b && !CJK_STOP_WORDS.has(b));
+  if (required.length === 0) return true;
+  const hit = required.filter(w => text.includes(w)).length;
+  return (hit / required.length) >= threshold;
+}
+
 // ─── FTS5 Token Formatting ──────────────────────────────────────────────────
 
 // Format a term for FTS5: quote if it contains spaces, hyphens, or special chars

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "claude-mem-lite",
-  "version": "2.49.1",
+  "version": "2.50.0",
   "description": "Lightweight persistent memory system for Claude Code",
   "type": "module",
   "engines": {

package/scripts/user-prompt-search.js

@@ -5,6 +5,7 @@
 
 import { ensureDb, DB_DIR, REGISTRY_DB_PATH } from '../schema.mjs';
 import { sanitizeFtsQuery, relaxFtsQueryToOr, truncate, typeIcon, inferProject, OBS_BM25, TYPE_DECAY_CASE, TYPE_QUALITY_CASE, notLowSignalTitleClause, noisePenaltyClause } from '../utils.mjs';
+import { cjkPrecisionOk } from '../nlp.mjs';
 import { writeFileSync, readFileSync, existsSync, renameSync } from 'fs';
 import { join } from 'path';
 import Database from 'better-sqlite3';
@@ -242,7 +243,11 @@ function searchByUserPrompts(db, queryText, project, limit) {
     }
   }
 
-  return rows;
+  // CJK precision filter (parity with server.mjs + mem-cli.mjs): unicode61
+  // FTS degrades CJK bigram queries to single-char AND, letting any prose
+  // sharing common chars leak through. Drop rows that miss < 20% of query
+  // bigrams/keywords as contiguous substrings. Non-CJK queries bypass.
+  return rows.filter(r => cjkPrecisionOk(queryText, r.prompt_text));
 }
 
 function searchRecent(db, project, limit) {

package/server.mjs

@@ -6,7 +6,7 @@ import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
 import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
 import { ListToolsRequestSchema } from '@modelcontextprotocol/sdk/types.js';
 import { jaccardSimilarity, truncate, typeIcon, sanitizeFtsQuery, relaxFtsQueryToOr, inferProject, computeMinHash, estimateJaccardFromMinHash, scrubSecrets, cjkBigrams, fmtDate, isoWeekKey, debugLog, debugCatch, COMPRESSED_PENDING_PURGE, OBS_BM25, SESS_BM25, TYPE_DECAY_CASE, TYPE_QUALITY_CASE, getCurrentBranch, DEFAULT_DECAY_HALF_LIFE_MS, isPathConfined, notLowSignalTitleClause, LOW_SIGNAL_TITLE } from './utils.mjs';
-import { extractCjkLikePatterns } from './nlp.mjs';
+import { extractCjkLikePatterns, cjkPrecisionOk } from './nlp.mjs';
 import { resolveProject as _resolveProjectShared } from './project-utils.mjs';
 import { ensureDb, DB_PATH, REGISTRY_DB_PATH } from './schema.mjs';
 import { reRankWithContext, markSuperseded, extractPRFTerms, expandQueryByConcepts, autoBoostIfNeeded, runIdleCleanup, buildServerInstructions } from './server-internals.mjs';
@@ -463,11 +463,15 @@ function searchPrompts(ctx) {
       epochTo, epochTo,
       perSourceLimit, perSourceOffset
     );
-    for (const r of rows) {
+    // CJK precision filter: unicode61 FTS degrades CJK bigram queries to
+    // single-char AND, letting any prose sharing common chars leak through.
+    // Require ≥30% of query's CJK bigrams/keywords as contiguous substrings.
+    const keptRows = args.query ? rows.filter(r => cjkPrecisionOk(args.query, r.prompt_text)) : rows;
+    for (const r of keptRows) {
       results.push({ source: 'prompt', id: r.id, text: r.prompt_text, session: r.content_session_id, date: r.created_at, created_at_epoch: r.created_at_epoch, score: r.score });
     }
     // CJK LIKE fallback: FTS5 unicode61 can't tokenize CJK substrings in prompts
-    if (rows.length === 0 && args.query) {
+    if (keptRows.length === 0 && args.query) {
       const cjkPatterns = extractCjkLikePatterns(args.query);
       if (cjkPatterns.length > 0) {
         const likeConds = cjkPatterns.map(() => 'p.prompt_text LIKE ?');
@@ -490,7 +494,13 @@ function searchPrompts(ctx) {
           epochTo, epochTo,
           perSourceLimit, perSourceOffset
         );
-        for (const r of fallbackRows) {
+        // Parity with mem-cli.mjs: the LIKE fallback is an OR'd bigram
+        // substring scan with no scoring gate. The precision filter must
+        // apply here too — without it, queries whose FTS set is empty
+        // re-admit the full common-char noise band that FTS would have
+        // dropped downstream anyway.
+        const keptFallback = args.query ? fallbackRows.filter(r => cjkPrecisionOk(args.query, r.prompt_text)) : fallbackRows;
+        for (const r of keptFallback) {
           results.push({ source: 'prompt', id: r.id, text: r.prompt_text, session: r.content_session_id, date: r.created_at, created_at_epoch: r.created_at_epoch, score: 0 });
         }
       }