claude-mem-lite 3.56.0 → 3.57.0

package/tfidf.mjs

@@ -9,6 +9,13 @@ import { createHash } from 'crypto';
 import { rrfAccumulate } from './lib/rrf.mjs';
 
 export const VOCAB_DIM = 512;
+// Fraction of the vocab dimension reserved for the highest-IDF (rarest) terms when the
+// candidate set exceeds `dim` and must be truncated. df×idf ("information gain") ranking
+// peaks at MID document-frequency, so a pure-IG cut drops the rare freq≈2 / high-IDF tail
+// FIRST — but those are exactly the discriminative identifiers (a filename / proper noun in
+// 2-3 memories) a high-precision query needs. The quota guarantees that tail a slot budget;
+// the rest fill by IG. No effect when candidates <= dim (audit P2-11 2026-07-24).
+const VOCAB_RARE_TERM_FRACTION = 0.25;
 export const MIN_COSINE_SIMILARITY = 0.05;
 export const VECTOR_SCAN_LIMIT = 500;
 // Reciprocal Rank Fusion constant. Higher k flattens the rank-position weighting
@@ -37,7 +44,12 @@ const VOCAB_STOP_WORDS = new Set([
 ]);
 
 // ─── Porter Stemmer ──────────────────────────────────────────────────────────
-// Minimal Porter stemmer (1980) aligned with SQLite FTS5's built-in porter tokenizer.
+// Minimal Porter stemmer (1980). Used to normalize tokens for the TF-IDF vocabulary and
+// vectors (query + doc are BOTH stemmed here, so the vector arm is internally consistent).
+// NOTE: this does NOT align with the FTS5 side — observations_fts uses FTS5's DEFAULT
+// unicode61 tokenizer (no stemming). Anything that stems a term and then feeds it into an
+// FTS5 MATCH must emit a SURFACE form, not a stem, or it matches nothing (see
+// extractPRFTerms in search-scoring.mjs, audit P2-24 2026-07-24).
 
 const step2map = {
   ational:'ate', tional:'tion', enci:'ence', anci:'ance', izer:'ize',
@@ -247,13 +259,37 @@ export function buildVocabulary(db, { dim = VOCAB_DIM } = {}) {
     }
   }
 
-  // Compute IDF and sort by information gain (df × idf) for balanced discriminativeness
+  // Compute IDF; rank by information gain (df × idf) for balanced discriminativeness.
   const idf = (freq) => Math.log(1 + N / (1 + freq));
-  const sortedTerms = [...df.entries()]
+  const candidates = [...df.entries()]
     .filter(([term, freq]) => !isNoiseTerm(term) && freq >= 2)
-    .map(([term, freq]) => ({ term, df: freq, idf: idf(freq), ig: freq * idf(freq) }))
-    .sort((a, b) => b.ig - a.ig)
-    .slice(0, dim);
+    .map(([term, freq]) => ({ term, df: freq, idf: idf(freq), ig: freq * idf(freq) }));
+
+  // SELECTION (which terms survive when candidates exceed `dim`): a pure df×idf cut drops
+  // the rarest, highest-IDF terms first (IG peaks at mid-df), blinding the arm to the
+  // discriminative tail. Reserve VOCAB_RARE_TERM_FRACTION of the budget for the highest-IDF
+  // terms, fill the rest by IG (audit P2-11). No-op when candidates <= dim.
+  let selected;
+  if (candidates.length <= dim) {
+    selected = candidates;
+  } else {
+    const igBudget = dim - Math.floor(dim * VOCAB_RARE_TERM_FRACTION);
+    const pick = new Map();
+    for (const e of [...candidates].sort((a, b) => b.ig - a.ig)) {
+      if (pick.size >= igBudget) break;
+      pick.set(e.term, e);
+    }
+    for (const e of [...candidates].sort((a, b) => b.idf - a.idf)) {
+      if (pick.size >= dim) break;
+      if (!pick.has(e.term)) pick.set(e.term, e);
+    }
+    selected = [...pick.values()];
+  }
+
+  // INDEX ORDERING is by IG desc. Index order is cosmetic for retrieval — cosine is
+  // permutation-invariant as long as query + doc share this mapping — but keeping the
+  // frequent workhorses at low indices preserves the historical slot layout.
+  const sortedTerms = selected.sort((a, b) => b.ig - a.ig).slice(0, dim);
 
   // Build terms map with index and IDF
   const terms = new Map();

package/utils.mjs

@@ -11,7 +11,7 @@ import { buildLowSignalRegex } from './lib/low-signal-patterns.mjs';
 
 export { DECAY_HALF_LIFE_BY_TYPE, DEFAULT_DECAY_HALF_LIFE_MS, OBS_BM25, SESS_BM25, EVT_BM25, TYPE_DECAY_CASE, TYPE_QUALITY_CASE, OBS_FTS_COLUMNS, notLowSignalTitleClause, noisePenaltyClause } from './scoring-sql.mjs';
 export { cjkBigrams, extractCjkSynonymTokens, extractCjkKeywords, extractCjkLikePatterns, SYNONYM_MAP, expandToken, sanitizeFtsQuery, relaxFtsQueryToOr, FTS_STOP_WORDS, CJK_COMPOUNDS } from './nlp.mjs';
-export { resolveProject, _resetProjectCache } from './project-utils.mjs';
+export { inferProject, resolveProject, _resetProjectCache } from './project-utils.mjs';
 export { scrubSecrets, SECRET_PATTERNS } from './secret-scrub.mjs';
 export { stripPrivate } from './lib/private-strip.mjs';
 export { truncate, typeIcon, fmtDate, fmtTime, isoWeekKey, formatErrorRecallHints, neutralizeContextDelimiters } from './format-utils.mjs';
@@ -44,6 +44,21 @@ export function isPathConfined(candidate, allowedBase) {
   return resolved === base || resolved.startsWith(base + sep);
 }
 
+/**
+ * Basename that treats BOTH '/' and '\' as separators on every host OS.
+ * `path.basename` follows the HOST's rules, so on POSIX it returns a Windows
+ * path unchanged. Hook payloads carry the CLIENT's paths and
+ * observation_files.filename stores either separator (lib/file-edge-match.mjs),
+ * so DB search keys derived from them must be host-independent.
+ * Not for filesystem access — '\' is a legal POSIX filename character.
+ * @param {string} p Path in any separator style
+ * @returns {string} Last segment, trailing separators ignored; '' if none
+ */
+export function basenameAnySep(p) {
+  const s = String(p ?? '').replace(/[/\\]+$/, '');
+  return s.slice(Math.max(s.lastIndexOf('/'), s.lastIndexOf('\\')) + 1);
+}
+
 // ─── Token Estimation ─────────────────────────────────────────────────────
 
 /**
@@ -151,23 +166,6 @@ export function computeRuleImportance(episode) {
   return importance;
 }
 
-// ─── Project Inference ───────────────────────────────────────────────────────
-
-/**
- * Infer a sanitized project name from CLAUDE_PROJECT_DIR, PWD, or cwd.
- * Format: "parent--basename" with non-alphanumeric chars replaced by hyphens.
- * @returns {string} Sanitized project identifier safe for use in filenames
- */
-export function inferProject() {
-  const p = process.env.CLAUDE_PROJECT_DIR || process.env.PWD || process.cwd();
-  const base = basename(p);
-  const parent = basename(dirname(p));
-  const raw = parent && parent !== '.' && parent !== '/' ? `${parent}--${base}` : base;
-  // Sanitize to prevent path traversal when used in filenames (ep-<project>.json)
-  // Truncate to 100 chars to avoid exceeding filesystem name limits (255 bytes)
-  return raw.replace(/[^a-zA-Z0-9_.-]/g, '-').slice(0, 100);
-}
-
 // ─── Episode Logic ───────────────────────────────────────────────────────────
 
 /**