@pcircle/memesh 4.2.8 → 4.2.10

package/scripts/hooks/_shared.js

@@ -1,148 +1,42 @@
 import { appendFileSync, chmodSync, closeSync, existsSync, mkdirSync, openSync, readFileSync, writeFileSync } from 'fs';
-import { spawn, execFileSync } from 'child_process';
+import { spawn } from 'child_process';
 import { createRequire } from 'module';
 import { homedir } from 'os';
-import { basename, dirname, join } from 'path';
+import { dirname, join } from 'path';
 import { fileURLToPath, pathToFileURL } from 'url';
 
-const require = createRequire(import.meta.url);
-
 // =============================================================================
-// Path helpers — MIRROR of src/core/paths.ts
+// Path helpers + FTS primitives — GENERATED from src/core (do not hand-mirror)
 // =============================================================================
 //
-// Hooks cannot import from `dist/` (the F5 security boundary — `dist/` may
-// be stale or absent at hook execution time), so the path-resolution logic
-// is duplicated here. The contract MUST stay in lockstep with
-// `src/core/paths.ts`. Any change to the function shapes / precedence
-// rules in that file MUST be reflected here too.
+// These were once a 965-line hand-mirror of `src/core`, kept in lockstep by
+// human review — until the copies drifted and shipped the P0 FTS bug (a hook
+// wrote an entity+observations but the mirror's reindex step diverged, leaving
+// the memory unrecallable).
 //
-// Unlike the schema duplication (which has a build-time diff guard via
-// `scripts/check-schema-drift.mjs`), these helpers are short enough that
-// human review at code-review time is sufficient. If they grow, add a
-// programmatic guard.
-
-/**
- * Return the user's home directory, honoring HOME env var first.
- *
- * On POSIX, os.homedir() already consults HOME. On Windows, it ignores
- * env vars and reads GetUserProfileDirectoryW directly — which makes
- * tests unable to redirect home-dir lookups to a tmp dir. Honoring HOME
- * first lets tests set HOME=<tmpdir> and have it actually take effect
- * across platforms. Production users on Windows almost never set HOME,
- * so this falls through to os.homedir() as before.
- *
- * Mirror of: src/core/paths.ts → homeDir()
- *
- * @returns {string}
- */
-function homeDir() {
-  // Mirror src/core/paths.ts homeDir() — same three-step fallback to
-  // handle HOME="" environments. `os.homedir()` itself reads HOME on
-  // POSIX, so HOME="" makes it return "". `os.userInfo().homedir`
-  // reads pw_dir via getpwuid syscall, bypassing env vars entirely.
-  const home = process.env.HOME;
-  if (home && home.length > 0) return home;
-  const fromOs = homedir();
-  if (fromOs && fromOs.length > 0) return fromOs;
-  // userInfo is the final defence — re-import here to keep the
-  // top-of-file `import { homedir } from 'os'` line stable.
-  return require('os').userInfo().homedir;
-}
-
-/**
- * Resolve the memesh data directory.
- *
- * Precedence: MEMESH_DIR env var > <home>/.memesh.
- * Mirror of: src/core/paths.ts → memeshDir()
- *
- * No-arg to mirror the core helper exactly. Earlier drafts accepted a
- * custom `env` parameter for symmetry with `getMemeshDirFromDbPath(env)`,
- * but the inner `homeDir()` only ever read `process.env.HOME`, so a
- * caller that passed `{HOME: '/tmp/x'}` would be silently ignored — a
- * footgun. Tests redirect via `process.env.HOME`; that's the supported
- * extension point.
- *
- * @returns {string}
- */
-export function memeshDir() {
-  return process.env.MEMESH_DIR ?? join(homeDir(), '.memesh');
-}
-
-/**
- * Resolve the active memesh DB path.
- *
- * Precedence: MEMESH_DB_PATH env var > <memeshDir>/knowledge-graph.db.
- * Mirror of: src/core/paths.ts → getDbPath() — no-arg, see memeshDir().
- *
- * @returns {string}
- */
-export function getDbPath() {
-  return process.env.MEMESH_DB_PATH ?? join(memeshDir(), 'knowledge-graph.db');
-}
-
-/**
- * Derive the project name from a working directory.
- *
- * Hooks historically used `basename(data.cwd || process.cwd())`. Core
- * had two variants (`basename(context.cwd)` and `basename(process.cwd())`).
- * This helper unifies the contract — explicit cwd wins, falls through to
- * process.cwd() — matching the most permissive caller's behaviour.
- *
- * Mirror of: src/core/paths.ts → getProjectName() + resolveProjectIdentity().
- * The layered git resolution MUST stay identical to that file — a divergence
- * means hooks (which write project tags) and core (which reads them) would
- * disagree on identity, re-creating the split this change fixes.
- *
- * @param {string|null|undefined} [cwdInput]
- * @returns {string}
- */
-const _projectNameCache = new Map();
-
-export function getProjectName(cwdInput) {
-  const cwd = cwdInput && cwdInput.length > 0 ? cwdInput : process.cwd();
-  const cached = _projectNameCache.get(cwd);
-  if (cached !== undefined) return cached;
-  const resolved = _resolveProjectIdentity(cwd);
-  _projectNameCache.set(cwd, resolved);
-  return resolved;
-}
-
-// Layered identity: git remote slug > git repo root basename > cwd basename.
-// See src/core/paths.ts resolveProjectIdentity for the full rationale. git
-// failures at any layer fall through to the next; capture must never break.
-function _resolveProjectIdentity(cwd) {
-  const remote = _tryGit(cwd, ['config', '--get', 'remote.origin.url']);
-  if (remote) {
-    const slug = slugFromRemoteUrl(remote);
-    if (slug) return slug;
-  }
-  const root = _tryGit(cwd, ['rev-parse', '--show-toplevel']);
-  if (root) return basename(root);
-  return basename(cwd);
-}
-
-function _tryGit(cwd, args) {
-  try {
-    const out = execFileSync('git', ['-C', cwd, ...args], {
-      encoding: 'utf8',
-      timeout: 2000,
-      stdio: ['ignore', 'pipe', 'ignore'],
-    });
-    const trimmed = out.trim();
-    return trimmed.length > 0 ? trimmed : null;
-  } catch {
-    return null;
-  }
-}
+// `src/core/paths.ts` and `src/storage/fts-index.ts` are runtime-LEAF modules
+// (paths.ts imports only node builtins; fts-index.ts has only a type-only
+// import), so `tsc` emits self-contained JS for them. `scripts/generate-hook-core.mjs`
+// copies that compiled JS to `_generated/` at build time — committed, shipped in
+// the tarball, and version-locked to its own install. So the hook path still
+// survives a missing/stale `dist/` (the F5 constraint) exactly as the hand-mirror
+// did, but the copy is byte-locked to core and CI-gated (`git diff` on rebuild +
+// `tests/hooks/mirror-parity.test.ts`), making drift structurally impossible.
+//
+// Re-exported here so all 7 hooks keep importing these names from `_shared.js`
+// unchanged.
+import {
+  memeshDir,
+  getDbPath,
+  getMemeshDirFromDbPath,
+  getProjectName,
+  slugFromRemoteUrl,
+} from './_generated/core-paths.js';
+import { removeFromFts, insertFtsRow } from './_generated/fts-index.js';
+
+export { memeshDir, getDbPath, getMemeshDirFromDbPath, getProjectName, slugFromRemoteUrl };
 
-/** Mirror of paths.ts slugFromRemoteUrl. */
-export function slugFromRemoteUrl(url) {
-  const cleaned = url.trim().replace(/\.git$/i, '').replace(/[/\\]+$/, '');
-  if (!cleaned) return null;
-  const seg = cleaned.split(/[/:\\]/).filter(Boolean).pop();
-  return seg && seg.length > 0 ? seg : null;
-}
+const require = createRequire(import.meta.url);
 
 /**
  * Resolve the package root from a hook file's `import.meta.url`.
@@ -623,25 +517,73 @@ export function openHookDb(env = process.env, opts = {}) {
   return { db, dbPath };
 }
 
-const PRIVATE_DIR_MODE = 0o700;
-const PRIVATE_FILE_MODE = 0o600;
-
 /**
- * Resolve the directory containing the active DB file.
+ * Single owner of the hook-side entity write dance: upsert entity, append
+ * observations + tags, and — critically — keep the contentless `entities_fts`
+ * index in sync so the memory is recallable via the FTS keyword hot path.
+ *
+ * Why this exists: post-commit.js, pre-compact.js, and session-summary.js each
+ * hand-rolled this dance inline. Three copies drifted — session-summary's copy
+ * omitted the FTS reindex entirely, so every `session-insight` memory it wrote
+ * was invisible to `recall` and pre-edit-recall (no FTS trigger, no self-heal
+ * rebuild on open back it up). Centralising the dance here makes the FTS step
+ * impossible to forget in a future hook. This mirrors, on the hook side, what
+ * `src/storage/fts-index.ts` already does for core (the F5 boundary keeps them
+ * as two implementations of the same contract).
  *
- * When MEMESH_DB_PATH is set, returns its parent directory (used for
- * sibling files next to the DB). Otherwise returns memeshDir().
- * Mirror of: src/core/paths.ts → getMemeshDirFromDbPath()
+ * The caller MUST open its DB with `openHookDb(env, { fts: true })` so the FTS
+ * table is guaranteed present. Embeddings + auto-tagging + signal-scoring are
+ * deliberately NOT done here: hooks are cheap always-on capture, and those are
+ * the heavier, user-initiated `remember` concerns (core owns them).
  *
- * Renamed from the legacy `getMemeshDir` to match the core helper —
- * sibling helper `memeshDir()` returns the GLOBAL data directory, so a
- * second function called `getMemeshDir` was confusing. Callers updated
- * in lockstep.
+ * @param {import('better-sqlite3').Database} db - an open hook DB handle
+ * @param {{name: string, type: string, observations?: string[], tags?: string[]}} entity
+ * @returns {{ id: number, isNew: boolean } | null} null if the row could not be resolved
  */
-export function getMemeshDirFromDbPath() {
-  return process.env.MEMESH_DB_PATH ? dirname(process.env.MEMESH_DB_PATH) : memeshDir();
+export function captureEntity(db, { name, type, observations = [], tags = [] }) {
+  const insertResult = db
+    .prepare('INSERT OR IGNORE INTO entities (name, type) VALUES (?, ?)')
+    .run(name, type);
+  const isNew = insertResult.changes > 0;
+  const row = db.prepare('SELECT id FROM entities WHERE name = ?').get(name);
+  if (!row) return null;
+  const id = row.id;
+
+  // Capture the previously-indexed observation text BEFORE inserting new rows,
+  // so the contentless-FTS 'delete' below matches what was indexed. Only for
+  // existing entities — a brand-new row has no prior FTS entry to remove.
+  const prevObsText = isNew
+    ? undefined
+    : db
+        .prepare('SELECT content FROM observations WHERE entity_id = ?')
+        .all(id)
+        .map((o) => o.content)
+        .join(' ');
+
+  const insertObs = db.prepare('INSERT INTO observations (entity_id, content) VALUES (?, ?)');
+  for (const obs of observations) insertObs.run(id, obs);
+  const insertTag = db.prepare('INSERT OR IGNORE INTO tags (entity_id, tag) VALUES (?, ?)');
+  for (const tag of tags) insertTag.run(id, tag);
+
+  // Reindex FTS: delete the stale entry (if any) then insert the full,
+  // current observation set. Uses the generated copy of src/storage/fts-index.ts
+  // so the contentless-FTS5 delete+insert dance can no longer drift from core.
+  if (prevObsText !== undefined) {
+    removeFromFts(db, id, name, prevObsText);
+  }
+  const allObsText = db
+    .prepare('SELECT content FROM observations WHERE entity_id = ?')
+    .all(id)
+    .map((o) => o.content)
+    .join(' ');
+  insertFtsRow(db, id, name, allObsText);
+
+  return { id, isNew };
 }
 
+const PRIVATE_DIR_MODE = 0o700;
+const PRIVATE_FILE_MODE = 0o600;
+
 export function ensurePrivateDir(dirPath) {
   mkdirSync(dirPath, { recursive: true, mode: PRIVATE_DIR_MODE });
   try {

package/scripts/hooks/post-commit.js

@@ -1,7 +1,7 @@
 #!/usr/bin/env node
 
 import { execFileSync } from 'child_process';
-import { getProjectName, openHookDb } from './_shared.js';
+import { captureEntity, getProjectName, openHookDb } from './_shared.js';
 
 let input = '';
 process.stdin.setEncoding('utf8');
@@ -77,53 +77,32 @@ process.stdin.on('end', () => {
     try {
       const entityName = `commit-${commitHash}`;
 
-      // Check if this is a new or existing entity
-      const insertResult = db.prepare('INSERT OR IGNORE INTO entities (name, type) VALUES (?, ?)').run(entityName, 'commit');
-      const isNew = insertResult.changes > 0;
-      const entity = db.prepare('SELECT id FROM entities WHERE name = ?').get(entityName);
-      if (entity) {
-        // Capture existing observations for FTS delete (before inserting new one)
-        const prevObs = isNew
-          ? []
-          : db.prepare('SELECT content FROM observations WHERE entity_id = ?').all(entity.id);
-        const prevObsText = isNew ? undefined : prevObs.map(o => o.content).join(' ');
-
-        // Add observations
-        db.prepare('INSERT INTO observations (entity_id, content) VALUES (?, ?)').run(entity.id, commitMsg);
-        db.prepare('INSERT INTO observations (entity_id, content) VALUES (?, ?)').run(entity.id, `Branch: ${branch}`);
-
-        // Add richer diff stats as an observation (backward compatible — failures are silently ignored)
-        try {
-          const stat = execFileSync('git', ['show', '--stat', '--format=', commitHash], {
-            cwd: data.cwd || process.cwd(),
-            encoding: 'utf8',
-            timeout: 5000,
-          }).trim();
-          if (stat) {
-            // Last non-empty line is the summary, e.g. "3 files changed, 45 insertions(+), 12 deletions(-)"
-            const statLines = stat.split('\n').filter(l => l.trim());
-            const summary = statLines[statLines.length - 1]?.trim() || '';
-            if (summary) {
-              db.prepare('INSERT INTO observations (entity_id, content) VALUES (?, ?)').run(entity.id, `Diff stats: ${summary}`);
-            }
-          }
-        } catch {
-          // git show failed — no diff stats recorded, existing behavior unchanged
-        }
-
-        // Add project tag
-        const projectTag = `project:${projectName}`;
-        db.prepare('INSERT OR IGNORE INTO tags (entity_id, tag) VALUES (?, ?)').run(entity.id, projectTag);
-
-        // Update FTS index — delete old entry first if entity existed
-        if (prevObsText !== undefined) {
-          db.prepare("INSERT INTO entities_fts(entities_fts, rowid, name, observations) VALUES('delete', ?, ?, ?)").run(entity.id, entityName, prevObsText);
+      // Build the observation set: commit message, branch, and — best-effort —
+      // richer diff stats (git failures are silently ignored, unchanged behavior).
+      const observations = [commitMsg, `Branch: ${branch}`];
+      try {
+        const stat = execFileSync('git', ['show', '--stat', '--format=', commitHash], {
+          cwd: data.cwd || process.cwd(),
+          encoding: 'utf8',
+          timeout: 5000,
+        }).trim();
+        if (stat) {
+          // Last non-empty line is the summary, e.g. "3 files changed, 45 insertions(+), 12 deletions(-)"
+          const statLines = stat.split('\n').filter(l => l.trim());
+          const summary = statLines[statLines.length - 1]?.trim() || '';
+          if (summary) observations.push(`Diff stats: ${summary}`);
         }
-        // Fetch all observations (including the one just added) for the new FTS entry
-        const allObs = db.prepare('SELECT content FROM observations WHERE entity_id = ?').all(entity.id);
-        const allObsText = allObs.map(o => o.content).join(' ');
-        db.prepare('INSERT INTO entities_fts(rowid, name, observations) VALUES(?, ?, ?)').run(entity.id, entityName, allObsText);
+      } catch {
+        // git show failed — no diff stats recorded, existing behavior unchanged
       }
+
+      // Shared write dance — upsert entity + observations + tags AND reindex FTS.
+      captureEntity(db, {
+        name: entityName,
+        type: 'commit',
+        observations,
+        tags: [`project:${projectName}`],
+      });
     } finally {
       db.close();
     }

package/scripts/hooks/pre-compact.js

@@ -2,7 +2,7 @@
 
 import { basename } from 'path';
 import { existsSync, readFileSync } from 'fs';
-import { getProjectName, isAutoCaptureEnabled, openHookDb } from './_shared.js';
+import { captureEntity, getProjectName, isAutoCaptureEnabled, openHookDb } from './_shared.js';
 
 // Timeout guard: always exit within 10 seconds
 const TIMEOUT_MS = 10000;
@@ -99,37 +99,14 @@ process.stdin.on('end', () => {
     if (!handle) return;
     const { db } = handle;
     try {
-      // Upsert entity
-      const insertResult = db.prepare('INSERT OR IGNORE INTO entities (name, type) VALUES (?, ?)').run(entityName, 'session-summary');
-      const isNew = insertResult.changes > 0;
-      const entity = db.prepare('SELECT id FROM entities WHERE name = ?').get(entityName);
-
-      if (entity) {
-        // Capture existing observations for FTS delete
-        const prevObs = isNew
-          ? []
-          : db.prepare('SELECT content FROM observations WHERE entity_id = ?').all(entity.id);
-        const prevObsText = isNew ? undefined : prevObs.map(o => o.content).join(' ');
-
-        // Insert each observation line
-        for (const line of obsLines) {
-          db.prepare('INSERT INTO observations (entity_id, content) VALUES (?, ?)').run(entity.id, line);
-        }
-
-        // Add tags
-        const tags = ['source:auto-capture', 'urgency:pre-compact', `project:${projectName}`];
-        for (const tag of tags) {
-          db.prepare('INSERT OR IGNORE INTO tags (entity_id, tag) VALUES (?, ?)').run(entity.id, tag);
-        }
-
-        // Update FTS
-        if (prevObsText !== undefined) {
-          db.prepare("INSERT INTO entities_fts(entities_fts, rowid, name, observations) VALUES('delete', ?, ?, ?)").run(entity.id, entityName, prevObsText);
-        }
-        const allObs = db.prepare('SELECT content FROM observations WHERE entity_id = ?').all(entity.id);
-        const allObsText = allObs.map(o => o.content).join(' ');
-        db.prepare('INSERT INTO entities_fts(rowid, name, observations) VALUES(?, ?, ?)').run(entity.id, entityName, allObsText);
-      }
+      // Shared write dance — upsert entity + observations + tags AND reindex FTS
+      // so the pre-compact memory is recallable via the FTS keyword path.
+      captureEntity(db, {
+        name: entityName,
+        type: 'session-summary',
+        observations: obsLines,
+        tags: ['source:auto-capture', 'urgency:pre-compact', `project:${projectName}`],
+      });
     } finally {
       db.close();
     }

package/scripts/hooks/session-summary.js

@@ -11,6 +11,7 @@ import { spawn } from 'child_process';
 import os from 'os';
 import { pathToFileURL } from 'url';
 import {
+  captureEntity,
   decideAutoUpdateHook,
   getMemeshDirFromDbPath,
   getProjectName,
@@ -233,7 +234,9 @@ process.stdin.on('end', async () => {
     // Open DB via shared helper — applies SCHEMA_SQL + status migration.
     // sqlite-vec is loaded separately because only this hook needs it
     // (for embedding-aware recall-effectiveness tracking).
-    const handle = openHookDb(process.env);
+    // { fts: true } guarantees the entities_fts table exists so captureEntity()
+    // can keep it in sync — session-insight memories must be FTS-recallable.
+    const handle = openHookDb(process.env, { fts: true });
     if (!handle) {
       // Native module unavailable (plugin-marketplace cache install with no
       // node_modules). Skip session-capture work, but still let the
@@ -291,17 +294,12 @@ process.stdin.on('end', async () => {
         return [...tags];
       }
 
-      const insertEntity = db.prepare('INSERT OR IGNORE INTO entities (name, type) VALUES (?, ?)');
-      const selectEntity = db.prepare('SELECT id FROM entities WHERE name = ?');
-      const insertObs = db.prepare('INSERT INTO observations (entity_id, content) VALUES (?, ?)');
-      const insertTag = db.prepare('INSERT OR IGNORE INTO tags (entity_id, tag) VALUES (?, ?)');
-
+      // Delegate the write to the shared captureEntity() so entities land in
+      // entities_fts too. This copy used to insert entity + observations + tags
+      // only, skipping the FTS reindex the sibling hooks did — which left every
+      // session-insight memory unrecallable via the FTS keyword path.
       function storeMemory(name, type, observations, tags) {
-        insertEntity.run(name, type);
-        const row = selectEntity.get(name);
-        if (!row) return;
-        for (const obs of observations) insertObs.run(row.id, obs);
-        for (const tag of tags) insertTag.run(row.id, tag);
+        captureEntity(db, { name, type, observations, tags });
       }
 
       // Rule 1: File editing session summary
@@ -426,8 +424,11 @@ process.stdin.on('end', async () => {
 
               for (let i = 0; i < entityIds.length; i++) {
                 const name = (entityNames[i] || '').toLowerCase();
-                // Skip very short names to avoid false positives
-                if (name.length < 4) continue;
+                // Skip names that carry no recall signal: too short, or a
+                // machine identifier (auto-capture entities) that can never
+                // substring-match prose. Scoring those would be a guaranteed
+                // unearned miss — see isMeasurableRecallName.
+                if (!isMeasurableRecallName(name)) continue;
                 if (isRecallHit(sessionText, name)) {
                   updateHit.run(entityIds[i]);
                 } else {
@@ -761,6 +762,28 @@ export function isRecallHit(sessionText, name) {
   return String(sessionText ?? '').toLowerCase().includes(String(name).toLowerCase());
 }
 
+/**
+ * Whether an injected entity's NAME can serve as a recall-effectiveness signal.
+ *
+ * Recall-effectiveness decides "was this injected memory used?" by substring-
+ * matching the entity NAME in the session transcript (isRecallHit). That only
+ * works for names a human might type. Auto-capture entities are named with
+ * machine identifiers — `session-<pid>-<ts>-files`, `commit-<hash>`,
+ * `pre-compact-<id>` — which never appear verbatim in conversation prose, so
+ * they take a `recall_miss` they didn't earn on every injection. Over repeated
+ * sessions that drags their Laplace-smoothed impact factor (scoring.ts, 10%
+ * weight) down and quietly suppresses auto-captured memories from future recall.
+ *
+ * We can't measure their usefulness by name, so we don't count them either way —
+ * they keep the neutral 0.5 impact. The prefix set is coupled to the auto-capture
+ * producers' `<kind>-<id>` naming (post-commit / session-summary / pre-compact);
+ * a new auto-capture producer should add its prefix here.
+ */
+export function isMeasurableRecallName(name) {
+  if (!name || name.length < 4) return false;
+  return !/^(session-|commit-|pre-compact-)/i.test(name);
+}
+
 export function maybeTriggerDream(projectName, config, pluginRoot) {
   dreamTrigTrace('enter', { projectName, hasLlm: Boolean(config?.llm) });
   if (!projectName || projectName === 'unknown') {