@ijfw/memory-server 1.4.4 → 1.5.1

package/src/memory/migrations/009-obsidian-backfill.js

@@ -0,0 +1,50 @@
+// IJFW v1.5.1 -- memory migration 009: M1 obsidian-index backfill.
+//
+// Source authority: Trident r5 finding 1.2 (HIGH).
+//
+// Round-4 Fix-1 (commit 3218812) wired M1 (Obsidian wikilink/tag/meta
+// indexing -- indexObsidianRelations) and M2 (A-Mem auto-linking -- autoLink)
+// into the production memory-write path (handleStore, search.js#autoIndex).
+// But the fix is forward-only: every memory_entries row written during
+// v1.5.0 -- when M1/M2 were bypassed -- has empty memory_links / memory_tags
+// / memory_meta. An existing user upgrading to v1.5.1 got auto-linking +
+// wikilink indexing only on NEW entries; their accumulated memory stayed
+// un-indexed.
+//
+// This migration runs a ONE-TIME M1 backfill: it walks every existing
+// memory_entries row and runs indexObsidianRelations over its body. M1 is:
+//   - free       -- pure markdown parse, zero LLM / network / cost
+//   - idempotent -- indexObsidianRelations does DELETE-then-INSERT per id,
+//                   so re-applying produces identical aux rows
+// which is exactly what makes it safe to run inside a schema migration: it
+// runs once (user_version gates re-application), deterministically, and a
+// crash rolls the whole txn back to user_version 8.
+//
+// M2 (autoLink) is NOT backfilled here -- it makes one LLM call per row and
+// can cost real money over a large memory. M2 backfill is opt-in via the
+// `ijfw memory reindex --m2` CLI verb, which is budget-gated (respects
+// IJFW_AUTOLINK_BUDGET_USD / IJFW_AUTOLINK_OFF / IJFW_AUTOLINK_BACKFILL).
+//
+// Ordering: migration 001 creates memory_entries; migration 006 creates
+// memory_links / memory_tags / memory_meta. Both run before 009, so by the
+// time up() executes the source table and the three aux tables all exist.
+//
+// Crash safety: the migration runner wraps up() in BEGIN IMMEDIATE.
+// backfillObsidianIndex's per-row indexObsidianRelations opens a nested
+// SQLite transaction (savepoint) -- valid inside the outer txn -- and the
+// whole thing rolls back to user_version 8 on any failure.
+
+import { backfillObsidianIndex } from '../obsidian-parser.js';
+
+export const VERSION = 9;
+export const DESCRIPTION =
+  'memory v1.5.1 -- one-time M1 obsidian-index backfill for pre-fix rows (Trident r5 1.2)';
+
+export function up(db) {
+  // backfillObsidianIndex tolerates a missing memory_entries table (returns
+  // zero counts) so a brand-new db that jumps straight to v9 is a clean
+  // no-op -- there are no pre-fix rows to backfill on a fresh install.
+  backfillObsidianIndex(db);
+}
+
+export default { version: VERSION, description: DESCRIPTION, up };

package/src/memory/obsidian-parser.js

@@ -0,0 +1,152 @@
+// IJFW v1.5.0 -- Obsidian-grade markdown extractor.
+//
+// Pulls three structured signals out of memory body text:
+//   1. [[wikilinks]]            -> links: [{ target, line }]
+//   2. #nested/tags             -> tags:  [{ path, depth }]
+//   3. [key:: value] (Dataview) -> meta:  [{ key, value }]
+//
+// Code fences and inline-code spans are masked before extraction so that
+// example syntax never produces fake edges. DB writes happen in
+// indexObsidianRelations (idempotent re-index: clears prior rows for the
+// memory id before re-inserting).
+
+const FENCE_RE = /```[\s\S]*?```/g;
+const INLINE_CODE_RE = /`[^`\n]+`/g;
+// eslint-disable-next-line security/detect-unsafe-regex -- parses developer-authored markdown notes on local disk; negated [^\]\n] classes bound match to one line per token
+const WIKILINK_RE = /\[\[([^\]\n|]+)(?:\|[^\]\n]+)?\]\]/g;
+const TAG_RE = /(?:^|[^\w&])#([\w/-]+)/g;
+const META_RE = /\[([A-Za-z_][\w-]*)::\s*([^\]\n]+?)\]/g;
+
+function maskCode(text) {
+  return text
+    .replace(FENCE_RE, (m) => ' '.repeat(m.length))
+    .replace(INLINE_CODE_RE, (m) => ' '.repeat(m.length));
+}
+
+function lineOf(masked, idx) {
+  let line = 1;
+  for (let i = 0; i < idx; i++) if (masked.charCodeAt(i) === 10) line++;
+  return line;
+}
+
+function normaliseLinkTarget(raw) {
+  return raw
+    .trim()
+    .toLowerCase()
+    .replace(/\s+/g, '-')
+    .replace(/[^a-z0-9/_-]/g, '');
+}
+
+export function parseObsidian(text) {
+  if (typeof text !== 'string' || text.length === 0) {
+    return { links: [], tags: [], meta: [] };
+  }
+  const masked = maskCode(text);
+
+  const links = [];
+  for (const m of masked.matchAll(WIKILINK_RE)) {
+    const target = normaliseLinkTarget(m[1]);
+    if (target) links.push({ target, line: lineOf(masked, m.index) });
+  }
+
+  const tagSet = new Map();
+  for (const m of masked.matchAll(TAG_RE)) {
+    const path = m[1].toLowerCase().replace(/^\/+|\/+$/g, '');
+    if (!path) continue;
+    const depth = path.split('/').length;
+    if (!tagSet.has(path)) tagSet.set(path, { path, depth });
+  }
+
+  const meta = [];
+  for (const m of masked.matchAll(META_RE)) {
+    meta.push({ key: m[1].toLowerCase(), value: m[2].trim() });
+  }
+
+  return { links, tags: [...tagSet.values()], meta };
+}
+
+export function indexObsidianRelations(db, memoryId, text) {
+  const parsed = parseObsidian(text);
+  const insLink = db.prepare(
+    'INSERT OR IGNORE INTO memory_links (from_id, to_target, line) VALUES (?, ?, ?)',
+  );
+  const insTag = db.prepare(
+    'INSERT OR IGNORE INTO memory_tags (memory_id, tag_path, depth) VALUES (?, ?, ?)',
+  );
+  const insMeta = db.prepare(
+    'INSERT OR IGNORE INTO memory_meta (memory_id, key, value) VALUES (?, ?, ?)',
+  );
+  const tx = db.transaction(() => {
+    db.prepare('DELETE FROM memory_links WHERE from_id=?').run(memoryId);
+    db.prepare('DELETE FROM memory_tags WHERE memory_id=?').run(memoryId);
+    db.prepare('DELETE FROM memory_meta WHERE memory_id=?').run(memoryId);
+    for (const l of parsed.links) insLink.run(memoryId, l.target, l.line);
+    for (const t of parsed.tags) insTag.run(memoryId, t.path, t.depth);
+    for (const m of parsed.meta) insMeta.run(memoryId, m.key, m.value);
+  });
+  tx();
+  return parsed;
+}
+
+// v1.5.1 R5-1.2 -- one-time M1 backfill for memory written during v1.5.0,
+// when indexObsidianRelations was NOT wired into the production write path.
+// Round-4 Fix-1 (commit 3218812) wired M1+M2 into handleStore/autoIndex but
+// forward-only: rows already in memory_entries have empty memory_links /
+// memory_tags / memory_meta. This walks EVERY row and re-runs M1 over it.
+//
+// Safe to run over everything:
+//   - free      -- pure markdown parse, zero LLM / network
+//   - idempotent-- indexObsidianRelations clears prior aux rows per id before
+//                  re-inserting, so a re-run produces identical state
+//
+// The walk reads ids in batches so a very large memory_entries doesn't pin
+// the whole table in memory; each row's indexObsidianRelations call carries
+// its own transaction (DELETE-then-INSERT) so a single bad row never aborts
+// the rest of the backfill.
+//
+// Returns { rows, links, tags, meta } -- counts re-indexed across the run.
+export function backfillObsidianIndex(db, opts = {}) {
+  if (!db || typeof db.prepare !== 'function') {
+    throw new Error('backfillObsidianIndex: db handle is invalid.');
+  }
+  const batchSize = Math.max(1, opts.batchSize || 500);
+  const result = { rows: 0, links: 0, tags: 0, meta: 0, errors: 0 };
+  let lastId = 0;
+  // eslint-disable-next-line no-constant-condition
+  while (true) {
+    let batch;
+    try {
+      batch = db
+        .prepare(
+          'SELECT id, body FROM memory_entries WHERE id > ? ORDER BY id ASC LIMIT ?',
+        )
+        .all(lastId, batchSize);
+    } catch {
+      // memory_entries missing (fresh db before migration 001) -- nothing to do.
+      break;
+    }
+    if (!batch || batch.length === 0) break;
+    for (const row of batch) {
+      lastId = row.id;
+      if (typeof row.body !== 'string' || row.body.length === 0) continue;
+      try {
+        const parsed = indexObsidianRelations(db, String(row.id), row.body);
+        result.rows += 1;
+        result.links += parsed.links.length;
+        result.tags += parsed.tags.length;
+        result.meta += parsed.meta.length;
+      } catch (e) {
+        result.errors += 1;
+        try {
+          console.error(
+            '[obsidian] backfill failed for id', row.id, ':', e?.message || e,
+          );
+        } catch { /* never throw out of the backfill */ }
+      }
+    }
+    if (batch.length < batchSize) break;
+  }
+  return result;
+}
+
+export default { parseObsidian, indexObsidianRelations, backfillObsidianIndex };

package/src/memory/query-dataview.js

@@ -0,0 +1,86 @@
+// IJFW v1.5.0 -- Dataview-grade declarative query for memory_entries.
+//
+// Supported grammar (v1 — intentionally minimal; expand in v1.6+):
+//   tag = #path[/sub]*           prefix-matches memory_tags.tag_path
+//   linked_to = "target"         matches memory_links.to_target
+//   created_after = <unix-secs>  memory_entries.created_at > N
+//   created_before = <unix-secs> memory_entries.created_at < N
+//
+// Multiple filters AND together via case-insensitive " and ".
+// Whitespace-tolerant. Unrecognised clauses are silently skipped
+// (caller can inspect parsed.filters for __unrecognised entries).
+
+const CLAUSE_AND = /\s+and\s+/i;
+const TAG_RE = /^\s*tag\s*=\s*#?([\w/_-]+)\s*$/i;
+const LINKED_RE = /^\s*linked_to\s*=\s*"([^"\n]+)"\s*$/i;
+const CREATED_AFTER_RE = /^\s*created_after\s*=\s*(\d+)\s*$/i;
+const CREATED_BEFORE_RE = /^\s*created_before\s*=\s*(\d+)\s*$/i;
+
+export function parseDataviewQuery(input) {
+  const clauses = String(input || '').split(CLAUSE_AND);
+  const out = { filters: [] };
+  for (const raw of clauses) {
+    const c = raw.trim();
+    if (!c) continue;
+    let m;
+    if ((m = c.match(TAG_RE))) {
+      out.tag = m[1].toLowerCase().replace(/^\/+|\/+$/g, '');
+      continue;
+    }
+    if ((m = c.match(LINKED_RE))) {
+      out.filters.push({ field: 'linked_to', op: '=', value: m[1] });
+      continue;
+    }
+    if ((m = c.match(CREATED_AFTER_RE))) {
+      out.filters.push({ field: 'created_after', op: '=', value: Number(m[1]) });
+      continue;
+    }
+    if ((m = c.match(CREATED_BEFORE_RE))) {
+      out.filters.push({ field: 'created_before', op: '=', value: Number(m[1]) });
+      continue;
+    }
+    out.filters.push({ field: '__unrecognised', op: '=', value: c });
+  }
+  return out;
+}
+
+export function runDataviewQuery(db, parsed) {
+  const where = [];
+  const params = [];
+  let join = '';
+  if (parsed.tag) {
+    join += ' JOIN memory_tags t ON t.memory_id = e.id ';
+    where.push('(t.tag_path = ? OR t.tag_path LIKE ?)');
+    params.push(parsed.tag, `${parsed.tag}/%`);
+  }
+  for (const f of parsed.filters) {
+    switch (f.field) {
+      case 'linked_to':
+        join += ' JOIN memory_links l ON l.from_id = e.id ';
+        where.push('l.to_target = ?');
+        params.push(f.value);
+        break;
+      case 'created_after':
+        where.push('e.created_at > ?');
+        params.push(f.value);
+        break;
+      case 'created_before':
+        where.push('e.created_at < ?');
+        params.push(f.value);
+        break;
+      default: /* __unrecognised silently skipped */
+    }
+  }
+  // Note: memory_entries (migration 001 schema) has columns:
+  // id (INTEGER PK), body, source, session_id, created_at. No title column.
+  // memory_links/_tags/_meta store TEXT memory_id — SQLite coerces across
+  // the type boundary at JOIN time (SQLite is permissive without strict FKs).
+  const sql = `SELECT DISTINCT e.id, e.body, e.source, e.created_at
+               FROM memory_entries e ${join}
+               ${where.length ? 'WHERE ' + where.join(' AND ') : ''}
+               ORDER BY e.created_at DESC`;
+  const rows = db.prepare(sql).all(...params);
+  return { rows, rowCount: rows.length, parsed, sql };
+}
+
+export default { parseDataviewQuery, runDataviewQuery };

package/src/memory/search.js

@@ -31,6 +31,13 @@ import { readFileSync, existsSync, mkdirSync } from 'node:fs';
 import { dirname, join, resolve, normalize, isAbsolute } from 'node:path';
 
 import { expandQuery } from '../compute/synonyms.js';
+import { loadMigrations } from './migration-runner.js';
+// v1.5.1 R4-H2 — auto-index rows must flow through indexEntry so the
+// v1.5.0 memory-moat (M1 Obsidian indexing + M2 A-Mem auto-linking) fires
+// for warm-tier rebuilds, not just the benchmark harness. obsidian-parser
+// is imported directly so M1 runs synchronously inside the same txn batch.
+import { indexObsidianRelations } from './obsidian-parser.js';
+import { autoLink } from './auto-linker.js';
 
 const MAX_RESULTS  = 50;
 const SNIPPET_HALF = 60;
@@ -50,20 +57,16 @@ try {
 }
 
 // Resolve migration modules synchronously at module load via top-level
-// await. Replayed inside searchMemory's sync path. Keep in lockstep with
-// ./migrations/.
-const MEMORY_MIGRATIONS = await loadMemoryMigrationsSync();
-
-async function loadMemoryMigrationsSync() {
-  const v1 = await import('./migrations/001-fts5-init.js');
-  const v2 = await import('./migrations/002-tier-semantic.js');
-  const v3 = await import('./migrations/003-stale-candidate.js');
-  return [
-    { version: v1.VERSION, description: v1.DESCRIPTION, up: v1.up },
-    { version: v2.VERSION, description: v2.DESCRIPTION, up: v2.up },
-    { version: v3.VERSION, description: v3.DESCRIPTION, up: v3.up },
-  ].sort((a, b) => a.version - b.version);
-}
+// await. Replayed inside searchMemory's sync path.
+//
+// v1.5.1 W3.B: discovery is delegated to memory/migration-runner.js
+// (readdirSync over ./migrations/) so a single source of truth governs
+// which migrations search.js knows about. Prior to this, search.js
+// carried its OWN hardcoded list -- the v1.5.0 INT.7 hotfix patched
+// the symptom (006/007/008 missing); this kills the dual-registry bug
+// class outright. Drop migration 009 into ./migrations/, and search.js
+// will pick it up automatically.
+const MEMORY_MIGRATIONS = await loadMigrations();
 
 function highestMigrationVersion() {
   if (!MEMORY_MIGRATIONS.length) return 0;
@@ -210,12 +213,20 @@ function runMemoryMigrationsSync(db, currentVersion, targetVersion) {
 
 function autoIndex(db, files) {
   let n = 0;
+  // v1.5.1 R4-H2 — capture the rowid of every inserted entry so the
+  // memory-moat aux indexing (M1 Obsidian relations, M2 auto-link) can run
+  // over the warm-tier rebuild, not just the benchmark harness. The bulk
+  // INSERT stays in one transaction for FTS write performance; M1/M2 run
+  // AFTER commit so a parse/link failure can never abort the rebuild.
+  const inserted = [];
   const txfn = db.transaction((batch) => {
     const stmt = db.prepare(
       'INSERT INTO memory_entries (body, source, session_id, created_at) VALUES (?, ?, ?, ?)'
     );
     for (const item of batch) {
-      stmt.run(item.body, item.source, null, item.created_at);
+      const info = stmt.run(item.body, item.source, null, item.created_at);
+      const id = info && info.lastInsertRowid != null ? Number(info.lastInsertRowid) : null;
+      inserted.push({ id, body: item.body });
       n++;
     }
   });
@@ -232,6 +243,26 @@ function autoIndex(db, files) {
   }
   if (batch.length === 0) return 0;
   try { txfn.immediate(batch); } catch { /* one bad batch should not abort the search */ }
+
+  // v1.5.1 R4-H2 — M1: Obsidian wikilink/tag/meta indexing into
+  // memory_links/_tags/_meta. Synchronous + idempotent (indexObsidianRelations
+  // clears prior rows for the id before re-inserting). Best-effort: a missing
+  // migration-006 schema or a parse failure must never break the search path.
+  // M2: A-Mem auto-linking — fire-and-forget, env-gated (IJFW_AUTOLINK_OFF),
+  // budget-capped (IJFW_AUTOLINK_BUDGET_USD); returns skipped cleanly when no
+  // API key, so a bulk rebuild without credentials does no LLM work.
+  for (const row of inserted) {
+    if (row.id == null) continue;
+    try {
+      indexObsidianRelations(db, String(row.id), row.body);
+    } catch { /* M1 best-effort -- never abort the search */ }
+    try {
+      const p = autoLink(db, { id: row.id, body: row.body });
+      if (p && typeof p.catch === 'function') p.catch(() => {});
+      // expose for tests that want deterministic completion
+      autoIndex.__lastAutoLinkPromise = p;
+    } catch { /* M2 dispatch best-effort */ }
+  }
   return n;
 }