@ijfw/memory-server 1.5.0 → 1.5.1

package/src/memory/auto-linker.js

@@ -149,4 +149,119 @@ export async function autoLink(db, entry, opts = {}) {
   return { skipped: false, neighbors, proposal, applied };
 }
 
-export default { autoLink };
+// v1.5.1 R5-1.2 -- one-time M2 (A-Mem auto-link) backfill for memory written
+// during v1.5.0, when autoLink was NOT wired into the production write path.
+//
+// UNLIKE the M1 backfill (free, always-on), M2 backfill makes one LLM call
+// per row -- backfilling over a large memory can cost real money. So M2
+// backfill is OPT-IN and budget-gated:
+//
+//   - IJFW_AUTOLINK_OFF=1            -> backfill is a no-op (kill switch)
+//   - IJFW_AUTOLINK_BACKFILL!=1      -> backfill is a no-op by default
+//                                       (M1-always, M2-opt-in is the safe
+//                                        default per R5-1.2)
+//   - IJFW_AUTOLINK_BUDGET_USD unset -> backfill is a no-op. A budget MUST be
+//     OR <= 0                           explicitly configured. The per-call
+//                                       llm-call.js path treats an unset
+//                                       budget as "uncapped"; for a bulk
+//                                       backfill that is unsafe -- a large
+//                                       memory could spend without bound. So
+//                                       the backfill REQUIRES a positive cap.
+//   - no API key                    -> backfill is a no-op (autoLink skips)
+//
+// The per-row autoLink call independently re-checks the SAME env gates (off /
+// budget / key), so even mid-run the backfill respects a budget that drops to
+// zero or a kill switch that flips. Returns aggregate counts.
+export async function backfillAutoLink(db, opts = {}) {
+  if (!db || typeof db.prepare !== 'function') {
+    throw new Error('backfillAutoLink: db handle is invalid.');
+  }
+  const force = opts.force === true;
+  // Opt-in gate: M2 backfill only runs when explicitly enabled. M1 backfill
+  // (obsidian-parser.js) is the always-on default; M2 costs money so it is
+  // off unless the operator opts in via IJFW_AUTOLINK_BACKFILL=1.
+  if (!force && process.env.IJFW_AUTOLINK_BACKFILL !== '1') {
+    return { skipped: true, reason: 'backfill_not_enabled', rows: 0 };
+  }
+  if (process.env.IJFW_AUTOLINK_OFF === '1') {
+    return { skipped: true, reason: 'autolink_off', rows: 0 };
+  }
+  // Budget cap is MANDATORY for the backfill. An unset budget means
+  // llm-call.js runs uncapped -- fine for one-off write-time autoLink, but a
+  // bulk backfill over thousands of rows would spend without bound. Refuse
+  // unless the operator has set a positive IJFW_AUTOLINK_BUDGET_USD.
+  const budget = process.env.IJFW_AUTOLINK_BUDGET_USD;
+  if (budget === undefined || !(Number(budget) > 0)) {
+    return {
+      skipped: true,
+      reason: budget === undefined ? 'budget_not_set' : 'budget_exhausted',
+      rows: 0,
+    };
+  }
+  const hasKey = !!(process.env.IJFW_AUTOLINK_API_KEY || process.env.ANTHROPIC_API_KEY);
+  if (!hasKey) {
+    return { skipped: true, reason: 'no_key', rows: 0 };
+  }
+
+  const batchSize = Math.max(1, opts.batchSize || 200);
+  const result = {
+    skipped: false, rows: 0, linked: 0, links_added: 0,
+    neighbor_tags_added: 0, stopped_early: false,
+  };
+  let lastId = 0;
+  // eslint-disable-next-line no-constant-condition
+  while (true) {
+    let batch;
+    try {
+      batch = db
+        .prepare(
+          'SELECT id, body FROM memory_entries WHERE id > ? ORDER BY id ASC LIMIT ?',
+        )
+        .all(lastId, batchSize);
+    } catch {
+      break;
+    }
+    if (!batch || batch.length === 0) break;
+    for (const row of batch) {
+      lastId = row.id;
+      if (typeof row.body !== 'string' || row.body.length === 0) continue;
+      // Per-row re-check: a budget that drops to zero or a kill switch that
+      // flips mid-run stops the backfill before the next paid call. autoLink
+      // itself ALSO re-checks, but stopping here avoids the wasted SELECT.
+      if (process.env.IJFW_AUTOLINK_OFF === '1') {
+        result.stopped_early = true;
+        return result;
+      }
+      const b = process.env.IJFW_AUTOLINK_BUDGET_USD;
+      if (b === undefined || !(Number(b) > 0)) {
+        result.stopped_early = true;
+        return result;
+      }
+      result.rows += 1;
+      let res;
+      try {
+        res = await autoLink(db, { id: row.id, body: row.body });
+      } catch {
+        continue;
+      }
+      if (res && res.skipped) {
+        // autoLink skipped (budget exhausted / off / no key / parse fail).
+        // budget_exhausted + autolink_off mean stop the whole run.
+        if (res.reason === 'budget_exhausted' || res.reason === 'autolink_off') {
+          result.stopped_early = true;
+          return result;
+        }
+        continue;
+      }
+      result.linked += 1;
+      if (res && res.applied) {
+        result.links_added += res.applied.links_added || 0;
+        result.neighbor_tags_added += res.applied.neighbor_tags_added || 0;
+      }
+    }
+    if (batch.length < batchSize) break;
+  }
+  return result;
+}
+
+export default { autoLink, backfillAutoLink };

package/src/memory/migration-runner.js

@@ -29,7 +29,12 @@ export class SchemaVersionError extends Error {
 // Discover and load every migration module under ./migrations/, sorted by
 // numeric prefix ascending. Each module must export VERSION (integer),
 // DESCRIPTION (string), and up(db) (function).
-async function loadMigrations() {
+//
+// Exported (v1.5.1 W3.B) so search.js (and any other consumer that needs
+// the sync migration pipeline) can reuse the SAME discovery path instead
+// of maintaining a parallel hardcoded list. Single source of truth: drop
+// a new NNN-name.js into ./migrations/ and every consumer sees it.
+export async function loadMigrations() {
   let files;
   try {
     files = readdirSync(MIGRATIONS_DIR);

package/src/memory/migrations/009-obsidian-backfill.js

@@ -0,0 +1,50 @@
+// IJFW v1.5.1 -- memory migration 009: M1 obsidian-index backfill.
+//
+// Source authority: Trident r5 finding 1.2 (HIGH).
+//
+// Round-4 Fix-1 (commit 3218812) wired M1 (Obsidian wikilink/tag/meta
+// indexing -- indexObsidianRelations) and M2 (A-Mem auto-linking -- autoLink)
+// into the production memory-write path (handleStore, search.js#autoIndex).
+// But the fix is forward-only: every memory_entries row written during
+// v1.5.0 -- when M1/M2 were bypassed -- has empty memory_links / memory_tags
+// / memory_meta. An existing user upgrading to v1.5.1 got auto-linking +
+// wikilink indexing only on NEW entries; their accumulated memory stayed
+// un-indexed.
+//
+// This migration runs a ONE-TIME M1 backfill: it walks every existing
+// memory_entries row and runs indexObsidianRelations over its body. M1 is:
+//   - free       -- pure markdown parse, zero LLM / network / cost
+//   - idempotent -- indexObsidianRelations does DELETE-then-INSERT per id,
+//                   so re-applying produces identical aux rows
+// which is exactly what makes it safe to run inside a schema migration: it
+// runs once (user_version gates re-application), deterministically, and a
+// crash rolls the whole txn back to user_version 8.
+//
+// M2 (autoLink) is NOT backfilled here -- it makes one LLM call per row and
+// can cost real money over a large memory. M2 backfill is opt-in via the
+// `ijfw memory reindex --m2` CLI verb, which is budget-gated (respects
+// IJFW_AUTOLINK_BUDGET_USD / IJFW_AUTOLINK_OFF / IJFW_AUTOLINK_BACKFILL).
+//
+// Ordering: migration 001 creates memory_entries; migration 006 creates
+// memory_links / memory_tags / memory_meta. Both run before 009, so by the
+// time up() executes the source table and the three aux tables all exist.
+//
+// Crash safety: the migration runner wraps up() in BEGIN IMMEDIATE.
+// backfillObsidianIndex's per-row indexObsidianRelations opens a nested
+// SQLite transaction (savepoint) -- valid inside the outer txn -- and the
+// whole thing rolls back to user_version 8 on any failure.
+
+import { backfillObsidianIndex } from '../obsidian-parser.js';
+
+export const VERSION = 9;
+export const DESCRIPTION =
+  'memory v1.5.1 -- one-time M1 obsidian-index backfill for pre-fix rows (Trident r5 1.2)';
+
+export function up(db) {
+  // backfillObsidianIndex tolerates a missing memory_entries table (returns
+  // zero counts) so a brand-new db that jumps straight to v9 is a clean
+  // no-op -- there are no pre-fix rows to backfill on a fresh install.
+  backfillObsidianIndex(db);
+}
+
+export default { version: VERSION, description: DESCRIPTION, up };

package/src/memory/obsidian-parser.js

@@ -88,4 +88,65 @@ export function indexObsidianRelations(db, memoryId, text) {
   return parsed;
 }
 
-export default { parseObsidian, indexObsidianRelations };
+// v1.5.1 R5-1.2 -- one-time M1 backfill for memory written during v1.5.0,
+// when indexObsidianRelations was NOT wired into the production write path.
+// Round-4 Fix-1 (commit 3218812) wired M1+M2 into handleStore/autoIndex but
+// forward-only: rows already in memory_entries have empty memory_links /
+// memory_tags / memory_meta. This walks EVERY row and re-runs M1 over it.
+//
+// Safe to run over everything:
+//   - free      -- pure markdown parse, zero LLM / network
+//   - idempotent-- indexObsidianRelations clears prior aux rows per id before
+//                  re-inserting, so a re-run produces identical state
+//
+// The walk reads ids in batches so a very large memory_entries doesn't pin
+// the whole table in memory; each row's indexObsidianRelations call carries
+// its own transaction (DELETE-then-INSERT) so a single bad row never aborts
+// the rest of the backfill.
+//
+// Returns { rows, links, tags, meta } -- counts re-indexed across the run.
+export function backfillObsidianIndex(db, opts = {}) {
+  if (!db || typeof db.prepare !== 'function') {
+    throw new Error('backfillObsidianIndex: db handle is invalid.');
+  }
+  const batchSize = Math.max(1, opts.batchSize || 500);
+  const result = { rows: 0, links: 0, tags: 0, meta: 0, errors: 0 };
+  let lastId = 0;
+  // eslint-disable-next-line no-constant-condition
+  while (true) {
+    let batch;
+    try {
+      batch = db
+        .prepare(
+          'SELECT id, body FROM memory_entries WHERE id > ? ORDER BY id ASC LIMIT ?',
+        )
+        .all(lastId, batchSize);
+    } catch {
+      // memory_entries missing (fresh db before migration 001) -- nothing to do.
+      break;
+    }
+    if (!batch || batch.length === 0) break;
+    for (const row of batch) {
+      lastId = row.id;
+      if (typeof row.body !== 'string' || row.body.length === 0) continue;
+      try {
+        const parsed = indexObsidianRelations(db, String(row.id), row.body);
+        result.rows += 1;
+        result.links += parsed.links.length;
+        result.tags += parsed.tags.length;
+        result.meta += parsed.meta.length;
+      } catch (e) {
+        result.errors += 1;
+        try {
+          console.error(
+            '[obsidian] backfill failed for id', row.id, ':', e?.message || e,
+          );
+        } catch { /* never throw out of the backfill */ }
+      }
+    }
+    if (batch.length < batchSize) break;
+  }
+  return result;
+}
+
+export default { parseObsidian, indexObsidianRelations, backfillObsidianIndex };

package/src/memory/search.js

@@ -31,6 +31,13 @@ import { readFileSync, existsSync, mkdirSync } from 'node:fs';
 import { dirname, join, resolve, normalize, isAbsolute } from 'node:path';
 
 import { expandQuery } from '../compute/synonyms.js';
+import { loadMigrations } from './migration-runner.js';
+// v1.5.1 R4-H2 — auto-index rows must flow through indexEntry so the
+// v1.5.0 memory-moat (M1 Obsidian indexing + M2 A-Mem auto-linking) fires
+// for warm-tier rebuilds, not just the benchmark harness. obsidian-parser
+// is imported directly so M1 runs synchronously inside the same txn batch.
+import { indexObsidianRelations } from './obsidian-parser.js';
+import { autoLink } from './auto-linker.js';
 
 const MAX_RESULTS  = 50;
 const SNIPPET_HALF = 60;
@@ -50,30 +57,16 @@ try {
 }
 
 // Resolve migration modules synchronously at module load via top-level
-// await. Replayed inside searchMemory's sync path. Keep in lockstep with
-// ./migrations/.
-const MEMORY_MIGRATIONS = await loadMemoryMigrationsSync();
-
-async function loadMemoryMigrationsSync() {
-  const v1 = await import('./migrations/001-fts5-init.js');
-  const v2 = await import('./migrations/002-tier-semantic.js');
-  const v3 = await import('./migrations/003-stale-candidate.js');
-  const v4 = await import('./migrations/004-bitemporal.js');
-  const v5 = await import('./migrations/005-vector-cache.js');
-  const v6 = await import('./migrations/006-obsidian-graph.js');
-  const v7 = await import('./migrations/007-skill-telemetry.js');
-  const v8 = await import('./migrations/008-write-provenance.js');
-  return [
-    { version: v1.VERSION, description: v1.DESCRIPTION, up: v1.up },
-    { version: v2.VERSION, description: v2.DESCRIPTION, up: v2.up },
-    { version: v3.VERSION, description: v3.DESCRIPTION, up: v3.up },
-    { version: v4.VERSION, description: v4.DESCRIPTION, up: v4.up },
-    { version: v5.VERSION, description: v5.DESCRIPTION, up: v5.up },
-    { version: v6.VERSION, description: v6.DESCRIPTION, up: v6.up },
-    { version: v7.VERSION, description: v7.DESCRIPTION, up: v7.up },
-    { version: v8.VERSION, description: v8.DESCRIPTION, up: v8.up },
-  ].sort((a, b) => a.version - b.version);
-}
+// await. Replayed inside searchMemory's sync path.
+//
+// v1.5.1 W3.B: discovery is delegated to memory/migration-runner.js
+// (readdirSync over ./migrations/) so a single source of truth governs
+// which migrations search.js knows about. Prior to this, search.js
+// carried its OWN hardcoded list -- the v1.5.0 INT.7 hotfix patched
+// the symptom (006/007/008 missing); this kills the dual-registry bug
+// class outright. Drop migration 009 into ./migrations/, and search.js
+// will pick it up automatically.
+const MEMORY_MIGRATIONS = await loadMigrations();
 
 function highestMigrationVersion() {
   if (!MEMORY_MIGRATIONS.length) return 0;
@@ -220,12 +213,20 @@ function runMemoryMigrationsSync(db, currentVersion, targetVersion) {
 
 function autoIndex(db, files) {
   let n = 0;
+  // v1.5.1 R4-H2 — capture the rowid of every inserted entry so the
+  // memory-moat aux indexing (M1 Obsidian relations, M2 auto-link) can run
+  // over the warm-tier rebuild, not just the benchmark harness. The bulk
+  // INSERT stays in one transaction for FTS write performance; M1/M2 run
+  // AFTER commit so a parse/link failure can never abort the rebuild.
+  const inserted = [];
   const txfn = db.transaction((batch) => {
     const stmt = db.prepare(
       'INSERT INTO memory_entries (body, source, session_id, created_at) VALUES (?, ?, ?, ?)'
     );
     for (const item of batch) {
-      stmt.run(item.body, item.source, null, item.created_at);
+      const info = stmt.run(item.body, item.source, null, item.created_at);
+      const id = info && info.lastInsertRowid != null ? Number(info.lastInsertRowid) : null;
+      inserted.push({ id, body: item.body });
       n++;
     }
   });
@@ -242,6 +243,26 @@ function autoIndex(db, files) {
   }
   if (batch.length === 0) return 0;
   try { txfn.immediate(batch); } catch { /* one bad batch should not abort the search */ }
+
+  // v1.5.1 R4-H2 — M1: Obsidian wikilink/tag/meta indexing into
+  // memory_links/_tags/_meta. Synchronous + idempotent (indexObsidianRelations
+  // clears prior rows for the id before re-inserting). Best-effort: a missing
+  // migration-006 schema or a parse failure must never break the search path.
+  // M2: A-Mem auto-linking — fire-and-forget, env-gated (IJFW_AUTOLINK_OFF),
+  // budget-capped (IJFW_AUTOLINK_BUDGET_USD); returns skipped cleanly when no
+  // API key, so a bulk rebuild without credentials does no LLM work.
+  for (const row of inserted) {
+    if (row.id == null) continue;
+    try {
+      indexObsidianRelations(db, String(row.id), row.body);
+    } catch { /* M1 best-effort -- never abort the search */ }
+    try {
+      const p = autoLink(db, { id: row.id, body: row.body });
+      if (p && typeof p.catch === 'function') p.catch(() => {});
+      // expose for tests that want deterministic completion
+      autoIndex.__lastAutoLinkPromise = p;
+    } catch { /* M2 dispatch best-effort */ }
+  }
   return n;
 }