@ijfw/memory-server 1.5.0 → 1.5.3

package/src/memory/search.js

@@ -31,6 +31,13 @@ import { readFileSync, existsSync, mkdirSync } from 'node:fs';
 import { dirname, join, resolve, normalize, isAbsolute } from 'node:path';
 
 import { expandQuery } from '../compute/synonyms.js';
+import { loadMigrations } from './migration-runner.js';
+// v1.5.1 R4-H2 — auto-index rows must flow through indexEntry so the
+// v1.5.0 memory-moat (M1 Obsidian indexing + M2 A-Mem auto-linking) fires
+// for warm-tier rebuilds, not just the benchmark harness. obsidian-parser
+// is imported directly so M1 runs synchronously inside the same txn batch.
+import { indexObsidianRelations } from './obsidian-parser.js';
+import { autoLink } from './auto-linker.js';
 
 const MAX_RESULTS  = 50;
 const SNIPPET_HALF = 60;
@@ -50,30 +57,16 @@ try {
 }
 
 // Resolve migration modules synchronously at module load via top-level
-// await. Replayed inside searchMemory's sync path. Keep in lockstep with
-// ./migrations/.
-const MEMORY_MIGRATIONS = await loadMemoryMigrationsSync();
-
-async function loadMemoryMigrationsSync() {
-  const v1 = await import('./migrations/001-fts5-init.js');
-  const v2 = await import('./migrations/002-tier-semantic.js');
-  const v3 = await import('./migrations/003-stale-candidate.js');
-  const v4 = await import('./migrations/004-bitemporal.js');
-  const v5 = await import('./migrations/005-vector-cache.js');
-  const v6 = await import('./migrations/006-obsidian-graph.js');
-  const v7 = await import('./migrations/007-skill-telemetry.js');
-  const v8 = await import('./migrations/008-write-provenance.js');
-  return [
-    { version: v1.VERSION, description: v1.DESCRIPTION, up: v1.up },
-    { version: v2.VERSION, description: v2.DESCRIPTION, up: v2.up },
-    { version: v3.VERSION, description: v3.DESCRIPTION, up: v3.up },
-    { version: v4.VERSION, description: v4.DESCRIPTION, up: v4.up },
-    { version: v5.VERSION, description: v5.DESCRIPTION, up: v5.up },
-    { version: v6.VERSION, description: v6.DESCRIPTION, up: v6.up },
-    { version: v7.VERSION, description: v7.DESCRIPTION, up: v7.up },
-    { version: v8.VERSION, description: v8.DESCRIPTION, up: v8.up },
-  ].sort((a, b) => a.version - b.version);
-}
+// await. Replayed inside searchMemory's sync path.
+//
+// v1.5.1 W3.B: discovery is delegated to memory/migration-runner.js
+// (readdirSync over ./migrations/) so a single source of truth governs
+// which migrations search.js knows about. Prior to this, search.js
+// carried its OWN hardcoded list -- the v1.5.0 INT.7 hotfix patched
+// the symptom (006/007/008 missing); this kills the dual-registry bug
+// class outright. Drop migration 009 into ./migrations/, and search.js
+// will pick it up automatically.
+const MEMORY_MIGRATIONS = await loadMigrations();
 
 function highestMigrationVersion() {
   if (!MEMORY_MIGRATIONS.length) return 0;
@@ -220,12 +213,20 @@ function runMemoryMigrationsSync(db, currentVersion, targetVersion) {
 
 function autoIndex(db, files) {
   let n = 0;
+  // v1.5.1 R4-H2 — capture the rowid of every inserted entry so the
+  // memory-moat aux indexing (M1 Obsidian relations, M2 auto-link) can run
+  // over the warm-tier rebuild, not just the benchmark harness. The bulk
+  // INSERT stays in one transaction for FTS write performance; M1/M2 run
+  // AFTER commit so a parse/link failure can never abort the rebuild.
+  const inserted = [];
   const txfn = db.transaction((batch) => {
     const stmt = db.prepare(
       'INSERT INTO memory_entries (body, source, session_id, created_at) VALUES (?, ?, ?, ?)'
     );
     for (const item of batch) {
-      stmt.run(item.body, item.source, null, item.created_at);
+      const info = stmt.run(item.body, item.source, null, item.created_at);
+      const id = info && info.lastInsertRowid != null ? Number(info.lastInsertRowid) : null;
+      inserted.push({ id, body: item.body });
       n++;
     }
   });
@@ -242,6 +243,26 @@ function autoIndex(db, files) {
   }
   if (batch.length === 0) return 0;
   try { txfn.immediate(batch); } catch { /* one bad batch should not abort the search */ }
+
+  // v1.5.1 R4-H2 — M1: Obsidian wikilink/tag/meta indexing into
+  // memory_links/_tags/_meta. Synchronous + idempotent (indexObsidianRelations
+  // clears prior rows for the id before re-inserting). Best-effort: a missing
+  // migration-006 schema or a parse failure must never break the search path.
+  // M2: A-Mem auto-linking — fire-and-forget, env-gated (IJFW_AUTOLINK_OFF),
+  // budget-capped (IJFW_AUTOLINK_BUDGET_USD); returns skipped cleanly when no
+  // API key, so a bulk rebuild without credentials does no LLM work.
+  for (const row of inserted) {
+    if (row.id == null) continue;
+    try {
+      indexObsidianRelations(db, String(row.id), row.body);
+    } catch { /* M1 best-effort -- never abort the search */ }
+    try {
+      const p = autoLink(db, { id: row.id, body: row.body });
+      if (p && typeof p.catch === 'function') p.catch(() => {});
+      // expose for tests that want deterministic completion
+      autoIndex.__lastAutoLinkPromise = p;
+    } catch { /* M2 dispatch best-effort */ }
+  }
   return n;
 }
 
@@ -325,6 +346,108 @@ function rowCount(db) {
   }
 }
 
+// --- Structured provenance helpers -----------------------------------------
+
+/**
+ * Convert a raw FTS row + fileBySource map to a structured provenance object.
+ * Used when opts.format === 'structured'.
+ *
+ * Fields that aren't computed in the existing pipeline are returned as
+ * null / 0 rather than introducing new compute work (Task 28 spec).
+ *
+ * @param {object} row  - raw DB row from searchFts5
+ * @param {Map}    fileBySource
+ * @param {string} rawQuery  - original user query (for whyMatched extraction)
+ * @param {object} db        - open DB handle (for backlink count query)
+ * @returns {object}
+ */
+function ftsRowToStructured(row, fileBySource, rawQuery, db) {
+  const src = row.source || '';
+  const meta = fileBySource.get(src) || null;
+  const source = (meta && meta.path) || src;
+  const text = String(row.body || '');
+  const snip = text.slice(0, 200).replace(/\s+/g, ' ').trim();
+
+  // confidence: bm25 rank is negative (more negative = better). Convert to 0..1.
+  // rank returned from searchFts5 can be 0 or negative; we use the same
+  // score formula as ftsRowToResult (100 - rank) but normalise to 0..1 by
+  // clamping to [0, 100] and dividing.
+  const rawRank = Number(row.rank || 0);
+  const scoreRaw = 100 - rawRank;         // same as ftsRowToResult
+  const confidence = Math.min(1, Math.max(0, scoreRaw / 100));
+
+  // ageDays: created_at is unix ms
+  const createdAt = Number(row.created_at || 0);
+  const ageDays = createdAt > 0
+    ? Math.max(0, (Date.now() - createdAt) / 86400000)
+    : 0;
+
+  // decayFactor: not yet computed in pipeline — return null per spec
+  const decayFactor = null;
+
+  // whyMatched: tokenise the raw query into distinct non-trivial terms
+  const whyMatched = rawQuery
+    .trim()
+    .split(/\s+/)
+    .map(t => t.replace(/['"*()]/g, '').toLowerCase())
+    .filter(t => t.length > 0);
+
+  // backlinkCount: count rows in memory_links where to_target matches source
+  let backlinkCount = 0;
+  if (db && row.id != null) {
+    try {
+      const idStr = String(row.id);
+      const r = db.prepare(
+        'SELECT COUNT(*) AS n FROM memory_links WHERE to_target = ?'
+      ).get(idStr);
+      backlinkCount = r ? Number(r.n) : 0;
+    } catch { /* memory_links may not exist in older dbs */ }
+  }
+
+  return {
+    source,
+    anchor: null,
+    snippet: snip,
+    confidence,
+    ageDays,
+    decayFactor,
+    whyMatched,
+    backlinkCount,
+  };
+}
+
+/**
+ * Convert a hot-linear result to a structured provenance object.
+ * Linear results lack a DB row id so backlinkCount is always 0.
+ *
+ * @param {object} result  - from searchLinear
+ * @param {string} rawQuery
+ * @returns {object}
+ */
+function linearResultToStructured(result, rawQuery) {
+  const scoreRaw = Number(result.score || 0);
+  // Hot-linear score is titleMatches*3 + bodyMatches; normalise loosely to 0..1
+  // by capping at 50 matches (arbitrary but safe)
+  const confidence = Math.min(1, scoreRaw / 50);
+
+  const whyMatched = rawQuery
+    .trim()
+    .split(/\s+/)
+    .map(t => t.replace(/['"*()]/g, '').toLowerCase())
+    .filter(t => t.length > 0);
+
+  return {
+    source: result.path || result.relpath || '',
+    anchor: null,
+    snippet: result.snippet || '',
+    confidence,
+    ageDays: 0,
+    decayFactor: null,
+    whyMatched,
+    backlinkCount: 0,
+  };
+}
+
 // --- Public API -------------------------------------------------------------
 
 /**
@@ -344,30 +467,41 @@ function rowCount(db) {
  *   the hot-linear fallback is unfiltered (D1 does not yet write tier
  *   metadata into the markdown surface).
  *
+ * format option (Task 28 — structured provenance):
+ *   opts.format === 'structured' returns an Array of provenance objects:
+ *     [{source, anchor, snippet, confidence, ageDays, decayFactor,
+ *       whyMatched, backlinkCount}]
+ *   Default (no format) returns Array<{path,relpath,title,snippet,score,
+ *   tier_semantic}> — byte-identical to pre-Task-28 behaviour.
+ *
  * @param {string} q
  * @param {Array<{path,relpath,title,preview}>} files
  * @param {number} limit
  * @param {object|undefined} options
- * @returns {Array<{path,relpath,title,snippet,score,tier_semantic}>}
+ * @returns {Array<{path,relpath,title,snippet,score,tier_semantic}>|Array<provenance>}
  */
 export function searchMemory(q, files, limit = MAX_RESULTS, options) {
   if (!q || !q.trim() || !files || files.length === 0) return [];
 
-  // Normalise options. Allow undefined / { tier_semantic, include_stale } /
-  // a bare string (treated as the tier_semantic value) for ergonomic call
-  // sites. include_stale defaults to false -- D4 GA-B2 retrieval guard.
+  // Normalise options. Allow undefined / { tier_semantic, include_stale,
+  // format } / a bare string (treated as the tier_semantic value) for
+  // ergonomic call sites. include_stale defaults to false -- D4 GA-B2
+  // retrieval guard. format === 'structured' enables Task-28 provenance.
   let tier_semantic;
   let include_stale = false;
+  let format;
   if (typeof options === 'string') {
     tier_semantic = options;
   } else if (options && typeof options === 'object') {
     tier_semantic = options.tier_semantic;
     include_stale = options.include_stale === true;
+    format = options.format;
   }
 
   const { expanded, synonym_matches, applied } = expandQuery(q);
 
   let warmHits = null;
+  let warmRawRows = null;  // preserved for structured format (Task 28)
   let warmEmpty = false;
   let db = null;
 
@@ -403,6 +537,11 @@ export function searchMemory(q, files, limit = MAX_RESULTS, options) {
             if (f.relpath) fileBySource.set(f.relpath, f);
             if (f.path) fileBySource.set(f.path, f);
           }
+          if (format === 'structured') {
+            // Task 28: map raw rows to provenance objects while db is still open
+            // (backlinkCount query needs the handle)
+            warmRawRows = rows.map(r => ftsRowToStructured(r, fileBySource, q, db));
+          }
           warmHits = rows.map(r => ftsRowToResult(r, fileBySource));
         } else {
           warmEmpty = true;
@@ -411,31 +550,41 @@ export function searchMemory(q, files, limit = MAX_RESULTS, options) {
     }
   } catch {
     warmHits = null;
+    warmRawRows = null;
   } finally {
     if (db) { try { db.close(); } catch { /* ignore */ } }
   }
 
   let results;
   if (warmHits && warmHits.length > 0) {
-    results = warmHits.slice(0, limit);
+    results = format === 'structured'
+      ? warmRawRows.slice(0, limit)
+      : warmHits.slice(0, limit);
   } else if (tier_semantic) {
     // Tier filter active and warm tier has no matches -- the hot-linear
     // tier doesn't carry tier metadata so it can't honour the filter.
     // Returning [] here keeps the contract honest ("only matching tier").
     results = [];
   } else {
-    results = searchLinear(q, files, limit);
+    const linearResults = searchLinear(q, files, limit);
+    results = format === 'structured'
+      ? linearResults.map(r => linearResultToStructured(r, q))
+      : linearResults;
   }
 
-  Object.defineProperty(results, 'synonym_matches', {
-    value: applied ? synonym_matches : {},
-    enumerable: false,
-  });
-  Object.defineProperty(results, 'tier', {
-    value: warmHits && warmHits.length > 0
-      ? 'warm-fts5'
-      : (warmEmpty ? 'hot-linear-empty-fts5' : 'hot-linear'),
-    enumerable: false,
-  });
+  // Structured results are plain arrays — no non-enumerable decorations needed.
+  // Legacy path: attach non-enumerable metadata as before (byte-identical).
+  if (format !== 'structured') {
+    Object.defineProperty(results, 'synonym_matches', {
+      value: applied ? synonym_matches : {},
+      enumerable: false,
+    });
+    Object.defineProperty(results, 'tier', {
+      value: warmHits && warmHits.length > 0
+        ? 'warm-fts5'
+        : (warmEmpty ? 'hot-linear-empty-fts5' : 'hot-linear'),
+      enumerable: false,
+    });
+  }
   return results;
 }

package/src/memory/temporal.js

@@ -333,7 +333,13 @@ export function storeFactBitemporal(db, fact, now) {
     const factId = insertFact(db, f, t);
     return { invalidated, factId };
   });
-  const r = txn(fact, ts);
+  // F2.7: .immediate() issues BEGIN IMMEDIATE — see brain-handler conflict.resolve.
+  // Sister writers (conflict.resolve) hold IMMEDIATE locks; if we ran DEFERRED here
+  // we would hit SQLITE_BUSY when upgrading SHARED→RESERVED on the first write
+  // inside the txn body and the user's memory write would silently drop (or
+  // throw SQLITE_BUSY at the caller). Lock-mode alignment with sister writers
+  // is what makes the cross-connection contract honest.
+  const r = txn.immediate(fact, ts);
   return { invalidated: r.invalidated, factId: r.factId, deduped: false };
 }
 
@@ -513,6 +519,38 @@ export function applyDecayToFacts(rows, now, options = {}) {
   });
 }
 
+/**
+ * getHistoryWindow -- bounded slice of facts about (subject[, predicate]) for
+ * the wiki compiler's "history" section. Returns at most `limit` rows ordered
+ * by valid_from DESC. When rollupOlder is true and rows.length === limit,
+ * also returns an `older` rollup of facts beyond the window so the wiki page
+ * can show "Older: 55 events between 2024-03-01 and 2025-06-30" without
+ * bloating the page.
+ *
+ * Trident F-B2 protection: prevents page-bloat for hot subjects with hundreds
+ * of facts.
+ */
+export function getHistoryWindow(db, subject, predicate, { limit = 50, since = null, rollupOlder = true } = {}) {
+  const params = [subject];
+  let where = 'subject = ?';
+  if (predicate != null) { where += ' AND predicate = ?'; params.push(predicate); }
+  if (since) { where += ' AND valid_from >= ?'; params.push(since); }
+  const rows = db.prepare(
+    `SELECT id, predicate, object, valid_from, valid_to, memory_id, source, confidence
+     FROM facts WHERE ${where} ORDER BY valid_from DESC LIMIT ?`
+  ).all(...params, limit);
+  let older = null;
+  if (rollupOlder && rows.length === limit) {
+    const earliest = rows[rows.length - 1].valid_from;
+    const r = db.prepare(
+      `SELECT COUNT(*) AS count, MIN(valid_from) AS fromIso, MAX(valid_from) AS toIso
+       FROM facts WHERE ${where} AND valid_from < ?`
+    ).get(...params, earliest);
+    if (r.count > 0) older = r;
+  }
+  return { rows, older };
+}
+
 export default {
   openTemporalDb,
   openTemporalDbSync,
@@ -524,6 +562,7 @@ export default {
   getHistory,
   getAllFactsWithWindows,
   applyDecayToFacts,
+  getHistoryWindow,
   DECAY_HALFLIFE_DAYS,
   DECAY_HALFLIFE_SESSION_DAYS,
 };

package/src/orchestrator/agents-md-blackboard.js

@@ -46,12 +46,17 @@
  * AGENTS.md hiccup — see `wave-state.js#checkpointWave`).
  */
 
-import { join } from 'node:path';
+import { join, resolve as pathResolve, dirname as pathDirname } from 'node:path';
 
 import { withFsLock, lockPathFor } from '../fs-lock.js';
 import { readWaveState } from './wave-state.js';
 import { mergeFile, MergeBlockAwareError } from './merge-block-aware.js';
 import { query } from './state-sdk.js';
+import {
+  selectDisciplineTemplate,
+  detectProjectTypeFromRepo,
+} from './discipline-selector.js';
+import { validateSafeRepoPath } from '../brain/path-guard.js';
 
 /**
  * Render the BLACKBOARD marker-block payload from a wave's STATE.md
@@ -93,8 +98,21 @@ export async function populateBlackboardBlock(waveId, projectRoot) {
   const state = await readWaveState(waveId, projectRoot);
   if (!state) return { ok: false, reason: 'no-state' };
 
+  // Defense-in-depth: refuse to operate at a filesystem root (e.g. '/' on
+  // POSIX, 'C:\\' on Windows). validateSafeRepoPath alone accepts these
+  // because '/AGENTS.md' is technically "inside" '/'; OS permissions would
+  // then catch the actual write, but the failure mode would be 'merge-error'
+  // not 'unsafe-path'. We want a clean structured rejection upstream of any
+  // I/O attempt. Test at integration/test-discipline-integration.js:189.
+  const resolvedRoot = pathResolve(projectRoot || '.');
+  if (resolvedRoot === pathDirname(resolvedRoot)) {
+    return { ok: false, reason: 'unsafe-path', error: 'projectRoot is a filesystem root' };
+  }
+
   const payload = renderBlackboardPayload(waveId, state);
   const agentsMdPath = join(projectRoot, 'AGENTS.md');
+  const guard = validateSafeRepoPath(projectRoot, agentsMdPath);
+  if (!guard.ok) return { ok: false, reason: 'unsafe-path', error: guard.error };
   const lockPath = lockPathFor(agentsMdPath);
 
   let mergeResult;
@@ -150,3 +168,98 @@ export async function populateBlackboardBlock(waveId, projectRoot) {
 
   return { ok: true };
 }
+
+/**
+ * Populate the DISCIPLINE marker block in `<projectRoot>/AGENTS.md` with the
+ * project-appropriate discipline template body. Held under the §3 #8 AGENTS.md
+ * lock; in-process (no spawn). Best-effort SDK event emit after lock release.
+ *
+ * If `projectType` is not supplied, `detectProjectTypeFromRepo(projectRoot)`
+ * is called to infer it. For `unknown` / `mixed` types the DISCIPLINE block
+ * is written with an empty body (marker present, body empty) -- this is the
+ * correct state, not an error.
+ *
+ * Return shapes:
+ *   `{ ok: true }`                               -- wrote AGENTS.md
+ *   `{ ok: false, reason: 'merge-error', error }` -- merger threw
+ *   `{ ok: false, reason: 'template-missing', error }` -- template file absent
+ *
+ * @param {string} projectRoot
+ * @param {string} [projectType]  optional; inferred when absent
+ * @param {{waveId?: string}} [opts]  optional; waveId defaults to 'system'
+ * @returns {Promise<{ok: boolean, reason?: string, error?: string}>}
+ */
+export async function populateDisciplineBlock(projectRoot, projectType, opts = {}) {
+  const waveId = opts.waveId || 'system';
+  const type = projectType !== undefined && projectType !== null
+    ? String(projectType)
+    : detectProjectTypeFromRepo(projectRoot);
+
+  let content;
+  try {
+    content = selectDisciplineTemplate(type);
+  } catch (err) {
+    return {
+      ok: false,
+      reason: 'template-missing',
+      error: String(err.message || err),
+    };
+  }
+
+  // Defense-in-depth: same filesystem-root rejection as populateBlackboardBlock.
+  const resolvedRoot = pathResolve(projectRoot || '.');
+  if (resolvedRoot === pathDirname(resolvedRoot)) {
+    return { ok: false, reason: 'unsafe-path', error: 'projectRoot is a filesystem root' };
+  }
+
+  const agentsMdPath = join(projectRoot, 'AGENTS.md');
+  const guard = validateSafeRepoPath(projectRoot, agentsMdPath);
+  if (!guard.ok) return { ok: false, reason: 'unsafe-path', error: guard.error };
+  const lockPath = lockPathFor(agentsMdPath);
+
+  let mergeResult;
+  let mergeError = null;
+  try {
+    mergeResult = await withFsLock(lockPath, async () => mergeFile(
+      agentsMdPath,
+      [{ block: 'DISCIPLINE', content }],
+    ));
+  } catch (err) {
+    mergeError = err;
+  }
+
+  if (mergeError) {
+    const code = mergeError instanceof MergeBlockAwareError ? mergeError.code : null;
+    return {
+      ok: false,
+      reason: code === 'ERR_TEMPLATE_MISSING' ? 'template-missing' : 'merge-error',
+      error: String(mergeError.message || mergeError),
+    };
+  }
+
+  // SDK-routed observability event -- fire-and-forget AFTER lock release.
+  // Mirrors the agents-md.blackboard.set emit pattern above.
+  try {
+    await query('event.emit', {
+      subagentId: 'parent',
+      waveId,
+      eventType: 'agents-md.discipline.set',
+      data: {
+        path: mergeResult?.path ?? agentsMdPath,
+        bytes: mergeResult?.bytes ?? 0,
+        seeded: !!mergeResult?.seeded,
+        project_type: type,
+      },
+      dedupKey: `agents-md.discipline.set:${waveId}:${type}`,
+    }, { projectRoot, subagentId: 'parent' });
+  } catch {
+    // Observability is best-effort; never demote a successful AGENTS.md
+    // rewrite because the event tap had a hiccup.
+  }
+
+  // Forward noop flag from the short-circuit so callers can detect idempotent
+  // calls (5B-L2-05). When mergeResult.noop is true the file was unchanged.
+  const result = { ok: true };
+  if (mergeResult?.noop) result.noop = true;
+  return result;
+}