@ijfw/memory-server 1.5.0 → 1.5.3

package/src/brain/promotion-suggester.js

@@ -0,0 +1,41 @@
+// IJFW v1.5.2 -- brain promotion suggester.
+//
+// Walks per-project db paths and surfaces (subject, predicate, object) triples
+// that appear across >= minProjects (default 3) projects. Promotion is ALWAYS
+// operator-confirmed -- this module only surfaces candidates.
+
+import Database from 'better-sqlite3';
+
+export function suggestPromotions({ projectDbs, minProjects = 3 } = {}) {
+  if (!Array.isArray(projectDbs) || projectDbs.length === 0) return [];
+  const tally = new Map();
+  for (const entry of projectDbs) {
+    const { name, dbPath } = entry;
+    let db;
+    try { db = new Database(dbPath, { readonly: true, fileMustExist: true }); }
+    catch { continue; }
+    try {
+      const rows = db.prepare(`
+        SELECT DISTINCT subject, predicate, object
+        FROM facts
+        WHERE valid_to IS NULL
+      `).all();
+      for (const r of rows) {
+        const key = `${r.subject || ''}|${r.predicate || ''}|${r.object || ''}`;
+        if (!tally.has(key)) tally.set(key, new Set());
+        tally.get(key).add(name);
+      }
+    } finally { try { db.close(); } catch {} }
+  }
+  const out = [];
+  for (const [key, projects] of tally) {
+    if (projects.size < minProjects) continue;
+    const [subject, predicate, object] = key.split('|');
+    out.push({
+      subject, predicate, object,
+      projects: [...projects].sort(),
+      confidence: projects.size >= 5 ? 'high' : 'medium',
+    });
+  }
+  return out.sort((a, b) => b.projects.length - a.projects.length);
+}

package/src/brain/stub-detector.js

@@ -0,0 +1,33 @@
+// IJFW v1.5.2 -- brain stub detector.
+//
+// A "stub" is a wikilink target with N+ incoming references but no actual
+// wiki page at ijfw/wiki/<type>s/<slug>.md. Surfaces gaps the operator may
+// want to fill. Checks both visible ijfw/ and legacy .ijfw/ paths during
+// the v1->v2 transition.
+
+import { existsSync } from 'node:fs';
+import { join } from 'node:path';
+
+const WIKI_TYPES = ['concepts', 'entities', 'decisions', 'milestones'];
+
+function pageExistsAnywhere(repoRoot, target) {
+  for (const t of WIKI_TYPES) {
+    if (existsSync(join(repoRoot, 'ijfw', 'wiki', t, `${target}.md`))) return true;
+    if (existsSync(join(repoRoot, '.ijfw', 'wiki', t, `${target}.md`))) return true;
+  }
+  return false;
+}
+
+export function detectStubs(db, repoRoot, { minIncomingLinks = 3 } = {}) {
+  const rows = db.prepare(`
+    SELECT to_target AS target, COUNT(*) AS incomingLinks
+    FROM memory_links
+    WHERE to_target IS NOT NULL AND to_target != ''
+    GROUP BY to_target
+    HAVING COUNT(*) >= ?
+  `).all(minIncomingLinks);
+  return rows
+    .filter((r) => !pageExistsAnywhere(repoRoot, r.target))
+    .map((r) => ({ target: r.target, incomingLinks: r.incomingLinks }))
+    .sort((a, b) => b.incomingLinks - a.incomingLinks);
+}

package/src/brain/tiered-llm.js

@@ -0,0 +1,83 @@
+// IJFW v1.5.2 -- brain tiered LLM router.
+//
+// Two tiers used by the dream-cycle pipeline:
+//   - extract: cheap, high-volume fact extraction (default: claude-haiku-4-5-20251001)
+//   - synth:   mid-tier reconciliation / page synthesis (default: claude-sonnet-4-6)
+//
+// When IJFW_BRAIN_LOCAL_URL is set, we try the local endpoint first (Ollama-
+// compatible /api/generate). On any local-call error, we fall back to the
+// Anthropic API. This makes "local-first" cost discipline opt-in via env.
+//
+// Tests inject custom callers via opts._callers to avoid hitting real APIs.
+
+const DEFAULTS = {
+  extract: 'claude-haiku-4-5-20251001',
+  synth: 'claude-sonnet-4-6',
+};
+
+const DEFAULT_MAX_TOKENS = {
+  extract: 512,
+  synth: 1500,
+};
+
+export function resolveTierModel(tier, env = process.env) {
+  if (tier === 'extract') return env.IJFW_BRAIN_EXTRACT_MODEL || DEFAULTS.extract;
+  if (tier === 'synth') return env.IJFW_BRAIN_SYNTH_MODEL || DEFAULTS.synth;
+  throw new Error(`tiered-llm: unknown tier '${tier}'`);
+}
+
+function defaultCallers() {
+  return {
+    async local({ url, model, prompt, maxTokens }) {
+      // Ollama-compatible /api/generate -- streamless single-response mode.
+      const res = await fetch(url.replace(/\/$/, '') + '/api/generate', {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        body: JSON.stringify({ model, prompt, stream: false, options: { num_predict: maxTokens } }),
+      });
+      if (!res.ok) throw new Error(`local LLM HTTP ${res.status}`);
+      const data = await res.json();
+      return { text: data.response || '', usage: { input: data.prompt_eval_count, output: data.eval_count }, model, via: 'local' };
+    },
+    async anthropic({ model, prompt, maxTokens, apiKey }) {
+      if (!apiKey) throw new Error('tiered-llm: ANTHROPIC_API_KEY (or IJFW_BRAIN_API_KEY) required for Anthropic fallback');
+      const res = await fetch('https://api.anthropic.com/v1/messages', {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+          'x-api-key': apiKey,
+          'anthropic-version': '2023-06-01',
+        },
+        body: JSON.stringify({
+          model,
+          max_tokens: maxTokens,
+          messages: [{ role: 'user', content: prompt }],
+        }),
+      });
+      if (!res.ok) throw new Error(`Anthropic HTTP ${res.status}`);
+      const data = await res.json();
+      const text = (data.content || []).map((b) => b.text || '').join('');
+      return { text, usage: data.usage, model, via: 'anthropic' };
+    },
+  };
+}
+
+export async function callTiered(tier, prompt, opts = {}) {
+  const env = opts.env || process.env;
+  const model = resolveTierModel(tier, env);
+  const maxTokens = opts.maxTokens || DEFAULT_MAX_TOKENS[tier] || 512;
+  const callers = opts._callers || defaultCallers();
+  if (env.IJFW_BRAIN_LOCAL_URL) {
+    try {
+      return await callers.local({ url: env.IJFW_BRAIN_LOCAL_URL, model, prompt, maxTokens });
+    } catch {
+      // fall through to Anthropic
+    }
+  }
+  return callers.anthropic({
+    model,
+    prompt,
+    maxTokens,
+    apiKey: env.IJFW_BRAIN_API_KEY || env.ANTHROPIC_API_KEY,
+  });
+}

package/src/brain/wiki-compiler.js

@@ -0,0 +1,144 @@
+// IJFW v1.5.2 -- brain wiki compiler (Trident F-B1 + F-F2 enforcement).
+//
+// compileWikiPage(db, { repoRoot, type, subject }) renders the page from
+// structured facts + history + backlinks + sources, runs the result through
+// resolveCitations(), and ATOMICALLY writes (.tmp + rename) only if every
+// citation resolves. Returns {ok:false, unresolved[]} when any cite is
+// dangling -- this is the hallucination defense.
+//
+// Wiki path: <ijfw|.ijfw>/wiki/<type>s/<slug>.md per the layout sentinel.
+// Slug is the subject lowercased + non-alphanum -> '-'.
+
+import { existsSync, mkdirSync, readFileSync, writeFileSync, renameSync, openSync, closeSync, unlinkSync, statSync } from 'node:fs';
+import { join } from 'node:path';
+
+const STALE_LOCK_MS = 60_000; // 60s — locks older than this are assumed dead-process orphans
+import { resolveBrainPaths } from './paths.js';
+import { applyTemplate } from './wiki-templates.js';
+import { resolveCitations } from './citation-resolver.js';
+import { getHistoryWindow } from '../memory/temporal.js';
+import { validateSafeRepoPath } from './path-guard.js';
+
+export function slugify(s) {
+  return String(s || '').toLowerCase().trim()
+    .replace(/[^a-z0-9]+/g, '-')
+    .replace(/^-+|-+$/g, '') || 'untitled';
+}
+
+function pluralType(type) {
+  // Templates store under <type>s/ -- e.g. type='entity' -> 'entities'.
+  if (type === 'entity') return 'entities';
+  if (type === 'concept') return 'concepts';
+  if (type === 'decision') return 'decisions';
+  if (type === 'milestone') return 'milestones';
+  return `${type}s`;
+}
+
+function queryFacts(db, subject) {
+  try {
+    return db.prepare(
+      'SELECT id, predicate, object, valid_from, valid_to, memory_id, source, confidence FROM facts WHERE subject = ? AND valid_to IS NULL ORDER BY id DESC'
+    ).all(subject);
+  } catch { return []; }
+}
+
+function queryBacklinks(db, target) {
+  try {
+    return db.prepare(
+      `SELECT to_target AS target, COUNT(*) AS count
+       FROM memory_links
+       WHERE to_target IS NOT NULL AND to_target = ?
+       GROUP BY to_target`
+    ).all(target);
+  } catch { return []; }
+}
+
+function querySources(db, subject) {
+  // Top-5 memory entries whose body mentions the subject.
+  try {
+    const rows = db.prepare(
+      `SELECT path, kind, COUNT(*) AS mentions
+       FROM memory_entries
+       WHERE body LIKE ?
+       GROUP BY path, kind
+       ORDER BY mentions DESC
+       LIMIT 5`
+    ).all(`%${subject}%`);
+    return rows.map((r) => ({ path: r.path, kind: r.kind, mentions: r.mentions }));
+  } catch { return []; }
+}
+
+export function compileWikiPage(db, { repoRoot, type, subject } = {}) {
+  if (!subject) return { ok: false, error: 'missing-subject' };
+  const paths = resolveBrainPaths(repoRoot);
+  const slug = slugify(subject);
+  const pageDir = join(paths.wikiDir, pluralType(type));
+  const pagePath = join(pageDir, `${slug}.md`);
+
+  // F-LENS2-05: enforce containment + reserved-name on the compile target.
+  // A symlinked wiki dir or a maliciously-crafted subject (slugify removes
+  // most danger, but defense-in-depth) could otherwise direct the atomic
+  // rename outside the repo. mkdirSync(pageDir,…) below runs only if guard
+  // passes so we never even create a parent directory outside repoRoot.
+  const guard = validateSafeRepoPath(repoRoot, pagePath);
+  if (!guard.ok) return guard;
+
+  // F8: per-page advisory lock prevents two concurrent compiles for the
+  // same subject from interleaving (both read existing → both render →
+  // both rename → operator NOTES from the intermediate state could be
+  // lost). EEXIST-based exclusive open is portable + atomic.
+  mkdirSync(pageDir, { recursive: true });
+  const lockPath = pagePath + '.lock';
+  let lockFd;
+  try {
+    lockFd = openSync(lockPath, 'wx');
+  } catch (e) {
+    if (e.code === 'EEXIST') {
+      // FLAG-8: stale-lock recovery. If a prior compile process was SIGKILL'd
+      // between acquire and finally, the lockfile orphans and every subsequent
+      // compile fails. Check the lockfile's age; if older than STALE_LOCK_MS,
+      // assume it's a dead-process orphan and reclaim.
+      let stale = false;
+      try {
+        const age = Date.now() - statSync(lockPath).mtimeMs;
+        if (age > STALE_LOCK_MS) stale = true;
+      } catch { /* lockfile vanished while we checked — race won by us */ stale = true; }
+      if (stale) {
+        try { unlinkSync(lockPath); } catch {}
+        try { lockFd = openSync(lockPath, 'wx'); }
+        catch (e2) { return { ok: false, error: 'page-locked-by-concurrent-compile', pagePath, staleReclaimFailed: e2.code }; }
+      } else {
+        return { ok: false, error: 'page-locked-by-concurrent-compile', pagePath };
+      }
+    } else {
+      throw e;
+    }
+  }
+
+  try {
+    // Read existing AFTER acquiring the lock so we see the freshest committed
+    // state (no race with another compile that just landed its rename).
+    const existing = existsSync(pagePath) ? readFileSync(pagePath, 'utf8') : '';
+    const facts = queryFacts(db, subject);
+    const history = getHistoryWindow(db, subject, null, { limit: 50 });
+    const backlinks = queryBacklinks(db, slug);
+    const sources = querySources(db, subject);
+
+    const candidate = applyTemplate(type, existing, { subject, facts, history, backlinks, sources });
+
+    const verdict = resolveCitations(db, candidate);
+    if (!verdict.ok) {
+      return { ok: false, error: 'unresolved-citations', unresolved: verdict.unresolved, pagePath };
+    }
+
+    // Atomic write
+    const tmp = pagePath + '.tmp';
+    writeFileSync(tmp, candidate);
+    renameSync(tmp, pagePath);
+
+    return { ok: true, pagePath, factsCount: facts.length, historyRows: history.rows.length };
+  } finally {
+    try { closeSync(lockFd); } catch {}
+    try { unlinkSync(lockPath); } catch {}
+  }
+}

package/src/brain/wiki-sentinels.js

@@ -0,0 +1,45 @@
+// IJFW v1.5.2 -- brain wiki section sentinels (Trident F-F2).
+//
+// Wiki pages carry LLM-managed regions delimited by HTML comments:
+//   <!-- ijfw:auto:begin section="current-state" -->
+//   ...AUTO content (replaced by the compiler)...
+//   <!-- ijfw:auto:end section="current-state" -->
+//
+// Everything OUTSIDE these blocks is operator-owned ("NOTES") and the
+// compiler MUST preserve it verbatim. extractSections() returns the AUTO
+// regions; replaceSection() swaps a named region's body while preserving
+// the NOTES content and all other AUTO regions unchanged.
+
+const SECTION_RE = /<!--\s*ijfw:auto:begin\s+section="([^"]+)"\s*-->([\s\S]*?)<!--\s*ijfw:auto:end\s+section="\1"\s*-->/g;
+
+export function extractSections(md) {
+  if (typeof md !== 'string') return [];
+  const out = [];
+  for (const m of md.matchAll(SECTION_RE)) {
+    out.push({ name: m[1], body: m[2] });
+  }
+  return out;
+}
+
+function escapeRe(s) {
+  return s.replace(/[-/\\^$*+?.()|[\]{}]/g, '\\$&');
+}
+
+function makeBlock(name, body) {
+  return `<!-- ijfw:auto:begin section="${name}" -->\n${body}\n<!-- ijfw:auto:end section="${name}" -->`;
+}
+
+export function replaceSection(md, name, newBody) {
+  if (typeof md !== 'string') md = '';
+  const escaped = escapeRe(name);
+  const targetRe = new RegExp(
+    `<!--\\s*ijfw:auto:begin\\s+section="${escaped}"\\s*-->[\\s\\S]*?<!--\\s*ijfw:auto:end\\s+section="${escaped}"\\s*-->`
+  );
+  const replacement = makeBlock(name, newBody);
+  if (targetRe.test(md)) {
+    return md.replace(targetRe, replacement);
+  }
+  // Section absent -- append it. If md ends without trailing newline, add one.
+  const sep = md.length === 0 || md.endsWith('\n') ? '' : '\n';
+  return md + sep + '\n' + replacement + '\n';
+}

package/src/brain/wiki-templates.js

@@ -0,0 +1,94 @@
+// IJFW v1.5.2 -- brain wiki page templates.
+//
+// applyTemplate(type, existingMd, { subject, facts, history, backlinks, sources })
+// returns markdown with 4 AUTO sections (current-state, history, backlinks,
+// sources) wrapped in ijfw:auto sentinels. Operator-owned NOTES regions OUTSIDE
+// the sentinels are preserved verbatim by the underlying replaceSection().
+//
+// Per Trident F-B2, the history section is windowed (consumer passes
+// {rows, older} from getHistoryWindow); when older is present we append a
+// rollup line ("Older: 55 events between A and B") instead of pasting more rows.
+
+import { replaceSection } from './wiki-sentinels.js';
+
+function isoDate(s) {
+  if (!s) return '';
+  try {
+    const d = new Date(s);
+    if (Number.isNaN(d.getTime())) return s;
+    return d.toISOString().slice(0, 10);
+  } catch { return s; }
+}
+
+function renderCurrentState({ subject, facts }) {
+  if (!facts || facts.length === 0) {
+    return `_No facts about **${subject}** yet._`;
+  }
+  const lines = [];
+  for (const f of facts) {
+    const cite = f.id != null ? ` [fact:${f.id}]` : '';
+    const memCite = f.memory_id != null ? ` [mem:${f.memory_id}]` : '';
+    lines.push(`- ${f.predicate || '(no predicate)'}: **${f.object ?? ''}**${cite}${memCite}`);
+  }
+  return lines.join('\n');
+}
+
+function renderHistory({ history }) {
+  if (!history) return '_No history available._';
+  const { rows = [], older = null } = history;
+  if (rows.length === 0) return '_No history available._';
+  const lines = [];
+  for (const r of rows) {
+    const date = isoDate(r.valid_from);
+    const closed = r.valid_to ? ` (closed ${isoDate(r.valid_to)})` : '';
+    const memCite = r.memory_id != null ? ` [mem:${r.memory_id}]` : '';
+    const factCite = r.id != null ? ` [fact:${r.id}]` : '';
+    lines.push(`- ${date}: ${r.predicate || '(no predicate)'} = **${r.object ?? ''}**${closed}${factCite}${memCite}`);
+  }
+  if (older) {
+    lines.push('');
+    lines.push(`_Older: ${older.count} event${older.count === 1 ? '' : 's'} between ${isoDate(older.fromIso)} and ${isoDate(older.toIso)}._`);
+  }
+  return lines.join('\n');
+}
+
+function renderBacklinks({ backlinks }) {
+  if (!backlinks || backlinks.length === 0) return '_No backlinks._';
+  const lines = [];
+  for (const b of backlinks) {
+    const target = b.target || b.to_target || '';
+    const n = b.count ?? b.incomingLinks ?? 1;
+    if (!target) continue;
+    lines.push(`- [[${target}]] (${n} link${n === 1 ? '' : 's'})`);
+  }
+  return lines.length ? lines.join('\n') : '_No backlinks._';
+}
+
+function renderSources({ sources }) {
+  if (!sources || sources.length === 0) return '_No sources._';
+  const top = sources.slice(0, 5);
+  const lines = [];
+  for (const s of top) {
+    const path = s.path || '(no path)';
+    const kind = s.kind ? ` (${s.kind})` : '';
+    const mentions = s.mentions != null ? ` — ${s.mentions} mention${s.mentions === 1 ? '' : 's'}` : '';
+    lines.push(`- \`${path}\`${kind}${mentions}`);
+  }
+  return lines.join('\n');
+}
+
+export function applyTemplate(type, existingMd, data) {
+  let md = existingMd || '';
+  md = replaceSection(md, 'current-state', renderCurrentState(data));
+  md = replaceSection(md, 'history', renderHistory(data));
+  md = replaceSection(md, 'backlinks', renderBacklinks(data));
+  md = replaceSection(md, 'sources', renderSources(data));
+  return md;
+}
+
+export {
+  renderCurrentState as _renderCurrentState,
+  renderHistory as _renderHistory,
+  renderBacklinks as _renderBacklinks,
+  renderSources as _renderSources,
+};