metame-cli 1.5.26 → 1.6.1

package/scripts/wiki-reflect.js

@@ -0,0 +1,338 @@
+'use strict';
+
+/**
+ * wiki-reflect.js — Wiki page rebuild orchestrator
+ *
+ * Process flow:
+ *   1. Acquire process lock (O_EXCL file flag, 10-min staleness detection)
+ *   2. Read all wiki_topics from DB
+ *   3. Per topic: query → build → export (failure per page does not stop others)
+ *   4. Rebuild _index.md
+ *   5. Release lock
+ *   6. Append audit log entry to wiki_reflect_log.jsonl
+ *
+ * Exports:
+ *   runWikiReflect(db, { outputDir, capsulesDir, logPath, providers, staleness }) → { built, failed, exportFailed }
+ */
+
+const fs = require('fs');
+const path = require('path');
+const os = require('os');
+
+const { listWikiTopics, getWikiPageBySlug, listWikiPages, listRecentSessionSummaries } = require('./core/wiki-db');
+const { queryRawFacts } = require('./wiki-reflect-query');
+const { buildWikiPage } = require('./wiki-reflect-build');
+const {
+  exportWikiPage,
+  rebuildIndex,
+  exportSessionSummary,
+  rebuildSessionsIndex,
+  exportCapsuleFile,
+  rebuildCapsulesIndex,
+  exportReflectDir,
+  rebuildReflectDirIndex,
+  exportDocPages,
+} = require('./wiki-reflect-export');
+
+const DEFAULT_WIKI_DIR = path.join(os.homedir(), '.metame', 'wiki');
+const DEFAULT_CAPSULES_DIR = path.join(os.homedir(), '.metame', 'memory', 'capsules');
+const DEFAULT_LOG_PATH = path.join(os.homedir(), '.metame', 'wiki_reflect_log.jsonl');
+const DEFAULT_DECISIONS_DIR = path.join(os.homedir(), '.metame', 'memory', 'decisions');
+const DEFAULT_LESSONS_DIR   = path.join(os.homedir(), '.metame', 'memory', 'lessons');
+const LOCK_FILE = path.join(os.homedir(), '.metame', 'wiki-reflect.lock');
+const LOCK_MAX_AGE_MS = 10 * 60 * 1000; // 10 minutes
+const STALENESS_THRESHOLD = 0.4;
+const MAX_RETRIES = 3;
+
+/**
+ * Run wiki reflect pipeline.
+ *
+ * @param {object} db - DatabaseSync instance
+ * @param {{
+ *   outputDir?: string,
+ *   capsulesDir?: string,
+ *   logPath?: string,
+ *   providers: { callHaiku: Function, buildDistillEnv: Function },
+ *   threshold?: number,
+ * }} opts
+ * @returns {{ built: string[], failed: object[], exportFailed: string[] }}
+ */
+async function runWikiReflect(db, {
+  outputDir = DEFAULT_WIKI_DIR,
+  capsulesDir = DEFAULT_CAPSULES_DIR,
+  decisionsDir = DEFAULT_DECISIONS_DIR,
+  lessonsDir   = DEFAULT_LESSONS_DIR,
+  logPath = DEFAULT_LOG_PATH,
+  providers,
+  threshold = STALENESS_THRESHOLD,
+} = {}) {
+  const startMs = Date.now();
+
+  // 1. Acquire lock
+  if (!_acquireLock(LOCK_FILE)) {
+    throw new Error('wiki-reflect: another instance is running (lock file exists and is recent)');
+  }
+
+  const built = [];
+  const failed = [];
+  const exportFailed = [];
+  const strippedLinksMap = {};
+  let docsExported = 0;
+  let reflectExported = 0;
+
+  try {
+    // 2. Load previous failed_slugs for retry logic
+    const failedSlugsMap = _loadFailedSlugs(logPath);
+
+    // 3. Get all registered topics and their allowed slugs (for wikilink whitelist)
+    const topics = listWikiTopics(db);
+    const allowedSlugs = topics.map(t => t.slug);
+
+    // 4. Process each topic
+    for (const topic of topics) {
+      const slug = topic.slug;
+
+      // Determine if this page should be rebuilt
+      const existingPage = getWikiPageBySlug(db, slug);
+      const staleness = existingPage ? existingPage.staleness : 1.0;
+      const failedEntry = failedSlugsMap.get(slug);
+
+      const needsBuild = _shouldBuild(staleness, failedEntry, threshold);
+      if (!needsBuild) continue;
+
+      try {
+        // Query raw facts
+        const queryResult = queryRawFacts(db, topic.tag, { capsulesDir });
+
+        if (queryResult.totalCount === 0) {
+          // No facts for this topic yet — skip without marking as failed
+          continue;
+        }
+
+        // Build (LLM + DB write)
+        const buildResult = await buildWikiPage(db, topic, queryResult, {
+          allowedSlugs,
+          providers,
+        });
+
+        if (buildResult === null) {
+          // LLM failure
+          const retries = failedEntry ? failedEntry.retries + 1 : 1;
+          failed.push({
+            slug,
+            retries,
+            next_retry: retries >= MAX_RETRIES ? null : _nextRetryISO(retries),
+            permanent_error: retries >= MAX_RETRIES,
+          });
+          continue;
+        }
+
+        // Track stripped links for audit log
+        if (buildResult.strippedLinks.length > 0) {
+          strippedLinksMap[slug] = buildResult.strippedLinks;
+        }
+
+        // Export (file write)
+        const updatedPage = getWikiPageBySlug(db, slug);
+        const frontmatter = {
+          title: updatedPage.title,
+          slug,
+          tags: _parseTags(updatedPage.topic_tags),
+          created: (updatedPage.created_at || '').slice(0, 10),
+          last_built: (updatedPage.last_built_at || '').slice(0, 10),
+          raw_sources: updatedPage.raw_source_count,
+          staleness: updatedPage.staleness,
+        };
+
+        try {
+          exportWikiPage(slug, frontmatter, buildResult.content, outputDir);
+          built.push(slug);
+        } catch (exportErr) {
+          // DB write succeeded, file write failed — log separately.
+          // Do NOT push to built: callers must not assume the file exists.
+          exportFailed.push(slug);
+        }
+
+      } catch (err) {
+        // Unexpected error (DB failure from buildWikiPage throws)
+        const retries = failedEntry ? failedEntry.retries + 1 : 1;
+        failed.push({
+          slug,
+          retries,
+          next_retry: retries >= MAX_RETRIES ? null : _nextRetryISO(retries),
+          permanent_error: retries >= MAX_RETRIES,
+        });
+      }
+    }
+
+    // 5. Rebuild index — per-operation try/catch so one failure doesn't suppress the rest
+    let allPages = [];
+    let sessions = [];
+    try { allPages = listWikiPages(db, { limit: 1000, orderBy: 'title' }); } catch { /* non-fatal */ }
+    try { sessions = listRecentSessionSummaries(db, { limit: 200 }); } catch { /* non-fatal */ }
+    const capsuleFiles = _listCapsuleFiles(capsulesDir);
+
+    try {
+      rebuildIndex(allPages, outputDir, { sessionCount: sessions.length, capsuleCount: capsuleFiles.length });
+    } catch { /* non-fatal — _index.md not updated */ }
+
+    for (const entry of sessions) {
+      try { exportSessionSummary(entry, outputDir, { wikiPages: allPages, capsuleFiles }); }
+      catch { /* non-fatal — skip this session */ }
+    }
+    try { rebuildSessionsIndex(sessions, outputDir); } catch { /* non-fatal */ }
+
+    for (const capsuleFile of capsuleFiles) {
+      try { exportCapsuleFile(capsuleFile, outputDir); }
+      catch { /* non-fatal — skip this capsule */ }
+    }
+    try { rebuildCapsulesIndex(capsuleFiles, outputDir); } catch { /* non-fatal */ }
+
+    // Step 6: Export doc/cluster pages from DB
+    try {
+      const { exported } = exportDocPages(db, outputDir);
+      docsExported = exported.length;
+    } catch { /* non-fatal */ }
+
+    // Step 7: Mirror decisions and lessons to vault
+    try {
+      const decWritten = exportReflectDir(decisionsDir, 'decisions', outputDir);
+      const lesWritten = exportReflectDir(lessonsDir, 'lessons', outputDir);
+      reflectExported = decWritten.length + lesWritten.length;
+
+      const decFiles = fs.existsSync(decisionsDir) && fs.statSync(decisionsDir).isDirectory()
+        ? fs.readdirSync(decisionsDir).filter(f => f.endsWith('.md'))
+        : [];
+      const lesFiles = fs.existsSync(lessonsDir) && fs.statSync(lessonsDir).isDirectory()
+        ? fs.readdirSync(lessonsDir).filter(f => f.endsWith('.md'))
+        : [];
+      if (decFiles.length > 0) rebuildReflectDirIndex(decFiles, 'decisions', outputDir);
+      if (lesFiles.length > 0) rebuildReflectDirIndex(lesFiles, 'lessons', outputDir);
+    } catch { /* non-fatal */ }
+
+  } finally {
+    // 6. Release lock
+    _releaseLock(LOCK_FILE);
+
+    // 7. Write audit log
+    const entry = {
+      ts: new Date().toISOString(),
+      slugs_built: built,
+      export_failed_slugs: exportFailed,
+      failed_slugs: failed,
+      stripped_links: strippedLinksMap,
+      docs_exported: docsExported,
+      reflect_exported: reflectExported,
+      duration_ms: Date.now() - startMs,
+    };
+    try {
+      fs.appendFileSync(logPath, JSON.stringify(entry) + '\n', 'utf8');
+    } catch { /* non-fatal */ }
+  }
+
+  return { built, failed, exportFailed, docsExported, reflectExported };
+}
+
+// ── Lock helpers ──────────────────────────────────────────────────────────────
+
+function _acquireLock(lockFile) {
+  // Check if lock file exists and is recent
+  try {
+    const stat = fs.statSync(lockFile);
+    const age = Date.now() - stat.mtimeMs;
+    if (age < LOCK_MAX_AGE_MS) return false; // Lock is held
+    // Stale lock — remove it
+    fs.unlinkSync(lockFile);
+  } catch {
+    // Lock file doesn't exist — proceed
+  }
+
+  try {
+    // O_EXCL ensures atomic creation (no race condition)
+    fs.writeFileSync(lockFile, String(process.pid), { flag: 'wx' });
+    return true;
+  } catch {
+    return false; // Another process created the lock between our check and write
+  }
+}
+
+function _releaseLock(lockFile) {
+  try { fs.unlinkSync(lockFile); } catch { /* ignore */ }
+}
+
+function _listCapsuleFiles(capsulesDir) {
+  try {
+    if (!fs.existsSync(capsulesDir)) return [];
+    return fs.readdirSync(capsulesDir)
+      .filter(name => name.endsWith('.md'))
+      .map(name => path.join(capsulesDir, name));
+  } catch {
+    return [];
+  }
+}
+
+// ── failed_slugs helpers ──────────────────────────────────────────────────────
+
+/**
+ * Load the most recent failed_slugs from the audit log.
+ * @param {string} logPath
+ * @returns {Map<string, { retries: number, next_retry: string|null, permanent_error?: boolean }>}
+ */
+function _loadFailedSlugs(logPath) {
+  const map = new Map();
+  if (!fs.existsSync(logPath)) return map;
+
+  try {
+    const lines = fs.readFileSync(logPath, 'utf8').trim().split('\n').filter(Boolean);
+    if (lines.length === 0) return map;
+
+    // Use the most recent log entry
+    const last = JSON.parse(lines[lines.length - 1]);
+    for (const entry of (last.failed_slugs || [])) {
+      map.set(entry.slug, {
+        retries: entry.retries || 0,
+        next_retry: entry.next_retry || null,
+        permanent_error: entry.permanent_error || false,
+      });
+    }
+  } catch { /* corrupted log — start fresh */ }
+
+  return map;
+}
+
+/**
+ * Determine if a page should be rebuilt this round.
+ */
+function _shouldBuild(staleness, failedEntry, threshold) {
+  // Permanent error → skip
+  if (failedEntry && failedEntry.permanent_error) return false;
+
+  // Retry queue: retries < MAX_RETRIES AND next_retry has passed
+  if (failedEntry && failedEntry.retries < MAX_RETRIES && failedEntry.next_retry) {
+    if (Date.now() >= Date.parse(failedEntry.next_retry)) return true;
+    return false; // Not yet time to retry
+  }
+
+  // Normal staleness gate
+  return staleness >= threshold;
+}
+
+/**
+ * Calculate next retry time using exponential backoff (2^retries days).
+ */
+function _nextRetryISO(retries) {
+  const daysMs = Math.pow(2, retries) * 24 * 60 * 60 * 1000;
+  return new Date(Date.now() + daysMs).toISOString();
+}
+
+// ── Tag helpers ────────────────────────────────────────────────────────────────
+
+function _parseTags(raw) {
+  if (Array.isArray(raw)) return raw;
+  if (typeof raw === 'string') {
+    try { const p = JSON.parse(raw); return Array.isArray(p) ? p : []; } catch { return []; }
+  }
+  return [];
+}
+
+module.exports = { runWikiReflect };

package/scripts/wiki-synthesis.js

@@ -0,0 +1,224 @@
+'use strict';
+
+/**
+ * wiki-synthesis.js — Evidence synthesis engines for Tier 2 cluster pages
+ *
+ * Single responsibility: DB queries + pure computation to produce structured
+ * intermediate artifacts. Zero LLM calls. All functions are synchronous and
+ * take a DatabaseSync instance + array of doc_source ids.
+ *
+ * Exports:
+ *   buildComparisonMatrix(db, docSourceIds) → string (markdown table)
+ *   buildTimeline(db, docSourceIds)         → string (markdown list)
+ *   detectContradictions(db, docSourceIds)  → object[]
+ *   buildCoverageReport(db, docSourceIds)   → string (markdown list)
+ */
+
+const EXPECTED_TYPES = ['problem', 'method', 'result', 'dataset', 'limitation'];
+const MAX_COLS = 8;       // max papers in comparison table before truncation
+const MAX_ROWS = 20;      // max predicate groups in comparison table
+const TRUNCATE_TITLE = 28; // char limit for column headers
+
+// ── helpers ───────────────────────────────────────────────────────────────────
+
+function ph(ids) {
+  return ids.map(() => '?').join(',');
+}
+
+function shortTitle(t, len = TRUNCATE_TITLE) {
+  if (!t) return '?';
+  return t.length > len ? t.slice(0, len - 1) + '…' : t;
+}
+
+// ── buildComparisonMatrix ─────────────────────────────────────────────────────
+
+/**
+ * Build a markdown comparison table of results/metrics across papers.
+ * Groups by predicate; columns are papers (up to MAX_COLS).
+ *
+ * @param {object} db - DatabaseSync
+ * @param {number[]} docSourceIds
+ * @returns {string} markdown table, or empty string if no result facts
+ */
+function buildComparisonMatrix(db, docSourceIds) {
+  if (docSourceIds.length === 0) return '';
+
+  const rows = db.prepare(`
+    SELECT pf.predicate, pf.subject, pf.object, pf.value, pf.unit, pf.context,
+           ds.id as doc_id, ds.title
+    FROM paper_facts pf
+    JOIN doc_sources ds ON ds.id = pf.doc_source_id
+    WHERE pf.doc_source_id IN (${ph(docSourceIds)})
+      AND pf.fact_type IN ('result','metric','baseline')
+      AND pf.predicate IS NOT NULL
+    ORDER BY pf.predicate, ds.id
+  `).all(...docSourceIds);
+
+  if (rows.length === 0) return '';
+
+  // Collect ordered unique papers (cap at MAX_COLS)
+  const paperOrder = [];
+  const paperTitles = {};
+  for (const r of rows) {
+    if (!paperTitles[r.doc_id]) {
+      paperOrder.push(r.doc_id);
+      paperTitles[r.doc_id] = r.title;
+    }
+  }
+  const papers = paperOrder.slice(0, MAX_COLS);
+
+  // Group by predicate → { docId → cell text }
+  const groups = {};
+  for (const r of rows) {
+    if (!papers.includes(r.doc_id)) continue;
+    const key = r.predicate;
+    if (!groups[key]) groups[key] = {};
+    const parts = [r.subject, r.object].filter(Boolean);
+    if (r.value) parts.push(r.value + (r.unit ? ' ' + r.unit : ''));
+    if (r.context) parts.push(`*(${r.context})*`);
+    // Keep first occurrence per (predicate, docId)
+    if (!groups[key][r.doc_id]) groups[key][r.doc_id] = parts.join(' — ');
+  }
+
+  const predicates = Object.keys(groups).slice(0, MAX_ROWS);
+  if (predicates.length === 0) return '';
+
+  // Build table
+  const header = ['Metric / Result', ...papers.map(id => shortTitle(paperTitles[id]))];
+  const separator = header.map(() => '---');
+  const tableRows = predicates.map(pred => {
+    const cells = papers.map(id => groups[pred][id] || '—');
+    return [pred, ...cells];
+  });
+
+  const fmt = (row) => '| ' + row.join(' | ') + ' |';
+  return [fmt(header), fmt(separator), ...tableRows.map(fmt)].join('\n');
+}
+
+// ── buildTimeline ─────────────────────────────────────────────────────────────
+
+/**
+ * Build a chronological timeline of core method contributions per paper.
+ *
+ * @param {object} db
+ * @param {number[]} docSourceIds
+ * @returns {string} markdown list
+ */
+function buildTimeline(db, docSourceIds) {
+  if (docSourceIds.length === 0) return '';
+
+  // Get top method/claim fact per paper (by confidence desc)
+  const rows = db.prepare(`
+    SELECT ds.year, ds.title, ds.slug, ds.id as doc_id,
+           pf.subject, pf.predicate, pf.object, pf.evidence_text, pf.confidence
+    FROM doc_sources ds
+    LEFT JOIN paper_facts pf ON pf.doc_source_id = ds.id
+      AND pf.fact_type IN ('method','claim')
+    WHERE ds.id IN (${ph(docSourceIds)})
+    ORDER BY ds.year ASC NULLS LAST, ds.id ASC, pf.confidence DESC
+  `).all(...docSourceIds);
+
+  if (rows.length === 0) return '';
+
+  // Deduplicate: one entry per doc (keep first = highest confidence)
+  const seen = new Set();
+  const entries = [];
+  for (const r of rows) {
+    if (seen.has(r.doc_id)) continue;
+    seen.add(r.doc_id);
+    entries.push(r);
+  }
+
+  return entries.map(r => {
+    const year = r.year ? `**${r.year}**` : '**year unknown**';
+    const slug  = r.slug ? `[[${r.slug}]]` : shortTitle(r.title);
+    let claim = '';
+    if (r.subject && r.predicate && r.object) {
+      claim = ` — ${r.subject} ${r.predicate} ${r.object}`;
+    } else if (r.evidence_text) {
+      claim = ` — "${r.evidence_text.slice(0, 120)}"`;
+    }
+    return `- ${year} ${slug}${claim}`;
+  }).join('\n');
+}
+
+// ── detectContradictions ──────────────────────────────────────────────────────
+
+/**
+ * Detect fact pairs where same (subject, predicate) yields different objects
+ * across different papers.
+ *
+ * @param {object} db
+ * @param {number[]} docSourceIds
+ * @returns {{ slugA, titleA, factA, slugB, titleB, factB }[]}
+ */
+function detectContradictions(db, docSourceIds) {
+  if (docSourceIds.length < 2) return [];
+
+  const rows = db.prepare(`
+    SELECT
+      a.id as id_a, a.subject, a.predicate, a.object as object_a,
+      a.evidence_text as ev_a, a.confidence as conf_a,
+      b.id as id_b, b.object as object_b,
+      b.evidence_text as ev_b, b.confidence as conf_b,
+      ds_a.slug as slug_a, ds_a.title as title_a,
+      ds_b.slug as slug_b, ds_b.title as title_b
+    FROM paper_facts a
+    JOIN paper_facts b ON (
+      a.subject IS NOT NULL AND a.subject = b.subject AND
+      a.predicate IS NOT NULL AND a.predicate = b.predicate AND
+      a.object IS NOT NULL AND b.object IS NOT NULL AND
+      a.object != b.object AND
+      a.doc_source_id < b.doc_source_id
+    )
+    JOIN doc_sources ds_a ON ds_a.id = a.doc_source_id
+    JOIN doc_sources ds_b ON ds_b.id = b.doc_source_id
+    WHERE a.doc_source_id IN (${ph(docSourceIds)})
+      AND b.doc_source_id IN (${ph(docSourceIds)})
+      AND a.fact_type IN ('result','claim','metric')
+      AND b.fact_type IN ('result','claim','metric')
+    LIMIT 20
+  `).all(...docSourceIds, ...docSourceIds);
+
+  return rows.map(r => ({
+    slugA: r.slug_a, titleA: r.title_a,
+    factA: { subject: r.subject, predicate: r.predicate, object: r.object_a, evidence: r.ev_a },
+    slugB: r.slug_b, titleB: r.title_b,
+    factB: { subject: r.subject, predicate: r.predicate, object: r.object_b, evidence: r.ev_b },
+  }));
+}
+
+// ── buildCoverageReport ───────────────────────────────────────────────────────
+
+/**
+ * Report which fact types are covered per paper, highlighting gaps.
+ *
+ * @param {object} db
+ * @param {number[]} docSourceIds
+ * @returns {string} markdown list
+ */
+function buildCoverageReport(db, docSourceIds) {
+  if (docSourceIds.length === 0) return '';
+
+  const rows = db.prepare(`
+    SELECT ds.id, ds.title, ds.slug,
+           GROUP_CONCAT(DISTINCT pf.fact_type) as covered_types
+    FROM doc_sources ds
+    LEFT JOIN paper_facts pf ON pf.doc_source_id = ds.id
+    WHERE ds.id IN (${ph(docSourceIds)})
+    GROUP BY ds.id
+    ORDER BY ds.id
+  `).all(...docSourceIds);
+
+  const lines = rows.map(r => {
+    const covered = new Set((r.covered_types || '').split(',').filter(Boolean));
+    const missing = EXPECTED_TYPES.filter(t => !covered.has(t));
+    const covStr  = EXPECTED_TYPES.map(t => covered.has(t) ? `✓${t}` : `✗${t}`).join(' ');
+    const gapNote = missing.length ? ` — **gaps: ${missing.join(', ')}**` : ' — complete';
+    return `- [[${r.slug || '?'}]] ${covStr}${gapNote}`;
+  });
+
+  return lines.join('\n');
+}
+
+module.exports = { buildComparisonMatrix, buildTimeline, detectContradictions, buildCoverageReport };