metame-cli 1.6.0 → 1.6.1

package/scripts/wiki-reflect-export.js

@@ -226,6 +226,118 @@ function rebuildCapsulesIndex(capsuleFiles, outputDir = DEFAULT_WIKI_DIR) {
   fs.renameSync(tmpPath, filePath);
 }
 
+/**
+ * Mirror all .md files from srcDir into outputDir/subdir (atomic write).
+ * Pattern mirrors exportCapsuleFile.
+ *
+ * @param {string} srcDir   — e.g. ~/.metame/memory/decisions
+ * @param {string} subdir   — vault subdirectory name, e.g. 'decisions'
+ * @param {string} [outputDir]
+ * @returns {string[]}      — list of destination file paths written
+ */
+function exportReflectDir(srcDir, subdir, outputDir = DEFAULT_WIKI_DIR) {
+  if (!fs.existsSync(srcDir) || !fs.statSync(srcDir).isDirectory()) return [];
+  const destDir = path.join(outputDir, subdir);
+  _ensureDir(destDir);
+
+  const written = [];
+  for (const name of fs.readdirSync(srcDir)) {
+    if (!name.endsWith('.md')) continue;
+    const src = path.join(srcDir, name);
+    const dest = path.join(destDir, name);
+    const tmp = `${dest}.tmp`;
+    try {
+      const content = fs.readFileSync(src, 'utf8');
+      try { fs.unlinkSync(tmp); } catch { /* not present */ }
+      fs.writeFileSync(tmp, content.endsWith('\n') ? content : `${content}\n`, 'utf8');
+      fs.renameSync(tmp, dest);
+      written.push(dest);
+    } catch { /* skip unreadable file */ }
+  }
+  return written;
+}
+
+/**
+ * Write _index.md for a reflect subdirectory (decisions or lessons).
+ *
+ * @param {string[]} fileNames  — bare filenames (not full paths)
+ * @param {string}   subdir     — 'decisions' | 'lessons'
+ * @param {string}   [outputDir]
+ */
+function rebuildReflectDirIndex(fileNames, subdir, outputDir = DEFAULT_WIKI_DIR) {
+  const destDir = path.join(outputDir, subdir);
+  _ensureDir(destDir);
+
+  const label = subdir === 'decisions' ? 'Architecture Decisions' : 'Operational Lessons';
+  const lines = [
+    '---',
+    `title: ${label}`,
+    `updated: ${new Date().toISOString().slice(0, 10)}`,
+    'type: reflect-index',
+    '---',
+    '',
+    `# ${label}`,
+    '',
+    `> ${fileNames.length} entries · 自动生成，勿手动编辑`,
+    '',
+  ];
+
+  for (const name of [...fileNames].sort().reverse()) {
+    const base = path.basename(name, '.md');
+    lines.push(`- [[${subdir}/${base}|${base}]]`);
+  }
+
+  const filePath = path.join(destDir, '_index.md');
+  const tmpPath = `${filePath}.tmp`;
+  try { fs.unlinkSync(tmpPath); } catch { /* not present */ }
+  fs.writeFileSync(tmpPath, lines.join('\n') + '\n', 'utf8');
+  fs.renameSync(tmpPath, filePath);
+}
+
+/**
+ * Export all doc/cluster wiki pages from DB to Obsidian vault.
+ * Called by runWikiReflect after the memory-topic loop.
+ * Pages with empty content are skipped.
+ *
+ * @param {object} db — DatabaseSync instance
+ * @param {string} [outputDir]
+ * @returns {{ exported: string[], skipped: string[] }}
+ */
+function exportDocPages(db, outputDir = DEFAULT_WIKI_DIR) {
+  _ensureDir(outputDir);
+  const rows = db.prepare(
+    `SELECT slug, title, primary_topic, source_type, content,
+            topic_tags, created_at, last_built_at, raw_source_count, staleness
+     FROM wiki_pages
+     WHERE source_type IN ('doc', 'topic_cluster')
+       AND content IS NOT NULL AND content != ''`
+  ).all();
+
+  const exported = [];
+  const skipped = [];
+
+  for (const row of rows) {
+    try {
+      const tags = _safeJsonArray(row.topic_tags);
+      const frontmatter = {
+        title: row.title || row.slug,
+        slug: row.slug,
+        tags,
+        created: (row.created_at || '').slice(0, 10),
+        last_built: (row.last_built_at || '').slice(0, 10),
+        raw_sources: row.raw_source_count || 0,
+        staleness: row.staleness || 0,
+      };
+      exportWikiPage(row.slug, frontmatter, row.content, outputDir);
+      exported.push(row.slug);
+    } catch {
+      skipped.push(row.slug);
+    }
+  }
+
+  return { exported, skipped };
+}
+
 // ── helpers ───────────────────────────────────────────────────────────────────
 
 function _ensureDir(dir) {
@@ -330,4 +442,7 @@ module.exports = {
   rebuildSessionsIndex,
   exportCapsuleFile,
   rebuildCapsulesIndex,
+  exportReflectDir,
+  rebuildReflectDirIndex,
+  exportDocPages,          // new
 };

package/scripts/wiki-reflect.js

@@ -29,11 +29,16 @@ const {
   rebuildSessionsIndex,
   exportCapsuleFile,
   rebuildCapsulesIndex,
+  exportReflectDir,
+  rebuildReflectDirIndex,
+  exportDocPages,
 } = require('./wiki-reflect-export');
 
 const DEFAULT_WIKI_DIR = path.join(os.homedir(), '.metame', 'wiki');
 const DEFAULT_CAPSULES_DIR = path.join(os.homedir(), '.metame', 'memory', 'capsules');
 const DEFAULT_LOG_PATH = path.join(os.homedir(), '.metame', 'wiki_reflect_log.jsonl');
+const DEFAULT_DECISIONS_DIR = path.join(os.homedir(), '.metame', 'memory', 'decisions');
+const DEFAULT_LESSONS_DIR   = path.join(os.homedir(), '.metame', 'memory', 'lessons');
 const LOCK_FILE = path.join(os.homedir(), '.metame', 'wiki-reflect.lock');
 const LOCK_MAX_AGE_MS = 10 * 60 * 1000; // 10 minutes
 const STALENESS_THRESHOLD = 0.4;
@@ -55,6 +60,8 @@ const MAX_RETRIES = 3;
 async function runWikiReflect(db, {
   outputDir = DEFAULT_WIKI_DIR,
   capsulesDir = DEFAULT_CAPSULES_DIR,
+  decisionsDir = DEFAULT_DECISIONS_DIR,
+  lessonsDir   = DEFAULT_LESSONS_DIR,
   logPath = DEFAULT_LOG_PATH,
   providers,
   threshold = STALENESS_THRESHOLD,
@@ -70,6 +77,8 @@ async function runWikiReflect(db, {
   const failed = [];
   const exportFailed = [];
   const strippedLinksMap = {};
+  let docsExported = 0;
+  let reflectExported = 0;
 
   try {
     // 2. Load previous failed_slugs for retry logic
@@ -179,6 +188,28 @@ async function runWikiReflect(db, {
     }
     try { rebuildCapsulesIndex(capsuleFiles, outputDir); } catch { /* non-fatal */ }
 
+    // Step 6: Export doc/cluster pages from DB
+    try {
+      const { exported } = exportDocPages(db, outputDir);
+      docsExported = exported.length;
+    } catch { /* non-fatal */ }
+
+    // Step 7: Mirror decisions and lessons to vault
+    try {
+      const decWritten = exportReflectDir(decisionsDir, 'decisions', outputDir);
+      const lesWritten = exportReflectDir(lessonsDir, 'lessons', outputDir);
+      reflectExported = decWritten.length + lesWritten.length;
+
+      const decFiles = fs.existsSync(decisionsDir) && fs.statSync(decisionsDir).isDirectory()
+        ? fs.readdirSync(decisionsDir).filter(f => f.endsWith('.md'))
+        : [];
+      const lesFiles = fs.existsSync(lessonsDir) && fs.statSync(lessonsDir).isDirectory()
+        ? fs.readdirSync(lessonsDir).filter(f => f.endsWith('.md'))
+        : [];
+      if (decFiles.length > 0) rebuildReflectDirIndex(decFiles, 'decisions', outputDir);
+      if (lesFiles.length > 0) rebuildReflectDirIndex(lesFiles, 'lessons', outputDir);
+    } catch { /* non-fatal */ }
+
   } finally {
     // 6. Release lock
     _releaseLock(LOCK_FILE);
@@ -190,6 +221,8 @@ async function runWikiReflect(db, {
       export_failed_slugs: exportFailed,
       failed_slugs: failed,
       stripped_links: strippedLinksMap,
+      docs_exported: docsExported,
+      reflect_exported: reflectExported,
       duration_ms: Date.now() - startMs,
     };
     try {
@@ -197,7 +230,7 @@ async function runWikiReflect(db, {
     } catch { /* non-fatal */ }
   }
 
-  return { built, failed, exportFailed };
+  return { built, failed, exportFailed, docsExported, reflectExported };
 }
 
 // ── Lock helpers ──────────────────────────────────────────────────────────────

package/scripts/wiki-synthesis.js

@@ -0,0 +1,224 @@
+'use strict';
+
+/**
+ * wiki-synthesis.js — Evidence synthesis engines for Tier 2 cluster pages
+ *
+ * Single responsibility: DB queries + pure computation to produce structured
+ * intermediate artifacts. Zero LLM calls. All functions are synchronous and
+ * take a DatabaseSync instance + array of doc_source ids.
+ *
+ * Exports:
+ *   buildComparisonMatrix(db, docSourceIds) → string (markdown table)
+ *   buildTimeline(db, docSourceIds)         → string (markdown list)
+ *   detectContradictions(db, docSourceIds)  → object[]
+ *   buildCoverageReport(db, docSourceIds)   → string (markdown list)
+ */
+
+const EXPECTED_TYPES = ['problem', 'method', 'result', 'dataset', 'limitation'];
+const MAX_COLS = 8;       // max papers in comparison table before truncation
+const MAX_ROWS = 20;      // max predicate groups in comparison table
+const TRUNCATE_TITLE = 28; // char limit for column headers
+
+// ── helpers ───────────────────────────────────────────────────────────────────
+
+function ph(ids) {
+  return ids.map(() => '?').join(',');
+}
+
+function shortTitle(t, len = TRUNCATE_TITLE) {
+  if (!t) return '?';
+  return t.length > len ? t.slice(0, len - 1) + '…' : t;
+}
+
+// ── buildComparisonMatrix ─────────────────────────────────────────────────────
+
+/**
+ * Build a markdown comparison table of results/metrics across papers.
+ * Groups by predicate; columns are papers (up to MAX_COLS).
+ *
+ * @param {object} db - DatabaseSync
+ * @param {number[]} docSourceIds
+ * @returns {string} markdown table, or empty string if no result facts
+ */
+function buildComparisonMatrix(db, docSourceIds) {
+  if (docSourceIds.length === 0) return '';
+
+  const rows = db.prepare(`
+    SELECT pf.predicate, pf.subject, pf.object, pf.value, pf.unit, pf.context,
+           ds.id as doc_id, ds.title
+    FROM paper_facts pf
+    JOIN doc_sources ds ON ds.id = pf.doc_source_id
+    WHERE pf.doc_source_id IN (${ph(docSourceIds)})
+      AND pf.fact_type IN ('result','metric','baseline')
+      AND pf.predicate IS NOT NULL
+    ORDER BY pf.predicate, ds.id
+  `).all(...docSourceIds);
+
+  if (rows.length === 0) return '';
+
+  // Collect ordered unique papers (cap at MAX_COLS)
+  const paperOrder = [];
+  const paperTitles = {};
+  for (const r of rows) {
+    if (!paperTitles[r.doc_id]) {
+      paperOrder.push(r.doc_id);
+      paperTitles[r.doc_id] = r.title;
+    }
+  }
+  const papers = paperOrder.slice(0, MAX_COLS);
+
+  // Group by predicate → { docId → cell text }
+  const groups = {};
+  for (const r of rows) {
+    if (!papers.includes(r.doc_id)) continue;
+    const key = r.predicate;
+    if (!groups[key]) groups[key] = {};
+    const parts = [r.subject, r.object].filter(Boolean);
+    if (r.value) parts.push(r.value + (r.unit ? ' ' + r.unit : ''));
+    if (r.context) parts.push(`*(${r.context})*`);
+    // Keep first occurrence per (predicate, docId)
+    if (!groups[key][r.doc_id]) groups[key][r.doc_id] = parts.join(' — ');
+  }
+
+  const predicates = Object.keys(groups).slice(0, MAX_ROWS);
+  if (predicates.length === 0) return '';
+
+  // Build table
+  const header = ['Metric / Result', ...papers.map(id => shortTitle(paperTitles[id]))];
+  const separator = header.map(() => '---');
+  const tableRows = predicates.map(pred => {
+    const cells = papers.map(id => groups[pred][id] || '—');
+    return [pred, ...cells];
+  });
+
+  const fmt = (row) => '| ' + row.join(' | ') + ' |';
+  return [fmt(header), fmt(separator), ...tableRows.map(fmt)].join('\n');
+}
+
+// ── buildTimeline ─────────────────────────────────────────────────────────────
+
+/**
+ * Build a chronological timeline of core method contributions per paper.
+ *
+ * @param {object} db
+ * @param {number[]} docSourceIds
+ * @returns {string} markdown list
+ */
+function buildTimeline(db, docSourceIds) {
+  if (docSourceIds.length === 0) return '';
+
+  // Get top method/claim fact per paper (by confidence desc)
+  const rows = db.prepare(`
+    SELECT ds.year, ds.title, ds.slug, ds.id as doc_id,
+           pf.subject, pf.predicate, pf.object, pf.evidence_text, pf.confidence
+    FROM doc_sources ds
+    LEFT JOIN paper_facts pf ON pf.doc_source_id = ds.id
+      AND pf.fact_type IN ('method','claim')
+    WHERE ds.id IN (${ph(docSourceIds)})
+    ORDER BY ds.year ASC NULLS LAST, ds.id ASC, pf.confidence DESC
+  `).all(...docSourceIds);
+
+  if (rows.length === 0) return '';
+
+  // Deduplicate: one entry per doc (keep first = highest confidence)
+  const seen = new Set();
+  const entries = [];
+  for (const r of rows) {
+    if (seen.has(r.doc_id)) continue;
+    seen.add(r.doc_id);
+    entries.push(r);
+  }
+
+  return entries.map(r => {
+    const year = r.year ? `**${r.year}**` : '**year unknown**';
+    const slug  = r.slug ? `[[${r.slug}]]` : shortTitle(r.title);
+    let claim = '';
+    if (r.subject && r.predicate && r.object) {
+      claim = ` — ${r.subject} ${r.predicate} ${r.object}`;
+    } else if (r.evidence_text) {
+      claim = ` — "${r.evidence_text.slice(0, 120)}"`;
+    }
+    return `- ${year} ${slug}${claim}`;
+  }).join('\n');
+}
+
+// ── detectContradictions ──────────────────────────────────────────────────────
+
+/**
+ * Detect fact pairs where same (subject, predicate) yields different objects
+ * across different papers.
+ *
+ * @param {object} db
+ * @param {number[]} docSourceIds
+ * @returns {{ slugA, titleA, factA, slugB, titleB, factB }[]}
+ */
+function detectContradictions(db, docSourceIds) {
+  if (docSourceIds.length < 2) return [];
+
+  const rows = db.prepare(`
+    SELECT
+      a.id as id_a, a.subject, a.predicate, a.object as object_a,
+      a.evidence_text as ev_a, a.confidence as conf_a,
+      b.id as id_b, b.object as object_b,
+      b.evidence_text as ev_b, b.confidence as conf_b,
+      ds_a.slug as slug_a, ds_a.title as title_a,
+      ds_b.slug as slug_b, ds_b.title as title_b
+    FROM paper_facts a
+    JOIN paper_facts b ON (
+      a.subject IS NOT NULL AND a.subject = b.subject AND
+      a.predicate IS NOT NULL AND a.predicate = b.predicate AND
+      a.object IS NOT NULL AND b.object IS NOT NULL AND
+      a.object != b.object AND
+      a.doc_source_id < b.doc_source_id
+    )
+    JOIN doc_sources ds_a ON ds_a.id = a.doc_source_id
+    JOIN doc_sources ds_b ON ds_b.id = b.doc_source_id
+    WHERE a.doc_source_id IN (${ph(docSourceIds)})
+      AND b.doc_source_id IN (${ph(docSourceIds)})
+      AND a.fact_type IN ('result','claim','metric')
+      AND b.fact_type IN ('result','claim','metric')
+    LIMIT 20
+  `).all(...docSourceIds, ...docSourceIds);
+
+  return rows.map(r => ({
+    slugA: r.slug_a, titleA: r.title_a,
+    factA: { subject: r.subject, predicate: r.predicate, object: r.object_a, evidence: r.ev_a },
+    slugB: r.slug_b, titleB: r.title_b,
+    factB: { subject: r.subject, predicate: r.predicate, object: r.object_b, evidence: r.ev_b },
+  }));
+}
+
+// ── buildCoverageReport ───────────────────────────────────────────────────────
+
+/**
+ * Report which fact types are covered per paper, highlighting gaps.
+ *
+ * @param {object} db
+ * @param {number[]} docSourceIds
+ * @returns {string} markdown list
+ */
+function buildCoverageReport(db, docSourceIds) {
+  if (docSourceIds.length === 0) return '';
+
+  const rows = db.prepare(`
+    SELECT ds.id, ds.title, ds.slug,
+           GROUP_CONCAT(DISTINCT pf.fact_type) as covered_types
+    FROM doc_sources ds
+    LEFT JOIN paper_facts pf ON pf.doc_source_id = ds.id
+    WHERE ds.id IN (${ph(docSourceIds)})
+    GROUP BY ds.id
+    ORDER BY ds.id
+  `).all(...docSourceIds);
+
+  const lines = rows.map(r => {
+    const covered = new Set((r.covered_types || '').split(',').filter(Boolean));
+    const missing = EXPECTED_TYPES.filter(t => !covered.has(t));
+    const covStr  = EXPECTED_TYPES.map(t => covered.has(t) ? `✓${t}` : `✗${t}`).join(' ');
+    const gapNote = missing.length ? ` — **gaps: ${missing.join(', ')}**` : ' — complete';
+    return `- [[${r.slug || '?'}]] ${covStr}${gapNote}`;
+  });
+
+  return lines.join('\n');
+}
+
+module.exports = { buildComparisonMatrix, buildTimeline, detectContradictions, buildCoverageReport };