metame-cli 1.6.0 → 1.6.2

package/scripts/memory-search.js

@@ -3,14 +3,15 @@
  * memory-search.js — Cross-session memory recall CLI
  *
  * Usage:
- *   node memory-search.js "<query>"                # hybrid search (QMD + FTS5)
+ *   node memory-search.js "<query>"                # hybrid search (FTS5 + vector + RRF)
  *   node memory-search.js "<q1>" "<q2>" "<q3>"     # multi-keyword parallel search
  *   node memory-search.js --facts "<query>"         # search facts only
  *   node memory-search.js --sessions "<query>"      # search sessions only
+ *   node memory-search.js --fts-only "<query>"      # force pure FTS5 (no vector)
  *   node memory-search.js --recent                  # show recent sessions
  *
  * Multi-keyword: results are deduplicated by fact ID, best rank wins.
- * Async: uses QMD hybrid search (BM25 + vector) when available, falls back to FTS5.
+ * Hybrid: uses FTS5 + vector embeddings + RRF fusion when available, falls back to FTS5.
  */
 
 'use strict';
@@ -31,8 +32,20 @@ if (!memoryPath) {
 const memory = require(memoryPath);
 
 const args = process.argv.slice(2);
-const mode = args[0] && args[0].startsWith('--') ? args[0] : null;
-const queries = mode ? args.slice(1) : args;
+// Parse flags: allow multiple -- flags before queries
+const flags = new Set();
+let firstQueryIdx = 0;
+for (let i = 0; i < args.length; i++) {
+  if (args[i].startsWith('--')) { flags.add(args[i]); firstQueryIdx = i + 1; }
+  else break;
+}
+const mode = flags.has('--facts') ? '--facts'
+  : flags.has('--sessions') ? '--sessions'
+  : flags.has('--recent') ? '--recent'
+  : flags.has('--fts-only') ? '--fts-only'
+  : null;
+const ftsOnly = flags.has('--fts-only');
+const queries = args.slice(firstQueryIdx);
 
 async function main() {
   try {
@@ -79,20 +92,35 @@ async function main() {
       limit: 3,
     });
 
-    // Wiki pages (if available)
+    // Wiki pages — hybrid search (FTS5 + vector + RRF) when available
     let wikiResults = [];
-    if (typeof memory.searchWikiAndFacts === 'function') {
-      try {
-        const allWiki = [];
-        for (const q of queries) {
-          const { wikiPages } = memory.searchWikiAndFacts(q, { trackSearch: true });
-          for (const p of (wikiPages || [])) {
-            allWiki.push({ type: 'wiki', slug: p.slug, title: p.title, excerpt: p.excerpt, last_built_at: p.last_built_at });
+    const useHybrid = typeof memory.hybridSearchWiki === 'function';
+    try {
+      const allWiki = [];
+      const seen = new Set();
+      for (const q of queries) {
+        const { wikiPages } = useHybrid
+          ? await memory.hybridSearchWiki(q, { ftsOnly, trackSearch: true })
+          : (typeof memory.searchWikiAndFacts === 'function'
+            ? memory.searchWikiAndFacts(q, { trackSearch: true })
+            : { wikiPages: [] });
+        for (const p of (wikiPages || [])) {
+          if (!seen.has(p.slug)) {
+            seen.add(p.slug);
+            allWiki.push({
+              type: 'wiki',
+              slug: p.slug,
+              title: p.title,
+              excerpt: p.excerpt,
+              score: p.score,
+              stale: p.stale,
+              source: p.source,
+            });
           }
         }
-        wikiResults = allWiki.slice(0, 5);
-      } catch { /* wiki not available */ }
-    }
+      }
+      wikiResults = allWiki.slice(0, 5);
+    } catch { /* wiki not available */ }
 
     console.log(JSON.stringify([...wikiResults, ...factResults, ...sessionResults], null, 2));
 

package/scripts/memory-wiki-schema.js

@@ -11,6 +11,8 @@
  *   wiki_pages       — topic knowledge pages
  *   wiki_topics      — controlled topic registry
  *   wiki_pages_fts   — FTS5 virtual table (content table, trigram tokenizer)
+ *   content_chunks   — chunked page content with optional vector embeddings
+ *   embedding_queue  — durable async queue for embedding generation
  *
  * Triggers:
  *   wiki_pages_fts_insert / wiki_pages_fts_update / wiki_pages_fts_delete
@@ -42,6 +44,9 @@ function applyWikiSchema(db) {
     )
   `);
 
+  // Migration: add timeline column for Compiled Truth + Timeline model (existing DBs)
+  try { db.exec("ALTER TABLE wiki_pages ADD COLUMN timeline TEXT DEFAULT ''"); } catch { /* column already exists */ }
+
   // ── wiki_topics ─────────────────────────────────────────────────────────────
   db.exec(`
     CREATE TABLE IF NOT EXISTS wiki_topics (
@@ -74,9 +79,14 @@ function applyWikiSchema(db) {
     END
   `);
 
+  // DROP+CREATE to upgrade existing unguarded trigger on deployed DBs
+  db.exec('DROP TRIGGER IF EXISTS wiki_pages_fts_update');
   db.exec(`
-    CREATE TRIGGER IF NOT EXISTS wiki_pages_fts_update
-      AFTER UPDATE ON wiki_pages BEGIN
+    CREATE TRIGGER wiki_pages_fts_update
+      AFTER UPDATE ON wiki_pages
+      WHEN old.slug IS NOT new.slug OR old.title IS NOT new.title
+        OR old.content IS NOT new.content OR old.topic_tags IS NOT new.topic_tags
+    BEGIN
       INSERT INTO wiki_pages_fts(wiki_pages_fts, rowid, slug, title, content, topic_tags)
         VALUES ('delete', old.rowid, old.slug, old.title, old.content, old.topic_tags);
       INSERT INTO wiki_pages_fts(rowid, slug, title, content, topic_tags)
@@ -91,6 +101,155 @@ function applyWikiSchema(db) {
         VALUES ('delete', old.rowid, old.slug, old.title, old.content, old.topic_tags);
     END
   `);
+
+  // ── content_chunks (vector embedding storage for wiki pages) ────────────────
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS content_chunks (
+      id              TEXT PRIMARY KEY,
+      page_slug       TEXT NOT NULL,
+      chunk_text      TEXT NOT NULL,
+      chunk_idx       INTEGER NOT NULL,
+      embedding       BLOB,
+      embedding_model TEXT,
+      embedding_dim   INTEGER,
+      created_at      TEXT DEFAULT (datetime('now'))
+    )
+  `);
+  try { db.exec('CREATE INDEX IF NOT EXISTS idx_chunks_slug ON content_chunks(page_slug)'); } catch { }
+
+  // ── embedding_queue (durable async queue for embedding generation) ──────────
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS embedding_queue (
+      id          INTEGER PRIMARY KEY AUTOINCREMENT,
+      item_type   TEXT NOT NULL,
+      item_id     TEXT NOT NULL,
+      model       TEXT DEFAULT 'text-embedding-3-small',
+      attempts    INTEGER DEFAULT 0,
+      last_error  TEXT,
+      created_at  TEXT DEFAULT (datetime('now'))
+    )
+  `);
+
+  // ── doc_sources ───────────────────────────────────────────────────────────
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS doc_sources (
+      id                  INTEGER PRIMARY KEY,
+      file_path           TEXT UNIQUE NOT NULL,
+      file_hash           TEXT NOT NULL,
+      mtime_ms            INTEGER,
+      size_bytes          INTEGER,
+      extracted_text_hash TEXT,
+      file_type           TEXT NOT NULL CHECK (file_type IN ('md','txt','pdf')),
+      extractor           TEXT,
+      extract_status      TEXT DEFAULT 'pending'
+                          CHECK (extract_status IN ('ok','empty_or_scanned','error','pending')),
+      title               TEXT,
+      slug                TEXT UNIQUE NOT NULL,
+      status              TEXT DEFAULT 'active'
+                          CHECK (status IN ('active','orphaned','missing')),
+      error_message       TEXT,
+      indexed_at          TEXT NOT NULL,
+      last_seen_at        TEXT,
+      built_at            TEXT,
+      content_stale       INTEGER DEFAULT 1
+    )
+  `);
+  db.exec(`CREATE INDEX IF NOT EXISTS idx_doc_sources_status        ON doc_sources(status)`);
+  db.exec(`CREATE INDEX IF NOT EXISTS idx_doc_sources_file_hash     ON doc_sources(file_hash)`);
+  db.exec(`CREATE INDEX IF NOT EXISTS idx_doc_sources_slug          ON doc_sources(slug)`);
+  db.exec(`CREATE INDEX IF NOT EXISTS idx_doc_sources_content_stale ON doc_sources(content_stale)`);
+
+  // ── wiki_page_doc_sources ─────────────────────────────────────────────────
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS wiki_page_doc_sources (
+      page_slug     TEXT NOT NULL,
+      doc_source_id INTEGER NOT NULL,
+      role          TEXT NOT NULL CHECK (role IN ('primary','cluster_member')),
+      PRIMARY KEY (page_slug, doc_source_id, role),
+      FOREIGN KEY (page_slug)     REFERENCES wiki_pages(slug)  ON DELETE CASCADE,
+      FOREIGN KEY (doc_source_id) REFERENCES doc_sources(id)   ON DELETE CASCADE
+    )
+  `);
+
+  // ── wiki_pages additions (idempotent ALTER) ───────────────────────────────
+  for (const [col, def] of [
+    ['source_type',    "TEXT DEFAULT 'memory'"],
+    ['membership_hash','TEXT'],
+    ['cluster_size',   'INTEGER'],
+  ]) {
+    try { db.exec(`ALTER TABLE wiki_pages ADD COLUMN ${col} ${def}`); } catch { /* already exists */ }
+  }
+  db.exec("UPDATE wiki_pages SET source_type = 'memory' WHERE source_type IS NULL");
+
+  // ── doc_sources additions (idempotent ALTER) ──────────────────────────────
+  for (const [col, def] of [
+    ['doi',            'TEXT'],
+    ['year',           'INTEGER'],
+    ['venue',          'TEXT'],
+    ['zotero_key',     'TEXT'],
+    ['citation_count', 'INTEGER'],
+  ]) {
+    try { db.exec(`ALTER TABLE doc_sources ADD COLUMN ${col} ${def}`); } catch { /* already exists */ }
+  }
+
+  // ── paper_facts ───────────────────────────────────────────────────────────
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS paper_facts (
+      id                TEXT PRIMARY KEY,
+      doc_source_id     INTEGER NOT NULL,
+      fact_type         TEXT NOT NULL CHECK (fact_type IN (
+                          'problem','method','claim','assumption',
+                          'dataset','metric','result','baseline',
+                          'limitation','future_work','contradiction_note'
+                        )),
+      subject           TEXT,
+      predicate         TEXT,
+      object            TEXT,
+      value             TEXT,
+      unit              TEXT,
+      context           TEXT,
+      evidence_text     TEXT NOT NULL,
+      section           TEXT,
+      extraction_source TEXT DEFAULT 'pdf_llm_section'
+                        CHECK (extraction_source IN (
+                          'pdf_llm_section',
+                          'zotero_deep_read',
+                          'manual'
+                        )),
+      confidence        REAL DEFAULT 0.7,
+      created_at        TEXT DEFAULT (datetime('now')),
+      FOREIGN KEY (doc_source_id) REFERENCES doc_sources(id) ON DELETE CASCADE
+    )
+  `);
+  db.exec('CREATE INDEX IF NOT EXISTS idx_paper_facts_doc     ON paper_facts(doc_source_id)');
+  db.exec('CREATE INDEX IF NOT EXISTS idx_paper_facts_type    ON paper_facts(fact_type)');
+  db.exec('CREATE INDEX IF NOT EXISTS idx_paper_facts_subject ON paper_facts(subject)');
+
+  // ── research_entities ─────────────────────────────────────────────────────
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS research_entities (
+      id          TEXT PRIMARY KEY,
+      entity_type TEXT NOT NULL CHECK (entity_type IN (
+                    'problem','concept','method_family','dataset','metric','application'
+                  )),
+      name        TEXT NOT NULL UNIQUE,
+      aliases     TEXT DEFAULT '[]',
+      description TEXT,
+      created_at  TEXT DEFAULT (datetime('now'))
+    )
+  `);
+
+  // ── fact_entity_links ─────────────────────────────────────────────────────
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS fact_entity_links (
+      fact_id   TEXT NOT NULL,
+      entity_id TEXT NOT NULL,
+      role      TEXT,
+      PRIMARY KEY (fact_id, entity_id),
+      FOREIGN KEY (fact_id)   REFERENCES paper_facts(id)       ON DELETE CASCADE,
+      FOREIGN KEY (entity_id) REFERENCES research_entities(id) ON DELETE CASCADE
+    )
+  `);
 }
 
 module.exports = { applyWikiSchema };

package/scripts/memory.js

@@ -47,6 +47,7 @@ function getDb() {
 
   _db.exec('PRAGMA journal_mode = WAL');
   _db.exec('PRAGMA busy_timeout = 3000');
+  _db.exec('PRAGMA foreign_keys = ON');
 
   _db.exec(`
     CREATE TABLE IF NOT EXISTS memory_items (
@@ -547,6 +548,19 @@ function searchWikiAndFacts(query, { trackSearch = true } = {}) {
   }
 }
 
+/**
+ * Hybrid wiki search (FTS5 + vector + RRF fusion).
+ * Falls back to pure FTS5 if hybrid-search module is unavailable.
+ */
+async function hybridSearchWiki(query, { ftsOnly = false, expand = false, trackSearch = true } = {}) {
+  try {
+    const { hybridSearchWiki: fn } = require('./core/hybrid-search');
+    return await fn(getDb(), query, { ftsOnly, trackSearch });
+  } catch {
+    return searchWikiAndFacts(query, { trackSearch });
+  }
+}
+
 module.exports = {
   // core
   saveMemoryItem,
@@ -558,6 +572,7 @@ module.exports = {
   assembleContext,
   // wiki
   searchWikiAndFacts,
+  hybridSearchWiki,
   // compatibility
   saveSession,
   saveFacts,

package/scripts/providers.js

@@ -159,23 +159,53 @@ function saveProviders(config) {
 // PROVIDER ENV BUILDER (Core mechanism)
 // ---------------------------------------------------------
 
+/**
+ * Read the env mapping defined in ~/.claude/settings.json.
+ * Returns a plain string→string object (only string values are kept).
+ * Returns {} on any error or if the file/env block is missing.
+ */
+function readClaudeSettingsEnv() {
+  const home = process.env.HOME || os.homedir();
+  const settingsPath = path.join(home, '.claude', 'settings.json');
+  try {
+    if (!fs.existsSync(settingsPath)) return {};
+    const data = JSON.parse(fs.readFileSync(settingsPath, 'utf8'));
+    if (!data || typeof data.env !== 'object' || data.env === null) return {};
+    const out = {};
+    for (const [k, v] of Object.entries(data.env)) {
+      if (typeof v === 'string') out[k] = v;
+    }
+    return out;
+  } catch {
+    return {};
+  }
+}
+
 /**
  * Build env var overrides for a named provider.
- * Returns {} for 'anthropic' (official) — use Claude Code defaults.
- * Returns { ANTHROPIC_BASE_URL, ANTHROPIC_API_KEY } for relays.
+ *
+ * Always inherits the env mapping from ~/.claude/settings.json (slot mappings
+ * like ANTHROPIC_DEFAULT_*_MODEL stay in place across providers).
+ * For 'anthropic' (official): returns the inherited Claude settings env unchanged.
+ * For custom providers: overrides ANTHROPIC_BASE_URL plus both
+ * ANTHROPIC_API_KEY and ANTHROPIC_AUTH_TOKEN with the provider's credentials.
  */
 function buildEnv(providerName) {
   const config = loadProviders();
   const name = providerName || config.active;
 
-  if (name === 'anthropic') return {};
+  const env = readClaudeSettingsEnv();
+
+  if (name === 'anthropic') return env;
 
   const provider = config.providers[name];
-  if (!provider) return {};
+  if (!provider) return env;
 
-  const env = {};
   if (provider.base_url) env.ANTHROPIC_BASE_URL = provider.base_url;
-  if (provider.api_key) env.ANTHROPIC_API_KEY = provider.api_key;
+  if (provider.api_key) {
+    env.ANTHROPIC_API_KEY = provider.api_key;
+    env.ANTHROPIC_AUTH_TOKEN = provider.api_key;
+  }
   return env;
 }
 
@@ -390,6 +420,7 @@ function getEngine() { return _currentEngine; }
 const api = {
   loadProviders,
   saveProviders,
+  readClaudeSettingsEnv,
   buildEnv,
   buildSpawnEnv,
   buildActiveEnv,

package/scripts/wiki-cluster.js

@@ -0,0 +1,121 @@
+'use strict';
+
+const crypto = require('node:crypto');
+
+function cosineSimilarity(a, b) {
+  if (a.length !== b.length) return 0;
+  let dot = 0, normA = 0, normB = 0;
+  for (let i = 0; i < a.length; i++) {
+    dot += a[i] * b[i];
+    normA += a[i] * a[i];
+    normB += b[i] * b[i];
+  }
+  const denom = Math.sqrt(normA) * Math.sqrt(normB);
+  return denom === 0 ? 0 : dot / denom;
+}
+
+function membershipHash(slugs) {
+  const sorted = [...slugs].sort().join(',');
+  return crypto.createHash('sha256').update(sorted).digest('hex');
+}
+
+function jaccardOverlap(setA, setB) {
+  const a = new Set(setA);
+  const b = new Set(setB);
+  let intersection = 0;
+  for (const x of a) if (b.has(x)) intersection++;
+  const union = a.size + b.size - intersection;
+  return union === 0 ? 0 : intersection / union;
+}
+
+/**
+ * Find existing cluster with Jaccard overlap > 0.5 with newMemberIds.
+ * Tie-break: prefer larger stored cluster.
+ */
+function findMatchingCluster(existingClusters, newMemberIds) {
+  let best = null;
+  let bestScore = 0.5; // strict threshold: must exceed 0.5
+  for (const cluster of existingClusters) {
+    const score = jaccardOverlap(cluster.memberIds, newMemberIds);
+    if (score > bestScore) {
+      best = cluster;
+      bestScore = score;
+    } else if (score === bestScore && best && cluster.memberIds.length > best.memberIds.length) {
+      best = cluster;
+    }
+  }
+  return best;
+}
+
+/**
+ * Build connected components from embeddings using cosine similarity threshold.
+ * @param {Array<{ slug: string, vector: Float32Array|number[] }>} embeddings
+ * @param {{ threshold?: number, minSize?: number }} options
+ * Uses union-find.
+ */
+function buildConnectedComponents(embeddings, { threshold = 0.75, minSize = 3 } = {}) {
+  const n = embeddings.length;
+  const slugs = embeddings.map(e => e.slug);
+  const parent = Object.fromEntries(slugs.map(s => [s, s]));
+
+  function find(x) {
+    if (parent[x] !== x) parent[x] = find(parent[x]);
+    return parent[x];
+  }
+  function union(x, y) { parent[find(x)] = find(y); }
+
+  for (let i = 0; i < n; i++) {
+    for (let j = i + 1; j < n; j++) {
+      if (cosineSimilarity(embeddings[i].vector, embeddings[j].vector) >= threshold) {
+        union(slugs[i], slugs[j]);
+      }
+    }
+  }
+
+  const groups = {};
+  for (const s of slugs) {
+    const root = find(s);
+    if (!groups[root]) groups[root] = [];
+    groups[root].push(s);
+  }
+
+  return Object.values(groups).filter(g => g.length >= minSize);
+}
+
+/**
+ * Fetch doc-level embeddings from content_chunks by averaging all chunk embeddings per page.
+ * @param {object} db - node:sqlite DatabaseSync instance
+ * @param {string[]} slugs
+ * @returns {Array<{ slug: string, vector: Float32Array }>} one entry per slug that has embeddings
+ */
+function getDocEmbeddings(db, slugs) {
+  if (slugs.length === 0) return [];
+  const placeholders = ',?'.repeat(slugs.length).slice(1);
+  const rows = db.prepare(
+    `SELECT page_slug, embedding FROM content_chunks WHERE page_slug IN (${placeholders}) AND embedding IS NOT NULL`
+  ).all(...slugs);
+
+  // Group rows by slug
+  const bySlug = {};
+  for (const row of rows) {
+    if (!bySlug[row.page_slug]) bySlug[row.page_slug] = [];
+    const buf = Buffer.isBuffer(row.embedding) ? row.embedding : Buffer.from(row.embedding);
+    if (buf.byteLength % 4 !== 0) continue; // skip corrupt/truncated embedding row
+    bySlug[row.page_slug].push(new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4));
+  }
+
+  // Average chunk vectors per slug
+  const result = [];
+  for (const [slug, vecs] of Object.entries(bySlug)) {
+    const dim = vecs[0].length;
+    const avg = new Float32Array(dim);
+    for (const v of vecs) {
+      for (let i = 0; i < dim; i++) avg[i] += v[i] / vecs.length;
+    }
+    result.push({ slug, vector: avg });
+  }
+  return result;
+}
+
+module.exports = { cosineSimilarity, buildConnectedComponents, jaccardOverlap,
+                   findMatchingCluster, membershipHash, getDocEmbeddings };

package/scripts/wiki-extract.js

@@ -0,0 +1,171 @@
+'use strict';
+
+const fs = require('node:fs');
+const path = require('node:path');
+const crypto = require('node:crypto');
+const { execFile } = require('node:child_process');
+const { promisify } = require('node:util');
+const execFileAsync = promisify(execFile);
+
+function slugFromFilename(filePath) {
+  const base = path.basename(filePath, path.extname(filePath));
+  return base
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, '-')
+    .replace(/^-+|-+$/g, '');
+}
+
+function extractMarkdownTitle(text) {
+  const m = text.match(/^#\s+(.+)$/m);
+  return m ? m[1].trim() : null;
+}
+
+async function extractText(filePath) {
+  const ext = path.extname(filePath).toLowerCase();
+
+  if (ext === '.md' || ext === '.txt') {
+    try {
+      const text = fs.readFileSync(filePath, 'utf8');
+      const title = ext === '.md' ? extractMarkdownTitle(text) : null;
+      return { text, title, extractor: 'direct', extractStatus: 'ok' };
+    } catch (err) {
+      return { text: '', title: null, extractor: 'direct', extractStatus: 'error', errorMessage: err.message };
+    }
+  }
+
+  if (ext === '.pdf') {
+    return extractPdf(filePath);
+  }
+
+  return {
+    text: '', title: null, extractor: 'unknown', extractStatus: 'error',
+    errorMessage: `Unsupported file type: ${ext}`,
+  };
+}
+
+async function extractPdf(filePath) {
+  const hasPdftotext = await checkCommand('pdftotext');
+
+  if (hasPdftotext) {
+    try {
+      const { stdout } = await execFileAsync('pdftotext', [filePath, '-'], { maxBuffer: 10 * 1024 * 1024 });
+      if (!stdout.trim()) {
+        return {
+          text: '', title: null, extractor: 'pdftotext', extractStatus: 'empty_or_scanned',
+          errorMessage: 'PDF produced no text — may be a scanned image. Install OCR for support.',
+        };
+      }
+      return { text: stdout, title: null, extractor: 'pdftotext', extractStatus: 'ok' };
+    } catch {
+      // fall through to pdf-parse
+    }
+  }
+
+  // Fallback: pdf-parse
+  try {
+    const pdfParse = require('pdf-parse');
+    const buf = fs.readFileSync(filePath);
+    const data = await pdfParse(buf);
+    if (!data.text.trim()) {
+      return {
+        text: '', title: null, extractor: 'pdf-parse', extractStatus: 'empty_or_scanned',
+        errorMessage: 'PDF produced no text — may be a scanned image.',
+      };
+    }
+    return { text: data.text, title: null, extractor: 'pdf-parse', extractStatus: 'ok' };
+  } catch (err) {
+    const hint = hasPdftotext ? '' : ' Install poppler for better PDF support: brew install poppler';
+    return {
+      text: '', title: null, extractor: 'pdf-parse', extractStatus: 'error',
+      errorMessage: err.message + hint,
+    };
+  }
+}
+
+/**
+ * Parse flat paper text into named sections.
+ *
+ * @param {string} text — raw extracted text (e.g. from pdftotext)
+ * @returns {{
+ *   abstract: string, introduction: string, method: string,
+ *   experiments: string, results: string, discussion: string,
+ *   conclusion: string, references: string, _fallback: boolean
+ * }}
+ * _fallback is true when fewer than 2 section headers were found;
+ * in that case the text is split into three equal chunks and returned
+ * under 'introduction', 'method', 'results' to guarantee non-empty input
+ * for downstream fact extraction.
+ */
+function extractSections(text) {
+  // Map from canonical key → regex patterns (case-insensitive, optional number prefix)
+  const PATTERNS = {
+    abstract:     /^(?:\d+[\.\s]+)?(?:abstract)\s*$/i,
+    introduction: /^(?:\d+[\.\s]+)?(?:introduction|background|overview)\s*$/i,
+    method:       /^(?:\d+[\.\s]+)?(?:method(?:s|ology)?|approach|proposed\s+method|framework|model|architecture)\s*$/i,
+    experiments:  /^(?:\d+[\.\s]+)?(?:experiments?|experimental\s+(?:setup|design)|evaluation|setup)\s*$/i,
+    results:      /^(?:\d+[\.\s]+)?(?:results?|findings|performance)\s*$/i,
+    discussion:   /^(?:\d+[\.\s]+)?(?:discussion|analysis|ablation)\s*$/i,
+    conclusion:   /^(?:\d+[\.\s]+)?(?:conclusions?|summary|future\s+work)\s*$/i,
+    references:   /^(?:\d+[\.\s]+)?(?:references|bibliography)\s*$/i,
+  };
+
+  const lines = text.split('\n');
+  const hits = []; // { key, lineIdx }
+
+  for (let i = 0; i < lines.length; i++) {
+    const trimmed = lines[i].trim();
+    if (!trimmed || trimmed.length > 80) continue; // section titles are short
+    for (const [key, re] of Object.entries(PATTERNS)) {
+      if (re.test(trimmed)) {
+        hits.push({ key, lineIdx: i });
+        break;
+      }
+    }
+  }
+
+  // Fallback: fewer than 2 distinct headers detected
+  if (hits.length < 2) {
+    const third = Math.floor(lines.length / 3);
+    return {
+      abstract: '',
+      introduction: lines.slice(0, third).join('\n'),
+      method: lines.slice(third, 2 * third).join('\n'),
+      experiments: '',
+      results: lines.slice(2 * third).join('\n'),
+      discussion: '',
+      conclusion: '',
+      references: '',
+      _fallback: true,
+    };
+  }
+
+  // Build sections from hits
+  const out = { abstract: '', introduction: '', method: '', experiments: '',
+                results: '', discussion: '', conclusion: '', references: '', _fallback: false };
+
+  for (let h = 0; h < hits.length; h++) {
+    const { key, lineIdx } = hits[h];
+    const endLine = h + 1 < hits.length ? hits[h + 1].lineIdx : lines.length;
+    // Deduplicate: keep the longest slice if same key appears twice
+    const chunk = lines.slice(lineIdx + 1, endLine).join('\n').trim();
+    if (chunk.length > (out[key] || '').length) out[key] = chunk;
+  }
+
+  return out;
+}
+
+async function checkCommand(cmd) {
+  // Use 'which' without shell:true to avoid shell injection
+  try {
+    await execFileAsync('which', [cmd]);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+function sha256(input) {
+  return crypto.createHash('sha256').update(input).digest('hex');
+}
+
+module.exports = { extractText, extractSections, slugFromFilename, sha256 };