botrun-mcli 0.2.2 → 0.4.0

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "botrun-mcli",
-  "version": "0.2.2",
-  "description": "Git-backed memory CLI for AI agents",
+  "version": "0.4.0",
+  "description": "Local SQLite full-text search CLI for plain-text memory",
   "type": "module",
   "bin": {
     "bm": "./src/bin.mjs"
@@ -16,7 +16,9 @@
     "ai",
     "agent",
     "memory",
-    "git",
+    "sqlite",
+    "fts5",
+    "search",
     "cli"
   ],
   "repository": {
@@ -25,6 +27,8 @@
   },
   "license": "MIT",
   "dependencies": {
-    "commander": "^13.0.0"
+    "better-sqlite3": "^12.9.0",
+    "commander": "^13.0.0",
+    "gray-matter": "^4.0.3"
   }
 }

package/src/bin.mjs

@@ -1,13 +1,15 @@
 #!/usr/bin/env node
 
 import { Command } from 'commander';
-import { getConfigPath } from './config.mjs';
-import { addScope } from './commands/config/add-scope.mjs';
-import { removeScope } from './commands/config/remove-scope.mjs';
+import { indexMemory } from './commands/memory/index.mjs';
+import { searchMemory } from './commands/memory/search.mjs';
+import { ingestFile } from './commands/memory/ingest.mjs';
+import { removeMemory } from './commands/memory/remove.mjs';
+import { resetMemory } from './commands/memory/reset.mjs';
+import { addSource } from './commands/config/add-source.mjs';
+import { removeSource } from './commands/config/remove-source.mjs';
 import { showConfig } from './commands/config/show.mjs';
-import { initMemory } from './commands/memory/init.mjs';
-import { syncMemory } from './commands/memory/sync.mjs';
-import { listScopes } from './commands/memory/scopes.mjs';
+import { getConfigPath } from './paths.mjs';
 
 function jsonHelp(cmd) {
   const commands = cmd.commands.map(c => ({
@@ -26,10 +28,9 @@ const program = new Command();
 
 program
   .name('bm')
-  .description('Git-backed memory management for agents')
-  .version('0.1.0')
-  .helpCommand(false)
-  .option('--bm-path <path>', 'Base directory for all bm data')
+  .description('Local SQLite full-text search over plain-text memory (multi-source)')
+  .version('0.4.0')
+  .option('--bm-path <path>', 'Base directory for bm state (config + db)')
   .configureHelp({ formatHelp: (cmd) => JSON.stringify(jsonHelp(cmd), null, 2) })
   .hook('preAction', (thisCommand) => {
     const bmPath = thisCommand.opts().bmPath;
@@ -37,63 +38,98 @@ program
   });
 
 // --- config ---
-const configCmd = program.command('config').description('Manage configuration');
+const configCmd = program.command('config').description('Manage source configuration');
 configCmd.configureHelp({ formatHelp: (cmd) => JSON.stringify(jsonHelp(cmd), null, 2) });
 
 configCmd
-  .command('add-scope <name>')
-  .description('Add a scope to the configuration')
-  .requiredOption('--repo <url>', 'Git repo URL')
-  .option('--branch <branch>', 'Git branch to use')
-  .option('--token-env <envVar>', 'Environment variable name for this scope token')
-  .option('--description <text>', 'Description of this scope for agent context')
-  .option('--access <mode>', 'Access mode hint for agent: readwrite or readonly', 'readwrite')
+  .command('add-source <name>')
+  .description('Register a new source (a raw + raw_text directory pair)')
+  .requiredOption('--raw <path>', 'Directory holding original files (PDF/DOCX/...)')
+  .requiredOption('--raw-text <path>', 'Directory holding plain-text .md files')
+  .option('--access <mode>', 'readwrite | readonly', 'readwrite')
   .action(async (name, opts) => {
-    const result = await addScope(name, opts, getConfigPath());
+    const result = await addSource(name, {
+      raw: opts.raw,
+      rawText: opts.rawText,
+      access: opts.access,
+    }, getConfigPath());
     console.log(JSON.stringify(result, null, 2));
   });
 
 configCmd
-  .command('remove-scope <name>')
-  .description('Remove a scope from the configuration')
+  .command('remove-source <name>')
+  .description('Remove a source from configuration (files untouched)')
   .action(async (name) => {
-    const result = await removeScope(name, getConfigPath());
+    const result = await removeSource(name, getConfigPath());
     console.log(JSON.stringify(result, null, 2));
   });
 
 configCmd
   .command('show')
-  .description('Show current configuration')
+  .description('Show current source configuration')
   .action(async () => {
     const result = await showConfig(getConfigPath());
     console.log(JSON.stringify(result, null, 2));
   });
 
 // --- memory ---
-const memoryCmd = program.command('memory').description('Manage memory repos');
+const memoryCmd = program.command('memory').description('Index, search, and manage memory');
 memoryCmd.configureHelp({ formatHelp: (cmd) => JSON.stringify(jsonHelp(cmd), null, 2) });
 
 memoryCmd
-  .command('init')
-  .description('Clone all scope repos')
-  .action(async () => {
-    const result = await initMemory({ configPath: getConfigPath() });
+  .command('index')
+  .description('Build or update SQLite FTS5 index across all configured sources')
+  .option('--force', 'Force full reindex (ignore mtime)')
+  .action(async (opts) => {
+    const result = await indexMemory({ force: opts.force });
     console.log(JSON.stringify(result, null, 2));
   });
 
 memoryCmd
-  .command('sync')
-  .description('Sync memory changes to remote repos')
-  .action(async () => {
-    const result = await syncMemory({ configPath: getConfigPath() });
+  .command('search')
+  .description('Search memory using BM25 full-text search across all sources')
+  .option('--query <text>', 'Search query (optional; omit for list mode)')
+  .option('--type <type>', 'Filter by source_type')
+  .option('--tags <tags>', 'Comma-separated tag filter')
+  .option('--after <date>', 'Filter created_at >= date')
+  .option('--before <date>', 'Filter created_at <= date')
+  .option('--limit <n>', 'Max results (default 10)')
+  .action(async (opts) => {
+    const result = await searchMemory({
+      query: opts.query,
+      type: opts.type,
+      tags: opts.tags,
+      after: opts.after,
+      before: opts.before,
+      limit: opts.limit,
+    });
+    console.log(JSON.stringify(result, null, 2));
+  });
+
+memoryCmd
+  .command('ingest')
+  .description('Index a single file (file must already live under a configured source)')
+  .requiredOption('--file <path>', 'Absolute (or relative) path to the file to index')
+  .action(async (opts) => {
+    const result = await ingestFile({ file: opts.file });
+    console.log(JSON.stringify(result, null, 2));
+  });
+
+memoryCmd
+  .command('remove')
+  .description('Delete files from readwrite sources and drop their rows (readonly blocked)')
+  .option('--file <path...>', 'Absolute file path(s) to remove')
+  .action(async (opts) => {
+    const files = opts.file || [];
+    const result = await removeMemory({ files });
     console.log(JSON.stringify(result, null, 2));
   });
 
 memoryCmd
-  .command('scopes')
-  .description('List available scopes and local paths')
+  .command('reset')
+  .description('Delete SQLite index (config.json and files unchanged)')
   .action(async () => {
-    const result = await listScopes({ configPath: getConfigPath() });
+    const result = await resetMemory();
     console.log(JSON.stringify(result, null, 2));
   });
 

package/src/commands/config/add-source.mjs

@@ -0,0 +1,38 @@
+import { resolve } from 'node:path';
+import { loadConfig, saveConfig } from '../../config/io.mjs';
+
+const INVALID_NAME_RE = /[\s/\uff1a]/;
+
+export async function addSource(name, options, configPath) {
+  if (!name || INVALID_NAME_RE.test(name)) {
+    return { error: 'invalid_name', message: 'Source name cannot contain whitespace, "/", or fullwidth colon' };
+  }
+  if (!options.raw) {
+    return { error: 'missing_raw', message: '--raw is required' };
+  }
+  if (!options.rawText) {
+    return { error: 'missing_raw_text', message: '--raw-text is required' };
+  }
+  const access = options.access || 'readwrite';
+  if (access !== 'readwrite' && access !== 'readonly') {
+    return { error: 'invalid_access', message: 'access must be "readwrite" or "readonly"' };
+  }
+
+  const config = await loadConfig(configPath);
+  if (config.sources[name]) {
+    return {
+      error: 'source_exists',
+      existing: config.sources[name],
+      message: `Source "${name}" already exists`,
+    };
+  }
+
+  const entry = {
+    raw: resolve(options.raw),
+    raw_text: resolve(options.rawText),
+    access,
+  };
+  config.sources[name] = entry;
+  await saveConfig(configPath, config);
+  return { added: name, source: entry };
+}

package/src/commands/config/remove-source.mjs

@@ -0,0 +1,11 @@
+import { loadConfig, saveConfig } from '../../config/io.mjs';
+
+export async function removeSource(name, configPath) {
+  const config = await loadConfig(configPath);
+  if (!config.sources[name]) {
+    return { error: 'source_not_found', message: `Source "${name}" not found` };
+  }
+  delete config.sources[name];
+  await saveConfig(configPath, config);
+  return { removed: name };
+}

package/src/commands/config/show.mjs

@@ -1,4 +1,4 @@
-import { loadConfig } from '../../config.mjs';
+import { loadConfig } from '../../config/io.mjs';
 
 export async function showConfig(configPath) {
   return await loadConfig(configPath);

package/src/commands/memory/index.mjs

@@ -0,0 +1,19 @@
+import { openDb } from '../../db/connection.mjs';
+import { indexAll } from '../../indexer/index-all.mjs';
+import { loadConfig } from '../../config/io.mjs';
+import { getBmPath, getConfigPath } from '../../paths.mjs';
+
+export async function indexMemory(options = {}) {
+  const bmPath = options.bmPath || getBmPath();
+  const configPath = options.configPath || getConfigPath();
+  const force = options.force === true;
+
+  const config = await loadConfig(configPath);
+  const db = openDb(bmPath);
+  try {
+    const stats = await indexAll(db, config, { force });
+    return { indexed: stats };
+  } finally {
+    db.close();
+  }
+}

package/src/commands/memory/ingest.mjs

@@ -0,0 +1,75 @@
+import { readFile, stat, access } from 'node:fs/promises';
+import { resolve } from 'node:path';
+import { openDb } from '../../db/connection.mjs';
+import { upsertDocument } from '../../db/queries.mjs';
+import { parseMarkdown } from '../../indexer/parse.mjs';
+import { loadConfig } from '../../config/io.mjs';
+import { findSourceByFilePath } from '../../config/lookup.mjs';
+import { getBmPath, getConfigPath } from '../../paths.mjs';
+
+async function exists(p) {
+  try { await access(p); return true; } catch { return false; }
+}
+
+function mtimeToDate(mtimeMs) {
+  return new Date(mtimeMs).toISOString().slice(0, 10);
+}
+
+export async function ingestFile(options = {}) {
+  const bmPath = options.bmPath || getBmPath();
+  const configPath = options.configPath || getConfigPath();
+
+  if (!options.file) {
+    return { error: 'missing_file', message: '--file is required' };
+  }
+
+  const absPath = resolve(options.file);
+
+  if (!(await exists(absPath))) {
+    return {
+      error: 'source_not_found',
+      file: absPath,
+      message: `File does not exist: ${absPath}`,
+    };
+  }
+
+  const config = await loadConfig(configPath);
+  const hit = findSourceByFilePath(config, absPath);
+  if (!hit) {
+    return {
+      error: 'not_under_source',
+      file: absPath,
+      message: `File is not under any configured source's raw_text. Run 'bm config show' to see sources.`,
+    };
+  }
+
+  const raw = await readFile(absPath, 'utf-8');
+  const s = await stat(absPath);
+  const mtime = Math.floor(s.mtimeMs);
+
+  const parsed = parseMarkdown(raw, absPath, { mtimeDate: mtimeToDate(mtime) });
+
+  const db = openDb(bmPath);
+  try {
+    upsertDocument(db, {
+      file_path: absPath,
+      title: parsed.title,
+      content: parsed.content,
+      tags: parsed.tags,
+      source_type: parsed.source_type,
+      source_url: parsed.source_url,
+      original: parsed.original,
+      synthesized_from: parsed.synthesized_from,
+      created_at: parsed.created_at,
+      mtime,
+      word_count: parsed.word_count,
+    });
+  } finally {
+    db.close();
+  }
+
+  return {
+    ingested: absPath,
+    indexed: true,
+  };
+}

package/src/commands/memory/remove.mjs

@@ -0,0 +1,52 @@
+import { rm, access } from 'node:fs/promises';
+import { resolve } from 'node:path';
+import { openDb } from '../../db/connection.mjs';
+import { deleteDocument } from '../../db/queries.mjs';
+import { loadConfig } from '../../config/io.mjs';
+import { findSourceByFilePath } from '../../config/lookup.mjs';
+import { getBmPath, getConfigPath } from '../../paths.mjs';
+
+async function exists(p) {
+  try { await access(p); return true; } catch { return false; }
+}
+
+export async function removeMemory(options = {}) {
+  const bmPath = options.bmPath || getBmPath();
+  const configPath = options.configPath || getConfigPath();
+  const files = options.files || [];
+
+  const config = await loadConfig(configPath);
+
+  const removed = [];
+  const not_found = [];
+  const not_under_source = [];
+  const readonly_blocked = [];
+
+  const db = openDb(bmPath);
+  try {
+    for (const file of files) {
+      const absPath = resolve(file);
+      const hit = findSourceByFilePath(config, absPath);
+      if (!hit) {
+        not_under_source.push(absPath);
+        continue;
+      }
+      if (hit.source.access === 'readonly') {
+        readonly_blocked.push(absPath);
+        continue;
+      }
+      if (await exists(absPath)) {
+        await rm(absPath);
+        deleteDocument(db, absPath);
+        removed.push(absPath);
+      } else {
+        deleteDocument(db, absPath);
+        not_found.push(absPath);
+      }
+    }
+  } finally {
+    db.close();
+  }
+
+  return { removed, not_found, not_under_source, readonly_blocked };
+}

package/src/commands/memory/reset.mjs

@@ -0,0 +1,20 @@
+import { rm, access } from 'node:fs/promises';
+import { getBmPath, getDbPath } from '../../paths.mjs';
+
+async function exists(p) {
+  try { await access(p); return true; } catch { return false; }
+}
+
+export async function resetMemory(options = {}) {
+  const bmPath = options.bmPath || getBmPath();
+  const dbPath = getDbPath(bmPath);
+
+  for (const suffix of ['', '-wal', '-shm', '-journal']) {
+    const p = `${dbPath}${suffix}`;
+    if (await exists(p)) {
+      await rm(p);
+    }
+  }
+
+  return { reset: true };
+}

package/src/commands/memory/search.mjs

@@ -0,0 +1,73 @@
+import { isAbsolute, join, basename } from 'node:path';
+import { openDb } from '../../db/connection.mjs';
+import { searchDocuments } from '../../db/queries.mjs';
+import { loadConfig } from '../../config/io.mjs';
+import { findSourceByFilePath, resolveOriginalByStem } from '../../config/lookup.mjs';
+import { getBmPath, getConfigPath } from '../../paths.mjs';
+
+async function expandOriginal(row, hit) {
+  // 1. frontmatter has explicit original:
+  if (row.original) {
+    if (isAbsolute(row.original)) return row.original;
+    if (hit && hit.source && hit.source.raw) {
+      return join(hit.source.raw, row.original);
+    }
+    return row.original;
+  }
+  // 2. Stem match fallback: look for <stem>.* in source.raw
+  if (hit && hit.source && hit.source.raw) {
+    const stem = basename(row.file_path).replace(/\.md$/, '');
+    const match = await resolveOriginalByStem(hit.source.raw, stem);
+    return match; // null if 0 or 2+ matches
+  }
+  // 3. No source → give up
+  return null;
+}
+
+export async function searchMemory(options = {}) {
+  const bmPath = options.bmPath || getBmPath();
+  const configPath = options.configPath || getConfigPath();
+  const query = options.query || null;
+  const sourceType = options.type || null;
+  const after = options.after || null;
+  const before = options.before || null;
+  const limit = options.limit ? Number(options.limit) : 10;
+  const tags = options.tags
+    ? options.tags.split(',').map(t => t.trim()).filter(Boolean)
+    : null;
+
+  const config = await loadConfig(configPath);
+  const db = openDb(bmPath);
+  let rows;
+  try {
+    rows = searchDocuments(db, {
+      query,
+      sourceType,
+      tags,
+      after,
+      before,
+      limit,
+    });
+  } finally {
+    db.close();
+  }
+
+  const results = [];
+  for (const r of rows) {
+    const hit = findSourceByFilePath(config, r.file_path);
+    const original = await expandOriginal(r, hit);
+    results.push({
+      source: hit ? hit.name : null,
+      file: r.file_path,
+      title: r.title,
+      original,
+      source_type: r.source_type,
+      source_url: r.source_url,
+      created_at: r.created_at,
+      tags: r.tags,
+      snippet: r.snippet,
+      score: r.score,
+    });
+  }
+  return { query, results };
+}

package/src/config/io.mjs

@@ -0,0 +1,27 @@
+import { readFile, writeFile, mkdir } from 'node:fs/promises';
+import { dirname } from 'node:path';
+
+export async function loadConfig(configPath) {
+  let raw;
+  try {
+    raw = await readFile(configPath, 'utf-8');
+  } catch (err) {
+    if (err.code === 'ENOENT') return { sources: {} };
+    throw err;
+  }
+  let data;
+  try {
+    data = JSON.parse(raw);
+  } catch {
+    return { sources: {} };
+  }
+  if (!data || typeof data !== 'object' || !data.sources) {
+    return { sources: {} };
+  }
+  return data;
+}
+
+export async function saveConfig(configPath, config) {
+  await mkdir(dirname(configPath), { recursive: true });
+  await writeFile(configPath, JSON.stringify(config, null, 2) + '\n');
+}

package/src/config/lookup.mjs

@@ -0,0 +1,59 @@
+import { readdir } from 'node:fs/promises';
+import { join } from 'node:path';
+
+export function findSourceByFilePath(config, absFilePath) {
+  const sources = (config && config.sources) || {};
+  let best = null;
+  let bestLen = -1;
+  for (const [name, source] of Object.entries(sources)) {
+    const rawText = source.raw_text;
+    if (!rawText) continue;
+    if (absFilePath === rawText || absFilePath.startsWith(rawText + '/')) {
+      if (rawText.length > bestLen) {
+        best = { name, source };
+        bestLen = rawText.length;
+      }
+    }
+  }
+  return best;
+}
+
+export function listReadwriteSources(config) {
+  const sources = (config && config.sources) || {};
+  const out = [];
+  for (const [name, source] of Object.entries(sources)) {
+    if (source.access === 'readwrite') {
+      out.push({ name, source });
+    }
+  }
+  return out;
+}
+
+/**
+ * Look for a file in rawDir whose basename without extension matches `stem`.
+ * Ignores .md files (the raw_text side owns .md). Does not recurse.
+ *
+ * Returns:
+ *   - the absolute path if exactly one non-.md file matches
+ *   - null if rawDir is missing, 0 matches, or 2+ matches (ambiguous)
+ */
+export async function resolveOriginalByStem(rawDir, stem) {
+  let entries;
+  try {
+    entries = await readdir(rawDir, { withFileTypes: true });
+  } catch {
+    return null;
+  }
+  const matches = [];
+  for (const e of entries) {
+    if (!e.isFile()) continue;
+    if (e.name.endsWith('.md')) continue;
+    const dot = e.name.lastIndexOf('.');
+    const entryStem = dot === -1 ? e.name : e.name.slice(0, dot);
+    if (entryStem === stem) {
+      matches.push(join(rawDir, e.name));
+    }
+  }
+  if (matches.length === 1) return matches[0];
+  return null;
+}

package/src/db/connection.mjs

@@ -0,0 +1,16 @@
+import Database from 'better-sqlite3';
+import { mkdirSync } from 'node:fs';
+import { getDbPath } from '../paths.mjs';
+import { initSchema } from './schema.mjs';
+
+export function openDb(bmPath) {
+  mkdirSync(bmPath, { recursive: true });
+  const db = new Database(getDbPath(bmPath));
+  db.pragma('journal_mode = WAL');
+  db.pragma('synchronous = NORMAL');
+  db.pragma('cache_size = -64000');
+  db.pragma('mmap_size = 268435456');
+  db.pragma('temp_store = MEMORY');
+  initSchema(db);
+  return db;
+}

package/src/db/queries.mjs

@@ -0,0 +1,157 @@
+const UPSERT_SQL = `
+  INSERT INTO documents (
+    file_path, title, content, tags, source_type,
+    source_url, original, synthesized_from,
+    created_at, indexed_at, mtime, word_count
+  ) VALUES (
+    @file_path, @title, @content, @tags, @source_type,
+    @source_url, @original, @synthesized_from,
+    @created_at, @indexed_at, @mtime, @word_count
+  )
+  ON CONFLICT(file_path) DO UPDATE SET
+    title=excluded.title,
+    content=excluded.content,
+    tags=excluded.tags,
+    source_type=excluded.source_type,
+    source_url=excluded.source_url,
+    original=excluded.original,
+    synthesized_from=excluded.synthesized_from,
+    created_at=excluded.created_at,
+    indexed_at=excluded.indexed_at,
+    mtime=excluded.mtime,
+    word_count=excluded.word_count
+`;
+
+export function upsertDocument(db, doc) {
+  const stmt = db.prepare(UPSERT_SQL);
+  stmt.run({
+    file_path: doc.file_path,
+    title: doc.title ?? null,
+    content: doc.content,
+    tags: JSON.stringify(doc.tags ?? []),
+    source_type: doc.source_type ?? null,
+    source_url: doc.source_url ?? null,
+    original: doc.original ?? null,
+    synthesized_from: doc.synthesized_from ? JSON.stringify(doc.synthesized_from) : null,
+    created_at: doc.created_at ?? null,
+    indexed_at: doc.indexed_at ?? new Date().toISOString(),
+    mtime: doc.mtime,
+    word_count: doc.word_count ?? 0,
+  });
+}
+
+export function deleteDocument(db, filePath) {
+  db.prepare('DELETE FROM documents WHERE file_path = ?').run(filePath);
+}
+
+export function listIndexedFiles(db) {
+  const rows = db.prepare('SELECT file_path, mtime FROM documents').all();
+  const map = new Map();
+  for (const r of rows) map.set(r.file_path, r.mtime);
+  return map;
+}
+
+function parseTags(tagsJson) {
+  if (!tagsJson) return [];
+  try { return JSON.parse(tagsJson); } catch { return []; }
+}
+
+export function searchDocuments(db, options) {
+  const {
+    query = null,
+    sourceType = null,
+    tags = null,
+    after = null,
+    before = null,
+    limit = 10,
+  } = options;
+
+  if (query) {
+    // Escape double-quotes inside the query and wrap in double-quotes so FTS5
+    // treats the whole string as a phrase / literal token sequence rather than
+    // interpreting hyphens as NOT-operators or "col:" as column filters.
+    const ftsQuery = '"' + query.replace(/"/g, '""') + '"';
+    const where = ['documents_fts MATCH ?'];
+    const params = [ftsQuery];
+
+    if (sourceType) {
+      where.push('d.source_type = ?');
+      params.push(sourceType);
+    }
+    if (after) {
+      where.push('d.created_at >= ?');
+      params.push(after);
+    }
+    if (before) {
+      where.push('d.created_at <= ?');
+      params.push(before);
+    }
+    if (tags && tags.length > 0) {
+      const tagConds = tags.map(() => 'd.tags LIKE ?').join(' OR ');
+      where.push(`(${tagConds})`);
+      for (const t of tags) params.push(`%"${t}"%`);
+    }
+
+    const sql = `
+      SELECT
+        d.file_path,
+        d.title,
+        d.original,
+        d.source_type,
+        d.source_url,
+        d.created_at,
+        d.tags,
+        snippet(documents_fts, 1, '...', '...', '', 30) AS snippet,
+        rank AS score
+      FROM documents_fts
+      JOIN documents d ON d.id = documents_fts.rowid
+      WHERE ${where.join(' AND ')}
+      ORDER BY rank
+      LIMIT ?
+    `;
+    params.push(limit);
+    return db.prepare(sql).all(...params).map(r => ({ ...r, tags: parseTags(r.tags) }));
+  }
+
+  // No query: list mode, order by created_at DESC
+  const where = [];
+  const params = [];
+
+  if (sourceType) {
+    where.push('source_type = ?');
+    params.push(sourceType);
+  }
+  if (after) {
+    where.push('created_at >= ?');
+    params.push(after);
+  }
+  if (before) {
+    where.push('created_at <= ?');
+    params.push(before);
+  }
+  if (tags && tags.length > 0) {
+    const tagConds = tags.map(() => 'tags LIKE ?').join(' OR ');
+    where.push(`(${tagConds})`);
+    for (const t of tags) params.push(`%"${t}"%`);
+  }
+
+  const whereClause = where.length > 0 ? `WHERE ${where.join(' AND ')}` : '';
+  const sql = `
+    SELECT
+      file_path,
+      title,
+      original,
+      source_type,
+      source_url,
+      created_at,
+      tags,
+      NULL AS snippet,
+      NULL AS score
+    FROM documents
+    ${whereClause}
+    ORDER BY created_at DESC
+    LIMIT ?
+  `;
+  params.push(limit);
+  return db.prepare(sql).all(...params).map(r => ({ ...r, tags: parseTags(r.tags) }));
+}

package/src/db/schema.mjs

@@ -0,0 +1,48 @@
+export function initSchema(db) {
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS documents (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      file_path TEXT NOT NULL UNIQUE,
+      title TEXT,
+      content TEXT NOT NULL,
+      tags TEXT,
+      source_type TEXT,
+      source_url TEXT,
+      original TEXT,
+      synthesized_from TEXT,
+      created_at TEXT,
+      indexed_at TEXT NOT NULL,
+      mtime INTEGER NOT NULL,
+      word_count INTEGER
+    );
+
+    CREATE INDEX IF NOT EXISTS idx_documents_source_type ON documents(source_type);
+    CREATE INDEX IF NOT EXISTS idx_documents_created_at ON documents(created_at);
+
+    CREATE VIRTUAL TABLE IF NOT EXISTS documents_fts USING fts5(
+      title,
+      content,
+      tags,
+      content=documents,
+      content_rowid=id,
+      tokenize='trigram'
+    );
+
+    CREATE TRIGGER IF NOT EXISTS documents_ai AFTER INSERT ON documents BEGIN
+      INSERT INTO documents_fts(rowid, title, content, tags)
+      VALUES (new.id, new.title, new.content, new.tags);
+    END;
+
+    CREATE TRIGGER IF NOT EXISTS documents_ad AFTER DELETE ON documents BEGIN
+      INSERT INTO documents_fts(documents_fts, rowid, title, content, tags)
+      VALUES ('delete', old.id, old.title, old.content, old.tags);
+    END;
+
+    CREATE TRIGGER IF NOT EXISTS documents_au AFTER UPDATE ON documents BEGIN
+      INSERT INTO documents_fts(documents_fts, rowid, title, content, tags)
+      VALUES ('delete', old.id, old.title, old.content, old.tags);
+      INSERT INTO documents_fts(rowid, title, content, tags)
+      VALUES (new.id, new.title, new.content, new.tags);
+    END;
+  `);
+}

package/src/indexer/index-all.mjs

@@ -0,0 +1,74 @@
+import { readFile } from 'node:fs/promises';
+import { scanRawText } from './scan.mjs';
+import { parseMarkdown } from './parse.mjs';
+import { upsertDocument, deleteDocument, listIndexedFiles } from '../db/queries.mjs';
+
+function mtimeToDate(mtime) {
+  return new Date(mtime).toISOString().slice(0, 10);
+}
+
+export async function indexAll(db, config, options = {}) {
+  const force = options.force === true;
+  const start = Date.now();
+
+  const rawTextDirs = Object.values(config.sources || {}).map(s => s.raw_text);
+  const { files, warnings } = await scanRawText(rawTextDirs);
+
+  const indexed = listIndexedFiles(db);
+  const stats = {
+    sources: rawTextDirs.length,
+    files: files.length,
+    added: 0,
+    updated: 0,
+    removed: 0,
+    skipped: 0,
+    duration_ms: 0,
+    warnings,
+  };
+
+  const seen = new Set();
+  for (const f of files) {
+    seen.add(f.absPath);
+    const prevMtime = indexed.get(f.absPath);
+
+    if (!force && prevMtime !== undefined && prevMtime === f.mtime) {
+      stats.skipped++;
+      continue;
+    }
+
+    const raw = await readFile(f.absPath, 'utf-8');
+    const parsed = parseMarkdown(raw, f.absPath, { mtimeDate: mtimeToDate(f.mtime) });
+
+    upsertDocument(db, {
+      file_path: f.absPath,
+      title: parsed.title,
+      content: parsed.content,
+      tags: parsed.tags,
+      source_type: parsed.source_type,
+      source_url: parsed.source_url,
+      original: parsed.original,
+      synthesized_from: parsed.synthesized_from,
+      created_at: parsed.created_at,
+      mtime: f.mtime,
+      word_count: parsed.word_count,
+    });
+
+    if (prevMtime === undefined) stats.added++;
+    else stats.updated++;
+  }
+
+  // Orphan cleanup: any indexed row whose file_path wasn't encountered in
+  // this scan gets dropped. Covers both "file deleted from disk" and
+  // "source removed from config" (the latter because we simply don't scan
+  // the removed source's raw_text anymore, so its files won't appear in
+  // `seen` and will look like orphans).
+  for (const [filePath] of indexed) {
+    if (!seen.has(filePath)) {
+      deleteDocument(db, filePath);
+      stats.removed++;
+    }
+  }
+
+  stats.duration_ms = Date.now() - start;
+  return stats;
+}

package/src/indexer/parse.mjs

@@ -0,0 +1,55 @@
+import matter from 'gray-matter';
+import { basename } from 'node:path';
+
+export function extractDateFromFilename(filename) {
+  const fname = basename(filename);
+  const hyphenated = fname.match(/^(\d{4}-\d{2}-\d{2})/);
+  if (hyphenated) return hyphenated[1];
+  const compact = fname.match(/^(\d{4})(\d{2})(\d{2})(?:[_T]|$)/);
+  if (compact) return `${compact[1]}-${compact[2]}-${compact[3]}`;
+  return null;
+}
+
+export function parseMarkdown(raw, filePath, options = {}) {
+  const { overrides = {}, mtimeDate = null } = options;
+
+  const parsed = matter(raw);
+  const fm = parsed.data || {};
+  const content = (parsed.content || '').trim();
+
+  const fname = basename(filePath).replace(/\.md$/, '');
+
+  const title = overrides.title ?? fm.title ?? fname;
+  const source_type = overrides.source_type ?? fm.source_type ?? 'unknown';
+  const tags = overrides.tags ?? (Array.isArray(fm.tags) ? fm.tags : []);
+
+  // gray-matter may auto-parse YAML dates as Date objects; normalize to ISO date string
+  let fmCreatedAt = fm.created_at ?? null;
+  if (fmCreatedAt instanceof Date) {
+    fmCreatedAt = fmCreatedAt.toISOString().slice(0, 10);
+  }
+
+  const created_at =
+    overrides.created_at ??
+    fmCreatedAt ??
+    extractDateFromFilename(fname) ??
+    mtimeDate ??
+    null;
+  const original = overrides.original ?? fm.original ?? null;
+  const source_url = overrides.source_url ?? fm.source_url ?? null;
+  const synthesized_from = Array.isArray(fm.synthesized_from)
+    ? fm.synthesized_from
+    : null;
+
+  return {
+    title,
+    content,
+    tags,
+    source_type,
+    source_url,
+    original,
+    synthesized_from,
+    created_at,
+    word_count: content.length,
+  };
+}

package/src/indexer/scan.mjs

@@ -0,0 +1,37 @@
+import { readdir, stat } from 'node:fs/promises';
+import { join } from 'node:path';
+
+async function walk(dir, out) {
+  let entries;
+  try {
+    entries = await readdir(dir, { withFileTypes: true });
+  } catch {
+    return false; // directory missing
+  }
+  for (const e of entries) {
+    if (e.name.startsWith('.')) continue;
+    const p = join(dir, e.name);
+    if (e.isDirectory()) {
+      await walk(p, out);
+    } else if (e.isFile() && e.name.endsWith('.md')) {
+      const s = await stat(p);
+      out.push({
+        absPath: p,
+        mtime: Math.floor(s.mtimeMs),
+      });
+    }
+  }
+  return true;
+}
+
+export async function scanRawText(rawTextDirs) {
+  const files = [];
+  const warnings = [];
+  for (const dir of rawTextDirs) {
+    const ok = await walk(dir, files);
+    if (!ok) {
+      warnings.push({ dir, message: `raw_text directory not found: ${dir}` });
+    }
+  }
+  return { files, warnings };
+}

package/src/paths.mjs

@@ -0,0 +1,21 @@
+import { join } from 'node:path';
+import { homedir } from 'node:os';
+import { mkdir } from 'node:fs/promises';
+
+const DEFAULT_BM_PATH = join(homedir(), '.botrun', 'bm');
+
+export function getBmPath() {
+  return process.env.BM_PATH || DEFAULT_BM_PATH;
+}
+
+export function getConfigPath() {
+  return process.env.BM_CONFIG || join(getBmPath(), 'config.json');
+}
+
+export function getDbPath(bmPath = getBmPath()) {
+  return join(bmPath, 'search.db');
+}
+
+export async function ensureBmDirs(bmPath = getBmPath()) {
+  await mkdir(bmPath, { recursive: true });
+}

package/src/commands/config/add-scope.mjs

@@ -1,13 +0,0 @@
-import { loadConfig, saveConfig } from '../../config.mjs';
-
-export async function addScope(name, options, configPath) {
-  const config = await loadConfig(configPath);
-  const scopeEntry = { repo: options.repo };
-  if (options.branch) scopeEntry.branch = options.branch;
-  if (options.tokenEnv) scopeEntry.token_env = options.tokenEnv;
-  if (options.description) scopeEntry.description = options.description;
-  if (options.access) scopeEntry.access = options.access;
-  config.scopes[name] = scopeEntry;
-  await saveConfig(configPath, config);
-  return { added: name, scope: scopeEntry };
-}

package/src/commands/config/remove-scope.mjs

@@ -1,11 +0,0 @@
-import { loadConfig, saveConfig } from '../../config.mjs';
-
-export async function removeScope(name, configPath) {
-  const config = await loadConfig(configPath);
-  if (!config.scopes[name]) {
-    throw new Error(`Scope "${name}" not found`);
-  }
-  delete config.scopes[name];
-  await saveConfig(configPath, config);
-  return { removed: name };
-}

package/src/commands/memory/init.mjs

@@ -1,72 +0,0 @@
-import { mkdir, lstat } from 'node:fs/promises';
-import { join } from 'node:path';
-import { loadConfig, getBasePath } from '../../config.mjs';
-import { gitExec } from '../../git-cmd.mjs';
-import { detectProvider, getProvider, resolveToken } from '../../git/provider.mjs';
-
-async function exists(path) {
-  try {
-    await lstat(path);
-    return true;
-  } catch {
-    return false;
-  }
-}
-
-async function hasWorkingTree(dir) {
-  try {
-    const result = await gitExec(['-C', dir, 'rev-parse', '--is-inside-work-tree']);
-    return result === 'true';
-  } catch {
-    return false;
-  }
-}
-
-async function cloneOrPull(repo, cloneDir, token, localMode, branch) {
-  if (await exists(cloneDir) && await hasWorkingTree(cloneDir)) {
-    await gitExec(['-C', cloneDir, 'pull', '--rebase']);
-    return;
-  }
-
-  // If directory exists but broken (no working tree), remove and re-clone
-  if (await exists(cloneDir)) {
-    const { rm } = await import('node:fs/promises');
-    await rm(cloneDir, { recursive: true });
-  }
-
-  let cloneUrl;
-  if (localMode) {
-    cloneUrl = repo;
-  } else {
-    const providerName = detectProvider(repo);
-    const provider = getProvider(providerName);
-    cloneUrl = provider.buildCloneUrl(repo, token);
-  }
-
-  const cloneArgs = ['clone', '--depth', '1'];
-  if (branch) cloneArgs.push('--branch', branch);
-  cloneArgs.push(cloneUrl, cloneDir);
-  await gitExec(cloneArgs);
-}
-
-export async function initMemory(options = {}) {
-  const configPath = options.configPath;
-  const dataDir = options.dataDir || join(getBasePath(), 'data');
-  const localMode = options.localMode || false;
-
-  const config = await loadConfig(configPath);
-  const result = { scopes: {} };
-
-  await mkdir(dataDir, { recursive: true });
-
-  for (const [name, scope] of Object.entries(config.scopes)) {
-    const cloneDir = join(dataDir, name);
-    const token = localMode ? undefined : resolveToken(scope);
-
-    await cloneOrPull(scope.repo, cloneDir, token, localMode, scope.branch);
-
-    result.scopes[name] = { local: cloneDir };
-  }
-
-  return result;
-}

package/src/commands/memory/scopes.mjs

@@ -1,27 +0,0 @@
-import { join } from 'node:path';
-import { lstat } from 'node:fs/promises';
-import { loadConfig, getBasePath } from '../../config.mjs';
-
-export async function listScopes(options = {}) {
-  const config = await loadConfig(options.configPath);
-  const dataDir = options.dataDir || join(getBasePath(), 'data');
-  const result = { scopes: {} };
-
-  for (const [name, scope] of Object.entries(config.scopes)) {
-    const entry = { repo: scope.repo };
-    if (scope.description) entry.description = scope.description;
-    if (scope.access) entry.access = scope.access;
-
-    const scopeDir = join(dataDir, name);
-    try {
-      await lstat(scopeDir);
-      entry.local = scopeDir;
-    } catch {
-      entry.local = null;
-    }
-
-    result.scopes[name] = entry;
-  }
-
-  return result;
-}

package/src/commands/memory/sync.mjs

@@ -1,54 +0,0 @@
-import { join } from 'node:path';
-import { loadConfig, getBasePath } from '../../config.mjs';
-import { gitExec } from '../../git-cmd.mjs';
-
-export async function syncMemory(options = {}) {
-  const config = await loadConfig(options.configPath);
-  const dataDir = options.dataDir || join(getBasePath(), 'data');
-
-  const result = { synced: [], pulled: [], skipped: [] };
-
-  for (const [name, scope] of Object.entries(config.scopes)) {
-    const cloneDir = join(dataDir, name);
-    let didPull = false;
-    let didPush = false;
-
-    // 1. Always pull remote changes first
-    try {
-      const before = await gitExec(['-C', cloneDir, 'rev-parse', 'HEAD']);
-      await gitExec(['-C', cloneDir, 'pull', '--rebase']);
-      const after = await gitExec(['-C', cloneDir, 'rev-parse', 'HEAD']);
-      if (before !== after) didPull = true;
-    } catch {
-      // pull may fail if no upstream set; try setting it
-      try {
-        const branch = await gitExec(['-C', cloneDir, 'rev-parse', '--abbrev-ref', 'HEAD']);
-        await gitExec(['-C', cloneDir, 'branch', '--set-upstream-to', `origin/${branch}`, branch]);
-        const before = await gitExec(['-C', cloneDir, 'rev-parse', 'HEAD']);
-        await gitExec(['-C', cloneDir, 'pull', '--rebase']);
-        const after = await gitExec(['-C', cloneDir, 'rev-parse', 'HEAD']);
-        if (before !== after) didPull = true;
-      } catch {
-        // still failed, continue
-      }
-    }
-
-    // 2. Check for local uncommitted changes and push
-    const status = await gitExec(['-C', cloneDir, 'status', '--porcelain']);
-    if (status) {
-      await gitExec(['-C', cloneDir, 'add', '-A']);
-      await gitExec(['-C', cloneDir, 'commit', '-m', `bm: update ${name} memories`]);
-      await gitExec(['-C', cloneDir, 'push']);
-      didPush = true;
-    }
-
-    if (didPull || didPush) {
-      result.synced.push(name);
-      if (didPull) result.pulled.push(name);
-    } else {
-      result.skipped.push(name);
-    }
-  }
-
-  return result;
-}

package/src/config.mjs

@@ -1,30 +0,0 @@
-import { readFile, writeFile, mkdir } from 'node:fs/promises';
-import { homedir } from 'node:os';
-import { join, dirname } from 'node:path';
-
-const DEFAULT_BASE_PATH = join(homedir(), '.botrun', 'bm');
-
-export function getBasePath() {
-  return process.env.BM_PATH || DEFAULT_BASE_PATH;
-}
-
-export function getConfigPath() {
-  return process.env.BM_CONFIG || join(getBasePath(), 'config.json');
-}
-
-export async function loadConfig(configPath = getConfigPath()) {
-  try {
-    const content = await readFile(configPath, 'utf-8');
-    return JSON.parse(content);
-  } catch (err) {
-    if (err.code === 'ENOENT') {
-      return { scopes: {} };
-    }
-    throw err;
-  }
-}
-
-export async function saveConfig(configPath = getConfigPath(), config) {
-  await mkdir(dirname(configPath), { recursive: true });
-  await writeFile(configPath, JSON.stringify(config, null, 2) + '\n');
-}

package/src/git/github.mjs

@@ -1,7 +0,0 @@
-export function buildCloneUrl(repo, token) {
-  // Normalize: strip https://, trailing .git
-  const cleaned = repo
-    .replace(/^https?:\/\//, '')
-    .replace(/\.git$/, '');
-  return `https://x-access-token:${token}@${cleaned}.git`;
-}

package/src/git/gitlab.mjs

@@ -1,3 +0,0 @@
-export function buildCloneUrl() {
-  throw new Error('GitLab support not yet implemented');
-}

package/src/git/provider.mjs

@@ -1,23 +0,0 @@
-import * as github from './github.mjs';
-import * as gitlab from './gitlab.mjs';
-
-const providers = { github, gitlab };
-
-export function detectProvider(repoUrl) {
-  if (repoUrl.includes('github.com')) return 'github';
-  if (repoUrl.includes('gitlab.com')) return 'gitlab';
-  throw new Error(`Unknown git provider for URL: ${repoUrl}`);
-}
-
-export function getProvider(name) {
-  const provider = providers[name];
-  if (!provider) throw new Error(`Unknown provider: ${name}`);
-  // trigger "not yet implemented" for gitlab
-  if (name === 'gitlab') provider.buildCloneUrl();
-  return provider;
-}
-
-export function resolveToken(scope) {
-  if (!scope.token_env) return undefined;
-  return process.env[scope.token_env] || undefined;
-}

package/src/git-cmd.mjs

@@ -1,12 +0,0 @@
-import { execFile } from 'node:child_process';
-import { promisify } from 'node:util';
-
-const execFileAsync = promisify(execFile);
-
-export async function gitExec(args, options = {}) {
-  const { stdout } = await execFileAsync('git', args, {
-    maxBuffer: 10 * 1024 * 1024,
-    ...options,
-  });
-  return stdout.trim();
-}