@geekbeer/minion 3.36.0 → 3.40.1

package/core/db.js

@@ -1,583 +0,0 @@
-/**
- * SQLite Database Module
- * Provides a singleton SQLite database connection for all minion stores.
- * Uses better-sqlite3 (synchronous API) with WAL mode for performance.
- * Falls back to Node.js built-in node:sqlite when better-sqlite3 is unavailable
- * (e.g., no prebuilt binaries for the current Node.js version on Windows).
- *
- * Database file: $DATA_DIR/minion.db
- *
- * FTS5 uses the trigram tokenizer for Japanese/CJK language support.
- * Trigram splits text into 3-character sequences, enabling substring search
- * in any language without a language-specific tokenizer.
- * Note: trigram requires search queries of 3+ characters.
- * Shorter queries fall back to LIKE-based search in the store layer.
- */
-
-const path = require('path')
-const fs = require('fs')
-
-const { DATA_DIR } = require('./lib/platform')
-const { config } = require('./config')
-
-let _db = null
-
-/**
- * Resolve the database file path.
- * Uses DATA_DIR if accessible, otherwise falls back to HOME_DIR.
- */
-function resolveDbPath() {
-  try {
-    fs.accessSync(DATA_DIR, fs.constants.W_OK)
-    return path.join(DATA_DIR, 'minion.db')
-  } catch {
-    return path.join(config.HOME_DIR, 'minion.db')
-  }
-}
-
-/**
- * Open a database connection using better-sqlite3 or node:sqlite fallback.
- * @param {string} dbPath
- * @returns {import('better-sqlite3').Database}
- */
-function openDatabase(dbPath) {
-  // Try better-sqlite3 first (fastest, has prebuilt binaries for most platforms)
-  try {
-    const Database = require('better-sqlite3')
-    const db = new Database(dbPath)
-    db.pragma('journal_mode = WAL')
-    db.pragma('foreign_keys = ON')
-    console.log('[DB] Using better-sqlite3')
-    return db
-  } catch (e) {
-    console.log(`[DB] better-sqlite3 unavailable (${e.message}), trying node:sqlite...`)
-  }
-
-  // Fallback to Node.js built-in sqlite (available in Node 22.5.0+)
-  try {
-    const { DatabaseSync } = require('node:sqlite')
-    const db = new DatabaseSync(dbPath)
-    db.exec('PRAGMA journal_mode = WAL')
-    db.exec('PRAGMA foreign_keys = ON')
-
-    // Polyfill db.pragma() for compatibility with better-sqlite3 API
-    db.pragma = function (str) {
-      const stmt = db.prepare(`PRAGMA ${str}`)
-      return stmt.get()
-    }
-
-    // Polyfill db.transaction() for compatibility with better-sqlite3 API
-    db.transaction = function (fn) {
-      return function (...args) {
-        db.exec('BEGIN')
-        try {
-          const result = fn(...args)
-          db.exec('COMMIT')
-          return result
-        } catch (e) {
-          db.exec('ROLLBACK')
-          throw e
-        }
-      }
-    }
-
-    console.log('[DB] Using node:sqlite (built-in)')
-    return db
-  } catch (e) {
-    throw new Error(
-      `[DB] No SQLite driver available. Install better-sqlite3 or use Node.js 22.5.0+ for built-in sqlite support. ` +
-      `better-sqlite3 error: check that Python and build tools are installed, or use Node.js 22 LTS which has prebuilt binaries. ` +
-      `node:sqlite error: ${e.message}`
-    )
-  }
-}
-
-/**
- * Get the singleton database connection.
- * Initializes the database and schema on first call.
- * @returns {import('better-sqlite3').Database}
- */
-function getDb() {
-  if (_db) return _db
-
-  const dbPath = resolveDbPath()
-  fs.mkdirSync(path.dirname(dbPath), { recursive: true })
-
-  _db = openDatabase(dbPath)
-
-  initSchema(_db)
-  migrateSchema(_db)
-
-  console.log(`[DB] SQLite database initialized: ${dbPath}`)
-  return _db
-}
-
-/**
- * Initialize all tables, indices, FTS virtual tables, and triggers.
- * All statements use IF NOT EXISTS for idempotency.
- */
-function initSchema(db) {
-  db.exec(`
-    -- ==================== memories ====================
-    CREATE TABLE IF NOT EXISTS memories (
-      id TEXT PRIMARY KEY,
-      title TEXT NOT NULL DEFAULT 'Untitled',
-      category TEXT NOT NULL DEFAULT 'reference'
-        CHECK (category IN ('user', 'feedback', 'project', 'reference')),
-      content TEXT NOT NULL DEFAULT '',
-      created_at TEXT NOT NULL,
-      updated_at TEXT NOT NULL
-    );
-
-    CREATE INDEX IF NOT EXISTS idx_memories_category ON memories(category);
-    CREATE INDEX IF NOT EXISTS idx_memories_updated_at ON memories(updated_at DESC);
-
-    -- FTS5 with trigram tokenizer for Japanese/CJK support
-    CREATE VIRTUAL TABLE IF NOT EXISTS memories_fts USING fts5(
-      title,
-      content,
-      content=memories,
-      content_rowid=rowid,
-      tokenize='trigram'
-    );
-
-    -- Triggers to keep FTS index in sync with memories table
-    CREATE TRIGGER IF NOT EXISTS memories_ai AFTER INSERT ON memories BEGIN
-      INSERT INTO memories_fts(rowid, title, content)
-        VALUES (new.rowid, new.title, new.content);
-    END;
-
-    CREATE TRIGGER IF NOT EXISTS memories_ad AFTER DELETE ON memories BEGIN
-      INSERT INTO memories_fts(memories_fts, rowid, title, content)
-        VALUES ('delete', old.rowid, old.title, old.content);
-    END;
-
-    CREATE TRIGGER IF NOT EXISTS memories_au AFTER UPDATE ON memories BEGIN
-      INSERT INTO memories_fts(memories_fts, rowid, title, content)
-        VALUES ('delete', old.rowid, old.title, old.content);
-      INSERT INTO memories_fts(rowid, title, content)
-        VALUES (new.rowid, new.title, new.content);
-    END;
-
-    -- ==================== daily_logs ====================
-    CREATE TABLE IF NOT EXISTS daily_logs (
-      date TEXT PRIMARY KEY,
-      content TEXT NOT NULL DEFAULT '',
-      created_at TEXT NOT NULL DEFAULT (datetime('now')),
-      updated_at TEXT NOT NULL DEFAULT (datetime('now'))
-    );
-
-    -- FTS5 with trigram tokenizer for Japanese/CJK support
-    CREATE VIRTUAL TABLE IF NOT EXISTS daily_logs_fts USING fts5(
-      content,
-      content=daily_logs,
-      content_rowid=rowid,
-      tokenize='trigram'
-    );
-
-    CREATE TRIGGER IF NOT EXISTS daily_logs_ai AFTER INSERT ON daily_logs BEGIN
-      INSERT INTO daily_logs_fts(rowid, content)
-        VALUES (new.rowid, new.content);
-    END;
-
-    CREATE TRIGGER IF NOT EXISTS daily_logs_ad AFTER DELETE ON daily_logs BEGIN
-      INSERT INTO daily_logs_fts(daily_logs_fts, rowid, content)
-        VALUES ('delete', old.rowid, old.content);
-    END;
-
-    CREATE TRIGGER IF NOT EXISTS daily_logs_au AFTER UPDATE ON daily_logs BEGIN
-      INSERT INTO daily_logs_fts(daily_logs_fts, rowid, content)
-        VALUES ('delete', old.rowid, old.content);
-      INSERT INTO daily_logs_fts(rowid, content)
-        VALUES (new.rowid, new.content);
-    END;
-
-    -- ==================== workspaces (cache from HQ) ====================
-    CREATE TABLE IF NOT EXISTS workspaces (
-      id TEXT PRIMARY KEY,
-      name TEXT NOT NULL,
-      slug TEXT NOT NULL,
-      updated_at INTEGER NOT NULL
-    );
-
-    -- ==================== chat_sessions ====================
-    CREATE TABLE IF NOT EXISTS chat_sessions (
-      session_id TEXT PRIMARY KEY,
-      workspace_id TEXT,
-      turn_count INTEGER NOT NULL DEFAULT 0,
-      created_at INTEGER NOT NULL,
-      updated_at INTEGER NOT NULL
-    );
-
-    CREATE TABLE IF NOT EXISTS chat_messages (
-      id INTEGER PRIMARY KEY AUTOINCREMENT,
-      session_id TEXT NOT NULL REFERENCES chat_sessions(session_id) ON DELETE CASCADE,
-      role TEXT NOT NULL,
-      content TEXT NOT NULL,
-      timestamp INTEGER NOT NULL
-    );
-
-    CREATE INDEX IF NOT EXISTS idx_chat_messages_session ON chat_messages(session_id, id);
-
-    -- ==================== executions ====================
-    CREATE TABLE IF NOT EXISTS executions (
-      id TEXT PRIMARY KEY,
-      skill_name TEXT,
-      workflow_id TEXT,
-      status TEXT,
-      created_at TEXT NOT NULL DEFAULT (datetime('now')),
-      data TEXT NOT NULL
-    );
-
-    CREATE INDEX IF NOT EXISTS idx_executions_workflow ON executions(workflow_id);
-    CREATE INDEX IF NOT EXISTS idx_executions_created ON executions(created_at DESC);
-
-    -- ==================== routines ====================
-    CREATE TABLE IF NOT EXISTS routines (
-      id TEXT PRIMARY KEY,
-      name TEXT NOT NULL,
-      data TEXT NOT NULL
-    );
-
-    CREATE INDEX IF NOT EXISTS idx_routines_name ON routines(name);
-
-    -- ==================== workflows ====================
-    CREATE TABLE IF NOT EXISTS workflows (
-      id TEXT PRIMARY KEY,
-      name TEXT NOT NULL,
-      data TEXT NOT NULL
-    );
-
-    CREATE INDEX IF NOT EXISTS idx_workflows_name ON workflows(name);
-
-    -- ==================== todos ====================
-    CREATE TABLE IF NOT EXISTS todos (
-      id TEXT PRIMARY KEY,
-      title TEXT NOT NULL,
-      description TEXT,
-      status TEXT NOT NULL DEFAULT 'pending'
-        CHECK (status IN ('pending', 'in_progress', 'done', 'cancelled')),
-      priority TEXT NOT NULL DEFAULT 'normal'
-        CHECK (priority IN ('low', 'normal', 'high', 'urgent')),
-      source_type TEXT
-        CHECK (source_type IS NULL OR source_type IN ('thread', 'workflow', 'directive', 'user', 'self')),
-      source_id TEXT,
-      project_id TEXT,
-      due_at TEXT,
-      created_at TEXT NOT NULL DEFAULT (datetime('now')),
-      updated_at TEXT NOT NULL DEFAULT (datetime('now')),
-      completed_at TEXT,
-      data TEXT,
-      session_id TEXT,
-      injection_count INTEGER NOT NULL DEFAULT 0
-    );
-
-    CREATE INDEX IF NOT EXISTS idx_todos_status ON todos(status);
-    CREATE INDEX IF NOT EXISTS idx_todos_project ON todos(project_id);
-    CREATE INDEX IF NOT EXISTS idx_todos_priority ON todos(priority);
-
-    -- ==================== emails ====================
-    CREATE TABLE IF NOT EXISTS emails (
-      id TEXT PRIMARY KEY,
-      from_address TEXT NOT NULL,
-      to_address TEXT NOT NULL,
-      subject TEXT DEFAULT '',
-      body_text TEXT DEFAULT '',
-      body_html TEXT DEFAULT '',
-      received_at TEXT NOT NULL,
-      is_read INTEGER DEFAULT 0,
-      labels TEXT DEFAULT '[]'
-    );
-
-    CREATE INDEX IF NOT EXISTS idx_emails_received_at ON emails(received_at DESC);
-    CREATE INDEX IF NOT EXISTS idx_emails_from ON emails(from_address);
-    CREATE INDEX IF NOT EXISTS idx_emails_is_read ON emails(is_read);
-
-    -- ==================== email_attachments ====================
-    CREATE TABLE IF NOT EXISTS email_attachments (
-      id TEXT PRIMARY KEY,
-      email_id TEXT NOT NULL REFERENCES emails(id) ON DELETE CASCADE,
-      filename TEXT NOT NULL,
-      content_type TEXT,
-      size_bytes INTEGER DEFAULT 0,
-      approved INTEGER DEFAULT 0,
-      data BLOB
-    );
-
-    CREATE INDEX IF NOT EXISTS idx_email_attachments_email ON email_attachments(email_id);
-
-    -- ==================== schema_version ====================
-    CREATE TABLE IF NOT EXISTS schema_version (
-      version INTEGER PRIMARY KEY,
-      applied_at TEXT NOT NULL DEFAULT (datetime('now'))
-    );
-  `)
-}
-
-/**
- * Run schema migrations for existing databases.
- * Each migration checks schema_version to avoid re-running.
- */
-function migrateSchema(db) {
-  const currentVersion = db.prepare(
-    'SELECT COALESCE(MAX(version), 0) as v FROM schema_version'
-  ).get().v
-
-  if (currentVersion < 1) {
-    // Migration 1: Recreate FTS tables with trigram tokenizer for Japanese support.
-    // The default unicode61 tokenizer doesn't tokenize CJK text correctly.
-    console.log('[DB] Migration 1: Switching FTS5 to trigram tokenizer...')
-
-    db.exec(`
-      -- Drop old FTS tables and triggers (may use unicode61 tokenizer)
-      DROP TRIGGER IF EXISTS memories_ai;
-      DROP TRIGGER IF EXISTS memories_ad;
-      DROP TRIGGER IF EXISTS memories_au;
-      DROP TABLE IF EXISTS memories_fts;
-
-      DROP TRIGGER IF EXISTS daily_logs_ai;
-      DROP TRIGGER IF EXISTS daily_logs_ad;
-      DROP TRIGGER IF EXISTS daily_logs_au;
-      DROP TABLE IF EXISTS daily_logs_fts;
-
-      -- Recreate with trigram tokenizer
-      CREATE VIRTUAL TABLE memories_fts USING fts5(
-        title,
-        content,
-        content=memories,
-        content_rowid=rowid,
-        tokenize='trigram'
-      );
-
-      CREATE TRIGGER memories_ai AFTER INSERT ON memories BEGIN
-        INSERT INTO memories_fts(rowid, title, content)
-          VALUES (new.rowid, new.title, new.content);
-      END;
-
-      CREATE TRIGGER memories_ad AFTER DELETE ON memories BEGIN
-        INSERT INTO memories_fts(memories_fts, rowid, title, content)
-          VALUES ('delete', old.rowid, old.title, old.content);
-      END;
-
-      CREATE TRIGGER memories_au AFTER UPDATE ON memories BEGIN
-        INSERT INTO memories_fts(memories_fts, rowid, title, content)
-          VALUES ('delete', old.rowid, old.title, old.content);
-        INSERT INTO memories_fts(rowid, title, content)
-          VALUES (new.rowid, new.title, new.content);
-      END;
-
-      CREATE VIRTUAL TABLE daily_logs_fts USING fts5(
-        content,
-        content=daily_logs,
-        content_rowid=rowid,
-        tokenize='trigram'
-      );
-
-      CREATE TRIGGER daily_logs_ai AFTER INSERT ON daily_logs BEGIN
-        INSERT INTO daily_logs_fts(rowid, content)
-          VALUES (new.rowid, new.content);
-      END;
-
-      CREATE TRIGGER daily_logs_ad AFTER DELETE ON daily_logs BEGIN
-        INSERT INTO daily_logs_fts(daily_logs_fts, rowid, content)
-          VALUES ('delete', old.rowid, old.content);
-      END;
-
-      CREATE TRIGGER daily_logs_au AFTER UPDATE ON daily_logs BEGIN
-        INSERT INTO daily_logs_fts(daily_logs_fts, rowid, content)
-          VALUES ('delete', old.rowid, old.content);
-        INSERT INTO daily_logs_fts(rowid, content)
-          VALUES (new.rowid, new.content);
-      END;
-
-      -- Rebuild FTS index from existing data
-      INSERT INTO memories_fts(memories_fts) VALUES ('rebuild');
-      INSERT INTO daily_logs_fts(daily_logs_fts) VALUES ('rebuild');
-
-      INSERT INTO schema_version (version) VALUES (1);
-    `)
-
-    console.log('[DB] Migration 1 complete: FTS5 trigram tokenizer applied')
-  }
-
-  if (currentVersion < 2) {
-    try {
-      console.log('[DB] Migration 2: Adding emails FTS5...')
-
-      db.exec(`
-        CREATE VIRTUAL TABLE IF NOT EXISTS emails_fts USING fts5(
-          subject,
-          body_text,
-          content=emails,
-          content_rowid=rowid,
-          tokenize='trigram'
-        );
-
-        CREATE TRIGGER IF NOT EXISTS emails_ai AFTER INSERT ON emails BEGIN
-          INSERT INTO emails_fts(rowid, subject, body_text)
-            VALUES (new.rowid, new.subject, new.body_text);
-        END;
-
-        CREATE TRIGGER IF NOT EXISTS emails_ad AFTER DELETE ON emails BEGIN
-          INSERT INTO emails_fts(emails_fts, rowid, subject, body_text)
-            VALUES ('delete', old.rowid, old.subject, old.body_text);
-        END;
-
-        CREATE TRIGGER IF NOT EXISTS emails_au AFTER UPDATE ON emails BEGIN
-          INSERT INTO emails_fts(emails_fts, rowid, subject, body_text)
-            VALUES ('delete', old.rowid, old.subject, old.body_text);
-          INSERT INTO emails_fts(rowid, subject, body_text)
-            VALUES (new.rowid, new.subject, new.body_text);
-        END;
-
-        INSERT INTO emails_fts(emails_fts) VALUES ('rebuild');
-
-        INSERT INTO schema_version (version) VALUES (2);
-      `)
-
-      console.log('[DB] Migration 2 complete: emails FTS5 added')
-    } catch (err) {
-      console.warn(`[DB] Migration 2 skipped (emails table may not exist yet): ${err.message}`)
-      // Mark as applied so it doesn't block subsequent migrations.
-      // emails FTS will be created by initSchema on next fresh DB.
-      try { db.exec("INSERT OR IGNORE INTO schema_version (version) VALUES (2)") } catch {}
-    }
-  }
-
-  if (currentVersion < 3) {
-    try {
-      console.log('[DB] Migration 3: Adding project_id to memories...')
-
-      db.exec(`
-        ALTER TABLE memories ADD COLUMN project_id TEXT DEFAULT NULL;
-
-        CREATE INDEX IF NOT EXISTS idx_memories_project_id ON memories(project_id);
-
-        INSERT INTO schema_version (version) VALUES (3);
-      `)
-
-      console.log('[DB] Migration 3 complete: memories.project_id added')
-    } catch (err) {
-      if (hasColumn(db, 'memories', 'project_id')) {
-        console.warn(`[DB] Migration 3 skipped (column already exists): ${err.message}`)
-        try { db.exec("INSERT OR IGNORE INTO schema_version (version) VALUES (3)") } catch {}
-      } else {
-        console.error(`[DB] Migration 3 FAILED — memories.project_id not added: ${err.message}`)
-      }
-    }
-  }
-
-  if (currentVersion < 4) {
-    try {
-      console.log('[DB] Migration 4: Adding workspace_id to chat_sessions...')
-
-      db.exec(`
-        ALTER TABLE chat_sessions ADD COLUMN workspace_id TEXT DEFAULT NULL;
-
-        CREATE INDEX IF NOT EXISTS idx_chat_sessions_workspace
-          ON chat_sessions(workspace_id, updated_at DESC);
-
-        INSERT INTO schema_version (version) VALUES (4);
-      `)
-
-      console.log('[DB] Migration 4 complete: chat_sessions.workspace_id added')
-    } catch (err) {
-      if (hasColumn(db, 'chat_sessions', 'workspace_id')) {
-        console.warn(`[DB] Migration 4 skipped (column already exists): ${err.message}`)
-        try { db.exec("INSERT OR IGNORE INTO schema_version (version) VALUES (4)") } catch {}
-      } else {
-        console.error(`[DB] Migration 4 FAILED — chat_sessions.workspace_id not added: ${err.message}`)
-      }
-    }
-  }
-
-  if (currentVersion < 5) {
-    try {
-      console.log('[DB] Migration 5: Adding session_id / injection_count to todos...')
-
-      db.exec(`
-        ALTER TABLE todos ADD COLUMN session_id TEXT;
-        ALTER TABLE todos ADD COLUMN injection_count INTEGER NOT NULL DEFAULT 0;
-
-        CREATE INDEX IF NOT EXISTS idx_todos_session ON todos(session_id);
-
-        INSERT INTO schema_version (version) VALUES (5);
-      `)
-
-      console.log('[DB] Migration 5 complete: todos.session_id / injection_count added')
-    } catch (err) {
-      if (hasColumn(db, 'todos', 'session_id') && hasColumn(db, 'todos', 'injection_count')) {
-        console.warn(`[DB] Migration 5 skipped (columns already exist): ${err.message}`)
-        try { db.exec("INSERT OR IGNORE INTO schema_version (version) VALUES (5)") } catch {}
-      } else {
-        console.error(`[DB] Migration 5 FAILED — todos columns not added: ${err.message}`)
-      }
-    }
-  }
-
-  // Repair step: fix DBs where migrations were incorrectly marked as done
-  // but columns were never actually added (caused by the old catch-all error handling)
-  repairMissingColumns(db)
-}
-
-/**
- * Check if a table has a specific column.
- * @param {import('better-sqlite3').Database} db
- * @param {string} table
- * @param {string} column
- * @returns {boolean}
- */
-function hasColumn(db, table, column) {
-  const cols = db.prepare(`PRAGMA table_info(${table})`).all()
-  return cols.some(c => c.name === column)
-}
-
-/**
- * Repair missing columns that should have been added by migrations.
- * Handles DBs where migrations were marked as done but ALTER TABLE actually failed.
- */
-function repairMissingColumns(db) {
-  const repairs = [
-    { table: 'memories', column: 'project_id', sql: 'ALTER TABLE memories ADD COLUMN project_id TEXT DEFAULT NULL' },
-    { table: 'chat_sessions', column: 'workspace_id', sql: 'ALTER TABLE chat_sessions ADD COLUMN workspace_id TEXT DEFAULT NULL' },
-    { table: 'todos', column: 'session_id', sql: 'ALTER TABLE todos ADD COLUMN session_id TEXT' },
-    { table: 'todos', column: 'injection_count', sql: 'ALTER TABLE todos ADD COLUMN injection_count INTEGER NOT NULL DEFAULT 0' },
-  ]
-
-  for (const { table, column, sql } of repairs) {
-    if (!hasColumn(db, table, column)) {
-      try {
-        console.warn(`[DB] Repair: adding missing column ${table}.${column}`)
-        db.exec(sql)
-        console.log(`[DB] Repair: ${table}.${column} added successfully`)
-      } catch (err) {
-        console.error(`[DB] Repair FAILED for ${table}.${column}: ${err.message}`)
-      }
-    }
-  }
-
-  // Ensure indexes exist after repair
-  try {
-    db.exec(`
-      CREATE INDEX IF NOT EXISTS idx_memories_project_id ON memories(project_id);
-      CREATE INDEX IF NOT EXISTS idx_chat_sessions_workspace ON chat_sessions(workspace_id, updated_at DESC);
-      CREATE INDEX IF NOT EXISTS idx_todos_session ON todos(session_id);
-    `)
-  } catch (err) {
-    console.warn(`[DB] Repair index creation skipped: ${err.message}`)
-  }
-}
-
-/**
- * Close the database connection.
- * Call this during graceful shutdown to flush WAL.
- */
-function closeDb() {
-  if (_db) {
-    _db.close()
-    _db = null
-    console.log('[DB] SQLite database closed')
-  }
-}
-
-module.exports = { getDb, closeDb }