@context-vault/core 2.8.3

package/src/core/files.js

@@ -0,0 +1,108 @@
+/**
+ * files.js — Shared file system utilities used across layers
+ *
+ * ULID generation, slugify, kind/dir mapping, directory walking.
+ */
+
+import { readdirSync } from "node:fs";
+import { join, resolve, sep } from "node:path";
+import { categoryDirFor } from "./categories.js";
+
+const CROCKFORD = "0123456789ABCDEFGHJKMNPQRSTVWXYZ";
+
+export function ulid() {
+  const now = Date.now();
+  let ts = "";
+  let t = now;
+  for (let i = 0; i < 10; i++) {
+    ts = CROCKFORD[t & 31] + ts;
+    t = Math.floor(t / 32);
+  }
+  let rand = "";
+  for (let i = 0; i < 16; i++) {
+    rand += CROCKFORD[Math.floor(Math.random() * 32)];
+  }
+  return ts + rand;
+}
+
+export function slugify(text, maxLen = 60) {
+  let slug = text
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, "-")
+    .replace(/^-+|-+$/g, "");
+  if (slug.length > maxLen) {
+    slug =
+      slug.slice(0, maxLen).replace(/-[^-]*$/, "") || slug.slice(0, maxLen);
+  }
+  return slug;
+}
+
+const PLURAL_MAP = {
+  insight: "insights",
+  decision: "decisions",
+  pattern: "patterns",
+  status: "statuses",
+  analysis: "analyses",
+  contact: "contacts",
+  project: "projects",
+  tool: "tools",
+  source: "sources",
+  conversation: "conversations",
+  message: "messages",
+  session: "sessions",
+  log: "logs",
+  feedback: "feedbacks",
+};
+
+const SINGULAR_MAP = Object.fromEntries(
+  Object.entries(PLURAL_MAP).map(([k, v]) => [v, k]),
+);
+
+export function kindToDir(kind) {
+  if (PLURAL_MAP[kind]) return PLURAL_MAP[kind];
+  return kind.endsWith("s") ? kind : kind + "s";
+}
+
+export function dirToKind(dirName) {
+  if (SINGULAR_MAP[dirName]) return SINGULAR_MAP[dirName];
+  return dirName.replace(/s$/, "");
+}
+
+/** Normalize a kind input (singular or plural) to its canonical singular form. */
+export function normalizeKind(input) {
+  if (PLURAL_MAP[input]) return input; // Already a known singular kind
+  if (SINGULAR_MAP[input]) return SINGULAR_MAP[input]; // Known plural → singular
+  return input; // Unknown — use as-is (don't strip 's')
+}
+
+/** Returns relative path from vault root → kind dir: "knowledge/insights", "events/sessions", etc. */
+export function kindToPath(kind) {
+  return `${categoryDirFor(kind)}/${kindToDir(kind)}`;
+}
+
+export function safeJoin(base, ...parts) {
+  const resolvedBase = resolve(base);
+  const result = resolve(join(base, ...parts));
+  if (!result.startsWith(resolvedBase + sep) && result !== resolvedBase) {
+    throw new Error(
+      `Path traversal blocked: resolved path escapes base directory`,
+    );
+  }
+  return result;
+}
+
+export function walkDir(dir) {
+  const results = [];
+  function walk(currentDir, relDir) {
+    for (const entry of readdirSync(currentDir, { withFileTypes: true })) {
+      const fullPath = join(currentDir, entry.name);
+      if (entry.isDirectory() && !entry.name.startsWith("_")) {
+        walk(fullPath, relDir ? join(relDir, entry.name) : entry.name);
+      } else if (entry.isFile() && entry.name.endsWith(".md")) {
+        results.push({ filePath: fullPath, relDir });
+      }
+    }
+  }
+  walk(dir, "");
+  return results;
+}

package/src/core/frontmatter.js

@@ -0,0 +1,120 @@
+/**
+ * frontmatter.js — YAML frontmatter parsing and formatting
+ */
+
+const NEEDS_QUOTING = /[:#'"{}[\],>|&*?!@`]/;
+
+export function formatFrontmatter(meta) {
+  const lines = ["---"];
+  for (const [k, v] of Object.entries(meta)) {
+    if (v === undefined || v === null) continue;
+    if (Array.isArray(v)) {
+      lines.push(`${k}: [${v.map((i) => JSON.stringify(i)).join(", ")}]`);
+    } else {
+      const str = String(v);
+      lines.push(
+        `${k}: ${NEEDS_QUOTING.test(str) ? JSON.stringify(str) : str}`,
+      );
+    }
+  }
+  lines.push("---");
+  return lines.join("\n");
+}
+
+export function parseFrontmatter(text) {
+  const normalized = text.replace(/\r\n/g, "\n");
+  const match = normalized.match(/^---\n([\s\S]*?)\n---\n?([\s\S]*)$/);
+  if (!match) return { meta: {}, body: normalized.trim() };
+  const meta = {};
+  for (const line of match[1].split("\n")) {
+    const idx = line.indexOf(":");
+    if (idx === -1) continue;
+    const key = line.slice(0, idx).trim();
+    let val = line.slice(idx + 1).trim();
+    // Unquote JSON-quoted strings from formatFrontmatter
+    if (
+      val.length >= 2 &&
+      val.startsWith('"') &&
+      val.endsWith('"') &&
+      !val.startsWith('["')
+    ) {
+      try {
+        val = JSON.parse(val);
+      } catch {
+        /* keep as-is */
+      }
+    }
+    // Parse arrays: [a, b, c]
+    if (val.startsWith("[") && val.endsWith("]")) {
+      try {
+        val = JSON.parse(val);
+      } catch {
+        val = val
+          .slice(1, -1)
+          .split(",")
+          .map((s) => s.trim().replace(/^"|"$/g, ""));
+      }
+    }
+    meta[key] = val;
+  }
+  return { meta, body: match[2].trim() };
+}
+
+const RESERVED_FM_KEYS = new Set([
+  "id",
+  "tags",
+  "source",
+  "created",
+  "identity_key",
+  "expires_at",
+]);
+
+export function extractCustomMeta(fmMeta) {
+  const custom = {};
+  for (const [k, v] of Object.entries(fmMeta)) {
+    if (!RESERVED_FM_KEYS.has(k)) custom[k] = v;
+  }
+  return Object.keys(custom).length ? custom : null;
+}
+
+export function parseEntryFromMarkdown(kind, body, fmMeta) {
+  if (kind === "insight") {
+    return {
+      title: null,
+      body,
+      meta: extractCustomMeta(fmMeta),
+    };
+  }
+
+  if (kind === "decision") {
+    const titleMatch = body.match(/^## Decision\s*\n+([\s\S]*?)(?=\n## |\n*$)/);
+    const rationaleMatch = body.match(/## Rationale\s*\n+([\s\S]*?)$/);
+    const title = titleMatch ? titleMatch[1].trim() : body.slice(0, 100);
+    const rationale = rationaleMatch ? rationaleMatch[1].trim() : body;
+    return {
+      title,
+      body: rationale,
+      meta: extractCustomMeta(fmMeta),
+    };
+  }
+
+  if (kind === "pattern") {
+    const titleMatch = body.match(/^# (.+)/);
+    const title = titleMatch ? titleMatch[1].trim() : body.slice(0, 80);
+    const codeMatch = body.match(/```[\w]*\n([\s\S]*?)```/);
+    const content = codeMatch ? codeMatch[1].trim() : body;
+    return {
+      title,
+      body: content,
+      meta: extractCustomMeta(fmMeta),
+    };
+  }
+
+  // Generic: use first heading as title, rest as body
+  const headingMatch = body.match(/^#+ (.+)/);
+  return {
+    title: headingMatch ? headingMatch[1].trim() : null,
+    body,
+    meta: extractCustomMeta(fmMeta),
+  };
+}

package/src/core/status.js

@@ -0,0 +1,146 @@
+/**
+ * status.js — Vault status/diagnostics data gathering
+ */
+
+import { existsSync, readdirSync, statSync } from "node:fs";
+import { join } from "node:path";
+import { walkDir } from "./files.js";
+import { isEmbedAvailable } from "../index/embed.js";
+
+/**
+ * Gather raw vault status data for formatting by consumers.
+ *
+ * @param {import('../server/types.js').BaseCtx} ctx
+ * @param {{ userId?: string }} opts — optional userId for per-user stats
+ * @returns {{ fileCount, subdirs, kindCounts, dbSize, stalePaths, resolvedFrom, embeddingStatus, errors }}
+ */
+export function gatherVaultStatus(ctx, opts = {}) {
+  const { db, config } = ctx;
+  const { userId } = opts;
+  const errors = [];
+
+  // Build user filter clause for DB queries
+  const hasUser = userId !== undefined;
+  const userWhere = hasUser ? "WHERE user_id = ?" : "";
+  const userAnd = hasUser ? "AND user_id = ?" : "";
+  const userParams = hasUser ? [userId] : [];
+
+  // Count files in vault subdirs (auto-discover)
+  let fileCount = 0;
+  const subdirs = [];
+  try {
+    if (existsSync(config.vaultDir)) {
+      for (const d of readdirSync(config.vaultDir, { withFileTypes: true })) {
+        if (d.isDirectory()) {
+          const dir = join(config.vaultDir, d.name);
+          const count = walkDir(dir).length;
+          fileCount += count;
+          if (count > 0) subdirs.push({ name: d.name, count });
+        }
+      }
+    }
+  } catch (e) {
+    errors.push(`File scan failed: ${e.message}`);
+  }
+
+  // Count DB rows by kind
+  let kindCounts = [];
+  try {
+    kindCounts = db
+      .prepare(
+        `SELECT kind, COUNT(*) as c FROM vault ${userWhere} GROUP BY kind`,
+      )
+      .all(...userParams);
+  } catch (e) {
+    errors.push(`Kind count query failed: ${e.message}`);
+  }
+
+  // Count DB rows by category
+  let categoryCounts = [];
+  try {
+    categoryCounts = db
+      .prepare(
+        `SELECT category, COUNT(*) as c FROM vault ${userWhere} GROUP BY category`,
+      )
+      .all(...userParams);
+  } catch (e) {
+    errors.push(`Category count query failed: ${e.message}`);
+  }
+
+  // DB file size
+  let dbSize = "n/a";
+  let dbSizeBytes = 0;
+  try {
+    if (existsSync(config.dbPath)) {
+      dbSizeBytes = statSync(config.dbPath).size;
+      dbSize =
+        dbSizeBytes > 1024 * 1024
+          ? `${(dbSizeBytes / 1024 / 1024).toFixed(1)}MB`
+          : `${(dbSizeBytes / 1024).toFixed(1)}KB`;
+    }
+  } catch (e) {
+    errors.push(`DB size check failed: ${e.message}`);
+  }
+
+  // Check for stale paths (count all mismatches, not just a sample)
+  let stalePaths = false;
+  let staleCount = 0;
+  try {
+    const result = db
+      .prepare(
+        `SELECT COUNT(*) as c FROM vault WHERE file_path NOT LIKE ? || '%' ${userAnd}`,
+      )
+      .get(config.vaultDir, ...userParams);
+    staleCount = result.c;
+    stalePaths = staleCount > 0;
+  } catch (e) {
+    errors.push(`Stale path check failed: ${e.message}`);
+  }
+
+  // Count expired entries pending pruning
+  let expiredCount = 0;
+  try {
+    expiredCount = db
+      .prepare(
+        `SELECT COUNT(*) as c FROM vault WHERE expires_at IS NOT NULL AND expires_at <= datetime('now') ${userAnd}`,
+      )
+      .get(...userParams).c;
+  } catch (e) {
+    errors.push(`Expired count failed: ${e.message}`);
+  }
+
+  // Embedding/vector status
+  let embeddingStatus = null;
+  try {
+    const total = db
+      .prepare(`SELECT COUNT(*) as c FROM vault ${userWhere}`)
+      .get(...userParams).c;
+    const indexed = db
+      .prepare(
+        `SELECT COUNT(*) as c FROM vault WHERE rowid IN (SELECT rowid FROM vault_vec) ${userAnd}`,
+      )
+      .get(...userParams).c;
+    embeddingStatus = { indexed, total, missing: total - indexed };
+  } catch (e) {
+    errors.push(`Embedding status check failed: ${e.message}`);
+  }
+
+  // Embedding model availability
+  const embedModelAvailable = isEmbedAvailable();
+
+  return {
+    fileCount,
+    subdirs,
+    kindCounts,
+    categoryCounts,
+    dbSize,
+    dbSizeBytes,
+    stalePaths,
+    staleCount,
+    expiredCount,
+    embeddingStatus,
+    embedModelAvailable,
+    resolvedFrom: config.resolvedFrom,
+    errors,
+  };
+}

package/src/index/db.js

@@ -0,0 +1,268 @@
+import { unlinkSync, copyFileSync, existsSync } from "node:fs";
+
+export class NativeModuleError extends Error {
+  constructor(originalError) {
+    const diagnostic = formatNativeModuleError(originalError);
+    super(diagnostic);
+    this.name = "NativeModuleError";
+    this.originalError = originalError;
+  }
+}
+
+function formatNativeModuleError(err) {
+  const msg = err.message || "";
+  const versionMatch = msg.match(
+    /was compiled against a different Node\.js version using\s+NODE_MODULE_VERSION (\d+)\. This version of Node\.js requires\s+NODE_MODULE_VERSION (\d+)/,
+  );
+
+  const lines = [
+    `Native module failed to load: ${msg}`,
+    "",
+    `  Running Node.js: ${process.version} (${process.execPath})`,
+  ];
+
+  if (versionMatch) {
+    lines.push(`  Module compiled for: NODE_MODULE_VERSION ${versionMatch[1]}`);
+    lines.push(`  Current runtime:     NODE_MODULE_VERSION ${versionMatch[2]}`);
+  }
+
+  lines.push(
+    "",
+    "  Fix: Rebuild native modules for your current Node.js:",
+    "    npm rebuild better-sqlite3 sqlite-vec",
+    "",
+    "  Or reinstall:",
+    "    npm install -g context-vault",
+  );
+
+  return lines.join("\n");
+}
+
+let _Database = null;
+let _sqliteVec = null;
+
+async function loadNativeModules() {
+  if (_Database && _sqliteVec)
+    return { Database: _Database, sqliteVec: _sqliteVec };
+
+  try {
+    const dbMod = await import("better-sqlite3");
+    _Database = dbMod.default;
+  } catch (e) {
+    throw new NativeModuleError(e);
+  }
+
+  try {
+    const vecMod = await import("sqlite-vec");
+    _sqliteVec = vecMod;
+  } catch (e) {
+    throw new NativeModuleError(e);
+  }
+
+  return { Database: _Database, sqliteVec: _sqliteVec };
+}
+
+export const SCHEMA_DDL = `
+  CREATE TABLE IF NOT EXISTS vault (
+    id              TEXT PRIMARY KEY,
+    kind            TEXT NOT NULL,
+    category        TEXT NOT NULL DEFAULT 'knowledge',
+    title           TEXT,
+    body            TEXT NOT NULL,
+    meta            TEXT,
+    tags            TEXT,
+    source          TEXT,
+    file_path       TEXT UNIQUE,
+    identity_key    TEXT,
+    expires_at      TEXT,
+    created_at      TEXT DEFAULT (datetime('now')),
+    user_id         TEXT,
+    team_id         TEXT,
+    body_encrypted  BLOB,
+    title_encrypted BLOB,
+    meta_encrypted  BLOB,
+    iv              BLOB
+  );
+
+  CREATE INDEX IF NOT EXISTS idx_vault_kind ON vault(kind);
+  CREATE INDEX IF NOT EXISTS idx_vault_category ON vault(category);
+  CREATE INDEX IF NOT EXISTS idx_vault_category_created ON vault(category, created_at DESC);
+  CREATE INDEX IF NOT EXISTS idx_vault_user ON vault(user_id);
+  CREATE INDEX IF NOT EXISTS idx_vault_team ON vault(team_id);
+  CREATE UNIQUE INDEX IF NOT EXISTS idx_vault_identity ON vault(user_id, kind, identity_key) WHERE identity_key IS NOT NULL;
+
+  -- Single FTS5 table
+  CREATE VIRTUAL TABLE IF NOT EXISTS vault_fts USING fts5(
+    title, body, tags, kind,
+    content='vault', content_rowid='rowid'
+  );
+
+  -- FTS sync triggers
+  CREATE TRIGGER IF NOT EXISTS vault_ai AFTER INSERT ON vault BEGIN
+    INSERT INTO vault_fts(rowid, title, body, tags, kind)
+      VALUES (new.rowid, new.title, new.body, new.tags, new.kind);
+  END;
+  CREATE TRIGGER IF NOT EXISTS vault_ad AFTER DELETE ON vault BEGIN
+    INSERT INTO vault_fts(vault_fts, rowid, title, body, tags, kind)
+      VALUES ('delete', old.rowid, old.title, old.body, old.tags, old.kind);
+  END;
+  CREATE TRIGGER IF NOT EXISTS vault_au AFTER UPDATE ON vault BEGIN
+    INSERT INTO vault_fts(vault_fts, rowid, title, body, tags, kind)
+      VALUES ('delete', old.rowid, old.title, old.body, old.tags, old.kind);
+    INSERT INTO vault_fts(rowid, title, body, tags, kind)
+      VALUES (new.rowid, new.title, new.body, new.tags, new.kind);
+  END;
+
+  -- Single vec table (384-dim float32 for all-MiniLM-L6-v2)
+  CREATE VIRTUAL TABLE IF NOT EXISTS vault_vec USING vec0(embedding float[384]);
+`;
+
+export async function initDatabase(dbPath) {
+  const { Database, sqliteVec } = await loadNativeModules();
+
+  function createDb(path) {
+    const db = new Database(path);
+    db.pragma("journal_mode = WAL");
+    db.pragma("foreign_keys = ON");
+    try {
+      sqliteVec.load(db);
+    } catch (e) {
+      throw new NativeModuleError(e);
+    }
+    return db;
+  }
+
+  const db = createDb(dbPath);
+  const version = db.pragma("user_version", { simple: true });
+
+  // Enforce fresh-DB-only — old schemas get a full rebuild (with backup)
+  if (version > 0 && version < 5) {
+    console.error(
+      `[context-vault] Schema v${version} is outdated. Rebuilding database...`,
+    );
+
+    // Backup old DB before destroying it
+    const backupPath = `${dbPath}.v${version}.backup`;
+    try {
+      db.close();
+      if (existsSync(dbPath)) {
+        copyFileSync(dbPath, backupPath);
+        console.error(
+          `[context-vault] Backed up old database to: ${backupPath}`,
+        );
+      }
+    } catch (backupErr) {
+      console.error(
+        `[context-vault] Warning: could not backup old database: ${backupErr.message}`,
+      );
+    }
+
+    unlinkSync(dbPath);
+    try {
+      unlinkSync(dbPath + "-wal");
+    } catch {}
+    try {
+      unlinkSync(dbPath + "-shm");
+    } catch {}
+
+    const freshDb = createDb(dbPath);
+    freshDb.exec(SCHEMA_DDL);
+    freshDb.pragma("user_version = 7");
+    return freshDb;
+  }
+
+  if (version < 5) {
+    db.exec(SCHEMA_DDL);
+    db.pragma("user_version = 7");
+  } else if (version === 5) {
+    // v5 -> v6 migration: add multi-tenancy + encryption columns
+    // Wrapped in transaction with duplicate-column guards for idempotent retry
+    const migrate = db.transaction(() => {
+      const addColumnSafe = (sql) => {
+        try {
+          db.exec(sql);
+        } catch (e) {
+          if (!e.message.includes("duplicate column")) throw e;
+        }
+      };
+      addColumnSafe(`ALTER TABLE vault ADD COLUMN user_id TEXT`);
+      addColumnSafe(`ALTER TABLE vault ADD COLUMN body_encrypted BLOB`);
+      addColumnSafe(`ALTER TABLE vault ADD COLUMN title_encrypted BLOB`);
+      addColumnSafe(`ALTER TABLE vault ADD COLUMN meta_encrypted BLOB`);
+      addColumnSafe(`ALTER TABLE vault ADD COLUMN iv BLOB`);
+      addColumnSafe(`ALTER TABLE vault ADD COLUMN team_id TEXT`);
+      db.exec(`CREATE INDEX IF NOT EXISTS idx_vault_user ON vault(user_id)`);
+      db.exec(`CREATE INDEX IF NOT EXISTS idx_vault_team ON vault(team_id)`);
+      db.exec(`DROP INDEX IF EXISTS idx_vault_identity`);
+      db.exec(
+        `CREATE UNIQUE INDEX IF NOT EXISTS idx_vault_identity ON vault(user_id, kind, identity_key) WHERE identity_key IS NOT NULL`,
+      );
+      db.pragma("user_version = 7");
+    });
+    migrate();
+  } else if (version === 6) {
+    // v6 -> v7 migration: add team_id column
+    const migrate = db.transaction(() => {
+      try {
+        db.exec(`ALTER TABLE vault ADD COLUMN team_id TEXT`);
+      } catch (e) {
+        if (!e.message.includes("duplicate column")) throw e;
+      }
+      db.exec(`CREATE INDEX IF NOT EXISTS idx_vault_team ON vault(team_id)`);
+      db.pragma("user_version = 7");
+    });
+    migrate();
+  }
+
+  return db;
+}
+
+export function prepareStatements(db) {
+  try {
+    return {
+      insertEntry: db.prepare(
+        `INSERT INTO vault (id, user_id, kind, category, title, body, meta, tags, source, file_path, identity_key, expires_at, created_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+      ),
+      insertEntryEncrypted: db.prepare(
+        `INSERT INTO vault (id, user_id, kind, category, title, body, meta, tags, source, file_path, identity_key, expires_at, created_at, body_encrypted, title_encrypted, meta_encrypted, iv) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+      ),
+      updateEntry: db.prepare(
+        `UPDATE vault SET title = ?, body = ?, meta = ?, tags = ?, source = ?, category = ?, identity_key = ?, expires_at = ? WHERE file_path = ?`,
+      ),
+      deleteEntry: db.prepare(`DELETE FROM vault WHERE id = ?`),
+      getRowid: db.prepare(`SELECT rowid FROM vault WHERE id = ?`),
+      getRowidByPath: db.prepare(`SELECT rowid FROM vault WHERE file_path = ?`),
+      getEntryById: db.prepare(`SELECT * FROM vault WHERE id = ?`),
+      getByIdentityKey: db.prepare(
+        `SELECT * FROM vault WHERE kind = ? AND identity_key = ? AND user_id IS ?`,
+      ),
+      upsertByIdentityKey: db.prepare(
+        `UPDATE vault SET title = ?, body = ?, meta = ?, tags = ?, source = ?, category = ?, file_path = ?, expires_at = ? WHERE kind = ? AND identity_key = ? AND user_id IS ?`,
+      ),
+      insertVecStmt: db.prepare(
+        `INSERT INTO vault_vec (rowid, embedding) VALUES (?, ?)`,
+      ),
+      deleteVecStmt: db.prepare(`DELETE FROM vault_vec WHERE rowid = ?`),
+    };
+  } catch (e) {
+    throw new Error(
+      `Failed to prepare database statements. The database may be corrupted.\n` +
+        `Try deleting and rebuilding: rm "${db.name}" && context-vault reindex\n` +
+        `Original error: ${e.message}`,
+    );
+  }
+}
+
+export function insertVec(stmts, rowid, embedding) {
+  // sqlite-vec requires BigInt for primary key — better-sqlite3 binds Number as REAL,
+  // but vec0 virtual tables only accept INTEGER rowids
+  const safeRowid = BigInt(rowid);
+  if (safeRowid < 1n) throw new Error(`Invalid rowid: ${rowid}`);
+  stmts.insertVecStmt.run(safeRowid, embedding);
+}
+
+export function deleteVec(stmts, rowid) {
+  const safeRowid = BigInt(rowid);
+  if (safeRowid < 1n) throw new Error(`Invalid rowid: ${rowid}`);
+  stmts.deleteVecStmt.run(safeRowid);
+}