@context-vault/core 2.17.0 → 3.0.0

package/src/db.ts

@@ -0,0 +1,229 @@
+import { unlinkSync, copyFileSync, existsSync } from "node:fs";
+import { DatabaseSync } from "node:sqlite";
+import type { PreparedStatements } from "./types.js";
+
+export class NativeModuleError extends Error {
+  originalError: Error;
+  constructor(originalError: Error) {
+    const diagnostic = formatNativeModuleError(originalError);
+    super(diagnostic);
+    this.name = "NativeModuleError";
+    this.originalError = originalError;
+  }
+}
+
+function formatNativeModuleError(err: Error): string {
+  const msg = err.message || "";
+  return [
+    `sqlite-vec extension failed to load: ${msg}`,
+    "",
+    `  Running Node.js: ${process.version} (${process.execPath})`,
+    "",
+    "  Fix: Reinstall context-vault:",
+    "    npx -y context-vault@latest setup",
+  ].join("\n");
+}
+
+let _sqliteVec: { load: (db: DatabaseSync) => void } | null = null;
+
+async function loadSqliteVec() {
+  if (_sqliteVec) return _sqliteVec;
+  const vecMod = await import("sqlite-vec");
+  _sqliteVec = vecMod;
+  return _sqliteVec;
+}
+
+function runTransaction(db: DatabaseSync, fn: () => void): void {
+  db.exec("BEGIN");
+  try {
+    fn();
+    db.exec("COMMIT");
+  } catch (e) {
+    db.exec("ROLLBACK");
+    throw e;
+  }
+}
+
+export const SCHEMA_DDL = `
+  CREATE TABLE IF NOT EXISTS vault (
+    id              TEXT PRIMARY KEY,
+    kind            TEXT NOT NULL,
+    category        TEXT NOT NULL DEFAULT 'knowledge',
+    title           TEXT,
+    body            TEXT NOT NULL,
+    meta            TEXT,
+    tags            TEXT,
+    source          TEXT,
+    file_path       TEXT UNIQUE,
+    identity_key    TEXT,
+    expires_at      TEXT,
+    superseded_by   TEXT,
+    created_at      TEXT DEFAULT (datetime('now')),
+    updated_at      TEXT,
+    hit_count       INTEGER DEFAULT 0,
+    last_accessed_at TEXT,
+    source_files    TEXT,
+    tier            TEXT DEFAULT 'working' CHECK(tier IN ('ephemeral', 'working', 'durable')),
+    related_to      TEXT
+  );
+
+  CREATE INDEX IF NOT EXISTS idx_vault_kind ON vault(kind);
+  CREATE INDEX IF NOT EXISTS idx_vault_category ON vault(category);
+  CREATE INDEX IF NOT EXISTS idx_vault_category_created ON vault(category, created_at DESC);
+  CREATE INDEX IF NOT EXISTS idx_vault_updated ON vault(updated_at DESC);
+  CREATE UNIQUE INDEX IF NOT EXISTS idx_vault_identity ON vault(kind, identity_key) WHERE identity_key IS NOT NULL AND category = 'entity';
+  CREATE INDEX IF NOT EXISTS idx_vault_superseded ON vault(superseded_by) WHERE superseded_by IS NOT NULL;
+  CREATE INDEX IF NOT EXISTS idx_vault_tier ON vault(tier);
+
+  CREATE VIRTUAL TABLE IF NOT EXISTS vault_fts USING fts5(
+    title, body, tags, kind,
+    content='vault', content_rowid='rowid'
+  );
+
+  CREATE TRIGGER IF NOT EXISTS vault_ai AFTER INSERT ON vault BEGIN
+    INSERT INTO vault_fts(rowid, title, body, tags, kind)
+      VALUES (new.rowid, new.title, new.body, new.tags, new.kind);
+  END;
+  CREATE TRIGGER IF NOT EXISTS vault_ad AFTER DELETE ON vault BEGIN
+    INSERT INTO vault_fts(vault_fts, rowid, title, body, tags, kind)
+      VALUES ('delete', old.rowid, old.title, old.body, old.tags, old.kind);
+  END;
+  CREATE TRIGGER IF NOT EXISTS vault_au AFTER UPDATE ON vault BEGIN
+    INSERT INTO vault_fts(vault_fts, rowid, title, body, tags, kind)
+      VALUES ('delete', old.rowid, old.title, old.body, old.tags, old.kind);
+    INSERT INTO vault_fts(rowid, title, body, tags, kind)
+      VALUES (new.rowid, new.title, new.body, new.tags, new.kind);
+  END;
+
+  CREATE VIRTUAL TABLE IF NOT EXISTS vault_vec USING vec0(embedding float[384]);
+`;
+
+const CURRENT_VERSION = 15;
+
+export async function initDatabase(dbPath: string): Promise<DatabaseSync> {
+  const sqliteVec = await loadSqliteVec();
+
+  function createDb(path: string): DatabaseSync {
+    const db = new DatabaseSync(path, { allowExtension: true });
+    db.exec("PRAGMA journal_mode = WAL");
+    db.exec("PRAGMA foreign_keys = ON");
+    try {
+      sqliteVec.load(db);
+    } catch (e) {
+      throw new NativeModuleError(e as Error);
+    }
+    return db;
+  }
+
+  const db = createDb(dbPath);
+  const version = (db.prepare("PRAGMA user_version").get() as { user_version: number }).user_version;
+
+  if (version > 0 && version < 15) {
+    console.error(
+      `[context-vault] Schema v${version} is outdated. Rebuilding database...`,
+    );
+
+    const backupPath = `${dbPath}.v${version}.backup`;
+    try {
+      db.close();
+      if (existsSync(dbPath)) {
+        copyFileSync(dbPath, backupPath);
+        console.error(
+          `[context-vault] Backed up old database to: ${backupPath}`,
+        );
+      }
+    } catch (backupErr) {
+      console.error(
+        `[context-vault] Warning: could not backup old database: ${(backupErr as Error).message}`,
+      );
+    }
+
+    unlinkSync(dbPath);
+    try { unlinkSync(dbPath + "-wal"); } catch {}
+    try { unlinkSync(dbPath + "-shm"); } catch {}
+
+    const freshDb = createDb(dbPath);
+    freshDb.exec(SCHEMA_DDL);
+    freshDb.exec(`PRAGMA user_version = ${CURRENT_VERSION}`);
+    return freshDb;
+  }
+
+  if (version < 15) {
+    db.exec(SCHEMA_DDL);
+    db.exec(`PRAGMA user_version = ${CURRENT_VERSION}`);
+  }
+
+  return db;
+}
+
+export function prepareStatements(db: DatabaseSync): PreparedStatements {
+  try {
+    return {
+      insertEntry: db.prepare(
+        `INSERT INTO vault (id, kind, category, title, body, meta, tags, source, file_path, identity_key, expires_at, created_at, updated_at, source_files, tier) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+      ),
+      updateEntry: db.prepare(
+        `UPDATE vault SET title = ?, body = ?, meta = ?, tags = ?, source = ?, category = ?, identity_key = ?, expires_at = ?, updated_at = datetime('now') WHERE file_path = ?`,
+      ),
+      deleteEntry: db.prepare(`DELETE FROM vault WHERE id = ?`),
+      getRowid: db.prepare(`SELECT rowid FROM vault WHERE id = ?`),
+      getRowidByPath: db.prepare(
+        `SELECT rowid FROM vault WHERE file_path = ?`,
+      ),
+      getEntryById: db.prepare(`SELECT * FROM vault WHERE id = ?`),
+      getByIdentityKey: db.prepare(
+        `SELECT * FROM vault WHERE kind = ? AND identity_key = ?`,
+      ),
+      upsertByIdentityKey: db.prepare(
+        `UPDATE vault SET title = ?, body = ?, meta = ?, tags = ?, source = ?, category = ?, file_path = ?, expires_at = ?, source_files = ?, updated_at = datetime('now') WHERE kind = ? AND identity_key = ?`,
+      ),
+      updateSourceFiles: db.prepare(
+        `UPDATE vault SET source_files = ? WHERE id = ?`,
+      ),
+      updateRelatedTo: db.prepare(
+        `UPDATE vault SET related_to = ? WHERE id = ?`,
+      ),
+      insertVecStmt: db.prepare(
+        `INSERT INTO vault_vec (rowid, embedding) VALUES (?, ?)`,
+      ),
+      deleteVecStmt: db.prepare(`DELETE FROM vault_vec WHERE rowid = ?`),
+      updateSupersededBy: db.prepare(
+        `UPDATE vault SET superseded_by = ? WHERE id = ?`,
+      ),
+      clearSupersededByRef: db.prepare(
+        `UPDATE vault SET superseded_by = NULL WHERE superseded_by = ?`,
+      ),
+    };
+  } catch (e) {
+    throw new Error(
+      `Failed to prepare database statements. The database may be corrupted.\n` +
+        `Try deleting and rebuilding: context-vault reindex\n` +
+        `Original error: ${(e as Error).message}`,
+    );
+  }
+}
+
+export function insertVec(
+  stmts: PreparedStatements,
+  rowid: number,
+  embedding: Float32Array,
+): void {
+  const safeRowid = BigInt(rowid);
+  if (safeRowid < 1n) throw new Error(`Invalid rowid: ${rowid}`);
+  stmts.insertVecStmt.run(safeRowid, embedding);
+}
+
+export function deleteVec(stmts: PreparedStatements, rowid: number): void {
+  const safeRowid = BigInt(rowid);
+  if (safeRowid < 1n) throw new Error(`Invalid rowid: ${rowid}`);
+  stmts.deleteVecStmt.run(safeRowid);
+}
+
+export function testConnection(db: DatabaseSync): boolean {
+  try {
+    db.prepare("SELECT 1").get();
+    return true;
+  } catch {
+    return false;
+  }
+}

package/src/{index/embed.js → embed.ts}

@@ -1,36 +1,21 @@
-/**
- * embed.js — Text embedding via HuggingFace transformers
- *
- * Graceful degradation: if the embedding model fails to load (offline, first run,
- * disk issues), semantic search is disabled but FTS still works.
- */
-
 import { join } from "node:path";
 import { homedir } from "node:os";
 import { mkdirSync } from "node:fs";
 
-let extractor = null;
-
-/** @type {null | true | false} null = uninitialized/retry, true = ready, false = permanently failed */
-let embedAvailable = null;
-
-/** In-flight load promise — coalesces concurrent callers onto a single pipeline() call */
-let loadingPromise = null;
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+let extractor: any = null;
+let embedAvailable: boolean | null = null;
+let loadingPromise: Promise<typeof extractor> | null = null;
 
-async function ensurePipeline() {
+async function ensurePipeline(): Promise<typeof extractor> {
   if (embedAvailable === false) return null;
   if (extractor) return extractor;
   if (loadingPromise) return loadingPromise;
 
   loadingPromise = (async () => {
     try {
-      // Dynamic import — @huggingface/transformers is optional (its transitive
-      // dep `sharp` can fail to install on some platforms).  When missing, the
-      // server still works with full-text search only.
       const { pipeline, env } = await import("@huggingface/transformers");
 
-      // Redirect model cache to ~/.context-mcp/models/ so it works when the
-      // package is installed globally in a root-owned directory (e.g. /usr/lib/node_modules/).
       const modelCacheDir = join(homedir(), ".context-mcp", "models");
       mkdirSync(modelCacheDir, { recursive: true });
       env.cacheDir = modelCacheDir;
@@ -47,7 +32,7 @@ async function ensurePipeline() {
     } catch (e) {
       embedAvailable = false;
       console.error(
-        `[context-vault] Failed to load embedding model: ${e.message}`,
+        `[context-vault] Failed to load embedding model: ${(e as Error).message}`,
       );
       console.error(
         `[context-vault] Semantic search disabled. Full-text search still works.`,
@@ -61,12 +46,11 @@ async function ensurePipeline() {
   return loadingPromise;
 }
 
-export async function embed(text) {
+export async function embed(text: string): Promise<Float32Array | null> {
   const ext = await ensurePipeline();
   if (!ext) return null;
 
   const result = await ext([text], { pooling: "mean", normalize: true });
-  // Health check — force re-init on empty results
   if (!result?.data?.length) {
     extractor = null;
     embedAvailable = null;
@@ -76,12 +60,7 @@ export async function embed(text) {
   return new Float32Array(result.data);
 }
 
-/**
- * Batch embedding — embed multiple texts in a single pipeline call.
- * Returns an array of Float32Array embeddings (one per input text).
- * Returns array of nulls if embedding is unavailable.
- */
-export async function embedBatch(texts) {
+export async function embedBatch(texts: string[]): Promise<(Float32Array | null)[]> {
   if (!texts.length) return [];
   const ext = await ensurePipeline();
   if (!ext) return texts.map(() => null);
@@ -99,19 +78,15 @@ export async function embedBatch(texts) {
       `Unexpected embedding dimension: ${result.data.length} / ${texts.length} = ${dim}`,
     );
   }
-  // subarray() creates a view into result.data's index-space, correctly
-  // accounting for any non-zero byteOffset on the source typed array.
   return texts.map((_, i) => result.data.subarray(i * dim, (i + 1) * dim));
 }
 
-/** Force re-initialization on next embed call. */
-export function resetEmbedPipeline() {
+export function resetEmbedPipeline(): void {
   extractor = null;
   embedAvailable = null;
   loadingPromise = null;
 }
 
-/** Check if embedding is currently available. */
-export function isEmbedAvailable() {
+export function isEmbedAvailable(): boolean | null {
   return embedAvailable;
 }

package/src/files.ts

@@ -0,0 +1,80 @@
+import { readdirSync } from "node:fs";
+import { join, resolve, sep } from "node:path";
+import { categoryDirFor } from "./categories.js";
+
+const CROCKFORD = "0123456789ABCDEFGHJKMNPQRSTVWXYZ";
+
+export function ulid(): string {
+  const now = Date.now();
+  let ts = "";
+  let t = now;
+  for (let i = 0; i < 10; i++) {
+    ts = CROCKFORD[t & 31] + ts;
+    t = Math.floor(t / 32);
+  }
+  let rand = "";
+  for (let i = 0; i < 16; i++) {
+    rand += CROCKFORD[Math.floor(Math.random() * 32)];
+  }
+  return ts + rand;
+}
+
+export function slugify(text: string, maxLen = 60): string {
+  let slug = text
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, "-")
+    .replace(/^-+|-+$/g, "");
+  if (slug.length > maxLen) {
+    slug =
+      slug.slice(0, maxLen).replace(/-[^-]*$/, "") || slug.slice(0, maxLen);
+  }
+  return slug;
+}
+
+export function kindToDir(kind: string): string {
+  return kind;
+}
+
+export function dirToKind(dirName: string): string {
+  return dirName;
+}
+
+export function normalizeKind(input: string): string {
+  return input;
+}
+
+export function kindToPath(kind: string): string {
+  return `${categoryDirFor(kind)}/${kindToDir(kind)}`;
+}
+
+export function safeJoin(base: string, ...parts: string[]): string {
+  const resolvedBase = resolve(base);
+  const result = resolve(join(base, ...parts));
+  if (!result.startsWith(resolvedBase + sep) && result !== resolvedBase) {
+    throw new Error(
+      `Path traversal blocked: resolved path escapes base directory`,
+    );
+  }
+  return result;
+}
+
+export interface WalkResult {
+  filePath: string;
+  relDir: string;
+}
+
+export function walkDir(dir: string): WalkResult[] {
+  const results: WalkResult[] = [];
+  function walk(currentDir: string, relDir: string) {
+    for (const entry of readdirSync(currentDir, { withFileTypes: true })) {
+      const fullPath = join(currentDir, entry.name);
+      if (entry.isDirectory() && !entry.name.startsWith("_")) {
+        walk(fullPath, relDir ? join(relDir, entry.name) : entry.name);
+      } else if (entry.isFile() && entry.name.endsWith(".md")) {
+        results.push({ filePath: fullPath, relDir });
+      }
+    }
+  }
+  walk(dir, "");
+  return results;
+}

package/src/{capture/formatters.js → formatters.ts}

@@ -1,11 +1,10 @@
-/**
- * formatters.js — Kind-specific markdown body templates
- *
- * Maps entry kinds to their markdown body format.
- * Default formatter used for unknown kinds.
- */
+interface FormatInput {
+  title?: string | null;
+  body: string;
+  meta?: Record<string, unknown>;
+}
 
-const FORMATTERS = {
+const FORMATTERS: Record<string, (input: FormatInput) => string> = {
   insight: ({ body }) => "\n" + body + "\n",
 
   decision: ({ title, body }) => {
@@ -15,15 +14,18 @@ const FORMATTERS = {
 
   pattern: ({ title, body, meta }) => {
     const t = title || body.slice(0, 80);
-    const lang = meta?.language || "";
+    const lang = (meta?.language as string) || "";
     return "\n# " + t + "\n\n```" + lang + "\n" + body + "\n```\n";
   },
 };
 
-const DEFAULT_FORMATTER = ({ title, body }) =>
+const DEFAULT_FORMATTER = ({ title, body }: FormatInput): string =>
   title ? "\n# " + title + "\n\n" + body + "\n" : "\n" + body + "\n";
 
-export function formatBody(kind, { title, body, meta }) {
+export function formatBody(
+  kind: string,
+  input: FormatInput,
+): string {
   const fn = FORMATTERS[kind] || DEFAULT_FORMATTER;
-  return fn({ title, body, meta });
+  return fn(input);
 }

package/src/{core/frontmatter.js → frontmatter.ts}

@@ -1,10 +1,6 @@
-/**
- * frontmatter.js — YAML frontmatter parsing and formatting
- */
-
 const NEEDS_QUOTING = /[:#'"{}[\],>|&*?!@`]/;
 
-export function formatFrontmatter(meta) {
+export function formatFrontmatter(meta: Record<string, unknown>): string {
   const lines = ["---"];
   for (const [k, v] of Object.entries(meta)) {
     if (v === undefined || v === null) continue;
@@ -21,18 +17,21 @@ export function formatFrontmatter(meta) {
   return lines.join("\n");
 }
 
-export function parseFrontmatter(text) {
+export function parseFrontmatter(text: string): {
+  meta: Record<string, unknown>;
+  body: string;
+} {
   const normalized = text.replace(/\r\n/g, "\n");
   const match = normalized.match(/^---\n([\s\S]*?)\n---\n?([\s\S]*)$/);
   if (!match) return { meta: {}, body: normalized.trim() };
-  const meta = {};
+  const meta: Record<string, unknown> = {};
   for (const line of match[1].split("\n")) {
     const idx = line.indexOf(":");
     if (idx === -1) continue;
     const key = line.slice(0, idx).trim();
-    let val = line.slice(idx + 1).trim();
-    // Unquote JSON-quoted strings from formatFrontmatter
+    let val: unknown = line.slice(idx + 1).trim() as string;
     if (
+      typeof val === "string" &&
       val.length >= 2 &&
       val.startsWith('"') &&
       val.endsWith('"') &&
@@ -44,15 +43,14 @@ export function parseFrontmatter(text) {
         /* keep as-is */
       }
     }
-    // Parse arrays: [a, b, c]
-    if (val.startsWith("[") && val.endsWith("]")) {
+    if (typeof val === "string" && val.startsWith("[") && val.endsWith("]")) {
       try {
         val = JSON.parse(val);
       } catch {
-        val = val
+        val = (val as string)
           .slice(1, -1)
           .split(",")
-          .map((s) => s.trim().replace(/^"|"$/g, ""));
+          .map((s: string) => s.trim().replace(/^"|"$/g, ""));
       }
     }
     meta[key] = val;
@@ -69,35 +67,36 @@ const RESERVED_FM_KEYS = new Set([
   "identity_key",
   "expires_at",
   "supersedes",
+  "related_to",
 ]);
 
-export function extractCustomMeta(fmMeta) {
-  const custom = {};
+export function extractCustomMeta(
+  fmMeta: Record<string, unknown>,
+): Record<string, unknown> | null {
+  const custom: Record<string, unknown> = {};
   for (const [k, v] of Object.entries(fmMeta)) {
     if (!RESERVED_FM_KEYS.has(k)) custom[k] = v;
   }
   return Object.keys(custom).length ? custom : null;
 }
 
-export function parseEntryFromMarkdown(kind, body, fmMeta) {
+export function parseEntryFromMarkdown(
+  kind: string,
+  body: string,
+  fmMeta: Record<string, unknown>,
+): { title: string | null; body: string; meta: Record<string, unknown> | null } {
   if (kind === "insight") {
-    return {
-      title: null,
-      body,
-      meta: extractCustomMeta(fmMeta),
-    };
+    return { title: null, body, meta: extractCustomMeta(fmMeta) };
   }
 
   if (kind === "decision") {
-    const titleMatch = body.match(/^## Decision\s*\n+([\s\S]*?)(?=\n## |\n*$)/);
+    const titleMatch = body.match(
+      /^## Decision\s*\n+([\s\S]*?)(?=\n## |\n*$)/,
+    );
     const rationaleMatch = body.match(/## Rationale\s*\n+([\s\S]*?)$/);
     const title = titleMatch ? titleMatch[1].trim() : body.slice(0, 100);
     const rationale = rationaleMatch ? rationaleMatch[1].trim() : body;
-    return {
-      title,
-      body: rationale,
-      meta: extractCustomMeta(fmMeta),
-    };
+    return { title, body: rationale, meta: extractCustomMeta(fmMeta) };
   }
 
   if (kind === "pattern") {
@@ -105,14 +104,9 @@ export function parseEntryFromMarkdown(kind, body, fmMeta) {
     const title = titleMatch ? titleMatch[1].trim() : body.slice(0, 80);
     const codeMatch = body.match(/```[\w]*\n([\s\S]*?)```/);
     const content = codeMatch ? codeMatch[1].trim() : body;
-    return {
-      title,
-      body: content,
-      meta: extractCustomMeta(fmMeta),
-    };
+    return { title, body: content, meta: extractCustomMeta(fmMeta) };
   }
 
-  // Generic: use first heading as title, rest as body
   const headingMatch = body.match(/^#+ (.+)/);
   return {
     title: headingMatch ? headingMatch[1].trim() : null,