coding-friend-cli 1.16.0 → 1.17.1

package/lib/cf-memory/src/backends/sqlite/migrations.ts

@@ -0,0 +1,188 @@
+/**
+ * SQLite schema migrations.
+ *
+ * Each migration is a function that takes a Database instance and applies
+ * the necessary schema changes. Migrations are idempotent.
+ */
+
+import {
+  SCHEMA_V1,
+  SCHEMA_VERSION,
+  SCHEMA_V2_METADATA,
+  PRAGMA_SETTINGS,
+  getVecTableSQL,
+} from "./schema.js";
+
+export type DatabaseLike = {
+  // better-sqlite3's exec method — runs SQL, NOT child_process.exec
+  exec(sql: string): void;
+  pragma(pragma: string): unknown;
+  prepare(sql: string): {
+    get(...params: unknown[]): unknown;
+    run(...params: unknown[]): unknown;
+  };
+};
+
+/**
+ * Get the current schema version from the database.
+ * Returns 0 if the schema hasn't been initialized yet.
+ */
+export function getSchemaVersion(db: DatabaseLike): number {
+  try {
+    const row = db
+      .prepare("SELECT version FROM schema_version LIMIT 1")
+      .get() as { version: number } | undefined;
+    return row?.version ?? 0;
+  } catch {
+    // Table doesn't exist yet
+    return 0;
+  }
+}
+
+/**
+ * Apply PRAGMA settings for performance.
+ */
+export function applyPragmas(db: DatabaseLike): void {
+  for (const pragma of PRAGMA_SETTINGS) {
+    db.exec(pragma);
+  }
+}
+
+/**
+ * Initialize the schema from scratch (version 0 → 1).
+ */
+function migrateV0ToV1(db: DatabaseLike): void {
+  for (const sql of SCHEMA_V1) {
+    db.exec(sql);
+  }
+}
+
+/**
+ * Migrate from schema v1 to v2: add metadata table.
+ */
+function migrateV1ToV2(db: DatabaseLike): void {
+  // better-sqlite3 Database.exec — runs SQL, not shell commands
+  db.exec(SCHEMA_V2_METADATA);
+  // Insert default embedding metadata (must match DEFAULT_TRANSFORMERS_MODEL)
+  db.prepare("INSERT OR REPLACE INTO metadata (key, value) VALUES (?, ?)").run(
+    "embedding_model",
+    "Xenova/all-MiniLM-L6-v2",
+  );
+  db.prepare("INSERT OR REPLACE INTO metadata (key, value) VALUES (?, ?)").run(
+    "embedding_dims",
+    "384",
+  );
+  // Update schema version
+  db.prepare("UPDATE schema_version SET version = ?").run(2);
+}
+
+/**
+ * Get a metadata value by key. Returns null if not found.
+ */
+export function getMetadata(db: DatabaseLike, key: string): string | null {
+  const row = db
+    .prepare("SELECT value FROM metadata WHERE key = ?")
+    .get(key) as { value: string } | undefined;
+  return row?.value ?? null;
+}
+
+/**
+ * Set a metadata key-value pair (insert or update).
+ */
+export function setMetadata(
+  db: DatabaseLike,
+  key: string,
+  value: string,
+): void {
+  db.prepare("INSERT OR REPLACE INTO metadata (key, value) VALUES (?, ?)").run(
+    key,
+    value,
+  );
+}
+
+/**
+ * Check if the current embedding model/dims differ from what is stored.
+ * Returns mismatched=false for fresh databases (no metadata yet).
+ */
+export function checkEmbeddingMismatch(
+  db: DatabaseLike,
+  currentModel: string,
+  currentDims: number,
+): {
+  mismatched: boolean;
+  storedModel: string | null;
+  storedDims: number | null;
+  currentModel: string;
+  currentDims: number;
+} {
+  const storedModel = getMetadata(db, "embedding_model");
+  const storedDimsStr = getMetadata(db, "embedding_dims");
+  const storedDims =
+    storedDimsStr !== null ? parseInt(storedDimsStr, 10) : null;
+
+  // Fresh database or partial metadata -- not a mismatch
+  if (storedModel === null || storedDims === null) {
+    return {
+      mismatched: false,
+      storedModel,
+      storedDims,
+      currentModel,
+      currentDims,
+    };
+  }
+
+  const mismatched = storedModel !== currentModel || storedDims !== currentDims;
+
+  return {
+    mismatched,
+    storedModel,
+    storedDims,
+    currentModel,
+    currentDims,
+  };
+}
+
+/**
+ * Try to create the sqlite-vec virtual table.
+ * Returns true if successful, false if extension not loaded.
+ */
+export function createVecTable(db: DatabaseLike, dims?: number): boolean {
+  try {
+    // better-sqlite3 Database.exec -- runs SQL, not shell commands
+    db.exec(getVecTableSQL(dims));
+    return true;
+  } catch {
+    // sqlite-vec extension not loaded -- vector search disabled
+    return false;
+  }
+}
+
+/**
+ * Run all necessary migrations to bring the database to the current version.
+ *
+ * This is idempotent: running it multiple times on the same database
+ * produces the same result.
+ */
+export function migrate(db: DatabaseLike): {
+  version: number;
+  migrated: boolean;
+} {
+  applyPragmas(db);
+
+  const currentVersion = getSchemaVersion(db);
+
+  if (currentVersion >= SCHEMA_VERSION) {
+    return { version: currentVersion, migrated: false };
+  }
+
+  // Apply migrations in order
+  if (currentVersion < 1) {
+    migrateV0ToV1(db);
+  }
+
+  if (currentVersion < 2) {
+    migrateV1ToV2(db);
+  }
+
+  return { version: SCHEMA_VERSION, migrated: true };
+}

package/lib/cf-memory/src/backends/sqlite/schema.ts

@@ -0,0 +1,120 @@
+/**
+ * SQLite schema definitions for Tier 1 memory backend.
+ *
+ * Tables:
+ * - memories: main table storing memory metadata + content
+ * - memories_fts: FTS5 virtual table for full-text search
+ * - vec_memories: sqlite-vec virtual table for vector similarity
+ * - embedding_cache: content hash → embedding mapping
+ * - schema_version: single-row version tracker
+ */
+
+/** Current schema version — bump when adding migrations */
+export const SCHEMA_VERSION = 2;
+
+/** Embedding dimensions for all-MiniLM-L6-v2 */
+export const EMBEDDING_DIMS = 384;
+
+/**
+ * SQL statements to create the initial schema (version 1).
+ */
+export const SCHEMA_V1: string[] = [
+  // Version tracking
+  `CREATE TABLE IF NOT EXISTS schema_version (
+    version INTEGER NOT NULL
+  )`,
+
+  // Main memories table
+  `CREATE TABLE IF NOT EXISTS memories (
+    id TEXT PRIMARY KEY,
+    slug TEXT NOT NULL,
+    category TEXT NOT NULL,
+    title TEXT NOT NULL,
+    description TEXT NOT NULL,
+    type TEXT NOT NULL,
+    tags TEXT NOT NULL DEFAULT '[]',
+    importance INTEGER NOT NULL DEFAULT 3,
+    created TEXT NOT NULL,
+    updated TEXT NOT NULL,
+    source TEXT NOT NULL DEFAULT 'conversation',
+    content TEXT NOT NULL DEFAULT '',
+    content_hash TEXT NOT NULL DEFAULT ''
+  )`,
+
+  // Indexes for common queries
+  `CREATE INDEX IF NOT EXISTS idx_memories_type ON memories(type)`,
+  `CREATE INDEX IF NOT EXISTS idx_memories_category ON memories(category)`,
+  `CREATE INDEX IF NOT EXISTS idx_memories_updated ON memories(updated DESC)`,
+  `CREATE INDEX IF NOT EXISTS idx_memories_content_hash ON memories(content_hash)`,
+
+  // FTS5 virtual table for full-text search (BM25 ranking)
+  `CREATE VIRTUAL TABLE IF NOT EXISTS memories_fts USING fts5(
+    title,
+    description,
+    tags,
+    content,
+    content=memories,
+    content_rowid=rowid,
+    tokenize='porter unicode61'
+  )`,
+
+  // FTS5 triggers to keep the index in sync
+  `CREATE TRIGGER IF NOT EXISTS memories_ai AFTER INSERT ON memories BEGIN
+    INSERT INTO memories_fts(rowid, title, description, tags, content)
+    VALUES (new.rowid, new.title, new.description, new.tags, new.content);
+  END`,
+
+  `CREATE TRIGGER IF NOT EXISTS memories_ad AFTER DELETE ON memories BEGIN
+    INSERT INTO memories_fts(memories_fts, rowid, title, description, tags, content)
+    VALUES ('delete', old.rowid, old.title, old.description, old.tags, old.content);
+  END`,
+
+  `CREATE TRIGGER IF NOT EXISTS memories_au AFTER UPDATE ON memories BEGIN
+    INSERT INTO memories_fts(memories_fts, rowid, title, description, tags, content)
+    VALUES ('delete', old.rowid, old.title, old.description, old.tags, old.content);
+    INSERT INTO memories_fts(rowid, title, description, tags, content)
+    VALUES (new.rowid, new.title, new.description, new.tags, new.content);
+  END`,
+
+  // Embedding cache: avoids re-embedding unchanged content
+  `CREATE TABLE IF NOT EXISTS embedding_cache (
+    content_hash TEXT PRIMARY KEY,
+    embedding BLOB NOT NULL,
+    model TEXT NOT NULL,
+    created TEXT NOT NULL
+  )`,
+
+  // Insert schema version
+  `INSERT INTO schema_version (version) VALUES (1)`,
+];
+
+/**
+ * SQL to create the metadata table (added in schema v2).
+ */
+export const SCHEMA_V2_METADATA = `CREATE TABLE IF NOT EXISTS metadata (
+  key TEXT PRIMARY KEY,
+  value TEXT NOT NULL
+)`;
+
+/**
+ * PRAGMA settings for optimal performance.
+ */
+export const PRAGMA_SETTINGS: string[] = [
+  "PRAGMA journal_mode = WAL",
+  "PRAGMA synchronous = NORMAL",
+  "PRAGMA cache_size = -64000", // 64MB cache
+  "PRAGMA foreign_keys = ON",
+  "PRAGMA temp_store = MEMORY",
+  "PRAGMA mmap_size = 268435456", // 256MB mmap
+];
+
+/**
+ * SQL to create the sqlite-vec virtual table.
+ * This is separate because sqlite-vec extension must be loaded first.
+ */
+export function getVecTableSQL(dims: number = EMBEDDING_DIMS): string {
+  return `CREATE VIRTUAL TABLE IF NOT EXISTS vec_memories USING vec0(
+    memory_id TEXT PRIMARY KEY,
+    embedding float[${dims}]
+  )`;
+}

package/lib/cf-memory/src/backends/sqlite/search.ts

@@ -0,0 +1,296 @@
+/**
+ * Hybrid search: FTS5 BM25 + sqlite-vec cosine similarity + RRF fusion.
+ *
+ * Query routing:
+ * - Quoted strings or code patterns → keyword only (FTS5)
+ * - Questions (starts with who/what/why/how/when/where) → semantic only (vector)
+ * - Default → hybrid (FTS5 + vector, fused with RRF)
+ */
+
+import type { DatabaseLike } from "./migrations.js";
+import {
+  EmbeddingPipeline,
+  EmbeddingCache,
+  contentHash,
+  prepareEmbeddingText,
+} from "./embeddings.js";
+import { applyTemporalDecay } from "../../lib/temporal-decay.js";
+
+/** RRF fusion constant — higher = more weight to individual rankings */
+const RRF_K = 60;
+
+export type SearchMode = "keyword" | "semantic" | "hybrid";
+
+interface RankedResult {
+  id: string;
+  score: number;
+  matchedOn: string[];
+}
+
+/**
+ * Detect the best search mode based on query shape.
+ */
+export function detectSearchMode(query: string): SearchMode {
+  const trimmed = query.trim();
+
+  // Quoted strings → keyword
+  if (/^["'].*["']$/.test(trimmed)) return "keyword";
+
+  // Code-like patterns (dotted identifiers, ::, ->, path separators) → keyword
+  if (/\w\.\w|->|::|\/\w+\//.test(trimmed)) return "keyword";
+
+  // Questions → semantic
+  if (
+    /^(who|what|why|how|when|where|is|are|can|does|do|should|which)\b/i.test(
+      trimmed,
+    )
+  ) {
+    return "semantic";
+  }
+
+  return "hybrid";
+}
+
+/**
+ * FTS5 keyword search using BM25 ranking.
+ */
+export function ftsSearch(
+  db: DatabaseLike,
+  query: string,
+  limit: number,
+  typeFilter?: string,
+): RankedResult[] {
+  // Escape FTS5 special characters and wrap tokens in quotes to prevent
+  // FTS5 operator injection (AND, OR, NOT, NEAR, column: filters)
+  const cleaned = query.replace(/['"*()]/g, " ").trim();
+  if (!cleaned) return [];
+  const escaped = cleaned
+    .split(/\s+/)
+    .filter(Boolean)
+    .map((t) => `"${t}"`)
+    .join(" ");
+
+  // Build FTS5 query with column weights
+  // BM25 weights: title(10), description(4), tags(6), content(1)
+  let sql = `
+    SELECT
+      m.id,
+      m.updated,
+      bm25(memories_fts, 10.0, 4.0, 6.0, 1.0) AS rank,
+      memories_fts.title AS fts_title,
+      memories_fts.description AS fts_desc,
+      memories_fts.tags AS fts_tags,
+      memories_fts.content AS fts_content
+    FROM memories_fts
+    JOIN memories m ON m.rowid = memories_fts.rowid
+    WHERE memories_fts MATCH ?
+  `;
+
+  const params: unknown[] = [escaped];
+
+  if (typeFilter) {
+    sql += ` AND m.type = ?`;
+    params.push(typeFilter);
+  }
+
+  sql += ` ORDER BY rank LIMIT ?`;
+  params.push(limit);
+
+  try {
+    const stmt = db.prepare(sql);
+    const results: RankedResult[] = [];
+    const allRows = (
+      stmt as unknown as {
+        all(...p: unknown[]): Array<Record<string, unknown>>;
+      }
+    ).all(...params);
+
+    for (const row of allRows) {
+      const matchedOn: string[] = [];
+      const q = escaped.toLowerCase();
+      if (
+        String(row.fts_title ?? "")
+          .toLowerCase()
+          .includes(q)
+      )
+        matchedOn.push("title");
+      if (
+        String(row.fts_desc ?? "")
+          .toLowerCase()
+          .includes(q)
+      )
+        matchedOn.push("description");
+      if (
+        String(row.fts_tags ?? "")
+          .toLowerCase()
+          .includes(q)
+      )
+        matchedOn.push("tags");
+      if (
+        matchedOn.length === 0 &&
+        String(row.fts_content ?? "")
+          .toLowerCase()
+          .includes(q)
+      ) {
+        matchedOn.push("content");
+      }
+      if (matchedOn.length === 0) matchedOn.push("content");
+
+      const rawScore = -(row.rank as number);
+      const decayedScore = applyTemporalDecay(rawScore, String(row.updated));
+
+      results.push({
+        id: String(row.id),
+        score: decayedScore,
+        matchedOn,
+      });
+    }
+
+    return results;
+  } catch {
+    // FTS5 query syntax error — return empty
+    return [];
+  }
+}
+
+/**
+ * Vector similarity search using sqlite-vec cosine distance.
+ */
+export async function vecSearch(
+  db: DatabaseLike,
+  query: string,
+  limit: number,
+  pipeline: EmbeddingPipeline,
+  typeFilter?: string,
+): Promise<RankedResult[]> {
+  const embedding = await pipeline.embed(query);
+  const buffer = Buffer.from(
+    embedding.buffer,
+    embedding.byteOffset,
+    embedding.byteLength,
+  );
+
+  try {
+    let sql: string;
+    let params: unknown[];
+
+    if (typeFilter) {
+      sql = `
+        SELECT
+          v.memory_id AS id,
+          v.distance,
+          m.updated
+        FROM vec_memories v
+        JOIN memories m ON m.id = v.memory_id
+        WHERE v.embedding MATCH ? AND k = ? AND m.type = ?
+        ORDER BY v.distance
+      `;
+      params = [buffer, limit, typeFilter];
+    } else {
+      sql = `
+        SELECT
+          v.memory_id AS id,
+          v.distance,
+          m.updated
+        FROM vec_memories v
+        JOIN memories m ON m.id = v.memory_id
+        WHERE v.embedding MATCH ? AND k = ?
+        ORDER BY v.distance
+      `;
+      params = [buffer, limit];
+    }
+
+    const stmt = db.prepare(sql);
+    const rows = (
+      stmt as unknown as {
+        all(...p: unknown[]): Array<Record<string, unknown>>;
+      }
+    ).all(...params);
+
+    return rows.map((row) => {
+      const rawScore = 1 - (row.distance as number);
+      return {
+        id: String(row.id),
+        score: applyTemporalDecay(rawScore, String(row.updated)),
+        matchedOn: ["semantic"],
+      };
+    });
+  } catch {
+    // sqlite-vec not available or query failed
+    return [];
+  }
+}
+
+/**
+ * Reciprocal Rank Fusion (RRF) — merge two ranked lists.
+ *
+ * For each document, RRF score = sum over all lists of: 1 / (k + rank_in_list)
+ * where k is a constant (default 60) that controls how much individual rank matters.
+ */
+export function rrfFuse(...lists: RankedResult[][]): RankedResult[] {
+  const scores = new Map<string, { score: number; matchedOn: Set<string> }>();
+
+  for (const list of lists) {
+    for (let rank = 0; rank < list.length; rank++) {
+      const item = list[rank];
+      const existing = scores.get(item.id) ?? {
+        score: 0,
+        matchedOn: new Set<string>(),
+      };
+      existing.score += 1 / (RRF_K + rank + 1);
+      for (const m of item.matchedOn) {
+        existing.matchedOn.add(m);
+      }
+      scores.set(item.id, existing);
+    }
+  }
+
+  return Array.from(scores.entries())
+    .map(([id, { score, matchedOn }]) => ({
+      id,
+      score,
+      matchedOn: Array.from(matchedOn),
+    }))
+    .sort((a, b) => b.score - a.score);
+}
+
+export interface HybridSearchOptions {
+  db: DatabaseLike;
+  query: string;
+  limit: number;
+  pipeline: EmbeddingPipeline | null;
+  vecEnabled: boolean;
+  typeFilter?: string;
+  modeOverride?: SearchMode;
+}
+
+/**
+ * Perform a hybrid search combining FTS5 and vector similarity.
+ */
+export async function hybridSearch(
+  opts: HybridSearchOptions,
+): Promise<RankedResult[]> {
+  const { db, query, limit, pipeline, vecEnabled, typeFilter, modeOverride } =
+    opts;
+  const mode = modeOverride ?? detectSearchMode(query);
+
+  // Fetch more results than needed for fusion, then trim
+  const fetchLimit = Math.max(limit * 3, 20);
+
+  if (mode === "keyword" || !vecEnabled || !pipeline) {
+    return ftsSearch(db, query, limit, typeFilter);
+  }
+
+  if (mode === "semantic") {
+    return vecSearch(db, query, limit, pipeline, typeFilter);
+  }
+
+  // Hybrid: run both in parallel, fuse with RRF
+  const [ftsResults, vecResults] = await Promise.all([
+    ftsSearch(db, query, fetchLimit, typeFilter),
+    vecSearch(db, query, fetchLimit, pipeline, typeFilter),
+  ]);
+
+  const fused = rrfFuse(ftsResults, vecResults);
+  return fused.slice(0, limit);
+}

package/lib/cf-memory/src/bin/cf-memory.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+import "../index.js";

package/lib/cf-memory/src/daemon/entry.ts

@@ -0,0 +1,99 @@
+/**
+ * Daemon entry point — spawned as a detached child process.
+ *
+ * Usage: node daemon/entry.js <docsDir> [idleTimeoutMs] [--tier=full|lite]
+ *
+ * When --tier=full, uses SqliteBackend (requires lazy-installed deps).
+ * Otherwise defaults to MiniSearchBackend.
+ */
+import path from "node:path";
+import { MiniSearchBackend } from "../backends/minisearch.js";
+import { areSqliteDepsAvailable } from "../lib/lazy-install.js";
+import type { MemoryBackend } from "../lib/backend.js";
+import { startDaemonServer } from "./process.js";
+import { setupWatcher } from "./watcher.js";
+
+const docsDir = process.argv[2];
+if (!docsDir) {
+  process.stderr.write(
+    "Usage: node daemon/entry.js <docsDir> [idleTimeoutMs] [--tier=full|lite]\n",
+  );
+  process.exit(1);
+}
+
+const resolvedDir = path.resolve(docsDir);
+const idleTimeoutMs =
+  process.argv[3] && !process.argv[3].startsWith("--")
+    ? parseInt(process.argv[3], 10)
+    : undefined;
+
+// Parse --tier flag from any position
+const tierArg = process.argv.find((a) => a.startsWith("--tier="));
+const requestedTier = tierArg?.split("=")[1];
+
+// Parse embedding flags from any position
+const embeddingProviderArg = process.argv.find((a) =>
+  a.startsWith("--embedding-provider="),
+);
+const embeddingModelArg = process.argv.find((a) =>
+  a.startsWith("--embedding-model="),
+);
+const embeddingOllamaUrlArg = process.argv.find((a) =>
+  a.startsWith("--embedding-ollama-url="),
+);
+
+// Use substring to avoid truncating values that contain "="
+function flagValue(arg: string): string {
+  return arg.substring(arg.indexOf("=") + 1);
+}
+
+const embeddingConfig = {
+  ...(embeddingProviderArg && {
+    provider: flagValue(embeddingProviderArg) as "transformers" | "ollama",
+  }),
+  ...(embeddingModelArg && { model: flagValue(embeddingModelArg) }),
+  ...(embeddingOllamaUrlArg && {
+    ollamaUrl: flagValue(embeddingOllamaUrlArg),
+  }),
+};
+const hasEmbeddingConfig = Object.keys(embeddingConfig).length > 0;
+
+async function createBackend(): Promise<MemoryBackend> {
+  // Use SqliteBackend if requested or if deps are available and not forced to lite
+  if (
+    requestedTier === "full" ||
+    (requestedTier !== "lite" && areSqliteDepsAvailable())
+  ) {
+    try {
+      const { SqliteBackend } = await import("../backends/sqlite/index.js");
+      return new SqliteBackend(
+        resolvedDir,
+        hasEmbeddingConfig ? { embedding: embeddingConfig } : undefined,
+      );
+    } catch (err) {
+      process.stderr.write(
+        `[cf-memory] SqliteBackend failed, falling back to MiniSearch: ${err instanceof Error ? err.message : String(err)}\n`,
+      );
+    }
+  }
+  return new MiniSearchBackend(resolvedDir);
+}
+
+const backend = await createBackend();
+
+// Only set up file watcher if backend supports rebuild
+const watcher = backend.rebuild
+  ? setupWatcher(
+      resolvedDir,
+      backend as Required<Pick<MemoryBackend, "rebuild">>,
+    )
+  : { close() {} };
+
+const { close } = startDaemonServer(backend, {
+  idleTimeoutMs,
+  onShutdown: () => watcher.close(),
+});
+
+// Single place for signal handling — no duplicates
+process.on("SIGTERM", close);
+process.on("SIGINT", close);