autoctxd 0.4.1

package/src/db/vector/embeddings.ts

@@ -0,0 +1,119 @@
+// Public embedding API. Delegates to the configured provider (see providers/factory.ts).
+//
+// Vectors are L2-normalized by every provider, so cosineSimilarity reduces to a
+// dot product. The cache table is partitioned by provider name — switching
+// providers does not invalidate cached vectors of the previous one, but
+// queries only see the rows that match the active provider.
+
+import { getDb } from "../sqlite/schema";
+import { createHash } from "crypto";
+import { getProvider } from "./providers/factory";
+import type { EmbeddingProvider, EmbeddingProviderName } from "./providers/types";
+
+export type { EmbeddingProvider, EmbeddingProviderName };
+
+/** Dim of the active provider's vectors. Use this instead of a constant. */
+export function getActiveDim(): number {
+  return getProvider().dim;
+}
+
+export function getActiveProviderName(): EmbeddingProviderName {
+  return getProvider().name;
+}
+
+export async function generateEmbedding(text: string): Promise<Float32Array> {
+  const provider = getProvider();
+  const hash = hashText(text);
+  const cached = getCachedEmbedding(hash, provider.name, provider.dim);
+  if (cached) return cached;
+
+  const embedding = await provider.embed(text);
+  cacheEmbedding(hash, provider.name, embedding);
+  return embedding;
+}
+
+export async function generateEmbeddingsBatch(texts: string[]): Promise<Float32Array[]> {
+  const provider = getProvider();
+  const out: Float32Array[] = new Array(texts.length);
+  const missingIdx: number[] = [];
+  const missingTexts: string[] = [];
+
+  for (let i = 0; i < texts.length; i++) {
+    const cached = getCachedEmbedding(hashText(texts[i]), provider.name, provider.dim);
+    if (cached) {
+      out[i] = cached;
+    } else {
+      missingIdx.push(i);
+      missingTexts.push(texts[i]);
+    }
+  }
+
+  if (missingTexts.length > 0) {
+    const fresh = provider.embedBatch
+      ? await provider.embedBatch(missingTexts)
+      : await Promise.all(missingTexts.map((t) => provider.embed(t)));
+    for (let k = 0; k < missingIdx.length; k++) {
+      const i = missingIdx[k];
+      out[i] = fresh[k];
+      cacheEmbedding(hashText(texts[i]), provider.name, fresh[k]);
+    }
+  }
+
+  return out;
+}
+
+function hashText(text: string): string {
+  return createHash("sha256").update(text).digest("hex").slice(0, 16);
+}
+
+function getCachedEmbedding(textHash: string, provider: string, expectedDim: number): Float32Array | null {
+  try {
+    const db = getDb();
+    const row = db
+      .prepare("SELECT embedding FROM embeddings_cache WHERE text_hash = ? AND provider = ?")
+      .get(textHash, provider) as { embedding?: Buffer } | undefined;
+    if (row?.embedding) {
+      const buffer = row.embedding;
+      const vec = new Float32Array(buffer.buffer, buffer.byteOffset, buffer.byteLength / 4);
+      if (vec.length !== expectedDim) return null; // stale cache from a different model variant
+      return vec;
+    }
+  } catch {
+    // cache miss
+  }
+  return null;
+}
+
+function cacheEmbedding(textHash: string, provider: string, embedding: Float32Array): void {
+  try {
+    const db = getDb();
+    const buffer = Buffer.from(embedding.buffer, embedding.byteOffset, embedding.byteLength);
+    db.prepare(
+      "INSERT OR IGNORE INTO embeddings_cache (text_hash, provider, embedding) VALUES (?, ?, ?)"
+    ).run(textHash, provider, buffer);
+  } catch {
+    // ignore cache write errors
+  }
+}
+
+export function clearEmbeddingCache(provider?: EmbeddingProviderName): void {
+  try {
+    const db = getDb();
+    if (provider) {
+      db.prepare("DELETE FROM embeddings_cache WHERE provider = ?").run(provider);
+    } else {
+      db.prepare("DELETE FROM embeddings_cache").run();
+    }
+  } catch {
+    // ignore
+  }
+}
+
+export function cosineSimilarity(a: Float32Array, b: Float32Array): number {
+  let dot = 0;
+  for (let i = 0; i < a.length; i++) dot += a[i] * b[i];
+  return dot; // vectors are L2-normalized
+}
+
+/** @deprecated Kept for backwards compatibility with old callers. Use getActiveDim(). */
+export const VECTOR_DIM = 128;

package/src/db/vector/providers/factory.ts

@@ -0,0 +1,99 @@
+// Provider selection and config persistence.
+//
+// Resolution order (highest priority first):
+//   1. AUTOCTXD_EMBEDDING env var
+//   2. data/config.json -> embeddingProvider
+//   3. Default: "tfidf"
+//
+// Switching providers means subsequent embeddings live in a different vector
+// space — re-embed existing summaries via `autoctxd embeddings switch <name>`
+// (handled in the CLI).
+
+import { join } from "path";
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
+import type { EmbeddingProvider, EmbeddingProviderName } from "./types";
+import { TfidfProvider } from "./tfidf";
+import { MiniLMProvider } from "./minilm";
+import { OllamaProvider } from "./ollama";
+
+interface Config {
+  embeddingProvider?: EmbeddingProviderName;
+}
+
+function dataDir(): string {
+  return process.env.AUTOCTXD_DATA_DIR || join(import.meta.dir, "..", "..", "..", "..", "data");
+}
+
+function configPath(): string {
+  return process.env.AUTOCTXD_CONFIG_PATH || join(dataDir(), "config.json");
+}
+
+function readConfig(): Config {
+  const path = configPath();
+  if (!existsSync(path)) return {};
+  try {
+    return JSON.parse(readFileSync(path, "utf8")) as Config;
+  } catch {
+    return {};
+  }
+}
+
+function writeConfig(cfg: Config): void {
+  const path = configPath();
+  mkdirSync(join(path, ".."), { recursive: true });
+  writeFileSync(path, JSON.stringify(cfg, null, 2) + "\n");
+}
+
+export const ALL_PROVIDER_NAMES: readonly EmbeddingProviderName[] = ["tfidf", "minilm", "ollama"] as const;
+
+function instantiate(name: EmbeddingProviderName): EmbeddingProvider {
+  switch (name) {
+    case "tfidf":
+      return new TfidfProvider();
+    case "minilm":
+      return new MiniLMProvider();
+    case "ollama":
+      return new OllamaProvider();
+  }
+}
+
+export function resolveProviderName(): EmbeddingProviderName {
+  const env = process.env.AUTOCTXD_EMBEDDING?.toLowerCase();
+  if (env === "tfidf" || env === "minilm" || env === "ollama") return env;
+
+  const cfg = readConfig();
+  if (cfg.embeddingProvider && ALL_PROVIDER_NAMES.includes(cfg.embeddingProvider)) {
+    return cfg.embeddingProvider;
+  }
+
+  return "tfidf";
+}
+
+let _cached: EmbeddingProvider | null = null;
+let _cachedName: EmbeddingProviderName | null = null;
+
+export function getProvider(): EmbeddingProvider {
+  const name = resolveProviderName();
+  if (_cached && _cachedName === name) return _cached;
+  _cached = instantiate(name);
+  _cachedName = name;
+  return _cached;
+}
+
+/** Build a fresh provider instance without affecting the singleton. For benchmarks. */
+export function buildProvider(name: EmbeddingProviderName): EmbeddingProvider {
+  return instantiate(name);
+}
+
+export async function setProvider(name: EmbeddingProviderName): Promise<void> {
+  const cfg = readConfig();
+  cfg.embeddingProvider = name;
+  writeConfig(cfg);
+  if (_cached?.dispose) await _cached.dispose().catch(() => {});
+  _cached = null;
+  _cachedName = null;
+}
+
+export function getConfiguredProviderName(): EmbeddingProviderName {
+  return resolveProviderName();
+}

package/src/db/vector/providers/minilm.ts

@@ -0,0 +1,90 @@
+// MiniLM (sentence-transformers/all-MiniLM-L6-v2) provider via @xenova/transformers.
+//
+// 384-dim, mean-pooled, L2-normalized. Runs on CPU via onnxruntime-web bundled
+// with transformers.js. First use downloads the quantized model (~25MB) into
+// AUTOCTXD_MODEL_DIR (default: ~/.claude/autoctxd/models).
+//
+// @xenova/transformers is an *optional* dependency — if it isn't installed,
+// `check()` returns a friendly error pointing at `autoctxd embeddings install minilm`.
+
+import { join } from "path";
+import { homedir } from "os";
+import type { EmbeddingProvider } from "./types";
+
+const MODEL_ID = "Xenova/all-MiniLM-L6-v2";
+const VECTOR_DIM = 384;
+
+function modelDir(): string {
+  return process.env.AUTOCTXD_MODEL_DIR || join(homedir(), ".claude", "autoctxd", "models");
+}
+
+let _extractor: any = null;
+let _loadPromise: Promise<any> | null = null;
+
+async function loadExtractor(): Promise<any> {
+  if (_extractor) return _extractor;
+  if (_loadPromise) return _loadPromise;
+
+  _loadPromise = (async () => {
+    let mod: any;
+    try {
+      mod = await import("@xenova/transformers");
+    } catch (e) {
+      throw new Error(
+        "@xenova/transformers is not installed. Run `autoctxd embeddings install minilm` to add it."
+      );
+    }
+
+    mod.env.allowLocalModels = false;
+    mod.env.cacheDir = modelDir();
+
+    _extractor = await mod.pipeline("feature-extraction", MODEL_ID, {
+      quantized: true,
+    });
+    return _extractor;
+  })();
+
+  try {
+    return await _loadPromise;
+  } finally {
+    _loadPromise = null;
+  }
+}
+
+export class MiniLMProvider implements EmbeddingProvider {
+  readonly name = "minilm" as const;
+  readonly dim = VECTOR_DIM;
+  readonly label = "MiniLM-L6-v2 via transformers.js (384-dim, ~25MB local model)";
+
+  async check() {
+    try {
+      await import("@xenova/transformers");
+      return { ok: true } as const;
+    } catch {
+      return {
+        ok: false as const,
+        reason: "@xenova/transformers not installed — run `autoctxd embeddings install minilm`",
+      };
+    }
+  }
+
+  async embed(text: string): Promise<Float32Array> {
+    const extractor = await loadExtractor();
+    const out = await extractor(text, { pooling: "mean", normalize: true });
+    return new Float32Array(out.data as Float32Array);
+  }
+
+  async embedBatch(texts: string[]): Promise<Float32Array[]> {
+    const extractor = await loadExtractor();
+    const results: Float32Array[] = [];
+    for (const t of texts) {
+      const out = await extractor(t, { pooling: "mean", normalize: true });
+      results.push(new Float32Array(out.data as Float32Array));
+    }
+    return results;
+  }
+
+  async dispose() {
+    _extractor = null;
+  }
+}

package/src/db/vector/providers/ollama.ts

@@ -0,0 +1,92 @@
+// Ollama provider — speaks to a local Ollama daemon at $AUTOCTXD_OLLAMA_URL
+// (default http://localhost:11434) using the embedding model in
+// $AUTOCTXD_OLLAMA_MODEL (default nomic-embed-text, 768-dim).
+//
+// Zero install on our side: we just call /api/embeddings. The user is expected
+// to have Ollama installed and the model pulled (`ollama pull nomic-embed-text`).
+// The dim is read from the first response and cached.
+
+import type { EmbeddingProvider } from "./types";
+
+const DEFAULT_URL = "http://localhost:11434";
+const DEFAULT_MODEL = "nomic-embed-text";
+
+function ollamaUrl(): string {
+  return process.env.AUTOCTXD_OLLAMA_URL || DEFAULT_URL;
+}
+
+function ollamaModel(): string {
+  return process.env.AUTOCTXD_OLLAMA_MODEL || DEFAULT_MODEL;
+}
+
+let _dim: number | null = null;
+
+async function fetchEmbedding(text: string): Promise<Float32Array> {
+  const res = await fetch(`${ollamaUrl()}/api/embeddings`, {
+    method: "POST",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify({ model: ollamaModel(), prompt: text }),
+  });
+  if (!res.ok) {
+    throw new Error(`Ollama returned ${res.status}: ${await res.text().catch(() => "")}`);
+  }
+  const data = (await res.json()) as { embedding?: number[] };
+  if (!data.embedding || !Array.isArray(data.embedding)) {
+    throw new Error("Ollama response missing 'embedding' array");
+  }
+
+  // L2-normalize so cosine === dot product downstream
+  const raw = data.embedding;
+  let norm = 0;
+  for (let i = 0; i < raw.length; i++) norm += raw[i] * raw[i];
+  norm = Math.sqrt(norm);
+  const out = new Float32Array(raw.length);
+  if (norm > 0) {
+    for (let i = 0; i < raw.length; i++) out[i] = raw[i] / norm;
+  } else {
+    for (let i = 0; i < raw.length; i++) out[i] = raw[i];
+  }
+  return out;
+}
+
+export class OllamaProvider implements EmbeddingProvider {
+  readonly name = "ollama" as const;
+  readonly label = `Ollama (${ollamaModel()} at ${ollamaUrl()})`;
+
+  get dim(): number {
+    // Default to nomic-embed-text's known dim; updated after first successful embed.
+    return _dim ?? 768;
+  }
+
+  async check() {
+    try {
+      const res = await fetch(`${ollamaUrl()}/api/tags`, {
+        signal: AbortSignal.timeout(2000),
+      });
+      if (!res.ok) {
+        return { ok: false as const, reason: `Ollama responded ${res.status} at ${ollamaUrl()}` };
+      }
+      const data = (await res.json()) as { models?: Array<{ name: string }> };
+      const want = ollamaModel();
+      const found = data.models?.some((m) => m.name === want || m.name.startsWith(want + ":"));
+      if (!found) {
+        return {
+          ok: false as const,
+          reason: `Ollama is up but model '${want}' isn't pulled — run \`ollama pull ${want}\``,
+        };
+      }
+      return { ok: true } as const;
+    } catch (e) {
+      return {
+        ok: false as const,
+        reason: `Ollama not reachable at ${ollamaUrl()} — install from https://ollama.com and start the daemon`,
+      };
+    }
+  }
+
+  async embed(text: string): Promise<Float32Array> {
+    const v = await fetchEmbedding(text);
+    if (_dim === null) _dim = v.length;
+    return v;
+  }
+}

package/src/db/vector/providers/tfidf.ts

@@ -0,0 +1,98 @@
+// TF-IDF feature-hashing provider — the zero-dep default.
+//
+// 128-dim vectors built from token + bigram hashing with log-scaled TF and a
+// sign hash. Fast, deterministic, no network, no model download. Quality is
+// adequate for keyword/semantic overlap on developer text — for richer
+// retrieval, switch to MiniLM or Ollama.
+
+import type { EmbeddingProvider } from "./types";
+
+const VECTOR_DIM = 128;
+
+const STOP_WORDS = new Set([
+  "the", "is", "at", "of", "on", "and", "or", "to", "in", "for",
+  "it", "an", "be", "as", "by", "was", "are", "do", "if", "so",
+  "no", "not", "but", "from", "this", "that", "with", "have", "has",
+  "had", "will", "would", "could", "should", "been", "being", "which",
+  "their", "there", "then", "than", "them", "they", "what", "when",
+  "where", "who", "how", "all", "each", "every", "both", "few",
+  "more", "most", "other", "some", "such", "only", "own", "same",
+  "also", "can", "just", "any", "now", "new", "one", "two",
+]);
+
+function murmurhash(key: string, seed: number): number {
+  let h = seed;
+  for (let i = 0; i < key.length; i++) {
+    h = Math.imul(h ^ key.charCodeAt(i), 0x5bd1e995);
+    h ^= h >>> 13;
+  }
+  return Math.abs(h);
+}
+
+function tokenize(text: string): string[] {
+  return text
+    .toLowerCase()
+    .replace(/[^\w\s/._-]/g, " ")
+    .split(/\s+/)
+    .filter((t) => t.length > 1 && t.length < 50)
+    .map((t) =>
+      t
+        .replace(/([a-z])([A-Z])/g, "$1 $2")
+        .replace(/_/g, " ")
+        .toLowerCase()
+        .split(/\s+/)
+    )
+    .flat()
+    .filter((t) => t.length > 1 && !STOP_WORDS.has(t));
+}
+
+export function tfidfEmbed(text: string): Float32Array {
+  const vec = new Float32Array(VECTOR_DIM);
+  const tokens = tokenize(text);
+  const tokenFreq = new Map<string, number>();
+
+  for (const token of tokens) {
+    tokenFreq.set(token, (tokenFreq.get(token) || 0) + 1);
+  }
+
+  for (const [token, freq] of tokenFreq) {
+    const h1 = murmurhash(token, 0) % VECTOR_DIM;
+    const h2 = murmurhash(token, 1) % 2 === 0 ? 1 : -1;
+    const tf = Math.log(1 + freq);
+    vec[h1] += h2 * tf;
+  }
+
+  for (let i = 0; i < tokens.length - 1; i++) {
+    const bigram = tokens[i] + " " + tokens[i + 1];
+    const h1 = murmurhash(bigram, 2) % VECTOR_DIM;
+    const h2 = murmurhash(bigram, 3) % 2 === 0 ? 1 : -1;
+    vec[h1] += h2 * 0.5;
+  }
+
+  let norm = 0;
+  for (let i = 0; i < VECTOR_DIM; i++) norm += vec[i] * vec[i];
+  norm = Math.sqrt(norm);
+  if (norm > 0) {
+    for (let i = 0; i < VECTOR_DIM; i++) vec[i] /= norm;
+  }
+
+  return vec;
+}
+
+export class TfidfProvider implements EmbeddingProvider {
+  readonly name = "tfidf" as const;
+  readonly dim = VECTOR_DIM;
+  readonly label = "TF-IDF feature hashing (128-dim, zero deps)";
+
+  async check() {
+    return { ok: true } as const;
+  }
+
+  async embed(text: string): Promise<Float32Array> {
+    return tfidfEmbed(text);
+  }
+
+  async embedBatch(texts: string[]): Promise<Float32Array[]> {
+    return texts.map(tfidfEmbed);
+  }
+}

package/src/db/vector/providers/types.ts

@@ -0,0 +1,39 @@
+// Pluggable embedding providers.
+//
+// Adding a provider:
+//   1. Implement EmbeddingProvider in a new file.
+//   2. Register it in factory.ts under PROVIDERS.
+//   3. Add a row to the docs table in README and the `embeddings list` output.
+//
+// Vectors must be L2-normalized so cosine similarity reduces to a dot product
+// downstream. Providers that don't normalize natively must do it themselves.
+
+export type EmbeddingProviderName = "tfidf" | "minilm" | "ollama";
+
+export interface EmbeddingProvider {
+  /** Stable identifier persisted to disk and the cache table. */
+  readonly name: EmbeddingProviderName;
+
+  /** Vector dimension. Must be stable for the lifetime of the provider instance. */
+  readonly dim: number;
+
+  /** Human label for `embeddings list` / status output. */
+  readonly label: string;
+
+  /**
+   * Returns `{ ok: true }` if the provider can run on this machine right now,
+   * otherwise `{ ok: false, reason }` with an actionable message
+   * (e.g. "run `autoctxd embeddings install minilm`").
+   * Should be cheap — no network calls beyond a localhost ping.
+   */
+  check(): Promise<{ ok: true } | { ok: false; reason: string }>;
+
+  /** Compute one embedding. Must return a Float32Array of length `dim`, L2-normalized. */
+  embed(text: string): Promise<Float32Array>;
+
+  /** Optional: compute many in one go. Default impl serializes via embed(). */
+  embedBatch?(texts: string[]): Promise<Float32Array[]>;
+
+  /** Optional: free model resources. Called by the CLI when switching providers. */
+  dispose?(): Promise<void>;
+}

package/src/db/vector/search.ts

@@ -0,0 +1,131 @@
+// High-level semantic search combining LanceDB and SQLite FTS
+
+import { generateEmbedding, cosineSimilarity } from "./embeddings";
+import { searchSimilar, type VectorRecord } from "./client";
+import { searchObservations } from "../sqlite/observations";
+import { searchDecisions } from "../sqlite/decisions";
+
+export interface SearchResult {
+  source: "vector" | "fts_observations" | "fts_decisions";
+  text: string;
+  score: number;
+  metadata: Record<string, any>;
+}
+
+export async function hybridSearch(
+  query: string,
+  options: {
+    limit?: number;
+    projectPath?: string;
+    includeObservations?: boolean;
+    includeDecisions?: boolean;
+  } = {}
+): Promise<SearchResult[]> {
+  const {
+    limit = 10,
+    projectPath,
+    includeObservations = true,
+    includeDecisions = true,
+  } = options;
+
+  const results: SearchResult[] = [];
+
+  // 1. Vector/semantic search
+  try {
+    const queryEmbedding = await generateEmbedding(query);
+    const vectorResults = await searchSimilar(
+      Array.from(queryEmbedding),
+      limit,
+      projectPath
+    );
+
+    for (const vr of vectorResults) {
+      results.push({
+        source: "vector",
+        text: vr.text,
+        score: (vr as any)._distance !== undefined ? 1 / (1 + (vr as any)._distance) : 0.5,
+        metadata: {
+          session_id: vr.session_id,
+          project_path: vr.project_path,
+          level: vr.level,
+          created_at: vr.created_at,
+        },
+      });
+    }
+  } catch {
+    // Vector search failed, continue with FTS
+  }
+
+  // 2. FTS search on observations
+  if (includeObservations) {
+    try {
+      const ftsQuery = sanitizeFtsQuery(query);
+      const obsResults = searchObservations(ftsQuery, limit);
+      for (const obs of obsResults) {
+        results.push({
+          source: "fts_observations",
+          text: obs.summary,
+          score: 0.6, // FTS results get moderate score
+          metadata: {
+            type: obs.type,
+            tool_name: obs.tool_name,
+            session_id: obs.session_id,
+            importance: obs.importance_score,
+            timestamp: obs.timestamp,
+          },
+        });
+      }
+    } catch {
+      // FTS search failed
+    }
+  }
+
+  // 3. FTS search on decisions
+  if (includeDecisions) {
+    try {
+      const ftsQuery = sanitizeFtsQuery(query);
+      const decResults = searchDecisions(ftsQuery, limit);
+      for (const dec of decResults) {
+        results.push({
+          source: "fts_decisions",
+          text: `${dec.title}: ${dec.decision_text}`,
+          score: 0.8, // Decisions are high value
+          metadata: {
+            project_path: dec.project_path,
+            alternatives: dec.alternatives,
+            rationale: dec.rationale,
+            created_at: dec.created_at,
+          },
+        });
+      }
+    } catch {
+      // FTS search failed
+    }
+  }
+
+  // Sort by score descending and deduplicate
+  results.sort((a, b) => b.score - a.score);
+
+  // Simple dedup by text similarity
+  const seen = new Set<string>();
+  const deduped: SearchResult[] = [];
+  for (const r of results) {
+    const key = r.text.slice(0, 80).toLowerCase();
+    if (!seen.has(key)) {
+      seen.add(key);
+      deduped.push(r);
+    }
+  }
+
+  return deduped.slice(0, limit);
+}
+
+function sanitizeFtsQuery(query: string): string {
+  // FTS5 query syntax: wrap each word in quotes to avoid syntax errors
+  return query
+    .replace(/[^\w\s]/g, " ")
+    .split(/\s+/)
+    .filter(w => w.length > 1)
+    .map(w => `"${w}"`)
+    .join(" OR ");
+}