autoctxd 0.4.1

package/src/db/sqlite/schema.ts

@@ -0,0 +1,366 @@
+import { Database } from "bun:sqlite";
+import { join } from "path";
+import { mkdirSync } from "fs";
+
+const DATA_DIR = process.env.AUTOCTXD_DATA_DIR || join(import.meta.dir, "..", "..", "..", "data");
+const DB_PATH = process.env.AUTOCTXD_DB_PATH || join(DATA_DIR, "autoctxd.db");
+
+let _db: Database | null = null;
+
+export function getDb(): Database {
+  if (_db) return _db;
+
+  mkdirSync(DATA_DIR, { recursive: true });
+
+  _db = new Database(DB_PATH, { create: true });
+  _db.exec("PRAGMA journal_mode = WAL");
+  _db.exec("PRAGMA busy_timeout = 5000");
+  initSchema(_db);
+  return _db;
+}
+
+function initSchema(db: Database) {
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS sessions (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      session_id TEXT UNIQUE NOT NULL,
+      project_path TEXT,
+      git_repo TEXT,
+      git_branch TEXT,
+      started_at TEXT NOT NULL DEFAULT (datetime('now')),
+      ended_at TEXT,
+      total_observations INTEGER DEFAULT 0,
+      tokens_injected INTEGER DEFAULT 0
+    );
+
+    CREATE TABLE IF NOT EXISTS observations (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      session_id TEXT NOT NULL,
+      type TEXT NOT NULL,
+      tool_name TEXT,
+      summary TEXT NOT NULL,
+      file_paths TEXT,
+      timestamp TEXT NOT NULL DEFAULT (datetime('now')),
+      importance_score INTEGER DEFAULT 5,
+      FOREIGN KEY (session_id) REFERENCES sessions(session_id)
+    );
+
+    CREATE TABLE IF NOT EXISTS summaries (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      session_id TEXT,
+      level INTEGER NOT NULL DEFAULT 1,
+      text TEXT NOT NULL,
+      embedding_id TEXT,
+      project_path TEXT,
+      created_at TEXT NOT NULL DEFAULT (datetime('now'))
+    );
+
+    CREATE TABLE IF NOT EXISTS decisions (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      project_path TEXT,
+      title TEXT NOT NULL,
+      decision_text TEXT NOT NULL,
+      alternatives TEXT,
+      rationale TEXT,
+      files_affected TEXT,
+      embedding_id TEXT,
+      created_at TEXT NOT NULL DEFAULT (datetime('now')),
+      last_referenced TEXT
+    );
+
+    CREATE TABLE IF NOT EXISTS patterns (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      project_path TEXT,
+      pattern_type TEXT NOT NULL,
+      description TEXT NOT NULL,
+      frequency INTEGER DEFAULT 1,
+      last_seen TEXT NOT NULL DEFAULT (datetime('now')),
+      examples TEXT
+    );
+
+    CREATE TABLE IF NOT EXISTS embeddings_cache (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      text_hash TEXT NOT NULL,
+      provider TEXT NOT NULL DEFAULT 'tfidf',
+      embedding BLOB NOT NULL,
+      created_at TEXT NOT NULL DEFAULT (datetime('now')),
+      UNIQUE(text_hash, provider)
+    );
+
+    CREATE TABLE IF NOT EXISTS feedback (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      target_type TEXT NOT NULL,
+      target_id TEXT NOT NULL,
+      target_text TEXT,
+      verdict TEXT NOT NULL,
+      reason TEXT,
+      project_path TEXT,
+      session_id TEXT,
+      created_at TEXT NOT NULL DEFAULT (datetime('now'))
+    );
+
+    CREATE TABLE IF NOT EXISTS mcp_access_log (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      session_id TEXT,
+      project_path TEXT,
+      tool_name TEXT NOT NULL,
+      args TEXT,
+      result_count INTEGER,
+      ts TEXT NOT NULL DEFAULT (datetime('now'))
+    );
+
+    CREATE TABLE IF NOT EXISTS token_metrics (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      session_id TEXT,
+      tokens_injected INTEGER DEFAULT 0,
+      estimated_tokens_saved INTEGER DEFAULT 0,
+      context_relevance_score REAL DEFAULT 0,
+      exploration_calls_before_edit INTEGER DEFAULT 0,
+      time_to_first_edit_sec REAL,
+      context_hit_count INTEGER DEFAULT 0,
+      context_miss_count INTEGER DEFAULT 0,
+      injected_files TEXT,
+      first_edit_recorded INTEGER DEFAULT 0,
+      FOREIGN KEY (session_id) REFERENCES sessions(session_id)
+    );
+
+    CREATE INDEX IF NOT EXISTS idx_observations_session ON observations(session_id);
+    CREATE INDEX IF NOT EXISTS idx_observations_type ON observations(type);
+    CREATE INDEX IF NOT EXISTS idx_observations_importance ON observations(importance_score DESC);
+    CREATE INDEX IF NOT EXISTS idx_summaries_project ON summaries(project_path);
+    CREATE INDEX IF NOT EXISTS idx_summaries_level ON summaries(level);
+    CREATE INDEX IF NOT EXISTS idx_decisions_project ON decisions(project_path);
+    CREATE INDEX IF NOT EXISTS idx_sessions_project ON sessions(project_path);
+    CREATE INDEX IF NOT EXISTS idx_sessions_started ON sessions(started_at DESC);
+    CREATE INDEX IF NOT EXISTS idx_feedback_target ON feedback(target_type, target_id);
+    CREATE INDEX IF NOT EXISTS idx_feedback_project ON feedback(project_path);
+    CREATE INDEX IF NOT EXISTS idx_mcp_log_session ON mcp_access_log(session_id, ts DESC);
+  `);
+
+  // Migrate embeddings_cache to include `provider` column (legacy DBs were UNIQUE on
+  // text_hash alone, before provider partitioning). Add the column with default
+  // 'tfidf' so existing cached vectors are still attributable.
+  try {
+    const cols = db.prepare("PRAGMA table_info(embeddings_cache)").all() as Array<{ name: string }>;
+    const colNames = new Set(cols.map(c => c.name));
+    if (!colNames.has("provider")) {
+      db.exec(`ALTER TABLE embeddings_cache ADD COLUMN provider TEXT NOT NULL DEFAULT 'tfidf'`);
+      // Drop the legacy single-column UNIQUE index; the composite UNIQUE in the
+      // CREATE TABLE will not apply to the existing table, but a unique index
+      // on (text_hash, provider) keeps semantics intact going forward.
+      try { db.exec(`DROP INDEX IF EXISTS sqlite_autoindex_embeddings_cache_1`); } catch {}
+      db.exec(`CREATE UNIQUE INDEX IF NOT EXISTS idx_embeddings_cache_text_provider
+                 ON embeddings_cache(text_hash, provider)`);
+    }
+  } catch {
+    // best-effort migration
+  }
+
+  // Migrate token_metrics if columns are missing (existing DBs)
+  try {
+    const cols = db.prepare("PRAGMA table_info(token_metrics)").all() as Array<{ name: string }>;
+    const colNames = new Set(cols.map(c => c.name));
+    if (!colNames.has("exploration_calls_before_edit")) {
+      db.exec(`ALTER TABLE token_metrics ADD COLUMN exploration_calls_before_edit INTEGER DEFAULT 0`);
+      db.exec(`ALTER TABLE token_metrics ADD COLUMN time_to_first_edit_sec REAL`);
+      db.exec(`ALTER TABLE token_metrics ADD COLUMN context_hit_count INTEGER DEFAULT 0`);
+      db.exec(`ALTER TABLE token_metrics ADD COLUMN context_miss_count INTEGER DEFAULT 0`);
+      db.exec(`ALTER TABLE token_metrics ADD COLUMN injected_files TEXT`);
+      db.exec(`ALTER TABLE token_metrics ADD COLUMN first_edit_recorded INTEGER DEFAULT 0`);
+    }
+  } catch {
+    // Migration already applied or table doesn't exist yet
+  }
+
+  // Collapse duplicate (session_id, level) summary rows produced before the
+  // unique index existed: keep the most recent, drop the rest. Required so
+  // CREATE UNIQUE INDEX below succeeds on legacy DBs.
+  try {
+    db.exec(`
+      DELETE FROM summaries WHERE id IN (
+        SELECT id FROM summaries s1
+        WHERE session_id IS NOT NULL
+          AND EXISTS (
+            SELECT 1 FROM summaries s2
+            WHERE s2.session_id = s1.session_id
+              AND s2.level = s1.level
+              AND (s2.created_at > s1.created_at OR (s2.created_at = s1.created_at AND s2.id > s1.id))
+          )
+      )
+    `);
+  } catch {
+    // table missing on first run, nothing to clean
+  }
+
+  // One Level-1 (per-session) summary per session. Stop hook can fire multiple
+  // times (compaction, sub-agents) — without this, the same session shows up
+  // 3x in "RECENT SESSIONS". Digests (level 2/3) have NULL session_id and are
+  // not constrained by this partial index.
+  try {
+    db.exec(`
+      CREATE UNIQUE INDEX IF NOT EXISTS idx_summaries_session_level
+        ON summaries(session_id, level) WHERE session_id IS NOT NULL
+    `);
+  } catch {
+    // index already exists or unsupported
+  }
+
+  // Normalize legacy pattern descriptions: older versions baked a per-session
+  // counter into the description ("Heavily uses Bash (32 times this session)",
+  // "Session focused on refactor (84/200 actions)"), which broke the upsert
+  // dedupe — every session inserted a fresh row. Strip the volatile tail so
+  // future upserts collapse correctly, then merge duplicates that already exist.
+  try {
+    cleanupLegacyPatterns(db);
+  } catch {
+    // best-effort migration
+  }
+
+  // Purge legacy "decisions" that aren't actually decisions. Older versions
+  // promoted Agent spawns / tool-prefixed observations to architectural
+  // decisions whenever a stray keyword matched. New code blocks this at the
+  // source; this cleans up rows already persisted.
+  try {
+    db.exec(`
+      DELETE FROM decisions WHERE
+        decision_text LIKE 'Spawned agent:%' OR
+        decision_text LIKE 'Wrote %' OR
+        decision_text LIKE 'mcp__%' OR
+        decision_text LIKE 'WebFetch %' OR
+        decision_text LIKE 'WebSearch:%' OR
+        decision_text LIKE 'Notebook %'
+    `);
+  } catch {
+    // best-effort
+  }
+
+  // Collapse duplicate (project_path, title) decision rows before creating the
+  // unique index. Older versions of the extractor inserted the same dep
+  // decision on every session — keep the earliest row, drop the rest.
+  try {
+    db.exec(`
+      DELETE FROM decisions WHERE id NOT IN (
+        SELECT MIN(id) FROM decisions
+        GROUP BY COALESCE(project_path, ''), title
+      )
+    `);
+  } catch {
+    // table missing on first run
+  }
+
+  // Once duplicates are gone, enforce the constraint going forward. Combined
+  // with INSERT OR IGNORE in decisions.ts this is what makes Bug 3
+  // (decisions duplicated across sessions) actually go away.
+  try {
+    db.exec(`
+      CREATE UNIQUE INDEX IF NOT EXISTS idx_decisions_project_title
+        ON decisions(COALESCE(project_path, ''), title)
+    `);
+  } catch {
+    // index already exists or unsupported expression-index syntax
+  }
+
+  // FTS5 virtual tables for full-text search
+  try {
+    db.exec(`
+      CREATE VIRTUAL TABLE IF NOT EXISTS observations_fts USING fts5(
+        summary,
+        content='observations',
+        content_rowid='id'
+      );
+
+      CREATE VIRTUAL TABLE IF NOT EXISTS decisions_fts USING fts5(
+        title,
+        decision_text,
+        content='decisions',
+        content_rowid='id'
+      );
+    `);
+  } catch {
+    // FTS5 tables may already exist
+  }
+
+  // Triggers to keep FTS in sync
+  try {
+    db.exec(`
+      CREATE TRIGGER IF NOT EXISTS observations_ai AFTER INSERT ON observations BEGIN
+        INSERT INTO observations_fts(rowid, summary) VALUES (new.id, new.summary);
+      END;
+
+      CREATE TRIGGER IF NOT EXISTS decisions_ai AFTER INSERT ON decisions BEGIN
+        INSERT INTO decisions_fts(rowid, title, decision_text) VALUES (new.id, new.title, new.decision_text);
+      END;
+    `);
+  } catch {
+    // Triggers may already exist
+  }
+}
+
+function cleanupLegacyPatterns(db: Database) {
+  const VOLATILE_TAIL = / \((?:\d+ times this session|\d+\/\d+ actions)\)\s*$/;
+  const rows = db.prepare(
+    `SELECT id, project_path, pattern_type, description, frequency, last_seen, examples FROM patterns`
+  ).all() as Array<{
+    id: number;
+    project_path: string | null;
+    pattern_type: string;
+    description: string;
+    frequency: number;
+    last_seen: string | null;
+    examples: string | null;
+  }>;
+
+  type Group = { keepId: number; freq: number; idsToDelete: number[]; latestExamples: string | null; latestSeen: string | null };
+  const groups = new Map<string, Group>();
+
+  for (const r of rows) {
+    const cleaned = r.description.replace(VOLATILE_TAIL, "").trim();
+    if (cleaned !== r.description) {
+      db.prepare(`UPDATE patterns SET description = ? WHERE id = ?`).run(cleaned, r.id);
+      r.description = cleaned;
+    }
+    const key = `${r.project_path ?? ""}${r.pattern_type}${r.description}`;
+    const existing = groups.get(key);
+    if (!existing) {
+      groups.set(key, {
+        keepId: r.id,
+        freq: r.frequency,
+        idsToDelete: [],
+        latestExamples: r.examples,
+        latestSeen: r.last_seen,
+      });
+    } else {
+      existing.freq += r.frequency;
+      if ((r.last_seen || "") > (existing.latestSeen || "")) {
+        existing.latestSeen = r.last_seen;
+        existing.latestExamples = r.examples;
+      }
+      existing.idsToDelete.push(r.id);
+    }
+  }
+
+  for (const g of groups.values()) {
+    if (g.idsToDelete.length === 0) continue;
+    db.prepare(
+      `UPDATE patterns SET frequency = ?, last_seen = COALESCE(?, last_seen), examples = COALESCE(?, examples) WHERE id = ?`
+    ).run(g.freq, g.latestSeen, g.latestExamples, g.keepId);
+    const placeholders = g.idsToDelete.map(() => "?").join(",");
+    db.prepare(`DELETE FROM patterns WHERE id IN (${placeholders})`).run(...g.idsToDelete);
+  }
+}
+
+export function closeDb() {
+  if (_db) {
+    _db.close();
+    _db = null;
+  }
+}
+
+// Run directly to initialize
+if (import.meta.main) {
+  const db = getDb();
+  console.log("Database initialized at", DB_PATH);
+  const tables = db.prepare("SELECT name FROM sqlite_master WHERE type='table'").all() as Array<{ name: string }>;
+  console.log("  Tables:", tables.map(r => r.name).join(", "));
+  closeDb();
+}

package/src/db/sqlite/sessions.ts

@@ -0,0 +1,50 @@
+import { getDb } from "./schema";
+
+export interface Session {
+  id?: number;
+  session_id: string;
+  project_path?: string;
+  git_repo?: string;
+  git_branch?: string;
+  started_at?: string;
+  ended_at?: string;
+  total_observations?: number;
+  tokens_injected?: number;
+}
+
+export function createSession(session: Session): void {
+  const db = getDb();
+  db.prepare(`
+    INSERT OR IGNORE INTO sessions (session_id, project_path, git_repo, git_branch)
+    VALUES (?, ?, ?, ?)
+  `).run(
+    session.session_id,
+    session.project_path || null,
+    session.git_repo || null,
+    session.git_branch || null
+  );
+}
+
+export function endSession(sessionId: string, totalObs: number) {
+  const db = getDb();
+  db.prepare(`
+    UPDATE sessions SET ended_at = datetime('now'), total_observations = ? WHERE session_id = ?
+  `).run(totalObs, sessionId);
+}
+
+export function getRecentSessions(projectPath: string, limit = 3): Session[] {
+  const db = getDb();
+  return db.prepare(`
+    SELECT * FROM sessions WHERE project_path = ? ORDER BY started_at DESC LIMIT ?
+  `).all(projectPath, limit) as Session[];
+}
+
+export function getSessionById(sessionId: string): Session | undefined {
+  const db = getDb();
+  return db.prepare(`SELECT * FROM sessions WHERE session_id = ?`).get(sessionId) as Session | undefined;
+}
+
+export function getAllSessions(limit = 50): Session[] {
+  const db = getDb();
+  return db.prepare(`SELECT * FROM sessions ORDER BY started_at DESC LIMIT ?`).all(limit) as Session[];
+}

package/src/db/sqlite/summaries.ts

@@ -0,0 +1,69 @@
+import { getDb } from "./schema";
+
+export interface Summary {
+  id?: number;
+  session_id?: string;
+  level: number;
+  text: string;
+  embedding_id?: string;
+  project_path?: string;
+  created_at?: string;
+}
+
+export function insertSummary(summary: Summary): void {
+  const db = getDb();
+  // For per-session summaries, replace any existing (session_id, level) row so a
+  // Stop hook that fires more than once doesn't duplicate. SQLite doesn't
+  // accept ON CONFLICT against the partial unique index used here, so we
+  // do the delete + insert explicitly inside a transaction. Digests have a
+  // null session_id and just append.
+  if (summary.session_id) {
+    const tx = db.transaction((s: Summary) => {
+      db.prepare(`DELETE FROM summaries WHERE session_id = ? AND level = ?`).run(
+        s.session_id!,
+        s.level
+      );
+      db.prepare(`
+        INSERT INTO summaries (session_id, level, text, embedding_id, project_path)
+        VALUES (?, ?, ?, ?, ?)
+      `).run(
+        s.session_id!,
+        s.level,
+        s.text,
+        s.embedding_id || null,
+        s.project_path || null
+      );
+    });
+    tx(summary);
+    return;
+  }
+  db.prepare(`
+    INSERT INTO summaries (session_id, level, text, embedding_id, project_path)
+    VALUES (?, ?, ?, ?, ?)
+  `).run(
+    null,
+    summary.level,
+    summary.text,
+    summary.embedding_id || null,
+    summary.project_path || null
+  );
+}
+
+export function getRecentSummaries(projectPath: string, level = 1, limit = 3): Summary[] {
+  const db = getDb();
+  return db.prepare(`
+    SELECT * FROM summaries
+    WHERE project_path = ? AND level = ?
+    ORDER BY created_at DESC
+    LIMIT ?
+  `).all(projectPath, level, limit) as Summary[];
+}
+
+export function getAllSummariesForDigest(projectPath: string, since: string): Summary[] {
+  const db = getDb();
+  return db.prepare(`
+    SELECT * FROM summaries
+    WHERE project_path = ? AND level = 1 AND created_at >= ?
+    ORDER BY created_at ASC
+  `).all(projectPath, since) as Summary[];
+}

package/src/db/vector/client.ts

@@ -0,0 +1,134 @@
+// LanceDB vector database client for semantic search.
+//
+// The table holds vectors at whatever dimension the active embedding provider
+// emits. If a query vector arrives with a mismatched dim (e.g. user just
+// switched providers without re-embedding), searchSimilar returns [] rather
+// than throwing — the CLI's `embeddings switch` command rebuilds the table.
+
+import * as lancedb from "@lancedb/lancedb";
+import { join } from "path";
+import { mkdirSync } from "fs";
+import { getActiveDim } from "./embeddings";
+
+const DATA_DIR = process.env.AUTOCTXD_VECTOR_DIR
+  || (process.env.AUTOCTXD_DATA_DIR ? join(process.env.AUTOCTXD_DATA_DIR, "vector") : join(import.meta.dir, "..", "..", "..", "data", "vector"));
+const TABLE_NAME = "summaries";
+
+let _db: any = null;
+let _table: any = null;
+let _tableDim: number | null = null;
+
+export async function getVectorDb() {
+  if (_db) return _db;
+  mkdirSync(DATA_DIR, { recursive: true });
+  _db = await lancedb.connect(DATA_DIR);
+  return _db;
+}
+
+async function detectTableDim(table: any): Promise<number | null> {
+  try {
+    const sample = await table.query().limit(1).toArray();
+    const v = sample?.[0]?.vector;
+    if (Array.isArray(v)) return v.length;
+    if (v && typeof v.length === "number") return v.length;
+  } catch {
+    // ignore
+  }
+  return null;
+}
+
+export async function getOrCreateTable() {
+  const expectedDim = getActiveDim();
+  if (_table && _tableDim === expectedDim) return _table;
+
+  const db = await getVectorDb();
+
+  try {
+    _table = await db.openTable(TABLE_NAME);
+    _tableDim = await detectTableDim(_table);
+    // If existing table was built for a different dim, signal to caller via
+    // _tableDim so dropTable() can be invoked from the CLI migration path.
+  } catch {
+    _table = await db.createTable(TABLE_NAME, [
+      {
+        id: "__init__",
+        session_id: "",
+        project_path: "",
+        text: "",
+        level: 0,
+        created_at: "",
+        vector: new Array(expectedDim).fill(0),
+      },
+    ]);
+    _tableDim = expectedDim;
+  }
+
+  return _table;
+}
+
+export async function dropTable(): Promise<void> {
+  const db = await getVectorDb();
+  try {
+    await db.dropTable(TABLE_NAME);
+  } catch {
+    // table may not exist
+  }
+  _table = null;
+  _tableDim = null;
+}
+
+export function getCurrentTableDim(): number | null {
+  return _tableDim;
+}
+
+export interface VectorRecord {
+  id: string;
+  session_id: string;
+  project_path: string;
+  text: string;
+  level: number;
+  created_at: string;
+  vector: number[];
+}
+
+export async function addVector(record: VectorRecord): Promise<void> {
+  const table = await getOrCreateTable();
+  await table.add([record]);
+}
+
+export async function searchSimilar(
+  queryVector: number[],
+  limit = 5,
+  projectFilter?: string
+): Promise<VectorRecord[]> {
+  const table = await getOrCreateTable();
+
+  let query = table.search(queryVector).limit(limit);
+
+  if (projectFilter) {
+    query = query.where(`project_path = '${projectFilter.replace(/'/g, "''")}'`);
+  }
+
+  try {
+    const results = await query.toArray();
+    return results
+      .filter((r: any) => r.id !== "__init__")
+      .map((r: any) => ({
+        id: r.id,
+        session_id: r.session_id,
+        project_path: r.project_path,
+        text: r.text,
+        level: r.level,
+        created_at: r.created_at,
+        vector: r.vector,
+        _distance: r._distance,
+      }));
+  } catch {
+    return [];
+  }
+}
+
+export async function closeVectorDb(): Promise<void> {
+  _table = null;
+  _db = null;
+}