@drewpayment/mink 0.11.0 → 0.12.0-beta.2

package/src/repositories/bug-memory-repo.ts

@@ -0,0 +1,268 @@
+// Bug-memory repository. Wraps the bug_memory + bug_tags + bug_related +
+// bug_memory_fts tables in mink.db. The CLI surface (loadBugMemory,
+// findDuplicate, lookupBugsForFile, searchBugs, hasBugForFileInSession)
+// is preserved by the thin wrapper in src/core/bug-memory.ts; this file
+// is where the SQLite queries live.
+//
+// Search uses FTS5 (porter+unicode61 tokenization) so the per-query cost
+// stays sublinear in bug count. The score-vs-false-positive guards from
+// the v1 Jaccard implementation are preserved: a 0.3 score threshold,
+// file-path or tag-overlap match required when score is borderline,
+// same-file matches get a 0.2 boost.
+
+import { randomUUID } from "crypto";
+import type { DbDriver } from "../storage/driver";
+import type { BugEntry, BugMemory, SimilarityMatch } from "../types/bug-memory";
+import { openProjectDb } from "../storage/db";
+import { getOrCreateDeviceId } from "../core/device";
+
+interface BugRow {
+  id: string;
+  created_at: string;
+  last_seen_at: string;
+  error_message: string;
+  file_path: string;
+  line_number: number | null;
+  root_cause: string;
+  fix_description: string;
+  occurrence_count: number;
+}
+
+function tokenize(text: string): Set<string> {
+  return new Set(
+    text.toLowerCase().split(/\W+/).filter((w) => w.length > 0)
+  );
+}
+
+export class BugMemoryRepo {
+  constructor(private readonly db: DbDriver) {}
+
+  static for(cwd: string): BugMemoryRepo {
+    return new BugMemoryRepo(openProjectDb(cwd));
+  }
+
+  // ── Insert / upsert ────────────────────────────────────────────────────
+
+  // Detect an exact-text duplicate of (errorMessage, filePath). Mirrors
+  // the v1 `findDuplicate` semantics — same (errorMessage, filePath)
+  // pair counts as a re-occurrence of the same bug.
+  findDuplicate(errorMessage: string, filePath: string): BugEntry | null {
+    const row = this.db
+      .prepare(
+        "SELECT * FROM bug_memory WHERE error_message = ? AND file_path = ? LIMIT 1"
+      )
+      .get(errorMessage, filePath);
+    if (!row) return null;
+    return this.hydrate(row as unknown as BugRow);
+  }
+
+  add(
+    fields: Omit<BugEntry, "id" | "createdAt" | "lastSeenAt" | "occurrenceCount">
+  ): BugEntry {
+    const existing = this.findDuplicate(fields.errorMessage, fields.filePath);
+    if (existing) {
+      this.incrementOccurrence(existing.id);
+      return this.lookup(existing.id) ?? existing;
+    }
+
+    const id = `bug-${randomUUID().slice(0, 8)}`;
+    const now = new Date().toISOString();
+    const deviceId = getOrCreateDeviceId();
+
+    this.db.transaction(() => {
+      this.db.prepare(`
+        INSERT INTO bug_memory
+          (id, created_at, last_seen_at, error_message, file_path, line_number,
+           root_cause, fix_description, occurrence_count, device_id)
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?, 1, ?)
+      `).run(
+        id, now, now,
+        fields.errorMessage, fields.filePath, fields.lineNumber ?? null,
+        fields.rootCause, fields.fixDescription, deviceId
+      );
+      const insertTag = this.db.prepare(
+        "INSERT OR IGNORE INTO bug_tags (bug_id, tag) VALUES (?, ?)"
+      );
+      for (const tag of fields.tags ?? []) insertTag.run(id, tag);
+      const insertRelated = this.db.prepare(
+        "INSERT OR IGNORE INTO bug_related (bug_id, related_bug_id) VALUES (?, ?)"
+      );
+      for (const rel of fields.relatedBugIds ?? []) insertRelated.run(id, rel);
+    });
+
+    return this.lookup(id)!;
+  }
+
+  incrementOccurrence(id: string): void {
+    const now = new Date().toISOString();
+    this.db.prepare(
+      "UPDATE bug_memory SET occurrence_count = occurrence_count + 1, last_seen_at = ? WHERE id = ?"
+    ).run(now, id);
+  }
+
+  // ── Read ───────────────────────────────────────────────────────────────
+
+  lookup(id: string): BugEntry | null {
+    const row = this.db
+      .prepare("SELECT * FROM bug_memory WHERE id = ?")
+      .get(id);
+    if (!row) return null;
+    return this.hydrate(row as unknown as BugRow);
+  }
+
+  lookupForFile(filePath: string): BugEntry[] {
+    const rows = this.db
+      .prepare(
+        "SELECT * FROM bug_memory WHERE file_path = ? ORDER BY last_seen_at DESC"
+      )
+      .all(filePath);
+    return (rows as unknown as BugRow[]).map((r) => this.hydrate(r));
+  }
+
+  listAll(): BugEntry[] {
+    const rows = this.db
+      .prepare("SELECT * FROM bug_memory ORDER BY last_seen_at DESC")
+      .all();
+    return (rows as unknown as BugRow[]).map((r) => this.hydrate(r));
+  }
+
+  count(): number {
+    const row = this.db.prepare("SELECT COUNT(*) AS n FROM bug_memory").get();
+    return Number((row as { n: number }).n);
+  }
+
+  hasBugForFileInSession(filePath: string, sessionStartIso: string): boolean {
+    const row = this.db
+      .prepare(
+        "SELECT 1 FROM bug_memory WHERE file_path = ? AND created_at >= ? LIMIT 1"
+      )
+      .get(filePath, sessionStartIso);
+    return row !== undefined;
+  }
+
+  // ── Search (FTS5) ──────────────────────────────────────────────────────
+
+  // Preserves the v1 contract: scores in (0, 1+) range, 0.3 threshold,
+  // file-path/tag boost. FTS5's bm25 returns negative scores (lower =
+  // better), so we normalize via `1 / (1 + abs(bm25))` to land in (0, 1].
+  // The boost for same-file matches stays at +0.2 and the same false-
+  // positive guards (require file-path or tag overlap when borderline)
+  // apply.
+  searchBugs(
+    query: string,
+    options?: { filePath?: string }
+  ): SimilarityMatch[] {
+    if (query.trim().length === 0) return [];
+
+    // FTS5 MATCH requires escaped phrase quoting for queries with
+    // punctuation. Build a phrase query if the input has anything
+    // weirder than alphanum + spaces.
+    const ftsQuery = buildFtsQuery(query);
+    if (ftsQuery === null) return [];
+
+    type FtsRow = { bug_id: string; bm25: number };
+    let ftsRows: FtsRow[] = [];
+    try {
+      ftsRows = this.db
+        .prepare(
+          "SELECT bug_id, bm25(bug_memory_fts) AS bm25 FROM bug_memory_fts WHERE bug_memory_fts MATCH ? ORDER BY bm25"
+        )
+        .all(ftsQuery) as unknown as FtsRow[];
+    } catch {
+      // FTS query parse error — fall back silently to no matches.
+      return [];
+    }
+
+    const queryTokens = tokenize(query);
+    const results: SimilarityMatch[] = [];
+
+    for (const row of ftsRows) {
+      const entry = this.lookup(row.bug_id);
+      if (!entry) continue;
+
+      // bm25 is negative; smaller magnitude == better match.
+      const ftsScore = 1 / (1 + Math.abs(row.bm25));
+      const matchReasons: string[] = ["fts"];
+
+      // Exact substring boost (matches v1 behavior).
+      let score = ftsScore;
+      if (entry.errorMessage.length > 0 && entry.errorMessage.includes(query)) {
+        score += 1.0;
+        matchReasons.unshift("exact_error_match");
+      }
+
+      const fileMatch = options?.filePath ? entry.filePath === options.filePath : false;
+      const tagMatch = entry.tags.some((tag) => queryTokens.has(tag.toLowerCase()));
+
+      // Same false-positive guard as v1: when the score is borderline
+      // (<= 0.3), only keep matches that also satisfy file-path or
+      // tag-overlap.
+      if (score <= 0.3 && !fileMatch && !tagMatch) continue;
+
+      if (fileMatch) {
+        score += 0.2;
+        matchReasons.push("file_path");
+      }
+      if (tagMatch) matchReasons.push("tags");
+
+      if (score > 0.3) {
+        results.push({ entry, score, matchReasons });
+      }
+    }
+
+    return results.sort((a, b) => b.score - a.score);
+  }
+
+  // ── Helpers ────────────────────────────────────────────────────────────
+
+  private hydrate(row: BugRow): BugEntry {
+    const tags = this.db
+      .prepare("SELECT tag FROM bug_tags WHERE bug_id = ? ORDER BY tag")
+      .all(row.id)
+      .map((r) => (r as { tag: string }).tag);
+    const relatedBugIds = this.db
+      .prepare(
+        "SELECT related_bug_id FROM bug_related WHERE bug_id = ? ORDER BY related_bug_id"
+      )
+      .all(row.id)
+      .map((r) => (r as { related_bug_id: string }).related_bug_id);
+    return {
+      id: row.id,
+      createdAt: row.created_at,
+      lastSeenAt: row.last_seen_at,
+      errorMessage: row.error_message,
+      filePath: row.file_path,
+      lineNumber: row.line_number ?? undefined,
+      rootCause: row.root_cause,
+      fixDescription: row.fix_description,
+      tags,
+      occurrenceCount: row.occurrence_count,
+      relatedBugIds,
+    };
+  }
+
+  // Convert the entire repo to the legacy BugMemory snapshot shape. Used
+  // by callers (dashboard, status) that still expect `{ entries, nextId }`.
+  snapshot(): BugMemory {
+    return {
+      entries: this.listAll(),
+      // nextId was only used by the in-memory generator; new ids come
+      // from randomUUID, so any value > current count is safe.
+      nextId: this.count() + 1,
+    };
+  }
+}
+
+// Build an FTS5 query string from arbitrary user input. FTS5's grammar
+// treats colons, parens, quotes, etc. as operators — we phrase-quote the
+// whole query to avoid syntax errors. Returns null for inputs that have
+// no searchable tokens.
+function buildFtsQuery(raw: string): string | null {
+  const trimmed = raw.trim();
+  if (trimmed.length === 0) return null;
+  // Drop characters that can't appear inside FTS5 phrase quotes.
+  const safe = trimmed.replace(/"/g, " ").trim();
+  if (safe.length === 0) return null;
+  // Quote so punctuation/colons/parens don't become operators.
+  return `"${safe}"`;
+}

package/src/repositories/counters-repo.ts

@@ -0,0 +1,88 @@
+// Per-device hit/miss counters for the file index. Replaces the legacy
+// `.mink-state-counters.json` file (read by the dashboard and by
+// `mink status`) with a SQLite-backed table that's queryable per device
+// or aggregated across all devices in a single SQL statement.
+
+import type { DbDriver } from "../storage/driver";
+import { openProjectDb } from "../storage/db";
+import { getOrCreateDeviceId } from "../core/device";
+
+const INCREMENT_HIT = `
+  INSERT INTO counters (device_id, file_index_hits, file_index_misses)
+  VALUES (?, 1, 0)
+  ON CONFLICT(device_id) DO UPDATE SET
+    file_index_hits = counters.file_index_hits + 1
+`;
+
+const INCREMENT_MISS = `
+  INSERT INTO counters (device_id, file_index_hits, file_index_misses)
+  VALUES (?, 0, 1)
+  ON CONFLICT(device_id) DO UPDATE SET
+    file_index_misses = counters.file_index_misses + 1
+`;
+
+export class CountersRepo {
+  constructor(private readonly db: DbDriver) {}
+
+  static for(cwd: string): CountersRepo {
+    return new CountersRepo(openProjectDb(cwd));
+  }
+
+  incrementHit(deviceId: string = getOrCreateDeviceId()): void {
+    this.db.prepare(INCREMENT_HIT).run(deviceId);
+  }
+
+  incrementMiss(deviceId: string = getOrCreateDeviceId()): void {
+    this.db.prepare(INCREMENT_MISS).run(deviceId);
+  }
+
+  // Returns this device's hit + miss counts (zero for either if no row
+  // exists yet). The dashboard and `mink status` show per-device totals,
+  // but callers that want a project-wide view use totals().
+  forDevice(deviceId: string = getOrCreateDeviceId()): { hits: number; misses: number } {
+    const row = this.db
+      .prepare(
+        "SELECT file_index_hits, file_index_misses FROM counters WHERE device_id = ?"
+      )
+      .get(deviceId);
+    if (!row) return { hits: 0, misses: 0 };
+    return {
+      hits: Number((row as { file_index_hits: number }).file_index_hits),
+      misses: Number((row as { file_index_misses: number }).file_index_misses),
+    };
+  }
+
+  totals(): { hits: number; misses: number } {
+    const row = this.db
+      .prepare(
+        "SELECT COALESCE(SUM(file_index_hits), 0) AS h, COALESCE(SUM(file_index_misses), 0) AS m FROM counters"
+      )
+      .get();
+    if (!row) return { hits: 0, misses: 0 };
+    return {
+      hits: Number((row as { h: number }).h),
+      misses: Number((row as { m: number }).m),
+    };
+  }
+
+  perDevice(): Record<string, { hits: number; misses: number }> {
+    const rows = this.db
+      .prepare(
+        "SELECT device_id, file_index_hits, file_index_misses FROM counters"
+      )
+      .all();
+    const out: Record<string, { hits: number; misses: number }> = {};
+    for (const r of rows) {
+      const row = r as {
+        device_id: string;
+        file_index_hits: number;
+        file_index_misses: number;
+      };
+      out[row.device_id] = {
+        hits: Number(row.file_index_hits),
+        misses: Number(row.file_index_misses),
+      };
+    }
+    return out;
+  }
+}

package/src/repositories/file-index-repo.ts

@@ -0,0 +1,238 @@
+// File-index repository. Wraps the file_index table in `mink.db` behind a
+// stable function-based API that the wrapper in src/core/index-store.ts
+// delegates to. Hook hot paths (pre-read, post-write) call exactly one
+// method per hook invocation — no full-index load.
+//
+// All writes attribute the calling device via device_id so the cross-device
+// sync merge driver (mink-db-merge) can reconcile origin. Counters that
+// were previously kept in file-index.json's header (lifetimeHits /
+// lifetimeMisses) live in the `counters` table indexed by device_id.
+
+import type { DbDriver } from "../storage/driver";
+import type { FileIndexEntry, StalenessReport } from "../types/file-index";
+import { openProjectDb } from "../storage/db";
+import { getOrCreateDeviceId } from "../core/device";
+
+interface FileIndexRow {
+  file_path: string;
+  description: string;
+  estimated_tokens: number;
+  last_modified: string;
+  last_indexed: string;
+  mtime_ms: number;
+  content_hash: string | null;
+  size_bytes: number | null;
+  device_id: string;
+}
+
+function rowToEntry(row: FileIndexRow): FileIndexEntry {
+  return {
+    filePath: row.file_path,
+    description: row.description,
+    estimatedTokens: row.estimated_tokens,
+    lastModified: row.last_modified,
+    lastIndexed: row.last_indexed,
+  };
+}
+
+// Mirror of upsertEntry's semantics under the JSON store, expressed as a
+// single SQL upsert. Conflict resolution picks the more recent
+// last_modified — matches the merge driver's per-row rule so a hook that
+// runs concurrently with sync converges deterministically.
+const UPSERT_SQL = `
+  INSERT INTO file_index
+    (file_path, description, estimated_tokens, last_modified, last_indexed,
+     mtime_ms, content_hash, size_bytes, device_id)
+  VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+  ON CONFLICT(file_path) DO UPDATE SET
+    description       = excluded.description,
+    estimated_tokens  = excluded.estimated_tokens,
+    last_modified     = excluded.last_modified,
+    last_indexed      = excluded.last_indexed,
+    mtime_ms          = excluded.mtime_ms,
+    content_hash      = COALESCE(excluded.content_hash, file_index.content_hash),
+    size_bytes        = COALESCE(excluded.size_bytes,   file_index.size_bytes),
+    device_id         = excluded.device_id
+`;
+
+export interface UpsertOptions {
+  mtimeMs?: number;
+  contentHash?: string | null;
+  sizeBytes?: number | null;
+  deviceId?: string;
+}
+
+export interface IndexLookup {
+  lookupEntry(filePath: string): FileIndexEntry | null;
+}
+
+export class FileIndexRepo implements IndexLookup {
+  constructor(private readonly db: DbDriver) {}
+
+  static for(cwd: string): FileIndexRepo {
+    return new FileIndexRepo(openProjectDb(cwd));
+  }
+
+  upsert(entry: FileIndexEntry, opts: UpsertOptions = {}): void {
+    const deviceId = opts.deviceId ?? getOrCreateDeviceId();
+    this.db.prepare(UPSERT_SQL).run(
+      entry.filePath,
+      entry.description,
+      entry.estimatedTokens,
+      entry.lastModified,
+      entry.lastIndexed,
+      opts.mtimeMs ?? 0,
+      opts.contentHash ?? null,
+      opts.sizeBytes ?? null,
+      deviceId
+    );
+  }
+
+  // Bulk upsert — used by `mink scan` to push hundreds-to-thousands of
+  // entries in a single transaction. ~50x faster than individual upserts
+  // because SQLite skips per-row WAL fsync.
+  upsertMany(entries: Array<{ entry: FileIndexEntry; opts?: UpsertOptions }>): void {
+    if (entries.length === 0) return;
+    const defaultDevice = getOrCreateDeviceId();
+    const stmt = this.db.prepare(UPSERT_SQL);
+    this.db.transaction(() => {
+      for (const { entry, opts } of entries) {
+        stmt.run(
+          entry.filePath,
+          entry.description,
+          entry.estimatedTokens,
+          entry.lastModified,
+          entry.lastIndexed,
+          opts?.mtimeMs ?? 0,
+          opts?.contentHash ?? null,
+          opts?.sizeBytes ?? null,
+          opts?.deviceId ?? defaultDevice
+        );
+      }
+    });
+  }
+
+  lookupEntry(filePath: string): FileIndexEntry | null {
+    const row = this.db
+      .prepare(
+        "SELECT file_path, description, estimated_tokens, last_modified, last_indexed, mtime_ms, content_hash, size_bytes, device_id FROM file_index WHERE file_path = ?"
+      )
+      .get(filePath);
+    if (!row) return null;
+    return rowToEntry(row as unknown as FileIndexRow);
+  }
+
+  remove(filePath: string): void {
+    this.db.prepare("DELETE FROM file_index WHERE file_path = ?").run(filePath);
+  }
+
+  // Remove every entry that's NOT in `keep`. Used by `mink scan` to prune
+  // orphaned entries for files that have been deleted from disk. Expressed
+  // as a single statement using a temp table to avoid the SQLite parameter
+  // limit (defaults to 999).
+  retainOnly(keep: Iterable<string>): number {
+    const keepArr = [...keep];
+    this.db.transaction(() => {
+      this.db.exec("CREATE TEMP TABLE IF NOT EXISTS _retain (path TEXT PRIMARY KEY)");
+      this.db.exec("DELETE FROM _retain");
+      const stmt = this.db.prepare("INSERT OR IGNORE INTO _retain VALUES (?)");
+      for (const p of keepArr) stmt.run(p);
+    });
+    const r = this.db
+      .prepare("DELETE FROM file_index WHERE file_path NOT IN (SELECT path FROM _retain)")
+      .run();
+    return Number(r.changes);
+  }
+
+  // Total count of indexed files. Cheap — backed by the PRIMARY KEY index.
+  totalFiles(): number {
+    const row = this.db.prepare("SELECT COUNT(*) AS n FROM file_index").get();
+    return Number((row as { n: number }).n);
+  }
+
+  // Bulk list — used by analytics, dashboard, and `mink status`. Stays
+  // off the hook hot path. Returns rows already shaped to the public
+  // FileIndexEntry type.
+  listAll(): FileIndexEntry[] {
+    const rows = this.db
+      .prepare(
+        "SELECT file_path, description, estimated_tokens, last_modified, last_indexed, mtime_ms, content_hash, size_bytes, device_id FROM file_index ORDER BY file_path"
+      )
+      .all();
+    return rows.map((r) => rowToEntry(r as unknown as FileIndexRow));
+  }
+
+  // For Phase 5's incremental scan. Returns the subset of `scanned` whose
+  // mtime differs from what we have stored — i.e. needs re-extraction.
+  // Done as one query per chunk to avoid a 20k-row IN list, but still much
+  // cheaper than reading every file's content.
+  staleSet(scanned: Array<{ relativePath: string; mtimeMs: number }>): string[] {
+    if (scanned.length === 0) return [];
+    const stmt = this.db.prepare(
+      "SELECT mtime_ms FROM file_index WHERE file_path = ?"
+    );
+    const stale: string[] = [];
+    for (const f of scanned) {
+      const row = stmt.get(f.relativePath);
+      if (!row) {
+        stale.push(f.relativePath); // never seen before
+        continue;
+      }
+      const storedMs = Number((row as { mtime_ms: number }).mtime_ms);
+      if (storedMs !== Math.floor(f.mtimeMs)) {
+        stale.push(f.relativePath);
+      }
+    }
+    return stale;
+  }
+
+  // Cheap content_hash lookup — used by `mink scan` to detect
+  // touch-without-edit cases (mtime changed, content didn't) and skip
+  // re-extraction in that case.
+  contentHashFor(filePath: string): string | null {
+    const row = this.db
+      .prepare("SELECT content_hash FROM file_index WHERE file_path = ?")
+      .get(filePath);
+    if (!row) return null;
+    const hash = (row as { content_hash: string | null }).content_hash;
+    return hash ?? null;
+  }
+
+  // Mirrors checkStaleness() under the JSON store: which files are on disk
+  // but not in the index (missing), and which are in the index but absent
+  // from disk (orphaned).
+  checkStaleness(scannedRelativePaths: string[]): StalenessReport {
+    const scannedSet = new Set(scannedRelativePaths);
+    const allIndexed = this.db
+      .prepare("SELECT file_path FROM file_index")
+      .all()
+      .map((r) => (r as { file_path: string }).file_path);
+    const indexedSet = new Set(allIndexed);
+    const missingFromIndex = scannedRelativePaths.filter((p) => !indexedSet.has(p));
+    const orphanedEntries = allIndexed.filter((p) => !scannedSet.has(p));
+    return {
+      missingFromIndex,
+      orphanedEntries,
+      isStale: missingFromIndex.length > 0 || orphanedEntries.length > 0,
+    };
+  }
+
+  // Header analogues. lastScanTimestamp is the only header field that's
+  // genuinely a project-wide state value; hit/miss counters live in the
+  // counters table and are per-device. Stored in the meta table.
+  setLastScanTimestamp(iso: string): void {
+    this.db
+      .prepare(
+        "INSERT INTO meta (key, value) VALUES ('last_scan_timestamp', ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value"
+      )
+      .run(iso);
+  }
+
+  getLastScanTimestamp(): string {
+    const row = this.db
+      .prepare("SELECT value FROM meta WHERE key = 'last_scan_timestamp'")
+      .get();
+    if (!row) return "";
+    return String((row as { value: string }).value);
+  }
+}