@drewpayment/mink 0.11.0-beta.4 → 0.12.0-beta.1

package/src/commands/status.ts

@@ -1,18 +1,14 @@
 import { existsSync, readFileSync, statSync } from "fs";
 import {
   sessionPath,
-  fileIndexPath,
+  projectDbPath,
   configPath,
   learningMemoryPath,
-  tokenLedgerPath,
-  bugMemoryPath,
   actionLogPath,
 } from "../core/paths";
 import { safeReadJson } from "../core/fs-utils";
-import { isFileIndex } from "../core/index-store";
-import { loadLedger } from "../core/token-ledger";
-import { parseLearningMemory, totalEntryCount } from "../core/learning-memory";
-import { loadBugMemory } from "../core/bug-memory";
+import { FileIndexRepo } from "../repositories/file-index-repo";
+import { openProjectDb } from "../storage/db";
 import {
   aggregateTokenLedger,
   aggregateBugMemory,
@@ -20,6 +16,7 @@ import {
 } from "../core/state-aggregator";
 import { loadCounters } from "../core/state-counters";
 import { getDaemonStatus } from "../core/daemon";
+import { totalEntryCount } from "../core/learning-memory";
 
 interface FileCheck {
   name: string;
@@ -45,18 +42,42 @@ function checkTextFile(name: string, filePath: string): FileCheck {
   }
 }
 
+function checkDbFile(name: string, filePath: string): FileCheck {
+  if (!existsSync(filePath)) return { name, path: filePath, status: "missing" };
+  try {
+    const header = readFileSync(filePath).slice(0, 16).toString("utf-8");
+    if (!header.startsWith("SQLite format 3")) {
+      return { name, path: filePath, status: "corrupt" };
+    }
+    return { name, path: filePath, status: "ok" };
+  } catch {
+    return { name, path: filePath, status: "corrupt" };
+  }
+}
+
 export function status(cwd: string): void {
   console.log("[mink] project status");
   console.log();
 
-  // Section 1: State directory integrity
+  // Open the project DB up-front so the lazy JSON → SQLite migration
+  // runs before checkDbFile inspects mink.db. Otherwise the integrity
+  // section would always report "missing" on a project that has only
+  // legacy JSON state.
+  try {
+    openProjectDb(cwd);
+  } catch {
+    // Migration failures are non-fatal — report the DB as corrupt below.
+  }
+
+  // Section 1: State directory integrity. file-index, bug-memory, and
+  // token-ledger all live inside mink.db. The remaining JSON/MD files
+  // are intentionally separate — session is per-process, config is
+  // user-editable, learning-memory and action-log are human-readable.
   const checks: FileCheck[] = [
     checkJsonFile("session.json", sessionPath(cwd)),
-    checkJsonFile("file-index.json", fileIndexPath(cwd), isFileIndex),
+    checkDbFile("mink.db", projectDbPath(cwd)),
     checkJsonFile("config.json", configPath(cwd)),
     checkTextFile("learning-memory.md", learningMemoryPath(cwd)),
-    checkJsonFile("token-ledger.json", tokenLedgerPath(cwd)),
-    checkJsonFile("bug-memory.json", bugMemoryPath(cwd)),
     checkTextFile("action-log.md", actionLogPath(cwd)),
   ];
 
@@ -73,24 +94,22 @@ export function status(cwd: string): void {
   }
   console.log();
 
-  // Section 2: File index
+  // Section 2: File index — sourced from mink.db.
   try {
-    const raw = safeReadJson(fileIndexPath(cwd));
-    if (raw && isFileIndex(raw)) {
-      const h = raw.header;
-      // Hit/miss counters live in the per-device counter file, fall back to
-      // legacy header counters for unmigrated repos.
+    const repo = FileIndexRepo.for(cwd);
+    const total = repo.totalFiles();
+    if (total === 0) {
+      console.log("  File index: not available");
+    } else {
       const counters = loadCounters(cwd);
-      const hits = counters.fileIndexHits || h.lifetimeHits;
-      const misses = counters.fileIndexMisses || h.lifetimeMisses;
-      const total = hits + misses;
-      const ratio = total > 0 ? ((hits / total) * 100).toFixed(1) : "N/A";
+      const hits = counters.fileIndexHits;
+      const misses = counters.fileIndexMisses;
+      const totalLookups = hits + misses;
+      const ratio = totalLookups > 0 ? ((hits / totalLookups) * 100).toFixed(1) : "N/A";
       console.log("  File index:");
-      console.log(`    Files: ${h.totalFiles}`);
-      console.log(`    Last scan: ${h.lastScanTimestamp || "never"}`);
-      console.log(`    Hit/miss ratio: ${ratio}${total > 0 ? "%" : ""} (${hits} hits, ${misses} misses)`);
-    } else {
-      console.log("  File index: not available");
+      console.log(`    Files: ${total}`);
+      console.log(`    Last scan: ${repo.getLastScanTimestamp() || "never"}`);
+      console.log(`    Hit/miss ratio: ${ratio}${totalLookups > 0 ? "%" : ""} (${hits} hits, ${misses} misses)`);
     }
   } catch {
     console.log("  File index: error reading");

package/src/core/bug-memory.ts

@@ -1,20 +1,17 @@
-import { safeReadJson, atomicWriteJson } from "./fs-utils";
+// Wrapper over the SQLite bug-memory storage layer. The function-based
+// API below stays compatible with existing unit tests (which build
+// in-memory BugMemory objects), while the repo-aware paths route through
+// BugMemoryRepo so 20k+ bug histories stay searchable in milliseconds.
+//
+// FTS5 is the search backbone — see BugMemoryRepo.searchBugs for the
+// query semantics, score normalization, and false-positive guards.
+
 import type { BugEntry, BugMemory, SimilarityMatch } from "../types/bug-memory";
 
 export function createEmptyBugMemory(): BugMemory {
   return { entries: [], nextId: 1 };
 }
 
-export function loadBugMemory(path: string): BugMemory {
-  const raw = safeReadJson(path);
-  if (raw !== null && isBugMemory(raw)) return raw;
-  return createEmptyBugMemory();
-}
-
-export function saveBugMemory(path: string, memory: BugMemory): void {
-  atomicWriteJson(path, memory);
-}
-
 export function isBugMemory(value: unknown): value is BugMemory {
   if (value === null || typeof value !== "object") return false;
   const obj = value as Record<string, unknown>;
@@ -25,6 +22,8 @@ export function generateBugId(nextId: number): string {
   return `bug-${String(nextId).padStart(3, "0")}`;
 }
 
+// ── In-memory operations (used by unit tests and the JSON migration importer) ──
+
 export function findDuplicate(
   memory: BugMemory,
   errorMessage: string,
@@ -42,9 +41,7 @@ export function addBugEntry(
   fields: Omit<BugEntry, "id" | "createdAt" | "lastSeenAt" | "occurrenceCount">
 ): BugMemory {
   const existing = findDuplicate(memory, fields.errorMessage, fields.filePath);
-  if (existing) {
-    return updateOccurrence(memory, existing.id);
-  }
+  if (existing) return updateOccurrence(memory, existing.id);
 
   const now = new Date().toISOString();
   const entry: BugEntry = {
@@ -73,14 +70,11 @@ export function updateOccurrence(memory: BugMemory, id: string): BugMemory {
   };
 }
 
-// --- Similarity scoring ---
+// ── Similarity scoring (in-memory) ────────────────────────────────────────
 
 function tokenize(text: string): Set<string> {
   return new Set(
-    text
-      .toLowerCase()
-      .split(/\W+/)
-      .filter((w) => w.length > 0)
+    text.toLowerCase().split(/\W+/).filter((w) => w.length > 0)
   );
 }
 
@@ -101,18 +95,12 @@ export function computeSimilarity(
   const matchReasons: string[] = [];
   let score = 0;
 
-  // 1. Exact substring match on error message
-  if (
-    entry.errorMessage.length > 0 &&
-    entry.errorMessage.includes(query)
-  ) {
+  if (entry.errorMessage.length > 0 && entry.errorMessage.includes(query)) {
     score += 1.0;
     matchReasons.push("exact_error_match");
   }
 
-  // 2. Word overlap (Jaccard) across searchable fields
   const queryTokens = tokenize(query);
-
   const fields: [string, string][] = [
     ["error_message", entry.errorMessage],
     ["root_cause", entry.rootCause],
@@ -142,6 +130,8 @@ function hasTagOverlap(entry: BugEntry, query: string): boolean {
   return entry.tags.some((tag) => queryTokens.has(tag.toLowerCase()));
 }
 
+// Pure-memory search — kept for unit tests that don't open a DB. Production
+// call sites use BugMemoryRepo.searchBugs which goes through FTS5.
 export function searchBugs(
   memory: BugMemory,
   query: string,
@@ -153,23 +143,16 @@ export function searchBugs(
 
   for (const entry of memory.entries) {
     const match = computeSimilarity(query, entry);
-
-    // False positive guard: require file-path match OR tag overlap
     const fileMatch = hasFilePathMatch(entry, options?.filePath);
     const tagMatch = hasTagOverlap(entry, query);
     if (!fileMatch && !tagMatch && match.score <= 0.3) continue;
-
-    // Boost same-file matches
     if (fileMatch) {
       match.score += 0.2;
       if (!match.matchReasons.includes("file_path")) {
         match.matchReasons.push("file_path");
       }
     }
-
-    if (match.score > 0.3) {
-      results.push(match);
-    }
+    if (match.score > 0.3) results.push(match);
   }
 
   return results.sort((a, b) => b.score - a.score);
@@ -221,3 +204,18 @@ export function hasBugForFileInSession(
       new Date(e.createdAt).getTime() >= sessionStart
   );
 }
+
+// ── JSON shim — kept so the migration importer + state-aggregator can still
+// read legacy files during the rollout window. New writes go through the repo.
+
+import { safeReadJson, atomicWriteJson } from "./fs-utils";
+
+export function loadBugMemory(path: string): BugMemory {
+  const raw = safeReadJson(path);
+  if (raw !== null && isBugMemory(raw)) return raw;
+  return createEmptyBugMemory();
+}
+
+export function saveBugMemory(path: string, memory: BugMemory): void {
+  atomicWriteJson(path, memory);
+}

package/src/core/dashboard-api.ts

@@ -1,9 +1,7 @@
 import { existsSync, readFileSync } from "fs";
 import {
   projectDir,
-  fileIndexPath,
-  tokenLedgerPath,
-  bugMemoryPath,
+  projectDbPath,
   actionLogPath,
   learningMemoryPath,
   projectMetaPath,
@@ -13,7 +11,8 @@ import {
   designReportPath,
 } from "./paths";
 import { safeReadJson } from "./fs-utils";
-import { isFileIndex } from "./index-store";
+import { FileIndexRepo } from "../repositories/file-index-repo";
+import { CountersRepo } from "../repositories/counters-repo";
 import { loadLedger } from "./token-ledger";
 import { parseLearningMemory } from "./learning-memory";
 import { loadBugMemory } from "./bug-memory";
@@ -125,6 +124,18 @@ function checkTextFile(name: string, filePath: string): FileStatus {
   }
 }
 
+function checkDbFile(name: string, filePath: string): FileStatus {
+  if (!existsSync(filePath)) return { name, status: "missing" };
+  try {
+    const header = readFileSync(filePath).slice(0, 16).toString("utf-8");
+    return header.startsWith("SQLite format 3")
+      ? { name, status: "ok" }
+      : { name, status: "corrupt" };
+  } catch {
+    return { name, status: "corrupt" };
+  }
+}
+
 // ── Panel Loaders ──────────────────────────────────────────────────────────
 
 export function loadOverview(cwd: string): OverviewPayload {
@@ -163,14 +174,13 @@ export function loadOverview(cwd: string): OverviewPayload {
     estimatedSavings: ledger.lifetime.totalEstimatedSavings,
   };
 
-  // State file health
+  // State file health. mink.db replaced file-index.json in Phase 2; the
+  // other JSON checks remain until Phases 3 (bug-memory) and 4 (ledger).
   const stateFiles: FileStatus[] = [
     checkJsonFile("session.json", sessionPath(cwd)),
-    checkJsonFile("file-index.json", fileIndexPath(cwd), isFileIndex),
+    checkDbFile("mink.db", projectDbPath(cwd)),
     checkJsonFile("config.json", configPath(cwd)),
     checkTextFile("learning-memory.md", learningMemoryPath(cwd)),
-    checkJsonFile("token-ledger.json", tokenLedgerPath(cwd)),
-    checkJsonFile("bug-memory.json", bugMemoryPath(cwd)),
     checkTextFile("action-log.md", actionLogPath(cwd)),
     checkJsonFile("scheduler-manifest.json", schedulerManifestPath(cwd)),
   ];
@@ -188,21 +198,33 @@ export function loadTokenLedgerPanel(cwd: string): TokenLedgerPayload {
 }
 
 export function loadFileIndexPanel(cwd: string): FileIndexPayload {
-  const raw = safeReadJson(fileIndexPath(cwd));
-  if (raw && isFileIndex(raw)) {
-    const index = raw as FileIndex;
-    const entries: FileIndexEntry[] = Object.values(index.entries);
-    return { header: index.header, entries };
+  // file_index + per-device counters now live in mink.db; we synthesize
+  // the FileIndexPayload shape the dashboard expects so the frontend
+  // doesn't need to change in this phase.
+  try {
+    const repo = FileIndexRepo.for(cwd);
+    const entries: FileIndexEntry[] = repo.listAll();
+    const totals = CountersRepo.for(cwd).totals();
+    return {
+      header: {
+        lastScanTimestamp: repo.getLastScanTimestamp(),
+        totalFiles: repo.totalFiles(),
+        lifetimeHits: totals.hits,
+        lifetimeMisses: totals.misses,
+      },
+      entries,
+    };
+  } catch {
+    return {
+      header: {
+        lastScanTimestamp: "",
+        totalFiles: 0,
+        lifetimeHits: 0,
+        lifetimeMisses: 0,
+      },
+      entries: [],
+    };
   }
-  return {
-    header: {
-      lastScanTimestamp: "",
-      totalFiles: 0,
-      lifetimeHits: 0,
-      lifetimeMisses: 0,
-    },
-    entries: [],
-  };
 }
 
 export function loadSchedulerPanel(cwd: string): SchedulerPayload {

package/src/core/index-store.ts

@@ -1,6 +1,19 @@
+// Wrapper over the file-index storage layer. Hooks and `mink scan` write
+// through to the SQLite repository at `src/repositories/file-index-repo.ts`;
+// the in-memory `FileIndex` helpers below stay around so unit tests can
+// build fixtures without spinning up a DB. The on-disk file-index.json is
+// no longer the source of truth — Phase 1's migrator moves any existing
+// JSON into the DB on first open.
+//
+// Callers that need lookup-only access should prefer the IndexLookup
+// interface (see src/types/file-index.ts) — it's the minimal surface the
+// hook hot path depends on and is satisfied by both FileIndexRepo and the
+// in-memory adapter below.
+
 import type {
   FileIndex,
   FileIndexEntry,
+  IndexLookup,
   StalenessReport,
 } from "../types/file-index";
 
@@ -70,3 +83,13 @@ export function checkStaleness(
     isStale: missingFromIndex.length > 0 || orphanedEntries.length > 0,
   };
 }
+
+// Adapter: lets analyzers that have an in-memory FileIndex (typically
+// unit tests) pass it to functions expecting the IndexLookup contract
+// without duplicating the lookup logic.
+export function indexAsLookup(index: FileIndex | null): IndexLookup | null {
+  if (index === null) return null;
+  return {
+    lookupEntry: (filePath: string) => lookupEntry(index, filePath),
+  };
+}

package/src/core/paths.ts

@@ -68,6 +68,13 @@ export function bugMemoryPath(cwd: string): string {
   return join(projectDir(cwd), "bug-memory.json");
 }
 
+// SQLite-backed state introduced in spec 17 (file index + bug memory +
+// token ledger). The legacy JSON path helpers above remain valid for
+// backup / rollback during the migration window.
+export function projectDbPath(cwd: string): string {
+  return join(projectDir(cwd), "mink.db");
+}
+
 export function actionLogPath(cwd: string): string {
   return join(projectDir(cwd), "action-log.md");
 }

package/src/core/scanner.ts

@@ -21,7 +21,10 @@ export const DEFAULT_EXCLUDES: string[] = [
   ".mink",
 ];
 
-const DEFAULT_MAX_FILES = 500;
+// Phase 5 of the SQLite migration drops the legacy default cap; callers
+// that want one set `maxFiles` explicitly. SQLite's per-row write cost is
+// flat in index size, so capping by default no longer pays off.
+const NO_CAP = Number.POSITIVE_INFINITY;
 
 function matchesPattern(name: string, pattern: string): boolean {
   if (pattern.includes("*")) {
@@ -96,20 +99,21 @@ export interface ScanStats {
 export function scanProjectWithStats(
   projectRoot: string,
   excludes: string[],
-  maxFiles: number = DEFAULT_MAX_FILES
+  maxFiles: number | undefined = NO_CAP
 ): ScanStats {
   const results: ScannedFile[] = [];
   walkDirectory(projectRoot, projectRoot, excludes, results);
   results.sort((a, b) => b.mtimeMs - a.mtimeMs);
   const totalScanned = results.length;
-  const files = results.slice(0, maxFiles);
+  const cap = maxFiles ?? NO_CAP;
+  const files = Number.isFinite(cap) ? results.slice(0, cap) : results;
   return { files, totalScanned, truncated: totalScanned - files.length };
 }
 
 export function scanProject(
   projectRoot: string,
   excludes: string[],
-  maxFiles: number = DEFAULT_MAX_FILES
+  maxFiles: number | undefined = NO_CAP
 ): ScannedFile[] {
   return scanProjectWithStats(projectRoot, excludes, maxFiles).files;
 }

package/src/core/state-aggregator.ts

@@ -75,14 +75,29 @@ function addLifetime(target: LifetimeCounters, source: LifetimeCounters): void {
 }
 
 export function aggregateTokenLedgerAt(projDir: string): TokenLedger {
+  // Phase 4: token_ledger lives in mink.db. The legacy JSON aggregation
+  // is preserved as a fallback for unit tests and pre-migration projects.
+  const dbPath = join(projDir, "mink.db");
+  if (existsSync(dbPath)) {
+    try {
+      const { openDriver } = require("../storage/driver") as typeof import("../storage/driver");
+      const { applySchema } = require("../storage/schema") as typeof import("../storage/schema");
+      const { TokenLedgerRepo } = require("../repositories/token-ledger-repo") as typeof import("../repositories/token-ledger-repo");
+      const db = openDriver(dbPath);
+      try {
+        applySchema(db);
+        return new TokenLedgerRepo(db).snapshot();
+      } finally {
+        db.close();
+      }
+    } catch {
+      // Fall through to JSON aggregation if the DB read fails.
+    }
+  }
+
   const merged = createEmptyLedger();
   const seenSessions = new Set<string>();
 
-  // Sum lifetime counters from every source (each shard + legacy). Lifetime
-  // persists across archive cycles, so deriving from active sessions alone
-  // would lose archived totals. Migration atomically moves legacy → shard
-  // (`git mv`), so a session never lives in both simultaneously and lifetime
-  // counters do not double-count in production.
   const sources = [
     ...listDeviceShardsAt(projDir).map((id) =>
       shardPath(projDir, id, "token-ledger.json")
@@ -90,8 +105,6 @@ export function aggregateTokenLedgerAt(projDir: string): TokenLedger {
     join(projDir, "token-ledger.json"),
   ];
 
-  // Track waste-flags across sources, deduped by (pattern, detectedAt) so
-  // each device's flags remain visible without spamming duplicates.
   const seenFlagKeys = new Set<string>();
   const wasteFlags: NonNullable<TokenLedger["wasteFlags"]> = [];
 
@@ -131,6 +144,25 @@ export function aggregateTokenLedger(cwd: string): TokenLedger {
 export function aggregateTokenLedgerArchiveAt(
   projDir: string
 ): LedgerSession[] {
+  // Phase 4: archive is `archived = 1` in ledger_sessions.
+  const dbPath = join(projDir, "mink.db");
+  if (existsSync(dbPath)) {
+    try {
+      const { openDriver } = require("../storage/driver") as typeof import("../storage/driver");
+      const { applySchema } = require("../storage/schema") as typeof import("../storage/schema");
+      const { TokenLedgerRepo } = require("../repositories/token-ledger-repo") as typeof import("../repositories/token-ledger-repo");
+      const db = openDriver(dbPath);
+      try {
+        applySchema(db);
+        return new TokenLedgerRepo(db).archivedSessions();
+      } finally {
+        db.close();
+      }
+    } catch {
+      // Fall through to JSON aggregation.
+    }
+  }
+
   const seen = new Set<string>();
   const archived: LedgerSession[] = [];
 
@@ -161,6 +193,31 @@ export function aggregateTokenLedgerArchive(cwd: string): LedgerSession[] {
 // ── Bug memory ─────────────────────────────────────────────────────────────
 
 export function aggregateBugMemoryAt(projDir: string): BugMemory {
+  // Phase 3 of the SQLite migration: bug_memory lives in mink.db. The
+  // legacy JSON aggregation below is preserved as a fallback for tests /
+  // pre-migration projects, but new call sites should read from
+  // BugMemoryRepo directly.
+  const dbPath = join(projDir, "mink.db");
+  if (existsSync(dbPath)) {
+    try {
+      // Use a fresh handle so we don't disturb the per-process cache used
+      // by hook commands. Lazy-require to keep state-aggregator free of
+      // a hard storage-layer dependency for tests that mock paths.
+      const { openDriver } = require("../storage/driver") as typeof import("../storage/driver");
+      const { applySchema } = require("../storage/schema") as typeof import("../storage/schema");
+      const { BugMemoryRepo } = require("../repositories/bug-memory-repo") as typeof import("../repositories/bug-memory-repo");
+      const db = openDriver(dbPath);
+      try {
+        applySchema(db); // tolerate older DBs missing newer tables
+        return new BugMemoryRepo(db).snapshot();
+      } finally {
+        db.close();
+      }
+    } catch {
+      // Fall through to JSON aggregation if the DB read fails.
+    }
+  }
+
   const byId = new Map<string, BugEntry>();
   let maxNextId = 1;
 

package/src/core/state-counters.ts

@@ -1,46 +1,26 @@
-import { atomicWriteJson, safeReadJson } from "./fs-utils";
-import { fileIndexCountersPath } from "./paths";
+// Wrapper over the per-device counters table. The legacy implementation
+// kept these in projects/<id>/.mink-state-counters.json; Phase 1's
+// importer copies that file's contents into the `counters` table the
+// first time the project DB opens, and the file is moved to
+// legacy-backup/. Both APIs (totals and per-device) remain available so
+// the dashboard and `mink status` keep their existing surface.
 
-// Per-device telemetry counters. Lives at projects/<id>/.mink-state-counters.json
-// and is gitignored so each device's counts never collide. Aggregated views
-// (dashboard, status) sum across devices via aggregateStateCounters().
+import { CountersRepo } from "../repositories/counters-repo";
 
 export interface StateCounters {
   fileIndexHits: number;
   fileIndexMisses: number;
 }
 
-function emptyCounters(): StateCounters {
-  return { fileIndexHits: 0, fileIndexMisses: 0 };
-}
-
-function isStateCounters(value: unknown): value is StateCounters {
-  if (value === null || typeof value !== "object") return false;
-  const obj = value as Record<string, unknown>;
-  return (
-    typeof obj.fileIndexHits === "number" &&
-    typeof obj.fileIndexMisses === "number"
-  );
-}
-
 export function loadCounters(cwd: string): StateCounters {
-  const raw = safeReadJson(fileIndexCountersPath(cwd));
-  if (raw !== null && isStateCounters(raw)) return raw;
-  return emptyCounters();
-}
-
-export function saveCounters(cwd: string, counters: StateCounters): void {
-  atomicWriteJson(fileIndexCountersPath(cwd), counters);
+  const t = CountersRepo.for(cwd).totals();
+  return { fileIndexHits: t.hits, fileIndexMisses: t.misses };
 }
 
 export function incrementFileIndexHit(cwd: string): void {
-  const c = loadCounters(cwd);
-  c.fileIndexHits++;
-  saveCounters(cwd, c);
+  CountersRepo.for(cwd).incrementHit();
 }
 
 export function incrementFileIndexMiss(cwd: string): void {
-  const c = loadCounters(cwd);
-  c.fileIndexMisses++;
-  saveCounters(cwd, c);
+  CountersRepo.for(cwd).incrementMiss();
 }