@drewpayment/mink 0.11.0 → 0.12.0-beta.2

package/src/core/state-aggregator.ts

@@ -75,14 +75,29 @@ function addLifetime(target: LifetimeCounters, source: LifetimeCounters): void {
 }
 
 export function aggregateTokenLedgerAt(projDir: string): TokenLedger {
+  // Phase 4: token_ledger lives in mink.db. The legacy JSON aggregation
+  // is preserved as a fallback for unit tests and pre-migration projects.
+  const dbPath = join(projDir, "mink.db");
+  if (existsSync(dbPath)) {
+    try {
+      const { openDriver } = require("../storage/driver") as typeof import("../storage/driver");
+      const { applySchema } = require("../storage/schema") as typeof import("../storage/schema");
+      const { TokenLedgerRepo } = require("../repositories/token-ledger-repo") as typeof import("../repositories/token-ledger-repo");
+      const db = openDriver(dbPath);
+      try {
+        applySchema(db);
+        return new TokenLedgerRepo(db).snapshot();
+      } finally {
+        db.close();
+      }
+    } catch {
+      // Fall through to JSON aggregation if the DB read fails.
+    }
+  }
+
   const merged = createEmptyLedger();
   const seenSessions = new Set<string>();
 
-  // Sum lifetime counters from every source (each shard + legacy). Lifetime
-  // persists across archive cycles, so deriving from active sessions alone
-  // would lose archived totals. Migration atomically moves legacy → shard
-  // (`git mv`), so a session never lives in both simultaneously and lifetime
-  // counters do not double-count in production.
   const sources = [
     ...listDeviceShardsAt(projDir).map((id) =>
       shardPath(projDir, id, "token-ledger.json")
@@ -90,8 +105,6 @@ export function aggregateTokenLedgerAt(projDir: string): TokenLedger {
     join(projDir, "token-ledger.json"),
   ];
 
-  // Track waste-flags across sources, deduped by (pattern, detectedAt) so
-  // each device's flags remain visible without spamming duplicates.
   const seenFlagKeys = new Set<string>();
   const wasteFlags: NonNullable<TokenLedger["wasteFlags"]> = [];
 
@@ -131,6 +144,25 @@ export function aggregateTokenLedger(cwd: string): TokenLedger {
 export function aggregateTokenLedgerArchiveAt(
   projDir: string
 ): LedgerSession[] {
+  // Phase 4: archive is `archived = 1` in ledger_sessions.
+  const dbPath = join(projDir, "mink.db");
+  if (existsSync(dbPath)) {
+    try {
+      const { openDriver } = require("../storage/driver") as typeof import("../storage/driver");
+      const { applySchema } = require("../storage/schema") as typeof import("../storage/schema");
+      const { TokenLedgerRepo } = require("../repositories/token-ledger-repo") as typeof import("../repositories/token-ledger-repo");
+      const db = openDriver(dbPath);
+      try {
+        applySchema(db);
+        return new TokenLedgerRepo(db).archivedSessions();
+      } finally {
+        db.close();
+      }
+    } catch {
+      // Fall through to JSON aggregation.
+    }
+  }
+
   const seen = new Set<string>();
   const archived: LedgerSession[] = [];
 
@@ -161,6 +193,31 @@ export function aggregateTokenLedgerArchive(cwd: string): LedgerSession[] {
 // ── Bug memory ─────────────────────────────────────────────────────────────
 
 export function aggregateBugMemoryAt(projDir: string): BugMemory {
+  // Phase 3 of the SQLite migration: bug_memory lives in mink.db. The
+  // legacy JSON aggregation below is preserved as a fallback for tests /
+  // pre-migration projects, but new call sites should read from
+  // BugMemoryRepo directly.
+  const dbPath = join(projDir, "mink.db");
+  if (existsSync(dbPath)) {
+    try {
+      // Use a fresh handle so we don't disturb the per-process cache used
+      // by hook commands. Lazy-require to keep state-aggregator free of
+      // a hard storage-layer dependency for tests that mock paths.
+      const { openDriver } = require("../storage/driver") as typeof import("../storage/driver");
+      const { applySchema } = require("../storage/schema") as typeof import("../storage/schema");
+      const { BugMemoryRepo } = require("../repositories/bug-memory-repo") as typeof import("../repositories/bug-memory-repo");
+      const db = openDriver(dbPath);
+      try {
+        applySchema(db); // tolerate older DBs missing newer tables
+        return new BugMemoryRepo(db).snapshot();
+      } finally {
+        db.close();
+      }
+    } catch {
+      // Fall through to JSON aggregation if the DB read fails.
+    }
+  }
+
   const byId = new Map<string, BugEntry>();
   let maxNextId = 1;
 

package/src/core/state-counters.ts

@@ -1,46 +1,26 @@
-import { atomicWriteJson, safeReadJson } from "./fs-utils";
-import { fileIndexCountersPath } from "./paths";
+// Wrapper over the per-device counters table. The legacy implementation
+// kept these in projects/<id>/.mink-state-counters.json; Phase 1's
+// importer copies that file's contents into the `counters` table the
+// first time the project DB opens, and the file is moved to
+// legacy-backup/. Both APIs (totals and per-device) remain available so
+// the dashboard and `mink status` keep their existing surface.
 
-// Per-device telemetry counters. Lives at projects/<id>/.mink-state-counters.json
-// and is gitignored so each device's counts never collide. Aggregated views
-// (dashboard, status) sum across devices via aggregateStateCounters().
+import { CountersRepo } from "../repositories/counters-repo";
 
 export interface StateCounters {
   fileIndexHits: number;
   fileIndexMisses: number;
 }
 
-function emptyCounters(): StateCounters {
-  return { fileIndexHits: 0, fileIndexMisses: 0 };
-}
-
-function isStateCounters(value: unknown): value is StateCounters {
-  if (value === null || typeof value !== "object") return false;
-  const obj = value as Record<string, unknown>;
-  return (
-    typeof obj.fileIndexHits === "number" &&
-    typeof obj.fileIndexMisses === "number"
-  );
-}
-
 export function loadCounters(cwd: string): StateCounters {
-  const raw = safeReadJson(fileIndexCountersPath(cwd));
-  if (raw !== null && isStateCounters(raw)) return raw;
-  return emptyCounters();
-}
-
-export function saveCounters(cwd: string, counters: StateCounters): void {
-  atomicWriteJson(fileIndexCountersPath(cwd), counters);
+  const t = CountersRepo.for(cwd).totals();
+  return { fileIndexHits: t.hits, fileIndexMisses: t.misses };
 }
 
 export function incrementFileIndexHit(cwd: string): void {
-  const c = loadCounters(cwd);
-  c.fileIndexHits++;
-  saveCounters(cwd, c);
+  CountersRepo.for(cwd).incrementHit();
 }
 
 export function incrementFileIndexMiss(cwd: string): void {
-  const c = loadCounters(cwd);
-  c.fileIndexMisses++;
-  saveCounters(cwd, c);
+  CountersRepo.for(cwd).incrementMiss();
 }

package/src/core/sync-merge-drivers.ts

@@ -1,7 +1,9 @@
-import { readFileSync, writeFileSync, appendFileSync } from "fs";
+import { readFileSync, writeFileSync, appendFileSync, copyFileSync, unlinkSync } from "fs";
 import { join } from "path";
 import { minkRoot } from "./paths";
 import { parseLearningMemory, serializeLearningMemory } from "./learning-memory";
+import { openDriver } from "../storage/driver";
+import { applySchema } from "../storage/schema";
 import type { LearningMemory, SectionName } from "../types/learning-memory";
 import type { FileIndex, FileIndexEntry } from "../types/file-index";
 import type { DeviceInfo, DeviceRegistry } from "../types/config";
@@ -220,6 +222,161 @@ export function mergeDevicesDriver(args: DriverArgs): void {
   }
 }
 
+// ── mink-db-merge: projects/*/mink.db ──────────────────────────────────────
+// Two-DB reconciliation: open `ours.db` for read/write, ATTACH `theirs.db`
+// as a read-only schema, replay rows via INSERT OR ... ON CONFLICT using
+// the same per-store conflict rules the JSON aggregator uses today.
+// Sessions / counters / ledger_lifetime are append-merge keyed by device.
+// Falls back to "keep ours" on any failure so a merge driver never blocks
+// the rebase.
+
+interface DbHandle {
+  exec(sql: string): void;
+  prepare(sql: string): { run(...args: unknown[]): unknown };
+  close(): void;
+}
+
+function attachAndReplay(ours: DbHandle, theirsPath: string): void {
+  // ATTACH requires the path to be a literal string parameter; bind it
+  // with the prepare binding to avoid SQL injection from a weird filename.
+  ours.prepare("ATTACH DATABASE ? AS theirs").run(theirsPath);
+  try {
+    ours.exec(`
+      -- file_index: keep the side with the newer last_modified.
+      INSERT INTO file_index
+        (file_path, description, estimated_tokens, last_modified, last_indexed,
+         mtime_ms, content_hash, size_bytes, device_id)
+      SELECT file_path, description, estimated_tokens, last_modified, last_indexed,
+             mtime_ms, content_hash, size_bytes, device_id
+      FROM theirs.file_index
+      WHERE TRUE
+      ON CONFLICT(file_path) DO UPDATE SET
+        description       = CASE WHEN excluded.last_modified > file_index.last_modified THEN excluded.description       ELSE file_index.description       END,
+        estimated_tokens  = CASE WHEN excluded.last_modified > file_index.last_modified THEN excluded.estimated_tokens  ELSE file_index.estimated_tokens  END,
+        last_indexed      = CASE WHEN excluded.last_modified > file_index.last_modified THEN excluded.last_indexed      ELSE file_index.last_indexed      END,
+        last_modified     = CASE WHEN excluded.last_modified > file_index.last_modified THEN excluded.last_modified     ELSE file_index.last_modified     END,
+        mtime_ms          = CASE WHEN excluded.last_modified > file_index.last_modified THEN excluded.mtime_ms          ELSE file_index.mtime_ms          END,
+        content_hash      = COALESCE(excluded.content_hash, file_index.content_hash),
+        size_bytes        = COALESCE(excluded.size_bytes,   file_index.size_bytes),
+        device_id         = CASE WHEN excluded.last_modified > file_index.last_modified THEN excluded.device_id         ELSE file_index.device_id         END;
+
+      -- bug_memory: oldest createdAt + latest lastSeenAt + max occurrence_count.
+      INSERT INTO bug_memory
+        (id, created_at, last_seen_at, error_message, file_path, line_number,
+         root_cause, fix_description, occurrence_count, device_id)
+      SELECT id, created_at, last_seen_at, error_message, file_path, line_number,
+             root_cause, fix_description, occurrence_count, device_id
+      FROM theirs.bug_memory
+      WHERE TRUE
+      ON CONFLICT(id) DO UPDATE SET
+        created_at       = CASE WHEN excluded.created_at < bug_memory.created_at THEN excluded.created_at ELSE bug_memory.created_at END,
+        last_seen_at     = CASE WHEN excluded.last_seen_at > bug_memory.last_seen_at THEN excluded.last_seen_at ELSE bug_memory.last_seen_at END,
+        occurrence_count = MAX(bug_memory.occurrence_count, excluded.occurrence_count);
+
+      INSERT OR IGNORE INTO bug_tags    (bug_id, tag)             SELECT bug_id, tag             FROM theirs.bug_tags;
+      INSERT OR IGNORE INTO bug_related (bug_id, related_bug_id)  SELECT bug_id, related_bug_id  FROM theirs.bug_related;
+
+      -- Ledger sessions are insert-only and device-isolated, so first writer
+      -- wins is correct (shards never overlap session_id in production).
+      INSERT OR IGNORE INTO ledger_sessions
+        (session_id, device_id, start_timestamp, end_timestamp, read_count,
+         write_count, estimated_tokens, repeated_reads, file_index_hits,
+         file_index_misses, estimated_savings, archived)
+      SELECT session_id, device_id, start_timestamp, end_timestamp, read_count,
+             write_count, estimated_tokens, repeated_reads, file_index_hits,
+             file_index_misses, estimated_savings, archived
+      FROM theirs.ledger_sessions;
+
+      INSERT INTO ledger_reads  (session_id, file_path, estimated_tokens, read_count)
+        SELECT session_id, file_path, estimated_tokens, read_count
+        FROM theirs.ledger_reads
+        WHERE session_id IN (SELECT session_id FROM theirs.ledger_sessions
+                             WHERE session_id NOT IN (SELECT session_id FROM ledger_reads));
+
+      INSERT INTO ledger_writes (session_id, file_path, estimated_tokens, action)
+        SELECT session_id, file_path, estimated_tokens, action
+        FROM theirs.ledger_writes
+        WHERE session_id IN (SELECT session_id FROM theirs.ledger_sessions
+                             WHERE session_id NOT IN (SELECT session_id FROM ledger_writes));
+
+      -- Per-device lifetime sums and counters: take the MAX so concurrent
+      -- increments on different devices don't double-count when the same
+      -- device's row is shared (it shouldn't be, but MAX is a safe upper
+      -- bound under concurrent shard mutation).
+      INSERT INTO ledger_lifetime
+        (device_id, total_tokens, total_reads, total_writes, total_sessions,
+         total_file_index_hits, total_file_index_misses, total_repeated_reads,
+         total_estimated_savings)
+      SELECT device_id, total_tokens, total_reads, total_writes, total_sessions,
+             total_file_index_hits, total_file_index_misses, total_repeated_reads,
+             total_estimated_savings
+      FROM theirs.ledger_lifetime
+      WHERE TRUE
+      ON CONFLICT(device_id) DO UPDATE SET
+        total_tokens             = MAX(ledger_lifetime.total_tokens,             excluded.total_tokens),
+        total_reads              = MAX(ledger_lifetime.total_reads,              excluded.total_reads),
+        total_writes             = MAX(ledger_lifetime.total_writes,             excluded.total_writes),
+        total_sessions           = MAX(ledger_lifetime.total_sessions,           excluded.total_sessions),
+        total_file_index_hits    = MAX(ledger_lifetime.total_file_index_hits,    excluded.total_file_index_hits),
+        total_file_index_misses  = MAX(ledger_lifetime.total_file_index_misses,  excluded.total_file_index_misses),
+        total_repeated_reads     = MAX(ledger_lifetime.total_repeated_reads,     excluded.total_repeated_reads),
+        total_estimated_savings  = MAX(ledger_lifetime.total_estimated_savings,  excluded.total_estimated_savings);
+
+      INSERT INTO counters (device_id, file_index_hits, file_index_misses)
+      SELECT device_id, file_index_hits, file_index_misses
+      FROM theirs.counters
+      WHERE TRUE
+      ON CONFLICT(device_id) DO UPDATE SET
+        file_index_hits   = MAX(counters.file_index_hits,   excluded.file_index_hits),
+        file_index_misses = MAX(counters.file_index_misses, excluded.file_index_misses);
+
+      INSERT OR IGNORE INTO waste_flags (pattern, detected_at, details, device_id)
+        SELECT pattern, detected_at, details, device_id FROM theirs.waste_flags;
+    `);
+  } finally {
+    ours.exec("DETACH DATABASE theirs");
+  }
+}
+
+export function mergeDbDriver(args: DriverArgs): void {
+  // Run the merge in a side-DB so a crash mid-replay never leaves ours in
+  // a half-merged state. We copy ours -> a temp file, replay theirs into
+  // the copy, then atomically replace ours via rename. Sidecar WAL/SHM
+  // files in the temp location are cleaned up before rename.
+  const tmp = `${args.oursPath}.merge-${process.pid}-${Date.now()}.tmp`;
+  let ours: ReturnType<typeof openDriver> | null = null;
+  try {
+    copyFileSync(args.oursPath, tmp);
+    ours = openDriver(tmp);
+    ours.exec("PRAGMA journal_mode = WAL");
+    ours.exec("PRAGMA foreign_keys = ON");
+    applySchema(ours); // tolerate `theirs` carrying tables we don't yet have
+
+    attachAndReplay(ours, args.theirsPath);
+
+    ours.exec("PRAGMA wal_checkpoint(TRUNCATE)");
+    ours.close();
+    ours = null;
+
+    // Clean up WAL/SHM sidecars left by the temp DB so the rename target
+    // is a single self-contained file. SQLite truncates the WAL above; we
+    // remove the empty sidecar entries explicitly.
+    for (const suffix of ["-wal", "-shm", "-journal"]) {
+      try { unlinkSync(`${tmp}${suffix}`); } catch { /* not present */ }
+    }
+
+    // Atomic replace.
+    copyFileSync(tmp, args.oursPath);
+    try { unlinkSync(tmp); } catch { /* best effort */ }
+  } catch (err) {
+    logWarning("mink-db-merge", args, err);
+    try { if (ours) ours.close(); } catch { /* ignore */ }
+    try { unlinkSync(tmp); } catch { /* ignore */ }
+    // Fall back to ours by doing nothing — the original args.oursPath is
+    // untouched.
+  }
+}
+
 // ── Dispatcher ─────────────────────────────────────────────────────────────
 
 export function runMergeDriver(
@@ -232,8 +389,14 @@ export function runMergeDriver(
   const args: DriverArgs = { basePath, oursPath, theirsPath, filePath };
   switch (name) {
     case "mink-json-union":
+      // Legacy driver — still registered for projects pre-Phase 2 of the
+      // SQLite migration where file-index.json may resurface during sync
+      // of legacy-backup contents. New projects don't use it.
       mergeJsonUnion(args);
       return 0;
+    case "mink-db-merge":
+      mergeDbDriver(args);
+      return 0;
     case "mink-learning-memory":
       mergeLearningMemoryDriver(args);
       return 0;

package/src/core/sync.ts

@@ -65,6 +65,12 @@ projects/*/scheduler-manifest.json
 projects/*/design-captures/
 projects/*/.mink-state-counters.json
 
+# SQLite write-ahead log + shared-memory sidecars — local-only, must not
+# travel with the main mink.db (WAL is checkpointed before push).
+projects/*/mink.db-wal
+projects/*/mink.db-shm
+projects/*/mink.db-journal
+
 # Wiki derived/regenerable pages — each device rebuilds locally
 wiki/_index.md
 wiki/.mink-index.json
@@ -74,6 +80,8 @@ wiki/projects/*/architecture.md
 
 const GITATTRIBUTES_CONTENTS = `# Sync v2 — merge drivers eliminate conflicts on shared files.
 # Drivers are registered in .git/config by ensureMergeDriversRegistered().
+projects/*/mink.db merge=mink-db-merge
+projects/*/mink.db binary
 projects/*/file-index.json merge=mink-json-union
 projects/*/learning-memory.*.md merge=union
 projects/*/learning-memory.md merge=mink-learning-memory
@@ -120,6 +128,7 @@ export function ensureGitAttributes(): void {
 
 const MERGE_DRIVERS = [
   "mink-json-union",
+  "mink-db-merge",
   "mink-learning-memory",
   "mink-devices",
 ] as const;

package/src/core/token-ledger.ts

@@ -1,3 +1,4 @@
+import { existsSync } from "fs";
 import { join } from "path";
 import type { TokenLedger, LedgerSession, LifetimeCounters } from "../types/token-ledger";
 import type { SessionFinalizer, SessionSummary } from "../types/session";
@@ -158,15 +159,60 @@ export function saveArchive(archivePath: string, newlyArchived: LedgerSession[])
 
 // ── Task 6: Ledger Finalizer Factory ─────────────────────────────────────────
 
+// Phase 4 of the SQLite migration: every projectDir that has a mink.db
+// uses TokenLedgerRepo. Legacy JSON paths still get a finalizer for
+// pre-migration projects (and for unit tests that don't open a DB).
 export function createLedgerFinalizer(
   projectDir: string,
   deviceIdOrThreshold?: string | number,
   archiveThreshold: number = 1000
 ): SessionFinalizer {
-  // Backward compat: callers that pass `(projectDir)` or
-  // `(projectDir, threshold)` still work and write to the legacy path. New
-  // callers pass `(projectDir, deviceId, threshold?)` to write into the
-  // per-device shard at projectDir/state/<deviceId>/...
+  const dbPath = join(projectDir, "mink.db");
+
+  if (existsSync(dbPath)) {
+    // Route through the SQLite repo. We open a fresh handle so this
+    // module doesn't depend on the per-process cache used by hooks.
+    const { openDriver } = require("../storage/driver") as typeof import("../storage/driver");
+    const { applySchema } = require("../storage/schema") as typeof import("../storage/schema");
+    const { TokenLedgerRepo } = require("../repositories/token-ledger-repo") as typeof import("../repositories/token-ledger-repo");
+
+    const deviceId = typeof deviceIdOrThreshold === "string" ? deviceIdOrThreshold : undefined;
+    const threshold = typeof deviceIdOrThreshold === "number"
+      ? deviceIdOrThreshold
+      : archiveThreshold;
+
+    return {
+      appendSession(summary: SessionSummary): void {
+        const db = openDriver(dbPath);
+        try {
+          db.exec("PRAGMA journal_mode = WAL");
+          db.exec("PRAGMA foreign_keys = ON");
+          applySchema(db);
+          const repo = new TokenLedgerRepo(db);
+          repo.appendSession(summary, deviceId);
+          repo.archive(threshold);
+        } finally {
+          db.close();
+        }
+      },
+
+      updateSession(summary: SessionSummary): void {
+        const db = openDriver(dbPath);
+        try {
+          db.exec("PRAGMA journal_mode = WAL");
+          db.exec("PRAGMA foreign_keys = ON");
+          applySchema(db);
+          new TokenLedgerRepo(db).updateSession(summary, deviceId);
+        } finally {
+          db.close();
+        }
+      },
+    };
+  }
+
+  // ── Legacy JSON fallback ────────────────────────────────────────────
+  // Tests + pre-migration projects continue to write to disk. The
+  // `(projectDir)` and `(projectDir, threshold)` signatures still work.
   let ledgerPath: string;
   let archivePath: string;
   let threshold: number;