@drewpayment/mink 0.11.0 → 0.12.0-beta.2

package/src/repositories/token-ledger-repo.ts

@@ -0,0 +1,412 @@
+// Token-ledger repository. Sessions + reads + writes + per-device lifetime
+// counters all live in mink.db. The legacy archive file is subsumed by the
+// `archived` column on ledger_sessions — "archive" becomes
+// `UPDATE ledger_sessions SET archived = 1 WHERE ...`.
+//
+// Sessions are insert-only from the perspective of the merge driver
+// (first-writer-wins keyed on session_id). The only post-insert mutations
+// allowed are:
+//   - updateSession(): replaces totals/lists for the latest session before
+//     it's been seen by any other device — the in-process device knows
+//     its own session_id is exclusive until session_stop persists it.
+//   - archive(): flips `archived = 1` once the active-session count
+//     exceeds the retention threshold.
+//
+// Lifetime counters are also per-device; the merge driver MAX-merges them
+// across shards so concurrent activity on different devices keeps the
+// project-wide total monotonic.
+
+import type { DbDriver } from "../storage/driver";
+import type {
+  TokenLedger,
+  LedgerSession,
+  LifetimeCounters,
+} from "../types/token-ledger";
+import type { SessionSummary } from "../types/session";
+import type { WasteFlag, WastePattern } from "../types/waste-detection";
+import { openProjectDb } from "../storage/db";
+import { getOrCreateDeviceId } from "../core/device";
+
+function emptyLifetime(): LifetimeCounters {
+  return {
+    totalTokens: 0,
+    totalReads: 0,
+    totalWrites: 0,
+    totalSessions: 0,
+    totalFileIndexHits: 0,
+    totalFileIndexMisses: 0,
+    totalRepeatedReads: 0,
+    totalEstimatedSavings: 0,
+  };
+}
+
+export class TokenLedgerRepo {
+  constructor(private readonly db: DbDriver) {}
+
+  static for(cwd: string): TokenLedgerRepo {
+    return new TokenLedgerRepo(openProjectDb(cwd));
+  }
+
+  // ── Append / update sessions ──────────────────────────────────────────
+
+  // Insert a session as the active (archived = 0) ledger entry, append its
+  // reads/writes, and bump this device's lifetime counters. Wrapped in a
+  // transaction so a partial write never leaves the lifetime out of sync
+  // with the per-session rows.
+  appendSession(summary: SessionSummary, deviceId: string = getOrCreateDeviceId()): void {
+    this.db.transaction(() => {
+      this.insertSessionRow(summary, deviceId, 0);
+      this.appendChildRows(summary);
+      this.addToLifetime(deviceId, summary);
+    });
+  }
+
+  // Replace a previously-inserted active session. Used when a session is
+  // stopped multiple times — only the last stop's totals are authoritative
+  // for this device, so we subtract the old contribution and add the new.
+  // We don't allow updating an archived session (the merge driver assumes
+  // archived rows are immutable).
+  updateSession(summary: SessionSummary, deviceId: string = getOrCreateDeviceId()): void {
+    this.db.transaction(() => {
+      const existing = this.fetchSession(summary.sessionId);
+      if (!existing) {
+        this.insertSessionRow(summary, deviceId, 0);
+        this.appendChildRows(summary);
+        this.addToLifetime(deviceId, summary);
+        return;
+      }
+      this.subtractFromLifetime(existing.device_id, existing);
+      this.db.prepare(
+        "DELETE FROM ledger_reads  WHERE session_id = ?"
+      ).run(summary.sessionId);
+      this.db.prepare(
+        "DELETE FROM ledger_writes WHERE session_id = ?"
+      ).run(summary.sessionId);
+      this.db.prepare(`
+        UPDATE ledger_sessions SET
+          start_timestamp     = ?,
+          end_timestamp       = ?,
+          read_count          = ?,
+          write_count         = ?,
+          estimated_tokens    = ?,
+          repeated_reads      = ?,
+          file_index_hits     = ?,
+          file_index_misses   = ?,
+          estimated_savings   = ?
+        WHERE session_id = ?
+      `).run(
+        summary.startTimestamp, summary.endTimestamp,
+        summary.totals.readCount, summary.totals.writeCount,
+        summary.totals.estimatedTokens, summary.totals.repeatedReads,
+        summary.totals.fileIndexHits, summary.totals.fileIndexMisses,
+        summary.estimatedSavings, summary.sessionId
+      );
+      this.appendChildRows(summary);
+      this.addToLifetime(existing.device_id, summary);
+    });
+  }
+
+  // Archive everything past the retention threshold. Returns the number of
+  // sessions newly archived. We sort by start_timestamp ASC and flip the
+  // oldest ones so the most recent N stay active — same intent as the v1
+  // JSON archive flow.
+  archive(threshold: number = 1000): number {
+    if (threshold <= 0) return 0;
+    const active = Number(
+      (this.db.prepare(
+        "SELECT COUNT(*) AS n FROM ledger_sessions WHERE archived = 0"
+      ).get() as { n: number }).n
+    );
+    if (active <= threshold) return 0;
+    const excess = active - threshold;
+    const r = this.db.prepare(`
+      UPDATE ledger_sessions SET archived = 1
+      WHERE session_id IN (
+        SELECT session_id FROM ledger_sessions
+        WHERE archived = 0
+        ORDER BY start_timestamp ASC
+        LIMIT ?
+      )
+    `).run(excess);
+    return Number(r.changes);
+  }
+
+  // ── Read ──────────────────────────────────────────────────────────────
+
+  lifetime(): LifetimeCounters {
+    // Sum across every device's row — gives the project-wide total.
+    const row = this.db.prepare(`
+      SELECT
+        COALESCE(SUM(total_tokens),             0) AS totalTokens,
+        COALESCE(SUM(total_reads),              0) AS totalReads,
+        COALESCE(SUM(total_writes),             0) AS totalWrites,
+        COALESCE(SUM(total_sessions),           0) AS totalSessions,
+        COALESCE(SUM(total_file_index_hits),    0) AS totalFileIndexHits,
+        COALESCE(SUM(total_file_index_misses),  0) AS totalFileIndexMisses,
+        COALESCE(SUM(total_repeated_reads),     0) AS totalRepeatedReads,
+        COALESCE(SUM(total_estimated_savings),  0) AS totalEstimatedSavings
+      FROM ledger_lifetime
+    `).get();
+    if (!row) return emptyLifetime();
+    const r = row as Record<string, number>;
+    return {
+      totalTokens:             Number(r.totalTokens),
+      totalReads:              Number(r.totalReads),
+      totalWrites:             Number(r.totalWrites),
+      totalSessions:           Number(r.totalSessions),
+      totalFileIndexHits:      Number(r.totalFileIndexHits),
+      totalFileIndexMisses:    Number(r.totalFileIndexMisses),
+      totalRepeatedReads:      Number(r.totalRepeatedReads),
+      totalEstimatedSavings:   Number(r.totalEstimatedSavings),
+    };
+  }
+
+  // All active (archived = 0) sessions, hydrated with their reads + writes.
+  // Sorted by start_timestamp to match the JSON aggregator's ordering.
+  activeSessions(): LedgerSession[] {
+    return this.hydrateSessions(
+      "SELECT * FROM ledger_sessions WHERE archived = 0 ORDER BY start_timestamp"
+    );
+  }
+
+  archivedSessions(): LedgerSession[] {
+    return this.hydrateSessions(
+      "SELECT * FROM ledger_sessions WHERE archived = 1 ORDER BY start_timestamp"
+    );
+  }
+
+  // Project-wide snapshot in the legacy TokenLedger shape — used by the
+  // dashboard, status, and detect-waste. wasteFlags are pulled from
+  // waste_flags and deduped by (pattern, detected_at).
+  snapshot(): TokenLedger {
+    const ledger: TokenLedger = {
+      lifetime: this.lifetime(),
+      sessions: this.activeSessions(),
+    };
+    const flagRows = this.db
+      .prepare(
+        "SELECT pattern, detected_at, details FROM waste_flags ORDER BY detected_at"
+      )
+      .all();
+    if (flagRows.length > 0) {
+      ledger.wasteFlags = flagRows.map((r) => {
+        const row = r as { pattern: string; detected_at: string; details: string | null };
+        const flag: WasteFlag = {
+          pattern: row.pattern as WastePattern,
+          detectedAt: row.detected_at,
+          description: "",
+          estimatedTokensWasted: 0,
+          suggestion: "",
+        };
+        if (row.details) {
+          try {
+            const parsed = JSON.parse(row.details) as Partial<WasteFlag>;
+            Object.assign(flag, parsed);
+          } catch {
+            // ignore bad JSON — keep base flag
+          }
+        }
+        return flag;
+      });
+    }
+    return ledger;
+  }
+
+  // Replace all waste_flags rows for THIS device with the provided set.
+  // detect-waste re-runs and overwrites every cycle, so we don't try to
+  // merge with previous flags.
+  replaceWasteFlagsForDevice(
+    deviceId: string,
+    flags: NonNullable<TokenLedger["wasteFlags"]>
+  ): void {
+    this.db.transaction(() => {
+      this.db.prepare(
+        "DELETE FROM waste_flags WHERE device_id = ?"
+      ).run(deviceId);
+      const stmt = this.db.prepare(
+        "INSERT OR REPLACE INTO waste_flags (pattern, detected_at, details, device_id) VALUES (?, ?, ?, ?)"
+      );
+      for (const flag of flags) {
+        const { pattern, detectedAt, ...rest } = flag;
+        stmt.run(pattern, detectedAt, JSON.stringify(rest), deviceId);
+      }
+    });
+  }
+
+  // ── Helpers ───────────────────────────────────────────────────────────
+
+  private insertSessionRow(
+    summary: SessionSummary,
+    deviceId: string,
+    archived: 0 | 1
+  ): void {
+    this.db.prepare(`
+      INSERT OR REPLACE INTO ledger_sessions
+        (session_id, device_id, start_timestamp, end_timestamp,
+         read_count, write_count, estimated_tokens, repeated_reads,
+         file_index_hits, file_index_misses, estimated_savings, archived)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    `).run(
+      summary.sessionId, deviceId,
+      summary.startTimestamp, summary.endTimestamp,
+      summary.totals.readCount, summary.totals.writeCount,
+      summary.totals.estimatedTokens, summary.totals.repeatedReads,
+      summary.totals.fileIndexHits, summary.totals.fileIndexMisses,
+      summary.estimatedSavings, archived
+    );
+  }
+
+  private appendChildRows(summary: SessionSummary): void {
+    const insertRead = this.db.prepare(
+      "INSERT INTO ledger_reads (session_id, file_path, estimated_tokens, read_count) VALUES (?, ?, ?, ?)"
+    );
+    for (const r of summary.reads ?? []) {
+      insertRead.run(summary.sessionId, r.filePath, r.estimatedTokens, r.readCount);
+    }
+    const insertWrite = this.db.prepare(
+      "INSERT INTO ledger_writes (session_id, file_path, estimated_tokens, action) VALUES (?, ?, ?, ?)"
+    );
+    for (const w of summary.writes ?? []) {
+      insertWrite.run(summary.sessionId, w.filePath, w.estimatedTokens, w.action);
+    }
+  }
+
+  private addToLifetime(deviceId: string, summary: SessionSummary): void {
+    this.adjustLifetime(deviceId, summary, +1);
+  }
+
+  private subtractFromLifetime(
+    deviceId: string,
+    existing: { estimated_tokens: number; read_count: number; write_count: number; file_index_hits: number; file_index_misses: number; repeated_reads: number; estimated_savings: number }
+  ): void {
+    // Reconstruct a SessionSummary-shaped delta from the stored row.
+    const synthetic: SessionSummary = {
+      sessionId: "",
+      startTimestamp: "",
+      endTimestamp: "",
+      reads: [],
+      writes: [],
+      totals: {
+        readCount: existing.read_count,
+        writeCount: existing.write_count,
+        estimatedTokens: existing.estimated_tokens,
+        repeatedReads: existing.repeated_reads,
+        fileIndexHits: existing.file_index_hits,
+        fileIndexMisses: existing.file_index_misses,
+      },
+      estimatedSavings: existing.estimated_savings,
+    };
+    this.adjustLifetime(deviceId, synthetic, -1);
+  }
+
+  private adjustLifetime(
+    deviceId: string,
+    summary: SessionSummary,
+    sign: 1 | -1
+  ): void {
+    const s = sign;
+    this.db.prepare(`
+      INSERT INTO ledger_lifetime
+        (device_id, total_tokens, total_reads, total_writes, total_sessions,
+         total_file_index_hits, total_file_index_misses, total_repeated_reads,
+         total_estimated_savings)
+      VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+      ON CONFLICT(device_id) DO UPDATE SET
+        total_tokens             = ledger_lifetime.total_tokens             + excluded.total_tokens,
+        total_reads              = ledger_lifetime.total_reads              + excluded.total_reads,
+        total_writes             = ledger_lifetime.total_writes             + excluded.total_writes,
+        total_sessions           = ledger_lifetime.total_sessions           + excluded.total_sessions,
+        total_file_index_hits    = ledger_lifetime.total_file_index_hits    + excluded.total_file_index_hits,
+        total_file_index_misses  = ledger_lifetime.total_file_index_misses  + excluded.total_file_index_misses,
+        total_repeated_reads     = ledger_lifetime.total_repeated_reads     + excluded.total_repeated_reads,
+        total_estimated_savings  = ledger_lifetime.total_estimated_savings  + excluded.total_estimated_savings
+    `).run(
+      deviceId,
+      s * summary.totals.estimatedTokens,
+      s * summary.totals.readCount,
+      s * summary.totals.writeCount,
+      s * 1,
+      s * summary.totals.fileIndexHits,
+      s * summary.totals.fileIndexMisses,
+      s * summary.totals.repeatedReads,
+      s * summary.estimatedSavings
+    );
+  }
+
+  private fetchSession(sessionId: string): {
+    session_id: string;
+    device_id: string;
+    estimated_tokens: number;
+    read_count: number;
+    write_count: number;
+    file_index_hits: number;
+    file_index_misses: number;
+    repeated_reads: number;
+    estimated_savings: number;
+  } | null {
+    const row = this.db
+      .prepare("SELECT * FROM ledger_sessions WHERE session_id = ?")
+      .get(sessionId);
+    if (!row) return null;
+    return row as never;
+  }
+
+  private hydrateSessions(sql: string): LedgerSession[] {
+    const rows = this.db.prepare(sql).all() as Array<Record<string, unknown>>;
+    if (rows.length === 0) return [];
+    const ids = rows.map((r) => String(r.session_id));
+    const readsBySession = this.groupChildren(
+      "SELECT session_id, file_path, estimated_tokens, read_count FROM ledger_reads WHERE session_id IN (" +
+        ids.map(() => "?").join(",") + ")",
+      ids
+    );
+    const writesBySession = this.groupChildren(
+      "SELECT session_id, file_path, estimated_tokens, action FROM ledger_writes WHERE session_id IN (" +
+        ids.map(() => "?").join(",") + ")",
+      ids
+    );
+    return rows.map((r) => {
+      const sid = String(r.session_id);
+      return {
+        sessionId: sid,
+        startTimestamp: String(r.start_timestamp),
+        endTimestamp: String(r.end_timestamp),
+        reads: (readsBySession.get(sid) ?? []).map((x) => ({
+          filePath: String(x.file_path),
+          estimatedTokens: Number(x.estimated_tokens),
+          readCount: Number(x.read_count),
+        })),
+        writes: (writesBySession.get(sid) ?? []).map((x) => ({
+          filePath: String(x.file_path),
+          estimatedTokens: Number(x.estimated_tokens),
+          action: x.action as "create" | "edit",
+        })),
+        totals: {
+          readCount:        Number(r.read_count),
+          writeCount:       Number(r.write_count),
+          estimatedTokens:  Number(r.estimated_tokens),
+          repeatedReads:    Number(r.repeated_reads),
+          fileIndexHits:    Number(r.file_index_hits),
+          fileIndexMisses:  Number(r.file_index_misses),
+        },
+        estimatedSavings: Number(r.estimated_savings),
+      };
+    });
+  }
+
+  private groupChildren(sql: string, ids: string[]): Map<string, Array<Record<string, unknown>>> {
+    const out = new Map<string, Array<Record<string, unknown>>>();
+    if (ids.length === 0) return out;
+    const rows = this.db.prepare(sql).all(...ids) as Array<Record<string, unknown>>;
+    for (const r of rows) {
+      const sid = String(r.session_id);
+      let list = out.get(sid);
+      if (!list) {
+        list = [];
+        out.set(sid, list);
+      }
+      list.push(r);
+    }
+    return out;
+  }
+}

package/src/storage/db.ts

@@ -0,0 +1,121 @@
+// Project database lifecycle. The handle is opened lazily and cached per
+// process (hook commands are short-lived; their first call to a repository
+// triggers the open, and the handle is closed via the registered exit hook).
+//
+// On first open for a project that has on-disk JSON state, the lazy JSON
+// importer runs (see `migrate-json.ts`). The importer is idempotent — once
+// `meta.migrated_from_json_at` is set, it returns immediately.
+
+import { mkdirSync } from "fs";
+import { dirname } from "path";
+import { projectDbPath } from "../core/paths";
+import { openDriver, type DbDriver } from "./driver";
+import { applySchema } from "./schema";
+import { migrateJsonIfNeeded } from "./migrate-json";
+
+export { projectDbPath } from "../core/paths";
+
+interface ConnectionEntry {
+  driver: DbDriver;
+  closed: boolean;
+}
+
+const handles = new Map<string, ConnectionEntry>();
+let exitHookInstalled = false;
+
+function installExitHook(): void {
+  if (exitHookInstalled) return;
+  exitHookInstalled = true;
+  const closeAll = (): void => {
+    for (const entry of handles.values()) {
+      if (entry.closed) continue;
+      try {
+        entry.driver.close();
+      } catch {
+        // best effort — process is shutting down
+      }
+      entry.closed = true;
+    }
+  };
+  process.on("exit", closeAll);
+}
+
+// Test-only — drop cached handles between tests that wipe MINK_ROOT_OVERRIDE.
+// Production code never calls this; the exit hook handles real shutdown.
+export function _resetDbCacheForTests(): void {
+  for (const entry of handles.values()) {
+    if (entry.closed) continue;
+    try {
+      entry.driver.close();
+    } catch {
+      // ignore
+    }
+    entry.closed = true;
+  }
+  handles.clear();
+}
+
+function applyPragmas(db: DbDriver): void {
+  // WAL: enables concurrent readers during a writer; survives crashes.
+  // synchronous=NORMAL: safe with WAL, ~2-5x faster than FULL.
+  // foreign_keys=ON: required for bug_tags / bug_related cascades.
+  // busy_timeout: matches the existing 5s hook safety timeout in
+  // src/core/runtime.ts — under contention SQLite will retry rather than
+  // throw SQLITE_BUSY immediately.
+  db.exec("PRAGMA journal_mode = WAL");
+  db.exec("PRAGMA synchronous = NORMAL");
+  db.exec("PRAGMA foreign_keys = ON");
+  db.exec("PRAGMA busy_timeout = 5000");
+}
+
+export function openProjectDb(cwd: string): DbDriver {
+  const path = projectDbPath(cwd);
+  const cached = handles.get(path);
+  if (cached && !cached.closed) return cached.driver;
+
+  mkdirSync(dirname(path), { recursive: true });
+  const driver = openDriver(path);
+  applyPragmas(driver);
+  applySchema(driver);
+
+  // Run migration AFTER applySchema so the importer can write into existing
+  // tables. The importer no-ops once `meta.migrated_from_json_at` is set.
+  try {
+    migrateJsonIfNeeded(driver, cwd);
+  } catch (err) {
+    // Migration failures should not block the process — log and continue
+    // with an empty DB. Phase 2 callers will fall back to legacy JSON reads.
+    // (We rethrow for tests via MINK_DB_STRICT_MIGRATE=1.)
+    if (process.env.MINK_DB_STRICT_MIGRATE === "1") throw err;
+    console.warn(
+      `[mink] JSON → SQLite migration failed for ${cwd}: ${
+        (err as Error).message
+      }`
+    );
+  }
+
+  installExitHook();
+  handles.set(path, { driver, closed: false });
+  return driver;
+}
+
+// Force a WAL checkpoint and close the handle for the given cwd. Used by
+// `mink sync` before pushing so the .db is self-contained (the -wal/-shm
+// sidecars are not synced).
+export function checkpointAndClose(cwd: string): void {
+  const path = projectDbPath(cwd);
+  const entry = handles.get(path);
+  if (!entry || entry.closed) return;
+  try {
+    entry.driver.exec("PRAGMA wal_checkpoint(TRUNCATE)");
+  } catch {
+    // best effort
+  }
+  try {
+    entry.driver.close();
+  } catch {
+    // best effort
+  }
+  entry.closed = true;
+  handles.delete(path);
+}

package/src/storage/driver.bun.ts

@@ -0,0 +1,99 @@
+// bun:sqlite implementation of the DbDriver interface.
+// Selected at build time when MINK_RUNTIME === "bun" and via `typeof Bun`
+// detection when running unbundled.
+
+import type { DbDriver, DriverModule, SqlParam, Statement } from "./driver";
+
+// Use require() so the type-only import path doesn't trip Node when this
+// file is loaded under the wrong runtime by mistake (the runtime dispatcher
+// in driver.ts is supposed to prevent that).
+const { Database } = require("bun:sqlite") as typeof import("bun:sqlite");
+
+class BunStatement implements Statement {
+  constructor(private readonly stmt: import("bun:sqlite").Statement) {}
+
+  run(...params: SqlParam[]): { changes: number | bigint; lastInsertRowid: number | bigint } {
+    const r = this.stmt.run(...(params as never[]));
+    return { changes: r.changes, lastInsertRowid: r.lastInsertRowid };
+  }
+
+  get(...params: SqlParam[]) {
+    const row = this.stmt.get(...(params as never[]));
+    return (row ?? undefined) as Record<string, unknown> | undefined;
+  }
+
+  all(...params: SqlParam[]) {
+    return this.stmt.all(...(params as never[])) as Record<string, unknown>[];
+  }
+}
+
+class BunDriver implements DbDriver {
+  readonly filename: string;
+  private readonly db: import("bun:sqlite").Database;
+  private readonly txnDepth = { value: 0 };
+
+  constructor(filename: string) {
+    this.filename = filename;
+    this.db = new Database(filename, { create: true });
+  }
+
+  prepare(sql: string): Statement {
+    return new BunStatement(this.db.prepare(sql));
+  }
+
+  exec(sql: string): void {
+    this.db.exec(sql);
+  }
+
+  // We implement transactions manually (rather than using bun:sqlite's
+  // `db.transaction(fn)` wrapper) so semantics match node:sqlite exactly:
+  // synchronous, nestable via savepoints, IMMEDIATE locking to fail fast
+  // when another writer is mid-transaction.
+  transaction<T>(fn: () => T): T {
+    if (this.txnDepth.value > 0) {
+      const sp = `sp_${this.txnDepth.value}`;
+      this.db.exec(`SAVEPOINT ${sp}`);
+      this.txnDepth.value++;
+      try {
+        const result = fn();
+        this.db.exec(`RELEASE SAVEPOINT ${sp}`);
+        this.txnDepth.value--;
+        return result;
+      } catch (err) {
+        this.db.exec(`ROLLBACK TO SAVEPOINT ${sp}`);
+        this.db.exec(`RELEASE SAVEPOINT ${sp}`);
+        this.txnDepth.value--;
+        throw err;
+      }
+    }
+    this.db.exec("BEGIN IMMEDIATE");
+    this.txnDepth.value++;
+    try {
+      const result = fn();
+      this.db.exec("COMMIT");
+      this.txnDepth.value--;
+      return result;
+    } catch (err) {
+      this.db.exec("ROLLBACK");
+      this.txnDepth.value--;
+      throw err;
+    }
+  }
+
+  close(): void {
+    this.db.close();
+  }
+
+  pragma(stmt: string): unknown {
+    // bun:sqlite has no dedicated pragma() helper; route through exec/query.
+    // Pragmas that return a value (e.g. `journal_mode`) are SELECT-shaped.
+    if (/^[a-z_]+\s*=/i.test(stmt) || /^[a-z_]+\s*\([^)]*\)/i.test(stmt)) {
+      // Assignment or call form — no result expected, but the sqlite engine
+      // still returns the new value. Query so callers can read it.
+      return this.db.prepare(`PRAGMA ${stmt}`).all();
+    }
+    return this.db.prepare(`PRAGMA ${stmt}`).all();
+  }
+}
+
+export const open: DriverModule["open"] = (filename) => new BunDriver(filename);