engrm 0.1.0

package/src/storage/migrations.ts

@@ -0,0 +1,261 @@
+import { Database } from "bun:sqlite";
+
+/**
+ * Schema version tracking and migrations for engrm.
+ *
+ * All schema changes go through numbered migrations. The current version
+ * is stored in the `schema_version` pragma. Migrations run sequentially
+ * on startup if the DB is behind.
+ */
+
+interface Migration {
+  version: number;
+  description: string;
+  sql: string;
+  /** If provided, migration is skipped when this returns false. */
+  condition?: (db: Database) => boolean;
+}
+
+const MIGRATIONS: Migration[] = [
+  {
+    version: 1,
+    description: "Initial schema: projects, observations, sessions, sync, FTS5",
+    sql: `
+      -- Projects (canonical identity across machines)
+      CREATE TABLE projects (
+          id              INTEGER PRIMARY KEY AUTOINCREMENT,
+          canonical_id    TEXT UNIQUE NOT NULL,
+          name            TEXT NOT NULL,
+          local_path      TEXT,
+          remote_url      TEXT,
+          first_seen_epoch INTEGER NOT NULL,
+          last_active_epoch INTEGER NOT NULL
+      );
+
+      -- Core observations table
+      CREATE TABLE observations (
+          id              INTEGER PRIMARY KEY AUTOINCREMENT,
+          session_id      TEXT,
+          project_id      INTEGER NOT NULL REFERENCES projects(id),
+          type            TEXT NOT NULL CHECK (type IN (
+              'bugfix', 'discovery', 'decision', 'pattern',
+              'change', 'feature', 'refactor', 'digest'
+          )),
+          title           TEXT NOT NULL,
+          narrative       TEXT,
+          facts           TEXT,
+          concepts        TEXT,
+          files_read      TEXT,
+          files_modified  TEXT,
+          quality         REAL DEFAULT 0.5 CHECK (quality BETWEEN 0.0 AND 1.0),
+          lifecycle       TEXT DEFAULT 'active' CHECK (lifecycle IN (
+              'active', 'aging', 'archived', 'purged', 'pinned'
+          )),
+          sensitivity     TEXT DEFAULT 'shared' CHECK (sensitivity IN (
+              'shared', 'personal', 'secret'
+          )),
+          user_id         TEXT NOT NULL,
+          device_id       TEXT NOT NULL,
+          agent           TEXT DEFAULT 'claude-code',
+          created_at      TEXT NOT NULL,
+          created_at_epoch INTEGER NOT NULL,
+          archived_at_epoch INTEGER,
+          compacted_into  INTEGER REFERENCES observations(id) ON DELETE SET NULL
+      );
+
+      -- Session tracking
+      CREATE TABLE sessions (
+          id              INTEGER PRIMARY KEY AUTOINCREMENT,
+          session_id      TEXT UNIQUE NOT NULL,
+          project_id      INTEGER REFERENCES projects(id),
+          user_id         TEXT NOT NULL,
+          device_id       TEXT NOT NULL,
+          agent           TEXT DEFAULT 'claude-code',
+          status          TEXT DEFAULT 'active' CHECK (status IN ('active', 'completed')),
+          observation_count INTEGER DEFAULT 0,
+          started_at_epoch INTEGER,
+          completed_at_epoch INTEGER
+      );
+
+      -- Session summaries (generated on Stop hook)
+      CREATE TABLE session_summaries (
+          id              INTEGER PRIMARY KEY AUTOINCREMENT,
+          session_id      TEXT UNIQUE NOT NULL,
+          project_id      INTEGER REFERENCES projects(id),
+          user_id         TEXT NOT NULL,
+          request         TEXT,
+          investigated    TEXT,
+          learned         TEXT,
+          completed       TEXT,
+          next_steps      TEXT,
+          created_at_epoch INTEGER
+      );
+
+      -- Sync outbox (offline-first queue)
+      CREATE TABLE sync_outbox (
+          id              INTEGER PRIMARY KEY AUTOINCREMENT,
+          record_type     TEXT NOT NULL CHECK (record_type IN ('observation', 'summary')),
+          record_id       INTEGER NOT NULL,
+          status          TEXT DEFAULT 'pending' CHECK (status IN (
+              'pending', 'syncing', 'synced', 'failed'
+          )),
+          retry_count     INTEGER DEFAULT 0,
+          max_retries     INTEGER DEFAULT 10,
+          last_error      TEXT,
+          created_at_epoch INTEGER NOT NULL,
+          synced_at_epoch  INTEGER,
+          next_retry_epoch INTEGER
+      );
+
+      -- Sync high-water mark and lifecycle job tracking
+      CREATE TABLE sync_state (
+          key   TEXT PRIMARY KEY,
+          value TEXT NOT NULL
+      );
+
+      -- FTS5 for local offline search (external content mode)
+      CREATE VIRTUAL TABLE observations_fts USING fts5(
+          title, narrative, facts, concepts,
+          content=observations,
+          content_rowid=id
+      );
+
+      -- Indexes: observations
+      CREATE INDEX idx_observations_project ON observations(project_id);
+      CREATE INDEX idx_observations_project_lifecycle ON observations(project_id, lifecycle);
+      CREATE INDEX idx_observations_type ON observations(type);
+      CREATE INDEX idx_observations_created ON observations(created_at_epoch);
+      CREATE INDEX idx_observations_session ON observations(session_id);
+      CREATE INDEX idx_observations_lifecycle ON observations(lifecycle);
+      CREATE INDEX idx_observations_quality ON observations(quality);
+      CREATE INDEX idx_observations_user ON observations(user_id);
+
+      -- Indexes: sessions
+      CREATE INDEX idx_sessions_project ON sessions(project_id);
+
+      -- Indexes: sync outbox
+      CREATE INDEX idx_outbox_status ON sync_outbox(status, next_retry_epoch);
+      CREATE INDEX idx_outbox_record ON sync_outbox(record_type, record_id);
+    `,
+  },
+  {
+    version: 2,
+    description: "Add superseded_by for knowledge supersession",
+    sql: `
+      ALTER TABLE observations ADD COLUMN superseded_by INTEGER REFERENCES observations(id) ON DELETE SET NULL;
+      CREATE INDEX idx_observations_superseded ON observations(superseded_by);
+    `,
+  },
+  {
+    version: 3,
+    description: "Add remote_source_id for pull deduplication",
+    sql: `
+      ALTER TABLE observations ADD COLUMN remote_source_id TEXT;
+      CREATE UNIQUE INDEX idx_observations_remote_source ON observations(remote_source_id) WHERE remote_source_id IS NOT NULL;
+    `,
+  },
+  {
+    version: 4,
+    description: "Add sqlite-vec for local semantic search",
+    sql: `
+      CREATE VIRTUAL TABLE vec_observations USING vec0(
+        observation_id INTEGER PRIMARY KEY,
+        embedding float[384]
+      );
+    `,
+    condition: (db) => isVecExtensionLoaded(db),
+  },
+  {
+    version: 5,
+    description: "Session metrics and security findings",
+    sql: `
+      ALTER TABLE sessions ADD COLUMN files_touched_count INTEGER DEFAULT 0;
+      ALTER TABLE sessions ADD COLUMN searches_performed INTEGER DEFAULT 0;
+      ALTER TABLE sessions ADD COLUMN tool_calls_count INTEGER DEFAULT 0;
+
+      CREATE TABLE security_findings (
+          id              INTEGER PRIMARY KEY AUTOINCREMENT,
+          session_id      TEXT,
+          project_id      INTEGER NOT NULL REFERENCES projects(id),
+          finding_type    TEXT NOT NULL,
+          severity        TEXT NOT NULL DEFAULT 'medium' CHECK (severity IN ('critical', 'high', 'medium', 'low')),
+          pattern_name    TEXT NOT NULL,
+          file_path       TEXT,
+          snippet         TEXT,
+          tool_name       TEXT,
+          user_id         TEXT NOT NULL,
+          device_id       TEXT NOT NULL,
+          created_at_epoch INTEGER NOT NULL
+      );
+
+      CREATE INDEX idx_security_findings_session ON security_findings(session_id);
+      CREATE INDEX idx_security_findings_project ON security_findings(project_id, created_at_epoch);
+      CREATE INDEX idx_security_findings_severity ON security_findings(severity);
+    `,
+  },
+];
+
+/**
+ * Check if the sqlite-vec extension is loaded in this database.
+ */
+function isVecExtensionLoaded(db: Database): boolean {
+  try {
+    db.exec("SELECT vec_version()");
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Run all pending migrations on the given database.
+ * Uses SQLite's user_version pragma to track schema version.
+ */
+export function runMigrations(db: Database): void {
+  const currentVersion = db.query("PRAGMA user_version").get() as {
+    user_version: number;
+  };
+  let version = currentVersion.user_version;
+
+  for (const migration of MIGRATIONS) {
+    if (migration.version <= version) continue;
+
+    // Skip conditional migrations when condition is not met
+    if (migration.condition && !migration.condition(db)) {
+      continue;
+    }
+
+    db.exec("BEGIN TRANSACTION");
+    try {
+      db.exec(migration.sql);
+      db.exec(`PRAGMA user_version = ${migration.version}`);
+      db.exec("COMMIT");
+      version = migration.version;
+    } catch (error) {
+      db.exec("ROLLBACK");
+      throw new Error(
+        `Migration ${migration.version} (${migration.description}) failed: ${
+          error instanceof Error ? error.message : String(error)
+        }`
+      );
+    }
+  }
+}
+
+/**
+ * Get the current schema version.
+ */
+export function getSchemaVersion(db: Database): number {
+  const result = db.query("PRAGMA user_version").get() as {
+    user_version: number;
+  };
+  return result.user_version;
+}
+
+/**
+ * Expected schema version after all unconditional migrations have run.
+ * Conditional migrations (e.g., sqlite-vec) may bump this higher at runtime.
+ */
+export const LATEST_SCHEMA_VERSION = MIGRATIONS
+  .filter((m) => !m.condition)
+  .reduce((max, m) => Math.max(max, m.version), 0);

package/src/storage/outbox.test.ts

@@ -0,0 +1,229 @@
+import { describe, expect, test, beforeEach, afterEach } from "bun:test";
+import { mkdtempSync, rmSync } from "node:fs";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import { MemDatabase } from "./sqlite.js";
+import {
+  getPendingEntries,
+  markSyncing,
+  markSynced,
+  markFailed,
+  purgeSynced,
+  getOutboxStats,
+} from "./outbox.js";
+
+let db: MemDatabase;
+let tmpDir: string;
+let projectId: number;
+
+beforeEach(() => {
+  tmpDir = mkdtempSync(join(tmpdir(), "candengo-mem-outbox-test-"));
+  db = new MemDatabase(join(tmpDir, "test.db"));
+  const project = db.upsertProject({
+    canonical_id: "github.com/org/repo",
+    name: "repo",
+  });
+  projectId = project.id;
+});
+
+afterEach(() => {
+  db.close();
+  rmSync(tmpDir, { recursive: true, force: true });
+});
+
+function createObsAndOutboxEntry(): number {
+  const obs = db.insertObservation({
+    project_id: projectId,
+    type: "bugfix",
+    title: "Fix something",
+    quality: 0.5,
+    user_id: "david",
+    device_id: "laptop-abc",
+  });
+  // insertObservation does NOT add to outbox — that's the save tool's job.
+  // We add manually here for outbox-specific tests.
+  db.addToOutbox("observation", obs.id);
+
+  const entry = db.db
+    .query<{ id: number }, [number]>(
+      "SELECT id FROM sync_outbox WHERE record_id = ? ORDER BY id DESC LIMIT 1"
+    )
+    .get(obs.id);
+  return entry!.id;
+}
+
+describe("getPendingEntries", () => {
+  test("returns pending entries", () => {
+    createObsAndOutboxEntry();
+    const entries = getPendingEntries(db);
+    expect(entries.length).toBe(1);
+    expect(entries[0]!.status).toBe("pending");
+  });
+
+  test("returns empty when no pending entries", () => {
+    expect(getPendingEntries(db)).toEqual([]);
+  });
+
+  test("respects limit", () => {
+    createObsAndOutboxEntry();
+    createObsAndOutboxEntry();
+    createObsAndOutboxEntry();
+    const entries = getPendingEntries(db, 2);
+    expect(entries.length).toBe(2);
+  });
+
+  test("includes failed entries past retry time", () => {
+    const entryId = createObsAndOutboxEntry();
+    markFailed(db, entryId, "network error");
+
+    // Set next_retry_epoch to the past
+    db.db
+      .query("UPDATE sync_outbox SET next_retry_epoch = 0 WHERE id = ?")
+      .run(entryId);
+
+    const entries = getPendingEntries(db);
+    expect(entries.length).toBe(1);
+  });
+
+  test("excludes failed entries before retry time", () => {
+    const entryId = createObsAndOutboxEntry();
+    markFailed(db, entryId, "network error");
+    // next_retry_epoch is in the future by default
+
+    const entries = getPendingEntries(db);
+    // Should only include failed entries whose retry time has passed
+    // Since we just marked it failed, next_retry is in the future
+    const failedEntries = entries.filter((e) => e.status === "failed");
+    expect(failedEntries.length).toBe(0);
+  });
+});
+
+describe("markSyncing", () => {
+  test("sets status to syncing", () => {
+    const entryId = createObsAndOutboxEntry();
+    markSyncing(db, entryId);
+
+    const entry = db.db
+      .query<{ status: string }, [number]>(
+        "SELECT status FROM sync_outbox WHERE id = ?"
+      )
+      .get(entryId);
+    expect(entry!.status).toBe("syncing");
+  });
+});
+
+describe("markSynced", () => {
+  test("sets status to synced with timestamp", () => {
+    const entryId = createObsAndOutboxEntry();
+    markSynced(db, entryId);
+
+    const entry = db.db
+      .query<{ status: string; synced_at_epoch: number | null }, [number]>(
+        "SELECT status, synced_at_epoch FROM sync_outbox WHERE id = ?"
+      )
+      .get(entryId);
+    expect(entry!.status).toBe("synced");
+    expect(entry!.synced_at_epoch).not.toBeNull();
+  });
+});
+
+describe("markFailed", () => {
+  test("increments retry count atomically", () => {
+    const entryId = createObsAndOutboxEntry();
+
+    markFailed(db, entryId, "error 1");
+    let entry = db.db
+      .query<{ retry_count: number }, [number]>(
+        "SELECT retry_count FROM sync_outbox WHERE id = ?"
+      )
+      .get(entryId);
+    expect(entry!.retry_count).toBe(1);
+
+    markFailed(db, entryId, "error 2");
+    entry = db.db
+      .query<{ retry_count: number }, [number]>(
+        "SELECT retry_count FROM sync_outbox WHERE id = ?"
+      )
+      .get(entryId);
+    expect(entry!.retry_count).toBe(2);
+  });
+
+  test("sets last_error", () => {
+    const entryId = createObsAndOutboxEntry();
+    markFailed(db, entryId, "connection refused");
+
+    const entry = db.db
+      .query<{ last_error: string | null }, [number]>(
+        "SELECT last_error FROM sync_outbox WHERE id = ?"
+      )
+      .get(entryId);
+    expect(entry!.last_error).toBe("connection refused");
+  });
+
+  test("sets next_retry_epoch in the future", () => {
+    const entryId = createObsAndOutboxEntry();
+    const before = Math.floor(Date.now() / 1000);
+    markFailed(db, entryId, "error");
+
+    const entry = db.db
+      .query<{ next_retry_epoch: number | null }, [number]>(
+        "SELECT next_retry_epoch FROM sync_outbox WHERE id = ?"
+      )
+      .get(entryId);
+    expect(entry!.next_retry_epoch).not.toBeNull();
+    expect(entry!.next_retry_epoch!).toBeGreaterThan(before);
+  });
+});
+
+describe("purgeSynced", () => {
+  test("deletes old synced entries", () => {
+    const entryId = createObsAndOutboxEntry();
+    markSynced(db, entryId);
+
+    // Set synced_at to far in the past
+    db.db
+      .query("UPDATE sync_outbox SET synced_at_epoch = 1000 WHERE id = ?")
+      .run(entryId);
+
+    const deleted = purgeSynced(db, 2000);
+    expect(deleted).toBe(1);
+  });
+
+  test("does not delete recent synced entries", () => {
+    const entryId = createObsAndOutboxEntry();
+    markSynced(db, entryId);
+
+    const deleted = purgeSynced(db, 1000); // cutoff before sync time
+    expect(deleted).toBe(0);
+  });
+
+  test("does not delete pending entries", () => {
+    createObsAndOutboxEntry();
+    const deleted = purgeSynced(db, Math.floor(Date.now() / 1000) + 9999);
+    expect(deleted).toBe(0);
+  });
+});
+
+describe("getOutboxStats", () => {
+  test("returns zeroes when empty", () => {
+    const stats = getOutboxStats(db);
+    expect(stats["pending"]).toBe(0);
+    expect(stats["syncing"]).toBe(0);
+    expect(stats["synced"]).toBe(0);
+    expect(stats["failed"]).toBe(0);
+  });
+
+  test("counts by status", () => {
+    const id1 = createObsAndOutboxEntry();
+    const id2 = createObsAndOutboxEntry();
+    createObsAndOutboxEntry(); // stays pending
+
+    markSynced(db, id1);
+    markFailed(db, id2, "err");
+
+    const stats = getOutboxStats(db);
+    expect(stats["pending"]).toBe(1);
+    expect(stats["synced"]).toBe(1);
+    expect(stats["failed"]).toBe(1);
+  });
+});

package/src/storage/outbox.ts

@@ -0,0 +1,131 @@
+/**
+ * Sync outbox queue management.
+ *
+ * The outbox queues observations and summaries for push to Candengo Vector.
+ * Supports exponential backoff on retries and batch processing.
+ *
+ * Flow: pending → syncing → synced | failed (with retry)
+ */
+
+import type { MemDatabase } from "./sqlite.js";
+
+export interface OutboxEntry {
+  id: number;
+  record_type: "observation" | "summary";
+  record_id: number;
+  status: string;
+  retry_count: number;
+  max_retries: number;
+  last_error: string | null;
+  created_at_epoch: number;
+  synced_at_epoch: number | null;
+  next_retry_epoch: number | null;
+}
+
+/**
+ * Get pending outbox entries that are ready to sync.
+ * Returns entries that are 'pending' or 'failed' with next_retry_epoch in the past.
+ */
+export function getPendingEntries(
+  db: MemDatabase,
+  limit: number = 50
+): OutboxEntry[] {
+  const now = Math.floor(Date.now() / 1000);
+  return db.db
+    .query<OutboxEntry, [number, number]>(
+      `SELECT * FROM sync_outbox
+      WHERE (status = 'pending')
+        OR (status = 'failed' AND retry_count < max_retries AND (next_retry_epoch IS NULL OR next_retry_epoch <= ?))
+      ORDER BY created_at_epoch ASC
+      LIMIT ?`
+    )
+    .all(now, limit);
+}
+
+/**
+ * Mark an entry as syncing (in-progress).
+ */
+export function markSyncing(db: MemDatabase, entryId: number): void {
+  db.db
+    .query("UPDATE sync_outbox SET status = 'syncing' WHERE id = ?")
+    .run(entryId);
+}
+
+/**
+ * Mark an entry as successfully synced.
+ */
+export function markSynced(db: MemDatabase, entryId: number): void {
+  const now = Math.floor(Date.now() / 1000);
+  db.db
+    .query(
+      "UPDATE sync_outbox SET status = 'synced', synced_at_epoch = ? WHERE id = ?"
+    )
+    .run(now, entryId);
+}
+
+/**
+ * Mark an entry as failed with exponential backoff for retry.
+ * Backoff: 30s, 60s, 120s, 240s, ... capped at 1 hour.
+ */
+export function markFailed(
+  db: MemDatabase,
+  entryId: number,
+  error: string
+): void {
+  const now = Math.floor(Date.now() / 1000);
+
+  // Atomic increment — avoids TOCTOU race on retry_count.
+  // Backoff: 30 * 2^(retry_count) seconds, capped at 3600.
+  db.db
+    .query(
+      `UPDATE sync_outbox SET
+        status = 'failed',
+        retry_count = retry_count + 1,
+        last_error = ?,
+        next_retry_epoch = ? + MIN(30 * (1 << retry_count), 3600)
+      WHERE id = ?`
+    )
+    .run(error, now, entryId);
+}
+
+/**
+ * Clean up old synced entries (older than the given epoch).
+ * Prevents the outbox table from growing unbounded.
+ */
+export function purgeSynced(
+  db: MemDatabase,
+  olderThanEpoch: number
+): number {
+  const result = db.db
+    .query(
+      "DELETE FROM sync_outbox WHERE status = 'synced' AND synced_at_epoch < ?"
+    )
+    .run(olderThanEpoch);
+  return result.changes;
+}
+
+/**
+ * Get outbox stats for diagnostics.
+ */
+export function getOutboxStats(
+  db: MemDatabase
+): Record<string, number> {
+  const rows = db.db
+    .query<{ status: string; count: number }, []>(
+      "SELECT status, COUNT(*) as count FROM sync_outbox GROUP BY status"
+    )
+    .all();
+
+  const stats: Record<string, number> = {
+    pending: 0,
+    syncing: 0,
+    synced: 0,
+    failed: 0,
+  };
+
+  for (const row of rows) {
+    stats[row.status] = row.count;
+  }
+
+  return stats;
+}