@loreai/core 0.15.0 → 0.17.0

package/src/db.ts

@@ -2,8 +2,26 @@ import { Database } from "#db/driver";
 import { join, dirname } from "path";
 import { mkdirSync } from "fs";
 import { homedir } from "os";
+import { getGitRemote } from "./git";
 
-const SCHEMA_VERSION = 13;
+/**
+ * Extract the repository name from a normalized git remote URL.
+ *
+ * Examples:
+ *   "github.com/BYK/LoreAI" → "LoreAI"
+ *   "github.com/org/repo"    → "repo"
+ *   "github.com"             → null (no path components)
+ *   null                     → null
+ */
+export function repoNameFromRemote(remote: string | null): string | null {
+  if (!remote) return null;
+  const lastSlash = remote.lastIndexOf("/");
+  if (lastSlash < 0) return null;
+  const name = remote.slice(lastSlash + 1);
+  return name.length > 0 ? name : null;
+}
+
+const SCHEMA_VERSION = 16;
 
 const MIGRATIONS: string[] = [
   `
@@ -362,6 +380,79 @@ const MIGRATIONS: string[] = [
     updated_at INTEGER NOT NULL
   );
   `,
+  `
+  -- Version 14: Git-based project identification.
+  --
+  -- Projects can now be identified by their git remote URL in addition to
+  -- filesystem path. This enables worktree, clone, and fork awareness:
+  -- the same repository accessed from different paths shares one project.
+  --
+  -- git_remote: Normalized canonical remote URL (e.g. "github.com/user/repo").
+  -- NULL for non-git directories or repos with no remotes.
+  --
+  -- project_path_aliases: Maps additional filesystem paths to existing
+  -- projects. When ensureProject() finds a match by git_remote, the
+  -- alternate path is registered here for O(1) subsequent lookups.
+  ALTER TABLE projects ADD COLUMN git_remote TEXT;
+  CREATE INDEX IF NOT EXISTS idx_projects_git_remote ON projects(git_remote);
+
+  CREATE TABLE IF NOT EXISTS project_path_aliases (
+    path TEXT PRIMARY KEY,
+    project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE
+  );
+  `,
+
+  `
+  -- Version 15: Cache warming survival histograms.
+  --
+  -- Persists global (per-project, per-time-slot) inter-turn gap histograms
+  -- across gateway restarts. These histograms feed the survival analysis
+  -- model that decides whether to send speculative cache-warming pings.
+  -- Without persistence, the model has no data until enough turns rebuild
+  -- the histogram from scratch (cold start problem).
+  --
+  -- counts: JSON array of bin counts (21 elements: 20 bins + 1 overflow).
+  -- total: Sum of counts (denormalized for fast reads).
+  CREATE TABLE IF NOT EXISTS warmup_histograms (
+    project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE,
+    time_slot TEXT NOT NULL,
+    counts TEXT NOT NULL DEFAULT '[]',
+    total INTEGER NOT NULL DEFAULT 0,
+    updated_at INTEGER NOT NULL DEFAULT 0,
+    PRIMARY KEY (project_id, time_slot)
+  );
+  `,
+  `
+  -- Version 16: Embedding BLOB column for temporal message vector search.
+  -- Same pattern as knowledge (v8) and distillation (v9) embeddings.
+  -- Only undistilled messages are embedded; the column is NULLed when
+  -- a message is marked as distilled (its semantic content is captured
+  -- by the distillation embedding at that point).
+  -- No backfill — new messages get embedded lazily at write time.
+  ALTER TABLE temporal_messages ADD COLUMN embedding BLOB;
+  `,
+  `
+  -- Version 17: Track whether distillation used batch API pricing.
+  -- NULL for pre-migration rows (treated as 'direct' for conservative estimates).
+  -- 'batch' = 50% discount on input+output, 'direct' = full price.
+  ALTER TABLE distillations ADD COLUMN call_type TEXT;
+  `,
+  `
+  -- Version 18: Persist live session cost data so historical estimates
+  -- include cache warming, 1h TTL savings, and batch API savings — metrics
+  -- that were previously lost on gateway restart.
+  -- All cost columns are in USD. Token columns are raw counts.
+  ALTER TABLE session_state ADD COLUMN conversation_cost REAL NOT NULL DEFAULT 0;
+  ALTER TABLE session_state ADD COLUMN worker_cost REAL NOT NULL DEFAULT 0;
+  ALTER TABLE session_state ADD COLUMN conversation_turns INTEGER NOT NULL DEFAULT 0;
+  ALTER TABLE session_state ADD COLUMN cache_read_tokens INTEGER NOT NULL DEFAULT 0;
+  ALTER TABLE session_state ADD COLUMN cache_write_tokens INTEGER NOT NULL DEFAULT 0;
+  ALTER TABLE session_state ADD COLUMN warmup_savings REAL NOT NULL DEFAULT 0;
+  ALTER TABLE session_state ADD COLUMN warmup_hits INTEGER NOT NULL DEFAULT 0;
+  ALTER TABLE session_state ADD COLUMN ttl_savings REAL NOT NULL DEFAULT 0;
+  ALTER TABLE session_state ADD COLUMN ttl_hits INTEGER NOT NULL DEFAULT 0;
+  ALTER TABLE session_state ADD COLUMN batch_savings REAL NOT NULL DEFAULT 0;
+  `,
 ];
 
 function dataDir() {
@@ -370,6 +461,13 @@ function dataDir() {
   return join(base, "opencode-lore");
 }
 
+/** Return the resolved path of the SQLite database file. */
+export function dbPath(): string {
+  const envPath = process.env.LORE_DB_PATH;
+  if (envPath) return envPath;
+  return join(dataDir(), "lore.db");
+}
+
 let instance: Database | undefined;
 
 export function db(): Database {
@@ -380,6 +478,18 @@ export function db(): Database {
     mkdirSync(dirname(envPath), { recursive: true });
     path = envPath;
   } else {
+    // Guard: refuse to open the production DB during test runs.
+    // The test preload (setup.ts) sets LORE_DB_PATH to a temp directory.
+    // If we reach here with NODE_ENV=test, the preload didn't fire
+    // (e.g. bun test invoked from outside the repo). Throw instead of
+    // silently writing test fixtures into the user's live database.
+    if (process.env.NODE_ENV === "test") {
+      throw new Error(
+        "LORE_DB_PATH is not set but NODE_ENV=test. " +
+          "Run tests via `bun test` from the repo root, or set " +
+          "LORE_DB_PATH to a temp path to avoid polluting the production DB.",
+      );
+    }
     const dir = dataDir();
     mkdirSync(dir, { recursive: true });
     path = join(dir, "lore.db");
@@ -507,7 +617,72 @@ function recoverMissingObjects(database: Database) {
       value TEXT NOT NULL,
       updated_at INTEGER NOT NULL
     );
+    CREATE TABLE IF NOT EXISTS project_path_aliases (
+      path TEXT PRIMARY KEY,
+      project_id TEXT NOT NULL REFERENCES projects(id) ON DELETE CASCADE
+    );
   `);
+
+  // Recover missing columns from partial migration runs.
+  // Version 17 added call_type to distillations but the ALTER could have been
+  // skipped if the version was bumped without the column being created.
+  const cols = database
+    .query("PRAGMA table_info(distillations)")
+    .all() as Array<{ name: string }>;
+  if (!cols.some((c) => c.name === "call_type")) {
+    database.exec("ALTER TABLE distillations ADD COLUMN call_type TEXT;");
+  }
+}
+
+/**
+ * Merge all data from `sourceId` project into `targetId` project.
+ *
+ * Moves knowledge, temporal messages, distillations, LAT sections, and
+ * path aliases from source to target. Registers the source project's path
+ * as an alias of the target. Deletes the source project row.
+ *
+ * Used internally during lazy git-remote backfill when two path-only
+ * projects are discovered to share the same git remote.
+ */
+export function mergeProjectInternal(
+  sourceId: string,
+  targetId: string,
+): void {
+  const d = db();
+  d.exec("BEGIN IMMEDIATE");
+  try {
+    d.query("UPDATE knowledge SET project_id = ? WHERE project_id = ?").run(
+      targetId,
+      sourceId,
+    );
+    d.query(
+      "UPDATE temporal_messages SET project_id = ? WHERE project_id = ?",
+    ).run(targetId, sourceId);
+    d.query(
+      "UPDATE distillations SET project_id = ? WHERE project_id = ?",
+    ).run(targetId, sourceId);
+    d.query("UPDATE lat_sections SET project_id = ? WHERE project_id = ?").run(
+      targetId,
+      sourceId,
+    );
+    d.query(
+      "UPDATE OR IGNORE project_path_aliases SET project_id = ? WHERE project_id = ?",
+    ).run(targetId, sourceId);
+    // Register source's path as alias of target
+    const sourceRow = d
+      .query("SELECT path FROM projects WHERE id = ?")
+      .get(sourceId) as { path: string } | null;
+    if (sourceRow) {
+      d.query(
+        "INSERT OR IGNORE INTO project_path_aliases (path, project_id) VALUES (?, ?)",
+      ).run(sourceRow.path, targetId);
+    }
+    d.query("DELETE FROM projects WHERE id = ?").run(sourceId);
+    d.exec("COMMIT");
+  } catch (e) {
+    d.exec("ROLLBACK");
+    throw e;
+  }
 }
 
 export function close() {
@@ -518,17 +693,85 @@ export function close() {
 }
 
 // Project management
+
+/**
+ * Look up or create a project by filesystem path, with git-remote awareness.
+ *
+ * Resolution order:
+ *  1. Exact path match in `projects` table (fast path, O(1) index scan)
+ *  2. Path alias match in `project_path_aliases` (worktree/clone re-visits)
+ *  3. Git remote match — runs `git remote -v` (once per unique path, cached),
+ *     finds an existing project with the same normalized remote URL
+ *  4. Create a new project row
+ *
+ * When a git-remote match is found (step 3), the new path is registered as
+ * an alias so subsequent calls skip the subprocess. If the matched project's
+ * git_remote was not yet populated (pre-v14 rows), it is backfilled lazily.
+ */
 export function ensureProject(path: string, name?: string): string {
+  // 1. Exact path match (fast path)
   const existing = db()
-    .query("SELECT id FROM projects WHERE path = ?")
-    .get(path) as { id: string } | null;
-  if (existing) return existing.id;
+    .query("SELECT id, git_remote FROM projects WHERE path = ?")
+    .get(path) as { id: string; git_remote: string | null } | null;
+  if (existing) {
+    // Lazy backfill: populate git_remote on pre-v14 rows
+    if (!existing.git_remote) {
+      const gitRemote = getGitRemote(path);
+      if (gitRemote) {
+        // Check for conflict: another project already has this git_remote.
+        // If so, merge the conflicting project into this one (one-time).
+        const conflict = db()
+          .query(
+            "SELECT id FROM projects WHERE git_remote = ? AND id != ? LIMIT 1",
+          )
+          .get(gitRemote, existing.id) as { id: string } | null;
+        if (conflict) {
+          mergeProjectInternal(conflict.id, existing.id);
+        }
+        db()
+          .query("UPDATE projects SET git_remote = ? WHERE id = ?")
+          .run(gitRemote, existing.id);
+      }
+    }
+    return existing.id;
+  }
+
+  // 2. Check path aliases (worktree/clone re-visits)
+  const alias = db()
+    .query("SELECT project_id FROM project_path_aliases WHERE path = ?")
+    .get(path) as { project_id: string } | null;
+  if (alias) return alias.project_id;
+
+  // 3. Git remote identification
+  const gitRemote = getGitRemote(path);
+  if (gitRemote) {
+    const byRemote = db()
+      .query("SELECT id FROM projects WHERE git_remote = ? LIMIT 1")
+      .get(gitRemote) as { id: string } | null;
+    if (byRemote) {
+      // Register this path as an alias for O(1) future lookups
+      db()
+        .query(
+          "INSERT OR IGNORE INTO project_path_aliases (path, project_id) VALUES (?, ?)",
+        )
+        .run(path, byRemote.id);
+      return byRemote.id;
+    }
+  }
+
+  // 4. Create new project
   const id = crypto.randomUUID();
   db()
     .query(
-      "INSERT INTO projects (id, path, name, created_at) VALUES (?, ?, ?, ?)",
+      "INSERT INTO projects (id, path, name, git_remote, created_at) VALUES (?, ?, ?, ?, ?)",
     )
-    .run(id, path, name ?? path.split("/").pop() ?? "unknown", Date.now());
+    .run(
+      id,
+      path,
+      name ?? repoNameFromRemote(gitRemote) ?? path.split("/").pop() ?? "unknown",
+      gitRemote,
+      Date.now(),
+    );
   return id;
 }
 
@@ -536,7 +779,13 @@ export function projectId(path: string): string | undefined {
   const row = db()
     .query("SELECT id FROM projects WHERE path = ?")
     .get(path) as { id: string } | null;
-  return row?.id;
+  if (row) return row.id;
+
+  // Check path aliases (worktree/clone paths registered by ensureProject)
+  const alias = db()
+    .query("SELECT project_id FROM project_path_aliases WHERE path = ?")
+    .get(path) as { project_id: string } | null;
+  return alias?.project_id;
 }
 
 /** Look up a project's display name by its internal ID. */
@@ -590,6 +839,136 @@ export function saveForceMinLayer(sessionID: string, layer: number): void {
   }
 }
 
+/** Persisted cost snapshot for a session. */
+export type SessionCostSnapshot = {
+  conversationCost: number;
+  workerCost: number;
+  conversationTurns: number;
+  cacheReadTokens: number;
+  cacheWriteTokens: number;
+  warmupSavings: number;
+  warmupHits: number;
+  ttlSavings: number;
+  ttlHits: number;
+  batchSavings: number;
+};
+
+/**
+ * Persist a session's cost snapshot. Uses INSERT OR REPLACE so it works
+ * whether or not a row already exists (forceMinLayer may have created one).
+ */
+export function saveSessionCosts(sessionID: string, costs: SessionCostSnapshot): void {
+  db()
+    .query(
+      `INSERT INTO session_state (session_id, force_min_layer, updated_at,
+         conversation_cost, worker_cost, conversation_turns,
+         cache_read_tokens, cache_write_tokens,
+         warmup_savings, warmup_hits, ttl_savings, ttl_hits, batch_savings)
+       VALUES (?, COALESCE((SELECT force_min_layer FROM session_state WHERE session_id = ?), 0), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+       ON CONFLICT(session_id) DO UPDATE SET
+         conversation_cost = excluded.conversation_cost,
+         worker_cost = excluded.worker_cost,
+         conversation_turns = excluded.conversation_turns,
+         cache_read_tokens = excluded.cache_read_tokens,
+         cache_write_tokens = excluded.cache_write_tokens,
+         warmup_savings = excluded.warmup_savings,
+         warmup_hits = excluded.warmup_hits,
+         ttl_savings = excluded.ttl_savings,
+         ttl_hits = excluded.ttl_hits,
+         batch_savings = excluded.batch_savings,
+         updated_at = excluded.updated_at`,
+    )
+    .run(
+      sessionID, sessionID, Date.now(),
+      costs.conversationCost, costs.workerCost, costs.conversationTurns,
+      costs.cacheReadTokens, costs.cacheWriteTokens,
+      costs.warmupSavings, costs.warmupHits, costs.ttlSavings, costs.ttlHits, costs.batchSavings,
+    );
+}
+
+/**
+ * Load persisted cost snapshot for a session. Returns null if not stored
+ * or if all cost columns are zero (pre-migration row from forceMinLayer only).
+ */
+export function loadSessionCosts(sessionID: string): SessionCostSnapshot | null {
+  const row = db()
+    .query(
+      `SELECT conversation_cost, worker_cost, conversation_turns,
+              cache_read_tokens, cache_write_tokens,
+              warmup_savings, warmup_hits, ttl_savings, ttl_hits, batch_savings
+       FROM session_state WHERE session_id = ?`,
+    )
+    .get(sessionID) as {
+      conversation_cost: number;
+      worker_cost: number;
+      conversation_turns: number;
+      cache_read_tokens: number;
+      cache_write_tokens: number;
+      warmup_savings: number;
+      warmup_hits: number;
+      ttl_savings: number;
+      ttl_hits: number;
+      batch_savings: number;
+    } | null;
+  if (!row) return null;
+  return {
+    conversationCost: row.conversation_cost,
+    workerCost: row.worker_cost,
+    conversationTurns: row.conversation_turns,
+    cacheReadTokens: row.cache_read_tokens,
+    cacheWriteTokens: row.cache_write_tokens,
+    warmupSavings: row.warmup_savings,
+    warmupHits: row.warmup_hits,
+    ttlSavings: row.ttl_savings,
+    ttlHits: row.ttl_hits,
+    batchSavings: row.batch_savings,
+  };
+}
+
+/**
+ * Load cost snapshots for all sessions that have non-zero cost data.
+ * Returns a map of sessionID → SessionCostSnapshot.
+ */
+export function loadAllSessionCosts(): Map<string, SessionCostSnapshot> {
+  const rows = db()
+    .query(
+      `SELECT session_id, conversation_cost, worker_cost, conversation_turns,
+              cache_read_tokens, cache_write_tokens,
+              warmup_savings, warmup_hits, ttl_savings, ttl_hits, batch_savings
+       FROM session_state
+       WHERE conversation_turns > 0 OR warmup_savings > 0 OR ttl_savings > 0 OR batch_savings > 0`,
+    )
+    .all() as Array<{
+      session_id: string;
+      conversation_cost: number;
+      worker_cost: number;
+      conversation_turns: number;
+      cache_read_tokens: number;
+      cache_write_tokens: number;
+      warmup_savings: number;
+      warmup_hits: number;
+      ttl_savings: number;
+      ttl_hits: number;
+      batch_savings: number;
+    }>;
+  const result = new Map<string, SessionCostSnapshot>();
+  for (const row of rows) {
+    result.set(row.session_id, {
+      conversationCost: row.conversation_cost,
+      workerCost: row.worker_cost,
+      conversationTurns: row.conversation_turns,
+      cacheReadTokens: row.cache_read_tokens,
+      cacheWriteTokens: row.cache_write_tokens,
+      warmupSavings: row.warmup_savings,
+      warmupHits: row.warmup_hits,
+      ttlSavings: row.ttl_savings,
+      ttlHits: row.ttl_hits,
+      batchSavings: row.batch_savings,
+    });
+  }
+  return result;
+}
+
 // ---------------------------------------------------------------------------
 // Installation metadata (metadata table)
 // ---------------------------------------------------------------------------