@loreai/core 0.18.0 → 0.20.0

package/src/db.ts

@@ -515,6 +515,55 @@ const MIGRATIONS: string[] = [
       AND ih.source_id = '__declined__'
   );
   `,
+  `
+  -- Version 23: Persist volatile session tracking state across restarts.
+  -- Previously these were in-memory only, causing duplicate processing,
+  -- false compaction detection, and expensive prompt cache busts on restart.
+  ALTER TABLE session_state ADD COLUMN last_curated_at INTEGER NOT NULL DEFAULT 0;
+  ALTER TABLE session_state ADD COLUMN message_count INTEGER NOT NULL DEFAULT 0;
+  ALTER TABLE session_state ADD COLUMN turns_since_curation INTEGER NOT NULL DEFAULT 0;
+  ALTER TABLE session_state ADD COLUMN ltm_cache_text TEXT;
+  ALTER TABLE session_state ADD COLUMN ltm_cache_tokens INTEGER;
+  ALTER TABLE session_state ADD COLUMN ltm_pin_text TEXT;
+  ALTER TABLE session_state ADD COLUMN ltm_pin_tokens INTEGER;
+  ALTER TABLE session_state ADD COLUMN consecutive_text_only_turns INTEGER NOT NULL DEFAULT 0;
+  `,
+  `
+  -- Version 24: Persist remaining volatile session state across restarts.
+  -- Session identity (Tier 1/2/3 session correlation)
+  ALTER TABLE session_state ADD COLUMN fingerprint TEXT NOT NULL DEFAULT '';
+  ALTER TABLE session_state ADD COLUMN header_session_id TEXT;
+  ALTER TABLE session_state ADD COLUMN header_name TEXT;
+  -- Cache warming state
+  ALTER TABLE session_state ADD COLUMN resolved_conversation_ttl TEXT NOT NULL DEFAULT '5m';
+  ALTER TABLE session_state ADD COLUMN warmup_state TEXT;
+  -- Gradient calibration state (survives restarts to avoid uncalibrated busts)
+  ALTER TABLE session_state ADD COLUMN dynamic_context_cap REAL NOT NULL DEFAULT 0;
+  ALTER TABLE session_state ADD COLUMN bust_rate_ema REAL NOT NULL DEFAULT -1;
+  ALTER TABLE session_state ADD COLUMN inter_bust_interval_ema REAL NOT NULL DEFAULT -1;
+  ALTER TABLE session_state ADD COLUMN last_layer INTEGER NOT NULL DEFAULT 0;
+  ALTER TABLE session_state ADD COLUMN last_known_input INTEGER NOT NULL DEFAULT 0;
+  ALTER TABLE session_state ADD COLUMN last_turn_at INTEGER NOT NULL DEFAULT 0;
+  ALTER TABLE session_state ADD COLUMN last_bust_at INTEGER NOT NULL DEFAULT 0;
+  `,
+  `
+  -- Version 25: Adaptive dedup threshold — store accept/reject feedback
+  -- on embedding-based duplicate pairs for per-project threshold calibration.
+  -- Titles stored instead of FK IDs because entries are deleted during dedup;
+  -- the similarity float is the actual calibration input.
+  CREATE TABLE IF NOT EXISTS dedup_feedback (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    project_id TEXT,
+    entry_a_title TEXT NOT NULL,
+    entry_b_title TEXT NOT NULL,
+    similarity REAL NOT NULL,
+    accepted INTEGER NOT NULL,
+    source TEXT NOT NULL DEFAULT 'manual',
+    created_at INTEGER NOT NULL
+  );
+  CREATE INDEX IF NOT EXISTS idx_dedup_feedback_project
+    ON dedup_feedback(project_id);
+  `,
 ];
 
 /** Return the resolved path of the SQLite database file. */
@@ -764,7 +813,22 @@ export function close() {
  * an alias so subsequent calls skip the subprocess. If the matched project's
  * git_remote was not yet populated (pre-v14 rows), it is backfilled lazily.
  */
-export function ensureProject(path: string, name?: string): string {
+export function ensureProject(path: string, name?: string, suppliedGitRemote?: string | null): string {
+  // Guard: reject synthetic test paths when targeting the production DB.
+  // Test paths like "/test/ltm/project" are absolute paths that don't exist
+  // on any real filesystem — they're only valid in test suites running against
+  // a temp DB (LORE_DB_PATH set by test preload). If we see such a path
+  // without LORE_DB_PATH being set, a test is likely hitting the production DB.
+  // Note: LORE_DB_PATH unset is used as a proxy for "production DB". This
+  // wouldn't catch the unlikely case of someone explicitly setting LORE_DB_PATH
+  // to the default production path, but that's not a realistic scenario.
+  if (!process.env.LORE_DB_PATH && /^\/test\//.test(path)) {
+    throw new Error(
+      `Refusing to create project with test path "${path}" in the production DB. ` +
+        `Set LORE_DB_PATH to a temp path, or run tests via \`bun test\` from the repo root.`,
+    );
+  }
+
   // 1. Exact path match (fast path)
   const existing = db()
     .query("SELECT id, git_remote FROM projects WHERE path = ?")
@@ -772,21 +836,21 @@ export function ensureProject(path: string, name?: string): string {
   if (existing) {
     // Lazy backfill: populate git_remote on pre-v14 rows
     if (!existing.git_remote) {
-      const gitRemote = getGitRemote(path);
-      if (gitRemote) {
+      const resolvedRemote = suppliedGitRemote ?? getGitRemote(path);
+      if (resolvedRemote) {
         // Check for conflict: another project already has this git_remote.
         // If so, merge the conflicting project into this one (one-time).
         const conflict = db()
           .query(
             "SELECT id FROM projects WHERE git_remote = ? AND id != ? LIMIT 1",
           )
-          .get(gitRemote, existing.id) as { id: string } | null;
+          .get(resolvedRemote, existing.id) as { id: string } | null;
         if (conflict) {
           mergeProjectInternal(conflict.id, existing.id);
         }
         db()
           .query("UPDATE projects SET git_remote = ? WHERE id = ?")
-          .run(gitRemote, existing.id);
+          .run(resolvedRemote, existing.id);
       }
     }
     return existing.id;
@@ -799,7 +863,7 @@ export function ensureProject(path: string, name?: string): string {
   if (alias) return alias.project_id;
 
   // 3. Git remote identification
-  const gitRemote = getGitRemote(path);
+  const gitRemote = suppliedGitRemote ?? getGitRemote(path);
   if (gitRemote) {
     const byRemote = db()
       .query("SELECT id FROM projects WHERE git_remote = ? LIMIT 1")
@@ -844,6 +908,39 @@ export function projectId(path: string): string | undefined {
   return alias?.project_id;
 }
 
+/**
+ * Look up a project by git_remote (preferred) or path. Returns the project ID
+ * or null if not found. Unlike `ensureProject()`, this is read-only — it never
+ * creates a project or registers path aliases.
+ */
+export function resolveProjectByRemoteOrPath(
+  gitRemote?: string,
+  path?: string,
+): string | null {
+  if (gitRemote) {
+    const row = db()
+      .query("SELECT id FROM projects WHERE git_remote = ? LIMIT 1")
+      .get(gitRemote) as { id: string } | null;
+    if (row) return row.id;
+  }
+  if (path) {
+    return projectId(path) ?? null;
+  }
+  return null;
+}
+
+/**
+ * Look up the path for a project by its internal ID.
+ * Used by the REST API to resolve project UUID → path for core functions
+ * that require a path argument.
+ */
+export function projectPath(id: string): string | null {
+  const row = db()
+    .query("SELECT path FROM projects WHERE id = ?")
+    .get(id) as { path: string } | null;
+  return row?.path ?? null;
+}
+
 /** Look up a project's display name by its internal ID. */
 export function projectName(id: string): string | null {
   const row = db()
@@ -1068,6 +1165,289 @@ export function loadAllSessionCosts(): Map<string, SessionCostSnapshot> {
   return result;
 }
 
+// ---------------------------------------------------------------------------
+// Session tracking state (session_state table, v23 columns)
+// ---------------------------------------------------------------------------
+
+/** Fields that can be persisted for session tracking state. */
+export type SessionTrackingState = {
+  lastCuratedAt?: number;
+  messageCount?: number;
+  turnsSinceCuration?: number;
+  consecutiveTextOnlyTurns?: number;
+  ltmCacheText?: string | null;
+  ltmCacheTokens?: number | null;
+  ltmPinText?: string | null;
+  ltmPinTokens?: number | null;
+  // v24: session identity
+  fingerprint?: string;
+  headerSessionId?: string | null;
+  headerName?: string | null;
+  // v24: cache warming
+  resolvedConversationTTL?: string;
+  warmupState?: string | null; // JSON blob
+  // v24: gradient calibration
+  dynamicContextCap?: number;
+  bustRateEMA?: number;
+  interBustIntervalEMA?: number;
+  lastLayer?: number;
+  lastKnownInput?: number;
+  lastTurnAt?: number;
+  lastBustAt?: number;
+};
+
+/**
+ * Persist session tracking state. Ensures the row exists, then updates
+ * only the fields that are explicitly provided (not undefined).
+ */
+export function saveSessionTracking(sessionID: string, state: SessionTrackingState): void {
+  const now = Date.now();
+
+  // Ensure row exists (no-op if it already does)
+  db()
+    .query(
+      "INSERT OR IGNORE INTO session_state (session_id, force_min_layer, updated_at) VALUES (?, 0, ?)",
+    )
+    .run(sessionID, now);
+
+  // Build SET clauses for only the provided fields
+  const sets: string[] = ["updated_at = ?"];
+  const vals: (string | number | null)[] = [now];
+
+  if (state.lastCuratedAt !== undefined) {
+    sets.push("last_curated_at = ?");
+    vals.push(state.lastCuratedAt);
+  }
+  if (state.messageCount !== undefined) {
+    sets.push("message_count = ?");
+    vals.push(state.messageCount);
+  }
+  if (state.turnsSinceCuration !== undefined) {
+    sets.push("turns_since_curation = ?");
+    vals.push(state.turnsSinceCuration);
+  }
+  if (state.consecutiveTextOnlyTurns !== undefined) {
+    sets.push("consecutive_text_only_turns = ?");
+    vals.push(state.consecutiveTextOnlyTurns);
+  }
+  if (state.ltmCacheText !== undefined) {
+    sets.push("ltm_cache_text = ?");
+    vals.push(state.ltmCacheText);
+  }
+  if (state.ltmCacheTokens !== undefined) {
+    sets.push("ltm_cache_tokens = ?");
+    vals.push(state.ltmCacheTokens);
+  }
+  if (state.ltmPinText !== undefined) {
+    sets.push("ltm_pin_text = ?");
+    vals.push(state.ltmPinText);
+  }
+  if (state.ltmPinTokens !== undefined) {
+    sets.push("ltm_pin_tokens = ?");
+    vals.push(state.ltmPinTokens);
+  }
+  // v24: session identity
+  if (state.fingerprint !== undefined) {
+    sets.push("fingerprint = ?");
+    vals.push(state.fingerprint);
+  }
+  if (state.headerSessionId !== undefined) {
+    sets.push("header_session_id = ?");
+    vals.push(state.headerSessionId);
+  }
+  if (state.headerName !== undefined) {
+    sets.push("header_name = ?");
+    vals.push(state.headerName);
+  }
+  // v24: cache warming
+  if (state.resolvedConversationTTL !== undefined) {
+    sets.push("resolved_conversation_ttl = ?");
+    vals.push(state.resolvedConversationTTL);
+  }
+  if (state.warmupState !== undefined) {
+    sets.push("warmup_state = ?");
+    vals.push(state.warmupState);
+  }
+  // v24: gradient calibration
+  if (state.dynamicContextCap !== undefined) {
+    sets.push("dynamic_context_cap = ?");
+    vals.push(state.dynamicContextCap);
+  }
+  if (state.bustRateEMA !== undefined) {
+    sets.push("bust_rate_ema = ?");
+    vals.push(state.bustRateEMA);
+  }
+  if (state.interBustIntervalEMA !== undefined) {
+    sets.push("inter_bust_interval_ema = ?");
+    vals.push(state.interBustIntervalEMA);
+  }
+  if (state.lastLayer !== undefined) {
+    sets.push("last_layer = ?");
+    vals.push(state.lastLayer);
+  }
+  if (state.lastKnownInput !== undefined) {
+    sets.push("last_known_input = ?");
+    vals.push(state.lastKnownInput);
+  }
+  if (state.lastTurnAt !== undefined) {
+    sets.push("last_turn_at = ?");
+    vals.push(state.lastTurnAt);
+  }
+  if (state.lastBustAt !== undefined) {
+    sets.push("last_bust_at = ?");
+    vals.push(state.lastBustAt);
+  }
+
+  // Update only the specified columns
+  db()
+    .query(
+      "UPDATE session_state SET " + sets.join(", ") + " WHERE session_id = ?",
+    )
+    .run(...vals, sessionID);
+}
+
+/** Loaded session tracking state. */
+export type LoadedSessionTracking = {
+  lastCuratedAt: number;
+  messageCount: number;
+  turnsSinceCuration: number;
+  consecutiveTextOnlyTurns: number;
+  ltmCacheText: string | null;
+  ltmCacheTokens: number | null;
+  ltmPinText: string | null;
+  ltmPinTokens: number | null;
+  // v24: session identity
+  fingerprint: string;
+  headerSessionId: string | null;
+  headerName: string | null;
+  // v24: cache warming
+  resolvedConversationTTL: string;
+  warmupState: string | null;
+  // v24: gradient calibration
+  dynamicContextCap: number;
+  bustRateEMA: number;
+  interBustIntervalEMA: number;
+  lastLayer: number;
+  lastKnownInput: number;
+  lastTurnAt: number;
+  lastBustAt: number;
+};
+
+/**
+ * Load persisted session tracking state. Returns null if no row exists.
+ */
+export function loadSessionTracking(sessionID: string): LoadedSessionTracking | null {
+  const row = db()
+    .query(
+      `SELECT last_curated_at, message_count, turns_since_curation,
+              consecutive_text_only_turns,
+              ltm_cache_text, ltm_cache_tokens, ltm_pin_text, ltm_pin_tokens,
+              fingerprint, header_session_id, header_name,
+              resolved_conversation_ttl, warmup_state,
+              dynamic_context_cap, bust_rate_ema, inter_bust_interval_ema,
+              last_layer, last_known_input, last_turn_at, last_bust_at
+       FROM session_state WHERE session_id = ?`,
+    )
+    .get(sessionID) as {
+      last_curated_at: number;
+      message_count: number;
+      turns_since_curation: number;
+      consecutive_text_only_turns: number;
+      ltm_cache_text: string | null;
+      ltm_cache_tokens: number | null;
+      ltm_pin_text: string | null;
+      ltm_pin_tokens: number | null;
+      fingerprint: string;
+      header_session_id: string | null;
+      header_name: string | null;
+      resolved_conversation_ttl: string;
+      warmup_state: string | null;
+      dynamic_context_cap: number;
+      bust_rate_ema: number;
+      inter_bust_interval_ema: number;
+      last_layer: number;
+      last_known_input: number;
+      last_turn_at: number;
+      last_bust_at: number;
+    } | null;
+  if (!row) return null;
+  return {
+    lastCuratedAt: row.last_curated_at,
+    messageCount: row.message_count,
+    turnsSinceCuration: row.turns_since_curation,
+    consecutiveTextOnlyTurns: row.consecutive_text_only_turns,
+    ltmCacheText: row.ltm_cache_text,
+    ltmCacheTokens: row.ltm_cache_tokens,
+    ltmPinText: row.ltm_pin_text,
+    ltmPinTokens: row.ltm_pin_tokens,
+    fingerprint: row.fingerprint,
+    headerSessionId: row.header_session_id,
+    headerName: row.header_name,
+    resolvedConversationTTL: row.resolved_conversation_ttl,
+    warmupState: row.warmup_state,
+    dynamicContextCap: row.dynamic_context_cap,
+    bustRateEMA: row.bust_rate_ema,
+    interBustIntervalEMA: row.inter_bust_interval_ema,
+    lastLayer: row.last_layer,
+    lastKnownInput: row.last_known_input,
+    lastTurnAt: row.last_turn_at,
+    lastBustAt: row.last_bust_at,
+  };
+}
+
+/**
+ * Load all persisted header → session ID mappings from the session_state table.
+ *
+ * Used on gateway startup (in initIfNeeded) to pre-populate the in-memory
+ * headerSessionIndex so Tier 1 session identification works immediately
+ * after a process restart — without this, the first post-restart request
+ * with a known session header would generate a new session ID and orphan
+ * the old session's persisted state.
+ */
+export function loadHeaderSessionIndex(): Array<{
+  sessionId: string;
+  headerSessionId: string;
+  headerName: string;
+}> {
+  const rows = db()
+    .query(
+      `SELECT session_id, header_session_id, header_name
+       FROM session_state
+       WHERE header_session_id IS NOT NULL AND header_name IS NOT NULL`,
+    )
+    .all() as Array<{
+      session_id: string;
+      header_session_id: string;
+      header_name: string;
+    }>;
+  return rows.map((row) => ({
+    sessionId: row.session_id,
+    headerSessionId: row.header_session_id,
+    headerName: row.header_name,
+  }));
+}
+
+// ---------------------------------------------------------------------------
+// Key-value store (kv_meta table)
+// ---------------------------------------------------------------------------
+
+/** Get a kv_meta value by key. Returns null if not found. */
+export function getKV(key: string): string | null {
+  const row = db()
+    .query("SELECT value FROM kv_meta WHERE key = ?")
+    .get(key) as { value: string } | null;
+  return row?.value ?? null;
+}
+
+/** Set a kv_meta value (upsert). */
+export function setKV(key: string, value: string): void {
+  db()
+    .query(
+      "INSERT INTO kv_meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = ?",
+    )
+    .run(key, value, value);
+}
+
 // ---------------------------------------------------------------------------
 // Installation metadata (metadata table)
 // ---------------------------------------------------------------------------

package/src/distillation.ts

@@ -14,6 +14,7 @@ import {
 } from "./prompt";
 import { toolStripAnnotation } from "./gradient";
 import { workerSessionIDs } from "./worker";
+import { distillLimiter } from "./session-limiter";
 import type { LLMClient } from "./types";
 
 // Re-export for backwards compat — index.ts and others may still import from here.
@@ -610,8 +611,23 @@ function resetOrphans(projectPath: string, sessionID: string): number {
   return orphans.length;
 }
 
-// Main distillation entry point — called on session.idle or when urgent
+// Main distillation entry point — called on session.idle or when urgent.
+// Serialized per session via p-limit(1) to prevent concurrent runs from
+// reading the same undistilled messages and producing duplicate rows.
 export async function run(input: {
+  llm: LLMClient;
+  projectPath: string;
+  sessionID: string;
+  model?: { providerID: string; modelID: string };
+  force?: boolean;
+  skipMeta?: boolean;
+  urgent?: boolean;
+  callType?: "batch" | "direct";
+}): Promise<{ rounds: number; distilled: number }> {
+  return distillLimiter.get(input.sessionID)(() => runInner(input));
+}
+
+async function runInner(input: {
   llm: LLMClient;
   projectPath: string;
   sessionID: string;
@@ -697,7 +713,8 @@ export async function run(input: {
       gen0Count(input.projectPath, input.sessionID) >=
       cfg.distillation.metaThreshold
     ) {
-      await metaDistill({
+      // Call inner directly — we're already under the per-session limiter.
+      await metaDistillInner({
         llm: input.llm,
         projectPath: input.projectPath,
         sessionID: input.sessionID,
@@ -776,17 +793,29 @@ async function distillSegment(input: {
     return null;
   }
 
-  const distillId = storeDistillation({
-    projectPath: input.projectPath,
-    sessionID: input.sessionID,
-    observations: result.observations,
-    sourceIDs: input.messages.map((m) => m.id),
-    generation: 0,
-    rCompression: rComp,
-    cNorm,
-    callType: input.callType,
-  });
-  temporal.markDistilled(input.messages.map((m) => m.id));
+  // Atomic: store distillation + mark source messages as distilled in one
+  // transaction. Without this, a crash between the two statements would leave
+  // messages undistilled but with an existing distillation row, causing
+  // re-processing on restart and duplicate distillation content.
+  let distillId: string;
+  db().exec("BEGIN IMMEDIATE");
+  try {
+    distillId = storeDistillation({
+      projectPath: input.projectPath,
+      sessionID: input.sessionID,
+      observations: result.observations,
+      sourceIDs: input.messages.map((m) => m.id),
+      generation: 0,
+      rCompression: rComp,
+      cNorm,
+      callType: input.callType,
+    });
+    temporal.markDistilled(input.messages.map((m) => m.id));
+    db().exec("COMMIT");
+  } catch (e) {
+    db().exec("ROLLBACK");
+    throw e;
+  }
 
   log.info(
     `distill segment: ${input.messages.length} msgs, ` +
@@ -840,7 +869,8 @@ async function distillSegment(input: {
  * via `<previous-meta-summary>` so the LLM updates in place rather than
  * re-deriving from scratch.
  *
- * Exported for tests; `run()` is the production entry point.
+ * Serialized per session via the same p-limit(1) as `run()`. Exported for
+ * the idle handler which calls metaDistill() independently of run().
  */
 export async function metaDistill(input: {
   llm: LLMClient;
@@ -849,6 +879,17 @@ export async function metaDistill(input: {
   model?: { providerID: string; modelID: string };
   urgent?: boolean;
   callType?: "batch" | "direct";
+}): Promise<DistillationResult | null> {
+  return distillLimiter.get(input.sessionID)(() => metaDistillInner(input));
+}
+
+async function metaDistillInner(input: {
+  llm: LLMClient;
+  projectPath: string;
+  sessionID: string;
+  model?: { providerID: string; modelID: string };
+  urgent?: boolean;
+  callType?: "batch" | "direct";
 }): Promise<DistillationResult | null> {
   const existing = loadGen0(input.projectPath, input.sessionID);