@vellumai/assistant 0.10.0 → 0.10.1-staging.1

package/src/memory/migrations/256-memory-v2-injection-events.ts

@@ -1,9 +1,7 @@
 import { getLogger } from "../../util/logger.js";
 import type { DrizzleDb } from "../db-connection.js";
 import { getSqliteFrom } from "../db-connection.js";
-import { withCrashRecovery } from "./validate-migration-state.js";
 
-const CHECKPOINT_KEY = "migration_memory_v2_injection_events_v1";
 const log = getLogger("memory-v2-injection-events-migration");
 
 /**
@@ -26,85 +24,83 @@ const log = getLogger("memory-v2-injection-events-migration");
  * - `(injected_at)` for time-range pruning later.
  */
 export function migrateMemoryV2InjectionEvents(database: DrizzleDb): void {
-  withCrashRecovery(database, CHECKPOINT_KEY, () => {
-    const raw = getSqliteFrom(database);
-    raw.exec(/*sql*/ `
-      CREATE TABLE IF NOT EXISTS memory_v2_injection_events (
-        id INTEGER PRIMARY KEY,
-        slug TEXT NOT NULL,
-        injected_at INTEGER NOT NULL
-      )
-    `);
-    raw.exec(/*sql*/ `
-      CREATE INDEX IF NOT EXISTS idx_memory_v2_injection_events_slug_time
-        ON memory_v2_injection_events (slug, injected_at)
-    `);
-    raw.exec(/*sql*/ `
-      CREATE INDEX IF NOT EXISTS idx_memory_v2_injection_events_time
-        ON memory_v2_injection_events (injected_at)
-    `);
+  const raw = getSqliteFrom(database);
+  raw.exec(/*sql*/ `
+    CREATE TABLE IF NOT EXISTS memory_v2_injection_events (
+      id INTEGER PRIMARY KEY,
+      slug TEXT NOT NULL,
+      injected_at INTEGER NOT NULL
+    )
+  `);
+  raw.exec(/*sql*/ `
+    CREATE INDEX IF NOT EXISTS idx_memory_v2_injection_events_slug_time
+      ON memory_v2_injection_events (slug, injected_at)
+  `);
+  raw.exec(/*sql*/ `
+    CREATE INDEX IF NOT EXISTS idx_memory_v2_injection_events_time
+      ON memory_v2_injection_events (injected_at)
+  `);
 
-    // Bail before backfill on databases that predate memory_v2_activation_logs
-    // (migration 234) — there's no historical signal to replay.
-    const logsTable = raw
-      .query(
-        `SELECT name FROM sqlite_master WHERE type='table' AND name='memory_v2_activation_logs'`,
-      )
-      .get();
-    if (!logsTable) return;
+  // Bail before backfill on databases that predate memory_v2_activation_logs
+  // (migration 234) — there's no historical signal to replay.
+  const logsTable = raw
+    .query(
+      `SELECT name FROM sqlite_master WHERE type='table' AND name='memory_v2_activation_logs'`,
+    )
+    .get();
+  if (!logsTable) return;
 
-    // Re-run safety net independent of the checkpoint: if events already
-    // exist, do not append duplicates. The checkpoint normally short-circuits
-    // re-runs in withCrashRecovery; this guards the manual-clear path.
-    const existing = raw
-      .query(`SELECT COUNT(*) as n FROM memory_v2_injection_events`)
-      .get() as { n: number };
-    if (existing.n > 0) return;
+  // Re-run safety net independent of the checkpoint: if events already
+  // exist, do not append duplicates. The step runner normally skips
+  // already-applied migrations; this guards the manual-clear path.
+  const existing = raw
+    .query(`SELECT COUNT(*) as n FROM memory_v2_injection_events`)
+    .get() as { n: number };
+  if (existing.n > 0) return;
 
-    const rows = raw
-      .query(
-        `SELECT concepts_json, created_at FROM memory_v2_activation_logs ORDER BY created_at ASC`,
-      )
-      .all() as Array<{ concepts_json: string; created_at: number }>;
-    if (rows.length === 0) return;
+  const rows = raw
+    .query(
+      `SELECT concepts_json, created_at FROM memory_v2_activation_logs ORDER BY created_at ASC`,
+    )
+    .all() as Array<{ concepts_json: string; created_at: number }>;
+  if (rows.length === 0) return;
 
-    const insert = raw.prepare(
-      `INSERT INTO memory_v2_injection_events (slug, injected_at) VALUES (?, ?)`,
-    );
-    const replay = raw.transaction(
-      (events: ReadonlyArray<{ slug: string; t: number }>) => {
-        for (const e of events) insert.run(e.slug, e.t);
-      },
-    );
+  const insert = raw.prepare(
+    `INSERT INTO memory_v2_injection_events (slug, injected_at) VALUES (?, ?)`,
+  );
+  const replay = raw.transaction(
+    (events: ReadonlyArray<{ slug: string; t: number }>) => {
+      for (const e of events) insert.run(e.slug, e.t);
+    },
+  );
 
-    const buffer: Array<{ slug: string; t: number }> = [];
-    let parseFailures = 0;
-    for (const row of rows) {
-      let concepts: Array<{ slug?: unknown; source?: unknown }>;
-      try {
-        concepts = JSON.parse(row.concepts_json) as Array<{
-          slug?: unknown;
-          source?: unknown;
-        }>;
-      } catch {
-        parseFailures += 1;
-        continue;
-      }
-      if (!Array.isArray(concepts)) continue;
-      for (const c of concepts) {
-        if (typeof c.slug !== "string") continue;
-        // Carry-over entries were not router-selected this turn — exclude.
-        if (c.source !== "router") continue;
-        buffer.push({ slug: c.slug, t: row.created_at });
-      }
+  const buffer: Array<{ slug: string; t: number }> = [];
+  let parseFailures = 0;
+  for (const row of rows) {
+    let concepts: Array<{ slug?: unknown; source?: unknown }>;
+    try {
+      concepts = JSON.parse(row.concepts_json) as Array<{
+        slug?: unknown;
+        source?: unknown;
+      }>;
+    } catch {
+      parseFailures += 1;
+      continue;
+    }
+    if (!Array.isArray(concepts)) continue;
+    for (const c of concepts) {
+      if (typeof c.slug !== "string") continue;
+      // Carry-over entries were not router-selected this turn — exclude.
+      if (c.source !== "router") continue;
+      buffer.push({ slug: c.slug, t: row.created_at });
     }
+  }
 
-    if (buffer.length > 0) replay(buffer);
-    log.info(
-      { inserted: buffer.length, rowsScanned: rows.length, parseFailures },
-      `Backfilled ${buffer.length} injection events from ${rows.length} activation log rows`,
-    );
-  });
+  if (buffer.length > 0) replay(buffer);
+  log.info(
+    { inserted: buffer.length, rowsScanned: rows.length, parseFailures },
+    `Backfilled ${buffer.length} injection events from ${rows.length} activation log rows`,
+  );
 }
 
 export function downMemoryV2InjectionEvents(database: DrizzleDb): void {

package/src/memory/migrations/259-conversation-cleaned-at.ts

@@ -1,8 +1,5 @@
 import type { DrizzleDb } from "../db-connection.js";
 import { tableHasColumn } from "./schema-introspection.js";
-import { withCrashRecovery } from "./validate-migration-state.js";
-
-const CHECKPOINT_KEY = "migration_conversation_cleaned_at_v1";
 
 const COLUMN_NAME = "cleaned_at";
 const COLUMN_DEFINITION = "cleaned_at INTEGER";
@@ -17,12 +14,10 @@ const COLUMN_DEFINITION = "cleaned_at INTEGER";
  * to the child only when the fork point is at-or-after the clean.
  */
 export function migrateConversationCleanedAt(database: DrizzleDb): void {
-  withCrashRecovery(database, CHECKPOINT_KEY, () => {
-    if (tableHasColumn(database, "conversations", COLUMN_NAME)) {
-      return;
-    }
-    database.run(`ALTER TABLE conversations ADD COLUMN ${COLUMN_DEFINITION}`);
-  });
+  if (tableHasColumn(database, "conversations", COLUMN_NAME)) {
+    return;
+  }
+  database.run(`ALTER TABLE conversations ADD COLUMN ${COLUMN_DEFINITION}`);
 }
 
 export function downConversationCleanedAt(database: DrizzleDb): void {

package/src/memory/migrations/260-rename-cleaned-at.ts

@@ -1,8 +1,5 @@
 import type { DrizzleDb } from "../db-connection.js";
 import { tableHasColumn } from "./schema-introspection.js";
-import { withCrashRecovery } from "./validate-migration-state.js";
-
-const CHECKPOINT_KEY = "migration_rename_cleaned_at_v1";
 
 const OLD_COLUMN = "cleaned_at";
 const NEW_COLUMN = "history_stripped_at";
@@ -16,19 +13,17 @@ const NEW_COLUMN = "history_stripped_at";
  * message metadata.
  */
 export function migrateRenameCleanedAt(database: DrizzleDb): void {
-  withCrashRecovery(database, CHECKPOINT_KEY, () => {
-    if (tableHasColumn(database, "conversations", NEW_COLUMN)) {
-      return;
-    }
-    if (!tableHasColumn(database, "conversations", OLD_COLUMN)) {
-      // 259 didn't run (fresh install on a newer schema where 259's column
-      // was added under the new name) — nothing to rename.
-      return;
-    }
-    database.run(
-      `ALTER TABLE conversations RENAME COLUMN ${OLD_COLUMN} TO ${NEW_COLUMN}`,
-    );
-  });
+  if (tableHasColumn(database, "conversations", NEW_COLUMN)) {
+    return;
+  }
+  if (!tableHasColumn(database, "conversations", OLD_COLUMN)) {
+    // 259 didn't run (fresh install on a newer schema where 259's column
+    // was added under the new name) — nothing to rename.
+    return;
+  }
+  database.run(
+    `ALTER TABLE conversations RENAME COLUMN ${OLD_COLUMN} TO ${NEW_COLUMN}`,
+  );
 }
 
 export function downRenameCleanedAt(database: DrizzleDb): void {

package/src/memory/migrations/261-llm-usage-add-raw-usage.ts

@@ -1,8 +1,5 @@
 import type { DrizzleDb } from "../db-connection.js";
 import { tableHasColumn } from "./schema-introspection.js";
-import { withCrashRecovery } from "./validate-migration-state.js";
-
-const CHECKPOINT_KEY = "migration_llm_usage_add_raw_usage_v1";
 
 const TABLE = "llm_usage_events";
 const COLUMN = "raw_usage";
@@ -22,11 +19,9 @@ const COLUMN = "raw_usage";
  * payload.
  */
 export function migrateLlmUsageAddRawUsage(database: DrizzleDb): void {
-  withCrashRecovery(database, CHECKPOINT_KEY, () => {
-    if (!tableHasColumn(database, TABLE, COLUMN)) {
-      database.run(`ALTER TABLE ${TABLE} ADD COLUMN ${COLUMN} TEXT`);
-    }
-  });
+  if (!tableHasColumn(database, TABLE, COLUMN)) {
+    database.run(`ALTER TABLE ${TABLE} ADD COLUMN ${COLUMN} TEXT`);
+  }
 }
 
 export function downLlmUsageAddRawUsage(database: DrizzleDb): void {

package/src/memory/migrations/262-memory-v3-coactivation.ts

@@ -1,8 +1,5 @@
 import type { DrizzleDb } from "../db-connection.js";
 import { getSqliteFrom } from "../db-connection.js";
-import { withCrashRecovery } from "./validate-migration-state.js";
-
-const CHECKPOINT_KEY = "migration_memory_v3_coactivation_v1";
 
 /**
  * Create the memory_v3_coactivation table — an append-only log of
@@ -26,29 +23,27 @@ const CHECKPOINT_KEY = "migration_memory_v3_coactivation_v1";
  * - `(created_at)` for time-range pruning later.
  */
 export function migrateMemoryV3Coactivation(database: DrizzleDb): void {
-  withCrashRecovery(database, CHECKPOINT_KEY, () => {
-    const raw = getSqliteFrom(database);
-    raw.exec(/*sql*/ `
-      CREATE TABLE IF NOT EXISTS memory_v3_coactivation (
-        id INTEGER PRIMARY KEY,
-        conversation_id TEXT NOT NULL,
-        turn INTEGER NOT NULL,
-        source_slug TEXT NOT NULL,
-        target_slug TEXT NOT NULL,
-        pass_gap INTEGER NOT NULL,
-        used INTEGER NOT NULL,
-        created_at INTEGER NOT NULL
-      )
-    `);
-    raw.exec(/*sql*/ `
-      CREATE INDEX IF NOT EXISTS idx_memory_v3_coactivation_pair
-        ON memory_v3_coactivation (source_slug, target_slug)
-    `);
-    raw.exec(/*sql*/ `
-      CREATE INDEX IF NOT EXISTS idx_memory_v3_coactivation_time
-        ON memory_v3_coactivation (created_at)
-    `);
-  });
+  const raw = getSqliteFrom(database);
+  raw.exec(/*sql*/ `
+    CREATE TABLE IF NOT EXISTS memory_v3_coactivation (
+      id INTEGER PRIMARY KEY,
+      conversation_id TEXT NOT NULL,
+      turn INTEGER NOT NULL,
+      source_slug TEXT NOT NULL,
+      target_slug TEXT NOT NULL,
+      pass_gap INTEGER NOT NULL,
+      used INTEGER NOT NULL,
+      created_at INTEGER NOT NULL
+    )
+  `);
+  raw.exec(/*sql*/ `
+    CREATE INDEX IF NOT EXISTS idx_memory_v3_coactivation_pair
+      ON memory_v3_coactivation (source_slug, target_slug)
+  `);
+  raw.exec(/*sql*/ `
+    CREATE INDEX IF NOT EXISTS idx_memory_v3_coactivation_time
+      ON memory_v3_coactivation (created_at)
+  `);
 }
 
 export function downMemoryV3Coactivation(database: DrizzleDb): void {

package/src/memory/migrations/263-memory-v3-auto-edges.ts

@@ -1,8 +1,5 @@
 import type { DrizzleDb } from "../db-connection.js";
 import { getSqliteFrom } from "../db-connection.js";
-import { withCrashRecovery } from "./validate-migration-state.js";
-
-const CHECKPOINT_KEY = "migration_memory_v3_auto_edges_v1";
 
 /**
  * Create the memory_v3_auto_edges table — the **learned** edge graph, a
@@ -26,22 +23,20 @@ const CHECKPOINT_KEY = "migration_memory_v3_auto_edges_v1";
  * graph grows.
  */
 export function migrateMemoryV3AutoEdges(database: DrizzleDb): void {
-  withCrashRecovery(database, CHECKPOINT_KEY, () => {
-    const raw = getSqliteFrom(database);
-    raw.exec(/*sql*/ `
-      CREATE TABLE IF NOT EXISTS memory_v3_auto_edges (
-        source_slug TEXT NOT NULL,
-        target_slug TEXT NOT NULL,
-        weight REAL NOT NULL,
-        last_reinforced_at INTEGER NOT NULL,
-        PRIMARY KEY (source_slug, target_slug)
-      )
-    `);
-    raw.exec(/*sql*/ `
-      CREATE INDEX IF NOT EXISTS idx_memory_v3_auto_edges_weight
-        ON memory_v3_auto_edges (weight)
-    `);
-  });
+  const raw = getSqliteFrom(database);
+  raw.exec(/*sql*/ `
+    CREATE TABLE IF NOT EXISTS memory_v3_auto_edges (
+      source_slug TEXT NOT NULL,
+      target_slug TEXT NOT NULL,
+      weight REAL NOT NULL,
+      last_reinforced_at INTEGER NOT NULL,
+      PRIMARY KEY (source_slug, target_slug)
+    )
+  `);
+  raw.exec(/*sql*/ `
+    CREATE INDEX IF NOT EXISTS idx_memory_v3_auto_edges_weight
+      ON memory_v3_auto_edges (weight)
+  `);
 }
 
 export function downMemoryV3AutoEdges(database: DrizzleDb): void {

package/src/memory/migrations/270-schedule-description.ts

@@ -1,7 +1,4 @@
 import { type DrizzleDb, getSqliteFrom } from "../db-connection.js";
-import { withCrashRecovery } from "./validate-migration-state.js";
-
-const CHECKPOINT_KEY = "migration_schedule_description_backfill_v1";
 
 export function migrateScheduleDescription(database: DrizzleDb): void {
   const raw = getSqliteFrom(database);
@@ -20,15 +17,13 @@ export function migrateScheduleDescription(database: DrizzleDb): void {
     // Column already exists.
   }
 
-  withCrashRecovery(database, CHECKPOINT_KEY, () => {
-    raw
-      .query(
-        `UPDATE cron_jobs
-         SET description = name
-         WHERE created_by <> 'defer' AND description = ''`,
-      )
-      .run();
-  });
+  raw
+    .query(
+      `UPDATE cron_jobs
+       SET description = name
+       WHERE created_by <> 'defer' AND description = ''`,
+    )
+    .run();
 }
 
 export function downScheduleDescription(): void {

package/src/memory/migrations/272-acp-session-history-cwd.ts

@@ -1,8 +1,5 @@
 import type { DrizzleDb } from "../db-connection.js";
 import { getSqliteFrom } from "../db-connection.js";
-import { withCrashRecovery } from "./validate-migration-state.js";
-
-const CHECKPOINT_KEY = "migration_acp_session_history_cwd_v1";
 
 const TABLE = "acp_session_history";
 const COLUMN = "cwd";
@@ -21,16 +18,14 @@ const COLUMN = "cwd";
  * exists.
  */
 export function migrateAcpSessionHistoryCwd(database: DrizzleDb): void {
-  withCrashRecovery(database, CHECKPOINT_KEY, () => {
-    const raw = getSqliteFrom(database);
+  const raw = getSqliteFrom(database);
 
-    const columns = raw.query(`PRAGMA table_info(${TABLE})`).all() as Array<{
-      name: string;
-    }>;
-    const columnNames = new Set(columns.map((c) => c.name));
+  const columns = raw.query(`PRAGMA table_info(${TABLE})`).all() as Array<{
+    name: string;
+  }>;
+  const columnNames = new Set(columns.map((c) => c.name));
 
-    if (!columnNames.has(COLUMN)) {
-      raw.exec(`ALTER TABLE ${TABLE} ADD COLUMN ${COLUMN} TEXT`);
-    }
-  });
+  if (!columnNames.has(COLUMN)) {
+    raw.exec(`ALTER TABLE ${TABLE} ADD COLUMN ${COLUMN} TEXT`);
+  }
 }

package/src/memory/migrations/281-memory-retrospective-remembered-log.ts

@@ -1,8 +1,5 @@
 import type { DrizzleDb } from "../db-connection.js";
 import { getSqliteFrom } from "../db-connection.js";
-import { withCrashRecovery } from "./validate-migration-state.js";
-
-const CHECKPOINT_KEY = "migration_memory_retrospective_remembered_log_v1";
 
 const TABLE = "memory_retrospective_state";
 const COLUMN = "remembered_log";
@@ -25,16 +22,14 @@ const COLUMN = "remembered_log";
 export function migrateMemoryRetrospectiveRememberedLog(
   database: DrizzleDb,
 ): void {
-  withCrashRecovery(database, CHECKPOINT_KEY, () => {
-    const raw = getSqliteFrom(database);
+  const raw = getSqliteFrom(database);
 
-    const columns = raw.query(`PRAGMA table_info(${TABLE})`).all() as Array<{
-      name: string;
-    }>;
-    const columnNames = new Set(columns.map((c) => c.name));
+  const columns = raw.query(`PRAGMA table_info(${TABLE})`).all() as Array<{
+    name: string;
+  }>;
+  const columnNames = new Set(columns.map((c) => c.name));
 
-    if (!columnNames.has(COLUMN)) {
-      raw.exec(`ALTER TABLE ${TABLE} ADD COLUMN ${COLUMN} TEXT`);
-    }
-  });
+  if (!columnNames.has(COLUMN)) {
+    raw.exec(`ALTER TABLE ${TABLE} ADD COLUMN ${COLUMN} TEXT`);
+  }
 }

package/src/memory/migrations/297-move-llm-request-logs-to-logs-db.ts

@@ -0,0 +1,111 @@
+import type { Database } from "bun:sqlite";
+
+import { getLogsDbPath } from "../../util/logs-db-path.js";
+import {
+  type DrizzleDb,
+  getLogsSqlite,
+  getSqliteFrom,
+} from "../db-connection.js";
+import {
+  drainStagedTable,
+  type RelocationSpec,
+  stageTableForRelocation,
+} from "./helpers/relocation.js";
+
+/** How to drain `llm_request_logs` from `main` into the logs DB. */
+const RELOCATION: RelocationSpec = {
+  table: "llm_request_logs",
+  targetDbPath: getLogsDbPath,
+  columns: [
+    "id",
+    "conversation_id",
+    "message_id",
+    "provider",
+    "request_payload",
+    "response_payload",
+    "created_at",
+    "agent_loop_exit_reason",
+    "call_site",
+  ],
+};
+
+const CREATE_TABLE = /*sql*/ `
+  CREATE TABLE IF NOT EXISTS llm_request_logs (
+    id TEXT PRIMARY KEY,
+    conversation_id TEXT NOT NULL,
+    message_id TEXT,
+    provider TEXT,
+    request_payload TEXT NOT NULL,
+    response_payload TEXT NOT NULL,
+    created_at INTEGER NOT NULL,
+    agent_loop_exit_reason TEXT,
+    call_site TEXT
+  )
+`;
+
+/** Create the three indexes on the logs connection's `llm_request_logs`. */
+function createIndexes(raw: Database) {
+  raw.exec(
+    `CREATE INDEX IF NOT EXISTS idx_llm_request_logs_message_id ON llm_request_logs(message_id)`,
+  );
+  raw.exec(
+    `CREATE INDEX IF NOT EXISTS idx_llm_request_logs_created_at ON llm_request_logs(created_at)`,
+  );
+  raw.exec(
+    `CREATE INDEX IF NOT EXISTS idx_llm_request_logs_conv_created ON llm_request_logs(conversation_id, created_at)`,
+  );
+}
+
+/**
+ * Create the `llm_request_logs` table and its indexes on the logs connection.
+ * Idempotent (`IF NOT EXISTS`) — the dedicated connection itself performs no DDL
+ * on open, so this migration owns the schema.
+ */
+function ensureLlmRequestLogsSchema(logsRaw: Database): void {
+  logsRaw.exec(CREATE_TABLE);
+  createIndexes(logsRaw);
+}
+
+/**
+ * House `llm_request_logs` in its own append-only database
+ * (`assistant-logs.db`) rather than the main DB. Keeping this heavy table — and
+ * the request/response payloads it stores — in its own file stops it from
+ * bloating the main DB and its WAL, and lets the two files VACUUM and
+ * checkpoint independently. The Drizzle store reads/writes the table over the
+ * dedicated logs connection (see `getLogsDb()`).
+ *
+ * The move is incremental: this step does metadata-only work — create the table
+ * (and indexes) on the logs connection, rename any populated
+ * `main.llm_request_logs` aside to `llm_request_logs__relocating` — then drains
+ * the staged rows into the logs file in awaited batches (see
+ * `helpers/relocation.ts`), keeping the event loop responsive between batches
+ * rather than stalling it with a one-shot `INSERT … SELECT` + `DROP` of a
+ * multi-GB table.
+ *
+ * Idempotent and safe to re-run: `stageTableForRelocation` drops a freshly
+ * recreated empty shadow, renames a populated table only once, and leaves an
+ * in-flight staging table alone; `drainStagedTable` resumes from the remaining
+ * rows.
+ *
+ * Throws (rather than returning) if the logs database cannot be opened, so the
+ * runner records the step as failed instead of applied — leaving it to retry on
+ * a later boot. The throw is caught per-step by the runner, so startup is not
+ * aborted.
+ */
+export async function migrateMoveLlmRequestLogsToLogsDb(
+  database: DrizzleDb,
+): Promise<void> {
+  const logsRaw = getLogsSqlite();
+  if (!logsRaw) {
+    throw new Error(
+      "logs database unavailable — deferring llm_request_logs relocation",
+    );
+  }
+
+  ensureLlmRequestLogsSchema(logsRaw);
+
+  const raw = getSqliteFrom(database);
+  const needsDrain = stageTableForRelocation(raw, RELOCATION.table);
+
+  if (needsDrain) await drainStagedTable(raw, RELOCATION);
+}