@vellumai/assistant 0.10.0 → 0.10.1-dev.202606240317.ea25efe

package/src/memory/migrations/272-acp-session-history-cwd.ts

@@ -1,8 +1,5 @@
 import type { DrizzleDb } from "../db-connection.js";
 import { getSqliteFrom } from "../db-connection.js";
-import { withCrashRecovery } from "./validate-migration-state.js";
-
-const CHECKPOINT_KEY = "migration_acp_session_history_cwd_v1";
 
 const TABLE = "acp_session_history";
 const COLUMN = "cwd";
@@ -21,16 +18,14 @@ const COLUMN = "cwd";
  * exists.
  */
 export function migrateAcpSessionHistoryCwd(database: DrizzleDb): void {
-  withCrashRecovery(database, CHECKPOINT_KEY, () => {
-    const raw = getSqliteFrom(database);
+  const raw = getSqliteFrom(database);
 
-    const columns = raw.query(`PRAGMA table_info(${TABLE})`).all() as Array<{
-      name: string;
-    }>;
-    const columnNames = new Set(columns.map((c) => c.name));
+  const columns = raw.query(`PRAGMA table_info(${TABLE})`).all() as Array<{
+    name: string;
+  }>;
+  const columnNames = new Set(columns.map((c) => c.name));
 
-    if (!columnNames.has(COLUMN)) {
-      raw.exec(`ALTER TABLE ${TABLE} ADD COLUMN ${COLUMN} TEXT`);
-    }
-  });
+  if (!columnNames.has(COLUMN)) {
+    raw.exec(`ALTER TABLE ${TABLE} ADD COLUMN ${COLUMN} TEXT`);
+  }
 }

package/src/memory/migrations/281-memory-retrospective-remembered-log.ts

@@ -1,8 +1,5 @@
 import type { DrizzleDb } from "../db-connection.js";
 import { getSqliteFrom } from "../db-connection.js";
-import { withCrashRecovery } from "./validate-migration-state.js";
-
-const CHECKPOINT_KEY = "migration_memory_retrospective_remembered_log_v1";
 
 const TABLE = "memory_retrospective_state";
 const COLUMN = "remembered_log";
@@ -25,16 +22,14 @@ const COLUMN = "remembered_log";
 export function migrateMemoryRetrospectiveRememberedLog(
   database: DrizzleDb,
 ): void {
-  withCrashRecovery(database, CHECKPOINT_KEY, () => {
-    const raw = getSqliteFrom(database);
+  const raw = getSqliteFrom(database);
 
-    const columns = raw.query(`PRAGMA table_info(${TABLE})`).all() as Array<{
-      name: string;
-    }>;
-    const columnNames = new Set(columns.map((c) => c.name));
+  const columns = raw.query(`PRAGMA table_info(${TABLE})`).all() as Array<{
+    name: string;
+  }>;
+  const columnNames = new Set(columns.map((c) => c.name));
 
-    if (!columnNames.has(COLUMN)) {
-      raw.exec(`ALTER TABLE ${TABLE} ADD COLUMN ${COLUMN} TEXT`);
-    }
-  });
+  if (!columnNames.has(COLUMN)) {
+    raw.exec(`ALTER TABLE ${TABLE} ADD COLUMN ${COLUMN} TEXT`);
+  }
 }

package/src/memory/migrations/297-move-llm-request-logs-to-logs-db.ts

@@ -0,0 +1,111 @@
+import type { Database } from "bun:sqlite";
+
+import { getLogsDbPath } from "../../util/logs-db-path.js";
+import {
+  type DrizzleDb,
+  getLogsSqlite,
+  getSqliteFrom,
+} from "../db-connection.js";
+import {
+  drainStagedTable,
+  type RelocationSpec,
+  stageTableForRelocation,
+} from "./helpers/relocation.js";
+
+/** How to drain `llm_request_logs` from `main` into the logs DB. */
+const RELOCATION: RelocationSpec = {
+  table: "llm_request_logs",
+  targetDbPath: getLogsDbPath,
+  columns: [
+    "id",
+    "conversation_id",
+    "message_id",
+    "provider",
+    "request_payload",
+    "response_payload",
+    "created_at",
+    "agent_loop_exit_reason",
+    "call_site",
+  ],
+};
+
+const CREATE_TABLE = /*sql*/ `
+  CREATE TABLE IF NOT EXISTS llm_request_logs (
+    id TEXT PRIMARY KEY,
+    conversation_id TEXT NOT NULL,
+    message_id TEXT,
+    provider TEXT,
+    request_payload TEXT NOT NULL,
+    response_payload TEXT NOT NULL,
+    created_at INTEGER NOT NULL,
+    agent_loop_exit_reason TEXT,
+    call_site TEXT
+  )
+`;
+
+/** Create the three indexes on the logs connection's `llm_request_logs`. */
+function createIndexes(raw: Database) {
+  raw.exec(
+    `CREATE INDEX IF NOT EXISTS idx_llm_request_logs_message_id ON llm_request_logs(message_id)`,
+  );
+  raw.exec(
+    `CREATE INDEX IF NOT EXISTS idx_llm_request_logs_created_at ON llm_request_logs(created_at)`,
+  );
+  raw.exec(
+    `CREATE INDEX IF NOT EXISTS idx_llm_request_logs_conv_created ON llm_request_logs(conversation_id, created_at)`,
+  );
+}
+
+/**
+ * Create the `llm_request_logs` table and its indexes on the logs connection.
+ * Idempotent (`IF NOT EXISTS`) — the dedicated connection itself performs no DDL
+ * on open, so this migration owns the schema.
+ */
+function ensureLlmRequestLogsSchema(logsRaw: Database): void {
+  logsRaw.exec(CREATE_TABLE);
+  createIndexes(logsRaw);
+}
+
+/**
+ * House `llm_request_logs` in its own append-only database
+ * (`assistant-logs.db`) rather than the main DB. Keeping this heavy table — and
+ * the request/response payloads it stores — in its own file stops it from
+ * bloating the main DB and its WAL, and lets the two files VACUUM and
+ * checkpoint independently. The Drizzle store reads/writes the table over the
+ * dedicated logs connection (see `getLogsDb()`).
+ *
+ * The move is incremental: this step does metadata-only work — create the table
+ * (and indexes) on the logs connection, rename any populated
+ * `main.llm_request_logs` aside to `llm_request_logs__relocating` — then drains
+ * the staged rows into the logs file in awaited batches (see
+ * `helpers/relocation.ts`), keeping the event loop responsive between batches
+ * rather than stalling it with a one-shot `INSERT … SELECT` + `DROP` of a
+ * multi-GB table.
+ *
+ * Idempotent and safe to re-run: `stageTableForRelocation` drops a freshly
+ * recreated empty shadow, renames a populated table only once, and leaves an
+ * in-flight staging table alone; `drainStagedTable` resumes from the remaining
+ * rows.
+ *
+ * Throws (rather than returning) if the logs database cannot be opened, so the
+ * runner records the step as failed instead of applied — leaving it to retry on
+ * a later boot. The throw is caught per-step by the runner, so startup is not
+ * aborted.
+ */
+export async function migrateMoveLlmRequestLogsToLogsDb(
+  database: DrizzleDb,
+): Promise<void> {
+  const logsRaw = getLogsSqlite();
+  if (!logsRaw) {
+    throw new Error(
+      "logs database unavailable — deferring llm_request_logs relocation",
+    );
+  }
+
+  ensureLlmRequestLogsSchema(logsRaw);
+
+  const raw = getSqliteFrom(database);
+  const needsDrain = stageTableForRelocation(raw, RELOCATION.table);
+
+  if (needsDrain) await drainStagedTable(raw, RELOCATION);
+}

package/src/memory/migrations/298-move-memory-jobs-to-memory-db.ts

@@ -0,0 +1,128 @@
+import type { Database } from "bun:sqlite";
+
+import { getMemoryDbPath } from "../../util/memory-db-path.js";
+import {
+  type DrizzleDb,
+  getMemorySqlite,
+  getSqliteFrom,
+} from "../db-connection.js";
+import {
+  drainStagedTable,
+  type RelocationSpec,
+  stageTableForRelocation,
+} from "./helpers/relocation.js";
+
+/**
+ * How to drain `memory_jobs` from `main` into the memory DB. Only
+ * `pending`/`running` jobs are worth keeping; the terminal (`completed`/`failed`)
+ * rows that make up the bulk of a runaway queue are purged without copying.
+ *
+ * A `running` job is copied as `pending`: the worker's startup
+ * `resetRunningJobsToPending()` runs against the (empty) new table before this
+ * drain copies rows over, so a row left `running` would never be re-claimed.
+ * Resetting it here makes the relocated job re-claimable in its new home.
+ */
+export const MEMORY_JOBS_RELOCATION: RelocationSpec = {
+  table: "memory_jobs",
+  targetDbPath: getMemoryDbPath,
+  columns: [
+    "id",
+    "type",
+    "payload",
+    "status",
+    "attempts",
+    "deferrals",
+    "run_after",
+    "last_error",
+    "started_at",
+    "created_at",
+    "updated_at",
+  ],
+  copyWhere: "status IN ('pending','running')",
+  columnExpr: {
+    status: "CASE WHEN status = 'running' THEN 'pending' ELSE status END",
+  },
+};
+
+const CREATE_TABLE = /*sql*/ `
+  CREATE TABLE IF NOT EXISTS memory_jobs (
+    id TEXT PRIMARY KEY,
+    type TEXT NOT NULL,
+    payload TEXT NOT NULL,
+    status TEXT NOT NULL,
+    attempts INTEGER NOT NULL DEFAULT 0,
+    deferrals INTEGER NOT NULL DEFAULT 0,
+    run_after INTEGER NOT NULL,
+    last_error TEXT,
+    started_at INTEGER,
+    created_at INTEGER NOT NULL,
+    updated_at INTEGER NOT NULL
+  )
+`;
+
+/**
+ * Create the `memory_jobs` table and its indexes on the memory connection.
+ * Idempotent (`IF NOT EXISTS`) — the dedicated connection itself performs no DDL
+ * on open, so this migration owns the schema.
+ */
+function ensureMemoryJobsSchema(memoryRaw: Database): void {
+  memoryRaw.exec(CREATE_TABLE);
+  createIndexes(memoryRaw);
+}
+
+/** Create the two `memory_jobs` indexes on the memory connection. */
+function createIndexes(raw: Database) {
+  raw.exec(
+    `CREATE INDEX IF NOT EXISTS idx_memory_jobs_status_run_after ON memory_jobs(status, run_after)`,
+  );
+  raw.exec(/*sql*/ `
+    CREATE INDEX IF NOT EXISTS idx_memory_jobs_conflict_resolve_dedupe
+    ON memory_jobs(
+      type,
+      status,
+      json_extract(payload, '$.messageId'),
+      COALESCE(json_extract(payload, '$.scopeId'), 'default')
+    )
+  `);
+}
+
+/**
+ * Move the `memory_jobs` work queue into its own database
+ * (`assistant-memory.db`). The queue is the heaviest, highest-churn table —
+ * left in the main DB a runaway backlog bloats it (and its WAL) to many GB.
+ * Splitting it out lets the queue grow, VACUUM, and checkpoint independently.
+ * The jobs store reads/writes the queue over the dedicated memory connection
+ * (see `getMemoryDb()`).
+ *
+ * Like the `llm_request_logs` move (migration 297) this is incremental: create
+ * the table (and indexes) on the memory connection, rename any populated
+ * `main.memory_jobs` aside to `memory_jobs__relocating`, then drain it in
+ * awaited batches (see `helpers/relocation.ts`) per {@link MEMORY_JOBS_RELOCATION}.
+ *
+ * Because the drain is awaited as part of this step, the pending/running jobs
+ * are in their new home before the memory jobs worker starts later in daemon
+ * startup — the worker never observes a transiently empty queue.
+ *
+ * Throws (rather than returning) if the memory database cannot be opened, so
+ * the runner records the step as failed instead of applied and retries it on a
+ * later boot — never renaming the source aside without a target to write to,
+ * and never marking the relocation done while it has not happened. The throw is
+ * caught per-step by the runner, so startup is not aborted.
+ */
+export async function migrateMoveMemoryJobsToMemoryDb(
+  database: DrizzleDb,
+): Promise<void> {
+  const memoryRaw = getMemorySqlite();
+  if (!memoryRaw) {
+    throw new Error(
+      "memory database unavailable — deferring memory_jobs relocation",
+    );
+  }
+
+  ensureMemoryJobsSchema(memoryRaw);
+
+  const raw = getSqliteFrom(database);
+  const needsDrain = stageTableForRelocation(raw, MEMORY_JOBS_RELOCATION.table);
+
+  if (needsDrain) await drainStagedTable(raw, MEMORY_JOBS_RELOCATION);
+}

package/src/memory/migrations/299-canonical-guardian-deliveries-conversation-index.ts

@@ -0,0 +1,19 @@
+import type { DrizzleDb } from "../db-connection.js";
+
+/**
+ * Add index on canonical_guardian_deliveries(destination_conversation_id).
+ *
+ * Guardian cards record their associated internal conversation on every
+ * delivery — channel cards as well as in-app — and the conversation-scope
+ * readers (candidate enrichment, decision gate, reply-routing seed) look up
+ * pending requests by destination_conversation_id. Candidate enrichment runs on
+ * the per-notification routing path, so without this index those lookups
+ * degrade to full table scans as delivery history grows.
+ */
+export function migrateCanonicalGuardianDeliveriesConversationIndex(
+  database: DrizzleDb,
+): void {
+  database.run(
+    /*sql*/ `CREATE INDEX IF NOT EXISTS idx_canonical_guardian_deliveries_dest_conversation ON canonical_guardian_deliveries(destination_conversation_id)`,
+  );
+}

package/src/memory/migrations/__tests__/014-backfill-inbox-thread-state.test.ts

@@ -0,0 +1,108 @@
+import { Database } from "bun:sqlite";
+import { describe, expect, test } from "bun:test";
+
+import { drizzle } from "drizzle-orm/bun-sqlite";
+
+import { getSqliteFrom } from "../../db-connection.js";
+import * as schema from "../../schema.js";
+import { migrateBackfillInboxThreadStateFromBindings } from "../014-backfill-inbox-thread-state.js";
+
+function createTestDb() {
+  const sqlite = new Database(":memory:");
+  sqlite.exec("PRAGMA journal_mode=WAL");
+  return drizzle(sqlite, { schema });
+}
+
+function bootstrap(raw: Database, opts: { withAssistantId: boolean }): void {
+  raw.exec(/*sql*/ `
+    CREATE TABLE memory_checkpoints (
+      key TEXT PRIMARY KEY,
+      value TEXT NOT NULL,
+      updated_at INTEGER NOT NULL
+    )
+  `);
+  raw.exec(/*sql*/ `
+    CREATE TABLE external_conversation_bindings (
+      conversation_id TEXT PRIMARY KEY,
+      source_channel TEXT NOT NULL,
+      external_chat_id TEXT NOT NULL,
+      external_user_id TEXT,
+      display_name TEXT,
+      username TEXT,
+      last_inbound_at INTEGER,
+      last_outbound_at INTEGER,
+      created_at INTEGER NOT NULL,
+      updated_at INTEGER NOT NULL
+    )
+  `);
+  raw.exec(/*sql*/ `
+    CREATE TABLE assistant_inbox_thread_state (
+      conversation_id TEXT PRIMARY KEY,
+      ${opts.withAssistantId ? "assistant_id TEXT NOT NULL DEFAULT 'self'," : ""}
+      source_channel TEXT NOT NULL,
+      external_chat_id TEXT NOT NULL,
+      external_user_id TEXT,
+      display_name TEXT,
+      username TEXT,
+      last_inbound_at INTEGER,
+      last_outbound_at INTEGER,
+      last_message_at INTEGER,
+      unread_count INTEGER NOT NULL DEFAULT 0,
+      pending_escalation_count INTEGER NOT NULL DEFAULT 0,
+      has_pending_escalation INTEGER NOT NULL DEFAULT 0,
+      created_at INTEGER NOT NULL,
+      updated_at INTEGER NOT NULL
+    )
+  `);
+  raw.run(
+    `INSERT INTO external_conversation_bindings
+       (conversation_id, source_channel, external_chat_id, external_user_id,
+        display_name, username, last_inbound_at, last_outbound_at,
+        created_at, updated_at)
+     VALUES ('c1','telegram','chat-1','U-1','Name One','one',1000,2000,500,2500)`,
+  );
+}
+
+describe("migration 014 — backfill inbox thread state", () => {
+  test("seeds rows when assistant_id column is present", () => {
+    const db = createTestDb();
+    const raw = getSqliteFrom(db);
+    bootstrap(raw, { withAssistantId: true });
+
+    migrateBackfillInboxThreadStateFromBindings(db);
+
+    const row = raw
+      .query(
+        `SELECT assistant_id, last_message_at FROM assistant_inbox_thread_state WHERE conversation_id = 'c1'`,
+      )
+      .get() as { assistant_id: string; last_message_at: number } | undefined;
+    expect(row?.assistant_id).toBe("self");
+    expect(row?.last_message_at).toBe(2000);
+  });
+
+  test("seeds rows without throwing when assistant_id column was dropped", () => {
+    const db = createTestDb();
+    const raw = getSqliteFrom(db);
+    // Simulate a DB where drop-assistant-id-columns already removed the column
+    // but the backfill checkpoint was never written.
+    bootstrap(raw, { withAssistantId: false });
+
+    expect(() => migrateBackfillInboxThreadStateFromBindings(db)).not.toThrow();
+
+    const count = raw
+      .query(`SELECT COUNT(*) AS n FROM assistant_inbox_thread_state`)
+      .get() as { n: number };
+    expect(count.n).toBe(1);
+  });
+
+  test("is a no-op once the checkpoint is recorded", () => {
+    const db = createTestDb();
+    const raw = getSqliteFrom(db);
+    bootstrap(raw, { withAssistantId: false });
+
+    migrateBackfillInboxThreadStateFromBindings(db);
+    // Removing the source rows would surface any second-run insert attempt.
+    raw.exec(`DELETE FROM external_conversation_bindings`);
+    expect(() => migrateBackfillInboxThreadStateFromBindings(db)).not.toThrow();
+  });
+});

package/src/memory/migrations/__tests__/136-drop-assistant-id-columns.test.ts

@@ -0,0 +1,82 @@
+import { Database } from "bun:sqlite";
+import { describe, expect, test } from "bun:test";
+
+import { drizzle } from "drizzle-orm/bun-sqlite";
+
+import { getSqliteFrom } from "../../db-connection.js";
+import * as schema from "../../schema.js";
+import { migrateDropAssistantIdColumns } from "../136-drop-assistant-id-columns.js";
+
+function createTestDb() {
+  const sqlite = new Database(":memory:");
+  sqlite.exec("PRAGMA journal_mode=WAL");
+  return drizzle(sqlite, { schema });
+}
+
+function indexExists(raw: Database, name: string): boolean {
+  return !!raw
+    .prepare(`SELECT 1 FROM sqlite_master WHERE type = 'index' AND name = ?`)
+    .get(name);
+}
+
+/**
+ * Create a minimal assistant_inbox_thread_state carrying assistant_id plus its
+ * composite index, so the column-drop + index-recreate path has something to
+ * act on.
+ */
+function createInboxThreadState(raw: Database): void {
+  raw.exec(/*sql*/ `
+    CREATE TABLE assistant_inbox_thread_state (
+      conversation_id TEXT PRIMARY KEY,
+      assistant_id TEXT NOT NULL DEFAULT 'self',
+      source_channel TEXT NOT NULL,
+      external_chat_id TEXT NOT NULL,
+      last_message_at INTEGER,
+      has_pending_escalation INTEGER NOT NULL DEFAULT 0
+    )
+  `);
+  raw.exec(/*sql*/ `
+    CREATE UNIQUE INDEX idx_inbox_thread_state_channel
+      ON assistant_inbox_thread_state(assistant_id, source_channel, external_chat_id)
+  `);
+}
+
+describe("migration 136 — drop assistant_id columns", () => {
+  test("drops the column and recreates the index without assistant_id", () => {
+    const db = createTestDb();
+    const raw = getSqliteFrom(db);
+    createInboxThreadState(raw);
+
+    migrateDropAssistantIdColumns(db);
+
+    const cols = (
+      raw.query(`PRAGMA table_info(assistant_inbox_thread_state)`).all() as {
+        name: string;
+      }[]
+    ).map((c) => c.name);
+    expect(cols).not.toContain("assistant_id");
+    expect(indexExists(raw, "idx_inbox_thread_state_channel")).toBe(true);
+  });
+
+  test("does not throw when some scoped tables were already dropped", () => {
+    const db = createTestDb();
+    const raw = getSqliteFrom(db);
+    // Only the inbox table exists — actor_token_records and friends were
+    // removed by an earlier migration. Index recreation must skip the missing
+    // tables instead of failing with "no such table".
+    createInboxThreadState(raw);
+
+    expect(() => migrateDropAssistantIdColumns(db)).not.toThrow();
+    // Sanity: the index targeting the missing table was not created.
+    expect(indexExists(raw, "idx_actor_tokens_active_device")).toBe(false);
+  });
+
+  test("is idempotent across repeated runs", () => {
+    const db = createTestDb();
+    const raw = getSqliteFrom(db);
+    createInboxThreadState(raw);
+
+    migrateDropAssistantIdColumns(db);
+    expect(() => migrateDropAssistantIdColumns(db)).not.toThrow();
+  });
+});