@vellumai/assistant 0.10.3 → 0.10.4-staging.1

package/src/memory/migrations/302-create-compaction-events.ts

@@ -0,0 +1,107 @@
+import { type DrizzleDb, getSqliteFrom } from "../db-connection.js";
+
+/**
+ * Create the append-only `conversation_compaction_events` ledger and backfill
+ * one event per already-compacted conversation from its current cache columns.
+ *
+ * The `conversations` row keeps the latest compaction
+ * (`context_summary` / `context_compacted_message_count` /
+ * `context_compacted_at`) as the hot-path cache the load path reads; this table
+ * preserves the full history so a fork can inherit the most recent compaction
+ * whose event time predates the message it forks from. Pre-feature history
+ * beyond the latest compaction is unrecoverable, so existing rows seed exactly
+ * one event.
+ *
+ * Idempotent: table creation is guarded on sqlite_master; the backfill is
+ * guarded by a memory_checkpoints key and a per-conversation NOT EXISTS so a
+ * lost checkpoint cannot duplicate rows.
+ */
+export function migrateCreateCompactionEvents(database: DrizzleDb): void {
+  const raw = getSqliteFrom(database);
+
+  const tableExists = raw
+    .query(
+      `SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'conversation_compaction_events'`,
+    )
+    .get();
+
+  if (!tableExists) {
+    try {
+      raw.exec("BEGIN");
+
+      raw.exec(/*sql*/ `
+        CREATE TABLE conversation_compaction_events (
+          id                       TEXT PRIMARY KEY,
+          conversation_id          TEXT NOT NULL REFERENCES conversations(id) ON DELETE CASCADE,
+          compacted_at             INTEGER NOT NULL,
+          summary                  TEXT NOT NULL,
+          compacted_message_count  INTEGER NOT NULL,
+          created_at               INTEGER NOT NULL
+        )
+      `);
+
+      raw.exec(/*sql*/ `
+        CREATE INDEX idx_compaction_events_conv_at
+          ON conversation_compaction_events(conversation_id, compacted_at)
+      `);
+
+      raw.exec("COMMIT");
+    } catch (e) {
+      try {
+        raw.exec("ROLLBACK");
+      } catch {
+        /* no active transaction */
+      }
+      throw e;
+    }
+  }
+
+  const checkpointKey = "backfill_conversation_compaction_events_v1";
+  const checkpoint = raw
+    .query(`SELECT 1 FROM memory_checkpoints WHERE key = ?`)
+    .get(checkpointKey);
+  if (checkpoint) return;
+
+  try {
+    raw.exec("BEGIN");
+
+    raw
+      .query(
+        /*sql*/ `
+        INSERT INTO conversation_compaction_events
+          (id, conversation_id, compacted_at, summary, compacted_message_count, created_at)
+        SELECT
+          lower(hex(randomblob(16))),
+          c.id,
+          c.context_compacted_at,
+          c.context_summary,
+          c.context_compacted_message_count,
+          ?
+        FROM conversations c
+        WHERE c.context_compacted_message_count > 0
+          AND c.context_compacted_at IS NOT NULL
+          AND c.context_summary IS NOT NULL
+          AND NOT EXISTS (
+            SELECT 1 FROM conversation_compaction_events e
+            WHERE e.conversation_id = c.id
+          )
+      `,
+      )
+      .run(Date.now());
+
+    raw
+      .query(
+        `INSERT OR IGNORE INTO memory_checkpoints (key, value, updated_at) VALUES (?, '1', ?)`,
+      )
+      .run(checkpointKey, Date.now());
+
+    raw.exec("COMMIT");
+  } catch (e) {
+    try {
+      raw.exec("ROLLBACK");
+    } catch {
+      /* no active transaction */
+    }
+    throw e;
+  }
+}

package/src/memory/migrations/303-add-conversation-creation-seq.ts

@@ -0,0 +1,33 @@
+import type { DrizzleDb } from "../db-connection.js";
+import { tableHasColumn } from "./schema-introspection.js";
+
+const COLUMN_NAME = "seq";
+const COLUMN_DEFINITION = "seq INTEGER";
+
+/**
+ * Add the `seq` column to the `conversations` table.
+ *
+ * The column holds the highest stream `seq` whose content is durably persisted
+ * to the conversation's message rows: seeded with the global high-water seq
+ * when the row is inserted and advanced on each persistence flush. It is the
+ * durable snapshot↔stream alignment baseline returned by `/messages` — a
+ * client aligns its snapshot with the `/events` stream by applying only events
+ * with a higher `seq`.
+ *
+ * Nullable: NULL means the conversation was created before any stream activity
+ * (global seq was 0) or predates this column, in which case the client
+ * cold-starts.
+ *
+ * No backfill is needed — existing rows default to NULL, which correctly maps
+ * to "no recorded baseline, cold-start".
+ *
+ * Idempotent: guarded with `tableHasColumn` so a crash between the `ALTER
+ * TABLE` and the checkpoint write doesn't cause a duplicate-column error on
+ * the next boot.
+ */
+export function migrateAddConversationCreationSeq(database: DrizzleDb): void {
+  if (tableHasColumn(database, "conversations", COLUMN_NAME)) {
+    return;
+  }
+  database.run(`ALTER TABLE conversations ADD COLUMN ${COLUMN_DEFINITION}`);
+}

package/src/memory/migrations/__tests__/209-strip-thinking-from-consolidated.test.ts

@@ -12,8 +12,11 @@ import { describe, expect, test } from "bun:test";
 
 const { getDb, getSqlite } = await import("../../db-connection.js");
 const { initializeDb } = await import("../../db-init.js");
-const { migrateStripThinkingFromConsolidated } =
-  await import("../209-strip-thinking-from-consolidated.js");
+const {
+  migrateStripThinkingFromConsolidated,
+  ROWID_WINDOW,
+  WINDOW_TIMEOUT_MS,
+} = await import("../209-strip-thinking-from-consolidated.js");
 
 await initializeDb();
 
@@ -40,6 +43,22 @@ function insert(role: string, content: string): { id: string; rowid: number } {
   return { id, rowid };
 }
 
+/** Insert at an explicit rowid so a test can place rows on either side of a
+ *  window boundary without materializing every intervening row. */
+function insertAt(
+  rowid: number,
+  role: string,
+  content: string,
+): { id: string; rowid: number } {
+  const id = `m209-at-${rowid}`;
+  getSqlite()
+    .query(
+      `INSERT INTO messages (rowid, id, conversation_id, role, content, created_at) VALUES (?, ?, ?, ?, ?, ?)`,
+    )
+    .run(rowid, id, CONV, role, content, Date.now());
+  return { id, rowid };
+}
+
 function content(id: string): string {
   return (
     getSqlite().query(`SELECT content FROM messages WHERE id = ?`).get(id) as {
@@ -52,6 +71,13 @@ function blocks(id: string): Array<Record<string, unknown>> {
   return JSON.parse(content(id));
 }
 
+/** The persisted resume watermark, or null once the sweep has cleared it. */
+function watermark(): unknown {
+  return getSqlite()
+    .query(`SELECT value FROM memory_checkpoints WHERE key = ?`)
+    .get("migration_209_strip_thinking_watermark");
+}
+
 describe("migration 209 — strip thinking from consolidated assistant messages", () => {
   test("strips thinking blocks but keeps text and tool_use, preserving order", async () => {
     const { id } = insert(
@@ -216,9 +242,56 @@ describe("migration 209 — strip thinking from consolidated assistant messages"
     expect(blocks(above.id)).toEqual([{ type: "text", text: "above" }]);
 
     // The watermark is cleared once the sweep reaches the end of the table.
-    const wm = getSqlite()
-      .query(`SELECT value FROM memory_checkpoints WHERE key = ?`)
-      .get("migration_209_strip_thinking_watermark");
-    expect(wm).toBeNull();
+    expect(watermark()).toBeNull();
+  });
+
+  test("sweeps a rowid span wider than one window across multiple windows", async () => {
+    // Place rows on either side of a window boundary so the sweep must iterate
+    // the window loop more than once to reach them. The original 100k window
+    // collapsed any table smaller than itself into a single subprocess sweep —
+    // this exercises the bounded-window loop that replaces it.
+    const base =
+      (
+        getSqlite().query(`SELECT MAX(rowid) AS m FROM messages`).get() as {
+          m: number | null;
+        }
+      ).m ?? 0;
+    const first = insertAt(
+      base + ROWID_WINDOW,
+      "assistant",
+      JSON.stringify([
+        { type: "thinking", thinking: "a", signature: "s" },
+        { type: "text", text: "first" },
+      ]),
+    );
+    const second = insertAt(
+      base + 2 * ROWID_WINDOW + 1,
+      "assistant",
+      JSON.stringify([
+        { type: "thinking", thinking: "b", signature: "s" },
+        { type: "text", text: "second" },
+      ]),
+    );
+
+    await migrateStripThinkingFromConsolidated(getDb());
+
+    // Both rows are cleaned even though they sit in different windows, and the
+    // sweep runs to completion (watermark cleared) rather than stalling on the
+    // first window.
+    expect(blocks(first.id)).toEqual([{ type: "text", text: "first" }]);
+    expect(blocks(second.id)).toEqual([{ type: "text", text: "second" }]);
+    expect(watermark()).toBeNull();
+  });
+
+  test("keeps the sweep window and timeout bounded so it cannot overrun a whole table", () => {
+    // Regression guard: ROWID_WINDOW was 100_000, larger than a typical
+    // messages table, so the entire table swept in one subprocess that overran
+    // runAsyncSqlite's 1h cap, never advanced the watermark, and re-ran the
+    // identical doomed window every boot. The window must stay small enough that
+    // a single window finishes well within its timeout, and the per-window cap
+    // must stay below the 1h whole-process default so a stuck window surfaces in
+    // minutes instead of blocking startup for an hour.
+    expect(ROWID_WINDOW).toBeLessThanOrEqual(10_000);
+    expect(WINDOW_TIMEOUT_MS).toBeLessThan(60 * 60 * 1000);
   });
 });

package/src/memory/schema/contacts.ts

@@ -8,10 +8,14 @@ export const contacts = sqliteTable("contacts", {
   notes: text("notes"),
   createdAt: integer("created_at").notNull(),
   updatedAt: integer("updated_at").notNull(),
-  role: text("role").notNull().default("contact"), // 'guardian' | 'contact'
+  role: text("role", { enum: ["guardian", "contact"] })
+    .notNull()
+    .default("contact"),
   principalId: text("principal_id"), // internal auth principal (nullable)
   userFile: text("user_file"), // workspace-relative path to per-user persona file
-  contactType: text("contact_type").notNull().default("human"), // 'human' | 'assistant'
+  contactType: text("contact_type", { enum: ["human", "assistant"] })
+    .notNull()
+    .default("human"),
 });
 
 export const contactChannels = sqliteTable(

package/src/memory/schema/conversations.ts

@@ -61,6 +61,17 @@ export const conversations = sqliteTable(
      * callers read this column directly.
      */
     processingStartedAt: integer("processing_started_at"),
+    /**
+     * Highest stream `seq` whose content is durably persisted to this
+     * conversation's message rows. Seeded with the global high-water seq when
+     * the row is inserted and advanced on each persistence flush
+     * (`recordConversationPersistedSeq`). Returned by `/messages` as the
+     * snapshot↔stream alignment baseline so a client applies only stream
+     * events with a higher `seq`. NULL means the conversation was created
+     * before any stream activity (global seq 0) or predates this column — the
+     * client cold-starts in that case.
+     */
+    seq: integer("seq"),
   },
   (table) => [
     index("idx_conversations_updated_at").on(table.updatedAt),
@@ -170,6 +181,34 @@ export const conversationGraphMemoryState = sqliteTable(
   },
 );
 
+/**
+ * Append-only ledger of every compaction event for a conversation. The
+ * `conversations` row keeps only the latest compaction (`context_summary` /
+ * `context_compacted_message_count` / `context_compacted_at`) as the hot-path
+ * cache the load path reads; this table preserves the full history so a fork
+ * can inherit the most recent compaction whose event time (`compacted_at`)
+ * is at-or-before the boundary message it forks from.
+ */
+export const conversationCompactionEvents = sqliteTable(
+  "conversation_compaction_events",
+  {
+    id: text("id").primaryKey(),
+    conversationId: text("conversation_id")
+      .notNull()
+      .references(() => conversations.id, { onDelete: "cascade" }),
+    compactedAt: integer("compacted_at").notNull(),
+    summary: text("summary").notNull(),
+    compactedMessageCount: integer("compacted_message_count").notNull(),
+    createdAt: integer("created_at").notNull(),
+  },
+  (table) => [
+    index("idx_compaction_events_conv_at").on(
+      table.conversationId,
+      table.compactedAt,
+    ),
+  ],
+);
+
 export const channelInboundEvents = sqliteTable("channel_inbound_events", {
   id: text("id").primaryKey(),
   sourceChannel: text("source_channel").notNull(),