@vellumai/assistant 0.8.1 → 0.8.3

package/src/memory/migrations/109-external-conversation-bindings.ts

@@ -1,8 +1,7 @@
 import type { DrizzleDb } from "../db-connection.js";
-import { migrateExtConvBindingsChannelChatUnique } from "./010-ext-conv-bindings-channel-chat-unique.js";
 
 /**
- * External conversation bindings table with indexes and unique constraint migration.
+ * External conversation bindings table with indexes.
  */
 export function createExternalConversationBindingsTables(
   database: DrizzleDb,
@@ -12,6 +11,7 @@ export function createExternalConversationBindingsTables(
       conversation_id TEXT PRIMARY KEY REFERENCES conversations(id) ON DELETE CASCADE,
       source_channel TEXT NOT NULL,
       external_chat_id TEXT NOT NULL,
+      external_thread_id TEXT,
       external_user_id TEXT,
       display_name TEXT,
       username TEXT,
@@ -25,9 +25,20 @@ export function createExternalConversationBindingsTables(
   database.run(
     /*sql*/ `CREATE INDEX IF NOT EXISTS idx_ext_conv_bindings_channel_chat ON external_conversation_bindings(source_channel, external_chat_id)`,
   );
+  database.run(
+    /*sql*/ `CREATE INDEX IF NOT EXISTS idx_ext_conv_bindings_channel_chat_thread ON external_conversation_bindings(source_channel, external_chat_id, external_thread_id)`,
+  );
   database.run(
     /*sql*/ `CREATE INDEX IF NOT EXISTS idx_ext_conv_bindings_channel ON external_conversation_bindings(source_channel)`,
   );
-
-  migrateExtConvBindingsChannelChatUnique(database);
+  database.run(/*sql*/ `
+    CREATE UNIQUE INDEX IF NOT EXISTS idx_ext_conv_bindings_channel_chat_no_thread_unique
+    ON external_conversation_bindings(source_channel, external_chat_id)
+    WHERE external_thread_id IS NULL
+  `);
+  database.run(/*sql*/ `
+    CREATE UNIQUE INDEX IF NOT EXISTS idx_ext_conv_bindings_channel_chat_thread_unique
+    ON external_conversation_bindings(source_channel, external_chat_id, external_thread_id)
+    WHERE external_thread_id IS NOT NULL
+  `);
 }

package/src/memory/migrations/229-delete-private-conversations.test.ts

@@ -1133,7 +1133,7 @@ describe("migrateDeletePrivateConversations", () => {
         1,
         'text',
         'eA==',
-        ${now}
+        ${now - 60_000}
       );
     `);
 
@@ -1153,4 +1153,41 @@ describe("migrateDeletePrivateConversations", () => {
       countWhere(raw, "attachments", `id = 'conv-standard-attachment'`),
     ).toBe(1);
   });
+
+  test("preserves pre-staged uploads (unlinked attachments) created after migration starts", () => {
+    const db = createTestDb();
+    const raw = getSqliteFrom(db);
+    const now = Date.now();
+
+    bootstrapTables(raw);
+    seedConversation(raw, "conv-standard", "standard");
+    // created_at in the future ensures it lands after the migration's start
+    // snapshot regardless of clock resolution / test-runner scheduling.
+    raw.exec(/*sql*/ `
+      INSERT INTO attachments (
+        id,
+        original_filename,
+        mime_type,
+        size_bytes,
+        kind,
+        data_base64,
+        created_at
+      ) VALUES (
+        'pre-staged-upload',
+        'pending.txt',
+        'text/plain',
+        1,
+        'text',
+        'eA==',
+        ${now + 60_000}
+      );
+    `);
+
+    migrateDeletePrivateConversations(db);
+
+    expect(countWhere(raw, "attachments", `id = 'pre-staged-upload'`)).toBe(1);
+    expect(
+      countWhere(raw, "attachments", `id = 'conv-standard-attachment'`),
+    ).toBe(1);
+  });
 });

package/src/memory/migrations/229-delete-private-conversations.ts

@@ -12,6 +12,12 @@ const PRIVATE_GRAPH_NODE_IDS = /*sql*/ `
 `;
 
 export function migrateDeletePrivateConversations(database: DrizzleDb): void {
+  // Snapshot the migration's start time. The trailing orphan-attachment sweep
+  // uses this as an upper bound so it cleans up leaks from prior runs of this
+  // migration (those rows were created before this run started) without
+  // touching pre-staged uploads created during or after the migration.
+  const migrationStartTs = Date.now();
+
   database.run(/*sql*/ `
     DELETE FROM tool_invocations
     WHERE conversation_id IN (${PRIVATE_CONVERSATION_IDS})
@@ -204,6 +210,7 @@ export function migrateDeletePrivateConversations(database: DrizzleDb): void {
       FROM message_attachments ma
       WHERE ma.attachment_id = attachments.id
     )
+      AND created_at <= ${migrationStartTs}
   `);
   database.run(/*sql*/ `
     DELETE FROM memory_summaries

package/src/memory/migrations/247-external-conversation-binding-thread-id.ts

@@ -0,0 +1,78 @@
+import { type DrizzleDb, getSqliteFrom } from "../db-connection.js";
+
+/**
+ * Add a provider thread anchor to external conversation bindings.
+ *
+ * Slack conversations are keyed by `(channel_id, thread_ts)`, while legacy
+ * channels still use only `(source_channel, external_chat_id)`. SQLite treats
+ * NULLs as distinct in unique indexes, so use two partial unique indexes:
+ * one for legacy/no-thread bindings and one for threaded bindings.
+ */
+export function migrateExternalConversationBindingThreadId(
+  database: DrizzleDb,
+): void {
+  const raw = getSqliteFrom(database);
+
+  const tableExists = raw
+    .query(
+      `SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'external_conversation_bindings'`,
+    )
+    .get();
+  if (!tableExists) return;
+
+  const columns = raw
+    .query(`PRAGMA table_info(external_conversation_bindings)`)
+    .all() as Array<{ name: string }>;
+  if (!columns.some((column) => column.name === "external_thread_id")) {
+    raw.exec(
+      `ALTER TABLE external_conversation_bindings ADD COLUMN external_thread_id TEXT`,
+    );
+  }
+
+  try {
+    raw.exec("BEGIN");
+
+    raw.exec(`DROP INDEX IF EXISTS idx_ext_conv_bindings_channel_chat_unique`);
+
+    raw.exec(/*sql*/ `
+      DELETE FROM external_conversation_bindings
+      WHERE rowid NOT IN (
+        SELECT rowid FROM (
+          SELECT rowid,
+                 ROW_NUMBER() OVER (
+                   PARTITION BY source_channel, external_chat_id, COALESCE(external_thread_id, '')
+                   ORDER BY updated_at DESC, created_at DESC, rowid DESC
+                 ) AS rn
+          FROM external_conversation_bindings
+        )
+        WHERE rn = 1
+      )
+    `);
+
+    raw.exec(/*sql*/ `
+      CREATE INDEX IF NOT EXISTS idx_ext_conv_bindings_channel_chat_thread
+      ON external_conversation_bindings(source_channel, external_chat_id, external_thread_id)
+    `);
+
+    raw.exec(/*sql*/ `
+      CREATE UNIQUE INDEX IF NOT EXISTS idx_ext_conv_bindings_channel_chat_no_thread_unique
+      ON external_conversation_bindings(source_channel, external_chat_id)
+      WHERE external_thread_id IS NULL
+    `);
+
+    raw.exec(/*sql*/ `
+      CREATE UNIQUE INDEX IF NOT EXISTS idx_ext_conv_bindings_channel_chat_thread_unique
+      ON external_conversation_bindings(source_channel, external_chat_id, external_thread_id)
+      WHERE external_thread_id IS NOT NULL
+    `);
+
+    raw.exec("COMMIT");
+  } catch (err) {
+    try {
+      raw.exec("ROLLBACK");
+    } catch {
+      /* no active transaction */
+    }
+    throw err;
+  }
+}

package/src/memory/migrations/248-create-onboarding-events.ts

@@ -0,0 +1,21 @@
+import type { DrizzleDb } from "../db-connection.js";
+
+/**
+ * Create the onboarding_events table for tracking pre-chat onboarding
+ * selections (tools, tasks, tone, Google connect status).
+ */
+export function createOnboardingEventsTable(database: DrizzleDb): void {
+  database.run(/*sql*/ `
+    CREATE TABLE IF NOT EXISTS onboarding_events (
+      id TEXT PRIMARY KEY,
+      created_at INTEGER NOT NULL,
+      screen TEXT NOT NULL,
+      tools_json TEXT,
+      tasks_json TEXT,
+      tone TEXT,
+      google_connected INTEGER,
+      google_scopes_json TEXT,
+      ab_variant TEXT
+    )
+  `);
+}

package/src/memory/migrations/249-normalize-slack-external-content.ts

@@ -0,0 +1,240 @@
+import {
+  readSlackMetadata,
+  type SlackMessageMetadata,
+} from "../../messaging/providers/slack/message-metadata.js";
+import {
+  parseExternalContentEnvelope,
+  unwrapExternalContentForDisplay,
+} from "../../security/untrusted-content.js";
+import { type DrizzleDb, getSqliteFrom } from "../db-connection.js";
+import { withCrashRecovery } from "./validate-migration-state.js";
+
+const CHECKPOINT_KEY = "migration_normalize_slack_external_content_v1";
+const BATCH_SIZE = 100;
+
+interface CandidateMessageRow {
+  rowid: number;
+  id: string;
+  role: string;
+  content: string;
+  metadata: string;
+}
+
+interface NormalizedMessageRow {
+  content: string;
+  metadata: string;
+}
+
+export function migrateNormalizeSlackExternalContent(
+  database: DrizzleDb,
+): void {
+  withCrashRecovery(database, CHECKPOINT_KEY, () => {
+    const raw = getSqliteFrom(database);
+
+    const tableExists = raw
+      .query(
+        `SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'messages'`,
+      )
+      .get();
+    if (!tableExists) return;
+
+    let lastRowid = 0;
+
+    for (;;) {
+      const rows = raw
+        .query(
+          /*sql*/ `
+            SELECT rowid, id, role, content, metadata
+            FROM messages
+            WHERE rowid > ?
+              AND metadata LIKE '%"slackMeta"%'
+              AND (
+                content LIKE '%<external_content%'
+                OR metadata NOT LIKE '%"provenanceTrustClass"%'
+              )
+            ORDER BY rowid
+            LIMIT ?
+          `,
+        )
+        .all(lastRowid, BATCH_SIZE) as CandidateMessageRow[];
+
+      if (rows.length === 0) break;
+
+      for (const row of rows) {
+        lastRowid = row.rowid;
+        const normalized = normalizeSlackMessageRow(row);
+        if (!normalized) continue;
+
+        raw
+          .query(`UPDATE messages SET content = ?, metadata = ? WHERE id = ?`)
+          .run(normalized.content, normalized.metadata, row.id);
+      }
+    }
+  });
+}
+
+export function downNormalizeSlackExternalContent(_database: DrizzleDb): void {
+  // Irreversible by design: this migration discards redundant persisted
+  // wrappers and leaves runtime assembly responsible for model boundaries.
+}
+
+function normalizeSlackMessageRow(
+  row: CandidateMessageRow,
+): NormalizedMessageRow | null {
+  const parsed = parseSlackMetadataEnvelope(row.metadata);
+  if (!parsed) return null;
+
+  const normalizedContent = normalizeMessageContent(row.content);
+  if (normalizedContent !== null) {
+    const { metadata } = parsed;
+    if (
+      !Object.prototype.hasOwnProperty.call(metadata, "provenanceTrustClass")
+    ) {
+      metadata.provenanceTrustClass = "unknown";
+    }
+
+    return {
+      content: normalizedContent,
+      metadata: JSON.stringify(metadata),
+    };
+  }
+
+  if (isLegacyGuardianBackfillRow(row, parsed)) {
+    const { metadata } = parsed;
+    metadata.provenanceTrustClass = "guardian";
+    if (
+      !Object.prototype.hasOwnProperty.call(metadata, "provenanceSourceChannel")
+    ) {
+      metadata.provenanceSourceChannel = "slack";
+    }
+
+    return {
+      content: row.content,
+      metadata: JSON.stringify(metadata),
+    };
+  }
+
+  return null;
+}
+
+function parseSlackMetadataEnvelope(rawMetadata: string): {
+  metadata: Record<string, unknown>;
+  slackMeta: SlackMessageMetadata;
+} | null {
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(rawMetadata);
+  } catch {
+    return null;
+  }
+
+  if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) {
+    return null;
+  }
+
+  const metadata = parsed as Record<string, unknown>;
+  if (typeof metadata.slackMeta !== "string") return null;
+  const slackMeta = readSlackMetadata(metadata.slackMeta);
+  if (!slackMeta) return null;
+  return { metadata, slackMeta };
+}
+
+function normalizeMessageContent(content: string): string | null {
+  const wholeEnvelope = parseExternalContentEnvelope(content);
+  if (wholeEnvelope) {
+    return wholeEnvelope.content;
+  }
+
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(content);
+  } catch {
+    return null;
+  }
+
+  if (!Array.isArray(parsed)) {
+    return null;
+  }
+
+  let changed = false;
+  const normalizedBlocks = parsed.map((block) => {
+    if (block === null || typeof block !== "object" || Array.isArray(block)) {
+      return block;
+    }
+    const record = block as Record<string, unknown>;
+    if (record.type !== "text" || typeof record.text !== "string") {
+      return block;
+    }
+
+    const unwrapped = unwrapExternalContentForDisplay(record.text);
+    if (unwrapped === record.text) {
+      return block;
+    }
+
+    changed = true;
+    return { ...record, text: unwrapped };
+  });
+
+  return changed ? JSON.stringify(normalizedBlocks) : null;
+}
+
+function isLegacyGuardianBackfillRow(
+  row: CandidateMessageRow,
+  parsed: {
+    metadata: Record<string, unknown>;
+    slackMeta: SlackMessageMetadata;
+  },
+): boolean {
+  if (row.role !== "user") return false;
+  if (parsed.slackMeta.eventKind !== "message") return false;
+  if (
+    Object.prototype.hasOwnProperty.call(
+      parsed.metadata,
+      "provenanceTrustClass",
+    )
+  ) {
+    return false;
+  }
+
+  // Old live Slack turns were written with turn-channel metadata. Old
+  // backfill rows were written directly with only `slackMeta`, and the old
+  // backfill invariant was: non-guardian non-empty text was stored wrapped,
+  // while guardian-authored non-empty text was stored raw. Only stamp the
+  // non-empty raw-text case; attachment-only / empty rows stay conservative.
+  if (
+    Object.prototype.hasOwnProperty.call(parsed.metadata, "userMessageChannel")
+  ) {
+    return false;
+  }
+
+  return hasNonEmptyRawText(row.content);
+}
+
+function hasNonEmptyRawText(content: string): boolean {
+  if (parseExternalContentEnvelope(content)) return false;
+
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(content);
+  } catch {
+    return content.trim().length > 0;
+  }
+
+  if (typeof parsed === "string") {
+    return parsed.trim().length > 0 && !parseExternalContentEnvelope(parsed);
+  }
+
+  if (!Array.isArray(parsed)) return false;
+
+  return parsed.some((block) => {
+    if (block === null || typeof block !== "object" || Array.isArray(block)) {
+      return false;
+    }
+    const text = (block as Record<string, unknown>).text;
+    return (
+      typeof text === "string" &&
+      text.trim().length > 0 &&
+      !parseExternalContentEnvelope(text)
+    );
+  });
+}

package/src/memory/migrations/250-provider-connection-base-url-and-models.ts

@@ -0,0 +1,28 @@
+import { type DrizzleDb, getSqliteFrom } from "../db-connection.js";
+
+/**
+ * Adds `base_url` (nullable) and `models` (nullable, JSON-encoded array of
+ * model identifiers) columns to the `provider_connections` table.
+ *
+ * Required by openai-compatible connections, which carry a user-supplied
+ * endpoint and model list per row instead of inheriting them from the catalog.
+ * Idempotent — re-running is a no-op once the columns exist.
+ */
+export function migrateProviderConnectionBaseUrlAndModels(
+  database: DrizzleDb,
+): void {
+  const raw = getSqliteFrom(database);
+
+  const columns = raw
+    .query(`PRAGMA table_info(provider_connections)`)
+    .all() as Array<{ name: string }>;
+  const columnNames = new Set(columns.map((c) => c.name));
+
+  if (!columnNames.has("base_url")) {
+    raw.exec(`ALTER TABLE provider_connections ADD COLUMN base_url TEXT`);
+  }
+
+  if (!columnNames.has("models")) {
+    raw.exec(`ALTER TABLE provider_connections ADD COLUMN models TEXT`);
+  }
+}

package/src/memory/migrations/251-a2a-tasks.ts

@@ -0,0 +1,49 @@
+import type { DrizzleDb } from "../db-connection.js";
+import { getSqliteFrom } from "../db-connection.js";
+import { tableHasColumn } from "./schema-introspection.js";
+import { withCrashRecovery } from "./validate-migration-state.js";
+
+const CHECKPOINT_KEY = "migration_a2a_tasks_v1";
+
+/**
+ * Create the a2a_tasks table for tracking A2A request/response lifecycle.
+ *
+ * Each row represents one inbound A2A task, tracking its state machine
+ * progression through submitted -> working -> completed/failed/canceled/rejected.
+ */
+export function migrateA2ATasks(database: DrizzleDb): void {
+  withCrashRecovery(database, CHECKPOINT_KEY, () => {
+    if (tableHasColumn(database, "a2a_tasks", "id")) {
+      return;
+    }
+    const raw = getSqliteFrom(database);
+    raw.exec(/*sql*/ `
+      CREATE TABLE IF NOT EXISTS a2a_tasks (
+        id TEXT PRIMARY KEY,
+        context_id TEXT,
+        conversation_id TEXT,
+        state TEXT NOT NULL DEFAULT 'submitted',
+        status_message TEXT,
+        request_message_json TEXT NOT NULL,
+        artifacts_json TEXT,
+        push_url TEXT,
+        sender_assistant_id TEXT NOT NULL,
+        created_at INTEGER NOT NULL,
+        updated_at INTEGER NOT NULL
+      )
+    `);
+    raw.exec(/*sql*/ `
+      CREATE INDEX IF NOT EXISTS idx_a2a_tasks_context
+        ON a2a_tasks (context_id)
+    `);
+    raw.exec(/*sql*/ `
+      CREATE INDEX IF NOT EXISTS idx_a2a_tasks_conversation
+        ON a2a_tasks (conversation_id)
+    `);
+  });
+}
+
+export function downA2ATasks(database: DrizzleDb): void {
+  const raw = getSqliteFrom(database);
+  raw.exec(/*sql*/ `DROP TABLE IF EXISTS a2a_tasks`);
+}

package/src/memory/migrations/252-llm-request-log-agent-loop-exit-reason.ts

@@ -0,0 +1,32 @@
+import { type DrizzleDb, getSqliteFrom } from "../db-connection.js";
+
+/**
+ * Adds `agent_loop_exit_reason` (nullable TEXT) to the `llm_request_logs`
+ * table.
+ *
+ * The agent loop sets this column on its final log row via
+ * `setAgentLoopExitReasonOnLatestLog` once the `while (true)` body exits.
+ * Intermediate rows keep NULL — downstream tooling (notably the LLM
+ * Context Inspector) reads "row has non-null value" as "this is the final
+ * call of a complete agent-loop run". Encoding the run-end via row state
+ * keeps the schema additive: no new tables, no FK churn.
+ *
+ * Idempotent — re-running is a no-op once the column exists. Modeled on
+ * migration 250 (`provider-connection-base-url-and-models`).
+ */
+export function migrateLlmRequestLogAgentLoopExitReason(
+  database: DrizzleDb,
+): void {
+  const raw = getSqliteFrom(database);
+
+  const columns = raw
+    .query(`PRAGMA table_info(llm_request_logs)`)
+    .all() as Array<{ name: string }>;
+  const columnNames = new Set(columns.map((c) => c.name));
+
+  if (!columnNames.has("agent_loop_exit_reason")) {
+    raw.exec(
+      `ALTER TABLE llm_request_logs ADD COLUMN agent_loop_exit_reason TEXT`,
+    );
+  }
+}

package/src/memory/migrations/index.ts

@@ -208,6 +208,15 @@ export { migrateCreateProviderConnections } from "./243-provider-connections.js"
 export { migrateProviderConnectionStatusLabel } from "./244-provider-connection-status-label.js";
 export { migrateMemoryRetrospectiveState } from "./245-memory-retrospective-state.js";
 export { migrateBackfillProviderConnectionLabel } from "./246-backfill-provider-connection-label.js";
+export { migrateExternalConversationBindingThreadId } from "./247-external-conversation-binding-thread-id.js";
+export { createOnboardingEventsTable } from "./248-create-onboarding-events.js";
+export {
+  downNormalizeSlackExternalContent,
+  migrateNormalizeSlackExternalContent,
+} from "./249-normalize-slack-external-content.js";
+export { migrateProviderConnectionBaseUrlAndModels } from "./250-provider-connection-base-url-and-models.js";
+export { downA2ATasks, migrateA2ATasks } from "./251-a2a-tasks.js";
+export { migrateLlmRequestLogAgentLoopExitReason } from "./252-llm-request-log-agent-loop-exit-reason.js";
 export {
   MIGRATION_REGISTRY,
   type MigrationRegistryEntry,

package/src/memory/migrations/registry.ts

@@ -48,6 +48,8 @@ import { downMemoryV2ActivationLogs } from "./234-memory-v2-activation-logs.js";
 import { downSlackCompactionWatermark } from "./235-slack-compaction-watermark.js";
 import { downToolInvocationsMatchedRuleId } from "./236-tool-invocations-matched-rule-id.js";
 import { downHeartbeatRuns } from "./237-heartbeat-runs.js";
+import { downNormalizeSlackExternalContent } from "./249-normalize-slack-external-content.js";
+import { downA2ATasks } from "./251-a2a-tasks.js";
 
 export interface MigrationRegistryEntry {
   /** The checkpoint key written to memory_checkpoints on completion. */
@@ -412,6 +414,20 @@ export const MIGRATION_REGISTRY: MigrationRegistryEntry[] = [
       "Create heartbeat_runs table for tracking heartbeat execution lifecycle with CAS state transitions",
     down: downHeartbeatRuns,
   },
+  {
+    key: "migration_normalize_slack_external_content_v1",
+    version: 48,
+    description:
+      "Normalize legacy persisted Slack external_content wrappers back to raw message content",
+    down: downNormalizeSlackExternalContent,
+  },
+  {
+    key: "migration_a2a_tasks_v1",
+    version: 49,
+    description:
+      "Create a2a_tasks table for tracking A2A request/response lifecycle",
+    down: downA2ATasks,
+  },
 ];
 
 export function getMaxMigrationVersion(): number {