@vellumai/assistant 0.8.1 → 0.8.2

package/src/memory/migrations/249-normalize-slack-external-content.ts

@@ -0,0 +1,240 @@
+import {
+  readSlackMetadata,
+  type SlackMessageMetadata,
+} from "../../messaging/providers/slack/message-metadata.js";
+import {
+  parseExternalContentEnvelope,
+  unwrapExternalContentForDisplay,
+} from "../../security/untrusted-content.js";
+import { type DrizzleDb, getSqliteFrom } from "../db-connection.js";
+import { withCrashRecovery } from "./validate-migration-state.js";
+
+const CHECKPOINT_KEY = "migration_normalize_slack_external_content_v1";
+const BATCH_SIZE = 100;
+
+interface CandidateMessageRow {
+  rowid: number;
+  id: string;
+  role: string;
+  content: string;
+  metadata: string;
+}
+
+interface NormalizedMessageRow {
+  content: string;
+  metadata: string;
+}
+
+export function migrateNormalizeSlackExternalContent(
+  database: DrizzleDb,
+): void {
+  withCrashRecovery(database, CHECKPOINT_KEY, () => {
+    const raw = getSqliteFrom(database);
+
+    const tableExists = raw
+      .query(
+        `SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'messages'`,
+      )
+      .get();
+    if (!tableExists) return;
+
+    let lastRowid = 0;
+
+    for (;;) {
+      const rows = raw
+        .query(
+          /*sql*/ `
+            SELECT rowid, id, role, content, metadata
+            FROM messages
+            WHERE rowid > ?
+              AND metadata LIKE '%"slackMeta"%'
+              AND (
+                content LIKE '%<external_content%'
+                OR metadata NOT LIKE '%"provenanceTrustClass"%'
+              )
+            ORDER BY rowid
+            LIMIT ?
+          `,
+        )
+        .all(lastRowid, BATCH_SIZE) as CandidateMessageRow[];
+
+      if (rows.length === 0) break;
+
+      for (const row of rows) {
+        lastRowid = row.rowid;
+        const normalized = normalizeSlackMessageRow(row);
+        if (!normalized) continue;
+
+        raw
+          .query(`UPDATE messages SET content = ?, metadata = ? WHERE id = ?`)
+          .run(normalized.content, normalized.metadata, row.id);
+      }
+    }
+  });
+}
+
+export function downNormalizeSlackExternalContent(_database: DrizzleDb): void {
+  // Irreversible by design: this migration discards redundant persisted
+  // wrappers and leaves runtime assembly responsible for model boundaries.
+}
+
+function normalizeSlackMessageRow(
+  row: CandidateMessageRow,
+): NormalizedMessageRow | null {
+  const parsed = parseSlackMetadataEnvelope(row.metadata);
+  if (!parsed) return null;
+
+  const normalizedContent = normalizeMessageContent(row.content);
+  if (normalizedContent !== null) {
+    const { metadata } = parsed;
+    if (
+      !Object.prototype.hasOwnProperty.call(metadata, "provenanceTrustClass")
+    ) {
+      metadata.provenanceTrustClass = "unknown";
+    }
+
+    return {
+      content: normalizedContent,
+      metadata: JSON.stringify(metadata),
+    };
+  }
+
+  if (isLegacyGuardianBackfillRow(row, parsed)) {
+    const { metadata } = parsed;
+    metadata.provenanceTrustClass = "guardian";
+    if (
+      !Object.prototype.hasOwnProperty.call(metadata, "provenanceSourceChannel")
+    ) {
+      metadata.provenanceSourceChannel = "slack";
+    }
+
+    return {
+      content: row.content,
+      metadata: JSON.stringify(metadata),
+    };
+  }
+
+  return null;
+}
+
+function parseSlackMetadataEnvelope(rawMetadata: string): {
+  metadata: Record<string, unknown>;
+  slackMeta: SlackMessageMetadata;
+} | null {
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(rawMetadata);
+  } catch {
+    return null;
+  }
+
+  if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) {
+    return null;
+  }
+
+  const metadata = parsed as Record<string, unknown>;
+  if (typeof metadata.slackMeta !== "string") return null;
+  const slackMeta = readSlackMetadata(metadata.slackMeta);
+  if (!slackMeta) return null;
+  return { metadata, slackMeta };
+}
+
+function normalizeMessageContent(content: string): string | null {
+  const wholeEnvelope = parseExternalContentEnvelope(content);
+  if (wholeEnvelope) {
+    return wholeEnvelope.content;
+  }
+
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(content);
+  } catch {
+    return null;
+  }
+
+  if (!Array.isArray(parsed)) {
+    return null;
+  }
+
+  let changed = false;
+  const normalizedBlocks = parsed.map((block) => {
+    if (block === null || typeof block !== "object" || Array.isArray(block)) {
+      return block;
+    }
+    const record = block as Record<string, unknown>;
+    if (record.type !== "text" || typeof record.text !== "string") {
+      return block;
+    }
+
+    const unwrapped = unwrapExternalContentForDisplay(record.text);
+    if (unwrapped === record.text) {
+      return block;
+    }
+
+    changed = true;
+    return { ...record, text: unwrapped };
+  });
+
+  return changed ? JSON.stringify(normalizedBlocks) : null;
+}
+
+function isLegacyGuardianBackfillRow(
+  row: CandidateMessageRow,
+  parsed: {
+    metadata: Record<string, unknown>;
+    slackMeta: SlackMessageMetadata;
+  },
+): boolean {
+  if (row.role !== "user") return false;
+  if (parsed.slackMeta.eventKind !== "message") return false;
+  if (
+    Object.prototype.hasOwnProperty.call(
+      parsed.metadata,
+      "provenanceTrustClass",
+    )
+  ) {
+    return false;
+  }
+
+  // Old live Slack turns were written with turn-channel metadata. Old
+  // backfill rows were written directly with only `slackMeta`, and the old
+  // backfill invariant was: non-guardian non-empty text was stored wrapped,
+  // while guardian-authored non-empty text was stored raw. Only stamp the
+  // non-empty raw-text case; attachment-only / empty rows stay conservative.
+  if (
+    Object.prototype.hasOwnProperty.call(parsed.metadata, "userMessageChannel")
+  ) {
+    return false;
+  }
+
+  return hasNonEmptyRawText(row.content);
+}
+
+function hasNonEmptyRawText(content: string): boolean {
+  if (parseExternalContentEnvelope(content)) return false;
+
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(content);
+  } catch {
+    return content.trim().length > 0;
+  }
+
+  if (typeof parsed === "string") {
+    return parsed.trim().length > 0 && !parseExternalContentEnvelope(parsed);
+  }
+
+  if (!Array.isArray(parsed)) return false;
+
+  return parsed.some((block) => {
+    if (block === null || typeof block !== "object" || Array.isArray(block)) {
+      return false;
+    }
+    const text = (block as Record<string, unknown>).text;
+    return (
+      typeof text === "string" &&
+      text.trim().length > 0 &&
+      !parseExternalContentEnvelope(text)
+    );
+  });
+}

package/src/memory/migrations/index.ts

@@ -208,6 +208,12 @@ export { migrateCreateProviderConnections } from "./243-provider-connections.js"
 export { migrateProviderConnectionStatusLabel } from "./244-provider-connection-status-label.js";
 export { migrateMemoryRetrospectiveState } from "./245-memory-retrospective-state.js";
 export { migrateBackfillProviderConnectionLabel } from "./246-backfill-provider-connection-label.js";
+export { migrateExternalConversationBindingThreadId } from "./247-external-conversation-binding-thread-id.js";
+export { createOnboardingEventsTable } from "./248-create-onboarding-events.js";
+export {
+  downNormalizeSlackExternalContent,
+  migrateNormalizeSlackExternalContent,
+} from "./249-normalize-slack-external-content.js";
 export {
   MIGRATION_REGISTRY,
   type MigrationRegistryEntry,

package/src/memory/migrations/registry.ts

@@ -48,6 +48,7 @@ import { downMemoryV2ActivationLogs } from "./234-memory-v2-activation-logs.js";
 import { downSlackCompactionWatermark } from "./235-slack-compaction-watermark.js";
 import { downToolInvocationsMatchedRuleId } from "./236-tool-invocations-matched-rule-id.js";
 import { downHeartbeatRuns } from "./237-heartbeat-runs.js";
+import { downNormalizeSlackExternalContent } from "./249-normalize-slack-external-content.js";
 
 export interface MigrationRegistryEntry {
   /** The checkpoint key written to memory_checkpoints on completion. */
@@ -412,6 +413,13 @@ export const MIGRATION_REGISTRY: MigrationRegistryEntry[] = [
       "Create heartbeat_runs table for tracking heartbeat execution lifecycle with CAS state transitions",
     down: downHeartbeatRuns,
   },
+  {
+    key: "migration_normalize_slack_external_content_v1",
+    version: 48,
+    description:
+      "Normalize legacy persisted Slack external_content wrappers back to raw message content",
+    down: downNormalizeSlackExternalContent,
+  },
 ];
 
 export function getMaxMigrationVersion(): number {

package/src/memory/onboarding-events-store.ts

@@ -0,0 +1,106 @@
+import { and, asc, eq, gt, or } from "drizzle-orm";
+import { v4 as uuid } from "uuid";
+
+import { getConfig } from "../config/loader.js";
+import { getDb } from "./db-connection.js";
+import { onboardingEvents } from "./schema.js";
+
+export interface OnboardingEvent {
+  id: string;
+  createdAt: number;
+  screen: string;
+  toolsJson: string | null;
+  tasksJson: string | null;
+  tone: string | null;
+  googleConnected: boolean | null;
+  googleScopesJson: string | null;
+  abVariant: string | null;
+}
+
+export interface RecordOnboardingEventParams {
+  screen: string;
+  tools?: string[];
+  tasks?: string[];
+  tone?: string;
+  googleConnected?: boolean;
+  googleScopes?: string[];
+  abVariant?: string;
+}
+
+/**
+ * Record an onboarding event (pre-chat selections and Google connect status).
+ * Returns null when usage data collection is disabled.
+ */
+export function recordOnboardingEvent(
+  params: RecordOnboardingEventParams,
+): OnboardingEvent | null {
+  if (!getConfig().collectUsageData) return null;
+  const db = getDb();
+  const event: OnboardingEvent = {
+    id: uuid(),
+    createdAt: Date.now(),
+    screen: params.screen,
+    toolsJson: params.tools ? JSON.stringify(params.tools) : null,
+    tasksJson: params.tasks ? JSON.stringify(params.tasks) : null,
+    tone: params.tone ?? null,
+    googleConnected: params.googleConnected ?? null,
+    googleScopesJson: params.googleScopes
+      ? JSON.stringify(params.googleScopes)
+      : null,
+    abVariant: params.abVariant ?? null,
+  };
+  db.insert(onboardingEvents)
+    .values({
+      id: event.id,
+      createdAt: event.createdAt,
+      screen: event.screen,
+      toolsJson: event.toolsJson,
+      tasksJson: event.tasksJson,
+      tone: event.tone,
+      googleConnected: event.googleConnected,
+      googleScopesJson: event.googleScopesJson,
+      abVariant: event.abVariant,
+    })
+    .run();
+  return event;
+}
+
+/**
+ * Query onboarding events that haven't been reported to telemetry yet.
+ * Uses a compound cursor (createdAt + id) for reliable watermarking.
+ */
+export function queryUnreportedOnboardingEvents(
+  afterCreatedAt: number,
+  afterId: string | undefined,
+  limit: number,
+): OnboardingEvent[] {
+  const db = getDb();
+  const rows = db
+    .select({
+      id: onboardingEvents.id,
+      createdAt: onboardingEvents.createdAt,
+      screen: onboardingEvents.screen,
+      toolsJson: onboardingEvents.toolsJson,
+      tasksJson: onboardingEvents.tasksJson,
+      tone: onboardingEvents.tone,
+      googleConnected: onboardingEvents.googleConnected,
+      googleScopesJson: onboardingEvents.googleScopesJson,
+      abVariant: onboardingEvents.abVariant,
+    })
+    .from(onboardingEvents)
+    .where(
+      afterId
+        ? or(
+            gt(onboardingEvents.createdAt, afterCreatedAt),
+            and(
+              eq(onboardingEvents.createdAt, afterCreatedAt),
+              gt(onboardingEvents.id, afterId),
+            ),
+          )
+        : gt(onboardingEvents.createdAt, afterCreatedAt),
+    )
+    .orderBy(asc(onboardingEvents.createdAt), asc(onboardingEvents.id))
+    .limit(limit)
+    .all();
+  return rows;
+}

package/src/memory/schema/bookmarks.ts

@@ -34,5 +34,3 @@ export const messageBookmarks = sqliteTable(
     index("message_bookmarks_created_at_idx").on(table.createdAt),
   ],
 );
-
-export type MessageBookmarkRow = typeof messageBookmarks.$inferSelect;

package/src/memory/schema/calls.ts

@@ -78,6 +78,7 @@ export const externalConversationBindings = sqliteTable(
       .references(() => conversations.id, { onDelete: "cascade" }),
     sourceChannel: text("source_channel").notNull(),
     externalChatId: text("external_chat_id").notNull(),
+    externalThreadId: text("external_thread_id"),
     externalUserId: text("external_user_id"),
     displayName: text("display_name"),
     username: text("username"),

package/src/memory/schema/inference.ts

@@ -20,9 +20,7 @@ export const providerConnections = sqliteTable(
     createdAt: integer("created_at").notNull(),
     updatedAt: integer("updated_at").notNull(),
   },
-  (table) => [
-    index("idx_provider_connections_provider").on(table.provider),
-  ],
+  (table) => [index("idx_provider_connections_provider").on(table.provider)],
 );
 
 export type ProviderConnectionRow = typeof providerConnections.$inferSelect;

package/src/memory/schema/infrastructure.ts

@@ -232,6 +232,18 @@ export const lifecycleEvents = sqliteTable("lifecycle_events", {
   createdAt: integer("created_at").notNull(),
 });
 
+export const onboardingEvents = sqliteTable("onboarding_events", {
+  id: text("id").primaryKey(),
+  createdAt: integer("created_at").notNull(),
+  screen: text("screen").notNull(),
+  toolsJson: text("tools_json"),
+  tasksJson: text("tasks_json"),
+  tone: text("tone"),
+  googleConnected: integer("google_connected", { mode: "boolean" }),
+  googleScopesJson: text("google_scopes_json"),
+  abVariant: text("ab_variant"),
+});
+
 export const traceEvents = sqliteTable(
   "trace_events",
   {

package/src/memory/turn-events-store.ts

@@ -1,16 +1,102 @@
 import { and, asc, eq, gt, or, sql } from "drizzle-orm";
 
 import { getDb } from "./db-connection.js";
-import { messages } from "./schema.js";
+import { conversations, messages } from "./schema.js";
 
 export interface TurnEvent {
   id: string;
   createdAt: number;
+  /**
+   * Parent conversation id. Lets downstream analytics group turns by
+   * conversation (e.g. avg turns per conversation).
+   */
+  conversationId: string;
+  /**
+   * Conversation type of the parent conversation. Used downstream to
+   * distinguish user-initiated turns (`"standard"`) from system-generated
+   * prompts in `"background"` / `"scheduled"` conversations so analytics
+   * (e.g. DAU) can exclude the latter.
+   */
+  conversationType: string;
+  /**
+   * 1-indexed position of this user turn within the parent conversation,
+   * counting only real user turns (tool-result rows persisted with
+   * role="user" are excluded — same filter as the eligibility predicate
+   * below). The first user turn in a conversation is `1`.
+   *
+   * Computed via correlated subquery on the same filtered set used for
+   * eligibility; this scales with batch size (≤ BATCH_SIZE turns per
+   * flush) and uses the `idx_messages_conversation_id` index for the
+   * partition lookup.
+   */
+  turnIndex: number;
+  /**
+   * Canonical `InterfaceId` enum value identifying the UI surface the user
+   * was interacting from at message-creation time (`"macos"`, `"ios"`,
+   * `"cli"`, `"web"`, `"chrome-extension"`, `"slack"`, `"telegram"`,
+   * `"whatsapp"`, `"email"`, `"phone"`). Sourced from
+   * `messages.metadata.userMessageInterface` (stamped on insert by every
+   * `persistUserMessage` path that flows through `TurnChannelContext`).
+   *
+   * Null when the metadata didn't carry the field — historical rows
+   * predating the threading, or system-initiated turns with no inbound
+   * client context. Downstream analytics should treat null as
+   * `"unknown"`.
+   */
+  interfaceId: string | null;
+  /**
+   * Canonical `ChannelId` enum value identifying the messaging fabric the
+   * user message arrived on (`"vellum"` for in-app messaging from
+   * macos/ios/web/cli; `"slack"`/`"telegram"`/`"whatsapp"`/`"email"`/
+   * `"phone"` for channel-based interfaces). Sourced from
+   * `messages.metadata.userMessageChannel`.
+   *
+   * The 7th `ChannelId` value (`"platform"`) is APNs-push outbound-only
+   * and should never appear on a user-message row.
+   */
+  channelId: string | null;
+  /**
+   * Flexible client metadata stashed under `messages.metadata.client` by
+   * the HTTP header middleware. Carries optional `browserFamily`,
+   * `browserVersion`, `os`, `interfaceVersion` (and is extensible without
+   * a migration since it lives inside the JSON column). Null when no
+   * client headers were attached.
+   *
+   * Returned as raw JSON text — the reporter parses + re-shapes for the
+   * wire format.
+   */
+  clientMetadata: string | null;
+}
+
+/**
+ * SQL fragment that excludes tool-result rows persisted with role="user".
+ * Kept as a single source of truth so the eligibility predicate and the
+ * correlated `turn_index` count stay in lockstep — otherwise the index
+ * can drift from the visible turn stream and break "first turn" /
+ * "turns per conversation" math.
+ *
+ * `<alias>` is interpolated as the SQL identifier for the table whose
+ * `content` column should be filtered (e.g. `messages` for the outer
+ * query, `m2` for the correlated subquery).
+ */
+function realUserTurnContentFilter(alias: string): ReturnType<typeof sql> {
+  return sql.raw(
+    `${alias}.content NOT LIKE '%"type":"tool\\_result"%' ESCAPE '\\' ` +
+      `AND ${alias}.content NOT LIKE '%"type":"web\\_search\\_tool\\_result"%' ESCAPE '\\'`,
+  );
 }
 
 /**
  * Query user messages (turns) that haven't been reported to telemetry yet.
  * Uses a compound cursor (createdAt + id) for reliable watermarking.
+ *
+ * Joins to `conversations` so each turn carries its `conversationType`.
+ * The inner join is safe because `messages.conversationId` has a
+ * not-null FK to `conversations.id` (cascade on delete): every message
+ * row has a matching conversation row.
+ *
+ * `turnIndex` is computed via a correlated subquery counting the real
+ * user turns in the same conversation up to and including this row.
  */
 export function queryUnreportedTurnEvents(
   afterCreatedAt: number,
@@ -19,8 +105,47 @@ export function queryUnreportedTurnEvents(
 ): TurnEvent[] {
   const db = getDb();
   const rows = db
-    .select({ id: messages.id, createdAt: messages.createdAt })
+    .select({
+      id: messages.id,
+      createdAt: messages.createdAt,
+      conversationId: messages.conversationId,
+      conversationType: conversations.conversationType,
+      // 1-indexed turn position within the parent conversation. Counts
+      // only real user turns (same filter applied to the outer query).
+      // `(created_at, id)` lex-comparison matches the watermark cursor
+      // ordering so ties on `created_at` are broken deterministically.
+      turnIndex: sql<number>`(
+        SELECT COUNT(*) FROM messages AS m2
+        WHERE m2.conversation_id = ${messages.conversationId}
+          AND m2.role = 'user'
+          AND ${realUserTurnContentFilter("m2")}
+          AND (m2.created_at < ${messages.createdAt}
+               OR (m2.created_at = ${messages.createdAt}
+                   AND m2.id <= ${messages.id}))
+      )`.as("turn_index"),
+      // Client attribution: extract from `messages.metadata` JSON.
+      // `userMessageInterface` and `userMessageChannel` are stamped on
+      // insert by every `persistUserMessage` path that flows through
+      // `TurnChannelContext`. `client` is the flexible namespace for
+      // browser/os/version metadata attached by HTTP header middleware.
+      // `json_extract` returns SQL NULL when the JSON path is absent —
+      // exactly the null semantics we want on the wire.
+      interfaceId: sql<
+        string | null
+      >`json_extract(${messages.metadata}, '$.userMessageInterface')`.as(
+        "interface_id",
+      ),
+      channelId: sql<
+        string | null
+      >`json_extract(${messages.metadata}, '$.userMessageChannel')`.as(
+        "channel_id",
+      ),
+      clientMetadata: sql<
+        string | null
+      >`json_extract(${messages.metadata}, '$.client')`.as("client_metadata"),
+    })
     .from(messages)
+    .innerJoin(conversations, eq(messages.conversationId, conversations.id))
     .where(
       and(
         eq(messages.role, "user"),

package/src/memory/v2/__tests__/activation.test.ts

@@ -52,9 +52,7 @@ mock.module("../../qdrant-client.js", () => ({
 
 const state = {
   embedCalls: [] as Array<{ inputs: unknown[] }>,
-  sparseCalls: [] as string[],
   embedReturn: [[0.1, 0.2, 0.3]] as number[][],
-  sparseReturn: { indices: [1, 2, 3], values: [0.5, 0.5, 0.5] },
   /**
    * Programmable Qdrant query response queues — one per channel. Each test
    * stages whatever ordered hits it needs and lets `simBatch` /
@@ -87,10 +85,6 @@ mock.module("../../embedding-backend.js", () => ({
       vectors: state.embedReturn,
     };
   },
-  generateSparseEmbedding: (text: string) => {
-    state.sparseCalls.push(text);
-    return state.sparseReturn;
-  },
 }));
 
 class MockQdrantClient {
@@ -176,9 +170,7 @@ const { _resetMemoryV2QdrantForTests } = await import("../qdrant.js");
 
 function resetState(): void {
   state.embedCalls.length = 0;
-  state.sparseCalls.length = 0;
   state.embedReturn = [[0.1, 0.2, 0.3]];
-  state.sparseReturn = { indices: [1, 2, 3], values: [0.5, 0.5, 0.5] };
   state.queryResponses.dense.length = 0;
   state.queryResponses.sparse.length = 0;
   state.queryCalls.length = 0;