@vellumai/assistant 0.8.1 → 0.8.3

package/src/memory/conversation-crud.ts

@@ -14,6 +14,7 @@ import {
   like,
   lt,
   lte,
+  or,
   sql,
 } from "drizzle-orm";
 import { v4 as uuid } from "uuid";
@@ -86,6 +87,7 @@ const subagentNotificationSchema = z.object({
   status: z.enum(["running", "completed", "failed", "aborted"]),
   error: z.string().optional(),
   conversationId: z.string().optional(),
+  objective: z.string().optional(),
 });
 
 export const messageMetadataSchema = z
@@ -94,6 +96,16 @@ export const messageMetadataSchema = z
     assistantMessageChannel: channelIdSchema.optional(),
     userMessageInterface: interfaceIdSchema.optional(),
     assistantMessageInterface: interfaceIdSchema.optional(),
+    /**
+     * Optional client-side metadata bag attached to user messages by HTTP
+     * header middleware (reads `x-vellum-browser-family`,
+     * `x-vellum-browser-version`, `x-vellum-client-os`,
+     * `x-vellum-interface-version`). Forwarded verbatim onto
+     * `TurnTelemetryEvent.client` for downstream analytics. Kept as a
+     * permissive `record` so adding a new client field doesn't require a
+     * migration -- dbt can unpack later via JSON_VALUE.
+     */
+    client: z.record(z.string(), z.unknown()).optional(),
     subagentNotification: subagentNotificationSchema.optional(),
     /**
      * Trust class of the actor at the time this message was persisted.
@@ -727,9 +739,14 @@ export function forkConversation(params: {
 
     // Carry the parent's per-conversation memory state into the child so the
     // forked thread resumes with the same activation/injection log and
-    // in-context tracker the parent had at fork time.
-    forkActivationState(db, sourceConversation.id, fc.id);
-    forkGraphMemoryState(sourceConversation.id, fc.id);
+    // in-context tracker the parent had at fork time. Only valid for
+    // full-history forks: a truncated fork would inherit activation/tracker
+    // entries for turns the child does not actually contain.
+    const isFullHistoryFork = copyBoundaryIndex === sourceMessages.length - 1;
+    if (isFullHistoryFork) {
+      forkActivationState(db, sourceConversation.id, fc.id);
+      forkGraphMemoryState(sourceConversation.id, fc.id);
+    }
     forkRetrospectiveState({
       database: db,
       sourceConversationId: sourceConversation.id,
@@ -1040,22 +1057,23 @@ export function getMessages(conversationId: string): MessageRow[] {
 }
 
 /**
- * Return raw `metadata` strings for messages whose metadata contains the
- * literal substring `"slackMeta"`, capped at `limit` and skipping the first
- * `offset` matches. Pushes `LIKE` + `LIMIT`/`OFFSET` into SQL so warm Slack
- * DM conversations don't require a full-table scan on the webhook critical
- * path. The substring match is an indexable prefilter only — callers must
- * parse and validate each returned string against the Slack metadata schema,
- * because a malformed row (partial write, legacy format, unrelated key
- * accidentally containing the literal) can still slip through the substring
- * match. Callers that need a fixed number of *valid* rows should iterate
- * with increasing offsets until the target is reached (capped at a
- * reasonable maximum to bound scan cost).
+ * Return raw `metadata` strings for messages whose metadata looks like it may
+ * contain Slack metadata, capped at `limit` and skipping the first `offset`
+ * matches. Pushes `LIKE` + `LIMIT`/`OFFSET` into SQL so warm Slack DM
+ * conversations don't require a full-table scan on the webhook critical path.
+ * The substring match is an indexable prefilter only — callers must parse and
+ * validate each returned string against the Slack metadata schema, because a
+ * malformed row (partial write, legacy format, unrelated key accidentally
+ * containing the literal) can still slip through the substring match. Callers
+ * that need a fixed number of *valid* rows should iterate with increasing
+ * offsets until the target is reached (capped at a reasonable maximum to bound
+ * scan cost).
  */
 export function selectSlackMetaCandidateMetadata(
   conversationId: string,
   limit: number,
   offset = 0,
+  opts?: { includeFlatLegacy?: boolean },
 ): string[] {
   const db = getDb();
   const rows = db
@@ -1064,7 +1082,12 @@ export function selectSlackMetaCandidateMetadata(
     .where(
       and(
         eq(messages.conversationId, conversationId),
-        like(messages.metadata, '%"slackMeta"%'),
+        opts?.includeFlatLegacy
+          ? or(
+              like(messages.metadata, '%"slackMeta"%'),
+              like(messages.metadata, '%"source":"slack"%'),
+            )
+          : like(messages.metadata, '%"slackMeta"%'),
       ),
     )
     .orderBy(asc(messages.createdAt))
@@ -1110,13 +1133,23 @@ export function countMessagesAfter(
     .where(eq(messages.id, afterMessageId))
     .get();
   if (!ref) return 0;
+  // Tie-breaker on `messages.id` so rows that share a millisecond timestamp
+  // with the reference are not permanently skipped. Mirrors the
+  // `(createdAt, id)` cursor pattern used by the backfill job-handler and
+  // turn-events-store.
   const row = db
     .select({ c: count() })
     .from(messages)
     .where(
       and(
         eq(messages.conversationId, conversationId),
-        gt(messages.createdAt, ref.createdAt),
+        or(
+          gt(messages.createdAt, ref.createdAt),
+          and(
+            eq(messages.createdAt, ref.createdAt),
+            gt(messages.id, afterMessageId),
+          ),
+        ),
       ),
     )
     .get();
@@ -1136,11 +1169,14 @@ export function getMessagesAfter(
 ): MessageRow[] {
   const db = getDb();
   if (afterMessageId === null || afterMessageId === "") {
+    // Secondary `asc(messages.id)` matches the non-null path's cursor
+    // ordering, so callers tracking `cutoffMessageId` across runs see a
+    // consistent ordering when multiple rows share a millisecond timestamp.
     return db
       .select()
       .from(messages)
       .where(eq(messages.conversationId, conversationId))
-      .orderBy(asc(messages.createdAt))
+      .orderBy(asc(messages.createdAt), asc(messages.id))
       .all()
       .map(parseMessage);
   }
@@ -1150,16 +1186,24 @@ export function getMessagesAfter(
     .where(eq(messages.id, afterMessageId))
     .get();
   if (!ref) return [];
+  // Same `(createdAt, id)` cursor as `countMessagesAfter` — rows sharing
+  // the reference's millisecond timestamp would otherwise be skipped.
   return db
     .select()
     .from(messages)
     .where(
       and(
         eq(messages.conversationId, conversationId),
-        gt(messages.createdAt, ref.createdAt),
+        or(
+          gt(messages.createdAt, ref.createdAt),
+          and(
+            eq(messages.createdAt, ref.createdAt),
+            gt(messages.id, afterMessageId),
+          ),
+        ),
       ),
     )
-    .orderBy(asc(messages.createdAt))
+    .orderBy(asc(messages.createdAt), asc(messages.id))
     .all()
     .map(parseMessage);
 }

package/src/memory/conversation-queries.ts

@@ -1,5 +1,11 @@
-import { and, count, desc, eq, sql } from "drizzle-orm";
+import { and, count, desc, eq, inArray, isNull, sql } from "drizzle-orm";
 
+import {
+  parseExternalContentEnvelope,
+  type UntrustedContentSource,
+  unwrapExternalContentForDisplay,
+  wrapUntrustedContent,
+} from "../security/untrusted-content.js";
 import { getLogger } from "../util/logger.js";
 import type { ConversationRow } from "./conversation-crud.js";
 import { parseConversation } from "./conversation-crud.js";
@@ -55,9 +61,14 @@ export function listConversations(
   // SQLite file without running migrations in-process, so legacy private rows
   // can briefly exist before migration cleanup. Hide them from foreground
   // lists until the next migration pass deletes them.
+  //
+  // group_id is checked alongside conversationType so that conversations
+  // routed to system:background (e.g. heartbeat) via conversationMetadata
+  // but created with conversationType "standard" (vellum channel strategy)
+  // appear in the correct bucket.
   const typeCond = backgroundOnly
-    ? sql`${conversations.conversationType} IN ('background', 'scheduled') AND (${conversations.source} IS NULL OR ${conversations.source} != 'subagent')`
-    : sql`${conversations.conversationType} NOT IN ('background', 'scheduled', 'private')`;
+    ? sql`(${conversations.conversationType} IN ('background', 'scheduled') OR group_id IN ('system:background', 'system:scheduled')) AND (${conversations.source} IS NULL OR ${conversations.source} != 'subagent')`
+    : sql`${conversations.conversationType} NOT IN ('background', 'scheduled', 'private') AND COALESCE(group_id, 'system:all') NOT IN ('system:background', 'system:scheduled')`;
   const where = includeArchived
     ? typeCond
     : sql`${typeCond} AND ${conversations.archivedAt} IS NULL`;
@@ -75,6 +86,92 @@ export function listConversations(
   return query.all().map(parseConversation);
 }
 
+/**
+ * List conversations matching an exact `source` value, ordered by `createdAt`
+ * descending. The surgical filter for "find every background run produced by
+ * job X" — heartbeat, memory_v2_consolidation, watcher-engine, etc. — since
+ * `source` is the canonical job-class distinguisher across the background
+ * bucket. `conversationType` + `group_id` only narrow to "background vs
+ * scheduled vs standard"; neither identifies which job produced the row.
+ *
+ * Filter is exact (no `LIKE`, no implicit exclusions): the route layer is
+ * responsible for knowing which source constants exist and passing one. The
+ * defensive `source != 'subagent'` carve-out applied by `listConversations`
+ * is deliberately NOT replicated here — a caller asking for an exact source
+ * gets exactly that source.
+ *
+ * @param source        Exact match against `conversations.source`. Pass the
+ *                      canonical constant (e.g. `MEMORY_V2_CONSOLIDATION_SOURCE`).
+ * @param limit         Maximum rows to return (default 20).
+ * @param opts.includeArchived  Include rows with non-null `archivedAt`.
+ *                              Defaults to `true` so callers that want a full
+ *                              run history get one; pass `false` for views
+ *                              that hide archived rows.
+ */
+export function listConversationsBySource(
+  source: string,
+  limit = 20,
+  opts?: { includeArchived?: boolean },
+): ConversationRow[] {
+  const db = getDb();
+  const includeArchived = opts?.includeArchived ?? true;
+  const where = includeArchived
+    ? eq(conversations.source, source)
+    : and(eq(conversations.source, source), isNull(conversations.archivedAt));
+  const rows = db
+    .select()
+    .from(conversations)
+    .where(where)
+    .orderBy(desc(conversations.createdAt))
+    .limit(limit)
+    .all();
+  return rows.map(parseConversation);
+}
+
+/**
+ * Per-conversation aggregate of messages with a specific role. Powers
+ * heartbeat-shaped run endpoints (e.g. `consolidation/runs`) that need a
+ * "did the agent emit any output?" signal stronger than
+ * `conversations.lastMessageAt` — which is bumped by the kickoff user
+ * prompt and so cannot distinguish "agent ran" from "agent dispatched but
+ * crashed before responding".
+ *
+ * Single batched aggregate query (no N+1). Conversations with zero matching
+ * messages are NOT present in the returned map — callers should treat a
+ * missing key as `{ count: 0, lastAt: null }`.
+ *
+ * @param conversationIds  Conversation ids to look up (empty → empty map).
+ * @param role             Message role to count (default `"assistant"`).
+ */
+export function getMessageRoleStatsByConversation(
+  conversationIds: string[],
+  role: string = "assistant",
+): Map<string, { count: number; lastAt: number }> {
+  if (conversationIds.length === 0) return new Map();
+  const db = getDb();
+  const rows = db
+    .select({
+      conversationId: messages.conversationId,
+      count: sql<number>`COUNT(*)`.as("count"),
+      lastAt: sql<number>`MAX(${messages.createdAt})`.as("last_at"),
+    })
+    .from(messages)
+    .where(
+      and(
+        inArray(messages.conversationId, conversationIds),
+        eq(messages.role, role),
+      ),
+    )
+    .groupBy(messages.conversationId)
+    .all();
+  return new Map(
+    rows.map((r) => [
+      r.conversationId,
+      { count: Number(r.count), lastAt: Number(r.lastAt) },
+    ]),
+  );
+}
+
 export function listPinnedConversations(): ConversationRow[] {
   ensureDisplayOrderMigration();
   ensureGroupMigration();
@@ -151,10 +248,11 @@ export function listConversationsByTitlePrefix(
 }
 
 export function countConversations(backgroundOnly = false): number {
+  ensureGroupMigration();
   const db = getDb();
   const where = backgroundOnly
-    ? sql`${conversations.conversationType} IN ('background', 'scheduled') AND (${conversations.source} IS NULL OR ${conversations.source} != 'subagent')`
-    : sql`${conversations.conversationType} NOT IN ('background', 'scheduled')`;
+    ? sql`(${conversations.conversationType} IN ('background', 'scheduled') OR group_id IN ('system:background', 'system:scheduled')) AND (${conversations.source} IS NULL OR ${conversations.source} != 'subagent')`
+    : sql`${conversations.conversationType} NOT IN ('background', 'scheduled', 'private') AND COALESCE(group_id, 'system:all') NOT IN ('system:background', 'system:scheduled')`;
   const [{ total }] = db
     .select({ total: count() })
     .from(conversations)
@@ -233,6 +331,7 @@ export function searchConversations(
 ): ConversationSearchResult[] {
   if (!query.trim()) return [];
 
+  ensureGroupMigration();
   const db = getDb();
   const limit = opts?.limit ?? 20;
   const maxMsgsPerConv = opts?.maxMessagesPerConversation ?? 3;
@@ -262,7 +361,7 @@ export function searchConversations(
         FROM messages_fts f
         JOIN messages m ON m.id = f.message_id
         JOIN conversations c ON c.id = m.conversation_id
-        WHERE messages_fts MATCH ? AND c.conversation_type NOT IN ('background', 'scheduled', 'private') AND c.archived_at IS NULL
+        WHERE messages_fts MATCH ? AND c.conversation_type NOT IN ('background', 'scheduled', 'private') AND COALESCE(c.group_id, 'system:all') NOT IN ('system:background', 'system:scheduled') AND c.archived_at IS NULL
         LIMIT 1000
       `,
         ftsMatch,
@@ -287,7 +386,7 @@ export function searchConversations(
       SELECT DISTINCT m.conversation_id
       FROM messages m
       JOIN conversations c ON c.id = m.conversation_id
-      WHERE m.content LIKE ? ESCAPE '\\' AND c.conversation_type NOT IN ('background', 'scheduled', 'private') AND c.archived_at IS NULL
+      WHERE m.content LIKE ? ESCAPE '\\' AND c.conversation_type NOT IN ('background', 'scheduled', 'private') AND COALESCE(c.group_id, 'system:all') NOT IN ('system:background', 'system:scheduled') AND c.archived_at IS NULL
       LIMIT 1000
     `,
       likePattern,
@@ -302,6 +401,7 @@ export function searchConversations(
     .where(
       and(
         sql`${conversations.conversationType} NOT IN ('background', 'scheduled', 'private')`,
+        sql`COALESCE(group_id, 'system:all') NOT IN ('system:background', 'system:scheduled')`,
         sql`${conversations.title} LIKE ${titlePattern} ESCAPE '\\'`,
         sql`${conversations.archivedAt} IS NULL`,
       ),
@@ -403,29 +503,68 @@ export function searchConversations(
  * text; we extract a readable snippet in either case.
  */
 export function buildExcerpt(rawContent: string, query: string): string {
+  return buildExcerptWithExternalContentMode(rawContent, query, "display");
+}
+
+/**
+ * Build an excerpt for model-facing recall evidence. Unlike display excerpts,
+ * this keeps complete external_content envelopes around untrusted snippets so
+ * the model still sees clear third-party content boundaries.
+ */
+export function buildRecallEvidenceExcerpt(
+  rawContent: string,
+  query: string,
+): string {
+  return buildExcerptWithExternalContentMode(rawContent, query, "preserve");
+}
+
+function buildExcerptWithExternalContentMode(
+  rawContent: string,
+  query: string,
+  externalContentMode: "display" | "preserve",
+): string {
   // Try to extract plain text from JSON content blocks first.
   let text = rawContent;
   try {
     const parsed = JSON.parse(rawContent);
     if (Array.isArray(parsed)) {
       const parts: string[] = [];
+      let preservedExternalContent = false;
       for (const block of parsed) {
         if (typeof block === "object" && block != null) {
           if (block.type === "text" && typeof block.text === "string") {
-            parts.push(block.text);
+            if (externalContentMode === "display") {
+              parts.push(unwrapExternalContentForDisplay(block.text));
+            } else {
+              const excerpt = buildRecallEvidenceText(block.text, query);
+              parts.push(excerpt.text);
+              preservedExternalContent ||= excerpt.preservedExternalContent;
+            }
           } else if (
             block.type === "tool_result" ||
             block.type === "web_search_tool_result"
           ) {
             const inner = Array.isArray(block.content) ? block.content : [];
             for (const ib of inner) {
-              if (ib?.type === "text" && typeof ib.text === "string")
-                parts.push(ib.text);
+              if (ib?.type === "text" && typeof ib.text === "string") {
+                if (externalContentMode === "display") {
+                  parts.push(unwrapExternalContentForDisplay(ib.text));
+                } else {
+                  const excerpt = buildRecallEvidenceText(ib.text, query);
+                  parts.push(excerpt.text);
+                  preservedExternalContent ||= excerpt.preservedExternalContent;
+                }
+              }
             }
           }
         }
       }
-      if (parts.length > 0) text = parts.join(" ");
+      if (parts.length > 0) {
+        text = parts.join(" ");
+        if (externalContentMode === "preserve" && preservedExternalContent) {
+          return text;
+        }
+      }
     } else if (typeof parsed === "string") {
       text = parsed;
     }
@@ -433,6 +572,53 @@ export function buildExcerpt(rawContent: string, query: string): string {
     // Not JSON — use as-is
   }
 
+  if (externalContentMode === "display") {
+    text = unwrapExternalContentForDisplay(text);
+  } else {
+    const envelope = parseExternalContentEnvelope(text);
+    if (envelope) {
+      const innerExcerpt = buildExcerptFromText(envelope.content, query);
+      return wrapRecallEvidenceExcerpt(
+        innerExcerpt,
+        envelope.source,
+        envelope.origin,
+      );
+    }
+  }
+
+  return buildExcerptFromText(text, query);
+}
+
+function buildRecallEvidenceText(
+  text: string,
+  query: string,
+): { text: string; preservedExternalContent: boolean } {
+  const envelope = parseExternalContentEnvelope(text);
+  if (!envelope) {
+    return { text, preservedExternalContent: false };
+  }
+  const innerExcerpt = buildExcerptFromText(envelope.content, query);
+  return {
+    text: wrapRecallEvidenceExcerpt(
+      innerExcerpt,
+      envelope.source,
+      envelope.origin,
+    ),
+    preservedExternalContent: true,
+  };
+}
+
+function wrapRecallEvidenceExcerpt(
+  excerpt: string,
+  source: UntrustedContentSource,
+  origin?: string,
+): string {
+  return origin
+    ? wrapUntrustedContent(excerpt, { source, sourceDetail: origin })
+    : wrapUntrustedContent(excerpt, { source });
+}
+
+function buildExcerptFromText(text: string, query: string): string {
   const WINDOW = 100;
   const lowerText = text.toLowerCase();
   const lowerQuery = query.toLowerCase();

package/src/memory/conversation-title-service.ts

@@ -298,9 +298,11 @@ function buildTitleSystemPrompt(): string {
     "You generate ultra-concise conversation titles. Output ONLY the title text — no explanation, no quotes, no markdown, no preamble.",
     "",
     "Rules:",
-    "- 2–6 words. Titles longer than 6 words are unacceptable — ruthlessly compress",
-    "- Summarize the TOPIC, not the request or instructions",
-    "- Noun phrases are ideal (e.g. 'Auth Middleware Rewrite', 'Docker Volume Mounts')",
+    "- 2–5 words maximum. Titles longer than 5 words are unacceptable — ruthlessly compress to a short noun phrase",
+    "- 40 characters absolute maximum — if your title exceeds 40 characters it will be truncated and look broken",
+    "- Summarize only the TOPIC, not the request or instructions",
+    "- Noun phrases are ideal (e.g. 'Auth Middleware Rewrite', 'Docker Volume Mounts', 'Onboarding Flow')",
+    "- Think: what would make a scannable sidebar label?",
     "- Do NOT echo back what the user asked you to do",
     "- Do NOT respond to the conversation content",
     "- Do NOT assess feasibility or comment on capabilities",
@@ -353,13 +355,33 @@ const META_FAILURE_TITLES = new Set([
   "no content",
 ]);
 
+const MAX_TITLE_LENGTH = 40;
+const MAX_TITLE_WORDS = 7;
+
+function truncateTitle(title: string): string {
+  if (title.length <= MAX_TITLE_LENGTH) return title;
+  const words = title.split(/\s+/);
+  if (words.length <= MAX_TITLE_WORDS) {
+    // Long words but few of them — truncate to char limit at word boundary
+    let result = "";
+    for (const word of words) {
+      const candidate = result ? result + " " + word : word;
+      if (candidate.length > MAX_TITLE_LENGTH) break;
+      result = candidate;
+    }
+    return result || title.slice(0, MAX_TITLE_LENGTH);
+  }
+  // Too many words — trim to 5 words
+  return words.slice(0, 5).join(" ");
+}
+
 function normalizeTitle(raw: string): string {
   let title = raw.trim().replace(/^["']|["']$/g, "");
   title = stripMarkdown(title);
   if (META_FAILURE_TITLES.has(title.toLowerCase())) {
     return "";
   }
-  return title;
+  return truncateTitle(title);
 }
 
 /** Strip common markdown formatting so titles render as plain text. */

package/src/memory/db-init.ts

@@ -32,12 +32,14 @@ import {
   createMessagesFts,
   createNotificationTables,
   createOAuthTables,
+  createOnboardingEventsTable,
   createScopedApprovalGrantsTable,
   createSequenceTables,
   createTasksAndWorkItemsTables,
   createWatchersAndLogsTables,
   migrate230AcpSessionHistory,
   migrate231RepairMemoryGraphEventDates,
+  migrateA2ATasks,
   migrateActivationState,
   migrateActivationStateFkCascade,
   migrateAddConversationInferenceProfile,
@@ -94,6 +96,7 @@ import {
   migrateDropSetupSkillIdColumn,
   migrateDropSimplifiedMemory,
   migrateDropUsageCompositeIndexes,
+  migrateExternalConversationBindingThreadId,
   migrateFkCascadeRebuilds,
   migrateGuardianActionFollowup,
   migrateGuardianActionSupersession,
@@ -109,6 +112,7 @@ import {
   migrateHeartbeatRuns,
   migrateInviteCodeHashColumn,
   migrateInviteContactId,
+  migrateLlmRequestLogAgentLoopExitReason,
   migrateLlmRequestLogMessageId,
   migrateLlmRequestLogProvider,
   migrateLlmRequestLogsCreatedAtIndex,
@@ -122,6 +126,7 @@ import {
   migrateMessagesConversationCreatedAtIndex,
   migrateMessagesFtsBackfill,
   migrateNormalizePhoneIdentities,
+  migrateNormalizeSlackExternalContent,
   migrateNormalizeUserFileByPrincipal,
   migrateNotificationDeliveryThreadDecision,
   migrateOAuthAppsClientSecretPath,
@@ -139,6 +144,7 @@ import {
   migrateOAuthProvidersScopeSeparator,
   migrateOAuthProvidersTokenAuthMethodDefault,
   migrateOAuthProvidersTokenExchangeBodyFormat,
+  migrateProviderConnectionBaseUrlAndModels,
   migrateProviderConnectionStatusLabel,
   migrateReminderRoutingIntent,
   migrateRemindersToSchedules,
@@ -424,6 +430,12 @@ export function initializeDb(): void {
     migrateProviderConnectionStatusLabel,
     migrateMemoryRetrospectiveState,
     migrateBackfillProviderConnectionLabel,
+    migrateExternalConversationBindingThreadId,
+    createOnboardingEventsTable,
+    migrateNormalizeSlackExternalContent,
+    migrateProviderConnectionBaseUrlAndModels,
+    migrateA2ATasks,
+    migrateLlmRequestLogAgentLoopExitReason,
   ];
 
   // Run each migration step, catching and logging individual failures so one