@vellumai/assistant 0.5.1 → 0.5.2

package/src/memory/db-init.ts

@@ -37,6 +37,7 @@ import {
   migrateAssistantContactMetadata,
   migrateBackfillContactInteractionStats,
   migrateBackfillGuardianPrincipalId,
+  migrateBackfillInlineAttachmentsToDisk,
   migrateBackfillUsageCacheAccounting,
   migrateCallSessionInviteMetadata,
   migrateCallSessionMode,
@@ -50,6 +51,7 @@ import {
   migrateContactsAssistantId,
   migrateContactsNotesColumn,
   migrateContactsRolePrincipal,
+  migrateConversationForkLineage,
   migrateConversationsThreadTypeIndex,
   migrateCreateThreadStartersTable,
   migrateCreateTraceEventsTable,
@@ -78,11 +80,15 @@ import {
   migrateGuardianVerificationSessions,
   migrateInviteCodeHashColumn,
   migrateInviteContactId,
+  migrateLlmRequestLogMessageId,
+  migrateLlmRequestLogProvider,
   migrateMemoryItemSupersession,
   migrateMessagesFtsBackfill,
   migrateNormalizePhoneIdentities,
   migrateNotificationDeliveryThreadDecision,
   migrateOAuthAppsClientSecretPath,
+  migrateOAuthProvidersDisplayMetadata,
+  migrateOAuthProvidersManagedServiceConfigKey,
   migrateOAuthProvidersPingUrl,
   migrateReminderRoutingIntent,
   migrateRemindersToSchedules,
@@ -96,6 +102,7 @@ import {
   migrateRenameSequenceEnrollmentsThreadIdColumn,
   migrateRenameSequenceStepsReplyKey,
   migrateRenameSourceSessionIdColumn,
+  migrateRenameThreadStartersCheckpoints,
   migrateRenameThreadStartersTable,
   migrateRenameVerificationSessionIdColumn,
   migrateRenameVerificationTable,
@@ -447,6 +454,9 @@ export function initializeDb(): void {
   // 77. Rename thread_starters → conversation_starters table and indexes
   migrateRenameThreadStartersTable(database);
 
+  // 77b. Rename checkpoint keys from thread_starters: → conversation_starters: prefix
+  migrateRenameThreadStartersCheckpoints(database);
+
   // 78. Lifecycle events table for app_open / hatch telemetry
   createLifecycleEventsTable(database);
 
@@ -456,6 +466,24 @@ export function initializeDb(): void {
   // 80. Trace events table for persistent trace/activity storage across sessions
   migrateCreateTraceEventsTable(database);
 
+  // 81. Add managed_service_config_key column to oauth_providers
+  migrateOAuthProvidersManagedServiceConfigKey(database);
+
+  // 81b. Add display metadata columns to oauth_providers (display_name, description, dashboard_url, etc.)
+  migrateOAuthProvidersDisplayMetadata(database);
+
+  // 82. Add message_id column to llm_request_logs for per-message LLM context lookup
+  migrateLlmRequestLogMessageId(database);
+
+  // 82b. Add provider column to llm_request_logs for runtime provider lookup
+  migrateLlmRequestLogProvider(database);
+
+  // 83. Backfill existing inline (base64-in-DB) attachments to on-disk storage
+  migrateBackfillInlineAttachmentsToDisk(database);
+
+  // 84. Add nullable conversation fork lineage columns and parent lookup index
+  migrateConversationForkLineage(database);
+
   validateMigrationState(database);
 
   if (process.env.BUN_TEST === "1") {

package/src/memory/embedding-backend.ts

@@ -203,7 +203,14 @@ function getCachedOrCreate<T extends EmbeddingBackend>(
   return instance;
 }
 
-/** Check if a backend instance already exists in the cache. */
+/**
+ * Look up a previously cached backend instance. Returns undefined when no
+ * cached entry exists. Used as a fallback when a provider key lookup
+ * returns undefined — a transient credential-store outage should not
+ * disable a provider whose backend is already warmed in memory. Explicit
+ * key deletion triggers `clearEmbeddingBackendCache()` which empties the
+ * cache, so a stale backend is never returned after intentional removal.
+ */
 function getCached(
   provider: string,
   model: string,
@@ -259,9 +266,6 @@ export async function selectEmbeddingBackend(
     };
   }
   if (requested === "ollama") {
-    // Check cache first to avoid unnecessary async key fetch on cache hits
-    const cached = getCached("ollama", config.memory.embeddings.ollamaModel);
-    if (cached) return { backend: cached, reason: null };
     const ollamaKey = (await getProviderKeyAsync("ollama")) ?? undefined;
     return {
       backend: getCachedOrCreate(
@@ -298,14 +302,17 @@ export async function selectEmbeddingBackend(
           reason: null,
         };
       case "openai": {
-        // Check cache first to avoid unnecessary async key fetch on cache hits
-        const cachedOpenai = getCached(
-          "openai",
-          config.memory.embeddings.openaiModel,
-        );
-        if (cachedOpenai) return { backend: cachedOpenai, reason: null };
         const openaiKey = await getProviderKeyAsync("openai");
-        if (!openaiKey) continue;
+        if (!openaiKey) {
+          // Preserve cached backend on transient credential-store failures.
+          // Explicit key deletion clears the cache via clearEmbeddingBackendCache().
+          const cached = getCached(
+            "openai",
+            config.memory.embeddings.openaiModel,
+          );
+          if (cached) return { backend: cached, reason: null };
+          continue;
+        }
         return {
           backend: getCachedOrCreate(
             "openai",
@@ -320,15 +327,16 @@ export async function selectEmbeddingBackend(
         };
       }
       case "gemini": {
-        // Check cache first to avoid unnecessary async key fetch on cache hits
-        const cachedGemini = getCached(
-          "gemini",
-          config.memory.embeddings.geminiModel,
-          geminiCacheExtras(config),
-        );
-        if (cachedGemini) return { backend: cachedGemini, reason: null };
         const geminiKey = await getProviderKeyAsync("gemini");
-        if (!geminiKey) continue;
+        if (!geminiKey) {
+          const cached = getCached(
+            "gemini",
+            config.memory.embeddings.geminiModel,
+            geminiCacheExtras(config),
+          );
+          if (cached) return { backend: cached, reason: null };
+          continue;
+        }
         return {
           backend: getCachedOrCreate(
             "gemini",
@@ -348,12 +356,6 @@ export async function selectEmbeddingBackend(
         };
       }
       case "ollama": {
-        // Check cache first to avoid unnecessary async key fetch on cache hits
-        const cachedOllama = getCached(
-          "ollama",
-          config.memory.embeddings.ollamaModel,
-        );
-        if (cachedOllama) return { backend: cachedOllama, reason: null };
         if (!(await isOllamaConfigured(config))) continue;
         const ollamaKey = (await getProviderKeyAsync("ollama")) ?? undefined;
         return {
@@ -570,15 +572,6 @@ async function selectFallbackBackends(
     if (provider === exclude) continue;
     switch (provider) {
       case "openai": {
-        // Check cache first to avoid unnecessary async key fetch on cache hits
-        const cachedOpenai = getCached(
-          "openai",
-          config.memory.embeddings.openaiModel,
-        );
-        if (cachedOpenai) {
-          backends.push(cachedOpenai);
-          break;
-        }
         const openaiKey = await getProviderKeyAsync("openai");
         if (openaiKey) {
           backends.push(
@@ -592,20 +585,17 @@ async function selectFallbackBackends(
                 ),
             ),
           );
+        } else {
+          // Preserve cached backend on transient credential-store failures.
+          const cached = getCached(
+            "openai",
+            config.memory.embeddings.openaiModel,
+          );
+          if (cached) backends.push(cached);
         }
         break;
       }
       case "gemini": {
-        // Check cache first to avoid unnecessary async key fetch on cache hits
-        const cachedGemini = getCached(
-          "gemini",
-          config.memory.embeddings.geminiModel,
-          geminiCacheExtras(config),
-        );
-        if (cachedGemini) {
-          backends.push(cachedGemini);
-          break;
-        }
         const geminiKey = await getProviderKeyAsync("gemini");
         if (geminiKey) {
           backends.push(
@@ -624,19 +614,18 @@ async function selectFallbackBackends(
               geminiCacheExtras(config),
             ),
           );
+        } else {
+          // Preserve cached backend on transient credential-store failures.
+          const cached = getCached(
+            "gemini",
+            config.memory.embeddings.geminiModel,
+            geminiCacheExtras(config),
+          );
+          if (cached) backends.push(cached);
         }
         break;
       }
       case "ollama": {
-        // Check cache first to avoid unnecessary async key fetch on cache hits
-        const cachedOllama = getCached(
-          "ollama",
-          config.memory.embeddings.ollamaModel,
-        );
-        if (cachedOllama) {
-          backends.push(cachedOllama);
-          break;
-        }
         if (await isOllamaConfigured(config)) {
           const ollamaKey = (await getProviderKeyAsync("ollama")) ?? undefined;
           backends.push(

package/src/memory/embedding-gemini.test.ts

@@ -14,7 +14,9 @@ describe("GeminiEmbeddingBackend", () => {
   let mockFetch: ReturnType<typeof mock>;
 
   beforeEach(() => {
-    mockFetch = mock(() => Promise.resolve(makeSuccessResponse([0.1, 0.2, 0.3])));
+    mockFetch = mock(() =>
+      Promise.resolve(makeSuccessResponse([0.1, 0.2, 0.3])),
+    );
     globalThis.fetch = mockFetch as unknown as typeof fetch;
   });
 
@@ -186,9 +188,7 @@ describe("GeminiEmbeddingBackend", () => {
   describe("error handling", () => {
     test("throws on non-OK response", async () => {
       mockFetch = mock(() =>
-        Promise.resolve(
-          new Response("Internal Server Error", { status: 500 }),
-        ),
+        Promise.resolve(new Response("Internal Server Error", { status: 500 })),
       );
       globalThis.fetch = mockFetch as unknown as typeof fetch;
 

package/src/memory/embedding-local.ts

@@ -66,9 +66,7 @@ export class LocalEmbeddingBackend implements EmbeddingBackend {
     const texts = inputs.map((i) => {
       const n = normalizeEmbeddingInput(i);
       if (n.type !== "text") {
-        throw new Error(
-          "Local embedding backend only supports text inputs",
-        );
+        throw new Error("Local embedding backend only supports text inputs");
       }
       return n.text;
     });

package/src/memory/embedding-ollama.ts

@@ -34,9 +34,7 @@ export class OllamaEmbeddingBackend implements EmbeddingBackend {
     const texts = inputs.map((i) => {
       const n = normalizeEmbeddingInput(i);
       if (n.type !== "text") {
-        throw new Error(
-          "Ollama embedding backend only supports text inputs",
-        );
+        throw new Error("Ollama embedding backend only supports text inputs");
       }
       return n.text;
     });

package/src/memory/embedding-openai.ts

@@ -26,9 +26,7 @@ export class OpenAIEmbeddingBackend implements EmbeddingBackend {
     const texts = inputs.map((i) => {
       const n = normalizeEmbeddingInput(i);
       if (n.type !== "text") {
-        throw new Error(
-          "OpenAI embedding backend only supports text inputs",
-        );
+        throw new Error("OpenAI embedding backend only supports text inputs");
       }
       return n.text;
     });

package/src/memory/indexer.ts

@@ -9,7 +9,7 @@ import { getDb } from "./db.js";
 import { selectedBackendSupportsMultimodal } from "./embedding-backend.js";
 import { enqueueMemoryJob } from "./jobs-store.js";
 import {
-  extractMediaBlocks,
+  extractMediaBlockMeta,
   extractTextFromStoredMessageContent,
 } from "./message-content.js";
 import { memorySegments } from "./schema.js";
@@ -71,16 +71,18 @@ export async function indexMessageNow(
     input.role === "user" ||
     (input.role === "assistant" && config.extraction.extractFromAssistant);
   // Check if the message has any image blocks before probing the backend.
-  // extractMediaBlocks is synchronous and cheap, while
-  // selectedBackendSupportsMultimodal requires async key resolution that
-  // would add unnecessary latency for text-only messages.
-  const candidateMediaBlocks = extractMediaBlocks(input.content).filter(
+  // extractMediaBlockMeta is synchronous and lightweight — it detects image
+  // blocks without decoding base64 data into Buffers, avoiding CPU/memory
+  // overhead for messages on non-multimodal backends.
+  // selectedBackendSupportsMultimodal requires async key resolution, so we
+  // skip it entirely for text-only messages.
+  const candidateMediaMeta = extractMediaBlockMeta(input.content).filter(
     (b) => b.type === "image",
   );
   const mediaBlocks =
-    candidateMediaBlocks.length > 0 &&
+    candidateMediaMeta.length > 0 &&
     (await selectedBackendSupportsMultimodal(getConfig()))
-      ? candidateMediaBlocks
+      ? candidateMediaMeta
       : [];
 
   // Wrap all segment inserts and job enqueues in a single transaction so they

package/src/memory/items-extractor.ts

@@ -135,6 +135,13 @@ function hasSemanticDensity(text: string): boolean {
 
 // ── LLM-powered extraction ────────────────────────────────────────────
 
+// Budget for the extraction system prompt (in characters). This is a
+// conservative estimate that fits comfortably within even small model
+// context windows (latency-optimized models like Haiku). The remaining
+// context budget is consumed by the user message, tool schema, and response
+// tokens. ~6000 tokens ≈ 24 000 chars is a safe ceiling.
+const EXTRACTION_SYSTEM_PROMPT_CHAR_BUDGET = 24_000;
+
 function buildExtractionSystemPrompt(
   existingItems: Array<{
     id: string;
@@ -144,16 +151,9 @@ function buildExtractionSystemPrompt(
   }>,
   messageRole: string,
 ): string {
-  // Inject identity context so extracted memories use real names instead of
-  // generic "User ..." labels.
-  const identityContext = buildCoreIdentityContext();
-
-  let prompt = "";
-  if (identityContext) {
-    prompt += `# Identity Context\n\n${identityContext}\n\n---\n\n`;
-  }
-
-  prompt += `You are a memory extraction system. Given a message from a conversation, extract structured memory items that would be valuable to remember for future interactions.
+  // Build the fixed instruction body first so we can measure it and allocate
+  // the remaining budget to identity context.
+  let instructions = `You are a memory extraction system. Given a message from a conversation, extract structured memory items that would be valuable to remember for future interactions.
 
 Extract items in these categories:
 - identity: Personal info (name, role, location, timezone, background), notable facts, relationships between people/teams/systems
@@ -183,22 +183,51 @@ Rules:
 - Only extract genuinely memorable information. Skip pleasantries, filler, and transient discussion.
 - Do NOT extract information about what tools the assistant used or what files it read — only extract substantive facts about the user, their projects, and their preferences.
 - Do NOT extract claims about actions the assistant performed, outcomes it achieved, or progress it reported (e.g., "I booked an appointment", "I sent the email"). Only extract facts stated by the user or from external sources — the assistant's self-reports are not reliable memory material.
+- Do NOT extract raw code snippets, JSON fragments, YAML, configuration values, log output, or data structures. Only extract the human-readable meaning or intent behind such content, not the literal syntax.
 - Prefer fewer high-quality items over many low-quality ones.
 - If the message contains no memorable information, return an empty array.`;
 
   if (messageRole === "assistant") {
-    prompt += `
+    instructions += `
 
 IMPORTANT: The message below is from the ASSISTANT, not the user. Do NOT attribute the assistant's own statements, feelings, self-descriptions, or introspection to the user. Only extract facts about the user, the world, or the project that the assistant is referencing or relaying — NOT the assistant's own identity, uncertainty, or behavior. If the assistant is simply talking about itself (e.g., introducing itself, expressing uncertainty about its own purpose), extract nothing.`;
   }
 
   if (existingItems.length > 0) {
-    prompt += `\n\nExisting memory items (use these to identify supersession targets — set \`supersedes\` to the item ID if the new information replaces one of these):\n`;
+    instructions += `\n\nExisting memory items (use these to identify supersession targets — set \`supersedes\` to the item ID if the new information replaces one of these):\n`;
     for (const item of existingItems) {
-      prompt += `- [${item.id}] (${item.kind}) ${item.subject}: ${item.statement}\n`;
+      instructions += `- [${item.id}] (${item.kind}) ${item.subject}: ${item.statement}\n`;
+    }
+  }
+
+  // Inject identity context so extracted memories use real names instead of
+  // generic "User ..." labels. Budget is dynamically computed: whatever
+  // remains after the fixed instructions fits within the system prompt
+  // ceiling, preventing oversized prompts from exceeding the provider input
+  // window (which would cause sendMessage to error and fall back to
+  // lower-quality pattern-based extraction).
+  const rawIdentityContext = buildCoreIdentityContext();
+
+  let prompt = "";
+  if (rawIdentityContext) {
+    // Reserve space for the wrapping text: "# Identity Context\n\n" + "\n\n---\n\n"
+    const wrapperOverhead = "# Identity Context\n\n\n\n---\n\n".length;
+    const identityBudget =
+      EXTRACTION_SYSTEM_PROMPT_CHAR_BUDGET -
+      instructions.length -
+      wrapperOverhead;
+
+    if (identityBudget > 0) {
+      const identityContext = truncate(
+        rawIdentityContext,
+        identityBudget,
+        "\n...[identity context truncated]",
+      );
+      prompt += `# Identity Context\n\n${identityContext}\n\n---\n\n`;
     }
   }
 
+  prompt += instructions;
   return prompt;
 }
 

package/src/memory/job-handlers/conversation-starters.ts

@@ -169,7 +169,12 @@ async function generateStarters(scopeId: string): Promise<GeneratedStarter[]> {
   const now = new Date();
   const timeContext = `Current time: ${now.toLocaleString("en-US", { weekday: "long", year: "numeric", month: "long", day: "numeric", hour: "numeric", minute: "2-digit", hour12: true })}`;
 
-  const identityContext = buildCoreIdentityContext();
+  // Truncate identity context to prevent oversized prompts when SOUL.md /
+  // IDENTITY.md / USER.md are large.
+  const rawIdentityContext = buildCoreIdentityContext();
+  const identityContext = rawIdentityContext
+    ? truncate(rawIdentityContext, 2000, "\n…[truncated]")
+    : null;
 
   const systemPrompt = `You are generating 4 conversation starters for a personal assistant app. These appear as clickable chips on the empty conversation page — the first thing the user sees when they open the app.
 

package/src/memory/job-handlers/embedding.test.ts

@@ -142,10 +142,7 @@ describe("embedMediaJob", () => {
       })
       .run();
 
-    await embedMediaJob(
-      makeJob({ assetId: "asset-registered" }),
-      TEST_CONFIG,
-    );
+    await embedMediaJob(makeJob({ assetId: "asset-registered" }), TEST_CONFIG);
     expect(embedAndUpsertCalls).toHaveLength(0);
   });
 

package/src/memory/llm-request-log-store.ts

@@ -1,6 +1,11 @@
-import { and, eq, gte, lte } from "drizzle-orm";
+import { and, eq, gte, inArray, isNull, lte } from "drizzle-orm";
 import { v4 as uuid } from "uuid";
 
+import {
+  getAssistantMessageIdsInTurn,
+  getMessageById,
+  messageMetadataSchema,
+} from "./conversation-crud.js";
 import { getDb } from "./db.js";
 import { llmRequestLogs } from "./schema.js";
 
@@ -8,12 +13,16 @@ export function recordRequestLog(
   conversationId: string,
   requestPayload: string,
   responsePayload: string,
+  messageId?: string,
+  provider?: string,
 ): void {
   const db = getDb();
   db.insert(llmRequestLogs)
     .values({
       id: uuid(),
       conversationId,
+      messageId: messageId ?? null,
+      provider: provider ?? null,
       requestPayload,
       responsePayload,
       createdAt: Date.now(),
@@ -46,3 +55,93 @@ export function queryRequestLogs(
     .orderBy(llmRequestLogs.createdAt)
     .all();
 }
+
+export function backfillMessageIdOnLogs(
+  conversationId: string,
+  messageId: string,
+): void {
+  const db = getDb();
+  db.update(llmRequestLogs)
+    .set({ messageId })
+    .where(
+      and(
+        eq(llmRequestLogs.conversationId, conversationId),
+        isNull(llmRequestLogs.messageId),
+      ),
+    )
+    .run();
+}
+
+/**
+ * Internal helper: query `llm_request_logs` for rows matching any of the
+ * given message IDs, ordered by `createdAt ASC`. Uses the existing
+ * `idx_llm_request_logs_message_id` index via `inArray`.
+ */
+function selectLogsByMessageIds(messageIds: string[]): Array<{
+  id: string;
+  conversationId: string;
+  messageId: string | null;
+  provider: string | null;
+  requestPayload: string;
+  responsePayload: string;
+  createdAt: number;
+}> {
+  if (messageIds.length === 0) return [];
+  const db = getDb();
+  return db
+    .select({
+      id: llmRequestLogs.id,
+      conversationId: llmRequestLogs.conversationId,
+      messageId: llmRequestLogs.messageId,
+      provider: llmRequestLogs.provider,
+      requestPayload: llmRequestLogs.requestPayload,
+      responsePayload: llmRequestLogs.responsePayload,
+      createdAt: llmRequestLogs.createdAt,
+    })
+    .from(llmRequestLogs)
+    .where(inArray(llmRequestLogs.messageId, messageIds))
+    .orderBy(llmRequestLogs.createdAt)
+    .all();
+}
+
+export function getRequestLogsByMessageId(messageId: string): Array<{
+  id: string;
+  conversationId: string;
+  messageId: string | null;
+  provider: string | null;
+  requestPayload: string;
+  responsePayload: string;
+  createdAt: number;
+}> {
+  // Resolve all assistant message IDs in the same turn so the inspector
+  // shows every LLM call from the entire agent turn, not just the queried message.
+  const turnMessageIds = getAssistantMessageIdsInTurn(messageId);
+  const turnLogs = selectLogsByMessageIds(turnMessageIds);
+  if (turnLogs.length > 0) {
+    return turnLogs;
+  }
+
+  // Fork-source fallback: if no logs found for the turn, check whether
+  // the queried message was forked from a source and resolve that source's turn.
+  const message = getMessageById(messageId);
+  if (!message?.metadata) {
+    return [];
+  }
+
+  try {
+    const parsed = messageMetadataSchema.safeParse(
+      JSON.parse(message.metadata),
+    );
+    const sourceMessageId =
+      parsed.success && typeof parsed.data.forkSourceMessageId === "string"
+        ? parsed.data.forkSourceMessageId
+        : null;
+    if (!sourceMessageId || sourceMessageId === messageId) {
+      return [];
+    }
+    const sourceTurnIds = getAssistantMessageIdsInTurn(sourceMessageId);
+    return selectLogsByMessageIds(sourceTurnIds);
+  } catch {
+    return [];
+  }
+}

package/src/memory/migrations/102-alter-table-columns.ts

@@ -261,6 +261,11 @@ export function addCoreColumns(database: DrizzleDb): void {
   } catch {
     /* already exists */
   }
+  try {
+    database.run(/*sql*/ `ALTER TABLE attachments ADD COLUMN source_path TEXT`);
+  } catch {
+    /* already exists */
+  }
 
   // cron_jobs
   try {

package/src/memory/migrations/146-schedule-oneshot-routing.ts

@@ -15,9 +15,9 @@ export function migrateScheduleOneShotRouting(database: DrizzleDb): void {
   const raw = getSqliteFrom(database);
 
   // Check if migration is already done by inspecting whether the status column exists
-  const tableInfo = raw
-    .query("PRAGMA table_info(cron_jobs)")
-    .all() as Array<{ name: string }>;
+  const tableInfo = raw.query("PRAGMA table_info(cron_jobs)").all() as Array<{
+    name: string;
+  }>;
   const hasStatusColumn = tableInfo.some((col) => col.name === "status");
   if (hasStatusColumn) {
     // Ensure all indexes exist even if the column migration already ran