@vellumai/assistant 0.5.16 → 0.6.0

package/src/daemon/conversation-media-retry.ts

@@ -6,13 +6,25 @@
  * text stubs to shrink the payload before retrying.
  */
 
+import { estimateContentBlockTokens } from "../context/token-estimator.js";
 import { getSummaryFromContextMessage } from "../context/window-manager.js";
 import type { ContentBlock, Message } from "../providers/types.js";
 
+export interface StripMediaOptions {
+  /** Token budget available for media in the latest user message. Keeps as
+   *  many images as fit within this budget instead of the hardcoded limit. */
+  mediaTokenBudget?: number;
+  /** Provider name for per-image token estimation. */
+  providerName?: string;
+}
+
 const RETRY_KEEP_LATEST_MEDIA_BLOCKS = 3;
 const MAX_MEDIA_STUB_TEXT = 2_000;
 
-export function stripMediaPayloadsForRetry(messages: Message[]): {
+export function stripMediaPayloadsForRetry(
+  messages: Message[],
+  options?: StripMediaOptions,
+): {
   messages: Message[];
   modified: boolean;
   replacedBlocks: number;
@@ -31,6 +43,8 @@ export function stripMediaPayloadsForRetry(messages: Message[]): {
   let modified = false;
   let replacedBlocks = 0;
   let keptLatestMediaBlocks = 0;
+  let cumulativeMediaTokens = 0;
+  const useBudget = options?.mediaTokenBudget != null;
 
   const nextMessages = messages.map((msg, msgIndex) => {
     const nextContent: ContentBlock[] = [];
@@ -39,9 +53,20 @@ export function stripMediaPayloadsForRetry(messages: Message[]): {
       // few (in the most recent user message) and strip older ones so the
       // retry can actually reduce context size when images are the cause.
       if (block.type === "image") {
-        const keep =
-          latestUserIndex === msgIndex &&
-          keptLatestMediaBlocks < RETRY_KEEP_LATEST_MEDIA_BLOCKS;
+        let keep = false;
+        if (latestUserIndex === msgIndex) {
+          if (useBudget) {
+            const cost = estimateContentBlockTokens(block, {
+              providerName: options!.providerName,
+            });
+            if (cumulativeMediaTokens + cost <= options!.mediaTokenBudget!) {
+              cumulativeMediaTokens += cost;
+              keep = true;
+            }
+          } else {
+            keep = keptLatestMediaBlocks < RETRY_KEEP_LATEST_MEDIA_BLOCKS;
+          }
+        }
         if (keep) {
           keptLatestMediaBlocks += 1;
           nextContent.push(block);
@@ -54,9 +79,20 @@ export function stripMediaPayloadsForRetry(messages: Message[]): {
       }
 
       if (block.type === "file") {
-        const keep =
-          latestUserIndex === msgIndex &&
-          keptLatestMediaBlocks < RETRY_KEEP_LATEST_MEDIA_BLOCKS;
+        let keep = false;
+        if (latestUserIndex === msgIndex) {
+          if (useBudget) {
+            const cost = estimateContentBlockTokens(block, {
+              providerName: options!.providerName,
+            });
+            if (cumulativeMediaTokens + cost <= options!.mediaTokenBudget!) {
+              cumulativeMediaTokens += cost;
+              keep = true;
+            }
+          } else {
+            keep = keptLatestMediaBlocks < RETRY_KEEP_LATEST_MEDIA_BLOCKS;
+          }
+        }
         if (keep) {
           keptLatestMediaBlocks += 1;
           nextContent.push(block);
@@ -112,6 +148,48 @@ export function stripMediaPayloadsForRetry(messages: Message[]): {
   };
 }
 
+/**
+ * Estimate the total token cost of stubs that will replace unconditionally
+ * stubbed media blocks (non-latest-user-message images/files and all
+ * tool_result-nested media). This lets callers account for stub overhead
+ * when computing the available media token budget.
+ */
+export function estimateUnconditionalStubTokens(
+  messages: Message[],
+  options?: { providerName?: string },
+): number {
+  let latestUserIndex: number | null = null;
+  for (let i = messages.length - 1; i >= 0; i--) {
+    const msg = messages[i];
+    if (msg.role !== "user") continue;
+    if (isToolResultOnlyMessage(msg)) continue;
+    if (getSummaryFromContextMessage(msg) != null) continue;
+    latestUserIndex = i;
+    break;
+  }
+
+  let stubTokens = 0;
+  for (let msgIndex = 0; msgIndex < messages.length; msgIndex++) {
+    const msg = messages[msgIndex];
+    for (const block of msg.content) {
+      if (block.type === "image" && msgIndex !== latestUserIndex) {
+        stubTokens += estimateContentBlockTokens(imageBlockToStub(block), options);
+      } else if (block.type === "file" && msgIndex !== latestUserIndex) {
+        stubTokens += estimateContentBlockTokens(fileBlockToStub(block), options);
+      } else if (block.type === "tool_result" && block.contentBlocks) { // guard:allow-tool-result-only — web_search_tool_result has no contentBlocks
+        for (const cb of block.contentBlocks) {
+          if (cb.type === "image") {
+            stubTokens += estimateContentBlockTokens(imageBlockToStub(cb), options);
+          } else if (cb.type === "file") {
+            stubTokens += estimateContentBlockTokens(fileBlockToStub(cb), options);
+          }
+        }
+      }
+    }
+  }
+  return stubTokens;
+}
+
 function imageBlockToStub(
   block: Extract<ContentBlock, { type: "image" }>,
 ): Extract<ContentBlock, { type: "text" }> {

package/src/daemon/conversation-notifiers.ts

@@ -99,6 +99,7 @@ export function registerConversationNotifiers(
         ctx.sendToClient({
           type: "message_complete",
           conversationId: conversationId,
+          messageId: msg.id,
         });
       }
     },
@@ -133,6 +134,7 @@ export function registerConversationNotifiers(
         ctx.sendToClient({
           type: "message_complete",
           conversationId: conversationId,
+          messageId: msg.id,
         });
       }
     },
@@ -145,7 +147,7 @@ export function registerConversationNotifiers(
       const callee = callSession?.toNumber ?? "the caller";
       const questionText = `**Live call question** (to ${callee}):\n\n${question}\n\n_Use the call answer API to respond._`;
 
-      await addMessage(
+      const msg = await addMessage(
         conversationId,
         "assistant",
         JSON.stringify([{ type: "text", text: questionText }]),
@@ -168,6 +170,7 @@ export function registerConversationNotifiers(
       ctx.sendToClient({
         type: "message_complete",
         conversationId: conversationId,
+        messageId: msg.id,
       });
     },
   );

package/src/daemon/conversation-runtime-assembly.ts

@@ -1049,6 +1049,11 @@ const RUNTIME_INJECTION_PREFIXES = [
   "<turn_context>",
   "<memory_context __injected>",
   "<memory_context>", // backward-compat: strip legacy blocks from pre-__injected history
+  // NOTE: <memory __injected> is intentionally NOT stripped — memory
+  // injections persist in history so the assistant can reference them.
+  // Context compaction handles these blocks during history reduction, and
+  // the InContextTracker deduplicates nodes across turns, so accumulation
+  // does not cause unbounded context growth.
   "<voice_call_control>",
   "<workspace_top_level>",
   TEMPORAL_INJECTED_PREFIX,

package/src/daemon/conversation.ts

@@ -44,6 +44,7 @@ import { registerToolTraceListener } from "../events/tool-trace-listener.js";
 import { getHookManager } from "../hooks/manager.js";
 import { resolveCanonicalGuardianRequest } from "../memory/canonical-guardian-store.js";
 import { updateConversationContextWindow } from "../memory/conversation-crud.js";
+import { ConversationGraphMemory } from "../memory/graph/conversation-graph-memory.js";
 import { PermissionPrompter } from "../permissions/prompter.js";
 import { SecretPrompter } from "../permissions/secret-prompter.js";
 import { patternMatchesCandidate } from "../permissions/trust-store.js";
@@ -57,6 +58,7 @@ import type { AuthContext } from "../runtime/auth/types.js";
 import * as approvalOverrides from "../runtime/conversation-approval-overrides.js";
 import * as pendingInteractions from "../runtime/pending-interactions.js";
 import { ToolExecutor } from "../tools/executor.js";
+import { getLogger } from "../util/logger.js";
 import type { AssistantAttachmentDraft } from "./assistant-attachments.js";
 import { runAgentLoopImpl } from "./conversation-agent-loop.js";
 import type { HistoryConversationContext } from "./conversation-history.js";
@@ -118,6 +120,8 @@ import type {
 } from "./message-types/messages.js";
 import { TraceEmitter } from "./trace-emitter.js";
 
+const log = getLogger("conversation");
+
 export interface ConversationMemoryPolicy {
   scopeId: string;
   includeDefaultFallback: boolean;
@@ -243,6 +247,7 @@ export class Conversation {
   public readonly traceEmitter: TraceEmitter;
   public readonly hasSystemPromptOverride: boolean;
   public memoryPolicy: ConversationMemoryPolicy;
+  /** @internal */ readonly graphMemory: ConversationGraphMemory;
   /** @internal */ streamThinking: boolean;
   /** @internal */ turnCount = 0;
   public lastAssistantAttachments: AssistantAttachmentDraft[] = [];
@@ -251,6 +256,8 @@ export class Conversation {
   /** @internal */ currentTurnInterfaceContext: TurnInterfaceContext | null =
     null;
   /** @internal */ activityVersion = 0;
+  /** Last emitted activity state message, retained for replay on SSE reconnection. */
+  /** @internal */ lastActivityStateMsg: ServerMessage | null = null;
   /** Set by the agent loop to track confirmation outcomes for persistence. */
   onConfirmationOutcome?: (
     requestId: string,
@@ -280,6 +287,10 @@ export class Conversation {
     this.memoryPolicy = memoryPolicy
       ? { ...memoryPolicy }
       : { ...DEFAULT_MEMORY_POLICY };
+    this.graphMemory = new ConversationGraphMemory(
+      this.memoryPolicy.scopeId,
+      conversationId,
+    );
     this.traceEmitter = new TraceEmitter(conversationId, sendToClient);
     this.prompter = new PermissionPrompter(sendToClient);
     this.prompter.setOnStateChanged((requestId, state, source, toolUseId) => {
@@ -486,6 +497,19 @@ export class Conversation {
       this.hostCuProxy?.updateSender(sendToClient, !hasNoClient);
       this.hostFileProxy?.updateSender(sendToClient, !hasNoClient);
     }
+
+    // Replay last activity state so a reconnecting client sees the current phase
+    // instead of being stuck on the last state it received before disconnection.
+    if (!hasNoClient && this.lastActivityStateMsg) {
+      try {
+        sendToClient(this.lastActivityStateMsg);
+      } catch (err) {
+        log.warn(
+          { err, conversationId: this.conversationId },
+          "Failed to replay activity state on client reconnection",
+        );
+      }
+    }
   }
 
   /** Returns the current sendToClient reference for identity comparison. */
@@ -862,7 +886,14 @@ export class Conversation {
       type: "confirmation_state_changed",
       ...params,
     } as ServerMessage;
-    this.sendToClient(msg);
+    try {
+      this.sendToClient(msg);
+    } catch (err) {
+      log.warn(
+        { err, conversationId: this.conversationId },
+        "sendToClient threw in emitConfirmationStateChanged",
+      );
+    }
   }
 
   emitActivityState(
@@ -883,7 +914,15 @@ export class Conversation {
       reason,
       ...(statusText ? { statusText } : {}),
     } as ServerMessage;
-    this.sendToClient(msg);
+    this.lastActivityStateMsg = msg;
+    try {
+      this.sendToClient(msg);
+    } catch (err) {
+      log.warn(
+        { err, conversationId: this.conversationId },
+        "sendToClient threw in emitActivityState",
+      );
+    }
   }
 
   async forceCompact(): Promise<ContextWindowResult> {

package/src/daemon/daemon-control.ts

@@ -12,12 +12,14 @@ import { resolve } from "node:path";
 import { getRuntimeHttpHost, getRuntimeHttpPort } from "../config/env.js";
 import { getIsContainerized } from "../config/env-registry.js";
 import { loadOrCreateSigningKey } from "../runtime/auth/token-service.js";
+import { ensureBun } from "../util/bun-runtime.js";
 import { DaemonError } from "../util/errors.js";
 import { getLogger } from "../util/logger.js";
 import {
   ensureDataDir,
   getDaemonStartupLockPath,
   getDaemonStderrLogPath,
+  getDataDir,
   getPidPath,
   getWorkspaceConfigPath,
 } from "../util/platform.js";
@@ -371,6 +373,10 @@ async function startDaemonLocked(): Promise<{
     }
   }
 
+  // Resolve bun before opening stderrFd to avoid leaking the file
+  // descriptor if ensureBun() throws (same pattern as loadOrCreateSigningKey).
+  const bunPath = await ensureBun();
+
   // Redirect the child's stderr to a file instead of piping it back to the
   // parent. A pipe's read end is destroyed when the parent exits, leaving
   // fd 2 broken in the child. Bun (unlike Node.js) does not ignore SIGPIPE,
@@ -378,7 +384,7 @@ async function startDaemonLocked(): Promise<{
   const stderrPath = getDaemonStderrLogPath();
   const stderrFd = openSync(stderrPath, "w");
 
-  const child = spawn("bun", ["run", mainPath], {
+  const child = spawn(bunPath, ["run", mainPath], {
     detached: true,
     stdio: ["ignore", "ignore", stderrFd],
     env: spawnEnv,
@@ -413,7 +419,7 @@ async function startDaemonLocked(): Promise<{
       const stderr = readFileSync(stderrPath, "utf-8").trim();
       const detail = stderr
         ? `\n${stderr}`
-        : `\nCheck logs at ~/.vellum/workspace/data/logs/ for details.`;
+        : `\nCheck logs at ${getDataDir()}/logs/ for details.`;
       throw new DaemonError(
         `Daemon exited immediately (code ${
           childExitCode ?? "unknown"

package/src/daemon/handlers/shared.ts

@@ -60,8 +60,10 @@ export interface HistoryToolCall {
   input: Record<string, unknown>;
   result?: string;
   isError?: boolean;
-  /** Base64-encoded image data from tool contentBlocks (e.g. browser_screenshot). */
+  /** Base64-encoded image data from tool contentBlocks (e.g. browser_screenshot). @deprecated Use imageDataList. */
   imageData?: string;
+  /** Base64-encoded image data from tool contentBlocks (e.g. browser_screenshot, image generation). */
+  imageDataList?: string[];
   /** Unix ms when the tool started executing. */
   startedAt?: number;
   /** Unix ms when the tool completed. */
@@ -341,16 +343,19 @@ export function renderHistoryContent(content: unknown): RenderedHistoryContent {
       const resultContent =
         typeof block.content === "string" ? block.content : "";
       const isError = block.is_error === true;
-      // Extract base64 image data from persisted contentBlocks (e.g. browser_screenshot)
-      let imageData: string | undefined;
+      // Extract base64 image data from persisted contentBlocks (e.g. browser_screenshot, image generation)
+      const imageDataList: string[] = [];
       if (Array.isArray(block.contentBlocks)) {
-        const imgBlock = block.contentBlocks.find(
-          (b: Record<string, unknown>) => isRecord(b) && b.type === "image",
-        );
-        if (imgBlock && isRecord(imgBlock) && isRecord(imgBlock.source)) {
-          const src = imgBlock.source as Record<string, unknown>;
-          if (typeof src.data === "string") {
-            imageData = src.data;
+        for (const cb of block.contentBlocks) {
+          if (
+            isRecord(cb) &&
+            cb.type === "image" &&
+            isRecord(cb.source) &&
+            typeof (cb.source as Record<string, unknown>).data === "string"
+          ) {
+            imageDataList.push(
+              (cb.source as Record<string, unknown>).data as string,
+            );
           }
         }
       }
@@ -358,14 +363,19 @@ export function renderHistoryContent(content: unknown): RenderedHistoryContent {
       if (matched) {
         matched.result = resultContent;
         matched.isError = isError;
-        if (imageData) matched.imageData = imageData;
+        if (imageDataList.length > 0) {
+          matched.imageData = imageDataList[0];
+          matched.imageDataList = imageDataList;
+        }
       } else {
         toolCalls.push({
           name: "unknown",
           input: {},
           result: resultContent,
           isError,
-          ...(imageData ? { imageData } : {}),
+          ...(imageDataList.length > 0
+            ? { imageData: imageDataList[0], imageDataList }
+            : {}),
         });
       }
       continue;