@vellumai/assistant 0.5.16 → 0.6.1

package/src/daemon/conversation-history.ts

@@ -68,14 +68,13 @@ export function findLastUndoableUserMessageIndex(messages: Message[]): number {
 // ── Qdrant Vector Cleanup ────────────────────────────────────────────
 
 /**
- * Delete Qdrant vector entries for the given segment and item IDs.
+ * Delete Qdrant vector entries for the given segment IDs.
  * Individual deletion failures are logged and enqueued as retry jobs
  * to prevent silently orphaned vectors.
  */
 export async function cleanupQdrantVectors(
   conversationId: string,
   segmentIds: string[],
-  orphanedItemIds: string[],
 ): Promise<void> {
   let qdrant: ReturnType<typeof getQdrantClient>;
   try {
@@ -84,15 +83,12 @@ export async function cleanupQdrantVectors(
     return; // Qdrant not initialized — nothing to clean up.
   }
 
-  if (segmentIds.length === 0 && orphanedItemIds.length === 0) return;
+  if (segmentIds.length === 0) return;
 
   const targets: Array<{ targetType: string; targetId: string }> = [];
   for (const segId of segmentIds) {
     targets.push({ targetType: "segment", targetId: segId });
   }
-  for (const itemId of orphanedItemIds) {
-    targets.push({ targetType: "item", targetId: itemId });
-  }
 
   const results = await Promise.allSettled(
     targets.map((t) =>
@@ -124,7 +120,6 @@ export async function cleanupQdrantVectors(
         succeeded,
         failed,
         segments: segmentIds.length,
-        items: orphanedItemIds.length,
       },
       "Cleaned up Qdrant vectors after regenerate",
     );
@@ -187,20 +182,17 @@ export function consolidateAssistantMessages(
     // Still delete internal tool_result messages even if only one assistant message,
     // and collect IDs for vector cleanup
     const allSegmentIds: string[] = [];
-    const allOrphanedItemIds: string[] = [];
     for (const id of messagesToDelete) {
       const deleted = deleteMessageById(id);
       didMutate = true;
       allSegmentIds.push(...deleted.segmentIds);
-      allOrphanedItemIds.push(...deleted.orphanedItemIds);
     }
 
     // Clean up Qdrant vectors (fire-and-forget)
-    if (allSegmentIds.length > 0 || allOrphanedItemIds.length > 0) {
+    if (allSegmentIds.length > 0) {
       cleanupQdrantVectors(
         conversationId,
         allSegmentIds,
-        allOrphanedItemIds,
       ).catch((err) => {
         log.warn(
           { err, conversationId },
@@ -322,24 +314,20 @@ export function consolidateAssistantMessages(
   // Delete the other assistant messages and internal tool_result messages,
   // and collect IDs for vector cleanup
   const allSegmentIds: string[] = [];
-  const allOrphanedItemIds: string[] = [];
   for (let i = 1; i < messagesToConsolidate.length; i++) {
     const deleted = deleteMessageById(messagesToConsolidate[i].id);
     allSegmentIds.push(...deleted.segmentIds);
-    allOrphanedItemIds.push(...deleted.orphanedItemIds);
   }
   for (const id of messagesToDelete) {
     const deleted = deleteMessageById(id);
     allSegmentIds.push(...deleted.segmentIds);
-    allOrphanedItemIds.push(...deleted.orphanedItemIds);
   }
 
   // Clean up Qdrant vectors (fire-and-forget)
-  if (allSegmentIds.length > 0 || allOrphanedItemIds.length > 0) {
+  if (allSegmentIds.length > 0) {
     cleanupQdrantVectors(
       conversationId,
       allSegmentIds,
-      allOrphanedItemIds,
     ).catch((err) => {
       log.warn(
         { err, conversationId },
@@ -539,18 +527,15 @@ export async function regenerate(
 
   // Delete each message via deleteMessageById and collect IDs for Qdrant cleanup.
   const allSegmentIds: string[] = [];
-  const allOrphanedItemIds: string[] = [];
   for (const msg of messagesToDelete) {
     const deleted = deleteMessageById(msg.id);
     allSegmentIds.push(...deleted.segmentIds);
-    allOrphanedItemIds.push(...deleted.orphanedItemIds);
   }
 
   // Clean up Qdrant vectors (fire-and-forget).
   cleanupQdrantVectors(
     conversation.conversationId,
     allSegmentIds,
-    allOrphanedItemIds,
   ).catch((err) => {
     log.warn(
       { err, conversationId: conversation.conversationId },

package/src/daemon/conversation-lifecycle.ts

@@ -9,7 +9,6 @@ import type { EventBus } from "../events/bus.js";
 import type { AssistantDomainEvents } from "../events/domain-events.js";
 import type { ToolProfiler } from "../events/tool-profiling-listener.js";
 import { getHookManager } from "../hooks/manager.js";
-import { getMemoryCheckpoint } from "../memory/checkpoints.js";
 import {
   getConversation,
   getMessages,
@@ -150,6 +149,11 @@ export interface DisposeContext extends AbortContext {
   currentTurnSurfaces: Array<unknown>;
   lastSurfaceAction: Map<string, unknown>;
   workspaceTopLevelContext: string | null;
+  trustContext?: { trustClass: TrustClass };
+  /** Active memory node IDs snapshotted from the conversation's InContextTracker before disposal. */
+  activeContextNodeIds?: string[];
+  /** Memory scope for extraction — defaults to "default" if omitted. */
+  memoryScopeId?: string;
   abort(): void;
 }
 
@@ -199,6 +203,25 @@ export async function loadFromDb(ctx: LoadFromDbContext): Promise<void> {
 
       content = reinjectImageSourcePaths(content, role, m.metadata);
 
+      // Re-inject persisted memory block from metadata so it survives
+      // conversation reloads (eviction, restart, fork).
+      if (role === "user" && m.metadata) {
+        try {
+          const meta = JSON.parse(m.metadata);
+          if (typeof meta.memoryInjectedBlock === "string") {
+            content = [
+              {
+                type: "text" as const,
+                text: `<memory __injected>\n${meta.memoryInjectedBlock}\n</memory>`,
+              },
+              ...content,
+            ];
+          }
+        } catch {
+          /* ignore parse errors — metadata may be malformed */
+        }
+      }
+
       return { role, content };
     });
 
@@ -269,17 +292,21 @@ export function disposeConversation(ctx: DisposeContext): void {
     conversationId: ctx.conversationId,
   });
 
-  // Trigger batch extraction for any remaining unextracted messages
-  try {
-    const pendingKey = `batch_extract:${ctx.conversationId}:pending_count`;
-    const pending = getMemoryCheckpoint(pendingKey);
-    if (pending && parseInt(pending, 10) > 0) {
-      enqueueMemoryJob("batch_extract", {
+  // Trigger graph extraction for end-of-conversation sweep.
+  // Only extract from guardian conversations to preserve the memory trust
+  // boundary — untrusted content must not influence future memory retrieval.
+  if (!isUntrustedTrustClass(ctx.trustContext?.trustClass)) {
+    try {
+      enqueueMemoryJob("graph_extract", {
         conversationId: ctx.conversationId,
+        scopeId: ctx.memoryScopeId ?? "default",
+        ...(ctx.activeContextNodeIds?.length
+          ? { activeContextNodeIds: ctx.activeContextNodeIds }
+          : {}),
       });
+    } catch {
+      // Best-effort — don't block conversation disposal
     }
-  } catch {
-    // Best-effort — don't block conversation disposal
   }
 
   ctx.abort();

package/src/daemon/conversation-media-retry.ts

@@ -6,13 +6,25 @@
  * text stubs to shrink the payload before retrying.
  */
 
+import { estimateContentBlockTokens } from "../context/token-estimator.js";
 import { getSummaryFromContextMessage } from "../context/window-manager.js";
 import type { ContentBlock, Message } from "../providers/types.js";
 
+export interface StripMediaOptions {
+  /** Token budget available for media in the latest user message. Keeps as
+   *  many images as fit within this budget instead of the hardcoded limit. */
+  mediaTokenBudget?: number;
+  /** Provider name for per-image token estimation. */
+  providerName?: string;
+}
+
 const RETRY_KEEP_LATEST_MEDIA_BLOCKS = 3;
 const MAX_MEDIA_STUB_TEXT = 2_000;
 
-export function stripMediaPayloadsForRetry(messages: Message[]): {
+export function stripMediaPayloadsForRetry(
+  messages: Message[],
+  options?: StripMediaOptions,
+): {
   messages: Message[];
   modified: boolean;
   replacedBlocks: number;
@@ -31,6 +43,8 @@ export function stripMediaPayloadsForRetry(messages: Message[]): {
   let modified = false;
   let replacedBlocks = 0;
   let keptLatestMediaBlocks = 0;
+  let cumulativeMediaTokens = 0;
+  const useBudget = options?.mediaTokenBudget != null;
 
   const nextMessages = messages.map((msg, msgIndex) => {
     const nextContent: ContentBlock[] = [];
@@ -39,9 +53,20 @@ export function stripMediaPayloadsForRetry(messages: Message[]): {
       // few (in the most recent user message) and strip older ones so the
       // retry can actually reduce context size when images are the cause.
       if (block.type === "image") {
-        const keep =
-          latestUserIndex === msgIndex &&
-          keptLatestMediaBlocks < RETRY_KEEP_LATEST_MEDIA_BLOCKS;
+        let keep = false;
+        if (latestUserIndex === msgIndex) {
+          if (useBudget) {
+            const cost = estimateContentBlockTokens(block, {
+              providerName: options!.providerName,
+            });
+            if (cumulativeMediaTokens + cost <= options!.mediaTokenBudget!) {
+              cumulativeMediaTokens += cost;
+              keep = true;
+            }
+          } else {
+            keep = keptLatestMediaBlocks < RETRY_KEEP_LATEST_MEDIA_BLOCKS;
+          }
+        }
         if (keep) {
           keptLatestMediaBlocks += 1;
           nextContent.push(block);
@@ -54,9 +79,20 @@ export function stripMediaPayloadsForRetry(messages: Message[]): {
       }
 
       if (block.type === "file") {
-        const keep =
-          latestUserIndex === msgIndex &&
-          keptLatestMediaBlocks < RETRY_KEEP_LATEST_MEDIA_BLOCKS;
+        let keep = false;
+        if (latestUserIndex === msgIndex) {
+          if (useBudget) {
+            const cost = estimateContentBlockTokens(block, {
+              providerName: options!.providerName,
+            });
+            if (cumulativeMediaTokens + cost <= options!.mediaTokenBudget!) {
+              cumulativeMediaTokens += cost;
+              keep = true;
+            }
+          } else {
+            keep = keptLatestMediaBlocks < RETRY_KEEP_LATEST_MEDIA_BLOCKS;
+          }
+        }
         if (keep) {
           keptLatestMediaBlocks += 1;
           nextContent.push(block);
@@ -112,6 +148,48 @@ export function stripMediaPayloadsForRetry(messages: Message[]): {
   };
 }
 
+/**
+ * Estimate the total token cost of stubs that will replace unconditionally
+ * stubbed media blocks (non-latest-user-message images/files and all
+ * tool_result-nested media). This lets callers account for stub overhead
+ * when computing the available media token budget.
+ */
+export function estimateUnconditionalStubTokens(
+  messages: Message[],
+  options?: { providerName?: string },
+): number {
+  let latestUserIndex: number | null = null;
+  for (let i = messages.length - 1; i >= 0; i--) {
+    const msg = messages[i];
+    if (msg.role !== "user") continue;
+    if (isToolResultOnlyMessage(msg)) continue;
+    if (getSummaryFromContextMessage(msg) != null) continue;
+    latestUserIndex = i;
+    break;
+  }
+
+  let stubTokens = 0;
+  for (let msgIndex = 0; msgIndex < messages.length; msgIndex++) {
+    const msg = messages[msgIndex];
+    for (const block of msg.content) {
+      if (block.type === "image" && msgIndex !== latestUserIndex) {
+        stubTokens += estimateContentBlockTokens(imageBlockToStub(block), options);
+      } else if (block.type === "file" && msgIndex !== latestUserIndex) {
+        stubTokens += estimateContentBlockTokens(fileBlockToStub(block), options);
+      } else if (block.type === "tool_result" && block.contentBlocks) { // guard:allow-tool-result-only — web_search_tool_result has no contentBlocks
+        for (const cb of block.contentBlocks) {
+          if (cb.type === "image") {
+            stubTokens += estimateContentBlockTokens(imageBlockToStub(cb), options);
+          } else if (cb.type === "file") {
+            stubTokens += estimateContentBlockTokens(fileBlockToStub(cb), options);
+          }
+        }
+      }
+    }
+  }
+  return stubTokens;
+}
+
 function imageBlockToStub(
   block: Extract<ContentBlock, { type: "image" }>,
 ): Extract<ContentBlock, { type: "text" }> {

package/src/daemon/conversation-notifiers.ts

@@ -99,6 +99,7 @@ export function registerConversationNotifiers(
         ctx.sendToClient({
           type: "message_complete",
           conversationId: conversationId,
+          messageId: msg.id,
         });
       }
     },
@@ -133,6 +134,7 @@ export function registerConversationNotifiers(
         ctx.sendToClient({
           type: "message_complete",
           conversationId: conversationId,
+          messageId: msg.id,
         });
       }
     },
@@ -145,7 +147,7 @@ export function registerConversationNotifiers(
       const callee = callSession?.toNumber ?? "the caller";
       const questionText = `**Live call question** (to ${callee}):\n\n${question}\n\n_Use the call answer API to respond._`;
 
-      await addMessage(
+      const msg = await addMessage(
         conversationId,
         "assistant",
         JSON.stringify([{ type: "text", text: questionText }]),
@@ -168,6 +170,7 @@ export function registerConversationNotifiers(
       ctx.sendToClient({
         type: "message_complete",
         conversationId: conversationId,
+        messageId: msg.id,
       });
     },
   );

package/src/daemon/conversation-process.ts

@@ -54,7 +54,8 @@ export function formatCompactResult(result: ContextWindowResult): string {
   if (!result.compacted) {
     return `Context compaction skipped — ${result.reason ?? "nothing to compact"}.`;
   }
-  const saved = result.previousEstimatedInputTokens - result.estimatedInputTokens;
+  const saved =
+    result.previousEstimatedInputTokens - result.estimatedInputTokens;
   return [
     "Context Compacted\n",
     `Tokens:   ${fmt(result.previousEstimatedInputTokens)} → ${fmt(result.estimatedInputTokens)} (${fmt(saved)} saved)`,
@@ -295,13 +296,13 @@ export async function drainQueue(
   if (next.isInteractive === false) {
     conversation.clearProxyAvailability();
   } else {
-    // Restore proxy availability only for desktop-originating turns (macos/ios)
+    // Restore proxy availability only for desktop-originating turns (macos)
     // in case a prior non-interactive drain disabled it. Non-desktop interactive
     // interfaces (CLI, Vellum) should not re-enable desktop host proxies.
     const interfaceCtx =
       queuedInterfaceCtx ?? conversation.getTurnInterfaceContext();
     const sourceInterface = interfaceCtx?.userMessageInterface;
-    if (sourceInterface === "macos" || sourceInterface === "ios") {
+    if (sourceInterface === "macos") {
       conversation.restoreProxyAvailability();
       conversation.addPreactivatedSkillId("computer-use");
     }
@@ -310,7 +311,8 @@ export async function drainQueue(
   // Snapshot persona context at turn start so later tool turns can't pick up
   // a different actor's context if a concurrent request mutates the live fields.
   conversation.currentTurnTrustContext = conversation.trustContext;
-  conversation.currentTurnChannelCapabilities = conversation.channelCapabilities;
+  conversation.currentTurnChannelCapabilities =
+    conversation.channelCapabilities;
 
   // Resolve slash commands for queued messages
   const slashResult = await resolveSlash(
@@ -671,7 +673,8 @@ export async function processMessage(
   // Snapshot persona context at turn start so later tool turns can't pick up
   // a different actor's context if a concurrent request mutates the live fields.
   conversation.currentTurnTrustContext = conversation.trustContext;
-  conversation.currentTurnChannelCapabilities = conversation.channelCapabilities;
+  conversation.currentTurnChannelCapabilities =
+    conversation.channelCapabilities;
   conversation.currentActiveSurfaceId = activeSurfaceId;
   conversation.currentPage = currentPage;
   const trimmedContent = content.trim();
@@ -880,7 +883,9 @@ export async function processMessage(
     try {
       const pmTurnCtx = conversation.getTurnChannelContext();
       const pmInterfaceCtx = conversation.getTurnInterfaceContext();
-      const pmProvenance = provenanceFromTrustContext(conversation.trustContext);
+      const pmProvenance = provenanceFromTrustContext(
+        conversation.trustContext,
+      );
       const pmChannelMeta = {
         ...pmProvenance,
         ...(pmTurnCtx
@@ -892,7 +897,8 @@ export async function processMessage(
         ...(pmInterfaceCtx
           ? {
               userMessageInterface: pmInterfaceCtx.userMessageInterface,
-              assistantMessageInterface: pmInterfaceCtx.assistantMessageInterface,
+              assistantMessageInterface:
+                pmInterfaceCtx.assistantMessageInterface,
             }
           : {}),
       };