@vellumai/assistant 0.10.7 → 0.10.8-dev.202607102228.5945895

package/src/context/compactor.ts

@@ -170,6 +170,65 @@ active decisions, open questions, commitments, project states.
 </compaction_result>
 </compaction_instructions>`;
 
+/**
+ * Fixed-boundary compaction instruction. Used when the caller pins the cut
+ * via `fixedTailStartIndex` — the user chose where the summary ends, so the
+ * model writes the summary but does not pick a `tail_start`.
+ *
+ * Interpolation points: `{image_manifest}` (same as the default prompt) and
+ * `{boundary_description}` (a description of the first preserved message so
+ * the model knows exactly where "before" ends).
+ */
+const FIXED_BOUNDARY_COMPACTION_PROMPT = `<compaction_instructions>
+The user has asked to summarize this conversation up to a fixed point they
+chose. Everything BEFORE that point is replaced by your summary; the boundary
+message and everything after it are preserved verbatim. You do not choose the
+cut — it is already fixed.
+
+The first message to be preserved verbatim is {boundary_description}.
+Summarize ONLY what comes before it.
+
+Write the summary in YOUR voice — as if you're remembering this conversation,
+not writing meeting notes about it. Prioritize:
+- Decisions made and commitments given
+- Key context that's still relevant going forward
+- Emotional moments that shaped the conversation's direction
+- Exact quotes when the specific wording matters
+- Project/task state changes
+
+Compress aggressively:
+- Repeated debugging or troubleshooting attempts → just the outcome
+- Tool call outputs → results only, not raw data
+- Intermediate states superseded by later states
+- Back-and-forth deliberation → just the conclusion
+
+IMAGE MANIFEST (images in this conversation):
+{image_manifest}
+
+If any images from the summarized portion are still relevant to the
+ongoing conversation, include them in retained_images by filename.
+
+Output your result in this exact format:
+
+<compaction_result>
+<summary>
+Your summary in your voice. Aim for 2000-4000 tokens — rich enough
+to preserve what matters, compact enough to free real space.
+</summary>
+
+<key_state>
+Short structured list of anything PENDING from this conversation:
+active decisions, open questions, commitments, project states.
+</key_state>
+
+<retained_images>
+<image file="filename.ext" />
+(only images from BEFORE the boundary that are still contextually important)
+(omit this section entirely if no images need retention)
+</retained_images>
+</compaction_result>
+</compaction_instructions>`;
+
 // ---------------------------------------------------------------------------
 // Public types
 // ---------------------------------------------------------------------------
@@ -211,6 +270,16 @@ export interface CompactionRunArgs {
    * DB counterparts.
    */
   nonPersistedPrefixCount?: number;
+  /**
+   * Index into `messages` of the first message to KEEP verbatim. When set,
+   * everything before it is summarized and the model does not choose the
+   * cut: the `<tail_start>` output contract, tail resolution, and the
+   * token-budget forward-cut are all skipped. The boundary is authoritative
+   * (the caller has already turn-snapped it); only the tool-pairing
+   * back-walk still applies, as defense-in-depth. Must be an integer in
+   * `[1, messages.length)` — otherwise the run returns `compacted: false`.
+   */
+  fixedTailStartIndex?: number;
 }
 
 export interface CompactionRunResult {
@@ -277,10 +346,14 @@ export interface ParsedCompactionResult {
  * Lenient by design — the model may wrap the block in narration, may omit
  * `<retained_images>`, and may produce slightly malformed inner tags. We
  * accept any of those. Returns `null` only when the required fields
- * (summary + tail_start.timestamp) are missing.
+ * (summary + tail_start.timestamp) are missing. With
+ * `requireTailStart: false` (the fixed-boundary mode, where the caller
+ * already owns the cut) an absent `<tail_start>` is tolerated and its
+ * fields come back empty.
  */
 export function parseCompactionResult(
   raw: string,
+  opts: { requireTailStart?: boolean } = {},
 ): ParsedCompactionResult | null {
   const openIdx = raw.indexOf(RESULT_TAG_OPEN);
   if (openIdx < 0) return null;
@@ -296,7 +369,12 @@ export function parseCompactionResult(
   const keyState = extractTagContent(inner, "key_state")?.trim() ?? "";
 
   const tail = extractTailStart(inner);
-  if (!tail || tail.timestamp.length === 0) return null;
+  if (
+    opts.requireTailStart !== false &&
+    (!tail || tail.timestamp.length === 0)
+  ) {
+    return null;
+  }
 
   const retainedImageFilenames = extractRetainedImages(inner);
 
@@ -304,8 +382,8 @@ export function parseCompactionResult(
     summary,
     keyState,
     retainedImageFilenames,
-    tailStartTimestamp: tail.timestamp,
-    tailStartPreview: tail.preview,
+    tailStartTimestamp: tail?.timestamp ?? "",
+    tailStartPreview: tail?.preview ?? "",
   };
 }
 
@@ -468,6 +546,23 @@ function extractFirstTextPreview(message: Message, maxChars = 120): string {
   return "";
 }
 
+/**
+ * Human-readable description of the first preserved message for the
+ * fixed-boundary instruction, so the model knows exactly where "before"
+ * ends: the message's `<turn_context>` timestamp when available, else a
+ * short preview of its text.
+ */
+function describeFixedBoundary(message: Message): string {
+  const timestamp = extractTurnContextTimestamp(message);
+  if (timestamp) {
+    return `the message whose turn_context timestamp is "${timestamp}"`;
+  }
+  const preview = extractFirstTextPreview(message, 80);
+  return preview.length > 0
+    ? `the ${message.role} message beginning: "${preview}"`
+    : `the first preserved ${message.role} message`;
+}
+
 // ---------------------------------------------------------------------------
 // Tail resolution
 // ---------------------------------------------------------------------------
@@ -749,7 +844,21 @@ export function buildInstructionMessage(
   customPrompt: string | null | undefined,
   imageManifest: string,
   tailBudgetTokens?: number,
+  fixedBoundaryDescription?: string,
 ): Message {
+  // Fixed-boundary mode always uses the fixed variant — custom prompts are
+  // authored against the tail-choosing `<tail_start>` contract, which does
+  // not apply when the caller pins the cut.
+  if (fixedBoundaryDescription != null) {
+    const text = FIXED_BOUNDARY_COMPACTION_PROMPT.replace(
+      "{image_manifest}",
+      imageManifest,
+    ).replace("{boundary_description}", fixedBoundaryDescription);
+    return {
+      role: "user",
+      content: [{ type: "text", text }],
+    };
+  }
   const template =
     customPrompt && customPrompt.trim().length > 0
       ? customPrompt
@@ -774,6 +883,39 @@ export function buildInstructionMessage(
 // Summary message construction
 // ---------------------------------------------------------------------------
 
+export const CONTEXT_SUMMARY_MARKER = "<context_summary>";
+export const CONTEXT_SUMMARY_CLOSE = "</context_summary>";
+
+/**
+ * Wrap durable summary text in the `<context_summary>` tags that identify a
+ * synthetic summary head. Every summary head in a live history carries this
+ * wrapper — the reload/fork rehydration path (`createContextSummaryMessage`
+ * in the compaction plugin) and the compactor-minted heads below — so
+ * persisted-count accounting can recognize a head structurally even after
+ * history repair rebuilds message wrappers and WeakSet identity is lost.
+ * The durable `summaryText` result field stays unwrapped; rehydration adds
+ * the wrapper back when it rebuilds the head from the DB column.
+ */
+export function wrapContextSummaryText(summary: string): string {
+  return `${CONTEXT_SUMMARY_MARKER}\n${summary}\n${CONTEXT_SUMMARY_CLOSE}`;
+}
+
+/**
+ * Synthetic messages a compaction pass prepends to the rebuilt history —
+ * the summary head and the retained-images message. They have no DB row,
+ * so the persisted-count accounting of a later pass over the same
+ * in-memory history must exclude them (see
+ * `CompactionRunArgs.nonPersistedPrefixCount`). WeakSet-gated: only the
+ * exact objects minted here are recognized, never lookalike content.
+ */
+const SYNTHETIC_COMPACTION_MESSAGES = new WeakSet<Message>();
+
+export function isSyntheticCompactionMessage(
+  message: Message | undefined,
+): boolean {
+  return message != null && SYNTHETIC_COMPACTION_MESSAGES.has(message);
+}
+
 /**
  * Stitch summary + key_state into the assistant-role memory message that
  * heads the compacted context. Kept as a single block so downstream
@@ -933,6 +1075,24 @@ export async function runAssistantDrivenCompaction(
     return emptyResult(args, thresholdTokens, "no messages to compact");
   }
 
+  // A caller-fixed boundary must land strictly inside the array: index 0
+  // would leave nothing to summarize, and anything past the end preserves
+  // nothing. Validated before the provider call so an invalid index never
+  // burns a full-context summary pass.
+  const fixedTailStartIndex = args.fixedTailStartIndex;
+  if (
+    fixedTailStartIndex != null &&
+    (!Number.isInteger(fixedTailStartIndex) ||
+      fixedTailStartIndex < 1 ||
+      fixedTailStartIndex >= args.messages.length)
+  ) {
+    log.warn(
+      { fixedTailStartIndex, messageCount: args.messages.length },
+      "Fixed compaction boundary out of range — skipping compaction",
+    );
+    return emptyResult(args, thresholdTokens, "fixed boundary out of range");
+  }
+
   // Build image manifest from the DB before invoking the model so the
   // instruction message carries a faithful picture of available images.
   // Filtered by actor trust so untrusted turns never see guardian-only
@@ -946,6 +1106,9 @@ export async function runAssistantDrivenCompaction(
     args.compaction.prompt ?? null,
     manifestText,
     args.targetTokens,
+    fixedTailStartIndex != null
+      ? describeFixedBoundary(args.messages[fixedTailStartIndex])
+      : undefined,
   );
 
   // Bound the summary call's own input to the context window. With no tool
@@ -999,7 +1162,9 @@ export async function runAssistantDrivenCompaction(
   recordCompactionRequestLog(args.conversationId, response, args.provider);
 
   const rawText = extractTextFromResponse(response.content);
-  const parsed = parseCompactionResult(rawText);
+  const parsed = parseCompactionResult(rawText, {
+    requireTailStart: fixedTailStartIndex == null,
+  });
   if (!parsed) {
     log.warn(
       { rawPreview: rawText.slice(0, 200) },
@@ -1021,34 +1186,43 @@ export async function runAssistantDrivenCompaction(
     };
   }
 
-  const timestamps = buildTimestampIndex(args.messages);
-  const resolvedTailIndex = resolveTailStartIndex(
-    args.messages,
-    timestamps,
-    parsed,
-  );
-  if (resolvedTailIndex == null) {
-    log.warn(
-      {
-        timestamp: parsed.tailStartTimestamp,
-        preview: parsed.tailStartPreview.slice(0, 60),
-      },
-      "Compaction tail_start did not match any message — aborting compaction",
+  // The caller's fixed boundary is authoritative — any `<tail_start>` the
+  // model emitted anyway is ignored. Without a fixed boundary, resolve the
+  // model's choice against the live messages.
+  let resolvedTailIndex: number;
+  if (fixedTailStartIndex != null) {
+    resolvedTailIndex = fixedTailStartIndex;
+  } else {
+    const timestamps = buildTimestampIndex(args.messages);
+    const modelTailIndex = resolveTailStartIndex(
+      args.messages,
+      timestamps,
+      parsed,
     );
-    return {
-      ...emptyResult(args, thresholdTokens, "tail_start unresolved"),
-      summaryFailed: false,
-      summaryInputTokens: response.usage.inputTokens,
-      summaryOutputTokens: response.usage.outputTokens,
-      summaryModel: response.model,
-      summaryCacheCreationInputTokens:
-        response.usage.cacheCreationInputTokens ?? 0,
-      summaryCacheReadInputTokens: response.usage.cacheReadInputTokens ?? 0,
-      summaryCallSite: COMPACTION_CALL_SITE,
-      summaryOverrideProfile: args.overrideProfile ?? null,
-      summaryRawResponses: response.rawResponse ? [response.rawResponse] : [],
-      summaryCalls: 1,
-    };
+    if (modelTailIndex == null) {
+      log.warn(
+        {
+          timestamp: parsed.tailStartTimestamp,
+          preview: parsed.tailStartPreview.slice(0, 60),
+        },
+        "Compaction tail_start did not match any message — aborting compaction",
+      );
+      return {
+        ...emptyResult(args, thresholdTokens, "tail_start unresolved"),
+        summaryFailed: false,
+        summaryInputTokens: response.usage.inputTokens,
+        summaryOutputTokens: response.usage.outputTokens,
+        summaryModel: response.model,
+        summaryCacheCreationInputTokens:
+          response.usage.cacheCreationInputTokens ?? 0,
+        summaryCacheReadInputTokens: response.usage.cacheReadInputTokens ?? 0,
+        summaryCallSite: COMPACTION_CALL_SITE,
+        summaryOverrideProfile: args.overrideProfile ?? null,
+        summaryRawResponses: response.rawResponse ? [response.rawResponse] : [],
+        summaryCalls: 1,
+      };
+    }
+    resolvedTailIndex = modelTailIndex;
   }
 
   const pairedTailIndex = adjustTailIndexForToolPairing(
@@ -1076,7 +1250,7 @@ export async function runAssistantDrivenCompaction(
   let finalSummaryText = summaryText;
   let summaryMessage: Message = {
     role: "assistant",
-    content: [{ type: "text", text: finalSummaryText }],
+    content: [{ type: "text", text: wrapContextSummaryText(finalSummaryText) }],
   };
 
   const {
@@ -1116,6 +1290,10 @@ export async function runAssistantDrivenCompaction(
   // between the model's cut and the enforced cut is acknowledged with an
   // explicit truncation notice appended to the summary message (see below),
   // so the loss is visible in-context rather than silent.
+  //
+  // A caller-fixed boundary skips the enforcement entirely: the user's cut is
+  // authoritative and no token-budget forward-cut applies, however large the
+  // preserved tail.
   let tailIndex = pairedTailIndex;
   // Whether the deterministic forward-cut hit the tail floor while still over
   // `targetTokens` — propagated onto the success result so the window-manager's
@@ -1123,7 +1301,11 @@ export async function runAssistantDrivenCompaction(
   // next time, so it cannot do better). Only meaningful when a `targetTokens`
   // budget drove the forward-cut.
   let tailFloorReached = false;
-  if (args.targetTokens != null && pairedTailIndex > 0) {
+  if (
+    fixedTailStartIndex == null &&
+    args.targetTokens != null &&
+    pairedTailIndex > 0
+  ) {
     const providerName =
       args.provider.tokenEstimationProvider ?? args.provider.name;
     // Mirror the window-manager's post-compaction estimate (system prompt +
@@ -1169,7 +1351,9 @@ export async function runAssistantDrivenCompaction(
       finalSummaryText = summaryText + truncationNote;
       summaryMessage = {
         role: "assistant",
-        content: [{ type: "text", text: finalSummaryText }],
+        content: [
+          { type: "text", text: wrapContextSummaryText(finalSummaryText) },
+        ],
       };
       log.info(
         {
@@ -1221,7 +1405,11 @@ export async function runAssistantDrivenCompaction(
   );
 
   const compactedMessages: Message[] = [summaryMessage];
-  if (retainedImageMessage) compactedMessages.push(retainedImageMessage);
+  SYNTHETIC_COMPACTION_MESSAGES.add(summaryMessage);
+  if (retainedImageMessage) {
+    compactedMessages.push(retainedImageMessage);
+    SYNTHETIC_COMPACTION_MESSAGES.add(retainedImageMessage);
+  }
   compactedMessages.push(...tailMessages);
 
   const nonPersistedCompactedAway = Math.min(
@@ -1430,7 +1618,10 @@ export async function runEmergencyCompaction(
   recordCompactionRequestLog(args.conversationId, response, args.provider);
 
   const rawText = extractTextFromResponse(response.content);
-  const parsed = parseCompactionResult(rawText);
+  // The emergency prompt asks for summary + key_state only — the split
+  // point is already fixed at `splitIndex`, so no `<tail_start>` is
+  // requested and a prompt-following response must still parse.
+  const parsed = parseCompactionResult(rawText, { requireTailStart: false });
   if (!parsed) {
     log.warn(
       { rawPreview: rawText.slice(0, 200) },
@@ -1449,10 +1640,11 @@ export async function runEmergencyCompaction(
   const summaryText = buildSummaryMemoryText(parsed.summary, parsed.keyState);
   const summaryMessage: Message = {
     role: "assistant",
-    content: [{ type: "text", text: summaryText }],
+    content: [{ type: "text", text: wrapContextSummaryText(summaryText) }],
   };
 
   const compactedMessages: Message[] = [summaryMessage, ...keptTail];
+  SYNTHETIC_COMPACTION_MESSAGES.add(summaryMessage);
 
   const compactedCount = splitIndex;
   const nonPersistedAway = Math.min(

package/src/context/image-dimensions.ts

@@ -1,12 +1,34 @@
+import type { MediaSource } from "../providers/types.js";
+
 /**
- * Parses image dimensions from base64-encoded image data by reading binary headers.
- * Supports PNG, JPEG, GIF, and WebP formats.
- * Returns null if parsing fails for any reason (corrupt, truncated, unrecognized).
+ * Parse image pixel dimensions, either from a base64 payload (reading binary
+ * headers — PNG, JPEG, GIF, WebP) or from an image block's media `source`.
+ *
+ * For a `workspace_ref` source the dimensions come from the hints captured at
+ * persist time (no disk read); for an inline `base64` source they are parsed
+ * from the header. Returns null when parsing fails or the reference carries no
+ * dimension hints.
  */
 export function parseImageDimensions(
   base64Data: string,
   mediaType: string,
+): { width: number; height: number } | null;
+export function parseImageDimensions(
+  source: MediaSource,
+): { width: number; height: number } | null;
+export function parseImageDimensions(
+  arg: string | MediaSource,
+  mediaType?: string,
 ): { width: number; height: number } | null {
+  if (typeof arg !== "string") {
+    if (arg.type === "workspace_ref") {
+      return arg.width != null && arg.height != null
+        ? { width: arg.width, height: arg.height }
+        : null;
+    }
+    return parseImageDimensions(arg.data, arg.media_type);
+  }
+  const base64Data = arg;
   try {
     switch (mediaType) {
       case "image/png":

package/src/context/token-estimator.ts

@@ -2,6 +2,7 @@ import {
   estimateGeminiAudioTokens,
   normalizeGeminiAudioMime,
 } from "../providers/gemini/inline-media.js";
+import { mediaSourceByteLength } from "../providers/media-resolve.js";
 import type {
   ContentBlock,
   Message,
@@ -98,8 +99,7 @@ export function estimateTextTokens(text: string | undefined): number {
   return Math.ceil(text.length / CHARS_PER_TOKEN);
 }
 
-function estimateAnthropicPdfTokens(base64Data: string): number {
-  const rawBytes = Math.ceil((base64Data.length * 3) / 4);
+function estimateAnthropicPdfTokens(rawBytes: number): number {
   return Math.max(
     ANTHROPIC_PDF_MIN_TOKENS,
     Math.ceil(rawBytes * ANTHROPIC_PDF_TOKENS_PER_BYTE),
@@ -111,13 +111,16 @@ function estimateFileDataTokens(
   options?: TokenEstimatorOptions,
 ): number {
   const providerName = options?.providerName;
+  // Size the payload from the source hint, resolving neither base64 nor a
+  // reference back to bytes — this runs on the per-turn hot path.
+  const byteLength = mediaSourceByteLength(block.source);
 
   // Anthropic sends PDFs as native document blocks and renders each page as an image
   if (
     providerName === "anthropic" &&
     block.source.media_type === "application/pdf"
   ) {
-    return estimateAnthropicPdfTokens(block.source.data);
+    return estimateAnthropicPdfTokens(byteLength);
   }
 
   // Gemini hears audio natively (inline base64) but bills it at ~32 tokens/sec.
@@ -127,15 +130,16 @@ function estimateFileDataTokens(
     providerName === "gemini" &&
     normalizeGeminiAudioMime(block.source.media_type) !== null
   ) {
-    return estimateGeminiAudioTokens(block.source.data);
+    return estimateGeminiAudioTokens(byteLength);
   }
 
-  // Gemini sends certain file types inline as base64
+  // Gemini sends certain file types inline as base64; cost the encoded payload
+  // (~4 base64 chars per 3 bytes) as text.
   if (
     providerName === "gemini" &&
     GEMINI_INLINE_FILE_MIME_TYPES.has(block.source.media_type)
   ) {
-    return estimateTextTokens(block.source.data);
+    return Math.ceil((Math.ceil(byteLength / 3) * 4) / CHARS_PER_TOKEN);
   }
 
   return 0;
@@ -185,7 +189,7 @@ function estimateImageTokens(
   block: Extract<ContentBlock, { type: "image" }>,
   options?: TokenEstimatorOptions,
 ): number {
-  const dims = parseImageDimensions(block.source.data, block.source.media_type);
+  const dims = parseImageDimensions(block.source);
   if (dims) {
     if (options?.providerName === "gemini") {
       return estimateGeminiImageTokens(dims.width, dims.height);

package/src/conversations/__tests__/message-consolidation.test.ts

@@ -1,6 +1,7 @@
 import { describe, expect, test } from "bun:test";
 
 import type { MessageRow } from "../../persistence/conversation-crud.js";
+import { resolveMessageContentBlocks } from "../../persistence/message-content-file.js";
 import {
   findDisplayTurnEndIndex,
   isToolResultOnlyUserMessage,
@@ -17,7 +18,7 @@ function makeMsg(
     id: `msg-${Math.random().toString(36).slice(2, 10)}`,
     conversationId: "conv-1",
     role,
-    content,
+    content: resolveMessageContentBlocks(content),
     createdAt: Date.now(),
     displayOrder: 0,
     seen: 1,
@@ -254,7 +255,7 @@ describe("mergeToolResultsIntoAssistantMessages", () => {
     const merged = mergeToolResultsIntoAssistantMessages(messages);
     expect(merged).toHaveLength(1);
     expect(merged[0].role).toBe("assistant");
-    const blocks = JSON.parse(merged[0].content) as Array<{ type: string }>;
+    const blocks = merged[0].content as Array<{ type: string }>;
     expect(blocks.map((b) => b.type)).toEqual(["tool_use", "tool_result"]);
   });
 
@@ -275,7 +276,7 @@ describe("mergeToolResultsIntoAssistantMessages", () => {
     const merged = mergeToolResultsIntoAssistantMessages(messages);
     expect(merged).toHaveLength(2);
     expect(merged[1].role).toBe("user");
-    const userBlocks = JSON.parse(merged[1].content) as Array<{ type: string }>;
+    const userBlocks = merged[1].content as Array<{ type: string }>;
     expect(userBlocks.map((b) => b.type)).toEqual(["text"]);
   });
 
@@ -283,7 +284,7 @@ describe("mergeToolResultsIntoAssistantMessages", () => {
     const messages = [makeMsg("user", "hi"), makeMsg("assistant", "hello")];
     const merged = mergeToolResultsIntoAssistantMessages(messages);
     expect(merged).toHaveLength(2);
-    expect(merged[0].content).toBe("hi");
+    expect(merged[0].content).toEqual([{ type: "text", text: "hi" }]);
   });
 });
 
@@ -306,7 +307,7 @@ describe("mergeConsecutiveAssistantMessages", () => {
     ]);
     expect(messages).toHaveLength(2);
     expect(messages[1].id).toBe("anchor");
-    const blocks = JSON.parse(messages[1].content) as Array<{ text: string }>;
+    const blocks = messages[1].content as Array<{ text: string }>;
     expect(blocks.map((blk) => blk.text)).toEqual(["part 1", "part 2"]);
     expect(mergedIdMap.get("anchor")).toEqual(["tail"]);
   });

package/src/conversations/job-handlers/summarization.ts

@@ -109,7 +109,6 @@ export async function buildConversationSummaryJob(
       id: summaryId,
       scope: "conversation",
       scopeKey: conversationId,
-      scopeId: "default",
       summary: summaryText,
       tokenEstimate: estimateTextTokens(summaryText),
       version: nextVersion,
@@ -124,7 +123,6 @@ export async function buildConversationSummaryJob(
         summary: summaryText,
         tokenEstimate: estimateTextTokens(summaryText),
         version: sql`${memorySummaries.version} + 1`,
-        scopeId: "default",
         startAt: earliestCovered,
         endAt: latestCovered,
         updatedAt: now,