@myagentroam/agent 0.11.21 → 0.11.22

package/dist/agent-core/model/openai-responses.js

@@ -1,9 +1,11 @@
 import { observeModelRequestUsage } from './usage-observer.js';
 import { observationModelParts } from '../tools/code-mode/observations.js';
+import { modelImageKey, modelImageUrl } from './image.js';
 import { stableToolSchema } from '../tools/definition-projection.js';
 import { isDeepStrictEqual } from 'node:util';
 import { createHash, randomUUID } from 'node:crypto';
 import { describeMarAgentError, MarAgentError } from '../error.js';
+import { assertInlineMediaBudget } from '../session/inline-media.js';
 import { marAgentReasoningEffortSchema, resolveModelDefaultReasoningEffort, usesResponsesConfigurationUpdates } from './configuration.js';
 import WebSocket from 'ws';
 import { isRetryableUpstreamOverloadCode, number, parseSse, record, text, observeModelStreamEvent, modelStreamEventObservedAt } from './sse.js';
@@ -95,7 +97,7 @@ export class OpenAiResponsesAdapter {
             const attemptTransport = this.configuration.responsesTransport?.transport === 'WEBSOCKET' ? 'WEBSOCKET' : 'HTTP';
             const requestPrefixDiagnostic = request.onAttemptDiagnostic
                 ? {
-                    requestInputItems: prepared.fullBody.input.length
+                    requestInputItems: prepared.inputItemCount
                 }
                 : {};
             const prelude = [];
@@ -667,7 +669,6 @@ class OpenAiResponsesTurnSession {
     #active = false;
     #closed = false;
     #turnState;
-    #fullBody;
     #completedResponseId;
     #connectionDiagnostic;
     #webSocketDiagnostic;
@@ -686,11 +687,15 @@ class OpenAiResponsesTurnSession {
         const outputItems = [];
         let completed = false;
         let sawUnrepresentedOutput = false;
+        let outputMessageCount = 0;
         try {
             const prepared = this.prepareRequest(request);
-            this.#fullBody = prepared.fullBody;
             this.#completedResponseId = undefined;
             for await (const event of this.adapter.startTurn(request, signal, this, prepared)) {
+                if (event.type === 'response.item' ||
+                    event.type === 'text.completed' ||
+                    event.type === 'client_tool.call')
+                    outputMessageCount++;
                 if (event.type === 'response.started' && event.responseId !== 'unknown')
                     responseId = event.responseId;
                 else if (event.type === 'response.item')
@@ -713,7 +718,13 @@ class OpenAiResponsesTurnSession {
                 completed &&
                 responseId !== undefined &&
                 (outputItems.length > 0 || !sawUnrepresentedOutput)) {
-                this.state.lastRequest = prepared.fullBody;
+                this.state.lastRequest =
+                    this.adapter.configuration.responsesTransport?.transport === 'WEBSOCKET'
+                        ? { ...prepared.body, input: [] }
+                        : prepared.fullBody;
+                this.state.lastMessageCount = request.messages.length + outputMessageCount;
+                this.state.lastInputItemCount = prepared.inputItemCount;
+                this.state.lastSystem = JSON.stringify([request.system, request.systemBlocks]);
                 this.state.lastResponse = { responseId, outputItems };
             }
             else
@@ -725,7 +736,6 @@ class OpenAiResponsesTurnSession {
         }
         finally {
             this.#completedResponseId = undefined;
-            this.#fullBody = undefined;
             this.#active = false;
         }
     }
@@ -738,11 +748,19 @@ class OpenAiResponsesTurnSession {
     resetContinuation() {
         clearResponsesContinuation(this.state);
     }
+    continuationInputCount() {
+        return this.state.lastMessageCount ?? 0;
+    }
+    commitContinuationInput(messageCount) {
+        if (this.state.lastResponse !== undefined)
+            this.state.lastMessageCount = messageCount;
+    }
     captureCompletedResponseId(responseId) {
         if (responseId !== undefined)
             this.#completedResponseId = responseId;
     }
     prepareRequest(request) {
+        assertInlineMediaBudget(request);
         if (this.adapter.configuration.responsesTransport?.transport === 'WEBSOCKET' &&
             this.state.socket !== undefined &&
             this.state.socket.readyState !== WebSocket.OPEN) {
@@ -750,16 +768,44 @@ class OpenAiResponsesTurnSession {
             clearResponsesContinuation(this.state);
         }
         // The continuation baseline must not share mutable schemas or items with callers.
-        const fullBody = JSON.parse(JSON.stringify(responsesRequestBody(this.adapter.configuration, request, request.messages)));
+        const snapshot = (body) => JSON.parse(JSON.stringify(body));
+        let full;
+        const fullBody = () => (full ??= snapshot(responsesRequestBody(this.adapter.configuration, request, request.messages)));
         if (responsesContinuationEnabled(this.adapter.configuration)) {
-            const incremental = incrementalResponsesInput(this.state, fullBody);
+            const websocket = this.adapter.configuration.responsesTransport?.transport === 'WEBSOCKET';
+            const suffixMessages = websocket && this.state.lastMessageCount !== undefined
+                ? request.messages.slice(this.state.lastMessageCount)
+                : undefined;
+            const suffixBody = websocket &&
+                this.state.lastResponse !== undefined &&
+                suffixMessages !== undefined &&
+                this.state.lastRequest !== undefined &&
+                this.state.lastSystem === JSON.stringify([request.system, request.systemBlocks]) &&
+                // Top-level images attach to the last real user, which must be in this suffix.
+                (!request.images?.length || suffixMessages.some(isRealUserMessage))
+                ? snapshot(responsesRequestBody(this.adapter.configuration, { ...request, system: '', systemBlocks: [] }, suffixMessages, this.state.lastResponse.outputItems))
+                : undefined;
+            const incremental = websocket
+                ? suffixBody !== undefined &&
+                    responsesContinuationPropertiesMatch(this.state.lastRequest, suffixBody)
+                    ? suffixBody.input
+                    : undefined
+                : incrementalResponsesInput(this.state, fullBody());
             if (incremental !== undefined) {
                 const responseId = this.state.lastResponse.responseId;
+                const inputItemCount = websocket
+                    ? (this.state.lastInputItemCount ?? 0) +
+                        this.state.lastResponse.outputItems.length +
+                        incremental.length
+                    : fullBody().input.length;
                 delete this.state.lastResponse;
                 return {
-                    fullBody,
+                    get fullBody() {
+                        return fullBody();
+                    },
+                    inputItemCount,
                     body: {
-                        ...fullBody,
+                        ...(suffixBody ?? fullBody()),
                         input: incremental,
                         previous_response_id: responseId
                     },
@@ -768,16 +814,12 @@ class OpenAiResponsesTurnSession {
             }
         }
         return {
-            fullBody,
-            body: fullBody,
+            fullBody: fullBody(),
+            inputItemCount: fullBody().input.length,
+            body: fullBody(),
             continuation: false
         };
     }
-    fullRequestBody() {
-        if (this.#fullBody === undefined)
-            throw new MarAgentError('MAR_AGENT_MODEL_PROTOCOL_ERROR', 'Full Responses request is unavailable.');
-        return this.#fullBody;
-    }
     currentTurnState() {
         return this.#turnState;
     }
@@ -906,7 +948,7 @@ function localContinuationReset() {
 function isLocalContinuationReset(error) {
     return error instanceof MarAgentError && error.code === 'MAR_AGENT_RESPONSES_CONTINUATION_RESET';
 }
-function responsesRequestBody(configuration, request, messages) {
+function responsesRequestBody(configuration, request, messages, previousOutputItems = []) {
     const systemBlocks = resolveModelTextBlocks(request.system, request.systemBlocks ?? []);
     if ((request.systemBlocks !== undefined && systemBlocks.some((block) => block.cache)) ||
         messages.some((message) => message.textBlocks !== undefined &&
@@ -940,7 +982,7 @@ function responsesRequestBody(configuration, request, messages) {
         configuration.hostedWebSearch)
         tools.push({ type: 'web_search' });
     const toolChoice = request.toolChoice ?? 'auto';
-    const input = openAiInput(messages, request.images, configuration.id, true, usesResponsesConfigurationUpdates(configuration), configuration.responsesOptionalParameters?.nativeTools !== false, request.tools);
+    const input = openAiInput(messages, request.images, configuration.id, true, usesResponsesConfigurationUpdates(configuration), configuration.responsesOptionalParameters?.nativeTools !== false, request.tools, previousOutputItems);
     const systemInput = responsesSystemInput(request, systemBlocks);
     if (systemInput !== undefined)
         input.unshift(systemInput);
@@ -1021,11 +1063,14 @@ function incrementalResponsesInput(state, current) {
     return current.input.slice(baseline.length);
 }
 function responsesContinuationPropertiesMatch(previous, current) {
-    return (JSON.stringify({ ...previous, input: undefined }) ===
-        JSON.stringify({ ...current, input: undefined }));
+    return (JSON.stringify({ ...previous, input: undefined, previous_response_id: undefined }) ===
+        JSON.stringify({ ...current, input: undefined, previous_response_id: undefined }));
 }
 function clearResponsesContinuation(state) {
     delete state.lastRequest;
+    delete state.lastMessageCount;
+    delete state.lastInputItemCount;
+    delete state.lastSystem;
     delete state.lastResponse;
 }
 function hashResponsesValue(value) {
@@ -1846,7 +1891,7 @@ function responseCitations(response) {
 function responseEventId(event) {
     return text(record(event.response).id);
 }
-export function openAiInput(messages, images, modelId, requireEncryptedReasoning, configurationUpdates = true, nativeTools = true, tools = []) {
+export function openAiInput(messages, images, modelId, requireEncryptedReasoning, configurationUpdates = true, nativeTools = true, tools = [], previousOutputItems = []) {
     const compatibleMessages = messages.filter((message) => message.role !== 'provider' ||
         (message.provider === 'OPENAI_RESPONSES' &&
             (message.providerModelId === undefined || message.providerModelId === modelId) &&
@@ -1861,6 +1906,15 @@ export function openAiInput(messages, images, modelId, requireEncryptedReasoning
     const functionCallIds = new Set();
     const customCallIds = new Set();
     const providerMessageTexts = new Set();
+    // Incremental input omits the calls already retained by the WebSocket response.
+    for (const item of previousOutputItems) {
+        if (typeof item.call_id !== 'string')
+            continue;
+        if (item.type === 'function_call')
+            functionCallIds.add(item.call_id);
+        if (nativeTools && item.type === 'custom_tool_call')
+            customCallIds.add(item.call_id);
+    }
     for (const message of compatibleMessages) {
         if (message.role !== 'provider' || !message.item)
             continue;
@@ -1970,7 +2024,7 @@ function openAiInputItem(message, images, customCallIds = new Set(), functionCal
                 ? { type: 'input_text', text: part.text }
                 : {
                     type: 'input_image',
-                    image_url: `data:${part.image.mimeType};base64,${part.image.dataBase64}`
+                    image_url: modelImageUrl(part.image)
                 }) ??
                 (message.textBlocks !== undefined
                     ? [
@@ -1980,7 +2034,7 @@ function openAiInputItem(message, images, customCallIds = new Set(), functionCal
                         })),
                         ...(message.images ?? []).map((image) => ({
                             type: 'input_image',
-                            image_url: `data:${image.mimeType};base64,${image.dataBase64}`
+                            image_url: modelImageUrl(image)
                         }))
                     ]
                     : message.images?.length
@@ -1988,7 +2042,7 @@ function openAiInputItem(message, images, customCallIds = new Set(), functionCal
                             { type: 'input_text', text: message.content },
                             ...message.images.map((image) => ({
                                 type: 'input_image',
-                                image_url: `data:${image.mimeType};base64,${image.dataBase64}`
+                                image_url: modelImageUrl(image)
                             }))
                         ]
                         : message.content)
@@ -2005,7 +2059,7 @@ function openAiInputItem(message, images, customCallIds = new Set(), functionCal
                 })),
                 ...userImages.map((image) => ({
                     type: 'input_image',
-                    image_url: `data:${image.mimeType};base64,${image.dataBase64}`
+                    image_url: modelImageUrl(image)
                 }))
             ]
             : userImages.length > 0
@@ -2013,7 +2067,7 @@ function openAiInputItem(message, images, customCallIds = new Set(), functionCal
                     { type: 'input_text', text: message.content },
                     ...userImages.map((image) => ({
                         type: 'input_image',
-                        image_url: `data:${image.mimeType};base64,${image.dataBase64}`
+                        image_url: modelImageUrl(image)
                     }))
                 ]
                 : message.content
@@ -2060,7 +2114,7 @@ function hasOpenFunctionObject(value) {
 function uniqueModelImages(...groups) {
     const seen = new Set();
     return groups.flatMap((group) => (group ?? []).flatMap((image) => {
-        const key = `${image.mimeType}\u0000${image.dataBase64}`;
+        const key = modelImageKey(image);
         if (seen.has(key))
             return [];
         seen.add(key);

package/dist/agent-core/runtime/compaction/retained-context.js

@@ -1,5 +1,6 @@
 import { parseModelInputMetadata } from '../../session/store/execution-state.js';
 import { restoreModelContext } from '../context/projection.js';
+import { parseModelImages } from '../../model/image.js';
 /** Use a public native identity, never private context text, as the visible boundary. */
 export function compactSnapshotAnchor(messages, sources) {
     for (let index = messages.length - 1; index >= 0; index--) {
@@ -39,15 +40,7 @@ export function restoreCompactTail(value) {
                     : {}),
             ...(metadata ? { metadata } : {}),
             ...(projected?.textBlocks ? { textBlocks: projected.textBlocks } : {}),
-            ...(Array.isArray(input.images) &&
-                input.images.every((image) => image && typeof image.mimeType === 'string' && typeof image.dataBase64 === 'string')
-                ? {
-                    images: input.images.map((image) => ({
-                        mimeType: image.mimeType,
-                        dataBase64: image.dataBase64
-                    }))
-                }
-                : {})
+            ...(parseModelImages(input.images)?.length ? { images: parseModelImages(input.images) } : {})
         };
     }
     return restoreModelContext(input);

package/dist/agent-core/runtime/context/gc-runtime.d.ts

@@ -1,11 +1,13 @@
-import type { ClientToolDefinition, ModelUsage } from '../../model/contracts.js';
+import type { ClientToolDefinition, ModelMessage, ModelUsage } from '../../model/contracts.js';
 import { type ContextGcPlatform } from '../../session/context-gc-checkpoint.js';
 import type { SessionStore, SessionRecord } from '../../session/store/contracts.js';
 import { type ContextProjection } from './projection.js';
 export interface ContextGcRuntimeInput {
+    readonly inlineMedia?: import('../../session/inline-media.js').InlineMediaGcCandidate | undefined;
     readonly store: SessionStore;
     readonly sessionId: string;
     readonly records?: readonly SessionRecord[];
+    readonly mediaView?: (messages: readonly ModelMessage[]) => readonly ModelMessage[];
     readonly system: string;
     readonly tools: readonly ClientToolDefinition[];
     readonly modelId: string;

package/dist/agent-core/runtime/context/gc-runtime.js

@@ -29,7 +29,10 @@ export class ContextGcRuntime {
             this.reset();
             this.#scope = scope;
         }
-        const usage = contextGcEligibleUsage(input.usage, input.contextWindowTokens);
+        const mediaRequired = input.inlineMedia !== undefined && input.inlineMedia.reason !== 'TEXT_GC';
+        const usage = mediaRequired
+            ? (input.usage.contextInputTokens ?? input.usage.inputTokens ?? 0)
+            : contextGcEligibleUsage(input.usage, input.contextWindowTokens);
         if (usage === undefined) {
             this.#lastUsage = undefined;
             this.#failures = 0;
@@ -45,11 +48,12 @@ export class ContextGcRuntime {
             this.#rolloutsSinceCheckpoint++;
         const maximumInterval = usage / input.contextWindowTokens >= 0.8 ? 2 : 8;
         this.#remaining = Math.min(this.#remaining, maximumInterval - 1);
-        if (this.#remaining > 0) {
+        if (this.#remaining > 0 && !mediaRequired) {
             this.#remaining--;
             return false;
         }
-        if (this.#checkpoint?.trigger.modelId === input.modelId &&
+        if (!mediaRequired &&
+            this.#checkpoint?.trigger.modelId === input.modelId &&
             this.#checkpoint.trigger.contextWindowTokens === input.contextWindowTokens &&
             !this.#checkpoint.protectedCandidatesPending &&
             this.#rolloutsSinceCheckpoint < CONTEXT_GC_POLICY.largeExecProtectionRollouts &&
@@ -90,7 +94,10 @@ export async function tryCommitContextGc(input) {
     return (await attemptContextGc(input)).result;
 }
 async function attemptContextGc(input) {
-    const contextUsage = contextGcEligibleUsage(input.usage, input.contextWindowTokens);
+    const mediaRequired = input.inlineMedia !== undefined && input.inlineMedia.reason !== 'TEXT_GC';
+    const contextUsage = mediaRequired
+        ? (input.usage.contextInputTokens ?? input.usage.inputTokens ?? 0)
+        : contextGcEligibleUsage(input.usage, input.contextWindowTokens);
     if (contextUsage === undefined)
         return { result: false };
     let deferred = { result: false };
@@ -108,7 +115,8 @@ async function attemptContextGc(input) {
                 protectedCandidatesPending: previous.payload.metrics.protectedCandidateTokens > 0
             };
         }
-        if (previous?.payload.trigger.modelId === input.modelId &&
+        if (!mediaRequired &&
+            previous?.payload.trigger.modelId === input.modelId &&
             previous.payload.trigger.contextWindowTokens === input.contextWindowTokens) {
             const rolloutGrowth = records.filter((record) => record.sequence > previous.record.sequence &&
                 record.type === 'model.attempt' &&
@@ -124,10 +132,12 @@ async function attemptContextGc(input) {
                 return deferred;
         }
         const projection = buildContextProjection(records, checkpointCache);
+        const messages = projection.entries.map((entry) => entry.message);
         const inspection = inspectContextGc({
+            ...(input.inlineMedia ? { inlineMedia: input.inlineMedia } : {}),
             records,
             checkpointCache,
-            messages: projection.entries.map((entry) => entry.message),
+            messages: input.mediaView?.(messages) ?? messages,
             messageSources: projection.entries.map((entry) => entry.record),
             system: input.system,
             tools: input.tools,
@@ -197,6 +207,7 @@ async function matchingCheckpointWasPersisted(store, sessionId, draft, records,
         !isDeepStrictEqual(latest.payload.base, draft.base) ||
         !isDeepStrictEqual(latest.payload.trigger, draft.trigger) ||
         !isDeepStrictEqual(latest.payload.economics, draft.economics) ||
+        !isDeepStrictEqual(latest.payload.inlineMedia, draft.inlineMedia) ||
         !isDeepStrictEqual(latest.payload.metrics, draft.metrics))
         return false;
     const persisted = new Map(latest.payload.replacements.map((replacement) => [

package/dist/agent-core/runtime/context/gc-tokens.js

@@ -10,6 +10,19 @@ export function contextGcTokenEntries(input, messages) {
     }));
 }
 function serializedEntries(input, messages) {
+    // Media bytes have a separate budget. Keep identity for cache boundaries without
+    // loading resources or counting base64 payloads as text tokens.
+    messages = messages.map((message) => message.images?.length
+        ? {
+            ...message,
+            images: message.images.map((image) => ({
+                mimeType: image.mimeType,
+                url: image.resourceId
+                    ? `https://mar.invalid/resource/${image.resourceId}`
+                    : (image.url ?? 'https://mar.invalid/inline-image')
+            }))
+        }
+        : message);
     const responses = input.modelProtocol === 'OPENAI_RESPONSES' ||
         (input.modelProtocol === undefined &&
             (input.messages ?? messages).some((message) => message.provider === 'OPENAI_RESPONSES'));

package/dist/agent-core/runtime/context/gc.d.ts

@@ -24,7 +24,7 @@ export interface ContextGcEconomicsResult {
     readonly passes: boolean;
 }
 export interface ContextGcPlan {
-    readonly stage: 'STAGE_A' | 'STAGE_A_AND_EXEC';
+    readonly stage: ContextGcMetrics['stage'];
     readonly draft: ContextGcCheckpointDraft;
     readonly metrics: ContextGcMetrics;
 }
@@ -38,6 +38,7 @@ export interface ContextGcInspection {
     readonly retryAfterRollouts?: number;
 }
 export interface ContextGcPlannerInput {
+    readonly inlineMedia?: import('../../session/inline-media.js').InlineMediaGcCandidate;
     readonly records: readonly SessionRecord[];
     readonly messages: readonly ModelMessage[];
     readonly messageSources: readonly SessionRecord[];

package/dist/agent-core/runtime/context/gc.js

@@ -118,6 +118,55 @@ export function planContextGc(input) {
     return inspectContextGc(input).plan;
 }
 export function inspectContextGc(input) {
+    const text = inspectTextGc(input);
+    const media = input.inlineMedia;
+    if (!media || (media.reason === 'TEXT_GC' && !text.plan))
+        return text;
+    const usageRecord = latestUsageRecord(input.records, input.modelId) ?? input.records.at(-1);
+    const contextUsageTokens = modelContextUsage(input.usage) ?? 0;
+    if (!usageRecord || contextUsageTokens === undefined)
+        return text;
+    input = {
+        ...input,
+        tokenCache: new Map(),
+        checkpointCache: input.checkpointCache ?? new ContextGcCheckpointCache()
+    };
+    const state = {
+        stage: text.plan?.stage ?? 'INLINE_MEDIA',
+        contextUsageTokens,
+        usageRatio: contextUsageTokens / input.contextWindowTokens,
+        usageRecord,
+        normalRolloutOrdinal: countSuccessfulNormalRollouts(input.records, input.modelId, input.contextWindowTokens),
+        protectedCandidateTokens: text.plan?.metrics.protectedCandidateTokens ?? 0
+    };
+    if (text.plan) {
+        const combined = evaluateCandidateSet(input, [], [
+            {
+                replacements: text.plan.draft.replacements,
+                observedRollouts: text.plan.metrics.minimumObservedRollouts,
+                evidenceSequence: text.plan.draft.boundarySequence
+            }
+        ], state, media.project);
+        if (combined?.economics.passes) {
+            const plan = finalizePlan(input, combined, true);
+            if (plan)
+                return { plan };
+        }
+        if (media.reason === 'TEXT_GC')
+            return text;
+    }
+    // Byte pressure has its own admission signal; it never relaxes text GC eligibility.
+    const onlyMedia = evaluateCandidateSet(input, input.checkpointCache.read(input.records)?.payload.replacements ?? [], [
+        {
+            replacements: [],
+            observedRollouts: media.reason === 'HARD_LIMIT' ? 0 : 2,
+            evidenceSequence: input.records.at(-1).sequence
+        }
+    ], { ...state, stage: 'INLINE_MEDIA', mediaOnly: true }, media.project);
+    const plan = onlyMedia && finalizePlan(input, onlyMedia, true);
+    return plan ? { plan } : text;
+}
+function inspectTextGc(input) {
     const contextUsageTokens = contextGcEligibleUsage(input.usage, input.contextWindowTokens);
     if (contextUsageTokens === undefined)
         return {};
@@ -396,14 +445,19 @@ export function contextGcEarliestReplacementSequence(replacements) {
             : [])
     ]));
 }
-function finalizePlan(input, evaluated) {
+function finalizePlan(input, evaluated, includeMedia = false) {
     if (!evaluated)
         return undefined;
+    let inlineMedia;
+    if (includeMedia && input.inlineMedia) {
+        const { project: _project, ...checkpoint } = input.inlineMedia;
+        inlineMedia = checkpoint;
+    }
     const base = contextGcBaseFromRecords(input.records, evaluated.boundarySequence);
     if (!base)
         return undefined;
     const replacements = normalizeContextGcReplacements(evaluated.replacements);
-    if (!replacements || replacements.length === 0)
+    if (!replacements || (replacements.length === 0 && !inlineMedia))
         return undefined;
     const metrics = {
         stage: evaluated.stage,
@@ -418,8 +472,12 @@ function finalizePlan(input, evaluated) {
         compactDeferralCreditCostUnits: evaluated.compactDeferralCreditCostUnits,
         imminentCompactAvoidanceCreditCostUnits: evaluated.imminentCompactAvoidanceCreditCostUnits,
         appliedCompactCreditCostUnits: evaluated.appliedCompactCreditCostUnits,
-        requiredPositiveNetRollouts: evaluated.economics.requiredPositiveNetRollouts,
-        requiredTargetRoiRollouts: evaluated.economics.requiredTargetRoiRollouts,
+        requiredPositiveNetRollouts: Number.isFinite(evaluated.economics.requiredPositiveNetRollouts)
+            ? evaluated.economics.requiredPositiveNetRollouts
+            : null,
+        requiredTargetRoiRollouts: Number.isFinite(evaluated.economics.requiredTargetRoiRollouts)
+            ? evaluated.economics.requiredTargetRoiRollouts
+            : null,
         projectedRolloutsUntilCompactAfterGc: evaluated.projectedRolloutsUntilCompactAfterGc,
         projectedTokensBeforeGc: evaluated.projectedTokensBeforeGc,
         projectedTokensAfterGc: evaluated.projectedTokensAfterGc,
@@ -436,12 +494,16 @@ function finalizePlan(input, evaluated) {
         cachePenaltyKnown: input.usage.cacheReadTokens !== undefined
     };
     const draft = {
+        ...(inlineMedia ? { inlineMedia } : {}),
         version: 1,
         base,
         boundarySequence: evaluated.boundarySequence,
         replacements,
         trigger: {
-            usageSequence: evaluated.usageRecord.sequence,
+            usageSequence: isRecord(evaluated.usageRecord.payload) &&
+                evaluated.usageRecord.payload.type === 'usage.updated'
+                ? evaluated.usageRecord.sequence
+                : null,
             contextInputTokens: evaluated.contextUsageTokens,
             contextWindowTokens: input.contextWindowTokens,
             normalRolloutOrdinal: evaluated.normalRolloutOrdinal,
@@ -453,7 +515,7 @@ function finalizePlan(input, evaluated) {
     };
     return { stage: evaluated.stage, draft, metrics };
 }
-function evaluateCandidateSet(input, existing, candidates, state) {
+function evaluateCandidateSet(input, existing, candidates, state, mediaProjection) {
     if (candidates.length === 0)
         return undefined;
     const replacements = normalizeContextGcReplacements([
@@ -463,16 +525,17 @@ function evaluateCandidateSet(input, existing, candidates, state) {
     if (!replacements)
         return undefined;
     const projected = applyContextGcReplacementsToMessages(input.messages, input.messageSources, replacements);
-    const reclaimedTokens = contextGcReclaimedTokens(input, input.messages, projected.messages);
+    const projectedMessages = mediaProjection?.(projected.messages) ?? projected.messages;
+    const reclaimedTokens = contextGcReclaimedTokens(input, input.messages, projectedMessages);
     const minimumGrossReductionRate = contextGcMinimumGrossReductionRate(state.usageRatio);
     const minimumReclaimedTokens = contextGcMinimumReclaimedTokens(state.contextUsageTokens, input.contextWindowTokens);
-    if (reclaimedTokens < minimumReclaimedTokens)
+    if (!state.mediaOnly && reclaimedTokens < minimumReclaimedTokens)
         return undefined;
     const beforeTokens = contextGcPrefixTokens(input) + contextGcMessageTokens(input, input.messages);
-    const afterTokens = contextGcPrefixTokens(input) + contextGcMessageTokens(input, projected.messages);
-    if (reclaimedTokens === 0)
+    const afterTokens = contextGcPrefixTokens(input) + contextGcMessageTokens(input, projectedMessages);
+    if (!state.mediaOnly && reclaimedTokens === 0)
         return undefined;
-    const cache = contextGcCacheImpact(input, input.messages, projected.messages, input.usage.cacheReadTokens ?? 0, reclaimedTokens);
+    const cache = contextGcCacheImpact(input, input.messages, projectedMessages, input.usage.cacheReadTokens ?? 0, reclaimedTokens);
     const growthSamples = collectContextGrowthSamples(input.records, input.modelId, input.contextWindowTokens, input.checkpointCache);
     const contextGrowthP75Tokens = percentile75(growthSamples);
     const compactThresholdTokens = Math.floor(input.contextWindowTokens * COMPACT_THRESHOLD_RATIO);
@@ -481,7 +544,7 @@ function evaluateCandidateSet(input, existing, candidates, state) {
     const pendingTokens = Math.max(0, input.usage.outputTokens ?? 0) + contextGcMessageTokens(input, pendingMessages);
     const projectedTokensBeforeGc = state.contextUsageTokens + pendingTokens;
     const projectedTokensAfterGc = Math.max(0, projectedTokensBeforeGc - reclaimedTokens);
-    if (projectedTokensAfterGc >= compactThresholdTokens)
+    if (!state.mediaOnly && projectedTokensAfterGc >= compactThresholdTokens)
         return undefined;
     const projectedRolloutsUntilCompactAfterGc = contextGrowthP75Tokens <= 0
         ? 0

package/dist/agent-core/runtime/context/projection.js

@@ -1,6 +1,7 @@
 import { restoreNativeModelContext } from '../../model/native-context.js';
 import { appliedInputRecord } from '../../session/deferred-input.js';
 import { inputMetadataFromRecord } from '../../session/store/execution-state.js';
+import { parseModelImages } from '../../model/image.js';
 import { contextGcTargetKey, latestValidContextGcCheckpoint } from '../../session/context-gc-checkpoint.js';
 import { TOOL_EXECUTION_LIMITS } from '../../tools/execution-limits.js';
 import { replaceReadContextItems } from '../../tools/local/read-context.js';
@@ -187,6 +188,9 @@ export function restoreModelContext(payload, providerModelId) {
             role: 'tool',
             content: payload.content,
             callId: payload.callId,
+            ...(parseModelImages(payload.images)?.length
+                ? { images: parseModelImages(payload.images) }
+                : {}),
             ...(typeof payload.name === 'string' ? { name: payload.name } : {}),
             ...(typeof payload.isError === 'boolean' ? { isError: payload.isError } : {}),
             ...(Array.isArray(payload.textBlocks) &&

package/dist/agent-core/runtime/context/request-budget.d.ts

@@ -1,6 +1,4 @@
 import type { ClientToolDefinition, ModelMessage } from '../../model/contracts.js';
-export declare function estimatedRequestTokens(system: string, messages: readonly ModelMessage[], tools: readonly ClientToolDefinition[], images?: readonly {
-    mimeType: string;
-    dataBase64: string;
-}[]): number;
+import type { ModelImage } from '../../model/image.js';
+export declare function estimatedRequestTokens(system: string, messages: readonly ModelMessage[], tools: readonly ClientToolDefinition[], images?: readonly ModelImage[]): number;
 export declare function stableJson(value: unknown): string;

package/dist/agent-core/runtime/context/restore.d.ts

@@ -1,8 +1,10 @@
 import type { SessionStore } from '../../session/store/contracts.js';
 import type { ModelMessage } from '../../model/contracts.js';
+import type { ModelImage } from '../../model/image.js';
 import type { CompactUserInput } from '../../session/store/contracts.js';
 import type { EnvironmentContextState } from './contracts.js';
 import { type ContextProjection } from './projection.js';
+export declare function readResourceImages(store: Pick<SessionStore, 'readResource'>, sessionId: string, images: readonly ModelImage[]): Promise<ModelImage[]>;
 export declare function restoreMessages(store: SessionStore, sessionId: string, records: readonly import('../../session/store/index.js').SessionRecord[], options: {
     includeImages: boolean;
     strictImages?: boolean;

package/dist/agent-core/runtime/context/restore.js

@@ -5,6 +5,16 @@ import { compactedHistoryMessages, continuationMessage } from '../compaction/his
 import { restoreCompactTail } from '../compaction/retained-context.js';
 import { environmentContextUpdate, restoreEnvironmentState } from './environment.js';
 import { buildContextProjection } from './projection.js';
+export async function readResourceImages(store, sessionId, images) {
+    return Promise.all(images.map(async (image) => {
+        const resource = image.resourceId === undefined
+            ? undefined
+            : await store.readResource(sessionId, image.resourceId);
+        if (!resource)
+            throw new MarAgentError('MAR_AGENT_SESSION_CORRUPT', 'Session input images cannot be fully restored.');
+        return { ...image, dataBase64: resource.dataBase64 };
+    }));
+}
 export async function restoreMessages(store, sessionId, records, options) {
     const messages = [];
     const outcomes = new Map();
@@ -41,8 +51,9 @@ export async function restoreMessages(store, sessionId, records, options) {
                 try {
                     images = await store.readInputImages(sessionId, payload.images);
                     if (options.strictImages &&
-                        payload.images !== undefined &&
-                        (!Array.isArray(payload.images) || images.length !== payload.images.length))
+                        Array.isArray(payload.images) &&
+                        images.length !==
+                            payload.images.filter((image) => image?.resourceInputVersion === 1).length)
                         throw new MarAgentError('MAR_AGENT_SESSION_CORRUPT', 'Session input images cannot be fully restored.');
                 }
                 catch (error) {
@@ -145,8 +156,13 @@ export async function restoreMessages(store, sessionId, records, options) {
                 }
             }
         }
-        for (const message of messages.slice(messageStart))
+        for (const message of messages.slice(messageStart)) {
+            if (!options.includeImages)
+                delete message.images;
+            else if (options.strictImages && message.images?.length)
+                message.images = await readResourceImages(store, sessionId, message.images);
             options.messageSources?.set(message, record);
+        }
     }
     const completedToolCalls = new Set(messages.filter((message) => message.role === 'tool').map((message) => message.callId));
     return {