dominds 1.16.2 → 1.16.4

package/dist/dialog-display-state.d.ts

@@ -34,7 +34,9 @@ export declare function broadcastRunControlCountsSnapshot(): Promise<void>;
 export declare function hasActiveRun(dialogId: DialogID): boolean;
 export declare function getActiveRunSignal(dialogId: DialogID): AbortSignal | undefined;
 export declare function createActiveRun(dialogId: DialogID): AbortSignal;
-export declare function clearActiveRun(dialogId: DialogID): void;
+export declare function clearActiveRun(dialogId: DialogID, options?: Readonly<{
+    notifyBackendLoop?: boolean;
+}>): void;
 export declare function clearRootDialogQuarantiningIfIdle(rootDialogId: DialogID): void;
 export declare function markRootDialogQuarantining(rootDialogId: DialogID): void;
 export declare function clearRootDialogQuarantining(rootDialogId: DialogID): void;

package/dist/dialog-display-state.js

@@ -44,6 +44,7 @@ exports.requestInterruptDialog = requestInterruptDialog;
 exports.requestEmergencyStopAll = requestEmergencyStopAll;
 const time_1 = require("@longrun-ai/kernel/utils/time");
 const dialog_1 = require("./dialog");
+const dialog_global_registry_1 = require("./dialog-global-registry");
 const dialog_interruption_1 = require("./dialog-interruption");
 const evt_registry_1 = require("./evt-registry");
 const log_1 = require("./log");
@@ -178,13 +179,19 @@ function createActiveRun(dialogId) {
     syncRunControlCountsAfterActiveRunChange('create_active_run', dialogId);
     return run.abortController.signal;
 }
-function clearActiveRun(dialogId) {
+function clearActiveRun(dialogId, options) {
     const deleted = activeRunsByDialogKey.delete(dialogId.key());
     if (!deleted) {
         clearQuarantiningRootDialogIfIdle(dialogId.rootId);
         return;
     }
     clearQuarantiningRootDialogIfIdle(dialogId.rootId);
+    if (dialogId.selfId === dialogId.rootId && options?.notifyBackendLoop !== false) {
+        dialog_global_registry_1.globalDialogRegistry.notifyActiveRunCleared(dialogId.rootId, {
+            source: 'dialog_display_state_active_run_clear',
+            reason: 'root_active_run_cleared',
+        });
+    }
     syncRunControlCountsAfterActiveRunChange('clear_active_run', dialogId);
 }
 function clearQuarantiningRootDialogIfIdle(rootId) {

package/dist/dialog-fork.js

@@ -113,6 +113,7 @@ function isPersistedMessageRecord(record) {
         // not contribute to message-count / context reconstruction semantics.
         case 'web_search_call_record':
         case 'native_tool_call_record':
+        case 'tool_result_image_ingest_record':
         case 'quest_for_sup_record':
         case 'tellask_reply_resolution_record':
         case 'tellask_call_anchor_record':
@@ -199,6 +200,14 @@ function rewriteRecordForFork(record, newRootId) {
                 ...record,
                 contentItems: rewriteFuncResultContentItems(record.contentItems, newRootId),
             };
+        case 'tool_result_image_ingest_record':
+            return {
+                ...record,
+                artifact: {
+                    ...record.artifact,
+                    rootId: newRootId,
+                },
+            };
         case 'subdialog_created_record':
         case 'reminders_reconciled_record':
         case 'questions4human_reconciled_record':

package/dist/dialog-global-registry.d.ts

@@ -1,7 +1,7 @@
 import type { RootDialog } from './dialog';
 export type DriveTriggerEvent = Readonly<{
     type: 'drive_trigger_evt';
-    action: 'mark_needs_drive' | 'mark_not_needing_drive';
+    action: 'mark_needs_drive' | 'mark_not_needing_drive' | 'active_run_cleared';
     rootId: string;
     entryFound: boolean;
     previousNeedsDrive: boolean | null;
@@ -28,6 +28,10 @@ declare class GlobalDialogRegistry {
     waitForDriveTrigger(): Promise<DriveTriggerEvent>;
     markNeedsDrive(rootId: string, meta?: DriveTriggerMeta): void;
     markNotNeedingDrive(rootId: string, meta?: DriveTriggerMeta): void;
+    notifyActiveRunCleared(rootId: string, meta?: DriveTriggerMeta): void;
+    noteActiveRunBlockedQueuedDrive(rootId: string): void;
+    hasPendingActiveRunClearedWake(rootId: string): boolean;
+    isMarkedNeedingDrive(rootId: string): boolean;
     getDialogsNeedingDrive(): RootDialog[];
     getLastDriveTrigger(rootId: string): DriveTriggerEvent | undefined;
     getAll(): RootDialog[];

package/dist/dialog-global-registry.js

@@ -29,7 +29,11 @@ class GlobalDialogRegistry {
         if (existing) {
             return;
         }
-        this.entries.set(rootDialog.id.rootId, { rootDialog, needsDrive: false });
+        this.entries.set(rootDialog.id.rootId, {
+            rootDialog,
+            needsDrive: false,
+            activeRunClearedWakePending: false,
+        });
         void (async () => {
             try {
                 const needsDrive = await persistence_1.DialogPersistence.getNeedsDrive(rootDialog.id);
@@ -82,6 +86,8 @@ class GlobalDialogRegistry {
         const previousNeedsDrive = entry ? entry.needsDrive : null;
         if (entry) {
             entry.needsDrive = true;
+            // A fresh queueing trigger supersedes any earlier "wake me once active run clears" debt.
+            entry.activeRunClearedWakePending = false;
         }
         this.publishDriveTrigger({
             action: 'mark_needs_drive',
@@ -101,6 +107,7 @@ class GlobalDialogRegistry {
         const previousNeedsDrive = entry ? entry.needsDrive : null;
         if (entry) {
             entry.needsDrive = false;
+            entry.activeRunClearedWakePending = false;
         }
         this.publishDriveTrigger({
             action: 'mark_not_needing_drive',
@@ -111,6 +118,43 @@ class GlobalDialogRegistry {
             meta: triggerMeta,
         });
     }
+    notifyActiveRunCleared(rootId, meta) {
+        const triggerMeta = meta ?? {
+            source: 'unknown',
+            reason: 'unspecified',
+        };
+        const entry = this.entries.get(rootId);
+        if (!entry) {
+            return;
+        }
+        if (!entry.activeRunClearedWakePending || !entry.needsDrive) {
+            entry.activeRunClearedWakePending = false;
+            return;
+        }
+        const currentNeedsDrive = entry ? entry.needsDrive : null;
+        entry.activeRunClearedWakePending = false;
+        this.publishDriveTrigger({
+            action: 'active_run_cleared',
+            rootId,
+            entryFound: true,
+            previousNeedsDrive: currentNeedsDrive,
+            nextNeedsDrive: entry.needsDrive,
+            meta: triggerMeta,
+        });
+    }
+    noteActiveRunBlockedQueuedDrive(rootId) {
+        const entry = this.entries.get(rootId);
+        if (!entry || !entry.needsDrive) {
+            return;
+        }
+        entry.activeRunClearedWakePending = true;
+    }
+    hasPendingActiveRunClearedWake(rootId) {
+        return this.entries.get(rootId)?.activeRunClearedWakePending === true;
+    }
+    isMarkedNeedingDrive(rootId) {
+        return this.entries.get(rootId)?.needsDrive === true;
+    }
     getDialogsNeedingDrive() {
         return Array.from(this.entries.values())
             .filter((entry) => entry.needsDrive)

package/dist/dialog.d.ts

@@ -18,6 +18,7 @@ import type { DialogPrompt, DialogRunControlSpec, DialogSubdialogReplyTarget, Dr
 import type { LanguageCode } from '@longrun-ai/kernel/types/language';
 import type { CalleeCourseNumber, CalleeGenerationSeqNumber, CallingCourseNumber, DialogMetadataFile, HumanQuestion, ProviderData, ReasoningPayload, TellaskCallRecordName, TellaskReplyDirective } from '@longrun-ai/kernel/types/storage';
 import { ChatMessage, FuncResultMsg, TellaskCarryoverMsg, TellaskResultMsg } from './llm/client';
+import type { ToolResultImageIngest } from './llm/gen';
 import type { JsonValue } from './tool';
 import { Reminder, ReminderOptions, ReminderOwner } from './tool';
 type NewCourseHookResult = {
@@ -406,6 +407,7 @@ export declare abstract class Dialog {
         action?: WebSearchCallAction;
     }): Promise<void>;
     nativeToolCall(payload: NativeToolCallPayload): Promise<void>;
+    toolResultImageIngest(payload: ToolResultImageIngest): Promise<void>;
     callingStart(payload: {
         callName: 'tellaskBack' | 'tellask' | 'tellaskSessionless' | 'askHuman' | 'freshBootsReasoning';
         callId: string;
@@ -624,6 +626,7 @@ export declare abstract class DialogStore {
         action?: WebSearchCallAction;
     }): Promise<void>;
     nativeToolCall(_dialog: Dialog, _payload: NativeToolCallPayload): Promise<void>;
+    toolResultImageIngest(_dialog: Dialog, _payload: ToolResultImageIngest): Promise<void>;
     /**
      * Load current course number from persisted metadata
      * This method should be implemented by subclasses to read from storage

package/dist/dialog.js

@@ -1144,6 +1144,9 @@ class Dialog {
     async nativeToolCall(payload) {
         await this.dlgStore.nativeToolCall(this, payload);
     }
+    async toolResultImageIngest(payload) {
+        await this.dlgStore.toolResultImageIngest(this, payload);
+    }
     // Tellask-special call lifecycle events
     async callingStart(payload) {
         // Store callId for inline call-result correlation
@@ -1578,6 +1581,7 @@ class DialogStore {
     async funcCallRequested(_dialog, _funcId, _funcName, _argumentsStr) { }
     async webSearchCall(_dialog, _payload) { }
     async nativeToolCall(_dialog, _payload) { }
+    async toolResultImageIngest(_dialog, _payload) { }
     /**
      * Load current course number from persisted metadata
      * This method should be implemented by subclasses to read from storage

package/dist/llm/gen/anthropic.d.ts

@@ -8,7 +8,7 @@ export type AnthropicStreamConsumeResult = {
     usage: LlmUsageStats;
     llmGenModel?: string;
 };
-export declare function buildAnthropicRequestMessagesWrapper(context: ChatMessage[], providerConfig?: ProviderConfig): Promise<MessageParam[]>;
+export declare function buildAnthropicRequestMessagesWrapper(context: ChatMessage[], requestContextOrProviderConfig?: LlmRequestContext | ProviderConfig, providerConfigMaybe?: ProviderConfig): Promise<MessageParam[]>;
 export declare function consumeAnthropicStream(stream: AsyncIterable<MessageStreamEvent>, receiver: LlmStreamReceiver, abortSignal?: AbortSignal, forcedJsonToolName?: string): Promise<AnthropicStreamConsumeResult>;
 /**
  * Reconstruct Anthropic context from persisted messages with genseq tracking.
@@ -29,6 +29,6 @@ export declare function reconstructAnthropicContextWrapperAsync(persistedMessage
 export declare class AnthropicGen implements LlmGenerator {
     get apiType(): string;
     classifyFailure(error: unknown): LlmFailureDisposition | undefined;
-    genToReceiver(providerConfig: ProviderConfig, agent: Team.Member, systemPrompt: string, funcTools: FuncTool[], _requestContext: LlmRequestContext, context: ChatMessage[], receiver: LlmStreamReceiver, _genseq: number, abortSignal?: AbortSignal): Promise<LlmStreamResult>;
-    genMoreMessages(providerConfig: ProviderConfig, agent: Team.Member, systemPrompt: string, funcTools: FuncTool[], _requestContext: LlmRequestContext, context: ChatMessage[], genseq: number, abortSignal?: AbortSignal): Promise<LlmBatchResult>;
+    genToReceiver(providerConfig: ProviderConfig, agent: Team.Member, systemPrompt: string, funcTools: FuncTool[], requestContext: LlmRequestContext, context: ChatMessage[], receiver: LlmStreamReceiver, _genseq: number, abortSignal?: AbortSignal): Promise<LlmStreamResult>;
+    genMoreMessages(providerConfig: ProviderConfig, agent: Team.Member, systemPrompt: string, funcTools: FuncTool[], requestContext: LlmRequestContext, context: ChatMessage[], genseq: number, abortSignal?: AbortSignal): Promise<LlmBatchResult>;
 }

package/dist/llm/gen/anthropic.js

@@ -18,6 +18,7 @@ const artifacts_1 = require("./artifacts");
 const failure_classifier_1 = require("./failure-classifier");
 const tool_call_context_1 = require("./tool-call-context");
 const tool_output_limit_1 = require("./tool-output-limit");
+const tool_result_image_ingest_1 = require("./tool-result-image-ingest");
 const log = (0, log_1.createLogger)('llm/anthropic');
 const ANTHROPIC_JSON_RESPONSE_TOOL_NAME = 'dominds_json_response';
 const ANTHROPIC_JSON_RESPONSE_TOOL_DESCRIPTION = 'Return the final answer as a JSON object. Do not include any non-JSON text.';
@@ -75,6 +76,13 @@ function isRecord(value) {
 function isNonArrayRecord(value) {
     return isRecord(value) && !Array.isArray(value);
 }
+function isLlmRequestContext(value) {
+    return (isNonArrayRecord(value) &&
+        typeof value.dialogSelfId === 'string' &&
+        typeof value.dialogRootId === 'string' &&
+        typeof value.providerKey === 'string' &&
+        typeof value.modelKey === 'string');
+}
 function tryExtractApiReturnedModel(value) {
     // NOTE: External API payload; a runtime check is unavoidable.
     if (!isRecord(value))
@@ -177,7 +185,7 @@ function parseForcedJsonToolInput(rawJson, fallbackInput, at) {
     }
     throw new Error(`Invalid ${at}: ${errorText}; raw=${JSON.stringify(preview)}`);
 }
-async function funcResultToAnthropicToolResultBlock(chatMsg, limitChars) {
+async function funcResultToAnthropicToolResultBlock(chatMsg, limitChars, requestContext, allowedImageKeys, onToolResultImageIngest) {
     const items = chatMsg.contentItems;
     if (!Array.isArray(items) || items.length === 0) {
         return {
@@ -194,22 +202,85 @@ async function funcResultToAnthropicToolResultBlock(chatMsg, limitChars) {
         }
         if (item.type === 'input_image') {
             if (!(0, artifacts_1.isVisionImageMimeType)(item.mimeType)) {
+                if (onToolResultImageIngest) {
+                    await onToolResultImageIngest((0, tool_result_image_ingest_1.buildToolResultImageIngest)({
+                        requestContext,
+                        toolCallId: chatMsg.id,
+                        toolName: chatMsg.name,
+                        artifact: item.artifact,
+                        disposition: 'filtered_mime_unsupported',
+                        mimeType: item.mimeType,
+                        providerPathLabel: 'Anthropic Messages path',
+                    }));
+                }
                 content.push({
                     type: 'text',
                     text: `[image omitted: unsupported mimeType=${item.mimeType}]`,
                 });
                 continue;
             }
-            const bytes = await (0, artifacts_1.readDialogArtifactBytes)({
-                rootId: item.artifact.rootId,
-                selfId: item.artifact.selfId,
-                status: item.artifact.status,
-                relPath: item.artifact.relPath,
-            });
-            if (!bytes) {
+            if (!allowedImageKeys.has((0, tool_result_image_ingest_1.buildToolResultImageBudgetKeyForMsg)(chatMsg, item.artifact))) {
+                if (onToolResultImageIngest) {
+                    await onToolResultImageIngest((0, tool_result_image_ingest_1.buildToolResultImageIngest)({
+                        requestContext,
+                        toolCallId: chatMsg.id,
+                        toolName: chatMsg.name,
+                        artifact: item.artifact,
+                        disposition: 'filtered_size_limit',
+                        detail: (0, tool_result_image_ingest_1.buildToolResultImageBudgetLimitDetail)({
+                            byteLength: item.byteLength,
+                            budgetBytes: tool_result_image_ingest_1.ANTHROPIC_TOOL_RESULT_IMAGE_BUDGET_BYTES,
+                        }),
+                        providerPathLabel: 'Anthropic Messages path',
+                    }));
+                }
+                content.push({
+                    type: 'text',
+                    text: `[image omitted: request image budget exceeded bytes=${String(item.byteLength)} budget=${String(tool_result_image_ingest_1.ANTHROPIC_TOOL_RESULT_IMAGE_BUDGET_BYTES)}]`,
+                });
+                continue;
+            }
+            const bytesResult = await (0, tool_result_image_ingest_1.readToolResultImageBytesSafe)(item.artifact);
+            if (bytesResult.kind === 'missing') {
+                if (onToolResultImageIngest) {
+                    await onToolResultImageIngest((0, tool_result_image_ingest_1.buildToolResultImageIngest)({
+                        requestContext,
+                        toolCallId: chatMsg.id,
+                        toolName: chatMsg.name,
+                        artifact: item.artifact,
+                        disposition: 'filtered_missing',
+                        providerPathLabel: 'Anthropic Messages path',
+                    }));
+                }
                 content.push({ type: 'text', text: `[image missing: ${item.artifact.relPath}]` });
                 continue;
             }
+            if (bytesResult.kind === 'read_failed') {
+                if (onToolResultImageIngest) {
+                    await onToolResultImageIngest((0, tool_result_image_ingest_1.buildToolResultImageIngest)({
+                        requestContext,
+                        toolCallId: chatMsg.id,
+                        toolName: chatMsg.name,
+                        artifact: item.artifact,
+                        disposition: 'filtered_read_failed',
+                        detail: bytesResult.detail,
+                        providerPathLabel: 'Anthropic Messages path',
+                    }));
+                }
+                content.push({ type: 'text', text: `[image unreadable: ${item.artifact.relPath}]` });
+                continue;
+            }
+            if (onToolResultImageIngest) {
+                await onToolResultImageIngest((0, tool_result_image_ingest_1.buildToolResultImageIngest)({
+                    requestContext,
+                    toolCallId: chatMsg.id,
+                    toolName: chatMsg.name,
+                    artifact: item.artifact,
+                    disposition: 'fed_native',
+                    providerPathLabel: 'Anthropic Messages path',
+                }));
+            }
+            const bytes = bytesResult.bytes;
             const base64 = bytes.toString('base64');
             content.push({
                 type: 'image',
@@ -237,14 +308,16 @@ async function funcResultToAnthropicToolResultBlock(chatMsg, limitChars) {
         content: limitAnthropicToolOutputBlocks(content, chatMsg, limitChars),
     };
 }
-async function chatMessageToContentBlocksAsync(chatMsg, limitChars) {
+async function chatMessageToContentBlocksAsync(chatMsg, limitChars, requestContext, allowedImageKeys, onToolResultImageIngest) {
     if (chatMsg.type !== 'func_result_msg') {
         return chatMessageToContentBlocks(chatMsg);
     }
-    return [await funcResultToAnthropicToolResultBlock(chatMsg, limitChars)];
+    return [
+        await funcResultToAnthropicToolResultBlock(chatMsg, limitChars, requestContext, allowedImageKeys, onToolResultImageIngest),
+    ];
 }
-async function chatMessageToAnthropicAsync(chatMsg, limitChars) {
-    const contentBlocks = await chatMessageToContentBlocksAsync(chatMsg, limitChars);
+async function chatMessageToAnthropicAsync(chatMsg, limitChars, requestContext, allowedImageKeys, onToolResultImageIngest) {
+    const contentBlocks = await chatMessageToContentBlocksAsync(chatMsg, limitChars, requestContext, allowedImageKeys, onToolResultImageIngest);
     if (contentBlocks.length === 0) {
         throw new Error(`No content blocks generated for message: ${JSON.stringify(chatMsg)}`);
     }
@@ -257,7 +330,7 @@ async function chatMessageToAnthropicAsync(chatMsg, limitChars) {
         content: contentBlocks.length === 1 ? contentBlocks : contentBlocks,
     };
 }
-async function buildAnthropicRequestMessages(context, providerConfig) {
+async function buildAnthropicRequestMessages(context, requestContext, providerConfig, onToolResultImageIngest) {
     // We keep the async path for func_result_msg because it may contain image artifacts.
     const normalized = (0, tool_call_context_1.normalizeToolCallPairs)(context);
     const violation = (0, tool_call_context_1.findFirstToolCallAdjacencyViolation)(normalized);
@@ -273,13 +346,25 @@ async function buildAnthropicRequestMessages(context, providerConfig) {
     }
     const messages = [];
     const toolResultMaxChars = (0, tool_output_limit_1.resolveProviderToolResultMaxChars)(providerConfig);
+    const allowedImageKeys = (0, tool_result_image_ingest_1.selectLatestToolResultImagesWithinBudget)(normalized, tool_result_image_ingest_1.ANTHROPIC_TOOL_RESULT_IMAGE_BUDGET_BYTES);
     for (const msg of normalized) {
-        messages.push(await chatMessageToAnthropicAsync(msg, toolResultMaxChars));
+        messages.push(await chatMessageToAnthropicAsync(msg, toolResultMaxChars, requestContext, allowedImageKeys, onToolResultImageIngest));
     }
     return assembleAnthropicTurns(messages);
 }
-async function buildAnthropicRequestMessagesWrapper(context, providerConfig) {
-    return await buildAnthropicRequestMessages(context, providerConfig);
+async function buildAnthropicRequestMessagesWrapper(context, requestContextOrProviderConfig, providerConfigMaybe) {
+    const requestContext = isLlmRequestContext(requestContextOrProviderConfig)
+        ? requestContextOrProviderConfig
+        : {
+            dialogSelfId: '',
+            dialogRootId: '',
+            providerKey: 'anthropic',
+            modelKey: 'unknown',
+        };
+    const providerConfig = isLlmRequestContext(requestContextOrProviderConfig)
+        ? providerConfigMaybe
+        : requestContextOrProviderConfig;
+    return await buildAnthropicRequestMessages(context, requestContext, providerConfig);
 }
 /**
  * Reconstruct Anthropic context from persisted messages.
@@ -800,7 +885,12 @@ function reconstructAnthropicContextWrapper(persistedMessages) {
     return reconstructed;
 }
 async function reconstructAnthropicContextWrapperAsync(persistedMessages) {
-    const reconstructed = await buildAnthropicRequestMessages(persistedMessages);
+    const reconstructed = await buildAnthropicRequestMessages(persistedMessages, {
+        dialogSelfId: '',
+        dialogRootId: '',
+        providerKey: 'anthropic',
+        modelKey: 'unknown',
+    });
     // Validate the reconstructed context
     try {
         validateReconstructedContext(reconstructed);
@@ -824,12 +914,12 @@ class AnthropicGen {
     classifyFailure(error) {
         return (0, failure_classifier_1.classifyAnthropicFailure)(error);
     }
-    async genToReceiver(providerConfig, agent, systemPrompt, funcTools, _requestContext, context, receiver, _genseq, abortSignal) {
+    async genToReceiver(providerConfig, agent, systemPrompt, funcTools, requestContext, context, receiver, _genseq, abortSignal) {
         const apiKey = process.env[providerConfig.apiKeyEnvVar];
         if (!apiKey)
             throw new Error(`Missing API key env var ${providerConfig.apiKeyEnvVar}`);
         const client = new sdk_1.Anthropic({ apiKey, baseURL: providerConfig.baseUrl });
-        const requestMessages = await buildAnthropicRequestMessages(context, providerConfig);
+        const requestMessages = await buildAnthropicRequestMessages(context, requestContext, providerConfig, receiver.toolResultImageIngest);
         const anthropicParams = agent.model_params?.anthropic || {};
         const forceJsonResponse = resolveAnthropicJsonResponseEnabled(agent);
         const maxTokens = agent.model_params?.max_tokens;
@@ -874,12 +964,15 @@ class AnthropicGen {
         const stream = client.messages.stream(streamParams);
         return consumeAnthropicStream(stream, receiver, abortSignal, forceJsonResponse ? ANTHROPIC_JSON_RESPONSE_TOOL_NAME : undefined);
     }
-    async genMoreMessages(providerConfig, agent, systemPrompt, funcTools, _requestContext, context, genseq, abortSignal) {
+    async genMoreMessages(providerConfig, agent, systemPrompt, funcTools, requestContext, context, genseq, abortSignal) {
         const apiKey = process.env[providerConfig.apiKeyEnvVar];
         if (!apiKey)
             throw new Error(`Missing API key env var ${providerConfig.apiKeyEnvVar}`);
         const client = new sdk_1.Anthropic({ apiKey, baseURL: providerConfig.baseUrl });
-        const requestMessages = await buildAnthropicRequestMessages(context, providerConfig);
+        const outputs = [];
+        const requestMessages = await buildAnthropicRequestMessages(context, requestContext, providerConfig, async (ingest) => {
+            outputs.push({ kind: 'tool_result_image_ingest', ingest });
+        });
         const anthropicParams = agent.model_params?.anthropic || {};
         const forceJsonResponse = resolveAnthropicJsonResponseEnabled(agent);
         const maxTokens = agent.model_params?.max_tokens;
@@ -940,6 +1033,7 @@ class AnthropicGen {
         };
         return {
             messages: anthropicToChatMessages(response, genseq, forceJsonResponse ? ANTHROPIC_JSON_RESPONSE_TOOL_NAME : undefined),
+            ...(outputs.length > 0 ? { outputs } : {}),
             usage,
             llmGenModel: returnedModel,
         };

package/dist/llm/gen/codex.js

@@ -44,6 +44,7 @@ const artifacts_1 = require("./artifacts");
 const failure_classifier_1 = require("./failure-classifier");
 const tool_call_context_1 = require("./tool-call-context");
 const tool_output_limit_1 = require("./tool-output-limit");
+const tool_result_image_ingest_1 = require("./tool-result-image-ingest");
 const log = (0, log_1.createLogger)('llm/codex');
 const codexFallbackInstructions = 'You are Codex CLI.';
 const CODEX_SYSTEM_PROMPT_DIRECTIVE_PATTERN = /^([ \t]*)@codex-system-prompt(?::([A-Za-z0-9._-]+))?([ \t]*)$/gm;
@@ -331,7 +332,7 @@ function chatMessageToCodexItems(msg) {
         }
     }
 }
-async function buildCodexFunctionCallOutput(msg, limitChars) {
+async function buildCodexFunctionCallOutput(msg, limitChars, requestContext, allowedImageKeys, onToolResultImageIngest) {
     const items = msg.contentItems;
     if (!Array.isArray(items) || items.length === 0) {
         return limitCodexToolOutputText(msg.content, msg, limitChars);
@@ -344,22 +345,85 @@ async function buildCodexFunctionCallOutput(msg, limitChars) {
         }
         if (item.type === 'input_image') {
             if (!(0, artifacts_1.isVisionImageMimeType)(item.mimeType)) {
+                if (onToolResultImageIngest) {
+                    await onToolResultImageIngest((0, tool_result_image_ingest_1.buildToolResultImageIngest)({
+                        requestContext,
+                        toolCallId: msg.id,
+                        toolName: msg.name,
+                        artifact: item.artifact,
+                        disposition: 'filtered_mime_unsupported',
+                        mimeType: item.mimeType,
+                        providerPathLabel: 'Codex path',
+                    }));
+                }
                 out.push({
                     type: 'input_text',
                     text: `[image omitted: unsupported mimeType=${item.mimeType}]`,
                 });
                 continue;
             }
-            const bytes = await (0, artifacts_1.readDialogArtifactBytes)({
-                rootId: item.artifact.rootId,
-                selfId: item.artifact.selfId,
-                status: item.artifact.status,
-                relPath: item.artifact.relPath,
-            });
-            if (!bytes) {
+            if (!allowedImageKeys.has((0, tool_result_image_ingest_1.buildToolResultImageBudgetKeyForMsg)(msg, item.artifact))) {
+                if (onToolResultImageIngest) {
+                    await onToolResultImageIngest((0, tool_result_image_ingest_1.buildToolResultImageIngest)({
+                        requestContext,
+                        toolCallId: msg.id,
+                        toolName: msg.name,
+                        artifact: item.artifact,
+                        disposition: 'filtered_size_limit',
+                        detail: (0, tool_result_image_ingest_1.buildToolResultImageBudgetLimitDetail)({
+                            byteLength: item.byteLength,
+                            budgetBytes: tool_result_image_ingest_1.CODEX_TOOL_RESULT_IMAGE_BUDGET_BYTES,
+                        }),
+                        providerPathLabel: 'Codex path',
+                    }));
+                }
+                out.push({
+                    type: 'input_text',
+                    text: `[image omitted: request image budget exceeded bytes=${String(item.byteLength)} budget=${String(tool_result_image_ingest_1.CODEX_TOOL_RESULT_IMAGE_BUDGET_BYTES)}]`,
+                });
+                continue;
+            }
+            const bytesResult = await (0, tool_result_image_ingest_1.readToolResultImageBytesSafe)(item.artifact);
+            if (bytesResult.kind === 'missing') {
+                if (onToolResultImageIngest) {
+                    await onToolResultImageIngest((0, tool_result_image_ingest_1.buildToolResultImageIngest)({
+                        requestContext,
+                        toolCallId: msg.id,
+                        toolName: msg.name,
+                        artifact: item.artifact,
+                        disposition: 'filtered_missing',
+                        providerPathLabel: 'Codex path',
+                    }));
+                }
                 out.push({ type: 'input_text', text: `[image missing: ${item.artifact.relPath}]` });
                 continue;
             }
+            if (bytesResult.kind === 'read_failed') {
+                if (onToolResultImageIngest) {
+                    await onToolResultImageIngest((0, tool_result_image_ingest_1.buildToolResultImageIngest)({
+                        requestContext,
+                        toolCallId: msg.id,
+                        toolName: msg.name,
+                        artifact: item.artifact,
+                        disposition: 'filtered_read_failed',
+                        detail: bytesResult.detail,
+                        providerPathLabel: 'Codex path',
+                    }));
+                }
+                out.push({ type: 'input_text', text: `[image unreadable: ${item.artifact.relPath}]` });
+                continue;
+            }
+            if (onToolResultImageIngest) {
+                await onToolResultImageIngest((0, tool_result_image_ingest_1.buildToolResultImageIngest)({
+                    requestContext,
+                    toolCallId: msg.id,
+                    toolName: msg.name,
+                    artifact: item.artifact,
+                    disposition: 'fed_native',
+                    providerPathLabel: 'Codex path',
+                }));
+            }
+            const bytes = bytesResult.bytes;
             out.push({
                 type: 'input_image',
                 image_url: (0, artifacts_1.bytesToDataUrl)({ mimeType: item.mimeType, bytes }),
@@ -373,7 +437,7 @@ async function buildCodexFunctionCallOutput(msg, limitChars) {
         ? limitCodexToolOutputItems(out, msg, limitChars)
         : limitCodexToolOutputText(msg.content, msg, limitChars);
 }
-async function buildCodexInput(context, providerConfig) {
+async function buildCodexInput(context, requestContext, providerConfig, onToolResultImageIngest) {
     const normalized = (0, tool_call_context_1.normalizeToolCallPairs)(context);
     const violation = (0, tool_call_context_1.findFirstToolCallAdjacencyViolation)(normalized);
     if (violation) {
@@ -388,6 +452,7 @@ async function buildCodexInput(context, providerConfig) {
     }
     const input = [];
     const toolResultMaxChars = (0, tool_output_limit_1.resolveProviderToolResultMaxChars)(providerConfig);
+    const allowedImageKeys = (0, tool_result_image_ingest_1.selectLatestToolResultImagesWithinBudget)(normalized, tool_result_image_ingest_1.CODEX_TOOL_RESULT_IMAGE_BUDGET_BYTES);
     for (const msg of normalized) {
         if (msg.type === 'func_call_msg') {
             input.push({
@@ -402,7 +467,7 @@ async function buildCodexInput(context, providerConfig) {
             input.push({
                 type: 'function_call_output',
                 call_id: msg.id,
-                output: await buildCodexFunctionCallOutput(msg, toolResultMaxChars),
+                output: await buildCodexFunctionCallOutput(msg, toolResultMaxChars, requestContext, allowedImageKeys, onToolResultImageIngest),
             });
             continue;
         }
@@ -413,11 +478,11 @@ async function buildCodexInput(context, providerConfig) {
     }
     return input;
 }
-async function buildCodexRequest(providerConfig, agent, instructions, funcTools, requestContext, context) {
+async function buildCodexRequest(providerConfig, agent, instructions, funcTools, requestContext, context, onToolResultImageIngest) {
     if (!agent.model) {
         throw new Error(`Internal error: Model is undefined for agent '${agent.id}'`);
     }
-    const input = await buildCodexInput(context, providerConfig);
+    const input = await buildCodexInput(context, requestContext, providerConfig, onToolResultImageIngest);
     // Provider isolation rule: request construction must only read Codex-native params here.
     const codexParams = agent.model_params?.codex;
     const parallelToolCalls = codexParams?.parallel_tool_calls ?? true;
@@ -473,7 +538,7 @@ class CodexGen {
             defaultModel: agent.model,
             loadPrompt: codexAuth.loadCodexPromptSync,
         });
-        const payload = await buildCodexRequest(providerConfig, agent, instructions, funcTools, requestContext, context);
+        const payload = await buildCodexRequest(providerConfig, agent, instructions, funcTools, requestContext, context, receiver.toolResultImageIngest);
         let sayingStarted = false;
         let thinkingStarted = false;
         let sawOutputText = false;

package/dist/llm/gen/failure-classifier.js

@@ -231,6 +231,16 @@ function isOpenAiLikeRateLimitFailure(error) {
     if (lowerMessage.includes('rate limit')) {
         return true;
     }
+    if (lowerMessage.includes('concurrency limit exceeded')) {
+        return true;
+    }
+    if (lowerMessage.includes('concurrent request limit')) {
+        return true;
+    }
+    if (lowerMessage.includes('concurrency limit') &&
+        (lowerMessage.includes('retry later') || lowerMessage.includes('try again later'))) {
+        return true;
+    }
     if (lowerMessage.includes('requests per min')) {
         return true;
     }

package/dist/llm/gen/openai-compatible.d.ts

@@ -15,13 +15,16 @@ import type { Team } from '../../team';
 import type { FuncTool } from '../../tool';
 import type { ChatMessage, ProviderConfig } from '../client';
 import { type LlmBatchResult, type LlmFailureDisposition, type LlmGenerator, type LlmRequestContext, type LlmStreamReceiver, type LlmStreamResult } from '../gen';
-export declare function buildOpenAiCompatibleRequestMessagesWrapper(systemPrompt: string, context: ChatMessage[], options?: {
+export declare function buildOpenAiCompatibleRequestMessagesWrapper(systemPrompt: string, context: ChatMessage[], requestContextOrOptions?: LlmRequestContext | {
+    reasoningContentMode?: boolean;
+    providerConfig?: ProviderConfig;
+}, optionsMaybe?: {
     reasoningContentMode?: boolean;
     providerConfig?: ProviderConfig;
 }): Promise<ChatCompletionMessageParam[]>;
 export declare class OpenAiCompatibleGen implements LlmGenerator {
     get apiType(): string;
     classifyFailure(error: unknown): LlmFailureDisposition | undefined;
-    genToReceiver(providerConfig: ProviderConfig, agent: Team.Member, systemPrompt: string, funcTools: FuncTool[], _requestContext: LlmRequestContext, context: ChatMessage[], receiver: LlmStreamReceiver, genseq: number, abortSignal?: AbortSignal): Promise<LlmStreamResult>;
-    genMoreMessages(providerConfig: ProviderConfig, agent: Team.Member, systemPrompt: string, funcTools: FuncTool[], _requestContext: LlmRequestContext, context: ChatMessage[], genseq: number, abortSignal?: AbortSignal): Promise<LlmBatchResult>;
+    genToReceiver(providerConfig: ProviderConfig, agent: Team.Member, systemPrompt: string, funcTools: FuncTool[], requestContext: LlmRequestContext, context: ChatMessage[], receiver: LlmStreamReceiver, genseq: number, abortSignal?: AbortSignal): Promise<LlmStreamResult>;
+    genMoreMessages(providerConfig: ProviderConfig, agent: Team.Member, systemPrompt: string, funcTools: FuncTool[], requestContext: LlmRequestContext, context: ChatMessage[], genseq: number, abortSignal?: AbortSignal): Promise<LlmBatchResult>;
 }