@inflexa-ai/harness 0.19.0 → 0.19.2

package/dist/execution/run-canceler.js

@@ -0,0 +1,130 @@
+/**
+ * RunCanceler — the encapsulated external-cancel path for an `executeAnalysis` run.
+ *
+ * A cancelled DBOS workflow can never execute another step, so the body's own
+ * terminal block (`collectAndComplete`) is unreachable on external cancel and
+ * nothing converges: the run row stays `running` (blocking same-plan relaunch
+ * through `queryActiveRun`'s active-status dedup), pending step rows never
+ * sweep, the running charge never closes, and the run mandate is never
+ * revoked. This module is the host-side counterpart: cancel the engine's
+ * parent + children, then converge each ledger the workflow would have.
+ *
+ * The engine capability stays internal — the production default is the
+ * purger's `DBOSClient`-backed child-cascading cancel, which needs no launched
+ * engine (see `dbos-workflow-purger.ts` for why the static `DBOS` facade is
+ * the wrong tool here). `cancelWorkflows` is an optional test seam over it.
+ *
+ * No stream writes: `DBOS.writeStream` is body-only, so hosts synthesize the
+ * terminal frame read-side.
+ */
+import { createNoopLogger } from "../lib/console-logger.js";
+import { unwrapOrThrow } from "../lib/result.js";
+import { markRunCanceledIfActive, queryRun } from "../state/runs.js";
+import { queryStepsByRun, sweepPendingStepExecutions } from "../state/step-executions.js";
+import { createDbosWorkflowPurger } from "./dbos-workflow-purger.js";
+/** The reason stamped on the run row and the revoke — the same literal
+ * `collectAndComplete` records for a cancel it observes from inside. */
+const EXTERNAL_CANCEL_REASON = "external_cancel";
+export class UnknownRunError extends Error {
+    runId;
+    constructor(runId) {
+        super(`no run exists for id ${runId}`);
+        this.runId = runId;
+        this.name = "UnknownRunError";
+    }
+}
+const TERMINAL = new Set(["completed", "partial", "failed", "canceled"]);
+export function createRunCanceler(deps) {
+    const { pool, runCharge, runAuthorizer } = deps;
+    const logger = (deps.logger ?? createNoopLogger()).named("run-canceler");
+    // Lazy so a canceler built for a host that never cancels constructs no client;
+    // one purger per factory (its client registers pool handlers — see its comment).
+    let engineCancel = deps.cancelWorkflows;
+    const cancelWorkflows = async (workflowIds) => {
+        engineCancel ??= (() => {
+            const purger = createDbosWorkflowPurger({ pool, logger });
+            return async (ids) => {
+                unwrapOrThrow(await purger.cancel([...ids]));
+            };
+        })();
+        await engineCancel(workflowIds);
+    };
+    /** Persisted child workflow ids of steps not yet completed. */
+    const incompleteChildIds = async (runId) => unwrapOrThrow(await queryStepsByRun(pool, runId)).flatMap((step) => step.childWorkflowId !== null && step.completedAt === null ? [step.childWorkflowId] : []);
+    return {
+        async cancel(runId, session) {
+            const row = unwrapOrThrow(await queryRun(pool, runId));
+            if (row === null)
+                throw new UnknownRunError(runId);
+            const workflowId = runId;
+            if (TERMINAL.has(row.status)) {
+                return {
+                    runId,
+                    workflowId,
+                    outcome: "already_terminal",
+                    finalStatus: row.status,
+                    converged: { steps: false, charge: false, mandate: false },
+                };
+            }
+            // The child-cascading cancel walks the engine's own parent ledger, which
+            // has a row the moment a child starts — that covers a child whose
+            // mark-running step has not yet committed its child_workflow_id. The
+            // persisted ids and the post-cancel re-query are belt-and-braces over
+            // both ledgers' lag.
+            const knownChildren = await incompleteChildIds(runId);
+            await cancelWorkflows([workflowId, ...knownChildren]);
+            try {
+                const late = (await incompleteChildIds(runId)).filter((id) => !knownChildren.includes(id));
+                if (late.length > 0)
+                    await cancelWorkflows(late);
+            }
+            catch (err) {
+                logger.error("late-child cancel sweep failed", { runId, ...logger.errorFields(err) });
+            }
+            // Convergence — each phase isolated so one failure skips nothing after it.
+            let transitioned = false;
+            try {
+                transitioned = unwrapOrThrow(await markRunCanceledIfActive(pool, runId, EXTERNAL_CANCEL_REASON));
+            }
+            catch (err) {
+                logger.error("markRunCanceledIfActive failed", { runId, ...logger.errorFields(err) });
+            }
+            let steps = false;
+            try {
+                unwrapOrThrow(await sweepPendingStepExecutions(pool, runId));
+                steps = true;
+            }
+            catch (err) {
+                logger.error("pending-step sweep failed", { runId, ...logger.errorFields(err) });
+            }
+            // Best-effort: the billing authority self-heals (defensive open + stale
+            // reaper), so a failed close loses attribution only.
+            let charge = false;
+            try {
+                await runCharge.close({ analysisId: row.analysisId, runId, reason: "canceled", session });
+                charge = true;
+            }
+            catch (err) {
+                logger.error("running-charge close failed", { runId, ...logger.errorFields(err) });
+            }
+            let mandate = row.mandateJti === null;
+            if (row.mandateJti !== null) {
+                try {
+                    await runAuthorizer.revokeByJti({ jti: row.mandateJti, auth: session.auth }, EXTERNAL_CANCEL_REASON);
+                    mandate = true;
+                }
+                catch (err) {
+                    logger.error("mandate revoke failed", { runId, jti: row.mandateJti, ...logger.errorFields(err) });
+                }
+            }
+            // No transition means the run reached a terminal state on its own (or the
+            // write failed) — report the row as it stands rather than claiming `canceled`.
+            let finalStatus = "canceled";
+            if (!transitioned) {
+                const reread = unwrapOrThrow(await queryRun(pool, runId));
+                finalStatus = reread?.status ?? "canceled";
+            }
+            return { runId, workflowId, outcome: "canceled", finalStatus, converged: { steps, charge, mandate } };
+        },
+    };
+}

package/dist/index.d.ts

@@ -5,14 +5,16 @@
  * the agent with its own seam realizations.
  */
 export { assembleCoreRuntime } from "./runtime/assemble.js";
-export type { CoreRuntime, CoreRuntimeDeps, CoreWorkflowDeps, RegisteredWorkflows, ConversationAssemblyDeps, SandboxStepCallable } from "./runtime/assemble.js";
+export type { CoreRuntime, CoreRuntimeDeps, CoreWorkflowDeps, RegisteredWorkflows, ConversationAssemblyDeps, SandboxStepCallable, MakeReportReferenceResolver, ReportResolverScope, } from "./runtime/assemble.js";
+export type { ReferenceResolver, ReportSnapshot, ResolvedValue } from "./report-model/reference-resolver.js";
+export type { ResolvePageAsset } from "./tools/report-session/index.js";
 export type { ThreadAgentResolver, UnregisteredThreadType } from "./runtime/assemble.js";
 export { bootHarness } from "./runtime/boot.js";
 export type { BootHarnessDeps, BootedHarness } from "./runtime/boot.js";
 export { createConversationAgent } from "./agents/conversation-agent.js";
 export type { ConversationAgentDeps } from "./agents/conversation-agent.js";
 export { createLocalRunAuthorizer } from "./auth/local-run-authorizer.js";
-export type { RunAuthorizer, AuthorizeRunInput, RunAuthorization } from "./execution/run-authorizer.js";
+export type { RunAuthorizer, AuthorizeRunInput, RunAuthorization, RevokeByJtiRef } from "./execution/run-authorizer.js";
 export { createNoopBillingResolver } from "./billing/noop-resolver.js";
 export type { ResolveBilling, ResolvableSession, BillingMap, BillingHeaders, BillingFetchResult } from "./billing/resolver.js";
 export { createNoopRunCharge } from "./billing/noop-run-charge.js";
@@ -30,6 +32,8 @@ export type { RunLauncher, LaunchOptions } from "./execution/run-launcher.js";
 export { createDbosWorkflowPurger } from "./execution/dbos-workflow-purger.js";
 export type { DbosWorkflowPurgerDeps } from "./execution/dbos-workflow-purger.js";
 export type { WorkflowPurger } from "./execution/workflow-purger.js";
+export { createRunCanceler, UnknownRunError } from "./execution/run-canceler.js";
+export type { RunCanceler, RunCancelerDeps, CancelRunResult } from "./execution/run-canceler.js";
 export { defineTool, isToolError } from "./tools/define-tool.js";
 export type { Tool, ToolDefinition, ToolContext, ToolError } from "./tools/define-tool.js";
 export { normalizeDetail, DETAIL_MAX_LENGTH } from "./loop/tool-detail.js";
@@ -136,7 +140,7 @@ export type { ValidatePlanOptions } from "./schemas/validate-plan.js";
 export type { ValidationResult } from "./schemas/validate-plan.js";
 export { upsertPlan, loadPlan } from "./state/plans.js";
 export type { UpsertPlanInput } from "./state/plans.js";
-export { insertRun, reserveRunById, queryActiveRun, queryActiveRunsByAnalysis, updateRunStatus, queryRun, queryRunsByAnalysis, RunDedupCollisionError, RunIdentityCollisionError, } from "./state/runs.js";
+export { insertRun, reserveRunById, queryActiveRun, queryActiveRunsByAnalysis, updateRunStatus, markRunCanceledIfActive, queryRun, queryRunsByAnalysis, RunDedupCollisionError, RunIdentityCollisionError, } from "./state/runs.js";
 export type { InsertRunInput, RunReservation } from "./state/runs.js";
 export { queryStepsByRun } from "./state/step-executions.js";
 export type { CortexRunRow, StepExecutionRow, RunStatus } from "./state/schema.js";

package/dist/index.js

@@ -31,6 +31,11 @@ export { createDbosRunLauncher } from "./execution/dbos-run-launcher.js";
 // Seam: workflow-ledger reclamation — cancels and deletes an analysis's rows in
 // the durability engine's own ledger, so no caller names an engine type.
 export { createDbosWorkflowPurger } from "./execution/dbos-workflow-purger.js";
+// Run cancellation — the encapsulated external-cancel path: engine cancel
+// (parent + children) plus convergence of the run row, pending step rows,
+// running charge, and run mandate. Hosts call this from their cancel route
+// instead of hand-rolling `DBOS.cancelWorkflow` + partial cleanup.
+export { createRunCanceler, UnknownRunError } from "./execution/run-canceler.js";
 // Tool primitive.
 export { defineTool, isToolError } from "./tools/define-tool.js";
 // Tool-call-detail authoring aids for an embedder that contributes a host tool.
@@ -224,7 +229,7 @@ export { validatePlan } from "./schemas/validate-plan.js";
 // caller-derived id and inserts-if-absent (the deterministic-id intake path);
 // the run-state calls mirror the chat trigger's reserve → dedup → status flow.
 export { upsertPlan, loadPlan } from "./state/plans.js";
-export { insertRun, reserveRunById, queryActiveRun, queryActiveRunsByAnalysis, updateRunStatus, queryRun, queryRunsByAnalysis, RunDedupCollisionError, RunIdentityCollisionError, } from "./state/runs.js";
+export { insertRun, reserveRunById, queryActiveRun, queryActiveRunsByAnalysis, updateRunStatus, markRunCanceledIfActive, queryRun, queryRunsByAnalysis, RunDedupCollisionError, RunIdentityCollisionError, } from "./state/runs.js";
 export { queryStepsByRun } from "./state/step-executions.js";
 // Host-neutral reference-data catalog and shared installation receipts. Each
 // artifact names the third-party upstream it is fetched from; embedders own all

package/dist/lib/chrome.d.ts

@@ -11,8 +11,22 @@ import type { Logger } from "./logger.js";
 export interface ChromeConfig {
     /** Operational logging seam; omitted falls back to no-op. */
     readonly logger?: Logger;
+    /**
+     * The endpoint of the Chrome sidecar. The connection is out of process, thus the sidecar has its own
+     * filesystem and its own network position. A `file://` URL resolves against the filesystem of the
+     * sidecar and never against the filesystem of the harness host. A caller that navigates to a workspace
+     * file must run a sidecar that mounts the same workspace tree at the same path.
+     *
+     * An absent value means that the composition gives no browser at all. A caller that needs a look must
+     * report that condition, not attempt a connection.
+     */
     readonly browserUrl?: string;
     readonly maxPages?: number;
 }
+/**
+ * Whether the config names a browser to connect to. A composition with no endpoint has no eyes, and a
+ * caller reads that up front instead of failing once for each attempted connection.
+ */
+export declare function hasBrowserUrl(cfg: ChromeConfig): boolean;
 export declare function getBrowser(browserUrl?: string, injected?: Logger): Promise<Browser>;
 export declare function withPage<T>(cfg: ChromeConfig, fn: (page: Page, context: BrowserContext) => Promise<T>): Promise<T>;

package/dist/lib/chrome.js

@@ -8,6 +8,13 @@
  */
 import puppeteer from "puppeteer-core";
 import { createNoopLogger } from "./console-logger.js";
+/**
+ * Whether the config names a browser to connect to. A composition with no endpoint has no eyes, and a
+ * caller reads that up front instead of failing once for each attempted connection.
+ */
+export function hasBrowserUrl(cfg) {
+    return cfg.browserUrl !== undefined && cfg.browserUrl.trim().length > 0;
+}
 let browser;
 let connecting;
 let semaphore;

package/dist/lib/page-capture.d.ts

@@ -0,0 +1,47 @@
+/**
+ * The one page capture over the Chrome sidecar.
+ *
+ * A capture navigates to a page URL, collects the console errors and the failed requests, waits for the
+ * readiness signal of the page, and gives back a base64 screenshot. Two tools capture a report page: the
+ * preview snapshot of the old report path and the eyes tool of a report session. Both read this module.
+ *
+ * The readiness contract is the reason for one shared body. The renderer emits the event name and the
+ * sentinel name, and `report-render/page.ts` owns both constants together with the two budgets. This module
+ * imports them, thus a rename or a retime reaches the waiter at compile time. A second copy of the wait
+ * script would instead degrade every capture to the readiness timeout, and no type would break.
+ *
+ * The capture speaks the throw protocol, because the chrome connection does. A caller that needs a typed
+ * outcome guards the call.
+ */
+import { type ChromeConfig } from "./chrome.js";
+/** One request that the page could not load, with the reason that the browser gave. */
+export interface FailedRequest {
+    url: string;
+    reason: string;
+}
+/** The picture and the faults of one page capture. */
+export interface PageCapture {
+    screenshotBase64: string;
+    consoleErrors: string[];
+    failedRequests: FailedRequest[];
+}
+/**
+ * The capture seam. It navigates to a page URL, and it gives back the screenshot and the faults. The seam
+ * speaks the throw protocol, because the chrome connection does. A test injects a seam that reads no
+ * browser, thus a tool orchestration runs with no chrome sidecar.
+ */
+export type CapturePage = (url: string) => Promise<PageCapture>;
+/** The extra settle steps that one call site needs. A call site that needs neither passes nothing. */
+export interface CaptureOptions {
+    /** A CSS selector to wait for after the readiness signal. The wait is best-effort. */
+    readonly waitForSelector?: string;
+    /** An extra settle time in milliseconds after the readiness signal, for a late paint. */
+    readonly waitMs?: number;
+}
+/**
+ * Capture one page: the screenshot, the console errors, and the failed requests.
+ *
+ * The readiness wait is best-effort. A page that never signals still captures at the readiness budget, thus
+ * a broken page gives a picture that shows what broke.
+ */
+export declare function capturePage(chrome: ChromeConfig, url: string, options?: CaptureOptions): Promise<PageCapture>;

package/dist/lib/page-capture.js

@@ -0,0 +1,83 @@
+/**
+ * The one page capture over the Chrome sidecar.
+ *
+ * A capture navigates to a page URL, collects the console errors and the failed requests, waits for the
+ * readiness signal of the page, and gives back a base64 screenshot. Two tools capture a report page: the
+ * preview snapshot of the old report path and the eyes tool of a report session. Both read this module.
+ *
+ * The readiness contract is the reason for one shared body. The renderer emits the event name and the
+ * sentinel name, and `report-render/page.ts` owns both constants together with the two budgets. This module
+ * imports them, thus a rename or a retime reaches the waiter at compile time. A second copy of the wait
+ * script would instead degrade every capture to the readiness timeout, and no type would break.
+ *
+ * The capture speaks the throw protocol, because the chrome connection does. A caller that needs a typed
+ * outcome guards the call.
+ */
+import { withPage } from "./chrome.js";
+import { PAGE_NAV_TIMEOUT_MS, PAGE_READY_TIMEOUT_MS, THEME_READY_EVENT, THEME_READY_SENTINEL } from "../report-render/page.js";
+/** The budget of the optional selector wait. The wait is best-effort, thus a miss captures the page anyway. */
+const SELECTOR_TIMEOUT_MS = 5_000;
+/**
+ * The body of the readiness wait, in the browser context. The sentinel arm resolves a page that dispatched
+ * the event before this wait registered, thus a plain listener never blocks forever. The timer arm bounds a
+ * page that never signals.
+ *
+ * The browser runs this body as source text. Thus the body reads no module binding, and each value that it
+ * needs arrives as a parameter. The page evaluation sends a parameter as data, thus no value becomes code.
+ */
+function waitForThemeReady(sentinel, event, timeout) {
+    return new Promise((resolve) => {
+        // The sentinel name arrives as a value, thus no declared property of the global object describes it.
+        if (window[sentinel]) {
+            resolve();
+            return;
+        }
+        const timer = setTimeout(() => {
+            resolve();
+        }, timeout);
+        document.addEventListener(event, () => {
+            clearTimeout(timer);
+            resolve();
+        }, { once: true });
+    });
+}
+/**
+ * Capture one page: the screenshot, the console errors, and the failed requests.
+ *
+ * The readiness wait is best-effort. A page that never signals still captures at the readiness budget, thus
+ * a broken page gives a picture that shows what broke.
+ */
+export function capturePage(chrome, url, options = {}) {
+    return withPage(chrome, async (page) => {
+        const consoleErrors = [];
+        const failedRequests = [];
+        page.on("console", (msg) => {
+            if (msg.type() === "error")
+                consoleErrors.push(msg.text());
+        });
+        page.on("pageerror", (err) => consoleErrors.push(`pageerror: ${err.message}`));
+        page.on("requestfailed", (req) => {
+            failedRequests.push({ url: req.url(), reason: req.failure()?.errorText ?? "unknown" });
+        });
+        await page.goto(url, { waitUntil: "networkidle2", timeout: PAGE_NAV_TIMEOUT_MS });
+        // The two names and the budget ride as arguments, because the browser context has no module scope.
+        await page.evaluate(waitForThemeReady, THEME_READY_SENTINEL, THEME_READY_EVENT, PAGE_READY_TIMEOUT_MS).catch(() => {
+            /* the picture captures as it stands */
+        });
+        if (options.waitForSelector !== undefined) {
+            await page.waitForSelector(options.waitForSelector, { timeout: SELECTOR_TIMEOUT_MS }).catch(() => {
+                /* the picture captures as it stands */
+            });
+        }
+        if (options.waitMs !== undefined && options.waitMs > 0) {
+            const settle = options.waitMs;
+            await new Promise((resolve) => setTimeout(resolve, settle));
+        }
+        const screenshot = await page.screenshot({ encoding: "base64", fullPage: false });
+        return {
+            screenshotBase64: typeof screenshot === "string" ? screenshot : Buffer.from(screenshot).toString("base64"),
+            consoleErrors,
+            failedRequests,
+        };
+    });
+}

package/dist/loop/run-agent.js

@@ -1,5 +1,5 @@
 import { randomUUID } from "node:crypto";
-import { jsonSchema, tool as aiTool } from "ai";
+import { jsonSchema, tool as aiTool, } from "ai";
 import { createNoopUsageRecorder } from "../billing/noop-usage-recorder.js";
 import { stripNulCharacters } from "../input-sanitization.js";
 import { createNoopLogger } from "../lib/console-logger.js";
@@ -9,7 +9,7 @@ import { classifyProviderError } from "../providers/errors.js";
 import { DEFAULT_PROMPT_CACHE, promptCacheProviderOptions } from "../providers/prompt-cache.js";
 import { resultStep } from "./run-step.js";
 import { AskRejectedError, UnavailableAsk } from "../tools/approval/contract.js";
-import { isToolError } from "../tools/define-tool.js";
+import { isToolError, readToolResultImage } from "../tools/define-tool.js";
 import { addChatUsage, hasReportedUsage, recordAgentRun } from "./metrics.js";
 import { computeDetail } from "./tool-detail.js";
 import { toolOutcomeForOutputType } from "./tool-outcome.js";
@@ -40,6 +40,10 @@ export async function runAgent(agent, initial, session, opts) {
     // `callPath` rides as an array rather than a joined string — the queryable form;
     // rendering `parent > child` is the sink's choice, not a published package's.
     const log = (opts.logger ?? createNoopLogger()).named("loop").with({ agentId: source.agentId, callPath: source.callPath });
+    // The picture of a tool result rides where the wire renders it. Absent flags
+    // mean "cannot carry", thus the loop degrades to text and records the drop.
+    // The collector belongs to this run, thus two loops never share one.
+    const encoding = { placement: imagePlacementFor(provider.capabilities), deferredImages: [], log };
     const toolsById = new Map(agent.tools.map((t) => [t.id, t]));
     const toolDefs = Object.fromEntries(agent.tools.map((t) => [
         t.id,
@@ -238,7 +242,7 @@ export async function runAgent(agent, initial, session, opts) {
             for (const [idx, tu] of earlier.entries()) {
                 await emit({ type: "tool-started", source, toolUseId: tu.toolCallId, name: tu.toolName, input: tu.input, ...detailField(earlierDetails[idx]) });
             }
-            const { results, durations } = await dispatchTools(earlier, toolsById, toolCtx, isFatalLoopError, runStep, formatStepName.tool);
+            const { results, durations } = await dispatchTools(earlier, toolsById, toolCtx, isFatalLoopError, runStep, formatStepName.tool, encoding);
             const errored = await settleRound(earlier, results, earlierDetails, durations);
             results.push(errorResult(trailing, TRUNCATED_TOOL_USE_ERROR));
             // The trailing call was never dispatched, but it reaches the model as an error
@@ -250,6 +254,7 @@ export async function runAgent(agent, initial, session, opts) {
             errored.push(trailing.toolName);
             log.debug("tool results returned errors", { iteration: i, tools: errored });
             messages.push({ role: "tool", content: results });
+            appendDeferredImages(messages, results, encoding.deferredImages);
             if (hasDenial(results))
                 return stopOnDenial(i);
             if (opts.resolved?.())
@@ -268,11 +273,12 @@ export async function runAgent(agent, initial, session, opts) {
         for (const [idx, tu] of toolCalls.entries()) {
             await emit({ type: "tool-started", source, toolUseId: tu.toolCallId, name: tu.toolName, input: tu.input, ...detailField(details[idx]) });
         }
-        const { results, durations } = await dispatchTools(toolCalls, toolsById, toolCtx, isFatalLoopError, runStep, formatStepName.tool);
+        const { results, durations } = await dispatchTools(toolCalls, toolsById, toolCtx, isFatalLoopError, runStep, formatStepName.tool, encoding);
         const errored = await settleRound(toolCalls, results, details, durations);
         if (errored.length > 0)
             log.debug("tool results returned errors", { iteration: i, tools: errored });
         messages.push({ role: "tool", content: results });
+        appendDeferredImages(messages, results, encoding.deferredImages);
         if (hasDenial(results))
             return stopOnDenial(i);
         if (opts.resolved?.())
@@ -385,6 +391,50 @@ function markLastLoopAssistant(messages, initialCount) {
         }
     }
 }
+/**
+ * Where a tool picture goes for this wire. The tool result wins, because it is
+ * the native place and the correlation to the tool call stays implicit. The user
+ * message is the fallback. When the wire renders neither, the loop drops the
+ * picture.
+ */
+function imagePlacementFor(capabilities) {
+    if (capabilities.imageToolResults === true)
+        return "tool-result";
+    if (capabilities.imageUserMessages === true)
+        return "user-message";
+    return "drop";
+}
+/**
+ * Append one user message that carries each deferred picture of the round, then
+ * empty the collector.
+ *
+ * The tool message must come directly after the assistant message with the
+ * tool calls. Thus a picture rides a separate message after the whole tool
+ * message, and one message batches the round. The parts obey the order of
+ * `results`, which is the order of the tool calls. Each deferred picture has a
+ * result of this round, because the collector fills during the dispatch and it
+ * empties here. A text part names the tool call of each picture, because the wire
+ * holds no structural link between a user message and a tool call.
+ *
+ * The message carries the synthetic marker, because a `user` message opens a
+ * conversation turn. An unmarked one is loop machinery that reads as a turn
+ * boundary, and it splits one stored turn in two.
+ */
+function appendDeferredImages(messages, results, deferredImages) {
+    if (deferredImages.length === 0)
+        return;
+    const byToolCallId = new Map(deferredImages.map((deferred) => [deferred.toolCallId, deferred]));
+    const content = [];
+    for (const result of results) {
+        const deferred = byToolCallId.get(result.toolCallId);
+        if (deferred === undefined)
+            continue;
+        content.push({ type: "text", text: `The picture of the tool result ${deferred.toolCallId} of ${deferred.toolName}.` });
+        content.push({ type: "file", mediaType: deferred.image.mediaType, data: { type: "data", data: deferred.image.base64 } });
+    }
+    messages.push(syntheticUserMessage(content));
+    deferredImages.length = 0;
+}
 /**
  * Dispatch one round of tool calls, and measure the time of each call.
  *
@@ -395,7 +445,7 @@ function markLastLoopAssistant(messages, initialCount) {
  * wrapper. The wrapper is part of what the call cost, and a cached replay of a step
  * is genuinely fast. A bracket inside the step would report a body that did not run.
  */
-async function dispatchTools(toolUses, toolsById, toolCtx, isFatalLoopError, runStep, toolStepName) {
+async function dispatchTools(toolUses, toolsById, toolCtx, isFatalLoopError, runStep, toolStepName, encoding) {
     const results = new Array(toolUses.length);
     // The array starts with holes, which read as `undefined`. The element type
     // says so, thus it agrees with what `settleRound` accepts. Every index is in
@@ -415,19 +465,19 @@ async function dispatchTools(toolUses, toolsById, toolCtx, isFatalLoopError, run
     }
     await Promise.all(stepTools.map(({ tu, idx }) => {
         const startedAt = performance.now();
-        return runStep(toolStepName(tu.toolName, tu.toolCallId), () => dispatchTool(tu, toolsById, toolCtx(tu), isFatalLoopError)).then((r) => {
+        return runStep(toolStepName(tu.toolName, tu.toolCallId), () => dispatchTool(tu, toolsById, toolCtx(tu), isFatalLoopError, encoding)).then((r) => {
             durations[idx] = elapsedMs(startedAt);
             results[idx] = r;
         });
     }));
     for (const { tu, idx } of workflowTools) {
         const startedAt = performance.now();
-        results[idx] = await dispatchTool(tu, toolsById, toolCtx(tu), isFatalLoopError);
+        results[idx] = await dispatchTool(tu, toolsById, toolCtx(tu), isFatalLoopError, encoding);
         durations[idx] = elapsedMs(startedAt);
     }
     for (const { tu, idx } of inlineTools) {
         const startedAt = performance.now();
-        results[idx] = await dispatchTool(tu, toolsById, toolCtx(tu), isFatalLoopError);
+        results[idx] = await dispatchTool(tu, toolsById, toolCtx(tu), isFatalLoopError, encoding);
         durations[idx] = elapsedMs(startedAt);
     }
     return { results, durations };
@@ -436,14 +486,14 @@ async function dispatchTools(toolUses, toolsById, toolCtx, isFatalLoopError, run
 function elapsedMs(startedAt) {
     return Math.round(performance.now() - startedAt);
 }
-async function dispatchTool(tu, toolsById, ctx, isFatalLoopError) {
+async function dispatchTool(tu, toolsById, ctx, isFatalLoopError, encoding) {
     const tool = toolsById.get(tu.toolName);
     if (tool === undefined) {
         return errorResult(tu, `unknown tool: ${tu.toolName}`);
     }
     const parsed = tool.inputSchema.safeParse(tu.input);
     if (parsed.success)
-        return execute(tu, tool, parsed.data, ctx, isFatalLoopError);
+        return execute(tu, tool, parsed.data, ctx, isFatalLoopError, encoding);
     // A complete JSON argument can arrive as a string behind function-call
     // markup or a code fence. Repair only makes the schema reachable — the
     // repaired value is validated in full, and the tool's own semantic checks
@@ -451,15 +501,15 @@ async function dispatchTool(tu, toolsById, ctx, isFatalLoopError) {
     const repairedInput = repairToolInput(tu.input, parsed.error);
     const repaired = repairedInput === undefined ? undefined : tool.inputSchema.safeParse(repairedInput);
     if (repaired?.success === true)
-        return execute(tu, tool, repaired.data, ctx, isFatalLoopError);
+        return execute(tu, tool, repaired.data, ctx, isFatalLoopError, encoding);
     return errorResult(tu, `input validation failed: ${formatZodIssues(parsed.error, tu.input)}`);
 }
-async function execute(tu, tool, input, ctx, isFatalLoopError) {
+async function execute(tu, tool, input, ctx, isFatalLoopError, encoding) {
     try {
         const output = await tool.execute(input, ctx);
         if (output.isErr())
             return errorResult(tu, toolErrorContent(output.error));
-        return successResult(tu, output.value);
+        return successResult(tu, output.value, encoding);
     }
     catch (err) {
         if (isFatalLoopError(err))
@@ -478,12 +528,51 @@ async function execute(tu, tool, input, ctx, isFatalLoopError) {
 function jsonValue(value) {
     return JSON.parse(JSON.stringify(value ?? null), (_key, v) => (typeof v === "string" ? stripNulCharacters(v) : v));
 }
-function successResult(toolCall, value) {
+/**
+ * Encode a tool ok value onto a tool-result message. A value with no picture
+ * stays a plain JSON result, byte-identical to a value that never carried one.
+ *
+ * A value that carries a picture splits: the JSON data goes to a text part, and
+ * the bytes ride as an image content block. Thus the model sees the picture, and
+ * the JSON text holds no bytes. When the wire carries no picture, the block drops
+ * and the text stays, because base64 text floods the context and the model cannot
+ * see it. The drop rides the log, thus an operator sees that the picture did not
+ * reach the model.
+ *
+ * A wire that renders a picture on a user message only gets the fallback: the
+ * result keeps its JSON text, and the picture goes to the collector of the round.
+ * The round then sends the bytes after the tool message.
+ */
+function successResult(toolCall, value, encoding) {
+    const jsonData = jsonValue(value);
+    const jsonOnly = {
+        type: "tool-result",
+        toolCallId: toolCall.toolCallId,
+        toolName: toolCall.toolName,
+        output: { type: "json", value: jsonData },
+    };
+    const image = readToolResultImage(value);
+    if (image === undefined)
+        return jsonOnly;
+    if (encoding.placement === "drop") {
+        encoding.log.warn("the wired provider carries no picture in a tool result, thus the picture is dropped", { toolName: toolCall.toolName });
+        return jsonOnly;
+    }
+    if (encoding.placement === "user-message") {
+        encoding.deferredImages.push({ toolCallId: toolCall.toolCallId, toolName: toolCall.toolName, image });
+        return jsonOnly;
+    }
     return {
         type: "tool-result",
         toolCallId: toolCall.toolCallId,
         toolName: toolCall.toolName,
-        output: { type: "json", value: jsonValue(value) },
+        output: {
+            type: "content",
+            value: [
+                { type: "text", text: JSON.stringify(jsonData) },
+                { type: "file", mediaType: image.mediaType, data: { type: "data", data: image.base64 } },
+            ],
+        },
     };
 }
 // NUL is stripped from the prose BEFORE it is serialized: past the stringify it

package/dist/memory/ai-sdk-message-storage.d.ts

@@ -1,4 +1,4 @@
-import { type ModelMessage } from "ai";
+import { type ModelMessage, type UserContent } from "ai";
 import { z } from "zod";
 export declare const SUPPORTED_AI_SDK_MAJOR = 7;
 export declare const StoredMessageEnvelopeSchema: z.ZodObject<{
@@ -50,8 +50,9 @@ export declare function markInterruptedMessage(message: ModelMessage): ModelMess
 /** Whether a client abort cut off this message's production (see {@link markInterruptedMessage}); an absent key reads as not interrupted. */
 export declare function isInterruptedMessage(message: ModelMessage): boolean;
 /**
- * Build a `user` message the LOOP synthesized rather than a human typing it — today, the nudge that
- * continues a reply the model truncated at its output-token limit.
+ * Build a `user` message the LOOP synthesized rather than a human typing it — the nudge that
+ * continues a reply the model truncated at its output-token limit, and the message that carries a
+ * tool picture that the tool result cannot carry.
  *
  * It has to carry the `user` role because the wire format requires a user turn after a truncated
  * assistant message, but it is not user input, and the difference is load-bearing: a `user` message is
@@ -59,11 +60,15 @@ export declare function isInterruptedMessage(message: ModelMessage): boolean;
  * splitting one turn into two for the token window and, worse, giving a tail-turn removal a cut point in
  * the middle of a turn. Marking it is what lets {@link isSyntheticUserMessage} keep those readers honest.
  *
+ * `content` takes prose or the parts of a user message, because a picture rides a file part and no
+ * string can carry one. The marker is the same either way: what makes a message synthetic is who wrote
+ * it, not what it holds.
+ *
  * This one is loop MACHINERY and is not rendered: showing it would put words on screen nobody typed.
  * An embedder recording out-of-band work — an analysis run's outcome — wants
  * {@link syntheticRecordMessage} instead, which is equally non-turn-opening but IS displayed.
  */
-export declare function syntheticUserMessage(text: string): ModelMessage;
+export declare function syntheticUserMessage(content: UserContent): ModelMessage;
 /**
  * Build a record of out-of-band work for an embedder to append between turns — an analysis run's
  * outcome, which the model should read on its next turn but which nobody said.