@truefoundry/assistant-ui-runtime 0.1.0-rc.1

package/src/modelMessageImageContent.ts

@@ -0,0 +1,193 @@
+import type { CompleteAttachment } from "@assistant-ui/core";
+import type {
+    ModelMessageEvent,
+    TurnEvent,
+    TurnStreamingEvent,
+} from "truefoundry-gateway-sdk/agents";
+import { isEventDelta, mergeEventDelta } from "truefoundry-gateway-sdk/agents";
+
+import type { AssistantContentPart } from "./modelMessageContent.js";
+
+export type ImageUrlContentPart = {
+    type: "image_url";
+    image_url: { url: string };
+};
+
+type ModelMessageContentPart =
+    | { type: "text"; text: string }
+    | { type: "refusal"; refusal: string }
+    | ImageUrlContentPart;
+
+type ContentBlockDelta = {
+    index: number;
+    delta:
+        | { type: "text"; text?: string }
+        | { type: "image_url"; image_url?: { url?: string } };
+};
+
+type ModelMessageDeltaWithContentBlocks = {
+    type: "model.message.delta";
+    id: string;
+    contentBlocks?: ContentBlockDelta[];
+    content_blocks?: ContentBlockDelta[];
+};
+
+function parseDataUriMime(data: string): string {
+    if (!data.startsWith("data:")) {
+        return "image/png";
+    }
+    const match = /^data:([^;,]+)/.exec(data);
+    return match?.[1] ?? "image/png";
+}
+
+function imageFilenameFromUrl(url: string, index: number): string {
+    const mimeType = parseDataUriMime(url);
+    const ext = mimeType.split("/")[1] ?? "png";
+    return `image-${index + 1}.${ext}`;
+}
+
+export function isImageUrlContentPart(
+    part: unknown,
+): part is ImageUrlContentPart {
+    return (
+        part != null &&
+        typeof part === "object" &&
+        (part as ImageUrlContentPart).type === "image_url" &&
+        typeof (part as ImageUrlContentPart).image_url?.url === "string"
+    );
+}
+
+function normalizeModelMessageContent(
+    message: ModelMessageEvent,
+): ModelMessageContentPart[] {
+    const { content } = message;
+    if (content == null) {
+        return [];
+    }
+    if (typeof content === "string") {
+        return content.length > 0 ? [{ type: "text", text: content }] : [];
+    }
+    return content as ModelMessageContentPart[];
+}
+
+function ensureModelMessageContentArray(message: ModelMessageEvent): void {
+    if (Array.isArray(message.content)) {
+        return;
+    }
+    message.content = normalizeModelMessageContent(message);
+}
+
+function mergeContentBlockDeltas(
+    message: ModelMessageEvent,
+    blocks: readonly ContentBlockDelta[],
+): void {
+    ensureModelMessageContentArray(message);
+    const content = message.content as ModelMessageContentPart[];
+
+    for (const block of blocks) {
+        const index = block.index;
+        while (content.length <= index) {
+            content.push({ type: "text", text: "" });
+        }
+
+        const delta = block.delta;
+        if (delta.type === "text") {
+            const existing = content[index];
+            if (existing?.type === "text") {
+                existing.text += delta.text ?? "";
+            } else {
+                content[index] = { type: "text", text: delta.text ?? "" };
+            }
+            continue;
+        }
+
+        if (delta.type !== "image_url") {
+            continue;
+        }
+
+        const chunk = delta.image_url?.url ?? "";
+        const existing = content[index];
+        if (isImageUrlContentPart(existing)) {
+            existing.image_url.url += chunk;
+        } else {
+            content[index] = { type: "image_url", image_url: { url: chunk } };
+        }
+    }
+}
+
+export function mergeStreamEventDelta(
+    base: TurnEvent,
+    delta: TurnStreamingEvent,
+): void {
+    if (!isEventDelta(delta)) {
+        return;
+    }
+
+    mergeEventDelta(base, delta);
+
+    if (base.type !== "model.message" || delta.type !== "model.message.delta") {
+        return;
+    }
+
+    const extended = delta as ModelMessageDeltaWithContentBlocks;
+    const blocks = extended.contentBlocks ?? extended.content_blocks;
+    if (blocks == null || blocks.length === 0) {
+        return;
+    }
+
+    mergeContentBlockDeltas(base, blocks);
+}
+
+export function imageUrlToAttachment(
+    url: string,
+    attachmentId: string,
+): CompleteAttachment {
+    const mimeType = parseDataUriMime(url);
+    return {
+        id: attachmentId,
+        type: "image",
+        name: imageFilenameFromUrl(url, 0),
+        contentType: mimeType,
+        status: { type: "complete" },
+        content: [{ type: "image", image: url, filename: imageFilenameFromUrl(url, 0) }],
+    };
+}
+
+export function imagePartToAssistantImage(url: string, index: number): AssistantContentPart {
+    return {
+        type: "image",
+        image: url,
+        filename: imageFilenameFromUrl(url, index),
+    };
+}
+
+export function extractImagePartsFromModelMessage(
+    message: ModelMessageEvent,
+): AssistantContentPart[] {
+    const parts: AssistantContentPart[] = [];
+    let imageIndex = 0;
+
+    for (const part of normalizeModelMessageContent(message)) {
+        if (!isImageUrlContentPart(part)) {
+            continue;
+        }
+        const url = part.image_url.url.trim();
+        if (url.length === 0) {
+            continue;
+        }
+        parts.push(imagePartToAssistantImage(url, imageIndex));
+        imageIndex += 1;
+    }
+
+    return parts;
+}
+
+export function extractImageUrlFromUserContentItem(
+    part: unknown,
+): string | undefined {
+    if (isImageUrlContentPart(part)) {
+        const url = part.image_url.url.trim();
+        return url.length > 0 ? url : undefined;
+    }
+    return undefined;
+}

package/src/requiredActionInputs.test.ts

@@ -0,0 +1,394 @@
+import { describe, expect, it } from "vitest";
+
+import { ROOT_THREAD_ID } from "./constants.js";
+import {
+    applyApprovalDecisionsToMessage,
+    collectApprovalInputs,
+} from "./toolApproval.js";
+import {
+    applyToolResponseToMessage,
+    collectResponseInputs,
+} from "./toolResponse.js";
+import {
+    collectRequiredActionInputs,
+    messageHasPendingRequiredActions,
+} from "./requiredActionInputs.js";
+
+describe("requiredActionInputs", () => {
+    describe("batched resume invariant", () => {
+        it("bundles approvals and responses once nothing is pending", () => {
+        const message = {
+            id: "turn-1-assistant",
+            role: "assistant" as const,
+            content: [
+                {
+                    type: "tool-call" as const,
+                    toolCallId: "approval-1",
+                    toolName: "bash",
+                    args: {},
+                    argsText: "{}",
+                    approval: { id: "approval-1", approved: true },
+                },
+                {
+                    type: "tool-call" as const,
+                    toolCallId: "question-1",
+                    toolName: "ask_user_question",
+                    args: {},
+                    argsText: "{}",
+                    interrupt: { type: "human" as const, payload: { question: "Q?" } },
+                    result: "answer",
+                },
+            ],
+            status: { type: "requires-action" as const, reason: "tool-calls" as const },
+            createdAt: new Date(),
+            metadata: {
+                unstable_state: null,
+                unstable_annotations: [],
+                unstable_data: [],
+                steps: [],
+                custom: {
+                    toolApprovalThreadId: ROOT_THREAD_ID,
+                    toolResponseThreadId: ROOT_THREAD_ID,
+                },
+            },
+        };
+
+        expect(messageHasPendingRequiredActions(message)).toBe(false);
+        expect(collectRequiredActionInputs(message)).toEqual([
+            {
+                type: "user.tool_approval",
+                threadId: ROOT_THREAD_ID,
+                toolCallId: "approval-1",
+                approval: { status: "allow" },
+            },
+            {
+                type: "user.tool_response",
+                threadId: ROOT_THREAD_ID,
+                toolCallId: "question-1",
+                content: "answer",
+            },
+        ]);
+    });
+
+    it("returns empty inputs while any required action is still pending", () => {
+        const pendingApproval = {
+            id: "turn-1-assistant",
+            role: "assistant" as const,
+            content: [
+                {
+                    type: "tool-call" as const,
+                    toolCallId: "approval-1",
+                    toolName: "bash",
+                    args: {},
+                    argsText: "{}",
+                    approval: { id: "approval-1" },
+                },
+                {
+                    type: "tool-call" as const,
+                    toolCallId: "question-1",
+                    toolName: "ask_user_question",
+                    args: {},
+                    argsText: "{}",
+                    interrupt: { type: "human" as const, payload: { question: "Q?" } },
+                    result: "answer",
+                },
+            ],
+            status: { type: "requires-action" as const, reason: "tool-calls" as const },
+            createdAt: new Date(),
+            metadata: {
+                unstable_state: null,
+                unstable_annotations: [],
+                unstable_data: [],
+                steps: [],
+                custom: {},
+            },
+        };
+
+        expect(messageHasPendingRequiredActions(pendingApproval)).toBe(true);
+        expect(collectRequiredActionInputs(pendingApproval)).toEqual([]);
+    });
+
+    it("collects nested thread ids independently", () => {
+        const message = {
+            id: "root-assistant",
+            role: "assistant" as const,
+            content: [
+                {
+                    type: "tool-call" as const,
+                    toolCallId: "spawn-1",
+                    toolName: "create_sub_agent",
+                    args: {},
+                    argsText: "{}",
+                    messages: [
+                        {
+                            id: "child-assistant",
+                            role: "assistant" as const,
+                            content: [
+                                {
+                                    type: "tool-call" as const,
+                                    toolCallId: "approval-sub",
+                                    toolName: "bash",
+                                    args: {},
+                                    argsText: "{}",
+                                    approval: { id: "approval-sub", approved: true },
+                                },
+                            ],
+                            status: {
+                                type: "requires-action" as const,
+                                reason: "tool-calls" as const,
+                            },
+                            createdAt: new Date(),
+                            metadata: {
+                                unstable_state: null,
+                                unstable_annotations: [],
+                                unstable_data: [],
+                                steps: [],
+                                custom: { toolApprovalThreadId: "child-1" },
+                            },
+                        },
+                    ],
+                },
+                {
+                    type: "tool-call" as const,
+                    toolCallId: "approval-root",
+                    toolName: "bash",
+                    args: {},
+                    argsText: "{}",
+                    approval: { id: "approval-root", approved: true },
+                },
+            ],
+            status: { type: "requires-action" as const, reason: "tool-calls" as const },
+            createdAt: new Date(),
+            metadata: {
+                unstable_state: null,
+                unstable_annotations: [],
+                unstable_data: [],
+                steps: [],
+                custom: { toolApprovalThreadId: ROOT_THREAD_ID },
+            },
+        };
+
+        expect(collectApprovalInputs(message, ROOT_THREAD_ID)).toEqual([
+            {
+                type: "user.tool_approval",
+                threadId: "child-1",
+                toolCallId: "approval-sub",
+                approval: { status: "allow" },
+            },
+            {
+                type: "user.tool_approval",
+                threadId: ROOT_THREAD_ID,
+                toolCallId: "approval-root",
+                approval: { status: "allow" },
+            },
+        ]);
+    });
+
+    it("collects both action types after approvals and responses are staged", () => {
+        const pending = {
+            id: "turn-1-assistant",
+            role: "assistant" as const,
+            content: [
+                {
+                    type: "tool-call" as const,
+                    toolCallId: "approval-1",
+                    toolName: "bash",
+                    args: {},
+                    argsText: "{}",
+                    approval: { id: "approval-1" },
+                },
+                {
+                    type: "tool-call" as const,
+                    toolCallId: "question-1",
+                    toolName: "ask_user_question",
+                    args: {},
+                    argsText: "{}",
+                    interrupt: {
+                        type: "human" as const,
+                        payload: { question: "Pick one", options: ["A", "B"] },
+                    },
+                },
+            ],
+            status: { type: "requires-action" as const, reason: "tool-calls" as const },
+            createdAt: new Date(),
+            metadata: {
+                unstable_state: null,
+                unstable_annotations: [],
+                unstable_data: [],
+                steps: [],
+                custom: {
+                    toolApprovalThreadId: ROOT_THREAD_ID,
+                    toolResponseThreadId: ROOT_THREAD_ID,
+                },
+            },
+        };
+
+        const decided = applyApprovalDecisionsToMessage(pending, {
+            approvalId: "approval-1",
+            approved: true,
+        });
+        expect(messageHasPendingRequiredActions(decided)).toBe(true);
+        expect(collectRequiredActionInputs(decided)).toEqual([]);
+
+        const answered = applyToolResponseToMessage(decided, {
+            toolCallId: "question-1",
+            content: "A",
+        });
+        expect(messageHasPendingRequiredActions(answered)).toBe(false);
+        expect(collectRequiredActionInputs(answered)).toEqual([
+            {
+                type: "user.tool_approval",
+                threadId: ROOT_THREAD_ID,
+                toolCallId: "approval-1",
+                approval: { status: "allow" },
+            },
+            {
+                type: "user.tool_response",
+                threadId: ROOT_THREAD_ID,
+                toolCallId: "question-1",
+                content: "A",
+            },
+        ]);
+    });
+
+    it("does not emit approval-only inputs while a response is still pending", () => {
+        const approvalDecidedResponsePending = applyApprovalDecisionsToMessage(
+            {
+                id: "turn-1-assistant",
+                role: "assistant" as const,
+                content: [
+                    {
+                        type: "tool-call" as const,
+                        toolCallId: "approval-1",
+                        toolName: "bash",
+                        args: {},
+                        argsText: "{}",
+                        approval: { id: "approval-1" },
+                    },
+                    {
+                        type: "tool-call" as const,
+                        toolCallId: "question-1",
+                        toolName: "ask_user_question",
+                        args: {},
+                        argsText: "{}",
+                        interrupt: {
+                            type: "human" as const,
+                            payload: { question: "Pick one" },
+                        },
+                    },
+                ],
+                status: { type: "requires-action" as const, reason: "tool-calls" as const },
+                createdAt: new Date(),
+                metadata: {
+                    unstable_state: null,
+                    unstable_annotations: [],
+                    unstable_data: [],
+                    steps: [],
+                    custom: {
+                        toolApprovalThreadId: ROOT_THREAD_ID,
+                        toolResponseThreadId: ROOT_THREAD_ID,
+                    },
+                },
+            },
+            { approvalId: "approval-1", approved: true },
+        );
+
+        expect(collectApprovalInputs(approvalDecidedResponsePending, ROOT_THREAD_ID)).toEqual([
+            {
+                type: "user.tool_approval",
+                threadId: ROOT_THREAD_ID,
+                toolCallId: "approval-1",
+                approval: { status: "allow" },
+            },
+        ]);
+        expect(collectRequiredActionInputs(approvalDecidedResponsePending)).toEqual([]);
+    });
+
+    it("bundles nested multi-thread approvals and responses in one input array", () => {
+        const pending = {
+            id: "root-assistant",
+            role: "assistant" as const,
+            content: [
+                {
+                    type: "tool-call" as const,
+                    toolCallId: "spawn-1",
+                    toolName: "create_sub_agent",
+                    args: {},
+                    argsText: "{}",
+                    messages: [
+                        {
+                            id: "child-assistant",
+                            role: "assistant" as const,
+                            content: [
+                                {
+                                    type: "tool-call" as const,
+                                    toolCallId: "question-sub",
+                                    toolName: "ask_user_question",
+                                    args: {},
+                                    argsText: "{}",
+                                    interrupt: {
+                                        type: "human" as const,
+                                        payload: { question: "Sub?" },
+                                    },
+                                },
+                            ],
+                            status: {
+                                type: "requires-action" as const,
+                                reason: "tool-calls" as const,
+                            },
+                            createdAt: new Date(),
+                            metadata: {
+                                unstable_state: null,
+                                unstable_annotations: [],
+                                unstable_data: [],
+                                steps: [],
+                                custom: { toolResponseThreadId: "child-1" },
+                            },
+                        },
+                    ],
+                },
+                {
+                    type: "tool-call" as const,
+                    toolCallId: "approval-root",
+                    toolName: "bash",
+                    args: {},
+                    argsText: "{}",
+                    approval: { id: "approval-root" },
+                },
+            ],
+            status: { type: "requires-action" as const, reason: "tool-calls" as const },
+            createdAt: new Date(),
+            metadata: {
+                unstable_state: null,
+                unstable_annotations: [],
+                unstable_data: [],
+                steps: [],
+                custom: { toolApprovalThreadId: ROOT_THREAD_ID },
+            },
+        };
+
+        const answered = applyToolResponseToMessage(
+            applyApprovalDecisionsToMessage(pending, {
+                approvalId: "approval-root",
+                approved: true,
+            }),
+            { toolCallId: "question-sub", content: "sub-answer" },
+        );
+
+        expect(collectRequiredActionInputs(answered)).toEqual([
+            {
+                type: "user.tool_approval",
+                threadId: ROOT_THREAD_ID,
+                toolCallId: "approval-root",
+                approval: { status: "allow" },
+            },
+            {
+                type: "user.tool_response",
+                threadId: "child-1",
+                toolCallId: "question-sub",
+                content: "sub-answer",
+            },
+        ]);
+    });
+    });
+});

package/src/requiredActionInputs.ts

@@ -0,0 +1,65 @@
+import type { ThreadMessage } from "@assistant-ui/core";
+import type {
+    TurnInputItem,
+    UserToolApprovalEvent,
+    UserToolResponseEvent,
+} from "truefoundry-gateway-sdk/agents";
+
+import { ROOT_THREAD_ID } from "./constants.js";
+import {
+    collectApprovalInputs,
+    messageHasPendingApprovals,
+} from "./toolApproval.js";
+import {
+    collectResponseInputs,
+    messageHasPendingResponses,
+} from "./toolResponse.js";
+
+export type RequiredActionInput = Extract<
+    TurnInputItem,
+    UserToolApprovalEvent | UserToolResponseEvent
+>;
+
+export function messageHasPendingRequiredActions(
+    message: ThreadMessage | undefined,
+): boolean {
+    return (
+        messageHasPendingApprovals(message) || messageHasPendingResponses(message)
+    );
+}
+
+export function collectRequiredActionInputs(
+    message: ThreadMessage,
+    defaultThreadId: string = ROOT_THREAD_ID,
+): RequiredActionInput[] {
+    if (messageHasPendingRequiredActions(message)) {
+        return [];
+    }
+    return [
+        ...collectApprovalInputs(message, defaultThreadId),
+        ...collectResponseInputs(message, defaultThreadId),
+    ];
+}
+
+export function isRequiredActionInput(
+    item: TurnInputItem,
+): item is RequiredActionInput {
+    return (
+        item.type === "user.tool_approval" || item.type === "user.tool_response"
+    );
+}
+
+export function findPausedAssistantMessage(
+    messages: readonly ThreadMessage[],
+): Extract<ThreadMessage, { role: "assistant" }> | undefined {
+    for (let i = messages.length - 1; i >= 0; i--) {
+        const candidate = messages[i];
+        if (
+            candidate?.role === "assistant" &&
+            candidate.status?.type === "requires-action"
+        ) {
+            return candidate;
+        }
+    }
+    return undefined;
+}