@adecore/agents 0.20.0 → 0.21.0

package/dist/chat/codex-backend.js

@@ -39,6 +39,7 @@ export class CodexBackend {
     acknowledgesTurns = true;
     promptId = null;
     launch;
+    imageWrites = null;
     host;
     protocol;
     client;
@@ -114,7 +115,7 @@ export class CodexBackend {
             this.emit(event);
         }
     }
-    sendTurn(input) {
+    turnInput(input) {
         const images = input.attachments.filter((attachment) => attachmentImageMime(attachment) !== null);
         if (images.length > 0 && this.imageInputSupported === false) {
             throw new Error(`${this.launch.modelName} does not support image input. Choose a model that accepts images.`);
@@ -140,18 +141,40 @@ export class CodexBackend {
         if (note !== '') {
             parts.push(`\n\n${note}`);
         }
+        return [...textInput(parts.join('')), ...images.map((attachment) => ({ type: 'localImage', path: attachment.path }))];
+    }
+    sendTurn(input) {
+        const nativeInput = this.turnInput(input);
         const effort = this.launch.selection.options.effort;
         const tier = codexServiceTier(this.launch.selection);
         this.turnUnnamed = true;
         this.promptId = input.promptId ?? null;
         this.request('turn/start', {
             threadId: this.threadId,
-            input: [...textInput(parts.join('')), ...images.map((attachment) => ({ type: 'localImage', path: attachment.path }))],
+            input: nativeInput,
             model: this.launch.selection.model,
             ...(typeof effort === 'string' ? { effort } : {}),
             ...(tier === null ? {} : { serviceTier: tier })
         }, () => this.acceptPrompt(input.promptId));
     }
+    async steerTurn(input) {
+        const transport = this.transport;
+        const turnId = this.protocol.turnId;
+        if (!transport || !turnId || this.turnUnnamed) {
+            return false;
+        }
+        try {
+            await transport.request('turn/steer', { threadId: this.threadId, expectedTurnId: turnId, input: this.turnInput(input) });
+            return true;
+        }
+        catch (error) {
+            const rejected = error instanceof Error && error.name === 'CodexRpcError';
+            if (rejected && /no active turn|not.*active|turn.*mismatch|expected.*turn|method not found/i.test(error.message)) {
+                return false;
+            }
+            throw error;
+        }
+    }
     acceptPrompt(promptId) {
         if (promptId !== undefined && this.promptId === promptId) {
             this.emit({ type: 'turn.accepted', promptId });
@@ -282,9 +305,9 @@ export class CodexBackend {
         this.request('turn/steer', { threadId: this.threadId, expectedTurnId: turnId, input: textInput(answer.text) });
         return true;
     }
-    /* Codex asks its async question once and waits; forgetting it locally is all a dismissal is. */
     dismissRequest(requestId) {
-        return this.protocol.dismissQuestion(requestId);
+        const answer = this.protocol.dismissQuestionAnswer(requestId);
+        return answer?.kind === 'respond' ? this.respond(requestId, answer.rpcId, answer.result) : this.protocol.dismissQuestion(requestId);
     }
     declineRequest(requestId, _message) {
         const answer = this.protocol.declineRequest(requestId);
@@ -299,7 +322,7 @@ export class CodexBackend {
     dispose() {
         const transport = this.transport;
         this.transport = null;
-        return transport?.dispose() ?? Promise.resolve();
+        return Promise.all([transport?.dispose(), this.imageWrites]).then(() => undefined);
     }
     /* A request whose failure the person has to hear about, since it is the turn that cannot go on. */
     request(method, params, accepted) {
@@ -359,6 +382,55 @@ export class CodexBackend {
             this.interruptOwed = false;
             this.promptId = null;
         }
+        if (event.type === 'image.generated' || this.imageWrites !== null) {
+            const pending = (this.imageWrites ?? Promise.resolve()).then(async () => {
+                if (event.type === 'image.generated') {
+                    await this.storeImage(event);
+                }
+                else {
+                    this.host.onEvent(event);
+                }
+            });
+            this.imageWrites = pending;
+            void pending
+                .finally(() => {
+                if (this.imageWrites === pending) {
+                    this.imageWrites = null;
+                }
+            })
+                .catch(() => undefined);
+            return;
+        }
         this.host.onEvent(event);
     }
+    async storeImage(event) {
+        const input = { revisedPrompt: event.revisedPrompt, transparentBackground: event.transparentBackground };
+        let attachment;
+        try {
+            if (!this.launch.saveGeneratedImage) {
+                throw new Error('This machine cannot save generated images');
+            }
+            attachment = await this.launch.saveGeneratedImage(event.ref, event.upload);
+        }
+        catch (error) {
+            this.host.onEvent({ type: 'tool.started', ref: event.ref, name: 'ImageGeneration', input, parentRef: null });
+            this.host.onEvent({ type: 'tool.done', ref: event.ref, output: errorText(error), state: 'error' });
+            return;
+        }
+        this.host.onEvent({
+            type: 'tool.started',
+            ref: event.ref,
+            name: 'ImageGeneration',
+            input: {
+                ...input,
+                attachment: {
+                    ...attachment,
+                    ...(event.width === undefined ? {} : { width: event.width }),
+                    ...(event.height === undefined ? {} : { height: event.height })
+                }
+            },
+            parentRef: null
+        });
+        this.host.onEvent({ type: 'tool.done', ref: event.ref, output: null, state: 'done' });
+    }
 }

package/dist/chat/codex-protocol.d.ts

@@ -29,6 +29,7 @@ export declare class CodexProtocol {
         result: unknown;
     } | null;
     questionAnswer(requestId: string, answers: Record<string, string>): CodexAnswer | null;
+    dismissQuestionAnswer(requestId: string): CodexAnswer | null;
     dismissQuestion(requestId: string): boolean;
     forgetPending(): void;
     declineRequest(requestId: string): {

package/dist/chat/codex-protocol.js

@@ -1,3 +1,4 @@
+import { generatedImage } from './generated-image.js';
 import { readCodexLimits } from '../usage/limits/normalize.js';
 import { codexRuntimeMode } from '../providers/codex.js';
 import { cleanTitle } from '../title-file.js';
@@ -315,6 +316,13 @@ export class CodexProtocol {
     /* The reply that settles an approval, or null when nothing waits under that id. */
     approvalDecision(requestId, decision) {
         const pending = this.pending.get(requestId);
+        if (pending?.type === 'permissions') {
+            this.pending.delete(requestId);
+            return {
+                rpcId: pending.rpcId,
+                result: { permissions: decision === 'deny' ? {} : pending.permissions, scope: decision === 'allow-always' ? 'session' : 'turn' }
+            };
+        }
         if (pending?.type === 'elicitation') {
             this.pending.delete(requestId);
             return { rpcId: pending.rpcId, result: elicitationAnswer(decision, pending.persist) };
@@ -346,6 +354,10 @@ export class CodexProtocol {
         }
         return { kind: 'respond', rpcId: pending.rpcId, result: { answers: byId } };
     }
+    dismissQuestionAnswer(requestId) {
+        const pending = this.pending.get(requestId);
+        return pending?.type === 'question' && pending.async === true && pending.rpcId !== null ? this.questionAnswer(requestId, {}) : null;
+    }
     /* Drops an asynchronous question the person walked away from; a blocking one has to be answered. */
     dismissQuestion(requestId) {
         const pending = this.pending.get(requestId);
@@ -427,8 +439,39 @@ export class CodexProtocol {
             if (questions.length === 0) {
                 return;
             }
-            this.pending.set(requestId, { ...this.owner(params), type: 'question', rpcId, questionIds: questions.map((question) => question.id) });
-            events.push({ type: 'question.requested', requestId, questions, ...(this.isChildRequest(params) ? { background: true } : {}) });
+            this.pending.set(requestId, {
+                ...this.owner(params),
+                type: 'question',
+                rpcId,
+                questionIds: questions.map((question) => question.id),
+                async: params.isBlocking === false
+            });
+            events.push({
+                type: 'question.requested',
+                requestId,
+                questions,
+                ...(params.isBlocking === false ? { async: true } : {}),
+                ...(this.isChildRequest(params) ? { background: true } : {})
+            });
+            return;
+        }
+        if (method === 'item/permissions/requestApproval') {
+            if (!isRecord(params.permissions)) {
+                return;
+            }
+            const permissions = params.permissions;
+            this.pending.set(requestId, { ...this.owner(params), type: 'permissions', rpcId, permissions });
+            events.push({
+                type: 'approval.requested',
+                requestId,
+                ref: str(params.itemId),
+                toolName: 'RequestPermissions',
+                input: { cwd: params.cwd, permissions },
+                description: this.requestDescription(params, str(params.reason)),
+                canAllowAlways: true,
+                allowAlways: { label: 'Allow for this session', description: 'Grant these permissions for the rest of this session.' },
+                ...(this.isChildRequest(params) ? { background: true } : {})
+            });
             return;
         }
         if (method === 'mcpServer/elicitation/request') {
@@ -525,7 +568,7 @@ export class CodexProtocol {
                     events.push({ type: 'question.requested', requestId: ref, questions, async: true });
                     return;
                 }
-                this.text(ref, str(item.text) ?? '', completed, events);
+                this.text(ref, str(item.text) ?? '', completed, events, str(item.phase) ?? undefined);
                 return;
             }
             case 'plan':
@@ -574,6 +617,32 @@ export class CodexProtocol {
             case 'dynamicToolCall':
                 this.tool(ref, str(item.tool) ?? 'tool', item.arguments ?? {}, completed, textOf(item.contentItems), item.success !== false, events);
                 return;
+            case 'imageGeneration': {
+                const revisedPrompt = str(item.revisedPrompt) ?? undefined;
+                const transparentBackground = typeof item.transparentBackground === 'boolean' ? item.transparentBackground : undefined;
+                const input = {
+                    ...(revisedPrompt === undefined ? {} : { revisedPrompt }),
+                    ...(transparentBackground === undefined ? {} : { transparentBackground })
+                };
+                if (!completed) {
+                    this.tool(ref, 'ImageGeneration', input, false, '', true, events);
+                    return;
+                }
+                try {
+                    if (item.failure || item.status !== 'completed') {
+                        throw new Error(isRecord(item.failure) ? (str(item.failure.message) ?? 'Image generation failed') : (str(item.failure) ?? 'Image generation failed'));
+                    }
+                    const image = generatedImage(str(item.result) ?? '', revisedPrompt);
+                    events.push({ type: 'image.generated', ref, ...image, ...input });
+                }
+                catch (error) {
+                    this.tool(ref, 'ImageGeneration', input, true, error instanceof Error ? error.message : 'Image generation failed', false, events);
+                }
+                return;
+            }
+            case 'imageView':
+                this.tool(ref, 'ImageView', { path: str(item.path) ?? '' }, completed, '', true, events);
+                return;
             case 'webSearch': {
                 const action = isRecord(item.action) ? item.action : {};
                 this.tool(ref, 'WebSearch', { query: str(item.query) ?? str(action.query) ?? '' }, completed, '', true, events);
@@ -641,8 +710,8 @@ export class CodexProtocol {
         }
         return null;
     }
-    text(ref, text, completed, events) {
-        events.push(completed ? { type: 'text.done', ref, text } : { type: 'text.delta', ref, text });
+    text(ref, text, completed, events, phase) {
+        events.push({ type: completed ? 'text.done' : 'text.delta', ref, text, ...(phase === undefined ? {} : { phase }) });
     }
     /*
      * An agent Codex spawned, as a spawnAgent call reports it. It works in a thread of its own, so this row

package/dist/chat/delta-coalescer.js

@@ -16,7 +16,7 @@ export class DeltaCoalescer {
         this.tickMs = tickMs;
     }
     push(event) {
-        if (event.type !== 'delta') {
+        if (event.type !== 'delta' || event.uiQueries !== undefined) {
             this.flush();
             this.send(event);
             return;

package/dist/chat/errors.d.ts

@@ -1,4 +1,4 @@
 import { CodedError } from '../coded-error.ts';
-export type ChatErrorCode = 'history-expired' | 'chat-not-found' | 'chat-unreadable' | 'chat-busy' | 'request-not-found' | 'chat-unsupported' | 'invalid-attachments' | 'subagent-not-found' | 'task-not-found' | 'transcript-missing' | 'transcript-format' | 'fork-failed' | 'turn-not-found' | 'item-not-found' | 'bookmark-not-found' | 'too-many-bookmarks' | 'visual-invalid' | 'visual-too-large';
+export type ChatErrorCode = 'refused-query' | 'stale-ui-query' | 'history-expired' | 'chat-not-found' | 'chat-unreadable' | 'chat-busy' | 'request-not-found' | 'chat-unsupported' | 'invalid-attachments' | 'subagent-not-found' | 'task-not-found' | 'transcript-missing' | 'transcript-format' | 'fork-failed' | 'turn-not-found' | 'item-not-found' | 'bookmark-not-found' | 'too-many-bookmarks' | 'visual-invalid' | 'visual-too-large';
 export declare class ChatError<TCode extends string = ChatErrorCode> extends CodedError<TCode> {
 }

package/dist/chat/fake-codex.js

@@ -169,6 +169,39 @@ export function fakeCodexWith(options) {
                 turnCompleted('failed', "You've hit your usage limit.", 'usageLimitExceeded');
                 return;
             }
+            if (text.startsWith('image:')) {
+                const scenario = text.slice(6);
+                const entry = item('imageGeneration', {
+                    status: 'inProgress',
+                    result: '',
+                    savedPath: '/never-read-generated-image.png',
+                    revisedPrompt: 'A rabbit',
+                    transparentBackground: true
+                });
+                started(entry);
+                const result = scenario === 'empty'
+                    ? ''
+                    : scenario === 'invalid'
+                        ? 'not base64'
+                        : scenario === 'not-image'
+                            ? Buffer.from('text').toString('base64')
+                            : scenario === 'too-large'
+                                ? Buffer.alloc(10 * 1024 * 1024 + 1).toString('base64')
+                                : 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+jRZkAAAAASUVORK5CYII=';
+                const done = {
+                    ...entry,
+                    status: scenario === 'failed' ? 'failed' : 'completed',
+                    result,
+                    ...(scenario === 'failed' ? { failure: { message: 'Image generation failed' } } : {})
+                };
+                completed(done);
+                if (scenario === 'replay') {
+                    completed(done);
+                }
+                agentMessage('Here is the generated image.');
+                turnCompleted('completed');
+                return;
+            }
             if (text === 'slow') {
                 slow = true;
                 started(item('agentMessage', { text: '', phase: 'final_answer', memoryCitation: null, delivery: null, questions: null }));

package/dist/chat/generated-image.d.ts

@@ -0,0 +1,6 @@
+import { type ChatAttachmentUpload } from '@adecore/agent-contracts';
+export declare function generatedImage(data: string, prompt?: string): {
+    upload: ChatAttachmentUpload;
+    width?: number;
+    height?: number;
+};

package/dist/chat/generated-image.js

@@ -0,0 +1,74 @@
+import { ChatAttachmentUploadSchema } from '@adecore/agent-contracts';
+function imageInfo(bytes) {
+    if (bytes.length >= 33 &&
+        bytes.subarray(0, 8).equals(Buffer.from([137, 80, 78, 71, 13, 10, 26, 10])) &&
+        bytes.readUInt32BE(8) === 13 &&
+        bytes.toString('ascii', 12, 16) === 'IHDR') {
+        return { mime: 'image/png', extension: 'png', width: bytes.readUInt32BE(16), height: bytes.readUInt32BE(20) };
+    }
+    if (bytes.length >= 13 && /GIF8[79]a/.test(bytes.toString('ascii', 0, 6))) {
+        return { mime: 'image/gif', extension: 'gif', width: bytes.readUInt16LE(6), height: bytes.readUInt16LE(8) };
+    }
+    if (bytes.length >= 20 &&
+        bytes.toString('ascii', 0, 4) === 'RIFF' &&
+        bytes.toString('ascii', 8, 12) === 'WEBP' &&
+        bytes.readUInt32LE(4) + 8 === bytes.length) {
+        const kind = bytes.toString('ascii', 12, 16);
+        if (kind === 'VP8X' && bytes.length >= 30) {
+            return { mime: 'image/webp', extension: 'webp', width: bytes.readUIntLE(24, 3) + 1, height: bytes.readUIntLE(27, 3) + 1 };
+        }
+        if (kind === 'VP8 ' && bytes.length >= 30 && bytes.subarray(23, 26).equals(Buffer.from([157, 1, 42]))) {
+            return { mime: 'image/webp', extension: 'webp', width: bytes.readUInt16LE(26) & 0x3fff, height: bytes.readUInt16LE(28) & 0x3fff };
+        }
+        if (kind === 'VP8L' && bytes.length >= 25 && bytes[20] === 0x2f) {
+            const size = bytes.readUInt32LE(21);
+            return { mime: 'image/webp', extension: 'webp', width: (size & 0x3fff) + 1, height: ((size >> 14) & 0x3fff) + 1 };
+        }
+        return null;
+    }
+    if (bytes.length >= 4 && bytes[0] === 255 && bytes[1] === 216 && bytes[2] === 255) {
+        let offset = 2;
+        while (offset + 4 <= bytes.length && bytes[offset] === 255) {
+            const marker = bytes[offset + 1];
+            if (marker === 218 || marker === 217) {
+                break;
+            }
+            if (marker === 255) {
+                offset++;
+                continue;
+            }
+            const size = bytes.readUInt16BE(offset + 2);
+            if (size < 2 || offset + 2 + size > bytes.length) {
+                return null;
+            }
+            if ([192, 193, 194, 195, 197, 198, 199, 201, 202, 203, 205, 206, 207].includes(marker) && size >= 8) {
+                return { mime: 'image/jpeg', extension: 'jpg', width: bytes.readUInt16BE(offset + 7), height: bytes.readUInt16BE(offset + 5) };
+            }
+            offset += size + 2;
+        }
+    }
+    return null;
+}
+export function generatedImage(data, prompt) {
+    const parsed = ChatAttachmentUploadSchema.safeParse({ name: 'image', mime: 'application/octet-stream', data });
+    if (!parsed.success) {
+        throw new Error('The generated image is empty, invalid base64 or exceeds 10 MiB');
+    }
+    const bytes = Buffer.from(data, 'base64');
+    // Buffer's decoder accepts malformed padding and trailing bits; the protocol requires canonical base64.
+    if (bytes.toString('base64') !== data) {
+        throw new Error('The generated image contains invalid base64');
+    }
+    const info = imageInfo(bytes);
+    if (info === null || (info.width !== undefined && info.width === 0) || (info.height !== undefined && info.height === 0)) {
+        throw new Error('The generated bytes are not a supported image');
+    }
+    const stem = (prompt ?? '')
+        .normalize('NFKD')
+        .replace(/[^a-zA-Z0-9]+/g, '-')
+        .replace(/^-|-$/g, '')
+        .slice(0, 80)
+        .toLowerCase() || 'generated-image';
+    const upload = { name: `${stem}.${info.extension}`, mime: info.mime, data };
+    return { upload, ...(info.width === undefined ? {} : { width: info.width }), ...(info.height === undefined ? {} : { height: info.height }) };
+}

package/dist/chat/projector.js

@@ -150,7 +150,7 @@ export class ThreadProjector {
                 break;
             case 'text.delta':
                 this.closeThinking(events);
-                this.appendText(this.itemId(generation, event.ref), event.text, events);
+                this.appendText(this.itemId(generation, event.ref), event.text, events, event.phase);
                 break;
             case 'text.done': {
                 if (event.parentRef) {
@@ -168,6 +168,12 @@ export class ThreadProjector {
                     createdAt: existing?.createdAt ?? this.now(),
                     turnId: existing?.turnId ?? info.activeTurnId,
                     text: event.text,
+                    ...(existing?.kind === 'assistant' ? { uiAnswers: existing.uiAnswers, uiQueries: existing.uiQueries } : {}),
+                    ...(event.phase === undefined
+                        ? existing?.kind === 'assistant' && existing.phase !== undefined
+                            ? { phase: existing.phase }
+                            : {}
+                        : { phase: event.phase }),
                     streaming: false
                 }));
                 break;
@@ -452,7 +458,7 @@ export class ThreadProjector {
             events.push(this.thread.upsert({ ...item, streaming: false, endedAt: this.now() }));
         }
     }
-    appendText(id, text, events) {
+    appendText(id, text, events, phase) {
         const existing = this.thread.get(id);
         if (!existing) {
             events.push(this.thread.upsert({
@@ -461,6 +467,7 @@ export class ThreadProjector {
                 createdAt: this.now(),
                 turnId: this.thread.info.activeTurnId,
                 text,
+                ...(phase === undefined ? {} : { phase }),
                 streaming: true
             }));
             return;

package/dist/chat/thread.js

@@ -82,6 +82,12 @@ export class ChatThread {
                 break;
             case 'delta':
                 this.appendText(event.itemId, event.text);
+                if (event.uiQueries) {
+                    const item = this.items.get(event.itemId);
+                    if (item?.kind === 'assistant') {
+                        this.items.set(item.id, { ...item, uiQueries: event.uiQueries });
+                    }
+                }
                 break;
             case 'info':
                 this.info = event.info;

package/dist/chat/ui-queries.d.ts

@@ -0,0 +1,38 @@
+import type { z } from 'zod';
+import type { ChatUiLinkPayload, ChatUiLinkReading, ChatAssistantItem, ChatInfo, ChatUiChoicePayload, ChatUiQueryPayload, ChatUiQueryReading } from '@adecore/agent-contracts';
+import { type UiValue } from '@adecore/intelligent-ui';
+import { type UiLinkTarget } from '@adecore/intelligent-ui/links';
+import type { ChatSession } from './chat-session.ts';
+export interface ChatUiSource {
+    args: z.ZodType;
+    result: z.ZodType;
+    authorize(info: ChatInfo, access: unknown, args: Record<string, UiValue>): Promise<void>;
+    read(info: ChatInfo, args: Record<string, UiValue>, signal: AbortSignal, access: unknown): Promise<unknown>;
+}
+export interface ChatUiHost {
+    capture(info: ChatInfo): Promise<unknown>;
+    sources: Readonly<Record<string, ChatUiSource>>;
+    link?(info: ChatInfo, access: unknown, target: UiLinkTarget): Promise<ChatUiLinkReading>;
+}
+export declare class ChatUiQueries {
+    readonly schemas: Record<string, z.ZodType>;
+    private readonly captures;
+    private readonly freezing;
+    private readonly cache;
+    private readonly reading;
+    private readonly attempts;
+    private readonly active;
+    private totalActive;
+    private readonly host;
+    constructor(host: ChatUiHost);
+    observe(session: ChatSession, item: ChatAssistantItem, final: boolean): void;
+    private access;
+    private storedBlock;
+    private stored;
+    query(session: ChatSession, payload: ChatUiQueryPayload): Promise<ChatUiQueryReading>;
+    choiceValues(session: ChatSession, payload: ChatUiChoicePayload | ChatUiLinkPayload): Promise<Record<string, unknown>>;
+    link(session: ChatSession, payload: ChatUiLinkPayload): Promise<ChatUiLinkReading>;
+    forget(chatId: string): void;
+    private linkReading;
+    private reason;
+}