@adecore/agents 0.20.1 → 0.21.1

package/src/chat/codex-protocol.ts

@@ -1,3 +1,4 @@
+import { generatedImage } from './generated-image.ts';
 import type { ChatFileChange, ChatQuestion, ChatTurnLimit, UsageWindow } from '@adecore/agent-contracts';
 import { readCodexLimits } from '../usage/limits/normalize.ts';
 import { codexRuntimeMode } from '../providers/codex.ts';
@@ -632,7 +633,7 @@ export class CodexProtocol {
                     events.push({ type: 'question.requested', requestId: ref, questions, async: true });
                     return;
                 }
-                this.text(ref, str(item.text) ?? '', completed, events);
+                this.text(ref, str(item.text) ?? '', completed, events, str(item.phase) ?? undefined);
                 return;
             }
             case 'plan':
@@ -705,6 +706,33 @@ export class CodexProtocol {
             case 'dynamicToolCall':
                 this.tool(ref, str(item.tool) ?? 'tool', item.arguments ?? {}, completed, textOf(item.contentItems), item.success !== false, events);
                 return;
+            case 'imageGeneration': {
+                const revisedPrompt = str(item.revisedPrompt) ?? undefined;
+                const transparentBackground = typeof item.transparentBackground === 'boolean' ? item.transparentBackground : undefined;
+                const input = {
+                    ...(revisedPrompt === undefined ? {} : { revisedPrompt }),
+                    ...(transparentBackground === undefined ? {} : { transparentBackground })
+                };
+                if (!completed) {
+                    this.tool(ref, 'ImageGeneration', input, false, '', true, events);
+                    return;
+                }
+                try {
+                    if (item.failure || item.status !== 'completed') {
+                        throw new Error(
+                            isRecord(item.failure) ? (str(item.failure.message) ?? 'Image generation failed') : (str(item.failure) ?? 'Image generation failed')
+                        );
+                    }
+                    const image = generatedImage(str(item.result) ?? '', revisedPrompt);
+                    events.push({ type: 'image.generated', ref, ...image, ...input });
+                } catch (error) {
+                    this.tool(ref, 'ImageGeneration', input, true, error instanceof Error ? error.message : 'Image generation failed', false, events);
+                }
+                return;
+            }
+            case 'imageView':
+                this.tool(ref, 'ImageView', { path: str(item.path) ?? '' }, completed, '', true, events);
+                return;
             case 'webSearch': {
                 const action = isRecord(item.action) ? item.action : {};
                 this.tool(ref, 'WebSearch', { query: str(item.query) ?? str(action.query) ?? '' }, completed, '', true, events);
@@ -775,8 +803,8 @@ export class CodexProtocol {
         return null;
     }
 
-    private text(ref: string, text: string, completed: boolean, events: BackendEvent[]): void {
-        events.push(completed ? { type: 'text.done', ref, text } : { type: 'text.delta', ref, text });
+    private text(ref: string, text: string, completed: boolean, events: BackendEvent[], phase?: string): void {
+        events.push({ type: completed ? 'text.done' : 'text.delta', ref, text, ...(phase === undefined ? {} : { phase }) });
     }
 
     /*

package/src/chat/delta-coalescer.ts

@@ -21,7 +21,7 @@ export class DeltaCoalescer {
     }
 
     push(event: ChatEvent): void {
-        if (event.type !== 'delta') {
+        if (event.type !== 'delta' || event.uiQueries !== undefined) {
             this.flush();
             this.send(event);
             return;

package/src/chat/errors.ts

@@ -1,6 +1,8 @@
 import { CodedError } from '../coded-error.ts';
 
 export type ChatErrorCode =
+    | 'refused-query'
+    | 'stale-ui-query'
     | 'history-expired'
     | 'chat-not-found'
     // A record on disk this version cannot read, which is never written over.

package/src/chat/fake-codex.ts

@@ -197,6 +197,40 @@ export function fakeCodexWith(options: FakeCodexOptions): FakeCli {
                 turnCompleted('failed', "You've hit your usage limit.", 'usageLimitExceeded');
                 return;
             }
+            if (text.startsWith('image:')) {
+                const scenario = text.slice(6);
+                const entry = item('imageGeneration', {
+                    status: 'inProgress',
+                    result: '',
+                    savedPath: '/never-read-generated-image.png',
+                    revisedPrompt: 'A rabbit',
+                    transparentBackground: true
+                });
+                started(entry);
+                const result =
+                    scenario === 'empty'
+                        ? ''
+                        : scenario === 'invalid'
+                          ? 'not base64'
+                          : scenario === 'not-image'
+                            ? Buffer.from('text').toString('base64')
+                            : scenario === 'too-large'
+                              ? Buffer.alloc(10 * 1024 * 1024 + 1).toString('base64')
+                              : 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+jRZkAAAAASUVORK5CYII=';
+                const done = {
+                    ...entry,
+                    status: scenario === 'failed' ? 'failed' : 'completed',
+                    result,
+                    ...(scenario === 'failed' ? { failure: { message: 'Image generation failed' } } : {})
+                };
+                completed(done);
+                if (scenario === 'replay') {
+                    completed(done);
+                }
+                agentMessage('Here is the generated image.');
+                turnCompleted('completed');
+                return;
+            }
             if (text === 'slow') {
                 slow = true;
                 started(item('agentMessage', { text: '', phase: 'final_answer', memoryCitation: null, delivery: null, questions: null }));

package/src/chat/generated-image.ts

@@ -0,0 +1,88 @@
+import { ChatAttachmentUploadSchema, type ChatAttachmentUpload } from '@adecore/agent-contracts';
+
+interface ImageInfo {
+    mime: string;
+    extension: string;
+    width?: number;
+    height?: number;
+}
+
+function imageInfo(bytes: Buffer): ImageInfo | null {
+    if (
+        bytes.length >= 33 &&
+        bytes.subarray(0, 8).equals(Buffer.from([137, 80, 78, 71, 13, 10, 26, 10])) &&
+        bytes.readUInt32BE(8) === 13 &&
+        bytes.toString('ascii', 12, 16) === 'IHDR'
+    ) {
+        return { mime: 'image/png', extension: 'png', width: bytes.readUInt32BE(16), height: bytes.readUInt32BE(20) };
+    }
+    if (bytes.length >= 13 && /GIF8[79]a/.test(bytes.toString('ascii', 0, 6))) {
+        return { mime: 'image/gif', extension: 'gif', width: bytes.readUInt16LE(6), height: bytes.readUInt16LE(8) };
+    }
+    if (
+        bytes.length >= 20 &&
+        bytes.toString('ascii', 0, 4) === 'RIFF' &&
+        bytes.toString('ascii', 8, 12) === 'WEBP' &&
+        bytes.readUInt32LE(4) + 8 === bytes.length
+    ) {
+        const kind = bytes.toString('ascii', 12, 16);
+        if (kind === 'VP8X' && bytes.length >= 30) {
+            return { mime: 'image/webp', extension: 'webp', width: bytes.readUIntLE(24, 3) + 1, height: bytes.readUIntLE(27, 3) + 1 };
+        }
+        if (kind === 'VP8 ' && bytes.length >= 30 && bytes.subarray(23, 26).equals(Buffer.from([157, 1, 42]))) {
+            return { mime: 'image/webp', extension: 'webp', width: bytes.readUInt16LE(26) & 0x3fff, height: bytes.readUInt16LE(28) & 0x3fff };
+        }
+        if (kind === 'VP8L' && bytes.length >= 25 && bytes[20] === 0x2f) {
+            const size = bytes.readUInt32LE(21);
+            return { mime: 'image/webp', extension: 'webp', width: (size & 0x3fff) + 1, height: ((size >> 14) & 0x3fff) + 1 };
+        }
+        return null;
+    }
+    if (bytes.length >= 4 && bytes[0] === 255 && bytes[1] === 216 && bytes[2] === 255) {
+        let offset = 2;
+        while (offset + 4 <= bytes.length && bytes[offset] === 255) {
+            const marker = bytes[offset + 1]!;
+            if (marker === 218 || marker === 217) {
+                break;
+            }
+            if (marker === 255) {
+                offset++;
+                continue;
+            }
+            const size = bytes.readUInt16BE(offset + 2);
+            if (size < 2 || offset + 2 + size > bytes.length) {
+                return null;
+            }
+            if ([192, 193, 194, 195, 197, 198, 199, 201, 202, 203, 205, 206, 207].includes(marker) && size >= 8) {
+                return { mime: 'image/jpeg', extension: 'jpg', width: bytes.readUInt16BE(offset + 7), height: bytes.readUInt16BE(offset + 5) };
+            }
+            offset += size + 2;
+        }
+    }
+    return null;
+}
+
+export function generatedImage(data: string, prompt?: string): { upload: ChatAttachmentUpload; width?: number; height?: number } {
+    const parsed = ChatAttachmentUploadSchema.safeParse({ name: 'image', mime: 'application/octet-stream', data });
+    if (!parsed.success) {
+        throw new Error('The generated image is empty, invalid base64 or exceeds 10 MiB');
+    }
+    const bytes = Buffer.from(data, 'base64');
+    // Buffer's decoder accepts malformed padding and trailing bits; the protocol requires canonical base64.
+    if (bytes.toString('base64') !== data) {
+        throw new Error('The generated image contains invalid base64');
+    }
+    const info = imageInfo(bytes);
+    if (info === null || (info.width !== undefined && info.width === 0) || (info.height !== undefined && info.height === 0)) {
+        throw new Error('The generated bytes are not a supported image');
+    }
+    const stem =
+        (prompt ?? '')
+            .normalize('NFKD')
+            .replace(/[^a-zA-Z0-9]+/g, '-')
+            .replace(/^-|-$/g, '')
+            .slice(0, 80)
+            .toLowerCase() || 'generated-image';
+    const upload = { name: `${stem}.${info.extension}`, mime: info.mime, data };
+    return { upload, ...(info.width === undefined ? {} : { width: info.width }), ...(info.height === undefined ? {} : { height: info.height }) };
+}

package/src/chat/projector.ts

@@ -187,7 +187,7 @@ export class ThreadProjector {
                 break;
             case 'text.delta':
                 this.closeThinking(events);
-                this.appendText(this.itemId(generation, event.ref), event.text, events);
+                this.appendText(this.itemId(generation, event.ref), event.text, events, event.phase);
                 break;
             case 'text.done': {
                 if (event.parentRef) {
@@ -206,6 +206,12 @@ export class ThreadProjector {
                         createdAt: existing?.createdAt ?? this.now(),
                         turnId: existing?.turnId ?? info.activeTurnId,
                         text: event.text,
+                        ...(existing?.kind === 'assistant' ? { uiAnswers: existing.uiAnswers, uiQueries: existing.uiQueries } : {}),
+                        ...(event.phase === undefined
+                            ? existing?.kind === 'assistant' && existing.phase !== undefined
+                                ? { phase: existing.phase }
+                                : {}
+                            : { phase: event.phase }),
                         streaming: false
                     })
                 );
@@ -507,7 +513,7 @@ export class ThreadProjector {
         }
     }
 
-    private appendText(id: string, text: string, events: ChatEvent[]): void {
+    private appendText(id: string, text: string, events: ChatEvent[], phase?: string): void {
         const existing = this.thread.get(id);
         if (!existing) {
             events.push(
@@ -517,6 +523,7 @@ export class ThreadProjector {
                     createdAt: this.now(),
                     turnId: this.thread.info.activeTurnId,
                     text,
+                    ...(phase === undefined ? {} : { phase }),
                     streaming: true
                 })
             );

package/src/chat/readable-text.ts

@@ -0,0 +1,18 @@
+import type { ChatAssistantItem } from '@adecore/agent-contracts';
+import { uiFallbackText } from '@adecore/intelligent-ui/text';
+
+/*
+ * An assistant's words as another agent or a person outside the thread reads them: each UI block becomes
+ * its text fallback, with the first query values when they were frozen for the block as stored. It works
+ * from the stored block ranges, so it holds whatever fence language the host compiled.
+ */
+export function readableAssistantText(item: Pick<ChatAssistantItem, 'text' | 'ui' | 'uiQueries'>): string {
+    if (!item.ui?.length) {
+        return item.text;
+    }
+    const blocks = item.ui.map((block) => {
+        const frozen = item.uiQueries?.blocks[block.id];
+        return frozen && frozen.revision === block.revision ? { ...block, fallback: frozen.fallback } : block;
+    });
+    return uiFallbackText(item.text, blocks);
+}

package/src/chat/thread.ts

@@ -93,6 +93,12 @@ export class ChatThread {
                 break;
             case 'delta':
                 this.appendText(event.itemId, event.text);
+                if (event.uiQueries) {
+                    const item = this.items.get(event.itemId);
+                    if (item?.kind === 'assistant') {
+                        this.items.set(item.id, { ...item, uiQueries: event.uiQueries });
+                    }
+                }
                 break;
             case 'info':
                 this.info = event.info;