@adecore/agents 0.20.1 → 0.21.1

package/src/chat/attachment-store.ts

@@ -1,5 +1,5 @@
-import { randomBytes } from 'node:crypto';
-import { mkdir, rm, writeFile } from 'node:fs/promises';
+import { createHash, randomBytes } from 'node:crypto';
+import { link, mkdir, readFile, rm, writeFile } from 'node:fs/promises';
 import { extname, join } from 'node:path';
 import { attachmentImageMime, type ChatAttachment, type ChatAttachmentUpload } from '@adecore/agent-contracts';
 
@@ -57,6 +57,42 @@ export class AttachmentStore {
         return { id, name: upload.name, mime, size: bytes.byteLength, path };
     }
 
+    async saveGenerated(chatId: string, ref: string, upload: ChatAttachmentUpload): Promise<ChatAttachment> {
+        const id = createHash('sha256').update(`codex-image:${ref}`).digest('hex');
+        const bytes = Buffer.from(upload.data, 'base64');
+        const folder = this.folderOf(chatId);
+        await mkdir(folder, { recursive: true, mode: 0o700 });
+        const path = join(folder, `${id}.${extensionFor('', upload.mime)}`);
+        const temporary = join(folder, `.${id}-${randomBytes(8).toString('hex')}.tmp`);
+        try {
+            await writeFile(temporary, bytes, { mode: 0o600, flag: 'wx' });
+            try {
+                // Publish complete bytes without replacing an image another replay has already saved.
+                await link(temporary, path);
+            } catch (error) {
+                if ((error as NodeJS.ErrnoException).code !== 'EEXIST') {
+                    throw error;
+                }
+                const previous = await readFile(path);
+                if (!previous.equals(bytes)) {
+                    throw new Error('The generated image id already belongs to another image');
+                }
+            }
+        } finally {
+            await rm(temporary, { force: true });
+        }
+        return { id, name: upload.name, mime: upload.mime, size: bytes.byteLength, path };
+    }
+
+    async copy(chatId: string, attachment: ChatAttachment): Promise<ChatAttachment> {
+        const bytes = await readFile(attachment.path);
+        const folder = this.folderOf(chatId);
+        await mkdir(folder, { recursive: true, mode: 0o700 });
+        const path = join(folder, `${createHash('sha256').update(attachment.id).digest('hex')}.${extensionFor(attachment.name, attachment.mime)}`);
+        await writeFile(path, bytes, { mode: 0o600 });
+        return { ...attachment, path };
+    }
+
     /* Everything a chat attached; used when the chat itself is deleted. */
     async removeAll(chatId: string): Promise<void> {
         await rm(this.folderOf(chatId), { recursive: true, force: true });

package/src/chat/backend.ts

@@ -1,5 +1,6 @@
 import type {
     ChatAttachment,
+    ChatAttachmentUpload,
     ChatFileChange,
     ChatQuestion,
     ChatSkill,
@@ -42,6 +43,7 @@ export interface BackendLaunch {
     folders?: readonly string[];
     // How the CLI is started; a test runs a fake in the same process, everything else spawns it.
     spawn?: SpawnChatProcess;
+    saveGeneratedImage?(ref: string, upload: ChatAttachmentUpload): Promise<ChatAttachment>;
 }
 
 export interface TurnInput {
@@ -78,9 +80,19 @@ export type BackendEvent =
     // The CLI renamed its thread, for a protocol that says so.
     | { type: 'title'; title: string }
     | { type: 'permissions'; permissionMode: string; effectiveRuntimeMode?: RuntimeMode }
-    | { type: 'text.delta'; ref: string; text: string }
+    | { type: 'text.delta'; ref: string; text: string; phase?: string }
     // `parentRef` is set for text a subagent wrote; it belongs to that agent's row, not to the thread.
-    | { type: 'text.done'; ref: string; text: string; parentRef?: string | null }
+    | { type: 'text.done'; ref: string; text: string; parentRef?: string | null; phase?: string }
+    // Bytes are consumed by the backend before anything reaches the thread or its log.
+    | {
+          type: 'image.generated';
+          ref: string;
+          upload: ChatAttachmentUpload;
+          width?: number;
+          height?: number;
+          revisedPrompt?: string;
+          transparentBackground?: boolean;
+      }
     // What the model thought before it answered; consecutive blocks become one thinking item.
     | { type: 'thinking.delta'; ref: string; text: string }
     | { type: 'thinking.done'; ref: string; text: string }

package/src/chat/chat-core.ts

@@ -2,6 +2,7 @@ import { homedir } from 'node:os';
 import { join } from 'node:path';
 import {
     ChatAttachmentUploadsSchema,
+    generatedImageAttachment,
     type AgentKind,
     type ChatAttachment,
     type ChatAttachResult,
@@ -15,6 +16,9 @@ import {
     type ChatInfo,
     type ChatItem,
     type ChatQueuedMessage,
+    type ChatUiChoicePayload,
+    type ChatUiQueryPayload,
+    type ChatUiLinkPayload,
     type ChatSkill,
     type ChatSubagentPayload,
     type ChatSubagentResult,
@@ -23,6 +27,7 @@ import {
     type ModelSelection,
     type RuntimeMode
 } from '@adecore/agent-contracts';
+import { ChatUiQueries, type ChatUiHost } from './ui-queries.ts';
 import { ClientSinks } from '../client-sinks.ts';
 import { errorText } from '../error-text.ts';
 import type { AgentEvent, AgentSink } from '../events.ts';
@@ -52,6 +57,7 @@ import type { CodexProcessSpec } from './codex-thread.ts';
 import type { CodexClientInfo } from './codex-transport.ts';
 import { ComposerPreferences } from './composer-preferences.ts';
 import { DeltaCoalescer } from './delta-coalescer.ts';
+import { readableAssistantText } from './readable-text.ts';
 import { ChatError } from './errors.ts';
 import { commandLabel, isBackgroundWork, runsInBackground, type BackgroundWork } from './background-work.ts';
 import { SubagentReader, type SubagentReaderOptions } from './subagent-reader.ts';
@@ -79,7 +85,8 @@ function isBookmarkable(item: ChatItem): item is BookmarkableItem {
 /* The start of a message on one line; a message of only attachments is its file names. */
 function excerptOf(item: BookmarkableItem): string {
     const files = item.kind === 'user' ? (item.attachments ?? []).map((attachment) => attachment.name).join(', ') : '';
-    return (item.text.trim() === '' ? files : item.text).replace(/\s+/g, ' ').trim();
+    const text = item.kind === 'assistant' ? readableAssistantText(item) : item.text;
+    return (text.trim() === '' ? files : text).replace(/\s+/g, ' ').trim();
 }
 
 // One process worked on the turn and one more may take it up after a restart; a loop of resumes could redo a command forever.
@@ -96,6 +103,9 @@ const DEFAULT_RESUME_WORDS: ResumeWords = {
 };
 
 export interface ChatCoreOptions {
+    intelligentUi?: ChatUiHost;
+    // The info string that marks a UI fence in a reply, for a host that keeps a name of its own.
+    uiFenceLanguage?: string;
     providers: ProviderRegistry;
     // Where the threads are kept; without it a chat lives as long as the process.
     store?: ChatStore;
@@ -168,6 +178,8 @@ export class ChatCore {
     /* Where Claude Code keeps the projects of its default account on this machine; empty when it has none. */
     readonly claudeProjectsDir: string;
     readonly composerPreferences = new ComposerPreferences();
+    private readonly uiQueries: ChatUiQueries | null;
+    private readonly uiFenceLanguage: string | undefined;
     private readonly checkpoints: TurnCheckpoints | null;
     private readonly instructions: string | null;
     private readonly skillIndex: SkillIndex;
@@ -181,7 +193,7 @@ export class ChatCore {
     // How big each record was the last time it went to disk, and the writes waiting for a big one.
     private readonly sizes = new Map<string, number>();
     private readonly waiting = new Map<string, ReturnType<typeof setTimeout>>();
-    // What the last record held that no event carries (preambles, the host's extras), as its JSON.
+    // What the last record held that no event carries (preambles, the host's extras, captured UI access), as its JSON.
     private readonly unlogged = new Map<string, string>();
     // The write in flight per chat, so the next one queues behind it instead of racing it.
     private readonly writes = new Map<string, Promise<void>>();
@@ -194,6 +206,8 @@ export class ChatCore {
     private readonly limitResume: ChatCoreOptions['limitResume'] | null;
 
     constructor(options: ChatCoreOptions) {
+        this.uiQueries = options.intelligentUi ? new ChatUiQueries(options.intelligentUi) : null;
+        this.uiFenceLanguage = options.uiFenceLanguage;
         this.providers = options.providers;
         this.store = options.store ?? null;
         this.bookmarks = options.bookmarks ?? null;
@@ -390,8 +404,11 @@ export class ChatCore {
         const references = this.referencesFor(payload.chatId);
         const session = new ChatSession({
             info,
+            ...(this.uiFenceLanguage ? { uiFenceLanguage: this.uiFenceLanguage } : {}),
+            ...(this.uiQueries ? { ui: { schemas: this.uiQueries.schemas, observe: (item, final) => this.uiQueries!.observe(session, item, final) } } : {}),
             items: stored?.items ?? [],
             preambles: stored?.preambles ?? [],
+            uiAccess: stored?.uiAccess ?? {},
             provider,
             command: this.commands[kind] ?? provider.command,
             ...(this.spawn ? { spawn: this.spawn } : {}),
@@ -400,6 +417,7 @@ export class ChatCore {
                 this.accounts?.launched?.(kind, chatAccount);
                 return env;
             },
+            saveGeneratedImage: (ref, upload) => this.attachments.saveGenerated(payload.chatId, ref, upload),
             instructions: () => this.instructionsFor(payload.chatId),
             resumeNote: () => this.resumeNoteFor(payload.chatId),
             folders: () => this.foldersFor(payload.chatId),
@@ -495,11 +513,16 @@ export class ChatCore {
         const log = this.logs.get(chatId)!;
         const events = since === undefined ? null : log.after(since);
         if (events !== null) {
-            return { info: session.info, items: [], events, seq: log.seq };
+            return { info: session.info, items: [], events: [...events, ...session.uiPreviews], seq: log.seq };
         }
-        return historyLimit === undefined
-            ? { ...session.thread.snapshot(), seq: log.seq }
-            : { info: session.info, ...session.thread.history(historyLimit), pending: session.thread.pending(), seq: log.seq };
+        const page = historyLimit === undefined ? session.thread.snapshot() : session.thread.history(historyLimit);
+        return {
+            ...page,
+            info: session.info,
+            items: session.previewItems(page.items),
+            ...(historyLimit === undefined ? {} : { pending: session.previewItems(session.thread.pending()) }),
+            seq: log.seq
+        };
     }
 
     /*
@@ -703,6 +726,34 @@ export class ChatCore {
         return session.sendInput(text, { ...extras, ...(attachments.length > 0 ? { attachments } : {}) });
     }
 
+    async linkUi(payload: ChatUiLinkPayload, clientId: string) {
+        if (!this.attached.get(payload.chatId)?.has(clientId) || !this.uiQueries) {
+            throw new ChatError('refused-query', 'Attach to the chat before resolving its UI links.');
+        }
+        return this.uiQueries.link(this.require(payload.chatId), payload);
+    }
+
+    async queryUi(payload: ChatUiQueryPayload, clientId: string) {
+        if (!this.attached.get(payload.chatId)?.has(clientId)) {
+            throw new ChatError('refused-query', 'Attach to the chat before reading its UI queries.');
+        }
+        if (!this.uiQueries) {
+            throw new ChatError('refused-query', 'This host does not provide UI query sources.');
+        }
+        return this.uiQueries.query(this.require(payload.chatId), payload);
+    }
+
+    async choose(payload: ChatUiChoicePayload, clientId: string): Promise<{ queued: boolean; turnId: string }> {
+        if (!this.attached.get(payload.chatId)?.has(clientId)) {
+            throw new ChatError('refused-query', 'Attach to the chat before answering its UI.');
+        }
+        const session = this.require(payload.chatId);
+        const queries = await this.uiQueries?.choiceValues(session, payload);
+        const result = session.choose(payload, queries);
+        await this.persisted(payload.chatId);
+        return result;
+    }
+
     /* The file behind an attachment id: what this chat's thread or queue says it is, or a visual's page. */
     attachment(chatId: string, id: string): ChatAttachment | null {
         const session = this.chats.get(chatId);
@@ -710,7 +761,8 @@ export class ChatCore {
             return null;
         }
         for (const item of session.thread.list()) {
-            const found = item.kind === 'user' ? item.attachments?.find((attachment) => attachment.id === id) : undefined;
+            const generated = generatedImageAttachment(item);
+            const found = item.kind === 'user' ? item.attachments?.find((attachment) => attachment.id === id) : generated?.id === id ? generated : null;
             if (found) {
                 return found;
             }
@@ -759,7 +811,7 @@ export class ChatCore {
             throw new ChatError('chat-busy', `Chat ${chatId} is still working on the previous message`);
         }
         await this.clearing(chatId);
-        session.clear(force);
+        const ended = session.clear(force);
         // Before the record is written, so what it takes along from the record is gone from the write as well.
         const cleared = this.cleared(chatId);
         // A debounced write still waiting holds the old thread and must not land after the empty one.
@@ -767,7 +819,7 @@ export class ChatCore {
         // Folded right away: every line before the reset describes a thread that is gone.
         await Promise.all([
             this.persistNow(chatId, true),
-            this.attachments.removeAll(chatId),
+            ended.then(() => this.attachments.removeAll(chatId)),
             this.bookmarks?.removeChat(chatId),
             this.visuals?.removeChat(chatId),
             cleared
@@ -812,8 +864,9 @@ export class ChatCore {
         const session = this.require(chatId);
         // A write already out would put the record back after the delete.
         const writing = this.writes.get(chatId);
-        void session.dispose();
+        const ended = session.dispose();
         this.subagents.releaseChat(chatId);
+        this.uiQueries?.forget(chatId);
         this.coalescers.get(chatId)?.dispose();
         this.coalescers.delete(chatId);
         this.logs.get(chatId)?.close();
@@ -829,6 +882,7 @@ export class ChatCore {
         const snapshot = session.thread.snapshot();
         this.forgotten(chatId);
         await writing;
+        await ended;
         await Promise.all([
             this.store?.delete(chatId),
             this.attachments.removeAll(chatId),
@@ -979,7 +1033,15 @@ export class ChatCore {
             const log = this.logs.get(chatId);
             // Not folded here: an older write still in flight may land after this one, and the log is what covers for it.
             try {
-                this.store.writeSync(chatId, info, items, { seq: log?.seq ?? 0, resetSeq: log?.resetSeq ?? 0 }, session.preambles, this.recordExtras(chatId));
+                this.store.writeSync(
+                    chatId,
+                    info,
+                    items,
+                    { seq: log?.seq ?? 0, resetSeq: log?.resetSeq ?? 0 },
+                    session.preambles,
+                    this.recordExtras(chatId),
+                    session.uiAccessRecord
+                );
             } catch (e) {
                 console.error(`Chat record for ${chatId} failed:`, errorText(e));
             }
@@ -1138,7 +1200,8 @@ export class ChatCore {
             // A delta held back is already in the thread, so it gets its seq before the snapshot says where it ends.
             this.coalescers.get(chatId)?.flush();
             const extras = this.recordExtras(chatId);
-            const unlogged = JSON.stringify([session.preambles, extras]);
+            const uiAccess = session.uiAccessRecord;
+            const unlogged = JSON.stringify([session.preambles, extras, uiAccess]);
             const folds = fold || log.size > COMPACT_ABOVE_BYTES || !log.onDisk;
             // The log already holds every change to the thread, so the whole record is rewritten only to fold
             // the log, on a chat's first write, or for what no event carries.
@@ -1147,7 +1210,7 @@ export class ChatCore {
             }
             const { info, items } = session.thread.snapshot();
             const at = { seq: log.seq, resetSeq: log.resetSeq };
-            this.sizes.set(chatId, await this.store.write(chatId, info, items, at, session.preambles, extras));
+            this.sizes.set(chatId, await this.store.write(chatId, info, items, at, session.preambles, extras, uiAccess));
             this.unlogged.set(chatId, unlogged);
             // A chat killed while the write was out has no log left to fold.
             if (this.logs.get(chatId) === log && folds) {
@@ -1162,6 +1225,14 @@ export class ChatCore {
     }
 
     private emit(chatId: string, event: ChatEvent): void {
+        if (event.type === 'delta' && event.ui !== undefined) {
+            // Previews follow all pending text but have no log position or observer side effects.
+            this.coalescers.get(chatId)?.flush();
+            for (const clientId of this.attached.get(chatId) ?? []) {
+                this.sinks.to(clientId, { event: 'chat.event', payload: { chatId, event } });
+            }
+            return;
+        }
         this.activity.set(chatId, Date.now());
         let coalescer = this.coalescers.get(chatId);
         if (!coalescer) {