@adecore/agents 0.20.1 → 0.21.1

package/dist/chat/attachment-store.d.ts

@@ -4,6 +4,8 @@ export declare class AttachmentStore {
     readonly dir: string;
     constructor(home: string);
     save(chatId: string, upload: ChatAttachmentUpload): Promise<ChatAttachment>;
+    saveGenerated(chatId: string, ref: string, upload: ChatAttachmentUpload): Promise<ChatAttachment>;
+    copy(chatId: string, attachment: ChatAttachment): Promise<ChatAttachment>;
     removeAll(chatId: string): Promise<void>;
     folderOf(chatId: string): string;
 }

package/dist/chat/attachment-store.js

@@ -1,5 +1,5 @@
-import { randomBytes } from 'node:crypto';
-import { mkdir, rm, writeFile } from 'node:fs/promises';
+import { createHash, randomBytes } from 'node:crypto';
+import { link, mkdir, readFile, rm, writeFile } from 'node:fs/promises';
 import { extname, join } from 'node:path';
 import { attachmentImageMime } from '@adecore/agent-contracts';
 // The extension the file gets on disk when its name does not carry a usable one.
@@ -50,6 +50,42 @@ export class AttachmentStore {
         await writeFile(path, bytes, { mode: 0o600 });
         return { id, name: upload.name, mime, size: bytes.byteLength, path };
     }
+    async saveGenerated(chatId, ref, upload) {
+        const id = createHash('sha256').update(`codex-image:${ref}`).digest('hex');
+        const bytes = Buffer.from(upload.data, 'base64');
+        const folder = this.folderOf(chatId);
+        await mkdir(folder, { recursive: true, mode: 0o700 });
+        const path = join(folder, `${id}.${extensionFor('', upload.mime)}`);
+        const temporary = join(folder, `.${id}-${randomBytes(8).toString('hex')}.tmp`);
+        try {
+            await writeFile(temporary, bytes, { mode: 0o600, flag: 'wx' });
+            try {
+                // Publish complete bytes without replacing an image another replay has already saved.
+                await link(temporary, path);
+            }
+            catch (error) {
+                if (error.code !== 'EEXIST') {
+                    throw error;
+                }
+                const previous = await readFile(path);
+                if (!previous.equals(bytes)) {
+                    throw new Error('The generated image id already belongs to another image');
+                }
+            }
+        }
+        finally {
+            await rm(temporary, { force: true });
+        }
+        return { id, name: upload.name, mime: upload.mime, size: bytes.byteLength, path };
+    }
+    async copy(chatId, attachment) {
+        const bytes = await readFile(attachment.path);
+        const folder = this.folderOf(chatId);
+        await mkdir(folder, { recursive: true, mode: 0o700 });
+        const path = join(folder, `${createHash('sha256').update(attachment.id).digest('hex')}.${extensionFor(attachment.name, attachment.mime)}`);
+        await writeFile(path, bytes, { mode: 0o600 });
+        return { ...attachment, path };
+    }
     /* Everything a chat attached; used when the chat itself is deleted. */
     async removeAll(chatId) {
         await rm(this.folderOf(chatId), { recursive: true, force: true });

package/dist/chat/backend.d.ts

@@ -1,4 +1,4 @@
-import type { ChatAttachment, ChatFileChange, ChatQuestion, ChatSkill, ChatSubagentUsage, ChatTurnLimit, ChatWorkflow, ModelSelection, RuntimeMode } from '@adecore/agent-contracts';
+import type { ChatAttachment, ChatAttachmentUpload, ChatFileChange, ChatQuestion, ChatSkill, ChatSubagentUsage, ChatTurnLimit, ChatWorkflow, ModelSelection, RuntimeMode } from '@adecore/agent-contracts';
 import type { LimitsUpdate } from '../usage/limits/normalize.ts';
 import type { SpawnChatProcess } from './chat-process.ts';
 export interface BackendLaunch {
@@ -14,6 +14,7 @@ export interface BackendLaunch {
     resumeNote: string | null;
     folders?: readonly string[];
     spawn?: SpawnChatProcess;
+    saveGeneratedImage?(ref: string, upload: ChatAttachmentUpload): Promise<ChatAttachment>;
 }
 export interface TurnInput {
     promptId?: string;
@@ -47,11 +48,21 @@ export type BackendEvent = {
     type: 'text.delta';
     ref: string;
     text: string;
+    phase?: string;
 } | {
     type: 'text.done';
     ref: string;
     text: string;
     parentRef?: string | null;
+    phase?: string;
+} | {
+    type: 'image.generated';
+    ref: string;
+    upload: ChatAttachmentUpload;
+    width?: number;
+    height?: number;
+    revisedPrompt?: string;
+    transparentBackground?: boolean;
 } | {
     type: 'thinking.delta';
     ref: string;

package/dist/chat/chat-core.d.ts

@@ -1,4 +1,5 @@
-import { type AgentKind, type ChatAttachment, type ChatAttachResult, type ChatAttachmentUpload, type ChatBookmark, type ChatCheckpointDiff, type ChatConfigurePayload, type ChatCreatePayload, type ChatEvent, type ChatHistoryResult, type ChatInfo, type ChatItem, type ChatQueuedMessage, type ChatSkill, type ChatSubagentPayload, type ChatSubagentResult, type ChatTurnItem, type ChatVisual, type ModelSelection, type RuntimeMode } from '@adecore/agent-contracts';
+import { type AgentKind, type ChatAttachment, type ChatAttachResult, type ChatAttachmentUpload, type ChatBookmark, type ChatCheckpointDiff, type ChatConfigurePayload, type ChatCreatePayload, type ChatEvent, type ChatHistoryResult, type ChatInfo, type ChatItem, type ChatQueuedMessage, type ChatUiChoicePayload, type ChatUiQueryPayload, type ChatUiLinkPayload, type ChatSkill, type ChatSubagentPayload, type ChatSubagentResult, type ChatTurnItem, type ChatVisual, type ModelSelection, type RuntimeMode } from '@adecore/agent-contracts';
+import { type ChatUiHost } from './ui-queries.ts';
 import { ClientSinks } from '../client-sinks.ts';
 import type { AgentEvent, AgentSink } from '../events.ts';
 import type { OutboxOutcome } from '../outbox/outbox-worker.ts';
@@ -25,6 +26,8 @@ export interface InterruptedRun {
     attempt: number;
 }
 export interface ChatCoreOptions {
+    intelligentUi?: ChatUiHost;
+    uiFenceLanguage?: string;
     providers: ProviderRegistry;
     store?: ChatStore;
     checkpoints?: TurnCheckpoints;
@@ -95,8 +98,21 @@ export declare class ChatCore {
                 mime: string;
                 size: number;
                 path: string;
+                width?: number | undefined;
+                height?: number | undefined;
             }[] | undefined;
             createdAt: number;
+            uiChoice?: {
+                itemId: string;
+                blockId: string;
+                revision: string;
+                choiceId: string;
+                label: string;
+                sourceAt: number;
+                older: boolean;
+                values?: Record<string, unknown> | undefined;
+                at: number;
+            } | undefined;
         }[] | undefined;
         queuePaused?: boolean | undefined;
         background?: {
@@ -174,6 +190,8 @@ export declare class ChatCore {
     protected readonly subagents: SubagentReader;
     readonly claudeProjectsDir: string;
     readonly composerPreferences: ComposerPreferences;
+    private readonly uiQueries;
+    private readonly uiFenceLanguage;
     private readonly checkpoints;
     private readonly instructions;
     private readonly skillIndex;
@@ -251,6 +269,44 @@ export declare class ChatCore {
         queued: boolean;
         turnId: string;
     }>;
+    linkUi(payload: ChatUiLinkPayload, clientId: string): Promise<{
+        state: "chip" | "plain";
+        target?: {
+            type: "File";
+            path: string;
+            line?: number | undefined;
+        } | {
+            type: "Diff";
+            path: string;
+        } | {
+            type: "Commit";
+            sha: string;
+        } | {
+            type: "Node";
+            id: string;
+        } | undefined;
+        cwd?: string | undefined;
+        viewId?: string | undefined;
+        projectId?: string | undefined;
+        staged?: boolean | undefined;
+        conflicted?: boolean | undefined;
+        relativePath?: string | undefined;
+        label?: string | undefined;
+        code?: string | undefined;
+        reason?: string | undefined;
+    }>;
+    queryUi(payload: ChatUiQueryPayload, clientId: string): Promise<{
+        state: "failed" | "fresh" | "refused";
+        value?: unknown;
+        readId?: string | undefined;
+        readAt: number;
+        code?: string | undefined;
+        reason?: string | undefined;
+    }>;
+    choose(payload: ChatUiChoicePayload, clientId: string): Promise<{
+        queued: boolean;
+        turnId: string;
+    }>;
     attachment(chatId: string, id: string): ChatAttachment | null;
     unqueue(chatId: string, messageId: string): ChatQueuedMessage;
     sendNow(chatId: string, messageId: string): void;

package/dist/chat/chat-core.js

@@ -1,6 +1,7 @@
 import { homedir } from 'node:os';
 import { join } from 'node:path';
-import { ChatAttachmentUploadsSchema } from '@adecore/agent-contracts';
+import { ChatAttachmentUploadsSchema, generatedImageAttachment } from '@adecore/agent-contracts';
+import { ChatUiQueries } from './ui-queries.js';
 import { ClientSinks } from '../client-sinks.js';
 import { errorText } from '../error-text.js';
 import { AccountError, definedEnv, isDefaultAccountOf, launchEnv, storedAccount } from '../providers/accounts/launch.js';
@@ -10,6 +11,7 @@ import { ChatLog, COMPACT_ABOVE_BYTES } from './chat-log.js';
 import { ChatSession } from './chat-session.js';
 import { ComposerPreferences } from './composer-preferences.js';
 import { DeltaCoalescer } from './delta-coalescer.js';
+import { readableAssistantText } from './readable-text.js';
 import { ChatError } from './errors.js';
 import { commandLabel, isBackgroundWork, runsInBackground } from './background-work.js';
 import { SubagentReader } from './subagent-reader.js';
@@ -24,7 +26,8 @@ function isBookmarkable(item) {
 /* The start of a message on one line; a message of only attachments is its file names. */
 function excerptOf(item) {
     const files = item.kind === 'user' ? (item.attachments ?? []).map((attachment) => attachment.name).join(', ') : '';
-    return (item.text.trim() === '' ? files : item.text).replace(/\s+/g, ' ').trim();
+    const text = item.kind === 'assistant' ? readableAssistantText(item) : item.text;
+    return (text.trim() === '' ? files : text).replace(/\s+/g, ' ').trim();
 }
 // One process worked on the turn and one more may take it up after a restart; a loop of resumes could redo a command forever.
 const MAX_ATTEMPTS = 2;
@@ -59,6 +62,8 @@ export class ChatCore {
     /* Where Claude Code keeps the projects of its default account on this machine; empty when it has none. */
     claudeProjectsDir;
     composerPreferences = new ComposerPreferences();
+    uiQueries;
+    uiFenceLanguage;
     checkpoints;
     instructions;
     skillIndex;
@@ -72,7 +77,7 @@ export class ChatCore {
     // How big each record was the last time it went to disk, and the writes waiting for a big one.
     sizes = new Map();
     waiting = new Map();
-    // What the last record held that no event carries (preambles, the host's extras), as its JSON.
+    // What the last record held that no event carries (preambles, the host's extras, captured UI access), as its JSON.
     unlogged = new Map();
     // The write in flight per chat, so the next one queues behind it instead of racing it.
     writes = new Map();
@@ -84,6 +89,8 @@ export class ChatCore {
     onInterruptedRun;
     limitResume;
     constructor(options) {
+        this.uiQueries = options.intelligentUi ? new ChatUiQueries(options.intelligentUi) : null;
+        this.uiFenceLanguage = options.uiFenceLanguage;
         this.providers = options.providers;
         this.store = options.store ?? null;
         this.bookmarks = options.bookmarks ?? null;
@@ -269,8 +276,11 @@ export class ChatCore {
         const references = this.referencesFor(payload.chatId);
         const session = new ChatSession({
             info,
+            ...(this.uiFenceLanguage ? { uiFenceLanguage: this.uiFenceLanguage } : {}),
+            ...(this.uiQueries ? { ui: { schemas: this.uiQueries.schemas, observe: (item, final) => this.uiQueries.observe(session, item, final) } } : {}),
             items: stored?.items ?? [],
             preambles: stored?.preambles ?? [],
+            uiAccess: stored?.uiAccess ?? {},
             provider,
             command: this.commands[kind] ?? provider.command,
             ...(this.spawn ? { spawn: this.spawn } : {}),
@@ -279,6 +289,7 @@ export class ChatCore {
                 this.accounts?.launched?.(kind, chatAccount);
                 return env;
             },
+            saveGeneratedImage: (ref, upload) => this.attachments.saveGenerated(payload.chatId, ref, upload),
             instructions: () => this.instructionsFor(payload.chatId),
             resumeNote: () => this.resumeNoteFor(payload.chatId),
             folders: () => this.foldersFor(payload.chatId),
@@ -362,11 +373,16 @@ export class ChatCore {
         const log = this.logs.get(chatId);
         const events = since === undefined ? null : log.after(since);
         if (events !== null) {
-            return { info: session.info, items: [], events, seq: log.seq };
+            return { info: session.info, items: [], events: [...events, ...session.uiPreviews], seq: log.seq };
         }
-        return historyLimit === undefined
-            ? { ...session.thread.snapshot(), seq: log.seq }
-            : { info: session.info, ...session.thread.history(historyLimit), pending: session.thread.pending(), seq: log.seq };
+        const page = historyLimit === undefined ? session.thread.snapshot() : session.thread.history(historyLimit);
+        return {
+            ...page,
+            info: session.info,
+            items: session.previewItems(page.items),
+            ...(historyLimit === undefined ? {} : { pending: session.previewItems(session.thread.pending()) }),
+            seq: log.seq
+        };
     }
     /*
      * `attach` with the chat's bookmarks and visuals. They are read before the client joins, so a
@@ -548,6 +564,31 @@ export class ChatCore {
         const attachments = await Promise.all(uploads.map((upload) => this.attachments.save(chatId, upload)));
         return session.sendInput(text, { ...extras, ...(attachments.length > 0 ? { attachments } : {}) });
     }
+    async linkUi(payload, clientId) {
+        if (!this.attached.get(payload.chatId)?.has(clientId) || !this.uiQueries) {
+            throw new ChatError('refused-query', 'Attach to the chat before resolving its UI links.');
+        }
+        return this.uiQueries.link(this.require(payload.chatId), payload);
+    }
+    async queryUi(payload, clientId) {
+        if (!this.attached.get(payload.chatId)?.has(clientId)) {
+            throw new ChatError('refused-query', 'Attach to the chat before reading its UI queries.');
+        }
+        if (!this.uiQueries) {
+            throw new ChatError('refused-query', 'This host does not provide UI query sources.');
+        }
+        return this.uiQueries.query(this.require(payload.chatId), payload);
+    }
+    async choose(payload, clientId) {
+        if (!this.attached.get(payload.chatId)?.has(clientId)) {
+            throw new ChatError('refused-query', 'Attach to the chat before answering its UI.');
+        }
+        const session = this.require(payload.chatId);
+        const queries = await this.uiQueries?.choiceValues(session, payload);
+        const result = session.choose(payload, queries);
+        await this.persisted(payload.chatId);
+        return result;
+    }
     /* The file behind an attachment id: what this chat's thread or queue says it is, or a visual's page. */
     attachment(chatId, id) {
         const session = this.chats.get(chatId);
@@ -555,7 +596,8 @@ export class ChatCore {
             return null;
         }
         for (const item of session.thread.list()) {
-            const found = item.kind === 'user' ? item.attachments?.find((attachment) => attachment.id === id) : undefined;
+            const generated = generatedImageAttachment(item);
+            const found = item.kind === 'user' ? item.attachments?.find((attachment) => attachment.id === id) : generated?.id === id ? generated : null;
             if (found) {
                 return found;
             }
@@ -599,7 +641,7 @@ export class ChatCore {
             throw new ChatError('chat-busy', `Chat ${chatId} is still working on the previous message`);
         }
         await this.clearing(chatId);
-        session.clear(force);
+        const ended = session.clear(force);
         // Before the record is written, so what it takes along from the record is gone from the write as well.
         const cleared = this.cleared(chatId);
         // A debounced write still waiting holds the old thread and must not land after the empty one.
@@ -607,7 +649,7 @@ export class ChatCore {
         // Folded right away: every line before the reset describes a thread that is gone.
         await Promise.all([
             this.persistNow(chatId, true),
-            this.attachments.removeAll(chatId),
+            ended.then(() => this.attachments.removeAll(chatId)),
             this.bookmarks?.removeChat(chatId),
             this.visuals?.removeChat(chatId),
             cleared
@@ -646,8 +688,9 @@ export class ChatCore {
         const session = this.require(chatId);
         // A write already out would put the record back after the delete.
         const writing = this.writes.get(chatId);
-        void session.dispose();
+        const ended = session.dispose();
         this.subagents.releaseChat(chatId);
+        this.uiQueries?.forget(chatId);
         this.coalescers.get(chatId)?.dispose();
         this.coalescers.delete(chatId);
         this.logs.get(chatId)?.close();
@@ -663,6 +706,7 @@ export class ChatCore {
         const snapshot = session.thread.snapshot();
         this.forgotten(chatId);
         await writing;
+        await ended;
         await Promise.all([
             this.store?.delete(chatId),
             this.attachments.removeAll(chatId),
@@ -802,7 +846,7 @@ export class ChatCore {
             const log = this.logs.get(chatId);
             // Not folded here: an older write still in flight may land after this one, and the log is what covers for it.
             try {
-                this.store.writeSync(chatId, info, items, { seq: log?.seq ?? 0, resetSeq: log?.resetSeq ?? 0 }, session.preambles, this.recordExtras(chatId));
+                this.store.writeSync(chatId, info, items, { seq: log?.seq ?? 0, resetSeq: log?.resetSeq ?? 0 }, session.preambles, this.recordExtras(chatId), session.uiAccessRecord);
             }
             catch (e) {
                 console.error(`Chat record for ${chatId} failed:`, errorText(e));
@@ -935,7 +979,8 @@ export class ChatCore {
             // A delta held back is already in the thread, so it gets its seq before the snapshot says where it ends.
             this.coalescers.get(chatId)?.flush();
             const extras = this.recordExtras(chatId);
-            const unlogged = JSON.stringify([session.preambles, extras]);
+            const uiAccess = session.uiAccessRecord;
+            const unlogged = JSON.stringify([session.preambles, extras, uiAccess]);
             const folds = fold || log.size > COMPACT_ABOVE_BYTES || !log.onDisk;
             // The log already holds every change to the thread, so the whole record is rewritten only to fold
             // the log, on a chat's first write, or for what no event carries.
@@ -944,7 +989,7 @@ export class ChatCore {
             }
             const { info, items } = session.thread.snapshot();
             const at = { seq: log.seq, resetSeq: log.resetSeq };
-            this.sizes.set(chatId, await this.store.write(chatId, info, items, at, session.preambles, extras));
+            this.sizes.set(chatId, await this.store.write(chatId, info, items, at, session.preambles, extras, uiAccess));
             this.unlogged.set(chatId, unlogged);
             // A chat killed while the write was out has no log left to fold.
             if (this.logs.get(chatId) === log && folds) {
@@ -955,6 +1000,14 @@ export class ChatCore {
         return next;
     }
     emit(chatId, event) {
+        if (event.type === 'delta' && event.ui !== undefined) {
+            // Previews follow all pending text but have no log position or observer side effects.
+            this.coalescers.get(chatId)?.flush();
+            for (const clientId of this.attached.get(chatId) ?? []) {
+                this.sinks.to(clientId, { event: 'chat.event', payload: { chatId, event } });
+            }
+            return;
+        }
         this.activity.set(chatId, Date.now());
         let coalescer = this.coalescers.get(chatId);
         if (!coalescer) {

package/dist/chat/chat-session.d.ts

@@ -1,6 +1,8 @@
-import type { ChatAttachment, ChatCheckpointDiff, ChatEvent, ChatInfo, ChatItem, ChatQueuedMessage, ChatSkill, ModelSelection, RuntimeMode } from '@adecore/agent-contracts';
+import type { ChatAttachment, ChatAssistantItem, ChatCheckpointDiff, ChatEvent, ChatInfo, ChatItem, ChatQueuedMessage, ChatSkill, ChatUiChoiceOrigin, ChatUiChoicePayload, ChatUiQueryState, ModelSelection, RuntimeMode } from '@adecore/agent-contracts';
+import type { UiCompileOptions } from '@adecore/intelligent-ui/compiler';
 import type { ChatProvider } from '../providers/provider.ts';
 import type { LimitsUpdate } from '../usage/limits/normalize.ts';
+import type { BackendLaunch } from './backend.ts';
 import type { SpawnChatProcess } from './chat-process.ts';
 import type { ChatTitleInput } from './chat-title.ts';
 import type { SubagentSettlement } from './subagent-settlement.ts';
@@ -30,16 +32,23 @@ export interface ChatSessionOptions {
     clock?: OutboxClock;
     items?: ChatItem[];
     preambles?: string[];
+    uiAccess?: Readonly<Record<string, unknown>>;
     provider: ChatProvider;
     command: string[];
     env(account: string | undefined): Record<string, string>;
     spawn?: SpawnChatProcess;
+    saveGeneratedImage?: BackendLaunch['saveGeneratedImage'];
     instructions?(): string | null;
     resumeNote?(): string | null;
     folders?(): readonly string[];
     promptNotes?: PromptNotes;
     references?: ChatReferences;
     checkpoints?: TurnCheckpoints;
+    uiFenceLanguage?: string;
+    ui?: {
+        schemas: UiCompileOptions['querySchemas'];
+        observe(item: ChatAssistantItem, final: boolean): void;
+    };
     emit(event: ChatEvent): void;
     onLimits?(update: LimitsUpdate): void;
     persist(): void;
@@ -58,6 +67,7 @@ export interface LimitResumeHooks {
     owed(): boolean;
 }
 export interface ChatSendExtras {
+    uiChoice?: ChatUiChoiceOrigin;
     delivery?: 'steer' | 'queue';
     mentions?: string[];
     skills?: string[];
@@ -95,7 +105,16 @@ export declare class ChatSession {
     private pendingPreambles;
     private submission;
     private readonly steering;
+    private readonly uiStreams;
+    private readonly uiCompiled;
+    private readonly capturedUiAccess;
+    private readonly uiScanned;
+    private readonly uiReferencing;
+    private readonly uiAttachments;
+    private readonly uiFenceLanguage;
     constructor(options: ChatSessionOptions);
+    get uiPreviews(): ChatEvent[];
+    previewItems(items: ChatItem[]): ChatItem[];
     get id(): string;
     get info(): ChatInfo;
     get running(): boolean;
@@ -112,6 +131,15 @@ export declare class ChatSession {
         queued: boolean;
         turnId: string;
     };
+    uiAccess(itemId: string): unknown;
+    setUiAccess(itemId: string, access: unknown): void;
+    get uiAccessRecord(): Record<string, unknown>;
+    updateUiQueries(itemId: string, uiQueries: ChatUiQueryState): void;
+    choose(payload: ChatUiChoicePayload, queries?: Readonly<Record<string, unknown>>): {
+        queued: boolean;
+        turnId: string;
+    };
+    private recordUiAnswer;
     sendInput(text: string, extras?: ChatSendExtras): Promise<{
         queued: boolean;
         turnId: string;
@@ -129,7 +157,7 @@ export declare class ChatSession {
     approve(requestId: string, decision: 'allow' | 'allow-always' | 'deny', message?: string): boolean;
     answer(requestId: string, answers: Record<string, string>): boolean;
     dismiss(itemId: string): boolean;
-    clear(force: boolean): void;
+    clear(force: boolean): Promise<void>;
     settleStored(selection: ModelSelection, resume: ResumeDecision): void;
     resume(turnId: string, attempt: number, words: ResumeWords): Promise<void>;
     abandon(turnId: string, reason: string | null): void;
@@ -208,6 +236,11 @@ export declare class ChatSession {
     private settleFromTranscripts;
     private settleRows;
     private emit;
+    private unscannedUiText;
+    private forgetUiScan;
+    private latestUiAttachment;
+    private findUiAttachment;
+    private clearUi;
     private delegatingChange;
 }
 export interface ResumeDecision {