@adecore/agents 0.20.1 → 0.21.0

package/dist/chat/attachment-store.d.ts

@@ -4,6 +4,8 @@ export declare class AttachmentStore {
     readonly dir: string;
     constructor(home: string);
     save(chatId: string, upload: ChatAttachmentUpload): Promise<ChatAttachment>;
+    saveGenerated(chatId: string, ref: string, upload: ChatAttachmentUpload): Promise<ChatAttachment>;
+    copy(chatId: string, attachment: ChatAttachment): Promise<ChatAttachment>;
     removeAll(chatId: string): Promise<void>;
     folderOf(chatId: string): string;
 }

package/dist/chat/attachment-store.js

@@ -1,5 +1,5 @@
-import { randomBytes } from 'node:crypto';
-import { mkdir, rm, writeFile } from 'node:fs/promises';
+import { createHash, randomBytes } from 'node:crypto';
+import { link, mkdir, readFile, rm, writeFile } from 'node:fs/promises';
 import { extname, join } from 'node:path';
 import { attachmentImageMime } from '@adecore/agent-contracts';
 // The extension the file gets on disk when its name does not carry a usable one.
@@ -50,6 +50,42 @@ export class AttachmentStore {
         await writeFile(path, bytes, { mode: 0o600 });
         return { id, name: upload.name, mime, size: bytes.byteLength, path };
     }
+    async saveGenerated(chatId, ref, upload) {
+        const id = createHash('sha256').update(`codex-image:${ref}`).digest('hex');
+        const bytes = Buffer.from(upload.data, 'base64');
+        const folder = this.folderOf(chatId);
+        await mkdir(folder, { recursive: true, mode: 0o700 });
+        const path = join(folder, `${id}.${extensionFor('', upload.mime)}`);
+        const temporary = join(folder, `.${id}-${randomBytes(8).toString('hex')}.tmp`);
+        try {
+            await writeFile(temporary, bytes, { mode: 0o600, flag: 'wx' });
+            try {
+                // Publish complete bytes without replacing an image another replay has already saved.
+                await link(temporary, path);
+            }
+            catch (error) {
+                if (error.code !== 'EEXIST') {
+                    throw error;
+                }
+                const previous = await readFile(path);
+                if (!previous.equals(bytes)) {
+                    throw new Error('The generated image id already belongs to another image');
+                }
+            }
+        }
+        finally {
+            await rm(temporary, { force: true });
+        }
+        return { id, name: upload.name, mime: upload.mime, size: bytes.byteLength, path };
+    }
+    async copy(chatId, attachment) {
+        const bytes = await readFile(attachment.path);
+        const folder = this.folderOf(chatId);
+        await mkdir(folder, { recursive: true, mode: 0o700 });
+        const path = join(folder, `${createHash('sha256').update(attachment.id).digest('hex')}.${extensionFor(attachment.name, attachment.mime)}`);
+        await writeFile(path, bytes, { mode: 0o600 });
+        return { ...attachment, path };
+    }
     /* Everything a chat attached; used when the chat itself is deleted. */
     async removeAll(chatId) {
         await rm(this.folderOf(chatId), { recursive: true, force: true });

package/dist/chat/backend.d.ts

@@ -1,4 +1,4 @@
-import type { ChatAttachment, ChatFileChange, ChatQuestion, ChatSkill, ChatSubagentUsage, ChatTurnLimit, ChatWorkflow, ModelSelection, RuntimeMode } from '@adecore/agent-contracts';
+import type { ChatAttachment, ChatAttachmentUpload, ChatFileChange, ChatQuestion, ChatSkill, ChatSubagentUsage, ChatTurnLimit, ChatWorkflow, ModelSelection, RuntimeMode } from '@adecore/agent-contracts';
 import type { LimitsUpdate } from '../usage/limits/normalize.ts';
 import type { SpawnChatProcess } from './chat-process.ts';
 export interface BackendLaunch {
@@ -14,6 +14,7 @@ export interface BackendLaunch {
     resumeNote: string | null;
     folders?: readonly string[];
     spawn?: SpawnChatProcess;
+    saveGeneratedImage?(ref: string, upload: ChatAttachmentUpload): Promise<ChatAttachment>;
 }
 export interface TurnInput {
     promptId?: string;
@@ -47,11 +48,21 @@ export type BackendEvent = {
     type: 'text.delta';
     ref: string;
     text: string;
+    phase?: string;
 } | {
     type: 'text.done';
     ref: string;
     text: string;
     parentRef?: string | null;
+    phase?: string;
+} | {
+    type: 'image.generated';
+    ref: string;
+    upload: ChatAttachmentUpload;
+    width?: number;
+    height?: number;
+    revisedPrompt?: string;
+    transparentBackground?: boolean;
 } | {
     type: 'thinking.delta';
     ref: string;

package/dist/chat/chat-core.d.ts

@@ -1,4 +1,5 @@
-import { type AgentKind, type ChatAttachment, type ChatAttachResult, type ChatAttachmentUpload, type ChatBookmark, type ChatCheckpointDiff, type ChatConfigurePayload, type ChatCreatePayload, type ChatEvent, type ChatHistoryResult, type ChatInfo, type ChatItem, type ChatQueuedMessage, type ChatSkill, type ChatSubagentPayload, type ChatSubagentResult, type ChatTurnItem, type ChatVisual, type ModelSelection, type RuntimeMode } from '@adecore/agent-contracts';
+import { type AgentKind, type ChatAttachment, type ChatAttachResult, type ChatAttachmentUpload, type ChatBookmark, type ChatCheckpointDiff, type ChatConfigurePayload, type ChatCreatePayload, type ChatEvent, type ChatHistoryResult, type ChatInfo, type ChatItem, type ChatQueuedMessage, type ChatUiChoicePayload, type ChatUiQueryPayload, type ChatUiLinkPayload, type ChatSkill, type ChatSubagentPayload, type ChatSubagentResult, type ChatTurnItem, type ChatVisual, type ModelSelection, type RuntimeMode } from '@adecore/agent-contracts';
+import { type ChatUiHost } from './ui-queries.ts';
 import { ClientSinks } from '../client-sinks.ts';
 import type { AgentEvent, AgentSink } from '../events.ts';
 import type { OutboxOutcome } from '../outbox/outbox-worker.ts';
@@ -25,6 +26,7 @@ export interface InterruptedRun {
     attempt: number;
 }
 export interface ChatCoreOptions {
+    intelligentUi?: ChatUiHost;
     providers: ProviderRegistry;
     store?: ChatStore;
     checkpoints?: TurnCheckpoints;
@@ -95,8 +97,21 @@ export declare class ChatCore {
                 mime: string;
                 size: number;
                 path: string;
+                width?: number | undefined;
+                height?: number | undefined;
             }[] | undefined;
             createdAt: number;
+            uiChoice?: {
+                itemId: string;
+                blockId: string;
+                revision: string;
+                choiceId: string;
+                label: string;
+                sourceAt: number;
+                older: boolean;
+                values?: Record<string, unknown> | undefined;
+                at: number;
+            } | undefined;
         }[] | undefined;
         queuePaused?: boolean | undefined;
         background?: {
@@ -174,6 +189,7 @@ export declare class ChatCore {
     protected readonly subagents: SubagentReader;
     readonly claudeProjectsDir: string;
     readonly composerPreferences: ComposerPreferences;
+    private readonly uiQueries;
     private readonly checkpoints;
     private readonly instructions;
     private readonly skillIndex;
@@ -251,6 +267,42 @@ export declare class ChatCore {
         queued: boolean;
         turnId: string;
     }>;
+    linkUi(payload: ChatUiLinkPayload, clientId: string): Promise<{
+        state: "chip" | "plain";
+        target?: {
+            type: "File";
+            path: string;
+            line?: number | undefined;
+        } | {
+            type: "Diff";
+            path: string;
+        } | {
+            type: "Commit";
+            sha: string;
+        } | {
+            type: "Node";
+            id: string;
+        } | undefined;
+        cwd?: string | undefined;
+        viewId?: string | undefined;
+        projectId?: string | undefined;
+        staged?: boolean | undefined;
+        conflicted?: boolean | undefined;
+        relativePath?: string | undefined;
+        label?: string | undefined;
+        reason?: string | undefined;
+    }>;
+    queryUi(payload: ChatUiQueryPayload, clientId: string): Promise<{
+        state: "failed" | "fresh" | "refused";
+        value?: unknown;
+        readId?: string | undefined;
+        readAt: number;
+        reason?: string | undefined;
+    }>;
+    choose(payload: ChatUiChoicePayload): Promise<{
+        queued: boolean;
+        turnId: string;
+    }>;
     attachment(chatId: string, id: string): ChatAttachment | null;
     unqueue(chatId: string, messageId: string): ChatQueuedMessage;
     sendNow(chatId: string, messageId: string): void;

package/dist/chat/chat-core.js

@@ -1,6 +1,7 @@
 import { homedir } from 'node:os';
 import { join } from 'node:path';
-import { ChatAttachmentUploadsSchema } from '@adecore/agent-contracts';
+import { ChatAttachmentUploadsSchema, generatedImageAttachment } from '@adecore/agent-contracts';
+import { ChatUiQueries } from './ui-queries.js';
 import { ClientSinks } from '../client-sinks.js';
 import { errorText } from '../error-text.js';
 import { AccountError, definedEnv, isDefaultAccountOf, launchEnv, storedAccount } from '../providers/accounts/launch.js';
@@ -59,6 +60,7 @@ export class ChatCore {
     /* Where Claude Code keeps the projects of its default account on this machine; empty when it has none. */
     claudeProjectsDir;
     composerPreferences = new ComposerPreferences();
+    uiQueries;
     checkpoints;
     instructions;
     skillIndex;
@@ -84,6 +86,7 @@ export class ChatCore {
     onInterruptedRun;
     limitResume;
     constructor(options) {
+        this.uiQueries = options.intelligentUi ? new ChatUiQueries(options.intelligentUi) : null;
         this.providers = options.providers;
         this.store = options.store ?? null;
         this.bookmarks = options.bookmarks ?? null;
@@ -269,6 +272,7 @@ export class ChatCore {
         const references = this.referencesFor(payload.chatId);
         const session = new ChatSession({
             info,
+            ...(this.uiQueries ? { ui: { schemas: this.uiQueries.schemas, observe: (item, final) => this.uiQueries.observe(session, item, final) } } : {}),
             items: stored?.items ?? [],
             preambles: stored?.preambles ?? [],
             provider,
@@ -279,6 +283,7 @@ export class ChatCore {
                 this.accounts?.launched?.(kind, chatAccount);
                 return env;
             },
+            saveGeneratedImage: (ref, upload) => this.attachments.saveGenerated(payload.chatId, ref, upload),
             instructions: () => this.instructionsFor(payload.chatId),
             resumeNote: () => this.resumeNoteFor(payload.chatId),
             folders: () => this.foldersFor(payload.chatId),
@@ -362,11 +367,16 @@ export class ChatCore {
         const log = this.logs.get(chatId);
         const events = since === undefined ? null : log.after(since);
         if (events !== null) {
-            return { info: session.info, items: [], events, seq: log.seq };
+            return { info: session.info, items: [], events: [...events, ...session.uiPreviews], seq: log.seq };
         }
-        return historyLimit === undefined
-            ? { ...session.thread.snapshot(), seq: log.seq }
-            : { info: session.info, ...session.thread.history(historyLimit), pending: session.thread.pending(), seq: log.seq };
+        const page = historyLimit === undefined ? session.thread.snapshot() : session.thread.history(historyLimit);
+        return {
+            ...page,
+            info: session.info,
+            items: session.previewItems(page.items),
+            ...(historyLimit === undefined ? {} : { pending: session.previewItems(session.thread.pending()) }),
+            seq: log.seq
+        };
     }
     /*
      * `attach` with the chat's bookmarks and visuals. They are read before the client joins, so a
@@ -548,6 +558,28 @@ export class ChatCore {
         const attachments = await Promise.all(uploads.map((upload) => this.attachments.save(chatId, upload)));
         return session.sendInput(text, { ...extras, ...(attachments.length > 0 ? { attachments } : {}) });
     }
+    async linkUi(payload, clientId) {
+        if (!this.attached.get(payload.chatId)?.has(clientId) || !this.uiQueries) {
+            throw new ChatError('refused-query', 'Attach to the chat before resolving its UI links.');
+        }
+        return this.uiQueries.link(this.require(payload.chatId), payload);
+    }
+    async queryUi(payload, clientId) {
+        if (!this.attached.get(payload.chatId)?.has(clientId)) {
+            throw new ChatError('refused-query', 'Attach to the chat before reading its UI queries.');
+        }
+        if (!this.uiQueries) {
+            throw new ChatError('refused-query', 'This host does not provide UI query sources.');
+        }
+        return this.uiQueries.query(this.require(payload.chatId), payload);
+    }
+    async choose(payload) {
+        const session = this.require(payload.chatId);
+        const queries = await this.uiQueries?.choiceValues(session, payload);
+        const result = session.choose(payload, queries);
+        await this.persisted(payload.chatId);
+        return result;
+    }
     /* The file behind an attachment id: what this chat's thread or queue says it is, or a visual's page. */
     attachment(chatId, id) {
         const session = this.chats.get(chatId);
@@ -555,7 +587,8 @@ export class ChatCore {
             return null;
         }
         for (const item of session.thread.list()) {
-            const found = item.kind === 'user' ? item.attachments?.find((attachment) => attachment.id === id) : undefined;
+            const generated = generatedImageAttachment(item);
+            const found = item.kind === 'user' ? item.attachments?.find((attachment) => attachment.id === id) : generated?.id === id ? generated : null;
             if (found) {
                 return found;
             }
@@ -599,7 +632,7 @@ export class ChatCore {
             throw new ChatError('chat-busy', `Chat ${chatId} is still working on the previous message`);
         }
         await this.clearing(chatId);
-        session.clear(force);
+        const ended = session.clear(force);
         // Before the record is written, so what it takes along from the record is gone from the write as well.
         const cleared = this.cleared(chatId);
         // A debounced write still waiting holds the old thread and must not land after the empty one.
@@ -607,7 +640,7 @@ export class ChatCore {
         // Folded right away: every line before the reset describes a thread that is gone.
         await Promise.all([
             this.persistNow(chatId, true),
-            this.attachments.removeAll(chatId),
+            ended.then(() => this.attachments.removeAll(chatId)),
             this.bookmarks?.removeChat(chatId),
             this.visuals?.removeChat(chatId),
             cleared
@@ -646,8 +679,9 @@ export class ChatCore {
         const session = this.require(chatId);
         // A write already out would put the record back after the delete.
         const writing = this.writes.get(chatId);
-        void session.dispose();
+        const ended = session.dispose();
         this.subagents.releaseChat(chatId);
+        this.uiQueries?.forget(chatId);
         this.coalescers.get(chatId)?.dispose();
         this.coalescers.delete(chatId);
         this.logs.get(chatId)?.close();
@@ -663,6 +697,7 @@ export class ChatCore {
         const snapshot = session.thread.snapshot();
         this.forgotten(chatId);
         await writing;
+        await ended;
         await Promise.all([
             this.store?.delete(chatId),
             this.attachments.removeAll(chatId),
@@ -955,6 +990,14 @@ export class ChatCore {
         return next;
     }
     emit(chatId, event) {
+        if (event.type === 'delta' && event.ui !== undefined) {
+            // Previews follow all pending text but have no log position or observer side effects.
+            this.coalescers.get(chatId)?.flush();
+            for (const clientId of this.attached.get(chatId) ?? []) {
+                this.sinks.to(clientId, { event: 'chat.event', payload: { chatId, event } });
+            }
+            return;
+        }
         this.activity.set(chatId, Date.now());
         let coalescer = this.coalescers.get(chatId);
         if (!coalescer) {

package/dist/chat/chat-session.d.ts

@@ -1,6 +1,8 @@
-import type { ChatAttachment, ChatCheckpointDiff, ChatEvent, ChatInfo, ChatItem, ChatQueuedMessage, ChatSkill, ModelSelection, RuntimeMode } from '@adecore/agent-contracts';
+import type { ChatAttachment, ChatAssistantItem, ChatCheckpointDiff, ChatEvent, ChatInfo, ChatItem, ChatQueuedMessage, ChatSkill, ChatUiChoiceOrigin, ChatUiChoicePayload, ChatUiQueryState, ModelSelection, RuntimeMode } from '@adecore/agent-contracts';
+import type { UiCompileOptions } from '@adecore/intelligent-ui/compiler';
 import type { ChatProvider } from '../providers/provider.ts';
 import type { LimitsUpdate } from '../usage/limits/normalize.ts';
+import type { BackendLaunch } from './backend.ts';
 import type { SpawnChatProcess } from './chat-process.ts';
 import type { ChatTitleInput } from './chat-title.ts';
 import type { SubagentSettlement } from './subagent-settlement.ts';
@@ -34,12 +36,17 @@ export interface ChatSessionOptions {
     command: string[];
     env(account: string | undefined): Record<string, string>;
     spawn?: SpawnChatProcess;
+    saveGeneratedImage?: BackendLaunch['saveGeneratedImage'];
     instructions?(): string | null;
     resumeNote?(): string | null;
     folders?(): readonly string[];
     promptNotes?: PromptNotes;
     references?: ChatReferences;
     checkpoints?: TurnCheckpoints;
+    ui?: {
+        schemas: UiCompileOptions['querySchemas'];
+        observe(item: ChatAssistantItem, final: boolean): void;
+    };
     emit(event: ChatEvent): void;
     onLimits?(update: LimitsUpdate): void;
     persist(): void;
@@ -58,6 +65,7 @@ export interface LimitResumeHooks {
     owed(): boolean;
 }
 export interface ChatSendExtras {
+    uiChoice?: ChatUiChoiceOrigin;
     delivery?: 'steer' | 'queue';
     mentions?: string[];
     skills?: string[];
@@ -95,7 +103,11 @@ export declare class ChatSession {
     private pendingPreambles;
     private submission;
     private readonly steering;
+    private readonly uiStreams;
+    private readonly uiCompiled;
     constructor(options: ChatSessionOptions);
+    get uiPreviews(): ChatEvent[];
+    previewItems(items: ChatItem[]): ChatItem[];
     get id(): string;
     get info(): ChatInfo;
     get running(): boolean;
@@ -112,6 +124,12 @@ export declare class ChatSession {
         queued: boolean;
         turnId: string;
     };
+    updateUiQueries(itemId: string, uiQueries: ChatUiQueryState): void;
+    choose(payload: ChatUiChoicePayload, queries?: Readonly<Record<string, unknown>>): {
+        queued: boolean;
+        turnId: string;
+    };
+    private recordUiAnswer;
     sendInput(text: string, extras?: ChatSendExtras): Promise<{
         queued: boolean;
         turnId: string;
@@ -129,7 +147,7 @@ export declare class ChatSession {
     approve(requestId: string, decision: 'allow' | 'allow-always' | 'deny', message?: string): boolean;
     answer(requestId: string, answers: Record<string, string>): boolean;
     dismiss(itemId: string): boolean;
-    clear(force: boolean): void;
+    clear(force: boolean): Promise<void>;
     settleStored(selection: ModelSelection, resume: ResumeDecision): void;
     resume(turnId: string, attempt: number, words: ResumeWords): Promise<void>;
     abandon(turnId: string, reason: string | null): void;
@@ -208,6 +226,8 @@ export declare class ChatSession {
     private settleFromTranscripts;
     private settleRows;
     private emit;
+    private latestUiAttachment;
+    private clearUi;
     private delegatingChange;
 }
 export interface ResumeDecision {