@adecore/agents 0.20.0 → 0.21.0

package/dist/chat/chat-session.js

@@ -1,5 +1,7 @@
-import { randomUUID } from 'node:crypto';
-import { notResumedNote } from '@adecore/agent-contracts';
+import { createHash, randomUUID } from 'node:crypto';
+import { GeneratedImageInputSchema, notResumedNote } from '@adecore/agent-contracts';
+import { resolveUiChoice, UiFailure } from '@adecore/intelligent-ui';
+import { UiStream } from '@adecore/intelligent-ui/stream';
 import { commandLabel, isBackgroundWork, reportsOnBackgroundWork, runningInBackground } from './background-work.js';
 import { ChatError } from './errors.js';
 import { limitedTurn, limitResumeAt, limitResumeWake } from './limit-resume.js';
@@ -69,12 +71,31 @@ export class ChatSession {
     orphans = new Set();
     pendingPreambles;
     submission = null;
+    steering = new Set();
+    uiStreams = new Map();
+    uiCompiled = new Map();
     constructor(options) {
         this.options = options;
         this.pendingPreambles = options.preambles ?? [];
         this.thread = new ChatThread(options.info, options.items);
         this.projector = new ThreadProjector(this.thread, { providerName: options.provider.name });
     }
+    get uiPreviews() {
+        const events = [];
+        for (const [itemId, preview] of this.uiCompiled) {
+            const item = this.thread.get(itemId);
+            if (item?.kind === 'assistant' && item.streaming && item.text.length === preview.textLength) {
+                events.push({ type: 'delta', itemId, text: '', ui: preview.blocks, textLength: preview.textLength });
+            }
+        }
+        return events;
+    }
+    previewItems(items) {
+        return items.map((item) => {
+            const preview = this.uiCompiled.get(item.id);
+            return item.kind === 'assistant' && preview?.textLength === item.text.length ? { ...item, ui: preview.blocks } : item;
+        });
+    }
     get id() {
         return this.thread.info.chatId;
     }
@@ -133,9 +154,7 @@ export class ChatSession {
         return this.thread.info;
     }
     /*
-     * A message while a turn runs joins the queue instead of being refused; the host sends it when
-     * that turn settles. One queue for both providers: Claude's steer and Codex's own queue have
-     * different semantics, and one rule is easier to reason about than a rule per CLI.
+     * Internal follow-ups wait for a new turn. User input goes through `sendInput`, which can steer native work.
      */
     send(text, extras = {}) {
         const turnId = newId('turn');
@@ -149,6 +168,7 @@ export class ChatSession {
                 turnId,
                 text,
                 createdAt: Date.now(),
+                ...(extras.uiChoice ? { uiChoice: extras.uiChoice } : {}),
                 ...(extras.mentions?.length ? { mentions: extras.mentions } : {}),
                 ...(extras.skills?.length ? { skills: extras.skills } : {}),
                 ...(extras.chats?.length ? { chats: extras.chats } : {}),
@@ -162,11 +182,144 @@ export class ChatSession {
         this.dispatch(text, extras, turnId);
         return { queued: false, turnId };
     }
+    updateUiQueries(itemId, uiQueries) {
+        const item = this.thread.get(itemId);
+        if (this.frozen || item?.kind !== 'assistant' || item.parentToolUseId) {
+            return;
+        }
+        this.thread.upsert({ ...item, uiQueries });
+        this.options.emit({ type: 'delta', itemId, text: '', uiQueries });
+        this.options.persistSoon();
+    }
+    choose(payload, queries = {}) {
+        const item = this.thread.get(payload.itemId);
+        const block = item?.kind === 'assistant' && !item.streaming && !item.parentToolUseId ? item.ui?.find((block) => block.id === payload.blockId) : undefined;
+        if (item?.kind !== 'assistant' || !block?.revision || block.revision !== payload.revision) {
+            throw new ChatError('stale-ui-block', 'This completed UI block is no longer available at the requested revision');
+        }
+        const sameBlock = (origin) => origin?.itemId === item.id && origin.blockId === block.id && origin.revision === block.revision;
+        let previous = item.uiAnswers?.[block.id];
+        if (previous?.revision !== block.revision) {
+            previous = undefined;
+        }
+        // The message is logged first: a crash before its answer marker must not send it a second time.
+        if (!previous) {
+            const sent = this.thread.list().find((entry) => entry.kind === 'user' && sameBlock(entry.uiChoice));
+            const queued = this.queue.find((entry) => sameBlock(entry.uiChoice));
+            const origin = sent?.kind === 'user' ? sent.uiChoice : queued?.uiChoice;
+            const turnId = sent?.turnId ?? queued?.turnId;
+            if (origin && turnId) {
+                previous = { ...origin, turnId, queued: !sent };
+                this.recordUiAnswer(previous);
+            }
+        }
+        if (previous) {
+            if (previous.choiceId !== payload.choiceId) {
+                throw new ChatError('ui-already-answered', 'This UI block was already answered');
+            }
+            return { queued: previous.queued, turnId: previous.turnId };
+        }
+        let selected;
+        try {
+            selected = resolveUiChoice(block, payload.choiceId, payload.values, queries);
+        }
+        catch (error) {
+            throw new ChatError(error instanceof UiFailure ? error.code : 'refused-choice', error instanceof Error ? error.message : 'This choice could not be read');
+        }
+        const origin = {
+            itemId: item.id,
+            blockId: block.id,
+            revision: block.revision,
+            choiceId: payload.choiceId,
+            label: selected.label,
+            sourceAt: item.createdAt,
+            older: this.thread.list().findLast((entry) => entry.kind === 'assistant' && !entry.parentToolUseId)?.id !== item.id,
+            values: selected.values,
+            at: Date.now()
+        };
+        const result = this.send(selected.context, { uiChoice: origin });
+        this.recordUiAnswer({ ...origin, ...result });
+        return result;
+    }
+    recordUiAnswer(answer) {
+        const item = this.thread.get(answer.itemId);
+        if (item?.kind !== 'assistant') {
+            return;
+        }
+        if (JSON.stringify(item.uiAnswers?.[answer.blockId]) === JSON.stringify(answer)) {
+            return;
+        }
+        this.options.emit(this.thread.upsert({ ...item, uiAnswers: { ...item.uiAnswers, [answer.blockId]: answer } }));
+        this.options.persist();
+    }
+    sendInput(text, extras = {}) {
+        return this.submitInput(text, extras);
+    }
+    async submitInput(text, extras) {
+        const backend = this.backend;
+        const turnId = this.info.activeTurnId;
+        if (extras.delivery !== 'steer' ||
+            this.info.queuePaused ||
+            text.startsWith('/') ||
+            !this.busy ||
+            !turnId ||
+            !backend?.steerTurn ||
+            this.pendingStart ||
+            this.replacementWaits()) {
+            return this.send(text, extras);
+        }
+        const queued = this.send(text, extras);
+        const message = this.queue.find((entry) => entry.turnId === queued.turnId);
+        if (!message) {
+            return queued;
+        }
+        this.steering.add(message.id);
+        try {
+            const references = this.options.references?.(extras.chats) ?? { ids: extras.chats ?? [], note: null };
+            const accepted = await backend.steerTurn({
+                promptId: message.id,
+                text,
+                preamble: references.note,
+                attachments: extras.attachments ?? [],
+                mentions: extras.mentions ?? [],
+                skills: extras.skills ?? []
+            });
+            if (!accepted) {
+                return queued;
+            }
+            this.emit([
+                this.thread.upsert({
+                    id: `user-${message.id}`,
+                    kind: 'user',
+                    createdAt: message.createdAt,
+                    turnId,
+                    text,
+                    mentions: extras.mentions,
+                    skills: extras.skills,
+                    chats: references.ids,
+                    attachments: extras.attachments
+                }),
+                this.thread.patchInfo({ queue: this.queue.filter((entry) => entry.id !== message.id) })
+            ]);
+            this.options.persist();
+            return { queued: false, turnId };
+        }
+        catch (error) {
+            // A lost acknowledgement does not prove rejection. Keep the message for an explicit retry.
+            this.pauseQueue(true);
+            this.options.persist();
+            throw error;
+        }
+        finally {
+            this.steering.delete(message.id);
+            this.drainQueue();
+        }
+    }
     /* Drops a queued message and hands it back; null when nothing waits under that id, as once it went out. */
     unqueue(messageId) {
         const queue = this.queue;
         const message = queue.find((entry) => entry.id === messageId);
-        if (!message) {
+        if (!message || this.steering.has(message.id)) {
             return null;
         }
         this.setQueue(queue.filter((entry) => entry.id !== messageId));
@@ -176,7 +329,7 @@ export class ChatSession {
     sendNow(messageId) {
         const queue = this.queue;
         const message = queue.find((entry) => entry.id === messageId);
-        if (!message) {
+        if (!message || this.steering.has(message.id)) {
             return false;
         }
         this.setQueue([message, ...queue.filter((entry) => entry.id !== messageId)]);
@@ -201,6 +354,7 @@ export class ChatSession {
         const limited = limitedTurn(this.thread.list());
         const explicit = asked || this.sendAfterReplacement;
         if (!next ||
+            this.steering.size > 0 ||
             this.busy ||
             (!explicit && (this.info.queuePaused || (limited !== null && (limited.limit.kind === 'usage' || this.info.resumeAt !== undefined))))) {
             return;
@@ -226,7 +380,7 @@ export class ChatSession {
         this.sendAfterReplacement = false;
         this.pauseQueue(false);
         this.setQueue(rest);
-        this.dispatch(next.text, { mentions: next.mentions, skills: next.skills, chats: next.chats, attachments: next.attachments }, next.turnId);
+        this.dispatch(next.text, { mentions: next.mentions, skills: next.skills, chats: next.chats, attachments: next.attachments, uiChoice: next.uiChoice }, next.turnId);
     }
     dispatch(text, extras, requestedTurnId) {
         this.pauseQueue(false);
@@ -235,7 +389,10 @@ export class ChatSession {
         // A slash command must stay the first thing the CLI reads, as `contextNote` keeps it.
         const references = text.startsWith('/') || !this.options.references ? { ids: [], note: null } : this.options.references(extras.chats);
         const turnId = this.openTurn(text, note, { ...extras, chats: references.ids }, requestedTurnId);
-        const said = [preamble, references.note].filter((part) => part !== null);
+        const choiceNote = extras.uiChoice
+            ? `The person chose ${JSON.stringify(extras.uiChoice.label)} from your UI reply. Their message is the visible context of that choice.`
+            : null;
+        const said = [preamble, references.note, choiceNote].filter((part) => part !== null);
         const input = {
             text,
             preamble: said.length === 0 ? null : said.join('\n\n'),
@@ -347,14 +504,14 @@ export class ChatSession {
     }
     /*
      * Starts the chat over: the CLI goes, and the next send starts one without a session to resume.
-     * A turn in the way is refused unless forced, and a forced clear does not wait for it to end,
-     * since the turn disappears with the thread anyway. Writing the empty thread is the caller's.
+     * The thread resets at once; the returned promise lets the host wait for pending attachment
+     * writes before removing their files. Writing the empty thread is the caller's.
      */
     clear(force) {
         if (this.busy && !force) {
             throw new ChatError('chat-busy', `Chat ${this.id} is still working on the previous message`);
         }
-        void this.dispose();
+        const ended = this.dispose();
         this.generation = nextGeneration();
         this.projector.reset();
         // The fresh CLI hears what the host tells it at launch, as it would on a first turn.
@@ -393,6 +550,7 @@ export class ChatSession {
                 }
             })
         ]);
+        return ended;
     }
     /*
      * A thread read from disk cannot still be streaming or waiting, so whatever was open is closed,
@@ -773,6 +931,7 @@ export class ChatSession {
     }
     /* Ends the process and stops listening to it, as the host goes down; the thread stays as it is. */
     freeze() {
+        this.clearUi();
         this.clearStartWarning();
         this.clearWorkflowWarnings();
         this.frozen = true;
@@ -780,6 +939,7 @@ export class ChatSession {
     }
     /* Ends the CLI and everything it started; settles once it exited or was sent the SIGKILL. */
     dispose() {
+        this.clearUi();
         this.clearStartWarning();
         this.clearWorkflowWarnings();
         if (this.titleTimer !== null) {
@@ -925,10 +1085,24 @@ export class ChatSession {
             const skills = extras.skills?.length ? extras.skills : undefined;
             const chats = extras.chats?.length ? extras.chats : undefined;
             const attachments = extras.attachments?.length ? extras.attachments : undefined;
-            events.push(this.thread.upsert({ id: newId('user'), kind: 'user', createdAt: now, turnId, text, mentions, skills, chats, attachments }));
+            events.push(this.thread.upsert({
+                id: newId('user'),
+                kind: 'user',
+                createdAt: now,
+                turnId,
+                text,
+                mentions,
+                skills,
+                chats,
+                attachments,
+                ...(extras.uiChoice ? { uiChoice: extras.uiChoice } : {})
+            }));
         }
         events.push(this.thread.patchInfo({ status: this.thread.statusFor(turnId), activeTurnId: turnId }));
         this.emit(events);
+        if (extras.uiChoice) {
+            this.recordUiAnswer({ ...extras.uiChoice, turnId, queued: false });
+        }
         // On disk before the CLI answers, so a host that goes down mid-turn still shows the question.
         this.options.persist();
         return turnId;
@@ -1143,6 +1317,7 @@ export class ChatSession {
             runtimeMode: info.runtimeMode,
             resume: info.agentSessionId,
             generation,
+            saveGeneratedImage: this.options.saveGeneratedImage,
             instructions: this.options.instructions?.() ?? null,
             resumeNote: this.options.resumeNote?.() ?? null,
             ...(folders.length > 0 ? { folders } : {}),
@@ -1473,9 +1648,77 @@ export class ChatSession {
     }
     emit(events) {
         const delegating = this.delegatingChange(events);
-        for (const event of delegating === null ? events : [...events, delegating]) {
-            this.options.emit(event);
+        for (let event of delegating === null ? events : [...events, delegating]) {
+            if (event.type === 'reset') {
+                this.clearUi();
+            }
+            const item = event.type === 'item' ? event.item : event.type === 'delta' ? this.thread.get(event.itemId) : null;
+            if (item?.kind !== 'assistant' || item.parentToolUseId) {
+                this.options.emit(event);
+                continue;
+            }
+            let stream = this.uiStreams.get(item.id);
+            if (!stream && !item.text.includes('ruimte-ui')) {
+                this.options.emit(event);
+                continue;
+            }
+            if (!stream) {
+                stream = new UiStream({
+                    querySchemas: this.options.ui?.schemas,
+                    id: item.id,
+                    clock: this.options.clock ?? systemClock,
+                    emit: (preview) => {
+                        this.uiCompiled.set(item.id, preview);
+                        this.options.emit({ type: 'delta', itemId: item.id, text: '', ui: preview.blocks, textLength: preview.textLength });
+                    }
+                });
+                this.uiStreams.set(item.id, stream);
+            }
+            this.options.ui?.observe(item, false);
+            const latestAttachment = this.latestUiAttachment(item.turnId);
+            if (event.type === 'item' && !item.streaming) {
+                const ui = stream
+                    .finish(item.text, latestAttachment)
+                    .map((block) => ({ ...block, revision: createHash('sha256').update(JSON.stringify(block)).digest('hex') }));
+                this.uiStreams.delete(item.id);
+                this.uiCompiled.delete(item.id);
+                const diagnostics = [...new Set(ui.flatMap((block) => block.diagnostics.map((diagnostic) => diagnostic.message)))].slice(0, 6);
+                if (diagnostics.length > 0) {
+                    const repair = `Your last ruimte-ui block had these diagnostics: ${diagnostics.join(' ')} Use the catalog's supported props and components.`.slice(0, 1200);
+                    if (!this.pendingPreambles.includes(repair)) {
+                        this.pendingPreambles = [...this.pendingPreambles, repair];
+                    }
+                }
+                const current = this.thread.get(item.id);
+                const completed = { ...item, ui, ...(current?.kind === 'assistant' && current.uiQueries ? { uiQueries: current.uiQueries } : {}) };
+                event = this.thread.upsert(completed);
+                this.options.emit(event);
+                this.options.ui?.observe(completed, true);
+            }
+            else {
+                this.options.emit(event);
+                stream.update(item.text, latestAttachment);
+            }
+        }
+    }
+    latestUiAttachment(turnId) {
+        for (const item of this.thread.list().toReversed()) {
+            if (item.kind !== 'tool' || item.turnId !== turnId || item.name !== 'ImageGeneration' || item.state !== 'done') {
+                continue;
+            }
+            const input = GeneratedImageInputSchema.safeParse(item.input);
+            if (input.success && input.data.attachment?.mime.startsWith('image/')) {
+                return input.data.attachment.id;
+            }
+        }
+        return undefined;
+    }
+    clearUi() {
+        for (const stream of this.uiStreams.values()) {
+            stream.dispose();
         }
+        this.uiStreams.clear();
+        this.uiCompiled.clear();
     }
     /* Said on the info as well as by the rows, since a client with the thread closed sees only the info. */
     delegatingChange(events) {

package/dist/chat/claude-backend.d.ts

@@ -11,11 +11,14 @@ export declare class ClaudeBackend implements ChatBackend {
     private child;
     private stdinClosed;
     private interrupted;
+    private readonly steers;
+    private active;
     constructor(launch: BackendLaunch, host: BackendHost, options?: ClaudeBackendOptions);
     get running(): boolean;
     get pid(): number | null;
     start(): Promise<void>;
     sendTurn(input: TurnInput): void;
+    steerTurn(input: TurnInput): Promise<boolean>;
     compact(): void;
     interrupt(): void;
     stopTask(taskId: string): void;
@@ -27,6 +30,7 @@ export declare class ClaudeBackend implements ChatBackend {
     private readLines;
     private handleLine;
     private readLine;
+    private rejectSteers;
     private failed;
     private respond;
     private write;

package/dist/chat/claude-backend.js

@@ -20,6 +20,8 @@ export class ClaudeBackend {
     stdinClosed = false;
     // The person asked to stop; the result frame that follows closes the turn as aborted.
     interrupted = false;
+    steers = new Map();
+    active = false;
     constructor(launch, host, options = {}) {
         this.launch = launch;
         this.host = host;
@@ -60,9 +62,28 @@ export class ClaudeBackend {
         // The prefix is what the CLI must see first (ultrathink), then the note about the links, then what was typed.
         const prefix = `${promptPrefix(this.launch.selection)}${input.preamble === null ? '' : `${input.preamble}\n\n`}`;
         const promptId = input.promptId ?? randomUUID();
+        this.active = true;
         this.protocol.beginPrompt(promptId);
         this.write(buildUserMessage({ text: input.text, attachments: input.attachments, prefix, skills: input.skills, promptId }));
     }
+    steerTurn(input) {
+        if (!this.child || !this.active || this.stdinClosed) {
+            return Promise.resolve(false);
+        }
+        const promptId = input.promptId ?? randomUUID();
+        return new Promise((resolve, reject) => {
+            this.steers.set(promptId, { resolve, reject });
+            this.protocol.beginSteer(promptId);
+            try {
+                const prefix = `${promptPrefix(this.launch.selection)}${input.preamble === null ? '' : `${input.preamble}\n\n`}`;
+                this.write(buildUserMessage({ text: input.text, attachments: input.attachments, prefix, skills: input.skills, promptId }));
+            }
+            catch (error) {
+                this.steers.delete(promptId);
+                reject(error instanceof Error ? error : new Error(errorText(error)));
+            }
+        });
+    }
     /* Claude Code folds its context through a slash command, so the session sends it as a turn. */
     compact() {
         this.sendTurn({ text: '/compact', preamble: null, attachments: [], mentions: [], skills: [] });
@@ -100,6 +121,7 @@ export class ClaudeBackend {
         this.child?.endAfterGrace();
     }
     dispose() {
+        this.rejectSteers('Claude stopped before acknowledging your message');
         const child = this.child;
         this.child = null;
         return child?.end() ?? Promise.resolve();
@@ -146,7 +168,15 @@ export class ClaudeBackend {
         catch {
             return;
         }
+        const echoed = typeof frame === 'object' && frame !== null ? frame : null;
+        if (echoed?.type === 'user' && typeof echoed.uuid === 'string' && echoed.parent_tool_use_id == null) {
+            this.steers.get(echoed.uuid)?.resolve(true);
+            this.steers.delete(echoed.uuid);
+        }
         for (const event of this.protocol.handle(frame)) {
+            if (event.type === 'turn.done') {
+                this.active = false;
+            }
             if (event.type === 'turn.done' && this.interrupted) {
                 this.interrupted = false;
                 this.host.onEvent({ ...event, state: 'aborted' });
@@ -163,7 +193,15 @@ export class ClaudeBackend {
             this.failed(error);
         }
     }
+    rejectSteers(reason) {
+        this.active = false;
+        for (const pending of this.steers.values()) {
+            pending.reject(new Error(reason));
+        }
+        this.steers.clear();
+    }
     failed(error) {
+        this.rejectSteers(`Claude transport failed: ${errorText(error)}`);
         this.protocol.forgetPending();
         try {
             this.host.onEvent({ type: 'failed', message: `Claude transport failed: ${errorText(error)}`, processAlive: this.running });
@@ -208,6 +246,7 @@ export class ClaudeBackend {
             return;
         }
         this.child = null;
+        this.rejectSteers('Claude exited before acknowledging your message');
         this.protocol.forgetPending();
         this.host.onEvent({ type: 'exit', exitCode, ...(stderr === null ? {} : { stderr }) });
     }

package/dist/chat/claude-protocol.d.ts

@@ -15,6 +15,8 @@ export declare class ClaudeProtocol {
     private readonly subagentModels;
     private apiError;
     private readonly refusedWindows;
+    private readonly steerIds;
+    beginSteer(promptId: string): void;
     beginPrompt(promptId: string): void;
     handle(frame: unknown): BackendEvent[];
     approvalResponse(requestId: string, decision: ApprovalDecision, message?: string): unknown | null;

package/dist/chat/claude-protocol.js

@@ -117,7 +117,12 @@ export class ClaudeProtocol {
     // Only main-chain errors classify the turn; each plan window can refuse it independently.
     apiError = null;
     refusedWindows = new Map();
+    steerIds = new Set();
+    beginSteer(promptId) {
+        this.steerIds.add(promptId);
+    }
     beginPrompt(promptId) {
+        this.steerIds.clear();
         this.promptId = promptId;
         this.apiError = null;
         this.refusedWindows.clear();
@@ -538,7 +543,10 @@ export class ClaudeProtocol {
     handleResult(frame, events) {
         const echoed = Array.isArray(frame.user_message_uuids) ? frame.user_message_uuids.filter((value) => typeof value === 'string') : [];
         const origin = str(frame.origin) ?? (isRecord(frame.origin) ? str(frame.origin.kind) : null);
-        if (this.promptId !== null && (echoed.length > 0 ? !echoed.includes(this.promptId) : origin !== null && origin !== 'human' && origin !== 'user')) {
+        if (this.promptId !== null &&
+            (echoed.length > 0
+                ? !echoed.includes(this.promptId) && !echoed.some((id) => this.steerIds.has(id))
+                : origin !== null && origin !== 'human' && origin !== 'user')) {
             if (typeof frame.total_cost_usd === 'number') {
                 events.push({ type: 'usage', costUsd: num(frame.total_cost_usd) });
             }
@@ -549,6 +557,9 @@ export class ClaudeProtocol {
         }
         const promptId = this.promptId !== null && echoed.includes(this.promptId) ? this.promptId : null;
         this.promptId = null;
+        for (const id of echoed) {
+            this.steerIds.delete(id);
+        }
         const failed = frame.is_error === true || (typeof frame.subtype === 'string' && frame.subtype.startsWith('error'));
         const errors = Array.isArray(frame.errors) ? frame.errors.filter((error) => typeof error === 'string') : [];
         const lastUuid = this.lastUuid;

package/dist/chat/codex-backend.d.ts

@@ -9,6 +9,7 @@ export declare class CodexBackend implements ChatBackend {
     readonly acknowledgesTurns = true;
     private promptId;
     private readonly launch;
+    private imageWrites;
     private readonly host;
     private readonly protocol;
     private readonly client;
@@ -22,7 +23,9 @@ export declare class CodexBackend implements ChatBackend {
     get running(): boolean;
     get pid(): number | null;
     start(): Promise<void>;
+    private turnInput;
     sendTurn(input: TurnInput): void;
+    steerTurn(input: TurnInput): Promise<boolean>;
     private acceptPrompt;
     private fail;
     private respond;
@@ -48,4 +51,5 @@ export declare class CodexBackend implements ChatBackend {
     private handleFrame;
     private handleExit;
     private emit;
+    private storeImage;
 }