@adecore/agents 0.20.0 → 0.21.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. package/dist/chat/attachment-store.d.ts +2 -0
  2. package/dist/chat/attachment-store.js +38 -2
  3. package/dist/chat/backend.d.ts +13 -1
  4. package/dist/chat/chat-core.d.ts +53 -1
  5. package/dist/chat/chat-core.js +53 -10
  6. package/dist/chat/chat-session.d.ts +29 -2
  7. package/dist/chat/chat-session.js +258 -15
  8. package/dist/chat/claude-backend.d.ts +4 -0
  9. package/dist/chat/claude-backend.js +39 -0
  10. package/dist/chat/claude-protocol.d.ts +2 -0
  11. package/dist/chat/claude-protocol.js +12 -1
  12. package/dist/chat/codex-backend.d.ts +4 -0
  13. package/dist/chat/codex-backend.js +77 -5
  14. package/dist/chat/codex-protocol.d.ts +1 -0
  15. package/dist/chat/codex-protocol.js +74 -5
  16. package/dist/chat/delta-coalescer.js +1 -1
  17. package/dist/chat/errors.d.ts +1 -1
  18. package/dist/chat/fake-codex.js +33 -0
  19. package/dist/chat/generated-image.d.ts +6 -0
  20. package/dist/chat/generated-image.js +74 -0
  21. package/dist/chat/projector.js +9 -2
  22. package/dist/chat/thread.js +6 -0
  23. package/dist/chat/ui-queries.d.ts +38 -0
  24. package/dist/chat/ui-queries.js +311 -0
  25. package/dist/host/handlers.js +4 -1
  26. package/package.json +5 -4
  27. package/src/chat/attachment-store.ts +38 -2
  28. package/src/chat/backend.ts +16 -2
  29. package/src/chat/chat-core.ts +60 -9
  30. package/src/chat/chat-session.ts +294 -16
  31. package/src/chat/claude-backend.ts +40 -0
  32. package/src/chat/claude-protocol.ts +16 -1
  33. package/src/chat/codex-backend.ts +77 -5
  34. package/src/chat/codex-protocol.ts +78 -6
  35. package/src/chat/delta-coalescer.ts +1 -1
  36. package/src/chat/errors.ts +2 -0
  37. package/src/chat/fake-codex.ts +34 -0
  38. package/src/chat/generated-image.ts +88 -0
  39. package/src/chat/projector.ts +9 -2
  40. package/src/chat/thread.ts +6 -0
  41. package/src/chat/ui-queries.ts +360 -0
  42. package/src/host/handlers.ts +9 -1
@@ -0,0 +1,311 @@
1
+ import { randomUUID } from 'node:crypto';
2
+ import { copyUiValue, UiBudget } from '@adecore/intelligent-ui';
3
+ import { UiLinkResolutionSchema, uiLinkTargets } from '@adecore/intelligent-ui/links';
4
+ import { uiQueryArguments, uiQueryFallback } from '@adecore/intelligent-ui/query';
5
+ import { ChatError } from './errors.js';
6
+ const MAX_QUERY_BYTES = 64 * 1024;
7
+ const MAX_SNAPSHOT_BYTES = 128 * 1024;
8
+ const MAX_QUERIES = 8;
9
+ const REFRESH_MS = 10_000;
10
+ const READ_TIMEOUT_MS = 8_000;
11
+ export class ChatUiQueries {
12
+ schemas;
13
+ captures = new Map();
14
+ freezing = new Set();
15
+ cache = new Map();
16
+ reading = new Map();
17
+ attempts = new Map();
18
+ active = new Map();
19
+ totalActive = 0;
20
+ host;
21
+ constructor(host) {
22
+ this.host = host;
23
+ this.schemas = Object.fromEntries(Object.entries(host.sources).map(([name, source]) => [name, source.args]));
24
+ }
25
+ observe(session, item, final) {
26
+ if (!item.text.includes('@Query') && !/<(?:File|Diff|Commit|Node)\b/.test(item.text) && !item.ui?.some((block) => Object.keys(block.queries).length)) {
27
+ return;
28
+ }
29
+ const capture = this.access(session, item);
30
+ if (!final || !item.ui?.length) {
31
+ return;
32
+ }
33
+ const key = `${session.id}:${item.id}`;
34
+ if (this.freezing.has(key)) {
35
+ return;
36
+ }
37
+ this.freezing.add(key);
38
+ void capture
39
+ .then(async () => {
40
+ for (const block of (item.ui ?? []).slice(0, 16)) {
41
+ const current = session.thread.get(item.id);
42
+ if (current?.kind !== 'assistant' || current.ui?.find((entry) => entry.id === block.id)?.revision !== block.revision) {
43
+ return;
44
+ }
45
+ if (current.uiQueries?.blocks[block.id]?.revision === block.revision) {
46
+ continue;
47
+ }
48
+ const readings = Object.create(null);
49
+ const values = Object.create(null);
50
+ for (const name of Object.keys(block.queries).slice(0, MAX_QUERIES)) {
51
+ const reading = await this.query(session, {
52
+ chatId: session.id,
53
+ itemId: item.id,
54
+ blockId: block.id,
55
+ revision: block.revision,
56
+ query: name
57
+ });
58
+ readings[name] = reading;
59
+ if (reading.state === 'fresh') {
60
+ values[name] = reading.value;
61
+ }
62
+ }
63
+ const links = Object.create(null);
64
+ if (this.host.link) {
65
+ const targets = uiLinkTargets(block, {}, values);
66
+ const access = await this.access(session, current);
67
+ for (const [id, target] of Object.entries(targets).slice(0, 64)) {
68
+ try {
69
+ links[id] = this.linkReading(await this.host.link(session.info, access.access, target));
70
+ }
71
+ catch (error) {
72
+ links[id] = { state: 'plain', reason: this.reason(error) };
73
+ }
74
+ }
75
+ }
76
+ const latest = session.thread.get(item.id);
77
+ if (latest?.kind !== 'assistant' || !latest.uiQueries || latest.ui?.find((entry) => entry.id === block.id)?.revision !== block.revision) {
78
+ return;
79
+ }
80
+ const names = Object.entries(block.queries)
81
+ .map(([name, query]) => `${name}: ${query.source}`)
82
+ .join(', ');
83
+ const failures = Object.entries(readings)
84
+ .filter(([, reading]) => reading.state !== 'fresh')
85
+ .map(([name, reading]) => `${name}: ${reading.reason}`)
86
+ .join('\n');
87
+ const frozen = {
88
+ revision: block.revision,
89
+ readings,
90
+ links,
91
+ fallback: `${uiQueryFallback(block, values).slice(0, 8192)}${names ? `\nSources: ${names}` : ''}${failures ? `\n${failures}` : ''}`
92
+ };
93
+ const blocks = { ...latest.uiQueries.blocks, [block.id]: frozen };
94
+ if (Buffer.byteLength(JSON.stringify(blocks)) > MAX_SNAPSHOT_BYTES) {
95
+ frozen.readings = Object.fromEntries(Object.keys(readings).map((name) => [
96
+ name,
97
+ { state: 'failed', readAt: Date.now(), reason: 'The frozen query values are too large.' }
98
+ ]));
99
+ frozen.links = Object.fromEntries(Object.keys(links).map((id) => [id, { state: 'plain', reason: 'This link must be checked again.' }]));
100
+ frozen.fallback = `${block.fallback.slice(0, 2048)}\nSources: ${names}\nThe frozen query values are too large.`;
101
+ }
102
+ if (Buffer.byteLength(JSON.stringify(blocks)) > MAX_SNAPSHOT_BYTES) {
103
+ delete blocks[block.id];
104
+ }
105
+ session.updateUiQueries(item.id, { ...latest.uiQueries, blocks });
106
+ }
107
+ })
108
+ .catch(() => { })
109
+ .finally(() => this.freezing.delete(key));
110
+ }
111
+ access(session, item) {
112
+ const key = `${session.id}:${item.id}`;
113
+ const existing = this.captures.get(key);
114
+ if (existing) {
115
+ return existing;
116
+ }
117
+ if (item.uiQueries?.authorChatId === session.id) {
118
+ return Promise.resolve(item.uiQueries);
119
+ }
120
+ // A pending capture is persisted as denied, so a restart cannot widen the writer's rights.
121
+ session.updateUiQueries(item.id, { authorChatId: session.id, access: null, blocks: {} });
122
+ const captured = this.host
123
+ .capture(session.info)
124
+ .then((access) => {
125
+ const captured = copyUiValue(access, new UiBudget());
126
+ if (Buffer.byteLength(JSON.stringify(captured)) > 32 * 1024) {
127
+ throw new Error('The UI access snapshot is too large.');
128
+ }
129
+ const state = { authorChatId: session.id, access: captured, blocks: {} };
130
+ session.updateUiQueries(item.id, state);
131
+ return state;
132
+ })
133
+ .catch(() => ({ authorChatId: session.id, access: null, blocks: {} }))
134
+ .finally(() => this.captures.delete(key));
135
+ this.captures.set(key, captured);
136
+ return captured;
137
+ }
138
+ storedBlock(session, payload) {
139
+ const item = session.thread.get(payload.itemId);
140
+ const block = item?.kind === 'assistant' && !item.streaming && !item.parentToolUseId ? item.ui?.find((entry) => entry.id === payload.blockId) : undefined;
141
+ if (item?.kind !== 'assistant' || !block?.complete || block.revision !== payload.revision) {
142
+ throw new ChatError('stale-ui-block', 'This completed UI block is no longer available.');
143
+ }
144
+ return { item, block };
145
+ }
146
+ stored(session, payload) {
147
+ const { item, block } = this.storedBlock(session, payload);
148
+ if (!Object.hasOwn(block.queries, payload.query) || Object.keys(block.queries).length > MAX_QUERIES) {
149
+ throw new ChatError('refused-query', 'This block does not declare an allowed query.');
150
+ }
151
+ const name = block.queries[payload.query].source;
152
+ if (!Object.hasOwn(this.host.sources, name)) {
153
+ throw new ChatError('refused-query', 'This query source is not registered.');
154
+ }
155
+ return { item, block, source: this.host.sources[name] };
156
+ }
157
+ async query(session, payload) {
158
+ let args;
159
+ let source;
160
+ let block;
161
+ let item;
162
+ let authorAccess;
163
+ try {
164
+ ({ source, block, item } = this.stored(session, payload));
165
+ args = uiQueryArguments(block, payload.query, payload.values);
166
+ args = source.args.parse(args);
167
+ const access = await this.access(session, item);
168
+ if (access.access === null) {
169
+ throw new ChatError('refused-query', 'The writer’s original access is unavailable.');
170
+ }
171
+ authorAccess = access.access;
172
+ await source.authorize(session.info, authorAccess, args);
173
+ }
174
+ catch (error) {
175
+ return { state: 'refused', readAt: Date.now(), reason: this.reason(error) };
176
+ }
177
+ const key = JSON.stringify([session.id, item.id, block.id, block.revision, payload.query, args]);
178
+ const cached = [...this.cache.values()].find((entry) => entry.key === key);
179
+ if (cached && Date.now() - cached.reading.readAt < REFRESH_MS) {
180
+ return cached.reading;
181
+ }
182
+ const pending = this.reading.get(key);
183
+ if (pending) {
184
+ return pending;
185
+ }
186
+ const identity = JSON.stringify([session.id, item.id, block.id, block.revision, payload.query]);
187
+ if (Date.now() - (this.attempts.get(identity) ?? 0) < REFRESH_MS) {
188
+ return { state: 'failed', readAt: Date.now(), reason: 'This query may refresh once every ten seconds.' };
189
+ }
190
+ if ((this.active.get(session.id) ?? 0) >= 2 || this.totalActive >= 16) {
191
+ return { state: 'failed', readAt: Date.now(), reason: 'Too many UI queries are being read.' };
192
+ }
193
+ this.attempts.set(identity, Date.now());
194
+ while (this.attempts.size > 128) {
195
+ this.attempts.delete(this.attempts.keys().next().value);
196
+ }
197
+ this.active.set(session.id, (this.active.get(session.id) ?? 0) + 1);
198
+ this.totalActive++;
199
+ const controller = new AbortController();
200
+ const work = Promise.resolve().then(() => source.read(session.info, args, controller.signal, authorAccess));
201
+ const timeout = setTimeout(() => controller.abort(), READ_TIMEOUT_MS);
202
+ timeout.unref?.();
203
+ const read = Promise.race([
204
+ work,
205
+ new Promise((_, reject) => controller.signal.addEventListener('abort', () => reject(new Error('The UI query timed out.')), { once: true }))
206
+ ])
207
+ .then((result) => {
208
+ const value = copyUiValue(source.result.parse(result), new UiBudget());
209
+ if (Buffer.byteLength(JSON.stringify(value)) > MAX_QUERY_BYTES) {
210
+ throw new Error('The UI query result is too large.');
211
+ }
212
+ const reading = { state: 'fresh', value, readId: randomUUID(), readAt: Date.now() };
213
+ this.cache.set(reading.readId, { key, chatId: session.id, reading });
214
+ while (this.cache.size > 128) {
215
+ this.cache.delete(this.cache.keys().next().value);
216
+ }
217
+ return reading;
218
+ })
219
+ .catch((error) => ({ state: 'failed', readAt: Date.now(), reason: this.reason(error) }))
220
+ .finally(() => {
221
+ clearTimeout(timeout);
222
+ this.reading.delete(key);
223
+ });
224
+ // Timed-out sources still occupy their slot until their actual work ends.
225
+ void work
226
+ .catch(() => { })
227
+ .finally(() => {
228
+ const count = (this.active.get(session.id) ?? 1) - 1;
229
+ if (count) {
230
+ this.active.set(session.id, count);
231
+ }
232
+ else {
233
+ this.active.delete(session.id);
234
+ }
235
+ this.totalActive--;
236
+ });
237
+ this.reading.set(key, read);
238
+ return read;
239
+ }
240
+ async choiceValues(session, payload) {
241
+ const item = session.thread.get(payload.itemId);
242
+ const block = item?.kind === 'assistant' ? item.ui?.find((entry) => entry.id === payload.blockId) : undefined;
243
+ if (!block || item?.kind !== 'assistant' || ('choiceId' in payload && item.uiAnswers?.[block.id]?.revision === payload.revision)) {
244
+ return {};
245
+ }
246
+ const values = Object.create(null);
247
+ for (const name of Object.keys(payload.reads ?? {})) {
248
+ const identity = { ...payload, query: name };
249
+ const { source } = this.stored(session, identity);
250
+ const args = source.args.parse(uiQueryArguments(block, name, payload.values));
251
+ const access = await this.access(session, item);
252
+ await source.authorize(session.info, access.access, args);
253
+ const key = JSON.stringify([session.id, item.id, block.id, block.revision, name, args]);
254
+ const readId = payload.reads[name];
255
+ const cached = this.cache.get(readId);
256
+ const frozen = item.uiQueries?.blocks[block.id];
257
+ const snapshot = frozen?.revision === block.revision ? frozen?.readings[name] : undefined;
258
+ const reading = cached?.key === key
259
+ ? cached.reading
260
+ : snapshot?.readId === readId && JSON.stringify(args) === JSON.stringify(source.args.parse(uiQueryArguments(block, name)))
261
+ ? snapshot
262
+ : undefined;
263
+ if (!reading || reading.state !== 'fresh') {
264
+ throw new ChatError('stale-ui-query', 'Refresh this query before sending its choice.');
265
+ }
266
+ values[name] = reading.value;
267
+ }
268
+ return values;
269
+ }
270
+ async link(session, payload) {
271
+ try {
272
+ if (!this.host.link) {
273
+ throw new ChatError('refused-query', 'This host does not provide UI links.');
274
+ }
275
+ const { item, block } = this.storedBlock(session, payload);
276
+ if (!item.uiQueries) {
277
+ throw new ChatError('refused-query', 'The writer’s original access is unavailable.');
278
+ }
279
+ const access = await this.access(session, item);
280
+ if (access.access === null) {
281
+ throw new ChatError('refused-query', 'The writer’s original access is unavailable.');
282
+ }
283
+ const queries = await this.choiceValues(session, payload);
284
+ const target = uiLinkTargets(block, payload.values, queries)[payload.nodeId];
285
+ if (!target) {
286
+ throw new ChatError('refused-query', 'This visible node is not a supported link.');
287
+ }
288
+ return this.linkReading(await this.host.link(session.info, access.access, target));
289
+ }
290
+ catch (error) {
291
+ return { state: 'plain', reason: this.reason(error) };
292
+ }
293
+ }
294
+ forget(chatId) {
295
+ for (const [id, cached] of this.cache) {
296
+ if (cached.chatId === chatId) {
297
+ this.cache.delete(id);
298
+ }
299
+ }
300
+ }
301
+ linkReading(value) {
302
+ const reading = UiLinkResolutionSchema.parse(copyUiValue(value, new UiBudget()));
303
+ if (reading.state === 'chip' && !reading.target) {
304
+ throw new Error('The host did not resolve a link target.');
305
+ }
306
+ return reading;
307
+ }
308
+ reason(error) {
309
+ return (error instanceof Error ? error.message : 'This query could not be read.').slice(0, 240);
310
+ }
311
+ }
@@ -31,7 +31,10 @@ export function chatHandlers(chats, providers) {
31
31
  chats.detach(payload.chatId, clientId);
32
32
  return {};
33
33
  },
34
- 'chat.send': (payload) => chats.send(payload.chatId, payload.text, { mentions: payload.mentions, skills: payload.skills, chats: payload.chats }, payload.attachments),
34
+ 'chat.send': (payload) => chats.send(payload.chatId, payload.text, { delivery: payload.delivery, mentions: payload.mentions, skills: payload.skills, chats: payload.chats }, payload.attachments),
35
+ 'chat.uiChoice': (payload) => chats.choose(payload),
36
+ 'ui.link': (payload, clientId) => chats.linkUi(payload, clientId),
37
+ 'ui.query': (payload, clientId) => chats.queryUi(payload, clientId),
35
38
  'chat.unqueue': (payload) => ({ message: chats.unqueue(payload.chatId, payload.messageId) }),
36
39
  'chat.sendNow': (payload) => {
37
40
  chats.sendNow(payload.chatId, payload.messageId);
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@adecore/agents",
3
- "version": "0.20.0",
3
+ "version": "0.21.0",
4
4
  "description": "Node-compatible agent chat hosts, providers, accounts, usage and durable task coordination.",
5
5
  "type": "module",
6
6
  "exports": {
@@ -20,13 +20,14 @@
20
20
  "test": "bun --conditions=source test"
21
21
  },
22
22
  "dependencies": {
23
- "@adecore/agent-contracts": "0.20.0",
23
+ "@adecore/agent-contracts": "0.21.0",
24
+ "@adecore/intelligent-ui": "0.21.0",
24
25
  "zod": "^4.6.5"
25
26
  },
26
27
  "devDependencies": {
27
28
  "@types/bun": "^1.4.2",
28
- "typescript": "^7.0.2",
29
- "@types/node": "^26.6.3"
29
+ "@types/node": "^26.6.3",
30
+ "typescript": "^7.0.2"
30
31
  },
31
32
  "license": "FSL-1.1-MIT",
32
33
  "author": "Bas Milius",
@@ -1,5 +1,5 @@
1
- import { randomBytes } from 'node:crypto';
2
- import { mkdir, rm, writeFile } from 'node:fs/promises';
1
+ import { createHash, randomBytes } from 'node:crypto';
2
+ import { link, mkdir, readFile, rm, writeFile } from 'node:fs/promises';
3
3
  import { extname, join } from 'node:path';
4
4
  import { attachmentImageMime, type ChatAttachment, type ChatAttachmentUpload } from '@adecore/agent-contracts';
5
5
 
@@ -57,6 +57,42 @@ export class AttachmentStore {
57
57
  return { id, name: upload.name, mime, size: bytes.byteLength, path };
58
58
  }
59
59
 
60
+ async saveGenerated(chatId: string, ref: string, upload: ChatAttachmentUpload): Promise<ChatAttachment> {
61
+ const id = createHash('sha256').update(`codex-image:${ref}`).digest('hex');
62
+ const bytes = Buffer.from(upload.data, 'base64');
63
+ const folder = this.folderOf(chatId);
64
+ await mkdir(folder, { recursive: true, mode: 0o700 });
65
+ const path = join(folder, `${id}.${extensionFor('', upload.mime)}`);
66
+ const temporary = join(folder, `.${id}-${randomBytes(8).toString('hex')}.tmp`);
67
+ try {
68
+ await writeFile(temporary, bytes, { mode: 0o600, flag: 'wx' });
69
+ try {
70
+ // Publish complete bytes without replacing an image another replay has already saved.
71
+ await link(temporary, path);
72
+ } catch (error) {
73
+ if ((error as NodeJS.ErrnoException).code !== 'EEXIST') {
74
+ throw error;
75
+ }
76
+ const previous = await readFile(path);
77
+ if (!previous.equals(bytes)) {
78
+ throw new Error('The generated image id already belongs to another image');
79
+ }
80
+ }
81
+ } finally {
82
+ await rm(temporary, { force: true });
83
+ }
84
+ return { id, name: upload.name, mime: upload.mime, size: bytes.byteLength, path };
85
+ }
86
+
87
+ async copy(chatId: string, attachment: ChatAttachment): Promise<ChatAttachment> {
88
+ const bytes = await readFile(attachment.path);
89
+ const folder = this.folderOf(chatId);
90
+ await mkdir(folder, { recursive: true, mode: 0o700 });
91
+ const path = join(folder, `${createHash('sha256').update(attachment.id).digest('hex')}.${extensionFor(attachment.name, attachment.mime)}`);
92
+ await writeFile(path, bytes, { mode: 0o600 });
93
+ return { ...attachment, path };
94
+ }
95
+
60
96
  /* Everything a chat attached; used when the chat itself is deleted. */
61
97
  async removeAll(chatId: string): Promise<void> {
62
98
  await rm(this.folderOf(chatId), { recursive: true, force: true });
@@ -1,5 +1,6 @@
1
1
  import type {
2
2
  ChatAttachment,
3
+ ChatAttachmentUpload,
3
4
  ChatFileChange,
4
5
  ChatQuestion,
5
6
  ChatSkill,
@@ -42,6 +43,7 @@ export interface BackendLaunch {
42
43
  folders?: readonly string[];
43
44
  // How the CLI is started; a test runs a fake in the same process, everything else spawns it.
44
45
  spawn?: SpawnChatProcess;
46
+ saveGeneratedImage?(ref: string, upload: ChatAttachmentUpload): Promise<ChatAttachment>;
45
47
  }
46
48
 
47
49
  export interface TurnInput {
@@ -78,9 +80,19 @@ export type BackendEvent =
78
80
  // The CLI renamed its thread, for a protocol that says so.
79
81
  | { type: 'title'; title: string }
80
82
  | { type: 'permissions'; permissionMode: string; effectiveRuntimeMode?: RuntimeMode }
81
- | { type: 'text.delta'; ref: string; text: string }
83
+ | { type: 'text.delta'; ref: string; text: string; phase?: string }
82
84
  // `parentRef` is set for text a subagent wrote; it belongs to that agent's row, not to the thread.
83
- | { type: 'text.done'; ref: string; text: string; parentRef?: string | null }
85
+ | { type: 'text.done'; ref: string; text: string; parentRef?: string | null; phase?: string }
86
+ // Bytes are consumed by the backend before anything reaches the thread or its log.
87
+ | {
88
+ type: 'image.generated';
89
+ ref: string;
90
+ upload: ChatAttachmentUpload;
91
+ width?: number;
92
+ height?: number;
93
+ revisedPrompt?: string;
94
+ transparentBackground?: boolean;
95
+ }
84
96
  // What the model thought before it answered; consecutive blocks become one thinking item.
85
97
  | { type: 'thinking.delta'; ref: string; text: string }
86
98
  | { type: 'thinking.done'; ref: string; text: string }
@@ -175,6 +187,8 @@ export interface ChatBackend {
175
187
  // Starts the process if it has not started; resolves once the CLI can take a turn.
176
188
  start(): Promise<void>;
177
189
  sendTurn(input: TurnInput): void;
190
+ // Resolves after native acceptance; false means the turn ended before the input could join it.
191
+ steerTurn?(input: TurnInput): Promise<boolean>;
178
192
  compact(): void;
179
193
  interrupt(): void;
180
194
  // False when nothing waits under that id, so the caller can answer the client with an error.
@@ -2,6 +2,7 @@ import { homedir } from 'node:os';
2
2
  import { join } from 'node:path';
3
3
  import {
4
4
  ChatAttachmentUploadsSchema,
5
+ generatedImageAttachment,
5
6
  type AgentKind,
6
7
  type ChatAttachment,
7
8
  type ChatAttachResult,
@@ -15,6 +16,9 @@ import {
15
16
  type ChatInfo,
16
17
  type ChatItem,
17
18
  type ChatQueuedMessage,
19
+ type ChatUiChoicePayload,
20
+ type ChatUiQueryPayload,
21
+ type ChatUiLinkPayload,
18
22
  type ChatSkill,
19
23
  type ChatSubagentPayload,
20
24
  type ChatSubagentResult,
@@ -23,6 +27,7 @@ import {
23
27
  type ModelSelection,
24
28
  type RuntimeMode
25
29
  } from '@adecore/agent-contracts';
30
+ import { ChatUiQueries, type ChatUiHost } from './ui-queries.ts';
26
31
  import { ClientSinks } from '../client-sinks.ts';
27
32
  import { errorText } from '../error-text.ts';
28
33
  import type { AgentEvent, AgentSink } from '../events.ts';
@@ -96,6 +101,7 @@ const DEFAULT_RESUME_WORDS: ResumeWords = {
96
101
  };
97
102
 
98
103
  export interface ChatCoreOptions {
104
+ intelligentUi?: ChatUiHost;
99
105
  providers: ProviderRegistry;
100
106
  // Where the threads are kept; without it a chat lives as long as the process.
101
107
  store?: ChatStore;
@@ -168,6 +174,7 @@ export class ChatCore {
168
174
  /* Where Claude Code keeps the projects of its default account on this machine; empty when it has none. */
169
175
  readonly claudeProjectsDir: string;
170
176
  readonly composerPreferences = new ComposerPreferences();
177
+ private readonly uiQueries: ChatUiQueries | null;
171
178
  private readonly checkpoints: TurnCheckpoints | null;
172
179
  private readonly instructions: string | null;
173
180
  private readonly skillIndex: SkillIndex;
@@ -194,6 +201,7 @@ export class ChatCore {
194
201
  private readonly limitResume: ChatCoreOptions['limitResume'] | null;
195
202
 
196
203
  constructor(options: ChatCoreOptions) {
204
+ this.uiQueries = options.intelligentUi ? new ChatUiQueries(options.intelligentUi) : null;
197
205
  this.providers = options.providers;
198
206
  this.store = options.store ?? null;
199
207
  this.bookmarks = options.bookmarks ?? null;
@@ -390,6 +398,7 @@ export class ChatCore {
390
398
  const references = this.referencesFor(payload.chatId);
391
399
  const session = new ChatSession({
392
400
  info,
401
+ ...(this.uiQueries ? { ui: { schemas: this.uiQueries.schemas, observe: (item, final) => this.uiQueries!.observe(session, item, final) } } : {}),
393
402
  items: stored?.items ?? [],
394
403
  preambles: stored?.preambles ?? [],
395
404
  provider,
@@ -400,6 +409,7 @@ export class ChatCore {
400
409
  this.accounts?.launched?.(kind, chatAccount);
401
410
  return env;
402
411
  },
412
+ saveGeneratedImage: (ref, upload) => this.attachments.saveGenerated(payload.chatId, ref, upload),
403
413
  instructions: () => this.instructionsFor(payload.chatId),
404
414
  resumeNote: () => this.resumeNoteFor(payload.chatId),
405
415
  folders: () => this.foldersFor(payload.chatId),
@@ -495,11 +505,16 @@ export class ChatCore {
495
505
  const log = this.logs.get(chatId)!;
496
506
  const events = since === undefined ? null : log.after(since);
497
507
  if (events !== null) {
498
- return { info: session.info, items: [], events, seq: log.seq };
508
+ return { info: session.info, items: [], events: [...events, ...session.uiPreviews], seq: log.seq };
499
509
  }
500
- return historyLimit === undefined
501
- ? { ...session.thread.snapshot(), seq: log.seq }
502
- : { info: session.info, ...session.thread.history(historyLimit), pending: session.thread.pending(), seq: log.seq };
510
+ const page = historyLimit === undefined ? session.thread.snapshot() : session.thread.history(historyLimit);
511
+ return {
512
+ ...page,
513
+ info: session.info,
514
+ items: session.previewItems(page.items),
515
+ ...(historyLimit === undefined ? {} : { pending: session.previewItems(session.thread.pending()) }),
516
+ seq: log.seq
517
+ };
503
518
  }
504
519
 
505
520
  /*
@@ -700,7 +715,32 @@ export class ChatCore {
700
715
  throw new ChatError('invalid-attachments', checked.error.issues[0]!.message);
701
716
  }
702
717
  const attachments = await Promise.all(uploads.map((upload) => this.attachments.save(chatId, upload)));
703
- return session.send(text, { ...extras, ...(attachments.length > 0 ? { attachments } : {}) });
718
+ return session.sendInput(text, { ...extras, ...(attachments.length > 0 ? { attachments } : {}) });
719
+ }
720
+
721
+ async linkUi(payload: ChatUiLinkPayload, clientId: string) {
722
+ if (!this.attached.get(payload.chatId)?.has(clientId) || !this.uiQueries) {
723
+ throw new ChatError('refused-query', 'Attach to the chat before resolving its UI links.');
724
+ }
725
+ return this.uiQueries.link(this.require(payload.chatId), payload);
726
+ }
727
+
728
+ async queryUi(payload: ChatUiQueryPayload, clientId: string) {
729
+ if (!this.attached.get(payload.chatId)?.has(clientId)) {
730
+ throw new ChatError('refused-query', 'Attach to the chat before reading its UI queries.');
731
+ }
732
+ if (!this.uiQueries) {
733
+ throw new ChatError('refused-query', 'This host does not provide UI query sources.');
734
+ }
735
+ return this.uiQueries.query(this.require(payload.chatId), payload);
736
+ }
737
+
738
+ async choose(payload: ChatUiChoicePayload): Promise<{ queued: boolean; turnId: string }> {
739
+ const session = this.require(payload.chatId);
740
+ const queries = await this.uiQueries?.choiceValues(session, payload);
741
+ const result = session.choose(payload, queries);
742
+ await this.persisted(payload.chatId);
743
+ return result;
704
744
  }
705
745
 
706
746
  /* The file behind an attachment id: what this chat's thread or queue says it is, or a visual's page. */
@@ -710,7 +750,8 @@ export class ChatCore {
710
750
  return null;
711
751
  }
712
752
  for (const item of session.thread.list()) {
713
- const found = item.kind === 'user' ? item.attachments?.find((attachment) => attachment.id === id) : undefined;
753
+ const generated = generatedImageAttachment(item);
754
+ const found = item.kind === 'user' ? item.attachments?.find((attachment) => attachment.id === id) : generated?.id === id ? generated : null;
714
755
  if (found) {
715
756
  return found;
716
757
  }
@@ -759,7 +800,7 @@ export class ChatCore {
759
800
  throw new ChatError('chat-busy', `Chat ${chatId} is still working on the previous message`);
760
801
  }
761
802
  await this.clearing(chatId);
762
- session.clear(force);
803
+ const ended = session.clear(force);
763
804
  // Before the record is written, so what it takes along from the record is gone from the write as well.
764
805
  const cleared = this.cleared(chatId);
765
806
  // A debounced write still waiting holds the old thread and must not land after the empty one.
@@ -767,7 +808,7 @@ export class ChatCore {
767
808
  // Folded right away: every line before the reset describes a thread that is gone.
768
809
  await Promise.all([
769
810
  this.persistNow(chatId, true),
770
- this.attachments.removeAll(chatId),
811
+ ended.then(() => this.attachments.removeAll(chatId)),
771
812
  this.bookmarks?.removeChat(chatId),
772
813
  this.visuals?.removeChat(chatId),
773
814
  cleared
@@ -812,8 +853,9 @@ export class ChatCore {
812
853
  const session = this.require(chatId);
813
854
  // A write already out would put the record back after the delete.
814
855
  const writing = this.writes.get(chatId);
815
- void session.dispose();
856
+ const ended = session.dispose();
816
857
  this.subagents.releaseChat(chatId);
858
+ this.uiQueries?.forget(chatId);
817
859
  this.coalescers.get(chatId)?.dispose();
818
860
  this.coalescers.delete(chatId);
819
861
  this.logs.get(chatId)?.close();
@@ -829,6 +871,7 @@ export class ChatCore {
829
871
  const snapshot = session.thread.snapshot();
830
872
  this.forgotten(chatId);
831
873
  await writing;
874
+ await ended;
832
875
  await Promise.all([
833
876
  this.store?.delete(chatId),
834
877
  this.attachments.removeAll(chatId),
@@ -1162,6 +1205,14 @@ export class ChatCore {
1162
1205
  }
1163
1206
 
1164
1207
  private emit(chatId: string, event: ChatEvent): void {
1208
+ if (event.type === 'delta' && event.ui !== undefined) {
1209
+ // Previews follow all pending text but have no log position or observer side effects.
1210
+ this.coalescers.get(chatId)?.flush();
1211
+ for (const clientId of this.attached.get(chatId) ?? []) {
1212
+ this.sinks.to(clientId, { event: 'chat.event', payload: { chatId, event } });
1213
+ }
1214
+ return;
1215
+ }
1165
1216
  this.activity.set(chatId, Date.now());
1166
1217
  let coalescer = this.coalescers.get(chatId);
1167
1218
  if (!coalescer) {