@adecore/agents 0.21.0 → 0.21.1

package/src/chat/chat-session.ts

@@ -18,7 +18,7 @@ import type {
     RuntimeMode
 } from '@adecore/agent-contracts';
 import { GeneratedImageInputSchema, notResumedNote } from '@adecore/agent-contracts';
-import { resolveUiChoice, UiFailure } from '@adecore/intelligent-ui';
+import { resolveUiChoice, UI_FENCE_LANGUAGE, UiFailure, uiHasFence, uiMayReferenceHost } from '@adecore/intelligent-ui';
 import type { UiCompileOptions } from '@adecore/intelligent-ui/compiler';
 import { UiStream, type UiStreamPreview } from '@adecore/intelligent-ui/stream';
 import type { ChatProvider } from '../providers/provider.ts';
@@ -30,6 +30,7 @@ import type { ChatTitleInput } from './chat-title.ts';
 import { ChatError } from './errors.ts';
 import { limitedTurn, limitResumeAt, limitResumeWake } from './limit-resume.ts';
 import { isMainAgentOutput, ThreadProjector } from './projector.ts';
+import { readableAssistantText } from './readable-text.ts';
 import type { SubagentSettlement } from './subagent-settlement.ts';
 import { ChatThread } from './thread.ts';
 import { errorText } from '../error-text.ts';
@@ -60,6 +61,8 @@ export interface ChatSessionOptions {
     items?: ChatItem[];
     // Said once, in front of the next real prompt, and kept in the record until then (a fork's note for its agent).
     preambles?: string[];
+    // The access a UI host captured per reply, from the record; never on the wire.
+    uiAccess?: Readonly<Record<string, unknown>>;
     provider: ChatProvider;
     // The executable and leading arguments; a test points this at a fake CLI.
     command: string[];
@@ -76,6 +79,8 @@ export interface ChatSessionOptions {
     references?: ChatReferences;
     // Trees per turn, so a settled turn can show what the working tree holds against its start.
     checkpoints?: TurnCheckpoints;
+    // The info string of the host's UI fences; `UI_FENCE_LANGUAGE` without one.
+    uiFenceLanguage?: string;
     ui?: {
         schemas: UiCompileOptions['querySchemas'];
         observe(item: ChatAssistantItem, final: boolean): void;
@@ -109,6 +114,8 @@ export interface LimitResumeHooks {
 
 // Claude Code names a session about six seconds after its first prompt, and a first turn can run for minutes.
 const TITLE_RECHECK_MS = 10_000;
+// Longer than any fence line or the name of a tag a scan looks for, so a match never falls between two deltas.
+const UI_SCAN_OVERLAP = 64;
 
 export interface ChatSendExtras {
     uiChoice?: ChatUiChoiceOrigin;
@@ -193,10 +200,18 @@ export class ChatSession {
     private readonly steering = new Set<string>();
     private readonly uiStreams = new Map<string, UiStream>();
     private readonly uiCompiled = new Map<string, UiStreamPreview>();
+    private readonly capturedUiAccess: Map<string, unknown>;
+    // How much of a streaming reply's text the fence and host scans already covered, so a delta scans only its tail.
+    private readonly uiScanned = new Map<string, number>();
+    private readonly uiReferencing = new Set<string>();
+    private readonly uiAttachments = new Map<string | null, string | undefined>();
+    private readonly uiFenceLanguage: string;
 
     constructor(options: ChatSessionOptions) {
         this.options = options;
+        this.uiFenceLanguage = options.uiFenceLanguage ?? UI_FENCE_LANGUAGE;
         this.pendingPreambles = options.preambles ?? [];
+        this.capturedUiAccess = new Map(Object.entries(options.uiAccess ?? {}));
         this.thread = new ChatThread(options.info, options.items);
         this.projector = new ThreadProjector(this.thread, { providerName: options.provider.name });
     }
@@ -314,6 +329,24 @@ export class ChatSession {
         return { queued: false, turnId };
     }
 
+    /* What a UI host captured for a reply; null where the capture was denied or is still out, undefined where none started. */
+    uiAccess(itemId: string): unknown {
+        return this.capturedUiAccess.get(itemId);
+    }
+
+    setUiAccess(itemId: string, access: unknown): void {
+        if (this.frozen) {
+            return;
+        }
+        this.capturedUiAccess.set(itemId, access);
+        this.options.persistSoon();
+    }
+
+    /* The captured access of the replies still in the thread, for the record. */
+    get uiAccessRecord(): Record<string, unknown> {
+        return Object.fromEntries([...this.capturedUiAccess].filter(([itemId]) => this.thread.get(itemId) !== undefined));
+    }
+
     updateUiQueries(itemId: string, uiQueries: ChatUiQueryState): void {
         const item = this.thread.get(itemId);
         if (this.frozen || item?.kind !== 'assistant' || item.parentToolUseId) {
@@ -1828,7 +1861,7 @@ export class ChatSession {
         this.naming = true;
         const turnId = done[0]!.id;
         const answer = items
-            .map((item) => (item.kind === 'assistant' && item.turnId === turnId ? item.text : ''))
+            .map((item) => (item.kind === 'assistant' && item.turnId === turnId ? readableAssistantText(item) : ''))
             .filter((text) => text !== '')
             .join('\n\n');
         const agentSessionId = this.thread.info.agentSessionId;
@@ -1886,19 +1919,28 @@ export class ChatSession {
             if (event.type === 'reset') {
                 this.clearUi();
             }
+            if (event.type === 'item' && event.item.kind === 'tool') {
+                this.uiAttachments.delete(event.item.turnId);
+            }
             const item = event.type === 'item' ? event.item : event.type === 'delta' ? this.thread.get(event.itemId) : null;
             if (item?.kind !== 'assistant' || item.parentToolUseId) {
                 this.options.emit(event);
                 continue;
             }
+            const tail = this.unscannedUiText(item, event.type === 'item');
+            if (uiMayReferenceHost(tail)) {
+                this.uiReferencing.add(item.id);
+            }
             let stream = this.uiStreams.get(item.id);
-            if (!stream && !item.text.includes('ruimte-ui')) {
+            if (!stream && !uiHasFence(tail, this.uiFenceLanguage)) {
+                this.forgetUiScan(item);
                 this.options.emit(event);
                 continue;
             }
             if (!stream) {
                 stream = new UiStream({
                     querySchemas: this.options.ui?.schemas,
+                    fenceLanguage: this.uiFenceLanguage,
                     id: item.id,
                     clock: this.options.clock ?? systemClock,
                     emit: (preview) => {
@@ -1908,7 +1950,10 @@ export class ChatSession {
                 });
                 this.uiStreams.set(item.id, stream);
             }
-            this.options.ui?.observe(item, false);
+            if (this.uiReferencing.has(item.id)) {
+                this.options.ui?.observe(item, false);
+            }
+            this.forgetUiScan(item);
             const latestAttachment = this.latestUiAttachment(item.turnId);
             if (event.type === 'item' && !item.streaming) {
                 const ui = stream
@@ -1919,7 +1964,7 @@ export class ChatSession {
                 const diagnostics = [...new Set(ui.flatMap((block) => block.diagnostics.map((diagnostic) => diagnostic.message)))].slice(0, 6);
                 if (diagnostics.length > 0) {
                     const repair =
-                        `Your last ruimte-ui block had these diagnostics: ${diagnostics.join(' ')} Use the catalog's supported props and components.`.slice(
+                        `Your last ${this.uiFenceLanguage} block had these diagnostics: ${diagnostics.join(' ')} Use the catalog's supported props and components.`.slice(
                             0,
                             1200
                         );
@@ -1939,7 +1984,39 @@ export class ChatSession {
         }
     }
 
+    /*
+     * The text a scan has not covered yet, from the start of the line the last scan ended in, since a fence
+     * is one line; a long line is cut short, behind a character no fence or tag can follow.
+     */
+    private unscannedUiText(item: ChatAssistantItem, whole: boolean): string {
+        const scanned = whole ? 0 : (this.uiScanned.get(item.id) ?? 0);
+        this.uiScanned.set(item.id, item.text.length);
+        if (scanned === 0 || scanned > item.text.length) {
+            return item.text;
+        }
+        const line = item.text.lastIndexOf('\n', scanned - 1) + 1;
+        const from = Math.max(line, scanned - UI_SCAN_OVERLAP);
+        return from === line ? item.text.slice(from) : `\0${item.text.slice(from)}`;
+    }
+
+    private forgetUiScan(item: ChatAssistantItem): void {
+        if (!item.streaming) {
+            this.uiScanned.delete(item.id);
+            this.uiReferencing.delete(item.id);
+        }
+    }
+
+    /* Looked up once per turn, and again only after one of its tool calls changed. */
     private latestUiAttachment(turnId: string | null): string | undefined {
+        if (this.uiAttachments.has(turnId)) {
+            return this.uiAttachments.get(turnId);
+        }
+        const attachment = this.findUiAttachment(turnId);
+        this.uiAttachments.set(turnId, attachment);
+        return attachment;
+    }
+
+    private findUiAttachment(turnId: string | null): string | undefined {
         for (const item of this.thread.list().toReversed()) {
             if (item.kind !== 'tool' || item.turnId !== turnId || item.name !== 'ImageGeneration' || item.state !== 'done') {
                 continue;
@@ -1958,6 +2035,9 @@ export class ChatSession {
         }
         this.uiStreams.clear();
         this.uiCompiled.clear();
+        this.uiScanned.clear();
+        this.uiReferencing.clear();
+        this.uiAttachments.clear();
     }
 
     /* Said on the info as well as by the rows, since a client with the thread closed sees only the info. */

package/src/chat/chat-store.ts

@@ -20,10 +20,12 @@ const RecordSchema = z.looseObject({
     seq: z.number().int().nonnegative().optional(),
     resetSeq: z.number().int().nonnegative().optional(),
     // Never on the wire: what goes in front of the next real prompt, once (a fork's note for its agent).
-    preambles: z.array(z.string()).optional()
+    preambles: z.array(z.string()).optional(),
+    // Never on the wire: the access a UI host captured for each reply that reads it, by item id.
+    uiAccess: z.record(z.string(), z.unknown()).optional()
 });
 
-const OWN_KEYS = new Set(['info', 'items', 'seq', 'resetSeq', 'preambles']);
+const OWN_KEYS = new Set(['info', 'items', 'seq', 'resetSeq', 'preambles', 'uiAccess']);
 
 /* What a host keeps in a chat's record beside the thread; it never reaches the wire. */
 export type ChatRecordExtras = Record<string, unknown>;
@@ -39,6 +41,7 @@ export interface ChatRecord {
     seq: number;
     resetSeq: number;
     preambles: string[];
+    uiAccess: Record<string, unknown>;
     extras: ChatRecordExtras;
     // The log as it is on disk, for the lines an attach with `since` may still be answered from.
     lines: ChatLogLine[];
@@ -54,8 +57,37 @@ function logName(chatId: string): string {
     return `${encodeURIComponent(chatId)}.log`;
 }
 
-function recordBody(info: ChatInfo, items: ChatItem[], at: ChatSeq, preambles: readonly string[], extras: ChatRecordExtras): string {
-    return JSON.stringify({ info, items, ...at, ...(preambles.length === 0 ? {} : { preambles }), ...withoutOwnKeys(extras) });
+function recordBody(
+    info: ChatInfo,
+    items: ChatItem[],
+    at: ChatSeq,
+    preambles: readonly string[],
+    extras: ChatRecordExtras,
+    uiAccess: Readonly<Record<string, unknown>>
+): string {
+    return JSON.stringify({
+        info,
+        items,
+        ...at,
+        ...(preambles.length === 0 ? {} : { preambles }),
+        ...(Object.keys(uiAccess).length === 0 ? {} : { uiAccess }),
+        ...withoutOwnKeys(extras)
+    });
+}
+
+/* A record from before `uiAccess` kept the access inside each reply's `uiQueries`, which the item schema now drops. */
+function liftUiAccess(record: unknown): unknown {
+    if (typeof record !== 'object' || record === null || 'uiAccess' in record || !Array.isArray((record as { items?: unknown }).items)) {
+        return record;
+    }
+    const uiAccess: Record<string, unknown> = {};
+    for (const item of (record as { items: unknown[] }).items) {
+        const queries = (item as { uiQueries?: unknown } | null)?.uiQueries;
+        if (typeof queries === 'object' && queries !== null && 'access' in queries) {
+            uiAccess[String((item as { id: unknown }).id)] = queries.access;
+        }
+    }
+    return Object.keys(uiAccess).length === 0 ? record : { ...record, uiAccess };
 }
 
 function withoutOwnKeys(extras: ChatRecordExtras): ChatRecordExtras {
@@ -127,9 +159,10 @@ export class ChatStore {
         items: ChatItem[],
         at: ChatSeq = { seq: 0, resetSeq: 0 },
         preambles: readonly string[] = [],
-        extras: ChatRecordExtras = {}
+        extras: ChatRecordExtras = {},
+        uiAccess: Readonly<Record<string, unknown>> = {}
     ): Promise<number> {
-        const body = recordBody(info, items, at, preambles, extras);
+        const body = recordBody(info, items, at, preambles, extras, uiAccess);
         await mkdir(this.dir, { recursive: true, mode: 0o700 });
         await writeAtomic(join(this.dir, recordFileName(chatId)), body);
         return body.length;
@@ -145,10 +178,11 @@ export class ChatStore {
         items: ChatItem[],
         at: ChatSeq = { seq: 0, resetSeq: 0 },
         preambles: readonly string[] = [],
-        extras: ChatRecordExtras = {}
+        extras: ChatRecordExtras = {},
+        uiAccess: Readonly<Record<string, unknown>> = {}
     ): void {
         mkdirSync(this.dir, { recursive: true, mode: 0o700 });
-        writeAtomicSync(join(this.dir, recordFileName(chatId)), recordBody(info, items, at, preambles, extras));
+        writeAtomicSync(join(this.dir, recordFileName(chatId)), recordBody(info, items, at, preambles, extras, uiAccess));
     }
 
     /* Null for a chat with nothing on disk to make it of, and for one whose record does not parse (see `unreadable`). */
@@ -174,7 +208,15 @@ export class ChatStore {
                 resetSeq = line.seq;
             }
         }
-        return { ...thread.snapshot(), seq, resetSeq, preambles: snapshot.preambles ?? [], extras: extrasOf(snapshot), lines };
+        return {
+            ...thread.snapshot(),
+            seq,
+            resetSeq,
+            preambles: snapshot.preambles ?? [],
+            uiAccess: snapshot.uiAccess ?? {},
+            extras: extrasOf(snapshot),
+            lines
+        };
     }
 
     /*
@@ -204,7 +246,7 @@ export class ChatStore {
                 await writeAtomic(join(this.dir, recordFileName(chatId)), JSON.stringify(migrated));
             }
         }
-        const parsed = RecordSchema.safeParse(record);
+        const parsed = RecordSchema.safeParse(liftUiAccess(record));
         return parsed.success ? parsed.data : UNREADABLE;
     }
 
@@ -260,5 +302,5 @@ function fromLogAlone(lines: ChatLogLine[]): ChatRecord | null {
             resetSeq = line.seq;
         }
     }
-    return { ...thread.snapshot(), seq: 0, resetSeq, preambles: [], extras: {}, lines };
+    return { ...thread.snapshot(), seq: 0, resetSeq, preambles: [], uiAccess: {}, extras: {}, lines };
 }

package/src/chat/readable-text.ts

@@ -0,0 +1,18 @@
+import type { ChatAssistantItem } from '@adecore/agent-contracts';
+import { uiFallbackText } from '@adecore/intelligent-ui/text';
+
+/*
+ * An assistant's words as another agent or a person outside the thread reads them: each UI block becomes
+ * its text fallback, with the first query values when they were frozen for the block as stored. It works
+ * from the stored block ranges, so it holds whatever fence language the host compiled.
+ */
+export function readableAssistantText(item: Pick<ChatAssistantItem, 'text' | 'ui' | 'uiQueries'>): string {
+    if (!item.ui?.length) {
+        return item.text;
+    }
+    const blocks = item.ui.map((block) => {
+        const frozen = item.uiQueries?.blocks[block.id];
+        return frozen && frozen.revision === block.revision ? { ...block, fallback: frozen.fallback } : block;
+    });
+    return uiFallbackText(item.text, blocks);
+}