@salsita-ai/sdk 0.50.0-develop.5a0bcdee42 → 0.50.0-develop.5ac9723d62

package/dist/configurator/application-runtime/extensions/systems/assistant-system/assistant-speech-player.d.ts

@@ -0,0 +1,69 @@
+import type { ObservableWithLastValue } from '@salsita-ai/common/utils';
+/**
+ * What the reading aloud of a reply is doing:
+ * - `idle`: no reply is read
+ * - `loading`: the reply's speech is on its way from the server
+ * - `playing`: the reply is read aloud
+ * - `blocked`: the browser refused to start reading the reply on its own; the reply's play button starts it
+ */
+export type AssistantPlaybackStatus = 'idle' | 'loading' | 'playing' | 'blocked';
+/** The reply read aloud, by the id of its message, and what its reading is doing; no reply while `idle`. */
+export interface AssistantPlayback {
+    readonly messageId: string | null;
+    readonly status: AssistantPlaybackStatus;
+}
+/** Starts the reading of a reply, by the id of its message, and how. */
+export interface AssistantSpeechPlayRequest {
+    readonly messageId: string;
+    /** Reads the reply's speech from the server. */
+    readonly loadSpeech: () => Promise<Blob>;
+    /**
+     * True when the panel reads the reply on its own, after a voice message: a browser that refuses to start it leaves
+     * the reply `blocked` for the user to start, where a refused tap is a failure.
+     */
+    readonly isAutoplay: boolean;
+}
+interface AssistantSpeechPlayerContext {
+    /** The audio element every reply is read through; see `AssistantSystemContext.createAudio`. */
+    readonly createAudio: () => HTMLAudioElement;
+    readonly logError: (message: string, error: Error) => void;
+}
+/**
+ * Reads the assistant's replies aloud, one at a time, through one audio element: starting a reply stops the one being
+ * read. The speech of a reply is read from the server once and kept as an object URL while the configuration stays
+ * loaded. Whether the user muted the replies that read themselves aloud is kept in the browser.
+ */
+export declare class AssistantSpeechPlayer {
+    private readonly _context;
+    private readonly _playback;
+    private readonly _muted;
+    private _audio;
+    private _isUnlocked;
+    /** Identity of the latest start; a reading whose speech arrives after another one started is dropped. */
+    private _attempt;
+    /** Object URLs of the speech read so far, by message id. */
+    private _urls;
+    constructor(_context: AssistantSpeechPlayerContext);
+    readonly playback$: ObservableWithLastValue<AssistantPlayback>;
+    readonly muted$: ObservableWithLastValue<boolean>;
+    /**
+     * Plays silence on the audio element, once, while the user taps: Safari lets a page start audio on its own only on
+     * an element that played during a tap before.
+     */
+    unlock: () => void;
+    /** Reads the reply aloud, stopping the one being read; the reply being read or loaded is stopped instead. */
+    toggle: (request: AssistantSpeechPlayRequest) => Promise<void>;
+    /** Reads the reply aloud, stopping the one being read. */
+    play: ({ messageId, loadSpeech, isAutoplay }: AssistantSpeechPlayRequest) => Promise<void>;
+    /** Stops the reply being read, and forgets a reply waiting to be started. */
+    stop: () => void;
+    /** Mutes the replies that read themselves aloud, and stops the one being read; the play buttons keep working. */
+    setMuted: (muted: boolean) => void;
+    /** Stops reading, and revokes the speech read for the configuration that was loaded. */
+    reset: () => void;
+    private _finish;
+    private _getAudio;
+    /** The object URL of a reply's speech, read once; a read that failed is forgotten, so it can be asked for again. */
+    private _loadUrl;
+}
+export {};

package/dist/configurator/application-runtime/extensions/systems/assistant-system/assistant-speech-player.js

@@ -0,0 +1 @@
+import{BehaviorSubject as t}from"rxjs";let e=Object.freeze({messageId:null,status:"idle"}),s="phx-assistant-muted";export class AssistantSpeechPlayer{_context;_playback=new t(e);_muted=new t((()=>{try{return"true"===localStorage.getItem(s)}catch{return!1}})());_audio=null;_isUnlocked=!1;_attempt=null;_urls=new Map;constructor(t){this._context=t}playback$=this._playback;muted$=this._muted;unlock=()=>{if(this._isUnlocked)return;this._isUnlocked=!0,this.stop();let t=this._getAudio();t.src="data:audio/wav;base64,UklGRiQAAABXQVZFZm10IBAAAAABAAEARKwAAIhYAQACABAAZGF0YQAAAAA=",t.play().catch(()=>void 0)};toggle=async t=>{let{messageId:e,status:s}=this._playback.value;e===t.messageId&&("loading"===s||"playing"===s)?this.stop():await this.play(t)};play=async({messageId:t,loadSpeech:e,isAutoplay:s})=>{let a={},i=this._getAudio();this._attempt=a,i.pause(),this._playback.next({messageId:t,status:"loading"});try{let s=await this._loadUrl(t,e);if(this._attempt!==a)return;i.src=s,await i.play(),this._attempt===a&&this._playback.next({messageId:t,status:"playing"})}catch(e){if(this._attempt!==a)return;if(s&&e instanceof DOMException&&"NotAllowedError"===e.name)return void this._playback.next({messageId:t,status:"blocked"});this._context.logError("The assistant reply could not be read aloud",e instanceof Error?e:Error(String(e))),this._finish()}};stop=()=>{this._audio?.pause(),this._finish()};setMuted=t=>{t&&this.stop(),t!==this._muted.value&&this._muted.next(t);try{localStorage.setItem(s,String(t))}catch{}};reset=()=>{for(let t of(this.stop(),this._urls.values()))t.then(URL.revokeObjectURL,()=>void 0);this._urls=new Map};_finish=()=>{this._attempt=null,this._playback.value!==e&&this._playback.next(e)};_getAudio=()=>{if(this._audio)return this._audio;let t=this._context.createAudio();return t.addEventListener("ended",()=>{"playing"===this._playback.value.status&&this._finish()}),this._audio=t,t};_loadUrl=(t,e)=>{let s=this._urls.get(t);if(s)return s;let a=this._urls,i=e().then(t=>URL.createObjectURL(t));return a.set(t,i),i.catch(()=>{a.get(t)===i&&a.delete(t)}),i}}

package/dist/configurator/application-runtime/extensions/systems/assistant-system/assistant-system.d.ts

@@ -1,24 +1,64 @@
 import type { AnyClientTool } from '@tanstack/ai';
 import { type UIMessage } from '@tanstack/ai-client';
 import { type ObservableWithLastValue } from '@salsita-ai/common/utils';
-import type { AuthorizedPhoenixApiClient } from '#package/client';
-import type { DomainContextSelectors, DomainServicesSelectors, InitializableSystem, SpecSelectors } from '#package/domain';
+import { type AssistantMessageAnnotation } from '@salsita-ai/contracts/assistant';
+import { type AuthorizedPhoenixApiClient } from '#package/client';
+import type { AssistantManifest, DomainContextSelectors, DomainServicesSelectors, InitializableSystem, SessionAssistantSetup } from '#package/domain';
 import type { ConfiguratorExtensionSelectors } from '../../selectors.js';
+import type { AssistantAnnotation, AssistantAnnotationOverlay } from './assistant-annotation.js';
+import type { AssistantImageView } from './assistant-image-view.js';
+import { type AssistantPlayback } from './assistant-speech-player.js';
+import { type AssistantDraftAttachment } from './composer-attachments.js';
 /** A tool of the assistant that the model calls and the browser runs, as the chat client accepts it. */
 export type AssistantClientTool = AnyClientTool;
 /**
  * Constructor context of an `AssistantSystem`: whole selector objects, like its peers take them, so
- * the tools reach the store, the session mode and the `GraphQLSystem` when they are called, and the
- * chat reaches the server of the current session when it sends a request.
+ * the tools reach the store, the session mode, the view mode and the `GraphQLSystem` when they are
+ * called, and the chat reaches the server of the current session when it sends a request.
  */
 export interface AssistantSystemContext {
     readonly configuratorExtensionSelectors: ConfiguratorExtensionSelectors;
     readonly domainContextSelectors: DomainContextSelectors;
     readonly domainServicesSelectors: DomainServicesSelectors;
-    readonly specSelectors: SpecSelectors;
     readonly getConfigurationId: () => string;
     readonly getAuthorizedClient: () => AuthorizedPhoenixApiClient<any, any, any>;
+    /** The assistant the initialized session runs: its tools, its exposed fields and whether chat runs at all. */
+    readonly getAssistantSetup: () => SessionAssistantSetup;
+    /**
+     * Copies the configurator view as the next frame draws it, for the user to annotate; see
+     * {@link AssistantSystem.annotate}. The only browser-bound step of an annotation the system takes,
+     * so a test can provide one that takes none.
+     */
+    readonly freezeView: () => Promise<HTMLCanvasElement>;
+    /**
+     * Draws an image the user annotates onto a canvas the size of the configurator view, contain-fitted
+     * over `background`; see {@link AssistantSystem.annotate}. Rejects when the image cannot be drawn.
+     */
+    readonly rasterizeImage: (image: Blob, background: string) => Promise<HTMLCanvasElement>;
+    /**
+     * Creates the one audio element the replies are read aloud through, the first time one is. The only other
+     * browser-bound step the system takes, so a test can provide an element that plays nothing.
+     */
+    readonly createAudio: () => HTMLAudioElement;
+}
+/** An image the user annotates instead of their view; see {@link AssistantSystem.annotate}. */
+export interface AssistantAnnotationImage {
+    readonly image: Blob;
+    /** A CSS colour behind the image and in the margins around it, such as the theme's background. */
+    readonly background: string;
 }
+/**
+ * An image attached to the message being written that the user annotates, so the annotation takes its
+ * place; see {@link AssistantSystem.annotate}.
+ */
+export interface AssistantAnnotationAttachment {
+    /** The id of the attachment in `composerAttachments$`. */
+    readonly attachmentId: string;
+    /** A CSS colour behind the image and in the margins around it, such as the theme's background. */
+    readonly background: string;
+}
+/** What the user attaches to a message: an image, or an annotation of their configurator view or of an image. */
+export type AssistantAttachment = Blob | AssistantAnnotation;
 /**
  * The conversation with the assistant as the panel paints it. One value is published per change, so
  * it changes with every chunk of a streaming reply.
@@ -35,10 +75,81 @@ export interface AssistantChatSnapshot {
      * the server was never reached.
      */
     readonly failedStatus: number | undefined;
+    /**
+     * The seconds the server asked to wait before sending again, from the `Retry-After` header of the
+     * latest chat request it answered with a failure: a message refused for the assistant's hourly
+     * limits. Nothing when that request carried none.
+     */
+    readonly retryAfterSeconds: number | undefined;
     /** False when the server said this session may read the thread but not add to it. */
     readonly isContinuable: boolean;
+    /**
+     * True when the server said a lock counted from the assistant's bans refuses the user here: on the thread read, or
+     * on a message it refused for it. The thread stays readable, but the server refuses every new message until the
+     * lock ends.
+     */
+    readonly isLocked: boolean;
+    /**
+     * True when that lock is one of the user's network address, which other users there share, and not one of the user:
+     * it ends on its own, sooner than a ban.
+     */
+    readonly isNetworkLocked: boolean;
     /** False until the first request of the chat, the thread read, was answered or failed. */
     readonly isLoaded: boolean;
+    /**
+     * True when the server said, on the thread read, that this session may send voice messages: the
+     * deployment's `assistant.ai.voice`, as the project's setup handler left it for the session.
+     */
+    readonly isVoiceEnabled: boolean;
+    /**
+     * True while a thread this session may not continue is being continued in a new one, which the
+     * user's next message starts; see {@link AssistantSystem.resume}.
+     */
+    readonly isResuming: boolean;
+    /**
+     * The closed threads the painted thread continues, oldest first: the ones that were resumed while
+     * the chat was open, and the ones loaded with {@link AssistantSystem.loadEarlierThread}.
+     */
+    readonly earlierThreads: ReadonlyArray<AssistantEarlierThreadSnapshot>;
+    /**
+     * The closed thread that {@link AssistantSystem.loadEarlierThread} loads next; `null` when the
+     * painted threads start the conversation. Usually the thread before the oldest painted one, but
+     * the oldest painted one itself when it was closed while only its newest messages were painted:
+     * loading it then paints it whole.
+     */
+    readonly earlierThreadId: string | null;
+    /** True while that thread is being loaded. */
+    readonly isLoadingEarlierThread: boolean;
+    /** True when the thread of the chat has older messages than the painted ones on the server; see {@link AssistantSystem.loadOlderMessages}. */
+    readonly hasOlderMessages: boolean;
+    /** True while those older messages are being loaded. */
+    readonly isLoadingOlderMessages: boolean;
+}
+/** What else the caller of {@link AssistantSystem.send} learns about the message. */
+export interface AssistantSendOptions {
+    /**
+     * Called once the message is handed to the chat, which paints it: after its images were uploaded,
+     * and not at all when it is not sent because the thread could not be resumed or an upload failed.
+     * The composer keeps the message until then, so a message that is not sent can be sent again.
+     */
+    readonly onSent?: () => void;
+    /**
+     * Called when a message that was handed to the chat is gone from it after all: the server refused
+     * it because the thread could not continue, the thread was resumed without it, and its images
+     * could not be uploaded into the new thread. The caller gets it back, so it can be sent again.
+     */
+    readonly onUnsent?: () => void;
+    /**
+     * A recording the user spoke, sent as the message in place of text: the server transcribes it and
+     * the chat shows the transcript once it arrives. The text is empty then. A voice message the server
+     * refuses is taken out of the chat and given back through `onUnsent`, unless nothing was heard in it.
+     */
+    readonly audio?: Blob;
+}
+/** A closed thread of the conversation as the panel paints it above the thread that continues it. */
+export interface AssistantEarlierThreadSnapshot {
+    readonly threadId: string;
+    readonly messages: ReadonlyArray<UIMessage>;
 }
 /**
  * The built-in assistant panel as a layout and the toolbar button see it: whether it is open, and
@@ -52,9 +163,9 @@ export interface AssistantPanelSnapshot {
     readonly isInputFocused: boolean;
 }
 /**
- * Owns the conversational assistant of a configurator: its client tools (`getBrowserDetails`,
- * `query` and `apply`), the chat with the server, the state of the built-in panel, and the record
- * of what changed in the configuration since the agent last acted, which every user message carries.
+ * Owns the conversational assistant of a configurator: its client tools (`query` and `apply`), the
+ * chat with the server, the state of the built-in panel, and the record of what changed in the
+ * configuration since the agent last acted, which every user message carries.
  *
  * The server owns the thread, so the chat keeps nothing in browser storage: its client hydrates
  * from the server and rejoins a reply that is still streaming.
@@ -68,6 +179,13 @@ export declare class AssistantSystem implements InitializableSystem {
     private readonly _tools;
     private readonly _chat;
     private readonly _panel;
+    private readonly _annotationOverlay;
+    private readonly _imageView;
+    private readonly _composerAttachments;
+    /** Releases the layout's inert hold, taken while there is an annotation overlay; nothing otherwise. */
+    private _releaseInert;
+    /** Identity of the `annotate()` call that runs; a new configuration load forgets it, which cancels it. */
+    private _annotationRun;
     /** Identity of the current `activate()` call; nothing while the chat is not active. */
     private _activation;
     /** The chat of the current activation; nothing while no chat is active. */
@@ -77,14 +195,93 @@ export declare class AssistantSystem implements InitializableSystem {
      * background. Each leaves the set when its reply ends or when a new chat takes its thread over.
      */
     private readonly _finishing;
+    /** The resume that is running, so two sends that find the thread locked share one. */
+    private _resuming;
+    /**
+     * The message the server refused on a chat because its thread cannot be continued, by where it
+     * starts in the chat's messages: it is painted, but it is not part of the thread, so a resume
+     * leaves it out of the closed thread it paints above the new one.
+     */
+    private _refused;
+    /**
+     * The chat a message is being sent on for the first time. When the server refuses that message
+     * because the thread cannot be continued, the refusal is not a failure of the chat: the thread is
+     * resumed and the message is sent again; see `send`.
+     */
+    private _firstAttempt;
+    /**
+     * Identity of the chain of threads that is painted. A resumed chat keeps it, because its new
+     * thread continues the same chain; any other new chat starts another one.
+     */
+    private _chain;
     private readonly _recorder;
     /** The recorder's subscription to the committed emissions of the store; made by the first `initialize()`. */
     private _recording;
+    /**
+     * The tool call of every `apply` by the state it committed or held, for as long as the state
+     * lives: an undone state that the undo history dropped is forgotten with it.
+     *
+     * Accepted: a commit the project's `registerIsUndoable` keeps out of the history never becomes a
+     * step, so the row of its `apply` never offers Undo.
+     */
+    private readonly _applyToolCallIds;
+    private readonly _undoTipToolCallId;
+    /**
+     * Object URLs of the screenshots read so far, by image id. An uploaded image never changes, so
+     * each is read once and kept while the configuration stays loaded.
+     */
+    private _screenshotUrls;
+    private readonly _player;
+    /**
+     * The chat whose reply answers a voice message, which reads it aloud once it has ended; see `send`. Nothing when
+     * the latest message was typed.
+     */
+    private _voiceReplyChat;
     constructor(_context: AssistantSystemContext);
     /** The conversation. It never completes, and holds the empty chat whenever no chat is active. */
     readonly chat$: ObservableWithLastValue<AssistantChatSnapshot>;
     /** The state of the built-in panel. It never completes. */
     readonly panel$: ObservableWithLastValue<AssistantPanelSnapshot>;
+    /**
+     * What covers the configurator for an annotation, as the configurator page renders it; nothing while no
+     * annotation is made or shown. See {@link AssistantSystem.annotate}. It never completes.
+     */
+    readonly annotationOverlay$: ObservableWithLastValue<AssistantAnnotationOverlay | null>;
+    /**
+     * The image of the chat the configurator page shows larger, beside the panel: a drawing of a reply,
+     * a project image of a reply or an image of a sent message; nothing while none is open. See
+     * {@link AssistantSystem.viewImage}. It never completes.
+     */
+    readonly imageView$: ObservableWithLastValue<AssistantImageView | null>;
+    /**
+     * The images and annotations attached to the message being written, in their order, as the composer
+     * shows and sends them. They outlive the composer: an annotation finished after it went away is
+     * still attached when it comes back. A new configuration load takes them off. It never completes.
+     */
+    readonly composerAttachments$: ObservableWithLastValue<ReadonlyArray<AssistantDraftAttachment>>;
+    /**
+     * Whether one more image or annotation fits in the message being written, up to 4 together, counting
+     * the images being prepared and the annotation being made. It never completes.
+     */
+    readonly hasFreeAttachmentSlot$: ObservableWithLastValue<boolean>;
+    /**
+     * The `apply` tool call whose committed state is the step of the undo history the configuration is
+     * at; nothing when that step is not a change of the assistant. It never completes.
+     */
+    readonly undoTipToolCallId$: ObservableWithLastValue<string | undefined>;
+    /** The reply being read aloud, and what its reading is doing. It never completes. */
+    readonly playback$: ObservableWithLastValue<AssistantPlayback>;
+    /**
+     * True when the user muted the replies that read themselves aloud after a voice message; the browser remembers it.
+     * The play buttons work either way. It never completes.
+     */
+    readonly muted$: ObservableWithLastValue<boolean>;
+    /**
+     * The manifest the initialized session runs, narrowed by the project's setup handler: what the
+     * agent can read, so the panel reads recipes and options the way the agent does. `null` while the
+     * assistant is off for the session or its setup failed.
+     */
+    getSessionManifest: () => AssistantManifest | null;
     /**
      * A new configuration load: the chat of the previous one is deactivated and both observables go
      * back to their empty values. An activation that is still held outlives it and gets the chat of
@@ -118,7 +315,7 @@ export declare class AssistantSystem implements InitializableSystem {
      * not interrupted, so an `apply` that has started may still commit. The background reply keeps
      * the configuration and the authorized client it started with.
      *
-     * The background reply ends early when a new chat starts for the same configuration, which is
+     * The background reply is disconnected when a new chat starts for the same configuration, which is
      * what opening the configuration again does: see `_takeOverThread`.
      */
     activate: () => (() => void);
@@ -126,8 +323,152 @@ export declare class AssistantSystem implements InitializableSystem {
      * Sends a user message with the record of what changed in the configuration since the agent last
      * acted, which the server puts in front of the message for the model. The record is taken only
      * when there is a chat to send it on, because taking it moves its baseline.
+     *
+     * The images the user attached go with the message: they are uploaded into the chat's thread
+     * first, and the message references them, one image part after its text each. The text part is
+     * left out of a message with images only. An annotation is attached like an image: its image is
+     * uploaded and referenced in its place among the images, and its pins go into the metadata of the
+     * message, by the id of its image, for the server to tell the model. Each attempt to send uploads them again, into the
+     * thread it sends on. An upload that fails leaves the message unsent, with the failure as the
+     * error of the chat; `onSent` is not called then.
+     *
+     * A thread can end in a tool call that nothing will answer: its reply parked while no page was
+     * open, or the page that ran the tool was left before the result was sent. The client refuses to
+     * send while it waits on such a call, so the wait is given up first, and the server records for
+     * the model that the call got no result.
+     *
+     * A thread this session may not continue is resumed first, in a new thread; see `resume`. When
+     * that fails the message is not sent.
+     *
+     * The chat can be out of date about that: the thread reached its size limit with the reply before
+     * this message, or the assistant's setup changed while the configurator stayed open. The server
+     * then refuses the message, or the upload of one of its images, and says that the thread cannot
+     * be continued. The thread is resumed, and the message is then sent again on the new thread, its
+     * images uploaded again, with the change record it was first sent with. This happens once: a
+     * refusal of the second attempt is the failure it is. When resuming fails, a refused message is
+     * left as any message that failed to send: painted, with the failure as the error of the chat; a
+     * refused upload leaves the message unsent. When the images of a refused message cannot be
+     * uploaded into the new thread, the message is given back through `onUnsent`.
+     *
+     * A chat that ends while the images upload, because another configuration was loaded or the
+     * thread is read again, sends nothing.
+     *
+     * Accepted: when the server refuses the compaction itself with a conflict, because another tab
+     * compacted the thread or a run is still running, the thread is read again (see `resume`) and
+     * the refused message is gone without a notice. It takes two tabs or a run racing the send, and
+     * the user writes the message again, as after any reload of a thread that failed.
+     *
+     * Accepted: images uploaded by an attempt that is not sent stay stored in that thread, which no
+     * message references.
+     *
+     * It never rejects: a message the client could not send is reported as the error of the chat.
+     */
+    send: (content: string, attachments?: ReadonlyArray<AssistantAttachment>, { onSent, onUnsent, audio }?: AssistantSendOptions) => Promise<void>;
+    /**
+     * Lets the user annotate their current view of the configurator, or `source.image`, and attaches what
+     * they made to the message being written (see `composerAttachments$`). Resolves with the id of the
+     * attachment, or with `null` when they cancel, the view cannot be frozen or the image drawn, or another
+     * configuration is loaded meanwhile. While the message is full, or another annotation is made or
+     * shown, nothing happens: it resolves with `null` at once. The annotation holds a slot of the
+     * message from the start, so nothing attached meanwhile can push it over the limit.
+     *
+     * With `source.attachmentId`, the user annotates that image of the message instead, and the annotation
+     * takes its place, at its position, so it needs no free slot. Nothing happens, and it resolves with
+     * `null`, when that attachment is not attached or is already an annotation; also when it is taken off
+     * while the user annotates it.
+     *
+     * The configurator is inert from the start, and `annotationOverlay$` says what covers it. For the
+     * view, once the camera is still, the view glides from the canvas padding to none, so the product is
+     * centred on the whole configurator area, and the next frame is frozen for the editor; when the editor
+     * finishes, the view glides back to the padding. An image is drawn contain-fitted over its background
+     * on a canvas the size of the configurator view, and the camera stays as it is.
+     */
+    annotate: (source?: AssistantAnnotationImage | AssistantAnnotationAttachment) => Promise<string | null>;
+    /**
+     * Shows an annotation of a sent message over the configurator, read-only, until its overlay is
+     * closed. Nothing happens while another annotation is made or shown.
+     */
+    viewAnnotation: (annotation: AssistantMessageAnnotation) => void;
+    /**
+     * Shows an image of the chat larger, beside the panel, in place of the one that is open, until
+     * {@link AssistantSystem.closeImageView} or a new configuration load. Unlike an annotation it leaves
+     * the configurator usable: the user can go on writing while it is open.
+     */
+    viewImage: (view: AssistantImageView) => void;
+    /** Closes the image that is open; nothing happens when none is. */
+    closeImageView: () => void;
+    /**
+     * Attaches an image, prepared for the upload, to the message being written, after the others, and
+     * returns its id; `null` when the message is full. Free a slot reserved for it first.
+     */
+    attach: (blob: Blob) => string | null;
+    /**
+     * Holds one attachment slot of the message being written, for an image that is being prepared, until
+     * the returned function frees it; `null` when the message is full. The function returns whether the
+     * image is still to be attached: `false` once a configuration load took the attachments off meanwhile.
+     */
+    reserveAttachmentSlot: () => (() => boolean) | null;
+    /**
+     * Puts attachments of a message the chat gave back in front of the others, as many as fit, the first
+     * ones first, and returns whether all of them fit.
+     */
+    restoreAttachments: (attachments: ReadonlyArray<AssistantDraftAttachment>) => boolean;
+    /** Takes an attachment off the message being written; nothing happens when it is not attached. */
+    removeAttachment: (id: string) => void;
+    /** Takes every attachment off the message being written. */
+    clearAttachments: () => void;
+    /**
+     * Makes the chat ready for the next message. A thread the server no longer lets this session
+     * continue, because the assistant's setup changed since it was started or the thread reached
+     * its size limit, is compacted there: the
+     * server closes it and opens a thread that starts from its summary. The messages painted so far
+     * move to `earlierThreads`, and a new client takes the new thread over.
+     *
+     * Resolves to whether a message can be sent now: at once for a thread that can continue, and
+     * `false` when there is no chat or the compaction failed, which is reported as the error of the
+     * chat. It never rejects. Calling it again runs the compaction again. A compaction the server
+     * refuses with a conflict is not a failure: the thread is read again and `false` is returned.
+     */
+    resume: () => Promise<boolean>;
+    /**
+     * An object URL of a screenshot of the conversation, by the id its tool result references it by.
+     * It is read from the server through the client of the current chat, once per image; a read that
+     * failed is forgotten, so it can be asked for again. Rejects without a chat or when the read fails.
      */
-    send: (content: string) => Promise<void>;
+    loadScreenshotUrl: (imageId: string) => Promise<string>;
+    /**
+     * Reads a reply of the conversation aloud, by the id of its message, stopping the one being read; a reply being
+     * read is stopped instead. Its speech is read from the server through the client of the current chat, once per
+     * reply, and nothing happens without a chat. A reply the server cannot read aloud is logged, and stays idle.
+     */
+    playReply: (messageId: string) => void;
+    /** Stops the reply being read aloud. */
+    stopPlayback: () => void;
+    /**
+     * Mutes or unmutes the replies that read themselves aloud after a voice message; muting stops the one being read.
+     * The browser remembers the choice.
+     */
+    setMuted: (muted: boolean) => void;
+    /**
+     * Lets the page read the reply to a voice message aloud on its own once it has ended. Call it while the user taps,
+     * such as on the tap that records or sends the message: Safari plays audio that no tap started only on an element
+     * that played during one before. Calls after the first do nothing.
+     */
+    unlockPlayback: () => void;
+    /**
+     * Loads the closed thread named by `earlierThreadId` of the chat, whole, and paints it above the
+     * painted ones; when it is the oldest painted thread itself, closed while only its newest messages
+     * were painted, it replaces that partial copy. Does nothing when there is none, one is being
+     * loaded, or the chat's own thread still has older messages: the earlier threads come above them.
+     * A load that fails is logged and can be asked for again.
+     */
+    loadEarlierThread: () => Promise<void>;
+    /**
+     * Loads the page of messages before the oldest painted one of the chat's thread and paints it
+     * above. Does nothing when the thread has no older messages or a page is being loaded. A load that
+     * fails is logged and can be asked for again; one that ends after its chat did is dropped.
+     */
+    loadOlderMessages: () => Promise<void>;
     /**
      * Stops the reply on both sides. Closing the stream alone keeps the server run going, because a
      * closed connection is also how a reload looks and a reload must be able to rejoin the run.
@@ -139,8 +480,75 @@ export declare class AssistantSystem implements InitializableSystem {
      * server is therefore gone from the thread and has to be written again.
      */
     reloadThread: () => void;
+    /**
+     * Undoes the changes of the given `apply` tool calls, one undo step at a time, while the step the
+     * configuration is at is one of them. A change the user made in between stops it, so it is never
+     * taken back; the non-undoable commits made on top of an undone change go with it, as with the
+     * toolbar's undo.
+     *
+     * Accepted: like the toolbar's undo, it checks neither the session mode nor the view mode; only
+     * `apply` follows the view mode and refuses a finalize run.
+     */
+    undoToolCalls: (toolCallIds: ReadonlySet<string>) => void;
     setOpen: (open: boolean) => void;
     setInputFocused: (isInputFocused: boolean) => void;
+    /** Every change of the annotation overlay goes through here: the configurator is inert while there is one. */
+    private _setAnnotationOverlay;
+    /** The camera path of `annotate`: still camera, glide, freeze, edit, glide back. */
+    private _annotateView;
+    /**
+     * Runs one annotation as the current `annotate()` call and resolves with what the user made, or with
+     * `null` when a configuration loaded meanwhile cancelled it: it belongs to the message of another one.
+     */
+    private _runAnnotation;
+    /** The attachment path of `annotate`: annotate an image of the message, which the annotation replaces. */
+    private _annotateAttachment;
+    /** The image path of `annotate`: draw the image, edit. The camera stays as it is. */
+    private _annotateImage;
+    /** Opens the editor on `view` and resolves with what the user made, or `null` when they cancel. */
+    private _edit;
+    private _logAnnotationFailure;
+    /** A new configuration load cancels an annotation that is made, and closes one that is shown. */
+    private _closeAnnotationOverlay;
+    /**
+     * Where the message starts that is being sent on the chat for the first time and that the server
+     * refused because its thread cannot be continued, so it never reached the thread; `-1` when there
+     * is no such message. A refused continuation of a reply is not one: its message is stored.
+     */
+    private _findRefusedMessageIndex;
+    /**
+     * The content of a user message: its text, when it has any, or its recording, and a reference to
+     * each of its images.
+     */
+    private _messageContent;
+    /**
+     * Uploads the images of a message into the chat's thread, in their order, and resolves with their
+     * ids. Resolves with `'refused'` when the server refused an upload because the thread cannot be
+     * continued, and marks the chat so; then with `null` for any other failure, which is reported as
+     * the error of the chat. It never rejects.
+     */
+    private _uploadImages;
+    /**
+     * Uploads the images of a message into the thread that was resumed for it. A message the chat
+     * refused shows nowhere meanwhile, so the chat stays resuming until the upload is over: nothing
+     * else is sent ahead of it.
+     */
+    private _uploadSuccessorImages;
+    private _reportUploadFailure;
+    private _sendOn;
+    private _compact;
+    /**
+     * The `apply` call whose committed state is the step the configuration is at. Nothing at the bottom
+     * of the history, where nothing can be undone: that step was not made by a change it could take back.
+     */
+    private _findTipToolCallId;
+    private _refreshUndoTip;
+    private _loadSpeech;
+    /**
+     * Reads aloud the reply to a voice message, its newest message with text, unless the user muted it or that message
+     * cannot be read aloud; a browser that refuses leaves its play button to start it.
+     */
+    private _readVoiceReply;
     private _committedState;
     private _patchPanel;
     /**
@@ -152,9 +560,14 @@ export declare class AssistantSystem implements InitializableSystem {
      * and the new one for real, and both would send the result back, of which the server accepts one.
      * The model could be told that nothing was run about a change that was made.
      *
-     * Stopping a client only closes its stream in the browser. Nothing is cancelled on the server, so
-     * the run goes on there and the new chat rejoins it. A chat that finishes a reply for another
-     * configuration is on another thread and is left alone.
+     * Stopping a client closes its stream in the browser and cancels nothing on the server: a run that
+     * is still producing goes on there and the new chat rejoins it. The stopped client also drops the
+     * result of a tool call that is still running in it, and any continuation it has not sent yet. Such
+     * a call still finishes, so an `apply` may commit, but the server never gets its result: the run is
+     * parked on the call with nothing running for the new chat to rejoin, and the thread waits on it
+     * until the user's next message gives the wait up; see `send`.
+     *
+     * A chat that finishes a reply for another configuration is on another thread and is left alone.
      */
     private _takeOverThread;
     private _startClient;
@@ -163,6 +576,8 @@ export declare class AssistantSystem implements InitializableSystem {
      * it is: whether it finishes its reply or is cut off is the caller's decision.
      */
     private _retireChat;
+    /** Revokes the object URLs of the screenshots read for the configuration that was loaded, also those still being read. */
+    private _releaseScreenshotUrls;
     /** Ends the current chat without cutting its reply off. */
     private _endChat;
 }