@salsita-ai/sdk 0.50.0-develop.ae9a830ef0 → 0.50.0-develop.afbc5157e1

package/dist/configurator/application-runtime/extensions/systems/assistant-system/assistant-speech-player.d.ts

@@ -0,0 +1,69 @@
+import type { ObservableWithLastValue } from '@salsita-ai/common/utils';
+/**
+ * What the reading aloud of a reply is doing:
+ * - `idle`: no reply is read
+ * - `loading`: the reply's speech is on its way from the server
+ * - `playing`: the reply is read aloud
+ * - `blocked`: the browser refused to start reading the reply on its own; the reply's play button starts it
+ */
+export type AssistantPlaybackStatus = 'idle' | 'loading' | 'playing' | 'blocked';
+/** The reply read aloud, by the id of its message, and what its reading is doing; no reply while `idle`. */
+export interface AssistantPlayback {
+    readonly messageId: string | null;
+    readonly status: AssistantPlaybackStatus;
+}
+/** Starts the reading of a reply, by the id of its message, and how. */
+export interface AssistantSpeechPlayRequest {
+    readonly messageId: string;
+    /** Reads the reply's speech from the server. */
+    readonly loadSpeech: () => Promise<Blob>;
+    /**
+     * True when the panel reads the reply on its own, after a voice message: a browser that refuses to start it leaves
+     * the reply `blocked` for the user to start, where a refused tap is a failure.
+     */
+    readonly isAutoplay: boolean;
+}
+interface AssistantSpeechPlayerContext {
+    /** The audio element every reply is read through; see `AssistantSystemContext.createAudio`. */
+    readonly createAudio: () => HTMLAudioElement;
+    readonly logError: (message: string, error: Error) => void;
+}
+/**
+ * Reads the assistant's replies aloud, one at a time, through one audio element: starting a reply stops the one being
+ * read. The speech of a reply is read from the server once and kept as an object URL while the configuration stays
+ * loaded. Whether the user muted the replies that read themselves aloud is kept in the browser.
+ */
+export declare class AssistantSpeechPlayer {
+    private readonly _context;
+    private readonly _playback;
+    private readonly _muted;
+    private _audio;
+    private _isUnlocked;
+    /** Identity of the latest start; a reading whose speech arrives after another one started is dropped. */
+    private _attempt;
+    /** Object URLs of the speech read so far, by message id. */
+    private _urls;
+    constructor(_context: AssistantSpeechPlayerContext);
+    readonly playback$: ObservableWithLastValue<AssistantPlayback>;
+    readonly muted$: ObservableWithLastValue<boolean>;
+    /**
+     * Plays silence on the audio element, once, while the user taps: Safari lets a page start audio on its own only on
+     * an element that played during a tap before.
+     */
+    unlock: () => void;
+    /** Reads the reply aloud, stopping the one being read; the reply being read or loaded is stopped instead. */
+    toggle: (request: AssistantSpeechPlayRequest) => Promise<void>;
+    /** Reads the reply aloud, stopping the one being read. */
+    play: ({ messageId, loadSpeech, isAutoplay }: AssistantSpeechPlayRequest) => Promise<void>;
+    /** Stops the reply being read, and forgets a reply waiting to be started. */
+    stop: () => void;
+    /** Mutes the replies that read themselves aloud, and stops the one being read; the play buttons keep working. */
+    setMuted: (muted: boolean) => void;
+    /** Stops reading, and revokes the speech read for the configuration that was loaded. */
+    reset: () => void;
+    private _finish;
+    private _getAudio;
+    /** The object URL of a reply's speech, read once; a read that failed is forgotten, so it can be asked for again. */
+    private _loadUrl;
+}
+export {};

package/dist/configurator/application-runtime/extensions/systems/assistant-system/assistant-speech-player.js

@@ -0,0 +1 @@
+import{BehaviorSubject as t}from"rxjs";let e=Object.freeze({messageId:null,status:"idle"}),s="phx-assistant-muted";export class AssistantSpeechPlayer{_context;_playback=new t(e);_muted=new t((()=>{try{return"true"===localStorage.getItem(s)}catch{return!1}})());_audio=null;_isUnlocked=!1;_attempt=null;_urls=new Map;constructor(t){this._context=t}playback$=this._playback;muted$=this._muted;unlock=()=>{if(this._isUnlocked)return;this._isUnlocked=!0,this.stop();let t=this._getAudio();t.src="data:audio/wav;base64,UklGRiQAAABXQVZFZm10IBAAAAABAAEARKwAAIhYAQACABAAZGF0YQAAAAA=",t.play().catch(()=>void 0)};toggle=async t=>{let{messageId:e,status:s}=this._playback.value;e===t.messageId&&("loading"===s||"playing"===s)?this.stop():await this.play(t)};play=async({messageId:t,loadSpeech:e,isAutoplay:s})=>{let a={},i=this._getAudio();this._attempt=a,i.pause(),this._playback.next({messageId:t,status:"loading"});try{let s=await this._loadUrl(t,e);if(this._attempt!==a)return;i.src=s,await i.play(),this._attempt===a&&this._playback.next({messageId:t,status:"playing"})}catch(e){if(this._attempt!==a)return;if(s&&e instanceof DOMException&&"NotAllowedError"===e.name)return void this._playback.next({messageId:t,status:"blocked"});this._context.logError("The assistant reply could not be read aloud",e instanceof Error?e:Error(String(e))),this._finish()}};stop=()=>{this._audio?.pause(),this._finish()};setMuted=t=>{t&&this.stop(),t!==this._muted.value&&this._muted.next(t);try{localStorage.setItem(s,String(t))}catch{}};reset=()=>{for(let t of(this.stop(),this._urls.values()))t.then(URL.revokeObjectURL,()=>void 0);this._urls=new Map};_finish=()=>{this._attempt=null,this._playback.value!==e&&this._playback.next(e)};_getAudio=()=>{if(this._audio)return this._audio;let t=this._context.createAudio();return t.addEventListener("ended",()=>{"playing"===this._playback.value.status&&this._finish()}),this._audio=t,t};_loadUrl=(t,e)=>{let s=this._urls.get(t);if(s)return s;let a=this._urls,i=e().then(t=>URL.createObjectURL(t));return a.set(t,i),i.catch(()=>{a.get(t)===i&&a.delete(t)}),i}}

package/dist/configurator/application-runtime/extensions/systems/assistant-system/assistant-system.d.ts

@@ -1,26 +1,64 @@
 import type { AnyClientTool } from '@tanstack/ai';
 import { type UIMessage } from '@tanstack/ai-client';
 import { type ObservableWithLastValue } from '@salsita-ai/common/utils';
+import { type AssistantMessageAnnotation } from '@salsita-ai/contracts/assistant';
 import { type AuthorizedPhoenixApiClient } from '#package/client';
-import type { DomainContextSelectors, DomainServicesSelectors, InitializableSystem, SessionAssistantSetup, SpecSelectors } from '#package/domain';
+import type { AssistantManifest, DomainContextSelectors, DomainServicesSelectors, InitializableSystem, SessionAssistantSetup } from '#package/domain';
 import type { ConfiguratorExtensionSelectors } from '../../selectors.js';
+import type { AssistantAnnotation, AssistantAnnotationOverlay } from './assistant-annotation.js';
+import type { AssistantImageView } from './assistant-image-view.js';
+import { type AssistantPlayback } from './assistant-speech-player.js';
+import { type AssistantDraftAttachment } from './composer-attachments.js';
 /** A tool of the assistant that the model calls and the browser runs, as the chat client accepts it. */
 export type AssistantClientTool = AnyClientTool;
 /**
  * Constructor context of an `AssistantSystem`: whole selector objects, like its peers take them, so
- * the tools reach the store, the session mode and the `GraphQLSystem` when they are called, and the
- * chat reaches the server of the current session when it sends a request.
+ * the tools reach the store, the session mode, the view mode and the `GraphQLSystem` when they are
+ * called, and the chat reaches the server of the current session when it sends a request.
  */
 export interface AssistantSystemContext {
     readonly configuratorExtensionSelectors: ConfiguratorExtensionSelectors;
     readonly domainContextSelectors: DomainContextSelectors;
     readonly domainServicesSelectors: DomainServicesSelectors;
-    readonly specSelectors: SpecSelectors;
     readonly getConfigurationId: () => string;
     readonly getAuthorizedClient: () => AuthorizedPhoenixApiClient<any, any, any>;
     /** The assistant the initialized session runs: its tools, its exposed fields and whether chat runs at all. */
     readonly getAssistantSetup: () => SessionAssistantSetup;
+    /**
+     * Copies the configurator view as the next frame draws it, for the user to annotate; see
+     * {@link AssistantSystem.annotate}. The only browser-bound step of an annotation the system takes,
+     * so a test can provide one that takes none.
+     */
+    readonly freezeView: () => Promise<HTMLCanvasElement>;
+    /**
+     * Draws an image the user annotates onto a canvas the size of the configurator view, contain-fitted
+     * over `background`; see {@link AssistantSystem.annotate}. Rejects when the image cannot be drawn.
+     */
+    readonly rasterizeImage: (image: Blob, background: string) => Promise<HTMLCanvasElement>;
+    /**
+     * Creates the one audio element the replies are read aloud through, the first time one is. The only other
+     * browser-bound step the system takes, so a test can provide an element that plays nothing.
+     */
+    readonly createAudio: () => HTMLAudioElement;
 }
+/** An image the user annotates instead of their view; see {@link AssistantSystem.annotate}. */
+export interface AssistantAnnotationImage {
+    readonly image: Blob;
+    /** A CSS colour behind the image and in the margins around it, such as the theme's background. */
+    readonly background: string;
+}
+/**
+ * An image attached to the message being written that the user annotates, so the annotation takes its
+ * place; see {@link AssistantSystem.annotate}.
+ */
+export interface AssistantAnnotationAttachment {
+    /** The id of the attachment in `composerAttachments$`. */
+    readonly attachmentId: string;
+    /** A CSS colour behind the image and in the margins around it, such as the theme's background. */
+    readonly background: string;
+}
+/** What the user attaches to a message: an image, or an annotation of their configurator view or of an image. */
+export type AssistantAttachment = Blob | AssistantAnnotation;
 /**
  * The conversation with the assistant as the panel paints it. One value is published per change, so
  * it changes with every chunk of a streaming reply.
@@ -37,10 +75,32 @@ export interface AssistantChatSnapshot {
      * the server was never reached.
      */
     readonly failedStatus: number | undefined;
+    /**
+     * The seconds the server asked to wait before sending again, from the `Retry-After` header of the
+     * latest chat request it answered with a failure: a message refused for the assistant's hourly
+     * limits. Nothing when that request carried none.
+     */
+    readonly retryAfterSeconds: number | undefined;
     /** False when the server said this session may read the thread but not add to it. */
     readonly isContinuable: boolean;
+    /**
+     * True when the server said a lock counted from the assistant's bans refuses the user here: on the thread read, or
+     * on a message it refused for it. The thread stays readable, but the server refuses every new message until the
+     * lock ends.
+     */
+    readonly isLocked: boolean;
+    /**
+     * True when that lock is one of the user's network address, which other users there share, and not one of the user:
+     * it ends on its own, sooner than a ban.
+     */
+    readonly isNetworkLocked: boolean;
     /** False until the first request of the chat, the thread read, was answered or failed. */
     readonly isLoaded: boolean;
+    /**
+     * True when the server said, on the thread read, that this session may send voice messages: the
+     * deployment's `assistant.ai.voice`, as the project's setup handler left it for the session.
+     */
+    readonly isVoiceEnabled: boolean;
     /**
      * True while a thread this session may not continue is being continued in a new one, which the
      * user's next message starts; see {@link AssistantSystem.resume}.
@@ -51,10 +111,40 @@ export interface AssistantChatSnapshot {
      * the chat was open, and the ones loaded with {@link AssistantSystem.loadEarlierThread}.
      */
     readonly earlierThreads: ReadonlyArray<AssistantEarlierThreadSnapshot>;
-    /** True when the server holds a thread older than the oldest painted one. */
-    readonly hasEarlierThread: boolean;
-    /** True while that older thread is being loaded. */
+    /**
+     * The closed thread that {@link AssistantSystem.loadEarlierThread} loads next; `null` when the
+     * painted threads start the conversation. Usually the thread before the oldest painted one, but
+     * the oldest painted one itself when it was closed while only its newest messages were painted:
+     * loading it then paints it whole.
+     */
+    readonly earlierThreadId: string | null;
+    /** True while that thread is being loaded. */
     readonly isLoadingEarlierThread: boolean;
+    /** True when the thread of the chat has older messages than the painted ones on the server; see {@link AssistantSystem.loadOlderMessages}. */
+    readonly hasOlderMessages: boolean;
+    /** True while those older messages are being loaded. */
+    readonly isLoadingOlderMessages: boolean;
+}
+/** What else the caller of {@link AssistantSystem.send} learns about the message. */
+export interface AssistantSendOptions {
+    /**
+     * Called once the message is handed to the chat, which paints it: after its images were uploaded,
+     * and not at all when it is not sent because the thread could not be resumed or an upload failed.
+     * The composer keeps the message until then, so a message that is not sent can be sent again.
+     */
+    readonly onSent?: () => void;
+    /**
+     * Called when a message that was handed to the chat is gone from it after all: the server refused
+     * it because the thread could not continue, the thread was resumed without it, and its images
+     * could not be uploaded into the new thread. The caller gets it back, so it can be sent again.
+     */
+    readonly onUnsent?: () => void;
+    /**
+     * A recording the user spoke, sent as the message in place of text: the server transcribes it and
+     * the chat shows the transcript once it arrives. The text is empty then. A voice message the server
+     * refuses is taken out of the chat and given back through `onUnsent`, unless nothing was heard in it.
+     */
+    readonly audio?: Blob;
 }
 /** A closed thread of the conversation as the panel paints it above the thread that continues it. */
 export interface AssistantEarlierThreadSnapshot {
@@ -73,9 +163,9 @@ export interface AssistantPanelSnapshot {
     readonly isInputFocused: boolean;
 }
 /**
- * Owns the conversational assistant of a configurator: its client tools (`getBrowserDetails`,
- * `query` and `apply`), the chat with the server, the state of the built-in panel, and the record
- * of what changed in the configuration since the agent last acted, which every user message carries.
+ * Owns the conversational assistant of a configurator: its client tools (`query` and `apply`), the
+ * chat with the server, the state of the built-in panel, and the record of what changed in the
+ * configuration since the agent last acted, which every user message carries.
  *
  * The server owns the thread, so the chat keeps nothing in browser storage: its client hydrates
  * from the server and rejoins a reply that is still streaming.
@@ -89,6 +179,13 @@ export declare class AssistantSystem implements InitializableSystem {
     private readonly _tools;
     private readonly _chat;
     private readonly _panel;
+    private readonly _annotationOverlay;
+    private readonly _imageView;
+    private readonly _composerAttachments;
+    /** Releases the layout's inert hold, taken while there is an annotation overlay; nothing otherwise. */
+    private _releaseInert;
+    /** Identity of the `annotate()` call that runs; a new configuration load forgets it, which cancels it. */
+    private _annotationRun;
     /** Identity of the current `activate()` call; nothing while the chat is not active. */
     private _activation;
     /** The chat of the current activation; nothing while no chat is active. */
@@ -98,11 +195,6 @@ export declare class AssistantSystem implements InitializableSystem {
      * background. Each leaves the set when its reply ends or when a new chat takes its thread over.
      */
     private readonly _finishing;
-    /**
-     * The thread before the oldest painted one, as the server named it; nothing when the painted
-     * threads start the conversation. `hasEarlierThread` of the chat says whether it is set.
-     */
-    private _olderThreadId;
     /** The resume that is running, so two sends that find the thread locked share one. */
     private _resuming;
     /**
@@ -122,16 +214,74 @@ export declare class AssistantSystem implements InitializableSystem {
      * thread continues the same chain; any other new chat starts another one.
      */
     private _chain;
-    /** The earlier thread that is being loaded, so a second press loads nothing twice. */
-    private _loadingThreadId;
     private readonly _recorder;
     /** The recorder's subscription to the committed emissions of the store; made by the first `initialize()`. */
     private _recording;
+    /**
+     * The tool call of every `apply` by the state it committed or held, for as long as the state
+     * lives: an undone state that the undo history dropped is forgotten with it.
+     *
+     * Accepted: a commit the project's `registerIsUndoable` keeps out of the history never becomes a
+     * step, so the row of its `apply` never offers Undo.
+     */
+    private readonly _applyToolCallIds;
+    private readonly _undoTipToolCallId;
+    /**
+     * Object URLs of the screenshots read so far, by image id. An uploaded image never changes, so
+     * each is read once and kept while the configuration stays loaded.
+     */
+    private _screenshotUrls;
+    private readonly _player;
+    /**
+     * The chat whose reply answers a voice message, which reads it aloud once it has ended; see `send`. Nothing when
+     * the latest message was typed.
+     */
+    private _voiceReplyChat;
     constructor(_context: AssistantSystemContext);
     /** The conversation. It never completes, and holds the empty chat whenever no chat is active. */
     readonly chat$: ObservableWithLastValue<AssistantChatSnapshot>;
     /** The state of the built-in panel. It never completes. */
     readonly panel$: ObservableWithLastValue<AssistantPanelSnapshot>;
+    /**
+     * What covers the configurator for an annotation, as the configurator page renders it; nothing while no
+     * annotation is made or shown. See {@link AssistantSystem.annotate}. It never completes.
+     */
+    readonly annotationOverlay$: ObservableWithLastValue<AssistantAnnotationOverlay | null>;
+    /**
+     * The image of the chat the configurator page shows larger, beside the panel: a drawing of a reply,
+     * a project image of a reply or an image of a sent message; nothing while none is open. See
+     * {@link AssistantSystem.viewImage}. It never completes.
+     */
+    readonly imageView$: ObservableWithLastValue<AssistantImageView | null>;
+    /**
+     * The images and annotations attached to the message being written, in their order, as the composer
+     * shows and sends them. They outlive the composer: an annotation finished after it went away is
+     * still attached when it comes back. A new configuration load takes them off. It never completes.
+     */
+    readonly composerAttachments$: ObservableWithLastValue<ReadonlyArray<AssistantDraftAttachment>>;
+    /**
+     * Whether one more image or annotation fits in the message being written, up to 4 together, counting
+     * the images being prepared and the annotation being made. It never completes.
+     */
+    readonly hasFreeAttachmentSlot$: ObservableWithLastValue<boolean>;
+    /**
+     * The `apply` tool call whose committed state is the step of the undo history the configuration is
+     * at; nothing when that step is not a change of the assistant. It never completes.
+     */
+    readonly undoTipToolCallId$: ObservableWithLastValue<string | undefined>;
+    /** The reply being read aloud, and what its reading is doing. It never completes. */
+    readonly playback$: ObservableWithLastValue<AssistantPlayback>;
+    /**
+     * True when the user muted the replies that read themselves aloud after a voice message; the browser remembers it.
+     * The play buttons work either way. It never completes.
+     */
+    readonly muted$: ObservableWithLastValue<boolean>;
+    /**
+     * The manifest the initialized session runs, narrowed by the project's setup handler: what the
+     * agent can read, so the panel reads recipes and options the way the agent does. `null` while the
+     * assistant is off for the session or its setup failed.
+     */
+    getSessionManifest: () => AssistantManifest | null;
     /**
      * A new configuration load: the chat of the previous one is deactivated and both observables go
      * back to their empty values. An activation that is still held outlives it and gets the chat of
@@ -174,6 +324,14 @@ export declare class AssistantSystem implements InitializableSystem {
      * acted, which the server puts in front of the message for the model. The record is taken only
      * when there is a chat to send it on, because taking it moves its baseline.
      *
+     * The images the user attached go with the message: they are uploaded into the chat's thread
+     * first, and the message references them, one image part after its text each. The text part is
+     * left out of a message with images only. An annotation is attached like an image: its image is
+     * uploaded and referenced in its place among the images, and its pins go into the metadata of the
+     * message, by the id of its image, for the server to tell the model. Each attempt to send uploads them again, into the
+     * thread it sends on. An upload that fails leaves the message unsent, with the failure as the
+     * error of the chat; `onSent` is not called then.
+     *
      * A thread can end in a tool call that nothing will answer: its reply parked while no page was
      * open, or the page that ran the tool was left before the result was sent. The client refuses to
      * send while it waits on such a call, so the wait is given up first, and the server records for
@@ -184,20 +342,81 @@ export declare class AssistantSystem implements InitializableSystem {
      *
      * The chat can be out of date about that: the thread reached its size limit with the reply before
      * this message, or the assistant's setup changed while the configurator stayed open. The server
-     * then refuses the message and says that the thread cannot be continued. The message stays
-     * painted while the thread is resumed, and is then sent again on the new thread, with the change
-     * record it was first sent with. This happens once: a refusal of the second attempt is the
-     * failure it is. When resuming fails, the message is left as any message that failed to send:
-     * painted, with the failure as the error of the chat.
+     * then refuses the message, or the upload of one of its images, and says that the thread cannot
+     * be continued. The thread is resumed, and the message is then sent again on the new thread, its
+     * images uploaded again, with the change record it was first sent with. This happens once: a
+     * refusal of the second attempt is the failure it is. When resuming fails, a refused message is
+     * left as any message that failed to send: painted, with the failure as the error of the chat; a
+     * refused upload leaves the message unsent. When the images of a refused message cannot be
+     * uploaded into the new thread, the message is given back through `onUnsent`.
+     *
+     * A chat that ends while the images upload, because another configuration was loaded or the
+     * thread is read again, sends nothing.
      *
      * Accepted: when the server refuses the compaction itself with a conflict, because another tab
      * compacted the thread or a run is still running, the thread is read again (see `resume`) and
      * the refused message is gone without a notice. It takes two tabs or a run racing the send, and
      * the user writes the message again, as after any reload of a thread that failed.
      *
+     * Accepted: images uploaded by an attempt that is not sent stay stored in that thread, which no
+     * message references.
+     *
      * It never rejects: a message the client could not send is reported as the error of the chat.
      */
-    send: (content: string) => Promise<void>;
+    send: (content: string, attachments?: ReadonlyArray<AssistantAttachment>, { onSent, onUnsent, audio }?: AssistantSendOptions) => Promise<void>;
+    /**
+     * Lets the user annotate their current view of the configurator, or `source.image`, and attaches what
+     * they made to the message being written (see `composerAttachments$`). Resolves with the id of the
+     * attachment, or with `null` when they cancel, the view cannot be frozen or the image drawn, or another
+     * configuration is loaded meanwhile. While the message is full, or another annotation is made or
+     * shown, nothing happens: it resolves with `null` at once. The annotation holds a slot of the
+     * message from the start, so nothing attached meanwhile can push it over the limit.
+     *
+     * With `source.attachmentId`, the user annotates that image of the message instead, and the annotation
+     * takes its place, at its position, so it needs no free slot. Nothing happens, and it resolves with
+     * `null`, when that attachment is not attached or is already an annotation; also when it is taken off
+     * while the user annotates it.
+     *
+     * The configurator is inert from the start, and `annotationOverlay$` says what covers it. For the
+     * view, once the camera is still, the view glides from the canvas padding to none, so the product is
+     * centred on the whole configurator area, and the next frame is frozen for the editor; when the editor
+     * finishes, the view glides back to the padding. An image is drawn contain-fitted over its background
+     * on a canvas the size of the configurator view, and the camera stays as it is.
+     */
+    annotate: (source?: AssistantAnnotationImage | AssistantAnnotationAttachment) => Promise<string | null>;
+    /**
+     * Shows an annotation of a sent message over the configurator, read-only, until its overlay is
+     * closed. Nothing happens while another annotation is made or shown.
+     */
+    viewAnnotation: (annotation: AssistantMessageAnnotation) => void;
+    /**
+     * Shows an image of the chat larger, beside the panel, in place of the one that is open, until
+     * {@link AssistantSystem.closeImageView} or a new configuration load. Unlike an annotation it leaves
+     * the configurator usable: the user can go on writing while it is open.
+     */
+    viewImage: (view: AssistantImageView) => void;
+    /** Closes the image that is open; nothing happens when none is. */
+    closeImageView: () => void;
+    /**
+     * Attaches an image, prepared for the upload, to the message being written, after the others, and
+     * returns its id; `null` when the message is full. Free a slot reserved for it first.
+     */
+    attach: (blob: Blob) => string | null;
+    /**
+     * Holds one attachment slot of the message being written, for an image that is being prepared, until
+     * the returned function frees it; `null` when the message is full. The function returns whether the
+     * image is still to be attached: `false` once a configuration load took the attachments off meanwhile.
+     */
+    reserveAttachmentSlot: () => (() => boolean) | null;
+    /**
+     * Puts attachments of a message the chat gave back in front of the others, as many as fit, the first
+     * ones first, and returns whether all of them fit.
+     */
+    restoreAttachments: (attachments: ReadonlyArray<AssistantDraftAttachment>) => boolean;
+    /** Takes an attachment off the message being written; nothing happens when it is not attached. */
+    removeAttachment: (id: string) => void;
+    /** Takes every attachment off the message being written. */
+    clearAttachments: () => void;
     /**
      * Makes the chat ready for the next message. A thread the server no longer lets this session
      * continue, because the assistant's setup changed since it was started or the thread reached
@@ -212,10 +431,44 @@ export declare class AssistantSystem implements InitializableSystem {
      */
     resume: () => Promise<boolean>;
     /**
-     * Loads the thread before the oldest painted one and paints it above. Does nothing when there is
-     * none or one is being loaded. A load that fails is logged and can be asked for again.
+     * An object URL of a screenshot of the conversation, by the id its tool result references it by.
+     * It is read from the server through the client of the current chat, once per image; a read that
+     * failed is forgotten, so it can be asked for again. Rejects without a chat or when the read fails.
+     */
+    loadScreenshotUrl: (imageId: string) => Promise<string>;
+    /**
+     * Reads a reply of the conversation aloud, by the id of its message, stopping the one being read; a reply being
+     * read is stopped instead. Its speech is read from the server through the client of the current chat, once per
+     * reply, and nothing happens without a chat. A reply the server cannot read aloud is logged, and stays idle.
+     */
+    playReply: (messageId: string) => void;
+    /** Stops the reply being read aloud. */
+    stopPlayback: () => void;
+    /**
+     * Mutes or unmutes the replies that read themselves aloud after a voice message; muting stops the one being read.
+     * The browser remembers the choice.
+     */
+    setMuted: (muted: boolean) => void;
+    /**
+     * Lets the page read the reply to a voice message aloud on its own once it has ended. Call it while the user taps,
+     * such as on the tap that records or sends the message: Safari plays audio that no tap started only on an element
+     * that played during one before. Calls after the first do nothing.
+     */
+    unlockPlayback: () => void;
+    /**
+     * Loads the closed thread named by `earlierThreadId` of the chat, whole, and paints it above the
+     * painted ones; when it is the oldest painted thread itself, closed while only its newest messages
+     * were painted, it replaces that partial copy. Does nothing when there is none, one is being
+     * loaded, or the chat's own thread still has older messages: the earlier threads come above them.
+     * A load that fails is logged and can be asked for again.
      */
     loadEarlierThread: () => Promise<void>;
+    /**
+     * Loads the page of messages before the oldest painted one of the chat's thread and paints it
+     * above. Does nothing when the thread has no older messages or a page is being loaded. A load that
+     * fails is logged and can be asked for again; one that ends after its chat did is dropped.
+     */
+    loadOlderMessages: () => Promise<void>;
     /**
      * Stops the reply on both sides. Closing the stream alone keeps the server run going, because a
      * closed connection is also how a reload looks and a reload must be able to rejoin the run.
@@ -227,16 +480,75 @@ export declare class AssistantSystem implements InitializableSystem {
      * server is therefore gone from the thread and has to be written again.
      */
     reloadThread: () => void;
+    /**
+     * Undoes the changes of the given `apply` tool calls, one undo step at a time, while the step the
+     * configuration is at is one of them. A change the user made in between stops it, so it is never
+     * taken back; the non-undoable commits made on top of an undone change go with it, as with the
+     * toolbar's undo.
+     *
+     * Accepted: like the toolbar's undo, it checks neither the session mode nor the view mode; only
+     * `apply` follows the view mode and refuses a finalize run.
+     */
+    undoToolCalls: (toolCallIds: ReadonlySet<string>) => void;
     setOpen: (open: boolean) => void;
     setInputFocused: (isInputFocused: boolean) => void;
+    /** Every change of the annotation overlay goes through here: the configurator is inert while there is one. */
+    private _setAnnotationOverlay;
+    /** The camera path of `annotate`: still camera, glide, freeze, edit, glide back. */
+    private _annotateView;
+    /**
+     * Runs one annotation as the current `annotate()` call and resolves with what the user made, or with
+     * `null` when a configuration loaded meanwhile cancelled it: it belongs to the message of another one.
+     */
+    private _runAnnotation;
+    /** The attachment path of `annotate`: annotate an image of the message, which the annotation replaces. */
+    private _annotateAttachment;
+    /** The image path of `annotate`: draw the image, edit. The camera stays as it is. */
+    private _annotateImage;
+    /** Opens the editor on `view` and resolves with what the user made, or `null` when they cancel. */
+    private _edit;
+    private _logAnnotationFailure;
+    /** A new configuration load cancels an annotation that is made, and closes one that is shown. */
+    private _closeAnnotationOverlay;
     /**
      * Where the message starts that is being sent on the chat for the first time and that the server
      * refused because its thread cannot be continued, so it never reached the thread; `-1` when there
      * is no such message. A refused continuation of a reply is not one: its message is stored.
      */
     private _findRefusedMessageIndex;
+    /**
+     * The content of a user message: its text, when it has any, or its recording, and a reference to
+     * each of its images.
+     */
+    private _messageContent;
+    /**
+     * Uploads the images of a message into the chat's thread, in their order, and resolves with their
+     * ids. Resolves with `'refused'` when the server refused an upload because the thread cannot be
+     * continued, and marks the chat so; then with `null` for any other failure, which is reported as
+     * the error of the chat. It never rejects.
+     */
+    private _uploadImages;
+    /**
+     * Uploads the images of a message into the thread that was resumed for it. A message the chat
+     * refused shows nowhere meanwhile, so the chat stays resuming until the upload is over: nothing
+     * else is sent ahead of it.
+     */
+    private _uploadSuccessorImages;
+    private _reportUploadFailure;
     private _sendOn;
-    private _resume;
+    private _compact;
+    /**
+     * The `apply` call whose committed state is the step the configuration is at. Nothing at the bottom
+     * of the history, where nothing can be undone: that step was not made by a change it could take back.
+     */
+    private _findTipToolCallId;
+    private _refreshUndoTip;
+    private _loadSpeech;
+    /**
+     * Reads aloud the reply to a voice message, its newest message with text, unless the user muted it or that message
+     * cannot be read aloud; a browser that refuses leaves its play button to start it.
+     */
+    private _readVoiceReply;
     private _committedState;
     private _patchPanel;
     /**
@@ -264,6 +576,8 @@ export declare class AssistantSystem implements InitializableSystem {
      * it is: whether it finishes its reply or is cut off is the caller's decision.
      */
     private _retireChat;
+    /** Revokes the object URLs of the screenshots read for the configuration that was loaded, also those still being read. */
+    private _releaseScreenshotUrls;
     /** Ends the current chat without cutting its reply off. */
     private _endChat;
 }

package/dist/configurator/application-runtime/extensions/systems/assistant-system/assistant-system.js

@@ -1 +1 @@
-import{ChatClient as e,fetchServerSentEvents as t}from"@tanstack/ai-client";import{BehaviorSubject as i}from"rxjs";import{skip as s}from"rxjs/operators";import{isShallowEqual as a}from"@salsita-ai/common/utils";import{AssistantRequestError as r}from"#package/client";import{createBrowserDetailsClientTool as n}from"./browser-details.js";import{ChangeRecorder as h}from"./change-recorder.js";import{createApplyClientTool as o,createQueryClientTool as l}from"./graphql-tools.js";let c=Object.freeze({messages:Object.freeze([]),isLoading:!1,error:void 0,failedStatus:void 0,isContinuable:!0,isLoaded:!1,isResuming:!1,earlierThreads:Object.freeze([]),hasEarlierThread:!1,isLoadingEarlierThread:!1}),d=Object.freeze({open:!1,hasUnreadReply:!1,isInputFocused:!1}),u=Object.freeze({}),_=Object.freeze({results:Object.freeze([]),committed:!1,sideEffects:Object.freeze([]),message:"Nothing was run, because the user closed the configurator while this reply was being written. Do not call any more tools in this reply and end it now. The tools work again as soon as the user writes the next message."}),g=e=>{if(e.getError())return!0;let t=e.getMessages().at(-1)?.parts.at(-1);return t?.type!=="tool-call"&&t?.type!=="tool-result"};export class AssistantSystem{_context;_tools;_chat=new i(c);_panel=new i(d);_activation=null;_current=null;_finishing=new Set;_olderThreadId=null;_resuming=null;_refused=null;_firstAttempt=null;_chain={};_loadingThreadId=null;_recorder;_recording=null;constructor(e){this._context=e;let{configuratorExtensionSelectors:t,domainContextSelectors:i,domainServicesSelectors:s,getAssistantSetup:a}=e;this._recorder=new h({maxEntities:30},{readers:{selectedEntityId:e=>i.getNavigationSystem().getActiveSelectionState(e),navigationState:e=>i.getNavigationSystem().getActiveNavigationState(e),viewState:e=>i.getNavigationSystem().getActiveViewState(e),toolState:e=>i.getNavigationSystem().getActiveToolState(e)},getExposedFields:()=>a().manifest?.fields??u,resolveLocalizable:e=>s.getResolveLocalizable()(e)});let r={getGraphQLSystem:()=>t.getGraphQLSystem(),getStoreSystem:()=>i.getStoreSystem(),getActiveMode:()=>i.getActiveMode(),clearChanges:()=>this._recorder.clear(this._committedState())};this._tools=[n(),l(r),o(r)]}chat$=this._chat;panel$=this._panel;initialize=()=>{let{committed:e}=this._context.domainContextSelectors.getStoreSystem().stateSlices;this._recording??=e.pipe(s(1)).subscribe(this._recorder.record),this._recorder.reset(e.value.state),this._endChat(),this._patchPanel(d),this._activation&&this._startClient()};activate=()=>{this._endChat();let e={};return this._activation=e,this._startClient(),()=>{this._activation===e&&(this._activation=null,this._endChat())}};send=async e=>{if(!this._chat.value.isContinuable&&!await this.resume())return;let t=this._current;if(!t)return;let i={changes:this._recorder.take(this._committedState())};this._firstAttempt=t;try{await this._sendOn(t,e,i);let s=this._findRefusedMessageIndex(t);if(-1===s)return;this._refused={chat:t,index:s}}finally{this._firstAttempt===t&&(this._firstAttempt=null)}if(!await this.resume())return;let s=this._current;s&&await this._sendOn(s,e,i)};resume=async()=>{let e=this._current;if(!e)return!1;if(this._chat.value.isContinuable)return!0;if(this._resuming)return this._resuming;let t=this._resume(e).finally(()=>{this._resuming===t&&(this._resuming=null)});return this._resuming=t,t};loadEarlierThread=async()=>{let e=this._current,t=this._olderThreadId;if(!e||null===t||this._loadingThreadId===t)return;let i=this._chain;this._loadingThreadId=t,this._chat.next({...this._chat.value,isLoadingEarlierThread:!0});try{let s=await e.authorizedClient.getAssistantEarlierThread(e.configurationId,t,50);if(this._chain!==i||this._olderThreadId!==t)return;this._olderThreadId=s.previousThreadId,this._chat.next({...this._chat.value,earlierThreads:[{threadId:t,messages:s.messages},...this._chat.value.earlierThreads],hasEarlierThread:null!==s.previousThreadId})}catch(e){this._context.domainServicesSelectors.getLogger().warn("Could not load the earlier assistant conversation",{error:e})}finally{this._chain===i&&this._loadingThreadId===t&&(this._loadingThreadId=null,this._chat.next({...this._chat.value,isLoadingEarlierThread:!1}))}};stop=async()=>{let e=this._current;if(!e)return;let t=e.client.getCurrentRunId();if(e.client.stop(),null!=t)try{let i=e.getConnection();await fetch(`${i.url}/runs/${encodeURIComponent(t)}/cancel`,{method:"POST",headers:await i.headers()})}catch(e){this._context.domainServicesSelectors.getLogger().warn("Could not cancel the assistant run on the server",{error:e})}};reloadThread=()=>{if(!this._activation)return;let e=this._retireChat();e?.client.stop(),e?.dispose(),this._recorder.reset(this._committedState()),this._startClient()};setOpen=e=>{this._patchPanel(e?{open:e,hasUnreadReply:!1}:{open:e,isInputFocused:!1})};setInputFocused=e=>{this._patchPanel({isInputFocused:e})};_findRefusedMessageIndex=e=>{var t;let i,{failedStatus:s,isContinuable:a}=this._chat.value;return this._firstAttempt===e&&e.generation.active&&409===s&&!a?(i=(t=e.client.getMessages()).map(e=>e.role).lastIndexOf("user"),t.slice(i+1).every(e=>0===e.parts.length)?i:-1):-1};_sendOn=async(e,t,i)=>{let{client:s}=e;s.getIsLoading()||null===s.getResumeState()||s.stop();try{await s.sendMessage({content:t,metadata:i})}catch(i){let t=i instanceof Error?i:Error(String(i));e.generation.active&&this._chat.next({...this._chat.value,error:t}),this._context.domainServicesSelectors.getLogger().error("The assistant message could not be sent",{},t)}};_resume=async e=>{this._chat.next({...this._chat.value,isResuming:!0,error:void 0,failedStatus:void 0});try{let{previousThreadId:t}=await e.authorizedClient.compactAssistantThread(e.configurationId);if(!e.generation.active)return!1;let{messages:i,earlierThreads:s,hasEarlierThread:a}=this._chat.value,r=this._refused?.chat===e?i.slice(0,this._refused.index):i;return e.generation.active=!1,this._current=null,e.client.stop(),e.dispose(),this._chat.next({...c,earlierThreads:[...s,{threadId:t,messages:r}],hasEarlierThread:a,isLoadingEarlierThread:null!==this._loadingThreadId}),this._startClient(),!0}catch(i){let t=i instanceof Error?i:Error(String(i));if(t instanceof r&&409===t.status)return e.generation.active&&this.reloadThread(),!1;return e.generation.active&&this._chat.next({...this._chat.value,isResuming:!1,error:t,failedStatus:t instanceof r?t.status:void 0}),this._context.domainServicesSelectors.getLogger().error("The assistant thread could not be resumed",{},t),!1}};_committedState=()=>this._context.domainContextSelectors.getStoreSystem().stateSlices.committed.value.state;_patchPanel=e=>{let t={...this._panel.value,...e};a(t,this._panel.value)||this._panel.next(t)};_takeOverThread=e=>{for(let t of this._finishing)t.configurationId===e&&(t.client.stop(),t.dispose())};_startClient=()=>{let i={active:!0},s=this._context.getConfigurationId();this._takeOverThread(s);let r=this._context.getAuthorizedClient(),n=()=>{if(i.active){let{manifest:e,error:t}=this._context.getAssistantSetup();if(t)throw t;if(!e)throw Error("The assistant is disabled for this session.")}return r.getAssistantChatConnection(s)},h=e=>{if(!i.active)return;let t={...this._chat.value,...e};a(t,this._chat.value)||this._chat.next(t)},o=async(e,t)=>{let s=await fetch(e,t).finally(()=>h({isLoaded:!0})),a=s.headers.get("x-assistant-thread-continuable");return null!=a&&s.ok&&i.active&&0===this._chat.value.earlierThreads.length&&(this._olderThreadId=s.headers.get("x-assistant-previous-thread")),h({failedStatus:s.ok?void 0:s.status,...null!=a&&{isContinuable:"false"!==a,hasEarlierThread:null!==this._olderThreadId}}),s},l=async()=>({headers:await n().headers(),fetchClient:o}),c=t(()=>n().url,l),d=t(()=>`${n().url}/runs`,l),u=this._tools.map(e=>({...e,execute:(t,s)=>{if(!i.active)return _;let{manifest:a,error:r}=this._context.getAssistantSetup();return r||!a?{error:r?.message??"The assistant is disabled for this session."}:e.execute?.(t,s)}})),f=new e({threadId:`assistant:${s}`,connection:{...c,joinRun:d.joinRun},persistence:!0,history:{pageSize:50},tools:u,onMessagesChange:e=>h({messages:e}),onLoadingChange:e=>{let t=i.active&&this._chat.value.isLoading;h({isLoading:e}),!e&&g(f)&&(i.active?t&&!this._panel.value.open&&this._patchPanel({hasUnreadReply:!0}):v())},onErrorChange:e=>{i.active&&e!==this._chat.value.error&&(!e||-1===this._findRefusedMessageIndex(p))&&(h({error:e}),e&&this._context.domainServicesSelectors.getLogger().error("The assistant request failed",{status:this._chat.value.failedStatus},e))}}),m=!1,v=()=>{m||(m=!0,this._finishing.delete(p),f.dispose())},p={client:f,configurationId:s,generation:i,authorizedClient:r,getConnection:n,dispose:v};this._current=p,f.attach()};_retireChat=()=>{let e=this._current;return this._current=null,e&&(e.generation.active=!1),this._chain={},this._olderThreadId=null,this._resuming=null,this._refused=null,this._loadingThreadId=null,this._chat.value!==c&&this._chat.next(c),e};_endChat=()=>{let e=this._retireChat();if(e){if(!e.client.getIsLoading()&&g(e.client))return void e.dispose();this._finishing.add(e)}}}
+import{ChatClient as t,fetchServerSentEvents as e}from"@tanstack/ai-client";import{BehaviorSubject as a,firstValueFrom as s,race as i,timer as n}from"rxjs";import{filter as r,skip as o}from"rxjs/operators";import{isShallowEqual as l}from"@salsita-ai/common/utils";import{ASSISTANT_TRANSCRIPT_EVENT as h,AssistantTranscriptEvent as c,assistantImageReference as u}from"@salsita-ai/contracts/assistant";import{AssistantRequestError as d}from"#package/client";import{AssistantSpeechPlayer as _}from"./assistant-speech-player.js";import{ChangeRecorder as g}from"./change-recorder.js";import{ComposerAttachments as m}from"./composer-attachments.js";import{createApplyClientTool as p,createQueryClientTool as f,toToolResult as v}from"./graphql-tools.js";import{isSpeakableReply as y}from"./speakable-reply.js";let S=Object.freeze({messages:Object.freeze([]),isLoading:!1,error:void 0,failedStatus:void 0,retryAfterSeconds:void 0,isContinuable:!0,isLocked:!1,isNetworkLocked:!1,isLoaded:!1,isVoiceEnabled:!1,isResuming:!1,earlierThreads:Object.freeze([]),earlierThreadId:null,isLoadingEarlierThread:!1,hasOlderMessages:!1,isLoadingOlderMessages:!1}),w=Object.freeze({top:0,right:0,bottom:0,left:0}),x=Object.freeze({kind:"transition"}),A=Object.freeze({open:!1,hasUnreadReply:!1,isInputFocused:!1}),C=Object.freeze({}),I=v({results:[],committed:!1,sideEffects:[],warnings:[],screenshots:new Map,message:"Nothing was run, because the user closed the configurator while this reply was being written. Do not call any more tools in this reply and end it now. The tools work again as soon as the user writes the next message."}),T=t=>{if(t.getError())return!0;let e=t.getMessages().at(-1)?.parts.at(-1);return e?.type!=="tool-call"&&e?.type!=="tool-result"},b=async t=>{try{let e=await t.clone().json(),a=e?.page;return a?.truncated===!0&&"string"==typeof a.cursor&&""!==a.cursor}catch{return!1}},O=t=>{let e=t.map(t=>t.role).lastIndexOf("user");return t.slice(e+1).every(t=>0===t.parts.length)?e:-1},k=async t=>{let e=new Uint8Array(await t.arrayBuffer()),a="";for(let t=0;t<e.length;t+=32768)a+=String.fromCharCode(...e.subarray(t,t+32768));return btoa(a)};export class AssistantSystem{_context;_tools;_chat=new a(S);_panel=new a(A);_annotationOverlay=new a(null);_imageView=new a(null);_composerAttachments=new m;_releaseInert=null;_annotationRun=null;_activation=null;_current=null;_finishing=new Set;_resuming=null;_refused=null;_firstAttempt=null;_chain={};_recorder;_recording=null;_applyToolCallIds=new WeakMap;_undoTipToolCallId=new a(void 0);_screenshotUrls=new Map;_player=new _({createAudio:()=>this._context.createAudio(),logError:(t,e)=>this._context.domainServicesSelectors.getLogger().error(t,{},e)});_voiceReplyChat=null;constructor(t){this._context=t;let{configuratorExtensionSelectors:e,domainContextSelectors:a,domainServicesSelectors:s,getAssistantSetup:i}=t;this._recorder=new g({maxEntities:30},{readers:{selectedEntityId:t=>a.getNavigationSystem().getActiveSelectionState(t),navigationState:t=>a.getNavigationSystem().getActiveNavigationState(t),viewState:t=>a.getNavigationSystem().getActiveViewState(t),toolState:t=>a.getNavigationSystem().getActiveToolState(t)},getExposedFields:()=>i().manifest?.fields??C,resolveLocalizable:t=>s.getResolveLocalizable()(t)});let n={getGraphQLSystem:()=>e.getGraphQLSystem(),getStoreSystem:()=>a.getStoreSystem(),getActiveMode:()=>a.getActiveMode(),getIsViewMode:()=>a.getIsViewMode(),clearChanges:()=>this._recorder.clear(this._committedState()),rememberCommit:(t,e)=>{this._applyToolCallIds.set(e,t),this._refreshUndoTip()}};this._tools=[f(n),p(n)],a.getUndoSystem().tip$.subscribe(this._refreshUndoTip)}chat$=this._chat;panel$=this._panel;annotationOverlay$=this._annotationOverlay;imageView$=this._imageView;composerAttachments$=this._composerAttachments.attachments$;hasFreeAttachmentSlot$=this._composerAttachments.hasFreeSlot$;undoTipToolCallId$=this._undoTipToolCallId;playback$=this._player.playback$;muted$=this._player.muted$;getSessionManifest=()=>this._context.getAssistantSetup().manifest;initialize=()=>{let{committed:t}=this._context.domainContextSelectors.getStoreSystem().stateSlices;this._recording??=t.pipe(o(1)).subscribe(this._recorder.record),this._recorder.reset(t.value.state),this._endChat(),this._patchPanel(A),this._releaseScreenshotUrls(),this._player.reset(),this._closeAnnotationOverlay(),this.closeImageView(),this.clearAttachments(),this._activation&&this._startClient()};activate=()=>{this._endChat();let t={};return this._activation=t,this._startClient(),()=>{this._activation===t&&(this._activation=null,this._endChat(),this._player.stop())}};send=async(t,e=[],{onSent:a,onUnsent:s,audio:i}={})=>{if(!this._chat.value.isContinuable&&!await this.resume())return;let n=this._current;if(!n)return;let r={changes:this._recorder.take(this._committedState()),locale:this._context.domainServicesSelectors.getI18n().locale},o=()=>this._recorder.reset(this._committedState()),l=e.map(t=>t instanceof Blob?t:t.blob),h=i?{type:"audio",source:{type:"data",value:await k(i),mimeType:i.type}}:null,c=!1,u=(s,i)=>{c||(c=!0,a?.());let n=e.flatMap((t,e)=>{let a=i[e];return t instanceof Blob||void 0===a?[]:[{imageId:a,pins:t.pins,source:t.source??"view"}]});return this._voiceReplyChat=h?s:null,this._sendOn(s,this._messageContent(h?"":t,i,h),0===n.length?r:{...r,annotations:n})},d=t=>{if(!h||!t.generation.active||t.client.getIsLoading())return;let e=t.client.getMessages(),{error:a,failedStatus:i}=this._chat.value,n=O(e);if(a&&void 0!==i&&-1!==n){t.client.setMessagesManually(e.slice(0,n)),o(),422!==i&&s?.();return}e.some(t=>"user"===t.role&&t.parts.some(t=>"audio"===t.type))&&this.reloadThread()};this._firstAttempt=n;try{let t=0===l.length?[]:await this._uploadImages(n,l);if(null===t||!n.generation.active)return void o();if("refused"!==t){await u(n,t);let e=this._findRefusedMessageIndex(n);if(-1===e)return void d(n);this._refused={chat:n,index:e}}}finally{this._firstAttempt===n&&(this._firstAttempt=null)}if(!await this.resume()){c||o(),this._current&&d(this._current);return}let _=this._current,g=_?await this._uploadSuccessorImages(_,l):null;if(!_||null===g||"refused"===g||!_.generation.active){_&&"refused"===g&&this._reportUploadFailure(_,this._chat.value.error),c&&s?.(),o();return}await u(_,g),d(_)};annotate=async t=>{let e;if(null!==this._annotationOverlay.value)return null;if(void 0!==t&&"attachmentId"in t)return this._annotateAttachment(t);let a=this._composerAttachments.reserveSlot();if(!a)return null;try{e=await this._runAnnotation(e=>t?this._annotateImage(e,t):this._annotateView(e))}finally{a()}return e&&this._composerAttachments.attach(e.blob,{pins:e.pins,source:t?"image":"view"})};viewAnnotation=t=>{if(null!==this._annotationOverlay.value)return;let e={kind:"viewer",annotation:t,close:()=>{this._annotationOverlay.value===e&&this._setAnnotationOverlay(null)}};this._setAnnotationOverlay(e)};viewImage=t=>{this._imageView.next(t)};closeImageView=()=>{null!==this._imageView.value&&this._imageView.next(null)};attach=t=>this._composerAttachments.attach(t);reserveAttachmentSlot=()=>this._composerAttachments.reserveSlot();restoreAttachments=t=>this._composerAttachments.restore(t);removeAttachment=t=>this._composerAttachments.remove(t);clearAttachments=()=>this._composerAttachments.clear();resume=async()=>{let t=this._current;if(!t)return!1;if(this._chat.value.isContinuable)return!0;if(this._resuming)return this._resuming;let e=this._compact(t).finally(()=>{this._resuming===e&&(this._resuming=null)});return this._resuming=e,e};loadScreenshotUrl=t=>{let e=this._screenshotUrls.get(t);if(e)return e;let a=this._current;if(!a)return Promise.reject(Error(`The screenshot ${t} cannot be read, because the assistant chat is not active. Open the assistant panel of the configuration first.`));let s=this._screenshotUrls,i=a.authorizedClient.getAssistantImage(a.configurationId,t).then(t=>URL.createObjectURL(t));return s.set(t,i),i.catch(()=>{s.get(t)===i&&s.delete(t)}),i};playReply=t=>{let e=this._current;e&&this._player.toggle({messageId:t,loadSpeech:()=>this._loadSpeech(e,t),isAutoplay:!1})};stopPlayback=()=>{this._player.stop()};setMuted=t=>{this._player.setMuted(t)};unlockPlayback=()=>{this._player.unlock()};loadEarlierThread=async()=>{let t=this._current,{earlierThreadId:e,isLoadingEarlierThread:a,hasOlderMessages:s}=this._chat.value;if(!t||null===e||a||s)return;let i=this._chain;this._chat.next({...this._chat.value,isLoadingEarlierThread:!0});try{let a=await t.authorizedClient.getAssistantEarlierThread(t.configurationId,e);if(this._chain!==i||this._chat.value.earlierThreadId!==e)return;let[s,...n]=this._chat.value.earlierThreads,r={threadId:e,messages:a.messages};this._chat.next({...this._chat.value,earlierThreads:s?.threadId===e?[r,...n]:[r,...this._chat.value.earlierThreads],earlierThreadId:a.previousThreadId})}catch(t){this._context.domainServicesSelectors.getLogger().warn("Could not load the earlier assistant conversation",{error:t})}finally{this._chain===i&&this._chat.next({...this._chat.value,isLoadingEarlierThread:!1})}};loadOlderMessages=async()=>{let t=this._current,{hasOlderMessages:e,isLoadingOlderMessages:a}=this._chat.value;if(t&&e&&!a){this._chat.next({...this._chat.value,isLoadingOlderMessages:!0});try{await t.client.loadOlderMessages()}catch(t){this._context.domainServicesSelectors.getLogger().warn("Could not load older assistant messages",{error:t})}finally{t===this._current&&this._chat.next({...this._chat.value,hasOlderMessages:t.client.getHasOlderMessages(),isLoadingOlderMessages:!1})}}};stop=async()=>{let t=this._current;if(!t)return;let e=t.client.getCurrentRunId();if(this._voiceReplyChat===t&&(this._voiceReplyChat=null),t.client.stop(),null!=e)try{let a=t.getConnection();await fetch(`${a.url}/runs/${encodeURIComponent(e)}/cancel`,{method:"POST",headers:await a.headers()})}catch(t){this._context.domainServicesSelectors.getLogger().warn("Could not cancel the assistant run on the server",{error:t})}};reloadThread=()=>{if(!this._activation)return;let t=this._retireChat();t?.client.stop(),t?.dispose(),this._recorder.reset(this._committedState()),this._startClient()};undoToolCalls=t=>{let e=this._context.domainContextSelectors.getUndoSystem(),a=()=>{let e=this._findTipToolCallId();return void 0!==e&&t.has(e)};for(;a();)e.undo()};setOpen=t=>{this._patchPanel(t?{open:t,hasUnreadReply:!1}:{open:t,isInputFocused:!1})};setInputFocused=t=>{this._patchPanel({isInputFocused:t})};_setAnnotationOverlay=t=>{null!==t&&null===this._releaseInert&&(this._releaseInert=this._context.configuratorExtensionSelectors.getLayoutSystem().makeInert()),this._annotationOverlay.next(t),null===t&&(this._releaseInert?.(),this._releaseInert=null)};_annotateView=async t=>{let{getCameraControlSystem:e,getCanvasSystem:a}=this._context.configuratorExtensionSelectors,o=e(),l=a().r3fHandle$.value?.get().frameloop==="demand",h=()=>this._annotationRun===t;if(this._setAnnotationOverlay(x),await s(i(o.transitionState$.pipe(r(t=>"still"===t||"sleep"===t&&!l)),n(2e3))),!h())return this._setAnnotationOverlay(null),null;let c=o.holdCameraOffset(),u=null;try{await o.glideCameraOffset(w);let t=await this._context.freezeView();h()&&(u=await this._edit(t))}catch(t){this._logAnnotationFailure("The configurator view could not be frozen for an annotation",t)}this._setAnnotationOverlay(x);try{await o.glideCameraOffset(o.canvasPadding$.value)}finally{c(),this._setAnnotationOverlay(null)}return u};_runAnnotation=async t=>{let e={},a=null;this._annotationRun=e;try{a=await t(e)}finally{this._annotationRun!==e?a=null:this._annotationRun=null}return a};_annotateAttachment=async({attachmentId:t,background:e})=>{let a=this._composerAttachments.attachments$.value.find(({id:e})=>e===t);if(!a||a.pins)return null;let s=await this._runAnnotation(t=>this._annotateImage(t,{image:a.blob,background:e}));return s&&this._composerAttachments.replace(t,s.blob,{pins:s.pins,source:"image"})};_annotateImage=async(t,{image:e,background:a})=>{this._setAnnotationOverlay(x);try{let s=await this._context.rasterizeImage(e,a);return this._annotationRun===t?await this._edit(s):null}catch(t){return this._logAnnotationFailure("The image could not be drawn for an annotation",t),null}finally{this._setAnnotationOverlay(null)}};_edit=t=>new Promise(e=>{this._setAnnotationOverlay({kind:"editor",view:t,finish:e})});_logAnnotationFailure=(t,e)=>{let a=e instanceof Error?e:Error(String(e));this._context.domainServicesSelectors.getLogger().error(t,{},a)};_closeAnnotationOverlay=()=>{let t=this._annotationOverlay.value;this._annotationRun=null,t?.kind==="editor"&&t.finish(null),t?.kind==="viewer"&&t.close()};_findRefusedMessageIndex=t=>{let{failedStatus:e,isContinuable:a}=this._chat.value;return this._firstAttempt===t&&t.generation.active&&409===e&&!a?O(t.client.getMessages()):-1};_messageContent=(t,e,a)=>0!==e.length||a?[...t?[{type:"text",content:t}]:[],...a?[a]:[],...e.map(t=>({type:"image",source:{type:"url",value:u(t)}}))]:t;_uploadImages=async(t,e)=>{if(0===e.length)return[];t.generation.active&&this._chat.next({...this._chat.value,error:void 0,failedStatus:void 0});try{return(await Promise.all(e.map(e=>t.authorizedClient.uploadAssistantImage(t.configurationId,e)))).map(({imageId:t})=>t)}catch(a){let e=a instanceof Error?a:Error(String(a));if(e instanceof d&&e.threadNotContinuable)return t.generation.active&&this._chat.next({...this._chat.value,isContinuable:!1,error:e,failedStatus:e.status}),"refused";return this._reportUploadFailure(t,e),null}};_uploadSuccessorImages=async(t,e)=>{if(0===e.length)return[];this._chat.next({...this._chat.value,isResuming:!0});try{return await this._uploadImages(t,e)}finally{t.generation.active&&this._chat.next({...this._chat.value,isResuming:!1})}};_reportUploadFailure=(t,e)=>{t.generation.active&&this._chat.next({...this._chat.value,error:e,failedStatus:e instanceof d?e.status:void 0}),this._context.domainServicesSelectors.getLogger().error("The images of the assistant message could not be uploaded",{},e)};_sendOn=async(t,e,a)=>{let{client:s}=t;s.getIsLoading()||null===s.getResumeState()||s.stop();try{await s.sendMessage({content:e,metadata:a})}catch(a){let e=a instanceof Error?a:Error(String(a));t.generation.active&&this._chat.next({...this._chat.value,error:e}),this._context.domainServicesSelectors.getLogger().error("The assistant message could not be sent",{},e)}};_compact=async t=>{this._chat.next({...this._chat.value,isResuming:!0,error:void 0,failedStatus:void 0});try{let{previousThreadId:e}=await t.authorizedClient.compactAssistantThread(t.configurationId);if(!t.generation.active)return!1;let{messages:a,earlierThreads:s,earlierThreadId:i,isLoadingEarlierThread:n}=this._chat.value,r=this._refused?.chat===t?a.slice(0,this._refused.index):a,o=t.client.getHasOlderMessages()?e:i;return t.generation.active=!1,this._current=null,t.client.stop(),t.dispose(),this._chat.next({...S,earlierThreads:[...s,{threadId:e,messages:r}],earlierThreadId:o,isLoadingEarlierThread:n}),this._startClient(),!0}catch(a){let e=a instanceof Error?a:Error(String(a));if(e instanceof d&&409===e.status)return t.generation.active&&this.reloadThread(),!1;return t.generation.active&&this._chat.next({...this._chat.value,isResuming:!1,error:e,failedStatus:e instanceof d?e.status:void 0,...e instanceof d&&e.locked&&{isLocked:!0,isNetworkLocked:e.networkLocked}}),this._context.domainServicesSelectors.getLogger().error("The assistant thread could not be resumed",{},e),!1}};_findTipToolCallId=()=>{let t=this._context.domainContextSelectors.getUndoSystem(),e=t.tip$.value?.committedState;return void 0!==e&&t.canUndo()?this._applyToolCallIds.get(e):void 0};_refreshUndoTip=()=>{let t=this._findTipToolCallId();t!==this._undoTipToolCallId.value&&this._undoTipToolCallId.next(t)};_loadSpeech=(t,e)=>t.authorizedClient.getAssistantReplySpeech(t.configurationId,e);_readVoiceReply=t=>{if(this._player.muted$.value||!this._chat.value.isVoiceEnabled)return;let e=t.client.getMessages(),a=e.map(t=>t.role).lastIndexOf("user")+1,s=e.slice(a).reverse().find(t=>t.parts.some(t=>"text"===t.type&&""!==t.content.trim()));s&&y(s)&&this._player.play({messageId:s.id,loadSpeech:()=>this._loadSpeech(t,s.id),isAutoplay:!0})};_committedState=()=>this._context.domainContextSelectors.getStoreSystem().stateSlices.committed.value.state;_patchPanel=t=>{let e={...this._panel.value,...t};l(e,this._panel.value)||this._panel.next(e)};_takeOverThread=t=>{for(let e of this._finishing)e.configurationId===t&&(e.client.stop(),e.dispose())};_startClient=()=>{let a={active:!0},s=this._context.getConfigurationId();this._takeOverThread(s);let i=this._context.getAuthorizedClient(),n=()=>{if(a.active){let{manifest:t,error:e}=this._context.getAssistantSetup();if(e)throw e;if(!t)throw Error("The assistant is disabled for this session.")}return i.getAssistantChatConnection(s)},r=t=>{if(!a.active)return;let e={...this._chat.value,...t};l(e,this._chat.value)||this._chat.next(e)},o=async(t,e)=>{let a,s=await fetch(t,e).finally(()=>r({isLoaded:!0})),i=s.headers.get("x-assistant-thread-continuable"),n=s.headers.get("x-assistant-locked"),o="true"===n||"network"===n,l=null!=i&&s.ok,h=l&&0===this._chat.value.earlierThreads.length,c=h?s.headers.get("x-assistant-previous-thread"):null;return r({failedStatus:s.ok?void 0:s.status,retryAfterSeconds:s.ok?void 0:null!==(a=s.headers.get("retry-after"))&&/^\d+$/.test(a.trim())?Number(a):void 0,...null!=i&&{isContinuable:"false"!==i},...(l||o)&&{isLocked:o,isNetworkLocked:"network"===n},...l&&{isVoiceEnabled:"true"===s.headers.get("x-assistant-voice")},...h&&{earlierThreadId:c,...null!==c&&{hasOlderMessages:await b(s)}}}),s},u=async()=>({headers:await n().headers(),fetchClient:o}),d=e(()=>n().url,u),_=e(()=>`${n().url}/runs`,u),g=this._tools.map(t=>({...t,execute:(e,s)=>{if(!a.active)return I;let{manifest:i,error:n}=this._context.getAssistantSetup();return n||!i?{error:n?.message??"The assistant is disabled for this session."}:t.execute?.(e,s)}})),m=new t({threadId:`assistant:${s}`,connection:{...d,joinRun:_.joinRun},persistence:!0,history:{pageSize:50},tools:g,onMessagesChange:t=>r({messages:t,hasOlderMessages:m.getHasOlderMessages()}),onLoadingChange:t=>{let e=a.active&&this._chat.value.isLoading;if(r({isLoading:t}),!t&&T(m)){if(!a.active)return void f();e&&!this._panel.value.open&&this._patchPanel({hasUnreadReply:!0}),this._voiceReplyChat===v&&(this._voiceReplyChat=null,m.getError()||this._readVoiceReply(v))}},onCustomEvent:(t,e)=>{let a=t===h?c.safeParse(e):null;if(!a?.success)return;let{messageId:s,transcript:i}=a.data;m.setMessagesManually(m.getMessages().map(t=>t.id===s?{...t,parts:t.parts.flatMap(t=>"audio"===t.type?[{type:"text",content:i}]:[t]),metadata:{...t.metadata,spoken:!0}}:t))},onErrorChange:t=>{a.active&&t!==this._chat.value.error&&(!t||-1===this._findRefusedMessageIndex(v))&&(r({error:t}),t&&this._context.domainServicesSelectors.getLogger().error("The assistant request failed",{status:this._chat.value.failedStatus},t))}}),p=!1,f=()=>{p||(p=!0,this._finishing.delete(v),m.dispose())},v={client:m,configurationId:s,generation:a,authorizedClient:i,getConnection:n,dispose:f};this._current=v,m.attach()};_retireChat=()=>{let t=this._current;return this._current=null,t&&(t.generation.active=!1),this._chain={},this._resuming=null,this._refused=null,this._voiceReplyChat=null,this._chat.value!==S&&this._chat.next(S),t};_releaseScreenshotUrls=()=>{for(let t of this._screenshotUrls.values())t.then(URL.revokeObjectURL,()=>void 0);this._screenshotUrls=new Map};_endChat=()=>{let t=this._retireChat();if(t){if(!t.client.getIsLoading()&&T(t.client))return void t.dispose();this._finishing.add(t)}}}