@salsita-ai/sdk 0.50.0-develop.f16492b14e → 0.50.0-develop.f354d3576d

package/dist/configurator/application-runtime/extensions/systems/assistant-system/assistant-system.d.ts

@@ -1,9 +1,11 @@
 import type { AnyClientTool } from '@tanstack/ai';
 import { type UIMessage } from '@tanstack/ai-client';
 import { type ObservableWithLastValue } from '@salsita-ai/common/utils';
+import { type AssistantMessageAnnotation } from '@salsita-ai/contracts/assistant';
 import { type AuthorizedPhoenixApiClient } from '#package/client';
-import type { AssistantManifest, DomainContextSelectors, DomainServicesSelectors, InitializableSystem, SessionAssistantSetup, SpecSelectors } from '#package/domain';
+import type { AssistantManifest, DomainContextSelectors, DomainServicesSelectors, InitializableSystem, SessionAssistantSetup } from '#package/domain';
 import type { ConfiguratorExtensionSelectors } from '../../selectors.js';
+import type { AssistantAnnotation, AssistantAnnotationOverlay } from './assistant-annotation.js';
 /** A tool of the assistant that the model calls and the browser runs, as the chat client accepts it. */
 export type AssistantClientTool = AnyClientTool;
 /**
@@ -15,12 +17,19 @@ export interface AssistantSystemContext {
     readonly configuratorExtensionSelectors: ConfiguratorExtensionSelectors;
     readonly domainContextSelectors: DomainContextSelectors;
     readonly domainServicesSelectors: DomainServicesSelectors;
-    readonly specSelectors: SpecSelectors;
     readonly getConfigurationId: () => string;
     readonly getAuthorizedClient: () => AuthorizedPhoenixApiClient<any, any, any>;
     /** The assistant the initialized session runs: its tools, its exposed fields and whether chat runs at all. */
     readonly getAssistantSetup: () => SessionAssistantSetup;
+    /**
+     * Copies the configurator view as the next frame draws it, for the user to annotate; see
+     * {@link AssistantSystem.annotate}. The only browser-bound step of an annotation the system takes,
+     * so a test can provide one that takes none.
+     */
+    readonly freezeView: () => Promise<HTMLCanvasElement>;
 }
+/** What the user attaches to a message: an image, or an annotation of their configurator view. */
+export type AssistantAttachment = Blob | AssistantAnnotation;
 /**
  * The conversation with the assistant as the panel paints it. One value is published per change, so
  * it changes with every chunk of a streaming reply.
@@ -37,6 +46,12 @@ export interface AssistantChatSnapshot {
      * the server was never reached.
      */
     readonly failedStatus: number | undefined;
+    /**
+     * The seconds the server asked to wait before sending again, from the `Retry-After` header of the
+     * latest chat request it answered with a failure: a message refused for the assistant's hourly
+     * limits. Nothing when that request carried none.
+     */
+    readonly retryAfterSeconds: number | undefined;
     /** False when the server said this session may read the thread but not add to it. */
     readonly isContinuable: boolean;
     /** False until the first request of the chat, the thread read, was answered or failed. */
@@ -51,10 +66,34 @@ export interface AssistantChatSnapshot {
      * the chat was open, and the ones loaded with {@link AssistantSystem.loadEarlierThread}.
      */
     readonly earlierThreads: ReadonlyArray<AssistantEarlierThreadSnapshot>;
-    /** True when the server holds a thread older than the oldest painted one. */
-    readonly hasEarlierThread: boolean;
-    /** True while that older thread is being loaded. */
+    /**
+     * The closed thread that {@link AssistantSystem.loadEarlierThread} loads next; `null` when the
+     * painted threads start the conversation. Usually the thread before the oldest painted one, but
+     * the oldest painted one itself when it was closed while only its newest messages were painted:
+     * loading it then paints it whole.
+     */
+    readonly earlierThreadId: string | null;
+    /** True while that thread is being loaded. */
     readonly isLoadingEarlierThread: boolean;
+    /** True when the thread of the chat has older messages than the painted ones on the server; see {@link AssistantSystem.loadOlderMessages}. */
+    readonly hasOlderMessages: boolean;
+    /** True while those older messages are being loaded. */
+    readonly isLoadingOlderMessages: boolean;
+}
+/** What else the caller of {@link AssistantSystem.send} learns about the message. */
+export interface AssistantSendOptions {
+    /**
+     * Called once the message is handed to the chat, which paints it: after its images were uploaded,
+     * and not at all when it is not sent because the thread could not be resumed or an upload failed.
+     * The composer keeps the message until then, so a message that is not sent can be sent again.
+     */
+    readonly onSent?: () => void;
+    /**
+     * Called when a message that was handed to the chat is gone from it after all: the server refused
+     * it because the thread could not continue, the thread was resumed without it, and its images
+     * could not be uploaded into the new thread. The caller gets it back, so it can be sent again.
+     */
+    readonly onUnsent?: () => void;
 }
 /** A closed thread of the conversation as the panel paints it above the thread that continues it. */
 export interface AssistantEarlierThreadSnapshot {
@@ -73,9 +112,9 @@ export interface AssistantPanelSnapshot {
     readonly isInputFocused: boolean;
 }
 /**
- * Owns the conversational assistant of a configurator: its client tools (`getBrowserDetails`,
- * `query` and `apply`), the chat with the server, the state of the built-in panel, and the record
- * of what changed in the configuration since the agent last acted, which every user message carries.
+ * Owns the conversational assistant of a configurator: its client tools (`query` and `apply`), the
+ * chat with the server, the state of the built-in panel, and the record of what changed in the
+ * configuration since the agent last acted, which every user message carries.
  *
  * The server owns the thread, so the chat keeps nothing in browser storage: its client hydrates
  * from the server and rejoins a reply that is still streaming.
@@ -89,6 +128,9 @@ export declare class AssistantSystem implements InitializableSystem {
     private readonly _tools;
     private readonly _chat;
     private readonly _panel;
+    private readonly _annotationOverlay;
+    /** Identity of the `annotate()` call that runs; a new configuration load forgets it, which cancels it. */
+    private _annotationRun;
     /** Identity of the current `activate()` call; nothing while the chat is not active. */
     private _activation;
     /** The chat of the current activation; nothing while no chat is active. */
@@ -98,11 +140,6 @@ export declare class AssistantSystem implements InitializableSystem {
      * background. Each leaves the set when its reply ends or when a new chat takes its thread over.
      */
     private readonly _finishing;
-    /**
-     * The thread before the oldest painted one, as the server named it; nothing when the painted
-     * threads start the conversation. `hasEarlierThread` of the chat says whether it is set.
-     */
-    private _olderThreadId;
     /** The resume that is running, so two sends that find the thread locked share one. */
     private _resuming;
     /**
@@ -122,8 +159,6 @@ export declare class AssistantSystem implements InitializableSystem {
      * thread continues the same chain; any other new chat starts another one.
      */
     private _chain;
-    /** The earlier thread that is being loaded, so a second press loads nothing twice. */
-    private _loadingThreadId;
     private readonly _recorder;
     /** The recorder's subscription to the committed emissions of the store; made by the first `initialize()`. */
     private _recording;
@@ -146,6 +181,11 @@ export declare class AssistantSystem implements InitializableSystem {
     readonly chat$: ObservableWithLastValue<AssistantChatSnapshot>;
     /** The state of the built-in panel. It never completes. */
     readonly panel$: ObservableWithLastValue<AssistantPanelSnapshot>;
+    /**
+     * What covers the configurator for an annotation, as the layout renders it; nothing while no
+     * annotation is made or shown. See {@link AssistantSystem.annotate}. It never completes.
+     */
+    readonly annotationOverlay$: ObservableWithLastValue<AssistantAnnotationOverlay | null>;
     /**
      * The `apply` tool call whose committed state is the step of the undo history the configuration is
      * at; nothing when that step is not a change of the assistant. It never completes.
@@ -199,6 +239,14 @@ export declare class AssistantSystem implements InitializableSystem {
      * acted, which the server puts in front of the message for the model. The record is taken only
      * when there is a chat to send it on, because taking it moves its baseline.
      *
+     * The images the user attached go with the message: they are uploaded into the chat's thread
+     * first, and the message references them, one image part after its text each. The text part is
+     * left out of a message with images only. An annotation is attached like an image: its image is
+     * uploaded and referenced in its place among the images, and its pins go into the metadata of the
+     * message, by the id of its image, for the server to tell the model. Each attempt to send uploads them again, into the
+     * thread it sends on. An upload that fails leaves the message unsent, with the failure as the
+     * error of the chat; `onSent` is not called then.
+     *
      * A thread can end in a tool call that nothing will answer: its reply parked while no page was
      * open, or the page that ran the tool was left before the result was sent. The client refuses to
      * send while it waits on such a call, so the wait is given up first, and the server records for
@@ -209,20 +257,44 @@ export declare class AssistantSystem implements InitializableSystem {
      *
      * The chat can be out of date about that: the thread reached its size limit with the reply before
      * this message, or the assistant's setup changed while the configurator stayed open. The server
-     * then refuses the message and says that the thread cannot be continued. The message stays
-     * painted while the thread is resumed, and is then sent again on the new thread, with the change
-     * record it was first sent with. This happens once: a refusal of the second attempt is the
-     * failure it is. When resuming fails, the message is left as any message that failed to send:
-     * painted, with the failure as the error of the chat.
+     * then refuses the message, or the upload of one of its images, and says that the thread cannot
+     * be continued. The thread is resumed, and the message is then sent again on the new thread, its
+     * images uploaded again, with the change record it was first sent with. This happens once: a
+     * refusal of the second attempt is the failure it is. When resuming fails, a refused message is
+     * left as any message that failed to send: painted, with the failure as the error of the chat; a
+     * refused upload leaves the message unsent. When the images of a refused message cannot be
+     * uploaded into the new thread, the message is given back through `onUnsent`.
+     *
+     * A chat that ends while the images upload, because another configuration was loaded or the
+     * thread is read again, sends nothing.
      *
      * Accepted: when the server refuses the compaction itself with a conflict, because another tab
      * compacted the thread or a run is still running, the thread is read again (see `resume`) and
      * the refused message is gone without a notice. It takes two tabs or a run racing the send, and
      * the user writes the message again, as after any reload of a thread that failed.
      *
+     * Accepted: images uploaded by an attempt that is not sent stay stored in that thread, which no
+     * message references.
+     *
      * It never rejects: a message the client could not send is reported as the error of the chat.
      */
-    send: (content: string) => Promise<void>;
+    send: (content: string, attachments?: ReadonlyArray<AssistantAttachment>, { onSent, onUnsent }?: AssistantSendOptions) => Promise<void>;
+    /**
+     * Lets the user annotate their current view of the configurator, and resolves with what they made,
+     * or with `null` when they cancel, the view cannot be frozen or another configuration is loaded
+     * meanwhile. Nothing happens while another annotation is made or shown: it resolves with `null` at once.
+     *
+     * The layout is inert from the start, and `annotationOverlay$` says what covers it. Once the camera
+     * is still, the view glides from the canvas padding to none, so the product is centred on the whole
+     * configurator area, and the next frame is frozen for the editor. When the editor finishes, the
+     * view glides back to the padding and the layout is released.
+     */
+    annotate: () => Promise<AssistantAnnotation | null>;
+    /**
+     * Shows an annotation of a sent message over the configurator, read-only, until its overlay is
+     * closed. Nothing happens while another annotation is made or shown.
+     */
+    viewAnnotation: (annotation: AssistantMessageAnnotation) => void;
     /**
      * Makes the chat ready for the next message. A thread the server no longer lets this session
      * continue, because the assistant's setup changed since it was started or the thread reached
@@ -243,10 +315,19 @@ export declare class AssistantSystem implements InitializableSystem {
      */
     loadScreenshotUrl: (imageId: string) => Promise<string>;
     /**
-     * Loads the thread before the oldest painted one and paints it above. Does nothing when there is
-     * none or one is being loaded. A load that fails is logged and can be asked for again.
+     * Loads the closed thread named by `earlierThreadId` of the chat, whole, and paints it above the
+     * painted ones; when it is the oldest painted thread itself, closed while only its newest messages
+     * were painted, it replaces that partial copy. Does nothing when there is none, one is being
+     * loaded, or the chat's own thread still has older messages: the earlier threads come above them.
+     * A load that fails is logged and can be asked for again.
      */
     loadEarlierThread: () => Promise<void>;
+    /**
+     * Loads the page of messages before the oldest painted one of the chat's thread and paints it
+     * above. Does nothing when the thread has no older messages or a page is being loaded. A load that
+     * fails is logged and can be asked for again; one that ends after its chat did is dropped.
+     */
+    loadOlderMessages: () => Promise<void>;
     /**
      * Stops the reply on both sides. Closing the stream alone keeps the server run going, because a
      * closed connection is also how a reload looks and a reload must be able to rejoin the run.
@@ -270,14 +351,32 @@ export declare class AssistantSystem implements InitializableSystem {
     undoToolCalls: (toolCallIds: ReadonlySet<string>) => void;
     setOpen: (open: boolean) => void;
     setInputFocused: (isInputFocused: boolean) => void;
+    /** A new configuration load cancels an annotation that is made, and closes one that is shown. */
+    private _closeAnnotationOverlay;
     /**
      * Where the message starts that is being sent on the chat for the first time and that the server
      * refused because its thread cannot be continued, so it never reached the thread; `-1` when there
      * is no such message. A refused continuation of a reply is not one: its message is stored.
      */
     private _findRefusedMessageIndex;
+    /** The content of a user message: its text, when it has any, and a reference to each of its images. */
+    private _messageContent;
+    /**
+     * Uploads the images of a message into the chat's thread, in their order, and resolves with their
+     * ids. Resolves with `'refused'` when the server refused an upload because the thread cannot be
+     * continued, and marks the chat so; then with `null` for any other failure, which is reported as
+     * the error of the chat. It never rejects.
+     */
+    private _uploadImages;
+    /**
+     * Uploads the images of a message into the thread that was resumed for it. A message the chat
+     * refused shows nowhere meanwhile, so the chat stays resuming until the upload is over: nothing
+     * else is sent ahead of it.
+     */
+    private _uploadSuccessorImages;
+    private _reportUploadFailure;
     private _sendOn;
-    private _resume;
+    private _compact;
     /**
      * The `apply` call whose committed state is the step the configuration is at. Nothing at the bottom
      * of the history, where nothing can be undone: that step was not made by a change it could take back.

package/dist/configurator/application-runtime/extensions/systems/assistant-system/assistant-system.js

@@ -1 +1 @@
-import{ChatClient as e,fetchServerSentEvents as t}from"@tanstack/ai-client";import{BehaviorSubject as s}from"rxjs";import{skip as i}from"rxjs/operators";import{isShallowEqual as a}from"@salsita-ai/common/utils";import{AssistantRequestError as r}from"#package/client";import{createBrowserDetailsClientTool as n}from"./browser-details.js";import{ChangeRecorder as o}from"./change-recorder.js";import{createApplyClientTool as l,createQueryClientTool as h,toToolResult as d}from"./graphql-tools.js";let c=Object.freeze({messages:Object.freeze([]),isLoading:!1,error:void 0,failedStatus:void 0,isContinuable:!0,isLoaded:!1,isResuming:!1,earlierThreads:Object.freeze([]),hasEarlierThread:!1,isLoadingEarlierThread:!1}),u=Object.freeze({open:!1,hasUnreadReply:!1,isInputFocused:!1}),_=Object.freeze({}),g=d({results:[],committed:!1,sideEffects:[],warnings:[],screenshots:new Map,message:"Nothing was run, because the user closed the configurator while this reply was being written. Do not call any more tools in this reply and end it now. The tools work again as soon as the user writes the next message."}),f=e=>{if(e.getError())return!0;let t=e.getMessages().at(-1)?.parts.at(-1);return t?.type!=="tool-call"&&t?.type!=="tool-result"};export class AssistantSystem{_context;_tools;_chat=new s(c);_panel=new s(u);_activation=null;_current=null;_finishing=new Set;_olderThreadId=null;_resuming=null;_refused=null;_firstAttempt=null;_chain={};_loadingThreadId=null;_recorder;_recording=null;_applyToolCallIds=new WeakMap;_undoTipToolCallId=new s(void 0);_screenshotUrls=new Map;constructor(e){this._context=e;let{configuratorExtensionSelectors:t,domainContextSelectors:s,domainServicesSelectors:i,getAssistantSetup:a}=e;this._recorder=new o({maxEntities:30},{readers:{selectedEntityId:e=>s.getNavigationSystem().getActiveSelectionState(e),navigationState:e=>s.getNavigationSystem().getActiveNavigationState(e),viewState:e=>s.getNavigationSystem().getActiveViewState(e),toolState:e=>s.getNavigationSystem().getActiveToolState(e)},getExposedFields:()=>a().manifest?.fields??_,resolveLocalizable:e=>i.getResolveLocalizable()(e)});let r={getGraphQLSystem:()=>t.getGraphQLSystem(),getStoreSystem:()=>s.getStoreSystem(),getActiveMode:()=>s.getActiveMode(),getIsViewMode:()=>s.getIsViewMode(),clearChanges:()=>this._recorder.clear(this._committedState()),rememberCommit:(e,t)=>{this._applyToolCallIds.set(t,e),this._refreshUndoTip()}};this._tools=[n(),h(r),l(r)],s.getUndoSystem().tip$.subscribe(this._refreshUndoTip)}chat$=this._chat;panel$=this._panel;undoTipToolCallId$=this._undoTipToolCallId;getSessionManifest=()=>this._context.getAssistantSetup().manifest;initialize=()=>{let{committed:e}=this._context.domainContextSelectors.getStoreSystem().stateSlices;this._recording??=e.pipe(i(1)).subscribe(this._recorder.record),this._recorder.reset(e.value.state),this._endChat(),this._patchPanel(u),this._releaseScreenshotUrls(),this._activation&&this._startClient()};activate=()=>{this._endChat();let e={};return this._activation=e,this._startClient(),()=>{this._activation===e&&(this._activation=null,this._endChat())}};send=async e=>{if(!this._chat.value.isContinuable&&!await this.resume())return;let t=this._current;if(!t)return;let s={changes:this._recorder.take(this._committedState())};this._firstAttempt=t;try{await this._sendOn(t,e,s);let i=this._findRefusedMessageIndex(t);if(-1===i)return;this._refused={chat:t,index:i}}finally{this._firstAttempt===t&&(this._firstAttempt=null)}if(!await this.resume())return;let i=this._current;i&&await this._sendOn(i,e,s)};resume=async()=>{let e=this._current;if(!e)return!1;if(this._chat.value.isContinuable)return!0;if(this._resuming)return this._resuming;let t=this._resume(e).finally(()=>{this._resuming===t&&(this._resuming=null)});return this._resuming=t,t};loadScreenshotUrl=e=>{let t=this._screenshotUrls.get(e);if(t)return t;let s=this._current;if(!s)return Promise.reject(Error(`The screenshot ${e} cannot be read, because the assistant chat is not active. Open the assistant panel of the configuration first.`));let i=this._screenshotUrls,a=s.authorizedClient.getAssistantImage(s.configurationId,e).then(e=>URL.createObjectURL(e));return i.set(e,a),a.catch(()=>{i.get(e)===a&&i.delete(e)}),a};loadEarlierThread=async()=>{let e=this._current,t=this._olderThreadId;if(!e||null===t||this._loadingThreadId===t)return;let s=this._chain;this._loadingThreadId=t,this._chat.next({...this._chat.value,isLoadingEarlierThread:!0});try{let i=await e.authorizedClient.getAssistantEarlierThread(e.configurationId,t,50);if(this._chain!==s||this._olderThreadId!==t)return;this._olderThreadId=i.previousThreadId,this._chat.next({...this._chat.value,earlierThreads:[{threadId:t,messages:i.messages},...this._chat.value.earlierThreads],hasEarlierThread:null!==i.previousThreadId})}catch(e){this._context.domainServicesSelectors.getLogger().warn("Could not load the earlier assistant conversation",{error:e})}finally{this._chain===s&&this._loadingThreadId===t&&(this._loadingThreadId=null,this._chat.next({...this._chat.value,isLoadingEarlierThread:!1}))}};stop=async()=>{let e=this._current;if(!e)return;let t=e.client.getCurrentRunId();if(e.client.stop(),null!=t)try{let s=e.getConnection();await fetch(`${s.url}/runs/${encodeURIComponent(t)}/cancel`,{method:"POST",headers:await s.headers()})}catch(e){this._context.domainServicesSelectors.getLogger().warn("Could not cancel the assistant run on the server",{error:e})}};reloadThread=()=>{if(!this._activation)return;let e=this._retireChat();e?.client.stop(),e?.dispose(),this._recorder.reset(this._committedState()),this._startClient()};undoToolCalls=e=>{let t=this._context.domainContextSelectors.getUndoSystem(),s=()=>{let t=this._findTipToolCallId();return void 0!==t&&e.has(t)};for(;s();)t.undo()};setOpen=e=>{this._patchPanel(e?{open:e,hasUnreadReply:!1}:{open:e,isInputFocused:!1})};setInputFocused=e=>{this._patchPanel({isInputFocused:e})};_findRefusedMessageIndex=e=>{var t;let s,{failedStatus:i,isContinuable:a}=this._chat.value;return this._firstAttempt===e&&e.generation.active&&409===i&&!a?(s=(t=e.client.getMessages()).map(e=>e.role).lastIndexOf("user"),t.slice(s+1).every(e=>0===e.parts.length)?s:-1):-1};_sendOn=async(e,t,s)=>{let{client:i}=e;i.getIsLoading()||null===i.getResumeState()||i.stop();try{await i.sendMessage({content:t,metadata:s})}catch(s){let t=s instanceof Error?s:Error(String(s));e.generation.active&&this._chat.next({...this._chat.value,error:t}),this._context.domainServicesSelectors.getLogger().error("The assistant message could not be sent",{},t)}};_resume=async e=>{this._chat.next({...this._chat.value,isResuming:!0,error:void 0,failedStatus:void 0});try{let{previousThreadId:t}=await e.authorizedClient.compactAssistantThread(e.configurationId);if(!e.generation.active)return!1;let{messages:s,earlierThreads:i,hasEarlierThread:a}=this._chat.value,r=this._refused?.chat===e?s.slice(0,this._refused.index):s;return e.generation.active=!1,this._current=null,e.client.stop(),e.dispose(),this._chat.next({...c,earlierThreads:[...i,{threadId:t,messages:r}],hasEarlierThread:a,isLoadingEarlierThread:null!==this._loadingThreadId}),this._startClient(),!0}catch(s){let t=s instanceof Error?s:Error(String(s));if(t instanceof r&&409===t.status)return e.generation.active&&this.reloadThread(),!1;return e.generation.active&&this._chat.next({...this._chat.value,isResuming:!1,error:t,failedStatus:t instanceof r?t.status:void 0}),this._context.domainServicesSelectors.getLogger().error("The assistant thread could not be resumed",{},t),!1}};_findTipToolCallId=()=>{let e=this._context.domainContextSelectors.getUndoSystem(),t=e.tip$.value?.committedState;return void 0!==t&&e.canUndo()?this._applyToolCallIds.get(t):void 0};_refreshUndoTip=()=>{let e=this._findTipToolCallId();e!==this._undoTipToolCallId.value&&this._undoTipToolCallId.next(e)};_committedState=()=>this._context.domainContextSelectors.getStoreSystem().stateSlices.committed.value.state;_patchPanel=e=>{let t={...this._panel.value,...e};a(t,this._panel.value)||this._panel.next(t)};_takeOverThread=e=>{for(let t of this._finishing)t.configurationId===e&&(t.client.stop(),t.dispose())};_startClient=()=>{let s={active:!0},i=this._context.getConfigurationId();this._takeOverThread(i);let r=this._context.getAuthorizedClient(),n=()=>{if(s.active){let{manifest:e,error:t}=this._context.getAssistantSetup();if(t)throw t;if(!e)throw Error("The assistant is disabled for this session.")}return r.getAssistantChatConnection(i)},o=e=>{if(!s.active)return;let t={...this._chat.value,...e};a(t,this._chat.value)||this._chat.next(t)},l=async(e,t)=>{let i=await fetch(e,t).finally(()=>o({isLoaded:!0})),a=i.headers.get("x-assistant-thread-continuable");return null!=a&&i.ok&&s.active&&0===this._chat.value.earlierThreads.length&&(this._olderThreadId=i.headers.get("x-assistant-previous-thread")),o({failedStatus:i.ok?void 0:i.status,...null!=a&&{isContinuable:"false"!==a,hasEarlierThread:null!==this._olderThreadId}}),i},h=async()=>({headers:await n().headers(),fetchClient:l}),d=t(()=>n().url,h),c=t(()=>`${n().url}/runs`,h),u=this._tools.map(e=>({...e,execute:(t,i)=>{if(!s.active)return g;let{manifest:a,error:r}=this._context.getAssistantSetup();return r||!a?{error:r?.message??"The assistant is disabled for this session."}:e.execute?.(t,i)}})),_=new e({threadId:`assistant:${i}`,connection:{...d,joinRun:c.joinRun},persistence:!0,history:{pageSize:50},tools:u,onMessagesChange:e=>o({messages:e}),onLoadingChange:e=>{let t=s.active&&this._chat.value.isLoading;o({isLoading:e}),!e&&f(_)&&(s.active?t&&!this._panel.value.open&&this._patchPanel({hasUnreadReply:!0}):p())},onErrorChange:e=>{s.active&&e!==this._chat.value.error&&(!e||-1===this._findRefusedMessageIndex(v))&&(o({error:e}),e&&this._context.domainServicesSelectors.getLogger().error("The assistant request failed",{status:this._chat.value.failedStatus},e))}}),m=!1,p=()=>{m||(m=!0,this._finishing.delete(v),_.dispose())},v={client:_,configurationId:i,generation:s,authorizedClient:r,getConnection:n,dispose:p};this._current=v,_.attach()};_retireChat=()=>{let e=this._current;return this._current=null,e&&(e.generation.active=!1),this._chain={},this._olderThreadId=null,this._resuming=null,this._refused=null,this._loadingThreadId=null,this._chat.value!==c&&this._chat.next(c),e};_releaseScreenshotUrls=()=>{for(let e of this._screenshotUrls.values())e.then(URL.revokeObjectURL,()=>void 0);this._screenshotUrls=new Map};_endChat=()=>{let e=this._retireChat();if(e){if(!e.client.getIsLoading()&&f(e.client))return void e.dispose();this._finishing.add(e)}}}
+import{ChatClient as t,fetchServerSentEvents as e}from"@tanstack/ai-client";import{BehaviorSubject as a,firstValueFrom as i,race as s,timer as n}from"rxjs";import{filter as r,skip as o}from"rxjs/operators";import{isShallowEqual as l}from"@salsita-ai/common/utils";import{assistantImageReference as h}from"@salsita-ai/contracts/assistant";import{AssistantRequestError as c}from"#package/client";import{ChangeRecorder as d}from"./change-recorder.js";import{createApplyClientTool as u,createQueryClientTool as _,toToolResult as g}from"./graphql-tools.js";let f=Object.freeze({messages:Object.freeze([]),isLoading:!1,error:void 0,failedStatus:void 0,retryAfterSeconds:void 0,isContinuable:!0,isLoaded:!1,isResuming:!1,earlierThreads:Object.freeze([]),earlierThreadId:null,isLoadingEarlierThread:!1,hasOlderMessages:!1,isLoadingOlderMessages:!1}),v=Object.freeze({top:0,right:0,bottom:0,left:0}),m=Object.freeze({kind:"transition"}),p=Object.freeze({open:!1,hasUnreadReply:!1,isInputFocused:!1}),S=Object.freeze({}),y=g({results:[],committed:!1,sideEffects:[],warnings:[],screenshots:new Map,message:"Nothing was run, because the user closed the configurator while this reply was being written. Do not call any more tools in this reply and end it now. The tools work again as soon as the user writes the next message."}),x=t=>{if(t.getError())return!0;let e=t.getMessages().at(-1)?.parts.at(-1);return e?.type!=="tool-call"&&e?.type!=="tool-result"},w=async t=>{try{let e=await t.clone().json(),a=e?.page;return a?.truncated===!0&&"string"==typeof a.cursor&&""!==a.cursor}catch{return!1}};export class AssistantSystem{_context;_tools;_chat=new a(f);_panel=new a(p);_annotationOverlay=new a(null);_annotationRun=null;_activation=null;_current=null;_finishing=new Set;_resuming=null;_refused=null;_firstAttempt=null;_chain={};_recorder;_recording=null;_applyToolCallIds=new WeakMap;_undoTipToolCallId=new a(void 0);_screenshotUrls=new Map;constructor(t){this._context=t;let{configuratorExtensionSelectors:e,domainContextSelectors:a,domainServicesSelectors:i,getAssistantSetup:s}=t;this._recorder=new d({maxEntities:30},{readers:{selectedEntityId:t=>a.getNavigationSystem().getActiveSelectionState(t),navigationState:t=>a.getNavigationSystem().getActiveNavigationState(t),viewState:t=>a.getNavigationSystem().getActiveViewState(t),toolState:t=>a.getNavigationSystem().getActiveToolState(t)},getExposedFields:()=>s().manifest?.fields??S,resolveLocalizable:t=>i.getResolveLocalizable()(t)});let n={getGraphQLSystem:()=>e.getGraphQLSystem(),getStoreSystem:()=>a.getStoreSystem(),getActiveMode:()=>a.getActiveMode(),getIsViewMode:()=>a.getIsViewMode(),clearChanges:()=>this._recorder.clear(this._committedState()),rememberCommit:(t,e)=>{this._applyToolCallIds.set(e,t),this._refreshUndoTip()}};this._tools=[_(n),u(n)],a.getUndoSystem().tip$.subscribe(this._refreshUndoTip)}chat$=this._chat;panel$=this._panel;annotationOverlay$=this._annotationOverlay;undoTipToolCallId$=this._undoTipToolCallId;getSessionManifest=()=>this._context.getAssistantSetup().manifest;initialize=()=>{let{committed:t}=this._context.domainContextSelectors.getStoreSystem().stateSlices;this._recording??=t.pipe(o(1)).subscribe(this._recorder.record),this._recorder.reset(t.value.state),this._endChat(),this._patchPanel(p),this._releaseScreenshotUrls(),this._closeAnnotationOverlay(),this._activation&&this._startClient()};activate=()=>{this._endChat();let t={};return this._activation=t,this._startClient(),()=>{this._activation===t&&(this._activation=null,this._endChat())}};send=async(t,e=[],{onSent:a,onUnsent:i}={})=>{if(!this._chat.value.isContinuable&&!await this.resume())return;let s=this._current;if(!s)return;let n={changes:this._recorder.take(this._committedState()),locale:this._context.domainServicesSelectors.getI18n().locale},r=()=>this._recorder.reset(this._committedState()),o=e.map(t=>t instanceof Blob?t:t.blob),l=!1,h=(i,s)=>{l||(l=!0,a?.());let r=e.flatMap((t,e)=>{let a=s[e];return t instanceof Blob||void 0===a?[]:[{imageId:a,pins:t.pins}]});return this._sendOn(i,this._messageContent(t,s),0===r.length?n:{...n,annotations:r})};this._firstAttempt=s;try{let t=0===o.length?[]:await this._uploadImages(s,o);if(null===t||!s.generation.active)return void r();if("refused"!==t){await h(s,t);let e=this._findRefusedMessageIndex(s);if(-1===e)return;this._refused={chat:s,index:e}}}finally{this._firstAttempt===s&&(this._firstAttempt=null)}if(!await this.resume()){l||r();return}let c=this._current,d=c?await this._uploadSuccessorImages(c,o):null;if(!c||null===d||"refused"===d||!c.generation.active){c&&"refused"===d&&this._reportUploadFailure(c,this._chat.value.error),l&&i?.(),r();return}await h(c,d)};annotate=async()=>{if(null!==this._annotationOverlay.value)return null;let{getCameraControlSystem:t,getCanvasSystem:e}=this._context.configuratorExtensionSelectors,a=t(),o=e().r3fHandle$.value?.get().frameloop==="demand",l={},h=()=>this._annotationRun===l;if(this._annotationRun=l,this._annotationOverlay.next(m),await i(s(a.transitionState$.pipe(r(t=>"still"===t||"sleep"===t&&!o)),n(2e3))),!h())return this._annotationOverlay.next(null),null;let c=a.holdCameraOffset(),d=null;try{await a.glideCameraOffset(v);let t=await this._context.freezeView();h()&&(d=await new Promise(e=>{this._annotationOverlay.next({kind:"editor",view:t,finish:e})}))}catch(e){let t=e instanceof Error?e:Error(String(e));this._context.domainServicesSelectors.getLogger().error("The configurator view could not be frozen for an annotation",{},t)}this._annotationOverlay.next(m);try{await a.glideCameraOffset(a.canvasPadding$.value)}finally{c(),this._annotationOverlay.next(null),h()&&(this._annotationRun=null)}return d};viewAnnotation=t=>{if(null!==this._annotationOverlay.value)return;let e={kind:"viewer",annotation:t,close:()=>{this._annotationOverlay.value===e&&this._annotationOverlay.next(null)}};this._annotationOverlay.next(e)};resume=async()=>{let t=this._current;if(!t)return!1;if(this._chat.value.isContinuable)return!0;if(this._resuming)return this._resuming;let e=this._compact(t).finally(()=>{this._resuming===e&&(this._resuming=null)});return this._resuming=e,e};loadScreenshotUrl=t=>{let e=this._screenshotUrls.get(t);if(e)return e;let a=this._current;if(!a)return Promise.reject(Error(`The screenshot ${t} cannot be read, because the assistant chat is not active. Open the assistant panel of the configuration first.`));let i=this._screenshotUrls,s=a.authorizedClient.getAssistantImage(a.configurationId,t).then(t=>URL.createObjectURL(t));return i.set(t,s),s.catch(()=>{i.get(t)===s&&i.delete(t)}),s};loadEarlierThread=async()=>{let t=this._current,{earlierThreadId:e,isLoadingEarlierThread:a,hasOlderMessages:i}=this._chat.value;if(!t||null===e||a||i)return;let s=this._chain;this._chat.next({...this._chat.value,isLoadingEarlierThread:!0});try{let a=await t.authorizedClient.getAssistantEarlierThread(t.configurationId,e);if(this._chain!==s||this._chat.value.earlierThreadId!==e)return;let[i,...n]=this._chat.value.earlierThreads,r={threadId:e,messages:a.messages};this._chat.next({...this._chat.value,earlierThreads:i?.threadId===e?[r,...n]:[r,...this._chat.value.earlierThreads],earlierThreadId:a.previousThreadId})}catch(t){this._context.domainServicesSelectors.getLogger().warn("Could not load the earlier assistant conversation",{error:t})}finally{this._chain===s&&this._chat.next({...this._chat.value,isLoadingEarlierThread:!1})}};loadOlderMessages=async()=>{let t=this._current,{hasOlderMessages:e,isLoadingOlderMessages:a}=this._chat.value;if(t&&e&&!a){this._chat.next({...this._chat.value,isLoadingOlderMessages:!0});try{await t.client.loadOlderMessages()}catch(t){this._context.domainServicesSelectors.getLogger().warn("Could not load older assistant messages",{error:t})}finally{t===this._current&&this._chat.next({...this._chat.value,hasOlderMessages:t.client.getHasOlderMessages(),isLoadingOlderMessages:!1})}}};stop=async()=>{let t=this._current;if(!t)return;let e=t.client.getCurrentRunId();if(t.client.stop(),null!=e)try{let a=t.getConnection();await fetch(`${a.url}/runs/${encodeURIComponent(e)}/cancel`,{method:"POST",headers:await a.headers()})}catch(t){this._context.domainServicesSelectors.getLogger().warn("Could not cancel the assistant run on the server",{error:t})}};reloadThread=()=>{if(!this._activation)return;let t=this._retireChat();t?.client.stop(),t?.dispose(),this._recorder.reset(this._committedState()),this._startClient()};undoToolCalls=t=>{let e=this._context.domainContextSelectors.getUndoSystem(),a=()=>{let e=this._findTipToolCallId();return void 0!==e&&t.has(e)};for(;a();)e.undo()};setOpen=t=>{this._patchPanel(t?{open:t,hasUnreadReply:!1}:{open:t,isInputFocused:!1})};setInputFocused=t=>{this._patchPanel({isInputFocused:t})};_closeAnnotationOverlay=()=>{let t=this._annotationOverlay.value;this._annotationRun=null,t?.kind==="editor"&&t.finish(null),t?.kind==="viewer"&&t.close()};_findRefusedMessageIndex=t=>{var e;let a,{failedStatus:i,isContinuable:s}=this._chat.value;return this._firstAttempt===t&&t.generation.active&&409===i&&!s?(a=(e=t.client.getMessages()).map(t=>t.role).lastIndexOf("user"),e.slice(a+1).every(t=>0===t.parts.length)?a:-1):-1};_messageContent=(t,e)=>0===e.length?t:[...t?[{type:"text",content:t}]:[],...e.map(t=>({type:"image",source:{type:"url",value:h(t)}}))];_uploadImages=async(t,e)=>{if(0===e.length)return[];t.generation.active&&this._chat.next({...this._chat.value,error:void 0,failedStatus:void 0});try{return(await Promise.all(e.map(e=>t.authorizedClient.uploadAssistantImage(t.configurationId,e)))).map(({imageId:t})=>t)}catch(a){let e=a instanceof Error?a:Error(String(a));if(e instanceof c&&e.threadNotContinuable)return t.generation.active&&this._chat.next({...this._chat.value,isContinuable:!1,error:e,failedStatus:e.status}),"refused";return this._reportUploadFailure(t,e),null}};_uploadSuccessorImages=async(t,e)=>{if(0===e.length)return[];this._chat.next({...this._chat.value,isResuming:!0});try{return await this._uploadImages(t,e)}finally{t.generation.active&&this._chat.next({...this._chat.value,isResuming:!1})}};_reportUploadFailure=(t,e)=>{t.generation.active&&this._chat.next({...this._chat.value,error:e,failedStatus:e instanceof c?e.status:void 0}),this._context.domainServicesSelectors.getLogger().error("The images of the assistant message could not be uploaded",{},e)};_sendOn=async(t,e,a)=>{let{client:i}=t;i.getIsLoading()||null===i.getResumeState()||i.stop();try{await i.sendMessage({content:e,metadata:a})}catch(a){let e=a instanceof Error?a:Error(String(a));t.generation.active&&this._chat.next({...this._chat.value,error:e}),this._context.domainServicesSelectors.getLogger().error("The assistant message could not be sent",{},e)}};_compact=async t=>{this._chat.next({...this._chat.value,isResuming:!0,error:void 0,failedStatus:void 0});try{let{previousThreadId:e}=await t.authorizedClient.compactAssistantThread(t.configurationId);if(!t.generation.active)return!1;let{messages:a,earlierThreads:i,earlierThreadId:s,isLoadingEarlierThread:n}=this._chat.value,r=this._refused?.chat===t?a.slice(0,this._refused.index):a,o=t.client.getHasOlderMessages()?e:s;return t.generation.active=!1,this._current=null,t.client.stop(),t.dispose(),this._chat.next({...f,earlierThreads:[...i,{threadId:e,messages:r}],earlierThreadId:o,isLoadingEarlierThread:n}),this._startClient(),!0}catch(a){let e=a instanceof Error?a:Error(String(a));if(e instanceof c&&409===e.status)return t.generation.active&&this.reloadThread(),!1;return t.generation.active&&this._chat.next({...this._chat.value,isResuming:!1,error:e,failedStatus:e instanceof c?e.status:void 0}),this._context.domainServicesSelectors.getLogger().error("The assistant thread could not be resumed",{},e),!1}};_findTipToolCallId=()=>{let t=this._context.domainContextSelectors.getUndoSystem(),e=t.tip$.value?.committedState;return void 0!==e&&t.canUndo()?this._applyToolCallIds.get(e):void 0};_refreshUndoTip=()=>{let t=this._findTipToolCallId();t!==this._undoTipToolCallId.value&&this._undoTipToolCallId.next(t)};_committedState=()=>this._context.domainContextSelectors.getStoreSystem().stateSlices.committed.value.state;_patchPanel=t=>{let e={...this._panel.value,...t};l(e,this._panel.value)||this._panel.next(e)};_takeOverThread=t=>{for(let e of this._finishing)e.configurationId===t&&(e.client.stop(),e.dispose())};_startClient=()=>{let a={active:!0},i=this._context.getConfigurationId();this._takeOverThread(i);let s=this._context.getAuthorizedClient(),n=()=>{if(a.active){let{manifest:t,error:e}=this._context.getAssistantSetup();if(e)throw e;if(!t)throw Error("The assistant is disabled for this session.")}return s.getAssistantChatConnection(i)},r=t=>{if(!a.active)return;let e={...this._chat.value,...t};l(e,this._chat.value)||this._chat.next(e)},o=async(t,e)=>{let a,i=await fetch(t,e).finally(()=>r({isLoaded:!0})),s=i.headers.get("x-assistant-thread-continuable"),n=null!=s&&i.ok&&0===this._chat.value.earlierThreads.length,o=n?i.headers.get("x-assistant-previous-thread"):null;return r({failedStatus:i.ok?void 0:i.status,retryAfterSeconds:i.ok?void 0:null!==(a=i.headers.get("retry-after"))&&/^\d+$/.test(a.trim())?Number(a):void 0,...null!=s&&{isContinuable:"false"!==s},...n&&{earlierThreadId:o,...null!==o&&{hasOlderMessages:await w(i)}}}),i},h=async()=>({headers:await n().headers(),fetchClient:o}),c=e(()=>n().url,h),d=e(()=>`${n().url}/runs`,h),u=this._tools.map(t=>({...t,execute:(e,i)=>{if(!a.active)return y;let{manifest:s,error:n}=this._context.getAssistantSetup();return n||!s?{error:n?.message??"The assistant is disabled for this session."}:t.execute?.(e,i)}})),_=new t({threadId:`assistant:${i}`,connection:{...c,joinRun:d.joinRun},persistence:!0,history:{pageSize:50},tools:u,onMessagesChange:t=>r({messages:t,hasOlderMessages:_.getHasOlderMessages()}),onLoadingChange:t=>{let e=a.active&&this._chat.value.isLoading;r({isLoading:t}),!t&&x(_)&&(a.active?e&&!this._panel.value.open&&this._patchPanel({hasUnreadReply:!0}):f())},onErrorChange:t=>{a.active&&t!==this._chat.value.error&&(!t||-1===this._findRefusedMessageIndex(v))&&(r({error:t}),t&&this._context.domainServicesSelectors.getLogger().error("The assistant request failed",{status:this._chat.value.failedStatus},t))}}),g=!1,f=()=>{g||(g=!0,this._finishing.delete(v),_.dispose())},v={client:_,configurationId:i,generation:a,authorizedClient:s,getConnection:n,dispose:f};this._current=v,_.attach()};_retireChat=()=>{let t=this._current;return this._current=null,t&&(t.generation.active=!1),this._chain={},this._resuming=null,this._refused=null,this._chat.value!==f&&this._chat.next(f),t};_releaseScreenshotUrls=()=>{for(let t of this._screenshotUrls.values())t.then(URL.revokeObjectURL,()=>void 0);this._screenshotUrls=new Map};_endChat=()=>{let t=this._retireChat();if(t){if(!t.client.getIsLoading()&&x(t.client))return void t.dispose();this._finishing.add(t)}}}

package/dist/configurator/application-runtime/extensions/systems/camera-control-system/camera-control-system.d.ts

@@ -65,6 +65,10 @@ export declare class CameraControlSystem {
      */
     private readonly _canvasPadding$;
     readonly canvasPadding$: ObservableWithLastValue<CanvasPadding>;
+    /** The padding the interactive cameras were last offset by, through `applyCameraOffset`. */
+    private _appliedCameraOffset;
+    /** How many holders keep CameraPadding from writing the offset; see `holdCameraOffset`. */
+    private _cameraOffsetHolds;
     private _generation;
     private _pendingAbortController;
     private _lastTransition;
@@ -118,6 +122,45 @@ export declare class CameraControlSystem {
     readonly fitRootCamera: () => void;
     readonly fitEntity: (id: DomainEntityId<string>) => void;
     readonly setCameraOffset: (props: CameraOffsetProps) => void;
+    /**
+     * The padding the interactive cameras were last offset by; a glide starts from it.
+     *
+     * @internal
+     */
+    get appliedCameraOffset(): CanvasPadding;
+    /**
+     * True while someone else than CameraPadding owns the offset of the interactive cameras.
+     *
+     * @internal
+     */
+    get isCameraOffsetHeld(): boolean;
+    /**
+     * Takes the offset of the interactive cameras over from CameraPadding, which stops writing it until
+     * every hold is released, so the holder can move the view without a refit (see
+     * `glideCameraOffset`). Releasing wakes the frameloop, so CameraPadding glides on from wherever the
+     * holder left the offset. The returned release does nothing when called again.
+     *
+     * @internal
+     */
+    readonly holdCameraOffset: () => (() => void);
+    /**
+     * Offsets both interactive cameras by `padding` against the measured canvas viewport. Nothing
+     * happens while no canvas has been measured: there is no viewport to offset against.
+     *
+     * @internal
+     */
+    readonly applyCameraOffset: (padding: CanvasPadding) => void;
+    /**
+     * Moves the offset of the interactive cameras from the applied padding to `to` in a fixed time, a
+     * frame at a time, and resolves once it is there. The offset is a function of the elapsed time, so
+     * a slow device shows fewer frames but takes no longer. With reduced motion it is set at once.
+     * Meant for the holder of `holdCameraOffset`; CameraPadding would otherwise overwrite every step.
+     *
+     * @internal
+     */
+    readonly glideCameraOffset: (to: CanvasPadding) => Promise<void>;
+    /** Wakes the demand frameloop of the mounted canvas, so an offset written outside a frame is drawn. */
+    private readonly _invalidate;
     initialize: () => void;
 }
 export {};

package/dist/configurator/application-runtime/extensions/systems/camera-control-system/camera-control-system.js

@@ -1 +1 @@
-import t from"promise-queue";import{BehaviorSubject as e,Subject as r,debounceTime as a,map as i,of as o,shareReplay as s,startWith as n,switchMap as h}from"rxjs";import{OrthographicCamera as l,PerspectiveCamera as p}from"three";import{match as c}from"ts-pattern";import{DEFAULT_DIMENSION_UNIT as C,distanceUnitToDefault as m}from"#package/domain";import{createCameraControls as g}from"./create-camera-controls.js";export const CameraControlType={Perspective:"perspective",Orthographic:"orthographic"};let d=t=>Number.isFinite(t.width)&&Number.isFinite(t.height)&&t.width>0&&t.height>0;export class CameraControlSystem{_context;perspectiveCamera=Object.assign(new p,{manual:!0});orthographicCamera=Object.assign(new l,{manual:!0});perspectiveCameraControls=g(this.perspectiveCamera);orthographicCameraControls=g(this.orthographicCamera);_transitionState$=new e("still");_rendered$=new r;_transitionQueue=new t(1,1/0);_activeCameraControls$=new e(void 0);activeCameraControls$=this._activeCameraControls$;_canvasPadding$=new e({top:0,right:0,bottom:0,left:0});canvasPadding$=this._canvasPadding$;_generation=null;_pendingAbortController=void 0;_lastTransition=void 0;_cameraControlState$=new e("idle");cameraControlState$=this._cameraControlState$;transitionState$=this._transitionState$.pipe(h(t=>"sleep"===t?this._rendered$.pipe(a(200),i(()=>"still"),n("sleep")):o(t)),s(1));constructor(t){this._context=t,this.perspectiveCamera.fov=50,this.perspectiveCamera.near=m(500,C),this.perspectiveCamera.far=m(4e4,C),this.perspectiveCamera.position.set(0,1,0),this.orthographicCamera.left=m(-500,C),this.orthographicCamera.bottom=m(-500,C),this.orthographicCamera.right=m(500,C),this.orthographicCamera.top=m(500,C),this.orthographicCamera.far=m(4e4,C),this.orthographicCamera.near=m(500,C),this.orthographicCamera.position.set(0,1,0),this._context.configuratorExtensionSelectors.getCanvasSystem().interactiveViewport$.subscribe(t=>{null!==t&&this.handleCanvasResize(t)})}_seedCameraControls=()=>{this.perspectiveCameraControls.enabled=!0,this.orthographicCameraControls.enabled=!0,this.perspectiveCameraControls.rotateTo(0,1.2,!1),this.orthographicCameraControls.rotateTo(0,1.2,!1),this.orthographicCameraControls.dollyTo(m(1e4,C),!1)};resetCamera=()=>{this._lastTransition&&this.scheduleCameraTransition(this._lastTransition,{bypassViewLocks:!0})};setTransitionState=t=>{this._transitionState$.next(t)};reportRender=()=>{this._rendered$.next()};scheduleCameraTransition(t,e={}){this._pendingAbortController?.abort();let r=new AbortController;this._pendingAbortController=r;let a=this._generation;return{finished:this._transitionQueue.add(async()=>{let i=r.signal;if(i.aborted||!a?.active)return;let o=c(t.cameraType).with(CameraControlType.Perspective,()=>this.perspectiveCameraControls).with(CameraControlType.Orthographic,()=>this.orthographicCameraControls).exhaustive(),s=null==this._activeCameraControls$.value,n=this._activeCameraControls$.value!==o;n&&this._activeCameraControls$.next(o),this._lastTransition=t,await t.run({controls:o,camera:o.camera,abortSignal:i,firstLoad:s,typeChanged:n,bypassViewLocks:!0===e.bypassViewLocks})}),abort:()=>{r.abort()}}}setCameraControlState=t=>{this._cameraControlState$.next(t)};handleCanvasResize=t=>{if(!d(t))return;let e=t.width/t.height;this.perspectiveCamera.aspect=e,this.perspectiveCamera.updateProjectionMatrix();let r=this.orthographicCamera,a=(r.left+r.right)/2,i=(r.top-r.bottom)/2*e;r.left=a-i,r.right=a+i,r.updateProjectionMatrix()};setCanvasPadding=t=>{let e=this._canvasPadding$.value;(e.top!==t.top||e.right!==t.right||e.bottom!==t.bottom||e.left!==t.left)&&this._canvasPadding$.next(t)};enable=()=>{this.perspectiveCameraControls.enabled=!0,this.orthographicCameraControls.enabled=!0};disable=()=>{this.perspectiveCameraControls.enabled=!1,this.orthographicCameraControls.enabled=!1};fitRootCamera=()=>{let t=this._context.configuratorExtensionSelectors.getCanvasSystem().ready$.value?.scene;t&&this.perspectiveCameraControls.fitToSphere(t,!1)};fitEntity=t=>{let e=this._context.configuratorExtensionSelectors.getCanvasSystem().ready$.value?.scene;if(!e)return;let r=e.getObjectByName(t.toString());null!=r&&this.perspectiveCameraControls.fitToBox(r,!1)};setCameraOffset=t=>{let{padding:e,viewPort:r,camera:a}=t;if(!d(r))return;if(![e.top,e.right,e.bottom,e.left].every(Number.isFinite)||e.left+e.right>=r.width||e.top+e.bottom>=r.height)return void a.clearViewOffset();let i=(e.right-e.left)/2,o=(e.bottom-e.top)/2;a.setViewOffset(r.width,r.height,i,o,r.width,r.height)};initialize=()=>{null!==this._generation&&(this._generation.active=!1),this._pendingAbortController?.abort(),this._pendingAbortController=void 0,this._lastTransition=void 0,this._transitionState$.next("still"),this._cameraControlState$.next("idle"),this._activeCameraControls$.next(void 0),this._generation={active:!0},this._seedCameraControls()}}
+import t from"promise-queue";import{BehaviorSubject as e,Subject as a,debounceTime as r,map as i,of as o,shareReplay as s,startWith as n,switchMap as h}from"rxjs";import{OrthographicCamera as l,PerspectiveCamera as m}from"three";import{match as p}from"ts-pattern";import{DEFAULT_DIMENSION_UNIT as c,distanceUnitToDefault as C}from"#package/domain";import{createCameraControls as d}from"./create-camera-controls.js";export const CameraControlType={Perspective:"perspective",Orthographic:"orthographic"};let f=Object.freeze({top:0,right:0,bottom:0,left:0}),g=t=>Number.isFinite(t.width)&&Number.isFinite(t.height)&&t.width>0&&t.height>0;export class CameraControlSystem{_context;perspectiveCamera=Object.assign(new m,{manual:!0});orthographicCamera=Object.assign(new l,{manual:!0});perspectiveCameraControls=d(this.perspectiveCamera);orthographicCameraControls=d(this.orthographicCamera);_transitionState$=new e("still");_rendered$=new a;_transitionQueue=new t(1,1/0);_activeCameraControls$=new e(void 0);activeCameraControls$=this._activeCameraControls$;_canvasPadding$=new e({top:0,right:0,bottom:0,left:0});canvasPadding$=this._canvasPadding$;_appliedCameraOffset=f;_cameraOffsetHolds=0;_generation=null;_pendingAbortController=void 0;_lastTransition=void 0;_cameraControlState$=new e("idle");cameraControlState$=this._cameraControlState$;transitionState$=this._transitionState$.pipe(h(t=>"sleep"===t?this._rendered$.pipe(r(200),i(()=>"still"),n("sleep")):o(t)),s(1));constructor(t){this._context=t,this.perspectiveCamera.fov=50,this.perspectiveCamera.near=C(500,c),this.perspectiveCamera.far=C(4e4,c),this.perspectiveCamera.position.set(0,1,0),this.orthographicCamera.left=C(-500,c),this.orthographicCamera.bottom=C(-500,c),this.orthographicCamera.right=C(500,c),this.orthographicCamera.top=C(500,c),this.orthographicCamera.far=C(4e4,c),this.orthographicCamera.near=C(500,c),this.orthographicCamera.position.set(0,1,0),this._context.configuratorExtensionSelectors.getCanvasSystem().interactiveViewport$.subscribe(t=>{null!==t&&this.handleCanvasResize(t)})}_seedCameraControls=()=>{this.perspectiveCameraControls.enabled=!0,this.orthographicCameraControls.enabled=!0,this.perspectiveCameraControls.rotateTo(0,1.2,!1),this.orthographicCameraControls.rotateTo(0,1.2,!1),this.orthographicCameraControls.dollyTo(C(1e4,c),!1)};resetCamera=()=>{this._lastTransition&&this.scheduleCameraTransition(this._lastTransition,{bypassViewLocks:!0})};setTransitionState=t=>{this._transitionState$.next(t)};reportRender=()=>{this._rendered$.next()};scheduleCameraTransition(t,e={}){this._pendingAbortController?.abort();let a=new AbortController;this._pendingAbortController=a;let r=this._generation;return{finished:this._transitionQueue.add(async()=>{let i=a.signal;if(i.aborted||!r?.active)return;let o=p(t.cameraType).with(CameraControlType.Perspective,()=>this.perspectiveCameraControls).with(CameraControlType.Orthographic,()=>this.orthographicCameraControls).exhaustive(),s=null==this._activeCameraControls$.value,n=this._activeCameraControls$.value!==o;n&&this._activeCameraControls$.next(o),this._lastTransition=t,await t.run({controls:o,camera:o.camera,abortSignal:i,firstLoad:s,typeChanged:n,bypassViewLocks:!0===e.bypassViewLocks})}),abort:()=>{a.abort()}}}setCameraControlState=t=>{this._cameraControlState$.next(t)};handleCanvasResize=t=>{if(!g(t))return;let e=t.width/t.height;this.perspectiveCamera.aspect=e,this.perspectiveCamera.updateProjectionMatrix();let a=this.orthographicCamera,r=(a.left+a.right)/2,i=(a.top-a.bottom)/2*e;a.left=r-i,a.right=r+i,a.updateProjectionMatrix()};setCanvasPadding=t=>{let e=this._canvasPadding$.value;(e.top!==t.top||e.right!==t.right||e.bottom!==t.bottom||e.left!==t.left)&&this._canvasPadding$.next(t)};enable=()=>{this.perspectiveCameraControls.enabled=!0,this.orthographicCameraControls.enabled=!0};disable=()=>{this.perspectiveCameraControls.enabled=!1,this.orthographicCameraControls.enabled=!1};fitRootCamera=()=>{let t=this._context.configuratorExtensionSelectors.getCanvasSystem().ready$.value?.scene;t&&this.perspectiveCameraControls.fitToSphere(t,!1)};fitEntity=t=>{let e=this._context.configuratorExtensionSelectors.getCanvasSystem().ready$.value?.scene;if(!e)return;let a=e.getObjectByName(t.toString());null!=a&&this.perspectiveCameraControls.fitToBox(a,!1)};setCameraOffset=t=>{let{padding:e,viewPort:a,camera:r}=t;if(!g(a))return;if(![e.top,e.right,e.bottom,e.left].every(Number.isFinite)||e.left+e.right>=a.width||e.top+e.bottom>=a.height)return void r.clearViewOffset();let i=(e.right-e.left)/2,o=(e.bottom-e.top)/2;r.setViewOffset(a.width,a.height,i,o,a.width,a.height)};get appliedCameraOffset(){return this._appliedCameraOffset}get isCameraOffsetHeld(){return this._cameraOffsetHolds>0}holdCameraOffset=()=>{this._cameraOffsetHolds+=1;let t=!1;return()=>{t||(t=!0,this._cameraOffsetHolds-=1,this._invalidate())}};applyCameraOffset=t=>{let e=this._context.configuratorExtensionSelectors.getCanvasSystem().interactiveViewport$.value;null!==e&&(this.setCameraOffset({padding:t,viewPort:e,camera:this.perspectiveCamera}),this.setCameraOffset({padding:t,viewPort:e,camera:this.orthographicCamera}),this._appliedCameraOffset=t)};glideCameraOffset=async t=>{let e=this._appliedCameraOffset;if(globalThis.matchMedia?.("(prefers-reduced-motion: reduce)").matches===!0){this.applyCameraOffset(t),this._invalidate();return}let a=performance.now();await new Promise(r=>{let i=o=>{let s,n=Math.min(1,Math.max(0,(o-a)/300));this.applyCameraOffset((s=n<.5?4*n*n*n:1-(-2*n+2)**3/2,{top:e.top+(t.top-e.top)*s,right:e.right+(t.right-e.right)*s,bottom:e.bottom+(t.bottom-e.bottom)*s,left:e.left+(t.left-e.left)*s})),this._invalidate(),n<1?requestAnimationFrame(i):r()};requestAnimationFrame(i)})};_invalidate=()=>{this._context.configuratorExtensionSelectors.getCanvasSystem().r3fHandle$.value?.get().invalidate()};initialize=()=>{null!==this._generation&&(this._generation.active=!1),this._pendingAbortController?.abort(),this._pendingAbortController=void 0,this._lastTransition=void 0,this._transitionState$.next("still"),this._cameraControlState$.next("idle"),this._activeCameraControls$.next(void 0),this._generation={active:!0},this._seedCameraControls()}}

package/dist/configurator/application-runtime/extensions/systems/graphql-system/graphql-system.d.ts

@@ -1,6 +1,6 @@
 import { type ExecutionResult } from 'graphql';
 import { type Localizable } from '@salsita-ai/common/utils';
-import { type $DomainEntity, type AgentScreenshotComponent, type ComponentSnapshot, type DomainContextSelectors, type DomainEntity, type DomainServicesSelectors, type DomainState, type InitializationGeneration, type SessionAssistantSetup, type SpecSelectors, type StateUpdateEmission } from '#package/domain';
+import { type $DomainEntity, type AgentScreenshotComponent, type ComponentSnapshot, type DomainContextSelectors, type DomainEntity, type DomainServicesSelectors, type DomainState, type InitializationGeneration, type SessionAssistantSetup, type StateUpdateEmission } from '#package/domain';
 import { type AssistantWarning } from './violations.js';
 /** One single-operation GraphQL document with its own variables, as executed by {@link GraphQLSystem}. */
 export interface GraphQLOperation {
@@ -31,7 +31,6 @@ export interface GraphQLSystemContext {
     readonly uploadScreenshot: (image: Blob) => Promise<{
         readonly imageId: string;
     }>;
-    readonly specSelectors: SpecSelectors;
     readonly domainContextSelectors: DomainContextSelectors;
     readonly domainServicesSelectors: DomainServicesSelectors;
 }
@@ -82,6 +81,13 @@ export interface GraphQLExecution {
  * `null` and values that are not localizable pass through, so the schema's nullability decides.
  */
 export declare const localize: (value: unknown, resolveLocalizable: (message: Localizable) => string) => unknown;
+/**
+ * The code of the error a screenshot that was not taken fails its own field with. Such an error is
+ * no failure of its operation: the screenshot is only a look at the configuration, so it neither
+ * stops the operations after it nor keeps `apply` from committing, and the assistant panel shows a
+ * query that failed only on it as done.
+ */
+export declare const SCREENSHOT_NOT_TAKEN = "SCREENSHOT_NOT_TAKEN";
 /**
  * Executes GraphQL operations against the schema generated from the application spec `assistant`
  * manifest, resolving it against the committed domain state. Agent-independent: the assistant's

package/dist/configurator/application-runtime/extensions/systems/graphql-system/graphql-system.js

@@ -1 +1 @@
-import{GraphQLError as e,NoSchemaIntrospectionCustomRule as t,defaultFieldResolver as i,execute as n,getNamedType as o,parse as r,specifiedRules as a,validate as s,validateSchema as l}from"graphql";import{assertExistence as c,createSelector as h,isMessageDescriptor as d,isMultilingualString as u}from"@salsita-ai/common/utils";import{DomainEntityIdSchema as m,agentScreenshotComponent as f,createAssistantExecutionSchema as p,findAssistantMutation as g}from"#package/domain";import{ExportBusyError as v}from"../export-system/export-busy-error.js";import{closeUnclosedBraces as y}from"./close-unclosed-braces.js";import{createEntityMeta as S}from"./entity-meta.js";import{injectAssistantLenses as _}from"./inject-assistant-lenses.js";import{readSideEffects as w}from"./side-effects.js";import{ViolationsError as x,readStateWarnings as b,readViolationEntries as E}from"./violations.js";let $={errors:[new e("The configurator does not provide this request. This is a fault in the configurator, not in your request. Tell the user this is not available.")]},T={errors:[new e("The configuration is not loaded yet, so there is no state to query. Run the query again once the configurator has finished loading.")]},C="Not taken: another configuration was loaded while this request ran. Query the current configuration before asking for screenshots again.",M=new RegExp(m.pattern),R=[...a,t],j=e=>"object"==typeof e&&null!==e&&"type"in e&&"string"==typeof e.type&&"id"in e&&"string"==typeof e.id,k=(e,t)=>"object"==typeof e&&null!==e?Reflect.get(e,t):void 0,N=e=>e instanceof Error?e.message:String(e),A=e=>/[.!?]$/.test(e)?e:`${e}.`,z=e=>"object"==typeof e&&null!==e&&Object.getPrototypeOf(e)===Object.prototype,L=(e,t)=>{let i=z(e)?e._raw:void 0;if(j(i))return i;if(!Array.isArray(e)&&!z(e))return e;if(t.has(e))throw Error("The value contains itself.");let n=new Set([...t,e]);return Array.isArray(e)?e.map(e=>L(e,n)):Object.fromEntries(Object.entries(e).map(([e,t])=>[e,L(t,n)]))};export const localize=(e,t)=>Array.isArray(e)?e.map(e=>localize(e,t)):"string"==typeof e||d(e)||u(e)?t(e):e;let q=t=>new e(`The configurator cannot handle this request: ${A(t)} This is a fault in the configurator, not in your request. Tell the user this is not available.`),O="SCREENSHOT_NOT_TAKEN",F=(t,i)=>new e(t,{originalError:i,extensions:{code:O}}),I=e=>(e.errors??[]).some(e=>e.extensions.code!==O),V=t=>({errors:[new e(`Not run: operation ${t+1} failed, so the operations after it were skipped and no change of the list takes effect. Fix the failing operation and send the whole list again.`)]});export class GraphQLSystem{_context;_selectSchema=h([h.leaf(e=>e)],e=>{try{let t=p(e),i=l(t);return i.length>0?q(i.map(e=>e.message).join(" ")):t}catch(e){return q(N(e))}});_selectLenses=h([h.leaf(e=>e)],e=>r(e));_selectFieldResolver=h([h.leaf(e=>e)],t=>(n,r,a,s)=>{if(s.parentType===s.schema.getQueryType())return i(n,r,a,s);if(s.parentType===s.schema.getMutationType())return this._resolveMutation(s.fieldName,String(s.path.key),r,a);if(!j(n))return i(n,r,a,s);if("meta"===s.fieldName){let e;return S({entity:n,state:("object"==typeof(e=k(n,"_state"))&&null!==e&&"entities"in e&&"rootEntityId"in e?e:void 0)??a.state},{manifest:a.manifest,recipes:this._recipeContext(),prepareStateEmission:(e,t,i)=>{let n=this._context.domainContextSelectors.getStoreSystem().prepareStateEmission(e,t,i);return a.validities.set(n.state,n.validity),n},find$Entity:(e,t)=>this._find$Entity(e,t),validateEntity:e=>this._context.domainContextSelectors.getDomain().validateEntity(e),takeScreenshot:(e,t)=>this._takeScreenshot(e,t,a),validateState:e=>this._validateCached(e,a)})}let{source:l,localized:c}=t[s.parentType.name]?.[s.fieldName]??{source:s.fieldName},h=((t,i)=>{try{return k(t,i)}catch(n){throw new e(`Expression "${i}" failed on ${j(t)?t.id:"an entity"}: ${A(N(n))} This is a fault in the configurator, not in your request. Tell the user this is not available.`,{originalError:n instanceof Error?n:void 0})}})(n,l);return c?localize(h,this._context.domainServicesSelectors.getResolveLocalizable()):l.startsWith("$")&&"JSON"===o(s.returnType).name?((t,i,n)=>{try{return L(n,new Set)}catch(n){throw new e(`Expression "${i}" returned a value that is not plain serializable data on ${j(t)?t.id:"an entity"}: ${A(N(n))} This is a fault in the configurator, not in your request. Tell the user this is not available.`,{originalError:n instanceof Error?n:void 0})}})(n,l,h):h});constructor(e){this._context=e}_get$Entity=(e,t)=>this._context.domainContextSelectors.getDomain().get$Entity(e,t);_find$Entity=(e,t)=>this._context.domainContextSelectors.getDomain().getEntity(e,t)?this._get$Entity(e,t):null;_validateState=e=>this._context.domainContextSelectors.getValidationProjections().validateState(e);_validateCached=(e,{validities:t})=>{let i=t.get(e);if(i)return i;let n=this._validateState(e);return t.set(e,n),n};_recipeContext=()=>({componentReader:this._context.domainContextSelectors.getComponentReader(),recipeCooker:this._context.domainContextSelectors.getRecipeCookerSystem(),resolveLocalizable:this._context.domainServicesSelectors.getResolveLocalizable()});_rootValue={warnings:(e,t)=>b(this._validateCached(t.state,t),void 0,this._context.domainServicesSelectors.getResolveLocalizable()),root:(e,{state:t})=>this._get$Entity(t.rootEntityId,t),entities:({ids:t},{state:i})=>{if(null==t)return i.entities.map(e=>this._get$Entity(e.id,i));var n=i.rootEntityId;let o=t.filter(e=>!M.test(e));if(o.length>0)throw new e(`${o.map(e=>`"${e}"`).join(", ")} ${1===o.length?"is not an entity id":"are not entity ids"}. Entity ids have the form "<Type>-<number>", e.g. "${n}". Use ids as returned by the "id" field.`);return t.map(e=>M.test(e)?this._find$Entity(e,i):null)},selectedEntity:(e,{state:t})=>this._find$Entity(this._context.domainContextSelectors.getNavigationSystem().getActiveSelectionState(t),t),navigation:(e,{state:t})=>{let i=this._context.domainContextSelectors.getNavigationSystem();return{navigationState:i.getActiveNavigationState(t),viewState:i.getActiveViewState(t),toolState:i.getActiveToolState(t)}}};_resolveType=e=>j(e)?e.type:void 0;_generation={active:!0};_renderScreenshot=async(e,t,i)=>{let n=this._context.domainContextSelectors.getComponentReader().readOneComponent({entityId:e,componentType:f,instanceName:t},i);if(!n)return null;let o=await this._context.renderScreenshot(n,i);return null===o?null:new Blob([o],{type:"JPEG"===n.state.fileFormat?"image/jpeg":"image/png"})};initialize=()=>{this._generation.active=!1,this._generation={active:!0}};_resolveMutation=(t,i,n,o)=>{if(void 0!==o.failedMutationField)throw new e(`Not run: the earlier field "${o.failedMutationField}" of this operation failed, so "${i}" was skipped and no change of the operation takes effect. Fix the failing field and send the operation again.`);try{return this._runMutation(t,n,o)}catch(e){throw o.failedMutationField=i,e}};_runMutation=(t,i,n)=>{let o=g(t,n.manifest);if(!o)throw new e(`The schema declares the mutation "${t}", but the configurator has no implementation for it. This is a fault in the configurator, not in your request. Tell the user this is not available.`);let r=n.state,a=o.implementation(i,r,{manifest:n.manifest,action:this._context.domainContextSelectors.getActionSystem().createContext(r),resolveLocalizable:this._context.domainServicesSelectors.getResolveLocalizable()});if(a.updates.length>0){let i=this._context.domainContextSelectors.getStoreSystem().prepareStateEmission(a.updates,r,"committed"),{validity:o}=i;if("uncaught"===o.severity)throw new e(`The changes of "${t}" were not applied, because the resulting configuration is invalid: validating it threw: ${A(N(o.error))} Change the request so the configuration stays valid.`,{originalError:o.error});if(!o.valid)throw new x(`The changes of "${t}" were not applied, because the resulting configuration has ${o.severity} violations.`,E(o,this._context.domainServicesSelectors.getResolveLocalizable()));n.emissions.push(i),n.validities.set(i.state,o),n.state=i.state}return a.readResult({previousState:r,state:n.state})};_takeScreenshot=({entity:t,state:i},n,o)=>{let r=o.manifest.screenshots?.[t.type]?.find(e=>e.view===n);if(!r)return Promise.reject(new e(`The schema declares the screenshot view "${n}" on ${t.type}, but the configurator has no view of that name. This is a fault in the configurator, not in your request. Tell the user this is not available.`));let a=o.lastScreenshot.then(async()=>{if(!o.generation.active)throw F(C);let e=await this._renderScreenshot(t.id,r.instanceName,i).catch(e=>{throw F(e instanceof v?"User made concurrent actions that prevented the screenshot from being taken, try again":`The screenshot of ${t.id} from "${n}" could not be taken: ${A(N(e))} Try again; if it fails again, tell the user the screenshot is not available.`,e instanceof Error?e:void 0)});if(!o.generation.active)throw F(C);if(null===e)return null;let{imageId:a}=await this._context.uploadScreenshot(e).catch(e=>{throw F(`The screenshot of ${t.id} from "${n}" was taken but could not be stored: ${A(N(e))} Try again; if it fails again, tell the user the screenshot is not available.`,e instanceof Error?e:void 0)}),s=`${t.id}/${n}`,l=[...o.screenshots.values()].filter(e=>e.entityId===t.id&&e.view===n).length,c=0===l?s:`${s}#${l+1}`;return o.screenshots.set(c,{entityId:t.id,view:n,imageId:a}),{image:c}});return o.lastScreenshot=a.catch(()=>void 0),a};_setup=()=>{let{manifest:t,error:i}=this._context.getAssistantSetup();if(i)return{ready:!1,result:{errors:[new e(i.message,{originalError:i})]}};if(!t)return{ready:!1,result:$};let n=this._selectSchema(t.sdl);return n instanceof e?{ready:!1,result:{errors:[n]}}:{ready:!0,assistant:t,schema:n,lenses:t.lenses?.definitions?this._selectLenses(t.lenses.definitions):void 0,fieldResolver:this._selectFieldResolver(t.fields)}};_run=async(t,i,o,a)=>{var l;let c;try{c=r(t)}catch(t){return{errors:[t instanceof e?t:new e(N(t))]}}let h=s(o.schema,c,R);return h.length>0?{errors:h}:(l=await n({schema:o.schema,document:c,rootValue:this._rootValue,contextValue:a,variableValues:i.variables,operationName:i.operationName,fieldResolver:o.fieldResolver,typeResolver:this._resolveType}),l.errors?.some(e=>e.originalError instanceof x)?{...l,errors:l.errors.flatMap(t=>t.originalError instanceof x?t.originalError.entries.map(({type:i,entityId:n,message:o})=>new e(o??i,{nodes:t.nodes,source:t.source,positions:t.positions,path:t.path,extensions:{code:"VIOLATION",type:i,entityId:n}})):[t])}:l)};prepare=async(t,i,n=this._generation)=>{let o,r=this._setup();if(!r.ready)return i.map(()=>({result:r.result,emissions:[],screenshots:new Map}));let a={manifest:r.assistant,state:t,emissions:[],failedMutationField:void 0,screenshots:new Map,lastScreenshot:Promise.resolve(),generation:n,validities:new WeakMap},s=[];for(let[t,n]of i.entries())if(void 0===o){let i=a.emissions.length,l=a.screenshots.size;a.failedMutationField=void 0;let c=y(n.document),h=r.lenses?_(c,r.lenses):c,d=h instanceof e?{errors:[h]}:await this._run(h,n,r,a);await a.lastScreenshot,s.push({result:d,emissions:a.emissions.slice(i),screenshots:new Map([...a.screenshots].slice(l))}),I(d)&&(o=t)}else s.push({result:V(o),emissions:[],screenshots:new Map});return s};readSideEffects=e=>{let t=this._context.domainServicesSelectors.getResolveLocalizable();return e.flatMap(({emissions:e})=>e.flatMap(e=>w(e.updates,t)))};readScreenshots=e=>new Map(e.flatMap(({screenshots:e})=>[...e]));readWarnings=(e,t)=>{let i=t.flatMap(({emissions:e})=>e).at(-1);return void 0===i||t.some(({result:e})=>I(e))?[]:b(i.validity,this._validateState(e),this._context.domainServicesSelectors.getResolveLocalizable())};_prepareAndCommit=async(e,t)=>{let i,n,o,r=this._context.domainContextSelectors.getStoreSystem(),a=r.stateSlices.committed.value.state,s=await this.prepare(a,e,t),l=s.map(({result:e})=>e),h=this.readSideEffects(s),d=this.readScreenshots(s),u=s.flatMap(e=>e.emissions);if(!t.active)return{results:l,sideEffects:h,screenshots:d,warnings:[],outcome:"conflict"};if(l.some(I)||0===u.length)return{results:l,sideEffects:h,screenshots:d,warnings:[],outcome:"none"};let m=(i=c(u[0]),n=c(u[u.length-1]),o=u.some(e=>void 0!==e.transformationLog),{kind:"committed",previousState:i.previousState,state:n.state,updates:u.flatMap(e=>e.updates),validity:n.validity,transformationLog:o?u.flatMap(e=>e.transformationLog??[]):void 0});return{results:l,sideEffects:h,screenshots:d,warnings:this.readWarnings(a,s),outcome:r.commitPrepared(m),preparedState:m.state}};execute=async e=>{let t=this._generation;if(!this._context.domainContextSelectors.getStoreSystem().isInitialized)return{results:e.map(()=>T),committed:!1,sideEffects:[],warnings:[],screenshots:new Map};let i=await this._prepareAndCommit(e,t),{results:n,sideEffects:o,warnings:r,screenshots:a,outcome:s,preparedState:l}="conflict"===i.outcome&&t.active?await this._prepareAndCommit(e,t):i;return"committed"===s?{results:n,committed:!0,sideEffects:o,warnings:r,screenshots:a,preparedState:l}:"held"===s?{results:n,committed:!1,message:"The changes wait for the user's approval and are not applied yet. Do not send them again; the next `[changes]` block lists them once approved.",sideEffects:o,warnings:r,screenshots:a,preparedState:l}:"conflict"===s?{results:n,committed:!1,message:"The changes were not committed, because the configuration changed while they were being applied. The results describe a state that was discarded. Query the current configuration and send the operations again if they still make sense.",sideEffects:o,warnings:r,screenshots:a}:{results:n,committed:!1,sideEffects:o,warnings:r,screenshots:a}}}
+import{GraphQLError as e,NoSchemaIntrospectionCustomRule as t,defaultFieldResolver as i,execute as n,getNamedType as o,parse as r,specifiedRules as s,validate as a,validateSchema as l}from"graphql";import{assertExistence as c,createSelector as h,isMessageDescriptor as d,isMultilingualString as u}from"@salsita-ai/common/utils";import{AGENT_SCREENSHOT_MAX_SIDE as m,DomainEntityIdSchema as f,agentScreenshotComponent as p,createAssistantExecutionSchema as g,findAssistantMutation as v}from"#package/domain";import{ExportBusyError as y}from"../export-system/export-busy-error.js";import{closeUnclosedBraces as S}from"./close-unclosed-braces.js";import{createEntityMeta as _}from"./entity-meta.js";import{injectAssistantLenses as w}from"./inject-assistant-lenses.js";import{readSideEffects as x}from"./side-effects.js";import{ViolationsError as E,readStateWarnings as T,readViolationEntries as $}from"./violations.js";let b={errors:[new e("The configurator does not provide this request. This is a fault in the configurator, not in your request. Tell the user this is not available.")]},C={errors:[new e("The configuration is not loaded yet, so there is no state to query. Run the query again once the configurator has finished loading.")]},N="Not taken: another configuration was loaded while this request ran. Query the current configuration before asking for screenshots again.",M=class ScreenshotTooLargeError extends Error{constructor(e,t){super(`The view is configured at ${e} \xd7 ${t} px, more than the ${m} \xd7 ${m} px a screenshot may have.`),this.name="ScreenshotTooLargeError"}},R=new RegExp(f.pattern),k=[...s,t],j=e=>"object"==typeof e&&null!==e&&"type"in e&&"string"==typeof e.type&&"id"in e&&"string"==typeof e.id,A=(e,t)=>"object"==typeof e&&null!==e?Reflect.get(e,t):void 0,L=e=>e instanceof Error?e.message:String(e),O=e=>/[.!?]$/.test(e)?e:`${e}.`,z=e=>"object"==typeof e&&null!==e&&Object.getPrototypeOf(e)===Object.prototype,q=(e,t)=>{let i=z(e)?e._raw:void 0;if(j(i))return i;if(!Array.isArray(e)&&!z(e))return e;if(t.has(e))throw Error("The value contains itself.");let n=new Set([...t,e]);return Array.isArray(e)?e.map(e=>q(e,n)):Object.fromEntries(Object.entries(e).map(([e,t])=>[e,q(t,n)]))};export const localize=(e,t)=>Array.isArray(e)?e.map(e=>localize(e,t)):"string"==typeof e||d(e)||u(e)?t(e):e;let F=t=>new e(`The configurator cannot handle this request: ${O(t)} This is a fault in the configurator, not in your request. Tell the user this is not available.`);export const SCREENSHOT_NOT_TAKEN="SCREENSHOT_NOT_TAKEN";let I=(t,i)=>new e(t,{originalError:i,extensions:{code:SCREENSHOT_NOT_TAKEN}}),V=e=>(e.errors??[]).some(e=>e.extensions.code!==SCREENSHOT_NOT_TAKEN),P=t=>({errors:[new e(`Not run: operation ${t+1} failed, so the operations after it were skipped and no change of the list takes effect. Fix the failing operation and send the whole list again.`)]});export class GraphQLSystem{_context;_selectSchema=h([h.leaf(e=>e)],e=>{try{let t=g(e),i=l(t);return i.length>0?F(i.map(e=>e.message).join(" ")):t}catch(e){return F(L(e))}});_selectLenses=h([h.leaf(e=>e)],e=>r(e));_selectFieldResolver=h([h.leaf(e=>e)],t=>(n,r,s,a)=>{if(a.parentType===a.schema.getQueryType())return i(n,r,s,a);if(a.parentType===a.schema.getMutationType())return this._resolveMutation(a.fieldName,String(a.path.key),r,s);if(!j(n))return i(n,r,s,a);if("meta"===a.fieldName){let e;return _({entity:n,state:("object"==typeof(e=A(n,"_state"))&&null!==e&&"entities"in e&&"rootEntityId"in e?e:void 0)??s.state},{manifest:s.manifest,recipes:this._recipeContext(),prepareStateEmission:(e,t,i)=>{let n=this._context.domainContextSelectors.getStoreSystem().prepareStateEmission(e,t,i);return s.validities.set(n.state,n.validity),n},find$Entity:(e,t)=>this._find$Entity(e,t),validateEntity:e=>this._context.domainContextSelectors.getDomain().validateEntity(e),takeScreenshot:(e,t)=>this._takeScreenshot(e,t,s),validateState:e=>this._validateCached(e,s)})}let{source:l,localized:c}=t[a.parentType.name]?.[a.fieldName]??{source:a.fieldName},h=((t,i)=>{try{return A(t,i)}catch(n){throw new e(`Expression "${i}" failed on ${j(t)?t.id:"an entity"}: ${O(L(n))} This is a fault in the configurator, not in your request. Tell the user this is not available.`,{originalError:n instanceof Error?n:void 0})}})(n,l);return c?localize(h,this._context.domainServicesSelectors.getResolveLocalizable()):l.startsWith("$")&&"JSON"===o(a.returnType).name?((t,i,n)=>{try{return q(n,new Set)}catch(n){throw new e(`Expression "${i}" returned a value that is not plain serializable data on ${j(t)?t.id:"an entity"}: ${O(L(n))} This is a fault in the configurator, not in your request. Tell the user this is not available.`,{originalError:n instanceof Error?n:void 0})}})(n,l,h):h});constructor(e){this._context=e}_get$Entity=(e,t)=>this._context.domainContextSelectors.getDomain().get$Entity(e,t);_find$Entity=(e,t)=>this._context.domainContextSelectors.getDomain().getEntity(e,t)?this._get$Entity(e,t):null;_validateState=e=>this._context.domainContextSelectors.getValidationProjections().validateState(e);_validateCached=(e,{validities:t})=>{let i=t.get(e);if(i)return i;let n=this._validateState(e);return t.set(e,n),n};_recipeContext=()=>({componentReader:this._context.domainContextSelectors.getComponentReader(),recipeCooker:this._context.domainContextSelectors.getRecipeCookerSystem(),resolveLocalizable:this._context.domainServicesSelectors.getResolveLocalizable()});_rootValue={warnings:(e,t)=>T(this._validateCached(t.state,t),void 0,this._context.domainServicesSelectors.getResolveLocalizable()),root:(e,{state:t})=>this._get$Entity(t.rootEntityId,t),entities:({ids:t},{state:i})=>{if(null==t)return i.entities.map(e=>this._get$Entity(e.id,i));var n=i.rootEntityId;let o=t.filter(e=>!R.test(e));if(o.length>0)throw new e(`${o.map(e=>`"${e}"`).join(", ")} ${1===o.length?"is not an entity id":"are not entity ids"}. Entity ids have the form "<Type>-<number>", e.g. "${n}". Use ids as returned by the "id" field.`);return t.map(e=>R.test(e)?this._find$Entity(e,i):null)},selectedEntity:(e,{state:t})=>this._find$Entity(this._context.domainContextSelectors.getNavigationSystem().getActiveSelectionState(t),t),navigation:(e,{state:t})=>{let i=this._context.domainContextSelectors.getNavigationSystem();return{navigationState:i.getActiveNavigationState(t),viewState:i.getActiveViewState(t),toolState:i.getActiveToolState(t)}}};_resolveType=e=>j(e)?e.type:void 0;_generation={active:!0};_renderScreenshot=async(e,t,i)=>{let n=this._context.domainContextSelectors.getComponentReader().readOneComponent({entityId:e,componentType:p,instanceName:t},i);if(!n)return null;let{width:o,height:r}=n.state;if(o>m||r>m)throw new M(o,r);let s=await this._context.renderScreenshot(n,i);return null===s?null:new Blob([s],{type:"JPEG"===n.state.fileFormat?"image/jpeg":"image/png"})};initialize=()=>{this._generation.active=!1,this._generation={active:!0}};_resolveMutation=(t,i,n,o)=>{if(void 0!==o.failedMutationField)throw new e(`Not run: the earlier field "${o.failedMutationField}" of this operation failed, so "${i}" was skipped and no change of the operation takes effect. Fix the failing field and send the operation again.`);try{return this._runMutation(t,n,o)}catch(e){throw o.failedMutationField=i,e}};_runMutation=(t,i,n)=>{let o=v(t,n.manifest);if(!o)throw new e(`The schema declares the mutation "${t}", but the configurator has no implementation for it. This is a fault in the configurator, not in your request. Tell the user this is not available.`);let r=n.state,s=o.implementation(i,r,{manifest:n.manifest,action:this._context.domainContextSelectors.getActionSystem().createContext(r),resolveLocalizable:this._context.domainServicesSelectors.getResolveLocalizable()});if(s.updates.length>0){let i=this._context.domainContextSelectors.getStoreSystem().prepareStateEmission(s.updates,r,"committed"),{validity:o}=i;if("uncaught"===o.severity)throw new e(`The changes of "${t}" were not applied, because the resulting configuration is invalid: validating it threw: ${O(L(o.error))} Change the request so the configuration stays valid.`,{originalError:o.error});if(!o.valid)throw new E(`The changes of "${t}" were not applied, because the resulting configuration has ${o.severity} violations.`,$(o,this._context.domainServicesSelectors.getResolveLocalizable()));n.emissions.push(i),n.validities.set(i.state,o),n.state=i.state}return s.readResult({previousState:r,state:n.state})};_takeScreenshot=({entity:t,state:i},n,o)=>{let r=o.manifest.screenshots?.[t.type]?.find(e=>e.view===n);if(!r)return Promise.reject(new e(`The schema declares the screenshot view "${n}" on ${t.type}, but the configurator has no view of that name. This is a fault in the configurator, not in your request. Tell the user this is not available.`));let s=o.lastScreenshot.then(async()=>{if(!o.generation.active)throw I(N);let e=await this._renderScreenshot(t.id,r.instanceName,i).catch(e=>{throw I(e instanceof y?"User made concurrent actions that prevented the screenshot from being taken, try again":e instanceof M?`The screenshot of ${t.id} from "${n}" cannot be taken: ${e.message} Tell the user the screenshot is not available.`:`The screenshot of ${t.id} from "${n}" could not be taken: ${O(L(e))} Try again; if it fails again, tell the user the screenshot is not available.`,e instanceof Error?e:void 0)});if(!o.generation.active)throw I(N);if(null===e)return null;let{imageId:s}=await this._context.uploadScreenshot(e).catch(e=>{throw I(`The screenshot of ${t.id} from "${n}" was taken but could not be stored: ${O(L(e))} Try again; if it fails again, tell the user the screenshot is not available.`,e instanceof Error?e:void 0)}),a=`${t.id}/${n}`,l=[...o.screenshots.values()].filter(e=>e.entityId===t.id&&e.view===n).length,c=0===l?a:`${a}#${l+1}`;return o.screenshots.set(c,{entityId:t.id,view:n,imageId:s}),{image:c}});return o.lastScreenshot=s.catch(()=>void 0),s};_setup=()=>{let{manifest:t,error:i}=this._context.getAssistantSetup();if(i)return{ready:!1,result:{errors:[new e(i.message,{originalError:i})]}};if(!t)return{ready:!1,result:b};let n=this._selectSchema(t.sdl);return n instanceof e?{ready:!1,result:{errors:[n]}}:{ready:!0,assistant:t,schema:n,lenses:t.lenses?.definitions?this._selectLenses(t.lenses.definitions):void 0,fieldResolver:this._selectFieldResolver(t.fields)}};_run=async(t,i,o,s)=>{var l;let c;try{c=r(t)}catch(t){return{errors:[t instanceof e?t:new e(L(t))]}}let h=a(o.schema,c,k);return h.length>0?{errors:h}:(l=await n({schema:o.schema,document:c,rootValue:this._rootValue,contextValue:s,variableValues:i.variables,operationName:i.operationName,fieldResolver:o.fieldResolver,typeResolver:this._resolveType}),l.errors?.some(e=>e.originalError instanceof E)?{...l,errors:l.errors.flatMap(t=>t.originalError instanceof E?t.originalError.entries.map(({type:i,entityId:n,message:o})=>new e(o??i,{nodes:t.nodes,source:t.source,positions:t.positions,path:t.path,extensions:{code:"VIOLATION",type:i,entityId:n}})):[t])}:l)};prepare=async(t,i,n=this._generation)=>{let o,r=this._setup();if(!r.ready)return i.map(()=>({result:r.result,emissions:[],screenshots:new Map}));let s={manifest:r.assistant,state:t,emissions:[],failedMutationField:void 0,screenshots:new Map,lastScreenshot:Promise.resolve(),generation:n,validities:new WeakMap},a=[];for(let[t,n]of i.entries())if(void 0===o){let i=s.emissions.length,l=s.screenshots.size;s.failedMutationField=void 0;let c=S(n.document),h=r.lenses?w(c,r.lenses):c,d=h instanceof e?{errors:[h]}:await this._run(h,n,r,s);await s.lastScreenshot,a.push({result:d,emissions:s.emissions.slice(i),screenshots:new Map([...s.screenshots].slice(l))}),V(d)&&(o=t)}else a.push({result:P(o),emissions:[],screenshots:new Map});return a};readSideEffects=e=>{let t=this._context.domainServicesSelectors.getResolveLocalizable();return e.flatMap(({emissions:e})=>e.flatMap(e=>x(e.updates,t)))};readScreenshots=e=>new Map(e.flatMap(({screenshots:e})=>[...e]));readWarnings=(e,t)=>{let i=t.flatMap(({emissions:e})=>e).at(-1);return void 0===i||t.some(({result:e})=>V(e))?[]:T(i.validity,this._validateState(e),this._context.domainServicesSelectors.getResolveLocalizable())};_prepareAndCommit=async(e,t)=>{let i,n,o,r=this._context.domainContextSelectors.getStoreSystem(),s=r.stateSlices.committed.value.state,a=await this.prepare(s,e,t),l=a.map(({result:e})=>e),h=this.readSideEffects(a),d=this.readScreenshots(a),u=a.flatMap(e=>e.emissions);if(!t.active)return{results:l,sideEffects:h,screenshots:d,warnings:[],outcome:"conflict"};if(l.some(V)||0===u.length)return{results:l,sideEffects:h,screenshots:d,warnings:[],outcome:"none"};let m=(i=c(u[0]),n=c(u[u.length-1]),o=u.some(e=>void 0!==e.transformationLog),{kind:"committed",previousState:i.previousState,state:n.state,updates:u.flatMap(e=>e.updates),validity:n.validity,transformationLog:o?u.flatMap(e=>e.transformationLog??[]):void 0});return{results:l,sideEffects:h,screenshots:d,warnings:this.readWarnings(s,a),outcome:r.commitPrepared(m),preparedState:m.state}};execute=async e=>{let t=this._generation;if(!this._context.domainContextSelectors.getStoreSystem().isInitialized)return{results:e.map(()=>C),committed:!1,sideEffects:[],warnings:[],screenshots:new Map};let i=await this._prepareAndCommit(e,t),{results:n,sideEffects:o,warnings:r,screenshots:s,outcome:a,preparedState:l}="conflict"===i.outcome&&t.active?await this._prepareAndCommit(e,t):i;return"committed"===a?{results:n,committed:!0,sideEffects:o,warnings:r,screenshots:s,preparedState:l}:"held"===a?{results:n,committed:!1,message:"The changes wait for the user's approval and are not applied yet. Do not send them again; the next `[changes]` block lists them once approved.",sideEffects:o,warnings:r,screenshots:s,preparedState:l}:"conflict"===a?{results:n,committed:!1,message:"The changes were not committed, because the configuration changed while they were being applied. The results describe a state that was discarded. Query the current configuration and send the operations again if they still make sense.",sideEffects:o,warnings:r,screenshots:s}:{results:n,committed:!1,sideEffects:o,warnings:r,screenshots:s}}}

package/dist/configurator/application-runtime/extensions/systems/image-rendering-system/capture-next-frame.d.ts

@@ -0,0 +1,10 @@
+import type { R3FHandle } from '../canvas-system.js';
+/**
+ * Copies the next regular frame the canvas draws into a 2D canvas of the same size, and resolves with
+ * it. `isRegularFrame` tells a frame drawn by the project's own render path (its postprocessing, its
+ * pixel ratio), which the copy shows as the user sees it, from one that is not, such as an export's,
+ * which is skipped. Without `preserveDrawingBuffer` the canvas can only be read in the task that drew
+ * it, so the copy is taken in an after-effect of that frame, which runs right after the render in the
+ * same animation frame. Rejects when no regular frame is drawn within a few seconds.
+ */
+export declare const captureNextFrame: (r3fHandle: R3FHandle, isRegularFrame: (state: ReturnType<R3FHandle['get']>) => boolean) => Promise<HTMLCanvasElement>;

package/dist/configurator/application-runtime/extensions/systems/image-rendering-system/capture-next-frame.js

@@ -0,0 +1 @@
+import{addAfterEffect as e}from"@react-three/fiber";export const captureNextFrame=(t,o)=>new Promise((r,a)=>{let i=setTimeout(()=>{n(),a(Error("The configurator view could not be copied, because the canvas drew no frame of it within 5000 ms. An export may hold the canvas; try again once it is over."))},5e3),n=e(()=>{let e=t.get();if(!o(e))return;n(),clearTimeout(i);let c=e.gl.domElement,h=document.createElement("canvas");h.width=c.width,h.height=c.height;let m=h.getContext("2d");null===m?a(Error("The browser gave no 2D canvas to copy the configurator view onto.")):(m.drawImage(c,0,0),r(h))});t.get().invalidate()});

package/dist/configurator/application-runtime/extensions/systems/image-rendering-system/image-rendering-system.d.ts

@@ -39,6 +39,14 @@ export declare class ImageRenderingSystem<ENTITIES extends DomainEntity<string>>
     registerEffectComposer(effectComposer: EffectComposer): () => void;
     private readonly _openDebugImage;
     private readonly _getRenderer;
+    /**
+     * A copy of the configurator view as the interactive canvas draws its next regular frame, at the
+     * canvas's resolution; waits for a running export to end and for a canvas to mount when none is.
+     * Unlike an export, it renders nothing of its own and leaves the canvas and the frameloop as they are.
+     *
+     * @internal
+     */
+    readonly captureView: () => Promise<HTMLCanvasElement>;
     /**
      * Generates an image for the given configuration and uploads it to s3 as finalization data.
      * This method uses the original canvas and camera to render the image.

package/dist/configurator/application-runtime/extensions/systems/image-rendering-system/image-rendering-system.js

@@ -1 +1 @@
-import{Color as e,HalfFloatType as t,NoToneMapping as a,OrthographicCamera as r,PerspectiveCamera as n,WebGLRenderTarget as o}from"three";import{preloadFont as i}from"troika-three-text";import{match as s}from"ts-pattern";import{clientSideImageComponent as l,dimensionIndicatorComponent as c,domainEntityTypeFromId as m,editableDistanceDimensionIndicatorComponent as g,isDomainEntityId as d,isVirtualEntityId as h,sketchingEditComponent as u}from"#package/domain";import{createCameraControls as f}from"../camera-control-system/create-camera-controls.js";import{createCanvasFreezeFrame as p}from"./canvas-freeze-frame.js";import{ExportCapturePass as w,readTargetToBlob as C,resolveExportBackground as _}from"./export-image-capture.js";export class ImageRenderingSystem{_context;_renderer=null;_effectComposer=null;_generation=null;_debugObjectUrls=new Set;_canvasFreeze=null;constructor(e){this._context=e}_captureGenerationAssertion=()=>{let e=this._generation,t=()=>{if(!e?.active)throw Error("Image rendering generation ended while images were being rendered! The configurator context was reinitialized before the renderer finished, so the results belong to a configuration that is no longer active. Await the render before leaving the configuration, or discard the rejection.")};return t(),t};_captureGenerationGuard=()=>{let e=this._captureGenerationAssertion();return{assertActive:e,waitFor:async t=>{let a=await t;return e(),a},runEffect:t=>(e(),t())}};register(e){return this._renderer=e,()=>{this._renderer===e&&(this._renderer=null)}}registerEffectComposer(e){return this._effectComposer=e,()=>{this._effectComposer===e&&(this._effectComposer=null)}}_openDebugImage=e=>{let t=URL.createObjectURL(e);this._debugObjectUrls.add(t),window.open(t,"_blank")};_getRenderer=async()=>{let e=this._generation,t=Date.now(),a=()=>{if(!e?.active)throw Error("Image rendering generation ended while waiting for the renderer! The configurator context was reinitialized before the renderer arrived, so the render would target a scene that is gone. Await the render before leaving the configuration, or discard the rejection.")};a();let r=this._renderer;for(;null===r;){if(await new Promise(e=>setTimeout(e,50)),a(),Date.now()-t>1e4)throw Error("Renderer could not be retrieved! Reached timeout.");r=this._renderer}return r};generateImageUsingOriginalCanvas=async(e,t,a=!1)=>{let r=this._captureGenerationGuard(),n=await r.waitFor(this._getRenderer());if(a){let{blob:e}=await r.waitFor(r.runEffect(()=>n.renderUsingOriginalCanvas(t)));r.runEffect(()=>this._openDebugImage(e));return}let o=await Promise.all([t].map(async e=>{let{blob:t}=await r.waitFor(r.runEffect(()=>n.renderUsingOriginalCanvas(e)));return{blob:t,name:e.name}})),i=await r.waitFor(r.runEffect(()=>this._context.getAuthorizedClient().generateUploadLinks(this._context.getConfigurationId(),e,[t.name])));await Promise.all(i.map(async e=>{let t=o.find(t=>t.name===e.name);if(null==t)throw Error("Blob not found!");await Promise.all(i.map(async e=>{await r.waitFor(r.runEffect(()=>fetch(e.url,{method:"PUT",body:t.blob,headers:{"Content-Type":"image/png","Content-Length":t.blob.size.toString()}})))}))}))};async generateImagesUsingOriginalCanvas(e,t,a=!1){let r=this._captureGenerationGuard(),n=await r.waitFor(this._getRenderer()),o=Array.isArray(t)?t:[t],i=await o.reduce(async(e,t)=>[...await e,await r.waitFor(r.runEffect(()=>n.renderUsingOriginalCanvas(t)))],Promise.resolve([]));if(a){for(let e of i)r.runEffect(()=>this._openDebugImage(e.blob)),await r.waitFor(new Promise(e=>setTimeout(e,100)));return}let s=o.map(e=>e.name),l=await r.waitFor(r.runEffect(()=>this._context.getAuthorizedClient().generateUploadLinks(this._context.getConfigurationId(),e,s)));await Promise.all(l.map(async e=>{let t=i.find(t=>t.name===e.name);if(null==t)throw Error("Blob not found!");await Promise.all(l.map(async e=>{await r.waitFor(r.runEffect(()=>fetch(e.url,{method:"PUT",body:t.blob,headers:{"Content-Type":"image/png","Content-Length":t.blob.size.toString()}})))}))}))}async generateImages(e,t,a=!1){let r=this._captureGenerationGuard(),n=await r.waitFor(this._getRenderer()),o=t.map(e=>e.name);if(a)return void await Promise.all(t.map(async e=>{let t=await r.waitFor(r.runEffect(()=>n.renderImage(e)));r.runEffect(()=>this._openDebugImage(t))}));let i=await Promise.all(t.map(async e=>({blob:await r.waitFor(r.runEffect(()=>n.renderImage(e))),name:e.name}))),s=await r.waitFor(r.runEffect(()=>this._context.getAuthorizedClient().generateUploadLinks(this._context.getConfigurationId(),e,o)));await Promise.all(s.map(async e=>{let t=i.find(t=>t.name===e.name);if(null==t)throw Error("Blob not found!");await Promise.all(s.map(async e=>{await r.waitFor(r.runEffect(()=>fetch(e.url,{method:"PUT",body:t.blob,headers:{"Content-Type":"image/png","Content-Length":t.blob.size.toString()}})))}))}))}async getGeneratedImages(e,t=!1){let a=this._captureGenerationAssertion(),r=await this._getRenderer();return(a(),t)?(await Promise.all(e.map(async e=>{let t=await r.renderImage(e);a(),this._openDebugImage(t)})),[]):Promise.all(e.map(async e=>{let t=await r.renderImage(e);return a(),{blob:t,name:e.name}}))}_renderCurrentView=e=>{let{gl:t,scene:a,camera:r}=e.get();null!==this._effectComposer?this._effectComposer.render():t.render(a,r)};_freezeCanvas=e=>{this._canvasFreeze?.handle===e?this._canvasFreeze.holders+=1:(this._canvasFreeze?.remove(),this._canvasFreeze={handle:e,remove:p(e.get().gl.domElement,()=>this._renderCurrentView(e)),resumeFrameloop:this._context.configuratorExtensionSelectors.getExportSystem().pauseFrameloop(e),holders:1});let t=this._canvasFreeze,a=!1;return async()=>{if(!a&&(a=!0,t.holders-=1,!(t.holders>0)))try{let a=this._context.configuratorExtensionSelectors.getCanvasSystem();a.r3fHandle$.value===e&&await a.waitForFlush(),a.r3fHandle$.value===e&&0===t.holders&&this._renderCurrentView(e)}catch{}finally{0===t.holders&&(t.remove(),t.resumeFrameloop(),this._canvasFreeze===t&&(this._canvasFreeze=null))}}};async renderImage(r){let n=this._context.configuratorExtensionSelectors.getCanvasSystem(),i=n.r3fHandle$.value??await n.waitForCanvas(),{gl:s,scene:l,camera:c,size:m,advance:g}=i.get(),d=s.getRenderTarget(),h=s.getClearColor(new e),u=s.getClearAlpha(),f=s.autoClearColor,p=e=>new w(r.width,r.height,e,s.outputColorSpace),y={composer:null,pass:null},b=null,x=null,v=e=>{let t=y.pass??p(a);return y.pass=t,y.composer===e&&e.passes.at(-1)===t||(y.composer?.removePass(t),e.addPass(t),y.composer=e),t},E=async()=>{if(await this._context.configuratorExtensionSelectors.getCanvasSystem().waitForFlush(),this._context.configuratorExtensionSelectors.getCanvasSystem().r3fHandle$.value!==i)throw Error("Canvas was remounted while an image was being rendered! The export borrowed a canvas that is gone, so it cannot capture the image. Retry the export once the configurator has settled.")},S=this._freezeCanvas(i);try{i.set({size:{height:r.height,width:r.width,top:0,left:0,updateStyle:!1}}),i.set({camera:r.camera}),await E(),null!==this._effectComposer&&v(this._effectComposer),g(void 0),await E();let a="JPEG"===r.fileFormat?"image/jpeg":"image/png",n=_({sceneBackground:l.background,backgroundColor:r.backgroundColor,fileFormat:r.fileFormat,outputColorSpace:s.outputColorSpace});if(null!==this._effectComposer){s.setRenderTarget(null),"JPEG"===r.fileFormat?s.setClearColor(r.backgroundColor??0xffffff):r.backgroundColor?s.setClearColor(r.backgroundColor):s.setClearAlpha(0),s.autoClearColor=!0;let e=v(this._effectComposer);return this._effectComposer.render(),await C(s,e.renderTarget,n,a)}b=p(s.toneMapping),x=new o(r.width,r.height,{samples:4,type:t,stencilBuffer:s.getContextAttributes().stencil??!1});let c=l.background;c instanceof e&&(l.background=null);try{s.setClearColor(0,0),s.setRenderTarget(x),s.clear(),s.render(l,r.camera)}finally{l.background=c}return b.render(s,x),await C(s,b.renderTarget,n,a)}finally{null!==y.pass&&(y.composer?.removePass(y.pass),y.pass.dispose()),b?.dispose(),x?.dispose(),this._context.configuratorExtensionSelectors.getCanvasSystem().r3fHandle$.value===i&&(s.setClearColor(h,u),s.autoClearColor=f,s.setRenderTarget(d),i.set({camera:c,size:{...m,updateStyle:!1}})),await S()}}async generateAndUploadDeclaredClientSideImages(e,t,a={}){let r=this._captureGenerationGuard(),n=await r.waitFor(this.generateDeclaredClientSideImages(e,a));if(a.debug)return n;let o=new Map((await r.waitFor(r.runEffect(()=>this._context.getAuthorizedClient().generateUploadLinks(this._context.getConfigurationId(),t,n.map(e=>e.name))))).map(e=>[e.name,e.url]));return await Promise.all(n.map(async e=>{let t=o.get(e.name);if(!t)throw Error(`Presigned link for image ${e.name} not found!`);await r.waitFor(r.runEffect(()=>fetch(t,{method:"PUT",body:e.blob})))})),n}generateDeclaredClientSideImages(e,t={}){let a=this._context.domainContextSelectors.getComponentReader().readManyComponents({componentTypes:[l]},e);return this.generateImagesFromComponents(a,e,t)}generateImagesFromComponents(e,t,a={},r={}){return this._generateImagesFromComponents(e,t,a,r)}async _generateImagesFromComponents(t,a,o,l){let p=this._captureGenerationGuard(),w=!1,C=this._context.domainContextSelectors.getExportStates(),_=t.map(t=>{if(!C.includes(t.state.exportState)){let e=d(t.componentInstance.entityId)?m(t.componentInstance.entityId):"unknown entity",a=t.componentInstance.componentType.name,r=t.componentInstance.instanceName;throw Error(`Export state ${t.state.exportState} declared in component ${a}/${r} on ${e} is not a valid export state. Expected one of: ${C.join(", ")}`)}let l=t.componentInstance.entityId;if(h(l))throw Error("Virtual entities cannot be used for export");return this._context.configuratorExtensionSelectors.getExportSystem().createExportJob(a,{exportState:t.state.exportState,exportedEntityId:l,toolState:t.state.toolState,viewState:t.state.viewState,navigationState:t.state.navigationState,selectedEntityId:a.rootEntityId},async a=>{let m=this._context.domainContextSelectors.getComponentReader().readManyComponents({componentTypes:[c,g,u]},a.state);!w&&m.length>0&&(await new Promise(e=>{i({font:void 0,characters:void 0},e)}),w=!0,await this._context.configuratorExtensionSelectors.getCanvasSystem().waitForFlush());let d=this._context.configuratorExtensionSelectors.getCameraSystem().getActiveCameraConfig(a.state);if(null==d)throw Error(`No camera config is attached for export state ${t.state.exportState} and exported entity ${l}. Whenever using a client-side image component, you need to attach a camera configuration to the export state defined on that componentin the domain spec. This camera will be used to render the image.`);let h=s(d.typeSpecificConfig.cameraType).with("perspective",()=>new n(45,t.state.width/t.state.height,1,1e4)).with("orthographic",()=>new r(0,0,1e4,0,1,1e4)).exhaustive(),C=f(h);await this._context.configuratorExtensionSelectors.getCameraSystem().applyConfig(d,C,!1,{width:t.state.width,height:t.state.height,padding:{top:0,right:0,bottom:0,left:0}}),C.update(1e6),await new Promise(e=>setTimeout(e,100));let _=await p.waitFor(p.runEffect(()=>this.renderImage({width:t.state.width,height:t.state.height,fileFormat:t.state.fileFormat,backgroundColor:t.state.backgroundColor?new e(t.state.backgroundColor):void 0,camera:h})));return o.debug&&p.runEffect(()=>this._openDebugImage(_)),{blob:_,imageComponent:t,name:`${t.componentInstance.componentType.name}-${t.componentInstance.entityId}-${t.componentInstance.instanceName}`}})});if(0===_.length)return this._context.configuratorExtensionSelectors.getExportSystem().runExportJobs(_,l);let y=this._context.configuratorExtensionSelectors.getCanvasSystem(),b=y.r3fHandle$.value??await p.waitFor(y.waitForCanvas().catch(e=>{throw p.assertActive(),e})),x=this._freezeCanvas(b);try{return await this._context.configuratorExtensionSelectors.getExportSystem().runExportJobs(_,l)}finally{await x()}}initialize=()=>{for(let e of(null!==this._generation&&(this._generation.active=!1),this._debugObjectUrls))URL.revokeObjectURL(e);this._debugObjectUrls.clear(),this._generation={active:!0}}}
+import{filter as e,firstValueFrom as t}from"rxjs";import{Color as r,HalfFloatType as a,NoToneMapping as n,OrthographicCamera as o,PerspectiveCamera as i,WebGLRenderTarget as s}from"three";import{preloadFont as l}from"troika-three-text";import{match as c}from"ts-pattern";import{clientSideImageComponent as m,dimensionIndicatorComponent as g,domainEntityTypeFromId as d,editableDistanceDimensionIndicatorComponent as h,isDomainEntityId as u,isVirtualEntityId as f,sketchingEditComponent as p}from"#package/domain";import{createCameraControls as w}from"../camera-control-system/create-camera-controls.js";import{createCanvasFreezeFrame as C}from"./canvas-freeze-frame.js";import{captureNextFrame as _}from"./capture-next-frame.js";import{ExportCapturePass as y,readTargetToBlob as x,resolveExportBackground as b}from"./export-image-capture.js";export class ImageRenderingSystem{_context;_renderer=null;_effectComposer=null;_generation=null;_debugObjectUrls=new Set;_canvasFreeze=null;constructor(e){this._context=e}_captureGenerationAssertion=()=>{let e=this._generation,t=()=>{if(!e?.active)throw Error("Image rendering generation ended while images were being rendered! The configurator context was reinitialized before the renderer finished, so the results belong to a configuration that is no longer active. Await the render before leaving the configuration, or discard the rejection.")};return t(),t};_captureGenerationGuard=()=>{let e=this._captureGenerationAssertion();return{assertActive:e,waitFor:async t=>{let r=await t;return e(),r},runEffect:t=>(e(),t())}};register(e){return this._renderer=e,()=>{this._renderer===e&&(this._renderer=null)}}registerEffectComposer(e){return this._effectComposer=e,()=>{this._effectComposer===e&&(this._effectComposer=null)}}_openDebugImage=e=>{let t=URL.createObjectURL(e);this._debugObjectUrls.add(t),window.open(t,"_blank")};_getRenderer=async()=>{let e=this._generation,t=Date.now(),r=()=>{if(!e?.active)throw Error("Image rendering generation ended while waiting for the renderer! The configurator context was reinitialized before the renderer arrived, so the render would target a scene that is gone. Await the render before leaving the configuration, or discard the rejection.")};r();let a=this._renderer;for(;null===a;){if(await new Promise(e=>setTimeout(e,50)),r(),Date.now()-t>1e4)throw Error("Renderer could not be retrieved! Reached timeout.");a=this._renderer}return a};captureView=async()=>{let{getCanvasSystem:r,getCameraControlSystem:a,getExportSystem:n}=this._context.configuratorExtensionSelectors,{perspectiveCamera:o,orthographicCamera:i}=a();return await t(n().exporting$.pipe(e(e=>!e))),_(await r().waitForCanvas(),({camera:e,frameloop:t})=>"never"!==t&&(e===o||e===i))};generateImageUsingOriginalCanvas=async(e,t,r=!1)=>{let a=this._captureGenerationGuard(),n=await a.waitFor(this._getRenderer());if(r){let{blob:e}=await a.waitFor(a.runEffect(()=>n.renderUsingOriginalCanvas(t)));a.runEffect(()=>this._openDebugImage(e));return}let o=await Promise.all([t].map(async e=>{let{blob:t}=await a.waitFor(a.runEffect(()=>n.renderUsingOriginalCanvas(e)));return{blob:t,name:e.name}})),i=await a.waitFor(a.runEffect(()=>this._context.getAuthorizedClient().generateUploadLinks(this._context.getConfigurationId(),e,[t.name])));await Promise.all(i.map(async e=>{let t=o.find(t=>t.name===e.name);if(null==t)throw Error("Blob not found!");await Promise.all(i.map(async e=>{await a.waitFor(a.runEffect(()=>fetch(e.url,{method:"PUT",body:t.blob,headers:{"Content-Type":"image/png","Content-Length":t.blob.size.toString()}})))}))}))};async generateImagesUsingOriginalCanvas(e,t,r=!1){let a=this._captureGenerationGuard(),n=await a.waitFor(this._getRenderer()),o=Array.isArray(t)?t:[t],i=await o.reduce(async(e,t)=>[...await e,await a.waitFor(a.runEffect(()=>n.renderUsingOriginalCanvas(t)))],Promise.resolve([]));if(r){for(let e of i)a.runEffect(()=>this._openDebugImage(e.blob)),await a.waitFor(new Promise(e=>setTimeout(e,100)));return}let s=o.map(e=>e.name),l=await a.waitFor(a.runEffect(()=>this._context.getAuthorizedClient().generateUploadLinks(this._context.getConfigurationId(),e,s)));await Promise.all(l.map(async e=>{let t=i.find(t=>t.name===e.name);if(null==t)throw Error("Blob not found!");await Promise.all(l.map(async e=>{await a.waitFor(a.runEffect(()=>fetch(e.url,{method:"PUT",body:t.blob,headers:{"Content-Type":"image/png","Content-Length":t.blob.size.toString()}})))}))}))}async generateImages(e,t,r=!1){let a=this._captureGenerationGuard(),n=await a.waitFor(this._getRenderer()),o=t.map(e=>e.name);if(r)return void await Promise.all(t.map(async e=>{let t=await a.waitFor(a.runEffect(()=>n.renderImage(e)));a.runEffect(()=>this._openDebugImage(t))}));let i=await Promise.all(t.map(async e=>({blob:await a.waitFor(a.runEffect(()=>n.renderImage(e))),name:e.name}))),s=await a.waitFor(a.runEffect(()=>this._context.getAuthorizedClient().generateUploadLinks(this._context.getConfigurationId(),e,o)));await Promise.all(s.map(async e=>{let t=i.find(t=>t.name===e.name);if(null==t)throw Error("Blob not found!");await Promise.all(s.map(async e=>{await a.waitFor(a.runEffect(()=>fetch(e.url,{method:"PUT",body:t.blob,headers:{"Content-Type":"image/png","Content-Length":t.blob.size.toString()}})))}))}))}async getGeneratedImages(e,t=!1){let r=this._captureGenerationAssertion(),a=await this._getRenderer();return(r(),t)?(await Promise.all(e.map(async e=>{let t=await a.renderImage(e);r(),this._openDebugImage(t)})),[]):Promise.all(e.map(async e=>{let t=await a.renderImage(e);return r(),{blob:t,name:e.name}}))}_renderCurrentView=e=>{let{gl:t,scene:r,camera:a}=e.get();null!==this._effectComposer?this._effectComposer.render():t.render(r,a)};_freezeCanvas=e=>{this._canvasFreeze?.handle===e?this._canvasFreeze.holders+=1:(this._canvasFreeze?.remove(),this._canvasFreeze={handle:e,remove:C(e.get().gl.domElement,()=>this._renderCurrentView(e)),resumeFrameloop:this._context.configuratorExtensionSelectors.getExportSystem().pauseFrameloop(e),holders:1});let t=this._canvasFreeze,r=!1;return async()=>{if(!r&&(r=!0,t.holders-=1,!(t.holders>0)))try{let r=this._context.configuratorExtensionSelectors.getCanvasSystem();r.r3fHandle$.value===e&&await r.waitForFlush(),r.r3fHandle$.value===e&&0===t.holders&&this._renderCurrentView(e)}catch{}finally{0===t.holders&&(t.remove(),t.resumeFrameloop(),this._canvasFreeze===t&&(this._canvasFreeze=null))}}};async renderImage(e){let t=this._context.configuratorExtensionSelectors.getCanvasSystem(),o=t.r3fHandle$.value??await t.waitForCanvas(),{gl:i,scene:l,camera:c,size:m,advance:g}=o.get(),d=i.getRenderTarget(),h=i.getClearColor(new r),u=i.getClearAlpha(),f=i.autoClearColor,p=t=>new y(e.width,e.height,t,i.outputColorSpace),w={composer:null,pass:null},C=null,_=null,v=e=>{let t=w.pass??p(n);return w.pass=t,w.composer===e&&e.passes.at(-1)===t||(w.composer?.removePass(t),e.addPass(t),w.composer=e),t},E=async()=>{if(await this._context.configuratorExtensionSelectors.getCanvasSystem().waitForFlush(),this._context.configuratorExtensionSelectors.getCanvasSystem().r3fHandle$.value!==o)throw Error("Canvas was remounted while an image was being rendered! The export borrowed a canvas that is gone, so it cannot capture the image. Retry the export once the configurator has settled.")},S=this._freezeCanvas(o);try{o.set({size:{height:e.height,width:e.width,top:0,left:0,updateStyle:!1}}),o.set({camera:e.camera}),await E(),null!==this._effectComposer&&v(this._effectComposer),g(void 0),await E();let t="JPEG"===e.fileFormat?"image/jpeg":"image/png",n=b({sceneBackground:l.background,backgroundColor:e.backgroundColor,fileFormat:e.fileFormat,outputColorSpace:i.outputColorSpace});if(null!==this._effectComposer){i.setRenderTarget(null),"JPEG"===e.fileFormat?i.setClearColor(e.backgroundColor??0xffffff):e.backgroundColor?i.setClearColor(e.backgroundColor):i.setClearAlpha(0),i.autoClearColor=!0;let r=v(this._effectComposer);return this._effectComposer.render(),await x(i,r.renderTarget,n,t)}C=p(i.toneMapping),_=new s(e.width,e.height,{samples:4,type:a,stencilBuffer:i.getContextAttributes().stencil??!1});let c=l.background;c instanceof r&&(l.background=null);try{i.setClearColor(0,0),i.setRenderTarget(_),i.clear(),i.render(l,e.camera)}finally{l.background=c}return C.render(i,_),await x(i,C.renderTarget,n,t)}finally{null!==w.pass&&(w.composer?.removePass(w.pass),w.pass.dispose()),C?.dispose(),_?.dispose(),this._context.configuratorExtensionSelectors.getCanvasSystem().r3fHandle$.value===o&&(i.setClearColor(h,u),i.autoClearColor=f,i.setRenderTarget(d),o.set({camera:c,size:{...m,updateStyle:!1}})),await S()}}async generateAndUploadDeclaredClientSideImages(e,t,r={}){let a=this._captureGenerationGuard(),n=await a.waitFor(this.generateDeclaredClientSideImages(e,r));if(r.debug)return n;let o=new Map((await a.waitFor(a.runEffect(()=>this._context.getAuthorizedClient().generateUploadLinks(this._context.getConfigurationId(),t,n.map(e=>e.name))))).map(e=>[e.name,e.url]));return await Promise.all(n.map(async e=>{let t=o.get(e.name);if(!t)throw Error(`Presigned link for image ${e.name} not found!`);await a.waitFor(a.runEffect(()=>fetch(t,{method:"PUT",body:e.blob})))})),n}generateDeclaredClientSideImages(e,t={}){let r=this._context.domainContextSelectors.getComponentReader().readManyComponents({componentTypes:[m]},e);return this.generateImagesFromComponents(r,e,t)}generateImagesFromComponents(e,t,r={},a={}){return this._generateImagesFromComponents(e,t,r,a)}async _generateImagesFromComponents(e,t,a,n){let s=this._captureGenerationGuard(),m=!1,C=this._context.domainContextSelectors.getExportStates(),_=e.map(e=>{if(!C.includes(e.state.exportState)){let t=u(e.componentInstance.entityId)?d(e.componentInstance.entityId):"unknown entity",r=e.componentInstance.componentType.name,a=e.componentInstance.instanceName;throw Error(`Export state ${e.state.exportState} declared in component ${r}/${a} on ${t} is not a valid export state. Expected one of: ${C.join(", ")}`)}let n=e.componentInstance.entityId;if(f(n))throw Error("Virtual entities cannot be used for export");return this._context.configuratorExtensionSelectors.getExportSystem().createExportJob(t,{exportState:e.state.exportState,exportedEntityId:n,toolState:e.state.toolState,viewState:e.state.viewState,navigationState:e.state.navigationState,selectedEntityId:t.rootEntityId},async t=>{let d=this._context.domainContextSelectors.getComponentReader().readManyComponents({componentTypes:[g,h,p]},t.state);!m&&d.length>0&&(await new Promise(e=>{l({font:void 0,characters:void 0},e)}),m=!0,await this._context.configuratorExtensionSelectors.getCanvasSystem().waitForFlush());let u=this._context.configuratorExtensionSelectors.getCameraSystem().getActiveCameraConfig(t.state);if(null==u)throw Error(`No camera config is attached for export state ${e.state.exportState} and exported entity ${n}. Whenever using a client-side image component, you need to attach a camera configuration to the export state defined on that componentin the domain spec. This camera will be used to render the image.`);let f=c(u.typeSpecificConfig.cameraType).with("perspective",()=>new i(45,e.state.width/e.state.height,1,1e4)).with("orthographic",()=>new o(0,0,1e4,0,1,1e4)).exhaustive(),C=w(f);await this._context.configuratorExtensionSelectors.getCameraSystem().applyConfig(u,C,!1,{width:e.state.width,height:e.state.height,padding:{top:0,right:0,bottom:0,left:0}}),C.update(1e6),await new Promise(e=>setTimeout(e,100));let _=await s.waitFor(s.runEffect(()=>this.renderImage({width:e.state.width,height:e.state.height,fileFormat:e.state.fileFormat,backgroundColor:e.state.backgroundColor?new r(e.state.backgroundColor):void 0,camera:f})));return a.debug&&s.runEffect(()=>this._openDebugImage(_)),{blob:_,imageComponent:e,name:`${e.componentInstance.componentType.name}-${e.componentInstance.entityId}-${e.componentInstance.instanceName}`}})});if(0===_.length)return this._context.configuratorExtensionSelectors.getExportSystem().runExportJobs(_,n);let y=this._context.configuratorExtensionSelectors.getCanvasSystem(),x=y.r3fHandle$.value??await s.waitFor(y.waitForCanvas().catch(e=>{throw s.assertActive(),e})),b=this._freezeCanvas(x);try{return await this._context.configuratorExtensionSelectors.getExportSystem().runExportJobs(_,n)}finally{await b()}}initialize=()=>{for(let e of(null!==this._generation&&(this._generation.active=!1),this._debugObjectUrls))URL.revokeObjectURL(e);this._debugObjectUrls.clear(),this._generation={active:!0}}}