@salsita-ai/sdk 0.50.0-develop.3f44338ee8 → 0.50.0-develop.4599bc222f

package/dist/client/authorized-client.d.ts

@@ -1,6 +1,6 @@
 import type { AccessLevel } from '@salsita-ai/contracts/api';
 import { type ChatStreamChunk } from '@salsita-ai/contracts/chat';
-import { type AssistantChatConnection, type AssistantEarlierThread, type AssistantThreadCompaction, type AuthorizedPhoenixApiClient, type ChatToolCall, type ChatToolCallResult, type ConfigurationPayload, type ConfigurationSchema, type waitForFinalizationResultSettings } from './contract.js';
+import { type AssistantChatConnection, type AssistantEarlierThread, type AssistantImageUpload, type AssistantThreadCompaction, type AuthorizedPhoenixApiClient, type ChatToolCall, type ChatToolCallResult, type ConfigurationPayload, type ConfigurationSchema, type waitForFinalizationResultSettings } from './contract.js';
 import type { components } from './phoenix-api.js';
 export declare class AuthorizedClient<FINALIZE_DATA extends Record<string, unknown>, FINALIZE_RESULT extends Record<string, unknown>, FRONTEND_DATA extends Record<string, unknown>> implements AuthorizedPhoenixApiClient<FINALIZE_DATA, FINALIZE_RESULT, FRONTEND_DATA> {
     private readonly _baseUrl;
@@ -28,6 +28,8 @@ export declare class AuthorizedClient<FINALIZE_DATA extends Record<string, unkno
     getAssistantChatConnection: (configurationId: string) => AssistantChatConnection;
     compactAssistantThread: (configurationId: string) => Promise<AssistantThreadCompaction>;
     getAssistantEarlierThread: (configurationId: string, threadId: string, limit: number) => Promise<AssistantEarlierThread>;
+    uploadAssistantImage: (configurationId: string, image: Blob) => Promise<AssistantImageUpload>;
+    getAssistantImage: (configurationId: string, imageId: string) => Promise<Blob>;
     sendMessage: (configurationId: string, content: string, context?: Record<string, string>) => Promise<ReadableStream<ChatStreamChunk>>;
     sendToolCallResults: (configurationId: string, calls: ChatToolCall[], results: ChatToolCallResult[], agent?: string) => Promise<ReadableStream<ChatStreamChunk>>;
     generateShareLink: (configurationId: string, revisionId: string, expiration: number, access: AccessLevel, baseUrl: string | undefined) => Promise<{

package/dist/client/authorized-client.js

@@ -1 +1 @@
-import{match as t}from"ts-pattern";import{InfoError as a}from"@salsita-ai/common/utils";import{isChatError as e}from"@salsita-ai/contracts/chat";import{createApiBaseClient as r}from"./base-client.js";import{AssistantRequestError as i,InvalidAccessKeyError as o}from"./contract.js";let n=()=>new TransformStream({transform(t,a){a.enqueue(t)}}),s=async(t,a)=>{for(;;){let{done:r,value:i}=await t.read();if(r){await a.close();break}for(let t of i.split("\n").filter(t=>""!==t.trim()).map(t=>JSON.parse(t))){if(e(t))return void await a.abort(Error(t.error));await a.write(t)}}},h=(e,r)=>{let i=`${r} API request failed: ${JSON.stringify(e)}`;return t(e).with({statusCode:403,invalidAccessKey:!0},()=>new o).with({statusCode:403},()=>new a(i)).otherwise(()=>Error(i))};export class AuthorizedClient{_baseUrl;_getAccessToken;_getSessionToken;_client;constructor(t,a,e){this._baseUrl=t,this._getAccessToken=a,this._getSessionToken=e,this._client=r(this._baseUrl)}_authHeaders=async()=>{let t=this._getAccessToken(),a=await this._getSessionToken();return{"X-Access-Key":t,Authorization:`Bearer ${a}`}};getConfiguration=async t=>{let a=await this._authHeaders(),e=await this._client.GET("/configurations/{configurationId}",{params:{path:{configurationId:t}},headers:a});if(e.error)throw h(e.error,"getConfiguration");return e.data};updateConfiguration=async(t,a)=>{let e=await this._authHeaders(),r=await this._client.PUT("/configurations/{configurationId}",{params:{path:{configurationId:t}},body:a,headers:e});if(r.error)throw h(r.error,"updateConfiguration");return r.data};finalize=async(t,a,e,r,i)=>{let o=await this._authHeaders(),n=await this._client.POST("/configurations/{configurationId}/{finalizationUuid}/finalize",{params:{path:{configurationId:t,finalizationUuid:a},query:{locale:r}},body:{finalizeData:e,frontendData:i},headers:o});if(n.error)throw h(n.error,"finalize")};waitForFinalizationToComplete=async(t,a,e)=>{let r=e?.pollingFrequency??500,i=Math.ceil((e?.pollingTimeout??3e4)/r);for(let e=0;e<i;e++){let e=await this._authHeaders(),i=await this._client.GET("/configurations/{configurationId}/{finalizationUuid}/state",{params:{path:{configurationId:t,finalizationUuid:a}},headers:e});if(i.error)throw h(i.error,"getFinalizeJobState");let o=i.data;if("completed"===o.state)return o.result[0];if("failed"===o.state)throw Error("finalization failed");if("unknown"===o.state)throw Error("finalization state is unknown");await new Promise(t=>setTimeout(t,r))}throw Error("finalization did not complete within the specified timeout")};getConfigurationRevision=async(t,a)=>{let e=await this._authHeaders(),r=await this._client.GET("/configurations/{configurationId}/revisions/{revisionId}",{params:{path:{configurationId:t,revisionId:a}},headers:e});if(r.error)throw h(r.error,"getConfigurationRevision");return r.data};getChatConversation=async t=>{let a=await this._authHeaders(),e=await this._client.GET("/configurations/{configurationId}/chat",{params:{path:{configurationId:t}},headers:a});if(e.error)throw h(e.error,"getChatConversation");return e.data};getSpeechToText=async(t,a)=>{let e=await this._authHeaders(),r=new FormData;r.append("voiceFile",a,"voice.wav");let i=await this._client.POST("/configurations/{configurationId}/chat/stt",{params:{path:{configurationId:t}},headers:e,body:r,bodySerializer:()=>r});if(i.error)throw h(i.error,"getSpeechToText");return i.data.text};getTextToSpeech=async(t,a)=>{let e=await this._authHeaders(),r=await this._client.POST("/configurations/{configurationId}/chat/tts",{params:{path:{configurationId:t}},body:{content:a},headers:e,parseAs:"arrayBuffer"});if(r.error)throw h(r.error,"getTextToSpeech");return new Blob([r.data],{type:"audio/mp3"})};getAssistantChatConnection=t=>({url:new URL(`configurations/${encodeURIComponent(t)}/assistant/chat`,`${this._baseUrl}/`).toString(),headers:this._authHeaders});compactAssistantThread=async t=>{let a=await this._authHeaders(),e=await this._client.POST("/configurations/{configurationId}/assistant/chat/compact",{params:{path:{configurationId:t}},headers:{...a,"Content-Type":null}});if(e.error)throw new i(`compactAssistantThread API request failed: ${JSON.stringify(e.error)}`,e.response.status);return e.data};getAssistantEarlierThread=async(t,a,e)=>{let r=await this._authHeaders(),o=await this._client.GET("/configurations/{configurationId}/assistant/chat/threads/{threadId}",{params:{path:{configurationId:t,threadId:a},query:{limit:e}},headers:r});if(o.error)throw new i(`getAssistantEarlierThread API request failed: ${JSON.stringify(o.error)}`,o.response.status);let{messages:n}=o.data;return{threadId:a,messages:Array.isArray(n)?n:[],previousThreadId:o.response.headers.get("x-assistant-previous-thread")}};sendMessage=async(t,a,e)=>{let r=await this._authHeaders(),i=await this._client.POST("/configurations/{configurationId}/chat/message",{params:{path:{configurationId:t}},body:{content:a,context:e},headers:r,parseAs:"stream"});if(i.error)throw h(i.error,"sendMessage");let o=i.data?.pipeThrough(new TextDecoderStream).getReader();if(!o)throw Error("sendMessage API response did not contain a readable stream");let c=n();return s(o,c.writable.getWriter()),c.readable};sendToolCallResults=async(t,a,e,r)=>{let i=await this._authHeaders(),o=await this._client.POST("/configurations/{configurationId}/chat/tool-calls",{params:{path:{configurationId:t}},body:{agent:r,calls:a,results:e},headers:i,parseAs:"stream"});if(o.error)throw h(o.error,"sendToolCallResults");let c=o.data?.pipeThrough(new TextDecoderStream).getReader();if(!c)throw Error("sendMessage API response did not contain a readable stream");let u=n();return s(c,u.writable.getWriter()),u.readable};generateShareLink=async(t,a,e,r,i)=>{let o=await this._authHeaders(),n=await this._client.POST("/configurations/{configurationId}/share-link",{params:{path:{configurationId:t}},body:{revisionId:a,expiration:e,access:r,baseUrl:i},headers:o});if(n.error)throw h(n.error,"generateShareLink");return n.data};generateUploadLinks=async(t,a,e)=>{let r=await this._authHeaders(),i=await this._client.POST("/configurations/{configurationId}/{finalizationUuid}/image-upload-links",{params:{path:{configurationId:t,finalizationUuid:a}},body:{names:e},headers:r});if(i.error)throw h(i.error,"generateUploadLinks");return i.data.links};getConfigurationPublicUploadLink=async(t,a)=>{let e=await this._authHeaders(),r=await this._client.GET("/configurations/{configurationId}/resources/{key}/public/upload",{params:{path:{configurationId:t,key:a}},headers:e});if(r.error)throw h(r.error,"getConfigurationPublicUploadLink");return r.data};getConfigurationPrivateUploadLink=async(t,a)=>{let e=await this._authHeaders(),r=await this._client.GET("/configurations/{configurationId}/resources/{key}/private/upload",{params:{path:{configurationId:t,key:a}},headers:e});if(r.error)throw h(r.error,"getConfigurationPrivateUploadLink");return r.data};getConfigurationPrivateResourceLink=async(t,a)=>{let e=await this._authHeaders(),r=await this._client.GET("/configurations/{configurationId}/resources/{key}/private",{params:{path:{configurationId:t,key:a}},headers:e});if(r.error)throw h(r.error,"getConfigurationPrivateUploadLink");return r.data};cloneConfiguration=async(t,a)=>{let e=await this._authHeaders(),r=await this._client.POST("/configurations/{configurationId}/revisions/{revisionId}/clone",{params:{path:{configurationId:t,revisionId:a}},headers:{...e,"Content-Type":null}});if(r.error)throw h(r.error,"cloneConfiguration");return r.data}}
+import{match as t}from"ts-pattern";import{InfoError as a}from"@salsita-ai/common/utils";import{isChatError as e}from"@salsita-ai/contracts/chat";import{createApiBaseClient as r}from"./base-client.js";import{AssistantRequestError as i,InvalidAccessKeyError as s}from"./contract.js";let o=()=>new TransformStream({transform(t,a){a.enqueue(t)}}),n=async(t,a)=>{for(;;){let{done:r,value:i}=await t.read();if(r){await a.close();break}for(let t of i.split("\n").filter(t=>""!==t.trim()).map(t=>JSON.parse(t))){if(e(t))return void await a.abort(Error(t.error));await a.write(t)}}},h=(e,r)=>{let i=`${r} API request failed: ${JSON.stringify(e)}`;return t(e).with({statusCode:403,invalidAccessKey:!0},()=>new s).with({statusCode:403},()=>new a(i)).otherwise(()=>Error(i))};export class AuthorizedClient{_baseUrl;_getAccessToken;_getSessionToken;_client;constructor(t,a,e){this._baseUrl=t,this._getAccessToken=a,this._getSessionToken=e,this._client=r(this._baseUrl)}_authHeaders=async()=>{let t=this._getAccessToken(),a=await this._getSessionToken();return{"X-Access-Key":t,Authorization:`Bearer ${a}`}};getConfiguration=async t=>{let a=await this._authHeaders(),e=await this._client.GET("/configurations/{configurationId}",{params:{path:{configurationId:t}},headers:a});if(e.error)throw h(e.error,"getConfiguration");return e.data};updateConfiguration=async(t,a)=>{let e=await this._authHeaders(),r=await this._client.PUT("/configurations/{configurationId}",{params:{path:{configurationId:t}},body:a,headers:e});if(r.error)throw h(r.error,"updateConfiguration");return r.data};finalize=async(t,a,e,r,i)=>{let s=await this._authHeaders(),o=await this._client.POST("/configurations/{configurationId}/{finalizationUuid}/finalize",{params:{path:{configurationId:t,finalizationUuid:a},query:{locale:r}},body:{finalizeData:e,frontendData:i},headers:s});if(o.error)throw h(o.error,"finalize")};waitForFinalizationToComplete=async(t,a,e)=>{let r=e?.pollingFrequency??500,i=Math.ceil((e?.pollingTimeout??3e4)/r);for(let e=0;e<i;e++){let e=await this._authHeaders(),i=await this._client.GET("/configurations/{configurationId}/{finalizationUuid}/state",{params:{path:{configurationId:t,finalizationUuid:a}},headers:e});if(i.error)throw h(i.error,"getFinalizeJobState");let s=i.data;if("completed"===s.state)return s.result[0];if("failed"===s.state)throw Error("finalization failed");if("unknown"===s.state)throw Error("finalization state is unknown");await new Promise(t=>setTimeout(t,r))}throw Error("finalization did not complete within the specified timeout")};getConfigurationRevision=async(t,a)=>{let e=await this._authHeaders(),r=await this._client.GET("/configurations/{configurationId}/revisions/{revisionId}",{params:{path:{configurationId:t,revisionId:a}},headers:e});if(r.error)throw h(r.error,"getConfigurationRevision");return r.data};getChatConversation=async t=>{let a=await this._authHeaders(),e=await this._client.GET("/configurations/{configurationId}/chat",{params:{path:{configurationId:t}},headers:a});if(e.error)throw h(e.error,"getChatConversation");return e.data};getSpeechToText=async(t,a)=>{let e=await this._authHeaders(),r=new FormData;r.append("voiceFile",a,"voice.wav");let i=await this._client.POST("/configurations/{configurationId}/chat/stt",{params:{path:{configurationId:t}},headers:e,body:r,bodySerializer:()=>r});if(i.error)throw h(i.error,"getSpeechToText");return i.data.text};getTextToSpeech=async(t,a)=>{let e=await this._authHeaders(),r=await this._client.POST("/configurations/{configurationId}/chat/tts",{params:{path:{configurationId:t}},body:{content:a},headers:e,parseAs:"arrayBuffer"});if(r.error)throw h(r.error,"getTextToSpeech");return new Blob([r.data],{type:"audio/mp3"})};getAssistantChatConnection=t=>({url:new URL(`configurations/${encodeURIComponent(t)}/assistant/chat`,`${this._baseUrl}/`).toString(),headers:this._authHeaders});compactAssistantThread=async t=>{let a=await this._authHeaders(),e=await this._client.POST("/configurations/{configurationId}/assistant/chat/compact",{params:{path:{configurationId:t}},headers:{...a,"Content-Type":null}});if(e.error)throw new i(`compactAssistantThread API request failed: ${JSON.stringify(e.error)}`,e.response.status);return e.data};getAssistantEarlierThread=async(t,a,e)=>{let r=await this._authHeaders(),s=await this._client.GET("/configurations/{configurationId}/assistant/chat/threads/{threadId}",{params:{path:{configurationId:t,threadId:a},query:{limit:e}},headers:r});if(s.error)throw new i(`getAssistantEarlierThread API request failed: ${JSON.stringify(s.error)}`,s.response.status);let{messages:o}=s.data;return{threadId:a,messages:Array.isArray(o)?o:[],previousThreadId:s.response.headers.get("x-assistant-previous-thread")}};uploadAssistantImage=async(t,a)=>{let e=await this._authHeaders(),r=await this._client.POST("/configurations/{configurationId}/assistant/images",{params:{path:{configurationId:t}},headers:{...e,"Content-Type":a.type},body:a.type,bodySerializer:()=>a});if(r.error)throw new i(`uploadAssistantImage API request failed: ${JSON.stringify(r.error)}`,r.response.status);return{imageId:r.data.imageId}};getAssistantImage=async(t,a)=>{let e=await this._authHeaders(),r=await this._client.GET("/configurations/{configurationId}/assistant/images/{imageId}",{params:{path:{configurationId:t,imageId:a}},headers:e,parseAs:"blob"});if(r.error)throw new i(`getAssistantImage API request failed: ${JSON.stringify(r.error)}`,r.response.status);return r.data};sendMessage=async(t,a,e)=>{let r=await this._authHeaders(),i=await this._client.POST("/configurations/{configurationId}/chat/message",{params:{path:{configurationId:t}},body:{content:a,context:e},headers:r,parseAs:"stream"});if(i.error)throw h(i.error,"sendMessage");let s=i.data?.pipeThrough(new TextDecoderStream).getReader();if(!s)throw Error("sendMessage API response did not contain a readable stream");let u=o();return n(s,u.writable.getWriter()),u.readable};sendToolCallResults=async(t,a,e,r)=>{let i=await this._authHeaders(),s=await this._client.POST("/configurations/{configurationId}/chat/tool-calls",{params:{path:{configurationId:t}},body:{agent:r,calls:a,results:e},headers:i,parseAs:"stream"});if(s.error)throw h(s.error,"sendToolCallResults");let u=s.data?.pipeThrough(new TextDecoderStream).getReader();if(!u)throw Error("sendMessage API response did not contain a readable stream");let c=o();return n(u,c.writable.getWriter()),c.readable};generateShareLink=async(t,a,e,r,i)=>{let s=await this._authHeaders(),o=await this._client.POST("/configurations/{configurationId}/share-link",{params:{path:{configurationId:t}},body:{revisionId:a,expiration:e,access:r,baseUrl:i},headers:s});if(o.error)throw h(o.error,"generateShareLink");return o.data};generateUploadLinks=async(t,a,e)=>{let r=await this._authHeaders(),i=await this._client.POST("/configurations/{configurationId}/{finalizationUuid}/image-upload-links",{params:{path:{configurationId:t,finalizationUuid:a}},body:{names:e},headers:r});if(i.error)throw h(i.error,"generateUploadLinks");return i.data.links};getConfigurationPublicUploadLink=async(t,a)=>{let e=await this._authHeaders(),r=await this._client.GET("/configurations/{configurationId}/resources/{key}/public/upload",{params:{path:{configurationId:t,key:a}},headers:e});if(r.error)throw h(r.error,"getConfigurationPublicUploadLink");return r.data};getConfigurationPrivateUploadLink=async(t,a)=>{let e=await this._authHeaders(),r=await this._client.GET("/configurations/{configurationId}/resources/{key}/private/upload",{params:{path:{configurationId:t,key:a}},headers:e});if(r.error)throw h(r.error,"getConfigurationPrivateUploadLink");return r.data};getConfigurationPrivateResourceLink=async(t,a)=>{let e=await this._authHeaders(),r=await this._client.GET("/configurations/{configurationId}/resources/{key}/private",{params:{path:{configurationId:t,key:a}},headers:e});if(r.error)throw h(r.error,"getConfigurationPrivateUploadLink");return r.data};cloneConfiguration=async(t,a)=>{let e=await this._authHeaders(),r=await this._client.POST("/configurations/{configurationId}/revisions/{revisionId}/clone",{params:{path:{configurationId:t,revisionId:a}},headers:{...e,"Content-Type":null}});if(r.error)throw h(r.error,"cloneConfiguration");return r.data}}

package/dist/client/contract.d.ts

@@ -100,6 +100,18 @@ export interface AuthorizedPhoenixApiClient<FINALIZE_DATA extends Record<string,
      * `limit` of its newest messages. Rejects with an {@link AssistantRequestError} when the server refuses.
      */
     getAssistantEarlierThread: (configurationId: string, threadId: string, limit: number) => Promise<AssistantEarlierThread>;
+    /**
+     * Stores a screenshot the assistant took in the open assistant thread of the configuration, while a
+     * reply runs there, and resolves with the id its tool result references it by. The image is the
+     * rendered JPEG or PNG blob, sent with its own type. Rejects with an {@link AssistantRequestError}
+     * when the server refuses.
+     */
+    uploadAssistantImage: (configurationId: string, image: Blob) => Promise<AssistantImageUpload>;
+    /**
+     * Reads a screenshot of an assistant thread of the configuration by the id its tool result
+     * references it by. Rejects with an {@link AssistantRequestError} when the server refuses.
+     */
+    getAssistantImage: (configurationId: string, imageId: string) => Promise<Blob>;
     getSpeechToText: (configurationId: string, blob: Blob) => Promise<string>;
     getTextToSpeech: (configurationId: string, text: string) => Promise<Blob>;
     sendMessage: (configurationId: string, message: string, state?: Record<string, string>) => Promise<ReadableStream<ChatStreamChunk>>;
@@ -137,6 +149,11 @@ export interface AssistantThreadCompaction {
     /** The closed thread the new one continues. */
     readonly previousThreadId: string;
 }
+/** A screenshot stored in an assistant thread. */
+export interface AssistantImageUpload {
+    /** The id a tool result references the image by, as `assistant-image:<imageId>`. */
+    readonly imageId: string;
+}
 /** A closed assistant thread, as the panel paints it above the thread that continues it. */
 export interface AssistantEarlierThread {
     readonly threadId: string;

package/dist/client/index.d.ts

@@ -9,5 +9,5 @@ export { MemoryAuthorizedPhoenixApiClient } from './memory-authorized-client.js'
 export { MemoryAuthenticatedClientPhoenixApiClient, memoryStorage } from './memory-authenticated-client.js';
 export { createUnsignedJWT } from './unsigned-jwt.js';
 export type { components as ApiTypes } from './phoenix-api.js';
-export type { ConfigurationPayload, ConfigurationSchema, ChatConfiguration, ChatConversation, ChatEmbedding, ChatMessageParam, ChatToolCallResult, Entity, UnauthenticatedPhoenixApiClient, AuthenticatedClientPhoenixApiClient, AuthorizedPhoenixApiClient, AssistantChatConnection, AssistantEarlierThread, AssistantThreadCompaction, OAuthProvider, OAuthAccessTokenResult, GoogleOAuthConfig, ReleaseToProductionResult, SharePointSiteConfig, } from './contract.js';
+export type { ConfigurationPayload, ConfigurationSchema, ChatConfiguration, ChatConversation, ChatEmbedding, ChatMessageParam, ChatToolCallResult, Entity, UnauthenticatedPhoenixApiClient, AuthenticatedClientPhoenixApiClient, AuthorizedPhoenixApiClient, AssistantChatConnection, AssistantEarlierThread, AssistantImageUpload, AssistantThreadCompaction, OAuthProvider, OAuthAccessTokenResult, GoogleOAuthConfig, ReleaseToProductionResult, SharePointSiteConfig, } from './contract.js';
 export { AssistantRequestError, InvalidAccessKeyError, InvalidDeploymentError, ChunkNotFoundError, OAuthStaleConnectionError, ConfigurationSchemaMismatchError, isConfigurationSchemaMismatchError, } from './contract.js';

package/dist/client/local-authorized-client.d.ts

@@ -1,5 +1,5 @@
 import type { ChatMessageContentChunk, ChatToolCalls } from '@salsita-ai/contracts/chat';
-import type { AssistantChatConnection, AssistantEarlierThread, AssistantThreadCompaction, AuthorizedPhoenixApiClient, ConfigurationPayload, ConfigurationSchema } from './contract.js';
+import type { AssistantChatConnection, AssistantEarlierThread, AssistantImageUpload, AssistantThreadCompaction, AuthorizedPhoenixApiClient, ConfigurationPayload, ConfigurationSchema } from './contract.js';
 import { LocalStorageAuthenticatedClientPhoenixApiClient } from './local-authenticated-client.js';
 /**
  * Local storage implementation for local dev, it does not check for schema changes, so you may need to create new configuration if the schema changes and
@@ -17,6 +17,8 @@ export declare class LocalStorageAuthorizedPhoenixApiClient<FINALIZE_DATA extend
     getAssistantChatConnection: () => AssistantChatConnection;
     compactAssistantThread: () => Promise<AssistantThreadCompaction>;
     getAssistantEarlierThread: () => Promise<AssistantEarlierThread>;
+    uploadAssistantImage: () => Promise<AssistantImageUpload>;
+    getAssistantImage: () => Promise<Blob>;
     getChatConversation: (configurationId: string) => Promise<readonly {
         readonly augmentation?: import("./phoenix-api.js").components['schemas']['ChatMessageAugmentation'];
         readonly content?: string;

package/dist/client/local-authorized-client.js

@@ -1 +1 @@
-import*as t from"uuid";import{ChunkType as e}from"@salsita-ai/contracts/chat";import{LocalStorageAuthenticatedClientPhoenixApiClient as o}from"./local-authenticated-client.js";let r=(t,o)=>{let r={role:"assistant",content:["I see","Yes","Definitely"][Math.round(2*Math.random())]};return localStorage.setItem(`${o}-chat`,JSON.stringify([...t,r])),Promise.resolve(new ReadableStream({start(t){t.enqueue({type:e.content,content:r.content??""}),t.close()}}))};export class LocalStorageAuthorizedPhoenixApiClient extends o{getConfigurationData=t=>{let e=JSON.parse(localStorage.getItem(t)??"[]");if(!e||0===e.length)throw Error("Configuration not found!");return e};getConfiguration=t=>{let e=this.getConfigurationData(t);return Promise.resolve(e[e.length-1])};updateConfiguration=(e,o)=>{let r=this.getConfigurationData(e),n=t.v4(),a={...o,configurationId:e,revisionId:n,modelVersion:this._modelVersion};return localStorage.setItem(e,JSON.stringify([...r,a])),Promise.resolve({revisionId:n})};finalize=()=>{throw Error("Not implemented!")};waitForFinalizationToComplete=()=>{throw Error("Not implemented!")};getConfigurationRevision=(t,e)=>{let o=this.getConfigurationData(t).find(t=>t.revisionId===e);if(!o)throw Error("Revision not found!");return Promise.resolve(o)};getAssistantChatConnection=()=>{throw Error("The assistant chat needs the Phoenix server. Run the configurator against a server deployment.")};compactAssistantThread=async()=>{throw Error("The assistant chat needs the Phoenix server. Run the configurator against a server deployment.")};getAssistantEarlierThread=async()=>{throw Error("The assistant chat needs the Phoenix server. Run the configurator against a server deployment.")};getChatConversation=async t=>JSON.parse(localStorage.getItem(`${t}-chat`)??"[]");sendMessage=async(t,e)=>{let o=await this.getChatConversation(t);return localStorage.setItem(`${t}-chat`,JSON.stringify([...o,e])),r(o,t)};getSpeechToText=async()=>{throw Error("Not implemented!")};getTextToSpeech=async()=>{throw Error("Not implemented!")};sendToolCallResults=async t=>r(await this.getChatConversation(t),t);generateShareLink=async()=>({link:"https://yourcompany.com/configurator/link-will-not-work-it-is-dummy-link"});generateUploadLinks=async()=>{throw Error("Not implemented!")};getConfigurationPublicUploadLink=async()=>{throw Error("Not implemented!")};getConfigurationPrivateUploadLink=async()=>{throw Error("Not implemented!")};getConfigurationPrivateResourceLink=async()=>{throw Error("Not implemented!")};cloneConfiguration=async(t,e)=>{let o=localStorage.getItem(t);if(!o)throw Error("Configuration not found");let r=JSON.parse(o).find(t=>t.revisionId===e);if(!r)throw Error("Revision not found");return this.createConfiguration(r)}}
+import*as t from"uuid";import{ChunkType as e}from"@salsita-ai/contracts/chat";import{LocalStorageAuthenticatedClientPhoenixApiClient as r}from"./local-authenticated-client.js";let o=(t,r)=>{let o={role:"assistant",content:["I see","Yes","Definitely"][Math.round(2*Math.random())]};return localStorage.setItem(`${r}-chat`,JSON.stringify([...t,o])),Promise.resolve(new ReadableStream({start(t){t.enqueue({type:e.content,content:o.content??""}),t.close()}}))};export class LocalStorageAuthorizedPhoenixApiClient extends r{getConfigurationData=t=>{let e=JSON.parse(localStorage.getItem(t)??"[]");if(!e||0===e.length)throw Error("Configuration not found!");return e};getConfiguration=t=>{let e=this.getConfigurationData(t);return Promise.resolve(e[e.length-1])};updateConfiguration=(e,r)=>{let o=this.getConfigurationData(e),n=t.v4(),a={...r,configurationId:e,revisionId:n,modelVersion:this._modelVersion};return localStorage.setItem(e,JSON.stringify([...o,a])),Promise.resolve({revisionId:n})};finalize=()=>{throw Error("Not implemented!")};waitForFinalizationToComplete=()=>{throw Error("Not implemented!")};getConfigurationRevision=(t,e)=>{let r=this.getConfigurationData(t).find(t=>t.revisionId===e);if(!r)throw Error("Revision not found!");return Promise.resolve(r)};getAssistantChatConnection=()=>{throw Error("The assistant chat needs the Phoenix server. Run the configurator against a server deployment.")};compactAssistantThread=async()=>{throw Error("The assistant chat needs the Phoenix server. Run the configurator against a server deployment.")};getAssistantEarlierThread=async()=>{throw Error("The assistant chat needs the Phoenix server. Run the configurator against a server deployment.")};uploadAssistantImage=async()=>{throw Error("The assistant chat needs the Phoenix server. Run the configurator against a server deployment.")};getAssistantImage=async()=>{throw Error("The assistant chat needs the Phoenix server. Run the configurator against a server deployment.")};getChatConversation=async t=>JSON.parse(localStorage.getItem(`${t}-chat`)??"[]");sendMessage=async(t,e)=>{let r=await this.getChatConversation(t);return localStorage.setItem(`${t}-chat`,JSON.stringify([...r,e])),o(r,t)};getSpeechToText=async()=>{throw Error("Not implemented!")};getTextToSpeech=async()=>{throw Error("Not implemented!")};sendToolCallResults=async t=>o(await this.getChatConversation(t),t);generateShareLink=async()=>({link:"https://yourcompany.com/configurator/link-will-not-work-it-is-dummy-link"});generateUploadLinks=async()=>{throw Error("Not implemented!")};getConfigurationPublicUploadLink=async()=>{throw Error("Not implemented!")};getConfigurationPrivateUploadLink=async()=>{throw Error("Not implemented!")};getConfigurationPrivateResourceLink=async()=>{throw Error("Not implemented!")};cloneConfiguration=async(t,e)=>{let r=localStorage.getItem(t);if(!r)throw Error("Configuration not found");let o=JSON.parse(r).find(t=>t.revisionId===e);if(!o)throw Error("Revision not found");return this.createConfiguration(o)}}

package/dist/client/memory-authorized-client.d.ts

@@ -1,4 +1,4 @@
-import type { AssistantChatConnection, AssistantEarlierThread, AssistantThreadCompaction, AuthorizedPhoenixApiClient, ConfigurationPayload, ConfigurationSchema } from './contract.js';
+import type { AssistantChatConnection, AssistantEarlierThread, AssistantImageUpload, AssistantThreadCompaction, AuthorizedPhoenixApiClient, ConfigurationPayload, ConfigurationSchema } from './contract.js';
 import { MemoryAuthenticatedClientPhoenixApiClient } from './memory-authenticated-client.js';
 /**
  * Memory implementation for local dev, it does not check for schema changes, so you may need to create new configuration if the schema changes and
@@ -17,6 +17,9 @@ export declare class MemoryAuthorizedPhoenixApiClient<FINALIZE_DATA extends Reco
     getAssistantChatConnection: () => AssistantChatConnection;
     compactAssistantThread: () => Promise<AssistantThreadCompaction>;
     getAssistantEarlierThread: () => Promise<AssistantEarlierThread>;
+    private readonly _assistantImages;
+    uploadAssistantImage: (configurationId: string, image: Blob) => Promise<AssistantImageUpload>;
+    getAssistantImage: (configurationId: string, imageId: string) => Promise<Blob>;
     sendMessage: () => Promise<never>;
     getSpeechToText: () => Promise<never>;
     getTextToSpeech: () => Promise<never>;

package/dist/client/memory-authorized-client.js

@@ -1 +1 @@
-import*as e from"uuid";import{MemoryAuthenticatedClientPhoenixApiClient as t,memoryStorage as r}from"./memory-authenticated-client.js";export class MemoryAuthorizedPhoenixApiClient extends t{getConfigurationData=e=>{let t=r.get(e);if(!t||!t||0===t.length)throw Error("Configuration not found!");return t};getConfiguration=e=>{let t=this.getConfigurationData(e);return Promise.resolve(t[t.length-1])};updateConfiguration=(t,o)=>{let n=this.getConfigurationData(t),i=e.v4(),a={...o,configurationId:t,revisionId:i,modelVersion:this._modelVersion};return r.set(t,[...n,a]),Promise.resolve({revisionId:i})};finalize=()=>{throw Error("Not implemented!")};waitForFinalizationToComplete=()=>{throw Error("Not implemented!")};getConfigurationRevision=(e,t)=>{let r=this.getConfigurationData(e).find(e=>e.revisionId===t);if(!r)throw Error("Revision not found!");return Promise.resolve(r)};getChatConversation=async()=>{throw Error("Not implemented!")};getAssistantChatConnection=()=>{throw Error("The assistant chat needs the Phoenix server. Run the configurator against a server deployment.")};compactAssistantThread=async()=>{throw Error("The assistant chat needs the Phoenix server. Run the configurator against a server deployment.")};getAssistantEarlierThread=async()=>{throw Error("The assistant chat needs the Phoenix server. Run the configurator against a server deployment.")};sendMessage=async()=>{throw Error("Not implemented!")};getSpeechToText=async()=>{throw Error("Not implemented!")};getTextToSpeech=async()=>{throw Error("Not implemented!")};sendToolCallResults=async()=>{throw Error("Not implemented!")};generateShareLink=async()=>({link:"https://yourcompany.com/configurator/link-will-not-work-it-is-dummy-link"});generateUploadLinks=async()=>{throw Error("Not implemented!")};getConfigurationPublicUploadLink=async()=>{throw Error("Not implemented!")};getConfigurationPrivateUploadLink=async()=>{throw Error("Not implemented!")};getConfigurationPrivateResourceLink=async()=>{throw Error("Not implemented!")};cloneConfiguration=async(e,t)=>{let o=r.get(e);if(!o)throw Error("Configuration not found");let n=o.find(e=>e.revisionId===t);if(!n)throw Error("Revision not found");return this.createConfiguration(n)}}
+import*as t from"uuid";import{MemoryAuthenticatedClientPhoenixApiClient as e,memoryStorage as r}from"./memory-authenticated-client.js";export class MemoryAuthorizedPhoenixApiClient extends e{getConfigurationData=t=>{let e=r.get(t);if(!e||!e||0===e.length)throw Error("Configuration not found!");return e};getConfiguration=t=>{let e=this.getConfigurationData(t);return Promise.resolve(e[e.length-1])};updateConfiguration=(e,o)=>{let n=this.getConfigurationData(e),i=t.v4(),a={...o,configurationId:e,revisionId:i,modelVersion:this._modelVersion};return r.set(e,[...n,a]),Promise.resolve({revisionId:i})};finalize=()=>{throw Error("Not implemented!")};waitForFinalizationToComplete=()=>{throw Error("Not implemented!")};getConfigurationRevision=(t,e)=>{let r=this.getConfigurationData(t).find(t=>t.revisionId===e);if(!r)throw Error("Revision not found!");return Promise.resolve(r)};getChatConversation=async()=>{throw Error("Not implemented!")};getAssistantChatConnection=()=>{throw Error("The assistant chat needs the Phoenix server. Run the configurator against a server deployment.")};compactAssistantThread=async()=>{throw Error("The assistant chat needs the Phoenix server. Run the configurator against a server deployment.")};getAssistantEarlierThread=async()=>{throw Error("The assistant chat needs the Phoenix server. Run the configurator against a server deployment.")};_assistantImages=new Map;uploadAssistantImage=async(e,r)=>{let o=t.v4();return this._assistantImages.set(`${e}/${o}`,r),{imageId:o}};getAssistantImage=async(t,e)=>{let r=this._assistantImages.get(`${t}/${e}`);if(!r)throw Error(`Assistant image ${e} was not uploaded for configuration ${t} through this client. Upload it before reading it.`);return r};sendMessage=async()=>{throw Error("Not implemented!")};getSpeechToText=async()=>{throw Error("Not implemented!")};getTextToSpeech=async()=>{throw Error("Not implemented!")};sendToolCallResults=async()=>{throw Error("Not implemented!")};generateShareLink=async()=>({link:"https://yourcompany.com/configurator/link-will-not-work-it-is-dummy-link"});generateUploadLinks=async()=>{throw Error("Not implemented!")};getConfigurationPublicUploadLink=async()=>{throw Error("Not implemented!")};getConfigurationPrivateUploadLink=async()=>{throw Error("Not implemented!")};getConfigurationPrivateResourceLink=async()=>{throw Error("Not implemented!")};cloneConfiguration=async(t,e)=>{let o=r.get(t);if(!o)throw Error("Configuration not found");let n=o.find(t=>t.revisionId===e);if(!n)throw Error("Revision not found");return this.createConfiguration(n)}}

package/dist/client/phoenix-api.d.ts

@@ -379,6 +379,40 @@ export interface paths {
     readonly patch?: never
     readonly trace?: never
   }
+  readonly '/configurations/{configurationId}/assistant/images': {
+    readonly parameters: {
+      readonly query?: never
+      readonly header?: never
+      readonly path?: never
+      readonly cookie?: never
+    }
+    readonly get?: never
+    readonly put?: never
+    /** @description Store a screenshot the browser took for the assistant in the caller's open thread on the configuration, while a run of the assistant answers there. The body is the raw JPEG or PNG image, with its type as the content type. The answer carries the id that tool results reference the image by, as `assistant-image:<imageId>`. Answers 409 when the caller has no open thread on the configuration or the thread is not in the middle of a reply (no run running and no client tool call waiting), and 415 for any other content type. */
+    readonly post: operations['uploadAssistantImage']
+    readonly delete?: never
+    readonly options?: never
+    readonly head?: never
+    readonly patch?: never
+    readonly trace?: never
+  }
+  readonly '/configurations/{configurationId}/assistant/images/{imageId}': {
+    readonly parameters: {
+      readonly query?: never
+      readonly header?: never
+      readonly path?: never
+      readonly cookie?: never
+    }
+    /** @description Load a screenshot of one of the caller's assistant threads on the configuration, the earlier threads of the chain included. The bytes never change, so the answer may be cached for good. Answers 404 for an image that is not in one of the caller's threads on this configuration. */
+    readonly get: operations['getAssistantImage']
+    readonly put?: never
+    readonly post?: never
+    readonly delete?: never
+    readonly options?: never
+    readonly head?: never
+    readonly patch?: never
+    readonly trace?: never
+  }
   readonly '/configurations/{configurationId}/chat': {
     readonly parameters: {
       readonly query?: never
@@ -1503,6 +1537,13 @@ export interface components {
       /** @enum {number} */
       readonly statusCode: 401
     }
+    /** @description Unsupported Media Type: The request body is of a content type the route does not accept. */
+    readonly UnsupportedMediaTypeError: {
+      /** @example Unsupported Media Type! */
+      readonly message: string
+      /** @enum {number} */
+      readonly statusCode: 415
+    }
     /** @description A webhook resource to which the deployment can send requests. */
     readonly Webhook: {
       /**
@@ -3397,6 +3438,193 @@ export interface operations {
       }
     }
   }
+  readonly uploadAssistantImage: {
+    readonly parameters: {
+      readonly query?: never
+      readonly header?: never
+      readonly path: {
+        /** @description The configuration on which the caller's assistant thread is used. */
+        readonly configurationId: components['schemas']['ConfigurationUuid']
+      }
+      readonly cookie?: never
+    }
+    /** @description The JPEG or PNG image. */
+    readonly requestBody: {
+      readonly content: {
+        readonly 'image/jpeg': string
+        readonly 'image/png': string
+      }
+    }
+    readonly responses: {
+      /** @description The image was stored in the thread. */
+      readonly 200: {
+        headers: {
+          readonly [name: string]: unknown
+        }
+        content: {
+          readonly 'application/json': {
+            /**
+             * Format: uuid
+             * @description The id tool results reference the image by.
+             */
+            readonly imageId: string
+          }
+        }
+      }
+      /** @description Bad Request! */
+      readonly 400: {
+        headers: {
+          readonly [name: string]: unknown
+        }
+        content: {
+          readonly 'application/json': components['schemas']['BadRequestError']
+        }
+      }
+      /** @description Unauthorized! */
+      readonly 401: {
+        headers: {
+          readonly [name: string]: unknown
+        }
+        content: {
+          readonly 'application/json': components['schemas']['UnauthorizedError']
+        }
+      }
+      /** @description Forbidden! */
+      readonly 403: {
+        headers: {
+          readonly [name: string]: unknown
+        }
+        content: {
+          readonly 'application/json': components['schemas']['ForbiddenError']
+        }
+      }
+      /** @description Not Found! */
+      readonly 404: {
+        headers: {
+          readonly [name: string]: unknown
+        }
+        content: {
+          readonly 'application/json': components['schemas']['NotFoundError']
+        }
+      }
+      /** @description Conflict! */
+      readonly 409: {
+        headers: {
+          readonly [name: string]: unknown
+        }
+        content: {
+          readonly 'application/json': components['schemas']['ConflictError']
+        }
+      }
+      /** @description Unsupported Media Type! */
+      readonly 415: {
+        headers: {
+          readonly [name: string]: unknown
+        }
+        content: {
+          readonly 'application/json': components['schemas']['UnsupportedMediaTypeError']
+        }
+      }
+      /** @description Bad Gateway! */
+      readonly 502: {
+        headers: {
+          readonly [name: string]: unknown
+        }
+        content: {
+          readonly 'application/json': components['schemas']['BadGatewayError']
+        }
+      }
+      /** @description An unexpected error occurred! */
+      readonly default: {
+        headers: {
+          readonly [name: string]: unknown
+        }
+        content: {
+          readonly 'application/json': components['schemas']['DefaultResponse']
+        }
+      }
+    }
+  }
+  readonly getAssistantImage: {
+    readonly parameters: {
+      readonly query?: never
+      readonly header?: never
+      readonly path: {
+        /** @description The configuration on which the caller's assistant thread is used. */
+        readonly configurationId: components['schemas']['ConfigurationUuid']
+        /** @description The image, as a tool result references it. */
+        readonly imageId: string
+      }
+      readonly cookie?: never
+    }
+    readonly requestBody?: never
+    readonly responses: {
+      /** @description The image. */
+      readonly 200: {
+        headers: {
+          readonly [name: string]: unknown
+        }
+        content: {
+          readonly 'image/jpeg': unknown
+          readonly 'image/png': unknown
+        }
+      }
+      /** @description Bad Request! */
+      readonly 400: {
+        headers: {
+          readonly [name: string]: unknown
+        }
+        content: {
+          readonly 'application/json': components['schemas']['BadRequestError']
+        }
+      }
+      /** @description Unauthorized! */
+      readonly 401: {
+        headers: {
+          readonly [name: string]: unknown
+        }
+        content: {
+          readonly 'application/json': components['schemas']['UnauthorizedError']
+        }
+      }
+      /** @description Forbidden! */
+      readonly 403: {
+        headers: {
+          readonly [name: string]: unknown
+        }
+        content: {
+          readonly 'application/json': components['schemas']['ForbiddenError']
+        }
+      }
+      /** @description Not Found! */
+      readonly 404: {
+        headers: {
+          readonly [name: string]: unknown
+        }
+        content: {
+          readonly 'application/json': components['schemas']['NotFoundError']
+        }
+      }
+      /** @description Bad Gateway! */
+      readonly 502: {
+        headers: {
+          readonly [name: string]: unknown
+        }
+        content: {
+          readonly 'application/json': components['schemas']['BadGatewayError']
+        }
+      }
+      /** @description An unexpected error occurred! */
+      readonly default: {
+        headers: {
+          readonly [name: string]: unknown
+        }
+        content: {
+          readonly 'application/json': components['schemas']['DefaultResponse']
+        }
+      }
+    }
+  }
   readonly getChatConversation: {
     readonly parameters: {
       readonly query?: never

package/dist/configurator/application-runtime/extensions/selectors.js

@@ -1 +1 @@
-import{createSelector as e}from"@salsita-ai/common/utils";import{createSessionAssistant as t,isAssistantSetupHandler as n}from"#package/domain";import{AnimationSystem as s,AssistantSystem as o,CameraControlSystem as r,CameraSystem as a,CanvasSystem as i,DragAndDropSystem as g,ExportSystem as m,GraphQLSystem as c,ImageRenderingSystem as l,LayoutSystem as S,SketchingSystem as d}from"./systems/index.js";export const createConfiguratorExtensionSelectors=u=>{let{getState:y,domainContextSelectors:w,specSelectors:C,domainServicesSelectors:f,accessSelectors:p,commonServicesSelectors:h}=u,x=e.withTypes()([C.selectConfiguratorSpec,w.selectInitializationContext],(e,s)=>t({resolved:e.assistant,handler:e.handlers.find(n),inputs:s&&s.access?{role:s.role,sessionData:s.sessionData,access:s.access}:null})),A={getConfigurationId:()=>p.getConfiguratorConfigurationId(),getAuthorizedClient:()=>h.getConfiguratorAuthorizedClient(),getAssistantSetup:()=>x(y()),domainContextSelectors:w,domainServicesSelectors:f,specSelectors:C},I={getSystems:()=>Q,getCanvasSystem:()=>D,getCameraControlSystem:()=>v,getAnimationSystem:()=>b,getDragAndDropSystem:()=>z,getImageRenderingSystem:()=>G,getLayoutSystem:()=>T,getSketchingSystem:()=>j,getExportSystem:()=>k,getCameraSystem:()=>F,getGraphQLSystem:()=>L,getAssistantSystem:()=>R},E={...A,configuratorExtensionSelectors:I},D=new i(A),v=new r({configuratorExtensionSelectors:I}),z=new g(E),b=new s(A),k=new m(E),L=new c({...A,renderScreenshot:async(e,t)=>{let[n]=await I.getImageRenderingSystem().generateImagesFromComponents([e],t,{},{awaitSettledState:!0});if(!n)throw Error("The canvas rendered no image.");return n.blob}}),R=new o(E),T=new S(A),j=new d(E),F=new a(E),G=new l(E),Q={cameraControlSystem:v,layoutSystem:T,sketchingSystem:j,dragAndDropSystem:z,canvasSystem:D,cameraSystem:F,exportSystem:k,imageRenderingSystem:G,animationSystem:b,graphQLSystem:L,assistantSystem:R};return I};
+import{createSelector as e}from"@salsita-ai/common/utils";import{createSessionAssistant as t,isAssistantSetupHandler as n}from"#package/domain";import{AnimationSystem as s,AssistantSystem as o,CameraControlSystem as r,CameraSystem as a,CanvasSystem as i,DragAndDropSystem as g,ExportSystem as m,GraphQLSystem as l,ImageRenderingSystem as c,LayoutSystem as S,SketchingSystem as d}from"./systems/index.js";export const createConfiguratorExtensionSelectors=u=>{let{getState:y,domainContextSelectors:C,specSelectors:p,domainServicesSelectors:w,accessSelectors:f,commonServicesSelectors:h}=u,A=e.withTypes()([p.selectConfiguratorSpec,C.selectInitializationContext],(e,s)=>t({resolved:e.assistant,handler:e.handlers.find(n),inputs:s&&s.access?{role:s.role,sessionData:s.sessionData,access:s.access}:null})),I={getConfigurationId:()=>f.getConfiguratorConfigurationId(),getAuthorizedClient:()=>h.getConfiguratorAuthorizedClient(),getAssistantSetup:()=>A(y()),domainContextSelectors:C,domainServicesSelectors:w,specSelectors:p},x={getSystems:()=>Q,getCanvasSystem:()=>z,getCameraControlSystem:()=>D,getAnimationSystem:()=>b,getDragAndDropSystem:()=>v,getImageRenderingSystem:()=>G,getLayoutSystem:()=>T,getSketchingSystem:()=>j,getExportSystem:()=>k,getCameraSystem:()=>F,getGraphQLSystem:()=>L,getAssistantSystem:()=>R},E={...I,configuratorExtensionSelectors:x},z=new i(I),D=new r({configuratorExtensionSelectors:x}),v=new g(E),b=new s(I),k=new m(E),L=new l({...I,renderScreenshot:async(e,t)=>{let[n]=await x.getImageRenderingSystem().generateImagesFromComponents([e],t,{},{awaitSettledState:!0});if(!n)throw Error("The canvas rendered no image.");return n.blob},uploadScreenshot:e=>I.getAuthorizedClient().uploadAssistantImage(I.getConfigurationId(),e)}),R=new o(E),T=new S(I),j=new d(E),F=new a(E),G=new c(E),Q={cameraControlSystem:D,layoutSystem:T,sketchingSystem:j,dragAndDropSystem:v,canvasSystem:z,cameraSystem:F,exportSystem:k,imageRenderingSystem:G,animationSystem:b,graphQLSystem:L,assistantSystem:R};return x};

package/dist/configurator/application-runtime/extensions/systems/assistant-system/assistant-system.d.ts

@@ -136,6 +136,11 @@ export declare class AssistantSystem implements InitializableSystem {
      */
     private readonly _applyToolCallIds;
     private readonly _undoTipToolCallId;
+    /**
+     * Object URLs of the screenshots read so far, by image id. An uploaded image never changes, so
+     * each is read once and kept while the configuration stays loaded.
+     */
+    private _screenshotUrls;
     constructor(_context: AssistantSystemContext);
     /** The conversation. It never completes, and holds the empty chat whenever no chat is active. */
     readonly chat$: ObservableWithLastValue<AssistantChatSnapshot>;
@@ -231,6 +236,12 @@ export declare class AssistantSystem implements InitializableSystem {
      * refuses with a conflict is not a failure: the thread is read again and `false` is returned.
      */
     resume: () => Promise<boolean>;
+    /**
+     * An object URL of a screenshot of the conversation, by the id its tool result references it by.
+     * It is read from the server through the client of the current chat, once per image; a read that
+     * failed is forgotten, so it can be asked for again. Rejects without a chat or when the read fails.
+     */
+    loadScreenshotUrl: (imageId: string) => Promise<string>;
     /**
      * Loads the thread before the oldest painted one and paints it above. Does nothing when there is
      * none or one is being loaded. A load that fails is logged and can be asked for again.
@@ -300,6 +311,8 @@ export declare class AssistantSystem implements InitializableSystem {
      * it is: whether it finishes its reply or is cut off is the caller's decision.
      */
     private _retireChat;
+    /** Revokes the object URLs of the screenshots read for the configuration that was loaded, also those still being read. */
+    private _releaseScreenshotUrls;
     /** Ends the current chat without cutting its reply off. */
     private _endChat;
 }

package/dist/configurator/application-runtime/extensions/systems/assistant-system/assistant-system.js

@@ -1 +1 @@
-import{ChatClient as e,fetchServerSentEvents as t}from"@tanstack/ai-client";import{BehaviorSubject as i}from"rxjs";import{skip as s}from"rxjs/operators";import{isShallowEqual as a}from"@salsita-ai/common/utils";import{AssistantRequestError as r}from"#package/client";import{createBrowserDetailsClientTool as n}from"./browser-details.js";import{ChangeRecorder as o}from"./change-recorder.js";import{createApplyClientTool as l,createQueryClientTool as h,toToolResult as d}from"./graphql-tools.js";let c=Object.freeze({messages:Object.freeze([]),isLoading:!1,error:void 0,failedStatus:void 0,isContinuable:!0,isLoaded:!1,isResuming:!1,earlierThreads:Object.freeze([]),hasEarlierThread:!1,isLoadingEarlierThread:!1}),u=Object.freeze({open:!1,hasUnreadReply:!1,isInputFocused:!1}),_=Object.freeze({}),g=d({results:[],committed:!1,sideEffects:[],screenshots:new Map,message:"Nothing was run, because the user closed the configurator while this reply was being written. Do not call any more tools in this reply and end it now. The tools work again as soon as the user writes the next message."}),m=e=>{if(e.getError())return!0;let t=e.getMessages().at(-1)?.parts.at(-1);return t?.type!=="tool-call"&&t?.type!=="tool-result"};export class AssistantSystem{_context;_tools;_chat=new i(c);_panel=new i(u);_activation=null;_current=null;_finishing=new Set;_olderThreadId=null;_resuming=null;_refused=null;_firstAttempt=null;_chain={};_loadingThreadId=null;_recorder;_recording=null;_applyToolCallIds=new WeakMap;_undoTipToolCallId=new i(void 0);constructor(e){this._context=e;let{configuratorExtensionSelectors:t,domainContextSelectors:i,domainServicesSelectors:s,getAssistantSetup:a}=e;this._recorder=new o({maxEntities:30},{readers:{selectedEntityId:e=>i.getNavigationSystem().getActiveSelectionState(e),navigationState:e=>i.getNavigationSystem().getActiveNavigationState(e),viewState:e=>i.getNavigationSystem().getActiveViewState(e),toolState:e=>i.getNavigationSystem().getActiveToolState(e)},getExposedFields:()=>a().manifest?.fields??_,resolveLocalizable:e=>s.getResolveLocalizable()(e)});let r={getGraphQLSystem:()=>t.getGraphQLSystem(),getStoreSystem:()=>i.getStoreSystem(),getActiveMode:()=>i.getActiveMode(),getIsViewMode:()=>i.getIsViewMode(),clearChanges:()=>this._recorder.clear(this._committedState()),rememberCommit:(e,t)=>{this._applyToolCallIds.set(t,e),this._refreshUndoTip()}};this._tools=[n(),h(r),l(r)],i.getUndoSystem().tip$.subscribe(this._refreshUndoTip)}chat$=this._chat;panel$=this._panel;undoTipToolCallId$=this._undoTipToolCallId;getSessionManifest=()=>this._context.getAssistantSetup().manifest;initialize=()=>{let{committed:e}=this._context.domainContextSelectors.getStoreSystem().stateSlices;this._recording??=e.pipe(s(1)).subscribe(this._recorder.record),this._recorder.reset(e.value.state),this._endChat(),this._patchPanel(u),this._activation&&this._startClient()};activate=()=>{this._endChat();let e={};return this._activation=e,this._startClient(),()=>{this._activation===e&&(this._activation=null,this._endChat())}};send=async e=>{if(!this._chat.value.isContinuable&&!await this.resume())return;let t=this._current;if(!t)return;let i={changes:this._recorder.take(this._committedState())};this._firstAttempt=t;try{await this._sendOn(t,e,i);let s=this._findRefusedMessageIndex(t);if(-1===s)return;this._refused={chat:t,index:s}}finally{this._firstAttempt===t&&(this._firstAttempt=null)}if(!await this.resume())return;let s=this._current;s&&await this._sendOn(s,e,i)};resume=async()=>{let e=this._current;if(!e)return!1;if(this._chat.value.isContinuable)return!0;if(this._resuming)return this._resuming;let t=this._resume(e).finally(()=>{this._resuming===t&&(this._resuming=null)});return this._resuming=t,t};loadEarlierThread=async()=>{let e=this._current,t=this._olderThreadId;if(!e||null===t||this._loadingThreadId===t)return;let i=this._chain;this._loadingThreadId=t,this._chat.next({...this._chat.value,isLoadingEarlierThread:!0});try{let s=await e.authorizedClient.getAssistantEarlierThread(e.configurationId,t,50);if(this._chain!==i||this._olderThreadId!==t)return;this._olderThreadId=s.previousThreadId,this._chat.next({...this._chat.value,earlierThreads:[{threadId:t,messages:s.messages},...this._chat.value.earlierThreads],hasEarlierThread:null!==s.previousThreadId})}catch(e){this._context.domainServicesSelectors.getLogger().warn("Could not load the earlier assistant conversation",{error:e})}finally{this._chain===i&&this._loadingThreadId===t&&(this._loadingThreadId=null,this._chat.next({...this._chat.value,isLoadingEarlierThread:!1}))}};stop=async()=>{let e=this._current;if(!e)return;let t=e.client.getCurrentRunId();if(e.client.stop(),null!=t)try{let i=e.getConnection();await fetch(`${i.url}/runs/${encodeURIComponent(t)}/cancel`,{method:"POST",headers:await i.headers()})}catch(e){this._context.domainServicesSelectors.getLogger().warn("Could not cancel the assistant run on the server",{error:e})}};reloadThread=()=>{if(!this._activation)return;let e=this._retireChat();e?.client.stop(),e?.dispose(),this._recorder.reset(this._committedState()),this._startClient()};undoToolCalls=e=>{let t=this._context.domainContextSelectors.getUndoSystem(),i=()=>{let t=this._findTipToolCallId();return void 0!==t&&e.has(t)};for(;i();)t.undo()};setOpen=e=>{this._patchPanel(e?{open:e,hasUnreadReply:!1}:{open:e,isInputFocused:!1})};setInputFocused=e=>{this._patchPanel({isInputFocused:e})};_findRefusedMessageIndex=e=>{var t;let i,{failedStatus:s,isContinuable:a}=this._chat.value;return this._firstAttempt===e&&e.generation.active&&409===s&&!a?(i=(t=e.client.getMessages()).map(e=>e.role).lastIndexOf("user"),t.slice(i+1).every(e=>0===e.parts.length)?i:-1):-1};_sendOn=async(e,t,i)=>{let{client:s}=e;s.getIsLoading()||null===s.getResumeState()||s.stop();try{await s.sendMessage({content:t,metadata:i})}catch(i){let t=i instanceof Error?i:Error(String(i));e.generation.active&&this._chat.next({...this._chat.value,error:t}),this._context.domainServicesSelectors.getLogger().error("The assistant message could not be sent",{},t)}};_resume=async e=>{this._chat.next({...this._chat.value,isResuming:!0,error:void 0,failedStatus:void 0});try{let{previousThreadId:t}=await e.authorizedClient.compactAssistantThread(e.configurationId);if(!e.generation.active)return!1;let{messages:i,earlierThreads:s,hasEarlierThread:a}=this._chat.value,r=this._refused?.chat===e?i.slice(0,this._refused.index):i;return e.generation.active=!1,this._current=null,e.client.stop(),e.dispose(),this._chat.next({...c,earlierThreads:[...s,{threadId:t,messages:r}],hasEarlierThread:a,isLoadingEarlierThread:null!==this._loadingThreadId}),this._startClient(),!0}catch(i){let t=i instanceof Error?i:Error(String(i));if(t instanceof r&&409===t.status)return e.generation.active&&this.reloadThread(),!1;return e.generation.active&&this._chat.next({...this._chat.value,isResuming:!1,error:t,failedStatus:t instanceof r?t.status:void 0}),this._context.domainServicesSelectors.getLogger().error("The assistant thread could not be resumed",{},t),!1}};_findTipToolCallId=()=>{let e=this._context.domainContextSelectors.getUndoSystem(),t=e.tip$.value?.committedState;return void 0!==t&&e.canUndo()?this._applyToolCallIds.get(t):void 0};_refreshUndoTip=()=>{let e=this._findTipToolCallId();e!==this._undoTipToolCallId.value&&this._undoTipToolCallId.next(e)};_committedState=()=>this._context.domainContextSelectors.getStoreSystem().stateSlices.committed.value.state;_patchPanel=e=>{let t={...this._panel.value,...e};a(t,this._panel.value)||this._panel.next(t)};_takeOverThread=e=>{for(let t of this._finishing)t.configurationId===e&&(t.client.stop(),t.dispose())};_startClient=()=>{let i={active:!0},s=this._context.getConfigurationId();this._takeOverThread(s);let r=this._context.getAuthorizedClient(),n=()=>{if(i.active){let{manifest:e,error:t}=this._context.getAssistantSetup();if(t)throw t;if(!e)throw Error("The assistant is disabled for this session.")}return r.getAssistantChatConnection(s)},o=e=>{if(!i.active)return;let t={...this._chat.value,...e};a(t,this._chat.value)||this._chat.next(t)},l=async(e,t)=>{let s=await fetch(e,t).finally(()=>o({isLoaded:!0})),a=s.headers.get("x-assistant-thread-continuable");return null!=a&&s.ok&&i.active&&0===this._chat.value.earlierThreads.length&&(this._olderThreadId=s.headers.get("x-assistant-previous-thread")),o({failedStatus:s.ok?void 0:s.status,...null!=a&&{isContinuable:"false"!==a,hasEarlierThread:null!==this._olderThreadId}}),s},h=async()=>({headers:await n().headers(),fetchClient:l}),d=t(()=>n().url,h),c=t(()=>`${n().url}/runs`,h),u=this._tools.map(e=>({...e,execute:(t,s)=>{if(!i.active)return g;let{manifest:a,error:r}=this._context.getAssistantSetup();return r||!a?{error:r?.message??"The assistant is disabled for this session."}:e.execute?.(t,s)}})),_=new e({threadId:`assistant:${s}`,connection:{...d,joinRun:c.joinRun},persistence:!0,history:{pageSize:50},tools:u,onMessagesChange:e=>o({messages:e}),onLoadingChange:e=>{let t=i.active&&this._chat.value.isLoading;o({isLoading:e}),!e&&m(_)&&(i.active?t&&!this._panel.value.open&&this._patchPanel({hasUnreadReply:!0}):p())},onErrorChange:e=>{i.active&&e!==this._chat.value.error&&(!e||-1===this._findRefusedMessageIndex(v))&&(o({error:e}),e&&this._context.domainServicesSelectors.getLogger().error("The assistant request failed",{status:this._chat.value.failedStatus},e))}}),f=!1,p=()=>{f||(f=!0,this._finishing.delete(v),_.dispose())},v={client:_,configurationId:s,generation:i,authorizedClient:r,getConnection:n,dispose:p};this._current=v,_.attach()};_retireChat=()=>{let e=this._current;return this._current=null,e&&(e.generation.active=!1),this._chain={},this._olderThreadId=null,this._resuming=null,this._refused=null,this._loadingThreadId=null,this._chat.value!==c&&this._chat.next(c),e};_endChat=()=>{let e=this._retireChat();if(e){if(!e.client.getIsLoading()&&m(e.client))return void e.dispose();this._finishing.add(e)}}}
+import{ChatClient as e,fetchServerSentEvents as t}from"@tanstack/ai-client";import{BehaviorSubject as s}from"rxjs";import{skip as i}from"rxjs/operators";import{isShallowEqual as a}from"@salsita-ai/common/utils";import{AssistantRequestError as r}from"#package/client";import{createBrowserDetailsClientTool as n}from"./browser-details.js";import{ChangeRecorder as o}from"./change-recorder.js";import{createApplyClientTool as l,createQueryClientTool as h,toToolResult as d}from"./graphql-tools.js";let c=Object.freeze({messages:Object.freeze([]),isLoading:!1,error:void 0,failedStatus:void 0,isContinuable:!0,isLoaded:!1,isResuming:!1,earlierThreads:Object.freeze([]),hasEarlierThread:!1,isLoadingEarlierThread:!1}),u=Object.freeze({open:!1,hasUnreadReply:!1,isInputFocused:!1}),_=Object.freeze({}),g=d({results:[],committed:!1,sideEffects:[],screenshots:new Map,message:"Nothing was run, because the user closed the configurator while this reply was being written. Do not call any more tools in this reply and end it now. The tools work again as soon as the user writes the next message."}),f=e=>{if(e.getError())return!0;let t=e.getMessages().at(-1)?.parts.at(-1);return t?.type!=="tool-call"&&t?.type!=="tool-result"};export class AssistantSystem{_context;_tools;_chat=new s(c);_panel=new s(u);_activation=null;_current=null;_finishing=new Set;_olderThreadId=null;_resuming=null;_refused=null;_firstAttempt=null;_chain={};_loadingThreadId=null;_recorder;_recording=null;_applyToolCallIds=new WeakMap;_undoTipToolCallId=new s(void 0);_screenshotUrls=new Map;constructor(e){this._context=e;let{configuratorExtensionSelectors:t,domainContextSelectors:s,domainServicesSelectors:i,getAssistantSetup:a}=e;this._recorder=new o({maxEntities:30},{readers:{selectedEntityId:e=>s.getNavigationSystem().getActiveSelectionState(e),navigationState:e=>s.getNavigationSystem().getActiveNavigationState(e),viewState:e=>s.getNavigationSystem().getActiveViewState(e),toolState:e=>s.getNavigationSystem().getActiveToolState(e)},getExposedFields:()=>a().manifest?.fields??_,resolveLocalizable:e=>i.getResolveLocalizable()(e)});let r={getGraphQLSystem:()=>t.getGraphQLSystem(),getStoreSystem:()=>s.getStoreSystem(),getActiveMode:()=>s.getActiveMode(),getIsViewMode:()=>s.getIsViewMode(),clearChanges:()=>this._recorder.clear(this._committedState()),rememberCommit:(e,t)=>{this._applyToolCallIds.set(t,e),this._refreshUndoTip()}};this._tools=[n(),h(r),l(r)],s.getUndoSystem().tip$.subscribe(this._refreshUndoTip)}chat$=this._chat;panel$=this._panel;undoTipToolCallId$=this._undoTipToolCallId;getSessionManifest=()=>this._context.getAssistantSetup().manifest;initialize=()=>{let{committed:e}=this._context.domainContextSelectors.getStoreSystem().stateSlices;this._recording??=e.pipe(i(1)).subscribe(this._recorder.record),this._recorder.reset(e.value.state),this._endChat(),this._patchPanel(u),this._releaseScreenshotUrls(),this._activation&&this._startClient()};activate=()=>{this._endChat();let e={};return this._activation=e,this._startClient(),()=>{this._activation===e&&(this._activation=null,this._endChat())}};send=async e=>{if(!this._chat.value.isContinuable&&!await this.resume())return;let t=this._current;if(!t)return;let s={changes:this._recorder.take(this._committedState())};this._firstAttempt=t;try{await this._sendOn(t,e,s);let i=this._findRefusedMessageIndex(t);if(-1===i)return;this._refused={chat:t,index:i}}finally{this._firstAttempt===t&&(this._firstAttempt=null)}if(!await this.resume())return;let i=this._current;i&&await this._sendOn(i,e,s)};resume=async()=>{let e=this._current;if(!e)return!1;if(this._chat.value.isContinuable)return!0;if(this._resuming)return this._resuming;let t=this._resume(e).finally(()=>{this._resuming===t&&(this._resuming=null)});return this._resuming=t,t};loadScreenshotUrl=e=>{let t=this._screenshotUrls.get(e);if(t)return t;let s=this._current;if(!s)return Promise.reject(Error(`The screenshot ${e} cannot be read, because the assistant chat is not active. Open the assistant panel of the configuration first.`));let i=this._screenshotUrls,a=s.authorizedClient.getAssistantImage(s.configurationId,e).then(e=>URL.createObjectURL(e));return i.set(e,a),a.catch(()=>{i.get(e)===a&&i.delete(e)}),a};loadEarlierThread=async()=>{let e=this._current,t=this._olderThreadId;if(!e||null===t||this._loadingThreadId===t)return;let s=this._chain;this._loadingThreadId=t,this._chat.next({...this._chat.value,isLoadingEarlierThread:!0});try{let i=await e.authorizedClient.getAssistantEarlierThread(e.configurationId,t,50);if(this._chain!==s||this._olderThreadId!==t)return;this._olderThreadId=i.previousThreadId,this._chat.next({...this._chat.value,earlierThreads:[{threadId:t,messages:i.messages},...this._chat.value.earlierThreads],hasEarlierThread:null!==i.previousThreadId})}catch(e){this._context.domainServicesSelectors.getLogger().warn("Could not load the earlier assistant conversation",{error:e})}finally{this._chain===s&&this._loadingThreadId===t&&(this._loadingThreadId=null,this._chat.next({...this._chat.value,isLoadingEarlierThread:!1}))}};stop=async()=>{let e=this._current;if(!e)return;let t=e.client.getCurrentRunId();if(e.client.stop(),null!=t)try{let s=e.getConnection();await fetch(`${s.url}/runs/${encodeURIComponent(t)}/cancel`,{method:"POST",headers:await s.headers()})}catch(e){this._context.domainServicesSelectors.getLogger().warn("Could not cancel the assistant run on the server",{error:e})}};reloadThread=()=>{if(!this._activation)return;let e=this._retireChat();e?.client.stop(),e?.dispose(),this._recorder.reset(this._committedState()),this._startClient()};undoToolCalls=e=>{let t=this._context.domainContextSelectors.getUndoSystem(),s=()=>{let t=this._findTipToolCallId();return void 0!==t&&e.has(t)};for(;s();)t.undo()};setOpen=e=>{this._patchPanel(e?{open:e,hasUnreadReply:!1}:{open:e,isInputFocused:!1})};setInputFocused=e=>{this._patchPanel({isInputFocused:e})};_findRefusedMessageIndex=e=>{var t;let s,{failedStatus:i,isContinuable:a}=this._chat.value;return this._firstAttempt===e&&e.generation.active&&409===i&&!a?(s=(t=e.client.getMessages()).map(e=>e.role).lastIndexOf("user"),t.slice(s+1).every(e=>0===e.parts.length)?s:-1):-1};_sendOn=async(e,t,s)=>{let{client:i}=e;i.getIsLoading()||null===i.getResumeState()||i.stop();try{await i.sendMessage({content:t,metadata:s})}catch(s){let t=s instanceof Error?s:Error(String(s));e.generation.active&&this._chat.next({...this._chat.value,error:t}),this._context.domainServicesSelectors.getLogger().error("The assistant message could not be sent",{},t)}};_resume=async e=>{this._chat.next({...this._chat.value,isResuming:!0,error:void 0,failedStatus:void 0});try{let{previousThreadId:t}=await e.authorizedClient.compactAssistantThread(e.configurationId);if(!e.generation.active)return!1;let{messages:s,earlierThreads:i,hasEarlierThread:a}=this._chat.value,r=this._refused?.chat===e?s.slice(0,this._refused.index):s;return e.generation.active=!1,this._current=null,e.client.stop(),e.dispose(),this._chat.next({...c,earlierThreads:[...i,{threadId:t,messages:r}],hasEarlierThread:a,isLoadingEarlierThread:null!==this._loadingThreadId}),this._startClient(),!0}catch(s){let t=s instanceof Error?s:Error(String(s));if(t instanceof r&&409===t.status)return e.generation.active&&this.reloadThread(),!1;return e.generation.active&&this._chat.next({...this._chat.value,isResuming:!1,error:t,failedStatus:t instanceof r?t.status:void 0}),this._context.domainServicesSelectors.getLogger().error("The assistant thread could not be resumed",{},t),!1}};_findTipToolCallId=()=>{let e=this._context.domainContextSelectors.getUndoSystem(),t=e.tip$.value?.committedState;return void 0!==t&&e.canUndo()?this._applyToolCallIds.get(t):void 0};_refreshUndoTip=()=>{let e=this._findTipToolCallId();e!==this._undoTipToolCallId.value&&this._undoTipToolCallId.next(e)};_committedState=()=>this._context.domainContextSelectors.getStoreSystem().stateSlices.committed.value.state;_patchPanel=e=>{let t={...this._panel.value,...e};a(t,this._panel.value)||this._panel.next(t)};_takeOverThread=e=>{for(let t of this._finishing)t.configurationId===e&&(t.client.stop(),t.dispose())};_startClient=()=>{let s={active:!0},i=this._context.getConfigurationId();this._takeOverThread(i);let r=this._context.getAuthorizedClient(),n=()=>{if(s.active){let{manifest:e,error:t}=this._context.getAssistantSetup();if(t)throw t;if(!e)throw Error("The assistant is disabled for this session.")}return r.getAssistantChatConnection(i)},o=e=>{if(!s.active)return;let t={...this._chat.value,...e};a(t,this._chat.value)||this._chat.next(t)},l=async(e,t)=>{let i=await fetch(e,t).finally(()=>o({isLoaded:!0})),a=i.headers.get("x-assistant-thread-continuable");return null!=a&&i.ok&&s.active&&0===this._chat.value.earlierThreads.length&&(this._olderThreadId=i.headers.get("x-assistant-previous-thread")),o({failedStatus:i.ok?void 0:i.status,...null!=a&&{isContinuable:"false"!==a,hasEarlierThread:null!==this._olderThreadId}}),i},h=async()=>({headers:await n().headers(),fetchClient:l}),d=t(()=>n().url,h),c=t(()=>`${n().url}/runs`,h),u=this._tools.map(e=>({...e,execute:(t,i)=>{if(!s.active)return g;let{manifest:a,error:r}=this._context.getAssistantSetup();return r||!a?{error:r?.message??"The assistant is disabled for this session."}:e.execute?.(t,i)}})),_=new e({threadId:`assistant:${i}`,connection:{...d,joinRun:c.joinRun},persistence:!0,history:{pageSize:50},tools:u,onMessagesChange:e=>o({messages:e}),onLoadingChange:e=>{let t=s.active&&this._chat.value.isLoading;o({isLoading:e}),!e&&f(_)&&(s.active?t&&!this._panel.value.open&&this._patchPanel({hasUnreadReply:!0}):p())},onErrorChange:e=>{s.active&&e!==this._chat.value.error&&(!e||-1===this._findRefusedMessageIndex(v))&&(o({error:e}),e&&this._context.domainServicesSelectors.getLogger().error("The assistant request failed",{status:this._chat.value.failedStatus},e))}}),m=!1,p=()=>{m||(m=!0,this._finishing.delete(v),_.dispose())},v={client:_,configurationId:i,generation:s,authorizedClient:r,getConnection:n,dispose:p};this._current=v,_.attach()};_retireChat=()=>{let e=this._current;return this._current=null,e&&(e.generation.active=!1),this._chain={},this._olderThreadId=null,this._resuming=null,this._refused=null,this._loadingThreadId=null,this._chat.value!==c&&this._chat.next(c),e};_releaseScreenshotUrls=()=>{for(let e of this._screenshotUrls.values())e.then(URL.revokeObjectURL,()=>void 0);this._screenshotUrls=new Map};_endChat=()=>{let e=this._retireChat();if(e){if(!e.client.getIsLoading()&&f(e.client))return void e.dispose();this._finishing.add(e)}}}

package/dist/configurator/application-runtime/extensions/systems/assistant-system/graphql-tools.d.ts

@@ -35,7 +35,8 @@ export interface AssistantToolDependencies {
 /**
  * A tool result as the chat client accepts it: content parts, the execution's JSON as a text part
  * followed, per screenshot, by its label as a text part and its picture as an image part, so the
- * model reads each picture under the label its `image` field returned. The client deep-clones a
+ * model reads each picture under the label its `image` field returned; the image part references the picture the browser uploaded, which the server
+ * inlines only for the model. The client deep-clones a
  * result and rejects anything that is not plain JSON, so the `GraphQLError` instances of a result
  * are formatted through their `toJSON` and `undefined` members are dropped. The prepared state stays
  * in the browser.

package/dist/configurator/application-runtime/extensions/systems/assistant-system/graphql-tools.js

@@ -1 +1 @@
-import{AssistantGraphQLToolInput as e,applyTool as t,queryTool as o}from"@salsita-ai/contracts/assistant";let a=e=>({results:[],committed:!1,message:e,sideEffects:[],screenshots:new Map});export const toToolResult=({preparedState:e,screenshots:t,...o})=>[{type:"text",content:JSON.stringify(o)},...[...t].flatMap(([e,{data:t,mimeType:o}])=>[{type:"text",content:e},{type:"image",source:{type:"data",value:t,mimeType:o}}])];let i=t=>{let o=e.safeParse(t);if(!o.success){let e=o.error.issues.map(e=>`${e.path.join(".")}: ${e.message}`).join("; ");return{ok:!1,refusal:a(`Nothing was run, because the tool input is not a list of operations (${e}). Send { "operations": [{ "query": "<one GraphQL operation>" }] } with at least one item.`)}}return{ok:!0,operations:o.data.operations.map(({query:e,variables:t,operationName:o})=>({document:e,variables:t??void 0,operationName:o??void 0}))}};export const createQueryClientTool=e=>o.client(async t=>{let o=i(t);if(!o.ok)return toToolResult(o.refusal);let s=e.getStoreSystem();if(!s.isInitialized)return toToolResult(a("Nothing was run, because the configuration is not loaded yet, so there is no state to query. Call the tool again once the configurator has finished loading."));let n=e.getGraphQLSystem(),r=await n.prepare(s.stateSlices.committed.value.state,o.operations);return toToolResult({results:r.map(({result:e})=>e),committed:!1,sideEffects:n.readSideEffects(r),screenshots:n.readScreenshots(r)})});export const createApplyClientTool=e=>t.client(async(t,o)=>{if("finalize"===e.getActiveMode())return toToolResult(a("Nothing was run, because changes cannot be applied in this session: the user may view this configuration but not change it. The query tool still works here, including mutations, to show what a change would do without keeping it."));if(e.getIsViewMode())return toToolResult(a("Nothing was run, because the configuration is read-only right now: the user may view it but not change it. The query tool still works here, including mutations, to show what a change would do without keeping it. Call apply again only once the configuration is editable."));let s=i(t);if(!s.ok)return toToolResult(s.refusal);let n=await e.getGraphQLSystem().execute(s.operations),r=o?.toolCallId;return void 0!==r&&void 0!==n.preparedState&&e.rememberCommit(r,n.preparedState),n.committed&&e.clearChanges(),toToolResult(n)});
+import{AssistantGraphQLToolInput as e,applyTool as t,assistantImageReference as o,queryTool as i}from"@salsita-ai/contracts/assistant";let a=e=>({results:[],committed:!1,message:e,sideEffects:[],screenshots:new Map});export const toToolResult=({preparedState:e,screenshots:t,...i})=>[{type:"text",content:JSON.stringify(i)},...[...t].flatMap(([e,{imageId:t}])=>[{type:"text",content:e},{type:"image",source:{type:"url",value:o(t)}}])];let s=t=>{let o=e.safeParse(t);if(!o.success){let e=o.error.issues.map(e=>`${e.path.join(".")}: ${e.message}`).join("; ");return{ok:!1,refusal:a(`Nothing was run, because the tool input is not a list of operations (${e}). Send { "operations": [{ "query": "<one GraphQL operation>" }] } with at least one item.`)}}return{ok:!0,operations:o.data.operations.map(({query:e,variables:t,operationName:o})=>({document:e,variables:t??void 0,operationName:o??void 0}))}};export const createQueryClientTool=e=>i.client(async t=>{let o=s(t);if(!o.ok)return toToolResult(o.refusal);let i=e.getStoreSystem();if(!i.isInitialized)return toToolResult(a("Nothing was run, because the configuration is not loaded yet, so there is no state to query. Call the tool again once the configurator has finished loading."));let n=e.getGraphQLSystem(),r=await n.prepare(i.stateSlices.committed.value.state,o.operations);return toToolResult({results:r.map(({result:e})=>e),committed:!1,sideEffects:n.readSideEffects(r),screenshots:n.readScreenshots(r)})});export const createApplyClientTool=e=>t.client(async(t,o)=>{if("finalize"===e.getActiveMode())return toToolResult(a("Nothing was run, because changes cannot be applied in this session: the user may view this configuration but not change it. The query tool still works here, including mutations, to show what a change would do without keeping it."));if(e.getIsViewMode())return toToolResult(a("Nothing was run, because the configuration is read-only right now: the user may view it but not change it. The query tool still works here, including mutations, to show what a change would do without keeping it. Call apply again only once the configuration is editable."));let i=s(t);if(!i.ok)return toToolResult(i.refusal);let n=await e.getGraphQLSystem().execute(i.operations),r=o?.toolCallId;return void 0!==r&&void 0!==n.preparedState&&e.rememberCommit(r,n.preparedState),n.committed&&e.clearChanges(),toToolResult(n)});

package/dist/configurator/application-runtime/extensions/systems/graphql-system/graphql-system.d.ts

@@ -11,19 +11,25 @@ export interface GraphQLOperation {
 export interface GraphQLScreenshot {
     readonly entityId: string;
     readonly view: string;
-    /** The picture as base64 data. */
-    readonly data: string;
-    readonly mimeType: 'image/jpeg' | 'image/png';
+    /** The id the server stored the picture under, which the tool result references it by. */
+    readonly imageId: string;
 }
 export interface GraphQLSystemContext {
     /** The assistant the initialized session runs: the operations are validated and executed against its schema. */
     readonly getAssistantSetup: () => SessionAssistantSetup;
     /**
      * Renders the picture an `agentScreenshot` component of a state declares, behind every export
-     * already queued; `null` from an executor that renders no pictures. The only async step of an
-     * execution, and the only one that needs a canvas.
+     * already queued, as a JPEG or PNG blob; `null` from an executor that renders no pictures. One of
+     * the two async steps of an execution, and the only one that needs a canvas.
      */
     readonly renderScreenshot: (component: ComponentSnapshot<AgentScreenshotComponent>, state: AnyDomainState) => Promise<Blob | null>;
+    /**
+     * Stores a rendered screenshot in the assistant thread on the server and resolves with the id the
+     * tool result references it by. The other async step of an execution, run right after each render.
+     */
+    readonly uploadScreenshot: (image: Blob) => Promise<{
+        readonly imageId: string;
+    }>;
     readonly specSelectors: SpecSelectors;
     readonly domainContextSelectors: DomainContextSelectors;
     readonly domainServicesSelectors: DomainServicesSelectors;
@@ -97,7 +103,7 @@ export declare class GraphQLSystem {
     private readonly _resolveType;
     private _generation;
     /**
-     * The picture of an entity of a state from one `agentScreenshot` view, as base64 of the view's
+     * The picture of an entity of a state from one `agentScreenshot` view, as a blob of the view's
      * file format; `null` when the view is not attached to the entity there or nothing renders it.
      */
     private readonly _renderScreenshot;
@@ -118,9 +124,9 @@ export declare class GraphQLSystem {
     private readonly _runMutation;
     /**
      * Takes a screenshot once every one asked for before it in the execution has been taken, so they
-     * render and spend the budget in document order. A failure nulls only its own field and fails
-     * nothing else: the user kept changing the configuration, the image did not fit the budget, or
-     * rendering failed. Once the budget is spent, no later screenshot renders.
+     * render in document order, and uploads it as soon as it has rendered. A failure nulls only its
+     * own field and fails nothing else: the user kept changing the configuration, rendering failed,
+     * or the server did not store the picture.
      */
     private readonly _takeScreenshot;
     private readonly _setup;

package/dist/configurator/application-runtime/extensions/systems/graphql-system/graphql-system.js

@@ -1 +1 @@
-import{GraphQLError as e,NoSchemaIntrospectionCustomRule as t,defaultFieldResolver as i,execute as n,getNamedType as r,parse as o,specifiedRules as s,validate as a,validateSchema as l}from"graphql";import{assertExistence as c,createSelector as h,isMessageDescriptor as d,isMultilingualString as u}from"@salsita-ai/common/utils";import{DomainEntityIdSchema as f,agentScreenshotComponent as m,createAssistantExecutionSchema as p,findAssistantMutation as g}from"#package/domain";import{ExportBusyError as y}from"../export-system/export-busy-error.js";import{closeUnclosedBraces as v}from"./close-unclosed-braces.js";import{createEntityMeta as S}from"./entity-meta.js";import{injectAssistantLenses as w}from"./inject-assistant-lenses.js";import{readSideEffects as _}from"./side-effects.js";let x={errors:[new e("The configurator does not provide this request. This is a fault in the configurator, not in your request. Tell the user this is not available.")]},$={errors:[new e("The configuration is not loaded yet, so there is no state to query. Run the query again once the configurator has finished loading.")]},b="Not taken: another configuration was loaded while this request ran. Query the current configuration before asking for screenshots again.",E="Not taken: the screenshots of this tool call have used up its image size budget. Ask for this screenshot in another call, and for fewer in one call.",T=async e=>{let t=new Uint8Array(await e.arrayBuffer()),i="";for(let e=0;e<t.length;e+=32768)i+=String.fromCharCode(...t.subarray(e,e+32768));return btoa(i)},C=new RegExp(f.pattern),M=[...s,t],k=e=>"object"==typeof e&&null!==e&&"type"in e&&"string"==typeof e.type&&"id"in e&&"string"==typeof e.id,j=(e,t)=>"object"==typeof e&&null!==e?Reflect.get(e,t):void 0,R=e=>e instanceof Error?e.message:String(e),A=e=>/[.!?]$/.test(e)?e:`${e}.`,N=e=>"object"==typeof e&&null!==e&&Object.getPrototypeOf(e)===Object.prototype,z=(e,t)=>{let i=N(e)?e._raw:void 0;if(k(i))return i;if(!Array.isArray(e)&&!N(e))return e;if(t.has(e))throw Error("The value contains itself.");let n=new Set([...t,e]);return Array.isArray(e)?e.map(e=>z(e,n)):Object.fromEntries(Object.entries(e).map(([e,t])=>[e,z(t,n)]))};export const localize=(e,t)=>Array.isArray(e)?e.map(e=>localize(e,t)):"string"==typeof e||d(e)||u(e)?t(e):e;let q=t=>new e(`The configurator cannot handle this request: ${A(t)} This is a fault in the configurator, not in your request. Tell the user this is not available.`),L="SCREENSHOT_NOT_TAKEN",F=(t,i)=>new e(t,{originalError:i,extensions:{code:L}}),O=e=>(e.errors??[]).some(e=>e.extensions.code!==L),V=t=>({errors:[new e(`Not run: operation ${t+1} failed, so the operations after it were skipped and no change of the list takes effect. Fix the failing operation and send the whole list again.`)]});export class GraphQLSystem{_context;_selectSchema=h([h.leaf(e=>e)],e=>{try{let t=p(e),i=l(t);return i.length>0?q(i.map(e=>e.message).join(" ")):t}catch(e){return q(R(e))}});_selectLenses=h([h.leaf(e=>e)],e=>o(e));_selectFieldResolver=h([h.leaf(e=>e)],t=>(n,o,s,a)=>{if(a.parentType===a.schema.getQueryType())return i(n,o,s,a);if(a.parentType===a.schema.getMutationType())return this._resolveMutation(a.fieldName,String(a.path.key),o,s);if(!k(n))return i(n,o,s,a);if("meta"===a.fieldName){let e;return S({entity:n,state:("object"==typeof(e=j(n,"_state"))&&null!==e&&"entities"in e&&"rootEntityId"in e?e:void 0)??s.state},{manifest:s.manifest,recipes:this._recipeContext(),prepareStateEmission:(e,t,i)=>this._context.domainContextSelectors.getStoreSystem().prepareStateEmission(e,t,i),find$Entity:(e,t)=>this._find$Entity(e,t),validateEntity:e=>this._context.domainContextSelectors.getDomain().validateEntity(e),takeScreenshot:(e,t)=>this._takeScreenshot(e,t,s)})}let{source:l,localized:c}=t[a.parentType.name]?.[a.fieldName]??{source:a.fieldName},h=((t,i)=>{try{return j(t,i)}catch(n){throw new e(`Expression "${i}" failed on ${k(t)?t.id:"an entity"}: ${A(R(n))} This is a fault in the configurator, not in your request. Tell the user this is not available.`,{originalError:n instanceof Error?n:void 0})}})(n,l);return c?localize(h,this._context.domainServicesSelectors.getResolveLocalizable()):l.startsWith("$")&&"JSON"===r(a.returnType).name?((t,i,n)=>{try{return z(n,new Set)}catch(n){throw new e(`Expression "${i}" returned a value that is not plain serializable data on ${k(t)?t.id:"an entity"}: ${A(R(n))} This is a fault in the configurator, not in your request. Tell the user this is not available.`,{originalError:n instanceof Error?n:void 0})}})(n,l,h):h});constructor(e){this._context=e}_get$Entity=(e,t)=>this._context.domainContextSelectors.getDomain().get$Entity(e,t);_find$Entity=(e,t)=>this._context.domainContextSelectors.getDomain().getEntity(e,t)?this._get$Entity(e,t):null;_recipeContext=()=>({componentReader:this._context.domainContextSelectors.getComponentReader(),recipeCooker:this._context.domainContextSelectors.getRecipeCookerSystem(),resolveLocalizable:this._context.domainServicesSelectors.getResolveLocalizable()});_rootValue={root:(e,{state:t})=>this._get$Entity(t.rootEntityId,t),entities:({ids:t},{state:i})=>{if(null==t)return i.entities.map(e=>this._get$Entity(e.id,i));var n=i.rootEntityId;let r=t.filter(e=>!C.test(e));if(r.length>0)throw new e(`${r.map(e=>`"${e}"`).join(", ")} ${1===r.length?"is not an entity id":"are not entity ids"}. Entity ids have the form "<Type>-<number>", e.g. "${n}". Use ids as returned by the "id" field.`);return t.map(e=>C.test(e)?this._find$Entity(e,i):null)},selectedEntity:(e,{state:t})=>this._find$Entity(this._context.domainContextSelectors.getNavigationSystem().getActiveSelectionState(t),t),navigation:(e,{state:t})=>{let i=this._context.domainContextSelectors.getNavigationSystem();return{navigationState:i.getActiveNavigationState(t),viewState:i.getActiveViewState(t),toolState:i.getActiveToolState(t)}}};_resolveType=e=>k(e)?e.type:void 0;_generation={active:!0};_renderScreenshot=async(e,t,i)=>{let n=this._context.domainContextSelectors.getComponentReader().readOneComponent({entityId:e,componentType:m,instanceName:t},i);if(!n)return null;let r=await this._context.renderScreenshot(n,i);return null===r?null:{data:await T(r),mimeType:"JPEG"===n.state.fileFormat?"image/jpeg":"image/png"}};initialize=()=>{this._generation.active=!1,this._generation={active:!0}};_resolveMutation=(t,i,n,r)=>{if(void 0!==r.failedMutationField)throw new e(`Not run: the earlier field "${r.failedMutationField}" of this operation failed, so "${i}" was skipped and no change of the operation takes effect. Fix the failing field and send the operation again.`);try{return this._runMutation(t,n,r)}catch(e){throw r.failedMutationField=i,e}};_runMutation=(t,i,n)=>{let r=g(t,n.manifest);if(!r)throw new e(`The schema declares the mutation "${t}", but the configurator has no implementation for it. This is a fault in the configurator, not in your request. Tell the user this is not available.`);let o=n.state,s=r.implementation(i,o,{manifest:n.manifest,action:this._context.domainContextSelectors.getActionSystem().createContext(o),resolveLocalizable:this._context.domainServicesSelectors.getResolveLocalizable()});if(s.updates.length>0){let i=this._context.domainContextSelectors.getStoreSystem().prepareStateEmission(s.updates,o,"committed");if(!i.validity.valid)throw new e(`The changes of "${t}" were not applied, because the resulting configuration is invalid: ${(e=>{if("uncaught"===e.severity)return`validating it threw: ${A(R(e.error))}`;let t=[...new Set(e.entityViolations.filter(({violation:t})=>t.severity===e.severity).map(({entityId:e})=>e))];return`it has ${e.severity} violations on ${t.join(", ")}.`})(i.validity)} Change the request so the configuration stays valid.`);n.emissions.push(i),n.state=i.state}return s.readResult({previousState:o,state:n.state})};_takeScreenshot=({entity:t,state:i},n,r)=>{let o=r.manifest.screenshots?.[t.type]?.find(e=>e.view===n);if(!o)return Promise.reject(new e(`The schema declares the screenshot view "${n}" on ${t.type}, but the configurator has no view of that name. This is a fault in the configurator, not in your request. Tell the user this is not available.`));let s=r.lastScreenshot.then(async()=>{if(!r.generation.active)throw F(b);if(r.screenshotBudget<=0)throw F(E);let e=await this._renderScreenshot(t.id,o.instanceName,i).catch(e=>{throw F(e instanceof y?"User made concurrent actions that prevented the screenshot from being taken, try again":`The screenshot of ${t.id} from "${n}" could not be taken: ${A(R(e))} Try again; if it fails again, tell the user the screenshot is not available.`,e instanceof Error?e:void 0)});if(!r.generation.active)throw F(b);if(null===e)return null;if(e.data.length>r.screenshotBudget)throw r.screenshotBudget=0,F(E);r.screenshotBudget-=e.data.length;let s=`${t.id}/${n}`,a=[...r.screenshots.values()].filter(e=>e.entityId===t.id&&e.view===n).length,l=0===a?s:`${s}#${a+1}`;return r.screenshots.set(l,{entityId:t.id,view:n,...e}),{image:l}});return r.lastScreenshot=s.catch(()=>void 0),s};_setup=()=>{let{manifest:t,error:i}=this._context.getAssistantSetup();if(i)return{ready:!1,result:{errors:[new e(i.message,{originalError:i})]}};if(!t)return{ready:!1,result:x};let n=this._selectSchema(t.sdl);return n instanceof e?{ready:!1,result:{errors:[n]}}:{ready:!0,assistant:t,schema:n,lenses:t.lenses?.definitions?this._selectLenses(t.lenses.definitions):void 0,fieldResolver:this._selectFieldResolver(t.fields)}};_run=(t,i,r,s)=>{let l;try{l=o(t)}catch(t){return{errors:[t instanceof e?t:new e(R(t))]}}let c=a(r.schema,l,M);return c.length>0?{errors:c}:n({schema:r.schema,document:l,rootValue:this._rootValue,contextValue:s,variableValues:i.variables,operationName:i.operationName,fieldResolver:r.fieldResolver,typeResolver:this._resolveType})};prepare=async(t,i,n=this._generation)=>{let r,o=this._setup();if(!o.ready)return i.map(()=>({result:o.result,emissions:[],screenshots:new Map}));let s={manifest:o.assistant,state:t,emissions:[],failedMutationField:void 0,screenshots:new Map,screenshotBudget:14e5,lastScreenshot:Promise.resolve(),generation:n},a=[];for(let[t,n]of i.entries())if(void 0===r){let i=s.emissions.length,l=s.screenshots.size;s.failedMutationField=void 0;let c=v(n.document),h=o.lenses?w(c,o.lenses):c,d=h instanceof e?{errors:[h]}:await this._run(h,n,o,s);await s.lastScreenshot,a.push({result:d,emissions:s.emissions.slice(i),screenshots:new Map([...s.screenshots].slice(l))}),O(d)&&(r=t)}else a.push({result:V(r),emissions:[],screenshots:new Map});return a};readSideEffects=e=>{let t=this._context.domainServicesSelectors.getResolveLocalizable();return e.flatMap(({emissions:e})=>e.flatMap(e=>_(e.updates,t)))};readScreenshots=e=>new Map(e.flatMap(({screenshots:e})=>[...e]));_prepareAndCommit=async(e,t)=>{let i,n,r,o=this._context.domainContextSelectors.getStoreSystem(),s=await this.prepare(o.stateSlices.committed.value.state,e,t),a=s.map(({result:e})=>e),l=this.readSideEffects(s),h=this.readScreenshots(s),d=s.flatMap(e=>e.emissions);if(!t.active)return{results:a,sideEffects:l,screenshots:h,outcome:"conflict"};if(a.some(O)||0===d.length)return{results:a,sideEffects:l,screenshots:h,outcome:"none"};let u=(i=c(d[0]),n=c(d[d.length-1]),r=d.some(e=>void 0!==e.transformationLog),{kind:"committed",previousState:i.previousState,state:n.state,updates:d.flatMap(e=>e.updates),validity:n.validity,transformationLog:r?d.flatMap(e=>e.transformationLog??[]):void 0});return{results:a,sideEffects:l,screenshots:h,outcome:o.commitPrepared(u),preparedState:u.state}};execute=async e=>{let t=this._generation;if(!this._context.domainContextSelectors.getStoreSystem().isInitialized)return{results:e.map(()=>$),committed:!1,sideEffects:[],screenshots:new Map};let i=await this._prepareAndCommit(e,t),{results:n,sideEffects:r,screenshots:o,outcome:s,preparedState:a}="conflict"===i.outcome&&t.active?await this._prepareAndCommit(e,t):i;return"committed"===s?{results:n,committed:!0,sideEffects:r,screenshots:o,preparedState:a}:"held"===s?{results:n,committed:!1,message:"The changes wait for the user's approval and are not applied yet. Do not send them again; the next `[changes]` block lists them once approved.",sideEffects:r,screenshots:o,preparedState:a}:"conflict"===s?{results:n,committed:!1,message:"The changes were not committed, because the configuration changed while they were being applied. The results describe a state that was discarded. Query the current configuration and send the operations again if they still make sense.",sideEffects:r,screenshots:o}:{results:n,committed:!1,sideEffects:r,screenshots:o}}}
+import{GraphQLError as e,NoSchemaIntrospectionCustomRule as t,defaultFieldResolver as i,execute as n,getNamedType as o,parse as r,specifiedRules as s,validate as a,validateSchema as l}from"graphql";import{assertExistence as c,createSelector as h,isMessageDescriptor as d,isMultilingualString as u}from"@salsita-ai/common/utils";import{DomainEntityIdSchema as f,agentScreenshotComponent as m,createAssistantExecutionSchema as p,findAssistantMutation as g}from"#package/domain";import{ExportBusyError as y}from"../export-system/export-busy-error.js";import{closeUnclosedBraces as v}from"./close-unclosed-braces.js";import{createEntityMeta as S}from"./entity-meta.js";import{injectAssistantLenses as _}from"./inject-assistant-lenses.js";import{readSideEffects as w}from"./side-effects.js";let x={errors:[new e("The configurator does not provide this request. This is a fault in the configurator, not in your request. Tell the user this is not available.")]},$={errors:[new e("The configuration is not loaded yet, so there is no state to query. Run the query again once the configurator has finished loading.")]},b="Not taken: another configuration was loaded while this request ran. Query the current configuration before asking for screenshots again.",E=new RegExp(f.pattern),T=[...s,t],C=e=>"object"==typeof e&&null!==e&&"type"in e&&"string"==typeof e.type&&"id"in e&&"string"==typeof e.id,M=(e,t)=>"object"==typeof e&&null!==e?Reflect.get(e,t):void 0,j=e=>e instanceof Error?e.message:String(e),k=e=>/[.!?]$/.test(e)?e:`${e}.`,R=e=>"object"==typeof e&&null!==e&&Object.getPrototypeOf(e)===Object.prototype,N=(e,t)=>{let i=R(e)?e._raw:void 0;if(C(i))return i;if(!Array.isArray(e)&&!R(e))return e;if(t.has(e))throw Error("The value contains itself.");let n=new Set([...t,e]);return Array.isArray(e)?e.map(e=>N(e,n)):Object.fromEntries(Object.entries(e).map(([e,t])=>[e,N(t,n)]))};export const localize=(e,t)=>Array.isArray(e)?e.map(e=>localize(e,t)):"string"==typeof e||d(e)||u(e)?t(e):e;let A=t=>new e(`The configurator cannot handle this request: ${k(t)} This is a fault in the configurator, not in your request. Tell the user this is not available.`),q="SCREENSHOT_NOT_TAKEN",z=(t,i)=>new e(t,{originalError:i,extensions:{code:q}}),L=e=>(e.errors??[]).some(e=>e.extensions.code!==q),F=t=>({errors:[new e(`Not run: operation ${t+1} failed, so the operations after it were skipped and no change of the list takes effect. Fix the failing operation and send the whole list again.`)]});export class GraphQLSystem{_context;_selectSchema=h([h.leaf(e=>e)],e=>{try{let t=p(e),i=l(t);return i.length>0?A(i.map(e=>e.message).join(" ")):t}catch(e){return A(j(e))}});_selectLenses=h([h.leaf(e=>e)],e=>r(e));_selectFieldResolver=h([h.leaf(e=>e)],t=>(n,r,s,a)=>{if(a.parentType===a.schema.getQueryType())return i(n,r,s,a);if(a.parentType===a.schema.getMutationType())return this._resolveMutation(a.fieldName,String(a.path.key),r,s);if(!C(n))return i(n,r,s,a);if("meta"===a.fieldName){let e;return S({entity:n,state:("object"==typeof(e=M(n,"_state"))&&null!==e&&"entities"in e&&"rootEntityId"in e?e:void 0)??s.state},{manifest:s.manifest,recipes:this._recipeContext(),prepareStateEmission:(e,t,i)=>this._context.domainContextSelectors.getStoreSystem().prepareStateEmission(e,t,i),find$Entity:(e,t)=>this._find$Entity(e,t),validateEntity:e=>this._context.domainContextSelectors.getDomain().validateEntity(e),takeScreenshot:(e,t)=>this._takeScreenshot(e,t,s)})}let{source:l,localized:c}=t[a.parentType.name]?.[a.fieldName]??{source:a.fieldName},h=((t,i)=>{try{return M(t,i)}catch(n){throw new e(`Expression "${i}" failed on ${C(t)?t.id:"an entity"}: ${k(j(n))} This is a fault in the configurator, not in your request. Tell the user this is not available.`,{originalError:n instanceof Error?n:void 0})}})(n,l);return c?localize(h,this._context.domainServicesSelectors.getResolveLocalizable()):l.startsWith("$")&&"JSON"===o(a.returnType).name?((t,i,n)=>{try{return N(n,new Set)}catch(n){throw new e(`Expression "${i}" returned a value that is not plain serializable data on ${C(t)?t.id:"an entity"}: ${k(j(n))} This is a fault in the configurator, not in your request. Tell the user this is not available.`,{originalError:n instanceof Error?n:void 0})}})(n,l,h):h});constructor(e){this._context=e}_get$Entity=(e,t)=>this._context.domainContextSelectors.getDomain().get$Entity(e,t);_find$Entity=(e,t)=>this._context.domainContextSelectors.getDomain().getEntity(e,t)?this._get$Entity(e,t):null;_recipeContext=()=>({componentReader:this._context.domainContextSelectors.getComponentReader(),recipeCooker:this._context.domainContextSelectors.getRecipeCookerSystem(),resolveLocalizable:this._context.domainServicesSelectors.getResolveLocalizable()});_rootValue={root:(e,{state:t})=>this._get$Entity(t.rootEntityId,t),entities:({ids:t},{state:i})=>{if(null==t)return i.entities.map(e=>this._get$Entity(e.id,i));var n=i.rootEntityId;let o=t.filter(e=>!E.test(e));if(o.length>0)throw new e(`${o.map(e=>`"${e}"`).join(", ")} ${1===o.length?"is not an entity id":"are not entity ids"}. Entity ids have the form "<Type>-<number>", e.g. "${n}". Use ids as returned by the "id" field.`);return t.map(e=>E.test(e)?this._find$Entity(e,i):null)},selectedEntity:(e,{state:t})=>this._find$Entity(this._context.domainContextSelectors.getNavigationSystem().getActiveSelectionState(t),t),navigation:(e,{state:t})=>{let i=this._context.domainContextSelectors.getNavigationSystem();return{navigationState:i.getActiveNavigationState(t),viewState:i.getActiveViewState(t),toolState:i.getActiveToolState(t)}}};_resolveType=e=>C(e)?e.type:void 0;_generation={active:!0};_renderScreenshot=async(e,t,i)=>{let n=this._context.domainContextSelectors.getComponentReader().readOneComponent({entityId:e,componentType:m,instanceName:t},i);if(!n)return null;let o=await this._context.renderScreenshot(n,i);return null===o?null:new Blob([o],{type:"JPEG"===n.state.fileFormat?"image/jpeg":"image/png"})};initialize=()=>{this._generation.active=!1,this._generation={active:!0}};_resolveMutation=(t,i,n,o)=>{if(void 0!==o.failedMutationField)throw new e(`Not run: the earlier field "${o.failedMutationField}" of this operation failed, so "${i}" was skipped and no change of the operation takes effect. Fix the failing field and send the operation again.`);try{return this._runMutation(t,n,o)}catch(e){throw o.failedMutationField=i,e}};_runMutation=(t,i,n)=>{let o=g(t,n.manifest);if(!o)throw new e(`The schema declares the mutation "${t}", but the configurator has no implementation for it. This is a fault in the configurator, not in your request. Tell the user this is not available.`);let r=n.state,s=o.implementation(i,r,{manifest:n.manifest,action:this._context.domainContextSelectors.getActionSystem().createContext(r),resolveLocalizable:this._context.domainServicesSelectors.getResolveLocalizable()});if(s.updates.length>0){let i=this._context.domainContextSelectors.getStoreSystem().prepareStateEmission(s.updates,r,"committed");if(!i.validity.valid)throw new e(`The changes of "${t}" were not applied, because the resulting configuration is invalid: ${(e=>{if("uncaught"===e.severity)return`validating it threw: ${k(j(e.error))}`;let t=[...new Set(e.entityViolations.filter(({violation:t})=>t.severity===e.severity).map(({entityId:e})=>e))];return`it has ${e.severity} violations on ${t.join(", ")}.`})(i.validity)} Change the request so the configuration stays valid.`);n.emissions.push(i),n.state=i.state}return s.readResult({previousState:r,state:n.state})};_takeScreenshot=({entity:t,state:i},n,o)=>{let r=o.manifest.screenshots?.[t.type]?.find(e=>e.view===n);if(!r)return Promise.reject(new e(`The schema declares the screenshot view "${n}" on ${t.type}, but the configurator has no view of that name. This is a fault in the configurator, not in your request. Tell the user this is not available.`));let s=o.lastScreenshot.then(async()=>{if(!o.generation.active)throw z(b);let e=await this._renderScreenshot(t.id,r.instanceName,i).catch(e=>{throw z(e instanceof y?"User made concurrent actions that prevented the screenshot from being taken, try again":`The screenshot of ${t.id} from "${n}" could not be taken: ${k(j(e))} Try again; if it fails again, tell the user the screenshot is not available.`,e instanceof Error?e:void 0)});if(!o.generation.active)throw z(b);if(null===e)return null;let{imageId:s}=await this._context.uploadScreenshot(e).catch(e=>{throw z(`The screenshot of ${t.id} from "${n}" was taken but could not be stored: ${k(j(e))} Try again; if it fails again, tell the user the screenshot is not available.`,e instanceof Error?e:void 0)}),a=`${t.id}/${n}`,l=[...o.screenshots.values()].filter(e=>e.entityId===t.id&&e.view===n).length,c=0===l?a:`${a}#${l+1}`;return o.screenshots.set(c,{entityId:t.id,view:n,imageId:s}),{image:c}});return o.lastScreenshot=s.catch(()=>void 0),s};_setup=()=>{let{manifest:t,error:i}=this._context.getAssistantSetup();if(i)return{ready:!1,result:{errors:[new e(i.message,{originalError:i})]}};if(!t)return{ready:!1,result:x};let n=this._selectSchema(t.sdl);return n instanceof e?{ready:!1,result:{errors:[n]}}:{ready:!0,assistant:t,schema:n,lenses:t.lenses?.definitions?this._selectLenses(t.lenses.definitions):void 0,fieldResolver:this._selectFieldResolver(t.fields)}};_run=(t,i,o,s)=>{let l;try{l=r(t)}catch(t){return{errors:[t instanceof e?t:new e(j(t))]}}let c=a(o.schema,l,T);return c.length>0?{errors:c}:n({schema:o.schema,document:l,rootValue:this._rootValue,contextValue:s,variableValues:i.variables,operationName:i.operationName,fieldResolver:o.fieldResolver,typeResolver:this._resolveType})};prepare=async(t,i,n=this._generation)=>{let o,r=this._setup();if(!r.ready)return i.map(()=>({result:r.result,emissions:[],screenshots:new Map}));let s={manifest:r.assistant,state:t,emissions:[],failedMutationField:void 0,screenshots:new Map,lastScreenshot:Promise.resolve(),generation:n},a=[];for(let[t,n]of i.entries())if(void 0===o){let i=s.emissions.length,l=s.screenshots.size;s.failedMutationField=void 0;let c=v(n.document),h=r.lenses?_(c,r.lenses):c,d=h instanceof e?{errors:[h]}:await this._run(h,n,r,s);await s.lastScreenshot,a.push({result:d,emissions:s.emissions.slice(i),screenshots:new Map([...s.screenshots].slice(l))}),L(d)&&(o=t)}else a.push({result:F(o),emissions:[],screenshots:new Map});return a};readSideEffects=e=>{let t=this._context.domainServicesSelectors.getResolveLocalizable();return e.flatMap(({emissions:e})=>e.flatMap(e=>w(e.updates,t)))};readScreenshots=e=>new Map(e.flatMap(({screenshots:e})=>[...e]));_prepareAndCommit=async(e,t)=>{let i,n,o,r=this._context.domainContextSelectors.getStoreSystem(),s=await this.prepare(r.stateSlices.committed.value.state,e,t),a=s.map(({result:e})=>e),l=this.readSideEffects(s),h=this.readScreenshots(s),d=s.flatMap(e=>e.emissions);if(!t.active)return{results:a,sideEffects:l,screenshots:h,outcome:"conflict"};if(a.some(L)||0===d.length)return{results:a,sideEffects:l,screenshots:h,outcome:"none"};let u=(i=c(d[0]),n=c(d[d.length-1]),o=d.some(e=>void 0!==e.transformationLog),{kind:"committed",previousState:i.previousState,state:n.state,updates:d.flatMap(e=>e.updates),validity:n.validity,transformationLog:o?d.flatMap(e=>e.transformationLog??[]):void 0});return{results:a,sideEffects:l,screenshots:h,outcome:r.commitPrepared(u),preparedState:u.state}};execute=async e=>{let t=this._generation;if(!this._context.domainContextSelectors.getStoreSystem().isInitialized)return{results:e.map(()=>$),committed:!1,sideEffects:[],screenshots:new Map};let i=await this._prepareAndCommit(e,t),{results:n,sideEffects:o,screenshots:r,outcome:s,preparedState:a}="conflict"===i.outcome&&t.active?await this._prepareAndCommit(e,t):i;return"committed"===s?{results:n,committed:!0,sideEffects:o,screenshots:r,preparedState:a}:"held"===s?{results:n,committed:!1,message:"The changes wait for the user's approval and are not applied yet. Do not send them again; the next `[changes]` block lists them once approved.",sideEffects:o,screenshots:r,preparedState:a}:"conflict"===s?{results:n,committed:!1,message:"The changes were not committed, because the configuration changed while they were being applied. The results describe a state that was discarded. Query the current configuration and send the operations again if they still make sense.",sideEffects:o,screenshots:r}:{results:n,committed:!1,sideEffects:o,screenshots:r}}}

package/dist/configurator/ui/assistant/screenshots/ScreenshotThumbnails.d.ts

@@ -4,6 +4,6 @@ interface ScreenshotThumbnailsProps {
     /** Called once a picture has loaded, which changes the size of the popover. */
     readonly onImageLoad: () => void;
 }
-/** The pictures of a tool call's screenshots, each captioned by its view and entity id. */
+/** The pictures of a tool call's screenshots. */
 export declare const ScreenshotThumbnails: ({ screenshots, onImageLoad }: ScreenshotThumbnailsProps) => import("react/jsx-runtime").JSX.Element;
 export {};

package/dist/configurator/ui/assistant/screenshots/ScreenshotThumbnails.js

@@ -1 +1 @@
-import{jsx as e,jsxs as i}from"react/jsx-runtime";import{styled as r}from"@mui/material";import{AssistantPanelName as o}from"../theme.js";let a=r("span",{name:o,slot:"screenshots"})(({theme:{palette:e,shape:i,shadows:r,spacing:o,typography:a}})=>({...a.caption,background:e.background.paper,borderRadius:i.borderRadius,boxShadow:r[4],color:e.text.secondary,display:"grid",gap:o(1.5),gridTemplateColumns:"repeat(auto-fill, minmax(140px, 1fr))",inlineSize:"min(80vw, 480px)",padding:o(1.5)})),n=r("span")(({theme:{spacing:e}})=>({display:"flex",flexDirection:"column",gap:e(.5),minInlineSize:0})),t=r("img")(({theme:{palette:e,shape:i}})=>({blockSize:"auto",borderRadius:i.borderRadius/2,boxShadow:`inset 0 0 0 1px ${e.divider}`,inlineSize:"100%"})),s=r("span")({overflow:"hidden",textOverflow:"ellipsis",whiteSpace:"nowrap"});export const ScreenshotThumbnails=({screenshots:r,onImageLoad:o})=>e(a,{children:r.map(({label:r,entityId:a,view:d,src:l})=>i(n,{children:[e(t,{src:l,alt:"",onLoad:o}),i(s,{title:r,children:[d," \xb7 ",a]})]},r))});
+import{jsx as e}from"react/jsx-runtime";import{styled as o}from"@mui/material";import{AssistantPanelName as r}from"../theme.js";import{useScreenshotSrc as a}from"./use-screenshot-src.js";let i=o("span",{name:r,slot:"screenshots",shouldForwardProp:e=>"columns"!==e})(({columns:e,theme:{palette:o,shape:r,shadows:a,spacing:i}})=>({background:o.background.paper,borderRadius:r.borderRadius,boxShadow:a[4],display:"grid",gap:i(1.5),gridTemplateColumns:`repeat(${e}, minmax(0, 200px))`,maxInlineSize:"80vw",padding:i(1.5)})),s=o("img")(({theme:{palette:e,shape:o}})=>({blockSize:"auto",borderRadius:o.borderRadius/2,boxShadow:`inset 0 0 0 1px ${e.divider}`,display:"block",inlineSize:"100%"})),d=o("span")(({theme:{palette:e,shape:o}})=>({aspectRatio:"4 / 3",background:e.action.hover,borderRadius:o.borderRadius/2,display:"block"})),t=({picture:o,onImageLoad:r})=>{let i=a(o);return void 0===i?e(d,{}):e(s,{src:i,alt:"",onLoad:r})};export const ScreenshotThumbnails=({screenshots:o,onImageLoad:r})=>e(i,{columns:Math.min(o.length,2),children:o.map(({label:o,picture:a})=>e(t,{picture:a,onImageLoad:r},o))});

package/dist/configurator/ui/assistant/screenshots/tool-screenshots.d.ts

@@ -1,11 +1,19 @@
+/**
+ * Where the panel gets a screenshot's picture from: the server, by the id the browser uploaded it
+ * under, or the data URL of a picture a thread from before the upload carries inline.
+ */
+export type AssistantScreenshotPicture = {
+    readonly kind: 'stored';
+    readonly imageId: string;
+} | {
+    readonly kind: 'inline';
+    readonly src: string;
+};
 /** A screenshot a `query` or `apply` call took, as the panel shows it. */
 export interface AssistantToolScreenshot {
     /** The label the call's `image` field returned, such as `Cabinet-2/doorLayout#2`. */
     readonly label: string;
-    readonly entityId: string;
-    readonly view: string;
-    /** The picture as a data URL. */
-    readonly src: string;
+    readonly picture: AssistantScreenshotPicture;
 }
 /**
  * The screenshots of a tool result: the image parts that follow the result's JSON, each after the

package/dist/configurator/ui/assistant/screenshots/tool-screenshots.js

@@ -1 +1 @@
-import{z as e}from"zod";let t=e.object({type:e.literal("text"),content:e.string()}),a=e.object({type:e.literal("image"),source:e.object({type:e.literal("data"),value:e.string(),mimeType:e.string()})}),s=/^([^/]+)\/([^#]+)(?:#\d+)?$/;export const readToolScreenshots=e=>Array.isArray(e)?e.slice(1).flatMap((e,r,c)=>{let l=a.safeParse(e),o=t.safeParse(c[r-1]),n=o.success?s.exec(o.data.content):null;if(!l.success||!o.success||!n?.[1]||!n[2])return[];let{value:i,mimeType:d}=l.data.source;return[{label:o.data.content,entityId:n[1],view:n[2],src:`data:${d};base64,${i}`}]}):[];
+import{z as t}from"zod";import{readAssistantImageReference as e}from"@salsita-ai/contracts/assistant";let a=t.object({type:t.literal("text"),content:t.string()}),r=t.object({type:t.literal("image"),source:t.discriminatedUnion("type",[t.object({type:t.literal("url"),value:t.string()}),t.object({type:t.literal("data"),value:t.string(),mimeType:t.string()})])}),s=/^[^/]+\/[^#]+(?:#\d+)?$/;export const readToolScreenshots=t=>Array.isArray(t)?t.slice(1).flatMap((t,l,i)=>{let n=r.safeParse(t),o=a.safeParse(i[l-1]),c=n.success?(({source:t})=>{if("data"===t.type)return{kind:"inline",src:`data:${t.mimeType};base64,${t.value}`};let a=e(t.value);return null===a?null:{kind:"stored",imageId:a}})(n.data):null;return c&&o.success&&s.test(o.data.content)?[{label:o.data.content,picture:c}]:[]}):[];

package/dist/configurator/ui/assistant/screenshots/use-screenshot-src.d.ts

@@ -0,0 +1,8 @@
+import type { AssistantScreenshotPicture } from './tool-screenshots.js';
+/**
+ * The `src` of a screenshot's picture: the data URL of an inline one at once, and for an uploaded
+ * one the object URL the assistant system reads it into, once it has been read; `undefined` until
+ * then and when the read fails. Only the open popover of a call's screenshots calls it, so nothing
+ * is read while the popover is closed.
+ */
+export declare const useScreenshotSrc: (picture: AssistantScreenshotPicture) => string | undefined;

package/dist/configurator/ui/assistant/screenshots/use-screenshot-src.js

@@ -0,0 +1 @@
+import{useEffect as r,useState as e}from"react";import{useConfigurator as n}from"../../../application-runtime/index.js";export const useScreenshotSrc=t=>{let{assistantSystem:i}=n(),o="stored"===t.kind?t.imageId:null,[l,d]=e();return(r(()=>{if(null===o)return;let r=!0;return i.loadScreenshotUrl(o).then(e=>{r&&d({imageId:o,src:e})},()=>void 0),()=>{r=!1}},[i,o]),"inline"===t.kind)?t.src:l?.imageId===o?l.src:void 0};

package/dist/docs/index.json

@@ -1,5 +1,5 @@
 {
-  "generatedAt": "2026-10-17T19:26:17.771Z",
+  "generatedAt": "2026-10-18T06:20:03.099Z",
   "categories": [
     {
       "name": "Phoenix",

package/dist/domain/application/assistant/build-assistant-manifest.js

@@ -1 +1 @@
-import{GraphQLBoolean as e,GraphQLEnumType as t,GraphQLID as n,GraphQLInputObjectType as i,GraphQLInterfaceType as s,GraphQLList as r,GraphQLNonNull as o,GraphQLObjectType as a,GraphQLSchema as p,GraphQLString as c,printSchema as d,validateSchema as l}from"graphql";import{GraphQLJSON as h,createAssistantValueType as w}from"./assistant-value-type.js";import{buildAssistantLenses as y}from"./build-assistant-lenses.js";import{createAssistantExecutionSchema as u}from"./create-assistant-execution-schema.js";let m="AnyDomainEntity",f="RootEntity",g=[["navigationState","Active navigation state.","navigationStates"],["viewState","Active view state.","viewStates"],["toolState","Active tool state.","toolStates"]],b={section:"childRecipes",mutation:"addChild"},v={section:"replaceRecipes",mutation:"replaceEntity"},j=(e,t)=>Object.entries(e).find(([e])=>e===t)?.[1],$=(e,t)=>{if(!t||0===t.length)return e;let n=`One of: ${t.map(e=>`"${e}"`).join(", ")}.`;return e?/[.!?]$/.test(e)?`${e} ${n}`:`${e}. ${n}`:n},O=({target:e,cardinality:t},n)=>{let i=n(e);switch(t){case"1-1":return new o(i);case"0-1":return i;case"0-N":case"1-N":return new o(new r(new o(i)))}},E=(e,t,i)=>{let s={id:{type:new o(n),description:i?`Unique id of the entity, assigned by the configurator, e.g. "${i}-1".`:void 0},type:{type:new o(c),description:i?`Entity type name, e.g. "${i}".`:void 0}};for(let n of e)s[n.name]={type:"edge"===n.type.kind?O(n.type,t):w(n.type.type),description:n.description};return s},R=e=>Object.fromEntries(e.map(({name:e,source:t,localized:n})=>[e,n?{source:t,localized:!0}:{source:t}])),k={sameForAllInstances:e=>`Same recipe groups and recipes for every ${e}.`,constantDuringSession:()=>"Fixed for the session."},T={sameForAllInstances:e=>`Same for every ${e}.`,constantDuringSession:()=>"Fixed for the session."},I=(e,t,n)=>e.map(e=>` ${t[e](n)}`).join(""),N=({section:e},{groups:t,hints:n},{entityType:i,purpose:s,recipeGroup:a})=>{let p=t.filter(({dryRuns:e})=>!e).map(({groupId:e})=>e);return 0===t.length?{}:{[e]:{type:new o(new r(new o(a))),description:`The recipe groups for ${s} this ${i}: ${t.map(({groupId:e})=>e).join(", ")}. A group is listed only while the configurator offers it on the entity.${p.length>0?` Recipes of these groups cannot be dry-run: ${p.join(", ")}.`:""}${I(n,k,i)}`}}},A=({entityType:e,childRecipes:n,replaceRecipes:i,optionsEdges:s,screenshots:p},{recipeGroup:c,fieldOption:d,screenshot:l})=>new a({name:`${e}Meta`,fields:Object.assign(N(b,n,{entityType:e,purpose:"adding children to",recipeGroup:c}),N(v,i,{entityType:e,purpose:"replacing",recipeGroup:c}),...s.map(({options:{edgeName:t,field:n,validate:i,dryRuns:s},hints:a})=>({[t]:{type:new r(new o(d)),description:`Values "${n}" accepts on this ${e}. Null when there is no choice to make.${i?"":" Not checked on write."}${s?"":" Cannot be dry-run."}${I(a,T,e)}`}})),0===p.length?{}:{screenshot:{type:l,args:{view:{type:new o(new t({name:`${e}ScreenshotView`,values:Object.fromEntries(p.map(({view:e,description:t})=>[e,{description:t}]))}))}},description:"Renders this entity from a predefined view. The picture follows the tool result as an image labelled with `image`. Each one costs tokens and blocks the user's configurator while it renders: ask only when seeing the design answers something the data cannot. The images of one call share a size budget; past it, further screenshots come back null. Null as well when the view does not apply to this entity right now."}})}),S=e=>Object.entries(e).map(([e,t])=>`${e} (${t.map(({groupId:e})=>e).join(", ")})`);export const buildAssistantManifest=t=>{var O,k,T,I,N;let M,D,F,W,{entityTypes:x,aliases:C,exampleType:q,fields:G,states:U,deletableTypes:K,writableTypes:V,actionFields:z,optionsEdges:P}=t,L=Object.fromEntries(Object.entries(t.childRecipes).map(([e,t])=>[e,t.groups])),Q=Object.fromEntries(Object.entries(t.replaceRecipes).map(([e,t])=>[e,t.groups])),B=new Map,H=new Map,J=new Map,X=e=>{let[t,...n]=Object.entries(C).find(([t])=>t===e)?.[1]??[];return(t&&0===n.length?H.get(t):void 0)??J.get(e)??Z},Y=e=>H.get(e)??X(e),Z=new s({name:m,description:"Every entity of the configuration.",fields:()=>E(G[m]??[],Y,q)});for(let e of(J.set(m,Z),Object.keys(C)))e!==m&&J.set(e,new s({name:e,interfaces:[Z],fields:()=>E(G[e]??[],Y)}));let _={};for(let e of x){let t=[m,...Object.entries(C).filter(([t,n])=>t!==m&&n.includes(e)).map(([e])=>e)],n=G[e]??[];_[e]=R(n),H.set(e,new a({name:e,interfaces:t.flatMap(e=>J.get(e)??[]),fields:()=>{let t=B.get(e);return{...E(n,Y),...t?{meta:{type:new o(t),description:"What can be done to this entity right now."}}:{}}}}))}let ee=(O=()=>X(J.has(f)?f:m),k=()=>X(m),M=new a({name:"DryRunError",description:"One reason a change cannot be made right now.",fields:{type:{type:new o(c),description:"Kind of the problem."},entityId:{type:n,description:"The entity the problem is on, when it is on one."},message:{type:c,description:"The problem as the user would be told it."}}}),new a({name:"DryRun",description:"What using a recipe or writing an option would do, tried out on the configuration the query reads. Nothing is changed.",fields:()=>({ok:{type:new o(e),description:"Whether the change would succeed on the configuration the query reads."},root:{type:O(),description:"The root entity of the configuration as it would be after the change; null when ok is false. Everything selected under it reads that configuration."},changedEntities:{type:new o(new r(new o(k()))),description:"Entities the change would add or modify; empty when ok is false. One whose id is not in the configuration would be added, and its real id may differ. An entity written to is listed even when its values stay the same."},deletedEntityIds:{type:new o(new r(new o(n))),description:"The ids of the entities the change would remove; empty when ok is false."},sideEffects:{type:new o(new r(new o(c))),description:"The major or unexpected things the configurator would change along with it; empty when ok is false."},errors:{type:new o(new r(new o(M))),description:"Why the change cannot be made; empty when ok is true."}})})),et={recipeGroup:(T=[...Object.keys(L).length>0?[b.mutation]:[],...Object.keys(Q).length>0?[v.mutation]:[]],D=T.join(" or "),F=new a({name:"Recipe",description:`${T.includes("addChild")?"One thing that can be added to the entity (under childRecipes)":"One thing the entity can be replaced with (under replaceRecipes)"}${T.length>1?" or that it can be replaced with (under replaceRecipes)":""}.`,fields:{key:{type:new o(c),description:`Pass to ${D} as recipeKey. Valid only for the listing it came from.`},name:{type:new o(c),description:"Name shown to the user."},description:{type:c},imageUrl:{type:c,description:"Thumbnail shown to the user."},dryRun:{type:ee,description:"What using this recipe would do. Null when it cannot be tried."}}}),new a({name:"RecipeGroup",description:"A group of recipes for this entity.",fields:{groupId:{type:new o(c),description:`Pass to ${D} as recipeGroupId.`},name:{type:new o(c),description:"Name shown to the user."},description:{type:c},recipes:{type:new o(new r(new o(F)))}}})),fieldOption:new a({name:"FieldOption",description:"One value a field accepts. Writing a value outside the list is rejected unless the field description says it is not checked.",fields:{value:{type:new o(h),description:"The value to write to the field."},name:{type:c,description:"Name shown to the user."},description:{type:c},imageUrl:{type:c,description:"Image shown to the user."},color:{type:c,description:'Color shown to the user, as "#rrggbb".'},dryRun:{type:ee,description:"What writing this value would do. Null when it cannot be tried."}}}),screenshot:new a({name:"Screenshot",description:"A screenshot of an entity. The picture itself follows the tool result.",fields:{image:{type:new o(n),description:"Label of the image that follows the tool result."}}})};for(let e of x){let n=j(t.childRecipes,e)??{groups:[],hints:[]},i=j(t.replaceRecipes,e)??{groups:[],hints:[]},s=j(P,e)??[],r=j(t.screenshots,e)??[];(n.groups.length>0||i.groups.length>0||s.length>0||r.length>0)&&B.set(e,A({entityType:e,childRecipes:n,replaceRecipes:i,optionsEdges:s.map(t=>({...t,hints:(C[f]??[]).includes(e)?t.hints.filter(e=>"sameForAllInstances"!==e):t.hints})),screenshots:r},et))}let en=new p({query:(I=X(J.has(f)?f:m),N=X(m),W=(e=>{let t={};for(let[n,i,s]of g){let r=e[s];r.length>1&&(t[n]={type:new o(c),description:$(i,r)})}if(0!==Object.keys(t).length)return new a({name:"Navigation",description:"The active UI states of the configurator.",fields:t})})(U),new a({name:"Query",fields:{root:{type:new o(I),description:"The root entity of the configuration."},entities:{type:new o(new r(N)),args:{ids:{type:new r(new o(n))}},description:"Every entity, or the given ids in order with null for an unknown id."},selectedEntity:{type:new o(N),description:"The selected entity, or the root entity when nothing is selected."},...W?{navigation:{type:new o(W)}}:{}}})),mutation:(({deletableTypes:e,writableTypes:t,childRecipes:s,replaceRecipes:p,actionFields:d,objectTypes:l,anyDomainEntity:h,targetOutputType:y})=>{let u,m;if(0!==e.length||0!==t.length||0!==Object.keys(s).length||0!==Object.keys(p).length||0!==d.length)return new a({name:"Mutation",fields:Object.assign(0===e.length?{}:{deleteEntity:{type:new o(new r(new o(n))),args:{id:{type:new o(n)}},description:`Deletes the entity and everything composed into it. Deletable types: ${e.join(", ")}. Returns the ids removed, the given one first.`}},0===(u=S(s)).length?{}:{addChild:{type:new o(new r(new o(h))),args:{parentId:{type:new o(n)},recipeGroupId:{type:new o(c)},recipeKey:{type:new o(c)}},description:`Adds a child to the entity parentId from a recipe listed under its meta { childRecipes }. Types with child recipes: ${u.join("; ")}. Returns the entities added, the recipe's own first; empty when they were removed again.`}},0===(m=S(p)).length?{}:{replaceEntity:{type:new o(new r(new o(h))),args:{entityId:{type:new o(n)},recipeGroupId:{type:new o(c)},recipeKey:{type:new o(c)}},description:`Replaces the entity entityId from a recipe listed under its meta { replaceRecipes }. Types with replace recipes: ${m.join("; ")}. The replacement is a new entity: it has a new id and the field values of its recipe, and entityId no longer exists. Returns the entities added, the replacement first; empty when they were removed again.`}},...t.flatMap(e=>{let t=l.get(e.entityType);return t?[(({entityType:e,fields:t},s)=>{let r=Object.fromEntries(t.map(({name:e,type:t,description:n})=>[e,{type:w(t),description:n}]));return{[`modify${e}`]:{type:s,args:{id:{type:new o(n)},delta:{type:new o(new i({name:`${e}Delta`,fields:r}))}},description:`Sets the fields named in delta on one ${e}. Returns the entity after the change, or null when the change removed it.`}}})(e,t)]:[]}),Object.fromEntries(d.map(e=>{var t;return[e.fieldName,{type:new o(new a({name:(t=e.fieldName,`${t.charAt(0).toUpperCase()+t.slice(1)}Result`),description:`What ${e.fieldName} returns, read from the configuration after the change.`,fields:()=>Object.fromEntries(Object.entries(e.result).map(([e,t])=>[e,{type:"edge"===t.kind?(({target:e,list:t,nullable:n},i)=>{let s=t?new r(new o(i(e))):i(e);return n?s:new o(s)})(t,y):w(t.type),description:t.description}]))})),args:Object.fromEntries(Object.entries(e.args).map(([e,t])=>[e,{type:w(t.type),description:t.description}])),description:e.description}]})))})})({deletableTypes:K,writableTypes:V,childRecipes:L,replaceRecipes:Q,actionFields:z,objectTypes:H,anyDomainEntity:X(m),targetOutputType:Y}),types:[...J.values(),...H.values()]}),ei=l(en);if(ei.length>0)throw Error(ei.map(e=>e.message).join(" "));let es=d(en);return{sdl:es,navigationFields:g.filter(([,,e])=>U[e].length>1).map(([e])=>e),fields:_,deletableTypes:K,writableFields:Object.fromEntries(V.map(({entityType:e,fields:t})=>[e,Object.fromEntries(t.map(({name:e,property:t})=>[e,t]))])),childRecipes:L,replaceRecipes:Q,fieldOptions:Object.fromEntries(x.flatMap(e=>{let t=j(P,e);return t?[[e,t.map(({options:e})=>e)]]:[]})),screenshots:Object.fromEntries(x.flatMap(e=>{let n=j(t.screenshots,e);return n?[[e,n]]:[]})),actions:Object.fromEntries(z.map(({fieldName:e,actionName:t})=>[e,t])),lenses:y(t,u(es)),formElements:t.formElements}};
+import{GraphQLBoolean as e,GraphQLEnumType as t,GraphQLID as n,GraphQLInputObjectType as i,GraphQLInterfaceType as s,GraphQLList as r,GraphQLNonNull as o,GraphQLObjectType as a,GraphQLSchema as p,GraphQLString as d,printSchema as c,validateSchema as l}from"graphql";import{GraphQLJSON as h,createAssistantValueType as w}from"./assistant-value-type.js";import{buildAssistantLenses as y}from"./build-assistant-lenses.js";import{createAssistantExecutionSchema as u}from"./create-assistant-execution-schema.js";let m="AnyDomainEntity",f="RootEntity",g=[["navigationState","Active navigation state.","navigationStates"],["viewState","Active view state.","viewStates"],["toolState","Active tool state.","toolStates"]],b={section:"childRecipes",mutation:"addChild"},v={section:"replaceRecipes",mutation:"replaceEntity"},j=(e,t)=>Object.entries(e).find(([e])=>e===t)?.[1],$=(e,t)=>{if(!t||0===t.length)return e;let n=`One of: ${t.map(e=>`"${e}"`).join(", ")}.`;return e?/[.!?]$/.test(e)?`${e} ${n}`:`${e}. ${n}`:n},O=({target:e,cardinality:t},n)=>{let i=n(e);switch(t){case"1-1":return new o(i);case"0-1":return i;case"0-N":case"1-N":return new o(new r(new o(i)))}},E=(e,t,i)=>{let s={id:{type:new o(n),description:i?`Unique id of the entity, assigned by the configurator, e.g. "${i}-1".`:void 0},type:{type:new o(d),description:i?`Entity type name, e.g. "${i}".`:void 0}};for(let n of e)s[n.name]={type:"edge"===n.type.kind?O(n.type,t):w(n.type.type),description:n.description};return s},R=e=>Object.fromEntries(e.map(({name:e,source:t,localized:n})=>[e,n?{source:t,localized:!0}:{source:t}])),k={sameForAllInstances:e=>`Same recipe groups and recipes for every ${e}.`,constantDuringSession:()=>"Fixed for the session."},T={sameForAllInstances:e=>`Same for every ${e}.`,constantDuringSession:()=>"Fixed for the session."},I=(e,t,n)=>e.map(e=>` ${t[e](n)}`).join(""),N=({section:e},{groups:t,hints:n},{entityType:i,purpose:s,recipeGroup:a})=>{let p=t.filter(({dryRuns:e})=>!e).map(({groupId:e})=>e);return 0===t.length?{}:{[e]:{type:new o(new r(new o(a))),description:`The recipe groups for ${s} this ${i}: ${t.map(({groupId:e})=>e).join(", ")}. A group is listed only while the configurator offers it on the entity.${p.length>0?` Recipes of these groups cannot be dry-run: ${p.join(", ")}.`:""}${I(n,k,i)}`}}},A=({entityType:e,childRecipes:n,replaceRecipes:i,optionsEdges:s,screenshots:p},{recipeGroup:d,fieldOption:c,screenshot:l})=>new a({name:`${e}Meta`,fields:Object.assign(N(b,n,{entityType:e,purpose:"adding children to",recipeGroup:d}),N(v,i,{entityType:e,purpose:"replacing",recipeGroup:d}),...s.map(({options:{edgeName:t,field:n,validate:i,dryRuns:s},hints:a})=>({[t]:{type:new r(new o(c)),description:`Values "${n}" accepts on this ${e}. Null when there is no choice to make.${i?"":" Not checked on write."}${s?"":" Cannot be dry-run."}${I(a,T,e)}`}})),0===p.length?{}:{screenshot:{type:l,args:{view:{type:new o(new t({name:`${e}ScreenshotView`,values:Object.fromEntries(p.map(({view:e,description:t})=>[e,{description:t}]))}))}},description:"Renders this entity from a predefined view. The picture follows the tool result as an image labelled with `image`. Null when the view does not apply to this entity right now."}})}),S=e=>Object.entries(e).map(([e,t])=>`${e} (${t.map(({groupId:e})=>e).join(", ")})`);export const buildAssistantManifest=t=>{var O,k,T,I,N;let M,D,F,W,{entityTypes:x,aliases:C,exampleType:q,fields:G,states:U,deletableTypes:K,writableTypes:V,actionFields:P,optionsEdges:z}=t,L=Object.fromEntries(Object.entries(t.childRecipes).map(([e,t])=>[e,t.groups])),Q=Object.fromEntries(Object.entries(t.replaceRecipes).map(([e,t])=>[e,t.groups])),B=new Map,H=new Map,J=new Map,X=e=>{let[t,...n]=Object.entries(C).find(([t])=>t===e)?.[1]??[];return(t&&0===n.length?H.get(t):void 0)??J.get(e)??Z},Y=e=>H.get(e)??X(e),Z=new s({name:m,description:"Every entity of the configuration.",fields:()=>E(G[m]??[],Y,q)});for(let e of(J.set(m,Z),Object.keys(C)))e!==m&&J.set(e,new s({name:e,interfaces:[Z],fields:()=>E(G[e]??[],Y)}));let _={};for(let e of x){let t=[m,...Object.entries(C).filter(([t,n])=>t!==m&&n.includes(e)).map(([e])=>e)],n=G[e]??[];_[e]=R(n),H.set(e,new a({name:e,interfaces:t.flatMap(e=>J.get(e)??[]),fields:()=>{let t=B.get(e);return{...E(n,Y),...t?{meta:{type:new o(t),description:"What can be done to this entity right now."}}:{}}}}))}let ee=(O=()=>X(J.has(f)?f:m),k=()=>X(m),M=new a({name:"DryRunError",description:"One reason a change cannot be made right now.",fields:{type:{type:new o(d),description:"Kind of the problem."},entityId:{type:n,description:"The entity the problem is on, when it is on one."},message:{type:d,description:"The problem as the user would be told it."}}}),new a({name:"DryRun",description:"What using a recipe or writing an option would do, tried out on the configuration the query reads. Nothing is changed.",fields:()=>({ok:{type:new o(e),description:"Whether the change would succeed on the configuration the query reads."},root:{type:O(),description:"The root entity of the configuration as it would be after the change; null when ok is false. Everything selected under it reads that configuration."},changedEntities:{type:new o(new r(new o(k()))),description:"Entities the change would add or modify; empty when ok is false. One whose id is not in the configuration would be added, and its real id may differ. An entity written to is listed even when its values stay the same."},deletedEntityIds:{type:new o(new r(new o(n))),description:"The ids of the entities the change would remove; empty when ok is false."},sideEffects:{type:new o(new r(new o(d))),description:"The major or unexpected things the configurator would change along with it; empty when ok is false."},errors:{type:new o(new r(new o(M))),description:"Why the change cannot be made; empty when ok is true."}})})),et={recipeGroup:(T=[...Object.keys(L).length>0?[b.mutation]:[],...Object.keys(Q).length>0?[v.mutation]:[]],D=T.join(" or "),F=new a({name:"Recipe",description:`${T.includes("addChild")?"One thing that can be added to the entity (under childRecipes)":"One thing the entity can be replaced with (under replaceRecipes)"}${T.length>1?" or that it can be replaced with (under replaceRecipes)":""}.`,fields:{key:{type:new o(d),description:`Pass to ${D} as recipeKey. Valid only for the listing it came from.`},name:{type:new o(d),description:"Name shown to the user."},description:{type:d},imageUrl:{type:d,description:"Thumbnail shown to the user."},dryRun:{type:ee,description:"What using this recipe would do. Null when it cannot be tried."}}}),new a({name:"RecipeGroup",description:"A group of recipes for this entity.",fields:{groupId:{type:new o(d),description:`Pass to ${D} as recipeGroupId.`},name:{type:new o(d),description:"Name shown to the user."},description:{type:d},recipes:{type:new o(new r(new o(F)))}}})),fieldOption:new a({name:"FieldOption",description:"One value a field accepts. Writing a value outside the list is rejected unless the field description says it is not checked.",fields:{value:{type:new o(h),description:"The value to write to the field."},name:{type:d,description:"Name shown to the user."},description:{type:d},imageUrl:{type:d,description:"Image shown to the user."},color:{type:d,description:'Color shown to the user, as "#rrggbb".'},dryRun:{type:ee,description:"What writing this value would do. Null when it cannot be tried."}}}),screenshot:new a({name:"Screenshot",description:"A screenshot of an entity. The picture itself follows the tool result.",fields:{image:{type:new o(n),description:"Label of the image that follows the tool result."}}})};for(let e of x){let n=j(t.childRecipes,e)??{groups:[],hints:[]},i=j(t.replaceRecipes,e)??{groups:[],hints:[]},s=j(z,e)??[],r=j(t.screenshots,e)??[];(n.groups.length>0||i.groups.length>0||s.length>0||r.length>0)&&B.set(e,A({entityType:e,childRecipes:n,replaceRecipes:i,optionsEdges:s.map(t=>({...t,hints:(C[f]??[]).includes(e)?t.hints.filter(e=>"sameForAllInstances"!==e):t.hints})),screenshots:r},et))}let en=new p({query:(I=X(J.has(f)?f:m),N=X(m),W=(e=>{let t={};for(let[n,i,s]of g){let r=e[s];r.length>1&&(t[n]={type:new o(d),description:$(i,r)})}if(0!==Object.keys(t).length)return new a({name:"Navigation",description:"The active UI states of the configurator.",fields:t})})(U),new a({name:"Query",fields:{root:{type:new o(I),description:"The root entity of the configuration."},entities:{type:new o(new r(N)),args:{ids:{type:new r(new o(n))}},description:"Every entity, or the given ids in order with null for an unknown id."},selectedEntity:{type:new o(N),description:"The selected entity, or the root entity when nothing is selected."},...W?{navigation:{type:new o(W)}}:{}}})),mutation:(({deletableTypes:e,writableTypes:t,childRecipes:s,replaceRecipes:p,actionFields:c,objectTypes:l,anyDomainEntity:h,targetOutputType:y})=>{let u,m;if(0!==e.length||0!==t.length||0!==Object.keys(s).length||0!==Object.keys(p).length||0!==c.length)return new a({name:"Mutation",fields:Object.assign(0===e.length?{}:{deleteEntity:{type:new o(new r(new o(n))),args:{id:{type:new o(n)}},description:`Deletes the entity and everything composed into it. Deletable types: ${e.join(", ")}. Returns the ids removed, the given one first.`}},0===(u=S(s)).length?{}:{addChild:{type:new o(new r(new o(h))),args:{parentId:{type:new o(n)},recipeGroupId:{type:new o(d)},recipeKey:{type:new o(d)}},description:`Adds a child to the entity parentId from a recipe listed under its meta { childRecipes }. Types with child recipes: ${u.join("; ")}. Returns the entities added, the recipe's own first; empty when they were removed again.`}},0===(m=S(p)).length?{}:{replaceEntity:{type:new o(new r(new o(h))),args:{entityId:{type:new o(n)},recipeGroupId:{type:new o(d)},recipeKey:{type:new o(d)}},description:`Replaces the entity entityId from a recipe listed under its meta { replaceRecipes }. Types with replace recipes: ${m.join("; ")}. The replacement is a new entity: it has a new id and the field values of its recipe, and entityId no longer exists. Returns the entities added, the replacement first; empty when they were removed again.`}},...t.flatMap(e=>{let t=l.get(e.entityType);return t?[(({entityType:e,fields:t},s)=>{let r=Object.fromEntries(t.map(({name:e,type:t,description:n})=>[e,{type:w(t),description:n}]));return{[`modify${e}`]:{type:s,args:{id:{type:new o(n)},delta:{type:new o(new i({name:`${e}Delta`,fields:r}))}},description:`Sets the fields named in delta on one ${e}. Returns the entity after the change, or null when the change removed it.`}}})(e,t)]:[]}),Object.fromEntries(c.map(e=>{var t;return[e.fieldName,{type:new o(new a({name:(t=e.fieldName,`${t.charAt(0).toUpperCase()+t.slice(1)}Result`),description:`What ${e.fieldName} returns, read from the configuration after the change.`,fields:()=>Object.fromEntries(Object.entries(e.result).map(([e,t])=>[e,{type:"edge"===t.kind?(({target:e,list:t,nullable:n},i)=>{let s=t?new r(new o(i(e))):i(e);return n?s:new o(s)})(t,y):w(t.type),description:t.description}]))})),args:Object.fromEntries(Object.entries(e.args).map(([e,t])=>[e,{type:w(t.type),description:t.description}])),description:e.description}]})))})})({deletableTypes:K,writableTypes:V,childRecipes:L,replaceRecipes:Q,actionFields:P,objectTypes:H,anyDomainEntity:X(m),targetOutputType:Y}),types:[...J.values(),...H.values()]}),ei=l(en);if(ei.length>0)throw Error(ei.map(e=>e.message).join(" "));let es=c(en);return{sdl:es,navigationFields:g.filter(([,,e])=>U[e].length>1).map(([e])=>e),fields:_,deletableTypes:K,writableFields:Object.fromEntries(V.map(({entityType:e,fields:t})=>[e,Object.fromEntries(t.map(({name:e,property:t})=>[e,t]))])),childRecipes:L,replaceRecipes:Q,fieldOptions:Object.fromEntries(x.flatMap(e=>{let t=j(z,e);return t?[[e,t.map(({options:e})=>e)]]:[]})),screenshots:Object.fromEntries(x.flatMap(e=>{let n=j(t.screenshots,e);return n?[[e,n]]:[]})),actions:Object.fromEntries(P.map(({fieldName:e,actionName:t})=>[e,t])),lenses:y(t,u(es)),formElements:t.formElements}};

package/dist/domain/application/assistant/generate-assistant-prompt.js

@@ -1 +1 @@
-import{ASSISTANT_BUILT_IN_FORM_ELEMENTS as e}from"./form/index.js";let t=[["Containers",["group","choice","choices"]],["Options",["option"]],["Inputs",["input"]]];export const formElementSignature=({name:e,role:t,attributes:a})=>{let n=["label",...Object.entries(a).map(([e,t])=>((e,{type:t,enum:a})=>a?`${e}="${a.join("|")}"`:"string"===t?e:`${e}="<${t}>"`)(e,t))];return"option"===t||"input"===t?`<${e} ${n.join(" ")} />`:`<${e} ${n.join(" ")}>…</${e}>`};export const generateAssistantPrompt=a=>{let n,s=Object.values(a.childRecipes??{}).some(e=>e.length>0),i=Object.values(a.replaceRecipes??{}).some(e=>e.length>0),o=Object.values(a.fieldOptions??{}).some(e=>e.length>0),r=Object.keys(a.actions??{}).length>0,h=(a.deletableTypes?.length??0)>0||Object.values(a.writableFields??{}).some(e=>Object.keys(e).length>0)||s||i||r,l=s||i||o,d=[...s?["`childRecipes` is what can be added to it"]:[],...i?["`replaceRecipes` what it can be replaced with"]:[],...o?["`<field>Options` the values a field accepts"]:[]],c=["`selected`",...(a.navigationFields??[]).map(e=>`\`${e}\``)].join(", ");return[["Tools","- Put related operations in one call and select only the fields you need.",...h?[`- Use a mutation inside \`query\` to answer a what-if about a change ${l?"the lists under `meta` do not show":"before making it"}.`,"- Use `apply` only for what the user asked for."]:["- Nothing in this configurator can be changed through you. Say so when the user asks for a change."],"- When an operation fails, its `errors` say why and what to send instead."],...h?[["After a change","- A change is made only when the `apply` result has `committed: true`. Otherwise `errors` or `message` says what happened; without either, nothing changed. Never say a change was made when it was not.","- `sideEffects` names the major or unexpected things the configurator changed along with your change. Tell the user about them when committed; otherwise they describe what would happen. Smaller adjustments are not listed.","- A mutation returns the state after the change: use the ids and values it returned, not the ones you sent.",...r?["- When one mutation does the whole of what the user asked, use it instead of building the same change from several others."]:[]]]:[],...l?[["What can be done to an entity",`- \`meta\` lists what is possible right now: ${d.join(", ")}. Read the list before such a change and before telling the user what is available.`,`- You can select \`dryRun\` on ${[...s||i?["recipes"]:[],...o?["options"]:[]].join(" or ")} to look ahead and pick one that meets what the user asked for. It is optional before a change.`,"- Read the lists again after a change, unless their descriptions say they stay the same."]]:[],["The `[changes]` block","- Every user message starts with it: a glimpse of what the user did in the configurator since your last turn.",`- \`[changes: none]\`, or only ${c} lines: what you know still holds. Do not query it again.`,"- Other lines are a hint at what to read again: `A` added, `D` deleted, `M` modified with the fields. Text after `#` says what caused the line; `reset: undo/redo` says the user undid or redid changes.","- Read computed fields again when a listed change may affect them.","- `[changes: unavailable]` and `[changes: too many (n)]`: read again before relying on anything you read earlier.",...h?["- Your applied changes are not listed, except those waiting for user approval: they appear once approved."]:[]],["Reading the schema","- Take ids from results.",...(a.deletableTypes?.length??0)>0||i?["- The id of a deleted or replaced entity is gone and may later be given to another entity."]:[],`- A field whose name starts with \`_\` is computed: read it${h?", never write it":""}.`,"- Numbers are in the unit the field description names. Convert when the user uses another unit."],["Links","- You may link an entity of the current configuration when you speak about that very entity: `[the left burner](entity:CookingTop-1)`. It helps the user find it in the configurator.",...l?["- Whenever you name recipes or options you read from `meta`, in a list, a table or a sentence, link each one: `[Single gas burner](recipe:KitchenRange-1/top/gas_single)` or `[Oak](option:Cabinet-2/finishOptions/oak)`. The link shows the user its picture or color and its description, so never draw previews yourself. The target is the entity id, the recipe group id or the options field you read, and the key or value, each encoded as in a URL."]:[]],["Replies","- Write Markdown only, never HTML: the panel does not render it.","- An image, `![alt](path)`, is shown only when its path is relative.","- To make a warning or a key note stand out, write a GitHub alert: a blockquote whose first line is only `[!WARNING]` or `[!NOTE]`. Use it sparingly."],["Forms","- To ask the user something they answer by picking or typing, end your reply with a form: a fenced code block with the language `form`. It holds only elements, written as lowercase tags with string attributes, without Markdown or text.","- Every element takes `label`, the text the user sees; on a container or an input it also names the answer. An option or an input whose signature lists `value` takes it: an option sends it instead of its label, and an input starts with it.",'- A bare attribute in a signature takes any text, `name="<type>"` takes a value of that type written as text, such as `min="<number>"`, and `name="a|b"` takes one of the words listed.',"- Options can also stand on their own, outside a `choice` or `choices`.","- An element or attribute not listed here, or an element other than an option inside a `choice` or `choices`, makes the whole form fail.","- The answer is the user's next message. Each container or input gives a line `Label: value`, or the bare value without a label; the lines of a labelled container are indented by two spaces under it, a `choices` lists each picked option on its own line, and `—` marks what was left unanswered.",'- Example: `<choice label="Plinth"><reply label="Black" /><reply label="Oak" /></choice>` is answered with `Plinth: Oak`.',"- The chat shows the answer as the user's own message, so a `value` reads as natural words, never as an id, key or code. Give `value` only when the label alone would be unclear as an answer, and map the answer's labels back to ids yourself.",...(n=[...e,...a.formElements??[]],t.flatMap(([e,t])=>{let a=n.filter(({role:e})=>t.includes(e));return 0===a.length?[]:[`- ${e}:`,...a.map(e=>`  - \`${formElementSignature(e)}\`: ${(({description:e,required:t=[]})=>0===t.length?e:`${e} Requires ${t.map(e=>`\`${e}\``).join(", ")}.`)(e)}`)]}))],...a.lenses?.declarations?[["Built-in fragments","- These fragments are already defined for every operation. Spread one by name on a field of its type, as in `field { ...FragmentName }`, instead of listing its fields.","- A fragment is a starting selection: add the other fields you need next to the spread, by name from the schema.","- Never define a fragment with one of these names.","",a.lenses.declarations]]:[]].map(e=>e.join("\n")).join("\n\n")};
+import{ASSISTANT_BUILT_IN_FORM_ELEMENTS as e}from"./form/index.js";let t=[["Containers",["group","choice","choices"]],["Options",["option"]],["Inputs",["input"]]];export const formElementSignature=({name:e,role:t,attributes:a})=>{let n=["label",...Object.entries(a).map(([e,t])=>((e,{type:t,enum:a})=>a?`${e}="${a.join("|")}"`:"string"===t?e:`${e}="<${t}>"`)(e,t))];return"option"===t||"input"===t?`<${e} ${n.join(" ")} />`:`<${e} ${n.join(" ")}>…</${e}>`};export const generateAssistantPrompt=a=>{let n,s=Object.values(a.childRecipes??{}).some(e=>e.length>0),o=Object.values(a.replaceRecipes??{}).some(e=>e.length>0),i=Object.values(a.fieldOptions??{}).some(e=>e.length>0),r=Object.keys(a.actions??{}).length>0,h=Object.values(a.screenshots??{}).some(e=>e.length>0),l=(a.deletableTypes?.length??0)>0||Object.values(a.writableFields??{}).some(e=>Object.keys(e).length>0)||s||o||r,d=s||o||i,u=[...s?["`childRecipes` is what can be added to it"]:[],...o?["`replaceRecipes` what it can be replaced with"]:[],...i?["`<field>Options` the values a field accepts"]:[]],c=["`selected`",...(a.navigationFields??[]).map(e=>`\`${e}\``)].join(", ");return[["Tools","- Put related operations in one call and select only the fields you need.",...l?[`- Use a mutation inside \`query\` to answer a what-if about a change ${d?"the lists under `meta` do not show":"before making it"}.`,"- Use `apply` only for what the user asked for."]:["- Nothing in this configurator can be changed through you. Say so when the user asks for a change."],"- When an operation fails, its `errors` say why and what to send instead."],...l?[["After a change","- A change is made only when the `apply` result has `committed: true`. Otherwise `errors` or `message` says what happened; without either, nothing changed. Never say a change was made when it was not.","- `sideEffects` names the major or unexpected things the configurator changed along with your change. Tell the user about them when committed; otherwise they describe what would happen. Smaller adjustments are not listed.","- A mutation returns the state after the change: use the ids and values it returned, not the ones you sent.",...r?["- When one mutation does the whole of what the user asked, use it instead of building the same change from several others."]:[]]]:[],...d?[["What can be done to an entity",`- \`meta\` lists what is possible right now: ${u.join(", ")}. Read the list before such a change and before telling the user what is available.`,`- You can select \`dryRun\` on ${[...s||o?["recipes"]:[],...i?["options"]:[]].join(" or ")} to look ahead and pick one that meets what the user asked for. It is optional before a change.`,"- Read the lists again after a change, unless their descriptions say they stay the same."]]:[],...h?[["Seeing the design","- `meta { screenshot(view: …) { image } }` renders an entity from one of the views its `<Type>ScreenshotView` lists. The picture follows the tool result.","- Take one when the user asks how the design looks, or when you are not sure something is laid out right: what sits where, what overlaps or lines up. Otherwise rely on the data.","- Each one costs tokens and blocks the user's configurator while it renders."]]:[],["The `[changes]` block","- Every user message starts with it: a glimpse of what the user did in the configurator since your last turn.",`- \`[changes: none]\`, or only ${c} lines: what you know still holds. Do not query it again.`,"- Other lines are a hint at what to read again: `A` added, `D` deleted, `M` modified with the fields. Text after `#` says what caused the line; `reset: undo/redo` says the user undid or redid changes.","- Read computed fields again when a listed change may affect them.","- `[changes: unavailable]` and `[changes: too many (n)]`: read again before relying on anything you read earlier.",...l?["- Your applied changes are not listed, except those waiting for user approval: they appear once approved."]:[]],["Reading the schema","- Take ids from results.",...(a.deletableTypes?.length??0)>0||o?["- The id of a deleted or replaced entity is gone and may later be given to another entity."]:[],`- A field whose name starts with \`_\` is computed: read it${l?", never write it":""}.`,"- Numbers are in the unit the field description names. Convert when the user uses another unit."],["Links","- You may link an entity of the current configuration when you speak about that very entity: `[the left burner](entity:CookingTop-1)`. It helps the user find it in the configurator.",...d?["- Whenever you name recipes or options you read from `meta`, in a list, a table or a sentence, link each one: `[Single gas burner](recipe:KitchenRange-1/top/gas_single)` or `[Oak](option:Cabinet-2/finishOptions/oak)`. The link shows the user its picture or color and its description, so never draw previews yourself. The target is the entity id, the recipe group id or the options field you read, and the key or value, each encoded as in a URL."]:[]],["Replies","- Write Markdown only, never HTML: the panel does not render it.","- An image, `![alt](path)`, is shown only when its path is relative.","- To make a warning or a key note stand out, write a GitHub alert: a blockquote whose first line is only `[!WARNING]` or `[!NOTE]`. Use it sparingly."],["Forms","- To ask the user something they answer by picking or typing, end your reply with a form: a fenced code block with the language `form`. It holds only elements, written as lowercase tags with string attributes, without Markdown or text.","- Every element takes `label`, the text the user sees; on a container or an input it also names the answer. An option or an input whose signature lists `value` takes it: an option sends it instead of its label, and an input starts with it.",'- A bare attribute in a signature takes any text, `name="<type>"` takes a value of that type written as text, such as `min="<number>"`, and `name="a|b"` takes one of the words listed.',"- Options can also stand on their own, outside a `choice` or `choices`.","- An element or attribute not listed here, or an element other than an option inside a `choice` or `choices`, makes the whole form fail.","- The answer is the user's next message. Each container or input gives a line `Label: value`, or the bare value without a label; the lines of a labelled container are indented by two spaces under it, a `choices` lists each picked option on its own line, and `—` marks what was left unanswered.",'- Example: `<choice label="Plinth"><reply label="Black" /><reply label="Oak" /></choice>` is answered with `Plinth: Oak`.',"- The chat shows the answer as the user's own message, so a `value` reads as natural words, never as an id, key or code. Give `value` only when the label alone would be unclear as an answer, and map the answer's labels back to ids yourself.",...(n=[...e,...a.formElements??[]],t.flatMap(([e,t])=>{let a=n.filter(({role:e})=>t.includes(e));return 0===a.length?[]:[`- ${e}:`,...a.map(e=>`  - \`${formElementSignature(e)}\`: ${(({description:e,required:t=[]})=>0===t.length?e:`${e} Requires ${t.map(e=>`\`${e}\``).join(", ")}.`)(e)}`)]}))],...a.lenses?.declarations?[["Built-in fragments","- These fragments are already defined for every operation. Spread one by name on a field of its type, as in `field { ...FragmentName }`, instead of listing its fields.","- A fragment is a starting selection: add the other fields you need next to the spread, by name from the schema.","- Never define a fragment with one of these names.","",a.lenses.declarations]]:[]].map(e=>e.join("\n")).join("\n\n")};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@salsita-ai/sdk",
-  "version": "0.50.0-develop.3f44338ee8",
+  "version": "0.50.0-develop.4599bc222f",
   "description": "TypeScript SDK and CLI for building, running, and deploying Phoenix configurator applications.",
   "keywords": [
     "salsita",
@@ -308,8 +308,8 @@
     "yaml": "2.9.0",
     "yargs": "17.7.2",
     "zod": "3.25.76",
-    "@salsita-ai/contracts": "0.50.0-develop.3f44338ee8",
-    "@salsita-ai/common": "0.50.0-develop.3f44338ee8"
+    "@salsita-ai/contracts": "0.50.0-develop.4599bc222f",
+    "@salsita-ai/common": "0.50.0-develop.4599bc222f"
   },
   "devDependencies": {
     "@clerk/clerk-react": "5.61.8",