@theoremjs/react 0.3.0 → 0.3.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (258) hide show
  1. package/README.md +728 -202
  2. package/client/composer-hints.js +1 -1
  3. package/client/decision-images.d.ts +76 -0
  4. package/client/decision-images.js +139 -0
  5. package/client/decision-transport.d.ts +26 -4
  6. package/client/decision-transport.js +79 -6
  7. package/client/embedding-input.d.ts +84 -0
  8. package/client/embedding-input.js +194 -0
  9. package/client/embedding-space.d.ts +82 -0
  10. package/client/embedding-space.js +389 -0
  11. package/client/embedding-transport.d.ts +40 -0
  12. package/client/embedding-transport.js +71 -0
  13. package/client/failure.d.ts +7 -10
  14. package/client/failure.js +9 -9
  15. package/client/host-call.d.ts +1 -1
  16. package/client/host-call.js +7 -2
  17. package/client/host-transport.d.ts +2 -0
  18. package/client/host-transport.js +3 -2
  19. package/client/index.d.ts +19 -11
  20. package/client/index.js +14 -10
  21. package/client/ink-waveform.d.ts +1 -1
  22. package/client/ink-waveform.js +1 -2
  23. package/client/live/apply-live-turn-tool-event.js +1 -1
  24. package/client/live/live-captions.d.ts +30 -0
  25. package/client/live/live-captions.js +105 -1
  26. package/client/live/live-mic-forward.d.ts +1 -0
  27. package/client/live/live-mic-forward.js +11 -2
  28. package/client/live/live-page-tool.d.ts +34 -0
  29. package/client/live/live-page-tool.js +11 -0
  30. package/client/live/live-state.d.ts +4 -2
  31. package/client/live/live-state.js +2 -0
  32. package/client/live/live-video.js +1 -1
  33. package/client/live/run-live-tool-call.d.ts +7 -2
  34. package/client/live/run-live-tool-call.js +48 -10
  35. package/client/live/use-live-caption-log.d.ts +17 -0
  36. package/client/live/use-live-caption-log.js +51 -0
  37. package/client/live-client.d.ts +52 -4
  38. package/client/live-client.js +250 -59
  39. package/client/live-messages.d.ts +51 -13
  40. package/client/live-messages.js +42 -9
  41. package/client/page-dispatch.d.ts +15 -0
  42. package/client/page-dispatch.js +63 -0
  43. package/client/run-commit.d.ts +16 -43
  44. package/client/run-commit.js +26 -101
  45. package/client/run-session.d.ts +40 -41
  46. package/client/run-session.js +104 -112
  47. package/client/schema-fields.js +27 -6
  48. package/client/shaped-data.js +37 -8
  49. package/client/source-citations.js +1 -1
  50. package/client/space-geometry.d.ts +39 -0
  51. package/client/space-geometry.js +183 -0
  52. package/client/state-fields.js +1 -1
  53. package/client/tool-result.js +30 -12
  54. package/client/tool-resume.d.ts +18 -1
  55. package/client/tool-resume.js +4 -1
  56. package/client/trace-feed.d.ts +1 -1
  57. package/client/trace-story.d.ts +2 -2
  58. package/client/trace-story.js +43 -19
  59. package/client/trace-view.d.ts +2 -2
  60. package/client/trace-view.js +56 -16
  61. package/client/transcript-groups.d.ts +28 -2
  62. package/client/transcript-groups.js +63 -13
  63. package/client/transcript-schema.d.ts +10 -0
  64. package/client/transcript-schema.js +120 -0
  65. package/client/transport.d.ts +114 -26
  66. package/client/transport.js +106 -32
  67. package/client/turn-client.d.ts +17 -8
  68. package/client/turn-client.js +26 -31
  69. package/client/voice-recorder.d.ts +1 -1
  70. package/client/voice-recorder.js +2 -5
  71. package/client/wire-line.js +5 -5
  72. package/client/worklets/mic-capture.js +9 -9
  73. package/components/InkWaveform.d.ts +12 -1
  74. package/components/InkWaveform.js +65 -42
  75. package/components/live/use-live-runner-controls.d.ts +3 -4
  76. package/components/live/use-live-runner-controls.js +6 -10
  77. package/components/live/use-live-runner-model.d.ts +10 -6
  78. package/components/live/use-live-runner-model.js +29 -14
  79. package/components/live/use-live-runner-ui.d.ts +10 -8
  80. package/components/live/use-live-runner-ui.js +19 -11
  81. package/components/live/use-live-session-client.d.ts +4 -4
  82. package/components/live/use-live-session-client.js +11 -8
  83. package/components/use-composer-voice.d.ts +4 -4
  84. package/components/use-composer-voice.js +8 -8
  85. package/components/use-editor-selection.js +4 -1
  86. package/components/use-voice-playback.d.ts +2 -1
  87. package/components/use-voice-playback.js +4 -4
  88. package/hooks/use-described.d.ts +5 -1
  89. package/hooks/use-described.js +11 -9
  90. package/hooks/use-embedding-index.d.ts +27 -0
  91. package/hooks/use-embedding-index.js +43 -0
  92. package/hooks/use-one-call.d.ts +31 -0
  93. package/hooks/use-one-call.js +76 -0
  94. package/hooks/use-page-tool-warnings.d.ts +6 -0
  95. package/hooks/use-page-tool-warnings.js +25 -0
  96. package/hooks/use-second-ticker.d.ts +2 -0
  97. package/hooks/use-second-ticker.js +13 -0
  98. package/hooks/use-theorem-chat-actions.d.ts +25 -21
  99. package/hooks/use-theorem-chat-actions.js +67 -45
  100. package/hooks/use-theorem-chat-state.d.ts +6 -6
  101. package/hooks/use-theorem-chat-state.js +13 -9
  102. package/hooks/use-theorem-chat.d.ts +48 -14
  103. package/hooks/use-theorem-chat.js +329 -141
  104. package/hooks/use-theorem-decision.d.ts +5 -10
  105. package/hooks/use-theorem-decision.js +4 -67
  106. package/hooks/use-theorem-embedding.d.ts +18 -0
  107. package/hooks/use-theorem-embedding.js +12 -0
  108. package/hooks/use-theorem-host.d.ts +5 -4
  109. package/hooks/use-theorem-host.js +26 -8
  110. package/hooks/use-theorem-interface.d.ts +6 -3
  111. package/hooks/use-theorem-interface.js +12 -22
  112. package/index.d.ts +6 -3
  113. package/index.js +5 -2
  114. package/live.d.ts +1 -0
  115. package/live.js +1 -1
  116. package/package.json +6 -5
  117. package/server/checkpoint-ownership.d.ts +6 -0
  118. package/server/checkpoint-ownership.js +42 -0
  119. package/server/conversation-store.d.ts +92 -0
  120. package/server/conversation-store.js +131 -0
  121. package/server/conversation.d.ts +193 -0
  122. package/server/conversation.js +552 -0
  123. package/server/decision-handler.d.ts +12 -4
  124. package/server/decision-handler.js +29 -38
  125. package/server/embedding-handler.d.ts +33 -0
  126. package/server/embedding-handler.js +45 -0
  127. package/server/handler.d.ts +58 -20
  128. package/server/handler.js +441 -238
  129. package/server/mod.d.ts +15 -6
  130. package/server/mod.js +13 -5
  131. package/server/page-dispatch.d.ts +22 -0
  132. package/server/page-dispatch.js +127 -0
  133. package/server/request-body.d.ts +2 -0
  134. package/server/request-body.js +42 -0
  135. package/server/request-check.d.ts +6 -1
  136. package/server/request-check.js +27 -12
  137. package/server/session-authority.d.ts +8 -0
  138. package/server/session-authority.js +78 -0
  139. package/server/session-boundary.d.ts +26 -0
  140. package/server/session-boundary.js +64 -0
  141. package/server/session-identity.d.ts +16 -0
  142. package/server/session-identity.js +41 -0
  143. package/server/session-limits.d.ts +12 -0
  144. package/server/session-limits.js +59 -0
  145. package/server/session-store.d.ts +69 -20
  146. package/server/session-store.js +220 -5
  147. package/server/single-action.d.ts +21 -0
  148. package/server/single-action.js +30 -0
  149. package/server/steer-inbox.js +1 -1
  150. package/server/tool-operations.d.ts +6 -0
  151. package/server/tool-operations.js +132 -0
  152. package/server/turn-input.d.ts +10 -0
  153. package/server/turn-input.js +29 -0
  154. package/styles/ink-controls.css +78 -78
  155. package/ui/CardWall.d.ts +10 -0
  156. package/ui/CardWall.js +61 -0
  157. package/ui/ChatComposerBar.d.ts +28 -5
  158. package/ui/ChatComposerBar.js +85 -30
  159. package/ui/ChatTranscript.d.ts +14 -1
  160. package/ui/ChatTranscript.js +110 -57
  161. package/ui/ConsoleFrame.d.ts +61 -4
  162. package/ui/ConsoleFrame.js +109 -9
  163. package/ui/DecisionAnswers.d.ts +40 -2
  164. package/ui/DecisionAnswers.js +166 -35
  165. package/ui/DecisionImages.d.ts +16 -0
  166. package/ui/DecisionImages.js +46 -0
  167. package/ui/EmbeddingDocuments.d.ts +78 -0
  168. package/ui/EmbeddingDocuments.js +278 -0
  169. package/ui/EmbeddingQuery.d.ts +27 -0
  170. package/ui/EmbeddingQuery.js +95 -0
  171. package/ui/EmbeddingResults.d.ts +41 -0
  172. package/ui/EmbeddingResults.js +165 -0
  173. package/ui/EmbeddingSpace.d.ts +26 -0
  174. package/ui/EmbeddingSpace.js +106 -0
  175. package/ui/LiveCaptionsPanel.d.ts +22 -0
  176. package/ui/LiveCaptionsPanel.js +27 -0
  177. package/ui/LiveRunner.d.ts +6 -4
  178. package/ui/LiveRunner.js +42 -55
  179. package/ui/PaneState.d.ts +33 -0
  180. package/ui/PaneState.js +50 -0
  181. package/ui/SchemaFields.d.ts +1 -1
  182. package/ui/SchemaFields.js +54 -26
  183. package/ui/ShapedData.js +73 -28
  184. package/ui/SidePanel.d.ts +74 -5
  185. package/ui/SidePanel.js +82 -9
  186. package/ui/StateFields.d.ts +10 -2
  187. package/ui/StateFields.js +56 -13
  188. package/ui/TheoremChat.d.ts +65 -14
  189. package/ui/TheoremChat.js +66 -78
  190. package/ui/TheoremDecision.d.ts +12 -20
  191. package/ui/TheoremDecision.js +78 -35
  192. package/ui/TheoremEmbedding.d.ts +27 -0
  193. package/ui/TheoremEmbedding.js +122 -0
  194. package/ui/TheoremHost.d.ts +22 -5
  195. package/ui/TheoremHost.js +127 -37
  196. package/ui/ToolGateCard.d.ts +5 -1
  197. package/ui/ToolGateCard.js +30 -23
  198. package/ui/ToolResult.js +12 -10
  199. package/ui/ToolResultCharts.js +29 -12
  200. package/ui/TraceCharts.d.ts +2 -2
  201. package/ui/TraceCharts.js +48 -17
  202. package/ui/TraceGuardrails.d.ts +3 -2
  203. package/ui/TraceGuardrails.js +24 -6
  204. package/ui/TraceInspector.d.ts +11 -1
  205. package/ui/TraceInspector.js +45 -28
  206. package/ui/TraceInspectorPanel.d.ts +31 -9
  207. package/ui/TraceInspectorPanel.js +77 -39
  208. package/ui/TraceOverview.d.ts +7 -3
  209. package/ui/TraceOverview.js +44 -14
  210. package/ui/TraceSpanDetail.js +24 -22
  211. package/ui/TraceStory.js +9 -8
  212. package/ui/TraceValues.js +19 -9
  213. package/ui/TraceWaterfall.js +12 -9
  214. package/ui/VoiceNote.js +4 -4
  215. package/ui/approval-heading.d.ts +4 -0
  216. package/ui/approval-heading.js +9 -0
  217. package/ui/arrive.d.ts +18 -0
  218. package/ui/arrive.js +46 -0
  219. package/ui/built/theme.css +86 -68
  220. package/ui/built/theme.css.d.ts +9 -0
  221. package/ui/built/theorem.d.ts +3 -3
  222. package/ui/built/theorem.js +34 -5
  223. package/ui/built/theorem.variants.d.ts +8 -2
  224. package/ui/code-language.d.ts +12 -0
  225. package/ui/code-language.js +50 -0
  226. package/ui/disclosure-motion.js +53 -9
  227. package/ui/embedding-space-gl.d.ts +37 -0
  228. package/ui/embedding-space-gl.js +663 -0
  229. package/ui/embedding-space-view.d.ts +8 -0
  230. package/ui/embedding-space-view.js +8 -0
  231. package/ui/glide.d.ts +7 -0
  232. package/ui/glide.js +44 -0
  233. package/ui/icons.js +2 -1
  234. package/ui/index.d.ts +20 -9
  235. package/ui/index.js +20 -10
  236. package/ui/labels-provider.js +5 -2
  237. package/ui/labels.d.ts +374 -85
  238. package/ui/labels.js +1028 -188
  239. package/ui/markdown-plugins.d.ts +9 -0
  240. package/ui/markdown-plugins.js +13 -0
  241. package/ui/mcp-icon.d.ts +6 -0
  242. package/ui/mcp-icon.js +8 -0
  243. package/ui/row-keys.d.ts +19 -0
  244. package/ui/row-keys.js +33 -0
  245. package/ui/theme.d.ts +1 -1
  246. package/ui/theme.js +2 -2
  247. package/ui/theorem-theme.js +32 -10
  248. package/ui/trace-palette.d.ts +2 -16
  249. package/ui/trace-palette.js +2 -16
  250. package/ui/trace-patterns.d.ts +11 -3
  251. package/ui/trace-patterns.js +16 -3
  252. package/ui/transcript-copy-text.js +1 -0
  253. package/ui/wall-columns.d.ts +8 -0
  254. package/ui/wall-columns.js +19 -0
  255. package/client/walk-away.d.ts +0 -34
  256. package/client/walk-away.js +0 -48
  257. package/server/walk-away.d.ts +0 -22
  258. package/server/walk-away.js +0 -41
@@ -1,3 +1,5 @@
1
+ // why: This is browser code; a Deno program that reaches it through `@theoremjs/react/client` checks it with the DOM.
2
+ /// <reference lib="dom" />
1
3
  /**
2
4
  * Pure client-side SDK for THEOREM Gemini 3.1 Flash Live sessions over WebSocket relay.
3
5
  *
@@ -11,15 +13,16 @@
11
13
  *
12
14
  * @module
13
15
  */
14
- import { describeError, TheoremError, } from '@theoremjs/agents';
16
+ import { describeError, labelText, TheoremError, } from '@theoremjs/agents';
15
17
  import { base64ToBytes, bytesToBase64 } from '@theoremjs/agents/kernel';
16
18
  import { float32Rms, float32RmsToLevel, timeDomainBytesToLevel } from "./audio-level.js";
17
- import { isPermissionDeniedError } from "./live-errors.js";
18
19
  import { applyLiveToolTurnEvent, liveTranscriptFromEvidence, shouldForwardMicFrame, } from "./live/live-mic-forward.js";
19
- import { parseLiveServerEnvelope, } from "./live-messages.js";
20
+ import { isPermissionDeniedError } from "./live-errors.js";
21
+ import { liveClientMessageSchema, parseLiveServerEnvelope, } from "./live-messages.js";
20
22
  import { MIC_CAPTURE_PROCESSOR, micCaptureWorkletUrl } from "./mic-capture.js";
21
23
  import { downsampleAndConvertToInt16, pcm16BytesToFloat32 } from "./pcm-downsample.js";
22
24
  import { hostError } from "./transport.js";
25
+ import { issuedOperationId } from "./turn-client.js";
23
26
  /** One message to the relay, in the shape a relay checks with `parseLiveClientMessage`. */
24
27
  function clientMessage(message) {
25
28
  return JSON.stringify(message);
@@ -38,9 +41,13 @@ function emptyInboundTurnAccum() {
38
41
  * fine-grained "less choppy barge-in" knob.
39
42
  */
40
43
  const BARGE_IN_RMS_WHILE_SPEAKING = 0.05;
44
+ /** How long the client waits before each try at taking a dropped call up again. After the last, the call fails. */
45
+ const LIVE_RECONNECT_DELAYS_MS = [500, 1000, 2000, 4000, 8000];
41
46
  /** A caught value as an `Error`; one that is not keeps its text as `internal` detail. */
42
47
  function asError(err) {
43
- return err instanceof Error ? err : new TheoremError('internal', describeError(err), { cause: err });
48
+ return err instanceof Error
49
+ ? err
50
+ : new TheoremError('internal', describeError(err), { cause: err });
44
51
  }
45
52
  function safeDisconnect(node) {
46
53
  if (!node || typeof node.disconnect !== 'function')
@@ -49,7 +56,7 @@ function safeDisconnect(node) {
49
56
  node.disconnect();
50
57
  }
51
58
  catch {
52
- /* ignore */
59
+ // why: disconnect() throws for a node that is not connected; it is disconnected either way.
53
60
  }
54
61
  }
55
62
  function stopMediaTracks(stream) {
@@ -109,8 +116,23 @@ export class LiveSessionClient {
109
116
  /** Bumped on barge-in / cancel so stale audioChain work is skipped. */
110
117
  audioEpoch = 0;
111
118
  pendingExecuteResults = new Map();
119
+ /** The page's context package, and the JSON of the one the relay last got. */
120
+ context;
121
+ sentContext;
122
+ /** The provider's latest handle for taking this call up again after a drop. */
123
+ history = [];
124
+ providerState;
125
+ /** The session said it ended, or failed: a close after that is the end, not a drop. */
126
+ sessionOver = false;
127
+ /** When the call dropped, for the time away a resume reports. */
128
+ droppedAt;
129
+ /** When the call that could not reconnect dropped: the next call takes it up again. */
130
+ lostAt;
131
+ reconnectTries = 0;
132
+ reconnectTimer = null;
112
133
  constructor(options) {
113
134
  this.options = options;
135
+ this.context = options.context;
114
136
  }
115
137
  setStatus(newStatus) {
116
138
  this.status = newStatus;
@@ -119,6 +141,10 @@ export class LiveSessionClient {
119
141
  }
120
142
  this.options.onStatusChange?.(newStatus);
121
143
  }
144
+ /** Still opening: the first connect, or a try at taking a dropped call up again. */
145
+ get opening() {
146
+ return this.status === 'connecting' || this.status === 'reconnecting';
147
+ }
122
148
  setConnectPhase(phase) {
123
149
  this.options.onConnectPhase?.(phase);
124
150
  }
@@ -130,7 +156,7 @@ export class LiveSessionClient {
130
156
  }
131
157
  detachWebSocket() {
132
158
  for (const [callId, pending] of this.pendingExecuteResults) {
133
- // lexicon-exempt: internal diagnostic; the user reads error.network
159
+ // labels-exempt: internal diagnostic; the user reads error.network
134
160
  pending.reject(new TheoremError('network', `live session closed before call ${callId} settled`));
135
161
  }
136
162
  this.pendingExecuteResults.clear();
@@ -144,10 +170,17 @@ export class LiveSessionClient {
144
170
  this.ws.close();
145
171
  }
146
172
  catch {
147
- /* ignore */
173
+ // why: closing a socket that already closed throws; the connection is over either way.
148
174
  }
149
175
  this.ws = null;
150
176
  }
177
+ clearReconnect() {
178
+ if (this.reconnectTimer)
179
+ clearTimeout(this.reconnectTimer);
180
+ this.reconnectTimer = null;
181
+ this.reconnectTries = 0;
182
+ this.droppedAt = undefined;
183
+ }
151
184
  teardownConnection() {
152
185
  this.clearConnectTimeout();
153
186
  this.micActivating = false;
@@ -159,6 +192,7 @@ export class LiveSessionClient {
159
192
  failSession(error) {
160
193
  if (this.status === 'error' || this.status === 'disconnected')
161
194
  return;
195
+ this.clearReconnect();
162
196
  this.teardownConnection();
163
197
  this.options.onError?.(error);
164
198
  this.setStatus('error');
@@ -166,11 +200,93 @@ export class LiveSessionClient {
166
200
  createSocket(url) {
167
201
  return this.options.createSocket?.() ?? new WebSocket(url);
168
202
  }
169
- // fallow-ignore-next-line unused-class-member -- called from useLiveRunnerControls via client refs
170
- async connect() {
203
+ /**
204
+ * Start a call. After one that dropped and could not reconnect, it takes that call up again,
205
+ * so the agent neither greets twice nor forgets; after a hang-up it is a new call.
206
+ */
207
+ connect() {
208
+ const lostAt = this.lostAt;
209
+ this.clearReconnect();
210
+ this.lostAt = undefined;
211
+ if (lostAt === undefined) {
212
+ this.providerState = undefined;
213
+ this.history = [];
214
+ }
215
+ else
216
+ this.droppedAt = lostAt;
217
+ return this.open('connecting');
218
+ }
219
+ /** A lost call that cannot be taken up again, its handle refused or expired, starts as a new one. */
220
+ startOver() {
221
+ if (this.status !== 'connecting' || !this.providerState)
222
+ return false;
223
+ this.providerState = undefined;
224
+ this.droppedAt = undefined;
225
+ void this.open('connecting');
226
+ return true;
227
+ }
228
+ /** The first message: the call's values, and the handle and time away when it takes a dropped call up again. */
229
+ openMessage() {
230
+ const { slots, openMessage: host } = this.options;
231
+ return {
232
+ type: 'open',
233
+ ...(this.history.length ? { history: structuredClone(this.history) } : {}),
234
+ ...(slots ? { slots } : {}),
235
+ ...(this.context === undefined ? {} : { context: this.context }),
236
+ ...(this.providerState
237
+ ? {
238
+ resume: {
239
+ providerState: this.providerState,
240
+ awayMs: this.droppedAt === undefined ? 0 : Date.now() - this.droppedAt,
241
+ },
242
+ }
243
+ : {}),
244
+ ...(host ? { host } : {}),
245
+ };
246
+ }
247
+ /** The socket closed while the call was open: take it up again if the provider gave a handle, else end. */
248
+ handleDrop() {
249
+ if (!this.providerState || this.sessionOver) {
250
+ this.teardownConnection();
251
+ this.setStatus('disconnected');
252
+ return;
253
+ }
254
+ this.droppedAt ??= Date.now();
171
255
  this.teardownConnection();
172
- this.setStatus('connecting');
173
- this.setConnectPhase('socket');
256
+ this.setStatus('reconnecting');
257
+ this.scheduleReconnect();
258
+ }
259
+ scheduleReconnect() {
260
+ const delay = LIVE_RECONNECT_DELAYS_MS[this.reconnectTries];
261
+ if (delay === undefined) {
262
+ const droppedAt = this.droppedAt;
263
+ // labels-exempt: internal diagnostic; the user reads error.network
264
+ this.failSession(new TheoremError('network', 'live call dropped and could not reconnect'));
265
+ this.lostAt = droppedAt;
266
+ return;
267
+ }
268
+ this.reconnectTries += 1;
269
+ this.reconnectTimer = setTimeout(() => {
270
+ this.reconnectTimer = null;
271
+ void this.open('reconnecting');
272
+ }, delay);
273
+ }
274
+ /** A try at opening failed: a reconnect waits and tries again, a first connect fails the call. */
275
+ failOpen(error) {
276
+ if (this.status !== 'reconnecting') {
277
+ if (!this.startOver())
278
+ this.failSession(error);
279
+ return;
280
+ }
281
+ this.teardownConnection();
282
+ this.scheduleReconnect();
283
+ }
284
+ async open(as) {
285
+ this.teardownConnection();
286
+ this.sessionOver = false;
287
+ this.setStatus(as);
288
+ if (as === 'connecting')
289
+ this.setConnectPhase('socket');
174
290
  try {
175
291
  const AudioContextClass = globalThis.AudioContext;
176
292
  this.audioContext = new AudioContextClass();
@@ -185,37 +301,39 @@ export class LiveSessionClient {
185
301
  const url = this.options.relayUrl || defaultUrl;
186
302
  this.ws = this.createSocket(url);
187
303
  this.ws.binaryType = 'arraybuffer';
188
- const { openMessage } = this.options;
189
- if (openMessage) {
190
- this.ws.onopen = () => {
191
- this.ws?.send(JSON.stringify(openMessage));
192
- };
193
- }
304
+ this.ws.onopen = () => {
305
+ const open = this.openMessage();
306
+ this.sentContext = JSON.stringify(open.context);
307
+ this.ws?.send(JSON.stringify(open));
308
+ };
194
309
  this.connectTimeout = setTimeout(() => {
195
- if (this.status === 'connecting') {
196
- // lexicon-exempt: internal diagnostic; the user reads error.timeout
197
- this.failSession(new TheoremError('timeout', 'live connect timed out'));
310
+ if (this.opening) {
311
+ // labels-exempt: internal diagnostic; the user reads error.timeout
312
+ this.failOpen(new TheoremError('timeout', 'live connect timed out'));
198
313
  }
199
314
  }, 20_000);
200
315
  this.ws.onmessage = (event) => {
201
316
  this.enqueueServerMessage(event.data);
202
317
  };
203
318
  this.ws.onclose = () => {
204
- if (this.status === 'connecting') {
205
- // lexicon-exempt: internal diagnostic; the user reads error.network
206
- this.failSession(new TheoremError('network', 'live socket closed before ready'));
319
+ if (this.opening) {
320
+ // labels-exempt: internal diagnostic; the user reads error.network
321
+ this.failOpen(new TheoremError('network', 'live socket closed before ready'));
207
322
  return;
208
323
  }
209
- this.teardownConnection();
210
- this.setStatus('disconnected');
324
+ this.handleDrop();
211
325
  };
212
326
  this.ws.onerror = () => {
213
- // lexicon-exempt: internal diagnostic; the user reads error.network
214
- this.failSession(new TheoremError('network', 'live socket error'));
327
+ if (!this.opening && this.providerState && !this.sessionOver) {
328
+ this.handleDrop();
329
+ return;
330
+ }
331
+ // labels-exempt: internal diagnostic; the user reads error.network
332
+ this.failOpen(new TheoremError('network', 'live socket error'));
215
333
  };
216
334
  }
217
335
  catch (err) {
218
- this.failSession(new TheoremError('internal', describeError(err), { cause: err }));
336
+ this.failOpen(new TheoremError('internal', describeError(err), { cause: err }));
219
337
  }
220
338
  }
221
339
  async ensureAudioContext() {
@@ -228,10 +346,11 @@ export class LiveSessionClient {
228
346
  }
229
347
  }
230
348
  async activateMicrophone() {
231
- if (this.micActivating || this.status !== 'connecting')
349
+ if (this.micActivating || !this.opening)
232
350
  return;
233
351
  this.micActivating = true;
234
- this.setConnectPhase('microphone');
352
+ if (this.status === 'connecting')
353
+ this.setConnectPhase('microphone');
235
354
  try {
236
355
  await this.ensureAudioContext();
237
356
  this.micStream = await navigator.mediaDevices.getUserMedia({
@@ -243,8 +362,7 @@ export class LiveSessionClient {
243
362
  },
244
363
  });
245
364
  await this.setupMicrophonePipeline();
246
- this.setConnectPhase(null);
247
- this.setStatus('listening');
365
+ this.becomeReady();
248
366
  }
249
367
  catch (err) {
250
368
  const denied = isPermissionDeniedError(err);
@@ -344,7 +462,9 @@ export class LiveSessionClient {
344
462
  const runnableTools = accum.toolCalls.filter((call) => !accum.cancelledToolIds.has(call.id));
345
463
  for (const call of runnableTools) {
346
464
  this.toolChain = this.toolChain
347
- .then(() => this.options.onToolCall(call.name, call.arguments, { callId: call.id,
465
+ .then(() => this.options.onToolCall(call.name, call.arguments, {
466
+ callId: call.id,
467
+ operationId: issuedOperationId(call),
348
468
  }))
349
469
  .catch((err) => {
350
470
  this.options.onError?.(asError(err));
@@ -358,7 +478,7 @@ export class LiveSessionClient {
358
478
  if (typeof data !== 'string')
359
479
  return;
360
480
  const payload = parseLiveServerEnvelope(data);
361
- // An envelope the client can't use reaches onTurnEvent like any event; the call goes on.
481
+ // why: An envelope the client can't use reaches onTurnEvent like any event; the call goes on.
362
482
  if (payload.type === 'unsupported' || payload.type === 'malformed') {
363
483
  this.options.onTurnEvent?.(payload);
364
484
  }
@@ -375,21 +495,34 @@ export class LiveSessionClient {
375
495
  profile: payload.profile,
376
496
  });
377
497
  if (this.options.voiceIngress === false) {
378
- this.setConnectPhase(null);
379
- this.setStatus('listening');
498
+ this.becomeReady();
380
499
  }
381
500
  else {
382
501
  await this.activateMicrophone();
383
502
  }
384
503
  }
504
+ /** The call is open, first or again: the page's context catches up with what changed meanwhile. */
505
+ becomeReady() {
506
+ this.clearReconnect();
507
+ this.setConnectPhase(null);
508
+ this.setStatus('listening');
509
+ this.flushContext();
510
+ }
385
511
  handleExecuteToolResultEnvelope(payload) {
386
- const pending = this.pendingExecuteResults.get(payload.callId);
387
- if (!pending)
512
+ const pending = this.pendingExecuteResults.get(payload.operationId);
513
+ if (!pending || pending.callId !== payload.callId)
388
514
  return;
389
- this.pendingExecuteResults.delete(payload.callId);
515
+ this.pendingExecuteResults.delete(payload.operationId);
390
516
  if (payload.status === 'refused') {
391
517
  pending.reject(hostError(payload.body, 'request'));
392
518
  }
519
+ else if (payload.status === 'dispatched') {
520
+ pending.resolve({
521
+ status: 'dispatched',
522
+ grantId: payload.grantId,
523
+ arguments: payload.arguments,
524
+ });
525
+ }
393
526
  else if (payload.status === 'gated') {
394
527
  pending.resolve({ status: 'gated', gate: payload.gate });
395
528
  }
@@ -410,7 +543,9 @@ export class LiveSessionClient {
410
543
  return true;
411
544
  }
412
545
  if (payload.type === 'error') {
413
- // The relay's kind and wording when it sent them, else `unavailable`.
546
+ if (this.startOver())
547
+ return true;
548
+ this.sessionOver = true;
414
549
  this.options.onError?.(hostError(payload, 'unavailable'));
415
550
  this.setStatus('error');
416
551
  return true;
@@ -425,6 +560,12 @@ export class LiveSessionClient {
425
560
  return accum;
426
561
  }
427
562
  processTurnEvent(event, accum) {
563
+ if (event.type === 'provider_checkpoint' && event.history)
564
+ this.history = structuredClone(event.history);
565
+ if ((event.type === 'provider_checkpoint' || event.type === 'done') && event.providerState)
566
+ this.providerState = event.providerState;
567
+ if (event.type === 'provider_warning')
568
+ this.providerState = undefined;
428
569
  this.options.onTurnEvent?.(event);
429
570
  switch (event.type) {
430
571
  case 'evidence':
@@ -447,7 +588,8 @@ export class LiveSessionClient {
447
588
  handleEvidenceTurnEvent(event) {
448
589
  const transcript = liveTranscriptFromEvidence(event);
449
590
  if (transcript) {
450
- this.options.onTranscript?.(transcript.text, transcript.isUser, { interim: transcript.interim,
591
+ this.options.onTranscript?.(transcript.text, transcript.isUser, {
592
+ interim: transcript.interim,
451
593
  });
452
594
  }
453
595
  }
@@ -467,6 +609,13 @@ export class LiveSessionClient {
467
609
  }
468
610
  if (session.kind !== 'ended')
469
611
  return false;
612
+ // why: The provider ends a call at its time limit. With a handle the call is taken up
613
+ // again, so the person is not told it ended.
614
+ if (this.providerState) {
615
+ this.handleDrop();
616
+ return true;
617
+ }
618
+ this.sessionOver = true;
470
619
  this.options.onSessionEnded?.(session);
471
620
  return true;
472
621
  }
@@ -510,14 +659,32 @@ export class LiveSessionClient {
510
659
  }
511
660
  /** The relay runs `LiveSession.executeTool`; see `ExecuteToolOnRelay`. */
512
661
  executeToolOnRelay(args) {
513
- if (!this.ws || this.ws.readyState !== WebSocket.OPEN) {
514
- throw new TheoremError('request', 'live session is not connected'); // lexicon-exempt: internal diagnostic
515
- }
516
- const resultPromise = new Promise((resolve, reject) => {
517
- this.pendingExecuteResults.set(args.callId, { resolve, reject });
662
+ const socket = this.ws;
663
+ if (!socket || socket.readyState !== WebSocket.OPEN) {
664
+ throw new TheoremError('request', 'live session is not connected'); // labels-exempt: internal diagnostic
665
+ }
666
+ const checked = liveClientMessageSchema.safeParse({ ...args, type: 'executeTool' });
667
+ if (!checked.success || checked.data.type !== 'executeTool') {
668
+ throw new TheoremError('request', labelText('error.request'));
669
+ }
670
+ const request = checked.data;
671
+ if (this.pendingExecuteResults.has(request.operationId)) {
672
+ throw new TheoremError('request', labelText('session.gate_expired'));
673
+ }
674
+ return new Promise((resolve, reject) => {
675
+ this.pendingExecuteResults.set(request.operationId, {
676
+ callId: request.callId,
677
+ resolve,
678
+ reject,
679
+ });
680
+ try {
681
+ socket.send(clientMessage(request));
682
+ }
683
+ catch (error) {
684
+ this.pendingExecuteResults.delete(request.operationId);
685
+ reject(asError(error));
686
+ }
518
687
  });
519
- this.ws.send(clientMessage({ type: 'executeTool', ...args }));
520
- return resultPromise;
521
688
  }
522
689
  async enqueueAudioChunk(base64Data, mimeType) {
523
690
  if (!this.audioContext)
@@ -533,7 +700,7 @@ export class LiveSessionClient {
533
700
  audioBuffer = await this.audioContext.decodeAudioData(wavBuffer);
534
701
  }
535
702
  else {
536
- // Raw PCM 24kHz 1-channel 16-bit LE
703
+ // why: Raw PCM 24kHz 1-channel 16-bit LE
537
704
  const float32 = pcm16BytesToFloat32(bytes);
538
705
  audioBuffer = this.audioContext.createBuffer(1, float32.length, 24000);
539
706
  audioBuffer.getChannelData(0).set(float32);
@@ -572,7 +739,7 @@ export class LiveSessionClient {
572
739
  node.disconnect();
573
740
  }
574
741
  catch {
575
- /* ignore */
742
+ // why: a node that already stopped throws on stop() and disconnect(); it leaves the list either way.
576
743
  }
577
744
  }
578
745
  this.playbackNodes = [];
@@ -581,31 +748,53 @@ export class LiveSessionClient {
581
748
  this.nextPlaybackTime = this.audioContext.currentTime;
582
749
  }
583
750
  }
584
- // fallow-ignore-next-line unused-class-member -- called from useLiveRunnerControls via client refs
585
751
  toggleMute() {
586
752
  this.isMuted = !this.isMuted;
587
753
  return this.isMuted;
588
754
  }
589
- // fallow-ignore-next-line unused-class-member -- called from useLiveRunnerControls via client refs
590
755
  sendText(text) {
591
756
  if (this.ws && this.ws.readyState === WebSocket.OPEN) {
592
757
  this.ws.send(clientMessage({ type: 'text', text }));
593
758
  }
594
759
  }
595
- // fallow-ignore-next-line unused-class-member -- called by hosts that tell the model where the visitor is
596
- sendContext(text) {
597
- if (this.ws && this.ws.readyState === WebSocket.OPEN) {
598
- this.ws.send(clientMessage({ type: 'context', text }));
760
+ /**
761
+ * The call's slot values. An open call is taken up again with them when the provider gave a
762
+ * handle; with none, the next call has them.
763
+ */
764
+ setSlots(slots) {
765
+ if (JSON.stringify(slots) === JSON.stringify(this.options.slots))
766
+ return;
767
+ this.options = { ...this.options, slots };
768
+ if (this.providerState && !this.opening && this.ws?.readyState === WebSocket.OPEN) {
769
+ this.handleDrop();
599
770
  }
600
771
  }
601
- // fallow-ignore-next-line unused-class-member -- called from useLiveRunnerControls via client refs
772
+ /**
773
+ * Replace the page's context package. In a call it reaches the agent as
774
+ * background, with no reply; before one, it goes with the call's opening.
775
+ */
776
+ setContext(context) {
777
+ this.context = context;
778
+ this.flushContext();
779
+ }
780
+ flushContext() {
781
+ if (this.opening || this.ws?.readyState !== WebSocket.OPEN)
782
+ return;
783
+ const next = JSON.stringify(this.context);
784
+ if (next === this.sentContext)
785
+ return;
786
+ this.sentContext = next;
787
+ this.ws.send(clientMessage({ type: 'context', context: this.context }));
788
+ }
602
789
  sendVideo(data, mimeType = 'image/jpeg') {
603
790
  if (this.ws && this.ws.readyState === WebSocket.OPEN) {
604
791
  this.ws.send(clientMessage({ type: 'video', data, mimeType }));
605
792
  }
606
793
  }
607
- // fallow-ignore-next-line unused-class-member -- called from useLiveSessionClient teardown
608
794
  disconnect() {
795
+ this.clearReconnect();
796
+ this.lostAt = undefined;
797
+ this.providerState = undefined;
609
798
  this.teardownConnection();
610
799
  this.setStatus('disconnected');
611
800
  }
@@ -630,9 +819,11 @@ export class LiveSessionClient {
630
819
  void this.audioContext.close();
631
820
  }
632
821
  catch {
633
- /* ignore */
822
+ // why: close() throws on a context that already closed.
634
823
  }
635
824
  this.audioContext = null;
825
+ // why: A worklet module belongs to its context; the next context loads it again.
826
+ this.micWorkletModuleLoaded = false;
636
827
  }
637
828
  }
638
829
  }
@@ -1,3 +1,4 @@
1
+ import { type ProviderCheckpoint } from '@theoremjs/agents';
1
2
  /**
2
3
  * The live wire between the browser and a host's relay: the messages the live
3
4
  * client sends (a relay reads them with `parseLiveClientMessage`) and the envelopes it reads
@@ -5,10 +6,31 @@
5
6
  *
6
7
  * @module
7
8
  */
8
- import { type GateDecision, type ToolGate, type TraceRecord, type TurnEvent, z } from '@theoremjs/agents';
9
+ import { type GateDecision, type ToolGate, type TraceRecord, type TurnEvent, type TurnHistoryMessage, z } from '@theoremjs/agents';
9
10
  import { type HostErrorBody } from './transport.ts';
10
11
  import { type MalformedEvent, type UnsupportedEvent } from './wire-line.ts';
11
- /** What the live client sends its relay, besides the host's own `openMessage`. */
12
+ /**
13
+ * The live client's first message, before the relay opens the session: the
14
+ * values the call starts with. A relay reads it with `parseLiveOpenMessage`.
15
+ */
16
+ export type LiveOpenMessage = {
17
+ type: 'open';
18
+ history?: TurnHistoryMessage[];
19
+ /** The value chosen for each of the profile's `inputs.slots`. */
20
+ slots?: Record<string, string>;
21
+ /** What the page tells the agent as the call opens: the client's context package. */
22
+ context?: unknown;
23
+ /** A call that dropped, taken up again: the provider's handle, and how long the caller was away. */
24
+ resume?: {
25
+ providerState: ProviderCheckpoint;
26
+ awayMs: number;
27
+ };
28
+ /** The host's own `openMessage`, for its relay to read. */
29
+ host?: Record<string, unknown>;
30
+ };
31
+ /** The live client's first message, as `parseLiveOpenMessage` checks it for a relay. */
32
+ export declare const liveOpenMessageSchema: z.ZodType<LiveOpenMessage>;
33
+ /** What the live client sends its relay once the call is open. */
12
34
  export type LiveClientMessage = {
13
35
  type: 'audio';
14
36
  data: string;
@@ -20,20 +42,23 @@ export type LiveClientMessage = {
20
42
  type: 'text';
21
43
  text: string;
22
44
  }
23
- /** Background the model reads without replying: the page the visitor is on, say. */
45
+ /** The page's context package, replacing the last: background the model reads without replying. */
24
46
  | {
25
47
  type: 'context';
26
- text: string;
48
+ context: unknown;
27
49
  }
28
- /** Run a call the model made, by its id; the live session holds its name, input and gate. */
50
+ /** Run a call the model made, by its id; the live session holds its name, input, and gate. */
29
51
  | {
30
52
  type: 'executeTool';
31
53
  callId: string;
54
+ operationId: string;
32
55
  decision?: GateDecision;
33
56
  input?: unknown;
34
57
  secret?: string;
35
- /** What the browser did for a tool that runs there; a relay hands it to the tool as `host.clientOutput`. */
58
+ /** What the browser did for a tool that runs there; a relay hands it to the session as `page.output`. */
36
59
  output?: unknown;
60
+ grantId?: string;
61
+ unanswered?: true;
37
62
  };
38
63
  /** The live client's messages, as `parseLiveClientMessage` checks them for a relay. */
39
64
  export declare const liveClientMessageSchema: z.ZodType<LiveClientMessage>;
@@ -48,7 +73,7 @@ export type LiveServerEnvelope = {
48
73
  type: 'events';
49
74
  events: (TurnEvent | UnsupportedEvent | MalformedEvent)[];
50
75
  }
51
- /** A trace record the session wrote, from a relay that delivers its traces (the playground). */
76
+ /** A trace record the session wrote, from a relay that delivers its traces (the studio). */
52
77
  | {
53
78
  type: 'trace';
54
79
  record: TraceRecord;
@@ -65,15 +90,25 @@ export type LiveServerEnvelope = {
65
90
  | {
66
91
  type: 'executeToolResult';
67
92
  callId: string;
93
+ operationId: string;
68
94
  status: 'settled';
69
95
  } | {
70
96
  type: 'executeToolResult';
71
97
  callId: string;
98
+ operationId: string;
99
+ status: 'dispatched';
100
+ grantId: string;
101
+ arguments: Record<string, unknown>;
102
+ } | {
103
+ type: 'executeToolResult';
104
+ callId: string;
105
+ operationId: string;
72
106
  status: 'gated';
73
107
  gate: ToolGate;
74
108
  } | {
75
109
  type: 'executeToolResult';
76
110
  callId: string;
111
+ operationId: string;
77
112
  status: 'refused';
78
113
  body: HostErrorBody;
79
114
  };
@@ -89,22 +124,25 @@ export type LiveToolStep = {
89
124
  } | {
90
125
  status: 'gated';
91
126
  gate: ToolGate;
127
+ } | {
128
+ status: 'dispatched';
129
+ grantId: string;
130
+ arguments: Record<string, unknown>;
92
131
  };
93
132
  /**
94
133
  * Ask the relay to run `LiveSession.executeTool` for a call the model made.
95
134
  * `decision` answers its gate; `input` is the user's edit to an approval;
96
135
  * `secret` is the key the user typed at a sign-in gate, sent once: the
97
136
  * session makes it the credential for the gate's slot. `output` is the
98
- * browser's result for a tool that runs in the page (see `browserToolHandler`).
137
+ * browser's result for a tool the page answers (`answeredBy: 'page'`).
99
138
  */
100
139
  export type ExecuteToolOnRelay = (args: Omit<Extract<LiveClientMessage, {
101
140
  type: 'executeTool';
102
141
  }>, 'type'>) => Promise<LiveToolStep>;
103
142
  export type LiveSocket = Pick<WebSocket, 'readyState' | 'binaryType' | 'send' | 'close' | 'onopen' | 'onmessage' | 'onclose' | 'onerror'>;
104
- export type LiveConnection = ({
105
- profile: string;
106
- } | {
107
- openMessage: Record<string, unknown>;
108
- }) & {
143
+ /** What a call connects to: a profile the relay knows by id, or the host's own message for its relay. */
144
+ export type LiveConnection = {
145
+ profile?: string;
146
+ openMessage?: Record<string, unknown>;
109
147
  createSocket?: () => LiveSocket;
110
148
  };