@theoremjs/react 0.3.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +728 -202
- package/client/composer-hints.js +1 -1
- package/client/decision-images.d.ts +76 -0
- package/client/decision-images.js +139 -0
- package/client/decision-transport.d.ts +26 -4
- package/client/decision-transport.js +79 -6
- package/client/embedding-input.d.ts +84 -0
- package/client/embedding-input.js +194 -0
- package/client/embedding-space.d.ts +82 -0
- package/client/embedding-space.js +389 -0
- package/client/embedding-transport.d.ts +40 -0
- package/client/embedding-transport.js +71 -0
- package/client/failure.d.ts +7 -10
- package/client/failure.js +9 -9
- package/client/host-call.d.ts +1 -1
- package/client/host-call.js +7 -2
- package/client/host-transport.d.ts +2 -0
- package/client/host-transport.js +3 -2
- package/client/index.d.ts +19 -11
- package/client/index.js +14 -10
- package/client/ink-waveform.d.ts +1 -1
- package/client/ink-waveform.js +1 -2
- package/client/live/apply-live-turn-tool-event.js +1 -1
- package/client/live/live-captions.d.ts +30 -0
- package/client/live/live-captions.js +105 -1
- package/client/live/live-mic-forward.d.ts +1 -0
- package/client/live/live-mic-forward.js +11 -2
- package/client/live/live-page-tool.d.ts +34 -0
- package/client/live/live-page-tool.js +11 -0
- package/client/live/live-state.d.ts +4 -2
- package/client/live/live-state.js +2 -0
- package/client/live/live-video.js +1 -1
- package/client/live/run-live-tool-call.d.ts +7 -2
- package/client/live/run-live-tool-call.js +48 -10
- package/client/live/use-live-caption-log.d.ts +17 -0
- package/client/live/use-live-caption-log.js +51 -0
- package/client/live-client.d.ts +52 -4
- package/client/live-client.js +250 -59
- package/client/live-messages.d.ts +51 -13
- package/client/live-messages.js +42 -9
- package/client/page-dispatch.d.ts +15 -0
- package/client/page-dispatch.js +63 -0
- package/client/run-commit.d.ts +16 -43
- package/client/run-commit.js +26 -101
- package/client/run-session.d.ts +40 -41
- package/client/run-session.js +104 -112
- package/client/schema-fields.js +27 -6
- package/client/shaped-data.js +37 -8
- package/client/source-citations.js +1 -1
- package/client/space-geometry.d.ts +39 -0
- package/client/space-geometry.js +183 -0
- package/client/state-fields.js +1 -1
- package/client/tool-result.js +30 -12
- package/client/tool-resume.d.ts +18 -1
- package/client/tool-resume.js +4 -1
- package/client/trace-feed.d.ts +1 -1
- package/client/trace-story.d.ts +2 -2
- package/client/trace-story.js +43 -19
- package/client/trace-view.d.ts +2 -2
- package/client/trace-view.js +56 -16
- package/client/transcript-groups.d.ts +28 -2
- package/client/transcript-groups.js +63 -13
- package/client/transcript-schema.d.ts +10 -0
- package/client/transcript-schema.js +120 -0
- package/client/transport.d.ts +114 -26
- package/client/transport.js +106 -32
- package/client/turn-client.d.ts +17 -8
- package/client/turn-client.js +26 -31
- package/client/voice-recorder.d.ts +1 -1
- package/client/voice-recorder.js +2 -5
- package/client/wire-line.js +5 -5
- package/client/worklets/mic-capture.js +9 -9
- package/components/InkWaveform.d.ts +12 -1
- package/components/InkWaveform.js +65 -42
- package/components/live/use-live-runner-controls.d.ts +3 -4
- package/components/live/use-live-runner-controls.js +6 -10
- package/components/live/use-live-runner-model.d.ts +10 -6
- package/components/live/use-live-runner-model.js +29 -14
- package/components/live/use-live-runner-ui.d.ts +10 -8
- package/components/live/use-live-runner-ui.js +19 -11
- package/components/live/use-live-session-client.d.ts +4 -4
- package/components/live/use-live-session-client.js +11 -8
- package/components/use-composer-voice.d.ts +4 -4
- package/components/use-composer-voice.js +8 -8
- package/components/use-editor-selection.js +4 -1
- package/components/use-voice-playback.d.ts +2 -1
- package/components/use-voice-playback.js +4 -4
- package/hooks/use-described.d.ts +5 -1
- package/hooks/use-described.js +11 -9
- package/hooks/use-embedding-index.d.ts +27 -0
- package/hooks/use-embedding-index.js +43 -0
- package/hooks/use-one-call.d.ts +31 -0
- package/hooks/use-one-call.js +76 -0
- package/hooks/use-page-tool-warnings.d.ts +6 -0
- package/hooks/use-page-tool-warnings.js +25 -0
- package/hooks/use-second-ticker.d.ts +2 -0
- package/hooks/use-second-ticker.js +13 -0
- package/hooks/use-theorem-chat-actions.d.ts +25 -21
- package/hooks/use-theorem-chat-actions.js +67 -45
- package/hooks/use-theorem-chat-state.d.ts +6 -6
- package/hooks/use-theorem-chat-state.js +13 -9
- package/hooks/use-theorem-chat.d.ts +48 -14
- package/hooks/use-theorem-chat.js +329 -141
- package/hooks/use-theorem-decision.d.ts +5 -10
- package/hooks/use-theorem-decision.js +4 -67
- package/hooks/use-theorem-embedding.d.ts +18 -0
- package/hooks/use-theorem-embedding.js +12 -0
- package/hooks/use-theorem-host.d.ts +5 -4
- package/hooks/use-theorem-host.js +26 -8
- package/hooks/use-theorem-interface.d.ts +6 -3
- package/hooks/use-theorem-interface.js +12 -22
- package/index.d.ts +6 -3
- package/index.js +5 -2
- package/live.d.ts +1 -0
- package/live.js +1 -1
- package/package.json +6 -5
- package/server/checkpoint-ownership.d.ts +6 -0
- package/server/checkpoint-ownership.js +42 -0
- package/server/conversation-store.d.ts +92 -0
- package/server/conversation-store.js +131 -0
- package/server/conversation.d.ts +193 -0
- package/server/conversation.js +552 -0
- package/server/decision-handler.d.ts +12 -4
- package/server/decision-handler.js +29 -38
- package/server/embedding-handler.d.ts +33 -0
- package/server/embedding-handler.js +45 -0
- package/server/handler.d.ts +58 -20
- package/server/handler.js +441 -238
- package/server/mod.d.ts +15 -6
- package/server/mod.js +13 -5
- package/server/page-dispatch.d.ts +22 -0
- package/server/page-dispatch.js +127 -0
- package/server/request-body.d.ts +2 -0
- package/server/request-body.js +42 -0
- package/server/request-check.d.ts +6 -1
- package/server/request-check.js +27 -12
- package/server/session-authority.d.ts +8 -0
- package/server/session-authority.js +78 -0
- package/server/session-boundary.d.ts +26 -0
- package/server/session-boundary.js +64 -0
- package/server/session-identity.d.ts +16 -0
- package/server/session-identity.js +41 -0
- package/server/session-limits.d.ts +12 -0
- package/server/session-limits.js +59 -0
- package/server/session-store.d.ts +69 -20
- package/server/session-store.js +220 -5
- package/server/single-action.d.ts +21 -0
- package/server/single-action.js +30 -0
- package/server/steer-inbox.js +1 -1
- package/server/tool-operations.d.ts +6 -0
- package/server/tool-operations.js +132 -0
- package/server/turn-input.d.ts +10 -0
- package/server/turn-input.js +29 -0
- package/styles/ink-controls.css +78 -78
- package/ui/CardWall.d.ts +10 -0
- package/ui/CardWall.js +61 -0
- package/ui/ChatComposerBar.d.ts +28 -5
- package/ui/ChatComposerBar.js +85 -30
- package/ui/ChatTranscript.d.ts +14 -1
- package/ui/ChatTranscript.js +110 -57
- package/ui/ConsoleFrame.d.ts +61 -4
- package/ui/ConsoleFrame.js +109 -9
- package/ui/DecisionAnswers.d.ts +40 -2
- package/ui/DecisionAnswers.js +166 -35
- package/ui/DecisionImages.d.ts +16 -0
- package/ui/DecisionImages.js +46 -0
- package/ui/EmbeddingDocuments.d.ts +78 -0
- package/ui/EmbeddingDocuments.js +278 -0
- package/ui/EmbeddingQuery.d.ts +27 -0
- package/ui/EmbeddingQuery.js +95 -0
- package/ui/EmbeddingResults.d.ts +41 -0
- package/ui/EmbeddingResults.js +165 -0
- package/ui/EmbeddingSpace.d.ts +26 -0
- package/ui/EmbeddingSpace.js +106 -0
- package/ui/LiveCaptionsPanel.d.ts +22 -0
- package/ui/LiveCaptionsPanel.js +27 -0
- package/ui/LiveRunner.d.ts +6 -4
- package/ui/LiveRunner.js +42 -55
- package/ui/PaneState.d.ts +33 -0
- package/ui/PaneState.js +50 -0
- package/ui/SchemaFields.d.ts +1 -1
- package/ui/SchemaFields.js +54 -26
- package/ui/ShapedData.js +73 -28
- package/ui/SidePanel.d.ts +74 -5
- package/ui/SidePanel.js +82 -9
- package/ui/StateFields.d.ts +10 -2
- package/ui/StateFields.js +56 -13
- package/ui/TheoremChat.d.ts +65 -14
- package/ui/TheoremChat.js +66 -78
- package/ui/TheoremDecision.d.ts +12 -20
- package/ui/TheoremDecision.js +78 -35
- package/ui/TheoremEmbedding.d.ts +27 -0
- package/ui/TheoremEmbedding.js +122 -0
- package/ui/TheoremHost.d.ts +22 -5
- package/ui/TheoremHost.js +127 -37
- package/ui/ToolGateCard.d.ts +5 -1
- package/ui/ToolGateCard.js +30 -23
- package/ui/ToolResult.js +12 -10
- package/ui/ToolResultCharts.js +29 -12
- package/ui/TraceCharts.d.ts +2 -2
- package/ui/TraceCharts.js +48 -17
- package/ui/TraceGuardrails.d.ts +3 -2
- package/ui/TraceGuardrails.js +24 -6
- package/ui/TraceInspector.d.ts +11 -1
- package/ui/TraceInspector.js +45 -28
- package/ui/TraceInspectorPanel.d.ts +31 -9
- package/ui/TraceInspectorPanel.js +77 -39
- package/ui/TraceOverview.d.ts +7 -3
- package/ui/TraceOverview.js +44 -14
- package/ui/TraceSpanDetail.js +24 -22
- package/ui/TraceStory.js +9 -8
- package/ui/TraceValues.js +19 -9
- package/ui/TraceWaterfall.js +12 -9
- package/ui/VoiceNote.js +4 -4
- package/ui/approval-heading.d.ts +4 -0
- package/ui/approval-heading.js +9 -0
- package/ui/arrive.d.ts +18 -0
- package/ui/arrive.js +46 -0
- package/ui/built/theme.css +86 -68
- package/ui/built/theme.css.d.ts +9 -0
- package/ui/built/theorem.d.ts +3 -3
- package/ui/built/theorem.js +34 -5
- package/ui/built/theorem.variants.d.ts +8 -2
- package/ui/code-language.d.ts +12 -0
- package/ui/code-language.js +50 -0
- package/ui/disclosure-motion.js +53 -9
- package/ui/embedding-space-gl.d.ts +37 -0
- package/ui/embedding-space-gl.js +663 -0
- package/ui/embedding-space-view.d.ts +8 -0
- package/ui/embedding-space-view.js +8 -0
- package/ui/glide.d.ts +7 -0
- package/ui/glide.js +44 -0
- package/ui/icons.js +2 -1
- package/ui/index.d.ts +20 -9
- package/ui/index.js +20 -10
- package/ui/labels-provider.js +5 -2
- package/ui/labels.d.ts +374 -85
- package/ui/labels.js +1028 -188
- package/ui/markdown-plugins.d.ts +9 -0
- package/ui/markdown-plugins.js +13 -0
- package/ui/mcp-icon.d.ts +6 -0
- package/ui/mcp-icon.js +8 -0
- package/ui/row-keys.d.ts +19 -0
- package/ui/row-keys.js +33 -0
- package/ui/theme.d.ts +1 -1
- package/ui/theme.js +2 -2
- package/ui/theorem-theme.js +32 -10
- package/ui/trace-palette.d.ts +2 -16
- package/ui/trace-palette.js +2 -16
- package/ui/trace-patterns.d.ts +11 -3
- package/ui/trace-patterns.js +16 -3
- package/ui/transcript-copy-text.js +1 -0
- package/ui/wall-columns.d.ts +8 -0
- package/ui/wall-columns.js +19 -0
- package/client/walk-away.d.ts +0 -34
- package/client/walk-away.js +0 -48
- package/server/walk-away.d.ts +0 -22
- package/server/walk-away.js +0 -41
package/client/live-client.js
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
// why: This is browser code; a Deno program that reaches it through `@theoremjs/react/client` checks it with the DOM.
|
|
2
|
+
/// <reference lib="dom" />
|
|
1
3
|
/**
|
|
2
4
|
* Pure client-side SDK for THEOREM Gemini 3.1 Flash Live sessions over WebSocket relay.
|
|
3
5
|
*
|
|
@@ -11,15 +13,16 @@
|
|
|
11
13
|
*
|
|
12
14
|
* @module
|
|
13
15
|
*/
|
|
14
|
-
import { describeError, TheoremError, } from '@theoremjs/agents';
|
|
16
|
+
import { describeError, labelText, TheoremError, } from '@theoremjs/agents';
|
|
15
17
|
import { base64ToBytes, bytesToBase64 } from '@theoremjs/agents/kernel';
|
|
16
18
|
import { float32Rms, float32RmsToLevel, timeDomainBytesToLevel } from "./audio-level.js";
|
|
17
|
-
import { isPermissionDeniedError } from "./live-errors.js";
|
|
18
19
|
import { applyLiveToolTurnEvent, liveTranscriptFromEvidence, shouldForwardMicFrame, } from "./live/live-mic-forward.js";
|
|
19
|
-
import {
|
|
20
|
+
import { isPermissionDeniedError } from "./live-errors.js";
|
|
21
|
+
import { liveClientMessageSchema, parseLiveServerEnvelope, } from "./live-messages.js";
|
|
20
22
|
import { MIC_CAPTURE_PROCESSOR, micCaptureWorkletUrl } from "./mic-capture.js";
|
|
21
23
|
import { downsampleAndConvertToInt16, pcm16BytesToFloat32 } from "./pcm-downsample.js";
|
|
22
24
|
import { hostError } from "./transport.js";
|
|
25
|
+
import { issuedOperationId } from "./turn-client.js";
|
|
23
26
|
/** One message to the relay, in the shape a relay checks with `parseLiveClientMessage`. */
|
|
24
27
|
function clientMessage(message) {
|
|
25
28
|
return JSON.stringify(message);
|
|
@@ -38,9 +41,13 @@ function emptyInboundTurnAccum() {
|
|
|
38
41
|
* fine-grained "less choppy barge-in" knob.
|
|
39
42
|
*/
|
|
40
43
|
const BARGE_IN_RMS_WHILE_SPEAKING = 0.05;
|
|
44
|
+
/** How long the client waits before each try at taking a dropped call up again. After the last, the call fails. */
|
|
45
|
+
const LIVE_RECONNECT_DELAYS_MS = [500, 1000, 2000, 4000, 8000];
|
|
41
46
|
/** A caught value as an `Error`; one that is not keeps its text as `internal` detail. */
|
|
42
47
|
function asError(err) {
|
|
43
|
-
return err instanceof Error
|
|
48
|
+
return err instanceof Error
|
|
49
|
+
? err
|
|
50
|
+
: new TheoremError('internal', describeError(err), { cause: err });
|
|
44
51
|
}
|
|
45
52
|
function safeDisconnect(node) {
|
|
46
53
|
if (!node || typeof node.disconnect !== 'function')
|
|
@@ -49,7 +56,7 @@ function safeDisconnect(node) {
|
|
|
49
56
|
node.disconnect();
|
|
50
57
|
}
|
|
51
58
|
catch {
|
|
52
|
-
|
|
59
|
+
// why: disconnect() throws for a node that is not connected; it is disconnected either way.
|
|
53
60
|
}
|
|
54
61
|
}
|
|
55
62
|
function stopMediaTracks(stream) {
|
|
@@ -109,8 +116,23 @@ export class LiveSessionClient {
|
|
|
109
116
|
/** Bumped on barge-in / cancel so stale audioChain work is skipped. */
|
|
110
117
|
audioEpoch = 0;
|
|
111
118
|
pendingExecuteResults = new Map();
|
|
119
|
+
/** The page's context package, and the JSON of the one the relay last got. */
|
|
120
|
+
context;
|
|
121
|
+
sentContext;
|
|
122
|
+
/** The provider's latest handle for taking this call up again after a drop. */
|
|
123
|
+
history = [];
|
|
124
|
+
providerState;
|
|
125
|
+
/** The session said it ended, or failed: a close after that is the end, not a drop. */
|
|
126
|
+
sessionOver = false;
|
|
127
|
+
/** When the call dropped, for the time away a resume reports. */
|
|
128
|
+
droppedAt;
|
|
129
|
+
/** When the call that could not reconnect dropped: the next call takes it up again. */
|
|
130
|
+
lostAt;
|
|
131
|
+
reconnectTries = 0;
|
|
132
|
+
reconnectTimer = null;
|
|
112
133
|
constructor(options) {
|
|
113
134
|
this.options = options;
|
|
135
|
+
this.context = options.context;
|
|
114
136
|
}
|
|
115
137
|
setStatus(newStatus) {
|
|
116
138
|
this.status = newStatus;
|
|
@@ -119,6 +141,10 @@ export class LiveSessionClient {
|
|
|
119
141
|
}
|
|
120
142
|
this.options.onStatusChange?.(newStatus);
|
|
121
143
|
}
|
|
144
|
+
/** Still opening: the first connect, or a try at taking a dropped call up again. */
|
|
145
|
+
get opening() {
|
|
146
|
+
return this.status === 'connecting' || this.status === 'reconnecting';
|
|
147
|
+
}
|
|
122
148
|
setConnectPhase(phase) {
|
|
123
149
|
this.options.onConnectPhase?.(phase);
|
|
124
150
|
}
|
|
@@ -130,7 +156,7 @@ export class LiveSessionClient {
|
|
|
130
156
|
}
|
|
131
157
|
detachWebSocket() {
|
|
132
158
|
for (const [callId, pending] of this.pendingExecuteResults) {
|
|
133
|
-
//
|
|
159
|
+
// labels-exempt: internal diagnostic; the user reads error.network
|
|
134
160
|
pending.reject(new TheoremError('network', `live session closed before call ${callId} settled`));
|
|
135
161
|
}
|
|
136
162
|
this.pendingExecuteResults.clear();
|
|
@@ -144,10 +170,17 @@ export class LiveSessionClient {
|
|
|
144
170
|
this.ws.close();
|
|
145
171
|
}
|
|
146
172
|
catch {
|
|
147
|
-
|
|
173
|
+
// why: closing a socket that already closed throws; the connection is over either way.
|
|
148
174
|
}
|
|
149
175
|
this.ws = null;
|
|
150
176
|
}
|
|
177
|
+
clearReconnect() {
|
|
178
|
+
if (this.reconnectTimer)
|
|
179
|
+
clearTimeout(this.reconnectTimer);
|
|
180
|
+
this.reconnectTimer = null;
|
|
181
|
+
this.reconnectTries = 0;
|
|
182
|
+
this.droppedAt = undefined;
|
|
183
|
+
}
|
|
151
184
|
teardownConnection() {
|
|
152
185
|
this.clearConnectTimeout();
|
|
153
186
|
this.micActivating = false;
|
|
@@ -159,6 +192,7 @@ export class LiveSessionClient {
|
|
|
159
192
|
failSession(error) {
|
|
160
193
|
if (this.status === 'error' || this.status === 'disconnected')
|
|
161
194
|
return;
|
|
195
|
+
this.clearReconnect();
|
|
162
196
|
this.teardownConnection();
|
|
163
197
|
this.options.onError?.(error);
|
|
164
198
|
this.setStatus('error');
|
|
@@ -166,11 +200,93 @@ export class LiveSessionClient {
|
|
|
166
200
|
createSocket(url) {
|
|
167
201
|
return this.options.createSocket?.() ?? new WebSocket(url);
|
|
168
202
|
}
|
|
169
|
-
|
|
170
|
-
|
|
203
|
+
/**
|
|
204
|
+
* Start a call. After one that dropped and could not reconnect, it takes that call up again,
|
|
205
|
+
* so the agent neither greets twice nor forgets; after a hang-up it is a new call.
|
|
206
|
+
*/
|
|
207
|
+
connect() {
|
|
208
|
+
const lostAt = this.lostAt;
|
|
209
|
+
this.clearReconnect();
|
|
210
|
+
this.lostAt = undefined;
|
|
211
|
+
if (lostAt === undefined) {
|
|
212
|
+
this.providerState = undefined;
|
|
213
|
+
this.history = [];
|
|
214
|
+
}
|
|
215
|
+
else
|
|
216
|
+
this.droppedAt = lostAt;
|
|
217
|
+
return this.open('connecting');
|
|
218
|
+
}
|
|
219
|
+
/** A lost call that cannot be taken up again, its handle refused or expired, starts as a new one. */
|
|
220
|
+
startOver() {
|
|
221
|
+
if (this.status !== 'connecting' || !this.providerState)
|
|
222
|
+
return false;
|
|
223
|
+
this.providerState = undefined;
|
|
224
|
+
this.droppedAt = undefined;
|
|
225
|
+
void this.open('connecting');
|
|
226
|
+
return true;
|
|
227
|
+
}
|
|
228
|
+
/** The first message: the call's values, and the handle and time away when it takes a dropped call up again. */
|
|
229
|
+
openMessage() {
|
|
230
|
+
const { slots, openMessage: host } = this.options;
|
|
231
|
+
return {
|
|
232
|
+
type: 'open',
|
|
233
|
+
...(this.history.length ? { history: structuredClone(this.history) } : {}),
|
|
234
|
+
...(slots ? { slots } : {}),
|
|
235
|
+
...(this.context === undefined ? {} : { context: this.context }),
|
|
236
|
+
...(this.providerState
|
|
237
|
+
? {
|
|
238
|
+
resume: {
|
|
239
|
+
providerState: this.providerState,
|
|
240
|
+
awayMs: this.droppedAt === undefined ? 0 : Date.now() - this.droppedAt,
|
|
241
|
+
},
|
|
242
|
+
}
|
|
243
|
+
: {}),
|
|
244
|
+
...(host ? { host } : {}),
|
|
245
|
+
};
|
|
246
|
+
}
|
|
247
|
+
/** The socket closed while the call was open: take it up again if the provider gave a handle, else end. */
|
|
248
|
+
handleDrop() {
|
|
249
|
+
if (!this.providerState || this.sessionOver) {
|
|
250
|
+
this.teardownConnection();
|
|
251
|
+
this.setStatus('disconnected');
|
|
252
|
+
return;
|
|
253
|
+
}
|
|
254
|
+
this.droppedAt ??= Date.now();
|
|
171
255
|
this.teardownConnection();
|
|
172
|
-
this.setStatus('
|
|
173
|
-
this.
|
|
256
|
+
this.setStatus('reconnecting');
|
|
257
|
+
this.scheduleReconnect();
|
|
258
|
+
}
|
|
259
|
+
scheduleReconnect() {
|
|
260
|
+
const delay = LIVE_RECONNECT_DELAYS_MS[this.reconnectTries];
|
|
261
|
+
if (delay === undefined) {
|
|
262
|
+
const droppedAt = this.droppedAt;
|
|
263
|
+
// labels-exempt: internal diagnostic; the user reads error.network
|
|
264
|
+
this.failSession(new TheoremError('network', 'live call dropped and could not reconnect'));
|
|
265
|
+
this.lostAt = droppedAt;
|
|
266
|
+
return;
|
|
267
|
+
}
|
|
268
|
+
this.reconnectTries += 1;
|
|
269
|
+
this.reconnectTimer = setTimeout(() => {
|
|
270
|
+
this.reconnectTimer = null;
|
|
271
|
+
void this.open('reconnecting');
|
|
272
|
+
}, delay);
|
|
273
|
+
}
|
|
274
|
+
/** A try at opening failed: a reconnect waits and tries again, a first connect fails the call. */
|
|
275
|
+
failOpen(error) {
|
|
276
|
+
if (this.status !== 'reconnecting') {
|
|
277
|
+
if (!this.startOver())
|
|
278
|
+
this.failSession(error);
|
|
279
|
+
return;
|
|
280
|
+
}
|
|
281
|
+
this.teardownConnection();
|
|
282
|
+
this.scheduleReconnect();
|
|
283
|
+
}
|
|
284
|
+
async open(as) {
|
|
285
|
+
this.teardownConnection();
|
|
286
|
+
this.sessionOver = false;
|
|
287
|
+
this.setStatus(as);
|
|
288
|
+
if (as === 'connecting')
|
|
289
|
+
this.setConnectPhase('socket');
|
|
174
290
|
try {
|
|
175
291
|
const AudioContextClass = globalThis.AudioContext;
|
|
176
292
|
this.audioContext = new AudioContextClass();
|
|
@@ -185,37 +301,39 @@ export class LiveSessionClient {
|
|
|
185
301
|
const url = this.options.relayUrl || defaultUrl;
|
|
186
302
|
this.ws = this.createSocket(url);
|
|
187
303
|
this.ws.binaryType = 'arraybuffer';
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
this.
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
}
|
|
304
|
+
this.ws.onopen = () => {
|
|
305
|
+
const open = this.openMessage();
|
|
306
|
+
this.sentContext = JSON.stringify(open.context);
|
|
307
|
+
this.ws?.send(JSON.stringify(open));
|
|
308
|
+
};
|
|
194
309
|
this.connectTimeout = setTimeout(() => {
|
|
195
|
-
if (this.
|
|
196
|
-
//
|
|
197
|
-
this.
|
|
310
|
+
if (this.opening) {
|
|
311
|
+
// labels-exempt: internal diagnostic; the user reads error.timeout
|
|
312
|
+
this.failOpen(new TheoremError('timeout', 'live connect timed out'));
|
|
198
313
|
}
|
|
199
314
|
}, 20_000);
|
|
200
315
|
this.ws.onmessage = (event) => {
|
|
201
316
|
this.enqueueServerMessage(event.data);
|
|
202
317
|
};
|
|
203
318
|
this.ws.onclose = () => {
|
|
204
|
-
if (this.
|
|
205
|
-
//
|
|
206
|
-
this.
|
|
319
|
+
if (this.opening) {
|
|
320
|
+
// labels-exempt: internal diagnostic; the user reads error.network
|
|
321
|
+
this.failOpen(new TheoremError('network', 'live socket closed before ready'));
|
|
207
322
|
return;
|
|
208
323
|
}
|
|
209
|
-
this.
|
|
210
|
-
this.setStatus('disconnected');
|
|
324
|
+
this.handleDrop();
|
|
211
325
|
};
|
|
212
326
|
this.ws.onerror = () => {
|
|
213
|
-
|
|
214
|
-
|
|
327
|
+
if (!this.opening && this.providerState && !this.sessionOver) {
|
|
328
|
+
this.handleDrop();
|
|
329
|
+
return;
|
|
330
|
+
}
|
|
331
|
+
// labels-exempt: internal diagnostic; the user reads error.network
|
|
332
|
+
this.failOpen(new TheoremError('network', 'live socket error'));
|
|
215
333
|
};
|
|
216
334
|
}
|
|
217
335
|
catch (err) {
|
|
218
|
-
this.
|
|
336
|
+
this.failOpen(new TheoremError('internal', describeError(err), { cause: err }));
|
|
219
337
|
}
|
|
220
338
|
}
|
|
221
339
|
async ensureAudioContext() {
|
|
@@ -228,10 +346,11 @@ export class LiveSessionClient {
|
|
|
228
346
|
}
|
|
229
347
|
}
|
|
230
348
|
async activateMicrophone() {
|
|
231
|
-
if (this.micActivating || this.
|
|
349
|
+
if (this.micActivating || !this.opening)
|
|
232
350
|
return;
|
|
233
351
|
this.micActivating = true;
|
|
234
|
-
this.
|
|
352
|
+
if (this.status === 'connecting')
|
|
353
|
+
this.setConnectPhase('microphone');
|
|
235
354
|
try {
|
|
236
355
|
await this.ensureAudioContext();
|
|
237
356
|
this.micStream = await navigator.mediaDevices.getUserMedia({
|
|
@@ -243,8 +362,7 @@ export class LiveSessionClient {
|
|
|
243
362
|
},
|
|
244
363
|
});
|
|
245
364
|
await this.setupMicrophonePipeline();
|
|
246
|
-
this.
|
|
247
|
-
this.setStatus('listening');
|
|
365
|
+
this.becomeReady();
|
|
248
366
|
}
|
|
249
367
|
catch (err) {
|
|
250
368
|
const denied = isPermissionDeniedError(err);
|
|
@@ -344,7 +462,9 @@ export class LiveSessionClient {
|
|
|
344
462
|
const runnableTools = accum.toolCalls.filter((call) => !accum.cancelledToolIds.has(call.id));
|
|
345
463
|
for (const call of runnableTools) {
|
|
346
464
|
this.toolChain = this.toolChain
|
|
347
|
-
.then(() => this.options.onToolCall(call.name, call.arguments, {
|
|
465
|
+
.then(() => this.options.onToolCall(call.name, call.arguments, {
|
|
466
|
+
callId: call.id,
|
|
467
|
+
operationId: issuedOperationId(call),
|
|
348
468
|
}))
|
|
349
469
|
.catch((err) => {
|
|
350
470
|
this.options.onError?.(asError(err));
|
|
@@ -358,7 +478,7 @@ export class LiveSessionClient {
|
|
|
358
478
|
if (typeof data !== 'string')
|
|
359
479
|
return;
|
|
360
480
|
const payload = parseLiveServerEnvelope(data);
|
|
361
|
-
// An envelope the client can't use reaches onTurnEvent like any event; the call goes on.
|
|
481
|
+
// why: An envelope the client can't use reaches onTurnEvent like any event; the call goes on.
|
|
362
482
|
if (payload.type === 'unsupported' || payload.type === 'malformed') {
|
|
363
483
|
this.options.onTurnEvent?.(payload);
|
|
364
484
|
}
|
|
@@ -375,21 +495,34 @@ export class LiveSessionClient {
|
|
|
375
495
|
profile: payload.profile,
|
|
376
496
|
});
|
|
377
497
|
if (this.options.voiceIngress === false) {
|
|
378
|
-
this.
|
|
379
|
-
this.setStatus('listening');
|
|
498
|
+
this.becomeReady();
|
|
380
499
|
}
|
|
381
500
|
else {
|
|
382
501
|
await this.activateMicrophone();
|
|
383
502
|
}
|
|
384
503
|
}
|
|
504
|
+
/** The call is open, first or again: the page's context catches up with what changed meanwhile. */
|
|
505
|
+
becomeReady() {
|
|
506
|
+
this.clearReconnect();
|
|
507
|
+
this.setConnectPhase(null);
|
|
508
|
+
this.setStatus('listening');
|
|
509
|
+
this.flushContext();
|
|
510
|
+
}
|
|
385
511
|
handleExecuteToolResultEnvelope(payload) {
|
|
386
|
-
const pending = this.pendingExecuteResults.get(payload.
|
|
387
|
-
if (!pending)
|
|
512
|
+
const pending = this.pendingExecuteResults.get(payload.operationId);
|
|
513
|
+
if (!pending || pending.callId !== payload.callId)
|
|
388
514
|
return;
|
|
389
|
-
this.pendingExecuteResults.delete(payload.
|
|
515
|
+
this.pendingExecuteResults.delete(payload.operationId);
|
|
390
516
|
if (payload.status === 'refused') {
|
|
391
517
|
pending.reject(hostError(payload.body, 'request'));
|
|
392
518
|
}
|
|
519
|
+
else if (payload.status === 'dispatched') {
|
|
520
|
+
pending.resolve({
|
|
521
|
+
status: 'dispatched',
|
|
522
|
+
grantId: payload.grantId,
|
|
523
|
+
arguments: payload.arguments,
|
|
524
|
+
});
|
|
525
|
+
}
|
|
393
526
|
else if (payload.status === 'gated') {
|
|
394
527
|
pending.resolve({ status: 'gated', gate: payload.gate });
|
|
395
528
|
}
|
|
@@ -410,7 +543,9 @@ export class LiveSessionClient {
|
|
|
410
543
|
return true;
|
|
411
544
|
}
|
|
412
545
|
if (payload.type === 'error') {
|
|
413
|
-
|
|
546
|
+
if (this.startOver())
|
|
547
|
+
return true;
|
|
548
|
+
this.sessionOver = true;
|
|
414
549
|
this.options.onError?.(hostError(payload, 'unavailable'));
|
|
415
550
|
this.setStatus('error');
|
|
416
551
|
return true;
|
|
@@ -425,6 +560,12 @@ export class LiveSessionClient {
|
|
|
425
560
|
return accum;
|
|
426
561
|
}
|
|
427
562
|
processTurnEvent(event, accum) {
|
|
563
|
+
if (event.type === 'provider_checkpoint' && event.history)
|
|
564
|
+
this.history = structuredClone(event.history);
|
|
565
|
+
if ((event.type === 'provider_checkpoint' || event.type === 'done') && event.providerState)
|
|
566
|
+
this.providerState = event.providerState;
|
|
567
|
+
if (event.type === 'provider_warning')
|
|
568
|
+
this.providerState = undefined;
|
|
428
569
|
this.options.onTurnEvent?.(event);
|
|
429
570
|
switch (event.type) {
|
|
430
571
|
case 'evidence':
|
|
@@ -447,7 +588,8 @@ export class LiveSessionClient {
|
|
|
447
588
|
handleEvidenceTurnEvent(event) {
|
|
448
589
|
const transcript = liveTranscriptFromEvidence(event);
|
|
449
590
|
if (transcript) {
|
|
450
|
-
this.options.onTranscript?.(transcript.text, transcript.isUser, {
|
|
591
|
+
this.options.onTranscript?.(transcript.text, transcript.isUser, {
|
|
592
|
+
interim: transcript.interim,
|
|
451
593
|
});
|
|
452
594
|
}
|
|
453
595
|
}
|
|
@@ -467,6 +609,13 @@ export class LiveSessionClient {
|
|
|
467
609
|
}
|
|
468
610
|
if (session.kind !== 'ended')
|
|
469
611
|
return false;
|
|
612
|
+
// why: The provider ends a call at its time limit. With a handle the call is taken up
|
|
613
|
+
// again, so the person is not told it ended.
|
|
614
|
+
if (this.providerState) {
|
|
615
|
+
this.handleDrop();
|
|
616
|
+
return true;
|
|
617
|
+
}
|
|
618
|
+
this.sessionOver = true;
|
|
470
619
|
this.options.onSessionEnded?.(session);
|
|
471
620
|
return true;
|
|
472
621
|
}
|
|
@@ -510,14 +659,32 @@ export class LiveSessionClient {
|
|
|
510
659
|
}
|
|
511
660
|
/** The relay runs `LiveSession.executeTool`; see `ExecuteToolOnRelay`. */
|
|
512
661
|
executeToolOnRelay(args) {
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
662
|
+
const socket = this.ws;
|
|
663
|
+
if (!socket || socket.readyState !== WebSocket.OPEN) {
|
|
664
|
+
throw new TheoremError('request', 'live session is not connected'); // labels-exempt: internal diagnostic
|
|
665
|
+
}
|
|
666
|
+
const checked = liveClientMessageSchema.safeParse({ ...args, type: 'executeTool' });
|
|
667
|
+
if (!checked.success || checked.data.type !== 'executeTool') {
|
|
668
|
+
throw new TheoremError('request', labelText('error.request'));
|
|
669
|
+
}
|
|
670
|
+
const request = checked.data;
|
|
671
|
+
if (this.pendingExecuteResults.has(request.operationId)) {
|
|
672
|
+
throw new TheoremError('request', labelText('session.gate_expired'));
|
|
673
|
+
}
|
|
674
|
+
return new Promise((resolve, reject) => {
|
|
675
|
+
this.pendingExecuteResults.set(request.operationId, {
|
|
676
|
+
callId: request.callId,
|
|
677
|
+
resolve,
|
|
678
|
+
reject,
|
|
679
|
+
});
|
|
680
|
+
try {
|
|
681
|
+
socket.send(clientMessage(request));
|
|
682
|
+
}
|
|
683
|
+
catch (error) {
|
|
684
|
+
this.pendingExecuteResults.delete(request.operationId);
|
|
685
|
+
reject(asError(error));
|
|
686
|
+
}
|
|
518
687
|
});
|
|
519
|
-
this.ws.send(clientMessage({ type: 'executeTool', ...args }));
|
|
520
|
-
return resultPromise;
|
|
521
688
|
}
|
|
522
689
|
async enqueueAudioChunk(base64Data, mimeType) {
|
|
523
690
|
if (!this.audioContext)
|
|
@@ -533,7 +700,7 @@ export class LiveSessionClient {
|
|
|
533
700
|
audioBuffer = await this.audioContext.decodeAudioData(wavBuffer);
|
|
534
701
|
}
|
|
535
702
|
else {
|
|
536
|
-
// Raw PCM 24kHz 1-channel 16-bit LE
|
|
703
|
+
// why: Raw PCM 24kHz 1-channel 16-bit LE
|
|
537
704
|
const float32 = pcm16BytesToFloat32(bytes);
|
|
538
705
|
audioBuffer = this.audioContext.createBuffer(1, float32.length, 24000);
|
|
539
706
|
audioBuffer.getChannelData(0).set(float32);
|
|
@@ -572,7 +739,7 @@ export class LiveSessionClient {
|
|
|
572
739
|
node.disconnect();
|
|
573
740
|
}
|
|
574
741
|
catch {
|
|
575
|
-
|
|
742
|
+
// why: a node that already stopped throws on stop() and disconnect(); it leaves the list either way.
|
|
576
743
|
}
|
|
577
744
|
}
|
|
578
745
|
this.playbackNodes = [];
|
|
@@ -581,31 +748,53 @@ export class LiveSessionClient {
|
|
|
581
748
|
this.nextPlaybackTime = this.audioContext.currentTime;
|
|
582
749
|
}
|
|
583
750
|
}
|
|
584
|
-
// fallow-ignore-next-line unused-class-member -- called from useLiveRunnerControls via client refs
|
|
585
751
|
toggleMute() {
|
|
586
752
|
this.isMuted = !this.isMuted;
|
|
587
753
|
return this.isMuted;
|
|
588
754
|
}
|
|
589
|
-
// fallow-ignore-next-line unused-class-member -- called from useLiveRunnerControls via client refs
|
|
590
755
|
sendText(text) {
|
|
591
756
|
if (this.ws && this.ws.readyState === WebSocket.OPEN) {
|
|
592
757
|
this.ws.send(clientMessage({ type: 'text', text }));
|
|
593
758
|
}
|
|
594
759
|
}
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
760
|
+
/**
|
|
761
|
+
* The call's slot values. An open call is taken up again with them when the provider gave a
|
|
762
|
+
* handle; with none, the next call has them.
|
|
763
|
+
*/
|
|
764
|
+
setSlots(slots) {
|
|
765
|
+
if (JSON.stringify(slots) === JSON.stringify(this.options.slots))
|
|
766
|
+
return;
|
|
767
|
+
this.options = { ...this.options, slots };
|
|
768
|
+
if (this.providerState && !this.opening && this.ws?.readyState === WebSocket.OPEN) {
|
|
769
|
+
this.handleDrop();
|
|
599
770
|
}
|
|
600
771
|
}
|
|
601
|
-
|
|
772
|
+
/**
|
|
773
|
+
* Replace the page's context package. In a call it reaches the agent as
|
|
774
|
+
* background, with no reply; before one, it goes with the call's opening.
|
|
775
|
+
*/
|
|
776
|
+
setContext(context) {
|
|
777
|
+
this.context = context;
|
|
778
|
+
this.flushContext();
|
|
779
|
+
}
|
|
780
|
+
flushContext() {
|
|
781
|
+
if (this.opening || this.ws?.readyState !== WebSocket.OPEN)
|
|
782
|
+
return;
|
|
783
|
+
const next = JSON.stringify(this.context);
|
|
784
|
+
if (next === this.sentContext)
|
|
785
|
+
return;
|
|
786
|
+
this.sentContext = next;
|
|
787
|
+
this.ws.send(clientMessage({ type: 'context', context: this.context }));
|
|
788
|
+
}
|
|
602
789
|
sendVideo(data, mimeType = 'image/jpeg') {
|
|
603
790
|
if (this.ws && this.ws.readyState === WebSocket.OPEN) {
|
|
604
791
|
this.ws.send(clientMessage({ type: 'video', data, mimeType }));
|
|
605
792
|
}
|
|
606
793
|
}
|
|
607
|
-
// fallow-ignore-next-line unused-class-member -- called from useLiveSessionClient teardown
|
|
608
794
|
disconnect() {
|
|
795
|
+
this.clearReconnect();
|
|
796
|
+
this.lostAt = undefined;
|
|
797
|
+
this.providerState = undefined;
|
|
609
798
|
this.teardownConnection();
|
|
610
799
|
this.setStatus('disconnected');
|
|
611
800
|
}
|
|
@@ -630,9 +819,11 @@ export class LiveSessionClient {
|
|
|
630
819
|
void this.audioContext.close();
|
|
631
820
|
}
|
|
632
821
|
catch {
|
|
633
|
-
|
|
822
|
+
// why: close() throws on a context that already closed.
|
|
634
823
|
}
|
|
635
824
|
this.audioContext = null;
|
|
825
|
+
// why: A worklet module belongs to its context; the next context loads it again.
|
|
826
|
+
this.micWorkletModuleLoaded = false;
|
|
636
827
|
}
|
|
637
828
|
}
|
|
638
829
|
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { type ProviderCheckpoint } from '@theoremjs/agents';
|
|
1
2
|
/**
|
|
2
3
|
* The live wire between the browser and a host's relay: the messages the live
|
|
3
4
|
* client sends (a relay reads them with `parseLiveClientMessage`) and the envelopes it reads
|
|
@@ -5,10 +6,31 @@
|
|
|
5
6
|
*
|
|
6
7
|
* @module
|
|
7
8
|
*/
|
|
8
|
-
import { type GateDecision, type ToolGate, type TraceRecord, type TurnEvent, z } from '@theoremjs/agents';
|
|
9
|
+
import { type GateDecision, type ToolGate, type TraceRecord, type TurnEvent, type TurnHistoryMessage, z } from '@theoremjs/agents';
|
|
9
10
|
import { type HostErrorBody } from './transport.ts';
|
|
10
11
|
import { type MalformedEvent, type UnsupportedEvent } from './wire-line.ts';
|
|
11
|
-
/**
|
|
12
|
+
/**
|
|
13
|
+
* The live client's first message, before the relay opens the session: the
|
|
14
|
+
* values the call starts with. A relay reads it with `parseLiveOpenMessage`.
|
|
15
|
+
*/
|
|
16
|
+
export type LiveOpenMessage = {
|
|
17
|
+
type: 'open';
|
|
18
|
+
history?: TurnHistoryMessage[];
|
|
19
|
+
/** The value chosen for each of the profile's `inputs.slots`. */
|
|
20
|
+
slots?: Record<string, string>;
|
|
21
|
+
/** What the page tells the agent as the call opens: the client's context package. */
|
|
22
|
+
context?: unknown;
|
|
23
|
+
/** A call that dropped, taken up again: the provider's handle, and how long the caller was away. */
|
|
24
|
+
resume?: {
|
|
25
|
+
providerState: ProviderCheckpoint;
|
|
26
|
+
awayMs: number;
|
|
27
|
+
};
|
|
28
|
+
/** The host's own `openMessage`, for its relay to read. */
|
|
29
|
+
host?: Record<string, unknown>;
|
|
30
|
+
};
|
|
31
|
+
/** The live client's first message, as `parseLiveOpenMessage` checks it for a relay. */
|
|
32
|
+
export declare const liveOpenMessageSchema: z.ZodType<LiveOpenMessage>;
|
|
33
|
+
/** What the live client sends its relay once the call is open. */
|
|
12
34
|
export type LiveClientMessage = {
|
|
13
35
|
type: 'audio';
|
|
14
36
|
data: string;
|
|
@@ -20,20 +42,23 @@ export type LiveClientMessage = {
|
|
|
20
42
|
type: 'text';
|
|
21
43
|
text: string;
|
|
22
44
|
}
|
|
23
|
-
/**
|
|
45
|
+
/** The page's context package, replacing the last: background the model reads without replying. */
|
|
24
46
|
| {
|
|
25
47
|
type: 'context';
|
|
26
|
-
|
|
48
|
+
context: unknown;
|
|
27
49
|
}
|
|
28
|
-
/** Run a call the model made, by its id; the live session holds its name, input and gate. */
|
|
50
|
+
/** Run a call the model made, by its id; the live session holds its name, input, and gate. */
|
|
29
51
|
| {
|
|
30
52
|
type: 'executeTool';
|
|
31
53
|
callId: string;
|
|
54
|
+
operationId: string;
|
|
32
55
|
decision?: GateDecision;
|
|
33
56
|
input?: unknown;
|
|
34
57
|
secret?: string;
|
|
35
|
-
/** What the browser did for a tool that runs there; a relay hands it to the
|
|
58
|
+
/** What the browser did for a tool that runs there; a relay hands it to the session as `page.output`. */
|
|
36
59
|
output?: unknown;
|
|
60
|
+
grantId?: string;
|
|
61
|
+
unanswered?: true;
|
|
37
62
|
};
|
|
38
63
|
/** The live client's messages, as `parseLiveClientMessage` checks them for a relay. */
|
|
39
64
|
export declare const liveClientMessageSchema: z.ZodType<LiveClientMessage>;
|
|
@@ -48,7 +73,7 @@ export type LiveServerEnvelope = {
|
|
|
48
73
|
type: 'events';
|
|
49
74
|
events: (TurnEvent | UnsupportedEvent | MalformedEvent)[];
|
|
50
75
|
}
|
|
51
|
-
/** A trace record the session wrote, from a relay that delivers its traces (the
|
|
76
|
+
/** A trace record the session wrote, from a relay that delivers its traces (the studio). */
|
|
52
77
|
| {
|
|
53
78
|
type: 'trace';
|
|
54
79
|
record: TraceRecord;
|
|
@@ -65,15 +90,25 @@ export type LiveServerEnvelope = {
|
|
|
65
90
|
| {
|
|
66
91
|
type: 'executeToolResult';
|
|
67
92
|
callId: string;
|
|
93
|
+
operationId: string;
|
|
68
94
|
status: 'settled';
|
|
69
95
|
} | {
|
|
70
96
|
type: 'executeToolResult';
|
|
71
97
|
callId: string;
|
|
98
|
+
operationId: string;
|
|
99
|
+
status: 'dispatched';
|
|
100
|
+
grantId: string;
|
|
101
|
+
arguments: Record<string, unknown>;
|
|
102
|
+
} | {
|
|
103
|
+
type: 'executeToolResult';
|
|
104
|
+
callId: string;
|
|
105
|
+
operationId: string;
|
|
72
106
|
status: 'gated';
|
|
73
107
|
gate: ToolGate;
|
|
74
108
|
} | {
|
|
75
109
|
type: 'executeToolResult';
|
|
76
110
|
callId: string;
|
|
111
|
+
operationId: string;
|
|
77
112
|
status: 'refused';
|
|
78
113
|
body: HostErrorBody;
|
|
79
114
|
};
|
|
@@ -89,22 +124,25 @@ export type LiveToolStep = {
|
|
|
89
124
|
} | {
|
|
90
125
|
status: 'gated';
|
|
91
126
|
gate: ToolGate;
|
|
127
|
+
} | {
|
|
128
|
+
status: 'dispatched';
|
|
129
|
+
grantId: string;
|
|
130
|
+
arguments: Record<string, unknown>;
|
|
92
131
|
};
|
|
93
132
|
/**
|
|
94
133
|
* Ask the relay to run `LiveSession.executeTool` for a call the model made.
|
|
95
134
|
* `decision` answers its gate; `input` is the user's edit to an approval;
|
|
96
135
|
* `secret` is the key the user typed at a sign-in gate, sent once: the
|
|
97
136
|
* session makes it the credential for the gate's slot. `output` is the
|
|
98
|
-
* browser's result for a tool
|
|
137
|
+
* browser's result for a tool the page answers (`answeredBy: 'page'`).
|
|
99
138
|
*/
|
|
100
139
|
export type ExecuteToolOnRelay = (args: Omit<Extract<LiveClientMessage, {
|
|
101
140
|
type: 'executeTool';
|
|
102
141
|
}>, 'type'>) => Promise<LiveToolStep>;
|
|
103
142
|
export type LiveSocket = Pick<WebSocket, 'readyState' | 'binaryType' | 'send' | 'close' | 'onopen' | 'onmessage' | 'onclose' | 'onerror'>;
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
openMessage
|
|
108
|
-
}) & {
|
|
143
|
+
/** What a call connects to: a profile the relay knows by id, or the host's own message for its relay. */
|
|
144
|
+
export type LiveConnection = {
|
|
145
|
+
profile?: string;
|
|
146
|
+
openMessage?: Record<string, unknown>;
|
|
109
147
|
createSocket?: () => LiveSocket;
|
|
110
148
|
};
|