@unoverse-platform/studio 0.1.101 → 0.1.103
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/local/sample-identity/brand.yaml +2 -0
- package/package.json +1 -1
- package/screens/TemplatesLibraryView.tsx +58 -98
- package/screens/TemplatesView.tsx +16 -0
- package/vendor/base/src/definitions/load/assets.ts +65 -0
- package/vendor/base/src/definitions/load/compile/component.ts +1 -0
- package/vendor/base/src/definitions/load/compile/index.ts +1 -1
- package/vendor/base/src/definitions/load/compile/sections.ts +26 -116
- package/vendor/base/src/definitions/load/compile/template.ts +41 -25
- package/vendor/base/src/definitions/load/expand/walk.ts +4 -1
- package/vendor/base/src/definitions/load/load.ts +6 -0
- package/vendor/base/src/definitions/model.ts +20 -41
- package/vendor/base/src/definitions/registry/homes.ts +9 -1
- package/vendor/base/src/lint/design/file.mjs +29 -5
- package/vendor/base/src/lint/design/vocabulary.mjs +1 -1
- package/vendor/base/src/lint/design/walk.mjs +31 -156
- package/vendor/base/src/manifests/executor/context.ts +1 -1
- package/vendor/base/src/manifests/executor/toolBridge.ts +3 -29
- package/vendor/base/src/manifests/runtime/duplex/audioLane.ts +35 -12
- package/vendor/base/src/manifests/runtime/duplex/session.ts +123 -48
- package/vendor/base/src/manifests/runtime/index.ts +1 -1
- package/vendor/base/src/platform/audioManager.ts +89 -14
- package/vendor/base/src/platform/clientTransport.ts +8 -0
- package/vendor/base/src/security/permissions.ts +2 -2
- package/vendor/sdk/appEngine.tsx +106 -46
- package/vendor/sdk/lib/UnoverseComponent.tsx +58 -5
- package/vendor/sdk/lib/app.tsx +9 -5
- package/vendor/sdk/lib/connection.tsx +14 -3
- package/vendor/sdk/lib/core/actions.ts +5 -2
- package/vendor/sdk/lib/index.ts +2 -2
- package/vendor/sdk/lib/primitives.tsx +38 -4
- package/vendor/sdk/lib/realtime/useAudioPlayback.ts +6 -1
- package/vendor/sdk/lib/voice.tsx +96 -13
- package/vendor/sdk/pageInterfaces.tsx +388 -0
|
@@ -95,20 +95,6 @@ export async function toolBridgeFor(executionContext: any): Promise<ToolBridge |
|
|
|
95
95
|
*/
|
|
96
96
|
const handoffTools = new Set<string>();
|
|
97
97
|
|
|
98
|
-
/**
|
|
99
|
-
* THE GUEST'S OWN WORDS, from the search that is about to unlock a page.
|
|
100
|
-
*
|
|
101
|
-
* A page's `query:` places ask half a sentence each (§9e rule 4) — the half true of every
|
|
102
|
-
* guest — and the platform joins the half that is different for each. This is where that
|
|
103
|
-
* half is picked up: the model wrote the guest's ask into its search args a moment ago, so
|
|
104
|
-
* nothing has to be re-derived or, worse, re-invented.
|
|
105
|
-
*
|
|
106
|
-
* Captured on the CALL and read on the ABSORB, because `absorb` receives only the result.
|
|
107
|
-
* Whoever knows the situation supplies it (§9d) — the shell-open passes it on rather than
|
|
108
|
-
* the server guessing what the conversation was about.
|
|
109
|
-
*/
|
|
110
|
-
let lastAsk = "";
|
|
111
|
-
|
|
112
98
|
const stringify = (r: unknown): string => (typeof r === "string" ? r : JSON.stringify(r));
|
|
113
99
|
|
|
114
100
|
/**
|
|
@@ -198,11 +184,6 @@ export async function toolBridgeFor(executionContext: any): Promise<ToolBridge |
|
|
|
198
184
|
async call(name, args) {
|
|
199
185
|
const app = appInvokers.get(name);
|
|
200
186
|
if (app) return stringify(await app(args));
|
|
201
|
-
if (harness.DISCOVERY_TOOL_NAMES.includes(name)) {
|
|
202
|
-
const a = (args ?? {}) as Record<string, unknown>;
|
|
203
|
-
const q = typeof a.query === "string" ? a.query : Array.isArray(a.queries) ? String(a.queries[0] ?? "") : "";
|
|
204
|
-
if (q.trim()) lastAsk = q.trim();
|
|
205
|
-
}
|
|
206
187
|
// THE ARGUMENTS GO OUT AS THE MODEL WROTE THEM. A wire-level rewrite used to strip
|
|
207
188
|
// queries the conversation had already searched; it was removed 2026-08-16 because it
|
|
208
189
|
// silently competed with the tool descriptions for the same decision, and the
|
|
@@ -248,18 +229,11 @@ export async function toolBridgeFor(executionContext: any): Promise<ToolBridge |
|
|
|
248
229
|
* 2026-09-06). This used to open the shell at MINT — every app a search unlocked
|
|
249
230
|
* was rendered before the model had read the results (observed 2026-09-07: two
|
|
250
231
|
* surfaces opened at once, neither asked for). Tasks are asked: the unlock mints the
|
|
251
|
-
* tool and nothing more. What the shell-open used to carry
|
|
252
|
-
* ranked rows for drafting, the
|
|
253
|
-
* ids for tracing — rides the model's CALL instead, beneath its own arguments.
|
|
232
|
+
* tool and nothing more. What the shell-open used to carry, the unlocking search's
|
|
233
|
+
* ranked rows for drafting, rides the model's CALL instead, beneath its own arguments.
|
|
254
234
|
*/
|
|
255
235
|
const draftRefs = harness.rankedRowIds(resultContent);
|
|
256
|
-
const ride = {
|
|
257
|
-
__draftRefs: draftRefs,
|
|
258
|
-
...(lastAsk ? { __ask: lastAsk } : {}),
|
|
259
|
-
...(executionContext?.workflowId ? { __workflowId: executionContext.workflowId } : {}),
|
|
260
|
-
...(executionContext?.executionId ? { __executionId: executionContext.executionId } : {}),
|
|
261
|
-
...(executionContext?.nodeId ? { __nodeId: executionContext.nodeId } : {}),
|
|
262
|
-
};
|
|
236
|
+
const ride = { __draftRefs: draftRefs };
|
|
263
237
|
appInvokers.set(def.name, (input: unknown) => invoke({ ...ride, ...((input ?? {}) as Record<string, unknown>) }));
|
|
264
238
|
minted.push({ name: def.name, description: def.description, parameters: def.parameters, ...(def.outputSchema ? { outputSchema: def.outputSchema } : {}) });
|
|
265
239
|
},
|
|
@@ -24,11 +24,33 @@
|
|
|
24
24
|
* error: a voice node with nowhere to send audio still holds a valid conversation, and the
|
|
25
25
|
* transcripts still reach the workflow.
|
|
26
26
|
*/
|
|
27
|
+
/** What a session hands the lane when it claims a conversation. */
|
|
28
|
+
export interface LaneSession {
|
|
29
|
+
/** A microphone frame for this conversation. */
|
|
30
|
+
audio(data: ArrayBuffer): void | Promise<void>;
|
|
31
|
+
/** A control message for this conversation (`stop` when the person hangs up). */
|
|
32
|
+
control(message: any): void | Promise<void>;
|
|
33
|
+
/** Another session has claimed this conversation: close. */
|
|
34
|
+
replaced(): void;
|
|
35
|
+
}
|
|
36
|
+
|
|
27
37
|
export interface AudioLane {
|
|
28
38
|
sendAudio(conversationId: string, bytes: Buffer): void;
|
|
29
39
|
sendControl(conversationId: string, message: unknown): void;
|
|
30
|
-
|
|
31
|
-
|
|
40
|
+
/**
|
|
41
|
+
* ONE LIVE SESSION PER CONVERSATION (docs/VOICE_STREAMING_GUIDE.md, "One Live Session Per
|
|
42
|
+
* Conversation"). A session claims its conversation when its vendor socket opens, and from then
|
|
43
|
+
* on that conversation's microphone frames and control messages reach it and nothing else.
|
|
44
|
+
*
|
|
45
|
+
* A second claim on the same conversation REPLACES the first: the lane calls the old session's
|
|
46
|
+
* `replaced`, and it closes. Two sessions never hold one conversation, so a page swapping from
|
|
47
|
+
* one task's voice to another's never bills for both, and two visitors calling at once never
|
|
48
|
+
* take each other's microphone.
|
|
49
|
+
*
|
|
50
|
+
* Returns the release, called when the session ends. It lets go of this claim only, so a
|
|
51
|
+
* replaced session finishing late cannot remove the one that replaced it.
|
|
52
|
+
*/
|
|
53
|
+
claim?(conversationId: string, session: LaneSession): () => void;
|
|
32
54
|
startAudioSession?(sessionId: string, conversationId: string): void;
|
|
33
55
|
|
|
34
56
|
/**
|
|
@@ -49,18 +71,19 @@ export interface AudioLane {
|
|
|
49
71
|
/**
|
|
50
72
|
* IS ANYONE THERE YET? Resolves true once the far end is listening, false if it never will be.
|
|
51
73
|
*
|
|
52
|
-
*
|
|
53
|
-
*
|
|
54
|
-
*
|
|
55
|
-
*
|
|
56
|
-
*
|
|
57
|
-
*
|
|
74
|
+
* Every lane has to reach someone. A telephone rings for twenty or thirty seconds; a page that
|
|
75
|
+
* only started a workflow is asked to join (`JOIN_CALL`) and its audio socket follows. Without
|
|
76
|
+
* this the vendor session opens immediately and the model speaks its opening line to nobody
|
|
77
|
+
* (observed live 2026-09-20 on the phone). A page that dialled us first is already there, and
|
|
78
|
+
* the browser lane answers true at once.
|
|
79
|
+
*
|
|
80
|
+
* `false` is a call nobody came to: busy, declined, rung out, or a page that never joined. The
|
|
81
|
+
* session closes for one of those, or a realtime model sits billing against an empty line.
|
|
58
82
|
*
|
|
59
|
-
* `
|
|
60
|
-
*
|
|
61
|
-
* telephone nobody picked up.
|
|
83
|
+
* `call` names who the call is for, so a lane can ask them to join. A lane that places its own
|
|
84
|
+
* call ignores it.
|
|
62
85
|
*/
|
|
63
|
-
whenConnected?(conversationId: string): Promise<boolean>;
|
|
86
|
+
whenConnected?(conversationId: string, call?: { userId?: string; workflowId?: string }): Promise<boolean>;
|
|
64
87
|
|
|
65
88
|
/**
|
|
66
89
|
* END THE CALL AT THE FAR END. Not closing our socket to the vendor: that stops the MODEL,
|
|
@@ -59,6 +59,13 @@ import { makeAudioBuffer, resamplePcm16, mulawToPcm16, CLIENT_RATE, type AudioLa
|
|
|
59
59
|
const HANG_UP_DRAIN_MS = 1500;
|
|
60
60
|
|
|
61
61
|
const IDLE_TIMEOUT_MS = 30_000;
|
|
62
|
+
/**
|
|
63
|
+
* NOBODY HAS SPOKEN FOR A MINUTE: the call ends (docs/VOICE_STREAMING_GUIDE.md, "Every Call
|
|
64
|
+
* Ends"). The idle timer above cannot catch this: the page's microphone streams frames the whole
|
|
65
|
+
* time, spoken into or not, so a call left open on a quiet page was billed until MAX. Only
|
|
66
|
+
* speech holds this one (`control.speech`), and so does work in progress away from the call.
|
|
67
|
+
*/
|
|
68
|
+
const SILENCE_TIMEOUT_MS = 60_000;
|
|
62
69
|
const MAX_SESSION_MS = 30 * 60_000;
|
|
63
70
|
|
|
64
71
|
import { offerDiscovered, parseMaybeJson, recordToolTrace, type ToolBridge } from "../tools/toolloop.js";
|
|
@@ -73,7 +80,9 @@ async function message(spec: unknown, ctx: RunContext, extra: Record<string, unk
|
|
|
73
80
|
|
|
74
81
|
export interface DuplexResult {
|
|
75
82
|
/** Why the session ended, for the caller to record. */
|
|
76
|
-
|
|
83
|
+
/** `replaced`: another session claimed this conversation (AudioLane.claim). `silence`: nobody
|
|
84
|
+
* spoke for SILENCE_TIMEOUT_MS. `stopped`: the run was stopped (a page that left). */
|
|
85
|
+
reason: "closed" | "ended" | "error" | "idle" | "maxDuration" | "notConnected" | "replaced" | "silence" | "stopped";
|
|
77
86
|
error?: string;
|
|
78
87
|
}
|
|
79
88
|
|
|
@@ -156,13 +165,14 @@ export async function runDuplexSession(opts: {
|
|
|
156
165
|
* generated early and flushes it the instant the call connects (twilioAudio.ts). So the only
|
|
157
166
|
* thing that still waits for the answer is the sound itself.
|
|
158
167
|
*
|
|
159
|
-
* The browser lane
|
|
160
|
-
*
|
|
168
|
+
* The browser lane is the same: a page that only started a workflow is asked to join, and the
|
|
169
|
+
* lane holds the greeting until its audio socket arrives. A page that dialled us is already
|
|
170
|
+
* there and answers at once.
|
|
161
171
|
*/
|
|
162
172
|
let answered: Promise<boolean> | null = null;
|
|
163
173
|
if (lane?.whenConnected) {
|
|
164
174
|
lane.startAudioSession?.(conversationId, conversationId);
|
|
165
|
-
answered = lane.whenConnected(conversationId);
|
|
175
|
+
answered = lane.whenConnected(conversationId, { userId: ctx.scope?.userId, workflowId: ctx.scope?.workflowId });
|
|
166
176
|
}
|
|
167
177
|
|
|
168
178
|
const ws = new WebSocket(url, { headers });
|
|
@@ -231,6 +241,32 @@ export async function runDuplexSession(opts: {
|
|
|
231
241
|
/** Set while the session is open and takes `CONTINUE` inputs; cleared when it ends. */
|
|
232
242
|
let unregisterLive: (() => void) | undefined;
|
|
233
243
|
|
|
244
|
+
/**
|
|
245
|
+
* THE PERSON'S TURN AND THE BACKEND'S WORK, said to the client once per change. A vendor with
|
|
246
|
+
* no start or end event of its own marks them with an event that repeats (GPT-Live's
|
|
247
|
+
* transcript deltas, one per word), and forwarding every repeat sent the client the same
|
|
248
|
+
* state dozens of times a second. A state that names a tool always goes, because the name is
|
|
249
|
+
* new information.
|
|
250
|
+
*/
|
|
251
|
+
const lastState: { user?: string; tool?: string } = {};
|
|
252
|
+
const sendState = (group: "user" | "tool", state: string, metadata?: Record<string, unknown>) => {
|
|
253
|
+
// Work starting or finishing away from the call is the call being alive.
|
|
254
|
+
if (group === "tool") heard();
|
|
255
|
+
if (lastState[group] === state && !metadata) return;
|
|
256
|
+
lastState[group] = state;
|
|
257
|
+
lane?.sendControl(conversationId, { type: "AUDIO_STATE", state, ...(metadata ? { metadata } : {}) });
|
|
258
|
+
};
|
|
259
|
+
|
|
260
|
+
/**
|
|
261
|
+
* SOMEONE SPOKE: the events that mean it, the node's own list or the person's speech and the
|
|
262
|
+
* assistant's audio. Restarts the silence timer; set once the session is open.
|
|
263
|
+
*/
|
|
264
|
+
const speechEvents: string[] = audio.control?.speech ?? [audio.control?.userSpeaking, outSpec?.match].filter(Boolean);
|
|
265
|
+
let heard: () => void = () => {};
|
|
266
|
+
|
|
267
|
+
/** This session's claim on its conversation's audio lane; let go when it ends. */
|
|
268
|
+
let release: (() => void) | undefined;
|
|
269
|
+
|
|
234
270
|
const send = (payload: unknown) => {
|
|
235
271
|
if (ws.readyState !== WebSocket.OPEN) return;
|
|
236
272
|
ws.send(JSON.stringify(payload));
|
|
@@ -263,6 +299,12 @@ export async function runDuplexSession(opts: {
|
|
|
263
299
|
void shutdown("maxDuration").then(() => finish({ reason: "maxDuration" }));
|
|
264
300
|
}, MAX_SESSION_MS);
|
|
265
301
|
|
|
302
|
+
// THE RUN WAS STOPPED (a page that left, a reaped run): the call ends with it, closed the
|
|
303
|
+
// way a hang-up closes it. Without this a stopped run left the vendor billing until silence.
|
|
304
|
+
const stop = () => void shutdown("stopped").then(() => finish({ reason: "stopped" }));
|
|
305
|
+
if (ctx.abort?.aborted) stop();
|
|
306
|
+
else ctx.abort?.addEventListener("abort", stop, { once: true });
|
|
307
|
+
|
|
266
308
|
/**
|
|
267
309
|
* Restarted by every inbound vendor frame AND by every outbound mic frame, so it only fires
|
|
268
310
|
* when BOTH sides have gone quiet. The vendor is legitimately silent while the person speaks
|
|
@@ -278,6 +320,17 @@ export async function runDuplexSession(opts: {
|
|
|
278
320
|
}, IDLE_TIMEOUT_MS);
|
|
279
321
|
};
|
|
280
322
|
|
|
323
|
+
// SILENCE (SILENCE_TIMEOUT_MS). A tool or hand-off still running is not silence: the timer
|
|
324
|
+
// waits another round for it, so a slow answer is never cut off.
|
|
325
|
+
let silenceTimer: ReturnType<typeof setTimeout> | undefined;
|
|
326
|
+
heard = () => {
|
|
327
|
+
if (silenceTimer) clearTimeout(silenceTimer);
|
|
328
|
+
silenceTimer = setTimeout(() => {
|
|
329
|
+
if (lastState.tool === "TOOL_USE") return heard();
|
|
330
|
+
void shutdown("silence").then(() => finish({ reason: "silence" }));
|
|
331
|
+
}, SILENCE_TIMEOUT_MS);
|
|
332
|
+
};
|
|
333
|
+
|
|
281
334
|
/**
|
|
282
335
|
* THE ONE PLACE A SESSION ENDS, and now the one place a CALL ends with it.
|
|
283
336
|
*
|
|
@@ -292,7 +345,18 @@ export async function runDuplexSession(opts: {
|
|
|
292
345
|
if (settled) return;
|
|
293
346
|
settled = true;
|
|
294
347
|
unregisterLive?.();
|
|
348
|
+
release?.();
|
|
295
349
|
if (idleTimer) clearTimeout(idleTimer);
|
|
350
|
+
if (silenceTimer) clearTimeout(silenceTimer);
|
|
351
|
+
// THE PAGE IS TOLD, so its microphone stops and its avatar goes idle. Not when replaced:
|
|
352
|
+
// the next session is already on the call, and the call goes on.
|
|
353
|
+
if (result.reason !== "replaced") {
|
|
354
|
+
try {
|
|
355
|
+
lane?.sendControl(conversationId, { type: "AUDIO_STATE", state: "SESSION_ENDED" });
|
|
356
|
+
} catch {
|
|
357
|
+
/* a lane already gone has nobody to tell */
|
|
358
|
+
}
|
|
359
|
+
}
|
|
296
360
|
if (drainTimer) clearTimeout(drainTimer);
|
|
297
361
|
clearTimeout(maxTimer);
|
|
298
362
|
try {
|
|
@@ -342,6 +406,8 @@ export async function runDuplexSession(opts: {
|
|
|
342
406
|
|
|
343
407
|
ws.on("open", async () => {
|
|
344
408
|
touch();
|
|
409
|
+
// The silence clock starts with the call: a call nobody ever speaks on still ends.
|
|
410
|
+
heard();
|
|
345
411
|
try {
|
|
346
412
|
/**
|
|
347
413
|
* THE TOOLS THE CALL OPENS WITH, discovered before the handshake goes out because the
|
|
@@ -400,6 +466,8 @@ export async function runDuplexSession(opts: {
|
|
|
400
466
|
unregisterLive = registerLiveInput(key, async (handle, value) => {
|
|
401
467
|
if (ws.readyState !== WebSocket.OPEN) return false;
|
|
402
468
|
touch();
|
|
469
|
+
// The task moving on (a step handed back) is the call being worked.
|
|
470
|
+
heard();
|
|
403
471
|
const scope = { input: { handle, value }, seen, said };
|
|
404
472
|
let sent = 0;
|
|
405
473
|
for (const rule of call.send ?? []) {
|
|
@@ -428,44 +496,54 @@ export async function runDuplexSession(opts: {
|
|
|
428
496
|
// lane twice would place a second call.
|
|
429
497
|
if (!lane.whenConnected) lane.startAudioSession?.(conversationId, conversationId);
|
|
430
498
|
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
499
|
+
// THE CONVERSATION IS THIS SESSION'S NOW (AudioLane.claim): its microphone and its
|
|
500
|
+
// control reach this session only, and a session that held it before is told to close.
|
|
501
|
+
release = lane.claim?.(conversationId, {
|
|
502
|
+
audio: async (data) => {
|
|
503
|
+
if (ws.readyState !== WebSocket.OPEN) return;
|
|
504
|
+
touch();
|
|
505
|
+
const raw = Buffer.from(data);
|
|
506
|
+
const rate = inSpec.resampleTo ?? CLIENT_RATE;
|
|
507
|
+
|
|
508
|
+
// NOTHING TO DO: the vendor asked for this lane's own codec (see `passthrough`
|
|
509
|
+
// above), so these are the bytes to send. The rate travels as the LANE's, because
|
|
510
|
+
// that is what the samples actually are.
|
|
511
|
+
if (passthrough) {
|
|
512
|
+
send(
|
|
513
|
+
await message(inSpec.message, ctx, {
|
|
514
|
+
audio: { base64: raw.toString("base64"), bytes: raw.length, rate: lane.rate ?? CLIENT_RATE },
|
|
515
|
+
}),
|
|
516
|
+
);
|
|
517
|
+
return;
|
|
518
|
+
}
|
|
519
|
+
/**
|
|
520
|
+
* DECODE BEFORE RESAMPLING. A mu-law byte is one sample, not half of one, so
|
|
521
|
+
* interpolating the raw bytes would treat pairs of samples as single ones and
|
|
522
|
+
* produce half-speed noise. And mu-law is logarithmic, so even the arithmetic
|
|
523
|
+
* would be wrong: the midpoint of two bytes is not the midpoint of two sounds.
|
|
524
|
+
*
|
|
525
|
+
* The browser lane declares neither, so both lines are identity for it.
|
|
526
|
+
*/
|
|
527
|
+
const decoded = lane.codec === "mulaw" ? mulawToPcm16(raw) : raw;
|
|
528
|
+
const pcm = resamplePcm16(decoded, lane.rate ?? CLIENT_RATE, rate);
|
|
441
529
|
send(
|
|
442
530
|
await message(inSpec.message, ctx, {
|
|
443
|
-
audio: { base64:
|
|
531
|
+
audio: { base64: pcm.toString("base64"), bytes: pcm.length, rate },
|
|
444
532
|
}),
|
|
445
533
|
);
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
await message(inSpec.message, ctx, {
|
|
460
|
-
audio: { base64: pcm.toString("base64"), bytes: pcm.length, rate },
|
|
461
|
-
}),
|
|
462
|
-
);
|
|
463
|
-
});
|
|
464
|
-
|
|
465
|
-
lane.setControlMessageHandler?.(async (sessionId, msg) => {
|
|
466
|
-
if (sessionId !== conversationId) return;
|
|
467
|
-
const endOn: string[] = audio.control?.endOn ?? [];
|
|
468
|
-
if (endOn.includes(String(msg?.type))) finish({ reason: await shutdown("ended") ?? "ended" });
|
|
534
|
+
},
|
|
535
|
+
control: async (msg) => {
|
|
536
|
+
const endOn: string[] = audio.control?.endOn ?? [];
|
|
537
|
+
if (endOn.includes(String(msg?.type))) finish({ reason: await shutdown("ended") ?? "ended" });
|
|
538
|
+
},
|
|
539
|
+
// ANOTHER SESSION TOOK THIS CONVERSATION: the page moved to a task another voice works.
|
|
540
|
+
// Closed the way a hang-up closes it, through the `close` list. What it had not yet
|
|
541
|
+
// played is thrown away, not flushed: the person has moved on, and the next voice
|
|
542
|
+
// must not be talked over by the last one.
|
|
543
|
+
replaced: () => {
|
|
544
|
+
buffer?.discard();
|
|
545
|
+
void shutdown("replaced").then(() => finish({ reason: "replaced" }));
|
|
546
|
+
},
|
|
469
547
|
});
|
|
470
548
|
}
|
|
471
549
|
} catch (err: any) {
|
|
@@ -484,6 +562,7 @@ export async function runDuplexSession(opts: {
|
|
|
484
562
|
return; // a frame that is not JSON is not something a manifest can describe
|
|
485
563
|
}
|
|
486
564
|
const type = String(event?.type ?? "");
|
|
565
|
+
if (speechEvents.includes(type)) heard();
|
|
487
566
|
|
|
488
567
|
// A REJECTED SESSION MUST SAY WHY. Every vendor this runtime speaks names its complaint in
|
|
489
568
|
// an `error` event and then closes; without this line a refused handshake looks exactly like
|
|
@@ -592,17 +671,17 @@ export async function runDuplexSession(opts: {
|
|
|
592
671
|
// Separate from `interruptOn` even when both name the same event: this tells the client the
|
|
593
672
|
// person is talking, that one throws away the audio they talked over.
|
|
594
673
|
if (audio.control?.userSpeaking && type === audio.control.userSpeaking) {
|
|
595
|
-
|
|
674
|
+
sendState("user", "USER_SPEECH_STARTED");
|
|
596
675
|
}
|
|
597
676
|
if (audio.control?.userStopped && type === audio.control.userStopped) {
|
|
598
|
-
|
|
677
|
+
sendState("user", "USER_SPEECH_ENDED");
|
|
599
678
|
}
|
|
600
679
|
// Work done off the call (a vendor's hand-off to a backend), for the call state's "thinking".
|
|
601
680
|
if (audio.control?.toolUse && type === audio.control.toolUse) {
|
|
602
|
-
|
|
681
|
+
sendState("tool", "TOOL_USE");
|
|
603
682
|
}
|
|
604
683
|
if (audio.control?.toolUseCompleted && type === audio.control.toolUseCompleted) {
|
|
605
|
-
|
|
684
|
+
sendState("tool", "TOOL_USE_COMPLETED");
|
|
606
685
|
}
|
|
607
686
|
|
|
608
687
|
/**
|
|
@@ -670,11 +749,7 @@ export async function runDuplexSession(opts: {
|
|
|
670
749
|
let output: string;
|
|
671
750
|
// A lookup takes 8-9s, and silence on a call reads as a dropped line. The client,
|
|
672
751
|
// the lookup-indicator and the docs all had this wired already; nothing ever sent it.
|
|
673
|
-
|
|
674
|
-
type: "AUDIO_STATE",
|
|
675
|
-
state: "TOOL_USE",
|
|
676
|
-
metadata: { toolName: name },
|
|
677
|
-
});
|
|
752
|
+
sendState("tool", "TOOL_USE", { toolName: name });
|
|
678
753
|
// Started BEFORE the call, so the timeline bar measures the wait the caller
|
|
679
754
|
// actually sat through, not the bookkeeping after it.
|
|
680
755
|
const toolStart = Date.now();
|
|
@@ -758,7 +833,7 @@ export async function runDuplexSession(opts: {
|
|
|
758
833
|
recordToolTrace(node, ctx, name, parseMaybeJson(rawArgs), output, toolStart, toolOk, tools.kindOf?.(name) ?? "tool");
|
|
759
834
|
// After the catch, so a tool that THREW still stops the dots. Spinning forever is
|
|
760
835
|
// worse than no indicator: it promises something still coming.
|
|
761
|
-
|
|
836
|
+
sendState("tool", "TOOL_USE_COMPLETED");
|
|
762
837
|
// PARSED for the connector lane only (readable mcpResult, same as the HTTP
|
|
763
838
|
// loop); the model message two lines down keeps the raw `output` string —
|
|
764
839
|
// the vendor wire requires text.
|
|
@@ -52,7 +52,7 @@ export { performDoc, MUTATING_DOC_OPS, docKeyFor } from "./docstore/index.js";
|
|
|
52
52
|
export { runDuplexSession } from "./duplex/session.js";
|
|
53
53
|
// The audio lane is its own module: session.ts owns the VENDOR socket, audioLane.ts owns the bytes
|
|
54
54
|
// and the browser. Two sockets, two files.
|
|
55
|
-
export { resamplePcm16, mulawToPcm16, pcm16ToMulaw, makeAudioBuffer, CLIENT_RATE, type AudioLane } from "./duplex/audioLane.js";
|
|
55
|
+
export { resamplePcm16, mulawToPcm16, pcm16ToMulaw, makeAudioBuffer, CLIENT_RATE, type AudioLane, type LaneSession } from "./duplex/audioLane.js";
|
|
56
56
|
export { performApi, type RunResult } from "./channels/workflow.js";
|
|
57
57
|
export { runFinal } from "./channels/final.js";
|
|
58
58
|
export { performService } from "./channels/service.js";
|
|
@@ -8,10 +8,14 @@
|
|
|
8
8
|
*/
|
|
9
9
|
|
|
10
10
|
import { clientTransport } from "./clientTransport.js";
|
|
11
|
+
import type { LaneSession } from "../manifests/runtime/duplex/audioLane.js";
|
|
12
|
+
|
|
13
|
+
/** How long a call the workflow started waits for its page to join before nobody is there. */
|
|
14
|
+
const JOIN_WAIT_MS = 15_000;
|
|
11
15
|
|
|
12
16
|
export interface AudioWebSocketManager {
|
|
13
|
-
|
|
14
|
-
|
|
17
|
+
claim(conversationId: string, session: LaneSession): () => void;
|
|
18
|
+
whenConnected(conversationId: string, call?: { userId?: string; workflowId?: string }): Promise<boolean>;
|
|
15
19
|
startAudioSession(conversationId: string, novaSessionId?: string): void;
|
|
16
20
|
endAudioSession(conversationId: string): void;
|
|
17
21
|
isConnected(conversationId: string): boolean;
|
|
@@ -23,26 +27,86 @@ class DistributedAudioManager implements AudioWebSocketManager {
|
|
|
23
27
|
private audioSessions: Map<string, { novaSessionId?: string; active: boolean }> = new Map();
|
|
24
28
|
|
|
25
29
|
/**
|
|
26
|
-
*
|
|
30
|
+
* THE LIVE SESSION OF EACH CONVERSATION, one each (docs/VOICE_STREAMING_GUIDE.md, "One Live
|
|
31
|
+
* Session Per Conversation"). The transport has ONE audio handler and ONE control handler for
|
|
32
|
+
* the whole process, so they are installed once, here, and hand each frame and each control
|
|
33
|
+
* message to the session that holds that conversation and to no other.
|
|
34
|
+
*
|
|
35
|
+
* Installing a session's own handler into the transport instead, as this did, meant the
|
|
36
|
+
* newest session anywhere on the server took every conversation's microphone: the one before
|
|
37
|
+
* went deaf, never heard `stop`, and idled out, and two visitors calling at once broke each
|
|
38
|
+
* other.
|
|
39
|
+
*/
|
|
40
|
+
private live = new Map<string, LaneSession>();
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* CALLS WAITING FOR THEIR PAGE (docs/VOICE_STREAMING_GUIDE.md, "Call Initialization"). What the
|
|
44
|
+
* session sends meanwhile, its greeting first, is held in order and played once the page's
|
|
45
|
+
* audio socket arrives, so nothing it says is spoken to nobody.
|
|
27
46
|
*/
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
47
|
+
private joining = new Map<string, { held: Array<() => void>; answer: (ok: boolean) => void }>();
|
|
48
|
+
|
|
49
|
+
/** Conversations whose microphone has been reported, so each is said once, not per frame. */
|
|
50
|
+
private heard = new Map<string, "session" | "nobody">();
|
|
51
|
+
|
|
52
|
+
private install(): void {
|
|
31
53
|
clientTransport().setAudioDataHandler(async (conversationId, audioData) => {
|
|
32
|
-
const session = this.
|
|
33
|
-
|
|
34
|
-
|
|
54
|
+
const session = this.live.get(conversationId);
|
|
55
|
+
// THE MICROPHONE, SAID ONCE: where a conversation's frames first go. A frame nobody holds
|
|
56
|
+
// is dropped silently otherwise, and heard only as an assistant who never listens.
|
|
57
|
+
const now = session ? "session" : "nobody";
|
|
58
|
+
if (this.heard.get(conversationId) !== now) {
|
|
59
|
+
this.heard.set(conversationId, now);
|
|
60
|
+
console.log(`[DistributedAudioManager] ${conversationId}: microphone frames reach ${now === "session" ? "the live voice session" : "NO session (dropped)"}`);
|
|
35
61
|
}
|
|
62
|
+
await session?.audio(audioData);
|
|
63
|
+
});
|
|
64
|
+
clientTransport().setAudioControlHandler(async (conversationId, message) => {
|
|
65
|
+
await this.live.get(conversationId)?.control(message);
|
|
36
66
|
});
|
|
37
|
-
|
|
67
|
+
clientTransport().setAudioSocketHandler((conversationId) => this.joining.get(conversationId)?.answer(true));
|
|
38
68
|
}
|
|
39
69
|
|
|
40
70
|
/**
|
|
41
|
-
*
|
|
71
|
+
* IS THE PAGE THERE? A page that dialled us already has its audio socket. One that only started
|
|
72
|
+
* a workflow is asked to join on the stream it already holds, and has JOIN_WAIT_MS to do it.
|
|
42
73
|
*/
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
74
|
+
whenConnected(conversationId: string, call?: { userId?: string; workflowId?: string }): Promise<boolean> {
|
|
75
|
+
this.install();
|
|
76
|
+
if (clientTransport().hasAudioSocket(conversationId)) return Promise.resolve(true);
|
|
77
|
+
return new Promise<boolean>((resolve) => {
|
|
78
|
+
const answer = (ok: boolean) => {
|
|
79
|
+
clearTimeout(timer);
|
|
80
|
+
const waiting = this.joining.get(conversationId);
|
|
81
|
+
if (waiting?.answer !== answer) return;
|
|
82
|
+
this.joining.delete(conversationId);
|
|
83
|
+
console.log(`[DistributedAudioManager] ${conversationId}: ${ok ? "the page joined the call" : "the page never joined; the call ends"}`);
|
|
84
|
+
if (ok) for (const send of waiting.held) send();
|
|
85
|
+
resolve(ok);
|
|
86
|
+
};
|
|
87
|
+
const timer = setTimeout(() => answer(false), JOIN_WAIT_MS);
|
|
88
|
+
this.joining.set(conversationId, { held: [], answer });
|
|
89
|
+
const asked = !!call?.userId && clientTransport().pushToClient(call.userId, conversationId, { type: "JOIN_CALL", workflowId: call.workflowId });
|
|
90
|
+
console.log(`[DistributedAudioManager] ${conversationId}: ${asked ? "asked the page to join" : "no page stream to ask; waiting for its audio socket"}`);
|
|
91
|
+
});
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
claim(conversationId: string, session: LaneSession): () => void {
|
|
95
|
+
this.install();
|
|
96
|
+
const before = this.live.get(conversationId);
|
|
97
|
+
this.live.set(conversationId, session);
|
|
98
|
+
console.log(`[DistributedAudioManager] ${conversationId}: a voice session claimed the call`);
|
|
99
|
+
if (before && before !== session) {
|
|
100
|
+
console.log(`[DistributedAudioManager] ${conversationId}: a new voice session took the call; the one before it closes`);
|
|
101
|
+
try {
|
|
102
|
+
before.replaced();
|
|
103
|
+
} catch (err: any) {
|
|
104
|
+
console.error(`[DistributedAudioManager] ${conversationId}: the replaced session did not close: ${err?.message ?? err}`);
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
return () => {
|
|
108
|
+
if (this.live.get(conversationId) === session) this.live.delete(conversationId);
|
|
109
|
+
};
|
|
46
110
|
}
|
|
47
111
|
|
|
48
112
|
/**
|
|
@@ -75,6 +139,12 @@ class DistributedAudioManager implements AudioWebSocketManager {
|
|
|
75
139
|
* Send audio to client via server
|
|
76
140
|
*/
|
|
77
141
|
sendAudio(conversationId: string, audioData: Buffer | ArrayBuffer): boolean {
|
|
142
|
+
const waiting = this.joining.get(conversationId);
|
|
143
|
+
if (waiting) {
|
|
144
|
+
const bytes = Buffer.isBuffer(audioData) ? Buffer.from(audioData) : audioData.slice(0);
|
|
145
|
+
waiting.held.push(() => clientTransport().sendAudioToClient(conversationId, bytes));
|
|
146
|
+
return true;
|
|
147
|
+
}
|
|
78
148
|
return clientTransport().sendAudioToClient(conversationId, audioData);
|
|
79
149
|
}
|
|
80
150
|
|
|
@@ -83,6 +153,11 @@ class DistributedAudioManager implements AudioWebSocketManager {
|
|
|
83
153
|
*/
|
|
84
154
|
sendControl(conversationId: string, message: Record<string, any>): boolean {
|
|
85
155
|
if (message.type === "AUDIO_STATE" || message.state) {
|
|
156
|
+
const waiting = this.joining.get(conversationId);
|
|
157
|
+
if (waiting) {
|
|
158
|
+
waiting.held.push(() => clientTransport().sendAudioStateToClient(conversationId, message.state || message.type, message.metadata));
|
|
159
|
+
return true;
|
|
160
|
+
}
|
|
86
161
|
return clientTransport().sendAudioStateToClient(conversationId, message.state || message.type, message.metadata);
|
|
87
162
|
}
|
|
88
163
|
return false;
|
|
@@ -38,6 +38,12 @@ export interface ClientTransport {
|
|
|
38
38
|
|
|
39
39
|
/** Register the handler for audio control messages from a client. */
|
|
40
40
|
setAudioControlHandler(handler: (conversationId: string, message: Record<string, unknown>) => Promise<void>): void;
|
|
41
|
+
|
|
42
|
+
/** Is a client's audio socket on this conversation, and not one a call has already ended on? */
|
|
43
|
+
hasAudioSocket(conversationId: string): boolean;
|
|
44
|
+
|
|
45
|
+
/** Register the handler told each time a client's audio socket arrives for a conversation. */
|
|
46
|
+
setAudioSocketHandler(handler: (conversationId: string) => void): void;
|
|
41
47
|
}
|
|
42
48
|
|
|
43
49
|
/**
|
|
@@ -50,6 +56,8 @@ const NONE: ClientTransport = {
|
|
|
50
56
|
sendAudioStateToClient: () => false,
|
|
51
57
|
setAudioDataHandler: () => {},
|
|
52
58
|
setAudioControlHandler: () => {},
|
|
59
|
+
hasAudioSocket: () => false,
|
|
60
|
+
setAudioSocketHandler: () => {},
|
|
53
61
|
};
|
|
54
62
|
|
|
55
63
|
let transport: ClientTransport = NONE;
|
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
* had no role check at all, so "who is an administrator" had two answers and one of them was
|
|
14
14
|
* "nobody". One map, imported twice.
|
|
15
15
|
*
|
|
16
|
-
* ROLES ARE THE TOKEN'S SHAPE; PERMISSIONS ARE THE CODE'S SHAPE.
|
|
16
|
+
* ROLES ARE THE TOKEN'S SHAPE; PERMISSIONS ARE THE CODE'S SHAPE. The IdP issues this
|
|
17
17
|
* platform `roles: ["admin"]` / `["author"]`, and the tokens carry no permissions claim we
|
|
18
18
|
* can rely on. So `ROLE_GRANTS` is the bridge, and it is deliberately the ONLY place the
|
|
19
19
|
* two vocabularies meet: one map, read once, so switching to real permission claims later
|
|
@@ -59,7 +59,7 @@ export const PERMISSIONS = [
|
|
|
59
59
|
export type Permission = (typeof PERMISSIONS)[number];
|
|
60
60
|
|
|
61
61
|
/**
|
|
62
|
-
* WHICH ROLE GRANTS WHAT. The one bridge between
|
|
62
|
+
* WHICH ROLE GRANTS WHAT. The one bridge between the IdP's vocabulary and this code's.
|
|
63
63
|
*
|
|
64
64
|
* `admin` is `author` plus the secrets. That is the entire difference between them, stated
|
|
65
65
|
* as data rather than described in a comment somewhere, so it cannot drift.
|