@unoverse-platform/studio 0.1.100 → 0.1.102

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. package/local/sample-identity/brand.yaml +2 -0
  2. package/package.json +1 -1
  3. package/screens/TemplatesLibraryView.tsx +58 -98
  4. package/screens/TemplatesView.tsx +18 -2
  5. package/vendor/base/src/auth/auth.ts +7 -5
  6. package/vendor/base/src/auth/authConfig.ts +1 -1
  7. package/vendor/base/src/definitions/load/assets.ts +65 -0
  8. package/vendor/base/src/definitions/load/compile/component.ts +1 -0
  9. package/vendor/base/src/definitions/load/compile/index.ts +1 -1
  10. package/vendor/base/src/definitions/load/compile/sections.ts +26 -116
  11. package/vendor/base/src/definitions/load/compile/template.ts +41 -25
  12. package/vendor/base/src/definitions/load/expand/walk.ts +4 -1
  13. package/vendor/base/src/definitions/load/load.ts +6 -0
  14. package/vendor/base/src/definitions/model.ts +20 -41
  15. package/vendor/base/src/definitions/registry/homes.ts +9 -1
  16. package/vendor/base/src/lint/design/file.mjs +29 -5
  17. package/vendor/base/src/lint/design/vocabulary.mjs +1 -1
  18. package/vendor/base/src/lint/design/walk.mjs +31 -156
  19. package/vendor/base/src/manifests/executor/context.ts +2 -2
  20. package/vendor/base/src/manifests/executor/toolBridge.ts +3 -29
  21. package/vendor/base/src/manifests/runtime/duplex/audioLane.ts +35 -12
  22. package/vendor/base/src/manifests/runtime/duplex/session.ts +123 -48
  23. package/vendor/base/src/manifests/runtime/index.ts +1 -1
  24. package/vendor/base/src/platform/audioManager.ts +89 -14
  25. package/vendor/base/src/platform/clientTransport.ts +8 -0
  26. package/vendor/base/src/security/permissions.ts +2 -2
  27. package/vendor/sdk/appEngine.tsx +104 -46
  28. package/vendor/sdk/lib/UnoverseComponent.tsx +58 -5
  29. package/vendor/sdk/lib/app.tsx +9 -5
  30. package/vendor/sdk/lib/connection.tsx +15 -4
  31. package/vendor/sdk/lib/core/actions.ts +5 -2
  32. package/vendor/sdk/lib/index.ts +2 -2
  33. package/vendor/sdk/lib/primitives.tsx +38 -4
  34. package/vendor/sdk/lib/realtime/useAudioPlayback.ts +6 -1
  35. package/vendor/sdk/lib/voice.tsx +96 -13
  36. package/vendor/sdk/main.tsx +1 -1
  37. package/vendor/sdk/pageInterfaces.tsx +383 -0
@@ -95,20 +95,6 @@ export async function toolBridgeFor(executionContext: any): Promise<ToolBridge |
95
95
  */
96
96
  const handoffTools = new Set<string>();
97
97
 
98
- /**
99
- * THE GUEST'S OWN WORDS, from the search that is about to unlock a page.
100
- *
101
- * A page's `query:` places ask half a sentence each (§9e rule 4) — the half true of every
102
- * guest — and the platform joins the half that is different for each. This is where that
103
- * half is picked up: the model wrote the guest's ask into its search args a moment ago, so
104
- * nothing has to be re-derived or, worse, re-invented.
105
- *
106
- * Captured on the CALL and read on the ABSORB, because `absorb` receives only the result.
107
- * Whoever knows the situation supplies it (§9d) — the shell-open passes it on rather than
108
- * the server guessing what the conversation was about.
109
- */
110
- let lastAsk = "";
111
-
112
98
  const stringify = (r: unknown): string => (typeof r === "string" ? r : JSON.stringify(r));
113
99
 
114
100
  /**
@@ -198,11 +184,6 @@ export async function toolBridgeFor(executionContext: any): Promise<ToolBridge |
198
184
  async call(name, args) {
199
185
  const app = appInvokers.get(name);
200
186
  if (app) return stringify(await app(args));
201
- if (harness.DISCOVERY_TOOL_NAMES.includes(name)) {
202
- const a = (args ?? {}) as Record<string, unknown>;
203
- const q = typeof a.query === "string" ? a.query : Array.isArray(a.queries) ? String(a.queries[0] ?? "") : "";
204
- if (q.trim()) lastAsk = q.trim();
205
- }
206
187
  // THE ARGUMENTS GO OUT AS THE MODEL WROTE THEM. A wire-level rewrite used to strip
207
188
  // queries the conversation had already searched; it was removed 2026-08-16 because it
208
189
  // silently competed with the tool descriptions for the same decision, and the
@@ -248,18 +229,11 @@ export async function toolBridgeFor(executionContext: any): Promise<ToolBridge |
248
229
  * 2026-09-06). This used to open the shell at MINT — every app a search unlocked
249
230
  * was rendered before the model had read the results (observed 2026-09-07: two
250
231
  * surfaces opened at once, neither asked for). Tasks are asked: the unlock mints the
251
- * tool and nothing more. What the shell-open used to carry — the unlocking search's
252
- * ranked rows for drafting, the guest's ask for the page's own searches, the run's
253
- * ids for tracing — rides the model's CALL instead, beneath its own arguments.
232
+ * tool and nothing more. What the shell-open used to carry, the unlocking search's
233
+ * ranked rows for drafting, rides the model's CALL instead, beneath its own arguments.
254
234
  */
255
235
  const draftRefs = harness.rankedRowIds(resultContent);
256
- const ride = {
257
- __draftRefs: draftRefs,
258
- ...(lastAsk ? { __ask: lastAsk } : {}),
259
- ...(executionContext?.workflowId ? { __workflowId: executionContext.workflowId } : {}),
260
- ...(executionContext?.executionId ? { __executionId: executionContext.executionId } : {}),
261
- ...(executionContext?.nodeId ? { __nodeId: executionContext.nodeId } : {}),
262
- };
236
+ const ride = { __draftRefs: draftRefs };
263
237
  appInvokers.set(def.name, (input: unknown) => invoke({ ...ride, ...((input ?? {}) as Record<string, unknown>) }));
264
238
  minted.push({ name: def.name, description: def.description, parameters: def.parameters, ...(def.outputSchema ? { outputSchema: def.outputSchema } : {}) });
265
239
  },
@@ -24,11 +24,33 @@
24
24
  * error: a voice node with nowhere to send audio still holds a valid conversation, and the
25
25
  * transcripts still reach the workflow.
26
26
  */
27
+ /** What a session hands the lane when it claims a conversation. */
28
+ export interface LaneSession {
29
+ /** A microphone frame for this conversation. */
30
+ audio(data: ArrayBuffer): void | Promise<void>;
31
+ /** A control message for this conversation (`stop` when the person hangs up). */
32
+ control(message: any): void | Promise<void>;
33
+ /** Another session has claimed this conversation: close. */
34
+ replaced(): void;
35
+ }
36
+
27
37
  export interface AudioLane {
28
38
  sendAudio(conversationId: string, bytes: Buffer): void;
29
39
  sendControl(conversationId: string, message: unknown): void;
30
- setAudioDataHandler?(fn: (sessionId: string, data: ArrayBuffer) => void | Promise<void>): void;
31
- setControlMessageHandler?(fn: (sessionId: string, message: any) => void | Promise<void>): void;
40
+ /**
41
+ * ONE LIVE SESSION PER CONVERSATION (docs/VOICE_STREAMING_GUIDE.md, "One Live Session Per
42
+ * Conversation"). A session claims its conversation when its vendor socket opens, and from then
43
+ * on that conversation's microphone frames and control messages reach it and nothing else.
44
+ *
45
+ * A second claim on the same conversation REPLACES the first: the lane calls the old session's
46
+ * `replaced`, and it closes. Two sessions never hold one conversation, so a page swapping from
47
+ * one task's voice to another's never bills for both, and two visitors calling at once never
48
+ * take each other's microphone.
49
+ *
50
+ * Returns the release, called when the session ends. It lets go of this claim only, so a
51
+ * replaced session finishing late cannot remove the one that replaced it.
52
+ */
53
+ claim?(conversationId: string, session: LaneSession): () => void;
32
54
  startAudioSession?(sessionId: string, conversationId: string): void;
33
55
 
34
56
  /**
@@ -49,18 +71,19 @@ export interface AudioLane {
49
71
  /**
50
72
  * IS ANYONE THERE YET? Resolves true once the far end is listening, false if it never will be.
51
73
  *
52
- * ABSENT MEANS ALREADY CONNECTED, which is the browser: the page dialled us, so its socket
53
- * exists before the node runs and there is nothing to wait for. A lane that has to REACH
54
- * someone is the other case, and the difference matters more than it sounds. A telephone rings
55
- * for twenty or thirty seconds, and without this the vendor session opens immediately, the
56
- * model speaks its opening line to a ringing tone, and by the time a person says hello the
57
- * greeting has already happened and been thrown away (observed live 2026-09-20).
74
+ * Every lane has to reach someone. A telephone rings for twenty or thirty seconds; a page that
75
+ * only started a workflow is asked to join (`JOIN_CALL`) and its audio socket follows. Without
76
+ * this the vendor session opens immediately and the model speaks its opening line to nobody
77
+ * (observed live 2026-09-20 on the phone). A page that dialled us first is already there, and
78
+ * the browser lane answers true at once.
79
+ *
80
+ * `false` is a call nobody came to: busy, declined, rung out, or a page that never joined. The
81
+ * session closes for one of those, or a realtime model sits billing against an empty line.
58
82
  *
59
- * `false` is a call that was never answered: busy, declined, or rung out. The session must not
60
- * open at all for one of those, or a realtime model sits billing by the minute against a
61
- * telephone nobody picked up.
83
+ * `call` names who the call is for, so a lane can ask them to join. A lane that places its own
84
+ * call ignores it.
62
85
  */
63
- whenConnected?(conversationId: string): Promise<boolean>;
86
+ whenConnected?(conversationId: string, call?: { userId?: string; workflowId?: string }): Promise<boolean>;
64
87
 
65
88
  /**
66
89
  * END THE CALL AT THE FAR END. Not closing our socket to the vendor: that stops the MODEL,
@@ -59,6 +59,13 @@ import { makeAudioBuffer, resamplePcm16, mulawToPcm16, CLIENT_RATE, type AudioLa
59
59
  const HANG_UP_DRAIN_MS = 1500;
60
60
 
61
61
  const IDLE_TIMEOUT_MS = 30_000;
62
+ /**
63
+ * NOBODY HAS SPOKEN FOR A MINUTE: the call ends (docs/VOICE_STREAMING_GUIDE.md, "Every Call
64
+ * Ends"). The idle timer above cannot catch this: the page's microphone streams frames the whole
65
+ * time, spoken into or not, so a call left open on a quiet page was billed until MAX. Only
66
+ * speech holds this one (`control.speech`), and so does work in progress away from the call.
67
+ */
68
+ const SILENCE_TIMEOUT_MS = 60_000;
62
69
  const MAX_SESSION_MS = 30 * 60_000;
63
70
 
64
71
  import { offerDiscovered, parseMaybeJson, recordToolTrace, type ToolBridge } from "../tools/toolloop.js";
@@ -73,7 +80,9 @@ async function message(spec: unknown, ctx: RunContext, extra: Record<string, unk
73
80
 
74
81
  export interface DuplexResult {
75
82
  /** Why the session ended, for the caller to record. */
76
- reason: "closed" | "ended" | "error" | "idle" | "maxDuration" | "notConnected";
83
+ /** `replaced`: another session claimed this conversation (AudioLane.claim). `silence`: nobody
84
+ * spoke for SILENCE_TIMEOUT_MS. `stopped`: the run was stopped (a page that left). */
85
+ reason: "closed" | "ended" | "error" | "idle" | "maxDuration" | "notConnected" | "replaced" | "silence" | "stopped";
77
86
  error?: string;
78
87
  }
79
88
 
@@ -156,13 +165,14 @@ export async function runDuplexSession(opts: {
156
165
  * generated early and flushes it the instant the call connects (twilioAudio.ts). So the only
157
166
  * thing that still waits for the answer is the sound itself.
158
167
  *
159
- * The browser lane declares no `whenConnected` at all, because the page dialled US and its
160
- * socket is already there. Nothing below changes for it.
168
+ * The browser lane is the same: a page that only started a workflow is asked to join, and the
169
+ * lane holds the greeting until its audio socket arrives. A page that dialled us is already
170
+ * there and answers at once.
161
171
  */
162
172
  let answered: Promise<boolean> | null = null;
163
173
  if (lane?.whenConnected) {
164
174
  lane.startAudioSession?.(conversationId, conversationId);
165
- answered = lane.whenConnected(conversationId);
175
+ answered = lane.whenConnected(conversationId, { userId: ctx.scope?.userId, workflowId: ctx.scope?.workflowId });
166
176
  }
167
177
 
168
178
  const ws = new WebSocket(url, { headers });
@@ -231,6 +241,32 @@ export async function runDuplexSession(opts: {
231
241
  /** Set while the session is open and takes `CONTINUE` inputs; cleared when it ends. */
232
242
  let unregisterLive: (() => void) | undefined;
233
243
 
244
+ /**
245
+ * THE PERSON'S TURN AND THE BACKEND'S WORK, said to the client once per change. A vendor with
246
+ * no start or end event of its own marks them with an event that repeats (GPT-Live's
247
+ * transcript deltas, one per word), and forwarding every repeat sent the client the same
248
+ * state dozens of times a second. A state that names a tool always goes, because the name is
249
+ * new information.
250
+ */
251
+ const lastState: { user?: string; tool?: string } = {};
252
+ const sendState = (group: "user" | "tool", state: string, metadata?: Record<string, unknown>) => {
253
+ // Work starting or finishing away from the call is the call being alive.
254
+ if (group === "tool") heard();
255
+ if (lastState[group] === state && !metadata) return;
256
+ lastState[group] = state;
257
+ lane?.sendControl(conversationId, { type: "AUDIO_STATE", state, ...(metadata ? { metadata } : {}) });
258
+ };
259
+
260
+ /**
261
+ * SOMEONE SPOKE: the events that mean it, the node's own list or the person's speech and the
262
+ * assistant's audio. Restarts the silence timer; set once the session is open.
263
+ */
264
+ const speechEvents: string[] = audio.control?.speech ?? [audio.control?.userSpeaking, outSpec?.match].filter(Boolean);
265
+ let heard: () => void = () => {};
266
+
267
+ /** This session's claim on its conversation's audio lane; let go when it ends. */
268
+ let release: (() => void) | undefined;
269
+
234
270
  const send = (payload: unknown) => {
235
271
  if (ws.readyState !== WebSocket.OPEN) return;
236
272
  ws.send(JSON.stringify(payload));
@@ -263,6 +299,12 @@ export async function runDuplexSession(opts: {
263
299
  void shutdown("maxDuration").then(() => finish({ reason: "maxDuration" }));
264
300
  }, MAX_SESSION_MS);
265
301
 
302
+ // THE RUN WAS STOPPED (a page that left, a reaped run): the call ends with it, closed the
303
+ // way a hang-up closes it. Without this a stopped run left the vendor billing until silence.
304
+ const stop = () => void shutdown("stopped").then(() => finish({ reason: "stopped" }));
305
+ if (ctx.abort?.aborted) stop();
306
+ else ctx.abort?.addEventListener("abort", stop, { once: true });
307
+
266
308
  /**
267
309
  * Restarted by every inbound vendor frame AND by every outbound mic frame, so it only fires
268
310
  * when BOTH sides have gone quiet. The vendor is legitimately silent while the person speaks
@@ -278,6 +320,17 @@ export async function runDuplexSession(opts: {
278
320
  }, IDLE_TIMEOUT_MS);
279
321
  };
280
322
 
323
+ // SILENCE (SILENCE_TIMEOUT_MS). A tool or hand-off still running is not silence: the timer
324
+ // waits another round for it, so a slow answer is never cut off.
325
+ let silenceTimer: ReturnType<typeof setTimeout> | undefined;
326
+ heard = () => {
327
+ if (silenceTimer) clearTimeout(silenceTimer);
328
+ silenceTimer = setTimeout(() => {
329
+ if (lastState.tool === "TOOL_USE") return heard();
330
+ void shutdown("silence").then(() => finish({ reason: "silence" }));
331
+ }, SILENCE_TIMEOUT_MS);
332
+ };
333
+
281
334
  /**
282
335
  * THE ONE PLACE A SESSION ENDS, and now the one place a CALL ends with it.
283
336
  *
@@ -292,7 +345,18 @@ export async function runDuplexSession(opts: {
292
345
  if (settled) return;
293
346
  settled = true;
294
347
  unregisterLive?.();
348
+ release?.();
295
349
  if (idleTimer) clearTimeout(idleTimer);
350
+ if (silenceTimer) clearTimeout(silenceTimer);
351
+ // THE PAGE IS TOLD, so its microphone stops and its avatar goes idle. Not when replaced:
352
+ // the next session is already on the call, and the call goes on.
353
+ if (result.reason !== "replaced") {
354
+ try {
355
+ lane?.sendControl(conversationId, { type: "AUDIO_STATE", state: "SESSION_ENDED" });
356
+ } catch {
357
+ /* a lane already gone has nobody to tell */
358
+ }
359
+ }
296
360
  if (drainTimer) clearTimeout(drainTimer);
297
361
  clearTimeout(maxTimer);
298
362
  try {
@@ -342,6 +406,8 @@ export async function runDuplexSession(opts: {
342
406
 
343
407
  ws.on("open", async () => {
344
408
  touch();
409
+ // The silence clock starts with the call: a call nobody ever speaks on still ends.
410
+ heard();
345
411
  try {
346
412
  /**
347
413
  * THE TOOLS THE CALL OPENS WITH, discovered before the handshake goes out because the
@@ -400,6 +466,8 @@ export async function runDuplexSession(opts: {
400
466
  unregisterLive = registerLiveInput(key, async (handle, value) => {
401
467
  if (ws.readyState !== WebSocket.OPEN) return false;
402
468
  touch();
469
+ // The task moving on (a step handed back) is the call being worked.
470
+ heard();
403
471
  const scope = { input: { handle, value }, seen, said };
404
472
  let sent = 0;
405
473
  for (const rule of call.send ?? []) {
@@ -428,44 +496,54 @@ export async function runDuplexSession(opts: {
428
496
  // lane twice would place a second call.
429
497
  if (!lane.whenConnected) lane.startAudioSession?.(conversationId, conversationId);
430
498
 
431
- lane.setAudioDataHandler?.(async (sessionId, data) => {
432
- if (sessionId !== conversationId || ws.readyState !== WebSocket.OPEN) return;
433
- touch();
434
- const raw = Buffer.from(data);
435
- const rate = inSpec.resampleTo ?? CLIENT_RATE;
436
-
437
- // NOTHING TO DO: the vendor asked for this lane's own codec (see `passthrough`
438
- // above), so these are the bytes to send. The rate travels as the LANE's, because
439
- // that is what the samples actually are.
440
- if (passthrough) {
499
+ // THE CONVERSATION IS THIS SESSION'S NOW (AudioLane.claim): its microphone and its
500
+ // control reach this session only, and a session that held it before is told to close.
501
+ release = lane.claim?.(conversationId, {
502
+ audio: async (data) => {
503
+ if (ws.readyState !== WebSocket.OPEN) return;
504
+ touch();
505
+ const raw = Buffer.from(data);
506
+ const rate = inSpec.resampleTo ?? CLIENT_RATE;
507
+
508
+ // NOTHING TO DO: the vendor asked for this lane's own codec (see `passthrough`
509
+ // above), so these are the bytes to send. The rate travels as the LANE's, because
510
+ // that is what the samples actually are.
511
+ if (passthrough) {
512
+ send(
513
+ await message(inSpec.message, ctx, {
514
+ audio: { base64: raw.toString("base64"), bytes: raw.length, rate: lane.rate ?? CLIENT_RATE },
515
+ }),
516
+ );
517
+ return;
518
+ }
519
+ /**
520
+ * DECODE BEFORE RESAMPLING. A mu-law byte is one sample, not half of one, so
521
+ * interpolating the raw bytes would treat pairs of samples as single ones and
522
+ * produce half-speed noise. And mu-law is logarithmic, so even the arithmetic
523
+ * would be wrong: the midpoint of two bytes is not the midpoint of two sounds.
524
+ *
525
+ * The browser lane declares neither, so both lines are identity for it.
526
+ */
527
+ const decoded = lane.codec === "mulaw" ? mulawToPcm16(raw) : raw;
528
+ const pcm = resamplePcm16(decoded, lane.rate ?? CLIENT_RATE, rate);
441
529
  send(
442
530
  await message(inSpec.message, ctx, {
443
- audio: { base64: raw.toString("base64"), bytes: raw.length, rate: lane.rate ?? CLIENT_RATE },
531
+ audio: { base64: pcm.toString("base64"), bytes: pcm.length, rate },
444
532
  }),
445
533
  );
446
- return;
447
- }
448
- /**
449
- * DECODE BEFORE RESAMPLING. A mu-law byte is one sample, not half of one, so
450
- * interpolating the raw bytes would treat pairs of samples as single ones and
451
- * produce half-speed noise. And mu-law is logarithmic, so even the arithmetic
452
- * would be wrong: the midpoint of two bytes is not the midpoint of two sounds.
453
- *
454
- * The browser lane declares neither, so both lines are identity for it.
455
- */
456
- const decoded = lane.codec === "mulaw" ? mulawToPcm16(raw) : raw;
457
- const pcm = resamplePcm16(decoded, lane.rate ?? CLIENT_RATE, rate);
458
- send(
459
- await message(inSpec.message, ctx, {
460
- audio: { base64: pcm.toString("base64"), bytes: pcm.length, rate },
461
- }),
462
- );
463
- });
464
-
465
- lane.setControlMessageHandler?.(async (sessionId, msg) => {
466
- if (sessionId !== conversationId) return;
467
- const endOn: string[] = audio.control?.endOn ?? [];
468
- if (endOn.includes(String(msg?.type))) finish({ reason: await shutdown("ended") ?? "ended" });
534
+ },
535
+ control: async (msg) => {
536
+ const endOn: string[] = audio.control?.endOn ?? [];
537
+ if (endOn.includes(String(msg?.type))) finish({ reason: await shutdown("ended") ?? "ended" });
538
+ },
539
+ // ANOTHER SESSION TOOK THIS CONVERSATION: the page moved to a task another voice works.
540
+ // Closed the way a hang-up closes it, through the `close` list. What it had not yet
541
+ // played is thrown away, not flushed: the person has moved on, and the next voice
542
+ // must not be talked over by the last one.
543
+ replaced: () => {
544
+ buffer?.discard();
545
+ void shutdown("replaced").then(() => finish({ reason: "replaced" }));
546
+ },
469
547
  });
470
548
  }
471
549
  } catch (err: any) {
@@ -484,6 +562,7 @@ export async function runDuplexSession(opts: {
484
562
  return; // a frame that is not JSON is not something a manifest can describe
485
563
  }
486
564
  const type = String(event?.type ?? "");
565
+ if (speechEvents.includes(type)) heard();
487
566
 
488
567
  // A REJECTED SESSION MUST SAY WHY. Every vendor this runtime speaks names its complaint in
489
568
  // an `error` event and then closes; without this line a refused handshake looks exactly like
@@ -592,17 +671,17 @@ export async function runDuplexSession(opts: {
592
671
  // Separate from `interruptOn` even when both name the same event: this tells the client the
593
672
  // person is talking, that one throws away the audio they talked over.
594
673
  if (audio.control?.userSpeaking && type === audio.control.userSpeaking) {
595
- lane?.sendControl(conversationId, { type: "AUDIO_STATE", state: "USER_SPEECH_STARTED" });
674
+ sendState("user", "USER_SPEECH_STARTED");
596
675
  }
597
676
  if (audio.control?.userStopped && type === audio.control.userStopped) {
598
- lane?.sendControl(conversationId, { type: "AUDIO_STATE", state: "USER_SPEECH_ENDED" });
677
+ sendState("user", "USER_SPEECH_ENDED");
599
678
  }
600
679
  // Work done off the call (a vendor's hand-off to a backend), for the call state's "thinking".
601
680
  if (audio.control?.toolUse && type === audio.control.toolUse) {
602
- lane?.sendControl(conversationId, { type: "AUDIO_STATE", state: "TOOL_USE" });
681
+ sendState("tool", "TOOL_USE");
603
682
  }
604
683
  if (audio.control?.toolUseCompleted && type === audio.control.toolUseCompleted) {
605
- lane?.sendControl(conversationId, { type: "AUDIO_STATE", state: "TOOL_USE_COMPLETED" });
684
+ sendState("tool", "TOOL_USE_COMPLETED");
606
685
  }
607
686
 
608
687
  /**
@@ -670,11 +749,7 @@ export async function runDuplexSession(opts: {
670
749
  let output: string;
671
750
  // A lookup takes 8-9s, and silence on a call reads as a dropped line. The client,
672
751
  // the lookup-indicator and the docs all had this wired already; nothing ever sent it.
673
- lane?.sendControl(conversationId, {
674
- type: "AUDIO_STATE",
675
- state: "TOOL_USE",
676
- metadata: { toolName: name },
677
- });
752
+ sendState("tool", "TOOL_USE", { toolName: name });
678
753
  // Started BEFORE the call, so the timeline bar measures the wait the caller
679
754
  // actually sat through, not the bookkeeping after it.
680
755
  const toolStart = Date.now();
@@ -758,7 +833,7 @@ export async function runDuplexSession(opts: {
758
833
  recordToolTrace(node, ctx, name, parseMaybeJson(rawArgs), output, toolStart, toolOk, tools.kindOf?.(name) ?? "tool");
759
834
  // After the catch, so a tool that THREW still stops the dots. Spinning forever is
760
835
  // worse than no indicator: it promises something still coming.
761
- lane?.sendControl(conversationId, { type: "AUDIO_STATE", state: "TOOL_USE_COMPLETED" });
836
+ sendState("tool", "TOOL_USE_COMPLETED");
762
837
  // PARSED for the connector lane only (readable mcpResult, same as the HTTP
763
838
  // loop); the model message two lines down keeps the raw `output` string —
764
839
  // the vendor wire requires text.
@@ -52,7 +52,7 @@ export { performDoc, MUTATING_DOC_OPS, docKeyFor } from "./docstore/index.js";
52
52
  export { runDuplexSession } from "./duplex/session.js";
53
53
  // The audio lane is its own module: session.ts owns the VENDOR socket, audioLane.ts owns the bytes
54
54
  // and the browser. Two sockets, two files.
55
- export { resamplePcm16, mulawToPcm16, pcm16ToMulaw, makeAudioBuffer, CLIENT_RATE, type AudioLane } from "./duplex/audioLane.js";
55
+ export { resamplePcm16, mulawToPcm16, pcm16ToMulaw, makeAudioBuffer, CLIENT_RATE, type AudioLane, type LaneSession } from "./duplex/audioLane.js";
56
56
  export { performApi, type RunResult } from "./channels/workflow.js";
57
57
  export { runFinal } from "./channels/final.js";
58
58
  export { performService } from "./channels/service.js";
@@ -8,10 +8,14 @@
8
8
  */
9
9
 
10
10
  import { clientTransport } from "./clientTransport.js";
11
+ import type { LaneSession } from "../manifests/runtime/duplex/audioLane.js";
12
+
13
+ /** How long a call the workflow started waits for its page to join before nobody is there. */
14
+ const JOIN_WAIT_MS = 15_000;
11
15
 
12
16
  export interface AudioWebSocketManager {
13
- setAudioDataHandler(handler: (conversationId: string, audioData: ArrayBuffer, metadata?: any) => Promise<void>): void;
14
- setControlMessageHandler(handler: (conversationId: string, message: any) => Promise<void>): void;
17
+ claim(conversationId: string, session: LaneSession): () => void;
18
+ whenConnected(conversationId: string, call?: { userId?: string; workflowId?: string }): Promise<boolean>;
15
19
  startAudioSession(conversationId: string, novaSessionId?: string): void;
16
20
  endAudioSession(conversationId: string): void;
17
21
  isConnected(conversationId: string): boolean;
@@ -23,26 +27,86 @@ class DistributedAudioManager implements AudioWebSocketManager {
23
27
  private audioSessions: Map<string, { novaSessionId?: string; active: boolean }> = new Map();
24
28
 
25
29
  /**
26
- * Set audio data handler (called by WebSocketAudioSubscriber)
30
+ * THE LIVE SESSION OF EACH CONVERSATION, one each (docs/VOICE_STREAMING_GUIDE.md, "One Live
31
+ * Session Per Conversation"). The transport has ONE audio handler and ONE control handler for
32
+ * the whole process, so they are installed once, here, and hand each frame and each control
33
+ * message to the session that holds that conversation and to no other.
34
+ *
35
+ * Installing a session's own handler into the transport instead, as this did, meant the
36
+ * newest session anywhere on the server took every conversation's microphone: the one before
37
+ * went deaf, never heard `stop`, and idled out, and two visitors calling at once broke each
38
+ * other.
39
+ */
40
+ private live = new Map<string, LaneSession>();
41
+
42
+ /**
43
+ * CALLS WAITING FOR THEIR PAGE (docs/VOICE_STREAMING_GUIDE.md, "Call Initialization"). What the
44
+ * session sends meanwhile, its greeting first, is held in order and played once the page's
45
+ * audio socket arrives, so nothing it says is spoken to nobody.
27
46
  */
28
- setAudioDataHandler(
29
- handler: (conversationId: string, audioData: ArrayBuffer, metadata?: any) => Promise<void>
30
- ): void {
47
+ private joining = new Map<string, { held: Array<() => void>; answer: (ok: boolean) => void }>();
48
+
49
+ /** Conversations whose microphone has been reported, so each is said once, not per frame. */
50
+ private heard = new Map<string, "session" | "nobody">();
51
+
52
+ private install(): void {
31
53
  clientTransport().setAudioDataHandler(async (conversationId, audioData) => {
32
- const session = this.audioSessions.get(conversationId);
33
- if (session?.active) {
34
- await handler(conversationId, audioData, { novaSessionId: session.novaSessionId });
54
+ const session = this.live.get(conversationId);
55
+ // THE MICROPHONE, SAID ONCE: where a conversation's frames first go. A frame nobody holds
56
+ // is dropped silently otherwise, and heard only as an assistant who never listens.
57
+ const now = session ? "session" : "nobody";
58
+ if (this.heard.get(conversationId) !== now) {
59
+ this.heard.set(conversationId, now);
60
+ console.log(`[DistributedAudioManager] ${conversationId}: microphone frames reach ${now === "session" ? "the live voice session" : "NO session (dropped)"}`);
35
61
  }
62
+ await session?.audio(audioData);
63
+ });
64
+ clientTransport().setAudioControlHandler(async (conversationId, message) => {
65
+ await this.live.get(conversationId)?.control(message);
36
66
  });
37
- // Wiring, not news.
67
+ clientTransport().setAudioSocketHandler((conversationId) => this.joining.get(conversationId)?.answer(true));
38
68
  }
39
69
 
40
70
  /**
41
- * Set control message handler (called by WebSocketAudioSubscriber)
71
+ * IS THE PAGE THERE? A page that dialled us already has its audio socket. One that only started
72
+ * a workflow is asked to join on the stream it already holds, and has JOIN_WAIT_MS to do it.
42
73
  */
43
- setControlMessageHandler(handler: (conversationId: string, message: any) => Promise<void>): void {
44
- clientTransport().setAudioControlHandler(handler);
45
- // Wiring, not news.
74
+ whenConnected(conversationId: string, call?: { userId?: string; workflowId?: string }): Promise<boolean> {
75
+ this.install();
76
+ if (clientTransport().hasAudioSocket(conversationId)) return Promise.resolve(true);
77
+ return new Promise<boolean>((resolve) => {
78
+ const answer = (ok: boolean) => {
79
+ clearTimeout(timer);
80
+ const waiting = this.joining.get(conversationId);
81
+ if (waiting?.answer !== answer) return;
82
+ this.joining.delete(conversationId);
83
+ console.log(`[DistributedAudioManager] ${conversationId}: ${ok ? "the page joined the call" : "the page never joined; the call ends"}`);
84
+ if (ok) for (const send of waiting.held) send();
85
+ resolve(ok);
86
+ };
87
+ const timer = setTimeout(() => answer(false), JOIN_WAIT_MS);
88
+ this.joining.set(conversationId, { held: [], answer });
89
+ const asked = !!call?.userId && clientTransport().pushToClient(call.userId, conversationId, { type: "JOIN_CALL", workflowId: call.workflowId });
90
+ console.log(`[DistributedAudioManager] ${conversationId}: ${asked ? "asked the page to join" : "no page stream to ask; waiting for its audio socket"}`);
91
+ });
92
+ }
93
+
94
+ claim(conversationId: string, session: LaneSession): () => void {
95
+ this.install();
96
+ const before = this.live.get(conversationId);
97
+ this.live.set(conversationId, session);
98
+ console.log(`[DistributedAudioManager] ${conversationId}: a voice session claimed the call`);
99
+ if (before && before !== session) {
100
+ console.log(`[DistributedAudioManager] ${conversationId}: a new voice session took the call; the one before it closes`);
101
+ try {
102
+ before.replaced();
103
+ } catch (err: any) {
104
+ console.error(`[DistributedAudioManager] ${conversationId}: the replaced session did not close: ${err?.message ?? err}`);
105
+ }
106
+ }
107
+ return () => {
108
+ if (this.live.get(conversationId) === session) this.live.delete(conversationId);
109
+ };
46
110
  }
47
111
 
48
112
  /**
@@ -75,6 +139,12 @@ class DistributedAudioManager implements AudioWebSocketManager {
75
139
  * Send audio to client via server
76
140
  */
77
141
  sendAudio(conversationId: string, audioData: Buffer | ArrayBuffer): boolean {
142
+ const waiting = this.joining.get(conversationId);
143
+ if (waiting) {
144
+ const bytes = Buffer.isBuffer(audioData) ? Buffer.from(audioData) : audioData.slice(0);
145
+ waiting.held.push(() => clientTransport().sendAudioToClient(conversationId, bytes));
146
+ return true;
147
+ }
78
148
  return clientTransport().sendAudioToClient(conversationId, audioData);
79
149
  }
80
150
 
@@ -83,6 +153,11 @@ class DistributedAudioManager implements AudioWebSocketManager {
83
153
  */
84
154
  sendControl(conversationId: string, message: Record<string, any>): boolean {
85
155
  if (message.type === "AUDIO_STATE" || message.state) {
156
+ const waiting = this.joining.get(conversationId);
157
+ if (waiting) {
158
+ waiting.held.push(() => clientTransport().sendAudioStateToClient(conversationId, message.state || message.type, message.metadata));
159
+ return true;
160
+ }
86
161
  return clientTransport().sendAudioStateToClient(conversationId, message.state || message.type, message.metadata);
87
162
  }
88
163
  return false;
@@ -38,6 +38,12 @@ export interface ClientTransport {
38
38
 
39
39
  /** Register the handler for audio control messages from a client. */
40
40
  setAudioControlHandler(handler: (conversationId: string, message: Record<string, unknown>) => Promise<void>): void;
41
+
42
+ /** Is a client's audio socket on this conversation, and not one a call has already ended on? */
43
+ hasAudioSocket(conversationId: string): boolean;
44
+
45
+ /** Register the handler told each time a client's audio socket arrives for a conversation. */
46
+ setAudioSocketHandler(handler: (conversationId: string) => void): void;
41
47
  }
42
48
 
43
49
  /**
@@ -50,6 +56,8 @@ const NONE: ClientTransport = {
50
56
  sendAudioStateToClient: () => false,
51
57
  setAudioDataHandler: () => {},
52
58
  setAudioControlHandler: () => {},
59
+ hasAudioSocket: () => false,
60
+ setAudioSocketHandler: () => {},
53
61
  };
54
62
 
55
63
  let transport: ClientTransport = NONE;
@@ -13,7 +13,7 @@
13
13
  * had no role check at all, so "who is an administrator" had two answers and one of them was
14
14
  * "nobody". One map, imported twice.
15
15
  *
16
- * ROLES ARE THE TOKEN'S SHAPE; PERMISSIONS ARE THE CODE'S SHAPE. Auth0 issues this
16
+ * ROLES ARE THE TOKEN'S SHAPE; PERMISSIONS ARE THE CODE'S SHAPE. The IdP issues this
17
17
  * platform `roles: ["admin"]` / `["author"]`, and the tokens carry no permissions claim we
18
18
  * can rely on. So `ROLE_GRANTS` is the bridge, and it is deliberately the ONLY place the
19
19
  * two vocabularies meet: one map, read once, so switching to real permission claims later
@@ -59,7 +59,7 @@ export const PERMISSIONS = [
59
59
  export type Permission = (typeof PERMISSIONS)[number];
60
60
 
61
61
  /**
62
- * WHICH ROLE GRANTS WHAT. The one bridge between Auth0's vocabulary and this code's.
62
+ * WHICH ROLE GRANTS WHAT. The one bridge between the IdP's vocabulary and this code's.
63
63
  *
64
64
  * `admin` is `author` plus the secrets. That is the entire difference between them, stated
65
65
  * as data rather than described in a comment somewhere, so it cannot drift.