hci-atrium 0.3.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +75 -24
- package/dist/atrium.d.ts +126 -42
- package/dist/atrium.d.ts.map +1 -1
- package/dist/atrium.js +192 -87
- package/dist/atrium.js.map +1 -1
- package/dist/dash.d.ts +3 -3
- package/dist/dash.d.ts.map +1 -1
- package/dist/dash.js +1 -2
- package/dist/dash.js.map +1 -1
- package/dist/env.d.ts +14 -0
- package/dist/env.d.ts.map +1 -1
- package/dist/env.js +22 -0
- package/dist/env.js.map +1 -1
- package/dist/index.d.ts +4 -4
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +11 -3
- package/dist/index.js.map +1 -1
- package/dist/models.d.ts +84 -31
- package/dist/models.d.ts.map +1 -1
- package/dist/models.js +97 -25
- package/dist/models.js.map +1 -1
- package/dist/player/play-command.d.ts +6 -6
- package/dist/player/play-command.d.ts.map +1 -1
- package/dist/player/play-intent.d.ts.map +1 -1
- package/dist/player/play-intent.js +2 -1
- package/dist/player/play-intent.js.map +1 -1
- package/dist/player/runtime.d.ts +7 -3
- package/dist/player/runtime.d.ts.map +1 -1
- package/dist/player/runtime.js.map +1 -1
- package/dist/react/use-room-connection.d.ts +4 -1
- package/dist/react/use-room-connection.d.ts.map +1 -1
- package/dist/react/use-room-connection.js +5 -1
- package/dist/react/use-room-connection.js.map +1 -1
- package/dist/rooms/index.d.ts +3 -1
- package/dist/rooms/index.d.ts.map +1 -1
- package/dist/rooms/index.js +4 -3
- package/dist/rooms/index.js.map +1 -1
- package/dist/rooms/offer-transcription.d.ts +1 -1
- package/dist/rooms/offer-transcription.js +1 -1
- package/dist/rooms/owned-room.d.ts +251 -0
- package/dist/rooms/owned-room.d.ts.map +1 -0
- package/dist/rooms/owned-room.js +844 -0
- package/dist/rooms/owned-room.js.map +1 -0
- package/dist/rooms/recorder.d.ts +1 -1
- package/dist/rooms/recorder.js +1 -1
- package/dist/rooms/room.d.ts +82 -278
- package/dist/rooms/room.d.ts.map +1 -1
- package/dist/rooms/room.js +60 -854
- package/dist/rooms/room.js.map +1 -1
- package/dist/rooms/standing-source.d.ts +1 -1
- package/dist/rooms/standing-source.js +1 -1
- package/dist/rooms/transcriber.d.ts +3 -3
- package/dist/rooms/transcriber.d.ts.map +1 -1
- package/dist/rooms/transcriber.js +2 -2
- package/dist/rooms/transport.d.ts +1 -1
- package/dist/rooms/transport.js +2 -2
- package/dist/shaping/assets.d.ts +13 -0
- package/dist/shaping/assets.d.ts.map +1 -1
- package/dist/shaping/assets.js +27 -0
- package/dist/shaping/assets.js.map +1 -1
- package/dist/shaping/chat.d.ts +3 -1
- package/dist/shaping/chat.d.ts.map +1 -1
- package/dist/shaping/chat.js +4 -2
- package/dist/shaping/chat.js.map +1 -1
- package/dist/shaping/rooms.d.ts +78 -1
- package/dist/shaping/rooms.d.ts.map +1 -1
- package/dist/shaping/rooms.js +58 -6
- package/dist/shaping/rooms.js.map +1 -1
- package/dist/shaping/sources.d.ts +7 -3
- package/dist/shaping/sources.d.ts.map +1 -1
- package/dist/shaping/sources.js +9 -10
- package/dist/shaping/sources.js.map +1 -1
- package/dist/transcribe-stream.d.ts +1 -1
- package/dist/transcribe-stream.js +1 -1
- package/dist/transcribe.d.ts +2 -2
- package/dist/transcribe.js +2 -2
- package/package.json +11 -2
package/dist/rooms/room.js
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* The live room handle — pure transport over the room WebSocket
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
2
|
+
* The live room handle — pure transport over the room WebSocket, and the **participant** half of the
|
|
3
|
+
* room surface. Returned by the loginless {@link ./index.joinRoom} and by ``client.rooms.join(code)``;
|
|
4
|
+
* the host-side half lives on {@link ./owned-room.OwnedRoom}, which extends this. Feature-equivalent
|
|
5
|
+
* to the Python SDK's ``Room`` (rooms.py) + ``AsyncRoom`` (aio.py), but JS needs no sync/async split —
|
|
6
|
+
* there is one API: ``room.on(...)`` registers, ``await room.request(...)`` correlates.
|
|
6
7
|
*
|
|
7
8
|
* Transport: everything rides one WebSocket (``/v1/rooms/{id}/ws``) — text signals *and* the
|
|
8
9
|
* ephemeral binary audio stream. That differs from the Python SDK (SSE receive + a REST ``/signal``
|
|
@@ -10,40 +11,26 @@
|
|
|
10
11
|
* that carries binary. The REST ``/signal`` wire body is still the canonical cross-SDK shape
|
|
11
12
|
* ({@link ../shaping/rooms.buildRoomSignal}); the fields match field-for-field.
|
|
12
13
|
*
|
|
13
|
-
* Correlation (mirror of ``_RoomDispatcher``): dispatch starts with {@link connect}. Each inbound
|
|
14
|
+
* Correlation (mirror of ``_RoomDispatcher``): dispatch starts with {@link Room.connect}. Each inbound
|
|
14
15
|
* signal routes to the handler registered for its ``type``; a handler's return value is sent back as
|
|
15
|
-
* the response when the signal was a {@link request} (carried an ``ackId``) — so a responder
|
|
16
|
-
* no correlation plumbing, it just returns. Pending
|
|
17
|
-
*
|
|
18
|
-
* Durable & reactive surfaces (wave 2): {@link Room.record} bridges the room into a session over an
|
|
19
|
-
* SSE-observe {@link ./recorder.Recorder} (every WS send is ephemeral, so durable capture rides the
|
|
20
|
-
* observe stream); {@link Room.audioVectors} embeds the live snippet stream and {@link Room.audioScores}
|
|
21
|
-
* fuses it against named anchors for a {@link ../decider.Decider}. Still deferred to later waves: the
|
|
22
|
-
* ``offer_embeds`` / ``offer_chat`` broker protocol, and ``offerAudioCapture`` (which will bind to the
|
|
23
|
-
* active recorder via {@link Room.activeRecorders} / {@link Room.onRecordersChanged}).
|
|
16
|
+
* the response when the signal was a {@link Room.request} (carried an ``ackId``) — so a responder
|
|
17
|
+
* writes no correlation plumbing, it just returns. Pending requests resolve by ``replyTo``.
|
|
24
18
|
*/
|
|
25
|
-
import {
|
|
26
|
-
import {
|
|
27
|
-
import {
|
|
28
|
-
import {
|
|
29
|
-
import { buildLayerClearBody, buildLayerEffectsBody, buildLayerPositionBody, buildPlayBody, buildRoomRotateCode, buildRoomSignal, buildShowBody, } from "../shaping/rooms.js";
|
|
30
|
-
import { RoomTrace, summarizePlay, traced, traceEnvelope, traceStateType, } from "../trace.js";
|
|
31
|
-
import { BrokerError, CAPTURE_OFFER_ROLE, CHAT_BROKER_ROLE, DEFAULT_CHAT_MAX_TOKENS, DEFAULT_MAX_ANCHORS, EMBED_BROKER_ROLE, parseAnchorDescriptor, TRANSCRIPTION_OFFER_ROLE, validateChatMessages, } from "./brokers.js";
|
|
19
|
+
import { AtriumError } from "../errors.js";
|
|
20
|
+
import { isBrowser } from "../http.js";
|
|
21
|
+
import { buildLayerClearBody, buildLayerConfigureBody, buildLayerEffectsBody, buildLayerPositionBody, buildPlayBody, buildShowBody, } from "../shaping/rooms.js";
|
|
22
|
+
import { summarizePlay, traced } from "../trace.js";
|
|
32
23
|
import { AUDIO_ROLE, AUDIO_SIGNAL, decodeAudioFrame, } from "./codec.js";
|
|
33
|
-
import { OfferedCapture } from "./offer-capture.js";
|
|
34
|
-
import { OfferedTranscription } from "./offer-transcription.js";
|
|
35
24
|
import { parseParticipant, participantMatches, } from "./participant.js";
|
|
36
|
-
import { Recorder } from "./recorder.js";
|
|
37
25
|
import { AUDIO_SNIPPET_CONSUMER_ROLE, decodeAudioSnippetFrame, } from "./snippet.js";
|
|
38
26
|
import { RoomEventStream, readObserveRoster } from "./sse.js";
|
|
39
|
-
import { TRANSCRIBER_SOURCE, Transcriber } from "./transcriber.js";
|
|
40
27
|
import { RoomSocket, } from "./transport.js";
|
|
41
28
|
const DEFAULT_REQUEST_TIMEOUT_MS = 10_000;
|
|
42
|
-
// A raw audio-consumer socket ({@link Room.audioClips} / {@link
|
|
43
|
-
// frames for a consumer that pulls in its own time; a slow consumer must not grow this
|
|
44
|
-
// so it drops the OLDEST buffered frame past this cap (live audio favours recency —
|
|
45
|
-
// offers' per-source ClipQueue). A prompt consumer never reaches it.
|
|
46
|
-
const RAW_CONSUMER_QUEUE_MAX = 32;
|
|
29
|
+
// A raw audio-consumer socket ({@link Room.audioClips} / {@link OwnedRoom.snippetStream}) buffers
|
|
30
|
+
// decoded frames for a consumer that pulls in its own time; a slow consumer must not grow this
|
|
31
|
+
// without bound, so it drops the OLDEST buffered frame past this cap (live audio favours recency —
|
|
32
|
+
// mirror of the offers' per-source ClipQueue). A prompt consumer never reaches it.
|
|
33
|
+
export const RAW_CONSUMER_QUEUE_MAX = 32;
|
|
47
34
|
function toRoomEvent(sig) {
|
|
48
35
|
return {
|
|
49
36
|
type: sig.type ?? "",
|
|
@@ -67,7 +54,7 @@ export function randomId() {
|
|
|
67
54
|
return Math.random().toString(36).slice(2) + Date.now().toString(36);
|
|
68
55
|
}
|
|
69
56
|
/** Drop one entry from a mutable registry array (a no-op when absent / the list is undefined). */
|
|
70
|
-
function remove(list, item) {
|
|
57
|
+
export function remove(list, item) {
|
|
71
58
|
const at = list?.indexOf(item) ?? -1;
|
|
72
59
|
if (at >= 0)
|
|
73
60
|
list?.splice(at, 1);
|
|
@@ -80,6 +67,16 @@ function openTab(url) {
|
|
|
80
67
|
}
|
|
81
68
|
window.open(url, "_blank", "noopener");
|
|
82
69
|
}
|
|
70
|
+
/**
|
|
71
|
+
* The **participant** room surface — everything a handle can do with nothing but a connect code:
|
|
72
|
+
* connect/close, send/on/onAny/request/onEnd, the presence roster, the Player + Viewer dialects,
|
|
73
|
+
* and the audio-consumption streams. This is what the loginless {@link ./index.joinRoom} returns.
|
|
74
|
+
*
|
|
75
|
+
* The host-side surfaces that genuinely need an account credential — ``record`` / ``transcribe`` /
|
|
76
|
+
* ``audioVectors`` / ``dashboard`` / ``trace`` / the standing offers / ``rotateCode`` / ``delete`` —
|
|
77
|
+
* live on {@link ./owned-room.OwnedRoom}, which ``client.rooms.open/get/list`` returns: a method you
|
|
78
|
+
* cannot call is a method you cannot see.
|
|
79
|
+
*/
|
|
83
80
|
export class Room {
|
|
84
81
|
id;
|
|
85
82
|
connectCode;
|
|
@@ -108,10 +105,9 @@ export class Room {
|
|
|
108
105
|
// the Player/tools `use-audio-signal` consumer connection.
|
|
109
106
|
audioSocket = null;
|
|
110
107
|
layerWatchers = new Set();
|
|
111
|
-
// Durable-capture recorders started by `record(into)
|
|
112
|
-
//
|
|
108
|
+
// Durable-capture recorders started by OwnedRoom's `record(into)` — the base holds them because
|
|
109
|
+
// `teardown` stops whatever is registered.
|
|
113
110
|
recorders = [];
|
|
114
|
-
recorderListeners = new Set();
|
|
115
111
|
// Live transcription brokers started by `transcribe()` — stopped on room close (a side effect,
|
|
116
112
|
// like `record`). Mirror of Python's `AsyncRoom._transcribers`.
|
|
117
113
|
transcribers = [];
|
|
@@ -181,7 +177,7 @@ export class Room {
|
|
|
181
177
|
announce: this.mergedAnnounce(),
|
|
182
178
|
reconnect: this.deps.reconnect,
|
|
183
179
|
probe: mode === "auto",
|
|
184
|
-
|
|
180
|
+
webSocketCtor: this.deps.webSocketCtor,
|
|
185
181
|
onSignal: (sig) => this.handleSignal(sig),
|
|
186
182
|
// Binary audio + the audio text signals ride the dedicated audio socket (below), never this
|
|
187
183
|
// one — so a frame isn't dispatched twice (both sockets are separate room subscribers).
|
|
@@ -272,12 +268,22 @@ export class Room {
|
|
|
272
268
|
this.endListeners.delete(cb);
|
|
273
269
|
};
|
|
274
270
|
}
|
|
275
|
-
/**
|
|
276
|
-
*
|
|
277
|
-
*
|
|
271
|
+
/**
|
|
272
|
+
* Send a live message — fire-and-forget, never stored. ``to`` (a connection id/identity, or a
|
|
273
|
+
* list) targets peers; omit it to broadcast to everyone else. ``replyTo`` answers a peer's
|
|
274
|
+
* {@link request} out-of-band — the correlation an ``on`` handler gets for free by *returning* a
|
|
275
|
+
* value.
|
|
276
|
+
*
|
|
277
|
+
* Both fields are all this carries: the room transport forces ``ephemeral: false`` on everything
|
|
278
|
+
* it accepts inbound, so a signal sent here is **durable** (a {@link OwnedRoom.record} bridge
|
|
279
|
+
* stores it), and a reply correlation needs a registered pending entry, which only
|
|
280
|
+
* {@link request} creates. An ephemeral or awaited-reply signal is {@link request} or, host-side,
|
|
281
|
+
* the REST ``/signal`` POST ({@link ../shaping/rooms.buildRoomSignal}) — what
|
|
282
|
+
* {@link OwnedRoom.dashboard} / {@link OwnedRoom.trace} publish through.
|
|
283
|
+
*/
|
|
278
284
|
async send(type, payload, opts = {}) {
|
|
279
285
|
await this.connect();
|
|
280
|
-
this.socket?.sendSignal({ type, payload, to: opts.to });
|
|
286
|
+
this.socket?.sendSignal({ type, payload, to: opts.to, replyTo: opts.replyTo });
|
|
281
287
|
}
|
|
282
288
|
/** Send a request and await one reply (a peer's handler return). ``to`` targets one peer; omit it
|
|
283
289
|
* to broadcast — first matching reply wins. Rejects if no reply arrives within the timeout. */
|
|
@@ -397,7 +403,7 @@ export class Room {
|
|
|
397
403
|
// Announce audio-consumer demand so an elected producer starts broadcasting (analyzer gate).
|
|
398
404
|
announce: { ...this.deps.announce, role: AUDIO_ROLE.consumer },
|
|
399
405
|
reconnect: this.deps.reconnect,
|
|
400
|
-
|
|
406
|
+
webSocketCtor: this.deps.webSocketCtor,
|
|
401
407
|
onSignal: (sig) => this.dispatchAudio(sig),
|
|
402
408
|
onBinary: (data) => this.handleAudioBinary(data),
|
|
403
409
|
onReady: () => this.audioSocket?.sendSignal({ type: AUDIO_SIGNAL.discover, ackId: `disc-${Date.now()}` }),
|
|
@@ -443,7 +449,7 @@ export class Room {
|
|
|
443
449
|
identity: `audio-clips:${randomId().slice(0, 8)}`,
|
|
444
450
|
announce: { ...this.deps.announce, role: AUDIO_SNIPPET_CONSUMER_ROLE },
|
|
445
451
|
reconnect: this.deps.reconnect,
|
|
446
|
-
|
|
452
|
+
webSocketCtor: this.deps.webSocketCtor,
|
|
447
453
|
onReady: () => {
|
|
448
454
|
// Freeze the source-name lookup at the first subscribe-time roster snapshot (the "joined
|
|
449
455
|
// after subscribe → null name" caveat) — a plain consumer socket has no live roster view.
|
|
@@ -506,229 +512,7 @@ export class Room {
|
|
|
506
512
|
socket.close();
|
|
507
513
|
}
|
|
508
514
|
}
|
|
509
|
-
// ──
|
|
510
|
-
/**
|
|
511
|
-
* Start capturing this room's live signals into a durable ``session`` — the single durable binding
|
|
512
|
-
* between a room (pure transport) and a session (an append-only log). A **side effect**: it starts
|
|
513
|
-
* immediately in the background and returns a live {@link Recorder} handle. Stop it with
|
|
514
|
-
* ``rec.stop()``, or let {@link close} (an ``await using`` room) stop it. It skips ``ephemeral``
|
|
515
|
-
* signals and presence traffic — only durable-worthy signals are recorded, each stamped with
|
|
516
|
-
* ``via`` provenance (``{ room_id, room_name, from, to }``). Mirror of Python's ``room.record``.
|
|
517
|
-
*
|
|
518
|
-
* A dropped stream reconnects with backoff (the gap is lost — rooms have no replay); only a
|
|
519
|
-
* terminal end stops it cleanly with a {@link Recorder.stoppedReason} (a code rotation, or the
|
|
520
|
-
* room being gone / its code rejected — the latter also sets {@link Recorder.error}). Needs an
|
|
521
|
-
* owned room (durable sessions require an account) — throws on a loginless {@link joinRoom} handle.
|
|
522
|
-
*/
|
|
523
|
-
record(into) {
|
|
524
|
-
const rest = this.deps.rest;
|
|
525
|
-
if (!rest) {
|
|
526
|
-
throw new AtriumError("this room handle has no account credential — join(code) can only send/receive; " +
|
|
527
|
-
"record(into) needs an owned room from client.rooms");
|
|
528
|
-
}
|
|
529
|
-
// The `trace.state.recorder` line rides the same change seam, on TRANSITIONS only (the seam also
|
|
530
|
-
// fires on a no-op stop) — mirror of Python's start/stop emissions in the recorder's own task.
|
|
531
|
-
let tracedAlive = false;
|
|
532
|
-
const recorder = new Recorder({
|
|
533
|
-
roomId: this.id,
|
|
534
|
-
code: this.connectCode,
|
|
535
|
-
roomName: this.name,
|
|
536
|
-
baseUrl: this.deps.baseUrl,
|
|
537
|
-
rest,
|
|
538
|
-
reconnect: this.deps.reconnect,
|
|
539
|
-
fetchImpl: this.deps.fetchImpl,
|
|
540
|
-
onChange: () => {
|
|
541
|
-
void this.notifyRecordersChanged();
|
|
542
|
-
if (recorder.alive === tracedAlive)
|
|
543
|
-
return;
|
|
544
|
-
tracedAlive = recorder.alive;
|
|
545
|
-
void this.traceState("recorder", tracedAlive
|
|
546
|
-
? `recording ${into.id}: started`
|
|
547
|
-
: `recording ${into.id}: stopped (${recorder.events} events)`);
|
|
548
|
-
},
|
|
549
|
-
}, into);
|
|
550
|
-
this.recorders.push(recorder);
|
|
551
|
-
recorder.start();
|
|
552
|
-
return recorder;
|
|
553
|
-
}
|
|
554
|
-
/** The recorders currently running on this handle (membership **and** liveness — a stopped one no
|
|
555
|
-
* longer counts) — the source of truth a wave-4 ``offerAudioCapture`` reads for audio-capture
|
|
556
|
-
* availability, and binds its capture to the first active recorder's session. */
|
|
557
|
-
activeRecorders() {
|
|
558
|
-
return this.recorders.filter((r) => r.alive);
|
|
559
|
-
}
|
|
560
|
-
/** Register a listener fired whenever the active-recorder set changes (a {@link record} started, or
|
|
561
|
-
* a recorder stopped/terminated) — the seam a wave-4 ``offerAudioCapture`` binds to, so it can
|
|
562
|
-
* re-derive availability and disengage when the recorder it wrote under is gone. Returns an
|
|
563
|
-
* unsubscribe function. Mirror of Python's ``_notify_recorders_changed`` registration. */
|
|
564
|
-
onRecordersChanged(listener) {
|
|
565
|
-
this.recorderListeners.add(listener);
|
|
566
|
-
return () => {
|
|
567
|
-
this.recorderListeners.delete(listener);
|
|
568
|
-
};
|
|
569
|
-
}
|
|
570
|
-
async notifyRecordersChanged() {
|
|
571
|
-
// Isolate each hook — one misbehaving listener must not starve the others nor throw out of a
|
|
572
|
-
// recorder's teardown path (this fires from a recorder's terminal wind-down).
|
|
573
|
-
for (const listener of [...this.recorderListeners]) {
|
|
574
|
-
try {
|
|
575
|
-
await listener();
|
|
576
|
-
}
|
|
577
|
-
catch (err) {
|
|
578
|
-
console.error("Atrium room: a recorders-changed listener failed:", err);
|
|
579
|
-
}
|
|
580
|
-
}
|
|
581
|
-
}
|
|
582
|
-
// ── Live transcription (the transcript.* broker) ─────────────────────────
|
|
583
|
-
/**
|
|
584
|
-
* Transcribe this room's live audio into ``transcript.*`` signals — the in-room broker. Consumes the
|
|
585
|
-
* room's ``audio.snippet`` stream (announcing the snippet-consumer role, so a Player starts
|
|
586
|
-
* capturing), pushes it through the same engine as {@link ../atrium.Atrium.transcribeLive} on
|
|
587
|
-
* **your** credentials/compute, and publishes back:
|
|
588
|
-
*
|
|
589
|
-
* * ``transcript.partial`` — ``{ text }``, **ephemeral** (a live caption the recorder skips).
|
|
590
|
-
* Wholesale-replaced — render the latest, don't append.
|
|
591
|
-
* * ``transcript.final`` — ``{ text, start, end, seq }``, durable (the {@link record} bridge persists
|
|
592
|
-
* it — no new recording API). ``seq`` is a per-handle counter from 0; the signal's ``source`` is
|
|
593
|
-
* ``"transcriber"``.
|
|
594
|
-
*
|
|
595
|
-
* Any consumer — including a loginless code-joined tool — just does ``room.on("transcript.final")``.
|
|
596
|
-
* Like {@link record}, it's a **side effect**: it starts immediately in the background and is stopped
|
|
597
|
-
* when the room closes, or via ``await handle.stop()`` / ``await using``. Needs an owned room
|
|
598
|
-
* (transcription runs on an account credential) — throws on a loginless {@link joinRoom} handle.
|
|
599
|
-
*/
|
|
600
|
-
transcribe(opts = {}) {
|
|
601
|
-
const client = this.requireClient("transcribe()");
|
|
602
|
-
const transcriber = new Transcriber({
|
|
603
|
-
model: opts.model,
|
|
604
|
-
language: opts.language,
|
|
605
|
-
snippetBytes: (signal) => this.snippetBytesFor(signal),
|
|
606
|
-
engine: (clips, o) => client.transcribeStream(clips, {
|
|
607
|
-
model: o.model,
|
|
608
|
-
language: o.language,
|
|
609
|
-
// The producer ships Opus-in-WebM; the backend decodes each clip. Partials are published
|
|
610
|
-
// as ephemeral live captions, so the engine must surface them.
|
|
611
|
-
format: "webm",
|
|
612
|
-
partials: true,
|
|
613
|
-
skipEmpty: true,
|
|
614
|
-
signal: o.signal,
|
|
615
|
-
webSocketCtor: this.deps.WebSocketCtor,
|
|
616
|
-
}),
|
|
617
|
-
publish: (type, payload, o) => client.request(buildRoomSignal(this.id, {
|
|
618
|
-
type,
|
|
619
|
-
payload,
|
|
620
|
-
source: TRANSCRIBER_SOURCE,
|
|
621
|
-
ephemeral: o.ephemeral,
|
|
622
|
-
})),
|
|
623
|
-
onEnd: (err) => void this.traceState("transcriber", err ? `transcribing: failed — ${err.message}` : "transcribing: stopped"),
|
|
624
|
-
});
|
|
625
|
-
this.transcribers.push(transcriber);
|
|
626
|
-
transcriber.start();
|
|
627
|
-
void this.traceState("transcriber", `transcribing (${opts.model ?? "default"}): started`);
|
|
628
|
-
return transcriber;
|
|
629
|
-
}
|
|
630
|
-
/**
|
|
631
|
-
* Subscribe to the room's live ``audio.snippet`` stream and transcribe it, yielding a
|
|
632
|
-
* {@link ../models.Transcript} per finalized sentence — the speech-to-text sibling of
|
|
633
|
-
* {@link audioVectors}, and the *raw* counterpart of the {@link transcribe} broker (which publishes
|
|
634
|
-
* ``transcript.*`` signals for every peer instead of handing you the stream).
|
|
635
|
-
*
|
|
636
|
-
* Subscribing *is* the demand signal (it announces the ``audio-snippet-consumer`` role, so a
|
|
637
|
-
* producer's own default-off "Share audio clips" toggle actually captures). Break out of the
|
|
638
|
-
* ``for await`` loop — or abort ``signal`` — to end the subscription and close the socket.
|
|
639
|
-
*
|
|
640
|
-
* Snippets run through the same engine as {@link ../atrium.Atrium.transcribeLive}: true streaming
|
|
641
|
-
* (per-sentence finals, plus revisable partials with ``partials``) when the backend's live route is
|
|
642
|
-
* reachable, degrading silently to per-clip batch transcription otherwise (finals only).
|
|
643
|
-
* Transcription runs on *your* credentials/compute — a rejected credential raises rather than
|
|
644
|
-
* falling back, and a loginless {@link joinRoom} handle throws.
|
|
645
|
-
*
|
|
646
|
-
* The two paths differ in one detail (mirror of Python's ``audio_transcripts``): on the **fallback**
|
|
647
|
-
* path each transcript carries its source frame's ``sequence`` / ``window_seconds`` under
|
|
648
|
-
* ``transcript.raw.snippet`` (a cheap ordering/correlation handle) and the clip's format comes from
|
|
649
|
-
* that frame's own MIME type; on the **streaming** path transcripts are per-sentence, so there is no
|
|
650
|
-
* 1:1 snippet correlation and ``raw.snippet`` is omitted (``raw`` carries ``start`` / ``end``).
|
|
651
|
-
*/
|
|
652
|
-
async *audioTranscripts(opts = {}) {
|
|
653
|
-
const client = this.requireClient("audioTranscripts");
|
|
654
|
-
const skipEmpty = opts.skipEmpty ?? true;
|
|
655
|
-
// Own abort, chained to the caller's: a consumer that just `break`s out of the loop must tear the
|
|
656
|
-
// whole chain down, and JS can't cancel the engine's pending feeder `await` the way Python cancels
|
|
657
|
-
// its task — aborting closes the snippet socket, which is what unblocks it (same teardown seam as
|
|
658
|
-
// the `transcribe()` handle's stop()).
|
|
659
|
-
const ctrl = new AbortController();
|
|
660
|
-
const chain = () => ctrl.abort();
|
|
661
|
-
if (opts.signal) {
|
|
662
|
-
if (opts.signal.aborted)
|
|
663
|
-
ctrl.abort();
|
|
664
|
-
else
|
|
665
|
-
opts.signal.addEventListener("abort", chain, { once: true });
|
|
666
|
-
}
|
|
667
|
-
// One shared snippet source: the streaming path draws its bytes, the fallback its frames + their
|
|
668
|
-
// per-frame metadata. Only one of the two ever iterates it (the engine picks a path).
|
|
669
|
-
const frames = this.snippetStream(`audio-transcripts:${randomId().slice(0, 8)}`, ctrl.signal);
|
|
670
|
-
async function* snippetBytes() {
|
|
671
|
-
for await (const frame of frames)
|
|
672
|
-
yield frame.data;
|
|
673
|
-
}
|
|
674
|
-
async function* snippetFallback() {
|
|
675
|
-
for await (const frame of frames) {
|
|
676
|
-
const format = frame.mimeType.split(";")[0]?.split("/").pop();
|
|
677
|
-
let said;
|
|
678
|
-
try {
|
|
679
|
-
said = await client.transcribe(new Blob([frame.data], { type: frame.mimeType }), {
|
|
680
|
-
model: opts.model,
|
|
681
|
-
language: opts.language,
|
|
682
|
-
format,
|
|
683
|
-
});
|
|
684
|
-
}
|
|
685
|
-
catch (err) {
|
|
686
|
-
if (err instanceof APIError && (err.status === 401 || err.status === 403))
|
|
687
|
-
throw err;
|
|
688
|
-
continue; // a frame the server couldn't transcribe — skip, keep the stream alive
|
|
689
|
-
}
|
|
690
|
-
if (!said.text && skipEmpty)
|
|
691
|
-
continue;
|
|
692
|
-
said.raw.snippet = { sequence: frame.sequence, window_seconds: frame.windowSeconds };
|
|
693
|
-
yield said;
|
|
694
|
-
}
|
|
695
|
-
}
|
|
696
|
-
const stream = client.transcribeStream(snippetBytes(), {
|
|
697
|
-
model: opts.model,
|
|
698
|
-
language: opts.language,
|
|
699
|
-
format: "webm", // the producer ships Opus-in-WebM; the backend decodes each clip
|
|
700
|
-
partials: opts.partials ?? false,
|
|
701
|
-
skipEmpty,
|
|
702
|
-
signal: ctrl.signal,
|
|
703
|
-
webSocketCtor: this.deps.WebSocketCtor,
|
|
704
|
-
fallback: snippetFallback,
|
|
705
|
-
});
|
|
706
|
-
const engine = stream[Symbol.asyncIterator]();
|
|
707
|
-
// Pulled by hand (not `yield*`) so teardown runs in the order that actually unwinds: abort the
|
|
708
|
-
// socket FIRST, then wind the engine down — `yield*` would forward the consumer's `return()`
|
|
709
|
-
// into the engine while its feeder is still parked on a snippet that will never arrive.
|
|
710
|
-
try {
|
|
711
|
-
for (;;) {
|
|
712
|
-
const next = await engine.next();
|
|
713
|
-
if (next.done)
|
|
714
|
-
break;
|
|
715
|
-
yield next.value;
|
|
716
|
-
}
|
|
717
|
-
}
|
|
718
|
-
finally {
|
|
719
|
-
opts.signal?.removeEventListener("abort", chain);
|
|
720
|
-
ctrl.abort();
|
|
721
|
-
await engine.return?.().catch(() => { });
|
|
722
|
-
}
|
|
723
|
-
}
|
|
724
|
-
// The raw snippet bytes for the transcription broker — a dedicated consumer socket, ended when the
|
|
725
|
-
// handle's stop() aborts the signal (which closes the socket).
|
|
726
|
-
async *snippetBytesFor(signal) {
|
|
727
|
-
for await (const frame of this.snippetStream(`transcriber:${randomId().slice(0, 8)}`, signal)) {
|
|
728
|
-
yield frame.data;
|
|
729
|
-
}
|
|
730
|
-
}
|
|
731
|
-
// ── Standing-service offers + capability brokers ─────────────────────────
|
|
515
|
+
// ── Presence announce (the broker/offer role seam) ───────────────────────
|
|
732
516
|
/** Merge the registered broker/offer roles into the presence announce (mirror of Python's
|
|
733
517
|
* ``_events_params``) — so registering a broker before ``connect()`` rides the first connection. */
|
|
734
518
|
mergedAnnounce() {
|
|
@@ -742,577 +526,12 @@ export class Room {
|
|
|
742
526
|
merged.roles = roles;
|
|
743
527
|
return merged;
|
|
744
528
|
}
|
|
529
|
+
/** Register a discoverable role for the presence announce — what an {@link OwnedRoom}'s brokers
|
|
530
|
+
* and standing offers advertise. */
|
|
745
531
|
announceRole(role) {
|
|
746
532
|
if (!this.brokerRoles.includes(role))
|
|
747
533
|
this.brokerRoles.push(role);
|
|
748
534
|
}
|
|
749
|
-
requirePreConnect() {
|
|
750
|
-
if (this.connectionId !== null) {
|
|
751
|
-
throw new AtriumError("register offer_*/offerEmbeds/offerChat before connect() — the broker role is only " +
|
|
752
|
-
"advertised when the connection opens");
|
|
753
|
-
}
|
|
754
|
-
}
|
|
755
|
-
/** The single capability-injection guard — the authed client slice (embed / chat / compare /
|
|
756
|
-
* transcribe) an owned room carries and a loginless ``join(code)`` handle lacks. Always throws
|
|
757
|
-
* {@link AtriumError} (never a bare ``Error``), so every SDK refusal is catchable under the one
|
|
758
|
-
* hierarchy. ``feature`` names the surface for the message. */
|
|
759
|
-
requireClient(feature) {
|
|
760
|
-
if (!this.deps.client) {
|
|
761
|
-
throw new AtriumError(`${feature} needs an owned room from client.rooms (open/get/join) — this handle has no ` +
|
|
762
|
-
"account credential (a loginless joinRoom can only send/receive)");
|
|
763
|
-
}
|
|
764
|
-
return this.deps.client;
|
|
765
|
-
}
|
|
766
|
-
offerDeps(client) {
|
|
767
|
-
return {
|
|
768
|
-
roomId: this.id,
|
|
769
|
-
getCode: () => this.connectCode,
|
|
770
|
-
baseUrl: this.deps.baseUrl,
|
|
771
|
-
reconnect: this.deps.reconnect,
|
|
772
|
-
WebSocketCtor: this.deps.WebSocketCtor,
|
|
773
|
-
// The offer consults the room's latched transport decision — refuse fast on an SSE handle.
|
|
774
|
-
requireWsAudio: () => this.requireWsAudio("this standing offer"),
|
|
775
|
-
// The ``*.available`` broadcast (and per-source ``transcript.*``) ride the REST /signal POST so
|
|
776
|
-
// an ephemeral hint survives — the WS inbound path forces ``ephemeral: false``.
|
|
777
|
-
publishSignal: (type, payload, o) => client.request(buildRoomSignal(this.id, { type, payload, source: o.source, ephemeral: o.ephemeral })),
|
|
778
|
-
// The armed/active transition an offer broadcasts also surfaces to any room tracer (deduped
|
|
779
|
-
// with the broadcast itself) — Python's `_trace_state` call in `_StandingSource._broadcast`.
|
|
780
|
-
traceState: (label, summary, detail) => this.traceState(label, summary, detail),
|
|
781
|
-
};
|
|
782
|
-
}
|
|
783
|
-
// Wire a peer-toggleable standing service (the third broker kind) — announce ``role`` and route the
|
|
784
|
-
// ``<name>.start`` / ``.stop`` / ``.status`` requests to ``service`` (mirror of ``_offer_service``).
|
|
785
|
-
// Engage is sender-scoped (the server-stamped ``from``) unless the payload carries ``all: true``.
|
|
786
|
-
offerService(name, role, service) {
|
|
787
|
-
this.requirePreConnect();
|
|
788
|
-
this.announceRole(role);
|
|
789
|
-
this.offeredServices.push(service);
|
|
790
|
-
this.on(`${name}.start`, async (e) => {
|
|
791
|
-
try {
|
|
792
|
-
await service.start({ source: e.from, all: Boolean(e.payload.all) });
|
|
793
|
-
}
|
|
794
|
-
catch (err) {
|
|
795
|
-
return { error: asError(err).message };
|
|
796
|
-
}
|
|
797
|
-
return { ok: true };
|
|
798
|
-
});
|
|
799
|
-
this.on(`${name}.stop`, async (e) => {
|
|
800
|
-
try {
|
|
801
|
-
await service.stopRequest({ source: e.from, all: Boolean(e.payload.all) });
|
|
802
|
-
}
|
|
803
|
-
catch (err) {
|
|
804
|
-
return { error: asError(err).message };
|
|
805
|
-
}
|
|
806
|
-
return { ok: true };
|
|
807
|
-
});
|
|
808
|
-
this.on(`${name}.status`, () => ({ ok: true, ...service.status() }));
|
|
809
|
-
}
|
|
810
|
-
/**
|
|
811
|
-
* Offer live transcription as a peer-toggleable standing service — a code-joined tool turns it on
|
|
812
|
-
* with a ``transcription.start`` request and off with ``transcription.stop`` (both idempotent).
|
|
813
|
-
* Announces the discoverable ``transcription-offer`` role. **Scoped, multi-source**: ``start`` /
|
|
814
|
-
* ``stop`` are scoped to the requesting connection unless the payload carries ``{ all: true }``;
|
|
815
|
-
* only armed sources are transcribed (up to ``maxSources``, default 8). While on it runs the same
|
|
816
|
-
* engine as {@link transcribe} on **your** credentials, publishing source-attributed ``transcript.*``
|
|
817
|
-
* signals. Register **before** ``connect()``; needs an owned room (throws on a loginless handle).
|
|
818
|
-
* Returns the handle (for ``handle.start({ all: true })`` from the host). Mirror of Python's
|
|
819
|
-
* ``offer_transcription``.
|
|
820
|
-
*/
|
|
821
|
-
offerTranscription(opts = {}) {
|
|
822
|
-
const client = this.requireClient("offerTranscription");
|
|
823
|
-
const service = new OfferedTranscription({
|
|
824
|
-
...this.offerDeps(client),
|
|
825
|
-
engine: (clips, o) => client.transcribeStream(clips, {
|
|
826
|
-
model: o.model,
|
|
827
|
-
language: o.language,
|
|
828
|
-
format: "webm", // the producer ships Opus-in-WebM; the backend decodes each clip
|
|
829
|
-
partials: true,
|
|
830
|
-
skipEmpty: true,
|
|
831
|
-
signal: o.signal,
|
|
832
|
-
webSocketCtor: this.deps.WebSocketCtor,
|
|
833
|
-
}),
|
|
834
|
-
}, { model: opts.model, language: opts.language, maxSources: opts.maxSources });
|
|
835
|
-
this.offerService("transcription", TRANSCRIPTION_OFFER_ROLE, service);
|
|
836
|
-
return service;
|
|
837
|
-
}
|
|
838
|
-
/**
|
|
839
|
-
* Offer audio capture as a peer-toggleable standing service — **session-less**: it writes into the
|
|
840
|
-
* **active recorder's** session ({@link record} is the single durable binding), so a ``capture.start``
|
|
841
|
-
* with no active recorder replies an error. A code-joined tool toggles it via ``capture.start`` /
|
|
842
|
-
* ``capture.stop`` (idempotent). Announces the discoverable ``capture-offer`` role. **Scoped,
|
|
843
|
-
* multi-source** (up to ``maxSources``, default 8): only armed sources are recorded; a same-name
|
|
844
|
-
* reconnect within grace resumes one file, two simultaneous same-name producers get their own files.
|
|
845
|
-
* Register **before** ``connect()``; needs an owned room (throws on a loginless handle). Returns the
|
|
846
|
-
* handle (for ``handle.start({ all: true })``). Mirror of Python's ``offer_audio_capture``.
|
|
847
|
-
*/
|
|
848
|
-
offerAudioCapture(opts = {}) {
|
|
849
|
-
const client = this.requireClient("offerAudioCapture");
|
|
850
|
-
const service = new OfferedCapture({
|
|
851
|
-
...this.offerDeps(client),
|
|
852
|
-
request: (req) => client.request(req),
|
|
853
|
-
activeRecorders: () => this.activeRecorders(),
|
|
854
|
-
onRecordersChanged: (listener) => this.onRecordersChanged(listener),
|
|
855
|
-
}, { maxSources: opts.maxSources });
|
|
856
|
-
this.offerService("capture", CAPTURE_OFFER_ROLE, service);
|
|
857
|
-
return service;
|
|
858
|
-
}
|
|
859
|
-
async resolveEmbedSpace(client, requested) {
|
|
860
|
-
const spaces = await client.library.spaces();
|
|
861
|
-
if (requested !== undefined) {
|
|
862
|
-
const matched = spaces.find((s) => s.id === requested);
|
|
863
|
-
return { space: requested, model: matched?.model ?? null };
|
|
864
|
-
}
|
|
865
|
-
const matched = spaces.find((s) => s.defaultModalities.includes("text"));
|
|
866
|
-
if (!matched) {
|
|
867
|
-
throw new AtriumError("no default text search space is configured on this deployment — the embed.resolve request " +
|
|
868
|
-
"must pass space");
|
|
869
|
-
}
|
|
870
|
-
return { space: matched.id, model: matched.model ?? null };
|
|
871
|
-
}
|
|
872
|
-
async resolveAnchorVector(client, desc, space, model, cache) {
|
|
873
|
-
const anchor = parseAnchorDescriptor(desc); // throws BrokerError on a malformed anchor
|
|
874
|
-
if (anchor.kind === "vector")
|
|
875
|
-
return [...anchor.vector];
|
|
876
|
-
if (anchor.kind === "asset") {
|
|
877
|
-
const key = `${space}|asset:${anchor.asset}:${anchor.matchBy ?? ""}`;
|
|
878
|
-
let vec = cache.get(key);
|
|
879
|
-
if (!vec) {
|
|
880
|
-
const query = { space };
|
|
881
|
-
if (anchor.matchBy !== null)
|
|
882
|
-
query.match_by = anchor.matchBy;
|
|
883
|
-
const rows = (await client.request({
|
|
884
|
-
method: "GET",
|
|
885
|
-
path: `/v1/assets/${anchor.asset}/vectors`,
|
|
886
|
-
query,
|
|
887
|
-
}));
|
|
888
|
-
const first = Array.isArray(rows) ? rows[0] : undefined;
|
|
889
|
-
if (!first || !Array.isArray(first.vector)) {
|
|
890
|
-
throw new BrokerError(`asset '${anchor.asset}' has no stored vector in space '${space}'`);
|
|
891
|
-
}
|
|
892
|
-
vec = first.vector.map(Number);
|
|
893
|
-
cache.set(key, vec);
|
|
894
|
-
}
|
|
895
|
-
return vec;
|
|
896
|
-
}
|
|
897
|
-
if (anchor.kind === "text") {
|
|
898
|
-
const key = `${space}|text:${anchor.text}`;
|
|
899
|
-
let vec = cache.get(key);
|
|
900
|
-
if (!vec) {
|
|
901
|
-
if (model === null) {
|
|
902
|
-
throw new BrokerError(`space '${space}' has no model configured — cannot embed text`);
|
|
903
|
-
}
|
|
904
|
-
vec = await client.embed({ text: anchor.text, model });
|
|
905
|
-
cache.set(key, vec);
|
|
906
|
-
}
|
|
907
|
-
return vec;
|
|
908
|
-
}
|
|
909
|
-
const key = `${space}|concept:${anchor.concept}:${anchor.collection ?? ""}`;
|
|
910
|
-
let vec = cache.get(key);
|
|
911
|
-
if (!vec) {
|
|
912
|
-
const query = { concept: anchor.concept, space };
|
|
913
|
-
if (anchor.collection !== null)
|
|
914
|
-
query.collection = anchor.collection;
|
|
915
|
-
const data = (await client.request({
|
|
916
|
-
method: "GET",
|
|
917
|
-
path: "/v1/library/concept-vector",
|
|
918
|
-
query,
|
|
919
|
-
}));
|
|
920
|
-
vec = data.vector.map(Number);
|
|
921
|
-
cache.set(key, vec);
|
|
922
|
-
}
|
|
923
|
-
return vec;
|
|
924
|
-
}
|
|
925
|
-
/**
|
|
926
|
-
* Handle ``embed.resolve`` requests from code-joined peers (e.g. a loginless Unity scene) — resolve
|
|
927
|
-
* named operand descriptors (``{ text }`` / ``{ concept, collection? }`` / ``{ vector }`` /
|
|
928
|
-
* ``{ asset, match_by? }``) to vectors in ONE embedding space, so a peer with no credentials can
|
|
929
|
-
* compare live signals against library concepts/text/assets. **The requester chooses the space**
|
|
930
|
-
* (omitted → the default text space). A per-anchor
|
|
931
|
-
* failure lands in ``errors`` without failing the others; a malformed / over-``maxAnchors`` request
|
|
932
|
-
* or a ``space`` outside ``spaces`` gets a whole-request ``{ error }``. Announces the discoverable
|
|
933
|
-
* ``embed-broker`` role — call before ``connect()``. Mirror of Python's ``offer_embeds``.
|
|
934
|
-
*/
|
|
935
|
-
offerEmbeds(opts = {}) {
|
|
936
|
-
const client = this.requireClient("offerEmbeds");
|
|
937
|
-
this.requirePreConnect();
|
|
938
|
-
const allowed = opts.spaces ? new Set(opts.spaces) : null;
|
|
939
|
-
const maxAnchors = opts.maxAnchors ?? DEFAULT_MAX_ANCHORS;
|
|
940
|
-
const cache = new Map();
|
|
941
|
-
this.announceRole(EMBED_BROKER_ROLE);
|
|
942
|
-
this.on("embed.resolve", async (e) => {
|
|
943
|
-
const anchors = e.payload.anchors;
|
|
944
|
-
if (typeof anchors !== "object" ||
|
|
945
|
-
anchors === null ||
|
|
946
|
-
Array.isArray(anchors) ||
|
|
947
|
-
Object.keys(anchors).length === 0) {
|
|
948
|
-
return { error: "anchors must be a non-empty object" };
|
|
949
|
-
}
|
|
950
|
-
const entries = Object.entries(anchors);
|
|
951
|
-
if (entries.length > maxAnchors) {
|
|
952
|
-
return { error: `too many anchors (${entries.length} > ${maxAnchors})` };
|
|
953
|
-
}
|
|
954
|
-
const requested = typeof e.payload.space === "string" ? e.payload.space : undefined;
|
|
955
|
-
if (requested !== undefined && allowed !== null && !allowed.has(requested)) {
|
|
956
|
-
return { error: `space '${requested}' is not served by this broker` };
|
|
957
|
-
}
|
|
958
|
-
let space;
|
|
959
|
-
let model;
|
|
960
|
-
try {
|
|
961
|
-
;
|
|
962
|
-
({ space, model } = await this.resolveEmbedSpace(client, requested));
|
|
963
|
-
}
|
|
964
|
-
catch (err) {
|
|
965
|
-
return { error: asError(err).message };
|
|
966
|
-
}
|
|
967
|
-
if (allowed !== null && !allowed.has(space)) {
|
|
968
|
-
return { error: `space '${space}' is not served by this broker` };
|
|
969
|
-
}
|
|
970
|
-
const vectors = {};
|
|
971
|
-
const errors = {};
|
|
972
|
-
for (const [nm, desc] of entries) {
|
|
973
|
-
try {
|
|
974
|
-
vectors[nm] = await this.resolveAnchorVector(client, desc, space, model, cache);
|
|
975
|
-
}
|
|
976
|
-
catch (err) {
|
|
977
|
-
errors[nm] = asError(err).message;
|
|
978
|
-
}
|
|
979
|
-
}
|
|
980
|
-
const reply = { space, vectors };
|
|
981
|
-
if (Object.keys(errors).length > 0)
|
|
982
|
-
reply.errors = errors;
|
|
983
|
-
return reply;
|
|
984
|
-
});
|
|
985
|
-
}
|
|
986
|
-
/**
|
|
987
|
-
* Handle ``chat.generate`` requests from code-joined peers — a stateless chat completion (no history
|
|
988
|
-
* broker-side), so a loginless peer gets an LLM reply without its own credentials. Reply:
|
|
989
|
-
* ``{ text, model }``, or ``{ error }`` if ``messages`` is malformed / over the payload cap. The
|
|
990
|
-
* **broker** pins ``model`` (the requester cannot choose it — cost control); ``maxTokens`` caps each
|
|
991
|
-
* reply. Announces the discoverable ``chat-broker`` role — call before ``connect()``. Every peer
|
|
992
|
-
* holding the connect code can trigger a reply on **your** billing. Mirror of Python's ``offer_chat``.
|
|
993
|
-
*/
|
|
994
|
-
offerChat(opts = {}) {
|
|
995
|
-
const client = this.requireClient("offerChat");
|
|
996
|
-
this.requirePreConnect();
|
|
997
|
-
const maxTokens = opts.maxTokens ?? DEFAULT_CHAT_MAX_TOKENS;
|
|
998
|
-
this.announceRole(CHAT_BROKER_ROLE);
|
|
999
|
-
this.on("chat.generate", async (e) => {
|
|
1000
|
-
let messages;
|
|
1001
|
-
try {
|
|
1002
|
-
messages = validateChatMessages(e.payload.messages);
|
|
1003
|
-
}
|
|
1004
|
-
catch (err) {
|
|
1005
|
-
return { error: asError(err).message };
|
|
1006
|
-
}
|
|
1007
|
-
try {
|
|
1008
|
-
const reply = await client.chat(messages, { model: opts.model, max_tokens: maxTokens });
|
|
1009
|
-
return { text: reply.content, model: reply.model };
|
|
1010
|
-
}
|
|
1011
|
-
catch (err) {
|
|
1012
|
-
return { error: asError(err).message };
|
|
1013
|
-
}
|
|
1014
|
-
});
|
|
1015
|
-
}
|
|
1016
|
-
// ── Observation dashboard + tracing (host-side helpers) ──────────────────
|
|
1017
|
-
/**
|
|
1018
|
-
* Build the host-side {@link ../dash.Dashboard} for this room — the live observation-dashboard
|
|
1019
|
-
* state an app holds as a grid of tiles. It answers each joining ``/tools/dashboard`` with a full
|
|
1020
|
-
* ``dash.snapshot`` and broadcasts ``dash.delta`` patches as things change, all **ephemeral** (the
|
|
1021
|
-
* room stores nothing — the durable record is the session).
|
|
1022
|
-
*
|
|
1023
|
-
* Prefer this over ``new Dashboard(room)``: it publishes over the REST ``/signal`` POST, which
|
|
1024
|
-
* preserves the dialect's ephemerality (the WS inbound path forces ``ephemeral: false``). Needs an
|
|
1025
|
-
* owned room — throws on a loginless {@link joinRoom} handle. Mirror of Python's ``Dashboard(room)``.
|
|
1026
|
-
*/
|
|
1027
|
-
dashboard(opts = {}) {
|
|
1028
|
-
const client = this.requireClient("dashboard()");
|
|
1029
|
-
return new Dashboard(this, {
|
|
1030
|
-
...opts,
|
|
1031
|
-
publish: (type, payload, o) => client.request(buildRoomSignal(this.id, {
|
|
1032
|
-
type,
|
|
1033
|
-
payload,
|
|
1034
|
-
to: o.to,
|
|
1035
|
-
ephemeral: o.ephemeral,
|
|
1036
|
-
connectionId: this.connectionId,
|
|
1037
|
-
})),
|
|
1038
|
-
});
|
|
1039
|
-
}
|
|
1040
|
-
/**
|
|
1041
|
-
* Mirror the SDK's own activity into this room as ``trace.*`` signals — one line turns it on. The
|
|
1042
|
-
* operation folds into the signal type (``label`` still rides the payload), and ephemerality is
|
|
1043
|
-
* per-signal (see {@link ../trace.isTraceEphemeral}). While tracing:
|
|
1044
|
-
*
|
|
1045
|
-
* * **library/model calls** made through the same client (``search`` / ``searchByExamples`` /
|
|
1046
|
-
* ``enrich`` / ``play`` / ``chat``) surface a ``trace.<label>`` (``trace.library.search`` /
|
|
1047
|
-
* ``trace.chat`` / ``trace.play``) — a glanceable summary (``"rain" → 8 hits (top: Forest Rain
|
|
1048
|
-
* 0.83)``) plus a capped detail (top-N results, never full payloads);
|
|
1049
|
-
* * **SDK-internal state** — a recorder start/stop (with its session + event count), a transcriber
|
|
1050
|
-
* start/stop, a standing offer's armed/active roster — surfaces a ``trace.state.<label>``.
|
|
1051
|
-
*
|
|
1052
|
-
* Default OFF (it broadcasts app internals to anyone with the code, and it's noise on the happy
|
|
1053
|
-
* path). Returns a handle: ``const t = room.trace()`` then ``t.stop()``, or ``using t =
|
|
1054
|
-
* room.trace()`` for a scope; stopped when the room closes. A trace failure never breaks the traced
|
|
1055
|
-
* call. ``play`` and the state seam are scoped to **this** room; client-wide calls mirror into every
|
|
1056
|
-
* armed room. Needs an owned room (the client's calls are what it mirrors) — throws on a loginless
|
|
1057
|
-
* handle. Mirror of Python's ``room.trace()``.
|
|
1058
|
-
*/
|
|
1059
|
-
trace() {
|
|
1060
|
-
const client = this.requireClient("trace()");
|
|
1061
|
-
const handle = new RoomTrace({
|
|
1062
|
-
// Trace signals ride the REST /signal POST, like the transcriber's: the WS inbound path forces
|
|
1063
|
-
// `ephemeral: false`, and trace ephemerality is per-signal (a durable `trace.library.search` IS
|
|
1064
|
-
// the study's record; an ephemeral `trace.play` must not double-record the play).
|
|
1065
|
-
publish: (type, payload, o) => client.request(buildRoomSignal(this.id, {
|
|
1066
|
-
type,
|
|
1067
|
-
payload,
|
|
1068
|
-
ephemeral: o.ephemeral,
|
|
1069
|
-
connectionId: this.connectionId,
|
|
1070
|
-
})),
|
|
1071
|
-
onStop: () => {
|
|
1072
|
-
remove(client.traceSinks, handle);
|
|
1073
|
-
remove(this.traceHandles, handle);
|
|
1074
|
-
},
|
|
1075
|
-
});
|
|
1076
|
-
client.traceSinks?.push(handle);
|
|
1077
|
-
this.traceHandles.push(handle);
|
|
1078
|
-
return handle;
|
|
1079
|
-
}
|
|
1080
|
-
// Emit a `trace.state.<label>` into every tracing session on this room (no-op when none — the
|
|
1081
|
-
// zero-cost-when-off contract). Mirror of Python's `_trace_state`.
|
|
1082
|
-
async traceState(label, summary, detail) {
|
|
1083
|
-
if (this.traceHandles.length === 0)
|
|
1084
|
-
return;
|
|
1085
|
-
const payload = traceEnvelope(label, summary, detail);
|
|
1086
|
-
for (const handle of [...this.traceHandles])
|
|
1087
|
-
await handle.emit(traceStateType(label), payload);
|
|
1088
|
-
}
|
|
1089
|
-
// ── Audio embedding (vectors + reactive scores) ──────────────────────────
|
|
1090
|
-
// Resolve the (space, model) pair for the audio surfaces (mirror of Python's `_resolve_audio_space`):
|
|
1091
|
-
// an explicit space+model skips the lookup; otherwise the member-facing spaces list resolves the
|
|
1092
|
-
// deployment's default audio space and/or reads the space's configured model.
|
|
1093
|
-
async resolveAudioSpace(space, model) {
|
|
1094
|
-
if (space !== undefined && model !== undefined)
|
|
1095
|
-
return { space, model };
|
|
1096
|
-
const spaces = await this.requireClient("audioVectors / audioScores").library.spaces();
|
|
1097
|
-
let matched;
|
|
1098
|
-
if (space === undefined) {
|
|
1099
|
-
matched = spaces.find((s) => s.defaultModalities.includes("audio"));
|
|
1100
|
-
if (!matched) {
|
|
1101
|
-
throw new Error("no default audio search space is configured on this deployment — pass space to audioVectors()");
|
|
1102
|
-
}
|
|
1103
|
-
space = matched.id;
|
|
1104
|
-
}
|
|
1105
|
-
else {
|
|
1106
|
-
matched = spaces.find((s) => s.id === space);
|
|
1107
|
-
if (!matched)
|
|
1108
|
-
throw new Error(`space '${space}' not found`);
|
|
1109
|
-
}
|
|
1110
|
-
return { space, model: model ?? matched?.model ?? null };
|
|
1111
|
-
}
|
|
1112
|
-
// A dedicated snippet-consumer socket (announcing the demand role so a producer captures), decoding
|
|
1113
|
-
// its binary frames — the shared source behind audioVectors/audioScores (mirror of audioClips).
|
|
1114
|
-
async *snippetStream(identity, signal) {
|
|
1115
|
-
this.requireWsAudio("live audio streaming");
|
|
1116
|
-
const queue = [];
|
|
1117
|
-
let wake = null;
|
|
1118
|
-
let ended = false;
|
|
1119
|
-
const socket = new RoomSocket({
|
|
1120
|
-
roomId: this.id,
|
|
1121
|
-
code: this.connectCode,
|
|
1122
|
-
baseUrl: this.deps.baseUrl,
|
|
1123
|
-
identity,
|
|
1124
|
-
announce: { ...this.deps.announce, role: AUDIO_SNIPPET_CONSUMER_ROLE },
|
|
1125
|
-
reconnect: this.deps.reconnect,
|
|
1126
|
-
WebSocketCtor: this.deps.WebSocketCtor,
|
|
1127
|
-
onBinary: (data) => {
|
|
1128
|
-
const frame = decodeAudioSnippetFrame(data);
|
|
1129
|
-
if (!frame)
|
|
1130
|
-
return; // a foreign / DSP binary frame — not ours
|
|
1131
|
-
if (queue.length >= RAW_CONSUMER_QUEUE_MAX)
|
|
1132
|
-
queue.shift(); // drop oldest — bound a slow consumer
|
|
1133
|
-
queue.push(frame);
|
|
1134
|
-
wake?.();
|
|
1135
|
-
wake = null;
|
|
1136
|
-
},
|
|
1137
|
-
onEnd: () => {
|
|
1138
|
-
ended = true;
|
|
1139
|
-
wake?.();
|
|
1140
|
-
wake = null;
|
|
1141
|
-
},
|
|
1142
|
-
});
|
|
1143
|
-
// An abort (a `transcribe()` handle's stop) closes the socket → ends this generator at once.
|
|
1144
|
-
const onAbort = () => {
|
|
1145
|
-
ended = true;
|
|
1146
|
-
socket.close();
|
|
1147
|
-
wake?.();
|
|
1148
|
-
wake = null;
|
|
1149
|
-
};
|
|
1150
|
-
if (signal) {
|
|
1151
|
-
if (signal.aborted)
|
|
1152
|
-
onAbort();
|
|
1153
|
-
else
|
|
1154
|
-
signal.addEventListener("abort", onAbort);
|
|
1155
|
-
}
|
|
1156
|
-
try {
|
|
1157
|
-
try {
|
|
1158
|
-
await socket.connect();
|
|
1159
|
-
}
|
|
1160
|
-
catch (err) {
|
|
1161
|
-
// A pre-/mid-connect abort closed the socket → connect() rejects "room closed"; that's an
|
|
1162
|
-
// expected teardown, not a failure. Only a genuine connect error (bad code) propagates.
|
|
1163
|
-
if (!ended)
|
|
1164
|
-
throw err;
|
|
1165
|
-
}
|
|
1166
|
-
while (!ended) {
|
|
1167
|
-
while (queue.length > 0)
|
|
1168
|
-
yield queue.shift();
|
|
1169
|
-
if (ended)
|
|
1170
|
-
break;
|
|
1171
|
-
await new Promise((resolve) => {
|
|
1172
|
-
wake = resolve;
|
|
1173
|
-
});
|
|
1174
|
-
}
|
|
1175
|
-
while (queue.length > 0)
|
|
1176
|
-
yield queue.shift();
|
|
1177
|
-
}
|
|
1178
|
-
finally {
|
|
1179
|
-
signal?.removeEventListener("abort", onAbort);
|
|
1180
|
-
socket.close();
|
|
1181
|
-
}
|
|
1182
|
-
}
|
|
1183
|
-
/**
|
|
1184
|
-
* Subscribe to the room's live ``audio.snippet`` stream and embed each ~10s window, yielding an
|
|
1185
|
-
* {@link AudioVector} per snippet — a live vector of "what the room sounds like right now".
|
|
1186
|
-
* Subscribing *is* the demand signal (announces the ``audio-snippet-consumer`` role, so a
|
|
1187
|
-
* producer's default-off "Share audio clips" toggle actually captures). Break out of the
|
|
1188
|
-
* ``for await`` loop to end it. Mirror of Python's ``audio_vectors``.
|
|
1189
|
-
*
|
|
1190
|
-
* ``space`` pins the search space snippets are labeled with (default: the deployment's default
|
|
1191
|
-
* audio space, so a no-arg vector is comparable against library-derived vectors); ``model`` the
|
|
1192
|
-
* embedding model (default: the resolved space's configured model). ``relay: true`` also relays
|
|
1193
|
-
* each vector into the room as an ephemeral ``audio.vector`` signal — so a loginless code-joined
|
|
1194
|
-
* peer can consume the live mood vector without credentials of its own. Needs an authenticated
|
|
1195
|
-
* client to embed with (throws on a loginless handle); binary rides the WebSocket only.
|
|
1196
|
-
*/
|
|
1197
|
-
async *audioVectors(opts = {}) {
|
|
1198
|
-
const client = this.requireClient("audioVectors / audioScores");
|
|
1199
|
-
const { space, model } = await this.resolveAudioSpace(opts.space, opts.model);
|
|
1200
|
-
if (model === null) {
|
|
1201
|
-
throw new Error(`the audio search space '${space}' has no model configured — pass model to audioVectors()`);
|
|
1202
|
-
}
|
|
1203
|
-
for await (const frame of this.snippetStream(`audio-vectors:${randomId().slice(0, 8)}`)) {
|
|
1204
|
-
const fmt = frame.mimeType.split(";")[0]?.split("/").pop();
|
|
1205
|
-
const vector = await client.embed({
|
|
1206
|
-
audio: new Blob([frame.data], { type: frame.mimeType }),
|
|
1207
|
-
model,
|
|
1208
|
-
format: fmt,
|
|
1209
|
-
});
|
|
1210
|
-
const vec = {
|
|
1211
|
-
vector,
|
|
1212
|
-
space,
|
|
1213
|
-
sequence: frame.sequence,
|
|
1214
|
-
windowSeconds: frame.windowSeconds,
|
|
1215
|
-
timestamp: Date.now() / 1000,
|
|
1216
|
-
};
|
|
1217
|
-
if (opts.relay) {
|
|
1218
|
-
await this.send("audio.vector", {
|
|
1219
|
-
vector: vec.vector,
|
|
1220
|
-
space: vec.space,
|
|
1221
|
-
sequence: vec.sequence,
|
|
1222
|
-
window_seconds: vec.windowSeconds,
|
|
1223
|
-
timestamp: vec.timestamp,
|
|
1224
|
-
}, { ephemeral: true });
|
|
1225
|
-
}
|
|
1226
|
-
yield vec;
|
|
1227
|
-
}
|
|
1228
|
-
}
|
|
1229
|
-
async preflightConceptAnchors(members, space) {
|
|
1230
|
-
// Resolve every concept anchor's vector once before waiting on any frame — a typo'd/unresolvable
|
|
1231
|
-
// concept fails immediately, naming the anchor, rather than surfacing mid-stream.
|
|
1232
|
-
const client = this.requireClient("audioVectors / audioScores");
|
|
1233
|
-
for (const [name, ops] of Object.entries(members)) {
|
|
1234
|
-
for (const op of ops) {
|
|
1235
|
-
if (op.kind !== "concept")
|
|
1236
|
-
continue;
|
|
1237
|
-
const query = { concept: op.name, space };
|
|
1238
|
-
if (op.collection !== undefined)
|
|
1239
|
-
query.collection = op.collection;
|
|
1240
|
-
try {
|
|
1241
|
-
await client.request({ method: "GET", path: "/v1/library/concept-vector", query });
|
|
1242
|
-
}
|
|
1243
|
-
catch (err) {
|
|
1244
|
-
throw new Error(`anchor '${name}': ${err.message}`);
|
|
1245
|
-
}
|
|
1246
|
-
}
|
|
1247
|
-
}
|
|
1248
|
-
}
|
|
1249
|
-
/**
|
|
1250
|
-
* Fuse {@link audioVectors} with per-anchor ``library.compare`` scoring — a live "which anchor does
|
|
1251
|
-
* the room sound like right now" signal. ``anchors`` maps a name to anything the ``by`` grammar
|
|
1252
|
-
* accepts (a bare string / {@link "../by".Asset} / ``by.*`` marker), **or an array** of those (a
|
|
1253
|
-
* few-shot category, scored as the mean of its top-``topN`` members' calibrated compare scores;
|
|
1254
|
-
* default ``topN: 1`` — the nearest member wins). Each frame's anchors are scored concurrently.
|
|
1255
|
-
*
|
|
1256
|
-
* Each {@link ScoredAudio} carries the calibrated ``scores`` plus ``mix`` (normalized to sum 1,
|
|
1257
|
-
* uniform when every score is 0 — gate on ``max(scores)`` for silence). No smoothing — ease the
|
|
1258
|
-
* stream with a {@link ../decider.Decider}. ``relay: true`` forwards to {@link audioVectors} (it
|
|
1259
|
-
* relays the raw ``audio.vector``, not the computed scores). Concept anchors are validated once up
|
|
1260
|
-
* front (a typo raises before any frame); a mid-stream compare failure re-throws named by anchor.
|
|
1261
|
-
* Mirror of Python's ``audio_scores``.
|
|
1262
|
-
*/
|
|
1263
|
-
async *audioScores(anchors, opts = {}) {
|
|
1264
|
-
const client = this.requireClient("audioVectors / audioScores");
|
|
1265
|
-
const names = Object.keys(anchors);
|
|
1266
|
-
const members = {};
|
|
1267
|
-
for (const name of names)
|
|
1268
|
-
members[name] = toAnchorMembers(anchors[name]);
|
|
1269
|
-
const { space, model } = await this.resolveAudioSpace(opts.space, opts.model);
|
|
1270
|
-
if (model === null) {
|
|
1271
|
-
throw new Error(`the audio search space '${space}' has no model configured — pass model to audioScores()`);
|
|
1272
|
-
}
|
|
1273
|
-
await this.preflightConceptAnchors(members, space);
|
|
1274
|
-
const topN = Math.max(1, opts.topN ?? 1);
|
|
1275
|
-
for await (const vec of this.audioVectors({ space, model, relay: opts.relay })) {
|
|
1276
|
-
const vectorOp = by.vector(vec.vector);
|
|
1277
|
-
// Score every member of every anchor concurrently — a handful of compares is cheap vs. the ~10s
|
|
1278
|
-
// window. Flatten to (name, op) pairs, then fold back into per-anchor top-N means below.
|
|
1279
|
-
const flat = names.flatMap((name) => members[name].map((op) => ({ name, op })));
|
|
1280
|
-
const results = await Promise.all(flat.map(async ({ name, op }) => {
|
|
1281
|
-
try {
|
|
1282
|
-
const r = await client.library.compare(vectorOp, op, { space: vec.space });
|
|
1283
|
-
return r.score;
|
|
1284
|
-
}
|
|
1285
|
-
catch (err) {
|
|
1286
|
-
throw new Error(`anchor '${name}': ${err.message}`);
|
|
1287
|
-
}
|
|
1288
|
-
}));
|
|
1289
|
-
const scores = {};
|
|
1290
|
-
let i = 0;
|
|
1291
|
-
for (const name of names) {
|
|
1292
|
-
const n = members[name].length;
|
|
1293
|
-
const top = results
|
|
1294
|
-
.slice(i, i + n)
|
|
1295
|
-
.sort((a, b) => b - a)
|
|
1296
|
-
.slice(0, topN);
|
|
1297
|
-
scores[name] = top.reduce((a, b) => a + b, 0) / top.length;
|
|
1298
|
-
i += n;
|
|
1299
|
-
}
|
|
1300
|
-
const total = Object.values(scores).reduce((a, b) => a + b, 0);
|
|
1301
|
-
const mix = {};
|
|
1302
|
-
for (const name of names) {
|
|
1303
|
-
mix[name] = total > 0 ? scores[name] / total : 1 / names.length;
|
|
1304
|
-
}
|
|
1305
|
-
yield {
|
|
1306
|
-
scores,
|
|
1307
|
-
mix,
|
|
1308
|
-
vector: vec.vector,
|
|
1309
|
-
space: vec.space,
|
|
1310
|
-
sequence: vec.sequence,
|
|
1311
|
-
windowSeconds: vec.windowSeconds,
|
|
1312
|
-
timestamp: vec.timestamp,
|
|
1313
|
-
};
|
|
1314
|
-
}
|
|
1315
|
-
}
|
|
1316
535
|
/**
|
|
1317
536
|
* The room's current presence roster as typed {@link Participant}s (merged ``roles`` / ``name`` /
|
|
1318
537
|
* ``connectedAt`` as a ``Date``). Opens one short **observe** SSE stream, reads the roster off its
|
|
@@ -1344,10 +563,11 @@ export class Room {
|
|
|
1344
563
|
return roster.some((p) => participantMatches(p, pred));
|
|
1345
564
|
}
|
|
1346
565
|
// ── Player controls (the `layer.*` / `play` dialect) ─────────────────────
|
|
1347
|
-
/** Play a sound (or several) on a layer — the one playback verb.
|
|
1348
|
-
*
|
|
1349
|
-
*
|
|
1350
|
-
*
|
|
566
|
+
/** Play a sound (or several) on a layer — the one playback verb. ``source`` is an
|
|
567
|
+
* {@link ../models.Asset} / {@link ../models.SearchHit} / an id-or-URL string, a typed
|
|
568
|
+
* {@link ../shaping/rooms.PlayTrack} for the per-sound options (``effects``, ``spatial`` — including
|
|
569
|
+
* its optional ``distance`` attenuation — ``delaySeconds``, ``loop`` …), or a list of any of those.
|
|
570
|
+
* Put the asset under the track's ``asset`` key to keep its loudness facts —
|
|
1351
571
|
* ``{ asset: hit, gain: 2 }``. ``opts.gain`` is the same emphasis multiplier for every track that
|
|
1352
572
|
* doesn't carry its own: a trim **on top of** loudness normalization (``2`` ≈ +6 dB over the
|
|
1353
573
|
* normalized baseline), per-sound — the layers keep their own volume. */
|
|
@@ -1359,10 +579,13 @@ export class Room {
|
|
|
1359
579
|
this.socket?.sendSignal({ type: "play", payload: buildPlayBody({ source, ...opts }) });
|
|
1360
580
|
});
|
|
1361
581
|
}
|
|
1362
|
-
/** Declare (or replace) the Player's layer set — idempotent.
|
|
582
|
+
/** Declare (or replace) the Player's layer set — idempotent. Each layer is a typed
|
|
583
|
+
* {@link ../shaping/rooms.LayerDeclaration} (``{ id, label?, volume?, crossfadeMs?, effects? }``)
|
|
584
|
+
* or the dialect's own snake_case spelling; the shaping layer translates. The read-back
|
|
585
|
+
* ({@link queryLayers} / {@link watchLayers}) is still the raw wire snapshot — snake_case. */
|
|
1363
586
|
async configureLayers(layers) {
|
|
1364
587
|
await this.connect();
|
|
1365
|
-
this.socket?.sendSignal({ type: "layer.configure", payload:
|
|
588
|
+
this.socket?.sendSignal({ type: "layer.configure", payload: buildLayerConfigureBody(layers) });
|
|
1366
589
|
}
|
|
1367
590
|
/** Fade out and empty a layer — its queue and one-shots (omit ``layer`` → ``"default"``). */
|
|
1368
591
|
async clearLayer(opts = {}) {
|
|
@@ -1493,23 +716,6 @@ export class Room {
|
|
|
1493
716
|
toolBase() {
|
|
1494
717
|
return this.deps.baseUrl.replace(/\/+$/, "");
|
|
1495
718
|
}
|
|
1496
|
-
// ── Management (authenticated handles only) ──────────────────────────────
|
|
1497
|
-
rest(req) {
|
|
1498
|
-
if (!this.deps.rest) {
|
|
1499
|
-
throw new AtriumError("this room handle has no account credential — join(code) can only send/receive");
|
|
1500
|
-
}
|
|
1501
|
-
return this.deps.rest(req);
|
|
1502
|
-
}
|
|
1503
|
-
/** Rotate the connect code (owner only); updates ``connectCode`` and returns it. */
|
|
1504
|
-
async rotateCode() {
|
|
1505
|
-
const data = (await this.rest(buildRoomRotateCode(this.id)));
|
|
1506
|
-
this.connectCode = data.connect_code ?? this.connectCode;
|
|
1507
|
-
return this.connectCode;
|
|
1508
|
-
}
|
|
1509
|
-
/** Delete the room for good (its connect code dies with it). */
|
|
1510
|
-
async delete() {
|
|
1511
|
-
await this.rest({ method: "DELETE", path: `/v1/rooms/${this.id}` });
|
|
1512
|
-
}
|
|
1513
719
|
// ── Lifecycle ────────────────────────────────────────────────────────────
|
|
1514
720
|
/** Disconnect the socket, stop dispatch, and fail any in-flight requests — the fire-and-forget
|
|
1515
721
|
* teardown. Idempotent. The synchronous state (sockets closed, requests failed, listeners cleared)
|