hci-atrium 0.3.1 β 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/GUIDE.md +1182 -0
- package/PARITY.md +385 -0
- package/README.md +96 -24
- package/bin/atrium.mjs +35 -0
- package/dist/atrium.d.ts +126 -42
- package/dist/atrium.d.ts.map +1 -1
- package/dist/atrium.js +192 -87
- package/dist/atrium.js.map +1 -1
- package/dist/dash.d.ts +3 -3
- package/dist/dash.d.ts.map +1 -1
- package/dist/dash.js +1 -2
- package/dist/dash.js.map +1 -1
- package/dist/env.d.ts +14 -0
- package/dist/env.d.ts.map +1 -1
- package/dist/env.js +22 -0
- package/dist/env.js.map +1 -1
- package/dist/index.d.ts +4 -4
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +11 -3
- package/dist/index.js.map +1 -1
- package/dist/models.d.ts +84 -31
- package/dist/models.d.ts.map +1 -1
- package/dist/models.js +97 -25
- package/dist/models.js.map +1 -1
- package/dist/player/play-command.d.ts +6 -6
- package/dist/player/play-command.d.ts.map +1 -1
- package/dist/player/play-intent.d.ts.map +1 -1
- package/dist/player/play-intent.js +2 -1
- package/dist/player/play-intent.js.map +1 -1
- package/dist/player/runtime.d.ts +7 -3
- package/dist/player/runtime.d.ts.map +1 -1
- package/dist/player/runtime.js.map +1 -1
- package/dist/react/use-room-connection.d.ts +4 -1
- package/dist/react/use-room-connection.d.ts.map +1 -1
- package/dist/react/use-room-connection.js +5 -1
- package/dist/react/use-room-connection.js.map +1 -1
- package/dist/rooms/index.d.ts +3 -1
- package/dist/rooms/index.d.ts.map +1 -1
- package/dist/rooms/index.js +4 -3
- package/dist/rooms/index.js.map +1 -1
- package/dist/rooms/offer-transcription.d.ts +1 -1
- package/dist/rooms/offer-transcription.js +1 -1
- package/dist/rooms/owned-room.d.ts +251 -0
- package/dist/rooms/owned-room.d.ts.map +1 -0
- package/dist/rooms/owned-room.js +844 -0
- package/dist/rooms/owned-room.js.map +1 -0
- package/dist/rooms/recorder.d.ts +1 -1
- package/dist/rooms/recorder.js +1 -1
- package/dist/rooms/room.d.ts +82 -278
- package/dist/rooms/room.d.ts.map +1 -1
- package/dist/rooms/room.js +60 -854
- package/dist/rooms/room.js.map +1 -1
- package/dist/rooms/standing-source.d.ts +1 -1
- package/dist/rooms/standing-source.js +1 -1
- package/dist/rooms/transcriber.d.ts +3 -3
- package/dist/rooms/transcriber.d.ts.map +1 -1
- package/dist/rooms/transcriber.js +2 -2
- package/dist/rooms/transport.d.ts +1 -1
- package/dist/rooms/transport.js +2 -2
- package/dist/shaping/assets.d.ts +13 -0
- package/dist/shaping/assets.d.ts.map +1 -1
- package/dist/shaping/assets.js +27 -0
- package/dist/shaping/assets.js.map +1 -1
- package/dist/shaping/chat.d.ts +3 -1
- package/dist/shaping/chat.d.ts.map +1 -1
- package/dist/shaping/chat.js +4 -2
- package/dist/shaping/chat.js.map +1 -1
- package/dist/shaping/rooms.d.ts +78 -1
- package/dist/shaping/rooms.d.ts.map +1 -1
- package/dist/shaping/rooms.js +58 -6
- package/dist/shaping/rooms.js.map +1 -1
- package/dist/shaping/sources.d.ts +7 -3
- package/dist/shaping/sources.d.ts.map +1 -1
- package/dist/shaping/sources.js +9 -10
- package/dist/shaping/sources.js.map +1 -1
- package/dist/transcribe-stream.d.ts +1 -1
- package/dist/transcribe-stream.js +1 -1
- package/dist/transcribe.d.ts +2 -2
- package/dist/transcribe.js +2 -2
- package/package.json +18 -3
|
@@ -0,0 +1,844 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The **owned** room handle β {@link ./room.Room} plus every surface that needs the account
|
|
3
|
+
* credential behind it. Returned by ``client.rooms.open`` / ``.get`` / ``.list``, which inject the
|
|
4
|
+
* authenticated REST caller and the client slice this class is built on.
|
|
5
|
+
*
|
|
6
|
+
* The split is deliberate: a loginless ``joinRoom(code)`` handle is a plain {@link ./room.Room}, so
|
|
7
|
+
* the members here are not merely guarded at runtime β they are absent from its type. Everything on
|
|
8
|
+
* this class assumes a credential and calls it directly ({@link OwnedRoomDeps} makes ``rest`` /
|
|
9
|
+
* ``client`` required), so nothing here needs a runtime credential guard.
|
|
10
|
+
*
|
|
11
|
+
* Durable & reactive: {@link OwnedRoom.record} bridges the room into a session over an SSE-observe
|
|
12
|
+
* {@link ./recorder.Recorder} (a room stores nothing itself, so durable capture rides the observe
|
|
13
|
+
* stream); {@link OwnedRoom.audioVectors} embeds the live snippet stream and
|
|
14
|
+
* {@link OwnedRoom.audioScores} fuses it against named anchors for a {@link ../decider.Decider};
|
|
15
|
+
* the standing offers and capability brokers serve code-joined peers on your credential.
|
|
16
|
+
*/
|
|
17
|
+
import { by, toAnchorMembers } from "../by.js";
|
|
18
|
+
import { Dashboard } from "../dash.js";
|
|
19
|
+
import { APIError, AtriumError } from "../errors.js";
|
|
20
|
+
import { asError } from "../http.js";
|
|
21
|
+
import { buildRoomRotateCode, buildRoomSignal } from "../shaping/rooms.js";
|
|
22
|
+
import { RoomTrace, traceEnvelope, traceStateType } from "../trace.js";
|
|
23
|
+
import { BrokerError, CAPTURE_OFFER_ROLE, CHAT_BROKER_ROLE, DEFAULT_CHAT_MAX_TOKENS, DEFAULT_MAX_ANCHORS, EMBED_BROKER_ROLE, parseAnchorDescriptor, TRANSCRIPTION_OFFER_ROLE, validateChatMessages, } from "./brokers.js";
|
|
24
|
+
import { OfferedCapture } from "./offer-capture.js";
|
|
25
|
+
import { OfferedTranscription } from "./offer-transcription.js";
|
|
26
|
+
import { Recorder } from "./recorder.js";
|
|
27
|
+
import { RAW_CONSUMER_QUEUE_MAX, Room, randomId, remove, } from "./room.js";
|
|
28
|
+
import { AUDIO_SNIPPET_CONSUMER_ROLE, decodeAudioSnippetFrame, } from "./snippet.js";
|
|
29
|
+
import { TRANSCRIBER_SOURCE, Transcriber } from "./transcriber.js";
|
|
30
|
+
import { RoomSocket } from "./transport.js";
|
|
31
|
+
export class OwnedRoom extends Room {
|
|
32
|
+
// The change-listeners a capture offer subscribes with β the recorder-registration seam
|
|
33
|
+
// (`_notify_recorders_changed` in Python). Cleared by `teardown`; an offer also unsubscribes
|
|
34
|
+
// itself when the room winds it down.
|
|
35
|
+
recorderListeners = new Set();
|
|
36
|
+
// The constructor exists to narrow the parameter: the inherited one would accept a bare RoomDeps,
|
|
37
|
+
// whose missing `rest`/`client` every member here assumes.
|
|
38
|
+
// biome-ignore lint/complexity/noUselessConstructor: it narrows the constructor's parameter type.
|
|
39
|
+
constructor(deps) {
|
|
40
|
+
super(deps);
|
|
41
|
+
}
|
|
42
|
+
// ββ Durable capture (record β session) βββββββββββββββββββββββββββββββββββ
|
|
43
|
+
/**
|
|
44
|
+
* Start capturing this room's live signals into a durable ``session`` β the single durable binding
|
|
45
|
+
* between a room (pure transport) and a session (an append-only log). A **side effect**: it starts
|
|
46
|
+
* immediately in the background and returns a live {@link Recorder} handle. Stop it with
|
|
47
|
+
* ``rec.stop()``, or let {@link close} (an ``await using`` room) stop it. It skips ``ephemeral``
|
|
48
|
+
* signals and presence traffic β only durable-worthy signals are recorded, each stamped with
|
|
49
|
+
* ``via`` provenance (``{ room_id, room_name, from, to }``). Mirror of Python's ``room.record``.
|
|
50
|
+
*
|
|
51
|
+
* A dropped stream reconnects with backoff (the gap is lost β rooms have no replay); only a
|
|
52
|
+
* terminal end stops it cleanly with a {@link Recorder.stoppedReason} (a code rotation, or the
|
|
53
|
+
* room being gone / its code rejected β the latter also sets {@link Recorder.error}).
|
|
54
|
+
*/
|
|
55
|
+
record(into) {
|
|
56
|
+
const rest = this.deps.rest;
|
|
57
|
+
// The `trace.state.recorder` line rides the same change seam, on TRANSITIONS only (the seam also
|
|
58
|
+
// fires on a no-op stop) β mirror of Python's start/stop emissions in the recorder's own task.
|
|
59
|
+
let tracedAlive = false;
|
|
60
|
+
const recorder = new Recorder({
|
|
61
|
+
roomId: this.id,
|
|
62
|
+
code: this.connectCode,
|
|
63
|
+
roomName: this.name,
|
|
64
|
+
baseUrl: this.deps.baseUrl,
|
|
65
|
+
rest,
|
|
66
|
+
reconnect: this.deps.reconnect,
|
|
67
|
+
fetchImpl: this.deps.fetchImpl,
|
|
68
|
+
onChange: () => {
|
|
69
|
+
void this.notifyRecordersChanged();
|
|
70
|
+
if (recorder.alive === tracedAlive)
|
|
71
|
+
return;
|
|
72
|
+
tracedAlive = recorder.alive;
|
|
73
|
+
void this.traceState("recorder", tracedAlive
|
|
74
|
+
? `recording ${into.id}: started`
|
|
75
|
+
: `recording ${into.id}: stopped (${recorder.events} events)`);
|
|
76
|
+
},
|
|
77
|
+
}, into);
|
|
78
|
+
this.recorders.push(recorder);
|
|
79
|
+
recorder.start();
|
|
80
|
+
return recorder;
|
|
81
|
+
}
|
|
82
|
+
/** The recorders currently running on this handle (membership **and** liveness β a stopped one no
|
|
83
|
+
* longer counts) β the source of truth a wave-4 ``offerAudioCapture`` reads for audio-capture
|
|
84
|
+
* availability, and binds its capture to the first active recorder's session. */
|
|
85
|
+
activeRecorders() {
|
|
86
|
+
return this.recorders.filter((r) => r.alive);
|
|
87
|
+
}
|
|
88
|
+
/** Register a listener fired whenever the active-recorder set changes (a {@link record} started, or
|
|
89
|
+
* a recorder stopped/terminated) β the seam a wave-4 ``offerAudioCapture`` binds to, so it can
|
|
90
|
+
* re-derive availability and disengage when the recorder it wrote under is gone. Returns an
|
|
91
|
+
* unsubscribe function. Mirror of Python's ``_notify_recorders_changed`` registration. */
|
|
92
|
+
onRecordersChanged(listener) {
|
|
93
|
+
this.recorderListeners.add(listener);
|
|
94
|
+
return () => {
|
|
95
|
+
this.recorderListeners.delete(listener);
|
|
96
|
+
};
|
|
97
|
+
}
|
|
98
|
+
async notifyRecordersChanged() {
|
|
99
|
+
// Isolate each hook β one misbehaving listener must not starve the others nor throw out of a
|
|
100
|
+
// recorder's teardown path (this fires from a recorder's terminal wind-down).
|
|
101
|
+
for (const listener of [...this.recorderListeners]) {
|
|
102
|
+
try {
|
|
103
|
+
await listener();
|
|
104
|
+
}
|
|
105
|
+
catch (err) {
|
|
106
|
+
console.error("Atrium room: a recorders-changed listener failed:", err);
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
// ββ Live transcription (the transcript.* broker) βββββββββββββββββββββββββ
|
|
111
|
+
/**
|
|
112
|
+
* Transcribe this room's live audio into ``transcript.*`` signals β the in-room broker. Consumes the
|
|
113
|
+
* room's ``audio.snippet`` stream (announcing the snippet-consumer role, so a Player starts
|
|
114
|
+
* capturing), pushes it through the same engine as {@link ../atrium.Atrium.transcribeLive} on
|
|
115
|
+
* **your** credentials/compute, and publishes back:
|
|
116
|
+
*
|
|
117
|
+
* * ``transcript.partial`` β ``{ text }``, **ephemeral** (a live caption the recorder skips).
|
|
118
|
+
* Wholesale-replaced β render the latest, don't append.
|
|
119
|
+
* * ``transcript.final`` β ``{ text, start, end, seq }``, durable (the {@link record} bridge persists
|
|
120
|
+
* it β no new recording API). ``seq`` is a per-handle counter from 0; the signal's ``source`` is
|
|
121
|
+
* ``"transcriber"``.
|
|
122
|
+
*
|
|
123
|
+
* Any consumer β including a loginless code-joined tool β just does ``room.on("transcript.final")``.
|
|
124
|
+
* Like {@link record}, it's a **side effect**: it starts immediately in the background and is stopped
|
|
125
|
+
* when the room closes, or via ``await handle.stop()`` / ``await using``.
|
|
126
|
+
*/
|
|
127
|
+
transcribe(opts = {}) {
|
|
128
|
+
const client = this.deps.client;
|
|
129
|
+
const transcriber = new Transcriber({
|
|
130
|
+
model: opts.model,
|
|
131
|
+
language: opts.language,
|
|
132
|
+
snippetBytes: (signal) => this.snippetBytesFor(signal),
|
|
133
|
+
engine: (clips, o) => client.transcribeStream(clips, {
|
|
134
|
+
model: o.model,
|
|
135
|
+
language: o.language,
|
|
136
|
+
// The producer ships Opus-in-WebM; the backend decodes each clip. Partials are published
|
|
137
|
+
// as ephemeral live captions, so the engine must surface them.
|
|
138
|
+
format: "webm",
|
|
139
|
+
partials: true,
|
|
140
|
+
skipEmpty: true,
|
|
141
|
+
signal: o.signal,
|
|
142
|
+
webSocketCtor: this.deps.webSocketCtor,
|
|
143
|
+
}),
|
|
144
|
+
publish: (type, payload, o) => client.request(buildRoomSignal(this.id, {
|
|
145
|
+
type,
|
|
146
|
+
payload,
|
|
147
|
+
source: TRANSCRIBER_SOURCE,
|
|
148
|
+
ephemeral: o.ephemeral,
|
|
149
|
+
})),
|
|
150
|
+
onEnd: (err) => void this.traceState("transcriber", err ? `transcribing: failed β ${err.message}` : "transcribing: stopped"),
|
|
151
|
+
});
|
|
152
|
+
this.transcribers.push(transcriber);
|
|
153
|
+
transcriber.start();
|
|
154
|
+
void this.traceState("transcriber", `transcribing (${opts.model ?? "default"}): started`);
|
|
155
|
+
return transcriber;
|
|
156
|
+
}
|
|
157
|
+
/**
|
|
158
|
+
* Subscribe to the room's live ``audio.snippet`` stream and transcribe it, yielding a
|
|
159
|
+
* {@link ../models.Transcript} per finalized sentence β the speech-to-text sibling of
|
|
160
|
+
* {@link audioVectors}, and the *raw* counterpart of the {@link transcribe} broker (which publishes
|
|
161
|
+
* ``transcript.*`` signals for every peer instead of handing you the stream).
|
|
162
|
+
*
|
|
163
|
+
* Subscribing *is* the demand signal (it announces the ``audio-snippet-consumer`` role, so a
|
|
164
|
+
* producer's own default-off "Share audio clips" toggle actually captures). Break out of the
|
|
165
|
+
* ``for await`` loop β or abort ``signal`` β to end the subscription and close the socket.
|
|
166
|
+
*
|
|
167
|
+
* Snippets run through the same engine as {@link ../atrium.Atrium.transcribeLive}: true streaming
|
|
168
|
+
* (per-sentence finals, plus revisable partials with ``partials``) when the backend's live route is
|
|
169
|
+
* reachable, degrading silently to per-clip batch transcription otherwise (finals only).
|
|
170
|
+
* Transcription runs on *your* credentials/compute β a rejected credential raises rather than
|
|
171
|
+
* falling back.
|
|
172
|
+
*
|
|
173
|
+
* The two paths differ in one detail (mirror of Python's ``audio_transcripts``): on the **fallback**
|
|
174
|
+
* path each transcript carries its source frame's ``sequence`` / ``window_seconds`` under
|
|
175
|
+
* ``transcript.raw.snippet`` (a cheap ordering/correlation handle) and the clip's format comes from
|
|
176
|
+
* that frame's own MIME type; on the **streaming** path transcripts are per-sentence, so there is no
|
|
177
|
+
* 1:1 snippet correlation and ``raw.snippet`` is omitted (``raw`` carries ``start`` / ``end``).
|
|
178
|
+
*/
|
|
179
|
+
async *audioTranscripts(opts = {}) {
|
|
180
|
+
const client = this.deps.client;
|
|
181
|
+
const skipEmpty = opts.skipEmpty ?? true;
|
|
182
|
+
// Own abort, chained to the caller's: a consumer that just `break`s out of the loop must tear the
|
|
183
|
+
// whole chain down, and JS can't cancel the engine's pending feeder `await` the way Python cancels
|
|
184
|
+
// its task β aborting closes the snippet socket, which is what unblocks it (same teardown seam as
|
|
185
|
+
// the `transcribe()` handle's stop()).
|
|
186
|
+
const ctrl = new AbortController();
|
|
187
|
+
const chain = () => ctrl.abort();
|
|
188
|
+
if (opts.signal) {
|
|
189
|
+
if (opts.signal.aborted)
|
|
190
|
+
ctrl.abort();
|
|
191
|
+
else
|
|
192
|
+
opts.signal.addEventListener("abort", chain, { once: true });
|
|
193
|
+
}
|
|
194
|
+
// One shared snippet source: the streaming path draws its bytes, the fallback its frames + their
|
|
195
|
+
// per-frame metadata. Only one of the two ever iterates it (the engine picks a path).
|
|
196
|
+
const frames = this.snippetStream(`audio-transcripts:${randomId().slice(0, 8)}`, ctrl.signal);
|
|
197
|
+
async function* snippetBytes() {
|
|
198
|
+
for await (const frame of frames)
|
|
199
|
+
yield frame.data;
|
|
200
|
+
}
|
|
201
|
+
async function* snippetFallback() {
|
|
202
|
+
for await (const frame of frames) {
|
|
203
|
+
const format = frame.mimeType.split(";")[0]?.split("/").pop();
|
|
204
|
+
let said;
|
|
205
|
+
try {
|
|
206
|
+
said = await client.transcribe(new Blob([frame.data], { type: frame.mimeType }), {
|
|
207
|
+
model: opts.model,
|
|
208
|
+
language: opts.language,
|
|
209
|
+
format,
|
|
210
|
+
});
|
|
211
|
+
}
|
|
212
|
+
catch (err) {
|
|
213
|
+
if (err instanceof APIError && (err.status === 401 || err.status === 403))
|
|
214
|
+
throw err;
|
|
215
|
+
continue; // a frame the server couldn't transcribe β skip, keep the stream alive
|
|
216
|
+
}
|
|
217
|
+
if (!said.text && skipEmpty)
|
|
218
|
+
continue;
|
|
219
|
+
said.raw.snippet = { sequence: frame.sequence, window_seconds: frame.windowSeconds };
|
|
220
|
+
yield said;
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
const stream = client.transcribeStream(snippetBytes(), {
|
|
224
|
+
model: opts.model,
|
|
225
|
+
language: opts.language,
|
|
226
|
+
format: "webm", // the producer ships Opus-in-WebM; the backend decodes each clip
|
|
227
|
+
partials: opts.partials ?? false,
|
|
228
|
+
skipEmpty,
|
|
229
|
+
signal: ctrl.signal,
|
|
230
|
+
webSocketCtor: this.deps.webSocketCtor,
|
|
231
|
+
fallback: snippetFallback,
|
|
232
|
+
});
|
|
233
|
+
const engine = stream[Symbol.asyncIterator]();
|
|
234
|
+
// Pulled by hand (not `yield*`) so teardown runs in the order that actually unwinds: abort the
|
|
235
|
+
// socket FIRST, then wind the engine down β `yield*` would forward the consumer's `return()`
|
|
236
|
+
// into the engine while its feeder is still parked on a snippet that will never arrive.
|
|
237
|
+
try {
|
|
238
|
+
for (;;) {
|
|
239
|
+
const next = await engine.next();
|
|
240
|
+
if (next.done)
|
|
241
|
+
break;
|
|
242
|
+
yield next.value;
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
finally {
|
|
246
|
+
opts.signal?.removeEventListener("abort", chain);
|
|
247
|
+
ctrl.abort();
|
|
248
|
+
await engine.return?.().catch(() => { });
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
// The raw snippet bytes for the transcription broker β a dedicated consumer socket, ended when the
|
|
252
|
+
// handle's stop() aborts the signal (which closes the socket).
|
|
253
|
+
async *snippetBytesFor(signal) {
|
|
254
|
+
for await (const frame of this.snippetStream(`transcriber:${randomId().slice(0, 8)}`, signal)) {
|
|
255
|
+
yield frame.data;
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
requirePreConnect() {
|
|
259
|
+
if (this.connectionId !== null) {
|
|
260
|
+
throw new AtriumError("register offer_*/offerEmbeds/offerChat before connect() β the broker role is only " +
|
|
261
|
+
"advertised when the connection opens");
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
offerDeps(client) {
|
|
265
|
+
return {
|
|
266
|
+
roomId: this.id,
|
|
267
|
+
getCode: () => this.connectCode,
|
|
268
|
+
baseUrl: this.deps.baseUrl,
|
|
269
|
+
reconnect: this.deps.reconnect,
|
|
270
|
+
webSocketCtor: this.deps.webSocketCtor,
|
|
271
|
+
// The offer consults the room's latched transport decision β refuse fast on an SSE handle.
|
|
272
|
+
requireWsAudio: () => this.requireWsAudio("this standing offer"),
|
|
273
|
+
// The ``*.available`` broadcast (and per-source ``transcript.*``) ride the REST /signal POST so
|
|
274
|
+
// an ephemeral hint survives β the WS inbound path forces ``ephemeral: false``.
|
|
275
|
+
publishSignal: (type, payload, o) => client.request(buildRoomSignal(this.id, { type, payload, source: o.source, ephemeral: o.ephemeral })),
|
|
276
|
+
// The armed/active transition an offer broadcasts also surfaces to any room tracer (deduped
|
|
277
|
+
// with the broadcast itself) β Python's `_trace_state` call in `_StandingSource._broadcast`.
|
|
278
|
+
traceState: (label, summary, detail) => this.traceState(label, summary, detail),
|
|
279
|
+
};
|
|
280
|
+
}
|
|
281
|
+
// Wire a peer-toggleable standing service (the third broker kind) β announce ``role`` and route the
|
|
282
|
+
// ``<name>.start`` / ``.stop`` / ``.status`` requests to ``service`` (mirror of ``_offer_service``).
|
|
283
|
+
// Engage is sender-scoped (the server-stamped ``from``) unless the payload carries ``all: true``.
|
|
284
|
+
offerService(name, role, service) {
|
|
285
|
+
this.requirePreConnect();
|
|
286
|
+
this.announceRole(role);
|
|
287
|
+
this.offeredServices.push(service);
|
|
288
|
+
this.on(`${name}.start`, async (e) => {
|
|
289
|
+
try {
|
|
290
|
+
await service.start({ source: e.from, all: Boolean(e.payload.all) });
|
|
291
|
+
}
|
|
292
|
+
catch (err) {
|
|
293
|
+
return { error: asError(err).message };
|
|
294
|
+
}
|
|
295
|
+
return { ok: true };
|
|
296
|
+
});
|
|
297
|
+
this.on(`${name}.stop`, async (e) => {
|
|
298
|
+
try {
|
|
299
|
+
await service.stopRequest({ source: e.from, all: Boolean(e.payload.all) });
|
|
300
|
+
}
|
|
301
|
+
catch (err) {
|
|
302
|
+
return { error: asError(err).message };
|
|
303
|
+
}
|
|
304
|
+
return { ok: true };
|
|
305
|
+
});
|
|
306
|
+
this.on(`${name}.status`, () => ({ ok: true, ...service.status() }));
|
|
307
|
+
}
|
|
308
|
+
/**
|
|
309
|
+
* Offer live transcription as a peer-toggleable standing service β a code-joined tool turns it on
|
|
310
|
+
* with a ``transcription.start`` request and off with ``transcription.stop`` (both idempotent).
|
|
311
|
+
* Announces the discoverable ``transcription-offer`` role. **Scoped, multi-source**: ``start`` /
|
|
312
|
+
* ``stop`` are scoped to the requesting connection unless the payload carries ``{ all: true }``;
|
|
313
|
+
* only armed sources are transcribed (up to ``maxSources``, default 8). While on it runs the same
|
|
314
|
+
* engine as {@link transcribe} on **your** credentials, publishing source-attributed ``transcript.*``
|
|
315
|
+
* signals. Register **before** ``connect()``.
|
|
316
|
+
* Returns the handle (for ``handle.start({ all: true })`` from the host). Mirror of Python's
|
|
317
|
+
* ``offer_transcription``.
|
|
318
|
+
*/
|
|
319
|
+
offerTranscription(opts = {}) {
|
|
320
|
+
const client = this.deps.client;
|
|
321
|
+
const service = new OfferedTranscription({
|
|
322
|
+
...this.offerDeps(client),
|
|
323
|
+
engine: (clips, o) => client.transcribeStream(clips, {
|
|
324
|
+
model: o.model,
|
|
325
|
+
language: o.language,
|
|
326
|
+
format: "webm", // the producer ships Opus-in-WebM; the backend decodes each clip
|
|
327
|
+
partials: true,
|
|
328
|
+
skipEmpty: true,
|
|
329
|
+
signal: o.signal,
|
|
330
|
+
webSocketCtor: this.deps.webSocketCtor,
|
|
331
|
+
}),
|
|
332
|
+
}, { model: opts.model, language: opts.language, maxSources: opts.maxSources });
|
|
333
|
+
this.offerService("transcription", TRANSCRIPTION_OFFER_ROLE, service);
|
|
334
|
+
return service;
|
|
335
|
+
}
|
|
336
|
+
/**
|
|
337
|
+
* Offer audio capture as a peer-toggleable standing service β **session-less**: it writes into the
|
|
338
|
+
* **active recorder's** session ({@link record} is the single durable binding), so a ``capture.start``
|
|
339
|
+
* with no active recorder replies an error. A code-joined tool toggles it via ``capture.start`` /
|
|
340
|
+
* ``capture.stop`` (idempotent). Announces the discoverable ``capture-offer`` role. **Scoped,
|
|
341
|
+
* multi-source** (up to ``maxSources``, default 8): only armed sources are recorded; a same-name
|
|
342
|
+
* reconnect within grace resumes one file, two simultaneous same-name producers get their own files.
|
|
343
|
+
* Register **before** ``connect()``. Returns the
|
|
344
|
+
* handle (for ``handle.start({ all: true })``). Mirror of Python's ``offer_audio_capture``.
|
|
345
|
+
*/
|
|
346
|
+
offerAudioCapture(opts = {}) {
|
|
347
|
+
const client = this.deps.client;
|
|
348
|
+
const service = new OfferedCapture({
|
|
349
|
+
...this.offerDeps(client),
|
|
350
|
+
request: (req) => client.request(req),
|
|
351
|
+
activeRecorders: () => this.activeRecorders(),
|
|
352
|
+
onRecordersChanged: (listener) => this.onRecordersChanged(listener),
|
|
353
|
+
}, { maxSources: opts.maxSources });
|
|
354
|
+
this.offerService("capture", CAPTURE_OFFER_ROLE, service);
|
|
355
|
+
return service;
|
|
356
|
+
}
|
|
357
|
+
async resolveEmbedSpace(client, requested) {
|
|
358
|
+
const spaces = await client.library.spaces();
|
|
359
|
+
if (requested !== undefined) {
|
|
360
|
+
const matched = spaces.find((s) => s.id === requested);
|
|
361
|
+
return { space: requested, model: matched?.model ?? null };
|
|
362
|
+
}
|
|
363
|
+
const matched = spaces.find((s) => s.defaultModalities.includes("text"));
|
|
364
|
+
if (!matched) {
|
|
365
|
+
throw new AtriumError("no default text search space is configured on this deployment β the embed.resolve request " +
|
|
366
|
+
"must pass space");
|
|
367
|
+
}
|
|
368
|
+
return { space: matched.id, model: matched.model ?? null };
|
|
369
|
+
}
|
|
370
|
+
async resolveAnchorVector(client, desc, space, model, cache) {
|
|
371
|
+
const anchor = parseAnchorDescriptor(desc); // throws BrokerError on a malformed anchor
|
|
372
|
+
if (anchor.kind === "vector")
|
|
373
|
+
return [...anchor.vector];
|
|
374
|
+
if (anchor.kind === "asset") {
|
|
375
|
+
const key = `${space}|asset:${anchor.asset}:${anchor.matchBy ?? ""}`;
|
|
376
|
+
let vec = cache.get(key);
|
|
377
|
+
if (!vec) {
|
|
378
|
+
const query = { space };
|
|
379
|
+
if (anchor.matchBy !== null)
|
|
380
|
+
query.match_by = anchor.matchBy;
|
|
381
|
+
const rows = (await client.request({
|
|
382
|
+
method: "GET",
|
|
383
|
+
path: `/v1/assets/${anchor.asset}/vectors`,
|
|
384
|
+
query,
|
|
385
|
+
}));
|
|
386
|
+
const first = Array.isArray(rows) ? rows[0] : undefined;
|
|
387
|
+
if (!first || !Array.isArray(first.vector)) {
|
|
388
|
+
throw new BrokerError(`asset '${anchor.asset}' has no stored vector in space '${space}'`);
|
|
389
|
+
}
|
|
390
|
+
vec = first.vector.map(Number);
|
|
391
|
+
cache.set(key, vec);
|
|
392
|
+
}
|
|
393
|
+
return vec;
|
|
394
|
+
}
|
|
395
|
+
if (anchor.kind === "text") {
|
|
396
|
+
const key = `${space}|text:${anchor.text}`;
|
|
397
|
+
let vec = cache.get(key);
|
|
398
|
+
if (!vec) {
|
|
399
|
+
if (model === null) {
|
|
400
|
+
throw new BrokerError(`space '${space}' has no model configured β cannot embed text`);
|
|
401
|
+
}
|
|
402
|
+
vec = await client.embed({ text: anchor.text, model });
|
|
403
|
+
cache.set(key, vec);
|
|
404
|
+
}
|
|
405
|
+
return vec;
|
|
406
|
+
}
|
|
407
|
+
const key = `${space}|concept:${anchor.concept}:${anchor.collection ?? ""}`;
|
|
408
|
+
let vec = cache.get(key);
|
|
409
|
+
if (!vec) {
|
|
410
|
+
const query = { concept: anchor.concept, space };
|
|
411
|
+
if (anchor.collection !== null)
|
|
412
|
+
query.collection = anchor.collection;
|
|
413
|
+
const data = (await client.request({
|
|
414
|
+
method: "GET",
|
|
415
|
+
path: "/v1/library/concept-vector",
|
|
416
|
+
query,
|
|
417
|
+
}));
|
|
418
|
+
vec = data.vector.map(Number);
|
|
419
|
+
cache.set(key, vec);
|
|
420
|
+
}
|
|
421
|
+
return vec;
|
|
422
|
+
}
|
|
423
|
+
/**
|
|
424
|
+
* Handle ``embed.resolve`` requests from code-joined peers (e.g. a loginless Unity scene) β resolve
|
|
425
|
+
* named operand descriptors (``{ text }`` / ``{ concept, collection? }`` / ``{ vector }`` /
|
|
426
|
+
* ``{ asset, match_by? }``) to vectors in ONE embedding space, so a peer with no credentials can
|
|
427
|
+
* compare live signals against library concepts/text/assets. **The requester chooses the space**
|
|
428
|
+
* (omitted β the default text space). A per-anchor
|
|
429
|
+
* failure lands in ``errors`` without failing the others; a malformed / over-``maxAnchors`` request
|
|
430
|
+
* or a ``space`` outside ``spaces`` gets a whole-request ``{ error }``. Announces the discoverable
|
|
431
|
+
* ``embed-broker`` role β call before ``connect()``. Mirror of Python's ``offer_embeds``.
|
|
432
|
+
*/
|
|
433
|
+
offerEmbeds(opts = {}) {
|
|
434
|
+
const client = this.deps.client;
|
|
435
|
+
this.requirePreConnect();
|
|
436
|
+
const allowed = opts.spaces ? new Set(opts.spaces) : null;
|
|
437
|
+
const maxAnchors = opts.maxAnchors ?? DEFAULT_MAX_ANCHORS;
|
|
438
|
+
const cache = new Map();
|
|
439
|
+
this.announceRole(EMBED_BROKER_ROLE);
|
|
440
|
+
this.on("embed.resolve", async (e) => {
|
|
441
|
+
const anchors = e.payload.anchors;
|
|
442
|
+
if (typeof anchors !== "object" ||
|
|
443
|
+
anchors === null ||
|
|
444
|
+
Array.isArray(anchors) ||
|
|
445
|
+
Object.keys(anchors).length === 0) {
|
|
446
|
+
return { error: "anchors must be a non-empty object" };
|
|
447
|
+
}
|
|
448
|
+
const entries = Object.entries(anchors);
|
|
449
|
+
if (entries.length > maxAnchors) {
|
|
450
|
+
return { error: `too many anchors (${entries.length} > ${maxAnchors})` };
|
|
451
|
+
}
|
|
452
|
+
const requested = typeof e.payload.space === "string" ? e.payload.space : undefined;
|
|
453
|
+
if (requested !== undefined && allowed !== null && !allowed.has(requested)) {
|
|
454
|
+
return { error: `space '${requested}' is not served by this broker` };
|
|
455
|
+
}
|
|
456
|
+
let space;
|
|
457
|
+
let model;
|
|
458
|
+
try {
|
|
459
|
+
;
|
|
460
|
+
({ space, model } = await this.resolveEmbedSpace(client, requested));
|
|
461
|
+
}
|
|
462
|
+
catch (err) {
|
|
463
|
+
return { error: asError(err).message };
|
|
464
|
+
}
|
|
465
|
+
if (allowed !== null && !allowed.has(space)) {
|
|
466
|
+
return { error: `space '${space}' is not served by this broker` };
|
|
467
|
+
}
|
|
468
|
+
const vectors = {};
|
|
469
|
+
const errors = {};
|
|
470
|
+
for (const [nm, desc] of entries) {
|
|
471
|
+
try {
|
|
472
|
+
vectors[nm] = await this.resolveAnchorVector(client, desc, space, model, cache);
|
|
473
|
+
}
|
|
474
|
+
catch (err) {
|
|
475
|
+
errors[nm] = asError(err).message;
|
|
476
|
+
}
|
|
477
|
+
}
|
|
478
|
+
const reply = { space, vectors };
|
|
479
|
+
if (Object.keys(errors).length > 0)
|
|
480
|
+
reply.errors = errors;
|
|
481
|
+
return reply;
|
|
482
|
+
});
|
|
483
|
+
}
|
|
484
|
+
/**
|
|
485
|
+
* Handle ``chat.generate`` requests from code-joined peers β a stateless chat completion (no history
|
|
486
|
+
* broker-side), so a loginless peer gets an LLM reply without its own credentials. Reply:
|
|
487
|
+
* ``{ text, model }``, or ``{ error }`` if ``messages`` is malformed / over the payload cap. The
|
|
488
|
+
* **broker** pins ``model`` (the requester cannot choose it β cost control); ``maxTokens`` caps each
|
|
489
|
+
* reply. Announces the discoverable ``chat-broker`` role β call before ``connect()``. Every peer
|
|
490
|
+
* holding the connect code can trigger a reply on **your** billing. Mirror of Python's ``offer_chat``.
|
|
491
|
+
*/
|
|
492
|
+
offerChat(opts = {}) {
|
|
493
|
+
const client = this.deps.client;
|
|
494
|
+
this.requirePreConnect();
|
|
495
|
+
const maxTokens = opts.maxTokens ?? DEFAULT_CHAT_MAX_TOKENS;
|
|
496
|
+
this.announceRole(CHAT_BROKER_ROLE);
|
|
497
|
+
this.on("chat.generate", async (e) => {
|
|
498
|
+
let messages;
|
|
499
|
+
try {
|
|
500
|
+
messages = validateChatMessages(e.payload.messages);
|
|
501
|
+
}
|
|
502
|
+
catch (err) {
|
|
503
|
+
return { error: asError(err).message };
|
|
504
|
+
}
|
|
505
|
+
try {
|
|
506
|
+
const reply = await client.chat(messages, {
|
|
507
|
+
model: opts.model,
|
|
508
|
+
params: { max_tokens: maxTokens },
|
|
509
|
+
});
|
|
510
|
+
return { text: reply.content, model: reply.model };
|
|
511
|
+
}
|
|
512
|
+
catch (err) {
|
|
513
|
+
return { error: asError(err).message };
|
|
514
|
+
}
|
|
515
|
+
});
|
|
516
|
+
}
|
|
517
|
+
// ββ Observation dashboard + tracing (host-side helpers) ββββββββββββββββββ
|
|
518
|
+
/**
|
|
519
|
+
* Build the host-side {@link ../dash.Dashboard} for this room β the live observation-dashboard
|
|
520
|
+
* state an app holds as a grid of tiles. It answers each joining ``/tools/dashboard`` with a full
|
|
521
|
+
* ``dash.snapshot`` and broadcasts ``dash.delta`` patches as things change, all **ephemeral** (the
|
|
522
|
+
* room stores nothing β the durable record is the session).
|
|
523
|
+
*
|
|
524
|
+
* Prefer this over ``new Dashboard(room)``: it publishes over the REST ``/signal`` POST, which
|
|
525
|
+
* preserves the dialect's ephemerality (the WS inbound path forces ``ephemeral: false``). Mirror of
|
|
526
|
+
* Python's ``Dashboard(room)``.
|
|
527
|
+
*/
|
|
528
|
+
dashboard(opts = {}) {
|
|
529
|
+
const client = this.deps.client;
|
|
530
|
+
return new Dashboard(this, {
|
|
531
|
+
...opts,
|
|
532
|
+
publish: (type, payload, o) => client.request(buildRoomSignal(this.id, {
|
|
533
|
+
type,
|
|
534
|
+
payload,
|
|
535
|
+
to: o.to,
|
|
536
|
+
ephemeral: o.ephemeral,
|
|
537
|
+
connectionId: this.connectionId,
|
|
538
|
+
})),
|
|
539
|
+
});
|
|
540
|
+
}
|
|
541
|
+
/**
|
|
542
|
+
* Mirror the SDK's own activity into this room as ``trace.*`` signals β one line turns it on. The
|
|
543
|
+
* operation folds into the signal type (``label`` still rides the payload), and ephemerality is
|
|
544
|
+
* per-signal (see {@link ../trace.isTraceEphemeral}). While tracing:
|
|
545
|
+
*
|
|
546
|
+
* * **library/model calls** made through the same client (``search`` / ``searchByExamples`` /
|
|
547
|
+
* ``enrich`` / ``play`` / ``chat``) surface a ``trace.<label>`` (``trace.library.search`` /
|
|
548
|
+
* ``trace.chat`` / ``trace.play``) β a glanceable summary (``"rain" β 8 hits (top: Forest Rain
|
|
549
|
+
* 0.83)``) plus a capped detail (top-N results, never full payloads);
|
|
550
|
+
* * **SDK-internal state** β a recorder start/stop (with its session + event count), a transcriber
|
|
551
|
+
* start/stop, a standing offer's armed/active roster β surfaces a ``trace.state.<label>``.
|
|
552
|
+
*
|
|
553
|
+
* Default OFF (it broadcasts app internals to anyone with the code, and it's noise on the happy
|
|
554
|
+
* path). Returns a handle: ``const t = room.trace()`` then ``t.stop()``, or ``using t =
|
|
555
|
+
* room.trace()`` for a scope; stopped when the room closes. A trace failure never breaks the traced
|
|
556
|
+
* call. ``play`` and the state seam are scoped to **this** room; client-wide calls mirror into every
|
|
557
|
+
* armed room. Mirror of Python's ``room.trace()``.
|
|
558
|
+
*/
|
|
559
|
+
trace() {
|
|
560
|
+
const client = this.deps.client;
|
|
561
|
+
const handle = new RoomTrace({
|
|
562
|
+
// Trace signals ride the REST /signal POST, like the transcriber's: the WS inbound path forces
|
|
563
|
+
// `ephemeral: false`, and trace ephemerality is per-signal (a durable `trace.library.search` IS
|
|
564
|
+
// the study's record; an ephemeral `trace.play` must not double-record the play).
|
|
565
|
+
publish: (type, payload, o) => client.request(buildRoomSignal(this.id, {
|
|
566
|
+
type,
|
|
567
|
+
payload,
|
|
568
|
+
ephemeral: o.ephemeral,
|
|
569
|
+
connectionId: this.connectionId,
|
|
570
|
+
})),
|
|
571
|
+
onStop: () => {
|
|
572
|
+
remove(client.traceSinks, handle);
|
|
573
|
+
remove(this.traceHandles, handle);
|
|
574
|
+
},
|
|
575
|
+
});
|
|
576
|
+
client.traceSinks?.push(handle);
|
|
577
|
+
this.traceHandles.push(handle);
|
|
578
|
+
return handle;
|
|
579
|
+
}
|
|
580
|
+
// Emit a `trace.state.<label>` into every tracing session on this room (no-op when none β the
|
|
581
|
+
// zero-cost-when-off contract). Mirror of Python's `_trace_state`.
|
|
582
|
+
async traceState(label, summary, detail) {
|
|
583
|
+
if (this.traceHandles.length === 0)
|
|
584
|
+
return;
|
|
585
|
+
const payload = traceEnvelope(label, summary, detail);
|
|
586
|
+
for (const handle of [...this.traceHandles])
|
|
587
|
+
await handle.emit(traceStateType(label), payload);
|
|
588
|
+
}
|
|
589
|
+
// ββ Audio embedding (vectors + reactive scores) ββββββββββββββββββββββββββ
|
|
590
|
+
// Resolve the (space, model) pair for the audio surfaces (mirror of Python's `_resolve_audio_space`):
|
|
591
|
+
// an explicit space+model skips the lookup; otherwise the member-facing spaces list resolves the
|
|
592
|
+
// deployment's default audio space and/or reads the space's configured model.
|
|
593
|
+
async resolveAudioSpace(space, model) {
|
|
594
|
+
if (space !== undefined && model !== undefined)
|
|
595
|
+
return { space, model };
|
|
596
|
+
const spaces = await this.deps.client.library.spaces();
|
|
597
|
+
let matched;
|
|
598
|
+
if (space === undefined) {
|
|
599
|
+
matched = spaces.find((s) => s.defaultModalities.includes("audio"));
|
|
600
|
+
if (!matched) {
|
|
601
|
+
throw new Error("no default audio search space is configured on this deployment β pass space to audioVectors()");
|
|
602
|
+
}
|
|
603
|
+
space = matched.id;
|
|
604
|
+
}
|
|
605
|
+
else {
|
|
606
|
+
matched = spaces.find((s) => s.id === space);
|
|
607
|
+
if (!matched)
|
|
608
|
+
throw new Error(`space '${space}' not found`);
|
|
609
|
+
}
|
|
610
|
+
return { space, model: model ?? matched?.model ?? null };
|
|
611
|
+
}
|
|
612
|
+
// A dedicated snippet-consumer socket (announcing the demand role so a producer captures), decoding
|
|
613
|
+
// its binary frames β the shared source behind audioVectors/audioScores (mirror of audioClips).
|
|
614
|
+
async *snippetStream(identity, signal) {
|
|
615
|
+
this.requireWsAudio("live audio streaming");
|
|
616
|
+
const queue = [];
|
|
617
|
+
let wake = null;
|
|
618
|
+
let ended = false;
|
|
619
|
+
const socket = new RoomSocket({
|
|
620
|
+
roomId: this.id,
|
|
621
|
+
code: this.connectCode,
|
|
622
|
+
baseUrl: this.deps.baseUrl,
|
|
623
|
+
identity,
|
|
624
|
+
announce: { ...this.deps.announce, role: AUDIO_SNIPPET_CONSUMER_ROLE },
|
|
625
|
+
reconnect: this.deps.reconnect,
|
|
626
|
+
webSocketCtor: this.deps.webSocketCtor,
|
|
627
|
+
onBinary: (data) => {
|
|
628
|
+
const frame = decodeAudioSnippetFrame(data);
|
|
629
|
+
if (!frame)
|
|
630
|
+
return; // a foreign / DSP binary frame β not ours
|
|
631
|
+
if (queue.length >= RAW_CONSUMER_QUEUE_MAX)
|
|
632
|
+
queue.shift(); // drop oldest β bound a slow consumer
|
|
633
|
+
queue.push(frame);
|
|
634
|
+
wake?.();
|
|
635
|
+
wake = null;
|
|
636
|
+
},
|
|
637
|
+
onEnd: () => {
|
|
638
|
+
ended = true;
|
|
639
|
+
wake?.();
|
|
640
|
+
wake = null;
|
|
641
|
+
},
|
|
642
|
+
});
|
|
643
|
+
// An abort (a `transcribe()` handle's stop) closes the socket β ends this generator at once.
|
|
644
|
+
const onAbort = () => {
|
|
645
|
+
ended = true;
|
|
646
|
+
socket.close();
|
|
647
|
+
wake?.();
|
|
648
|
+
wake = null;
|
|
649
|
+
};
|
|
650
|
+
if (signal) {
|
|
651
|
+
if (signal.aborted)
|
|
652
|
+
onAbort();
|
|
653
|
+
else
|
|
654
|
+
signal.addEventListener("abort", onAbort);
|
|
655
|
+
}
|
|
656
|
+
try {
|
|
657
|
+
try {
|
|
658
|
+
await socket.connect();
|
|
659
|
+
}
|
|
660
|
+
catch (err) {
|
|
661
|
+
// A pre-/mid-connect abort closed the socket β connect() rejects "room closed"; that's an
|
|
662
|
+
// expected teardown, not a failure. Only a genuine connect error (bad code) propagates.
|
|
663
|
+
if (!ended)
|
|
664
|
+
throw err;
|
|
665
|
+
}
|
|
666
|
+
while (!ended) {
|
|
667
|
+
while (queue.length > 0)
|
|
668
|
+
yield queue.shift();
|
|
669
|
+
if (ended)
|
|
670
|
+
break;
|
|
671
|
+
await new Promise((resolve) => {
|
|
672
|
+
wake = resolve;
|
|
673
|
+
});
|
|
674
|
+
}
|
|
675
|
+
while (queue.length > 0)
|
|
676
|
+
yield queue.shift();
|
|
677
|
+
}
|
|
678
|
+
finally {
|
|
679
|
+
signal?.removeEventListener("abort", onAbort);
|
|
680
|
+
socket.close();
|
|
681
|
+
}
|
|
682
|
+
}
|
|
683
|
+
/**
|
|
684
|
+
* Subscribe to the room's live ``audio.snippet`` stream and embed each ~10s window, yielding an
|
|
685
|
+
* {@link AudioVector} per snippet β a live vector of "what the room sounds like right now".
|
|
686
|
+
* Subscribing *is* the demand signal (announces the ``audio-snippet-consumer`` role, so a
|
|
687
|
+
* producer's default-off "Share audio clips" toggle actually captures). Break out of the
|
|
688
|
+
* ``for await`` loop to end it. Mirror of Python's ``audio_vectors``.
|
|
689
|
+
*
|
|
690
|
+
* ``space`` pins the search space snippets are labeled with (default: the deployment's default
|
|
691
|
+
* audio space, so a no-arg vector is comparable against library-derived vectors); ``model`` the
|
|
692
|
+
* embedding model (default: the resolved space's configured model). ``relay: true`` also relays
|
|
693
|
+
* each vector into the room as an ephemeral ``audio.vector`` signal β so a loginless code-joined
|
|
694
|
+
* peer can consume the live mood vector without credentials of its own. Binary rides the
|
|
695
|
+
* WebSocket only.
|
|
696
|
+
*/
|
|
697
|
+
async *audioVectors(opts = {}) {
|
|
698
|
+
const client = this.deps.client;
|
|
699
|
+
const { space, model } = await this.resolveAudioSpace(opts.space, opts.model);
|
|
700
|
+
if (model === null) {
|
|
701
|
+
throw new Error(`the audio search space '${space}' has no model configured β pass model to audioVectors()`);
|
|
702
|
+
}
|
|
703
|
+
for await (const frame of this.snippetStream(`audio-vectors:${randomId().slice(0, 8)}`)) {
|
|
704
|
+
const fmt = frame.mimeType.split(";")[0]?.split("/").pop();
|
|
705
|
+
const vector = await client.embed({
|
|
706
|
+
audio: new Blob([frame.data], { type: frame.mimeType }),
|
|
707
|
+
model,
|
|
708
|
+
format: fmt,
|
|
709
|
+
});
|
|
710
|
+
const vec = {
|
|
711
|
+
vector,
|
|
712
|
+
space,
|
|
713
|
+
sequence: frame.sequence,
|
|
714
|
+
windowSeconds: frame.windowSeconds,
|
|
715
|
+
timestamp: Date.now() / 1000,
|
|
716
|
+
};
|
|
717
|
+
if (opts.relay) {
|
|
718
|
+
// Over the REST /signal POST, like the dashboard's and the transcriber's publishes: the WS
|
|
719
|
+
// inbound path forces ``ephemeral: false``, and a durable relay would write one full vector
|
|
720
|
+
// per ~10s window into every recording session.
|
|
721
|
+
await client.request(buildRoomSignal(this.id, {
|
|
722
|
+
type: "audio.vector",
|
|
723
|
+
payload: {
|
|
724
|
+
vector: vec.vector,
|
|
725
|
+
space: vec.space,
|
|
726
|
+
sequence: vec.sequence,
|
|
727
|
+
window_seconds: vec.windowSeconds,
|
|
728
|
+
timestamp: vec.timestamp,
|
|
729
|
+
},
|
|
730
|
+
ephemeral: true,
|
|
731
|
+
connectionId: this.connectionId,
|
|
732
|
+
}));
|
|
733
|
+
}
|
|
734
|
+
yield vec;
|
|
735
|
+
}
|
|
736
|
+
}
|
|
737
|
+
async preflightConceptAnchors(members, space) {
|
|
738
|
+
// Resolve every concept anchor's vector once before waiting on any frame β a typo'd/unresolvable
|
|
739
|
+
// concept fails immediately, naming the anchor, rather than surfacing mid-stream.
|
|
740
|
+
const client = this.deps.client;
|
|
741
|
+
for (const [name, ops] of Object.entries(members)) {
|
|
742
|
+
for (const op of ops) {
|
|
743
|
+
if (op.kind !== "concept")
|
|
744
|
+
continue;
|
|
745
|
+
const query = { concept: op.name, space };
|
|
746
|
+
if (op.collection !== undefined)
|
|
747
|
+
query.collection = op.collection;
|
|
748
|
+
try {
|
|
749
|
+
await client.request({ method: "GET", path: "/v1/library/concept-vector", query });
|
|
750
|
+
}
|
|
751
|
+
catch (err) {
|
|
752
|
+
throw new Error(`anchor '${name}': ${err.message}`);
|
|
753
|
+
}
|
|
754
|
+
}
|
|
755
|
+
}
|
|
756
|
+
}
|
|
757
|
+
/**
|
|
758
|
+
* Fuse {@link audioVectors} with per-anchor ``library.compare`` scoring β a live "which anchor does
|
|
759
|
+
* the room sound like right now" signal. ``anchors`` maps a name to anything the ``by`` grammar
|
|
760
|
+
* accepts (a bare string / {@link "../by".Asset} / ``by.*`` marker), **or an array** of those (a
|
|
761
|
+
* few-shot category, scored as the mean of its top-``topN`` members' calibrated compare scores;
|
|
762
|
+
* default ``topN: 1`` β the nearest member wins). Each frame's anchors are scored concurrently.
|
|
763
|
+
*
|
|
764
|
+
* Each {@link ScoredAudio} carries the calibrated ``scores`` plus ``mix`` (normalized to sum 1,
|
|
765
|
+
* uniform when every score is 0 β gate on ``max(scores)`` for silence). No smoothing β ease the
|
|
766
|
+
* stream with a {@link ../decider.Decider}. ``relay: true`` forwards to {@link audioVectors} (it
|
|
767
|
+
* relays the raw ``audio.vector``, not the computed scores). Concept anchors are validated once up
|
|
768
|
+
* front (a typo raises before any frame); a mid-stream compare failure re-throws named by anchor.
|
|
769
|
+
* Mirror of Python's ``audio_scores``.
|
|
770
|
+
*/
|
|
771
|
+
async *audioScores(anchors, opts = {}) {
|
|
772
|
+
const client = this.deps.client;
|
|
773
|
+
const names = Object.keys(anchors);
|
|
774
|
+
const members = {};
|
|
775
|
+
for (const name of names)
|
|
776
|
+
members[name] = toAnchorMembers(anchors[name]);
|
|
777
|
+
const { space, model } = await this.resolveAudioSpace(opts.space, opts.model);
|
|
778
|
+
if (model === null) {
|
|
779
|
+
throw new Error(`the audio search space '${space}' has no model configured β pass model to audioScores()`);
|
|
780
|
+
}
|
|
781
|
+
await this.preflightConceptAnchors(members, space);
|
|
782
|
+
const topN = Math.max(1, opts.topN ?? 1);
|
|
783
|
+
for await (const vec of this.audioVectors({ space, model, relay: opts.relay })) {
|
|
784
|
+
const vectorOp = by.vector(vec.vector);
|
|
785
|
+
// Score every member of every anchor concurrently β a handful of compares is cheap vs. the ~10s
|
|
786
|
+
// window. Flatten to (name, op) pairs, then fold back into per-anchor top-N means below.
|
|
787
|
+
const flat = names.flatMap((name) => members[name].map((op) => ({ name, op })));
|
|
788
|
+
const results = await Promise.all(flat.map(async ({ name, op }) => {
|
|
789
|
+
try {
|
|
790
|
+
const r = await client.library.compare(vectorOp, op, { space: vec.space });
|
|
791
|
+
return r.score;
|
|
792
|
+
}
|
|
793
|
+
catch (err) {
|
|
794
|
+
throw new Error(`anchor '${name}': ${err.message}`);
|
|
795
|
+
}
|
|
796
|
+
}));
|
|
797
|
+
const scores = {};
|
|
798
|
+
let i = 0;
|
|
799
|
+
for (const name of names) {
|
|
800
|
+
const n = members[name].length;
|
|
801
|
+
const top = results
|
|
802
|
+
.slice(i, i + n)
|
|
803
|
+
.sort((a, b) => b - a)
|
|
804
|
+
.slice(0, topN);
|
|
805
|
+
scores[name] = top.reduce((a, b) => a + b, 0) / top.length;
|
|
806
|
+
i += n;
|
|
807
|
+
}
|
|
808
|
+
const total = Object.values(scores).reduce((a, b) => a + b, 0);
|
|
809
|
+
const mix = {};
|
|
810
|
+
for (const name of names) {
|
|
811
|
+
mix[name] = total > 0 ? scores[name] / total : 1 / names.length;
|
|
812
|
+
}
|
|
813
|
+
yield {
|
|
814
|
+
scores,
|
|
815
|
+
mix,
|
|
816
|
+
vector: vec.vector,
|
|
817
|
+
space: vec.space,
|
|
818
|
+
sequence: vec.sequence,
|
|
819
|
+
windowSeconds: vec.windowSeconds,
|
|
820
|
+
timestamp: vec.timestamp,
|
|
821
|
+
};
|
|
822
|
+
}
|
|
823
|
+
}
|
|
824
|
+
// ββ Management (authenticated handles only) ββββββββββββββββββββββββββββββ
|
|
825
|
+
/** Rotate the connect code (owner only); updates ``connectCode`` and returns it. */
|
|
826
|
+
async rotateCode() {
|
|
827
|
+
const data = (await this.deps.rest(buildRoomRotateCode(this.id)));
|
|
828
|
+
this.connectCode = data.connect_code ?? this.connectCode;
|
|
829
|
+
return this.connectCode;
|
|
830
|
+
}
|
|
831
|
+
/** Delete the room for good (its connect code dies with it). */
|
|
832
|
+
async delete() {
|
|
833
|
+
await this.deps.rest({ method: "DELETE", path: `/v1/rooms/${this.id}` });
|
|
834
|
+
}
|
|
835
|
+
// The base tears the transport + recorders down; the recorder-registration seam is this class's,
|
|
836
|
+
// so it is dropped here (a listener outliving the handle would be notified about nothing).
|
|
837
|
+
teardown() {
|
|
838
|
+
// Base first: stopping the recorders fires the seam one last time, and only then is it dropped.
|
|
839
|
+
const done = super.teardown();
|
|
840
|
+
this.recorderListeners.clear();
|
|
841
|
+
return done;
|
|
842
|
+
}
|
|
843
|
+
}
|
|
844
|
+
//# sourceMappingURL=owned-room.js.map
|