@vocalstack/js-sdk 1.0.13 → 2.0.0-alpha.0

package/dist/realtime.js

@@ -0,0 +1,450 @@
+/**
+ * The realtime session: `Realtime` (`vs.realtime`: create a session, or
+ * transcribe a file or a URL over one) and `RealtimeSession` (its socket).
+ */
+import { parseServerFrame } from '@vocalstack/contract';
+import { capacityExceeded, fromEnvelope, VocalStackError } from './errors.js';
+import { fileChunks } from './file.js';
+import { openWebSocket, WS_OPEN } from './ws.js';
+const DEFAULT_READY_DEADLINE_MS = 30_000;
+/** A refused socket upgrade as an error.  A 503 carrying the error body
+ * with `unavailable` (the database is waking; retry shortly) says so; any
+ * other 503 — a bare one from a saturated proxy, or no body at all — is
+ * `capacity_exceeded`.  So is a 502 or 504 without the API's error body:
+ * the proxy in front of the engines found none free in time.  Every other
+ * status is its own envelope. */
+export function upgradeRefused(status, body) {
+    let parsed;
+    try {
+        parsed = body ? JSON.parse(body) : undefined;
+    }
+    catch {
+        parsed = undefined;
+    }
+    if (status === 503) {
+        const code = parsed?.code;
+        if (code === 'unavailable')
+            return fromEnvelope(parsed, status);
+        return capacityExceeded('stream host answered 503 (fleet saturated)');
+    }
+    const ours = typeof parsed?.code === 'string';
+    if ((status === 502 || status === 504) && !ours)
+        return capacityExceeded(`stream host answered ${status} (no engine answered in time)`);
+    return fromEnvelope(parsed, status);
+}
+export class RealtimeSession {
+    sessionId;
+    #ws;
+    #queue = [];
+    #wake = null;
+    #endReason = null;
+    #stats = null;
+    #error = null;
+    #closed = false;
+    #onFrame;
+    constructor(sessionId, ws, onFrame) {
+        this.sessionId = sessionId;
+        this.#ws = ws;
+        this.#onFrame = onFrame;
+    }
+    static async connect(params) {
+        const deadlineMs = params.readyDeadlineMs ?? DEFAULT_READY_DEADLINE_MS;
+        const ws = await openWebSocket(params.streamUrl);
+        ws.binaryType = 'arraybuffer';
+        const session = new RealtimeSession(params.sessionId, ws, params.onFrame);
+        await new Promise((resolve, reject) => {
+            let settled = false;
+            let sawFrame = false;
+            let rejecting = false;
+            let timer;
+            const fail = (err) => {
+                if (settled)
+                    return;
+                settled = true;
+                clearTimeout(timer);
+                try {
+                    ws.close();
+                }
+                catch {
+                }
+                reject(err);
+            };
+            const arm = () => {
+                clearTimeout(timer);
+                timer = setTimeout(() => {
+                    fail(sawFrame
+                        ? new VocalStackError('engine_unavailable', `engine did not become ready within ${deadlineMs}ms`, { retryable: true })
+                        : capacityExceeded(`no response from the engine within ${deadlineMs}ms — every machine is ` +
+                            `busy with another session, or one is stuck holding a slot`));
+                }, deadlineMs);
+            };
+            arm();
+            ws.addEventListener('message', (ev) => {
+                // Once ready, #listen reads the socket; this one stops re-arming
+                // the deadline (it used to leave a timer running for 30 s).
+                if (settled || typeof ev.data !== 'string')
+                    return;
+                sawFrame = true;
+                arm();
+                let frame;
+                try {
+                    frame = parseServerFrame(ev.data);
+                }
+                catch {
+                    return;
+                }
+                if (!frame)
+                    return;
+                session.#frame(frame);
+                if (frame.type === 'session.loading') {
+                    params.onLoading?.(frame.stage);
+                }
+                else if (frame.type === 'session.ready') {
+                    if (!settled) {
+                        settled = true;
+                        clearTimeout(timer);
+                        session.#listen();
+                        resolve();
+                    }
+                }
+                else if (frame.type === 'error') {
+                    fail(new VocalStackError(frame.code, frame.message, {
+                        retryable: frame.retryable,
+                    }));
+                }
+            });
+            const emitterOn = ws
+                .on?.bind(ws);
+            emitterOn?.('unexpected-response', (_req, res) => {
+                rejecting = true;
+                const status = res?.statusCode;
+                let body = '';
+                if (res && typeof res.on === 'function') {
+                    res.on('data', (c) => (body += c.toString()));
+                    res.on('end', () => fail(upgradeRefused(status, body)));
+                }
+                else {
+                    fail(upgradeRefused(status, ''));
+                }
+            });
+            ws.addEventListener('close', () => {
+                if (rejecting)
+                    return;
+                // No HTTP answer at all (a refused upgrade is `unexpected-response`
+                // above): the connection dropped on the way, as the Python and PHP
+                // SDKs say it — retryable, not a bad token.
+                fail(new VocalStackError('engine_unavailable', 'connection closed before ready', {
+                    retryable: true,
+                }));
+            });
+            ws.addEventListener('error', () => {
+            });
+        });
+        return session;
+    }
+    /** Hands a frame to `onFrame`; a throwing callback never breaks the socket. */
+    #frame(frame) {
+        try {
+            this.#onFrame?.(frame);
+        }
+        catch {
+        }
+    }
+    #listen() {
+        this.#ws.addEventListener('message', (ev) => {
+            if (typeof ev.data !== 'string')
+                return;
+            let frame;
+            try {
+                frame = parseServerFrame(ev.data);
+            }
+            catch {
+                return;
+            }
+            if (!frame)
+                return;
+            this.#frame(frame);
+            switch (frame.type) {
+                case 'transcript.committed':
+                case 'transcript.draft':
+                case 'transcript.words':
+                case 'transcript.translated':
+                case 'transcript.amended':
+                case 'transcript.latency':
+                case 'transcript.speaker':
+                case 'transcript.sentence':
+                case 'transcript.progress':
+                    this.#queue.push(frame);
+                    this.#wake?.();
+                    break;
+                case 'session.done':
+                    this.#endReason = frame.reason;
+                    this.#stats = frame.stats;
+                    this.#wake?.();
+                    break;
+                case 'error':
+                    this.#error = new VocalStackError(frame.code, frame.message, {
+                        retryable: frame.retryable,
+                    });
+                    this.#wake?.();
+                    break;
+                default:
+                    break;
+            }
+        });
+        this.#ws.addEventListener('close', () => {
+            this.#closed = true;
+            this.#wake?.();
+        });
+    }
+    sendAudio(chunk) {
+        if (this.#ws.readyState !== WS_OPEN)
+            return;
+        const data = chunk instanceof Float32Array
+            ? new Uint8Array(chunk.buffer, chunk.byteOffset, chunk.byteLength)
+            : chunk;
+        this.#ws.send(data);
+    }
+    async sendFile(source, chunkBytes = 256 * 1024) {
+        const MAX_BUFFERED = 4 * 1024 * 1024;
+        for await (const chunk of source) {
+            for (let off = 0; off < chunk.byteLength; off += chunkBytes) {
+                if (this.#ws.readyState !== WS_OPEN)
+                    return;
+                while (this.#ws.bufferedAmount > MAX_BUFFERED) {
+                    await new Promise((r) => setTimeout(r, 10));
+                    if (this.#ws.readyState !== WS_OPEN)
+                        return;
+                }
+                this.#ws.send(chunk.subarray(off, Math.min(off + chunkBytes, chunk.byteLength)));
+            }
+        }
+        this.eof();
+    }
+    async pipe(source) {
+        const iter = Symbol.asyncIterator in source
+            ? source
+            : readableToAsyncIterable(source);
+        for await (const chunk of iter) {
+            this.sendAudio(chunk);
+        }
+        this.eof();
+    }
+    /**
+     * Capture clock heartbeat: `pcm` = cumulative samples captured so far
+     * (16kHz-equivalent), `wall` = seconds since capture start.  Lets the
+     * server detect capture loss (throttled tab, device switch) and mark
+     * the transcript with honest gaps instead of gluing across them.
+     * `streamMicrophone`/`captureMicrophone` call this automatically.
+     */
+    sendClock(pcm, wall) {
+        if (this.#ws.readyState === WS_OPEN) {
+            this.#ws.send(JSON.stringify({ type: 'clock', pcm: Math.floor(pcm), wall }));
+        }
+    }
+    /**
+     * Options update during the realtime session. Omitted fields are unchanged.
+     * `latencyMode` switches the decode model policy from the next
+     * engine step; `spokenLanguage` changes the transcription language from the
+     * next engine step (`''` or `'auto'` = auto-detect); `speakerLabels`
+     * turns speaker labelling on (`'on'` or `'auto'`) or off (`'off'`) from the next
+     * engine step — enabling labels speakers from that point on;
+     * `translate` replaces the whole target-language list (`[]` turns
+     * translation off). Applies to subsequent transcript —
+     * already-delivered segments and settled sentences are not
+     * re-transcribed, re-attributed or re-translated.
+     */
+    update(changes) {
+        if (this.#ws.readyState !== WS_OPEN)
+            return;
+        this.#ws.send(JSON.stringify({
+            type: 'session.update',
+            ...(changes.latencyMode !== undefined && { latencyMode: changes.latencyMode }),
+            ...(changes.spokenLanguage !== undefined && { spokenLanguage: changes.spokenLanguage }),
+            ...(changes.speakerLabels !== undefined && { speakerLabels: changes.speakerLabels }),
+            ...(changes.translate !== undefined && { translate: changes.translate }),
+        }));
+    }
+    eof() {
+        if (this.#ws.readyState === WS_OPEN) {
+            this.#ws.send(JSON.stringify({ type: 'eof' }));
+        }
+    }
+    cancel() {
+        if (this.#ws.readyState === WS_OPEN) {
+            this.#ws.send(JSON.stringify({ type: 'cancel' }));
+        }
+    }
+    get doneReason() {
+        return this.#endReason;
+    }
+    get stats() {
+        return this.#stats;
+    }
+    close() {
+        this.#closed = true;
+        try {
+            this.#ws.close();
+        }
+        catch {
+        }
+        this.#wake?.();
+    }
+    async *[Symbol.asyncIterator]() {
+        for (;;) {
+            const item = this.#queue.shift();
+            if (item) {
+                yield item;
+                continue;
+            }
+            if (this.#error)
+                throw this.#error;
+            if (this.#endReason || this.#closed)
+                return;
+            await new Promise((resolve) => {
+                this.#wake = resolve;
+            });
+            this.#wake = null;
+        }
+    }
+    [Symbol.dispose]() {
+        this.close();
+    }
+}
+async function* readableToAsyncIterable(stream) {
+    const reader = stream.getReader();
+    try {
+        for (;;) {
+            const { done, value } = await reader.read();
+            if (done)
+                return;
+            if (value)
+                yield value;
+        }
+    }
+    finally {
+        reader.releaseLock();
+    }
+}
+/** `vs.realtime`: a realtime session is a signed token plus one socket.
+ * Nothing is stored: the transcript is what the socket sends. */
+export class Realtime {
+    #t;
+    constructor(t) {
+        this.#t = t;
+    }
+    /** `POST /v1/realtime/sessions`: validate the options, get a signed `streamUrl`. */
+    createSession(options = {}, call = {}) {
+        return this.#t.json('POST', '/v1/realtime/sessions', { body: options, ...call });
+    }
+    /** `createSession`, then open the socket and wait for `session.ready`. */
+    async connect(options = {}, connect = {}) {
+        const created = await this.createSession(options);
+        return RealtimeSession.connect({
+            sessionId: created.sessionId,
+            streamUrl: created.streamUrl,
+            options,
+            readyDeadlineMs: connect.readyDeadlineMs ?? this.#t.opts.readyDeadlineMs,
+            onLoading: connect.onLoading,
+            onFrame: connect.onFrame,
+        });
+    }
+    /** `GET /v1/realtime/capabilities`: languages, audio formats, limits. */
+    capabilities() {
+        return this.#t.json('GET', '/v1/realtime/capabilities');
+    }
+    /** Stream an encoded file (mp3, wav, flac, ogg, m4a…) through a realtime
+     * session and collect the transcript.  Nothing is stored; for a stored,
+     * shareable transcription use `vs.transcriptions.create`. */
+    async transcribeFile(file, opts = {}) {
+        const { onSession, readyDeadlineMs, handlers, options } = split(opts);
+        const session = await this.connect({ ...options, input: { kind: 'file' } }, { readyDeadlineMs });
+        onSession?.(session);
+        await session.sendFile(fileChunks(file));
+        return collectTranscript(session, handlers);
+    }
+    /** The same for a link the engine fetches itself. */
+    async transcribeUrl(url, opts = {}) {
+        const { onSession, readyDeadlineMs, handlers, options } = split(opts);
+        const session = await this.connect({ ...options, input: { kind: 'url', url } }, { readyDeadlineMs });
+        onSession?.(session);
+        return collectTranscript(session, handlers);
+    }
+}
+function split(opts) {
+    const { onProgress, onCommit, onSession, onTranslated, onAmended, onSpeakerSettled, onSentence, readyDeadlineMs, ...options } = opts;
+    const handlers = { onProgress, onCommit, onTranslated, onAmended, onSpeakerSettled, onSentence };
+    return { onSession, readyDeadlineMs, handlers, options };
+}
+/** Upsert one sentence frame into a list kept in sentenceId order. */
+export function applySentence(list, sentence) {
+    const i = list.findIndex((s) => s.sentenceId === sentence.sentenceId);
+    if (sentence.status === 'dropped') {
+        return i === -1 ? list : [...list.slice(0, i), ...list.slice(i + 1)];
+    }
+    if (i === -1) {
+        const next = [...list, sentence];
+        next.sort((a, b) => a.sentenceId - b.sentenceId);
+        return next;
+    }
+    const next = list.slice();
+    next[i] = sentence;
+    return next;
+}
+async function collectTranscript(session, h) {
+    const segments = [];
+    let sentences = [];
+    for await (const event of session) {
+        if (event.type === 'transcript.committed') {
+            segments.push(event.segment);
+            h.onCommit?.(event.segment);
+        }
+        else if (event.type === 'transcript.translated') {
+            const seg = segments.find((s) => s.seq === event.seq);
+            if (seg) {
+                seg.translations = event.translations;
+                seg.translationSources = event.sources;
+            }
+            h.onTranslated?.(event.seq, event.translations, event.sources ?? undefined);
+        }
+        else if (event.type === 'transcript.amended') {
+            const seg = segments.find((s) => s.seq === event.seq);
+            if (seg) {
+                seg.text = event.text;
+                if (event.speaker >= 0)
+                    seg.speaker = event.speaker;
+            }
+            h.onAmended?.(event.seq, event.text, event.speaker, event.fixes ?? []);
+        }
+        else if (event.type === 'transcript.latency') {
+            const seg = segments.find((s) => s.seq === event.seq);
+            if (seg)
+                seg.latencyS = event.latencyS;
+        }
+        else if (event.type === 'transcript.speaker') {
+            const seg = segments.find((s) => s.seq === event.seq);
+            if (seg) {
+                if (event.speaker >= 0)
+                    seg.speaker = event.speaker;
+                if (event.speakerConfidence != null)
+                    seg.speakerConfidence = event.speakerConfidence;
+                seg.speakerProvisional = false;
+            }
+            h.onSpeakerSettled?.(event.seq, event.speaker, event.verdict, event.speakerConfidence ?? undefined);
+        }
+        else if (event.type === 'transcript.sentence') {
+            sentences = applySentence(sentences, event.sentence);
+            h.onSentence?.(event.sentence);
+        }
+        else if (event.type === 'transcript.progress') {
+            h.onProgress?.({ processedS: event.processedS, totalS: event.totalS ?? undefined });
+        }
+    }
+    return {
+        segments,
+        sentences,
+        text: (sentences.length ? sentences : segments).map((s) => s.text).join(' '),
+        doneReason: session.doneReason,
+        stats: session.stats,
+    };
+}