@vocalstack/js-sdk 2.0.0-alpha.0 → 2.0.0-alpha.2

package/dist/realtime.d.ts

@@ -1,4 +1,4 @@
-import type { Capabilities, CommittedSegment, CreateSessionResponse, Fix, LatencyMode, SentenceSegment, ServerFrame, SessionOptions, SessionStats, SpeakerLabels } from '@vocalstack/contract';
+import type { Capabilities, CommittedSegment, CreateSessionResponse, Fix, LatencyMode, SentenceSegment, ServerFrame, SessionOptions, SessionStats, SpeakerLabels, WarmRequest } from '@vocalstack/contract';
 import { VocalStackError } from './errors.js';
 import { type FileInput } from './file.js';
 import type { CallOptions, Transport } from './http.js';
@@ -10,26 +10,12 @@ export interface ConnectParams {
     streamUrl: string;
     options?: SessionOptions;
     readyDeadlineMs?: number;
-    /** Fires on every `session.loading` frame while the engine warms up
-     * (model download / load after a machine wake) — lets a UI show
-     * "warming up" instead of a generic connecting state.  Each frame also
-     * re-arms the ready deadline, so a long cold load only fails once the
-     * engine goes silent, not merely slow. */
     onLoading?(stage: 'download' | 'model'): void;
-    /** Every server text frame as it arrives, parsed, before the iterator
-     * sees it: the lifecycle frames the iterator keeps to itself
-     * (`session.loading`, `session.ready`, `session.done`, `error`) and the
-     * transcript frames alike.  For logs and protocol views; the iterator
-     * is still the way to read the transcript. */
     onFrame?(frame: ServerFrame): void;
 }
-/** A refused socket upgrade as an error.  A 503 carrying the error body
- * with `unavailable` (the database is waking; retry shortly) says so; any
- * other 503 — a bare one from a saturated proxy, or no body at all — is
- * `capacity_exceeded`.  So is a 502 or 504 without the API's error body:
- * the proxy in front of the engines found none free in time.  Every other
- * status is its own envelope. */
+export declare function engineLost(closeCode?: number): VocalStackError;
 export declare function upgradeRefused(status: number | undefined, body: string): VocalStackError;
+export declare const RESTART_CLOSE = 1012;
 export declare class RealtimeSession implements AsyncIterable<SessionEvent> {
     #private;
     readonly sessionId: string;
@@ -38,26 +24,7 @@ export declare class RealtimeSession implements AsyncIterable<SessionEvent> {
     sendAudio(chunk: Float32Array | ArrayBuffer | Uint8Array): void;
     sendFile(source: AsyncIterable<Uint8Array>, chunkBytes?: number): Promise<void>;
     pipe(source: AsyncIterable<Float32Array | ArrayBuffer | Uint8Array> | ReadableStream<Uint8Array>): Promise<void>;
-    /**
-     * Capture clock heartbeat: `pcm` = cumulative samples captured so far
-     * (16kHz-equivalent), `wall` = seconds since capture start.  Lets the
-     * server detect capture loss (throttled tab, device switch) and mark
-     * the transcript with honest gaps instead of gluing across them.
-     * `streamMicrophone`/`captureMicrophone` call this automatically.
-     */
     sendClock(pcm: number, wall: number): void;
-    /**
-     * Options update during the realtime session. Omitted fields are unchanged.
-     * `latencyMode` switches the decode model policy from the next
-     * engine step; `spokenLanguage` changes the transcription language from the
-     * next engine step (`''` or `'auto'` = auto-detect); `speakerLabels`
-     * turns speaker labelling on (`'on'` or `'auto'`) or off (`'off'`) from the next
-     * engine step — enabling labels speakers from that point on;
-     * `translate` replaces the whole target-language list (`[]` turns
-     * translation off). Applies to subsequent transcript —
-     * already-delivered segments and settled sentences are not
-     * re-transcribed, re-attributed or re-translated.
-     */
     update(changes: {
         latencyMode?: LatencyMode;
         spokenLanguage?: string;
@@ -73,60 +40,38 @@ export declare class RealtimeSession implements AsyncIterable<SessionEvent> {
     [Symbol.dispose](): void;
 }
 export interface ConnectOptions {
-    /** See {@link ConnectParams.onLoading}: engine warm-up progress. */
     onLoading?(stage: 'download' | 'model'): void;
-    /** See {@link ConnectParams.onFrame}: every server frame, lifecycle included. */
     onFrame?(frame: ServerFrame): void;
-    /** Overrides the client-wide `readyDeadlineMs` for this socket. */
     readyDeadlineMs?: number;
 }
 export interface TranscribeOptions extends Omit<SessionOptions, 'input' | 'audio'> {
-    /** The realtime session, once connected (to `cancel()` it, say). */
     onSession?(session: RealtimeSession): void;
     onProgress?(progress: {
         processedS: number;
         totalS?: number;
     }): void;
     onCommit?(segment: CommittedSegment): void;
-    /** A committed segment's translation arrived; the segment (by `seq`)
-     * already has its `translations` filled in. */
     onTranslated?(seq: number, translations: Record<string, string>, sources?: Record<string, string>): void;
-    /** The engine rewrote a committed segment; it is patched in place first. */
     onAmended?(seq: number, text: string, speaker: number, fixes: Fix[]): void;
-    /** A tentative speaker label (`speakerProvisional`) settled: `verdict` is
-     * "confirmed", "amended" or "unverified".  The segment is patched first. */
     onSpeakerSettled?(seq: number, speaker: number, verdict: string, confidence?: number): void;
-    /** Every sentence upsert (`sentences.enabled`); the returned `sentences`
-     * list is kept up to date (dropped sentences are removed). */
     onSentence?(sentence: SentenceSegment): void;
     readyDeadlineMs?: number;
 }
 export interface RealtimeTranscript {
     segments: CommittedSegment[];
-    /** Sentence upserts folded into their final state (empty unless
-     * `sentences.enabled`). */
     sentences: SentenceSegment[];
     text: string;
     doneReason: 'eof' | 'max_duration' | 'cancelled' | null;
     stats: SessionStats | null;
 }
-/** `vs.realtime`: a realtime session is a signed token plus one socket.
- * Nothing is stored: the transcript is what the socket sends. */
 export declare class Realtime {
     #private;
     constructor(t: Transport);
-    /** `POST /v1/realtime/sessions`: validate the options, get a signed `streamUrl`. */
     createSession(options?: SessionOptions, call?: CallOptions): Promise<CreateSessionResponse>;
-    /** `createSession`, then open the socket and wait for `session.ready`. */
     connect(options?: SessionOptions, connect?: ConnectOptions): Promise<RealtimeSession>;
-    /** `GET /v1/realtime/capabilities`: languages, audio formats, limits. */
     capabilities(): Promise<Capabilities>;
-    /** Stream an encoded file (mp3, wav, flac, ogg, m4a…) through a realtime
-     * session and collect the transcript.  Nothing is stored; for a stored,
-     * shareable transcription use `vs.transcriptions.create`. */
+    warm(req: WarmRequest): Promise<void>;
     transcribeFile(file: FileInput, opts?: TranscribeOptions): Promise<RealtimeTranscript>;
-    /** The same for a link the engine fetches itself. */
     transcribeUrl(url: string, opts?: TranscribeOptions): Promise<RealtimeTranscript>;
 }
-/** Upsert one sentence frame into a list kept in sentenceId order. */
 export declare function applySentence(list: SentenceSegment[], sentence: SentenceSegment): SentenceSegment[];

package/dist/realtime.js

@@ -1,18 +1,12 @@
-/**
- * The realtime session: `Realtime` (`vs.realtime`: create a session, or
- * transcribe a file or a URL over one) and `RealtimeSession` (its socket).
- */
 import { parseServerFrame } from '@vocalstack/contract';
 import { capacityExceeded, fromEnvelope, VocalStackError } from './errors.js';
 import { fileChunks } from './file.js';
 import { openWebSocket, WS_OPEN } from './ws.js';
 const DEFAULT_READY_DEADLINE_MS = 30_000;
-/** A refused socket upgrade as an error.  A 503 carrying the error body
- * with `unavailable` (the database is waking; retry shortly) says so; any
- * other 503 — a bare one from a saturated proxy, or no body at all — is
- * `capacity_exceeded`.  So is a 502 or 504 without the API's error body:
- * the proxy in front of the engines found none free in time.  Every other
- * status is its own envelope. */
+export function engineLost(closeCode) {
+    const how = closeCode ? ` (close ${closeCode})` : '';
+    return new VocalStackError('engine_lost', `The connection to the transcription engine closed before the realtime session finished${how}; open a new realtime session to continue.`, { retryable: true });
+}
 export function upgradeRefused(status, body) {
     let parsed;
     try {
@@ -32,6 +26,96 @@ export function upgradeRefused(status, body) {
         return capacityExceeded(`stream host answered ${status} (no engine answered in time)`);
     return fromEnvelope(parsed, status);
 }
+export const RESTART_CLOSE = 1012;
+const RESTART_TRIES = 3;
+const MAX_PENDING_BYTES = 4 * 1024 * 1024;
+async function openReady(url, deadlineMs, onLoading, onFrame, attach) {
+    const ws = await openWebSocket(url);
+    ws.binaryType = 'arraybuffer';
+    await new Promise((resolve, reject) => {
+        let settled = false;
+        let sawFrame = false;
+        let rejecting = false;
+        let timer;
+        const fail = (err) => {
+            if (settled)
+                return;
+            settled = true;
+            clearTimeout(timer);
+            try {
+                ws.close();
+            }
+            catch {
+            }
+            reject(err);
+        };
+        const arm = () => {
+            clearTimeout(timer);
+            timer = setTimeout(() => {
+                fail(sawFrame
+                    ? new VocalStackError('engine_unavailable', `engine did not become ready within ${deadlineMs}ms`, { retryable: true })
+                    : capacityExceeded(`no response from the engine within ${deadlineMs}ms — every machine is ` +
+                        `busy with another session, or one is stuck holding a slot`));
+            }, deadlineMs);
+        };
+        arm();
+        ws.addEventListener('message', (ev) => {
+            if (settled || typeof ev.data !== 'string')
+                return;
+            sawFrame = true;
+            arm();
+            let frame;
+            try {
+                frame = parseServerFrame(ev.data);
+            }
+            catch {
+                return;
+            }
+            if (!frame)
+                return;
+            onFrame(frame);
+            if (frame.type === 'session.loading') {
+                onLoading?.(frame.stage);
+            }
+            else if (frame.type === 'session.ready') {
+                if (!settled) {
+                    settled = true;
+                    clearTimeout(timer);
+                    attach(ws);
+                    resolve();
+                }
+            }
+            else if (frame.type === 'error') {
+                fail(new VocalStackError(frame.code, frame.message, {
+                    retryable: frame.retryable,
+                }));
+            }
+        });
+        const emitterOn = ws
+            .on?.bind(ws);
+        emitterOn?.('unexpected-response', (_req, res) => {
+            rejecting = true;
+            const status = res?.statusCode;
+            let body = '';
+            if (res && typeof res.on === 'function') {
+                res.on('data', (c) => (body += c.toString()));
+                res.on('end', () => fail(upgradeRefused(status, body)));
+            }
+            else {
+                fail(upgradeRefused(status, ''));
+            }
+        });
+        ws.addEventListener('close', () => {
+            if (rejecting)
+                return;
+            fail(new VocalStackError('engine_unavailable', 'connection closed before ready', {
+                retryable: true,
+            }));
+        });
+        ws.addEventListener('error', () => {
+        });
+    });
+}
 export class RealtimeSession {
     sessionId;
     #ws;
@@ -41,108 +125,85 @@ export class RealtimeSession {
     #stats = null;
     #error = null;
     #closed = false;
+    #closedByUs = false;
     #onFrame;
-    constructor(sessionId, ws, onFrame) {
+    #params;
+    #reconnecting = false;
+    #pending = [];
+    #pendingBytes = 0;
+    #reconnects = 0;
+    constructor(sessionId, ws, params) {
         this.sessionId = sessionId;
         this.#ws = ws;
-        this.#onFrame = onFrame;
+        this.#onFrame = params.onFrame;
+        this.#params = params;
     }
     static async connect(params) {
         const deadlineMs = params.readyDeadlineMs ?? DEFAULT_READY_DEADLINE_MS;
-        const ws = await openWebSocket(params.streamUrl);
-        ws.binaryType = 'arraybuffer';
-        const session = new RealtimeSession(params.sessionId, ws, params.onFrame);
-        await new Promise((resolve, reject) => {
-            let settled = false;
-            let sawFrame = false;
-            let rejecting = false;
-            let timer;
-            const fail = (err) => {
-                if (settled)
-                    return;
-                settled = true;
-                clearTimeout(timer);
-                try {
-                    ws.close();
-                }
-                catch {
-                }
-                reject(err);
-            };
-            const arm = () => {
-                clearTimeout(timer);
-                timer = setTimeout(() => {
-                    fail(sawFrame
-                        ? new VocalStackError('engine_unavailable', `engine did not become ready within ${deadlineMs}ms`, { retryable: true })
-                        : capacityExceeded(`no response from the engine within ${deadlineMs}ms — every machine is ` +
-                            `busy with another session, or one is stuck holding a slot`));
-                }, deadlineMs);
-            };
-            arm();
-            ws.addEventListener('message', (ev) => {
-                // Once ready, #listen reads the socket; this one stops re-arming
-                // the deadline (it used to leave a timer running for 30 s).
-                if (settled || typeof ev.data !== 'string')
-                    return;
-                sawFrame = true;
-                arm();
-                let frame;
-                try {
-                    frame = parseServerFrame(ev.data);
-                }
-                catch {
-                    return;
-                }
-                if (!frame)
-                    return;
-                session.#frame(frame);
-                if (frame.type === 'session.loading') {
-                    params.onLoading?.(frame.stage);
-                }
-                else if (frame.type === 'session.ready') {
-                    if (!settled) {
-                        settled = true;
-                        clearTimeout(timer);
-                        session.#listen();
-                        resolve();
-                    }
-                }
-                else if (frame.type === 'error') {
-                    fail(new VocalStackError(frame.code, frame.message, {
-                        retryable: frame.retryable,
-                    }));
-                }
-            });
-            const emitterOn = ws
-                .on?.bind(ws);
-            emitterOn?.('unexpected-response', (_req, res) => {
-                rejecting = true;
-                const status = res?.statusCode;
-                let body = '';
-                if (res && typeof res.on === 'function') {
-                    res.on('data', (c) => (body += c.toString()));
-                    res.on('end', () => fail(upgradeRefused(status, body)));
-                }
-                else {
-                    fail(upgradeRefused(status, ''));
-                }
-            });
-            ws.addEventListener('close', () => {
-                if (rejecting)
-                    return;
-                // No HTTP answer at all (a refused upgrade is `unexpected-response`
-                // above): the connection dropped on the way, as the Python and PHP
-                // SDKs say it — retryable, not a bad token.
-                fail(new VocalStackError('engine_unavailable', 'connection closed before ready', {
-                    retryable: true,
-                }));
-            });
-            ws.addEventListener('error', () => {
-            });
+        const early = [];
+        let session;
+        await openReady(params.streamUrl, deadlineMs, params.onLoading, (f) => early.push(f), (ws) => {
+            const s = new RealtimeSession(params.sessionId, ws, params);
+            for (const f of early)
+                s.#frame(f);
+            s.#listen();
+            session = s;
         });
         return session;
     }
-    /** Hands a frame to `onFrame`; a throwing callback never breaks the socket. */
+    async #reconnect(code) {
+        this.#reconnecting = true;
+        const deadlineMs = this.#params.readyDeadlineMs ?? DEFAULT_READY_DEADLINE_MS;
+        while (this.#reconnects < RESTART_TRIES && !this.#closedByUs) {
+            await new Promise((r) => setTimeout(r, 1000 * 2 ** this.#reconnects));
+            this.#reconnects++;
+            if (this.#closedByUs)
+                break;
+            try {
+                await openReady(this.#params.streamUrl, deadlineMs, this.#params.onLoading, (f) => this.#frame(f), (ws) => {
+                    this.#ws = ws;
+                    this.#reconnecting = false;
+                    this.#listen();
+                    const waiting = this.#pending;
+                    this.#pending = [];
+                    this.#pendingBytes = 0;
+                    for (const d of waiting)
+                        this.#send(d);
+                });
+                if (this.#closedByUs)
+                    this.close();
+                return;
+            }
+            catch (e) {
+                if (e instanceof VocalStackError && !e.retryable) {
+                    this.#fail(e);
+                    return;
+                }
+            }
+        }
+        this.#fail(engineLost(code));
+    }
+    #fail(err) {
+        this.#reconnecting = false;
+        this.#pending = [];
+        this.#closed = true;
+        if (!this.#endReason && !this.#error && !this.#closedByUs)
+            this.#error = err;
+        this.#wake?.();
+    }
+    #send(data) {
+        if (this.#reconnecting) {
+            const n = typeof data === 'string' ? data.length : data.byteLength;
+            if (this.#pendingBytes + n <= MAX_PENDING_BYTES) {
+                this.#pending.push(data);
+                this.#pendingBytes += n;
+            }
+            return;
+        }
+        if (this.#ws.readyState !== WS_OPEN)
+            return;
+        this.#ws.send(data);
+    }
     #frame(frame) {
         try {
             this.#onFrame?.(frame);
@@ -151,8 +212,9 @@ export class RealtimeSession {
         }
     }
     #listen() {
-        this.#ws.addEventListener('message', (ev) => {
-            if (typeof ev.data !== 'string')
+        const sock = this.#ws;
+        sock.addEventListener('message', (ev) => {
+            if (sock !== this.#ws || typeof ev.data !== 'string')
                 return;
             let frame;
             try {
@@ -192,31 +254,42 @@ export class RealtimeSession {
                     break;
             }
         });
-        this.#ws.addEventListener('close', () => {
+        const ws = this.#ws;
+        ws.addEventListener('close', (ev) => {
+            if (ws !== this.#ws)
+                return;
+            const code = typeof ev?.code === 'number' ? ev.code : undefined;
+            if (code === RESTART_CLOSE && !this.#endReason && !this.#error && !this.#closedByUs) {
+                void this.#reconnect(code);
+                return;
+            }
             this.#closed = true;
+            if (!this.#endReason && !this.#error && !this.#closedByUs) {
+                this.#error = engineLost(code);
+            }
             this.#wake?.();
         });
     }
     sendAudio(chunk) {
-        if (this.#ws.readyState !== WS_OPEN)
-            return;
         const data = chunk instanceof Float32Array
             ? new Uint8Array(chunk.buffer, chunk.byteOffset, chunk.byteLength)
             : chunk;
-        this.#ws.send(data);
+        this.#send(this.#reconnecting ? (data instanceof Uint8Array ? data.slice() : data.slice(0)) : data);
     }
     async sendFile(source, chunkBytes = 256 * 1024) {
         const MAX_BUFFERED = 4 * 1024 * 1024;
         for await (const chunk of source) {
             for (let off = 0; off < chunk.byteLength; off += chunkBytes) {
+                while (this.#reconnecting)
+                    await new Promise((r) => setTimeout(r, 50));
                 if (this.#ws.readyState !== WS_OPEN)
                     return;
                 while (this.#ws.bufferedAmount > MAX_BUFFERED) {
                     await new Promise((r) => setTimeout(r, 10));
-                    if (this.#ws.readyState !== WS_OPEN)
+                    if (this.#ws.readyState !== WS_OPEN && !this.#reconnecting)
                         return;
                 }
-                this.#ws.send(chunk.subarray(off, Math.min(off + chunkBytes, chunk.byteLength)));
+                this.#send(chunk.subarray(off, Math.min(off + chunkBytes, chunk.byteLength)));
             }
         }
         this.eof();
@@ -230,34 +303,11 @@ export class RealtimeSession {
         }
         this.eof();
     }
-    /**
-     * Capture clock heartbeat: `pcm` = cumulative samples captured so far
-     * (16kHz-equivalent), `wall` = seconds since capture start.  Lets the
-     * server detect capture loss (throttled tab, device switch) and mark
-     * the transcript with honest gaps instead of gluing across them.
-     * `streamMicrophone`/`captureMicrophone` call this automatically.
-     */
     sendClock(pcm, wall) {
-        if (this.#ws.readyState === WS_OPEN) {
-            this.#ws.send(JSON.stringify({ type: 'clock', pcm: Math.floor(pcm), wall }));
-        }
+        this.#send(JSON.stringify({ type: 'clock', pcm: Math.floor(pcm), wall }));
     }
-    /**
-     * Options update during the realtime session. Omitted fields are unchanged.
-     * `latencyMode` switches the decode model policy from the next
-     * engine step; `spokenLanguage` changes the transcription language from the
-     * next engine step (`''` or `'auto'` = auto-detect); `speakerLabels`
-     * turns speaker labelling on (`'on'` or `'auto'`) or off (`'off'`) from the next
-     * engine step — enabling labels speakers from that point on;
-     * `translate` replaces the whole target-language list (`[]` turns
-     * translation off). Applies to subsequent transcript —
-     * already-delivered segments and settled sentences are not
-     * re-transcribed, re-attributed or re-translated.
-     */
     update(changes) {
-        if (this.#ws.readyState !== WS_OPEN)
-            return;
-        this.#ws.send(JSON.stringify({
+        this.#send(JSON.stringify({
             type: 'session.update',
             ...(changes.latencyMode !== undefined && { latencyMode: changes.latencyMode }),
             ...(changes.spokenLanguage !== undefined && { spokenLanguage: changes.spokenLanguage }),
@@ -266,14 +316,10 @@ export class RealtimeSession {
         }));
     }
     eof() {
-        if (this.#ws.readyState === WS_OPEN) {
-            this.#ws.send(JSON.stringify({ type: 'eof' }));
-        }
+        this.#send(JSON.stringify({ type: 'eof' }));
     }
     cancel() {
-        if (this.#ws.readyState === WS_OPEN) {
-            this.#ws.send(JSON.stringify({ type: 'cancel' }));
-        }
+        this.#send(JSON.stringify({ type: 'cancel' }));
     }
     get doneReason() {
         return this.#endReason;
@@ -283,6 +329,7 @@ export class RealtimeSession {
     }
     close() {
         this.#closed = true;
+        this.#closedByUs = true;
         try {
             this.#ws.close();
         }
@@ -326,18 +373,14 @@ async function* readableToAsyncIterable(stream) {
         reader.releaseLock();
     }
 }
-/** `vs.realtime`: a realtime session is a signed token plus one socket.
- * Nothing is stored: the transcript is what the socket sends. */
 export class Realtime {
     #t;
     constructor(t) {
         this.#t = t;
     }
-    /** `POST /v1/realtime/sessions`: validate the options, get a signed `streamUrl`. */
     createSession(options = {}, call = {}) {
         return this.#t.json('POST', '/v1/realtime/sessions', { body: options, ...call });
     }
-    /** `createSession`, then open the socket and wait for `session.ready`. */
     async connect(options = {}, connect = {}) {
         const created = await this.createSession(options);
         return RealtimeSession.connect({
@@ -349,13 +392,12 @@ export class Realtime {
             onFrame: connect.onFrame,
         });
     }
-    /** `GET /v1/realtime/capabilities`: languages, audio formats, limits. */
     capabilities() {
         return this.#t.json('GET', '/v1/realtime/capabilities');
     }
-    /** Stream an encoded file (mp3, wav, flac, ogg, m4a…) through a realtime
-     * session and collect the transcript.  Nothing is stored; for a stored,
-     * shareable transcription use `vs.transcriptions.create`. */
+    async warm(req) {
+        await this.#t.json('POST', '/v1/realtime/warm', { body: req });
+    }
     async transcribeFile(file, opts = {}) {
         const { onSession, readyDeadlineMs, handlers, options } = split(opts);
         const session = await this.connect({ ...options, input: { kind: 'file' } }, { readyDeadlineMs });
@@ -363,7 +405,6 @@ export class Realtime {
         await session.sendFile(fileChunks(file));
         return collectTranscript(session, handlers);
     }
-    /** The same for a link the engine fetches itself. */
     async transcribeUrl(url, opts = {}) {
         const { onSession, readyDeadlineMs, handlers, options } = split(opts);
         const session = await this.connect({ ...options, input: { kind: 'url', url } }, { readyDeadlineMs });
@@ -376,7 +417,6 @@ function split(opts) {
     const handlers = { onProgress, onCommit, onTranslated, onAmended, onSpeakerSettled, onSentence };
     return { onSession, readyDeadlineMs, handlers, options };
 }
-/** Upsert one sentence frame into a list kept in sentenceId order. */
 export function applySentence(list, sentence) {
     const i = list.findIndex((s) => s.sentenceId === sentence.sentenceId);
     if (sentence.status === 'dropped') {