@vocalstack/js-sdk 1.0.13 → 2.0.0-alpha.0

package/dist/mic.js

@@ -0,0 +1,258 @@
+// The worklet runs on the browser's realtime audio thread and does one
+// allocation-light thing: post every 128-frame block to the main thread.
+// MessagePort queues are unbounded and never overwrite, so a busy main
+// thread (a big transcript re-render) delivers audio late, never drops it.
+const WORKLET_SOURCE = `
+class PCMProcessor extends AudioWorkletProcessor {
+  process(inputs) {
+    const input = inputs[0];
+    if (!input || input.length === 0) return true;
+    const ch = input[0];
+    if (!ch || ch.length === 0) return true;
+    this.port.postMessage(ch.slice(0));
+    return true;
+  }
+}
+registerProcessor('pcm-processor', PCMProcessor);
+`;
+/** Retry cadence for context resume / track re-acquire while stalled. */
+const RECOVER_MS = 1000;
+/** Capture clock heartbeat cadence. */
+const CLOCK_MS = 500;
+export async function captureMicrophone(onChunk, opts = {}) {
+    if (typeof navigator === 'undefined' || !navigator.mediaDevices) {
+        throw new Error('captureMicrophone is browser-only (no mediaDevices)');
+    }
+    const targetRate = opts.sampleRate ?? 16000;
+    const chunkSamples = Math.round((opts.chunkSeconds ?? 0.25) * targetRate);
+    const constraints = opts.audio ?? {
+        channelCount: 1,
+        echoCancellation: true,
+        noiseSuppression: true,
+    };
+    let muted = false;
+    let media = await navigator.mediaDevices.getUserMedia({ audio: constraints });
+    // Ask for the target rate directly: Chrome/Firefox/Safari 14.1+ then
+    // resample on the audio thread and our converter is a pass-through.
+    // Browsers that refuse the option fall back to the default rate and the
+    // linear-interpolation resampler below.
+    let ctx;
+    try {
+        ctx = new AudioContext({ sampleRate: targetRate });
+    }
+    catch {
+        ctx = new AudioContext();
+    }
+    const workletUrl = URL.createObjectURL(new Blob([WORKLET_SOURCE], { type: 'application/javascript' }));
+    try {
+        await ctx.audioWorklet.addModule(workletUrl);
+    }
+    finally {
+        URL.revokeObjectURL(workletUrl);
+    }
+    const node = new AudioWorkletNode(ctx, 'pcm-processor');
+    let source = ctx.createMediaStreamSource(media);
+    source.connect(node);
+    const inputRate = ctx.sampleRate;
+    const passThrough = inputRate === targetRate;
+    let buffered = [];
+    let readPos = 0;
+    let pending = null;
+    // Capture clock: cumulative resampled output samples vs wall time.
+    const t0 = performance.now();
+    let outSamples = 0;
+    // Zero-run detector state (input-rate samples).
+    const silentLimit = Math.round((opts.silentSeconds ?? 3) * inputRate);
+    let zeroRun = 0;
+    let silentFlagged = false;
+    // Stall bookkeeping: one active reason at a time; `onRecover` fires
+    // once when it clears.
+    let stopped = false;
+    let stall = null;
+    let recoverTimer = null;
+    const setStall = (reason) => {
+        if (stopped || stall === reason)
+            return;
+        stall = reason;
+        opts.onStall?.(reason);
+    };
+    const clearStall = () => {
+        if (stall === null)
+            return;
+        stall = null;
+        opts.onRecover?.();
+    };
+    node.port.onmessage = (ev) => {
+        const block = ev.data;
+        if (silentLimit > 0) {
+            let allZero = true;
+            for (const v of block) {
+                if (v !== 0) {
+                    allZero = false;
+                    break;
+                }
+            }
+            if (allZero) {
+                zeroRun += block.length;
+                if (!silentFlagged && zeroRun >= silentLimit) {
+                    silentFlagged = true;
+                    setStall('silent');
+                }
+            }
+            else {
+                zeroRun = 0;
+                if (silentFlagged) {
+                    silentFlagged = false;
+                    if (stall === 'silent')
+                        clearStall();
+                }
+            }
+        }
+        let out;
+        if (passThrough) {
+            out = block;
+        }
+        else {
+            for (const v of block)
+                buffered.push(v);
+            const step = inputRate / targetRate;
+            const res = [];
+            while (readPos + 1 < buffered.length) {
+                const i = Math.floor(readPos);
+                const frac = readPos - i;
+                res.push(buffered[i] * (1 - frac) + buffered[i + 1] * frac);
+                readPos += step;
+            }
+            const keepFrom = Math.max(0, Math.floor(readPos) - 1);
+            buffered = buffered.slice(keepFrom);
+            readPos -= keepFrom;
+            out = res;
+        }
+        if (out.length) {
+            outSamples += out.length;
+            // Samples flowing again clears a track/context stall; the silent
+            // stall clears only on a nonzero sample (handled above).
+            if (stall !== null && stall !== 'silent')
+                clearStall();
+            pending = pending ? concat(pending, out) : Float32Array.from(out);
+            while (pending.length >= chunkSamples) {
+                const chunk = pending.slice(0, chunkSamples);
+                // Silence, not nothing: see `MicCapture.setMuted`.
+                onChunk(muted ? new Float32Array(chunk.length) : chunk);
+                pending = pending.slice(chunkSamples);
+            }
+        }
+    };
+    // Heartbeat on a timer so a stalled capture is still reported: pcm
+    // stops while wall advances, and the server declares the gap.
+    const clockTimer = opts.onClock
+        ? setInterval(() => {
+            opts.onClock?.(outSamples, (performance.now() - t0) / 1000);
+        }, CLOCK_MS)
+        : null;
+    // --- recovery -------------------------------------------------------
+    const scheduleRecover = (fn) => {
+        if (stopped || recoverTimer)
+            return;
+        recoverTimer = setTimeout(() => {
+            recoverTimer = null;
+            fn();
+        }, RECOVER_MS);
+    };
+    const resumeContext = () => {
+        if (stopped || ctx.state !== 'suspended')
+            return;
+        void ctx.resume().catch(() => { }).then(() => {
+            if (!stopped && ctx.state === 'suspended')
+                scheduleRecover(resumeContext);
+        });
+    };
+    const reacquireTrack = () => {
+        if (stopped)
+            return;
+        navigator.mediaDevices
+            .getUserMedia({ audio: constraints })
+            .then((next) => {
+            if (stopped) {
+                for (const t of next.getTracks())
+                    t.stop();
+                return;
+            }
+            source.disconnect();
+            for (const t of media.getTracks())
+                t.stop();
+            media = next;
+            source = ctx.createMediaStreamSource(media);
+            source.connect(node);
+            watchTracks(media);
+            // `onRecover` fires from the first delivered block.
+        })
+            .catch(() => scheduleRecover(reacquireTrack));
+    };
+    const watchTracks = (stream) => {
+        for (const track of stream.getAudioTracks()) {
+            track.addEventListener('ended', () => {
+                if (stopped)
+                    return;
+                setStall('track_ended');
+                scheduleRecover(reacquireTrack);
+            });
+            track.addEventListener('mute', () => {
+                if (!stopped)
+                    setStall('track_muted');
+            });
+            // `unmute` needs no handler: the next block clears the stall.
+        }
+    };
+    watchTracks(media);
+    ctx.addEventListener('statechange', () => {
+        if (stopped)
+            return;
+        if (ctx.state === 'suspended') {
+            setStall('context_suspended');
+            resumeContext();
+        }
+        else if (ctx.state === 'closed') {
+            setStall('context_closed');
+        }
+    });
+    return {
+        sampleRate: targetRate,
+        get device() {
+            const t = media.getAudioTracks()[0];
+            const st = (t?.getSettings?.() ?? {});
+            return { label: t?.label ?? '', sampleRate: st.sampleRate, channelCount: st.channelCount, noiseSuppression: st.noiseSuppression, echoCancellation: st.echoCancellation };
+        },
+        get muted() {
+            return muted;
+        },
+        setMuted(on) {
+            muted = !!on;
+        },
+        async stop() {
+            stopped = true;
+            if (clockTimer)
+                clearInterval(clockTimer);
+            if (recoverTimer)
+                clearTimeout(recoverTimer);
+            node.port.onmessage = null;
+            source.disconnect();
+            node.disconnect();
+            for (const t of media.getTracks())
+                t.stop();
+            await ctx.close();
+        },
+    };
+}
+export async function streamMicrophone(session, opts = {}) {
+    return captureMicrophone((pcm) => session.sendAudio(pcm), {
+        onClock: (pcm, wall) => session.sendClock(pcm, wall),
+        ...opts,
+    });
+}
+function concat(a, b) {
+    const out = new Float32Array(a.length + b.length);
+    out.set(a, 0);
+    out.set(b, a.length);
+    return out;
+}

package/dist/realtime.d.ts

@@ -0,0 +1,132 @@
+import type { Capabilities, CommittedSegment, CreateSessionResponse, Fix, LatencyMode, SentenceSegment, ServerFrame, SessionOptions, SessionStats, SpeakerLabels } from '@vocalstack/contract';
+import { VocalStackError } from './errors.js';
+import { type FileInput } from './file.js';
+import type { CallOptions, Transport } from './http.js';
+export type SessionEvent = Extract<ServerFrame, {
+    type: 'transcript.committed' | 'transcript.draft' | 'transcript.words' | 'transcript.translated' | 'transcript.amended' | 'transcript.latency' | 'transcript.speaker' | 'transcript.sentence' | 'transcript.progress';
+}>;
+export interface ConnectParams {
+    sessionId: string;
+    streamUrl: string;
+    options?: SessionOptions;
+    readyDeadlineMs?: number;
+    /** Fires on every `session.loading` frame while the engine warms up
+     * (model download / load after a machine wake) — lets a UI show
+     * "warming up" instead of a generic connecting state.  Each frame also
+     * re-arms the ready deadline, so a long cold load only fails once the
+     * engine goes silent, not merely slow. */
+    onLoading?(stage: 'download' | 'model'): void;
+    /** Every server text frame as it arrives, parsed, before the iterator
+     * sees it: the lifecycle frames the iterator keeps to itself
+     * (`session.loading`, `session.ready`, `session.done`, `error`) and the
+     * transcript frames alike.  For logs and protocol views; the iterator
+     * is still the way to read the transcript. */
+    onFrame?(frame: ServerFrame): void;
+}
+/** A refused socket upgrade as an error.  A 503 carrying the error body
+ * with `unavailable` (the database is waking; retry shortly) says so; any
+ * other 503 — a bare one from a saturated proxy, or no body at all — is
+ * `capacity_exceeded`.  So is a 502 or 504 without the API's error body:
+ * the proxy in front of the engines found none free in time.  Every other
+ * status is its own envelope. */
+export declare function upgradeRefused(status: number | undefined, body: string): VocalStackError;
+export declare class RealtimeSession implements AsyncIterable<SessionEvent> {
+    #private;
+    readonly sessionId: string;
+    private constructor();
+    static connect(params: ConnectParams): Promise<RealtimeSession>;
+    sendAudio(chunk: Float32Array | ArrayBuffer | Uint8Array): void;
+    sendFile(source: AsyncIterable<Uint8Array>, chunkBytes?: number): Promise<void>;
+    pipe(source: AsyncIterable<Float32Array | ArrayBuffer | Uint8Array> | ReadableStream<Uint8Array>): Promise<void>;
+    /**
+     * Capture clock heartbeat: `pcm` = cumulative samples captured so far
+     * (16kHz-equivalent), `wall` = seconds since capture start.  Lets the
+     * server detect capture loss (throttled tab, device switch) and mark
+     * the transcript with honest gaps instead of gluing across them.
+     * `streamMicrophone`/`captureMicrophone` call this automatically.
+     */
+    sendClock(pcm: number, wall: number): void;
+    /**
+     * Options update during the realtime session. Omitted fields are unchanged.
+     * `latencyMode` switches the decode model policy from the next
+     * engine step; `spokenLanguage` changes the transcription language from the
+     * next engine step (`''` or `'auto'` = auto-detect); `speakerLabels`
+     * turns speaker labelling on (`'on'` or `'auto'`) or off (`'off'`) from the next
+     * engine step — enabling labels speakers from that point on;
+     * `translate` replaces the whole target-language list (`[]` turns
+     * translation off). Applies to subsequent transcript —
+     * already-delivered segments and settled sentences are not
+     * re-transcribed, re-attributed or re-translated.
+     */
+    update(changes: {
+        latencyMode?: LatencyMode;
+        spokenLanguage?: string;
+        speakerLabels?: SpeakerLabels;
+        translate?: string[];
+    }): void;
+    eof(): void;
+    cancel(): void;
+    get doneReason(): 'eof' | 'max_duration' | 'cancelled' | null;
+    get stats(): SessionStats | null;
+    close(): void;
+    [Symbol.asyncIterator](): AsyncIterator<SessionEvent>;
+    [Symbol.dispose](): void;
+}
+export interface ConnectOptions {
+    /** See {@link ConnectParams.onLoading}: engine warm-up progress. */
+    onLoading?(stage: 'download' | 'model'): void;
+    /** See {@link ConnectParams.onFrame}: every server frame, lifecycle included. */
+    onFrame?(frame: ServerFrame): void;
+    /** Overrides the client-wide `readyDeadlineMs` for this socket. */
+    readyDeadlineMs?: number;
+}
+export interface TranscribeOptions extends Omit<SessionOptions, 'input' | 'audio'> {
+    /** The realtime session, once connected (to `cancel()` it, say). */
+    onSession?(session: RealtimeSession): void;
+    onProgress?(progress: {
+        processedS: number;
+        totalS?: number;
+    }): void;
+    onCommit?(segment: CommittedSegment): void;
+    /** A committed segment's translation arrived; the segment (by `seq`)
+     * already has its `translations` filled in. */
+    onTranslated?(seq: number, translations: Record<string, string>, sources?: Record<string, string>): void;
+    /** The engine rewrote a committed segment; it is patched in place first. */
+    onAmended?(seq: number, text: string, speaker: number, fixes: Fix[]): void;
+    /** A tentative speaker label (`speakerProvisional`) settled: `verdict` is
+     * "confirmed", "amended" or "unverified".  The segment is patched first. */
+    onSpeakerSettled?(seq: number, speaker: number, verdict: string, confidence?: number): void;
+    /** Every sentence upsert (`sentences.enabled`); the returned `sentences`
+     * list is kept up to date (dropped sentences are removed). */
+    onSentence?(sentence: SentenceSegment): void;
+    readyDeadlineMs?: number;
+}
+export interface RealtimeTranscript {
+    segments: CommittedSegment[];
+    /** Sentence upserts folded into their final state (empty unless
+     * `sentences.enabled`). */
+    sentences: SentenceSegment[];
+    text: string;
+    doneReason: 'eof' | 'max_duration' | 'cancelled' | null;
+    stats: SessionStats | null;
+}
+/** `vs.realtime`: a realtime session is a signed token plus one socket.
+ * Nothing is stored: the transcript is what the socket sends. */
+export declare class Realtime {
+    #private;
+    constructor(t: Transport);
+    /** `POST /v1/realtime/sessions`: validate the options, get a signed `streamUrl`. */
+    createSession(options?: SessionOptions, call?: CallOptions): Promise<CreateSessionResponse>;
+    /** `createSession`, then open the socket and wait for `session.ready`. */
+    connect(options?: SessionOptions, connect?: ConnectOptions): Promise<RealtimeSession>;
+    /** `GET /v1/realtime/capabilities`: languages, audio formats, limits. */
+    capabilities(): Promise<Capabilities>;
+    /** Stream an encoded file (mp3, wav, flac, ogg, m4a…) through a realtime
+     * session and collect the transcript.  Nothing is stored; for a stored,
+     * shareable transcription use `vs.transcriptions.create`. */
+    transcribeFile(file: FileInput, opts?: TranscribeOptions): Promise<RealtimeTranscript>;
+    /** The same for a link the engine fetches itself. */
+    transcribeUrl(url: string, opts?: TranscribeOptions): Promise<RealtimeTranscript>;
+}
+/** Upsert one sentence frame into a list kept in sentenceId order. */
+export declare function applySentence(list: SentenceSegment[], sentence: SentenceSegment): SentenceSegment[];