@framers/agentos 0.13.77 → 0.13.79

package/dist/io/voice-pipeline/ListenSession.js

@@ -0,0 +1,242 @@
+/**
+ * @module voice-pipeline/ListenSession
+ *
+ * A session that listens and writes: a transport's audio into a streaming
+ * transcriber, each final (or each detector turn) into a sink as a numbered
+ * turn. No synthesis, no barge-in, no agent turn. Built for a notetaker, a
+ * meeting bot or a transcription job, where {@link VoicePipelineOrchestrator}
+ * is built for a conversation.
+ */
+import { EventEmitter } from 'node:events';
+import { ReconnectingSTT } from './ReconnectingSTT.js';
+/**
+ * Starts listening. Emits `turn` (a {@link TranscriptTurn}), `interim` (a {@link TranscriptEvent}), `stt:reconnected`,
+ * `stt:gap`, `paused`, `resumed` and `ended` (`{ reason, error? }`).
+ *
+ * The transport's frames go to the transcriber. Each final, or with an endpoint detector each `turn_complete`, goes
+ * to the sink, one write at a time in arrival order; the sink numbers it. When `sinkQueueLimit` finals wait for the
+ * sink, the session emits `pause` on the transport and `paused` on itself, and `resume` and `resumed` once the queue
+ * is down to half or the session ends; a final is never dropped for a slow sink. The transcriber's `speech_start`
+ * and `speech_end` reach the detector as voice activity, as the orchestrator gives them. The session ends once, for the first
+ * {@link ListenEndReason}; it closes the transcriber and leaves the transport open, since the transport is the
+ * caller's. A transport error ends it with `transport_failed` and the error in `ended`. Its transport listeners
+ * are removed on every end. An empty detector turn uses the accumulated STT finals and their timing.
+ *
+ * @param transport - Where the audio comes from: the session listens for its `audio`, `close` and `error`.
+ * @param sink - Where the turns go; it assigns each turn's number and id.
+ * @param overrides - The transcriber, the run and the session's options.
+ * @returns The session, listening.
+ * @throws {Error} When the transcriber's first session fails to open.
+ *
+ * @example
+ * ```typescript
+ * const session = await startListenSession(transport, sink, {
+ *   streamingSTT: new DeepgramStreamingSTT({ apiKey: process.env.DEEPGRAM_API_KEY! }),
+ *   runId: 'run-1',
+ * });
+ * session.on('ended', ({ reason }: { reason: ListenEndReason }) => console.log(`listening ended: ${reason}`));
+ * await session.close('stopped');
+ * ```
+ */
+export async function startListenSession(transport, sink, overrides) {
+    const stt = overrides.reconnect === false ? overrides.streamingSTT : new ReconnectingSTT(overrides.streamingSTT, typeof overrides.reconnect === 'object' ? overrides.reconnect : {});
+    const sttSession = await stt.startSession(overrides.sttConfig);
+    const detector = overrides.endpointDetector;
+    const queueLimit = overrides.sinkQueueLimit ?? 8;
+    const drainMs = overrides.drainMs ?? 250;
+    const emitter = new EventEmitter();
+    let state = 'listening';
+    let turns = 0;
+    let dropped = 0;
+    let reconnects = 0;
+    let paused = false;
+    let sinkFailed = false;
+    const pending = [];
+    const finals = [];
+    let speechEnded = false;
+    let writing = Promise.resolve();
+    let endPromise;
+    const accepting = () => state === 'listening' || state === 'draining';
+    const resume = () => {
+        if (!paused)
+            return;
+        paused = false;
+        transport.emit('resume');
+        emitter.emit('resumed');
+    };
+    const drainDetector = async () => {
+        if (!detector || finals.length === 0)
+            return;
+        await new Promise((resolve) => {
+            const done = () => {
+                clearTimeout(timer);
+                detector.off('turn_complete', done);
+                resolve();
+            };
+            // One timer tick lets a detector scheduled by the synthetic speech_end
+            // below emit at its bound before the fallback removes its listener.
+            const timer = setTimeout(done, (detector.turnCompletionTimeoutMs ?? 3250) + 1);
+            detector.on('turn_complete', done);
+            // A provider need not report speech_end when flushed. End the input so
+            // the detector can start its silence timer, without restarting a pending one.
+            if (!speechEnded)
+                detector.pushVadEvent({ type: 'speech_end', timestamp: Date.now(), source: 'stt' });
+        });
+    };
+    /** One end for every path. The writer calls it with `fromWriter` and never waits for its own chain. */
+    const end = (reason, error, fromWriter = false) => {
+        if (endPromise)
+            return endPromise;
+        // Reserve the end before cleanup emits resume: its listeners may call close again.
+        let resolveEnd = () => { };
+        let rejectEnd = () => { };
+        endPromise = new Promise((resolve, reject) => { resolveEnd = resolve; rejectEnd = reject; });
+        void (async () => {
+            if (reason === 'stopped') {
+                // the transcriber is flushed and its last finals are still written
+                state = 'draining';
+                try {
+                    await sttSession.flush();
+                }
+                catch {
+                    // the transcriber is gone; nothing to drain
+                }
+                await new Promise((r) => setTimeout(r, drainMs));
+                await drainDetector();
+            }
+            state = 'ending';
+            transport.off('audio', onAudio);
+            transport.off('close', onTransportClose);
+            detector?.off('turn_complete', onTurnComplete);
+            finals.length = 0;
+            if (reason === 'purged')
+                pending.length = 0;
+            resume();
+            sttSession.close();
+            detector?.reset();
+            if (!fromWriter)
+                await writing;
+            transport.off('error', onTransportError);
+            state = 'ended';
+            emitter.emit('ended', error ? { reason, error } : { reason });
+        })().then(resolveEnd, rejectEnd);
+        return endPromise;
+    };
+    const write = (text, event) => {
+        if (!accepting() || sinkFailed)
+            return;
+        const turn = { runId: overrides.runId, at: Date.now(), text, startMs: event.startMs, endMs: event.endMs, speaker: overrides.speaker?.(event) };
+        pending.push(turn);
+        if (pending.length >= queueLimit && !paused) {
+            paused = true;
+            transport.emit('pause');
+            emitter.emit('paused');
+        }
+        writing = writing.then(async () => {
+            const next = pending.shift();
+            if (!next || sinkFailed)
+                return;
+            try {
+                const stored = await sink.appendNext(next);
+                if (stored) {
+                    turns += 1;
+                    emitter.emit('turn', stored);
+                }
+            }
+            catch (err) {
+                sinkFailed = true;
+                pending.length = 0;
+                void end('sink_failed', err instanceof Error ? err : new Error(String(err)), true);
+                return;
+            }
+            if (paused && pending.length < Math.ceil(queueLimit / 2)) {
+                resume();
+            }
+        });
+    };
+    const onAudio = (frame) => {
+        if (state !== 'listening')
+            return;
+        if (paused)
+            dropped += 1; // the transport was asked to pause; a frame that still comes is counted, and the transcriber keeps its own buffer
+        sttSession.pushAudio(frame);
+    };
+    const onTransportClose = () => {
+        void end('transport_closed');
+    };
+    const onTransportError = (error) => {
+        void end('transport_failed', error);
+    };
+    const onTurnComplete = (turn) => {
+        if (!accepting())
+            return;
+        const accumulated = finals.splice(0);
+        if (!turn.transcript.trim()) {
+            const text = accumulated.map((event) => event.text).filter(Boolean).join(' ');
+            if (text)
+                write(text, {
+                    text, confidence: accumulated[accumulated.length - 1].confidence,
+                    words: accumulated.flatMap((event) => event.words), isFinal: true,
+                    startMs: accumulated[0].startMs, endMs: accumulated[accumulated.length - 1].endMs,
+                });
+            return;
+        }
+        // a detector turn has its text and confidence, and no audio boundaries of its own
+        write(turn.transcript, { text: turn.transcript, confidence: turn.confidence, words: [], isFinal: true, durationMs: turn.durationMs });
+    };
+    sttSession.on('transcript', (event) => {
+        if (!accepting())
+            return;
+        if (detector) {
+            if (event.isFinal)
+                finals.push(event);
+            detector.pushTranscript(event);
+            if (!event.isFinal)
+                emitter.emit('interim', event);
+            return;
+        }
+        if (event.isFinal)
+            write(event.text, event);
+        else
+            emitter.emit('interim', event);
+    });
+    const onVad = (event) => {
+        if (!accepting())
+            return;
+        if (event.type === 'speech_start')
+            speechEnded = false;
+        else if (event.type === 'speech_end')
+            speechEnded = true;
+        detector?.pushVadEvent(event);
+    };
+    sttSession.on('vad', onVad);
+    // the transcriber's own speech events reach the detector as voice activity, as the orchestrator gives them
+    sttSession.on('speech_start', () => onVad({ type: 'speech_start', timestamp: Date.now(), source: 'stt' }));
+    sttSession.on('speech_end', () => onVad({ type: 'speech_end', timestamp: Date.now(), source: 'stt' }));
+    sttSession.on('stt:reconnected', (e) => {
+        reconnects += 1;
+        emitter.emit('stt:reconnected', e);
+    });
+    sttSession.on('stt:gap', (e) => emitter.emit('stt:gap', e));
+    sttSession.on('error', (err) => {
+        if (state === 'listening')
+            void end('stt_failed', err);
+    });
+    sttSession.on('close', () => {
+        if (state === 'listening')
+            void end('stt_failed', new Error('the transcriber closed'));
+    });
+    detector?.on('turn_complete', onTurnComplete);
+    transport.on('audio', onAudio);
+    transport.on('close', onTransportClose);
+    transport.on('error', onTransportError);
+    const session = Object.assign(emitter, {
+        runId: overrides.runId,
+        transport,
+        stats: () => ({ turns, dropped, reconnects }),
+        close: (reason = 'stopped') => end(reason),
+    });
+    Object.defineProperty(session, 'state', { get: () => state, enumerable: true });
+    return session;
+}
+//# sourceMappingURL=ListenSession.js.map

package/dist/io/voice-pipeline/ListenSession.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ListenSession.js","sourceRoot":"","sources":["../../../src/io/voice-pipeline/ListenSession.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,eAAe,EAA2E,MAAM,sBAAsB,CAAC;AA6EhI;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,SAA2B,EAAE,IAA0B,EAAE,SAAiC;IACjI,MAAM,GAAG,GAAG,SAAS,CAAC,SAAS,KAAK,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI,eAAe,CAAC,SAAS,CAAC,YAAY,EAAE,OAAO,SAAS,CAAC,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACrL,MAAM,UAAU,GAAwB,MAAM,GAAG,CAAC,YAAY,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC;IACpF,MAAM,QAAQ,GAAG,SAAS,CAAC,gBAAgB,CAAC;IAC5C,MAAM,UAAU,GAAG,SAAS,CAAC,cAAc,IAAI,CAAC,CAAC;IACjD,MAAM,OAAO,GAAG,SAAS,CAAC,OAAO,IAAI,GAAG,CAAC;IACzC,MAAM,OAAO,GAAG,IAAI,YAAY,EAAE,CAAC;IACnC,IAAI,KAAK,GAAuB,WAAW,CAAC;IAC5C,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,MAAM,GAAG,KAAK,CAAC;IACnB,IAAI,UAAU,GAAG,KAAK,CAAC;IACvB,MAAM,OAAO,GAAsB,EAAE,CAAC;IACtC,MAAM,MAAM,GAAsB,EAAE,CAAC;IACrC,IAAI,WAAW,GAAG,KAAK,CAAC;IACxB,IAAI,OAAO,GAAkB,OAAO,CAAC,OAAO,EAAE,CAAC;IAC/C,IAAI,UAAqC,CAAC;IAE1C,MAAM,SAAS,GAAG,GAAY,EAAE,CAAC,KAAK,KAAK,WAAW,IAAI,KAAK,KAAK,UAAU,CAAC;IAE/E,MAAM,MAAM,GAAG,GAAS,EAAE;QACxB,IAAI,CAAC,MAAM;YAAE,OAAO;QACpB,MAAM,GAAG,KAAK,CAAC;QACf,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACzB,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAC1B,CAAC,CAAC;IAEF,MAAM,aAAa,GAAG,KAAK,IAAmB,EAAE;QAC9C,IAAI,CAAC,QAAQ,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAC7C,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;YAClC,MAAM,IAAI,GAAG,GAAS,EAAE;gBACtB,YAAY,CAAC,KAAK,CAAC,CAAC;gBACpB,QAAQ,CAAC,GAAG,CAAC,eAAe,EAAE,IAAI,CAAC,CAAC;gBACpC,OAAO,EAAE,CAAC;YACZ,CAAC,CAAC;YACF,uEAAuE;YACvE,oEAAoE;YACpE,MAAM,KAAK,GAAG,UAAU,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,uBAAuB,IAAI,IAAK,CAAC,GAAG,CAAC,CAAC,CAAC;YAChF,QAAQ,CAAC,EAAE,CAAC,eAAe,EAAE,IAAI,CAAC,CAAC;YACnC,uEAAuE;YACvE,8EAA8E;YAC9E,IAAI,CAAC,WAAW;gBAAE,QAAQ,CAAC,YAAY,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;QACxG,CAAC,CAAC,CAAC;IACL,CAAC,CAAC;IAEF,uGAAuG;IACvG,MAAM,GAAG,GAAG,CAAC,MAAuB,EAAE,KAAa,EAAE,UAAU,GAAG,KAAK,EAAiB,EAAE;QACxF,IAAI,UAAU;YAAE,OAAO,UAAU,CAAC;QAClC,mFAAmF;QACnF,IAAI,UAAU,GAAe,GAAG,EAAE,GAAE,CAAC,CAAC;QACtC,IAAI,SAAS,GAA6B,GAAG,EAAE,GAAE,CAAC,CAAC;QACnD,UAAU,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE,GAAG,UAAU,GAAG,OAAO,CAAC,CAAC,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QACnG,KAAK,CAAC,KAAK,IAAI,EAAE;YACf,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;gBACzB,mEAAmE;gBACnE,KAAK,GAAG,UAAU,CAAC;gBACnB,IAAI,CAAC;oBACH,MAAM,UAAU,CAAC,KAAK,EAAE,CAAC;gBAC3B,CAAC;gBAAC,MAAM,CAAC;oBACP,4CAA4C;gBAC9C,CAAC;gBACD,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;gBACjD,MAAM,aAAa,EAAE,CAAC;YACxB,CAAC;YACD,KAAK,GAAG,QAAQ,CAAC;YACjB,SAAS,CAAC,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAChC,SAAS,CAAC,GAAG,CAAC,OAAO,EAAE,gBAAgB,CAAC,CAAC;YACzC,QAAQ,EAAE,GAAG,CAAC,eAAe,EAAE,cAAc,CAAC,CAAC;YAC/C,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;YAClB,IAAI,MAAM,KAAK,QAAQ;gBAAE,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;YAC5C,MAAM,EAAE,CAAC;YACT,UAAU,CAAC,KAAK,EAAE,CAAC;YACnB,QAAQ,EAAE,KAAK,EAAE,CAAC;YAClB,IAAI,CAAC,UAAU;gBAAE,MAAM,OAAO,CAAC;YAC/B,SAAS,CAAC,GAAG,CAAC,OAAO,EAAE,gBAAgB,CAAC,CAAC;YACzC,KAAK,GAAG,OAAO,CAAC;YAChB,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;QAChE,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;QACjC,OAAO,UAAU,CAAC;IACpB,CAAC,CAAC;IAEF,MAAM,KAAK,GAAG,CAAC,IAAY,EAAE,KAAsB,EAAQ,EAAE;QAC3D,IAAI,CAAC,SAAS,EAAE,IAAI,UAAU;YAAE,OAAO;QACvC,MAAM,IAAI,GAAoB,EAAE,KAAK,EAAE,SAAS,CAAC,KAAK,EAAE,EAAE,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,CAAC,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,OAAO,EAAE,SAAS,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC;QAChK,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACnB,IAAI,OAAO,CAAC,MAAM,IAAI,UAAU,IAAI,CAAC,MAAM,EAAE,CAAC;YAC5C,MAAM,GAAG,IAAI,CAAC;YACd,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACxB,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACzB,CAAC;QACD,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;YAChC,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;YAC7B,IAAI,CAAC,IAAI,IAAI,UAAU;gBAAE,OAAO;YAChC,IAAI,CAAC;gBACH,MAAM,MAAM,GAA+B,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;gBACvE,IAAI,MAAM,EAAE,CAAC;oBACX,KAAK,IAAI,CAAC,CAAC;oBACX,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;gBAC/B,CAAC;YACH,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,UAAU,GAAG,IAAI,CAAC;gBAClB,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;gBACnB,KAAK,GAAG,CAAC,aAAa,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;gBACnF,OAAO;YACT,CAAC;YACD,IAAI,MAAM,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC,EAAE,CAAC;gBACzD,MAAM,EAAE,CAAC;YACX,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC,CAAC;IAEF,MAAM,OAAO,GAAG,CAAC,KAAiB,EAAQ,EAAE;QAC1C,IAAI,KAAK,KAAK,WAAW;YAAE,OAAO;QAClC,IAAI,MAAM;YAAE,OAAO,IAAI,CAAC,CAAC,CAAC,kHAAkH;QAC5I,UAAU,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IAC9B,CAAC,CAAC;IACF,MAAM,gBAAgB,GAAG,GAAS,EAAE;QAClC,KAAK,GAAG,CAAC,kBAAkB,CAAC,CAAC;IAC/B,CAAC,CAAC;IACF,MAAM,gBAAgB,GAAG,CAAC,KAAY,EAAQ,EAAE;QAC9C,KAAK,GAAG,CAAC,kBAAkB,EAAE,KAAK,CAAC,CAAC;IACtC,CAAC,CAAC;IACF,MAAM,cAAc,GAAG,CAAC,IAAuB,EAAQ,EAAE;QACvD,IAAI,CAAC,SAAS,EAAE;YAAE,OAAO;QACzB,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QACrC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,EAAE,CAAC;YAC5B,MAAM,IAAI,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC9E,IAAI,IAAI;gBAAE,KAAK,CAAC,IAAI,EAAE;oBACpB,IAAI,EAAE,UAAU,EAAE,WAAW,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,UAAU;oBAChE,KAAK,EAAE,WAAW,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,IAAI;oBACjE,OAAO,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,WAAW,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,KAAK;iBAClF,CAAC,CAAC;YACH,OAAO;QACT,CAAC;QACD,kFAAkF;QAClF,KAAK,CAAC,IAAI,CAAC,UAAU,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,UAAU,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,KAAK,EAAE,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;IACxI,CAAC,CAAC;IAEF,UAAU,CAAC,EAAE,CAAC,YAAY,EAAE,CAAC,KAAsB,EAAE,EAAE;QACrD,IAAI,CAAC,SAAS,EAAE;YAAE,OAAO;QACzB,IAAI,QAAQ,EAAE,CAAC;YACb,IAAI,KAAK,CAAC,OAAO;gBAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACtC,QAAQ,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;YAC/B,IAAI,CAAC,KAAK,CAAC,OAAO;gBAAE,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;YACnD,OAAO;QACT,CAAC;QACD,IAAI,KAAK,CAAC,OAAO;YAAE,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;;YACvC,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IACtC,CAAC,CAAC,CAAC;IACH,MAAM,KAAK,GAAG,CAAC,KAAe,EAAQ,EAAE;QACtC,IAAI,CAAC,SAAS,EAAE;YAAE,OAAO;QACzB,IAAI,KAAK,CAAC,IAAI,KAAK,cAAc;YAAE,WAAW,GAAG,KAAK,CAAC;aAClD,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY;YAAE,WAAW,GAAG,IAAI,CAAC;QACzD,QAAQ,EAAE,YAAY,CAAC,KAAK,CAAC,CAAC;IAChC,CAAC,CAAC;IACF,UAAU,CAAC,EAAE,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;IAC5B,2GAA2G;IAC3G,UAAU,CAAC,EAAE,CAAC,cAAc,EAAE,GAAG,EAAE,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;IAC3G,UAAU,CAAC,EAAE,CAAC,YAAY,EAAE,GAAG,EAAE,CAAC,KAAK,CAAC,EAAE,IAAI,EAAE,YAAY,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;IACvG,UAAU,CAAC,EAAE,CAAC,iBAAiB,EAAE,CAAC,CAAsB,EAAE,EAAE;QAC1D,UAAU,IAAI,CAAC,CAAC;QAChB,OAAO,CAAC,IAAI,CAAC,iBAAiB,EAAE,CAAC,CAAC,CAAC;IACrC,CAAC,CAAC,CAAC;IACH,UAAU,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,CAAc,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC;IACzE,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAU,EAAE,EAAE;QACpC,IAAI,KAAK,KAAK,WAAW;YAAE,KAAK,GAAG,CAAC,YAAY,EAAE,GAAG,CAAC,CAAC;IACzD,CAAC,CAAC,CAAC;IACH,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;QAC1B,IAAI,KAAK,KAAK,WAAW;YAAE,KAAK,GAAG,CAAC,YAAY,EAAE,IAAI,KAAK,CAAC,wBAAwB,CAAC,CAAC,CAAC;IACzF,CAAC,CAAC,CAAC;IACH,QAAQ,EAAE,EAAE,CAAC,eAAe,EAAE,cAAc,CAAC,CAAC;IAC9C,SAAS,CAAC,EAAE,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAC/B,SAAS,CAAC,EAAE,CAAC,OAAO,EAAE,gBAAgB,CAAC,CAAC;IACxC,SAAS,CAAC,EAAE,CAAC,OAAO,EAAE,gBAAgB,CAAC,CAAC;IAExC,MAAM,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,EAAE;QACrC,KAAK,EAAE,SAAS,CAAC,KAAK;QACtB,SAAS;QACT,KAAK,EAAE,GAAuB,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC;QACjE,KAAK,EAAE,CAAC,SAA0B,SAAS,EAAE,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC;KAC5D,CAAkB,CAAC;IACpB,MAAM,CAAC,cAAc,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;IAChF,OAAO,OAAO,CAAC;AACjB,CAAC"}

package/dist/io/voice-pipeline/ReconnectingSTT.d.ts

@@ -0,0 +1,90 @@
+import type { IStreamingSTT, StreamingSTTConfig, StreamingSTTSession } from './types.js';
+/** Options of {@link ReconnectingSTT}. */
+export interface ReconnectingSTTOptions {
+    /** How much audio is kept for replay. Default 3000. */
+    ringBufferCapacityMs?: number;
+    /** Replays below this are skipped. Default 400. */
+    minReplayMs?: number;
+    /** Attempts allowed in any sixty seconds before the session gives up. Default 3. */
+    maxAttemptsPerMinute?: number;
+    /** Frames queued while a reconnect runs; beyond it the oldest are dropped and counted. Default 2000 ms. */
+    queueCapacityMs?: number;
+    /** Milliseconds now, for the attempt window and the gap's length. Default `Date.now`. */
+    clock?: () => number;
+}
+/** Payload of a session's `stt:reconnected`: a new session of the provider took over. */
+export interface SttReconnectedEvent {
+    /** The attempt's number among the attempts of the last sixty seconds, from 1. */
+    attempt: number;
+    /** Milliseconds of buffered audio replayed into the new session; 0 when there was less than `minReplayMs`. */
+    replayedMs: number;
+}
+/** Payload of a session's `stt:gap`, emitted right after `stt:reconnected`. */
+export interface SttGapEvent {
+    /** The attempt's number, as in {@link SttReconnectedEvent}. */
+    attempt: number;
+    /**
+     * Milliseconds the reconnect took, by the options' clock. The audio pushed in that time was queued for the new
+     * session, less the oldest frames a full queue dropped (the session's `droppedFrames`).
+     */
+    lostMs: number;
+}
+/**
+ * An {@link IStreamingSTT} over one provider whose session is reconnected.
+ *
+ * An `error` or `close` of the provider's session that the caller did not ask
+ * for opens a new session of the same provider, at most
+ * `maxAttemptsPerMinute` times in any sixty seconds. The new session hears the
+ * buffered audio first (the last `ringBufferCapacityMs`, when there is at least
+ * `minReplayMs` of it), then the frames pushed while it opened, then live
+ * audio. Its `startMs` and `endMs` are moved onto one timeline, the audio
+ * pushed since the session began, and a final that repeats one already
+ * emitted over the same audio is dropped. Only finals with finite start and end
+ * bounds and a positive duration enter the dedupe history; others pass through.
+ * Queue overflow leaves a gap after the replay, preserving the original times
+ * of both replayed and retained audio. Each reconnect emits
+ * `stt:reconnected` ({@link SttReconnectedEvent}) and `stt:gap`
+ * ({@link SttGapEvent}). Past the cap the session emits `error`, then `close`.
+ *
+ * The session forwards `transcript`, `vad`, `speech_start`, `speech_end`,
+ * `usage` and `warning` from the provider's sessions; `usage` comes from every
+ * one of them, a replaced one too, since a session reports its audio as it
+ * closes. `flush()` during a reconnect waits for the replacement, replay and
+ * queued audio before flushing it. A close during that flush, or after it with
+ * no further provider activity or pushed audio, ends the stream without a
+ * reconnect. Activity after a completed flush makes a later close a failure.
+ *
+ * @example
+ * ```typescript
+ * const stt = new ReconnectingSTT(new DeepgramStreamingSTT({ apiKey: process.env.DEEPGRAM_API_KEY! }));
+ * const session = await stt.startSession({ language: 'en-US' });
+ * session.on('stt:gap', ({ lostMs }: SttGapEvent) => console.log(`the transcriber was away for ${lostMs} ms`));
+ * ```
+ */
+export declare class ReconnectingSTT implements IStreamingSTT {
+    private readonly inner;
+    /** `reconnecting:` followed by the inner provider's id. */
+    readonly providerId: string;
+    private readonly capacityMs;
+    private readonly minReplayMs;
+    private readonly maxAttempts;
+    private readonly queueCapacityMs;
+    private readonly clock;
+    /**
+     * @param inner - The provider each session opens, again on every reconnect.
+     * @param options - The replay buffer, the attempt cap, the queue and the clock.
+     */
+    constructor(inner: IStreamingSTT, options?: ReconnectingSTTOptions);
+    /** Whether the inner provider has a session open. */
+    get isStreaming(): boolean;
+    /**
+     * Opens a session of the inner provider behind a session that reconnects it.
+     *
+     * @param config - Passed to the inner provider for the first session and for every reconnect.
+     * @returns The session, with a read-only `droppedFrames`: the queued frames dropped during reconnects so far.
+     * @throws {Error} When the first session fails to open; a session that fails to open on a reconnect is retried
+     *   within the cap.
+     */
+    startSession(config?: StreamingSTTConfig): Promise<StreamingSTTSession>;
+}
+//# sourceMappingURL=ReconnectingSTT.d.ts.map

package/dist/io/voice-pipeline/ReconnectingSTT.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ReconnectingSTT.d.ts","sourceRoot":"","sources":["../../../src/io/voice-pipeline/ReconnectingSTT.ts"],"names":[],"mappings":"AAcA,OAAO,KAAK,EAAc,aAAa,EAAE,kBAAkB,EAAE,mBAAmB,EAAmB,MAAM,YAAY,CAAC;AAEtH,0CAA0C;AAC1C,MAAM,WAAW,sBAAsB;IACrC,uDAAuD;IACvD,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,mDAAmD;IACnD,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,oFAAoF;IACpF,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,2GAA2G;IAC3G,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,yFAAyF;IACzF,KAAK,CAAC,EAAE,MAAM,MAAM,CAAC;CACtB;AAED,yFAAyF;AACzF,MAAM,WAAW,mBAAmB;IAClC,iFAAiF;IACjF,OAAO,EAAE,MAAM,CAAC;IAChB,8GAA8G;IAC9G,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,+EAA+E;AAC/E,MAAM,WAAW,WAAW;IAC1B,+DAA+D;IAC/D,OAAO,EAAE,MAAM,CAAC;IAChB;;;OAGG;IACH,MAAM,EAAE,MAAM,CAAC;CAChB;AAID;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AACH,qBAAa,eAAgB,YAAW,aAAa;IAavC,OAAO,CAAC,QAAQ,CAAC,KAAK;IAZlC,2DAA2D;IAC3D,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;IACpC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAS;IACzC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAe;IAErC;;;OAGG;gBAC0B,KAAK,EAAE,aAAa,EAAE,OAAO,GAAE,sBAA2B;IASvF,qDAAqD;IACrD,IAAI,WAAW,IAAI,OAAO,CAEzB;IAED;;;;;;;OAOG;IACG,YAAY,CAAC,MAAM,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,mBAAmB,CAAC;CA6N9E"}

package/dist/io/voice-pipeline/ReconnectingSTT.js

@@ -0,0 +1,318 @@
+/**
+ * @module voice-pipeline/ReconnectingSTT
+ *
+ * One streaming transcriber, reconnected when its session errors or closes:
+ * the last seconds of audio are replayed into the new session before the
+ * frames that arrived meanwhile, a replayed duplicate is dropped, each new
+ * session's clock is moved onto the one session timeline, and the caller
+ * hears `stt:reconnected` and `stt:gap`. After the cap, `error` then `close`.
+ * A chain of several providers fails over instead ({@link StreamingSTTChain});
+ * this wraps a chain of one.
+ */
+import { EventEmitter } from 'node:events';
+import { AudioRingBuffer } from './AudioRingBuffer.js';
+import { TranscriptDedupe } from './TranscriptDedupe.js';
+const frameMs = (f) => (f.samples.length / f.sampleRate) * 1000;
+/**
+ * An {@link IStreamingSTT} over one provider whose session is reconnected.
+ *
+ * An `error` or `close` of the provider's session that the caller did not ask
+ * for opens a new session of the same provider, at most
+ * `maxAttemptsPerMinute` times in any sixty seconds. The new session hears the
+ * buffered audio first (the last `ringBufferCapacityMs`, when there is at least
+ * `minReplayMs` of it), then the frames pushed while it opened, then live
+ * audio. Its `startMs` and `endMs` are moved onto one timeline, the audio
+ * pushed since the session began, and a final that repeats one already
+ * emitted over the same audio is dropped. Only finals with finite start and end
+ * bounds and a positive duration enter the dedupe history; others pass through.
+ * Queue overflow leaves a gap after the replay, preserving the original times
+ * of both replayed and retained audio. Each reconnect emits
+ * `stt:reconnected` ({@link SttReconnectedEvent}) and `stt:gap`
+ * ({@link SttGapEvent}). Past the cap the session emits `error`, then `close`.
+ *
+ * The session forwards `transcript`, `vad`, `speech_start`, `speech_end`,
+ * `usage` and `warning` from the provider's sessions; `usage` comes from every
+ * one of them, a replaced one too, since a session reports its audio as it
+ * closes. `flush()` during a reconnect waits for the replacement, replay and
+ * queued audio before flushing it. A close during that flush, or after it with
+ * no further provider activity or pushed audio, ends the stream without a
+ * reconnect. Activity after a completed flush makes a later close a failure.
+ *
+ * @example
+ * ```typescript
+ * const stt = new ReconnectingSTT(new DeepgramStreamingSTT({ apiKey: process.env.DEEPGRAM_API_KEY! }));
+ * const session = await stt.startSession({ language: 'en-US' });
+ * session.on('stt:gap', ({ lostMs }: SttGapEvent) => console.log(`the transcriber was away for ${lostMs} ms`));
+ * ```
+ */
+export class ReconnectingSTT {
+    /**
+     * @param inner - The provider each session opens, again on every reconnect.
+     * @param options - The replay buffer, the attempt cap, the queue and the clock.
+     */
+    constructor(inner, options = {}) {
+        this.inner = inner;
+        this.providerId = `reconnecting:${inner.providerId}`;
+        this.capacityMs = options.ringBufferCapacityMs ?? 3000;
+        this.minReplayMs = options.minReplayMs ?? 400;
+        this.maxAttempts = options.maxAttemptsPerMinute ?? 3;
+        this.queueCapacityMs = options.queueCapacityMs ?? 2000;
+        this.clock = options.clock ?? Date.now;
+    }
+    /** Whether the inner provider has a session open. */
+    get isStreaming() {
+        return this.inner.isStreaming;
+    }
+    /**
+     * Opens a session of the inner provider behind a session that reconnects it.
+     *
+     * @param config - Passed to the inner provider for the first session and for every reconnect.
+     * @returns The session, with a read-only `droppedFrames`: the queued frames dropped during reconnects so far.
+     * @throws {Error} When the first session fails to open; a session that fails to open on a reconnect is retried
+     *   within the cap.
+     */
+    async startSession(config) {
+        const facade = new EventEmitter();
+        const ring = new AudioRingBuffer({ capacityMs: this.capacityMs, sampleRate: 16000 });
+        const dedupe = new TranscriptDedupe();
+        const attempts = [];
+        const queue = [];
+        let queuedMs = 0;
+        let droppedQueuedMs = 0;
+        let dropped = 0;
+        let current = await this.inner.startSession(config);
+        // the connection generation: the dedupe skips observations of one provider id, so each session gets an id of its own
+        let generation = 0;
+        // where each session's clock 0 sits on the session timeline (the audio pushed by the caller since the start)
+        let pushedMs = 0;
+        let offsetMs = 0;
+        let reconnecting = false;
+        let reconnectPromise;
+        let closedByCaller = false;
+        let ended = false;
+        // The flush is the last operation until audio or provider activity follows it.
+        let flushed = false;
+        let flushing;
+        /** Ends the session once: `close`, after which nothing but `usage` is forwarded. */
+        const finish = () => {
+            if (ended)
+                return;
+            ended = true;
+            facade.emit('close');
+        };
+        const attach = (session, sessionGeneration, sessionOffsetMs, replayedMs = 0, gapMs = 0) => {
+            const providerId = `${this.inner.providerId}#${sessionGeneration}`;
+            const activity = () => {
+                if (session === current)
+                    flushed = false;
+            };
+            // End bounds are exclusive: an utterance ending exactly at the replay
+            // boundary still belongs to the audio before the gap.
+            const mapTime = (time, end = false) => time + sessionOffsetMs + ((end ? time > replayedMs : time >= replayedMs) ? gapMs : 0);
+            session.on('transcript', (raw) => {
+                if (ended)
+                    return;
+                activity();
+                const evt = {
+                    ...raw,
+                    ...(raw.startMs !== undefined ? { startMs: mapTime(raw.startMs) } : {}),
+                    ...(raw.endMs !== undefined ? { endMs: mapTime(raw.endMs, true) } : {}),
+                };
+                if (evt.isFinal && evt.startMs !== undefined && evt.endMs !== undefined && Number.isFinite(evt.startMs) && Number.isFinite(evt.endMs) && evt.endMs > evt.startMs) {
+                    const r = dedupe.evaluate({ provider: providerId, text: evt.text, audioStartMs: evt.startMs, audioEndMs: evt.endMs, isFinal: true });
+                    if (r.isDuplicate)
+                        return;
+                }
+                facade.emit('transcript', evt);
+            });
+            for (const name of ['vad', 'speech_start', 'speech_end']) {
+                session.on(name, (...args) => {
+                    if (ended)
+                        return;
+                    activity();
+                    facade.emit(name, ...args);
+                });
+            }
+            // from every session, a replaced or ended one too: a session reports the audio of its connections as it closes
+            session.on('usage', (usage) => {
+                activity();
+                facade.emit('usage', usage);
+            });
+            session.on('warning', (warning) => {
+                if (ended)
+                    return;
+                activity();
+                // Listening here must not silence a warning: a provider session prints one nobody listens for, so the
+                // session prints it when its own caller does not listen either.
+                if (facade.listenerCount('warning') > 0)
+                    facade.emit('warning', warning);
+                else
+                    console.warn(`[${this.inner.providerId}] ${warning.message}`);
+            });
+            session.on('error', (err) => {
+                if (closedByCaller || session !== current)
+                    return;
+                activity();
+                void reconnect(err);
+            });
+            session.on('close', () => {
+                if (closedByCaller || session !== current || reconnecting)
+                    return;
+                if (flushed || flushing === session) {
+                    // the provider sent its last final and ended its stream, as the caller's flush asked
+                    finish();
+                    return;
+                }
+                void reconnect(undefined);
+            });
+        };
+        const giveUp = (err) => {
+            if (ended)
+                return;
+            ended = true;
+            // the provider's session does not stay open behind a session that reports itself closed
+            try {
+                current.close();
+            }
+            catch {
+                // already gone
+            }
+            facade.emit('error', err);
+            facade.emit('close');
+        };
+        const runReconnect = async (err) => {
+            if (reconnecting || closedByCaller || ended)
+                return;
+            reconnecting = true;
+            const startedAt = this.clock();
+            const minuteAgo = startedAt - 60000;
+            while (attempts.length && attempts[0] < minuteAgo)
+                attempts.shift();
+            if (attempts.length >= this.maxAttempts) {
+                reconnecting = false;
+                giveUp(err ?? new Error(`${this.inner.providerId}: session ended after ${this.maxAttempts} reconnects in a minute`));
+                return;
+            }
+            attempts.push(startedAt);
+            const attempt = attempts.length;
+            try {
+                current.close();
+            }
+            catch {
+                // already gone
+            }
+            let next;
+            try {
+                next = await this.inner.startSession(config);
+            }
+            catch (startErr) {
+                reconnecting = false;
+                await runReconnect(startErr instanceof Error ? startErr : new Error(String(startErr)));
+                return;
+            }
+            if (closedByCaller) {
+                next.close();
+                reconnecting = false;
+                return;
+            }
+            const replay = ring.snapshot();
+            const bufferedMs = replay.reduce((duration, frame) => duration + frameMs(frame), 0);
+            const replayedMs = bufferedMs >= this.minReplayMs ? bufferedMs : 0;
+            // Clock zero belongs to the replay before the failure. Dropped queue
+            // audio shifts only the retained queue and the live audio after it.
+            generation += 1;
+            offsetMs = pushedMs - queuedMs - droppedQueuedMs - replayedMs;
+            current = next;
+            flushed = false;
+            attach(next, generation, offsetMs, replayedMs, droppedQueuedMs);
+            if (replayedMs > 0)
+                for (const f of replay)
+                    next.pushAudio(f);
+            // After a gap the ring's old span and the retained queue are not contiguous;
+            // keep only the audio a later replay can place on one timeline.
+            if (droppedQueuedMs > 0)
+                ring.clear();
+            for (const f of queue.splice(0)) {
+                ring.push(f);
+                next.pushAudio(f);
+            }
+            queuedMs = 0;
+            droppedQueuedMs = 0;
+            reconnecting = false;
+            facade.emit('stt:reconnected', { attempt, replayedMs });
+            facade.emit('stt:gap', { attempt, lostMs: this.clock() - startedAt });
+        };
+        const reconnect = (err) => {
+            if (reconnecting || closedByCaller || ended)
+                return reconnectPromise ?? Promise.resolve();
+            reconnectPromise = runReconnect(err);
+            return reconnectPromise;
+        };
+        attach(current, generation, offsetMs);
+        const session = Object.assign(facade, {
+            pushAudio: (frame) => {
+                if (ended || closedByCaller)
+                    return;
+                flushed = false;
+                flushing = undefined;
+                pushedMs += frameMs(frame);
+                if (reconnecting) {
+                    queue.push(frame);
+                    queuedMs += frameMs(frame);
+                    while (queuedMs > this.queueCapacityMs && queue.length) {
+                        const old = queue.shift();
+                        queuedMs -= frameMs(old);
+                        droppedQueuedMs += frameMs(old);
+                        dropped += 1;
+                    }
+                    return;
+                }
+                ring.push(frame);
+                try {
+                    current.pushAudio(frame);
+                }
+                catch (err) {
+                    void reconnect(err instanceof Error ? err : new Error(String(err)));
+                }
+            },
+            /**
+             * Waits for any reconnect and its replay, then drains the adopted session. A close during the flush or
+             * before subsequent provider activity or audio ends the stream without reconnecting.
+             */
+            flush: async () => {
+                if (ended || closedByCaller)
+                    return;
+                while (reconnecting && reconnectPromise)
+                    await reconnectPromise;
+                if (ended || closedByCaller)
+                    return;
+                const target = current;
+                flushed = true;
+                flushing = target;
+                try {
+                    await target.flush();
+                }
+                finally {
+                    if (flushing === target)
+                        flushing = undefined;
+                }
+            },
+            /** Closes the live session and emits `close` once, from the facade itself: what the provider does after is ignored. */
+            close: () => {
+                if (closedByCaller)
+                    return;
+                closedByCaller = true;
+                try {
+                    current.close();
+                }
+                catch {
+                    // already gone
+                }
+                finish();
+            },
+        });
+        Object.defineProperty(session, 'droppedFrames', { get: () => dropped, enumerable: true });
+        return session;
+    }
+}
+//# sourceMappingURL=ReconnectingSTT.js.map