livekit-plugin-parrot-stt 0.1.0

package/src/stt.ts

@@ -0,0 +1,280 @@
+// Ringg Parrot STT plugin for LiveKit Agents.
+//
+// Implements the streaming STT interface (`stt.STT` + `stt.SpeechStream`) by
+// speaking the Ringg WebSocket protocol directly (there is no JS SDK).
+
+import {
+  type APIConnectOptions,
+  type AudioBuffer,
+  AudioByteStream,
+  asLanguageCode,
+  log,
+  stt,
+} from '@livekit/agents';
+import type { AudioFrame } from '@livekit/rtc-node';
+import { type RawData, WebSocket } from 'ws';
+import {
+  type RinggSTTOptions,
+  type RinggServerEvent,
+  type RinggTranscriptEvent,
+  buildStartPayload,
+  defaultRinggSTTOptions,
+} from './models.js';
+
+const sleep = (ms: number): Promise<void> => new Promise((r) => setTimeout(r, ms));
+
+export class STT extends stt.STT {
+  label = 'ringg.STT';
+  #opts: RinggSTTOptions;
+
+  constructor(opts: Partial<RinggSTTOptions> = {}) {
+    super({ streaming: true, interimResults: true });
+    this.#opts = { ...defaultRinggSTTOptions, ...opts };
+    if (!this.#opts.apiKey) {
+      throw new Error(
+        'Ringg STT: API key is required — set $RINGG_STT_API_KEY or pass { apiKey }.',
+      );
+    }
+    if (!this.#opts.url) {
+      throw new Error('Ringg STT: url is required — set $RINGG_STT_URL or pass { url }.');
+    }
+  }
+
+  get provider(): string {
+    return 'ringg';
+  }
+
+  get model(): string {
+    return `ringg-parrot-${this.#opts.language}`;
+  }
+
+  updateOptions(opts: Partial<RinggSTTOptions>): void {
+    this.#opts = { ...this.#opts, ...opts };
+  }
+
+  async _recognize(_frame: AudioBuffer): Promise<stt.SpeechEvent> {
+    throw new Error(
+      'Ringg STT plugin is streaming-only; use stream(). ' +
+        '(Ringg also exposes a REST /transcriptions endpoint, but it is not wired into this plugin.)',
+    );
+  }
+
+  stream(options?: { connOptions?: APIConnectOptions }): stt.SpeechStream {
+    return new SpeechStream(this, this.#opts, options?.connOptions);
+  }
+}
+
+export class SpeechStream extends stt.SpeechStream {
+  label = 'ringg.SpeechStream';
+  #opts: RinggSTTOptions;
+  #logger = log();
+  #speaking = false;
+  #requestId?: string;
+
+  constructor(sttImpl: STT, opts: RinggSTTOptions, connOptions?: APIConnectOptions) {
+    // Passing sampleRate as `neededSampleRate` makes the base class resample
+    // incoming (typically 48kHz) LiveKit frames down to Ringg's rate for us.
+    super(sttImpl, opts.sampleRate, connOptions);
+    this.#opts = opts;
+  }
+
+  protected async run(): Promise<void> {
+    const ws = new WebSocket(this.#opts.url);
+    ws.binaryType = 'nodebuffer';
+
+    // 1) wait for the socket to open
+    await new Promise<void>((resolve, reject) => {
+      const onErr = (err: Error) => reject(err);
+      ws.once('open', () => {
+        ws.off('error', onErr);
+        resolve();
+      });
+      ws.once('error', onErr);
+    });
+
+    // 2) send the `start` handshake and wait for the `ready` reply
+    ws.send(JSON.stringify(buildStartPayload(this.#opts)));
+    await new Promise<void>((resolve, reject) => {
+      const onMsg = (data: RawData, isBinary: boolean) => {
+        if (isBinary) return;
+        let ev: RinggServerEvent;
+        try {
+          ev = JSON.parse(data.toString());
+        } catch {
+          return;
+        }
+        if (ev.type === 'ready') {
+          this.#requestId = (ev as { request_id?: string }).request_id;
+          cleanup();
+          resolve();
+        } else if (ev.type === 'error') {
+          cleanup();
+          reject(
+            new Error(
+              `Ringg STT: handshake rejected — ${(ev as { detail?: string }).detail ?? 'unknown error'}`,
+            ),
+          );
+        }
+      };
+      const onClose = (code: number) => {
+        cleanup();
+        reject(new Error(`Ringg STT: connection closed during handshake (code ${code})`));
+      };
+      const cleanup = () => {
+        ws.off('message', onMsg);
+        ws.off('close', onClose);
+      };
+      ws.on('message', onMsg);
+      ws.once('close', onClose);
+    });
+
+    this.#logger
+      .child({ url: this.#opts.url, language: this.#opts.language, requestId: this.#requestId })
+      .debug('ringg stt: connected');
+
+    let closed = false;
+    const closeWs = () => {
+      if (!closed) {
+        closed = true;
+        try {
+          ws.close();
+        } catch {
+          /* ignore */
+        }
+      }
+    };
+
+    // keepalive: application-level JSON ping (Ringg replies with `pong`)
+    let keepalive: ReturnType<typeof setInterval> | undefined;
+    if (this.#opts.keepaliveIntervalMs > 0) {
+      keepalive = setInterval(() => {
+        if (ws.readyState === WebSocket.OPEN) {
+          try {
+            ws.send(JSON.stringify({ type: 'ping' }));
+          } catch {
+            /* ignore */
+          }
+        }
+      }, this.#opts.keepaliveIntervalMs);
+    }
+
+    // 3) receive loop — resolves when the server closes the socket
+    const recvTask = new Promise<void>((resolve, reject) => {
+      ws.on('message', (data: RawData, isBinary: boolean) => {
+        if (isBinary) return; // Ringg only sends JSON text frames
+        let ev: RinggServerEvent;
+        try {
+          ev = JSON.parse(data.toString());
+        } catch {
+          return;
+        }
+        this.#handleEvent(ev);
+      });
+      ws.once('close', () => resolve());
+      ws.once('error', (err) => reject(err));
+    });
+
+    // 4) send loop — reads resampled frames from `this.input`, ships raw PCM16
+    const sendTask = async () => {
+      const samples20ms = Math.floor(this.#opts.sampleRate / 50); // 20ms chunks
+      const bstream = new AudioByteStream(this.#opts.sampleRate, 1, samples20ms);
+      const shipFrames = (frames: AudioFrame[]) => {
+        for (const f of frames) {
+          if (ws.readyState === WebSocket.OPEN) {
+            // send exactly this frame's bytes (raw PCM16 LE), regardless of view offset
+            ws.send(new Uint8Array(f.data.buffer, f.data.byteOffset, f.data.byteLength));
+          }
+        }
+      };
+
+      for (;;) {
+        if (closed) break;
+        const { done, value } = await this.input.next();
+        if (done) break;
+        if (value === SpeechStream.FLUSH_SENTINEL) {
+          shipFrames(bstream.flush());
+        } else {
+          shipFrames(bstream.write((value as AudioFrame).data.buffer));
+        }
+      }
+
+      // input exhausted: flush remaining audio and ask the server to finalize
+      shipFrames(bstream.flush());
+      if (ws.readyState === WebSocket.OPEN) {
+        ws.send(JSON.stringify({ type: 'end' }));
+      }
+    };
+
+    // abort (session close / cancel) → stop promptly
+    const abortPromise = new Promise<void>((resolve) => {
+      const signal = this.abortController.signal;
+      if (signal.aborted) return resolve();
+      signal.addEventListener('abort', () => resolve(), { once: true });
+    });
+
+    // finite input (a file): after the send loop finishes, wait a grace period
+    // for trailing final transcripts, then let the socket close.
+    const finiteInputPath = (async () => {
+      await sendTask();
+      await sleep(this.#opts.finalizeGraceMs);
+    })();
+
+    try {
+      await Promise.race([abortPromise, recvTask, finiteInputPath]);
+    } finally {
+      if (keepalive) clearInterval(keepalive);
+      closeWs();
+      // let recvTask settle so its listeners are cleaned up
+      await recvTask.catch(() => {});
+    }
+  }
+
+  #handleEvent(ev: RinggServerEvent): void {
+    switch (ev.type) {
+      case 'transcript': {
+        const t = ev as RinggTranscriptEvent;
+        const text = (t.transcription ?? '').trim();
+        if (!text) return;
+
+        if (!this.#speaking) {
+          this.#speaking = true;
+          this.queue.put({ type: stt.SpeechEventType.START_OF_SPEECH });
+        }
+
+        const data: stt.SpeechData = {
+          language: asLanguageCode(t.language ?? this.#opts.language),
+          text,
+          startTime: 0,
+          endTime: 0,
+          confidence: 1.0,
+        };
+
+        if (t.is_final) {
+          this.queue.put({
+            type: stt.SpeechEventType.FINAL_TRANSCRIPT,
+            alternatives: [data],
+            requestId: this.#requestId,
+          });
+          if (this.#speaking) {
+            this.#speaking = false;
+            this.queue.put({ type: stt.SpeechEventType.END_OF_SPEECH });
+          }
+        } else {
+          this.queue.put({
+            type: stt.SpeechEventType.INTERIM_TRANSCRIPT,
+            alternatives: [data],
+            requestId: this.#requestId,
+          });
+        }
+        break;
+      }
+      case 'error': {
+        this.#logger.child({ event: ev }).error('ringg stt: server error');
+        break;
+      }
+      // ready (duplicate), ack, pong, and unknown types are ignored
+      default:
+        break;
+    }
+  }
+}