@pipecat-ai/moq-transport 0.1.0

package/LICENSE

@@ -0,0 +1,24 @@
+BSD 2-Clause License
+
+Copyright (c) 2024, Daily
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

package/README.md

@@ -0,0 +1,125 @@
+# MoQ (Media over QUIC) Transport
+
+[![Docs](https://img.shields.io/badge/documentation-blue)](https://docs.pipecat.ai/client/js/transports)
+![NPM Version](https://img.shields.io/npm/v/@pipecat-ai/moq-transport)
+[![Demo](https://img.shields.io/badge/Demo-forestgreen)](https://github.com/pipecat-ai/voice-ui-kit/tree/main/examples/01-console)
+
+Media-over-QUIC transport package for use with `@pipecat-ai/client-js`.
+
+## Installation
+
+```bash copy
+npm install \
+@pipecat-ai/client-js \
+@pipecat-ai/moq-transport
+```
+
+## Overview
+
+The `MoqTransport` class connects a `PipecatClient` to a Pipecat MoQ bot, either through a MoQ relay or directly to a bot running in serve mode. It publishes the local microphone as an Opus broadcast and consumes the bot's broadcast — both audio and a bot→client RTVI message track — using catalog discovery, so codec and sample rate are negotiated at connect time rather than pinned in code. RTVI messages also flow client→bot over a matching transcript track on the client's own broadcast, carrying `client-ready`, typed text input, function-call results, and other client→server RTVI traffic.
+
+Connection management uses WebTransport with a WebSocket fallback (raced by `@moq/net`), with auto-reconnect via `Connection.Reload`.
+
+## Features
+
+- 🎤 Microphone capture and Opus publish (`@moq/publish`)
+- 📡 WebTransport with WebSocket fallback and auto-reconnect (`@moq/net`)
+- 🎧 Catalog-driven bot audio playback with bounded-latency jitter buffering (`@moq/watch`)
+- 💬 Bidirectional RTVI messages over dedicated transcript tracks (`@moq/json`) — bot→client events/transcripts, and client→bot messages like `client-ready`
+- 🔐 `serverCertificateHashes` pinning for self-signed dev relays
+
+## Usage
+
+### Basic Setup
+
+```javascript
+import { PipecatClient } from "@pipecat-ai/client-js";
+import { MoqTransport } from "@pipecat-ai/moq-transport";
+
+const transport = new MoqTransport({
+  relayUrl: "https://relay.example.com:4080/moq",
+});
+
+const pcClient = new PipecatClient({
+  transport,
+  callbacks: {
+    // Event handlers
+  },
+});
+
+await pcClient.connect();
+```
+
+### Self-signed dev relay
+
+```javascript
+const transport = new MoqTransport({
+  relayUrl: "https://localhost:4080/moq",
+  serverCertificateHashes: [
+    { algorithm: "sha-256", value: certHashBytes },
+  ],
+});
+```
+
+### Configuration Options
+
+```typescript
+interface MoqTransportOptions {
+  relayUrl: string;                              // Required: full URL of the MoQ peer
+  serverCertificateHashes?: WebTransportHash[];  // Optional: pinned cert hashes for self-signed dev setups
+  clientId?: string;                             // Optional: this client's participant id (default "client0")
+  botId?: string;                                // Optional: peer (bot) participant id to consume (default "bot0")
+  namespace?: string;                            // Optional: top-level namespace / room name (default "pipecat")
+  transcriptTrack?: string;                      // Optional: track name for the bidirectional RTVI transcript channel (default "transcript.json.z")
+  audioLatencyMs?: number;                       // Optional: jitter buffer floor latency in ms (default 80)
+  audioBufferMaxMs?: number | "real-time";       // Optional: buffered-playback latency ceiling in ms, or "real-time" to collapse to the floor (default 30000)
+  audioSampleRate?: number;                      // Optional: mic publish sample rate in Hz; one of 8000/12000/16000/24000/48000 (default 48000)
+}
+```
+
+Broadcast paths are derived as `<namespace>/<clientId>` (publish) and `<namespace>/<botId>` (subscribe). Audio track names inside each broadcast are discovered from the bot's catalog, so they aren't configured directly.
+
+### Connecting via a bot `/start` endpoint
+
+If your `/start` endpoint returns the bot's MoQ config nested under a `moq` key (the shape the bot's `pipecat.transports.moq.transport` returns), pass that response straight to `connect()` / `PipecatClient.startBotAndConnect()` — `MoqTransport` unwraps it (including base64-decoding `certHash` into `serverCertificateHashes`) in `_validateConnectionParams`, no app-side transform needed:
+
+```json
+{
+  "moq": {
+    "relayUrl": "https://relay.example.com:4080/moq",
+    "certHash": "base64-encoded-sha-256-or-null",
+    "namespace": "pipecat",
+    "clientId": "client0",
+    "botId": "bot0",
+    "transcriptTrack": "transcript.json.z"
+  }
+}
+```
+
+### Handling Events
+
+The transport implements the various [Pipecat event handlers](https://docs.pipecat.ai/client/js/api-reference/callbacks). Check out the docs or samples for more info.
+
+## API Reference
+
+### States
+
+The transport can be in one of these states:
+- "disconnected"
+- "initialized"
+- "connecting"
+- "connected"
+- "ready"
+- "disconnecting"
+- "error"
+
+## Error Handling
+
+The transport includes error handling for:
+- Microphone acquisition failures (`initDevices`, `_connect`)
+- Invalid `relayUrl`
+- WebTransport / WebSocket connection failures (surfaced via `@moq/net` auto-reconnect)
+- Catalog decode and audio decode errors (logged; the consume loop continues)
+
+## License
+BSD-2 Clause

package/dist/index.d.ts

@@ -0,0 +1,156 @@
+import { PipecatClientOptions, RTVIMessage, Transport, Tracks, TransportState } from "@pipecat-ai/client-js";
+/**
+ * Constructor options for the MoQ transport.
+ *
+ * The browser dials a MOQ peer (relay or bot in serve mode) at
+ * ``relayUrl`` and publishes its mic under ``<namespace>/<clientId>``
+ * while consuming the bot under ``<namespace>/<botId>``. Audio tracks
+ * inside each broadcast are catalog-driven (codec, sample rate, channel
+ * count are discovered at connect time), so they aren't pinned here.
+ */
+export interface MoqTransportOptions {
+    /**
+     * Full URL of the MOQ peer (e.g. ``https://relay.example.com:4080/moq``).
+     * WebTransport (HTTPS/HTTP3); @moq/net races WebSocket as a fallback
+     * if WebTransport isn't reachable.
+     */
+    relayUrl: string;
+    /**
+     * Pinned cert hashes for self-signed dev setups. Same shape as the
+     * native WebTransport ``serverCertificateHashes`` option.
+     */
+    serverCertificateHashes?: WebTransportHash[];
+    /**
+     * This client's participant id. Combined with ``namespace`` it forms
+     * the broadcast path the client publishes under: ``<namespace>/<clientId>``.
+     */
+    clientId?: string;
+    /**
+     * The peer (bot) participant id to consume. Subscriptions target
+     * ``<namespace>/<botId>``; track names come from the bot's catalog.
+     */
+    botId?: string;
+    /**
+     * Top-level namespace (analogous to a room name). Defaults to ``"pipecat"``.
+     */
+    namespace?: string;
+    /**
+     * Track name for RTVI server→client messages (transcripts, bot-ready,
+     * speech events, etc.). The client subscribes at
+     * ``<namespace>/<botId>/<transcriptTrack>``. Defaults to
+     * ``"transcript.json.z"``.
+     *
+     * Stays pinned because the transcript is a ``@moq/json`` stream track
+     * (an ordered, lossless append-log, wire-compatible with the bot's Rust
+     * ``moq_json::stream``), not a catalog rendition — the catalog only
+     * describes media tracks (audio/video). The ``.z`` suffix marks that the
+     * stream is compressed.
+     */
+    transcriptTrack?: string;
+    /**
+     * Latency floor (ms) — the jitter buffer the player keeps before
+     * playback. Lower = more interactive, more drops; higher = smoother,
+     * more glass-to-glass delay. Maps to the `min` bound of the
+     * ``@moq/watch`` ``Sync.latency`` range.
+     */
+    audioLatencyMs?: number;
+    /**
+     * Latency ceiling for buffered playback (the `max` bound of the
+     * ``@moq/watch`` ``Sync.latency`` range).
+     *
+     * The bot writes TTS audio faster than real-time with future-dated
+     * timestamps; the player buffers it and plays at the encoded pace
+     * instead of skipping ahead. This sets how much it's allowed to
+     * build up before re-anchoring:
+     *
+     * - a number (default: 30 s) — cap the buffer at that many ms. The
+     *   player retains up to this much faster-than-real-time audio before
+     *   dropping; an interruption (``user-started-speaking``) flushes early
+     *   via ``reset()``. The bot paces a little under this so it never
+     *   actually overruns the cap.
+     * - ``"real-time"`` — collapse to the floor (minimize latency, the old
+     *   skip-ahead behavior; only useful with a live, real-time publisher).
+     */
+    audioBufferMaxMs?: number | "real-time";
+    /**
+     * Sample rate (Hz) the client publishes its mic audio at. Must be one
+     * of Opus's supported rates: 8000, 12000, 16000, 24000, or 48000.
+     * Defaults to ``48000`` to match the typical browser mic rate.
+     *
+     * The bot reads this from the client's catalog and resamples to its
+     * own ``audio_in_sample_rate`` after Opus decode, so exact agreement
+     * isn't required — pinning a known value just keeps the catalog
+     * unambiguous.
+     *
+     * Bot-side playback sample rate isn't an option here: ``MoqTransport``
+     * reads whatever the bot advertises in its catalog and configures its
+     * playback ``AudioContext`` accordingly.
+     */
+    audioSampleRate?: number;
+}
+/**
+ * ``MoqTransport`` — Pipecat Client SDK transport plugin for Media-over-QUIC.
+ *
+ * Built on the official ``moq`` library family:
+ *
+ * - ``@moq/net`` for connection management (``Connection.Reload``
+ *   auto-reconnects on drops; races WebTransport + WebSocket).
+ * - ``@moq/publish`` for mic capture and Opus encoding via
+ *   ``Publish.Source.Microphone`` + ``Publish.Broadcast``.
+ * - ``@moq/watch`` for bot audio playback: ``Watch.Broadcast`` discovers
+ *   the catalog, ``Watch.Audio.Source`` picks a rendition, and
+ *   ``Watch.Audio.Decoder`` + ``Watch.Audio.Emitter`` drive an
+ *   ``AudioWorklet`` with a ring buffer (loss-tolerant, low-latency).
+ * - ``@moq/json`` for the transcript tracks — the ``Stream`` mode: an
+ *   ordered, lossless append-log (wire-compatible with the bot's Rust
+ *   ``moq_json::stream``), so no RTVI event is dropped. Not the snapshot
+ *   mode, which collapses to the latest value.
+ * - ``@moq/signals`` for reactive plumbing.
+ *
+ * Two transcript tracks flow in opposite directions, both named
+ * ``transcript.json.z`` (or whatever the shared ``transcriptTrack`` option
+ * is set to) on their respective broadcasts:
+ *
+ * - Bot → client: bot publishes RTVI events (``bot-output``,
+ *   ``bot-transcription``, ``bot-started-speaking``, ``metrics``, …);
+ *   client subscribes and delivers to ``PipecatClient`` via ``_onMessage``.
+ * - Client → bot: ``sendMessage`` writes each RTVI message on the client's
+ *   own transcript track; the bot's Python transport
+ *   (``pipecat.transports.moq.transport``) subscribes and dispatches them
+ *   through ``RTVIProcessor`` the same way any other transport does. This
+ *   is what carries ``client-ready`` for protocol-version negotiation,
+ *   typed text input, function-call results, and any other client→server
+ *   RTVI traffic.
+ */
+export class MoqTransport extends Transport {
+    static SERVICE_NAME: string;
+    protected _state: TransportState;
+    constructor(options: MoqTransportOptions);
+    initialize(options: PipecatClientOptions, messageHandler: (ev: RTVIMessage) => void): void;
+    initDevices(): Promise<void>;
+    _validateConnectionParams(connectParams?: unknown): MoqTransportOptions;
+    _connect(connectParams?: MoqTransportOptions): Promise<void>;
+    _disconnect(): Promise<void>;
+    sendReadyMessage(): void;
+    get state(): TransportState;
+    set state(next: TransportState);
+    getAllMics(): Promise<MediaDeviceInfo[]>;
+    getAllCams(): Promise<MediaDeviceInfo[]>;
+    getAllSpeakers(): Promise<MediaDeviceInfo[]>;
+    updateMic(micId: string): void;
+    updateCam(_camId: string): void;
+    updateSpeaker(_speakerId: string): void;
+    get selectedMic(): MediaDeviceInfo | Record<string, never>;
+    get selectedCam(): MediaDeviceInfo | Record<string, never>;
+    get selectedSpeaker(): MediaDeviceInfo | Record<string, never>;
+    enableMic(enable: boolean): void;
+    enableCam(_enable: boolean): void;
+    enableScreenShare(_enable: boolean): void;
+    get isCamEnabled(): boolean;
+    get isMicEnabled(): boolean;
+    get isSharingScreen(): boolean;
+    sendMessage(message: RTVIMessage): void;
+    tracks(): Tracks;
+}
+
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"mappings":";AA4CA;;;;;;;;GAQG;AACH;IACE;;;;OAIG;IACH,QAAQ,EAAE,MAAM,CAAC;IAEjB;;;OAGG;IACH,uBAAuB,CAAC,EAAE,gBAAgB,EAAE,CAAC;IAE7C;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB;;;OAGG;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf;;OAEG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;;;;;;;;;;OAWG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;IAEzB;;;;;OAKG;IACH,cAAc,CAAC,EAAE,MAAM,CAAC;IAExB;;;;;;;;;;;;;;;;OAgBG;IACH,gBAAgB,CAAC,EAAE,MAAM,GAAG,WAAW,CAAC;IAExC;;;;;;;;;;;;;OAaG;IACH,eAAe,CAAC,EAAE,MAAM,CAAC;CAC1B;AA2DD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AACH,yBAA0B,SAAQ,SAAS;IACzC,OAAc,YAAY,SAAmB;IAmD7C,UAAkB,MAAM,EAAE,cAAc,CAAC;gBAE7B,OAAO,EAAE,mBAAmB;IAUxC,UAAU,CACR,OAAO,EAAE,oBAAoB,EAC7B,cAAc,EAAE,CAAC,EAAE,EAAE,WAAW,KAAK,IAAI,GACxC,IAAI;IAOD,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAelC,yBAAyB,CAAC,aAAa,CAAC,EAAE,OAAO,GAAG,mBAAmB;IAsCjE,QAAQ,CAAC,aAAa,CAAC,EAAE,mBAAmB,GAAG,OAAO,CAAC,IAAI,CAAC;IAwP5D,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IA2ClC,gBAAgB,IAAI,IAAI;IAaxB,IAAI,KAAK,IAAI,cAAc,CAE1B;IAED,IAAI,KAAK,CAAC,IAAI,EAAE,cAAc,EAI7B;IAMK,UAAU,IAAI,OAAO,CAAC,eAAe,EAAE,CAAC;IAIxC,UAAU,IAAI,OAAO,CAAC,eAAe,EAAE,CAAC;IAIxC,cAAc,IAAI,OAAO,CAAC,eAAe,EAAE,CAAC;IAIlD,SAAS,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;IAI9B,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IAE/B,aAAa,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI;IAEvC,IAAI,WAAW,IAAI,eAAe,GAAG,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,CAOzD;IAED,IAAI,WAAW,IAAI,eAAe,GAAG,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,CAEzD;IAED,IAAI,eAAe,IAAI,eAAe,GAAG,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,CAE7D;IAED,SAAS,CAAC,MAAM,EAAE,OAAO,GAAG,IAAI;IAIhC,SAAS,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI;IAEjC,iBAAiB,CAAC,OAAO,EAAE,OAAO,GAAG,IAAI;IAEzC,IAAI,YAAY,IAAI,OAAO,CAE1B;IAED,IAAI,YAAY,IAAI,OAAO,CAE1B;IAED,IAAI,eAAe,IAAI,OAAO,CAE7B;IAMD,WAAW,CAAC,OAAO,EAAE,WAAW,GAAG,IAAI;IAevC,MAAM,IAAI,MAAM;CA4DjB","sources":["transports/moq-transport/src/src/moqTransport.ts","transports/moq-transport/src/src/index.ts","transports/moq-transport/src/index.ts"],"sourcesContent":[null,null,"export * from \"./moqTransport\";\n"],"names":[],"version":3,"file":"index.d.ts.map"}