@ariaflowagents/livekit-plugin-transport-twilio 0.9.0

package/README.md

@@ -0,0 +1,221 @@
+# @ariaflow/livekit-plugin-transport-twilio
+
+Twilio Media Streams transport adapter for AriaFlow voice agents. Works with Node.js, Bun, Deno, and any WebSocket-enabled runtime. Cloudflare Workers support is currently unstable.
+
+## Features
+
+- ⚠️ **Cloudflare Workers Compatible** (Unstable) - No Node.js dependencies
+- ✅ **Twilio Media Streams Support** - Native integration with Twilio's API
+- ✅ **Automatic Codec Conversion** - G.711 μ-law ↔ PCM with resampling (8kHz ↔ 24kHz)
+- ✅ **AriaFlow Native** - Works with `createAriaFlowSession` and AriaFlow Runtime
+- ✅ **Session Management** - Full lifecycle management with SessionManager
+
+## Installation
+
+```bash
+# Using npm
+npm install @ariaflow/livekit-plugin @ariaflow/livekit-plugin-transport-twilio
+
+# Using bun
+bun add @ariaflow/livekit-plugin @ariaflow/livekit-plugin-transport-twilio
+```
+
+## Quick Start
+
+### Cloudflare Workers (Unstable)
+
+> ⚠️ **Note:** The Cloudflare Workers integration is currently unstable and not officially supported. Use at your own risk.
+
+```typescript
+// wrangler.toml:
+// name = "ariaflow-twilio-agent"
+// compatibility_date = "2024-01-01"
+// compatibility_flags = ["nodejs_compat"]
+
+import { createTwilioWorker } from '@ariaflow/livekit-plugin-transport-twilio/cloudflare';
+import { createAriaFlowSession } from '@ariaflow/livekit-plugin';
+import { GeminiLiveSTT, GeminiLiveTTS } from '@ariaflow/livekit-plugin/gemini';
+
+const agentConfig = {
+  agents: [{ id: 'assistant', prompt: 'You are helpful.' }],
+  defaultAgentId: 'assistant',
+};
+
+export default createTwilioWorker({
+  agent: () => createAriaFlowSession({
+    runtime: agentConfig,
+    stt: new GeminiLiveSTT(),
+    tts: new GeminiLiveTTS(),
+    greeting: 'Hello! How can I help?',
+  }),
+});
+```
+
+### Node.js / Deno / Any WebSocket
+
+```typescript
+import { WebSocketServer } from 'ws';
+import { TwilioTransportAdapter } from '@ariaflow/livekit-plugin-transport-twilio';
+import { createAriaFlowSession } from '@ariaflow/livekit-plugin';
+import { SessionManager } from '@ariaflow/livekit-plugin';
+import { GeminiLiveSTT, GeminiLiveTTS } from '@ariaflow/livekit-plugin/gemini';
+
+const wss = new WebSocketServer({ port: 8080 });
+const sessionManager = new SessionManager();
+
+const agentConfig = {
+  agents: [{ id: 'assistant', prompt: 'You are helpful.' }],
+  defaultAgentId: 'assistant',
+};
+
+wss.on('connection', (ws) => {
+  const transport = new TwilioTransportAdapter({
+    send: (msg) => ws.send(msg),
+  });
+
+  ws.on('message', (data) => {
+    transport.handleMessage(data.toString());
+  });
+
+  ws.on('close', async () => {
+    await transport.close();
+  });
+
+  // Wait for 'connected' event before starting session
+  ws.once('message', async (data) => {
+    const event = JSON.parse(data.toString());
+    if (event.event === 'connected') {
+      const { agent, sessionOptions } = createAriaFlowSession({
+        runtime: agentConfig,
+        stt: new GeminiLiveSTT(),
+        tts: new GeminiLiveTTS(),
+      });
+
+      await sessionManager.startSession(transport, agent, sessionOptions);
+    }
+  });
+});
+```
+
+## TwiML Setup
+
+Create a TwiML bin in your Twilio console:
+
+```xml
+<?xml version="1.0" encoding="UTF-8"?>
+<Response>
+  <Say>Connecting to AI assistant...</Say>
+  <Connect>
+    <Stream url="wss://your-worker.workers.dev/voice/stream" />
+  </Connect>
+</Response>
+```
+
+## Audio Specifications
+
+| Direction | Twilio Format | Internal Format | Sample Rate |
+|-----------|---------------|-----------------|-------------|
+| Inbound | G.711 μ-law | PCM S16 LE | 8kHz → 24kHz |
+| Outbound | PCM S16 LE | G.711 μ-law | 24kHz → 8kHz |
+
+The transport handles all codec conversion and resampling automatically.
+
+## API Reference
+
+### `TwilioTransportAdapter`
+
+Main transport adapter class.
+
+```typescript
+const transport = new TwilioTransportAdapter({
+  id: 'optional-custom-id',
+  send: (message: string) => {
+    // Send message to WebSocket
+    ws.send(message);
+  },
+});
+```
+
+**Properties:**
+- `id: string` - Unique identifier for this transport
+- `audioInput: TwilioAudioInput` - Audio input stream
+- `audioOutput: TwilioAudioOutput` - Audio output stream
+- `textOutput: TwilioTextOutput` - Text output stream
+- `config: TransportAdapterConfig` - Configuration object
+- `isOpen: boolean` - Whether the transport is active
+
+**Methods:**
+- `handleMessage(message: string)` - Process incoming Twilio message
+- `clearAudio()` - Clear audio buffer
+- `close()` - Close the transport
+
+### `createTwilioWorker(options)` (Unstable)
+
+> ⚠️ **Note:** This Cloudflare Workers integration helper is currently unstable and not officially supported.
+
+Cloudflare Workers integration helper.
+
+```typescript
+export default createTwilioWorker({
+  agent: () => createAriaFlowSession({...}),
+});
+```
+
+**Options:**
+- `agent: () => Agent | { agent: Agent; sessionOptions: any }` - Agent factory
+- `stt?: any` - STT instance (for raw LiveKit pattern)
+- `llm?: any` - LLM instance
+- `tts?: any` - TTS instance
+- `vad?: any` - VAD instance
+- `turnDetection?: any` - Turn detection config
+
+## Environment Compatibility
+
+| Runtime | WebSocket Support | Status |
+|---------|------------------|--------|
+| Cloudflare Workers | ✅ Native | ⚠️ Unstable |
+| Node.js | ✅ via `ws` package | ✅ Supported |
+| Deno | ✅ Native | ✅ Supported |
+| Bun | ✅ Native | ✅ Supported |
+| Browser | ✅ Native | ✅ Supported (development only) |
+
+## Limitations
+
+1. **Twilio Sample Rate**: Twilio uses 8kHz G.711 μ-law. The transport resamples to 24kHz for AriaFlow, but this adds slight latency.
+2. **Codec**: Only G.711 μ-law is supported (Twilio's standard).
+3. **Latency**: Expect 500-1000ms additional latency compared to direct WebRTC.
+4. **Full Duplex**: Twilio Media Streams is full-duplex, but the implementation treats inbound/outbound as separate streams.
+
+## Troubleshooting
+
+### Audio cutting out
+- Check your WebSocket connection is stable
+- Verify `send` callback is working correctly
+- Check for errors in the console
+
+### Poor audio quality
+- The 8kHz sample rate is lower than typical 24kHz
+- This is a Twilio limitation, not the transport
+- Consider using Twilio's programmable voice SDK for higher quality
+
+### Connection issues
+- Ensure your WebSocket URL is publicly accessible
+- Check Twilio can reach your server (no firewall blocking)
+- Verify CORS headers if using browser-based testing
+
+## Examples
+
+See `examples/` directory for:
+- `cloudflare_worker.ts` - Complete Cloudflare Worker example (⚠️ Unstable)
+- `wrangler.toml` - Deployment configuration
+
+## License
+
+Apache-2.0
+
+## Related Packages
+
+- `@ariaflow/livekit-plugin` - Core AriaFlow LiveKit integration
+- `@ariaflow/livekit-plugin/gemini` - Gemini STT/TTS plugins
+- `@ariaflow/livekit-plugin-transport-ws` - WebSocket transport
+- `@ariaflow/livekit-plugin-transport-http` - HTTP/SSE transport

package/dist/audio_input.d.ts

@@ -0,0 +1,42 @@
+/**
+ * Audio input for Twilio Media Streams.
+ *
+ * Receives mu-law encoded audio at 8kHz from Twilio, decodes to PCM,
+ * resamples to 24kHz via a streaming Sox sinc resampler, and provides
+ * AudioFrame objects to the STT pipeline.
+ *
+ * The resampler is allocated once and reused across all media events
+ * for the lifetime of this input (local Rust FFI, no LiveKit server).
+ */
+import { AudioInput } from '@ariaflow/livekit-plugin';
+import type { TwilioMediaEvent } from './twilio_protocol.js';
+/**
+ * Receives audio from Twilio Media Streams and provides it as a
+ * ReadableStream<AudioFrame> for the STT pipeline.
+ *
+ * Flow:
+ * 1. Twilio sends mu-law audio at 8kHz (base64 encoded)
+ * 2. Decode mu-law to PCM Int16
+ * 3. Wrap in AudioFrame at 8kHz
+ * 4. Push through streaming resampler -> 24kHz AudioFrames
+ * 5. Write frames to STT stream
+ */
+export declare class TwilioAudioInput extends AudioInput {
+    private currentStreamId;
+    private streamWriter;
+    private closed;
+    private resampler;
+    /**
+     * Process a media event from Twilio.
+     *
+     * @param event - Twilio media event with base64 mu-law payload
+     */
+    handleMediaEvent(event: TwilioMediaEvent): void;
+    private startNewStream;
+    /**
+     * End the current audio stream (flush remaining audio from resampler).
+     */
+    endCurrentStream(): void;
+    close(): Promise<void>;
+}
+//# sourceMappingURL=audio_input.d.ts.map

package/dist/audio_input.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"audio_input.d.ts","sourceRoot":"","sources":["../src/audio_input.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AACH,OAAO,EAAE,UAAU,EAAc,MAAM,0BAA0B,CAAC;AAIlE,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,sBAAsB,CAAC;AAK7D;;;;;;;;;;GAUG;AACH,qBAAa,gBAAiB,SAAQ,UAAU;IAC9C,OAAO,CAAC,eAAe,CAAuB;IAC9C,OAAO,CAAC,YAAY,CAAwD;IAC5E,OAAO,CAAC,MAAM,CAAkB;IAChC,OAAO,CAAC,SAAS,CAA2D;IAE5E;;;;OAIG;IACH,gBAAgB,CAAC,KAAK,EAAE,gBAAgB,GAAG,IAAI;IAgD/C,OAAO,CAAC,cAAc;IAMtB;;OAEG;IACH,gBAAgB,IAAI,IAAI;IAuBlB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAK7B"}

package/dist/audio_input.js

@@ -0,0 +1,117 @@
+/**
+ * Audio input for Twilio Media Streams.
+ *
+ * Receives mu-law encoded audio at 8kHz from Twilio, decodes to PCM,
+ * resamples to 24kHz via a streaming Sox sinc resampler, and provides
+ * AudioFrame objects to the STT pipeline.
+ *
+ * The resampler is allocated once and reused across all media events
+ * for the lifetime of this input (local Rust FFI, no LiveKit server).
+ */
+import { AudioInput, AudioFrame } from '@ariaflow/livekit-plugin';
+import { createResampler } from '@ariaflow/livekit-plugin/utils/resample';
+import { TransformStream } from 'stream/web';
+import { mulawDecodeArray } from '@ariaflow/livekit-plugin/codec/g711';
+const TWILIO_SAMPLE_RATE = 8000;
+const TARGET_SAMPLE_RATE = 24000;
+/**
+ * Receives audio from Twilio Media Streams and provides it as a
+ * ReadableStream<AudioFrame> for the STT pipeline.
+ *
+ * Flow:
+ * 1. Twilio sends mu-law audio at 8kHz (base64 encoded)
+ * 2. Decode mu-law to PCM Int16
+ * 3. Wrap in AudioFrame at 8kHz
+ * 4. Push through streaming resampler -> 24kHz AudioFrames
+ * 5. Write frames to STT stream
+ */
+export class TwilioAudioInput extends AudioInput {
+    currentStreamId = null;
+    streamWriter = null;
+    closed = false;
+    resampler = createResampler(TWILIO_SAMPLE_RATE, TARGET_SAMPLE_RATE);
+    /**
+     * Process a media event from Twilio.
+     *
+     * @param event - Twilio media event with base64 mu-law payload
+     */
+    handleMediaEvent(event) {
+        if (this.closed)
+            return;
+        const payload = event.media.payload;
+        if (!payload)
+            return;
+        try {
+            // Decode base64 -> mu-law bytes
+            const mulawData = Uint8Array.from(atob(payload), (c) => c.charCodeAt(0));
+            // Decode mu-law to PCM at 8kHz
+            const pcm8kHz = mulawDecodeArray(mulawData);
+            // Wrap in AudioFrame at the input rate
+            const inputFrame = new AudioFrame(pcm8kHz, TWILIO_SAMPLE_RATE, 1, pcm8kHz.length);
+            // Resample 8kHz -> 24kHz via streaming resampler
+            const outputFrames = this.resampler.push(inputFrame);
+            if (outputFrames.length === 0)
+                return;
+            // Ensure we have an active stream
+            if (!this.currentStreamId) {
+                this.startNewStream();
+            }
+            // Write resampled frames to STT stream
+            for (const frame of outputFrames) {
+                if (this.streamWriter) {
+                    this.streamWriter.write(frame).catch((err) => {
+                        if (!this.closed) {
+                            console.error('[TwilioAudioInput] Error writing frame:', {
+                                error: err instanceof Error ? err.message : String(err),
+                                streamId: this.currentStreamId,
+                                timestamp: new Date().toISOString(),
+                            });
+                        }
+                    });
+                }
+            }
+        }
+        catch (error) {
+            console.error('[TwilioAudioInput] Error processing media event:', {
+                error: error instanceof Error ? error.message : String(error),
+                payloadLength: payload?.length || 0,
+                timestamp: new Date().toISOString(),
+            });
+        }
+    }
+    startNewStream() {
+        const { readable, writable } = new TransformStream();
+        this.streamWriter = writable.getWriter();
+        this.currentStreamId = this.multiStream.addInputStream(readable);
+    }
+    /**
+     * End the current audio stream (flush remaining audio from resampler).
+     */
+    endCurrentStream() {
+        // Flush resampler to emit any remaining buffered samples
+        if (this.streamWriter) {
+            for (const frame of this.resampler.flush()) {
+                this.streamWriter.write(frame).catch(() => { });
+            }
+            this.streamWriter.close().catch((err) => {
+                if (!this.closed) {
+                    console.error('[TwilioAudioInput] Error closing stream writer:', {
+                        error: err instanceof Error ? err.message : String(err),
+                        streamId: this.currentStreamId,
+                        timestamp: new Date().toISOString(),
+                    });
+                }
+            });
+            this.streamWriter = null;
+        }
+        this.currentStreamId = null;
+        // Re-create the resampler for the next stream (fresh state)
+        this.resampler = createResampler(TWILIO_SAMPLE_RATE, TARGET_SAMPLE_RATE);
+    }
+    async close() {
+        this.closed = true;
+        this.endCurrentStream();
+        await super.close();
+    }
+}
+//# sourceMappingURL=audio_input.js.map

package/dist/audio_input.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"audio_input.js","sourceRoot":"","sources":["../src/audio_input.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AACH,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AAClE,OAAO,EAAE,eAAe,EAAE,MAAM,yCAAyC,CAAC;AAC1E,OAAO,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAC7C,OAAO,EAAE,gBAAgB,EAAE,MAAM,qCAAqC,CAAC;AAGvE,MAAM,kBAAkB,GAAG,IAAI,CAAC;AAChC,MAAM,kBAAkB,GAAG,KAAK,CAAC;AAEjC;;;;;;;;;;GAUG;AACH,MAAM,OAAO,gBAAiB,SAAQ,UAAU;IACtC,eAAe,GAAkB,IAAI,CAAC;IACtC,YAAY,GAAmD,IAAI,CAAC;IACpE,MAAM,GAAY,KAAK,CAAC;IACxB,SAAS,GAAG,eAAe,CAAC,kBAAkB,EAAE,kBAAkB,CAAC,CAAC;IAE5E;;;;OAIG;IACH,gBAAgB,CAAC,KAAuB;QACtC,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO;QAExB,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC;QACpC,IAAI,CAAC,OAAO;YAAE,OAAO;QAErB,IAAI,CAAC;YACH,gCAAgC;YAChC,MAAM,SAAS,GAAG,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,CAAC;YAEzE,+BAA+B;YAC/B,MAAM,OAAO,GAAG,gBAAgB,CAAC,SAAS,CAAC,CAAC;YAE5C,uCAAuC;YACvC,MAAM,UAAU,GAAG,IAAI,UAAU,CAAC,OAAO,EAAE,kBAAkB,EAAE,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;YAElF,iDAAiD;YACjD,MAAM,YAAY,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YACrD,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC;gBAAE,OAAO;YAEtC,kCAAkC;YAClC,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;gBAC1B,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,CAAC;YAED,uCAAuC;YACvC,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;gBACjC,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;oBACtB,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;wBAC3C,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;4BACjB,OAAO,CAAC,KAAK,CAAC,yCAAyC,EAAE;gCACvD,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;gCACvD,QAAQ,EAAE,IAAI,CAAC,eAAe;gCAC9B,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;6BACpC,CAAC,CAAC;wBACL,CAAC;oBACH,CAAC,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,kDAAkD,EAAE;gBAChE,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;gBAC7D,aAAa,EAAE,OAAO,EAAE,MAAM,IAAI,CAAC;gBACnC,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;aACpC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAEO,cAAc;QACpB,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,IAAI,eAAe,EAAc,CAAC;QACjE,IAAI,CAAC,YAAY,GAAG,QAAQ,CAAC,SAAS,EAAE,CAAC;QACzC,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC,WAAW,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;IACnE,CAAC;IAED;;OAEG;IACH,gBAAgB;QACd,yDAAyD;QACzD,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtB,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,EAAE,CAAC;gBAC3C,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;YACjD,CAAC;YACD,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;gBACtC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;oBACjB,OAAO,CAAC,KAAK,CAAC,iDAAiD,EAAE;wBAC/D,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;wBACvD,QAAQ,EAAE,IAAI,CAAC,eAAe;wBAC9B,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;qBACpC,CAAC,CAAC;gBACL,CAAC;YACH,CAAC,CAAC,CAAC;YACH,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QAC3B,CAAC;QACD,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC;QAE5B,4DAA4D;QAC5D,IAAI,CAAC,SAAS,GAAG,eAAe,CAAC,kBAAkB,EAAE,kBAAkB,CAAC,CAAC;IAC3E,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACxB,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;IACtB,CAAC;CACF"}

package/dist/audio_output.d.ts

@@ -0,0 +1,47 @@
+/**
+ * Audio output for Twilio Media Streams.
+ *
+ * Receives AudioFrame objects at 24kHz from the TTS pipeline,
+ * resamples to 8kHz via a streaming Sox sinc resampler, encodes
+ * to mu-law, and sends to Twilio via WebSocket.
+ *
+ * The resampler is allocated once and reused across all frames
+ * for the lifetime of this output (local Rust FFI, no LiveKit server).
+ */
+import { AudioOutput, AudioFrame } from '@ariaflow/livekit-plugin';
+/**
+ * Sends audio frames from the TTS pipeline to Twilio Media Streams.
+ *
+ * Flow:
+ * 1. TTS pipeline sends AudioFrame at 24kHz
+ * 2. Push through streaming resampler -> 8kHz AudioFrames
+ * 3. Encode each frame to mu-law
+ * 4. Base64 encode
+ * 5. Send as Twilio media event
+ */
+export declare class TwilioAudioOutput extends AudioOutput {
+    private sendQueue;
+    private sending;
+    private segmentSamplesSent;
+    private flushed;
+    private closed;
+    private resampler;
+    private sendCallback;
+    constructor();
+    /**
+     * Set the callback for sending messages to Twilio.
+     *
+     * @param callback - Function that sends JSON string to WebSocket
+     */
+    setSendCallback(callback: (message: string) => void): void;
+    captureFrame(frame: AudioFrame): Promise<void>;
+    private processSendQueue;
+    /**
+     * Encode and send a single 8kHz audio frame to Twilio.
+     */
+    private sendFrame;
+    flush(): void;
+    clearBuffer(): void;
+    close(): Promise<void>;
+}
+//# sourceMappingURL=audio_output.d.ts.map

package/dist/audio_output.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"audio_output.d.ts","sourceRoot":"","sources":["../src/audio_output.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AACH,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AAOnE;;;;;;;;;GASG;AACH,qBAAa,iBAAkB,SAAQ,WAAW;IAChD,OAAO,CAAC,SAAS,CAAoB;IACrC,OAAO,CAAC,OAAO,CAAkB;IACjC,OAAO,CAAC,kBAAkB,CAAa;IACvC,OAAO,CAAC,OAAO,CAAkB;IACjC,OAAO,CAAC,MAAM,CAAkB;IAChC,OAAO,CAAC,SAAS,CAAwD;IAGzE,OAAO,CAAC,YAAY,CAAuC;;IAM3D;;;;OAIG;IACH,eAAe,CAAC,QAAQ,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,GAAG,IAAI;IAIpD,YAAY,CAAC,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAcpD,OAAO,CAAC,gBAAgB;IAyCxB;;OAEG;IACH,OAAO,CAAC,SAAS;IAoBjB,KAAK,IAAI,IAAI;IA0Bb,WAAW,IAAI,IAAI;IA6Bb,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAI7B"}

package/dist/audio_output.js

@@ -0,0 +1,162 @@
+/**
+ * Audio output for Twilio Media Streams.
+ *
+ * Receives AudioFrame objects at 24kHz from the TTS pipeline,
+ * resamples to 8kHz via a streaming Sox sinc resampler, encodes
+ * to mu-law, and sends to Twilio via WebSocket.
+ *
+ * The resampler is allocated once and reused across all frames
+ * for the lifetime of this output (local Rust FFI, no LiveKit server).
+ */
+import { AudioOutput } from '@ariaflow/livekit-plugin';
+import { createResampler } from '@ariaflow/livekit-plugin/utils/resample';
+import { mulawEncodeArray } from '@ariaflow/livekit-plugin/codec/g711';
+const TTS_SAMPLE_RATE = 24000;
+const TARGET_SAMPLE_RATE = 8000;
+/**
+ * Sends audio frames from the TTS pipeline to Twilio Media Streams.
+ *
+ * Flow:
+ * 1. TTS pipeline sends AudioFrame at 24kHz
+ * 2. Push through streaming resampler -> 8kHz AudioFrames
+ * 3. Encode each frame to mu-law
+ * 4. Base64 encode
+ * 5. Send as Twilio media event
+ */
+export class TwilioAudioOutput extends AudioOutput {
+    sendQueue = [];
+    sending = false;
+    segmentSamplesSent = 0;
+    flushed = false;
+    closed = false;
+    resampler = createResampler(TTS_SAMPLE_RATE, TARGET_SAMPLE_RATE);
+    // Callback to send messages to Twilio
+    sendCallback = () => { };
+    constructor() {
+        super(TARGET_SAMPLE_RATE);
+    }
+    /**
+     * Set the callback for sending messages to Twilio.
+     *
+     * @param callback - Function that sends JSON string to WebSocket
+     */
+    setSendCallback(callback) {
+        this.sendCallback = callback;
+    }
+    async captureFrame(frame) {
+        await super.captureFrame(frame);
+        if (this.closed)
+            return;
+        // Push 24kHz frame through the streaming resampler -> 8kHz frames
+        for (const outFrame of this.resampler.push(frame)) {
+            this.sendQueue.push(outFrame);
+        }
+        if (!this.sending) {
+            this.processSendQueue();
+        }
+    }
+    processSendQueue() {
+        if (this.sending || this.closed)
+            return;
+        this.sending = true;
+        while (this.sendQueue.length > 0) {
+            const frame = this.sendQueue.shift();
+            if (this.segmentSamplesSent === 0) {
+                this.onPlaybackStarted(Date.now());
+            }
+            try {
+                this.sendFrame(frame);
+            }
+            catch (error) {
+                console.error('[TwilioAudioOutput] Error sending frame:', {
+                    error: error instanceof Error ? error.message : String(error),
+                    samplesPerChannel: frame.samplesPerChannel,
+                    timestamp: new Date().toISOString(),
+                });
+                break;
+            }
+            this.segmentSamplesSent += frame.samplesPerChannel;
+        }
+        this.sending = false;
+        if (this.flushed && this.sendQueue.length === 0) {
+            const sampleRate = this.sampleRate || TARGET_SAMPLE_RATE;
+            const playbackDuration = this.segmentSamplesSent / sampleRate;
+            this.onPlaybackFinished({
+                playbackPosition: playbackDuration,
+                interrupted: false,
+            });
+            this.segmentSamplesSent = 0;
+            this.flushed = false;
+        }
+    }
+    /**
+     * Encode and send a single 8kHz audio frame to Twilio.
+     */
+    sendFrame(frame) {
+        // Encode to mu-law
+        const mulawData = mulawEncodeArray(new Int16Array(frame.data));
+        // Base64 encode
+        const base64 = Buffer.from(mulawData).toString('base64');
+        // Create Twilio media event
+        const message = JSON.stringify({
+            event: 'media',
+            streamSid: '',
+            sequenceNumber: `${Date.now()}`,
+            media: {
+                payload: base64,
+            },
+        });
+        this.sendCallback(message);
+    }
+    flush() {
+        super.flush();
+        // Drain any buffered samples from the resampler
+        for (const outFrame of this.resampler.flush()) {
+            this.sendQueue.push(outFrame);
+        }
+        this.flushed = true;
+        if (this.sendQueue.length === 0) {
+            const sampleRate = this.sampleRate || TARGET_SAMPLE_RATE;
+            const playbackDuration = this.segmentSamplesSent / sampleRate;
+            this.onPlaybackFinished({
+                playbackPosition: playbackDuration,
+                interrupted: false,
+            });
+            this.segmentSamplesSent = 0;
+            this.flushed = false;
+        }
+        else if (!this.sending) {
+            this.processSendQueue();
+        }
+    }
+    clearBuffer() {
+        const sampleRate = this.sampleRate || TARGET_SAMPLE_RATE;
+        const playbackDuration = this.segmentSamplesSent / sampleRate;
+        this.sendQueue = [];
+        this.onPlaybackFinished({
+            playbackPosition: playbackDuration,
+            interrupted: true,
+        });
+        this.segmentSamplesSent = 0;
+        this.flushed = false;
+        // Send clear message to Twilio
+        try {
+            this.sendCallback(JSON.stringify({
+                event: 'clear',
+                streamSid: '',
+                sequenceNumber: `${Date.now()}`,
+            }));
+        }
+        catch (error) {
+            console.error('[TwilioAudioOutput] Error sending clear message:', {
+                error: error instanceof Error ? error.message : String(error),
+                timestamp: new Date().toISOString(),
+            });
+        }
+    }
+    async close() {
+        this.closed = true;
+        this.sendQueue = [];
+    }
+}
+//# sourceMappingURL=audio_output.js.map

package/dist/audio_output.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"audio_output.js","sourceRoot":"","sources":["../src/audio_output.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AACH,OAAO,EAAE,WAAW,EAAc,MAAM,0BAA0B,CAAC;AACnE,OAAO,EAAE,eAAe,EAAE,MAAM,yCAAyC,CAAC;AAC1E,OAAO,EAAE,gBAAgB,EAAE,MAAM,qCAAqC,CAAC;AAEvE,MAAM,eAAe,GAAG,KAAK,CAAC;AAC9B,MAAM,kBAAkB,GAAG,IAAI,CAAC;AAEhC;;;;;;;;;GASG;AACH,MAAM,OAAO,iBAAkB,SAAQ,WAAW;IACxC,SAAS,GAAiB,EAAE,CAAC;IAC7B,OAAO,GAAY,KAAK,CAAC;IACzB,kBAAkB,GAAW,CAAC,CAAC;IAC/B,OAAO,GAAY,KAAK,CAAC;IACzB,MAAM,GAAY,KAAK,CAAC;IACxB,SAAS,GAAG,eAAe,CAAC,eAAe,EAAE,kBAAkB,CAAC,CAAC;IAEzE,sCAAsC;IAC9B,YAAY,GAA8B,GAAG,EAAE,GAAE,CAAC,CAAC;IAE3D;QACE,KAAK,CAAC,kBAAkB,CAAC,CAAC;IAC5B,CAAC;IAED;;;;OAIG;IACH,eAAe,CAAC,QAAmC;QACjD,IAAI,CAAC,YAAY,GAAG,QAAQ,CAAC;IAC/B,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,KAAiB;QAClC,MAAM,KAAK,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;QAChC,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO;QAExB,kEAAkE;QAClE,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YAClD,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAChC,CAAC;QAED,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAClB,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAC1B,CAAC;IACH,CAAC;IAEO,gBAAgB;QACtB,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO;QACxC,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QAEpB,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACjC,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,EAAG,CAAC;YAEtC,IAAI,IAAI,CAAC,kBAAkB,KAAK,CAAC,EAAE,CAAC;gBAClC,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;YACrC,CAAC;YAED,IAAI,CAAC;gBACH,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;YACxB,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,CAAC,KAAK,CAAC,0CAA0C,EAAE;oBACxD,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;oBAC7D,iBAAiB,EAAE,KAAK,CAAC,iBAAiB;oBAC1C,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;iBACpC,CAAC,CAAC;gBACH,MAAM;YACR,CAAC;YAED,IAAI,CAAC,kBAAkB,IAAI,KAAK,CAAC,iBAAiB,CAAC;QACrD,CAAC;QAED,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;QAErB,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAChD,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,kBAAkB,CAAC;YACzD,MAAM,gBAAgB,GAAG,IAAI,CAAC,kBAAkB,GAAG,UAAU,CAAC;YAE9D,IAAI,CAAC,kBAAkB,CAAC;gBACtB,gBAAgB,EAAE,gBAAgB;gBAClC,WAAW,EAAE,KAAK;aACnB,CAAC,CAAC;YAEH,IAAI,CAAC,kBAAkB,GAAG,CAAC,CAAC;YAC5B,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;QACvB,CAAC;IACH,CAAC;IAED;;OAEG;IACK,SAAS,CAAC,KAAiB;QACjC,mBAAmB;QACnB,MAAM,SAAS,GAAG,gBAAgB,CAAC,IAAI,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;QAE/D,gBAAgB;QAChB,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAEzD,4BAA4B;QAC5B,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC;YAC7B,KAAK,EAAE,OAAO;YACd,SAAS,EAAE,EAAE;YACb,cAAc,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,EAAE;YAC/B,KAAK,EAAE;gBACL,OAAO,EAAE,MAAM;aAChB;SACF,CAAC,CAAC;QAEH,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;IAC7B,CAAC;IAED,KAAK;QACH,KAAK,CAAC,KAAK,EAAE,CAAC;QAEd,gDAAgD;QAChD,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,EAAE,CAAC;YAC9C,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAChC,CAAC;QAED,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QAEpB,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAChC,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,kBAAkB,CAAC;YACzD,MAAM,gBAAgB,GAAG,IAAI,CAAC,kBAAkB,GAAG,UAAU,CAAC;YAE9D,IAAI,CAAC,kBAAkB,CAAC;gBACtB,gBAAgB,EAAE,gBAAgB;gBAClC,WAAW,EAAE,KAAK;aACnB,CAAC,CAAC;YAEH,IAAI,CAAC,kBAAkB,GAAG,CAAC,CAAC;YAC5B,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;QACvB,CAAC;aAAM,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YACzB,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAC1B,CAAC;IACH,CAAC;IAED,WAAW;QACT,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,kBAAkB,CAAC;QACzD,MAAM,gBAAgB,GAAG,IAAI,CAAC,kBAAkB,GAAG,UAAU,CAAC;QAE9D,IAAI,CAAC,SAAS,GAAG,EAAE,CAAC;QAEpB,IAAI,CAAC,kBAAkB,CAAC;YACtB,gBAAgB,EAAE,gBAAgB;YAClC,WAAW,EAAE,IAAI;SAClB,CAAC,CAAC;QAEH,IAAI,CAAC,kBAAkB,GAAG,CAAC,CAAC;QAC5B,IAAI,CAAC,OAAO,GAAG,KAAK,CAAC;QAErB,+BAA+B;QAC/B,IAAI,CAAC;YACH,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC;gBAC/B,KAAK,EAAE,OAAO;gBACd,SAAS,EAAE,EAAE;gBACb,cAAc,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,EAAE;aAChC,CAAC,CAAC,CAAC;QACN,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,kDAAkD,EAAE;gBAChE,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;gBAC7D,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;aACpC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,IAAI,CAAC,SAAS,GAAG,EAAE,CAAC;IACtB,CAAC;CACF"}

package/dist/codec/g711.d.ts

@@ -0,0 +1,36 @@
+/**
+ * G.711 μ-law codec implementation for Twilio Media Streams.
+ *
+ * Twilio uses G.711 μ-law (mu-law) encoding at 8kHz.
+ * This codec provides bidirectional conversion between PCM Int16
+ * and μ-law encoded Uint8.
+ */
+/**
+ * Encode a PCM Int16 sample to μ-law Uint8.
+ *
+ * @param sample - 16-bit PCM sample (-32768 to 32767)
+ * @returns μ-law encoded byte (0-255)
+ */
+export declare function mulawEncode(sample: number): number;
+/**
+ * Decode a μ-law Uint8 to PCM Int16.
+ *
+ * @param mulaw - μ-law encoded byte (0-255)
+ * @returns 16-bit PCM sample
+ */
+export declare function mulawDecode(mulaw: number): number;
+/**
+ * Encode an array of PCM samples to μ-law.
+ *
+ * @param pcm - Int16Array of PCM samples
+ * @returns Uint8Array of μ-law encoded bytes
+ */
+export declare function mulawEncodeArray(pcm: Int16Array): Uint8Array;
+/**
+ * Decode an array of μ-law bytes to PCM.
+ *
+ * @param mulaw - Uint8Array of μ-law encoded bytes
+ * @returns Int16Array of PCM samples
+ */
+export declare function mulawDecodeArray(mulaw: Uint8Array): Int16Array;
+//# sourceMappingURL=g711.d.ts.map

package/dist/codec/g711.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"g711.d.ts","sourceRoot":"","sources":["../../src/codec/g711.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAmEH;;;;;GAKG;AACH,wBAAgB,WAAW,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CA2BlD;AAED;;;;;GAKG;AACH,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAGjD;AAED;;;;;GAKG;AACH,wBAAgB,gBAAgB,CAAC,GAAG,EAAE,UAAU,GAAG,UAAU,CAM5D;AAED;;;;;GAKG;AACH,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,UAAU,GAAG,UAAU,CAM9D"}