@astropods/messaging 0.0.1 → 0.0.3

package/dist/messaging-client.d.ts

@@ -45,6 +45,9 @@ export interface AgentResponse {
     threadMetadata?: ThreadMetadata;
     error?: ErrorResponse;
     contextRequest?: ThreadHistoryRequest;
+    transcript?: Transcript;
+    audioConfig?: AudioStreamConfig;
+    audioChunk?: AudioChunk;
 }
 export interface StatusUpdate {
     status: 'THINKING' | 'SEARCHING' | 'GENERATING' | 'PROCESSING' | 'ANALYZING' | 'CUSTOM';
@@ -77,6 +80,11 @@ export interface ErrorResponse {
     details?: string;
     retryable?: boolean;
 }
+export interface Transcript {
+    text: string;
+    messageId?: string;
+    language?: string;
+}
 export interface ThreadHistoryRequest {
     conversationId: string;
     maxMessages?: number;
@@ -129,11 +137,68 @@ export interface AgentConfig {
     systemPrompt: string;
     tools: AgentToolConfig[];
 }
+/**
+ * Supported audio encoding formats. Matches the AudioEncoding protobuf enum.
+ *
+ * Common sources:
+ * - LINEAR16: Universal PCM baseline (any platform)
+ * - MULAW: Twilio / telephony (G.711 mu-law, 8kHz)
+ * - WEBM_OPUS: Browser MediaRecorder default
+ * - AAC: iOS native recording
+ */
+export type AudioEncoding = 'LINEAR16' | 'MULAW' | 'OPUS' | 'MP3' | 'WEBM_OPUS' | 'OGG_OPUS' | 'FLAC' | 'AAC';
+/**
+ * Configuration sent at the start of an audio segment to describe the format.
+ * Maps to the AudioStreamConfig protobuf message.
+ */
+export interface AudioStreamConfig {
+    encoding: AudioEncoding;
+    sampleRate: number;
+    channels: number;
+    language?: string;
+    conversationId: string;
+    source?: string;
+}
+/**
+ * A chunk of raw audio bytes. Maps to the AudioChunk protobuf message.
+ *
+ * Chunks arrive sequentially during a segment. When done=true, the segment
+ * is complete and the agent should run STT on the accumulated audio.
+ */
+export interface AudioChunk {
+    data: Buffer | Uint8Array;
+    sequence?: number;
+    done?: boolean;
+}
+/**
+ * Maps an AudioEncoding to the filetype string expected by Mastra's voice.listen().
+ *
+ * Usage:
+ *   const filetype = audioEncodingToFiletype(config.encoding);
+ *   const transcript = await agent.voice.listen(audioStream, { filetype });
+ */
+export declare function audioEncodingToFiletype(encoding: AudioEncoding): string;
 export interface ConversationRequest {
     message?: Message;
     feedback?: any;
     agentConfig?: AgentConfig;
     agentResponse?: AgentResponse;
+    audioConfig?: AudioStreamConfig;
+    audio?: AudioChunk;
+}
+export interface ReconnectOptions {
+    /** Maximum number of reconnect attempts. Default: Infinity */
+    maxRetries?: number;
+    /** Initial delay before first retry in ms. Default: 500 */
+    initialDelayMs?: number;
+    /** Maximum delay between retries in ms. Default: 30_000 */
+    maxDelayMs?: number;
+    /** Apply full jitter to backoff delay. Default: true */
+    jitter?: boolean;
+    /** Maximum number of writes to queue during reconnect. Default: 1000 */
+    maxBufferSize?: number;
+    /** gRPC status codes that trigger a reconnect attempt. Default: UNAVAILABLE, DEADLINE_EXCEEDED, INTERNAL, RESOURCE_EXHAUSTED */
+    retryableStatusCodes?: number[];
 }
 /**
  * MessagingClient provides a TypeScript interface to the Astro Messaging gRPC service
@@ -149,9 +214,14 @@ export declare class MessagingClient extends EventEmitter {
      */
     connect(): Promise<void>;
     /**
-     * Create a bidirectional conversation stream
+     * Connect with automatic retry on failure (exponential backoff).
+     * Emits 'reconnecting' before each retry and 'reconnected' on success after failures.
      */
-    createConversationStream(): ConversationStream;
+    connectWithRetry(options?: ReconnectOptions): Promise<void>;
+    /**
+     * Create a bidirectional conversation stream with optional reconnect support
+     */
+    createConversationStream(options?: ReconnectOptions): ConversationStream;
     /**
      * Process a single message (server-side streaming)
      */
@@ -176,11 +246,31 @@ export declare class MessagingClient extends EventEmitter {
     close(): void;
 }
 /**
- * ConversationStream wraps a bidirectional gRPC stream
+ * ConversationStream wraps a bidirectional gRPC stream with automatic reconnection.
+ *
+ * Events:
+ *   - 'response'     — AgentResponse received from server
+ *   - 'reconnecting' — { attempt, reason, delayMs } — before each retry delay
+ *   - 'reconnected'  — { attempt } — after a successful stream recreation
+ *   - 'error'        — non-retryable error OR max retries exceeded
+ *   - 'end'          — only on intentional close(), not on unexpected stream drop
  */
 export declare class ConversationStream extends EventEmitter {
+    private streamFactory;
     private stream;
-    constructor(stream: any);
+    private writeBuffer;
+    private reconnecting;
+    private closed;
+    private retryCount;
+    private readonly opts;
+    constructor(streamFactory: () => any, options?: ReconnectOptions);
+    private attachHandlers;
+    private isRetryable;
+    private calculateDelay;
+    private scheduleReconnect;
+    private doReconnect;
+    private flushBuffer;
+    private write;
     /**
      * Send a message through the stream
      */
@@ -206,7 +296,62 @@ export declare class ConversationStream extends EventEmitter {
      */
     sendStatusUpdate(conversationId: string, status: StatusUpdate): void;
     /**
-     * End the stream
+     * Send a transcript of the user's audio input back to the platform.
+     *
+     * After the agent runs STT on the audio, it calls this to send the transcribed
+     * text back to the platform (web adapter). The platform uses it to replace the
+     * "[audio]" placeholder message with the actual spoken text in the chat UI.
+     *
+     * @param conversationId - The conversation this transcript belongs to
+     * @param text - The transcribed text from STT
+     * @param messageId - Optional: the original "[audio]" message ID to update
+     * @param language - Optional: BCP-47 language detected by STT (e.g. "en-US")
+     */
+    sendTranscript(conversationId: string, text: string, messageId?: string, language?: string): void;
+    /**
+     * Send an audio stream config through the bidi stream.
+     * Must be called before sendAudioChunk() so the receiver knows the encoding.
+     */
+    sendAudioConfig(config: AudioStreamConfig): void;
+    /**
+     * Send a raw audio chunk through the bidi stream.
+     * The chunk's sequence number should increase monotonically.
+     */
+    sendAudioChunk(chunk: AudioChunk): void;
+    /**
+     * Signal end of the current audio segment by sending an empty chunk with done=true.
+     * The receiver should process all accumulated audio (e.g. run STT).
+     * After this, more audio can follow — either new config or more chunks.
+     */
+    endAudio(): void;
+    /**
+     * Converts incoming audioChunk events into a Web Streams API ReadableStream.
+     *
+     * This is the primary integration point with Mastra's voice system. The agent
+     * listens for the 'audioConfig' event to know the format, then calls this
+     * method to get a stream it can pass directly to voice.listen():
+     *
+     * ```typescript
+     * conversation.on('audioConfig', async (config) => {
+     *   const audioStream = conversation.audioAsReadable();
+     *   const filetype = audioEncodingToFiletype(config.encoding);
+     *   const transcript = await agent.voice.listen(audioStream, { filetype });
+     *   // ... process transcript
+     * });
+     * ```
+     *
+     * The ReadableStream:
+     * - Yields Uint8Array chunks as audioChunk events arrive
+     * - Closes when an AudioChunk with done=true arrives (end of segment)
+     * - Closes when the ConversationStream emits 'end' (intentional close)
+     * - Errors when the ConversationStream emits 'error'
+     * - Properly cleans up all event listeners on close, error, or cancel
+     *
+     * @returns A ReadableStream<Uint8Array> suitable for Mastra voice.listen()
+     */
+    audioAsReadable(): ReadableStream<Uint8Array>;
+    /**
+     * End the stream intentionally. Emits 'end' and prevents any further reconnects.
      */
     end(): void;
 }

package/dist/messaging-client.js

@@ -34,10 +34,43 @@ var __importStar = (this && this.__importStar) || (function () {
 })();
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Helpers = exports.MessageStream = exports.ConversationStream = exports.MessagingClient = void 0;
+exports.audioEncodingToFiletype = audioEncodingToFiletype;
 const grpc = __importStar(require("@grpc/grpc-js"));
 const protoLoader = __importStar(require("@grpc/proto-loader"));
 const path_1 = require("path");
 const events_1 = require("events");
+/**
+ * Maps an AudioEncoding to the filetype string expected by Mastra's voice.listen().
+ *
+ * Usage:
+ *   const filetype = audioEncodingToFiletype(config.encoding);
+ *   const transcript = await agent.voice.listen(audioStream, { filetype });
+ */
+function audioEncodingToFiletype(encoding) {
+    const map = {
+        LINEAR16: 'wav',
+        MULAW: 'wav',
+        OPUS: 'opus',
+        MP3: 'mp3',
+        WEBM_OPUS: 'webm',
+        OGG_OPUS: 'ogg',
+        FLAC: 'flac',
+        AAC: 'm4a',
+    };
+    return map[encoding] ?? 'wav';
+}
+// gRPC status codes: DEADLINE_EXCEEDED=4, INTERNAL=13, UNAVAILABLE=14, RESOURCE_EXHAUSTED=8
+const DEFAULT_RETRYABLE_STATUS_CODES = [4, 8, 13, 14];
+function resolveReconnectOptions(options) {
+    return {
+        maxRetries: options.maxRetries ?? Infinity,
+        initialDelayMs: options.initialDelayMs ?? 500,
+        maxDelayMs: options.maxDelayMs ?? 30000,
+        jitter: options.jitter ?? true,
+        maxBufferSize: options.maxBufferSize ?? 1000,
+        retryableStatusCodes: options.retryableStatusCodes ?? DEFAULT_RETRYABLE_STATUS_CODES,
+    };
+}
 /**
  * MessagingClient provides a TypeScript interface to the Astro Messaging gRPC service
  */
@@ -45,6 +78,7 @@ class MessagingClient extends events_1.EventEmitter {
     constructor(serverAddress) {
         super();
         this.serverAddress = serverAddress;
+        this.conversationStream = null;
         this.isConnected = false;
     }
     /**
@@ -67,14 +101,42 @@ class MessagingClient extends events_1.EventEmitter {
         this.emit('connected');
     }
     /**
-     * Create a bidirectional conversation stream
+     * Connect with automatic retry on failure (exponential backoff).
+     * Emits 'reconnecting' before each retry and 'reconnected' on success after failures.
+     */
+    async connectWithRetry(options = {}) {
+        const opts = resolveReconnectOptions(options);
+        let retryCount = 0;
+        while (true) {
+            try {
+                await this.connect();
+                if (retryCount > 0) {
+                    this.emit('reconnected', { attempt: retryCount });
+                }
+                return;
+            }
+            catch (err) {
+                if (retryCount >= opts.maxRetries) {
+                    throw err;
+                }
+                const base = Math.min(opts.initialDelayMs * Math.pow(2, retryCount), opts.maxDelayMs);
+                const delayMs = opts.jitter ? base * (0.5 + Math.random() * 0.5) : base;
+                this.emit('reconnecting', { attempt: retryCount + 1, reason: err, delayMs });
+                await new Promise(resolve => setTimeout(resolve, delayMs));
+                retryCount++;
+            }
+        }
+    }
+    /**
+     * Create a bidirectional conversation stream with optional reconnect support
      */
-    createConversationStream() {
+    createConversationStream(options) {
         if (!this.isConnected) {
             throw new Error('Client not connected. Call connect() first.');
         }
-        this.conversationStream = this.client.ProcessConversation();
-        return new ConversationStream(this.conversationStream);
+        const factory = () => this.client.ProcessConversation();
+        this.conversationStream = new ConversationStream(factory, options);
+        return this.conversationStream;
     }
     /**
      * Process a single message (server-side streaming)
@@ -169,57 +231,133 @@ class MessagingClient extends events_1.EventEmitter {
 }
 exports.MessagingClient = MessagingClient;
 /**
- * ConversationStream wraps a bidirectional gRPC stream
+ * ConversationStream wraps a bidirectional gRPC stream with automatic reconnection.
+ *
+ * Events:
+ *   - 'response'     — AgentResponse received from server
+ *   - 'reconnecting' — { attempt, reason, delayMs } — before each retry delay
+ *   - 'reconnected'  — { attempt } — after a successful stream recreation
+ *   - 'error'        — non-retryable error OR max retries exceeded
+ *   - 'end'          — only on intentional close(), not on unexpected stream drop
  */
 class ConversationStream extends events_1.EventEmitter {
-    constructor(stream) {
+    constructor(streamFactory, options = {}) {
         super();
-        this.stream = stream;
-        this.stream.on('data', (response) => {
+        this.streamFactory = streamFactory;
+        this.writeBuffer = [];
+        this.reconnecting = false;
+        this.closed = false;
+        this.retryCount = 0;
+        this.opts = resolveReconnectOptions(options);
+        this.stream = this.streamFactory();
+        this.attachHandlers(this.stream);
+    }
+    attachHandlers(stream) {
+        stream.on('data', (response) => {
+            this.retryCount = 0;
+            // Emit audio-specific events if present.
+            // The server sends audio data through the bidi stream as AgentResponse
+            // messages with audioConfig or audioChunk payloads. We emit dedicated
+            // events for these so the agent can handle audio separately from text,
+            // while still emitting the generic 'response' event for observability.
+            if (response.audioConfig) {
+                this.emit('audioConfig', response.audioConfig);
+            }
+            else if (response.audioChunk) {
+                this.emit('audioChunk', response.audioChunk);
+            }
             this.emit('response', response);
         });
-        this.stream.on('end', () => {
-            this.emit('end');
+        stream.on('error', (error) => {
+            if (!this.closed && this.isRetryable(error)) {
+                this.scheduleReconnect(error);
+            }
+            else {
+                this.emit('error', error);
+            }
         });
-        this.stream.on('error', (error) => {
-            this.emit('error', error);
+        stream.on('end', () => {
+            if (!this.closed) {
+                this.scheduleReconnect(new Error('Stream ended unexpectedly'));
+            }
+            // If closed, 'end' was already emitted by end() — do nothing
         });
     }
+    isRetryable(error) {
+        return this.opts.retryableStatusCodes.includes(error.code);
+    }
+    calculateDelay() {
+        const base = Math.min(this.opts.initialDelayMs * Math.pow(2, this.retryCount), this.opts.maxDelayMs);
+        return this.opts.jitter ? base * (0.5 + Math.random() * 0.5) : base;
+    }
+    scheduleReconnect(reason) {
+        if (this.reconnecting || this.closed)
+            return;
+        if (this.retryCount >= this.opts.maxRetries) {
+            this.emit('error', new Error(`Max reconnection attempts (${this.opts.maxRetries}) exceeded`));
+            return;
+        }
+        const delayMs = this.calculateDelay();
+        this.reconnecting = true;
+        this.emit('reconnecting', { attempt: this.retryCount + 1, reason, delayMs });
+        setTimeout(() => this.doReconnect(), delayMs);
+    }
+    doReconnect() {
+        if (this.closed)
+            return;
+        this.retryCount++;
+        try {
+            this.stream = this.streamFactory();
+            this.attachHandlers(this.stream);
+            this.reconnecting = false;
+            this.emit('reconnected', { attempt: this.retryCount });
+            this.flushBuffer();
+        }
+        catch (err) {
+            this.reconnecting = false;
+            this.scheduleReconnect(err);
+        }
+    }
+    flushBuffer() {
+        const toFlush = this.writeBuffer.splice(0);
+        for (const request of toFlush) {
+            this.stream.write(request);
+        }
+    }
+    write(request) {
+        if (this.reconnecting || this.closed) {
+            if (this.writeBuffer.length >= this.opts.maxBufferSize) {
+                this.writeBuffer.shift(); // drop oldest
+            }
+            this.writeBuffer.push(request);
+        }
+        else {
+            this.stream.write(request);
+        }
+    }
     /**
      * Send a message through the stream
      */
     sendMessage(message) {
-        const request = {
-            message,
-        };
-        this.stream.write(request);
+        this.write({ message });
     }
     /**
      * Send platform feedback through the stream
      */
     sendFeedback(feedback) {
-        const request = {
-            feedback,
-        };
-        this.stream.write(request);
+        this.write({ feedback });
     }
     /**
      * Send agent configuration through the stream
      */
     sendAgentConfig(config) {
-        const request = {
-            agentConfig: config,
-        };
-        this.stream.write(request);
+        this.write({ agentConfig: config });
     }
     /**
      * Send a typed AgentResponse through the stream
      */
     sendAgentResponse(response) {
-        const request = {
-            agentResponse: response,
-        };
-        this.stream.write(request);
+        this.write({ agentResponse: response });
     }
     /**
      * Send a content chunk (START/DELTA/END) for a conversation
@@ -240,10 +378,138 @@ class ConversationStream extends events_1.EventEmitter {
         });
     }
     /**
-     * End the stream
+     * Send a transcript of the user's audio input back to the platform.
+     *
+     * After the agent runs STT on the audio, it calls this to send the transcribed
+     * text back to the platform (web adapter). The platform uses it to replace the
+     * "[audio]" placeholder message with the actual spoken text in the chat UI.
+     *
+     * @param conversationId - The conversation this transcript belongs to
+     * @param text - The transcribed text from STT
+     * @param messageId - Optional: the original "[audio]" message ID to update
+     * @param language - Optional: BCP-47 language detected by STT (e.g. "en-US")
+     */
+    sendTranscript(conversationId, text, messageId, language) {
+        this.sendAgentResponse({
+            conversationId,
+            transcript: { text, messageId, language },
+        });
+    }
+    // --- Audio support ---
+    //
+    // These methods handle sending audio data through the gRPC bidi stream.
+    // Two directions:
+    //   - Agent → Server (sendAudioConfig/sendAudioChunk/endAudio): used when the
+    //     agent needs to forward audio upstream (less common)
+    //   - Server → Agent (audioConfig/audioChunk events + audioAsReadable): the main
+    //     path where the server forwards client mic audio to the agent for STT
+    /**
+     * Send an audio stream config through the bidi stream.
+     * Must be called before sendAudioChunk() so the receiver knows the encoding.
+     */
+    sendAudioConfig(config) {
+        this.write({ audioConfig: config });
+    }
+    /**
+     * Send a raw audio chunk through the bidi stream.
+     * The chunk's sequence number should increase monotonically.
+     */
+    sendAudioChunk(chunk) {
+        this.write({ audio: chunk });
+    }
+    /**
+     * Signal end of the current audio segment by sending an empty chunk with done=true.
+     * The receiver should process all accumulated audio (e.g. run STT).
+     * After this, more audio can follow — either new config or more chunks.
+     */
+    endAudio() {
+        this.write({ audio: { data: Buffer.alloc(0), done: true } });
+    }
+    /**
+     * Converts incoming audioChunk events into a Web Streams API ReadableStream.
+     *
+     * This is the primary integration point with Mastra's voice system. The agent
+     * listens for the 'audioConfig' event to know the format, then calls this
+     * method to get a stream it can pass directly to voice.listen():
+     *
+     * ```typescript
+     * conversation.on('audioConfig', async (config) => {
+     *   const audioStream = conversation.audioAsReadable();
+     *   const filetype = audioEncodingToFiletype(config.encoding);
+     *   const transcript = await agent.voice.listen(audioStream, { filetype });
+     *   // ... process transcript
+     * });
+     * ```
+     *
+     * The ReadableStream:
+     * - Yields Uint8Array chunks as audioChunk events arrive
+     * - Closes when an AudioChunk with done=true arrives (end of segment)
+     * - Closes when the ConversationStream emits 'end' (intentional close)
+     * - Errors when the ConversationStream emits 'error'
+     * - Properly cleans up all event listeners on close, error, or cancel
+     *
+     * @returns A ReadableStream<Uint8Array> suitable for Mastra voice.listen()
+     */
+    audioAsReadable() {
+        // Centralized cleanup to prevent listener leaks. Called on:
+        // - done=true chunk (normal completion)
+        // - stream 'end' event (intentional close)
+        // - stream 'error' event
+        // - ReadableStream cancel() (consumer gave up, e.g. reader.cancel())
+        const cleanup = () => {
+            this.removeListener('audioChunk', onChunk);
+            this.removeListener('end', onEnd);
+            this.removeListener('error', onError);
+        };
+        const onChunk = (chunk) => {
+            if (chunk.done) {
+                cleanup();
+                try {
+                    controller.close();
+                }
+                catch { }
+            }
+            else {
+                controller.enqueue(new Uint8Array(chunk.data));
+            }
+        };
+        const onEnd = () => {
+            cleanup();
+            try {
+                controller.close();
+            }
+            catch { }
+        };
+        const onError = (err) => {
+            cleanup();
+            try {
+                controller.error(err);
+            }
+            catch { }
+        };
+        let controller;
+        return new ReadableStream({
+            start: (ctrl) => {
+                controller = ctrl;
+                this.on('audioChunk', onChunk);
+                this.once('end', onEnd);
+                this.once('error', onError);
+            },
+            cancel: () => {
+                // Consumer cancelled (e.g. reader.cancel()) — remove all listeners
+                // to prevent memory leaks
+                cleanup();
+            },
+        });
+    }
+    /**
+     * End the stream intentionally. Emits 'end' and prevents any further reconnects.
      */
     end() {
+        this.closed = true;
+        this.writeBuffer = [];
         this.stream.end();
+        this.emit('end');
     }
 }
 exports.ConversationStream = ConversationStream;

package/dist/proto/astro/messaging/v1/audio.proto

@@ -0,0 +1,71 @@
+// Audio streaming types for the messaging system.
+//
+// These types enable raw audio input from any frontend (browser, phone, mobile app)
+// to be streamed through the messaging server to an agent. The messaging system is
+// a pass-through — it does NOT perform speech-to-text, transcoding, or voice activity
+// detection. The agent handles STT via Mastra's voice provider abstraction.
+//
+// Data flow:
+//   Client (mic) → WebSocket → Server → gRPC (these types) → Agent → Mastra voice.listen()
+//
+// Two ways audio enters the system:
+//   1. ProcessAudioStream RPC: dedicated audio-only streaming (AudioStreamRequest)
+//   2. ProcessConversation RPC: audio mixed into the bidi stream (ConversationRequest.audio_config/audio)
+//
+// Both converge on the same types: AudioStreamConfig describes the format,
+// AudioChunk carries the bytes, and done=true signals end of an utterance.
+
+syntax = "proto3";
+
+package astro.messaging.v1;
+
+option go_package = "github.com/postman/astro/messaging/v1;messagingv1";
+
+// Audio encoding format — covers browser, telephony, and mobile sources.
+// The agent uses this to configure the STT provider (e.g. Whisper, Deepgram).
+enum AudioEncoding {
+  AUDIO_ENCODING_UNSPECIFIED = 0;
+  LINEAR16 = 1;          // PCM signed 16-bit little-endian — universal baseline, any platform
+  MULAW = 2;             // G.711 mu-law — Twilio and traditional telephony (8kHz)
+  OPUS = 3;              // Raw Opus frames — low-latency codec
+  MP3 = 4;               // MP3 — batch uploads, pre-recorded audio
+  WEBM_OPUS = 5;         // WebM container with Opus — browser MediaRecorder default
+  OGG_OPUS = 6;          // OGG container with Opus — Firefox MediaRecorder
+  FLAC = 7;              // FLAC lossless — high-quality uploads
+  AAC = 8;               // AAC — iOS native recording
+}
+
+// Sent once at the start of an audio segment to tell the agent what format
+// the subsequent AudioChunk bytes are in. Without this, the agent can't
+// decode the raw bytes.
+message AudioStreamConfig {
+  AudioEncoding encoding = 1;     // What codec the audio bytes use
+  int32 sample_rate = 2;          // Hz: 8000 (telephony), 16000 (speech), 48000 (browser)
+  int32 channels = 3;             // 1 = mono (speech default), 2 = stereo
+  string language = 4;            // BCP-47 hint for STT, e.g. "en-US" (optional)
+  string conversation_id = 5;     // Links this audio to an existing conversation
+
+  // Source metadata — helps the agent pick the right STT config.
+  // Examples: "browser", "twilio", "vonage", "mobile", "upload"
+  string source = 6;
+}
+
+// A chunk of raw audio bytes in the encoding specified by AudioStreamConfig.
+//
+// Chunks arrive in order during a segment. When done=true, the segment is
+// complete and the agent should run STT on all accumulated chunks.
+// The data field may be empty on the final done=true chunk.
+message AudioChunk {
+  bytes data = 1;                 // Raw audio bytes (pass-through, no transcoding)
+  int64 sequence = 2;             // Monotonic sequence number for ordering
+  bool done = 3;                  // true = end of segment, process accumulated audio
+}
+
+// Wrapper for the ProcessAudioStream RPC (dedicated audio streaming).
+// The first message MUST be config, all subsequent messages are audio chunks.
+message AudioStreamRequest {
+  oneof request {
+    AudioStreamConfig config = 1;    // First message: tells agent the audio format
+    AudioChunk audio = 2;            // Subsequent: raw audio bytes
+  }
+}

package/dist/proto/astro/messaging/v1/response.proto

@@ -4,6 +4,7 @@ package astro.messaging.v1;
 
 import "google/protobuf/timestamp.proto";
 import "astro/messaging/v1/message.proto";
+import "astro/messaging/v1/audio.proto";
 
 option go_package = "github.com/postman/astro/messaging/v1;messagingv1";
 
@@ -22,6 +23,9 @@ message AgentResponse {
     ThreadMetadata thread_metadata = 7;   // Thread title, creation
     ErrorResponse error = 8;              // Error during processing
     ThreadHistoryRequest context_request = 9;  // Request cached context (optional)
+    Transcript transcript = 10;           // Audio transcript (agent → platform)
+    AudioStreamConfig audio_config = 11;  // Audio session config (server → agent)
+    AudioChunk audio_chunk = 12;          // Audio data (server → agent)
   }
 }
 
@@ -144,6 +148,14 @@ message ThreadMetadata {
   bool create_new = 3;                    // Create new thread
 }
 
+// Transcript of user audio input (agent → platform after STT)
+// Used to update a placeholder message with the actual transcribed text
+message Transcript {
+  string text = 1;                        // Transcribed text
+  string message_id = 2;                  // User message ID to update (optional)
+  string language = 3;                    // Detected language BCP-47 (optional)
+}
+
 // Error response from agent
 message ErrorResponse {
   enum ErrorCode {

package/dist/proto/astro/messaging/v1/service.proto

@@ -5,6 +5,7 @@ package astro.messaging.v1;
 import "astro/messaging/v1/message.proto";
 import "astro/messaging/v1/response.proto";
 import "astro/messaging/v1/feedback.proto";
+import "astro/messaging/v1/audio.proto";
 import "astro/messaging/v1/config.proto";
 import "google/protobuf/timestamp.proto";
 
@@ -29,6 +30,11 @@ service AgentMessaging {
   rpc GetConversationMetadata(ConversationMetadataRequest)
       returns (ConversationMetadataResponse);
 
+  // Audio: client streams raw audio, server responds with text
+  // First message MUST be AudioStreamConfig, rest are AudioChunks
+  rpc ProcessAudioStream(stream AudioStreamRequest)
+      returns (stream AgentResponse);
+
   // Health check
   rpc HealthCheck(HealthCheckRequest)
       returns (HealthCheckResponse);
@@ -41,6 +47,8 @@ message ConversationRequest {
     PlatformFeedback feedback = 2;
     AgentConfig agent_config = 3;
     AgentResponse agent_response = 4;
+    AudioStreamConfig audio_config = 5;  // Start audio within conversation
+    AudioChunk audio = 6;                // Audio data within conversation
   }
 }
 

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@astropods/messaging",
   "license": "Apache-2.0",
-  "version": "0.0.1",
+  "version": "0.0.3",
   "description": "TypeScript SDK for Astro Messaging",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -9,8 +9,8 @@
     "dist"
   ],
   "scripts": {
-    "postinstall": "ln -sf ../../proto proto",
-    "build": "tsc && cp -r ../../proto dist/proto",
+    "postinstall": "rm -rf proto && ln -sf ../../proto proto",
+    "build": "tsc && rm -rf dist/proto && cp -r ../../proto dist/proto",
     "watch": "tsc --watch",
     "test": "bun test",
     "test:watch": "bun test --watch"