@realtimex/sdk 1.2.0 → 1.2.1

package/dist/index.d.mts

@@ -80,16 +80,53 @@ interface Task {
     runs: TaskRun[];
 }
 interface TTSOptions {
+    /** Voice ID (provider-specific) */
     voice?: string;
+    /** Model ID (provider-specific) */
     model?: string;
+    /** Speech speed (0.5-2.0) */
     speed?: number;
+    /** TTS provider ID */
     provider?: string;
+    /** Language code (e.g., 'en', 'es', 'fr') - for Supertonic */
+    language?: string;
+    /** Quality level (1-20) - for Supertonic num_inference_steps */
+    num_inference_steps?: number;
+}
+interface TTSProviderConfig {
+    /** Available voice/speaker IDs */
+    voices: string[];
+    /** Supported languages (for multilingual providers) */
+    languages?: string[];
+    /** Speed range */
+    speed?: {
+        min: number;
+        max: number;
+        default: number;
+    };
+    /** Quality range (for providers that support it) */
+    quality?: {
+        min: number;
+        max: number;
+        default: number;
+        description?: string;
+    };
 }
 interface TTSProvider {
+    /** Provider ID (e.g., 'elevenlabs', 'supertonic_local') */
     id: string;
+    /** Display name */
     name: string;
-    type: 'remote' | 'local';
-    voices: string[];
+    /** Provider type: 'server' (remote API) or 'client' (local) */
+    type: 'server' | 'client';
+    /** Whether provider is configured and ready */
+    configured: boolean;
+    /** Whether streaming is supported */
+    supportsStreaming: boolean;
+    /** Optional note about provider requirements */
+    note?: string;
+    /** Configuration options */
+    config?: TTSProviderConfig;
 }
 interface TTSProvidersResponse {
     success: boolean;
@@ -97,6 +134,24 @@ interface TTSProvidersResponse {
     default: string;
     error?: string;
 }
+interface TTSChunk {
+    /** Chunk index (0-based) */
+    index: number;
+    /** Total number of chunks */
+    total: number;
+    /** Decoded audio data (ArrayBuffer) - ready for playback */
+    audio: ArrayBuffer;
+    /** Audio MIME type */
+    mimeType: string;
+}
+interface TTSChunkEvent {
+    type: 'info' | 'chunk' | 'error' | 'done';
+    data: TTSChunk | {
+        totalChunks: number;
+    } | {
+        error: string;
+    };
+}
 
 /**
  * Activities Module - HTTP Proxy to RealtimeX Main App
@@ -680,19 +735,22 @@ declare class TTSModule {
      */
     speak(text: string, options?: TTSOptions): Promise<ArrayBuffer>;
     /**
-     * Generate speech from text (returns stream)
+     * Generate speech from text with streaming (yields decoded audio chunks)
+     * Uses SSE internally but returns decoded ArrayBuffer chunks for easy playback.
      *
      * @example
      * ```ts
-     * const stream = await sdk.tts.speakStream("Hello world");
-     * for await (const chunk of stream) {
-     *   // Play chunk...
+     * for await (const chunk of sdk.tts.speakStream("Hello world")) {
+     *   // chunk.audio is ArrayBuffer (already decoded!)
+     *   const blob = new Blob([chunk.audio], { type: chunk.mimeType });
+     *   const audio = new Audio(URL.createObjectURL(blob));
+     *   await audio.play();
      * }
      * ```
      */
-    speakStream(text: string, options?: TTSOptions): AsyncGenerator<Uint8Array>;
+    speakStream(text: string, options?: TTSOptions): AsyncGenerator<TTSChunk>;
     /**
-     * List available TTS providers
+     * List available TTS providers with configuration options
      */
     listProviders(): Promise<TTSProvider[]>;
 }
@@ -745,4 +803,4 @@ declare class RealtimeXSDK {
     getAppDataDir(): Promise<string>;
 }
 
-export { ActivitiesModule, type Activity, type Agent, ApiModule, type ChatMessage, type ChatOptions, type ChatResponse, type EmbedOptions, type EmbedResponse, LLMModule, LLMPermissionError, LLMProviderError, PermissionDeniedError, PermissionRequiredError, PortModule, type Provider, type ProvidersResponse, RealtimeXSDK, type SDKConfig, type StreamChunk, TTSModule, type TTSOptions, type TTSProvider, type TTSProvidersResponse, type Task, TaskModule, type TaskRun, type Thread, type TriggerAgentPayload, type TriggerAgentResponse, type VectorDeleteOptions, type VectorDeleteResponse, type VectorQueryOptions, type VectorQueryResponse, type VectorQueryResult, type VectorRecord, VectorStore, type VectorUpsertOptions, type VectorUpsertResponse, WebhookModule, type Workspace };
+export { ActivitiesModule, type Activity, type Agent, ApiModule, type ChatMessage, type ChatOptions, type ChatResponse, type EmbedOptions, type EmbedResponse, LLMModule, LLMPermissionError, LLMProviderError, PermissionDeniedError, PermissionRequiredError, PortModule, type Provider, type ProvidersResponse, RealtimeXSDK, type SDKConfig, type StreamChunk, type TTSChunk, type TTSChunkEvent, TTSModule, type TTSOptions, type TTSProvider, type TTSProviderConfig, type TTSProvidersResponse, type Task, TaskModule, type TaskRun, type Thread, type TriggerAgentPayload, type TriggerAgentResponse, type VectorDeleteOptions, type VectorDeleteResponse, type VectorQueryOptions, type VectorQueryResponse, type VectorQueryResult, type VectorRecord, VectorStore, type VectorUpsertOptions, type VectorUpsertResponse, WebhookModule, type Workspace };

package/dist/index.d.ts

@@ -80,16 +80,53 @@ interface Task {
     runs: TaskRun[];
 }
 interface TTSOptions {
+    /** Voice ID (provider-specific) */
     voice?: string;
+    /** Model ID (provider-specific) */
     model?: string;
+    /** Speech speed (0.5-2.0) */
     speed?: number;
+    /** TTS provider ID */
     provider?: string;
+    /** Language code (e.g., 'en', 'es', 'fr') - for Supertonic */
+    language?: string;
+    /** Quality level (1-20) - for Supertonic num_inference_steps */
+    num_inference_steps?: number;
+}
+interface TTSProviderConfig {
+    /** Available voice/speaker IDs */
+    voices: string[];
+    /** Supported languages (for multilingual providers) */
+    languages?: string[];
+    /** Speed range */
+    speed?: {
+        min: number;
+        max: number;
+        default: number;
+    };
+    /** Quality range (for providers that support it) */
+    quality?: {
+        min: number;
+        max: number;
+        default: number;
+        description?: string;
+    };
 }
 interface TTSProvider {
+    /** Provider ID (e.g., 'elevenlabs', 'supertonic_local') */
     id: string;
+    /** Display name */
     name: string;
-    type: 'remote' | 'local';
-    voices: string[];
+    /** Provider type: 'server' (remote API) or 'client' (local) */
+    type: 'server' | 'client';
+    /** Whether provider is configured and ready */
+    configured: boolean;
+    /** Whether streaming is supported */
+    supportsStreaming: boolean;
+    /** Optional note about provider requirements */
+    note?: string;
+    /** Configuration options */
+    config?: TTSProviderConfig;
 }
 interface TTSProvidersResponse {
     success: boolean;
@@ -97,6 +134,24 @@ interface TTSProvidersResponse {
     default: string;
     error?: string;
 }
+interface TTSChunk {
+    /** Chunk index (0-based) */
+    index: number;
+    /** Total number of chunks */
+    total: number;
+    /** Decoded audio data (ArrayBuffer) - ready for playback */
+    audio: ArrayBuffer;
+    /** Audio MIME type */
+    mimeType: string;
+}
+interface TTSChunkEvent {
+    type: 'info' | 'chunk' | 'error' | 'done';
+    data: TTSChunk | {
+        totalChunks: number;
+    } | {
+        error: string;
+    };
+}
 
 /**
  * Activities Module - HTTP Proxy to RealtimeX Main App
@@ -680,19 +735,22 @@ declare class TTSModule {
      */
     speak(text: string, options?: TTSOptions): Promise<ArrayBuffer>;
     /**
-     * Generate speech from text (returns stream)
+     * Generate speech from text with streaming (yields decoded audio chunks)
+     * Uses SSE internally but returns decoded ArrayBuffer chunks for easy playback.
      *
      * @example
      * ```ts
-     * const stream = await sdk.tts.speakStream("Hello world");
-     * for await (const chunk of stream) {
-     *   // Play chunk...
+     * for await (const chunk of sdk.tts.speakStream("Hello world")) {
+     *   // chunk.audio is ArrayBuffer (already decoded!)
+     *   const blob = new Blob([chunk.audio], { type: chunk.mimeType });
+     *   const audio = new Audio(URL.createObjectURL(blob));
+     *   await audio.play();
      * }
      * ```
      */
-    speakStream(text: string, options?: TTSOptions): AsyncGenerator<Uint8Array>;
+    speakStream(text: string, options?: TTSOptions): AsyncGenerator<TTSChunk>;
     /**
-     * List available TTS providers
+     * List available TTS providers with configuration options
      */
     listProviders(): Promise<TTSProvider[]>;
 }
@@ -745,4 +803,4 @@ declare class RealtimeXSDK {
     getAppDataDir(): Promise<string>;
 }
 
-export { ActivitiesModule, type Activity, type Agent, ApiModule, type ChatMessage, type ChatOptions, type ChatResponse, type EmbedOptions, type EmbedResponse, LLMModule, LLMPermissionError, LLMProviderError, PermissionDeniedError, PermissionRequiredError, PortModule, type Provider, type ProvidersResponse, RealtimeXSDK, type SDKConfig, type StreamChunk, TTSModule, type TTSOptions, type TTSProvider, type TTSProvidersResponse, type Task, TaskModule, type TaskRun, type Thread, type TriggerAgentPayload, type TriggerAgentResponse, type VectorDeleteOptions, type VectorDeleteResponse, type VectorQueryOptions, type VectorQueryResponse, type VectorQueryResult, type VectorRecord, VectorStore, type VectorUpsertOptions, type VectorUpsertResponse, WebhookModule, type Workspace };
+export { ActivitiesModule, type Activity, type Agent, ApiModule, type ChatMessage, type ChatOptions, type ChatResponse, type EmbedOptions, type EmbedResponse, LLMModule, LLMPermissionError, LLMProviderError, PermissionDeniedError, PermissionRequiredError, PortModule, type Provider, type ProvidersResponse, RealtimeXSDK, type SDKConfig, type StreamChunk, type TTSChunk, type TTSChunkEvent, TTSModule, type TTSOptions, type TTSProvider, type TTSProviderConfig, type TTSProvidersResponse, type Task, TaskModule, type TaskRun, type Thread, type TriggerAgentPayload, type TriggerAgentResponse, type VectorDeleteOptions, type VectorDeleteResponse, type VectorQueryOptions, type VectorQueryResponse, type VectorQueryResult, type VectorRecord, VectorStore, type VectorUpsertOptions, type VectorUpsertResponse, WebhookModule, type Workspace };

package/dist/index.js

@@ -1068,35 +1068,94 @@ var TTSModule = class {
     });
   }
   /**
-   * Generate speech from text (returns stream)
+   * Generate speech from text with streaming (yields decoded audio chunks)
+   * Uses SSE internally but returns decoded ArrayBuffer chunks for easy playback.
    * 
    * @example
    * ```ts
-   * const stream = await sdk.tts.speakStream("Hello world");
-   * for await (const chunk of stream) {
-   *   // Play chunk...
+   * for await (const chunk of sdk.tts.speakStream("Hello world")) {
+   *   // chunk.audio is ArrayBuffer (already decoded!)
+   *   const blob = new Blob([chunk.audio], { type: chunk.mimeType });
+   *   const audio = new Audio(URL.createObjectURL(blob));
+   *   await audio.play();
    * }
    * ```
    */
   async *speakStream(text, options = {}) {
-    const body = await this.request("POST", "/sdk/tts/stream", {
-      text,
-      ...options
-    }, true);
-    if (!body) throw new Error("No response body");
-    const reader = body.getReader();
+    const response = await fetch(`${this.baseUrl}/sdk/tts/stream`, {
+      method: "POST",
+      headers: this.headers,
+      body: JSON.stringify({ text, ...options })
+    });
+    if (!response.ok) {
+      const data = await response.json();
+      if (data.code === "PERMISSION_REQUIRED") {
+        const permission = data.permission || "tts.generate";
+        const granted = await this.requestPermission(permission);
+        if (granted) {
+          yield* this.speakStream(text, options);
+          return;
+        }
+        throw new PermissionDeniedError(permission);
+      }
+      throw new Error(data.error || `Streaming failed: ${response.status}`);
+    }
+    const reader = response.body?.getReader();
+    if (!reader) throw new Error("No response body");
+    const decoder = new TextDecoder();
+    let buffer = "";
+    let eventType = "";
     try {
       while (true) {
         const { done, value } = await reader.read();
         if (done) break;
-        yield value;
+        buffer += decoder.decode(value, { stream: true });
+        const lines = buffer.split("\n");
+        buffer = lines.pop() || "";
+        for (const line of lines) {
+          const trimmedLine = line.trim();
+          if (!trimmedLine) continue;
+          if (trimmedLine.startsWith("event:")) {
+            eventType = trimmedLine.slice(6).trim();
+          } else if (trimmedLine.startsWith("data:")) {
+            const eventData = trimmedLine.slice(5).trim();
+            if (eventType === "chunk" && eventData) {
+              try {
+                const parsed = JSON.parse(eventData);
+                const binaryString = atob(parsed.audio);
+                const bytes = new Uint8Array(binaryString.length);
+                for (let i = 0; i < binaryString.length; i++) {
+                  bytes[i] = binaryString.charCodeAt(i);
+                }
+                yield {
+                  index: parsed.index,
+                  total: parsed.total,
+                  audio: bytes.buffer,
+                  mimeType: parsed.mimeType
+                };
+              } catch (e) {
+                console.warn("[TTS SDK] Failed to parse chunk:", e);
+              }
+            } else if (eventType === "error" && eventData) {
+              try {
+                const err = JSON.parse(eventData);
+                throw new Error(err.error || "TTS streaming error");
+              } catch (e) {
+                if (e instanceof Error && e.message !== "TTS streaming error") {
+                  throw e;
+                }
+              }
+            }
+            eventType = "";
+          }
+        }
       }
     } finally {
       reader.releaseLock();
     }
   }
   /**
-   * List available TTS providers
+   * List available TTS providers with configuration options
    */
   async listProviders() {
     const data = await this.request("GET", "/sdk/tts/providers");

package/dist/index.mjs

@@ -1020,35 +1020,94 @@ var TTSModule = class {
     });
   }
   /**
-   * Generate speech from text (returns stream)
+   * Generate speech from text with streaming (yields decoded audio chunks)
+   * Uses SSE internally but returns decoded ArrayBuffer chunks for easy playback.
    * 
    * @example
    * ```ts
-   * const stream = await sdk.tts.speakStream("Hello world");
-   * for await (const chunk of stream) {
-   *   // Play chunk...
+   * for await (const chunk of sdk.tts.speakStream("Hello world")) {
+   *   // chunk.audio is ArrayBuffer (already decoded!)
+   *   const blob = new Blob([chunk.audio], { type: chunk.mimeType });
+   *   const audio = new Audio(URL.createObjectURL(blob));
+   *   await audio.play();
    * }
    * ```
    */
   async *speakStream(text, options = {}) {
-    const body = await this.request("POST", "/sdk/tts/stream", {
-      text,
-      ...options
-    }, true);
-    if (!body) throw new Error("No response body");
-    const reader = body.getReader();
+    const response = await fetch(`${this.baseUrl}/sdk/tts/stream`, {
+      method: "POST",
+      headers: this.headers,
+      body: JSON.stringify({ text, ...options })
+    });
+    if (!response.ok) {
+      const data = await response.json();
+      if (data.code === "PERMISSION_REQUIRED") {
+        const permission = data.permission || "tts.generate";
+        const granted = await this.requestPermission(permission);
+        if (granted) {
+          yield* this.speakStream(text, options);
+          return;
+        }
+        throw new PermissionDeniedError(permission);
+      }
+      throw new Error(data.error || `Streaming failed: ${response.status}`);
+    }
+    const reader = response.body?.getReader();
+    if (!reader) throw new Error("No response body");
+    const decoder = new TextDecoder();
+    let buffer = "";
+    let eventType = "";
     try {
       while (true) {
         const { done, value } = await reader.read();
         if (done) break;
-        yield value;
+        buffer += decoder.decode(value, { stream: true });
+        const lines = buffer.split("\n");
+        buffer = lines.pop() || "";
+        for (const line of lines) {
+          const trimmedLine = line.trim();
+          if (!trimmedLine) continue;
+          if (trimmedLine.startsWith("event:")) {
+            eventType = trimmedLine.slice(6).trim();
+          } else if (trimmedLine.startsWith("data:")) {
+            const eventData = trimmedLine.slice(5).trim();
+            if (eventType === "chunk" && eventData) {
+              try {
+                const parsed = JSON.parse(eventData);
+                const binaryString = atob(parsed.audio);
+                const bytes = new Uint8Array(binaryString.length);
+                for (let i = 0; i < binaryString.length; i++) {
+                  bytes[i] = binaryString.charCodeAt(i);
+                }
+                yield {
+                  index: parsed.index,
+                  total: parsed.total,
+                  audio: bytes.buffer,
+                  mimeType: parsed.mimeType
+                };
+              } catch (e) {
+                console.warn("[TTS SDK] Failed to parse chunk:", e);
+              }
+            } else if (eventType === "error" && eventData) {
+              try {
+                const err = JSON.parse(eventData);
+                throw new Error(err.error || "TTS streaming error");
+              } catch (e) {
+                if (e instanceof Error && e.message !== "TTS streaming error") {
+                  throw e;
+                }
+              }
+            }
+            eventType = "";
+          }
+        }
       }
     } finally {
       reader.releaseLock();
     }
   }
   /**
-   * List available TTS providers
+   * List available TTS providers with configuration options
    */
   async listProviders() {
     const data = await this.request("GET", "/sdk/tts/providers");

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@realtimex/sdk",
-    "version": "1.2.0",
+    "version": "1.2.1",
     "description": "SDK for building Local Apps that integrate with RealtimeX",
     "main": "dist/index.js",
     "module": "dist/index.mjs",