@realtimex/sdk 1.3.2 → 1.3.4

package/README.md

@@ -167,6 +167,19 @@ const response = await sdk.llm.chat(
 );
 console.log(response.response?.content);
 
+// Multimodal Chat (text + file/image blocks)
+const multimodal = await sdk.llm.chat([
+  {
+    role: 'user',
+    content: [
+      { type: 'text', text: 'Summarize the attached document' },
+      { type: 'input_file', file_url: 'https://example.com/report.pdf' },
+      { type: 'input_image', image_url: 'https://example.com/chart.png' }
+    ]
+  }
+]);
+console.log(multimodal.response?.content);
+
 // Streaming Chat
 for await (const chunk of sdk.llm.chatStream(messages, options)) {
   process.stdout.write(chunk.textResponse || '');

package/dist/index.d.mts

@@ -380,9 +380,36 @@ declare class PortModule {
  * - Vector storage (upsert, query, delete)
  */
 
+interface ChatTextBlock {
+    type: 'text' | 'input_text';
+    text: string;
+    [key: string]: unknown;
+}
+interface ChatImageUrlBlock {
+    type: 'image_url' | 'input_image';
+    image_url: string | {
+        url: string;
+        detail?: 'auto' | 'low' | 'high';
+        [key: string]: unknown;
+    };
+    [key: string]: unknown;
+}
+interface ChatFileBlock {
+    type: 'input_file' | 'file';
+    file_url?: string;
+    file_id?: string;
+    filename?: string;
+    [key: string]: unknown;
+}
+interface ChatCustomBlock {
+    type: string;
+    [key: string]: unknown;
+}
+type ChatContentBlock = ChatTextBlock | ChatImageUrlBlock | ChatFileBlock | ChatCustomBlock;
+type ChatMessageContent = string | ChatContentBlock[];
 interface ChatMessage {
     role: 'system' | 'user' | 'assistant';
-    content: string;
+    content: ChatMessageContent;
 }
 interface ChatOptions {
     model?: string;
@@ -955,6 +982,74 @@ declare class AgentModule {
     }>;
 }
 
+/**
+ * MCP Module - Interact with MCP servers via RealtimeX SDK
+ */
+
+interface MCPServer {
+    /** Unique server name (slug) */
+    name: string;
+    /** User-friendly display name */
+    display_name: string;
+    /** Server description */
+    description: string | null;
+    /** Server type: 'stdio', 'http', 'sse', or 'remote' */
+    server_type: string;
+    /** Whether the server is enabled */
+    enabled: boolean;
+    /** Provider: 'local' or 'remote' */
+    provider: 'local' | 'remote';
+    /** Tags / categories */
+    tags: string[];
+}
+interface MCPTool {
+    /** Tool name */
+    name: string;
+    /** Tool description */
+    description: string | null;
+    /** JSON Schema describing the tool's input parameters */
+    input_schema: Record<string, any>;
+}
+interface MCPToolResult {
+    /** Whether the execution was successful */
+    success: boolean;
+    /** Server that executed the tool */
+    server: string;
+    /** Tool that was executed */
+    tool: string;
+    /** Provider used */
+    provider: string;
+    /** Execution result data */
+    result: any;
+    /** Error message if failed */
+    error?: string;
+}
+declare class MCPModule extends ApiModule {
+    constructor(realtimexUrl: string, appId: string, appName?: string, apiKey?: string);
+    /**
+     * List configured MCP servers.
+     * @param provider - Filter by provider: 'local', 'remote', or 'all' (default: 'all')
+     * @returns Array of MCP server objects
+     */
+    getServers(provider?: 'local' | 'remote' | 'all'): Promise<MCPServer[]>;
+    /**
+     * List available tools for a specific MCP server.
+     * @param serverName - The server name (slug)
+     * @param provider - Provider: 'local' or 'remote' (default: 'local')
+     * @returns Array of tool objects with name, description, and input schema
+     */
+    getTools(serverName: string, provider?: 'local' | 'remote'): Promise<MCPTool[]>;
+    /**
+     * Execute a tool on an MCP server.
+     * @param serverName - The server name (slug)
+     * @param toolName - The tool name to execute
+     * @param args - Arguments to pass to the tool (matches tool's input_schema)
+     * @param provider - Provider: 'local' or 'remote' (default: 'local')
+     * @returns Tool execution result
+     */
+    executeTool(serverName: string, toolName: string, args?: Record<string, any>, provider?: 'local' | 'remote'): Promise<any>;
+}
+
 /**
  * RealtimeX Local App SDK
  *
@@ -972,6 +1067,7 @@ declare class RealtimeXSDK {
     tts: TTSModule;
     stt: STTModule;
     agent: AgentModule;
+    mcp: MCPModule;
     readonly appId: string;
     readonly appName: string | undefined;
     readonly apiKey: string | undefined;
@@ -1006,4 +1102,4 @@ declare class RealtimeXSDK {
     getAppDataDir(): Promise<string>;
 }
 
-export { ActivitiesModule, type Activity, type Agent, type AgentChatOptions, type AgentChatResponse, AgentModule, type AgentSession, type AgentSessionInfo, type AgentSessionOptions, ApiModule, type ChatMessage, type ChatOptions, type ChatResponse, type EmbedOptions, type EmbedResponse, LLMModule, LLMPermissionError, LLMProviderError, PermissionDeniedError, PermissionRequiredError, PortModule, type Provider, type ProvidersResponse, RealtimeXSDK, type SDKConfig, type STTListenOptions, type STTModel, type STTModelsResponse, STTModule, type STTProvider, type STTProvidersResponse, type STTResponse, type StreamChunk, type StreamChunkEvent, type TTSChunk, type TTSChunkEvent, TTSModule, type TTSOptions, type TTSProvider, type TTSProviderConfig, type TTSProvidersResponse, type Task, TaskModule, type TaskRun, type Thread, type TriggerAgentPayload, type TriggerAgentResponse, type VectorDeleteOptions, type VectorDeleteResponse, type VectorQueryOptions, type VectorQueryResponse, type VectorQueryResult, type VectorRecord, VectorStore, type VectorUpsertOptions, type VectorUpsertResponse, WebhookModule, type Workspace };
+export { ActivitiesModule, type Activity, type Agent, type AgentChatOptions, type AgentChatResponse, AgentModule, type AgentSession, type AgentSessionInfo, type AgentSessionOptions, ApiModule, type ChatContentBlock, type ChatCustomBlock, type ChatFileBlock, type ChatImageUrlBlock, type ChatMessage, type ChatMessageContent, type ChatOptions, type ChatResponse, type ChatTextBlock, type EmbedOptions, type EmbedResponse, LLMModule, LLMPermissionError, LLMProviderError, MCPModule, type MCPServer, type MCPTool, type MCPToolResult, PermissionDeniedError, PermissionRequiredError, PortModule, type Provider, type ProvidersResponse, RealtimeXSDK, type SDKConfig, type STTListenOptions, type STTModel, type STTModelsResponse, STTModule, type STTProvider, type STTProvidersResponse, type STTResponse, type StreamChunk, type StreamChunkEvent, type TTSChunk, type TTSChunkEvent, TTSModule, type TTSOptions, type TTSProvider, type TTSProviderConfig, type TTSProvidersResponse, type Task, TaskModule, type TaskRun, type Thread, type TriggerAgentPayload, type TriggerAgentResponse, type VectorDeleteOptions, type VectorDeleteResponse, type VectorQueryOptions, type VectorQueryResponse, type VectorQueryResult, type VectorRecord, VectorStore, type VectorUpsertOptions, type VectorUpsertResponse, WebhookModule, type Workspace };

package/dist/index.d.ts

@@ -380,9 +380,36 @@ declare class PortModule {
  * - Vector storage (upsert, query, delete)
  */
 
+interface ChatTextBlock {
+    type: 'text' | 'input_text';
+    text: string;
+    [key: string]: unknown;
+}
+interface ChatImageUrlBlock {
+    type: 'image_url' | 'input_image';
+    image_url: string | {
+        url: string;
+        detail?: 'auto' | 'low' | 'high';
+        [key: string]: unknown;
+    };
+    [key: string]: unknown;
+}
+interface ChatFileBlock {
+    type: 'input_file' | 'file';
+    file_url?: string;
+    file_id?: string;
+    filename?: string;
+    [key: string]: unknown;
+}
+interface ChatCustomBlock {
+    type: string;
+    [key: string]: unknown;
+}
+type ChatContentBlock = ChatTextBlock | ChatImageUrlBlock | ChatFileBlock | ChatCustomBlock;
+type ChatMessageContent = string | ChatContentBlock[];
 interface ChatMessage {
     role: 'system' | 'user' | 'assistant';
-    content: string;
+    content: ChatMessageContent;
 }
 interface ChatOptions {
     model?: string;
@@ -955,6 +982,74 @@ declare class AgentModule {
     }>;
 }
 
+/**
+ * MCP Module - Interact with MCP servers via RealtimeX SDK
+ */
+
+interface MCPServer {
+    /** Unique server name (slug) */
+    name: string;
+    /** User-friendly display name */
+    display_name: string;
+    /** Server description */
+    description: string | null;
+    /** Server type: 'stdio', 'http', 'sse', or 'remote' */
+    server_type: string;
+    /** Whether the server is enabled */
+    enabled: boolean;
+    /** Provider: 'local' or 'remote' */
+    provider: 'local' | 'remote';
+    /** Tags / categories */
+    tags: string[];
+}
+interface MCPTool {
+    /** Tool name */
+    name: string;
+    /** Tool description */
+    description: string | null;
+    /** JSON Schema describing the tool's input parameters */
+    input_schema: Record<string, any>;
+}
+interface MCPToolResult {
+    /** Whether the execution was successful */
+    success: boolean;
+    /** Server that executed the tool */
+    server: string;
+    /** Tool that was executed */
+    tool: string;
+    /** Provider used */
+    provider: string;
+    /** Execution result data */
+    result: any;
+    /** Error message if failed */
+    error?: string;
+}
+declare class MCPModule extends ApiModule {
+    constructor(realtimexUrl: string, appId: string, appName?: string, apiKey?: string);
+    /**
+     * List configured MCP servers.
+     * @param provider - Filter by provider: 'local', 'remote', or 'all' (default: 'all')
+     * @returns Array of MCP server objects
+     */
+    getServers(provider?: 'local' | 'remote' | 'all'): Promise<MCPServer[]>;
+    /**
+     * List available tools for a specific MCP server.
+     * @param serverName - The server name (slug)
+     * @param provider - Provider: 'local' or 'remote' (default: 'local')
+     * @returns Array of tool objects with name, description, and input schema
+     */
+    getTools(serverName: string, provider?: 'local' | 'remote'): Promise<MCPTool[]>;
+    /**
+     * Execute a tool on an MCP server.
+     * @param serverName - The server name (slug)
+     * @param toolName - The tool name to execute
+     * @param args - Arguments to pass to the tool (matches tool's input_schema)
+     * @param provider - Provider: 'local' or 'remote' (default: 'local')
+     * @returns Tool execution result
+     */
+    executeTool(serverName: string, toolName: string, args?: Record<string, any>, provider?: 'local' | 'remote'): Promise<any>;
+}
+
 /**
  * RealtimeX Local App SDK
  *
@@ -972,6 +1067,7 @@ declare class RealtimeXSDK {
     tts: TTSModule;
     stt: STTModule;
     agent: AgentModule;
+    mcp: MCPModule;
     readonly appId: string;
     readonly appName: string | undefined;
     readonly apiKey: string | undefined;
@@ -1006,4 +1102,4 @@ declare class RealtimeXSDK {
     getAppDataDir(): Promise<string>;
 }
 
-export { ActivitiesModule, type Activity, type Agent, type AgentChatOptions, type AgentChatResponse, AgentModule, type AgentSession, type AgentSessionInfo, type AgentSessionOptions, ApiModule, type ChatMessage, type ChatOptions, type ChatResponse, type EmbedOptions, type EmbedResponse, LLMModule, LLMPermissionError, LLMProviderError, PermissionDeniedError, PermissionRequiredError, PortModule, type Provider, type ProvidersResponse, RealtimeXSDK, type SDKConfig, type STTListenOptions, type STTModel, type STTModelsResponse, STTModule, type STTProvider, type STTProvidersResponse, type STTResponse, type StreamChunk, type StreamChunkEvent, type TTSChunk, type TTSChunkEvent, TTSModule, type TTSOptions, type TTSProvider, type TTSProviderConfig, type TTSProvidersResponse, type Task, TaskModule, type TaskRun, type Thread, type TriggerAgentPayload, type TriggerAgentResponse, type VectorDeleteOptions, type VectorDeleteResponse, type VectorQueryOptions, type VectorQueryResponse, type VectorQueryResult, type VectorRecord, VectorStore, type VectorUpsertOptions, type VectorUpsertResponse, WebhookModule, type Workspace };
+export { ActivitiesModule, type Activity, type Agent, type AgentChatOptions, type AgentChatResponse, AgentModule, type AgentSession, type AgentSessionInfo, type AgentSessionOptions, ApiModule, type ChatContentBlock, type ChatCustomBlock, type ChatFileBlock, type ChatImageUrlBlock, type ChatMessage, type ChatMessageContent, type ChatOptions, type ChatResponse, type ChatTextBlock, type EmbedOptions, type EmbedResponse, LLMModule, LLMPermissionError, LLMProviderError, MCPModule, type MCPServer, type MCPTool, type MCPToolResult, PermissionDeniedError, PermissionRequiredError, PortModule, type Provider, type ProvidersResponse, RealtimeXSDK, type SDKConfig, type STTListenOptions, type STTModel, type STTModelsResponse, STTModule, type STTProvider, type STTProvidersResponse, type STTResponse, type StreamChunk, type StreamChunkEvent, type TTSChunk, type TTSChunkEvent, TTSModule, type TTSOptions, type TTSProvider, type TTSProviderConfig, type TTSProvidersResponse, type Task, TaskModule, type TaskRun, type Thread, type TriggerAgentPayload, type TriggerAgentResponse, type VectorDeleteOptions, type VectorDeleteResponse, type VectorQueryOptions, type VectorQueryResponse, type VectorQueryResult, type VectorRecord, VectorStore, type VectorUpsertOptions, type VectorUpsertResponse, WebhookModule, type Workspace };

package/dist/index.js

@@ -36,6 +36,7 @@ __export(index_exports, {
   LLMModule: () => LLMModule,
   LLMPermissionError: () => LLMPermissionError,
   LLMProviderError: () => LLMProviderError,
+  MCPModule: () => MCPModule,
   PermissionDeniedError: () => PermissionDeniedError,
   PermissionRequiredError: () => PermissionRequiredError,
   PortModule: () => PortModule,
@@ -1358,6 +1359,54 @@ var AgentModule = class {
   }
 };
 
+// src/modules/mcp.ts
+var MCPModule = class extends ApiModule {
+  constructor(realtimexUrl, appId, appName, apiKey) {
+    super(realtimexUrl, appId, appName, apiKey);
+  }
+  /**
+   * List configured MCP servers.
+   * @param provider - Filter by provider: 'local', 'remote', or 'all' (default: 'all')
+   * @returns Array of MCP server objects
+   */
+  async getServers(provider = "all") {
+    const params = provider !== "all" ? `?provider=${provider}` : "";
+    const data = await this.apiCall("GET", `/sdk/mcp/servers${params}`);
+    return data.servers;
+  }
+  /**
+   * List available tools for a specific MCP server.
+   * @param serverName - The server name (slug)
+   * @param provider - Provider: 'local' or 'remote' (default: 'local')
+   * @returns Array of tool objects with name, description, and input schema
+   */
+  async getTools(serverName, provider = "local") {
+    const data = await this.apiCall(
+      "GET",
+      `/sdk/mcp/servers/${encodeURIComponent(serverName)}/tools?provider=${provider}`
+    );
+    return data.tools;
+  }
+  /**
+   * Execute a tool on an MCP server.
+   * @param serverName - The server name (slug)
+   * @param toolName - The tool name to execute
+   * @param args - Arguments to pass to the tool (matches tool's input_schema)
+   * @param provider - Provider: 'local' or 'remote' (default: 'local')
+   * @returns Tool execution result
+   */
+  async executeTool(serverName, toolName, args = {}, provider = "local") {
+    const data = await this.apiCall(
+      "POST",
+      `/sdk/mcp/servers/${encodeURIComponent(serverName)}/tools/${encodeURIComponent(toolName)}/execute?provider=${provider}`,
+      {
+        body: JSON.stringify({ arguments: args })
+      }
+    );
+    return data.result;
+  }
+};
+
 // src/modules/http.ts
 var HttpClient = class {
   constructor(baseUrl, appId, apiKey) {
@@ -1436,6 +1485,7 @@ var _RealtimeXSDK = class _RealtimeXSDK {
     this.tts = new TTSModule(this.realtimexUrl, this.appId, this.appName, this.apiKey);
     this.stt = new STTModule(this.realtimexUrl, this.appId, this.appName, this.apiKey);
     this.agent = new AgentModule(this.httpClient);
+    this.mcp = new MCPModule(this.realtimexUrl, this.appId, this.appName, this.apiKey);
     if (this.permissions.length > 0 && this.appId && !this.apiKey) {
       this.register().catch((err) => {
         console.error("[RealtimeX SDK] Auto-registration failed:", err.message);
@@ -1543,6 +1593,7 @@ var RealtimeXSDK = _RealtimeXSDK;
   LLMModule,
   LLMPermissionError,
   LLMProviderError,
+  MCPModule,
   PermissionDeniedError,
   PermissionRequiredError,
   PortModule,

package/dist/index.mjs

@@ -1308,6 +1308,54 @@ var AgentModule = class {
   }
 };
 
+// src/modules/mcp.ts
+var MCPModule = class extends ApiModule {
+  constructor(realtimexUrl, appId, appName, apiKey) {
+    super(realtimexUrl, appId, appName, apiKey);
+  }
+  /**
+   * List configured MCP servers.
+   * @param provider - Filter by provider: 'local', 'remote', or 'all' (default: 'all')
+   * @returns Array of MCP server objects
+   */
+  async getServers(provider = "all") {
+    const params = provider !== "all" ? `?provider=${provider}` : "";
+    const data = await this.apiCall("GET", `/sdk/mcp/servers${params}`);
+    return data.servers;
+  }
+  /**
+   * List available tools for a specific MCP server.
+   * @param serverName - The server name (slug)
+   * @param provider - Provider: 'local' or 'remote' (default: 'local')
+   * @returns Array of tool objects with name, description, and input schema
+   */
+  async getTools(serverName, provider = "local") {
+    const data = await this.apiCall(
+      "GET",
+      `/sdk/mcp/servers/${encodeURIComponent(serverName)}/tools?provider=${provider}`
+    );
+    return data.tools;
+  }
+  /**
+   * Execute a tool on an MCP server.
+   * @param serverName - The server name (slug)
+   * @param toolName - The tool name to execute
+   * @param args - Arguments to pass to the tool (matches tool's input_schema)
+   * @param provider - Provider: 'local' or 'remote' (default: 'local')
+   * @returns Tool execution result
+   */
+  async executeTool(serverName, toolName, args = {}, provider = "local") {
+    const data = await this.apiCall(
+      "POST",
+      `/sdk/mcp/servers/${encodeURIComponent(serverName)}/tools/${encodeURIComponent(toolName)}/execute?provider=${provider}`,
+      {
+        body: JSON.stringify({ arguments: args })
+      }
+    );
+    return data.result;
+  }
+};
+
 // src/modules/http.ts
 var HttpClient = class {
   constructor(baseUrl, appId, apiKey) {
@@ -1386,6 +1434,7 @@ var _RealtimeXSDK = class _RealtimeXSDK {
     this.tts = new TTSModule(this.realtimexUrl, this.appId, this.appName, this.apiKey);
     this.stt = new STTModule(this.realtimexUrl, this.appId, this.appName, this.apiKey);
     this.agent = new AgentModule(this.httpClient);
+    this.mcp = new MCPModule(this.realtimexUrl, this.appId, this.appName, this.apiKey);
     if (this.permissions.length > 0 && this.appId && !this.apiKey) {
       this.register().catch((err) => {
         console.error("[RealtimeX SDK] Auto-registration failed:", err.message);
@@ -1492,6 +1541,7 @@ export {
   LLMModule,
   LLMPermissionError,
   LLMProviderError,
+  MCPModule,
   PermissionDeniedError,
   PermissionRequiredError,
   PortModule,

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@realtimex/sdk",
-    "version": "1.3.2",
+    "version": "1.3.4",
     "description": "SDK for building Local Apps that integrate with RealtimeX",
     "main": "dist/index.js",
     "module": "dist/index.mjs",