@runanywhere/core 0.16.0

package/src/Public/Extensions/RunAnywhere+VoiceAgent.ts

@@ -0,0 +1,225 @@
+/**
+ * RunAnywhere+VoiceAgent.ts
+ *
+ * Voice Agent extension for the full voice pipeline.
+ * Delegates to native VoiceAgentBridge.
+ *
+ * Reference: sdk/runanywhere-swift/Sources/RunAnywhere/Public/Extensions/VoiceAgent/RunAnywhere+VoiceAgent.swift
+ */
+
+import { requireNativeModule, isNativeModuleAvailable } from '../../native';
+import { SDKLogger } from '../../Foundation/Logging/Logger/SDKLogger';
+import type {
+  VoiceAgentConfig,
+  VoiceAgentComponentStates,
+  VoiceTurnResult,
+} from '../../types/VoiceAgentTypes';
+
+const logger = new SDKLogger('RunAnywhere.VoiceAgent');
+
+/**
+ * Get voice agent component states
+ * @returns Component load states for STT, LLM, TTS
+ */
+export async function getVoiceAgentComponentStates(): Promise<VoiceAgentComponentStates> {
+  if (!isNativeModuleAvailable()) {
+    throw new Error('Native module not available');
+  }
+
+  const native = requireNativeModule();
+
+  try {
+    const resultJson = await native.getVoiceAgentComponentStates();
+    return JSON.parse(resultJson) as VoiceAgentComponentStates;
+  } catch (error) {
+    const msg = error instanceof Error ? error.message : String(error);
+    logger.error(`Failed to get component states: ${msg}`);
+    throw error;
+  }
+}
+
+/**
+ * Check if all voice components are ready
+ */
+export async function areAllVoiceComponentsReady(): Promise<boolean> {
+  const states = await getVoiceAgentComponentStates();
+  return states.isFullyReady;
+}
+
+/**
+ * Initialize voice agent with configuration
+ * @param config Voice agent configuration
+ * @returns true if initialized successfully
+ */
+export async function initializeVoiceAgent(
+  config: VoiceAgentConfig
+): Promise<boolean> {
+  if (!isNativeModuleAvailable()) {
+    throw new Error('Native module not available');
+  }
+
+  const native = requireNativeModule();
+
+  try {
+    logger.info('Initializing voice agent...');
+    const result = await native.initializeVoiceAgent(JSON.stringify(config));
+    if (result) {
+      logger.info('Voice agent initialized successfully');
+    }
+    return result;
+  } catch (error) {
+    const msg = error instanceof Error ? error.message : String(error);
+    logger.error(`Failed to initialize voice agent: ${msg}`);
+    throw error;
+  }
+}
+
+/**
+ * Initialize voice agent using already-loaded models
+ * Uses the current STT, LLM, and TTS models
+ * @returns true if initialized successfully
+ */
+export async function initializeVoiceAgentWithLoadedModels(): Promise<boolean> {
+  if (!isNativeModuleAvailable()) {
+    throw new Error('Native module not available');
+  }
+
+  const native = requireNativeModule();
+
+  try {
+    logger.info('Initializing voice agent with loaded models...');
+    const result = await native.initializeVoiceAgentWithLoadedModels();
+    if (result) {
+      logger.info('Voice agent initialized with loaded models');
+    }
+    return result;
+  } catch (error) {
+    const msg = error instanceof Error ? error.message : String(error);
+    logger.error(`Failed to initialize voice agent: ${msg}`);
+    throw error;
+  }
+}
+
+/**
+ * Check if voice agent is ready
+ */
+export async function isVoiceAgentReady(): Promise<boolean> {
+  if (!isNativeModuleAvailable()) {
+    return false;
+  }
+
+  const native = requireNativeModule();
+  return native.isVoiceAgentReady();
+}
+
+/**
+ * Process a complete voice turn: audio -> transcription -> response -> speech
+ * @param audioData Audio data as ArrayBuffer or base64 string
+ * @returns Voice turn result
+ */
+export async function processVoiceTurn(
+  audioData: ArrayBuffer | string
+): Promise<VoiceTurnResult> {
+  if (!isNativeModuleAvailable()) {
+    throw new Error('Native module not available');
+  }
+
+  const native = requireNativeModule();
+
+  try {
+    // Convert to base64 if ArrayBuffer
+    let base64Audio: string;
+    if (audioData instanceof ArrayBuffer) {
+      const bytes = new Uint8Array(audioData);
+      base64Audio = btoa(String.fromCharCode(...bytes));
+    } else {
+      base64Audio = audioData;
+    }
+
+    const resultJson = await native.processVoiceTurn(base64Audio);
+    const result = JSON.parse(resultJson);
+
+    return {
+      speechDetected: result.speechDetected === true || result.speechDetected === 'true',
+      transcription: result.transcription || '',
+      response: result.response || '',
+      synthesizedAudio: result.synthesizedAudio || undefined,
+      sampleRate: result.sampleRate || 16000,
+    };
+  } catch (error) {
+    const msg = error instanceof Error ? error.message : String(error);
+    logger.error(`Voice turn failed: ${msg}`);
+    throw error;
+  }
+}
+
+/**
+ * Transcribe audio using voice agent (voice agent must be initialized)
+ * @param audioData Audio data as ArrayBuffer or base64 string
+ * @returns Transcription text
+ */
+export async function voiceAgentTranscribe(
+  audioData: ArrayBuffer | string
+): Promise<string> {
+  if (!isNativeModuleAvailable()) {
+    throw new Error('Native module not available');
+  }
+
+  const native = requireNativeModule();
+
+  let base64Audio: string;
+  if (audioData instanceof ArrayBuffer) {
+    const bytes = new Uint8Array(audioData);
+    base64Audio = btoa(String.fromCharCode(...bytes));
+  } else {
+    base64Audio = audioData;
+  }
+
+  return native.voiceAgentTranscribe(base64Audio);
+}
+
+/**
+ * Generate response using voice agent LLM
+ * @param prompt Input text
+ * @returns Generated response text
+ */
+export async function voiceAgentGenerateResponse(
+  prompt: string
+): Promise<string> {
+  if (!isNativeModuleAvailable()) {
+    throw new Error('Native module not available');
+  }
+
+  const native = requireNativeModule();
+  return native.voiceAgentGenerateResponse(prompt);
+}
+
+/**
+ * Synthesize speech using voice agent TTS
+ * @param text Text to synthesize
+ * @returns Base64-encoded audio data
+ */
+export async function voiceAgentSynthesizeSpeech(
+  text: string
+): Promise<string> {
+  if (!isNativeModuleAvailable()) {
+    throw new Error('Native module not available');
+  }
+
+  const native = requireNativeModule();
+  return native.voiceAgentSynthesizeSpeech(text);
+}
+
+/**
+ * Cleanup voice agent resources
+ */
+export async function cleanupVoiceAgent(): Promise<void> {
+  if (!isNativeModuleAvailable()) {
+    return;
+  }
+
+  const native = requireNativeModule();
+  logger.info('Cleaning up voice agent...');
+  await native.cleanupVoiceAgent();
+  logger.info('Voice agent cleaned up');
+}

package/src/Public/Extensions/RunAnywhere+VoiceSession.ts

@@ -0,0 +1,155 @@
+/**
+ * RunAnywhere+VoiceSession.ts
+ *
+ * High-level voice session API for simplified voice assistant integration.
+ * Handles audio capture, VAD, and processing internally.
+ *
+ * Reference: sdk/runanywhere-swift/Sources/RunAnywhere/Public/Extensions/VoiceAgent/RunAnywhere+VoiceSession.swift
+ *
+ * Usage:
+ * ```typescript
+ * // Start a voice session with async iterator
+ * const session = await startVoiceSession();
+ *
+ * for await (const event of session.events()) {
+ *   switch (event.type) {
+ *     case 'listening':
+ *       updateAudioMeter(event.audioLevel);
+ *       break;
+ *     case 'processing':
+ *       showProcessingIndicator();
+ *       break;
+ *     case 'turnCompleted':
+ *       updateUI(event.transcription, event.response);
+ *       break;
+ *   }
+ * }
+ *
+ * // Or use callbacks
+ * const session = await startVoiceSessionWithCallback({}, (event) => {
+ *   // Handle event
+ * });
+ *
+ * // Stop the session
+ * session.stop();
+ * ```
+ */
+
+import { SDKLogger } from '../../Foundation/Logging/Logger/SDKLogger';
+import {
+  VoiceSessionHandle,
+  DEFAULT_VOICE_SESSION_CONFIG,
+  type VoiceSessionConfig,
+  type VoiceSessionEvent,
+  type VoiceSessionEventCallback,
+} from '../../Features/VoiceSession';
+
+const logger = new SDKLogger('RunAnywhere.VoiceSession');
+
+// Re-export types for convenience
+export type {
+  VoiceSessionConfig,
+  VoiceSessionEvent,
+  VoiceSessionEventCallback
+};
+export { DEFAULT_VOICE_SESSION_CONFIG };
+
+/**
+ * Start a voice session with async event iteration
+ *
+ * This is the simplest way to integrate voice assistant.
+ * The session handles audio capture, VAD, and processing internally.
+ *
+ * Example:
+ * ```typescript
+ * const session = await startVoiceSession();
+ *
+ * // Consume events using async iteration
+ * for await (const event of session.events()) {
+ *   switch (event.type) {
+ *     case 'listening':
+ *       audioMeter = event.audioLevel ?? 0;
+ *       break;
+ *     case 'processing':
+ *       status = 'Processing...';
+ *       break;
+ *     case 'turnCompleted':
+ *       userText = event.transcription ?? '';
+ *       assistantText = event.response ?? '';
+ *       break;
+ *     case 'stopped':
+ *       // Session ended
+ *       break;
+ *   }
+ * }
+ * ```
+ *
+ * @param config Session configuration (optional)
+ * @returns Session handle with events iterator
+ */
+export async function startVoiceSession(
+  config: VoiceSessionConfig = {}
+): Promise<VoiceSessionHandle> {
+  logger.info('Starting voice session...');
+
+  const session = new VoiceSessionHandle(config);
+  await session.start();
+
+  logger.info('Voice session started');
+  return session;
+}
+
+/**
+ * Start a voice session with callback-based event handling
+ *
+ * Alternative API using callbacks instead of async iterator.
+ *
+ * Example:
+ * ```typescript
+ * const session = await startVoiceSessionWithCallback({}, (event) => {
+ *   switch (event.type) {
+ *     case 'listening':
+ *       setAudioLevel(event.audioLevel ?? 0);
+ *       break;
+ *     case 'turnCompleted':
+ *       setUserText(event.transcription ?? '');
+ *       setAssistantText(event.response ?? '');
+ *       break;
+ *   }
+ * });
+ *
+ * // Later...
+ * session.stop();
+ * ```
+ *
+ * @param config Session configuration
+ * @param onEvent Callback for each event
+ * @returns Session handle for control
+ */
+export async function startVoiceSessionWithCallback(
+  config: VoiceSessionConfig = {},
+  onEvent: VoiceSessionEventCallback
+): Promise<VoiceSessionHandle> {
+  logger.info('Starting voice session with callback...');
+
+  const session = new VoiceSessionHandle(config);
+  session.setEventCallback(onEvent);
+  await session.start();
+
+  logger.info('Voice session with callback started');
+  return session;
+}
+
+/**
+ * Create a voice session handle without starting it
+ *
+ * Useful when you want to configure the session before starting.
+ *
+ * @param config Session configuration
+ * @returns Session handle (not started)
+ */
+export function createVoiceSession(
+  config: VoiceSessionConfig = {}
+): VoiceSessionHandle {
+  return new VoiceSessionHandle(config);
+}

package/src/Public/Extensions/index.ts

@@ -0,0 +1,126 @@
+/**
+ * RunAnywhere Extensions
+ *
+ * Re-exports all extension modules for convenient importing.
+ */
+
+// Text Generation (LLM)
+export {
+  loadModel,
+  isModelLoaded,
+  unloadModel,
+  chat,
+  generate,
+  generateStream,
+  cancelGeneration,
+} from './RunAnywhere+TextGeneration';
+
+// Speech-to-Text
+export {
+  loadSTTModel,
+  isSTTModelLoaded,
+  unloadSTTModel,
+  transcribe,
+  transcribeSimple,
+  transcribeBuffer,
+  transcribeStream,
+  transcribeFile,
+  startStreamingSTT,
+  stopStreamingSTT,
+  isStreamingSTT,
+} from './RunAnywhere+STT';
+
+// Text-to-Speech
+export {
+  loadTTSModel,
+  loadTTSVoice,
+  unloadTTSVoice,
+  isTTSModelLoaded,
+  isTTSVoiceLoaded,
+  unloadTTSModel,
+  synthesize,
+  synthesizeStream,
+  speak,
+  isSpeaking,
+  stopSpeaking,
+  availableTTSVoices,
+  getTTSVoices,
+  getTTSVoiceInfo,
+  stopSynthesis,
+  cancelTTS,
+  cleanupTTS,
+} from './RunAnywhere+TTS';
+
+// Voice Activity Detection
+export {
+  initializeVAD,
+  isVADReady,
+  loadVADModel,
+  isVADModelLoaded,
+  unloadVADModel,
+  detectSpeech,
+  processVAD,
+  startVAD,
+  stopVAD,
+  resetVAD,
+  setVADSpeechActivityCallback,
+  setVADAudioBufferCallback,
+  cleanupVAD,
+  getVADState,
+} from './RunAnywhere+VAD';
+
+// Voice Agent
+export {
+  initializeVoiceAgent,
+  initializeVoiceAgentWithLoadedModels,
+  isVoiceAgentReady,
+  getVoiceAgentComponentStates,
+  areAllVoiceComponentsReady,
+  processVoiceTurn,
+  voiceAgentTranscribe,
+  voiceAgentGenerateResponse,
+  voiceAgentSynthesizeSpeech,
+  cleanupVoiceAgent,
+} from './RunAnywhere+VoiceAgent';
+
+// Voice Session
+export {
+  startVoiceSession,
+  startVoiceSessionWithCallback,
+  createVoiceSession,
+  DEFAULT_VOICE_SESSION_CONFIG,
+} from './RunAnywhere+VoiceSession';
+export type {
+  VoiceSessionConfig,
+  VoiceSessionEvent,
+  VoiceSessionEventCallback
+} from './RunAnywhere+VoiceSession';
+
+// Structured Output
+export {
+  generateStructured,
+  generateStructuredStream,
+  generate as generateStructuredType,
+  extractEntities,
+  classify,
+} from './RunAnywhere+StructuredOutput';
+export type {
+  StreamToken,
+  StructuredOutputStreamResult
+} from './RunAnywhere+StructuredOutput';
+
+// Logging
+export { setLogLevel } from './RunAnywhere+Logging';
+
+// Storage
+export { getStorageInfo, clearCache } from './RunAnywhere+Storage';
+
+// Models
+export {
+  getAvailableModels,
+  getModelInfo,
+  isModelDownloaded,
+  downloadModel,
+  cancelDownload,
+  deleteModel,
+} from './RunAnywhere+Models';