@runanywhere/core 0.16.0

package/src/Public/Extensions/RunAnywhere+TextGeneration.ts

@@ -0,0 +1,320 @@
+/**
+ * RunAnywhere+TextGeneration.ts
+ *
+ * Text generation (LLM) extension for RunAnywhere SDK.
+ * Uses backend-agnostic rac_llm_component_* C++ APIs via the core native module.
+ * The actual backend (LlamaCPP, etc.) must be registered by installing
+ * and importing the appropriate backend package (e.g., @runanywhere/llamacpp).
+ *
+ * Matches iOS: RunAnywhere+TextGeneration.swift
+ */
+
+import { EventBus } from '../Events';
+import {
+  requireNativeModule,
+  isNativeModuleAvailable,
+} from '../../native';
+import type { GenerationOptions, GenerationResult } from '../../types';
+import { ExecutionTarget, HardwareAcceleration } from '../../types';
+import { SDKLogger } from '../../Foundation/Logging/Logger/SDKLogger';
+import type {
+  LLMStreamingResult,
+  LLMGenerationResult,
+} from '../../types/LLMTypes';
+
+const logger = new SDKLogger('RunAnywhere.TextGeneration');
+
+// ============================================================================
+// Text Generation (LLM) Extension - Backend Agnostic
+// ============================================================================
+
+/**
+ * Load an LLM model by ID or path
+ *
+ * Matches iOS: `RunAnywhere.loadModel(_:)`
+ * @throws Error if no LLM backend is registered
+ */
+export async function loadModel(
+  modelPathOrId: string,
+  config?: Record<string, unknown>
+): Promise<boolean> {
+  if (!isNativeModuleAvailable()) {
+    logger.warning('Native module not available for loadModel');
+    return false;
+  }
+  const native = requireNativeModule();
+  return native.loadTextModel(
+    modelPathOrId,
+    config ? JSON.stringify(config) : undefined
+  );
+}
+
+/**
+ * Check if an LLM model is loaded
+ * Matches iOS: `RunAnywhere.isModelLoaded`
+ */
+export async function isModelLoaded(): Promise<boolean> {
+  if (!isNativeModuleAvailable()) {
+    return false;
+  }
+  const native = requireNativeModule();
+  return native.isTextModelLoaded();
+}
+
+/**
+ * Unload the currently loaded LLM model
+ * Matches iOS: `RunAnywhere.unloadModel()`
+ */
+export async function unloadModel(): Promise<boolean> {
+  if (!isNativeModuleAvailable()) {
+    return false;
+  }
+  const native = requireNativeModule();
+  return native.unloadTextModel();
+}
+
+/**
+ * Simple chat - returns just the text response
+ * Matches Swift SDK: RunAnywhere.chat(_:)
+ */
+export async function chat(prompt: string): Promise<string> {
+  const result = await generate(prompt);
+  return result.text;
+}
+
+/**
+ * Text generation with options and full metrics
+ * Matches Swift SDK: RunAnywhere.generate(_:options:)
+ */
+export async function generate(
+  prompt: string,
+  options?: GenerationOptions
+): Promise<GenerationResult> {
+  if (!isNativeModuleAvailable()) {
+    throw new Error('Native module not available');
+  }
+  const native = requireNativeModule();
+
+  const optionsJson = JSON.stringify({
+    max_tokens: options?.maxTokens ?? 1000,
+    temperature: options?.temperature ?? 0.7,
+    system_prompt: options?.systemPrompt ?? null,
+  });
+
+  const resultJson = await native.generate(prompt, optionsJson);
+
+  try {
+    const result = JSON.parse(resultJson);
+    return {
+      text: result.text ?? '',
+      thinkingContent: result.thinkingContent,
+      tokensUsed: result.tokensUsed ?? 0,
+      modelUsed: result.modelUsed ?? 'unknown',
+      latencyMs: result.latencyMs ?? 0,
+      executionTarget: result.executionTarget ?? 0,
+      savedAmount: result.savedAmount ?? 0,
+      framework: result.framework,
+      hardwareUsed: result.hardwareUsed ?? 0,
+      memoryUsed: result.memoryUsed ?? 0,
+      performanceMetrics: {
+        timeToFirstTokenMs: result.performanceMetrics?.timeToFirstTokenMs,
+        tokensPerSecond: result.performanceMetrics?.tokensPerSecond,
+        inferenceTimeMs:
+          result.performanceMetrics?.inferenceTimeMs ?? result.latencyMs ?? 0,
+      },
+      thinkingTokens: result.thinkingTokens,
+      responseTokens: result.responseTokens ?? result.tokensUsed ?? 0,
+    };
+  } catch {
+    if (resultJson.includes('error')) {
+      throw new Error(resultJson);
+    }
+    return {
+      text: resultJson,
+      tokensUsed: 0,
+      modelUsed: 'unknown',
+      latencyMs: 0,
+      executionTarget: ExecutionTarget.OnDevice,
+      savedAmount: 0,
+      hardwareUsed: HardwareAcceleration.CPU,
+      memoryUsed: 0,
+      performanceMetrics: {
+        inferenceTimeMs: 0,
+      },
+      responseTokens: 0,
+    };
+  }
+}
+
+/**
+ * Streaming text generation with async iterator
+ *
+ * Returns a LLMStreamingResult containing:
+ * - stream: AsyncIterable<string> for consuming tokens
+ * - result: Promise<LLMGenerationResult> for final metrics
+ * - cancel: Function to cancel generation
+ *
+ * Matches Swift SDK: RunAnywhere.generateStream(_:options:)
+ *
+ * Example usage:
+ * ```typescript
+ * const streaming = await generateStream(prompt);
+ *
+ * // Display tokens in real-time
+ * for await (const token of streaming.stream) {
+ *   console.log(token);
+ * }
+ *
+ * // Get complete analytics after streaming finishes
+ * const metrics = await streaming.result;
+ * console.log(`Speed: ${metrics.tokensPerSecond} tok/s`);
+ * ```
+ */
+export async function generateStream(
+  prompt: string,
+  options?: GenerationOptions
+): Promise<LLMStreamingResult> {
+  if (!isNativeModuleAvailable()) {
+    throw new Error('Native module not available');
+  }
+
+  const native = requireNativeModule();
+  const startTime = Date.now();
+  let firstTokenTime: number | null = null;
+  let cancelled = false;
+  let fullText = '';
+  let tokenCount = 0;
+  let resolveResult: ((result: LLMGenerationResult) => void) | null = null;
+  let rejectResult: ((error: Error) => void) | null = null;
+
+  const optionsJson = JSON.stringify({
+    max_tokens: options?.maxTokens ?? 1000,
+    temperature: options?.temperature ?? 0.7,
+    system_prompt: options?.systemPrompt ?? null,
+  });
+
+  // Create the result promise
+  const resultPromise = new Promise<LLMGenerationResult>((resolve, reject) => {
+    resolveResult = resolve;
+    rejectResult = reject;
+  });
+
+  // Create async generator for tokens
+  async function* tokenGenerator(): AsyncGenerator<string> {
+    const tokenQueue: string[] = [];
+    let resolver: ((value: IteratorResult<string>) => void) | null = null;
+    let done = false;
+    let error: Error | null = null;
+
+    // Start streaming
+    native.generateStream(
+      prompt,
+      optionsJson,
+      (token: string, isComplete: boolean) => {
+        if (cancelled) return;
+
+        if (!isComplete && token) {
+          // Track first token time
+          if (firstTokenTime === null) {
+            firstTokenTime = Date.now();
+          }
+
+          fullText += token;
+          tokenCount++;
+
+          if (resolver) {
+            resolver({ value: token, done: false });
+            resolver = null;
+          } else {
+            tokenQueue.push(token);
+          }
+        }
+
+        if (isComplete) {
+          done = true;
+
+          // Build final result
+          const endTime = Date.now();
+          const latencyMs = endTime - startTime;
+          const timeToFirstTokenMs = firstTokenTime ? firstTokenTime - startTime : undefined;
+          const tokensPerSecond = latencyMs > 0 ? (tokenCount / latencyMs) * 1000 : 0;
+
+          const finalResult: LLMGenerationResult = {
+            text: fullText,
+            thinkingContent: undefined,
+            inputTokens: Math.ceil(prompt.length / 4),
+            tokensUsed: tokenCount,
+            modelUsed: 'unknown',
+            latencyMs,
+            framework: 'unknown', // Backend-agnostic
+            tokensPerSecond,
+            timeToFirstTokenMs,
+            thinkingTokens: 0,
+            responseTokens: tokenCount,
+          };
+
+          if (resolveResult) {
+            resolveResult(finalResult);
+          }
+
+          if (resolver) {
+            resolver({ value: undefined as unknown as string, done: true });
+            resolver = null;
+          }
+
+          EventBus.publish('Generation', { type: 'completed' });
+        }
+      }
+    ).catch((err: Error) => {
+      error = err;
+      done = true;
+      if (rejectResult) {
+        rejectResult(err);
+      }
+      if (resolver) {
+        resolver({ value: undefined as unknown as string, done: true });
+      }
+      EventBus.publish('Generation', { type: 'failed', error: err.message });
+    });
+
+    // Yield tokens
+    while (!done || tokenQueue.length > 0) {
+      if (tokenQueue.length > 0) {
+        yield tokenQueue.shift()!;
+      } else if (!done) {
+        const result = await new Promise<IteratorResult<string>>((resolve) => {
+          resolver = resolve;
+        });
+        if (result.done) break;
+        yield result.value;
+      }
+    }
+
+    if (error) {
+      throw error;
+    }
+  }
+
+  // Cancel function
+  const cancel = (): void => {
+    cancelled = true;
+    cancelGeneration();
+  };
+
+  return {
+    stream: tokenGenerator(),
+    result: resultPromise,
+    cancel,
+  };
+}
+
+/**
+ * Cancel ongoing text generation
+ */
+export function cancelGeneration(): void {
+  if (!isNativeModuleAvailable()) {
+    return;
+  }
+  const native = requireNativeModule();
+  native.cancelGeneration();
+}

package/src/Public/Extensions/RunAnywhere+VAD.ts

@@ -0,0 +1,359 @@
+/**
+ * RunAnywhere+VAD.ts
+ *
+ * Voice Activity Detection extension for RunAnywhere SDK.
+ * Matches iOS: RunAnywhere+VAD.swift
+ */
+
+import { requireNativeModule, isNativeModuleAvailable } from '../../native';
+import { EventBus } from '../Events';
+import { SDKLogger } from '../../Foundation/Logging/Logger/SDKLogger';
+import type {
+  VADConfiguration,
+  VADResult,
+  SpeechActivityEvent,
+  VADSpeechActivityCallback,
+  VADAudioBufferCallback,
+  VADState,
+} from '../../types/VADTypes';
+
+const logger = new SDKLogger('RunAnywhere.VAD');
+
+// ============================================================================
+// VAD State Management
+// ============================================================================
+
+// Internal VAD state
+let vadState: VADState = {
+  isInitialized: false,
+  isRunning: false,
+  isSpeechActive: false,
+  currentProbability: 0,
+};
+
+// Callbacks
+let speechActivityCallback: VADSpeechActivityCallback | null = null;
+let audioBufferCallback: VADAudioBufferCallback | null = null;
+
+// ============================================================================
+// VAD Initialization
+// ============================================================================
+
+/**
+ * Initialize VAD with default configuration
+ * Matches Swift SDK: RunAnywhere.initializeVAD()
+ */
+export async function initializeVAD(config?: VADConfiguration): Promise<void> {
+  if (!isNativeModuleAvailable()) {
+    throw new Error('Native module not available');
+  }
+
+  logger.info('Initializing VAD...');
+
+  const native = requireNativeModule();
+
+  // If a config is provided, configure VAD first
+  if (config) {
+    const configJson = JSON.stringify({
+      sampleRate: config.sampleRate ?? 16000,
+      frameLength: config.frameLength ?? 0.1,
+      energyThreshold: config.energyThreshold ?? 0.005,
+    });
+
+    // Load VAD model if path provided, otherwise use default
+    const loaded = await native.loadVADModel('default', configJson);
+    if (!loaded) {
+      throw new Error('Failed to initialize VAD');
+    }
+  }
+
+  vadState.isInitialized = true;
+  logger.info('VAD initialized');
+
+  EventBus.publish('Voice', { type: 'vadInitialized' });
+}
+
+/**
+ * Check if VAD is ready
+ * Matches Swift SDK: RunAnywhere.isVADReady
+ */
+export async function isVADReady(): Promise<boolean> {
+  if (!isNativeModuleAvailable()) {
+    return false;
+  }
+
+  const native = requireNativeModule();
+  return native.isVADModelLoaded();
+}
+
+/**
+ * Get current VAD state
+ */
+export function getVADState(): VADState {
+  return { ...vadState };
+}
+
+// ============================================================================
+// VAD Model Loading
+// ============================================================================
+
+/**
+ * Load a VAD model
+ */
+export async function loadVADModel(
+  modelPath: string,
+  config?: VADConfiguration
+): Promise<boolean> {
+  if (!isNativeModuleAvailable()) {
+    return false;
+  }
+
+  logger.info(`Loading VAD model: ${modelPath}`);
+  const native = requireNativeModule();
+
+  const configJson = config ? JSON.stringify(config) : undefined;
+  const result = await native.loadVADModel(modelPath, configJson);
+
+  if (result) {
+    vadState.isInitialized = true;
+    logger.info('VAD model loaded');
+  }
+
+  return result;
+}
+
+/**
+ * Check if a VAD model is loaded
+ */
+export async function isVADModelLoaded(): Promise<boolean> {
+  if (!isNativeModuleAvailable()) {
+    return false;
+  }
+  const native = requireNativeModule();
+  return native.isVADModelLoaded();
+}
+
+/**
+ * Unload the current VAD model
+ */
+export async function unloadVADModel(): Promise<boolean> {
+  if (!isNativeModuleAvailable()) {
+    return false;
+  }
+
+  const native = requireNativeModule();
+  const result = await native.unloadVADModel();
+
+  if (result) {
+    vadState.isInitialized = false;
+    vadState.isRunning = false;
+    vadState.isSpeechActive = false;
+    logger.info('VAD model unloaded');
+  }
+
+  return result;
+}
+
+// ============================================================================
+// Speech Detection
+// ============================================================================
+
+/**
+ * Detect speech in audio samples
+ * Matches Swift SDK: RunAnywhere.detectSpeech(in:)
+ *
+ * @param samples Float32Array of audio samples
+ * @returns Whether speech was detected
+ */
+export async function detectSpeech(samples: Float32Array): Promise<boolean> {
+  if (!isNativeModuleAvailable()) {
+    throw new Error('Native module not available');
+  }
+
+  // Convert Float32Array to base64
+  const bytes = new Uint8Array(samples.buffer);
+  let binary = '';
+  for (let i = 0; i < bytes.byteLength; i++) {
+    const byte = bytes[i];
+    if (byte !== undefined) {
+      binary += String.fromCharCode(byte);
+    }
+  }
+  const audioBase64 = btoa(binary);
+
+  const result = await processVAD(audioBase64);
+
+  // Update state
+  const wasSpeechActive = vadState.isSpeechActive;
+  vadState.isSpeechActive = result.isSpeech;
+  vadState.currentProbability = result.probability;
+
+  // Emit speech activity events
+  if (result.isSpeech && !wasSpeechActive) {
+    if (speechActivityCallback) {
+      speechActivityCallback('started');
+    }
+    EventBus.publish('Voice', { type: 'speechStarted' });
+  } else if (!result.isSpeech && wasSpeechActive) {
+    if (speechActivityCallback) {
+      speechActivityCallback('ended');
+    }
+    EventBus.publish('Voice', { type: 'speechEnded' });
+  }
+
+  // Forward to audio buffer callback if set
+  if (audioBufferCallback) {
+    audioBufferCallback(samples);
+  }
+
+  return result.isSpeech;
+}
+
+/**
+ * Process audio for voice activity detection
+ * Returns detailed VAD result
+ */
+export async function processVAD(
+  audioData: string | ArrayBuffer,
+  sampleRate: number = 16000
+): Promise<VADResult> {
+  if (!isNativeModuleAvailable()) {
+    return { isSpeech: false, probability: 0 };
+  }
+  const native = requireNativeModule();
+
+  let audioBase64: string;
+  if (typeof audioData === 'string') {
+    audioBase64 = audioData;
+  } else {
+    const bytes = new Uint8Array(audioData);
+    let binary = '';
+    for (let i = 0; i < bytes.byteLength; i++) {
+      const byte = bytes[i];
+      if (byte !== undefined) {
+        binary += String.fromCharCode(byte);
+      }
+    }
+    audioBase64 = btoa(binary);
+  }
+
+  const optionsJson = JSON.stringify({ sampleRate });
+  const resultJson = await native.processVAD(audioBase64, optionsJson);
+
+  try {
+    const result = JSON.parse(resultJson);
+    return {
+      isSpeech: result.isSpeech ?? false,
+      probability: result.speechProbability ?? result.probability ?? 0,
+      startTime: result.startTime,
+      endTime: result.endTime,
+    };
+  } catch {
+    return { isSpeech: false, probability: 0 };
+  }
+}
+
+// ============================================================================
+// VAD Control
+// ============================================================================
+
+/**
+ * Start VAD processing
+ * Matches Swift SDK: RunAnywhere.startVAD()
+ */
+export async function startVAD(): Promise<void> {
+  if (!vadState.isInitialized) {
+    await initializeVAD();
+  }
+
+  vadState.isRunning = true;
+  logger.info('VAD started');
+
+  EventBus.publish('Voice', { type: 'vadStarted' });
+}
+
+/**
+ * Stop VAD processing
+ * Matches Swift SDK: RunAnywhere.stopVAD()
+ */
+export async function stopVAD(): Promise<void> {
+  vadState.isRunning = false;
+  vadState.isSpeechActive = false;
+  vadState.currentProbability = 0;
+
+  logger.info('VAD stopped');
+  EventBus.publish('Voice', { type: 'vadStopped' });
+}
+
+/**
+ * Reset VAD state
+ */
+export async function resetVAD(): Promise<void> {
+  if (!isNativeModuleAvailable()) {
+    return;
+  }
+
+  const native = requireNativeModule();
+  await native.resetVAD();
+
+  vadState.isSpeechActive = false;
+  vadState.currentProbability = 0;
+
+  logger.debug('VAD state reset');
+}
+
+// ============================================================================
+// Callbacks
+// ============================================================================
+
+/**
+ * Set VAD speech activity callback
+ * Matches Swift SDK: RunAnywhere.setVADSpeechActivityCallback(_:)
+ *
+ * @param callback Callback invoked when speech state changes
+ */
+export function setVADSpeechActivityCallback(
+  callback: VADSpeechActivityCallback | null
+): void {
+  speechActivityCallback = callback;
+  logger.debug('VAD speech activity callback set');
+}
+
+/**
+ * Set VAD audio buffer callback
+ * Matches Swift SDK: RunAnywhere.setVADAudioBufferCallback(_:)
+ *
+ * @param callback Callback invoked with audio samples
+ */
+export function setVADAudioBufferCallback(
+  callback: VADAudioBufferCallback | null
+): void {
+  audioBufferCallback = callback;
+  logger.debug('VAD audio buffer callback set');
+}
+
+// ============================================================================
+// Cleanup
+// ============================================================================
+
+/**
+ * Cleanup VAD resources
+ * Matches Swift SDK: RunAnywhere.cleanupVAD()
+ */
+export async function cleanupVAD(): Promise<void> {
+  await stopVAD();
+  await unloadVADModel();
+
+  speechActivityCallback = null;
+  audioBufferCallback = null;
+
+  vadState = {
+    isInitialized: false,
+    isRunning: false,
+    isSpeechActive: false,
+    currentProbability: 0,
+  };
+
+  logger.info('VAD cleaned up');
+  EventBus.publish('Voice', { type: 'vadCleanedUp' });
+}