@runanywhere/web 0.1.0-beta.1

package/dist/Public/Extensions/RunAnywhere+VAD.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"RunAnywhere+VAD.d.ts","sourceRoot":"","sources":["../../../src/Public/Extensions/RunAnywhere+VAD.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AASH,OAAO,KAAK,EAAE,sBAAsB,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAMxF,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAC5C,YAAY,EAAE,sBAAsB,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAiBxF,cAAM,OAAO;IACX,QAAQ,CAAC,aAAa,SAAS;IAC/B,OAAO,CAAC,UAAU,CAAK;IACvB,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,mBAAmB,CAAuC;IAClE,OAAO,CAAC,gBAAgB,CAAS;IAEjC;;;OAGG;IACG,SAAS,CAAC,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC;IAmEtD,mCAAmC;IACnC,IAAI,aAAa,IAAI,OAAO,CAE3B;IAED;;;OAGG;IACH,gBAAgB,CAAC,QAAQ,EAAE,sBAAsB,GAAG,MAAM,IAAI;IAK9D;;;;;;;;;OASG;IACH,cAAc,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO;IAqC9C;;;;;;OAMG;IACH,gBAAgB,IAAI,aAAa,GAAG,IAAI;IAiCxC,4CAA4C;IAC5C,IAAI,cAAc,IAAI,OAAO,CAG5B;IAED,uBAAuB;IACvB,KAAK,IAAI,IAAI;IAMb,yCAAyC;IACzC,KAAK,IAAI,IAAI;IAKb,kCAAkC;IAClC,OAAO,IAAI,IAAI;CAUhB;AAED,eAAO,MAAM,GAAG,SAAgB,CAAC"}

package/dist/Public/Extensions/RunAnywhere+VAD.js

@@ -0,0 +1,231 @@
+/**
+ * RunAnywhere Web SDK - Voice Activity Detection Extension
+ *
+ * Adds VAD capabilities via sherpa-onnx WASM using Silero VAD model.
+ * Detects speech segments in audio streams with high accuracy.
+ *
+ * Mirrors: sdk/runanywhere-swift/Sources/RunAnywhere/Public/Extensions/VAD/
+ *
+ * Usage:
+ *   import { VAD } from '@runanywhere/web';
+ *
+ *   await VAD.loadModel({
+ *     modelPath: '/models/vad/silero_vad.onnx',
+ *     threshold: 0.5,
+ *   });
+ *
+ *   const hasVoice = VAD.processSamples(audioFloat32Array);
+ *   if (hasVoice) console.log('Speech detected!');
+ */
+import { RunAnywhere } from '../RunAnywhere';
+import { SherpaONNXBridge } from '../../Foundation/SherpaONNXBridge';
+import { SDKError, SDKErrorCode } from '../../Foundation/ErrorTypes';
+import { SDKLogger } from '../../Foundation/SDKLogger';
+import { EventBus } from '../../Foundation/EventBus';
+import { SDKEventType } from '../../types/enums';
+import { SpeechActivity } from './VADTypes';
+// @ts-expect-error -- sherpa-onnx JS wrappers ship without type declarations
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+import { initSherpaOnnxVadModelConfig, freeConfig } from '../../../wasm/sherpa/sherpa-onnx-vad.js';
+export { SpeechActivity } from './VADTypes';
+const logger = new SDKLogger('VAD');
+// ---------------------------------------------------------------------------
+// Internal Helpers
+// ---------------------------------------------------------------------------
+function requireSherpa() {
+    if (!RunAnywhere.isInitialized)
+        throw SDKError.notInitialized();
+    return SherpaONNXBridge.shared;
+}
+// ---------------------------------------------------------------------------
+// VAD Extension
+// ---------------------------------------------------------------------------
+class VADImpl {
+    extensionName = 'VAD';
+    _vadHandle = 0;
+    _sampleRate = 16000;
+    _jsActivityCallback = null;
+    _lastSpeechState = false;
+    /**
+     * Load the Silero VAD model via sherpa-onnx.
+     * The model file must already be in the sherpa-onnx virtual FS.
+     */
+    async loadModel(config) {
+        const sherpa = requireSherpa();
+        await sherpa.ensureLoaded();
+        const m = sherpa.module;
+        // Clean up previous
+        this.cleanup();
+        this._sampleRate = config.sampleRate ?? 16000;
+        logger.info('Loading Silero VAD model');
+        EventBus.shared.emit('model.loadStarted', SDKEventType.Model, {
+            modelId: 'silero-vad', component: 'vad',
+        });
+        const startMs = performance.now();
+        const bufferSizeInSeconds = 30; // 30 second circular buffer
+        // Build the struct-based config matching sherpa-onnx C API layout.
+        // Uses initSherpaOnnxVadModelConfig from sherpa-onnx-vad.js which
+        // allocates a C struct in WASM memory (not JSON).
+        const configObj = {
+            sileroVad: {
+                model: config.modelPath,
+                threshold: config.threshold ?? 0.5,
+                minSilenceDuration: config.minSilenceDuration ?? 0.5,
+                minSpeechDuration: config.minSpeechDuration ?? 0.25,
+                maxSpeechDuration: config.maxSpeechDuration ?? 5.0,
+                windowSize: config.windowSize ?? 512,
+            },
+            tenVad: {
+                model: '',
+                threshold: 0.5,
+                minSilenceDuration: 0.5,
+                minSpeechDuration: 0.25,
+                maxSpeechDuration: 20,
+                windowSize: 256,
+            },
+            sampleRate: this._sampleRate,
+            numThreads: 1,
+            provider: 'cpu',
+            debug: 0,
+        };
+        const configStruct = initSherpaOnnxVadModelConfig(configObj, m);
+        try {
+            this._vadHandle = m._SherpaOnnxCreateVoiceActivityDetector(configStruct.ptr, bufferSizeInSeconds);
+            freeConfig(configStruct, m);
+            if (this._vadHandle === 0) {
+                throw new SDKError(SDKErrorCode.ModelLoadFailed, 'Failed to create VAD from Silero model');
+            }
+            const loadTimeMs = Math.round(performance.now() - startMs);
+            logger.info(`Silero VAD loaded in ${loadTimeMs}ms`);
+            EventBus.shared.emit('model.loadCompleted', SDKEventType.Model, {
+                modelId: 'silero-vad', component: 'vad', loadTimeMs,
+            });
+        }
+        catch (error) {
+            this.cleanup();
+            throw error;
+        }
+    }
+    /** Whether VAD model is loaded. */
+    get isInitialized() {
+        return this._vadHandle !== 0;
+    }
+    /**
+     * Register a callback for speech activity events.
+     * Called when speech starts, ends, or is ongoing.
+     */
+    onSpeechActivity(callback) {
+        this._jsActivityCallback = callback;
+        return () => { this._jsActivityCallback = null; };
+    }
+    /**
+     * Process audio samples through VAD.
+     * Returns whether speech was detected in this frame.
+     *
+     * The Silero VAD expects 512-sample windows at 16kHz.
+     * This method handles arbitrary-length input by feeding in chunks.
+     *
+     * @param samples - Float32Array of PCM audio samples (mono, 16kHz)
+     * @returns Whether speech is currently detected
+     */
+    processSamples(samples) {
+        if (this._vadHandle === 0) {
+            logger.warning('VAD not initialized. Call loadModel() first.');
+            return false;
+        }
+        const m = SherpaONNXBridge.shared.module;
+        // Copy samples to WASM memory
+        const audioPtr = m._malloc(samples.length * 4);
+        m.HEAPF32.set(samples, audioPtr / 4);
+        try {
+            // Feed samples to VAD
+            m._SherpaOnnxVoiceActivityDetectorAcceptWaveform(this._vadHandle, audioPtr, samples.length);
+            // Check detection state
+            const detected = m._SherpaOnnxVoiceActivityDetectorDetected(this._vadHandle) !== 0;
+            // Emit speech activity callbacks
+            if (detected && !this._lastSpeechState) {
+                this._jsActivityCallback?.(SpeechActivity.Started);
+                EventBus.shared.emit('vad.speechStarted', SDKEventType.Voice, { activity: SpeechActivity.Started });
+            }
+            else if (!detected && this._lastSpeechState) {
+                this._jsActivityCallback?.(SpeechActivity.Ended);
+                EventBus.shared.emit('vad.speechEnded', SDKEventType.Voice, { activity: SpeechActivity.Ended });
+            }
+            else if (detected) {
+                this._jsActivityCallback?.(SpeechActivity.Ongoing);
+            }
+            this._lastSpeechState = detected;
+            return detected;
+        }
+        finally {
+            m._free(audioPtr);
+        }
+    }
+    /**
+     * Get the next available speech segment (if any).
+     * Returns null if no complete segments are available.
+     *
+     * After calling processSamples(), check for available segments
+     * using this method. Call repeatedly until it returns null.
+     */
+    popSpeechSegment() {
+        if (this._vadHandle === 0)
+            return null;
+        const m = SherpaONNXBridge.shared.module;
+        // Check if there's a segment available
+        if (m._SherpaOnnxVoiceActivityDetectorEmpty(this._vadHandle) !== 0) {
+            return null;
+        }
+        // Get the front segment
+        const segmentPtr = m._SherpaOnnxVoiceActivityDetectorFront(this._vadHandle);
+        if (segmentPtr === 0)
+            return null;
+        // Read segment struct: { int32_t start; const float* samples; int32_t n; }
+        // (matches sherpa-onnx-vad.js Vad.front() layout)
+        const startTime = m.HEAP32[segmentPtr / 4];
+        const samplesPtr = m.HEAP32[segmentPtr / 4 + 1];
+        const numSamples = m.HEAP32[segmentPtr / 4 + 2];
+        // Copy samples from WASM heap
+        const samples = new Float32Array(numSamples);
+        if (samplesPtr && numSamples > 0) {
+            samples.set(m.HEAPF32.subarray(samplesPtr / 4, samplesPtr / 4 + numSamples));
+        }
+        // Destroy the segment and pop
+        m._SherpaOnnxDestroySpeechSegment(segmentPtr);
+        m._SherpaOnnxVoiceActivityDetectorPop(this._vadHandle);
+        return { startTime, samples };
+    }
+    /** Whether speech is currently detected. */
+    get isSpeechActive() {
+        if (this._vadHandle === 0)
+            return false;
+        return SherpaONNXBridge.shared.module._SherpaOnnxVoiceActivityDetectorDetected(this._vadHandle) !== 0;
+    }
+    /** Reset VAD state. */
+    reset() {
+        if (this._vadHandle === 0)
+            return;
+        SherpaONNXBridge.shared.module._SherpaOnnxVoiceActivityDetectorReset(this._vadHandle);
+        this._lastSpeechState = false;
+    }
+    /** Flush remaining audio through VAD. */
+    flush() {
+        if (this._vadHandle === 0)
+            return;
+        SherpaONNXBridge.shared.module._SherpaOnnxVoiceActivityDetectorFlush(this._vadHandle);
+    }
+    /** Clean up the VAD resources. */
+    cleanup() {
+        if (this._vadHandle !== 0) {
+            try {
+                SherpaONNXBridge.shared.module._SherpaOnnxDestroyVoiceActivityDetector(this._vadHandle);
+            }
+            catch { /* ignore */ }
+            this._vadHandle = 0;
+        }
+        this._jsActivityCallback = null;
+        this._lastSpeechState = false;
+    }
+}
+export const VAD = new VADImpl();
+//# sourceMappingURL=RunAnywhere+VAD.js.map

package/dist/Public/Extensions/RunAnywhere+VAD.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"RunAnywhere+VAD.js","sourceRoot":"","sources":["../../../src/Public/Extensions/RunAnywhere+VAD.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,OAAO,EAAE,gBAAgB,EAAE,MAAM,mCAAmC,CAAC;AACrE,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AACrE,OAAO,EAAE,SAAS,EAAE,MAAM,4BAA4B,CAAC;AACvD,OAAO,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AACrD,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACjD,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAG5C,6EAA6E;AAC7E,8DAA8D;AAC9D,OAAO,EAAE,4BAA4B,EAAE,UAAU,EAAE,MAAM,yCAAyC,CAAC;AAEnG,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAG5C,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC,KAAK,CAAC,CAAC;AAEpC,8EAA8E;AAC9E,mBAAmB;AACnB,8EAA8E;AAE9E,SAAS,aAAa;IACpB,IAAI,CAAC,WAAW,CAAC,aAAa;QAAE,MAAM,QAAQ,CAAC,cAAc,EAAE,CAAC;IAChE,OAAO,gBAAgB,CAAC,MAAM,CAAC;AACjC,CAAC;AAED,8EAA8E;AAC9E,gBAAgB;AAChB,8EAA8E;AAE9E,MAAM,OAAO;IACF,aAAa,GAAG,KAAK,CAAC;IACvB,UAAU,GAAG,CAAC,CAAC;IACf,WAAW,GAAG,KAAK,CAAC;IACpB,mBAAmB,GAAkC,IAAI,CAAC;IAC1D,gBAAgB,GAAG,KAAK,CAAC;IAEjC;;;OAGG;IACH,KAAK,CAAC,SAAS,CAAC,MAAsB;QACpC,MAAM,MAAM,GAAG,aAAa,EAAE,CAAC;QAC/B,MAAM,MAAM,CAAC,YAAY,EAAE,CAAC;QAC5B,MAAM,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;QAExB,oBAAoB;QACpB,IAAI,CAAC,OAAO,EAAE,CAAC;QAEf,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,UAAU,IAAI,KAAK,CAAC;QAE9C,MAAM,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;QACxC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,mBAAmB,EAAE,YAAY,CAAC,KAAK,EAAE;YAC5D,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,KAAK;SACxC,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAClC,MAAM,mBAAmB,GAAG,EAAE,CAAC,CAAC,4BAA4B;QAE5D,mEAAmE;QACnE,kEAAkE;QAClE,kDAAkD;QAClD,MAAM,SAAS,GAAG;YAChB,SAAS,EAAE;gBACT,KAAK,EAAE,MAAM,CAAC,SAAS;gBACvB,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,GAAG;gBAClC,kBAAkB,EAAE,MAAM,CAAC,kBAAkB,IAAI,GAAG;gBACpD,iBAAiB,EAAE,MAAM,CAAC,iBAAiB,IAAI,IAAI;gBACnD,iBAAiB,EAAE,MAAM,CAAC,iBAAiB,IAAI,GAAG;gBAClD,UAAU,EAAE,MAAM,CAAC,UAAU,IAAI,GAAG;aACrC;YACD,MAAM,EAAE;gBACN,KAAK,EAAE,EAAE;gBACT,SAAS,EAAE,GAAG;gBACd,kBAAkB,EAAE,GAAG;gBACvB,iBAAiB,EAAE,IAAI;gBACvB,iBAAiB,EAAE,EAAE;gBACrB,UAAU,EAAE,GAAG;aAChB;YACD,UAAU,EAAE,IAAI,CAAC,WAAW;YAC5B,UAAU,EAAE,CAAC;YACb,QAAQ,EAAE,KAAK;YACf,KAAK,EAAE,CAAC;SACT,CAAC;QAEF,MAAM,YAAY,GAAG,4BAA4B,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;QAEhE,IAAI,CAAC;YACH,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC,sCAAsC,CACxD,YAAY,CAAC,GAAG,EAAE,mBAAmB,CACtC,CAAC;YACF,UAAU,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC;YAE5B,IAAI,IAAI,CAAC,UAAU,KAAK,CAAC,EAAE,CAAC;gBAC1B,MAAM,IAAI,QAAQ,CAAC,YAAY,CAAC,eAAe,EAAE,wCAAwC,CAAC,CAAC;YAC7F,CAAC;YAED,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,CAAC;YAC3D,MAAM,CAAC,IAAI,CAAC,wBAAwB,UAAU,IAAI,CAAC,CAAC;YACpD,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,qBAAqB,EAAE,YAAY,CAAC,KAAK,EAAE;gBAC9D,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,KAAK,EAAE,UAAU;aACpD,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,OAAO,EAAE,CAAC;YACf,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED,mCAAmC;IACnC,IAAI,aAAa;QACf,OAAO,IAAI,CAAC,UAAU,KAAK,CAAC,CAAC;IAC/B,CAAC;IAED;;;OAGG;IACH,gBAAgB,CAAC,QAAgC;QAC/C,IAAI,CAAC,mBAAmB,GAAG,QAAQ,CAAC;QACpC,OAAO,GAAG,EAAE,GAAG,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;IACpD,CAAC;IAED;;;;;;;;;OASG;IACH,cAAc,CAAC,OAAqB;QAClC,IAAI,IAAI,CAAC,UAAU,KAAK,CAAC,EAAE,CAAC;YAC1B,MAAM,CAAC,OAAO,CAAC,8CAA8C,CAAC,CAAC;YAC/D,OAAO,KAAK,CAAC;QACf,CAAC;QAED,MAAM,CAAC,GAAG,gBAAgB,CAAC,MAAM,CAAC,MAAM,CAAC;QAEzC,8BAA8B;QAC9B,MAAM,QAAQ,GAAG,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC/C,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,QAAQ,GAAG,CAAC,CAAC,CAAC;QAErC,IAAI,CAAC;YACH,sBAAsB;YACtB,CAAC,CAAC,8CAA8C,CAAC,IAAI,CAAC,UAAU,EAAE,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;YAE5F,wBAAwB;YACxB,MAAM,QAAQ,GAAG,CAAC,CAAC,wCAAwC,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;YAEnF,iCAAiC;YACjC,IAAI,QAAQ,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBACvC,IAAI,CAAC,mBAAmB,EAAE,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;gBACnD,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,mBAAmB,EAAE,YAAY,CAAC,KAAK,EAAE,EAAE,QAAQ,EAAE,cAAc,CAAC,OAAO,EAAE,CAAC,CAAC;YACtG,CAAC;iBAAM,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBAC9C,IAAI,CAAC,mBAAmB,EAAE,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC;gBACjD,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,iBAAiB,EAAE,YAAY,CAAC,KAAK,EAAE,EAAE,QAAQ,EAAE,cAAc,CAAC,KAAK,EAAE,CAAC,CAAC;YAClG,CAAC;iBAAM,IAAI,QAAQ,EAAE,CAAC;gBACpB,IAAI,CAAC,mBAAmB,EAAE,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;YACrD,CAAC;YAED,IAAI,CAAC,gBAAgB,GAAG,QAAQ,CAAC;YACjC,OAAO,QAAQ,CAAC;QAClB,CAAC;gBAAS,CAAC;YACT,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QACpB,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACH,gBAAgB;QACd,IAAI,IAAI,CAAC,UAAU,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QAEvC,MAAM,CAAC,GAAG,gBAAgB,CAAC,MAAM,CAAC,MAAM,CAAC;QAEzC,uCAAuC;QACvC,IAAI,CAAC,CAAC,qCAAqC,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;YACnE,OAAO,IAAI,CAAC;QACd,CAAC;QAED,wBAAwB;QACxB,MAAM,UAAU,GAAG,CAAC,CAAC,qCAAqC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAC5E,IAAI,UAAU,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QAElC,2EAA2E;QAC3E,kDAAkD;QAClD,MAAM,SAAS,GAAG,CAAC,CAAC,MAAM,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC;QAC3C,MAAM,UAAU,GAAG,CAAC,CAAC,MAAM,CAAC,UAAU,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;QAChD,MAAM,UAAU,GAAG,CAAC,CAAC,MAAM,CAAC,UAAU,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;QAEhD,8BAA8B;QAC9B,MAAM,OAAO,GAAG,IAAI,YAAY,CAAC,UAAU,CAAC,CAAC;QAC7C,IAAI,UAAU,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;YACjC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,UAAU,GAAG,CAAC,EAAE,UAAU,GAAG,CAAC,GAAG,UAAU,CAAC,CAAC,CAAC;QAC/E,CAAC;QAED,8BAA8B;QAC9B,CAAC,CAAC,+BAA+B,CAAC,UAAU,CAAC,CAAC;QAC9C,CAAC,CAAC,mCAAmC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAEvD,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC;IAChC,CAAC;IAED,4CAA4C;IAC5C,IAAI,cAAc;QAChB,IAAI,IAAI,CAAC,UAAU,KAAK,CAAC;YAAE,OAAO,KAAK,CAAC;QACxC,OAAO,gBAAgB,CAAC,MAAM,CAAC,MAAM,CAAC,wCAAwC,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;IACxG,CAAC;IAED,uBAAuB;IACvB,KAAK;QACH,IAAI,IAAI,CAAC,UAAU,KAAK,CAAC;YAAE,OAAO;QAClC,gBAAgB,CAAC,MAAM,CAAC,MAAM,CAAC,qCAAqC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACtF,IAAI,CAAC,gBAAgB,GAAG,KAAK,CAAC;IAChC,CAAC;IAED,yCAAyC;IACzC,KAAK;QACH,IAAI,IAAI,CAAC,UAAU,KAAK,CAAC;YAAE,OAAO;QAClC,gBAAgB,CAAC,MAAM,CAAC,MAAM,CAAC,qCAAqC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IACxF,CAAC;IAED,kCAAkC;IAClC,OAAO;QACL,IAAI,IAAI,CAAC,UAAU,KAAK,CAAC,EAAE,CAAC;YAC1B,IAAI,CAAC;gBACH,gBAAgB,CAAC,MAAM,CAAC,MAAM,CAAC,uCAAuC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAC1F,CAAC;YAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;YACxB,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC;QACtB,CAAC;QACD,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;QAChC,IAAI,CAAC,gBAAgB,GAAG,KAAK,CAAC;IAChC,CAAC;CACF;AAED,MAAM,CAAC,MAAM,GAAG,GAAG,IAAI,OAAO,EAAE,CAAC"}

package/dist/Public/Extensions/RunAnywhere+VLM.d.ts

@@ -0,0 +1,58 @@
+/**
+ * RunAnywhere Web SDK - Vision Language Model Extension
+ *
+ * Adds VLM capabilities for image understanding + text generation.
+ * Uses the RACommons rac_vlm_component_* C API (llama.cpp mtmd backend).
+ *
+ * Mirrors: sdk/runanywhere-swift/Sources/RunAnywhere/Public/Extensions/VLM/
+ *
+ * Usage:
+ *   import { VLM } from '@runanywhere/web';
+ *
+ *   await VLM.loadModel('/models/qwen2-vl.gguf', '/models/qwen2-vl-mmproj.gguf', 'qwen2-vl');
+ *   const result = await VLM.process(imageData, 'Describe this image');
+ *   console.log(result.text);
+ */
+import type { VLMImage, VLMGenerationOptions, VLMGenerationResult } from './VLMTypes';
+export type { VLMImage, VLMGenerationOptions, VLMGenerationResult, VLMStreamingResult } from './VLMTypes';
+export { VLMImageFormat, VLMModelFamily } from './VLMTypes';
+declare class VLMImpl {
+    readonly extensionName = "VLM";
+    private _vlmComponentHandle;
+    private _vlmBackendRegistered;
+    private requireBridge;
+    /**
+     * Ensure the llama.cpp VLM backend is registered with the service registry.
+     * Must be called before creating the VLM component so it can find a provider.
+     */
+    private ensureVLMBackendRegistered;
+    private ensureVLMComponent;
+    /**
+     * Load a VLM model (GGUF model + multimodal projector).
+     *
+     * @param modelPath - Path to the GGUF model file in WASM FS
+     * @param mmprojPath - Path to the mmproj file in WASM FS
+     * @param modelId - Unique model identifier
+     * @param modelName - Display name (optional)
+     */
+    loadModel(modelPath: string, mmprojPath: string, modelId: string, modelName?: string): Promise<void>;
+    /** Unload the VLM model. */
+    unloadModel(): Promise<void>;
+    /** Check if a VLM model is loaded. */
+    get isModelLoaded(): boolean;
+    /**
+     * Process an image with a text prompt.
+     *
+     * @param image - Image input (file path, pixel data, or base64)
+     * @param prompt - Text prompt describing what to do with the image
+     * @param options - Generation options
+     * @returns VLM generation result
+     */
+    process(image: VLMImage, prompt: string, options?: VLMGenerationOptions): Promise<VLMGenerationResult>;
+    /** Cancel in-progress VLM generation. */
+    cancel(): void;
+    /** Clean up the VLM component and unregister backend. */
+    cleanup(): void;
+}
+export declare const VLM: VLMImpl;
+//# sourceMappingURL=RunAnywhere+VLM.d.ts.map

package/dist/Public/Extensions/RunAnywhere+VLM.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"RunAnywhere+VLM.d.ts","sourceRoot":"","sources":["../../../src/Public/Extensions/RunAnywhere+VLM.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAUH,OAAO,KAAK,EAAE,QAAQ,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAC;AAItF,YAAY,EAAE,QAAQ,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAC1G,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAM5D,cAAM,OAAO;IACX,QAAQ,CAAC,aAAa,SAAS;IAC/B,OAAO,CAAC,mBAAmB,CAAK;IAChC,OAAO,CAAC,qBAAqB,CAAS;IAEtC,OAAO,CAAC,aAAa;IAKrB;;;OAGG;IACH,OAAO,CAAC,0BAA0B;IAyBlC,OAAO,CAAC,kBAAkB;IAsB1B;;;;;;;OAOG;IACG,SAAS,CACb,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,MAAM,EACf,SAAS,CAAC,EAAE,MAAM,GACjB,OAAO,CAAC,IAAI,CAAC;IA8BhB,4BAA4B;IACtB,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAUlC,sCAAsC;IACtC,IAAI,aAAa,IAAI,OAAO,CAO3B;IAED;;;;;;;OAOG;IACG,OAAO,CACX,KAAK,EAAE,QAAQ,EACf,MAAM,EAAE,MAAM,EACd,OAAO,GAAE,oBAAyB,GACjC,OAAO,CAAC,mBAAmB,CAAC;IA+G/B,yCAAyC;IACzC,MAAM,IAAI,IAAI;IAOd,yDAAyD;IACzD,OAAO,IAAI,IAAI;CAqBhB;AAED,eAAO,MAAM,GAAG,SAAgB,CAAC"}

package/dist/Public/Extensions/RunAnywhere+VLM.js

@@ -0,0 +1,262 @@
+/**
+ * RunAnywhere Web SDK - Vision Language Model Extension
+ *
+ * Adds VLM capabilities for image understanding + text generation.
+ * Uses the RACommons rac_vlm_component_* C API (llama.cpp mtmd backend).
+ *
+ * Mirrors: sdk/runanywhere-swift/Sources/RunAnywhere/Public/Extensions/VLM/
+ *
+ * Usage:
+ *   import { VLM } from '@runanywhere/web';
+ *
+ *   await VLM.loadModel('/models/qwen2-vl.gguf', '/models/qwen2-vl-mmproj.gguf', 'qwen2-vl');
+ *   const result = await VLM.process(imageData, 'Describe this image');
+ *   console.log(result.text);
+ */
+import { RunAnywhere } from '../RunAnywhere';
+import { WASMBridge } from '../../Foundation/WASMBridge';
+import { Offsets } from '../../Foundation/StructOffsets';
+import { SDKError, SDKErrorCode } from '../../Foundation/ErrorTypes';
+import { SDKLogger } from '../../Foundation/SDKLogger';
+import { EventBus } from '../../Foundation/EventBus';
+import { SDKEventType } from '../../types/enums';
+import { VLMImageFormat, VLMModelFamily } from './VLMTypes';
+const logger = new SDKLogger('VLM');
+export { VLMImageFormat, VLMModelFamily } from './VLMTypes';
+// ---------------------------------------------------------------------------
+// VLM Extension
+// ---------------------------------------------------------------------------
+class VLMImpl {
+    extensionName = 'VLM';
+    _vlmComponentHandle = 0;
+    _vlmBackendRegistered = false;
+    requireBridge() {
+        if (!RunAnywhere.isInitialized)
+            throw SDKError.notInitialized();
+        return WASMBridge.shared;
+    }
+    /**
+     * Ensure the llama.cpp VLM backend is registered with the service registry.
+     * Must be called before creating the VLM component so it can find a provider.
+     */
+    ensureVLMBackendRegistered() {
+        if (this._vlmBackendRegistered)
+            return;
+        const bridge = this.requireBridge();
+        const m = bridge.module;
+        // Check if the backend registration function exists (only when built with --vlm)
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        const fn = m['_rac_backend_llamacpp_vlm_register'];
+        if (!fn) {
+            throw new SDKError(SDKErrorCode.BackendNotAvailable, 'VLM backend not available. Rebuild WASM with --vlm flag.');
+        }
+        const result = m.ccall('rac_backend_llamacpp_vlm_register', 'number', [], []);
+        if (result !== 0) {
+            bridge.checkResult(result, 'rac_backend_llamacpp_vlm_register');
+        }
+        this._vlmBackendRegistered = true;
+        logger.info('VLM backend (llama.cpp mtmd) registered');
+    }
+    ensureVLMComponent() {
+        if (this._vlmComponentHandle !== 0)
+            return this._vlmComponentHandle;
+        // Register the VLM backend first
+        this.ensureVLMBackendRegistered();
+        const bridge = this.requireBridge();
+        const m = bridge.module;
+        const handlePtr = m._malloc(4);
+        const result = m.ccall('rac_vlm_component_create', 'number', ['number'], [handlePtr]);
+        if (result !== 0) {
+            m._free(handlePtr);
+            bridge.checkResult(result, 'rac_vlm_component_create');
+        }
+        this._vlmComponentHandle = m.getValue(handlePtr, 'i32');
+        m._free(handlePtr);
+        logger.debug('VLM component created');
+        return this._vlmComponentHandle;
+    }
+    /**
+     * Load a VLM model (GGUF model + multimodal projector).
+     *
+     * @param modelPath - Path to the GGUF model file in WASM FS
+     * @param mmprojPath - Path to the mmproj file in WASM FS
+     * @param modelId - Unique model identifier
+     * @param modelName - Display name (optional)
+     */
+    async loadModel(modelPath, mmprojPath, modelId, modelName) {
+        const bridge = this.requireBridge();
+        const m = bridge.module;
+        const handle = this.ensureVLMComponent();
+        logger.info(`Loading VLM model: ${modelId}`);
+        EventBus.shared.emit('model.loadStarted', SDKEventType.Model, { modelId, component: 'vlm' });
+        const pathPtr = bridge.allocString(modelPath);
+        const projPtr = bridge.allocString(mmprojPath);
+        const idPtr = bridge.allocString(modelId);
+        const namePtr = bridge.allocString(modelName ?? modelId);
+        try {
+            const result = m.ccall('rac_vlm_component_load_model', 'number', ['number', 'number', 'number', 'number', 'number'], [handle, pathPtr, projPtr, idPtr, namePtr]);
+            bridge.checkResult(result, 'rac_vlm_component_load_model');
+            logger.info(`VLM model loaded: ${modelId}`);
+            EventBus.shared.emit('model.loadCompleted', SDKEventType.Model, { modelId, component: 'vlm' });
+        }
+        finally {
+            bridge.free(pathPtr);
+            bridge.free(projPtr);
+            bridge.free(idPtr);
+            bridge.free(namePtr);
+        }
+    }
+    /** Unload the VLM model. */
+    async unloadModel() {
+        if (this._vlmComponentHandle === 0)
+            return;
+        const bridge = this.requireBridge();
+        const result = bridge.module.ccall('rac_vlm_component_unload', 'number', ['number'], [this._vlmComponentHandle]);
+        bridge.checkResult(result, 'rac_vlm_component_unload');
+        logger.info('VLM model unloaded');
+    }
+    /** Check if a VLM model is loaded. */
+    get isModelLoaded() {
+        if (this._vlmComponentHandle === 0)
+            return false;
+        try {
+            return WASMBridge.shared.module.ccall('rac_vlm_component_is_loaded', 'number', ['number'], [this._vlmComponentHandle]) === 1;
+        }
+        catch {
+            return false;
+        }
+    }
+    /**
+     * Process an image with a text prompt.
+     *
+     * @param image - Image input (file path, pixel data, or base64)
+     * @param prompt - Text prompt describing what to do with the image
+     * @param options - Generation options
+     * @returns VLM generation result
+     */
+    async process(image, prompt, options = {}) {
+        const bridge = this.requireBridge();
+        const m = bridge.module;
+        const handle = this.ensureVLMComponent();
+        if (!this.isModelLoaded) {
+            throw new SDKError(SDKErrorCode.ModelNotLoaded, 'No VLM model loaded. Call loadModel() first.');
+        }
+        logger.debug(`VLM process: "${prompt.substring(0, 50)}..."`);
+        // Build rac_vlm_image_t struct
+        const imageSize = m._rac_wasm_sizeof_vlm_image();
+        const imagePtr = m._malloc(imageSize);
+        for (let i = 0; i < imageSize; i++)
+            m.setValue(imagePtr + i, 0, 'i8');
+        let filePathPtr = 0;
+        let base64Ptr = 0;
+        let pixelPtr = 0;
+        const vi = Offsets.vlmImage;
+        m.setValue(imagePtr + vi.format, image.format, 'i32');
+        if (image.format === VLMImageFormat.FilePath && image.filePath) {
+            filePathPtr = bridge.allocString(image.filePath);
+            m.setValue(imagePtr + vi.filePath, filePathPtr, '*');
+        }
+        else if (image.format === VLMImageFormat.Base64 && image.base64Data) {
+            base64Ptr = bridge.allocString(image.base64Data);
+            m.setValue(imagePtr + vi.base64Data, base64Ptr, '*');
+        }
+        else if (image.format === VLMImageFormat.RGBPixels && image.pixelData) {
+            pixelPtr = m._malloc(image.pixelData.length);
+            bridge.writeBytes(image.pixelData, pixelPtr);
+            m.setValue(imagePtr + vi.pixelData, pixelPtr, '*');
+        }
+        m.setValue(imagePtr + vi.width, image.width ?? 0, 'i32');
+        m.setValue(imagePtr + vi.height, image.height ?? 0, 'i32');
+        // data_size: use pixel data length for RGB, base64 string length for base64
+        const dataSize = image.pixelData?.length ?? image.base64Data?.length ?? 0;
+        m.setValue(imagePtr + vi.dataSize, dataSize, 'i32');
+        // Build rac_vlm_options_t
+        const optSize = m._rac_wasm_sizeof_vlm_options();
+        const optPtr = m._malloc(optSize);
+        for (let i = 0; i < optSize; i++)
+            m.setValue(optPtr + i, 0, 'i8');
+        const vo = Offsets.vlmOptions;
+        m.setValue(optPtr + vo.maxTokens, options.maxTokens ?? 512, 'i32');
+        m.setValue(optPtr + vo.temperature, options.temperature ?? 0.7, 'float');
+        m.setValue(optPtr + vo.topP, options.topP ?? 0.9, 'float');
+        m.setValue(optPtr + vo.streamingEnabled, options.streaming ? 1 : 0, 'i32');
+        let sysPtr = 0;
+        if (options.systemPrompt) {
+            sysPtr = bridge.allocString(options.systemPrompt);
+            m.setValue(optPtr + vo.systemPrompt, sysPtr, '*');
+        }
+        m.setValue(optPtr + vo.modelFamily, options.modelFamily ?? VLMModelFamily.Auto, 'i32');
+        const promptPtr = bridge.allocString(prompt);
+        // Result struct
+        const resSize = m._rac_wasm_sizeof_vlm_result();
+        const resPtr = m._malloc(resSize);
+        try {
+            const r = m.ccall('rac_vlm_component_process', 'number', ['number', 'number', 'number', 'number', 'number'], [handle, imagePtr, promptPtr, optPtr, resPtr]);
+            bridge.checkResult(r, 'rac_vlm_component_process');
+            // Read rac_vlm_result_t (offsets from compiler via StructOffsets)
+            const vr = Offsets.vlmResult;
+            const textPtr = m.getValue(resPtr + vr.text, '*');
+            const vlmResult = {
+                text: bridge.readString(textPtr),
+                promptTokens: m.getValue(resPtr + vr.promptTokens, 'i32'),
+                imageTokens: m.getValue(resPtr + vr.imageTokens, 'i32'),
+                completionTokens: m.getValue(resPtr + vr.completionTokens, 'i32'),
+                totalTokens: m.getValue(resPtr + vr.totalTokens, 'i32'),
+                timeToFirstTokenMs: m.getValue(resPtr + vr.timeToFirstTokenMs, 'i32'),
+                imageEncodeTimeMs: m.getValue(resPtr + vr.imageEncodeTimeMs, 'i32'),
+                totalTimeMs: m.getValue(resPtr + vr.totalTimeMs, 'i32'),
+                tokensPerSecond: m.getValue(resPtr + vr.tokensPerSecond, 'float'),
+                hardwareUsed: bridge.accelerationMode,
+            };
+            m.ccall('rac_vlm_result_free', null, ['number'], [resPtr]);
+            EventBus.shared.emit('vlm.processed', SDKEventType.Generation, {
+                tokensPerSecond: vlmResult.tokensPerSecond,
+                totalTokens: vlmResult.totalTokens,
+                hardwareUsed: vlmResult.hardwareUsed,
+            });
+            return vlmResult;
+        }
+        finally {
+            bridge.free(promptPtr);
+            m._free(imagePtr);
+            m._free(optPtr);
+            if (filePathPtr)
+                bridge.free(filePathPtr);
+            if (base64Ptr)
+                bridge.free(base64Ptr);
+            if (pixelPtr)
+                m._free(pixelPtr);
+            if (sysPtr)
+                bridge.free(sysPtr);
+        }
+    }
+    /** Cancel in-progress VLM generation. */
+    cancel() {
+        if (this._vlmComponentHandle === 0)
+            return;
+        WASMBridge.shared.module.ccall('rac_vlm_component_cancel', 'number', ['number'], [this._vlmComponentHandle]);
+    }
+    /** Clean up the VLM component and unregister backend. */
+    cleanup() {
+        if (this._vlmComponentHandle !== 0) {
+            try {
+                WASMBridge.shared.module.ccall('rac_vlm_component_destroy', null, ['number'], [this._vlmComponentHandle]);
+            }
+            catch { /* ignore */ }
+            this._vlmComponentHandle = 0;
+        }
+        if (this._vlmBackendRegistered) {
+            try {
+                // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                const fn = WASMBridge.shared.module['_rac_backend_llamacpp_vlm_unregister'];
+                if (fn) {
+                    WASMBridge.shared.module.ccall('rac_backend_llamacpp_vlm_unregister', 'number', [], []);
+                }
+            }
+            catch { /* ignore */ }
+            this._vlmBackendRegistered = false;
+        }
+    }
+}
+export const VLM = new VLMImpl();
+//# sourceMappingURL=RunAnywhere+VLM.js.map

package/dist/Public/Extensions/RunAnywhere+VLM.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"RunAnywhere+VLM.js","sourceRoot":"","sources":["../../../src/Public/Extensions/RunAnywhere+VLM.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,OAAO,EAAE,UAAU,EAAE,MAAM,6BAA6B,CAAC;AACzD,OAAO,EAAE,OAAO,EAAE,MAAM,gCAAgC,CAAC;AACzD,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,6BAA6B,CAAC;AACrE,OAAO,EAAE,SAAS,EAAE,MAAM,4BAA4B,CAAC;AACvD,OAAO,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAC;AACrD,OAAO,EAAE,YAAY,EAAwB,MAAM,mBAAmB,CAAC;AACvE,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAG5D,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC,KAAK,CAAC,CAAC;AAGpC,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAE5D,8EAA8E;AAC9E,gBAAgB;AAChB,8EAA8E;AAE9E,MAAM,OAAO;IACF,aAAa,GAAG,KAAK,CAAC;IACvB,mBAAmB,GAAG,CAAC,CAAC;IACxB,qBAAqB,GAAG,KAAK,CAAC;IAE9B,aAAa;QACnB,IAAI,CAAC,WAAW,CAAC,aAAa;YAAE,MAAM,QAAQ,CAAC,cAAc,EAAE,CAAC;QAChE,OAAO,UAAU,CAAC,MAAM,CAAC;IAC3B,CAAC;IAED;;;OAGG;IACK,0BAA0B;QAChC,IAAI,IAAI,CAAC,qBAAqB;YAAE,OAAO;QAEvC,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QACpC,MAAM,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;QAExB,iFAAiF;QACjF,8DAA8D;QAC9D,MAAM,EAAE,GAAI,CAAS,CAAC,oCAAoC,CAAC,CAAC;QAC5D,IAAI,CAAC,EAAE,EAAE,CAAC;YACR,MAAM,IAAI,QAAQ,CAChB,YAAY,CAAC,mBAAmB,EAChC,0DAA0D,CAC3D,CAAC;QACJ,CAAC;QAED,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,mCAAmC,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,CAAW,CAAC;QACxF,IAAI,MAAM,KAAK,CAAC,EAAE,CAAC;YACjB,MAAM,CAAC,WAAW,CAAC,MAAM,EAAE,mCAAmC,CAAC,CAAC;QAClE,CAAC;QAED,IAAI,CAAC,qBAAqB,GAAG,IAAI,CAAC;QAClC,MAAM,CAAC,IAAI,CAAC,yCAAyC,CAAC,CAAC;IACzD,CAAC;IAEO,kBAAkB;QACxB,IAAI,IAAI,CAAC,mBAAmB,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC,mBAAmB,CAAC;QAEpE,iCAAiC;QACjC,IAAI,CAAC,0BAA0B,EAAE,CAAC;QAElC,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QACpC,MAAM,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;QACxB,MAAM,SAAS,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAC/B,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CAAC,0BAA0B,EAAE,QAAQ,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,SAAS,CAAC,CAAW,CAAC;QAEhG,IAAI,MAAM,KAAK,CAAC,EAAE,CAAC;YACjB,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;YACnB,MAAM,CAAC,WAAW,CAAC,MAAM,EAAE,0BAA0B,CAAC,CAAC;QACzD,CAAC;QAED,IAAI,CAAC,mBAAmB,GAAG,CAAC,CAAC,QAAQ,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;QACxD,CAAC,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QACnB,MAAM,CAAC,KAAK,CAAC,uBAAuB,CAAC,CAAC;QACtC,OAAO,IAAI,CAAC,mBAAmB,CAAC;IAClC,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,SAAS,CACb,SAAiB,EACjB,UAAkB,EAClB,OAAe,EACf,SAAkB;QAElB,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QACpC,MAAM,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;QACxB,MAAM,MAAM,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAEzC,MAAM,CAAC,IAAI,CAAC,sBAAsB,OAAO,EAAE,CAAC,CAAC;QAC7C,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,mBAAmB,EAAE,YAAY,CAAC,KAAK,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;QAE7F,MAAM,OAAO,GAAG,MAAM,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;QAC9C,MAAM,OAAO,GAAG,MAAM,CAAC,WAAW,CAAC,UAAU,CAAC,CAAC;QAC/C,MAAM,KAAK,GAAG,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;QAC1C,MAAM,OAAO,GAAG,MAAM,CAAC,WAAW,CAAC,SAAS,IAAI,OAAO,CAAC,CAAC;QAEzD,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,CAAC,CAAC,KAAK,CACpB,8BAA8B,EAAE,QAAQ,EACxC,CAAC,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,CAAC,EAClD,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC,CACjC,CAAC;YACZ,MAAM,CAAC,WAAW,CAAC,MAAM,EAAE,8BAA8B,CAAC,CAAC;YAC3D,MAAM,CAAC,IAAI,CAAC,qBAAqB,OAAO,EAAE,CAAC,CAAC;YAC5C,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,qBAAqB,EAAE,YAAY,CAAC,KAAK,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;QACjG,CAAC;gBAAS,CAAC;YACT,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACrB,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACrB,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACnB,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACvB,CAAC;IACH,CAAC;IAED,4BAA4B;IAC5B,KAAK,CAAC,WAAW;QACf,IAAI,IAAI,CAAC,mBAAmB,KAAK,CAAC;YAAE,OAAO;QAC3C,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QACpC,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAChC,0BAA0B,EAAE,QAAQ,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC,mBAAmB,CAAC,CACnE,CAAC;QACZ,MAAM,CAAC,WAAW,CAAC,MAAM,EAAE,0BAA0B,CAAC,CAAC;QACvD,MAAM,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;IACpC,CAAC;IAED,sCAAsC;IACtC,IAAI,aAAa;QACf,IAAI,IAAI,CAAC,mBAAmB,KAAK,CAAC;YAAE,OAAO,KAAK,CAAC;QACjD,IAAI,CAAC;YACH,OAAQ,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CACpC,6BAA6B,EAAE,QAAQ,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC,mBAAmB,CAAC,CACrE,KAAK,CAAC,CAAC;QACrB,CAAC;QAAC,MAAM,CAAC;YAAC,OAAO,KAAK,CAAC;QAAC,CAAC;IAC3B,CAAC;IAED;;;;;;;OAOG;IACH,KAAK,CAAC,OAAO,CACX,KAAe,EACf,MAAc,EACd,UAAgC,EAAE;QAElC,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QACpC,MAAM,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;QACxB,MAAM,MAAM,GAAG,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAEzC,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,CAAC;YACxB,MAAM,IAAI,QAAQ,CAAC,YAAY,CAAC,cAAc,EAAE,8CAA8C,CAAC,CAAC;QAClG,CAAC;QAED,MAAM,CAAC,KAAK,CAAC,iBAAiB,MAAM,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC;QAE7D,+BAA+B;QAC/B,MAAM,SAAS,GAAG,CAAC,CAAC,0BAA0B,EAAE,CAAC;QACjD,MAAM,QAAQ,GAAG,CAAC,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;QACtC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,EAAE,CAAC,EAAE;YAAE,CAAC,CAAC,QAAQ,CAAC,QAAQ,GAAG,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC;QAEtE,IAAI,WAAW,GAAG,CAAC,CAAC;QACpB,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,IAAI,QAAQ,GAAG,CAAC,CAAC;QAEjB,MAAM,EAAE,GAAG,OAAO,CAAC,QAAQ,CAAC;QAC5B,CAAC,CAAC,QAAQ,CAAC,QAAQ,GAAG,EAAE,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;QAEtD,IAAI,KAAK,CAAC,MAAM,KAAK,cAAc,CAAC,QAAQ,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;YAC/D,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;YACjD,CAAC,CAAC,QAAQ,CAAC,QAAQ,GAAG,EAAE,CAAC,QAAQ,EAAE,WAAW,EAAE,GAAG,CAAC,CAAC;QACvD,CAAC;aAAM,IAAI,KAAK,CAAC,MAAM,KAAK,cAAc,CAAC,MAAM,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;YACtE,SAAS,GAAG,MAAM,CAAC,WAAW,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YACjD,CAAC,CAAC,QAAQ,CAAC,QAAQ,GAAG,EAAE,CAAC,UAAU,EAAE,SAAS,EAAE,GAAG,CAAC,CAAC;QACvD,CAAC;aAAM,IAAI,KAAK,CAAC,MAAM,KAAK,cAAc,CAAC,SAAS,IAAI,KAAK,CAAC,SAAS,EAAE,CAAC;YACxE,QAAQ,GAAG,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;YAC7C,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;YAC7C,CAAC,CAAC,QAAQ,CAAC,QAAQ,GAAG,EAAE,CAAC,SAAS,EAAE,QAAQ,EAAE,GAAG,CAAC,CAAC;QACrD,CAAC;QAED,CAAC,CAAC,QAAQ,CAAC,QAAQ,GAAG,EAAE,CAAC,KAAK,EAAE,KAAK,CAAC,KAAK,IAAI,CAAC,EAAE,KAAK,CAAC,CAAC;QACzD,CAAC,CAAC,QAAQ,CAAC,QAAQ,GAAG,EAAE,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,KAAK,CAAC,CAAC;QAE3D,4EAA4E;QAC5E,MAAM,QAAQ,GAAG,KAAK,CAAC,SAAS,EAAE,MAAM,IAAI,KAAK,CAAC,UAAU,EAAE,MAAM,IAAI,CAAC,CAAC;QAC1E,CAAC,CAAC,QAAQ,CAAC,QAAQ,GAAG,EAAE,CAAC,QAAQ,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;QAEpD,0BAA0B;QAC1B,MAAM,OAAO,GAAG,CAAC,CAAC,4BAA4B,EAAE,CAAC;QACjD,MAAM,MAAM,GAAG,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAClC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,EAAE,CAAC,EAAE;YAAE,CAAC,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC;QAElE,MAAM,EAAE,GAAG,OAAO,CAAC,UAAU,CAAC;QAC9B,CAAC,CAAC,QAAQ,CAAC,MAAM,GAAG,EAAE,CAAC,SAAS,EAAE,OAAO,CAAC,SAAS,IAAI,GAAG,EAAE,KAAK,CAAC,CAAC;QACnE,CAAC,CAAC,QAAQ,CAAC,MAAM,GAAG,EAAE,CAAC,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,GAAG,EAAE,OAAO,CAAC,CAAC;QACzE,CAAC,CAAC,QAAQ,CAAC,MAAM,GAAG,EAAE,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,GAAG,EAAE,OAAO,CAAC,CAAC;QAC3D,CAAC,CAAC,QAAQ,CAAC,MAAM,GAAG,EAAE,CAAC,gBAAgB,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;QAE3E,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;YACzB,MAAM,GAAG,MAAM,CAAC,WAAW,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;YAClD,CAAC,CAAC,QAAQ,CAAC,MAAM,GAAG,EAAE,CAAC,YAAY,EAAE,MAAM,EAAE,GAAG,CAAC,CAAC;QACpD,CAAC;QAED,CAAC,CAAC,QAAQ,CAAC,MAAM,GAAG,EAAE,CAAC,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,cAAc,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAEvF,MAAM,SAAS,GAAG,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;QAE7C,gBAAgB;QAChB,MAAM,OAAO,GAAG,CAAC,CAAC,2BAA2B,EAAE,CAAC;QAChD,MAAM,MAAM,GAAG,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAElC,IAAI,CAAC;YACH,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,CACf,2BAA2B,EAAE,QAAQ,EACrC,CAAC,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,CAAC,EAClD,CAAC,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,CACpC,CAAC;YACZ,MAAM,CAAC,WAAW,CAAC,CAAC,EAAE,2BAA2B,CAAC,CAAC;YAEnD,kEAAkE;YAClE,MAAM,EAAE,GAAG,OAAO,CAAC,SAAS,CAAC;YAC7B,MAAM,OAAO,GAAG,CAAC,CAAC,QAAQ,CAAC,MAAM,GAAG,EAAE,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YAClD,MAAM,SAAS,GAAwB;gBACrC,IAAI,EAAE,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC;gBAChC,YAAY,EAAE,CAAC,CAAC,QAAQ,CAAC,MAAM,GAAG,EAAE,CAAC,YAAY,EAAE,KAAK,CAAC;gBACzD,WAAW,EAAE,CAAC,CAAC,QAAQ,CAAC,MAAM,GAAG,EAAE,CAAC,WAAW,EAAE,KAAK,CAAC;gBACvD,gBAAgB,EAAE,CAAC,CAAC,QAAQ,CAAC,MAAM,GAAG,EAAE,CAAC,gBAAgB,EAAE,KAAK,CAAC;gBACjE,WAAW,EAAE,CAAC,CAAC,QAAQ,CAAC,MAAM,GAAG,EAAE,CAAC,WAAW,EAAE,KAAK,CAAC;gBACvD,kBAAkB,EAAE,CAAC,CAAC,QAAQ,CAAC,MAAM,GAAG,EAAE,CAAC,kBAAkB,EAAE,KAAK,CAAC;gBACrE,iBAAiB,EAAE,CAAC,CAAC,QAAQ,CAAC,MAAM,GAAG,EAAE,CAAC,iBAAiB,EAAE,KAAK,CAAC;gBACnE,WAAW,EAAE,CAAC,CAAC,QAAQ,CAAC,MAAM,GAAG,EAAE,CAAC,WAAW,EAAE,KAAK,CAAC;gBACvD,eAAe,EAAE,CAAC,CAAC,QAAQ,CAAC,MAAM,GAAG,EAAE,CAAC,eAAe,EAAE,OAAO,CAAC;gBACjE,YAAY,EAAE,MAAM,CAAC,gBAAwC;aAC9D,CAAC;YAEF,CAAC,CAAC,KAAK,CAAC,qBAAqB,EAAE,IAAI,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC;YAE3D,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,eAAe,EAAE,YAAY,CAAC,UAAU,EAAE;gBAC7D,eAAe,EAAE,SAAS,CAAC,eAAe;gBAC1C,WAAW,EAAE,SAAS,CAAC,WAAW;gBAClC,YAAY,EAAE,SAAS,CAAC,YAAY;aACrC,CAAC,CAAC;YAEH,OAAO,SAAS,CAAC;QACnB,CAAC;gBAAS,CAAC;YACT,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACvB,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;YAClB,CAAC,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YAChB,IAAI,WAAW;gBAAE,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YAC1C,IAAI,SAAS;gBAAE,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACtC,IAAI,QAAQ;gBAAE,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;YAChC,IAAI,MAAM;gBAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAClC,CAAC;IACH,CAAC;IAED,yCAAyC;IACzC,MAAM;QACJ,IAAI,IAAI,CAAC,mBAAmB,KAAK,CAAC;YAAE,OAAO;QAC3C,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAC5B,0BAA0B,EAAE,QAAQ,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAC7E,CAAC;IACJ,CAAC;IAED,yDAAyD;IACzD,OAAO;QACL,IAAI,IAAI,CAAC,mBAAmB,KAAK,CAAC,EAAE,CAAC;YACnC,IAAI,CAAC;gBACH,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAC5B,2BAA2B,EAAE,IAAI,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAC1E,CAAC;YACJ,CAAC;YAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;YACxB,IAAI,CAAC,mBAAmB,GAAG,CAAC,CAAC;QAC/B,CAAC;QAED,IAAI,IAAI,CAAC,qBAAqB,EAAE,CAAC;YAC/B,IAAI,CAAC;gBACH,8DAA8D;gBAC9D,MAAM,EAAE,GAAI,UAAU,CAAC,MAAM,CAAC,MAAc,CAAC,sCAAsC,CAAC,CAAC;gBACrF,IAAI,EAAE,EAAE,CAAC;oBACP,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,qCAAqC,EAAE,QAAQ,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;gBAC1F,CAAC;YACH,CAAC;YAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC;YACxB,IAAI,CAAC,qBAAqB,GAAG,KAAK,CAAC;QACrC,CAAC;IACH,CAAC;CACF;AAED,MAAM,CAAC,MAAM,GAAG,GAAG,IAAI,OAAO,EAAE,CAAC"}

package/dist/Public/Extensions/RunAnywhere+VoiceAgent.d.ts

@@ -0,0 +1,49 @@
+/**
+ * RunAnywhere Web SDK - VoiceAgent Extension
+ *
+ * Orchestrates the complete voice pipeline: VAD -> STT -> LLM -> TTS.
+ * Uses the RACommons rac_voice_agent_* C API for pipeline management.
+ *
+ * Mirrors: sdk/runanywhere-swift/Sources/RunAnywhere/Public/Extensions/VoiceAgent/
+ *
+ * Usage:
+ *   import { VoiceAgent } from '@runanywhere/web';
+ *
+ *   const agent = await VoiceAgent.create();
+ *   await agent.loadModels({ stt: '/models/whisper.bin', llm: '/models/llama.gguf', tts: '/models/piper.onnx' });
+ *   const result = await agent.processVoiceTurn(audioData);
+ *   console.log('Transcription:', result.transcription);
+ *   console.log('Response:', result.response);
+ */
+import type { VoiceAgentModels, VoiceTurnResult } from './VoiceAgentTypes';
+export { PipelineState } from './VoiceAgentTypes';
+export type { VoiceAgentModels, VoiceTurnResult, VoiceAgentEventData, VoiceAgentEventCallback } from './VoiceAgentTypes';
+export declare class VoiceAgentSession {
+    private handle;
+    private eventCallbackPtr;
+    constructor(handle: number);
+    /**
+     * Load models for all components.
+     */
+    loadModels(models: VoiceAgentModels): Promise<void>;
+    /**
+     * Process a complete voice turn (audio in -> text response + audio out).
+     */
+    processVoiceTurn(audioData: Uint8Array): Promise<VoiceTurnResult>;
+    /** Check if the voice agent is ready. */
+    get isReady(): boolean;
+    /** Transcribe audio without the full pipeline. */
+    transcribe(audioData: Uint8Array): Promise<string>;
+    /** Generate LLM response without the full pipeline. */
+    generateResponse(prompt: string): Promise<string>;
+    /** Destroy the voice agent session. */
+    destroy(): void;
+}
+export declare const VoiceAgent: {
+    /**
+     * Create a standalone VoiceAgent session.
+     * The agent manages its own STT, LLM, TTS, and VAD components.
+     */
+    create(): Promise<VoiceAgentSession>;
+};
+//# sourceMappingURL=RunAnywhere+VoiceAgent.d.ts.map

package/dist/Public/Extensions/RunAnywhere+VoiceAgent.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"RunAnywhere+VoiceAgent.d.ts","sourceRoot":"","sources":["../../../src/Public/Extensions/RunAnywhere+VoiceAgent.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAQH,OAAO,KAAK,EAAE,gBAAgB,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAE3E,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAClD,YAAY,EAAE,gBAAgB,EAAE,eAAe,EAAE,mBAAmB,EAAE,uBAAuB,EAAE,MAAM,mBAAmB,CAAC;AAQzH,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,gBAAgB,CAAK;gBAEjB,MAAM,EAAE,MAAM;IAI1B;;OAEG;IACG,UAAU,CAAC,MAAM,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC;IAoDzD;;OAEG;IACG,gBAAgB,CAAC,SAAS,EAAE,UAAU,GAAG,OAAO,CAAC,eAAe,CAAC;IAgDvE,yCAAyC;IACzC,IAAI,OAAO,IAAI,OAAO,CAUrB;IAED,kDAAkD;IAC5C,UAAU,CAAC,SAAS,EAAE,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC;IAsBxD,uDAAuD;IACjD,gBAAgB,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IAqBvD,uCAAuC;IACvC,OAAO,IAAI,IAAI;CAYhB;AAMD,eAAO,MAAM,UAAU;IACrB;;;OAGG;cACa,OAAO,CAAC,iBAAiB,CAAC;CAuB3C,CAAC"}