@runanywhere/web 0.1.0-beta.6 → 0.1.0-beta.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/Foundation/ErrorTypes.d.ts +1 -0
- package/dist/Foundation/ErrorTypes.d.ts.map +1 -1
- package/dist/Foundation/ErrorTypes.js +3 -0
- package/dist/Foundation/ErrorTypes.js.map +1 -1
- package/dist/Foundation/EventBus.d.ts +0 -1
- package/dist/Foundation/EventBus.d.ts.map +1 -1
- package/dist/Foundation/StructOffsets.d.ts +5 -37
- package/dist/Foundation/StructOffsets.d.ts.map +1 -1
- package/dist/Foundation/StructOffsets.js +6 -157
- package/dist/Foundation/StructOffsets.js.map +1 -1
- package/dist/Foundation/WASMBridge.d.ts +8 -236
- package/dist/Foundation/WASMBridge.d.ts.map +1 -1
- package/dist/Foundation/WASMBridge.js +7 -388
- package/dist/Foundation/WASMBridge.js.map +1 -1
- package/dist/Infrastructure/DeviceCapabilities.d.ts.map +1 -1
- package/dist/Infrastructure/DeviceCapabilities.js +1 -3
- package/dist/Infrastructure/DeviceCapabilities.js.map +1 -1
- package/dist/Infrastructure/ExtensionPoint.d.ts +114 -0
- package/dist/Infrastructure/ExtensionPoint.d.ts.map +1 -0
- package/dist/Infrastructure/ExtensionPoint.js +178 -0
- package/dist/Infrastructure/ExtensionPoint.js.map +1 -0
- package/dist/Infrastructure/LocalFileStorage.d.ts +134 -0
- package/dist/Infrastructure/LocalFileStorage.d.ts.map +1 -0
- package/dist/Infrastructure/LocalFileStorage.js +428 -0
- package/dist/Infrastructure/LocalFileStorage.js.map +1 -0
- package/dist/Infrastructure/ModelDownloader.d.ts +21 -5
- package/dist/Infrastructure/ModelDownloader.d.ts.map +1 -1
- package/dist/Infrastructure/ModelDownloader.js +79 -7
- package/dist/Infrastructure/ModelDownloader.js.map +1 -1
- package/dist/Infrastructure/ModelFileInference.d.ts +39 -0
- package/dist/Infrastructure/ModelFileInference.d.ts.map +1 -0
- package/dist/Infrastructure/ModelFileInference.js +119 -0
- package/dist/Infrastructure/ModelFileInference.js.map +1 -0
- package/dist/Infrastructure/ModelLoaderTypes.d.ts +91 -12
- package/dist/Infrastructure/ModelLoaderTypes.d.ts.map +1 -1
- package/dist/Infrastructure/ModelLoaderTypes.js +7 -1
- package/dist/Infrastructure/ModelLoaderTypes.js.map +1 -1
- package/dist/Infrastructure/ModelManager.d.ts +31 -104
- package/dist/Infrastructure/ModelManager.d.ts.map +1 -1
- package/dist/Infrastructure/ModelManager.js +207 -568
- package/dist/Infrastructure/ModelManager.js.map +1 -1
- package/dist/Infrastructure/ModelRegistry.d.ts +6 -8
- package/dist/Infrastructure/ModelRegistry.d.ts.map +1 -1
- package/dist/Infrastructure/ModelRegistry.js +11 -4
- package/dist/Infrastructure/ModelRegistry.js.map +1 -1
- package/dist/Infrastructure/OPFSStorage.d.ts +8 -0
- package/dist/Infrastructure/OPFSStorage.d.ts.map +1 -1
- package/dist/Infrastructure/OPFSStorage.js +37 -0
- package/dist/Infrastructure/OPFSStorage.js.map +1 -1
- package/dist/Public/Extensions/RunAnywhere+ModelManagement.d.ts +12 -4
- package/dist/Public/Extensions/RunAnywhere+ModelManagement.d.ts.map +1 -1
- package/dist/Public/Extensions/RunAnywhere+ModelManagement.js +23 -51
- package/dist/Public/Extensions/RunAnywhere+ModelManagement.js.map +1 -1
- package/dist/Public/Extensions/RunAnywhere+VoiceAgent.d.ts +42 -10
- package/dist/Public/Extensions/RunAnywhere+VoiceAgent.d.ts.map +1 -1
- package/dist/Public/Extensions/RunAnywhere+VoiceAgent.js +63 -161
- package/dist/Public/Extensions/RunAnywhere+VoiceAgent.js.map +1 -1
- package/dist/Public/Extensions/RunAnywhere+VoicePipeline.d.ts +3 -29
- package/dist/Public/Extensions/RunAnywhere+VoicePipeline.d.ts.map +1 -1
- package/dist/Public/Extensions/RunAnywhere+VoicePipeline.js +26 -42
- package/dist/Public/Extensions/RunAnywhere+VoicePipeline.js.map +1 -1
- package/dist/Public/Extensions/VoicePipelineTypes.d.ts +28 -37
- package/dist/Public/Extensions/VoicePipelineTypes.d.ts.map +1 -1
- package/dist/Public/Extensions/VoicePipelineTypes.js +4 -1
- package/dist/Public/Extensions/VoicePipelineTypes.js.map +1 -1
- package/dist/Public/RunAnywhere.d.ts +29 -85
- package/dist/Public/RunAnywhere.d.ts.map +1 -1
- package/dist/Public/RunAnywhere.js +169 -211
- package/dist/Public/RunAnywhere.js.map +1 -1
- package/dist/index.d.ts +19 -39
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +15 -31
- package/dist/index.js.map +1 -1
- package/dist/types/index.d.ts +0 -1
- package/dist/types/index.d.ts.map +1 -1
- package/dist/types/index.js +1 -0
- package/dist/types/index.js.map +1 -1
- package/package.json +4 -10
- package/dist/Foundation/PlatformAdapter.d.ts +0 -101
- package/dist/Foundation/PlatformAdapter.d.ts.map +0 -1
- package/dist/Foundation/PlatformAdapter.js +0 -417
- package/dist/Foundation/PlatformAdapter.js.map +0 -1
- package/dist/Foundation/SherpaONNXBridge.d.ts +0 -147
- package/dist/Foundation/SherpaONNXBridge.d.ts.map +0 -1
- package/dist/Foundation/SherpaONNXBridge.js +0 -345
- package/dist/Foundation/SherpaONNXBridge.js.map +0 -1
- package/dist/Infrastructure/AudioCapture.d.ts +0 -99
- package/dist/Infrastructure/AudioCapture.d.ts.map +0 -1
- package/dist/Infrastructure/AudioCapture.js +0 -264
- package/dist/Infrastructure/AudioCapture.js.map +0 -1
- package/dist/Infrastructure/AudioPlayback.d.ts +0 -53
- package/dist/Infrastructure/AudioPlayback.d.ts.map +0 -1
- package/dist/Infrastructure/AudioPlayback.js +0 -117
- package/dist/Infrastructure/AudioPlayback.js.map +0 -1
- package/dist/Infrastructure/VLMWorkerBridge.d.ts +0 -211
- package/dist/Infrastructure/VLMWorkerBridge.d.ts.map +0 -1
- package/dist/Infrastructure/VLMWorkerBridge.js +0 -264
- package/dist/Infrastructure/VLMWorkerBridge.js.map +0 -1
- package/dist/Infrastructure/VLMWorkerRuntime.d.ts +0 -38
- package/dist/Infrastructure/VLMWorkerRuntime.d.ts.map +0 -1
- package/dist/Infrastructure/VLMWorkerRuntime.js +0 -503
- package/dist/Infrastructure/VLMWorkerRuntime.js.map +0 -1
- package/dist/Infrastructure/VideoCapture.d.ts +0 -130
- package/dist/Infrastructure/VideoCapture.d.ts.map +0 -1
- package/dist/Infrastructure/VideoCapture.js +0 -236
- package/dist/Infrastructure/VideoCapture.js.map +0 -1
- package/dist/Public/Extensions/DiffusionTypes.d.ts +0 -64
- package/dist/Public/Extensions/DiffusionTypes.d.ts.map +0 -1
- package/dist/Public/Extensions/DiffusionTypes.js +0 -28
- package/dist/Public/Extensions/DiffusionTypes.js.map +0 -1
- package/dist/Public/Extensions/EmbeddingsTypes.d.ts +0 -33
- package/dist/Public/Extensions/EmbeddingsTypes.d.ts.map +0 -1
- package/dist/Public/Extensions/EmbeddingsTypes.js +0 -13
- package/dist/Public/Extensions/EmbeddingsTypes.js.map +0 -1
- package/dist/Public/Extensions/RunAnywhere+Diffusion.d.ts +0 -44
- package/dist/Public/Extensions/RunAnywhere+Diffusion.d.ts.map +0 -1
- package/dist/Public/Extensions/RunAnywhere+Diffusion.js +0 -189
- package/dist/Public/Extensions/RunAnywhere+Diffusion.js.map +0 -1
- package/dist/Public/Extensions/RunAnywhere+Embeddings.d.ts +0 -56
- package/dist/Public/Extensions/RunAnywhere+Embeddings.d.ts.map +0 -1
- package/dist/Public/Extensions/RunAnywhere+Embeddings.js +0 -240
- package/dist/Public/Extensions/RunAnywhere+Embeddings.js.map +0 -1
- package/dist/Public/Extensions/RunAnywhere+STT.d.ts +0 -97
- package/dist/Public/Extensions/RunAnywhere+STT.d.ts.map +0 -1
- package/dist/Public/Extensions/RunAnywhere+STT.js +0 -417
- package/dist/Public/Extensions/RunAnywhere+STT.js.map +0 -1
- package/dist/Public/Extensions/RunAnywhere+StructuredOutput.d.ts +0 -69
- package/dist/Public/Extensions/RunAnywhere+StructuredOutput.d.ts.map +0 -1
- package/dist/Public/Extensions/RunAnywhere+StructuredOutput.js +0 -196
- package/dist/Public/Extensions/RunAnywhere+StructuredOutput.js.map +0 -1
- package/dist/Public/Extensions/RunAnywhere+TTS.d.ts +0 -55
- package/dist/Public/Extensions/RunAnywhere+TTS.d.ts.map +0 -1
- package/dist/Public/Extensions/RunAnywhere+TTS.js +0 -253
- package/dist/Public/Extensions/RunAnywhere+TTS.js.map +0 -1
- package/dist/Public/Extensions/RunAnywhere+TextGeneration.d.ts +0 -80
- package/dist/Public/Extensions/RunAnywhere+TextGeneration.d.ts.map +0 -1
- package/dist/Public/Extensions/RunAnywhere+TextGeneration.js +0 -470
- package/dist/Public/Extensions/RunAnywhere+TextGeneration.js.map +0 -1
- package/dist/Public/Extensions/RunAnywhere+ToolCalling.d.ts +0 -82
- package/dist/Public/Extensions/RunAnywhere+ToolCalling.d.ts.map +0 -1
- package/dist/Public/Extensions/RunAnywhere+ToolCalling.js +0 -576
- package/dist/Public/Extensions/RunAnywhere+ToolCalling.js.map +0 -1
- package/dist/Public/Extensions/RunAnywhere+VAD.d.ts +0 -70
- package/dist/Public/Extensions/RunAnywhere+VAD.d.ts.map +0 -1
- package/dist/Public/Extensions/RunAnywhere+VAD.js +0 -231
- package/dist/Public/Extensions/RunAnywhere+VAD.js.map +0 -1
- package/dist/Public/Extensions/RunAnywhere+VLM.d.ts +0 -58
- package/dist/Public/Extensions/RunAnywhere+VLM.d.ts.map +0 -1
- package/dist/Public/Extensions/RunAnywhere+VLM.js +0 -262
- package/dist/Public/Extensions/RunAnywhere+VLM.js.map +0 -1
- package/dist/Public/Extensions/STTTypes.d.ts +0 -61
- package/dist/Public/Extensions/STTTypes.d.ts.map +0 -1
- package/dist/Public/Extensions/STTTypes.js +0 -16
- package/dist/Public/Extensions/STTTypes.js.map +0 -1
- package/dist/Public/Extensions/TTSTypes.d.ts +0 -31
- package/dist/Public/Extensions/TTSTypes.d.ts.map +0 -1
- package/dist/Public/Extensions/TTSTypes.js +0 -3
- package/dist/Public/Extensions/TTSTypes.js.map +0 -1
- package/dist/Public/Extensions/ToolCallingTypes.d.ts +0 -78
- package/dist/Public/Extensions/ToolCallingTypes.d.ts.map +0 -1
- package/dist/Public/Extensions/ToolCallingTypes.js +0 -8
- package/dist/Public/Extensions/ToolCallingTypes.js.map +0 -1
- package/dist/Public/Extensions/VADTypes.d.ts +0 -30
- package/dist/Public/Extensions/VADTypes.d.ts.map +0 -1
- package/dist/Public/Extensions/VADTypes.js +0 -8
- package/dist/Public/Extensions/VADTypes.js.map +0 -1
- package/dist/Public/Extensions/VLMTypes.d.ts +0 -56
- package/dist/Public/Extensions/VLMTypes.d.ts.map +0 -1
- package/dist/Public/Extensions/VLMTypes.js +0 -24
- package/dist/Public/Extensions/VLMTypes.js.map +0 -1
- package/dist/types/LLMTypes.d.ts +0 -48
- package/dist/types/LLMTypes.d.ts.map +0 -1
- package/dist/types/LLMTypes.js +0 -8
- package/dist/types/LLMTypes.js.map +0 -1
- package/dist/workers/vlm-worker.d.ts +0 -9
- package/dist/workers/vlm-worker.d.ts.map +0 -1
- package/dist/workers/vlm-worker.js +0 -10
- package/dist/workers/vlm-worker.js.map +0 -1
- package/wasm/racommons-webgpu.js +0 -156
- package/wasm/racommons-webgpu.wasm +0 -0
- package/wasm/racommons.js +0 -126
- package/wasm/racommons.wasm +0 -0
- package/wasm/sherpa/sherpa-onnx-asr.js +0 -1538
- package/wasm/sherpa/sherpa-onnx-glue-original.js +0 -19
- package/wasm/sherpa/sherpa-onnx-glue.js +0 -17
- package/wasm/sherpa/sherpa-onnx-tts.js +0 -657
- package/wasm/sherpa/sherpa-onnx-vad.js +0 -337
- package/wasm/sherpa/sherpa-onnx-wave.js +0 -88
- package/wasm/sherpa/sherpa-onnx.wasm +0 -0
|
@@ -1,264 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* RunAnywhere Web SDK - Audio Capture
|
|
3
|
-
*
|
|
4
|
-
* Captures microphone audio using Web Audio API and provides
|
|
5
|
-
* Float32Array PCM samples suitable for STT and VAD processing.
|
|
6
|
-
*
|
|
7
|
-
* Supports:
|
|
8
|
-
* - Real-time microphone capture via getUserMedia
|
|
9
|
-
* - Configurable sample rate (resampling via AudioContext)
|
|
10
|
-
* - Chunk-based callbacks for streaming STT/VAD
|
|
11
|
-
* - Buffer accumulation (getAudioBuffer, drainBuffer, clearBuffer)
|
|
12
|
-
* - Audio level monitoring via AnalyserNode (currentLevel getter)
|
|
13
|
-
*/
|
|
14
|
-
import { SDKLogger } from '../Foundation/SDKLogger';
|
|
15
|
-
const logger = new SDKLogger('AudioCapture');
|
|
16
|
-
/**
|
|
17
|
-
* AudioCapture - Captures microphone audio for STT/VAD processing.
|
|
18
|
-
*
|
|
19
|
-
* Uses Web Audio API (AudioContext + ScriptProcessorNode) to capture
|
|
20
|
-
* microphone audio, resample to target rate, and deliver Float32Array
|
|
21
|
-
* PCM chunks to the consumer.
|
|
22
|
-
*
|
|
23
|
-
* Includes buffer accumulation for batch STT and an AnalyserNode for
|
|
24
|
-
* real-time audio level metering (0-1 range).
|
|
25
|
-
*/
|
|
26
|
-
export class AudioCapture {
|
|
27
|
-
audioContext = null;
|
|
28
|
-
mediaStream = null;
|
|
29
|
-
sourceNode = null;
|
|
30
|
-
processorNode = null;
|
|
31
|
-
_analyser = null;
|
|
32
|
-
_animFrameId = null;
|
|
33
|
-
_isCapturing = false;
|
|
34
|
-
_currentLevel = 0;
|
|
35
|
-
_pcmChunks = [];
|
|
36
|
-
config;
|
|
37
|
-
chunkCallback = null;
|
|
38
|
-
levelCallback = null;
|
|
39
|
-
/**
|
|
40
|
-
* Callback invoked when the microphone track ends unexpectedly.
|
|
41
|
-
*
|
|
42
|
-
* On mobile browsers, the OS can revoke mic access due to power management,
|
|
43
|
-
* screen lock, another app requesting the mic, or AudioContext suspension.
|
|
44
|
-
* Set this to detect and recover from such interruptions.
|
|
45
|
-
*/
|
|
46
|
-
onTrackEnded = null;
|
|
47
|
-
/** Bound handler for the track 'ended' event (stored for cleanup). */
|
|
48
|
-
_boundOnTrackEnded = null;
|
|
49
|
-
constructor(config = {}) {
|
|
50
|
-
this.config = {
|
|
51
|
-
sampleRate: config.sampleRate ?? 16000,
|
|
52
|
-
chunkSize: config.chunkSize ?? 1600,
|
|
53
|
-
channels: config.channels ?? 1,
|
|
54
|
-
};
|
|
55
|
-
}
|
|
56
|
-
get isCapturing() {
|
|
57
|
-
return this._isCapturing;
|
|
58
|
-
}
|
|
59
|
-
/** Current normalized audio level (0..1), updated per animation frame. */
|
|
60
|
-
get currentLevel() {
|
|
61
|
-
return this._currentLevel;
|
|
62
|
-
}
|
|
63
|
-
/**
|
|
64
|
-
* Get the actual sample rate of the audio context.
|
|
65
|
-
* May differ from requested rate if browser doesn't support it.
|
|
66
|
-
*/
|
|
67
|
-
get actualSampleRate() {
|
|
68
|
-
return this.audioContext?.sampleRate ?? this.config.sampleRate;
|
|
69
|
-
}
|
|
70
|
-
/** Duration of collected audio in seconds based on configured sample rate. */
|
|
71
|
-
get bufferDurationSeconds() {
|
|
72
|
-
const samples = this._pcmChunks.reduce((acc, c) => acc + c.length, 0);
|
|
73
|
-
return samples / this.config.sampleRate;
|
|
74
|
-
}
|
|
75
|
-
/**
|
|
76
|
-
* Start capturing microphone audio.
|
|
77
|
-
*
|
|
78
|
-
* @param onChunk - Optional callback receiving Float32Array chunks of PCM audio (streaming)
|
|
79
|
-
* @param onLevel - Optional callback invoked per animation frame with audio level 0..1
|
|
80
|
-
* @throws If microphone permission is denied
|
|
81
|
-
*/
|
|
82
|
-
async start(onChunk, onLevel) {
|
|
83
|
-
if (this._isCapturing) {
|
|
84
|
-
logger.debug('Already capturing');
|
|
85
|
-
return;
|
|
86
|
-
}
|
|
87
|
-
this.chunkCallback = onChunk ?? null;
|
|
88
|
-
this.levelCallback = onLevel ?? null;
|
|
89
|
-
this._pcmChunks = [];
|
|
90
|
-
this._currentLevel = 0;
|
|
91
|
-
logger.info(`Starting audio capture (${this.config.sampleRate}Hz, chunk=${this.config.chunkSize})`);
|
|
92
|
-
try {
|
|
93
|
-
// Request microphone access
|
|
94
|
-
this.mediaStream = await navigator.mediaDevices.getUserMedia({
|
|
95
|
-
audio: {
|
|
96
|
-
sampleRate: { ideal: this.config.sampleRate },
|
|
97
|
-
channelCount: { exact: this.config.channels },
|
|
98
|
-
echoCancellation: true,
|
|
99
|
-
noiseSuppression: true,
|
|
100
|
-
autoGainControl: true,
|
|
101
|
-
},
|
|
102
|
-
});
|
|
103
|
-
// Listen for the audio track ending unexpectedly (OS/browser killed it).
|
|
104
|
-
// Mobile browsers kill mic tracks on screen lock, power management,
|
|
105
|
-
// or when another app requests the microphone.
|
|
106
|
-
const audioTrack = this.mediaStream.getAudioTracks()[0];
|
|
107
|
-
if (audioTrack) {
|
|
108
|
-
this._boundOnTrackEnded = () => {
|
|
109
|
-
logger.info('Audio track ended unexpectedly (OS/browser revoked access)');
|
|
110
|
-
const wasCapturing = this._isCapturing;
|
|
111
|
-
this._isCapturing = false;
|
|
112
|
-
this._currentLevel = 0;
|
|
113
|
-
this.cleanupResources();
|
|
114
|
-
if (wasCapturing) {
|
|
115
|
-
this.onTrackEnded?.();
|
|
116
|
-
}
|
|
117
|
-
};
|
|
118
|
-
audioTrack.addEventListener('ended', this._boundOnTrackEnded);
|
|
119
|
-
}
|
|
120
|
-
// Create AudioContext at target sample rate
|
|
121
|
-
this.audioContext = new AudioContext({
|
|
122
|
-
sampleRate: this.config.sampleRate,
|
|
123
|
-
});
|
|
124
|
-
// Connect microphone to AudioContext
|
|
125
|
-
this.sourceNode = this.audioContext.createMediaStreamSource(this.mediaStream);
|
|
126
|
-
// --- AnalyserNode for level metering ---
|
|
127
|
-
this._analyser = this.audioContext.createAnalyser();
|
|
128
|
-
this._analyser.fftSize = 256;
|
|
129
|
-
this.sourceNode.connect(this._analyser);
|
|
130
|
-
// --- ScriptProcessorNode for chunk-based processing ---
|
|
131
|
-
// Note: ScriptProcessorNode is deprecated but AudioWorklet requires
|
|
132
|
-
// cross-origin isolation. We use ScriptProcessor as a fallback.
|
|
133
|
-
const bufferSize = this.nearestPowerOf2(this.config.chunkSize);
|
|
134
|
-
this.processorNode = this.audioContext.createScriptProcessor(bufferSize, this.config.channels, this.config.channels);
|
|
135
|
-
this.processorNode.onaudioprocess = (event) => {
|
|
136
|
-
if (!this._isCapturing)
|
|
137
|
-
return;
|
|
138
|
-
const inputData = event.inputBuffer.getChannelData(0);
|
|
139
|
-
// Copy to avoid buffer reuse issues
|
|
140
|
-
const samples = new Float32Array(inputData.length);
|
|
141
|
-
samples.set(inputData);
|
|
142
|
-
// Accumulate into internal buffer
|
|
143
|
-
this._pcmChunks.push(samples);
|
|
144
|
-
// Notify chunk callback if provided
|
|
145
|
-
this.chunkCallback?.(samples);
|
|
146
|
-
};
|
|
147
|
-
this.sourceNode.connect(this.processorNode);
|
|
148
|
-
this.processorNode.connect(this.audioContext.destination);
|
|
149
|
-
this._isCapturing = true;
|
|
150
|
-
// Start level monitoring loop
|
|
151
|
-
this.startLevelMonitoring();
|
|
152
|
-
logger.info('Audio capture started');
|
|
153
|
-
}
|
|
154
|
-
catch (error) {
|
|
155
|
-
this.cleanupResources();
|
|
156
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
157
|
-
logger.error(`Failed to start audio capture: ${message}`);
|
|
158
|
-
throw new Error(`Microphone access failed: ${message}`);
|
|
159
|
-
}
|
|
160
|
-
}
|
|
161
|
-
/**
|
|
162
|
-
* Stop capturing audio and release resources.
|
|
163
|
-
*/
|
|
164
|
-
stop() {
|
|
165
|
-
if (!this._isCapturing)
|
|
166
|
-
return;
|
|
167
|
-
this._isCapturing = false;
|
|
168
|
-
this._currentLevel = 0;
|
|
169
|
-
this.chunkCallback = null;
|
|
170
|
-
this.levelCallback = null;
|
|
171
|
-
this.cleanupResources();
|
|
172
|
-
logger.info('Audio capture stopped');
|
|
173
|
-
}
|
|
174
|
-
/**
|
|
175
|
-
* Get all collected PCM audio as a single Float32Array.
|
|
176
|
-
* Does NOT clear the buffer — call `clearBuffer()` separately.
|
|
177
|
-
*/
|
|
178
|
-
getAudioBuffer() {
|
|
179
|
-
if (this._pcmChunks.length === 0)
|
|
180
|
-
return new Float32Array(0);
|
|
181
|
-
const totalLength = this._pcmChunks.reduce((acc, c) => acc + c.length, 0);
|
|
182
|
-
const merged = new Float32Array(totalLength);
|
|
183
|
-
let offset = 0;
|
|
184
|
-
for (const chunk of this._pcmChunks) {
|
|
185
|
-
merged.set(chunk, offset);
|
|
186
|
-
offset += chunk.length;
|
|
187
|
-
}
|
|
188
|
-
return merged;
|
|
189
|
-
}
|
|
190
|
-
/**
|
|
191
|
-
* Drain: return the current buffer and clear it for the next segment.
|
|
192
|
-
* Useful for live mode where we transcribe segments incrementally.
|
|
193
|
-
*/
|
|
194
|
-
drainBuffer() {
|
|
195
|
-
const buffer = this.getAudioBuffer();
|
|
196
|
-
this._pcmChunks = [];
|
|
197
|
-
return buffer;
|
|
198
|
-
}
|
|
199
|
-
/** Clear collected PCM data without stopping capture. */
|
|
200
|
-
clearBuffer() {
|
|
201
|
-
this._pcmChunks = [];
|
|
202
|
-
}
|
|
203
|
-
/** Start the requestAnimationFrame loop that reads the AnalyserNode. */
|
|
204
|
-
startLevelMonitoring() {
|
|
205
|
-
const analyser = this._analyser;
|
|
206
|
-
if (!analyser)
|
|
207
|
-
return;
|
|
208
|
-
const dataArray = new Uint8Array(analyser.frequencyBinCount);
|
|
209
|
-
const tick = () => {
|
|
210
|
-
if (!this._isCapturing || !this._analyser)
|
|
211
|
-
return;
|
|
212
|
-
this._analyser.getByteFrequencyData(dataArray);
|
|
213
|
-
let sum = 0;
|
|
214
|
-
for (let i = 0; i < dataArray.length; i++)
|
|
215
|
-
sum += dataArray[i];
|
|
216
|
-
const avg = sum / dataArray.length / 255;
|
|
217
|
-
this._currentLevel = avg;
|
|
218
|
-
this.levelCallback?.(avg);
|
|
219
|
-
this._animFrameId = requestAnimationFrame(tick);
|
|
220
|
-
};
|
|
221
|
-
this._animFrameId = requestAnimationFrame(tick);
|
|
222
|
-
}
|
|
223
|
-
cleanupResources() {
|
|
224
|
-
if (this._animFrameId !== null) {
|
|
225
|
-
cancelAnimationFrame(this._animFrameId);
|
|
226
|
-
this._animFrameId = null;
|
|
227
|
-
}
|
|
228
|
-
if (this.processorNode) {
|
|
229
|
-
this.processorNode.disconnect();
|
|
230
|
-
this.processorNode.onaudioprocess = null;
|
|
231
|
-
this.processorNode = null;
|
|
232
|
-
}
|
|
233
|
-
if (this._analyser) {
|
|
234
|
-
this._analyser.disconnect();
|
|
235
|
-
this._analyser = null;
|
|
236
|
-
}
|
|
237
|
-
if (this.sourceNode) {
|
|
238
|
-
this.sourceNode.disconnect();
|
|
239
|
-
this.sourceNode = null;
|
|
240
|
-
}
|
|
241
|
-
if (this.audioContext) {
|
|
242
|
-
this.audioContext.close().catch(() => { });
|
|
243
|
-
this.audioContext = null;
|
|
244
|
-
}
|
|
245
|
-
if (this.mediaStream) {
|
|
246
|
-
// Remove the track-ended listener before stopping to avoid double-fire
|
|
247
|
-
if (this._boundOnTrackEnded) {
|
|
248
|
-
const audioTrack = this.mediaStream.getAudioTracks()[0];
|
|
249
|
-
audioTrack?.removeEventListener('ended', this._boundOnTrackEnded);
|
|
250
|
-
this._boundOnTrackEnded = null;
|
|
251
|
-
}
|
|
252
|
-
this.mediaStream.getTracks().forEach((track) => track.stop());
|
|
253
|
-
this.mediaStream = null;
|
|
254
|
-
}
|
|
255
|
-
}
|
|
256
|
-
nearestPowerOf2(n) {
|
|
257
|
-
let power = 256;
|
|
258
|
-
while (power < n && power < 16384) {
|
|
259
|
-
power *= 2;
|
|
260
|
-
}
|
|
261
|
-
return power;
|
|
262
|
-
}
|
|
263
|
-
}
|
|
264
|
-
//# sourceMappingURL=AudioCapture.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"AudioCapture.js","sourceRoot":"","sources":["../../src/Infrastructure/AudioCapture.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC;AAEpD,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC,cAAc,CAAC,CAAC;AAgB7C;;;;;;;;;GASG;AACH,MAAM,OAAO,YAAY;IACf,YAAY,GAAwB,IAAI,CAAC;IACzC,WAAW,GAAuB,IAAI,CAAC;IACvC,UAAU,GAAsC,IAAI,CAAC;IACrD,aAAa,GAA+B,IAAI,CAAC;IACjD,SAAS,GAAwB,IAAI,CAAC;IACtC,YAAY,GAAkB,IAAI,CAAC;IACnC,YAAY,GAAG,KAAK,CAAC;IACrB,aAAa,GAAG,CAAC,CAAC;IAClB,UAAU,GAAmB,EAAE,CAAC;IAEvB,MAAM,CAA+B;IAC9C,aAAa,GAA8B,IAAI,CAAC;IAChD,aAAa,GAA8B,IAAI,CAAC;IAExD;;;;;;OAMG;IACH,YAAY,GAAmC,IAAI,CAAC;IAEpD,sEAAsE;IAC9D,kBAAkB,GAAwB,IAAI,CAAC;IAEvD,YAAY,SAA6B,EAAE;QACzC,IAAI,CAAC,MAAM,GAAG;YACZ,UAAU,EAAE,MAAM,CAAC,UAAU,IAAI,KAAK;YACtC,SAAS,EAAE,MAAM,CAAC,SAAS,IAAI,IAAI;YACnC,QAAQ,EAAE,MAAM,CAAC,QAAQ,IAAI,CAAC;SAC/B,CAAC;IACJ,CAAC;IAED,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAED,0EAA0E;IAC1E,IAAI,YAAY;QACd,OAAO,IAAI,CAAC,aAAa,CAAC;IAC5B,CAAC;IAED;;;OAGG;IACH,IAAI,gBAAgB;QAClB,OAAO,IAAI,CAAC,YAAY,EAAE,UAAU,IAAI,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC;IACjE,CAAC;IAED,8EAA8E;IAC9E,IAAI,qBAAqB;QACvB,MAAM,OAAO,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QACtE,OAAO,OAAO,GAAG,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC;IAC1C,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,KAAK,CAAC,OAA4B,EAAE,OAA4B;QACpE,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtB,MAAM,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC;YAClC,OAAO;QACT,CAAC;QAED,IAAI,CAAC,aAAa,GAAG,OAAO,IAAI,IAAI,CAAC;QACrC,IAAI,CAAC,aAAa,GAAG,OAAO,IAAI,IAAI,CAAC;QACrC,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC;QACrB,IAAI,CAAC,aAAa,GAAG,CAAC,CAAC;QAEvB,MAAM,CAAC,IAAI,CAAC,2BAA2B,IAAI,CAAC,MAAM,CAAC,UAAU,aAAa,IAAI,CAAC,MAAM,CAAC,SAAS,GAAG,CAAC,CAAC;QAEpG,IAAI,CAAC;YACH,4BAA4B;YAC5B,IAAI,CAAC,WAAW,GAAG,MAAM,SAAS,CAAC,YAAY,CAAC,YAAY,CAAC;gBAC3D,KAAK,EAAE;oBACL,UAAU,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,UAAU,EAAE;oBAC7C,YAAY,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE;oBAC7C,gBAAgB,EAAE,IAAI;oBACtB,gBAAgB,EAAE,IAAI;oBACtB,eAAe,EAAE,IAAI;iBACtB;aACF,CAAC,CAAC;YAEH,yEAAyE;YACzE,oEAAoE;YACpE,+CAA+C;YAC/C,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,CAAC;YACxD,IAAI,UAAU,EAAE,CAAC;gBACf,IAAI,CAAC,kBAAkB,GAAG,GAAG,EAAE;oBAC7B,MAAM,CAAC,IAAI,CAAC,4DAA4D,CAAC,CAAC;oBAC1E,MAAM,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC;oBACvC,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;oBAC1B,IAAI,CAAC,aAAa,GAAG,CAAC,CAAC;oBACvB,IAAI,CAAC,gBAAgB,EAAE,CAAC;oBACxB,IAAI,YAAY,EAAE,CAAC;wBACjB,IAAI,CAAC,YAAY,EAAE,EAAE,CAAC;oBACxB,CAAC;gBACH,CAAC,CAAC;gBACF,UAAU,CAAC,gBAAgB,CAAC,OAAO,EAAE,IAAI,CAAC,kBAAkB,CAAC,CAAC;YAChE,CAAC;YAED,4CAA4C;YAC5C,IAAI,CAAC,YAAY,GAAG,IAAI,YAAY,CAAC;gBACnC,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,UAAU;aACnC,CAAC,CAAC;YAEH,qCAAqC;YACrC,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,YAAY,CAAC,uBAAuB,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YAE9E,0CAA0C;YAC1C,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,cAAc,EAAE,CAAC;YACpD,IAAI,CAAC,SAAS,CAAC,OAAO,GAAG,GAAG,CAAC;YAC7B,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YAExC,yDAAyD;YACzD,oEAAoE;YACpE,gEAAgE;YAChE,MAAM,UAAU,GAAG,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YAC/D,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,YAAY,CAAC,qBAAqB,CAC1D,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ,CACvD,CAAC;YAEF,IAAI,CAAC,aAAa,CAAC,cAAc,GAAG,CAAC,KAAK,EAAE,EAAE;gBAC5C,IAAI,CAAC,IAAI,CAAC,YAAY;oBAAE,OAAO;gBAE/B,MAAM,SAAS,GAAG,KAAK,CAAC,WAAW,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC;gBACtD,oCAAoC;gBACpC,MAAM,OAAO,GAAG,IAAI,YAAY,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;gBACnD,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;gBAEvB,kCAAkC;gBAClC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBAE9B,oCAAoC;gBACpC,IAAI,CAAC,aAAa,EAAE,CAAC,OAAO,CAAC,CAAC;YAChC,CAAC,CAAC;YAEF,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;YAC5C,IAAI,CAAC,aAAa,CAAC,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,WAAW,CAAC,CAAC;YAE1D,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;YAEzB,8BAA8B;YAC9B,IAAI,CAAC,oBAAoB,EAAE,CAAC;YAE5B,MAAM,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;QACvC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACxB,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACvE,MAAM,CAAC,KAAK,CAAC,kCAAkC,OAAO,EAAE,CAAC,CAAC;YAC1D,MAAM,IAAI,KAAK,CAAC,6BAA6B,OAAO,EAAE,CAAC,CAAC;QAC1D,CAAC;IACH,CAAC;IAED;;OAEG;IACH,IAAI;QACF,IAAI,CAAC,IAAI,CAAC,YAAY;YAAE,OAAO;QAE/B,IAAI,CAAC,YAAY,GAAG,KAAK,CAAC;QAC1B,IAAI,CAAC,aAAa,GAAG,CAAC,CAAC;QACvB,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;QAC1B,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;QAC1B,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAExB,MAAM,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;IACvC,CAAC;IAED;;;OAGG;IACH,cAAc;QACZ,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,IAAI,YAAY,CAAC,CAAC,CAAC,CAAC;QAC7D,MAAM,WAAW,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QAC1E,MAAM,MAAM,GAAG,IAAI,YAAY,CAAC,WAAW,CAAC,CAAC;QAC7C,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;YAC1B,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC;QACzB,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAED;;;OAGG;IACH,WAAW;QACT,MAAM,MAAM,GAAG,IAAI,CAAC,cAAc,EAAE,CAAC;QACrC,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC;QACrB,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,yDAAyD;IACzD,WAAW;QACT,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC;IACvB,CAAC;IAED,wEAAwE;IAChE,oBAAoB;QAC1B,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC;QAChC,IAAI,CAAC,QAAQ;YAAE,OAAO;QAEtB,MAAM,SAAS,GAAG,IAAI,UAAU,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC;QAC7D,MAAM,IAAI,GAAG,GAAG,EAAE;YAChB,IAAI,CAAC,IAAI,CAAC,YAAY,IAAI,CAAC,IAAI,CAAC,SAAS;gBAAE,OAAO;YAClD,IAAI,CAAC,SAAS,CAAC,oBAAoB,CAAC,SAAS,CAAC,CAAC;YAC/C,IAAI,GAAG,GAAG,CAAC,CAAC;YACZ,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,MAAM,EAAE,CAAC,EAAE;gBAAE,GAAG,IAAI,SAAS,CAAC,CAAC,CAAC,CAAC;YAC/D,MAAM,GAAG,GAAG,GAAG,GAAG,SAAS,CAAC,MAAM,GAAG,GAAG,CAAC;YACzC,IAAI,CAAC,aAAa,GAAG,GAAG,CAAC;YACzB,IAAI,CAAC,aAAa,EAAE,CAAC,GAAG,CAAC,CAAC;YAC1B,IAAI,CAAC,YAAY,GAAG,qBAAqB,CAAC,IAAI,CAAC,CAAC;QAClD,CAAC,CAAC;QACF,IAAI,CAAC,YAAY,GAAG,qBAAqB,CAAC,IAAI,CAAC,CAAC;IAClD,CAAC;IAEO,gBAAgB;QACtB,IAAI,IAAI,CAAC,YAAY,KAAK,IAAI,EAAE,CAAC;YAC/B,oBAAoB,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;YACxC,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QAC3B,CAAC;QACD,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACvB,IAAI,CAAC,aAAa,CAAC,UAAU,EAAE,CAAC;YAChC,IAAI,CAAC,aAAa,CAAC,cAAc,GAAG,IAAI,CAAC;YACzC,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;QAC5B,CAAC;QACD,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,CAAC;YAC5B,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACxB,CAAC;QACD,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,IAAI,CAAC,UAAU,CAAC,UAAU,EAAE,CAAC;YAC7B,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QACzB,CAAC;QACD,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtB,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAgB,CAAC,CAAC,CAAC;YACxD,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QAC3B,CAAC;QACD,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACrB,uEAAuE;YACvE,IAAI,IAAI,CAAC,kBAAkB,EAAE,CAAC;gBAC5B,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,CAAC;gBACxD,UAAU,EAAE,mBAAmB,CAAC,OAAO,EAAE,IAAI,CAAC,kBAAkB,CAAC,CAAC;gBAClE,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC;YACjC,CAAC;YACD,IAAI,CAAC,WAAW,CAAC,SAAS,EAAE,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;YAC9D,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QAC1B,CAAC;IACH,CAAC;IAEO,eAAe,CAAC,CAAS;QAC/B,IAAI,KAAK,GAAG,GAAG,CAAC;QAChB,OAAO,KAAK,GAAG,CAAC,IAAI,KAAK,GAAG,KAAK,EAAE,CAAC;YAClC,KAAK,IAAI,CAAC,CAAC;QACb,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;CACF"}
|
|
@@ -1,53 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* RunAnywhere Web SDK - Audio Playback
|
|
3
|
-
*
|
|
4
|
-
* Plays synthesized audio (TTS output) using Web Audio API.
|
|
5
|
-
* Accepts Float32Array PCM samples and plays them through
|
|
6
|
-
* the browser's audio output.
|
|
7
|
-
*
|
|
8
|
-
* Supports:
|
|
9
|
-
* - One-shot playback of PCM audio buffers
|
|
10
|
-
* - Streaming playback (queued chunks)
|
|
11
|
-
* - Playback controls (stop, pause, resume)
|
|
12
|
-
* - Completion callbacks
|
|
13
|
-
*/
|
|
14
|
-
export type PlaybackCompleteCallback = () => void;
|
|
15
|
-
export interface PlaybackConfig {
|
|
16
|
-
/** Sample rate of the audio (default: 22050 for Piper TTS) */
|
|
17
|
-
sampleRate?: number;
|
|
18
|
-
/** Volume (0.0 - 1.0, default: 1.0) */
|
|
19
|
-
volume?: number;
|
|
20
|
-
}
|
|
21
|
-
/**
|
|
22
|
-
* AudioPlayback - Plays synthesized audio through browser speakers.
|
|
23
|
-
*/
|
|
24
|
-
export declare class AudioPlayback {
|
|
25
|
-
private audioContext;
|
|
26
|
-
private gainNode;
|
|
27
|
-
private currentSource;
|
|
28
|
-
private _isPlaying;
|
|
29
|
-
private config;
|
|
30
|
-
constructor(config?: PlaybackConfig);
|
|
31
|
-
get isPlaying(): boolean;
|
|
32
|
-
/**
|
|
33
|
-
* Play a Float32Array of PCM audio samples.
|
|
34
|
-
*
|
|
35
|
-
* @param samples - PCM audio data (Float32Array)
|
|
36
|
-
* @param sampleRate - Sample rate (overrides config)
|
|
37
|
-
* @returns Promise that resolves when playback completes
|
|
38
|
-
*/
|
|
39
|
-
play(samples: Float32Array, sampleRate?: number): Promise<void>;
|
|
40
|
-
/**
|
|
41
|
-
* Stop playback immediately.
|
|
42
|
-
*/
|
|
43
|
-
stop(): void;
|
|
44
|
-
/**
|
|
45
|
-
* Set playback volume.
|
|
46
|
-
*/
|
|
47
|
-
setVolume(volume: number): void;
|
|
48
|
-
/**
|
|
49
|
-
* Release all audio resources.
|
|
50
|
-
*/
|
|
51
|
-
dispose(): void;
|
|
52
|
-
}
|
|
53
|
-
//# sourceMappingURL=AudioPlayback.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"AudioPlayback.d.ts","sourceRoot":"","sources":["../../src/Infrastructure/AudioPlayback.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAQH,MAAM,MAAM,wBAAwB,GAAG,MAAM,IAAI,CAAC;AAElD,MAAM,WAAW,cAAc;IAC7B,8DAA8D;IAC9D,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,uCAAuC;IACvC,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED;;GAEG;AACH,qBAAa,aAAa;IACxB,OAAO,CAAC,YAAY,CAA6B;IACjD,OAAO,CAAC,QAAQ,CAAyB;IACzC,OAAO,CAAC,aAAa,CAAsC;IAC3D,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,MAAM,CAA2B;gBAE7B,MAAM,GAAE,cAAmB;IAOvC,IAAI,SAAS,IAAI,OAAO,CAEvB;IAED;;;;;;OAMG;IACG,IAAI,CAAC,OAAO,EAAE,YAAY,EAAE,UAAU,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAiDrE;;OAEG;IACH,IAAI,IAAI,IAAI;IAYZ;;OAEG;IACH,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IAO/B;;OAEG;IACH,OAAO,IAAI,IAAI;CAQhB"}
|
|
@@ -1,117 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* RunAnywhere Web SDK - Audio Playback
|
|
3
|
-
*
|
|
4
|
-
* Plays synthesized audio (TTS output) using Web Audio API.
|
|
5
|
-
* Accepts Float32Array PCM samples and plays them through
|
|
6
|
-
* the browser's audio output.
|
|
7
|
-
*
|
|
8
|
-
* Supports:
|
|
9
|
-
* - One-shot playback of PCM audio buffers
|
|
10
|
-
* - Streaming playback (queued chunks)
|
|
11
|
-
* - Playback controls (stop, pause, resume)
|
|
12
|
-
* - Completion callbacks
|
|
13
|
-
*/
|
|
14
|
-
import { SDKLogger } from '../Foundation/SDKLogger';
|
|
15
|
-
import { EventBus } from '../Foundation/EventBus';
|
|
16
|
-
import { SDKEventType } from '../types/enums';
|
|
17
|
-
const logger = new SDKLogger('AudioPlayback');
|
|
18
|
-
/**
|
|
19
|
-
* AudioPlayback - Plays synthesized audio through browser speakers.
|
|
20
|
-
*/
|
|
21
|
-
export class AudioPlayback {
|
|
22
|
-
audioContext = null;
|
|
23
|
-
gainNode = null;
|
|
24
|
-
currentSource = null;
|
|
25
|
-
_isPlaying = false;
|
|
26
|
-
config;
|
|
27
|
-
constructor(config = {}) {
|
|
28
|
-
this.config = {
|
|
29
|
-
sampleRate: config.sampleRate ?? 22050,
|
|
30
|
-
volume: config.volume ?? 1.0,
|
|
31
|
-
};
|
|
32
|
-
}
|
|
33
|
-
get isPlaying() {
|
|
34
|
-
return this._isPlaying;
|
|
35
|
-
}
|
|
36
|
-
/**
|
|
37
|
-
* Play a Float32Array of PCM audio samples.
|
|
38
|
-
*
|
|
39
|
-
* @param samples - PCM audio data (Float32Array)
|
|
40
|
-
* @param sampleRate - Sample rate (overrides config)
|
|
41
|
-
* @returns Promise that resolves when playback completes
|
|
42
|
-
*/
|
|
43
|
-
async play(samples, sampleRate) {
|
|
44
|
-
this.stop(); // Stop any current playback
|
|
45
|
-
const rate = sampleRate ?? this.config.sampleRate;
|
|
46
|
-
const durationMs = (samples.length / rate) * 1000;
|
|
47
|
-
logger.debug(`Playing ${samples.length} samples at ${rate}Hz (${(durationMs / 1000).toFixed(1)}s)`);
|
|
48
|
-
// Create or resume AudioContext
|
|
49
|
-
if (!this.audioContext || this.audioContext.state === 'closed') {
|
|
50
|
-
this.audioContext = new AudioContext({ sampleRate: rate });
|
|
51
|
-
}
|
|
52
|
-
if (this.audioContext.state === 'suspended') {
|
|
53
|
-
await this.audioContext.resume();
|
|
54
|
-
}
|
|
55
|
-
// Create gain node for volume control
|
|
56
|
-
if (!this.gainNode || this.gainNode.context !== this.audioContext) {
|
|
57
|
-
this.gainNode = this.audioContext.createGain();
|
|
58
|
-
this.gainNode.connect(this.audioContext.destination);
|
|
59
|
-
}
|
|
60
|
-
this.gainNode.gain.value = this.config.volume;
|
|
61
|
-
// Create audio buffer from samples
|
|
62
|
-
const audioBuffer = this.audioContext.createBuffer(1, samples.length, rate);
|
|
63
|
-
audioBuffer.getChannelData(0).set(samples);
|
|
64
|
-
// Create and play source node
|
|
65
|
-
return new Promise((resolve) => {
|
|
66
|
-
const source = this.audioContext.createBufferSource();
|
|
67
|
-
source.buffer = audioBuffer;
|
|
68
|
-
source.connect(this.gainNode);
|
|
69
|
-
source.onended = () => {
|
|
70
|
-
this._isPlaying = false;
|
|
71
|
-
this.currentSource = null;
|
|
72
|
-
EventBus.shared.emit('playback.completed', SDKEventType.Voice, { durationMs });
|
|
73
|
-
resolve();
|
|
74
|
-
};
|
|
75
|
-
this.currentSource = source;
|
|
76
|
-
this._isPlaying = true;
|
|
77
|
-
EventBus.shared.emit('playback.started', SDKEventType.Voice, { durationMs, sampleRate: rate });
|
|
78
|
-
source.start();
|
|
79
|
-
});
|
|
80
|
-
}
|
|
81
|
-
/**
|
|
82
|
-
* Stop playback immediately.
|
|
83
|
-
*/
|
|
84
|
-
stop() {
|
|
85
|
-
if (this.currentSource) {
|
|
86
|
-
try {
|
|
87
|
-
this.currentSource.stop();
|
|
88
|
-
}
|
|
89
|
-
catch {
|
|
90
|
-
// Already stopped
|
|
91
|
-
}
|
|
92
|
-
this.currentSource = null;
|
|
93
|
-
}
|
|
94
|
-
this._isPlaying = false;
|
|
95
|
-
}
|
|
96
|
-
/**
|
|
97
|
-
* Set playback volume.
|
|
98
|
-
*/
|
|
99
|
-
setVolume(volume) {
|
|
100
|
-
this.config.volume = Math.max(0, Math.min(1, volume));
|
|
101
|
-
if (this.gainNode) {
|
|
102
|
-
this.gainNode.gain.value = this.config.volume;
|
|
103
|
-
}
|
|
104
|
-
}
|
|
105
|
-
/**
|
|
106
|
-
* Release all audio resources.
|
|
107
|
-
*/
|
|
108
|
-
dispose() {
|
|
109
|
-
this.stop();
|
|
110
|
-
if (this.audioContext && this.audioContext.state !== 'closed') {
|
|
111
|
-
this.audioContext.close().catch(() => { });
|
|
112
|
-
}
|
|
113
|
-
this.audioContext = null;
|
|
114
|
-
this.gainNode = null;
|
|
115
|
-
}
|
|
116
|
-
}
|
|
117
|
-
//# sourceMappingURL=AudioPlayback.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"AudioPlayback.js","sourceRoot":"","sources":["../../src/Infrastructure/AudioPlayback.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC;AACpD,OAAO,EAAE,QAAQ,EAAE,MAAM,wBAAwB,CAAC;AAClD,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAE9C,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC,eAAe,CAAC,CAAC;AAW9C;;GAEG;AACH,MAAM,OAAO,aAAa;IAChB,YAAY,GAAwB,IAAI,CAAC;IACzC,QAAQ,GAAoB,IAAI,CAAC;IACjC,aAAa,GAAiC,IAAI,CAAC;IACnD,UAAU,GAAG,KAAK,CAAC;IACnB,MAAM,CAA2B;IAEzC,YAAY,SAAyB,EAAE;QACrC,IAAI,CAAC,MAAM,GAAG;YACZ,UAAU,EAAE,MAAM,CAAC,UAAU,IAAI,KAAK;YACtC,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,GAAG;SAC7B,CAAC;IACJ,CAAC;IAED,IAAI,SAAS;QACX,OAAO,IAAI,CAAC,UAAU,CAAC;IACzB,CAAC;IAED;;;;;;OAMG;IACH,KAAK,CAAC,IAAI,CAAC,OAAqB,EAAE,UAAmB;QACnD,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,4BAA4B;QAEzC,MAAM,IAAI,GAAG,UAAU,IAAI,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC;QAClD,MAAM,UAAU,GAAG,CAAC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC;QAElD,MAAM,CAAC,KAAK,CAAC,WAAW,OAAO,CAAC,MAAM,eAAe,IAAI,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAEpG,gCAAgC;QAChC,IAAI,CAAC,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,YAAY,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC/D,IAAI,CAAC,YAAY,GAAG,IAAI,YAAY,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;QAC7D,CAAC;QAED,IAAI,IAAI,CAAC,YAAY,CAAC,KAAK,KAAK,WAAW,EAAE,CAAC;YAC5C,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC;QACnC,CAAC;QAED,sCAAsC;QACtC,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ,CAAC,OAAO,KAAK,IAAI,CAAC,YAAY,EAAE,CAAC;YAClE,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,YAAY,CAAC,UAAU,EAAE,CAAC;YAC/C,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,WAAW,CAAC,CAAC;QACvD,CAAC;QACD,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;QAE9C,mCAAmC;QACnC,MAAM,WAAW,GAAG,IAAI,CAAC,YAAY,CAAC,YAAY,CAAC,CAAC,EAAE,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QAC5E,WAAW,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QAE3C,8BAA8B;QAC9B,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE;YACnC,MAAM,MAAM,GAAG,IAAI,CAAC,YAAa,CAAC,kBAAkB,EAAE,CAAC;YACvD,MAAM,CAAC,MAAM,GAAG,WAAW,CAAC;YAC5B,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,QAAS,CAAC,CAAC;YAE/B,MAAM,CAAC,OAAO,GAAG,GAAG,EAAE;gBACpB,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;gBACxB,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;gBAC1B,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,oBAAoB,EAAE,YAAY,CAAC,KAAK,EAAE,EAAE,UAAU,EAAE,CAAC,CAAC;gBAC/E,OAAO,EAAE,CAAC;YACZ,CAAC,CAAC;YAEF,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC;YAC5B,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;YAEvB,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,kBAAkB,EAAE,YAAY,CAAC,KAAK,EAAE,EAAE,UAAU,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;YAC/F,MAAM,CAAC,KAAK,EAAE,CAAC;QACjB,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,IAAI;QACF,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACvB,IAAI,CAAC;gBACH,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC;YAC5B,CAAC;YAAC,MAAM,CAAC;gBACP,kBAAkB;YACpB,CAAC;YACD,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;QAC5B,CAAC;QACD,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;IAC1B,CAAC;IAED;;OAEG;IACH,SAAS,CAAC,MAAc;QACtB,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC;QACtD,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;QAChD,CAAC;IACH,CAAC;IAED;;OAEG;IACH,OAAO;QACL,IAAI,CAAC,IAAI,EAAE,CAAC;QACZ,IAAI,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,YAAY,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC9D,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAgB,CAAC,CAAC,CAAC;QAC1D,CAAC;QACD,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC;QACzB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;IACvB,CAAC;CACF"}
|
|
@@ -1,211 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* RunAnywhere Web SDK - VLM Worker Bridge
|
|
3
|
-
*
|
|
4
|
-
* Main-thread proxy for the VLM Web Worker. All VLM inference runs off the
|
|
5
|
-
* main thread so the camera, UI animations, and event loop stay responsive.
|
|
6
|
-
*
|
|
7
|
-
* Usage:
|
|
8
|
-
* ```typescript
|
|
9
|
-
* import { VLMWorkerBridge } from '@runanywhere/web';
|
|
10
|
-
*
|
|
11
|
-
* const vlm = VLMWorkerBridge.shared;
|
|
12
|
-
* await vlm.init();
|
|
13
|
-
* await vlm.loadModel({ ... });
|
|
14
|
-
* const result = await vlm.process(rgbPixels, width, height, prompt, { maxTokens: 100 });
|
|
15
|
-
* ```
|
|
16
|
-
*/
|
|
17
|
-
/**
|
|
18
|
-
* RPC commands sent from the main thread to the Worker.
|
|
19
|
-
*/
|
|
20
|
-
export type VLMWorkerCommand = {
|
|
21
|
-
type: 'init';
|
|
22
|
-
id: number;
|
|
23
|
-
payload: {
|
|
24
|
-
/** URL to the WASM glue JS (racommons.js or racommons-webgpu.js) */
|
|
25
|
-
wasmJsUrl: string;
|
|
26
|
-
/** Whether the loaded module is the WebGPU variant */
|
|
27
|
-
useWebGPU?: boolean;
|
|
28
|
-
};
|
|
29
|
-
} | {
|
|
30
|
-
type: 'load-model';
|
|
31
|
-
id: number;
|
|
32
|
-
payload: {
|
|
33
|
-
modelOpfsKey: string;
|
|
34
|
-
modelFilename: string;
|
|
35
|
-
mmprojOpfsKey: string;
|
|
36
|
-
mmprojFilename: string;
|
|
37
|
-
modelId: string;
|
|
38
|
-
modelName: string;
|
|
39
|
-
/** Optional: raw model data when OPFS doesn't have it (memory-cache fallback). */
|
|
40
|
-
modelData?: ArrayBuffer;
|
|
41
|
-
/** Optional: raw mmproj data when OPFS doesn't have it. */
|
|
42
|
-
mmprojData?: ArrayBuffer;
|
|
43
|
-
};
|
|
44
|
-
} | {
|
|
45
|
-
type: 'process';
|
|
46
|
-
id: number;
|
|
47
|
-
payload: {
|
|
48
|
-
rgbPixels: ArrayBuffer;
|
|
49
|
-
width: number;
|
|
50
|
-
height: number;
|
|
51
|
-
prompt: string;
|
|
52
|
-
maxTokens: number;
|
|
53
|
-
temperature: number;
|
|
54
|
-
};
|
|
55
|
-
} | {
|
|
56
|
-
type: 'cancel';
|
|
57
|
-
id: number;
|
|
58
|
-
} | {
|
|
59
|
-
type: 'unload';
|
|
60
|
-
id: number;
|
|
61
|
-
};
|
|
62
|
-
/**
|
|
63
|
-
* Result of a VLM inference operation.
|
|
64
|
-
*/
|
|
65
|
-
export interface VLMWorkerResult {
|
|
66
|
-
text: string;
|
|
67
|
-
totalTokens: number;
|
|
68
|
-
promptTokens: number;
|
|
69
|
-
completionTokens: number;
|
|
70
|
-
imageTokens: number;
|
|
71
|
-
}
|
|
72
|
-
/**
|
|
73
|
-
* RPC responses from the Worker to the main thread.
|
|
74
|
-
*/
|
|
75
|
-
export type VLMWorkerResponse = {
|
|
76
|
-
id: number;
|
|
77
|
-
type: 'result';
|
|
78
|
-
payload: any;
|
|
79
|
-
} | {
|
|
80
|
-
id: number;
|
|
81
|
-
type: 'error';
|
|
82
|
-
payload: {
|
|
83
|
-
message: string;
|
|
84
|
-
};
|
|
85
|
-
} | {
|
|
86
|
-
id: number;
|
|
87
|
-
type: 'progress';
|
|
88
|
-
payload: {
|
|
89
|
-
stage: string;
|
|
90
|
-
};
|
|
91
|
-
};
|
|
92
|
-
/**
|
|
93
|
-
* Parameters for loading a VLM model in the Worker.
|
|
94
|
-
*/
|
|
95
|
-
export interface VLMLoadModelParams {
|
|
96
|
-
modelOpfsKey: string;
|
|
97
|
-
modelFilename: string;
|
|
98
|
-
mmprojOpfsKey: string;
|
|
99
|
-
mmprojFilename: string;
|
|
100
|
-
modelId: string;
|
|
101
|
-
modelName: string;
|
|
102
|
-
/** Optional: raw model data when OPFS doesn't have it (memory-cache fallback). */
|
|
103
|
-
modelData?: ArrayBuffer;
|
|
104
|
-
/** Optional: raw mmproj data when OPFS doesn't have it. */
|
|
105
|
-
mmprojData?: ArrayBuffer;
|
|
106
|
-
}
|
|
107
|
-
/**
|
|
108
|
-
* Options for VLM image processing.
|
|
109
|
-
*/
|
|
110
|
-
export interface VLMProcessOptions {
|
|
111
|
-
maxTokens?: number;
|
|
112
|
-
temperature?: number;
|
|
113
|
-
}
|
|
114
|
-
/**
|
|
115
|
-
* Callback for VLM progress updates (model loading stages, etc.).
|
|
116
|
-
*/
|
|
117
|
-
export type ProgressListener = (stage: string) => void;
|
|
118
|
-
/**
|
|
119
|
-
* VLMWorkerBridge - Main-thread proxy for VLM Web Worker inference.
|
|
120
|
-
*
|
|
121
|
-
* Manages the lifecycle of a dedicated Web Worker that runs VLM inference
|
|
122
|
-
* in its own WASM instance. Provides:
|
|
123
|
-
* - RPC protocol with message ID tracking and promise correlation
|
|
124
|
-
* - Auto-recovery from WASM crashes (OOB, stack overflow, etc.)
|
|
125
|
-
* - Progress listeners for model loading stages
|
|
126
|
-
* - Transferable pixel data for zero-copy image transfer
|
|
127
|
-
*/
|
|
128
|
-
export declare class VLMWorkerBridge {
|
|
129
|
-
private static _instance;
|
|
130
|
-
static get shared(): VLMWorkerBridge;
|
|
131
|
-
private worker;
|
|
132
|
-
private nextId;
|
|
133
|
-
private pending;
|
|
134
|
-
private _isInitialized;
|
|
135
|
-
private _isModelLoaded;
|
|
136
|
-
private _progressListeners;
|
|
137
|
-
/** Saved for auto-recovery after WASM crash */
|
|
138
|
-
private _lastModelParams;
|
|
139
|
-
private _needsRecovery;
|
|
140
|
-
/**
|
|
141
|
-
* Optional: the app can provide a custom Worker URL or factory.
|
|
142
|
-
* If not set, the bridge uses the bundled SDK worker entry point.
|
|
143
|
-
*/
|
|
144
|
-
private _workerUrl;
|
|
145
|
-
get isInitialized(): boolean;
|
|
146
|
-
get isModelLoaded(): boolean;
|
|
147
|
-
/**
|
|
148
|
-
* Set a custom Worker URL.
|
|
149
|
-
*
|
|
150
|
-
* By default the bridge creates a Worker using the SDK's bundled entry point
|
|
151
|
-
* (`workers/vlm-worker.js`). Apps that need to customise the Worker location
|
|
152
|
-
* (e.g. different deploy path, or a worker that wraps the runtime) can call
|
|
153
|
-
* this before `init()`.
|
|
154
|
-
*/
|
|
155
|
-
set workerUrl(url: URL | string);
|
|
156
|
-
/**
|
|
157
|
-
* Subscribe to progress updates from the Worker.
|
|
158
|
-
* Returns an unsubscribe function.
|
|
159
|
-
*/
|
|
160
|
-
onProgress(fn: ProgressListener): () => void;
|
|
161
|
-
private emitProgress;
|
|
162
|
-
/**
|
|
163
|
-
* Initialize the Worker and its WASM instance.
|
|
164
|
-
* Must be called once before loadModel/process.
|
|
165
|
-
*
|
|
166
|
-
* Reads the WASM URL and acceleration mode from WASMBridge internally — the
|
|
167
|
-
* app does not need to pass these.
|
|
168
|
-
*
|
|
169
|
-
* @param wasmJsUrl - Optional explicit URL for the WASM glue JS.
|
|
170
|
-
* When omitted, the SDK's WASMBridge.workerWasmUrl is used
|
|
171
|
-
* so the worker loads the exact same variant (WebGPU or CPU)
|
|
172
|
-
* that the main thread successfully loaded.
|
|
173
|
-
*/
|
|
174
|
-
init(wasmJsUrl?: string): Promise<void>;
|
|
175
|
-
/**
|
|
176
|
-
* Load a VLM model in the Worker's WASM instance.
|
|
177
|
-
*
|
|
178
|
-
* Normally the Worker reads model files directly from OPFS (zero-copy).
|
|
179
|
-
* When OPFS quota is exceeded and models are only in the main-thread memory
|
|
180
|
-
* cache, the data is transferred via postMessage (still zero-copy via
|
|
181
|
-
* Transferable ArrayBuffers).
|
|
182
|
-
*/
|
|
183
|
-
loadModel(params: VLMLoadModelParams): Promise<void>;
|
|
184
|
-
/**
|
|
185
|
-
* Process an image with the VLM.
|
|
186
|
-
* Returns a promise that resolves when inference is complete.
|
|
187
|
-
* The main thread stays responsive during processing.
|
|
188
|
-
*
|
|
189
|
-
* The pixel buffer is transferred (zero-copy) to the Worker.
|
|
190
|
-
*/
|
|
191
|
-
process(rgbPixels: Uint8Array, width: number, height: number, prompt: string, options?: VLMProcessOptions): Promise<VLMWorkerResult>;
|
|
192
|
-
/**
|
|
193
|
-
* Recover from a WASM crash by terminating the old Worker,
|
|
194
|
-
* creating a fresh one, and reloading the model.
|
|
195
|
-
*/
|
|
196
|
-
private recover;
|
|
197
|
-
/** Cancel in-progress VLM generation. */
|
|
198
|
-
cancel(): void;
|
|
199
|
-
/** Unload the VLM model. */
|
|
200
|
-
unloadModel(): Promise<void>;
|
|
201
|
-
/**
|
|
202
|
-
* Terminate the Worker entirely.
|
|
203
|
-
*
|
|
204
|
-
* Rejects any in-flight RPC promises so callers aren't left hanging,
|
|
205
|
-
* then terminates the underlying Web Worker.
|
|
206
|
-
*/
|
|
207
|
-
terminate(): void;
|
|
208
|
-
private send;
|
|
209
|
-
private handleMessage;
|
|
210
|
-
}
|
|
211
|
-
//# sourceMappingURL=VLMWorkerBridge.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"VLMWorkerBridge.d.ts","sourceRoot":"","sources":["../../src/Infrastructure/VLMWorkerBridge.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AASH;;GAEG;AACH,MAAM,MAAM,gBAAgB,GACxB;IACE,IAAI,EAAE,MAAM,CAAC;IAAC,EAAE,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE;QACjC,oEAAoE;QACpE,SAAS,EAAE,MAAM,CAAC;QAClB,sDAAsD;QACtD,SAAS,CAAC,EAAE,OAAO,CAAC;KACrB,CAAC;CACH,GACD;IACE,IAAI,EAAE,YAAY,CAAC;IAAC,EAAE,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE;QACvC,YAAY,EAAE,MAAM,CAAC;QAAC,aAAa,EAAE,MAAM,CAAC;QAC5C,aAAa,EAAE,MAAM,CAAC;QAAC,cAAc,EAAE,MAAM,CAAC;QAC9C,OAAO,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QACnC,kFAAkF;QAClF,SAAS,CAAC,EAAE,WAAW,CAAC;QACxB,2DAA2D;QAC3D,UAAU,CAAC,EAAE,WAAW,CAAC;KAC1B,CAAC;CACH,GACD;IACE,IAAI,EAAE,SAAS,CAAC;IAAC,EAAE,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE;QACpC,SAAS,EAAE,WAAW,CAAC;QAAC,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QACtD,MAAM,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAC;KACxD,CAAC;CACH,GACD;IAAE,IAAI,EAAE,QAAQ,CAAC;IAAC,EAAE,EAAE,MAAM,CAAA;CAAE,GAC9B;IAAE,IAAI,EAAE,QAAQ,CAAC;IAAC,EAAE,EAAE,MAAM,CAAA;CAAE,CAAC;AAEnC;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,gBAAgB,EAAE,MAAM,CAAC;IACzB,WAAW,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,MAAM,MAAM,iBAAiB,GACzB;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,QAAQ,CAAC;IAAC,OAAO,EAAE,GAAG,CAAA;CAAE,GAC5C;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,OAAO,CAAC;IAAC,OAAO,EAAE;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,CAAA;CAAE,GAC3D;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,UAAU,CAAC;IAAC,OAAO,EAAE;QAAE,KAAK,EAAE,MAAM,CAAA;KAAE,CAAA;CAAE,CAAC;AAEjE;;GAEG;AACH,MAAM,WAAW,kBAAkB;IACjC,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,cAAc,EAAE,MAAM,CAAC;IACvB,OAAO,EAAE,MAAM,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;IAClB,kFAAkF;IAClF,SAAS,CAAC,EAAE,WAAW,CAAC;IACxB,2DAA2D;IAC3D,UAAU,CAAC,EAAE,WAAW,CAAC;CAC1B;AAED;;GAEG;AACH,MAAM,WAAW,iBAAiB;IAChC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAED;;GAEG;AACH,MAAM,MAAM,gBAAgB,GAAG,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;AAQvD;;;;;;;;;GASG;AACH,qBAAa,eAAe;IAE1B,OAAO,CAAC,MAAM,CAAC,SAAS,CAAgC;IAExD,MAAM,KAAK,MAAM,IAAI,eAAe,CAKnC;IAGD,OAAO,CAAC,MAAM,CAAuB;IACrC,OAAO,CAAC,MAAM,CAAK;IACnB,OAAO,CAAC,OAAO,CAAgF;IAC/F,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,kBAAkB,CAA0B;IACpD,+CAA+C;IAC/C,OAAO,CAAC,gBAAgB,CAAmC;IAC3D,OAAO,CAAC,cAAc,CAAS;IAE/B;;;OAGG;IACH,OAAO,CAAC,UAAU,CAA6B;IAE/C,IAAI,aAAa,IAAI,OAAO,CAAgC;IAC5D,IAAI,aAAa,IAAI,OAAO,CAAgC;IAI5D;;;;;;;OAOG;IACH,IAAI,SAAS,CAAC,GAAG,EAAE,GAAG,GAAG,MAAM,EAE9B;IAID;;;OAGG;IACH,UAAU,CAAC,EAAE,EAAE,gBAAgB,GAAG,MAAM,IAAI;IAO5C,OAAO,CAAC,YAAY;IAMpB;;;;;;;;;;;OAWG;IACG,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IA2B7C;;;;;;;OAOG;IACG,SAAS,CAAC,MAAM,EAAE,kBAAkB,GAAG,OAAO,CAAC,IAAI,CAAC;IAiB1D;;;;;;OAMG;IACG,OAAO,CACX,SAAS,EAAE,UAAU,EACrB,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,MAAM,EACd,OAAO,GAAE,iBAAsB,GAC9B,OAAO,CAAC,eAAe,CAAC;IA2C3B;;;OAGG;YACW,OAAO;IAiBrB,yCAAyC;IACzC,MAAM,IAAI,IAAI;IAMd,4BAA4B;IACtB,WAAW,IAAI,OAAO,CAAC,IAAI,CAAC;IAMlC;;;;;OAKG;IACH,SAAS,IAAI,IAAI;IAgBjB,OAAO,CAAC,IAAI;IAaZ,OAAO,CAAC,aAAa;CAoBtB"}
|