@runanywhere/web 0.1.0-beta.6 → 0.1.0-beta.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/Foundation/ErrorTypes.d.ts +1 -0
- package/dist/Foundation/ErrorTypes.d.ts.map +1 -1
- package/dist/Foundation/ErrorTypes.js +3 -0
- package/dist/Foundation/ErrorTypes.js.map +1 -1
- package/dist/Foundation/EventBus.d.ts +0 -1
- package/dist/Foundation/EventBus.d.ts.map +1 -1
- package/dist/Foundation/StructOffsets.d.ts +5 -37
- package/dist/Foundation/StructOffsets.d.ts.map +1 -1
- package/dist/Foundation/StructOffsets.js +6 -157
- package/dist/Foundation/StructOffsets.js.map +1 -1
- package/dist/Foundation/WASMBridge.d.ts +8 -236
- package/dist/Foundation/WASMBridge.d.ts.map +1 -1
- package/dist/Foundation/WASMBridge.js +7 -388
- package/dist/Foundation/WASMBridge.js.map +1 -1
- package/dist/Infrastructure/DeviceCapabilities.d.ts.map +1 -1
- package/dist/Infrastructure/DeviceCapabilities.js +1 -3
- package/dist/Infrastructure/DeviceCapabilities.js.map +1 -1
- package/dist/Infrastructure/ExtensionPoint.d.ts +114 -0
- package/dist/Infrastructure/ExtensionPoint.d.ts.map +1 -0
- package/dist/Infrastructure/ExtensionPoint.js +178 -0
- package/dist/Infrastructure/ExtensionPoint.js.map +1 -0
- package/dist/Infrastructure/LocalFileStorage.d.ts +134 -0
- package/dist/Infrastructure/LocalFileStorage.d.ts.map +1 -0
- package/dist/Infrastructure/LocalFileStorage.js +428 -0
- package/dist/Infrastructure/LocalFileStorage.js.map +1 -0
- package/dist/Infrastructure/ModelDownloader.d.ts +21 -5
- package/dist/Infrastructure/ModelDownloader.d.ts.map +1 -1
- package/dist/Infrastructure/ModelDownloader.js +79 -7
- package/dist/Infrastructure/ModelDownloader.js.map +1 -1
- package/dist/Infrastructure/ModelFileInference.d.ts +39 -0
- package/dist/Infrastructure/ModelFileInference.d.ts.map +1 -0
- package/dist/Infrastructure/ModelFileInference.js +119 -0
- package/dist/Infrastructure/ModelFileInference.js.map +1 -0
- package/dist/Infrastructure/ModelLoaderTypes.d.ts +91 -12
- package/dist/Infrastructure/ModelLoaderTypes.d.ts.map +1 -1
- package/dist/Infrastructure/ModelLoaderTypes.js +7 -1
- package/dist/Infrastructure/ModelLoaderTypes.js.map +1 -1
- package/dist/Infrastructure/ModelManager.d.ts +31 -104
- package/dist/Infrastructure/ModelManager.d.ts.map +1 -1
- package/dist/Infrastructure/ModelManager.js +207 -568
- package/dist/Infrastructure/ModelManager.js.map +1 -1
- package/dist/Infrastructure/ModelRegistry.d.ts +6 -8
- package/dist/Infrastructure/ModelRegistry.d.ts.map +1 -1
- package/dist/Infrastructure/ModelRegistry.js +11 -4
- package/dist/Infrastructure/ModelRegistry.js.map +1 -1
- package/dist/Infrastructure/OPFSStorage.d.ts +8 -0
- package/dist/Infrastructure/OPFSStorage.d.ts.map +1 -1
- package/dist/Infrastructure/OPFSStorage.js +37 -0
- package/dist/Infrastructure/OPFSStorage.js.map +1 -1
- package/dist/Public/Extensions/RunAnywhere+ModelManagement.d.ts +12 -4
- package/dist/Public/Extensions/RunAnywhere+ModelManagement.d.ts.map +1 -1
- package/dist/Public/Extensions/RunAnywhere+ModelManagement.js +23 -51
- package/dist/Public/Extensions/RunAnywhere+ModelManagement.js.map +1 -1
- package/dist/Public/Extensions/RunAnywhere+VoiceAgent.d.ts +42 -10
- package/dist/Public/Extensions/RunAnywhere+VoiceAgent.d.ts.map +1 -1
- package/dist/Public/Extensions/RunAnywhere+VoiceAgent.js +63 -161
- package/dist/Public/Extensions/RunAnywhere+VoiceAgent.js.map +1 -1
- package/dist/Public/Extensions/RunAnywhere+VoicePipeline.d.ts +3 -29
- package/dist/Public/Extensions/RunAnywhere+VoicePipeline.d.ts.map +1 -1
- package/dist/Public/Extensions/RunAnywhere+VoicePipeline.js +26 -42
- package/dist/Public/Extensions/RunAnywhere+VoicePipeline.js.map +1 -1
- package/dist/Public/Extensions/VoicePipelineTypes.d.ts +28 -37
- package/dist/Public/Extensions/VoicePipelineTypes.d.ts.map +1 -1
- package/dist/Public/Extensions/VoicePipelineTypes.js +4 -1
- package/dist/Public/Extensions/VoicePipelineTypes.js.map +1 -1
- package/dist/Public/RunAnywhere.d.ts +29 -85
- package/dist/Public/RunAnywhere.d.ts.map +1 -1
- package/dist/Public/RunAnywhere.js +169 -211
- package/dist/Public/RunAnywhere.js.map +1 -1
- package/dist/index.d.ts +19 -39
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +15 -31
- package/dist/index.js.map +1 -1
- package/dist/types/index.d.ts +0 -1
- package/dist/types/index.d.ts.map +1 -1
- package/dist/types/index.js +1 -0
- package/dist/types/index.js.map +1 -1
- package/package.json +4 -10
- package/dist/Foundation/PlatformAdapter.d.ts +0 -101
- package/dist/Foundation/PlatformAdapter.d.ts.map +0 -1
- package/dist/Foundation/PlatformAdapter.js +0 -417
- package/dist/Foundation/PlatformAdapter.js.map +0 -1
- package/dist/Foundation/SherpaONNXBridge.d.ts +0 -147
- package/dist/Foundation/SherpaONNXBridge.d.ts.map +0 -1
- package/dist/Foundation/SherpaONNXBridge.js +0 -345
- package/dist/Foundation/SherpaONNXBridge.js.map +0 -1
- package/dist/Infrastructure/AudioCapture.d.ts +0 -99
- package/dist/Infrastructure/AudioCapture.d.ts.map +0 -1
- package/dist/Infrastructure/AudioCapture.js +0 -264
- package/dist/Infrastructure/AudioCapture.js.map +0 -1
- package/dist/Infrastructure/AudioPlayback.d.ts +0 -53
- package/dist/Infrastructure/AudioPlayback.d.ts.map +0 -1
- package/dist/Infrastructure/AudioPlayback.js +0 -117
- package/dist/Infrastructure/AudioPlayback.js.map +0 -1
- package/dist/Infrastructure/VLMWorkerBridge.d.ts +0 -211
- package/dist/Infrastructure/VLMWorkerBridge.d.ts.map +0 -1
- package/dist/Infrastructure/VLMWorkerBridge.js +0 -264
- package/dist/Infrastructure/VLMWorkerBridge.js.map +0 -1
- package/dist/Infrastructure/VLMWorkerRuntime.d.ts +0 -38
- package/dist/Infrastructure/VLMWorkerRuntime.d.ts.map +0 -1
- package/dist/Infrastructure/VLMWorkerRuntime.js +0 -503
- package/dist/Infrastructure/VLMWorkerRuntime.js.map +0 -1
- package/dist/Infrastructure/VideoCapture.d.ts +0 -130
- package/dist/Infrastructure/VideoCapture.d.ts.map +0 -1
- package/dist/Infrastructure/VideoCapture.js +0 -236
- package/dist/Infrastructure/VideoCapture.js.map +0 -1
- package/dist/Public/Extensions/DiffusionTypes.d.ts +0 -64
- package/dist/Public/Extensions/DiffusionTypes.d.ts.map +0 -1
- package/dist/Public/Extensions/DiffusionTypes.js +0 -28
- package/dist/Public/Extensions/DiffusionTypes.js.map +0 -1
- package/dist/Public/Extensions/EmbeddingsTypes.d.ts +0 -33
- package/dist/Public/Extensions/EmbeddingsTypes.d.ts.map +0 -1
- package/dist/Public/Extensions/EmbeddingsTypes.js +0 -13
- package/dist/Public/Extensions/EmbeddingsTypes.js.map +0 -1
- package/dist/Public/Extensions/RunAnywhere+Diffusion.d.ts +0 -44
- package/dist/Public/Extensions/RunAnywhere+Diffusion.d.ts.map +0 -1
- package/dist/Public/Extensions/RunAnywhere+Diffusion.js +0 -189
- package/dist/Public/Extensions/RunAnywhere+Diffusion.js.map +0 -1
- package/dist/Public/Extensions/RunAnywhere+Embeddings.d.ts +0 -56
- package/dist/Public/Extensions/RunAnywhere+Embeddings.d.ts.map +0 -1
- package/dist/Public/Extensions/RunAnywhere+Embeddings.js +0 -240
- package/dist/Public/Extensions/RunAnywhere+Embeddings.js.map +0 -1
- package/dist/Public/Extensions/RunAnywhere+STT.d.ts +0 -97
- package/dist/Public/Extensions/RunAnywhere+STT.d.ts.map +0 -1
- package/dist/Public/Extensions/RunAnywhere+STT.js +0 -417
- package/dist/Public/Extensions/RunAnywhere+STT.js.map +0 -1
- package/dist/Public/Extensions/RunAnywhere+StructuredOutput.d.ts +0 -69
- package/dist/Public/Extensions/RunAnywhere+StructuredOutput.d.ts.map +0 -1
- package/dist/Public/Extensions/RunAnywhere+StructuredOutput.js +0 -196
- package/dist/Public/Extensions/RunAnywhere+StructuredOutput.js.map +0 -1
- package/dist/Public/Extensions/RunAnywhere+TTS.d.ts +0 -55
- package/dist/Public/Extensions/RunAnywhere+TTS.d.ts.map +0 -1
- package/dist/Public/Extensions/RunAnywhere+TTS.js +0 -253
- package/dist/Public/Extensions/RunAnywhere+TTS.js.map +0 -1
- package/dist/Public/Extensions/RunAnywhere+TextGeneration.d.ts +0 -80
- package/dist/Public/Extensions/RunAnywhere+TextGeneration.d.ts.map +0 -1
- package/dist/Public/Extensions/RunAnywhere+TextGeneration.js +0 -470
- package/dist/Public/Extensions/RunAnywhere+TextGeneration.js.map +0 -1
- package/dist/Public/Extensions/RunAnywhere+ToolCalling.d.ts +0 -82
- package/dist/Public/Extensions/RunAnywhere+ToolCalling.d.ts.map +0 -1
- package/dist/Public/Extensions/RunAnywhere+ToolCalling.js +0 -576
- package/dist/Public/Extensions/RunAnywhere+ToolCalling.js.map +0 -1
- package/dist/Public/Extensions/RunAnywhere+VAD.d.ts +0 -70
- package/dist/Public/Extensions/RunAnywhere+VAD.d.ts.map +0 -1
- package/dist/Public/Extensions/RunAnywhere+VAD.js +0 -231
- package/dist/Public/Extensions/RunAnywhere+VAD.js.map +0 -1
- package/dist/Public/Extensions/RunAnywhere+VLM.d.ts +0 -58
- package/dist/Public/Extensions/RunAnywhere+VLM.d.ts.map +0 -1
- package/dist/Public/Extensions/RunAnywhere+VLM.js +0 -262
- package/dist/Public/Extensions/RunAnywhere+VLM.js.map +0 -1
- package/dist/Public/Extensions/STTTypes.d.ts +0 -61
- package/dist/Public/Extensions/STTTypes.d.ts.map +0 -1
- package/dist/Public/Extensions/STTTypes.js +0 -16
- package/dist/Public/Extensions/STTTypes.js.map +0 -1
- package/dist/Public/Extensions/TTSTypes.d.ts +0 -31
- package/dist/Public/Extensions/TTSTypes.d.ts.map +0 -1
- package/dist/Public/Extensions/TTSTypes.js +0 -3
- package/dist/Public/Extensions/TTSTypes.js.map +0 -1
- package/dist/Public/Extensions/ToolCallingTypes.d.ts +0 -78
- package/dist/Public/Extensions/ToolCallingTypes.d.ts.map +0 -1
- package/dist/Public/Extensions/ToolCallingTypes.js +0 -8
- package/dist/Public/Extensions/ToolCallingTypes.js.map +0 -1
- package/dist/Public/Extensions/VADTypes.d.ts +0 -30
- package/dist/Public/Extensions/VADTypes.d.ts.map +0 -1
- package/dist/Public/Extensions/VADTypes.js +0 -8
- package/dist/Public/Extensions/VADTypes.js.map +0 -1
- package/dist/Public/Extensions/VLMTypes.d.ts +0 -56
- package/dist/Public/Extensions/VLMTypes.d.ts.map +0 -1
- package/dist/Public/Extensions/VLMTypes.js +0 -24
- package/dist/Public/Extensions/VLMTypes.js.map +0 -1
- package/dist/types/LLMTypes.d.ts +0 -48
- package/dist/types/LLMTypes.d.ts.map +0 -1
- package/dist/types/LLMTypes.js +0 -8
- package/dist/types/LLMTypes.js.map +0 -1
- package/dist/workers/vlm-worker.d.ts +0 -9
- package/dist/workers/vlm-worker.d.ts.map +0 -1
- package/dist/workers/vlm-worker.js +0 -10
- package/dist/workers/vlm-worker.js.map +0 -1
- package/wasm/racommons-webgpu.js +0 -156
- package/wasm/racommons-webgpu.wasm +0 -0
- package/wasm/racommons.js +0 -126
- package/wasm/racommons.wasm +0 -0
- package/wasm/sherpa/sherpa-onnx-asr.js +0 -1538
- package/wasm/sherpa/sherpa-onnx-glue-original.js +0 -19
- package/wasm/sherpa/sherpa-onnx-glue.js +0 -17
- package/wasm/sherpa/sherpa-onnx-tts.js +0 -657
- package/wasm/sherpa/sherpa-onnx-vad.js +0 -337
- package/wasm/sherpa/sherpa-onnx-wave.js +0 -88
- package/wasm/sherpa/sherpa-onnx.wasm +0 -0
|
@@ -16,181 +16,90 @@
|
|
|
16
16
|
* console.log('Response:', result.response);
|
|
17
17
|
*/
|
|
18
18
|
import { RunAnywhere } from '../RunAnywhere';
|
|
19
|
-
import { WASMBridge } from '../../Foundation/WASMBridge';
|
|
20
19
|
import { SDKError } from '../../Foundation/ErrorTypes';
|
|
21
20
|
import { SDKLogger } from '../../Foundation/SDKLogger';
|
|
22
|
-
import { EventBus } from '../../Foundation/EventBus';
|
|
23
|
-
import { SDKEventType } from '../../types/enums';
|
|
24
21
|
export { PipelineState } from './VoiceAgentTypes';
|
|
25
22
|
const logger = new SDKLogger('VoiceAgent');
|
|
26
23
|
// ---------------------------------------------------------------------------
|
|
27
24
|
// VoiceAgent Instance
|
|
28
25
|
// ---------------------------------------------------------------------------
|
|
26
|
+
/**
|
|
27
|
+
* VoiceAgentSession orchestrates the complete voice pipeline (VAD → STT → LLM → TTS).
|
|
28
|
+
*
|
|
29
|
+
* TODO: Refactor to use the ExtensionPoint/provider pattern.
|
|
30
|
+
* The previous implementation called rac_voice_agent_* C functions via WASMBridge,
|
|
31
|
+
* which has been removed from the core package. Each backend package (e.g.
|
|
32
|
+
* @runanywhere/web-llamacpp) should register a VoiceAgent provider through
|
|
33
|
+
* ExtensionPoint so that this session can delegate to it.
|
|
34
|
+
*/
|
|
29
35
|
export class VoiceAgentSession {
|
|
30
|
-
|
|
31
|
-
eventCallbackPtr = 0;
|
|
36
|
+
_handle;
|
|
32
37
|
constructor(handle) {
|
|
33
|
-
this.
|
|
38
|
+
this._handle = handle;
|
|
34
39
|
}
|
|
35
40
|
/**
|
|
36
41
|
* Load models for all components.
|
|
42
|
+
*
|
|
43
|
+
* TODO: Delegate to backend provider via ExtensionPoint.
|
|
37
44
|
*/
|
|
38
45
|
async loadModels(models) {
|
|
39
|
-
|
|
40
|
-
if (models.stt) {
|
|
46
|
+
if (models.stt)
|
|
41
47
|
logger.info(`Loading STT model: ${models.stt.id}`);
|
|
42
|
-
|
|
43
|
-
const idPtr = bridge.allocString(models.stt.id);
|
|
44
|
-
const namePtr = bridge.allocString(models.stt.name ?? models.stt.id);
|
|
45
|
-
try {
|
|
46
|
-
bridge.callWithCheck('rac_voice_agent_load_stt_model', ['number', 'number', 'number', 'number'], [this.handle, pathPtr, idPtr, namePtr]);
|
|
47
|
-
}
|
|
48
|
-
finally {
|
|
49
|
-
bridge.free(pathPtr);
|
|
50
|
-
bridge.free(idPtr);
|
|
51
|
-
bridge.free(namePtr);
|
|
52
|
-
}
|
|
53
|
-
}
|
|
54
|
-
if (models.llm) {
|
|
48
|
+
if (models.llm)
|
|
55
49
|
logger.info(`Loading LLM model: ${models.llm.id}`);
|
|
56
|
-
|
|
57
|
-
const idPtr = bridge.allocString(models.llm.id);
|
|
58
|
-
const namePtr = bridge.allocString(models.llm.name ?? models.llm.id);
|
|
59
|
-
try {
|
|
60
|
-
bridge.callWithCheck('rac_voice_agent_load_llm_model', ['number', 'number', 'number', 'number'], [this.handle, pathPtr, idPtr, namePtr]);
|
|
61
|
-
}
|
|
62
|
-
finally {
|
|
63
|
-
bridge.free(pathPtr);
|
|
64
|
-
bridge.free(idPtr);
|
|
65
|
-
bridge.free(namePtr);
|
|
66
|
-
}
|
|
67
|
-
}
|
|
68
|
-
if (models.tts) {
|
|
50
|
+
if (models.tts)
|
|
69
51
|
logger.info(`Loading TTS voice: ${models.tts.id}`);
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
const namePtr = bridge.allocString(models.tts.name ?? models.tts.id);
|
|
73
|
-
try {
|
|
74
|
-
bridge.callWithCheck('rac_voice_agent_load_tts_voice', ['number', 'number', 'number', 'number'], [this.handle, pathPtr, idPtr, namePtr]);
|
|
75
|
-
}
|
|
76
|
-
finally {
|
|
77
|
-
bridge.free(pathPtr);
|
|
78
|
-
bridge.free(idPtr);
|
|
79
|
-
bridge.free(namePtr);
|
|
80
|
-
}
|
|
81
|
-
}
|
|
82
|
-
// Initialize with loaded models
|
|
83
|
-
bridge.callWithCheck('rac_voice_agent_initialize_with_loaded_models', ['number'], [this.handle]);
|
|
84
|
-
logger.info('VoiceAgent initialized with loaded models');
|
|
52
|
+
// TODO: Invoke backend-specific voice agent provider to load models.
|
|
53
|
+
throw SDKError.componentNotReady('VoiceAgent', 'No WASM backend registered — use a backend package (e.g. @runanywhere/web-llamacpp)');
|
|
85
54
|
}
|
|
86
55
|
/**
|
|
87
|
-
* Process a complete voice turn (audio in
|
|
56
|
+
* Process a complete voice turn (audio in → text response + audio out).
|
|
57
|
+
*
|
|
58
|
+
* TODO: Delegate to backend provider via ExtensionPoint.
|
|
88
59
|
*/
|
|
89
|
-
async processVoiceTurn(
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
const audioPtr = m._malloc(audioData.length);
|
|
93
|
-
bridge.writeBytes(audioData, audioPtr);
|
|
94
|
-
// rac_voice_agent_result_t: { speech_detected, transcription, response, synthesized_audio, audio_size }
|
|
95
|
-
const resultSize = m._rac_wasm_sizeof_voice_agent_result();
|
|
96
|
-
const resultPtr = m._malloc(resultSize);
|
|
97
|
-
try {
|
|
98
|
-
bridge.callWithCheck('rac_voice_agent_process_voice_turn', ['number', 'number', 'number', 'number'], [this.handle, audioPtr, audioData.length, resultPtr]);
|
|
99
|
-
const speechDetected = m.getValue(resultPtr, 'i32') === 1;
|
|
100
|
-
const transcriptionPtr = m.getValue(resultPtr + 4, '*');
|
|
101
|
-
const responsePtr = m.getValue(resultPtr + 8, '*');
|
|
102
|
-
const audioDataPtr = m.getValue(resultPtr + 12, '*');
|
|
103
|
-
const audioSize = m.getValue(resultPtr + 16, 'i32');
|
|
104
|
-
const result = {
|
|
105
|
-
speechDetected,
|
|
106
|
-
transcription: transcriptionPtr ? bridge.readString(transcriptionPtr) : undefined,
|
|
107
|
-
response: responsePtr ? bridge.readString(responsePtr) : undefined,
|
|
108
|
-
};
|
|
109
|
-
if (audioDataPtr && audioSize > 0) {
|
|
110
|
-
const numSamples = audioSize / 4;
|
|
111
|
-
result.synthesizedAudio = bridge.readFloat32Array(audioDataPtr, numSamples);
|
|
112
|
-
}
|
|
113
|
-
// Free C result
|
|
114
|
-
bridge.callFunction('rac_voice_agent_result_free', null, ['number'], [resultPtr]);
|
|
115
|
-
EventBus.shared.emit('voice.turnCompleted', SDKEventType.Voice, {
|
|
116
|
-
speechDetected,
|
|
117
|
-
transcription: result.transcription ?? '',
|
|
118
|
-
response: result.response ?? '',
|
|
119
|
-
});
|
|
120
|
-
return result;
|
|
121
|
-
}
|
|
122
|
-
finally {
|
|
123
|
-
m._free(audioPtr);
|
|
124
|
-
}
|
|
60
|
+
async processVoiceTurn(_audioData) {
|
|
61
|
+
// TODO: Invoke backend-specific voice agent provider.
|
|
62
|
+
throw SDKError.componentNotReady('VoiceAgent', 'No WASM backend registered — use a backend package (e.g. @runanywhere/web-llamacpp)');
|
|
125
63
|
}
|
|
126
|
-
/**
|
|
64
|
+
/**
|
|
65
|
+
* Check if the voice agent is ready.
|
|
66
|
+
*
|
|
67
|
+
* TODO: Delegate to backend provider via ExtensionPoint.
|
|
68
|
+
*/
|
|
127
69
|
get isReady() {
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
const outPtr = m._malloc(4);
|
|
131
|
-
try {
|
|
132
|
-
bridge.callFunction('rac_voice_agent_is_ready', 'number', ['number', 'number'], [this.handle, outPtr]);
|
|
133
|
-
return m.getValue(outPtr, 'i32') === 1;
|
|
134
|
-
}
|
|
135
|
-
finally {
|
|
136
|
-
m._free(outPtr);
|
|
137
|
-
}
|
|
70
|
+
// TODO: Query backend provider readiness.
|
|
71
|
+
return false;
|
|
138
72
|
}
|
|
139
|
-
/**
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
bridge.callWithCheck('rac_voice_agent_transcribe', ['number', 'number', 'number', 'number'], [this.handle, audioPtr, audioData.length, outPtr]);
|
|
148
|
-
const textPtr = m.getValue(outPtr, '*');
|
|
149
|
-
const text = bridge.readString(textPtr);
|
|
150
|
-
if (textPtr)
|
|
151
|
-
m._free(textPtr);
|
|
152
|
-
return text;
|
|
153
|
-
}
|
|
154
|
-
finally {
|
|
155
|
-
m._free(audioPtr);
|
|
156
|
-
m._free(outPtr);
|
|
157
|
-
}
|
|
73
|
+
/**
|
|
74
|
+
* Transcribe audio without the full pipeline.
|
|
75
|
+
*
|
|
76
|
+
* TODO: Delegate to backend provider via ExtensionPoint.
|
|
77
|
+
*/
|
|
78
|
+
async transcribe(_audioData) {
|
|
79
|
+
// TODO: Invoke backend-specific STT provider.
|
|
80
|
+
throw SDKError.componentNotReady('VoiceAgent', 'No WASM backend registered — use a backend package (e.g. @runanywhere/web-llamacpp)');
|
|
158
81
|
}
|
|
159
|
-
/**
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
const textPtr = m.getValue(outPtr, '*');
|
|
168
|
-
const text = bridge.readString(textPtr);
|
|
169
|
-
if (textPtr)
|
|
170
|
-
m._free(textPtr);
|
|
171
|
-
return text;
|
|
172
|
-
}
|
|
173
|
-
finally {
|
|
174
|
-
bridge.free(promptPtr);
|
|
175
|
-
m._free(outPtr);
|
|
176
|
-
}
|
|
82
|
+
/**
|
|
83
|
+
* Generate LLM response without the full pipeline.
|
|
84
|
+
*
|
|
85
|
+
* TODO: Delegate to backend provider via ExtensionPoint.
|
|
86
|
+
*/
|
|
87
|
+
async generateResponse(_prompt) {
|
|
88
|
+
// TODO: Invoke backend-specific LLM provider.
|
|
89
|
+
throw SDKError.componentNotReady('VoiceAgent', 'No WASM backend registered — use a backend package (e.g. @runanywhere/web-llamacpp)');
|
|
177
90
|
}
|
|
178
|
-
/**
|
|
91
|
+
/** Get the native handle (used by backend providers). */
|
|
92
|
+
get handle() {
|
|
93
|
+
return this._handle;
|
|
94
|
+
}
|
|
95
|
+
/**
|
|
96
|
+
* Destroy the voice agent session.
|
|
97
|
+
*
|
|
98
|
+
* TODO: Delegate cleanup to backend provider via ExtensionPoint.
|
|
99
|
+
*/
|
|
179
100
|
destroy() {
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
WASMBridge.shared.module.removeFunction(this.eventCallbackPtr);
|
|
183
|
-
}
|
|
184
|
-
catch { /* ignore */ }
|
|
185
|
-
this.eventCallbackPtr = 0;
|
|
186
|
-
}
|
|
187
|
-
if (this.handle !== 0) {
|
|
188
|
-
try {
|
|
189
|
-
WASMBridge.shared.callFunction('rac_voice_agent_destroy', null, ['number'], [this.handle]);
|
|
190
|
-
}
|
|
191
|
-
catch { /* ignore */ }
|
|
192
|
-
this.handle = 0;
|
|
193
|
-
}
|
|
101
|
+
// TODO: Invoke backend-specific cleanup.
|
|
102
|
+
this._handle = 0;
|
|
194
103
|
}
|
|
195
104
|
}
|
|
196
105
|
// ---------------------------------------------------------------------------
|
|
@@ -200,23 +109,16 @@ export const VoiceAgent = {
|
|
|
200
109
|
/**
|
|
201
110
|
* Create a standalone VoiceAgent session.
|
|
202
111
|
* The agent manages its own STT, LLM, TTS, and VAD components.
|
|
112
|
+
*
|
|
113
|
+
* TODO: Delegate to backend provider via ExtensionPoint.
|
|
203
114
|
*/
|
|
204
115
|
async create() {
|
|
205
116
|
if (!RunAnywhere.isInitialized) {
|
|
206
117
|
throw SDKError.notInitialized();
|
|
207
118
|
}
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
const result = bridge.callFunction('rac_voice_agent_create_standalone', 'number', ['number'], [handlePtr]);
|
|
212
|
-
if (result !== 0) {
|
|
213
|
-
m._free(handlePtr);
|
|
214
|
-
throw SDKError.fromRACResult(result, 'rac_voice_agent_create_standalone');
|
|
215
|
-
}
|
|
216
|
-
const handle = m.getValue(handlePtr, 'i32');
|
|
217
|
-
m._free(handlePtr);
|
|
218
|
-
logger.info('VoiceAgent session created');
|
|
219
|
-
return new VoiceAgentSession(handle);
|
|
119
|
+
// TODO: Look up a registered VoiceAgent provider from ExtensionPoint
|
|
120
|
+
// and delegate session creation to it.
|
|
121
|
+
throw SDKError.componentNotReady('VoiceAgent', 'No WASM backend registered — use a backend package (e.g. @runanywhere/web-llamacpp)');
|
|
220
122
|
},
|
|
221
123
|
};
|
|
222
124
|
//# sourceMappingURL=RunAnywhere+VoiceAgent.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"RunAnywhere+VoiceAgent.js","sourceRoot":"","sources":["../../../src/Public/Extensions/RunAnywhere+VoiceAgent.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"RunAnywhere+VoiceAgent.js","sourceRoot":"","sources":["../../../src/Public/Extensions/RunAnywhere+VoiceAgent.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,OAAO,EAAE,QAAQ,EAAE,MAAM,6BAA6B,CAAC;AACvD,OAAO,EAAE,SAAS,EAAE,MAAM,4BAA4B,CAAC;AAGvD,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAGlD,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC,YAAY,CAAC,CAAC;AAE3C,8EAA8E;AAC9E,sBAAsB;AACtB,8EAA8E;AAE9E;;;;;;;;GAQG;AACH,MAAM,OAAO,iBAAiB;IACpB,OAAO,CAAS;IAExB,YAAY,MAAc;QACxB,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;IACxB,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,UAAU,CAAC,MAAwB;QACvC,IAAI,MAAM,CAAC,GAAG;YAAE,MAAM,CAAC,IAAI,CAAC,sBAAsB,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC;QACnE,IAAI,MAAM,CAAC,GAAG;YAAE,MAAM,CAAC,IAAI,CAAC,sBAAsB,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC;QACnE,IAAI,MAAM,CAAC,GAAG;YAAE,MAAM,CAAC,IAAI,CAAC,sBAAsB,MAAM,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC;QAEnE,qEAAqE;QACrE,MAAM,QAAQ,CAAC,iBAAiB,CAAC,YAAY,EAAE,qFAAqF,CAAC,CAAC;IACxI,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,gBAAgB,CAAC,UAAsB;QAC3C,sDAAsD;QACtD,MAAM,QAAQ,CAAC,iBAAiB,CAAC,YAAY,EAAE,qFAAqF,CAAC,CAAC;IACxI,CAAC;IAED;;;;OAIG;IACH,IAAI,OAAO;QACT,0CAA0C;QAC1C,OAAO,KAAK,CAAC;IACf,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,UAAU,CAAC,UAAsB;QACrC,8CAA8C;QAC9C,MAAM,QAAQ,CAAC,iBAAiB,CAAC,YAAY,EAAE,qFAAqF,CAAC,CAAC;IACxI,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,gBAAgB,CAAC,OAAe;QACpC,8CAA8C;QAC9C,MAAM,QAAQ,CAAC,iBAAiB,CAAC,YAAY,EAAE,qFAAqF,CAAC,CAAC;IACxI,CAAC;IAED,yDAAyD;IACzD,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,OAAO,CAAC;IACtB,CAAC;IAED;;;;OAIG;IACH,OAAO;QACL,yCAAyC;QACzC,IAAI,CAAC,OAAO,GAAG,CAAC,CAAC;IACnB,CAAC;CACF;AAED,8EAA8E;AAC9E,qBAAqB;AACrB,8EAA8E;AAE9E,MAAM,CAAC,MAAM,UAAU,GAAG;IACxB;;;;;OAKG;IACH,KAAK,CAAC,MAAM;QACV,IAAI,CAAC,WAAW,CAAC,aAAa,EAAE,CAAC;YAC/B,MAAM,QAAQ,CAAC,cAAc,EAAE,CAAC;QAClC,CAAC;QAED,qEAAqE;QACrE,uCAAuC;QACvC,MAAM,QAAQ,CAAC,iBAAiB,CAAC,YAAY,EAAE,qFAAqF,CAAC,CAAC;IACxI,CAAC;CACF,CAAC"}
|
|
@@ -3,10 +3,9 @@
|
|
|
3
3
|
*
|
|
4
4
|
* High-level streaming voice orchestrator: STT -> LLM (streaming) -> TTS.
|
|
5
5
|
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
* stage progresses.
|
|
6
|
+
* Uses runtime capability lookups via ExtensionPoint, so it doesn't import
|
|
7
|
+
* backend packages directly. Requires both @runanywhere/web-llamacpp and
|
|
8
|
+
* @runanywhere/web-onnx to be registered.
|
|
10
9
|
*
|
|
11
10
|
* Usage:
|
|
12
11
|
* ```typescript
|
|
@@ -22,41 +21,16 @@
|
|
|
22
21
|
* onSynthesisComplete: (audio, sr) => playAudio(audio, sr),
|
|
23
22
|
* });
|
|
24
23
|
* ```
|
|
25
|
-
*
|
|
26
|
-
* The pipeline does NOT handle audio capture, VAD, or playback — those are
|
|
27
|
-
* app-level concerns (mic control, VAD thresholds, speaker output).
|
|
28
24
|
*/
|
|
29
25
|
import { PipelineState } from './VoiceAgentTypes';
|
|
30
26
|
import type { VoicePipelineCallbacks, VoicePipelineOptions, VoicePipelineTurnResult } from './VoicePipelineTypes';
|
|
31
27
|
export { PipelineState } from './VoiceAgentTypes';
|
|
32
28
|
export type { VoicePipelineCallbacks, VoicePipelineOptions, VoicePipelineTurnResult, } from './VoicePipelineTypes';
|
|
33
|
-
/**
|
|
34
|
-
* Streaming voice pipeline that orchestrates STT -> LLM -> TTS.
|
|
35
|
-
*
|
|
36
|
-
* Create one instance and reuse it across conversation turns. Each call to
|
|
37
|
-
* `processTurn()` runs the full pipeline once and returns the result.
|
|
38
|
-
*
|
|
39
|
-
* The LLM step uses streaming generation, so the `onResponseToken` callback
|
|
40
|
-
* fires for each token — enabling real-time UI updates.
|
|
41
|
-
*/
|
|
42
29
|
export declare class VoicePipeline {
|
|
43
30
|
private _cancelGeneration;
|
|
44
31
|
private _state;
|
|
45
|
-
/** Current pipeline state. */
|
|
46
32
|
get state(): PipelineState;
|
|
47
|
-
/**
|
|
48
|
-
* Run a complete voice turn: audio in -> transcription -> LLM response -> TTS audio.
|
|
49
|
-
*
|
|
50
|
-
* @param audioData Float32Array of PCM samples (mono, typically 16 kHz).
|
|
51
|
-
* @param options LLM / TTS / STT configuration for this turn.
|
|
52
|
-
* @param callbacks Streaming callbacks for real-time UI updates.
|
|
53
|
-
* @returns The full turn result once all stages complete.
|
|
54
|
-
*/
|
|
55
33
|
processTurn(audioData: Float32Array, options?: VoicePipelineOptions, callbacks?: VoicePipelineCallbacks): Promise<VoicePipelineTurnResult>;
|
|
56
|
-
/**
|
|
57
|
-
* Cancel in-progress LLM generation.
|
|
58
|
-
* Safe to call at any time — no-ops if nothing is in progress.
|
|
59
|
-
*/
|
|
60
34
|
cancel(): void;
|
|
61
35
|
private transition;
|
|
62
36
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"RunAnywhere+VoicePipeline.d.ts","sourceRoot":"","sources":["../../../src/Public/Extensions/RunAnywhere+VoicePipeline.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"RunAnywhere+VoicePipeline.d.ts","sourceRoot":"","sources":["../../../src/Public/Extensions/RunAnywhere+VoicePipeline.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAIH,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAClD,OAAO,KAAK,EACV,sBAAsB,EACtB,oBAAoB,EACpB,uBAAuB,EACxB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAClD,YAAY,EACV,sBAAsB,EACtB,oBAAoB,EACpB,uBAAuB,GACxB,MAAM,sBAAsB,CAAC;AAyD9B,qBAAa,aAAa;IACxB,OAAO,CAAC,iBAAiB,CAA6B;IACtD,OAAO,CAAC,MAAM,CAAqC;IAEnD,IAAI,KAAK,IAAI,aAAa,CAEzB;IAEK,WAAW,CACf,SAAS,EAAE,YAAY,EACvB,OAAO,CAAC,EAAE,oBAAoB,EAC9B,SAAS,CAAC,EAAE,sBAAsB,GACjC,OAAO,CAAC,uBAAuB,CAAC;IAmGnC,MAAM,IAAI,IAAI;IAQd,OAAO,CAAC,UAAU;CAInB"}
|
|
@@ -3,10 +3,9 @@
|
|
|
3
3
|
*
|
|
4
4
|
* High-level streaming voice orchestrator: STT -> LLM (streaming) -> TTS.
|
|
5
5
|
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
* stage progresses.
|
|
6
|
+
* Uses runtime capability lookups via ExtensionPoint, so it doesn't import
|
|
7
|
+
* backend packages directly. Requires both @runanywhere/web-llamacpp and
|
|
8
|
+
* @runanywhere/web-onnx to be registered.
|
|
10
9
|
*
|
|
11
10
|
* Usage:
|
|
12
11
|
* ```typescript
|
|
@@ -22,18 +21,25 @@
|
|
|
22
21
|
* onSynthesisComplete: (audio, sr) => playAudio(audio, sr),
|
|
23
22
|
* });
|
|
24
23
|
* ```
|
|
25
|
-
*
|
|
26
|
-
* The pipeline does NOT handle audio capture, VAD, or playback — those are
|
|
27
|
-
* app-level concerns (mic control, VAD thresholds, speaker output).
|
|
28
24
|
*/
|
|
29
|
-
import { STT } from './RunAnywhere+STT';
|
|
30
|
-
import { TextGeneration } from './RunAnywhere+TextGeneration';
|
|
31
|
-
import { TTS } from './RunAnywhere+TTS';
|
|
32
25
|
import { SDKLogger } from '../../Foundation/SDKLogger';
|
|
26
|
+
import { ExtensionPoint, ServiceKey } from '../../Infrastructure/ExtensionPoint';
|
|
33
27
|
import { PipelineState } from './VoiceAgentTypes';
|
|
34
28
|
export { PipelineState } from './VoiceAgentTypes';
|
|
35
29
|
const logger = new SDKLogger('VoicePipeline');
|
|
36
30
|
// ---------------------------------------------------------------------------
|
|
31
|
+
// Dynamic backend access helpers (typed via ExtensionPoint service registry)
|
|
32
|
+
// ---------------------------------------------------------------------------
|
|
33
|
+
function requireSTT() {
|
|
34
|
+
return ExtensionPoint.requireService(ServiceKey.STT, '@runanywhere/web-onnx');
|
|
35
|
+
}
|
|
36
|
+
function requireTextGeneration() {
|
|
37
|
+
return ExtensionPoint.requireService(ServiceKey.TextGeneration, '@runanywhere/web-llamacpp');
|
|
38
|
+
}
|
|
39
|
+
function requireTTS() {
|
|
40
|
+
return ExtensionPoint.requireService(ServiceKey.TTS, '@runanywhere/web-onnx');
|
|
41
|
+
}
|
|
42
|
+
// ---------------------------------------------------------------------------
|
|
37
43
|
// Default options
|
|
38
44
|
// ---------------------------------------------------------------------------
|
|
39
45
|
const DEFAULT_OPTIONS = {
|
|
@@ -46,38 +52,21 @@ const DEFAULT_OPTIONS = {
|
|
|
46
52
|
// ---------------------------------------------------------------------------
|
|
47
53
|
// VoicePipeline
|
|
48
54
|
// ---------------------------------------------------------------------------
|
|
49
|
-
/**
|
|
50
|
-
* Streaming voice pipeline that orchestrates STT -> LLM -> TTS.
|
|
51
|
-
*
|
|
52
|
-
* Create one instance and reuse it across conversation turns. Each call to
|
|
53
|
-
* `processTurn()` runs the full pipeline once and returns the result.
|
|
54
|
-
*
|
|
55
|
-
* The LLM step uses streaming generation, so the `onResponseToken` callback
|
|
56
|
-
* fires for each token — enabling real-time UI updates.
|
|
57
|
-
*/
|
|
58
55
|
export class VoicePipeline {
|
|
59
56
|
_cancelGeneration = null;
|
|
60
57
|
_state = PipelineState.Idle;
|
|
61
|
-
/** Current pipeline state. */
|
|
62
58
|
get state() {
|
|
63
59
|
return this._state;
|
|
64
60
|
}
|
|
65
|
-
/**
|
|
66
|
-
* Run a complete voice turn: audio in -> transcription -> LLM response -> TTS audio.
|
|
67
|
-
*
|
|
68
|
-
* @param audioData Float32Array of PCM samples (mono, typically 16 kHz).
|
|
69
|
-
* @param options LLM / TTS / STT configuration for this turn.
|
|
70
|
-
* @param callbacks Streaming callbacks for real-time UI updates.
|
|
71
|
-
* @returns The full turn result once all stages complete.
|
|
72
|
-
*/
|
|
73
61
|
async processTurn(audioData, options, callbacks) {
|
|
74
62
|
const opts = { ...DEFAULT_OPTIONS, ...options };
|
|
75
63
|
const totalStart = performance.now();
|
|
76
|
-
//
|
|
64
|
+
// Step 1: STT
|
|
77
65
|
this.transition(PipelineState.ProcessingSTT, callbacks);
|
|
78
66
|
const sttStart = performance.now();
|
|
79
67
|
logger.info(`STT: ${(audioData.length / opts.sampleRate).toFixed(1)}s of audio`);
|
|
80
|
-
const
|
|
68
|
+
const stt = requireSTT();
|
|
69
|
+
const sttResult = await stt.transcribe(audioData, {
|
|
81
70
|
sampleRate: opts.sampleRate,
|
|
82
71
|
});
|
|
83
72
|
const sttMs = performance.now() - sttStart;
|
|
@@ -92,10 +81,11 @@ export class VoicePipeline {
|
|
|
92
81
|
timing: { sttMs, llmMs: 0, ttsMs: 0, totalMs: performance.now() - totalStart },
|
|
93
82
|
};
|
|
94
83
|
}
|
|
95
|
-
//
|
|
84
|
+
// Step 2: LLM (streaming)
|
|
96
85
|
this.transition(PipelineState.GeneratingResponse, callbacks);
|
|
97
86
|
const llmStart = performance.now();
|
|
98
|
-
const
|
|
87
|
+
const textGen = requireTextGeneration();
|
|
88
|
+
const { stream, result: llmResultPromise, cancel } = await textGen.generateStream(userText, {
|
|
99
89
|
maxTokens: opts.maxTokens,
|
|
100
90
|
temperature: opts.temperature,
|
|
101
91
|
systemPrompt: opts.systemPrompt,
|
|
@@ -121,16 +111,17 @@ export class VoicePipeline {
|
|
|
121
111
|
llmResult,
|
|
122
112
|
};
|
|
123
113
|
}
|
|
124
|
-
//
|
|
114
|
+
// Step 3: TTS
|
|
125
115
|
this.transition(PipelineState.PlayingTTS, callbacks);
|
|
126
116
|
const ttsStart = performance.now();
|
|
127
|
-
const
|
|
117
|
+
const tts = requireTTS();
|
|
118
|
+
const ttsResult = await tts.synthesize(fullResponse.trim(), {
|
|
128
119
|
speed: opts.ttsSpeed,
|
|
129
120
|
});
|
|
130
121
|
const ttsMs = performance.now() - ttsStart;
|
|
131
122
|
logger.info(`TTS complete: ${ttsResult.durationMs}ms audio in ${ttsResult.processingTimeMs}ms`);
|
|
132
123
|
callbacks?.onSynthesisComplete?.(ttsResult.audioData, ttsResult.sampleRate, ttsResult);
|
|
133
|
-
//
|
|
124
|
+
// Done
|
|
134
125
|
this.transition(PipelineState.Idle, callbacks);
|
|
135
126
|
return {
|
|
136
127
|
transcription: userText,
|
|
@@ -146,10 +137,6 @@ export class VoicePipeline {
|
|
|
146
137
|
llmResult,
|
|
147
138
|
};
|
|
148
139
|
}
|
|
149
|
-
/**
|
|
150
|
-
* Cancel in-progress LLM generation.
|
|
151
|
-
* Safe to call at any time — no-ops if nothing is in progress.
|
|
152
|
-
*/
|
|
153
140
|
cancel() {
|
|
154
141
|
if (this._cancelGeneration) {
|
|
155
142
|
this._cancelGeneration();
|
|
@@ -157,9 +144,6 @@ export class VoicePipeline {
|
|
|
157
144
|
logger.info('Generation cancelled');
|
|
158
145
|
}
|
|
159
146
|
}
|
|
160
|
-
// ---------------------------------------------------------------------------
|
|
161
|
-
// Internals
|
|
162
|
-
// ---------------------------------------------------------------------------
|
|
163
147
|
transition(newState, callbacks) {
|
|
164
148
|
this._state = newState;
|
|
165
149
|
callbacks?.onStateChange?.(newState);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"RunAnywhere+VoicePipeline.js","sourceRoot":"","sources":["../../../src/Public/Extensions/RunAnywhere+VoicePipeline.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"RunAnywhere+VoicePipeline.js","sourceRoot":"","sources":["../../../src/Public/Extensions/RunAnywhere+VoicePipeline.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,4BAA4B,CAAC;AACvD,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,MAAM,qCAAqC,CAAC;AACjF,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAOlD,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAOlD,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC,eAAe,CAAC,CAAC;AAsB9C,8EAA8E;AAC9E,6EAA6E;AAC7E,8EAA8E;AAE9E,SAAS,UAAU;IACjB,OAAO,cAAc,CAAC,cAAc,CAAa,UAAU,CAAC,GAAG,EAAE,uBAAuB,CAAC,CAAC;AAC5F,CAAC;AAED,SAAS,qBAAqB;IAC5B,OAAO,cAAc,CAAC,cAAc,CAAwB,UAAU,CAAC,cAAc,EAAE,2BAA2B,CAAC,CAAC;AACtH,CAAC;AAED,SAAS,UAAU;IACjB,OAAO,cAAc,CAAC,cAAc,CAAa,UAAU,CAAC,GAAG,EAAE,uBAAuB,CAAC,CAAC;AAC5F,CAAC;AAED,8EAA8E;AAC9E,kBAAkB;AAClB,8EAA8E;AAE9E,MAAM,eAAe,GAAmC;IACtD,SAAS,EAAE,EAAE;IACb,WAAW,EAAE,GAAG;IAChB,YAAY,EACV,gFAAgF;IAClF,QAAQ,EAAE,GAAG;IACb,UAAU,EAAE,KAAK;CAClB,CAAC;AAEF,8EAA8E;AAC9E,gBAAgB;AAChB,8EAA8E;AAE9E,MAAM,OAAO,aAAa;IAChB,iBAAiB,GAAwB,IAAI,CAAC;IAC9C,MAAM,GAAkB,aAAa,CAAC,IAAI,CAAC;IAEnD,IAAI,KAAK;QACP,OAAO,IAAI,CAAC,MAAM,CAAC;IACrB,CAAC;IAED,KAAK,CAAC,WAAW,CACf,SAAuB,EACvB,OAA8B,EAC9B,SAAkC;QAElC,MAAM,IAAI,GAAG,EAAE,GAAG,eAAe,EAAE,GAAG,OAAO,EAAE,CAAC;QAChD,MAAM,UAAU,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QAErC,cAAc;QACd,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,aAAa,EAAE,SAAS,CAAC,CAAC;QAExD,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QACnC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;QAEjF,MAAM,GAAG,GAAG,UAAU,EAAE,CAAC;QACzB,MAAM,SAAS,GAAG,MAAM,GAAG,CAAC,UAAU,CAAC,SAAS,EAAE;YAChD,UAAU,EAAE,IAAI,CAAC,UAAU;SAC5B,CAAC,CAAC;QACH,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,QAAQ,CAAC;QAC3C,MAAM,QAAQ,GAAG,SAAS,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;QAEvC,MAAM,CAAC,IAAI,CAAC,kBAAkB,QAAQ,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QACnE,SAAS,EAAE,eAAe,EAAE,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;QAElD,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;YAC/C,OAAO;gBACL,aAAa,EAAE,EAAE;gBACjB,QAAQ,EAAE,EAAE;gBACZ,MAAM,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,WAAW,CAAC,GAAG,EAAE,GAAG,UAAU,EAAE;aAC/E,CAAC;QACJ,CAAC;QAED,0BAA0B;QAC1B,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,kBAAkB,EAAE,SAAS,CAAC,CAAC;QAE7D,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QACnC,MAAM,OAAO,GAAG,qBAAqB,EAAE,CAAC;QACxC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,gBAAgB,EAAE,MAAM,EAAE,GAAG,MAAM,OAAO,CAAC,cAAc,CAC/E,QAAQ,EACR;YACE,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,WAAW,EAAE,IAAI,CAAC,WAAW;YAC7B,YAAY,EAAE,IAAI,CAAC,YAAY;SAChC,CACF,CAAC;QACF,IAAI,CAAC,iBAAiB,GAAG,MAAM,CAAC;QAEhC,IAAI,WAAW,GAAG,EAAE,CAAC;QACrB,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YACjC,WAAW,IAAI,KAAK,CAAC;YACrB,SAAS,EAAE,eAAe,EAAE,CAAC,KAAK,EAAE,WAAW,CAAC,CAAC;QACnD,CAAC;QACD,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC;QAE9B,MAAM,SAAS,GAAG,MAAM,gBAAgB,CAAC;QACzC,MAAM,YAAY,GAAG,SAAS,CAAC,IAAI,IAAI,WAAW,CAAC;QACnD,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,QAAQ,CAAC;QAE3C,MAAM,CAAC,IAAI,CAAC,iBAAiB,SAAS,CAAC,UAAU,YAAY,SAAS,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;QACnI,SAAS,EAAE,kBAAkB,EAAE,CAAC,YAAY,EAAE,SAAS,CAAC,CAAC;QAEzD,IAAI,CAAC,YAAY,CAAC,IAAI,EAAE,EAAE,CAAC;YACzB,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;YAC/C,OAAO;gBACL,aAAa,EAAE,QAAQ;gBACvB,QAAQ,EAAE,EAAE;gBACZ,MAAM,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,EAAE,WAAW,CAAC,GAAG,EAAE,GAAG,UAAU,EAAE;gBAC3E,SAAS;aACV,CAAC;QACJ,CAAC;QAED,cAAc;QACd,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;QAErD,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,EAAE,CAAC;QACnC,MAAM,GAAG,GAAG,UAAU,EAAE,CAAC;QACzB,MAAM,SAAS,GAAG,MAAM,GAAG,CAAC,UAAU,CAAC,YAAY,CAAC,IAAI,EAAE,EAAE;YAC1D,KAAK,EAAE,IAAI,CAAC,QAAQ;SACrB,CAAC,CAAC;QACH,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,QAAQ,CAAC;QAE3C,MAAM,CAAC,IAAI,CAAC,iBAAiB,SAAS,CAAC,UAAU,eAAe,SAAS,CAAC,gBAAgB,IAAI,CAAC,CAAC;QAChG,SAAS,EAAE,mBAAmB,EAAE,CAAC,SAAS,CAAC,SAAS,EAAE,SAAS,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;QAEvF,OAAO;QACP,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;QAE/C,OAAO;YACL,aAAa,EAAE,QAAQ;YACvB,QAAQ,EAAE,YAAY;YACtB,gBAAgB,EAAE,SAAS,CAAC,SAAS;YACrC,UAAU,EAAE,SAAS,CAAC,UAAU;YAChC,MAAM,EAAE;gBACN,KAAK;gBACL,KAAK;gBACL,KAAK;gBACL,OAAO,EAAE,WAAW,CAAC,GAAG,EAAE,GAAG,UAAU;aACxC;YACD,SAAS;SACV,CAAC;IACJ,CAAC;IAED,MAAM;QACJ,IAAI,IAAI,CAAC,iBAAiB,EAAE,CAAC;YAC3B,IAAI,CAAC,iBAAiB,EAAE,CAAC;YACzB,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC;YAC9B,MAAM,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;QACtC,CAAC;IACH,CAAC;IAEO,UAAU,CAAC,QAAuB,EAAE,SAAkC;QAC5E,IAAI,CAAC,MAAM,GAAG,QAAQ,CAAC;QACvB,SAAS,EAAE,aAAa,EAAE,CAAC,QAAQ,CAAC,CAAC;IACvC,CAAC;CACF"}
|
|
@@ -2,68 +2,59 @@
|
|
|
2
2
|
* RunAnywhere Web SDK - VoicePipeline Types
|
|
3
3
|
*
|
|
4
4
|
* Types for the high-level streaming voice pipeline that orchestrates
|
|
5
|
-
* STT -> LLM (streaming) -> TTS using
|
|
5
|
+
* STT -> LLM (streaming) -> TTS using backend extensions.
|
|
6
|
+
*
|
|
7
|
+
* These types use generic result shapes so the core VoicePipeline doesn't
|
|
8
|
+
* need to import types from backend packages.
|
|
6
9
|
*/
|
|
7
10
|
import { PipelineState } from './VoiceAgentTypes';
|
|
8
|
-
import type { STTTranscriptionResult } from './STTTypes';
|
|
9
|
-
import type { LLMGenerationResult } from '../../types/LLMTypes';
|
|
10
|
-
import type { TTSSynthesisResult } from './TTSTypes';
|
|
11
11
|
export { PipelineState };
|
|
12
|
-
/**
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
*/
|
|
12
|
+
/** Generic STT result shape (matches STTTranscriptionResult). */
|
|
13
|
+
export interface VoicePipelineSTTResult {
|
|
14
|
+
text: string;
|
|
15
|
+
[key: string]: unknown;
|
|
16
|
+
}
|
|
17
|
+
/** Generic LLM result shape (matches LLMGenerationResult). */
|
|
18
|
+
export interface VoicePipelineLLMResult {
|
|
19
|
+
text: string;
|
|
20
|
+
tokensUsed: number;
|
|
21
|
+
tokensPerSecond: number;
|
|
22
|
+
[key: string]: unknown;
|
|
23
|
+
}
|
|
24
|
+
/** Generic TTS result shape (matches TTSSynthesisResult). */
|
|
25
|
+
export interface VoicePipelineTTSResult {
|
|
26
|
+
audioData: Float32Array;
|
|
27
|
+
sampleRate: number;
|
|
28
|
+
durationMs: number;
|
|
29
|
+
processingTimeMs: number;
|
|
30
|
+
[key: string]: unknown;
|
|
31
|
+
}
|
|
18
32
|
export interface VoicePipelineCallbacks {
|
|
19
|
-
/** Fires when the pipeline transitions between stages. */
|
|
20
33
|
onStateChange?: (state: PipelineState) => void;
|
|
21
|
-
|
|
22
|
-
onTranscription?: (text: string, result: STTTranscriptionResult) => void;
|
|
23
|
-
/**
|
|
24
|
-
* Fires for each LLM token during streaming generation.
|
|
25
|
-
* @param token - The individual new token
|
|
26
|
-
* @param accumulated - The full response so far
|
|
27
|
-
*/
|
|
34
|
+
onTranscription?: (text: string, result: VoicePipelineSTTResult) => void;
|
|
28
35
|
onResponseToken?: (token: string, accumulated: string) => void;
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
/** Fires when TTS synthesis is complete with playable audio. */
|
|
32
|
-
onSynthesisComplete?: (audio: Float32Array, sampleRate: number, result: TTSSynthesisResult) => void;
|
|
33
|
-
/** Fires on any error during the pipeline. */
|
|
36
|
+
onResponseComplete?: (text: string, result: VoicePipelineLLMResult) => void;
|
|
37
|
+
onSynthesisComplete?: (audio: Float32Array, sampleRate: number, result: VoicePipelineTTSResult) => void;
|
|
34
38
|
onError?: (error: Error, stage: PipelineState) => void;
|
|
35
39
|
}
|
|
36
|
-
/** Options for a single voice pipeline turn. */
|
|
37
40
|
export interface VoicePipelineOptions {
|
|
38
|
-
/** Max tokens for LLM generation (default: 150). */
|
|
39
41
|
maxTokens?: number;
|
|
40
|
-
/** LLM temperature (default: 0.7). */
|
|
41
42
|
temperature?: number;
|
|
42
|
-
/** System prompt for the LLM (default: conversational assistant). */
|
|
43
43
|
systemPrompt?: string;
|
|
44
|
-
/** TTS speech speed factor (default: 1.0). */
|
|
45
44
|
ttsSpeed?: number;
|
|
46
|
-
/** STT audio sample rate (default: 16000). */
|
|
47
45
|
sampleRate?: number;
|
|
48
46
|
}
|
|
49
|
-
/** Result of a complete voice pipeline turn. */
|
|
50
47
|
export interface VoicePipelineTurnResult {
|
|
51
|
-
/** Transcribed user speech. Empty string if no speech detected. */
|
|
52
48
|
transcription: string;
|
|
53
|
-
/** LLM response text. */
|
|
54
49
|
response: string;
|
|
55
|
-
/** Synthesized audio (PCM Float32 samples). */
|
|
56
50
|
synthesizedAudio?: Float32Array;
|
|
57
|
-
/** Audio sample rate of synthesized audio. */
|
|
58
51
|
sampleRate?: number;
|
|
59
|
-
/** Time in ms for each stage. */
|
|
60
52
|
timing: {
|
|
61
53
|
sttMs: number;
|
|
62
54
|
llmMs: number;
|
|
63
55
|
ttsMs: number;
|
|
64
56
|
totalMs: number;
|
|
65
57
|
};
|
|
66
|
-
|
|
67
|
-
llmResult?: LLMGenerationResult;
|
|
58
|
+
llmResult?: VoicePipelineLLMResult;
|
|
68
59
|
}
|
|
69
60
|
//# sourceMappingURL=VoicePipelineTypes.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"VoicePipelineTypes.d.ts","sourceRoot":"","sources":["../../../src/Public/Extensions/VoicePipelineTypes.ts"],"names":[],"mappings":"AAAA
|
|
1
|
+
{"version":3,"file":"VoicePipelineTypes.d.ts","sourceRoot":"","sources":["../../../src/Public/Extensions/VoicePipelineTypes.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAElD,OAAO,EAAE,aAAa,EAAE,CAAC;AAMzB,iEAAiE;AACjE,MAAM,WAAW,sBAAsB;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED,8DAA8D;AAC9D,MAAM,WAAW,sBAAsB;IACrC,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,eAAe,EAAE,MAAM,CAAC;IACxB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED,6DAA6D;AAC7D,MAAM,WAAW,sBAAsB;IACrC,SAAS,EAAE,YAAY,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,gBAAgB,EAAE,MAAM,CAAC;IACzB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAMD,MAAM,WAAW,sBAAsB;IACrC,aAAa,CAAC,EAAE,CAAC,KAAK,EAAE,aAAa,KAAK,IAAI,CAAC;IAC/C,eAAe,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,sBAAsB,KAAK,IAAI,CAAC;IACzE,eAAe,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,KAAK,IAAI,CAAC;IAC/D,kBAAkB,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,sBAAsB,KAAK,IAAI,CAAC;IAC5E,mBAAmB,CAAC,EAAE,CAAC,KAAK,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,EAAE,sBAAsB,KAAK,IAAI,CAAC;IACxG,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,aAAa,KAAK,IAAI,CAAC;CACxD;AAMD,MAAM,WAAW,oBAAoB;IACnC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAMD,MAAM,WAAW,uBAAuB;IACtC,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;IACjB,gBAAgB,CAAC,EAAE,YAAY,CAAC;IAChC,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,MAAM,EAAE;QACN,KAAK,EAAE,MAAM,CAAC;QACd,KAAK,EAAE,MAAM,CAAC;QACd,KAAK,EAAE,MAAM,CAAC;QACd,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC;IACF,SAAS,CAAC,EAAE,sBAAsB,CAAC;CACpC"}
|