arcane-os 0.94.0 → 0.95.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/README.md +1 -1
- package/browser-runtime/core/client.mjs +195 -0
- package/docs/reference/inventory/package-api.json +1 -1
- package/docs/reference/native-magpie.md +98 -0
- package/docs/reference/native-nemotron.md +204 -0
- package/docs/reference/native-speech.md +145 -1
- package/package.json +5 -1
- package/runtime/arcane/modules/AI.js +114 -14
- package/runtime/arcane/modules/AIProviderRuntime.js +32 -15
- package/runtime/arcane/modules/ContinuousVoiceCapture.js +118 -21
- package/runtime/arcane/modules/NativeSpeechCapture.js +198 -0
- package/runtime/arcane/modules/NativeSpeechPlayback.js +304 -0
- package/runtime/arcane/modules/VoiceCaptureWorklet.js +62 -2
- package/src/core/services/speech.mjs +235 -2
- package/src/local-ai/magpie/build.mjs +36 -0
- package/src/local-ai/magpie/helper.mjs +254 -0
- package/src/local-ai/magpie/index.mjs +488 -0
- package/src/local-ai/magpie/native/CMakeLists.txt +10 -0
- package/src/local-ai/magpie/native/main.cpp +270 -0
- package/src/local-ai/nemotron/build.mjs +53 -0
- package/src/local-ai/nemotron/helper.mjs +308 -0
- package/src/local-ai/nemotron/index.mjs +637 -0
- package/src/local-ai/nemotron/native/CMakeLists.txt +17 -0
- package/src/local-ai/nemotron/native/main.cpp +282 -0
|
@@ -7,7 +7,8 @@ from `arcane-os/core/speech`.
|
|
|
7
7
|
The host supplies an STT engine, a TTS engine, or both. Construction performs no
|
|
8
8
|
installation, model download, model loading, or inference. Core registration
|
|
9
9
|
exposes `speech.status`, `speech.load`, `speech.unload`, `speech.transcribe`,
|
|
10
|
-
and `speech.synthesize
|
|
10
|
+
and `speech.synthesize`, plus the optional streaming operations described below;
|
|
11
|
+
an ordinary operation whose engine was omitted rejects with
|
|
11
12
|
`SPEECH_ENGINE_UNAVAILABLE`.
|
|
12
13
|
The optional `signal` controls the service lifetime.
|
|
13
14
|
|
|
@@ -35,6 +36,15 @@ continues calling `speech.transcribe` without loading native libraries itself.
|
|
|
35
36
|
|
|
36
37
|
## Startup and status
|
|
37
38
|
|
|
39
|
+
A speech-only interface can construct the shared browser AI API with explicit
|
|
40
|
+
`null` for both the language-model service and model:
|
|
41
|
+
`new AI(null, 'LOCAL_SPEACH', 'LOCAL_SPEACH', null, ttsModel, sttModel)`.
|
|
42
|
+
This leaves both LLM routes unselected from construction onward; it does not
|
|
43
|
+
register or activate the historical default language provider. Select a real
|
|
44
|
+
language model later through `configureProviders`. Omitted or empty constructor
|
|
45
|
+
arguments keep their historical defaults, and `setAI` retains its existing
|
|
46
|
+
omitted/null/empty-field semantics.
|
|
47
|
+
|
|
38
48
|
`start(context)` subscribes to each configured engine and starts their loads
|
|
39
49
|
independently. It returns immediately, allowing Core and `speech.status` to
|
|
40
50
|
respond while models load. A failure in one role leaves the other role usable.
|
|
@@ -60,6 +70,12 @@ model readiness. Existing readiness consumers additionally recognize
|
|
|
60
70
|
the engine IDs `whisper.cpp` and `kokoro-onnx`; another engine ID requires the
|
|
61
71
|
corresponding consumer integration.
|
|
62
72
|
|
|
73
|
+
Each role also reports `supportsStreaming`: STT requires the configured
|
|
74
|
+
engine's `openStream()` method, and TTS requires its `synthesizeStream()` method.
|
|
75
|
+
This capability is independent of `loaded` and `available`; a facade method
|
|
76
|
+
alone does not establish support in the selected engine. Complete engine state,
|
|
77
|
+
including `defaultVoice` and `voices` when supplied, remains present.
|
|
78
|
+
|
|
63
79
|
## Explicit role activation
|
|
64
80
|
|
|
65
81
|
`speech.load({role, ...selection})` and `speech.unload({role, ...selection})`
|
|
@@ -100,6 +116,127 @@ cancelled requests publish no later progress.
|
|
|
100
116
|
Role load/unload acknowledgements additionally carry `operation: 'load'` or
|
|
101
117
|
`'unload'` in that progress record.
|
|
102
118
|
|
|
119
|
+
## Continuous transcription and streaming synthesis
|
|
120
|
+
|
|
121
|
+
`Arcane.speech.openStream(request, {signal, onUpdate, onProgress, onRequest})`
|
|
122
|
+
returns an owned handle immediately:
|
|
123
|
+
|
|
124
|
+
```js
|
|
125
|
+
const recognition = Arcane.speech.openStream({sampleRate: 16000}, {
|
|
126
|
+
signal,
|
|
127
|
+
onUpdate: function retainNativeSnapshot(result) {
|
|
128
|
+
// Keep the complete snapshot. The application decides when to take a turn.
|
|
129
|
+
showRecognition(result);
|
|
130
|
+
}
|
|
131
|
+
});
|
|
132
|
+
await recognition.ready;
|
|
133
|
+
await recognition.push(pcm); // Float32Array, supplied in capture order.
|
|
134
|
+
await recognition.forceEndpoint(); // Finalizes recognition, keeps the stream.
|
|
135
|
+
const complete = await recognition.finish(); // Drains and closes this stream.
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
The handle exposes `streamId`, `ready`, `completion`, `current()`,
|
|
139
|
+
`push(Float32Array)`, `forceEndpoint()`, `finish()`, and `cancel(reason)`.
|
|
140
|
+
`ready` resolves with the native-open acknowledgement; `completion` resolves
|
|
141
|
+
with the engine's complete terminal result or rejects with its actual failure.
|
|
142
|
+
`current()` returns the latest delivered complete snapshot. Pushes and endpoint
|
|
143
|
+
controls are ordered, and each returned promise lets the producer apply
|
|
144
|
+
backpressure without dropping captured samples. `forceEndpoint()` waits for
|
|
145
|
+
ordered update callbacks preceding its control response, so an external Reply
|
|
146
|
+
control can consume those final segments. `finish()` follows every accepted
|
|
147
|
+
push and waits for final callback delivery. An update callback must not await
|
|
148
|
+
its own `forceEndpoint()`, `finish()`, or `completion` delivery join; invoke these
|
|
149
|
+
controls outside the callback. `cancel()` targets this
|
|
150
|
+
stream, suppresses later callbacks immediately, and awaits its explicit native
|
|
151
|
+
cleanup control; it leaves the other speech role alone. Aborting the caller's
|
|
152
|
+
signal uses ordinary Core request cancellation and can settle the browser's
|
|
153
|
+
request before native cleanup finishes.
|
|
154
|
+
Creation opens no microphone: the selected capture owner supplies PCM.
|
|
155
|
+
|
|
156
|
+
Updates are the engine's unchanged snapshots, including provisional/final
|
|
157
|
+
status, words, timestamps, and speaker scope when supplied. A provisional
|
|
158
|
+
snapshot replaces the preceding provisional snapshot; it is not appended text.
|
|
159
|
+
An ASR final or silence boundary is recognition output, not a completed human
|
|
160
|
+
turn, an instruction, or approval. `forceEndpoint()` preserves the native
|
|
161
|
+
stream's speaker scope according to its engine contract. The application owns
|
|
162
|
+
conversational meaning, turn completion, tool selection, and whether to issue
|
|
163
|
+
an LLM request. The SDK starts no chat request from an update.
|
|
164
|
+
|
|
165
|
+
The underlying `speech.open` request remains pending for the entire stream,
|
|
166
|
+
with no fixed request timeout. `speech.opened` and `speech.transcription`
|
|
167
|
+
events use that request's scoped emitter, carrying `{streamId, ...metadata}`
|
|
168
|
+
and `{streamId, result}` respectively. They reach only the owning connection.
|
|
169
|
+
`speech.push`, `speech.endpoint`, `speech.finish`, and `speech.cancel` address
|
|
170
|
+
the same stream. Client close, document retirement, transport disconnect, or
|
|
171
|
+
the request signal cancels the native stream through normal Core request
|
|
172
|
+
ownership. This stream is not accepted service-lifetime background work.
|
|
173
|
+
|
|
174
|
+
`Arcane.speech.synthesizeStream(request, {signal, onAudio, onProgress,
|
|
175
|
+
onRequest})` returns a promise for the complete native terminal metadata.
|
|
176
|
+
The complete synthesis payload reaches the selected engine unchanged.
|
|
177
|
+
The required `onAudio(audio)` consumer receives every ordered native PCM contribution, preserving
|
|
178
|
+
its metadata; `audio.pcm` is a `Uint8Array`, interpreted using the supplied
|
|
179
|
+
`format`, `channels`, and `sampleRate`. Magpie supplies signed little-endian
|
|
180
|
+
16-bit mono PCM. The facade does not play audio, infer audible completion, or
|
|
181
|
+
add a voice, language, speed, or output limit. The selected playback owner
|
|
182
|
+
consumes this stream and owns its actual playback completion and interruption.
|
|
183
|
+
Give each utterance its own signal to stop precisely that utterance without
|
|
184
|
+
stopping microphone capture or unrelated synthesis.
|
|
185
|
+
|
|
186
|
+
The synthesis request has no fixed timeout. Its request-scoped `speech.audio`
|
|
187
|
+
events contain `{streamId, sequence, audio}`; `speech.progress` also carries the
|
|
188
|
+
stream correlation. The facade serializes asynchronous callbacks and sends
|
|
189
|
+
`speech.audioConsumed` for that exact stream and sequence only after `onAudio`
|
|
190
|
+
completes. The native producer awaits this acknowledgement before contributing
|
|
191
|
+
the next PCM chunk; the event parser remains free to handle control messages.
|
|
192
|
+
Sequence and acknowledgement metadata stay outside the audio payload.
|
|
193
|
+
|
|
194
|
+
Aborting the utterance signal suppresses later callback delivery and sends
|
|
195
|
+
`speech.cancelSynthesis`. That scoped control releases pending audio
|
|
196
|
+
acknowledgements and joins the native synthesis task. The facade keeps the
|
|
197
|
+
original response pending and joins this cleanup control before settling;
|
|
198
|
+
ordinary cancellation rejects with the caller's reason. Callback failure
|
|
199
|
+
cancels the same owned operation and retains its complete failure together with
|
|
200
|
+
any native cleanup failure. Successful completion waits for all callback
|
|
201
|
+
delivery. A callback may abort its caller-owned signal, while its own completion
|
|
202
|
+
must not await the same synthesis promise. A transport failure remains an error,
|
|
203
|
+
not evidence of completed native cleanup. Playback completion remains with the
|
|
204
|
+
selected playback owner, independently of synthesis completion.
|
|
205
|
+
|
|
206
|
+
The Core transport uses base64 only for PCM framing: STT transfers exact
|
|
207
|
+
Float32 bit patterns as little-endian samples, and TTS transfers the engine's
|
|
208
|
+
complete PCM before decoding it back to the original sample representation.
|
|
209
|
+
Routing and callbacks stay outside transcript and synthesis payloads.
|
|
210
|
+
Existing file transcription, complete-audio synthesis, model loading, and
|
|
211
|
+
unloading methods keep their original contracts. A selected engine without the
|
|
212
|
+
requested streaming method reports `SPEECH_OPERATION_UNAVAILABLE`; the SDK
|
|
213
|
+
selects no replacement engine or capture path.
|
|
214
|
+
|
|
215
|
+
### Shared AI capture and playback
|
|
216
|
+
|
|
217
|
+
The built-in native AI adapter advertises continuous capture only when the
|
|
218
|
+
selected STT engine reports streaming support. `ai.createTranscriptionCapture`
|
|
219
|
+
uses the shared microphone owner, forwards the caller's media constraints and
|
|
220
|
+
actual sample rate, and preserves every ordered PCM sample. Its `onUpdate`,
|
|
221
|
+
`onInterim`, and `onSegment` callbacks receive the complete recognition record
|
|
222
|
+
and a second `{requestStop}` argument. A callback can call `requestStop()` and
|
|
223
|
+
return; the external owner joins `capture.stop()` or `capture.done`. Do not
|
|
224
|
+
await those joins or `forceEndpoint()` from their own recognition callback.
|
|
225
|
+
Stop drains accepted audio and final recognition; cancel joins microphone and
|
|
226
|
+
native cleanup without inventing a completed transcript. Neither action sends
|
|
227
|
+
a chat turn automatically.
|
|
228
|
+
|
|
229
|
+
For a selected streaming TTS engine, `ai.prepareTTSPlayback()` returns an inert
|
|
230
|
+
playback descriptor. Its `play()` consumes native PCM through the existing AI
|
|
231
|
+
provider lane and shared audio output. Playback stop affects only that utterance;
|
|
232
|
+
the control's `released` promise joins native cleanup. Existing full-audio
|
|
233
|
+
`fetchTTS()` remains available. `streamTTS(text, end, {languageCode})`,
|
|
234
|
+
`prepareTTSPlayback({input, languageCode, ...})`, and
|
|
235
|
+
`fetchTTS({input, languageCode, ...})` carry the caller's per-utterance language
|
|
236
|
+
to the native engine separately from the unchanged text. The SDK does not infer
|
|
237
|
+
language or create conversation replies. See the
|
|
238
|
+
[Nemotron](native-nemotron.md) and [Magpie](native-magpie.md) engine contracts.
|
|
239
|
+
|
|
103
240
|
## Engine interface
|
|
104
241
|
|
|
105
242
|
Each supplied engine implements:
|
|
@@ -115,6 +252,13 @@ Each supplied engine implements:
|
|
|
115
252
|
- `transcribe(request, {signal, onProgress, requestId})` for STT, or
|
|
116
253
|
`synthesize(request, {signal, onProgress, requestId})` for TTS. `requestId` is
|
|
117
254
|
the actual Core request's control metadata, separate from the unchanged payload.
|
|
255
|
+
- Optional `openStream({...options, signal, onUpdate})` for continuous STT,
|
|
256
|
+
returning `{push, forceEndpoint, finish, cancel, completion, current}`.
|
|
257
|
+
Its cancellation must settle `completion` without requiring service shutdown.
|
|
258
|
+
- Optional `synthesizeStream(request, {signal, onProgress, onAudio, requestId})`
|
|
259
|
+
for streaming TTS. Each `onAudio` value contains the complete `pcm` Buffer
|
|
260
|
+
and its format metadata; the engine awaits each callback before producing the
|
|
261
|
+
next contribution and joins native cleanup before settling cancellation.
|
|
118
262
|
- `close()`, cancelling and joining its native work and releasing its resources.
|
|
119
263
|
|
|
120
264
|
The service forwards each complete transcription/synthesis request and result
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "arcane-os",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.95.0",
|
|
4
4
|
"description": "Arcane OS JavaScript SDK, project-local CLI, browser runtime, and repository-portable application packager.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./src/index.mjs",
|
|
@@ -90,6 +90,10 @@
|
|
|
90
90
|
"./local-ai/image": "./src/local-ai/image-runtime.mjs",
|
|
91
91
|
"./local-ai/whisper": "./src/local-ai/whisper/index.mjs",
|
|
92
92
|
"./local-ai/kokoro": "./src/local-ai/kokoro/index.mjs",
|
|
93
|
+
"./local-ai/nemotron": "./src/local-ai/nemotron/index.mjs",
|
|
94
|
+
"./local-ai/nemotron/build": "./src/local-ai/nemotron/build.mjs",
|
|
95
|
+
"./local-ai/magpie": "./src/local-ai/magpie/index.mjs",
|
|
96
|
+
"./local-ai/magpie/build": "./src/local-ai/magpie/build.mjs",
|
|
93
97
|
"./local-ai/whisper/build": "./src/local-ai/whisper/build.mjs",
|
|
94
98
|
"./local-ai/whisper/openvino-build": "./src/local-ai/whisper/openvino-build.mjs",
|
|
95
99
|
"./integrated-provider": "./src/integrated-provider-loader.mjs",
|
|
@@ -28,6 +28,8 @@ import {
|
|
|
28
28
|
import {MarkdownSpeech,stripSpeechFormatting} from 'arcane-os/speech-text';
|
|
29
29
|
import {prepareSpeech, readPreparedSpeech} from './PreparedSpeech.js';
|
|
30
30
|
import {createToolTextObserver} from 'arcane-os/ai/tool-text-stream';
|
|
31
|
+
import createNativeSpeechCapture from './NativeSpeechCapture.js';
|
|
32
|
+
import prepareNativeSpeechPlayback from './NativeSpeechPlayback.js';
|
|
31
33
|
|
|
32
34
|
const completeValue=(value)=>value;
|
|
33
35
|
|
|
@@ -1150,14 +1152,18 @@ class AI {
|
|
|
1150
1152
|
}
|
|
1151
1153
|
);
|
|
1152
1154
|
|
|
1155
|
+
const speechOnly=llmService===null&&model===null;
|
|
1153
1156
|
const preferences=[
|
|
1154
|
-
llmService||'TWIN',
|
|
1157
|
+
speechOnly?'':llmService||'TWIN',
|
|
1155
1158
|
sttService||'LOCAL_SPEACH',
|
|
1156
1159
|
ttsService||'LOCAL_SPEACH',
|
|
1157
|
-
model||'TWIN',
|
|
1160
|
+
speechOnly?'':model||'TWIN',
|
|
1158
1161
|
modelTTS||'LOCAL_SPEACH',
|
|
1159
1162
|
modelSTT||'LOCAL_SPEACH'
|
|
1160
1163
|
];
|
|
1164
|
+
// Explicitly absent LLM selection must not briefly register or load a
|
|
1165
|
+
// default provider. setAI retains empty entries from this initial tuple.
|
|
1166
|
+
if(speechOnly)this.#preferenceTuple=preferences.slice();
|
|
1161
1167
|
this.setAI(
|
|
1162
1168
|
...preferences
|
|
1163
1169
|
);
|
|
@@ -2008,13 +2014,15 @@ class AI {
|
|
|
2008
2014
|
if(runtime.#builtInSpeechService(role)!==providerId||!model){
|
|
2009
2015
|
return completeValue([]);
|
|
2010
2016
|
}
|
|
2011
|
-
const defaultVoice=runtime.#builtInSpeechDefaultVoice(
|
|
2017
|
+
const defaultVoice=nativeWatch?.snapshot?.roles?.[role]?.defaultVoice??runtime.#builtInSpeechDefaultVoice(
|
|
2012
2018
|
role,
|
|
2013
2019
|
providerId
|
|
2014
2020
|
);
|
|
2015
2021
|
return completeValue([
|
|
2016
2022
|
completeValue({
|
|
2017
2023
|
id:model,
|
|
2024
|
+
...(nativeWatch?.snapshot?.roles?.[role]?.speech
|
|
2025
|
+
?{speech:nativeWatch.snapshot.roles[role].speech}:{}),
|
|
2018
2026
|
...(defaultVoice?{defaultVoice}:{})
|
|
2019
2027
|
})
|
|
2020
2028
|
]);
|
|
@@ -2026,6 +2034,39 @@ class AI {
|
|
|
2026
2034
|
return runtime.#builtInSpeechInspection(role,providerId,selection);
|
|
2027
2035
|
},
|
|
2028
2036
|
status:statusBuiltInSpeechProvider,
|
|
2037
|
+
...(role==='stt'?{
|
|
2038
|
+
supportsCapture:function supportsNativeCapture(){
|
|
2039
|
+
return nativeWatch?.snapshot?.roles?.stt?.supportsStreaming===true;
|
|
2040
|
+
},
|
|
2041
|
+
createCapture:function createBuiltInNativeCapture(options={}){
|
|
2042
|
+
if(nativeWatch?.snapshot?.roles?.stt?.supportsStreaming!==true)return null;
|
|
2043
|
+
const capture=createNativeSpeechCapture({
|
|
2044
|
+
...options,
|
|
2045
|
+
speech:nativeWatch.speech,
|
|
2046
|
+
model:nativeWatch.modelId
|
|
2047
|
+
});
|
|
2048
|
+
const operation={watch:nativeWatch,requestId:null,signal:null};
|
|
2049
|
+
const done=capture.done.finally(function releaseNativeCapture(){
|
|
2050
|
+
if(nativeRequest===operation)releaseBuiltInSpeechRequest();
|
|
2051
|
+
});
|
|
2052
|
+
done.catch(function observeNativeCaptureCompletion(error){
|
|
2053
|
+
if(!isAIRequestAbort(error))arcaneLogging.error('Native transcription capture failed.',error);
|
|
2054
|
+
});
|
|
2055
|
+
return {
|
|
2056
|
+
...capture,
|
|
2057
|
+
done,
|
|
2058
|
+
start:function startBuiltInNativeCapture(options={}){
|
|
2059
|
+
if(!nativeSpeechReady()||busy){
|
|
2060
|
+
throw aiProviderError('The native transcription model is not ready.','ARCANE_AI_ROLE_NOT_READY');
|
|
2061
|
+
}
|
|
2062
|
+
busy=true;
|
|
2063
|
+
operation.signal=options.signal;
|
|
2064
|
+
nativeRequest=operation;
|
|
2065
|
+
return capture.start(options);
|
|
2066
|
+
}
|
|
2067
|
+
};
|
|
2068
|
+
}
|
|
2069
|
+
}:{}),
|
|
2029
2070
|
load:async function loadBuiltInSpeechProvider(context={}){
|
|
2030
2071
|
if(context.signal?.aborted){
|
|
2031
2072
|
throw normalizeAIRequestAbort(context.signal.reason);
|
|
@@ -2086,7 +2127,7 @@ class AI {
|
|
|
2086
2127
|
throw error;
|
|
2087
2128
|
}
|
|
2088
2129
|
},
|
|
2089
|
-
request:function requestBuiltInSpeechProvider(context={}){
|
|
2130
|
+
request:function requestBuiltInSpeechProvider(context={}, {speechStream=null}={}){
|
|
2090
2131
|
if(context.signal?.aborted){
|
|
2091
2132
|
throw normalizeAIRequestAbort(context.signal.reason);
|
|
2092
2133
|
}
|
|
@@ -2126,7 +2167,8 @@ class AI {
|
|
|
2126
2167
|
)
|
|
2127
2168
|
:runtime.#requestBuiltInSpeechSynthesis(
|
|
2128
2169
|
context.payload,
|
|
2129
|
-
context.signal
|
|
2170
|
+
context.signal,
|
|
2171
|
+
speechStream
|
|
2130
2172
|
);
|
|
2131
2173
|
return Promise.resolve(request).finally(releaseBuiltInSpeechRequest);
|
|
2132
2174
|
},
|
|
@@ -2548,6 +2590,7 @@ class AI {
|
|
|
2548
2590
|
|
|
2549
2591
|
audioMessageChunks='';
|
|
2550
2592
|
#markdownSpeech=new MarkdownSpeech();
|
|
2593
|
+
#streamSpeechOptions=null;
|
|
2551
2594
|
sourceNodes=[];
|
|
2552
2595
|
isSpeaking=false;
|
|
2553
2596
|
audioContext=null;
|
|
@@ -4684,7 +4727,7 @@ class AI {
|
|
|
4684
4727
|
return response.text();
|
|
4685
4728
|
}
|
|
4686
4729
|
|
|
4687
|
-
async #requestBuiltInSpeechSynthesis(payload={},signal=null){
|
|
4730
|
+
async #requestBuiltInSpeechSynthesis(payload={},signal=null,speechStream=null){
|
|
4688
4731
|
const input=is.string(payload?.input)?payload.input:'';
|
|
4689
4732
|
if(!input){
|
|
4690
4733
|
throw new TypeError('Speech synthesis requires nonempty input.');
|
|
@@ -4703,9 +4746,15 @@ class AI {
|
|
|
4703
4746
|
const speed=is.finite(payload.speed)?payload.speed:this.voiceSpeed;
|
|
4704
4747
|
const nativeSpeech=this.#nativeSpeech(this.ttsService,'tts');
|
|
4705
4748
|
if(nativeSpeech){
|
|
4749
|
+
if(speechStream){
|
|
4750
|
+
return nativeSpeech.synthesizeStream({
|
|
4751
|
+
...payload,model,voice,input,speed
|
|
4752
|
+
},{...speechStream,signal});
|
|
4753
|
+
}
|
|
4706
4754
|
let response;
|
|
4707
4755
|
try{
|
|
4708
4756
|
response=await nativeSpeech.synthesize({
|
|
4757
|
+
...payload,
|
|
4709
4758
|
model,
|
|
4710
4759
|
voice,
|
|
4711
4760
|
input,
|
|
@@ -6871,12 +6920,24 @@ class AI {
|
|
|
6871
6920
|
if(end){
|
|
6872
6921
|
this.audioMessageChunks='';
|
|
6873
6922
|
this.#markdownSpeech.reset();
|
|
6923
|
+
this.#streamSpeechOptions=null;
|
|
6874
6924
|
}
|
|
6875
6925
|
this.#traceSpeech('streamTTS.result',{callId,result:false,reason:'muted'});
|
|
6876
6926
|
return Promise.resolve(false);
|
|
6877
6927
|
}
|
|
6878
6928
|
|
|
6879
|
-
|
|
6929
|
+
// A final flush belongs to the same utterance selection. An explicit
|
|
6930
|
+
// language change drains the previous text before accepting new text.
|
|
6931
|
+
let preceding=null;
|
|
6932
|
+
if(this.#streamSpeechOptions&&Object.hasOwn(options,'languageCode')
|
|
6933
|
+
&&options.languageCode!==this.#streamSpeechOptions.languageCode){
|
|
6934
|
+
preceding=this.streamTTS('',true,this.#streamSpeechOptions);
|
|
6935
|
+
}
|
|
6936
|
+
const selectedOptions={...this.#streamSpeechOptions,...options};
|
|
6937
|
+
this.#streamSpeechOptions={
|
|
6938
|
+
voice:selectedOptions.voice,speed:selectedOptions.speed,languageCode:selectedOptions.languageCode
|
|
6939
|
+
};
|
|
6940
|
+
const {voice,speed,languageCode,waitForPlayback=false}=selectedOptions;
|
|
6880
6941
|
const pauseAfterMs=Number(options.pauseAfterMs??0);
|
|
6881
6942
|
if(!is.finite(pauseAfterMs)||pauseAfterMs<0){
|
|
6882
6943
|
throw new RangeError('Speech pauses must be a nonnegative number of milliseconds.');
|
|
@@ -6885,6 +6946,7 @@ class AI {
|
|
|
6885
6946
|
const speechText=this.#markdownSpeech.append(String(text||''),end);
|
|
6886
6947
|
this.audioMessageChunks+=speechText;
|
|
6887
6948
|
const outputs=this.#extractSpeechSegments(end);
|
|
6949
|
+
if(end)this.#streamSpeechOptions=null;
|
|
6888
6950
|
this.#traceSpeech('streamTTS.segments',{
|
|
6889
6951
|
callId,segments:outputs,remainder:this.audioMessageChunks,
|
|
6890
6952
|
segmentation:this.ttsSegmentation,speechText
|
|
@@ -6892,7 +6954,7 @@ class AI {
|
|
|
6892
6954
|
|
|
6893
6955
|
if(!outputs.length){
|
|
6894
6956
|
this.#traceSpeech('streamTTS.result',{callId,result:true,reason:'buffered'});
|
|
6895
|
-
return Promise.resolve(true);
|
|
6957
|
+
return preceding??Promise.resolve(true);
|
|
6896
6958
|
}
|
|
6897
6959
|
|
|
6898
6960
|
try{
|
|
@@ -6909,7 +6971,7 @@ class AI {
|
|
|
6909
6971
|
}
|
|
6910
6972
|
|
|
6911
6973
|
const generation=this.speechGeneration;
|
|
6912
|
-
const jobs=[];
|
|
6974
|
+
const jobs=preceding?[preceding]:[];
|
|
6913
6975
|
const runtime=this;
|
|
6914
6976
|
|
|
6915
6977
|
for(const [index,output] of outputs.entries()){
|
|
@@ -6917,6 +6979,7 @@ class AI {
|
|
|
6917
6979
|
diagnosticCallId:callId,
|
|
6918
6980
|
voice,
|
|
6919
6981
|
speed,
|
|
6982
|
+
languageCode,
|
|
6920
6983
|
waitForPlayback,
|
|
6921
6984
|
pauseAfterMs:index===outputs.length-1?pauseAfterMs:0
|
|
6922
6985
|
}));
|
|
@@ -6938,6 +7001,7 @@ class AI {
|
|
|
6938
7001
|
for(const playback of this.#preparedSpeechPlaybacks) playback.stop();
|
|
6939
7002
|
this.audioMessageChunks='';
|
|
6940
7003
|
this.#markdownSpeech.reset();
|
|
7004
|
+
this.#streamSpeechOptions=null;
|
|
6941
7005
|
return;
|
|
6942
7006
|
}
|
|
6943
7007
|
this.finishTTS();
|
|
@@ -7111,6 +7175,7 @@ class AI {
|
|
|
7111
7175
|
text,
|
|
7112
7176
|
voice:options.voice,
|
|
7113
7177
|
speed:options.speed,
|
|
7178
|
+
languageCode:options.languageCode,
|
|
7114
7179
|
pauseAfterMs:options.pauseAfterMs??0,
|
|
7115
7180
|
resolvePlayback:null
|
|
7116
7181
|
};
|
|
@@ -7206,6 +7271,7 @@ class AI {
|
|
|
7206
7271
|
const payload={
|
|
7207
7272
|
model:selection?.modelId||this.modelTTS,
|
|
7208
7273
|
input:job.text,
|
|
7274
|
+
...(job.languageCode!==undefined?{languageCode:job.languageCode}:{}),
|
|
7209
7275
|
...(job.voice!==undefined||voice?{voice}:{}),
|
|
7210
7276
|
...(!native?{responseFormat:selection
|
|
7211
7277
|
?this.#providerSpeechResponseFormat()
|
|
@@ -7570,6 +7636,34 @@ class AI {
|
|
|
7570
7636
|
|
|
7571
7637
|
async prepareTTSPlayback(payload={},signal=null,preparation={}){
|
|
7572
7638
|
const model=this.#providerSpeechModel();
|
|
7639
|
+
if(model?.speech?.streaming===true){
|
|
7640
|
+
this.#assertServiceConfigured(this.ttsService,'tts');
|
|
7641
|
+
if(signal?.aborted)throw normalizeAIRequestAbort(signal.reason);
|
|
7642
|
+
const runtime=this;
|
|
7643
|
+
const selection=this.#providerRuntime.selection('tts');
|
|
7644
|
+
return prepareNativeSpeechPlayback({
|
|
7645
|
+
request:{
|
|
7646
|
+
...payload,
|
|
7647
|
+
model:payload.model??selection.modelId,
|
|
7648
|
+
voice:payload.voice??this.#providerSpeechVoice(),
|
|
7649
|
+
speed:payload.speed??this.voiceSpeed
|
|
7650
|
+
},
|
|
7651
|
+
signal,
|
|
7652
|
+
getAudioContext:function getOwnedSpeechOutput(){
|
|
7653
|
+
return runtime.#getSpeechAudioContext();
|
|
7654
|
+
},
|
|
7655
|
+
speech:{
|
|
7656
|
+
synthesizeStream:function requestNativeSpeechStream(request,options){
|
|
7657
|
+
return runtime.#providerRuntime.request('tts',{
|
|
7658
|
+
operation:'synthesize',payload:request,localOnly:false,signal:options.signal
|
|
7659
|
+
},{
|
|
7660
|
+
...preparation,
|
|
7661
|
+
speechStream:{onAudio:options.onAudio,onProgress:options.onProgress,onRequest:options.onRequest}
|
|
7662
|
+
});
|
|
7663
|
+
}
|
|
7664
|
+
}
|
|
7665
|
+
});
|
|
7666
|
+
}
|
|
7573
7667
|
if(model?.speech?.playback!=='native'){
|
|
7574
7668
|
return this.fetchTTS(payload,signal,preparation);
|
|
7575
7669
|
}
|
|
@@ -7618,6 +7712,7 @@ class AI {
|
|
|
7618
7712
|
'input',
|
|
7619
7713
|
'textFormat',
|
|
7620
7714
|
'responseFormat',
|
|
7715
|
+
'languageCode',
|
|
7621
7716
|
'speed'
|
|
7622
7717
|
]);
|
|
7623
7718
|
for(const key of Reflect.ownKeys(descriptors)){
|
|
@@ -7625,7 +7720,7 @@ class AI {
|
|
|
7625
7720
|
||!acceptedKeys.has(key)
|
|
7626
7721
|
||!Object.hasOwn(descriptors[key],'value')){
|
|
7627
7722
|
const error=new TypeError(
|
|
7628
|
-
'AI.fetchTTS accepts only model, voice, input, textFormat, responseFormat, and speed data properties.'
|
|
7723
|
+
'AI.fetchTTS accepts only model, voice, input, textFormat, responseFormat, languageCode, and speed data properties.'
|
|
7629
7724
|
);
|
|
7630
7725
|
error.code='ARCANE_AI_TTS_REQUEST_INVALID';
|
|
7631
7726
|
throw error;
|
|
@@ -7732,15 +7827,19 @@ class AI {
|
|
|
7732
7827
|
throw error;
|
|
7733
7828
|
}
|
|
7734
7829
|
|
|
7830
|
+
const speechRequest={
|
|
7831
|
+
model,voice,input,responseFormat,speed,
|
|
7832
|
+
...(descriptors.languageCode?{languageCode:descriptors.languageCode.value}:{})
|
|
7833
|
+
};
|
|
7735
7834
|
if(selection){
|
|
7736
7835
|
this.#traceSpeech('fetchTTS.dispatch',{
|
|
7737
|
-
callId,selection,payload:
|
|
7836
|
+
callId,selection,payload:speechRequest
|
|
7738
7837
|
});
|
|
7739
7838
|
const response=await this.#providerRuntime.request(
|
|
7740
7839
|
'tts',
|
|
7741
7840
|
{
|
|
7742
7841
|
operation:'synthesize',
|
|
7743
|
-
payload:
|
|
7842
|
+
payload:speechRequest,
|
|
7744
7843
|
localOnly:false,
|
|
7745
7844
|
signal
|
|
7746
7845
|
},
|
|
@@ -7755,10 +7854,10 @@ class AI {
|
|
|
7755
7854
|
}
|
|
7756
7855
|
|
|
7757
7856
|
this.#traceSpeech('fetchTTS.dispatch',{
|
|
7758
|
-
callId,service:this.ttsService,payload:
|
|
7857
|
+
callId,service:this.ttsService,payload:speechRequest
|
|
7759
7858
|
});
|
|
7760
7859
|
const audio=await this.#requestBuiltInSpeechSynthesis(
|
|
7761
|
-
|
|
7860
|
+
speechRequest,
|
|
7762
7861
|
signal
|
|
7763
7862
|
);
|
|
7764
7863
|
this.#traceSpeech('fetchTTS.result',{callId,audio});
|
|
@@ -7859,6 +7958,7 @@ class AI {
|
|
|
7859
7958
|
this.speechScheduleTime=0;
|
|
7860
7959
|
this.audioMessageChunks='';
|
|
7861
7960
|
this.#markdownSpeech.reset();
|
|
7961
|
+
this.#streamSpeechOptions=null;
|
|
7862
7962
|
this.#clearSpeechUnlock();
|
|
7863
7963
|
|
|
7864
7964
|
for(const job of this.speechJobs){
|
|
@@ -1368,6 +1368,9 @@ function validateProvider(provider) {
|
|
|
1368
1368
|
}
|
|
1369
1369
|
if (provider.role === 'stt' && is.function(provider.createCapture)) {
|
|
1370
1370
|
record.createCapture = provider.createCapture.bind(provider);
|
|
1371
|
+
if (is.function(provider.supportsCapture)) {
|
|
1372
|
+
record.supportsCapture = provider.supportsCapture.bind(provider);
|
|
1373
|
+
}
|
|
1371
1374
|
}
|
|
1372
1375
|
const admitted = completeValue(record);
|
|
1373
1376
|
PROVIDER_RECORDS.set(provider, admitted);
|
|
@@ -3360,14 +3363,14 @@ export class AIProviderRuntime {
|
|
|
3360
3363
|
return this.#disposeAllPromise;
|
|
3361
3364
|
}
|
|
3362
3365
|
|
|
3363
|
-
request(role, options = {}, {speechInputPrepared = false, observeToolText = null, onRetry = null} = {}) {
|
|
3366
|
+
request(role, options = {}, {speechInputPrepared = false, observeToolText = null, onRetry = null, speechStream = null} = {}) {
|
|
3364
3367
|
if (SPEECH_ROLES.includes(role)) {
|
|
3365
3368
|
arcaneLogging.debug('[Arcane speech runtime] request', role, options);
|
|
3366
3369
|
}
|
|
3367
|
-
return this.#requestRole(role, options, false, speechInputPrepared, observeToolText, onRetry);
|
|
3370
|
+
return this.#requestRole(role, options, false, speechInputPrepared, observeToolText, onRetry, speechStream);
|
|
3368
3371
|
}
|
|
3369
3372
|
|
|
3370
|
-
#requestRole(role, options, queued, speechInputPrepared = false, observeToolText = null, onRetry = null) {
|
|
3373
|
+
#requestRole(role, options, queued, speechInputPrepared = false, observeToolText = null, onRetry = null, speechStream = null) {
|
|
3371
3374
|
this.#assertOpen();
|
|
3372
3375
|
this.#assertNotConfiguring();
|
|
3373
3376
|
assertRole(role);
|
|
@@ -3503,7 +3506,7 @@ export class AIProviderRuntime {
|
|
|
3503
3506
|
}
|
|
3504
3507
|
if ((!queued && slot.requestQueue.length)
|
|
3505
3508
|
|| slot.activeRequests.size >= maxConcurrentRequests) {
|
|
3506
|
-
return this.#enqueueRoleRequest(slot, options, observeToolText, onRetry);
|
|
3509
|
+
return this.#enqueueRoleRequest(slot, options, observeToolText, onRetry, speechStream);
|
|
3507
3510
|
}
|
|
3508
3511
|
if (!slot.ready
|
|
3509
3512
|
|| providerStatus.state !== 'ready'
|
|
@@ -4064,7 +4067,7 @@ export class AIProviderRuntime {
|
|
|
4064
4067
|
arcaneLogging.debug('[Arcane speech runtime] provider.request', {generation, requestSequence, operationId}, providerRequest);
|
|
4065
4068
|
}
|
|
4066
4069
|
const result = role === 'tts'
|
|
4067
|
-
? await provider.request(providerRequest, {speechInputPrepared: true})
|
|
4070
|
+
? await provider.request(providerRequest, {speechInputPrepared: true, speechStream})
|
|
4068
4071
|
: role === 'llm'
|
|
4069
4072
|
? await provider.request(
|
|
4070
4073
|
providerRequest,
|
|
@@ -4088,9 +4091,14 @@ export class AIProviderRuntime {
|
|
|
4088
4091
|
)
|
|
4089
4092
|
: result;
|
|
4090
4093
|
} catch (error) {
|
|
4091
|
-
|
|
4092
|
-
|
|
4093
|
-
|
|
4094
|
+
// Streaming speech joins its native cancellation before release.
|
|
4095
|
+
// An aborted signal alone must not conceal a cleanup failure.
|
|
4096
|
+
const speechCancelled = speechStream && !error?.errors
|
|
4097
|
+
&& (error === controller.signal.reason || error?.name === 'AbortError'
|
|
4098
|
+
|| error?.code === 'ARCANE_REQUEST_ABORTED');
|
|
4099
|
+
const normalized = speechStream
|
|
4100
|
+
? speechCancelled ? normalizedAbort(controller.signal.reason ?? error) : error
|
|
4101
|
+
: isAbort(error, controller.signal) ? normalizedAbort(error) : error;
|
|
4094
4102
|
if (SPEECH_ROLES.includes(role)) {
|
|
4095
4103
|
arcaneLogging.debug('[Arcane speech runtime] provider.request.error', {role, generation, requestSequence, operationId}, error);
|
|
4096
4104
|
}
|
|
@@ -4134,7 +4142,8 @@ export class AIProviderRuntime {
|
|
|
4134
4142
|
const provider = providerId === null
|
|
4135
4143
|
? this.#providerFor(slot)
|
|
4136
4144
|
: this.#providers.get(providerKey('stt', providerId));
|
|
4137
|
-
return is.function(provider?.createCapture)
|
|
4145
|
+
return is.function(provider?.createCapture)
|
|
4146
|
+
&& (!is.function(provider.supportsCapture) || provider.supportsCapture());
|
|
4138
4147
|
}
|
|
4139
4148
|
|
|
4140
4149
|
createTranscriptionCapture(options = {}) {
|
|
@@ -4147,6 +4156,7 @@ export class AIProviderRuntime {
|
|
|
4147
4156
|
const runtime = this;
|
|
4148
4157
|
const generation = slot.generation;
|
|
4149
4158
|
const capture = provider.createCapture(options);
|
|
4159
|
+
if (!capture) return null;
|
|
4150
4160
|
const controller = new AbortController();
|
|
4151
4161
|
let requestRecord = null;
|
|
4152
4162
|
let detachSignal = null;
|
|
@@ -4185,7 +4195,7 @@ export class AIProviderRuntime {
|
|
|
4185
4195
|
// permission retains the originating user gesture. It is never queued.
|
|
4186
4196
|
return {
|
|
4187
4197
|
done,
|
|
4188
|
-
start({signal} = {}) {
|
|
4198
|
+
start({signal, ...captureOptions} = {}) {
|
|
4189
4199
|
if (started || closed) return Promise.resolve(false);
|
|
4190
4200
|
try {
|
|
4191
4201
|
runtime.#assertOpen();
|
|
@@ -4217,7 +4227,7 @@ export class AIProviderRuntime {
|
|
|
4217
4227
|
};
|
|
4218
4228
|
slot.activeRequests.set(requestSequence, requestRecord);
|
|
4219
4229
|
runtime.#publishRoleRequestState(slot);
|
|
4220
|
-
const starting = capture.start({signal: controller.signal});
|
|
4230
|
+
const starting = capture.start({...captureOptions, signal: controller.signal});
|
|
4221
4231
|
return Promise.resolve(starting).catch(
|
|
4222
4232
|
function releaseFailedTranscriptionCaptureStart(error) {
|
|
4223
4233
|
capture.cancel();
|
|
@@ -4230,16 +4240,22 @@ export class AIProviderRuntime {
|
|
|
4230
4240
|
}
|
|
4231
4241
|
},
|
|
4232
4242
|
stop() {
|
|
4233
|
-
capture.stop();
|
|
4234
|
-
return done;
|
|
4243
|
+
return Promise.resolve(capture.stop()).then(function captureStopped() { return done; });
|
|
4235
4244
|
},
|
|
4245
|
+
...(is.function(capture.forceEndpoint) ? {
|
|
4246
|
+
forceEndpoint() {
|
|
4247
|
+
return capture.forceEndpoint();
|
|
4248
|
+
}
|
|
4249
|
+
} : {}),
|
|
4236
4250
|
cancel() {
|
|
4237
4251
|
controller.abort();
|
|
4238
4252
|
capture.cancel();
|
|
4253
|
+
return done;
|
|
4239
4254
|
},
|
|
4240
4255
|
destroy() {
|
|
4241
4256
|
controller.abort();
|
|
4242
4257
|
capture.destroy();
|
|
4258
|
+
return done;
|
|
4243
4259
|
}
|
|
4244
4260
|
};
|
|
4245
4261
|
}
|
|
@@ -4432,7 +4448,7 @@ export class AIProviderRuntime {
|
|
|
4432
4448
|
);
|
|
4433
4449
|
}
|
|
4434
4450
|
|
|
4435
|
-
#enqueueRoleRequest(slot, options, observeToolText = null, onRetry = null) {
|
|
4451
|
+
#enqueueRoleRequest(slot, options, observeToolText = null, onRetry = null, speechStream = null) {
|
|
4436
4452
|
if (SPEECH_ROLES.includes(slot.role)) {
|
|
4437
4453
|
arcaneLogging.debug('[Arcane speech runtime] queue.enqueue', {
|
|
4438
4454
|
role: slot.role,
|
|
@@ -4448,6 +4464,7 @@ export class AIProviderRuntime {
|
|
|
4448
4464
|
options,
|
|
4449
4465
|
observeToolText,
|
|
4450
4466
|
onRetry,
|
|
4467
|
+
speechStream,
|
|
4451
4468
|
resolve,
|
|
4452
4469
|
reject,
|
|
4453
4470
|
detachSignal: null
|
|
@@ -4520,7 +4537,7 @@ export class AIProviderRuntime {
|
|
|
4520
4537
|
let operation;
|
|
4521
4538
|
try {
|
|
4522
4539
|
operation = runtime.#requestRole(
|
|
4523
|
-
slot.role, entry.options, true, false, entry.observeToolText, entry.onRetry
|
|
4540
|
+
slot.role, entry.options, true, false, entry.observeToolText, entry.onRetry, entry.speechStream
|
|
4524
4541
|
);
|
|
4525
4542
|
} catch (error) {
|
|
4526
4543
|
if (SPEECH_ROLES.includes(slot.role)) {
|