arcane-os 0.93.0 → 0.95.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +33 -0
- package/README.md +1 -1
- package/browser-runtime/ai/twin-cloud.mjs +109 -46
- package/browser-runtime/core/client.mjs +197 -0
- package/docs/reference/ai/twin-cloud.md +35 -14
- package/docs/reference/availability-and-normalization.md +1 -1
- package/docs/reference/inventory/package-api.json +96 -12
- package/docs/reference/local-ai.md +81 -5
- package/docs/reference/native-magpie.md +98 -0
- package/docs/reference/native-nemotron.md +204 -0
- package/docs/reference/native-speech.md +364 -11
- package/docs/reference/runtime-modules.md +21 -18
- package/docs/reference/sdk-api.md +152 -11
- package/package.json +6 -1
- package/runtime/arcane/modules/AI.js +114 -14
- package/runtime/arcane/modules/AIProviderRuntime.js +32 -15
- package/runtime/arcane/modules/ContinuousVoiceCapture.js +118 -21
- package/runtime/arcane/modules/NativeSpeechCapture.js +198 -0
- package/runtime/arcane/modules/NativeSpeechPlayback.js +304 -0
- package/runtime/arcane/modules/VoiceCaptureWorklet.js +62 -2
- package/schemas/arcane-app.schema.json +2 -2
- package/schemas/arcane-package.schema.json +2 -2
- package/src/core/services/speech.mjs +320 -25
- package/src/local-ai/config.mjs +12 -1
- package/src/local-ai/dev.mjs +71 -4
- package/src/local-ai/install.mjs +113 -5
- package/src/local-ai/kokoro/frontend.mjs +219 -0
- package/src/local-ai/kokoro/helper.mjs +168 -0
- package/src/local-ai/kokoro/index.mjs +645 -0
- package/src/local-ai/kokoro/native/CMakeLists.txt +45 -0
- package/src/local-ai/kokoro/native/main.cpp +249 -0
- package/src/local-ai/magpie/build.mjs +36 -0
- package/src/local-ai/magpie/helper.mjs +254 -0
- package/src/local-ai/magpie/index.mjs +488 -0
- package/src/local-ai/magpie/native/CMakeLists.txt +10 -0
- package/src/local-ai/magpie/native/main.cpp +270 -0
- package/src/local-ai/nemotron/build.mjs +53 -0
- package/src/local-ai/nemotron/helper.mjs +308 -0
- package/src/local-ai/nemotron/index.mjs +637 -0
- package/src/local-ai/nemotron/native/CMakeLists.txt +17 -0
- package/src/local-ai/nemotron/native/main.cpp +282 -0
- package/src/native/portable-layout.mjs +67 -9
|
@@ -0,0 +1,637 @@
|
|
|
1
|
+
import {AsyncLocalStorage} from 'node:async_hooks';
|
|
2
|
+
import {randomUUID} from 'node:crypto';
|
|
3
|
+
import {createArcaneEventSource} from '../../event-manager.mjs';
|
|
4
|
+
import {createNemotronHelper, deferred, encodeNemotronAudio, nemotronError} from './helper.mjs';
|
|
5
|
+
|
|
6
|
+
/** Retain one selected recognizer and diarizer across explicit utterance endpoints. */
|
|
7
|
+
export function createNemotronRuntime(
|
|
8
|
+
{runtime, prepare, modelId = 'nemotron-3.5-asr-streaming-0.6b', modelPath,
|
|
9
|
+
diarizationModelPath, gpu, onEvent} = {}
|
|
10
|
+
) {
|
|
11
|
+
if (!runtime && typeof prepare !== 'function') {
|
|
12
|
+
throw new TypeError('Nemotron requires a prepared runtime or a prepare function.');
|
|
13
|
+
}
|
|
14
|
+
const events = createArcaneEventSource(
|
|
15
|
+
{},
|
|
16
|
+
{source: 'nemotron', eventTypes: ['nemotron.state']}
|
|
17
|
+
);
|
|
18
|
+
const callbacks = new AsyncLocalStorage();
|
|
19
|
+
const instance = randomUUID();
|
|
20
|
+
let selected = modelId;
|
|
21
|
+
let selectedModelPath = modelPath ?? runtime?.modelPath;
|
|
22
|
+
let selectedDiarizationPath = diarizationModelPath ?? runtime?.diarizationModelPath;
|
|
23
|
+
let selectedGpu = gpu ?? runtime?.gpu;
|
|
24
|
+
let streamSequence = 0;
|
|
25
|
+
let state = 'unloaded';
|
|
26
|
+
let helper = null;
|
|
27
|
+
let loading = null;
|
|
28
|
+
let loadController = null;
|
|
29
|
+
let active = null;
|
|
30
|
+
let unloading = null;
|
|
31
|
+
let closing = null;
|
|
32
|
+
let disposed = false;
|
|
33
|
+
let error = null;
|
|
34
|
+
let reset = null;
|
|
35
|
+
|
|
36
|
+
function current() {
|
|
37
|
+
return {
|
|
38
|
+
providerId: 'nemotron', modelId: selected, state,
|
|
39
|
+
loaded: state === 'ready', busy: active !== null,
|
|
40
|
+
modelPath: selectedModelPath ?? null,
|
|
41
|
+
diarizationModelPath: selectedDiarizationPath ?? null,
|
|
42
|
+
requestedGpu: selectedGpu ?? null,
|
|
43
|
+
observedBackend: null,
|
|
44
|
+
nativeVersion: helper?.current().metadata?.version ?? null,
|
|
45
|
+
speakerScope: active?.speakerScope ?? null,
|
|
46
|
+
resetRequired: reset !== null,
|
|
47
|
+
reset,
|
|
48
|
+
error
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
function publish() {
|
|
53
|
+
if (!disposed) events.dispatch('nemotron.state', current());
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function notify(listener, snapshot) {
|
|
57
|
+
const result = listener(snapshot);
|
|
58
|
+
if (result && typeof result.then === 'function') {
|
|
59
|
+
Promise.resolve(result).catch(
|
|
60
|
+
function listenerFailed(failure) {
|
|
61
|
+
console.error('Nemotron state listener failed.', failure);
|
|
62
|
+
}
|
|
63
|
+
);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
function subscribe(listener, {replay = true, signal} = {}) {
|
|
68
|
+
if (typeof listener !== 'function') throw new TypeError('A Nemotron state listener must be a function.');
|
|
69
|
+
if (disposed) {
|
|
70
|
+
if (replay && !signal?.aborted) notify(listener, current());
|
|
71
|
+
return function closedSubscription() {};
|
|
72
|
+
}
|
|
73
|
+
const unsubscribe = events.on(
|
|
74
|
+
'nemotron.state',
|
|
75
|
+
function modelState(event) {
|
|
76
|
+
notify(listener, event.detail);
|
|
77
|
+
},
|
|
78
|
+
{signal}
|
|
79
|
+
);
|
|
80
|
+
try {
|
|
81
|
+
if (replay && !signal?.aborted) notify(listener, current());
|
|
82
|
+
} catch (failure) {
|
|
83
|
+
unsubscribe();
|
|
84
|
+
throw failure;
|
|
85
|
+
}
|
|
86
|
+
return unsubscribe;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
function load(options = {}) {
|
|
90
|
+
const {signal} = options;
|
|
91
|
+
signal?.throwIfAborted();
|
|
92
|
+
if (closing || disposed || unloading) {
|
|
93
|
+
return Promise.reject(nemotronError('NEMOTRON_CLOSING', 'Nemotron is closing.'));
|
|
94
|
+
}
|
|
95
|
+
const changed = (options.modelId !== undefined && options.modelId !== selected)
|
|
96
|
+
|| (options.modelPath !== undefined && options.modelPath !== selectedModelPath)
|
|
97
|
+
|| (options.diarizationModelPath !== undefined && options.diarizationModelPath !== selectedDiarizationPath)
|
|
98
|
+
|| (options.gpu !== undefined && options.gpu !== selectedGpu);
|
|
99
|
+
if ((loading || helper || active) && changed) {
|
|
100
|
+
return Promise.reject(nemotronError('NEMOTRON_MODEL_LOADED', 'Unload the current Nemotron selection before replacing it.'));
|
|
101
|
+
}
|
|
102
|
+
if (loading) return waitFor(loading, signal);
|
|
103
|
+
if (state === 'ready') return Promise.resolve(current());
|
|
104
|
+
if (helper || active) {
|
|
105
|
+
return Promise.reject(nemotronError('NEMOTRON_MODEL_ACTIVE', 'Unload the previous Nemotron operation before loading again.'));
|
|
106
|
+
}
|
|
107
|
+
selected = options.modelId ?? selected;
|
|
108
|
+
selectedModelPath = options.modelPath ?? selectedModelPath;
|
|
109
|
+
selectedDiarizationPath = options.diarizationModelPath ?? selectedDiarizationPath;
|
|
110
|
+
selectedGpu = options.gpu ?? selectedGpu;
|
|
111
|
+
state = 'loading';
|
|
112
|
+
error = null;
|
|
113
|
+
loadController = new AbortController();
|
|
114
|
+
const ownerSignal = loadController.signal;
|
|
115
|
+
async function loadModel() {
|
|
116
|
+
ownerSignal.throwIfAborted();
|
|
117
|
+
if (!runtime) {
|
|
118
|
+
runtime = await prepare(
|
|
119
|
+
{signal: ownerSignal, onEvent}
|
|
120
|
+
);
|
|
121
|
+
}
|
|
122
|
+
ownerSignal.throwIfAborted();
|
|
123
|
+
selectedModelPath ??= runtime.modelPath;
|
|
124
|
+
selectedDiarizationPath ??= runtime.diarizationModelPath;
|
|
125
|
+
selectedGpu ??= runtime.gpu;
|
|
126
|
+
if (typeof runtime.helperExecutable !== 'string' || typeof selectedModelPath !== 'string') {
|
|
127
|
+
throw nemotronError('NEMOTRON_MODEL_UNAVAILABLE', 'Select the prepared Nemotron helperExecutable and ASR modelPath.');
|
|
128
|
+
}
|
|
129
|
+
if (selectedDiarizationPath !== undefined && typeof selectedDiarizationPath !== 'string') {
|
|
130
|
+
throw new TypeError('diarizationModelPath must be the selected local model path.');
|
|
131
|
+
}
|
|
132
|
+
if (!Number.isInteger(selectedGpu) || selectedGpu < -1) {
|
|
133
|
+
throw new TypeError('Select the native GPU ordinal through gpu, or explicitly use -1 for CPU.');
|
|
134
|
+
}
|
|
135
|
+
const session = createNemotronHelper(
|
|
136
|
+
{
|
|
137
|
+
runtime, modelPath: selectedModelPath, diarizationModelPath: selectedDiarizationPath,
|
|
138
|
+
gpu: selectedGpu, onEvent
|
|
139
|
+
}
|
|
140
|
+
);
|
|
141
|
+
helper = session;
|
|
142
|
+
function cancelLoad() {
|
|
143
|
+
session.stop(ownerSignal.reason).catch(observeRetainedFailure);
|
|
144
|
+
}
|
|
145
|
+
ownerSignal.addEventListener(
|
|
146
|
+
'abort', cancelLoad,
|
|
147
|
+
{once: true}
|
|
148
|
+
);
|
|
149
|
+
session.completion.catch(
|
|
150
|
+
function modelExited(failure) {
|
|
151
|
+
if (helper === session && !unloading && !closing && !session.current().stopping) {
|
|
152
|
+
error = failure;
|
|
153
|
+
state = 'error';
|
|
154
|
+
active?.failed(failure);
|
|
155
|
+
publish();
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
);
|
|
159
|
+
try {
|
|
160
|
+
if (ownerSignal.aborted) cancelLoad();
|
|
161
|
+
await session.ready;
|
|
162
|
+
ownerSignal.throwIfAborted();
|
|
163
|
+
state = 'ready';
|
|
164
|
+
reset = null;
|
|
165
|
+
publish();
|
|
166
|
+
return current();
|
|
167
|
+
} catch (failure) {
|
|
168
|
+
try {
|
|
169
|
+
await session.stop();
|
|
170
|
+
} catch (cleanupFailure) {
|
|
171
|
+
if (cleanupFailure !== failure) {
|
|
172
|
+
throw new AggregateError([failure, cleanupFailure], 'Nemotron loading and cleanup failed.');
|
|
173
|
+
}
|
|
174
|
+
} finally {
|
|
175
|
+
if (helper === session) helper = null;
|
|
176
|
+
}
|
|
177
|
+
throw failure;
|
|
178
|
+
} finally {
|
|
179
|
+
ownerSignal.removeEventListener('abort', cancelLoad);
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
function loadFailed(failure) {
|
|
183
|
+
const cancelledLoad = ownerSignal.aborted && (
|
|
184
|
+
failure === ownerSignal.reason || failure?.code === 'ARCANE_CANCELLED' || failure?.name === 'AbortError'
|
|
185
|
+
);
|
|
186
|
+
error = cancelledLoad ? null : failure;
|
|
187
|
+
state = unloading || closing ? 'unloading' : ownerSignal.aborted ? 'unloaded' : 'error';
|
|
188
|
+
publish();
|
|
189
|
+
throw failure;
|
|
190
|
+
}
|
|
191
|
+
function loadSettled() {
|
|
192
|
+
loading = null;
|
|
193
|
+
loadController = null;
|
|
194
|
+
}
|
|
195
|
+
loading = Promise.resolve().then(loadModel).catch(loadFailed).finally(loadSettled);
|
|
196
|
+
loading.catch(observeRetainedFailure);
|
|
197
|
+
publish();
|
|
198
|
+
// A caller's wait never cancels shared model loading. Explicit unload
|
|
199
|
+
// and close own that cancellation and its joined native cleanup.
|
|
200
|
+
return waitFor(loading, signal);
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
function openStream({sampleRate = 16000, languageCode, model, onUpdate, signal} = {}) {
|
|
204
|
+
signal?.throwIfAborted();
|
|
205
|
+
if (model !== undefined && model !== selected) {
|
|
206
|
+
return Promise.reject(
|
|
207
|
+
nemotronError('NEMOTRON_MODEL_MISMATCH', `The selected Nemotron model is ${selected}.`)
|
|
208
|
+
);
|
|
209
|
+
}
|
|
210
|
+
if (closing || disposed || unloading) return Promise.reject(nemotronError('NEMOTRON_CLOSING', 'Nemotron is closing.'));
|
|
211
|
+
if (active) return Promise.reject(nemotronError('NEMOTRON_BUSY', 'This Nemotron engine already owns an audio stream.'));
|
|
212
|
+
if (!Number.isInteger(sampleRate) || (sampleRate !== 0 && (sampleRate < 8000 || sampleRate > 96000))) {
|
|
213
|
+
return Promise.reject(new TypeError('sampleRate must be 0 (model rate) or an integer from 8000 through 96000.'));
|
|
214
|
+
}
|
|
215
|
+
if (languageCode !== undefined && (typeof languageCode !== 'string' || languageCode.includes('\0'))) {
|
|
216
|
+
return Promise.reject(new TypeError('languageCode must be a string representable by the native C ABI.'));
|
|
217
|
+
}
|
|
218
|
+
if (onUpdate !== undefined && typeof onUpdate !== 'function') return Promise.reject(new TypeError('onUpdate must be a function.'));
|
|
219
|
+
const controller = new AbortController();
|
|
220
|
+
const operationSignal = signal ? AbortSignal.any(
|
|
221
|
+
[signal, controller.signal]
|
|
222
|
+
) : controller.signal;
|
|
223
|
+
const lifetime = deferred();
|
|
224
|
+
const record = {controller, speakerScope: `${instance}/${++streamSequence}`, opening: null, cancel: null, failed: null};
|
|
225
|
+
active = record;
|
|
226
|
+
let session = null;
|
|
227
|
+
let nativeOpening = false;
|
|
228
|
+
let nativeOpened = false;
|
|
229
|
+
let tail = Promise.resolve();
|
|
230
|
+
let finishing = null;
|
|
231
|
+
let cancelling = null;
|
|
232
|
+
let cancellationJoined = null;
|
|
233
|
+
let cancelled = null;
|
|
234
|
+
let terminalFailure = null;
|
|
235
|
+
let ended = false;
|
|
236
|
+
let latest = null;
|
|
237
|
+
|
|
238
|
+
function requireActive() {
|
|
239
|
+
if (cancelled) throw cancelled;
|
|
240
|
+
operationSignal.throwIfAborted();
|
|
241
|
+
if (ended) throw nemotronError('NEMOTRON_STREAM_CLOSED', 'The Nemotron stream has ended.');
|
|
242
|
+
if (helper !== session || state !== 'ready') throw nemotronError('NEMOTRON_MODEL_NOT_READY', 'The selected Nemotron model is no longer ready.');
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
function release() {
|
|
246
|
+
ended = true;
|
|
247
|
+
nativeOpened = false;
|
|
248
|
+
operationSignal.removeEventListener('abort', abort);
|
|
249
|
+
if (active === record) active = null;
|
|
250
|
+
publish();
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
function insideCallback() {
|
|
254
|
+
const context = callbacks.getStore();
|
|
255
|
+
return context?.active && context.record === record;
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
function requireOutsideCallback(operation) {
|
|
259
|
+
if (insideCallback()) {
|
|
260
|
+
throw nemotronError('NEMOTRON_CALLBACK_WAIT', `${operation} must wait until the output callback returns; abort the caller-owned signal synchronously and join outside the callback.`);
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
async function accept(results) {
|
|
265
|
+
const delivered = [];
|
|
266
|
+
try {
|
|
267
|
+
for (const result of results) {
|
|
268
|
+
requireActive();
|
|
269
|
+
const first = result.alternatives[0];
|
|
270
|
+
const snapshot = {...result, text: first?.text ?? '', words: first?.words ?? [], speakerScope: record.speakerScope};
|
|
271
|
+
if (onUpdate) {
|
|
272
|
+
const context = {record, active: true};
|
|
273
|
+
try {
|
|
274
|
+
await callbacks.run(context, onUpdate, snapshot);
|
|
275
|
+
} finally {
|
|
276
|
+
context.active = false;
|
|
277
|
+
}
|
|
278
|
+
requireActive();
|
|
279
|
+
}
|
|
280
|
+
latest = snapshot;
|
|
281
|
+
delivered.push(snapshot);
|
|
282
|
+
}
|
|
283
|
+
} catch (failure) {
|
|
284
|
+
// Cancellation suppresses later commits, while complete native
|
|
285
|
+
// output remains available to the operation's diagnostic owner.
|
|
286
|
+
throw nemotronError(
|
|
287
|
+
failure?.code ?? 'NEMOTRON_OUTPUT_FAILED',
|
|
288
|
+
failure?.message ?? 'Nemotron output delivery failed.',
|
|
289
|
+
{cause: failure, nativeResults: results}
|
|
290
|
+
);
|
|
291
|
+
}
|
|
292
|
+
return delivered;
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
function beginCancellation(reason) {
|
|
296
|
+
if (cancellationJoined) return cancellationJoined;
|
|
297
|
+
if (ended) return Promise.resolve({cancelled: false, resetRequired: false});
|
|
298
|
+
cancelled = nemotronError('NEMOTRON_CANCELLED', 'The Nemotron stream was cancelled.', {reason});
|
|
299
|
+
async function cancelStream() {
|
|
300
|
+
let outcome = {resetRequired: false};
|
|
301
|
+
if (session && (nativeOpening || nativeOpened)) outcome = await session.closeStream(cancelled);
|
|
302
|
+
if (outcome.resetRequired && helper === session) {
|
|
303
|
+
helper = null;
|
|
304
|
+
reset = {speakerScope: record.speakerScope, reason: cancelled, diagnostics: outcome.diagnostics};
|
|
305
|
+
state = unloading || closing ? 'unloading' : 'unloaded';
|
|
306
|
+
}
|
|
307
|
+
return {cancelled: true, ...outcome, speakerScope: record.speakerScope};
|
|
308
|
+
}
|
|
309
|
+
cancelling = Promise.resolve().then(cancelStream);
|
|
310
|
+
async function joinCancellation() {
|
|
311
|
+
// Native shutdown and the accepted operation/callback must both
|
|
312
|
+
// settle before releasing this stream's conversation ownership.
|
|
313
|
+
const [native] = await Promise.allSettled(
|
|
314
|
+
[cancelling, tail, record.opening]
|
|
315
|
+
);
|
|
316
|
+
try {
|
|
317
|
+
if (native.status === 'rejected') {
|
|
318
|
+
throw terminalFailure && terminalFailure !== native.reason
|
|
319
|
+
? new AggregateError(
|
|
320
|
+
[terminalFailure, native.reason],
|
|
321
|
+
'Nemotron operation and cleanup failed.'
|
|
322
|
+
)
|
|
323
|
+
: native.reason;
|
|
324
|
+
}
|
|
325
|
+
if (terminalFailure) throw terminalFailure;
|
|
326
|
+
lifetime.resolve(native.value);
|
|
327
|
+
return native.value;
|
|
328
|
+
} catch (failure) {
|
|
329
|
+
lifetime.reject(failure);
|
|
330
|
+
throw failure;
|
|
331
|
+
} finally {
|
|
332
|
+
release();
|
|
333
|
+
}
|
|
334
|
+
}
|
|
335
|
+
cancellationJoined = Promise.resolve().then(joinCancellation);
|
|
336
|
+
cancellationJoined.catch(observeRetainedFailure);
|
|
337
|
+
return cancellationJoined;
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
function cancel(reason) {
|
|
341
|
+
requireOutsideCallback('cancel');
|
|
342
|
+
return beginCancellation(reason);
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
function abort() {
|
|
346
|
+
beginCancellation(operationSignal.reason).catch(observeRetainedFailure);
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
record.cancel = cancel;
|
|
350
|
+
record.failed = function streamFailed(failure) {
|
|
351
|
+
terminalFailure = failure;
|
|
352
|
+
beginCancellation(failure).catch(observeRetainedFailure);
|
|
353
|
+
};
|
|
354
|
+
|
|
355
|
+
function enqueue(operation) {
|
|
356
|
+
async function operationFailed(failure) {
|
|
357
|
+
if (!cancelled) terminalFailure = failure;
|
|
358
|
+
try {
|
|
359
|
+
beginCancellation(failure);
|
|
360
|
+
await cancelling;
|
|
361
|
+
} catch (cleanupFailure) {
|
|
362
|
+
if (cleanupFailure !== failure) throw new AggregateError([failure, cleanupFailure], 'Nemotron operation and stream cleanup failed.');
|
|
363
|
+
}
|
|
364
|
+
throw failure;
|
|
365
|
+
}
|
|
366
|
+
const result = tail.then(operation).catch(operationFailed);
|
|
367
|
+
tail = result.then(observeRetainedFailure, observeRetainedFailure);
|
|
368
|
+
return result;
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
function push(audio) {
|
|
372
|
+
requireOutsideCallback('push');
|
|
373
|
+
if (finishing || ended || cancelling) return Promise.reject(nemotronError('NEMOTRON_STREAM_CLOSED', 'The Nemotron stream is ending.'));
|
|
374
|
+
const samples = decodeAudio(audio);
|
|
375
|
+
async function pushAudio() {
|
|
376
|
+
requireActive();
|
|
377
|
+
const results = [];
|
|
378
|
+
// Drive recognition between ordered 100 ms transport chunks.
|
|
379
|
+
// This schedules native work; it never shortens caller audio.
|
|
380
|
+
const count = Math.round((sampleRate || 16000) * 0.1);
|
|
381
|
+
for (let offset = 0; offset < samples.length; offset += count) {
|
|
382
|
+
requireActive();
|
|
383
|
+
const chunk = samples.subarray(offset, offset + count);
|
|
384
|
+
const updates = await session.command('push', String(chunk.length), encodeNemotronAudio(chunk));
|
|
385
|
+
for (const result of await accept(updates)) results.push(result);
|
|
386
|
+
}
|
|
387
|
+
return results;
|
|
388
|
+
}
|
|
389
|
+
return enqueue(pushAudio);
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
function forceEndpoint() {
|
|
393
|
+
requireOutsideCallback('forceEndpoint');
|
|
394
|
+
if (finishing || ended || cancelling) return Promise.reject(nemotronError('NEMOTRON_STREAM_CLOSED', 'The Nemotron stream is ending.'));
|
|
395
|
+
async function endpoint() {
|
|
396
|
+
requireActive();
|
|
397
|
+
// Native ASR segmentation preserves this exact stream and its
|
|
398
|
+
// speaker scope. It makes no conversational-completion decision.
|
|
399
|
+
return accept(await session.command('endpoint'));
|
|
400
|
+
}
|
|
401
|
+
return enqueue(endpoint);
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
function finish() {
|
|
405
|
+
requireOutsideCallback('finish');
|
|
406
|
+
if (finishing) return finishing;
|
|
407
|
+
if (cancelling || ended) return Promise.reject(cancelled ?? nemotronError('NEMOTRON_STREAM_CLOSED', 'The Nemotron stream has ended.'));
|
|
408
|
+
async function finishAudio() {
|
|
409
|
+
requireActive();
|
|
410
|
+
const result = await accept(await session.command('finish'));
|
|
411
|
+
release();
|
|
412
|
+
lifetime.resolve(result);
|
|
413
|
+
return result;
|
|
414
|
+
}
|
|
415
|
+
finishing = enqueue(finishAudio);
|
|
416
|
+
finishing.catch(observeRetainedFailure);
|
|
417
|
+
return finishing;
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
const stream = {
|
|
421
|
+
push, forceEndpoint, finish, cancel, completion: lifetime.promise,
|
|
422
|
+
speakerScope: record.speakerScope, sampleRate,
|
|
423
|
+
current: function streamResult() { return latest; }
|
|
424
|
+
};
|
|
425
|
+
async function openNativeStream() {
|
|
426
|
+
if (loading) await waitFor(loading, operationSignal);
|
|
427
|
+
operationSignal.throwIfAborted();
|
|
428
|
+
if (ended || cancelled) throw cancelled ?? nemotronError('NEMOTRON_STREAM_CLOSED', 'The Nemotron stream was cancelled before opening.');
|
|
429
|
+
if (state !== 'ready' || !helper) throw nemotronError('NEMOTRON_MODEL_NOT_READY', 'Explicitly load the selected Nemotron model before opening a stream.');
|
|
430
|
+
session = helper;
|
|
431
|
+
operationSignal.addEventListener(
|
|
432
|
+
'abort', abort,
|
|
433
|
+
{once: true}
|
|
434
|
+
);
|
|
435
|
+
const language = Buffer.from(languageCode ?? '', 'utf8');
|
|
436
|
+
nativeOpening = true;
|
|
437
|
+
if (operationSignal.aborted) abort();
|
|
438
|
+
requireActive();
|
|
439
|
+
await session.command('open', `${sampleRate} ${language.length}`, language);
|
|
440
|
+
nativeOpened = true;
|
|
441
|
+
nativeOpening = false;
|
|
442
|
+
requireActive();
|
|
443
|
+
publish();
|
|
444
|
+
return stream;
|
|
445
|
+
}
|
|
446
|
+
async function openFailed(failure) {
|
|
447
|
+
if (!cancelled) terminalFailure = failure;
|
|
448
|
+
try {
|
|
449
|
+
beginCancellation(failure);
|
|
450
|
+
await cancelling;
|
|
451
|
+
} catch (cleanupFailure) {
|
|
452
|
+
if (cleanupFailure !== failure) throw new AggregateError([failure, cleanupFailure], 'Nemotron stream opening and cleanup failed.');
|
|
453
|
+
}
|
|
454
|
+
throw failure;
|
|
455
|
+
}
|
|
456
|
+
record.opening = Promise.resolve().then(openNativeStream).catch(openFailed);
|
|
457
|
+
publish();
|
|
458
|
+
return record.opening;
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
async function transcribe(request, {signal, onProgress, onUpdate} = {}) {
|
|
462
|
+
signal?.throwIfAborted();
|
|
463
|
+
if (request?.model !== undefined && request.model !== selected) {
|
|
464
|
+
throw nemotronError('NEMOTRON_MODEL_MISMATCH', `The loaded Nemotron model is ${selected}.`);
|
|
465
|
+
}
|
|
466
|
+
const audio = decodeAudio(request?.audio ?? request);
|
|
467
|
+
const finals = [];
|
|
468
|
+
await onProgress?.(
|
|
469
|
+
{phase: 'accepted'}
|
|
470
|
+
);
|
|
471
|
+
const stream = await openStream(
|
|
472
|
+
{
|
|
473
|
+
sampleRate: request?.sampleRate ?? 16000,
|
|
474
|
+
languageCode: request?.languageCode ?? request?.language,
|
|
475
|
+
signal,
|
|
476
|
+
onUpdate: async function transcriptionUpdate(result) {
|
|
477
|
+
if (result.final) finals.push(result);
|
|
478
|
+
await onUpdate?.(result);
|
|
479
|
+
await onProgress?.(
|
|
480
|
+
{phase: 'recognizing', result}
|
|
481
|
+
);
|
|
482
|
+
}
|
|
483
|
+
}
|
|
484
|
+
);
|
|
485
|
+
try {
|
|
486
|
+
await stream.push(audio);
|
|
487
|
+
await stream.finish();
|
|
488
|
+
// Whole-recording mode has no intermediate endpoints. Preserve any
|
|
489
|
+
// native final boundaries as well as the complete visible text.
|
|
490
|
+
return {
|
|
491
|
+
text: finals.map(
|
|
492
|
+
function transcript(result) { return result.text; }
|
|
493
|
+
).join(''),
|
|
494
|
+
words: finals.flatMap(
|
|
495
|
+
function words(result) { return result.words; }
|
|
496
|
+
),
|
|
497
|
+
results: finals,
|
|
498
|
+
speakerScope: stream.speakerScope
|
|
499
|
+
};
|
|
500
|
+
} catch (failure) {
|
|
501
|
+
try {
|
|
502
|
+
await stream.cancel(failure);
|
|
503
|
+
} catch (cleanupFailure) {
|
|
504
|
+
throw new AggregateError([failure, cleanupFailure], 'Nemotron transcription and cleanup failed.');
|
|
505
|
+
}
|
|
506
|
+
if (finals.length) {
|
|
507
|
+
throw nemotronError(
|
|
508
|
+
failure?.code ?? 'NEMOTRON_TRANSCRIPTION_FAILED',
|
|
509
|
+
failure?.message ?? 'Nemotron transcription failed.',
|
|
510
|
+
{cause: failure, completedResults: finals}
|
|
511
|
+
);
|
|
512
|
+
}
|
|
513
|
+
throw failure;
|
|
514
|
+
}
|
|
515
|
+
}
|
|
516
|
+
|
|
517
|
+
function unload() {
|
|
518
|
+
if (callbacks.getStore()?.active) {
|
|
519
|
+
return Promise.reject(nemotronError('NEMOTRON_CALLBACK_WAIT', 'Unload Nemotron after the output callback returns; abort the caller-owned signal synchronously and join outside the callback.'));
|
|
520
|
+
}
|
|
521
|
+
if (unloading) return unloading;
|
|
522
|
+
if (disposed) return Promise.resolve(current());
|
|
523
|
+
state = 'unloading';
|
|
524
|
+
const operation = active;
|
|
525
|
+
const pendingLoad = loading;
|
|
526
|
+
const pendingLoadSignal = loadController?.signal;
|
|
527
|
+
loadController?.abort(nemotronError('NEMOTRON_UNLOADED', 'The Nemotron model was unloaded.'));
|
|
528
|
+
operation?.controller.abort(nemotronError('NEMOTRON_UNLOADED', 'The Nemotron model was unloaded.'));
|
|
529
|
+
async function unloadModel() {
|
|
530
|
+
const failures = [];
|
|
531
|
+
if (operation) {
|
|
532
|
+
try { await operation.cancel(); } catch (failure) { failures.push(failure); }
|
|
533
|
+
await Promise.allSettled(
|
|
534
|
+
[operation.opening]
|
|
535
|
+
);
|
|
536
|
+
}
|
|
537
|
+
if (pendingLoad) {
|
|
538
|
+
try {
|
|
539
|
+
await pendingLoad;
|
|
540
|
+
} catch (failure) {
|
|
541
|
+
const cancelledLoad = pendingLoadSignal?.aborted && (
|
|
542
|
+
failure === pendingLoadSignal.reason || failure?.code === 'ARCANE_CANCELLED' || failure?.name === 'AbortError'
|
|
543
|
+
);
|
|
544
|
+
if (!cancelledLoad) failures.push(failure);
|
|
545
|
+
}
|
|
546
|
+
}
|
|
547
|
+
const session = helper;
|
|
548
|
+
try { if (session) await session.stop(); } catch (failure) { failures.push(failure); }
|
|
549
|
+
if (helper === session) helper = null;
|
|
550
|
+
if (failures.length) throw new AggregateError(failures, 'Nemotron cleanup failed.');
|
|
551
|
+
error = null;
|
|
552
|
+
state = 'unloaded';
|
|
553
|
+
return current();
|
|
554
|
+
}
|
|
555
|
+
function unloadFailed(failure) {
|
|
556
|
+
state = 'error';
|
|
557
|
+
error = failure;
|
|
558
|
+
throw failure;
|
|
559
|
+
}
|
|
560
|
+
function unloadSettled() {
|
|
561
|
+
unloading = null;
|
|
562
|
+
publish();
|
|
563
|
+
}
|
|
564
|
+
unloading = Promise.resolve().then(unloadModel).catch(unloadFailed).finally(unloadSettled);
|
|
565
|
+
publish();
|
|
566
|
+
return unloading;
|
|
567
|
+
}
|
|
568
|
+
|
|
569
|
+
function close() {
|
|
570
|
+
if (callbacks.getStore()?.active) {
|
|
571
|
+
return Promise.reject(nemotronError('NEMOTRON_CALLBACK_WAIT', 'Close Nemotron after the output callback returns; abort the caller-owned signal synchronously and join outside the callback.'));
|
|
572
|
+
}
|
|
573
|
+
if (closing) return closing;
|
|
574
|
+
async function closeModel() {
|
|
575
|
+
try {
|
|
576
|
+
await unload();
|
|
577
|
+
} finally {
|
|
578
|
+
state = 'closed';
|
|
579
|
+
publish();
|
|
580
|
+
disposed = true;
|
|
581
|
+
events.dispose();
|
|
582
|
+
}
|
|
583
|
+
return current();
|
|
584
|
+
}
|
|
585
|
+
closing = Promise.resolve().then(closeModel);
|
|
586
|
+
return closing;
|
|
587
|
+
}
|
|
588
|
+
|
|
589
|
+
return {current, subscribe, load, unload, transcribe, openStream, close};
|
|
590
|
+
}
|
|
591
|
+
|
|
592
|
+
function decodeAudio(audio) {
|
|
593
|
+
if (audio instanceof Float32Array) return audio;
|
|
594
|
+
if (audio?.encoding !== 'f32le' || typeof audio.data !== 'string') {
|
|
595
|
+
throw nemotronError('NEMOTRON_AUDIO_FORMAT_UNSUPPORTED', 'Nemotron accepts mono Float32Array or {encoding: "f32le", data: BASE64}; encoded recordings require caller-owned decoding.');
|
|
596
|
+
}
|
|
597
|
+
const encoded = Buffer.from(audio.data, 'base64');
|
|
598
|
+
if (encoded.length % 4 !== 0) throw new TypeError('The f32le transport ends during a sample.');
|
|
599
|
+
const samples = new Float32Array(encoded.length / 4);
|
|
600
|
+
const bits = new Uint32Array(samples.buffer);
|
|
601
|
+
for (let index = 0; index < samples.length; index++) bits[index] = encoded.readUInt32LE(index * 4);
|
|
602
|
+
return samples;
|
|
603
|
+
}
|
|
604
|
+
|
|
605
|
+
function waitFor(task, signal) {
|
|
606
|
+
if (!signal) return task;
|
|
607
|
+
signal.throwIfAborted();
|
|
608
|
+
return new Promise(
|
|
609
|
+
function waitForModel(resolve, reject) {
|
|
610
|
+
function cancelled() {
|
|
611
|
+
signal.removeEventListener('abort', cancelled);
|
|
612
|
+
reject(signal.reason);
|
|
613
|
+
}
|
|
614
|
+
signal.addEventListener(
|
|
615
|
+
'abort', cancelled,
|
|
616
|
+
{once: true}
|
|
617
|
+
);
|
|
618
|
+
task.then(
|
|
619
|
+
function ready(value) {
|
|
620
|
+
signal.removeEventListener('abort', cancelled);
|
|
621
|
+
if (signal.aborted) reject(signal.reason);
|
|
622
|
+
else resolve(value);
|
|
623
|
+
},
|
|
624
|
+
function failed(failure) {
|
|
625
|
+
signal.removeEventListener('abort', cancelled);
|
|
626
|
+
reject(failure);
|
|
627
|
+
}
|
|
628
|
+
);
|
|
629
|
+
}
|
|
630
|
+
);
|
|
631
|
+
}
|
|
632
|
+
|
|
633
|
+
// Public promises and lifecycle snapshots retain these failures for their
|
|
634
|
+
// owners, including before a consumer has attached its own observer.
|
|
635
|
+
function observeRetainedFailure() {}
|
|
636
|
+
|
|
637
|
+
export default createNemotronRuntime;
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
cmake_minimum_required(VERSION 3.26)
|
|
2
|
+
project(ArcaneNemotron LANGUAGES CXX)
|
|
3
|
+
|
|
4
|
+
find_package(NeMoSpeech REQUIRED COMPONENTS ASR)
|
|
5
|
+
|
|
6
|
+
add_executable(arcane-nemotron main.cpp)
|
|
7
|
+
target_compile_features(arcane-nemotron PRIVATE cxx_std_17)
|
|
8
|
+
target_link_libraries(arcane-nemotron PRIVATE NeMoSpeech::ASR)
|
|
9
|
+
if(MSVC)
|
|
10
|
+
target_compile_options(arcane-nemotron PRIVATE /utf-8 /EHsc)
|
|
11
|
+
endif()
|
|
12
|
+
if(APPLE)
|
|
13
|
+
set_target_properties(arcane-nemotron PROPERTIES INSTALL_RPATH "@loader_path;@loader_path/../lib")
|
|
14
|
+
elseif(UNIX)
|
|
15
|
+
set_target_properties(arcane-nemotron PROPERTIES INSTALL_RPATH "$ORIGIN;$ORIGIN/../lib")
|
|
16
|
+
endif()
|
|
17
|
+
install(TARGETS arcane-nemotron RUNTIME DESTINATION bin)
|