arcane-os 0.94.0 → 0.95.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/README.md +1 -1
- package/browser-runtime/core/client.mjs +195 -0
- package/docs/reference/inventory/package-api.json +1 -1
- package/docs/reference/native-magpie.md +98 -0
- package/docs/reference/native-nemotron.md +204 -0
- package/docs/reference/native-speech.md +145 -1
- package/package.json +5 -1
- package/runtime/arcane/modules/AI.js +114 -14
- package/runtime/arcane/modules/AIProviderRuntime.js +32 -15
- package/runtime/arcane/modules/ContinuousVoiceCapture.js +118 -21
- package/runtime/arcane/modules/NativeSpeechCapture.js +198 -0
- package/runtime/arcane/modules/NativeSpeechPlayback.js +304 -0
- package/runtime/arcane/modules/VoiceCaptureWorklet.js +62 -2
- package/src/core/services/speech.mjs +235 -2
- package/src/local-ai/magpie/build.mjs +36 -0
- package/src/local-ai/magpie/helper.mjs +254 -0
- package/src/local-ai/magpie/index.mjs +488 -0
- package/src/local-ai/magpie/native/CMakeLists.txt +10 -0
- package/src/local-ai/magpie/native/main.cpp +270 -0
- package/src/local-ai/nemotron/build.mjs +53 -0
- package/src/local-ai/nemotron/helper.mjs +308 -0
- package/src/local-ai/nemotron/index.mjs +637 -0
- package/src/local-ai/nemotron/native/CMakeLists.txt +17 -0
- package/src/local-ai/nemotron/native/main.cpp +282 -0
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,23 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.95.0
|
|
4
|
+
|
|
5
|
+
- Add retained native Nemotron streaming recognition and optional diarization,
|
|
6
|
+
with ordered complete snapshots, explicit utterance endpoints, speaker scope,
|
|
7
|
+
and joined cancellation. Preserve existing Whisper and complete transcription.
|
|
8
|
+
- Add native Magpie/NanoCodec synthesis with model-owned voice selection,
|
|
9
|
+
per-request language, ordered PCM streaming, complete WAV/PCM export, and
|
|
10
|
+
exact-utterance cancellation. Preserve authored input and natural model rate;
|
|
11
|
+
the caller supplies upstream runtime and model assets without SDK vendoring.
|
|
12
|
+
- Extend shared Core speech and browser clients with correlated streaming
|
|
13
|
+
capture/output, consumed-audio acknowledgements and native cleanup joins.
|
|
14
|
+
Connect AI playback through its existing queues and shared AudioContext,
|
|
15
|
+
retaining per-utterance pause/resume/stop and complete-audio APIs.
|
|
16
|
+
- Keep capture ownership, recognition segments and model-owned conversational
|
|
17
|
+
decisions distinct. Windows x64 native helpers compiled and installed against
|
|
18
|
+
the approved NeMo 0.2.0 runtime; inference, microphone/audio acceptance and
|
|
19
|
+
other-platform execution remain separate evidence boundaries.
|
|
20
|
+
|
|
3
21
|
## 0.94.0
|
|
4
22
|
|
|
5
23
|
- Recover all TWiN HTTP 429 quota responses through the shared chat/System One
|
package/README.md
CHANGED
|
@@ -25,7 +25,7 @@ selected SDK projection or an explicit live SDK source mount. Both profiles
|
|
|
25
25
|
share the theme, packaging, event, cancellation,
|
|
26
26
|
and browser run contracts while retaining their selected application layout.
|
|
27
27
|
|
|
28
|
-
This checkout defines the `0.
|
|
28
|
+
This checkout defines the `0.95.0` SDK contract. Development applications declare
|
|
29
29
|
`arcane-os` as `latest` and retain their lockfile. Exact SDK pins begin when Roshi
|
|
30
30
|
designates a production build. Follow [Consumer updates](docs/consumer-updates.md)
|
|
31
31
|
to refresh and compare the installed SDK and lockfile with the published channel;
|
|
@@ -569,6 +569,199 @@ export function createCoreClient({
|
|
|
569
569
|
export function createCoreFacade(client){
|
|
570
570
|
const {invoke,events,uuid}=client;
|
|
571
571
|
const long={timeoutMs:LONG_OPERATION_TIMEOUT};
|
|
572
|
+
function speechStreamEvents(streamId,options){
|
|
573
|
+
const controller=new AbortController();
|
|
574
|
+
const signal=options.signal?AbortSignal.any([options.signal,controller.signal]):controller.signal;
|
|
575
|
+
const subscriptions=[];
|
|
576
|
+
let delivery=Promise.resolve();
|
|
577
|
+
let callbackFailure;
|
|
578
|
+
function listen(name,callback){
|
|
579
|
+
if(callback===undefined)return;
|
|
580
|
+
if(typeof callback!=='function')throw new TypeError(`${name} observer must be a function.`);
|
|
581
|
+
subscriptions.push(events.on(name,function receiveSpeechStreamEvent(event){
|
|
582
|
+
if(event?.streamId!==streamId||signal.aborted)return;
|
|
583
|
+
delivery=delivery.then(function deliverOrderedSpeechEvent(){
|
|
584
|
+
if(!signal.aborted)return callback(event);
|
|
585
|
+
});
|
|
586
|
+
delivery.catch(function cancelFailedSpeechObserver(error){
|
|
587
|
+
callbackFailure??=error;
|
|
588
|
+
controller.abort(error);
|
|
589
|
+
});
|
|
590
|
+
}));
|
|
591
|
+
}
|
|
592
|
+
function dispose(){
|
|
593
|
+
for(const unsubscribe of subscriptions.splice(0))unsubscribe();
|
|
594
|
+
}
|
|
595
|
+
async function delivered(){
|
|
596
|
+
await delivery;
|
|
597
|
+
signal.throwIfAborted();
|
|
598
|
+
}
|
|
599
|
+
async function settle(task,{preserveTaskFailure=false}={}){
|
|
600
|
+
let result;
|
|
601
|
+
let failure;
|
|
602
|
+
let failed=false;
|
|
603
|
+
let deliveryFailure;
|
|
604
|
+
let deliveryFailed=false;
|
|
605
|
+
try{
|
|
606
|
+
try{result=await task;}catch(error){failed=true;failure=error;}
|
|
607
|
+
try{await delivery;}catch(error){deliveryFailed=true;deliveryFailure=callbackFailure??error;}
|
|
608
|
+
if(preserveTaskFailure&&failed&&deliveryFailed&&failure!==deliveryFailure){
|
|
609
|
+
throw new AggregateError([deliveryFailure,failure],'Speech callback and native cleanup failed.');
|
|
610
|
+
}
|
|
611
|
+
if(failed||deliveryFailed)throw callbackFailure??deliveryFailure??failure;
|
|
612
|
+
signal.throwIfAborted();
|
|
613
|
+
return result;
|
|
614
|
+
}finally{dispose();}
|
|
615
|
+
}
|
|
616
|
+
return {controller,signal,listen,settle,dispose,delivered};
|
|
617
|
+
}
|
|
618
|
+
function encodeSpeechSamples(samples){
|
|
619
|
+
if(!(samples instanceof Float32Array))throw new TypeError('Speech stream push requires Float32Array PCM.');
|
|
620
|
+
const encoded=new Uint8Array(samples.length*4);
|
|
621
|
+
const view=new DataView(encoded.buffer);
|
|
622
|
+
const bits=new Uint32Array(samples.buffer,samples.byteOffset,samples.length);
|
|
623
|
+
for(let index=0;index<bits.length;index+=1)view.setUint32(index*4,bits[index],true);
|
|
624
|
+
let binary='';
|
|
625
|
+
for(const value of encoded)binary+=String.fromCharCode(value);
|
|
626
|
+
return {encoding:'f32le',data:globalThis.btoa(binary)};
|
|
627
|
+
}
|
|
628
|
+
function decodeSpeechAudio(audio){
|
|
629
|
+
if(audio?.pcm?.encoding!=='base64'||typeof audio.pcm.data!=='string'){
|
|
630
|
+
throw new TypeError('Streaming speech audio requires base64 PCM transport.');
|
|
631
|
+
}
|
|
632
|
+
const binary=globalThis.atob(audio.pcm.data);
|
|
633
|
+
const pcm=new Uint8Array(binary.length);
|
|
634
|
+
for(let index=0;index<binary.length;index+=1)pcm[index]=binary.charCodeAt(index);
|
|
635
|
+
return {...audio,pcm};
|
|
636
|
+
}
|
|
637
|
+
function openSpeechStream(request={},options={}){
|
|
638
|
+
const streamId=uuid();
|
|
639
|
+
const owned=speechStreamEvents(streamId,options);
|
|
640
|
+
let resolveReady;
|
|
641
|
+
let rejectReady;
|
|
642
|
+
let opened=false;
|
|
643
|
+
let ending=false;
|
|
644
|
+
let snapshot=null;
|
|
645
|
+
let controls=Promise.resolve();
|
|
646
|
+
let finishing=null;
|
|
647
|
+
let cancelling=null;
|
|
648
|
+
const ready=new Promise(function waitForSpeechStream(resolve,reject){
|
|
649
|
+
resolveReady=resolve;
|
|
650
|
+
rejectReady=reject;
|
|
651
|
+
});
|
|
652
|
+
ready.catch(function observeSpeechStreamReadinessFailure(){});
|
|
653
|
+
try{
|
|
654
|
+
owned.listen('speech.opened',function speechStreamOpened(event){
|
|
655
|
+
opened=true;
|
|
656
|
+
resolveReady(event);
|
|
657
|
+
});
|
|
658
|
+
owned.listen('speech.transcription',function transcriptionUpdated(event){
|
|
659
|
+
snapshot=event.result;
|
|
660
|
+
return options.onUpdate?.(event.result);
|
|
661
|
+
});
|
|
662
|
+
if(options.onProgress!==undefined){
|
|
663
|
+
owned.listen('speech.progress',function speechStreamProgress(event){return options.onProgress(event);});
|
|
664
|
+
}
|
|
665
|
+
}catch(error){owned.dispose();throw error;}
|
|
666
|
+
const completion=owned.settle(invoke('speech.open',{streamId,options:request},{
|
|
667
|
+
timeoutMs:0,signal:owned.signal,onRequest:options.onRequest
|
|
668
|
+
})).then(function speechStreamCompleted(result){
|
|
669
|
+
if(!opened)throw new CoreError({code:'SPEECH_STREAM_NOT_OPENED',message:'The speech stream ended before readiness.'});
|
|
670
|
+
return result;
|
|
671
|
+
}).catch(function speechStreamFailed(error){
|
|
672
|
+
rejectReady(error);
|
|
673
|
+
throw error;
|
|
674
|
+
}).finally(function releaseSpeechStream(){ending=true;});
|
|
675
|
+
completion.catch(function observeOwnedSpeechStreamFailure(){});
|
|
676
|
+
function control(method,parameters={}){
|
|
677
|
+
const task=controls.then(async function sendOrderedSpeechControl(){
|
|
678
|
+
await ready;
|
|
679
|
+
owned.signal.throwIfAborted();
|
|
680
|
+
return invoke(method,{streamId,...parameters},{timeoutMs:0,signal:owned.signal});
|
|
681
|
+
});
|
|
682
|
+
controls=task;
|
|
683
|
+
task.catch(function cancelFailedSpeechControl(error){owned.controller.abort(error);});
|
|
684
|
+
return task;
|
|
685
|
+
}
|
|
686
|
+
return {
|
|
687
|
+
streamId,ready,completion,
|
|
688
|
+
current:function currentSpeechTranscript(){return snapshot;},
|
|
689
|
+
push:function pushSpeechSamples(samples){
|
|
690
|
+
if(ending)return Promise.reject(new CoreError({code:'SPEECH_STREAM_ENDING',message:'The speech stream is finishing.'}));
|
|
691
|
+
return control('speech.push',{audio:encodeSpeechSamples(samples)});
|
|
692
|
+
},
|
|
693
|
+
forceEndpoint:function endpointSpeechStream(){
|
|
694
|
+
if(ending)return Promise.reject(new CoreError({code:'SPEECH_STREAM_ENDING',message:'The speech stream is finishing.'}));
|
|
695
|
+
return control('speech.endpoint').then(async function deliverEndpointSnapshots(result){
|
|
696
|
+
await owned.delivered();
|
|
697
|
+
return result;
|
|
698
|
+
});
|
|
699
|
+
},
|
|
700
|
+
finish:function finishSpeechStream(){
|
|
701
|
+
if(!ending){
|
|
702
|
+
ending=true;
|
|
703
|
+
finishing=control('speech.finish').then(function finishSpeechDelivery(){return completion;});
|
|
704
|
+
finishing.catch(function observeSpeechFinishFailure(){});
|
|
705
|
+
}
|
|
706
|
+
return finishing??completion;
|
|
707
|
+
},
|
|
708
|
+
cancel:function cancelSpeechStream(reason){
|
|
709
|
+
if(cancelling)return cancelling;
|
|
710
|
+
ending=true;
|
|
711
|
+
const stopping=invoke('speech.cancel',{streamId},{timeoutMs:0});
|
|
712
|
+
owned.controller.abort(reason);
|
|
713
|
+
// A transcript callback may await cancellation. Native cleanup
|
|
714
|
+
// must not wait for that same callback through completion.
|
|
715
|
+
cancelling=stopping;
|
|
716
|
+
return cancelling;
|
|
717
|
+
}
|
|
718
|
+
};
|
|
719
|
+
}
|
|
720
|
+
function synthesizeSpeechStream(request={},options={}){
|
|
721
|
+
if(typeof options.onAudio!=='function')throw new TypeError('Streaming speech synthesis requires an onAudio consumer.');
|
|
722
|
+
const streamId=uuid();
|
|
723
|
+
const owned=speechStreamEvents(streamId,options);
|
|
724
|
+
try{
|
|
725
|
+
owned.listen('speech.audio',async function synthesisAudio(event){
|
|
726
|
+
await options.onAudio(decodeSpeechAudio(event.audio));
|
|
727
|
+
owned.signal.throwIfAborted();
|
|
728
|
+
await invoke('speech.audioConsumed',{streamId,sequence:event.sequence},{timeoutMs:0,signal:owned.signal});
|
|
729
|
+
});
|
|
730
|
+
if(options.onProgress!==undefined){
|
|
731
|
+
owned.listen('speech.progress',function synthesisProgress(event){return options.onProgress(event);});
|
|
732
|
+
}
|
|
733
|
+
}catch(error){owned.dispose();throw error;}
|
|
734
|
+
if(owned.signal.aborted)return owned.settle(Promise.reject(owned.signal.reason));
|
|
735
|
+
// Keep the response pending until the native operation settles. Generic
|
|
736
|
+
// invoke cancellation settles locally before native cleanup can finish.
|
|
737
|
+
const operation=invoke('speech.synthesizeStream',{streamId,request},{
|
|
738
|
+
timeoutMs:0,onRequest:options.onRequest
|
|
739
|
+
});
|
|
740
|
+
let cancellation=null;
|
|
741
|
+
function cancelSynthesis(){
|
|
742
|
+
if(cancellation)return;
|
|
743
|
+
cancellation=invoke('speech.cancelSynthesis',{streamId},{timeoutMs:0});
|
|
744
|
+
cancellation.catch(function observeSynthesisCleanupFailure(){});
|
|
745
|
+
}
|
|
746
|
+
owned.signal.addEventListener('abort',cancelSynthesis,{once:true});
|
|
747
|
+
if(owned.signal.aborted)cancelSynthesis();
|
|
748
|
+
const joined=(async function joinSynthesisCleanup(){
|
|
749
|
+
let result;
|
|
750
|
+
const failures=[];
|
|
751
|
+
try{result=await operation;}catch(error){
|
|
752
|
+
if(!owned.signal.aborted||error?.name!=='AbortError')failures.push(error);
|
|
753
|
+
}
|
|
754
|
+
if(cancellation){
|
|
755
|
+
try{await cancellation;}catch(error){failures.push(error);}
|
|
756
|
+
}
|
|
757
|
+
if(failures.length===1)throw failures[0];
|
|
758
|
+
if(failures.length)throw new AggregateError(failures,'Speech synthesis and cleanup failed.');
|
|
759
|
+
return result;
|
|
760
|
+
})();
|
|
761
|
+
return owned.settle(joined,{preserveTaskFailure:true}).finally(function releaseSynthesisCancellation(){
|
|
762
|
+
owned.signal.removeEventListener('abort',cancelSynthesis);
|
|
763
|
+
});
|
|
764
|
+
}
|
|
572
765
|
function ollamaInvoke(operation,request={},options={}){
|
|
573
766
|
const onChunk=typeof options==='function'?options:options?.onChunk;
|
|
574
767
|
const timeoutMs=options?.timeoutMs??(['pull','push','create'].includes(operation)?LONG_OPERATION_TIMEOUT:10*60*1000);
|
|
@@ -598,6 +791,8 @@ export function createCoreFacade(client){
|
|
|
598
791
|
remove:name=>invoke('environment.delete',{name})},
|
|
599
792
|
mail:{send:request=>invoke('mail.send',request??{},{timeoutMs:450000})},
|
|
600
793
|
speech:{status:()=>invoke('speech.status',{},{timeoutMs:10000}),
|
|
794
|
+
openStream:openSpeechStream,
|
|
795
|
+
synthesizeStream:synthesizeSpeechStream,
|
|
601
796
|
load:function loadSpeech(request,options){return invoke('speech.load',request??{},{timeoutMs:0,signal:options?.signal,onRequest:options?.onRequest});},
|
|
602
797
|
unload:function unloadSpeech(request,options){return invoke('speech.unload',request??{},{timeoutMs:0,signal:options?.signal,onRequest:options?.onRequest});},
|
|
603
798
|
synthesize:(request,options)=>invoke('speech.synthesize',request??{},{timeoutMs:180000,signal:options?.signal}),
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
# Native Magpie speech
|
|
2
|
+
|
|
3
|
+
`arcane-os/local-ai/magpie` exports `createMagpieRuntime` as a named and default export. It retains the selected Magpie model and NanoCodec decoder in one owned native process. It produces complete encoded audio or ordered PCM callbacks; the caller owns playback. Existing Kokoro, browser speech, and complete-audio APIs remain separate capabilities.
|
|
4
|
+
|
|
5
|
+
## Prepare and select the runtime
|
|
6
|
+
|
|
7
|
+
Supply a prepared native executable and the external NeMo shared-library directory:
|
|
8
|
+
|
|
9
|
+
```js
|
|
10
|
+
import {createMagpieRuntime} from 'arcane-os/local-ai/magpie';
|
|
11
|
+
|
|
12
|
+
const speech = createMagpieRuntime({
|
|
13
|
+
runtime: {helperExecutable, libraryDirectory},
|
|
14
|
+
paths: {
|
|
15
|
+
model: magpieModel,
|
|
16
|
+
codec: nanoCodecModel,
|
|
17
|
+
tokenizerDirectory
|
|
18
|
+
},
|
|
19
|
+
executionTarget: {backend: 'cuda', cudaVisibleDevices: selectedCudaUuid},
|
|
20
|
+
voice: 'Aria',
|
|
21
|
+
languageCode: 'en-US'
|
|
22
|
+
});
|
|
23
|
+
await speech.load();
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
The paths identify an already selected Magpie GGUF, matching tokenizer assets, and matching NanoCodec decoder. Nothing is downloaded or copied by this factory. `executionTarget.backend` must explicitly select `cuda` or `cpu`. For CUDA, the native device owner supplies its selected visible device or UUID through `cudaVisibleDevices`; the child receives that process-local environment before loading CUDA. The adapter does not invent a GPU ordinal, select another device, or fall back to another provider.
|
|
27
|
+
|
|
28
|
+
Instead of `runtime`, supply `prepare({signal})`, returning the prepared descriptor directly: `{helperExecutable, libraryDirectory, ...}`. Preparation runs when loading needs the executable, alongside model metadata preparation. It must return the descriptor, not a `{runtime}` wrapper. Without `libraryDirectory`, the executable's directory is used.
|
|
29
|
+
|
|
30
|
+
Applications using model-assets ownership may replace `paths` with `modelAssets`, `assetProjectionId`, and `resourcePaths: {model, codec, tokenizerConfig}`. The resource paths identify members of the retained projection; the tokenizer directory is derived from the selected configuration member. The projection stays retained until its native helper closes.
|
|
31
|
+
|
|
32
|
+
Other factory options are `modelId` (default `magpie`), a lifetime `signal`, and `onEvent` for the existing process/event diagnostics. Factory defaults are voice `Aria` and language `en-US`; both are exposed by `current()`.
|
|
33
|
+
|
|
34
|
+
## State and lifecycle
|
|
35
|
+
|
|
36
|
+
- `current()` returns provider/model identity, `state`, `loaded`, `busy`, active `requestId`, `queuedRequests`, `defaultVoice`, `languageCode`, model-owned `voices`, configured execution information, `sampleRate`, progress, and a serialized error. Its speech catalog is `{streaming: true, responseFormats: ['wav', 'pcm'], defaultResponseFormat: 'wav'}`. Configured execution is not evidence of actual GPU inference.
|
|
37
|
+
- `subscribe(listener, {replay: true, signal})` observes lifecycle state and replays the current state by default; it returns an unsubscribe function.
|
|
38
|
+
- `load({paths, assetProjectionId, resourcePaths, executionTarget, modelId, signal})` loads or replaces the selected resources. Reusing the same selection reuses its loaded engine. The optional signal cancels that caller's load wait; the factory lifetime signal owns engine cancellation. Load the engine before requesting speech.
|
|
39
|
+
- `unload()` cancels owned work, joins native and callback cleanup, releases retained assets, and leaves the factory reusable.
|
|
40
|
+
- `close()` joins unloading and disposes the factory. Repeated close calls share completion.
|
|
41
|
+
|
|
42
|
+
Requests accepted by one engine are processed in order. Unloading or replacing that engine cancels its owned work; it does not cancel another engine or audio owner.
|
|
43
|
+
|
|
44
|
+
## Exact requests, voices, and languages
|
|
45
|
+
|
|
46
|
+
Both synthesis methods accept the same request:
|
|
47
|
+
|
|
48
|
+
```js
|
|
49
|
+
const request = {
|
|
50
|
+
input: 'The moon is closed for dragon maintenance.',
|
|
51
|
+
voice: 'Aria',
|
|
52
|
+
languageCode: 'en-US'
|
|
53
|
+
};
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
`input` is passed unchanged: no trimming, sentence splitting, prompt injection, tempo stage, or output cap. A NUL or an unpaired surrogate cannot be represented by the native UTF-8 C-string interface and produces an explicit incompatibility error.
|
|
57
|
+
|
|
58
|
+
Optional `model` must match the configured `modelId`. `voice` defaults to the factory selection and is resolved case-insensitively against the matching tokenizer's `model_config.yaml` speaker-map reference and its model-owned JSON IDs. Embedded GGUF voice labels are not used as the selection map.
|
|
59
|
+
|
|
60
|
+
`languageCode` (or `language_code`) is separate per-request control information; it never changes the authored input. Omission uses the configured language. The application/model owns language selection. Installed tokenizer support determines whether a language can run; a language listed in configuration is not proof of runtime support. Native errors are returned rather than switching languages or engines. The public native ABI supplies the natural speaking rate only; an optional `speed` must be `1`.
|
|
61
|
+
|
|
62
|
+
## Complete audio and streaming audio
|
|
63
|
+
|
|
64
|
+
`synthesize(request, {signal, requestId, onProgress})` resolves with complete `audioBase64`, `contentType`, `format`, `sampleFormat: 's16le'`, sample rate, channel count, model, voice, language code, speed, and request ID. Set `request.responseFormat` to `wav` (default) or `pcm`. WAV uses the external RIFF framing; when a complete result cannot fit that format, select PCM or streaming instead of truncating it.
|
|
65
|
+
|
|
66
|
+
`synthesizeStream(request, {signal, requestId, onProgress, onAudio})` requires `onAudio` and awaits each callback in original order:
|
|
67
|
+
|
|
68
|
+
```js
|
|
69
|
+
const controller = new AbortController();
|
|
70
|
+
const completion = speech.synthesizeStream(request, {
|
|
71
|
+
signal: controller.signal,
|
|
72
|
+
requestId: 'dragon-announcement',
|
|
73
|
+
async onAudio({pcm, format, sampleRate, channels, requestId}) {
|
|
74
|
+
await audioOutput.write({pcm, format, sampleRate, channels, requestId});
|
|
75
|
+
}
|
|
76
|
+
});
|
|
77
|
+
const result = await completion;
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
`pcm` is a Node `Buffer` of signed 16-bit little-endian interleaved PCM; the current native engine returns mono audio. The callback includes `format: 's16le'`, `sampleRate`, `channels`, and `requestId`. Awaited callback delivery applies backpressure to the native helper. The streaming terminal result contains metadata and `complete: true`, without accumulating or returning the audio again. Browser/Core transport and playback have their own delivery and cleanup owners.
|
|
81
|
+
|
|
82
|
+
`onProgress` receives synthesis progress. Callback failures cancel the matching utterance and remain observable. A caller-supplied `requestId` identifies its active or queued request; otherwise the runtime creates one.
|
|
83
|
+
|
|
84
|
+
## Cancellation and callback ownership
|
|
85
|
+
|
|
86
|
+
`await speech.stop({requestId})` stops only that active or queued utterance and joins native completion and callback cleanup. It returns `{requestId, stopped: true}`, or `stopped: false` when the ID is no longer owned. Cancellation of an active native call is observed at a PCM callback boundary: this ABI has no asynchronous pre-first-PCM interrupt. Model creation can be cancelled by retiring its owned process.
|
|
87
|
+
|
|
88
|
+
An output callback must return before joining its own stop, unload, close, or selection replacement. Those self-joining calls reject with `MAGPIE_CALLBACK_WAIT`; their return types do not change inside callbacks. To cancel from `onAudio` or `onProgress`, synchronously call the caller-owned `controller.abort()` and return. The external owner then awaits the synthesis completion and cleanup. Likewise, do not await a new queued synthesis from a callback that the current synthesis is awaiting.
|
|
89
|
+
|
|
90
|
+
The synthesis promise rejects on cancellation or failure. Native errors, callback failures, and cleanup failures remain observable; combined failures are preserved in aggregates. Complete audio is never reported for an interrupted request.
|
|
91
|
+
|
|
92
|
+
## Build the first-party helper
|
|
93
|
+
|
|
94
|
+
`arcane-os/local-ai/magpie/build` exports named/default `buildMagpieHelper({runtime, outputRoot, signal, onEvent})`. The selected upstream installation supplies `runtime.cmakeDirectory`, containing `NeMoSpeechConfig.cmake`, its public TTS headers, and its platform-matching libraries. Build prerequisites are CMake 3.26 or newer, a C++17 compiler, platform threads support, and that prepared NeMo TTS installation.
|
|
95
|
+
|
|
96
|
+
The builder configures, builds, and installs the SDK's `arcane-magpie` executable under `outputRoot`. It returns `{platform, architecture, root, helperExecutable, diagnostics: {configure, compile, install}}`. Combine its executable with the selected upstream `libraryDirectory` for the runtime descriptor. The SDK does not vendor NeMo libraries, models, tokenizer assets, or voices.
|
|
97
|
+
|
|
98
|
+
The source targets Windows, Linux, and macOS; Android requires its host adaptation and compatible upstream runtime. On October 17, 2026, the selected Windows x64 helper configured, compiled, and installed successfully against installed NeMo-Speech.cpp 0.2.0 using CMake 4.4.3, Visual Studio 2022, MSVC 19.44.35217, and Windows SDK 10.0.26100. Each operation exited 0, with complete diagnostics retained in the SDK's ignored selected-output directory. The helper was not launched: model inference, microphone capture, and audio playback remain unexecuted for this adapter. Linux and macOS compilation and execution remain unverified. Build success, actual selected-device inference, and audible playback are separate evidence boundaries.
|
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
# Native Nemotron streaming transcription
|
|
2
|
+
|
|
3
|
+
`arcane-os/local-ai/nemotron` provides `createNemotronRuntime`, a Node/Core
|
|
4
|
+
adapter for NVIDIA's public NeMo Speech ASR C ABI. One owned helper retains the
|
|
5
|
+
selected recognizer and optional diarization model. One active audio stream
|
|
6
|
+
belongs to that engine at a time. Existing Whisper transcription and standalone
|
|
7
|
+
diarization remain separate supported capabilities.
|
|
8
|
+
|
|
9
|
+
This adapter has no microphone, playback, conversational endpoint detector,
|
|
10
|
+
keyword classifier, cloud fallback or automatic engine replacement. The host
|
|
11
|
+
owns capture and the model owns conversational interpretation. A final ASR
|
|
12
|
+
segment does not establish that the person has finished thinking, yielded the
|
|
13
|
+
conversation, approved an action or issued an instruction.
|
|
14
|
+
|
|
15
|
+
## Preparation and selection
|
|
16
|
+
|
|
17
|
+
Select an upstream NeMo Speech runtime with its public ASR headers, library,
|
|
18
|
+
runtime dependencies and CMake package configuration. This implementation was
|
|
19
|
+
source-reviewed against the installed NVIDIA NeMo-Speech.cpp 0.2.0 public ABI,
|
|
20
|
+
`nemotron-3.5-asr-streaming-0.6b` and `Nemotron-3-Diarization`. Acquire runtime and
|
|
21
|
+
model assets through their upstream distribution; the SDK adapter does not
|
|
22
|
+
download or redistribute them.
|
|
23
|
+
|
|
24
|
+
`arcane-os/local-ai/nemotron/build` exports `buildNemotronHelper`:
|
|
25
|
+
|
|
26
|
+
```js
|
|
27
|
+
import {buildNemotronHelper} from 'arcane-os/local-ai/nemotron/build';
|
|
28
|
+
import {createNemotronRuntime} from 'arcane-os/local-ai/nemotron';
|
|
29
|
+
|
|
30
|
+
const built = await buildNemotronHelper(
|
|
31
|
+
{
|
|
32
|
+
runtime: {cmakeDirectory: selectedNeMoCmakeDirectory},
|
|
33
|
+
outputRoot: selectedHelperOutputDirectory,
|
|
34
|
+
signal,
|
|
35
|
+
onEvent
|
|
36
|
+
}
|
|
37
|
+
);
|
|
38
|
+
const engine = createNemotronRuntime(
|
|
39
|
+
{
|
|
40
|
+
runtime: {
|
|
41
|
+
helperExecutable: built.helperExecutable,
|
|
42
|
+
binaryDirectory: selectedNeMoBinaryDirectory,
|
|
43
|
+
libraryDirectory: selectedNeMoLibraryDirectory
|
|
44
|
+
},
|
|
45
|
+
modelPath: selectedAsrModelPath,
|
|
46
|
+
diarizationModelPath: selectedDiarizationModelPath,
|
|
47
|
+
gpu: selectedNativeGpuOrdinal,
|
|
48
|
+
onEvent
|
|
49
|
+
}
|
|
50
|
+
);
|
|
51
|
+
await engine.load();
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
The builder requires CMake 3.26 or later and a platform C++17 toolchain. It
|
|
55
|
+
configures, builds and installs only the first-party helper, returning
|
|
56
|
+
`helperExecutable`, its `executable` alias, `root`, `platform`, `architecture`
|
|
57
|
+
and complete configure/build/install diagnostics. The caller selects the output
|
|
58
|
+
directory and supplies the appropriate installed upstream runtime. Windows
|
|
59
|
+
loads its runtime through the child's `PATH`; Linux uses `LD_LIBRARY_PATH` and
|
|
60
|
+
macOS uses `DYLD_LIBRARY_PATH`, alongside the helper's loader-relative paths.
|
|
61
|
+
`runtime.env` supplies additional child-only environment settings.
|
|
62
|
+
|
|
63
|
+
`gpu` is required: supply the native runtime's selected GPU ordinal or explicitly
|
|
64
|
+
select `-1` for CPU. Device selection belongs to the host. When the host changes
|
|
65
|
+
`CUDA_VISIBLE_DEVICES`, the ordinal must describe that child's visible devices.
|
|
66
|
+
The adapter reports `requestedGpu`; `observedBackend` remains `null` because a
|
|
67
|
+
requested device alone does not prove GPU inference.
|
|
68
|
+
|
|
69
|
+
Instead of `runtime`, supply an asynchronous
|
|
70
|
+
`prepare({signal, onEvent})` function returning that prepared descriptor.
|
|
71
|
+
Preparation runs during `load`, never at construction. `modelPath`,
|
|
72
|
+
`diarizationModelPath` and `gpu` may also be supplied by the returned descriptor.
|
|
73
|
+
Omit `diarizationModelPath` when speaker labels are not requested.
|
|
74
|
+
|
|
75
|
+
## Engine lifecycle
|
|
76
|
+
|
|
77
|
+
- `current()` returns selected provider/model, lifecycle `state`, `loaded`,
|
|
78
|
+
`busy`, native version, requested device, active speaker scope, errors and any
|
|
79
|
+
explicit reset requirement.
|
|
80
|
+
- `subscribe(listener, {replay: true, signal} = {})` immediately replays current
|
|
81
|
+
state and observes subsequent lifecycle events; it returns an unsubscribe
|
|
82
|
+
function.
|
|
83
|
+
- `load({modelId, modelPath, diarizationModelPath, gpu, signal} = {})` loads the
|
|
84
|
+
selected models and resolves only when native readiness is reported.
|
|
85
|
+
`modelId` defaults to `nemotron-3.5-asr-streaming-0.6b`. Concurrent callers share
|
|
86
|
+
that load; one caller's signal cancels its wait, not the shared load. Unload
|
|
87
|
+
before replacing an active selection.
|
|
88
|
+
- `unload()` cancels and joins the active stream and model preparation, closes
|
|
89
|
+
the owned helper, and retains selection for a later explicit load.
|
|
90
|
+
- `close()` joins unload and disposes lifecycle subscriptions.
|
|
91
|
+
|
|
92
|
+
The host can compose this engine into the existing
|
|
93
|
+
[`createSpeechService`](./native-speech.md) STT engine seam. Readiness follows
|
|
94
|
+
the selected engine, not an application startup barrier.
|
|
95
|
+
|
|
96
|
+
## Ordered streaming
|
|
97
|
+
|
|
98
|
+
```js
|
|
99
|
+
const stream = await engine.openStream(
|
|
100
|
+
{
|
|
101
|
+
model: 'nemotron-3.5-asr-streaming-0.6b',
|
|
102
|
+
sampleRate: actualCaptureSampleRate,
|
|
103
|
+
signal,
|
|
104
|
+
onUpdate: async function showRecognition(result) {
|
|
105
|
+
await displayCompleteSnapshot(result);
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
);
|
|
109
|
+
await stream.push(monoFloat32Samples);
|
|
110
|
+
await stream.forceEndpoint();
|
|
111
|
+
// The same stream accepts the next utterance, preserving its speaker scope.
|
|
112
|
+
await stream.push(nextMonoFloat32Samples);
|
|
113
|
+
await stream.finish();
|
|
114
|
+
await stream.completion;
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
`openStream({sampleRate = 16000, languageCode, model, onUpdate, signal} = {})`
|
|
118
|
+
requires the selected model to be ready, or waits for its already-started load.
|
|
119
|
+
It never silently reloads after a reset. An explicit `model` must match the
|
|
120
|
+
loaded selection. `sampleRate` is the actual input rate: an integer from 8000
|
|
121
|
+
through 96000, or `0` for the model rate. It is passed unchanged to the native
|
|
122
|
+
API. The transport's ordered 100 ms scheduling does not reinterpret a 48 kHz
|
|
123
|
+
recording as 16 kHz or discard audio.
|
|
124
|
+
|
|
125
|
+
The returned handle contains `push`, `forceEndpoint`, `finish`, `cancel`,
|
|
126
|
+
`completion`, `current`, `sampleRate` and `speakerScope`:
|
|
127
|
+
|
|
128
|
+
- `push(audio)` accepts mono `Float32Array` or
|
|
129
|
+
`{encoding: 'f32le', data: BASE64}` and resolves to all ordered snapshots from
|
|
130
|
+
that input. The encoded form carries exact little-endian float32 samples.
|
|
131
|
+
Keep a supplied array unchanged until its push settles. Other encoded
|
|
132
|
+
recordings produce `NEMOTRON_AUDIO_FORMAT_UNSUPPORTED`; caller-owned decoding
|
|
133
|
+
is required.
|
|
134
|
+
- `forceEndpoint()` drains the current ASR utterance while retaining the native
|
|
135
|
+
stream, recognizer and diarization scope. Capture closure and conversational
|
|
136
|
+
turn completion remain the host's decisions.
|
|
137
|
+
- `finish()` queues behind all accepted input, terminally finishes and drains
|
|
138
|
+
the native stream, delivers remaining snapshots, and resolves with that final
|
|
139
|
+
drain's snapshot array. The loaded engine remains available for another
|
|
140
|
+
stream.
|
|
141
|
+
- `current()` returns the latest delivered snapshot, or `null` before one is
|
|
142
|
+
available.
|
|
143
|
+
- `completion` settles after terminal finish or joined cancellation. Successful
|
|
144
|
+
finish resolves to the final drain's snapshots; earlier snapshots have already
|
|
145
|
+
been delivered through `onUpdate` and each operation's result.
|
|
146
|
+
|
|
147
|
+
Each snapshot includes `sequence`, `final`, `audioProcessedSeconds`,
|
|
148
|
+
`channelTag`, `speakerScope`, and every native `alternatives` record with its
|
|
149
|
+
exact `text`, `confidence`, `languageCodes` and `words`. Every word preserves
|
|
150
|
+
`word`, `startTimeMs`, `endTimeMs`, `confidence` and numeric `speaker`. Top-level
|
|
151
|
+
`text` and `words` expose the first alternative without changing its content.
|
|
152
|
+
|
|
153
|
+
Partial snapshots are provisional replacements, not appended transcript text.
|
|
154
|
+
Native endpoint finals describe individual utterances in a continuing stream;
|
|
155
|
+
they are forwarded unchanged. Keep their boundaries distinct. Speaker numbers
|
|
156
|
+
distinguish voices within the reported scope, not named people or biometric
|
|
157
|
+
identity. A new stream has a new scope. Timestamp values retain the native
|
|
158
|
+
runtime's units and origin without an invented offset.
|
|
159
|
+
|
|
160
|
+
Callbacks run serially in native result order and may be asynchronous. A
|
|
161
|
+
callback failure stops the stream, remains observable, and retains complete
|
|
162
|
+
native result records in diagnostics. Inside an active `onUpdate` callback,
|
|
163
|
+
calling `push`, `forceEndpoint`, `finish`, `cancel`, `unload` or `close` raises
|
|
164
|
+
`NEMOTRON_CALLBACK_WAIT`: those joined operations would wait for the callback
|
|
165
|
+
itself. Abort the caller-owned signal synchronously if cancellation is needed
|
|
166
|
+
there, return from the callback, then join through the external owner.
|
|
167
|
+
|
|
168
|
+
## Cancellation and complete recordings
|
|
169
|
+
|
|
170
|
+
`cancel(reason)` and an aborted stream signal first request ordinary native
|
|
171
|
+
stream closure. If a noninterruptible public-ABI call is in flight, cancellation
|
|
172
|
+
terminates only this engine's owned helper and joins it. It also joins accepted
|
|
173
|
+
JavaScript operations and output callbacks before releasing stream ownership.
|
|
174
|
+
Explicit cancellation resolves `completion` and `cancel` with
|
|
175
|
+
`{cancelled: true, resetRequired, speakerScope, diagnostics?}`. Actual native,
|
|
176
|
+
callback or cleanup failures reject and retain their diagnostic causes.
|
|
177
|
+
|
|
178
|
+
After helper termination, `current()` reports `resetRequired: true`, a reset
|
|
179
|
+
record and `state: 'unloaded'`. The next caller must explicitly `load()`;
|
|
180
|
+
recognizer replacement and speaker-scope reset are never silent. Ordinary
|
|
181
|
+
stream closure keeps the loaded helper. No cancellation claims that unfinished
|
|
182
|
+
recognition produced a complete final.
|
|
183
|
+
|
|
184
|
+
`transcribe(request, {signal, onProgress, onUpdate} = {})` uses the same stream
|
|
185
|
+
implementation for complete recordings. Supply `{audio, sampleRate, model?,
|
|
186
|
+
languageCode?}` or a direct supported audio value at the default sample rate.
|
|
187
|
+
It returns `{text, words, results, speakerScope}`, retaining all final records
|
|
188
|
+
in `results`, and emits accepted/recognizing progress. Its final text is the
|
|
189
|
+
ordered concatenation of native final text without inserted separators. It
|
|
190
|
+
does not decode WAV, MP3 or other encoded recording formats.
|
|
191
|
+
|
|
192
|
+
## Evidence boundary
|
|
193
|
+
|
|
194
|
+
The adapter, transport and platform paths have been source-reviewed. On October
|
|
195
|
+
17, 2026, the selected Windows x64 helper configured, compiled and installed
|
|
196
|
+
successfully against installed NeMo-Speech.cpp 0.2.0 using CMake 4.4.3, Visual
|
|
197
|
+
Studio 2022, MSVC 19.44.35217 and Windows SDK 10.0.26100. Each operation exited 0;
|
|
198
|
+
complete diagnostics remain in the SDK's ignored selected-output directory. The
|
|
199
|
+
helper was not launched, and there is no measured latency, GPU-inference,
|
|
200
|
+
microphone, diarization, audio playback or multi-utterance acceptance result for
|
|
201
|
+
this adapter. Linux and macOS compilation and execution remain unverified.
|
|
202
|
+
Compilation, header review and upstream documentation remain distinct from a
|
|
203
|
+
built native application's observed behavior. Android requires the necessary
|
|
204
|
+
host integration and a matching native runtime.
|