arcane-os 0.93.0 → 0.94.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/README.md +1 -1
- package/browser-runtime/ai/twin-cloud.mjs +109 -46
- package/browser-runtime/core/client.mjs +2 -0
- package/docs/reference/ai/twin-cloud.md +35 -14
- package/docs/reference/availability-and-normalization.md +1 -1
- package/docs/reference/inventory/package-api.json +95 -11
- package/docs/reference/local-ai.md +81 -5
- package/docs/reference/native-speech.md +220 -11
- package/docs/reference/runtime-modules.md +21 -18
- package/docs/reference/sdk-api.md +152 -11
- package/package.json +2 -1
- package/schemas/arcane-app.schema.json +2 -2
- package/schemas/arcane-package.schema.json +2 -2
- package/src/core/services/speech.mjs +85 -23
- package/src/local-ai/config.mjs +12 -1
- package/src/local-ai/dev.mjs +71 -4
- package/src/local-ai/install.mjs +113 -5
- package/src/local-ai/kokoro/frontend.mjs +219 -0
- package/src/local-ai/kokoro/helper.mjs +168 -0
- package/src/local-ai/kokoro/index.mjs +645 -0
- package/src/local-ai/kokoro/native/CMakeLists.txt +45 -0
- package/src/local-ai/kokoro/native/main.cpp +249 -0
- package/src/native/portable-layout.mjs +67 -9
package/src/local-ai/install.mjs
CHANGED
|
@@ -10,7 +10,7 @@ import {ArcaneError,ERROR_CODES,normalizeError,throwIfAborted} from '../errors.m
|
|
|
10
10
|
import {createEventQueue} from '../event-queue.mjs';
|
|
11
11
|
import {runProcess} from '../process.mjs';
|
|
12
12
|
import {extractLocalAIArchive} from './archive.mjs';
|
|
13
|
-
import {normalizeImageRuntimeRequirement,normalizeLlamaRuntimeRequirement} from './config.mjs';
|
|
13
|
+
import {normalizeImageRuntimeRequirement,normalizeKokoroRuntimeRequirement,normalizeLlamaRuntimeRequirement} from './config.mjs';
|
|
14
14
|
import {normalizeWhisperRuntimeRequirement} from './whisper/config.mjs';
|
|
15
15
|
import {bundledWhisperRuntime, installedWhisperRuntime, installWhisperDistribution, whisperRuntimeSelection} from './whisper/install.mjs';
|
|
16
16
|
|
|
@@ -28,6 +28,21 @@ function selectedONNXNativeDistribution(runtime,platform,architecture){
|
|
|
28
28
|
?onnxWindowsDistribution:null;
|
|
29
29
|
}
|
|
30
30
|
|
|
31
|
+
function kokoroRuntimeSelection(runtime) {
|
|
32
|
+
return JSON.stringify(
|
|
33
|
+
{
|
|
34
|
+
source: runtime.helperRoot ? 'directory' : runtime.url ? 'archive' : 'sdk-release',
|
|
35
|
+
version: runtime.version ?? null,
|
|
36
|
+
url: runtime.url ?? null,
|
|
37
|
+
helperRoot: runtime.helperRoot ? path.resolve(runtime.helperRoot) : null,
|
|
38
|
+
helperExecutable: runtime.helperExecutable ?? null,
|
|
39
|
+
espeakDataDirectory: runtime.espeakDataDirectory ?? null,
|
|
40
|
+
libraryDirectory: runtime.libraryDirectory ?? null,
|
|
41
|
+
sdkVersion: !runtime.helperRoot && !runtime.url ? packageMetadata.version : null
|
|
42
|
+
}
|
|
43
|
+
);
|
|
44
|
+
}
|
|
45
|
+
|
|
31
46
|
function selectRuntimes(runtimes){
|
|
32
47
|
if(!is.array(runtimes)){
|
|
33
48
|
throw new ArcaneError(ERROR_CODES.usage,'Local AI runtimes must be an array.');
|
|
@@ -37,14 +52,23 @@ function selectRuntimes(runtimes){
|
|
|
37
52
|
const requirement=is.string(item)?{id:item}:item;
|
|
38
53
|
const runtime=requirement?.id==='stable-diffusion.cpp'?normalizeImageRuntimeRequirement(requirement)
|
|
39
54
|
:requirement?.id==='whisper.cpp'?normalizeWhisperRuntimeRequirement(requirement)
|
|
55
|
+
:requirement?.id==='kokoro-native'?normalizeKokoroRuntimeRequirement(requirement)
|
|
40
56
|
:requirement?.id==='llama.cpp'?normalizeLlamaRuntimeRequirement(requirement):requirement;
|
|
41
|
-
if(!runtime||!['llama.cpp','ollama','nemo-speech','onnx','stable-diffusion.cpp','whisper.cpp'].includes(runtime.id)){
|
|
57
|
+
if(!runtime||!['llama.cpp','ollama','nemo-speech','onnx','stable-diffusion.cpp','whisper.cpp','kokoro-native'].includes(runtime.id)){
|
|
42
58
|
throw new ArcaneError(ERROR_CODES.targetUnavailable,`Local AI runtime installation is unavailable for ${String(runtime?.id)}.`);
|
|
43
59
|
}
|
|
44
60
|
const prior=selected.get(runtime.id);
|
|
45
61
|
if(prior&&runtime.id==='whisper.cpp'&&whisperRuntimeSelection(prior)!==whisperRuntimeSelection(runtime)){
|
|
46
62
|
throw new ArcaneError(ERROR_CODES.usage,'Select one complete whisper.cpp runtime and model requirement for this operation.');
|
|
47
63
|
}
|
|
64
|
+
if (runtime.id === 'kokoro-native') {
|
|
65
|
+
if (runtime.helperRoot && runtime.url) {
|
|
66
|
+
throw new ArcaneError(ERROR_CODES.usage, 'Select either a kokoro-native helperRoot or an archive URL for this operation.');
|
|
67
|
+
}
|
|
68
|
+
if (prior && kokoroRuntimeSelection(prior) !== kokoroRuntimeSelection(runtime)) {
|
|
69
|
+
throw new ArcaneError(ERROR_CODES.usage, 'Select one complete kokoro-native helper requirement for this operation.');
|
|
70
|
+
}
|
|
71
|
+
}
|
|
48
72
|
if(prior&&(prior.version!==runtime.version||prior.url!==runtime.url
|
|
49
73
|
||(['stable-diffusion.cpp','llama.cpp'].includes(runtime.id)&&prior.backend!==runtime.backend)
|
|
50
74
|
||(runtime.id==='llama.cpp'&&prior.companionUrl!==runtime.companionUrl))){
|
|
@@ -111,6 +135,20 @@ async function selectRelease(runtime,platform,architecture,signal){
|
|
|
111
135
|
if (runtime.id === 'stable-diffusion.cpp') {
|
|
112
136
|
return selectImageRelease(runtime, platform, architecture, signal);
|
|
113
137
|
}
|
|
138
|
+
if (runtime.id === 'kokoro-native') {
|
|
139
|
+
if (runtime.helperRoot) return null;
|
|
140
|
+
if (!runtime.url) {
|
|
141
|
+
if (platform !== 'win32' || architecture !== 'x64') {
|
|
142
|
+
throw new ArcaneError(ERROR_CODES.targetUnavailable, `Supply a complete kokoro-native archive URL or helperRoot for ${platform}/${architecture}.`);
|
|
143
|
+
}
|
|
144
|
+
const name = 'arcane-kokoro-windows-x64.tar.gz';
|
|
145
|
+
return {
|
|
146
|
+
version: packageMetadata.version,
|
|
147
|
+
url: `https://github.com/TheWizardNexus/arcane-os-sdk/releases/download/${packageMetadata.version}/${name}`,
|
|
148
|
+
name
|
|
149
|
+
};
|
|
150
|
+
}
|
|
151
|
+
}
|
|
114
152
|
const backend=runtime.id==='llama.cpp'?llamaBackendCandidate(runtime,platform,architecture):undefined;
|
|
115
153
|
if(runtime.url){
|
|
116
154
|
const name=path.posix.basename(new URL(runtime.url).pathname);
|
|
@@ -432,6 +470,7 @@ async function installedRuntime(base,runtime,platform,architecture,signal,reuseW
|
|
|
432
470
|
throw error;
|
|
433
471
|
}
|
|
434
472
|
if(installation.requestVersion!==(runtime.version??null)||installation.requestUrl!==(runtime.url??null))continue;
|
|
473
|
+
if (runtime.id === 'kokoro-native' && installation.requestKokoro !== kokoroRuntimeSelection(runtime)) continue;
|
|
435
474
|
if (runtime.id === 'stable-diffusion.cpp' && installation.requestBackend !== runtime.backend) continue;
|
|
436
475
|
if(runtime.id==='llama.cpp'&&((installation.requestBackend??'auto')!==runtime.backend
|
|
437
476
|
||(installation.requestCompanionUrl??null)!==(runtime.companionUrl??null)))continue;
|
|
@@ -445,6 +484,10 @@ async function installedRuntime(base,runtime,platform,architecture,signal,reuseW
|
|
|
445
484
|
if(record?.id!==runtime.id||record.platform!==platform||record.architecture!==architecture)continue;
|
|
446
485
|
if(runtime.id==='llama.cpp'&&record.backend!==undefined&&record.backend!==llamaBackend)continue;
|
|
447
486
|
try{
|
|
487
|
+
if (runtime.id === 'kokoro-native') {
|
|
488
|
+
if (await installedKokoroRuntime(record, signal)) return record;
|
|
489
|
+
continue;
|
|
490
|
+
}
|
|
448
491
|
if(runtime.id==='whisper.cpp'){
|
|
449
492
|
if(await installedWhisperRuntime(record,signal,!reuseWhisper))return reuseWhisper?{runtime:record,requirement:whisperRequirement}:record;
|
|
450
493
|
continue;
|
|
@@ -507,6 +550,51 @@ async function installedRuntime(base,runtime,platform,architecture,signal,reuseW
|
|
|
507
550
|
return null;
|
|
508
551
|
}
|
|
509
552
|
|
|
553
|
+
function kokoroRuntimeLocations(runtime, root, platform) {
|
|
554
|
+
const sourceRoot = runtime.helperRoot ? path.resolve(runtime.helperRoot) : root;
|
|
555
|
+
const defaults = {
|
|
556
|
+
helperExecutable: path.join('bin', `arcane-kokoro${platform === 'win32' ? '.exe' : ''}`),
|
|
557
|
+
espeakDataDirectory: 'share',
|
|
558
|
+
libraryDirectory: 'bin'
|
|
559
|
+
};
|
|
560
|
+
const locations = {};
|
|
561
|
+
for (const [field, fallback] of Object.entries(defaults)) {
|
|
562
|
+
const selected = path.resolve(sourceRoot, runtime[field] ?? fallback);
|
|
563
|
+
const relative = path.relative(sourceRoot, selected);
|
|
564
|
+
if (relative === '..' || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative)) {
|
|
565
|
+
throw new ArcaneError(ERROR_CODES.usage, `kokoro-native ${field} must belong to the selected complete helper tree so it can be relocated.`);
|
|
566
|
+
}
|
|
567
|
+
locations[field] = path.join(root, relative);
|
|
568
|
+
}
|
|
569
|
+
return locations;
|
|
570
|
+
}
|
|
571
|
+
|
|
572
|
+
async function installedKokoroRuntime(record, signal) {
|
|
573
|
+
throwIfAborted(signal);
|
|
574
|
+
const locations = [record.helperExecutable, record.espeakDataDirectory, record.libraryDirectory];
|
|
575
|
+
const missingLocation = locations.some(
|
|
576
|
+
function missingKokoroLocation(location) {
|
|
577
|
+
return !is.string(location);
|
|
578
|
+
}
|
|
579
|
+
);
|
|
580
|
+
if (missingLocation) return false;
|
|
581
|
+
// eSpeak's data argument is the parent of its espeak-ng-data directory.
|
|
582
|
+
locations.push(path.join(record.espeakDataDirectory, 'espeak-ng-data'));
|
|
583
|
+
const entries = await Promise.all(
|
|
584
|
+
locations.map(
|
|
585
|
+
function inspectKokoroLocation(location) {
|
|
586
|
+
return stat(location);
|
|
587
|
+
}
|
|
588
|
+
)
|
|
589
|
+
);
|
|
590
|
+
throwIfAborted(signal);
|
|
591
|
+
return entries.every(
|
|
592
|
+
function kokoroLocationPresent(info, index) {
|
|
593
|
+
return index === 0 ? info.isFile() : info.isDirectory();
|
|
594
|
+
}
|
|
595
|
+
);
|
|
596
|
+
}
|
|
597
|
+
|
|
510
598
|
async function runtimeExecutable(root,id,platform,signal){
|
|
511
599
|
const filename=`${id==='llama.cpp'?'llama-server':'ollama'}${platform==='win32'?'.exe':''}`;
|
|
512
600
|
const directories=[root];
|
|
@@ -645,6 +733,22 @@ async function installRuntime(runtime,{directory,platform,architecture,signal,on
|
|
|
645
733
|
release,
|
|
646
734
|
{root, directory, platform, architecture, signal, onEvent}
|
|
647
735
|
);
|
|
736
|
+
} else if (runtime.id === 'kokoro-native' && runtime.helperRoot) {
|
|
737
|
+
const sourceRoot = path.resolve(runtime.helperRoot);
|
|
738
|
+
const destination = path.relative(sourceRoot, root);
|
|
739
|
+
if (destination !== '..' && !destination.startsWith(`..${path.sep}`) && !path.isAbsolute(destination)) {
|
|
740
|
+
throw new ArcaneError(ERROR_CODES.usage, 'The kokoro-native installation directory must be outside helperRoot so copying does not include its own output.');
|
|
741
|
+
}
|
|
742
|
+
locations = {version: runtime.version ?? 'custom', ...kokoroRuntimeLocations(runtime, root, platform)};
|
|
743
|
+
await onEvent(
|
|
744
|
+
{
|
|
745
|
+
type: 'local-ai.install.copying',
|
|
746
|
+
message: 'Copying the selected complete Kokoro helper tree.',
|
|
747
|
+
data: {id: runtime.id, platform, architecture}
|
|
748
|
+
}
|
|
749
|
+
);
|
|
750
|
+
const files = [];
|
|
751
|
+
await copyRuntimeTree(sourceRoot, root, root, files, signal);
|
|
648
752
|
}else{
|
|
649
753
|
const assets=[{url:release.url,name:release.name,archive}];
|
|
650
754
|
if(companionArchive){
|
|
@@ -679,13 +783,17 @@ async function installRuntime(runtime,{directory,platform,architecture,signal,on
|
|
|
679
783
|
...(runtime.id==='llama.cpp'?{backend:release.backend,requestedBackend:release.requestedBackend}:{}),
|
|
680
784
|
...(runtime.id==='nemo-speech'
|
|
681
785
|
?await nemoRuntimeDirectories(root,signal)
|
|
786
|
+
:runtime.id==='kokoro-native'?kokoroRuntimeLocations(runtime,root,platform)
|
|
682
787
|
:{executable:await runtimeExecutable(root,runtime.id,platform,signal)})};
|
|
683
788
|
}
|
|
684
789
|
const record={id:runtime.id,platform,architecture,root,...locations,...(runtime.version?{requestedVersion:runtime.version}:{})};
|
|
790
|
+
if (runtime.id === 'kokoro-native' && !await installedKokoroRuntime(record, signal)) {
|
|
791
|
+
throw new ArcaneError(ERROR_CODES.operationFailed, 'Kokoro preparation did not produce the selected helper executable, eSpeak data, and library directory.');
|
|
792
|
+
}
|
|
685
793
|
throwIfAborted(signal);
|
|
686
794
|
if(archive)await unlink(archive);
|
|
687
795
|
if(companionArchive)await unlink(companionArchive);
|
|
688
|
-
await writeFile(path.join(attempt,'installation.json'),`${JSON.stringify({requestVersion:runtime.version??null,requestUrl:runtime.url??null,...(['stable-diffusion.cpp','llama.cpp'].includes(runtime.id)?{requestBackend:runtime.backend}:{}),...(runtime.id==='llama.cpp'?{requestCompanionUrl:runtime.companionUrl??null}:{}),...(runtime.id==='whisper.cpp'?{requestWhisper:whisperRuntimeSelection(runtime)}:{}),runtime:record},null,2)}\n`,{flag:'wx',signal});
|
|
796
|
+
await writeFile(path.join(attempt,'installation.json'),`${JSON.stringify({requestVersion:runtime.version??null,requestUrl:runtime.url??null,...(['stable-diffusion.cpp','llama.cpp'].includes(runtime.id)?{requestBackend:runtime.backend}:{}),...(runtime.id==='llama.cpp'?{requestCompanionUrl:runtime.companionUrl??null}:{}),...(runtime.id==='whisper.cpp'?{requestWhisper:whisperRuntimeSelection(runtime)}:{}),...(runtime.id==='kokoro-native'?{requestKokoro:kokoroRuntimeSelection(runtime)}:{}),runtime:record},null,2)}\n`,{flag:'wx',signal});
|
|
689
797
|
completed=true;
|
|
690
798
|
await onEvent({type:'local-ai.install.completed',message:`${runtime.id} ${record.version} is available.`,data:record});
|
|
691
799
|
return record;
|
|
@@ -705,7 +813,7 @@ async function installRuntime(runtime,{directory,platform,architecture,signal,on
|
|
|
705
813
|
|
|
706
814
|
function shareInstallation(runtime,options){
|
|
707
815
|
throwIfAborted(options.signal);
|
|
708
|
-
const key=JSON.stringify([options.directory,options.platform,options.architecture,runtime.id,runtime.version??null,runtime.url??null,['stable-diffusion.cpp','llama.cpp'].includes(runtime.id)?runtime.backend:null,runtime.id==='whisper.cpp'?whisperRuntimeSelection(runtime):null,runtime.id==='llama.cpp'?runtime.companionUrl??null:null]);
|
|
816
|
+
const key=JSON.stringify([options.directory,options.platform,options.architecture,runtime.id,runtime.version??null,runtime.url??null,['stable-diffusion.cpp','llama.cpp'].includes(runtime.id)?runtime.backend:null,runtime.id==='whisper.cpp'?whisperRuntimeSelection(runtime):null,runtime.id==='llama.cpp'?runtime.companionUrl??null:null,runtime.id==='kokoro-native'?kokoroRuntimeSelection(runtime):null]);
|
|
709
817
|
let entry=installations.get(key);
|
|
710
818
|
if(entry?.controller.signal.aborted){
|
|
711
819
|
// The prior cancelled attempt owns its cleanup. A later request waits
|
|
@@ -855,7 +963,7 @@ export async function bundleLocalAIRuntimes({runtimes=[],directory,outputRoot,pl
|
|
|
855
963
|
const relative=`runtime/local-ai/${runtime.id}`;
|
|
856
964
|
const destination=path.join(root,relative);
|
|
857
965
|
const bundled=runtime.id==='whisper.cpp'?bundledWhisperRuntime(runtime,relative):{...runtime,root:relative};
|
|
858
|
-
for(const field of ['executable','modulePath','includeDirectory','libraryDirectory','binaryDirectory','cmakeDirectory','libraryPath','bindingModulePath']){
|
|
966
|
+
for(const field of ['executable','modulePath','includeDirectory','libraryDirectory','binaryDirectory','cmakeDirectory','libraryPath','bindingModulePath','helperExecutable','espeakDataDirectory']){
|
|
859
967
|
if(runtime[field]===undefined)continue;
|
|
860
968
|
bundled[field]=path.posix.join(relative,path.relative(runtime.root,runtime[field]).split(path.sep).join('/'));
|
|
861
969
|
}
|
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
import {readFile} from 'node:fs/promises';
|
|
2
|
+
import {endianness} from 'node:os';
|
|
3
|
+
import Is from 'strong-type';
|
|
4
|
+
|
|
5
|
+
const is = new Is(false);
|
|
6
|
+
|
|
7
|
+
export const KOKORO_MODEL = 'onnx-community/Kokoro-82M-v1.0-ONNX';
|
|
8
|
+
export const KOKORO_REVISION = '1939ad2a8e416c0acfeecc08a694d14ef25f2231';
|
|
9
|
+
export const KOKORO_SAMPLE_RATE = 24000;
|
|
10
|
+
export const KOKORO_VOICES = [
|
|
11
|
+
'af_heart', 'af_alloy', 'af_aoede', 'af_bella', 'af_jessica', 'af_kore',
|
|
12
|
+
'af_nicole', 'af_nova', 'af_river', 'af_sarah', 'af_sky',
|
|
13
|
+
'am_adam', 'am_echo', 'am_eric', 'am_fenrir', 'am_liam', 'am_michael',
|
|
14
|
+
'am_onyx', 'am_puck', 'am_santa', 'bf_emma', 'bf_isabella', 'bm_george',
|
|
15
|
+
'bm_lewis', 'bf_alice', 'bf_lily', 'bm_daniel', 'bm_fable'
|
|
16
|
+
];
|
|
17
|
+
|
|
18
|
+
/** Read the selected model's public tokenizer and voice tensors, without a second model store. */
|
|
19
|
+
export async function loadKokoroFrontend(paths, {signal} = {}) {
|
|
20
|
+
const resources = await Promise.allSettled([
|
|
21
|
+
readFile(paths.tokenizer, {encoding: 'utf8', signal}),
|
|
22
|
+
readFile(paths.tokenizerConfig, {encoding: 'utf8', signal}),
|
|
23
|
+
...KOKORO_VOICES.map(function readVoice(voice) { return readFile(paths.voices[voice], {signal}); })
|
|
24
|
+
]);
|
|
25
|
+
const failures = resources.filter(function failed(result) { return result.status === 'rejected'; })
|
|
26
|
+
.map(function cause(result) { return result.reason; });
|
|
27
|
+
if (signal?.aborted) {
|
|
28
|
+
const independent = failures.filter(function independentFailure(cause) {
|
|
29
|
+
return cause !== signal.reason && !(cause?.code === 'ABORT_ERR' && cause.cause === signal.reason);
|
|
30
|
+
});
|
|
31
|
+
if (independent.length) throw new AggregateError([signal.reason, ...independent], 'Kokoro resource loading was cancelled with errors.', {cause: signal.reason});
|
|
32
|
+
signal.throwIfAborted();
|
|
33
|
+
}
|
|
34
|
+
if (failures.length) throw new AggregateError(failures, 'Reading the complete Kokoro frontend resources failed.');
|
|
35
|
+
const tokenizer = JSON.parse(resources[0].value);
|
|
36
|
+
const configuration = JSON.parse(resources[1].value);
|
|
37
|
+
const vocabulary = tokenizer.model.vocab;
|
|
38
|
+
const context = configuration.model_max_length;
|
|
39
|
+
const boundary = vocabulary[configuration.pad_token];
|
|
40
|
+
if (!is.integer(context) || context < 3 || !is.integer(boundary)) {
|
|
41
|
+
throw new Error('The selected Kokoro tokenizer does not describe its context and boundary token.');
|
|
42
|
+
}
|
|
43
|
+
const voices = new Map();
|
|
44
|
+
// Independent files load concurrently; the activation owns every read until it settles.
|
|
45
|
+
for (let voiceIndex = 0; voiceIndex < KOKORO_VOICES.length; voiceIndex += 1) {
|
|
46
|
+
const voice = KOKORO_VOICES[voiceIndex];
|
|
47
|
+
signal?.throwIfAborted();
|
|
48
|
+
const encoded = resources[voiceIndex + 2].value;
|
|
49
|
+
const values = new Float32Array(Math.ceil(encoded.length / 4));
|
|
50
|
+
for (let index = 0; index < values.length; index += 1) {
|
|
51
|
+
values[index] = encoded.readFloatLE(index * 4);
|
|
52
|
+
}
|
|
53
|
+
voices.set(voice, values);
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
function tokens(clauses, voice) {
|
|
57
|
+
const language = voiceLanguage(voice);
|
|
58
|
+
const result = [];
|
|
59
|
+
for (let index = 0; index < clauses.length; index += 1) {
|
|
60
|
+
const clause = clauses[index];
|
|
61
|
+
const phonemes = kokoroPhonemes(clause.phonemes, language);
|
|
62
|
+
const beginning = clause.source.match(/^[\s;:,.!?—…"()“”]+/u)?.[0] ?? '';
|
|
63
|
+
let ending = clause.source.match(/[\s;:,.!?—…"()“”]+$/u);
|
|
64
|
+
// The public cursor can include one next-clause character saved by
|
|
65
|
+
// eSpeak's UngetC. Its phonemes belong to the next returned clause;
|
|
66
|
+
// recover only the preceding punctuation, leaving both records intact.
|
|
67
|
+
if (!ending && index + 1 < clauses.length) {
|
|
68
|
+
ending = clause.source.match(/[;:,.!?—…"()“”][\s;:,.!?—…"()“”]*(?=[^\s;:,.!?—…"()“”]$)/u);
|
|
69
|
+
}
|
|
70
|
+
const punctuation = ending && ending.index >= beginning.length ? ending[0] : '';
|
|
71
|
+
const complete = beginning.replace(/\s/gu, ' ') + phonemes + punctuation.replace(/\s/gu, ' ');
|
|
72
|
+
for (const symbol of complete) {
|
|
73
|
+
const id = vocabulary[symbol];
|
|
74
|
+
if (!is.integer(id)) {
|
|
75
|
+
const failure = new Error(`Kokoro has no token for the emitted phoneme ${JSON.stringify(symbol)}.`);
|
|
76
|
+
failure.code = 'KOKORO_PHONEME_UNSUPPORTED';
|
|
77
|
+
failure.data = {symbol, language, source: clause.source, phonemes: clause.phonemes, mappedPhonemes: complete};
|
|
78
|
+
throw failure;
|
|
79
|
+
}
|
|
80
|
+
result.push({id, symbol});
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
return result;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
function* segments(clauses, voice) {
|
|
87
|
+
const content = tokens(clauses, voice);
|
|
88
|
+
const style = voices.get(voice);
|
|
89
|
+
const capacity = context - 2;
|
|
90
|
+
let offset = 0;
|
|
91
|
+
while (offset < content.length) {
|
|
92
|
+
let end = Math.min(offset + capacity, content.length);
|
|
93
|
+
if (end < content.length) {
|
|
94
|
+
for (let candidate = end; candidate > offset; candidate -= 1) {
|
|
95
|
+
if (/[ ;:,.!?—…]/u.test(content[candidate - 1].symbol)) {
|
|
96
|
+
end = candidate;
|
|
97
|
+
break;
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
const count = end - offset;
|
|
102
|
+
const input = new BigInt64Array(count + 2);
|
|
103
|
+
input[0] = BigInt(boundary);
|
|
104
|
+
for (let index = 0; index < count; index += 1) input[index + 1] = BigInt(content[offset + index].id);
|
|
105
|
+
input[count + 1] = BigInt(boundary);
|
|
106
|
+
// The public model contract selects the style row by content-token count, excluding BOS/EOS.
|
|
107
|
+
const row = style.subarray(count * 256, (count + 1) * 256);
|
|
108
|
+
if (row.length !== 256) throw new Error(`The selected ${voice} tensor has no style row for ${count} phoneme tokens.`);
|
|
109
|
+
yield {input, style: row, tokenCount: count};
|
|
110
|
+
offset = end;
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
return {segments};
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
export function voiceLanguage(voice) {
|
|
118
|
+
if (!KOKORO_VOICES.includes(voice)) {
|
|
119
|
+
const failure = new Error(`The selected Kokoro voice ${String(voice)} is unavailable.`);
|
|
120
|
+
failure.code = 'KOKORO_VOICE_UNAVAILABLE';
|
|
121
|
+
throw failure;
|
|
122
|
+
}
|
|
123
|
+
return voice.startsWith('b') ? 'en-gb' : 'en-us';
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
/** Model-specific IPA spelling for Kokoro v1.0 English, including the versioned recipe:
|
|
127
|
+
* https://github.com/hexgrad/misaki/blob/main/misaki/espeak.py
|
|
128
|
+
* This is phoneme conversion; the caller's source text is never rewritten.
|
|
129
|
+
*/
|
|
130
|
+
export function kokoroPhonemes(ipa, language) {
|
|
131
|
+
const british = language === 'en-gb';
|
|
132
|
+
const phones = Array.from(ipa);
|
|
133
|
+
const output = [];
|
|
134
|
+
const diphthongs = new Map([
|
|
135
|
+
['aɪ', 'I'], ['aʊ', 'W'], ['eɪ', 'A'], ['ɔɪ', 'Y'],
|
|
136
|
+
['oʊ', 'O'], ['əʊ', 'Q'], ['tʃ', 'ʧ'], ['dʒ', 'ʤ']
|
|
137
|
+
]);
|
|
138
|
+
for (let index = 0; index < phones.length; index += 1) {
|
|
139
|
+
const phone = phones[index];
|
|
140
|
+
const tied = phones[index + 1] === '^';
|
|
141
|
+
const next = phones[index + (tied ? 2 : 1)];
|
|
142
|
+
const pair = phone + (next ?? '');
|
|
143
|
+
if (diphthongs.has(pair)) {
|
|
144
|
+
output.push(diphthongs.get(pair));
|
|
145
|
+
index += tied ? 2 : 1;
|
|
146
|
+
} else if (british && pair === 'eə') {
|
|
147
|
+
output.push('ɛː');
|
|
148
|
+
index += tied ? 2 : 1;
|
|
149
|
+
} else if (british && pair === 'iə') {
|
|
150
|
+
output.push('ɪə');
|
|
151
|
+
index += tied ? 2 : 1;
|
|
152
|
+
} else if (!british && pair === 'ɪə') {
|
|
153
|
+
output.push('iə');
|
|
154
|
+
index += tied ? 2 : 1;
|
|
155
|
+
} else if (tied && pair === 'əl') {
|
|
156
|
+
output.push('ᵊl');
|
|
157
|
+
index += 2;
|
|
158
|
+
} else if (phone === 'ʔ' && (next === 'n' || (next === 'ˌ' && phones[index + 2] === 'n'))
|
|
159
|
+
&& phones[index + (next === 'n' ? 2 : 3)] === '\u0329') {
|
|
160
|
+
output.push('tn');
|
|
161
|
+
index += next === 'n' ? 2 : 3;
|
|
162
|
+
} else if (phone === '\u0329') {
|
|
163
|
+
const consonant = output.pop();
|
|
164
|
+
if (consonant === undefined) throw new Error('eSpeak emitted a syllabic mark without its consonant.');
|
|
165
|
+
output.push('ᵊ', consonant);
|
|
166
|
+
} else if (phone === 'ɚ') output.push('əɹ');
|
|
167
|
+
else if (phone === 'ɝ') output.push('ɜɹ');
|
|
168
|
+
else if (phone === 'r') output.push('ɹ');
|
|
169
|
+
else if (phone === 'g') output.push('ɡ');
|
|
170
|
+
else if (phone === 'ɐ') output.push('ə');
|
|
171
|
+
else if (phone === 'ɬ') output.push('l');
|
|
172
|
+
else if (phone === 'x' || phone === 'ç') output.push('k');
|
|
173
|
+
else if (phone === 'ʔ') output.push('t');
|
|
174
|
+
else if (phone === 'ɾ') output.push('T');
|
|
175
|
+
else if (phone === 'ʲ') {
|
|
176
|
+
if (next === 'o' || next === 'ə') output.push('j');
|
|
177
|
+
} else if (phone === '\u0303') continue;
|
|
178
|
+
else if (phone === 'o') output.push('ɔ');
|
|
179
|
+
else if (phone === 'e') output.push('A');
|
|
180
|
+
else if (phone === '^') continue;
|
|
181
|
+
else if (phone === 'ː' && !british) {
|
|
182
|
+
if (phones[index - 1] === 'ɜ' && next !== 'ɹ' && next !== 'r') output.push('ɹ');
|
|
183
|
+
} else if (/\s/u.test(phone)) output.push(' ');
|
|
184
|
+
else output.push(phone);
|
|
185
|
+
}
|
|
186
|
+
return output.join('');
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
/** IEEE float WAV preserves the complete native waveform without amplitude clamps. */
|
|
190
|
+
export function encodeKokoroWav(chunks) {
|
|
191
|
+
const samples = chunks.reduce(function countSamples(total, chunk) { return total + chunk.length; }, 0);
|
|
192
|
+
const dataLength = samples * 4;
|
|
193
|
+
// RIFF's unsigned 32-bit chunk fields are an external format constraint, not a content policy.
|
|
194
|
+
if (dataLength + 48 > 0xffffffff) throw new RangeError('This complete waveform exceeds the WAV RIFF format; select Ogg Opus.');
|
|
195
|
+
const header = Buffer.alloc(56);
|
|
196
|
+
header.write('RIFF', 0);
|
|
197
|
+
header.writeUInt32LE(dataLength + 48, 4);
|
|
198
|
+
header.write('WAVEfmt ', 8);
|
|
199
|
+
header.writeUInt32LE(16, 16);
|
|
200
|
+
header.writeUInt16LE(3, 20);
|
|
201
|
+
header.writeUInt16LE(1, 22);
|
|
202
|
+
header.writeUInt32LE(KOKORO_SAMPLE_RATE, 24);
|
|
203
|
+
header.writeUInt32LE(KOKORO_SAMPLE_RATE * 4, 28);
|
|
204
|
+
header.writeUInt16LE(4, 32);
|
|
205
|
+
header.writeUInt16LE(32, 34);
|
|
206
|
+
header.write('fact', 36);
|
|
207
|
+
header.writeUInt32LE(4, 40);
|
|
208
|
+
header.writeUInt32LE(samples, 44);
|
|
209
|
+
header.write('data', 48);
|
|
210
|
+
header.writeUInt32LE(dataLength, 52);
|
|
211
|
+
return Buffer.concat([header, ...chunks.map(floatPCM)]);
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
export function floatPCM(samples) {
|
|
215
|
+
if (endianness() === 'LE') return Buffer.from(samples.buffer, samples.byteOffset, samples.byteLength);
|
|
216
|
+
const encoded = Buffer.allocUnsafe(samples.length * 4);
|
|
217
|
+
for (let index = 0; index < samples.length; index += 1) encoded.writeFloatLE(samples[index], index * 4);
|
|
218
|
+
return encoded;
|
|
219
|
+
}
|
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
import path from 'node:path';
|
|
2
|
+
import {PassThrough} from 'node:stream';
|
|
3
|
+
import Is from 'strong-type';
|
|
4
|
+
import {runProcess} from '../../process.mjs';
|
|
5
|
+
import {floatPCM} from './frontend.mjs';
|
|
6
|
+
|
|
7
|
+
const is = new Is(false);
|
|
8
|
+
|
|
9
|
+
/** One retained helper owns eSpeak's process-global state and the public Opus encoder. */
|
|
10
|
+
export function createKokoroHelper(runtime, {onEvent} = {}) {
|
|
11
|
+
const controller = new AbortController();
|
|
12
|
+
const input = new PassThrough();
|
|
13
|
+
const readiness = deferred();
|
|
14
|
+
let pending = null;
|
|
15
|
+
let fragment = Buffer.alloc(0);
|
|
16
|
+
let frame = null;
|
|
17
|
+
let sequence = 0;
|
|
18
|
+
let stopping = false;
|
|
19
|
+
let exited = false;
|
|
20
|
+
let terminalError = null;
|
|
21
|
+
const env = {...process.env};
|
|
22
|
+
const directory = runtime.libraryDirectory ?? path.dirname(runtime.helperExecutable);
|
|
23
|
+
const variable = process.platform === 'win32'
|
|
24
|
+
? Object.keys(env).find(function pathName(name) { return name.toLowerCase() === 'path'; }) ?? 'PATH'
|
|
25
|
+
: process.platform === 'darwin' ? 'DYLD_LIBRARY_PATH' : 'LD_LIBRARY_PATH';
|
|
26
|
+
env[variable] = [directory, env[variable]].filter(Boolean).join(path.delimiter);
|
|
27
|
+
const source = input[Symbol.asyncIterator]();
|
|
28
|
+
const incoming = {
|
|
29
|
+
[Symbol.asyncIterator]() { return this; },
|
|
30
|
+
next() { return source.next(); },
|
|
31
|
+
return() { input.end(); return source.return(); }
|
|
32
|
+
};
|
|
33
|
+
const completion = runProcess(runtime.helperExecutable, ['--data', runtime.espeakDataDirectory], {
|
|
34
|
+
env, signal: controller.signal, onEvent, input: incoming,
|
|
35
|
+
outputEncoding: {stdout: null, stderr: 'utf8'},
|
|
36
|
+
captureOutput: {stdout: false, stderr: true},
|
|
37
|
+
emitOutputEvents: {stdout: false, stderr: true},
|
|
38
|
+
onOutput: receive
|
|
39
|
+
}).then(function helperExited(result) {
|
|
40
|
+
if (!stopping) throw new Error('The Kokoro helper exited before being released.');
|
|
41
|
+
return result;
|
|
42
|
+
}).catch(function helperFailed(error) {
|
|
43
|
+
terminalError = error;
|
|
44
|
+
throw error;
|
|
45
|
+
}).finally(function helperSettled() {
|
|
46
|
+
exited = true;
|
|
47
|
+
input.end();
|
|
48
|
+
const failure = terminalError ?? new Error('The Kokoro helper has stopped.');
|
|
49
|
+
readiness.reject(failure);
|
|
50
|
+
pending?.response.reject(failure);
|
|
51
|
+
});
|
|
52
|
+
completion.catch(function observeHelperFailure() {});
|
|
53
|
+
|
|
54
|
+
function receive({stream, chunk}) {
|
|
55
|
+
if (stream !== 'stdout') return;
|
|
56
|
+
fragment = fragment.length ? Buffer.concat([fragment, chunk]) : chunk;
|
|
57
|
+
for (;;) {
|
|
58
|
+
if (!frame) {
|
|
59
|
+
const boundary = fragment.indexOf(10);
|
|
60
|
+
if (boundary < 0) return;
|
|
61
|
+
const [kind, id, length] = fragment.subarray(0, boundary).toString('utf8').split('\t');
|
|
62
|
+
frame = {kind, id, length: Number(length)};
|
|
63
|
+
if (!is.safeInteger(frame.length) || frame.length < 0) throw new Error('The Kokoro helper returned unreadable protocol framing.');
|
|
64
|
+
fragment = fragment.subarray(boundary + 1);
|
|
65
|
+
}
|
|
66
|
+
if (fragment.length < frame.length) return;
|
|
67
|
+
const payload = fragment.subarray(0, frame.length);
|
|
68
|
+
fragment = fragment.subarray(frame.length);
|
|
69
|
+
const record = frame;
|
|
70
|
+
frame = null;
|
|
71
|
+
if (record.kind === 'ready') {
|
|
72
|
+
readiness.resolve();
|
|
73
|
+
continue;
|
|
74
|
+
}
|
|
75
|
+
if (!pending || record.id !== pending.id) throw new Error('The Kokoro helper returned a response without its active request.');
|
|
76
|
+
if (record.kind === 'error') pending.response.reject(new Error(payload.toString('utf8')));
|
|
77
|
+
else if (record.kind === 'clause') pending.clauses.push(JSON.parse(payload.toString('utf8')));
|
|
78
|
+
else if (record.kind === 'page') pending.pages.push(Buffer.from(payload));
|
|
79
|
+
else if (record.kind === 'done') pending.response.resolve(pending);
|
|
80
|
+
else throw new Error(`The Kokoro helper returned an unknown record ${record.kind}.`);
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
async function command(kind, body, language, signal) {
|
|
85
|
+
signal?.throwIfAborted();
|
|
86
|
+
await readiness.promise;
|
|
87
|
+
signal?.throwIfAborted();
|
|
88
|
+
if (pending || stopping || exited) throw terminalError ?? new Error('The Kokoro helper is unavailable for a new operation.');
|
|
89
|
+
const operation = {id: String(++sequence), response: deferred(), clauses: [], pages: []};
|
|
90
|
+
pending = operation;
|
|
91
|
+
function cancel() { stopping = true; controller.abort(signal.reason); }
|
|
92
|
+
signal?.addEventListener('abort', cancel, {once: true});
|
|
93
|
+
try {
|
|
94
|
+
if (signal?.aborted) cancel();
|
|
95
|
+
const length = body.reduce(function frameLength(total, part) { return total + part.length; }, 0);
|
|
96
|
+
await write(Buffer.from(`${kind}\t${operation.id}\t${length}\t${language ?? ''}\n`));
|
|
97
|
+
for (const part of body) {
|
|
98
|
+
signal?.throwIfAborted();
|
|
99
|
+
await write(part);
|
|
100
|
+
}
|
|
101
|
+
const result = await operation.response.promise;
|
|
102
|
+
signal?.throwIfAborted();
|
|
103
|
+
return result;
|
|
104
|
+
} catch (failure) {
|
|
105
|
+
// A partial command cannot share this pipe with another request.
|
|
106
|
+
stopping = true;
|
|
107
|
+
controller.abort(failure);
|
|
108
|
+
let primary = failure;
|
|
109
|
+
if (signal?.aborted && expectedCancellation(failure)) primary = signal.reason;
|
|
110
|
+
try {
|
|
111
|
+
await completion;
|
|
112
|
+
} catch (completionFailure) {
|
|
113
|
+
if (completionFailure !== failure && !expectedCancellation(completionFailure)) {
|
|
114
|
+
throw new AggregateError([primary, completionFailure], 'The Kokoro command and helper shutdown failed.', {cause: primary});
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
throw primary;
|
|
118
|
+
} finally {
|
|
119
|
+
signal?.removeEventListener('abort', cancel);
|
|
120
|
+
if (pending === operation) pending = null;
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
function write(value) {
|
|
125
|
+
if (exited || controller.signal.aborted) return Promise.reject(terminalError ?? controller.signal.reason ?? new Error('The Kokoro helper stopped.'));
|
|
126
|
+
return new Promise(function writeInput(resolve, reject) {
|
|
127
|
+
input.write(value, function inputWritten(error) { if (error) reject(error); else resolve(); });
|
|
128
|
+
});
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
async function phonemize(text, language, {signal} = {}) {
|
|
132
|
+
if (text.includes('\0')) throw new TypeError('The public eSpeak text interface cannot consume an embedded NUL character.');
|
|
133
|
+
if (!text.isWellFormed()) throw new TypeError('The public eSpeak UTF-8 interface cannot represent an unpaired Unicode surrogate.');
|
|
134
|
+
const result = await command('phonemize', [Buffer.from(text, 'utf8')], language, signal);
|
|
135
|
+
return result.clauses;
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
async function encodeOpus(chunks, {signal} = {}) {
|
|
139
|
+
const result = await command('opus', chunks.map(floatPCM), null, signal);
|
|
140
|
+
return Buffer.concat(result.pages);
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
async function close(reason) {
|
|
144
|
+
stopping = true;
|
|
145
|
+
if (reason !== undefined) controller.abort(reason);
|
|
146
|
+
input.end();
|
|
147
|
+
try { await completion; }
|
|
148
|
+
catch (failure) {
|
|
149
|
+
if (!expectedCancellation(failure)) throw failure;
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
function expectedCancellation(failure) {
|
|
154
|
+
if (!controller.signal.aborted || failure?.code !== 'ARCANE_CANCELLED') return false;
|
|
155
|
+
if (is.array(failure.errors) && !failure.errors.every(expectedCancellation)) return false;
|
|
156
|
+
return failure.cause === controller.signal.reason || expectedCancellation(failure.cause);
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
return {ready: readiness.promise, completion, phonemize, encodeOpus, close,
|
|
160
|
+
get exited() { return exited; }};
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
function deferred() {
|
|
164
|
+
const value = {};
|
|
165
|
+
value.promise = new Promise(function pending(resolve, reject) { value.resolve = resolve; value.reject = reject; });
|
|
166
|
+
value.promise.catch(function observePendingFailure() {});
|
|
167
|
+
return value;
|
|
168
|
+
}
|