@qvac/inference 0.20.0 → 0.21.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +107 -33
- package/README.md +8 -5
- package/dist/api/assess-model-fit.d.ts +9 -4
- package/dist/api/assess-model-fit.js +9 -4
- package/dist/api/bci-transcribe.d.ts +4 -1
- package/dist/api/index.d.ts +1 -0
- package/dist/api/index.js +1 -0
- package/dist/api/rpc-server.d.ts +19 -0
- package/dist/api/rpc-server.js +44 -0
- package/dist/api/text-to-speech.d.ts +9 -0
- package/dist/api/text-to-speech.js +9 -0
- package/dist/api/transcribe.d.ts +11 -7
- package/dist/api/transcribe.js +47 -15
- package/dist/api/video-request.d.ts +5 -5
- package/dist/dispatch.js +39 -7
- package/dist/errors/index.d.ts +7 -0
- package/dist/errors/index.js +15 -0
- package/dist/handlers/assess-model-fit.d.ts +18 -6
- package/dist/handlers/assess-model-fit.js +215 -35
- package/dist/handlers/completion-orchestrate.js +17 -0
- package/dist/handlers/get-loaded-model-info.js +4 -1
- package/dist/handlers/get-model-info.js +29 -1
- package/dist/handlers/load-model/coreml-sidecars.d.ts +25 -0
- package/dist/handlers/load-model/coreml-sidecars.js +274 -0
- package/dist/handlers/load-model/hyperdrive.js +2 -2
- package/dist/handlers/load-model/registry-download-utils.d.ts +1 -0
- package/dist/handlers/load-model/registry-download-utils.js +2 -1
- package/dist/handlers/load-model/registry.js +59 -0
- package/dist/handlers/rpc-server.d.ts +23 -0
- package/dist/handlers/rpc-server.js +141 -0
- package/dist/index.d.ts +2 -1
- package/dist/index.js +2 -1
- package/dist/models/registry/models.d.ts +7731 -5672
- package/dist/models/registry/models.js +4652 -2526
- package/dist/models/registry/resource-profiles.js +481 -94
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +8 -6
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +2 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +73 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +37 -19
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +1 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +9 -5
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +16 -7
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +80 -8
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +9 -3
- package/dist/plugins/builtin/llamacpp-completion/transform.js +19 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +5 -3
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +2 -1
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +121 -0
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +56 -27
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +8 -2
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +7 -2
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.d.ts +2 -0
- package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +5 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +27 -10
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +181 -0
- package/dist/plugins/builtin/tts-ggml/plugin.js +84 -2
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +136 -0
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +18 -7
- package/dist/plugins/ops/bci-transcribe.d.ts +6 -2
- package/dist/plugins/ops/bci-transcribe.js +18 -13
- package/dist/plugins/ops/load-model.js +2 -2
- package/dist/plugins/ops/transcribe.d.ts +4 -2
- package/dist/plugins/ops/transcribe.js +39 -2
- package/dist/plugins/ops/translate-stats.d.ts +2 -1
- package/dist/plugins/ops/translate-stats.js +64 -7
- package/dist/plugins/ops/translate.js +20 -4
- package/dist/profiling/operation-metrics.js +56 -0
- package/dist/registry.js +4 -0
- package/dist/resources/model-fit/assess.d.ts +46 -2
- package/dist/resources/model-fit/assess.js +67 -3
- package/dist/resources/model-fit/estimators/llm.js +2 -2
- package/dist/resources/model-fit/estimators/whisper.js +3 -4
- package/dist/resources/model-fit/fit-stub/fetch-fit-stub.d.ts +102 -0
- package/dist/resources/model-fit/fit-stub/fetch-fit-stub.js +212 -0
- package/dist/resources/model-fit/fit-stub/project-fit-from-load.d.ts +27 -0
- package/dist/resources/model-fit/fit-stub/project-fit-from-load.js +31 -0
- package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.d.ts +38 -0
- package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.js +138 -0
- package/dist/resources/model-fit/floor.js +2 -2
- package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +9 -6
- package/dist/resources/model-fit/native-probe/advisory-fit.js +129 -80
- package/dist/resources/model-fit/native-probe/child-engine-fit.d.ts +2 -0
- package/dist/resources/model-fit/native-probe/child-engine-fit.js +41 -0
- package/dist/resources/model-fit/native-probe/classify-fit.d.ts +6 -0
- package/dist/resources/model-fit/native-probe/classify-fit.js +135 -0
- package/dist/resources/model-fit/native-probe/create-fit-request.d.ts +28 -0
- package/dist/resources/model-fit/native-probe/create-fit-request.js +51 -0
- package/dist/resources/model-fit/native-probe/engine-fit.d.ts +59 -0
- package/dist/resources/model-fit/native-probe/engine-fit.js +13 -0
- package/dist/resources/model-fit/native-probe/engines/asr.d.ts +10 -0
- package/dist/resources/model-fit/native-probe/engines/asr.js +103 -0
- package/dist/resources/model-fit/native-probe/engines/audiogen.d.ts +8 -0
- package/dist/resources/model-fit/native-probe/engines/audiogen.js +35 -0
- package/dist/resources/model-fit/native-probe/engines/diffusion.d.ts +7 -0
- package/dist/resources/model-fit/native-probe/engines/diffusion.js +55 -0
- package/dist/resources/model-fit/native-probe/engines/gpu.d.ts +10 -0
- package/dist/resources/model-fit/native-probe/engines/gpu.js +21 -0
- package/dist/resources/model-fit/native-probe/engines/llama.d.ts +13 -0
- package/dist/resources/model-fit/native-probe/engines/llama.js +145 -0
- package/dist/resources/model-fit/native-probe/engines/tts.d.ts +8 -0
- package/dist/resources/model-fit/native-probe/engines/tts.js +106 -0
- package/dist/resources/model-fit/native-probe/fit-outcome.d.ts +16 -0
- package/dist/resources/model-fit/native-probe/fit-outcome.js +1 -0
- package/dist/resources/model-fit/native-probe/fit-process.d.ts +27 -0
- package/dist/resources/model-fit/native-probe/fit-process.js +55 -0
- package/dist/resources/model-fit/native-probe/fit-runner.d.ts +1 -0
- package/dist/resources/model-fit/native-probe/fit-runner.js +58 -0
- package/dist/resources/model-fit/native-probe/run-fit.d.ts +24 -0
- package/dist/resources/model-fit/native-probe/run-fit.js +20 -0
- package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +14 -34
- package/dist/resources/model-fit/native-probe/run-in-process-fit.js +55 -139
- package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +5 -12
- package/dist/resources/model-fit/native-probe/run-isolated-fit.js +13 -8
- package/dist/resources/model-fit/platform.d.ts +9 -0
- package/dist/resources/model-fit/platform.js +28 -0
- package/dist/rpc/device-map.d.ts +13 -0
- package/dist/rpc/device-map.js +22 -0
- package/dist/rpc/discovery.d.ts +12 -0
- package/dist/rpc/discovery.js +225 -0
- package/dist/rpc/ggml-provider.d.ts +4 -0
- package/dist/rpc/ggml-provider.js +25 -0
- package/dist/rpc/index.d.ts +2 -0
- package/dist/rpc/index.js +1 -0
- package/dist/rpc/instance.d.ts +6 -0
- package/dist/rpc/instance.js +21 -0
- package/dist/rpc/inventory.d.ts +5 -0
- package/dist/rpc/inventory.js +144 -0
- package/dist/rpc/manager.d.ts +14 -0
- package/dist/rpc/manager.js +174 -0
- package/dist/rpc/network.d.ts +11 -0
- package/dist/rpc/network.js +85 -0
- package/dist/rpc/provider.d.ts +6 -0
- package/dist/rpc/provider.js +24 -0
- package/dist/runtime/lifecycle.js +31 -13
- package/dist/runtime/request-context.d.ts +1 -1
- package/dist/runtime/state.js +5 -0
- package/dist/schemas/assess-model-fit.d.ts +284 -77
- package/dist/schemas/assess-model-fit.js +93 -55
- package/dist/schemas/audio-gen.d.ts +6 -6
- package/dist/schemas/audio-gen.js +2 -2
- package/dist/schemas/batch-completion-stream.d.ts +13 -5
- package/dist/schemas/batch-completion-stream.js +5 -3
- package/dist/schemas/bci-config.js +1 -1
- package/dist/schemas/bci.d.ts +86 -1
- package/dist/schemas/bci.js +21 -2
- package/dist/schemas/cancel.d.ts +10 -0
- package/dist/schemas/cancel.js +2 -0
- package/dist/schemas/common.d.ts +765 -56
- package/dist/schemas/common.js +7 -0
- package/dist/schemas/completion-event.d.ts +6 -6
- package/dist/schemas/completion-stream.d.ts +27 -8
- package/dist/schemas/completion-stream.js +41 -4
- package/dist/schemas/config.d.ts +236 -30
- package/dist/schemas/config.js +10 -0
- package/dist/schemas/embed.d.ts +2 -2
- package/dist/schemas/error.js +13 -0
- package/dist/schemas/errors.d.ts +1 -0
- package/dist/schemas/errors.js +5 -0
- package/dist/schemas/index.d.ts +1 -0
- package/dist/schemas/index.js +1 -0
- package/dist/schemas/llamacpp-config.d.ts +105 -14
- package/dist/schemas/llamacpp-config.js +113 -3
- package/dist/schemas/load-model.d.ts +3102 -627
- package/dist/schemas/load-model.js +2 -2
- package/dist/schemas/mcp-adapter.d.ts +4 -0
- package/dist/schemas/model-config-schemas.d.ts +205 -5
- package/dist/schemas/model-info.d.ts +56 -0
- package/dist/schemas/model-info.js +5 -1
- package/dist/schemas/plugin.d.ts +8 -2
- package/dist/schemas/profiling.d.ts +4 -4
- package/dist/schemas/reload-config.d.ts +30 -0
- package/dist/schemas/rpc-server.d.ts +101 -0
- package/dist/schemas/rpc-server.js +87 -0
- package/dist/schemas/sdcpp-config.d.ts +7 -3
- package/dist/schemas/sdcpp-config.js +37 -2
- package/dist/schemas/system-resources.d.ts +4 -4
- package/dist/schemas/text-to-speech.d.ts +529 -1
- package/dist/schemas/text-to-speech.js +122 -12
- package/dist/schemas/tools.d.ts +2 -0
- package/dist/schemas/tools.js +8 -0
- package/dist/schemas/transcription-config.d.ts +5 -0
- package/dist/schemas/transcription-config.js +20 -2
- package/dist/schemas/transcription.d.ts +164 -0
- package/dist/schemas/transcription.js +71 -4
- package/dist/schemas/vla.d.ts +1 -1
- package/dist/surface.d.ts +4 -1
- package/dist/surface.js +3 -1
- package/dist/utils/addon-responses.d.ts +5 -1
- package/dist/utils/archive.js +5 -1
- package/dist/utils/asr-diagnostics.d.ts +12 -0
- package/dist/utils/asr-diagnostics.js +42 -0
- package/dist/utils/asr-events.d.ts +1 -0
- package/dist/utils/asr-events.js +3 -2
- package/dist/utils/cache/paths.js +3 -2
- package/dist/utils/mcp-adapter.js +7 -1
- package/dist/utils/multi-gpu-mobile.d.ts +1 -1
- package/dist/utils/multi-gpu-mobile.js +8 -6
- package/dist/utils/path-security.d.ts +5 -0
- package/dist/utils/path-security.js +14 -0
- package/dist/utils/tool-helpers.d.ts +4 -0
- package/dist/utils/tool-helpers.js +2 -0
- package/dist/utils/tools/defer.d.ts +93 -0
- package/dist/utils/tools/defer.js +290 -0
- package/dist/utils/tools/index.d.ts +1 -0
- package/dist/utils/tools/index.js +1 -0
- package/dist/utils/transcribe-metadata.d.ts +21 -7
- package/dist/utils/transcribe-metadata.js +24 -8
- package/dist/utils/tts-stats.d.ts +6 -0
- package/dist/utils/tts-stats.js +14 -1
- package/package.json +44 -30
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.d.ts +0 -26
- package/dist/resources/model-fit/native-probe/create-llama-fit-request.js +0 -171
|
@@ -66,7 +66,14 @@ export const TTS_SUPERTONIC_LANGUAGES = [
|
|
|
66
66
|
// The engines @qvac/tts-ggml exposes as `TTSGgml.ENGINE_*`, in the order the
|
|
67
67
|
// addon lists them. Exported so callers can discriminate a TTS config without
|
|
68
68
|
// re-declaring the union (the CLI used to).
|
|
69
|
-
export const TTS_ENGINES = [
|
|
69
|
+
export const TTS_ENGINES = [
|
|
70
|
+
'chatterbox',
|
|
71
|
+
'supertonic',
|
|
72
|
+
'parler',
|
|
73
|
+
'cosyvoice3',
|
|
74
|
+
'audio8',
|
|
75
|
+
'moss'
|
|
76
|
+
];
|
|
70
77
|
// Mirror of the addon's `SentenceDelimiterPreset` (./text-stream-accumulator).
|
|
71
78
|
export const TTS_SENTENCE_DELIMITER_PRESETS = ['latin', 'cjk', 'multilingual'];
|
|
72
79
|
export const TTS_PARLER_EMOTIONS = [
|
|
@@ -191,9 +198,10 @@ function refineGpuIntent(config, ctx) {
|
|
|
191
198
|
}
|
|
192
199
|
// Desired output sample rate in Hz. Matches the @qvac/tts-ggml addon's
|
|
193
200
|
// accepted range; omit to keep the engine's native rate (or 48 kHz when the
|
|
194
|
-
// LavaSR enhancer is active). Every engine
|
|
195
|
-
//
|
|
196
|
-
// seam-free (Parler, CosyVoice3) constrain it in
|
|
201
|
+
// LavaSR enhancer is active). Every engine but MOSS (24 kHz only) resamples,
|
|
202
|
+
// so this is exposed on the other five configs; the engines whose native chunk
|
|
203
|
+
// streaming cannot resample seam-free (Parler, CosyVoice3) constrain it in
|
|
204
|
+
// their own refinements.
|
|
197
205
|
const ttsOutputSampleRateSchema = ttsIntegerSchema
|
|
198
206
|
.min(8000)
|
|
199
207
|
.max(192000)
|
|
@@ -544,12 +552,53 @@ const ttsAudio8RuntimeConfigShape = {
|
|
|
544
552
|
export const ttsAudio8RuntimeConfigSchema = z
|
|
545
553
|
.object(ttsAudio8RuntimeConfigShape)
|
|
546
554
|
.superRefine(refineGpuIntent);
|
|
555
|
+
// @qvac/tts-ggml's MOSS_MAX_DURATION_TOKENS: the 2048-row generation budget
|
|
556
|
+
// less the 31 delay-pattern drain rows and the two termination rows.
|
|
557
|
+
const TTS_MOSS_MAX_DURATION_TOKENS = 2015;
|
|
558
|
+
const TTS_MOSS_MAX_DIALOGUE_SPEAKERS = 5;
|
|
559
|
+
// MOSS emits its native 24 kHz only (the addon rejects any other
|
|
560
|
+
// `outputSampleRate`) and supports neither the LavaSR stages nor a smaller
|
|
561
|
+
// first streaming chunk, so none of those fields exist on this arm.
|
|
562
|
+
const ttsMossRuntimeConfigShape = {
|
|
563
|
+
ttsEngine: z
|
|
564
|
+
.literal('moss')
|
|
565
|
+
.describe('TTS engine: MOSS (OpenMOSS MOSS-TTS v1.5 / MOSS-TTSD Delay; 24 kHz, 8B backbone, desktop only).'),
|
|
566
|
+
// The addon forwards it into the model prompt as a hint and does not
|
|
567
|
+
// validate it, so the vocabulary is the model's, not a fixed enum.
|
|
568
|
+
language: z
|
|
569
|
+
.string()
|
|
570
|
+
.trim()
|
|
571
|
+
.min(1)
|
|
572
|
+
.optional()
|
|
573
|
+
.describe('Language hint written into the model prompt, e.g. `en` or `zh`. Default `en`.'),
|
|
574
|
+
durationTokens: ttsIntegerSchema
|
|
575
|
+
.min(0)
|
|
576
|
+
.max(TTS_MOSS_MAX_DURATION_TOKENS)
|
|
577
|
+
.optional()
|
|
578
|
+
.describe('Target length of each synthesis in codec frames (12.5 per second, so 38 ≈ 3 s), 0–2015; 0 or unset keeps the length free.'),
|
|
579
|
+
streamChunkTokens: ttsNonNegativeInt32Schema
|
|
580
|
+
.optional()
|
|
581
|
+
.describe('Codec frames per native streaming chunk (12.5 per second, so 25 ≈ 2 s); 0 or unset synthesizes the whole text before emitting.'),
|
|
582
|
+
useGPU: z.boolean().optional().describe(TTS_USE_GPU_DESC),
|
|
583
|
+
threads: ttsPositiveInt32Schema.optional().describe(TTS_THREADS_DESC),
|
|
584
|
+
// Not TTS_NGPU_LAYERS_DESC: MOSS does not offload per layer, so a layer
|
|
585
|
+
// count would misdescribe it — the value only picks the backend.
|
|
586
|
+
nGpuLayers: ttsInt32Schema
|
|
587
|
+
.optional()
|
|
588
|
+
.describe('Any non-zero value selects the GPU backend, 0 keeps MOSS on the CPU. Wins over `useGPU`; when both are set they must agree.'),
|
|
589
|
+
seed: ttsInt32Schema.optional().describe(TTS_SEED_DESC),
|
|
590
|
+
...ttsBackendDirFieldsShape
|
|
591
|
+
};
|
|
592
|
+
export const ttsMossRuntimeConfigSchema = z
|
|
593
|
+
.object(ttsMossRuntimeConfigShape)
|
|
594
|
+
.superRefine(refineGpuIntent);
|
|
547
595
|
export const ttsRuntimeConfigSchema = z.discriminatedUnion('ttsEngine', [
|
|
548
596
|
ttsChatterboxRuntimeConfigSchema,
|
|
549
597
|
ttsSupertonicRuntimeConfigSchema,
|
|
550
598
|
ttsParlerRuntimeConfigSchema,
|
|
551
599
|
ttsCosyvoice3RuntimeConfigSchema,
|
|
552
|
-
ttsAudio8RuntimeConfigSchema
|
|
600
|
+
ttsAudio8RuntimeConfigSchema,
|
|
601
|
+
ttsMossRuntimeConfigSchema
|
|
553
602
|
]);
|
|
554
603
|
// Optional LavaSR post-processing model sources, shared across engines. Supply
|
|
555
604
|
// the enhancer GGUF to bandwidth-extend the output to 48 kHz, and/or the
|
|
@@ -725,6 +774,55 @@ export const ttsAudio8LoadConfigSchema = z
|
|
|
725
774
|
.describe('Audio8 voice-cloning reference recording source; pair with `referenceText`.')
|
|
726
775
|
})
|
|
727
776
|
.superRefine(refineAudio8LoadConfig);
|
|
777
|
+
// The addon enforces the same three rules at construction; catching them here
|
|
778
|
+
// names the SDK field instead of the addon's `files.*` option.
|
|
779
|
+
function refineMossLoadConfig(config, ctx) {
|
|
780
|
+
refineGpuIntent(config, ctx);
|
|
781
|
+
if (config.referenceAudioSrc !== undefined && config.dialogueReferenceSrcs !== undefined) {
|
|
782
|
+
ctx.addIssue({
|
|
783
|
+
code: 'custom',
|
|
784
|
+
path: ['dialogueReferenceSrcs'],
|
|
785
|
+
message: 'referenceAudioSrc and dialogueReferenceSrcs are exclusive; pass one recording per speaker in dialogueReferenceSrcs.'
|
|
786
|
+
});
|
|
787
|
+
}
|
|
788
|
+
const cloningField = config.referenceAudioSrc !== undefined
|
|
789
|
+
? 'referenceAudioSrc'
|
|
790
|
+
: config.dialogueReferenceSrcs !== undefined
|
|
791
|
+
? 'dialogueReferenceSrcs'
|
|
792
|
+
: undefined;
|
|
793
|
+
if (cloningField !== undefined && config.mossCodecEncoderModelSrc === undefined) {
|
|
794
|
+
ctx.addIssue({
|
|
795
|
+
code: 'custom',
|
|
796
|
+
path: ['mossCodecEncoderModelSrc'],
|
|
797
|
+
message: `${cloningField} requires the MOSS codec encoder GGUF (mossCodecEncoderModelSrc).`
|
|
798
|
+
});
|
|
799
|
+
}
|
|
800
|
+
}
|
|
801
|
+
// MOSS loads from explicit per-component GGUFs like Audio8: the primary
|
|
802
|
+
// `modelSrc` is the Delay backbone — MOSS-TTS (`moss-tts-delay-*.gguf`) or the
|
|
803
|
+
// MOSS-TTSD dialogue checkpoint (`moss-ttsd-*.gguf`) — the codec decoder is
|
|
804
|
+
// required, and the codec encoder is only needed to encode reference audio.
|
|
805
|
+
export const ttsMossLoadConfigSchema = z
|
|
806
|
+
.object({
|
|
807
|
+
...ttsMossRuntimeConfigShape,
|
|
808
|
+
mossCodecDecoderModelSrc: modelSrcInputSchema.describe('MOSS codec decoder model source (codes to 24 kHz waveform).'),
|
|
809
|
+
mossCodecEncoderModelSrc: modelSrcInputSchema
|
|
810
|
+
.optional()
|
|
811
|
+
.describe('MOSS codec encoder model source (waveform to codes); required only with `referenceAudioSrc` or `dialogueReferenceSrcs`.'),
|
|
812
|
+
referenceAudioSrc: modelSrcInputSchema
|
|
813
|
+
.optional()
|
|
814
|
+
.describe('MOSS voice-cloning reference recording source: a 24 kHz WAV (no resampling; multichannel is downmixed, at most 60 s). No transcript needed. Fixed for the loaded model.'),
|
|
815
|
+
// MOSS-TTSD covers one to five speakers. The engine itself sets no count
|
|
816
|
+
// (only a 60 s cap on the references combined), so without a bound here a
|
|
817
|
+
// single load could fan out into any number of parallel source downloads.
|
|
818
|
+
dialogueReferenceSrcs: z
|
|
819
|
+
.array(modelSrcInputSchema)
|
|
820
|
+
.min(1)
|
|
821
|
+
.max(TTS_MOSS_MAX_DIALOGUE_SPEAKERS)
|
|
822
|
+
.optional()
|
|
823
|
+
.describe('MOSS-TTSD dialogue: one 24 kHz WAV per speaker (one to five, at most 60 s combined), in the order the text tags them (`[S1]`, `[S2]`, …). The text must open with each recording’s transcript under its tag, followed by the lines to generate. Needs a MOSS-TTSD backbone as `modelSrc`; fixed for the loaded model.')
|
|
824
|
+
})
|
|
825
|
+
.superRefine(refineMossLoadConfig);
|
|
728
826
|
function refineChatterboxTokenizerAssets(data, ctx) {
|
|
729
827
|
if (data.ttsEngine !== 'chatterbox')
|
|
730
828
|
return;
|
|
@@ -749,7 +847,8 @@ export const ttsLoadConfigSchema = z
|
|
|
749
847
|
ttsSupertonicLoadConfigSchema,
|
|
750
848
|
ttsParlerLoadConfigSchema,
|
|
751
849
|
ttsCosyvoice3LoadConfigSchema,
|
|
752
|
-
ttsAudio8LoadConfigSchema
|
|
850
|
+
ttsAudio8LoadConfigSchema,
|
|
851
|
+
ttsMossLoadConfigSchema
|
|
753
852
|
])
|
|
754
853
|
.superRefine(refineChatterboxTokenizerAssets);
|
|
755
854
|
// === Legacy ONNX modelConfig fields (deprecated) ===
|
|
@@ -788,7 +887,8 @@ export const ttsConfigSchema = z
|
|
|
788
887
|
ttsSupertonicLoadConfigSchema.extend(legacyTtsOnnxFieldsShape).strict(),
|
|
789
888
|
ttsParlerLoadConfigSchema.strict(),
|
|
790
889
|
ttsCosyvoice3LoadConfigSchema.strict(),
|
|
791
|
-
ttsAudio8LoadConfigSchema.strict()
|
|
890
|
+
ttsAudio8LoadConfigSchema.strict(),
|
|
891
|
+
ttsMossLoadConfigSchema.strict()
|
|
792
892
|
])
|
|
793
893
|
.superRefine(refineChatterboxTokenizerAssets);
|
|
794
894
|
// Cancel targeting. Both TTS handlers declare `cancel: { scope: 'model', hard:
|
|
@@ -832,16 +932,18 @@ export const ttsRequestSchema = z
|
|
|
832
932
|
// The chunked paths (`stream: true`, `sentenceStream`, and the duplex session)
|
|
833
933
|
// run one native job per sentence and the addon's aggregate carries only the
|
|
834
934
|
// four summable fields (totalTime, audioDuration, totalSamples,
|
|
835
|
-
// generatedFrames)
|
|
836
|
-
//
|
|
935
|
+
// generatedFrames) and Audio8's two Core ML flags, which keep the value of the
|
|
936
|
+
// last sentence that reported them; the five backend fields below are
|
|
937
|
+
// reported by batch synthesis (`stream: false`) only.
|
|
837
938
|
export const ttsStatsSchema = z.object({
|
|
838
939
|
audioDuration: z.number().optional(),
|
|
839
940
|
totalTime: z.number().optional(),
|
|
840
941
|
realTimeFactor: z.number().optional(),
|
|
841
942
|
tokensPerSecond: z.number().optional(),
|
|
842
943
|
totalSamples: z.number().optional(),
|
|
843
|
-
// Audio8
|
|
844
|
-
//
|
|
944
|
+
// Audio8 (on a fixed 46 ms grid) and MOSS (12.5 per second) count codec
|
|
945
|
+
// frames rather than tokens, and report them as the unit behind their
|
|
946
|
+
// `tokensPerSecond`.
|
|
845
947
|
generatedFrames: z.number().optional(),
|
|
846
948
|
// Backend selection captured once at model load. `0` CPU / `1` GPU;
|
|
847
949
|
// `backendId` codes the family (0 CPU, 1 Metal, 2 CUDA, 3 Vulkan, 4 OpenCL,
|
|
@@ -851,7 +953,15 @@ export const ttsStatsSchema = z.object({
|
|
|
851
953
|
backendId: z.number().optional(),
|
|
852
954
|
gpuUnsupported: z.number().optional(),
|
|
853
955
|
enhancerBackendDevice: z.number().optional(),
|
|
854
|
-
enhancerBackendId: z.number().optional()
|
|
956
|
+
enhancerBackendId: z.number().optional(),
|
|
957
|
+
codecSidecarLoaded: z
|
|
958
|
+
.number()
|
|
959
|
+
.optional()
|
|
960
|
+
.describe('Audio8 on macOS / iOS: `1` while the Core ML codec sidecar is attached, `0` without one or once a failing sidecar was retired.'),
|
|
961
|
+
codecOnCoreml: z
|
|
962
|
+
.number()
|
|
963
|
+
.optional()
|
|
964
|
+
.describe('Audio8: `1` when this synthesis ran its codec on the Core ML sidecar, `0` when it ran on the ggml backend `backendId` reports.')
|
|
855
965
|
});
|
|
856
966
|
// The rate of `buffer`. It is not a constant of the engine: `outputSampleRate`
|
|
857
967
|
// and the LavaSR enhancer both move it, so a caller writing a WAV or feeding
|
package/dist/schemas/tools.d.ts
CHANGED
|
@@ -3,6 +3,8 @@ export declare const toolSchema: z.ZodObject<{
|
|
|
3
3
|
type: z.ZodLiteral<"function">;
|
|
4
4
|
name: z.ZodString;
|
|
5
5
|
description: z.ZodString;
|
|
6
|
+
deferLoading: z.ZodOptional<z.ZodBoolean>;
|
|
7
|
+
group: z.ZodOptional<z.ZodString>;
|
|
6
8
|
parameters: z.ZodObject<{
|
|
7
9
|
type: z.ZodLiteral<"object">;
|
|
8
10
|
properties: z.ZodRecord<z.ZodString, z.ZodObject<{
|
package/dist/schemas/tools.js
CHANGED
|
@@ -4,6 +4,14 @@ export const toolSchema = z.object({
|
|
|
4
4
|
type: z.literal('function'),
|
|
5
5
|
name: z.string(),
|
|
6
6
|
description: z.string(),
|
|
7
|
+
deferLoading: z
|
|
8
|
+
.boolean()
|
|
9
|
+
.optional()
|
|
10
|
+
.describe('Opt out of the initial prompt. A deferred tool is registered but only its name and description reach the model, in the catalog carried by the built-in `tool_search`; its parameter schema is appended to the conversation once the model searches for it. Tools without this field behave as before.'),
|
|
11
|
+
group: z
|
|
12
|
+
.string()
|
|
13
|
+
.optional()
|
|
14
|
+
.describe('Optional heading this tool is listed under in the deferred catalog — a skill or an MCP server name. Ignored for tools that are not deferred.'),
|
|
7
15
|
parameters: z.object({
|
|
8
16
|
type: z.literal('object'),
|
|
9
17
|
properties: z.record(z.string(), z.object({
|
|
@@ -43,6 +43,10 @@ export declare const whisperConfigSchema: z.ZodObject<{
|
|
|
43
43
|
logprob_thold: z.ZodOptional<z.ZodNumber>;
|
|
44
44
|
greedy_best_of: z.ZodOptional<z.ZodNumber>;
|
|
45
45
|
beam_search_beam_size: z.ZodOptional<z.ZodNumber>;
|
|
46
|
+
max_initial_ts: z.ZodOptional<z.ZodNumber>;
|
|
47
|
+
no_speech_thold: z.ZodOptional<z.ZodNumber>;
|
|
48
|
+
seed: z.ZodOptional<z.ZodNumber>;
|
|
49
|
+
backendsDir: z.ZodOptional<z.ZodString>;
|
|
46
50
|
vad_params: z.ZodOptional<z.ZodObject<{
|
|
47
51
|
threshold: z.ZodOptional<z.ZodNumber>;
|
|
48
52
|
min_speech_duration_ms: z.ZodOptional<z.ZodNumber>;
|
|
@@ -63,6 +67,7 @@ export declare const whisperConfigSchema: z.ZodObject<{
|
|
|
63
67
|
}, z.core.$strip>>;
|
|
64
68
|
miscConfig: z.ZodOptional<z.ZodObject<{
|
|
65
69
|
caption_enabled: z.ZodOptional<z.ZodBoolean>;
|
|
70
|
+
seed: z.ZodOptional<z.ZodNumber>;
|
|
66
71
|
}, z.core.$strip>>;
|
|
67
72
|
vadModelSrc: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
|
|
68
73
|
src: z.ZodString;
|
|
@@ -41,7 +41,12 @@ const contextParamsSchema = z
|
|
|
41
41
|
.optional();
|
|
42
42
|
const miscConfigSchema = z
|
|
43
43
|
.object({
|
|
44
|
-
caption_enabled: z.boolean().optional().describe('Format output segments as captions.')
|
|
44
|
+
caption_enabled: z.boolean().optional().describe('Format output segments as captions.'),
|
|
45
|
+
seed: z
|
|
46
|
+
.number()
|
|
47
|
+
.int()
|
|
48
|
+
.optional()
|
|
49
|
+
.describe('RNG seed applied to the whisper context; `-1` picks a random seed.')
|
|
45
50
|
})
|
|
46
51
|
.optional();
|
|
47
52
|
export const whisperConfigSchema = z.object({
|
|
@@ -135,7 +140,7 @@ export const whisperConfigSchema = z.object({
|
|
|
135
140
|
detect_language: z
|
|
136
141
|
.boolean()
|
|
137
142
|
.optional()
|
|
138
|
-
.describe("
|
|
143
|
+
.describe("Convenience flag: `true` is translated to `language: 'auto'` before the request reaches the addon, which has no `detect_language` key of its own."),
|
|
139
144
|
suppress_blank: z
|
|
140
145
|
.boolean()
|
|
141
146
|
.optional()
|
|
@@ -165,6 +170,19 @@ export const whisperConfigSchema = z.object({
|
|
|
165
170
|
.int()
|
|
166
171
|
.optional()
|
|
167
172
|
.describe('Beam size for beam-search decoding; `-1` = default.'),
|
|
173
|
+
max_initial_ts: z
|
|
174
|
+
.number()
|
|
175
|
+
.optional()
|
|
176
|
+
.describe('Maximum initial timestamp the decoder may emit, as a fraction of the window.'),
|
|
177
|
+
no_speech_thold: z
|
|
178
|
+
.number()
|
|
179
|
+
.optional()
|
|
180
|
+
.describe('No-speech probability above which a segment is treated as silence.'),
|
|
181
|
+
seed: z.number().int().optional().describe('Decoding RNG seed; `-1` picks a random seed.'),
|
|
182
|
+
backendsDir: z
|
|
183
|
+
.string()
|
|
184
|
+
.optional()
|
|
185
|
+
.describe("Root directory for dynamically-loaded ggml backend libraries. Defaults to the addon's own `prebuilds/`."),
|
|
168
186
|
vad_params: vadParamsSchema,
|
|
169
187
|
audio_format: audioFormatSchema
|
|
170
188
|
.optional()
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
2
|
import type Buffer from 'bare-buffer';
|
|
3
|
+
import { type InferenceBackendDiagnostics } from './system-resources.js';
|
|
3
4
|
export declare const audioInputSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
4
5
|
type: z.ZodLiteral<"base64">;
|
|
5
6
|
value: z.ZodString;
|
|
@@ -35,17 +36,54 @@ export declare const transcribeStatsSchema: z.ZodObject<{
|
|
|
35
36
|
gpuUnsupported: z.ZodOptional<z.ZodNumber>;
|
|
36
37
|
gpuMemTotalMb: z.ZodOptional<z.ZodNumber>;
|
|
37
38
|
gpuMemFreeMb: z.ZodOptional<z.ZodNumber>;
|
|
39
|
+
totalTime: z.ZodOptional<z.ZodNumber>;
|
|
40
|
+
totalWallMs: z.ZodOptional<z.ZodNumber>;
|
|
41
|
+
totalSamples: z.ZodOptional<z.ZodNumber>;
|
|
42
|
+
processCalls: z.ZodOptional<z.ZodNumber>;
|
|
43
|
+
whisperSampleMs: z.ZodOptional<z.ZodNumber>;
|
|
44
|
+
whisperBatchdMs: z.ZodOptional<z.ZodNumber>;
|
|
45
|
+
whisperPromptMs: z.ZodOptional<z.ZodNumber>;
|
|
46
|
+
totalTranscriptions: z.ZodOptional<z.ZodNumber>;
|
|
47
|
+
modelLoadMs: z.ZodOptional<z.ZodNumber>;
|
|
48
|
+
totalEncodedFrames: z.ZodOptional<z.ZodNumber>;
|
|
49
|
+
encoderOnCoreml: z.ZodOptional<z.ZodNumber>;
|
|
50
|
+
encoderUsedCoreml: z.ZodOptional<z.ZodNumber>;
|
|
38
51
|
}, z.core.$strip>;
|
|
52
|
+
/**
|
|
53
|
+
* Compute backends the ASR engines report through `stats.backendId`, as the
|
|
54
|
+
* addon's `BackendId` enum defines them. Exported so callers can decode the
|
|
55
|
+
* number instead of hardcoding it.
|
|
56
|
+
*
|
|
57
|
+
* Kept as a literal because it feeds the contract, and checked against the
|
|
58
|
+
* addon's enum by `satisfies`: a backend the addon adds or removes is a compile
|
|
59
|
+
* error here rather than a silently short vocabulary.
|
|
60
|
+
*/
|
|
61
|
+
export declare const ASR_BACKEND_IDS: Readonly<{
|
|
62
|
+
readonly CPU: 0;
|
|
63
|
+
readonly Metal: 1;
|
|
64
|
+
readonly CUDA: 2;
|
|
65
|
+
readonly Vulkan: 3;
|
|
66
|
+
readonly OpenCL: 4;
|
|
67
|
+
readonly Other: 99;
|
|
68
|
+
}>;
|
|
69
|
+
export type AsrBackendId = (typeof ASR_BACKEND_IDS)[keyof typeof ASR_BACKEND_IDS];
|
|
39
70
|
export declare const transcribeSegmentSchema: z.ZodObject<{
|
|
40
71
|
text: z.ZodString;
|
|
41
72
|
startMs: z.ZodNumber;
|
|
42
73
|
endMs: z.ZodNumber;
|
|
43
74
|
append: z.ZodBoolean;
|
|
44
75
|
id: z.ZodNumber;
|
|
76
|
+
isEndOfTurn: z.ZodOptional<z.ZodBoolean>;
|
|
77
|
+
startsWord: z.ZodOptional<z.ZodBoolean>;
|
|
45
78
|
}, z.core.$strip>;
|
|
46
79
|
export declare const vadStateEventSchema: z.ZodObject<{
|
|
47
80
|
speaking: z.ZodBoolean;
|
|
48
81
|
probability: z.ZodNumber;
|
|
82
|
+
source: z.ZodOptional<z.ZodEnum<{
|
|
83
|
+
silero: "silero";
|
|
84
|
+
energy: "energy";
|
|
85
|
+
sortformer: "sortformer";
|
|
86
|
+
}>>;
|
|
49
87
|
}, z.core.$strip>;
|
|
50
88
|
export declare const whisperEndOfTurnEventSchema: z.ZodObject<{
|
|
51
89
|
source: z.ZodLiteral<"whisper">;
|
|
@@ -93,6 +131,18 @@ export declare const transcribeResponseSchema: z.ZodObject<{
|
|
|
93
131
|
gpuUnsupported: z.ZodOptional<z.ZodNumber>;
|
|
94
132
|
gpuMemTotalMb: z.ZodOptional<z.ZodNumber>;
|
|
95
133
|
gpuMemFreeMb: z.ZodOptional<z.ZodNumber>;
|
|
134
|
+
totalTime: z.ZodOptional<z.ZodNumber>;
|
|
135
|
+
totalWallMs: z.ZodOptional<z.ZodNumber>;
|
|
136
|
+
totalSamples: z.ZodOptional<z.ZodNumber>;
|
|
137
|
+
processCalls: z.ZodOptional<z.ZodNumber>;
|
|
138
|
+
whisperSampleMs: z.ZodOptional<z.ZodNumber>;
|
|
139
|
+
whisperBatchdMs: z.ZodOptional<z.ZodNumber>;
|
|
140
|
+
whisperPromptMs: z.ZodOptional<z.ZodNumber>;
|
|
141
|
+
totalTranscriptions: z.ZodOptional<z.ZodNumber>;
|
|
142
|
+
modelLoadMs: z.ZodOptional<z.ZodNumber>;
|
|
143
|
+
totalEncodedFrames: z.ZodOptional<z.ZodNumber>;
|
|
144
|
+
encoderOnCoreml: z.ZodOptional<z.ZodNumber>;
|
|
145
|
+
encoderUsedCoreml: z.ZodOptional<z.ZodNumber>;
|
|
96
146
|
}, z.core.$strip>>;
|
|
97
147
|
error: z.ZodOptional<z.ZodString>;
|
|
98
148
|
segment: z.ZodOptional<z.ZodObject<{
|
|
@@ -101,10 +151,17 @@ export declare const transcribeResponseSchema: z.ZodObject<{
|
|
|
101
151
|
endMs: z.ZodNumber;
|
|
102
152
|
append: z.ZodBoolean;
|
|
103
153
|
id: z.ZodNumber;
|
|
154
|
+
isEndOfTurn: z.ZodOptional<z.ZodBoolean>;
|
|
155
|
+
startsWord: z.ZodOptional<z.ZodBoolean>;
|
|
104
156
|
}, z.core.$strip>>;
|
|
105
157
|
vad: z.ZodOptional<z.ZodObject<{
|
|
106
158
|
speaking: z.ZodBoolean;
|
|
107
159
|
probability: z.ZodNumber;
|
|
160
|
+
source: z.ZodOptional<z.ZodEnum<{
|
|
161
|
+
silero: "silero";
|
|
162
|
+
energy: "energy";
|
|
163
|
+
sortformer: "sortformer";
|
|
164
|
+
}>>;
|
|
108
165
|
}, z.core.$strip>>;
|
|
109
166
|
endOfTurn: z.ZodOptional<z.ZodPreprocess<z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
110
167
|
source: z.ZodLiteral<"whisper">;
|
|
@@ -112,6 +169,47 @@ export declare const transcribeResponseSchema: z.ZodObject<{
|
|
|
112
169
|
}, z.core.$strip>, z.ZodObject<{
|
|
113
170
|
source: z.ZodLiteral<"parakeet">;
|
|
114
171
|
}, z.core.$strip>], "source">, unknown>>;
|
|
172
|
+
diagnostics: z.ZodOptional<z.ZodObject<{
|
|
173
|
+
selectedBackend: z.ZodString;
|
|
174
|
+
selectedDevice: z.ZodEnum<{
|
|
175
|
+
cpu: "cpu";
|
|
176
|
+
gpu: "gpu";
|
|
177
|
+
}>;
|
|
178
|
+
graphicsApi: z.ZodOptional<z.ZodEnum<{
|
|
179
|
+
vulkan: "vulkan";
|
|
180
|
+
metal: "metal";
|
|
181
|
+
opencl: "opencl";
|
|
182
|
+
cuda: "cuda";
|
|
183
|
+
opengl: "opengl";
|
|
184
|
+
webgpu: "webgpu";
|
|
185
|
+
direct3d11: "direct3d11";
|
|
186
|
+
direct3d12: "direct3d12";
|
|
187
|
+
levelZero: "levelZero";
|
|
188
|
+
rocm: "rocm";
|
|
189
|
+
}>>;
|
|
190
|
+
driver: z.ZodOptional<z.ZodObject<{
|
|
191
|
+
name: z.ZodString;
|
|
192
|
+
version: z.ZodOptional<z.ZodString>;
|
|
193
|
+
}, z.core.$strip>>;
|
|
194
|
+
gpuId: z.ZodOptional<z.ZodString>;
|
|
195
|
+
fallback: z.ZodOptional<z.ZodObject<{
|
|
196
|
+
requestedBackend: z.ZodOptional<z.ZodString>;
|
|
197
|
+
requestedDevice: z.ZodOptional<z.ZodEnum<{
|
|
198
|
+
cpu: "cpu";
|
|
199
|
+
gpu: "gpu";
|
|
200
|
+
}>>;
|
|
201
|
+
reason: z.ZodString;
|
|
202
|
+
}, z.core.$strip>>;
|
|
203
|
+
probe: z.ZodOptional<z.ZodObject<{
|
|
204
|
+
status: z.ZodEnum<{
|
|
205
|
+
unknown: "unknown";
|
|
206
|
+
compatible: "compatible";
|
|
207
|
+
incompatible: "incompatible";
|
|
208
|
+
}>;
|
|
209
|
+
backend: z.ZodString;
|
|
210
|
+
reason: z.ZodOptional<z.ZodString>;
|
|
211
|
+
}, z.core.$strip>>;
|
|
212
|
+
}, z.core.$strip>>;
|
|
115
213
|
type: z.ZodLiteral<"transcribe">;
|
|
116
214
|
}, z.core.$strip>;
|
|
117
215
|
export type AudioInput = z.infer<typeof audioInputSchema>;
|
|
@@ -193,6 +291,18 @@ export declare const transcribeStreamResponseSchema: z.ZodObject<{
|
|
|
193
291
|
gpuUnsupported: z.ZodOptional<z.ZodNumber>;
|
|
194
292
|
gpuMemTotalMb: z.ZodOptional<z.ZodNumber>;
|
|
195
293
|
gpuMemFreeMb: z.ZodOptional<z.ZodNumber>;
|
|
294
|
+
totalTime: z.ZodOptional<z.ZodNumber>;
|
|
295
|
+
totalWallMs: z.ZodOptional<z.ZodNumber>;
|
|
296
|
+
totalSamples: z.ZodOptional<z.ZodNumber>;
|
|
297
|
+
processCalls: z.ZodOptional<z.ZodNumber>;
|
|
298
|
+
whisperSampleMs: z.ZodOptional<z.ZodNumber>;
|
|
299
|
+
whisperBatchdMs: z.ZodOptional<z.ZodNumber>;
|
|
300
|
+
whisperPromptMs: z.ZodOptional<z.ZodNumber>;
|
|
301
|
+
totalTranscriptions: z.ZodOptional<z.ZodNumber>;
|
|
302
|
+
modelLoadMs: z.ZodOptional<z.ZodNumber>;
|
|
303
|
+
totalEncodedFrames: z.ZodOptional<z.ZodNumber>;
|
|
304
|
+
encoderOnCoreml: z.ZodOptional<z.ZodNumber>;
|
|
305
|
+
encoderUsedCoreml: z.ZodOptional<z.ZodNumber>;
|
|
196
306
|
}, z.core.$strip>>;
|
|
197
307
|
error: z.ZodOptional<z.ZodString>;
|
|
198
308
|
segment: z.ZodOptional<z.ZodObject<{
|
|
@@ -201,10 +311,17 @@ export declare const transcribeStreamResponseSchema: z.ZodObject<{
|
|
|
201
311
|
endMs: z.ZodNumber;
|
|
202
312
|
append: z.ZodBoolean;
|
|
203
313
|
id: z.ZodNumber;
|
|
314
|
+
isEndOfTurn: z.ZodOptional<z.ZodBoolean>;
|
|
315
|
+
startsWord: z.ZodOptional<z.ZodBoolean>;
|
|
204
316
|
}, z.core.$strip>>;
|
|
205
317
|
vad: z.ZodOptional<z.ZodObject<{
|
|
206
318
|
speaking: z.ZodBoolean;
|
|
207
319
|
probability: z.ZodNumber;
|
|
320
|
+
source: z.ZodOptional<z.ZodEnum<{
|
|
321
|
+
silero: "silero";
|
|
322
|
+
energy: "energy";
|
|
323
|
+
sortformer: "sortformer";
|
|
324
|
+
}>>;
|
|
208
325
|
}, z.core.$strip>>;
|
|
209
326
|
endOfTurn: z.ZodOptional<z.ZodPreprocess<z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
210
327
|
source: z.ZodLiteral<"whisper">;
|
|
@@ -212,6 +329,47 @@ export declare const transcribeStreamResponseSchema: z.ZodObject<{
|
|
|
212
329
|
}, z.core.$strip>, z.ZodObject<{
|
|
213
330
|
source: z.ZodLiteral<"parakeet">;
|
|
214
331
|
}, z.core.$strip>], "source">, unknown>>;
|
|
332
|
+
diagnostics: z.ZodOptional<z.ZodObject<{
|
|
333
|
+
selectedBackend: z.ZodString;
|
|
334
|
+
selectedDevice: z.ZodEnum<{
|
|
335
|
+
cpu: "cpu";
|
|
336
|
+
gpu: "gpu";
|
|
337
|
+
}>;
|
|
338
|
+
graphicsApi: z.ZodOptional<z.ZodEnum<{
|
|
339
|
+
vulkan: "vulkan";
|
|
340
|
+
metal: "metal";
|
|
341
|
+
opencl: "opencl";
|
|
342
|
+
cuda: "cuda";
|
|
343
|
+
opengl: "opengl";
|
|
344
|
+
webgpu: "webgpu";
|
|
345
|
+
direct3d11: "direct3d11";
|
|
346
|
+
direct3d12: "direct3d12";
|
|
347
|
+
levelZero: "levelZero";
|
|
348
|
+
rocm: "rocm";
|
|
349
|
+
}>>;
|
|
350
|
+
driver: z.ZodOptional<z.ZodObject<{
|
|
351
|
+
name: z.ZodString;
|
|
352
|
+
version: z.ZodOptional<z.ZodString>;
|
|
353
|
+
}, z.core.$strip>>;
|
|
354
|
+
gpuId: z.ZodOptional<z.ZodString>;
|
|
355
|
+
fallback: z.ZodOptional<z.ZodObject<{
|
|
356
|
+
requestedBackend: z.ZodOptional<z.ZodString>;
|
|
357
|
+
requestedDevice: z.ZodOptional<z.ZodEnum<{
|
|
358
|
+
cpu: "cpu";
|
|
359
|
+
gpu: "gpu";
|
|
360
|
+
}>>;
|
|
361
|
+
reason: z.ZodString;
|
|
362
|
+
}, z.core.$strip>>;
|
|
363
|
+
probe: z.ZodOptional<z.ZodObject<{
|
|
364
|
+
status: z.ZodEnum<{
|
|
365
|
+
unknown: "unknown";
|
|
366
|
+
compatible: "compatible";
|
|
367
|
+
incompatible: "incompatible";
|
|
368
|
+
}>;
|
|
369
|
+
backend: z.ZodString;
|
|
370
|
+
reason: z.ZodOptional<z.ZodString>;
|
|
371
|
+
}, z.core.$strip>>;
|
|
372
|
+
}, z.core.$strip>>;
|
|
215
373
|
type: z.ZodLiteral<"transcribeStream">;
|
|
216
374
|
}, z.core.$strip>;
|
|
217
375
|
export type TranscribeStreamRequest = z.infer<typeof transcribeStreamRequestSchema>;
|
|
@@ -227,6 +385,8 @@ export type TranscribeStreamClientParams = {
|
|
|
227
385
|
};
|
|
228
386
|
export interface TranscribeStreamSession {
|
|
229
387
|
stats: Promise<TranscribeStats | undefined>;
|
|
388
|
+
/** Backend selection detail for the session, settled from the same terminal frame as `stats`. */
|
|
389
|
+
diagnostics: Promise<InferenceBackendDiagnostics | undefined>;
|
|
230
390
|
write(audioChunk: Uint8Array): void;
|
|
231
391
|
end(): void;
|
|
232
392
|
destroy(): void;
|
|
@@ -234,6 +394,8 @@ export interface TranscribeStreamSession {
|
|
|
234
394
|
}
|
|
235
395
|
export interface TranscribeStreamMetadataSession {
|
|
236
396
|
stats: Promise<TranscribeStats | undefined>;
|
|
397
|
+
/** Backend selection detail for the session, settled from the same terminal frame as `stats`. */
|
|
398
|
+
diagnostics: Promise<InferenceBackendDiagnostics | undefined>;
|
|
237
399
|
write(audioChunk: Uint8Array): void;
|
|
238
400
|
end(): void;
|
|
239
401
|
destroy(): void;
|
|
@@ -256,6 +418,8 @@ export type TranscribeStreamEvent = {
|
|
|
256
418
|
} & EndOfTurnEvent);
|
|
257
419
|
export interface TranscribeStreamConversationSession {
|
|
258
420
|
stats: Promise<TranscribeStats | undefined>;
|
|
421
|
+
/** Backend selection detail for the session, settled from the same terminal frame as `stats`. */
|
|
422
|
+
diagnostics: Promise<InferenceBackendDiagnostics | undefined>;
|
|
259
423
|
write(audioChunk: Uint8Array): void;
|
|
260
424
|
end(): void;
|
|
261
425
|
destroy(): void;
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
|
+
import { inferenceBackendDiagnosticsSchema } from './system-resources.js';
|
|
2
3
|
export const audioInputSchema = z.discriminatedUnion('type', [
|
|
3
4
|
z.object({
|
|
4
5
|
type: z.literal('base64'),
|
|
@@ -41,18 +42,81 @@ export const transcribeStatsSchema = z.object({
|
|
|
41
42
|
backendId: z.number().optional(),
|
|
42
43
|
gpuUnsupported: z.number().optional(),
|
|
43
44
|
gpuMemTotalMb: z.number().optional(),
|
|
44
|
-
gpuMemFreeMb: z.number().optional()
|
|
45
|
+
gpuMemFreeMb: z.number().optional(),
|
|
46
|
+
// Shared engine counters.
|
|
47
|
+
totalTime: z.number().optional().describe('Total native inference time for the run, in ms.'),
|
|
48
|
+
totalWallMs: z
|
|
49
|
+
.number()
|
|
50
|
+
.optional()
|
|
51
|
+
.describe('Wall-clock time spent inside the native engine for the run, in ms.'),
|
|
52
|
+
totalSamples: z.number().optional().describe('Audio samples processed across the run.'),
|
|
53
|
+
processCalls: z
|
|
54
|
+
.number()
|
|
55
|
+
.optional()
|
|
56
|
+
.describe('Native `process()` invocations made across the run.'),
|
|
57
|
+
// whisper.cpp stage timings: the whisper engine and the BCI engine, which
|
|
58
|
+
// runs on the same decoder.
|
|
59
|
+
whisperSampleMs: z.number().optional().describe('Whisper: sampling time, in ms.'),
|
|
60
|
+
whisperBatchdMs: z.number().optional().describe('Whisper: batched-decode time, in ms.'),
|
|
61
|
+
whisperPromptMs: z.number().optional().describe('Whisper: prompt-processing time, in ms.'),
|
|
62
|
+
// Parakeet-only counters.
|
|
63
|
+
totalTranscriptions: z
|
|
64
|
+
.number()
|
|
65
|
+
.optional()
|
|
66
|
+
.describe('Parakeet: transcriptions produced across the session.'),
|
|
67
|
+
modelLoadMs: z.number().optional().describe('Parakeet: model load time, in ms.'),
|
|
68
|
+
totalEncodedFrames: z
|
|
69
|
+
.number()
|
|
70
|
+
.optional()
|
|
71
|
+
.describe('Parakeet: encoder frames produced across the run.'),
|
|
72
|
+
encoderOnCoreml: z
|
|
73
|
+
.number()
|
|
74
|
+
.optional()
|
|
75
|
+
.describe('Parakeet: `1` when a Core ML encoder sidecar loaded, `0` otherwise.'),
|
|
76
|
+
encoderUsedCoreml: z
|
|
77
|
+
.number()
|
|
78
|
+
.optional()
|
|
79
|
+
.describe('Parakeet offline ASR: `1` when this run used Core ML for encoding, `0` on fallback.')
|
|
80
|
+
});
|
|
81
|
+
/**
|
|
82
|
+
* Compute backends the ASR engines report through `stats.backendId`, as the
|
|
83
|
+
* addon's `BackendId` enum defines them. Exported so callers can decode the
|
|
84
|
+
* number instead of hardcoding it.
|
|
85
|
+
*
|
|
86
|
+
* Kept as a literal because it feeds the contract, and checked against the
|
|
87
|
+
* addon's enum by `satisfies`: a backend the addon adds or removes is a compile
|
|
88
|
+
* error here rather than a silently short vocabulary.
|
|
89
|
+
*/
|
|
90
|
+
export const ASR_BACKEND_IDS = Object.freeze({
|
|
91
|
+
CPU: 0,
|
|
92
|
+
Metal: 1,
|
|
93
|
+
CUDA: 2,
|
|
94
|
+
Vulkan: 3,
|
|
95
|
+
OpenCL: 4,
|
|
96
|
+
Other: 99
|
|
45
97
|
});
|
|
46
98
|
export const transcribeSegmentSchema = z.object({
|
|
47
99
|
text: z.string(),
|
|
48
100
|
startMs: z.number(),
|
|
49
101
|
endMs: z.number(),
|
|
50
102
|
append: z.boolean(),
|
|
51
|
-
id: z.number()
|
|
103
|
+
id: z.number(),
|
|
104
|
+
isEndOfTurn: z
|
|
105
|
+
.boolean()
|
|
106
|
+
.optional()
|
|
107
|
+
.describe('Segment ends on a recognized end-of-utterance boundary. Parakeet engine only (EOU-capable checkpoints); absent on whisper.'),
|
|
108
|
+
startsWord: z
|
|
109
|
+
.boolean()
|
|
110
|
+
.optional()
|
|
111
|
+
.describe('Segment begins a new SentencePiece word, for joining partial segments without splitting words. Parakeet engine only; absent on whisper.')
|
|
52
112
|
});
|
|
53
113
|
export const vadStateEventSchema = z.object({
|
|
54
114
|
speaking: z.boolean(),
|
|
55
|
-
probability: z.number()
|
|
115
|
+
probability: z.number(),
|
|
116
|
+
source: z
|
|
117
|
+
.enum(['silero', 'energy', 'sortformer'])
|
|
118
|
+
.optional()
|
|
119
|
+
.describe("Detector behind the event: `'silero'` for Whisper, `'energy'` for Parakeet energy VAD, or `'sortformer'` for Sortformer speaker activity.")
|
|
56
120
|
});
|
|
57
121
|
export const whisperEndOfTurnEventSchema = z.object({
|
|
58
122
|
source: z.literal('whisper'),
|
|
@@ -94,7 +158,10 @@ const transcriptionResultBase = z.object({
|
|
|
94
158
|
error: z.string().optional(),
|
|
95
159
|
segment: transcribeSegmentSchema.optional(),
|
|
96
160
|
vad: vadStateEventSchema.optional(),
|
|
97
|
-
endOfTurn: endOfTurnEventSchema.optional()
|
|
161
|
+
endOfTurn: endOfTurnEventSchema.optional(),
|
|
162
|
+
diagnostics: inferenceBackendDiagnosticsSchema
|
|
163
|
+
.optional()
|
|
164
|
+
.describe('Backend selection detail for the completed run, on the terminal frame. Carries the same payload the engine attaches to the internal diagnostics symbol, so an RPC client can read it.')
|
|
98
165
|
});
|
|
99
166
|
export const transcribeResponseSchema = transcriptionResultBase.extend({
|
|
100
167
|
type: z.literal('transcribe')
|
package/dist/schemas/vla.d.ts
CHANGED
|
@@ -42,8 +42,8 @@ export type VlaEmbodimentSelection = string | number | {
|
|
|
42
42
|
};
|
|
43
43
|
export declare const vlaConfigSchema: z.ZodObject<{
|
|
44
44
|
backend: z.ZodOptional<z.ZodEnum<{
|
|
45
|
-
auto: "auto";
|
|
46
45
|
cpu: "cpu";
|
|
46
|
+
auto: "auto";
|
|
47
47
|
}>>;
|
|
48
48
|
verbosity: z.ZodOptional<z.ZodNumber>;
|
|
49
49
|
embodiment: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodNumber, z.ZodObject<{
|