@qvac/inference 0.20.0 → 0.21.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (213) hide show
  1. package/NOTICE +107 -33
  2. package/README.md +8 -5
  3. package/dist/api/assess-model-fit.d.ts +9 -4
  4. package/dist/api/assess-model-fit.js +9 -4
  5. package/dist/api/bci-transcribe.d.ts +4 -1
  6. package/dist/api/index.d.ts +1 -0
  7. package/dist/api/index.js +1 -0
  8. package/dist/api/rpc-server.d.ts +19 -0
  9. package/dist/api/rpc-server.js +44 -0
  10. package/dist/api/text-to-speech.d.ts +9 -0
  11. package/dist/api/text-to-speech.js +9 -0
  12. package/dist/api/transcribe.d.ts +11 -7
  13. package/dist/api/transcribe.js +47 -15
  14. package/dist/api/video-request.d.ts +5 -5
  15. package/dist/dispatch.js +39 -7
  16. package/dist/errors/index.d.ts +7 -0
  17. package/dist/errors/index.js +15 -0
  18. package/dist/handlers/assess-model-fit.d.ts +18 -6
  19. package/dist/handlers/assess-model-fit.js +215 -35
  20. package/dist/handlers/completion-orchestrate.js +17 -0
  21. package/dist/handlers/get-loaded-model-info.js +4 -1
  22. package/dist/handlers/get-model-info.js +29 -1
  23. package/dist/handlers/load-model/coreml-sidecars.d.ts +25 -0
  24. package/dist/handlers/load-model/coreml-sidecars.js +274 -0
  25. package/dist/handlers/load-model/hyperdrive.js +2 -2
  26. package/dist/handlers/load-model/registry-download-utils.d.ts +1 -0
  27. package/dist/handlers/load-model/registry-download-utils.js +2 -1
  28. package/dist/handlers/load-model/registry.js +59 -0
  29. package/dist/handlers/rpc-server.d.ts +23 -0
  30. package/dist/handlers/rpc-server.js +141 -0
  31. package/dist/index.d.ts +2 -1
  32. package/dist/index.js +2 -1
  33. package/dist/models/registry/models.d.ts +7731 -5672
  34. package/dist/models/registry/models.js +4652 -2526
  35. package/dist/models/registry/resource-profiles.js +481 -94
  36. package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +8 -6
  37. package/dist/plugins/builtin/audiogen-ggml/plugin.js +2 -1
  38. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +73 -1
  39. package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +37 -19
  40. package/dist/plugins/builtin/ggml-vla/plugin.d.ts +1 -1
  41. package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +9 -5
  42. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -1
  43. package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +16 -7
  44. package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +80 -8
  45. package/dist/plugins/builtin/llamacpp-completion/plugin.js +9 -3
  46. package/dist/plugins/builtin/llamacpp-completion/transform.js +19 -0
  47. package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +5 -3
  48. package/dist/plugins/builtin/llamacpp-embedding/plugin.js +2 -1
  49. package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +121 -0
  50. package/dist/plugins/builtin/parakeet-transcription/plugin.js +56 -27
  51. package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +8 -2
  52. package/dist/plugins/builtin/sdcpp-generation/plugin.js +7 -2
  53. package/dist/plugins/builtin/tts-ggml/ops/parler-options.d.ts +2 -0
  54. package/dist/plugins/builtin/tts-ggml/ops/parler-options.js +5 -1
  55. package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +27 -10
  56. package/dist/plugins/builtin/tts-ggml/plugin.d.ts +181 -0
  57. package/dist/plugins/builtin/tts-ggml/plugin.js +84 -2
  58. package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +136 -0
  59. package/dist/plugins/builtin/whispercpp-transcription/plugin.js +18 -7
  60. package/dist/plugins/ops/bci-transcribe.d.ts +6 -2
  61. package/dist/plugins/ops/bci-transcribe.js +18 -13
  62. package/dist/plugins/ops/load-model.js +2 -2
  63. package/dist/plugins/ops/transcribe.d.ts +4 -2
  64. package/dist/plugins/ops/transcribe.js +39 -2
  65. package/dist/plugins/ops/translate-stats.d.ts +2 -1
  66. package/dist/plugins/ops/translate-stats.js +64 -7
  67. package/dist/plugins/ops/translate.js +20 -4
  68. package/dist/profiling/operation-metrics.js +56 -0
  69. package/dist/registry.js +4 -0
  70. package/dist/resources/model-fit/assess.d.ts +46 -2
  71. package/dist/resources/model-fit/assess.js +67 -3
  72. package/dist/resources/model-fit/estimators/llm.js +2 -2
  73. package/dist/resources/model-fit/estimators/whisper.js +3 -4
  74. package/dist/resources/model-fit/fit-stub/fetch-fit-stub.d.ts +102 -0
  75. package/dist/resources/model-fit/fit-stub/fetch-fit-stub.js +212 -0
  76. package/dist/resources/model-fit/fit-stub/project-fit-from-load.d.ts +27 -0
  77. package/dist/resources/model-fit/fit-stub/project-fit-from-load.js +31 -0
  78. package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.d.ts +38 -0
  79. package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.js +138 -0
  80. package/dist/resources/model-fit/floor.js +2 -2
  81. package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +9 -6
  82. package/dist/resources/model-fit/native-probe/advisory-fit.js +129 -80
  83. package/dist/resources/model-fit/native-probe/child-engine-fit.d.ts +2 -0
  84. package/dist/resources/model-fit/native-probe/child-engine-fit.js +41 -0
  85. package/dist/resources/model-fit/native-probe/classify-fit.d.ts +6 -0
  86. package/dist/resources/model-fit/native-probe/classify-fit.js +135 -0
  87. package/dist/resources/model-fit/native-probe/create-fit-request.d.ts +28 -0
  88. package/dist/resources/model-fit/native-probe/create-fit-request.js +51 -0
  89. package/dist/resources/model-fit/native-probe/engine-fit.d.ts +59 -0
  90. package/dist/resources/model-fit/native-probe/engine-fit.js +13 -0
  91. package/dist/resources/model-fit/native-probe/engines/asr.d.ts +10 -0
  92. package/dist/resources/model-fit/native-probe/engines/asr.js +103 -0
  93. package/dist/resources/model-fit/native-probe/engines/audiogen.d.ts +8 -0
  94. package/dist/resources/model-fit/native-probe/engines/audiogen.js +35 -0
  95. package/dist/resources/model-fit/native-probe/engines/diffusion.d.ts +7 -0
  96. package/dist/resources/model-fit/native-probe/engines/diffusion.js +55 -0
  97. package/dist/resources/model-fit/native-probe/engines/gpu.d.ts +10 -0
  98. package/dist/resources/model-fit/native-probe/engines/gpu.js +21 -0
  99. package/dist/resources/model-fit/native-probe/engines/llama.d.ts +13 -0
  100. package/dist/resources/model-fit/native-probe/engines/llama.js +145 -0
  101. package/dist/resources/model-fit/native-probe/engines/tts.d.ts +8 -0
  102. package/dist/resources/model-fit/native-probe/engines/tts.js +106 -0
  103. package/dist/resources/model-fit/native-probe/fit-outcome.d.ts +16 -0
  104. package/dist/resources/model-fit/native-probe/fit-outcome.js +1 -0
  105. package/dist/resources/model-fit/native-probe/fit-process.d.ts +27 -0
  106. package/dist/resources/model-fit/native-probe/fit-process.js +55 -0
  107. package/dist/resources/model-fit/native-probe/fit-runner.d.ts +1 -0
  108. package/dist/resources/model-fit/native-probe/fit-runner.js +58 -0
  109. package/dist/resources/model-fit/native-probe/run-fit.d.ts +24 -0
  110. package/dist/resources/model-fit/native-probe/run-fit.js +20 -0
  111. package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +14 -34
  112. package/dist/resources/model-fit/native-probe/run-in-process-fit.js +55 -139
  113. package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +5 -12
  114. package/dist/resources/model-fit/native-probe/run-isolated-fit.js +13 -8
  115. package/dist/resources/model-fit/platform.d.ts +9 -0
  116. package/dist/resources/model-fit/platform.js +28 -0
  117. package/dist/rpc/device-map.d.ts +13 -0
  118. package/dist/rpc/device-map.js +22 -0
  119. package/dist/rpc/discovery.d.ts +12 -0
  120. package/dist/rpc/discovery.js +225 -0
  121. package/dist/rpc/ggml-provider.d.ts +4 -0
  122. package/dist/rpc/ggml-provider.js +25 -0
  123. package/dist/rpc/index.d.ts +2 -0
  124. package/dist/rpc/index.js +1 -0
  125. package/dist/rpc/instance.d.ts +6 -0
  126. package/dist/rpc/instance.js +21 -0
  127. package/dist/rpc/inventory.d.ts +5 -0
  128. package/dist/rpc/inventory.js +144 -0
  129. package/dist/rpc/manager.d.ts +14 -0
  130. package/dist/rpc/manager.js +174 -0
  131. package/dist/rpc/network.d.ts +11 -0
  132. package/dist/rpc/network.js +85 -0
  133. package/dist/rpc/provider.d.ts +6 -0
  134. package/dist/rpc/provider.js +24 -0
  135. package/dist/runtime/lifecycle.js +31 -13
  136. package/dist/runtime/request-context.d.ts +1 -1
  137. package/dist/runtime/state.js +5 -0
  138. package/dist/schemas/assess-model-fit.d.ts +284 -77
  139. package/dist/schemas/assess-model-fit.js +93 -55
  140. package/dist/schemas/audio-gen.d.ts +6 -6
  141. package/dist/schemas/audio-gen.js +2 -2
  142. package/dist/schemas/batch-completion-stream.d.ts +13 -5
  143. package/dist/schemas/batch-completion-stream.js +5 -3
  144. package/dist/schemas/bci-config.js +1 -1
  145. package/dist/schemas/bci.d.ts +86 -1
  146. package/dist/schemas/bci.js +21 -2
  147. package/dist/schemas/cancel.d.ts +10 -0
  148. package/dist/schemas/cancel.js +2 -0
  149. package/dist/schemas/common.d.ts +765 -56
  150. package/dist/schemas/common.js +7 -0
  151. package/dist/schemas/completion-event.d.ts +6 -6
  152. package/dist/schemas/completion-stream.d.ts +27 -8
  153. package/dist/schemas/completion-stream.js +41 -4
  154. package/dist/schemas/config.d.ts +236 -30
  155. package/dist/schemas/config.js +10 -0
  156. package/dist/schemas/embed.d.ts +2 -2
  157. package/dist/schemas/error.js +13 -0
  158. package/dist/schemas/errors.d.ts +1 -0
  159. package/dist/schemas/errors.js +5 -0
  160. package/dist/schemas/index.d.ts +1 -0
  161. package/dist/schemas/index.js +1 -0
  162. package/dist/schemas/llamacpp-config.d.ts +105 -14
  163. package/dist/schemas/llamacpp-config.js +113 -3
  164. package/dist/schemas/load-model.d.ts +3102 -627
  165. package/dist/schemas/load-model.js +2 -2
  166. package/dist/schemas/mcp-adapter.d.ts +4 -0
  167. package/dist/schemas/model-config-schemas.d.ts +205 -5
  168. package/dist/schemas/model-info.d.ts +56 -0
  169. package/dist/schemas/model-info.js +5 -1
  170. package/dist/schemas/plugin.d.ts +8 -2
  171. package/dist/schemas/profiling.d.ts +4 -4
  172. package/dist/schemas/reload-config.d.ts +30 -0
  173. package/dist/schemas/rpc-server.d.ts +101 -0
  174. package/dist/schemas/rpc-server.js +87 -0
  175. package/dist/schemas/sdcpp-config.d.ts +7 -3
  176. package/dist/schemas/sdcpp-config.js +37 -2
  177. package/dist/schemas/system-resources.d.ts +4 -4
  178. package/dist/schemas/text-to-speech.d.ts +529 -1
  179. package/dist/schemas/text-to-speech.js +122 -12
  180. package/dist/schemas/tools.d.ts +2 -0
  181. package/dist/schemas/tools.js +8 -0
  182. package/dist/schemas/transcription-config.d.ts +5 -0
  183. package/dist/schemas/transcription-config.js +20 -2
  184. package/dist/schemas/transcription.d.ts +164 -0
  185. package/dist/schemas/transcription.js +71 -4
  186. package/dist/schemas/vla.d.ts +1 -1
  187. package/dist/surface.d.ts +4 -1
  188. package/dist/surface.js +3 -1
  189. package/dist/utils/addon-responses.d.ts +5 -1
  190. package/dist/utils/archive.js +5 -1
  191. package/dist/utils/asr-diagnostics.d.ts +12 -0
  192. package/dist/utils/asr-diagnostics.js +42 -0
  193. package/dist/utils/asr-events.d.ts +1 -0
  194. package/dist/utils/asr-events.js +3 -2
  195. package/dist/utils/cache/paths.js +3 -2
  196. package/dist/utils/mcp-adapter.js +7 -1
  197. package/dist/utils/multi-gpu-mobile.d.ts +1 -1
  198. package/dist/utils/multi-gpu-mobile.js +8 -6
  199. package/dist/utils/path-security.d.ts +5 -0
  200. package/dist/utils/path-security.js +14 -0
  201. package/dist/utils/tool-helpers.d.ts +4 -0
  202. package/dist/utils/tool-helpers.js +2 -0
  203. package/dist/utils/tools/defer.d.ts +93 -0
  204. package/dist/utils/tools/defer.js +290 -0
  205. package/dist/utils/tools/index.d.ts +1 -0
  206. package/dist/utils/tools/index.js +1 -0
  207. package/dist/utils/transcribe-metadata.d.ts +21 -7
  208. package/dist/utils/transcribe-metadata.js +24 -8
  209. package/dist/utils/tts-stats.d.ts +6 -0
  210. package/dist/utils/tts-stats.js +14 -1
  211. package/package.json +44 -30
  212. package/dist/resources/model-fit/native-probe/create-llama-fit-request.d.ts +0 -26
  213. package/dist/resources/model-fit/native-probe/create-llama-fit-request.js +0 -171
@@ -66,7 +66,14 @@ export const TTS_SUPERTONIC_LANGUAGES = [
66
66
  // The engines @qvac/tts-ggml exposes as `TTSGgml.ENGINE_*`, in the order the
67
67
  // addon lists them. Exported so callers can discriminate a TTS config without
68
68
  // re-declaring the union (the CLI used to).
69
- export const TTS_ENGINES = ['chatterbox', 'supertonic', 'parler', 'cosyvoice3', 'audio8'];
69
+ export const TTS_ENGINES = [
70
+ 'chatterbox',
71
+ 'supertonic',
72
+ 'parler',
73
+ 'cosyvoice3',
74
+ 'audio8',
75
+ 'moss'
76
+ ];
70
77
  // Mirror of the addon's `SentenceDelimiterPreset` (./text-stream-accumulator).
71
78
  export const TTS_SENTENCE_DELIMITER_PRESETS = ['latin', 'cjk', 'multilingual'];
72
79
  export const TTS_PARLER_EMOTIONS = [
@@ -191,9 +198,10 @@ function refineGpuIntent(config, ctx) {
191
198
  }
192
199
  // Desired output sample rate in Hz. Matches the @qvac/tts-ggml addon's
193
200
  // accepted range; omit to keep the engine's native rate (or 48 kHz when the
194
- // LavaSR enhancer is active). Every engine resamples, so this is exposed on
195
- // all five configs; the engines whose native chunk streaming cannot resample
196
- // seam-free (Parler, CosyVoice3) constrain it in their own refinements.
201
+ // LavaSR enhancer is active). Every engine but MOSS (24 kHz only) resamples,
202
+ // so this is exposed on the other five configs; the engines whose native chunk
203
+ // streaming cannot resample seam-free (Parler, CosyVoice3) constrain it in
204
+ // their own refinements.
197
205
  const ttsOutputSampleRateSchema = ttsIntegerSchema
198
206
  .min(8000)
199
207
  .max(192000)
@@ -544,12 +552,53 @@ const ttsAudio8RuntimeConfigShape = {
544
552
  export const ttsAudio8RuntimeConfigSchema = z
545
553
  .object(ttsAudio8RuntimeConfigShape)
546
554
  .superRefine(refineGpuIntent);
555
+ // @qvac/tts-ggml's MOSS_MAX_DURATION_TOKENS: the 2048-row generation budget
556
+ // less the 31 delay-pattern drain rows and the two termination rows.
557
+ const TTS_MOSS_MAX_DURATION_TOKENS = 2015;
558
+ const TTS_MOSS_MAX_DIALOGUE_SPEAKERS = 5;
559
+ // MOSS emits its native 24 kHz only (the addon rejects any other
560
+ // `outputSampleRate`) and supports neither the LavaSR stages nor a smaller
561
+ // first streaming chunk, so none of those fields exist on this arm.
562
+ const ttsMossRuntimeConfigShape = {
563
+ ttsEngine: z
564
+ .literal('moss')
565
+ .describe('TTS engine: MOSS (OpenMOSS MOSS-TTS v1.5 / MOSS-TTSD Delay; 24 kHz, 8B backbone, desktop only).'),
566
+ // The addon forwards it into the model prompt as a hint and does not
567
+ // validate it, so the vocabulary is the model's, not a fixed enum.
568
+ language: z
569
+ .string()
570
+ .trim()
571
+ .min(1)
572
+ .optional()
573
+ .describe('Language hint written into the model prompt, e.g. `en` or `zh`. Default `en`.'),
574
+ durationTokens: ttsIntegerSchema
575
+ .min(0)
576
+ .max(TTS_MOSS_MAX_DURATION_TOKENS)
577
+ .optional()
578
+ .describe('Target length of each synthesis in codec frames (12.5 per second, so 38 ≈ 3 s), 0–2015; 0 or unset keeps the length free.'),
579
+ streamChunkTokens: ttsNonNegativeInt32Schema
580
+ .optional()
581
+ .describe('Codec frames per native streaming chunk (12.5 per second, so 25 ≈ 2 s); 0 or unset synthesizes the whole text before emitting.'),
582
+ useGPU: z.boolean().optional().describe(TTS_USE_GPU_DESC),
583
+ threads: ttsPositiveInt32Schema.optional().describe(TTS_THREADS_DESC),
584
+ // Not TTS_NGPU_LAYERS_DESC: MOSS does not offload per layer, so a layer
585
+ // count would misdescribe it — the value only picks the backend.
586
+ nGpuLayers: ttsInt32Schema
587
+ .optional()
588
+ .describe('Any non-zero value selects the GPU backend, 0 keeps MOSS on the CPU. Wins over `useGPU`; when both are set they must agree.'),
589
+ seed: ttsInt32Schema.optional().describe(TTS_SEED_DESC),
590
+ ...ttsBackendDirFieldsShape
591
+ };
592
+ export const ttsMossRuntimeConfigSchema = z
593
+ .object(ttsMossRuntimeConfigShape)
594
+ .superRefine(refineGpuIntent);
547
595
  export const ttsRuntimeConfigSchema = z.discriminatedUnion('ttsEngine', [
548
596
  ttsChatterboxRuntimeConfigSchema,
549
597
  ttsSupertonicRuntimeConfigSchema,
550
598
  ttsParlerRuntimeConfigSchema,
551
599
  ttsCosyvoice3RuntimeConfigSchema,
552
- ttsAudio8RuntimeConfigSchema
600
+ ttsAudio8RuntimeConfigSchema,
601
+ ttsMossRuntimeConfigSchema
553
602
  ]);
554
603
  // Optional LavaSR post-processing model sources, shared across engines. Supply
555
604
  // the enhancer GGUF to bandwidth-extend the output to 48 kHz, and/or the
@@ -725,6 +774,55 @@ export const ttsAudio8LoadConfigSchema = z
725
774
  .describe('Audio8 voice-cloning reference recording source; pair with `referenceText`.')
726
775
  })
727
776
  .superRefine(refineAudio8LoadConfig);
777
+ // The addon enforces the same three rules at construction; catching them here
778
+ // names the SDK field instead of the addon's `files.*` option.
779
+ function refineMossLoadConfig(config, ctx) {
780
+ refineGpuIntent(config, ctx);
781
+ if (config.referenceAudioSrc !== undefined && config.dialogueReferenceSrcs !== undefined) {
782
+ ctx.addIssue({
783
+ code: 'custom',
784
+ path: ['dialogueReferenceSrcs'],
785
+ message: 'referenceAudioSrc and dialogueReferenceSrcs are exclusive; pass one recording per speaker in dialogueReferenceSrcs.'
786
+ });
787
+ }
788
+ const cloningField = config.referenceAudioSrc !== undefined
789
+ ? 'referenceAudioSrc'
790
+ : config.dialogueReferenceSrcs !== undefined
791
+ ? 'dialogueReferenceSrcs'
792
+ : undefined;
793
+ if (cloningField !== undefined && config.mossCodecEncoderModelSrc === undefined) {
794
+ ctx.addIssue({
795
+ code: 'custom',
796
+ path: ['mossCodecEncoderModelSrc'],
797
+ message: `${cloningField} requires the MOSS codec encoder GGUF (mossCodecEncoderModelSrc).`
798
+ });
799
+ }
800
+ }
801
+ // MOSS loads from explicit per-component GGUFs like Audio8: the primary
802
+ // `modelSrc` is the Delay backbone — MOSS-TTS (`moss-tts-delay-*.gguf`) or the
803
+ // MOSS-TTSD dialogue checkpoint (`moss-ttsd-*.gguf`) — the codec decoder is
804
+ // required, and the codec encoder is only needed to encode reference audio.
805
+ export const ttsMossLoadConfigSchema = z
806
+ .object({
807
+ ...ttsMossRuntimeConfigShape,
808
+ mossCodecDecoderModelSrc: modelSrcInputSchema.describe('MOSS codec decoder model source (codes to 24 kHz waveform).'),
809
+ mossCodecEncoderModelSrc: modelSrcInputSchema
810
+ .optional()
811
+ .describe('MOSS codec encoder model source (waveform to codes); required only with `referenceAudioSrc` or `dialogueReferenceSrcs`.'),
812
+ referenceAudioSrc: modelSrcInputSchema
813
+ .optional()
814
+ .describe('MOSS voice-cloning reference recording source: a 24 kHz WAV (no resampling; multichannel is downmixed, at most 60 s). No transcript needed. Fixed for the loaded model.'),
815
+ // MOSS-TTSD covers one to five speakers. The engine itself sets no count
816
+ // (only a 60 s cap on the references combined), so without a bound here a
817
+ // single load could fan out into any number of parallel source downloads.
818
+ dialogueReferenceSrcs: z
819
+ .array(modelSrcInputSchema)
820
+ .min(1)
821
+ .max(TTS_MOSS_MAX_DIALOGUE_SPEAKERS)
822
+ .optional()
823
+ .describe('MOSS-TTSD dialogue: one 24 kHz WAV per speaker (one to five, at most 60 s combined), in the order the text tags them (`[S1]`, `[S2]`, …). The text must open with each recording’s transcript under its tag, followed by the lines to generate. Needs a MOSS-TTSD backbone as `modelSrc`; fixed for the loaded model.')
824
+ })
825
+ .superRefine(refineMossLoadConfig);
728
826
  function refineChatterboxTokenizerAssets(data, ctx) {
729
827
  if (data.ttsEngine !== 'chatterbox')
730
828
  return;
@@ -749,7 +847,8 @@ export const ttsLoadConfigSchema = z
749
847
  ttsSupertonicLoadConfigSchema,
750
848
  ttsParlerLoadConfigSchema,
751
849
  ttsCosyvoice3LoadConfigSchema,
752
- ttsAudio8LoadConfigSchema
850
+ ttsAudio8LoadConfigSchema,
851
+ ttsMossLoadConfigSchema
753
852
  ])
754
853
  .superRefine(refineChatterboxTokenizerAssets);
755
854
  // === Legacy ONNX modelConfig fields (deprecated) ===
@@ -788,7 +887,8 @@ export const ttsConfigSchema = z
788
887
  ttsSupertonicLoadConfigSchema.extend(legacyTtsOnnxFieldsShape).strict(),
789
888
  ttsParlerLoadConfigSchema.strict(),
790
889
  ttsCosyvoice3LoadConfigSchema.strict(),
791
- ttsAudio8LoadConfigSchema.strict()
890
+ ttsAudio8LoadConfigSchema.strict(),
891
+ ttsMossLoadConfigSchema.strict()
792
892
  ])
793
893
  .superRefine(refineChatterboxTokenizerAssets);
794
894
  // Cancel targeting. Both TTS handlers declare `cancel: { scope: 'model', hard:
@@ -832,16 +932,18 @@ export const ttsRequestSchema = z
832
932
  // The chunked paths (`stream: true`, `sentenceStream`, and the duplex session)
833
933
  // run one native job per sentence and the addon's aggregate carries only the
834
934
  // four summable fields (totalTime, audioDuration, totalSamples,
835
- // generatedFrames); the five backend fields below are reported by batch
836
- // synthesis (`stream: false`) only.
935
+ // generatedFrames) and Audio8's two Core ML flags, which keep the value of the
936
+ // last sentence that reported them; the five backend fields below are
937
+ // reported by batch synthesis (`stream: false`) only.
837
938
  export const ttsStatsSchema = z.object({
838
939
  audioDuration: z.number().optional(),
839
940
  totalTime: z.number().optional(),
840
941
  realTimeFactor: z.number().optional(),
841
942
  tokensPerSecond: z.number().optional(),
842
943
  totalSamples: z.number().optional(),
843
- // Audio8 counts codec frames on a fixed 46 ms grid rather than tokens, and
844
- // reports them as the unit behind its `tokensPerSecond`.
944
+ // Audio8 (on a fixed 46 ms grid) and MOSS (12.5 per second) count codec
945
+ // frames rather than tokens, and report them as the unit behind their
946
+ // `tokensPerSecond`.
845
947
  generatedFrames: z.number().optional(),
846
948
  // Backend selection captured once at model load. `0` CPU / `1` GPU;
847
949
  // `backendId` codes the family (0 CPU, 1 Metal, 2 CUDA, 3 Vulkan, 4 OpenCL,
@@ -851,7 +953,15 @@ export const ttsStatsSchema = z.object({
851
953
  backendId: z.number().optional(),
852
954
  gpuUnsupported: z.number().optional(),
853
955
  enhancerBackendDevice: z.number().optional(),
854
- enhancerBackendId: z.number().optional()
956
+ enhancerBackendId: z.number().optional(),
957
+ codecSidecarLoaded: z
958
+ .number()
959
+ .optional()
960
+ .describe('Audio8 on macOS / iOS: `1` while the Core ML codec sidecar is attached, `0` without one or once a failing sidecar was retired.'),
961
+ codecOnCoreml: z
962
+ .number()
963
+ .optional()
964
+ .describe('Audio8: `1` when this synthesis ran its codec on the Core ML sidecar, `0` when it ran on the ggml backend `backendId` reports.')
855
965
  });
856
966
  // The rate of `buffer`. It is not a constant of the engine: `outputSampleRate`
857
967
  // and the LavaSR enhancer both move it, so a caller writing a WAV or feeding
@@ -3,6 +3,8 @@ export declare const toolSchema: z.ZodObject<{
3
3
  type: z.ZodLiteral<"function">;
4
4
  name: z.ZodString;
5
5
  description: z.ZodString;
6
+ deferLoading: z.ZodOptional<z.ZodBoolean>;
7
+ group: z.ZodOptional<z.ZodString>;
6
8
  parameters: z.ZodObject<{
7
9
  type: z.ZodLiteral<"object">;
8
10
  properties: z.ZodRecord<z.ZodString, z.ZodObject<{
@@ -4,6 +4,14 @@ export const toolSchema = z.object({
4
4
  type: z.literal('function'),
5
5
  name: z.string(),
6
6
  description: z.string(),
7
+ deferLoading: z
8
+ .boolean()
9
+ .optional()
10
+ .describe('Opt out of the initial prompt. A deferred tool is registered but only its name and description reach the model, in the catalog carried by the built-in `tool_search`; its parameter schema is appended to the conversation once the model searches for it. Tools without this field behave as before.'),
11
+ group: z
12
+ .string()
13
+ .optional()
14
+ .describe('Optional heading this tool is listed under in the deferred catalog — a skill or an MCP server name. Ignored for tools that are not deferred.'),
7
15
  parameters: z.object({
8
16
  type: z.literal('object'),
9
17
  properties: z.record(z.string(), z.object({
@@ -43,6 +43,10 @@ export declare const whisperConfigSchema: z.ZodObject<{
43
43
  logprob_thold: z.ZodOptional<z.ZodNumber>;
44
44
  greedy_best_of: z.ZodOptional<z.ZodNumber>;
45
45
  beam_search_beam_size: z.ZodOptional<z.ZodNumber>;
46
+ max_initial_ts: z.ZodOptional<z.ZodNumber>;
47
+ no_speech_thold: z.ZodOptional<z.ZodNumber>;
48
+ seed: z.ZodOptional<z.ZodNumber>;
49
+ backendsDir: z.ZodOptional<z.ZodString>;
46
50
  vad_params: z.ZodOptional<z.ZodObject<{
47
51
  threshold: z.ZodOptional<z.ZodNumber>;
48
52
  min_speech_duration_ms: z.ZodOptional<z.ZodNumber>;
@@ -63,6 +67,7 @@ export declare const whisperConfigSchema: z.ZodObject<{
63
67
  }, z.core.$strip>>;
64
68
  miscConfig: z.ZodOptional<z.ZodObject<{
65
69
  caption_enabled: z.ZodOptional<z.ZodBoolean>;
70
+ seed: z.ZodOptional<z.ZodNumber>;
66
71
  }, z.core.$strip>>;
67
72
  vadModelSrc: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodObject<{
68
73
  src: z.ZodString;
@@ -41,7 +41,12 @@ const contextParamsSchema = z
41
41
  .optional();
42
42
  const miscConfigSchema = z
43
43
  .object({
44
- caption_enabled: z.boolean().optional().describe('Format output segments as captions.')
44
+ caption_enabled: z.boolean().optional().describe('Format output segments as captions.'),
45
+ seed: z
46
+ .number()
47
+ .int()
48
+ .optional()
49
+ .describe('RNG seed applied to the whisper context; `-1` picks a random seed.')
45
50
  })
46
51
  .optional();
47
52
  export const whisperConfigSchema = z.object({
@@ -135,7 +140,7 @@ export const whisperConfigSchema = z.object({
135
140
  detect_language: z
136
141
  .boolean()
137
142
  .optional()
138
- .describe("Not supported natively (rejected by the addon); use `language: 'auto'` to auto-detect the spoken language."),
143
+ .describe("Convenience flag: `true` is translated to `language: 'auto'` before the request reaches the addon, which has no `detect_language` key of its own."),
139
144
  suppress_blank: z
140
145
  .boolean()
141
146
  .optional()
@@ -165,6 +170,19 @@ export const whisperConfigSchema = z.object({
165
170
  .int()
166
171
  .optional()
167
172
  .describe('Beam size for beam-search decoding; `-1` = default.'),
173
+ max_initial_ts: z
174
+ .number()
175
+ .optional()
176
+ .describe('Maximum initial timestamp the decoder may emit, as a fraction of the window.'),
177
+ no_speech_thold: z
178
+ .number()
179
+ .optional()
180
+ .describe('No-speech probability above which a segment is treated as silence.'),
181
+ seed: z.number().int().optional().describe('Decoding RNG seed; `-1` picks a random seed.'),
182
+ backendsDir: z
183
+ .string()
184
+ .optional()
185
+ .describe("Root directory for dynamically-loaded ggml backend libraries. Defaults to the addon's own `prebuilds/`."),
168
186
  vad_params: vadParamsSchema,
169
187
  audio_format: audioFormatSchema
170
188
  .optional()
@@ -1,5 +1,6 @@
1
1
  import { z } from 'zod';
2
2
  import type Buffer from 'bare-buffer';
3
+ import { type InferenceBackendDiagnostics } from './system-resources.js';
3
4
  export declare const audioInputSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
4
5
  type: z.ZodLiteral<"base64">;
5
6
  value: z.ZodString;
@@ -35,17 +36,54 @@ export declare const transcribeStatsSchema: z.ZodObject<{
35
36
  gpuUnsupported: z.ZodOptional<z.ZodNumber>;
36
37
  gpuMemTotalMb: z.ZodOptional<z.ZodNumber>;
37
38
  gpuMemFreeMb: z.ZodOptional<z.ZodNumber>;
39
+ totalTime: z.ZodOptional<z.ZodNumber>;
40
+ totalWallMs: z.ZodOptional<z.ZodNumber>;
41
+ totalSamples: z.ZodOptional<z.ZodNumber>;
42
+ processCalls: z.ZodOptional<z.ZodNumber>;
43
+ whisperSampleMs: z.ZodOptional<z.ZodNumber>;
44
+ whisperBatchdMs: z.ZodOptional<z.ZodNumber>;
45
+ whisperPromptMs: z.ZodOptional<z.ZodNumber>;
46
+ totalTranscriptions: z.ZodOptional<z.ZodNumber>;
47
+ modelLoadMs: z.ZodOptional<z.ZodNumber>;
48
+ totalEncodedFrames: z.ZodOptional<z.ZodNumber>;
49
+ encoderOnCoreml: z.ZodOptional<z.ZodNumber>;
50
+ encoderUsedCoreml: z.ZodOptional<z.ZodNumber>;
38
51
  }, z.core.$strip>;
52
+ /**
53
+ * Compute backends the ASR engines report through `stats.backendId`, as the
54
+ * addon's `BackendId` enum defines them. Exported so callers can decode the
55
+ * number instead of hardcoding it.
56
+ *
57
+ * Kept as a literal because it feeds the contract, and checked against the
58
+ * addon's enum by `satisfies`: a backend the addon adds or removes is a compile
59
+ * error here rather than a silently short vocabulary.
60
+ */
61
+ export declare const ASR_BACKEND_IDS: Readonly<{
62
+ readonly CPU: 0;
63
+ readonly Metal: 1;
64
+ readonly CUDA: 2;
65
+ readonly Vulkan: 3;
66
+ readonly OpenCL: 4;
67
+ readonly Other: 99;
68
+ }>;
69
+ export type AsrBackendId = (typeof ASR_BACKEND_IDS)[keyof typeof ASR_BACKEND_IDS];
39
70
  export declare const transcribeSegmentSchema: z.ZodObject<{
40
71
  text: z.ZodString;
41
72
  startMs: z.ZodNumber;
42
73
  endMs: z.ZodNumber;
43
74
  append: z.ZodBoolean;
44
75
  id: z.ZodNumber;
76
+ isEndOfTurn: z.ZodOptional<z.ZodBoolean>;
77
+ startsWord: z.ZodOptional<z.ZodBoolean>;
45
78
  }, z.core.$strip>;
46
79
  export declare const vadStateEventSchema: z.ZodObject<{
47
80
  speaking: z.ZodBoolean;
48
81
  probability: z.ZodNumber;
82
+ source: z.ZodOptional<z.ZodEnum<{
83
+ silero: "silero";
84
+ energy: "energy";
85
+ sortformer: "sortformer";
86
+ }>>;
49
87
  }, z.core.$strip>;
50
88
  export declare const whisperEndOfTurnEventSchema: z.ZodObject<{
51
89
  source: z.ZodLiteral<"whisper">;
@@ -93,6 +131,18 @@ export declare const transcribeResponseSchema: z.ZodObject<{
93
131
  gpuUnsupported: z.ZodOptional<z.ZodNumber>;
94
132
  gpuMemTotalMb: z.ZodOptional<z.ZodNumber>;
95
133
  gpuMemFreeMb: z.ZodOptional<z.ZodNumber>;
134
+ totalTime: z.ZodOptional<z.ZodNumber>;
135
+ totalWallMs: z.ZodOptional<z.ZodNumber>;
136
+ totalSamples: z.ZodOptional<z.ZodNumber>;
137
+ processCalls: z.ZodOptional<z.ZodNumber>;
138
+ whisperSampleMs: z.ZodOptional<z.ZodNumber>;
139
+ whisperBatchdMs: z.ZodOptional<z.ZodNumber>;
140
+ whisperPromptMs: z.ZodOptional<z.ZodNumber>;
141
+ totalTranscriptions: z.ZodOptional<z.ZodNumber>;
142
+ modelLoadMs: z.ZodOptional<z.ZodNumber>;
143
+ totalEncodedFrames: z.ZodOptional<z.ZodNumber>;
144
+ encoderOnCoreml: z.ZodOptional<z.ZodNumber>;
145
+ encoderUsedCoreml: z.ZodOptional<z.ZodNumber>;
96
146
  }, z.core.$strip>>;
97
147
  error: z.ZodOptional<z.ZodString>;
98
148
  segment: z.ZodOptional<z.ZodObject<{
@@ -101,10 +151,17 @@ export declare const transcribeResponseSchema: z.ZodObject<{
101
151
  endMs: z.ZodNumber;
102
152
  append: z.ZodBoolean;
103
153
  id: z.ZodNumber;
154
+ isEndOfTurn: z.ZodOptional<z.ZodBoolean>;
155
+ startsWord: z.ZodOptional<z.ZodBoolean>;
104
156
  }, z.core.$strip>>;
105
157
  vad: z.ZodOptional<z.ZodObject<{
106
158
  speaking: z.ZodBoolean;
107
159
  probability: z.ZodNumber;
160
+ source: z.ZodOptional<z.ZodEnum<{
161
+ silero: "silero";
162
+ energy: "energy";
163
+ sortformer: "sortformer";
164
+ }>>;
108
165
  }, z.core.$strip>>;
109
166
  endOfTurn: z.ZodOptional<z.ZodPreprocess<z.ZodDiscriminatedUnion<[z.ZodObject<{
110
167
  source: z.ZodLiteral<"whisper">;
@@ -112,6 +169,47 @@ export declare const transcribeResponseSchema: z.ZodObject<{
112
169
  }, z.core.$strip>, z.ZodObject<{
113
170
  source: z.ZodLiteral<"parakeet">;
114
171
  }, z.core.$strip>], "source">, unknown>>;
172
+ diagnostics: z.ZodOptional<z.ZodObject<{
173
+ selectedBackend: z.ZodString;
174
+ selectedDevice: z.ZodEnum<{
175
+ cpu: "cpu";
176
+ gpu: "gpu";
177
+ }>;
178
+ graphicsApi: z.ZodOptional<z.ZodEnum<{
179
+ vulkan: "vulkan";
180
+ metal: "metal";
181
+ opencl: "opencl";
182
+ cuda: "cuda";
183
+ opengl: "opengl";
184
+ webgpu: "webgpu";
185
+ direct3d11: "direct3d11";
186
+ direct3d12: "direct3d12";
187
+ levelZero: "levelZero";
188
+ rocm: "rocm";
189
+ }>>;
190
+ driver: z.ZodOptional<z.ZodObject<{
191
+ name: z.ZodString;
192
+ version: z.ZodOptional<z.ZodString>;
193
+ }, z.core.$strip>>;
194
+ gpuId: z.ZodOptional<z.ZodString>;
195
+ fallback: z.ZodOptional<z.ZodObject<{
196
+ requestedBackend: z.ZodOptional<z.ZodString>;
197
+ requestedDevice: z.ZodOptional<z.ZodEnum<{
198
+ cpu: "cpu";
199
+ gpu: "gpu";
200
+ }>>;
201
+ reason: z.ZodString;
202
+ }, z.core.$strip>>;
203
+ probe: z.ZodOptional<z.ZodObject<{
204
+ status: z.ZodEnum<{
205
+ unknown: "unknown";
206
+ compatible: "compatible";
207
+ incompatible: "incompatible";
208
+ }>;
209
+ backend: z.ZodString;
210
+ reason: z.ZodOptional<z.ZodString>;
211
+ }, z.core.$strip>>;
212
+ }, z.core.$strip>>;
115
213
  type: z.ZodLiteral<"transcribe">;
116
214
  }, z.core.$strip>;
117
215
  export type AudioInput = z.infer<typeof audioInputSchema>;
@@ -193,6 +291,18 @@ export declare const transcribeStreamResponseSchema: z.ZodObject<{
193
291
  gpuUnsupported: z.ZodOptional<z.ZodNumber>;
194
292
  gpuMemTotalMb: z.ZodOptional<z.ZodNumber>;
195
293
  gpuMemFreeMb: z.ZodOptional<z.ZodNumber>;
294
+ totalTime: z.ZodOptional<z.ZodNumber>;
295
+ totalWallMs: z.ZodOptional<z.ZodNumber>;
296
+ totalSamples: z.ZodOptional<z.ZodNumber>;
297
+ processCalls: z.ZodOptional<z.ZodNumber>;
298
+ whisperSampleMs: z.ZodOptional<z.ZodNumber>;
299
+ whisperBatchdMs: z.ZodOptional<z.ZodNumber>;
300
+ whisperPromptMs: z.ZodOptional<z.ZodNumber>;
301
+ totalTranscriptions: z.ZodOptional<z.ZodNumber>;
302
+ modelLoadMs: z.ZodOptional<z.ZodNumber>;
303
+ totalEncodedFrames: z.ZodOptional<z.ZodNumber>;
304
+ encoderOnCoreml: z.ZodOptional<z.ZodNumber>;
305
+ encoderUsedCoreml: z.ZodOptional<z.ZodNumber>;
196
306
  }, z.core.$strip>>;
197
307
  error: z.ZodOptional<z.ZodString>;
198
308
  segment: z.ZodOptional<z.ZodObject<{
@@ -201,10 +311,17 @@ export declare const transcribeStreamResponseSchema: z.ZodObject<{
201
311
  endMs: z.ZodNumber;
202
312
  append: z.ZodBoolean;
203
313
  id: z.ZodNumber;
314
+ isEndOfTurn: z.ZodOptional<z.ZodBoolean>;
315
+ startsWord: z.ZodOptional<z.ZodBoolean>;
204
316
  }, z.core.$strip>>;
205
317
  vad: z.ZodOptional<z.ZodObject<{
206
318
  speaking: z.ZodBoolean;
207
319
  probability: z.ZodNumber;
320
+ source: z.ZodOptional<z.ZodEnum<{
321
+ silero: "silero";
322
+ energy: "energy";
323
+ sortformer: "sortformer";
324
+ }>>;
208
325
  }, z.core.$strip>>;
209
326
  endOfTurn: z.ZodOptional<z.ZodPreprocess<z.ZodDiscriminatedUnion<[z.ZodObject<{
210
327
  source: z.ZodLiteral<"whisper">;
@@ -212,6 +329,47 @@ export declare const transcribeStreamResponseSchema: z.ZodObject<{
212
329
  }, z.core.$strip>, z.ZodObject<{
213
330
  source: z.ZodLiteral<"parakeet">;
214
331
  }, z.core.$strip>], "source">, unknown>>;
332
+ diagnostics: z.ZodOptional<z.ZodObject<{
333
+ selectedBackend: z.ZodString;
334
+ selectedDevice: z.ZodEnum<{
335
+ cpu: "cpu";
336
+ gpu: "gpu";
337
+ }>;
338
+ graphicsApi: z.ZodOptional<z.ZodEnum<{
339
+ vulkan: "vulkan";
340
+ metal: "metal";
341
+ opencl: "opencl";
342
+ cuda: "cuda";
343
+ opengl: "opengl";
344
+ webgpu: "webgpu";
345
+ direct3d11: "direct3d11";
346
+ direct3d12: "direct3d12";
347
+ levelZero: "levelZero";
348
+ rocm: "rocm";
349
+ }>>;
350
+ driver: z.ZodOptional<z.ZodObject<{
351
+ name: z.ZodString;
352
+ version: z.ZodOptional<z.ZodString>;
353
+ }, z.core.$strip>>;
354
+ gpuId: z.ZodOptional<z.ZodString>;
355
+ fallback: z.ZodOptional<z.ZodObject<{
356
+ requestedBackend: z.ZodOptional<z.ZodString>;
357
+ requestedDevice: z.ZodOptional<z.ZodEnum<{
358
+ cpu: "cpu";
359
+ gpu: "gpu";
360
+ }>>;
361
+ reason: z.ZodString;
362
+ }, z.core.$strip>>;
363
+ probe: z.ZodOptional<z.ZodObject<{
364
+ status: z.ZodEnum<{
365
+ unknown: "unknown";
366
+ compatible: "compatible";
367
+ incompatible: "incompatible";
368
+ }>;
369
+ backend: z.ZodString;
370
+ reason: z.ZodOptional<z.ZodString>;
371
+ }, z.core.$strip>>;
372
+ }, z.core.$strip>>;
215
373
  type: z.ZodLiteral<"transcribeStream">;
216
374
  }, z.core.$strip>;
217
375
  export type TranscribeStreamRequest = z.infer<typeof transcribeStreamRequestSchema>;
@@ -227,6 +385,8 @@ export type TranscribeStreamClientParams = {
227
385
  };
228
386
  export interface TranscribeStreamSession {
229
387
  stats: Promise<TranscribeStats | undefined>;
388
+ /** Backend selection detail for the session, settled from the same terminal frame as `stats`. */
389
+ diagnostics: Promise<InferenceBackendDiagnostics | undefined>;
230
390
  write(audioChunk: Uint8Array): void;
231
391
  end(): void;
232
392
  destroy(): void;
@@ -234,6 +394,8 @@ export interface TranscribeStreamSession {
234
394
  }
235
395
  export interface TranscribeStreamMetadataSession {
236
396
  stats: Promise<TranscribeStats | undefined>;
397
+ /** Backend selection detail for the session, settled from the same terminal frame as `stats`. */
398
+ diagnostics: Promise<InferenceBackendDiagnostics | undefined>;
237
399
  write(audioChunk: Uint8Array): void;
238
400
  end(): void;
239
401
  destroy(): void;
@@ -256,6 +418,8 @@ export type TranscribeStreamEvent = {
256
418
  } & EndOfTurnEvent);
257
419
  export interface TranscribeStreamConversationSession {
258
420
  stats: Promise<TranscribeStats | undefined>;
421
+ /** Backend selection detail for the session, settled from the same terminal frame as `stats`. */
422
+ diagnostics: Promise<InferenceBackendDiagnostics | undefined>;
259
423
  write(audioChunk: Uint8Array): void;
260
424
  end(): void;
261
425
  destroy(): void;
@@ -1,4 +1,5 @@
1
1
  import { z } from 'zod';
2
+ import { inferenceBackendDiagnosticsSchema } from './system-resources.js';
2
3
  export const audioInputSchema = z.discriminatedUnion('type', [
3
4
  z.object({
4
5
  type: z.literal('base64'),
@@ -41,18 +42,81 @@ export const transcribeStatsSchema = z.object({
41
42
  backendId: z.number().optional(),
42
43
  gpuUnsupported: z.number().optional(),
43
44
  gpuMemTotalMb: z.number().optional(),
44
- gpuMemFreeMb: z.number().optional()
45
+ gpuMemFreeMb: z.number().optional(),
46
+ // Shared engine counters.
47
+ totalTime: z.number().optional().describe('Total native inference time for the run, in ms.'),
48
+ totalWallMs: z
49
+ .number()
50
+ .optional()
51
+ .describe('Wall-clock time spent inside the native engine for the run, in ms.'),
52
+ totalSamples: z.number().optional().describe('Audio samples processed across the run.'),
53
+ processCalls: z
54
+ .number()
55
+ .optional()
56
+ .describe('Native `process()` invocations made across the run.'),
57
+ // whisper.cpp stage timings: the whisper engine and the BCI engine, which
58
+ // runs on the same decoder.
59
+ whisperSampleMs: z.number().optional().describe('Whisper: sampling time, in ms.'),
60
+ whisperBatchdMs: z.number().optional().describe('Whisper: batched-decode time, in ms.'),
61
+ whisperPromptMs: z.number().optional().describe('Whisper: prompt-processing time, in ms.'),
62
+ // Parakeet-only counters.
63
+ totalTranscriptions: z
64
+ .number()
65
+ .optional()
66
+ .describe('Parakeet: transcriptions produced across the session.'),
67
+ modelLoadMs: z.number().optional().describe('Parakeet: model load time, in ms.'),
68
+ totalEncodedFrames: z
69
+ .number()
70
+ .optional()
71
+ .describe('Parakeet: encoder frames produced across the run.'),
72
+ encoderOnCoreml: z
73
+ .number()
74
+ .optional()
75
+ .describe('Parakeet: `1` when a Core ML encoder sidecar loaded, `0` otherwise.'),
76
+ encoderUsedCoreml: z
77
+ .number()
78
+ .optional()
79
+ .describe('Parakeet offline ASR: `1` when this run used Core ML for encoding, `0` on fallback.')
80
+ });
81
+ /**
82
+ * Compute backends the ASR engines report through `stats.backendId`, as the
83
+ * addon's `BackendId` enum defines them. Exported so callers can decode the
84
+ * number instead of hardcoding it.
85
+ *
86
+ * Kept as a literal because it feeds the contract, and checked against the
87
+ * addon's enum by `satisfies`: a backend the addon adds or removes is a compile
88
+ * error here rather than a silently short vocabulary.
89
+ */
90
+ export const ASR_BACKEND_IDS = Object.freeze({
91
+ CPU: 0,
92
+ Metal: 1,
93
+ CUDA: 2,
94
+ Vulkan: 3,
95
+ OpenCL: 4,
96
+ Other: 99
45
97
  });
46
98
  export const transcribeSegmentSchema = z.object({
47
99
  text: z.string(),
48
100
  startMs: z.number(),
49
101
  endMs: z.number(),
50
102
  append: z.boolean(),
51
- id: z.number()
103
+ id: z.number(),
104
+ isEndOfTurn: z
105
+ .boolean()
106
+ .optional()
107
+ .describe('Segment ends on a recognized end-of-utterance boundary. Parakeet engine only (EOU-capable checkpoints); absent on whisper.'),
108
+ startsWord: z
109
+ .boolean()
110
+ .optional()
111
+ .describe('Segment begins a new SentencePiece word, for joining partial segments without splitting words. Parakeet engine only; absent on whisper.')
52
112
  });
53
113
  export const vadStateEventSchema = z.object({
54
114
  speaking: z.boolean(),
55
- probability: z.number()
115
+ probability: z.number(),
116
+ source: z
117
+ .enum(['silero', 'energy', 'sortformer'])
118
+ .optional()
119
+ .describe("Detector behind the event: `'silero'` for Whisper, `'energy'` for Parakeet energy VAD, or `'sortformer'` for Sortformer speaker activity.")
56
120
  });
57
121
  export const whisperEndOfTurnEventSchema = z.object({
58
122
  source: z.literal('whisper'),
@@ -94,7 +158,10 @@ const transcriptionResultBase = z.object({
94
158
  error: z.string().optional(),
95
159
  segment: transcribeSegmentSchema.optional(),
96
160
  vad: vadStateEventSchema.optional(),
97
- endOfTurn: endOfTurnEventSchema.optional()
161
+ endOfTurn: endOfTurnEventSchema.optional(),
162
+ diagnostics: inferenceBackendDiagnosticsSchema
163
+ .optional()
164
+ .describe('Backend selection detail for the completed run, on the terminal frame. Carries the same payload the engine attaches to the internal diagnostics symbol, so an RPC client can read it.')
98
165
  });
99
166
  export const transcribeResponseSchema = transcriptionResultBase.extend({
100
167
  type: z.literal('transcribe')
@@ -42,8 +42,8 @@ export type VlaEmbodimentSelection = string | number | {
42
42
  };
43
43
  export declare const vlaConfigSchema: z.ZodObject<{
44
44
  backend: z.ZodOptional<z.ZodEnum<{
45
- auto: "auto";
46
45
  cpu: "cpu";
46
+ auto: "auto";
47
47
  }>>;
48
48
  verbosity: z.ZodOptional<z.ZodNumber>;
49
49
  embodiment: z.ZodOptional<z.ZodUnion<readonly [z.ZodString, z.ZodNumber, z.ZodObject<{