@qvac/inference 0.21.0 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +50 -25
- package/README.md +44 -1
- package/dist/api/assess-model-fit.d.ts +4 -4
- package/dist/api/assess-model-fit.js +4 -4
- package/dist/api/audio-gen.d.ts +4 -1
- package/dist/api/audio-gen.js +4 -1
- package/dist/api/batch-completion.js +1 -1
- package/dist/api/completion-stream.js +1 -1
- package/dist/api/decide.d.ts +26 -0
- package/dist/api/decide.js +26 -0
- package/dist/api/get-model-info.d.ts +1 -1
- package/dist/api/index.d.ts +1 -0
- package/dist/api/index.js +1 -0
- package/dist/api/text-to-speech.d.ts +1 -0
- package/dist/api/text-to-speech.js +15 -19
- package/dist/api/transcribe.js +5 -2
- package/dist/api/video-request.d.ts +5 -5
- package/dist/api/video.d.ts +5 -4
- package/dist/api/video.js +5 -4
- package/dist/errors/index.d.ts +3 -0
- package/dist/errors/index.js +5 -0
- package/dist/handlers/assess-model-fit.d.ts +5 -3
- package/dist/handlers/assess-model-fit.js +80 -28
- package/dist/handlers/get-loaded-model-info.js +1 -2
- package/dist/handlers/load-model/safe-fetch.js +5 -6
- package/dist/handlers/registry.d.ts +3 -1
- package/dist/handlers/registry.js +3 -3
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/logging/logger.js +1 -1
- package/dist/logging/utils.js +1 -1
- package/dist/models/registry/models.d.ts +563 -816
- package/dist/models/registry/models.js +2873 -3110
- package/dist/models/registry/resource-profiles.d.ts +1 -2
- package/dist/models/registry/resource-profiles.js +203 -2
- package/dist/plugins/builtin/asr-ggml/config.d.ts +7 -1
- package/dist/plugins/builtin/asr-ggml/config.js +6 -0
- package/dist/plugins/builtin/asr-ggml/logging.d.ts +1 -1
- package/dist/plugins/builtin/audiogen-ggml/config.d.ts +1 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +28 -10
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.d.ts +27 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.js +23 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +6 -27
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +147 -60
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +1 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +40 -24
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +1 -1
- package/dist/plugins/builtin/ggml-classification/plugin.d.ts +4 -4
- package/dist/plugins/builtin/ggml-ocr/params.d.ts +18 -0
- package/dist/plugins/builtin/ggml-ocr/params.js +37 -0
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +18 -10
- package/dist/plugins/builtin/ggml-ocr/plugin.js +16 -38
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +21 -14
- package/dist/plugins/builtin/ggml-vla/plugin.js +1 -1
- package/dist/plugins/builtin/index.d.ts +2 -0
- package/dist/plugins/builtin/index.js +2 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +9 -7
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +109 -189
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +32 -90
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +60 -211
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +2 -49
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +2 -46
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +185 -87
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +4 -5
- package/dist/plugins/builtin/llamacpp-completion/transform.js +1 -1
- package/dist/plugins/builtin/llamacpp-decision/helpers.d.ts +6 -0
- package/dist/plugins/builtin/llamacpp-decision/helpers.js +97 -0
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.d.ts +91 -0
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.js +129 -0
- package/dist/plugins/builtin/llamacpp-decision/plugin.d.ts +216 -0
- package/dist/plugins/builtin/llamacpp-decision/plugin.js +43 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +12 -5
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +1 -1
- package/dist/plugins/builtin/llamacpp-embedding/transform.js +3 -0
- package/dist/plugins/builtin/moss-transcribe/plugin.d.ts +143 -0
- package/dist/plugins/builtin/moss-transcribe/plugin.js +96 -0
- package/dist/plugins/builtin/nmtcpp-translation/load-files.d.ts +6 -0
- package/dist/plugins/builtin/nmtcpp-translation/load-files.js +70 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +25 -13
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +9 -73
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +52 -31
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +18 -3
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +10 -6
- package/dist/plugins/builtin/sdcpp-generation/ops/world.d.ts +2 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/world.js +1 -1
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +59 -33
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +12 -9
- package/dist/plugins/builtin/tts-ggml/ops/moss-options.d.ts +22 -0
- package/dist/plugins/builtin/tts-ggml/ops/moss-options.js +79 -0
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +4 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +17 -5
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +164 -29
- package/dist/plugins/builtin/tts-ggml/plugin.js +38 -1
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +58 -35
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +6 -2
- package/dist/plugins/ops/kv-cache-utils.d.ts +1 -1
- package/dist/plugins/ops/kv-cache-utils.js +10 -25
- package/dist/plugins/ops/load-model.js +21 -16
- package/dist/plugins/ops/transcribe.js +17 -1
- package/dist/registry.js +1 -0
- package/dist/resources/model-fit/assess.d.ts +15 -23
- package/dist/resources/model-fit/assess.js +207 -271
- package/dist/resources/model-fit/fit-stub/fetch-fit-stub.d.ts +3 -1
- package/dist/resources/model-fit/fit-stub/fetch-fit-stub.js +6 -1
- package/dist/resources/model-fit/fit-stub/project-fit-from-load.js +2 -0
- package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.d.ts +5 -3
- package/dist/resources/model-fit/fit-stub/resolve-load-from-stubs.js +9 -1
- package/dist/resources/model-fit/floor.d.ts +10 -10
- package/dist/resources/model-fit/floor.js +13 -13
- package/dist/resources/model-fit/kv-cache.d.ts +23 -0
- package/dist/resources/model-fit/kv-cache.js +85 -0
- package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +11 -6
- package/dist/resources/model-fit/native-probe/advisory-fit.js +12 -27
- package/dist/resources/model-fit/native-probe/child-engine-fit.js +8 -0
- package/dist/resources/model-fit/native-probe/classify-fit.js +32 -1
- package/dist/resources/model-fit/native-probe/create-fit-request.d.ts +4 -1
- package/dist/resources/model-fit/native-probe/create-fit-request.js +24 -14
- package/dist/resources/model-fit/native-probe/engine-fit.d.ts +14 -0
- package/dist/resources/model-fit/native-probe/engine-fit.js +1 -1
- package/dist/resources/model-fit/native-probe/engines/asr.d.ts +4 -0
- package/dist/resources/model-fit/native-probe/engines/asr.js +23 -0
- package/dist/resources/model-fit/native-probe/engines/audiogen.js +29 -1
- package/dist/resources/model-fit/native-probe/engines/diffusion.js +16 -6
- package/dist/resources/model-fit/native-probe/engines/llama.d.ts +0 -1
- package/dist/resources/model-fit/native-probe/engines/llama.js +19 -123
- package/dist/resources/model-fit/native-probe/engines/nmt.d.ts +8 -0
- package/dist/resources/model-fit/native-probe/engines/nmt.js +11 -0
- package/dist/resources/model-fit/native-probe/engines/ocr.d.ts +9 -0
- package/dist/resources/model-fit/native-probe/engines/ocr.js +13 -0
- package/dist/resources/model-fit/native-probe/engines/tts.d.ts +2 -0
- package/dist/resources/model-fit/native-probe/engines/tts.js +49 -3
- package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +5 -3
- package/dist/resources/model-fit/native-probe/run-in-process-fit.js +38 -23
- package/dist/resources/model-fit/platform.d.ts +4 -4
- package/dist/resources/model-fit/platform.js +6 -8
- package/dist/resources/model-fit/types.d.ts +3 -117
- package/dist/rpc/inventory.js +5 -7
- package/dist/runtime/env.d.ts +0 -2
- package/dist/runtime/env.js +2 -9
- package/dist/runtime/model-config-utils.js +6 -1
- package/dist/runtime/request-context.d.ts +1 -1
- package/dist/runtime/request-context.js +8 -0
- package/dist/runtime/state.js +5 -0
- package/dist/schemas/assess-model-fit.d.ts +100 -43
- package/dist/schemas/assess-model-fit.js +66 -29
- package/dist/schemas/audio-gen.d.ts +186 -18
- package/dist/schemas/audio-gen.js +180 -48
- package/dist/schemas/batch-completion-stream.d.ts +202 -34
- package/dist/schemas/bci-config.d.ts +2 -0
- package/dist/schemas/bci.d.ts +23 -2
- package/dist/schemas/cancel.d.ts +5 -0
- package/dist/schemas/cancel.js +1 -0
- package/dist/schemas/common.d.ts +1083 -158
- package/dist/schemas/common.js +5 -2
- package/dist/schemas/completion-stream.d.ts +232 -37
- package/dist/schemas/completion-stream.js +64 -20
- package/dist/schemas/config.d.ts +477 -12
- package/dist/schemas/config.js +21 -1
- package/dist/schemas/decision.d.ts +602 -0
- package/dist/schemas/decision.js +205 -0
- package/dist/schemas/download-asset.d.ts +7 -3
- package/dist/schemas/engine-addon-map.d.ts +7 -0
- package/dist/schemas/engine-addon-map.js +23 -3
- package/dist/schemas/errors.d.ts +1 -0
- package/dist/schemas/errors.js +5 -0
- package/dist/schemas/index.d.ts +2 -0
- package/dist/schemas/index.js +2 -0
- package/dist/schemas/llamacpp-config.d.ts +42 -2
- package/dist/schemas/llamacpp-config.js +27 -0
- package/dist/schemas/load-model.d.ts +5247 -1309
- package/dist/schemas/load-model.js +88 -9
- package/dist/schemas/model-config-schemas.d.ts +233 -8
- package/dist/schemas/model-config-schemas.js +4 -1
- package/dist/schemas/model-fit-policy.d.ts +17 -0
- package/dist/schemas/model-fit-policy.js +17 -0
- package/dist/schemas/model-info.d.ts +20 -2
- package/dist/schemas/model-info.js +2 -1
- package/dist/schemas/model-resource-profile.d.ts +3 -4
- package/dist/schemas/model-resource-profile.js +1 -4
- package/dist/schemas/model-src-utils.d.ts +10 -2
- package/dist/schemas/model-types.d.ts +10 -1
- package/dist/schemas/model-types.js +3 -0
- package/dist/schemas/ocr.d.ts +74 -0
- package/dist/schemas/ocr.js +42 -0
- package/dist/schemas/plugin.d.ts +8 -2
- package/dist/schemas/plugin.js +6 -0
- package/dist/schemas/profiling.d.ts +4 -4
- package/dist/schemas/registry.d.ts +26 -0
- package/dist/schemas/registry.js +3 -0
- package/dist/schemas/reload-config.d.ts +10 -2
- package/dist/schemas/rpc-server.js +3 -3
- package/dist/schemas/sdcpp-config.d.ts +26 -0
- package/dist/schemas/sdcpp-config.js +8 -7
- package/dist/schemas/system-resources.d.ts +4 -4
- package/dist/schemas/text-to-speech.d.ts +402 -3
- package/dist/schemas/text-to-speech.js +129 -11
- package/dist/schemas/tools.d.ts +13 -4
- package/dist/schemas/tools.js +18 -6
- package/dist/schemas/transcription-config.d.ts +15 -0
- package/dist/schemas/transcription-config.js +38 -3
- package/dist/schemas/transcription.d.ts +46 -4
- package/dist/schemas/transcription.js +51 -4
- package/dist/schemas/translation-config.d.ts +30 -10
- package/dist/schemas/vla.d.ts +9 -2
- package/dist/schemas/vla.js +9 -5
- package/dist/surface.d.ts +1 -1
- package/dist/surface.js +1 -1
- package/dist/utils/addon-responses.d.ts +1 -0
- package/dist/utils/asr-events.d.ts +1 -0
- package/dist/utils/asr-events.js +2 -1
- package/dist/utils/cache/types.d.ts +7 -0
- package/dist/utils/mcp-adapter.js +12 -8
- package/dist/utils/tool-helpers.d.ts +1 -1
- package/dist/utils/tool-helpers.js +20 -52
- package/dist/utils/tools/parsers/dsml.js +8 -5
- package/dist/utils/tools/parsers/qwen35.js +7 -4
- package/dist/utils/tools/shared.d.ts +3 -0
- package/dist/utils/tools/shared.js +62 -0
- package/dist/utils/transcribe-metadata.d.ts +8 -1
- package/dist/utils/transcribe-metadata.js +13 -3
- package/dist/utils/tts-stats.d.ts +1 -0
- package/package.json +41 -32
- package/dist/resources/model-fit/calibration/darwin-arm64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/darwin-arm64.js +0 -36
- package/dist/resources/model-fit/calibration/darwin-x64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/darwin-x64.js +0 -38
- package/dist/resources/model-fit/calibration/fit.d.ts +0 -73
- package/dist/resources/model-fit/calibration/fit.js +0 -152
- package/dist/resources/model-fit/calibration/index.d.ts +0 -37
- package/dist/resources/model-fit/calibration/index.js +0 -75
- package/dist/resources/model-fit/calibration/linux-arm64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/linux-arm64.js +0 -38
- package/dist/resources/model-fit/calibration/linux-x64-vulkan.d.ts +0 -8
- package/dist/resources/model-fit/calibration/linux-x64-vulkan.js +0 -39
- package/dist/resources/model-fit/calibration/linux-x64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/linux-x64.js +0 -38
- package/dist/resources/model-fit/calibration/win32-x64-vulkan-shared.d.ts +0 -8
- package/dist/resources/model-fit/calibration/win32-x64-vulkan-shared.js +0 -39
- package/dist/resources/model-fit/calibration/win32-x64-vulkan.d.ts +0 -8
- package/dist/resources/model-fit/calibration/win32-x64-vulkan.js +0 -36
- package/dist/resources/model-fit/calibration/win32-x64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/win32-x64.js +0 -38
- package/dist/resources/model-fit/estimators/llm.d.ts +0 -46
- package/dist/resources/model-fit/estimators/llm.js +0 -222
- package/dist/resources/model-fit/estimators/whisper.d.ts +0 -10
- package/dist/resources/model-fit/estimators/whisper.js +0 -76
|
@@ -10,6 +10,7 @@ import { attachModelExecutionMs } from '../../../profiling/model-execution.js';
|
|
|
10
10
|
// shapes cannot drift apart.
|
|
11
11
|
function ttsFrameMetadata(value) {
|
|
12
12
|
return {
|
|
13
|
+
...(value.text !== undefined ? { text: value.text } : {}),
|
|
13
14
|
...(value.sampleRate !== undefined ? { sampleRate: value.sampleRate } : {}),
|
|
14
15
|
...(value.chunkIndex !== undefined ? { chunkIndex: value.chunkIndex } : {}),
|
|
15
16
|
...(typeof value.sentenceChunk === 'string' && value.sentenceChunk.length > 0
|
|
@@ -457,6 +458,32 @@ function createMossModel(modelId, config, params, artifacts) {
|
|
|
457
458
|
registerAddonLogger(modelId, ModelType.ttsGgml, logger);
|
|
458
459
|
return { model };
|
|
459
460
|
}
|
|
461
|
+
function createMossBatchModel(params, config) {
|
|
462
|
+
if (!params.modelPath)
|
|
463
|
+
throw new TtsArtifactsRequiredError();
|
|
464
|
+
const codec = params.artifacts?.['mossSpeechCodecPath'];
|
|
465
|
+
if (config.ttsEngine === 'moss-speech' && !codec)
|
|
466
|
+
throw new TtsArtifactsRequiredError();
|
|
467
|
+
const logger = createStreamLogger(params.modelId, ModelType.ttsGgml);
|
|
468
|
+
const model = new TTSGgml({
|
|
469
|
+
engine: config.ttsEngine,
|
|
470
|
+
files: config.ttsEngine === 'moss-sfx'
|
|
471
|
+
? { mossSoundEffect: params.modelPath }
|
|
472
|
+
: { mossSpeechModel: params.modelPath, mossSpeechCodec: codec },
|
|
473
|
+
...(config.threads !== undefined ? { threads: config.threads } : {}),
|
|
474
|
+
...(config.nGpuLayers !== undefined ? { nGpuLayers: config.nGpuLayers } : {}),
|
|
475
|
+
...(config.seed !== undefined ? { seed: config.seed } : {}),
|
|
476
|
+
config: {
|
|
477
|
+
...(config.useGPU !== undefined ? { useGPU: config.useGPU } : {}),
|
|
478
|
+
...ggmlBackendConfig(config)
|
|
479
|
+
},
|
|
480
|
+
logger,
|
|
481
|
+
opts: { stats: true },
|
|
482
|
+
exclusiveRun: true
|
|
483
|
+
});
|
|
484
|
+
registerAddonLogger(params.modelId, ModelType.ttsGgml, logger);
|
|
485
|
+
return { model };
|
|
486
|
+
}
|
|
460
487
|
export const ttsPlugin = definePlugin({
|
|
461
488
|
modelType: ModelType.ttsGgml,
|
|
462
489
|
displayName: 'TTS (GGML)',
|
|
@@ -466,6 +493,13 @@ export const ttsPlugin = definePlugin({
|
|
|
466
493
|
async resolveConfig(cfg, ctx) {
|
|
467
494
|
const { ttsEngine } = cfg;
|
|
468
495
|
// Same default as the former onnx-tts plugin: omitting `ttsEngine` → Chatterbox.
|
|
496
|
+
if (ttsEngine === 'moss-sfx')
|
|
497
|
+
return { config: cfg };
|
|
498
|
+
if (ttsEngine === 'moss-speech') {
|
|
499
|
+
const { mossSpeechCodecModelSrc, ...runtime } = cfg;
|
|
500
|
+
const mossSpeechCodecPath = await ctx.resolveModelPath(mossSpeechCodecModelSrc);
|
|
501
|
+
return { config: runtime, artifacts: { mossSpeechCodecPath } };
|
|
502
|
+
}
|
|
469
503
|
if (ttsEngine === 'parler') {
|
|
470
504
|
return resolveParlerConfig(cfg, ctx);
|
|
471
505
|
}
|
|
@@ -486,6 +520,9 @@ export const ttsPlugin = definePlugin({
|
|
|
486
520
|
createModel(params) {
|
|
487
521
|
const config = (params.modelConfig ?? {});
|
|
488
522
|
const artifacts = params.artifacts ?? {};
|
|
523
|
+
if (config.ttsEngine === 'moss-sfx' || config.ttsEngine === 'moss-speech') {
|
|
524
|
+
return createMossBatchModel(params, config);
|
|
525
|
+
}
|
|
489
526
|
if (config.ttsEngine === 'parler') {
|
|
490
527
|
return createParlerModel(params.modelId, config, params, artifacts);
|
|
491
528
|
}
|
|
@@ -571,7 +608,7 @@ export const ttsPlugin = definePlugin({
|
|
|
571
608
|
})
|
|
572
609
|
},
|
|
573
610
|
logging: {
|
|
574
|
-
module: () => import('@qvac/tts-ggml/addonLogging'),
|
|
611
|
+
module: () => import('@qvac/tts-ggml/addonLogging.js'),
|
|
575
612
|
namespace: ModelType.ttsGgml
|
|
576
613
|
}
|
|
577
614
|
});
|
|
@@ -53,7 +53,7 @@ export declare const whisperPlugin: {
|
|
|
53
53
|
max_speech_duration_s: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
54
54
|
speech_pad_ms: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
55
55
|
samples_overlap: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
56
|
-
}, import("zod/v4/core").$strip>>;
|
|
56
|
+
}, import("zod/v4/core/index.js").$strip>>;
|
|
57
57
|
audio_format: import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
58
58
|
f32le: "f32le";
|
|
59
59
|
s16le: "s16le";
|
|
@@ -63,11 +63,11 @@ export declare const whisperPlugin: {
|
|
|
63
63
|
use_gpu: import("zod").ZodOptional<import("zod").ZodBoolean>;
|
|
64
64
|
flash_attn: import("zod").ZodOptional<import("zod").ZodBoolean>;
|
|
65
65
|
gpu_device: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
66
|
-
}, import("zod/v4/core").$strip>>;
|
|
66
|
+
}, import("zod/v4/core/index.js").$strip>>;
|
|
67
67
|
miscConfig: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
68
68
|
caption_enabled: import("zod").ZodOptional<import("zod").ZodBoolean>;
|
|
69
69
|
seed: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
70
|
-
}, import("zod/v4/core").$strip>>;
|
|
70
|
+
}, import("zod/v4/core/index.js").$strip>>;
|
|
71
71
|
vadModelSrc: import("zod").ZodOptional<import("zod").ZodUnion<readonly [import("zod").ZodString, import("zod").ZodObject<{
|
|
72
72
|
src: import("zod").ZodString;
|
|
73
73
|
name: import("zod").ZodOptional<import("zod").ZodString>;
|
|
@@ -90,10 +90,12 @@ export declare const whisperPlugin: {
|
|
|
90
90
|
"whispercpp-transcription": "whispercpp-transcription";
|
|
91
91
|
"bci-whispercpp-transcription": "bci-whispercpp-transcription";
|
|
92
92
|
"llamacpp-embedding": "llamacpp-embedding";
|
|
93
|
+
"llamacpp-decision": "llamacpp-decision";
|
|
93
94
|
"nmtcpp-translation": "nmtcpp-translation";
|
|
94
95
|
"onnx-tts": "onnx-tts";
|
|
95
96
|
"tts-ggml": "tts-ggml";
|
|
96
97
|
"parakeet-transcription": "parakeet-transcription";
|
|
98
|
+
"moss-transcribe": "moss-transcribe";
|
|
97
99
|
"ggml-ocr": "ggml-ocr";
|
|
98
100
|
"sdcpp-generation": "sdcpp-generation";
|
|
99
101
|
"audiogen-ggml": "audiogen-ggml";
|
|
@@ -106,8 +108,8 @@ export declare const whisperPlugin: {
|
|
|
106
108
|
parakeet: "parakeet";
|
|
107
109
|
classification: "classification";
|
|
108
110
|
}>, import("zod").ZodLiteral<"vad">]>>;
|
|
109
|
-
}, import("zod/v4/core").$strip>]>>;
|
|
110
|
-
}, import("zod/v4/core").$strip>;
|
|
111
|
+
}, import("zod/v4/core/index.js").$strip>]>>;
|
|
112
|
+
}, import("zod/v4/core/index.js").$strip>;
|
|
111
113
|
assessFit: typeof import("node_modules/@qvac/asr-ggml/lib/fit").assessFit;
|
|
112
114
|
resolveConfig(cfg: WhisperConfig, ctx: ResolveContext): Promise<{
|
|
113
115
|
config: {
|
|
@@ -245,13 +247,15 @@ export declare const whisperPlugin: {
|
|
|
245
247
|
audioChunk: import("zod").ZodDiscriminatedUnion<[import("zod").ZodObject<{
|
|
246
248
|
type: import("zod").ZodLiteral<"base64">;
|
|
247
249
|
value: import("zod").ZodString;
|
|
248
|
-
}, import("zod/v4/core").$strip>, import("zod").ZodObject<{
|
|
250
|
+
}, import("zod/v4/core/index.js").$strip>, import("zod").ZodObject<{
|
|
249
251
|
type: import("zod").ZodLiteral<"filePath">;
|
|
250
252
|
value: import("zod").ZodString;
|
|
251
|
-
}, import("zod/v4/core").$strip>], "type">;
|
|
253
|
+
}, import("zod/v4/core/index.js").$strip>], "type">;
|
|
254
|
+
hotwords: import("zod").ZodOptional<import("zod").ZodArray<import("zod").ZodString>>;
|
|
255
|
+
maxNewTokens: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
252
256
|
type: import("zod").ZodLiteral<"transcribe">;
|
|
253
257
|
requestId: import("zod").ZodOptional<import("zod").ZodString>;
|
|
254
|
-
}, import("zod/v4/core").$strip>, import("zod").ZodObject<{
|
|
258
|
+
}, import("zod/v4/core/index.js").$strip>, import("zod").ZodObject<{
|
|
255
259
|
text: import("zod").ZodOptional<import("zod").ZodString>;
|
|
256
260
|
done: import("zod").ZodOptional<import("zod").ZodBoolean>;
|
|
257
261
|
stats: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
@@ -282,7 +286,7 @@ export declare const whisperPlugin: {
|
|
|
282
286
|
totalEncodedFrames: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
283
287
|
encoderOnCoreml: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
284
288
|
encoderUsedCoreml: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
285
|
-
}, import("zod/v4/core").$strip>>;
|
|
289
|
+
}, import("zod/v4/core/index.js").$strip>>;
|
|
286
290
|
error: import("zod").ZodOptional<import("zod").ZodString>;
|
|
287
291
|
segment: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
288
292
|
text: import("zod").ZodString;
|
|
@@ -290,10 +294,18 @@ export declare const whisperPlugin: {
|
|
|
290
294
|
endMs: import("zod").ZodNumber;
|
|
291
295
|
append: import("zod").ZodBoolean;
|
|
292
296
|
id: import("zod").ZodNumber;
|
|
297
|
+
speakerId: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
298
|
+
speaker: import("zod").ZodOptional<import("zod").ZodString>;
|
|
299
|
+
speakerSegments: import("zod").ZodOptional<import("zod").ZodArray<import("zod").ZodObject<{
|
|
300
|
+
speakerId: import("zod").ZodNumber;
|
|
301
|
+
startMs: import("zod").ZodNumber;
|
|
302
|
+
endMs: import("zod").ZodNumber;
|
|
303
|
+
}, import("zod/v4/core/index.js").$strip>>>;
|
|
293
304
|
isEndOfTurn: import("zod").ZodOptional<import("zod").ZodBoolean>;
|
|
294
305
|
startsWord: import("zod").ZodOptional<import("zod").ZodBoolean>;
|
|
295
|
-
}, import("zod/v4/core").$strip>>;
|
|
306
|
+
}, import("zod/v4/core/index.js").$strip>>;
|
|
296
307
|
vad: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
308
|
+
speakerId: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
297
309
|
speaking: import("zod").ZodBoolean;
|
|
298
310
|
probability: import("zod").ZodNumber;
|
|
299
311
|
source: import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
@@ -301,13 +313,13 @@ export declare const whisperPlugin: {
|
|
|
301
313
|
energy: "energy";
|
|
302
314
|
sortformer: "sortformer";
|
|
303
315
|
}>>;
|
|
304
|
-
}, import("zod/v4/core").$strip>>;
|
|
316
|
+
}, import("zod/v4/core/index.js").$strip>>;
|
|
305
317
|
endOfTurn: import("zod").ZodOptional<import("zod").ZodPreprocess<import("zod").ZodDiscriminatedUnion<[import("zod").ZodObject<{
|
|
306
318
|
source: import("zod").ZodLiteral<"whisper">;
|
|
307
319
|
silenceDurationMs: import("zod").ZodNumber;
|
|
308
|
-
}, import("zod/v4/core").$strip>, import("zod").ZodObject<{
|
|
320
|
+
}, import("zod/v4/core/index.js").$strip>, import("zod").ZodObject<{
|
|
309
321
|
source: import("zod").ZodLiteral<"parakeet">;
|
|
310
|
-
}, import("zod/v4/core").$strip>], "source">, unknown>>;
|
|
322
|
+
}, import("zod/v4/core/index.js").$strip>], "source">, unknown>>;
|
|
311
323
|
diagnostics: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
312
324
|
selectedBackend: import("zod").ZodString;
|
|
313
325
|
selectedDevice: import("zod").ZodEnum<{
|
|
@@ -315,21 +327,21 @@ export declare const whisperPlugin: {
|
|
|
315
327
|
gpu: "gpu";
|
|
316
328
|
}>;
|
|
317
329
|
graphicsApi: import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
330
|
+
cuda: "cuda";
|
|
318
331
|
vulkan: "vulkan";
|
|
319
332
|
metal: "metal";
|
|
320
333
|
opencl: "opencl";
|
|
321
|
-
|
|
334
|
+
rocm: "rocm";
|
|
322
335
|
opengl: "opengl";
|
|
323
336
|
webgpu: "webgpu";
|
|
324
337
|
direct3d11: "direct3d11";
|
|
325
338
|
direct3d12: "direct3d12";
|
|
326
339
|
levelZero: "levelZero";
|
|
327
|
-
rocm: "rocm";
|
|
328
340
|
}>>;
|
|
329
341
|
driver: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
330
342
|
name: import("zod").ZodString;
|
|
331
343
|
version: import("zod").ZodOptional<import("zod").ZodString>;
|
|
332
|
-
}, import("zod/v4/core").$strip>>;
|
|
344
|
+
}, import("zod/v4/core/index.js").$strip>>;
|
|
333
345
|
gpuId: import("zod").ZodOptional<import("zod").ZodString>;
|
|
334
346
|
fallback: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
335
347
|
requestedBackend: import("zod").ZodOptional<import("zod").ZodString>;
|
|
@@ -338,7 +350,7 @@ export declare const whisperPlugin: {
|
|
|
338
350
|
gpu: "gpu";
|
|
339
351
|
}>>;
|
|
340
352
|
reason: import("zod").ZodString;
|
|
341
|
-
}, import("zod/v4/core").$strip>>;
|
|
353
|
+
}, import("zod/v4/core/index.js").$strip>>;
|
|
342
354
|
probe: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
343
355
|
status: import("zod").ZodEnum<{
|
|
344
356
|
unknown: "unknown";
|
|
@@ -347,10 +359,10 @@ export declare const whisperPlugin: {
|
|
|
347
359
|
}>;
|
|
348
360
|
backend: import("zod").ZodString;
|
|
349
361
|
reason: import("zod").ZodOptional<import("zod").ZodString>;
|
|
350
|
-
}, import("zod/v4/core").$strip>>;
|
|
351
|
-
}, import("zod/v4/core").$strip>>;
|
|
362
|
+
}, import("zod/v4/core/index.js").$strip>>;
|
|
363
|
+
}, import("zod/v4/core/index.js").$strip>>;
|
|
352
364
|
type: import("zod").ZodLiteral<"transcribe">;
|
|
353
|
-
}, import("zod/v4/core").$strip>>;
|
|
365
|
+
}, import("zod/v4/core/index.js").$strip>>;
|
|
354
366
|
transcribeStream: import("../../../schemas/index.js").PluginHandlerDefinition<import("zod").ZodObject<{
|
|
355
367
|
modelId: import("zod").ZodString;
|
|
356
368
|
prompt: import("zod").ZodOptional<import("zod").ZodString>;
|
|
@@ -366,15 +378,18 @@ export declare const whisperPlugin: {
|
|
|
366
378
|
rightLookaheadMs: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
367
379
|
emitPartials: import("zod").ZodOptional<import("zod").ZodBoolean>;
|
|
368
380
|
emitEnergyVad: import("zod").ZodOptional<import("zod").ZodBoolean>;
|
|
381
|
+
emitSpeakerVad: import("zod").ZodOptional<import("zod").ZodBoolean>;
|
|
382
|
+
diarizationThreshold: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
383
|
+
diarizationMinSegmentMs: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
369
384
|
spkCacheEnable: import("zod").ZodOptional<import("zod").ZodBoolean>;
|
|
370
385
|
spkCacheLen: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
371
386
|
fifoLen: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
372
387
|
chunkLeftContextMs: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
373
388
|
chunkRightContextMs: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
374
389
|
spkCacheUpdatePeriod: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
375
|
-
}, import("zod/v4/core").$strip>>;
|
|
390
|
+
}, import("zod/v4/core/index.js").$strip>>;
|
|
376
391
|
requestId: import("zod").ZodOptional<import("zod").ZodString>;
|
|
377
|
-
}, import("zod/v4/core").$strip>, import("zod").ZodObject<{
|
|
392
|
+
}, import("zod/v4/core/index.js").$strip>, import("zod").ZodObject<{
|
|
378
393
|
text: import("zod").ZodOptional<import("zod").ZodString>;
|
|
379
394
|
done: import("zod").ZodOptional<import("zod").ZodBoolean>;
|
|
380
395
|
stats: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
@@ -405,7 +420,7 @@ export declare const whisperPlugin: {
|
|
|
405
420
|
totalEncodedFrames: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
406
421
|
encoderOnCoreml: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
407
422
|
encoderUsedCoreml: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
408
|
-
}, import("zod/v4/core").$strip>>;
|
|
423
|
+
}, import("zod/v4/core/index.js").$strip>>;
|
|
409
424
|
error: import("zod").ZodOptional<import("zod").ZodString>;
|
|
410
425
|
segment: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
411
426
|
text: import("zod").ZodString;
|
|
@@ -413,10 +428,18 @@ export declare const whisperPlugin: {
|
|
|
413
428
|
endMs: import("zod").ZodNumber;
|
|
414
429
|
append: import("zod").ZodBoolean;
|
|
415
430
|
id: import("zod").ZodNumber;
|
|
431
|
+
speakerId: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
432
|
+
speaker: import("zod").ZodOptional<import("zod").ZodString>;
|
|
433
|
+
speakerSegments: import("zod").ZodOptional<import("zod").ZodArray<import("zod").ZodObject<{
|
|
434
|
+
speakerId: import("zod").ZodNumber;
|
|
435
|
+
startMs: import("zod").ZodNumber;
|
|
436
|
+
endMs: import("zod").ZodNumber;
|
|
437
|
+
}, import("zod/v4/core/index.js").$strip>>>;
|
|
416
438
|
isEndOfTurn: import("zod").ZodOptional<import("zod").ZodBoolean>;
|
|
417
439
|
startsWord: import("zod").ZodOptional<import("zod").ZodBoolean>;
|
|
418
|
-
}, import("zod/v4/core").$strip>>;
|
|
440
|
+
}, import("zod/v4/core/index.js").$strip>>;
|
|
419
441
|
vad: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
442
|
+
speakerId: import("zod").ZodOptional<import("zod").ZodNumber>;
|
|
420
443
|
speaking: import("zod").ZodBoolean;
|
|
421
444
|
probability: import("zod").ZodNumber;
|
|
422
445
|
source: import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
@@ -424,13 +447,13 @@ export declare const whisperPlugin: {
|
|
|
424
447
|
energy: "energy";
|
|
425
448
|
sortformer: "sortformer";
|
|
426
449
|
}>>;
|
|
427
|
-
}, import("zod/v4/core").$strip>>;
|
|
450
|
+
}, import("zod/v4/core/index.js").$strip>>;
|
|
428
451
|
endOfTurn: import("zod").ZodOptional<import("zod").ZodPreprocess<import("zod").ZodDiscriminatedUnion<[import("zod").ZodObject<{
|
|
429
452
|
source: import("zod").ZodLiteral<"whisper">;
|
|
430
453
|
silenceDurationMs: import("zod").ZodNumber;
|
|
431
|
-
}, import("zod/v4/core").$strip>, import("zod").ZodObject<{
|
|
454
|
+
}, import("zod/v4/core/index.js").$strip>, import("zod").ZodObject<{
|
|
432
455
|
source: import("zod").ZodLiteral<"parakeet">;
|
|
433
|
-
}, import("zod/v4/core").$strip>], "source">, unknown>>;
|
|
456
|
+
}, import("zod/v4/core/index.js").$strip>], "source">, unknown>>;
|
|
434
457
|
diagnostics: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
435
458
|
selectedBackend: import("zod").ZodString;
|
|
436
459
|
selectedDevice: import("zod").ZodEnum<{
|
|
@@ -438,21 +461,21 @@ export declare const whisperPlugin: {
|
|
|
438
461
|
gpu: "gpu";
|
|
439
462
|
}>;
|
|
440
463
|
graphicsApi: import("zod").ZodOptional<import("zod").ZodEnum<{
|
|
464
|
+
cuda: "cuda";
|
|
441
465
|
vulkan: "vulkan";
|
|
442
466
|
metal: "metal";
|
|
443
467
|
opencl: "opencl";
|
|
444
|
-
|
|
468
|
+
rocm: "rocm";
|
|
445
469
|
opengl: "opengl";
|
|
446
470
|
webgpu: "webgpu";
|
|
447
471
|
direct3d11: "direct3d11";
|
|
448
472
|
direct3d12: "direct3d12";
|
|
449
473
|
levelZero: "levelZero";
|
|
450
|
-
rocm: "rocm";
|
|
451
474
|
}>>;
|
|
452
475
|
driver: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
453
476
|
name: import("zod").ZodString;
|
|
454
477
|
version: import("zod").ZodOptional<import("zod").ZodString>;
|
|
455
|
-
}, import("zod/v4/core").$strip>>;
|
|
478
|
+
}, import("zod/v4/core/index.js").$strip>>;
|
|
456
479
|
gpuId: import("zod").ZodOptional<import("zod").ZodString>;
|
|
457
480
|
fallback: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
458
481
|
requestedBackend: import("zod").ZodOptional<import("zod").ZodString>;
|
|
@@ -461,7 +484,7 @@ export declare const whisperPlugin: {
|
|
|
461
484
|
gpu: "gpu";
|
|
462
485
|
}>>;
|
|
463
486
|
reason: import("zod").ZodString;
|
|
464
|
-
}, import("zod/v4/core").$strip>>;
|
|
487
|
+
}, import("zod/v4/core/index.js").$strip>>;
|
|
465
488
|
probe: import("zod").ZodOptional<import("zod").ZodObject<{
|
|
466
489
|
status: import("zod").ZodEnum<{
|
|
467
490
|
unknown: "unknown";
|
|
@@ -470,13 +493,13 @@ export declare const whisperPlugin: {
|
|
|
470
493
|
}>;
|
|
471
494
|
backend: import("zod").ZodString;
|
|
472
495
|
reason: import("zod").ZodOptional<import("zod").ZodString>;
|
|
473
|
-
}, import("zod/v4/core").$strip>>;
|
|
474
|
-
}, import("zod/v4/core").$strip>>;
|
|
496
|
+
}, import("zod/v4/core/index.js").$strip>>;
|
|
497
|
+
}, import("zod/v4/core/index.js").$strip>>;
|
|
475
498
|
type: import("zod").ZodLiteral<"transcribeStream">;
|
|
476
|
-
}, import("zod/v4/core").$strip>>;
|
|
499
|
+
}, import("zod/v4/core/index.js").$strip>>;
|
|
477
500
|
};
|
|
478
501
|
logging: {
|
|
479
|
-
module: () => Promise<typeof import("@qvac/asr-ggml/addonLogging")>;
|
|
502
|
+
module: () => Promise<typeof import("@qvac/asr-ggml/addonLogging.js")>;
|
|
480
503
|
namespace: "@qvac/asr-ggml";
|
|
481
504
|
};
|
|
482
505
|
};
|
|
@@ -55,12 +55,16 @@ export const whisperPlugin = definePlugin({
|
|
|
55
55
|
modelId: request.modelId,
|
|
56
56
|
audioChunk: request.audioChunk,
|
|
57
57
|
prompt: request.prompt,
|
|
58
|
+
hotwords: request.hotwords,
|
|
59
|
+
maxNewTokens: request.maxNewTokens,
|
|
58
60
|
metadata: true
|
|
59
61
|
}, request.requestId)
|
|
60
62
|
: transcribe({
|
|
61
63
|
modelId: request.modelId,
|
|
62
64
|
audioChunk: request.audioChunk,
|
|
63
|
-
prompt: request.prompt
|
|
65
|
+
prompt: request.prompt,
|
|
66
|
+
hotwords: request.hotwords,
|
|
67
|
+
maxNewTokens: request.maxNewTokens
|
|
64
68
|
}, request.requestId);
|
|
65
69
|
try {
|
|
66
70
|
let result = await stream.next();
|
|
@@ -182,7 +186,7 @@ export const whisperPlugin = definePlugin({
|
|
|
182
186
|
})
|
|
183
187
|
},
|
|
184
188
|
logging: {
|
|
185
|
-
module: () => import('@qvac/asr-ggml/addonLogging'),
|
|
189
|
+
module: () => import('@qvac/asr-ggml/addonLogging.js'),
|
|
186
190
|
namespace: ADDON_ASR
|
|
187
191
|
}
|
|
188
192
|
});
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { type CacheMessage } from '../../utils/index.js';
|
|
2
2
|
export declare function extractSystemPrompt(messages: CacheMessage[]): string | null;
|
|
3
|
-
export declare function generateConfigHash(systemPrompt: string | null
|
|
3
|
+
export declare function generateConfigHash(systemPrompt: string | null): string;
|
|
4
4
|
export declare function generateCacheKey(messages: CacheMessage[]): string;
|
|
5
5
|
export declare function getCacheFilePath(modelId: string, configHash: string, cacheKey: string): Promise<string>;
|
|
6
6
|
export declare function findMatchingCache(modelId: string, configHash: string, currentHistory: CacheMessage[]): Promise<{
|
|
@@ -7,36 +7,21 @@ import { PathTraversalError } from '../../errors/index.js';
|
|
|
7
7
|
import { Buffer } from 'bare-buffer';
|
|
8
8
|
import { markAutoCacheKey } from './kv-cache-retention.js';
|
|
9
9
|
const logger = getEngineLogger();
|
|
10
|
-
// In-memory KV-cache state lives in `KvCacheSession
|
|
11
|
-
//
|
|
12
|
-
//
|
|
13
|
-
// in-memory state.
|
|
10
|
+
// In-memory KV-cache state lives in `KvCacheSession`, its single mutation
|
|
11
|
+
// point. This module keeps only the pure path / hash utilities that don't
|
|
12
|
+
// touch in-memory state.
|
|
14
13
|
export function extractSystemPrompt(messages) {
|
|
15
14
|
const systemMessage = messages.find((msg) => msg.role === 'system');
|
|
16
15
|
return systemMessage ? systemMessage.content : null;
|
|
17
16
|
}
|
|
18
|
-
//
|
|
19
|
-
//
|
|
20
|
-
//
|
|
21
|
-
//
|
|
22
|
-
//
|
|
23
|
-
|
|
24
|
-
// that is the order sent to the model.
|
|
25
|
-
function canonicalizeHashInput(value) {
|
|
26
|
-
if (Array.isArray(value))
|
|
27
|
-
return value.map(canonicalizeHashInput);
|
|
28
|
-
if (typeof value !== 'object' || value === null)
|
|
29
|
-
return value;
|
|
30
|
-
const canonical = {};
|
|
31
|
-
for (const key of Object.keys(value).sort()) {
|
|
32
|
-
canonical[key] = canonicalizeHashInput(value[key]);
|
|
33
|
-
}
|
|
34
|
-
return canonical;
|
|
35
|
-
}
|
|
36
|
-
export function generateConfigHash(systemPrompt, tools) {
|
|
17
|
+
// Names the cache file by system prompt only. Tools stay out on purpose: the
|
|
18
|
+
// addon compares the rendered prompt against the tokens the file holds, so a
|
|
19
|
+
// changed tool set on the same key must reach the same file to be trimmed there.
|
|
20
|
+
// The empty `tools` keeps the path earlier releases gave a tool-less cache, so
|
|
21
|
+
// an upgrade overwrites those files instead of orphaning them.
|
|
22
|
+
export function generateConfigHash(systemPrompt) {
|
|
37
23
|
const hash = crypto.createHash('sha-256');
|
|
38
|
-
|
|
39
|
-
hash.update(Buffer.from(canonicalConfig, 'utf8'));
|
|
24
|
+
hash.update(Buffer.from(JSON.stringify({ systemPrompt, tools: [] }), 'utf8'));
|
|
40
25
|
return hash.digest('hex').substring(0, 16);
|
|
41
26
|
}
|
|
42
27
|
export function generateCacheKey(messages) {
|
|
@@ -4,16 +4,18 @@ import { forwardBackendDiagnostics } from '../../profiling/backend-diagnostics.j
|
|
|
4
4
|
import { isModelLoaded, registerModel } from '../../runtime/model-registry.js';
|
|
5
5
|
import { startLogBuffering, stopLogBufferingWithTimeout } from '../../runtime/logging-stream-registry.js';
|
|
6
6
|
import { detectShardedModel, generateShardFilenames, validateShardedModelCache } from '../../utils/index.js';
|
|
7
|
-
import { PluginNotFoundError, ModelFileNotFoundError, ModelFileNotFoundInDirError, ModelFileLocateFailedError } from '../../errors/index.js';
|
|
7
|
+
import { PluginNotFoundError, ModelFileNotFoundError, ModelFileNotFoundInDirError, ModelFileLocateFailedError, ModelFitRefusedError } from '../../errors/index.js';
|
|
8
8
|
import { ensureAddonLoggerReady, getPlugin } from '../index.js';
|
|
9
9
|
import { runAdvisoryFitCheck } from '../../resources/model-fit/native-probe/advisory-fit.js';
|
|
10
|
+
import { refusesLoad, resolveModelFitPolicy, runsProbe } from '../../schemas/model-fit-policy.js';
|
|
11
|
+
import { getConfig } from '../../runtime/state.js';
|
|
10
12
|
import { promises as fsPromises } from 'bare-fs';
|
|
11
13
|
import path from 'bare-path';
|
|
12
14
|
import { getEngineLogger } from '../../logging/index.js';
|
|
13
15
|
const logger = getEngineLogger();
|
|
14
16
|
export async function loadModel(params, options) {
|
|
15
17
|
const { modelId, modelPath, options: modelOptions, artifacts, modelName } = loadModelServerParamsSchema.parse(params);
|
|
16
|
-
const { modelConfig, modelType: rawModelType } = modelOptions;
|
|
18
|
+
const { modelConfig, modelType: rawModelType, modelFitPolicy } = modelOptions;
|
|
17
19
|
// Normalize modelType to canonical form (handles aliases and custom types)
|
|
18
20
|
const modelType = normalizeModelType(rawModelType);
|
|
19
21
|
// Check if model is already loaded
|
|
@@ -53,20 +55,23 @@ export async function loadModel(params, options) {
|
|
|
53
55
|
throw new ModelFileLocateFailedError(modelType, modelPath, error);
|
|
54
56
|
}
|
|
55
57
|
}
|
|
56
|
-
//
|
|
57
|
-
//
|
|
58
|
-
//
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
58
|
+
// Runs after config resolution and path validation so it sees the same state
|
|
59
|
+
// the real load uses, and before `createModel()` so it never competes with
|
|
60
|
+
// the native load for device memory.
|
|
61
|
+
const fitPolicy = resolveModelFitPolicy(modelFitPolicy, getConfig().modelFitPolicy);
|
|
62
|
+
const fitProbe = runsProbe(fitPolicy)
|
|
63
|
+
? await runAdvisoryFitCheck({
|
|
64
|
+
modelId,
|
|
65
|
+
modelType: modelType,
|
|
66
|
+
modelPath,
|
|
67
|
+
modelConfig,
|
|
68
|
+
artifacts,
|
|
69
|
+
isShardedModel
|
|
70
|
+
})
|
|
71
|
+
: undefined;
|
|
72
|
+
if (refusesLoad(fitPolicy, fitProbe?.verdict)) {
|
|
73
|
+
throw new ModelFitRefusedError(modelType, fitProbe.reason, fitProbe.message);
|
|
74
|
+
}
|
|
70
75
|
// Load the addon's logger before the addon itself runs. Plugins that defer
|
|
71
76
|
// their logging module do so to keep registration free of native loads, so
|
|
72
77
|
// this is the first point their addon is genuinely required.
|
|
@@ -191,6 +191,15 @@ export async function* transcribe(params, requestId) {
|
|
|
191
191
|
}), true);
|
|
192
192
|
const requestLogger = withRequestContext(getEngineLogger(), ctx);
|
|
193
193
|
const engineType = getEngineModelType(modelId);
|
|
194
|
+
const hasMossOptions = params.hotwords !== undefined || params.maxNewTokens !== undefined;
|
|
195
|
+
if (engineType !== ModelType.mossTranscribe && hasMossOptions) {
|
|
196
|
+
throw new TranscriptionFailedError('hotwords and maxNewTokens require a MOSS-Transcribe-Diarize model');
|
|
197
|
+
}
|
|
198
|
+
if (engineType === ModelType.mossTranscribe &&
|
|
199
|
+
params.prompt !== undefined &&
|
|
200
|
+
params.hotwords?.length) {
|
|
201
|
+
throw new TranscriptionFailedError('MOSS prompt cannot be combined with hotwords');
|
|
202
|
+
}
|
|
194
203
|
assertMetadataSupported(modelId, engineType, metadata);
|
|
195
204
|
const silenceMarker = SILENCE_MARKERS[engineType] ?? '';
|
|
196
205
|
const audioFormat = getAudioFormat(modelId, engineType);
|
|
@@ -224,7 +233,14 @@ export async function* transcribe(params, requestId) {
|
|
|
224
233
|
});
|
|
225
234
|
const audioStream = await createAudioStream(params.audioChunk, audioFormat);
|
|
226
235
|
const modelStart = nowMs();
|
|
227
|
-
const
|
|
236
|
+
const runOptions = engineType === ModelType.mossTranscribe
|
|
237
|
+
? {
|
|
238
|
+
...(params.prompt !== undefined && { prompt: params.prompt }),
|
|
239
|
+
...(params.hotwords !== undefined && { hotwords: params.hotwords }),
|
|
240
|
+
...(params.maxNewTokens !== undefined && { maxNewTokens: params.maxNewTokens })
|
|
241
|
+
}
|
|
242
|
+
: undefined;
|
|
243
|
+
const response = (await model.run(audioStream, runOptions));
|
|
228
244
|
for await (const output of response.iterate()) {
|
|
229
245
|
if (ctx.signal.aborted)
|
|
230
246
|
break;
|
package/dist/registry.js
CHANGED
|
@@ -56,6 +56,7 @@ export const registry = {
|
|
|
56
56
|
handler: handleUnloadModel
|
|
57
57
|
},
|
|
58
58
|
embed: { type: 'reply', pluginOp: true, handler: pluginReply('embed') },
|
|
59
|
+
decide: { type: 'reply', pluginOp: true, handler: pluginReply('decide') },
|
|
59
60
|
cancel: {
|
|
60
61
|
type: 'reply',
|
|
61
62
|
handler: cancelHandler
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { AssessModelFitResult, ModelFitBasis, ModelFitEstimateTarget, ModelFitExecution, NativeProbeFit } from '../../schemas/assess-model-fit.js';
|
|
2
2
|
import type { SystemResources } from '../../schemas/system-resources.js';
|
|
3
3
|
import type { ModelResourceProfile } from '../../schemas/model-resource-profile.js';
|
|
4
|
-
import type { ModelFitPlatform
|
|
4
|
+
import type { ModelFitPlatform } from './types.js';
|
|
5
5
|
/** Resolves a checksum to its catalog resource profile. */
|
|
6
6
|
export type ProfileResolver = (sha256Checksum: string) => ModelResourceProfile | undefined;
|
|
7
7
|
export interface AssessModelFitOptions {
|
|
@@ -9,40 +9,32 @@ export interface AssessModelFitOptions {
|
|
|
9
9
|
execution: ModelFitExecution;
|
|
10
10
|
resources: SystemResources;
|
|
11
11
|
/**
|
|
12
|
-
* `undefined` when the runtime's platform/arch pair is not
|
|
13
|
-
*
|
|
12
|
+
* `undefined` when the runtime's platform/arch pair is not one this
|
|
13
|
+
* assessment covers. Also selects the mobile headroom policy.
|
|
14
14
|
*/
|
|
15
15
|
platform: ModelFitPlatform | undefined;
|
|
16
|
-
/** `undefined` when the platform has no validated coefficients. */
|
|
17
|
-
calibration: PlatformCalibration | undefined;
|
|
18
|
-
/**
|
|
19
|
-
* Resolves GPU-resident coefficients once the target backend is known.
|
|
20
|
-
* Defaults to the built-in fixtures; injected in tests.
|
|
21
|
-
*/
|
|
22
|
-
resolveGpuCalibration?: (platform: ModelFitPlatform, backend: string) => PlatformCalibration | undefined;
|
|
23
|
-
/** Integrated-GPU coefficients, spent against the system budget. */
|
|
24
|
-
resolveSharedGpuCalibration?: (platform: ModelFitPlatform, backend: string) => PlatformCalibration | undefined;
|
|
25
16
|
/** Defaults to the generated catalog table; injected in tests. */
|
|
26
17
|
resolveProfile?: ProfileResolver;
|
|
27
18
|
/**
|
|
28
|
-
* The engine's own verdict
|
|
29
|
-
*
|
|
30
|
-
*
|
|
31
|
-
*
|
|
32
|
-
* model against the whole machine, and two such answers cannot be summed into
|
|
33
|
-
* a combined budget the way two byte estimates can.
|
|
19
|
+
* The engine's own verdict per candidate, in `models` order, from the
|
|
20
|
+
* registry's fit stubs. Resolved in the handler, which is where the network
|
|
21
|
+
* is. A candidate whose fitter reached no verdict falls back to the evidence
|
|
22
|
+
* the rest of this assessment can give it.
|
|
34
23
|
*/
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
24
|
+
nativeFits?: readonly NativeCandidateFit[] | undefined;
|
|
25
|
+
}
|
|
26
|
+
/** One candidate's engine verdict, or why it has none. */
|
|
27
|
+
export interface NativeCandidateFit {
|
|
28
|
+
fit?: NativeProbeFit | undefined;
|
|
29
|
+
unavailable?: string | undefined;
|
|
38
30
|
}
|
|
39
31
|
/**
|
|
40
32
|
* Turns catalog profiles plus a fresh system-memory sample into an advisory
|
|
41
33
|
* verdict.
|
|
42
34
|
*
|
|
43
35
|
* Pure: every input is passed in, so the same call is testable without a worker,
|
|
44
|
-
* a device, or a network. Sampling, platform detection and
|
|
45
|
-
* happen in the handler.
|
|
36
|
+
* a device, or a network. Sampling, platform detection and the engine fitter
|
|
37
|
+
* calls happen in the handler.
|
|
46
38
|
*/
|
|
47
39
|
export declare function assessModelFitFromResources(options: AssessModelFitOptions): AssessModelFitResult;
|
|
48
40
|
/**
|