@qvac/inference 0.23.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +60 -28
- package/README.md +3 -1
- package/dist/api/assess-model-fit.d.ts +14 -14
- package/dist/api/assess-model-fit.js +14 -14
- package/dist/api/audio-gen.d.ts +2 -2
- package/dist/api/audio-gen.js +135 -44
- package/dist/api/batch-completion.js +2 -1
- package/dist/api/calc-tokens.d.ts +30 -0
- package/dist/api/calc-tokens.js +41 -0
- package/dist/api/classify.d.ts +5 -2
- package/dist/api/classify.js +5 -2
- package/dist/api/completion-stream.d.ts +1 -1
- package/dist/api/completion-stream.js +2 -1
- package/dist/api/delete-cache.d.ts +7 -1
- package/dist/api/delete-cache.js +11 -1
- package/dist/api/diffusion.d.ts +14 -1
- package/dist/api/diffusion.js +14 -1
- package/dist/api/embed.d.ts +82 -29
- package/dist/api/embed.js +64 -18
- package/dist/api/get-model-info.d.ts +1 -1
- package/dist/api/index.d.ts +2 -1
- package/dist/api/index.js +2 -1
- package/dist/api/rag.d.ts +1 -1
- package/dist/api/rag.js +1 -1
- package/dist/api/registry.d.ts +16 -6
- package/dist/api/registry.js +32 -9
- package/dist/api/text-to-speech.d.ts +14 -5
- package/dist/api/text-to-speech.js +83 -29
- package/dist/api/transcribe.d.ts +6 -2
- package/dist/api/transcribe.js +7 -1
- package/dist/api/vector-index.d.ts +25 -3
- package/dist/api/vector-index.js +17 -2
- package/dist/api/video-request.d.ts +15 -15
- package/dist/dispatch.d.ts +10 -4
- package/dist/dispatch.js +65 -15
- package/dist/engine.d.ts +1 -1
- package/dist/errors/index.d.ts +70 -0
- package/dist/errors/index.js +128 -0
- package/dist/handlers/assess-model-fit.d.ts +3 -21
- package/dist/handlers/assess-model-fit.js +4 -303
- package/dist/handlers/cancelHandler.d.ts +10 -9
- package/dist/handlers/cancelHandler.js +17 -13
- package/dist/handlers/completion-orchestrate.d.ts +14 -2
- package/dist/handlers/completion-orchestrate.js +164 -22
- package/dist/handlers/delete-cache.d.ts +2 -1
- package/dist/handlers/delete-cache.js +6 -1
- package/dist/handlers/download-asset.js +11 -4
- package/dist/handlers/get-loaded-model-info.js +1 -0
- package/dist/handlers/get-model-info.js +71 -2
- package/dist/handlers/load-model/coreml-sidecars.d.ts +2 -13
- package/dist/handlers/load-model/coreml-sidecars.js +16 -119
- package/dist/handlers/load-model/download-lock.d.ts +20 -0
- package/dist/handlers/load-model/download-lock.js +213 -0
- package/dist/handlers/load-model/download-manager.d.ts +10 -2
- package/dist/handlers/load-model/download-manager.js +62 -11
- package/dist/handlers/load-model/handler.js +88 -35
- package/dist/handlers/load-model/http.js +327 -205
- package/dist/handlers/load-model/hyperdrive.d.ts +15 -0
- package/dist/handlers/load-model/hyperdrive.js +123 -50
- package/dist/handlers/load-model/registry-client-options.d.ts +2 -0
- package/dist/handlers/load-model/registry-client-options.js +4 -2
- package/dist/handlers/load-model/registry-companion-set.d.ts +5 -0
- package/dist/handlers/load-model/registry-companion-set.js +19 -1
- package/dist/handlers/load-model/registry-delete.d.ts +5 -0
- package/dist/handlers/load-model/registry-delete.js +269 -0
- package/dist/handlers/load-model/registry-download-utils.d.ts +4 -2
- package/dist/handlers/load-model/registry-download-utils.js +93 -22
- package/dist/handlers/load-model/registry.d.ts +20 -0
- package/dist/handlers/load-model/registry.js +308 -175
- package/dist/handlers/load-model/resolve.js +4 -1
- package/dist/handlers/load-model/types.d.ts +2 -0
- package/dist/handlers/logging-stream.js +17 -5
- package/dist/handlers/plugin-invoke.js +152 -39
- package/dist/handlers/registry.d.ts +2 -2
- package/dist/handlers/registry.js +105 -28
- package/dist/handlers/rpc-server.js +7 -7
- package/dist/handlers/session.d.ts +11 -0
- package/dist/handlers/session.js +40 -0
- package/dist/handlers/unload-model.js +8 -3
- package/dist/handlers/vector-index.js +118 -19
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/logging/utils.js +10 -2
- package/dist/model-cache-layout.d.ts +1 -0
- package/dist/model-cache-layout.js +2 -0
- package/dist/model-fit/assess.d.ts +18 -67
- package/dist/model-fit/assess.js +96 -665
- package/dist/model-fit/describe.d.ts +12 -0
- package/dist/model-fit/describe.js +29 -0
- package/dist/model-fit/description/fetch.d.ts +1 -1
- package/dist/model-fit/description/fetch.js +95 -16
- package/dist/model-fit/description/resolve-load.d.ts +5 -3
- package/dist/model-fit/description/resolve-load.js +14 -14
- package/dist/model-fit/fit-load.d.ts +23 -0
- package/dist/model-fit/fit-load.js +29 -0
- package/dist/model-fit/load-check.d.ts +15 -33
- package/dist/model-fit/load-check.js +50 -195
- package/dist/model-fit/run/engine.d.ts +12 -72
- package/dist/model-fit/run/engine.js +4 -8
- package/dist/model-fit/run/run.d.ts +10 -7
- package/dist/model-fit/run/run.js +22 -18
- package/dist/model-fit/sum.d.ts +32 -0
- package/dist/model-fit/sum.js +60 -0
- package/dist/models/registry/models.d.ts +865 -128
- package/dist/models/registry/models.js +3737 -2963
- package/dist/plugins/builtin/asr-ggml/config.d.ts +19 -4
- package/dist/plugins/builtin/asr-ggml/config.js +39 -3
- package/dist/plugins/builtin/asr-ggml/fit.d.ts +18 -0
- package/dist/{model-fit/request/engines/asr.js → plugins/builtin/asr-ggml/fit.js} +55 -71
- package/dist/plugins/builtin/audiogen-ggml/config.d.ts +6 -0
- package/dist/plugins/builtin/audiogen-ggml/config.js +9 -0
- package/dist/plugins/builtin/audiogen-ggml/fit.d.ts +3 -0
- package/dist/{model-fit/request/engines/audiogen.js → plugins/builtin/audiogen-ggml/fit.js} +28 -18
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +2 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.d.ts +17 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.js +68 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +11 -3
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +47 -11
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +2 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +2 -1
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +143 -18
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +13 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/fit.d.ts +3 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/fit.js +22 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +45 -4
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +8 -6
- package/dist/plugins/builtin/ggml-classification/plugin.d.ts +4 -1
- package/dist/plugins/builtin/ggml-classification/plugin.js +88 -8
- package/dist/plugins/builtin/ggml-image-embedding/ops/embed.d.ts +24 -0
- package/dist/plugins/builtin/ggml-image-embedding/ops/embed.js +184 -0
- package/dist/plugins/builtin/ggml-image-embedding/plugin.d.ts +120 -0
- package/dist/plugins/builtin/ggml-image-embedding/plugin.js +39 -0
- package/dist/plugins/builtin/ggml-ocr/fit.d.ts +6 -0
- package/dist/plugins/builtin/ggml-ocr/fit.js +19 -0
- package/dist/plugins/builtin/ggml-ocr/params.d.ts +1 -1
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +9 -3
- package/dist/plugins/builtin/ggml-ocr/plugin.js +107 -21
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.d.ts +2 -2
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.js +18 -2
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +3 -3
- package/dist/plugins/builtin/index.d.ts +1 -0
- package/dist/plugins/builtin/index.js +1 -0
- package/dist/plugins/builtin/llamacpp-completion/fit.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-completion/fit.js +28 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +1 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +15 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.d.ts +2 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.js +32 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +1 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +17 -3
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +61 -5
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.d.ts +5 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.js +54 -19
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +148 -19
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +26 -9
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.d.ts +3 -1
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.js +5 -2
- package/dist/plugins/builtin/llamacpp-decision/plugin.d.ts +4 -2
- package/dist/plugins/builtin/llamacpp-decision/plugin.js +2 -2
- package/dist/plugins/builtin/llamacpp-embedding/fit.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-embedding/fit.js +23 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +194 -4
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +37 -11
- package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +2 -2
- package/dist/plugins/builtin/llamacpp-embedding/transform.js +23 -2
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.d.ts +3 -1
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.js +5 -2
- package/dist/plugins/builtin/llamacpp-rerank/plugin.d.ts +6 -5
- package/dist/plugins/builtin/llamacpp-rerank/plugin.js +2 -3
- package/dist/plugins/builtin/moss-transcribe/plugin.d.ts +30 -3
- package/dist/plugins/builtin/moss-transcribe/plugin.js +5 -3
- package/dist/plugins/builtin/nmtcpp-translation/fit.d.ts +3 -0
- package/dist/plugins/builtin/nmtcpp-translation/fit.js +11 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +14 -5
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +4 -2
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +205 -9
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +39 -13
- package/dist/plugins/builtin/sdcpp-generation/fit.d.ts +3 -0
- package/dist/plugins/builtin/sdcpp-generation/fit.js +68 -0
- package/dist/plugins/builtin/sdcpp-generation/load-files.d.ts +30 -0
- package/dist/plugins/builtin/sdcpp-generation/load-files.js +121 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +4 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +8 -3
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.d.ts +4 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.js +107 -22
- package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +3 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +4 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/world.d.ts +4 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/world.js +7 -2
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +79 -16
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +29 -117
- package/dist/plugins/builtin/tts-ggml/fit.d.ts +3 -0
- package/dist/plugins/builtin/tts-ggml/fit.js +162 -0
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +4 -1
- package/dist/plugins/builtin/tts-ggml/ops/moss-options.d.ts +1 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +22 -9
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +51 -16
- package/dist/plugins/builtin/tts-ggml/ops/voice-options.d.ts +50 -0
- package/dist/plugins/builtin/tts-ggml/ops/voice-options.js +74 -0
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +368 -25
- package/dist/plugins/builtin/tts-ggml/plugin.js +180 -31
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +70 -7
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +8 -6
- package/dist/plugins/ops/bci-transcribe.d.ts +5 -4
- package/dist/plugins/ops/bci-transcribe.js +18 -4
- package/dist/plugins/ops/cancel.d.ts +1 -0
- package/dist/plugins/ops/cancel.js +11 -2
- package/dist/plugins/ops/embed.d.ts +5 -3
- package/dist/plugins/ops/embed.js +89 -34
- package/dist/plugins/ops/kv-cache-utils.d.ts +1 -0
- package/dist/plugins/ops/kv-cache-utils.js +6 -2
- package/dist/plugins/ops/load-model.d.ts +2 -0
- package/dist/plugins/ops/load-model.js +33 -16
- package/dist/plugins/ops/transcribe.d.ts +5 -4
- package/dist/plugins/ops/transcribe.js +62 -27
- package/dist/plugins/ops/translate.d.ts +2 -1
- package/dist/plugins/ops/translate.js +4 -1
- package/dist/plugins/ops/unload-model.d.ts +17 -1
- package/dist/plugins/ops/unload-model.js +31 -2
- package/dist/plugins/turbovec-backend.d.ts +1 -1
- package/dist/plugins/turbovec-backend.js +16 -1
- package/dist/profiling/operation-metrics.js +22 -19
- package/dist/rag/embedding-scope.d.ts +16 -0
- package/dist/rag/embedding-scope.js +81 -0
- package/dist/rag/handler.d.ts +25 -0
- package/dist/rag/handler.js +50 -16
- package/dist/rag/handlers/ingest.js +2 -6
- package/dist/rag/handlers/search.d.ts +10 -1
- package/dist/rag/handlers/search.js +16 -7
- package/dist/rag/rag-operation-manager.d.ts +27 -3
- package/dist/rag/rag-operation-manager.js +10 -27
- package/dist/registry.js +5 -0
- package/dist/resources/collector.d.ts +70 -36
- package/dist/resources/collector.js +8 -1
- package/dist/resources/types.d.ts +2 -0
- package/dist/rpc/inventory.js +1 -12
- package/dist/runtime/env.d.ts +5 -1
- package/dist/runtime/env.js +6 -21
- package/dist/runtime/index.d.ts +4 -1
- package/dist/runtime/index.js +2 -0
- package/dist/runtime/lifecycle.d.ts +6 -4
- package/dist/runtime/lifecycle.js +9 -5
- package/dist/runtime/model-config-utils.js +5 -1
- package/dist/runtime/model-registry.d.ts +28 -4
- package/dist/runtime/model-registry.js +46 -1
- package/dist/runtime/model-work-lock.d.ts +13 -0
- package/dist/runtime/model-work-lock.js +48 -0
- package/dist/runtime/registry-client.d.ts +47 -1
- package/dist/runtime/registry-client.js +446 -88
- package/dist/runtime/request-context.d.ts +9 -1
- package/dist/runtime/request-context.js +25 -1
- package/dist/runtime/request-registry.d.ts +55 -7
- package/dist/runtime/request-registry.js +189 -71
- package/dist/runtime/runtime-lifecycle.js +10 -1
- package/dist/runtime/session-id.d.ts +18 -0
- package/dist/runtime/session-id.js +20 -0
- package/dist/runtime/session-registry.d.ts +39 -0
- package/dist/runtime/session-registry.js +114 -0
- package/dist/runtime/vector-index-backend.d.ts +4 -0
- package/dist/runtime/vector-index-registry.d.ts +21 -0
- package/dist/runtime/vector-index-registry.js +56 -18
- package/dist/schemas/abort-signal.d.ts +8 -0
- package/dist/schemas/abort-signal.js +1 -0
- package/dist/schemas/assess-model-fit.d.ts +132 -282
- package/dist/schemas/assess-model-fit.js +82 -216
- package/dist/schemas/audio-gen.d.ts +278 -24
- package/dist/schemas/audio-gen.js +127 -23
- package/dist/schemas/batch-completion-stream.d.ts +105 -13
- package/dist/schemas/batch-completion-stream.js +8 -4
- package/dist/schemas/bci-config.d.ts +2 -1
- package/dist/schemas/bci.d.ts +54 -3
- package/dist/schemas/bci.js +11 -16
- package/dist/schemas/calc-tokens.d.ts +141 -0
- package/dist/schemas/calc-tokens.js +53 -0
- package/dist/schemas/cancel.d.ts +26 -0
- package/dist/schemas/cancel.js +10 -2
- package/dist/schemas/classification.d.ts +3 -0
- package/dist/schemas/classification.js +6 -2
- package/dist/schemas/common.d.ts +1831 -348
- package/dist/schemas/common.js +18 -2
- package/dist/schemas/completion-event.d.ts +3 -0
- package/dist/schemas/completion-event.js +1 -1
- package/dist/schemas/completion-stream.d.ts +162 -17
- package/dist/schemas/completion-stream.js +83 -13
- package/dist/schemas/config.d.ts +1212 -169
- package/dist/schemas/config.js +9 -13
- package/dist/schemas/decision.d.ts +4 -2
- package/dist/schemas/decision.js +2 -1
- package/dist/schemas/download-asset.d.ts +10 -5
- package/dist/schemas/download-asset.js +3 -6
- package/dist/schemas/embed.d.ts +286 -3
- package/dist/schemas/embed.js +162 -16
- package/dist/schemas/engine-addon-map.d.ts +1 -0
- package/dist/schemas/engine-addon-map.js +1 -0
- package/dist/schemas/errors.d.ts +12 -0
- package/dist/schemas/errors.js +66 -1
- package/dist/schemas/finetune.d.ts +14 -4
- package/dist/schemas/finetune.js +5 -6
- package/dist/schemas/heartbeat.d.ts +1 -0
- package/dist/schemas/heartbeat.js +5 -2
- package/dist/schemas/image-embedding.d.ts +27 -0
- package/dist/schemas/image-embedding.js +107 -0
- package/dist/schemas/index.d.ts +7 -3
- package/dist/schemas/index.js +7 -3
- package/dist/schemas/lifecycle.d.ts +3 -0
- package/dist/schemas/lifecycle.js +13 -6
- package/dist/schemas/llamacpp-config.d.ts +269 -22
- package/dist/schemas/llamacpp-config.js +86 -7
- package/dist/schemas/load-model.d.ts +6418 -1960
- package/dist/schemas/load-model.js +39 -8
- package/dist/schemas/logging-stream.d.ts +1 -0
- package/dist/schemas/logging-stream.js +5 -2
- package/dist/schemas/model-config-schemas.d.ts +542 -116
- package/dist/schemas/model-config-schemas.js +2 -0
- package/dist/schemas/model-fit-policy.d.ts +2 -2
- package/dist/schemas/model-info.d.ts +54 -56
- package/dist/schemas/model-info.js +17 -6
- package/dist/schemas/model-ops.d.ts +12 -0
- package/dist/schemas/model-ops.js +33 -5
- package/dist/schemas/model-src-utils.d.ts +10 -6
- package/dist/schemas/model-types.d.ts +8 -4
- package/dist/schemas/model-types.js +1 -0
- package/dist/schemas/ocr.d.ts +9 -4
- package/dist/schemas/ocr.js +6 -2
- package/dist/schemas/plugin.d.ts +62 -6
- package/dist/schemas/plugin.js +26 -4
- package/dist/schemas/profiling.d.ts +32 -32
- package/dist/schemas/rag.d.ts +9 -0
- package/dist/schemas/rag.js +10 -16
- package/dist/schemas/registry.d.ts +30 -10
- package/dist/schemas/registry.js +14 -6
- package/dist/schemas/reload-config.d.ts +41 -6
- package/dist/schemas/reload-config.js +11 -4
- package/dist/schemas/request-base.d.ts +39 -0
- package/dist/schemas/request-base.js +64 -0
- package/dist/schemas/rerank.d.ts +8 -7
- package/dist/schemas/rerank.js +6 -4
- package/dist/schemas/rpc-server.d.ts +3 -0
- package/dist/schemas/rpc-server.js +15 -15
- package/dist/schemas/sdcpp-config.d.ts +90 -20
- package/dist/schemas/sdcpp-config.js +96 -19
- package/dist/schemas/session.d.ts +37 -0
- package/dist/schemas/session.js +53 -0
- package/dist/schemas/system-resources.d.ts +364 -179
- package/dist/schemas/system-resources.js +8 -3
- package/dist/schemas/text-to-speech.d.ts +1343 -271
- package/dist/schemas/text-to-speech.js +547 -33
- package/dist/schemas/tools.d.ts +7 -0
- package/dist/schemas/tools.js +29 -0
- package/dist/schemas/transcription-config.d.ts +262 -2
- package/dist/schemas/transcription-config.js +122 -6
- package/dist/schemas/transcription.d.ts +93 -6
- package/dist/schemas/transcription.js +98 -21
- package/dist/schemas/translate.d.ts +2 -0
- package/dist/schemas/translate.js +5 -9
- package/dist/schemas/translation-config.d.ts +30 -20
- package/dist/schemas/tts-pcm.d.ts +35 -0
- package/dist/schemas/tts-pcm.js +69 -0
- package/dist/schemas/vector-index.d.ts +47 -1
- package/dist/schemas/vector-index.js +37 -10
- package/dist/schemas/vla.d.ts +3 -3
- package/dist/surface.d.ts +3 -3
- package/dist/surface.js +3 -3
- package/dist/utils/addon-responses.d.ts +69 -2
- package/dist/utils/archive.d.ts +7 -0
- package/dist/utils/archive.js +88 -29
- package/dist/utils/asr-events.d.ts +5 -0
- package/dist/utils/asr-events.js +10 -1
- package/dist/utils/cache/auto-history.js +3 -1
- package/dist/utils/cache/layout.d.ts +48 -0
- package/dist/utils/cache/layout.js +190 -0
- package/dist/utils/cache/paths.js +2 -4
- package/dist/utils/cache/types.d.ts +2 -1
- package/dist/utils/checksum.d.ts +5 -0
- package/dist/utils/checksum.js +18 -0
- package/dist/utils/completion-normalizer.js +286 -87
- package/dist/utils/gguf-tensor-extractor.d.ts +0 -6
- package/dist/utils/gguf-tensor-extractor.js +1 -50
- package/dist/utils/shard-utils.d.ts +0 -21
- package/dist/utils/shard-utils.js +12 -62
- package/dist/utils/tool-integration.d.ts +2 -2
- package/dist/utils/tools/defer.d.ts +39 -10
- package/dist/utils/tools/defer.js +72 -17
- package/dist/utils/tools/index.d.ts +1 -1
- package/dist/utils/tools/index.js +1 -1
- package/dist/utils/transcribe-metadata.d.ts +28 -2
- package/dist/utils/transcribe-metadata.js +16 -0
- package/dist/utils/tts-stats.d.ts +42 -0
- package/dist/utils/tts-stats.js +120 -22
- package/package.json +43 -39
- package/dist/model-fit/classify.d.ts +0 -6
- package/dist/model-fit/classify.js +0 -166
- package/dist/model-fit/description/project.d.ts +0 -27
- package/dist/model-fit/description/project.js +0 -33
- package/dist/model-fit/floor.d.ts +0 -31
- package/dist/model-fit/floor.js +0 -65
- package/dist/model-fit/kv-cache.d.ts +0 -23
- package/dist/model-fit/kv-cache.js +0 -85
- package/dist/model-fit/platform.d.ts +0 -14
- package/dist/model-fit/platform.js +0 -26
- package/dist/model-fit/request/create.d.ts +0 -31
- package/dist/model-fit/request/create.js +0 -61
- package/dist/model-fit/request/engines/asr.d.ts +0 -14
- package/dist/model-fit/request/engines/audiogen.d.ts +0 -8
- package/dist/model-fit/request/engines/diffusion.d.ts +0 -7
- package/dist/model-fit/request/engines/diffusion.js +0 -93
- package/dist/model-fit/request/engines/llama.d.ts +0 -12
- package/dist/model-fit/request/engines/llama.js +0 -44
- package/dist/model-fit/request/engines/nmt.d.ts +0 -8
- package/dist/model-fit/request/engines/nmt.js +0 -11
- package/dist/model-fit/request/engines/ocr.d.ts +0 -9
- package/dist/model-fit/request/engines/ocr.js +0 -13
- package/dist/model-fit/request/engines/tts.d.ts +0 -10
- package/dist/model-fit/request/engines/tts.js +0 -152
- package/dist/models/registry/resource-profiles.d.ts +0 -17
- package/dist/models/registry/resource-profiles.js +0 -5859
- package/dist/schemas/model-resource-profile.d.ts +0 -109
- package/dist/schemas/model-resource-profile.js +0 -80
- /package/dist/model-fit/{request/gpu.d.ts → gpu.d.ts} +0 -0
- /package/dist/model-fit/{request/gpu.js → gpu.js} +0 -0
|
@@ -74,14 +74,38 @@ export function createCompletionNormalizer(config) {
|
|
|
74
74
|
// (could be the built-in `<think>` or a dialect-specific frame like
|
|
75
75
|
// Harmony's `<|channel|>analysis<|message|>...<|end|>`).
|
|
76
76
|
let activeThinkingFrame = null;
|
|
77
|
+
// Qwen-style templates open `<think>` in the prompt and the model sometimes
|
|
78
|
+
// skips `</think>` before calling a tool. A call parsed inside the block is
|
|
79
|
+
// held until what follows shows whether it was made or only quoted.
|
|
80
|
+
let toolFrameOpenedInThinking = false;
|
|
81
|
+
let heldThinkingCalls = null;
|
|
82
|
+
// Index into `thinkingParts` where the current built-in block starts.
|
|
83
|
+
let thinkingBlockStart = 0;
|
|
84
|
+
// Calls in the current block from the first one followed by more reasoning.
|
|
85
|
+
// That kind was quoted if the block closes and made if the turn ends with it
|
|
86
|
+
// open; calls made after it wait here so they are emitted in order.
|
|
87
|
+
let deferredCalls = [];
|
|
88
|
+
let undecidedAnswerStart = 0;
|
|
77
89
|
// Content frame: inner streamed as it arrives; partial close tail buffered.
|
|
78
90
|
let activeContentFrame = null;
|
|
91
|
+
// Set when a thinking frame closes before any content. Reasoning templates
|
|
92
|
+
// put a blank line between the close marker and the answer; it belongs to
|
|
93
|
+
// the framing, not the answer, so whitespace is dropped until the answer's
|
|
94
|
+
// first visible character.
|
|
95
|
+
let trimAfterThinking = false;
|
|
96
|
+
// The same framing whitespace sits inside a thinking frame (`<think>\n` ...
|
|
97
|
+
// `\n</think>`). Leading whitespace is dropped and trailing whitespace held
|
|
98
|
+
// back until more reasoning follows, so the close drops it. Both reset when a
|
|
99
|
+
// frame opens; a later frame starts on a new line so frames don't run together.
|
|
100
|
+
let trimThinkingStart = false;
|
|
101
|
+
let pendingThinkingWhitespace = '';
|
|
79
102
|
// Trailing chars that could be a prefix of a watched marker, so markers
|
|
80
103
|
// split across token pushes aren't missed by indexOf.
|
|
81
104
|
let contentTail = '';
|
|
82
105
|
let toolFramingTail = '';
|
|
83
106
|
let thinkingFramingTail = '';
|
|
84
107
|
let contentFramingTail = '';
|
|
108
|
+
let afterThinkingToolTail = '';
|
|
85
109
|
const streamedToolCallKeys = new Set();
|
|
86
110
|
const toolsEnabled = config.tools.length > 0 && config.capabilities.toolCalling !== 'none';
|
|
87
111
|
const useToolFraming = toolsEnabled && config.capabilities.toolCalling === 'textParse';
|
|
@@ -110,6 +134,7 @@ export function createCompletionNormalizer(config) {
|
|
|
110
134
|
}
|
|
111
135
|
}
|
|
112
136
|
registerFrames(toolFrames, 'tool');
|
|
137
|
+
const toolOpeners = toolFrames.map((f) => f.open);
|
|
113
138
|
registerFrames(thinkingFrames, 'thinking');
|
|
114
139
|
registerFrames(contentFrames, 'content');
|
|
115
140
|
for (const t of dropTokens) {
|
|
@@ -126,6 +151,11 @@ export function createCompletionNormalizer(config) {
|
|
|
126
151
|
for (const token of contentDropTokens) {
|
|
127
152
|
normalized = normalized.split(token).join('');
|
|
128
153
|
}
|
|
154
|
+
if (trimAfterThinking) {
|
|
155
|
+
normalized = normalized.trimStart();
|
|
156
|
+
if (normalized)
|
|
157
|
+
trimAfterThinking = false;
|
|
158
|
+
}
|
|
129
159
|
if (!normalized)
|
|
130
160
|
return;
|
|
131
161
|
contentParts.push(normalized);
|
|
@@ -135,11 +165,25 @@ export function createCompletionNormalizer(config) {
|
|
|
135
165
|
// is false (e.g. harmony's analysis channel, to strip markers from
|
|
136
166
|
// `contentDelta`) — drop the inner here instead of buffering it.
|
|
137
167
|
function emitThinking(events, text) {
|
|
138
|
-
|
|
168
|
+
let normalized = text;
|
|
169
|
+
if (trimThinkingStart) {
|
|
170
|
+
normalized = normalized.trimStart();
|
|
171
|
+
if (!normalized)
|
|
172
|
+
return;
|
|
173
|
+
trimThinkingStart = false;
|
|
174
|
+
if (thinkingParts.length > 0)
|
|
175
|
+
normalized = '\n' + normalized;
|
|
176
|
+
}
|
|
177
|
+
const body = normalized.trimEnd();
|
|
178
|
+
if (!body) {
|
|
179
|
+
pendingThinkingWhitespace += normalized;
|
|
139
180
|
return;
|
|
181
|
+
}
|
|
182
|
+
const out = pendingThinkingWhitespace + body;
|
|
183
|
+
pendingThinkingWhitespace = normalized.slice(body.length);
|
|
140
184
|
if (config.captureThinking) {
|
|
141
|
-
thinkingParts.push(
|
|
142
|
-
events.push({ type: 'thinkingDelta', seq: nextSeq(), text });
|
|
185
|
+
thinkingParts.push(out);
|
|
186
|
+
events.push({ type: 'thinkingDelta', seq: nextSeq(), text: out });
|
|
143
187
|
}
|
|
144
188
|
}
|
|
145
189
|
function getToolCallKey(call) {
|
|
@@ -149,8 +193,16 @@ export function createCompletionNormalizer(config) {
|
|
|
149
193
|
streamedToolCallKeys.add(getToolCallKey(call));
|
|
150
194
|
events.push({ type: 'toolCall', seq: nextSeq(), call });
|
|
151
195
|
}
|
|
196
|
+
// A made call waits behind earlier undecided ones so the order is kept.
|
|
197
|
+
function emitMadeCall(events, call) {
|
|
198
|
+
if (deferredCalls.length > 0)
|
|
199
|
+
deferredCalls.push({ call, made: true });
|
|
200
|
+
else
|
|
201
|
+
emitStreamedToolCall(events, call);
|
|
202
|
+
}
|
|
152
203
|
function emitFinalToolCall(events, call) {
|
|
153
|
-
|
|
204
|
+
const key = getToolCallKey(call);
|
|
205
|
+
if (streamedToolCallKeys.has(key))
|
|
154
206
|
return;
|
|
155
207
|
events.push({ type: 'toolCall', seq: nextSeq(), call });
|
|
156
208
|
}
|
|
@@ -179,14 +231,37 @@ export function createCompletionNormalizer(config) {
|
|
|
179
231
|
}
|
|
180
232
|
return best;
|
|
181
233
|
}
|
|
182
|
-
function
|
|
183
|
-
|
|
184
|
-
|
|
234
|
+
function openToolFrame(marker, spec, fromThinking) {
|
|
235
|
+
toolCallBuffer = marker;
|
|
236
|
+
activeToolFrame = spec;
|
|
237
|
+
toolFrameOpenedInThinking = fromThinking;
|
|
238
|
+
state = 'toolFraming';
|
|
239
|
+
}
|
|
240
|
+
// Every exit from a tool frame goes through here, so no frame state outlives it.
|
|
241
|
+
function takeToolFrame() {
|
|
242
|
+
const taken = {
|
|
243
|
+
raw: toolCallBuffer + toolFramingTail,
|
|
244
|
+
frame: activeToolFrame,
|
|
245
|
+
openedInThinking: toolFrameOpenedInThinking
|
|
246
|
+
};
|
|
185
247
|
toolCallBuffer = '';
|
|
186
|
-
activeToolFrame = null;
|
|
187
248
|
toolFramingTail = '';
|
|
249
|
+
activeToolFrame = null;
|
|
250
|
+
toolFrameOpenedInThinking = false;
|
|
251
|
+
return taken;
|
|
252
|
+
}
|
|
253
|
+
function flushToolBuffer(events, closed) {
|
|
254
|
+
const { raw, frame, openedInThinking } = takeToolFrame();
|
|
188
255
|
if (!frame)
|
|
189
256
|
return;
|
|
257
|
+
// The reasoning only mentioned the marker: still inside `<think>`.
|
|
258
|
+
function returnToThinking() {
|
|
259
|
+
trimThinkingStart = false;
|
|
260
|
+
emitThinking(events, raw);
|
|
261
|
+
activeThinkingFrame = BUILTIN_THINKING;
|
|
262
|
+
if (closed)
|
|
263
|
+
state = 'thinkingFraming';
|
|
264
|
+
}
|
|
190
265
|
// Empty-payload guard. Strip frame markers and check the inner is
|
|
191
266
|
// non-empty; surface a toolError on empty payloads so the literal
|
|
192
267
|
// markers don't leak into contentDelta.
|
|
@@ -197,6 +272,8 @@ export function createCompletionNormalizer(config) {
|
|
|
197
272
|
inner = inner.slice(0, inner.length - frame.close.length);
|
|
198
273
|
}
|
|
199
274
|
if (inner.trim().length === 0) {
|
|
275
|
+
if (openedInThinking)
|
|
276
|
+
return returnToThinking();
|
|
200
277
|
events.push({
|
|
201
278
|
type: 'toolError',
|
|
202
279
|
seq: nextSeq(),
|
|
@@ -205,8 +282,17 @@ export function createCompletionNormalizer(config) {
|
|
|
205
282
|
return;
|
|
206
283
|
}
|
|
207
284
|
const { toolCalls, errors } = parseToolCalls(raw, config.tools, dialect);
|
|
285
|
+
if (openedInThinking) {
|
|
286
|
+
if (toolCalls.length === 0)
|
|
287
|
+
return returnToThinking();
|
|
288
|
+
if (closed) {
|
|
289
|
+
heldThinkingCalls = { raw, calls: toolCalls };
|
|
290
|
+
state = 'afterThinkingTool';
|
|
291
|
+
return;
|
|
292
|
+
}
|
|
293
|
+
}
|
|
208
294
|
for (const call of toolCalls)
|
|
209
|
-
|
|
295
|
+
emitMadeCall(events, call);
|
|
210
296
|
if (toolCalls.length === 0) {
|
|
211
297
|
if (errors.length > 0) {
|
|
212
298
|
for (const error of errors) {
|
|
@@ -230,10 +316,7 @@ export function createCompletionNormalizer(config) {
|
|
|
230
316
|
}
|
|
231
317
|
}
|
|
232
318
|
function abandonToolFrame(events) {
|
|
233
|
-
const raw =
|
|
234
|
-
toolCallBuffer = '';
|
|
235
|
-
toolFramingTail = '';
|
|
236
|
-
activeToolFrame = null;
|
|
319
|
+
const { raw } = takeToolFrame();
|
|
237
320
|
state = 'content';
|
|
238
321
|
events.push({
|
|
239
322
|
type: 'toolError',
|
|
@@ -255,7 +338,113 @@ export function createCompletionNormalizer(config) {
|
|
|
255
338
|
return toolFramingTail;
|
|
256
339
|
case 'contentFraming':
|
|
257
340
|
return contentFramingTail;
|
|
341
|
+
case 'afterThinkingTool':
|
|
342
|
+
return afterThinkingToolTail;
|
|
343
|
+
}
|
|
344
|
+
}
|
|
345
|
+
// Each step consumes what it can of `buf` in the current state and returns the rest.
|
|
346
|
+
function stepToolFraming(events, buf) {
|
|
347
|
+
if (!activeToolFrame) {
|
|
348
|
+
state = 'content';
|
|
349
|
+
return buf;
|
|
350
|
+
}
|
|
351
|
+
const close = activeToolFrame.close;
|
|
352
|
+
const hit = buf.indexOf(close);
|
|
353
|
+
// `</think>` before the frame closes: the reasoning only mentioned the
|
|
354
|
+
// opener, so the buffered text goes back to reasoning.
|
|
355
|
+
const thinkClose = toolFrameOpenedInThinking ? buf.indexOf(BUILTIN_THINKING.close) : -1;
|
|
356
|
+
if (thinkClose >= 0 && (hit < 0 || thinkClose < hit)) {
|
|
357
|
+
const { raw } = takeToolFrame();
|
|
358
|
+
trimThinkingStart = false;
|
|
359
|
+
emitThinking(events, raw + buf.slice(0, thinkClose));
|
|
360
|
+
closeThinkingBlock(events);
|
|
361
|
+
return buf.slice(thinkClose + BUILTIN_THINKING.close.length);
|
|
362
|
+
}
|
|
363
|
+
if (hit >= 0) {
|
|
364
|
+
toolCallBuffer += buf.slice(0, hit + close.length);
|
|
365
|
+
state = 'content';
|
|
366
|
+
flushToolBuffer(events, true);
|
|
367
|
+
return buf.slice(hit + close.length);
|
|
258
368
|
}
|
|
369
|
+
const tailLen = trailingPartialMarkerLen(buf, toolFrameOpenedInThinking ? [close, BUILTIN_THINKING.close] : [close]);
|
|
370
|
+
const safeLen = buf.length - tailLen;
|
|
371
|
+
toolCallBuffer += buf.slice(0, safeLen);
|
|
372
|
+
toolFramingTail = buf.slice(safeLen);
|
|
373
|
+
if (toolCallBuffer.length > MAX_TOOL_FRAME_CHARS)
|
|
374
|
+
abandonToolFrame(events);
|
|
375
|
+
return '';
|
|
376
|
+
}
|
|
377
|
+
function closeThinkingBlock(events) {
|
|
378
|
+
for (const { call, made } of deferredCalls)
|
|
379
|
+
if (made)
|
|
380
|
+
emitStreamedToolCall(events, call);
|
|
381
|
+
deferredCalls = [];
|
|
382
|
+
activeThinkingFrame = null;
|
|
383
|
+
trimAfterThinking = contentParts.length === 0;
|
|
384
|
+
state = 'content';
|
|
385
|
+
}
|
|
386
|
+
function releaseHeldCalls(events) {
|
|
387
|
+
const held = heldThinkingCalls;
|
|
388
|
+
heldThinkingCalls = null;
|
|
389
|
+
for (const call of held?.calls ?? [])
|
|
390
|
+
emitMadeCall(events, call);
|
|
391
|
+
}
|
|
392
|
+
// A held call was made if the turn ends, `</think>` closes the block, or
|
|
393
|
+
// another call follows. Any other text means the reasoning quoted it.
|
|
394
|
+
function stepAfterThinkingTool(events, buf) {
|
|
395
|
+
const rest = buf.trimStart();
|
|
396
|
+
const markers = [BUILTIN_THINKING.close, ...toolOpeners];
|
|
397
|
+
const marker = markers.find((m) => rest.startsWith(m));
|
|
398
|
+
if (marker) {
|
|
399
|
+
releaseHeldCalls(events);
|
|
400
|
+
if (marker === BUILTIN_THINKING.close) {
|
|
401
|
+
closeThinkingBlock(events);
|
|
402
|
+
}
|
|
403
|
+
else {
|
|
404
|
+
openToolFrame(marker, markerToEntry.get(marker).spec, true);
|
|
405
|
+
}
|
|
406
|
+
return rest.slice(marker.length);
|
|
407
|
+
}
|
|
408
|
+
if (markers.some((m) => m.startsWith(rest))) {
|
|
409
|
+
afterThinkingToolTail = buf;
|
|
410
|
+
return '';
|
|
411
|
+
}
|
|
412
|
+
const held = heldThinkingCalls;
|
|
413
|
+
heldThinkingCalls = null;
|
|
414
|
+
trimThinkingStart = false;
|
|
415
|
+
emitThinking(events, (held?.raw ?? '') + buf.slice(0, buf.length - rest.length));
|
|
416
|
+
for (const call of held?.calls ?? [])
|
|
417
|
+
deferredCalls.push({ call, made: false });
|
|
418
|
+
undecidedAnswerStart = thinkingParts.length;
|
|
419
|
+
activeThinkingFrame = BUILTIN_THINKING;
|
|
420
|
+
state = 'thinkingFraming';
|
|
421
|
+
return rest;
|
|
422
|
+
}
|
|
423
|
+
function stepThinkingFraming(events, buf) {
|
|
424
|
+
if (!activeThinkingFrame) {
|
|
425
|
+
state = 'content';
|
|
426
|
+
return buf;
|
|
427
|
+
}
|
|
428
|
+
const builtin = activeThinkingFrame === BUILTIN_THINKING;
|
|
429
|
+
const closeMarker = activeThinkingFrame.close;
|
|
430
|
+
const closeIdx = buf.indexOf(closeMarker);
|
|
431
|
+
const toolOpen = builtin ? findEarliestMarker(buf, toolOpeners) : null;
|
|
432
|
+
if (toolOpen && (closeIdx < 0 || toolOpen.idx < closeIdx)) {
|
|
433
|
+
emitThinking(events, buf.slice(0, toolOpen.idx));
|
|
434
|
+
activeThinkingFrame = null;
|
|
435
|
+
openToolFrame(toolOpen.marker, markerToEntry.get(toolOpen.marker).spec, true);
|
|
436
|
+
return buf.slice(toolOpen.idx + toolOpen.marker.length);
|
|
437
|
+
}
|
|
438
|
+
if (closeIdx >= 0) {
|
|
439
|
+
emitThinking(events, buf.slice(0, closeIdx));
|
|
440
|
+
closeThinkingBlock(events);
|
|
441
|
+
return buf.slice(closeIdx + closeMarker.length);
|
|
442
|
+
}
|
|
443
|
+
const tailLen = trailingPartialMarkerLen(buf, builtin ? [closeMarker, ...toolOpeners] : [closeMarker]);
|
|
444
|
+
const safeLen = buf.length - tailLen;
|
|
445
|
+
emitThinking(events, buf.slice(0, safeLen));
|
|
446
|
+
thinkingFramingTail = buf.slice(safeLen);
|
|
447
|
+
return '';
|
|
259
448
|
}
|
|
260
449
|
function processSegment(events, text) {
|
|
261
450
|
let buf = getStateTail() + text;
|
|
@@ -263,48 +452,18 @@ export function createCompletionNormalizer(config) {
|
|
|
263
452
|
thinkingFramingTail = '';
|
|
264
453
|
toolFramingTail = '';
|
|
265
454
|
contentFramingTail = '';
|
|
455
|
+
afterThinkingToolTail = '';
|
|
266
456
|
while (buf.length > 0) {
|
|
267
457
|
if (state === 'toolFraming') {
|
|
268
|
-
|
|
269
|
-
state = 'content';
|
|
270
|
-
continue;
|
|
271
|
-
}
|
|
272
|
-
const hit = buf.indexOf(activeToolFrame.close);
|
|
273
|
-
if (hit >= 0) {
|
|
274
|
-
toolCallBuffer += buf.slice(0, hit + activeToolFrame.close.length);
|
|
275
|
-
buf = buf.slice(hit + activeToolFrame.close.length);
|
|
276
|
-
state = 'content';
|
|
277
|
-
flushToolBuffer(events, true);
|
|
278
|
-
continue;
|
|
279
|
-
}
|
|
280
|
-
const tailLen = trailingPartialMarkerLen(buf, [activeToolFrame.close]);
|
|
281
|
-
const safeLen = buf.length - tailLen;
|
|
282
|
-
toolCallBuffer += buf.slice(0, safeLen);
|
|
283
|
-
toolFramingTail = buf.slice(safeLen);
|
|
284
|
-
buf = '';
|
|
285
|
-
if (toolCallBuffer.length > MAX_TOOL_FRAME_CHARS)
|
|
286
|
-
abandonToolFrame(events);
|
|
458
|
+
buf = stepToolFraming(events, buf);
|
|
287
459
|
continue;
|
|
288
460
|
}
|
|
289
461
|
if (state === 'thinkingFraming') {
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
const closeIdx = buf.indexOf(closeMarker);
|
|
296
|
-
if (closeIdx >= 0) {
|
|
297
|
-
emitThinking(events, buf.slice(0, closeIdx));
|
|
298
|
-
buf = buf.slice(closeIdx + closeMarker.length);
|
|
299
|
-
activeThinkingFrame = null;
|
|
300
|
-
state = 'content';
|
|
301
|
-
continue;
|
|
302
|
-
}
|
|
303
|
-
const tailLen = trailingPartialMarkerLen(buf, [closeMarker]);
|
|
304
|
-
const safeLen = buf.length - tailLen;
|
|
305
|
-
emitThinking(events, buf.slice(0, safeLen));
|
|
306
|
-
thinkingFramingTail = buf.slice(safeLen);
|
|
307
|
-
buf = '';
|
|
462
|
+
buf = stepThinkingFraming(events, buf);
|
|
463
|
+
continue;
|
|
464
|
+
}
|
|
465
|
+
if (state === 'afterThinkingTool') {
|
|
466
|
+
buf = stepAfterThinkingTool(events, buf);
|
|
308
467
|
continue;
|
|
309
468
|
}
|
|
310
469
|
if (state === 'contentFraming') {
|
|
@@ -338,13 +497,19 @@ export function createCompletionNormalizer(config) {
|
|
|
338
497
|
continue;
|
|
339
498
|
}
|
|
340
499
|
const entry = markerToEntry.get(hit.marker);
|
|
500
|
+
// A content frame (Harmony's final channel) carries the answer itself,
|
|
501
|
+
// so the trim carries into it; any other frame ends it.
|
|
502
|
+
if (entry.kind !== 'content')
|
|
503
|
+
trimAfterThinking = false;
|
|
341
504
|
if (entry.kind === 'tool') {
|
|
342
|
-
|
|
343
|
-
activeToolFrame = entry.spec;
|
|
344
|
-
state = 'toolFraming';
|
|
505
|
+
openToolFrame(hit.marker, entry.spec, false);
|
|
345
506
|
}
|
|
346
507
|
else if (entry.kind === 'thinking') {
|
|
347
508
|
activeThinkingFrame = entry.spec;
|
|
509
|
+
thinkingBlockStart = thinkingParts.length;
|
|
510
|
+
deferredCalls = [];
|
|
511
|
+
trimThinkingStart = true;
|
|
512
|
+
pendingThinkingWhitespace = '';
|
|
348
513
|
state = 'thinkingFraming';
|
|
349
514
|
}
|
|
350
515
|
else {
|
|
@@ -369,10 +534,8 @@ export function createCompletionNormalizer(config) {
|
|
|
369
534
|
processSegment(events, token);
|
|
370
535
|
return events;
|
|
371
536
|
}
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
// Drain pending tail bytes so we don't lose trailing chars when the
|
|
375
|
-
// stream ends mid-marker.
|
|
537
|
+
// Emits tail bytes held back while a marker might still be arriving.
|
|
538
|
+
function drainTails(events) {
|
|
376
539
|
if (state === 'content' && contentTail) {
|
|
377
540
|
emitContent(events, contentTail);
|
|
378
541
|
contentTail = '';
|
|
@@ -386,38 +549,79 @@ export function createCompletionNormalizer(config) {
|
|
|
386
549
|
toolFramingTail = '';
|
|
387
550
|
}
|
|
388
551
|
else if (state === 'contentFraming' && contentFramingTail) {
|
|
389
|
-
// Cutoff inside a partial close marker — emit tail as content.
|
|
390
552
|
emitContent(events, contentFramingTail);
|
|
391
553
|
contentFramingTail = '';
|
|
392
554
|
}
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
activeToolFrame = null;
|
|
398
|
-
}
|
|
399
|
-
else {
|
|
400
|
-
flushToolBuffer(events, false);
|
|
401
|
-
}
|
|
402
|
-
}
|
|
403
|
-
else if (state === 'contentFraming') {
|
|
555
|
+
}
|
|
556
|
+
// A frame still open at the end is parsed, or returned as text on error.
|
|
557
|
+
function closeOpenFrames(events, failed) {
|
|
558
|
+
if (state === 'contentFraming') {
|
|
404
559
|
activeContentFrame = null;
|
|
405
560
|
}
|
|
561
|
+
else if (state === 'toolFraming' && activeToolFrame && !failed) {
|
|
562
|
+
flushToolBuffer(events, false);
|
|
563
|
+
}
|
|
564
|
+
else if (state === 'toolFraming' && activeToolFrame) {
|
|
565
|
+
const { raw, openedInThinking } = takeToolFrame();
|
|
566
|
+
if (openedInThinking)
|
|
567
|
+
emitThinking(events, raw);
|
|
568
|
+
else
|
|
569
|
+
emitContent(events, raw);
|
|
570
|
+
}
|
|
571
|
+
}
|
|
572
|
+
// The turn ended right after a call inside `<think>`: it was made, unless
|
|
573
|
+
// the turn failed. A partial marker buffered after it stays reasoning.
|
|
574
|
+
function settleHeldCalls(events, failed) {
|
|
575
|
+
if (!heldThinkingCalls)
|
|
576
|
+
return;
|
|
577
|
+
if (failed) {
|
|
578
|
+
emitThinking(events, heldThinkingCalls.raw + afterThinkingToolTail);
|
|
579
|
+
heldThinkingCalls = null;
|
|
580
|
+
}
|
|
581
|
+
else {
|
|
582
|
+
releaseHeldCalls(events);
|
|
583
|
+
if (afterThinkingToolTail.trim())
|
|
584
|
+
emitThinking(events, afterThinkingToolTail);
|
|
585
|
+
}
|
|
586
|
+
afterThinkingToolTail = '';
|
|
587
|
+
}
|
|
588
|
+
// The model stopped on its own inside the prompt-opened `<think>`: calls it
|
|
589
|
+
// left there were made, and the reasoning after them, or the whole block
|
|
590
|
+
// when the turn has no call at all, is the answer.
|
|
591
|
+
function promoteOpenThinking(events, callsLeftInBlock, finalCalls) {
|
|
592
|
+
if (contentParts.length > 0)
|
|
593
|
+
return;
|
|
594
|
+
if (callsLeftInBlock) {
|
|
595
|
+
emitContent(events, thinkingParts.slice(undecidedAnswerStart).join('').trim());
|
|
596
|
+
}
|
|
597
|
+
else if (streamedToolCallKeys.size + finalCalls.length === 0) {
|
|
598
|
+
emitContent(events, thinkingParts.slice(thinkingBlockStart).join('').trim());
|
|
599
|
+
}
|
|
600
|
+
}
|
|
601
|
+
function finish(opts) {
|
|
602
|
+
const events = [];
|
|
603
|
+
const failed = !!opts?.error;
|
|
604
|
+
const finalCalls = failed ? [] : (opts?.toolCalls ?? []);
|
|
605
|
+
const naturalStop = opts?.stopReason === 'eos' || opts?.stopReason === 'stopSequence';
|
|
606
|
+
drainTails(events);
|
|
607
|
+
closeOpenFrames(events, failed);
|
|
608
|
+
settleHeldCalls(events, failed);
|
|
609
|
+
// The block never closed, so every call left in it was made. A cancelled
|
|
610
|
+
// turn says nothing about the ones the model had not confirmed.
|
|
611
|
+
const cancelled = opts?.stopReason === 'cancelled';
|
|
612
|
+
const leftInBlock = failed ? [] : deferredCalls.filter((d) => d.made || !cancelled);
|
|
613
|
+
deferredCalls = [];
|
|
614
|
+
for (const { call } of leftInBlock)
|
|
615
|
+
emitStreamedToolCall(events, call);
|
|
616
|
+
if (activeThinkingFrame === BUILTIN_THINKING && naturalStop && !failed) {
|
|
617
|
+
promoteOpenThinking(events, leftInBlock.length > 0, finalCalls);
|
|
618
|
+
}
|
|
406
619
|
activeThinkingFrame = null;
|
|
407
620
|
state = 'content';
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
for (const call of opts.toolCalls) {
|
|
411
|
-
emitFinalToolCall(events, call);
|
|
412
|
-
}
|
|
413
|
-
}
|
|
414
|
-
}
|
|
621
|
+
for (const call of finalCalls)
|
|
622
|
+
emitFinalToolCall(events, call);
|
|
415
623
|
if (opts?.stats) {
|
|
416
|
-
events.push({
|
|
417
|
-
type: 'completionStats',
|
|
418
|
-
seq: nextSeq(),
|
|
419
|
-
stats: opts.stats
|
|
420
|
-
});
|
|
624
|
+
events.push({ type: 'completionStats', seq: nextSeq(), stats: opts.stats });
|
|
421
625
|
}
|
|
422
626
|
const raw = { fullText: rawText };
|
|
423
627
|
if (opts?.error) {
|
|
@@ -430,12 +634,7 @@ export function createCompletionNormalizer(config) {
|
|
|
430
634
|
});
|
|
431
635
|
}
|
|
432
636
|
else if (opts?.stopReason) {
|
|
433
|
-
events.push({
|
|
434
|
-
type: 'completionDone',
|
|
435
|
-
seq: nextSeq(),
|
|
436
|
-
stopReason: opts.stopReason,
|
|
437
|
-
raw
|
|
438
|
-
});
|
|
637
|
+
events.push({ type: 'completionDone', seq: nextSeq(), stopReason: opts.stopReason, raw });
|
|
439
638
|
}
|
|
440
639
|
else {
|
|
441
640
|
events.push({ type: 'completionDone', seq: nextSeq(), raw });
|
|
@@ -1,9 +1,3 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Extract tensor names from all GGUF shards and write to {baseFilename}.tensors.txt
|
|
3
|
-
* Required for sharded models to enable incremental/async loading
|
|
4
|
-
* @throws ModelLoadFailedError if any shard fails extraction
|
|
5
|
-
*/
|
|
6
|
-
export declare function extractAndWriteTensorsFile(shardDir: string, shardFilenames: string[], baseFilename: string): Promise<string>;
|
|
7
1
|
/**
|
|
8
2
|
* Check if a file has a valid GGUF header structure.
|
|
9
3
|
* Returns true if the header parses successfully and contains tensor metadata.
|
|
@@ -1,8 +1,7 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { createReadStream } from 'bare-fs';
|
|
2
2
|
import Buffer from 'bare-buffer';
|
|
3
3
|
import { getEngineLogger } from '../logging/index.js';
|
|
4
4
|
import { ModelLoadFailedError } from '../errors/index.js';
|
|
5
|
-
import { validateAndJoinPath } from '../utils/path-security.js';
|
|
6
5
|
const logger = getEngineLogger();
|
|
7
6
|
// GGUF value types
|
|
8
7
|
const GGUF_TYPE = {
|
|
@@ -244,54 +243,6 @@ async function extractTensorNamesFromHeader(filePath) {
|
|
|
244
243
|
destroyStream();
|
|
245
244
|
}
|
|
246
245
|
}
|
|
247
|
-
/**
|
|
248
|
-
* Extract tensor names from all GGUF shards and write to {baseFilename}.tensors.txt
|
|
249
|
-
* Required for sharded models to enable incremental/async loading
|
|
250
|
-
* @throws ModelLoadFailedError if any shard fails extraction
|
|
251
|
-
*/
|
|
252
|
-
export async function extractAndWriteTensorsFile(shardDir, shardFilenames, baseFilename) {
|
|
253
|
-
const tensorsFilePath = validateAndJoinPath(shardDir, `${baseFilename}.tensors.txt`);
|
|
254
|
-
try {
|
|
255
|
-
await fsPromises.access(tensorsFilePath);
|
|
256
|
-
logger.info(`Tensors file already exists: ${tensorsFilePath}`);
|
|
257
|
-
return tensorsFilePath;
|
|
258
|
-
}
|
|
259
|
-
catch {
|
|
260
|
-
// Continue with extraction
|
|
261
|
-
}
|
|
262
|
-
logger.info(`Extracting tensors from ${shardFilenames.length} shards...`);
|
|
263
|
-
const allTensorNames = new Set();
|
|
264
|
-
const failedShards = [];
|
|
265
|
-
for (const filename of shardFilenames) {
|
|
266
|
-
const shardPath = validateAndJoinPath(shardDir, filename);
|
|
267
|
-
try {
|
|
268
|
-
logger.debug(`Extracting tensors from ${filename}...`);
|
|
269
|
-
const tensorNames = await extractTensorNamesFromHeader(shardPath);
|
|
270
|
-
if (tensorNames.length > 0) {
|
|
271
|
-
tensorNames.forEach((name) => allTensorNames.add(name));
|
|
272
|
-
logger.info(`Extracted ${tensorNames.length} tensors from ${filename}`);
|
|
273
|
-
}
|
|
274
|
-
else {
|
|
275
|
-
failedShards.push(filename);
|
|
276
|
-
logger.error(`No tensors found in ${filename}`);
|
|
277
|
-
}
|
|
278
|
-
}
|
|
279
|
-
catch (error) {
|
|
280
|
-
failedShards.push(filename);
|
|
281
|
-
logger.error(`Failed to extract tensors from ${filename}:`, error);
|
|
282
|
-
}
|
|
283
|
-
}
|
|
284
|
-
if (failedShards.length > 0) {
|
|
285
|
-
throw new ModelLoadFailedError(`Failed to extract tensors from ${failedShards.length}/${shardFilenames.length} shards: ${failedShards.join(', ')}`);
|
|
286
|
-
}
|
|
287
|
-
if (allTensorNames.size === 0) {
|
|
288
|
-
throw new ModelLoadFailedError(`Could not extract any tensors from shards.`);
|
|
289
|
-
}
|
|
290
|
-
const tensorNames = Array.from(allTensorNames).sort();
|
|
291
|
-
await fsPromises.writeFile(tensorsFilePath, tensorNames.join('\n') + '\n', 'utf8');
|
|
292
|
-
logger.info(`Tensors file created: ${tensorsFilePath} (${tensorNames.length} tensors from ${shardFilenames.length} shards)`);
|
|
293
|
-
return tensorsFilePath;
|
|
294
|
-
}
|
|
295
246
|
/**
|
|
296
247
|
* Check if a file has a valid GGUF header structure.
|
|
297
248
|
* Returns true if the header parses successfully and contains tensor metadata.
|
|
@@ -12,11 +12,6 @@ export declare function getFirstShardPath(modelPath: string): string;
|
|
|
12
12
|
* @returns Array of all numbered shard filenames
|
|
13
13
|
*/
|
|
14
14
|
export declare function generateShardFilenames(shardName: string): string[];
|
|
15
|
-
/**
|
|
16
|
-
* List the files a model path loads: the path itself, or for a sharded model
|
|
17
|
-
* its tensors file followed by every shard in order
|
|
18
|
-
*/
|
|
19
|
-
export declare function getModelShardFiles(modelPath: string): string[];
|
|
20
15
|
/**
|
|
21
16
|
* Parse pattern-based shard URL and generate all shard URLs with cache key
|
|
22
17
|
* Works with patterns like: https://example.com/path/model-00002-of-00005.gguf
|
|
@@ -28,15 +23,6 @@ export declare function parsePatternBasedShardUrl(shardUrl: string): {
|
|
|
28
23
|
shardUrls: ShardUrl[];
|
|
29
24
|
cacheKey: string;
|
|
30
25
|
};
|
|
31
|
-
/**
|
|
32
|
-
* Extract tensor information from all shards and write to baseFilename.tensors.txt
|
|
33
|
-
* Required for sharded models to enable incremental/async loading
|
|
34
|
-
* @param shardDir - Directory containing the shard files
|
|
35
|
-
* @param shardFilename - Filename of any shard (e.g., "model-00001-of-00002.gguf")
|
|
36
|
-
* @returns Path to the created tensors.txt file
|
|
37
|
-
* @throws If not a sharded model or extraction fails
|
|
38
|
-
*/
|
|
39
|
-
export declare function extractTensorsFromShards(shardDir: string, shardFilename: string): Promise<string>;
|
|
40
26
|
/**
|
|
41
27
|
* Check if all numbered shard files exist
|
|
42
28
|
* @param shardDir - Directory containing the shard files
|
|
@@ -44,10 +30,3 @@ export declare function extractTensorsFromShards(shardDir: string, shardFilename
|
|
|
44
30
|
* @returns true if all numbered shards exist, false otherwise
|
|
45
31
|
*/
|
|
46
32
|
export declare function checkAllShardsExist(shardDir: string, shardFilename: string): Promise<boolean>;
|
|
47
|
-
/**
|
|
48
|
-
* Validate that all shards and tensors.txt exist for a sharded model
|
|
49
|
-
* @param shardDir - Directory containing the shard files
|
|
50
|
-
* @param shardFilename - Filename of any shard (e.g., "model-00001-of-00002.gguf")
|
|
51
|
-
* @returns true if all shards and tensors.txt exist, false otherwise
|
|
52
|
-
*/
|
|
53
|
-
export declare function validateShardedModelCache(shardDir: string, shardFilename: string): Promise<boolean>;
|