@qvac/inference 0.22.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +111 -60
- package/README.md +4 -2
- package/dist/api/assess-model-fit.d.ts +14 -14
- package/dist/api/assess-model-fit.js +14 -14
- package/dist/api/audio-gen.d.ts +2 -2
- package/dist/api/audio-gen.js +136 -45
- package/dist/api/batch-completion.js +2 -1
- package/dist/api/calc-tokens.d.ts +30 -0
- package/dist/api/calc-tokens.js +41 -0
- package/dist/api/classify.d.ts +5 -2
- package/dist/api/classify.js +5 -2
- package/dist/api/completion-stream.d.ts +2 -1
- package/dist/api/completion-stream.js +4 -1
- package/dist/api/decide.js +1 -1
- package/dist/api/delete-cache.d.ts +7 -1
- package/dist/api/delete-cache.js +11 -1
- package/dist/api/diffusion.d.ts +32 -2
- package/dist/api/diffusion.js +32 -2
- package/dist/api/download-asset.js +1 -1
- package/dist/api/embed.d.ts +82 -29
- package/dist/api/embed.js +64 -18
- package/dist/api/finetune.js +1 -1
- package/dist/api/get-model-info.d.ts +1 -1
- package/dist/api/index.d.ts +40 -38
- package/dist/api/index.js +39 -37
- package/dist/api/load-model.js +2 -2
- package/dist/api/logging-stream-registry.js +11 -7
- package/dist/api/logging-stream.d.ts +11 -1
- package/dist/api/logging-stream.js +11 -2
- package/dist/api/rag.d.ts +1 -1
- package/dist/api/rag.js +1 -1
- package/dist/api/registry.d.ts +18 -7
- package/dist/api/registry.js +37 -13
- package/dist/api/rerank.d.ts +17 -0
- package/dist/api/rerank.js +36 -0
- package/dist/api/subscribe-logs.js +5 -3
- package/dist/api/text-to-speech.d.ts +14 -5
- package/dist/api/text-to-speech.js +83 -29
- package/dist/api/transcribe.d.ts +6 -2
- package/dist/api/transcribe.js +7 -1
- package/dist/api/unload-model.js +1 -1
- package/dist/api/vector-index.d.ts +25 -3
- package/dist/api/vector-index.js +17 -2
- package/dist/api/video-request.d.ts +15 -15
- package/dist/api/video.js +1 -1
- package/dist/api/vla.js +1 -1
- package/dist/api/world-result.js +1 -1
- package/dist/api/world.d.ts +1 -1
- package/dist/api/world.js +1 -1
- package/dist/config/resolve-config.d.ts +1 -1
- package/dist/config/resolve-config.js +1 -1
- package/dist/constants/index.d.ts +2 -2
- package/dist/constants/index.js +2 -2
- package/dist/dispatch.d.ts +11 -4
- package/dist/dispatch.js +80 -18
- package/dist/engine.d.ts +1 -1
- package/dist/errors/index.d.ts +76 -0
- package/dist/errors/index.js +141 -1
- package/dist/handlers/assess-model-fit.d.ts +3 -21
- package/dist/handlers/assess-model-fit.js +4 -303
- package/dist/handlers/cancelHandler.d.ts +10 -9
- package/dist/handlers/cancelHandler.js +18 -14
- package/dist/handlers/completion-orchestrate.d.ts +14 -2
- package/dist/handlers/completion-orchestrate.js +166 -24
- package/dist/handlers/delete-cache.d.ts +2 -1
- package/dist/handlers/delete-cache.js +6 -1
- package/dist/handlers/download-asset.js +13 -6
- package/dist/handlers/finetune.js +1 -1
- package/dist/handlers/get-loaded-model-info.js +3 -3
- package/dist/handlers/get-model-info.js +72 -3
- package/dist/handlers/load-model/coreml-sidecars.d.ts +15 -11
- package/dist/handlers/load-model/coreml-sidecars.js +51 -91
- package/dist/handlers/load-model/download-lock.d.ts +20 -0
- package/dist/handlers/load-model/download-lock.js +213 -0
- package/dist/handlers/load-model/download-manager.d.ts +11 -3
- package/dist/handlers/load-model/download-manager.js +62 -11
- package/dist/handlers/load-model/download-stats.d.ts +2 -2
- package/dist/handlers/load-model/download-stats.js +3 -3
- package/dist/handlers/load-model/handler.js +97 -41
- package/dist/handlers/load-model/http-verify.d.ts +1 -1
- package/dist/handlers/load-model/http.d.ts +1 -1
- package/dist/handlers/load-model/http.js +330 -208
- package/dist/handlers/load-model/hyperdrive.d.ts +16 -1
- package/dist/handlers/load-model/hyperdrive.js +125 -52
- package/dist/handlers/load-model/index.d.ts +2 -2
- package/dist/handlers/load-model/index.js +2 -2
- package/dist/handlers/load-model/registry-client-options.d.ts +2 -0
- package/dist/handlers/load-model/registry-client-options.js +4 -2
- package/dist/handlers/load-model/registry-companion-set.d.ts +6 -1
- package/dist/handlers/load-model/registry-companion-set.js +20 -2
- package/dist/handlers/load-model/registry-delete.d.ts +5 -0
- package/dist/handlers/load-model/registry-delete.js +269 -0
- package/dist/handlers/load-model/registry-download-utils.d.ts +7 -5
- package/dist/handlers/load-model/registry-download-utils.js +95 -24
- package/dist/handlers/load-model/registry.d.ts +22 -2
- package/dist/handlers/load-model/registry.js +312 -179
- package/dist/handlers/load-model/resolve-session.d.ts +1 -1
- package/dist/handlers/load-model/resolve-session.js +11 -3
- package/dist/handlers/load-model/resolve.d.ts +1 -1
- package/dist/handlers/load-model/resolve.js +9 -6
- package/dist/handlers/load-model/types.d.ts +2 -0
- package/dist/handlers/logging-stream.d.ts +2 -1
- package/dist/handlers/logging-stream.js +35 -7
- package/dist/handlers/plugin-invoke.js +152 -39
- package/dist/handlers/registry.d.ts +2 -2
- package/dist/handlers/registry.js +108 -29
- package/dist/handlers/rpc-server.js +7 -7
- package/dist/handlers/session.d.ts +11 -0
- package/dist/handlers/session.js +40 -0
- package/dist/handlers/types.d.ts +7 -1
- package/dist/handlers/unload-model.js +8 -3
- package/dist/handlers/vector-index.js +118 -19
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/logging/addon.d.ts +1 -1
- package/dist/logging/addon.js +1 -1
- package/dist/logging/index.d.ts +5 -5
- package/dist/logging/index.js +4 -4
- package/dist/logging/logger.d.ts +1 -1
- package/dist/logging/logger.js +3 -3
- package/dist/logging/utils.js +11 -3
- package/dist/model-cache-layout.d.ts +1 -0
- package/dist/model-cache-layout.js +2 -0
- package/dist/model-fit/assess.d.ts +23 -0
- package/dist/model-fit/assess.js +110 -0
- package/dist/model-fit/describe.d.ts +12 -0
- package/dist/model-fit/describe.js +29 -0
- package/dist/{resources/model-fit/fit-stub/fetch-fit-stub.d.ts → model-fit/description/fetch.d.ts} +2 -2
- package/dist/{resources/model-fit/fit-stub/fetch-fit-stub.js → model-fit/description/fetch.js} +98 -19
- package/dist/{resources/model-fit/fit-stub/resolve-load-from-stubs.d.ts → model-fit/description/resolve-load.d.ts} +8 -6
- package/dist/{resources/model-fit/fit-stub/resolve-load-from-stubs.js → model-fit/description/resolve-load.js} +20 -20
- package/dist/model-fit/fit-load.d.ts +23 -0
- package/dist/model-fit/fit-load.js +29 -0
- package/dist/model-fit/load-check.d.ts +34 -0
- package/dist/model-fit/load-check.js +77 -0
- package/dist/model-fit/run/engine.d.ts +12 -0
- package/dist/model-fit/run/engine.js +9 -0
- package/dist/model-fit/run/run.d.ts +46 -0
- package/dist/model-fit/run/run.js +208 -0
- package/dist/model-fit/sum.d.ts +32 -0
- package/dist/model-fit/sum.js +60 -0
- package/dist/models/registry/index.d.ts +1 -1
- package/dist/models/registry/index.js +2 -2
- package/dist/models/registry/models.d.ts +1377 -130
- package/dist/models/registry/models.js +3622 -2312
- package/dist/plugins/builtin/asr-ggml/config.d.ts +19 -4
- package/dist/plugins/builtin/asr-ggml/config.js +39 -3
- package/dist/plugins/builtin/asr-ggml/fit.d.ts +18 -0
- package/dist/plugins/builtin/asr-ggml/fit.js +118 -0
- package/dist/plugins/builtin/audiogen-ggml/config.d.ts +6 -0
- package/dist/plugins/builtin/audiogen-ggml/config.js +9 -0
- package/dist/plugins/builtin/audiogen-ggml/fit.d.ts +3 -0
- package/dist/{resources/model-fit/native-probe/engines/audiogen.js → plugins/builtin/audiogen-ggml/fit.js} +28 -18
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +4 -3
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.d.ts +17 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.js +68 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +11 -3
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +47 -11
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +4 -3
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +4 -3
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +203 -72
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +17 -5
- package/dist/plugins/builtin/bci-whispercpp-transcription/fit.d.ts +3 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/fit.js +22 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +71 -29
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +12 -10
- package/dist/plugins/builtin/ggml-classification/plugin.d.ts +8 -5
- package/dist/plugins/builtin/ggml-classification/plugin.js +89 -9
- package/dist/plugins/builtin/ggml-image-embedding/ops/embed.d.ts +24 -0
- package/dist/plugins/builtin/ggml-image-embedding/ops/embed.js +184 -0
- package/dist/plugins/builtin/ggml-image-embedding/plugin.d.ts +120 -0
- package/dist/plugins/builtin/ggml-image-embedding/plugin.js +39 -0
- package/dist/plugins/builtin/ggml-ocr/fit.d.ts +6 -0
- package/dist/plugins/builtin/ggml-ocr/fit.js +19 -0
- package/dist/plugins/builtin/ggml-ocr/params.d.ts +1 -1
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +20 -13
- package/dist/plugins/builtin/ggml-ocr/plugin.js +111 -25
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.d.ts +2 -2
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.js +18 -2
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +15 -15
- package/dist/plugins/builtin/ggml-vla/plugin.js +3 -3
- package/dist/plugins/builtin/index.d.ts +16 -14
- package/dist/plugins/builtin/index.js +16 -14
- package/dist/plugins/builtin/llamacpp-completion/fit.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-completion/fit.js +28 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +6 -8
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +15 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.d.ts +2 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.js +32 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +3 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +6 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +18 -3
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +94 -21
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.d.ts +5 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.js +54 -19
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +5 -3
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +9 -7
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +315 -95
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +51 -25
- package/dist/plugins/builtin/llamacpp-completion/transform.js +4 -0
- package/dist/plugins/builtin/llamacpp-decision/helpers.d.ts +0 -1
- package/dist/plugins/builtin/llamacpp-decision/helpers.js +7 -15
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.d.ts +3 -1
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.js +6 -3
- package/dist/plugins/builtin/llamacpp-decision/plugin.d.ts +43 -35
- package/dist/plugins/builtin/llamacpp-decision/plugin.js +4 -3
- package/dist/plugins/builtin/llamacpp-embedding/fit.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-embedding/fit.js +23 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +201 -10
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +52 -16
- package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +2 -2
- package/dist/plugins/builtin/llamacpp-embedding/transform.js +29 -2
- package/dist/plugins/builtin/llamacpp-rerank/helpers.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-rerank/helpers.js +16 -0
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.d.ts +7 -0
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.js +120 -0
- package/dist/plugins/builtin/llamacpp-rerank/plugin.d.ts +92 -0
- package/dist/plugins/builtin/llamacpp-rerank/plugin.js +44 -0
- package/dist/plugins/builtin/moss-transcribe/plugin.d.ts +46 -19
- package/dist/plugins/builtin/moss-transcribe/plugin.js +9 -7
- package/dist/plugins/builtin/nmtcpp-translation/fit.d.ts +3 -0
- package/dist/plugins/builtin/nmtcpp-translation/fit.js +11 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +32 -18
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +8 -6
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +234 -38
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +42 -16
- package/dist/plugins/builtin/sdcpp-generation/fit.d.ts +3 -0
- package/dist/plugins/builtin/sdcpp-generation/fit.js +68 -0
- package/dist/plugins/builtin/sdcpp-generation/load-files.d.ts +30 -0
- package/dist/plugins/builtin/sdcpp-generation/load-files.js +121 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +8 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +174 -53
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.d.ts +4 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.js +107 -22
- package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +7 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +23 -9
- package/dist/plugins/builtin/sdcpp-generation/ops/world.d.ts +6 -4
- package/dist/plugins/builtin/sdcpp-generation/ops/world.js +8 -3
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +166 -47
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +67 -155
- package/dist/plugins/builtin/sdcpp-generation/upscaler-config.d.ts +15 -0
- package/dist/plugins/builtin/sdcpp-generation/upscaler-config.js +19 -0
- package/dist/plugins/builtin/sdcpp-generation/world-config.d.ts +8 -0
- package/dist/plugins/builtin/sdcpp-generation/world-config.js +9 -0
- package/dist/plugins/builtin/tts-ggml/fit.d.ts +3 -0
- package/dist/plugins/builtin/tts-ggml/fit.js +162 -0
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +4 -1
- package/dist/plugins/builtin/tts-ggml/ops/moss-options.d.ts +1 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +3 -2
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +24 -11
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +53 -18
- package/dist/plugins/builtin/tts-ggml/ops/voice-options.d.ts +50 -0
- package/dist/plugins/builtin/tts-ggml/ops/voice-options.js +74 -0
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +415 -57
- package/dist/plugins/builtin/tts-ggml/plugin.js +183 -34
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +104 -40
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +12 -10
- package/dist/plugins/index.d.ts +1 -1
- package/dist/plugins/index.js +2 -2
- package/dist/plugins/ops/bci-transcribe.d.ts +5 -4
- package/dist/plugins/ops/bci-transcribe.js +18 -4
- package/dist/plugins/ops/cancel.d.ts +1 -0
- package/dist/plugins/ops/cancel.js +11 -2
- package/dist/plugins/ops/embed.d.ts +5 -3
- package/dist/plugins/ops/embed.js +89 -34
- package/dist/plugins/ops/kv-cache-utils.d.ts +5 -3
- package/dist/plugins/ops/kv-cache-utils.js +21 -10
- package/dist/plugins/ops/load-model.d.ts +10 -2
- package/dist/plugins/ops/load-model.js +45 -19
- package/dist/plugins/ops/transcribe.d.ts +5 -4
- package/dist/plugins/ops/transcribe.js +63 -28
- package/dist/plugins/ops/translate.d.ts +2 -1
- package/dist/plugins/ops/translate.js +7 -4
- package/dist/plugins/ops/unload-model.d.ts +17 -1
- package/dist/plugins/ops/unload-model.js +32 -3
- package/dist/plugins/turbovec-backend.d.ts +1 -1
- package/dist/plugins/turbovec-backend.js +21 -6
- package/dist/profiling/aggregator.d.ts +1 -1
- package/dist/profiling/aggregator.js +1 -1
- package/dist/profiling/controller.d.ts +1 -1
- package/dist/profiling/controller.js +2 -2
- package/dist/profiling/events.d.ts +1 -1
- package/dist/profiling/events.js +2 -2
- package/dist/profiling/exporters.d.ts +1 -1
- package/dist/profiling/exporters.js +3 -3
- package/dist/profiling/index.d.ts +13 -13
- package/dist/profiling/index.js +12 -12
- package/dist/profiling/operation-metrics.d.ts +1 -1
- package/dist/profiling/operation-metrics.js +24 -21
- package/dist/profiling/operation-wrappers.js +3 -3
- package/dist/profiling/profiler.js +1 -1
- package/dist/rag/embedding-scope.d.ts +16 -0
- package/dist/rag/embedding-scope.js +81 -0
- package/dist/rag/handler.d.ts +25 -0
- package/dist/rag/handler.js +51 -17
- package/dist/rag/handlers/close-workspace.js +1 -1
- package/dist/rag/handlers/delete-embeddings.js +1 -1
- package/dist/rag/handlers/delete-workspace.js +1 -1
- package/dist/rag/handlers/ingest.js +3 -7
- package/dist/rag/handlers/list-workspaces.d.ts +1 -1
- package/dist/rag/handlers/list-workspaces.js +1 -1
- package/dist/rag/handlers/reindex.js +1 -1
- package/dist/rag/handlers/save-embeddings.js +1 -1
- package/dist/rag/handlers/search.d.ts +10 -1
- package/dist/rag/handlers/search.js +17 -8
- package/dist/rag/index.d.ts +11 -11
- package/dist/rag/index.js +11 -11
- package/dist/rag/rag-operation-manager.d.ts +27 -3
- package/dist/rag/rag-operation-manager.js +10 -27
- package/dist/rag/rag-workspace-manager.js +1 -1
- package/dist/registry.js +7 -1
- package/dist/resources/collector.d.ts +71 -37
- package/dist/resources/collector.js +9 -2
- package/dist/resources/instance.d.ts +1 -1
- package/dist/resources/instance.js +1 -1
- package/dist/resources/native.d.ts +1 -1
- package/dist/resources/normalize.d.ts +1 -1
- package/dist/resources/types.d.ts +2 -0
- package/dist/rpc/inventory.js +1 -12
- package/dist/runtime/env.d.ts +5 -1
- package/dist/runtime/env.js +6 -21
- package/dist/runtime/index.d.ts +8 -5
- package/dist/runtime/index.js +4 -2
- package/dist/runtime/lifecycle.d.ts +6 -4
- package/dist/runtime/lifecycle.js +16 -12
- package/dist/runtime/model-config-utils.js +8 -1
- package/dist/runtime/model-registry.d.ts +30 -4
- package/dist/runtime/model-registry.js +48 -2
- package/dist/runtime/model-work-lock.d.ts +13 -0
- package/dist/runtime/model-work-lock.js +48 -0
- package/dist/runtime/registry-client.d.ts +47 -1
- package/dist/runtime/registry-client.js +446 -88
- package/dist/runtime/request-context.d.ts +11 -3
- package/dist/runtime/request-context.js +39 -2
- package/dist/runtime/request-registry.d.ts +56 -8
- package/dist/runtime/request-registry.js +190 -72
- package/dist/runtime/runtime-lifecycle.js +10 -1
- package/dist/runtime/session-id.d.ts +18 -0
- package/dist/runtime/session-id.js +20 -0
- package/dist/runtime/session-registry.d.ts +39 -0
- package/dist/runtime/session-registry.js +114 -0
- package/dist/runtime/state.d.ts +1 -1
- package/dist/runtime/state.js +2 -2
- package/dist/runtime/vector-index-backend.d.ts +6 -2
- package/dist/runtime/vector-index-registry.d.ts +26 -5
- package/dist/runtime/vector-index-registry.js +88 -27
- package/dist/schemas/abort-signal.d.ts +8 -0
- package/dist/schemas/assess-model-fit.d.ts +138 -282
- package/dist/schemas/assess-model-fit.js +85 -219
- package/dist/schemas/audio-gen.d.ts +291 -25
- package/dist/schemas/audio-gen.js +130 -26
- package/dist/schemas/batch-completion-stream.d.ts +113 -15
- package/dist/schemas/batch-completion-stream.js +10 -6
- package/dist/schemas/bci-config.d.ts +3 -1
- package/dist/schemas/bci-config.js +2 -2
- package/dist/schemas/bci.d.ts +55 -4
- package/dist/schemas/bci.js +13 -18
- package/dist/schemas/calc-tokens.d.ts +141 -0
- package/dist/schemas/calc-tokens.js +53 -0
- package/dist/schemas/cancel.d.ts +31 -0
- package/dist/schemas/cancel.js +11 -2
- package/dist/schemas/classification.d.ts +3 -0
- package/dist/schemas/classification.js +6 -2
- package/dist/schemas/common.d.ts +2209 -348
- package/dist/schemas/common.js +52 -33
- package/dist/schemas/completion-event.d.ts +11 -2
- package/dist/schemas/completion-event.js +5 -2
- package/dist/schemas/completion-stream.d.ts +173 -19
- package/dist/schemas/completion-stream.js +117 -20
- package/dist/schemas/config.d.ts +2064 -178
- package/dist/schemas/config.js +21 -23
- package/dist/schemas/decision.d.ts +12 -2
- package/dist/schemas/decision.js +16 -1
- package/dist/schemas/download-asset.d.ts +13 -6
- package/dist/schemas/download-asset.js +4 -7
- package/dist/schemas/embed.d.ts +286 -3
- package/dist/schemas/embed.js +162 -16
- package/dist/schemas/engine-addon-map.d.ts +3 -1
- package/dist/schemas/engine-addon-map.js +5 -3
- package/dist/schemas/errors.d.ts +13 -0
- package/dist/schemas/errors.js +71 -1
- package/dist/schemas/finetune.d.ts +14 -4
- package/dist/schemas/finetune.js +5 -6
- package/dist/schemas/heartbeat.d.ts +1 -0
- package/dist/schemas/heartbeat.js +5 -2
- package/dist/schemas/image-embedding.d.ts +27 -0
- package/dist/schemas/image-embedding.js +107 -0
- package/dist/schemas/index.d.ts +55 -50
- package/dist/schemas/index.js +54 -49
- package/dist/schemas/lifecycle.d.ts +3 -0
- package/dist/schemas/lifecycle.js +13 -6
- package/dist/schemas/llamacpp-config.d.ts +443 -20
- package/dist/schemas/llamacpp-config.js +188 -14
- package/dist/schemas/load-model.d.ts +8940 -1952
- package/dist/schemas/load-model.js +73 -9
- package/dist/schemas/logging-stream.d.ts +1 -0
- package/dist/schemas/logging-stream.js +5 -2
- package/dist/schemas/mcp-adapter.d.ts +1 -1
- package/dist/schemas/model-config-schemas.d.ts +731 -111
- package/dist/schemas/model-config-schemas.js +16 -12
- package/dist/schemas/model-fit-policy.d.ts +2 -2
- package/dist/schemas/model-info.d.ts +54 -56
- package/dist/schemas/model-info.js +18 -7
- package/dist/schemas/model-ops.d.ts +12 -0
- package/dist/schemas/model-ops.js +33 -5
- package/dist/schemas/model-src-utils.d.ts +14 -6
- package/dist/schemas/model-src-utils.js +2 -2
- package/dist/schemas/model-types.d.ts +12 -4
- package/dist/schemas/model-types.js +2 -0
- package/dist/schemas/ocr.d.ts +11 -4
- package/dist/schemas/ocr.js +7 -3
- package/dist/schemas/plugin.d.ts +70 -8
- package/dist/schemas/plugin.js +32 -4
- package/dist/schemas/profiling.d.ts +32 -32
- package/dist/schemas/profiling.js +1 -1
- package/dist/schemas/rag.d.ts +9 -0
- package/dist/schemas/rag.js +11 -17
- package/dist/schemas/registry.d.ts +38 -10
- package/dist/schemas/registry.js +16 -7
- package/dist/schemas/reload-config.d.ts +45 -6
- package/dist/schemas/reload-config.js +13 -6
- package/dist/schemas/request-base.d.ts +39 -0
- package/dist/schemas/request-base.js +64 -0
- package/dist/schemas/rerank.d.ts +132 -0
- package/dist/schemas/rerank.js +104 -0
- package/dist/schemas/rpc-server.d.ts +3 -0
- package/dist/schemas/rpc-server.js +15 -15
- package/dist/schemas/sdcpp-config.d.ts +144 -18
- package/dist/schemas/sdcpp-config.js +123 -28
- package/dist/schemas/session.d.ts +37 -0
- package/dist/schemas/session.js +53 -0
- package/dist/schemas/system-resources.d.ts +364 -179
- package/dist/schemas/system-resources.js +8 -3
- package/dist/schemas/text-to-speech.d.ts +1408 -275
- package/dist/schemas/text-to-speech.js +547 -33
- package/dist/schemas/tools.d.ts +7 -0
- package/dist/schemas/tools.js +29 -0
- package/dist/schemas/transcription-config.d.ts +267 -2
- package/dist/schemas/transcription-config.js +138 -7
- package/dist/schemas/transcription.d.ts +94 -7
- package/dist/schemas/transcription.js +99 -22
- package/dist/schemas/translate.d.ts +4 -2
- package/dist/schemas/translate.js +6 -10
- package/dist/schemas/translation-config.d.ts +40 -20
- package/dist/schemas/translation-config.js +1 -1
- package/dist/schemas/tts-pcm.d.ts +35 -0
- package/dist/schemas/tts-pcm.js +69 -0
- package/dist/schemas/vector-index.d.ts +47 -1
- package/dist/schemas/vector-index.js +37 -10
- package/dist/schemas/vla.d.ts +3 -3
- package/dist/surface.d.ts +4 -3
- package/dist/surface.js +4 -3
- package/dist/utils/addon-responses.d.ts +71 -2
- package/dist/utils/aggregate-events.d.ts +1 -1
- package/dist/utils/aggregate-events.js +2 -2
- package/dist/utils/archive.d.ts +7 -0
- package/dist/utils/archive.js +89 -30
- package/dist/utils/asr-events.d.ts +5 -0
- package/dist/utils/asr-events.js +10 -1
- package/dist/utils/audio/decoder.js +3 -1
- package/dist/utils/audio-input.js +1 -1
- package/dist/utils/cache/auto-history.d.ts +2 -1
- package/dist/utils/cache/auto-history.js +5 -3
- package/dist/utils/cache/layout.d.ts +48 -0
- package/dist/utils/cache/layout.js +190 -0
- package/dist/utils/cache/paths.js +8 -10
- package/dist/utils/cache/types.d.ts +2 -1
- package/dist/utils/checksum.d.ts +7 -0
- package/dist/utils/checksum.js +59 -2
- package/dist/utils/completion-normalizer.js +287 -88
- package/dist/utils/gguf-architecture.d.ts +7 -0
- package/dist/utils/gguf-architecture.js +142 -0
- package/dist/utils/gguf-tensor-extractor.d.ts +0 -6
- package/dist/utils/gguf-tensor-extractor.js +1 -50
- package/dist/utils/index.d.ts +13 -13
- package/dist/utils/index.js +13 -13
- package/dist/utils/mcp-adapter.d.ts +1 -1
- package/dist/utils/mcp-adapter.js +1 -1
- package/dist/utils/path-security.js +1 -1
- package/dist/utils/shard-utils.d.ts +0 -16
- package/dist/utils/shard-utils.js +14 -48
- package/dist/utils/tool-integration.d.ts +3 -2
- package/dist/utils/tool-integration.js +3 -6
- package/dist/utils/tools/defer.d.ts +50 -31
- package/dist/utils/tools/defer.js +115 -51
- package/dist/utils/tools/dialect.d.ts +15 -4
- package/dist/utils/tools/dialect.js +34 -5
- package/dist/utils/tools/index.d.ts +3 -3
- package/dist/utils/tools/index.js +3 -3
- package/dist/utils/tools/parser.js +8 -8
- package/dist/utils/tools/parsers/dsml.d.ts +1 -1
- package/dist/utils/tools/parsers/dsml.js +1 -1
- package/dist/utils/tools/parsers/gemma4native.d.ts +1 -1
- package/dist/utils/tools/parsers/gemma4native.js +1 -1
- package/dist/utils/tools/parsers/harmony.d.ts +1 -1
- package/dist/utils/tools/parsers/harmony.js +1 -1
- package/dist/utils/tools/parsers/hermes.d.ts +1 -1
- package/dist/utils/tools/parsers/hermes.js +1 -1
- package/dist/utils/tools/parsers/json.d.ts +1 -1
- package/dist/utils/tools/parsers/json.js +1 -1
- package/dist/utils/tools/parsers/pythonic.d.ts +1 -1
- package/dist/utils/tools/parsers/pythonic.js +1 -1
- package/dist/utils/tools/parsers/qwen35.d.ts +1 -1
- package/dist/utils/tools/parsers/qwen35.js +1 -1
- package/dist/utils/transcribe-metadata.d.ts +28 -2
- package/dist/utils/transcribe-metadata.js +16 -0
- package/dist/utils/tts-stats.d.ts +43 -1
- package/dist/utils/tts-stats.js +120 -22
- package/dist/utils/until-aborted.d.ts +7 -0
- package/dist/utils/until-aborted.js +34 -0
- package/package.json +67 -55
- package/dist/models/registry/resource-profiles.d.ts +0 -17
- package/dist/models/registry/resource-profiles.js +0 -5657
- package/dist/resources/model-fit/assess.d.ts +0 -72
- package/dist/resources/model-fit/assess.js +0 -679
- package/dist/resources/model-fit/fit-stub/project-fit-from-load.d.ts +0 -27
- package/dist/resources/model-fit/fit-stub/project-fit-from-load.js +0 -33
- package/dist/resources/model-fit/floor.d.ts +0 -31
- package/dist/resources/model-fit/floor.js +0 -65
- package/dist/resources/model-fit/kv-cache.d.ts +0 -23
- package/dist/resources/model-fit/kv-cache.js +0 -85
- package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +0 -54
- package/dist/resources/model-fit/native-probe/advisory-fit.js +0 -248
- package/dist/resources/model-fit/native-probe/child-engine-fit.d.ts +0 -2
- package/dist/resources/model-fit/native-probe/child-engine-fit.js +0 -49
- package/dist/resources/model-fit/native-probe/classify-fit.d.ts +0 -6
- package/dist/resources/model-fit/native-probe/classify-fit.js +0 -166
- package/dist/resources/model-fit/native-probe/create-fit-request.d.ts +0 -31
- package/dist/resources/model-fit/native-probe/create-fit-request.js +0 -61
- package/dist/resources/model-fit/native-probe/engine-fit.d.ts +0 -73
- package/dist/resources/model-fit/native-probe/engine-fit.js +0 -13
- package/dist/resources/model-fit/native-probe/engines/asr.d.ts +0 -14
- package/dist/resources/model-fit/native-probe/engines/asr.js +0 -126
- package/dist/resources/model-fit/native-probe/engines/audiogen.d.ts +0 -8
- package/dist/resources/model-fit/native-probe/engines/diffusion.d.ts +0 -7
- package/dist/resources/model-fit/native-probe/engines/diffusion.js +0 -65
- package/dist/resources/model-fit/native-probe/engines/llama.d.ts +0 -12
- package/dist/resources/model-fit/native-probe/engines/llama.js +0 -41
- package/dist/resources/model-fit/native-probe/engines/nmt.d.ts +0 -8
- package/dist/resources/model-fit/native-probe/engines/nmt.js +0 -11
- package/dist/resources/model-fit/native-probe/engines/ocr.d.ts +0 -9
- package/dist/resources/model-fit/native-probe/engines/ocr.js +0 -13
- package/dist/resources/model-fit/native-probe/engines/tts.d.ts +0 -10
- package/dist/resources/model-fit/native-probe/engines/tts.js +0 -152
- package/dist/resources/model-fit/native-probe/fit-outcome.d.ts +0 -16
- package/dist/resources/model-fit/native-probe/fit-process.d.ts +0 -27
- package/dist/resources/model-fit/native-probe/fit-process.js +0 -55
- package/dist/resources/model-fit/native-probe/fit-runner.d.ts +0 -1
- package/dist/resources/model-fit/native-probe/fit-runner.js +0 -58
- package/dist/resources/model-fit/native-probe/run-fit.d.ts +0 -24
- package/dist/resources/model-fit/native-probe/run-fit.js +0 -20
- package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +0 -24
- package/dist/resources/model-fit/native-probe/run-in-process-fit.js +0 -123
- package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +0 -63
- package/dist/resources/model-fit/native-probe/run-isolated-fit.js +0 -377
- package/dist/resources/model-fit/platform.d.ts +0 -9
- package/dist/resources/model-fit/platform.js +0 -26
- package/dist/resources/model-fit/types.d.ts +0 -6
- package/dist/resources/model-fit/types.js +0 -1
- package/dist/schemas/model-resource-profile.d.ts +0 -108
- package/dist/schemas/model-resource-profile.js +0 -80
- /package/dist/{resources/model-fit/native-probe/engines → model-fit}/gpu.d.ts +0 -0
- /package/dist/{resources/model-fit/native-probe/engines → model-fit}/gpu.js +0 -0
- /package/dist/{resources/model-fit/native-probe/fit-outcome.js → schemas/abort-signal.js} +0 -0
|
@@ -1,8 +1,7 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { createReadStream } from 'bare-fs';
|
|
2
2
|
import Buffer from 'bare-buffer';
|
|
3
3
|
import { getEngineLogger } from '../logging/index.js';
|
|
4
4
|
import { ModelLoadFailedError } from '../errors/index.js';
|
|
5
|
-
import { validateAndJoinPath } from './path-security.js';
|
|
6
5
|
const logger = getEngineLogger();
|
|
7
6
|
// GGUF value types
|
|
8
7
|
const GGUF_TYPE = {
|
|
@@ -244,54 +243,6 @@ async function extractTensorNamesFromHeader(filePath) {
|
|
|
244
243
|
destroyStream();
|
|
245
244
|
}
|
|
246
245
|
}
|
|
247
|
-
/**
|
|
248
|
-
* Extract tensor names from all GGUF shards and write to {baseFilename}.tensors.txt
|
|
249
|
-
* Required for sharded models to enable incremental/async loading
|
|
250
|
-
* @throws ModelLoadFailedError if any shard fails extraction
|
|
251
|
-
*/
|
|
252
|
-
export async function extractAndWriteTensorsFile(shardDir, shardFilenames, baseFilename) {
|
|
253
|
-
const tensorsFilePath = validateAndJoinPath(shardDir, `${baseFilename}.tensors.txt`);
|
|
254
|
-
try {
|
|
255
|
-
await fsPromises.access(tensorsFilePath);
|
|
256
|
-
logger.info(`Tensors file already exists: ${tensorsFilePath}`);
|
|
257
|
-
return tensorsFilePath;
|
|
258
|
-
}
|
|
259
|
-
catch {
|
|
260
|
-
// Continue with extraction
|
|
261
|
-
}
|
|
262
|
-
logger.info(`Extracting tensors from ${shardFilenames.length} shards...`);
|
|
263
|
-
const allTensorNames = new Set();
|
|
264
|
-
const failedShards = [];
|
|
265
|
-
for (const filename of shardFilenames) {
|
|
266
|
-
const shardPath = validateAndJoinPath(shardDir, filename);
|
|
267
|
-
try {
|
|
268
|
-
logger.debug(`Extracting tensors from ${filename}...`);
|
|
269
|
-
const tensorNames = await extractTensorNamesFromHeader(shardPath);
|
|
270
|
-
if (tensorNames.length > 0) {
|
|
271
|
-
tensorNames.forEach((name) => allTensorNames.add(name));
|
|
272
|
-
logger.info(`Extracted ${tensorNames.length} tensors from ${filename}`);
|
|
273
|
-
}
|
|
274
|
-
else {
|
|
275
|
-
failedShards.push(filename);
|
|
276
|
-
logger.error(`No tensors found in ${filename}`);
|
|
277
|
-
}
|
|
278
|
-
}
|
|
279
|
-
catch (error) {
|
|
280
|
-
failedShards.push(filename);
|
|
281
|
-
logger.error(`Failed to extract tensors from ${filename}:`, error);
|
|
282
|
-
}
|
|
283
|
-
}
|
|
284
|
-
if (failedShards.length > 0) {
|
|
285
|
-
throw new ModelLoadFailedError(`Failed to extract tensors from ${failedShards.length}/${shardFilenames.length} shards: ${failedShards.join(', ')}`);
|
|
286
|
-
}
|
|
287
|
-
if (allTensorNames.size === 0) {
|
|
288
|
-
throw new ModelLoadFailedError(`Could not extract any tensors from shards.`);
|
|
289
|
-
}
|
|
290
|
-
const tensorNames = Array.from(allTensorNames).sort();
|
|
291
|
-
await fsPromises.writeFile(tensorsFilePath, tensorNames.join('\n') + '\n', 'utf8');
|
|
292
|
-
logger.info(`Tensors file created: ${tensorsFilePath} (${tensorNames.length} tensors from ${shardFilenames.length} shards)`);
|
|
293
|
-
return tensorsFilePath;
|
|
294
|
-
}
|
|
295
246
|
/**
|
|
296
247
|
* Check if a file has a valid GGUF header structure.
|
|
297
248
|
* Returns true if the header parses successfully and contains tensor metadata.
|
package/dist/utils/index.d.ts
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
|
-
export * from '
|
|
2
|
-
export * from '
|
|
3
|
-
export * from '
|
|
4
|
-
export * from '
|
|
5
|
-
export * from '
|
|
6
|
-
export * from '
|
|
7
|
-
export * from '
|
|
8
|
-
export * from '
|
|
9
|
-
export * from '
|
|
10
|
-
export * from '
|
|
11
|
-
export * from '
|
|
12
|
-
export * from '
|
|
13
|
-
export * from '
|
|
1
|
+
export * from '../utils/archive.js';
|
|
2
|
+
export * from '../utils/cache/types.js';
|
|
3
|
+
export * from '../utils/cache/auto-history.js';
|
|
4
|
+
export * from '../utils/cache/paths.js';
|
|
5
|
+
export * from '../utils/checksum.js';
|
|
6
|
+
export * from '../utils/config-transform.js';
|
|
7
|
+
export * from '../utils/formatting.js';
|
|
8
|
+
export * from '../utils/gguf-tensor-extractor.js';
|
|
9
|
+
export * from '../utils/image-dimensions.js';
|
|
10
|
+
export * from '../utils/path-sanitize.js';
|
|
11
|
+
export * from '../utils/path-security.js';
|
|
12
|
+
export * from '../utils/shard-utils.js';
|
|
13
|
+
export * from '../utils/url-security.js';
|
package/dist/utils/index.js
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
|
-
export * from '
|
|
2
|
-
export * from '
|
|
3
|
-
export * from '
|
|
4
|
-
export * from '
|
|
5
|
-
export * from '
|
|
6
|
-
export * from '
|
|
7
|
-
export * from '
|
|
8
|
-
export * from '
|
|
9
|
-
export * from '
|
|
10
|
-
export * from '
|
|
11
|
-
export * from '
|
|
12
|
-
export * from '
|
|
13
|
-
export * from '
|
|
1
|
+
export * from '../utils/archive.js';
|
|
2
|
+
export * from '../utils/cache/types.js';
|
|
3
|
+
export * from '../utils/cache/auto-history.js';
|
|
4
|
+
export * from '../utils/cache/paths.js';
|
|
5
|
+
export * from '../utils/checksum.js';
|
|
6
|
+
export * from '../utils/config-transform.js';
|
|
7
|
+
export * from '../utils/formatting.js';
|
|
8
|
+
export * from '../utils/gguf-tensor-extractor.js';
|
|
9
|
+
export * from '../utils/image-dimensions.js';
|
|
10
|
+
export * from '../utils/path-sanitize.js';
|
|
11
|
+
export * from '../utils/path-security.js';
|
|
12
|
+
export * from '../utils/shard-utils.js';
|
|
13
|
+
export * from '../utils/url-security.js';
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import type { Tool } from '../schemas/tools.js';
|
|
2
2
|
import type { McpClientInput } from '../schemas/mcp-adapter.js';
|
|
3
|
-
import type { ToolHandler } from '
|
|
3
|
+
import type { ToolHandler } from '../utils/tool-helpers.js';
|
|
4
4
|
export type { McpClient, McpClientInput } from '../schemas/mcp-adapter.js';
|
|
5
5
|
export type ToolHandlerMap = Map<string, ToolHandler>;
|
|
6
6
|
export type McpToolsResult = {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import path from 'bare-path';
|
|
2
2
|
import { PathTraversalError } from '../errors/index.js';
|
|
3
|
-
import { sanitizePathComponent } from '
|
|
3
|
+
import { sanitizePathComponent } from '../utils/path-sanitize.js';
|
|
4
4
|
/**
|
|
5
5
|
* Check whether a resolved target path is contained within a base directory.
|
|
6
6
|
* Portable — caller provides the resolve function and separator for their runtime.
|
|
@@ -23,15 +23,6 @@ export declare function parsePatternBasedShardUrl(shardUrl: string): {
|
|
|
23
23
|
shardUrls: ShardUrl[];
|
|
24
24
|
cacheKey: string;
|
|
25
25
|
};
|
|
26
|
-
/**
|
|
27
|
-
* Extract tensor information from all shards and write to baseFilename.tensors.txt
|
|
28
|
-
* Required for sharded models to enable incremental/async loading
|
|
29
|
-
* @param shardDir - Directory containing the shard files
|
|
30
|
-
* @param shardFilename - Filename of any shard (e.g., "model-00001-of-00002.gguf")
|
|
31
|
-
* @returns Path to the created tensors.txt file
|
|
32
|
-
* @throws If not a sharded model or extraction fails
|
|
33
|
-
*/
|
|
34
|
-
export declare function extractTensorsFromShards(shardDir: string, shardFilename: string): Promise<string>;
|
|
35
26
|
/**
|
|
36
27
|
* Check if all numbered shard files exist
|
|
37
28
|
* @param shardDir - Directory containing the shard files
|
|
@@ -39,10 +30,3 @@ export declare function extractTensorsFromShards(shardDir: string, shardFilename
|
|
|
39
30
|
* @returns true if all numbered shards exist, false otherwise
|
|
40
31
|
*/
|
|
41
32
|
export declare function checkAllShardsExist(shardDir: string, shardFilename: string): Promise<boolean>;
|
|
42
|
-
/**
|
|
43
|
-
* Validate that all shards and tensors.txt exist for a sharded model
|
|
44
|
-
* @param shardDir - Directory containing the shard files
|
|
45
|
-
* @param shardFilename - Filename of any shard (e.g., "model-00001-of-00002.gguf")
|
|
46
|
-
* @returns true if all shards and tensors.txt exist, false otherwise
|
|
47
|
-
*/
|
|
48
|
-
export declare function validateShardedModelCache(shardDir: string, shardFilename: string): Promise<boolean>;
|
|
@@ -1,14 +1,13 @@
|
|
|
1
1
|
import { ModelLoadFailedError, InvalidShardUrlPatternError } from '../errors/index.js';
|
|
2
|
-
import { generateShortHash } from '
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
2
|
+
import { generateShortHash } from '../utils/formatting.js';
|
|
3
|
+
import { SHARD_FILENAME_PATTERN } from '../utils/cache/layout.js';
|
|
4
|
+
import { validateAndJoinPath } from '../utils/path-security.js';
|
|
5
5
|
import { promises as fsPromises } from 'bare-fs';
|
|
6
6
|
/**
|
|
7
7
|
* Detect if model filename follows shard pattern (00001-of-0000x)
|
|
8
8
|
*/
|
|
9
9
|
export function detectShardedModel(filename) {
|
|
10
|
-
const
|
|
11
|
-
const match = filename.match(shardPattern);
|
|
10
|
+
const match = filename.match(SHARD_FILENAME_PATTERN);
|
|
12
11
|
if (match && match[1] && match[2] && match[3] && match[4]) {
|
|
13
12
|
return {
|
|
14
13
|
isSharded: true,
|
|
@@ -79,22 +78,6 @@ export function parsePatternBasedShardUrl(shardUrl) {
|
|
|
79
78
|
cacheKey: generateShortHash(`${baseUrl}/${shardInfo.baseFilename}`)
|
|
80
79
|
};
|
|
81
80
|
}
|
|
82
|
-
/**
|
|
83
|
-
* Extract tensor information from all shards and write to baseFilename.tensors.txt
|
|
84
|
-
* Required for sharded models to enable incremental/async loading
|
|
85
|
-
* @param shardDir - Directory containing the shard files
|
|
86
|
-
* @param shardFilename - Filename of any shard (e.g., "model-00001-of-00002.gguf")
|
|
87
|
-
* @returns Path to the created tensors.txt file
|
|
88
|
-
* @throws If not a sharded model or extraction fails
|
|
89
|
-
*/
|
|
90
|
-
export async function extractTensorsFromShards(shardDir, shardFilename) {
|
|
91
|
-
const shardInfo = detectShardedModel(shardFilename);
|
|
92
|
-
if (!shardInfo.isSharded || !shardInfo.baseFilename || !shardInfo.totalShards) {
|
|
93
|
-
throw new ModelLoadFailedError(`Not a sharded model filename: ${shardFilename}`);
|
|
94
|
-
}
|
|
95
|
-
const allShardFilenames = generateShardFilenames(shardFilename);
|
|
96
|
-
return extractAndWriteTensorsFile(shardDir, allShardFilenames, shardInfo.baseFilename);
|
|
97
|
-
}
|
|
98
81
|
/**
|
|
99
82
|
* Check if all numbered shard files exist
|
|
100
83
|
* @param shardDir - Directory containing the shard files
|
|
@@ -106,32 +89,15 @@ export async function checkAllShardsExist(shardDir, shardFilename) {
|
|
|
106
89
|
if (!shardInfo.isSharded) {
|
|
107
90
|
return false;
|
|
108
91
|
}
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
* @param shardDir - Directory containing the shard files
|
|
119
|
-
* @param shardFilename - Filename of any shard (e.g., "model-00001-of-00002.gguf")
|
|
120
|
-
* @returns true if all shards and tensors.txt exist, false otherwise
|
|
121
|
-
*/
|
|
122
|
-
export async function validateShardedModelCache(shardDir, shardFilename) {
|
|
123
|
-
const shardInfo = detectShardedModel(shardFilename);
|
|
124
|
-
if (!shardInfo.isSharded || !shardInfo.baseFilename) {
|
|
125
|
-
return false;
|
|
126
|
-
}
|
|
127
|
-
const allShardsExist = await checkAllShardsExist(shardDir, shardFilename);
|
|
128
|
-
if (!allShardsExist) {
|
|
129
|
-
return false;
|
|
92
|
+
// Sequential with an early exit: the shard count comes from the filename, so a
|
|
93
|
+
// crafted name must not fan out one filesystem call per claimed shard.
|
|
94
|
+
for (const f of generateShardFilenames(shardFilename)) {
|
|
95
|
+
try {
|
|
96
|
+
await fsPromises.access(validateAndJoinPath(shardDir, f));
|
|
97
|
+
}
|
|
98
|
+
catch {
|
|
99
|
+
return false;
|
|
100
|
+
}
|
|
130
101
|
}
|
|
131
|
-
|
|
132
|
-
const tensorsExist = await fsPromises
|
|
133
|
-
.access(validateAndJoinPath(shardDir, tensorsFile))
|
|
134
|
-
.then(() => true)
|
|
135
|
-
.catch(() => false);
|
|
136
|
-
return tensorsExist;
|
|
102
|
+
return true;
|
|
137
103
|
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
import type { Tool, ToolDialect } from '../schemas/index.js';
|
|
1
|
+
import type { HistoryContent, Tool, ToolDialect } from '../schemas/index.js';
|
|
2
2
|
interface HistoryMessage {
|
|
3
3
|
role: string;
|
|
4
|
-
content:
|
|
4
|
+
content: HistoryContent;
|
|
5
5
|
attachments?: {
|
|
6
6
|
path: string;
|
|
7
7
|
}[] | undefined;
|
|
@@ -12,5 +12,6 @@ interface HistoryMessage {
|
|
|
12
12
|
* whole chat session.
|
|
13
13
|
*/
|
|
14
14
|
export declare function prependToolsToHistory(history: HistoryMessage[], tools: Tool[]): Array<HistoryMessage | Tool>;
|
|
15
|
+
/** The tool-call dialect resolved when the model loaded. */
|
|
15
16
|
export declare function detectToolDialect(modelId: string): ToolDialect;
|
|
16
17
|
export {};
|
|
@@ -1,5 +1,4 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import { getModelInfo } from '../runtime/model-registry.js';
|
|
1
|
+
import { getModelEntry } from '../runtime/model-registry.js';
|
|
3
2
|
/**
|
|
4
3
|
* Prepend tools right after the system message (or at the very start when no
|
|
5
4
|
* system message is present). The tool block stays in the kv-cache for the
|
|
@@ -12,9 +11,7 @@ export function prependToolsToHistory(history, tools) {
|
|
|
12
11
|
}
|
|
13
12
|
return [...tools, ...history];
|
|
14
13
|
}
|
|
14
|
+
/** The tool-call dialect resolved when the model loaded. */
|
|
15
15
|
export function detectToolDialect(modelId) {
|
|
16
|
-
|
|
17
|
-
if (!info)
|
|
18
|
-
return 'hermes';
|
|
19
|
-
return detectToolDialectFromName(info.name, info.path);
|
|
16
|
+
return getModelEntry(modelId)?.local.toolDialect ?? 'hermes';
|
|
20
17
|
}
|
|
@@ -1,5 +1,7 @@
|
|
|
1
|
-
import type {
|
|
2
|
-
import { type
|
|
1
|
+
import type { HistoryContent } from '../../schemas/completion-stream.js';
|
|
2
|
+
import { type Tool, type ToolSearchOptions } from '../../schemas/tools.js';
|
|
3
|
+
import type { McpClientInput } from '../../schemas/mcp-adapter.js';
|
|
4
|
+
import { type ToolInput } from '../../utils/tool-helpers.js';
|
|
3
5
|
/**
|
|
4
6
|
* Reserved name of the built-in search tool. A caller cannot register a tool
|
|
5
7
|
* under this name, and it is never itself deferred — the model needs one
|
|
@@ -8,8 +10,17 @@ import { type ToolInput } from '../tool-helpers.js';
|
|
|
8
10
|
export declare const TOOL_SEARCH_NAME = "tool_search";
|
|
9
11
|
export type HistoryMessage = {
|
|
10
12
|
role: string;
|
|
11
|
-
content:
|
|
13
|
+
content: HistoryContent;
|
|
12
14
|
};
|
|
15
|
+
export type ToolSearchLimits = {
|
|
16
|
+
defaultLimit: number;
|
|
17
|
+
maxLimit: number;
|
|
18
|
+
};
|
|
19
|
+
/**
|
|
20
|
+
* The limits a request's `toolSearch` options resolve to. Both the prompt and
|
|
21
|
+
* the search read them from here, so what the model is told is what applies.
|
|
22
|
+
*/
|
|
23
|
+
export declare function resolveToolSearchLimits(options?: ToolSearchOptions): ToolSearchLimits;
|
|
13
24
|
export declare function isDeferred(tool: Tool): boolean;
|
|
14
25
|
export declare function partitionTools(tools: readonly Tool[]): {
|
|
15
26
|
alwaysLoaded: Tool[];
|
|
@@ -34,12 +45,16 @@ export declare function buildCatalog(deferred: readonly Tool[]): string;
|
|
|
34
45
|
* tool whose schema the model has never seen. Description text cannot be
|
|
35
46
|
* called, which is the whole point.
|
|
36
47
|
*/
|
|
37
|
-
export declare function buildToolSearchTool(deferred: readonly Tool[]): Tool;
|
|
48
|
+
export declare function buildToolSearchTool(deferred: readonly Tool[], options?: ToolSearchOptions): Tool;
|
|
38
49
|
/**
|
|
39
50
|
* Rank the deferred inventory against a query. Exact name first, then text
|
|
40
51
|
* match. Ties keep registration order so repeated searches are deterministic.
|
|
52
|
+
*
|
|
53
|
+
* An exact name loads that tool alone. Otherwise a match scoring under half of
|
|
54
|
+
* the best is left out: one query word in a description or group is not enough
|
|
55
|
+
* to load a tool beside a strong match, while tools tied at the top load together.
|
|
41
56
|
*/
|
|
42
|
-
export declare function searchDeferredTools(deferred: readonly Tool[], query: string, limit?: number): Tool[];
|
|
57
|
+
export declare function searchDeferredTools(deferred: readonly Tool[], query: string, limit?: number, options?: ToolSearchOptions): Tool[];
|
|
43
58
|
export declare function buildToolSearchResult(query: string, matches: readonly Tool[], alreadyLoaded: readonly string[]): string;
|
|
44
59
|
/**
|
|
45
60
|
* Which deferred definitions this conversation already carries.
|
|
@@ -50,44 +65,48 @@ export declare function buildToolSearchResult(query: string, matches: readonly T
|
|
|
50
65
|
*/
|
|
51
66
|
export declare function loadedToolNames(history: readonly HistoryMessage[]): Set<string>;
|
|
52
67
|
export type DeferredResolution = {
|
|
53
|
-
/**
|
|
68
|
+
/**
|
|
69
|
+
* Definitions that go into the prompt and the parser: always-loaded tools,
|
|
70
|
+
* `tool_search`, and whatever history has loaded.
|
|
71
|
+
*/
|
|
54
72
|
toolsToRender: Tool[];
|
|
55
|
-
/**
|
|
56
|
-
callableTools: Tool[];
|
|
57
|
-
/** The full deferred inventory, for `tool_search` to search over. */
|
|
73
|
+
/** The deferred tools `tool_search` searches over. */
|
|
58
74
|
deferred: Tool[];
|
|
59
75
|
};
|
|
60
76
|
/**
|
|
61
|
-
* Split a registered tool list into what the prompt carries and what
|
|
62
|
-
*
|
|
77
|
+
* Split a registered tool list into what the prompt carries and what
|
|
78
|
+
* `tool_search` searches over.
|
|
63
79
|
*
|
|
64
|
-
*
|
|
65
|
-
*
|
|
66
|
-
*
|
|
67
|
-
*
|
|
80
|
+
* A loaded definition is declared in the tools block like any other tool, in
|
|
81
|
+
* registration order. A schema that sits only in a `tool` result, outside the
|
|
82
|
+
* block and the tool-call grammar, is not reliably called by small models: they
|
|
83
|
+
* search for it again instead. Declaring it changes the block at the front of
|
|
84
|
+
* the prompt, so the turn after a search re-prefills from there.
|
|
68
85
|
*
|
|
69
86
|
* Returns `null` when nothing defers, so the existing path is untouched.
|
|
70
87
|
*/
|
|
71
|
-
export declare function resolveDeferredTools(tools: readonly Tool[] | undefined, history: readonly HistoryMessage[]): DeferredResolution | null;
|
|
88
|
+
export declare function resolveDeferredTools(tools: readonly Tool[] | undefined, history: readonly HistoryMessage[], options?: ToolSearchOptions): DeferredResolution | null;
|
|
89
|
+
/** What `executeToolSearch` searches: the params passed to `completion()`. */
|
|
90
|
+
export type ToolSearchSource = {
|
|
91
|
+
tools?: readonly Tool[] | readonly ToolInput[] | undefined;
|
|
92
|
+
toolSearch?: ToolSearchOptions | undefined;
|
|
93
|
+
mcp?: readonly McpClientInput[] | undefined;
|
|
94
|
+
};
|
|
72
95
|
/**
|
|
73
|
-
*
|
|
74
|
-
*
|
|
75
|
-
*
|
|
76
|
-
* loaded definition lives in the history rather than that block, so under the
|
|
77
|
-
* default grammar the model can never produce the name it just loaded. Unless
|
|
78
|
-
* the caller asked for more than the default, those turns run with the grammar
|
|
79
|
-
* off and the parser, which does accept loaded names, validates the call.
|
|
96
|
+
* Resolve completion params into a source `executeToolSearch` can search,
|
|
97
|
+
* listing MCP tools first as `completion()` does. Needed only when the params
|
|
98
|
+
* carry `mcp`; resolve once per conversation and reuse the result.
|
|
80
99
|
*/
|
|
81
|
-
export declare function
|
|
82
|
-
tool_choice?: string | undefined;
|
|
83
|
-
}>(generationParams: T | undefined, resolution: DeferredResolution | null): T | undefined;
|
|
100
|
+
export declare function resolveToolSearchSource(params: ToolSearchSource): Promise<ToolSearchSource>;
|
|
84
101
|
/**
|
|
85
102
|
* Run a `tool_search` call and produce the `tool` message that carries its
|
|
86
103
|
* result. Pure over the history it is given; the caller appends the message.
|
|
87
104
|
*
|
|
88
|
-
*
|
|
89
|
-
*
|
|
90
|
-
* `
|
|
91
|
-
*
|
|
105
|
+
* Pass the same params given to `completion()`, so the search applies the
|
|
106
|
+
* `toolSearch` limits the model was told; with `mcp`, pass
|
|
107
|
+
* `resolveToolSearchSource(params)` instead. A bare tool list applies the default
|
|
108
|
+
* limits. Tools may be wire `Tool`s or the Zod-schema `ToolInput` form; both
|
|
109
|
+
* are converted to wire form first, so the size cap measures the JSON schemas
|
|
110
|
+
* the prompt will declare, not Zod objects.
|
|
92
111
|
*/
|
|
93
|
-
export declare function executeToolSearch(
|
|
112
|
+
export declare function executeToolSearch(source: readonly Tool[] | readonly ToolInput[] | ToolSearchSource, args: Record<string, unknown>, history: readonly HistoryMessage[]): string;
|