@qvac/inference 0.18.2 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +11 -2
- package/README.md +1 -5
- package/dist/api/assess-model-fit.d.ts +16 -0
- package/dist/api/assess-model-fit.js +27 -0
- package/dist/api/audio-gen.d.ts +4 -4
- package/dist/api/audio-gen.js +17 -6
- package/dist/api/completion-stream.d.ts +1 -1
- package/dist/api/completion-stream.js +1 -1
- package/dist/api/download-asset.d.ts +1 -1
- package/dist/api/download-asset.js +2 -2
- package/dist/api/finetune.js +1 -1
- package/dist/api/get-loaded-model-info.d.ts +6 -10
- package/dist/api/get-loaded-model-info.js +6 -10
- package/dist/api/heartbeat.d.ts +5 -22
- package/dist/api/heartbeat.js +6 -23
- package/dist/api/index.d.ts +36 -35
- package/dist/api/index.js +35 -35
- package/dist/api/load-model.d.ts +5 -1
- package/dist/api/load-model.js +3 -5
- package/dist/api/logging-stream-registry.js +1 -1
- package/dist/api/resume.d.ts +0 -3
- package/dist/api/resume.js +0 -3
- package/dist/api/subscribe-logs.js +1 -1
- package/dist/api/suspend.d.ts +0 -4
- package/dist/api/suspend.js +0 -4
- package/dist/api/translate.d.ts +3 -1
- package/dist/api/translate.js +16 -8
- package/dist/api/unload-model.js +3 -5
- package/dist/api/video-request.d.ts +207 -0
- package/dist/api/video-request.js +18 -0
- package/dist/api/video.d.ts +22 -2
- package/dist/api/video.js +25 -15
- package/dist/api/vla-helpers.d.ts +1 -2
- package/dist/api/vla-helpers.js +1 -2
- package/dist/api/vla.js +1 -1
- package/dist/api/world-result.d.ts +71 -0
- package/dist/api/world-result.js +293 -0
- package/dist/api/world.d.ts +143 -0
- package/dist/api/world.js +71 -0
- package/dist/config/resolve-config.d.ts +1 -1
- package/dist/config/resolve-config.js +1 -1
- package/dist/constants/index.d.ts +2 -2
- package/dist/constants/index.js +2 -2
- package/dist/dispatch.js +15 -28
- package/dist/errors/index.d.ts +35 -31
- package/dist/errors/index.js +66 -56
- package/dist/handlers/assess-model-fit.d.ts +10 -0
- package/dist/handlers/assess-model-fit.js +75 -0
- package/dist/handlers/cancelHandler.js +1 -1
- package/dist/handlers/completion-orchestrate.js +1 -1
- package/dist/handlers/download-asset.js +6 -5
- package/dist/handlers/finetune.js +1 -1
- package/dist/handlers/get-loaded-model-info.js +0 -12
- package/dist/handlers/get-model-info.js +1 -1
- package/dist/handlers/get-system-resources.js +8 -0
- package/dist/handlers/load-model/download-manager.d.ts +1 -1
- package/dist/handlers/load-model/download-stats.d.ts +3 -3
- package/dist/handlers/load-model/download-stats.js +5 -5
- package/dist/handlers/load-model/handler.js +6 -8
- package/dist/handlers/load-model/http-verify.d.ts +20 -0
- package/dist/handlers/load-model/http-verify.js +54 -0
- package/dist/handlers/load-model/http.d.ts +12 -2
- package/dist/handlers/load-model/http.js +119 -230
- package/dist/handlers/load-model/hyperdrive.d.ts +1 -1
- package/dist/handlers/load-model/hyperdrive.js +1 -1
- package/dist/handlers/load-model/index.d.ts +2 -2
- package/dist/handlers/load-model/index.js +2 -2
- package/dist/handlers/load-model/registry-companion-set.d.ts +1 -1
- package/dist/handlers/load-model/registry-companion-set.js +1 -1
- package/dist/handlers/load-model/registry-download-utils.d.ts +3 -3
- package/dist/handlers/load-model/registry-download-utils.js +2 -2
- package/dist/handlers/load-model/registry.d.ts +2 -2
- package/dist/handlers/load-model/registry.js +4 -4
- package/dist/handlers/load-model/resolve-session.d.ts +8 -1
- package/dist/handlers/load-model/resolve-session.js +12 -9
- package/dist/handlers/load-model/resolve.d.ts +4 -4
- package/dist/handlers/load-model/resolve.js +18 -16
- package/dist/handlers/load-model/safe-fetch.d.ts +44 -0
- package/dist/handlers/load-model/safe-fetch.js +143 -0
- package/dist/handlers/load-model/types.d.ts +9 -0
- package/dist/handlers/plugin-dispatch.js +1 -4
- package/dist/handlers/plugin-invoke.js +1 -4
- package/dist/handlers/types.d.ts +0 -2
- package/dist/handlers/unload-model.js +1 -4
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/logging/addon.d.ts +1 -1
- package/dist/logging/addon.js +1 -1
- package/dist/logging/index.d.ts +5 -5
- package/dist/logging/index.js +4 -4
- package/dist/logging/logger.d.ts +1 -1
- package/dist/logging/logger.js +2 -2
- package/dist/models/registry/index.d.ts +1 -1
- package/dist/models/registry/index.js +2 -2
- package/dist/models/registry/models.d.ts +250 -0
- package/dist/models/registry/models.js +2071 -1806
- package/dist/models/registry/resource-profiles.d.ts +18 -0
- package/dist/models/registry/resource-profiles.js +4934 -0
- package/dist/plugins/builtin/audiogen-ggml/config.d.ts +14 -0
- package/dist/plugins/builtin/audiogen-ggml/config.js +23 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +47 -5
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +134 -4
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +47 -28
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +1 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +3 -3
- package/dist/plugins/builtin/ggml-classification/plugin.js +1 -1
- package/dist/plugins/builtin/ggml-ocr/plugin.js +2 -2
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +1 -1
- package/dist/plugins/builtin/ggml-vla/plugin.js +3 -3
- package/dist/plugins/builtin/index.d.ts +12 -12
- package/dist/plugins/builtin/index.js +12 -12
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +2 -2
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +37 -26
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.d.ts +12 -11
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.js +133 -41
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +7 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +26 -4
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +19 -8
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +10 -13
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +4 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +13 -2
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +2 -2
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +2 -2
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +3 -3
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.js +1 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +32 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/world.d.ts +126 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/world.js +820 -0
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +153 -4
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +134 -13
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +1 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +1 -1
- package/dist/plugins/builtin/tts-ggml/plugin.js +2 -2
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +2 -2
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +3 -3
- package/dist/plugins/index.d.ts +1 -1
- package/dist/plugins/index.js +2 -2
- package/dist/plugins/ops/kv-cache-utils.js +1 -1
- package/dist/plugins/ops/load-model.js +1 -1
- package/dist/plugins/ops/transcribe.js +2 -2
- package/dist/plugins/ops/translate.js +4 -12
- package/dist/plugins/ops/unload-model.js +19 -21
- package/dist/plugins/registry.d.ts +1 -0
- package/dist/plugins/registry.js +12 -0
- package/dist/profiling/aggregator.d.ts +1 -1
- package/dist/profiling/aggregator.js +1 -1
- package/dist/profiling/controller.d.ts +1 -1
- package/dist/profiling/controller.js +2 -2
- package/dist/profiling/envelope.d.ts +0 -1
- package/dist/profiling/envelope.js +0 -10
- package/dist/profiling/events.d.ts +2 -3
- package/dist/profiling/events.js +2 -9
- package/dist/profiling/exporters.d.ts +1 -1
- package/dist/profiling/exporters.js +3 -3
- package/dist/profiling/index.d.ts +13 -14
- package/dist/profiling/index.js +12 -13
- package/dist/profiling/operation-metrics.d.ts +1 -1
- package/dist/profiling/operation-metrics.js +35 -2
- package/dist/profiling/operation-wrappers.js +12 -15
- package/dist/profiling/profiler.d.ts +1 -2
- package/dist/profiling/profiler.js +7 -26
- package/dist/profiling/types.d.ts +1 -1
- package/dist/rag/handler.js +1 -1
- package/dist/rag/handlers/close-workspace.js +1 -1
- package/dist/rag/handlers/delete-embeddings.js +1 -1
- package/dist/rag/handlers/delete-workspace.js +2 -6
- package/dist/rag/handlers/ingest.js +1 -1
- package/dist/rag/handlers/list-workspaces.d.ts +1 -1
- package/dist/rag/handlers/list-workspaces.js +1 -1
- package/dist/rag/handlers/reindex.js +1 -1
- package/dist/rag/handlers/save-embeddings.js +1 -1
- package/dist/rag/handlers/search.js +1 -1
- package/dist/rag/index.d.ts +11 -11
- package/dist/rag/index.js +11 -11
- package/dist/rag/rag-workspace-manager.d.ts +4 -2
- package/dist/rag/rag-workspace-manager.js +268 -15
- package/dist/registry.js +8 -58
- package/dist/resources/collector.d.ts +3 -1
- package/dist/resources/collector.js +91 -9
- package/dist/resources/instance.d.ts +1 -1
- package/dist/resources/instance.js +1 -1
- package/dist/resources/model-fit/assess.d.ts +36 -0
- package/dist/resources/model-fit/assess.js +578 -0
- package/dist/resources/model-fit/calibration/darwin-arm64.d.ts +8 -0
- package/dist/resources/model-fit/calibration/darwin-arm64.js +36 -0
- package/dist/resources/model-fit/calibration/darwin-x64.d.ts +8 -0
- package/dist/resources/model-fit/calibration/darwin-x64.js +38 -0
- package/dist/resources/model-fit/calibration/fit.d.ts +73 -0
- package/dist/resources/model-fit/calibration/fit.js +152 -0
- package/dist/resources/model-fit/calibration/index.d.ts +37 -0
- package/dist/resources/model-fit/calibration/index.js +75 -0
- package/dist/resources/model-fit/calibration/linux-arm64.d.ts +8 -0
- package/dist/resources/model-fit/calibration/linux-arm64.js +38 -0
- package/dist/resources/model-fit/calibration/linux-x64-vulkan.d.ts +8 -0
- package/dist/resources/model-fit/calibration/linux-x64-vulkan.js +39 -0
- package/dist/resources/model-fit/calibration/linux-x64.d.ts +8 -0
- package/dist/resources/model-fit/calibration/linux-x64.js +38 -0
- package/dist/resources/model-fit/calibration/win32-x64-vulkan-shared.d.ts +8 -0
- package/dist/resources/model-fit/calibration/win32-x64-vulkan-shared.js +39 -0
- package/dist/resources/model-fit/calibration/win32-x64-vulkan.d.ts +8 -0
- package/dist/resources/model-fit/calibration/win32-x64-vulkan.js +36 -0
- package/dist/resources/model-fit/calibration/win32-x64.d.ts +8 -0
- package/dist/resources/model-fit/calibration/win32-x64.js +38 -0
- package/dist/resources/model-fit/estimators/llm.d.ts +45 -0
- package/dist/resources/model-fit/estimators/llm.js +220 -0
- package/dist/resources/model-fit/estimators/whisper.d.ts +10 -0
- package/dist/resources/model-fit/estimators/whisper.js +77 -0
- package/dist/resources/model-fit/types.d.ts +120 -0
- package/dist/resources/model-fit/types.js +1 -0
- package/dist/resources/native.d.ts +1 -1
- package/dist/resources/native.js +14 -0
- package/dist/resources/normalize.d.ts +1 -1
- package/dist/resources/types.d.ts +16 -0
- package/dist/runtime/index.d.ts +5 -5
- package/dist/runtime/index.js +2 -2
- package/dist/runtime/lifecycle.js +5 -7
- package/dist/runtime/model-registry.d.ts +0 -14
- package/dist/runtime/model-registry.js +17 -40
- package/dist/runtime/registry-client.js +1 -1
- package/dist/runtime/request-context.d.ts +3 -3
- package/dist/runtime/request-context.js +13 -1
- package/dist/runtime/request-registry.d.ts +1 -1
- package/dist/runtime/request-registry.js +4 -5
- package/dist/runtime/state.d.ts +1 -1
- package/dist/runtime/state.js +18 -3
- package/dist/schemas/assess-model-fit.d.ts +271 -0
- package/dist/schemas/assess-model-fit.js +153 -0
- package/dist/schemas/audio-gen.d.ts +434 -4
- package/dist/schemas/audio-gen.js +131 -19
- package/dist/schemas/batch-completion-stream.d.ts +2 -2
- package/dist/schemas/batch-completion-stream.js +3 -3
- package/dist/schemas/bci-config.js +73 -27
- package/dist/schemas/bci.d.ts +2 -2
- package/dist/schemas/bci.js +5 -5
- package/dist/schemas/cancel.d.ts +5 -0
- package/dist/schemas/cancel.js +1 -0
- package/dist/schemas/classification.js +13 -6
- package/dist/schemas/common.d.ts +662 -331
- package/dist/schemas/common.js +33 -31
- package/dist/schemas/completion-event.d.ts +2 -2
- package/dist/schemas/completion-event.js +1 -1
- package/dist/schemas/completion-stream.d.ts +1 -5
- package/dist/schemas/completion-stream.js +4 -8
- package/dist/schemas/config.d.ts +640 -18
- package/dist/schemas/config.js +50 -8
- package/dist/schemas/download-asset.d.ts +19 -1
- package/dist/schemas/download-asset.js +20 -4
- package/dist/schemas/engine-addon-map.d.ts +1 -1
- package/dist/schemas/engine-addon-map.js +3 -3
- package/dist/schemas/error.d.ts +2 -2
- package/dist/schemas/error.js +2 -2
- package/dist/schemas/errors.d.ts +2 -6
- package/dist/schemas/errors.js +30 -40
- package/dist/schemas/heartbeat.d.ts +10 -0
- package/dist/schemas/heartbeat.js +8 -0
- package/dist/schemas/index.d.ts +46 -44
- package/dist/schemas/index.js +45 -43
- package/dist/schemas/llamacpp-config.d.ts +32 -10
- package/dist/schemas/llamacpp-config.js +11 -8
- package/dist/schemas/load-model.d.ts +2933 -1948
- package/dist/schemas/load-model.js +52 -52
- package/dist/schemas/mcp-adapter.d.ts +1 -1
- package/dist/schemas/model-config-schemas.d.ts +2354 -0
- package/dist/schemas/model-config-schemas.js +46 -0
- package/dist/schemas/model-info.d.ts +10 -20
- package/dist/schemas/model-info.js +4 -25
- package/dist/schemas/model-ops.d.ts +0 -1
- package/dist/schemas/model-ops.js +1 -2
- package/dist/schemas/model-resource-profile.d.ts +109 -0
- package/dist/schemas/model-resource-profile.js +83 -0
- package/dist/schemas/model-src-utils.d.ts +1 -0
- package/dist/schemas/model-src-utils.js +33 -13
- package/dist/schemas/ocr.js +48 -13
- package/dist/schemas/path-validation.js +8 -2
- package/dist/schemas/plugin.d.ts +14 -1
- package/dist/schemas/plugin.js +11 -0
- package/dist/schemas/profiling.d.ts +138 -43
- package/dist/schemas/profiling.js +1 -22
- package/dist/schemas/rag.js +1 -1
- package/dist/schemas/registry.js +1 -1
- package/dist/schemas/reload-config.d.ts +0 -2
- package/dist/schemas/reload-config.js +2 -3
- package/dist/schemas/sdcpp-config.d.ts +344 -4
- package/dist/schemas/sdcpp-config.js +504 -17
- package/dist/schemas/system-resources.d.ts +138 -0
- package/dist/schemas/system-resources.js +3 -1
- package/dist/schemas/text-to-speech.js +183 -76
- package/dist/schemas/transcription-config.js +253 -85
- package/dist/schemas/transcription.d.ts +3 -3
- package/dist/schemas/translate.d.ts +2 -2
- package/dist/schemas/translate.js +1 -1
- package/dist/schemas/translation-config.js +53 -25
- package/dist/schemas/vla.d.ts +1 -1
- package/dist/selection.d.ts +1 -5
- package/dist/selection.js +2 -6
- package/dist/surface.d.ts +1 -1
- package/dist/utils/aggregate-events.d.ts +1 -1
- package/dist/utils/aggregate-events.js +2 -2
- package/dist/utils/archive.js +2 -2
- package/dist/utils/audio-input.js +1 -1
- package/dist/utils/cache/auto-history.d.ts +1 -1
- package/dist/utils/cache/auto-history.js +1 -1
- package/dist/utils/cache/paths.js +5 -5
- package/dist/utils/completion-normalizer.js +1 -1
- package/dist/utils/gguf-tensor-extractor.js +1 -1
- package/dist/utils/image-dimensions.d.ts +26 -0
- package/dist/utils/image-dimensions.js +46 -0
- package/dist/utils/index.d.ts +13 -10
- package/dist/utils/index.js +13 -10
- package/dist/utils/mcp-adapter.d.ts +1 -1
- package/dist/utils/mcp-adapter.js +1 -1
- package/dist/utils/path-sanitize.d.ts +7 -0
- package/dist/utils/path-sanitize.js +36 -0
- package/dist/utils/path-security.d.ts +0 -7
- package/dist/utils/path-security.js +1 -36
- package/dist/utils/shard-utils.js +3 -3
- package/dist/utils/tool-integration.js +1 -1
- package/dist/utils/tools/dialect.js +1 -1
- package/dist/utils/tools/index.d.ts +2 -2
- package/dist/utils/tools/index.js +2 -2
- package/dist/utils/tools/parser.js +8 -8
- package/dist/utils/tools/parsers/dsml.d.ts +1 -1
- package/dist/utils/tools/parsers/dsml.js +1 -1
- package/dist/utils/tools/parsers/gemma4native.d.ts +1 -1
- package/dist/utils/tools/parsers/gemma4native.js +1 -1
- package/dist/utils/tools/parsers/harmony.d.ts +1 -1
- package/dist/utils/tools/parsers/harmony.js +1 -1
- package/dist/utils/tools/parsers/hermes.d.ts +1 -1
- package/dist/utils/tools/parsers/hermes.js +1 -1
- package/dist/utils/tools/parsers/json.d.ts +1 -1
- package/dist/utils/tools/parsers/json.js +1 -1
- package/dist/utils/tools/parsers/pythonic.d.ts +1 -1
- package/dist/utils/tools/parsers/pythonic.js +1 -1
- package/dist/utils/tools/parsers/qwen35.d.ts +1 -1
- package/dist/utils/tools/parsers/qwen35.js +3 -3
- package/dist/utils/url-security.d.ts +17 -0
- package/dist/utils/url-security.js +75 -0
- package/package.json +31 -26
- package/dist/api/provide.d.ts +0 -23
- package/dist/api/provide.js +0 -32
- package/dist/api/stop-provider.d.ts +0 -20
- package/dist/api/stop-provider.js +0 -30
- package/dist/p2p/cancel-delegated.d.ts +0 -3
- package/dist/p2p/cancel-delegated.js +0 -64
- package/dist/p2p/completion-stream-delegated.d.ts +0 -4
- package/dist/p2p/completion-stream-delegated.js +0 -51
- package/dist/p2p/delegate-client.d.ts +0 -9
- package/dist/p2p/delegate-client.js +0 -219
- package/dist/p2p/delegate-connect-diagnostics.d.ts +0 -8
- package/dist/p2p/delegate-connect-diagnostics.js +0 -38
- package/dist/p2p/delegate-transport.d.ts +0 -18
- package/dist/p2p/delegate-transport.js +0 -237
- package/dist/p2p/error-handlers.d.ts +0 -4
- package/dist/p2p/error-handlers.js +0 -31
- package/dist/p2p/handle-request.d.ts +0 -2
- package/dist/p2p/handle-request.js +0 -171
- package/dist/p2p/handler-utils.d.ts +0 -19
- package/dist/p2p/handler-utils.js +0 -180
- package/dist/p2p/heartbeat-delegated.d.ts +0 -3
- package/dist/p2p/heartbeat-delegated.js +0 -32
- package/dist/p2p/load-model-delegated.d.ts +0 -6
- package/dist/p2p/load-model-delegated.js +0 -132
- package/dist/p2p/progress-throttle.d.ts +0 -7
- package/dist/p2p/progress-throttle.js +0 -47
- package/dist/p2p/provideHandler/connection.d.ts +0 -2
- package/dist/p2p/provideHandler/connection.js +0 -38
- package/dist/p2p/provideHandler/handler.d.ts +0 -2
- package/dist/p2p/provideHandler/handler.js +0 -66
- package/dist/p2p/provideHandler/proxy.d.ts +0 -2
- package/dist/p2p/provideHandler/proxy.js +0 -24
- package/dist/p2p/provideHandler.d.ts +0 -1
- package/dist/p2p/provideHandler.js +0 -2
- package/dist/p2p/rpc-utils.d.ts +0 -4
- package/dist/p2p/rpc-utils.js +0 -8
- package/dist/p2p/stopProvideHandler.d.ts +0 -2
- package/dist/p2p/stopProvideHandler.js +0 -29
- package/dist/p2p/swarm.d.ts +0 -9
- package/dist/p2p/swarm.js +0 -103
- package/dist/p2p/transport-selector.d.ts +0 -2
- package/dist/p2p/transport-selector.js +0 -18
- package/dist/p2p/unload-model-delegated.d.ts +0 -2
- package/dist/p2p/unload-model-delegated.js +0 -31
- package/dist/profiling/delegation-profiler.d.ts +0 -31
- package/dist/profiling/delegation-profiler.js +0 -155
- package/dist/schemas/delegate.d.ts +0 -29
- package/dist/schemas/delegate.js +0 -40
- package/dist/schemas/provide.d.ts +0 -50
- package/dist/schemas/provide.js +0 -32
|
@@ -22,7 +22,13 @@ export declare const llmConfigBaseSchema: z.ZodObject<{
|
|
|
22
22
|
device: z.ZodOptional<z.ZodString>;
|
|
23
23
|
predict: z.ZodOptional<z.ZodUnion<readonly [z.ZodLiteral<-1>, z.ZodLiteral<-2>, z.ZodNumber]>>;
|
|
24
24
|
system_prompt: z.ZodOptional<z.ZodString>;
|
|
25
|
-
|
|
25
|
+
load_mode: z.ZodOptional<z.ZodEnum<{
|
|
26
|
+
none: "none";
|
|
27
|
+
mmap: "mmap";
|
|
28
|
+
mlock: "mlock";
|
|
29
|
+
"mmap+mlock": "mmap+mlock";
|
|
30
|
+
dio: "dio";
|
|
31
|
+
}>>;
|
|
26
32
|
verbosity: z.ZodOptional<z.ZodEnum<{
|
|
27
33
|
readonly ERROR: 0;
|
|
28
34
|
readonly WARN: 1;
|
|
@@ -33,7 +39,6 @@ export declare const llmConfigBaseSchema: z.ZodObject<{
|
|
|
33
39
|
frequency_penalty: z.ZodOptional<z.ZodNumber>;
|
|
34
40
|
repeat_penalty: z.ZodOptional<z.ZodNumber>;
|
|
35
41
|
stop_sequences: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
36
|
-
n_discarded: z.ZodOptional<z.ZodNumber>;
|
|
37
42
|
parallel: z.ZodOptional<z.ZodNumber>;
|
|
38
43
|
tools: z.ZodOptional<z.ZodBoolean>;
|
|
39
44
|
'cache-type-k': z.ZodOptional<z.ZodString>;
|
|
@@ -46,6 +51,12 @@ export declare const llmConfigBaseSchema: z.ZodObject<{
|
|
|
46
51
|
none: "none";
|
|
47
52
|
layer: "layer";
|
|
48
53
|
row: "row";
|
|
54
|
+
tensor: "tensor";
|
|
55
|
+
}>>;
|
|
56
|
+
'flash-attn': z.ZodOptional<z.ZodEnum<{
|
|
57
|
+
on: "on";
|
|
58
|
+
off: "off";
|
|
59
|
+
auto: "auto";
|
|
49
60
|
}>>;
|
|
50
61
|
'tensor-split': z.ZodOptional<z.ZodString>;
|
|
51
62
|
openclCacheDir: z.ZodOptional<z.ZodString>;
|
|
@@ -119,7 +130,13 @@ export declare const llmConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
119
130
|
device: z.ZodOptional<z.ZodString>;
|
|
120
131
|
predict: z.ZodOptional<z.ZodUnion<readonly [z.ZodLiteral<-1>, z.ZodLiteral<-2>, z.ZodNumber]>>;
|
|
121
132
|
system_prompt: z.ZodOptional<z.ZodString>;
|
|
122
|
-
|
|
133
|
+
load_mode: z.ZodOptional<z.ZodEnum<{
|
|
134
|
+
none: "none";
|
|
135
|
+
mmap: "mmap";
|
|
136
|
+
mlock: "mlock";
|
|
137
|
+
"mmap+mlock": "mmap+mlock";
|
|
138
|
+
dio: "dio";
|
|
139
|
+
}>>;
|
|
123
140
|
verbosity: z.ZodOptional<z.ZodEnum<{
|
|
124
141
|
readonly ERROR: 0;
|
|
125
142
|
readonly WARN: 1;
|
|
@@ -130,7 +147,6 @@ export declare const llmConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
130
147
|
frequency_penalty: z.ZodOptional<z.ZodNumber>;
|
|
131
148
|
repeat_penalty: z.ZodOptional<z.ZodNumber>;
|
|
132
149
|
stop_sequences: z.ZodOptional<z.ZodArray<z.ZodString>>;
|
|
133
|
-
n_discarded: z.ZodOptional<z.ZodNumber>;
|
|
134
150
|
parallel: z.ZodOptional<z.ZodNumber>;
|
|
135
151
|
tools: z.ZodOptional<z.ZodBoolean>;
|
|
136
152
|
'cache-type-k': z.ZodOptional<z.ZodString>;
|
|
@@ -143,6 +159,12 @@ export declare const llmConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
143
159
|
none: "none";
|
|
144
160
|
layer: "layer";
|
|
145
161
|
row: "row";
|
|
162
|
+
tensor: "tensor";
|
|
163
|
+
}>>;
|
|
164
|
+
'flash-attn': z.ZodOptional<z.ZodEnum<{
|
|
165
|
+
on: "on";
|
|
166
|
+
off: "off";
|
|
167
|
+
auto: "auto";
|
|
146
168
|
}>>;
|
|
147
169
|
'tensor-split': z.ZodOptional<z.ZodString>;
|
|
148
170
|
openclCacheDir: z.ZodOptional<z.ZodString>;
|
|
@@ -207,19 +229,19 @@ export declare const llmConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
207
229
|
device: string | undefined;
|
|
208
230
|
predict?: number | undefined;
|
|
209
231
|
system_prompt: string | undefined;
|
|
210
|
-
|
|
232
|
+
load_mode?: "none" | "mmap" | "mlock" | "mmap+mlock" | "dio" | undefined;
|
|
211
233
|
verbosity?: 0 | 3 | 2 | 1 | undefined;
|
|
212
234
|
presence_penalty?: number | undefined;
|
|
213
235
|
frequency_penalty?: number | undefined;
|
|
214
236
|
repeat_penalty?: number | undefined;
|
|
215
237
|
stop_sequences?: string[] | undefined;
|
|
216
|
-
n_discarded?: number | undefined;
|
|
217
238
|
parallel?: number | undefined;
|
|
218
239
|
tools?: boolean | undefined;
|
|
219
240
|
'cache-type-k'?: string | undefined;
|
|
220
241
|
'cache-type-v'?: string | undefined;
|
|
221
242
|
'main-gpu'?: number | "integrated" | "dedicated" | undefined;
|
|
222
|
-
'split-mode'?: "none" | "layer" | "row" | undefined;
|
|
243
|
+
'split-mode'?: "none" | "layer" | "row" | "tensor" | undefined;
|
|
244
|
+
'flash-attn'?: "on" | "off" | "auto" | undefined;
|
|
223
245
|
'tensor-split'?: string | undefined;
|
|
224
246
|
openclCacheDir?: string | undefined;
|
|
225
247
|
reasoning_budget?: number | undefined;
|
|
@@ -250,19 +272,19 @@ export declare const llmConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
|
250
272
|
device?: string | undefined;
|
|
251
273
|
predict?: number | undefined;
|
|
252
274
|
system_prompt?: string | undefined;
|
|
253
|
-
|
|
275
|
+
load_mode?: "none" | "mmap" | "mlock" | "mmap+mlock" | "dio" | undefined;
|
|
254
276
|
verbosity?: 0 | 3 | 2 | 1 | undefined;
|
|
255
277
|
presence_penalty?: number | undefined;
|
|
256
278
|
frequency_penalty?: number | undefined;
|
|
257
279
|
repeat_penalty?: number | undefined;
|
|
258
280
|
stop_sequences?: string[] | undefined;
|
|
259
|
-
n_discarded?: number | undefined;
|
|
260
281
|
parallel?: number | undefined;
|
|
261
282
|
tools?: boolean | undefined;
|
|
262
283
|
'cache-type-k'?: string | undefined;
|
|
263
284
|
'cache-type-v'?: string | undefined;
|
|
264
285
|
'main-gpu'?: number | "integrated" | "dedicated" | undefined;
|
|
265
|
-
'split-mode'?: "none" | "layer" | "row" | undefined;
|
|
286
|
+
'split-mode'?: "none" | "layer" | "row" | "tensor" | undefined;
|
|
287
|
+
'flash-attn'?: "on" | "off" | "auto" | undefined;
|
|
266
288
|
'tensor-split'?: string | undefined;
|
|
267
289
|
openclCacheDir?: string | undefined;
|
|
268
290
|
reasoning_budget?: number | undefined;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
|
-
import { modelSrcInputSchema } from '
|
|
2
|
+
import { modelSrcInputSchema } from './model-src-utils.js';
|
|
3
3
|
/**
|
|
4
4
|
* Upper bound for `reasoning_budget`. Mirrors the llm-llamacpp addon, which
|
|
5
5
|
* stores the budget as a 32-bit `int` and rejects values above
|
|
@@ -58,7 +58,10 @@ export const llmConfigBaseSchema = z.object({
|
|
|
58
58
|
.string()
|
|
59
59
|
.optional()
|
|
60
60
|
.describe("Seeds conversation history on the JS side only; never forwarded to the addon. Default `'You are a helpful assistant.'`"),
|
|
61
|
-
|
|
61
|
+
load_mode: z
|
|
62
|
+
.enum(['none', 'mmap', 'mlock', 'mmap+mlock', 'dio'])
|
|
63
|
+
.optional()
|
|
64
|
+
.describe("Model loading mode: `'none'`, `'mmap'`, `'mlock'`, `'mmap+mlock'`, or `'dio'`. Unset uses the addon's default (`'mmap'`)."),
|
|
62
65
|
verbosity: verbositySchema
|
|
63
66
|
.optional()
|
|
64
67
|
.describe('Native log verbosity: `0`=ERROR, `1`=WARN, `2`=INFO, `3`=DEBUG. Default 0.'),
|
|
@@ -78,10 +81,6 @@ export const llmConfigBaseSchema = z.object({
|
|
|
78
81
|
.array(z.string())
|
|
79
82
|
.optional()
|
|
80
83
|
.describe('Strings that stop generation when produced (forwarded to the addon as `reverse_prompt`).'),
|
|
81
|
-
n_discarded: z
|
|
82
|
-
.number()
|
|
83
|
-
.optional()
|
|
84
|
-
.describe('Tokens to discard from the front of the context when it fills (sliding window); `0` (default) disables sliding. In batch mode clamped to the per-slot window (`ctx_size / parallel`).'),
|
|
85
84
|
parallel: z
|
|
86
85
|
.number()
|
|
87
86
|
.int()
|
|
@@ -105,9 +104,13 @@ export const llmConfigBaseSchema = z.object({
|
|
|
105
104
|
.optional()
|
|
106
105
|
.describe("GPU to use on multi-GPU systems: a device index, or `'integrated'`/`'dedicated'` to restrict selection to that class."),
|
|
107
106
|
'split-mode': z
|
|
108
|
-
.enum(['none', 'layer', 'row'])
|
|
107
|
+
.enum(['none', 'layer', 'row', 'tensor'])
|
|
109
108
|
.optional()
|
|
110
|
-
.describe("How to split the model across GPUs: `'none'` (default, single GPU), `'layer'` (pipeline parallelism), or `'
|
|
109
|
+
.describe("How to split the model across GPUs: `'none'` (default, single GPU), `'layer'` (pipeline parallelism), `'row'` (legacy; degrades to `'layer'`), or `'tensor'` (EXPERIMENTAL tensor parallelism across all visible GPUs; desktop-only, requires flash attention, and disables auto-fit, so set `ctx_size` explicitly)."),
|
|
110
|
+
'flash-attn': z
|
|
111
|
+
.enum(['on', 'off', 'auto'])
|
|
112
|
+
.optional()
|
|
113
|
+
.describe("Flash attention: `'on'`, `'off'`, or `'auto'`. With `'auto'`, the backend decides. When unset, the addon defaults to `'off'` for BitNet models and `'on'` for other inference workloads. An explicit value overrides the BitNet default. Finetuning enforces its own setting. `'off'` is incompatible with `'split-mode': 'tensor'`."),
|
|
111
114
|
'tensor-split': z
|
|
112
115
|
.string()
|
|
113
116
|
.optional()
|