@qvac/inference 0.21.1 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +89 -41
- package/README.md +44 -1
- package/dist/api/assess-model-fit.d.ts +4 -4
- package/dist/api/assess-model-fit.js +4 -4
- package/dist/api/audio-gen.d.ts +4 -1
- package/dist/api/audio-gen.js +5 -2
- package/dist/api/batch-completion.js +1 -1
- package/dist/api/completion-stream.d.ts +2 -1
- package/dist/api/completion-stream.js +4 -2
- package/dist/api/decide.d.ts +26 -0
- package/dist/api/decide.js +26 -0
- package/dist/api/diffusion.d.ts +18 -1
- package/dist/api/diffusion.js +18 -1
- package/dist/api/download-asset.js +1 -1
- package/dist/api/finetune.js +1 -1
- package/dist/api/get-model-info.d.ts +1 -1
- package/dist/api/index.d.ts +39 -37
- package/dist/api/index.js +38 -36
- package/dist/api/load-model.js +2 -2
- package/dist/api/logging-stream-registry.js +11 -7
- package/dist/api/logging-stream.d.ts +11 -1
- package/dist/api/logging-stream.js +11 -2
- package/dist/api/registry.d.ts +2 -1
- package/dist/api/registry.js +5 -4
- package/dist/api/rerank.d.ts +17 -0
- package/dist/api/rerank.js +36 -0
- package/dist/api/subscribe-logs.js +5 -3
- package/dist/api/text-to-speech.d.ts +1 -0
- package/dist/api/text-to-speech.js +15 -19
- package/dist/api/transcribe.js +5 -2
- package/dist/api/unload-model.js +1 -1
- package/dist/api/video-request.d.ts +5 -5
- package/dist/api/video.d.ts +5 -4
- package/dist/api/video.js +6 -5
- package/dist/api/vla.js +1 -1
- package/dist/api/world-result.js +1 -1
- package/dist/api/world.d.ts +1 -1
- package/dist/api/world.js +1 -1
- package/dist/config/resolve-config.d.ts +1 -1
- package/dist/config/resolve-config.js +1 -1
- package/dist/constants/index.d.ts +2 -2
- package/dist/constants/index.js +2 -2
- package/dist/dispatch.d.ts +3 -2
- package/dist/dispatch.js +16 -4
- package/dist/errors/index.d.ts +9 -0
- package/dist/errors/index.js +18 -1
- package/dist/handlers/assess-model-fit.d.ts +5 -3
- package/dist/handlers/assess-model-fit.js +84 -32
- package/dist/handlers/cancelHandler.js +1 -1
- package/dist/handlers/completion-orchestrate.js +2 -2
- package/dist/handlers/download-asset.js +2 -2
- package/dist/handlers/finetune.js +1 -1
- package/dist/handlers/get-loaded-model-info.js +3 -5
- package/dist/handlers/get-model-info.js +1 -1
- package/dist/handlers/load-model/coreml-sidecars.d.ts +15 -0
- package/dist/handlers/load-model/coreml-sidecars.js +71 -8
- package/dist/handlers/load-model/download-manager.d.ts +1 -1
- package/dist/handlers/load-model/download-stats.d.ts +2 -2
- package/dist/handlers/load-model/download-stats.js +3 -3
- package/dist/handlers/load-model/handler.js +10 -7
- package/dist/handlers/load-model/http-verify.d.ts +1 -1
- package/dist/handlers/load-model/http.d.ts +1 -1
- package/dist/handlers/load-model/http.js +3 -3
- package/dist/handlers/load-model/hyperdrive.d.ts +1 -1
- package/dist/handlers/load-model/hyperdrive.js +2 -2
- package/dist/handlers/load-model/index.d.ts +2 -2
- package/dist/handlers/load-model/index.js +2 -2
- package/dist/handlers/load-model/registry-companion-set.d.ts +1 -1
- package/dist/handlers/load-model/registry-companion-set.js +1 -1
- package/dist/handlers/load-model/registry-download-utils.d.ts +3 -3
- package/dist/handlers/load-model/registry-download-utils.js +2 -2
- package/dist/handlers/load-model/registry.d.ts +2 -2
- package/dist/handlers/load-model/registry.js +5 -5
- package/dist/handlers/load-model/resolve-session.d.ts +1 -1
- package/dist/handlers/load-model/resolve-session.js +11 -3
- package/dist/handlers/load-model/resolve.d.ts +1 -1
- package/dist/handlers/load-model/resolve.js +5 -5
- package/dist/handlers/load-model/safe-fetch.js +5 -6
- package/dist/handlers/logging-stream.d.ts +2 -1
- package/dist/handlers/logging-stream.js +23 -7
- package/dist/handlers/registry.d.ts +3 -1
- package/dist/handlers/registry.js +7 -5
- package/dist/handlers/types.d.ts +7 -1
- package/dist/handlers/vector-index.js +2 -2
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/logging/addon.d.ts +1 -1
- package/dist/logging/addon.js +1 -1
- package/dist/logging/index.d.ts +5 -5
- package/dist/logging/index.js +4 -4
- package/dist/logging/logger.d.ts +1 -1
- package/dist/logging/logger.js +2 -2
- package/dist/{resources/model-fit → model-fit}/assess.d.ts +18 -26
- package/dist/{resources/model-fit → model-fit}/assess.js +210 -274
- package/dist/{resources/model-fit/native-probe/classify-fit.d.ts → model-fit/classify.d.ts} +2 -2
- package/dist/{resources/model-fit/native-probe/classify-fit.js → model-fit/classify.js} +36 -5
- package/dist/{resources/model-fit/fit-stub/fetch-fit-stub.d.ts → model-fit/description/fetch.d.ts} +4 -2
- package/dist/{resources/model-fit/fit-stub/fetch-fit-stub.js → model-fit/description/fetch.js} +11 -6
- package/dist/{resources/model-fit/fit-stub/project-fit-from-load.d.ts → model-fit/description/project.d.ts} +3 -3
- package/dist/{resources/model-fit/fit-stub/project-fit-from-load.js → model-fit/description/project.js} +4 -2
- package/dist/{resources/model-fit/fit-stub/resolve-load-from-stubs.d.ts → model-fit/description/resolve-load.d.ts} +6 -4
- package/dist/{resources/model-fit/fit-stub/resolve-load-from-stubs.js → model-fit/description/resolve-load.js} +14 -6
- package/dist/model-fit/floor.d.ts +31 -0
- package/dist/{resources/model-fit → model-fit}/floor.js +13 -13
- package/dist/model-fit/kv-cache.d.ts +23 -0
- package/dist/model-fit/kv-cache.js +85 -0
- package/dist/{resources/model-fit/native-probe/advisory-fit.d.ts → model-fit/load-check.d.ts} +17 -14
- package/dist/{resources/model-fit/native-probe/advisory-fit.js → model-fit/load-check.js} +23 -64
- package/dist/model-fit/platform.d.ts +14 -0
- package/dist/model-fit/platform.js +26 -0
- package/dist/{resources/model-fit/native-probe/create-fit-request.d.ts → model-fit/request/create.d.ts} +5 -2
- package/dist/model-fit/request/create.js +61 -0
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/asr.d.ts +5 -1
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/asr.js +32 -1
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/audiogen.d.ts +1 -1
- package/dist/model-fit/request/engines/audiogen.js +63 -0
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/diffusion.d.ts +1 -1
- package/dist/model-fit/request/engines/diffusion.js +93 -0
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/llama.d.ts +2 -3
- package/dist/model-fit/request/engines/llama.js +44 -0
- package/dist/model-fit/request/engines/nmt.d.ts +8 -0
- package/dist/model-fit/request/engines/nmt.js +11 -0
- package/dist/model-fit/request/engines/ocr.d.ts +9 -0
- package/dist/model-fit/request/engines/ocr.js +13 -0
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/tts.d.ts +3 -1
- package/dist/{resources/model-fit/native-probe → model-fit/request}/engines/tts.js +50 -4
- package/dist/{resources/model-fit/native-probe/engine-fit.d.ts → model-fit/run/engine.d.ts} +17 -4
- package/dist/{resources/model-fit/native-probe/engine-fit.js → model-fit/run/engine.js} +2 -2
- package/dist/model-fit/run/run.d.ts +43 -0
- package/dist/model-fit/run/run.js +204 -0
- package/dist/models/registry/index.d.ts +1 -1
- package/dist/models/registry/index.js +2 -2
- package/dist/models/registry/models.d.ts +1072 -815
- package/dist/models/registry/models.js +3704 -3405
- package/dist/models/registry/resource-profiles.d.ts +1 -2
- package/dist/models/registry/resource-profiles.js +406 -3
- package/dist/plugins/builtin/asr-ggml/config.d.ts +7 -1
- package/dist/plugins/builtin/asr-ggml/config.js +6 -0
- package/dist/plugins/builtin/asr-ggml/logging.d.ts +1 -1
- package/dist/plugins/builtin/audiogen-ggml/config.d.ts +1 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +29 -11
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.d.ts +27 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-input.js +23 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +7 -28
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +2 -2
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +106 -13
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +5 -4
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +20 -3
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +3 -3
- package/dist/plugins/builtin/ggml-classification/plugin.js +1 -1
- package/dist/plugins/builtin/ggml-ocr/params.d.ts +18 -0
- package/dist/plugins/builtin/ggml-ocr/params.js +37 -0
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +9 -0
- package/dist/plugins/builtin/ggml-ocr/plugin.js +17 -39
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +9 -2
- package/dist/plugins/builtin/ggml-vla/plugin.js +4 -4
- package/dist/plugins/builtin/index.d.ts +15 -12
- package/dist/plugins/builtin/index.js +15 -12
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +5 -7
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +2 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +6 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +10 -7
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +141 -204
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +36 -92
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +67 -216
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.d.ts +2 -49
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-state.js +2 -46
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +208 -19
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +28 -20
- package/dist/plugins/builtin/llamacpp-completion/transform.js +5 -1
- package/dist/plugins/builtin/llamacpp-decision/helpers.d.ts +5 -0
- package/dist/plugins/builtin/llamacpp-decision/helpers.js +89 -0
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.d.ts +91 -0
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.js +129 -0
- package/dist/plugins/builtin/llamacpp-decision/plugin.d.ts +222 -0
- package/dist/plugins/builtin/llamacpp-decision/plugin.js +44 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +9 -1
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +15 -5
- package/dist/plugins/builtin/llamacpp-embedding/transform.js +9 -0
- package/dist/plugins/builtin/llamacpp-rerank/helpers.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-rerank/helpers.js +16 -0
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.d.ts +5 -0
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.js +117 -0
- package/dist/plugins/builtin/llamacpp-rerank/plugin.d.ts +91 -0
- package/dist/plugins/builtin/llamacpp-rerank/plugin.js +45 -0
- package/dist/plugins/builtin/moss-transcribe/plugin.d.ts +143 -0
- package/dist/plugins/builtin/moss-transcribe/plugin.js +96 -0
- package/dist/plugins/builtin/nmtcpp-translation/load-files.d.ts +6 -0
- package/dist/plugins/builtin/nmtcpp-translation/load-files.js +70 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +17 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +10 -74
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +25 -4
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +20 -5
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +5 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +168 -52
- package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +5 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +28 -13
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +83 -1
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +55 -52
- package/dist/plugins/builtin/sdcpp-generation/upscaler-config.d.ts +15 -0
- package/dist/plugins/builtin/sdcpp-generation/upscaler-config.js +19 -0
- package/dist/plugins/builtin/sdcpp-generation/world-config.d.ts +8 -0
- package/dist/plugins/builtin/sdcpp-generation/world-config.js +9 -0
- package/dist/plugins/builtin/tts-ggml/ops/moss-options.d.ts +22 -0
- package/dist/plugins/builtin/tts-ggml/ops/moss-options.js +79 -0
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +1 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +5 -2
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +19 -7
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +151 -1
- package/dist/plugins/builtin/tts-ggml/plugin.js +39 -2
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +28 -4
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +8 -4
- package/dist/plugins/index.d.ts +1 -1
- package/dist/plugins/index.js +2 -2
- package/dist/plugins/ops/kv-cache-utils.d.ts +5 -4
- package/dist/plugins/ops/kv-cache-utils.js +25 -33
- package/dist/plugins/ops/load-model.d.ts +8 -2
- package/dist/plugins/ops/load-model.js +33 -19
- package/dist/plugins/ops/transcribe.js +18 -2
- package/dist/plugins/ops/translate.js +3 -3
- package/dist/plugins/ops/unload-model.js +1 -1
- package/dist/plugins/turbovec-backend.js +5 -5
- package/dist/profiling/aggregator.d.ts +1 -1
- package/dist/profiling/aggregator.js +1 -1
- package/dist/profiling/controller.d.ts +1 -1
- package/dist/profiling/controller.js +2 -2
- package/dist/profiling/events.d.ts +1 -1
- package/dist/profiling/events.js +2 -2
- package/dist/profiling/exporters.d.ts +1 -1
- package/dist/profiling/exporters.js +3 -3
- package/dist/profiling/index.d.ts +13 -13
- package/dist/profiling/index.js +12 -12
- package/dist/profiling/operation-metrics.d.ts +1 -1
- package/dist/profiling/operation-metrics.js +2 -2
- package/dist/profiling/operation-wrappers.js +3 -3
- package/dist/profiling/profiler.js +1 -1
- package/dist/rag/handler.js +1 -1
- package/dist/rag/handlers/close-workspace.js +1 -1
- package/dist/rag/handlers/delete-embeddings.js +1 -1
- package/dist/rag/handlers/delete-workspace.js +1 -1
- package/dist/rag/handlers/ingest.js +1 -1
- package/dist/rag/handlers/list-workspaces.d.ts +1 -1
- package/dist/rag/handlers/list-workspaces.js +1 -1
- package/dist/rag/handlers/reindex.js +1 -1
- package/dist/rag/handlers/save-embeddings.js +1 -1
- package/dist/rag/handlers/search.js +1 -1
- package/dist/rag/index.d.ts +11 -11
- package/dist/rag/index.js +11 -11
- package/dist/rag/rag-workspace-manager.js +1 -1
- package/dist/registry.js +3 -1
- package/dist/resources/collector.d.ts +1 -1
- package/dist/resources/collector.js +1 -1
- package/dist/resources/instance.d.ts +1 -1
- package/dist/resources/instance.js +1 -1
- package/dist/resources/native.d.ts +1 -1
- package/dist/resources/normalize.d.ts +1 -1
- package/dist/rpc/inventory.js +5 -7
- package/dist/runtime/env.d.ts +0 -2
- package/dist/runtime/env.js +2 -9
- package/dist/runtime/index.d.ts +5 -5
- package/dist/runtime/index.js +2 -2
- package/dist/runtime/lifecycle.js +7 -7
- package/dist/runtime/model-config-utils.js +9 -1
- package/dist/runtime/model-registry.d.ts +3 -1
- package/dist/runtime/model-registry.js +2 -1
- package/dist/runtime/registry-client.js +1 -1
- package/dist/runtime/request-context.d.ts +3 -3
- package/dist/runtime/request-context.js +22 -1
- package/dist/runtime/request-registry.d.ts +1 -1
- package/dist/runtime/request-registry.js +1 -1
- package/dist/runtime/state.d.ts +1 -1
- package/dist/runtime/state.js +7 -2
- package/dist/runtime/vector-index-backend.d.ts +2 -2
- package/dist/runtime/vector-index-registry.d.ts +5 -5
- package/dist/runtime/vector-index-registry.js +36 -13
- package/dist/schemas/assess-model-fit.d.ts +106 -43
- package/dist/schemas/assess-model-fit.js +69 -32
- package/dist/schemas/audio-gen.d.ts +199 -19
- package/dist/schemas/audio-gen.js +183 -51
- package/dist/schemas/batch-completion-stream.d.ts +210 -36
- package/dist/schemas/batch-completion-stream.js +3 -3
- package/dist/schemas/bci-config.d.ts +3 -0
- package/dist/schemas/bci-config.js +2 -2
- package/dist/schemas/bci.d.ts +24 -3
- package/dist/schemas/bci.js +2 -2
- package/dist/schemas/cancel.d.ts +10 -0
- package/dist/schemas/cancel.js +2 -0
- package/dist/schemas/common.d.ts +1465 -162
- package/dist/schemas/common.js +37 -31
- package/dist/schemas/completion-event.d.ts +8 -2
- package/dist/schemas/completion-event.js +4 -1
- package/dist/schemas/completion-stream.d.ts +243 -39
- package/dist/schemas/completion-stream.js +100 -29
- package/dist/schemas/config.d.ts +1326 -18
- package/dist/schemas/config.js +30 -8
- package/dist/schemas/decision.d.ts +610 -0
- package/dist/schemas/decision.js +219 -0
- package/dist/schemas/download-asset.d.ts +10 -4
- package/dist/schemas/download-asset.js +1 -1
- package/dist/schemas/engine-addon-map.d.ts +9 -1
- package/dist/schemas/engine-addon-map.js +26 -5
- package/dist/schemas/errors.d.ts +2 -0
- package/dist/schemas/errors.js +11 -1
- package/dist/schemas/index.d.ts +51 -48
- package/dist/schemas/index.js +50 -47
- package/dist/schemas/llamacpp-config.d.ts +222 -6
- package/dist/schemas/llamacpp-config.js +129 -7
- package/dist/schemas/load-model.d.ts +7960 -1492
- package/dist/schemas/load-model.js +122 -10
- package/dist/schemas/mcp-adapter.d.ts +1 -1
- package/dist/schemas/model-config-schemas.d.ts +428 -9
- package/dist/schemas/model-config-schemas.js +16 -11
- package/dist/schemas/model-fit-policy.d.ts +17 -0
- package/dist/schemas/model-fit-policy.js +17 -0
- package/dist/schemas/model-info.d.ts +20 -2
- package/dist/schemas/model-info.js +4 -3
- package/dist/schemas/model-resource-profile.d.ts +4 -4
- package/dist/schemas/model-resource-profile.js +2 -5
- package/dist/schemas/model-src-utils.d.ts +14 -2
- package/dist/schemas/model-src-utils.js +2 -2
- package/dist/schemas/model-types.d.ts +14 -1
- package/dist/schemas/model-types.js +4 -0
- package/dist/schemas/ocr.d.ts +76 -0
- package/dist/schemas/ocr.js +43 -1
- package/dist/schemas/plugin.d.ts +16 -4
- package/dist/schemas/plugin.js +12 -0
- package/dist/schemas/profiling.d.ts +4 -4
- package/dist/schemas/profiling.js +1 -1
- package/dist/schemas/rag.js +1 -1
- package/dist/schemas/registry.d.ts +34 -0
- package/dist/schemas/registry.js +5 -1
- package/dist/schemas/reload-config.d.ts +14 -2
- package/dist/schemas/reload-config.js +2 -2
- package/dist/schemas/rerank.d.ts +131 -0
- package/dist/schemas/rerank.js +102 -0
- package/dist/schemas/rpc-server.js +3 -3
- package/dist/schemas/sdcpp-config.d.ts +82 -0
- package/dist/schemas/sdcpp-config.js +41 -22
- package/dist/schemas/system-resources.d.ts +4 -4
- package/dist/schemas/text-to-speech.d.ts +463 -3
- package/dist/schemas/text-to-speech.js +129 -11
- package/dist/schemas/tools.d.ts +13 -4
- package/dist/schemas/tools.js +18 -6
- package/dist/schemas/transcription-config.d.ts +20 -0
- package/dist/schemas/transcription-config.js +54 -4
- package/dist/schemas/transcription.d.ts +47 -5
- package/dist/schemas/transcription.js +52 -5
- package/dist/schemas/translate.d.ts +2 -2
- package/dist/schemas/translate.js +1 -1
- package/dist/schemas/translation-config.d.ts +40 -10
- package/dist/schemas/translation-config.js +1 -1
- package/dist/schemas/vla.d.ts +9 -2
- package/dist/schemas/vla.js +9 -5
- package/dist/surface.d.ts +3 -2
- package/dist/surface.js +3 -2
- package/dist/utils/addon-responses.d.ts +3 -0
- package/dist/utils/aggregate-events.d.ts +1 -1
- package/dist/utils/aggregate-events.js +2 -2
- package/dist/utils/archive.js +2 -2
- package/dist/utils/asr-events.d.ts +1 -0
- package/dist/utils/asr-events.js +2 -1
- package/dist/utils/audio/decoder.js +3 -1
- package/dist/utils/audio-input.js +1 -1
- package/dist/utils/cache/auto-history.d.ts +2 -1
- package/dist/utils/cache/auto-history.js +2 -2
- package/dist/utils/cache/paths.js +6 -6
- package/dist/utils/cache/types.d.ts +7 -0
- package/dist/utils/checksum.d.ts +2 -0
- package/dist/utils/checksum.js +41 -2
- package/dist/utils/completion-normalizer.js +1 -1
- package/dist/utils/gguf-architecture.d.ts +7 -0
- package/dist/utils/gguf-architecture.js +142 -0
- package/dist/utils/gguf-tensor-extractor.js +1 -1
- package/dist/utils/index.d.ts +13 -13
- package/dist/utils/index.js +13 -13
- package/dist/utils/mcp-adapter.d.ts +1 -1
- package/dist/utils/mcp-adapter.js +13 -9
- package/dist/utils/path-security.js +1 -1
- package/dist/utils/shard-utils.d.ts +5 -0
- package/dist/utils/shard-utils.js +19 -3
- package/dist/utils/tool-helpers.d.ts +1 -1
- package/dist/utils/tool-helpers.js +20 -52
- package/dist/utils/tool-integration.d.ts +1 -0
- package/dist/utils/tool-integration.js +3 -6
- package/dist/utils/tools/defer.d.ts +16 -26
- package/dist/utils/tools/defer.js +50 -41
- package/dist/utils/tools/dialect.d.ts +15 -4
- package/dist/utils/tools/dialect.js +34 -5
- package/dist/utils/tools/index.d.ts +3 -3
- package/dist/utils/tools/index.js +3 -3
- package/dist/utils/tools/parser.js +8 -8
- package/dist/utils/tools/parsers/dsml.d.ts +1 -1
- package/dist/utils/tools/parsers/dsml.js +8 -5
- package/dist/utils/tools/parsers/gemma4native.d.ts +1 -1
- package/dist/utils/tools/parsers/gemma4native.js +1 -1
- package/dist/utils/tools/parsers/harmony.d.ts +1 -1
- package/dist/utils/tools/parsers/harmony.js +1 -1
- package/dist/utils/tools/parsers/hermes.d.ts +1 -1
- package/dist/utils/tools/parsers/hermes.js +1 -1
- package/dist/utils/tools/parsers/json.d.ts +1 -1
- package/dist/utils/tools/parsers/json.js +1 -1
- package/dist/utils/tools/parsers/pythonic.d.ts +1 -1
- package/dist/utils/tools/parsers/pythonic.js +1 -1
- package/dist/utils/tools/parsers/qwen35.d.ts +1 -1
- package/dist/utils/tools/parsers/qwen35.js +7 -4
- package/dist/utils/tools/shared.d.ts +3 -0
- package/dist/utils/tools/shared.js +62 -0
- package/dist/utils/transcribe-metadata.d.ts +8 -1
- package/dist/utils/transcribe-metadata.js +13 -3
- package/dist/utils/tts-stats.d.ts +2 -1
- package/dist/utils/until-aborted.d.ts +7 -0
- package/dist/utils/until-aborted.js +34 -0
- package/package.json +68 -51
- package/dist/resources/model-fit/calibration/darwin-arm64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/darwin-arm64.js +0 -36
- package/dist/resources/model-fit/calibration/darwin-x64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/darwin-x64.js +0 -38
- package/dist/resources/model-fit/calibration/fit.d.ts +0 -73
- package/dist/resources/model-fit/calibration/fit.js +0 -152
- package/dist/resources/model-fit/calibration/index.d.ts +0 -37
- package/dist/resources/model-fit/calibration/index.js +0 -75
- package/dist/resources/model-fit/calibration/linux-arm64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/linux-arm64.js +0 -38
- package/dist/resources/model-fit/calibration/linux-x64-vulkan.d.ts +0 -8
- package/dist/resources/model-fit/calibration/linux-x64-vulkan.js +0 -39
- package/dist/resources/model-fit/calibration/linux-x64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/linux-x64.js +0 -38
- package/dist/resources/model-fit/calibration/win32-x64-vulkan-shared.d.ts +0 -8
- package/dist/resources/model-fit/calibration/win32-x64-vulkan-shared.js +0 -39
- package/dist/resources/model-fit/calibration/win32-x64-vulkan.d.ts +0 -8
- package/dist/resources/model-fit/calibration/win32-x64-vulkan.js +0 -36
- package/dist/resources/model-fit/calibration/win32-x64.d.ts +0 -8
- package/dist/resources/model-fit/calibration/win32-x64.js +0 -38
- package/dist/resources/model-fit/estimators/llm.d.ts +0 -46
- package/dist/resources/model-fit/estimators/llm.js +0 -222
- package/dist/resources/model-fit/estimators/whisper.d.ts +0 -10
- package/dist/resources/model-fit/estimators/whisper.js +0 -76
- package/dist/resources/model-fit/floor.d.ts +0 -31
- package/dist/resources/model-fit/native-probe/child-engine-fit.d.ts +0 -2
- package/dist/resources/model-fit/native-probe/child-engine-fit.js +0 -41
- package/dist/resources/model-fit/native-probe/create-fit-request.js +0 -51
- package/dist/resources/model-fit/native-probe/engines/audiogen.js +0 -35
- package/dist/resources/model-fit/native-probe/engines/diffusion.js +0 -55
- package/dist/resources/model-fit/native-probe/engines/llama.js +0 -145
- package/dist/resources/model-fit/native-probe/fit-outcome.d.ts +0 -16
- package/dist/resources/model-fit/native-probe/fit-outcome.js +0 -1
- package/dist/resources/model-fit/native-probe/fit-process.d.ts +0 -27
- package/dist/resources/model-fit/native-probe/fit-process.js +0 -55
- package/dist/resources/model-fit/native-probe/fit-runner.d.ts +0 -1
- package/dist/resources/model-fit/native-probe/fit-runner.js +0 -58
- package/dist/resources/model-fit/native-probe/run-fit.d.ts +0 -24
- package/dist/resources/model-fit/native-probe/run-fit.js +0 -20
- package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +0 -22
- package/dist/resources/model-fit/native-probe/run-in-process-fit.js +0 -108
- package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +0 -63
- package/dist/resources/model-fit/native-probe/run-isolated-fit.js +0 -377
- package/dist/resources/model-fit/platform.d.ts +0 -9
- package/dist/resources/model-fit/platform.js +0 -28
- package/dist/resources/model-fit/types.d.ts +0 -120
- package/dist/resources/model-fit/types.js +0 -1
- /package/dist/{resources/model-fit/native-probe/engines → model-fit/request}/gpu.d.ts +0 -0
- /package/dist/{resources/model-fit/native-probe/engines → model-fit/request}/gpu.js +0 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@qvac/inference",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.23.0",
|
|
4
4
|
"description": "Bare-only in-process core of the QVAC SDK. Runs inference directly on the Bare runtime with explicit plugin assembly and no RPC, worker, or subprocess layer.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"repository": {
|
|
@@ -141,6 +141,18 @@
|
|
|
141
141
|
"./ggml-rpc-server/provider": {
|
|
142
142
|
"types": "./dist/rpc/ggml-provider.d.ts",
|
|
143
143
|
"import": "./dist/rpc/ggml-provider.js"
|
|
144
|
+
},
|
|
145
|
+
"./moss-transcribe/plugin": {
|
|
146
|
+
"types": "./dist/plugins/builtin/moss-transcribe/plugin.d.ts",
|
|
147
|
+
"import": "./dist/plugins/builtin/moss-transcribe/plugin.js"
|
|
148
|
+
},
|
|
149
|
+
"./llamacpp-decision/plugin": {
|
|
150
|
+
"types": "./dist/plugins/builtin/llamacpp-decision/plugin.d.ts",
|
|
151
|
+
"import": "./dist/plugins/builtin/llamacpp-decision/plugin.js"
|
|
152
|
+
},
|
|
153
|
+
"./llamacpp-rerank/plugin": {
|
|
154
|
+
"types": "./dist/plugins/builtin/llamacpp-rerank/plugin.d.ts",
|
|
155
|
+
"import": "./dist/plugins/builtin/llamacpp-rerank/plugin.js"
|
|
144
156
|
}
|
|
145
157
|
},
|
|
146
158
|
"files": [
|
|
@@ -149,29 +161,13 @@
|
|
|
149
161
|
"LICENSE",
|
|
150
162
|
"NOTICE"
|
|
151
163
|
],
|
|
152
|
-
"scripts": {
|
|
153
|
-
"lint": "lunte",
|
|
154
|
-
"lint:fix": "lunte --fix",
|
|
155
|
-
"format": "prettier --check .",
|
|
156
|
-
"format:fix": "prettier --write .",
|
|
157
|
-
"typecheck": "tsc --noEmit -p tsconfig.json && tsc --noEmit -p test/tsconfig.json",
|
|
158
|
-
"compile": "tsc -p tsconfig.build.json",
|
|
159
|
-
"postcompile:aliases": "tsc-alias -p tsconfig.alias.json",
|
|
160
|
-
"build": "npm run compile && npm run postcompile:aliases",
|
|
161
|
-
"prepare": "npm run build",
|
|
162
|
-
"build:test": "tsc -p test/tsconfig.build.json && tsc-alias -p test/tsconfig.alias.json",
|
|
163
|
-
"test": "npm run build:test && brittle-bare 'test/dist/test/**/*.test.js'",
|
|
164
|
-
"test:bare": "npm run test",
|
|
165
|
-
"check-models": "bare ./dist/models/update-models/index.js --check",
|
|
166
|
-
"update-models": "bare ./dist/models/update-models/index.js && prettier --write src/models/registry/models.ts src/models/registry/resource-profiles.ts"
|
|
167
|
-
},
|
|
168
164
|
"engines": {
|
|
169
|
-
"bare": ">=1.
|
|
165
|
+
"bare": ">=1.32.0"
|
|
170
166
|
},
|
|
171
167
|
"dependencies": {
|
|
172
168
|
"@qvac/error": "^0.1.1",
|
|
173
169
|
"@qvac/logging": "^0.1.1",
|
|
174
|
-
"@qvac/rag": "^0.
|
|
170
|
+
"@qvac/rag": "^0.9.0",
|
|
175
171
|
"@qvac/registry-client": "^0.7.0",
|
|
176
172
|
"bare-abort-controller": "^1.1.2",
|
|
177
173
|
"bare-buffer": "^3.6.2",
|
|
@@ -180,14 +176,12 @@
|
|
|
180
176
|
"bare-env": "^3.0.1",
|
|
181
177
|
"bare-fs": "^4.7.4",
|
|
182
178
|
"bare-gpu-info": "0.1.1",
|
|
183
|
-
"bare-http1": "^4.
|
|
179
|
+
"bare-http1": "^4.6.2",
|
|
184
180
|
"bare-https": "^3.0.0",
|
|
185
181
|
"bare-net": "^2.3.2",
|
|
186
182
|
"bare-os": "^3.9.3",
|
|
187
183
|
"bare-path": "^3.1.1",
|
|
188
|
-
"bare-process": "^4.5.1",
|
|
189
184
|
"bare-rpc": "^1.3.8",
|
|
190
|
-
"bare-runtime": "^1.30.3",
|
|
191
185
|
"bare-stream": "^2.13.3",
|
|
192
186
|
"bare-url": "^2.4.5",
|
|
193
187
|
"bare-zlib": "^1.4.0",
|
|
@@ -199,23 +193,24 @@
|
|
|
199
193
|
"semver": "^7.8.5",
|
|
200
194
|
"tar-stream": "^3.2.0",
|
|
201
195
|
"which-runtime": "^1.2.1",
|
|
202
|
-
"zod": "
|
|
196
|
+
"zod": "4.6.5"
|
|
203
197
|
},
|
|
204
198
|
"peerDependencies": {
|
|
205
|
-
"@qvac/asr-ggml": "^0.
|
|
206
|
-
"@qvac/audiogen-ggml": "^0.
|
|
207
|
-
"@qvac/bci-whispercpp": "^0.
|
|
208
|
-
"@qvac/classification-ggml": "^0.
|
|
209
|
-
"@qvac/decoder-audio": "^0.
|
|
210
|
-
"@qvac/diffusion-cpp": "^0.
|
|
211
|
-
"@qvac/embed-llamacpp": "^0.
|
|
199
|
+
"@qvac/asr-ggml": "^0.11.1",
|
|
200
|
+
"@qvac/audiogen-ggml": "^0.9.0",
|
|
201
|
+
"@qvac/bci-whispercpp": "^0.14.0",
|
|
202
|
+
"@qvac/classification-ggml": "^0.34.0",
|
|
203
|
+
"@qvac/decoder-audio": "^0.8.1",
|
|
204
|
+
"@qvac/diffusion-cpp": "^0.31.0",
|
|
205
|
+
"@qvac/embed-llamacpp": "^0.53.0",
|
|
212
206
|
"@qvac/langdetect-text": "^0.1.2",
|
|
213
|
-
"@qvac/llm-llamacpp": "^0.
|
|
214
|
-
"@qvac/ocr-ggml": "^0.
|
|
215
|
-
"@qvac/translation-nmtcpp": "^0.
|
|
216
|
-
"@qvac/tts-ggml": "^0.
|
|
217
|
-
"@qvac/
|
|
218
|
-
"@qvac/ggml
|
|
207
|
+
"@qvac/llm-llamacpp": "^0.65.0",
|
|
208
|
+
"@qvac/ocr-ggml": "^0.31.0",
|
|
209
|
+
"@qvac/translation-nmtcpp": "^0.23.0",
|
|
210
|
+
"@qvac/tts-ggml": "^0.14.0",
|
|
211
|
+
"@qvac/turbovec-ggml": "^0.3.0",
|
|
212
|
+
"@qvac/vla-ggml": "^0.34.0",
|
|
213
|
+
"@qvac/ggml-rpc-server": "^0.7.0"
|
|
219
214
|
},
|
|
220
215
|
"peerDependenciesMeta": {
|
|
221
216
|
"@qvac/asr-ggml": {
|
|
@@ -254,6 +249,9 @@
|
|
|
254
249
|
"@qvac/tts-ggml": {
|
|
255
250
|
"optional": true
|
|
256
251
|
},
|
|
252
|
+
"@qvac/turbovec-ggml": {
|
|
253
|
+
"optional": true
|
|
254
|
+
},
|
|
257
255
|
"@qvac/vla-ggml": {
|
|
258
256
|
"optional": true
|
|
259
257
|
},
|
|
@@ -262,22 +260,25 @@
|
|
|
262
260
|
}
|
|
263
261
|
},
|
|
264
262
|
"devDependencies": {
|
|
265
|
-
"@qvac/asr-ggml": "^0.
|
|
266
|
-
"@qvac/audiogen-ggml": "^0.
|
|
267
|
-
"@qvac/bci-whispercpp": "^0.
|
|
268
|
-
"@qvac/classification-ggml": "^0.
|
|
269
|
-
"@qvac/decoder-audio": "^0.
|
|
270
|
-
"@qvac/diffusion-cpp": "^0.
|
|
271
|
-
"@qvac/embed-llamacpp": "^0.
|
|
263
|
+
"@qvac/asr-ggml": "^0.11.1",
|
|
264
|
+
"@qvac/audiogen-ggml": "^0.9.0",
|
|
265
|
+
"@qvac/bci-whispercpp": "^0.14.0",
|
|
266
|
+
"@qvac/classification-ggml": "^0.34.0",
|
|
267
|
+
"@qvac/decoder-audio": "^0.8.1",
|
|
268
|
+
"@qvac/diffusion-cpp": "^0.31.0",
|
|
269
|
+
"@qvac/embed-llamacpp": "^0.53.0",
|
|
272
270
|
"@qvac/langdetect-text": "^0.1.2",
|
|
273
|
-
"@qvac/llm-llamacpp": "^0.
|
|
274
|
-
"@qvac/ocr-ggml": "^0.
|
|
275
|
-
"@qvac/translation-nmtcpp": "^0.
|
|
276
|
-
"@qvac/tts-ggml": "^0.
|
|
277
|
-
"@qvac/
|
|
271
|
+
"@qvac/llm-llamacpp": "^0.65.0",
|
|
272
|
+
"@qvac/ocr-ggml": "^0.31.0",
|
|
273
|
+
"@qvac/translation-nmtcpp": "^0.23.0",
|
|
274
|
+
"@qvac/tts-ggml": "^0.14.0",
|
|
275
|
+
"@qvac/turbovec-ggml": "^0.3.0",
|
|
276
|
+
"@qvac/vla-ggml": "^0.34.0",
|
|
278
277
|
"@types/brittle": "^3.5.0",
|
|
279
|
-
"bare": "
|
|
278
|
+
"bare": "1.34.1 || >=1.35.1",
|
|
280
279
|
"bare-console": "*",
|
|
280
|
+
"bare-events": "^2.9.1",
|
|
281
|
+
"bare-process": "^4.5.1",
|
|
281
282
|
"bare-stdio": "^1.0.3",
|
|
282
283
|
"bare-subprocess": "^6.1.0",
|
|
283
284
|
"bare-timers": "*",
|
|
@@ -287,6 +288,22 @@
|
|
|
287
288
|
"prettier-config-holepunch": "^2.0.0",
|
|
288
289
|
"tsc-alias": "^1.8.17",
|
|
289
290
|
"typescript": "^5.9.3",
|
|
290
|
-
"@qvac/ggml-rpc-server": "^0.
|
|
291
|
+
"@qvac/ggml-rpc-server": "^0.7.0"
|
|
292
|
+
},
|
|
293
|
+
"gitHead": "4badf6ff1900bbf099b63792fe663a33cc0e6c82",
|
|
294
|
+
"scripts": {
|
|
295
|
+
"lint": "lunte",
|
|
296
|
+
"lint:fix": "lunte --fix",
|
|
297
|
+
"format": "prettier --check .",
|
|
298
|
+
"format:fix": "prettier --write .",
|
|
299
|
+
"typecheck": "tsc --noEmit -p tsconfig.json && tsc --noEmit -p test/tsconfig.json",
|
|
300
|
+
"compile": "tsc -p tsconfig.build.json",
|
|
301
|
+
"postcompile:aliases": "tsc-alias -p tsconfig.alias.json",
|
|
302
|
+
"build": "npm run compile && npm run postcompile:aliases",
|
|
303
|
+
"build:test": "tsc -p test/tsconfig.build.json && tsc-alias -p test/tsconfig.alias.json",
|
|
304
|
+
"test": "npm run build:test && brittle-bare 'test/dist/test/**/*.test.js'",
|
|
305
|
+
"test:bare": "npm run test",
|
|
306
|
+
"check-models": "bare ./dist/models/update-models/index.js --check",
|
|
307
|
+
"update-models": "bare ./dist/models/update-models/index.js && prettier --write src/models/registry/models.ts src/models/registry/resource-profiles.ts"
|
|
291
308
|
}
|
|
292
|
-
}
|
|
309
|
+
}
|
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
import type { PlatformCalibration } from '../types.js';
|
|
2
|
-
/**
|
|
3
|
-
* darwin-arm64 coefficients.
|
|
4
|
-
*
|
|
5
|
-
* Generated by `scripts/calibrate-model-fit.ts`; see `METHODOLOGY.md` next
|
|
6
|
-
* to this file for how the numbers are derived and validated.
|
|
7
|
-
*/
|
|
8
|
-
export declare const DARWIN_ARM64_CALIBRATION: PlatformCalibration;
|
|
@@ -1,36 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* darwin-arm64 coefficients.
|
|
3
|
-
*
|
|
4
|
-
* Generated by `scripts/calibrate-model-fit.ts`; see `METHODOLOGY.md` next
|
|
5
|
-
* to this file for how the numbers are derived and validated.
|
|
6
|
-
*/
|
|
7
|
-
export const DARWIN_ARM64_CALIBRATION = {
|
|
8
|
-
weightUpperCoeff: 1.01,
|
|
9
|
-
workingPeakBytes: {
|
|
10
|
-
lower: 0,
|
|
11
|
-
upper: 2870477
|
|
12
|
-
},
|
|
13
|
-
fixedOverheadBytes: {
|
|
14
|
-
lower: 5703296,
|
|
15
|
-
upper: 85542755
|
|
16
|
-
},
|
|
17
|
-
computeBufferBytesPerToken: {
|
|
18
|
-
lower: 92,
|
|
19
|
-
upper: 138
|
|
20
|
-
},
|
|
21
|
-
audioWindowBytes: {
|
|
22
|
-
lower: 0,
|
|
23
|
-
upper: 0
|
|
24
|
-
},
|
|
25
|
-
audioStreamingBytes: {
|
|
26
|
-
lower: 0,
|
|
27
|
-
upper: 0
|
|
28
|
-
},
|
|
29
|
-
validated: true,
|
|
30
|
-
measuredAt: '2026-09-03',
|
|
31
|
-
measuredOn: {
|
|
32
|
-
backend: 'metal',
|
|
33
|
-
device: 'Apple M4 Max',
|
|
34
|
-
kvElementBytes: 1.0625
|
|
35
|
-
}
|
|
36
|
-
};
|
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
import type { PlatformCalibration } from '../types.js';
|
|
2
|
-
/**
|
|
3
|
-
* darwin-x64 coefficients.
|
|
4
|
-
*
|
|
5
|
-
* Generated by `scripts/calibrate-model-fit.ts`; see `METHODOLOGY.md` next
|
|
6
|
-
* to this file for how the numbers are derived and validated.
|
|
7
|
-
*/
|
|
8
|
-
export declare const DARWIN_X64_CALIBRATION: PlatformCalibration;
|
|
@@ -1,38 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* darwin-x64 coefficients.
|
|
3
|
-
*
|
|
4
|
-
* Generated by `scripts/calibrate-model-fit.ts`; see `METHODOLOGY.md` next
|
|
5
|
-
* to this file for how the numbers are derived and validated.
|
|
6
|
-
*/
|
|
7
|
-
export const DARWIN_X64_CALIBRATION = {
|
|
8
|
-
weightUpperCoeff: 1.027,
|
|
9
|
-
workingPeakBytes: {
|
|
10
|
-
lower: 0,
|
|
11
|
-
upper: 92474573
|
|
12
|
-
},
|
|
13
|
-
fixedOverheadBytes: {
|
|
14
|
-
lower: 7161979,
|
|
15
|
-
upper: 54888742
|
|
16
|
-
},
|
|
17
|
-
computeBufferBytesPerToken: {
|
|
18
|
-
lower: 0,
|
|
19
|
-
upper: 0
|
|
20
|
-
},
|
|
21
|
-
audioWindowBytes: {
|
|
22
|
-
lower: 0,
|
|
23
|
-
upper: 0
|
|
24
|
-
},
|
|
25
|
-
audioStreamingBytes: {
|
|
26
|
-
lower: 0,
|
|
27
|
-
upper: 0
|
|
28
|
-
},
|
|
29
|
-
validated: true,
|
|
30
|
-
notes: [
|
|
31
|
-
"weights were loaded with load_mode 'none' so RSS counted them in full at load; a mapped weight set keeps at most the artifact size resident, so weightUpperCoeff remains an upper bound for the default mmap load"
|
|
32
|
-
],
|
|
33
|
-
measuredAt: '2026-09-03',
|
|
34
|
-
measuredOn: {
|
|
35
|
-
backend: 'cpu',
|
|
36
|
-
kvElementBytes: 2
|
|
37
|
-
}
|
|
38
|
-
};
|
|
@@ -1,73 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* The least-squares core of the calibration harness
|
|
3
|
-
* (`scripts/calibrate-model-fit.ts`), kept in `src` so it is unit-testable —
|
|
4
|
-
* the harness itself needs real model loads, but the arithmetic that turns
|
|
5
|
-
* measurements into coefficients does not.
|
|
6
|
-
*
|
|
7
|
-
* The model is dictated by how llama.cpp actually allocates: everything is
|
|
8
|
-
* built when the model loads — weights, KV cache, engine overhead, and the
|
|
9
|
-
* context-scaled compute buffers — so a load's RSS delta is
|
|
10
|
-
*
|
|
11
|
-
* persistent = ratio × artifactBytes + kv + fixed + perToken × context
|
|
12
|
-
*
|
|
13
|
-
* The KV term is computed exactly per point and subtracted before fitting, so
|
|
14
|
-
* three unknowns remain: the weight-slack ratio, the fixed overhead, and the
|
|
15
|
-
* per-token slope. Three models at two contexts each give six points for a
|
|
16
|
-
* three-parameter plane; repeats add more.
|
|
17
|
-
*/
|
|
18
|
-
/** One measured load, with the KV cache the engine allocated for it. */
|
|
19
|
-
export interface CalibrationPoint {
|
|
20
|
-
artifactBytes: number;
|
|
21
|
-
contextTokens: number;
|
|
22
|
-
/** KV-cache bytes for this shape and context, exactly determined. */
|
|
23
|
-
kvBytes: number;
|
|
24
|
-
/** RSS delta across the load, after settle. */
|
|
25
|
-
persistentBytes: number;
|
|
26
|
-
}
|
|
27
|
-
/** Coefficients recovered from a set of calibration points. */
|
|
28
|
-
export interface ResidentFit {
|
|
29
|
-
/** Marginal resident bytes per artifact byte. */
|
|
30
|
-
weightRatio: number;
|
|
31
|
-
fixedBytes: number;
|
|
32
|
-
perTokenBytes: number;
|
|
33
|
-
/**
|
|
34
|
-
* Largest amount any point sits above the fitted (and clamped) plane. An
|
|
35
|
-
* upper bound that does not cover an observed point is not an upper bound,
|
|
36
|
-
* so this floors the fixed-overhead upper bound.
|
|
37
|
-
*/
|
|
38
|
-
worstExcessBytes: number;
|
|
39
|
-
}
|
|
40
|
-
/**
|
|
41
|
-
* Fits `persistent − kv = ratio × artifact + perToken × context + fixed` by
|
|
42
|
-
* ordinary least squares.
|
|
43
|
-
*
|
|
44
|
-
* Negative solutions are clamped to zero — a physical coefficient cannot be
|
|
45
|
-
* negative, and a slightly negative fit is measurement noise — but
|
|
46
|
-
* `worstExcessBytes` is computed against the clamped plane, so the clamp can
|
|
47
|
-
* only widen the eventual upper bound, never thin it.
|
|
48
|
-
*
|
|
49
|
-
* @returns The fit, or `undefined` when the points cannot determine three
|
|
50
|
-
* parameters (fewer than three points, or a degenerate design — e.g. a
|
|
51
|
-
* single model or a single context throughout).
|
|
52
|
-
*/
|
|
53
|
-
export declare function fitResidentMemory(points: readonly CalibrationPoint[]): ResidentFit | undefined;
|
|
54
|
-
/** One model's KV growth between its smallest and largest context, computed vs observed. */
|
|
55
|
-
export interface KvGrowth {
|
|
56
|
-
artifactBytes: number;
|
|
57
|
-
kvDeltaBytes: number;
|
|
58
|
-
observedDeltaBytes: number;
|
|
59
|
-
}
|
|
60
|
-
/** Whether the counter behind the measurements sees allocation at all. */
|
|
61
|
-
export interface KvObservation {
|
|
62
|
-
models: readonly KvGrowth[];
|
|
63
|
-
/** Observed over computed growth; `1` when no model has two contexts. */
|
|
64
|
-
ratio: number;
|
|
65
|
-
}
|
|
66
|
-
/**
|
|
67
|
-
* Checks the measurements against the one term the file sizes exactly: the KV
|
|
68
|
-
* cache grows by a known amount between contexts, and compute buffers only add
|
|
69
|
-
* to it. Growth below that means the wrong cache type was subtracted, or a
|
|
70
|
-
* counter that misses allocation. Summed across models, repeats as medians so a
|
|
71
|
-
* cold first load does not read as a shortfall.
|
|
72
|
-
*/
|
|
73
|
-
export declare function kvObservation(points: readonly CalibrationPoint[]): KvObservation;
|
|
@@ -1,152 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* The least-squares core of the calibration harness
|
|
3
|
-
* (`scripts/calibrate-model-fit.ts`), kept in `src` so it is unit-testable —
|
|
4
|
-
* the harness itself needs real model loads, but the arithmetic that turns
|
|
5
|
-
* measurements into coefficients does not.
|
|
6
|
-
*
|
|
7
|
-
* The model is dictated by how llama.cpp actually allocates: everything is
|
|
8
|
-
* built when the model loads — weights, KV cache, engine overhead, and the
|
|
9
|
-
* context-scaled compute buffers — so a load's RSS delta is
|
|
10
|
-
*
|
|
11
|
-
* persistent = ratio × artifactBytes + kv + fixed + perToken × context
|
|
12
|
-
*
|
|
13
|
-
* The KV term is computed exactly per point and subtracted before fitting, so
|
|
14
|
-
* three unknowns remain: the weight-slack ratio, the fixed overhead, and the
|
|
15
|
-
* per-token slope. Three models at two contexts each give six points for a
|
|
16
|
-
* three-parameter plane; repeats add more.
|
|
17
|
-
*/
|
|
18
|
-
/**
|
|
19
|
-
* Fits `persistent − kv = ratio × artifact + perToken × context + fixed` by
|
|
20
|
-
* ordinary least squares.
|
|
21
|
-
*
|
|
22
|
-
* Negative solutions are clamped to zero — a physical coefficient cannot be
|
|
23
|
-
* negative, and a slightly negative fit is measurement noise — but
|
|
24
|
-
* `worstExcessBytes` is computed against the clamped plane, so the clamp can
|
|
25
|
-
* only widen the eventual upper bound, never thin it.
|
|
26
|
-
*
|
|
27
|
-
* @returns The fit, or `undefined` when the points cannot determine three
|
|
28
|
-
* parameters (fewer than three points, or a degenerate design — e.g. a
|
|
29
|
-
* single model or a single context throughout).
|
|
30
|
-
*/
|
|
31
|
-
export function fitResidentMemory(points) {
|
|
32
|
-
if (points.length < 3)
|
|
33
|
-
return undefined;
|
|
34
|
-
// Normal equations for y = a·x1 + b·x2 + c.
|
|
35
|
-
let s11 = 0;
|
|
36
|
-
let s12 = 0;
|
|
37
|
-
let s1 = 0;
|
|
38
|
-
let s22 = 0;
|
|
39
|
-
let s2 = 0;
|
|
40
|
-
let s1y = 0;
|
|
41
|
-
let s2y = 0;
|
|
42
|
-
let sy = 0;
|
|
43
|
-
const n = points.length;
|
|
44
|
-
for (const point of points) {
|
|
45
|
-
const x1 = point.artifactBytes;
|
|
46
|
-
const x2 = point.contextTokens;
|
|
47
|
-
const y = point.persistentBytes - point.kvBytes;
|
|
48
|
-
s11 += x1 * x1;
|
|
49
|
-
s12 += x1 * x2;
|
|
50
|
-
s1 += x1;
|
|
51
|
-
s22 += x2 * x2;
|
|
52
|
-
s2 += x2;
|
|
53
|
-
s1y += x1 * y;
|
|
54
|
-
s2y += x2 * y;
|
|
55
|
-
sy += y;
|
|
56
|
-
}
|
|
57
|
-
const solution = solve3([
|
|
58
|
-
[s11, s12, s1],
|
|
59
|
-
[s12, s22, s2],
|
|
60
|
-
[s1, s2, n]
|
|
61
|
-
], [s1y, s2y, sy]);
|
|
62
|
-
if (!solution)
|
|
63
|
-
return undefined;
|
|
64
|
-
const weightRatio = Math.max(0, solution[0]);
|
|
65
|
-
const perTokenBytes = Math.max(0, solution[1]);
|
|
66
|
-
const fixedBytes = Math.max(0, solution[2]);
|
|
67
|
-
let worstExcessBytes = 0;
|
|
68
|
-
for (const point of points) {
|
|
69
|
-
const predicted = weightRatio * point.artifactBytes +
|
|
70
|
-
perTokenBytes * point.contextTokens +
|
|
71
|
-
fixedBytes +
|
|
72
|
-
point.kvBytes;
|
|
73
|
-
worstExcessBytes = Math.max(worstExcessBytes, point.persistentBytes - predicted);
|
|
74
|
-
}
|
|
75
|
-
const fit = { weightRatio, fixedBytes, perTokenBytes, worstExcessBytes };
|
|
76
|
-
return fit;
|
|
77
|
-
}
|
|
78
|
-
/**
|
|
79
|
-
* Checks the measurements against the one term the file sizes exactly: the KV
|
|
80
|
-
* cache grows by a known amount between contexts, and compute buffers only add
|
|
81
|
-
* to it. Growth below that means the wrong cache type was subtracted, or a
|
|
82
|
-
* counter that misses allocation. Summed across models, repeats as medians so a
|
|
83
|
-
* cold first load does not read as a shortfall.
|
|
84
|
-
*/
|
|
85
|
-
export function kvObservation(points) {
|
|
86
|
-
const byModel = new Map();
|
|
87
|
-
for (const point of points) {
|
|
88
|
-
const group = byModel.get(point.artifactBytes);
|
|
89
|
-
if (group)
|
|
90
|
-
group.push(point);
|
|
91
|
-
else
|
|
92
|
-
byModel.set(point.artifactBytes, [point]);
|
|
93
|
-
}
|
|
94
|
-
const models = [];
|
|
95
|
-
let kvDeltaTotal = 0;
|
|
96
|
-
let observedDeltaTotal = 0;
|
|
97
|
-
for (const [artifactBytes, group] of byModel) {
|
|
98
|
-
const contexts = [...new Set(group.map((p) => p.contextTokens))].sort((a, b) => a - b);
|
|
99
|
-
if (contexts.length < 2)
|
|
100
|
-
continue;
|
|
101
|
-
const low = group.filter((p) => p.contextTokens === contexts[0]);
|
|
102
|
-
const high = group.filter((p) => p.contextTokens === contexts[contexts.length - 1]);
|
|
103
|
-
const kvDeltaBytes = high[0].kvBytes - low[0].kvBytes;
|
|
104
|
-
const observedDeltaBytes = median(high.map((p) => p.persistentBytes)) - median(low.map((p) => p.persistentBytes));
|
|
105
|
-
models.push({ artifactBytes, kvDeltaBytes, observedDeltaBytes });
|
|
106
|
-
kvDeltaTotal += kvDeltaBytes;
|
|
107
|
-
observedDeltaTotal += observedDeltaBytes;
|
|
108
|
-
}
|
|
109
|
-
return { models, ratio: kvDeltaTotal > 0 ? observedDeltaTotal / kvDeltaTotal : 1 };
|
|
110
|
-
}
|
|
111
|
-
function median(values) {
|
|
112
|
-
const sorted = [...values].sort((a, b) => a - b);
|
|
113
|
-
const middle = Math.floor(sorted.length / 2);
|
|
114
|
-
return sorted.length % 2 === 1 ? sorted[middle] : (sorted[middle - 1] + sorted[middle]) / 2;
|
|
115
|
-
}
|
|
116
|
-
/**
|
|
117
|
-
* Solves a 3×3 linear system by Gaussian elimination with partial pivoting.
|
|
118
|
-
*
|
|
119
|
-
* @returns The solution vector, or `undefined` when the matrix is singular —
|
|
120
|
-
* which for the normal equations means the measurement design cannot
|
|
121
|
-
* separate the three coefficients.
|
|
122
|
-
*/
|
|
123
|
-
function solve3(matrix, rhs) {
|
|
124
|
-
const a = matrix.map((row, i) => [...row, rhs[i]]);
|
|
125
|
-
for (let col = 0; col < 3; col++) {
|
|
126
|
-
let pivot = col;
|
|
127
|
-
for (let row = col + 1; row < 3; row++) {
|
|
128
|
-
if (Math.abs(a[row][col]) > Math.abs(a[pivot][col]))
|
|
129
|
-
pivot = row;
|
|
130
|
-
}
|
|
131
|
-
if (Math.abs(a[pivot][col]) < 1e-9)
|
|
132
|
-
return undefined;
|
|
133
|
-
if (pivot !== col) {
|
|
134
|
-
const swap = a[col];
|
|
135
|
-
a[col] = a[pivot];
|
|
136
|
-
a[pivot] = swap;
|
|
137
|
-
}
|
|
138
|
-
for (let row = col + 1; row < 3; row++) {
|
|
139
|
-
const factor = a[row][col] / a[col][col];
|
|
140
|
-
for (let k = col; k <= 3; k++)
|
|
141
|
-
a[row][k] = a[row][k] - factor * a[col][k];
|
|
142
|
-
}
|
|
143
|
-
}
|
|
144
|
-
const x = [0, 0, 0];
|
|
145
|
-
for (let row = 2; row >= 0; row--) {
|
|
146
|
-
let sum = a[row][3];
|
|
147
|
-
for (let k = row + 1; k < 3; k++)
|
|
148
|
-
sum -= a[row][k] * x[k];
|
|
149
|
-
x[row] = sum / a[row][row];
|
|
150
|
-
}
|
|
151
|
-
return x;
|
|
152
|
-
}
|
|
@@ -1,37 +0,0 @@
|
|
|
1
|
-
import type { CalibrationFixture, ModelFitPlatform, PlatformCalibration } from '../types.js';
|
|
2
|
-
/**
|
|
3
|
-
* Every platform this feature has coefficients for.
|
|
4
|
-
*
|
|
5
|
-
* A platform is added here only once the calibration harness has measured it
|
|
6
|
-
* and a held-out model validated inside the bounds. Everything else assesses as
|
|
7
|
-
* `unknown` — see `METHODOLOGY.md` next to this file.
|
|
8
|
-
*/
|
|
9
|
-
export declare const CALIBRATION: CalibrationFixture;
|
|
10
|
-
/**
|
|
11
|
-
* Looks up the coefficients for a platform.
|
|
12
|
-
*
|
|
13
|
-
* @param platform - Runtime platform-arch pair.
|
|
14
|
-
* @returns The calibration, or `undefined` when the platform is absent or its
|
|
15
|
-
* coefficients are still provisional. Both cases assess as `unknown`.
|
|
16
|
-
*/
|
|
17
|
-
export declare function getPlatformCalibration(platform: ModelFitPlatform): PlatformCalibration | undefined;
|
|
18
|
-
/**
|
|
19
|
-
* Looks up GPU-resident coefficients for a platform on a specific backend.
|
|
20
|
-
*
|
|
21
|
-
* Keyed by backend as well as platform because a `linux-x64` host may run
|
|
22
|
-
* Vulkan, CUDA or ROCm, and their buffers differ — unlike the CPU fixtures,
|
|
23
|
-
* where one platform entry is accepted as covering every backend.
|
|
24
|
-
*
|
|
25
|
-
* @returns `undefined` when that pair has not been measured, which assesses as
|
|
26
|
-
* `unknown` rather than borrowing another backend's coefficients.
|
|
27
|
-
*/
|
|
28
|
-
export declare function getGpuCalibration(platform: ModelFitPlatform, backend: string): PlatformCalibration | undefined;
|
|
29
|
-
/**
|
|
30
|
-
* Looks up integrated-GPU coefficients, spent against the system budget. The
|
|
31
|
-
* gap from the device fixture is large: `win32-x64:vulkan` fits a weight ratio
|
|
32
|
-
* of 1.02 against the card's memory and 2.04 against system RAM.
|
|
33
|
-
*
|
|
34
|
-
* @returns `undefined` when unmeasured, which assesses as `unknown` rather
|
|
35
|
-
* than borrowing the platform's CPU-forced coefficients.
|
|
36
|
-
*/
|
|
37
|
-
export declare function getSharedGpuCalibration(platform: ModelFitPlatform, backend: string): PlatformCalibration | undefined;
|
|
@@ -1,75 +0,0 @@
|
|
|
1
|
-
import { DARWIN_ARM64_CALIBRATION } from './darwin-arm64.js';
|
|
2
|
-
import { DARWIN_X64_CALIBRATION } from './darwin-x64.js';
|
|
3
|
-
import { LINUX_ARM64_CALIBRATION } from './linux-arm64.js';
|
|
4
|
-
import { LINUX_X64_CALIBRATION } from './linux-x64.js';
|
|
5
|
-
import { LINUX_X64_VULKAN_CALIBRATION } from './linux-x64-vulkan.js';
|
|
6
|
-
import { WIN32_X64_CALIBRATION } from './win32-x64.js';
|
|
7
|
-
import { WIN32_X64_VULKAN_CALIBRATION } from './win32-x64-vulkan.js';
|
|
8
|
-
import { WIN32_X64_VULKAN_SHARED_CALIBRATION } from './win32-x64-vulkan-shared.js';
|
|
9
|
-
/**
|
|
10
|
-
* Every platform this feature has coefficients for.
|
|
11
|
-
*
|
|
12
|
-
* A platform is added here only once the calibration harness has measured it
|
|
13
|
-
* and a held-out model validated inside the bounds. Everything else assesses as
|
|
14
|
-
* `unknown` — see `METHODOLOGY.md` next to this file.
|
|
15
|
-
*/
|
|
16
|
-
export const CALIBRATION = {
|
|
17
|
-
schemaVersion: 1,
|
|
18
|
-
platforms: {
|
|
19
|
-
'darwin-arm64': DARWIN_ARM64_CALIBRATION,
|
|
20
|
-
'darwin-x64': DARWIN_X64_CALIBRATION,
|
|
21
|
-
'linux-arm64': LINUX_ARM64_CALIBRATION,
|
|
22
|
-
'linux-x64': LINUX_X64_CALIBRATION,
|
|
23
|
-
'win32-x64': WIN32_X64_CALIBRATION
|
|
24
|
-
},
|
|
25
|
-
gpuPlatforms: {
|
|
26
|
-
'linux-x64:vulkan': LINUX_X64_VULKAN_CALIBRATION,
|
|
27
|
-
'win32-x64:vulkan': WIN32_X64_VULKAN_CALIBRATION
|
|
28
|
-
},
|
|
29
|
-
sharedGpuPlatforms: {
|
|
30
|
-
'win32-x64:vulkan': WIN32_X64_VULKAN_SHARED_CALIBRATION
|
|
31
|
-
}
|
|
32
|
-
};
|
|
33
|
-
/**
|
|
34
|
-
* Looks up the coefficients for a platform.
|
|
35
|
-
*
|
|
36
|
-
* @param platform - Runtime platform-arch pair.
|
|
37
|
-
* @returns The calibration, or `undefined` when the platform is absent or its
|
|
38
|
-
* coefficients are still provisional. Both cases assess as `unknown`.
|
|
39
|
-
*/
|
|
40
|
-
export function getPlatformCalibration(platform) {
|
|
41
|
-
const calibration = CALIBRATION.platforms[platform];
|
|
42
|
-
if (!calibration || !calibration.validated)
|
|
43
|
-
return undefined;
|
|
44
|
-
return calibration;
|
|
45
|
-
}
|
|
46
|
-
/**
|
|
47
|
-
* Looks up GPU-resident coefficients for a platform on a specific backend.
|
|
48
|
-
*
|
|
49
|
-
* Keyed by backend as well as platform because a `linux-x64` host may run
|
|
50
|
-
* Vulkan, CUDA or ROCm, and their buffers differ — unlike the CPU fixtures,
|
|
51
|
-
* where one platform entry is accepted as covering every backend.
|
|
52
|
-
*
|
|
53
|
-
* @returns `undefined` when that pair has not been measured, which assesses as
|
|
54
|
-
* `unknown` rather than borrowing another backend's coefficients.
|
|
55
|
-
*/
|
|
56
|
-
export function getGpuCalibration(platform, backend) {
|
|
57
|
-
const calibration = CALIBRATION.gpuPlatforms?.[`${platform}:${backend}`];
|
|
58
|
-
if (!calibration || !calibration.validated)
|
|
59
|
-
return undefined;
|
|
60
|
-
return calibration;
|
|
61
|
-
}
|
|
62
|
-
/**
|
|
63
|
-
* Looks up integrated-GPU coefficients, spent against the system budget. The
|
|
64
|
-
* gap from the device fixture is large: `win32-x64:vulkan` fits a weight ratio
|
|
65
|
-
* of 1.02 against the card's memory and 2.04 against system RAM.
|
|
66
|
-
*
|
|
67
|
-
* @returns `undefined` when unmeasured, which assesses as `unknown` rather
|
|
68
|
-
* than borrowing the platform's CPU-forced coefficients.
|
|
69
|
-
*/
|
|
70
|
-
export function getSharedGpuCalibration(platform, backend) {
|
|
71
|
-
const calibration = CALIBRATION.sharedGpuPlatforms?.[`${platform}:${backend}`];
|
|
72
|
-
if (!calibration || !calibration.validated)
|
|
73
|
-
return undefined;
|
|
74
|
-
return calibration;
|
|
75
|
-
}
|
|
@@ -1,8 +0,0 @@
|
|
|
1
|
-
import type { PlatformCalibration } from '../types.js';
|
|
2
|
-
/**
|
|
3
|
-
* linux-arm64 coefficients.
|
|
4
|
-
*
|
|
5
|
-
* Generated by `scripts/calibrate-model-fit.ts`; see `METHODOLOGY.md` next
|
|
6
|
-
* to this file for how the numbers are derived and validated.
|
|
7
|
-
*/
|
|
8
|
-
export declare const LINUX_ARM64_CALIBRATION: PlatformCalibration;
|
|
@@ -1,38 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* linux-arm64 coefficients.
|
|
3
|
-
*
|
|
4
|
-
* Generated by `scripts/calibrate-model-fit.ts`; see `METHODOLOGY.md` next
|
|
5
|
-
* to this file for how the numbers are derived and validated.
|
|
6
|
-
*/
|
|
7
|
-
export const LINUX_ARM64_CALIBRATION = {
|
|
8
|
-
weightUpperCoeff: 1.088,
|
|
9
|
-
workingPeakBytes: {
|
|
10
|
-
lower: 0,
|
|
11
|
-
upper: 8178893
|
|
12
|
-
},
|
|
13
|
-
fixedOverheadBytes: {
|
|
14
|
-
lower: 108926309,
|
|
15
|
-
upper: 238075911
|
|
16
|
-
},
|
|
17
|
-
computeBufferBytesPerToken: {
|
|
18
|
-
lower: 0,
|
|
19
|
-
upper: 0
|
|
20
|
-
},
|
|
21
|
-
audioWindowBytes: {
|
|
22
|
-
lower: 0,
|
|
23
|
-
upper: 0
|
|
24
|
-
},
|
|
25
|
-
audioStreamingBytes: {
|
|
26
|
-
lower: 0,
|
|
27
|
-
upper: 0
|
|
28
|
-
},
|
|
29
|
-
validated: true,
|
|
30
|
-
notes: [
|
|
31
|
-
"weights were loaded with load_mode 'none' so RSS counted them in full at load; a mapped weight set keeps at most the artifact size resident, so weightUpperCoeff remains an upper bound for the default mmap load"
|
|
32
|
-
],
|
|
33
|
-
measuredAt: '2026-09-03',
|
|
34
|
-
measuredOn: {
|
|
35
|
-
backend: 'cpu',
|
|
36
|
-
kvElementBytes: 2
|
|
37
|
-
}
|
|
38
|
-
};
|