@qvac/inference 0.22.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +111 -60
- package/README.md +4 -2
- package/dist/api/assess-model-fit.d.ts +14 -14
- package/dist/api/assess-model-fit.js +14 -14
- package/dist/api/audio-gen.d.ts +2 -2
- package/dist/api/audio-gen.js +136 -45
- package/dist/api/batch-completion.js +2 -1
- package/dist/api/calc-tokens.d.ts +30 -0
- package/dist/api/calc-tokens.js +41 -0
- package/dist/api/classify.d.ts +5 -2
- package/dist/api/classify.js +5 -2
- package/dist/api/completion-stream.d.ts +2 -1
- package/dist/api/completion-stream.js +4 -1
- package/dist/api/decide.js +1 -1
- package/dist/api/delete-cache.d.ts +7 -1
- package/dist/api/delete-cache.js +11 -1
- package/dist/api/diffusion.d.ts +32 -2
- package/dist/api/diffusion.js +32 -2
- package/dist/api/download-asset.js +1 -1
- package/dist/api/embed.d.ts +82 -29
- package/dist/api/embed.js +64 -18
- package/dist/api/finetune.js +1 -1
- package/dist/api/get-model-info.d.ts +1 -1
- package/dist/api/index.d.ts +40 -38
- package/dist/api/index.js +39 -37
- package/dist/api/load-model.js +2 -2
- package/dist/api/logging-stream-registry.js +11 -7
- package/dist/api/logging-stream.d.ts +11 -1
- package/dist/api/logging-stream.js +11 -2
- package/dist/api/rag.d.ts +1 -1
- package/dist/api/rag.js +1 -1
- package/dist/api/registry.d.ts +18 -7
- package/dist/api/registry.js +37 -13
- package/dist/api/rerank.d.ts +17 -0
- package/dist/api/rerank.js +36 -0
- package/dist/api/subscribe-logs.js +5 -3
- package/dist/api/text-to-speech.d.ts +14 -5
- package/dist/api/text-to-speech.js +83 -29
- package/dist/api/transcribe.d.ts +6 -2
- package/dist/api/transcribe.js +7 -1
- package/dist/api/unload-model.js +1 -1
- package/dist/api/vector-index.d.ts +25 -3
- package/dist/api/vector-index.js +17 -2
- package/dist/api/video-request.d.ts +15 -15
- package/dist/api/video.js +1 -1
- package/dist/api/vla.js +1 -1
- package/dist/api/world-result.js +1 -1
- package/dist/api/world.d.ts +1 -1
- package/dist/api/world.js +1 -1
- package/dist/config/resolve-config.d.ts +1 -1
- package/dist/config/resolve-config.js +1 -1
- package/dist/constants/index.d.ts +2 -2
- package/dist/constants/index.js +2 -2
- package/dist/dispatch.d.ts +11 -4
- package/dist/dispatch.js +80 -18
- package/dist/engine.d.ts +1 -1
- package/dist/errors/index.d.ts +76 -0
- package/dist/errors/index.js +141 -1
- package/dist/handlers/assess-model-fit.d.ts +3 -21
- package/dist/handlers/assess-model-fit.js +4 -303
- package/dist/handlers/cancelHandler.d.ts +10 -9
- package/dist/handlers/cancelHandler.js +18 -14
- package/dist/handlers/completion-orchestrate.d.ts +14 -2
- package/dist/handlers/completion-orchestrate.js +166 -24
- package/dist/handlers/delete-cache.d.ts +2 -1
- package/dist/handlers/delete-cache.js +6 -1
- package/dist/handlers/download-asset.js +13 -6
- package/dist/handlers/finetune.js +1 -1
- package/dist/handlers/get-loaded-model-info.js +3 -3
- package/dist/handlers/get-model-info.js +72 -3
- package/dist/handlers/load-model/coreml-sidecars.d.ts +15 -11
- package/dist/handlers/load-model/coreml-sidecars.js +51 -91
- package/dist/handlers/load-model/download-lock.d.ts +20 -0
- package/dist/handlers/load-model/download-lock.js +213 -0
- package/dist/handlers/load-model/download-manager.d.ts +11 -3
- package/dist/handlers/load-model/download-manager.js +62 -11
- package/dist/handlers/load-model/download-stats.d.ts +2 -2
- package/dist/handlers/load-model/download-stats.js +3 -3
- package/dist/handlers/load-model/handler.js +97 -41
- package/dist/handlers/load-model/http-verify.d.ts +1 -1
- package/dist/handlers/load-model/http.d.ts +1 -1
- package/dist/handlers/load-model/http.js +330 -208
- package/dist/handlers/load-model/hyperdrive.d.ts +16 -1
- package/dist/handlers/load-model/hyperdrive.js +125 -52
- package/dist/handlers/load-model/index.d.ts +2 -2
- package/dist/handlers/load-model/index.js +2 -2
- package/dist/handlers/load-model/registry-client-options.d.ts +2 -0
- package/dist/handlers/load-model/registry-client-options.js +4 -2
- package/dist/handlers/load-model/registry-companion-set.d.ts +6 -1
- package/dist/handlers/load-model/registry-companion-set.js +20 -2
- package/dist/handlers/load-model/registry-delete.d.ts +5 -0
- package/dist/handlers/load-model/registry-delete.js +269 -0
- package/dist/handlers/load-model/registry-download-utils.d.ts +7 -5
- package/dist/handlers/load-model/registry-download-utils.js +95 -24
- package/dist/handlers/load-model/registry.d.ts +22 -2
- package/dist/handlers/load-model/registry.js +312 -179
- package/dist/handlers/load-model/resolve-session.d.ts +1 -1
- package/dist/handlers/load-model/resolve-session.js +11 -3
- package/dist/handlers/load-model/resolve.d.ts +1 -1
- package/dist/handlers/load-model/resolve.js +9 -6
- package/dist/handlers/load-model/types.d.ts +2 -0
- package/dist/handlers/logging-stream.d.ts +2 -1
- package/dist/handlers/logging-stream.js +35 -7
- package/dist/handlers/plugin-invoke.js +152 -39
- package/dist/handlers/registry.d.ts +2 -2
- package/dist/handlers/registry.js +108 -29
- package/dist/handlers/rpc-server.js +7 -7
- package/dist/handlers/session.d.ts +11 -0
- package/dist/handlers/session.js +40 -0
- package/dist/handlers/types.d.ts +7 -1
- package/dist/handlers/unload-model.js +8 -3
- package/dist/handlers/vector-index.js +118 -19
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/logging/addon.d.ts +1 -1
- package/dist/logging/addon.js +1 -1
- package/dist/logging/index.d.ts +5 -5
- package/dist/logging/index.js +4 -4
- package/dist/logging/logger.d.ts +1 -1
- package/dist/logging/logger.js +3 -3
- package/dist/logging/utils.js +11 -3
- package/dist/model-cache-layout.d.ts +1 -0
- package/dist/model-cache-layout.js +2 -0
- package/dist/model-fit/assess.d.ts +23 -0
- package/dist/model-fit/assess.js +110 -0
- package/dist/model-fit/describe.d.ts +12 -0
- package/dist/model-fit/describe.js +29 -0
- package/dist/{resources/model-fit/fit-stub/fetch-fit-stub.d.ts → model-fit/description/fetch.d.ts} +2 -2
- package/dist/{resources/model-fit/fit-stub/fetch-fit-stub.js → model-fit/description/fetch.js} +98 -19
- package/dist/{resources/model-fit/fit-stub/resolve-load-from-stubs.d.ts → model-fit/description/resolve-load.d.ts} +8 -6
- package/dist/{resources/model-fit/fit-stub/resolve-load-from-stubs.js → model-fit/description/resolve-load.js} +20 -20
- package/dist/model-fit/fit-load.d.ts +23 -0
- package/dist/model-fit/fit-load.js +29 -0
- package/dist/model-fit/load-check.d.ts +34 -0
- package/dist/model-fit/load-check.js +77 -0
- package/dist/model-fit/run/engine.d.ts +12 -0
- package/dist/model-fit/run/engine.js +9 -0
- package/dist/model-fit/run/run.d.ts +46 -0
- package/dist/model-fit/run/run.js +208 -0
- package/dist/model-fit/sum.d.ts +32 -0
- package/dist/model-fit/sum.js +60 -0
- package/dist/models/registry/index.d.ts +1 -1
- package/dist/models/registry/index.js +2 -2
- package/dist/models/registry/models.d.ts +1377 -130
- package/dist/models/registry/models.js +3622 -2312
- package/dist/plugins/builtin/asr-ggml/config.d.ts +19 -4
- package/dist/plugins/builtin/asr-ggml/config.js +39 -3
- package/dist/plugins/builtin/asr-ggml/fit.d.ts +18 -0
- package/dist/plugins/builtin/asr-ggml/fit.js +118 -0
- package/dist/plugins/builtin/audiogen-ggml/config.d.ts +6 -0
- package/dist/plugins/builtin/audiogen-ggml/config.js +9 -0
- package/dist/plugins/builtin/audiogen-ggml/fit.d.ts +3 -0
- package/dist/{resources/model-fit/native-probe/engines/audiogen.js → plugins/builtin/audiogen-ggml/fit.js} +28 -18
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +4 -3
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.d.ts +17 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.js +68 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +11 -3
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +47 -11
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +4 -3
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +4 -3
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +203 -72
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +17 -5
- package/dist/plugins/builtin/bci-whispercpp-transcription/fit.d.ts +3 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/fit.js +22 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +71 -29
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +12 -10
- package/dist/plugins/builtin/ggml-classification/plugin.d.ts +8 -5
- package/dist/plugins/builtin/ggml-classification/plugin.js +89 -9
- package/dist/plugins/builtin/ggml-image-embedding/ops/embed.d.ts +24 -0
- package/dist/plugins/builtin/ggml-image-embedding/ops/embed.js +184 -0
- package/dist/plugins/builtin/ggml-image-embedding/plugin.d.ts +120 -0
- package/dist/plugins/builtin/ggml-image-embedding/plugin.js +39 -0
- package/dist/plugins/builtin/ggml-ocr/fit.d.ts +6 -0
- package/dist/plugins/builtin/ggml-ocr/fit.js +19 -0
- package/dist/plugins/builtin/ggml-ocr/params.d.ts +1 -1
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +20 -13
- package/dist/plugins/builtin/ggml-ocr/plugin.js +111 -25
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.d.ts +2 -2
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.js +18 -2
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +15 -15
- package/dist/plugins/builtin/ggml-vla/plugin.js +3 -3
- package/dist/plugins/builtin/index.d.ts +16 -14
- package/dist/plugins/builtin/index.js +16 -14
- package/dist/plugins/builtin/llamacpp-completion/fit.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-completion/fit.js +28 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +6 -8
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +15 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.d.ts +2 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.js +32 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +3 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +6 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +18 -3
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +94 -21
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.d.ts +5 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.js +54 -19
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +5 -3
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +9 -7
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +315 -95
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +51 -25
- package/dist/plugins/builtin/llamacpp-completion/transform.js +4 -0
- package/dist/plugins/builtin/llamacpp-decision/helpers.d.ts +0 -1
- package/dist/plugins/builtin/llamacpp-decision/helpers.js +7 -15
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.d.ts +3 -1
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.js +6 -3
- package/dist/plugins/builtin/llamacpp-decision/plugin.d.ts +43 -35
- package/dist/plugins/builtin/llamacpp-decision/plugin.js +4 -3
- package/dist/plugins/builtin/llamacpp-embedding/fit.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-embedding/fit.js +23 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +201 -10
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +52 -16
- package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +2 -2
- package/dist/plugins/builtin/llamacpp-embedding/transform.js +29 -2
- package/dist/plugins/builtin/llamacpp-rerank/helpers.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-rerank/helpers.js +16 -0
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.d.ts +7 -0
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.js +120 -0
- package/dist/plugins/builtin/llamacpp-rerank/plugin.d.ts +92 -0
- package/dist/plugins/builtin/llamacpp-rerank/plugin.js +44 -0
- package/dist/plugins/builtin/moss-transcribe/plugin.d.ts +46 -19
- package/dist/plugins/builtin/moss-transcribe/plugin.js +9 -7
- package/dist/plugins/builtin/nmtcpp-translation/fit.d.ts +3 -0
- package/dist/plugins/builtin/nmtcpp-translation/fit.js +11 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +32 -18
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +8 -6
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +234 -38
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +42 -16
- package/dist/plugins/builtin/sdcpp-generation/fit.d.ts +3 -0
- package/dist/plugins/builtin/sdcpp-generation/fit.js +68 -0
- package/dist/plugins/builtin/sdcpp-generation/load-files.d.ts +30 -0
- package/dist/plugins/builtin/sdcpp-generation/load-files.js +121 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +8 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +174 -53
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.d.ts +4 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.js +107 -22
- package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +7 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +23 -9
- package/dist/plugins/builtin/sdcpp-generation/ops/world.d.ts +6 -4
- package/dist/plugins/builtin/sdcpp-generation/ops/world.js +8 -3
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +166 -47
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +67 -155
- package/dist/plugins/builtin/sdcpp-generation/upscaler-config.d.ts +15 -0
- package/dist/plugins/builtin/sdcpp-generation/upscaler-config.js +19 -0
- package/dist/plugins/builtin/sdcpp-generation/world-config.d.ts +8 -0
- package/dist/plugins/builtin/sdcpp-generation/world-config.js +9 -0
- package/dist/plugins/builtin/tts-ggml/fit.d.ts +3 -0
- package/dist/plugins/builtin/tts-ggml/fit.js +162 -0
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +4 -1
- package/dist/plugins/builtin/tts-ggml/ops/moss-options.d.ts +1 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +3 -2
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +24 -11
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +53 -18
- package/dist/plugins/builtin/tts-ggml/ops/voice-options.d.ts +50 -0
- package/dist/plugins/builtin/tts-ggml/ops/voice-options.js +74 -0
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +415 -57
- package/dist/plugins/builtin/tts-ggml/plugin.js +183 -34
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +104 -40
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +12 -10
- package/dist/plugins/index.d.ts +1 -1
- package/dist/plugins/index.js +2 -2
- package/dist/plugins/ops/bci-transcribe.d.ts +5 -4
- package/dist/plugins/ops/bci-transcribe.js +18 -4
- package/dist/plugins/ops/cancel.d.ts +1 -0
- package/dist/plugins/ops/cancel.js +11 -2
- package/dist/plugins/ops/embed.d.ts +5 -3
- package/dist/plugins/ops/embed.js +89 -34
- package/dist/plugins/ops/kv-cache-utils.d.ts +5 -3
- package/dist/plugins/ops/kv-cache-utils.js +21 -10
- package/dist/plugins/ops/load-model.d.ts +10 -2
- package/dist/plugins/ops/load-model.js +45 -19
- package/dist/plugins/ops/transcribe.d.ts +5 -4
- package/dist/plugins/ops/transcribe.js +63 -28
- package/dist/plugins/ops/translate.d.ts +2 -1
- package/dist/plugins/ops/translate.js +7 -4
- package/dist/plugins/ops/unload-model.d.ts +17 -1
- package/dist/plugins/ops/unload-model.js +32 -3
- package/dist/plugins/turbovec-backend.d.ts +1 -1
- package/dist/plugins/turbovec-backend.js +21 -6
- package/dist/profiling/aggregator.d.ts +1 -1
- package/dist/profiling/aggregator.js +1 -1
- package/dist/profiling/controller.d.ts +1 -1
- package/dist/profiling/controller.js +2 -2
- package/dist/profiling/events.d.ts +1 -1
- package/dist/profiling/events.js +2 -2
- package/dist/profiling/exporters.d.ts +1 -1
- package/dist/profiling/exporters.js +3 -3
- package/dist/profiling/index.d.ts +13 -13
- package/dist/profiling/index.js +12 -12
- package/dist/profiling/operation-metrics.d.ts +1 -1
- package/dist/profiling/operation-metrics.js +24 -21
- package/dist/profiling/operation-wrappers.js +3 -3
- package/dist/profiling/profiler.js +1 -1
- package/dist/rag/embedding-scope.d.ts +16 -0
- package/dist/rag/embedding-scope.js +81 -0
- package/dist/rag/handler.d.ts +25 -0
- package/dist/rag/handler.js +51 -17
- package/dist/rag/handlers/close-workspace.js +1 -1
- package/dist/rag/handlers/delete-embeddings.js +1 -1
- package/dist/rag/handlers/delete-workspace.js +1 -1
- package/dist/rag/handlers/ingest.js +3 -7
- package/dist/rag/handlers/list-workspaces.d.ts +1 -1
- package/dist/rag/handlers/list-workspaces.js +1 -1
- package/dist/rag/handlers/reindex.js +1 -1
- package/dist/rag/handlers/save-embeddings.js +1 -1
- package/dist/rag/handlers/search.d.ts +10 -1
- package/dist/rag/handlers/search.js +17 -8
- package/dist/rag/index.d.ts +11 -11
- package/dist/rag/index.js +11 -11
- package/dist/rag/rag-operation-manager.d.ts +27 -3
- package/dist/rag/rag-operation-manager.js +10 -27
- package/dist/rag/rag-workspace-manager.js +1 -1
- package/dist/registry.js +7 -1
- package/dist/resources/collector.d.ts +71 -37
- package/dist/resources/collector.js +9 -2
- package/dist/resources/instance.d.ts +1 -1
- package/dist/resources/instance.js +1 -1
- package/dist/resources/native.d.ts +1 -1
- package/dist/resources/normalize.d.ts +1 -1
- package/dist/resources/types.d.ts +2 -0
- package/dist/rpc/inventory.js +1 -12
- package/dist/runtime/env.d.ts +5 -1
- package/dist/runtime/env.js +6 -21
- package/dist/runtime/index.d.ts +8 -5
- package/dist/runtime/index.js +4 -2
- package/dist/runtime/lifecycle.d.ts +6 -4
- package/dist/runtime/lifecycle.js +16 -12
- package/dist/runtime/model-config-utils.js +8 -1
- package/dist/runtime/model-registry.d.ts +30 -4
- package/dist/runtime/model-registry.js +48 -2
- package/dist/runtime/model-work-lock.d.ts +13 -0
- package/dist/runtime/model-work-lock.js +48 -0
- package/dist/runtime/registry-client.d.ts +47 -1
- package/dist/runtime/registry-client.js +446 -88
- package/dist/runtime/request-context.d.ts +11 -3
- package/dist/runtime/request-context.js +39 -2
- package/dist/runtime/request-registry.d.ts +56 -8
- package/dist/runtime/request-registry.js +190 -72
- package/dist/runtime/runtime-lifecycle.js +10 -1
- package/dist/runtime/session-id.d.ts +18 -0
- package/dist/runtime/session-id.js +20 -0
- package/dist/runtime/session-registry.d.ts +39 -0
- package/dist/runtime/session-registry.js +114 -0
- package/dist/runtime/state.d.ts +1 -1
- package/dist/runtime/state.js +2 -2
- package/dist/runtime/vector-index-backend.d.ts +6 -2
- package/dist/runtime/vector-index-registry.d.ts +26 -5
- package/dist/runtime/vector-index-registry.js +88 -27
- package/dist/schemas/abort-signal.d.ts +8 -0
- package/dist/schemas/assess-model-fit.d.ts +138 -282
- package/dist/schemas/assess-model-fit.js +85 -219
- package/dist/schemas/audio-gen.d.ts +291 -25
- package/dist/schemas/audio-gen.js +130 -26
- package/dist/schemas/batch-completion-stream.d.ts +113 -15
- package/dist/schemas/batch-completion-stream.js +10 -6
- package/dist/schemas/bci-config.d.ts +3 -1
- package/dist/schemas/bci-config.js +2 -2
- package/dist/schemas/bci.d.ts +55 -4
- package/dist/schemas/bci.js +13 -18
- package/dist/schemas/calc-tokens.d.ts +141 -0
- package/dist/schemas/calc-tokens.js +53 -0
- package/dist/schemas/cancel.d.ts +31 -0
- package/dist/schemas/cancel.js +11 -2
- package/dist/schemas/classification.d.ts +3 -0
- package/dist/schemas/classification.js +6 -2
- package/dist/schemas/common.d.ts +2209 -348
- package/dist/schemas/common.js +52 -33
- package/dist/schemas/completion-event.d.ts +11 -2
- package/dist/schemas/completion-event.js +5 -2
- package/dist/schemas/completion-stream.d.ts +173 -19
- package/dist/schemas/completion-stream.js +117 -20
- package/dist/schemas/config.d.ts +2064 -178
- package/dist/schemas/config.js +21 -23
- package/dist/schemas/decision.d.ts +12 -2
- package/dist/schemas/decision.js +16 -1
- package/dist/schemas/download-asset.d.ts +13 -6
- package/dist/schemas/download-asset.js +4 -7
- package/dist/schemas/embed.d.ts +286 -3
- package/dist/schemas/embed.js +162 -16
- package/dist/schemas/engine-addon-map.d.ts +3 -1
- package/dist/schemas/engine-addon-map.js +5 -3
- package/dist/schemas/errors.d.ts +13 -0
- package/dist/schemas/errors.js +71 -1
- package/dist/schemas/finetune.d.ts +14 -4
- package/dist/schemas/finetune.js +5 -6
- package/dist/schemas/heartbeat.d.ts +1 -0
- package/dist/schemas/heartbeat.js +5 -2
- package/dist/schemas/image-embedding.d.ts +27 -0
- package/dist/schemas/image-embedding.js +107 -0
- package/dist/schemas/index.d.ts +55 -50
- package/dist/schemas/index.js +54 -49
- package/dist/schemas/lifecycle.d.ts +3 -0
- package/dist/schemas/lifecycle.js +13 -6
- package/dist/schemas/llamacpp-config.d.ts +443 -20
- package/dist/schemas/llamacpp-config.js +188 -14
- package/dist/schemas/load-model.d.ts +8940 -1952
- package/dist/schemas/load-model.js +73 -9
- package/dist/schemas/logging-stream.d.ts +1 -0
- package/dist/schemas/logging-stream.js +5 -2
- package/dist/schemas/mcp-adapter.d.ts +1 -1
- package/dist/schemas/model-config-schemas.d.ts +731 -111
- package/dist/schemas/model-config-schemas.js +16 -12
- package/dist/schemas/model-fit-policy.d.ts +2 -2
- package/dist/schemas/model-info.d.ts +54 -56
- package/dist/schemas/model-info.js +18 -7
- package/dist/schemas/model-ops.d.ts +12 -0
- package/dist/schemas/model-ops.js +33 -5
- package/dist/schemas/model-src-utils.d.ts +14 -6
- package/dist/schemas/model-src-utils.js +2 -2
- package/dist/schemas/model-types.d.ts +12 -4
- package/dist/schemas/model-types.js +2 -0
- package/dist/schemas/ocr.d.ts +11 -4
- package/dist/schemas/ocr.js +7 -3
- package/dist/schemas/plugin.d.ts +70 -8
- package/dist/schemas/plugin.js +32 -4
- package/dist/schemas/profiling.d.ts +32 -32
- package/dist/schemas/profiling.js +1 -1
- package/dist/schemas/rag.d.ts +9 -0
- package/dist/schemas/rag.js +11 -17
- package/dist/schemas/registry.d.ts +38 -10
- package/dist/schemas/registry.js +16 -7
- package/dist/schemas/reload-config.d.ts +45 -6
- package/dist/schemas/reload-config.js +13 -6
- package/dist/schemas/request-base.d.ts +39 -0
- package/dist/schemas/request-base.js +64 -0
- package/dist/schemas/rerank.d.ts +132 -0
- package/dist/schemas/rerank.js +104 -0
- package/dist/schemas/rpc-server.d.ts +3 -0
- package/dist/schemas/rpc-server.js +15 -15
- package/dist/schemas/sdcpp-config.d.ts +144 -18
- package/dist/schemas/sdcpp-config.js +123 -28
- package/dist/schemas/session.d.ts +37 -0
- package/dist/schemas/session.js +53 -0
- package/dist/schemas/system-resources.d.ts +364 -179
- package/dist/schemas/system-resources.js +8 -3
- package/dist/schemas/text-to-speech.d.ts +1408 -275
- package/dist/schemas/text-to-speech.js +547 -33
- package/dist/schemas/tools.d.ts +7 -0
- package/dist/schemas/tools.js +29 -0
- package/dist/schemas/transcription-config.d.ts +267 -2
- package/dist/schemas/transcription-config.js +138 -7
- package/dist/schemas/transcription.d.ts +94 -7
- package/dist/schemas/transcription.js +99 -22
- package/dist/schemas/translate.d.ts +4 -2
- package/dist/schemas/translate.js +6 -10
- package/dist/schemas/translation-config.d.ts +40 -20
- package/dist/schemas/translation-config.js +1 -1
- package/dist/schemas/tts-pcm.d.ts +35 -0
- package/dist/schemas/tts-pcm.js +69 -0
- package/dist/schemas/vector-index.d.ts +47 -1
- package/dist/schemas/vector-index.js +37 -10
- package/dist/schemas/vla.d.ts +3 -3
- package/dist/surface.d.ts +4 -3
- package/dist/surface.js +4 -3
- package/dist/utils/addon-responses.d.ts +71 -2
- package/dist/utils/aggregate-events.d.ts +1 -1
- package/dist/utils/aggregate-events.js +2 -2
- package/dist/utils/archive.d.ts +7 -0
- package/dist/utils/archive.js +89 -30
- package/dist/utils/asr-events.d.ts +5 -0
- package/dist/utils/asr-events.js +10 -1
- package/dist/utils/audio/decoder.js +3 -1
- package/dist/utils/audio-input.js +1 -1
- package/dist/utils/cache/auto-history.d.ts +2 -1
- package/dist/utils/cache/auto-history.js +5 -3
- package/dist/utils/cache/layout.d.ts +48 -0
- package/dist/utils/cache/layout.js +190 -0
- package/dist/utils/cache/paths.js +8 -10
- package/dist/utils/cache/types.d.ts +2 -1
- package/dist/utils/checksum.d.ts +7 -0
- package/dist/utils/checksum.js +59 -2
- package/dist/utils/completion-normalizer.js +287 -88
- package/dist/utils/gguf-architecture.d.ts +7 -0
- package/dist/utils/gguf-architecture.js +142 -0
- package/dist/utils/gguf-tensor-extractor.d.ts +0 -6
- package/dist/utils/gguf-tensor-extractor.js +1 -50
- package/dist/utils/index.d.ts +13 -13
- package/dist/utils/index.js +13 -13
- package/dist/utils/mcp-adapter.d.ts +1 -1
- package/dist/utils/mcp-adapter.js +1 -1
- package/dist/utils/path-security.js +1 -1
- package/dist/utils/shard-utils.d.ts +0 -16
- package/dist/utils/shard-utils.js +14 -48
- package/dist/utils/tool-integration.d.ts +3 -2
- package/dist/utils/tool-integration.js +3 -6
- package/dist/utils/tools/defer.d.ts +50 -31
- package/dist/utils/tools/defer.js +115 -51
- package/dist/utils/tools/dialect.d.ts +15 -4
- package/dist/utils/tools/dialect.js +34 -5
- package/dist/utils/tools/index.d.ts +3 -3
- package/dist/utils/tools/index.js +3 -3
- package/dist/utils/tools/parser.js +8 -8
- package/dist/utils/tools/parsers/dsml.d.ts +1 -1
- package/dist/utils/tools/parsers/dsml.js +1 -1
- package/dist/utils/tools/parsers/gemma4native.d.ts +1 -1
- package/dist/utils/tools/parsers/gemma4native.js +1 -1
- package/dist/utils/tools/parsers/harmony.d.ts +1 -1
- package/dist/utils/tools/parsers/harmony.js +1 -1
- package/dist/utils/tools/parsers/hermes.d.ts +1 -1
- package/dist/utils/tools/parsers/hermes.js +1 -1
- package/dist/utils/tools/parsers/json.d.ts +1 -1
- package/dist/utils/tools/parsers/json.js +1 -1
- package/dist/utils/tools/parsers/pythonic.d.ts +1 -1
- package/dist/utils/tools/parsers/pythonic.js +1 -1
- package/dist/utils/tools/parsers/qwen35.d.ts +1 -1
- package/dist/utils/tools/parsers/qwen35.js +1 -1
- package/dist/utils/transcribe-metadata.d.ts +28 -2
- package/dist/utils/transcribe-metadata.js +16 -0
- package/dist/utils/tts-stats.d.ts +43 -1
- package/dist/utils/tts-stats.js +120 -22
- package/dist/utils/until-aborted.d.ts +7 -0
- package/dist/utils/until-aborted.js +34 -0
- package/package.json +67 -55
- package/dist/models/registry/resource-profiles.d.ts +0 -17
- package/dist/models/registry/resource-profiles.js +0 -5657
- package/dist/resources/model-fit/assess.d.ts +0 -72
- package/dist/resources/model-fit/assess.js +0 -679
- package/dist/resources/model-fit/fit-stub/project-fit-from-load.d.ts +0 -27
- package/dist/resources/model-fit/fit-stub/project-fit-from-load.js +0 -33
- package/dist/resources/model-fit/floor.d.ts +0 -31
- package/dist/resources/model-fit/floor.js +0 -65
- package/dist/resources/model-fit/kv-cache.d.ts +0 -23
- package/dist/resources/model-fit/kv-cache.js +0 -85
- package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +0 -54
- package/dist/resources/model-fit/native-probe/advisory-fit.js +0 -248
- package/dist/resources/model-fit/native-probe/child-engine-fit.d.ts +0 -2
- package/dist/resources/model-fit/native-probe/child-engine-fit.js +0 -49
- package/dist/resources/model-fit/native-probe/classify-fit.d.ts +0 -6
- package/dist/resources/model-fit/native-probe/classify-fit.js +0 -166
- package/dist/resources/model-fit/native-probe/create-fit-request.d.ts +0 -31
- package/dist/resources/model-fit/native-probe/create-fit-request.js +0 -61
- package/dist/resources/model-fit/native-probe/engine-fit.d.ts +0 -73
- package/dist/resources/model-fit/native-probe/engine-fit.js +0 -13
- package/dist/resources/model-fit/native-probe/engines/asr.d.ts +0 -14
- package/dist/resources/model-fit/native-probe/engines/asr.js +0 -126
- package/dist/resources/model-fit/native-probe/engines/audiogen.d.ts +0 -8
- package/dist/resources/model-fit/native-probe/engines/diffusion.d.ts +0 -7
- package/dist/resources/model-fit/native-probe/engines/diffusion.js +0 -65
- package/dist/resources/model-fit/native-probe/engines/llama.d.ts +0 -12
- package/dist/resources/model-fit/native-probe/engines/llama.js +0 -41
- package/dist/resources/model-fit/native-probe/engines/nmt.d.ts +0 -8
- package/dist/resources/model-fit/native-probe/engines/nmt.js +0 -11
- package/dist/resources/model-fit/native-probe/engines/ocr.d.ts +0 -9
- package/dist/resources/model-fit/native-probe/engines/ocr.js +0 -13
- package/dist/resources/model-fit/native-probe/engines/tts.d.ts +0 -10
- package/dist/resources/model-fit/native-probe/engines/tts.js +0 -152
- package/dist/resources/model-fit/native-probe/fit-outcome.d.ts +0 -16
- package/dist/resources/model-fit/native-probe/fit-process.d.ts +0 -27
- package/dist/resources/model-fit/native-probe/fit-process.js +0 -55
- package/dist/resources/model-fit/native-probe/fit-runner.d.ts +0 -1
- package/dist/resources/model-fit/native-probe/fit-runner.js +0 -58
- package/dist/resources/model-fit/native-probe/run-fit.d.ts +0 -24
- package/dist/resources/model-fit/native-probe/run-fit.js +0 -20
- package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +0 -24
- package/dist/resources/model-fit/native-probe/run-in-process-fit.js +0 -123
- package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +0 -63
- package/dist/resources/model-fit/native-probe/run-isolated-fit.js +0 -377
- package/dist/resources/model-fit/platform.d.ts +0 -9
- package/dist/resources/model-fit/platform.js +0 -26
- package/dist/resources/model-fit/types.d.ts +0 -6
- package/dist/resources/model-fit/types.js +0 -1
- package/dist/schemas/model-resource-profile.d.ts +0 -108
- package/dist/schemas/model-resource-profile.js +0 -80
- /package/dist/{resources/model-fit/native-probe/engines → model-fit}/gpu.d.ts +0 -0
- /package/dist/{resources/model-fit/native-probe/engines → model-fit}/gpu.js +0 -0
- /package/dist/{resources/model-fit/native-probe/fit-outcome.js → schemas/abort-signal.js} +0 -0
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
|
-
import { whisperConfigSchema } from '
|
|
3
|
-
import { whisperModelTypeSchema } from '
|
|
2
|
+
import { whisperConfigSchema } from '../schemas/transcription-config.js';
|
|
3
|
+
import { whisperModelTypeSchema } from '../schemas/model-types.js';
|
|
4
|
+
import { trackedRequestSchema } from '../schemas/request-base.js';
|
|
4
5
|
export const modelIdSchema = z.string().regex(/^[0-9a-f]{16}$/, 'Invalid modelId format');
|
|
5
6
|
export const reloadConfigOptionsSchema = z.union([
|
|
6
7
|
z.object({
|
|
@@ -14,23 +15,29 @@ export const reloadConfigOptionsToRequestSchema = z.union([
|
|
|
14
15
|
.object({
|
|
15
16
|
modelId: modelIdSchema,
|
|
16
17
|
modelType: whisperModelTypeSchema,
|
|
17
|
-
modelConfig: whisperConfigSchema.partial().strict()
|
|
18
|
+
modelConfig: whisperConfigSchema.partial().strict(),
|
|
19
|
+
requestId: z.string().optional()
|
|
18
20
|
})
|
|
19
21
|
.transform((data) => ({
|
|
20
22
|
type: 'loadModel',
|
|
21
23
|
modelId: data.modelId,
|
|
22
24
|
modelType: data.modelType,
|
|
23
|
-
modelConfig: data.modelConfig
|
|
25
|
+
modelConfig: data.modelConfig,
|
|
26
|
+
// A reload is registered under this id, so `cancel({ requestId })`
|
|
27
|
+
// reaches it while it waits for another load of the same model.
|
|
28
|
+
...(data.requestId !== undefined && { requestId: data.requestId })
|
|
24
29
|
}))
|
|
25
30
|
]);
|
|
26
|
-
const reloadConfigRequestBaseSchema = z
|
|
31
|
+
const reloadConfigRequestBaseSchema = z
|
|
32
|
+
.object({
|
|
27
33
|
type: z.literal('loadModel'),
|
|
28
34
|
modelId: modelIdSchema,
|
|
29
35
|
// Explicitly exclude load-specific fields for type narrowing
|
|
30
36
|
modelSrc: z.never().optional(),
|
|
31
37
|
withProgress: z.never().optional(),
|
|
32
38
|
seed: z.never().optional()
|
|
33
|
-
})
|
|
39
|
+
})
|
|
40
|
+
.extend(trackedRequestSchema.shape);
|
|
34
41
|
export const reloadConfigWhisperRequestSchema = reloadConfigRequestBaseSchema.extend({
|
|
35
42
|
modelType: whisperModelTypeSchema,
|
|
36
43
|
modelConfig: whisperConfigSchema.partial()
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
/**
|
|
3
|
+
* Caller session identity. The SDK client stamps every request sent through a
|
|
4
|
+
* session handle with the session's id; a request without one runs as the
|
|
5
|
+
* owner session (the process that started the runtime).
|
|
6
|
+
*/
|
|
7
|
+
export declare const sessionIdSchema: z.ZodString;
|
|
8
|
+
/**
|
|
9
|
+
* Caller-minted request identity. Surfaces on the handle the SDK returns so the
|
|
10
|
+
* same caller can target the request with `cancel({ requestId })`. The engine
|
|
11
|
+
* keys its registry by `(sessionId, requestId)`, so ids only have to be unique
|
|
12
|
+
* within one session.
|
|
13
|
+
*/
|
|
14
|
+
export declare const requestIdSchema: z.ZodString;
|
|
15
|
+
/**
|
|
16
|
+
* Base of every request schema. A request schema is built as
|
|
17
|
+
* `<own fields>.extend(sessionScopedRequestSchema.shape)`, which appends the
|
|
18
|
+
* base fields after the method's own so the wire property order, and with it
|
|
19
|
+
* the generated Kotlin constructor order, stays stable.
|
|
20
|
+
* `test/request-schemas-base-fields.test.ts` fails for a member of
|
|
21
|
+
* `requestSchema` that lacks them.
|
|
22
|
+
*/
|
|
23
|
+
export declare const sessionScopedRequestSchema: z.ZodObject<{
|
|
24
|
+
sessionId: z.ZodOptional<z.ZodString>;
|
|
25
|
+
}, z.core.$strip>;
|
|
26
|
+
/** Base of every request schema that registers work the caller can cancel. */
|
|
27
|
+
export declare const trackedRequestSchema: z.ZodObject<{
|
|
28
|
+
sessionId: z.ZodOptional<z.ZodString>;
|
|
29
|
+
requestId: z.ZodOptional<z.ZodString>;
|
|
30
|
+
}, z.core.$strip>;
|
|
31
|
+
/**
|
|
32
|
+
* The request types that register no work of their own: lifecycle, lookups
|
|
33
|
+
* and the session methods. Their schemas extend `sessionScopedRequestSchema`;
|
|
34
|
+
* every other request type extends `trackedRequestSchema`.
|
|
35
|
+
* `test/request-schemas-base-fields.test.ts` holds the two lists to that.
|
|
36
|
+
*/
|
|
37
|
+
export declare const UNTRACKED_REQUEST_TYPES: ReadonlySet<string>;
|
|
38
|
+
export type SessionScopedRequest = z.infer<typeof sessionScopedRequestSchema>;
|
|
39
|
+
export type TrackedRequest = z.infer<typeof trackedRequestSchema>;
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
/**
|
|
3
|
+
* Caller session identity. The SDK client stamps every request sent through a
|
|
4
|
+
* session handle with the session's id; a request without one runs as the
|
|
5
|
+
* owner session (the process that started the runtime).
|
|
6
|
+
*/
|
|
7
|
+
export const sessionIdSchema = z
|
|
8
|
+
.string()
|
|
9
|
+
.min(1)
|
|
10
|
+
.describe('Identifier of the caller session this request belongs to. Omitted for ' +
|
|
11
|
+
'requests made by the runtime owner.');
|
|
12
|
+
/**
|
|
13
|
+
* Caller-minted request identity. Surfaces on the handle the SDK returns so the
|
|
14
|
+
* same caller can target the request with `cancel({ requestId })`. The engine
|
|
15
|
+
* keys its registry by `(sessionId, requestId)`, so ids only have to be unique
|
|
16
|
+
* within one session.
|
|
17
|
+
*/
|
|
18
|
+
export const requestIdSchema = z
|
|
19
|
+
.string()
|
|
20
|
+
.min(1)
|
|
21
|
+
.describe('Stable identifier for this in-flight request, generated by the caller at ' +
|
|
22
|
+
'call time. Surfaces on the returned handle so the caller can target it ' +
|
|
23
|
+
'with `cancel({ requestId })`. Optional: the engine generates one when the ' +
|
|
24
|
+
'field is missing. Scoped to the caller session; another session cannot ' +
|
|
25
|
+
'address it.');
|
|
26
|
+
/**
|
|
27
|
+
* Base of every request schema. A request schema is built as
|
|
28
|
+
* `<own fields>.extend(sessionScopedRequestSchema.shape)`, which appends the
|
|
29
|
+
* base fields after the method's own so the wire property order, and with it
|
|
30
|
+
* the generated Kotlin constructor order, stays stable.
|
|
31
|
+
* `test/request-schemas-base-fields.test.ts` fails for a member of
|
|
32
|
+
* `requestSchema` that lacks them.
|
|
33
|
+
*/
|
|
34
|
+
export const sessionScopedRequestSchema = z.object({
|
|
35
|
+
sessionId: sessionIdSchema.optional()
|
|
36
|
+
});
|
|
37
|
+
/** Base of every request schema that registers work the caller can cancel. */
|
|
38
|
+
export const trackedRequestSchema = sessionScopedRequestSchema.extend({
|
|
39
|
+
requestId: requestIdSchema.optional()
|
|
40
|
+
});
|
|
41
|
+
/**
|
|
42
|
+
* The request types that register no work of their own: lifecycle, lookups
|
|
43
|
+
* and the session methods. Their schemas extend `sessionScopedRequestSchema`;
|
|
44
|
+
* every other request type extends `trackedRequestSchema`.
|
|
45
|
+
* `test/request-schemas-base-fields.test.ts` holds the two lists to that.
|
|
46
|
+
*/
|
|
47
|
+
export const UNTRACKED_REQUEST_TYPES = new Set([
|
|
48
|
+
'assessModelFit',
|
|
49
|
+
'calcTokens',
|
|
50
|
+
'cancel',
|
|
51
|
+
'closeSession',
|
|
52
|
+
'deleteCache',
|
|
53
|
+
'getLoadedModelInfo',
|
|
54
|
+
'getModelInfo',
|
|
55
|
+
'getSystemResources',
|
|
56
|
+
'heartbeat',
|
|
57
|
+
'loggingStream',
|
|
58
|
+
'openSession',
|
|
59
|
+
'resume',
|
|
60
|
+
'state',
|
|
61
|
+
'stopRpcServer',
|
|
62
|
+
'suspend',
|
|
63
|
+
'unloadModel'
|
|
64
|
+
]);
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
export declare const rerankRequestBodySchema: z.ZodObject<{
|
|
3
|
+
query: z.ZodString;
|
|
4
|
+
documents: z.ZodArray<z.ZodString>;
|
|
5
|
+
}, z.core.$strict>;
|
|
6
|
+
export declare const rerankParamsSchema: z.ZodObject<{
|
|
7
|
+
query: z.ZodString;
|
|
8
|
+
documents: z.ZodArray<z.ZodString>;
|
|
9
|
+
modelId: z.ZodString;
|
|
10
|
+
}, z.core.$strict>;
|
|
11
|
+
export declare const rerankRequestSchema: z.ZodObject<{
|
|
12
|
+
query: z.ZodString;
|
|
13
|
+
documents: z.ZodArray<z.ZodString>;
|
|
14
|
+
type: z.ZodLiteral<"rerank">;
|
|
15
|
+
modelId: z.ZodString;
|
|
16
|
+
sessionId: z.ZodOptional<z.ZodString>;
|
|
17
|
+
requestId: z.ZodOptional<z.ZodString>;
|
|
18
|
+
}, z.core.$strict>;
|
|
19
|
+
export declare const rerankScoresSchema: z.ZodArray<z.ZodNumber>;
|
|
20
|
+
export declare const rerankResponseSchema: z.ZodObject<{
|
|
21
|
+
type: z.ZodLiteral<"rerank">;
|
|
22
|
+
scores: z.ZodArray<z.ZodNumber>;
|
|
23
|
+
}, z.core.$strip>;
|
|
24
|
+
export declare const rerankConfigBaseSchema: z.ZodObject<{
|
|
25
|
+
device: z.ZodOptional<z.ZodEnum<{
|
|
26
|
+
cpu: "cpu";
|
|
27
|
+
gpu: "gpu";
|
|
28
|
+
}>>;
|
|
29
|
+
gpu_layers: z.ZodOptional<z.ZodNumber>;
|
|
30
|
+
batch_size: z.ZodOptional<z.ZodNumber>;
|
|
31
|
+
backend: z.ZodOptional<z.ZodArray<z.ZodEnum<{
|
|
32
|
+
auto: "auto";
|
|
33
|
+
vulkan: "vulkan";
|
|
34
|
+
opencl: "opencl";
|
|
35
|
+
metal: "metal";
|
|
36
|
+
cuda: "cuda";
|
|
37
|
+
}>>>;
|
|
38
|
+
poll: z.ZodOptional<z.ZodNumber>;
|
|
39
|
+
pollBatch: z.ZodOptional<z.ZodNumber>;
|
|
40
|
+
threads: z.ZodOptional<z.ZodNumber>;
|
|
41
|
+
'threads-batch': z.ZodOptional<z.ZodNumber>;
|
|
42
|
+
flash_attn: z.ZodOptional<z.ZodEnum<{
|
|
43
|
+
auto: "auto";
|
|
44
|
+
on: "on";
|
|
45
|
+
off: "off";
|
|
46
|
+
}>>;
|
|
47
|
+
verbosity: z.ZodOptional<z.ZodNumber>;
|
|
48
|
+
'main-gpu': z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodEnum<{
|
|
49
|
+
integrated: "integrated";
|
|
50
|
+
dedicated: "dedicated";
|
|
51
|
+
}>]>>;
|
|
52
|
+
'split-mode': z.ZodOptional<z.ZodEnum<{
|
|
53
|
+
none: "none";
|
|
54
|
+
layer: "layer";
|
|
55
|
+
}>>;
|
|
56
|
+
'tensor-split': z.ZodOptional<z.ZodString>;
|
|
57
|
+
openclCacheDir: z.ZodOptional<z.ZodString>;
|
|
58
|
+
}, z.core.$strict>;
|
|
59
|
+
export type RerankConfigInput = z.infer<typeof rerankConfigBaseSchema>;
|
|
60
|
+
export declare const RERANK_CONFIG_DEFAULTS: {
|
|
61
|
+
readonly device: "gpu";
|
|
62
|
+
};
|
|
63
|
+
export declare const rerankConfigSchema: z.ZodPipe<z.ZodObject<{
|
|
64
|
+
device: z.ZodOptional<z.ZodEnum<{
|
|
65
|
+
cpu: "cpu";
|
|
66
|
+
gpu: "gpu";
|
|
67
|
+
}>>;
|
|
68
|
+
gpu_layers: z.ZodOptional<z.ZodNumber>;
|
|
69
|
+
batch_size: z.ZodOptional<z.ZodNumber>;
|
|
70
|
+
backend: z.ZodOptional<z.ZodArray<z.ZodEnum<{
|
|
71
|
+
auto: "auto";
|
|
72
|
+
vulkan: "vulkan";
|
|
73
|
+
opencl: "opencl";
|
|
74
|
+
metal: "metal";
|
|
75
|
+
cuda: "cuda";
|
|
76
|
+
}>>>;
|
|
77
|
+
poll: z.ZodOptional<z.ZodNumber>;
|
|
78
|
+
pollBatch: z.ZodOptional<z.ZodNumber>;
|
|
79
|
+
threads: z.ZodOptional<z.ZodNumber>;
|
|
80
|
+
'threads-batch': z.ZodOptional<z.ZodNumber>;
|
|
81
|
+
flash_attn: z.ZodOptional<z.ZodEnum<{
|
|
82
|
+
auto: "auto";
|
|
83
|
+
on: "on";
|
|
84
|
+
off: "off";
|
|
85
|
+
}>>;
|
|
86
|
+
verbosity: z.ZodOptional<z.ZodNumber>;
|
|
87
|
+
'main-gpu': z.ZodOptional<z.ZodUnion<readonly [z.ZodNumber, z.ZodEnum<{
|
|
88
|
+
integrated: "integrated";
|
|
89
|
+
dedicated: "dedicated";
|
|
90
|
+
}>]>>;
|
|
91
|
+
'split-mode': z.ZodOptional<z.ZodEnum<{
|
|
92
|
+
none: "none";
|
|
93
|
+
layer: "layer";
|
|
94
|
+
}>>;
|
|
95
|
+
'tensor-split': z.ZodOptional<z.ZodString>;
|
|
96
|
+
openclCacheDir: z.ZodOptional<z.ZodString>;
|
|
97
|
+
}, z.core.$strict>, z.ZodTransform<{
|
|
98
|
+
device: "cpu" | "gpu";
|
|
99
|
+
gpu_layers?: number | undefined;
|
|
100
|
+
batch_size?: number | undefined;
|
|
101
|
+
backend?: ("auto" | "vulkan" | "opencl" | "metal" | "cuda")[] | undefined;
|
|
102
|
+
poll?: number | undefined;
|
|
103
|
+
pollBatch?: number | undefined;
|
|
104
|
+
threads?: number | undefined;
|
|
105
|
+
'threads-batch'?: number | undefined;
|
|
106
|
+
flash_attn?: "auto" | "on" | "off" | undefined;
|
|
107
|
+
verbosity?: number | undefined;
|
|
108
|
+
'main-gpu'?: number | "integrated" | "dedicated" | undefined;
|
|
109
|
+
'split-mode'?: "none" | "layer" | undefined;
|
|
110
|
+
'tensor-split'?: string | undefined;
|
|
111
|
+
openclCacheDir?: string | undefined;
|
|
112
|
+
}, {
|
|
113
|
+
device?: "cpu" | "gpu" | undefined;
|
|
114
|
+
gpu_layers?: number | undefined;
|
|
115
|
+
batch_size?: number | undefined;
|
|
116
|
+
backend?: ("auto" | "vulkan" | "opencl" | "metal" | "cuda")[] | undefined;
|
|
117
|
+
poll?: number | undefined;
|
|
118
|
+
pollBatch?: number | undefined;
|
|
119
|
+
threads?: number | undefined;
|
|
120
|
+
'threads-batch'?: number | undefined;
|
|
121
|
+
flash_attn?: "auto" | "on" | "off" | undefined;
|
|
122
|
+
verbosity?: number | undefined;
|
|
123
|
+
'main-gpu'?: number | "integrated" | "dedicated" | undefined;
|
|
124
|
+
'split-mode'?: "none" | "layer" | undefined;
|
|
125
|
+
'tensor-split'?: string | undefined;
|
|
126
|
+
openclCacheDir?: string | undefined;
|
|
127
|
+
}>>;
|
|
128
|
+
export type RerankParams = z.infer<typeof rerankParamsSchema>;
|
|
129
|
+
export type RerankRequest = z.infer<typeof rerankRequestSchema>;
|
|
130
|
+
export type RerankScores = z.infer<typeof rerankScoresSchema>;
|
|
131
|
+
export type RerankResponse = z.infer<typeof rerankResponseSchema>;
|
|
132
|
+
export type RerankConfig = z.infer<typeof rerankConfigSchema>;
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
import { trackedRequestSchema } from '../schemas/request-base.js';
|
|
3
|
+
const rerankFields = {
|
|
4
|
+
query: z.string().min(1).describe('Query to score each document against.'),
|
|
5
|
+
documents: z
|
|
6
|
+
.array(z.string())
|
|
7
|
+
.min(1)
|
|
8
|
+
.describe('Documents to score. Each one, paired with the query, must fit in `batch_size`.')
|
|
9
|
+
};
|
|
10
|
+
export const rerankRequestBodySchema = z.object(rerankFields).strict();
|
|
11
|
+
const envelope = { modelId: z.string().min(1) };
|
|
12
|
+
export const rerankParamsSchema = rerankRequestBodySchema.extend(envelope);
|
|
13
|
+
export const rerankRequestSchema = rerankRequestBodySchema
|
|
14
|
+
.extend({
|
|
15
|
+
...envelope,
|
|
16
|
+
type: z.literal('rerank')
|
|
17
|
+
})
|
|
18
|
+
.extend(trackedRequestSchema.shape);
|
|
19
|
+
export const rerankScoresSchema = z
|
|
20
|
+
.array(z.number())
|
|
21
|
+
.describe("One relevance score per document, in input order. Higher is more relevant. Scores are the model's raw classifier output, so compare them within one model only.");
|
|
22
|
+
export const rerankResponseSchema = z.object({
|
|
23
|
+
type: z.literal('rerank'),
|
|
24
|
+
scores: rerankScoresSchema
|
|
25
|
+
});
|
|
26
|
+
export const rerankConfigBaseSchema = z
|
|
27
|
+
.object({
|
|
28
|
+
device: z.enum(['cpu', 'gpu']).optional().describe("Compute device. Default 'gpu'."),
|
|
29
|
+
gpu_layers: z
|
|
30
|
+
.number()
|
|
31
|
+
.int()
|
|
32
|
+
.optional()
|
|
33
|
+
.describe('Layers to offload. Unset lets the addon choose placement.'),
|
|
34
|
+
batch_size: z
|
|
35
|
+
.number()
|
|
36
|
+
.int()
|
|
37
|
+
.positive()
|
|
38
|
+
.optional()
|
|
39
|
+
.describe('Tokens per forward pass. Default 2048; each document paired with the query must fit.'),
|
|
40
|
+
backend: z
|
|
41
|
+
.array(z.enum(['cuda', 'vulkan', 'metal', 'opencl', 'auto']))
|
|
42
|
+
.min(1)
|
|
43
|
+
.optional()
|
|
44
|
+
.describe("Ordered GPU backend names, such as ['cuda', 'vulkan']. " +
|
|
45
|
+
'Accepted names: cuda, vulkan, metal, opencl, and auto for no preference. ' +
|
|
46
|
+
'If unset, uses the native default order. ' +
|
|
47
|
+
'If none of the specified names has a corresponding available device, selection falls back to the native default order. ' +
|
|
48
|
+
"Use device: 'cpu' for CPU. " +
|
|
49
|
+
"A GPU override cannot be combined with 'split-mode': 'layer'."),
|
|
50
|
+
poll: z
|
|
51
|
+
.number()
|
|
52
|
+
.int()
|
|
53
|
+
.min(0)
|
|
54
|
+
.max(100)
|
|
55
|
+
.optional()
|
|
56
|
+
.describe('CPU polling level, from 0 to 100. If omitted, uses the default native behavior. Higher values may reduce wake latency but consume more CPU during GPU work.'),
|
|
57
|
+
pollBatch: z
|
|
58
|
+
.number()
|
|
59
|
+
.int()
|
|
60
|
+
.min(0)
|
|
61
|
+
.max(100)
|
|
62
|
+
.optional()
|
|
63
|
+
.describe('CPU polling level for batch processing, from 0 to 100. If omitted, uses the default native behavior.'),
|
|
64
|
+
threads: z
|
|
65
|
+
.number()
|
|
66
|
+
.int()
|
|
67
|
+
.optional()
|
|
68
|
+
.describe('CPU threads, at most the CPU count. Zero or below uses all logical CPUs.'),
|
|
69
|
+
'threads-batch': z
|
|
70
|
+
.number()
|
|
71
|
+
.int()
|
|
72
|
+
.optional()
|
|
73
|
+
.describe('Batch CPU threads. Unset uses threads; zero or below uses all logical CPUs.'),
|
|
74
|
+
flash_attn: z
|
|
75
|
+
.enum(['on', 'off', 'auto'])
|
|
76
|
+
.optional()
|
|
77
|
+
.describe('Flash attention mode. Default auto.'),
|
|
78
|
+
verbosity: z
|
|
79
|
+
.number()
|
|
80
|
+
.int()
|
|
81
|
+
.min(0)
|
|
82
|
+
.max(3)
|
|
83
|
+
.optional()
|
|
84
|
+
.describe('Native log level: 0 error, 1 warn, 2 info, 3 debug.'),
|
|
85
|
+
'main-gpu': z
|
|
86
|
+
.union([z.number().int().nonnegative(), z.enum(['integrated', 'dedicated'])])
|
|
87
|
+
.optional()
|
|
88
|
+
.describe('GPU index or device class.'),
|
|
89
|
+
'split-mode': z.enum(['none', 'layer']).optional().describe('Multi-GPU split mode.'),
|
|
90
|
+
'tensor-split': z.string().optional().describe('Per-GPU tensor split proportions.'),
|
|
91
|
+
openclCacheDir: z
|
|
92
|
+
.string()
|
|
93
|
+
.optional()
|
|
94
|
+
.describe('Writable OpenCL kernel cache directory, required on Android.')
|
|
95
|
+
})
|
|
96
|
+
.strict();
|
|
97
|
+
export const RERANK_CONFIG_DEFAULTS = {
|
|
98
|
+
device: 'gpu'
|
|
99
|
+
};
|
|
100
|
+
export const rerankConfigSchema = rerankConfigBaseSchema.transform((data) => ({
|
|
101
|
+
...RERANK_CONFIG_DEFAULTS,
|
|
102
|
+
...data,
|
|
103
|
+
device: data.device ?? RERANK_CONFIG_DEFAULTS.device
|
|
104
|
+
}));
|
|
@@ -24,6 +24,7 @@ export declare const startRpcServerRequestSchema: z.ZodObject<{
|
|
|
24
24
|
allowNonLoopbackHost: z.ZodOptional<z.ZodBoolean>;
|
|
25
25
|
discoveryTopic: z.ZodOptional<z.ZodString>;
|
|
26
26
|
type: z.ZodLiteral<"startRpcServer">;
|
|
27
|
+
sessionId: z.ZodOptional<z.ZodString>;
|
|
27
28
|
requestId: z.ZodOptional<z.ZodString>;
|
|
28
29
|
}, z.core.$strip>;
|
|
29
30
|
export declare const startRpcServerResponseSchema: z.ZodObject<{
|
|
@@ -38,6 +39,7 @@ export declare const stopRpcServerOptionsSchema: z.ZodObject<{
|
|
|
38
39
|
export declare const stopRpcServerRequestSchema: z.ZodObject<{
|
|
39
40
|
serverId: z.ZodString;
|
|
40
41
|
type: z.ZodLiteral<"stopRpcServer">;
|
|
42
|
+
sessionId: z.ZodOptional<z.ZodString>;
|
|
41
43
|
}, z.core.$strip>;
|
|
42
44
|
export declare const stopRpcServerResponseSchema: z.ZodObject<{
|
|
43
45
|
type: z.ZodLiteral<"stopRpcServer">;
|
|
@@ -65,6 +67,7 @@ export declare const discoverRpcServersRequestSchema: z.ZodObject<{
|
|
|
65
67
|
topic: z.ZodString;
|
|
66
68
|
timeoutMs: z.ZodOptional<z.ZodNumber>;
|
|
67
69
|
type: z.ZodLiteral<"discoverRpcServers">;
|
|
70
|
+
sessionId: z.ZodOptional<z.ZodString>;
|
|
68
71
|
requestId: z.ZodOptional<z.ZodString>;
|
|
69
72
|
}, z.core.$strip>;
|
|
70
73
|
export declare const discoverRpcServersResponseSchema: z.ZodObject<{
|
|
@@ -1,10 +1,6 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
|
+
import { sessionScopedRequestSchema, trackedRequestSchema } from '../schemas/request-base.js';
|
|
2
3
|
const topicSchema = z.string().trim().min(1).max(256);
|
|
3
|
-
const requestIdSchema = z
|
|
4
|
-
.string()
|
|
5
|
-
.min(1)
|
|
6
|
-
.optional()
|
|
7
|
-
.describe('Caller-generated identifier for targeted cancellation with cancel({ requestId }). Generated by the engine when omitted.');
|
|
8
4
|
export const startRpcServerOptionsSchema = z.object({
|
|
9
5
|
host: z.string().min(1).optional().describe('IPv4 bind address. Defaults to 127.0.0.1.'),
|
|
10
6
|
port: z
|
|
@@ -39,17 +35,20 @@ export const rpcServerInfoSchema = z.object({
|
|
|
39
35
|
.boolean()
|
|
40
36
|
.describe('Whether the server will try RDMA. On Linux, this requires a capable native backend, libibverbs, an RDMA device, and GGML_RPC_NO_RDMA unset. Each connection can still fall back to TCP.')
|
|
41
37
|
});
|
|
42
|
-
export const startRpcServerRequestSchema = startRpcServerOptionsSchema
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
})
|
|
38
|
+
export const startRpcServerRequestSchema = startRpcServerOptionsSchema
|
|
39
|
+
.extend({
|
|
40
|
+
type: z.literal('startRpcServer')
|
|
41
|
+
})
|
|
42
|
+
.extend(trackedRequestSchema.shape);
|
|
46
43
|
export const startRpcServerResponseSchema = rpcServerInfoSchema.extend({
|
|
47
44
|
type: z.literal('startRpcServer')
|
|
48
45
|
});
|
|
49
46
|
export const stopRpcServerOptionsSchema = z.object({ serverId: z.string().min(1) });
|
|
50
|
-
export const stopRpcServerRequestSchema = stopRpcServerOptionsSchema
|
|
47
|
+
export const stopRpcServerRequestSchema = stopRpcServerOptionsSchema
|
|
48
|
+
.extend({
|
|
51
49
|
type: z.literal('stopRpcServer')
|
|
52
|
-
})
|
|
50
|
+
})
|
|
51
|
+
.extend(sessionScopedRequestSchema.shape);
|
|
53
52
|
export const stopRpcServerResponseSchema = z.object({ type: z.literal('stopRpcServer') });
|
|
54
53
|
export const discoverRpcServersOptionsSchema = z.object({
|
|
55
54
|
topic: topicSchema.describe('Shared discovery topic. Only use with trusted private-network participants.'),
|
|
@@ -77,10 +76,11 @@ export const rpcServerCandidateSchema = z.object({
|
|
|
77
76
|
.describe('Server advertised RDMA availability for this endpoint at discovery time. Actual transport depends on client support and negotiation.'),
|
|
78
77
|
devices: z.array(rpcDeviceSchema).describe('Native served devices in endpoint enumeration order.')
|
|
79
78
|
});
|
|
80
|
-
export const discoverRpcServersRequestSchema = discoverRpcServersOptionsSchema
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
})
|
|
79
|
+
export const discoverRpcServersRequestSchema = discoverRpcServersOptionsSchema
|
|
80
|
+
.extend({
|
|
81
|
+
type: z.literal('discoverRpcServers')
|
|
82
|
+
})
|
|
83
|
+
.extend(trackedRequestSchema.shape);
|
|
84
84
|
export const discoverRpcServersResponseSchema = z.object({
|
|
85
85
|
type: z.literal('discoverRpcServers'),
|
|
86
86
|
servers: z.array(rpcServerCandidateSchema)
|