@qvac/inference 0.22.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +111 -60
- package/README.md +4 -2
- package/dist/api/assess-model-fit.d.ts +14 -14
- package/dist/api/assess-model-fit.js +14 -14
- package/dist/api/audio-gen.d.ts +2 -2
- package/dist/api/audio-gen.js +136 -45
- package/dist/api/batch-completion.js +2 -1
- package/dist/api/calc-tokens.d.ts +30 -0
- package/dist/api/calc-tokens.js +41 -0
- package/dist/api/classify.d.ts +5 -2
- package/dist/api/classify.js +5 -2
- package/dist/api/completion-stream.d.ts +2 -1
- package/dist/api/completion-stream.js +4 -1
- package/dist/api/decide.js +1 -1
- package/dist/api/delete-cache.d.ts +7 -1
- package/dist/api/delete-cache.js +11 -1
- package/dist/api/diffusion.d.ts +32 -2
- package/dist/api/diffusion.js +32 -2
- package/dist/api/download-asset.js +1 -1
- package/dist/api/embed.d.ts +82 -29
- package/dist/api/embed.js +64 -18
- package/dist/api/finetune.js +1 -1
- package/dist/api/get-model-info.d.ts +1 -1
- package/dist/api/index.d.ts +40 -38
- package/dist/api/index.js +39 -37
- package/dist/api/load-model.js +2 -2
- package/dist/api/logging-stream-registry.js +11 -7
- package/dist/api/logging-stream.d.ts +11 -1
- package/dist/api/logging-stream.js +11 -2
- package/dist/api/rag.d.ts +1 -1
- package/dist/api/rag.js +1 -1
- package/dist/api/registry.d.ts +18 -7
- package/dist/api/registry.js +37 -13
- package/dist/api/rerank.d.ts +17 -0
- package/dist/api/rerank.js +36 -0
- package/dist/api/subscribe-logs.js +5 -3
- package/dist/api/text-to-speech.d.ts +14 -5
- package/dist/api/text-to-speech.js +83 -29
- package/dist/api/transcribe.d.ts +6 -2
- package/dist/api/transcribe.js +7 -1
- package/dist/api/unload-model.js +1 -1
- package/dist/api/vector-index.d.ts +25 -3
- package/dist/api/vector-index.js +17 -2
- package/dist/api/video-request.d.ts +15 -15
- package/dist/api/video.js +1 -1
- package/dist/api/vla.js +1 -1
- package/dist/api/world-result.js +1 -1
- package/dist/api/world.d.ts +1 -1
- package/dist/api/world.js +1 -1
- package/dist/config/resolve-config.d.ts +1 -1
- package/dist/config/resolve-config.js +1 -1
- package/dist/constants/index.d.ts +2 -2
- package/dist/constants/index.js +2 -2
- package/dist/dispatch.d.ts +11 -4
- package/dist/dispatch.js +80 -18
- package/dist/engine.d.ts +1 -1
- package/dist/errors/index.d.ts +76 -0
- package/dist/errors/index.js +141 -1
- package/dist/handlers/assess-model-fit.d.ts +3 -21
- package/dist/handlers/assess-model-fit.js +4 -303
- package/dist/handlers/cancelHandler.d.ts +10 -9
- package/dist/handlers/cancelHandler.js +18 -14
- package/dist/handlers/completion-orchestrate.d.ts +14 -2
- package/dist/handlers/completion-orchestrate.js +166 -24
- package/dist/handlers/delete-cache.d.ts +2 -1
- package/dist/handlers/delete-cache.js +6 -1
- package/dist/handlers/download-asset.js +13 -6
- package/dist/handlers/finetune.js +1 -1
- package/dist/handlers/get-loaded-model-info.js +3 -3
- package/dist/handlers/get-model-info.js +72 -3
- package/dist/handlers/load-model/coreml-sidecars.d.ts +15 -11
- package/dist/handlers/load-model/coreml-sidecars.js +51 -91
- package/dist/handlers/load-model/download-lock.d.ts +20 -0
- package/dist/handlers/load-model/download-lock.js +213 -0
- package/dist/handlers/load-model/download-manager.d.ts +11 -3
- package/dist/handlers/load-model/download-manager.js +62 -11
- package/dist/handlers/load-model/download-stats.d.ts +2 -2
- package/dist/handlers/load-model/download-stats.js +3 -3
- package/dist/handlers/load-model/handler.js +97 -41
- package/dist/handlers/load-model/http-verify.d.ts +1 -1
- package/dist/handlers/load-model/http.d.ts +1 -1
- package/dist/handlers/load-model/http.js +330 -208
- package/dist/handlers/load-model/hyperdrive.d.ts +16 -1
- package/dist/handlers/load-model/hyperdrive.js +125 -52
- package/dist/handlers/load-model/index.d.ts +2 -2
- package/dist/handlers/load-model/index.js +2 -2
- package/dist/handlers/load-model/registry-client-options.d.ts +2 -0
- package/dist/handlers/load-model/registry-client-options.js +4 -2
- package/dist/handlers/load-model/registry-companion-set.d.ts +6 -1
- package/dist/handlers/load-model/registry-companion-set.js +20 -2
- package/dist/handlers/load-model/registry-delete.d.ts +5 -0
- package/dist/handlers/load-model/registry-delete.js +269 -0
- package/dist/handlers/load-model/registry-download-utils.d.ts +7 -5
- package/dist/handlers/load-model/registry-download-utils.js +95 -24
- package/dist/handlers/load-model/registry.d.ts +22 -2
- package/dist/handlers/load-model/registry.js +312 -179
- package/dist/handlers/load-model/resolve-session.d.ts +1 -1
- package/dist/handlers/load-model/resolve-session.js +11 -3
- package/dist/handlers/load-model/resolve.d.ts +1 -1
- package/dist/handlers/load-model/resolve.js +9 -6
- package/dist/handlers/load-model/types.d.ts +2 -0
- package/dist/handlers/logging-stream.d.ts +2 -1
- package/dist/handlers/logging-stream.js +35 -7
- package/dist/handlers/plugin-invoke.js +152 -39
- package/dist/handlers/registry.d.ts +2 -2
- package/dist/handlers/registry.js +108 -29
- package/dist/handlers/rpc-server.js +7 -7
- package/dist/handlers/session.d.ts +11 -0
- package/dist/handlers/session.js +40 -0
- package/dist/handlers/types.d.ts +7 -1
- package/dist/handlers/unload-model.js +8 -3
- package/dist/handlers/vector-index.js +118 -19
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/logging/addon.d.ts +1 -1
- package/dist/logging/addon.js +1 -1
- package/dist/logging/index.d.ts +5 -5
- package/dist/logging/index.js +4 -4
- package/dist/logging/logger.d.ts +1 -1
- package/dist/logging/logger.js +3 -3
- package/dist/logging/utils.js +11 -3
- package/dist/model-cache-layout.d.ts +1 -0
- package/dist/model-cache-layout.js +2 -0
- package/dist/model-fit/assess.d.ts +23 -0
- package/dist/model-fit/assess.js +110 -0
- package/dist/model-fit/describe.d.ts +12 -0
- package/dist/model-fit/describe.js +29 -0
- package/dist/{resources/model-fit/fit-stub/fetch-fit-stub.d.ts → model-fit/description/fetch.d.ts} +2 -2
- package/dist/{resources/model-fit/fit-stub/fetch-fit-stub.js → model-fit/description/fetch.js} +98 -19
- package/dist/{resources/model-fit/fit-stub/resolve-load-from-stubs.d.ts → model-fit/description/resolve-load.d.ts} +8 -6
- package/dist/{resources/model-fit/fit-stub/resolve-load-from-stubs.js → model-fit/description/resolve-load.js} +20 -20
- package/dist/model-fit/fit-load.d.ts +23 -0
- package/dist/model-fit/fit-load.js +29 -0
- package/dist/model-fit/load-check.d.ts +34 -0
- package/dist/model-fit/load-check.js +77 -0
- package/dist/model-fit/run/engine.d.ts +12 -0
- package/dist/model-fit/run/engine.js +9 -0
- package/dist/model-fit/run/run.d.ts +46 -0
- package/dist/model-fit/run/run.js +208 -0
- package/dist/model-fit/sum.d.ts +32 -0
- package/dist/model-fit/sum.js +60 -0
- package/dist/models/registry/index.d.ts +1 -1
- package/dist/models/registry/index.js +2 -2
- package/dist/models/registry/models.d.ts +1377 -130
- package/dist/models/registry/models.js +3622 -2312
- package/dist/plugins/builtin/asr-ggml/config.d.ts +19 -4
- package/dist/plugins/builtin/asr-ggml/config.js +39 -3
- package/dist/plugins/builtin/asr-ggml/fit.d.ts +18 -0
- package/dist/plugins/builtin/asr-ggml/fit.js +118 -0
- package/dist/plugins/builtin/audiogen-ggml/config.d.ts +6 -0
- package/dist/plugins/builtin/audiogen-ggml/config.js +9 -0
- package/dist/plugins/builtin/audiogen-ggml/fit.d.ts +3 -0
- package/dist/{resources/model-fit/native-probe/engines/audiogen.js → plugins/builtin/audiogen-ggml/fit.js} +28 -18
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-edit-stream.js +4 -3
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.d.ts +17 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-encode.js +68 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.d.ts +11 -3
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-run.js +47 -11
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +4 -3
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.d.ts +3 -1
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-understand.js +4 -3
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +203 -72
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +17 -5
- package/dist/plugins/builtin/bci-whispercpp-transcription/fit.d.ts +3 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/fit.js +22 -0
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +71 -29
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +12 -10
- package/dist/plugins/builtin/ggml-classification/plugin.d.ts +8 -5
- package/dist/plugins/builtin/ggml-classification/plugin.js +89 -9
- package/dist/plugins/builtin/ggml-image-embedding/ops/embed.d.ts +24 -0
- package/dist/plugins/builtin/ggml-image-embedding/ops/embed.js +184 -0
- package/dist/plugins/builtin/ggml-image-embedding/plugin.d.ts +120 -0
- package/dist/plugins/builtin/ggml-image-embedding/plugin.js +39 -0
- package/dist/plugins/builtin/ggml-ocr/fit.d.ts +6 -0
- package/dist/plugins/builtin/ggml-ocr/fit.js +19 -0
- package/dist/plugins/builtin/ggml-ocr/params.d.ts +1 -1
- package/dist/plugins/builtin/ggml-ocr/plugin.d.ts +20 -13
- package/dist/plugins/builtin/ggml-ocr/plugin.js +111 -25
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.d.ts +2 -2
- package/dist/plugins/builtin/ggml-vla/ops/vla-run.js +18 -2
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +15 -15
- package/dist/plugins/builtin/ggml-vla/plugin.js +3 -3
- package/dist/plugins/builtin/index.d.ts +16 -14
- package/dist/plugins/builtin/index.js +16 -14
- package/dist/plugins/builtin/llamacpp-completion/fit.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-completion/fit.js +28 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +6 -8
- package/dist/plugins/builtin/llamacpp-completion/ops/cache-logger.js +15 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.d.ts +2 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/calc-tokens.js +32 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.d.ts +3 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stats.js +6 -0
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.d.ts +18 -3
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +94 -21
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.d.ts +5 -4
- package/dist/plugins/builtin/llamacpp-completion/ops/finetune.js +54 -19
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +5 -3
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +9 -7
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +315 -95
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +51 -25
- package/dist/plugins/builtin/llamacpp-completion/transform.js +4 -0
- package/dist/plugins/builtin/llamacpp-decision/helpers.d.ts +0 -1
- package/dist/plugins/builtin/llamacpp-decision/helpers.js +7 -15
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.d.ts +3 -1
- package/dist/plugins/builtin/llamacpp-decision/ops/decide.js +6 -3
- package/dist/plugins/builtin/llamacpp-decision/plugin.d.ts +43 -35
- package/dist/plugins/builtin/llamacpp-decision/plugin.js +4 -3
- package/dist/plugins/builtin/llamacpp-embedding/fit.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-embedding/fit.js +23 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +201 -10
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +52 -16
- package/dist/plugins/builtin/llamacpp-embedding/transform.d.ts +2 -2
- package/dist/plugins/builtin/llamacpp-embedding/transform.js +29 -2
- package/dist/plugins/builtin/llamacpp-rerank/helpers.d.ts +3 -0
- package/dist/plugins/builtin/llamacpp-rerank/helpers.js +16 -0
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.d.ts +7 -0
- package/dist/plugins/builtin/llamacpp-rerank/ops/rerank.js +120 -0
- package/dist/plugins/builtin/llamacpp-rerank/plugin.d.ts +92 -0
- package/dist/plugins/builtin/llamacpp-rerank/plugin.js +44 -0
- package/dist/plugins/builtin/moss-transcribe/plugin.d.ts +46 -19
- package/dist/plugins/builtin/moss-transcribe/plugin.js +9 -7
- package/dist/plugins/builtin/nmtcpp-translation/fit.d.ts +3 -0
- package/dist/plugins/builtin/nmtcpp-translation/fit.js +11 -0
- package/dist/plugins/builtin/nmtcpp-translation/plugin.d.ts +32 -18
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +8 -6
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +234 -38
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +42 -16
- package/dist/plugins/builtin/sdcpp-generation/fit.d.ts +3 -0
- package/dist/plugins/builtin/sdcpp-generation/fit.js +68 -0
- package/dist/plugins/builtin/sdcpp-generation/load-files.d.ts +30 -0
- package/dist/plugins/builtin/sdcpp-generation/load-files.js +121 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.d.ts +8 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/diffusion.js +174 -53
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.d.ts +4 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.js +107 -22
- package/dist/plugins/builtin/sdcpp-generation/ops/video.d.ts +7 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +23 -9
- package/dist/plugins/builtin/sdcpp-generation/ops/world.d.ts +6 -4
- package/dist/plugins/builtin/sdcpp-generation/ops/world.js +8 -3
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +166 -47
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +67 -155
- package/dist/plugins/builtin/sdcpp-generation/upscaler-config.d.ts +15 -0
- package/dist/plugins/builtin/sdcpp-generation/upscaler-config.js +19 -0
- package/dist/plugins/builtin/sdcpp-generation/world-config.d.ts +8 -0
- package/dist/plugins/builtin/sdcpp-generation/world-config.js +9 -0
- package/dist/plugins/builtin/tts-ggml/fit.d.ts +3 -0
- package/dist/plugins/builtin/tts-ggml/fit.js +162 -0
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/cancel-binding.js +4 -1
- package/dist/plugins/builtin/tts-ggml/ops/moss-options.d.ts +1 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.d.ts +3 -2
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +24 -11
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.d.ts +2 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +53 -18
- package/dist/plugins/builtin/tts-ggml/ops/voice-options.d.ts +50 -0
- package/dist/plugins/builtin/tts-ggml/ops/voice-options.js +74 -0
- package/dist/plugins/builtin/tts-ggml/plugin.d.ts +415 -57
- package/dist/plugins/builtin/tts-ggml/plugin.js +183 -34
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +104 -40
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +12 -10
- package/dist/plugins/index.d.ts +1 -1
- package/dist/plugins/index.js +2 -2
- package/dist/plugins/ops/bci-transcribe.d.ts +5 -4
- package/dist/plugins/ops/bci-transcribe.js +18 -4
- package/dist/plugins/ops/cancel.d.ts +1 -0
- package/dist/plugins/ops/cancel.js +11 -2
- package/dist/plugins/ops/embed.d.ts +5 -3
- package/dist/plugins/ops/embed.js +89 -34
- package/dist/plugins/ops/kv-cache-utils.d.ts +5 -3
- package/dist/plugins/ops/kv-cache-utils.js +21 -10
- package/dist/plugins/ops/load-model.d.ts +10 -2
- package/dist/plugins/ops/load-model.js +45 -19
- package/dist/plugins/ops/transcribe.d.ts +5 -4
- package/dist/plugins/ops/transcribe.js +63 -28
- package/dist/plugins/ops/translate.d.ts +2 -1
- package/dist/plugins/ops/translate.js +7 -4
- package/dist/plugins/ops/unload-model.d.ts +17 -1
- package/dist/plugins/ops/unload-model.js +32 -3
- package/dist/plugins/turbovec-backend.d.ts +1 -1
- package/dist/plugins/turbovec-backend.js +21 -6
- package/dist/profiling/aggregator.d.ts +1 -1
- package/dist/profiling/aggregator.js +1 -1
- package/dist/profiling/controller.d.ts +1 -1
- package/dist/profiling/controller.js +2 -2
- package/dist/profiling/events.d.ts +1 -1
- package/dist/profiling/events.js +2 -2
- package/dist/profiling/exporters.d.ts +1 -1
- package/dist/profiling/exporters.js +3 -3
- package/dist/profiling/index.d.ts +13 -13
- package/dist/profiling/index.js +12 -12
- package/dist/profiling/operation-metrics.d.ts +1 -1
- package/dist/profiling/operation-metrics.js +24 -21
- package/dist/profiling/operation-wrappers.js +3 -3
- package/dist/profiling/profiler.js +1 -1
- package/dist/rag/embedding-scope.d.ts +16 -0
- package/dist/rag/embedding-scope.js +81 -0
- package/dist/rag/handler.d.ts +25 -0
- package/dist/rag/handler.js +51 -17
- package/dist/rag/handlers/close-workspace.js +1 -1
- package/dist/rag/handlers/delete-embeddings.js +1 -1
- package/dist/rag/handlers/delete-workspace.js +1 -1
- package/dist/rag/handlers/ingest.js +3 -7
- package/dist/rag/handlers/list-workspaces.d.ts +1 -1
- package/dist/rag/handlers/list-workspaces.js +1 -1
- package/dist/rag/handlers/reindex.js +1 -1
- package/dist/rag/handlers/save-embeddings.js +1 -1
- package/dist/rag/handlers/search.d.ts +10 -1
- package/dist/rag/handlers/search.js +17 -8
- package/dist/rag/index.d.ts +11 -11
- package/dist/rag/index.js +11 -11
- package/dist/rag/rag-operation-manager.d.ts +27 -3
- package/dist/rag/rag-operation-manager.js +10 -27
- package/dist/rag/rag-workspace-manager.js +1 -1
- package/dist/registry.js +7 -1
- package/dist/resources/collector.d.ts +71 -37
- package/dist/resources/collector.js +9 -2
- package/dist/resources/instance.d.ts +1 -1
- package/dist/resources/instance.js +1 -1
- package/dist/resources/native.d.ts +1 -1
- package/dist/resources/normalize.d.ts +1 -1
- package/dist/resources/types.d.ts +2 -0
- package/dist/rpc/inventory.js +1 -12
- package/dist/runtime/env.d.ts +5 -1
- package/dist/runtime/env.js +6 -21
- package/dist/runtime/index.d.ts +8 -5
- package/dist/runtime/index.js +4 -2
- package/dist/runtime/lifecycle.d.ts +6 -4
- package/dist/runtime/lifecycle.js +16 -12
- package/dist/runtime/model-config-utils.js +8 -1
- package/dist/runtime/model-registry.d.ts +30 -4
- package/dist/runtime/model-registry.js +48 -2
- package/dist/runtime/model-work-lock.d.ts +13 -0
- package/dist/runtime/model-work-lock.js +48 -0
- package/dist/runtime/registry-client.d.ts +47 -1
- package/dist/runtime/registry-client.js +446 -88
- package/dist/runtime/request-context.d.ts +11 -3
- package/dist/runtime/request-context.js +39 -2
- package/dist/runtime/request-registry.d.ts +56 -8
- package/dist/runtime/request-registry.js +190 -72
- package/dist/runtime/runtime-lifecycle.js +10 -1
- package/dist/runtime/session-id.d.ts +18 -0
- package/dist/runtime/session-id.js +20 -0
- package/dist/runtime/session-registry.d.ts +39 -0
- package/dist/runtime/session-registry.js +114 -0
- package/dist/runtime/state.d.ts +1 -1
- package/dist/runtime/state.js +2 -2
- package/dist/runtime/vector-index-backend.d.ts +6 -2
- package/dist/runtime/vector-index-registry.d.ts +26 -5
- package/dist/runtime/vector-index-registry.js +88 -27
- package/dist/schemas/abort-signal.d.ts +8 -0
- package/dist/schemas/assess-model-fit.d.ts +138 -282
- package/dist/schemas/assess-model-fit.js +85 -219
- package/dist/schemas/audio-gen.d.ts +291 -25
- package/dist/schemas/audio-gen.js +130 -26
- package/dist/schemas/batch-completion-stream.d.ts +113 -15
- package/dist/schemas/batch-completion-stream.js +10 -6
- package/dist/schemas/bci-config.d.ts +3 -1
- package/dist/schemas/bci-config.js +2 -2
- package/dist/schemas/bci.d.ts +55 -4
- package/dist/schemas/bci.js +13 -18
- package/dist/schemas/calc-tokens.d.ts +141 -0
- package/dist/schemas/calc-tokens.js +53 -0
- package/dist/schemas/cancel.d.ts +31 -0
- package/dist/schemas/cancel.js +11 -2
- package/dist/schemas/classification.d.ts +3 -0
- package/dist/schemas/classification.js +6 -2
- package/dist/schemas/common.d.ts +2209 -348
- package/dist/schemas/common.js +52 -33
- package/dist/schemas/completion-event.d.ts +11 -2
- package/dist/schemas/completion-event.js +5 -2
- package/dist/schemas/completion-stream.d.ts +173 -19
- package/dist/schemas/completion-stream.js +117 -20
- package/dist/schemas/config.d.ts +2064 -178
- package/dist/schemas/config.js +21 -23
- package/dist/schemas/decision.d.ts +12 -2
- package/dist/schemas/decision.js +16 -1
- package/dist/schemas/download-asset.d.ts +13 -6
- package/dist/schemas/download-asset.js +4 -7
- package/dist/schemas/embed.d.ts +286 -3
- package/dist/schemas/embed.js +162 -16
- package/dist/schemas/engine-addon-map.d.ts +3 -1
- package/dist/schemas/engine-addon-map.js +5 -3
- package/dist/schemas/errors.d.ts +13 -0
- package/dist/schemas/errors.js +71 -1
- package/dist/schemas/finetune.d.ts +14 -4
- package/dist/schemas/finetune.js +5 -6
- package/dist/schemas/heartbeat.d.ts +1 -0
- package/dist/schemas/heartbeat.js +5 -2
- package/dist/schemas/image-embedding.d.ts +27 -0
- package/dist/schemas/image-embedding.js +107 -0
- package/dist/schemas/index.d.ts +55 -50
- package/dist/schemas/index.js +54 -49
- package/dist/schemas/lifecycle.d.ts +3 -0
- package/dist/schemas/lifecycle.js +13 -6
- package/dist/schemas/llamacpp-config.d.ts +443 -20
- package/dist/schemas/llamacpp-config.js +188 -14
- package/dist/schemas/load-model.d.ts +8940 -1952
- package/dist/schemas/load-model.js +73 -9
- package/dist/schemas/logging-stream.d.ts +1 -0
- package/dist/schemas/logging-stream.js +5 -2
- package/dist/schemas/mcp-adapter.d.ts +1 -1
- package/dist/schemas/model-config-schemas.d.ts +731 -111
- package/dist/schemas/model-config-schemas.js +16 -12
- package/dist/schemas/model-fit-policy.d.ts +2 -2
- package/dist/schemas/model-info.d.ts +54 -56
- package/dist/schemas/model-info.js +18 -7
- package/dist/schemas/model-ops.d.ts +12 -0
- package/dist/schemas/model-ops.js +33 -5
- package/dist/schemas/model-src-utils.d.ts +14 -6
- package/dist/schemas/model-src-utils.js +2 -2
- package/dist/schemas/model-types.d.ts +12 -4
- package/dist/schemas/model-types.js +2 -0
- package/dist/schemas/ocr.d.ts +11 -4
- package/dist/schemas/ocr.js +7 -3
- package/dist/schemas/plugin.d.ts +70 -8
- package/dist/schemas/plugin.js +32 -4
- package/dist/schemas/profiling.d.ts +32 -32
- package/dist/schemas/profiling.js +1 -1
- package/dist/schemas/rag.d.ts +9 -0
- package/dist/schemas/rag.js +11 -17
- package/dist/schemas/registry.d.ts +38 -10
- package/dist/schemas/registry.js +16 -7
- package/dist/schemas/reload-config.d.ts +45 -6
- package/dist/schemas/reload-config.js +13 -6
- package/dist/schemas/request-base.d.ts +39 -0
- package/dist/schemas/request-base.js +64 -0
- package/dist/schemas/rerank.d.ts +132 -0
- package/dist/schemas/rerank.js +104 -0
- package/dist/schemas/rpc-server.d.ts +3 -0
- package/dist/schemas/rpc-server.js +15 -15
- package/dist/schemas/sdcpp-config.d.ts +144 -18
- package/dist/schemas/sdcpp-config.js +123 -28
- package/dist/schemas/session.d.ts +37 -0
- package/dist/schemas/session.js +53 -0
- package/dist/schemas/system-resources.d.ts +364 -179
- package/dist/schemas/system-resources.js +8 -3
- package/dist/schemas/text-to-speech.d.ts +1408 -275
- package/dist/schemas/text-to-speech.js +547 -33
- package/dist/schemas/tools.d.ts +7 -0
- package/dist/schemas/tools.js +29 -0
- package/dist/schemas/transcription-config.d.ts +267 -2
- package/dist/schemas/transcription-config.js +138 -7
- package/dist/schemas/transcription.d.ts +94 -7
- package/dist/schemas/transcription.js +99 -22
- package/dist/schemas/translate.d.ts +4 -2
- package/dist/schemas/translate.js +6 -10
- package/dist/schemas/translation-config.d.ts +40 -20
- package/dist/schemas/translation-config.js +1 -1
- package/dist/schemas/tts-pcm.d.ts +35 -0
- package/dist/schemas/tts-pcm.js +69 -0
- package/dist/schemas/vector-index.d.ts +47 -1
- package/dist/schemas/vector-index.js +37 -10
- package/dist/schemas/vla.d.ts +3 -3
- package/dist/surface.d.ts +4 -3
- package/dist/surface.js +4 -3
- package/dist/utils/addon-responses.d.ts +71 -2
- package/dist/utils/aggregate-events.d.ts +1 -1
- package/dist/utils/aggregate-events.js +2 -2
- package/dist/utils/archive.d.ts +7 -0
- package/dist/utils/archive.js +89 -30
- package/dist/utils/asr-events.d.ts +5 -0
- package/dist/utils/asr-events.js +10 -1
- package/dist/utils/audio/decoder.js +3 -1
- package/dist/utils/audio-input.js +1 -1
- package/dist/utils/cache/auto-history.d.ts +2 -1
- package/dist/utils/cache/auto-history.js +5 -3
- package/dist/utils/cache/layout.d.ts +48 -0
- package/dist/utils/cache/layout.js +190 -0
- package/dist/utils/cache/paths.js +8 -10
- package/dist/utils/cache/types.d.ts +2 -1
- package/dist/utils/checksum.d.ts +7 -0
- package/dist/utils/checksum.js +59 -2
- package/dist/utils/completion-normalizer.js +287 -88
- package/dist/utils/gguf-architecture.d.ts +7 -0
- package/dist/utils/gguf-architecture.js +142 -0
- package/dist/utils/gguf-tensor-extractor.d.ts +0 -6
- package/dist/utils/gguf-tensor-extractor.js +1 -50
- package/dist/utils/index.d.ts +13 -13
- package/dist/utils/index.js +13 -13
- package/dist/utils/mcp-adapter.d.ts +1 -1
- package/dist/utils/mcp-adapter.js +1 -1
- package/dist/utils/path-security.js +1 -1
- package/dist/utils/shard-utils.d.ts +0 -16
- package/dist/utils/shard-utils.js +14 -48
- package/dist/utils/tool-integration.d.ts +3 -2
- package/dist/utils/tool-integration.js +3 -6
- package/dist/utils/tools/defer.d.ts +50 -31
- package/dist/utils/tools/defer.js +115 -51
- package/dist/utils/tools/dialect.d.ts +15 -4
- package/dist/utils/tools/dialect.js +34 -5
- package/dist/utils/tools/index.d.ts +3 -3
- package/dist/utils/tools/index.js +3 -3
- package/dist/utils/tools/parser.js +8 -8
- package/dist/utils/tools/parsers/dsml.d.ts +1 -1
- package/dist/utils/tools/parsers/dsml.js +1 -1
- package/dist/utils/tools/parsers/gemma4native.d.ts +1 -1
- package/dist/utils/tools/parsers/gemma4native.js +1 -1
- package/dist/utils/tools/parsers/harmony.d.ts +1 -1
- package/dist/utils/tools/parsers/harmony.js +1 -1
- package/dist/utils/tools/parsers/hermes.d.ts +1 -1
- package/dist/utils/tools/parsers/hermes.js +1 -1
- package/dist/utils/tools/parsers/json.d.ts +1 -1
- package/dist/utils/tools/parsers/json.js +1 -1
- package/dist/utils/tools/parsers/pythonic.d.ts +1 -1
- package/dist/utils/tools/parsers/pythonic.js +1 -1
- package/dist/utils/tools/parsers/qwen35.d.ts +1 -1
- package/dist/utils/tools/parsers/qwen35.js +1 -1
- package/dist/utils/transcribe-metadata.d.ts +28 -2
- package/dist/utils/transcribe-metadata.js +16 -0
- package/dist/utils/tts-stats.d.ts +43 -1
- package/dist/utils/tts-stats.js +120 -22
- package/dist/utils/until-aborted.d.ts +7 -0
- package/dist/utils/until-aborted.js +34 -0
- package/package.json +67 -55
- package/dist/models/registry/resource-profiles.d.ts +0 -17
- package/dist/models/registry/resource-profiles.js +0 -5657
- package/dist/resources/model-fit/assess.d.ts +0 -72
- package/dist/resources/model-fit/assess.js +0 -679
- package/dist/resources/model-fit/fit-stub/project-fit-from-load.d.ts +0 -27
- package/dist/resources/model-fit/fit-stub/project-fit-from-load.js +0 -33
- package/dist/resources/model-fit/floor.d.ts +0 -31
- package/dist/resources/model-fit/floor.js +0 -65
- package/dist/resources/model-fit/kv-cache.d.ts +0 -23
- package/dist/resources/model-fit/kv-cache.js +0 -85
- package/dist/resources/model-fit/native-probe/advisory-fit.d.ts +0 -54
- package/dist/resources/model-fit/native-probe/advisory-fit.js +0 -248
- package/dist/resources/model-fit/native-probe/child-engine-fit.d.ts +0 -2
- package/dist/resources/model-fit/native-probe/child-engine-fit.js +0 -49
- package/dist/resources/model-fit/native-probe/classify-fit.d.ts +0 -6
- package/dist/resources/model-fit/native-probe/classify-fit.js +0 -166
- package/dist/resources/model-fit/native-probe/create-fit-request.d.ts +0 -31
- package/dist/resources/model-fit/native-probe/create-fit-request.js +0 -61
- package/dist/resources/model-fit/native-probe/engine-fit.d.ts +0 -73
- package/dist/resources/model-fit/native-probe/engine-fit.js +0 -13
- package/dist/resources/model-fit/native-probe/engines/asr.d.ts +0 -14
- package/dist/resources/model-fit/native-probe/engines/asr.js +0 -126
- package/dist/resources/model-fit/native-probe/engines/audiogen.d.ts +0 -8
- package/dist/resources/model-fit/native-probe/engines/diffusion.d.ts +0 -7
- package/dist/resources/model-fit/native-probe/engines/diffusion.js +0 -65
- package/dist/resources/model-fit/native-probe/engines/llama.d.ts +0 -12
- package/dist/resources/model-fit/native-probe/engines/llama.js +0 -41
- package/dist/resources/model-fit/native-probe/engines/nmt.d.ts +0 -8
- package/dist/resources/model-fit/native-probe/engines/nmt.js +0 -11
- package/dist/resources/model-fit/native-probe/engines/ocr.d.ts +0 -9
- package/dist/resources/model-fit/native-probe/engines/ocr.js +0 -13
- package/dist/resources/model-fit/native-probe/engines/tts.d.ts +0 -10
- package/dist/resources/model-fit/native-probe/engines/tts.js +0 -152
- package/dist/resources/model-fit/native-probe/fit-outcome.d.ts +0 -16
- package/dist/resources/model-fit/native-probe/fit-process.d.ts +0 -27
- package/dist/resources/model-fit/native-probe/fit-process.js +0 -55
- package/dist/resources/model-fit/native-probe/fit-runner.d.ts +0 -1
- package/dist/resources/model-fit/native-probe/fit-runner.js +0 -58
- package/dist/resources/model-fit/native-probe/run-fit.d.ts +0 -24
- package/dist/resources/model-fit/native-probe/run-fit.js +0 -20
- package/dist/resources/model-fit/native-probe/run-in-process-fit.d.ts +0 -24
- package/dist/resources/model-fit/native-probe/run-in-process-fit.js +0 -123
- package/dist/resources/model-fit/native-probe/run-isolated-fit.d.ts +0 -63
- package/dist/resources/model-fit/native-probe/run-isolated-fit.js +0 -377
- package/dist/resources/model-fit/platform.d.ts +0 -9
- package/dist/resources/model-fit/platform.js +0 -26
- package/dist/resources/model-fit/types.d.ts +0 -6
- package/dist/resources/model-fit/types.js +0 -1
- package/dist/schemas/model-resource-profile.d.ts +0 -108
- package/dist/schemas/model-resource-profile.js +0 -80
- /package/dist/{resources/model-fit/native-probe/engines → model-fit}/gpu.d.ts +0 -0
- /package/dist/{resources/model-fit/native-probe/engines → model-fit}/gpu.js +0 -0
- /package/dist/{resources/model-fit/native-probe/fit-outcome.js → schemas/abort-signal.js} +0 -0
|
@@ -1,20 +1,26 @@
|
|
|
1
|
-
import { logCacheDisabled, logCacheSave, logMessagesToAddon } from '
|
|
2
|
-
import { extractSystemPrompt, getCurrentCacheInfo } from '
|
|
1
|
+
import { logCacheDisabled, logCacheSave, logMessagesToAddon } from '../../../../plugins/builtin/llamacpp-completion/ops/cache-logger.js';
|
|
2
|
+
import { extractSystemPrompt, getCurrentCacheInfo, resolveCacheFilePath } from '../../../../plugins/ops/kv-cache-utils.js';
|
|
3
3
|
import { getModel, getModelConfig } from '../../../../runtime/model-registry.js';
|
|
4
|
-
import { shouldCommitCachedTurn } from '
|
|
5
|
-
import { createKvCacheSession, generateConfigHash } from '
|
|
4
|
+
import { shouldCommitCachedTurn } from '../../../../plugins/builtin/llamacpp-completion/ops/kv-cache-state.js';
|
|
5
|
+
import { createKvCacheSession, generateConfigHash } from '../../../../plugins/builtin/llamacpp-completion/ops/kv-cache-session.js';
|
|
6
6
|
import { detectToolDialect, prependToolsToHistory } from '../../../../utils/tool-integration.js';
|
|
7
|
-
import { resolveDeferredTools, toWireTool
|
|
7
|
+
import { resolveDeferredTools, toWireTool } from '../../../../utils/tools/defer.js';
|
|
8
8
|
import { parseToolCalls } from '../../../../utils/tools/index.js';
|
|
9
9
|
import { getResponseFormatJsonSchema } from '../../../../utils/response-format.js';
|
|
10
|
-
import { buildAutoCacheSaveHistory } from '../../../../utils/index.js';
|
|
10
|
+
import { buildAutoCacheSaveHistory, normalizeCacheMessage } from '../../../../utils/index.js';
|
|
11
11
|
import { getEngineLogger } from '../../../../logging/index.js';
|
|
12
|
-
import { AttachmentNotFoundError } from '../../../../errors/index.js';
|
|
12
|
+
import { AttachmentNotFoundError, InvalidSeedFromParamsError, KvCacheSeedSharingDisabledError, KvCacheSeedSourceNotFoundError, RequestValidationFailedError } from '../../../../errors/index.js';
|
|
13
13
|
import { nowMs } from '../../../../profiling/index.js';
|
|
14
14
|
import { buildStreamResult } from '../../../../profiling/model-execution.js';
|
|
15
|
-
import { normalizeCompletionStats, withEmittedTokens } from '
|
|
15
|
+
import { normalizeCompletionStats, withEmittedTokens } from '../../../../plugins/builtin/llamacpp-completion/ops/completion-stats.js';
|
|
16
16
|
import fs from 'bare-fs';
|
|
17
17
|
const logger = getEngineLogger();
|
|
18
|
+
export function withParallelToolCalls(generationParams) {
|
|
19
|
+
return {
|
|
20
|
+
...generationParams,
|
|
21
|
+
parallel_tool_calls: generationParams?.parallel_tool_calls ?? true
|
|
22
|
+
};
|
|
23
|
+
}
|
|
18
24
|
function transformMessage(message) {
|
|
19
25
|
const transformed = [];
|
|
20
26
|
// Check if it's a tool definition (has type: "function")
|
|
@@ -28,6 +34,21 @@ function transformMessage(message) {
|
|
|
28
34
|
return transformed;
|
|
29
35
|
}
|
|
30
36
|
const msg = message;
|
|
37
|
+
// Ordered parts reach the addon as ONE message, so text and each image share
|
|
38
|
+
// a turn. Attachments cannot express that: they are all sent before the
|
|
39
|
+
// text, which leaves the model unable to tell which image is which option.
|
|
40
|
+
if (Array.isArray(msg.content)) {
|
|
41
|
+
const parts = msg.content.map((part) => {
|
|
42
|
+
if (part.type === 'text')
|
|
43
|
+
return { type: 'text', text: part.text };
|
|
44
|
+
if (!fs.existsSync(part.path)) {
|
|
45
|
+
throw new AttachmentNotFoundError(part.path);
|
|
46
|
+
}
|
|
47
|
+
return { type: 'media', content: part.path };
|
|
48
|
+
});
|
|
49
|
+
transformed.push({ role: msg.role, content: parts });
|
|
50
|
+
return transformed;
|
|
51
|
+
}
|
|
31
52
|
if (msg.attachments && msg.attachments.length > 0) {
|
|
32
53
|
for (const attachment of msg.attachments) {
|
|
33
54
|
if (!fs.existsSync(attachment.path)) {
|
|
@@ -134,6 +155,12 @@ async function* processModelResponse(model, messagesToSend, tools, generationPar
|
|
|
134
155
|
...(generationParams && { generationParams }),
|
|
135
156
|
...(cacheOptions?.cacheKey !== undefined && {
|
|
136
157
|
cacheKey: cacheOptions.cacheKey
|
|
158
|
+
}),
|
|
159
|
+
...(cacheOptions?.prefill !== undefined && {
|
|
160
|
+
prefill: cacheOptions.prefill
|
|
161
|
+
}),
|
|
162
|
+
...(cacheOptions?.baseCacheKey !== undefined && {
|
|
163
|
+
baseCacheKey: cacheOptions.baseCacheKey
|
|
137
164
|
})
|
|
138
165
|
};
|
|
139
166
|
const hasRunOptions = Object.keys(runOptions).length > 0;
|
|
@@ -167,23 +194,32 @@ async function* processModelResponse(model, messagesToSend, tools, generationPar
|
|
|
167
194
|
toolCalls: toolCallsResult,
|
|
168
195
|
responseText: accumulatedText,
|
|
169
196
|
producedTokens,
|
|
197
|
+
...(stopReason === 'eos' && { stopReason: 'eos' }),
|
|
198
|
+
...(stopReason === 'antiprompt' && { stopReason: 'stopSequence' }),
|
|
170
199
|
stoppedAtContextBoundary: stopReason === 'contextOverflow'
|
|
171
200
|
};
|
|
172
201
|
}
|
|
173
202
|
export async function* completion(params, opts) {
|
|
174
|
-
const { history: requestHistory, modelId, kvCache, tools, generationParams, responseFormat } = params;
|
|
203
|
+
const { history: requestHistory, modelId, kvCache, prefill, seedFrom, tools, toolSearch, generationParams, responseFormat } = params;
|
|
175
204
|
const { signal, scope } = opts;
|
|
176
205
|
const requestLogger = opts.logger ?? logger;
|
|
206
|
+
// seedFrom needs a named kvCache key other than itself: the auto key is
|
|
207
|
+
// never returned to the caller, so it cannot be named on either side.
|
|
208
|
+
if (seedFrom !== undefined && (typeof kvCache !== 'string' || seedFrom === kvCache)) {
|
|
209
|
+
throw new InvalidSeedFromParamsError();
|
|
210
|
+
}
|
|
177
211
|
const modelConfig = getModelConfig(modelId);
|
|
178
212
|
const history = seedConfiguredSystemPrompt(requestHistory, modelConfig);
|
|
213
|
+
// The addon cannot render a prompt without messages; tools alone are not enough.
|
|
214
|
+
if (prefill && history.length === 0) {
|
|
215
|
+
throw new RequestValidationFailedError('prefill needs at least one message: add a system message to history or set modelConfig.system_prompt');
|
|
216
|
+
}
|
|
179
217
|
const toolsEnabled = modelConfig.tools === true;
|
|
180
|
-
// With deferred tools the prompt
|
|
181
|
-
//
|
|
182
|
-
//
|
|
183
|
-
|
|
184
|
-
const deferred = resolveDeferredTools(tools, history);
|
|
218
|
+
// With deferred tools the prompt carries the always-loaded ones, `tool_search`
|
|
219
|
+
// and its catalog, and whatever earlier searches loaded. `null` when nothing
|
|
220
|
+
// defers, which leaves the existing path alone.
|
|
221
|
+
const deferred = resolveDeferredTools(tools, history, toolSearch);
|
|
185
222
|
const toolsToRender = deferred?.toolsToRender ?? tools?.map(toWireTool);
|
|
186
|
-
const callableTools = deferred?.callableTools ?? toolsToRender;
|
|
187
223
|
const toolsActive = !!toolsToRender?.length && toolsEnabled;
|
|
188
224
|
const dialect = tools && tools.length > 0 ? (params.toolDialect ?? detectToolDialect(modelId)) : undefined;
|
|
189
225
|
// `responseFormat` is forwarded to the addon as a per-request
|
|
@@ -203,7 +239,7 @@ export async function* completion(params, opts) {
|
|
|
203
239
|
};
|
|
204
240
|
}
|
|
205
241
|
}
|
|
206
|
-
mergedGenerationParams =
|
|
242
|
+
mergedGenerationParams = withParallelToolCalls(mergedGenerationParams);
|
|
207
243
|
const model = getModel(modelId);
|
|
208
244
|
// Per-request hard cancel: under continuous batching the model runs several
|
|
209
245
|
// jobs at once, so an abort must cancel only THIS request's job, not the
|
|
@@ -241,7 +277,7 @@ export async function* completion(params, opts) {
|
|
|
241
277
|
if (!kvCache) {
|
|
242
278
|
logCacheDisabled();
|
|
243
279
|
logMessagesToAddon(prompt, 'NO_CACHE');
|
|
244
|
-
return yield* processModelResponse(model, prompt,
|
|
280
|
+
return yield* processModelResponse(model, prompt, toolsToRender, mergedGenerationParams, undefined, dialect, setActiveResponse);
|
|
245
281
|
}
|
|
246
282
|
// ---- KV-cache path. The session owns every bookkeeping layer; the handler
|
|
247
283
|
// registers one deferred unwind that `commitTurn` short-circuits. The addon
|
|
@@ -272,7 +308,8 @@ export async function* completion(params, opts) {
|
|
|
272
308
|
turn = await session.beginTurn({
|
|
273
309
|
kind: 'auto',
|
|
274
310
|
configHash,
|
|
275
|
-
history: cacheMessages,
|
|
311
|
+
history: prefill ? cacheMessages.map(normalizeCacheMessage) : cacheMessages,
|
|
312
|
+
...(toolsActive && toolsToRender ? { tools: toolsToRender } : {}),
|
|
276
313
|
signal
|
|
277
314
|
});
|
|
278
315
|
}
|
|
@@ -280,8 +317,35 @@ export async function* completion(params, opts) {
|
|
|
280
317
|
// defer above so this runs before the listener detach.
|
|
281
318
|
let rollbackOnUnwind = false;
|
|
282
319
|
scope.defer(() => (rollbackOnUnwind ? session.rollback(turn) : session.releaseTurn(turn)));
|
|
320
|
+
// A seed starts a chat that has no cache yet; a chat that has one keeps it.
|
|
321
|
+
// The addon ignores `baseCacheKey` on a model loaded with sharing off and
|
|
322
|
+
// skips a missing source silently, so both are checked here first.
|
|
323
|
+
let baseCacheKey;
|
|
324
|
+
if (seedFrom !== undefined && turn.createdCache) {
|
|
325
|
+
if (modelConfig.cache_share_across_keys !== true) {
|
|
326
|
+
throw new KvCacheSeedSharingDisabledError(modelId);
|
|
327
|
+
}
|
|
328
|
+
const sourcePath = resolveCacheFilePath(modelId, configHash, seedFrom);
|
|
329
|
+
try {
|
|
330
|
+
await fs.promises.access(sourcePath);
|
|
331
|
+
}
|
|
332
|
+
catch (error) {
|
|
333
|
+
// Only a missing file is "not found"; a permission or I/O error keeps its cause.
|
|
334
|
+
if (error.code !== 'ENOENT')
|
|
335
|
+
throw error;
|
|
336
|
+
throw new KvCacheSeedSourceNotFoundError(seedFrom, modelId);
|
|
337
|
+
}
|
|
338
|
+
baseCacheKey = sourcePath;
|
|
339
|
+
}
|
|
340
|
+
else if (seedFrom !== undefined) {
|
|
341
|
+
requestLogger.debug(`[kv-cache] seedFrom ignored: "${kvCache}" already has a cache`);
|
|
342
|
+
}
|
|
283
343
|
logMessagesToAddon(prompt, 'PROMPT_SEND');
|
|
284
|
-
const result = yield* processModelResponse(model, prompt,
|
|
344
|
+
const result = yield* processModelResponse(model, prompt, toolsToRender, mergedGenerationParams, {
|
|
345
|
+
cacheKey: turn.cachePath,
|
|
346
|
+
...(prefill !== undefined && { prefill }),
|
|
347
|
+
...(baseCacheKey !== undefined && { baseCacheKey })
|
|
348
|
+
}, dialect, setActiveResponse);
|
|
285
349
|
if (typeof kvCache === 'string') {
|
|
286
350
|
// Custom-key path: the file holds whatever the addon kept, cancelled and
|
|
287
351
|
// stopped turns included.
|
|
@@ -290,7 +354,16 @@ export async function* completion(params, opts) {
|
|
|
290
354
|
return result;
|
|
291
355
|
}
|
|
292
356
|
// Auto-cache path.
|
|
293
|
-
|
|
357
|
+
if (prefill && !signal.aborted && !result.stoppedAtContextBoundary) {
|
|
358
|
+
await saveCacheFile(model, turn.cachePath, requestLogger);
|
|
359
|
+
const cacheInfo = await getCurrentCacheInfo(modelId, configHash, history.map(toCacheMessage).map(normalizeCacheMessage), toolsActive ? toolsToRender : undefined);
|
|
360
|
+
await session.commitTurn(turn,
|
|
361
|
+
// Renaming to the held path would reacquire its non-reentrant write lock.
|
|
362
|
+
cacheInfo.cachePath === turn.cachePath
|
|
363
|
+
? { kind: 'static' }
|
|
364
|
+
: { kind: 'autoRename', targetCachePath: cacheInfo.cachePath });
|
|
365
|
+
return result;
|
|
366
|
+
}
|
|
294
367
|
// Tool-call turns: the auto-cache key is derived from
|
|
295
368
|
// `result.responseText`, which here is raw tool-call markup rather
|
|
296
369
|
// than a clean assistant message. There's no safe post-response key
|
|
@@ -320,7 +393,7 @@ export async function* completion(params, opts) {
|
|
|
320
393
|
}
|
|
321
394
|
await saveCacheFile(model, turn.cachePath, requestLogger);
|
|
322
395
|
const savedHistory = buildAutoCacheSaveHistory(history.map(toCacheMessage), result.responseText);
|
|
323
|
-
const postResponseCacheInfo = await getCurrentCacheInfo(modelId, configHash, savedHistory);
|
|
396
|
+
const postResponseCacheInfo = await getCurrentCacheInfo(modelId, configHash, savedHistory, toolsActive ? toolsToRender : undefined);
|
|
324
397
|
await session.commitTurn(turn, {
|
|
325
398
|
kind: 'autoRename',
|
|
326
399
|
targetCachePath: postResponseCacheInfo.cachePath
|
|
@@ -1,10 +1,11 @@
|
|
|
1
|
+
import { type PluginHandlerContext } from '../../../../schemas/plugin.js';
|
|
1
2
|
import type { FinetuneRunParams, FinetuneRunRequest, FinetuneProgress, FinetuneRequest, FinetuneResult, FinetuneStatus, FinetuneGetStateRequest } from '../../../../schemas/index.js';
|
|
2
3
|
type FinetuneOptions = FinetuneRunParams['options'];
|
|
3
4
|
export declare function clearFinetuneRuntimeState(modelId: string): void;
|
|
4
5
|
export declare function getFinetuneStateFromCheckpoints(options: FinetuneOptions): FinetuneStatus;
|
|
5
|
-
export declare function startFinetune(request: FinetuneRunRequest, onProgress?: (progress: FinetuneProgress) => void): Promise<FinetuneResult>;
|
|
6
|
-
export declare function pauseFinetune(modelId: string): Promise<FinetuneResult>;
|
|
7
|
-
export declare function cancelFinetune(modelId: string): Promise<FinetuneResult>;
|
|
6
|
+
export declare function startFinetune(request: FinetuneRunRequest, onProgress?: (progress: FinetuneProgress) => void, context?: PluginHandlerContext): Promise<FinetuneResult>;
|
|
7
|
+
export declare function pauseFinetune(modelId: string, callerSessionId?: string): Promise<FinetuneResult>;
|
|
8
|
+
export declare function cancelFinetune(modelId: string, callerSessionId?: string): Promise<FinetuneResult>;
|
|
8
9
|
export declare function getFinetuneState(params: FinetuneGetStateRequest): FinetuneResult;
|
|
9
|
-
export declare function finetune(request: FinetuneRequest, onProgress?: (progress: FinetuneProgress) => void): Promise<FinetuneResult>;
|
|
10
|
+
export declare function finetune(request: FinetuneRequest, onProgress?: (progress: FinetuneProgress) => void, context?: PluginHandlerContext): Promise<FinetuneResult>;
|
|
10
11
|
export {};
|
|
@@ -50,11 +50,13 @@ var __disposeResources = (this && this.__disposeResources) || (function (Suppres
|
|
|
50
50
|
var e = new Error(message);
|
|
51
51
|
return e.name = "SuppressedError", e.error = error, e.suppressed = suppressed, e;
|
|
52
52
|
});
|
|
53
|
+
import { requestParent } from '../../../../schemas/plugin.js';
|
|
53
54
|
import fs from 'bare-fs';
|
|
54
55
|
import { getModel } from '../../../../runtime/model-registry.js';
|
|
55
56
|
import { CompletionFailedError } from '../../../../errors/index.js';
|
|
56
57
|
import { getRequestRegistry, withRequestContext } from '../../../../runtime/index.js';
|
|
57
58
|
import { generateRandomRequestId } from '../../../../runtime/request-id.js';
|
|
59
|
+
import { OWNER_SESSION_ID, resolveSessionId } from '../../../../runtime/session-id.js';
|
|
58
60
|
import { getEngineLogger } from '../../../../logging/index.js';
|
|
59
61
|
const PAUSE_CHECKPOINT_PREFIX = 'pause_checkpoint_step_';
|
|
60
62
|
// Ref-count, not a Set: with an exclusive writer a second finetune queues behind
|
|
@@ -94,15 +96,33 @@ const finetuneRequestIds = new Map();
|
|
|
94
96
|
// not settled. A new finetune is rejected during this window even if the runtime ref
|
|
95
97
|
// has already cleared, because handle.await() can settle before model.pause() resolves.
|
|
96
98
|
const pausingModels = new Set();
|
|
99
|
+
// A finetune may be stopped by the session that started it, or by the owner,
|
|
100
|
+
// who may stop any session's finetune. Another session's pause or cancel is a
|
|
101
|
+
// no-op, as its registry cancel would be.
|
|
102
|
+
function callerMayStop(tracked, callerSessionId) {
|
|
103
|
+
if (callerSessionId === OWNER_SESSION_ID)
|
|
104
|
+
return true;
|
|
105
|
+
return (tracked.sessionId ?? OWNER_SESSION_ID) === callerSessionId;
|
|
106
|
+
}
|
|
97
107
|
// Cancel the model's in-flight finetune by its tracked requestId (covers the
|
|
98
108
|
// registry grant→register handoff window via the cancel-before-begin tripwire),
|
|
99
|
-
// falling back to a broad cancel if the id is
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
109
|
+
// falling back to a broad cancel within the caller's own session if the id is
|
|
110
|
+
// unknown. Returns false when the finetune belongs to another session.
|
|
111
|
+
function cancelTrackedFinetune(modelId, callerSessionId) {
|
|
112
|
+
const tracked = finetuneRequestIds.get(modelId);
|
|
113
|
+
if (tracked !== undefined) {
|
|
114
|
+
if (!callerMayStop(tracked, callerSessionId))
|
|
115
|
+
return false;
|
|
116
|
+
getRequestRegistry().cancel(tracked);
|
|
117
|
+
return true;
|
|
118
|
+
}
|
|
119
|
+
if (callerSessionId === OWNER_SESSION_ID) {
|
|
120
|
+
getRequestRegistry().cancel({ modelId, kind: 'finetune', allSessions: true });
|
|
121
|
+
}
|
|
122
|
+
else {
|
|
123
|
+
getRequestRegistry().cancel({ modelId, kind: 'finetune', sessionId: callerSessionId });
|
|
124
|
+
}
|
|
125
|
+
return true;
|
|
106
126
|
}
|
|
107
127
|
export function getFinetuneStateFromCheckpoints(options) {
|
|
108
128
|
const checkpointDirectory = options.checkpointSaveDir ?? './checkpoints';
|
|
@@ -137,7 +157,7 @@ function validateExplicitFinetuneOperation(request) {
|
|
|
137
157
|
throw new CompletionFailedError(`Model "${request.modelId}" has no paused finetune checkpoint to resume`);
|
|
138
158
|
}
|
|
139
159
|
}
|
|
140
|
-
export async function startFinetune(request, onProgress) {
|
|
160
|
+
export async function startFinetune(request, onProgress, context) {
|
|
141
161
|
const env_1 = { stack: [], error: void 0, hasError: false };
|
|
142
162
|
try {
|
|
143
163
|
const model = getModel(request.modelId);
|
|
@@ -156,19 +176,25 @@ export async function startFinetune(request, onProgress) {
|
|
|
156
176
|
// observes RUNNING, not IDLE, and a concurrent start is rejected above. Released
|
|
157
177
|
// on a failed begin() or on scope unwind below.
|
|
158
178
|
registerRunningFinetune(request.modelId);
|
|
159
|
-
|
|
179
|
+
// Under an invoke the job runs as the invoke's session, the owner included,
|
|
180
|
+
// so the stop paths compare against that session and not one named inside
|
|
181
|
+
// the params.
|
|
182
|
+
const tracked = { requestId, sessionId: context ? context.sessionId : request.sessionId };
|
|
183
|
+
finetuneRequestIds.set(request.modelId, tracked);
|
|
160
184
|
// Scope the run into the registry so cancel({ requestId }) and
|
|
161
185
|
// cancel({ modelId, kind: "finetune" }) reach it; onAbort forwards to
|
|
162
186
|
// model.cancel().
|
|
163
187
|
const ctx = __addDisposableResource(env_1, await getRequestRegistry()
|
|
164
188
|
.begin({
|
|
165
189
|
requestId,
|
|
190
|
+
sessionId: request.sessionId,
|
|
191
|
+
...requestParent(context),
|
|
166
192
|
kind: 'finetune',
|
|
167
193
|
modelId: request.modelId
|
|
168
194
|
})
|
|
169
195
|
.catch((err) => {
|
|
170
196
|
clearFinetuneRuntimeState(request.modelId);
|
|
171
|
-
if (finetuneRequestIds.get(request.modelId) ===
|
|
197
|
+
if (finetuneRequestIds.get(request.modelId) === tracked) {
|
|
172
198
|
finetuneRequestIds.delete(request.modelId);
|
|
173
199
|
}
|
|
174
200
|
throw err;
|
|
@@ -178,7 +204,7 @@ export async function startFinetune(request, onProgress) {
|
|
|
178
204
|
// removes the listener first.
|
|
179
205
|
ctx.scope.defer(() => {
|
|
180
206
|
clearFinetuneRuntimeState(request.modelId);
|
|
181
|
-
if (finetuneRequestIds.get(request.modelId) ===
|
|
207
|
+
if (finetuneRequestIds.get(request.modelId) === tracked) {
|
|
182
208
|
finetuneRequestIds.delete(request.modelId);
|
|
183
209
|
}
|
|
184
210
|
});
|
|
@@ -237,10 +263,14 @@ export async function startFinetune(request, onProgress) {
|
|
|
237
263
|
await result_1;
|
|
238
264
|
}
|
|
239
265
|
}
|
|
240
|
-
export async function pauseFinetune(modelId) {
|
|
266
|
+
export async function pauseFinetune(modelId, callerSessionId = OWNER_SESSION_ID) {
|
|
241
267
|
// Validate the model is loaded on every path (matches main; getModel throws
|
|
242
268
|
// ModelNotFoundError for an unknown/unloaded model) before reporting any state.
|
|
243
269
|
const model = getModel(modelId);
|
|
270
|
+
const tracked = finetuneRequestIds.get(modelId);
|
|
271
|
+
if (tracked !== undefined && !callerMayStop(tracked, callerSessionId)) {
|
|
272
|
+
return { type: 'finetune', status: 'RUNNING' };
|
|
273
|
+
}
|
|
244
274
|
if (admittedFinetunes.has(modelId)) {
|
|
245
275
|
// The single finetune holds the model exclusively, so the addon-global pause is
|
|
246
276
|
// safe (nothing else runs on the model). Bar a new finetune until the native
|
|
@@ -260,7 +290,7 @@ export async function pauseFinetune(modelId) {
|
|
|
260
290
|
// pause would cancel that unrelated reader, so cancel the queued finetune by its
|
|
261
291
|
// own requestId. Nothing has trained, so report CANCELLED rather than a false
|
|
262
292
|
// PAUSED.
|
|
263
|
-
cancelTrackedFinetune(modelId);
|
|
293
|
+
cancelTrackedFinetune(modelId, callerSessionId);
|
|
264
294
|
return { type: 'finetune', status: 'CANCELLED' };
|
|
265
295
|
}
|
|
266
296
|
// No finetune on this model: a no-op pause, but the model was validated above.
|
|
@@ -268,8 +298,10 @@ export async function pauseFinetune(modelId) {
|
|
|
268
298
|
}
|
|
269
299
|
// Routes cancellation through the registry; the model.cancel() forward is
|
|
270
300
|
// installed by startFinetune, so never call model.cancel() here.
|
|
271
|
-
export function cancelFinetune(modelId) {
|
|
272
|
-
cancelTrackedFinetune(modelId)
|
|
301
|
+
export function cancelFinetune(modelId, callerSessionId = OWNER_SESSION_ID) {
|
|
302
|
+
if (!cancelTrackedFinetune(modelId, callerSessionId)) {
|
|
303
|
+
return Promise.resolve({ type: 'finetune', status: 'RUNNING' });
|
|
304
|
+
}
|
|
273
305
|
return Promise.resolve({
|
|
274
306
|
type: 'finetune',
|
|
275
307
|
status: 'CANCELLED'
|
|
@@ -282,18 +314,21 @@ export function getFinetuneState(params) {
|
|
|
282
314
|
status: runtimeState ? 'RUNNING' : getFinetuneStateFromCheckpoints(params.options)
|
|
283
315
|
};
|
|
284
316
|
}
|
|
285
|
-
export async function finetune(request, onProgress) {
|
|
317
|
+
export async function finetune(request, onProgress, context) {
|
|
286
318
|
if (request.operation === undefined ||
|
|
287
319
|
request.operation === 'start' ||
|
|
288
320
|
request.operation === 'resume') {
|
|
289
|
-
return startFinetune(request, onProgress);
|
|
321
|
+
return startFinetune(request, onProgress, context);
|
|
290
322
|
}
|
|
323
|
+
// Under a plugin invoke the caller is the invoke's session; the nested
|
|
324
|
+
// params name whatever the caller put there.
|
|
325
|
+
const callerSessionId = resolveSessionId(context ?? request);
|
|
291
326
|
switch (request.operation) {
|
|
292
327
|
case 'getState':
|
|
293
328
|
return getFinetuneState(request);
|
|
294
329
|
case 'pause':
|
|
295
|
-
return pauseFinetune(request.modelId);
|
|
330
|
+
return pauseFinetune(request.modelId, callerSessionId);
|
|
296
331
|
case 'cancel':
|
|
297
|
-
return cancelFinetune(request.modelId);
|
|
332
|
+
return cancelFinetune(request.modelId, callerSessionId);
|
|
298
333
|
}
|
|
299
334
|
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import { generateConfigHash } from '
|
|
1
|
+
import { generateConfigHash } from '../../../../plugins/ops/kv-cache-utils.js';
|
|
2
2
|
import { type CacheMessage } from '../../../../utils/index.js';
|
|
3
|
+
import type { Tool } from '../../../../schemas/tools.js';
|
|
3
4
|
import type { Logger } from '../../../../logging/types.js';
|
|
4
5
|
import { type AbortSignal } from 'bare-abort-controller';
|
|
5
6
|
export interface TurnHandle {
|
|
@@ -30,6 +31,8 @@ export interface BeginAutoTurnInput {
|
|
|
30
31
|
configHash: string;
|
|
31
32
|
/** Conversation history used to compute the pre-response cache key. */
|
|
32
33
|
history: CacheMessage[];
|
|
34
|
+
/** Rendered tools that distinguish prefixes with no assistant message. */
|
|
35
|
+
tools?: readonly Tool[];
|
|
33
36
|
/** See `BeginCustomTurnInput.signal`. */
|
|
34
37
|
signal?: AbortSignal;
|
|
35
38
|
}
|
|
@@ -40,8 +43,7 @@ export interface StaticCommitResult {
|
|
|
40
43
|
export interface AutoRenameCommitResult {
|
|
41
44
|
kind: 'autoRename';
|
|
42
45
|
/**
|
|
43
|
-
* Destination path the
|
|
44
|
-
* renamed to (computed from `cacheMessages + responseText`).
|
|
46
|
+
* Destination path for the history covered by the saved cache.
|
|
45
47
|
*/
|
|
46
48
|
targetCachePath: string;
|
|
47
49
|
}
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { promises as fsPromises } from 'bare-fs';
|
|
2
2
|
import path from 'bare-path';
|
|
3
|
-
import { findMatchingCache, generateConfigHash, getCacheFilePath, getCurrentCacheInfo, pruneEmptyCacheDirectories, renameCacheFile, deleteCache as deleteCacheUtil } from '
|
|
4
|
-
import { isCachePathWithinDirectory, markAutoCacheKey, planAutoCacheEvictions, removeAutoCacheMarker, removeAutoCacheMarkerIfMissing } from '
|
|
3
|
+
import { findMatchingCache, generateConfigHash, getCacheFilePath, getCurrentCacheInfo, pruneEmptyCacheDirectories, renameCacheFile, deleteCache as deleteCacheUtil } from '../../../../plugins/ops/kv-cache-utils.js';
|
|
4
|
+
import { isCachePathWithinDirectory, markAutoCacheKey, planAutoCacheEvictions, removeAutoCacheMarker, removeAutoCacheMarkerIfMissing } from '../../../../plugins/ops/kv-cache-retention.js';
|
|
5
5
|
import { isMobile } from '../../../../runtime/state.js';
|
|
6
6
|
import { getKVCacheDir } from '../../../../utils/index.js';
|
|
7
|
-
import { logCacheSaveError, logCacheStatus } from '
|
|
7
|
+
import { logCacheSaveError, logCacheStatus } from '../../../../plugins/builtin/llamacpp-completion/ops/cache-logger.js';
|
|
8
8
|
import { getEngineLogger } from '../../../../logging/index.js';
|
|
9
9
|
// Used by cross-model paths that have no `RequestContext` (e.g.
|
|
10
10
|
// `deleteKvCacheState`). Per-session call sites receive a logger from
|
|
@@ -323,8 +323,9 @@ export function createKvCacheSession(modelId, options) {
|
|
|
323
323
|
// the state lock, or commitTurn's rename could deadlock a waiter), then
|
|
324
324
|
// re-check existence under the lock in case a same-file peer just renamed it.
|
|
325
325
|
const discovered = await withCacheStateLock(async () => {
|
|
326
|
-
const existingCache = await findMatchingCache(modelId, input.configHash, input.history);
|
|
327
|
-
const cacheInfo = existingCache ??
|
|
326
|
+
const existingCache = await findMatchingCache(modelId, input.configHash, input.history, input.tools);
|
|
327
|
+
const cacheInfo = existingCache ??
|
|
328
|
+
(await getCurrentCacheInfo(modelId, input.configHash, input.history, input.tools));
|
|
328
329
|
return {
|
|
329
330
|
cachePath: cacheInfo.cachePath,
|
|
330
331
|
cacheKey: cacheInfo.cacheKey
|
|
@@ -414,8 +415,7 @@ export function createKvCacheSession(modelId, options) {
|
|
|
414
415
|
if (state.committed || state.rolledBack)
|
|
415
416
|
return;
|
|
416
417
|
if (result.kind === 'static') {
|
|
417
|
-
//
|
|
418
|
-
// same path. Verify the file persisted.
|
|
418
|
+
// The addon's save wrote the new cache state at the same path.
|
|
419
419
|
const ok = await verifySaveAndRecord(state.cachePath);
|
|
420
420
|
if (!ok) {
|
|
421
421
|
await runRollback(state);
|
|
@@ -424,6 +424,8 @@ export function createKvCacheSession(modelId, options) {
|
|
|
424
424
|
state.committed = true;
|
|
425
425
|
releaseCachePath(state.cachePath);
|
|
426
426
|
state.releaseWriteLock();
|
|
427
|
+
if (state.autoCacheKey !== undefined)
|
|
428
|
+
scheduleAutoCacheSweep(logger);
|
|
427
429
|
return;
|
|
428
430
|
}
|
|
429
431
|
const sourceCachePath = state.cachePath;
|