@qvac/inference 0.18.2 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +11 -2
- package/README.md +1 -5
- package/dist/api/assess-model-fit.d.ts +16 -0
- package/dist/api/assess-model-fit.js +27 -0
- package/dist/api/audio-gen.d.ts +4 -4
- package/dist/api/audio-gen.js +17 -6
- package/dist/api/completion-stream.d.ts +1 -1
- package/dist/api/completion-stream.js +1 -1
- package/dist/api/download-asset.d.ts +1 -1
- package/dist/api/download-asset.js +2 -2
- package/dist/api/finetune.js +1 -1
- package/dist/api/get-loaded-model-info.d.ts +6 -10
- package/dist/api/get-loaded-model-info.js +6 -10
- package/dist/api/heartbeat.d.ts +5 -22
- package/dist/api/heartbeat.js +6 -23
- package/dist/api/index.d.ts +36 -35
- package/dist/api/index.js +35 -35
- package/dist/api/load-model.d.ts +5 -1
- package/dist/api/load-model.js +3 -5
- package/dist/api/logging-stream-registry.js +1 -1
- package/dist/api/resume.d.ts +0 -3
- package/dist/api/resume.js +0 -3
- package/dist/api/subscribe-logs.js +1 -1
- package/dist/api/suspend.d.ts +0 -4
- package/dist/api/suspend.js +0 -4
- package/dist/api/translate.d.ts +3 -1
- package/dist/api/translate.js +16 -8
- package/dist/api/unload-model.js +3 -5
- package/dist/api/video-request.d.ts +207 -0
- package/dist/api/video-request.js +18 -0
- package/dist/api/video.d.ts +22 -2
- package/dist/api/video.js +25 -15
- package/dist/api/vla-helpers.d.ts +1 -2
- package/dist/api/vla-helpers.js +1 -2
- package/dist/api/vla.js +1 -1
- package/dist/api/world-result.d.ts +71 -0
- package/dist/api/world-result.js +293 -0
- package/dist/api/world.d.ts +143 -0
- package/dist/api/world.js +71 -0
- package/dist/config/resolve-config.d.ts +1 -1
- package/dist/config/resolve-config.js +1 -1
- package/dist/constants/index.d.ts +2 -2
- package/dist/constants/index.js +2 -2
- package/dist/dispatch.js +15 -28
- package/dist/errors/index.d.ts +35 -31
- package/dist/errors/index.js +66 -56
- package/dist/handlers/assess-model-fit.d.ts +10 -0
- package/dist/handlers/assess-model-fit.js +75 -0
- package/dist/handlers/cancelHandler.js +1 -1
- package/dist/handlers/completion-orchestrate.js +1 -1
- package/dist/handlers/download-asset.js +6 -5
- package/dist/handlers/finetune.js +1 -1
- package/dist/handlers/get-loaded-model-info.js +0 -12
- package/dist/handlers/get-model-info.js +1 -1
- package/dist/handlers/get-system-resources.js +8 -0
- package/dist/handlers/load-model/download-manager.d.ts +1 -1
- package/dist/handlers/load-model/download-stats.d.ts +3 -3
- package/dist/handlers/load-model/download-stats.js +5 -5
- package/dist/handlers/load-model/handler.js +6 -8
- package/dist/handlers/load-model/http-verify.d.ts +20 -0
- package/dist/handlers/load-model/http-verify.js +54 -0
- package/dist/handlers/load-model/http.d.ts +12 -2
- package/dist/handlers/load-model/http.js +119 -230
- package/dist/handlers/load-model/hyperdrive.d.ts +1 -1
- package/dist/handlers/load-model/hyperdrive.js +1 -1
- package/dist/handlers/load-model/index.d.ts +2 -2
- package/dist/handlers/load-model/index.js +2 -2
- package/dist/handlers/load-model/registry-companion-set.d.ts +1 -1
- package/dist/handlers/load-model/registry-companion-set.js +1 -1
- package/dist/handlers/load-model/registry-download-utils.d.ts +3 -3
- package/dist/handlers/load-model/registry-download-utils.js +2 -2
- package/dist/handlers/load-model/registry.d.ts +2 -2
- package/dist/handlers/load-model/registry.js +4 -4
- package/dist/handlers/load-model/resolve-session.d.ts +8 -1
- package/dist/handlers/load-model/resolve-session.js +12 -9
- package/dist/handlers/load-model/resolve.d.ts +4 -4
- package/dist/handlers/load-model/resolve.js +18 -16
- package/dist/handlers/load-model/safe-fetch.d.ts +44 -0
- package/dist/handlers/load-model/safe-fetch.js +143 -0
- package/dist/handlers/load-model/types.d.ts +9 -0
- package/dist/handlers/plugin-dispatch.js +1 -4
- package/dist/handlers/plugin-invoke.js +1 -4
- package/dist/handlers/types.d.ts +0 -2
- package/dist/handlers/unload-model.js +1 -4
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/logging/addon.d.ts +1 -1
- package/dist/logging/addon.js +1 -1
- package/dist/logging/index.d.ts +5 -5
- package/dist/logging/index.js +4 -4
- package/dist/logging/logger.d.ts +1 -1
- package/dist/logging/logger.js +2 -2
- package/dist/models/registry/index.d.ts +1 -1
- package/dist/models/registry/index.js +2 -2
- package/dist/models/registry/models.d.ts +250 -0
- package/dist/models/registry/models.js +2071 -1806
- package/dist/models/registry/resource-profiles.d.ts +18 -0
- package/dist/models/registry/resource-profiles.js +4934 -0
- package/dist/plugins/builtin/audiogen-ggml/config.d.ts +14 -0
- package/dist/plugins/builtin/audiogen-ggml/config.js +23 -0
- package/dist/plugins/builtin/audiogen-ggml/ops/audio-gen-stream.js +47 -5
- package/dist/plugins/builtin/audiogen-ggml/plugin.d.ts +134 -4
- package/dist/plugins/builtin/audiogen-ggml/plugin.js +47 -28
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.d.ts +1 -1
- package/dist/plugins/builtin/bci-whispercpp-transcription/plugin.js +3 -3
- package/dist/plugins/builtin/ggml-classification/plugin.js +1 -1
- package/dist/plugins/builtin/ggml-ocr/plugin.js +2 -2
- package/dist/plugins/builtin/ggml-vla/plugin.d.ts +1 -1
- package/dist/plugins/builtin/ggml-vla/plugin.js +3 -3
- package/dist/plugins/builtin/index.d.ts +12 -12
- package/dist/plugins/builtin/index.js +12 -12
- package/dist/plugins/builtin/llamacpp-completion/ops/batch-completion-stream.js +2 -2
- package/dist/plugins/builtin/llamacpp-completion/ops/completion-stream.js +37 -26
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.d.ts +12 -11
- package/dist/plugins/builtin/llamacpp-completion/ops/context-overflow.js +133 -41
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.d.ts +7 -1
- package/dist/plugins/builtin/llamacpp-completion/ops/kv-cache-session.js +26 -4
- package/dist/plugins/builtin/llamacpp-completion/plugin.d.ts +19 -8
- package/dist/plugins/builtin/llamacpp-completion/plugin.js +10 -13
- package/dist/plugins/builtin/llamacpp-embedding/plugin.d.ts +4 -0
- package/dist/plugins/builtin/llamacpp-embedding/plugin.js +13 -2
- package/dist/plugins/builtin/nmtcpp-translation/plugin.js +2 -2
- package/dist/plugins/builtin/parakeet-transcription/plugin.d.ts +2 -2
- package/dist/plugins/builtin/parakeet-transcription/plugin.js +3 -3
- package/dist/plugins/builtin/sdcpp-generation/ops/upscale.js +1 -1
- package/dist/plugins/builtin/sdcpp-generation/ops/video.js +32 -2
- package/dist/plugins/builtin/sdcpp-generation/ops/world.d.ts +126 -0
- package/dist/plugins/builtin/sdcpp-generation/ops/world.js +820 -0
- package/dist/plugins/builtin/sdcpp-generation/plugin.d.ts +153 -4
- package/dist/plugins/builtin/sdcpp-generation/plugin.js +134 -13
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech-stream.js +1 -1
- package/dist/plugins/builtin/tts-ggml/ops/text-to-speech.js +1 -1
- package/dist/plugins/builtin/tts-ggml/plugin.js +2 -2
- package/dist/plugins/builtin/whispercpp-transcription/plugin.d.ts +2 -2
- package/dist/plugins/builtin/whispercpp-transcription/plugin.js +3 -3
- package/dist/plugins/index.d.ts +1 -1
- package/dist/plugins/index.js +2 -2
- package/dist/plugins/ops/kv-cache-utils.js +1 -1
- package/dist/plugins/ops/load-model.js +1 -1
- package/dist/plugins/ops/transcribe.js +2 -2
- package/dist/plugins/ops/translate.js +4 -12
- package/dist/plugins/ops/unload-model.js +19 -21
- package/dist/plugins/registry.d.ts +1 -0
- package/dist/plugins/registry.js +12 -0
- package/dist/profiling/aggregator.d.ts +1 -1
- package/dist/profiling/aggregator.js +1 -1
- package/dist/profiling/controller.d.ts +1 -1
- package/dist/profiling/controller.js +2 -2
- package/dist/profiling/envelope.d.ts +0 -1
- package/dist/profiling/envelope.js +0 -10
- package/dist/profiling/events.d.ts +2 -3
- package/dist/profiling/events.js +2 -9
- package/dist/profiling/exporters.d.ts +1 -1
- package/dist/profiling/exporters.js +3 -3
- package/dist/profiling/index.d.ts +13 -14
- package/dist/profiling/index.js +12 -13
- package/dist/profiling/operation-metrics.d.ts +1 -1
- package/dist/profiling/operation-metrics.js +35 -2
- package/dist/profiling/operation-wrappers.js +12 -15
- package/dist/profiling/profiler.d.ts +1 -2
- package/dist/profiling/profiler.js +7 -26
- package/dist/profiling/types.d.ts +1 -1
- package/dist/rag/handler.js +1 -1
- package/dist/rag/handlers/close-workspace.js +1 -1
- package/dist/rag/handlers/delete-embeddings.js +1 -1
- package/dist/rag/handlers/delete-workspace.js +2 -6
- package/dist/rag/handlers/ingest.js +1 -1
- package/dist/rag/handlers/list-workspaces.d.ts +1 -1
- package/dist/rag/handlers/list-workspaces.js +1 -1
- package/dist/rag/handlers/reindex.js +1 -1
- package/dist/rag/handlers/save-embeddings.js +1 -1
- package/dist/rag/handlers/search.js +1 -1
- package/dist/rag/index.d.ts +11 -11
- package/dist/rag/index.js +11 -11
- package/dist/rag/rag-workspace-manager.d.ts +4 -2
- package/dist/rag/rag-workspace-manager.js +268 -15
- package/dist/registry.js +8 -58
- package/dist/resources/collector.d.ts +3 -1
- package/dist/resources/collector.js +91 -9
- package/dist/resources/instance.d.ts +1 -1
- package/dist/resources/instance.js +1 -1
- package/dist/resources/model-fit/assess.d.ts +36 -0
- package/dist/resources/model-fit/assess.js +578 -0
- package/dist/resources/model-fit/calibration/darwin-arm64.d.ts +8 -0
- package/dist/resources/model-fit/calibration/darwin-arm64.js +36 -0
- package/dist/resources/model-fit/calibration/darwin-x64.d.ts +8 -0
- package/dist/resources/model-fit/calibration/darwin-x64.js +38 -0
- package/dist/resources/model-fit/calibration/fit.d.ts +73 -0
- package/dist/resources/model-fit/calibration/fit.js +152 -0
- package/dist/resources/model-fit/calibration/index.d.ts +37 -0
- package/dist/resources/model-fit/calibration/index.js +75 -0
- package/dist/resources/model-fit/calibration/linux-arm64.d.ts +8 -0
- package/dist/resources/model-fit/calibration/linux-arm64.js +38 -0
- package/dist/resources/model-fit/calibration/linux-x64-vulkan.d.ts +8 -0
- package/dist/resources/model-fit/calibration/linux-x64-vulkan.js +39 -0
- package/dist/resources/model-fit/calibration/linux-x64.d.ts +8 -0
- package/dist/resources/model-fit/calibration/linux-x64.js +38 -0
- package/dist/resources/model-fit/calibration/win32-x64-vulkan-shared.d.ts +8 -0
- package/dist/resources/model-fit/calibration/win32-x64-vulkan-shared.js +39 -0
- package/dist/resources/model-fit/calibration/win32-x64-vulkan.d.ts +8 -0
- package/dist/resources/model-fit/calibration/win32-x64-vulkan.js +36 -0
- package/dist/resources/model-fit/calibration/win32-x64.d.ts +8 -0
- package/dist/resources/model-fit/calibration/win32-x64.js +38 -0
- package/dist/resources/model-fit/estimators/llm.d.ts +45 -0
- package/dist/resources/model-fit/estimators/llm.js +220 -0
- package/dist/resources/model-fit/estimators/whisper.d.ts +10 -0
- package/dist/resources/model-fit/estimators/whisper.js +77 -0
- package/dist/resources/model-fit/types.d.ts +120 -0
- package/dist/resources/model-fit/types.js +1 -0
- package/dist/resources/native.d.ts +1 -1
- package/dist/resources/native.js +14 -0
- package/dist/resources/normalize.d.ts +1 -1
- package/dist/resources/types.d.ts +16 -0
- package/dist/runtime/index.d.ts +5 -5
- package/dist/runtime/index.js +2 -2
- package/dist/runtime/lifecycle.js +5 -7
- package/dist/runtime/model-registry.d.ts +0 -14
- package/dist/runtime/model-registry.js +17 -40
- package/dist/runtime/registry-client.js +1 -1
- package/dist/runtime/request-context.d.ts +3 -3
- package/dist/runtime/request-context.js +13 -1
- package/dist/runtime/request-registry.d.ts +1 -1
- package/dist/runtime/request-registry.js +4 -5
- package/dist/runtime/state.d.ts +1 -1
- package/dist/runtime/state.js +18 -3
- package/dist/schemas/assess-model-fit.d.ts +271 -0
- package/dist/schemas/assess-model-fit.js +153 -0
- package/dist/schemas/audio-gen.d.ts +434 -4
- package/dist/schemas/audio-gen.js +131 -19
- package/dist/schemas/batch-completion-stream.d.ts +2 -2
- package/dist/schemas/batch-completion-stream.js +3 -3
- package/dist/schemas/bci-config.js +73 -27
- package/dist/schemas/bci.d.ts +2 -2
- package/dist/schemas/bci.js +5 -5
- package/dist/schemas/cancel.d.ts +5 -0
- package/dist/schemas/cancel.js +1 -0
- package/dist/schemas/classification.js +13 -6
- package/dist/schemas/common.d.ts +662 -331
- package/dist/schemas/common.js +33 -31
- package/dist/schemas/completion-event.d.ts +2 -2
- package/dist/schemas/completion-event.js +1 -1
- package/dist/schemas/completion-stream.d.ts +1 -5
- package/dist/schemas/completion-stream.js +4 -8
- package/dist/schemas/config.d.ts +640 -18
- package/dist/schemas/config.js +50 -8
- package/dist/schemas/download-asset.d.ts +19 -1
- package/dist/schemas/download-asset.js +20 -4
- package/dist/schemas/engine-addon-map.d.ts +1 -1
- package/dist/schemas/engine-addon-map.js +3 -3
- package/dist/schemas/error.d.ts +2 -2
- package/dist/schemas/error.js +2 -2
- package/dist/schemas/errors.d.ts +2 -6
- package/dist/schemas/errors.js +30 -40
- package/dist/schemas/heartbeat.d.ts +10 -0
- package/dist/schemas/heartbeat.js +8 -0
- package/dist/schemas/index.d.ts +46 -44
- package/dist/schemas/index.js +45 -43
- package/dist/schemas/llamacpp-config.d.ts +32 -10
- package/dist/schemas/llamacpp-config.js +11 -8
- package/dist/schemas/load-model.d.ts +2933 -1948
- package/dist/schemas/load-model.js +52 -52
- package/dist/schemas/mcp-adapter.d.ts +1 -1
- package/dist/schemas/model-config-schemas.d.ts +2354 -0
- package/dist/schemas/model-config-schemas.js +46 -0
- package/dist/schemas/model-info.d.ts +10 -20
- package/dist/schemas/model-info.js +4 -25
- package/dist/schemas/model-ops.d.ts +0 -1
- package/dist/schemas/model-ops.js +1 -2
- package/dist/schemas/model-resource-profile.d.ts +109 -0
- package/dist/schemas/model-resource-profile.js +83 -0
- package/dist/schemas/model-src-utils.d.ts +1 -0
- package/dist/schemas/model-src-utils.js +33 -13
- package/dist/schemas/ocr.js +48 -13
- package/dist/schemas/path-validation.js +8 -2
- package/dist/schemas/plugin.d.ts +14 -1
- package/dist/schemas/plugin.js +11 -0
- package/dist/schemas/profiling.d.ts +138 -43
- package/dist/schemas/profiling.js +1 -22
- package/dist/schemas/rag.js +1 -1
- package/dist/schemas/registry.js +1 -1
- package/dist/schemas/reload-config.d.ts +0 -2
- package/dist/schemas/reload-config.js +2 -3
- package/dist/schemas/sdcpp-config.d.ts +344 -4
- package/dist/schemas/sdcpp-config.js +504 -17
- package/dist/schemas/system-resources.d.ts +138 -0
- package/dist/schemas/system-resources.js +3 -1
- package/dist/schemas/text-to-speech.js +183 -76
- package/dist/schemas/transcription-config.js +253 -85
- package/dist/schemas/transcription.d.ts +3 -3
- package/dist/schemas/translate.d.ts +2 -2
- package/dist/schemas/translate.js +1 -1
- package/dist/schemas/translation-config.js +53 -25
- package/dist/schemas/vla.d.ts +1 -1
- package/dist/selection.d.ts +1 -5
- package/dist/selection.js +2 -6
- package/dist/surface.d.ts +1 -1
- package/dist/utils/aggregate-events.d.ts +1 -1
- package/dist/utils/aggregate-events.js +2 -2
- package/dist/utils/archive.js +2 -2
- package/dist/utils/audio-input.js +1 -1
- package/dist/utils/cache/auto-history.d.ts +1 -1
- package/dist/utils/cache/auto-history.js +1 -1
- package/dist/utils/cache/paths.js +5 -5
- package/dist/utils/completion-normalizer.js +1 -1
- package/dist/utils/gguf-tensor-extractor.js +1 -1
- package/dist/utils/image-dimensions.d.ts +26 -0
- package/dist/utils/image-dimensions.js +46 -0
- package/dist/utils/index.d.ts +13 -10
- package/dist/utils/index.js +13 -10
- package/dist/utils/mcp-adapter.d.ts +1 -1
- package/dist/utils/mcp-adapter.js +1 -1
- package/dist/utils/path-sanitize.d.ts +7 -0
- package/dist/utils/path-sanitize.js +36 -0
- package/dist/utils/path-security.d.ts +0 -7
- package/dist/utils/path-security.js +1 -36
- package/dist/utils/shard-utils.js +3 -3
- package/dist/utils/tool-integration.js +1 -1
- package/dist/utils/tools/dialect.js +1 -1
- package/dist/utils/tools/index.d.ts +2 -2
- package/dist/utils/tools/index.js +2 -2
- package/dist/utils/tools/parser.js +8 -8
- package/dist/utils/tools/parsers/dsml.d.ts +1 -1
- package/dist/utils/tools/parsers/dsml.js +1 -1
- package/dist/utils/tools/parsers/gemma4native.d.ts +1 -1
- package/dist/utils/tools/parsers/gemma4native.js +1 -1
- package/dist/utils/tools/parsers/harmony.d.ts +1 -1
- package/dist/utils/tools/parsers/harmony.js +1 -1
- package/dist/utils/tools/parsers/hermes.d.ts +1 -1
- package/dist/utils/tools/parsers/hermes.js +1 -1
- package/dist/utils/tools/parsers/json.d.ts +1 -1
- package/dist/utils/tools/parsers/json.js +1 -1
- package/dist/utils/tools/parsers/pythonic.d.ts +1 -1
- package/dist/utils/tools/parsers/pythonic.js +1 -1
- package/dist/utils/tools/parsers/qwen35.d.ts +1 -1
- package/dist/utils/tools/parsers/qwen35.js +3 -3
- package/dist/utils/url-security.d.ts +17 -0
- package/dist/utils/url-security.js +75 -0
- package/package.json +31 -26
- package/dist/api/provide.d.ts +0 -23
- package/dist/api/provide.js +0 -32
- package/dist/api/stop-provider.d.ts +0 -20
- package/dist/api/stop-provider.js +0 -30
- package/dist/p2p/cancel-delegated.d.ts +0 -3
- package/dist/p2p/cancel-delegated.js +0 -64
- package/dist/p2p/completion-stream-delegated.d.ts +0 -4
- package/dist/p2p/completion-stream-delegated.js +0 -51
- package/dist/p2p/delegate-client.d.ts +0 -9
- package/dist/p2p/delegate-client.js +0 -219
- package/dist/p2p/delegate-connect-diagnostics.d.ts +0 -8
- package/dist/p2p/delegate-connect-diagnostics.js +0 -38
- package/dist/p2p/delegate-transport.d.ts +0 -18
- package/dist/p2p/delegate-transport.js +0 -237
- package/dist/p2p/error-handlers.d.ts +0 -4
- package/dist/p2p/error-handlers.js +0 -31
- package/dist/p2p/handle-request.d.ts +0 -2
- package/dist/p2p/handle-request.js +0 -171
- package/dist/p2p/handler-utils.d.ts +0 -19
- package/dist/p2p/handler-utils.js +0 -180
- package/dist/p2p/heartbeat-delegated.d.ts +0 -3
- package/dist/p2p/heartbeat-delegated.js +0 -32
- package/dist/p2p/load-model-delegated.d.ts +0 -6
- package/dist/p2p/load-model-delegated.js +0 -132
- package/dist/p2p/progress-throttle.d.ts +0 -7
- package/dist/p2p/progress-throttle.js +0 -47
- package/dist/p2p/provideHandler/connection.d.ts +0 -2
- package/dist/p2p/provideHandler/connection.js +0 -38
- package/dist/p2p/provideHandler/handler.d.ts +0 -2
- package/dist/p2p/provideHandler/handler.js +0 -66
- package/dist/p2p/provideHandler/proxy.d.ts +0 -2
- package/dist/p2p/provideHandler/proxy.js +0 -24
- package/dist/p2p/provideHandler.d.ts +0 -1
- package/dist/p2p/provideHandler.js +0 -2
- package/dist/p2p/rpc-utils.d.ts +0 -4
- package/dist/p2p/rpc-utils.js +0 -8
- package/dist/p2p/stopProvideHandler.d.ts +0 -2
- package/dist/p2p/stopProvideHandler.js +0 -29
- package/dist/p2p/swarm.d.ts +0 -9
- package/dist/p2p/swarm.js +0 -103
- package/dist/p2p/transport-selector.d.ts +0 -2
- package/dist/p2p/transport-selector.js +0 -18
- package/dist/p2p/unload-model-delegated.d.ts +0 -2
- package/dist/p2p/unload-model-delegated.js +0 -31
- package/dist/profiling/delegation-profiler.d.ts +0 -31
- package/dist/profiling/delegation-profiler.js +0 -155
- package/dist/schemas/delegate.d.ts +0 -29
- package/dist/schemas/delegate.js +0 -40
- package/dist/schemas/provide.d.ts +0 -50
- package/dist/schemas/provide.js +0 -32
|
@@ -0,0 +1,293 @@
|
|
|
1
|
+
import { MAX_SCENE_IMAGE_BYTES, walkKeySchema, worldSceneStreamRequestSchema, worldSceneStreamResponseSchema, worldStepStreamRequestSchema, worldStepStreamResponseSchema } from '../schemas/sdcpp-config.js';
|
|
2
|
+
import { parseClientInput } from './parse-input.js';
|
|
3
|
+
import { generateRequestId } from '../runtime/request-id.js';
|
|
4
|
+
import { decodeBase64, encodeBase64 } from '../utils/encoding.js';
|
|
5
|
+
import { InvalidResponseError, RequestValidationFailedError, StreamEndedError, InferenceCancelledError } from '../errors/index.js';
|
|
6
|
+
import { ERROR_CODES } from '../schemas/errors.js';
|
|
7
|
+
/** Bit order the native action adapter expects: bit 0..7 = W,A,S,D,I,J,K,L. */
|
|
8
|
+
const KEY_ORDER = ['W', 'A', 'S', 'D', 'I', 'J', 'K', 'L'];
|
|
9
|
+
/**
|
|
10
|
+
* Normalizes the three accepted key forms to the array the wire schema takes.
|
|
11
|
+
*
|
|
12
|
+
* A raw mask is accepted because the native session is ultimately driven by
|
|
13
|
+
* one, and a key-state object is accepted because that is what a keyboard
|
|
14
|
+
* handler naturally holds — neither should force the caller to convert.
|
|
15
|
+
*/
|
|
16
|
+
export function toWalkKeys(keys) {
|
|
17
|
+
if (keys === undefined)
|
|
18
|
+
return [];
|
|
19
|
+
if (typeof keys === 'number') {
|
|
20
|
+
if (!Number.isInteger(keys) || keys < 0 || keys > 255) {
|
|
21
|
+
throw new RequestValidationFailedError(`walk key mask must be an integer in [0, 255], got: ${keys}`);
|
|
22
|
+
}
|
|
23
|
+
return KEY_ORDER.filter((_, bit) => (keys & (1 << bit)) !== 0);
|
|
24
|
+
}
|
|
25
|
+
const pressed = Array.isArray(keys)
|
|
26
|
+
? keys.map((key) => String(key))
|
|
27
|
+
: Object.entries(keys)
|
|
28
|
+
.filter(([, held]) => Boolean(held))
|
|
29
|
+
.map(([key]) => key);
|
|
30
|
+
const seen = new Set();
|
|
31
|
+
for (const key of pressed) {
|
|
32
|
+
const parsed = walkKeySchema.safeParse(key.toUpperCase());
|
|
33
|
+
if (!parsed.success) {
|
|
34
|
+
throw new RequestValidationFailedError(`unknown walk key '${key}' (valid: ${KEY_ORDER.join(', ')})`);
|
|
35
|
+
}
|
|
36
|
+
seen.add(parsed.data);
|
|
37
|
+
}
|
|
38
|
+
// Emit in bit order so an identical set of keys always produces an identical
|
|
39
|
+
// request, whatever order the caller supplied them in.
|
|
40
|
+
return KEY_ORDER.filter((key) => seen.has(key));
|
|
41
|
+
}
|
|
42
|
+
function deferred() {
|
|
43
|
+
let resolve = () => { };
|
|
44
|
+
let reject = () => { };
|
|
45
|
+
let settled = false;
|
|
46
|
+
const promise = new Promise((res, rej) => {
|
|
47
|
+
resolve = (value) => {
|
|
48
|
+
settled = true;
|
|
49
|
+
res(value);
|
|
50
|
+
};
|
|
51
|
+
reject = (error) => {
|
|
52
|
+
settled = true;
|
|
53
|
+
rej(error);
|
|
54
|
+
};
|
|
55
|
+
});
|
|
56
|
+
// Nothing else settles these, so an unobserved rejection must not surface as
|
|
57
|
+
// an unhandled rejection just because the caller only awaited one of them.
|
|
58
|
+
promise.catch(() => { });
|
|
59
|
+
return { promise, resolve, reject, isSettled: () => settled };
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* A stream that ends without a `done` frame has to fail the result promises
|
|
63
|
+
* explicitly. Nothing else settles them, so leaving them pending turns a
|
|
64
|
+
* dropped connection or a truncated stream into an `await` that never returns —
|
|
65
|
+
* a hang the caller cannot distinguish from slow generation, and cannot time
|
|
66
|
+
* out because no error is ever delivered.
|
|
67
|
+
*
|
|
68
|
+
* `StreamEndedError` rather than a bare `Error`, matching `upscale.ts`, so a
|
|
69
|
+
* caller can `instanceof` a dropped walk instead of matching on message text.
|
|
70
|
+
*/
|
|
71
|
+
function truncatedStream() {
|
|
72
|
+
return new StreamEndedError();
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* The world ops throw `InferenceCancelledError` on the server, so it crosses the
|
|
76
|
+
* RPC envelope and arrives as a generic `RPCError` — `instanceof` fails and a
|
|
77
|
+
* caller cannot tell a cancelled walk from a failed one. Rebuild it here rather
|
|
78
|
+
* than registering a reconstructor in `client/rpc/rpc-error.ts`: that file's
|
|
79
|
+
* maintenance contract keeps client-constructed typed errors out of the
|
|
80
|
+
* reconstructor map, because a global entry would also fire for the completion
|
|
81
|
+
* path, which builds this same class from its aggregated partial state.
|
|
82
|
+
*/
|
|
83
|
+
function asClientError(error, requestId) {
|
|
84
|
+
if (error instanceof InferenceCancelledError)
|
|
85
|
+
return error;
|
|
86
|
+
const code = error?.code;
|
|
87
|
+
if (code === ERROR_CODES.INFERENCE_CANCELLED) {
|
|
88
|
+
return new InferenceCancelledError(requestId, {}, error);
|
|
89
|
+
}
|
|
90
|
+
return error instanceof Error ? error : new Error(String(error));
|
|
91
|
+
}
|
|
92
|
+
export function createWorldStepResult(params, streamFactory) {
|
|
93
|
+
const requestId = generateRequestId();
|
|
94
|
+
const { keys, ...rest } = params;
|
|
95
|
+
// Validate before opening the stream so a bad key or a missing modelId fails
|
|
96
|
+
// here rather than costing a server round-trip, as audio-gen-result.ts does.
|
|
97
|
+
const request = parseClientInput(worldStepStreamRequestSchema, {
|
|
98
|
+
...rest,
|
|
99
|
+
keys: toWalkKeys(keys),
|
|
100
|
+
type: 'worldStepStream',
|
|
101
|
+
requestId
|
|
102
|
+
});
|
|
103
|
+
const collected = [];
|
|
104
|
+
const pending = [];
|
|
105
|
+
// Its own queue and its own waker: a progress tick must not wake the frame
|
|
106
|
+
// generator into an empty buffer, and a consumer may iterate either stream
|
|
107
|
+
// alone.
|
|
108
|
+
const progressPending = [];
|
|
109
|
+
let done = false;
|
|
110
|
+
// Set by the terminal `done` frame; later chunks for the same request are
|
|
111
|
+
// ignored rather than appended to a block that has already been delivered.
|
|
112
|
+
let terminated = false;
|
|
113
|
+
let streamError = null;
|
|
114
|
+
let wake = null;
|
|
115
|
+
let progressWake = null;
|
|
116
|
+
const framesOut = deferred();
|
|
117
|
+
const statsOut = deferred();
|
|
118
|
+
async function pump() {
|
|
119
|
+
try {
|
|
120
|
+
for await (const response of streamFactory(request)) {
|
|
121
|
+
if (response === null ||
|
|
122
|
+
typeof response !== 'object' ||
|
|
123
|
+
!('type' in response) ||
|
|
124
|
+
response.type !== 'worldStepStream') {
|
|
125
|
+
continue;
|
|
126
|
+
}
|
|
127
|
+
const parsed = worldStepStreamResponseSchema.parse(response);
|
|
128
|
+
// lunte-disable-next-line eqeqeq -- `!= null` intentionally matches null and undefined
|
|
129
|
+
if (parsed.step != null && parsed.totalSteps != null && parsed.elapsedMs != null) {
|
|
130
|
+
progressPending.push({
|
|
131
|
+
step: parsed.step,
|
|
132
|
+
totalSteps: parsed.totalSteps,
|
|
133
|
+
elapsedMs: parsed.elapsedMs
|
|
134
|
+
});
|
|
135
|
+
progressWake?.();
|
|
136
|
+
progressWake = null;
|
|
137
|
+
}
|
|
138
|
+
// `done` is terminal for this block. Anything after it is not part of
|
|
139
|
+
// the block the caller was handed, and accepting it would let
|
|
140
|
+
// `frameStream` and the `frames` promise disagree about the same block:
|
|
141
|
+
// `frames` is already resolved with what `collected` held at `done`,
|
|
142
|
+
// while the generator would keep yielding.
|
|
143
|
+
if (terminated)
|
|
144
|
+
continue;
|
|
145
|
+
if (parsed.data) {
|
|
146
|
+
const frame = decodeBase64(parsed.data);
|
|
147
|
+
collected.push(frame);
|
|
148
|
+
pending.push(frame);
|
|
149
|
+
wake?.();
|
|
150
|
+
wake = null;
|
|
151
|
+
}
|
|
152
|
+
if (parsed.done) {
|
|
153
|
+
terminated = true;
|
|
154
|
+
statsOut.resolve(parsed.stats);
|
|
155
|
+
framesOut.resolve(collected);
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
catch (error) {
|
|
160
|
+
streamError = asClientError(error, requestId);
|
|
161
|
+
statsOut.reject(streamError);
|
|
162
|
+
framesOut.reject(streamError);
|
|
163
|
+
}
|
|
164
|
+
if (!framesOut.isSettled()) {
|
|
165
|
+
const truncated = streamError ?? truncatedStream();
|
|
166
|
+
streamError = truncated;
|
|
167
|
+
statsOut.reject(truncated);
|
|
168
|
+
framesOut.reject(truncated);
|
|
169
|
+
}
|
|
170
|
+
done = true;
|
|
171
|
+
wake?.();
|
|
172
|
+
wake = null;
|
|
173
|
+
progressWake?.();
|
|
174
|
+
progressWake = null;
|
|
175
|
+
}
|
|
176
|
+
void pump();
|
|
177
|
+
const progressStream = (async function* () {
|
|
178
|
+
while (true) {
|
|
179
|
+
if (progressPending.length > 0) {
|
|
180
|
+
yield progressPending.shift();
|
|
181
|
+
}
|
|
182
|
+
else if (done) {
|
|
183
|
+
if (streamError)
|
|
184
|
+
throw streamError;
|
|
185
|
+
return;
|
|
186
|
+
}
|
|
187
|
+
else {
|
|
188
|
+
await new Promise((resolve) => {
|
|
189
|
+
progressWake = resolve;
|
|
190
|
+
});
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
})();
|
|
194
|
+
const frameStream = (async function* () {
|
|
195
|
+
while (true) {
|
|
196
|
+
if (pending.length > 0) {
|
|
197
|
+
yield pending.shift();
|
|
198
|
+
}
|
|
199
|
+
else if (done) {
|
|
200
|
+
// The cast satisfies @typescript-eslint/only-throw-error, which does not
|
|
201
|
+
// accept the `Error | null` declared type as throwable even though the
|
|
202
|
+
// guard narrows it. Same shape as client/api/video.ts.
|
|
203
|
+
if (streamError)
|
|
204
|
+
throw streamError;
|
|
205
|
+
return;
|
|
206
|
+
}
|
|
207
|
+
else {
|
|
208
|
+
await new Promise((resolve) => {
|
|
209
|
+
wake = resolve;
|
|
210
|
+
});
|
|
211
|
+
}
|
|
212
|
+
}
|
|
213
|
+
})();
|
|
214
|
+
return {
|
|
215
|
+
requestId,
|
|
216
|
+
frameStream,
|
|
217
|
+
progressStream,
|
|
218
|
+
frames: framesOut.promise,
|
|
219
|
+
stats: statsOut.promise
|
|
220
|
+
};
|
|
221
|
+
}
|
|
222
|
+
export function createWorldSceneResult(params, streamFactory) {
|
|
223
|
+
const requestId = generateRequestId();
|
|
224
|
+
const { image, ...rest } = params;
|
|
225
|
+
// Checked on the RAW bytes, before base64. The schema has a `.max()` on the
|
|
226
|
+
// encoded string, but it cannot be relied on to produce the error: Zod runs
|
|
227
|
+
// every check on a string regardless of order, so the base64 pattern runs too,
|
|
228
|
+
// and on V8 that pattern THROWS `RangeError: Maximum call stack size exceeded`
|
|
229
|
+
// rather than returning false once the input passes roughly 4.5M characters.
|
|
230
|
+
// `parseClientInput` only converts `ZodError`, so the caller would get a raw
|
|
231
|
+
// RangeError instead of the typed validation error this API promises.
|
|
232
|
+
if (image.byteLength > MAX_SCENE_IMAGE_BYTES) {
|
|
233
|
+
throw new RequestValidationFailedError(`The first-frame image is ${image.byteLength} bytes, over the ` +
|
|
234
|
+
`${MAX_SCENE_IMAGE_BYTES}-byte ceiling. It is cover-scaled and cropped to ` +
|
|
235
|
+
'width x height, so send a smaller one.');
|
|
236
|
+
}
|
|
237
|
+
// Empty prompts and dimensions that are not multiples of 32 are rejected
|
|
238
|
+
// locally rather than after a round-trip carrying ~10 MB of image bytes.
|
|
239
|
+
const request = parseClientInput(worldSceneStreamRequestSchema, {
|
|
240
|
+
...rest,
|
|
241
|
+
image: encodeBase64(image),
|
|
242
|
+
type: 'worldSceneStream',
|
|
243
|
+
requestId
|
|
244
|
+
});
|
|
245
|
+
const wantsPack = params.returnPack === true;
|
|
246
|
+
const sceneOut = deferred();
|
|
247
|
+
const statsOut = deferred();
|
|
248
|
+
async function pump() {
|
|
249
|
+
try {
|
|
250
|
+
let scene;
|
|
251
|
+
for await (const response of streamFactory(request)) {
|
|
252
|
+
if (response === null ||
|
|
253
|
+
typeof response !== 'object' ||
|
|
254
|
+
!('type' in response) ||
|
|
255
|
+
response.type !== 'worldSceneStream') {
|
|
256
|
+
continue;
|
|
257
|
+
}
|
|
258
|
+
const parsed = worldSceneStreamResponseSchema.parse(response);
|
|
259
|
+
if (parsed.data)
|
|
260
|
+
scene = decodeBase64(parsed.data);
|
|
261
|
+
if (parsed.done) {
|
|
262
|
+
statsOut.resolve(parsed.stats);
|
|
263
|
+
if (scene) {
|
|
264
|
+
sceneOut.resolve(scene);
|
|
265
|
+
}
|
|
266
|
+
else if (wantsPack) {
|
|
267
|
+
// Only a failure when the caller asked for the bytes; otherwise the
|
|
268
|
+
// server keeping them is the whole point of the default.
|
|
269
|
+
// Same typed error the other stream helpers raise for a response
|
|
270
|
+
// that is well-formed but missing what it promised.
|
|
271
|
+
sceneOut.reject(new InvalidResponseError('worldSceneStream scene pack'));
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
catch (error) {
|
|
277
|
+
const failure = asClientError(error, requestId);
|
|
278
|
+
statsOut.reject(failure);
|
|
279
|
+
sceneOut.reject(failure);
|
|
280
|
+
}
|
|
281
|
+
if (!statsOut.isSettled()) {
|
|
282
|
+
const failure = truncatedStream();
|
|
283
|
+
statsOut.reject(failure);
|
|
284
|
+
sceneOut.reject(failure);
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
void pump();
|
|
288
|
+
// `scene` is present only on the requested shape, so a caller who did not ask
|
|
289
|
+
// for the bytes has nothing to await by mistake.
|
|
290
|
+
return wantsPack
|
|
291
|
+
? { requestId, scene: sceneOut.promise, stats: statsOut.promise }
|
|
292
|
+
: { requestId, stats: statsOut.promise };
|
|
293
|
+
}
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
import type { WorldSceneClientParams, WorldStepClientParams } from '../schemas/sdcpp-config.js';
|
|
2
|
+
import { type WorldSceneResult, type WorldSceneResultWithPack, type WorldStepResult } from './world-result.js';
|
|
3
|
+
/**
|
|
4
|
+
* Builds a world for an ABot-World session: the prompt and first frame are
|
|
5
|
+
* encoded into a scene pack the walk then runs on.
|
|
6
|
+
*
|
|
7
|
+
* Run this once per world. The world is live on the session as soon as this
|
|
8
|
+
* completes — `stats` is the completion signal. Creating a world on a session
|
|
9
|
+
* that already has one replaces it and restarts the walk from the beginning.
|
|
10
|
+
*
|
|
11
|
+
* The pack itself is NOT returned by default: it is 10+ MB, a third larger again
|
|
12
|
+
* as base64, and the common create-then-walk-now flow never touches the bytes.
|
|
13
|
+
* Pass `returnPack: true` to get them, save them, and pass that file back as
|
|
14
|
+
* `modelConfig.sceneSrc` on a later `loadModel` to walk the same world again.
|
|
15
|
+
* `sceneSrc` takes a path or URL, so persist the bytes yourself.
|
|
16
|
+
*
|
|
17
|
+
* Scene creation cannot be interrupted — the engine exposes no abort hook for
|
|
18
|
+
* it — so `cancel({ requestId })` stops the SDK from yielding but the encode runs
|
|
19
|
+
* to completion. Await the result before unloading the model.
|
|
20
|
+
*
|
|
21
|
+
* @param params - Loaded world model ID, scene prompt, first-frame image bytes, optional dimensions, and `returnPack`.
|
|
22
|
+
* @returns `requestId` and `stats`; plus `scene` (promise of the pack) when `returnPack: true`.
|
|
23
|
+
* @throws {RequestValidationFailedError} Client-side, before any RPC, if the
|
|
24
|
+
* prompt is empty, the dimensions are not positive multiples of 32 within the
|
|
25
|
+
* per-axis and total-pixel ceilings, or the first frame exceeds 3 MB.
|
|
26
|
+
* @throws Server-side, before the session is touched, if the first frame's
|
|
27
|
+
* header declares more than 8192x8192 pixels or cannot be read. Raised as
|
|
28
|
+
* `PluginRequestValidationFailedError` on the engine; that class is not
|
|
29
|
+
* exported from the client package and is not in the RPC reconstructor map, so
|
|
30
|
+
* match it on message rather than with `instanceof`.
|
|
31
|
+
* @throws {ModelOperationNotSupportedError} If the model was not loaded with
|
|
32
|
+
* `mode: "world"`.
|
|
33
|
+
* @throws {RequestRejectedByPolicyError} If a world job is already running on
|
|
34
|
+
* this model — world refuses rather than queues.
|
|
35
|
+
* @throws {InferenceCancelledError} If `cancel({ requestId })` is accepted.
|
|
36
|
+
* Scene creation is uninterruptible, so this stops delivery while the encode
|
|
37
|
+
* runs to completion.
|
|
38
|
+
* @throws {StreamEndedError} If the RPC stream closes without a terminal
|
|
39
|
+
* `done` chunk.
|
|
40
|
+
*
|
|
41
|
+
* @example
|
|
42
|
+
* ```typescript
|
|
43
|
+
* const modelId = await loadModel({
|
|
44
|
+
* modelType: "diffusion",
|
|
45
|
+
* modelSrc: ABOT_WORLD_0_5B_Q8_0,
|
|
46
|
+
* modelConfig: {
|
|
47
|
+
* mode: "world",
|
|
48
|
+
* taehvModelSrc: ABOT_WORLD_0_5B_LF_VAE,
|
|
49
|
+
* t5XxlModelSrc: UMT5_XXL_ENC_Q8_0,
|
|
50
|
+
* vaeModelSrc: ABOT_WORLD_0_5B_LF_VAE_F16,
|
|
51
|
+
* world: { seed: 42, kvCache: true },
|
|
52
|
+
* },
|
|
53
|
+
* });
|
|
54
|
+
*
|
|
55
|
+
* // Walk it now and never persist it: no pack crosses the wire.
|
|
56
|
+
* const { stats } = worldCreateScene({
|
|
57
|
+
* modelId,
|
|
58
|
+
* prompt: "| unknown | A realistic outdoor world scene with a navigable path.",
|
|
59
|
+
* image: fs.readFileSync("first-frame.jpg"),
|
|
60
|
+
* });
|
|
61
|
+
* await stats;
|
|
62
|
+
*
|
|
63
|
+
* // Or keep it, to walk the same world after a reload.
|
|
64
|
+
* const { scene } = worldCreateScene({
|
|
65
|
+
* modelId,
|
|
66
|
+
* prompt: "| unknown | A realistic outdoor world scene with a navigable path.",
|
|
67
|
+
* image: fs.readFileSync("first-frame.jpg"),
|
|
68
|
+
* returnPack: true,
|
|
69
|
+
* });
|
|
70
|
+
* fs.writeFileSync("world.safetensors", await scene);
|
|
71
|
+
* ```
|
|
72
|
+
*/
|
|
73
|
+
export declare function worldCreateScene(params: WorldSceneClientParams & {
|
|
74
|
+
returnPack: true;
|
|
75
|
+
}): WorldSceneResultWithPack;
|
|
76
|
+
export declare function worldCreateScene(params: WorldSceneClientParams & {
|
|
77
|
+
returnPack?: false | undefined;
|
|
78
|
+
}): WorldSceneResult;
|
|
79
|
+
export declare function worldCreateScene(params: WorldSceneClientParams): WorldSceneResult | WorldSceneResultWithPack;
|
|
80
|
+
/**
|
|
81
|
+
* Generates the next block of an ABot-World walk under the keys held for it.
|
|
82
|
+
*
|
|
83
|
+
* One call produces one block: at the default `world.numFramePerBlock`, 9 frames
|
|
84
|
+
* for the first block after the session loads (decoder warmup) and 12
|
|
85
|
+
* thereafter. Both counts move with `numFramePerBlock`, so read `stats.frames`
|
|
86
|
+
* rather than assuming them. Frames arrive on `frameStream` as they are decoded,
|
|
87
|
+
* so a viewer can display them without waiting for the block.
|
|
88
|
+
*
|
|
89
|
+
* Only one block runs at a time per model — a second call while one is in
|
|
90
|
+
* flight is rejected rather than queued, so drive the next call off the
|
|
91
|
+
* previous one. Cancelling has block granularity: the current block finishes
|
|
92
|
+
* internally, delivery stops, and the call rejects rather than returning a
|
|
93
|
+
* truncated block.
|
|
94
|
+
*
|
|
95
|
+
* `keys` accepts an array, a key-state object, or a raw 8-bit mask. WASD move,
|
|
96
|
+
* IJKL steer the camera; omit for an idle block.
|
|
97
|
+
*
|
|
98
|
+
* `progressStream` forwards the engine's `{ step, totalSteps, elapsedMs }` tick,
|
|
99
|
+
* the same shape `video` and `diffusion` emit. The engine fires it once per
|
|
100
|
+
* block, AFTER the frames, so it is an end-of-block summary rather than
|
|
101
|
+
* mid-block liveness — one tick per `worldStep` call.
|
|
102
|
+
*
|
|
103
|
+
* @param params - Loaded world model ID and the keys held for this block.
|
|
104
|
+
* @returns `requestId`, `frameStream`, `progressStream`, `frames` (the whole block), and `stats`.
|
|
105
|
+
* @throws {RequestValidationFailedError} Client-side, before any RPC, on an
|
|
106
|
+
* unknown walk key.
|
|
107
|
+
* @throws {ModelOperationNotSupportedError} If the model was not loaded with
|
|
108
|
+
* `mode: "world"`.
|
|
109
|
+
* @throws {RequestRejectedByPolicyError} If a block is already in flight on
|
|
110
|
+
* this model.
|
|
111
|
+
* @throws {InferenceCancelledError} If `cancel({ requestId })` is accepted. The
|
|
112
|
+
* step rejects rather than resolving with a truncated block, and the session
|
|
113
|
+
* is rebuilt on the next call.
|
|
114
|
+
* @throws {StreamEndedError} If the RPC stream closes without a terminal
|
|
115
|
+
* `done` chunk.
|
|
116
|
+
*
|
|
117
|
+
* @example Walk forward, displaying frames as they arrive
|
|
118
|
+
* ```typescript
|
|
119
|
+
* const { frameStream } = worldStep({ modelId, keys: ["W"] });
|
|
120
|
+
* for await (const frame of frameStream) {
|
|
121
|
+
* display(frame); // PNG, or JPEG when world.frameJpegQuality is set
|
|
122
|
+
* }
|
|
123
|
+
* ```
|
|
124
|
+
*
|
|
125
|
+
* @example Report what each block did
|
|
126
|
+
* ```typescript
|
|
127
|
+
* const { progressStream, frames } = worldStep({ modelId, keys: ["W"] });
|
|
128
|
+
* for await (const { totalSteps, elapsedMs } of progressStream) {
|
|
129
|
+
* // Once per block, after its frames: a summary, not a spinner tick.
|
|
130
|
+
* report(`block delivered ${totalSteps} frames in ${elapsedMs}ms`);
|
|
131
|
+
* }
|
|
132
|
+
* for (const frame of await frames) display(frame);
|
|
133
|
+
* ```
|
|
134
|
+
*
|
|
135
|
+
* @example Drive a key loop off the one-block-at-a-time contract
|
|
136
|
+
* ```typescript
|
|
137
|
+
* while (walking) {
|
|
138
|
+
* const { frames } = worldStep({ modelId, keys: heldKeys }); // e.g. { W: true, L: true }
|
|
139
|
+
* for (const frame of await frames) display(frame);
|
|
140
|
+
* }
|
|
141
|
+
* ```
|
|
142
|
+
*/
|
|
143
|
+
export declare function worldStep(params: WorldStepClientParams): WorldStepResult;
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
import { stream as streamRpc } from '../dispatch.js';
|
|
2
|
+
import { createWorldSceneResult, createWorldStepResult } from './world-result.js';
|
|
3
|
+
export function worldCreateScene(params) {
|
|
4
|
+
return createWorldSceneResult(params, streamRpc);
|
|
5
|
+
}
|
|
6
|
+
/**
|
|
7
|
+
* Generates the next block of an ABot-World walk under the keys held for it.
|
|
8
|
+
*
|
|
9
|
+
* One call produces one block: at the default `world.numFramePerBlock`, 9 frames
|
|
10
|
+
* for the first block after the session loads (decoder warmup) and 12
|
|
11
|
+
* thereafter. Both counts move with `numFramePerBlock`, so read `stats.frames`
|
|
12
|
+
* rather than assuming them. Frames arrive on `frameStream` as they are decoded,
|
|
13
|
+
* so a viewer can display them without waiting for the block.
|
|
14
|
+
*
|
|
15
|
+
* Only one block runs at a time per model — a second call while one is in
|
|
16
|
+
* flight is rejected rather than queued, so drive the next call off the
|
|
17
|
+
* previous one. Cancelling has block granularity: the current block finishes
|
|
18
|
+
* internally, delivery stops, and the call rejects rather than returning a
|
|
19
|
+
* truncated block.
|
|
20
|
+
*
|
|
21
|
+
* `keys` accepts an array, a key-state object, or a raw 8-bit mask. WASD move,
|
|
22
|
+
* IJKL steer the camera; omit for an idle block.
|
|
23
|
+
*
|
|
24
|
+
* `progressStream` forwards the engine's `{ step, totalSteps, elapsedMs }` tick,
|
|
25
|
+
* the same shape `video` and `diffusion` emit. The engine fires it once per
|
|
26
|
+
* block, AFTER the frames, so it is an end-of-block summary rather than
|
|
27
|
+
* mid-block liveness — one tick per `worldStep` call.
|
|
28
|
+
*
|
|
29
|
+
* @param params - Loaded world model ID and the keys held for this block.
|
|
30
|
+
* @returns `requestId`, `frameStream`, `progressStream`, `frames` (the whole block), and `stats`.
|
|
31
|
+
* @throws {RequestValidationFailedError} Client-side, before any RPC, on an
|
|
32
|
+
* unknown walk key.
|
|
33
|
+
* @throws {ModelOperationNotSupportedError} If the model was not loaded with
|
|
34
|
+
* `mode: "world"`.
|
|
35
|
+
* @throws {RequestRejectedByPolicyError} If a block is already in flight on
|
|
36
|
+
* this model.
|
|
37
|
+
* @throws {InferenceCancelledError} If `cancel({ requestId })` is accepted. The
|
|
38
|
+
* step rejects rather than resolving with a truncated block, and the session
|
|
39
|
+
* is rebuilt on the next call.
|
|
40
|
+
* @throws {StreamEndedError} If the RPC stream closes without a terminal
|
|
41
|
+
* `done` chunk.
|
|
42
|
+
*
|
|
43
|
+
* @example Walk forward, displaying frames as they arrive
|
|
44
|
+
* ```typescript
|
|
45
|
+
* const { frameStream } = worldStep({ modelId, keys: ["W"] });
|
|
46
|
+
* for await (const frame of frameStream) {
|
|
47
|
+
* display(frame); // PNG, or JPEG when world.frameJpegQuality is set
|
|
48
|
+
* }
|
|
49
|
+
* ```
|
|
50
|
+
*
|
|
51
|
+
* @example Report what each block did
|
|
52
|
+
* ```typescript
|
|
53
|
+
* const { progressStream, frames } = worldStep({ modelId, keys: ["W"] });
|
|
54
|
+
* for await (const { totalSteps, elapsedMs } of progressStream) {
|
|
55
|
+
* // Once per block, after its frames: a summary, not a spinner tick.
|
|
56
|
+
* report(`block delivered ${totalSteps} frames in ${elapsedMs}ms`);
|
|
57
|
+
* }
|
|
58
|
+
* for (const frame of await frames) display(frame);
|
|
59
|
+
* ```
|
|
60
|
+
*
|
|
61
|
+
* @example Drive a key loop off the one-block-at-a-time contract
|
|
62
|
+
* ```typescript
|
|
63
|
+
* while (walking) {
|
|
64
|
+
* const { frames } = worldStep({ modelId, keys: heldKeys }); // e.g. { W: true, L: true }
|
|
65
|
+
* for (const frame of await frames) display(frame);
|
|
66
|
+
* }
|
|
67
|
+
* ```
|
|
68
|
+
*/
|
|
69
|
+
export function worldStep(params) {
|
|
70
|
+
return createWorldStepResult(params, streamRpc);
|
|
71
|
+
}
|
|
@@ -7,7 +7,7 @@ import path from 'bare-path';
|
|
|
7
7
|
import os from 'bare-os';
|
|
8
8
|
import env from 'bare-env';
|
|
9
9
|
import { pathToFileURL } from 'bare-url';
|
|
10
|
-
import { validateConfig } from '
|
|
10
|
+
import { validateConfig } from './config-utils.js';
|
|
11
11
|
import { ConfigFileInvalidError, ConfigFileParseFailedError } from '../errors/index.js';
|
|
12
12
|
import { getAppLogger } from '../logging/index.js';
|
|
13
13
|
const SUPPORTED_CONFIG_FILE_EXTS = ['.js', '.json'];
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
export * from '
|
|
2
|
-
export * from '
|
|
1
|
+
export * from './audio.js';
|
|
2
|
+
export * from './registry.js';
|
package/dist/constants/index.js
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
export * from '
|
|
2
|
-
export * from '
|
|
1
|
+
export * from './audio.js';
|
|
2
|
+
export * from './registry.js';
|
package/dist/dispatch.js
CHANGED
|
@@ -137,21 +137,8 @@ function prepareRequest(request) {
|
|
|
137
137
|
}
|
|
138
138
|
return validated;
|
|
139
139
|
}
|
|
140
|
-
function
|
|
141
|
-
const profilingMeta = getProfilingMeta(request);
|
|
142
|
-
if (!profilingMeta && !progressCallback)
|
|
143
|
-
return undefined;
|
|
144
|
-
const options = {};
|
|
145
|
-
if (progressCallback)
|
|
146
|
-
options.progressCallback = progressCallback;
|
|
147
|
-
if (profilingMeta)
|
|
148
|
-
options.profilingMeta = profilingMeta;
|
|
149
|
-
return options;
|
|
150
|
-
}
|
|
151
|
-
function invokeHandler(request, handler, isDelegated) {
|
|
140
|
+
function invokeHandler(request, handler) {
|
|
152
141
|
const directHandler = handler;
|
|
153
|
-
if (isDelegated)
|
|
154
|
-
return directHandler(request, delegatedOptions(request));
|
|
155
142
|
return directHandler(request);
|
|
156
143
|
}
|
|
157
144
|
function isAsyncGenerator(result) {
|
|
@@ -161,7 +148,7 @@ function isAsyncGenerator(result) {
|
|
|
161
148
|
* Bridge a progress-callback handler (which returns a single final response and
|
|
162
149
|
* pushes interim updates through a callback) into an async generator.
|
|
163
150
|
*/
|
|
164
|
-
async function* streamWithProgress(request, handler
|
|
151
|
+
async function* streamWithProgress(request, handler) {
|
|
165
152
|
const queue = [];
|
|
166
153
|
const errors = [];
|
|
167
154
|
let done = false;
|
|
@@ -169,7 +156,7 @@ async function* streamWithProgress(request, handler, isDelegated) {
|
|
|
169
156
|
queue.push(update);
|
|
170
157
|
}
|
|
171
158
|
const directHandler = handler;
|
|
172
|
-
Promise.resolve(directHandler(request,
|
|
159
|
+
Promise.resolve(directHandler(request, progressCallback))
|
|
173
160
|
.then((final) => {
|
|
174
161
|
queue.push(final);
|
|
175
162
|
done = true;
|
|
@@ -195,27 +182,27 @@ export async function send(request, _options) {
|
|
|
195
182
|
assertLifecycleAllowed(request);
|
|
196
183
|
const processed = prepareRequest(request);
|
|
197
184
|
const entry = getHandlerEntry(processed.type);
|
|
198
|
-
const
|
|
199
|
-
// Plugin capabilities profile themselves inside plugin dispatch
|
|
200
|
-
//
|
|
201
|
-
//
|
|
202
|
-
if (entry.pluginOp
|
|
203
|
-
return (await invokeHandler(processed, handler
|
|
185
|
+
const handler = selectHandler(entry);
|
|
186
|
+
// Plugin capabilities profile themselves inside plugin dispatch. Everything
|
|
187
|
+
// else is timed here so the `profiler` covers every operation. Profiling is a
|
|
188
|
+
// no-op unless enabled.
|
|
189
|
+
if (entry.pluginOp) {
|
|
190
|
+
return (await invokeHandler(processed, handler));
|
|
204
191
|
}
|
|
205
|
-
return profileReplyHandler({ op: processed.type, request: processed }, () => invokeHandler(processed, handler
|
|
192
|
+
return profileReplyHandler({ op: processed.type, request: processed }, () => invokeHandler(processed, handler));
|
|
206
193
|
}
|
|
207
194
|
export async function* stream(request, _options) {
|
|
208
195
|
await ensureReady();
|
|
209
196
|
assertLifecycleAllowed(request);
|
|
210
197
|
const processed = prepareRequest(request);
|
|
211
198
|
const entry = getHandlerEntry(processed.type);
|
|
212
|
-
const
|
|
199
|
+
const handler = selectHandler(entry);
|
|
213
200
|
async function* run() {
|
|
214
201
|
if (handlerSupportsProgress(entry, processed)) {
|
|
215
|
-
yield* streamWithProgress(processed, handler
|
|
202
|
+
yield* streamWithProgress(processed, handler);
|
|
216
203
|
return;
|
|
217
204
|
}
|
|
218
|
-
const result = invokeHandler(processed, handler
|
|
205
|
+
const result = invokeHandler(processed, handler);
|
|
219
206
|
if (isAsyncGenerator(result)) {
|
|
220
207
|
yield* result;
|
|
221
208
|
}
|
|
@@ -223,8 +210,8 @@ export async function* stream(request, _options) {
|
|
|
223
210
|
yield await result;
|
|
224
211
|
}
|
|
225
212
|
}
|
|
226
|
-
// See `send`: plugin capabilities
|
|
227
|
-
if (entry.pluginOp
|
|
213
|
+
// See `send`: plugin capabilities are timed elsewhere.
|
|
214
|
+
if (entry.pluginOp) {
|
|
228
215
|
yield* run();
|
|
229
216
|
}
|
|
230
217
|
else {
|