@atomic-chat/core 0.5.1 → 0.13.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +22 -0
- package/dist/app-daemon.js +20 -67
- package/dist/app-daemon.js.map +1 -1
- package/dist/backend/advisor/advisor.d.ts +132 -0
- package/dist/backend/advisor/advisor.js +537 -0
- package/dist/backend/advisor/advisor.js.map +1 -0
- package/dist/backend/advisor/index.d.ts +6 -0
- package/dist/backend/advisor/index.js +7 -0
- package/dist/backend/advisor/index.js.map +1 -0
- package/dist/backend/advisor/policy.d.ts +81 -0
- package/dist/backend/advisor/policy.js +154 -0
- package/dist/backend/advisor/policy.js.map +1 -0
- package/dist/backend/catalog/archive.d.ts +21 -7
- package/dist/backend/catalog/archive.js +57 -17
- package/dist/backend/catalog/archive.js.map +1 -1
- package/dist/backend/catalog/bundled-manifest-baseline.js +42 -26
- package/dist/backend/catalog/bundled-manifest-baseline.js.map +1 -1
- package/dist/backend/catalog/cuda-family.d.ts +12 -2
- package/dist/backend/catalog/cuda-family.js +30 -7
- package/dist/backend/catalog/cuda-family.js.map +1 -1
- package/dist/backend/catalog/index.d.ts +5 -0
- package/dist/backend/catalog/index.js +5 -0
- package/dist/backend/catalog/index.js.map +1 -1
- package/dist/backend/catalog/manifest.d.ts +5 -3
- package/dist/backend/catalog/manifest.js +12 -9
- package/dist/backend/catalog/manifest.js.map +1 -1
- package/dist/backend/catalog/migrate.d.ts +8 -6
- package/dist/backend/catalog/migrate.js +26 -15
- package/dist/backend/catalog/migrate.js.map +1 -1
- package/dist/backend/catalog/prism-catalog.d.ts +42 -0
- package/dist/backend/catalog/prism-catalog.js +101 -0
- package/dist/backend/catalog/prism-catalog.js.map +1 -0
- package/dist/backend/catalog/prism-manifest-baseline.d.ts +2 -0
- package/dist/backend/catalog/prism-manifest-baseline.js +11 -0
- package/dist/backend/catalog/prism-manifest-baseline.js.map +1 -0
- package/dist/backend/catalog/prism-manifest-baseline.json +134 -0
- package/dist/backend/catalog/prism-manifest.d.ts +90 -0
- package/dist/backend/catalog/prism-manifest.js +171 -0
- package/dist/backend/catalog/prism-manifest.js.map +1 -0
- package/dist/backend/catalog/turboquant-catalog.d.ts +60 -0
- package/dist/backend/catalog/turboquant-catalog.js +162 -0
- package/dist/backend/catalog/turboquant-catalog.js.map +1 -0
- package/dist/backend/catalog/turboquant-index.d.ts +104 -0
- package/dist/backend/catalog/turboquant-index.js +260 -0
- package/dist/backend/catalog/turboquant-index.js.map +1 -0
- package/dist/backend/index.d.ts +1 -0
- package/dist/backend/index.js +1 -0
- package/dist/backend/index.js.map +1 -1
- package/dist/backend/install/runtime-backend.d.ts +6 -6
- package/dist/backend/install/runtime-backend.js +23 -20
- package/dist/backend/install/runtime-backend.js.map +1 -1
- package/dist/backend/install/service.d.ts +106 -5
- package/dist/backend/install/service.js +247 -21
- package/dist/backend/install/service.js.map +1 -1
- package/dist/backend/installed/bundled.d.ts +19 -0
- package/dist/backend/installed/bundled.js +45 -0
- package/dist/backend/installed/bundled.js.map +1 -0
- package/dist/backend/installed/index.d.ts +1 -0
- package/dist/backend/installed/index.js +1 -0
- package/dist/backend/installed/index.js.map +1 -1
- package/dist/backend/installed/installed.js +3 -0
- package/dist/backend/installed/installed.js.map +1 -1
- package/dist/backend/optimal/optimal-cache.d.ts +27 -8
- package/dist/backend/optimal/optimal-cache.js +38 -19
- package/dist/backend/optimal/optimal-cache.js.map +1 -1
- package/dist/backend/optimal/optimal-store.js +1 -1
- package/dist/backend/optimal/optimal-store.js.map +1 -1
- package/dist/backend/select/categories.d.ts +2 -1
- package/dist/backend/select/categories.js +8 -1
- package/dist/backend/select/categories.js.map +1 -1
- package/dist/backend/select/features.d.ts +13 -5
- package/dist/backend/select/features.js +36 -6
- package/dist/backend/select/features.js.map +1 -1
- package/dist/backend/select/index.d.ts +2 -0
- package/dist/backend/select/index.js +2 -0
- package/dist/backend/select/index.js.map +1 -1
- package/dist/backend/select/prism.d.ts +54 -0
- package/dist/backend/select/prism.js +212 -0
- package/dist/backend/select/prism.js.map +1 -0
- package/dist/backend/select/recovery.d.ts +4 -2
- package/dist/backend/select/recovery.js +17 -5
- package/dist/backend/select/recovery.js.map +1 -1
- package/dist/backend/select/tiers.d.ts +20 -11
- package/dist/backend/select/tiers.js +49 -11
- package/dist/backend/select/tiers.js.map +1 -1
- package/dist/backend/select/turboquant-tiers.d.ts +45 -0
- package/dist/backend/select/turboquant-tiers.js +114 -0
- package/dist/backend/select/turboquant-tiers.js.map +1 -0
- package/dist/backend/types.d.ts +6 -69
- package/dist/cli/commands/backends.d.ts +10 -0
- package/dist/cli/commands/backends.js +142 -0
- package/dist/cli/commands/backends.js.map +1 -0
- package/dist/cli/commands/hardware.d.ts +13 -0
- package/dist/cli/commands/hardware.js +71 -0
- package/dist/cli/commands/hardware.js.map +1 -0
- package/dist/cli/commands/host-step.d.ts +28 -0
- package/dist/cli/commands/host-step.js +67 -0
- package/dist/cli/commands/host-step.js.map +1 -0
- package/dist/cli/commands/index.d.ts +4 -1
- package/dist/cli/commands/index.js +4 -1
- package/dist/cli/commands/index.js.map +1 -1
- package/dist/cli/commands/serve.js +4 -4
- package/dist/cli/commands/serve.js.map +1 -1
- package/dist/cli/commands/shared.d.ts +6 -1
- package/dist/cli/commands/shared.js +7 -0
- package/dist/cli/commands/shared.js.map +1 -1
- package/dist/cli/io.d.ts +11 -0
- package/dist/cli/io.js +28 -3
- package/dist/cli/io.js.map +1 -1
- package/dist/cli/launch.js +2 -2
- package/dist/cli/launch.js.map +1 -1
- package/dist/cli/main.d.ts +1 -1
- package/dist/cli/main.js +11 -1
- package/dist/cli/main.js.map +1 -1
- package/dist/client/control-client.d.ts +128 -7
- package/dist/client/control-client.js +211 -12
- package/dist/client/control-client.js.map +1 -1
- package/dist/cloud/registry.js +9 -1
- package/dist/cloud/registry.js.map +1 -1
- package/dist/config/paths.d.ts +137 -0
- package/dist/config/paths.js +163 -0
- package/dist/config/paths.js.map +1 -1
- package/dist/contracts/backend-advisor.d.ts +225 -0
- package/dist/contracts/backend-advisor.js +14 -0
- package/dist/contracts/backend-advisor.js.map +1 -0
- package/dist/contracts/control-api.d.ts +7 -1
- package/dist/contracts/control-api.js +7 -1
- package/dist/contracts/control-api.js.map +1 -1
- package/dist/contracts/decision.d.ts +394 -0
- package/dist/contracts/decision.js +69 -0
- package/dist/contracts/decision.js.map +1 -0
- package/dist/contracts/diffusion.d.ts +62 -9
- package/dist/contracts/embedding.d.ts +103 -0
- package/dist/contracts/embedding.js +24 -0
- package/dist/contracts/embedding.js.map +1 -0
- package/dist/contracts/engine-builds.d.ts +108 -0
- package/dist/contracts/engine-builds.js +11 -0
- package/dist/contracts/engine-builds.js.map +1 -0
- package/dist/contracts/engines.d.ts +157 -0
- package/dist/contracts/engines.js +50 -0
- package/dist/contracts/engines.js.map +1 -0
- package/dist/contracts/environment.d.ts +774 -0
- package/dist/contracts/environment.js +116 -0
- package/dist/contracts/environment.js.map +1 -0
- package/dist/contracts/errors.d.ts +43 -2
- package/dist/contracts/errors.js +41 -0
- package/dist/contracts/errors.js.map +1 -1
- package/dist/contracts/events.d.ts +82 -4
- package/dist/contracts/hardware.d.ts +79 -6
- package/dist/contracts/hardware.js +9 -5
- package/dist/contracts/hardware.js.map +1 -1
- package/dist/contracts/index.d.ts +8 -0
- package/dist/contracts/index.js +8 -0
- package/dist/contracts/index.js.map +1 -1
- package/dist/contracts/model-compatibility.d.ts +97 -0
- package/dist/contracts/model-compatibility.js +6 -0
- package/dist/contracts/model-compatibility.js.map +1 -0
- package/dist/contracts/model-setup.d.ts +115 -0
- package/dist/contracts/model-setup.js +33 -0
- package/dist/contracts/model-setup.js.map +1 -0
- package/dist/contracts/model-yml.d.ts +14 -0
- package/dist/contracts/model-yml.js +1 -0
- package/dist/contracts/model-yml.js.map +1 -1
- package/dist/contracts/session.d.ts +45 -2
- package/dist/contracts/session.js +14 -1
- package/dist/contracts/session.js.map +1 -1
- package/dist/contracts/settings.d.ts +44 -1
- package/dist/contracts/settings.js.map +1 -1
- package/dist/core/atomic-core.d.ts +46 -5
- package/dist/core/atomic-core.js +25 -0
- package/dist/core/atomic-core.js.map +1 -1
- package/dist/core/create.js +523 -23
- package/dist/core/create.js.map +1 -1
- package/dist/core/gpu/index.d.ts +6 -0
- package/dist/core/gpu/index.js +7 -0
- package/dist/core/gpu/index.js.map +1 -0
- package/dist/core/gpu/policy.d.ts +68 -0
- package/dist/core/gpu/policy.js +99 -0
- package/dist/core/gpu/policy.js.map +1 -0
- package/dist/core/gpu/residency.d.ts +44 -0
- package/dist/core/gpu/residency.js +101 -0
- package/dist/core/gpu/residency.js.map +1 -0
- package/dist/core/gpu-residency.d.ts +39 -0
- package/dist/core/gpu-residency.js +32 -0
- package/dist/core/gpu-residency.js.map +1 -0
- package/dist/core/index.d.ts +1 -1
- package/dist/core/managed-engines.d.ts +207 -0
- package/dist/core/managed-engines.js +579 -0
- package/dist/core/managed-engines.js.map +1 -0
- package/dist/core/managed-environment.d.ts +72 -0
- package/dist/core/managed-environment.js +218 -0
- package/dist/core/managed-environment.js.map +1 -0
- package/dist/core/model-setup/index.d.ts +4 -0
- package/dist/core/model-setup/index.js +5 -0
- package/dist/core/model-setup/index.js.map +1 -0
- package/dist/core/model-setup/plan.d.ts +56 -0
- package/dist/core/model-setup/plan.js +158 -0
- package/dist/core/model-setup/plan.js.map +1 -0
- package/dist/core/model-setup/service.d.ts +92 -0
- package/dist/core/model-setup/service.js +329 -0
- package/dist/core/model-setup/service.js.map +1 -0
- package/dist/core/model-setup/store.d.ts +29 -0
- package/dist/core/model-setup/store.js +116 -0
- package/dist/core/model-setup/store.js.map +1 -0
- package/dist/core/model-setup/wiring.d.ts +59 -0
- package/dist/core/model-setup/wiring.js +132 -0
- package/dist/core/model-setup/wiring.js.map +1 -0
- package/dist/core/prism.d.ts +38 -0
- package/dist/core/prism.js +43 -0
- package/dist/core/prism.js.map +1 -0
- package/dist/core/public-server.d.ts +7 -0
- package/dist/core/public-server.js +5 -2
- package/dist/core/public-server.js.map +1 -1
- package/dist/core/sessions.d.ts +6 -1
- package/dist/core/sessions.js +18 -14
- package/dist/core/sessions.js.map +1 -1
- package/dist/core/tensorrt-llm.d.ts +169 -0
- package/dist/core/tensorrt-llm.js +428 -0
- package/dist/core/tensorrt-llm.js.map +1 -0
- package/dist/core/types.d.ts +43 -1
- package/dist/core/types.js.map +1 -1
- package/dist/decision/args.d.ts +110 -0
- package/dist/decision/args.js +126 -0
- package/dist/decision/args.js.map +1 -0
- package/dist/decision/backoff.d.ts +25 -0
- package/dist/decision/backoff.js +33 -0
- package/dist/decision/backoff.js.map +1 -0
- package/dist/decision/engine-candidates.d.ts +52 -0
- package/dist/decision/engine-candidates.js +138 -0
- package/dist/decision/engine-candidates.js.map +1 -0
- package/dist/decision/engine-fallback.d.ts +41 -0
- package/dist/decision/engine-fallback.js +57 -0
- package/dist/decision/engine-fallback.js.map +1 -0
- package/dist/decision/engine-version.d.ts +27 -0
- package/dist/decision/engine-version.js +54 -0
- package/dist/decision/engine-version.js.map +1 -0
- package/dist/decision/engine.d.ts +114 -0
- package/dist/decision/engine.js +253 -0
- package/dist/decision/engine.js.map +1 -0
- package/dist/decision/http.d.ts +36 -0
- package/dist/decision/http.js +73 -0
- package/dist/decision/http.js.map +1 -0
- package/dist/decision/index.d.ts +20 -0
- package/dist/decision/index.js +21 -0
- package/dist/decision/index.js.map +1 -0
- package/dist/decision/model-facts.d.ts +32 -0
- package/dist/decision/model-facts.js +40 -0
- package/dist/decision/model-facts.js.map +1 -0
- package/dist/decision/outcome.d.ts +35 -0
- package/dist/decision/outcome.js +105 -0
- package/dist/decision/outcome.js.map +1 -0
- package/dist/decision/process.d.ts +71 -0
- package/dist/decision/process.js +142 -0
- package/dist/decision/process.js.map +1 -0
- package/dist/decision/readiness.d.ts +74 -0
- package/dist/decision/readiness.js +165 -0
- package/dist/decision/readiness.js.map +1 -0
- package/dist/decision/request.d.ts +11 -0
- package/dist/decision/request.js +56 -0
- package/dist/decision/request.js.map +1 -0
- package/dist/decision/service.d.ts +202 -0
- package/dist/decision/service.js +711 -0
- package/dist/decision/service.js.map +1 -0
- package/dist/decision/upstream-version.d.ts +23 -0
- package/dist/decision/upstream-version.js +34 -0
- package/dist/decision/upstream-version.js.map +1 -0
- package/dist/decision/wiring.d.ts +67 -0
- package/dist/decision/wiring.js +101 -0
- package/dist/decision/wiring.js.map +1 -0
- package/dist/diffusion/args.d.ts +7 -4
- package/dist/diffusion/args.js +31 -7
- package/dist/diffusion/args.js.map +1 -1
- package/dist/diffusion/compat.d.ts +1 -1
- package/dist/diffusion/compat.js +6 -2
- package/dist/diffusion/compat.js.map +1 -1
- package/dist/diffusion/image-job.js +1 -1
- package/dist/diffusion/image-job.js.map +1 -1
- package/dist/diffusion/install.d.ts +35 -11
- package/dist/diffusion/install.js +74 -49
- package/dist/diffusion/install.js.map +1 -1
- package/dist/diffusion/job-kind.d.ts +33 -7
- package/dist/diffusion/jobs.d.ts +6 -1
- package/dist/diffusion/jobs.js +64 -18
- package/dist/diffusion/jobs.js.map +1 -1
- package/dist/diffusion/parse.d.ts +1 -2
- package/dist/diffusion/parse.js +1 -13
- package/dist/diffusion/parse.js.map +1 -1
- package/dist/diffusion/progress.js +3 -0
- package/dist/diffusion/progress.js.map +1 -1
- package/dist/diffusion/server-process.d.ts +6 -0
- package/dist/diffusion/server-process.js +3 -2
- package/dist/diffusion/server-process.js.map +1 -1
- package/dist/diffusion/service.d.ts +52 -6
- package/dist/diffusion/service.js +215 -40
- package/dist/diffusion/service.js.map +1 -1
- package/dist/diffusion/session.d.ts +27 -3
- package/dist/diffusion/session.js +108 -17
- package/dist/diffusion/session.js.map +1 -1
- package/dist/diffusion/state.d.ts +19 -1
- package/dist/diffusion/state.js +10 -0
- package/dist/diffusion/state.js.map +1 -1
- package/dist/diffusion/tracker.d.ts +21 -1
- package/dist/diffusion/tracker.js +43 -6
- package/dist/diffusion/tracker.js.map +1 -1
- package/dist/diffusion/types.d.ts +7 -0
- package/dist/diffusion/types.js +11 -0
- package/dist/diffusion/types.js.map +1 -1
- package/dist/diffusion/validate.js +2 -2
- package/dist/diffusion/validate.js.map +1 -1
- package/dist/diffusion/video-estimate.d.ts +214 -0
- package/dist/diffusion/video-estimate.js +402 -0
- package/dist/diffusion/video-estimate.js.map +1 -0
- package/dist/diffusion/video-eta.d.ts +90 -0
- package/dist/diffusion/video-eta.js +248 -0
- package/dist/diffusion/video-eta.js.map +1 -0
- package/dist/diffusion/video-history.d.ts +45 -0
- package/dist/diffusion/video-history.js +121 -0
- package/dist/diffusion/video-history.js.map +1 -0
- package/dist/diffusion/video-job.d.ts +13 -5
- package/dist/diffusion/video-job.js +42 -15
- package/dist/diffusion/video-job.js.map +1 -1
- package/dist/diffusion/video-recipe.js +3 -0
- package/dist/diffusion/video-recipe.js.map +1 -1
- package/dist/diffusion/video-tiling.d.ts +54 -0
- package/dist/diffusion/video-tiling.js +70 -0
- package/dist/diffusion/video-tiling.js.map +1 -0
- package/dist/diffusion/wiring.d.ts +14 -1
- package/dist/diffusion/wiring.js +7 -0
- package/dist/diffusion/wiring.js.map +1 -1
- package/dist/diffusion/workflow.d.ts +8 -2
- package/dist/diffusion/workflow.js +13 -5
- package/dist/diffusion/workflow.js.map +1 -1
- package/dist/downloads/downloader.d.ts +22 -0
- package/dist/downloads/downloader.js +57 -2
- package/dist/downloads/downloader.js.map +1 -1
- package/dist/downloads/protocol.d.ts +3 -8
- package/dist/downloads/protocol.js.map +1 -1
- package/dist/downloads/segmented.d.ts +40 -0
- package/dist/downloads/segmented.js +170 -0
- package/dist/downloads/segmented.js.map +1 -0
- package/dist/embedding/args.d.ts +39 -0
- package/dist/embedding/args.js +46 -0
- package/dist/embedding/args.js.map +1 -0
- package/dist/embedding/engine.d.ts +50 -0
- package/dist/embedding/engine.js +100 -0
- package/dist/embedding/engine.js.map +1 -0
- package/dist/embedding/ffmpeg.d.ts +18 -0
- package/dist/embedding/ffmpeg.js +63 -0
- package/dist/embedding/ffmpeg.js.map +1 -0
- package/dist/embedding/index.d.ts +16 -0
- package/dist/embedding/index.js +17 -0
- package/dist/embedding/index.js.map +1 -0
- package/dist/embedding/model-facts.d.ts +51 -0
- package/dist/embedding/model-facts.js +84 -0
- package/dist/embedding/model-facts.js.map +1 -0
- package/dist/embedding/process.d.ts +70 -0
- package/dist/embedding/process.js +140 -0
- package/dist/embedding/process.js.map +1 -0
- package/dist/embedding/readiness.d.ts +56 -0
- package/dist/embedding/readiness.js +114 -0
- package/dist/embedding/readiness.js.map +1 -0
- package/dist/embedding/request.d.ts +43 -0
- package/dist/embedding/request.js +285 -0
- package/dist/embedding/request.js.map +1 -0
- package/dist/embedding/service.d.ts +152 -0
- package/dist/embedding/service.js +602 -0
- package/dist/embedding/service.js.map +1 -0
- package/dist/embedding/wiring.d.ts +51 -0
- package/dist/embedding/wiring.js +79 -0
- package/dist/embedding/wiring.js.map +1 -0
- package/dist/engine-builds/builds.d.ts +54 -0
- package/dist/engine-builds/builds.js +151 -0
- package/dist/engine-builds/builds.js.map +1 -0
- package/dist/engine-builds/index.d.ts +10 -0
- package/dist/engine-builds/index.js +11 -0
- package/dist/engine-builds/index.js.map +1 -0
- package/dist/engine-builds/install.d.ts +42 -0
- package/dist/engine-builds/install.js +100 -0
- package/dist/engine-builds/install.js.map +1 -0
- package/dist/engine-builds/manifest.d.ts +121 -0
- package/dist/engine-builds/manifest.js +254 -0
- package/dist/engine-builds/manifest.js.map +1 -0
- package/dist/engine-builds/order.d.ts +24 -0
- package/dist/engine-builds/order.js +44 -0
- package/dist/engine-builds/order.js.map +1 -0
- package/dist/engine-builds/select.d.ts +123 -0
- package/dist/engine-builds/select.js +223 -0
- package/dist/engine-builds/select.js.map +1 -0
- package/dist/engine-builds/service.d.ts +123 -0
- package/dist/engine-builds/service.js +392 -0
- package/dist/engine-builds/service.js.map +1 -0
- package/dist/engines/descriptor-order.d.ts +28 -0
- package/dist/engines/descriptor-order.js +60 -0
- package/dist/engines/descriptor-order.js.map +1 -0
- package/dist/engines/engine-build.d.ts +23 -0
- package/dist/engines/engine-build.js +58 -0
- package/dist/engines/engine-build.js.map +1 -0
- package/dist/engines/index.d.ts +10 -0
- package/dist/engines/index.js +11 -0
- package/dist/engines/index.js.map +1 -0
- package/dist/engines/llamacpp.d.ts +71 -0
- package/dist/engines/llamacpp.js +176 -0
- package/dist/engines/llamacpp.js.map +1 -0
- package/dist/engines/managed.d.ts +49 -0
- package/dist/engines/managed.js +88 -0
- package/dist/engines/managed.js.map +1 -0
- package/dist/engines/service.d.ts +46 -0
- package/dist/engines/service.js +61 -0
- package/dist/engines/service.js.map +1 -0
- package/dist/engines/versions.d.ts +65 -0
- package/dist/engines/versions.js +259 -0
- package/dist/engines/versions.js.map +1 -0
- package/dist/hardware/cpu-flags.d.ts +44 -0
- package/dist/hardware/cpu-flags.js +206 -0
- package/dist/hardware/cpu-flags.js.map +1 -0
- package/dist/hardware/drm-sysfs.d.ts +51 -0
- package/dist/hardware/drm-sysfs.js +67 -0
- package/dist/hardware/drm-sysfs.js.map +1 -0
- package/dist/hardware/facts.d.ts +42 -0
- package/dist/hardware/facts.js +92 -0
- package/dist/hardware/facts.js.map +1 -0
- package/dist/hardware/index.d.ts +19 -3
- package/dist/hardware/index.js +19 -3
- package/dist/hardware/index.js.map +1 -1
- package/dist/hardware/merge.d.ts +55 -0
- package/dist/hardware/merge.js +141 -0
- package/dist/hardware/merge.js.map +1 -0
- package/dist/hardware/nvidia-smi.d.ts +43 -0
- package/dist/hardware/nvidia-smi.js +118 -0
- package/dist/hardware/nvidia-smi.js.map +1 -0
- package/dist/hardware/probe-common.d.ts +72 -0
- package/dist/hardware/probe-common.js +106 -0
- package/dist/hardware/probe-common.js.map +1 -0
- package/dist/hardware/probe-darwin.d.ts +20 -0
- package/dist/hardware/probe-darwin.js +76 -0
- package/dist/hardware/probe-darwin.js.map +1 -0
- package/dist/hardware/probe-linux.d.ts +9 -0
- package/dist/hardware/probe-linux.js +133 -0
- package/dist/hardware/probe-linux.js.map +1 -0
- package/dist/hardware/probe-windows.d.ts +7 -0
- package/dist/hardware/probe-windows.js +50 -0
- package/dist/hardware/probe-windows.js.map +1 -0
- package/dist/hardware/probe.d.ts +16 -0
- package/dist/hardware/probe.js +65 -0
- package/dist/hardware/probe.js.map +1 -0
- package/dist/hardware/service.d.ts +60 -0
- package/dist/hardware/service.js +114 -0
- package/dist/hardware/service.js.map +1 -0
- package/dist/hardware/vulkan.d.ts +47 -0
- package/dist/hardware/vulkan.js +148 -0
- package/dist/hardware/vulkan.js.map +1 -0
- package/dist/hardware/windows-video.d.ts +97 -0
- package/dist/hardware/windows-video.js +214 -0
- package/dist/hardware/windows-video.js.map +1 -0
- package/dist/host/app-daemon.d.ts +47 -0
- package/dist/host/app-daemon.js +146 -0
- package/dist/host/app-daemon.js.map +1 -0
- package/dist/host/index.d.ts +5 -3
- package/dist/host/index.js +5 -3
- package/dist/host/index.js.map +1 -1
- package/dist/host/log-file.d.ts +74 -0
- package/dist/host/log-file.js +214 -0
- package/dist/host/log-file.js.map +1 -0
- package/dist/host/recipes/enable-wsl.d.ts +63 -0
- package/dist/host/recipes/enable-wsl.js +73 -0
- package/dist/host/recipes/enable-wsl.js.map +1 -0
- package/dist/host/recipes/executor-io-windows.d.ts +69 -0
- package/dist/host/recipes/executor-io-windows.js +235 -0
- package/dist/host/recipes/executor-io-windows.js.map +1 -0
- package/dist/host/recipes/executor-io.d.ts +80 -0
- package/dist/host/recipes/executor-io.js +211 -0
- package/dist/host/recipes/executor-io.js.map +1 -0
- package/dist/host/recipes/executor.d.ts +81 -0
- package/dist/host/recipes/executor.js +678 -0
- package/dist/host/recipes/executor.js.map +1 -0
- package/dist/host/recipes/guest-executor.d.ts +29 -0
- package/dist/host/recipes/guest-executor.js +76 -0
- package/dist/host/recipes/guest-executor.js.map +1 -0
- package/dist/host/recipes/index.d.ts +16 -0
- package/dist/host/recipes/index.js +17 -0
- package/dist/host/recipes/index.js.map +1 -0
- package/dist/host/recipes/install-container-runtime.d.ts +389 -0
- package/dist/host/recipes/install-container-runtime.js +796 -0
- package/dist/host/recipes/install-container-runtime.js.map +1 -0
- package/dist/host/recipes/openpgp.d.ts +23 -0
- package/dist/host/recipes/openpgp.js +153 -0
- package/dist/host/recipes/openpgp.js.map +1 -0
- package/dist/host/recipes/request-file.d.ts +96 -0
- package/dist/host/recipes/request-file.js +103 -0
- package/dist/host/recipes/request-file.js.map +1 -0
- package/dist/lock/process-journal.d.ts +5 -2
- package/dist/lock/process-journal.js.map +1 -1
- package/dist/models/capabilities.d.ts +7 -0
- package/dist/models/capabilities.js +17 -1
- package/dist/models/capabilities.js.map +1 -1
- package/dist/models/compatibility/index.d.ts +10 -0
- package/dist/models/compatibility/index.js +11 -0
- package/dist/models/compatibility/index.js.map +1 -0
- package/dist/models/compatibility/inspect.d.ts +28 -0
- package/dist/models/compatibility/inspect.js +90 -0
- package/dist/models/compatibility/inspect.js.map +1 -0
- package/dist/models/compatibility/resolve.d.ts +59 -0
- package/dist/models/compatibility/resolve.js +219 -0
- package/dist/models/compatibility/resolve.js.map +1 -0
- package/dist/models/compatibility/rules-baseline.d.ts +2 -0
- package/dist/models/compatibility/rules-baseline.js +10 -0
- package/dist/models/compatibility/rules-baseline.js.map +1 -0
- package/dist/models/compatibility/rules-baseline.json +292 -0
- package/dist/models/compatibility/rules-service.d.ts +36 -0
- package/dist/models/compatibility/rules-service.js +109 -0
- package/dist/models/compatibility/rules-service.js.map +1 -0
- package/dist/models/compatibility/rules.d.ts +81 -0
- package/dist/models/compatibility/rules.js +164 -0
- package/dist/models/compatibility/rules.js.map +1 -0
- package/dist/models/compatibility/service.d.ts +62 -0
- package/dist/models/compatibility/service.js +138 -0
- package/dist/models/compatibility/service.js.map +1 -0
- package/dist/models/decision-checkpoint.d.ts +22 -0
- package/dist/models/decision-checkpoint.js +33 -0
- package/dist/models/decision-checkpoint.js.map +1 -0
- package/dist/models/gguf/classify.d.ts +27 -1
- package/dist/models/gguf/classify.js +78 -2
- package/dist/models/gguf/classify.js.map +1 -1
- package/dist/models/gguf/index.d.ts +1 -0
- package/dist/models/gguf/index.js +1 -0
- package/dist/models/gguf/index.js.map +1 -1
- package/dist/models/gguf/tensors.d.ts +57 -0
- package/dist/models/gguf/tensors.js +239 -0
- package/dist/models/gguf/tensors.js.map +1 -0
- package/dist/models/index.d.ts +2 -0
- package/dist/models/index.js +2 -0
- package/dist/models/index.js.map +1 -1
- package/dist/router/resolve.d.ts +7 -1
- package/dist/router/resolve.js +14 -1
- package/dist/router/resolve.js.map +1 -1
- package/dist/runtime/container/argv.d.ts +88 -0
- package/dist/runtime/container/argv.js +406 -0
- package/dist/runtime/container/argv.js.map +1 -0
- package/dist/runtime/container/docker-binary.d.ts +4 -0
- package/dist/runtime/container/docker-binary.js +34 -0
- package/dist/runtime/container/docker-binary.js.map +1 -0
- package/dist/runtime/container/env.d.ts +17 -0
- package/dist/runtime/container/env.js +52 -0
- package/dist/runtime/container/env.js.map +1 -0
- package/dist/runtime/container/exec.d.ts +27 -0
- package/dist/runtime/container/exec.js +79 -0
- package/dist/runtime/container/exec.js.map +1 -0
- package/dist/runtime/container/execution-journal.d.ts +55 -0
- package/dist/runtime/container/execution-journal.js +167 -0
- package/dist/runtime/container/execution-journal.js.map +1 -0
- package/dist/runtime/container/heartbeat.d.ts +47 -0
- package/dist/runtime/container/heartbeat.js +78 -0
- package/dist/runtime/container/heartbeat.js.map +1 -0
- package/dist/runtime/container/index.d.ts +30 -0
- package/dist/runtime/container/index.js +31 -0
- package/dist/runtime/container/index.js.map +1 -0
- package/dist/runtime/container/operations.d.ts +40 -0
- package/dist/runtime/container/operations.js +233 -0
- package/dist/runtime/container/operations.js.map +1 -0
- package/dist/runtime/container/pull.d.ts +48 -0
- package/dist/runtime/container/pull.js +280 -0
- package/dist/runtime/container/pull.js.map +1 -0
- package/dist/runtime/container/reconcile.d.ts +34 -0
- package/dist/runtime/container/reconcile.js +79 -0
- package/dist/runtime/container/reconcile.js.map +1 -0
- package/dist/runtime/container/types.d.ts +124 -0
- package/dist/runtime/container/types.js +2 -0
- package/dist/runtime/container/types.js.map +1 -0
- package/dist/runtime/container/watchdog.d.ts +114 -0
- package/dist/runtime/container/watchdog.js +418 -0
- package/dist/runtime/container/watchdog.js.map +1 -0
- package/dist/runtime/container/wiring.d.ts +78 -0
- package/dist/runtime/container/wiring.js +92 -0
- package/dist/runtime/container/wiring.js.map +1 -0
- package/dist/runtime/container/wsl-exec.d.ts +13 -0
- package/dist/runtime/container/wsl-exec.js +18 -0
- package/dist/runtime/container/wsl-exec.js.map +1 -0
- package/dist/runtime/environment/cached-document.d.ts +128 -0
- package/dist/runtime/environment/cached-document.js +218 -0
- package/dist/runtime/environment/cached-document.js.map +1 -0
- package/dist/runtime/environment/canonical-json.d.ts +78 -0
- package/dist/runtime/environment/canonical-json.js +86 -0
- package/dist/runtime/environment/canonical-json.js.map +1 -0
- package/dist/runtime/environment/descriptor-provider.d.ts +121 -0
- package/dist/runtime/environment/descriptor-provider.js +159 -0
- package/dist/runtime/environment/descriptor-provider.js.map +1 -0
- package/dist/runtime/environment/descriptor.d.ts +35 -0
- package/dist/runtime/environment/descriptor.js +239 -0
- package/dist/runtime/environment/descriptor.js.map +1 -0
- package/dist/runtime/environment/diagnostics.d.ts +50 -0
- package/dist/runtime/environment/diagnostics.js +114 -0
- package/dist/runtime/environment/diagnostics.js.map +1 -0
- package/dist/runtime/environment/document-fields.d.ts +39 -0
- package/dist/runtime/environment/document-fields.js +96 -0
- package/dist/runtime/environment/document-fields.js.map +1 -0
- package/dist/runtime/environment/environment-manifest-provider.d.ts +65 -0
- package/dist/runtime/environment/environment-manifest-provider.js +89 -0
- package/dist/runtime/environment/environment-manifest-provider.js.map +1 -0
- package/dist/runtime/environment/environment-manifest.d.ts +35 -0
- package/dist/runtime/environment/environment-manifest.js +131 -0
- package/dist/runtime/environment/environment-manifest.js.map +1 -0
- package/dist/runtime/environment/guest-host.d.ts +43 -0
- package/dist/runtime/environment/guest-host.js +88 -0
- package/dist/runtime/environment/guest-host.js.map +1 -0
- package/dist/runtime/environment/host-exec.d.ts +38 -0
- package/dist/runtime/environment/host-exec.js +145 -0
- package/dist/runtime/environment/host-exec.js.map +1 -0
- package/dist/runtime/environment/host-step-failure.d.ts +33 -0
- package/dist/runtime/environment/host-step-failure.js +34 -0
- package/dist/runtime/environment/host-step-failure.js.map +1 -0
- package/dist/runtime/environment/index.d.ts +56 -0
- package/dist/runtime/environment/index.js +57 -0
- package/dist/runtime/environment/index.js.map +1 -0
- package/dist/runtime/environment/installations.d.ts +25 -0
- package/dist/runtime/environment/installations.js +70 -0
- package/dist/runtime/environment/installations.js.map +1 -0
- package/dist/runtime/environment/inventory.d.ts +42 -0
- package/dist/runtime/environment/inventory.js +57 -0
- package/dist/runtime/environment/inventory.js.map +1 -0
- package/dist/runtime/environment/linux-blockers.d.ts +130 -0
- package/dist/runtime/environment/linux-blockers.js +251 -0
- package/dist/runtime/environment/linux-blockers.js.map +1 -0
- package/dist/runtime/environment/linux-docker-facts.d.ts +200 -0
- package/dist/runtime/environment/linux-docker-facts.js +342 -0
- package/dist/runtime/environment/linux-docker-facts.js.map +1 -0
- package/dist/runtime/environment/linux-docker-network.d.ts +80 -0
- package/dist/runtime/environment/linux-docker-network.js +195 -0
- package/dist/runtime/environment/linux-docker-network.js.map +1 -0
- package/dist/runtime/environment/linux-host.d.ts +22 -0
- package/dist/runtime/environment/linux-host.js +98 -0
- package/dist/runtime/environment/linux-host.js.map +1 -0
- package/dist/runtime/environment/linux-plan.d.ts +126 -0
- package/dist/runtime/environment/linux-plan.js +392 -0
- package/dist/runtime/environment/linux-plan.js.map +1 -0
- package/dist/runtime/environment/linux-probe.d.ts +245 -0
- package/dist/runtime/environment/linux-probe.js +315 -0
- package/dist/runtime/environment/linux-probe.js.map +1 -0
- package/dist/runtime/environment/linux-provisioner.d.ts +102 -0
- package/dist/runtime/environment/linux-provisioner.js +734 -0
- package/dist/runtime/environment/linux-provisioner.js.map +1 -0
- package/dist/runtime/environment/provisioner-descriptors.d.ts +17 -0
- package/dist/runtime/environment/provisioner-descriptors.js +62 -0
- package/dist/runtime/environment/provisioner-descriptors.js.map +1 -0
- package/dist/runtime/environment/recovery.d.ts +74 -0
- package/dist/runtime/environment/recovery.js +121 -0
- package/dist/runtime/environment/recovery.js.map +1 -0
- package/dist/runtime/environment/rootfs-download.d.ts +2 -0
- package/dist/runtime/environment/rootfs-download.js +47 -0
- package/dist/runtime/environment/rootfs-download.js.map +1 -0
- package/dist/runtime/environment/service.d.ts +280 -0
- package/dist/runtime/environment/service.js +776 -0
- package/dist/runtime/environment/service.js.map +1 -0
- package/dist/runtime/environment/state.d.ts +163 -0
- package/dist/runtime/environment/state.js +479 -0
- package/dist/runtime/environment/state.js.map +1 -0
- package/dist/runtime/environment/store-models.d.ts +19 -0
- package/dist/runtime/environment/store-models.js +30 -0
- package/dist/runtime/environment/store-models.js.map +1 -0
- package/dist/runtime/environment/store.d.ts +210 -0
- package/dist/runtime/environment/store.js +438 -0
- package/dist/runtime/environment/store.js.map +1 -0
- package/dist/runtime/environment/windows-environment-record.d.ts +48 -0
- package/dist/runtime/environment/windows-environment-record.js +89 -0
- package/dist/runtime/environment/windows-environment-record.js.map +1 -0
- package/dist/runtime/environment/windows-forwarding.d.ts +27 -0
- package/dist/runtime/environment/windows-forwarding.js +84 -0
- package/dist/runtime/environment/windows-forwarding.js.map +1 -0
- package/dist/runtime/environment/windows-guest-setup.d.ts +48 -0
- package/dist/runtime/environment/windows-guest-setup.js +157 -0
- package/dist/runtime/environment/windows-guest-setup.js.map +1 -0
- package/dist/runtime/environment/windows-host.d.ts +46 -0
- package/dist/runtime/environment/windows-host.js +163 -0
- package/dist/runtime/environment/windows-host.js.map +1 -0
- package/dist/runtime/environment/windows-plan.d.ts +78 -0
- package/dist/runtime/environment/windows-plan.js +216 -0
- package/dist/runtime/environment/windows-plan.js.map +1 -0
- package/dist/runtime/environment/windows-probe.d.ts +135 -0
- package/dist/runtime/environment/windows-probe.js +261 -0
- package/dist/runtime/environment/windows-probe.js.map +1 -0
- package/dist/runtime/environment/windows-provisioner.d.ts +113 -0
- package/dist/runtime/environment/windows-provisioner.js +782 -0
- package/dist/runtime/environment/windows-provisioner.js.map +1 -0
- package/dist/runtime/environment/wiring.d.ts +117 -0
- package/dist/runtime/environment/wiring.js +354 -0
- package/dist/runtime/environment/wiring.js.map +1 -0
- package/dist/runtime/foundation-models/runtime.d.ts +8 -0
- package/dist/runtime/foundation-models/runtime.js +5 -2
- package/dist/runtime/foundation-models/runtime.js.map +1 -1
- package/dist/runtime/llamacpp/args.d.ts +10 -1
- package/dist/runtime/llamacpp/args.js +43 -7
- package/dist/runtime/llamacpp/args.js.map +1 -1
- package/dist/runtime/llamacpp/fit-margin.d.ts +46 -0
- package/dist/runtime/llamacpp/fit-margin.js +47 -0
- package/dist/runtime/llamacpp/fit-margin.js.map +1 -0
- package/dist/runtime/llamacpp/index.d.ts +1 -0
- package/dist/runtime/llamacpp/index.js +1 -0
- package/dist/runtime/llamacpp/index.js.map +1 -1
- package/dist/runtime/llamacpp/load-plan.d.ts +23 -1
- package/dist/runtime/llamacpp/load-plan.js +82 -1
- package/dist/runtime/llamacpp/load-plan.js.map +1 -1
- package/dist/runtime/llamacpp/policy.d.ts +16 -0
- package/dist/runtime/llamacpp/policy.js +20 -0
- package/dist/runtime/llamacpp/policy.js.map +1 -1
- package/dist/runtime/llamacpp/probe.d.ts +21 -1
- package/dist/runtime/llamacpp/probe.js +32 -6
- package/dist/runtime/llamacpp/probe.js.map +1 -1
- package/dist/runtime/llamacpp/runtime.d.ts +56 -1
- package/dist/runtime/llamacpp/runtime.js +142 -15
- package/dist/runtime/llamacpp/runtime.js.map +1 -1
- package/dist/runtime/managed-engines/index.d.ts +8 -0
- package/dist/runtime/managed-engines/index.js +9 -0
- package/dist/runtime/managed-engines/index.js.map +1 -0
- package/dist/runtime/managed-engines/runtime.d.ts +140 -0
- package/dist/runtime/managed-engines/runtime.js +402 -0
- package/dist/runtime/managed-engines/runtime.js.map +1 -0
- package/dist/runtime/managed-engines/spec.d.ts +70 -0
- package/dist/runtime/managed-engines/spec.js +39 -0
- package/dist/runtime/managed-engines/spec.js.map +1 -0
- package/dist/runtime/managed-models/check.d.ts +50 -0
- package/dist/runtime/managed-models/check.js +157 -0
- package/dist/runtime/managed-models/check.js.map +1 -0
- package/dist/runtime/managed-models/compatibility.d.ts +256 -0
- package/dist/runtime/managed-models/compatibility.js +594 -0
- package/dist/runtime/managed-models/compatibility.js.map +1 -0
- package/dist/runtime/managed-models/delete.d.ts +15 -0
- package/dist/runtime/managed-models/delete.js +60 -0
- package/dist/runtime/managed-models/delete.js.map +1 -0
- package/dist/runtime/managed-models/guest-files.d.ts +8 -0
- package/dist/runtime/managed-models/guest-files.js +49 -0
- package/dist/runtime/managed-models/guest-files.js.map +1 -0
- package/dist/runtime/managed-models/index.d.ts +17 -0
- package/dist/runtime/managed-models/index.js +18 -0
- package/dist/runtime/managed-models/index.js.map +1 -0
- package/dist/runtime/managed-models/location.d.ts +22 -0
- package/dist/runtime/managed-models/location.js +53 -0
- package/dist/runtime/managed-models/location.js.map +1 -0
- package/dist/runtime/managed-models/migrate.d.ts +30 -0
- package/dist/runtime/managed-models/migrate.js +119 -0
- package/dist/runtime/managed-models/migrate.js.map +1 -0
- package/dist/runtime/managed-models/model-dir.d.ts +36 -0
- package/dist/runtime/managed-models/model-dir.js +136 -0
- package/dist/runtime/managed-models/model-dir.js.map +1 -0
- package/dist/runtime/managed-models/quant-format.d.ts +71 -0
- package/dist/runtime/managed-models/quant-format.js +299 -0
- package/dist/runtime/managed-models/quant-format.js.map +1 -0
- package/dist/runtime/managed-models/registry.d.ts +26 -0
- package/dist/runtime/managed-models/registry.js +66 -0
- package/dist/runtime/managed-models/registry.js.map +1 -0
- package/dist/runtime/managed-text/adapter.d.ts +283 -0
- package/dist/runtime/managed-text/adapter.js +96 -0
- package/dist/runtime/managed-text/adapter.js.map +1 -0
- package/dist/runtime/managed-text/backend-target.d.ts +3 -0
- package/dist/runtime/managed-text/backend-target.js +52 -0
- package/dist/runtime/managed-text/backend-target.js.map +1 -0
- package/dist/runtime/managed-text/deployment.d.ts +14 -0
- package/dist/runtime/managed-text/deployment.js +32 -0
- package/dist/runtime/managed-text/deployment.js.map +1 -0
- package/dist/runtime/managed-text/engine-cache.d.ts +15 -0
- package/dist/runtime/managed-text/engine-cache.js +64 -0
- package/dist/runtime/managed-text/engine-cache.js.map +1 -0
- package/dist/runtime/managed-text/gateway.d.ts +158 -0
- package/dist/runtime/managed-text/gateway.js +535 -0
- package/dist/runtime/managed-text/gateway.js.map +1 -0
- package/dist/runtime/managed-text/generation-defaults.d.ts +23 -0
- package/dist/runtime/managed-text/generation-defaults.js +51 -0
- package/dist/runtime/managed-text/generation-defaults.js.map +1 -0
- package/dist/runtime/managed-text/index.d.ts +26 -0
- package/dist/runtime/managed-text/index.js +27 -0
- package/dist/runtime/managed-text/index.js.map +1 -0
- package/dist/runtime/managed-text/lifecycle.d.ts +284 -0
- package/dist/runtime/managed-text/lifecycle.js +896 -0
- package/dist/runtime/managed-text/lifecycle.js.map +1 -0
- package/dist/runtime/managed-text/load-policy.d.ts +47 -0
- package/dist/runtime/managed-text/load-policy.js +111 -0
- package/dist/runtime/managed-text/load-policy.js.map +1 -0
- package/dist/runtime/managed-text/mount-source.d.ts +14 -0
- package/dist/runtime/managed-text/mount-source.js +18 -0
- package/dist/runtime/managed-text/mount-source.js.map +1 -0
- package/dist/runtime/managed-text/readiness.d.ts +33 -0
- package/dist/runtime/managed-text/readiness.js +88 -0
- package/dist/runtime/managed-text/readiness.js.map +1 -0
- package/dist/runtime/managed-text/request-rules.d.ts +18 -0
- package/dist/runtime/managed-text/request-rules.js +62 -0
- package/dist/runtime/managed-text/request-rules.js.map +1 -0
- package/dist/runtime/managed-text/types.d.ts +88 -0
- package/dist/runtime/managed-text/types.js +12 -0
- package/dist/runtime/managed-text/types.js.map +1 -0
- package/dist/runtime/managed-text/wsl-deployment.d.ts +24 -0
- package/dist/runtime/managed-text/wsl-deployment.js +70 -0
- package/dist/runtime/managed-text/wsl-deployment.js.map +1 -0
- package/dist/runtime/mlx/errors.d.ts +6 -1
- package/dist/runtime/mlx/errors.js +7 -2
- package/dist/runtime/mlx/errors.js.map +1 -1
- package/dist/runtime/mlx/runtime.d.ts +36 -1
- package/dist/runtime/mlx/runtime.js +85 -4
- package/dist/runtime/mlx/runtime.js.map +1 -1
- package/dist/runtime/shared/backend-output.d.ts +23 -0
- package/dist/runtime/shared/backend-output.js +32 -0
- package/dist/runtime/shared/backend-output.js.map +1 -0
- package/dist/runtime/shared/bounded-output.d.ts +24 -0
- package/dist/runtime/shared/bounded-output.js +63 -0
- package/dist/runtime/shared/bounded-output.js.map +1 -0
- package/dist/runtime/shared/external-sessions.js +9 -1
- package/dist/runtime/shared/external-sessions.js.map +1 -1
- package/dist/runtime/shared/gpu-occupancy.d.ts +60 -0
- package/dist/runtime/shared/gpu-occupancy.js +10 -0
- package/dist/runtime/shared/gpu-occupancy.js.map +1 -0
- package/dist/runtime/shared/index.d.ts +4 -0
- package/dist/runtime/shared/index.js +4 -0
- package/dist/runtime/shared/index.js.map +1 -1
- package/dist/runtime/shared/local-runtime.d.ts +35 -0
- package/dist/runtime/shared/ports.d.ts +16 -0
- package/dist/runtime/shared/ports.js +42 -1
- package/dist/runtime/shared/ports.js.map +1 -1
- package/dist/runtime/shared/process.d.ts +11 -0
- package/dist/runtime/shared/process.js +20 -0
- package/dist/runtime/shared/process.js.map +1 -1
- package/dist/runtime/shared/redact-args.d.ts +7 -0
- package/dist/runtime/shared/redact-args.js +26 -0
- package/dist/runtime/shared/redact-args.js.map +1 -0
- package/dist/runtime/shared/sidecar.d.ts +15 -0
- package/dist/runtime/shared/sidecar.js +43 -8
- package/dist/runtime/shared/sidecar.js.map +1 -1
- package/dist/runtime/tensorrt-llm/adapter.d.ts +247 -0
- package/dist/runtime/tensorrt-llm/adapter.js +1028 -0
- package/dist/runtime/tensorrt-llm/adapter.js.map +1 -0
- package/dist/runtime/tensorrt-llm/check.d.ts +12 -0
- package/dist/runtime/tensorrt-llm/check.js +22 -0
- package/dist/runtime/tensorrt-llm/check.js.map +1 -0
- package/dist/runtime/tensorrt-llm/compatibility.d.ts +124 -0
- package/dist/runtime/tensorrt-llm/compatibility.js +213 -0
- package/dist/runtime/tensorrt-llm/compatibility.js.map +1 -0
- package/dist/runtime/tensorrt-llm/cuda-graphs.d.ts +32 -0
- package/dist/runtime/tensorrt-llm/cuda-graphs.js +33 -0
- package/dist/runtime/tensorrt-llm/cuda-graphs.js.map +1 -0
- package/dist/runtime/tensorrt-llm/delete.d.ts +15 -0
- package/dist/runtime/tensorrt-llm/delete.js +60 -0
- package/dist/runtime/tensorrt-llm/delete.js.map +1 -0
- package/dist/runtime/tensorrt-llm/engine.d.ts +3 -0
- package/dist/runtime/tensorrt-llm/engine.js +30 -0
- package/dist/runtime/tensorrt-llm/engine.js.map +1 -0
- package/dist/runtime/tensorrt-llm/guest-files.d.ts +8 -0
- package/dist/runtime/tensorrt-llm/guest-files.js +49 -0
- package/dist/runtime/tensorrt-llm/guest-files.js.map +1 -0
- package/dist/runtime/tensorrt-llm/host-facts.d.ts +55 -0
- package/dist/runtime/tensorrt-llm/host-facts.js +87 -0
- package/dist/runtime/tensorrt-llm/host-facts.js.map +1 -0
- package/dist/runtime/tensorrt-llm/index.d.ts +30 -0
- package/dist/runtime/tensorrt-llm/index.js +31 -0
- package/dist/runtime/tensorrt-llm/index.js.map +1 -0
- package/dist/runtime/tensorrt-llm/installation.d.ts +23 -0
- package/dist/runtime/tensorrt-llm/installation.js +53 -0
- package/dist/runtime/tensorrt-llm/installation.js.map +1 -0
- package/dist/runtime/tensorrt-llm/kv-cache.d.ts +41 -0
- package/dist/runtime/tensorrt-llm/kv-cache.js +48 -0
- package/dist/runtime/tensorrt-llm/kv-cache.js.map +1 -0
- package/dist/runtime/tensorrt-llm/location.d.ts +20 -0
- package/dist/runtime/tensorrt-llm/location.js +48 -0
- package/dist/runtime/tensorrt-llm/location.js.map +1 -0
- package/dist/runtime/tensorrt-llm/model-dir.d.ts +38 -0
- package/dist/runtime/tensorrt-llm/model-dir.js +137 -0
- package/dist/runtime/tensorrt-llm/model-dir.js.map +1 -0
- package/dist/runtime/tensorrt-llm/prelaunch.d.ts +20 -0
- package/dist/runtime/tensorrt-llm/prelaunch.js +168 -0
- package/dist/runtime/tensorrt-llm/prelaunch.js.map +1 -0
- package/dist/runtime/tensorrt-llm/quant-format.d.ts +65 -0
- package/dist/runtime/tensorrt-llm/quant-format.js +189 -0
- package/dist/runtime/tensorrt-llm/quant-format.js.map +1 -0
- package/dist/runtime/tensorrt-llm/registry.d.ts +26 -0
- package/dist/runtime/tensorrt-llm/registry.js +68 -0
- package/dist/runtime/tensorrt-llm/registry.js.map +1 -0
- package/dist/runtime/tensorrt-llm/route-policy.d.ts +19 -0
- package/dist/runtime/tensorrt-llm/route-policy.js +17 -0
- package/dist/runtime/tensorrt-llm/route-policy.js.map +1 -0
- package/dist/runtime/tensorrt-llm/runtime.d.ts +130 -0
- package/dist/runtime/tensorrt-llm/runtime.js +395 -0
- package/dist/runtime/tensorrt-llm/runtime.js.map +1 -0
- package/dist/runtime/tensorrt-llm/settings.d.ts +7 -0
- package/dist/runtime/tensorrt-llm/settings.js +48 -0
- package/dist/runtime/tensorrt-llm/settings.js.map +1 -0
- package/dist/runtime/vllm/adapter.d.ts +46 -0
- package/dist/runtime/vllm/adapter.js +425 -0
- package/dist/runtime/vllm/adapter.js.map +1 -0
- package/dist/runtime/vllm/engine.d.ts +16 -0
- package/dist/runtime/vllm/engine.js +37 -0
- package/dist/runtime/vllm/engine.js.map +1 -0
- package/dist/runtime/vllm/index.d.ts +11 -0
- package/dist/runtime/vllm/index.js +12 -0
- package/dist/runtime/vllm/index.js.map +1 -0
- package/dist/runtime/vllm/memory.d.ts +69 -0
- package/dist/runtime/vllm/memory.js +123 -0
- package/dist/runtime/vllm/memory.js.map +1 -0
- package/dist/runtime/vllm/settings.d.ts +52 -0
- package/dist/runtime/vllm/settings.js +151 -0
- package/dist/runtime/vllm/settings.js.map +1 -0
- package/dist/runtime/wsl/guest-paths.d.ts +33 -0
- package/dist/runtime/wsl/guest-paths.js +83 -0
- package/dist/runtime/wsl/guest-paths.js.map +1 -0
- package/dist/runtime/wsl/guest-scope.d.ts +14 -0
- package/dist/runtime/wsl/guest-scope.js +98 -0
- package/dist/runtime/wsl/guest-scope.js.map +1 -0
- package/dist/runtime/wsl/index.d.ts +12 -0
- package/dist/runtime/wsl/index.js +13 -0
- package/dist/runtime/wsl/index.js.map +1 -0
- package/dist/runtime/wsl/keeper.d.ts +29 -0
- package/dist/runtime/wsl/keeper.js +48 -0
- package/dist/runtime/wsl/keeper.js.map +1 -0
- package/dist/runtime/wsl/transport.d.ts +106 -0
- package/dist/runtime/wsl/transport.js +181 -0
- package/dist/runtime/wsl/transport.js.map +1 -0
- package/dist/server/control/index.d.ts +1 -1
- package/dist/server/control/router.js +16 -1
- package/dist/server/control/router.js.map +1 -1
- package/dist/server/control/routes/backends.js +26 -0
- package/dist/server/control/routes/backends.js.map +1 -1
- package/dist/server/control/routes/decision.d.ts +10 -0
- package/dist/server/control/routes/decision.js +43 -0
- package/dist/server/control/routes/decision.js.map +1 -0
- package/dist/server/control/routes/diffusion-video.js +4 -0
- package/dist/server/control/routes/diffusion-video.js.map +1 -1
- package/dist/server/control/routes/diffusion.d.ts +1 -1
- package/dist/server/control/routes/diffusion.js +4 -9
- package/dist/server/control/routes/diffusion.js.map +1 -1
- package/dist/server/control/routes/embedding.d.ts +9 -0
- package/dist/server/control/routes/embedding.js +43 -0
- package/dist/server/control/routes/embedding.js.map +1 -0
- package/dist/server/control/routes/engine-builds.d.ts +14 -0
- package/dist/server/control/routes/engine-builds.js +82 -0
- package/dist/server/control/routes/engine-builds.js.map +1 -0
- package/dist/server/control/routes/engines.d.ts +17 -0
- package/dist/server/control/routes/engines.js +152 -0
- package/dist/server/control/routes/engines.js.map +1 -0
- package/dist/server/control/routes/environments.d.ts +16 -0
- package/dist/server/control/routes/environments.js +207 -0
- package/dist/server/control/routes/environments.js.map +1 -0
- package/dist/server/control/routes/hardware.d.ts +1 -1
- package/dist/server/control/routes/hardware.js +11 -6
- package/dist/server/control/routes/hardware.js.map +1 -1
- package/dist/server/control/routes/model-setups.d.ts +11 -0
- package/dist/server/control/routes/model-setups.js +55 -0
- package/dist/server/control/routes/model-setups.js.map +1 -0
- package/dist/server/control/routes/models.d.ts +3 -2
- package/dist/server/control/routes/models.js +45 -2
- package/dist/server/control/routes/models.js.map +1 -1
- package/dist/server/control/types.d.ts +172 -10
- package/dist/server/control/types.js.map +1 -1
- package/dist/server/http.d.ts +9 -1
- package/dist/server/http.js +51 -1
- package/dist/server/http.js.map +1 -1
- package/dist/server/public/decision-preview.d.ts +17 -0
- package/dist/server/public/decision-preview.js +119 -0
- package/dist/server/public/decision-preview.js.map +1 -0
- package/dist/server/public/decision.d.ts +32 -0
- package/dist/server/public/decision.js +135 -0
- package/dist/server/public/decision.js.map +1 -0
- package/dist/server/public/embedding.d.ts +31 -0
- package/dist/server/public/embedding.js +126 -0
- package/dist/server/public/embedding.js.map +1 -0
- package/dist/server/public/errors.d.ts +2 -1
- package/dist/server/public/errors.js +5 -2
- package/dist/server/public/errors.js.map +1 -1
- package/dist/server/public/forward.js +75 -3
- package/dist/server/public/forward.js.map +1 -1
- package/dist/server/public/gates.d.ts +18 -0
- package/dist/server/public/gates.js +41 -14
- package/dist/server/public/gates.js.map +1 -1
- package/dist/server/public/index.d.ts +6 -2
- package/dist/server/public/index.js +23 -7
- package/dist/server/public/index.js.map +1 -1
- package/dist/server/public/listing.d.ts +13 -2
- package/dist/server/public/listing.js +33 -9
- package/dist/server/public/listing.js.map +1 -1
- package/dist/server/public/policy.d.ts +9 -0
- package/dist/server/public/policy.js +68 -0
- package/dist/server/public/policy.js.map +1 -0
- package/dist/server/public/responses.js +10 -0
- package/dist/server/public/responses.js.map +1 -1
- package/dist/server/public/trace.d.ts +2 -2
- package/dist/server/public/trace.js +6 -2
- package/dist/server/public/trace.js.map +1 -1
- package/dist/server/public/types.d.ts +91 -1
- package/dist/settings/decision.d.ts +15 -0
- package/dist/settings/decision.js +88 -0
- package/dist/settings/decision.js.map +1 -0
- package/dist/settings/embedding.d.ts +12 -0
- package/dist/settings/embedding.js +82 -0
- package/dist/settings/embedding.js.map +1 -0
- package/dist/settings/index.d.ts +2 -0
- package/dist/settings/index.js +2 -0
- package/dist/settings/index.js.map +1 -1
- package/dist/settings/schema/atomic-prism.json +433 -0
- package/dist/settings/schema/tensorrt-llm.json +253 -0
- package/dist/settings/schema/vllm.json +273 -0
- package/dist/settings/schema.js +58 -0
- package/dist/settings/schema.js.map +1 -1
- package/dist/settings/store.d.ts +13 -2
- package/dist/settings/store.js +44 -0
- package/dist/settings/store.js.map +1 -1
- package/dist/util/index.d.ts +1 -0
- package/dist/util/index.js +1 -0
- package/dist/util/index.js.map +1 -1
- package/dist/util/result.d.ts +19 -0
- package/dist/util/result.js +15 -0
- package/dist/util/result.js.map +1 -0
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/version.js.map +1 -1
- package/package.json +2 -2
- package/src/settings/schema/atomic-prism.json +433 -0
- package/src/settings/schema/tensorrt-llm.json +253 -0
- package/src/settings/schema/vllm.json +273 -0
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
* always received; the wording is part of the contract (the host message tells the user where in
|
|
10
10
|
* Settings to fix it).
|
|
11
11
|
*/
|
|
12
|
+
import { timingSafeEqual } from 'node:crypto';
|
|
12
13
|
/** Paths that answer without the host and key checks: the API documentation. */
|
|
13
14
|
export const WHITELISTED_PATHS = new Set([
|
|
14
15
|
'/',
|
|
@@ -112,6 +113,17 @@ function header(req, name) {
|
|
|
112
113
|
const value = req.headers[name];
|
|
113
114
|
return typeof value === 'string' ? value : Array.isArray(value) ? (value[0] ?? '') : '';
|
|
114
115
|
}
|
|
116
|
+
/**
|
|
117
|
+
* `a === b`, but on the time a key comparison takes rather than where the first differing byte is —
|
|
118
|
+
* a client guessing the key one byte at a time cannot use response latency to find it. Unequal
|
|
119
|
+
* lengths are rejected before `timingSafeEqual`, which throws rather than compares on those; the
|
|
120
|
+
* length itself is not the secret, so leaking it costs nothing a `===` check would not already.
|
|
121
|
+
*/
|
|
122
|
+
function timingSafeEqualString(a, b) {
|
|
123
|
+
const bufA = Buffer.from(a);
|
|
124
|
+
const bufB = Buffer.from(b);
|
|
125
|
+
return bufA.length === bufB.length && timingSafeEqual(bufA, bufB);
|
|
126
|
+
}
|
|
115
127
|
export function hostMessage(host) {
|
|
116
128
|
return (`Host '${host}' is not in Trusted Hosts. Add this server's address (e.g. its LAN IP or hostname) ` +
|
|
117
129
|
`in Settings → Local API Server → Trusted Hosts, or use '*' to allow all.`);
|
|
@@ -150,29 +162,44 @@ export function preflight(req, config) {
|
|
|
150
162
|
}
|
|
151
163
|
return { status: 200, headers, body: '' };
|
|
152
164
|
}
|
|
165
|
+
/**
|
|
166
|
+
* The Host and Bearer-key checks alone, with no path whitelist, CORS or hidden-path behaviour: what
|
|
167
|
+
* a listener that has no docs endpoints and must authenticate every request needs (the managed
|
|
168
|
+
* session gateway, `runtime/managed-text/gateway.ts`). `undefined` means both checks passed.
|
|
169
|
+
*/
|
|
170
|
+
export function hostAndKeyGate(req, config) {
|
|
171
|
+
const host = header(req, 'host');
|
|
172
|
+
if (host === '')
|
|
173
|
+
return { status: 400, headers: [], body: 'Missing host header', kind: 'bad_request' };
|
|
174
|
+
if (!isValidHost(host, config.trustedHosts))
|
|
175
|
+
return { status: 403, headers: [], body: hostMessage(host), kind: 'host' };
|
|
176
|
+
const keys = config.apiKeys ?? (config.apiKey !== '' ? [config.apiKey] : undefined);
|
|
177
|
+
if (keys !== undefined) {
|
|
178
|
+
// `Bearer ` is matched exactly, case included, as the proxy does.
|
|
179
|
+
const auth = header(req, 'authorization');
|
|
180
|
+
const bearer = auth.startsWith('Bearer ') ? auth.slice('Bearer '.length) : undefined;
|
|
181
|
+
const xApiKey = header(req, 'x-api-key');
|
|
182
|
+
const matches = (presented) => keys.some((key) => key !== '' && timingSafeEqualString(presented, key));
|
|
183
|
+
const bearerOk = bearer !== undefined && matches(bearer);
|
|
184
|
+
const keyOk = matches(xApiKey);
|
|
185
|
+
if (!bearerOk && !keyOk) {
|
|
186
|
+
return { status: 401, headers: [], body: 'Invalid or missing authorization token', kind: 'auth' };
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
return undefined;
|
|
190
|
+
}
|
|
153
191
|
/**
|
|
154
192
|
* The host, key and hidden-path checks, in the proxy's order. `undefined` means the request passes
|
|
155
193
|
* and should be routed.
|
|
156
194
|
*/
|
|
157
195
|
export function gate(req, path, config) {
|
|
158
|
-
const host = header(req, 'host');
|
|
159
196
|
const origin = header(req, 'origin');
|
|
160
197
|
const cors = corsHeaders(origin, config.trustedHosts);
|
|
161
198
|
const whitelisted = WHITELISTED_PATHS.has(path);
|
|
162
199
|
if (!whitelisted) {
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
return { status: 403, headers: cors, body: hostMessage(host), kind: 'host' };
|
|
167
|
-
if (config.apiKey !== '') {
|
|
168
|
-
// `Bearer ` is matched exactly, case included, as the proxy does.
|
|
169
|
-
const auth = header(req, 'authorization');
|
|
170
|
-
const bearerOk = auth.startsWith('Bearer ') && auth.slice('Bearer '.length) === config.apiKey;
|
|
171
|
-
const keyOk = header(req, 'x-api-key') === config.apiKey;
|
|
172
|
-
if (!bearerOk && !keyOk) {
|
|
173
|
-
return { status: 401, headers: cors, body: 'Invalid or missing authorization token', kind: 'auth' };
|
|
174
|
-
}
|
|
175
|
-
}
|
|
200
|
+
const refused = hostAndKeyGate(req, config);
|
|
201
|
+
if (refused)
|
|
202
|
+
return { ...refused, headers: cors };
|
|
176
203
|
}
|
|
177
204
|
if (path.includes('/configs'))
|
|
178
205
|
return { status: 404, headers: cors, body: 'Not Found', kind: 'hidden' };
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"gates.js","sourceRoot":"","sources":["../../../src/server/public/gates.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;
|
|
1
|
+
{"version":3,"file":"gates.js","sourceRoot":"","sources":["../../../src/server/public/gates.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAI7C,gFAAgF;AAChF,MAAM,CAAC,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC;IACvC,GAAG;IACH,eAAe;IACf,sBAAsB;IACtB,4BAA4B;IAC5B,uCAAuC;CACxC,CAAC,CAAA;AAEF,qDAAqD;AACrD,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,eAAe,CAAC,CAAC,CAAA;AAE3D,MAAM,mBAAmB,GAAG,CAAC,WAAW,EAAE,WAAW,EAAE,SAAS,EAAE,sBAAsB,CAAC,CAAA;AAEzF,MAAM,CAAC,MAAM,kBAAkB,GAAG,wCAAwC,CAAA;AAC1E,MAAM,CAAC,MAAM,kBAAkB,GAC7B,6QAA6Q,CAAA;AAE/Q,MAAM,iBAAiB,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,CAAC,CAAA;AAC9E,MAAM,iBAAiB,GAAG;IACxB,QAAQ;IACR,iBAAiB;IACjB,eAAe;IACf,eAAe;IACf,YAAY;IACZ,cAAc;IACd,KAAK;IACL,MAAM;IACN,mBAAmB;IACnB,YAAY;IACZ,QAAQ;IACR,YAAY;IACZ,WAAW;IACX,cAAc;IACd,iBAAiB;IACjB,kBAAkB;IAClB,mBAAmB;IACnB,kBAAkB;IAClB,kBAAkB;IAClB,kBAAkB;IAClB,gBAAgB;IAChB,6BAA6B;IAC7B,yBAAyB;IACzB,qBAAqB;IACrB,6BAA6B;IAC7B,qBAAqB;CACtB,CAAA;AAWD;;;;GAIG;AACH,MAAM,UAAU,YAAY,CAAC,IAAY,EAAE,MAAc;IACvD,IAAI,MAAM,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;QAAE,OAAO,IAAI,CAAA;IAC1D,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;IACtC,IAAI,IAAI,KAAK,EAAE;QAAE,OAAO,GAAG,CAAA;IAC3B,OAAO,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAA;AACjD,CAAC;AAED,iFAAiF;AACjF,MAAM,UAAU,qBAAqB,CAAC,MAAc;IAClD,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;IACjC,MAAM,WAAW,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;IAC5B,IAAI,WAAW,KAAK,SAAS;QAAE,OAAO,MAAM,CAAA;IAC5C,OAAO,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,WAAW,CAAA;AACjD,CAAC;AAED,SAAS,eAAe,CAAC,IAAY;IACnC,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;QAAE,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAA;IACjF,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAA;AACnC,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,WAAW,CAAC,IAAY,EAAE,YAA+B;IACvE,IAAI,YAAY,CAAC,QAAQ,CAAC,GAAG,CAAC;QAAE,OAAO,IAAI,CAAA;IAC3C,IAAI,IAAI,KAAK,EAAE;QAAE,OAAO,KAAK,CAAA;IAC7B,MAAM,IAAI,GAAG,eAAe,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,CAAA;IAChD,IAAI,mBAAmB,CAAC,QAAQ,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAA;IACnD,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,EAAE,CAAA;IAChC,OAAO,YAAY,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE;QACjC,MAAM,UAAU,GAAG,KAAK,CAAC,WAAW,EAAE,CAAA;QACtC,OAAO,KAAK,KAAK,UAAU,IAAI,IAAI,KAAK,eAAe,CAAC,KAAK,CAAC,CAAC,WAAW,EAAE,CAAA;IAC9E,CAAC,CAAC,CAAA;AACJ,CAAC;AAED,iGAAiG;AACjG,MAAM,UAAU,WAAW,CAAC,MAAc,EAAE,YAA+B;IACzE,MAAM,OAAO,GAA4B;QACvC,CAAC,8BAA8B,EAAE,kBAAkB,CAAC;QACpD,CAAC,8BAA8B,EAAE,kBAAkB,CAAC;QACpD,CAAC,MAAM,EAAE,QAAQ,CAAC;KACnB,CAAA;IACD,IAAI,MAAM,KAAK,EAAE,IAAI,WAAW,CAAC,qBAAqB,CAAC,MAAM,CAAC,EAAE,YAAY,CAAC,EAAE,CAAC;QAC9E,OAAO,CAAC,IAAI,CAAC,CAAC,6BAA6B,EAAE,MAAM,CAAC,EAAE,CAAC,kCAAkC,EAAE,MAAM,CAAC,CAAC,CAAA;IACrG,CAAC;IACD,OAAO,OAAO,CAAA;AAChB,CAAC;AAED,SAAS,MAAM,CAAC,GAAoB,EAAE,IAAY;IAChD,MAAM,KAAK,GAAG,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;IAC/B,OAAO,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;AACzF,CAAC;AAED;;;;;GAKG;AACH,SAAS,qBAAqB,CAAC,CAAS,EAAE,CAAS;IACjD,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAC3B,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAC3B,OAAO,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,IAAI,eAAe,CAAC,IAAI,EAAE,IAAI,CAAC,CAAA;AACnE,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,IAAY;IACtC,OAAO,CACL,SAAS,IAAI,qFAAqF;QAClG,0EAA0E,CAC3E,CAAA;AACH,CAAC;AAED,4FAA4F;AAC5F,MAAM,UAAU,SAAS,CAAC,GAAoB,EAAE,MAA0B;IACxE,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,CAAA;IAChC,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAA;IACpC,MAAM,eAAe,GAAG,MAAM,CAAC,GAAG,EAAE,+BAA+B,CAAC,CAAA;IACpE,IACE,eAAe,KAAK,EAAE;QACtB,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,KAAK,eAAe,CAAC,WAAW,EAAE,CAAC,EACjF,CAAC;QACD,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,oBAAoB,EAAE,CAAA;IACjE,CAAC;IAED,mGAAmG;IACnG,MAAM,IAAI,GAAG,CAAC,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,CAAA;IAClD,MAAM,OAAO,GAAG,mBAAmB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,KAAK,EAAE,IAAI,WAAW,CAAC,IAAI,EAAE,MAAM,CAAC,YAAY,CAAC,CAAC,CAAA;IACxG,IAAI,CAAC,OAAO;QAAE,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,WAAW,CAAC,IAAI,CAAC,EAAE,CAAA;IAE1E,MAAM,gBAAgB,GAAG,MAAM,CAAC,GAAG,EAAE,gCAAgC,CAAC,CAAA;IACtE,IAAI,gBAAgB,KAAK,EAAE,EAAE,CAAC;QAC5B,MAAM,EAAE,GAAG,gBAAgB;aACxB,KAAK,CAAC,GAAG,CAAC;aACV,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;aACpB,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAA;QAC/F,IAAI,CAAC,EAAE;YAAE,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,qBAAqB,EAAE,CAAA;IAC3E,CAAC;IAED,MAAM,OAAO,GAA4B;QACvC,CAAC,8BAA8B,EAAE,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC9D,CAAC,8BAA8B,EAAE,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC9D,CAAC,wBAAwB,EAAE,OAAO,CAAC;QACnC,CAAC,MAAM,EAAE,uEAAuE,CAAC;KAClF,CAAA;IACD,IAAI,MAAM,KAAK,EAAE,IAAI,WAAW,CAAC,qBAAqB,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,YAAY,CAAC,EAAE,CAAC;QACrF,OAAO,CAAC,IAAI,CAAC,CAAC,6BAA6B,EAAE,MAAM,CAAC,EAAE,CAAC,kCAAkC,EAAE,MAAM,CAAC,CAAC,CAAA;IACrG,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,CAAA;AAC3C,CAAC;AAeD;;;;GAIG;AACH,MAAM,UAAU,cAAc,CAAC,GAAoB,EAAE,MAAwB;IAC3E,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,CAAA;IAChC,IAAI,IAAI,KAAK,EAAE;QAAE,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,qBAAqB,EAAE,IAAI,EAAE,aAAa,EAAE,CAAA;IACtG,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,MAAM,CAAC,YAAY,CAAC;QACzC,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,WAAW,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAA;IAE5E,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAA;IACnF,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QACvB,kEAAkE;QAClE,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,EAAE,eAAe,CAAC,CAAA;QACzC,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,CAAA;QACpF,MAAM,OAAO,GAAG,MAAM,CAAC,GAAG,EAAE,WAAW,CAAC,CAAA;QACxC,MAAM,OAAO,GAAG,CAAC,SAAiB,EAAE,EAAE,CACpC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,KAAK,EAAE,IAAI,qBAAqB,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC,CAAA;QACzE,MAAM,QAAQ,GAAG,MAAM,KAAK,SAAS,IAAI,OAAO,CAAC,MAAM,CAAC,CAAA;QACxD,MAAM,KAAK,GAAG,OAAO,CAAC,OAAO,CAAC,CAAA;QAC9B,IAAI,CAAC,QAAQ,IAAI,CAAC,KAAK,EAAE,CAAC;YACxB,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,wCAAwC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAA;QACnG,CAAC;IACH,CAAC;IACD,OAAO,SAAS,CAAA;AAClB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,IAAI,CAClB,GAAoB,EACpB,IAAY,EACZ,MAA0B;IAE1B,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAA;IACpC,MAAM,IAAI,GAAG,WAAW,CAAC,MAAM,EAAE,MAAM,CAAC,YAAY,CAAC,CAAA;IACrD,MAAM,WAAW,GAAG,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IAE/C,IAAI,CAAC,WAAW,EAAE,CAAC;QACjB,MAAM,OAAO,GAAG,cAAc,CAAC,GAAG,EAAE,MAAM,CAAC,CAAA;QAC3C,IAAI,OAAO;YAAE,OAAO,EAAE,GAAG,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAA;IACnD,CAAC;IAED,IAAI,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC;QAAE,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAA;IACvG,OAAO,SAAS,CAAA;AAClB,CAAC"}
|
|
@@ -17,9 +17,12 @@
|
|
|
17
17
|
import type { IncomingMessage, ServerResponse } from 'node:http';
|
|
18
18
|
import type { LocalApiServerState } from '../../contracts/index.js';
|
|
19
19
|
import type { PublicServerConfig, PublicServerDeps } from './types.js';
|
|
20
|
-
export type { CtxIncreaseOutcome, ImagesBackend, LocalTarget, PublicServerConfig, PublicServerDeps, VideosBackend, } from './types.js';
|
|
21
|
-
export { isValidHost, removePrefix } from './gates.js';
|
|
20
|
+
export type { CtxIncreaseOutcome, DecisionBackend, DecisionTarget, EmbeddingBackend, EmbeddingTarget, ImagesBackend, LocalTarget, PublicServerConfig, PublicServerDeps, VideosBackend, } from './types.js';
|
|
21
|
+
export { hostAndKeyGate, isValidHost, removePrefix } from './gates.js';
|
|
22
|
+
export type { HostAndKeyConfig } from './gates.js';
|
|
22
23
|
export { DynamicTrustedHosts, socketAddressLiteral } from './dynamic-hosts.js';
|
|
24
|
+
export { forwardableHeaders, pipeBody, readBody, relay, relayedHeaders, sendUpstream, sendWhole, } from './wire.js';
|
|
25
|
+
export type { HeaderPairs, UpstreamResponse } from './wire.js';
|
|
23
26
|
export declare const DEFAULT_PUBLIC_PORT = 1337;
|
|
24
27
|
export declare const DEFAULT_PUBLIC_HOST = "127.0.0.1";
|
|
25
28
|
export declare const DEFAULT_PUBLIC_PREFIX = "/v1";
|
|
@@ -46,6 +49,7 @@ export declare function handlePublicRequest(req: IncomingMessage, res: ServerRes
|
|
|
46
49
|
export declare class PublicServer {
|
|
47
50
|
private readonly server;
|
|
48
51
|
private readonly config;
|
|
52
|
+
private readonly keyring?;
|
|
49
53
|
private constructor();
|
|
50
54
|
static start(deps: PublicServerDeps, options?: PublicServerOptions): Promise<PublicServer>;
|
|
51
55
|
get host(): string;
|
|
@@ -17,6 +17,8 @@
|
|
|
17
17
|
import { createServer } from 'node:http';
|
|
18
18
|
import { AtomicCoreError } from '../../contracts/index.js';
|
|
19
19
|
import { captureReport, internalErrorReport } from '../../telemetry/index.js';
|
|
20
|
+
import { DECISION_ROUTES, serveDecision } from './decision.js';
|
|
21
|
+
import { serveEmbeddingIfOwned } from './embedding.js';
|
|
20
22
|
import { answer, newExchange } from './exchange.js';
|
|
21
23
|
import { serveForward } from './forward.js';
|
|
22
24
|
import { serveImagesGenerations } from './images.js';
|
|
@@ -29,8 +31,9 @@ import { serveResponses } from './responses.js';
|
|
|
29
31
|
import { serveStatic } from './static.js';
|
|
30
32
|
import { RequestTrace, endpointFromPath } from './trace.js';
|
|
31
33
|
import { sendWhole } from './wire.js';
|
|
32
|
-
export { isValidHost, removePrefix } from './gates.js';
|
|
34
|
+
export { hostAndKeyGate, isValidHost, removePrefix } from './gates.js';
|
|
33
35
|
export { DynamicTrustedHosts, socketAddressLiteral } from './dynamic-hosts.js';
|
|
36
|
+
export { forwardableHeaders, pipeBody, readBody, relay, relayedHeaders, sendUpstream, sendWhole, } from './wire.js';
|
|
34
37
|
export const DEFAULT_PUBLIC_PORT = 1337;
|
|
35
38
|
export const DEFAULT_PUBLIC_HOST = '127.0.0.1';
|
|
36
39
|
export const DEFAULT_PUBLIC_PREFIX = '/v1';
|
|
@@ -52,6 +55,8 @@ const ALLOWED_METHODS = {
|
|
|
52
55
|
'/embeddings': 'POST',
|
|
53
56
|
'/messages/count_tokens': 'POST',
|
|
54
57
|
'/images/generations': 'POST',
|
|
58
|
+
'/systemone': 'POST',
|
|
59
|
+
'/router/score': 'POST',
|
|
55
60
|
};
|
|
56
61
|
const FORWARDED = new Set([
|
|
57
62
|
'/messages',
|
|
@@ -78,10 +83,11 @@ function requestPath(url) {
|
|
|
78
83
|
}
|
|
79
84
|
/** The configuration one request is checked against: the listener's, plus what is trusted just for it. */
|
|
80
85
|
function configFor(req, listener, deps) {
|
|
86
|
+
// The host's keyring is read for every request: a key it revokes is refused on the next one.
|
|
87
|
+
const keys = deps.apiKeys?.();
|
|
88
|
+
const keyed = keys === undefined ? listener : { ...listener, apiKeys: keys };
|
|
81
89
|
const dynamic = deps.dynamicTrustedHosts?.(req.socket.localAddress) ?? [];
|
|
82
|
-
return dynamic.length === 0
|
|
83
|
-
? listener
|
|
84
|
-
: { ...listener, trustedHosts: [...listener.trustedHosts, ...dynamic] };
|
|
90
|
+
return dynamic.length === 0 ? keyed : { ...keyed, trustedHosts: [...keyed.trustedHosts, ...dynamic] };
|
|
85
91
|
}
|
|
86
92
|
export async function handlePublicRequest(req, res, listener, deps) {
|
|
87
93
|
const trace = new RequestTrace(req.method ?? 'GET', deps);
|
|
@@ -114,6 +120,9 @@ export async function handlePublicRequest(req, res, listener, deps) {
|
|
|
114
120
|
// so it is answered in full here and never reaches the generic forwarder.
|
|
115
121
|
if (path === '/chat/completions' && deps.chatgpt && (await serveSubscriptionIfOwned(ex, deps.chatgpt)))
|
|
116
122
|
return;
|
|
123
|
+
// The embedding module's model is served by its own process; any other `model` goes on as before.
|
|
124
|
+
if (path === '/embeddings' && deps.embedding && (await serveEmbeddingIfOwned(ex, deps.embedding)))
|
|
125
|
+
return;
|
|
117
126
|
if (FORWARDED.has(path))
|
|
118
127
|
return serveForward(ex);
|
|
119
128
|
// Served here, never forwarded: the image model is the core's own, and it is not in `/models`.
|
|
@@ -121,6 +130,11 @@ export async function handlePublicRequest(req, res, listener, deps) {
|
|
|
121
130
|
trace.endpoint = endpointFromPath(path);
|
|
122
131
|
return serveImagesGenerations(ex);
|
|
123
132
|
}
|
|
133
|
+
// Served by the decision module's own process, byte for byte; never a chat session.
|
|
134
|
+
if (Object.hasOwn(DECISION_ROUTES, path)) {
|
|
135
|
+
trace.endpoint = endpointFromPath(path);
|
|
136
|
+
return serveDecision(ex);
|
|
137
|
+
}
|
|
124
138
|
}
|
|
125
139
|
if (ex.method === 'GET') {
|
|
126
140
|
// Model polling, metrics scraping and the docs are client bookkeeping: never reported.
|
|
@@ -159,9 +173,11 @@ export async function handlePublicRequest(req, res, listener, deps) {
|
|
|
159
173
|
export class PublicServer {
|
|
160
174
|
server;
|
|
161
175
|
config;
|
|
162
|
-
|
|
176
|
+
keyring;
|
|
177
|
+
constructor(server, config, keyring) {
|
|
163
178
|
this.server = server;
|
|
164
179
|
this.config = config;
|
|
180
|
+
this.keyring = keyring;
|
|
165
181
|
}
|
|
166
182
|
static async start(deps, options = {}) {
|
|
167
183
|
const host = options.host ?? DEFAULT_PUBLIC_HOST;
|
|
@@ -211,7 +227,7 @@ export class PublicServer {
|
|
|
211
227
|
const address = server.address();
|
|
212
228
|
if (typeof address === 'object' && address)
|
|
213
229
|
config.port = address.port;
|
|
214
|
-
return new PublicServer(server, config);
|
|
230
|
+
return new PublicServer(server, config, deps.apiKeys);
|
|
215
231
|
}
|
|
216
232
|
get host() {
|
|
217
233
|
return this.config.host;
|
|
@@ -231,7 +247,7 @@ export class PublicServer {
|
|
|
231
247
|
host: this.host,
|
|
232
248
|
port: this.port,
|
|
233
249
|
prefix: this.prefix,
|
|
234
|
-
requires_api_key: this.config.apiKey.length > 0,
|
|
250
|
+
requires_api_key: this.keyring?.() !== undefined || this.config.apiKey.length > 0,
|
|
235
251
|
pid: process.pid,
|
|
236
252
|
};
|
|
237
253
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/server/public/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAA;AAExC,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAA;AAE1D,OAAO,EAAE,aAAa,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAA;AAC7E,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,eAAe,CAAA;AACnD,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAA;AAC3C,OAAO,EAAE,sBAAsB,EAAE,MAAM,aAAa,CAAA;AACpD,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AACzC,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAA;AACnD,OAAO,EAAE,wBAAwB,EAAE,MAAM,mBAAmB,CAAA;AAC5D,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,YAAY,CAAA;AAC1D,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAA;AAC1E,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAA;AAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AACzC,OAAO,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAA;AAE3D,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAA;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/server/public/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,WAAW,CAAA;AAExC,OAAO,EAAE,eAAe,EAAE,MAAM,0BAA0B,CAAA;AAE1D,OAAO,EAAE,aAAa,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAA;AAC7E,OAAO,EAAE,eAAe,EAAE,aAAa,EAAE,MAAM,eAAe,CAAA;AAC9D,OAAO,EAAE,qBAAqB,EAAE,MAAM,gBAAgB,CAAA;AACtD,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,eAAe,CAAA;AACnD,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAA;AAC3C,OAAO,EAAE,sBAAsB,EAAE,MAAM,aAAa,CAAA;AACpD,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AACzC,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAA;AACnD,OAAO,EAAE,wBAAwB,EAAE,MAAM,mBAAmB,CAAA;AAC5D,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,YAAY,CAAA;AAC1D,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAA;AAC1E,OAAO,EAAE,cAAc,EAAE,MAAM,gBAAgB,CAAA;AAC/C,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAA;AACzC,OAAO,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAA;AAE3D,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAA;AAcrC,OAAO,EAAE,cAAc,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,YAAY,CAAA;AAEtE,OAAO,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAA;AAC9E,OAAO,EACL,kBAAkB,EAClB,QAAQ,EACR,QAAQ,EACR,KAAK,EACL,cAAc,EACd,YAAY,EACZ,SAAS,GACV,MAAM,WAAW,CAAA;AAGlB,MAAM,CAAC,MAAM,mBAAmB,GAAG,IAAI,CAAA;AACvC,MAAM,CAAC,MAAM,mBAAmB,GAAG,WAAW,CAAA;AAC9C,MAAM,CAAC,MAAM,qBAAqB,GAAG,KAAK,CAAA;AAC1C,yDAAyD;AACzD,MAAM,CAAC,MAAM,0BAA0B,GAAG,GAAG,CAAA;AAmB7C,+FAA+F;AAC/F,MAAM,eAAe,GAA2B;IAC9C,GAAG,EAAE,KAAK;IACV,eAAe,EAAE,KAAK;IACtB,sBAAsB,EAAE,KAAK;IAC7B,4BAA4B,EAAE,KAAK;IACnC,SAAS,EAAE,KAAK;IAChB,UAAU,EAAE,KAAK;IACjB,mBAAmB,EAAE,KAAK;IAC1B,WAAW,EAAE,MAAM;IACnB,mBAAmB,EAAE,MAAM;IAC3B,YAAY,EAAE,MAAM;IACpB,cAAc,EAAE,MAAM;IACtB,aAAa,EAAE,MAAM;IACrB,wBAAwB,EAAE,MAAM;IAChC,qBAAqB,EAAE,MAAM;IAC7B,YAAY,EAAE,MAAM;IACpB,eAAe,EAAE,MAAM;CACxB,CAAA;AAED,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC;IACxB,WAAW;IACX,mBAAmB;IACnB,cAAc;IACd,aAAa;IACb,wBAAwB;CACzB,CAAC,CAAA;AAEF,MAAM,UAAU,eAAe,CAAC,MAAc;IAC5C,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,EAAE,CAAA;IAC7B,IAAI,CAAC,OAAO,IAAI,OAAO,KAAK,GAAG;QAAE,OAAO,EAAE,CAAA;IAC1C,MAAM,SAAS,GAAG,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,OAAO,EAAE,CAAA;IACnE,OAAO,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAA;AACrE,CAAC;AAED,2FAA2F;AAC3F,SAAS,WAAW,CAAC,GAAuB;IAC1C,IAAI,MAAM,GAAG,GAAG,IAAI,GAAG,CAAA;IACvB,MAAM,MAAM,GAAG,+BAA+B,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;IAC3D,IAAI,MAAM;QAAE,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,GAAG,CAAA;IAC1D,MAAM,CAAC,GAAG,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;IAC7B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAA;AAC7C,CAAC;AAED,0GAA0G;AAC1G,SAAS,SAAS,CAChB,GAAoB,EACpB,QAA4B,EAC5B,IAAsB;IAEtB,6FAA6F;IAC7F,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,EAAE,EAAE,CAAA;IAC7B,MAAM,KAAK,GAAG,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,GAAG,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,CAAA;IAC5E,MAAM,OAAO,GAAG,IAAI,CAAC,mBAAmB,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,EAAE,CAAA;IACzE,OAAO,OAAO,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,GAAG,KAAK,EAAE,YAAY,EAAE,CAAC,GAAG,KAAK,CAAC,YAAY,EAAE,GAAG,OAAO,CAAC,EAAE,CAAA;AACvG,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,GAAoB,EACpB,GAAmB,EACnB,QAA4B,EAC5B,IAAsB;IAEtB,MAAM,KAAK,GAAG,IAAI,YAAY,CAAC,GAAG,CAAC,MAAM,IAAI,KAAK,EAAE,IAAI,CAAC,CAAA;IACzD,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;IACjB,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAA;IAE7C,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;QAC7B,gFAAgF;QAChF,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAA;QACrB,MAAM,CAAC,GAAG,SAAS,CAAC,GAAG,EAAE,MAAM,CAAC,CAAA;QAChC,SAAS,CAAC,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,CAAA;QAC3C,OAAM;IACR,CAAC;IAED,MAAM,IAAI,GAAG,YAAY,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,CAAC,CAAA;IAC9D,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,MAAM,CAAC,CAAA;IACvC,IAAI,OAAO,EAAE,CAAC;QACZ,IAAI,OAAO,CAAC,IAAI,KAAK,QAAQ;YAAE,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAA;aAC/C,CAAC;YACJ,KAAK,CAAC,QAAQ,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAA;YACvC,KAAK,CAAC,SAAS,GAAG,OAAO,CAAC,IAAI,IAAI,IAAI,CAAA;QACxC,CAAC;QACD,SAAS,CAAC,GAAG,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC,CAAA;QAC7D,OAAM;IACR,CAAC;IAED,MAAM,EAAE,GAAG,WAAW,CAAC,GAAG,EAAE,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,KAAK,CAAC,CAAA;IAC3D,IAAI,EAAE,CAAC,MAAM,KAAK,MAAM,EAAE,CAAC;QACzB,IAAI,IAAI,KAAK,YAAY;YAAE,OAAO,cAAc,CAAC,EAAE,CAAC,CAAA;QACpD,8FAA8F;QAC9F,0EAA0E;QAC1E,IAAI,IAAI,KAAK,mBAAmB,IAAI,IAAI,CAAC,OAAO,IAAI,CAAC,MAAM,wBAAwB,CAAC,EAAE,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;YACpG,OAAM;QACR,kGAAkG;QAClG,IAAI,IAAI,KAAK,aAAa,IAAI,IAAI,CAAC,SAAS,IAAI,CAAC,MAAM,qBAAqB,CAAC,EAAE,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;YAAE,OAAM;QACzG,IAAI,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,OAAO,YAAY,CAAC,EAAE,CAAC,CAAA;QAChD,+FAA+F;QAC/F,IAAI,IAAI,KAAK,qBAAqB,EAAE,CAAC;YACnC,KAAK,CAAC,QAAQ,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAA;YACvC,OAAO,sBAAsB,CAAC,EAAE,CAAC,CAAA;QACnC,CAAC;QACD,oFAAoF;QACpF,IAAI,MAAM,CAAC,MAAM,CAAC,eAAe,EAAE,IAAI,CAAC,EAAE,CAAC;YACzC,KAAK,CAAC,QAAQ,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAA;YACvC,OAAO,aAAa,CAAC,EAAE,CAAC,CAAA;QAC1B,CAAC;IACH,CAAC;IACD,IAAI,EAAE,CAAC,MAAM,KAAK,KAAK,EAAE,CAAC;QACxB,uFAAuF;QACvF,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,mBAAmB,IAAI,IAAI,KAAK,UAAU,EAAE,CAAC;YAC9E,KAAK,CAAC,QAAQ,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAA;YACvC,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAA;QACvB,CAAC;QACD,IAAI,IAAI,KAAK,SAAS;YAAE,OAAO,WAAW,CAAC,EAAE,CAAC,CAAA;QAC9C,IAAI,IAAI,KAAK,mBAAmB;YAAE,OAAO,gBAAgB,CAAC,EAAE,CAAC,CAAA;QAC7D,IAAI,IAAI,KAAK,UAAU;YAAE,OAAO,YAAY,CAAC,EAAE,CAAC,CAAA;QAChD,IAAI,WAAW,CAAC,EAAE,CAAC,EAAE,CAAC;YACpB,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAA;YACrB,OAAM;QACR,CAAC;IACH,CAAC;IAED,+FAA+F;IAC/F,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,CAAC,CAAA;IAClC,IAAI,KAAK,EAAE,CAAC;QACV,KAAK,CAAC,QAAQ,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAA;QACvC,OAAO,WAAW,CAAC,EAAE,EAAE,KAAK,CAAC,CAAA;IAC/B,CAAC;IAED,MAAM,OAAO,GAAG,eAAe,CAAC,IAAI,CAAC,CAAA;IACrC,IAAI,OAAO,EAAE,CAAC;QACZ,KAAK,CAAC,QAAQ,GAAG,gBAAgB,CAAC,IAAI,CAAC,CAAA;QACvC,KAAK,CAAC,SAAS,GAAG,oBAAoB,CAAA;QACtC,MAAM,CAAC,EAAE,EAAE,GAAG,EAAE,oBAAoB,EAAE,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,CAAA;QAC3D,OAAM;IACR,CAAC;IACD,mFAAmF;IACnF,KAAK,CAAC,QAAQ,GAAG,IAAI,CAAA;IACrB,MAAM,CAAC,EAAE,EAAE,GAAG,EAAE,WAAW,CAAC,CAAA;AAC9B,CAAC;AAED,MAAM,OAAO,YAAY;IAEJ;IACA;IACA;IAHnB,YACmB,MAAc,EACd,MAA0B,EAC1B,OAA6C;QAF7C,WAAM,GAAN,MAAM,CAAQ;QACd,WAAM,GAAN,MAAM,CAAoB;QAC1B,YAAO,GAAP,OAAO,CAAsC;IAC7D,CAAC;IAEJ,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,IAAsB,EAAE,UAA+B,EAAE;QAC1E,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,mBAAmB,CAAA;QAChD,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,mBAAmB,CAAA;QAChD,8FAA8F;QAC9F,MAAM,MAAM,GAAuB;YACjC,IAAI;YACJ,IAAI;YACJ,MAAM,EAAE,eAAe,CAAC,OAAO,CAAC,MAAM,IAAI,qBAAqB,CAAC;YAChE,MAAM,EAAE,OAAO,CAAC,MAAM,IAAI,EAAE;YAC5B,YAAY,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC;YAC/C,gBAAgB,EAAE,OAAO,CAAC,gBAAgB,IAAI,0BAA0B;SACzE,CAAA;QACD,iGAAiG;QACjG,kDAAkD;QAClD,MAAM,MAAM,GAAG,YAAY,CAAC,EAAE,iBAAiB,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE;YACrE,mBAAmB,CAAC,GAAG,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,CAAU,EAAE,EAAE;gBAC/D,aAAa,CAAC,IAAI,CAAC,MAAM,EAAE,mBAAmB,CAAC,EAAE,MAAM,EAAE,eAAe,EAAE,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC,CAAC,CAAA;gBACnG,IAAI,GAAG,CAAC,WAAW;oBAAE,GAAG,CAAC,OAAO,CAAC,CAAU,CAAC,CAAA;;oBACvC,SAAS,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,uBAAuB,CAAC,CAAA;YACvD,CAAC,CAAC,CAAA;QACJ,CAAC,CAAC,CAAA;QACF,MAAM,MAAM,GAAG,CAAC,QAAgB,EAAE,EAAE,CAClC,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACpC,MAAM,OAAO,GAAG,CAAC,CAAwB,EAAE,EAAE;gBAC3C,MAAM,CACJ,CAAC,CAAC,IAAI,KAAK,YAAY;oBACrB,CAAC,CAAC,IAAI,eAAe,CAAC,UAAU,EAAE,QAAQ,QAAQ,qBAAqB,EAAE,GAAG,IAAI,IAAI,QAAQ,EAAE,CAAC;oBAC/F,CAAC,CAAC,CAAC,CACN,CAAA;YACH,CAAC,CAAA;YACD,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;YAC7B,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,IAAI,EAAE,GAAG,EAAE;gBACjC,MAAM,CAAC,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC,CAAA;gBAC5B,OAAO,EAAE,CAAA;YACX,CAAC,CAAC,CAAA;QACJ,CAAC,CAAC,CAAA;QACJ,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,IAAI,CAAC,CAAA;QACpB,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAI,CAAC,OAAO,CAAC,YAAY,IAAI,IAAI,KAAK,CAAC;gBAAE,MAAM,CAAC,CAAA;YAChD,MAAM,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,QAAe,EAAE,EAAE;gBACxC,MAAM,IAAI,eAAe,CACvB,UAAU,EACV,oBAAoB,IAAI,IAAI,IAAI,+BAA+B,EAC/D,GAAI,CAAW,CAAC,OAAO,eAAe,QAAQ,CAAC,OAAO,EAAE,CACzD,CAAA;YACH,CAAC,CAAC,CAAA;QACJ,CAAC;QACD,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,EAAE,CAAA;QAChC,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,OAAO;YAAE,MAAM,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAA;QACtE,OAAO,IAAI,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,CAAA;IACvD,CAAC;IAED,IAAI,IAAI;QACN,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CAAA;IACzB,CAAC;IAED,IAAI,IAAI;QACN,OAAO,IAAI,CAAC,MAAM,CAAC,IAAI,CAAA;IACzB,CAAC;IAED,IAAI,MAAM;QACR,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAA;IAC3B,CAAC;IAED,IAAI,GAAG;QACL,OAAO,UAAU,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,MAAM,EAAE,CAAA;IACzD,CAAC;IAED,KAAK;QACH,OAAO;YACL,OAAO,EAAE,IAAI;YACb,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,gBAAgB,EAAE,IAAI,CAAC,OAAO,EAAE,EAAE,KAAK,SAAS,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC;YACjF,GAAG,EAAE,OAAO,CAAC,GAAG;SACjB,CAAA;IACH,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,MAAM,CAAC,mBAAmB,EAAE,EAAE,CAAA;QACnC,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC,CAAA;IAC1E,CAAC;CACF;AAED,MAAM,UAAU,YAAY,CAAC,QAA8B;IACzD,OAAO;QACL,OAAO,EAAE,KAAK;QACd,IAAI,EAAE,QAAQ,EAAE,IAAI,IAAI,mBAAmB;QAC3C,IAAI,EAAE,QAAQ,EAAE,IAAI,IAAI,mBAAmB;QAC3C,MAAM,EAAE,QAAQ,EAAE,MAAM,IAAI,qBAAqB;QACjD,gBAAgB,EAAE,QAAQ,EAAE,gBAAgB,IAAI,KAAK;QACrD,GAAG,EAAE,IAAI;KACV,CAAA;AACH,CAAC"}
|
|
@@ -7,21 +7,32 @@
|
|
|
7
7
|
*/
|
|
8
8
|
import type { JsonValue } from '../shims/index.js';
|
|
9
9
|
import type { Exchange } from './exchange.js';
|
|
10
|
+
/** What a declared session (`LocalTarget.policy`) says about itself, for `/muse-code/models`. */
|
|
11
|
+
interface MuseFacts {
|
|
12
|
+
toolCall?: boolean;
|
|
13
|
+
context?: number;
|
|
14
|
+
output?: number;
|
|
15
|
+
}
|
|
10
16
|
export declare function servedModels(ex: Exchange): Array<{
|
|
11
17
|
id: string;
|
|
12
18
|
ownedBy: string;
|
|
19
|
+
muse: MuseFacts;
|
|
13
20
|
}>;
|
|
14
21
|
export declare function serveModels(ex: Exchange): void;
|
|
15
22
|
export declare function serveMuseCatalog(ex: Exchange): void;
|
|
16
23
|
/**
|
|
17
24
|
* One row of Muse Code's catalogue. `modalities.input` claims text only and `reasoning` is false,
|
|
18
25
|
* because the server cannot know either per model and the safe answer never sends a text-only model
|
|
19
|
-
* an image or a reasoning parameter. `tool_call`
|
|
26
|
+
* an image or a reasoning parameter. `tool_call` stays true unless the session itself says otherwise:
|
|
27
|
+
* without it Muse has no agent loop, but a session that declares it has no tool-call parser
|
|
28
|
+
* (`tensorrt-llm`) would refuse every agent request, so it says so here too. A session that knows the
|
|
29
|
+
* context and output cap it was started with advertises those instead of the guesses above.
|
|
20
30
|
*/
|
|
21
|
-
export declare function museCatalogEntry(modelId: string, ownedBy: string): JsonValue;
|
|
31
|
+
export declare function museCatalogEntry(modelId: string, ownedBy: string, facts?: MuseFacts): JsonValue;
|
|
22
32
|
/**
|
|
23
33
|
* `GET /metrics?model=<id>` (or `X-Model`). llama.cpp is the only engine with a Prometheus exporter,
|
|
24
34
|
* so MLX sessions are deliberately not considered. llama-server serves it at its root, not under
|
|
25
35
|
* `/v1`, and behind the session key.
|
|
26
36
|
*/
|
|
27
37
|
export declare function serveMetrics(ex: Exchange): Promise<void>;
|
|
38
|
+
export {};
|
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
import { serdeToString } from '../shims/index.js';
|
|
9
9
|
import { answer, connectTimeoutMs, header } from './exchange.js';
|
|
10
10
|
import { readBody, sendUpstream } from './wire.js';
|
|
11
|
+
import { EMBEDDING_OWNED_BY } from './embedding.js';
|
|
11
12
|
/**
|
|
12
13
|
* Advertised to Muse Code, which compacts a conversation as it nears the advertised context. The
|
|
13
14
|
* server does not know a session's real window, so these under-estimate on purpose: guessing low
|
|
@@ -21,30 +22,51 @@ const OWNED_BY = {
|
|
|
21
22
|
'llamacpp': 'llama.cpp',
|
|
22
23
|
'llamacpp-upstream': 'llama.cpp-upstream',
|
|
23
24
|
'mlx': 'mlx',
|
|
25
|
+
'tensorrt-llm': 'tensorrt-llm',
|
|
26
|
+
'atomic-prism': 'prism-llama.cpp',
|
|
27
|
+
'vllm': 'vllm',
|
|
24
28
|
};
|
|
25
29
|
export function servedModels(ex) {
|
|
26
|
-
const models = ex.deps.listLocal().map((s) => ({
|
|
30
|
+
const models = ex.deps.listLocal().map((s) => ({
|
|
31
|
+
id: s.modelId,
|
|
32
|
+
ownedBy: OWNED_BY[s.provider],
|
|
33
|
+
muse: s.policy
|
|
34
|
+
? {
|
|
35
|
+
toolCall: s.policy.tools,
|
|
36
|
+
...(s.policy.contextLength === undefined ? {} : { context: s.policy.contextLength }),
|
|
37
|
+
...(s.policy.maxOutputTokens === undefined ? {} : { output: s.policy.maxOutputTokens }),
|
|
38
|
+
}
|
|
39
|
+
: {},
|
|
40
|
+
}));
|
|
27
41
|
for (const provider of ex.deps.providers().values()) {
|
|
28
42
|
for (const id of provider.models)
|
|
29
|
-
models.push({ id, ownedBy: 'remote' });
|
|
43
|
+
models.push({ id, ownedBy: 'remote', muse: {} });
|
|
30
44
|
}
|
|
31
45
|
return models;
|
|
32
46
|
}
|
|
33
47
|
export function serveModels(ex) {
|
|
34
48
|
const data = servedModels(ex).map((m) => ({ id: m.id, object: 'model', created: 1, owned_by: m.ownedBy }));
|
|
49
|
+
// The embedding module's model, while it is turned on: OpenAI clients pick their embedding model from
|
|
50
|
+
// this list. Not in `servedModels`, which also feeds Muse Code's catalogue of chat models.
|
|
51
|
+
const embedding = ex.deps.embedding?.modelId() ?? null;
|
|
52
|
+
if (embedding !== null && !data.some((m) => m.id === embedding))
|
|
53
|
+
data.push({ id: embedding, object: 'model', created: 1, owned_by: EMBEDDING_OWNED_BY });
|
|
35
54
|
answer(ex, 200, serdeToString({ object: 'list', data }), [['Content-Type', 'application/json']]);
|
|
36
55
|
}
|
|
37
56
|
export function serveMuseCatalog(ex) {
|
|
38
|
-
const data = servedModels(ex).map((m) => museCatalogEntry(m.id, m.ownedBy));
|
|
57
|
+
const data = servedModels(ex).map((m) => museCatalogEntry(m.id, m.ownedBy, m.muse));
|
|
39
58
|
answer(ex, 200, serdeToString({ object: 'list', data }), [['Content-Type', 'application/json']]);
|
|
40
59
|
}
|
|
41
60
|
/**
|
|
42
61
|
* One row of Muse Code's catalogue. `modalities.input` claims text only and `reasoning` is false,
|
|
43
62
|
* because the server cannot know either per model and the safe answer never sends a text-only model
|
|
44
|
-
* an image or a reasoning parameter. `tool_call`
|
|
63
|
+
* an image or a reasoning parameter. `tool_call` stays true unless the session itself says otherwise:
|
|
64
|
+
* without it Muse has no agent loop, but a session that declares it has no tool-call parser
|
|
65
|
+
* (`tensorrt-llm`) would refuse every agent request, so it says so here too. A session that knows the
|
|
66
|
+
* context and output cap it was started with advertises those instead of the guesses above.
|
|
45
67
|
*/
|
|
46
|
-
export function museCatalogEntry(modelId, ownedBy) {
|
|
47
|
-
const context = ownedBy === 'remote' ? MUSE_REMOTE_CONTEXT_LIMIT : MUSE_LOCAL_CONTEXT_LIMIT;
|
|
68
|
+
export function museCatalogEntry(modelId, ownedBy, facts = {}) {
|
|
69
|
+
const context = facts.context ?? (ownedBy === 'remote' ? MUSE_REMOTE_CONTEXT_LIMIT : MUSE_LOCAL_CONTEXT_LIMIT);
|
|
48
70
|
return {
|
|
49
71
|
id: modelId,
|
|
50
72
|
object: 'model',
|
|
@@ -59,9 +81,9 @@ export function museCatalogEntry(modelId, ownedBy) {
|
|
|
59
81
|
attachment: false,
|
|
60
82
|
reasoning: false,
|
|
61
83
|
temperature: false,
|
|
62
|
-
tool_call: true,
|
|
84
|
+
tool_call: facts.toolCall ?? true,
|
|
63
85
|
modalities: { input: ['text'], output: ['text'] },
|
|
64
|
-
limit: { context, output: MUSE_OUTPUT_LIMIT },
|
|
86
|
+
limit: { context, output: facts.output ?? MUSE_OUTPUT_LIMIT },
|
|
65
87
|
options: { include: [], temperature: 0.9, top_p: 0.9 },
|
|
66
88
|
variants: {},
|
|
67
89
|
description: `${modelId} via Atomic Chat`,
|
|
@@ -90,7 +112,9 @@ export async function serveMetrics(ex) {
|
|
|
90
112
|
answer(ex, 400, "Missing 'model' query parameter (use ?model=<model_id>)");
|
|
91
113
|
return;
|
|
92
114
|
}
|
|
93
|
-
const session = ex.deps.findLocal('llamacpp', modelId) ??
|
|
115
|
+
const session = ex.deps.findLocal('llamacpp', modelId) ??
|
|
116
|
+
ex.deps.findLocal('llamacpp-upstream', modelId) ??
|
|
117
|
+
ex.deps.findLocal('atomic-prism', modelId);
|
|
94
118
|
if (!session) {
|
|
95
119
|
answer(ex, 404, `No running llama.cpp session for model '${modelId}'`);
|
|
96
120
|
return;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"listing.js","sourceRoot":"","sources":["../../../src/server/public/listing.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAA;AAEjD,OAAO,EAAE,MAAM,EAAE,gBAAgB,EAAE,MAAM,EAAE,MAAM,eAAe,CAAA;AAEhE,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,WAAW,CAAA;
|
|
1
|
+
{"version":3,"file":"listing.js","sourceRoot":"","sources":["../../../src/server/public/listing.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAA;AAEjD,OAAO,EAAE,MAAM,EAAE,gBAAgB,EAAE,MAAM,EAAE,MAAM,eAAe,CAAA;AAEhE,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,WAAW,CAAA;AAClD,OAAO,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAA;AAGnD;;;;GAIG;AACH,MAAM,wBAAwB,GAAG,MAAM,CAAA;AACvC,MAAM,yBAAyB,GAAG,OAAO,CAAA;AACzC,MAAM,iBAAiB,GAAG,MAAM,CAAA;AAEhC,kFAAkF;AAClF,MAAM,QAAQ,GAAkC;IAC9C,UAAU,EAAE,WAAW;IACvB,mBAAmB,EAAE,oBAAoB;IACzC,KAAK,EAAE,KAAK;IACZ,cAAc,EAAE,cAAc;IAC9B,cAAc,EAAE,iBAAiB;IACjC,MAAM,EAAE,MAAM;CACf,CAAA;AASD,MAAM,UAAU,YAAY,CAAC,EAAY;IACvC,MAAM,MAAM,GAAG,EAAE,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAC7C,EAAE,EAAE,CAAC,CAAC,OAAO;QACb,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC;QAC7B,IAAI,EAAE,CAAC,CAAC,MAAM;YACZ,CAAC,CAAC;gBACE,QAAQ,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK;gBACxB,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,aAAa,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,CAAC,MAAM,CAAC,aAAa,EAAE,CAAC;gBACpF,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,eAAe,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,eAAe,EAAE,CAAC;aACxF;YACH,CAAC,CAAC,EAAE;KACP,CAAC,CAAC,CAAA;IACH,KAAK,MAAM,QAAQ,IAAI,EAAE,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC;QACpD,KAAK,MAAM,EAAE,IAAI,QAAQ,CAAC,MAAM;YAAE,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAA;IACpF,CAAC;IACD,OAAO,MAAM,CAAA;AACf,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,EAAY;IACtC,MAAM,IAAI,GAAG,YAAY,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAA;IAC1G,sGAAsG;IACtG,2FAA2F;IAC3F,MAAM,SAAS,GAAG,EAAE,CAAC,IAAI,CAAC,SAAS,EAAE,OAAO,EAAE,IAAI,IAAI,CAAA;IACtD,IAAI,SAAS,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,SAAS,CAAC;QAC7D,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,QAAQ,EAAE,kBAAkB,EAAE,CAAC,CAAA;IACzF,MAAM,CAAC,EAAE,EAAE,GAAG,EAAE,aAAa,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,cAAc,EAAE,kBAAkB,CAAC,CAAC,CAAC,CAAA;AAClG,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,EAAY;IAC3C,MAAM,IAAI,GAAG,YAAY,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAA;IACnF,MAAM,CAAC,EAAE,EAAE,GAAG,EAAE,aAAa,CAAC,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,cAAc,EAAE,kBAAkB,CAAC,CAAC,CAAC,CAAA;AAClG,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAAe,EAAE,OAAe,EAAE,QAAmB,EAAE;IACtF,MAAM,OAAO,GACX,KAAK,CAAC,OAAO,IAAI,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,yBAAyB,CAAC,CAAC,CAAC,wBAAwB,CAAC,CAAA;IAChG,OAAO;QACL,EAAE,EAAE,OAAO;QACX,MAAM,EAAE,OAAO;QACf,OAAO,EAAE,CAAC;QACV,QAAQ,EAAE,OAAO;QACjB,QAAQ,EAAE;YACR,WAAW,EAAE;gBACX,IAAI,EAAE,OAAO;gBACb,MAAM,EAAE,OAAO;gBACf,YAAY,EAAE,YAAY;gBAC1B,SAAS,EAAE,KAAK;gBAChB,UAAU,EAAE,KAAK;gBACjB,SAAS,EAAE,KAAK;gBAChB,WAAW,EAAE,KAAK;gBAClB,SAAS,EAAE,KAAK,CAAC,QAAQ,IAAI,IAAI;gBACjC,UAAU,EAAE,EAAE,KAAK,EAAE,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC,MAAM,CAAC,EAAE;gBACjD,KAAK,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,MAAM,IAAI,iBAAiB,EAAE;gBAC7D,OAAO,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,WAAW,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,EAAE;gBACtD,QAAQ,EAAE,EAAE;gBACZ,WAAW,EAAE,GAAG,OAAO,kBAAkB;gBACzC,IAAI,EAAE,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,QAAQ,EAAE,KAAK,EAAE;aAChE;SACF;KACF,CAAA;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,EAAY;IAC7C,gGAAgG;IAChG,sDAAsD;IACtD,MAAM,SAAS,GAAG,EAAE,CAAC,KAAK;QACxB,EAAE,KAAK,CAAC,GAAG,CAAC;SACX,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;QACZ,MAAM,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;QAC5B,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAA;IAC5F,CAAC,CAAC;SACD,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,SAAS,CAAC,CAAA;IAC/B,MAAM,OAAO,GAAG,SAAS,IAAI,MAAM,CAAC,EAAE,CAAC,GAAG,EAAE,SAAS,CAAC,CAAA;IACtD,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,MAAM,CAAC,EAAE,EAAE,GAAG,EAAE,yDAAyD,CAAC,CAAA;QAC1E,OAAM;IACR,CAAC;IAED,MAAM,OAAO,GACX,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,OAAO,CAAC;QACtC,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,mBAAmB,EAAE,OAAO,CAAC;QAC/C,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,cAAc,EAAE,OAAO,CAAC,CAAA;IAC5C,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,MAAM,CAAC,EAAE,EAAE,GAAG,EAAE,2CAA2C,OAAO,GAAG,CAAC,CAAA;QACtE,OAAM;IACR,CAAC;IAED,MAAM,OAAO,GAA4B,OAAO,CAAC,MAAM;QACrD,CAAC,CAAC,CAAC,CAAC,eAAe,EAAE,UAAU,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;QACjD,CAAC,CAAC,EAAE,CAAA;IACN,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,oBAAoB,OAAO,CAAC,IAAI,UAAU,EAAE;YAC9E,MAAM,EAAE,KAAK;YACb,OAAO;YACP,gBAAgB,EAAE,gBAAgB,CAAC,EAAE,CAAC;SACvC,CAAC,CAAA;QACF,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAA;QACxE,MAAM,CAAC,EAAE,EAAE,QAAQ,CAAC,MAAM,EAAE,KAAK,EAAE;YACjC,CAAC,cAAc,EAAE,QAAQ,CAAC,WAAW,IAAI,2BAA2B,CAAC;SACtE,CAAC,CAAA;IACJ,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,CAAC,EAAE,EAAE,GAAG,EAAE,8CAA+C,CAAW,CAAC,OAAO,EAAE,CAAC,CAAA;IACvF,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { JsonValue } from '../shims/index.js';
|
|
2
|
+
import type { LocalTargetPolicy } from './types.js';
|
|
3
|
+
export interface PolicyRefusal {
|
|
4
|
+
status: number;
|
|
5
|
+
/** A serialized OpenAI error envelope. */
|
|
6
|
+
body: string;
|
|
7
|
+
}
|
|
8
|
+
/** Why this POST must not reach the session, or `undefined` when it may. */
|
|
9
|
+
export declare function policyRefusal(policy: LocalTargetPolicy, path: string, modelId: string, json: JsonValue): PolicyRefusal | undefined;
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The checks the public server makes before it forwards a request to a session whose engine declares
|
|
3
|
+
* its routes (`LocalTarget.policy`; `tensorrt-llm`, spec `tensorrt-llm-runtime`): a route the session
|
|
4
|
+
* does not serve, tool calls to a model with no tool-call parser, or JSON output to a family without
|
|
5
|
+
* structured output, is answered here with an
|
|
6
|
+
* OpenAI-shaped `400` that says what is missing — never forwarded, and never silently degraded
|
|
7
|
+
* ("Публичный сервер MUST отвечать понятной ошибкой на маршрут, который провайдер не объявил"; "запрос
|
|
8
|
+
* с `tools` к этой модели получает ошибку о неподдерживаемой возможности, а не молча игнорируется").
|
|
9
|
+
*/
|
|
10
|
+
import { isJsonObject } from '../shims/index.js';
|
|
11
|
+
import { structuredErrorJson } from './errors.js';
|
|
12
|
+
/**
|
|
13
|
+
* The engine route each public path needs, and how an error names it. `/messages` needs chat
|
|
14
|
+
* completions: the forwarder translates an Anthropic request into one when the engine has no
|
|
15
|
+
* `/v1/messages` of its own.
|
|
16
|
+
*/
|
|
17
|
+
const NEEDS = {
|
|
18
|
+
'/chat/completions': { route: '/v1/chat/completions', what: 'chat completions' },
|
|
19
|
+
'/completions': { route: '/v1/completions', what: 'text completions' },
|
|
20
|
+
'/embeddings': { route: '/v1/embeddings', what: 'embeddings' },
|
|
21
|
+
'/messages': { route: '/v1/chat/completions', what: 'the Anthropic Messages API' },
|
|
22
|
+
'/messages/count_tokens': { route: '/v1/messages/count_tokens', what: 'token counting' },
|
|
23
|
+
'/responses': { route: '/v1/responses', what: 'the Responses API' },
|
|
24
|
+
};
|
|
25
|
+
function refusal(message, code) {
|
|
26
|
+
return { status: 400, body: structuredErrorJson(message, 'invalid_request_error', code) };
|
|
27
|
+
}
|
|
28
|
+
/** Why this POST must not reach the session, or `undefined` when it may. */
|
|
29
|
+
export function policyRefusal(policy, path, modelId, json) {
|
|
30
|
+
const needed = NEEDS[path];
|
|
31
|
+
const declared = needed !== undefined && policy.routes.some((r) => r.method === 'POST' && r.path === needed.route);
|
|
32
|
+
if (!declared) {
|
|
33
|
+
const what = needed?.what ?? path;
|
|
34
|
+
return refusal(`The model '${modelId}' does not support ${what}.`, 'unsupported_endpoint');
|
|
35
|
+
}
|
|
36
|
+
if (!isJsonObject(json))
|
|
37
|
+
return undefined;
|
|
38
|
+
if (!policy.tools && asksForTools(json)) {
|
|
39
|
+
return refusal(`The model '${modelId}' does not support tool calling.`, 'unsupported_capability');
|
|
40
|
+
}
|
|
41
|
+
if (!policy.structuredOutput && asksForStructuredOutput(json)) {
|
|
42
|
+
return refusal(`The model '${modelId}' does not support structured output.`, 'unsupported_capability');
|
|
43
|
+
}
|
|
44
|
+
return undefined;
|
|
45
|
+
}
|
|
46
|
+
/**
|
|
47
|
+
* A non-empty `tools` list, or a `tool_choice` other than `"none"` — the same rule the session
|
|
48
|
+
* gateway's own rewriter applies (`runtime/tensorrt-llm/adapter.ts`), so both ports agree.
|
|
49
|
+
*/
|
|
50
|
+
function asksForTools(json) {
|
|
51
|
+
const tools = json['tools'];
|
|
52
|
+
const choice = json['tool_choice'];
|
|
53
|
+
return ((Array.isArray(tools) && tools.length > 0) ||
|
|
54
|
+
(choice !== undefined && choice !== null && choice !== 'none'));
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* `response_format` asking for constrained output: anything but `{"type": "text"}` — OpenAI's
|
|
58
|
+
* `json_schema`/`json_object`, TensorRT-LLM's own `json`/`regex`/`ebnf`/`structural_tag`, and any
|
|
59
|
+
* type this core does not know (final review I-2). The session gateway's rewriter
|
|
60
|
+
* (`runtime/tensorrt-llm/adapter.ts`) applies the same rule, so both ports agree.
|
|
61
|
+
*/
|
|
62
|
+
function asksForStructuredOutput(json) {
|
|
63
|
+
const format = json['response_format'];
|
|
64
|
+
if (!isJsonObject(format))
|
|
65
|
+
return false;
|
|
66
|
+
return format['type'] !== 'text';
|
|
67
|
+
}
|
|
68
|
+
//# sourceMappingURL=policy.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"policy.js","sourceRoot":"","sources":["../../../src/server/public/policy.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAA;AAEhD,OAAO,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAA;AAGjD;;;;GAIG;AACH,MAAM,KAAK,GAAoD;IAC7D,mBAAmB,EAAE,EAAE,KAAK,EAAE,sBAAsB,EAAE,IAAI,EAAE,kBAAkB,EAAE;IAChF,cAAc,EAAE,EAAE,KAAK,EAAE,iBAAiB,EAAE,IAAI,EAAE,kBAAkB,EAAE;IACtE,aAAa,EAAE,EAAE,KAAK,EAAE,gBAAgB,EAAE,IAAI,EAAE,YAAY,EAAE;IAC9D,WAAW,EAAE,EAAE,KAAK,EAAE,sBAAsB,EAAE,IAAI,EAAE,4BAA4B,EAAE;IAClF,wBAAwB,EAAE,EAAE,KAAK,EAAE,2BAA2B,EAAE,IAAI,EAAE,gBAAgB,EAAE;IACxF,YAAY,EAAE,EAAE,KAAK,EAAE,eAAe,EAAE,IAAI,EAAE,mBAAmB,EAAE;CACpE,CAAA;AAQD,SAAS,OAAO,CAAC,OAAe,EAAE,IAAY;IAC5C,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,mBAAmB,CAAC,OAAO,EAAE,uBAAuB,EAAE,IAAI,CAAC,EAAE,CAAA;AAC3F,CAAC;AAED,4EAA4E;AAC5E,MAAM,UAAU,aAAa,CAC3B,MAAyB,EACzB,IAAY,EACZ,OAAe,EACf,IAAe;IAEf,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,CAAA;IAC1B,MAAM,QAAQ,GACZ,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,MAAM,IAAI,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,KAAK,CAAC,CAAA;IACnG,IAAI,CAAC,QAAQ,EAAE,CAAC;QACd,MAAM,IAAI,GAAG,MAAM,EAAE,IAAI,IAAI,IAAI,CAAA;QACjC,OAAO,OAAO,CAAC,cAAc,OAAO,sBAAsB,IAAI,GAAG,EAAE,sBAAsB,CAAC,CAAA;IAC5F,CAAC;IACD,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC;QAAE,OAAO,SAAS,CAAA;IACzC,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC;QACxC,OAAO,OAAO,CAAC,cAAc,OAAO,kCAAkC,EAAE,wBAAwB,CAAC,CAAA;IACnG,CAAC;IACD,IAAI,CAAC,MAAM,CAAC,gBAAgB,IAAI,uBAAuB,CAAC,IAAI,CAAC,EAAE,CAAC;QAC9D,OAAO,OAAO,CAAC,cAAc,OAAO,uCAAuC,EAAE,wBAAwB,CAAC,CAAA;IACxG,CAAC;IACD,OAAO,SAAS,CAAA;AAClB,CAAC;AAED;;;GAGG;AACH,SAAS,YAAY,CAAC,IAAkC;IACtD,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,CAAA;IAC3B,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC,CAAA;IAClC,OAAO,CACL,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;QAC1C,CAAC,MAAM,KAAK,SAAS,IAAI,MAAM,KAAK,IAAI,IAAI,MAAM,KAAK,MAAM,CAAC,CAC/D,CAAA;AACH,CAAC;AAED;;;;;GAKG;AACH,SAAS,uBAAuB,CAAC,IAAkC;IACjE,MAAM,MAAM,GAAG,IAAI,CAAC,iBAAiB,CAAC,CAAA;IACtC,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC;QAAE,OAAO,KAAK,CAAA;IACvC,OAAO,MAAM,CAAC,MAAM,CAAC,KAAK,MAAM,CAAA;AAClC,CAAC"}
|
|
@@ -11,6 +11,7 @@ import { ResponsesStreamConverter, chatResponseToResponses, newResponseId, respo
|
|
|
11
11
|
import { resolveRemoteProvider } from '../../router/index.js';
|
|
12
12
|
import { answer, clientGone, connectTimeoutMs } from './exchange.js';
|
|
13
13
|
import { outboundHeaders, readModelRequest, sseEvent } from './forward.js';
|
|
14
|
+
import { policyRefusal } from './policy.js';
|
|
14
15
|
import { SseLineReader } from './sse.js';
|
|
15
16
|
import { bodyIndicatesOom, isContextLimitError } from './errors.js';
|
|
16
17
|
import { pipeBody, readBody, readUpstreamText, relay, sendUpstream } from './wire.js';
|
|
@@ -36,8 +37,17 @@ export async function serveResponses(ex) {
|
|
|
36
37
|
// Unlike the chat routes, MLX is the passthrough here and llama.cpp is translated.
|
|
37
38
|
const session = ex.deps.findLocal('llamacpp', modelId) ??
|
|
38
39
|
ex.deps.findLocal('llamacpp-upstream', modelId) ??
|
|
40
|
+
ex.deps.findLocal('atomic-prism', modelId) ??
|
|
39
41
|
ex.deps.findLocal('mlx', modelId);
|
|
40
42
|
if (!session) {
|
|
43
|
+
// A session that declares its routes and has no Responses API says so, rather than "not found".
|
|
44
|
+
const declared = ex.deps.findLocal('tensorrt-llm', modelId) ?? ex.deps.findLocal('vllm', modelId);
|
|
45
|
+
const refused = declared?.policy && policyRefusal(declared.policy, ex.path, modelId, json);
|
|
46
|
+
if (refused) {
|
|
47
|
+
trace.errorKind = 'bad_request';
|
|
48
|
+
answer(ex, refused.status, refused.body, [['Content-Type', 'application/json']]);
|
|
49
|
+
return;
|
|
50
|
+
}
|
|
41
51
|
trace.errorKind = 'not_found';
|
|
42
52
|
answer(ex, 404, `No running session found for model '${modelId}'`);
|
|
43
53
|
return;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"responses.js","sourceRoot":"","sources":["../../../src/server/public/responses.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EACL,wBAAwB,EACxB,uBAAuB,EACvB,aAAa,EACb,sBAAsB,EACtB,aAAa,GACd,MAAM,mBAAmB,CAAA;AAE1B,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAA;AAC7D,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAA;AAEpE,OAAO,EAAE,eAAe,EAAE,gBAAgB,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAA;
|
|
1
|
+
{"version":3,"file":"responses.js","sourceRoot":"","sources":["../../../src/server/public/responses.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EACL,wBAAwB,EACxB,uBAAuB,EACvB,aAAa,EACb,sBAAsB,EACtB,aAAa,GACd,MAAM,mBAAmB,CAAA;AAE1B,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAA;AAC7D,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAA;AAEpE,OAAO,EAAE,eAAe,EAAE,gBAAgB,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAA;AAC1E,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAA;AAE3C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAA;AACxC,OAAO,EAAE,gBAAgB,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAA;AAEnE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,gBAAgB,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,WAAW,CAAA;AAGrF,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,EAAY;IAC/C,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,EAAE,CAAC,CAAA;IACzC,IAAI,CAAC,MAAM;QAAE,OAAM;IACnB,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,MAAM,CAAA;IAE7C,MAAM,KAAK,GAAG,EAAE,CAAC,KAAK,CAAA;IACtB,MAAM,MAAM,GAAG,qBAAqB,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC,CAAA;IAClE,IAAI,MAAM,EAAE,CAAC;QACX,KAAK,CAAC,OAAO,GAAG,QAAQ,CAAA;QACxB,KAAK,CAAC,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAA;QAChC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,KAAK,CAAC,SAAS,GAAG,gBAAgB,CAAA;YAClC,MAAM,CAAC,EAAE,EAAE,GAAG,EAAE,0BAA0B,CAAC,CAAA;YAC3C,OAAM;QACR,CAAC;QACD,MAAM,GAAG,GAAG,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,YAAY,CAAA;QAC7D,MAAM,WAAW,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,GAAG,EAAE,MAAM,CAAC,MAAM,IAAI,SAAS,EAAE,GAAG,CAAC,CAAA;QACjG,OAAM;IACR,CAAC;IAED,mFAAmF;IACnF,MAAM,OAAO,GACX,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,OAAO,CAAC;QACtC,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,mBAAmB,EAAE,OAAO,CAAC;QAC/C,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,cAAc,EAAE,OAAO,CAAC;QAC1C,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,OAAO,CAAC,CAAA;IACnC,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,gGAAgG;QAChG,MAAM,QAAQ,GAAG,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,cAAc,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;QACjG,MAAM,OAAO,GAAG,QAAQ,EAAE,MAAM,IAAI,aAAa,CAAC,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,CAAA;QAC1F,IAAI,OAAO,EAAE,CAAC;YACZ,KAAK,CAAC,SAAS,GAAG,aAAa,CAAA;YAC/B,MAAM,CAAC,EAAE,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,cAAc,EAAE,kBAAkB,CAAC,CAAC,CAAC,CAAA;YAChF,OAAM;QACR,CAAC;QACD,KAAK,CAAC,SAAS,GAAG,WAAW,CAAA;QAC7B,MAAM,CAAC,EAAE,EAAE,GAAG,EAAE,uCAAuC,OAAO,GAAG,CAAC,CAAA;QAClE,OAAM;IACR,CAAC;IACD,KAAK,CAAC,OAAO,GAAG,OAAO,CAAC,QAAQ,CAAA;IAChC,MAAM,OAAO,GAAY,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,CAAA;IACnD,IAAI,OAAO,CAAC,QAAQ,KAAK,KAAK,EAAE,CAAC;QAC/B,MAAM,WAAW,CAAC,EAAE,EAAE,OAAO,EAAE,oBAAoB,OAAO,CAAC,IAAI,eAAe,EAAE,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;QACpG,OAAM;IACR,CAAC;IACD,MAAM,SAAS,CACb,EAAE,EACF,oBAAoB,OAAO,CAAC,IAAI,sBAAsB,EACtD,OAAO,CAAC,MAAM,EACd,IAAI,EACJ,OAAO,EACP,MAAM,CACP,CAAA;AACH,CAAC;AAED,KAAK,UAAU,WAAW,CACxB,EAAY,EACZ,OAAgB,EAChB,GAAW,EACX,GAAuB,EACvB,GAAW;IAEX,MAAM,MAAM,GAAG,UAAU,CAAC,EAAE,CAAC,CAAA;IAC7B,IAAI,QAA0B,CAAA;IAC9B,IAAI,CAAC;QACH,QAAQ,GAAG,MAAM,YAAY,CAAC,GAAG,EAAE;YACjC,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,eAAe,CACtB,EAAE,EACF,CAAC,MAAM,EAAE,eAAe,EAAE,gBAAgB,EAAE,cAAc,CAAC,EAC3D,OAAO,EACP,GAAG,EACH,CAAC,CAAC,cAAc,EAAE,kBAAkB,CAAC,CAAC,CACvC;YACD,IAAI,EAAE,GAAG;YACT,gBAAgB,EAAE,gBAAgB,CAAC,EAAE,CAAC;YACtC,MAAM;SACP,CAAC,CAAA;IACJ,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,EAAE,CAAC,KAAK,CAAC,SAAS,GAAG,EAAE,CAAC,KAAK,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,uBAAuB,CAAC,CAAC,CAAC,yBAAyB,CAAA;QACxG,IAAI,CAAC,MAAM,CAAC,OAAO;YAAE,MAAM,CAAC,EAAE,EAAE,GAAG,EAAE,kCAAmC,CAAW,CAAC,OAAO,EAAE,CAAC,CAAA;QAC9F,OAAM;IACR,CAAC;IACD,MAAM,KAAK,CAAC,EAAE,CAAC,GAAG,EAAE,QAAQ,EAAE,EAAE,CAAC,IAAI,CAAC,CAAA;AACxC,CAAC;AAED,iGAAiG;AACjG,KAAK,UAAU,SAAS,CACtB,EAAY,EACZ,GAAW,EACX,GAAW,EACX,IAAe,EACf,OAAe,EACf,MAAe;IAEf,MAAM,MAAM,GAAG,UAAU,CAAC,EAAE,CAAC,CAAA;IAC7B,IAAI,QAA0B,CAAA;IAC9B,IAAI,CAAC;QACH,QAAQ,GAAG,MAAM,YAAY,CAAC,GAAG,EAAE;YACjC,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,CAAC,cAAc,EAAE,kBAAkB,CAAC;gBACpC,CAAC,iBAAiB,EAAE,UAAU,CAAC;gBAC/B,CAAC,eAAe,EAAE,UAAU,GAAG,EAAE,CAAC;aACnC;YACD,IAAI,EAAE,aAAa,CAAC,sBAAsB,CAAC,IAAI,CAAC,CAAC;YACjD,gBAAgB,EAAE,gBAAgB,CAAC,EAAE,CAAC;YACtC,MAAM;SACP,CAAC,CAAA;IACJ,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,EAAE,CAAC,KAAK,CAAC,SAAS,GAAG,EAAE,CAAC,KAAK,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,uBAAuB,CAAC,CAAC,CAAC,yBAAyB,CAAA;QACxG,IAAI,CAAC,MAAM,CAAC,OAAO;YAAE,MAAM,CAAC,EAAE,EAAE,GAAG,EAAE,kCAAmC,CAAW,CAAC,OAAO,EAAE,CAAC,CAAA;QAC9F,OAAM;IACR,CAAC;IAED,IAAI,QAAQ,CAAC,MAAM,GAAG,GAAG,IAAI,QAAQ,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC;QACpD,MAAM,SAAS,GAAG,MAAM,gBAAgB,CAAC,QAAQ,CAAC,CAAA;QAClD,EAAE,CAAC,KAAK,CAAC,SAAS,GAAG,mBAAmB,CAAA;QACxC,EAAE,CAAC,KAAK,CAAC,cAAc,GAAG,QAAQ,CAAC,MAAM,CAAA;QACzC,EAAE,CAAC,KAAK,CAAC,WAAW,GAAG,gBAAgB,CAAC,SAAS,CAAC,CAAA;QAClD,EAAE,CAAC,KAAK,CAAC,mBAAmB,GAAG,mBAAmB,CAAC,QAAQ,CAAC,MAAM,EAAE,SAAS,CAAC,CAAA;QAC9E,MAAM,CAAC,EAAE,EAAE,QAAQ,CAAC,MAAM,EAAE,SAAS,CAAC,CAAA;QACtC,OAAM;IACR,CAAC;IAED,MAAM,UAAU,GAAG,aAAa,EAAE,CAAA;IAClC,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,MAAM,KAAK,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAA;QACxE,IAAI,IAAI,GAAc,IAAI,CAAA;QAC1B,IAAI,CAAC;YACH,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAc,CAAA;QACxD,CAAC;QAAC,MAAM,CAAC;YACP,IAAI,GAAG,IAAI,CAAA;QACb,CAAC;QACD,MAAM,CAAC,EAAE,EAAE,GAAG,EAAE,aAAa,CAAC,uBAAuB,CAAC,IAAI,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC,EAAE;YACjF,CAAC,cAAc,EAAE,kBAAkB,CAAC;SACrC,CAAC,CAAA;QACF,OAAM;IACR,CAAC;IAED,EAAE,CAAC,GAAG,CAAC,SAAS,CACd,GAAG,EACH,CAAC,CAAC,cAAc,EAAE,mBAAmB,CAAC,EAAE,CAAC,eAAe,EAAE,UAAU,CAAC,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAC1F,CAAA;IACD,MAAM,QAAQ,CAAC,EAAE,CAAC,GAAG,EAAE,eAAe,CAAC,QAAQ,EAAE,UAAU,EAAE,OAAO,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,CACpF,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,CACxB,CAAA;AACH,CAAC;AAED,KAAK,SAAS,CAAC,CAAC,eAAe,CAC7B,QAA0B,EAC1B,UAAkB,EAClB,OAAe,EACf,KAAmB;IAEnB,MAAM,SAAS,GAAG,KAAK,CAAC,cAAc,EAAE,CAAA;IACxC,MAAM,IAAI,GAAG,IAAI,wBAAwB,CAAC,UAAU,EAAE,OAAO,CAAC,CAAA;IAC9D,MAAM,QAAQ,CAAC,IAAI,CAAC,YAAY,EAAE,CAAC,CAAA;IACnC,MAAM,MAAM,GAAG,IAAI,aAAa,EAAE,CAAA;IAClC,IAAI,KAA4B,CAAA;IAChC,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,QAAQ,CAAC,IAAI,EAAE,CAAC;QACxC,MAAM,CAAC,IAAI,CAAC,KAAe,CAAC,CAAA;QAC5B,KAAK,IAAI,IAAI,GAAG,MAAM,CAAC,QAAQ,EAAE,EAAE,IAAI,EAAE,IAAI,GAAG,MAAM,CAAC,QAAQ,EAAE,EAAE,CAAC;YAClE,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM;gBAAE,SAAQ;YAClC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAA;YAC5B,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBAC5B,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;oBAAE,MAAM,QAAQ,CAAC,KAAK,CAAC,CAAA;gBAC7D,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,CAAA;gBACvB,OAAM;YACR,CAAC;YACD,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBAC5B,MAAM,CAAC,GACL,OAAO,CAAC,IAAI,KAAK,IAAI,IAAI,OAAO,OAAO,CAAC,IAAI,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC;oBACvF,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC;oBACvB,CAAC,CAAC,SAAS,CAAA;gBACf,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,IAAI;oBAAE,KAAK,GAAG,CAAC,CAAA;gBAC5C,SAAS,EAAE,MAAM,CAAC,OAAO,CAAC,IAAI,EAAE,WAAW,CAAC,GAAG,EAAE,CAAC,CAAA;gBAClD,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC;oBAAE,MAAM,QAAQ,CAAC,KAAK,CAAC,CAAA;gBACrE,KAAK,CAAC,QAAQ,EAAE,CAAA;YAClB,CAAC;QACH,CAAC;IACH,CAAC;IACD,mEAAmE;IACnE,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC;QAAE,MAAM,QAAQ,CAAC,KAAK,CAAC,CAAA;AAC/D,CAAC"}
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
import type { ServerResponse } from 'node:http';
|
|
18
18
|
import type { JsonValue } from '../shims/index.js';
|
|
19
19
|
import { StreamTelemetry } from './telemetry.js';
|
|
20
|
-
import type { TelemetryFields } from './telemetry.js';
|
|
20
|
+
import type { PromptPreview, TelemetryFields } from './telemetry.js';
|
|
21
21
|
import type { PublicServerDeps } from './types.js';
|
|
22
22
|
export declare function endpointFromPath(path: string): string;
|
|
23
23
|
export declare class RequestTrace {
|
|
@@ -55,7 +55,7 @@ export declare class RequestTrace {
|
|
|
55
55
|
* The inspector's `request-started`, with the prompt preview when the body is known. Idempotent:
|
|
56
56
|
* a body-parse site announces with detail, the close announces without for everything else.
|
|
57
57
|
*/
|
|
58
|
-
announce(body?: JsonValue, raw?: Buffer): void;
|
|
58
|
+
announce(body?: JsonValue, raw?: Buffer, ownPreview?: PromptPreview): void;
|
|
59
59
|
/** A telemetry collector, when the inspector is watching. */
|
|
60
60
|
startTelemetry(): StreamTelemetry | undefined;
|
|
61
61
|
/** Telemetry from a whole (non-streamed) response body: there is no time to first token. */
|
|
@@ -38,6 +38,10 @@ export function endpointFromPath(path) {
|
|
|
38
38
|
return 'messages/count_tokens';
|
|
39
39
|
case '/images/generations':
|
|
40
40
|
return 'images/generations';
|
|
41
|
+
case '/systemone':
|
|
42
|
+
return 'systemone';
|
|
43
|
+
case '/router/score':
|
|
44
|
+
return 'router/score';
|
|
41
45
|
case '/videos':
|
|
42
46
|
return 'videos';
|
|
43
47
|
case '/models':
|
|
@@ -104,11 +108,11 @@ export class RequestTrace {
|
|
|
104
108
|
* The inspector's `request-started`, with the prompt preview when the body is known. Idempotent:
|
|
105
109
|
* a body-parse site announces with detail, the close announces without for everything else.
|
|
106
110
|
*/
|
|
107
|
-
announce(body, raw) {
|
|
111
|
+
announce(body, raw, ownPreview) {
|
|
108
112
|
if (this.seq === undefined || this.announced)
|
|
109
113
|
return;
|
|
110
114
|
this.announced = true;
|
|
111
|
-
const preview = body !== undefined ? promptPreview(body) : undefined;
|
|
115
|
+
const preview = ownPreview ?? (body !== undefined ? promptPreview(body) : undefined);
|
|
112
116
|
this.deps.emit?.('api:request', {
|
|
113
117
|
phase: 'started',
|
|
114
118
|
id: this.id,
|