mlx-bun 0.4.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1453 -88
- package/THIRD_PARTY_NOTICES.md +882 -0
- package/bin/mlx-bun.convert.mjs +4 -0
- package/bin/mlx-bun.fuse.mjs +4 -0
- package/bin/mlx-bun.generate.mjs +4 -0
- package/bin/mlx-bun.lora.mjs +4 -0
- package/bin/mlx-bun.mjs +13 -21
- package/bin/mlx-bun.server.mjs +4 -0
- package/bin/mlx-bun.upload.mjs +4 -0
- package/dist/web/app.js +9584 -0
- package/package.json +50 -53
- package/src/cli/ane-install.ts +76 -0
- package/src/cli/args.ts +179 -0
- package/src/cli/engine-entry.ts +68 -0
- package/src/cli/inference.ts +214 -0
- package/src/cli/job-entry.ts +97 -0
- package/src/cli/library-host.ts +117 -0
- package/src/cli/main.ts +57 -0
- package/src/cli/memory-engine.ts +163 -0
- package/src/cli/memory.ts +36 -0
- package/src/cli/mlx-lm-aliases.ts +309 -0
- package/src/cli/model-choice.ts +33 -0
- package/src/cli/model-selection.ts +84 -0
- package/src/cli/module-host.ts +11 -0
- package/src/cli/module-verbs.ts +50 -0
- package/src/cli/numerical-policy.ts +12 -0
- package/src/cli/perplexity.ts +89 -0
- package/src/cli/serve-host.ts +423 -0
- package/src/cli/serve-isolated.ts +217 -0
- package/src/cli/serve-options.ts +106 -0
- package/src/cli/serve-state.ts +205 -0
- package/src/cli/serve.ts +500 -0
- package/src/cli/served-model-host.ts +102 -0
- package/src/cli/server-entry.ts +148 -0
- package/src/cli/serving-unit.ts +177 -0
- package/src/{tui.ts → cli/terminal.ts} +13 -8
- package/src/cli/worker-adapters.ts +34 -0
- package/src/cli/worker-entry.ts +272 -0
- package/src/cli/worker-unit.ts +179 -0
- package/src/engine/adapter-operation.ts +26 -0
- package/src/engine/cache-services.ts +114 -0
- package/src/engine/completion.ts +39 -0
- package/src/engine/generation-gateway.ts +358 -0
- package/src/engine/index.ts +59 -0
- package/src/engine/kv-budget.ts +120 -0
- package/src/engine/media-preparation.ts +232 -0
- package/src/engine/model-binding.ts +24 -0
- package/src/engine/model-host.ts +513 -0
- package/src/engine/model-serving.ts +89 -0
- package/src/{serve → engine}/preparation.ts +10 -8
- package/src/{serve → engine}/session-completion-engine.ts +6 -6
- package/src/engine/telemetry.ts +109 -0
- package/src/hub/downloads.ts +116 -0
- package/src/jobs/db.ts +16 -7
- package/src/jobs/events.ts +29 -0
- package/src/jobs/executable.ts +5 -0
- package/src/jobs/host.ts +51 -0
- package/src/jobs/in-process.ts +55 -0
- package/src/{contracts/jobs.ts → jobs/protocol.ts} +7 -9
- package/src/jobs/runner.ts +150 -231
- package/src/jobs/service.ts +75 -0
- package/src/jobs/sse.ts +59 -23
- package/src/jobs/worker-memory.ts +46 -0
- package/src/jobs/worker-process.ts +163 -0
- package/src/jobs/worker-supervisor.ts +258 -0
- package/src/modules.ts +58 -0
- package/src/publishing/catalog-hub.ts +18 -0
- package/src/publishing/credentials.ts +31 -0
- package/src/publishing/upload.ts +37 -0
- package/src/residency/leased-adapters.ts +21 -0
- package/src/residency/local-records.ts +16 -0
- package/src/residency/model-residency.ts +405 -0
- package/src/residency/record-index.ts +28 -0
- package/src/residency/resident-estimate.ts +50 -0
- package/src/{anthropic.ts → server/anthropic.ts} +3 -81
- package/src/server/cache-routes.ts +36 -0
- package/src/{serve → server}/chat-request.ts +12 -10
- package/src/{serve → server}/chat-stage.ts +27 -22
- package/src/{client.ts → server/client.ts} +28 -2
- package/src/{serve → server}/completion-executor.ts +42 -42
- package/src/server/decision-body.ts +55 -0
- package/src/server/decision-routes.ts +134 -0
- package/src/server/decision-wire.ts +228 -0
- package/src/{serve → server}/discovery-routes.ts +42 -83
- package/src/server/generated-token-history.ts +57 -0
- package/src/{serve → server}/http.ts +25 -13
- package/src/{serve → server}/inference-request.ts +5 -5
- package/src/server/job-routes.ts +20 -0
- package/src/server/media-prompt.ts +63 -0
- package/src/server/memory-completion-client.ts +46 -0
- package/src/server/model-routes.ts +192 -0
- package/src/{serve → server}/openai-wire.ts +3 -2
- package/src/{serve → server}/pipeline.ts +1 -1
- package/src/server/prompt-builder.ts +13 -0
- package/src/server/prompt-contracts.ts +17 -0
- package/src/server/proxy-routes.ts +187 -0
- package/src/server/publishing-routes.ts +41 -0
- package/src/{serve → server}/request-plan.ts +8 -8
- package/src/{serve → server}/request-prep.ts +82 -68
- package/src/{serve → server}/responses-client.ts +30 -13
- package/src/{responses.ts → server/responses.ts} +12 -93
- package/src/server/routes.ts +205 -0
- package/src/server/start.ts +72 -0
- package/src/server/status-routes.ts +108 -0
- package/src/{serve → server}/text-completion-stage.ts +25 -4
- package/src/server/text-prompt.ts +12 -0
- package/src/{serve → server}/token-streams.ts +36 -38
- package/src/server/worker-routes.ts +311 -0
- package/src/storage/paths.ts +61 -0
- package/src/web/assets.ts +37 -0
- package/src/web/{src → browser}/api.ts +5 -23
- package/src/web/browser/chat-host.ts +17 -0
- package/src/web/browser/installed-panels.ts +6 -0
- package/src/web/browser/main.ts +55 -0
- package/src/web/browser/markdown.ts +279 -0
- package/src/web/browser/memory-panel.ts +9 -0
- package/src/web/browser/palette.ts +64 -0
- package/src/web/browser/protocol.ts +13 -0
- package/src/web/browser/shell.ts +467 -0
- package/src/web/{src → browser}/status.ts +32 -36
- package/src/web/build.ts +51 -0
- package/src/web/public/app.html +1054 -0
- package/src/web/{sw.js → public/sw.js} +4 -3
- package/src/web/{vendor → public/vendor}/hljs-theme.css +2 -2
- package/src/web/{vendor → public/vendor}/hljs.js +2 -2
- package/THIRD_PARTY_LICENSES.md +0 -51
- package/docs/README.md +0 -94
- package/docs/archive/hlg-sampling.md +0 -350
- package/docs/archive/investigations/audio-input-plan.md +0 -356
- package/docs/archive/investigations/chunk-finetune-experiment.md +0 -84
- package/docs/archive/investigations/colibri-glm52-port.md +0 -1486
- package/docs/archive/investigations/colibri-oracle-pin.md +0 -1284
- package/docs/archive/investigations/curve-bisector-routing.md +0 -123
- package/docs/archive/investigations/decode-roofline-lookagain.md +0 -348
- package/docs/archive/investigations/diffusion-gemma-port.md +0 -602
- package/docs/archive/investigations/dspark-handoff.md +0 -224
- package/docs/archive/investigations/expert-offload-single-user-moe.md +0 -292
- package/docs/archive/investigations/faithful-l1-consolidation.md +0 -310
- package/docs/archive/investigations/hlg-sampling-investigation.md +0 -970
- package/docs/archive/investigations/indexshare-performance-spike.md +0 -357
- package/docs/archive/investigations/kernel-perf-review-2026-07.md +0 -171
- package/docs/archive/investigations/lab-build-journal.md +0 -157
- package/docs/archive/investigations/minicpm5-decode-megakernel.md +0 -489
- package/docs/archive/investigations/openjarvis-fit.md +0 -198
- package/docs/archive/investigations/openwiki-evaluation.md +0 -249
- package/docs/archive/investigations/optimization_plan.md +0 -346
- package/docs/archive/investigations/orpo-base-uf-experiment-and-directions.md +0 -120
- package/docs/archive/investigations/orpo-flash-cce-pin-leak.md +0 -167
- package/docs/archive/investigations/orpo-uf-testing-handoff.md +0 -126
- package/docs/archive/investigations/pareto-specialized-runtime-findings.md +0 -92
- package/docs/archive/investigations/pi-builtin-investigation.md +0 -308
- package/docs/archive/investigations/pre-colibri-stabilization.md +0 -329
- package/docs/archive/investigations/qwen38-27b-campaign-log.md +0 -1092
- package/docs/archive/investigations/segmented-backward-handoff.md +0 -148
- package/docs/archive/investigations/starter-model-port-handoff.md +0 -342
- package/docs/archive/investigations/steel-flash-cce-handoff.md +0 -489
- package/docs/archive/investigations/the-dreaming-handoff.md +0 -141
- package/docs/archive/investigations/trainer-validation-experiment.md +0 -173
- package/docs/archive/planning/chat-ui-vision-memory-first.md +0 -199
- package/docs/archive/planning/chat-ui-vision.md +0 -50
- package/docs/archive/planning/chunk-task-roadmap.md +0 -77
- package/docs/archive/planning/curve-sampler-research-plan.md +0 -331
- package/docs/archive/planning/journal.md +0 -316
- package/docs/archive/planning/memory-docs-and-dag-plan.md +0 -194
- package/docs/archive/planning/pi-integration-review.md +0 -178
- package/docs/archive/planning/release-notes-v0.0.10.md +0 -57
- package/docs/archive/planning/release-notes-v0.0.11.md +0 -239
- package/docs/archive/planning/release-notes-v0.0.12.md +0 -80
- package/docs/archive/planning/release-notes-v0.0.13.md +0 -77
- package/docs/archive/planning/release-notes-v0.0.9.md +0 -122
- package/docs/archive/planning/release-notes-v0.2.0.md +0 -49
- package/docs/archive/planning/release-notes-v0.3.0.md +0 -35
- package/docs/archive/planning/web-ui-pass-plan.md +0 -213
- package/docs/archive/planning/website-readme-pass-plan.md +0 -224
- package/docs/design/batching.md +0 -1841
- package/docs/design/decode-speed-program.md +0 -2349
- package/docs/design/dreaming-nightly-pipeline.md +0 -684
- package/docs/design/generic-model-support.md +0 -705
- package/docs/design/kv-cache.md +0 -824
- package/docs/design/orpo-training.md +0 -680
- package/docs/design/speculative-decoding.md +0 -1990
- package/docs/design/turboquant.md +0 -2212
- package/docs/design/unified-engine-frontier-plan.md +0 -1880
- package/docs/design/web-chat-redesign.md +0 -695
- package/docs/planning/PRODUCT_ROADMAP.md +0 -594
- package/docs/planning/ResearchTopics.md +0 -119
- package/docs/planning/release-notes-v0.4.0.md +0 -45
- package/docs/planning/video-series.md +0 -421
- package/docs/reference/benchmarks.md +0 -3613
- package/docs/reference/cli.md +0 -585
- package/docs/reference/distribution.md +0 -355
- package/docs/reference/environment.md +0 -398
- package/docs/reference/glossary.md +0 -39
- package/docs/reference/library-api.md +0 -674
- package/docs/reference/memory.md +0 -260
- package/docs/reference/models.md +0 -334
- package/docs/reference/server-api.md +0 -1775
- package/docs/reference/server-config.md +0 -846
- package/docs/reference/training.md +0 -462
- package/docs/reference/troubleshooting.md +0 -158
- package/src/admission-plan.ts +0 -18
- package/src/audio/conformer.ts +0 -647
- package/src/audio/decode.ts +0 -133
- package/src/audio/features.ts +0 -202
- package/src/audio/transcode.ts +0 -67
- package/src/backends/mlx/assistant-target.ts +0 -46
- package/src/backends/mlx/autoregressive-draft-rows.ts +0 -120
- package/src/backends/mlx/autoregressive.ts +0 -108
- package/src/backends/mlx/batch-group.ts +0 -1790
- package/src/backends/mlx/cache-attachment.ts +0 -67
- package/src/backends/mlx/cache-layout.ts +0 -31
- package/src/backends/mlx/checkpoint-state.ts +0 -53
- package/src/backends/mlx/continuation-persistence.ts +0 -106
- package/src/backends/mlx/continuation-request.ts +0 -43
- package/src/backends/mlx/continuation.ts +0 -77
- package/src/backends/mlx/diffusion.ts +0 -39
- package/src/backends/mlx/draft-target.ts +0 -32
- package/src/backends/mlx/evaluation.ts +0 -105
- package/src/backends/mlx/fill-append.ts +0 -45
- package/src/backends/mlx/full-kv-row-append.ts +0 -32
- package/src/backends/mlx/full-kv-row-donor.ts +0 -54
- package/src/backends/mlx/full-transitioning-kv-rows.ts +0 -65
- package/src/backends/mlx/gateway-binding.ts +0 -130
- package/src/backends/mlx/graph.ts +0 -38
- package/src/backends/mlx/kv-codec.ts +0 -9
- package/src/backends/mlx/kv-maintenance.ts +0 -133
- package/src/backends/mlx/kv-tensor-rows.ts +0 -212
- package/src/backends/mlx/memory-client.ts +0 -189
- package/src/backends/mlx/methods.ts +0 -90
- package/src/backends/mlx/model-prompt.ts +0 -164
- package/src/backends/mlx/model-serving.ts +0 -79
- package/src/backends/mlx/prefill-cohort.ts +0 -89
- package/src/backends/mlx/prefill-rows.ts +0 -200
- package/src/backends/mlx/prefill.ts +0 -33
- package/src/backends/mlx/projected-context-rows.ts +0 -72
- package/src/backends/mlx/request-policy.ts +0 -20
- package/src/backends/mlx/request-state-policy.ts +0 -41
- package/src/backends/mlx/rollback.ts +0 -89
- package/src/backends/mlx/row-sampling.ts +0 -49
- package/src/backends/mlx/serial-executor.ts +0 -242
- package/src/backends/mlx/speculative-group.ts +0 -359
- package/src/backends/mlx/speculative-round.ts +0 -161
- package/src/backends/mlx/speculative.ts +0 -114
- package/src/backends/mlx/state-rows.ts +0 -104
- package/src/backends/mlx/state-views.ts +0 -46
- package/src/backends/mlx/target-layout-capability.ts +0 -35
- package/src/backends/mlx/text-engine.ts +0 -73
- package/src/backends/mlx/transitioning-kv-rows.ts +0 -165
- package/src/chat-template.ts +0 -382
- package/src/cli.ts +0 -3168
- package/src/config.ts +0 -399
- package/src/contracts/completion.ts +0 -16
- package/src/contracts/execution.ts +0 -60
- package/src/contracts/generation.ts +0 -103
- package/src/contracts/host.ts +0 -7
- package/src/contracts/pi-provider.ts +0 -30
- package/src/contracts/pi.ts +0 -259
- package/src/contracts/prefix-cache.ts +0 -23
- package/src/contracts/resources.ts +0 -25
- package/src/contracts/sampling.ts +0 -23
- package/src/contracts/scheduling.ts +0 -45
- package/src/contracts/tsconfig.json +0 -14
- package/src/dataset/generators.ts +0 -1087
- package/src/dataset/index.ts +0 -12
- package/src/dataset/job.ts +0 -81
- package/src/dataset/llm.ts +0 -71
- package/src/dataset/registry.ts +0 -528
- package/src/diffusion/diffusion-generate.ts +0 -507
- package/src/download.ts +0 -505
- package/src/embed.ts +0 -77
- package/src/engine/admission.ts +0 -59
- package/src/engine/cancellation.ts +0 -38
- package/src/engine/engine.ts +0 -56
- package/src/engine/execution-coordinator.ts +0 -70
- package/src/engine/execution-plan.ts +0 -53
- package/src/engine/resources.ts +0 -53
- package/src/engine/scheduler.ts +0 -72
- package/src/engine/session.ts +0 -247
- package/src/eval/README.md +0 -130
- package/src/eval/ifeval.ts +0 -81
- package/src/eval/kl-prompts.ts +0 -29
- package/src/eval/kl.ts +0 -590
- package/src/eval/perplexity.ts +0 -164
- package/src/eval/quality-db.ts +0 -92
- package/src/eval/reference-logits.ts +0 -207
- package/src/eval/runner.ts +0 -124
- package/src/eval/score.ts +0 -49
- package/src/eval/tasks/bfcl.ts +0 -535
- package/src/eval/tasks/gsm8k.ts +0 -98
- package/src/eval/tasks/hashhop.ts +0 -139
- package/src/eval/tasks/humaneval.ts +0 -275
- package/src/eval/tasks/ifeval.ts +0 -525
- package/src/eval/tasks/mmlu.ts +0 -157
- package/src/eval/ultrafeedback.ts +0 -60
- package/src/evaldb.ts +0 -199
- package/src/expert-io.ts +0 -393
- package/src/expert-offload-build.ts +0 -77
- package/src/expert-offload.ts +0 -76
- package/src/expert-residency.ts +0 -1134
- package/src/expert-usage.ts +0 -415
- package/src/fill/echo-index.ts +0 -188
- package/src/fill/fill-session.ts +0 -452
- package/src/fill/proposal.ts +0 -68
- package/src/fill/schema-rows.ts +0 -506
- package/src/fill/tool-boundary.ts +0 -103
- package/src/fit.ts +0 -249
- package/src/generate.ts +0 -1346
- package/src/grammar.ts +0 -591
- package/src/harness-pi.ts +0 -184
- package/src/hf-push.ts +0 -628
- package/src/hub-rest.ts +0 -242
- package/src/index.ts +0 -63
- package/src/inference/denoising.ts +0 -17
- package/src/inference/draft-acceptance.ts +0 -41
- package/src/inference/draft-output.ts +0 -49
- package/src/inference/graph.ts +0 -33
- package/src/inference/prefill.ts +0 -39
- package/src/inference/rollback.ts +0 -7
- package/src/jobs/index.ts +0 -31
- package/src/jobs/job-entry.ts +0 -107
- package/src/jobs/task-client.ts +0 -19
- package/src/jobs/types.ts +0 -2
- package/src/kv-scheme.ts +0 -257
- package/src/kv-store.ts +0 -1099
- package/src/lab/README.md +0 -18
- package/src/lab/curve/curve-designer.html +0 -303
- package/src/lab/curve/curve-sampler.ts +0 -122
- package/src/lab/expert-trace/expert-trace.ts +0 -69
- package/src/lab/paged-kv/paged-kv-rows.ts +0 -83
- package/src/lab/paged-kv/paged-kv.ts +0 -270
- package/src/library.ts +0 -40
- package/src/lora.ts +0 -408
- package/src/md-modules.d.ts +0 -49
- package/src/media-fetch.ts +0 -442
- package/src/memory/article.ts +0 -558
- package/src/memory/chunk-validation.ts +0 -150
- package/src/memory/chunk.ts +0 -318
- package/src/memory/cluster.ts +0 -351
- package/src/memory/coverage.ts +0 -134
- package/src/memory/crosslink.ts +0 -0
- package/src/memory/db.ts +0 -253
- package/src/memory/entity.ts +0 -203
- package/src/memory/gate.ts +0 -187
- package/src/memory/history.ts +0 -152
- package/src/memory/ingest.ts +0 -335
- package/src/memory/model.ts +0 -183
- package/src/memory/normalize.ts +0 -671
- package/src/memory/parse.ts +0 -22
- package/src/memory/pipeline.ts +0 -275
- package/src/memory/prompts.ts +0 -65
- package/src/memory/query.ts +0 -334
- package/src/memory/reconcile.ts +0 -734
- package/src/memory/redirect.ts +0 -108
- package/src/memory/reindex.ts +0 -325
- package/src/memory/resolve.ts +0 -307
- package/src/memory/rest.ts +0 -245
- package/src/memory/route.ts +0 -272
- package/src/memory/schedule.ts +0 -165
- package/src/memory/stages.ts +0 -777
- package/src/memory/synthesize.ts +0 -1638
- package/src/memory/tools.ts +0 -749
- package/src/memory/vault.ts +0 -1194
- package/src/memory/wikify.ts +0 -1064
- package/src/mlx/array.ts +0 -388
- package/src/mlx/autograd.ts +0 -304
- package/src/mlx/checkpoint.ts +0 -149
- package/src/mlx/compile.ts +0 -168
- package/src/mlx/custom-vjp.ts +0 -188
- package/src/mlx/ffi.ts +0 -512
- package/src/mlx/materialize.ts +0 -23
- package/src/mlx/metal-kernel.ts +0 -159
- package/src/mlx/ops.ts +0 -1021
- package/src/mlx/quantized-rows.ts +0 -36
- package/src/mlx/token-bitmask.ts +0 -52
- package/src/mlx/turboquant-kv-decode.ts +0 -82
- package/src/mlx/turboquant-kv-inverse.ts +0 -122
- package/src/mlx/turboquant-ops.ts +0 -861
- package/src/mlx/turboquant-tables.ts +0 -237
- package/src/mmap.ts +0 -90
- package/src/model/artifact-identity.ts +0 -73
- package/src/model/batched-kv.ts +0 -235
- package/src/model/batched-mask.ts +0 -279
- package/src/model/batched-quant.ts +0 -212
- package/src/model/batched-quantized-kv.ts +0 -144
- package/src/model/batched-rotating-quant.ts +0 -353
- package/src/model/batched-rotating-state.ts +0 -149
- package/src/model/batched-rotating.ts +0 -338
- package/src/model/batched-row-storage.ts +0 -179
- package/src/model/batched-ssm.ts +0 -52
- package/src/model/batched-turboquant-kv.ts +0 -93
- package/src/model/compiled-decode.ts +0 -712
- package/src/model/decoded-kv-donor.ts +0 -12
- package/src/model/delayed-quantized-kv.ts +0 -72
- package/src/model/delayed-rotating-quantized-kv.ts +0 -93
- package/src/model/delayed-turboquant-kv.ts +0 -55
- package/src/model/diffusion-gemma.ts +0 -1026
- package/src/model/factory.ts +0 -189
- package/src/model/fingerprint.ts +0 -33
- package/src/model/flash-attention.ts +0 -694
- package/src/model/full-prefill-padding.ts +0 -56
- package/src/model/full-prefill-row.ts +0 -105
- package/src/model/gemma4-base.ts +0 -2351
- package/src/model/gemma4.ts +0 -1315
- package/src/model/generated/gemma4-12b.ts +0 -429
- package/src/model/generated/gemma4-26b.ts +0 -448
- package/src/model/generated/gemma4-e4b.ts +0 -717
- package/src/model/generated/index.ts +0 -18
- package/src/model/glm52-atlas.ts +0 -511
- package/src/model/glm52-cache.ts +0 -792
- package/src/model/glm52-config.ts +0 -262
- package/src/model/glm52-container.ts +0 -305
- package/src/model/glm52-coupling.ts +0 -388
- package/src/model/glm52-dsa.ts +0 -606
- package/src/model/glm52-expert-layout.ts +0 -253
- package/src/model/glm52-layout.ts +0 -152
- package/src/model/glm52-memory.ts +0 -500
- package/src/model/glm52-mla.ts +0 -792
- package/src/model/glm52-moe.ts +0 -223
- package/src/model/glm52-pilot.ts +0 -537
- package/src/model/glm52-reference.ts +0 -445
- package/src/model/glm52-residency.ts +0 -557
- package/src/model/glm52-resident-weights.ts +0 -135
- package/src/model/glm52-streamed-experts.ts +0 -453
- package/src/model/glm52-streamed-metal.ts +0 -610
- package/src/model/glm52-weights.ts +0 -503
- package/src/model/glm52.ts +0 -1132
- package/src/model/implementation.ts +0 -69
- package/src/model/kv-attention-view.ts +0 -67
- package/src/model/minicpm5.ts +0 -376
- package/src/model/profile.ts +0 -443
- package/src/model/qwen-append.ts +0 -12
- package/src/model/qwen-conv.ts +0 -52
- package/src/model/qwen3-delta-state.ts +0 -74
- package/src/model/qwen3-delta.ts +0 -451
- package/src/model/qwen3-moe.ts +0 -404
- package/src/model/qwen3-mrope.ts +0 -252
- package/src/model/qwen3.ts +0 -277
- package/src/model/qwen3_5-checkpoint.ts +0 -164
- package/src/model/qwen3_5.ts +0 -819
- package/src/model/rotating-kv-donor.ts +0 -35
- package/src/model/rotating-kv-layout.ts +0 -262
- package/src/model/rotating-row-transaction.ts +0 -35
- package/src/model/speculative-rotating-kv.ts +0 -101
- package/src/model/ssm-prefill-padding.ts +0 -39
- package/src/model/support.ts +0 -102
- package/src/model/trellis-balanced-scatter.ts +0 -51
- package/src/model/trellis-linear.ts +0 -672
- package/src/model/trellis-shared-m.ts +0 -151
- package/src/model/trellis-shared-scatter.ts +0 -144
- package/src/model/trellis-splitk-prefill.ts +0 -158
- package/src/model/trellis-tiled-prefill.ts +0 -136
- package/src/model/trellis-vector-expand.ts +0 -62
- package/src/model/trellis-wide-prefill.ts +0 -94
- package/src/model/turboquant-codec.ts +0 -83
- package/src/model/universal/archs.ts +0 -332
- package/src/model/universal/dense.ts +0 -612
- package/src/model/universal/modules.ts +0 -170
- package/src/model/universal/rope.ts +0 -381
- package/src/native/expert_io.c +0 -523
- package/src/native/frame_extract.swift +0 -140
- package/src/native-pack.ts +0 -162
- package/src/pi-provider.ts +0 -53
- package/src/pi-session.ts +0 -81
- package/src/pi-terminal.ts +0 -274
- package/src/pi-web.ts +0 -1932
- package/src/preflight.ts +0 -158
- package/src/prompt-cache.ts +0 -460
- package/src/quantize/allocator.ts +0 -627
- package/src/quantize/atomic-output.ts +0 -38
- package/src/quantize/calibration.ts +0 -181
- package/src/quantize/config-writer.ts +0 -145
- package/src/quantize/index.ts +0 -84
- package/src/quantize/job.ts +0 -162
- package/src/quantize/quantizer.ts +0 -666
- package/src/quantize/rotate.ts +0 -603
- package/src/quantize/safetensors-writer.ts +0 -249
- package/src/quantize/sensitivity.ts +0 -216
- package/src/quantize/trellis.ts +0 -586
- package/src/quantize/weight-transform.ts +0 -292
- package/src/registry.ts +0 -602
- package/src/runtime-config.ts +0 -81
- package/src/safetensors.ts +0 -158
- package/src/sampler.ts +0 -1011
- package/src/serve/admin-routes.ts +0 -184
- package/src/serve/assets/curve-terrain.html +0 -105
- package/src/serve/assets/training-inference-map.html +0 -575
- package/src/serve/aux-routes.ts +0 -132
- package/src/serve/batch-scheduler.ts +0 -8
- package/src/serve/checkpoint-identity.ts +0 -64
- package/src/serve/generation-gateway.ts +0 -496
- package/src/serve/isolate.ts +0 -609
- package/src/serve/kv-budget.ts +0 -14
- package/src/serve/lab-routes.ts +0 -341
- package/src/serve/lane-registry.ts +0 -45
- package/src/serve/model-admin-routes.ts +0 -155
- package/src/serve/model-binding.ts +0 -43
- package/src/serve/model-host.ts +0 -511
- package/src/serve/parent-application.ts +0 -56
- package/src/serve/prompt-response-trace.ts +0 -147
- package/src/serve/response-lease.ts +0 -24
- package/src/serve/session-search.ts +0 -326
- package/src/serve/static-routes.ts +0 -101
- package/src/server.ts +0 -1314
- package/src/spec/assistant-rows.ts +0 -92
- package/src/spec/assistant-source.ts +0 -126
- package/src/spec/cache-identity.ts +0 -22
- package/src/spec/deepspec-rows.ts +0 -29
- package/src/spec/deepspec-source.ts +0 -145
- package/src/spec/dflash-source.ts +0 -149
- package/src/spec/drafter.ts +0 -304
- package/src/spec/dspark/ab-stats.ts +0 -136
- package/src/spec/dspark/calibration.ts +0 -112
- package/src/spec/dspark/data-dflash.ts +0 -156
- package/src/spec/dspark/data.ts +0 -201
- package/src/spec/dspark/deepspec-module.ts +0 -776
- package/src/spec/dspark/generate-dflash.ts +0 -245
- package/src/spec/dspark/generate.ts +0 -290
- package/src/spec/dspark/loader.ts +0 -23
- package/src/spec/dspark/loss.ts +0 -213
- package/src/spec/dspark/module-dflash.ts +0 -560
- package/src/spec/dspark/module.ts +0 -540
- package/src/spec/dspark/quantize-drafter.ts +0 -91
- package/src/spec/dspark/sample.ts +0 -112
- package/src/spec/generate.ts +0 -276
- package/src/spec/glm52-mtp-graph.ts +0 -29
- package/src/spec/glm52-mtp-rows.ts +0 -132
- package/src/spec/glm52-mtp-source.ts +0 -317
- package/src/spec/glm52-mtp-state.ts +0 -33
- package/src/spec/ngram-source.ts +0 -204
- package/src/spec/projected-draft-rows.ts +0 -73
- package/src/spec/qwen-mtp-module.ts +0 -171
- package/src/spec/qwen-mtp-rows.ts +0 -219
- package/src/spec/qwen-mtp-source.ts +0 -383
- package/src/spec/qwen-mtp-state.ts +0 -30
- package/src/spec/serve-loop.ts +0 -585
- package/src/spec/source.ts +0 -215
- package/src/spec/two-model-rows.ts +0 -61
- package/src/spec/two-model.ts +0 -113
- package/src/ssd-cache.ts +0 -372
- package/src/ssd-durability.ts +0 -211
- package/src/tokenizer.ts +0 -335
- package/src/tool-approvals.ts +0 -131
- package/src/tool-call.ts +0 -695
- package/src/train/dataset.ts +0 -484
- package/src/train/diffusion-lora.ts +0 -216
- package/src/train/export.ts +0 -32
- package/src/train/flash-cce.ts +0 -1234
- package/src/train/forward.ts +0 -266
- package/src/train/fuse.ts +0 -243
- package/src/train/index.ts +0 -46
- package/src/train/job.ts +0 -205
- package/src/train/lora-params.ts +0 -254
- package/src/train/loss.ts +0 -1591
- package/src/train/merge.ts +0 -195
- package/src/train/optimizer.ts +0 -183
- package/src/train/prefix-shared.ts +0 -490
- package/src/train/rank.ts +0 -137
- package/src/train/segmented.ts +0 -1828
- package/src/train/steel-qmm-header.ts +0 -1461
- package/src/train/trainer.ts +0 -1221
- package/src/train/watch.ts +0 -542
- package/src/vision/diffusion-vision.ts +0 -620
- package/src/vision/embedder.ts +0 -168
- package/src/vision/preprocess.ts +0 -247
- package/src/vision/prompt.ts +0 -373
- package/src/vision/qwen3vl-preprocess.ts +0 -409
- package/src/vision/qwen3vl-prompt.ts +0 -196
- package/src/vision/qwen3vl-tower.ts +0 -555
- package/src/vision/siglip.ts +0 -599
- package/src/vision/video-frames.ts +0 -148
- package/src/web/app.html +0 -2143
- package/src/web/app.js +0 -5703
- package/src/web/skills/memory/SKILL.md +0 -120
- package/src/web/skills/web-research/SKILL.md +0 -46
- package/src/web/skills.ts +0 -97
- package/src/web/src/adapters-panel.ts +0 -288
- package/src/web/src/assistant.ts +0 -451
- package/src/web/src/chat.ts +0 -1057
- package/src/web/src/composer.ts +0 -959
- package/src/web/src/dataset.ts +0 -132
- package/src/web/src/finetune.ts +0 -267
- package/src/web/src/hub.ts +0 -428
- package/src/web/src/main.ts +0 -98
- package/src/web/src/markdown.ts +0 -0
- package/src/web/src/memory-panel.ts +0 -572
- package/src/web/src/model-picker.ts +0 -159
- package/src/web/src/palette.ts +0 -346
- package/src/web/src/protocol.ts +0 -26
- package/src/web/src/quantize.ts +0 -167
- package/src/web/src/rag.ts +0 -274
- package/src/web/src/sessions.ts +0 -469
- package/src/web/src/shell.ts +0 -804
- package/src/web/src/ui-catalog.ts +0 -78
- package/src/web/vendor/README.md +0 -61
- package/src/web-assets.ts +0 -47
- package/src/web-tools.ts +0 -478
- package/src/weights.ts +0 -255
- /package/src/{serve → server}/completion-sink.ts +0 -0
- /package/src/web/{icon.svg → public/icon.svg} +0 -0
- /package/src/web/{manifest.webmanifest → public/manifest.webmanifest} +0 -0
- /package/src/web/{vendor → public/vendor}/hljs-LICENSE +0 -0
|
@@ -0,0 +1,358 @@
|
|
|
1
|
+
import { AdmissionPool } from "@mlx-bun/inference/execution/admission";
|
|
2
|
+
import { cleanupFailure, disposeResources } from "@mlx-bun/inference/runtime/resources";
|
|
3
|
+
import type { createRowSampling, MlxGatewayBinding, MlxBatchGroup } from "@mlx-bun/inference/execution";
|
|
4
|
+
import type { GenerateOptions, GenerateStats } from "@mlx-bun/inference/generation";
|
|
5
|
+
import type { RuntimeConfig } from "@mlx-bun/inference/runtime/config";
|
|
6
|
+
import type { PromptResponseTrace } from "@mlx-bun/inference/runtime/trace";
|
|
7
|
+
import type { KvCacheService, RequestStateCache } from "@mlx-bun/inference/state";
|
|
8
|
+
import type { DisposableResource } from "@mlx-bun/inference/contracts/portable";
|
|
9
|
+
import type { DeviceStepSampler, DeviceStepSamplerConfig } from "@mlx-bun/inference/sampling";
|
|
10
|
+
import type { CompletionEngine, RequestShape, GenerationPlacement, Vision, OnToken } from "./completion";
|
|
11
|
+
import { UnsupportedExecutionError } from "./completion";
|
|
12
|
+
import { acquireReservation } from "./preparation";
|
|
13
|
+
|
|
14
|
+
/** Async mutex: acquire() resolves to a release fn; releases run FIFO. */
|
|
15
|
+
class AsyncMutex {
|
|
16
|
+
#tail: Promise<void> = Promise.resolve();
|
|
17
|
+
/** Holders + waiters. > 0 means somebody owns or wants the engine. */
|
|
18
|
+
#pending = 0;
|
|
19
|
+
get locked(): boolean {
|
|
20
|
+
return this.#pending > 0;
|
|
21
|
+
}
|
|
22
|
+
acquire(signal?: AbortSignal): Promise<() => void> {
|
|
23
|
+
if (signal?.aborted) return Promise.reject(signal.reason);
|
|
24
|
+
this.#pending++;
|
|
25
|
+
let release!: () => void;
|
|
26
|
+
const gate = new Promise<void>((r) => { release = r; });
|
|
27
|
+
const wait = this.#tail;
|
|
28
|
+
this.#tail = this.#tail.then(() => gate);
|
|
29
|
+
let released = false;
|
|
30
|
+
const unlock = () => {
|
|
31
|
+
if (released) return; // idempotent: a double release must not skew #pending
|
|
32
|
+
released = true;
|
|
33
|
+
this.#pending--;
|
|
34
|
+
release();
|
|
35
|
+
};
|
|
36
|
+
return new Promise((resolve, reject) => {
|
|
37
|
+
let settled = false;
|
|
38
|
+
const abort = () => {
|
|
39
|
+
if (settled) return;
|
|
40
|
+
settled = true;
|
|
41
|
+
signal?.removeEventListener("abort", abort);
|
|
42
|
+
// Release this waiter's gate, not its predecessor's. Later acquirers
|
|
43
|
+
// still wait for the active holder, preserving mutual exclusion.
|
|
44
|
+
unlock();
|
|
45
|
+
reject(signal!.reason);
|
|
46
|
+
};
|
|
47
|
+
signal?.addEventListener("abort", abort, { once: true });
|
|
48
|
+
wait.then(() => {
|
|
49
|
+
if (settled) return;
|
|
50
|
+
settled = true;
|
|
51
|
+
signal?.removeEventListener("abort", abort);
|
|
52
|
+
resolve(unlock);
|
|
53
|
+
});
|
|
54
|
+
});
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
export function disposeUnstartedRequest(options: GenerateOptions, vision?: Vision): void {
|
|
59
|
+
disposeResources([options.grammar, vision?.embeddings, vision?.imageMask,
|
|
60
|
+
vision?.multimodalMask, options.visionPixels].filter((resource) => resource != null));
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/** After reservation, run() releases the request's media and pixels once on
|
|
64
|
+
* every path; a request that fails before submission releases only its grammar. */
|
|
65
|
+
function disposeReservedUnstarted(options: GenerateOptions): void {
|
|
66
|
+
options.grammar?.dispose();
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/** The app's one execution path: lone and concurrent requests use the same library
|
|
70
|
+
* scheduler. Exclusive leases protect mutations and managed jobs. */
|
|
71
|
+
export class GenerationGateway implements CompletionEngine {
|
|
72
|
+
#closed = false;
|
|
73
|
+
readonly #runtime: RuntimeConfig;
|
|
74
|
+
readonly #binding: MlxGatewayBinding;
|
|
75
|
+
readonly #requests: AdmissionPool;
|
|
76
|
+
readonly #mutex = new AsyncMutex();
|
|
77
|
+
readonly #batch: number;
|
|
78
|
+
#scheduler: MlxBatchGroup | null = null;
|
|
79
|
+
#rowsSubmitted = 0;
|
|
80
|
+
|
|
81
|
+
#exclusiveWaiters = 0;
|
|
82
|
+
constructor(
|
|
83
|
+
binding: MlxGatewayBinding,
|
|
84
|
+
capacity: number,
|
|
85
|
+
private readonly opts: {
|
|
86
|
+
/** The loaded model's cache service: the scheduler's request state, and
|
|
87
|
+
* saved prefixes read ahead of admission. Borrowed; its owner closes it. */
|
|
88
|
+
cache?: RequestStateCache & Pick<KvCacheService, "prefetch" | "checkpointEveryTokens">;
|
|
89
|
+
adapterNamespace?: (adapters: string[]) => string;
|
|
90
|
+
/** Builds the per-row sampler for ordinary rows (binding-owned method and
|
|
91
|
+
* continuation rows sample through their binding). Default: makeStepSampler. */
|
|
92
|
+
samplerFactory?: (options: GenerateOptions, config: DeviceStepSamplerConfig) => DeviceStepSampler;
|
|
93
|
+
} = {},
|
|
94
|
+
) {
|
|
95
|
+
this.#binding = binding;
|
|
96
|
+
this.#runtime = this.#binding.runtime;
|
|
97
|
+
this.opts = Object.freeze({ ...opts });
|
|
98
|
+
if (!Number.isSafeInteger(capacity) || capacity < 1) throw new Error("execution capacity must be a positive integer");
|
|
99
|
+
this.#batch = capacity;
|
|
100
|
+
this.#requests = new AdmissionPool(this.#batch);
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/** Rows currently decoding in the batch (0 if no scheduler / idle). */
|
|
104
|
+
get activeRows(): number {
|
|
105
|
+
return this.#scheduler?.activeRows ?? 0;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/** Queued + mid-prefill rows waiting behind the batch (0 when idle). */
|
|
109
|
+
get pendingRows(): number {
|
|
110
|
+
return this.#scheduler?.pendingRows ?? 0;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
get busy(): boolean {
|
|
114
|
+
return this.#requests.active > 0 || this.#requests.queued > 0 ||
|
|
115
|
+
this.#mutex.locked || this.activeRows > 0 || this.pendingRows > 0;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
async close(): Promise<void> {
|
|
119
|
+
this.#closed = true;
|
|
120
|
+
this.#requests.close();
|
|
121
|
+
try { await this.#scheduler?.close(); }
|
|
122
|
+
finally { await this.onIdle(); }
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/** Observe idleness. Call runWhenIdle for background work that must also
|
|
126
|
+
* acquire ownership atomically with the idle check. */
|
|
127
|
+
async onIdle(pollMs = 20): Promise<void> {
|
|
128
|
+
while (this.busy) await new Promise<void>((r) => setTimeout(r, pollMs));
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/** Background work waits without requesting a batch drain. The final busy
|
|
132
|
+
* check and mutex acquisition have no await between them, so a foreground
|
|
133
|
+
* request cannot enter between the check and ownership registration. */
|
|
134
|
+
async runWhenIdle<T>(fn: () => Promise<T>, pollMs = 20): Promise<T> {
|
|
135
|
+
while (this.busy) await new Promise<void>(resolve => setTimeout(resolve, pollMs));
|
|
136
|
+
return this.runExclusive(fn);
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
get submittedRows(): number {
|
|
140
|
+
return this.#rowsSubmitted;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
place(shape: RequestShape, options: GenerateOptions = {}): GenerationPlacement {
|
|
144
|
+
if (this.#closed) throw new Error("generation gateway is closed");
|
|
145
|
+
const frozenShape = Object.freeze(shape);
|
|
146
|
+
const execution = this.#binding.plan(frozenShape, options,
|
|
147
|
+
{ checkpoints: this.opts.cache?.checkpointEveryTokens !== undefined });
|
|
148
|
+
if (execution.mechanism !== "continuous")
|
|
149
|
+
throw new UnsupportedExecutionError(this.#binding.config.modelType, execution.method, execution.reasons);
|
|
150
|
+
return Object.freeze({ shape: frozenShape, mechanism: "continuous", execution });
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
/** Shared by inference, preparation and managed GPU subprocesses. */
|
|
154
|
+
async acquireExecutionLease(
|
|
155
|
+
signal?: AbortSignal,
|
|
156
|
+
trace?: PromptResponseTrace,
|
|
157
|
+
): Promise<DisposableResource> {
|
|
158
|
+
if (this.#closed) throw new Error("generation gateway is closed");
|
|
159
|
+
this.#exclusiveWaiters++;
|
|
160
|
+
const closeAdmission = trace?.begin("engine.admission_wait", { mechanism: "exclusive" });
|
|
161
|
+
const resume = () => {
|
|
162
|
+
this.#exclusiveWaiters--;
|
|
163
|
+
if (this.#exclusiveWaiters === 0) this.#scheduler?.kick();
|
|
164
|
+
};
|
|
165
|
+
try {
|
|
166
|
+
const release = await this.#mutex.acquire(signal);
|
|
167
|
+
let released = false;
|
|
168
|
+
return { dispose: () => {
|
|
169
|
+
if (released) return;
|
|
170
|
+
released = true;
|
|
171
|
+
release(); resume();
|
|
172
|
+
} };
|
|
173
|
+
} catch (error) { resume(); throw error; }
|
|
174
|
+
finally { closeAdmission?.(); }
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
async runExclusive<T>(
|
|
178
|
+
fn: () => Promise<T>,
|
|
179
|
+
trace?: PromptResponseTrace,
|
|
180
|
+
signal?: AbortSignal,
|
|
181
|
+
): Promise<T> {
|
|
182
|
+
const lease = await this.acquireExecutionLease(signal, trace);
|
|
183
|
+
try { return await fn(); } finally { lease.dispose(); }
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
/** Preparation reads model weights without replacing active model state.
|
|
187
|
+
* A compatible backend can perform it between existing decode iterations. */
|
|
188
|
+
runPreparation<T>(work: () => Promise<T>, signal?: AbortSignal): Promise<T> {
|
|
189
|
+
if (this.#closed) return Promise.reject(new Error("generation gateway is closed"));
|
|
190
|
+
if (signal?.aborted) return Promise.reject(signal.reason);
|
|
191
|
+
const group = this.#ensureScheduler();
|
|
192
|
+
return group?.runPreparation
|
|
193
|
+
? group.runPreparation(work, signal)
|
|
194
|
+
: this.runExclusive(work, undefined, signal);
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
get mediaBatchingEnabled(): boolean { return !!this.#binding.mediaInput; }
|
|
198
|
+
|
|
199
|
+
/** Run one generation through the shared scheduler. onToken is invoked per emitted
|
|
200
|
+
* token (its `false` halts); resolves with stats when the generation ends. */
|
|
201
|
+
async run(
|
|
202
|
+
promptIds: number[],
|
|
203
|
+
options: GenerateOptions & { stopSequences?: string[] },
|
|
204
|
+
onToken: OnToken,
|
|
205
|
+
vision: Vision | undefined,
|
|
206
|
+
shape: RequestShape,
|
|
207
|
+
placement: GenerationPlacement,
|
|
208
|
+
signal?: AbortSignal,
|
|
209
|
+
trace?: PromptResponseTrace,
|
|
210
|
+
): Promise<GenerateStats> {
|
|
211
|
+
let reservation;
|
|
212
|
+
const closeRequestWait = trace?.begin("engine.request_wait", { capacity: this.#batch });
|
|
213
|
+
try { reservation = await acquireReservation(this.#requests, signal); }
|
|
214
|
+
catch (error) { cleanupFailure(error, () => disposeUnstartedRequest(options, vision)); }
|
|
215
|
+
finally { closeRequestWait?.(); }
|
|
216
|
+
let releasePrefix: (() => void) | undefined;
|
|
217
|
+
try {
|
|
218
|
+
try {
|
|
219
|
+
const adapters = options.adapters?.length
|
|
220
|
+
? this.opts.adapterNamespace?.(options.adapters) ?? JSON.stringify(options.adapters) : "";
|
|
221
|
+
// A prepared media prefix is identified within the request's own namespace, which the
|
|
222
|
+
// binding places in its method's (the rows' cache namespace, below); the cache service adds its paging.
|
|
223
|
+
const base = vision?.prefixIdentity ? JSON.stringify(["prepared-prefix-v1", vision.prefixIdentity, adapters]) : adapters;
|
|
224
|
+
const namespace = this.#binding.prefixNamespace?.(placement.execution, options, base) ??
|
|
225
|
+
(placement.execution?.method === "speculative" ? null : base);
|
|
226
|
+
if ((!vision || placement.execution?.promptCache) && namespace !== null && this.opts.cache) {
|
|
227
|
+
const closePrefetch = trace?.begin("cache.prefetch");
|
|
228
|
+
try { releasePrefix = await this.opts.cache.prefetch(promptIds, namespace, options.cacheSessionId); }
|
|
229
|
+
finally { closePrefetch?.(); }
|
|
230
|
+
}
|
|
231
|
+
} catch (error) { cleanupFailure(error, () => disposeReservedUnstarted(options)); }
|
|
232
|
+
return await this.#run(promptIds, options, onToken, vision, shape, placement, signal, trace);
|
|
233
|
+
} finally {
|
|
234
|
+
// The request transferred its native inputs to this run; this is their
|
|
235
|
+
// one release after reservation. A denoising row borrows its pixels for
|
|
236
|
+
// its first unit and closes its run before the submission settles, so
|
|
237
|
+
// they are released here, as main's one-request run did.
|
|
238
|
+
disposeResources([{ dispose: () => releasePrefix?.() }, reservation,
|
|
239
|
+
...[vision?.embeddings, vision?.imageMask, vision?.multimodalMask, options.visionPixels]
|
|
240
|
+
.filter(value => value != null)]);
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
async #run(
|
|
245
|
+
promptIds: number[],
|
|
246
|
+
options: GenerateOptions & { stopSequences?: string[] },
|
|
247
|
+
onToken: OnToken,
|
|
248
|
+
vision: Vision | undefined,
|
|
249
|
+
shape: RequestShape,
|
|
250
|
+
placement: GenerationPlacement,
|
|
251
|
+
signal?: AbortSignal,
|
|
252
|
+
trace?: PromptResponseTrace,
|
|
253
|
+
): Promise<GenerateStats> {
|
|
254
|
+
const disposeUnstarted = () => disposeReservedUnstarted(options);
|
|
255
|
+
if (placement.shape !== shape) {
|
|
256
|
+
disposeUnstarted();
|
|
257
|
+
throw new Error("generation placement does not belong to this request shape");
|
|
258
|
+
}
|
|
259
|
+
try {
|
|
260
|
+
signal?.throwIfAborted();
|
|
261
|
+
} catch (e) {
|
|
262
|
+
// The gateway is the first component that accepts ownership of a
|
|
263
|
+
// compiled controller. An aborted request never reaches the scheduler.
|
|
264
|
+
disposeUnstarted();
|
|
265
|
+
throw e;
|
|
266
|
+
}
|
|
267
|
+
let sampling: ReturnType<typeof createRowSampling> | undefined;
|
|
268
|
+
let closeAdmission: (() => void) | undefined;
|
|
269
|
+
let st;
|
|
270
|
+
try {
|
|
271
|
+
const adapters = options.adapters?.length ? [...options.adapters] : undefined;
|
|
272
|
+
const adapterKey = adapters ? JSON.stringify(adapters) : "";
|
|
273
|
+
const adapterNamespace = () => adapters ? this.opts.adapterNamespace?.(adapters) ?? adapterKey : "";
|
|
274
|
+
const cacheNamespace = vision?.prefixIdentity
|
|
275
|
+
? () => JSON.stringify(["prepared-prefix-v1", vision.prefixIdentity, adapterNamespace()])
|
|
276
|
+
: adapters ? adapterNamespace : "";
|
|
277
|
+
const context = adapters
|
|
278
|
+
? this.#binding.bindAdapterContext?.(adapters, `adapters:${adapterKey}`) : undefined;
|
|
279
|
+
|
|
280
|
+
// The scheduler owns grammar ready/accept sequencing. StepSampler owns the
|
|
281
|
+
// shared processors -> mask -> logprobs -> sample -> history contract.
|
|
282
|
+
const method = this.#binding.methodRequest?.(placement.execution, options);
|
|
283
|
+
const continuation = method ? undefined : this.#binding.continuationRequest?.(placement.execution, options, promptIds, onToken,
|
|
284
|
+
this.opts.cache?.checkpointEveryTokens);
|
|
285
|
+
if (!method && !continuation) {
|
|
286
|
+
const { createRowSampling } = await import("@mlx-bun/inference/execution");
|
|
287
|
+
const samplerFactory = this.opts.samplerFactory ?? (await import("@mlx-bun/inference/sampling")).makeStepSampler;
|
|
288
|
+
sampling = createRowSampling(samplerFactory(options, {
|
|
289
|
+
tokenRepresentation: "device", grammarWait: "external",
|
|
290
|
+
historyUpdate: "after-sample", initialHistory: promptIds,
|
|
291
|
+
captureSelectedLogprob: options.logprobs === true,
|
|
292
|
+
captureTopLogprobs: options.topLogprobs,
|
|
293
|
+
}), onToken);
|
|
294
|
+
} else { sampling = continuation; }
|
|
295
|
+
|
|
296
|
+
signal?.throwIfAborted();
|
|
297
|
+
this.#rowsSubmitted++;
|
|
298
|
+
closeAdmission = trace?.begin("engine.admission_wait", {
|
|
299
|
+
mechanism: "continuous",
|
|
300
|
+
});
|
|
301
|
+
st = await this.#ensureScheduler().submit({
|
|
302
|
+
promptIds, context, cacheNamespace, cacheSessionId: options.cacheSessionId, prefillChunkSize: options.prefillChunkSize,
|
|
303
|
+
...(vision ? { promptInput: this.#binding.mediaInput!(vision) } : {}),
|
|
304
|
+
continuation: continuation?.continuation,
|
|
305
|
+
reusesPrefix: this.#binding.reusesPrefix(placement.execution),
|
|
306
|
+
compiledDecode: placement.execution?.compiledDecode,
|
|
307
|
+
maxTokens: options.maxTokens ?? 512,
|
|
308
|
+
eosTokenIds: options.eosTokenIds ?? this.#binding.config.eosTokenIds,
|
|
309
|
+
...(method ? { method, onToken } : { sample: sampling!.sample,
|
|
310
|
+
plainGreedy: sampling!.plainGreedy, onToken: sampling!.onToken }),
|
|
311
|
+
onAdmitted: closeAdmission,
|
|
312
|
+
trace,
|
|
313
|
+
...(signal ? { signal } : {}),
|
|
314
|
+
// B1: pass the per-row grammar controller through. The scheduler drives
|
|
315
|
+
// accept/ready/terminate; this gateway OWNS disposal (finally below)
|
|
316
|
+
// across resolve, reject, eviction, and the whole-batch-drop error path.
|
|
317
|
+
...(options.grammar ? { grammar: options.grammar } : {}),
|
|
318
|
+
// Capture reusable prefixes at the established stable boundary.
|
|
319
|
+
...(options.snapshotAt !== undefined ? { snapshotAt: options.snapshotAt } : {}),
|
|
320
|
+
});
|
|
321
|
+
} finally {
|
|
322
|
+
// Attempt all request-owned releases even if a destructor fails.
|
|
323
|
+
disposeResources([{ dispose: () => closeAdmission?.() }, sampling, options.grammar]
|
|
324
|
+
.filter((resource) => resource != null));
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
// Autoregressive decode time starts at the first token. Denoising computes
|
|
328
|
+
// every published token inside its decode span, as main's one-request method did.
|
|
329
|
+
const decodedTokens = placement.execution?.method === "denoising" ? st.generatedTokens : st.generatedTokens - 1;
|
|
330
|
+
return {
|
|
331
|
+
promptTokens: st.promptTokens,
|
|
332
|
+
cachedTokens: st.cachedTokens,
|
|
333
|
+
generatedTokens: st.generatedTokens,
|
|
334
|
+
finishReason: st.finishReason,
|
|
335
|
+
prefillMs: st.prefillMs,
|
|
336
|
+
decodeMs: st.decodeMs,
|
|
337
|
+
prefillTps: st.prefillMs > 0 ? ((st.promptTokens - st.cachedTokens) / st.prefillMs) * 1000 : 0,
|
|
338
|
+
decodeTps: st.decodeMs > 0 && decodedTokens > 0 ? (decodedTokens / st.decodeMs) * 1000 : 0,
|
|
339
|
+
cacheTokens: [],
|
|
340
|
+
...(st.spec ? { spec: st.spec } : {}),
|
|
341
|
+
...(st.fill ? { fill: st.fill } : {}),
|
|
342
|
+
};
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
#ensureScheduler(): MlxBatchGroup {
|
|
346
|
+
if (!this.#scheduler)
|
|
347
|
+
this.#scheduler = this.#binding.createBatchGroup({
|
|
348
|
+
runtime: this.#runtime,
|
|
349
|
+
maxBatch: this.#batch,
|
|
350
|
+
...(this.opts.cache ? { cache: this.opts.cache } : {}),
|
|
351
|
+
lock: { acquire: () => this.#mutex.acquire() },
|
|
352
|
+
|
|
353
|
+
// Exclusive mutations drain current rows before acquiring the model.
|
|
354
|
+
admissionHeld: () => this.#exclusiveWaiters > 0,
|
|
355
|
+
});
|
|
356
|
+
return this.#scheduler;
|
|
357
|
+
}
|
|
358
|
+
}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import { GenerationGateway, disposeUnstartedRequest } from "./generation-gateway";
|
|
2
|
+
import { createSessionCompletionEngine } from "./session-completion-engine";
|
|
3
|
+
import { createPreparationExecutor } from "./preparation";
|
|
4
|
+
import { modelServingBinding } from "./model-serving";
|
|
5
|
+
import { releaseContext, type ContextOwnership, type LoadedModelContext } from "./model-host";
|
|
6
|
+
import type { ModelBinding } from "./model-binding";
|
|
7
|
+
|
|
8
|
+
export * from "./model-host";
|
|
9
|
+
export * from "./model-binding";
|
|
10
|
+
export * from "./model-serving";
|
|
11
|
+
export * from "./generation-gateway";
|
|
12
|
+
export * from "./completion";
|
|
13
|
+
export * from "./preparation";
|
|
14
|
+
export * from "./session-completion-engine";
|
|
15
|
+
export * from "./cache-services";
|
|
16
|
+
|
|
17
|
+
/** Takes ownership of beforeModelDispose and, unless `ownership` is "borrowed",
|
|
18
|
+
* of context. Close cancels sessions, drains work, then runs the hook and
|
|
19
|
+
* releases the context by that rule, once. A supplied binding keeps replacement
|
|
20
|
+
* graphs independent of the built-in model classes. No server is started here. */
|
|
21
|
+
export async function createAppEngine(context: LoadedModelContext, options: {
|
|
22
|
+
capacity: number; binding?: ModelBinding;
|
|
23
|
+
gateway?: ConstructorParameters<typeof GenerationGateway>[2];
|
|
24
|
+
/** Owned cleanup, invoked once after execution drains and before model release,
|
|
25
|
+
* including construction failure. Use for cache flush and cache disposal. */
|
|
26
|
+
beforeModelDispose?: () => void | Promise<unknown>;
|
|
27
|
+
/** Default "owned": close and a failed construction dispose the context. */
|
|
28
|
+
ownership?: ContextOwnership;
|
|
29
|
+
}) {
|
|
30
|
+
let createdGateway: GenerationGateway | undefined;
|
|
31
|
+
const release = () => releaseContext(context, options.ownership ?? "owned");
|
|
32
|
+
try {
|
|
33
|
+
const binding = await modelServingBinding(context, options.binding);
|
|
34
|
+
const gateway = createdGateway = new GenerationGateway(binding.gateway, options.capacity, options.gateway);
|
|
35
|
+
const completion = createSessionCompletionEngine(gateway, disposeUnstartedRequest);
|
|
36
|
+
const preparation = createPreparationExecutor((work, signal) => gateway.runPreparation(work, signal),
|
|
37
|
+
options.capacity, gateway.mediaBatchingEnabled ? options.capacity : 1);
|
|
38
|
+
let closing: Promise<void> | undefined;
|
|
39
|
+
return {
|
|
40
|
+
context, binding, gateway, completion, preparation,
|
|
41
|
+
close() {
|
|
42
|
+
return closing ??= (async () => {
|
|
43
|
+
await releaseAll([() => preparation.close(), () => completion.close(),
|
|
44
|
+
() => gateway.close(), () => options.beforeModelDispose?.(), release]);
|
|
45
|
+
})();
|
|
46
|
+
},
|
|
47
|
+
};
|
|
48
|
+
} catch (error) {
|
|
49
|
+
await releaseAll([() => createdGateway?.close(), () => options.beforeModelDispose?.(), release], [error]);
|
|
50
|
+
throw error;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/** Attempt owned asynchronous releases in order, preserving every failure. */
|
|
55
|
+
async function releaseAll(releases: readonly (() => void | Promise<unknown>)[], errors: unknown[] = []): Promise<void> {
|
|
56
|
+
for (const release of releases) { try { await release(); } catch (error) { errors.push(error); } }
|
|
57
|
+
if (errors.length === 1) throw errors[0];
|
|
58
|
+
if (errors.length) throw new AggregateError(errors, "app engine cleanup failed");
|
|
59
|
+
}
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
// One byte budget over the saved-state root (`MLX_BUN_HOME/kv`). Every model's
|
|
2
|
+
// store lives in its own directory `<root>/<fingerprint>/`; a store enforces
|
|
3
|
+
// only its own directory, so the budget lends each live store the room the
|
|
4
|
+
// others leave and evicts the oldest files of directories no live store owns.
|
|
5
|
+
// Saved state is always droppable: an evicted file is a prefix miss, never an
|
|
6
|
+
// error. A live store never loses a file it indexed; it shrinks itself.
|
|
7
|
+
import { existsSync, readdirSync, rmdirSync, rmSync, statSync } from "node:fs";
|
|
8
|
+
import { join } from "node:path";
|
|
9
|
+
|
|
10
|
+
/** What the budget needs from one live store. */
|
|
11
|
+
export interface KvBudgetMember {
|
|
12
|
+
/** The store's own directory under the root. */
|
|
13
|
+
readonly dir: string;
|
|
14
|
+
/** Bytes the store holds now. */
|
|
15
|
+
bytes(): number;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export interface KvBudgetUsage {
|
|
19
|
+
/** Every byte under the root. */
|
|
20
|
+
readonly bytes: number;
|
|
21
|
+
/** Held by directories no live store owns. */
|
|
22
|
+
readonly idleBytes: number;
|
|
23
|
+
readonly maxBytes: number;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export interface KvBudget {
|
|
27
|
+
readonly root: string;
|
|
28
|
+
/** Infinity: unlimited. */
|
|
29
|
+
readonly maxBytes: number;
|
|
30
|
+
/** Count a live store against the budget; `limit()` is the cap it enforces on itself now. */
|
|
31
|
+
attach(member: KvBudgetMember): { limit(): number; detach(): void };
|
|
32
|
+
/** Evict the oldest files of idle directories until the root fits. Live stores are never touched. */
|
|
33
|
+
reclaim(): { removedFiles: number; removedBytes: number };
|
|
34
|
+
usage(): KvBudgetUsage;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
interface FileEntry { path: string; bytes: number; mtimeMs: number }
|
|
38
|
+
|
|
39
|
+
function walk(dir: string, into: FileEntry[]): void {
|
|
40
|
+
let entries;
|
|
41
|
+
try { entries = readdirSync(dir, { withFileTypes: true }); } catch { return; }
|
|
42
|
+
for (const entry of entries) {
|
|
43
|
+
const path = join(dir, entry.name);
|
|
44
|
+
if (entry.isDirectory()) { walk(path, into); continue; }
|
|
45
|
+
try { const stat = statSync(path); into.push({ path, bytes: stat.size, mtimeMs: stat.mtimeMs }); } catch { /* raced with a delete */ }
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** A live store keeps at least this share of the budget however full the others are. */
|
|
50
|
+
const LIVE_FLOOR = 0.25;
|
|
51
|
+
|
|
52
|
+
export function createKvBudget(root: string, maxBytes: number = Infinity): KvBudget {
|
|
53
|
+
if (!(maxBytes > 0)) throw new Error("a saved-state budget must be positive; use Infinity for none");
|
|
54
|
+
const members = new Set<KvBudgetMember>();
|
|
55
|
+
let idleBytes = 0, scanned = false;
|
|
56
|
+
const isLive = (dir: string) => [...members].some(member => member.dir === dir);
|
|
57
|
+
/** Files of every top-level directory that no live store owns. */
|
|
58
|
+
const idleFiles = (): FileEntry[] => {
|
|
59
|
+
const files: FileEntry[] = [];
|
|
60
|
+
let names: string[] = [];
|
|
61
|
+
try { names = readdirSync(root); } catch { /* no root yet: nothing saved */ }
|
|
62
|
+
for (const name of names) if (!isLive(join(root, name))) walk(join(root, name), files);
|
|
63
|
+
return files;
|
|
64
|
+
};
|
|
65
|
+
const refresh = (files = idleFiles()) => { idleBytes = files.reduce((sum, file) => sum + file.bytes, 0); scanned = true; };
|
|
66
|
+
const liveBytes = (except?: KvBudgetMember) => [...members].reduce((sum, member) => member === except ? sum : sum + member.bytes(), 0);
|
|
67
|
+
return {
|
|
68
|
+
root, maxBytes,
|
|
69
|
+
attach(member) {
|
|
70
|
+
members.add(member);
|
|
71
|
+
scanned = false;
|
|
72
|
+
return {
|
|
73
|
+
limit: () => {
|
|
74
|
+
if (!Number.isFinite(maxBytes)) return Infinity;
|
|
75
|
+
if (!scanned) refresh();
|
|
76
|
+
return Math.max(maxBytes * LIVE_FLOOR, maxBytes - idleBytes - liveBytes(member));
|
|
77
|
+
},
|
|
78
|
+
detach: () => { members.delete(member); scanned = false; },
|
|
79
|
+
};
|
|
80
|
+
},
|
|
81
|
+
reclaim() {
|
|
82
|
+
const files = idleFiles();
|
|
83
|
+
refresh(files);
|
|
84
|
+
let removedFiles = 0, removedBytes = 0;
|
|
85
|
+
if (Number.isFinite(maxBytes)) {
|
|
86
|
+
files.sort((a, b) => a.mtimeMs - b.mtimeMs);
|
|
87
|
+
for (const file of files) {
|
|
88
|
+
if (idleBytes + liveBytes() <= maxBytes) break;
|
|
89
|
+
try { rmSync(file.path, { force: true }); } catch { continue; }
|
|
90
|
+
idleBytes -= file.bytes; removedFiles++; removedBytes += file.bytes;
|
|
91
|
+
}
|
|
92
|
+
if (removedFiles) pruneEmpty(root, members);
|
|
93
|
+
}
|
|
94
|
+
return { removedFiles, removedBytes };
|
|
95
|
+
},
|
|
96
|
+
usage() {
|
|
97
|
+
refresh();
|
|
98
|
+
return { bytes: idleBytes + liveBytes(), idleBytes, maxBytes };
|
|
99
|
+
},
|
|
100
|
+
};
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/** Remove directories the eviction emptied, never a live store's own. */
|
|
104
|
+
function pruneEmpty(root: string, members: ReadonlySet<KvBudgetMember>): void {
|
|
105
|
+
const live = new Set([...members].map(member => member.dir));
|
|
106
|
+
const prune = (dir: string): boolean => {
|
|
107
|
+
let entries;
|
|
108
|
+
try { entries = readdirSync(dir, { withFileTypes: true }); } catch { return false; }
|
|
109
|
+
let empty = true;
|
|
110
|
+
for (const entry of entries) {
|
|
111
|
+
if (entry.isDirectory() && prune(join(dir, entry.name))) continue;
|
|
112
|
+
empty = false;
|
|
113
|
+
}
|
|
114
|
+
if (!empty || live.has(dir)) return false;
|
|
115
|
+
try { rmdirSync(dir); return true; } catch { return false; }
|
|
116
|
+
};
|
|
117
|
+
if (!existsSync(root)) return;
|
|
118
|
+
for (const entry of readdirSync(root, { withFileTypes: true }))
|
|
119
|
+
if (entry.isDirectory() && !live.has(join(root, entry.name))) prune(join(root, entry.name));
|
|
120
|
+
}
|