mlx-bun 0.4.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1453 -88
- package/THIRD_PARTY_NOTICES.md +882 -0
- package/bin/mlx-bun.convert.mjs +4 -0
- package/bin/mlx-bun.fuse.mjs +4 -0
- package/bin/mlx-bun.generate.mjs +4 -0
- package/bin/mlx-bun.lora.mjs +4 -0
- package/bin/mlx-bun.mjs +13 -21
- package/bin/mlx-bun.server.mjs +4 -0
- package/bin/mlx-bun.upload.mjs +4 -0
- package/dist/web/app.js +9584 -0
- package/package.json +50 -53
- package/src/cli/ane-install.ts +76 -0
- package/src/cli/args.ts +179 -0
- package/src/cli/engine-entry.ts +68 -0
- package/src/cli/inference.ts +214 -0
- package/src/cli/job-entry.ts +97 -0
- package/src/cli/library-host.ts +117 -0
- package/src/cli/main.ts +57 -0
- package/src/cli/memory-engine.ts +163 -0
- package/src/cli/memory.ts +36 -0
- package/src/cli/mlx-lm-aliases.ts +309 -0
- package/src/cli/model-choice.ts +33 -0
- package/src/cli/model-selection.ts +84 -0
- package/src/cli/module-host.ts +11 -0
- package/src/cli/module-verbs.ts +50 -0
- package/src/cli/numerical-policy.ts +12 -0
- package/src/cli/perplexity.ts +89 -0
- package/src/cli/serve-host.ts +423 -0
- package/src/cli/serve-isolated.ts +217 -0
- package/src/cli/serve-options.ts +106 -0
- package/src/cli/serve-state.ts +205 -0
- package/src/cli/serve.ts +500 -0
- package/src/cli/served-model-host.ts +102 -0
- package/src/cli/server-entry.ts +148 -0
- package/src/cli/serving-unit.ts +177 -0
- package/src/{tui.ts → cli/terminal.ts} +13 -8
- package/src/cli/worker-adapters.ts +34 -0
- package/src/cli/worker-entry.ts +272 -0
- package/src/cli/worker-unit.ts +179 -0
- package/src/engine/adapter-operation.ts +26 -0
- package/src/engine/cache-services.ts +114 -0
- package/src/engine/completion.ts +39 -0
- package/src/engine/generation-gateway.ts +358 -0
- package/src/engine/index.ts +59 -0
- package/src/engine/kv-budget.ts +120 -0
- package/src/engine/media-preparation.ts +232 -0
- package/src/engine/model-binding.ts +24 -0
- package/src/engine/model-host.ts +513 -0
- package/src/engine/model-serving.ts +89 -0
- package/src/{serve → engine}/preparation.ts +10 -8
- package/src/{serve → engine}/session-completion-engine.ts +6 -6
- package/src/engine/telemetry.ts +109 -0
- package/src/hub/downloads.ts +116 -0
- package/src/jobs/db.ts +16 -7
- package/src/jobs/events.ts +29 -0
- package/src/jobs/executable.ts +5 -0
- package/src/jobs/host.ts +51 -0
- package/src/jobs/in-process.ts +55 -0
- package/src/{contracts/jobs.ts → jobs/protocol.ts} +7 -9
- package/src/jobs/runner.ts +150 -231
- package/src/jobs/service.ts +75 -0
- package/src/jobs/sse.ts +59 -23
- package/src/jobs/worker-memory.ts +46 -0
- package/src/jobs/worker-process.ts +163 -0
- package/src/jobs/worker-supervisor.ts +258 -0
- package/src/modules.ts +58 -0
- package/src/publishing/catalog-hub.ts +18 -0
- package/src/publishing/credentials.ts +31 -0
- package/src/publishing/upload.ts +37 -0
- package/src/residency/leased-adapters.ts +21 -0
- package/src/residency/local-records.ts +16 -0
- package/src/residency/model-residency.ts +405 -0
- package/src/residency/record-index.ts +28 -0
- package/src/residency/resident-estimate.ts +50 -0
- package/src/{anthropic.ts → server/anthropic.ts} +3 -81
- package/src/server/cache-routes.ts +36 -0
- package/src/{serve → server}/chat-request.ts +12 -10
- package/src/{serve → server}/chat-stage.ts +27 -22
- package/src/{client.ts → server/client.ts} +28 -2
- package/src/{serve → server}/completion-executor.ts +42 -42
- package/src/server/decision-body.ts +55 -0
- package/src/server/decision-routes.ts +134 -0
- package/src/server/decision-wire.ts +228 -0
- package/src/{serve → server}/discovery-routes.ts +42 -83
- package/src/server/generated-token-history.ts +57 -0
- package/src/{serve → server}/http.ts +25 -13
- package/src/{serve → server}/inference-request.ts +5 -5
- package/src/server/job-routes.ts +20 -0
- package/src/server/media-prompt.ts +63 -0
- package/src/server/memory-completion-client.ts +46 -0
- package/src/server/model-routes.ts +192 -0
- package/src/{serve → server}/openai-wire.ts +3 -2
- package/src/{serve → server}/pipeline.ts +1 -1
- package/src/server/prompt-builder.ts +13 -0
- package/src/server/prompt-contracts.ts +17 -0
- package/src/server/proxy-routes.ts +187 -0
- package/src/server/publishing-routes.ts +41 -0
- package/src/{serve → server}/request-plan.ts +8 -8
- package/src/{serve → server}/request-prep.ts +82 -68
- package/src/{serve → server}/responses-client.ts +30 -13
- package/src/{responses.ts → server/responses.ts} +12 -93
- package/src/server/routes.ts +205 -0
- package/src/server/start.ts +72 -0
- package/src/server/status-routes.ts +108 -0
- package/src/{serve → server}/text-completion-stage.ts +25 -4
- package/src/server/text-prompt.ts +12 -0
- package/src/{serve → server}/token-streams.ts +36 -38
- package/src/server/worker-routes.ts +311 -0
- package/src/storage/paths.ts +61 -0
- package/src/web/assets.ts +37 -0
- package/src/web/{src → browser}/api.ts +5 -23
- package/src/web/browser/chat-host.ts +17 -0
- package/src/web/browser/installed-panels.ts +6 -0
- package/src/web/browser/main.ts +55 -0
- package/src/web/browser/markdown.ts +279 -0
- package/src/web/browser/memory-panel.ts +9 -0
- package/src/web/browser/palette.ts +64 -0
- package/src/web/browser/protocol.ts +13 -0
- package/src/web/browser/shell.ts +467 -0
- package/src/web/{src → browser}/status.ts +32 -36
- package/src/web/build.ts +51 -0
- package/src/web/public/app.html +1054 -0
- package/src/web/{sw.js → public/sw.js} +4 -3
- package/src/web/{vendor → public/vendor}/hljs-theme.css +2 -2
- package/src/web/{vendor → public/vendor}/hljs.js +2 -2
- package/THIRD_PARTY_LICENSES.md +0 -51
- package/docs/README.md +0 -94
- package/docs/archive/hlg-sampling.md +0 -350
- package/docs/archive/investigations/audio-input-plan.md +0 -356
- package/docs/archive/investigations/chunk-finetune-experiment.md +0 -84
- package/docs/archive/investigations/colibri-glm52-port.md +0 -1486
- package/docs/archive/investigations/colibri-oracle-pin.md +0 -1284
- package/docs/archive/investigations/curve-bisector-routing.md +0 -123
- package/docs/archive/investigations/decode-roofline-lookagain.md +0 -348
- package/docs/archive/investigations/diffusion-gemma-port.md +0 -602
- package/docs/archive/investigations/dspark-handoff.md +0 -224
- package/docs/archive/investigations/expert-offload-single-user-moe.md +0 -292
- package/docs/archive/investigations/faithful-l1-consolidation.md +0 -310
- package/docs/archive/investigations/hlg-sampling-investigation.md +0 -970
- package/docs/archive/investigations/indexshare-performance-spike.md +0 -357
- package/docs/archive/investigations/kernel-perf-review-2026-07.md +0 -171
- package/docs/archive/investigations/lab-build-journal.md +0 -157
- package/docs/archive/investigations/minicpm5-decode-megakernel.md +0 -489
- package/docs/archive/investigations/openjarvis-fit.md +0 -198
- package/docs/archive/investigations/openwiki-evaluation.md +0 -249
- package/docs/archive/investigations/optimization_plan.md +0 -346
- package/docs/archive/investigations/orpo-base-uf-experiment-and-directions.md +0 -120
- package/docs/archive/investigations/orpo-flash-cce-pin-leak.md +0 -167
- package/docs/archive/investigations/orpo-uf-testing-handoff.md +0 -126
- package/docs/archive/investigations/pareto-specialized-runtime-findings.md +0 -92
- package/docs/archive/investigations/pi-builtin-investigation.md +0 -308
- package/docs/archive/investigations/pre-colibri-stabilization.md +0 -329
- package/docs/archive/investigations/qwen38-27b-campaign-log.md +0 -1092
- package/docs/archive/investigations/segmented-backward-handoff.md +0 -148
- package/docs/archive/investigations/starter-model-port-handoff.md +0 -342
- package/docs/archive/investigations/steel-flash-cce-handoff.md +0 -489
- package/docs/archive/investigations/the-dreaming-handoff.md +0 -141
- package/docs/archive/investigations/trainer-validation-experiment.md +0 -173
- package/docs/archive/planning/chat-ui-vision-memory-first.md +0 -199
- package/docs/archive/planning/chat-ui-vision.md +0 -50
- package/docs/archive/planning/chunk-task-roadmap.md +0 -77
- package/docs/archive/planning/curve-sampler-research-plan.md +0 -331
- package/docs/archive/planning/journal.md +0 -316
- package/docs/archive/planning/memory-docs-and-dag-plan.md +0 -194
- package/docs/archive/planning/pi-integration-review.md +0 -178
- package/docs/archive/planning/release-notes-v0.0.10.md +0 -57
- package/docs/archive/planning/release-notes-v0.0.11.md +0 -239
- package/docs/archive/planning/release-notes-v0.0.12.md +0 -80
- package/docs/archive/planning/release-notes-v0.0.13.md +0 -77
- package/docs/archive/planning/release-notes-v0.0.9.md +0 -122
- package/docs/archive/planning/release-notes-v0.2.0.md +0 -49
- package/docs/archive/planning/release-notes-v0.3.0.md +0 -35
- package/docs/archive/planning/web-ui-pass-plan.md +0 -213
- package/docs/archive/planning/website-readme-pass-plan.md +0 -224
- package/docs/design/batching.md +0 -1841
- package/docs/design/decode-speed-program.md +0 -2349
- package/docs/design/dreaming-nightly-pipeline.md +0 -684
- package/docs/design/generic-model-support.md +0 -705
- package/docs/design/kv-cache.md +0 -824
- package/docs/design/orpo-training.md +0 -680
- package/docs/design/speculative-decoding.md +0 -1990
- package/docs/design/turboquant.md +0 -2212
- package/docs/design/unified-engine-frontier-plan.md +0 -1880
- package/docs/design/web-chat-redesign.md +0 -695
- package/docs/planning/PRODUCT_ROADMAP.md +0 -594
- package/docs/planning/ResearchTopics.md +0 -119
- package/docs/planning/release-notes-v0.4.0.md +0 -45
- package/docs/planning/video-series.md +0 -421
- package/docs/reference/benchmarks.md +0 -3613
- package/docs/reference/cli.md +0 -585
- package/docs/reference/distribution.md +0 -355
- package/docs/reference/environment.md +0 -398
- package/docs/reference/glossary.md +0 -39
- package/docs/reference/library-api.md +0 -674
- package/docs/reference/memory.md +0 -260
- package/docs/reference/models.md +0 -334
- package/docs/reference/server-api.md +0 -1775
- package/docs/reference/server-config.md +0 -846
- package/docs/reference/training.md +0 -462
- package/docs/reference/troubleshooting.md +0 -158
- package/src/admission-plan.ts +0 -18
- package/src/audio/conformer.ts +0 -647
- package/src/audio/decode.ts +0 -133
- package/src/audio/features.ts +0 -202
- package/src/audio/transcode.ts +0 -67
- package/src/backends/mlx/assistant-target.ts +0 -46
- package/src/backends/mlx/autoregressive-draft-rows.ts +0 -120
- package/src/backends/mlx/autoregressive.ts +0 -108
- package/src/backends/mlx/batch-group.ts +0 -1790
- package/src/backends/mlx/cache-attachment.ts +0 -67
- package/src/backends/mlx/cache-layout.ts +0 -31
- package/src/backends/mlx/checkpoint-state.ts +0 -53
- package/src/backends/mlx/continuation-persistence.ts +0 -106
- package/src/backends/mlx/continuation-request.ts +0 -43
- package/src/backends/mlx/continuation.ts +0 -77
- package/src/backends/mlx/diffusion.ts +0 -39
- package/src/backends/mlx/draft-target.ts +0 -32
- package/src/backends/mlx/evaluation.ts +0 -105
- package/src/backends/mlx/fill-append.ts +0 -45
- package/src/backends/mlx/full-kv-row-append.ts +0 -32
- package/src/backends/mlx/full-kv-row-donor.ts +0 -54
- package/src/backends/mlx/full-transitioning-kv-rows.ts +0 -65
- package/src/backends/mlx/gateway-binding.ts +0 -130
- package/src/backends/mlx/graph.ts +0 -38
- package/src/backends/mlx/kv-codec.ts +0 -9
- package/src/backends/mlx/kv-maintenance.ts +0 -133
- package/src/backends/mlx/kv-tensor-rows.ts +0 -212
- package/src/backends/mlx/memory-client.ts +0 -189
- package/src/backends/mlx/methods.ts +0 -90
- package/src/backends/mlx/model-prompt.ts +0 -164
- package/src/backends/mlx/model-serving.ts +0 -79
- package/src/backends/mlx/prefill-cohort.ts +0 -89
- package/src/backends/mlx/prefill-rows.ts +0 -200
- package/src/backends/mlx/prefill.ts +0 -33
- package/src/backends/mlx/projected-context-rows.ts +0 -72
- package/src/backends/mlx/request-policy.ts +0 -20
- package/src/backends/mlx/request-state-policy.ts +0 -41
- package/src/backends/mlx/rollback.ts +0 -89
- package/src/backends/mlx/row-sampling.ts +0 -49
- package/src/backends/mlx/serial-executor.ts +0 -242
- package/src/backends/mlx/speculative-group.ts +0 -359
- package/src/backends/mlx/speculative-round.ts +0 -161
- package/src/backends/mlx/speculative.ts +0 -114
- package/src/backends/mlx/state-rows.ts +0 -104
- package/src/backends/mlx/state-views.ts +0 -46
- package/src/backends/mlx/target-layout-capability.ts +0 -35
- package/src/backends/mlx/text-engine.ts +0 -73
- package/src/backends/mlx/transitioning-kv-rows.ts +0 -165
- package/src/chat-template.ts +0 -382
- package/src/cli.ts +0 -3168
- package/src/config.ts +0 -399
- package/src/contracts/completion.ts +0 -16
- package/src/contracts/execution.ts +0 -60
- package/src/contracts/generation.ts +0 -103
- package/src/contracts/host.ts +0 -7
- package/src/contracts/pi-provider.ts +0 -30
- package/src/contracts/pi.ts +0 -259
- package/src/contracts/prefix-cache.ts +0 -23
- package/src/contracts/resources.ts +0 -25
- package/src/contracts/sampling.ts +0 -23
- package/src/contracts/scheduling.ts +0 -45
- package/src/contracts/tsconfig.json +0 -14
- package/src/dataset/generators.ts +0 -1087
- package/src/dataset/index.ts +0 -12
- package/src/dataset/job.ts +0 -81
- package/src/dataset/llm.ts +0 -71
- package/src/dataset/registry.ts +0 -528
- package/src/diffusion/diffusion-generate.ts +0 -507
- package/src/download.ts +0 -505
- package/src/embed.ts +0 -77
- package/src/engine/admission.ts +0 -59
- package/src/engine/cancellation.ts +0 -38
- package/src/engine/engine.ts +0 -56
- package/src/engine/execution-coordinator.ts +0 -70
- package/src/engine/execution-plan.ts +0 -53
- package/src/engine/resources.ts +0 -53
- package/src/engine/scheduler.ts +0 -72
- package/src/engine/session.ts +0 -247
- package/src/eval/README.md +0 -130
- package/src/eval/ifeval.ts +0 -81
- package/src/eval/kl-prompts.ts +0 -29
- package/src/eval/kl.ts +0 -590
- package/src/eval/perplexity.ts +0 -164
- package/src/eval/quality-db.ts +0 -92
- package/src/eval/reference-logits.ts +0 -207
- package/src/eval/runner.ts +0 -124
- package/src/eval/score.ts +0 -49
- package/src/eval/tasks/bfcl.ts +0 -535
- package/src/eval/tasks/gsm8k.ts +0 -98
- package/src/eval/tasks/hashhop.ts +0 -139
- package/src/eval/tasks/humaneval.ts +0 -275
- package/src/eval/tasks/ifeval.ts +0 -525
- package/src/eval/tasks/mmlu.ts +0 -157
- package/src/eval/ultrafeedback.ts +0 -60
- package/src/evaldb.ts +0 -199
- package/src/expert-io.ts +0 -393
- package/src/expert-offload-build.ts +0 -77
- package/src/expert-offload.ts +0 -76
- package/src/expert-residency.ts +0 -1134
- package/src/expert-usage.ts +0 -415
- package/src/fill/echo-index.ts +0 -188
- package/src/fill/fill-session.ts +0 -452
- package/src/fill/proposal.ts +0 -68
- package/src/fill/schema-rows.ts +0 -506
- package/src/fill/tool-boundary.ts +0 -103
- package/src/fit.ts +0 -249
- package/src/generate.ts +0 -1346
- package/src/grammar.ts +0 -591
- package/src/harness-pi.ts +0 -184
- package/src/hf-push.ts +0 -628
- package/src/hub-rest.ts +0 -242
- package/src/index.ts +0 -63
- package/src/inference/denoising.ts +0 -17
- package/src/inference/draft-acceptance.ts +0 -41
- package/src/inference/draft-output.ts +0 -49
- package/src/inference/graph.ts +0 -33
- package/src/inference/prefill.ts +0 -39
- package/src/inference/rollback.ts +0 -7
- package/src/jobs/index.ts +0 -31
- package/src/jobs/job-entry.ts +0 -107
- package/src/jobs/task-client.ts +0 -19
- package/src/jobs/types.ts +0 -2
- package/src/kv-scheme.ts +0 -257
- package/src/kv-store.ts +0 -1099
- package/src/lab/README.md +0 -18
- package/src/lab/curve/curve-designer.html +0 -303
- package/src/lab/curve/curve-sampler.ts +0 -122
- package/src/lab/expert-trace/expert-trace.ts +0 -69
- package/src/lab/paged-kv/paged-kv-rows.ts +0 -83
- package/src/lab/paged-kv/paged-kv.ts +0 -270
- package/src/library.ts +0 -40
- package/src/lora.ts +0 -408
- package/src/md-modules.d.ts +0 -49
- package/src/media-fetch.ts +0 -442
- package/src/memory/article.ts +0 -558
- package/src/memory/chunk-validation.ts +0 -150
- package/src/memory/chunk.ts +0 -318
- package/src/memory/cluster.ts +0 -351
- package/src/memory/coverage.ts +0 -134
- package/src/memory/crosslink.ts +0 -0
- package/src/memory/db.ts +0 -253
- package/src/memory/entity.ts +0 -203
- package/src/memory/gate.ts +0 -187
- package/src/memory/history.ts +0 -152
- package/src/memory/ingest.ts +0 -335
- package/src/memory/model.ts +0 -183
- package/src/memory/normalize.ts +0 -671
- package/src/memory/parse.ts +0 -22
- package/src/memory/pipeline.ts +0 -275
- package/src/memory/prompts.ts +0 -65
- package/src/memory/query.ts +0 -334
- package/src/memory/reconcile.ts +0 -734
- package/src/memory/redirect.ts +0 -108
- package/src/memory/reindex.ts +0 -325
- package/src/memory/resolve.ts +0 -307
- package/src/memory/rest.ts +0 -245
- package/src/memory/route.ts +0 -272
- package/src/memory/schedule.ts +0 -165
- package/src/memory/stages.ts +0 -777
- package/src/memory/synthesize.ts +0 -1638
- package/src/memory/tools.ts +0 -749
- package/src/memory/vault.ts +0 -1194
- package/src/memory/wikify.ts +0 -1064
- package/src/mlx/array.ts +0 -388
- package/src/mlx/autograd.ts +0 -304
- package/src/mlx/checkpoint.ts +0 -149
- package/src/mlx/compile.ts +0 -168
- package/src/mlx/custom-vjp.ts +0 -188
- package/src/mlx/ffi.ts +0 -512
- package/src/mlx/materialize.ts +0 -23
- package/src/mlx/metal-kernel.ts +0 -159
- package/src/mlx/ops.ts +0 -1021
- package/src/mlx/quantized-rows.ts +0 -36
- package/src/mlx/token-bitmask.ts +0 -52
- package/src/mlx/turboquant-kv-decode.ts +0 -82
- package/src/mlx/turboquant-kv-inverse.ts +0 -122
- package/src/mlx/turboquant-ops.ts +0 -861
- package/src/mlx/turboquant-tables.ts +0 -237
- package/src/mmap.ts +0 -90
- package/src/model/artifact-identity.ts +0 -73
- package/src/model/batched-kv.ts +0 -235
- package/src/model/batched-mask.ts +0 -279
- package/src/model/batched-quant.ts +0 -212
- package/src/model/batched-quantized-kv.ts +0 -144
- package/src/model/batched-rotating-quant.ts +0 -353
- package/src/model/batched-rotating-state.ts +0 -149
- package/src/model/batched-rotating.ts +0 -338
- package/src/model/batched-row-storage.ts +0 -179
- package/src/model/batched-ssm.ts +0 -52
- package/src/model/batched-turboquant-kv.ts +0 -93
- package/src/model/compiled-decode.ts +0 -712
- package/src/model/decoded-kv-donor.ts +0 -12
- package/src/model/delayed-quantized-kv.ts +0 -72
- package/src/model/delayed-rotating-quantized-kv.ts +0 -93
- package/src/model/delayed-turboquant-kv.ts +0 -55
- package/src/model/diffusion-gemma.ts +0 -1026
- package/src/model/factory.ts +0 -189
- package/src/model/fingerprint.ts +0 -33
- package/src/model/flash-attention.ts +0 -694
- package/src/model/full-prefill-padding.ts +0 -56
- package/src/model/full-prefill-row.ts +0 -105
- package/src/model/gemma4-base.ts +0 -2351
- package/src/model/gemma4.ts +0 -1315
- package/src/model/generated/gemma4-12b.ts +0 -429
- package/src/model/generated/gemma4-26b.ts +0 -448
- package/src/model/generated/gemma4-e4b.ts +0 -717
- package/src/model/generated/index.ts +0 -18
- package/src/model/glm52-atlas.ts +0 -511
- package/src/model/glm52-cache.ts +0 -792
- package/src/model/glm52-config.ts +0 -262
- package/src/model/glm52-container.ts +0 -305
- package/src/model/glm52-coupling.ts +0 -388
- package/src/model/glm52-dsa.ts +0 -606
- package/src/model/glm52-expert-layout.ts +0 -253
- package/src/model/glm52-layout.ts +0 -152
- package/src/model/glm52-memory.ts +0 -500
- package/src/model/glm52-mla.ts +0 -792
- package/src/model/glm52-moe.ts +0 -223
- package/src/model/glm52-pilot.ts +0 -537
- package/src/model/glm52-reference.ts +0 -445
- package/src/model/glm52-residency.ts +0 -557
- package/src/model/glm52-resident-weights.ts +0 -135
- package/src/model/glm52-streamed-experts.ts +0 -453
- package/src/model/glm52-streamed-metal.ts +0 -610
- package/src/model/glm52-weights.ts +0 -503
- package/src/model/glm52.ts +0 -1132
- package/src/model/implementation.ts +0 -69
- package/src/model/kv-attention-view.ts +0 -67
- package/src/model/minicpm5.ts +0 -376
- package/src/model/profile.ts +0 -443
- package/src/model/qwen-append.ts +0 -12
- package/src/model/qwen-conv.ts +0 -52
- package/src/model/qwen3-delta-state.ts +0 -74
- package/src/model/qwen3-delta.ts +0 -451
- package/src/model/qwen3-moe.ts +0 -404
- package/src/model/qwen3-mrope.ts +0 -252
- package/src/model/qwen3.ts +0 -277
- package/src/model/qwen3_5-checkpoint.ts +0 -164
- package/src/model/qwen3_5.ts +0 -819
- package/src/model/rotating-kv-donor.ts +0 -35
- package/src/model/rotating-kv-layout.ts +0 -262
- package/src/model/rotating-row-transaction.ts +0 -35
- package/src/model/speculative-rotating-kv.ts +0 -101
- package/src/model/ssm-prefill-padding.ts +0 -39
- package/src/model/support.ts +0 -102
- package/src/model/trellis-balanced-scatter.ts +0 -51
- package/src/model/trellis-linear.ts +0 -672
- package/src/model/trellis-shared-m.ts +0 -151
- package/src/model/trellis-shared-scatter.ts +0 -144
- package/src/model/trellis-splitk-prefill.ts +0 -158
- package/src/model/trellis-tiled-prefill.ts +0 -136
- package/src/model/trellis-vector-expand.ts +0 -62
- package/src/model/trellis-wide-prefill.ts +0 -94
- package/src/model/turboquant-codec.ts +0 -83
- package/src/model/universal/archs.ts +0 -332
- package/src/model/universal/dense.ts +0 -612
- package/src/model/universal/modules.ts +0 -170
- package/src/model/universal/rope.ts +0 -381
- package/src/native/expert_io.c +0 -523
- package/src/native/frame_extract.swift +0 -140
- package/src/native-pack.ts +0 -162
- package/src/pi-provider.ts +0 -53
- package/src/pi-session.ts +0 -81
- package/src/pi-terminal.ts +0 -274
- package/src/pi-web.ts +0 -1932
- package/src/preflight.ts +0 -158
- package/src/prompt-cache.ts +0 -460
- package/src/quantize/allocator.ts +0 -627
- package/src/quantize/atomic-output.ts +0 -38
- package/src/quantize/calibration.ts +0 -181
- package/src/quantize/config-writer.ts +0 -145
- package/src/quantize/index.ts +0 -84
- package/src/quantize/job.ts +0 -162
- package/src/quantize/quantizer.ts +0 -666
- package/src/quantize/rotate.ts +0 -603
- package/src/quantize/safetensors-writer.ts +0 -249
- package/src/quantize/sensitivity.ts +0 -216
- package/src/quantize/trellis.ts +0 -586
- package/src/quantize/weight-transform.ts +0 -292
- package/src/registry.ts +0 -602
- package/src/runtime-config.ts +0 -81
- package/src/safetensors.ts +0 -158
- package/src/sampler.ts +0 -1011
- package/src/serve/admin-routes.ts +0 -184
- package/src/serve/assets/curve-terrain.html +0 -105
- package/src/serve/assets/training-inference-map.html +0 -575
- package/src/serve/aux-routes.ts +0 -132
- package/src/serve/batch-scheduler.ts +0 -8
- package/src/serve/checkpoint-identity.ts +0 -64
- package/src/serve/generation-gateway.ts +0 -496
- package/src/serve/isolate.ts +0 -609
- package/src/serve/kv-budget.ts +0 -14
- package/src/serve/lab-routes.ts +0 -341
- package/src/serve/lane-registry.ts +0 -45
- package/src/serve/model-admin-routes.ts +0 -155
- package/src/serve/model-binding.ts +0 -43
- package/src/serve/model-host.ts +0 -511
- package/src/serve/parent-application.ts +0 -56
- package/src/serve/prompt-response-trace.ts +0 -147
- package/src/serve/response-lease.ts +0 -24
- package/src/serve/session-search.ts +0 -326
- package/src/serve/static-routes.ts +0 -101
- package/src/server.ts +0 -1314
- package/src/spec/assistant-rows.ts +0 -92
- package/src/spec/assistant-source.ts +0 -126
- package/src/spec/cache-identity.ts +0 -22
- package/src/spec/deepspec-rows.ts +0 -29
- package/src/spec/deepspec-source.ts +0 -145
- package/src/spec/dflash-source.ts +0 -149
- package/src/spec/drafter.ts +0 -304
- package/src/spec/dspark/ab-stats.ts +0 -136
- package/src/spec/dspark/calibration.ts +0 -112
- package/src/spec/dspark/data-dflash.ts +0 -156
- package/src/spec/dspark/data.ts +0 -201
- package/src/spec/dspark/deepspec-module.ts +0 -776
- package/src/spec/dspark/generate-dflash.ts +0 -245
- package/src/spec/dspark/generate.ts +0 -290
- package/src/spec/dspark/loader.ts +0 -23
- package/src/spec/dspark/loss.ts +0 -213
- package/src/spec/dspark/module-dflash.ts +0 -560
- package/src/spec/dspark/module.ts +0 -540
- package/src/spec/dspark/quantize-drafter.ts +0 -91
- package/src/spec/dspark/sample.ts +0 -112
- package/src/spec/generate.ts +0 -276
- package/src/spec/glm52-mtp-graph.ts +0 -29
- package/src/spec/glm52-mtp-rows.ts +0 -132
- package/src/spec/glm52-mtp-source.ts +0 -317
- package/src/spec/glm52-mtp-state.ts +0 -33
- package/src/spec/ngram-source.ts +0 -204
- package/src/spec/projected-draft-rows.ts +0 -73
- package/src/spec/qwen-mtp-module.ts +0 -171
- package/src/spec/qwen-mtp-rows.ts +0 -219
- package/src/spec/qwen-mtp-source.ts +0 -383
- package/src/spec/qwen-mtp-state.ts +0 -30
- package/src/spec/serve-loop.ts +0 -585
- package/src/spec/source.ts +0 -215
- package/src/spec/two-model-rows.ts +0 -61
- package/src/spec/two-model.ts +0 -113
- package/src/ssd-cache.ts +0 -372
- package/src/ssd-durability.ts +0 -211
- package/src/tokenizer.ts +0 -335
- package/src/tool-approvals.ts +0 -131
- package/src/tool-call.ts +0 -695
- package/src/train/dataset.ts +0 -484
- package/src/train/diffusion-lora.ts +0 -216
- package/src/train/export.ts +0 -32
- package/src/train/flash-cce.ts +0 -1234
- package/src/train/forward.ts +0 -266
- package/src/train/fuse.ts +0 -243
- package/src/train/index.ts +0 -46
- package/src/train/job.ts +0 -205
- package/src/train/lora-params.ts +0 -254
- package/src/train/loss.ts +0 -1591
- package/src/train/merge.ts +0 -195
- package/src/train/optimizer.ts +0 -183
- package/src/train/prefix-shared.ts +0 -490
- package/src/train/rank.ts +0 -137
- package/src/train/segmented.ts +0 -1828
- package/src/train/steel-qmm-header.ts +0 -1461
- package/src/train/trainer.ts +0 -1221
- package/src/train/watch.ts +0 -542
- package/src/vision/diffusion-vision.ts +0 -620
- package/src/vision/embedder.ts +0 -168
- package/src/vision/preprocess.ts +0 -247
- package/src/vision/prompt.ts +0 -373
- package/src/vision/qwen3vl-preprocess.ts +0 -409
- package/src/vision/qwen3vl-prompt.ts +0 -196
- package/src/vision/qwen3vl-tower.ts +0 -555
- package/src/vision/siglip.ts +0 -599
- package/src/vision/video-frames.ts +0 -148
- package/src/web/app.html +0 -2143
- package/src/web/app.js +0 -5703
- package/src/web/skills/memory/SKILL.md +0 -120
- package/src/web/skills/web-research/SKILL.md +0 -46
- package/src/web/skills.ts +0 -97
- package/src/web/src/adapters-panel.ts +0 -288
- package/src/web/src/assistant.ts +0 -451
- package/src/web/src/chat.ts +0 -1057
- package/src/web/src/composer.ts +0 -959
- package/src/web/src/dataset.ts +0 -132
- package/src/web/src/finetune.ts +0 -267
- package/src/web/src/hub.ts +0 -428
- package/src/web/src/main.ts +0 -98
- package/src/web/src/markdown.ts +0 -0
- package/src/web/src/memory-panel.ts +0 -572
- package/src/web/src/model-picker.ts +0 -159
- package/src/web/src/palette.ts +0 -346
- package/src/web/src/protocol.ts +0 -26
- package/src/web/src/quantize.ts +0 -167
- package/src/web/src/rag.ts +0 -274
- package/src/web/src/sessions.ts +0 -469
- package/src/web/src/shell.ts +0 -804
- package/src/web/src/ui-catalog.ts +0 -78
- package/src/web/vendor/README.md +0 -61
- package/src/web-assets.ts +0 -47
- package/src/web-tools.ts +0 -478
- package/src/weights.ts +0 -255
- /package/src/{serve → server}/completion-sink.ts +0 -0
- /package/src/web/{icon.svg → public/icon.svg} +0 -0
- /package/src/web/{manifest.webmanifest → public/manifest.webmanifest} +0 -0
- /package/src/web/{vendor → public/vendor}/hljs-LICENSE +0 -0
package/src/ssd-cache.ts
DELETED
|
@@ -1,372 +0,0 @@
|
|
|
1
|
-
import type { CheckpointAttachment } from "./backends/mlx/checkpoint-state";
|
|
2
|
-
// SSD cold tier for the prompt/KV cache (docs/design/kv-cache.md).
|
|
3
|
-
//
|
|
4
|
-
// Files ARE the database: no sidecar index. Layout
|
|
5
|
-
// <dir>/<configFingerprint>/<nsHash>/<uuid>.mlxkv
|
|
6
|
-
// so two models (or two kv-quant schemes of one model) share a directory
|
|
7
|
-
// without ever colliding, and an incompatible fingerprint dir is simply
|
|
8
|
-
// ignored (never deleted — it may belong to the user's other model).
|
|
9
|
-
// Startup recovery = a header-only scan of OUR fingerprint dir: header hash
|
|
10
|
-
// verified (cheap), corrupt or metadata-mismatched files unlinked, `.tmp`
|
|
11
|
-
// write orphans reaped. LRU state = file mtime (bumped on hit); the byte
|
|
12
|
-
// cap is enforced at write time by oldest-mtime eviction. A cache is always
|
|
13
|
-
// droppable: every failure path degrades to "no hit" or "not stored" — the
|
|
14
|
-
// tier must never take serving down (oMLX paged_ssd_cache lesson, Apache-2.0
|
|
15
|
-
// idea port; ours spills whole prefix entries, not content-hashed blocks —
|
|
16
|
-
// see the design doc's D1 for why).
|
|
17
|
-
|
|
18
|
-
import {
|
|
19
|
-
existsSync, mkdirSync, readdirSync, rmSync, statSync, utimesSync,
|
|
20
|
-
} from "node:fs";
|
|
21
|
-
import { join } from "node:path";
|
|
22
|
-
import { randomUUID } from "node:crypto";
|
|
23
|
-
import { commonPrefixLength } from "./prompt-cache";
|
|
24
|
-
import {
|
|
25
|
-
saveKvCache, saveKvCacheAsync, loadKvCache, readKvHeader,
|
|
26
|
-
cacheHeadersTrimmable, cacheHeadersMinimumReusableOffset, legacyCacheCodecs, type CacheCodecProvider,
|
|
27
|
-
type KvSaveMeta, type KvLoadExpect, type LoadedKvCache,
|
|
28
|
-
} from "./kv-store";
|
|
29
|
-
import type { Cache } from "./model/gemma4-base";
|
|
30
|
-
import { minimumReusableOffset } from "./backends/mlx/state-views";
|
|
31
|
-
|
|
32
|
-
export interface SsdIndexEntry {
|
|
33
|
-
path: string;
|
|
34
|
-
ns: string;
|
|
35
|
-
tokens: number[];
|
|
36
|
-
bytes: number;
|
|
37
|
-
mtimeMs: number;
|
|
38
|
-
/** Whether every serialized cache can trim (derived from kinds+state at
|
|
39
|
-
* store/scan time). An untrimmable entry (wrapped ring, SSM) can only
|
|
40
|
-
* seed prompts it matches WITHOUT a trim — find() must not prefer it
|
|
41
|
-
* over a shorter usable one (the 2026-07-06 restart-0 defect: the
|
|
42
|
-
* [prompt+gen] file always won, always got rejected after restore). */
|
|
43
|
-
trimmable: boolean;
|
|
44
|
-
minimumReusableOffset?: number;
|
|
45
|
-
generationCheckpoint?: NonNullable<KvSaveMeta["generationCheckpoint"]>;
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
export interface SsdStoreOptions {
|
|
49
|
-
codecs?: CacheCodecProvider;
|
|
50
|
-
dir: string;
|
|
51
|
-
maxBytes: number;
|
|
52
|
-
/** Server identity — written into every file, enforced on scan/load. */
|
|
53
|
-
configFingerprint: string;
|
|
54
|
-
tokenizerHash: string;
|
|
55
|
-
modelId: string;
|
|
56
|
-
/** Verify every tensor hash on load (--ssd-cache-verify). */
|
|
57
|
-
verify?: boolean;
|
|
58
|
-
}
|
|
59
|
-
|
|
60
|
-
const nsHash = (ns: string): string =>
|
|
61
|
-
ns === "" ? "base" : Bun.hash(ns).toString(16);
|
|
62
|
-
|
|
63
|
-
export class SsdCacheStore {
|
|
64
|
-
readonly #opts: SsdStoreOptions;
|
|
65
|
-
readonly #codecs: CacheCodecProvider;
|
|
66
|
-
readonly #root: string; // <dir>/<configFingerprint>
|
|
67
|
-
#index: SsdIndexEntry[] = [];
|
|
68
|
-
#warnedWriteFailure = false;
|
|
69
|
-
stats = { restores: 0, spills: 0, restoreMsLast: 0 };
|
|
70
|
-
|
|
71
|
-
constructor(opts: SsdStoreOptions) {
|
|
72
|
-
this.#opts = opts;
|
|
73
|
-
this.#codecs = opts.codecs ?? legacyCacheCodecs;
|
|
74
|
-
this.#root = join(opts.dir, opts.configFingerprint);
|
|
75
|
-
}
|
|
76
|
-
|
|
77
|
-
get entries(): number {
|
|
78
|
-
return this.#index.length;
|
|
79
|
-
}
|
|
80
|
-
|
|
81
|
-
get totalBytes(): number {
|
|
82
|
-
return this.#index.reduce((a, e) => a + e.bytes, 0);
|
|
83
|
-
}
|
|
84
|
-
|
|
85
|
-
get maxBytes(): number {
|
|
86
|
-
return this.#opts.maxBytes;
|
|
87
|
-
}
|
|
88
|
-
|
|
89
|
-
/** Longest token prefix whose atomic cache file is present in the index. */
|
|
90
|
-
get longestDurablePrefixTokens(): number {
|
|
91
|
-
return this.#index.reduce((best, entry) => Math.max(best, entry.tokens.length), 0);
|
|
92
|
-
}
|
|
93
|
-
|
|
94
|
-
/** True when the SSD index can seed every token in this prefix. A longer
|
|
95
|
-
* trimmable descendant is equivalent to an exact boundary snapshot. */
|
|
96
|
-
hasDurablePrefix(tokens: number[], ns = ""): boolean {
|
|
97
|
-
return this.#index.some((entry) => {
|
|
98
|
-
if (entry.ns !== ns || entry.tokens.length < tokens.length || tokens.length < (entry.minimumReusableOffset ?? 0)) return false;
|
|
99
|
-
if (commonPrefixLength(entry.tokens, tokens) !== tokens.length) return false;
|
|
100
|
-
return entry.tokens.length === tokens.length || entry.trimmable;
|
|
101
|
-
});
|
|
102
|
-
}
|
|
103
|
-
|
|
104
|
-
/** Startup recovery: rebuild the in-memory index from disk. Corrupt
|
|
105
|
-
* headers and metadata mismatches are unlinked; `.tmp` orphans reaped.
|
|
106
|
-
* Only OUR fingerprint dir is touched. Returns entries indexed. */
|
|
107
|
-
scan(): number {
|
|
108
|
-
this.#index = [];
|
|
109
|
-
if (!existsSync(this.#root)) return 0;
|
|
110
|
-
for (const nsDir of readdirSync(this.#root)) {
|
|
111
|
-
const nsPath = join(this.#root, nsDir);
|
|
112
|
-
let files: string[];
|
|
113
|
-
try { files = readdirSync(nsPath); } catch { continue; }
|
|
114
|
-
for (const f of files) {
|
|
115
|
-
const path = join(nsPath, f);
|
|
116
|
-
if (f.endsWith(".tmp")) {
|
|
117
|
-
// A crash mid-write: the rename never happened. Ours by
|
|
118
|
-
// construction (uuid names), safe to reap.
|
|
119
|
-
try { rmSync(path, { force: true }); } catch {}
|
|
120
|
-
continue;
|
|
121
|
-
}
|
|
122
|
-
if (!f.endsWith(".mlxkv")) continue;
|
|
123
|
-
try {
|
|
124
|
-
const h = readKvHeader(path); // verifies the header hash
|
|
125
|
-
if (
|
|
126
|
-
h.modelId !== this.#opts.modelId ||
|
|
127
|
-
h.configFingerprint !== this.#opts.configFingerprint ||
|
|
128
|
-
h.tokenizerHash !== this.#opts.tokenizerHash
|
|
129
|
-
)
|
|
130
|
-
throw new Error("metadata mismatch");
|
|
131
|
-
if ((h.codecProvider ?? legacyCacheCodecs.id) !== this.#codecs.id) continue;
|
|
132
|
-
const st = statSync(path);
|
|
133
|
-
this.#index.push({
|
|
134
|
-
path, ns: h.ns ?? "", tokens: h.tokens, bytes: st.size, mtimeMs: st.mtimeMs,
|
|
135
|
-
trimmable: !h.attachments?.length && cacheHeadersTrimmable(h.caches, this.#codecs),
|
|
136
|
-
minimumReusableOffset: cacheHeadersMinimumReusableOffset(h.caches),
|
|
137
|
-
...(h.generationCheckpoint
|
|
138
|
-
? { generationCheckpoint: h.generationCheckpoint }
|
|
139
|
-
: {}),
|
|
140
|
-
});
|
|
141
|
-
} catch {
|
|
142
|
-
try { rmSync(path, { force: true }); } catch {}
|
|
143
|
-
}
|
|
144
|
-
}
|
|
145
|
-
}
|
|
146
|
-
return this.#index.length;
|
|
147
|
-
}
|
|
148
|
-
|
|
149
|
-
/** Best USABLE stored entry for `prompt` in `ns`: the longest common
|
|
150
|
-
* prefix (capped at prompt.length−1 — at least one token must forward).
|
|
151
|
-
* An entry LONGER than the matched prefix is returned only when it can
|
|
152
|
-
* actually seed the prompt: the divergent tail must be trimmable after
|
|
153
|
-
* restore. Untrimmable entries (ring post-wrap, SSM) that would need a
|
|
154
|
-
* trim are SKIPPED here — before this gate, the big [prompt+gen] file
|
|
155
|
-
* always outranked the usable boundary snapshot, got restored, and was
|
|
156
|
-
* thrown away by take()'s backstop (the 2026-07-06 restart-0 + wasted
|
|
157
|
-
* 84 s restore-then-re-prefill path). Pure index lookup; no I/O. */
|
|
158
|
-
find(prompt: number[], ns = ""): { entry: SsdIndexEntry; prefixLen: number } | null {
|
|
159
|
-
let best: SsdIndexEntry | null = null;
|
|
160
|
-
let bestLen = 0;
|
|
161
|
-
for (const e of this.#index) {
|
|
162
|
-
if (e.ns !== ns) continue;
|
|
163
|
-
const p = Math.min(commonPrefixLength(e.tokens, prompt), prompt.length - 1);
|
|
164
|
-
if (p === 0 || p <= bestLen || p < (e.minimumReusableOffset ?? 0)) continue;
|
|
165
|
-
if (e.tokens.length - p > 0 && !e.trimmable) continue;
|
|
166
|
-
bestLen = p;
|
|
167
|
-
best = e;
|
|
168
|
-
}
|
|
169
|
-
return best ? { entry: best, prefixLen: bestLen } : null;
|
|
170
|
-
}
|
|
171
|
-
|
|
172
|
-
/** Restore an entry via kv-store's STREAMED COPY (bounded host
|
|
173
|
-
* transient — live entry + one tensor; no mapping survives the call,
|
|
174
|
-
* the caches own their bytes). Bumps LRU mtime. On ANY failure the
|
|
175
|
-
* file is dropped and null returned. */
|
|
176
|
-
restore(entry: SsdIndexEntry, model: { makeCache(): Cache[] }): LoadedKvCache | null {
|
|
177
|
-
const t0 = performance.now();
|
|
178
|
-
try {
|
|
179
|
-
const expect: KvLoadExpect = {
|
|
180
|
-
modelId: this.#opts.modelId,
|
|
181
|
-
configFingerprint: this.#opts.configFingerprint,
|
|
182
|
-
tokenizerHash: this.#opts.tokenizerHash,
|
|
183
|
-
ns: entry.ns,
|
|
184
|
-
verify: this.#opts.verify,
|
|
185
|
-
};
|
|
186
|
-
const loaded = loadKvCache(entry.path, model, expect, this.#codecs);
|
|
187
|
-
const now = new Date();
|
|
188
|
-
try { utimesSync(entry.path, now, now); entry.mtimeMs = now.getTime(); } catch {}
|
|
189
|
-
this.stats.restores++;
|
|
190
|
-
this.stats.restoreMsLast = performance.now() - t0;
|
|
191
|
-
return loaded;
|
|
192
|
-
} catch (err) {
|
|
193
|
-
// Loud: a failed restore silently degraded to a full re-prefill for
|
|
194
|
-
// a month before the 2026-07-06 bench made it observable.
|
|
195
|
-
console.warn(`[ssd-cache] restore failed, dropping ${entry.path}: ${(err as Error).message}`);
|
|
196
|
-
this.remove(entry.path);
|
|
197
|
-
return null;
|
|
198
|
-
}
|
|
199
|
-
}
|
|
200
|
-
|
|
201
|
-
/** Persist an entry. Synchronous (the tier calls it on the idle serial
|
|
202
|
-
* lane between requests); atomic via kv-store's tmp+fsync+rename. An
|
|
203
|
-
* entry bigger than the cap is refused; disk/write failure is a warn-once
|
|
204
|
-
* soft-fail — serving never stalls on the cold tier. Existing entries in
|
|
205
|
-
* the same ns whose tokens are a PREFIX of the new entry are superseded
|
|
206
|
-
* (the agent-conversation pattern: one growing entry, not N generations
|
|
207
|
-
* of it). Returns true if stored. */
|
|
208
|
-
store(tokens: number[], caches: Cache[], ns = "", attachments?: CheckpointAttachment[]): boolean {
|
|
209
|
-
const dir = join(this.#root, nsHash(ns));
|
|
210
|
-
const path = join(dir, `${randomUUID()}.mlxkv`);
|
|
211
|
-
try {
|
|
212
|
-
mkdirSync(dir, { recursive: true });
|
|
213
|
-
saveKvCache(path, tokens, caches, { ...this.#meta(ns), attachments }, this.#codecs);
|
|
214
|
-
return this.#indexStored(path, tokens, caches, ns, undefined, attachments);
|
|
215
|
-
} catch (err) {
|
|
216
|
-
return this.#storeFailed(path, err);
|
|
217
|
-
}
|
|
218
|
-
}
|
|
219
|
-
|
|
220
|
-
/** Non-blocking store (the write-behind persistence path): same file,
|
|
221
|
-
* index, and supersede semantics as store(), but the tensor flush
|
|
222
|
-
* yields the event loop between tensors so serving interleaves.
|
|
223
|
-
* Caller passes zero-copy CLONES it owns (a consistent snapshot no
|
|
224
|
-
* matter what the live entry does meanwhile) and disposes them after.
|
|
225
|
-
* `runStep` owns every per-tensor step (see saveKvCacheAsync) — the
|
|
226
|
-
* server passes the gateway's exclusive runner so a request cannot start
|
|
227
|
-
* between an idle check and the blocking MLX readback. */
|
|
228
|
-
async storeAsync(
|
|
229
|
-
tokens: number[], caches: Cache[], ns = "",
|
|
230
|
-
runStep?: <T>(step: () => T) => Promise<T>,
|
|
231
|
-
attachments?: CheckpointAttachment[],
|
|
232
|
-
): Promise<boolean> {
|
|
233
|
-
const dir = join(this.#root, nsHash(ns));
|
|
234
|
-
const path = join(dir, `${randomUUID()}.mlxkv`);
|
|
235
|
-
try {
|
|
236
|
-
mkdirSync(dir, { recursive: true });
|
|
237
|
-
await saveKvCacheAsync(path, tokens, caches, { ...this.#meta(ns), attachments }, runStep, this.#codecs);
|
|
238
|
-
return this.#indexStored(path, tokens, caches, ns, undefined, attachments);
|
|
239
|
-
} catch (err) {
|
|
240
|
-
return this.#storeFailed(path, err);
|
|
241
|
-
}
|
|
242
|
-
}
|
|
243
|
-
|
|
244
|
-
/** Longest durable in-flight generation for an identical request. The
|
|
245
|
-
* checkpoint namespace is separate from the ordinary prompt-cache tier,
|
|
246
|
-
* so incomplete assistant output can never seed an unrelated prefill. */
|
|
247
|
-
findGenerationCheckpoint(
|
|
248
|
-
prompt: number[], key: string, cacheNs = "",
|
|
249
|
-
): SsdIndexEntry | null {
|
|
250
|
-
let best: SsdIndexEntry | null = null;
|
|
251
|
-
for (const entry of this.#index) {
|
|
252
|
-
const checkpoint = entry.generationCheckpoint;
|
|
253
|
-
if (!checkpoint || checkpoint.key !== key || checkpoint.cacheNs !== cacheNs)
|
|
254
|
-
continue;
|
|
255
|
-
if (checkpoint.originalPromptTokens !== prompt.length) continue;
|
|
256
|
-
if (entry.tokens.length <= prompt.length) continue;
|
|
257
|
-
if (commonPrefixLength(entry.tokens, prompt) !== prompt.length) continue;
|
|
258
|
-
if (!best || entry.tokens.length > best.tokens.length) best = entry;
|
|
259
|
-
}
|
|
260
|
-
return best;
|
|
261
|
-
}
|
|
262
|
-
|
|
263
|
-
/** Persist one in-flight generation atomically. A completed newer
|
|
264
|
-
* checkpoint supersedes older checkpoints for the same request only after
|
|
265
|
-
* its rename succeeds, so a crash during a write leaves the prior point. */
|
|
266
|
-
async storeGenerationCheckpoint(
|
|
267
|
-
tokens: number[], caches: Cache[], checkpoint: NonNullable<KvSaveMeta["generationCheckpoint"]>,
|
|
268
|
-
runStep?: <T>(step: () => T) => Promise<T>,
|
|
269
|
-
): Promise<boolean> {
|
|
270
|
-
const ns = `__generation_checkpoint__:${checkpoint.key}`;
|
|
271
|
-
const dir = join(this.#root, nsHash(ns));
|
|
272
|
-
const path = join(dir, `${randomUUID()}.mlxkv`);
|
|
273
|
-
try {
|
|
274
|
-
if (runStep) await runStep(() => mkdirSync(dir, { recursive: true }));
|
|
275
|
-
else mkdirSync(dir, { recursive: true });
|
|
276
|
-
await saveKvCacheAsync(path, tokens, caches, {
|
|
277
|
-
...this.#meta(ns), generationCheckpoint: checkpoint,
|
|
278
|
-
}, runStep, this.#codecs);
|
|
279
|
-
const index = () => {
|
|
280
|
-
const stored = this.#indexStored(path, tokens, caches, ns, checkpoint);
|
|
281
|
-
if (stored) {
|
|
282
|
-
for (const entry of [...this.#index]) {
|
|
283
|
-
if (entry.path === path) continue;
|
|
284
|
-
if (entry.generationCheckpoint?.key === checkpoint.key) this.remove(entry.path);
|
|
285
|
-
}
|
|
286
|
-
}
|
|
287
|
-
return stored;
|
|
288
|
-
};
|
|
289
|
-
return runStep ? await runStep(index) : index();
|
|
290
|
-
} catch (err) {
|
|
291
|
-
return runStep ? await runStep(() => this.#storeFailed(path, err)) : this.#storeFailed(path, err);
|
|
292
|
-
}
|
|
293
|
-
}
|
|
294
|
-
|
|
295
|
-
removeGenerationCheckpoints(key: string): void {
|
|
296
|
-
for (const entry of [...this.#index])
|
|
297
|
-
if (entry.generationCheckpoint?.key === key) this.remove(entry.path);
|
|
298
|
-
}
|
|
299
|
-
|
|
300
|
-
#meta(ns: string): KvSaveMeta {
|
|
301
|
-
return {
|
|
302
|
-
modelId: this.#opts.modelId,
|
|
303
|
-
configFingerprint: this.#opts.configFingerprint,
|
|
304
|
-
tokenizerHash: this.#opts.tokenizerHash,
|
|
305
|
-
ns,
|
|
306
|
-
};
|
|
307
|
-
}
|
|
308
|
-
|
|
309
|
-
#indexStored(
|
|
310
|
-
path: string, tokens: number[], caches: Cache[], ns: string,
|
|
311
|
-
generationCheckpoint?: NonNullable<KvSaveMeta["generationCheckpoint"]>,
|
|
312
|
-
attachments?: CheckpointAttachment[],
|
|
313
|
-
): boolean {
|
|
314
|
-
const st = statSync(path);
|
|
315
|
-
if (st.size > this.#opts.maxBytes) {
|
|
316
|
-
rmSync(path, { force: true });
|
|
317
|
-
console.warn(`[ssd-cache] entry not stored: ${st.size} bytes exceeds the ${this.#opts.maxBytes}-byte cap`);
|
|
318
|
-
return false;
|
|
319
|
-
}
|
|
320
|
-
const trimmable = !attachments?.length && caches.every((c) => c.isTrimmable());
|
|
321
|
-
// Exact duplicates (same tokens) are replaced regardless of
|
|
322
|
-
// trimmability — the new file serves exactly the old one's matches.
|
|
323
|
-
for (const e of [...this.#index]) {
|
|
324
|
-
if (e.ns !== ns || e.tokens.length !== tokens.length) continue;
|
|
325
|
-
if (commonPrefixLength(e.tokens, tokens) === tokens.length) this.remove(e.path);
|
|
326
|
-
}
|
|
327
|
-
// Supersede shadowed ancestors of this prefix (same ns) — ONLY when
|
|
328
|
-
// the new entry can serve their prefixes (all caches trimmable; the
|
|
329
|
-
// same guard as PromptCache.put()). An untrimmable [prompt+gen]
|
|
330
|
-
// entry deleting the boundary snapshot was the gemma restart-0
|
|
331
|
-
// defect (2026-07-06): the only restorable file died the moment the
|
|
332
|
-
// unrestorable one landed.
|
|
333
|
-
if (trimmable) {
|
|
334
|
-
for (const e of [...this.#index]) {
|
|
335
|
-
if (e.ns !== ns || e.tokens.length >= tokens.length || e.tokens.length < minimumReusableOffset(caches)) continue;
|
|
336
|
-
if (commonPrefixLength(e.tokens, tokens) === e.tokens.length) this.remove(e.path);
|
|
337
|
-
}
|
|
338
|
-
}
|
|
339
|
-
this.#index.push({
|
|
340
|
-
path, ns, tokens, bytes: st.size, mtimeMs: Date.now(), trimmable,
|
|
341
|
-
minimumReusableOffset: minimumReusableOffset(caches),
|
|
342
|
-
...(generationCheckpoint ? { generationCheckpoint } : {}),
|
|
343
|
-
});
|
|
344
|
-
this.stats.spills++;
|
|
345
|
-
this.evictToCap();
|
|
346
|
-
return true;
|
|
347
|
-
}
|
|
348
|
-
|
|
349
|
-
#storeFailed(path: string, err: unknown): boolean {
|
|
350
|
-
try { rmSync(path, { force: true }); rmSync(`${path}.tmp`, { force: true }); } catch {}
|
|
351
|
-
if (!this.#warnedWriteFailure) {
|
|
352
|
-
this.#warnedWriteFailure = true;
|
|
353
|
-
console.warn(`[ssd-cache] store failed (disk full or unwritable?) — cold tier disabled for this entry: ${(err as Error).message}`);
|
|
354
|
-
}
|
|
355
|
-
return false;
|
|
356
|
-
}
|
|
357
|
-
|
|
358
|
-
/** Enforce the byte cap: unlink oldest-mtime entries until under it. */
|
|
359
|
-
evictToCap(): void {
|
|
360
|
-
while (this.totalBytes > this.#opts.maxBytes && this.#index.length > 0) {
|
|
361
|
-
let oldest = 0;
|
|
362
|
-
for (let i = 1; i < this.#index.length; i++)
|
|
363
|
-
if (this.#index[i]!.mtimeMs < this.#index[oldest]!.mtimeMs) oldest = i;
|
|
364
|
-
this.remove(this.#index[oldest]!.path);
|
|
365
|
-
}
|
|
366
|
-
}
|
|
367
|
-
|
|
368
|
-
remove(path: string): void {
|
|
369
|
-
try { rmSync(path, { force: true }); } catch {}
|
|
370
|
-
this.#index = this.#index.filter((e) => e.path !== path);
|
|
371
|
-
}
|
|
372
|
-
}
|
package/src/ssd-durability.ts
DELETED
|
@@ -1,211 +0,0 @@
|
|
|
1
|
-
import { cloneAttachments } from "./backends/mlx/checkpoint-state";
|
|
2
|
-
import { cleanupFailure, disposeResources } from "./engine/resources";
|
|
3
|
-
import type { Cache } from "./model/gemma4";
|
|
4
|
-
import type { PromptCacheEntry } from "./prompt-cache";
|
|
5
|
-
import type { SpillItem, SpillQueue } from "./kv-store";
|
|
6
|
-
|
|
7
|
-
export interface DurabilityGateway {
|
|
8
|
-
readonly busy: boolean;
|
|
9
|
-
runWhenIdle<T>(fn: () => Promise<T>): Promise<T>;
|
|
10
|
-
}
|
|
11
|
-
|
|
12
|
-
export interface DurabilityPromptCache {
|
|
13
|
-
findExact(tokens: number[], ns?: string): PromptCacheEntry | null;
|
|
14
|
-
}
|
|
15
|
-
|
|
16
|
-
export interface DurabilitySnapshotStats {
|
|
17
|
-
pendingSnapshots: number;
|
|
18
|
-
pendingSpills: number;
|
|
19
|
-
pendingSpillBytes: number;
|
|
20
|
-
droppedSpills: number;
|
|
21
|
-
failedSpills: number;
|
|
22
|
-
}
|
|
23
|
-
|
|
24
|
-
export interface DurabilityFlushResult extends DurabilitySnapshotStats {
|
|
25
|
-
durable: boolean;
|
|
26
|
-
flushedSnapshots: number;
|
|
27
|
-
missingSnapshots: number;
|
|
28
|
-
elapsedMs: number;
|
|
29
|
-
}
|
|
30
|
-
|
|
31
|
-
interface DirtySnapshot {
|
|
32
|
-
key: string;
|
|
33
|
-
tokens: number[];
|
|
34
|
-
ns: string;
|
|
35
|
-
}
|
|
36
|
-
|
|
37
|
-
type StoreOutcome = "stored" | "missing" | "failed";
|
|
38
|
-
|
|
39
|
-
/**
|
|
40
|
-
* Turns debounced prompt-cache puts into a durability boundary.
|
|
41
|
-
*
|
|
42
|
-
* A dirty record stays present until SpillQueue reports that its atomic store
|
|
43
|
-
* completed. Queue drops and write failures therefore remain retryable during
|
|
44
|
-
* an explicit flush or graceful shutdown.
|
|
45
|
-
*/
|
|
46
|
-
export class SsdDurabilityCoordinator {
|
|
47
|
-
readonly #dirty = new Map<string, DirtySnapshot>();
|
|
48
|
-
readonly #timers = new Map<string, ReturnType<typeof setTimeout>>();
|
|
49
|
-
readonly #attempts = new Map<string, Promise<StoreOutcome>>();
|
|
50
|
-
#flush: Promise<DurabilityFlushResult> | null = null;
|
|
51
|
-
|
|
52
|
-
constructor(
|
|
53
|
-
readonly gateway: DurabilityGateway,
|
|
54
|
-
readonly promptCache: DurabilityPromptCache,
|
|
55
|
-
readonly spillQueue: SpillQueue,
|
|
56
|
-
readonly cloneCaches: (caches: Cache[]) => Cache[],
|
|
57
|
-
readonly isAlreadyDurable: (tokens: number[], ns: string) => boolean = () => false,
|
|
58
|
-
readonly debounceMs = 1_000,
|
|
59
|
-
readonly busyRetryMs = 5_000,
|
|
60
|
-
) {}
|
|
61
|
-
|
|
62
|
-
schedule(tokens: number[], ns = ""): void {
|
|
63
|
-
if (tokens.length === 0) return;
|
|
64
|
-
// PromptCache can hold unrelated entries with the same namespace and
|
|
65
|
-
// length. Include the tokens so one conversation cannot cancel another
|
|
66
|
-
// conversation's pending durability record.
|
|
67
|
-
const key = `${ns.length}:${ns}:${tokens.join(",")}`;
|
|
68
|
-
const rec: DirtySnapshot = { key, tokens: [...tokens], ns };
|
|
69
|
-
this.#dirty.set(key, rec);
|
|
70
|
-
this.#arm(key, this.debounceMs);
|
|
71
|
-
}
|
|
72
|
-
|
|
73
|
-
get stats(): DurabilitySnapshotStats {
|
|
74
|
-
return {
|
|
75
|
-
pendingSnapshots: this.#dirty.size,
|
|
76
|
-
pendingSpills: this.spillQueue.pendingCount,
|
|
77
|
-
pendingSpillBytes: this.spillQueue.pendingBytes,
|
|
78
|
-
droppedSpills: this.spillQueue.droppedCount,
|
|
79
|
-
failedSpills: this.spillQueue.failedCount,
|
|
80
|
-
};
|
|
81
|
-
}
|
|
82
|
-
|
|
83
|
-
/** Force every dirty RAM snapshot through the queue and wait for storage. */
|
|
84
|
-
flush(): Promise<DurabilityFlushResult> {
|
|
85
|
-
if (this.#flush) return this.#flush;
|
|
86
|
-
const started = performance.now();
|
|
87
|
-
this.#flush = this.#flushInner(started).finally(() => { this.#flush = null; });
|
|
88
|
-
return this.#flush;
|
|
89
|
-
}
|
|
90
|
-
|
|
91
|
-
#arm(key: string, delayMs: number): void {
|
|
92
|
-
const old = this.#timers.get(key);
|
|
93
|
-
if (old) clearTimeout(old);
|
|
94
|
-
const timer = setTimeout(() => {
|
|
95
|
-
this.#timers.delete(key);
|
|
96
|
-
void this.#attempt(key, false);
|
|
97
|
-
}, delayMs);
|
|
98
|
-
timer.unref?.();
|
|
99
|
-
this.#timers.set(key, timer);
|
|
100
|
-
}
|
|
101
|
-
|
|
102
|
-
async #attempt(key: string, force: boolean): Promise<StoreOutcome> {
|
|
103
|
-
const running = this.#attempts.get(key);
|
|
104
|
-
if (running) {
|
|
105
|
-
const outcome = await running;
|
|
106
|
-
if (!force || !this.#dirty.has(key)) return outcome;
|
|
107
|
-
}
|
|
108
|
-
|
|
109
|
-
const rec = this.#dirty.get(key);
|
|
110
|
-
if (!rec) return "stored";
|
|
111
|
-
if (!force && this.gateway.busy) {
|
|
112
|
-
this.#arm(key, this.busyRetryMs);
|
|
113
|
-
return "failed";
|
|
114
|
-
}
|
|
115
|
-
|
|
116
|
-
const task = this.#store(rec);
|
|
117
|
-
this.#attempts.set(key, task);
|
|
118
|
-
try {
|
|
119
|
-
const outcome = await task;
|
|
120
|
-
if (outcome === "stored" && this.#dirty.get(key) === rec)
|
|
121
|
-
this.#dirty.delete(key);
|
|
122
|
-
if (outcome !== "stored" && !force && this.#dirty.get(key) === rec)
|
|
123
|
-
this.#arm(key, this.busyRetryMs);
|
|
124
|
-
return outcome;
|
|
125
|
-
} finally {
|
|
126
|
-
if (this.#attempts.get(key) === task) this.#attempts.delete(key);
|
|
127
|
-
}
|
|
128
|
-
}
|
|
129
|
-
|
|
130
|
-
async #store(rec: DirtySnapshot): Promise<StoreOutcome> {
|
|
131
|
-
let snap: SpillItem | null = null;
|
|
132
|
-
try {
|
|
133
|
-
snap = await this.gateway.runWhenIdle(async () => {
|
|
134
|
-
if (this.isAlreadyDurable(rec.tokens, rec.ns)) return null;
|
|
135
|
-
const entry = this.promptCache.findExact(rec.tokens, rec.ns);
|
|
136
|
-
if (!entry) return null;
|
|
137
|
-
const caches = this.cloneCaches(entry.caches);
|
|
138
|
-
try {
|
|
139
|
-
return { tokens: [...entry.tokens], caches, ns: rec.ns,
|
|
140
|
-
attachments: cloneAttachments(entry.attachments) };
|
|
141
|
-
} catch (error) {
|
|
142
|
-
return cleanupFailure(error, () => disposeResources(caches));
|
|
143
|
-
}
|
|
144
|
-
});
|
|
145
|
-
} catch {
|
|
146
|
-
return "failed";
|
|
147
|
-
}
|
|
148
|
-
if (!snap)
|
|
149
|
-
return this.isAlreadyDurable(rec.tokens, rec.ns) ? "stored" : "missing";
|
|
150
|
-
return await this.spillQueue.enqueue(snap) ? "stored" : "failed";
|
|
151
|
-
}
|
|
152
|
-
|
|
153
|
-
async #flushInner(started: number): Promise<DurabilityFlushResult> {
|
|
154
|
-
for (const timer of this.#timers.values()) clearTimeout(timer);
|
|
155
|
-
this.#timers.clear();
|
|
156
|
-
|
|
157
|
-
let flushedSnapshots = 0;
|
|
158
|
-
const missing = new Map<string, DirtySnapshot>();
|
|
159
|
-
const droppedBefore = this.spillQueue.droppedCount;
|
|
160
|
-
const failedBefore = this.spillQueue.failedCount;
|
|
161
|
-
|
|
162
|
-
if (this.#attempts.size > 0)
|
|
163
|
-
await Promise.allSettled([...this.#attempts.values()]);
|
|
164
|
-
await this.spillQueue.drain();
|
|
165
|
-
|
|
166
|
-
// Flush one entry at a time. This prevents the queue cap from dropping a
|
|
167
|
-
// boundary snapshot while a large final snapshot is already in flight.
|
|
168
|
-
// Attempt each record version once. Missing state stays dirty, so a
|
|
169
|
-
// second flush cannot report success without a real durable snapshot.
|
|
170
|
-
const attempted = new Set<DirtySnapshot>();
|
|
171
|
-
while (this.#dirty.size > 0) {
|
|
172
|
-
const records = [...this.#dirty.entries()].filter(([, rec]) => !attempted.has(rec));
|
|
173
|
-
if (records.length === 0) break;
|
|
174
|
-
for (const [key, rec] of records) {
|
|
175
|
-
if (this.#dirty.get(key) !== rec) continue;
|
|
176
|
-
attempted.add(rec);
|
|
177
|
-
const outcome = await this.#attempt(key, true);
|
|
178
|
-
await this.spillQueue.drain();
|
|
179
|
-
if (outcome === "stored") {
|
|
180
|
-
flushedSnapshots++;
|
|
181
|
-
} else if (outcome === "missing") {
|
|
182
|
-
missing.set(key, rec);
|
|
183
|
-
}
|
|
184
|
-
}
|
|
185
|
-
}
|
|
186
|
-
|
|
187
|
-
// A later trimmable snapshot may now cover a superseded RAM ancestor.
|
|
188
|
-
// Reconcile against committed SSD coverage after all writes settle;
|
|
189
|
-
// never clear a missing prefix merely because another write succeeded.
|
|
190
|
-
for (const [key, rec] of missing) {
|
|
191
|
-
if (!this.isAlreadyDurable(rec.tokens, rec.ns)) continue;
|
|
192
|
-
if (this.#dirty.get(key) === rec) this.#dirty.delete(key);
|
|
193
|
-
missing.delete(key);
|
|
194
|
-
flushedSnapshots++;
|
|
195
|
-
}
|
|
196
|
-
|
|
197
|
-
const stats = this.stats;
|
|
198
|
-
return {
|
|
199
|
-
...stats,
|
|
200
|
-
durable:
|
|
201
|
-
stats.pendingSnapshots === 0 &&
|
|
202
|
-
stats.pendingSpills === 0 &&
|
|
203
|
-
stats.droppedSpills === droppedBefore &&
|
|
204
|
-
stats.failedSpills === failedBefore &&
|
|
205
|
-
missing.size === 0,
|
|
206
|
-
flushedSnapshots,
|
|
207
|
-
missingSnapshots: missing.size,
|
|
208
|
-
elapsedMs: performance.now() - started,
|
|
209
|
-
};
|
|
210
|
-
}
|
|
211
|
-
}
|