mlx-bun 0.4.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1453 -88
- package/THIRD_PARTY_NOTICES.md +882 -0
- package/bin/mlx-bun.convert.mjs +4 -0
- package/bin/mlx-bun.fuse.mjs +4 -0
- package/bin/mlx-bun.generate.mjs +4 -0
- package/bin/mlx-bun.lora.mjs +4 -0
- package/bin/mlx-bun.mjs +13 -21
- package/bin/mlx-bun.server.mjs +4 -0
- package/bin/mlx-bun.upload.mjs +4 -0
- package/dist/web/app.js +9584 -0
- package/package.json +50 -53
- package/src/cli/ane-install.ts +76 -0
- package/src/cli/args.ts +179 -0
- package/src/cli/engine-entry.ts +68 -0
- package/src/cli/inference.ts +214 -0
- package/src/cli/job-entry.ts +97 -0
- package/src/cli/library-host.ts +117 -0
- package/src/cli/main.ts +57 -0
- package/src/cli/memory-engine.ts +163 -0
- package/src/cli/memory.ts +36 -0
- package/src/cli/mlx-lm-aliases.ts +309 -0
- package/src/cli/model-choice.ts +33 -0
- package/src/cli/model-selection.ts +84 -0
- package/src/cli/module-host.ts +11 -0
- package/src/cli/module-verbs.ts +50 -0
- package/src/cli/numerical-policy.ts +12 -0
- package/src/cli/perplexity.ts +89 -0
- package/src/cli/serve-host.ts +423 -0
- package/src/cli/serve-isolated.ts +217 -0
- package/src/cli/serve-options.ts +106 -0
- package/src/cli/serve-state.ts +205 -0
- package/src/cli/serve.ts +500 -0
- package/src/cli/served-model-host.ts +102 -0
- package/src/cli/server-entry.ts +148 -0
- package/src/cli/serving-unit.ts +177 -0
- package/src/{tui.ts → cli/terminal.ts} +13 -8
- package/src/cli/worker-adapters.ts +34 -0
- package/src/cli/worker-entry.ts +272 -0
- package/src/cli/worker-unit.ts +179 -0
- package/src/engine/adapter-operation.ts +26 -0
- package/src/engine/cache-services.ts +114 -0
- package/src/engine/completion.ts +39 -0
- package/src/engine/generation-gateway.ts +358 -0
- package/src/engine/index.ts +59 -0
- package/src/engine/kv-budget.ts +120 -0
- package/src/engine/media-preparation.ts +232 -0
- package/src/engine/model-binding.ts +24 -0
- package/src/engine/model-host.ts +513 -0
- package/src/engine/model-serving.ts +89 -0
- package/src/{serve → engine}/preparation.ts +10 -8
- package/src/{serve → engine}/session-completion-engine.ts +6 -6
- package/src/engine/telemetry.ts +109 -0
- package/src/hub/downloads.ts +116 -0
- package/src/jobs/db.ts +16 -7
- package/src/jobs/events.ts +29 -0
- package/src/jobs/executable.ts +5 -0
- package/src/jobs/host.ts +51 -0
- package/src/jobs/in-process.ts +55 -0
- package/src/{contracts/jobs.ts → jobs/protocol.ts} +7 -9
- package/src/jobs/runner.ts +150 -231
- package/src/jobs/service.ts +75 -0
- package/src/jobs/sse.ts +59 -23
- package/src/jobs/worker-memory.ts +46 -0
- package/src/jobs/worker-process.ts +163 -0
- package/src/jobs/worker-supervisor.ts +258 -0
- package/src/modules.ts +58 -0
- package/src/publishing/catalog-hub.ts +18 -0
- package/src/publishing/credentials.ts +31 -0
- package/src/publishing/upload.ts +37 -0
- package/src/residency/leased-adapters.ts +21 -0
- package/src/residency/local-records.ts +16 -0
- package/src/residency/model-residency.ts +405 -0
- package/src/residency/record-index.ts +28 -0
- package/src/residency/resident-estimate.ts +50 -0
- package/src/{anthropic.ts → server/anthropic.ts} +3 -81
- package/src/server/cache-routes.ts +36 -0
- package/src/{serve → server}/chat-request.ts +12 -10
- package/src/{serve → server}/chat-stage.ts +27 -22
- package/src/{client.ts → server/client.ts} +28 -2
- package/src/{serve → server}/completion-executor.ts +42 -42
- package/src/server/decision-body.ts +55 -0
- package/src/server/decision-routes.ts +134 -0
- package/src/server/decision-wire.ts +228 -0
- package/src/{serve → server}/discovery-routes.ts +42 -83
- package/src/server/generated-token-history.ts +57 -0
- package/src/{serve → server}/http.ts +25 -13
- package/src/{serve → server}/inference-request.ts +5 -5
- package/src/server/job-routes.ts +20 -0
- package/src/server/media-prompt.ts +63 -0
- package/src/server/memory-completion-client.ts +46 -0
- package/src/server/model-routes.ts +192 -0
- package/src/{serve → server}/openai-wire.ts +3 -2
- package/src/{serve → server}/pipeline.ts +1 -1
- package/src/server/prompt-builder.ts +13 -0
- package/src/server/prompt-contracts.ts +17 -0
- package/src/server/proxy-routes.ts +187 -0
- package/src/server/publishing-routes.ts +41 -0
- package/src/{serve → server}/request-plan.ts +8 -8
- package/src/{serve → server}/request-prep.ts +82 -68
- package/src/{serve → server}/responses-client.ts +30 -13
- package/src/{responses.ts → server/responses.ts} +12 -93
- package/src/server/routes.ts +205 -0
- package/src/server/start.ts +72 -0
- package/src/server/status-routes.ts +108 -0
- package/src/{serve → server}/text-completion-stage.ts +25 -4
- package/src/server/text-prompt.ts +12 -0
- package/src/{serve → server}/token-streams.ts +36 -38
- package/src/server/worker-routes.ts +311 -0
- package/src/storage/paths.ts +61 -0
- package/src/web/assets.ts +37 -0
- package/src/web/{src → browser}/api.ts +5 -23
- package/src/web/browser/chat-host.ts +17 -0
- package/src/web/browser/installed-panels.ts +6 -0
- package/src/web/browser/main.ts +55 -0
- package/src/web/browser/markdown.ts +279 -0
- package/src/web/browser/memory-panel.ts +9 -0
- package/src/web/browser/palette.ts +64 -0
- package/src/web/browser/protocol.ts +13 -0
- package/src/web/browser/shell.ts +467 -0
- package/src/web/{src → browser}/status.ts +32 -36
- package/src/web/build.ts +51 -0
- package/src/web/public/app.html +1054 -0
- package/src/web/{sw.js → public/sw.js} +4 -3
- package/src/web/{vendor → public/vendor}/hljs-theme.css +2 -2
- package/src/web/{vendor → public/vendor}/hljs.js +2 -2
- package/THIRD_PARTY_LICENSES.md +0 -51
- package/docs/README.md +0 -94
- package/docs/archive/hlg-sampling.md +0 -350
- package/docs/archive/investigations/audio-input-plan.md +0 -356
- package/docs/archive/investigations/chunk-finetune-experiment.md +0 -84
- package/docs/archive/investigations/colibri-glm52-port.md +0 -1486
- package/docs/archive/investigations/colibri-oracle-pin.md +0 -1284
- package/docs/archive/investigations/curve-bisector-routing.md +0 -123
- package/docs/archive/investigations/decode-roofline-lookagain.md +0 -348
- package/docs/archive/investigations/diffusion-gemma-port.md +0 -602
- package/docs/archive/investigations/dspark-handoff.md +0 -224
- package/docs/archive/investigations/expert-offload-single-user-moe.md +0 -292
- package/docs/archive/investigations/faithful-l1-consolidation.md +0 -310
- package/docs/archive/investigations/hlg-sampling-investigation.md +0 -970
- package/docs/archive/investigations/indexshare-performance-spike.md +0 -357
- package/docs/archive/investigations/kernel-perf-review-2026-07.md +0 -171
- package/docs/archive/investigations/lab-build-journal.md +0 -157
- package/docs/archive/investigations/minicpm5-decode-megakernel.md +0 -489
- package/docs/archive/investigations/openjarvis-fit.md +0 -198
- package/docs/archive/investigations/openwiki-evaluation.md +0 -249
- package/docs/archive/investigations/optimization_plan.md +0 -346
- package/docs/archive/investigations/orpo-base-uf-experiment-and-directions.md +0 -120
- package/docs/archive/investigations/orpo-flash-cce-pin-leak.md +0 -167
- package/docs/archive/investigations/orpo-uf-testing-handoff.md +0 -126
- package/docs/archive/investigations/pareto-specialized-runtime-findings.md +0 -92
- package/docs/archive/investigations/pi-builtin-investigation.md +0 -308
- package/docs/archive/investigations/pre-colibri-stabilization.md +0 -329
- package/docs/archive/investigations/qwen38-27b-campaign-log.md +0 -1092
- package/docs/archive/investigations/segmented-backward-handoff.md +0 -148
- package/docs/archive/investigations/starter-model-port-handoff.md +0 -342
- package/docs/archive/investigations/steel-flash-cce-handoff.md +0 -489
- package/docs/archive/investigations/the-dreaming-handoff.md +0 -141
- package/docs/archive/investigations/trainer-validation-experiment.md +0 -173
- package/docs/archive/planning/chat-ui-vision-memory-first.md +0 -199
- package/docs/archive/planning/chat-ui-vision.md +0 -50
- package/docs/archive/planning/chunk-task-roadmap.md +0 -77
- package/docs/archive/planning/curve-sampler-research-plan.md +0 -331
- package/docs/archive/planning/journal.md +0 -316
- package/docs/archive/planning/memory-docs-and-dag-plan.md +0 -194
- package/docs/archive/planning/pi-integration-review.md +0 -178
- package/docs/archive/planning/release-notes-v0.0.10.md +0 -57
- package/docs/archive/planning/release-notes-v0.0.11.md +0 -239
- package/docs/archive/planning/release-notes-v0.0.12.md +0 -80
- package/docs/archive/planning/release-notes-v0.0.13.md +0 -77
- package/docs/archive/planning/release-notes-v0.0.9.md +0 -122
- package/docs/archive/planning/release-notes-v0.2.0.md +0 -49
- package/docs/archive/planning/release-notes-v0.3.0.md +0 -35
- package/docs/archive/planning/web-ui-pass-plan.md +0 -213
- package/docs/archive/planning/website-readme-pass-plan.md +0 -224
- package/docs/design/batching.md +0 -1841
- package/docs/design/decode-speed-program.md +0 -2349
- package/docs/design/dreaming-nightly-pipeline.md +0 -684
- package/docs/design/generic-model-support.md +0 -705
- package/docs/design/kv-cache.md +0 -824
- package/docs/design/orpo-training.md +0 -680
- package/docs/design/speculative-decoding.md +0 -1990
- package/docs/design/turboquant.md +0 -2212
- package/docs/design/unified-engine-frontier-plan.md +0 -1880
- package/docs/design/web-chat-redesign.md +0 -695
- package/docs/planning/PRODUCT_ROADMAP.md +0 -594
- package/docs/planning/ResearchTopics.md +0 -119
- package/docs/planning/release-notes-v0.4.0.md +0 -45
- package/docs/planning/video-series.md +0 -421
- package/docs/reference/benchmarks.md +0 -3613
- package/docs/reference/cli.md +0 -585
- package/docs/reference/distribution.md +0 -355
- package/docs/reference/environment.md +0 -398
- package/docs/reference/glossary.md +0 -39
- package/docs/reference/library-api.md +0 -674
- package/docs/reference/memory.md +0 -260
- package/docs/reference/models.md +0 -334
- package/docs/reference/server-api.md +0 -1775
- package/docs/reference/server-config.md +0 -846
- package/docs/reference/training.md +0 -462
- package/docs/reference/troubleshooting.md +0 -158
- package/src/admission-plan.ts +0 -18
- package/src/audio/conformer.ts +0 -647
- package/src/audio/decode.ts +0 -133
- package/src/audio/features.ts +0 -202
- package/src/audio/transcode.ts +0 -67
- package/src/backends/mlx/assistant-target.ts +0 -46
- package/src/backends/mlx/autoregressive-draft-rows.ts +0 -120
- package/src/backends/mlx/autoregressive.ts +0 -108
- package/src/backends/mlx/batch-group.ts +0 -1790
- package/src/backends/mlx/cache-attachment.ts +0 -67
- package/src/backends/mlx/cache-layout.ts +0 -31
- package/src/backends/mlx/checkpoint-state.ts +0 -53
- package/src/backends/mlx/continuation-persistence.ts +0 -106
- package/src/backends/mlx/continuation-request.ts +0 -43
- package/src/backends/mlx/continuation.ts +0 -77
- package/src/backends/mlx/diffusion.ts +0 -39
- package/src/backends/mlx/draft-target.ts +0 -32
- package/src/backends/mlx/evaluation.ts +0 -105
- package/src/backends/mlx/fill-append.ts +0 -45
- package/src/backends/mlx/full-kv-row-append.ts +0 -32
- package/src/backends/mlx/full-kv-row-donor.ts +0 -54
- package/src/backends/mlx/full-transitioning-kv-rows.ts +0 -65
- package/src/backends/mlx/gateway-binding.ts +0 -130
- package/src/backends/mlx/graph.ts +0 -38
- package/src/backends/mlx/kv-codec.ts +0 -9
- package/src/backends/mlx/kv-maintenance.ts +0 -133
- package/src/backends/mlx/kv-tensor-rows.ts +0 -212
- package/src/backends/mlx/memory-client.ts +0 -189
- package/src/backends/mlx/methods.ts +0 -90
- package/src/backends/mlx/model-prompt.ts +0 -164
- package/src/backends/mlx/model-serving.ts +0 -79
- package/src/backends/mlx/prefill-cohort.ts +0 -89
- package/src/backends/mlx/prefill-rows.ts +0 -200
- package/src/backends/mlx/prefill.ts +0 -33
- package/src/backends/mlx/projected-context-rows.ts +0 -72
- package/src/backends/mlx/request-policy.ts +0 -20
- package/src/backends/mlx/request-state-policy.ts +0 -41
- package/src/backends/mlx/rollback.ts +0 -89
- package/src/backends/mlx/row-sampling.ts +0 -49
- package/src/backends/mlx/serial-executor.ts +0 -242
- package/src/backends/mlx/speculative-group.ts +0 -359
- package/src/backends/mlx/speculative-round.ts +0 -161
- package/src/backends/mlx/speculative.ts +0 -114
- package/src/backends/mlx/state-rows.ts +0 -104
- package/src/backends/mlx/state-views.ts +0 -46
- package/src/backends/mlx/target-layout-capability.ts +0 -35
- package/src/backends/mlx/text-engine.ts +0 -73
- package/src/backends/mlx/transitioning-kv-rows.ts +0 -165
- package/src/chat-template.ts +0 -382
- package/src/cli.ts +0 -3168
- package/src/config.ts +0 -399
- package/src/contracts/completion.ts +0 -16
- package/src/contracts/execution.ts +0 -60
- package/src/contracts/generation.ts +0 -103
- package/src/contracts/host.ts +0 -7
- package/src/contracts/pi-provider.ts +0 -30
- package/src/contracts/pi.ts +0 -259
- package/src/contracts/prefix-cache.ts +0 -23
- package/src/contracts/resources.ts +0 -25
- package/src/contracts/sampling.ts +0 -23
- package/src/contracts/scheduling.ts +0 -45
- package/src/contracts/tsconfig.json +0 -14
- package/src/dataset/generators.ts +0 -1087
- package/src/dataset/index.ts +0 -12
- package/src/dataset/job.ts +0 -81
- package/src/dataset/llm.ts +0 -71
- package/src/dataset/registry.ts +0 -528
- package/src/diffusion/diffusion-generate.ts +0 -507
- package/src/download.ts +0 -505
- package/src/embed.ts +0 -77
- package/src/engine/admission.ts +0 -59
- package/src/engine/cancellation.ts +0 -38
- package/src/engine/engine.ts +0 -56
- package/src/engine/execution-coordinator.ts +0 -70
- package/src/engine/execution-plan.ts +0 -53
- package/src/engine/resources.ts +0 -53
- package/src/engine/scheduler.ts +0 -72
- package/src/engine/session.ts +0 -247
- package/src/eval/README.md +0 -130
- package/src/eval/ifeval.ts +0 -81
- package/src/eval/kl-prompts.ts +0 -29
- package/src/eval/kl.ts +0 -590
- package/src/eval/perplexity.ts +0 -164
- package/src/eval/quality-db.ts +0 -92
- package/src/eval/reference-logits.ts +0 -207
- package/src/eval/runner.ts +0 -124
- package/src/eval/score.ts +0 -49
- package/src/eval/tasks/bfcl.ts +0 -535
- package/src/eval/tasks/gsm8k.ts +0 -98
- package/src/eval/tasks/hashhop.ts +0 -139
- package/src/eval/tasks/humaneval.ts +0 -275
- package/src/eval/tasks/ifeval.ts +0 -525
- package/src/eval/tasks/mmlu.ts +0 -157
- package/src/eval/ultrafeedback.ts +0 -60
- package/src/evaldb.ts +0 -199
- package/src/expert-io.ts +0 -393
- package/src/expert-offload-build.ts +0 -77
- package/src/expert-offload.ts +0 -76
- package/src/expert-residency.ts +0 -1134
- package/src/expert-usage.ts +0 -415
- package/src/fill/echo-index.ts +0 -188
- package/src/fill/fill-session.ts +0 -452
- package/src/fill/proposal.ts +0 -68
- package/src/fill/schema-rows.ts +0 -506
- package/src/fill/tool-boundary.ts +0 -103
- package/src/fit.ts +0 -249
- package/src/generate.ts +0 -1346
- package/src/grammar.ts +0 -591
- package/src/harness-pi.ts +0 -184
- package/src/hf-push.ts +0 -628
- package/src/hub-rest.ts +0 -242
- package/src/index.ts +0 -63
- package/src/inference/denoising.ts +0 -17
- package/src/inference/draft-acceptance.ts +0 -41
- package/src/inference/draft-output.ts +0 -49
- package/src/inference/graph.ts +0 -33
- package/src/inference/prefill.ts +0 -39
- package/src/inference/rollback.ts +0 -7
- package/src/jobs/index.ts +0 -31
- package/src/jobs/job-entry.ts +0 -107
- package/src/jobs/task-client.ts +0 -19
- package/src/jobs/types.ts +0 -2
- package/src/kv-scheme.ts +0 -257
- package/src/kv-store.ts +0 -1099
- package/src/lab/README.md +0 -18
- package/src/lab/curve/curve-designer.html +0 -303
- package/src/lab/curve/curve-sampler.ts +0 -122
- package/src/lab/expert-trace/expert-trace.ts +0 -69
- package/src/lab/paged-kv/paged-kv-rows.ts +0 -83
- package/src/lab/paged-kv/paged-kv.ts +0 -270
- package/src/library.ts +0 -40
- package/src/lora.ts +0 -408
- package/src/md-modules.d.ts +0 -49
- package/src/media-fetch.ts +0 -442
- package/src/memory/article.ts +0 -558
- package/src/memory/chunk-validation.ts +0 -150
- package/src/memory/chunk.ts +0 -318
- package/src/memory/cluster.ts +0 -351
- package/src/memory/coverage.ts +0 -134
- package/src/memory/crosslink.ts +0 -0
- package/src/memory/db.ts +0 -253
- package/src/memory/entity.ts +0 -203
- package/src/memory/gate.ts +0 -187
- package/src/memory/history.ts +0 -152
- package/src/memory/ingest.ts +0 -335
- package/src/memory/model.ts +0 -183
- package/src/memory/normalize.ts +0 -671
- package/src/memory/parse.ts +0 -22
- package/src/memory/pipeline.ts +0 -275
- package/src/memory/prompts.ts +0 -65
- package/src/memory/query.ts +0 -334
- package/src/memory/reconcile.ts +0 -734
- package/src/memory/redirect.ts +0 -108
- package/src/memory/reindex.ts +0 -325
- package/src/memory/resolve.ts +0 -307
- package/src/memory/rest.ts +0 -245
- package/src/memory/route.ts +0 -272
- package/src/memory/schedule.ts +0 -165
- package/src/memory/stages.ts +0 -777
- package/src/memory/synthesize.ts +0 -1638
- package/src/memory/tools.ts +0 -749
- package/src/memory/vault.ts +0 -1194
- package/src/memory/wikify.ts +0 -1064
- package/src/mlx/array.ts +0 -388
- package/src/mlx/autograd.ts +0 -304
- package/src/mlx/checkpoint.ts +0 -149
- package/src/mlx/compile.ts +0 -168
- package/src/mlx/custom-vjp.ts +0 -188
- package/src/mlx/ffi.ts +0 -512
- package/src/mlx/materialize.ts +0 -23
- package/src/mlx/metal-kernel.ts +0 -159
- package/src/mlx/ops.ts +0 -1021
- package/src/mlx/quantized-rows.ts +0 -36
- package/src/mlx/token-bitmask.ts +0 -52
- package/src/mlx/turboquant-kv-decode.ts +0 -82
- package/src/mlx/turboquant-kv-inverse.ts +0 -122
- package/src/mlx/turboquant-ops.ts +0 -861
- package/src/mlx/turboquant-tables.ts +0 -237
- package/src/mmap.ts +0 -90
- package/src/model/artifact-identity.ts +0 -73
- package/src/model/batched-kv.ts +0 -235
- package/src/model/batched-mask.ts +0 -279
- package/src/model/batched-quant.ts +0 -212
- package/src/model/batched-quantized-kv.ts +0 -144
- package/src/model/batched-rotating-quant.ts +0 -353
- package/src/model/batched-rotating-state.ts +0 -149
- package/src/model/batched-rotating.ts +0 -338
- package/src/model/batched-row-storage.ts +0 -179
- package/src/model/batched-ssm.ts +0 -52
- package/src/model/batched-turboquant-kv.ts +0 -93
- package/src/model/compiled-decode.ts +0 -712
- package/src/model/decoded-kv-donor.ts +0 -12
- package/src/model/delayed-quantized-kv.ts +0 -72
- package/src/model/delayed-rotating-quantized-kv.ts +0 -93
- package/src/model/delayed-turboquant-kv.ts +0 -55
- package/src/model/diffusion-gemma.ts +0 -1026
- package/src/model/factory.ts +0 -189
- package/src/model/fingerprint.ts +0 -33
- package/src/model/flash-attention.ts +0 -694
- package/src/model/full-prefill-padding.ts +0 -56
- package/src/model/full-prefill-row.ts +0 -105
- package/src/model/gemma4-base.ts +0 -2351
- package/src/model/gemma4.ts +0 -1315
- package/src/model/generated/gemma4-12b.ts +0 -429
- package/src/model/generated/gemma4-26b.ts +0 -448
- package/src/model/generated/gemma4-e4b.ts +0 -717
- package/src/model/generated/index.ts +0 -18
- package/src/model/glm52-atlas.ts +0 -511
- package/src/model/glm52-cache.ts +0 -792
- package/src/model/glm52-config.ts +0 -262
- package/src/model/glm52-container.ts +0 -305
- package/src/model/glm52-coupling.ts +0 -388
- package/src/model/glm52-dsa.ts +0 -606
- package/src/model/glm52-expert-layout.ts +0 -253
- package/src/model/glm52-layout.ts +0 -152
- package/src/model/glm52-memory.ts +0 -500
- package/src/model/glm52-mla.ts +0 -792
- package/src/model/glm52-moe.ts +0 -223
- package/src/model/glm52-pilot.ts +0 -537
- package/src/model/glm52-reference.ts +0 -445
- package/src/model/glm52-residency.ts +0 -557
- package/src/model/glm52-resident-weights.ts +0 -135
- package/src/model/glm52-streamed-experts.ts +0 -453
- package/src/model/glm52-streamed-metal.ts +0 -610
- package/src/model/glm52-weights.ts +0 -503
- package/src/model/glm52.ts +0 -1132
- package/src/model/implementation.ts +0 -69
- package/src/model/kv-attention-view.ts +0 -67
- package/src/model/minicpm5.ts +0 -376
- package/src/model/profile.ts +0 -443
- package/src/model/qwen-append.ts +0 -12
- package/src/model/qwen-conv.ts +0 -52
- package/src/model/qwen3-delta-state.ts +0 -74
- package/src/model/qwen3-delta.ts +0 -451
- package/src/model/qwen3-moe.ts +0 -404
- package/src/model/qwen3-mrope.ts +0 -252
- package/src/model/qwen3.ts +0 -277
- package/src/model/qwen3_5-checkpoint.ts +0 -164
- package/src/model/qwen3_5.ts +0 -819
- package/src/model/rotating-kv-donor.ts +0 -35
- package/src/model/rotating-kv-layout.ts +0 -262
- package/src/model/rotating-row-transaction.ts +0 -35
- package/src/model/speculative-rotating-kv.ts +0 -101
- package/src/model/ssm-prefill-padding.ts +0 -39
- package/src/model/support.ts +0 -102
- package/src/model/trellis-balanced-scatter.ts +0 -51
- package/src/model/trellis-linear.ts +0 -672
- package/src/model/trellis-shared-m.ts +0 -151
- package/src/model/trellis-shared-scatter.ts +0 -144
- package/src/model/trellis-splitk-prefill.ts +0 -158
- package/src/model/trellis-tiled-prefill.ts +0 -136
- package/src/model/trellis-vector-expand.ts +0 -62
- package/src/model/trellis-wide-prefill.ts +0 -94
- package/src/model/turboquant-codec.ts +0 -83
- package/src/model/universal/archs.ts +0 -332
- package/src/model/universal/dense.ts +0 -612
- package/src/model/universal/modules.ts +0 -170
- package/src/model/universal/rope.ts +0 -381
- package/src/native/expert_io.c +0 -523
- package/src/native/frame_extract.swift +0 -140
- package/src/native-pack.ts +0 -162
- package/src/pi-provider.ts +0 -53
- package/src/pi-session.ts +0 -81
- package/src/pi-terminal.ts +0 -274
- package/src/pi-web.ts +0 -1932
- package/src/preflight.ts +0 -158
- package/src/prompt-cache.ts +0 -460
- package/src/quantize/allocator.ts +0 -627
- package/src/quantize/atomic-output.ts +0 -38
- package/src/quantize/calibration.ts +0 -181
- package/src/quantize/config-writer.ts +0 -145
- package/src/quantize/index.ts +0 -84
- package/src/quantize/job.ts +0 -162
- package/src/quantize/quantizer.ts +0 -666
- package/src/quantize/rotate.ts +0 -603
- package/src/quantize/safetensors-writer.ts +0 -249
- package/src/quantize/sensitivity.ts +0 -216
- package/src/quantize/trellis.ts +0 -586
- package/src/quantize/weight-transform.ts +0 -292
- package/src/registry.ts +0 -602
- package/src/runtime-config.ts +0 -81
- package/src/safetensors.ts +0 -158
- package/src/sampler.ts +0 -1011
- package/src/serve/admin-routes.ts +0 -184
- package/src/serve/assets/curve-terrain.html +0 -105
- package/src/serve/assets/training-inference-map.html +0 -575
- package/src/serve/aux-routes.ts +0 -132
- package/src/serve/batch-scheduler.ts +0 -8
- package/src/serve/checkpoint-identity.ts +0 -64
- package/src/serve/generation-gateway.ts +0 -496
- package/src/serve/isolate.ts +0 -609
- package/src/serve/kv-budget.ts +0 -14
- package/src/serve/lab-routes.ts +0 -341
- package/src/serve/lane-registry.ts +0 -45
- package/src/serve/model-admin-routes.ts +0 -155
- package/src/serve/model-binding.ts +0 -43
- package/src/serve/model-host.ts +0 -511
- package/src/serve/parent-application.ts +0 -56
- package/src/serve/prompt-response-trace.ts +0 -147
- package/src/serve/response-lease.ts +0 -24
- package/src/serve/session-search.ts +0 -326
- package/src/serve/static-routes.ts +0 -101
- package/src/server.ts +0 -1314
- package/src/spec/assistant-rows.ts +0 -92
- package/src/spec/assistant-source.ts +0 -126
- package/src/spec/cache-identity.ts +0 -22
- package/src/spec/deepspec-rows.ts +0 -29
- package/src/spec/deepspec-source.ts +0 -145
- package/src/spec/dflash-source.ts +0 -149
- package/src/spec/drafter.ts +0 -304
- package/src/spec/dspark/ab-stats.ts +0 -136
- package/src/spec/dspark/calibration.ts +0 -112
- package/src/spec/dspark/data-dflash.ts +0 -156
- package/src/spec/dspark/data.ts +0 -201
- package/src/spec/dspark/deepspec-module.ts +0 -776
- package/src/spec/dspark/generate-dflash.ts +0 -245
- package/src/spec/dspark/generate.ts +0 -290
- package/src/spec/dspark/loader.ts +0 -23
- package/src/spec/dspark/loss.ts +0 -213
- package/src/spec/dspark/module-dflash.ts +0 -560
- package/src/spec/dspark/module.ts +0 -540
- package/src/spec/dspark/quantize-drafter.ts +0 -91
- package/src/spec/dspark/sample.ts +0 -112
- package/src/spec/generate.ts +0 -276
- package/src/spec/glm52-mtp-graph.ts +0 -29
- package/src/spec/glm52-mtp-rows.ts +0 -132
- package/src/spec/glm52-mtp-source.ts +0 -317
- package/src/spec/glm52-mtp-state.ts +0 -33
- package/src/spec/ngram-source.ts +0 -204
- package/src/spec/projected-draft-rows.ts +0 -73
- package/src/spec/qwen-mtp-module.ts +0 -171
- package/src/spec/qwen-mtp-rows.ts +0 -219
- package/src/spec/qwen-mtp-source.ts +0 -383
- package/src/spec/qwen-mtp-state.ts +0 -30
- package/src/spec/serve-loop.ts +0 -585
- package/src/spec/source.ts +0 -215
- package/src/spec/two-model-rows.ts +0 -61
- package/src/spec/two-model.ts +0 -113
- package/src/ssd-cache.ts +0 -372
- package/src/ssd-durability.ts +0 -211
- package/src/tokenizer.ts +0 -335
- package/src/tool-approvals.ts +0 -131
- package/src/tool-call.ts +0 -695
- package/src/train/dataset.ts +0 -484
- package/src/train/diffusion-lora.ts +0 -216
- package/src/train/export.ts +0 -32
- package/src/train/flash-cce.ts +0 -1234
- package/src/train/forward.ts +0 -266
- package/src/train/fuse.ts +0 -243
- package/src/train/index.ts +0 -46
- package/src/train/job.ts +0 -205
- package/src/train/lora-params.ts +0 -254
- package/src/train/loss.ts +0 -1591
- package/src/train/merge.ts +0 -195
- package/src/train/optimizer.ts +0 -183
- package/src/train/prefix-shared.ts +0 -490
- package/src/train/rank.ts +0 -137
- package/src/train/segmented.ts +0 -1828
- package/src/train/steel-qmm-header.ts +0 -1461
- package/src/train/trainer.ts +0 -1221
- package/src/train/watch.ts +0 -542
- package/src/vision/diffusion-vision.ts +0 -620
- package/src/vision/embedder.ts +0 -168
- package/src/vision/preprocess.ts +0 -247
- package/src/vision/prompt.ts +0 -373
- package/src/vision/qwen3vl-preprocess.ts +0 -409
- package/src/vision/qwen3vl-prompt.ts +0 -196
- package/src/vision/qwen3vl-tower.ts +0 -555
- package/src/vision/siglip.ts +0 -599
- package/src/vision/video-frames.ts +0 -148
- package/src/web/app.html +0 -2143
- package/src/web/app.js +0 -5703
- package/src/web/skills/memory/SKILL.md +0 -120
- package/src/web/skills/web-research/SKILL.md +0 -46
- package/src/web/skills.ts +0 -97
- package/src/web/src/adapters-panel.ts +0 -288
- package/src/web/src/assistant.ts +0 -451
- package/src/web/src/chat.ts +0 -1057
- package/src/web/src/composer.ts +0 -959
- package/src/web/src/dataset.ts +0 -132
- package/src/web/src/finetune.ts +0 -267
- package/src/web/src/hub.ts +0 -428
- package/src/web/src/main.ts +0 -98
- package/src/web/src/markdown.ts +0 -0
- package/src/web/src/memory-panel.ts +0 -572
- package/src/web/src/model-picker.ts +0 -159
- package/src/web/src/palette.ts +0 -346
- package/src/web/src/protocol.ts +0 -26
- package/src/web/src/quantize.ts +0 -167
- package/src/web/src/rag.ts +0 -274
- package/src/web/src/sessions.ts +0 -469
- package/src/web/src/shell.ts +0 -804
- package/src/web/src/ui-catalog.ts +0 -78
- package/src/web/vendor/README.md +0 -61
- package/src/web-assets.ts +0 -47
- package/src/web-tools.ts +0 -478
- package/src/weights.ts +0 -255
- /package/src/{serve → server}/completion-sink.ts +0 -0
- /package/src/web/{icon.svg → public/icon.svg} +0 -0
- /package/src/web/{manifest.webmanifest → public/manifest.webmanifest} +0 -0
- /package/src/web/{vendor → public/vendor}/hljs-LICENSE +0 -0
|
@@ -1,165 +0,0 @@
|
|
|
1
|
-
import type { MlxArray } from "../../mlx/array";
|
|
2
|
-
import * as ops from "../../mlx/ops";
|
|
3
|
-
import type { BatchableCache, Cache, Mask } from "../../model/gemma4-base";
|
|
4
|
-
import { KvTensorRows } from "./kv-tensor-rows";
|
|
5
|
-
import { leaseCacheStates, minimumReusableOffset } from "./state-views";
|
|
6
|
-
import { withResource, disposeResources } from "../../engine/resources";
|
|
7
|
-
import { cloneKvCaches } from "../../kv-store";
|
|
8
|
-
|
|
9
|
-
/** Encoded layouts retain physical row alignment across precision changes. */
|
|
10
|
-
export interface TransitionedKvLayout extends BatchableCache {
|
|
11
|
-
alignRows?(leftPad: readonly number[]): void;
|
|
12
|
-
}
|
|
13
|
-
export interface SpeculativeTransitionedKvLayout extends TransitionedKvLayout {
|
|
14
|
-
specRoundBegin(): void;
|
|
15
|
-
specRoundCommit(): void;
|
|
16
|
-
specRoundRollback(keep: number | readonly number[]): void;
|
|
17
|
-
}
|
|
18
|
-
export interface KvRowTransition<Layout extends TransitionedKvLayout> {
|
|
19
|
-
readonly signature: string;
|
|
20
|
-
maintain(rows: Cache[]): void;
|
|
21
|
-
converted(row: Cache): boolean;
|
|
22
|
-
makeLayout(): Layout;
|
|
23
|
-
packRows?(layout: Layout, rows: readonly Cache[]): void;
|
|
24
|
-
prepareRows?(rows: Cache[]): Cache[];
|
|
25
|
-
extractRow?(row: Cache): Cache;
|
|
26
|
-
rollbackRow?(row: Cache, before: number, keep: number, preserveColumns: boolean): void;
|
|
27
|
-
}
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
export interface TransitioningKvPositions {
|
|
31
|
-
readonly rowOffsets: readonly number[];
|
|
32
|
-
readonly leftPad: readonly number[];
|
|
33
|
-
readonly offset: number;
|
|
34
|
-
readonly batchSize: number | null;
|
|
35
|
-
readonly ropeOffsetArr: MlxArray | undefined;
|
|
36
|
-
sync(rows: readonly Cache[], preserve: boolean): void;
|
|
37
|
-
filterRows(keep: readonly number[]): void;
|
|
38
|
-
makeMask(tokens: number, window: number | null): Mask;
|
|
39
|
-
dispose(): void;
|
|
40
|
-
}
|
|
41
|
-
export class FullKvPositions extends KvTensorRows implements TransitioningKvPositions {
|
|
42
|
-
sync(rows: readonly Cache[], preserve: boolean): void {
|
|
43
|
-
this.mergeRows(preserve
|
|
44
|
-
? [{ planes: [], rowOffsets: rows.map(row => row.offset), leftPad: [...this.leftPad] }]
|
|
45
|
-
: rows.map(row => ({ planes: [], rowOffsets: [row.offset], leftPad: [0] })));
|
|
46
|
-
}
|
|
47
|
-
}
|
|
48
|
-
|
|
49
|
-
/** Owns membership, committed positions and precision transitions. Codec and
|
|
50
|
-
* attention adapters own tensor access; scheduling and persistence use the
|
|
51
|
-
* existing batch and checkpoint contracts. Row conversion never examines an
|
|
52
|
-
* uncommitted speculative suffix. */
|
|
53
|
-
export abstract class TransitioningKvRows<Layout extends TransitionedKvLayout> implements BatchableCache {
|
|
54
|
-
protected readonly positions: TransitioningKvPositions;
|
|
55
|
-
protected rows: Cache[] = [];
|
|
56
|
-
protected packed?: Layout;
|
|
57
|
-
#prefilling = false;
|
|
58
|
-
|
|
59
|
-
readonly stateNeedsDispose = true;
|
|
60
|
-
constructor(readonly transition: KvRowTransition<Layout>, row?: Cache, positions: TransitioningKvPositions = new FullKvPositions()) {
|
|
61
|
-
this.positions = positions;
|
|
62
|
-
if (row) { this.rows = transition.prepareRows?.([row]) ?? [row]; this.syncPositions(); }
|
|
63
|
-
}
|
|
64
|
-
signature(): string { return this.transition.signature; }
|
|
65
|
-
get prefillMaintenance() { return this; }
|
|
66
|
-
beginPrefill(): void { this.#prefilling = true; }
|
|
67
|
-
commitPrefill(indices: readonly number[]): void {
|
|
68
|
-
if (this.packed) return;
|
|
69
|
-
for (const index of indices) {
|
|
70
|
-
const row = [this.rows[index]!];
|
|
71
|
-
try { this.transition.maintain(row); }
|
|
72
|
-
finally { this.rows[index] = row[0]!; }
|
|
73
|
-
}
|
|
74
|
-
}
|
|
75
|
-
endPrefill(): void { this.#prefilling = false; }
|
|
76
|
-
get minimumReusableOffset(): number { return this.packed?.minimumReusableOffset ?? minimumReusableOffset(this.rows); }
|
|
77
|
-
get rowOffsets(): readonly number[] { return this.packed?.rowOffsets ?? this.positions.rowOffsets; }
|
|
78
|
-
get leftPad(): readonly number[] { return this.packed?.leftPad ?? this.positions.leftPad; }
|
|
79
|
-
get offset(): number { return this.packed?.offset ?? this.positions.offset; }
|
|
80
|
-
get batchSize(): number | null { return this.packed?.batchSize ?? this.positions.batchSize; }
|
|
81
|
-
get ropeOffsetArr(): MlxArray | undefined { return this.packed ? this.packed.ropeOffsetArr : this.positions.ropeOffsetArr; }
|
|
82
|
-
bytesPerToken(): number { return this.packed?.bytesPerToken?.() ?? Math.max(0, ...this.rows.map(row => row.bytesPerToken?.() ?? 0)); }
|
|
83
|
-
projectedBytes(tokens: number): number { return this.bytesPerToken() * tokens; }
|
|
84
|
-
makeMask(tokens: number, window: number | null): Mask { return (this.packed ?? this.positions).makeMask(tokens, window); }
|
|
85
|
-
isTrimmable(): boolean { return this.packed?.isTrimmable() ?? this.rows.every(row => row.isTrimmable()); }
|
|
86
|
-
trim(count: number): void {
|
|
87
|
-
if (this.packed) this.packed.trim(count);
|
|
88
|
-
else { for (const row of this.rows) row.trim(count); this.syncPositions(true); }
|
|
89
|
-
}
|
|
90
|
-
protected syncPositions(preserve = false): void { this.positions.sync(this.rows, preserve); }
|
|
91
|
-
protected advance(): void {
|
|
92
|
-
if (this.packed) return;
|
|
93
|
-
if (!this.#prefilling) this.transition.maintain(this.rows);
|
|
94
|
-
if (this.rows.length && this.rows.every(row => this.transition.converted(row))) {
|
|
95
|
-
const packed = this.transition.makeLayout();
|
|
96
|
-
try {
|
|
97
|
-
if (this.transition.packRows) this.transition.packRows(packed, this.rows);
|
|
98
|
-
else packed.mergeRows(this.rows);
|
|
99
|
-
packed.alignRows?.(this.positions.leftPad);
|
|
100
|
-
}
|
|
101
|
-
catch (error) { packed.dispose(); throw error; }
|
|
102
|
-
const previous = this.rows; this.rows = []; this.packed = packed;
|
|
103
|
-
this.positions.dispose(); disposeResources(previous);
|
|
104
|
-
}
|
|
105
|
-
}
|
|
106
|
-
abstract updateAndFetch(k: MlxArray, v: MlxArray): [MlxArray, MlxArray];
|
|
107
|
-
abstract makeEmptyBatch(): TransitioningKvRows<Layout>;
|
|
108
|
-
mergeRows(rows: readonly Cache[]): void {
|
|
109
|
-
const owned: Cache[] = [];
|
|
110
|
-
try {
|
|
111
|
-
for (const row of rows) {
|
|
112
|
-
if (row instanceof TransitioningKvRows) {
|
|
113
|
-
for (let index = 0; index < row.batchSize!; index++) owned.push(row.extractRow(index));
|
|
114
|
-
} else owned.push(...cloneKvCaches([row]));
|
|
115
|
-
}
|
|
116
|
-
} catch (error) { disposeResources(owned); throw error; }
|
|
117
|
-
let prepared: Cache[];
|
|
118
|
-
try { prepared = this.transition.prepareRows?.(owned) ?? owned; }
|
|
119
|
-
catch (error) { disposeResources(owned); throw error; }
|
|
120
|
-
this.dispose(); this.rows = prepared; this.syncPositions();
|
|
121
|
-
}
|
|
122
|
-
filterRows(keep: readonly number[]): void {
|
|
123
|
-
if (this.packed) { this.packed.filterRows(keep); return; }
|
|
124
|
-
const previous = this.rows; this.rows = keep.map(index => previous[index]!);
|
|
125
|
-
disposeResources(previous.filter((_, index) => !keep.includes(index))); this.positions.filterRows(keep);
|
|
126
|
-
}
|
|
127
|
-
extractRow(row: number): Cache { return this.packed?.extractRow(row) ?? (this.transition.extractRow?.(this.rows[row]!) ?? cloneKvCaches([this.rows[row]!])[0]!); }
|
|
128
|
-
state(): MlxArray[] {
|
|
129
|
-
return withResource(leaseCacheStates(this.packed ? [this.packed] : this.rows), arrays => {
|
|
130
|
-
const views: MlxArray[] = [];
|
|
131
|
-
try { for (const array of arrays) views.push(ops.contiguous(array)); return views; }
|
|
132
|
-
catch (error) { disposeResources(views); throw error; }
|
|
133
|
-
});
|
|
134
|
-
}
|
|
135
|
-
dispose(): void {
|
|
136
|
-
this.#prefilling = false;
|
|
137
|
-
const previous = this.rows; this.rows = [];
|
|
138
|
-
this.packed?.dispose(); this.packed = undefined; this.positions.dispose(); disposeResources(previous);
|
|
139
|
-
}
|
|
140
|
-
}
|
|
141
|
-
|
|
142
|
-
/** Rollback is a method capability, separate from ordinary cache membership. */
|
|
143
|
-
export abstract class SpeculativeTransitioningKvRows<Layout extends SpeculativeTransitionedKvLayout> extends TransitioningKvRows<Layout> {
|
|
144
|
-
#beforeRound?: number[];
|
|
145
|
-
specRoundBegin(): void {
|
|
146
|
-
this.advance(); this.#beforeRound = [...this.rowOffsets]; this.packed?.specRoundBegin();
|
|
147
|
-
}
|
|
148
|
-
specRoundCommit(): void {
|
|
149
|
-
this.packed?.specRoundCommit(); this.#beforeRound = undefined; this.advance();
|
|
150
|
-
}
|
|
151
|
-
specRoundRollback(keep: number | readonly number[]): void {
|
|
152
|
-
if (this.packed) this.packed.specRoundRollback(keep);
|
|
153
|
-
else {
|
|
154
|
-
for (const [index, row] of this.rows.entries()) {
|
|
155
|
-
const count = typeof keep === "number" ? keep : keep[index]!;
|
|
156
|
-
const before = this.#beforeRound![index]!;
|
|
157
|
-
if (this.transition.rollbackRow) this.transition.rollbackRow(row, before, count, this.rows.length > 1);
|
|
158
|
-
else row.trim(row.offset - before - count);
|
|
159
|
-
}
|
|
160
|
-
this.syncPositions(true);
|
|
161
|
-
}
|
|
162
|
-
this.#beforeRound = undefined; this.advance();
|
|
163
|
-
}
|
|
164
|
-
override dispose(): void { this.#beforeRound = undefined; super.dispose(); }
|
|
165
|
-
}
|
package/src/chat-template.ts
DELETED
|
@@ -1,382 +0,0 @@
|
|
|
1
|
-
// Chat template rendering via @huggingface/jinja (pure JS, purpose-built
|
|
2
|
-
// for HF chat templates). Decision recorded in PLAN.md Phase 1 findings:
|
|
3
|
-
// rendering the model's own chat_template.jinja beats a hand-port because
|
|
4
|
-
// it can't rot when the model updates its template.
|
|
5
|
-
|
|
6
|
-
import { Template } from "@huggingface/jinja";
|
|
7
|
-
|
|
8
|
-
export interface ToolCall {
|
|
9
|
-
id?: string;
|
|
10
|
-
type?: "function";
|
|
11
|
-
function: {
|
|
12
|
-
name: string;
|
|
13
|
-
/** Object form preferred; OpenAI's JSON-string form is normalized by
|
|
14
|
-
* the server before rendering. */
|
|
15
|
-
arguments: Record<string, unknown> | string;
|
|
16
|
-
};
|
|
17
|
-
}
|
|
18
|
-
|
|
19
|
-
export interface ChatMessage {
|
|
20
|
-
role: string;
|
|
21
|
-
content?: string | Array<Record<string, unknown>> | null;
|
|
22
|
-
tool_calls?: ToolCall[];
|
|
23
|
-
tool_call_id?: string;
|
|
24
|
-
name?: string;
|
|
25
|
-
reasoning?: string;
|
|
26
|
-
reasoning_content?: string;
|
|
27
|
-
}
|
|
28
|
-
|
|
29
|
-
/** OpenAI-style tool definition (function type). */
|
|
30
|
-
export interface ToolDefinition {
|
|
31
|
-
type: "function";
|
|
32
|
-
function: {
|
|
33
|
-
name: string;
|
|
34
|
-
description?: string;
|
|
35
|
-
parameters?: Record<string, unknown>;
|
|
36
|
-
};
|
|
37
|
-
}
|
|
38
|
-
|
|
39
|
-
export interface RenderOptions {
|
|
40
|
-
addGenerationPrompt?: boolean;
|
|
41
|
-
tools?: ToolDefinition[] | null;
|
|
42
|
-
enableThinking?: boolean;
|
|
43
|
-
/** Template-level reasoning depth (Qwen3.8: xhigh|medium|low — the template
|
|
44
|
-
* RAISES on other values, so callers must map OpenAI levels first and only
|
|
45
|
-
* set this for templates that read it; see ChatTemplate.readsReasoningEffort). */
|
|
46
|
-
reasoningEffort?: "xhigh" | "medium" | "low";
|
|
47
|
-
/** Keep think blocks from historical assistant turns in the rendered prompt
|
|
48
|
-
* (Qwen3.8 `preserve_thinking`, template default true — better prompt-cache
|
|
49
|
-
* reuse and agent-trace continuity). Only passed when the template reads it. */
|
|
50
|
-
preserveThinking?: boolean;
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
type ChatRenderer = (
|
|
54
|
-
messages: ChatMessage[],
|
|
55
|
-
options: RenderOptions,
|
|
56
|
-
) => string;
|
|
57
|
-
|
|
58
|
-
function glm52ContentText(content: ChatMessage["content"], label: string): string {
|
|
59
|
-
if (typeof content === "string") return content;
|
|
60
|
-
if (!Array.isArray(content))
|
|
61
|
-
throw new Error(`${label} must be a string or an array of text parts`);
|
|
62
|
-
return content.map((part, index) => {
|
|
63
|
-
if (
|
|
64
|
-
!part ||
|
|
65
|
-
(part.type !== "text" && part.type !== "input_text") ||
|
|
66
|
-
typeof part.text !== "string"
|
|
67
|
-
) {
|
|
68
|
-
throw new Error(`${label}.${index}: GLM-5.2 supports text content only`);
|
|
69
|
-
}
|
|
70
|
-
return part.text;
|
|
71
|
-
}).join("");
|
|
72
|
-
}
|
|
73
|
-
|
|
74
|
-
/** Text/tool subset of the pinned GLM-5.2 template used by Colibri. */
|
|
75
|
-
export function renderGlm52Chat(
|
|
76
|
-
messages: ChatMessage[],
|
|
77
|
-
options: RenderOptions = {},
|
|
78
|
-
): string {
|
|
79
|
-
if (!Array.isArray(messages) || messages.length === 0)
|
|
80
|
-
throw new Error("GLM-5.2 messages must be a non-empty array");
|
|
81
|
-
const {
|
|
82
|
-
addGenerationPrompt = true,
|
|
83
|
-
tools = null,
|
|
84
|
-
enableThinking = false,
|
|
85
|
-
} = options;
|
|
86
|
-
const prompt = ["[gMASK]<sop>"];
|
|
87
|
-
if (enableThinking)
|
|
88
|
-
prompt.push("<|system|>Reasoning Effort: Max");
|
|
89
|
-
|
|
90
|
-
const normalizedTools = normalizeToolSchemas(tools);
|
|
91
|
-
if (normalizedTools && normalizedTools.length > 0) {
|
|
92
|
-
prompt.push(
|
|
93
|
-
"<|system|>\n# Tools\n\nYou may call one or more functions to assist with the " +
|
|
94
|
-
"user query.\n\nYou are provided with function signatures within <tools></tools> " +
|
|
95
|
-
"XML tags:\n<tools>\n",
|
|
96
|
-
);
|
|
97
|
-
for (const tool of normalizedTools) {
|
|
98
|
-
const fn = { ...tool.function } as Record<string, unknown>;
|
|
99
|
-
delete fn.defer_loading;
|
|
100
|
-
delete fn.strict;
|
|
101
|
-
prompt.push(`${JSON.stringify(fn)}\n`);
|
|
102
|
-
}
|
|
103
|
-
prompt.push(
|
|
104
|
-
"</tools>\n\nFor each function call, output the function name and arguments " +
|
|
105
|
-
"within the following XML format:\n<tool_call>{function-name}" +
|
|
106
|
-
"<arg_key>{arg-key-1}</arg_key><arg_value>{arg-value-1}</arg_value>" +
|
|
107
|
-
"<arg_key>{arg-key-2}</arg_key><arg_value>{arg-value-2}</arg_value>...</tool_call>",
|
|
108
|
-
);
|
|
109
|
-
}
|
|
110
|
-
|
|
111
|
-
let previousWasTool = false;
|
|
112
|
-
for (let index = 0; index < messages.length; index++) {
|
|
113
|
-
const message = messages[index]!;
|
|
114
|
-
const role = message.role;
|
|
115
|
-
if (role === "system" || role === "developer") {
|
|
116
|
-
prompt.push(
|
|
117
|
-
`<|system|>${glm52ContentText(message.content, `messages.${index}.content`)}`,
|
|
118
|
-
);
|
|
119
|
-
} else if (role === "user") {
|
|
120
|
-
prompt.push(
|
|
121
|
-
`<|user|>${glm52ContentText(message.content, `messages.${index}.content`)}`,
|
|
122
|
-
);
|
|
123
|
-
} else if (role === "assistant") {
|
|
124
|
-
const content = message.content === null || message.content === undefined
|
|
125
|
-
? ""
|
|
126
|
-
: glm52ContentText(message.content, `messages.${index}.content`);
|
|
127
|
-
prompt.push(`<|assistant|><think></think>${content.trim()}`);
|
|
128
|
-
for (const call of message.tool_calls ?? []) {
|
|
129
|
-
const raw = call.function.arguments;
|
|
130
|
-
let args: Record<string, unknown> = {};
|
|
131
|
-
if (typeof raw === "string") {
|
|
132
|
-
try {
|
|
133
|
-
const parsed = JSON.parse(raw);
|
|
134
|
-
if (parsed && typeof parsed === "object" && !Array.isArray(parsed))
|
|
135
|
-
args = parsed as Record<string, unknown>;
|
|
136
|
-
} catch {
|
|
137
|
-
// Match the pinned template path: malformed historical arguments
|
|
138
|
-
// contribute an empty call body rather than changing the prompt.
|
|
139
|
-
}
|
|
140
|
-
} else {
|
|
141
|
-
args = raw;
|
|
142
|
-
}
|
|
143
|
-
prompt.push(`<tool_call>${call.function.name}`);
|
|
144
|
-
for (const [key, value] of Object.entries(args)) {
|
|
145
|
-
prompt.push(
|
|
146
|
-
`<arg_key>${key}</arg_key><arg_value>` +
|
|
147
|
-
`${typeof value === "string" ? value : JSON.stringify(value)}</arg_value>`,
|
|
148
|
-
);
|
|
149
|
-
}
|
|
150
|
-
prompt.push("</tool_call>");
|
|
151
|
-
}
|
|
152
|
-
} else if (role === "tool") {
|
|
153
|
-
if (!previousWasTool) prompt.push("<|observation|>");
|
|
154
|
-
prompt.push(
|
|
155
|
-
`<tool_response>${glm52ContentText(message.content, `messages.${index}.content`)}` +
|
|
156
|
-
"</tool_response>",
|
|
157
|
-
);
|
|
158
|
-
} else {
|
|
159
|
-
throw new Error(`GLM-5.2 unsupported message role ${JSON.stringify(role)}`);
|
|
160
|
-
}
|
|
161
|
-
previousWasTool = role === "tool";
|
|
162
|
-
}
|
|
163
|
-
if (addGenerationPrompt) {
|
|
164
|
-
prompt.push(
|
|
165
|
-
enableThinking
|
|
166
|
-
? "<|assistant|><think>"
|
|
167
|
-
: "<|assistant|><think></think>",
|
|
168
|
-
);
|
|
169
|
-
}
|
|
170
|
-
return prompt.join("");
|
|
171
|
-
}
|
|
172
|
-
|
|
173
|
-
/** JSON-schema type implied by a literal value. */
|
|
174
|
-
function jsonTypeOf(v: unknown): string {
|
|
175
|
-
if (typeof v === "number") return Number.isInteger(v) ? "integer" : "number";
|
|
176
|
-
if (typeof v === "boolean") return "boolean";
|
|
177
|
-
if (Array.isArray(v)) return "array";
|
|
178
|
-
if (v !== null && typeof v === "object") return "object";
|
|
179
|
-
return "string";
|
|
180
|
-
}
|
|
181
|
-
|
|
182
|
-
/** Infer a JSON-schema `type` for a node that lacks one (enum/const/anyOf/…). */
|
|
183
|
-
function inferSchemaType(node: Record<string, unknown>): string {
|
|
184
|
-
if (Array.isArray(node.enum) && node.enum.length > 0) return jsonTypeOf(node.enum[0]);
|
|
185
|
-
if ("const" in node) return jsonTypeOf(node.const);
|
|
186
|
-
if (node.properties || node.additionalProperties) return "object";
|
|
187
|
-
if (node.items !== undefined || node.prefixItems !== undefined) return "array";
|
|
188
|
-
for (const key of ["anyOf", "oneOf", "allOf"] as const) {
|
|
189
|
-
const subs = node[key];
|
|
190
|
-
if (Array.isArray(subs) && subs.length > 0) {
|
|
191
|
-
const types = subs
|
|
192
|
-
.map((s) => (s && typeof s === "object" ? (s as Record<string, unknown>).type : undefined))
|
|
193
|
-
.filter((t): t is string => typeof t === "string");
|
|
194
|
-
const first = types[0];
|
|
195
|
-
if (first !== undefined) return first;
|
|
196
|
-
}
|
|
197
|
-
}
|
|
198
|
-
return "string";
|
|
199
|
-
}
|
|
200
|
-
|
|
201
|
-
/**
|
|
202
|
-
* Recursively ensure every JSON-schema node carries a `type`.
|
|
203
|
-
*
|
|
204
|
-
* Why: HF chat templates (e.g. Gemma's tool-declaration block) do
|
|
205
|
-
* `value['type'] | upper`, which throws "Cannot apply filter upper to
|
|
206
|
-
* UndefinedValue" for schemas that describe a parameter via `anyOf`/`enum`/
|
|
207
|
-
* `const` without a top-level `type` (TypeBox unions/literals emit exactly
|
|
208
|
-
* this). We synthesize a sensible `type` so any tool renders, regardless of
|
|
209
|
-
* how its schema was authored. Returns a new object; never mutates input.
|
|
210
|
-
*/
|
|
211
|
-
export function normalizeSchemaTypes(node: unknown): unknown {
|
|
212
|
-
if (Array.isArray(node)) return node.map(normalizeSchemaTypes);
|
|
213
|
-
if (!node || typeof node !== "object") return node;
|
|
214
|
-
const out: Record<string, unknown> = { ...(node as Record<string, unknown>) };
|
|
215
|
-
if (out.properties && typeof out.properties === "object") {
|
|
216
|
-
const props: Record<string, unknown> = {};
|
|
217
|
-
for (const [k, v] of Object.entries(out.properties as Record<string, unknown>)) props[k] = normalizeSchemaTypes(v);
|
|
218
|
-
out.properties = props;
|
|
219
|
-
}
|
|
220
|
-
if (out.items !== undefined) out.items = normalizeSchemaTypes(out.items);
|
|
221
|
-
if (out.prefixItems !== undefined) out.prefixItems = normalizeSchemaTypes(out.prefixItems);
|
|
222
|
-
if (out.additionalProperties && typeof out.additionalProperties === "object") {
|
|
223
|
-
out.additionalProperties = normalizeSchemaTypes(out.additionalProperties);
|
|
224
|
-
}
|
|
225
|
-
for (const key of ["anyOf", "oneOf", "allOf"] as const) {
|
|
226
|
-
if (Array.isArray(out[key])) out[key] = (out[key] as unknown[]).map(normalizeSchemaTypes);
|
|
227
|
-
}
|
|
228
|
-
if (typeof out.type !== "string") out.type = inferSchemaType(out);
|
|
229
|
-
return out;
|
|
230
|
-
}
|
|
231
|
-
|
|
232
|
-
/** Apply normalizeSchemaTypes to every tool's parameter schema. */
|
|
233
|
-
export function normalizeToolSchemas(tools: ToolDefinition[] | null): ToolDefinition[] | null {
|
|
234
|
-
if (!tools) return tools;
|
|
235
|
-
return tools.map((t) => ({
|
|
236
|
-
...t,
|
|
237
|
-
function: {
|
|
238
|
-
...t.function,
|
|
239
|
-
parameters: t.function.parameters
|
|
240
|
-
? (normalizeSchemaTypes(t.function.parameters) as Record<string, unknown>)
|
|
241
|
-
: t.function.parameters,
|
|
242
|
-
},
|
|
243
|
-
}));
|
|
244
|
-
}
|
|
245
|
-
|
|
246
|
-
export class ChatTemplate {
|
|
247
|
-
readonly #template: Template | null;
|
|
248
|
-
readonly #renderer: ChatRenderer | null;
|
|
249
|
-
readonly #bosToken: string | null;
|
|
250
|
-
readonly #eosToken: string | null;
|
|
251
|
-
/** True when the template reads `reasoning_effort` (Qwen3.8's xhigh/medium/
|
|
252
|
-
* low depth control). Callers must not pass the variable otherwise — and
|
|
253
|
-
* must never pass unmapped OpenAI level names; this template family raises
|
|
254
|
-
* on values outside its supported set. */
|
|
255
|
-
readonly readsReasoningEffort: boolean;
|
|
256
|
-
/** True when the template reads `preserve_thinking` (Qwen3.8: keep think
|
|
257
|
-
* blocks from historical turns; template default true). */
|
|
258
|
-
readonly readsPreserveThinking: boolean;
|
|
259
|
-
/** Which reasoning format the template's `enable_thinking` channel uses, or
|
|
260
|
-
* null if the model has no switchable reasoning:
|
|
261
|
-
* - "think-tag": `<think>…</think>` text markers (Qwen3.5, MiniCPM5) — split
|
|
262
|
-
* from decoded text by ThinkingTagSplitter.
|
|
263
|
-
* - "gemma-channel": `<|channel>thought\n…<channel|>` (Gemma 4) — the markers
|
|
264
|
-
* are SPECIAL TOKENS stripped at decode, so reasoning is split at the token
|
|
265
|
-
* level in ToolAwareStream (gemma-sentinel mode), not from decoded text.
|
|
266
|
-
* Drives both the parser path and the reasoning capability advertised to Pi. */
|
|
267
|
-
readonly thinkingFormat: "think-tag" | "gemma-channel" | null;
|
|
268
|
-
/** True when the model has a switchable reasoning channel we can parse. */
|
|
269
|
-
readonly supportsThinking: boolean;
|
|
270
|
-
|
|
271
|
-
private constructor(
|
|
272
|
-
source: string | null,
|
|
273
|
-
bosToken: string | null,
|
|
274
|
-
eosToken: string | null,
|
|
275
|
-
forceNoThinking = false,
|
|
276
|
-
renderer: ChatRenderer | null = null,
|
|
277
|
-
explicitThinkingFormat: "think-tag" | "gemma-channel" | null = null,
|
|
278
|
-
) {
|
|
279
|
-
if (source === null && renderer === null)
|
|
280
|
-
throw new Error("ChatTemplate needs a Jinja source or renderer");
|
|
281
|
-
this.#template = source === null ? null : new Template(source);
|
|
282
|
-
this.#renderer = renderer;
|
|
283
|
-
this.#bosToken = bosToken;
|
|
284
|
-
this.#eosToken = eosToken;
|
|
285
|
-
const gatesThinking = source?.includes("enable_thinking") ?? false;
|
|
286
|
-
this.readsReasoningEffort = source?.includes("reasoning_effort") ?? false;
|
|
287
|
-
this.readsPreserveThinking = source?.includes("preserve_thinking") ?? false;
|
|
288
|
-
// `forceNoThinking` suppresses the switchable channel even when the template
|
|
289
|
-
// carries the gemma markers. DiffusionGemma ships the shared gemma-family
|
|
290
|
-
// template (with the `<|channel>thought…<channel|>` reasoning channel), but
|
|
291
|
-
// its non-autoregressive canvas decode cannot reliably emit the `<channel|>`
|
|
292
|
-
// close, so an enabled channel never ends (everything is captured as
|
|
293
|
-
// reasoning, no answer). The OptiQ reference never enables thinking for this
|
|
294
|
-
// model — it always renders the template's `default(false)` pre-closed empty
|
|
295
|
-
// channel and decodes the whole canvas as plain text. We match that.
|
|
296
|
-
this.thinkingFormat = explicitThinkingFormat ?? (forceNoThinking
|
|
297
|
-
? null
|
|
298
|
-
: gatesThinking && source!.includes("<think>")
|
|
299
|
-
? "think-tag"
|
|
300
|
-
: gatesThinking && source!.includes("<|channel>")
|
|
301
|
-
? "gemma-channel"
|
|
302
|
-
: null);
|
|
303
|
-
this.supportsThinking = this.thinkingFormat !== null;
|
|
304
|
-
}
|
|
305
|
-
|
|
306
|
-
static async load(
|
|
307
|
-
modelDir: string,
|
|
308
|
-
opts: { disableThinking?: boolean } = {},
|
|
309
|
-
): Promise<ChatTemplate> {
|
|
310
|
-
const config = (await Bun.file(`${modelDir}/tokenizer_config.json`).json()) as Record<string, any>;
|
|
311
|
-
let source: string | undefined = config.chat_template;
|
|
312
|
-
if (!source) {
|
|
313
|
-
const jinjaFile = Bun.file(`${modelDir}/chat_template.jinja`);
|
|
314
|
-
if (await jinjaFile.exists()) source = await jinjaFile.text();
|
|
315
|
-
}
|
|
316
|
-
if (!source) {
|
|
317
|
-
const modelConfigFile = Bun.file(`${modelDir}/config.json`);
|
|
318
|
-
const modelConfig = await modelConfigFile.exists()
|
|
319
|
-
? await modelConfigFile.json() as Record<string, unknown>
|
|
320
|
-
: null;
|
|
321
|
-
if (modelConfig?.model_type === "glm_moe_dsa") {
|
|
322
|
-
const tokenText = (t: unknown): string | null =>
|
|
323
|
-
t == null ? null : typeof t === "string" ? t : (t as any).content;
|
|
324
|
-
return new ChatTemplate(
|
|
325
|
-
null,
|
|
326
|
-
tokenText(config.bos_token),
|
|
327
|
-
tokenText(config.eos_token),
|
|
328
|
-
false,
|
|
329
|
-
renderGlm52Chat,
|
|
330
|
-
"think-tag",
|
|
331
|
-
);
|
|
332
|
-
}
|
|
333
|
-
throw new Error(`${modelDir}: no chat template found`);
|
|
334
|
-
}
|
|
335
|
-
// @huggingface/jinja lacks the `min`/`max` array filters that real
|
|
336
|
-
// Jinja2 has. MiniCPM5's template uses `[a, b]|min` in its assistant
|
|
337
|
-
// tool-call history branch, so without this rewrite every multi-turn
|
|
338
|
-
// tool conversation fails at render time ("Unknown ArrayValue filter").
|
|
339
|
-
source = source
|
|
340
|
-
.replace(
|
|
341
|
-
/\[\s*([\w.]+)\s*,\s*([\w.]+)\s*\]\s*\|\s*min\b/g,
|
|
342
|
-
"($1 if $1 < $2 else $2)",
|
|
343
|
-
)
|
|
344
|
-
.replace(
|
|
345
|
-
/\[\s*([\w.]+)\s*,\s*([\w.]+)\s*\]\s*\|\s*max\b/g,
|
|
346
|
-
"($1 if $1 > $2 else $2)",
|
|
347
|
-
);
|
|
348
|
-
const tokenText = (t: unknown): string | null =>
|
|
349
|
-
t == null ? null : typeof t === "string" ? t : (t as any).content;
|
|
350
|
-
return new ChatTemplate(source, tokenText(config.bos_token), tokenText(config.eos_token));
|
|
351
|
-
}
|
|
352
|
-
|
|
353
|
-
render(messages: ChatMessage[], options: RenderOptions = {}): string {
|
|
354
|
-
if (this.#renderer) return this.#renderer(messages, options);
|
|
355
|
-
const {
|
|
356
|
-
addGenerationPrompt = true,
|
|
357
|
-
tools = null,
|
|
358
|
-
enableThinking,
|
|
359
|
-
reasoningEffort,
|
|
360
|
-
preserveThinking,
|
|
361
|
-
} = options;
|
|
362
|
-
return this.#template!.render({
|
|
363
|
-
messages,
|
|
364
|
-
add_generation_prompt: addGenerationPrompt,
|
|
365
|
-
// Guarantee every tool param schema has a `type` so templates that do
|
|
366
|
-
// `value['type'] | upper` (Gemma) never see UndefinedValue.
|
|
367
|
-
tools: normalizeToolSchemas(tools),
|
|
368
|
-
...(enableThinking !== undefined ? { enable_thinking: enableThinking } : {}),
|
|
369
|
-
// Only set for templates that read them: reasoning_effort values are
|
|
370
|
-
// template-validated (Qwen3.8 raises on unknown levels), and an unread
|
|
371
|
-
// variable in other templates is dead context at best.
|
|
372
|
-
...(reasoningEffort !== undefined && this.readsReasoningEffort
|
|
373
|
-
? { reasoning_effort: reasoningEffort }
|
|
374
|
-
: {}),
|
|
375
|
-
...(preserveThinking !== undefined && this.readsPreserveThinking
|
|
376
|
-
? { preserve_thinking: preserveThinking }
|
|
377
|
-
: {}),
|
|
378
|
-
bos_token: this.#bosToken,
|
|
379
|
-
eos_token: this.#eosToken,
|
|
380
|
-
});
|
|
381
|
-
}
|
|
382
|
-
}
|