mlx-bun 0.5.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1449 -109
- package/THIRD_PARTY_NOTICES.md +882 -0
- package/bin/mlx-bun.convert.mjs +4 -0
- package/bin/mlx-bun.fuse.mjs +4 -0
- package/bin/mlx-bun.generate.mjs +4 -0
- package/bin/mlx-bun.lora.mjs +4 -0
- package/bin/mlx-bun.mjs +13 -21
- package/bin/mlx-bun.server.mjs +4 -0
- package/bin/mlx-bun.upload.mjs +4 -0
- package/dist/web/app.js +9584 -0
- package/package.json +50 -53
- package/src/cli/ane-install.ts +76 -0
- package/src/cli/args.ts +179 -0
- package/src/cli/engine-entry.ts +68 -0
- package/src/cli/inference.ts +214 -0
- package/src/cli/job-entry.ts +97 -0
- package/src/cli/library-host.ts +117 -0
- package/src/cli/main.ts +57 -0
- package/src/cli/memory-engine.ts +163 -0
- package/src/cli/memory.ts +36 -0
- package/src/cli/mlx-lm-aliases.ts +309 -0
- package/src/cli/model-choice.ts +33 -0
- package/src/cli/model-selection.ts +84 -0
- package/src/cli/module-host.ts +11 -0
- package/src/cli/module-verbs.ts +50 -0
- package/src/cli/numerical-policy.ts +12 -0
- package/src/cli/perplexity.ts +89 -0
- package/src/cli/serve-host.ts +423 -0
- package/src/cli/serve-isolated.ts +217 -0
- package/src/cli/serve-options.ts +106 -0
- package/src/cli/serve-state.ts +205 -0
- package/src/cli/serve.ts +500 -0
- package/src/cli/served-model-host.ts +102 -0
- package/src/cli/server-entry.ts +148 -0
- package/src/cli/serving-unit.ts +177 -0
- package/src/{tui.ts → cli/terminal.ts} +13 -8
- package/src/cli/worker-adapters.ts +34 -0
- package/src/cli/worker-entry.ts +272 -0
- package/src/cli/worker-unit.ts +179 -0
- package/src/engine/adapter-operation.ts +26 -0
- package/src/engine/cache-services.ts +114 -0
- package/src/engine/completion.ts +39 -0
- package/src/engine/generation-gateway.ts +358 -0
- package/src/engine/index.ts +59 -0
- package/src/engine/kv-budget.ts +120 -0
- package/src/engine/media-preparation.ts +232 -0
- package/src/engine/model-binding.ts +24 -0
- package/src/engine/model-host.ts +513 -0
- package/src/engine/model-serving.ts +89 -0
- package/src/{serve → engine}/preparation.ts +7 -5
- package/src/{serve → engine}/session-completion-engine.ts +6 -6
- package/src/engine/telemetry.ts +109 -0
- package/src/hub/downloads.ts +116 -0
- package/src/jobs/db.ts +16 -7
- package/src/jobs/events.ts +29 -0
- package/src/jobs/executable.ts +5 -0
- package/src/jobs/host.ts +51 -0
- package/src/jobs/in-process.ts +55 -0
- package/src/{contracts/jobs.ts → jobs/protocol.ts} +7 -9
- package/src/jobs/runner.ts +150 -231
- package/src/jobs/service.ts +75 -0
- package/src/jobs/sse.ts +59 -23
- package/src/jobs/worker-memory.ts +46 -0
- package/src/jobs/worker-process.ts +163 -0
- package/src/jobs/worker-supervisor.ts +258 -0
- package/src/modules.ts +58 -0
- package/src/publishing/catalog-hub.ts +18 -0
- package/src/publishing/credentials.ts +31 -0
- package/src/publishing/upload.ts +37 -0
- package/src/residency/leased-adapters.ts +21 -0
- package/src/residency/local-records.ts +16 -0
- package/src/residency/model-residency.ts +405 -0
- package/src/residency/record-index.ts +28 -0
- package/src/residency/resident-estimate.ts +50 -0
- package/src/{anthropic.ts → server/anthropic.ts} +3 -81
- package/src/server/cache-routes.ts +36 -0
- package/src/{serve → server}/chat-request.ts +10 -10
- package/src/{serve → server}/chat-stage.ts +14 -14
- package/src/{client.ts → server/client.ts} +28 -2
- package/src/{serve → server}/completion-executor.ts +33 -41
- package/src/server/decision-body.ts +55 -0
- package/src/server/decision-routes.ts +134 -0
- package/src/server/decision-wire.ts +228 -0
- package/src/{serve → server}/discovery-routes.ts +25 -82
- package/src/{serve → server}/http.ts +25 -13
- package/src/{serve → server}/inference-request.ts +5 -5
- package/src/server/job-routes.ts +20 -0
- package/src/server/media-prompt.ts +63 -0
- package/src/server/memory-completion-client.ts +46 -0
- package/src/server/model-routes.ts +192 -0
- package/src/{serve → server}/openai-wire.ts +3 -2
- package/src/{serve → server}/pipeline.ts +1 -1
- package/src/server/prompt-builder.ts +13 -0
- package/src/server/prompt-contracts.ts +17 -0
- package/src/server/proxy-routes.ts +187 -0
- package/src/server/publishing-routes.ts +41 -0
- package/src/{serve → server}/request-plan.ts +6 -8
- package/src/{serve → server}/request-prep.ts +47 -50
- package/src/{serve → server}/responses-client.ts +30 -13
- package/src/{responses.ts → server/responses.ts} +12 -93
- package/src/server/routes.ts +205 -0
- package/src/server/start.ts +72 -0
- package/src/server/status-routes.ts +108 -0
- package/src/{serve → server}/text-completion-stage.ts +25 -4
- package/src/server/text-prompt.ts +12 -0
- package/src/{serve → server}/token-streams.ts +36 -38
- package/src/server/worker-routes.ts +311 -0
- package/src/storage/paths.ts +61 -0
- package/src/web/assets.ts +37 -0
- package/src/web/{src → browser}/api.ts +5 -23
- package/src/web/browser/chat-host.ts +17 -0
- package/src/web/browser/installed-panels.ts +6 -0
- package/src/web/browser/main.ts +55 -0
- package/src/web/browser/markdown.ts +279 -0
- package/src/web/browser/memory-panel.ts +9 -0
- package/src/web/browser/palette.ts +64 -0
- package/src/web/browser/protocol.ts +13 -0
- package/src/web/browser/shell.ts +467 -0
- package/src/web/{src → browser}/status.ts +32 -36
- package/src/web/build.ts +51 -0
- package/src/web/public/app.html +1054 -0
- package/src/web/{sw.js → public/sw.js} +4 -3
- package/src/web/{vendor → public/vendor}/hljs-theme.css +2 -2
- package/src/web/{vendor → public/vendor}/hljs.js +2 -2
- package/THIRD_PARTY_LICENSES.md +0 -51
- package/docs/README.md +0 -107
- package/docs/archive/hlg-sampling.md +0 -350
- package/docs/archive/investigations/audio-input-plan.md +0 -356
- package/docs/archive/investigations/benchmark-ledger-through-v0.4.0.md +0 -3617
- package/docs/archive/investigations/chunk-finetune-experiment.md +0 -84
- package/docs/archive/investigations/colibri-glm52-port.md +0 -1486
- package/docs/archive/investigations/colibri-oracle-pin.md +0 -1284
- package/docs/archive/investigations/curve-bisector-routing.md +0 -123
- package/docs/archive/investigations/decode-roofline-lookagain.md +0 -348
- package/docs/archive/investigations/diffusion-gemma-port.md +0 -602
- package/docs/archive/investigations/dspark-handoff.md +0 -224
- package/docs/archive/investigations/expert-offload-single-user-moe.md +0 -292
- package/docs/archive/investigations/faithful-l1-consolidation.md +0 -310
- package/docs/archive/investigations/hlg-sampling-investigation.md +0 -970
- package/docs/archive/investigations/indexshare-performance-spike.md +0 -357
- package/docs/archive/investigations/kernel-perf-review-2026-07.md +0 -171
- package/docs/archive/investigations/lab-build-journal.md +0 -157
- package/docs/archive/investigations/minicpm5-decode-megakernel.md +0 -489
- package/docs/archive/investigations/openjarvis-fit.md +0 -198
- package/docs/archive/investigations/openwiki-evaluation.md +0 -249
- package/docs/archive/investigations/optimization_plan.md +0 -346
- package/docs/archive/investigations/orpo-base-uf-experiment-and-directions.md +0 -120
- package/docs/archive/investigations/orpo-flash-cce-pin-leak.md +0 -167
- package/docs/archive/investigations/orpo-uf-testing-handoff.md +0 -126
- package/docs/archive/investigations/pareto-specialized-runtime-findings.md +0 -92
- package/docs/archive/investigations/pi-builtin-investigation.md +0 -308
- package/docs/archive/investigations/pre-colibri-stabilization.md +0 -329
- package/docs/archive/investigations/qwen38-27b-campaign-log.md +0 -1092
- package/docs/archive/investigations/segmented-backward-handoff.md +0 -148
- package/docs/archive/investigations/starter-model-port-handoff.md +0 -342
- package/docs/archive/investigations/steel-flash-cce-handoff.md +0 -489
- package/docs/archive/investigations/the-dreaming-handoff.md +0 -141
- package/docs/archive/investigations/trainer-validation-experiment.md +0 -173
- package/docs/archive/planning/chat-ui-vision-memory-first.md +0 -199
- package/docs/archive/planning/chat-ui-vision.md +0 -50
- package/docs/archive/planning/chunk-task-roadmap.md +0 -77
- package/docs/archive/planning/curve-sampler-research-plan.md +0 -331
- package/docs/archive/planning/journal.md +0 -316
- package/docs/archive/planning/memory-docs-and-dag-plan.md +0 -194
- package/docs/archive/planning/pi-integration-review.md +0 -178
- package/docs/archive/planning/release-notes-v0.0.10.md +0 -57
- package/docs/archive/planning/release-notes-v0.0.11.md +0 -239
- package/docs/archive/planning/release-notes-v0.0.12.md +0 -80
- package/docs/archive/planning/release-notes-v0.0.13.md +0 -77
- package/docs/archive/planning/release-notes-v0.0.9.md +0 -122
- package/docs/archive/planning/release-notes-v0.2.0.md +0 -49
- package/docs/archive/planning/release-notes-v0.3.0.md +0 -35
- package/docs/archive/planning/release-notes-v0.4.0.md +0 -45
- package/docs/archive/planning/release-notes-v0.5.0.md +0 -37
- package/docs/archive/planning/web-ui-pass-plan.md +0 -213
- package/docs/archive/planning/website-readme-pass-plan.md +0 -224
- package/docs/design/batching.md +0 -2054
- package/docs/design/decode-speed-program.md +0 -2404
- package/docs/design/dreaming-nightly-pipeline.md +0 -684
- package/docs/design/generic-model-support.md +0 -821
- package/docs/design/kv-cache.md +0 -993
- package/docs/design/orpo-training.md +0 -680
- package/docs/design/speculative-decoding.md +0 -2129
- package/docs/design/turboquant.md +0 -2225
- package/docs/design/unified-engine-frontier-plan.md +0 -1987
- package/docs/design/web-chat-redesign.md +0 -695
- package/docs/planning/PRODUCT_ROADMAP.md +0 -593
- package/docs/planning/ResearchTopics.md +0 -119
- package/docs/planning/public-readiness-handoff.md +0 -244
- package/docs/planning/public-readiness.md +0 -149
- package/docs/planning/video-series.md +0 -421
- package/docs/reference/benchmarks.md +0 -3587
- package/docs/reference/cli.md +0 -646
- package/docs/reference/distribution.md +0 -380
- package/docs/reference/environment.md +0 -494
- package/docs/reference/glossary.md +0 -39
- package/docs/reference/library-api.md +0 -688
- package/docs/reference/memory.md +0 -260
- package/docs/reference/models.md +0 -346
- package/docs/reference/server-api.md +0 -1945
- package/docs/reference/server-config.md +0 -973
- package/docs/reference/training.md +0 -462
- package/docs/reference/troubleshooting.md +0 -158
- package/src/admission-plan.ts +0 -18
- package/src/audio/audiotoolbox.ts +0 -112
- package/src/audio/conformer.ts +0 -655
- package/src/audio/decode.ts +0 -133
- package/src/audio/features.ts +0 -202
- package/src/audio/mic-capture.ts +0 -98
- package/src/audio/silero-vad.ts +0 -379
- package/src/audio/transcode.ts +0 -67
- package/src/audio/whisper-decode.ts +0 -774
- package/src/audio/whisper-fast.ts +0 -732
- package/src/audio/whisper-format.ts +0 -73
- package/src/audio/whisper-mel.ts +0 -185
- package/src/audio/whisper-timing.ts +0 -274
- package/src/audio/whisper-tokenizer.ts +0 -239
- package/src/audio/whisper-transcribe.ts +0 -410
- package/src/backends/mlx/assistant-target.ts +0 -46
- package/src/backends/mlx/autoregressive-draft-rows.ts +0 -120
- package/src/backends/mlx/autoregressive.ts +0 -129
- package/src/backends/mlx/batch-group.ts +0 -1874
- package/src/backends/mlx/cache-attachment.ts +0 -67
- package/src/backends/mlx/cache-layout.ts +0 -31
- package/src/backends/mlx/checkpoint-state.ts +0 -53
- package/src/backends/mlx/continuation-persistence.ts +0 -107
- package/src/backends/mlx/continuation-request.ts +0 -43
- package/src/backends/mlx/continuation.ts +0 -77
- package/src/backends/mlx/diffusion.ts +0 -39
- package/src/backends/mlx/draft-target.ts +0 -32
- package/src/backends/mlx/encoder-cache.ts +0 -25
- package/src/backends/mlx/evaluation.ts +0 -105
- package/src/backends/mlx/fill-append.ts +0 -58
- package/src/backends/mlx/fill-group.ts +0 -497
- package/src/backends/mlx/full-kv-row-append.ts +0 -32
- package/src/backends/mlx/full-kv-row-donor.ts +0 -54
- package/src/backends/mlx/full-transitioning-kv-rows.ts +0 -65
- package/src/backends/mlx/gateway-binding.ts +0 -174
- package/src/backends/mlx/graph.ts +0 -38
- package/src/backends/mlx/kv-codec.ts +0 -9
- package/src/backends/mlx/kv-maintenance.ts +0 -161
- package/src/backends/mlx/kv-tensor-rows.ts +0 -212
- package/src/backends/mlx/memory-client.ts +0 -189
- package/src/backends/mlx/methods.ts +0 -90
- package/src/backends/mlx/mixed-iteration.ts +0 -62
- package/src/backends/mlx/model-prompt.ts +0 -178
- package/src/backends/mlx/model-serving.ts +0 -80
- package/src/backends/mlx/paged-cache-codec.ts +0 -27
- package/src/backends/mlx/prefill-cohort.ts +0 -100
- package/src/backends/mlx/prefill-policy.ts +0 -26
- package/src/backends/mlx/prefill-rows.ts +0 -244
- package/src/backends/mlx/prefill.ts +0 -33
- package/src/backends/mlx/projected-context-rows.ts +0 -72
- package/src/backends/mlx/prompt-input.ts +0 -29
- package/src/backends/mlx/qwen-prompt-input.ts +0 -40
- package/src/backends/mlx/request-policy.ts +0 -20
- package/src/backends/mlx/request-state-policy.ts +0 -54
- package/src/backends/mlx/rollback.ts +0 -92
- package/src/backends/mlx/row-sampling.ts +0 -49
- package/src/backends/mlx/serial-executor.ts +0 -251
- package/src/backends/mlx/speculative-group.ts +0 -438
- package/src/backends/mlx/speculative-round.ts +0 -161
- package/src/backends/mlx/speculative.ts +0 -131
- package/src/backends/mlx/state-rows.ts +0 -104
- package/src/backends/mlx/state-views.ts +0 -46
- package/src/backends/mlx/target-layout-capability.ts +0 -35
- package/src/backends/mlx/text-engine.ts +0 -73
- package/src/backends/mlx/transitioning-kv-rows.ts +0 -173
- package/src/chat-template.ts +0 -382
- package/src/cli.ts +0 -3490
- package/src/config.ts +0 -399
- package/src/contracts/completion.ts +0 -16
- package/src/contracts/execution.ts +0 -70
- package/src/contracts/generation.ts +0 -103
- package/src/contracts/host.ts +0 -7
- package/src/contracts/object-cache.ts +0 -8
- package/src/contracts/pi-provider.ts +0 -30
- package/src/contracts/pi.ts +0 -261
- package/src/contracts/prefix-cache.ts +0 -33
- package/src/contracts/resources.ts +0 -25
- package/src/contracts/sampling.ts +0 -23
- package/src/contracts/scheduling.ts +0 -54
- package/src/contracts/tsconfig.json +0 -14
- package/src/dataset/generators.ts +0 -1087
- package/src/dataset/index.ts +0 -12
- package/src/dataset/job.ts +0 -81
- package/src/dataset/llm.ts +0 -71
- package/src/dataset/registry.ts +0 -528
- package/src/diffusion/diffusion-generate.ts +0 -507
- package/src/download.ts +0 -505
- package/src/embed.ts +0 -77
- package/src/engine/admission.ts +0 -59
- package/src/engine/cancellation.ts +0 -38
- package/src/engine/engine.ts +0 -56
- package/src/engine/execution-coordinator.ts +0 -70
- package/src/engine/execution-plan.ts +0 -64
- package/src/engine/execution-tasks.ts +0 -44
- package/src/engine/namespaced-cache.ts +0 -13
- package/src/engine/resources.ts +0 -53
- package/src/engine/scheduler.ts +0 -92
- package/src/engine/session.ts +0 -247
- package/src/eval/README.md +0 -130
- package/src/eval/ifeval.ts +0 -81
- package/src/eval/kl-prompts.ts +0 -29
- package/src/eval/kl.ts +0 -590
- package/src/eval/perplexity.ts +0 -164
- package/src/eval/quality-db.ts +0 -92
- package/src/eval/reference-logits.ts +0 -207
- package/src/eval/runner.ts +0 -124
- package/src/eval/score.ts +0 -49
- package/src/eval/tasks/bfcl.ts +0 -535
- package/src/eval/tasks/gsm8k.ts +0 -98
- package/src/eval/tasks/hashhop.ts +0 -139
- package/src/eval/tasks/humaneval.ts +0 -275
- package/src/eval/tasks/ifeval.ts +0 -525
- package/src/eval/tasks/mmlu.ts +0 -157
- package/src/eval/ultrafeedback.ts +0 -60
- package/src/evaldb.ts +0 -199
- package/src/expert-io.ts +0 -393
- package/src/expert-offload-build.ts +0 -77
- package/src/expert-offload.ts +0 -76
- package/src/expert-residency.ts +0 -1134
- package/src/expert-usage.ts +0 -415
- package/src/fill/echo-index.ts +0 -188
- package/src/fill/fill-session.ts +0 -463
- package/src/fill/proposal.ts +0 -68
- package/src/fill/schema-rows.ts +0 -506
- package/src/fill/tool-boundary.ts +0 -103
- package/src/fit.ts +0 -249
- package/src/generate.ts +0 -1352
- package/src/grammar.ts +0 -604
- package/src/harness-pi.ts +0 -184
- package/src/hf-push.ts +0 -628
- package/src/hub-rest.ts +0 -242
- package/src/index.ts +0 -63
- package/src/inference/denoising.ts +0 -17
- package/src/inference/draft-acceptance.ts +0 -41
- package/src/inference/draft-output.ts +0 -49
- package/src/inference/graph.ts +0 -33
- package/src/inference/prefill.ts +0 -45
- package/src/inference/rollback.ts +0 -7
- package/src/jobs/index.ts +0 -31
- package/src/jobs/job-entry.ts +0 -107
- package/src/jobs/task-client.ts +0 -19
- package/src/jobs/types.ts +0 -2
- package/src/kv-scheme.ts +0 -279
- package/src/kv-store.ts +0 -1135
- package/src/lab/README.md +0 -22
- package/src/lab/curve/curve-designer.html +0 -303
- package/src/lab/curve/curve-sampler.ts +0 -122
- package/src/lab/expert-trace/expert-trace.ts +0 -69
- package/src/lab/paged-kv/paged-attention.ts +0 -117
- package/src/lab/paged-kv/paged-kv-rows.ts +0 -111
- package/src/lab/paged-kv/paged-kv.ts +0 -336
- package/src/library.ts +0 -40
- package/src/lora.ts +0 -408
- package/src/md-modules.d.ts +0 -54
- package/src/media-fetch.ts +0 -442
- package/src/memory/article.ts +0 -558
- package/src/memory/chunk-validation.ts +0 -150
- package/src/memory/chunk.ts +0 -318
- package/src/memory/cluster.ts +0 -351
- package/src/memory/coverage.ts +0 -134
- package/src/memory/crosslink.ts +0 -0
- package/src/memory/db.ts +0 -253
- package/src/memory/entity.ts +0 -203
- package/src/memory/gate.ts +0 -187
- package/src/memory/history.ts +0 -152
- package/src/memory/ingest.ts +0 -335
- package/src/memory/model.ts +0 -183
- package/src/memory/normalize.ts +0 -671
- package/src/memory/parse.ts +0 -22
- package/src/memory/pipeline.ts +0 -275
- package/src/memory/prompts.ts +0 -65
- package/src/memory/query.ts +0 -334
- package/src/memory/reconcile.ts +0 -734
- package/src/memory/redirect.ts +0 -108
- package/src/memory/reindex.ts +0 -325
- package/src/memory/resolve.ts +0 -307
- package/src/memory/rest.ts +0 -245
- package/src/memory/route.ts +0 -272
- package/src/memory/schedule.ts +0 -165
- package/src/memory/stages.ts +0 -777
- package/src/memory/synthesize.ts +0 -1638
- package/src/memory/tools.ts +0 -749
- package/src/memory/vault.ts +0 -1194
- package/src/memory/wikify.ts +0 -1064
- package/src/mlx/array.ts +0 -425
- package/src/mlx/autograd.ts +0 -304
- package/src/mlx/checkpoint.ts +0 -149
- package/src/mlx/compile.ts +0 -168
- package/src/mlx/custom-vjp.ts +0 -188
- package/src/mlx/ffi.ts +0 -531
- package/src/mlx/materialize.ts +0 -23
- package/src/mlx/metal-kernel.ts +0 -159
- package/src/mlx/normalized-argmax.ts +0 -116
- package/src/mlx/ops.ts +0 -1108
- package/src/mlx/quantized-rows.ts +0 -36
- package/src/mlx/token-bitmask.ts +0 -52
- package/src/mlx/turboquant-kv-decode.ts +0 -82
- package/src/mlx/turboquant-kv-inverse.ts +0 -122
- package/src/mlx/turboquant-ops.ts +0 -861
- package/src/mlx/turboquant-tables.ts +0 -237
- package/src/mmap.ts +0 -90
- package/src/model/artifact-identity.ts +0 -73
- package/src/model/batched-kv.ts +0 -235
- package/src/model/batched-mask.ts +0 -279
- package/src/model/batched-quant.ts +0 -212
- package/src/model/batched-quantized-kv.ts +0 -144
- package/src/model/batched-rotating-quant.ts +0 -353
- package/src/model/batched-rotating-state.ts +0 -149
- package/src/model/batched-rotating.ts +0 -338
- package/src/model/batched-row-storage.ts +0 -179
- package/src/model/batched-ssm.ts +0 -52
- package/src/model/batched-turboquant-kv.ts +0 -94
- package/src/model/compiled-decode.ts +0 -712
- package/src/model/decoded-kv-donor.ts +0 -12
- package/src/model/delayed-quantized-kv.ts +0 -72
- package/src/model/delayed-rotating-quantized-kv.ts +0 -93
- package/src/model/delayed-turboquant-kv.ts +0 -57
- package/src/model/diffusion-gemma.ts +0 -1026
- package/src/model/factory.ts +0 -215
- package/src/model/fingerprint.ts +0 -45
- package/src/model/flash-attention.ts +0 -694
- package/src/model/full-prefill-padding.ts +0 -56
- package/src/model/full-prefill-row.ts +0 -105
- package/src/model/gemma4-base.ts +0 -2370
- package/src/model/gemma4.ts +0 -1395
- package/src/model/generated/gemma4-12b.ts +0 -429
- package/src/model/generated/gemma4-26b.ts +0 -448
- package/src/model/generated/gemma4-e4b.ts +0 -717
- package/src/model/generated/index.ts +0 -18
- package/src/model/glm52-atlas.ts +0 -511
- package/src/model/glm52-cache.ts +0 -792
- package/src/model/glm52-config.ts +0 -262
- package/src/model/glm52-container.ts +0 -305
- package/src/model/glm52-coupling.ts +0 -388
- package/src/model/glm52-dsa.ts +0 -606
- package/src/model/glm52-expert-layout.ts +0 -253
- package/src/model/glm52-layout.ts +0 -152
- package/src/model/glm52-memory.ts +0 -500
- package/src/model/glm52-mla.ts +0 -792
- package/src/model/glm52-moe.ts +0 -223
- package/src/model/glm52-pilot.ts +0 -537
- package/src/model/glm52-reference.ts +0 -445
- package/src/model/glm52-residency.ts +0 -557
- package/src/model/glm52-resident-weights.ts +0 -135
- package/src/model/glm52-streamed-experts.ts +0 -453
- package/src/model/glm52-streamed-metal.ts +0 -610
- package/src/model/glm52-weights.ts +0 -503
- package/src/model/glm52.ts +0 -1132
- package/src/model/implementation.ts +0 -69
- package/src/model/kv-attention-view.ts +0 -70
- package/src/model/minicpm5.ts +0 -376
- package/src/model/profile.ts +0 -468
- package/src/model/quantized-append-attention.ts +0 -31
- package/src/model/qwen-append.ts +0 -12
- package/src/model/qwen-conv.ts +0 -52
- package/src/model/qwen3-delta-state.ts +0 -74
- package/src/model/qwen3-delta.ts +0 -451
- package/src/model/qwen3-moe.ts +0 -404
- package/src/model/qwen3-mrope.ts +0 -250
- package/src/model/qwen3.ts +0 -277
- package/src/model/qwen3_5-checkpoint.ts +0 -164
- package/src/model/qwen3_5.ts +0 -881
- package/src/model/rotating-kv-donor.ts +0 -35
- package/src/model/rotating-kv-layout.ts +0 -262
- package/src/model/rotating-row-transaction.ts +0 -35
- package/src/model/speculative-rotating-kv.ts +0 -101
- package/src/model/ssm-prefill-padding.ts +0 -39
- package/src/model/support.ts +0 -110
- package/src/model/token-groups.ts +0 -66
- package/src/model/trellis-balanced-scatter.ts +0 -51
- package/src/model/trellis-linear.ts +0 -672
- package/src/model/trellis-shared-m.ts +0 -151
- package/src/model/trellis-shared-scatter.ts +0 -144
- package/src/model/trellis-splitk-prefill.ts +0 -158
- package/src/model/trellis-tiled-prefill.ts +0 -136
- package/src/model/trellis-vector-expand.ts +0 -62
- package/src/model/trellis-wide-prefill.ts +0 -94
- package/src/model/turboquant-codec.ts +0 -89
- package/src/model/universal/archs.ts +0 -332
- package/src/model/universal/dense.ts +0 -612
- package/src/model/universal/modules.ts +0 -170
- package/src/model/universal/rope.ts +0 -381
- package/src/model/whisper.ts +0 -435
- package/src/native/expert_io.c +0 -523
- package/src/native/frame_extract.swift +0 -140
- package/src/native/mic_capture.swift +0 -92
- package/src/native-pack.ts +0 -163
- package/src/pi-provider.ts +0 -53
- package/src/pi-session.ts +0 -81
- package/src/pi-terminal.ts +0 -274
- package/src/pi-web.ts +0 -1940
- package/src/preflight.ts +0 -158
- package/src/prompt-cache.ts +0 -682
- package/src/quantize/allocator.ts +0 -627
- package/src/quantize/atomic-output.ts +0 -38
- package/src/quantize/calibration.ts +0 -181
- package/src/quantize/config-writer.ts +0 -145
- package/src/quantize/index.ts +0 -84
- package/src/quantize/job.ts +0 -162
- package/src/quantize/quantizer.ts +0 -666
- package/src/quantize/rotate.ts +0 -603
- package/src/quantize/safetensors-writer.ts +0 -249
- package/src/quantize/sensitivity.ts +0 -216
- package/src/quantize/trellis.ts +0 -586
- package/src/quantize/weight-transform.ts +0 -292
- package/src/registry.ts +0 -602
- package/src/runtime-config.ts +0 -77
- package/src/safetensors.ts +0 -165
- package/src/sampler.ts +0 -1012
- package/src/serve/admin-routes.ts +0 -184
- package/src/serve/assets/curve-terrain.html +0 -105
- package/src/serve/assets/training-inference-map.html +0 -575
- package/src/serve/audio-routes.ts +0 -323
- package/src/serve/aux-routes.ts +0 -132
- package/src/serve/batch-scheduler.ts +0 -8
- package/src/serve/checkpoint-identity.ts +0 -64
- package/src/serve/generation-gateway.ts +0 -537
- package/src/serve/isolate.ts +0 -609
- package/src/serve/kv-budget.ts +0 -14
- package/src/serve/lab-routes.ts +0 -341
- package/src/serve/lane-registry.ts +0 -45
- package/src/serve/model-admin-routes.ts +0 -155
- package/src/serve/model-binding.ts +0 -49
- package/src/serve/model-host.ts +0 -511
- package/src/serve/parent-application.ts +0 -56
- package/src/serve/prompt-response-trace.ts +0 -161
- package/src/serve/response-lease.ts +0 -24
- package/src/serve/session-search.ts +0 -326
- package/src/serve/static-routes.ts +0 -101
- package/src/serve/transcription-server.ts +0 -64
- package/src/serve/transcription-service.ts +0 -502
- package/src/server.ts +0 -1282
- package/src/spec/assistant-rows.ts +0 -93
- package/src/spec/assistant-source.ts +0 -126
- package/src/spec/cache-identity.ts +0 -22
- package/src/spec/deepspec-rows.ts +0 -29
- package/src/spec/deepspec-source.ts +0 -145
- package/src/spec/dflash-source.ts +0 -149
- package/src/spec/drafter.ts +0 -304
- package/src/spec/dspark/ab-stats.ts +0 -136
- package/src/spec/dspark/calibration.ts +0 -112
- package/src/spec/dspark/data-dflash.ts +0 -156
- package/src/spec/dspark/data.ts +0 -201
- package/src/spec/dspark/deepspec-module.ts +0 -776
- package/src/spec/dspark/generate-dflash.ts +0 -245
- package/src/spec/dspark/generate.ts +0 -290
- package/src/spec/dspark/loader.ts +0 -23
- package/src/spec/dspark/loss.ts +0 -213
- package/src/spec/dspark/module-dflash.ts +0 -560
- package/src/spec/dspark/module.ts +0 -540
- package/src/spec/dspark/quantize-drafter.ts +0 -91
- package/src/spec/dspark/sample.ts +0 -112
- package/src/spec/generate.ts +0 -276
- package/src/spec/glm52-mtp-graph.ts +0 -29
- package/src/spec/glm52-mtp-rows.ts +0 -132
- package/src/spec/glm52-mtp-source.ts +0 -318
- package/src/spec/glm52-mtp-state.ts +0 -33
- package/src/spec/ngram-source.ts +0 -226
- package/src/spec/projected-draft-rows.ts +0 -73
- package/src/spec/qwen-mtp-module.ts +0 -171
- package/src/spec/qwen-mtp-rows.ts +0 -230
- package/src/spec/qwen-mtp-source.ts +0 -386
- package/src/spec/qwen-mtp-state.ts +0 -30
- package/src/spec/serve-loop.ts +0 -587
- package/src/spec/source.ts +0 -228
- package/src/spec/two-model-rows.ts +0 -62
- package/src/spec/two-model.ts +0 -113
- package/src/ssd-cache.ts +0 -431
- package/src/ssd-durability.ts +0 -209
- package/src/storage/host-buffer.ts +0 -24
- package/src/storage/kv-writer.ts +0 -83
- package/src/storage/kv-writer.worker.js +0 -264
- package/src/storage/retention-policy.ts +0 -37
- package/src/tiered-prompt-cache.ts +0 -45
- package/src/tokenizer.ts +0 -335
- package/src/tool-approvals.ts +0 -131
- package/src/tool-call.ts +0 -695
- package/src/train/dataset.ts +0 -484
- package/src/train/diffusion-lora.ts +0 -216
- package/src/train/export.ts +0 -32
- package/src/train/flash-cce.ts +0 -1234
- package/src/train/forward.ts +0 -266
- package/src/train/fuse.ts +0 -243
- package/src/train/index.ts +0 -46
- package/src/train/job.ts +0 -205
- package/src/train/lora-params.ts +0 -254
- package/src/train/loss.ts +0 -1591
- package/src/train/merge.ts +0 -195
- package/src/train/optimizer.ts +0 -183
- package/src/train/prefix-shared.ts +0 -490
- package/src/train/rank.ts +0 -137
- package/src/train/segmented.ts +0 -1828
- package/src/train/steel-qmm-header.ts +0 -1461
- package/src/train/trainer.ts +0 -1221
- package/src/train/watch.ts +0 -542
- package/src/vision/diffusion-vision.ts +0 -620
- package/src/vision/embedder.ts +0 -176
- package/src/vision/preprocess.ts +0 -247
- package/src/vision/prompt.ts +0 -409
- package/src/vision/qwen3vl-preprocess.ts +0 -409
- package/src/vision/qwen3vl-prompt.ts +0 -233
- package/src/vision/qwen3vl-tower.ts +0 -566
- package/src/vision/siglip.ts +0 -607
- package/src/vision/video-frames.ts +0 -148
- package/src/web/app.html +0 -2155
- package/src/web/app.js +0 -5863
- package/src/web/skills/memory/SKILL.md +0 -120
- package/src/web/skills/web-research/SKILL.md +0 -46
- package/src/web/skills.ts +0 -97
- package/src/web/src/adapters-panel.ts +0 -288
- package/src/web/src/assistant.ts +0 -451
- package/src/web/src/chat.ts +0 -1060
- package/src/web/src/composer.ts +0 -962
- package/src/web/src/dataset.ts +0 -132
- package/src/web/src/finetune.ts +0 -267
- package/src/web/src/hub.ts +0 -428
- package/src/web/src/main.ts +0 -98
- package/src/web/src/markdown.ts +0 -0
- package/src/web/src/memory-panel.ts +0 -572
- package/src/web/src/model-picker.ts +0 -159
- package/src/web/src/palette.ts +0 -346
- package/src/web/src/protocol.ts +0 -26
- package/src/web/src/quantize.ts +0 -167
- package/src/web/src/rag.ts +0 -274
- package/src/web/src/sessions.ts +0 -469
- package/src/web/src/shell.ts +0 -804
- package/src/web/src/ui-catalog.ts +0 -78
- package/src/web/src/voice.ts +0 -141
- package/src/web/vendor/README.md +0 -61
- package/src/web-assets.ts +0 -47
- package/src/web-tools.ts +0 -478
- package/src/weights.ts +0 -255
- /package/src/{serve → server}/completion-sink.ts +0 -0
- /package/src/{serve → server}/generated-token-history.ts +0 -0
- /package/src/web/{icon.svg → public/icon.svg} +0 -0
- /package/src/web/{manifest.webmanifest → public/manifest.webmanifest} +0 -0
- /package/src/web/{vendor → public/vendor}/hljs-LICENSE +0 -0
package/src/audio/decode.ts
DELETED
|
@@ -1,133 +0,0 @@
|
|
|
1
|
-
// WAV decode for audio input (A1 of docs/design/generic-model-support.md).
|
|
2
|
-
//
|
|
3
|
-
// Scope: RIFF/WAVE containers — PCM 16/24/32-bit and IEEE float32, any
|
|
4
|
-
// channel count (mean mixdown), any sample rate (linear resample to 16 kHz).
|
|
5
|
-
// PCM16 scales by 1/32768 to match the oracle golden generator
|
|
6
|
-
// (scripts/oracle/gen-e4b-audio-golden.py) exactly. Non-WAV containers (mp3/m4a/
|
|
7
|
-
// flac/ogg) are transcoded to WAV by the serve layer via macOS `afconvert`
|
|
8
|
-
// before reaching this parser.
|
|
9
|
-
//
|
|
10
|
-
// Resampling note: optiq's serve frontend never exposes audio, so there is
|
|
11
|
-
// no oracle behavior for non-16 kHz input — linear interpolation is our
|
|
12
|
-
// documented choice (the parity fixtures are native 16 kHz and skip it).
|
|
13
|
-
|
|
14
|
-
export const AUDIO_SAMPLE_RATE = 16_000;
|
|
15
|
-
export const AUDIO_MS_PER_TOKEN = 40;
|
|
16
|
-
export const AUDIO_SEQ_LENGTH = 750; // 30 s cap, matches the oracle processor
|
|
17
|
-
/** Raw-speech truncation bound — the oracle feature extractor's default
|
|
18
|
-
* (audio_feature_extractor.py __call__: truncation=True,
|
|
19
|
-
* max_length=480_000 → `w[:max_length]`, i.e. 30 s at 16 kHz) applied
|
|
20
|
-
* BEFORE feature extraction. Mirrored in decodeAudio so a >30 s clip
|
|
21
|
-
* yields exactly the capped 750 soft tokens AND the matching mel frame
|
|
22
|
-
* count — the prompt builder hard-checks that the tower's output length
|
|
23
|
-
* equals the splice's soft-token count. */
|
|
24
|
-
export const AUDIO_MAX_SAMPLES = 30 * AUDIO_SAMPLE_RATE; // 480_000
|
|
25
|
-
|
|
26
|
-
export interface DecodedWav {
|
|
27
|
-
samples: Float32Array; // mono
|
|
28
|
-
sampleRate: number;
|
|
29
|
-
}
|
|
30
|
-
|
|
31
|
-
/** Number of audio soft tokens for a clip — processing_gemma4.py
|
|
32
|
-
* `_compute_audio_num_tokens`: ceil(duration_ms / 40) capped at 750.
|
|
33
|
-
* MUST be computed from decoded samples, never from container metadata. */
|
|
34
|
-
export function audioSoftTokenCount(
|
|
35
|
-
numSamples: number,
|
|
36
|
-
sampleRate: number = AUDIO_SAMPLE_RATE,
|
|
37
|
-
): number {
|
|
38
|
-
const durationMs = (numSamples / sampleRate) * 1000;
|
|
39
|
-
return Math.min(Math.ceil(durationMs / AUDIO_MS_PER_TOKEN), AUDIO_SEQ_LENGTH);
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
const WAVE_FORMAT_PCM = 1;
|
|
43
|
-
const WAVE_FORMAT_IEEE_FLOAT = 3;
|
|
44
|
-
const WAVE_FORMAT_EXTENSIBLE = 0xfffe;
|
|
45
|
-
|
|
46
|
-
/** Parse a RIFF/WAVE file into mono float samples at its native rate. */
|
|
47
|
-
export function decodeWav(bytes: Uint8Array): DecodedWav {
|
|
48
|
-
const v = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
49
|
-
const tag = (o: number) => String.fromCharCode(bytes[o]!, bytes[o + 1]!, bytes[o + 2]!, bytes[o + 3]!);
|
|
50
|
-
if (bytes.length < 44 || tag(0) !== "RIFF" || tag(8) !== "WAVE")
|
|
51
|
-
throw new Error("not a RIFF/WAVE file");
|
|
52
|
-
|
|
53
|
-
let fmt: { format: number; channels: number; sampleRate: number; bits: number } | null = null;
|
|
54
|
-
let dataOff = -1;
|
|
55
|
-
let dataLen = 0;
|
|
56
|
-
for (let o = 12; o + 8 <= bytes.length; ) {
|
|
57
|
-
const id = tag(o);
|
|
58
|
-
const size = v.getUint32(o + 4, true);
|
|
59
|
-
const body = o + 8;
|
|
60
|
-
if (id === "fmt ") {
|
|
61
|
-
// the reads below reach body+16 (base fields) / body+26 (extensible)
|
|
62
|
-
if (size < 16 || body + 16 > bytes.length)
|
|
63
|
-
throw new Error("WAV: truncated fmt chunk");
|
|
64
|
-
let format = v.getUint16(body, true);
|
|
65
|
-
const channels = v.getUint16(body + 2, true);
|
|
66
|
-
const sampleRate = v.getUint32(body + 4, true);
|
|
67
|
-
const bits = v.getUint16(body + 14, true);
|
|
68
|
-
if (format === WAVE_FORMAT_EXTENSIBLE) {
|
|
69
|
-
if (size < 40 || body + 26 > bytes.length)
|
|
70
|
-
throw new Error("WAV: truncated WAVE_FORMAT_EXTENSIBLE fmt chunk");
|
|
71
|
-
format = v.getUint16(body + 24, true); // first 2 bytes of SubFormat GUID
|
|
72
|
-
}
|
|
73
|
-
fmt = { format, channels, sampleRate, bits };
|
|
74
|
-
} else if (id === "data") {
|
|
75
|
-
dataOff = body;
|
|
76
|
-
dataLen = Math.min(size, bytes.length - body);
|
|
77
|
-
}
|
|
78
|
-
o = body + size + (size & 1); // chunks are word-aligned
|
|
79
|
-
}
|
|
80
|
-
if (!fmt) throw new Error("WAV: missing fmt chunk");
|
|
81
|
-
if (dataOff < 0) throw new Error("WAV: missing data chunk");
|
|
82
|
-
const { format, channels, sampleRate, bits } = fmt;
|
|
83
|
-
if (channels < 1) throw new Error("WAV: zero channels");
|
|
84
|
-
|
|
85
|
-
const bytesPer = bits / 8;
|
|
86
|
-
const frames = Math.floor(dataLen / (bytesPer * channels));
|
|
87
|
-
const out = new Float32Array(frames);
|
|
88
|
-
|
|
89
|
-
const readSample = (off: number): number => {
|
|
90
|
-
if (format === WAVE_FORMAT_IEEE_FLOAT && bits === 32) return v.getFloat32(off, true);
|
|
91
|
-
if (format === WAVE_FORMAT_PCM && bits === 16) return v.getInt16(off, true) / 32768;
|
|
92
|
-
if (format === WAVE_FORMAT_PCM && bits === 24) {
|
|
93
|
-
const raw = bytes[off]! | (bytes[off + 1]! << 8) | (bytes[off + 2]! << 16);
|
|
94
|
-
return ((raw << 8) >> 8) / 8388608;
|
|
95
|
-
}
|
|
96
|
-
if (format === WAVE_FORMAT_PCM && bits === 32) return v.getInt32(off, true) / 2147483648;
|
|
97
|
-
throw new Error(`WAV: unsupported format ${format} / ${bits}-bit`);
|
|
98
|
-
};
|
|
99
|
-
|
|
100
|
-
for (let f = 0; f < frames; f++) {
|
|
101
|
-
const base = dataOff + f * bytesPer * channels;
|
|
102
|
-
let acc = 0;
|
|
103
|
-
for (let c = 0; c < channels; c++) acc += readSample(base + c * bytesPer);
|
|
104
|
-
out[f] = Math.fround(acc / channels);
|
|
105
|
-
}
|
|
106
|
-
return { samples: out, sampleRate };
|
|
107
|
-
}
|
|
108
|
-
|
|
109
|
-
/** Linear resample to the model rate (no-op passthrough at 16 kHz). */
|
|
110
|
-
export function resampleTo16k(samples: Float32Array, sampleRate: number): Float32Array {
|
|
111
|
-
if (sampleRate === AUDIO_SAMPLE_RATE) return samples;
|
|
112
|
-
if (sampleRate <= 0) throw new Error(`bad sample rate ${sampleRate}`);
|
|
113
|
-
const outLen = Math.max(1, Math.round((samples.length * AUDIO_SAMPLE_RATE) / sampleRate));
|
|
114
|
-
const out = new Float32Array(outLen);
|
|
115
|
-
const step = samples.length / outLen;
|
|
116
|
-
for (let i = 0; i < outLen; i++) {
|
|
117
|
-
const pos = i * step;
|
|
118
|
-
const j = Math.floor(pos);
|
|
119
|
-
const fracPart = pos - j;
|
|
120
|
-
const a = samples[Math.min(j, samples.length - 1)]!;
|
|
121
|
-
const b = samples[Math.min(j + 1, samples.length - 1)]!;
|
|
122
|
-
out[i] = Math.fround(a + (b - a) * fracPart);
|
|
123
|
-
}
|
|
124
|
-
return out;
|
|
125
|
-
}
|
|
126
|
-
|
|
127
|
-
/** WAV bytes → mono 16 kHz float samples (the feature-extractor input),
|
|
128
|
-
* truncated at 30 s exactly like the oracle (AUDIO_MAX_SAMPLES). */
|
|
129
|
-
export function decodeAudio(bytes: Uint8Array): Float32Array {
|
|
130
|
-
const { samples, sampleRate } = decodeWav(bytes);
|
|
131
|
-
const s = resampleTo16k(samples, sampleRate);
|
|
132
|
-
return s.length > AUDIO_MAX_SAMPLES ? s.subarray(0, AUDIO_MAX_SAMPLES) : s;
|
|
133
|
-
}
|
package/src/audio/features.ts
DELETED
|
@@ -1,202 +0,0 @@
|
|
|
1
|
-
// USM log-mel feature extraction for gemma-4 audio — a verbatim-semantics
|
|
2
|
-
// port of the oracle's Gemma4AudioFeatureExtractor (optiq/vlm/_mlxvlm/
|
|
3
|
-
// models/gemma4/audio_feature_extractor.py, constructor DEFAULTS — §3.3 Q3
|
|
4
|
-
// of docs/design/generic-model-support.md: no per-bin normalization, params are
|
|
5
|
-
// fixed for all gemma-4 models). Gated bit-close against the T0 mel goldens
|
|
6
|
-
// (goldens/e4b-audio-*-mel.bin) in tests/parity/audio-features.test.ts.
|
|
7
|
-
//
|
|
8
|
-
// Pipeline (single waveform, 16 kHz mono float32):
|
|
9
|
-
// pad to a multiple of 128 samples (validity mask over the original part)
|
|
10
|
-
// → semicausal left-pad of frame_length/2 = 160 zeros
|
|
11
|
-
// → unfold into 321-sample frames at hop 160, drop the 321st sample
|
|
12
|
-
// (preemphasis = 0 branch)
|
|
13
|
-
// → × periodic Hann window (numpy computes it in FLOAT32 — emulated with
|
|
14
|
-
// Math.fround so the products match the oracle's f32 rounding)
|
|
15
|
-
// → rfft n=512 (numpy promotes to float64; we FFT in f64)
|
|
16
|
-
// → |·| → @ mel filter bank (128 HTK bins over 0–8 kHz, float64, the
|
|
17
|
-
// transformers mel_filter_bank construction: linspace fft freqs,
|
|
18
|
-
// unguarded slope division, no norm)
|
|
19
|
-
// → log(x + 1e-3) → zero out invalid frames → float32 [T, 128].
|
|
20
|
-
//
|
|
21
|
-
// Frame validity: frame f is valid iff original-sample index
|
|
22
|
-
// f·hop + frame_length lands inside the un-padded waveform.
|
|
23
|
-
|
|
24
|
-
const SR = 16_000;
|
|
25
|
-
const FRAME = 320; // 20 ms
|
|
26
|
-
const HOP = 160; // 10 ms
|
|
27
|
-
const FFT_N = 512; // 2^ceil(log2(320)), fft_overdrive=false
|
|
28
|
-
const FREQ_BINS = FFT_N / 2 + 1; // 257
|
|
29
|
-
const MEL_BINS = 128;
|
|
30
|
-
const MEL_FMIN = 0.0;
|
|
31
|
-
const MEL_FMAX = 8000.0;
|
|
32
|
-
const MEL_FLOOR = 1e-3;
|
|
33
|
-
const PAD_MULTIPLE = 128;
|
|
34
|
-
|
|
35
|
-
export interface MelFeatures {
|
|
36
|
-
/** [frames, 128] row-major log-mel, float32 (invalid frames zeroed). */
|
|
37
|
-
features: Float32Array;
|
|
38
|
-
frames: number;
|
|
39
|
-
/** Per-frame validity (1 = real audio, 0 = padding). */
|
|
40
|
-
mask: Uint8Array;
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
// --- mel filter bank (float64, transformers semantics) -----------------------
|
|
44
|
-
const hzToMel = (f: number): number => 2595.0 * Math.log10(1.0 + f / 700.0);
|
|
45
|
-
const melToHz = (m: number): number => 700.0 * (10 ** (m / 2595.0) - 1.0);
|
|
46
|
-
|
|
47
|
-
function buildMelFilters(): Float64Array {
|
|
48
|
-
// filter_freqs: 130 points linspace in mel space, mapped back to Hz
|
|
49
|
-
const melMin = hzToMel(MEL_FMIN);
|
|
50
|
-
const melMax = hzToMel(MEL_FMAX);
|
|
51
|
-
const nPts = MEL_BINS + 2;
|
|
52
|
-
const filterFreqs = new Float64Array(nPts);
|
|
53
|
-
for (let i = 0; i < nPts; i++)
|
|
54
|
-
filterFreqs[i] = melToHz(melMin + ((melMax - melMin) * i) / (nPts - 1));
|
|
55
|
-
// fft_freqs: linspace(0, sr//2, 257)
|
|
56
|
-
const fftFreqs = new Float64Array(FREQ_BINS);
|
|
57
|
-
for (let k = 0; k < FREQ_BINS; k++) fftFreqs[k] = (8000.0 * k) / (FREQ_BINS - 1);
|
|
58
|
-
// triangular filters: max(0, min(down, up)) — unguarded diffs like the oracle
|
|
59
|
-
const filters = new Float64Array(FREQ_BINS * MEL_BINS);
|
|
60
|
-
for (let m = 0; m < MEL_BINS; m++) {
|
|
61
|
-
const lower = filterFreqs[m]!;
|
|
62
|
-
const center = filterFreqs[m + 1]!;
|
|
63
|
-
const upper = filterFreqs[m + 2]!;
|
|
64
|
-
for (let k = 0; k < FREQ_BINS; k++) {
|
|
65
|
-
const down = (fftFreqs[k]! - lower) / (center - lower);
|
|
66
|
-
const up = (upper - fftFreqs[k]!) / (upper - center);
|
|
67
|
-
filters[k * MEL_BINS + m] = Math.max(0, Math.min(down, up));
|
|
68
|
-
}
|
|
69
|
-
}
|
|
70
|
-
return filters;
|
|
71
|
-
}
|
|
72
|
-
|
|
73
|
-
// --- periodic Hann window: the ORACLE'S EXACT float32 values -----------------
|
|
74
|
-
// numpy builds this in float32 end-to-end (arange f32 → ×2π/320 → cos, all in
|
|
75
|
-
// numpy's vectorized f32 routines) and those cos values differ from
|
|
76
|
-
// fround(Math.cos(x)) by 1 ulp on some inputs. That 1 ulp is the ONLY
|
|
77
|
-
// divergence in the whole pipeline: with these exact values our extractor
|
|
78
|
-
// reproduces the oracle mel golden BIT-EXACTLY; with a recomputed window the
|
|
79
|
-
// log-amplified cancellation error reaches ~4.5e-4 (verified 2026-07-07).
|
|
80
|
-
// So the constant IS the spec. Regen (oracle venv):
|
|
81
|
-
// python -c "from optiq.vlm._mlxvlm.models.gemma4.audio_feature_extractor \
|
|
82
|
-
// import Gemma4AudioFeatureExtractor as F; import base64; \
|
|
83
|
-
// print(base64.b64encode(F().window.astype('<f4').tobytes()).decode())"
|
|
84
|
-
const WINDOW_B64 =
|
|
85
|
-
"AAAAAAAgyjgAHMo5AFRjOgAHyjqAyR07ACJjOwCDmjvAtsk7wCf/O0BoHTyAVT48gFhiPNC2hDyQ" +
|
|
86
|
-
"yJk8UF+wPPB4yDwQE+I8ECv9PDDfDD0Q5Rs9qKUrPXAfPD3IUE09+DdfPUjTcT1okII9XI+MPYDl" +
|
|
87
|
-
"lj3QkaE9PJOsPbDotz0MkcM9LIvPPeDV2z30b+g9JFj1PZZGAT7iBgg+SOwOPhr2FT6mIx0+PHQk" +
|
|
88
|
-
"PhrnKz6OezM+zjA7Ph4GQz64+ko+0A1TPp4+Wz5OjGM+EvZrPhR7dD54Gn0+uemCPopShz5Ix4s+" +
|
|
89
|
-
"fEeQPrbSlD6GaJk+dQiePhCyoj7gZKc+biCsPkTksD7or7U+4oK6Prdcvz7uPMQ+CiPJPpEOzj4F" +
|
|
90
|
-
"/9I+6fPXPsTs3D4S6eE+W+jmPhvq6z7Y7fA+EvP1Pkr5+j4AAAA/XIMCP3cGBT8ViQc/8woKP9OL" +
|
|
91
|
-
"DD93Cw8/oIkRPwwGFD9+gBY/uPgYP3xuGz+K4R0/pVEgP5C+Ij8OKCU/340nP8rvKT+RTSw/+KYu" +
|
|
92
|
-
"P8b7MD++SzM/pZY1P0PcNz9dHDo/u1Y8PySLPj9iuUA/PeFCP3wCRT/sHEc/WjBJP448Sz9SQU0/" +
|
|
93
|
-
"eD5PP84zUT8dIVM/OgZVP/LiVj8Yt1g/eoJaP+5EXD9I/l0/Wq5fP/xUYT8C8mI/RIVkP5oOZj/e" +
|
|
94
|
-
"jWc/6gJpP5ltaj/GzWs/UCNtPxVubj/0rW8/zOJwP4EMcj/0KnM/Cj50P6ZFdT+wQXY/DTJ3P6gW" +
|
|
95
|
-
"eD9o73g/OLx5PwZ9ej+8MXs/Stp7P552fD+qBn0/YIp9P7ABfj+SbH4/+sp+P94cfz82Yn8//Zp/" +
|
|
96
|
-
"PyvHfz+85n8/r/l/PwAAgD+v+X8/vOZ/PyvHfz/9mn8/NmJ/P94cfz/6yn4/kmx+P7ABfj9gin0/" +
|
|
97
|
-
"qgZ9P552fD9J2ns/uzF7PwV9ej84vHk/aO94P6gWeD8NMnc/r0F2P6RFdT8IPnQ/8ipzP4AMcj/M" +
|
|
98
|
-
"4nA/861vPxRubj9QI20/xc1rP5dtaj/rAmk/3o1nP5oOZj9EhWQ/AfJiP/tUYT9Yrl8/SP5dP+9E" +
|
|
99
|
-
"XD95glo/FbdYP/DiVj84BlU/HCFTP80zUT93Pk8/UkFNP4o8Sz9ZMEk/6xxHP3gCRT894UI/YrlA" +
|
|
100
|
-
"PyKLPj+6Vjw/Whw6P0LcNz+mljU/vEszP8b7MD/4pi4/jk0sP8jvKT/cjSc/DCglP5C+Ij+kUSA/" +
|
|
101
|
-
"iuEdP3puGz+1+Bg/fYAWPwwGFD+diRE/dwsPP9KLDD/zCgo/EokHP3MGBT9dgwI/AAAAP0X5+j4R" +
|
|
102
|
-
"8/U+0+3wPhrq6z5U6OY+EenhPsXs3D7o89c+/v7SPo4Ozj4EI8k+6zzEPrhcvz7fgro+6a+1PkHk" +
|
|
103
|
-
"sD5oIKw+3WSnPgqyoj5zCJ4+iGiZPrTSlD58R5A+RMeLPoRShz616YI+ehp9Pgx7dD4Q9ms+Roxj" +
|
|
104
|
-
"Ppw+Wz7IDVM+qvpKPiQGQz7MMDs+hHszPhrnKz4ydCQ+piMdPhL2FT5E7A4+5AYIPpRGAT4UWPU9" +
|
|
105
|
-
"7G/oPdDV2z0oi889EJHDPajotz08k6w9yJGhPXDllj1Uj4w9WJCCPTjTcT34N189uFBNPXAfPD2Y" +
|
|
106
|
-
"pSs98OQbPTjfDD0QK/088BLiPPB4yDxAX7A8gMiZPMC2hDxgWGI8gFU+PCBoHTyAJ/87wLbJO8CC" +
|
|
107
|
-
"mjsAImM7gMkdOwAGyjoAVGM6ABzKOQAgyjg=";
|
|
108
|
-
|
|
109
|
-
function buildWindow(): Float32Array {
|
|
110
|
-
const raw = Buffer.from(WINDOW_B64, "base64");
|
|
111
|
-
return new Float32Array(raw.buffer, raw.byteOffset, FRAME);
|
|
112
|
-
}
|
|
113
|
-
|
|
114
|
-
// --- iterative radix-2 complex FFT (float64), N = 512 ------------------------
|
|
115
|
-
const FFT_COS = new Float64Array(FFT_N / 2);
|
|
116
|
-
const FFT_SIN = new Float64Array(FFT_N / 2);
|
|
117
|
-
for (let i = 0; i < FFT_N / 2; i++) {
|
|
118
|
-
FFT_COS[i] = Math.cos((-2 * Math.PI * i) / FFT_N);
|
|
119
|
-
FFT_SIN[i] = Math.sin((-2 * Math.PI * i) / FFT_N);
|
|
120
|
-
}
|
|
121
|
-
|
|
122
|
-
function fft512(re: Float64Array, im: Float64Array): void {
|
|
123
|
-
// bit-reversal permutation
|
|
124
|
-
for (let i = 1, j = 0; i < FFT_N; i++) {
|
|
125
|
-
let bit = FFT_N >> 1;
|
|
126
|
-
for (; j & bit; bit >>= 1) j ^= bit;
|
|
127
|
-
j ^= bit;
|
|
128
|
-
if (i < j) {
|
|
129
|
-
const tr = re[i]!; re[i] = re[j]!; re[j] = tr;
|
|
130
|
-
const ti = im[i]!; im[i] = im[j]!; im[j] = ti;
|
|
131
|
-
}
|
|
132
|
-
}
|
|
133
|
-
for (let len = 2; len <= FFT_N; len <<= 1) {
|
|
134
|
-
const half = len >> 1;
|
|
135
|
-
const step = FFT_N / len;
|
|
136
|
-
for (let i = 0; i < FFT_N; i += len) {
|
|
137
|
-
for (let k = 0; k < half; k++) {
|
|
138
|
-
const c = FFT_COS[k * step]!;
|
|
139
|
-
const s = FFT_SIN[k * step]!;
|
|
140
|
-
const xr = re[i + k + half]! * c - im[i + k + half]! * s;
|
|
141
|
-
const xi = re[i + k + half]! * s + im[i + k + half]! * c;
|
|
142
|
-
re[i + k + half] = re[i + k]! - xr;
|
|
143
|
-
im[i + k + half] = im[i + k]! - xi;
|
|
144
|
-
re[i + k] = re[i + k]! + xr;
|
|
145
|
-
im[i + k] = im[i + k]! + xi;
|
|
146
|
-
}
|
|
147
|
-
}
|
|
148
|
-
}
|
|
149
|
-
}
|
|
150
|
-
|
|
151
|
-
let melFilters: Float64Array | null = null;
|
|
152
|
-
let hannWindow: Float32Array | null = null;
|
|
153
|
-
|
|
154
|
-
/** 16 kHz mono float32 waveform → USM log-mel features [T, 128]. */
|
|
155
|
-
export function extractMelFeatures(waveform: Float32Array): MelFeatures {
|
|
156
|
-
melFilters ??= buildMelFilters();
|
|
157
|
-
hannWindow ??= buildWindow();
|
|
158
|
-
|
|
159
|
-
const nReal = waveform.length;
|
|
160
|
-
const nPadded = Math.ceil(nReal / PAD_MULTIPLE) * PAD_MULTIPLE;
|
|
161
|
-
const PAD_LEFT = FRAME >> 1; // 160
|
|
162
|
-
const total = PAD_LEFT + nPadded;
|
|
163
|
-
const frames = Math.floor((total - (FRAME + 1)) / HOP) + 1;
|
|
164
|
-
if (frames <= 0) return { features: new Float32Array(0), frames: 0, mask: new Uint8Array(0) };
|
|
165
|
-
|
|
166
|
-
const features = new Float32Array(frames * MEL_BINS);
|
|
167
|
-
const mask = new Uint8Array(frames);
|
|
168
|
-
const re = new Float64Array(FFT_N);
|
|
169
|
-
const im = new Float64Array(FFT_N);
|
|
170
|
-
const mel = new Float64Array(MEL_BINS);
|
|
171
|
-
|
|
172
|
-
for (let f = 0; f < frames; f++) {
|
|
173
|
-
// frame f covers padded indices [f*HOP, f*HOP+FRAME) after the left pad;
|
|
174
|
-
// validity = the frame-END sample (index f*HOP+FRAME in the left-padded
|
|
175
|
-
// stream, i.e. f*HOP+FRAME-PAD_LEFT in the original) is real audio.
|
|
176
|
-
const endIdx = f * HOP + FRAME - PAD_LEFT;
|
|
177
|
-
const valid = endIdx >= 0 && endIdx < nReal;
|
|
178
|
-
mask[f] = valid ? 1 : 0;
|
|
179
|
-
if (!valid) continue; // oracle zeroes invalid frames; ours start zeroed
|
|
180
|
-
|
|
181
|
-
re.fill(0);
|
|
182
|
-
im.fill(0);
|
|
183
|
-
for (let n = 0; n < FRAME; n++) {
|
|
184
|
-
const srcIdx = f * HOP + n - PAD_LEFT; // <0 → semicausal zero pad
|
|
185
|
-
const sample = srcIdx >= 0 && srcIdx < nReal ? waveform[srcIdx]! : 0;
|
|
186
|
-
re[n] = Math.fround(sample * hannWindow[n]!); // numpy f32 product
|
|
187
|
-
}
|
|
188
|
-
fft512(re, im);
|
|
189
|
-
|
|
190
|
-
mel.fill(0);
|
|
191
|
-
for (let k = 0; k < FREQ_BINS; k++) {
|
|
192
|
-
const mag = Math.hypot(re[k]!, im[k]!);
|
|
193
|
-
if (mag === 0) continue;
|
|
194
|
-
const base = k * MEL_BINS;
|
|
195
|
-
for (let m = 0; m < MEL_BINS; m++) mel[m] = mel[m]! + mag * melFilters![base + m]!;
|
|
196
|
-
}
|
|
197
|
-
const out = f * MEL_BINS;
|
|
198
|
-
for (let m = 0; m < MEL_BINS; m++)
|
|
199
|
-
features[out + m] = Math.fround(Math.log(mel[m]! + MEL_FLOOR));
|
|
200
|
-
}
|
|
201
|
-
return { features, frames, mask };
|
|
202
|
-
}
|
package/src/audio/mic-capture.ts
DELETED
|
@@ -1,98 +0,0 @@
|
|
|
1
|
-
// Microphone capture for `mlx-bun dictate`: spawns the AVAudioEngine
|
|
2
|
-
// sidecar (src/native/mic_capture.swift, `mlx-bun-mic-capture`), yields
|
|
3
|
-
// 16 kHz float32 PCM chunks and hotkey down/up events. Resolution mirrors
|
|
4
|
-
// the video frame extractor: env override → beside the binary → the
|
|
5
|
-
// native-pack dir → dev-compile from source with swiftc.
|
|
6
|
-
|
|
7
|
-
import { existsSync, mkdirSync } from "node:fs";
|
|
8
|
-
import { dirname, join } from "node:path";
|
|
9
|
-
import { nativeRuntimeDir } from "../native-pack";
|
|
10
|
-
import { runtimeValue } from "../runtime-config";
|
|
11
|
-
|
|
12
|
-
const BIN = "mlx-bun-mic-capture";
|
|
13
|
-
let devCompile: Promise<string | null> | null = null;
|
|
14
|
-
|
|
15
|
-
function compileFromSource(): Promise<string | null> {
|
|
16
|
-
devCompile ??= (async () => {
|
|
17
|
-
const src = join(import.meta.dir, "..", "native", "mic_capture.swift");
|
|
18
|
-
if (!existsSync(src)) return null;
|
|
19
|
-
const out = join(import.meta.dir, "..", "..", "dist-native", BIN);
|
|
20
|
-
try {
|
|
21
|
-
mkdirSync(dirname(out), { recursive: true });
|
|
22
|
-
const proc = Bun.spawn(["swiftc", "-O", src, "-o", out], { stdout: "ignore", stderr: "pipe" });
|
|
23
|
-
if ((await proc.exited) === 0 && existsSync(out)) return out;
|
|
24
|
-
} catch { /* no swiftc */ }
|
|
25
|
-
return null;
|
|
26
|
-
})();
|
|
27
|
-
return devCompile;
|
|
28
|
-
}
|
|
29
|
-
|
|
30
|
-
export async function resolveMicCapture(): Promise<string | null> {
|
|
31
|
-
const explicit = runtimeValue("MLX_BUN_MIC_CAPTURE");
|
|
32
|
-
if (explicit) return explicit;
|
|
33
|
-
const beside = join(dirname(process.execPath), BIN);
|
|
34
|
-
if (existsSync(beside)) return beside;
|
|
35
|
-
const dir = nativeRuntimeDir();
|
|
36
|
-
if (dir && existsSync(join(dir, BIN))) return join(dir, BIN);
|
|
37
|
-
const dev = join(import.meta.dir, "..", "..", "dist-native", BIN);
|
|
38
|
-
if (existsSync(dev)) return dev;
|
|
39
|
-
return compileFromSource();
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
export type MicEvent =
|
|
43
|
-
| { kind: "ready"; info: string }
|
|
44
|
-
| { kind: "pcm"; samples: Float32Array }
|
|
45
|
-
| { kind: "hotkey"; down: boolean }
|
|
46
|
-
| { kind: "error"; message: string };
|
|
47
|
-
|
|
48
|
-
export interface MicCapture {
|
|
49
|
-
events: AsyncIterable<MicEvent>;
|
|
50
|
-
stop(): void;
|
|
51
|
-
}
|
|
52
|
-
|
|
53
|
-
/** Start capturing. `hotkey` is a macOS virtual keycode (61 = Right Option). */
|
|
54
|
-
export async function startMicCapture(opts: { hotkey?: number | null } = {}): Promise<MicCapture> {
|
|
55
|
-
const bin = await resolveMicCapture();
|
|
56
|
-
if (!bin) throw new Error("microphone capture needs the mlx-bun-mic-capture sidecar (build: scripts/build-mic-capture.sh, needs swiftc)");
|
|
57
|
-
const args = [bin, "--rate", "16000"];
|
|
58
|
-
if (opts.hotkey != null) args.push("--hotkey", String(opts.hotkey));
|
|
59
|
-
const proc = Bun.spawn(args, { stdin: "pipe", stdout: "pipe", stderr: "pipe" });
|
|
60
|
-
const queue: MicEvent[] = [];
|
|
61
|
-
const waiter: { fn: (() => void) | null } = { fn: null };
|
|
62
|
-
let done = false;
|
|
63
|
-
const push = (e: MicEvent) => { queue.push(e); const w = waiter.fn; waiter.fn = null; w?.(); };
|
|
64
|
-
(async () => {
|
|
65
|
-
let carry = new Uint8Array(0);
|
|
66
|
-
for await (const chunk of proc.stdout) {
|
|
67
|
-
const buf = new Uint8Array(carry.length + chunk.length);
|
|
68
|
-
buf.set(carry); buf.set(chunk, carry.length);
|
|
69
|
-
const usable = buf.length - (buf.length % 4);
|
|
70
|
-
if (usable > 0) push({ kind: "pcm", samples: new Float32Array(buf.buffer.slice(0, usable)) });
|
|
71
|
-
carry = buf.subarray(usable);
|
|
72
|
-
}
|
|
73
|
-
done = true; const w = waiter.fn; waiter.fn = null; w?.();
|
|
74
|
-
})();
|
|
75
|
-
(async () => {
|
|
76
|
-
let text = "";
|
|
77
|
-
for await (const chunk of proc.stderr) {
|
|
78
|
-
text += new TextDecoder().decode(chunk);
|
|
79
|
-
let nl: number;
|
|
80
|
-
while ((nl = text.indexOf("\n")) >= 0) {
|
|
81
|
-
const line = text.slice(0, nl).trim(); text = text.slice(nl + 1);
|
|
82
|
-
if (line.startsWith("ready")) push({ kind: "ready", info: line.slice(5).trim() });
|
|
83
|
-
else if (line === "hotkey down") push({ kind: "hotkey", down: true });
|
|
84
|
-
else if (line === "hotkey up") push({ kind: "hotkey", down: false });
|
|
85
|
-
else if (line.startsWith("error:")) push({ kind: "error", message: line.slice(6).trim() });
|
|
86
|
-
}
|
|
87
|
-
}
|
|
88
|
-
})();
|
|
89
|
-
const events: AsyncIterable<MicEvent> = {
|
|
90
|
-
async *[Symbol.asyncIterator]() {
|
|
91
|
-
while (!done || queue.length) {
|
|
92
|
-
if (queue.length) { yield queue.shift()!; continue; }
|
|
93
|
-
await new Promise<void>((r) => { waiter.fn = r; });
|
|
94
|
-
}
|
|
95
|
-
},
|
|
96
|
-
};
|
|
97
|
-
return { events, stop: () => { try { proc.stdin.end(); } catch { /* gone */ } proc.kill(); } };
|
|
98
|
-
}
|