doppler-gpu 0.4.3 → 0.4.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/README.md +67 -34
- package/assets/doppler.svg +57 -1601
- package/package.json +36 -5
- package/src/cli/cli-output.js +34 -1
- package/src/cli/doppler-cli.d.ts +18 -0
- package/src/cli/doppler-cli.js +68 -35
- package/src/cli/doppler-serve.d.ts +210 -0
- package/src/cli/doppler-serve.js +468 -42
- package/src/client/doppler-registry.json +3 -5
- package/src/client/provider.d.ts +1 -1
- package/src/client/provider.js +1 -1
- package/src/client/runtime/index.js +13 -0
- package/src/client/runtime/lora.d.ts +11 -1
- package/src/client/runtime/lora.js +28 -7
- package/src/client/runtime/model-manager.js +35 -11
- package/src/client/runtime/model-session.d.ts +2 -1
- package/src/client/runtime/model-session.js +2 -2
- package/src/client/runtime/model-source.js +20 -1
- package/src/client/runtime/types.d.ts +8 -0
- package/src/client/wrap-pipeline-handle.d.ts +62 -0
- package/src/client/wrap-pipeline-handle.js +188 -13
- package/src/config/conversion/README.md +5 -0
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +4 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +17 -9
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +14 -12
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +7 -7
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +7 -7
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +3 -3
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +3 -3
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +8 -4
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +8 -4
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +9 -5
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +8 -4
- package/src/config/goal-completion-matrix.json +552 -0
- package/src/config/kernel-path-loader.js +192 -38
- package/src/config/kernels/kernel-ref-digests.js +52 -18
- package/src/config/kernels/registry.json +3017 -744
- package/src/config/merge-contract-check.js +7 -3
- package/src/config/merge-helpers.d.ts +2 -0
- package/src/config/merge-helpers.js +1 -0
- package/src/config/merge.d.ts +1 -0
- package/src/config/merge.js +30 -0
- package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
- package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
- package/src/config/param-validator.js +4 -0
- package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
- package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
- package/src/config/schema/buffer-pool.schema.js +3 -3
- package/src/config/schema/claim-evidence-contract.schema.json +93 -0
- package/src/config/schema/command-surface-contract.schema.json +81 -0
- package/src/config/schema/converter.schema.d.ts +11 -0
- package/src/config/schema/converter.schema.js +8 -0
- package/src/config/schema/doppler.schema.d.ts +1 -0
- package/src/config/schema/doppler.schema.js +38 -6
- package/src/config/schema/execution-v1.schema.d.ts +27 -0
- package/src/config/schema/execution-v1.schema.js +28 -19
- package/src/config/schema/goal-completion-matrix.schema.json +223 -0
- package/src/config/schema/inference.schema.d.ts +11 -0
- package/src/config/schema/manifest.schema.d.ts +107 -1
- package/src/config/schema/manifest.schema.js +10 -1
- package/src/config/schema/model-artifact-contract.schema.json +119 -0
- package/src/config/schema/moe.schema.d.ts +13 -0
- package/src/config/schema/moe.schema.js +1 -1
- package/src/config/schema/policy-schema-registry.json +26 -0
- package/src/config/schema/subsystem-support-contract.schema.json +48 -0
- package/src/config/schema/training-metrics.schema.d.ts +1 -1
- package/src/config/schema/training-metrics.schema.js +4 -3
- package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +7 -7
- package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
- package/src/config/source-packages/registry.json +25 -0
- package/src/config/supported-operations.json +5 -0
- package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
- package/src/config/transforms/execution-graph-transforms.d.ts +42 -0
- package/src/config/transforms/execution-graph-transforms.js +227 -230
- package/src/converter/conversion-plan.d.ts +2 -0
- package/src/converter/conversion-plan.js +18 -4
- package/src/converter/core.d.ts +8 -0
- package/src/converter/core.js +574 -12
- package/src/converter/quantization-info.js +179 -2
- package/src/converter/quantizer.d.ts +5 -4
- package/src/converter/quantizer.js +16 -2
- package/src/experimental/adapters/lora-loader.js +109 -24
- package/src/experimental/diffusion/index.d.ts +24 -0
- package/src/experimental/diffusion/index.js +10 -0
- package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
- package/src/experimental/distribution/descriptor-transport.js +342 -0
- package/src/experimental/training/README.md +11 -0
- package/src/experimental/training/autograd.js +33 -8
- package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
- package/src/experimental/training/datasets/gepa-frontier.js +220 -0
- package/src/experimental/training/datasets/index.d.ts +13 -1
- package/src/experimental/training/datasets/index.js +8 -1
- package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
- package/src/experimental/training/datasets/teacher-traces.js +279 -0
- package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
- package/src/experimental/training/datasets/text-pairs.js +82 -5
- package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
- package/src/experimental/training/distillation/student-fixture.js +290 -32
- package/src/experimental/training/export.d.ts +9 -0
- package/src/experimental/training/export.js +113 -7
- package/src/experimental/training/index.d.ts +13 -1
- package/src/experimental/training/index.js +13 -1
- package/src/experimental/training/lora-pipeline.d.ts +143 -0
- package/src/experimental/training/lora-pipeline.js +1737 -49
- package/src/experimental/training/lora.js +23 -1
- package/src/experimental/training/loss.js +1 -1
- package/src/experimental/training/operator-agent-eval.d.ts +65 -0
- package/src/experimental/training/operator-agent-eval.js +326 -0
- package/src/experimental/training/operator-command.js +310 -51
- package/src/experimental/training/runner.d.ts +1 -0
- package/src/experimental/training/runner.js +4 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
- package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/registry.json +186 -6
- package/src/experimental/training/workloads.d.ts +37 -0
- package/src/experimental/training/workloads.js +110 -1
- package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
- package/src/formats/rdrr/functional-descriptor.js +230 -0
- package/src/formats/rdrr/index.d.ts +1 -0
- package/src/formats/rdrr/index.js +1 -0
- package/src/formats/rdrr/parsing.js +22 -6
- package/src/formats/rdrr/source-transform-contract.js +10 -1
- package/src/formats/rdrr/storage-descriptor.js +9 -2
- package/src/formats/rdrr/types.d.ts +5 -3
- package/src/formats/rdrr/validation.js +15 -1
- package/src/gpu/command-recorder.d.ts +43 -3
- package/src/gpu/command-recorder.js +122 -25
- package/src/gpu/device.js +111 -11
- package/src/gpu/kernel-tuner/cache.js +1 -5
- package/src/gpu/kernels/attention.js +17 -9
- package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
- package/src/gpu/kernels/attention_streaming_f16.wgsl +7 -5
- package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
- package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
- package/src/gpu/kernels/cast.js +74 -67
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +46 -1
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -20
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +59 -44
- package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
- package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
- package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
- package/src/gpu/kernels/dispatch.js +13 -3
- package/src/gpu/kernels/fused_ffn.js +3 -4
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
- package/src/gpu/kernels/gather.d.ts +84 -1
- package/src/gpu/kernels/gather.js +353 -5
- package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
- package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
- package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
- package/src/gpu/kernels/gelu.wgsl +11 -7
- package/src/gpu/kernels/gelu_f16.wgsl +11 -7
- package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
- package/src/gpu/kernels/gemma4-route-expert.js +126 -0
- package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
- package/src/gpu/kernels/index.d.ts +71 -0
- package/src/gpu/kernels/index.js +52 -0
- package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
- package/src/gpu/kernels/kv-cache-write.js +126 -0
- package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
- package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
- package/src/gpu/kernels/lm-head-argmax.js +218 -0
- package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
- package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
- package/src/gpu/kernels/matmul-dispatch.js +37 -1
- package/src/gpu/kernels/matmul-selection.d.ts +5 -5
- package/src/gpu/kernels/matmul-selection.js +104 -9
- package/src/gpu/kernels/matmul.d.ts +1 -1
- package/src/gpu/kernels/matmul.js +151 -43
- package/src/gpu/kernels/moe.d.ts +1 -0
- package/src/gpu/kernels/moe.js +46 -18
- package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
- package/src/gpu/kernels/pipeline-cache.js +24 -0
- package/src/gpu/kernels/rep-penalty.d.ts +21 -0
- package/src/gpu/kernels/rep-penalty.js +17 -9
- package/src/gpu/kernels/residual.d.ts +1 -0
- package/src/gpu/kernels/residual.js +13 -1
- package/src/gpu/kernels/residual.wgsl +3 -3
- package/src/gpu/kernels/residual_f16.wgsl +5 -3
- package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
- package/src/gpu/kernels/residual_vec4.wgsl +6 -6
- package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
- package/src/gpu/kernels/rmsnorm-pair.js +152 -0
- package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
- package/src/gpu/kernels/rmsnorm-qk.js +128 -0
- package/src/gpu/kernels/rmsnorm.d.ts +31 -0
- package/src/gpu/kernels/rmsnorm.js +32 -7
- package/src/gpu/kernels/rmsnorm.wgsl +10 -6
- package/src/gpu/kernels/rmsnorm_f16.wgsl +7 -3
- package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
- package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
- package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
- package/src/gpu/kernels/rope-qk.d.ts +41 -0
- package/src/gpu/kernels/rope-qk.js +143 -0
- package/src/gpu/kernels/rope_qk.wgsl +95 -0
- package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
- package/src/gpu/kernels/rule-registry.js +50 -10
- package/src/gpu/kernels/sample.js +212 -187
- package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
- package/src/gpu/kernels/shader-cache.js +1 -21
- package/src/gpu/kernels/soft-embedding.d.ts +28 -0
- package/src/gpu/kernels/soft-embedding.js +447 -0
- package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
- package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
- package/src/gpu/kernels/softmax.wgsl +5 -80
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
- package/src/gpu/kernels/uniform-utils.js +2 -5
- package/src/gpu/kernels/utils.d.ts +16 -1
- package/src/gpu/kernels/utils.js +42 -13
- package/src/gpu/readback-utils.js +10 -2
- package/src/gpu/uniform-cache.d.ts +10 -1
- package/src/gpu/uniform-cache.js +25 -7
- package/src/gpu/weight-buffer.d.ts +76 -7
- package/src/gpu/weight-buffer.js +85 -1
- package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
- package/src/inference/browser-harness-suite-helpers.d.ts +9 -0
- package/src/inference/browser-harness-suite-helpers.js +438 -2
- package/src/inference/browser-harness-text-helpers.d.ts +77 -0
- package/src/inference/browser-harness-text-helpers.js +236 -11
- package/src/inference/browser-harness.d.ts +8 -0
- package/src/inference/browser-harness.js +292 -8
- package/src/inference/decode-ring.d.ts +1 -0
- package/src/inference/decode-ring.js +7 -5
- package/src/inference/kv-cache/base.d.ts +25 -0
- package/src/inference/kv-cache/base.js +109 -1
- package/src/inference/kv-cache/mixed-geometry.d.ts +10 -0
- package/src/inference/kv-cache/mixed-geometry.js +109 -2
- package/src/inference/kv-cache/sliding-window.d.ts +14 -0
- package/src/inference/kv-cache/sliding-window.js +88 -0
- package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
- package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
- package/src/inference/moe-router.d.ts +14 -1
- package/src/inference/moe-router.js +24 -1
- package/src/inference/pipelines/audio/gemma4.js +52 -1
- package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
- package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
- package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
- package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
- package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
- package/src/inference/pipelines/text/attention/projections.d.ts +71 -2
- package/src/inference/pipelines/text/attention/projections.js +329 -49
- package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
- package/src/inference/pipelines/text/attention/record.js +446 -73
- package/src/inference/pipelines/text/attention/run.js +251 -49
- package/src/inference/pipelines/text/attention/types.d.ts +5 -1
- package/src/inference/pipelines/text/config.d.ts +22 -1
- package/src/inference/pipelines/text/config.js +199 -22
- package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
- package/src/inference/pipelines/text/dtype-contract.js +4 -1
- package/src/inference/pipelines/text/embed.d.ts +4 -3
- package/src/inference/pipelines/text/embed.js +16 -6
- package/src/inference/pipelines/text/execution-plan.d.ts +8 -0
- package/src/inference/pipelines/text/execution-plan.js +6 -0
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -0
- package/src/inference/pipelines/text/execution-runtime-builders.js +44 -28
- package/src/inference/pipelines/text/execution-v1.d.ts +2 -0
- package/src/inference/pipelines/text/execution-v1.js +373 -139
- package/src/inference/pipelines/text/ffn/dense.js +44 -13
- package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
- package/src/inference/pipelines/text/ffn/moe.js +9 -2
- package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
- package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/standard.js +44 -4
- package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
- package/src/inference/pipelines/text/generator-decode-policy.js +2 -12
- package/src/inference/pipelines/text/generator-helpers.js +5 -2
- package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
- package/src/inference/pipelines/text/generator-prefill-helpers.js +67 -1
- package/src/inference/pipelines/text/generator-runtime.d.ts +1 -0
- package/src/inference/pipelines/text/generator-steps.d.ts +38 -3
- package/src/inference/pipelines/text/generator-steps.js +268 -77
- package/src/inference/pipelines/text/generator.d.ts +39 -0
- package/src/inference/pipelines/text/generator.js +1105 -18
- package/src/inference/pipelines/text/init.d.ts +8 -1
- package/src/inference/pipelines/text/init.js +30 -1
- package/src/inference/pipelines/text/layer-plan-gpu.js +62 -14
- package/src/inference/pipelines/text/layer.js +272 -25
- package/src/inference/pipelines/text/linear-attention.js +6 -4
- package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
- package/src/inference/pipelines/text/logits/gpu.js +417 -66
- package/src/inference/pipelines/text/logits/index.d.ts +2 -2
- package/src/inference/pipelines/text/logits/index.js +96 -42
- package/src/inference/pipelines/text/logits/types.d.ts +2 -2
- package/src/inference/pipelines/text/lora-apply.js +17 -0
- package/src/inference/pipelines/text/model-load.js +5 -1
- package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
- package/src/inference/pipelines/text/moe-gpu.js +445 -16
- package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
- package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
- package/src/inference/pipelines/text/ops.d.ts +29 -2
- package/src/inference/pipelines/text/ops.js +19 -0
- package/src/inference/pipelines/text/per-layer-inputs.js +5 -0
- package/src/inference/pipelines/text/probes.js +4 -5
- package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
- package/src/inference/pipelines/text/sampling-config.js +103 -64
- package/src/inference/pipelines/text/sampling.js +13 -9
- package/src/inference/pipelines/text/state.js +17 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +8 -9
- package/src/inference/pipelines/text/types.d.ts +84 -3
- package/src/inference/pipelines/text/weights.d.ts +4 -2
- package/src/inference/pipelines/text/weights.js +4 -5
- package/src/inference/pipelines/text.d.ts +24 -0
- package/src/inference/pipelines/text.js +162 -38
- package/src/inference/pipelines/vision/gemma4.js +20 -9
- package/src/inference/tokenizer.d.ts +30 -0
- package/src/inference/tokenizer.js +340 -164
- package/src/inference/tokenizers/bundled.js +71 -49
- package/src/loader/doppler-loader.d.ts +11 -0
- package/src/loader/doppler-loader.js +290 -15
- package/src/loader/embedding-limit-preflight.d.ts +31 -0
- package/src/loader/embedding-limit-preflight.js +216 -0
- package/src/loader/embedding-loader.d.ts +8 -2
- package/src/loader/embedding-loader.js +263 -8
- package/src/loader/experts/expert-loader.d.ts +4 -1
- package/src/loader/experts/expert-loader.js +174 -5
- package/src/loader/final-weights-loader.d.ts +5 -0
- package/src/loader/final-weights-loader.js +167 -4
- package/src/loader/layer-loader.d.ts +2 -0
- package/src/loader/layer-loader.js +74 -6
- package/src/loader/loader-types.d.ts +59 -2
- package/src/loader/manifest-config.d.ts +8 -0
- package/src/loader/shard-resolver.js +37 -4
- package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
- package/src/loader/tensors/functional-descriptor-loader.js +377 -0
- package/src/loader/tensors/tensor-loader.d.ts +36 -1
- package/src/loader/tensors/tensor-loader.js +437 -21
- package/src/loader/tensors/tensor-reader.d.ts +4 -1
- package/src/loader/tensors/tensor-reader.js +11 -4
- package/src/loader/weights.d.ts +3 -1
- package/src/memory/buffer-pool.js +12 -9
- package/src/models/diffusiongemma.d.ts +7 -0
- package/src/models/diffusiongemma.js +25 -0
- package/src/models/gemma4.js +24 -0
- package/src/models/qwen3.js +2 -2
- package/src/rules/execution-rules-contract-check.js +93 -10
- package/src/rules/inference/capability-transforms.rules.json +26 -2
- package/src/rules/inference/config.rules.json +1 -0
- package/src/rules/inference/dtype.rules.json +7 -0
- package/src/rules/inference/execution.rules.json +108 -17
- package/src/rules/inference/ffn.rules.json +18 -0
- package/src/rules/kernels/gather.rules.json +81 -0
- package/src/rules/kernels/matmul.rules.json +19 -0
- package/src/rules/kernels/moe.rules.json +3 -0
- package/src/rules/kernels/rep-penalty.rules.json +6 -0
- package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
- package/src/rules/kernels/rmsnorm.rules.json +42 -7
- package/src/rules/kernels/rope-qk.rules.json +6 -0
- package/src/rules/loader/tensor-loader.rules.json +8 -0
- package/src/rules/rule-registry.js +6 -0
- package/src/storage/artifact-storage-context.d.ts +1 -0
- package/src/storage/artifact-storage-context.js +10 -0
- package/src/tooling/browser-command-runner.d.ts +0 -3
- package/src/tooling/browser-command-runner.js +2 -14
- package/src/tooling/command-api-family-normalizers.js +4 -2
- package/src/tooling/command-api-helpers.js +2 -1
- package/src/tooling/command-api.d.ts +2 -0
- package/src/tooling/command-api.js +1 -0
- package/src/tooling/command-runner-shared.d.ts +4 -0
- package/src/tooling/command-runner-shared.js +2 -1
- package/src/tooling/diagnose-runner.js +2 -0
- package/src/tooling/litert-package-runtime.js +553 -16
- package/src/tooling/node-browser-command-runner.js +10 -4
- package/src/tooling/node-command-runner.js +7 -2
- package/src/tooling/node-convert-worker-pool.js +3 -0
- package/src/tooling/node-convert-worker.js +1 -0
- package/src/tooling/node-converter.js +12 -0
- package/src/tooling/node-webgpu.d.ts +1 -0
- package/src/tooling/node-webgpu.js +23 -11
- package/src/tooling/program-bundle.d.ts +52 -1
- package/src/tooling/rdrr-integrity-refresh.d.ts +18 -0
- package/src/tooling/source-artifact-adapter.js +7 -2
- package/src/tooling/source-package-profiles.d.ts +18 -1
- package/src/tooling/source-package-profiles.js +109 -1
- package/src/tooling/source-runtime-bundle.d.ts +1 -0
- package/src/tooling/source-runtime-bundle.js +27 -0
- package/src/tooling/source-runtime-converter-config.js +12 -3
- package/src/tooling-experimental-exports.shared.d.ts +10 -0
- package/src/tooling-experimental-exports.shared.js +10 -0
- package/src/version.js +1 -1
|
@@ -707,15 +707,26 @@ export async function encodeGemma4Image(params) {
|
|
|
707
707
|
);
|
|
708
708
|
uploadData(pooledTensor.buffer, pooled.output, 0);
|
|
709
709
|
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
710
|
+
let projected = null;
|
|
711
|
+
if (weights.projector) {
|
|
712
|
+
projected = await runMatmul(
|
|
713
|
+
pooledTensor,
|
|
714
|
+
weights.projector,
|
|
715
|
+
pooled.outputLength,
|
|
716
|
+
weights.textHiddenSize,
|
|
717
|
+
hiddenSize,
|
|
718
|
+
{ outputDtype: 'f32', transposeB: 'auto' }
|
|
719
|
+
);
|
|
720
|
+
releaseBuffer(pooledTensor.buffer);
|
|
721
|
+
} else {
|
|
722
|
+
if (hiddenSize !== weights.textHiddenSize) {
|
|
723
|
+
throw new Error(
|
|
724
|
+
`[Vision] Gemma 4 vision encoder-free mode has no projector, but vision hiddenSize (${hiddenSize}) ` +
|
|
725
|
+
`does not match text hiddenSize (${weights.textHiddenSize}).`
|
|
726
|
+
);
|
|
727
|
+
}
|
|
728
|
+
projected = pooledTensor;
|
|
729
|
+
}
|
|
719
730
|
|
|
720
731
|
return {
|
|
721
732
|
features: projected.buffer,
|
|
@@ -32,6 +32,30 @@ export interface TokenizerInitOptions {
|
|
|
32
32
|
loadTokenizerModel?: ((path?: string) => Promise<ArrayBuffer | Uint8Array | null | undefined>) | null;
|
|
33
33
|
}
|
|
34
34
|
|
|
35
|
+
export type TokenizerLoadTimingPhase =
|
|
36
|
+
| 'configResolution'
|
|
37
|
+
| 'cacheLookup'
|
|
38
|
+
| 'backendCreate'
|
|
39
|
+
| 'assetLoad'
|
|
40
|
+
| 'assetParse'
|
|
41
|
+
| 'backendLoad'
|
|
42
|
+
| 'cacheStore';
|
|
43
|
+
|
|
44
|
+
export interface TokenizerLoadTiming {
|
|
45
|
+
schemaVersion: 1;
|
|
46
|
+
source: 'doppler-tokenizer';
|
|
47
|
+
modelId: string | null;
|
|
48
|
+
status: 'running' | 'complete' | 'failed';
|
|
49
|
+
tokenizerType: string | null;
|
|
50
|
+
tokenizerFile: string | null;
|
|
51
|
+
backend: string | null;
|
|
52
|
+
assetSource: string | null;
|
|
53
|
+
cacheHit: boolean;
|
|
54
|
+
phasesMs: Record<TokenizerLoadTimingPhase, number | null>;
|
|
55
|
+
totalMs: number | null;
|
|
56
|
+
error: string | null;
|
|
57
|
+
}
|
|
58
|
+
|
|
35
59
|
/**
|
|
36
60
|
* Tokenizer wrapper that auto-detects backend from model manifest
|
|
37
61
|
* This is a thin wrapper over the backend implementations
|
|
@@ -39,6 +63,7 @@ export interface TokenizerInitOptions {
|
|
|
39
63
|
export declare class Tokenizer {
|
|
40
64
|
private backend;
|
|
41
65
|
private config;
|
|
66
|
+
private loadTiming;
|
|
42
67
|
|
|
43
68
|
/**
|
|
44
69
|
* Initialize from model manifest.
|
|
@@ -72,6 +97,11 @@ export declare class Tokenizer {
|
|
|
72
97
|
* Get high-priority token IDs for bounded runtime hot caches, or null when unavailable
|
|
73
98
|
*/
|
|
74
99
|
getHotTokenIds(limit: number): number[] | null;
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* Get report-only tokenizer initialization timing.
|
|
103
|
+
*/
|
|
104
|
+
getLoadTiming(): TokenizerLoadTiming | null;
|
|
75
105
|
}
|
|
76
106
|
|
|
77
107
|
export default Tokenizer;
|
|
@@ -8,6 +8,15 @@ import { BPETokenizer } from './tokenizers/bpe.js';
|
|
|
8
8
|
import { toArrayBuffer } from '../utils/array-buffer.js';
|
|
9
9
|
|
|
10
10
|
const BUNDLED_TOKENIZER_CACHE = new Map();
|
|
11
|
+
const TOKENIZER_LOAD_PHASES = Object.freeze([
|
|
12
|
+
'configResolution',
|
|
13
|
+
'cacheLookup',
|
|
14
|
+
'backendCreate',
|
|
15
|
+
'assetLoad',
|
|
16
|
+
'assetParse',
|
|
17
|
+
'backendLoad',
|
|
18
|
+
'cacheStore',
|
|
19
|
+
]);
|
|
11
20
|
|
|
12
21
|
function hasScheme(value) {
|
|
13
22
|
return /^[a-zA-Z][a-zA-Z0-9+.-]*:/.test(value);
|
|
@@ -26,6 +35,69 @@ function parseTokenizerJsonPayload(value) {
|
|
|
26
35
|
throw new Error('Tokenizer JSON loader must return an object, JSON string, or null.');
|
|
27
36
|
}
|
|
28
37
|
|
|
38
|
+
function roundTokenizerTimingMs(value) {
|
|
39
|
+
return Number.isFinite(value) ? Number(value.toFixed(3)) : null;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function createTokenizerLoadTiming(modelId) {
|
|
43
|
+
return {
|
|
44
|
+
schemaVersion: 1,
|
|
45
|
+
source: 'doppler-tokenizer',
|
|
46
|
+
modelId: typeof modelId === 'string' ? modelId : null,
|
|
47
|
+
status: 'running',
|
|
48
|
+
tokenizerType: null,
|
|
49
|
+
tokenizerFile: null,
|
|
50
|
+
backend: null,
|
|
51
|
+
assetSource: null,
|
|
52
|
+
cacheHit: false,
|
|
53
|
+
phasesMs: Object.fromEntries(TOKENIZER_LOAD_PHASES.map((phase) => [phase, null])),
|
|
54
|
+
totalMs: null,
|
|
55
|
+
error: null,
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
function cloneTokenizerLoadTiming(loadTiming) {
|
|
60
|
+
if (!loadTiming || typeof loadTiming !== 'object') {
|
|
61
|
+
return null;
|
|
62
|
+
}
|
|
63
|
+
return JSON.parse(JSON.stringify(loadTiming));
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
function finishTokenizerLoadPhase(loadTiming, phase, startMs) {
|
|
67
|
+
if (!loadTiming?.phasesMs || !phase) return;
|
|
68
|
+
const elapsedMs = roundTokenizerTimingMs(performance.now() - startMs);
|
|
69
|
+
const currentMs = loadTiming.phasesMs[phase];
|
|
70
|
+
loadTiming.phasesMs[phase] = Number.isFinite(currentMs)
|
|
71
|
+
? roundTokenizerTimingMs(currentMs + elapsedMs)
|
|
72
|
+
: elapsedMs;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
async function timedTokenizerLoadPhase(loadTiming, phase, run) {
|
|
76
|
+
const startMs = performance.now();
|
|
77
|
+
try {
|
|
78
|
+
return await run();
|
|
79
|
+
} finally {
|
|
80
|
+
finishTokenizerLoadPhase(loadTiming, phase, startMs);
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function timedTokenizerLoadPhaseSync(loadTiming, phase, run) {
|
|
85
|
+
const startMs = performance.now();
|
|
86
|
+
try {
|
|
87
|
+
return run();
|
|
88
|
+
} finally {
|
|
89
|
+
finishTokenizerLoadPhase(loadTiming, phase, startMs);
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
function finishTokenizerLoadTiming(loadTiming, status, startMs, error = null) {
|
|
94
|
+
loadTiming.status = status;
|
|
95
|
+
loadTiming.totalMs = roundTokenizerTimingMs(performance.now() - startMs);
|
|
96
|
+
loadTiming.error = error == null
|
|
97
|
+
? null
|
|
98
|
+
: (error instanceof Error ? error.message : String(error));
|
|
99
|
+
}
|
|
100
|
+
|
|
29
101
|
function buildBundledTokenizerCacheKey(modelId, tokenizerConfig) {
|
|
30
102
|
const tokenizerType = typeof tokenizerConfig?.type === 'string'
|
|
31
103
|
? tokenizerConfig.type.trim().toLowerCase()
|
|
@@ -55,199 +127,299 @@ export class Tokenizer {
|
|
|
55
127
|
|
|
56
128
|
config = null;
|
|
57
129
|
|
|
130
|
+
loadTiming = null;
|
|
131
|
+
|
|
58
132
|
|
|
59
133
|
async initialize(manifest, options = {}) {
|
|
134
|
+
const loadStart = performance.now();
|
|
60
135
|
const modelId = manifest?.modelId || 'unknown';
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
}
|
|
71
|
-
if (tokenizerConfig.eosToken == null && typeof eosTokenId === 'number') {
|
|
72
|
-
tokenizerConfig.eosToken = eosTokenId;
|
|
73
|
-
}
|
|
74
|
-
if (tokenizerConfig.bosToken == null && tokenizerConfig.bosTokenId != null) {
|
|
75
|
-
tokenizerConfig.bosToken = tokenizerConfig.bosTokenId;
|
|
76
|
-
}
|
|
77
|
-
if (tokenizerConfig.padToken == null && tokenizerConfig.padTokenId != null) {
|
|
78
|
-
tokenizerConfig.padToken = tokenizerConfig.padTokenId;
|
|
79
|
-
}
|
|
80
|
-
if (tokenizerConfig.unkToken == null && tokenizerConfig.unkTokenId != null) {
|
|
81
|
-
tokenizerConfig.unkToken = tokenizerConfig.unkTokenId;
|
|
82
|
-
}
|
|
136
|
+
const loadTiming = createTokenizerLoadTiming(modelId);
|
|
137
|
+
this.loadTiming = loadTiming;
|
|
138
|
+
try {
|
|
139
|
+
const tokenizerConfig = timedTokenizerLoadPhaseSync(loadTiming, 'configResolution', () => {
|
|
140
|
+
const tokenizerHints = options.tokenizerHints || {};
|
|
141
|
+
const resolvedTokenizerConfig = { ...tokenizerHints, ...(manifest.tokenizer || {}) };
|
|
142
|
+
const eosTokenId = Array.isArray(manifest.eos_token_id)
|
|
143
|
+
? manifest.eos_token_id[0]
|
|
144
|
+
: manifest.eos_token_id;
|
|
83
145
|
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
146
|
+
if (resolvedTokenizerConfig.eosToken == null && Array.isArray(resolvedTokenizerConfig.eosTokens) && resolvedTokenizerConfig.eosTokens.length > 0) {
|
|
147
|
+
resolvedTokenizerConfig.eosToken = resolvedTokenizerConfig.eosTokens[0];
|
|
148
|
+
}
|
|
149
|
+
if (resolvedTokenizerConfig.eosToken == null && typeof eosTokenId === 'number') {
|
|
150
|
+
resolvedTokenizerConfig.eosToken = eosTokenId;
|
|
151
|
+
}
|
|
152
|
+
if (resolvedTokenizerConfig.bosToken == null && resolvedTokenizerConfig.bosTokenId != null) {
|
|
153
|
+
resolvedTokenizerConfig.bosToken = resolvedTokenizerConfig.bosTokenId;
|
|
154
|
+
}
|
|
155
|
+
if (resolvedTokenizerConfig.padToken == null && resolvedTokenizerConfig.padTokenId != null) {
|
|
156
|
+
resolvedTokenizerConfig.padToken = resolvedTokenizerConfig.padTokenId;
|
|
157
|
+
}
|
|
158
|
+
if (resolvedTokenizerConfig.unkToken == null && resolvedTokenizerConfig.unkTokenId != null) {
|
|
159
|
+
resolvedTokenizerConfig.unkToken = resolvedTokenizerConfig.unkTokenId;
|
|
160
|
+
}
|
|
161
|
+
return resolvedTokenizerConfig;
|
|
162
|
+
});
|
|
163
|
+
loadTiming.tokenizerType = typeof tokenizerConfig.type === 'string' ? tokenizerConfig.type : null;
|
|
164
|
+
loadTiming.tokenizerFile = typeof tokenizerConfig.file === 'string' ? tokenizerConfig.file : null;
|
|
165
|
+
|
|
166
|
+
// Check for bundled or HuggingFace tokenizer first (eliminates transformers.js dependency)
|
|
167
|
+
const isBundled = tokenizerConfig.type === 'bundled' || tokenizerConfig.type === 'huggingface';
|
|
168
|
+
if (isBundled && tokenizerConfig.file) {
|
|
169
|
+
loadTiming.backend = 'bundled';
|
|
170
|
+
tokenizerConfig.deferSpecialTokens = true;
|
|
171
|
+
log.info('Tokenizer', `Loading ${tokenizerConfig.type} tokenizer from ${tokenizerConfig.file}`);
|
|
172
|
+
const { cacheKey, cachedBackend } = timedTokenizerLoadPhaseSync(loadTiming, 'cacheLookup', () => {
|
|
173
|
+
const resolvedCacheKey = buildBundledTokenizerCacheKey(modelId, tokenizerConfig);
|
|
174
|
+
return {
|
|
175
|
+
cacheKey: resolvedCacheKey,
|
|
176
|
+
cachedBackend: resolvedCacheKey ? BUNDLED_TOKENIZER_CACHE.get(resolvedCacheKey) : null,
|
|
177
|
+
};
|
|
178
|
+
});
|
|
179
|
+
loadTiming.cacheHit = Boolean(cachedBackend);
|
|
180
|
+
if (cachedBackend) {
|
|
181
|
+
log.info('Tokenizer', `Bundled tokenizer cache hit: ${tokenizerConfig.file}`);
|
|
182
|
+
this.backend = cachedBackend;
|
|
183
|
+
this.config = tokenizerConfig;
|
|
184
|
+
loadTiming.assetSource = 'cache';
|
|
185
|
+
finishTokenizerLoadTiming(loadTiming, 'complete', loadStart);
|
|
186
|
+
return;
|
|
187
|
+
}
|
|
97
188
|
|
|
98
|
-
|
|
189
|
+
this.backend = timedTokenizerLoadPhaseSync(
|
|
190
|
+
loadTiming,
|
|
191
|
+
'backendCreate',
|
|
192
|
+
() => new BundledTokenizer(tokenizerConfig)
|
|
193
|
+
);
|
|
99
194
|
|
|
100
|
-
|
|
195
|
+
const baseUrl = options.baseUrl;
|
|
101
196
|
|
|
102
|
-
|
|
103
|
-
|
|
197
|
+
let tokenizerJson = null;
|
|
198
|
+
const attemptedPaths = [];
|
|
104
199
|
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
200
|
+
// Try to load tokenizer.json
|
|
201
|
+
if (typeof options.loadTokenizerJson === 'function') {
|
|
202
|
+
attemptedPaths.push('custom-loader');
|
|
203
|
+
try {
|
|
204
|
+
const loaded = await timedTokenizerLoadPhase(
|
|
205
|
+
loadTiming,
|
|
206
|
+
'assetLoad',
|
|
207
|
+
() => options.loadTokenizerJson()
|
|
208
|
+
);
|
|
209
|
+
tokenizerJson = timedTokenizerLoadPhaseSync(
|
|
210
|
+
loadTiming,
|
|
211
|
+
'assetParse',
|
|
212
|
+
() => parseTokenizerJsonPayload(loaded)
|
|
213
|
+
);
|
|
214
|
+
if (tokenizerJson) {
|
|
215
|
+
loadTiming.assetSource = 'custom-loader';
|
|
216
|
+
}
|
|
217
|
+
} catch (err) {
|
|
218
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
219
|
+
log.warn('Tokenizer', `Failed to load bundled tokenizer from custom loader: ${message}`);
|
|
220
|
+
}
|
|
114
221
|
}
|
|
115
|
-
}
|
|
116
222
|
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
223
|
+
if (!tokenizerJson && baseUrl) {
|
|
224
|
+
// Load from remote URL
|
|
225
|
+
const tokenizerUrl = `${baseUrl}/${tokenizerConfig.file}`;
|
|
226
|
+
attemptedPaths.push(tokenizerUrl);
|
|
227
|
+
try {
|
|
228
|
+
const tokenizerText = await timedTokenizerLoadPhase(loadTiming, 'assetLoad', async () => {
|
|
229
|
+
const response = await fetch(tokenizerUrl);
|
|
230
|
+
if (!response.ok) {
|
|
231
|
+
throw new Error(`Failed to fetch tokenizer: ${response.status}`);
|
|
232
|
+
}
|
|
233
|
+
return response.text();
|
|
234
|
+
});
|
|
235
|
+
tokenizerJson = timedTokenizerLoadPhaseSync(
|
|
236
|
+
loadTiming,
|
|
237
|
+
'assetParse',
|
|
238
|
+
() => JSON.parse(tokenizerText)
|
|
239
|
+
);
|
|
240
|
+
loadTiming.assetSource = tokenizerUrl;
|
|
241
|
+
} catch (err) {
|
|
242
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
243
|
+
log.warn('Tokenizer', `Failed to fetch bundled tokenizer from URL: ${message}`);
|
|
244
|
+
}
|
|
245
|
+
} else if (!tokenizerJson) {
|
|
246
|
+
// Try to load from OPFS (for cached models)
|
|
247
|
+
attemptedPaths.push('OPFS:tokenizer.json');
|
|
248
|
+
try {
|
|
249
|
+
const tokenizerStr = await timedTokenizerLoadPhase(loadTiming, 'assetLoad', async () => {
|
|
250
|
+
const { loadTokenizerFromStore } = await import('../storage/shard-manager.js');
|
|
251
|
+
return loadTokenizerFromStore();
|
|
252
|
+
});
|
|
253
|
+
if (tokenizerStr) {
|
|
254
|
+
tokenizerJson = timedTokenizerLoadPhaseSync(
|
|
255
|
+
loadTiming,
|
|
256
|
+
'assetParse',
|
|
257
|
+
() => JSON.parse(tokenizerStr)
|
|
258
|
+
);
|
|
259
|
+
loadTiming.assetSource = 'OPFS:tokenizer.json';
|
|
260
|
+
}
|
|
261
|
+
} catch (err) {
|
|
262
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
263
|
+
log.warn('Tokenizer', `Failed to load bundled tokenizer from OPFS: ${message}`);
|
|
125
264
|
}
|
|
126
|
-
tokenizerJson = await response.json();
|
|
127
|
-
} catch (err) {
|
|
128
|
-
const message = err instanceof Error ? err.message : String(err);
|
|
129
|
-
log.warn('Tokenizer', `Failed to fetch bundled tokenizer from URL: ${message}`);
|
|
130
265
|
}
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
266
|
+
|
|
267
|
+
if (tokenizerJson) {
|
|
268
|
+
timedTokenizerLoadPhaseSync(
|
|
269
|
+
loadTiming,
|
|
270
|
+
'backendLoad',
|
|
271
|
+
() => (this.backend).load(tokenizerJson)
|
|
272
|
+
);
|
|
273
|
+
if (cacheKey) {
|
|
274
|
+
timedTokenizerLoadPhaseSync(
|
|
275
|
+
loadTiming,
|
|
276
|
+
'cacheStore',
|
|
277
|
+
() => BUNDLED_TOKENIZER_CACHE.set(cacheKey, this.backend)
|
|
278
|
+
);
|
|
139
279
|
}
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
280
|
+
this.config = tokenizerConfig;
|
|
281
|
+
finishTokenizerLoadTiming(loadTiming, 'complete', loadStart);
|
|
282
|
+
return;
|
|
143
283
|
}
|
|
284
|
+
|
|
285
|
+
// No external fallback - bundled tokenizer is required
|
|
286
|
+
throw new Error(
|
|
287
|
+
`[Tokenizer] Bundled tokenizer not found for model "${modelId}". ` +
|
|
288
|
+
`Expected tokenizer file: "${tokenizerConfig.file}". ` +
|
|
289
|
+
`Attempted paths: [${attemptedPaths.join(', ')}]. ` +
|
|
290
|
+
'Ensure tokenizer.json is in OPFS or model directory. ' +
|
|
291
|
+
'Clear browser storage and re-download the model.'
|
|
292
|
+
);
|
|
144
293
|
}
|
|
145
294
|
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
return;
|
|
295
|
+
let hfModel = tokenizerConfig.hfModel ?? tokenizerConfig.modelId ?? null;
|
|
296
|
+
const allowArchFallback = tokenizerConfig.allowArchFallback === true;
|
|
297
|
+
if (allowArchFallback && !hfModel) {
|
|
298
|
+
throw new Error(
|
|
299
|
+
`[Tokenizer] tokenizer.allowArchFallback requires explicit tokenizer.hfModel or tokenizer.modelId for model "${modelId}".`
|
|
300
|
+
);
|
|
153
301
|
}
|
|
154
302
|
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
`
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
303
|
+
if (hfModel) {
|
|
304
|
+
// Use Transformers.js for HuggingFace models (fallback)
|
|
305
|
+
loadTiming.backend = 'transformersjs';
|
|
306
|
+
loadTiming.assetSource = hfModel;
|
|
307
|
+
log.info('Tokenizer', `Loading from HuggingFace: ${hfModel}`);
|
|
308
|
+
this.backend = timedTokenizerLoadPhaseSync(loadTiming, 'backendCreate', () => new TransformersTokenizer({
|
|
309
|
+
modelId: hfModel,
|
|
310
|
+
...tokenizerConfig
|
|
311
|
+
}));
|
|
312
|
+
await timedTokenizerLoadPhase(loadTiming, 'backendLoad', () => (this.backend).load(hfModel));
|
|
313
|
+
} else if (tokenizerConfig.sentencepieceModel) {
|
|
314
|
+
// Load SentencePiece model
|
|
315
|
+
loadTiming.backend = 'sentencepiece';
|
|
316
|
+
loadTiming.tokenizerFile = typeof tokenizerConfig.sentencepieceModel === 'string'
|
|
317
|
+
? tokenizerConfig.sentencepieceModel
|
|
318
|
+
: loadTiming.tokenizerFile;
|
|
319
|
+
this.backend = timedTokenizerLoadPhaseSync(
|
|
320
|
+
loadTiming,
|
|
321
|
+
'backendCreate',
|
|
322
|
+
() => new SentencePieceTokenizer(tokenizerConfig)
|
|
323
|
+
);
|
|
164
324
|
|
|
165
|
-
|
|
166
|
-
const allowArchFallback = tokenizerConfig.allowArchFallback === true;
|
|
167
|
-
if (allowArchFallback && !hfModel) {
|
|
168
|
-
throw new Error(
|
|
169
|
-
`[Tokenizer] tokenizer.allowArchFallback requires explicit tokenizer.hfModel or tokenizer.modelId for model "${modelId}".`
|
|
170
|
-
);
|
|
171
|
-
}
|
|
325
|
+
// Load the model data from the provided source
|
|
172
326
|
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
modelId: hfModel,
|
|
178
|
-
...tokenizerConfig
|
|
179
|
-
});
|
|
180
|
-
await (this.backend).load(hfModel);
|
|
181
|
-
} else if (tokenizerConfig.sentencepieceModel) {
|
|
182
|
-
// Load SentencePiece model
|
|
183
|
-
this.backend = new SentencePieceTokenizer(tokenizerConfig);
|
|
184
|
-
|
|
185
|
-
// Load the model data from the provided source
|
|
186
|
-
|
|
187
|
-
let modelData;
|
|
188
|
-
const spAttemptedPaths = [];
|
|
189
|
-
if (typeof options.loadTokenizerModel === 'function') {
|
|
190
|
-
spAttemptedPaths.push('custom-loader');
|
|
191
|
-
try {
|
|
192
|
-
const loaded = await options.loadTokenizerModel(
|
|
193
|
-
typeof tokenizerConfig.sentencepieceModel === 'string'
|
|
194
|
-
? tokenizerConfig.sentencepieceModel
|
|
195
|
-
: undefined
|
|
196
|
-
);
|
|
197
|
-
if (loaded != null) {
|
|
198
|
-
modelData = toArrayBuffer(loaded, 'options.loadTokenizerModel');
|
|
199
|
-
}
|
|
200
|
-
} catch (err) {
|
|
201
|
-
const message = err instanceof Error ? err.message : String(err);
|
|
202
|
-
log.warn('Tokenizer', `Failed to load sentencepiece model from custom loader: ${message}`);
|
|
203
|
-
}
|
|
204
|
-
}
|
|
205
|
-
if (!modelData && tokenizerConfig.sentencepieceModel instanceof ArrayBuffer) {
|
|
206
|
-
spAttemptedPaths.push('inline-ArrayBuffer');
|
|
207
|
-
modelData = tokenizerConfig.sentencepieceModel;
|
|
208
|
-
} else if (!modelData && tokenizerConfig.loadShard) {
|
|
209
|
-
// Use provided shard loader
|
|
210
|
-
spAttemptedPaths.push('shard-loader');
|
|
211
|
-
modelData = await tokenizerConfig.loadShard(tokenizerConfig.sentencepieceModel);
|
|
212
|
-
} else if (!modelData && typeof tokenizerConfig.sentencepieceModel === 'string') {
|
|
213
|
-
if (options.baseUrl && !hasScheme(tokenizerConfig.sentencepieceModel)) {
|
|
214
|
-
const url = `${options.baseUrl}/${tokenizerConfig.sentencepieceModel}`;
|
|
215
|
-
spAttemptedPaths.push(url);
|
|
216
|
-
const response = await fetch(url);
|
|
217
|
-
modelData = await response.arrayBuffer();
|
|
218
|
-
} else if (hasScheme(tokenizerConfig.sentencepieceModel)) {
|
|
219
|
-
spAttemptedPaths.push(tokenizerConfig.sentencepieceModel);
|
|
220
|
-
const response = await fetch(tokenizerConfig.sentencepieceModel);
|
|
221
|
-
modelData = await response.arrayBuffer();
|
|
222
|
-
} else {
|
|
223
|
-
spAttemptedPaths.push('OPFS:tokenizer.model');
|
|
327
|
+
let modelData;
|
|
328
|
+
const spAttemptedPaths = [];
|
|
329
|
+
if (typeof options.loadTokenizerModel === 'function') {
|
|
330
|
+
spAttemptedPaths.push('custom-loader');
|
|
224
331
|
try {
|
|
225
|
-
|
|
226
|
-
|
|
332
|
+
modelData = await timedTokenizerLoadPhase(loadTiming, 'assetLoad', async () => {
|
|
333
|
+
const loaded = await options.loadTokenizerModel(
|
|
334
|
+
typeof tokenizerConfig.sentencepieceModel === 'string'
|
|
335
|
+
? tokenizerConfig.sentencepieceModel
|
|
336
|
+
: undefined
|
|
337
|
+
);
|
|
338
|
+
return loaded == null
|
|
339
|
+
? null
|
|
340
|
+
: toArrayBuffer(loaded, 'options.loadTokenizerModel');
|
|
341
|
+
});
|
|
342
|
+
if (modelData) {
|
|
343
|
+
loadTiming.assetSource = 'custom-loader';
|
|
344
|
+
}
|
|
227
345
|
} catch (err) {
|
|
228
346
|
const message = err instanceof Error ? err.message : String(err);
|
|
229
|
-
log.warn('Tokenizer', `Failed to load
|
|
347
|
+
log.warn('Tokenizer', `Failed to load sentencepiece model from custom loader: ${message}`);
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
if (!modelData && tokenizerConfig.sentencepieceModel instanceof ArrayBuffer) {
|
|
351
|
+
spAttemptedPaths.push('inline-ArrayBuffer');
|
|
352
|
+
loadTiming.assetSource = 'inline-ArrayBuffer';
|
|
353
|
+
modelData = tokenizerConfig.sentencepieceModel;
|
|
354
|
+
} else if (!modelData && tokenizerConfig.loadShard) {
|
|
355
|
+
// Use provided shard loader
|
|
356
|
+
spAttemptedPaths.push('shard-loader');
|
|
357
|
+
modelData = await timedTokenizerLoadPhase(
|
|
358
|
+
loadTiming,
|
|
359
|
+
'assetLoad',
|
|
360
|
+
() => tokenizerConfig.loadShard(tokenizerConfig.sentencepieceModel)
|
|
361
|
+
);
|
|
362
|
+
loadTiming.assetSource = 'shard-loader';
|
|
363
|
+
} else if (!modelData && typeof tokenizerConfig.sentencepieceModel === 'string') {
|
|
364
|
+
if (options.baseUrl && !hasScheme(tokenizerConfig.sentencepieceModel)) {
|
|
365
|
+
const url = `${options.baseUrl}/${tokenizerConfig.sentencepieceModel}`;
|
|
366
|
+
spAttemptedPaths.push(url);
|
|
367
|
+
modelData = await timedTokenizerLoadPhase(loadTiming, 'assetLoad', async () => {
|
|
368
|
+
const response = await fetch(url);
|
|
369
|
+
return response.arrayBuffer();
|
|
370
|
+
});
|
|
371
|
+
loadTiming.assetSource = url;
|
|
372
|
+
} else if (hasScheme(tokenizerConfig.sentencepieceModel)) {
|
|
373
|
+
spAttemptedPaths.push(tokenizerConfig.sentencepieceModel);
|
|
374
|
+
modelData = await timedTokenizerLoadPhase(loadTiming, 'assetLoad', async () => {
|
|
375
|
+
const response = await fetch(tokenizerConfig.sentencepieceModel);
|
|
376
|
+
return response.arrayBuffer();
|
|
377
|
+
});
|
|
378
|
+
loadTiming.assetSource = tokenizerConfig.sentencepieceModel;
|
|
379
|
+
} else {
|
|
380
|
+
spAttemptedPaths.push('OPFS:tokenizer.model');
|
|
381
|
+
try {
|
|
382
|
+
modelData = await timedTokenizerLoadPhase(loadTiming, 'assetLoad', async () => {
|
|
383
|
+
const { loadTokenizerModelFromStore } = await import('../storage/shard-manager.js');
|
|
384
|
+
return loadTokenizerModelFromStore();
|
|
385
|
+
});
|
|
386
|
+
if (modelData) {
|
|
387
|
+
loadTiming.assetSource = 'OPFS:tokenizer.model';
|
|
388
|
+
}
|
|
389
|
+
} catch (err) {
|
|
390
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
391
|
+
log.warn('Tokenizer', `Failed to load tokenizer.model from OPFS: ${message}`);
|
|
392
|
+
}
|
|
230
393
|
}
|
|
231
394
|
}
|
|
232
|
-
}
|
|
233
395
|
|
|
234
|
-
|
|
235
|
-
|
|
396
|
+
if (modelData) {
|
|
397
|
+
await timedTokenizerLoadPhase(loadTiming, 'backendLoad', () => (this.backend).load(modelData));
|
|
398
|
+
} else {
|
|
399
|
+
throw new Error(`Could not load SentencePiece model data. Attempted paths: [${spAttemptedPaths.join(', ')}]`);
|
|
400
|
+
}
|
|
401
|
+
} else if (tokenizerConfig.vocab && tokenizerConfig.merges) {
|
|
402
|
+
// BPE with vocab + merges
|
|
403
|
+
loadTiming.backend = 'bpe';
|
|
404
|
+
this.backend = timedTokenizerLoadPhaseSync(loadTiming, 'backendCreate', () => new BPETokenizer(tokenizerConfig));
|
|
405
|
+
timedTokenizerLoadPhaseSync(
|
|
406
|
+
loadTiming,
|
|
407
|
+
'backendLoad',
|
|
408
|
+
() => (this.backend).load(tokenizerConfig.vocab, tokenizerConfig.merges)
|
|
409
|
+
);
|
|
236
410
|
} else {
|
|
237
|
-
throw new Error(
|
|
411
|
+
throw new Error(
|
|
412
|
+
`[Tokenizer] No valid tokenizer configuration in manifest for model "${modelId}". ` +
|
|
413
|
+
'Provide tokenizer.hfModel or bundle tokenizer.json (tokenizer.type="bundled", tokenizer.file="tokenizer.json").'
|
|
414
|
+
);
|
|
238
415
|
}
|
|
239
|
-
} else if (tokenizerConfig.vocab && tokenizerConfig.merges) {
|
|
240
|
-
// BPE with vocab + merges
|
|
241
|
-
this.backend = new BPETokenizer(tokenizerConfig);
|
|
242
|
-
(this.backend).load(tokenizerConfig.vocab, tokenizerConfig.merges);
|
|
243
|
-
} else {
|
|
244
|
-
throw new Error(
|
|
245
|
-
`[Tokenizer] No valid tokenizer configuration in manifest for model "${modelId}". ` +
|
|
246
|
-
'Provide tokenizer.hfModel or bundle tokenizer.json (tokenizer.type="bundled", tokenizer.file="tokenizer.json").'
|
|
247
|
-
);
|
|
248
|
-
}
|
|
249
416
|
|
|
250
|
-
|
|
417
|
+
this.config = tokenizerConfig;
|
|
418
|
+
finishTokenizerLoadTiming(loadTiming, 'complete', loadStart);
|
|
419
|
+
} catch (error) {
|
|
420
|
+
finishTokenizerLoadTiming(loadTiming, 'failed', loadStart, error);
|
|
421
|
+
throw error;
|
|
422
|
+
}
|
|
251
423
|
}
|
|
252
424
|
encode(text) {
|
|
253
425
|
if (!this.backend) {
|
|
@@ -280,6 +452,10 @@ export class Tokenizer {
|
|
|
280
452
|
}
|
|
281
453
|
return this.backend.getHotTokenIds(limit);
|
|
282
454
|
}
|
|
455
|
+
|
|
456
|
+
getLoadTiming() {
|
|
457
|
+
return cloneTokenizerLoadTiming(this.loadTiming);
|
|
458
|
+
}
|
|
283
459
|
}
|
|
284
460
|
|
|
285
461
|
export default Tokenizer;
|