doppler-gpu 0.4.3 → 0.4.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -0
- package/README.md +67 -34
- package/assets/doppler.svg +57 -1601
- package/package.json +36 -5
- package/src/cli/cli-output.js +34 -1
- package/src/cli/doppler-cli.d.ts +18 -0
- package/src/cli/doppler-cli.js +68 -35
- package/src/cli/doppler-serve.d.ts +210 -0
- package/src/cli/doppler-serve.js +468 -42
- package/src/client/doppler-registry.json +3 -5
- package/src/client/provider.d.ts +1 -1
- package/src/client/provider.js +1 -1
- package/src/client/runtime/index.js +13 -0
- package/src/client/runtime/lora.d.ts +11 -1
- package/src/client/runtime/lora.js +28 -7
- package/src/client/runtime/model-manager.js +35 -11
- package/src/client/runtime/model-session.d.ts +2 -1
- package/src/client/runtime/model-session.js +2 -2
- package/src/client/runtime/model-source.js +20 -1
- package/src/client/runtime/types.d.ts +8 -0
- package/src/client/wrap-pipeline-handle.d.ts +62 -0
- package/src/client/wrap-pipeline-handle.js +188 -13
- package/src/config/conversion/README.md +5 -0
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +4 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +17 -9
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +14 -12
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +7 -7
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +7 -7
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +3 -3
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +3 -3
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +8 -4
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +8 -4
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +9 -5
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +8 -4
- package/src/config/goal-completion-matrix.json +552 -0
- package/src/config/kernel-path-loader.js +192 -38
- package/src/config/kernels/kernel-ref-digests.js +52 -18
- package/src/config/kernels/registry.json +3017 -744
- package/src/config/merge-contract-check.js +7 -3
- package/src/config/merge-helpers.d.ts +2 -0
- package/src/config/merge-helpers.js +1 -0
- package/src/config/merge.d.ts +1 -0
- package/src/config/merge.js +30 -0
- package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
- package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
- package/src/config/param-validator.js +4 -0
- package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
- package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
- package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
- package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
- package/src/config/schema/buffer-pool.schema.js +3 -3
- package/src/config/schema/claim-evidence-contract.schema.json +93 -0
- package/src/config/schema/command-surface-contract.schema.json +81 -0
- package/src/config/schema/converter.schema.d.ts +11 -0
- package/src/config/schema/converter.schema.js +8 -0
- package/src/config/schema/doppler.schema.d.ts +1 -0
- package/src/config/schema/doppler.schema.js +38 -6
- package/src/config/schema/execution-v1.schema.d.ts +27 -0
- package/src/config/schema/execution-v1.schema.js +28 -19
- package/src/config/schema/goal-completion-matrix.schema.json +223 -0
- package/src/config/schema/inference.schema.d.ts +11 -0
- package/src/config/schema/manifest.schema.d.ts +107 -1
- package/src/config/schema/manifest.schema.js +10 -1
- package/src/config/schema/model-artifact-contract.schema.json +119 -0
- package/src/config/schema/moe.schema.d.ts +13 -0
- package/src/config/schema/moe.schema.js +1 -1
- package/src/config/schema/policy-schema-registry.json +26 -0
- package/src/config/schema/subsystem-support-contract.schema.json +48 -0
- package/src/config/schema/training-metrics.schema.d.ts +1 -1
- package/src/config/schema/training-metrics.schema.js +4 -3
- package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +7 -7
- package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
- package/src/config/source-packages/registry.json +25 -0
- package/src/config/supported-operations.json +5 -0
- package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
- package/src/config/transforms/execution-graph-transforms.d.ts +42 -0
- package/src/config/transforms/execution-graph-transforms.js +227 -230
- package/src/converter/conversion-plan.d.ts +2 -0
- package/src/converter/conversion-plan.js +18 -4
- package/src/converter/core.d.ts +8 -0
- package/src/converter/core.js +574 -12
- package/src/converter/quantization-info.js +179 -2
- package/src/converter/quantizer.d.ts +5 -4
- package/src/converter/quantizer.js +16 -2
- package/src/experimental/adapters/lora-loader.js +109 -24
- package/src/experimental/diffusion/index.d.ts +24 -0
- package/src/experimental/diffusion/index.js +10 -0
- package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
- package/src/experimental/distribution/descriptor-transport.js +342 -0
- package/src/experimental/training/README.md +11 -0
- package/src/experimental/training/autograd.js +33 -8
- package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
- package/src/experimental/training/datasets/gepa-frontier.js +220 -0
- package/src/experimental/training/datasets/index.d.ts +13 -1
- package/src/experimental/training/datasets/index.js +8 -1
- package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
- package/src/experimental/training/datasets/teacher-traces.js +279 -0
- package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
- package/src/experimental/training/datasets/text-pairs.js +82 -5
- package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
- package/src/experimental/training/distillation/student-fixture.js +290 -32
- package/src/experimental/training/export.d.ts +9 -0
- package/src/experimental/training/export.js +113 -7
- package/src/experimental/training/index.d.ts +13 -1
- package/src/experimental/training/index.js +13 -1
- package/src/experimental/training/lora-pipeline.d.ts +143 -0
- package/src/experimental/training/lora-pipeline.js +1737 -49
- package/src/experimental/training/lora.js +23 -1
- package/src/experimental/training/loss.js +1 -1
- package/src/experimental/training/operator-agent-eval.d.ts +65 -0
- package/src/experimental/training/operator-agent-eval.js +326 -0
- package/src/experimental/training/operator-command.js +310 -51
- package/src/experimental/training/runner.d.ts +1 -0
- package/src/experimental/training/runner.js +4 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
- package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
- package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
- package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
- package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
- package/src/experimental/training/workload-packs/registry.json +186 -6
- package/src/experimental/training/workloads.d.ts +37 -0
- package/src/experimental/training/workloads.js +110 -1
- package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
- package/src/formats/rdrr/functional-descriptor.js +230 -0
- package/src/formats/rdrr/index.d.ts +1 -0
- package/src/formats/rdrr/index.js +1 -0
- package/src/formats/rdrr/parsing.js +22 -6
- package/src/formats/rdrr/source-transform-contract.js +10 -1
- package/src/formats/rdrr/storage-descriptor.js +9 -2
- package/src/formats/rdrr/types.d.ts +5 -3
- package/src/formats/rdrr/validation.js +15 -1
- package/src/gpu/command-recorder.d.ts +43 -3
- package/src/gpu/command-recorder.js +122 -25
- package/src/gpu/device.js +111 -11
- package/src/gpu/kernel-tuner/cache.js +1 -5
- package/src/gpu/kernels/attention.js +17 -9
- package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
- package/src/gpu/kernels/attention_streaming_f16.wgsl +7 -5
- package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
- package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
- package/src/gpu/kernels/cast.js +74 -67
- package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +46 -1
- package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -20
- package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
- package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +59 -44
- package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
- package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
- package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
- package/src/gpu/kernels/dispatch.js +13 -3
- package/src/gpu/kernels/fused_ffn.js +3 -4
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
- package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
- package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
- package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
- package/src/gpu/kernels/gather.d.ts +84 -1
- package/src/gpu/kernels/gather.js +353 -5
- package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
- package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
- package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
- package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
- package/src/gpu/kernels/gelu.wgsl +11 -7
- package/src/gpu/kernels/gelu_f16.wgsl +11 -7
- package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
- package/src/gpu/kernels/gemma4-route-expert.js +126 -0
- package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
- package/src/gpu/kernels/index.d.ts +71 -0
- package/src/gpu/kernels/index.js +52 -0
- package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
- package/src/gpu/kernels/kv-cache-write.js +126 -0
- package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
- package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
- package/src/gpu/kernels/lm-head-argmax.js +218 -0
- package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
- package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
- package/src/gpu/kernels/matmul-dispatch.js +37 -1
- package/src/gpu/kernels/matmul-selection.d.ts +5 -5
- package/src/gpu/kernels/matmul-selection.js +104 -9
- package/src/gpu/kernels/matmul.d.ts +1 -1
- package/src/gpu/kernels/matmul.js +151 -43
- package/src/gpu/kernels/moe.d.ts +1 -0
- package/src/gpu/kernels/moe.js +46 -18
- package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
- package/src/gpu/kernels/pipeline-cache.js +24 -0
- package/src/gpu/kernels/rep-penalty.d.ts +21 -0
- package/src/gpu/kernels/rep-penalty.js +17 -9
- package/src/gpu/kernels/residual.d.ts +1 -0
- package/src/gpu/kernels/residual.js +13 -1
- package/src/gpu/kernels/residual.wgsl +3 -3
- package/src/gpu/kernels/residual_f16.wgsl +5 -3
- package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
- package/src/gpu/kernels/residual_vec4.wgsl +6 -6
- package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
- package/src/gpu/kernels/rmsnorm-pair.js +152 -0
- package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
- package/src/gpu/kernels/rmsnorm-qk.js +128 -0
- package/src/gpu/kernels/rmsnorm.d.ts +31 -0
- package/src/gpu/kernels/rmsnorm.js +32 -7
- package/src/gpu/kernels/rmsnorm.wgsl +10 -6
- package/src/gpu/kernels/rmsnorm_f16.wgsl +7 -3
- package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
- package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
- package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
- package/src/gpu/kernels/rope-qk.d.ts +41 -0
- package/src/gpu/kernels/rope-qk.js +143 -0
- package/src/gpu/kernels/rope_qk.wgsl +95 -0
- package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
- package/src/gpu/kernels/rule-registry.js +50 -10
- package/src/gpu/kernels/sample.js +212 -187
- package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
- package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
- package/src/gpu/kernels/shader-cache.js +1 -21
- package/src/gpu/kernels/soft-embedding.d.ts +28 -0
- package/src/gpu/kernels/soft-embedding.js +447 -0
- package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
- package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
- package/src/gpu/kernels/softmax.wgsl +5 -80
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
- package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
- package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
- package/src/gpu/kernels/uniform-utils.js +2 -5
- package/src/gpu/kernels/utils.d.ts +16 -1
- package/src/gpu/kernels/utils.js +42 -13
- package/src/gpu/readback-utils.js +10 -2
- package/src/gpu/uniform-cache.d.ts +10 -1
- package/src/gpu/uniform-cache.js +25 -7
- package/src/gpu/weight-buffer.d.ts +76 -7
- package/src/gpu/weight-buffer.js +85 -1
- package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
- package/src/inference/browser-harness-suite-helpers.d.ts +9 -0
- package/src/inference/browser-harness-suite-helpers.js +438 -2
- package/src/inference/browser-harness-text-helpers.d.ts +77 -0
- package/src/inference/browser-harness-text-helpers.js +236 -11
- package/src/inference/browser-harness.d.ts +8 -0
- package/src/inference/browser-harness.js +292 -8
- package/src/inference/decode-ring.d.ts +1 -0
- package/src/inference/decode-ring.js +7 -5
- package/src/inference/kv-cache/base.d.ts +25 -0
- package/src/inference/kv-cache/base.js +109 -1
- package/src/inference/kv-cache/mixed-geometry.d.ts +10 -0
- package/src/inference/kv-cache/mixed-geometry.js +109 -2
- package/src/inference/kv-cache/sliding-window.d.ts +14 -0
- package/src/inference/kv-cache/sliding-window.js +88 -0
- package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
- package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
- package/src/inference/moe-router.d.ts +14 -1
- package/src/inference/moe-router.js +24 -1
- package/src/inference/pipelines/audio/gemma4.js +52 -1
- package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
- package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
- package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
- package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
- package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
- package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
- package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
- package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
- package/src/inference/pipelines/text/attention/projections.d.ts +71 -2
- package/src/inference/pipelines/text/attention/projections.js +329 -49
- package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
- package/src/inference/pipelines/text/attention/record.js +446 -73
- package/src/inference/pipelines/text/attention/run.js +251 -49
- package/src/inference/pipelines/text/attention/types.d.ts +5 -1
- package/src/inference/pipelines/text/config.d.ts +22 -1
- package/src/inference/pipelines/text/config.js +199 -22
- package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
- package/src/inference/pipelines/text/dtype-contract.js +4 -1
- package/src/inference/pipelines/text/embed.d.ts +4 -3
- package/src/inference/pipelines/text/embed.js +16 -6
- package/src/inference/pipelines/text/execution-plan.d.ts +8 -0
- package/src/inference/pipelines/text/execution-plan.js +6 -0
- package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -0
- package/src/inference/pipelines/text/execution-runtime-builders.js +44 -28
- package/src/inference/pipelines/text/execution-v1.d.ts +2 -0
- package/src/inference/pipelines/text/execution-v1.js +373 -139
- package/src/inference/pipelines/text/ffn/dense.js +44 -13
- package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
- package/src/inference/pipelines/text/ffn/moe.js +9 -2
- package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
- package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
- package/src/inference/pipelines/text/ffn/standard.js +44 -4
- package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
- package/src/inference/pipelines/text/generator-decode-policy.js +2 -12
- package/src/inference/pipelines/text/generator-helpers.js +5 -2
- package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
- package/src/inference/pipelines/text/generator-prefill-helpers.js +67 -1
- package/src/inference/pipelines/text/generator-runtime.d.ts +1 -0
- package/src/inference/pipelines/text/generator-steps.d.ts +38 -3
- package/src/inference/pipelines/text/generator-steps.js +268 -77
- package/src/inference/pipelines/text/generator.d.ts +39 -0
- package/src/inference/pipelines/text/generator.js +1105 -18
- package/src/inference/pipelines/text/init.d.ts +8 -1
- package/src/inference/pipelines/text/init.js +30 -1
- package/src/inference/pipelines/text/layer-plan-gpu.js +62 -14
- package/src/inference/pipelines/text/layer.js +272 -25
- package/src/inference/pipelines/text/linear-attention.js +6 -4
- package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
- package/src/inference/pipelines/text/logits/gpu.js +417 -66
- package/src/inference/pipelines/text/logits/index.d.ts +2 -2
- package/src/inference/pipelines/text/logits/index.js +96 -42
- package/src/inference/pipelines/text/logits/types.d.ts +2 -2
- package/src/inference/pipelines/text/lora-apply.js +17 -0
- package/src/inference/pipelines/text/model-load.js +5 -1
- package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
- package/src/inference/pipelines/text/moe-gpu.js +445 -16
- package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
- package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
- package/src/inference/pipelines/text/ops.d.ts +29 -2
- package/src/inference/pipelines/text/ops.js +19 -0
- package/src/inference/pipelines/text/per-layer-inputs.js +5 -0
- package/src/inference/pipelines/text/probes.js +4 -5
- package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
- package/src/inference/pipelines/text/sampling-config.js +103 -64
- package/src/inference/pipelines/text/sampling.js +13 -9
- package/src/inference/pipelines/text/state.js +17 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
- package/src/inference/pipelines/text/tsir-fixture-writer.js +8 -9
- package/src/inference/pipelines/text/types.d.ts +84 -3
- package/src/inference/pipelines/text/weights.d.ts +4 -2
- package/src/inference/pipelines/text/weights.js +4 -5
- package/src/inference/pipelines/text.d.ts +24 -0
- package/src/inference/pipelines/text.js +162 -38
- package/src/inference/pipelines/vision/gemma4.js +20 -9
- package/src/inference/tokenizer.d.ts +30 -0
- package/src/inference/tokenizer.js +340 -164
- package/src/inference/tokenizers/bundled.js +71 -49
- package/src/loader/doppler-loader.d.ts +11 -0
- package/src/loader/doppler-loader.js +290 -15
- package/src/loader/embedding-limit-preflight.d.ts +31 -0
- package/src/loader/embedding-limit-preflight.js +216 -0
- package/src/loader/embedding-loader.d.ts +8 -2
- package/src/loader/embedding-loader.js +263 -8
- package/src/loader/experts/expert-loader.d.ts +4 -1
- package/src/loader/experts/expert-loader.js +174 -5
- package/src/loader/final-weights-loader.d.ts +5 -0
- package/src/loader/final-weights-loader.js +167 -4
- package/src/loader/layer-loader.d.ts +2 -0
- package/src/loader/layer-loader.js +74 -6
- package/src/loader/loader-types.d.ts +59 -2
- package/src/loader/manifest-config.d.ts +8 -0
- package/src/loader/shard-resolver.js +37 -4
- package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
- package/src/loader/tensors/functional-descriptor-loader.js +377 -0
- package/src/loader/tensors/tensor-loader.d.ts +36 -1
- package/src/loader/tensors/tensor-loader.js +437 -21
- package/src/loader/tensors/tensor-reader.d.ts +4 -1
- package/src/loader/tensors/tensor-reader.js +11 -4
- package/src/loader/weights.d.ts +3 -1
- package/src/memory/buffer-pool.js +12 -9
- package/src/models/diffusiongemma.d.ts +7 -0
- package/src/models/diffusiongemma.js +25 -0
- package/src/models/gemma4.js +24 -0
- package/src/models/qwen3.js +2 -2
- package/src/rules/execution-rules-contract-check.js +93 -10
- package/src/rules/inference/capability-transforms.rules.json +26 -2
- package/src/rules/inference/config.rules.json +1 -0
- package/src/rules/inference/dtype.rules.json +7 -0
- package/src/rules/inference/execution.rules.json +108 -17
- package/src/rules/inference/ffn.rules.json +18 -0
- package/src/rules/kernels/gather.rules.json +81 -0
- package/src/rules/kernels/matmul.rules.json +19 -0
- package/src/rules/kernels/moe.rules.json +3 -0
- package/src/rules/kernels/rep-penalty.rules.json +6 -0
- package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
- package/src/rules/kernels/rmsnorm.rules.json +42 -7
- package/src/rules/kernels/rope-qk.rules.json +6 -0
- package/src/rules/loader/tensor-loader.rules.json +8 -0
- package/src/rules/rule-registry.js +6 -0
- package/src/storage/artifact-storage-context.d.ts +1 -0
- package/src/storage/artifact-storage-context.js +10 -0
- package/src/tooling/browser-command-runner.d.ts +0 -3
- package/src/tooling/browser-command-runner.js +2 -14
- package/src/tooling/command-api-family-normalizers.js +4 -2
- package/src/tooling/command-api-helpers.js +2 -1
- package/src/tooling/command-api.d.ts +2 -0
- package/src/tooling/command-api.js +1 -0
- package/src/tooling/command-runner-shared.d.ts +4 -0
- package/src/tooling/command-runner-shared.js +2 -1
- package/src/tooling/diagnose-runner.js +2 -0
- package/src/tooling/litert-package-runtime.js +553 -16
- package/src/tooling/node-browser-command-runner.js +10 -4
- package/src/tooling/node-command-runner.js +7 -2
- package/src/tooling/node-convert-worker-pool.js +3 -0
- package/src/tooling/node-convert-worker.js +1 -0
- package/src/tooling/node-converter.js +12 -0
- package/src/tooling/node-webgpu.d.ts +1 -0
- package/src/tooling/node-webgpu.js +23 -11
- package/src/tooling/program-bundle.d.ts +52 -1
- package/src/tooling/rdrr-integrity-refresh.d.ts +18 -0
- package/src/tooling/source-artifact-adapter.js +7 -2
- package/src/tooling/source-package-profiles.d.ts +18 -1
- package/src/tooling/source-package-profiles.js +109 -1
- package/src/tooling/source-runtime-bundle.d.ts +1 -0
- package/src/tooling/source-runtime-bundle.js +27 -0
- package/src/tooling/source-runtime-converter-config.js +12 -3
- package/src/tooling-experimental-exports.shared.d.ts +10 -0
- package/src/tooling-experimental-exports.shared.js +10 -0
- package/src/version.js +1 -1
|
@@ -21,6 +21,38 @@ function normalizeText(value) {
|
|
|
21
21
|
return String(value || '').trim();
|
|
22
22
|
}
|
|
23
23
|
|
|
24
|
+
function normalizeLiteRTStorageEncoding(value, fieldName) {
|
|
25
|
+
const storageEncoding = normalizeText(value).toLowerCase();
|
|
26
|
+
if (storageEncoding !== 'signed' && storageEncoding !== 'offset_binary') {
|
|
27
|
+
throw new Error(
|
|
28
|
+
`direct-source runtime: ${fieldName} must be "signed" or "offset_binary", got "${value ?? 'missing'}".`
|
|
29
|
+
);
|
|
30
|
+
}
|
|
31
|
+
return storageEncoding;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function normalizeLayerIndexList(value, numLayers, fieldName) {
|
|
35
|
+
if (!Array.isArray(value)) {
|
|
36
|
+
throw new Error(`direct-source runtime: ${fieldName} must be an array of layer indices.`);
|
|
37
|
+
}
|
|
38
|
+
const indexes = [];
|
|
39
|
+
let previous = -1;
|
|
40
|
+
for (const rawIndex of value) {
|
|
41
|
+
const layerIndex = Number(rawIndex);
|
|
42
|
+
if (!Number.isInteger(layerIndex) || layerIndex < 0 || layerIndex >= numLayers) {
|
|
43
|
+
throw new Error(
|
|
44
|
+
`direct-source runtime: ${fieldName} contains invalid layer index "${String(rawIndex)}".`
|
|
45
|
+
);
|
|
46
|
+
}
|
|
47
|
+
if (layerIndex <= previous) {
|
|
48
|
+
throw new Error(`direct-source runtime: ${fieldName} must be sorted and unique.`);
|
|
49
|
+
}
|
|
50
|
+
indexes.push(layerIndex);
|
|
51
|
+
previous = layerIndex;
|
|
52
|
+
}
|
|
53
|
+
return indexes;
|
|
54
|
+
}
|
|
55
|
+
|
|
24
56
|
function createVirtualFile(path, offset, size, kind, options = {}) {
|
|
25
57
|
return {
|
|
26
58
|
path,
|
|
@@ -72,6 +104,22 @@ function resolvePackageTokenizerConfig(sourceKind, runtimeProfile) {
|
|
|
72
104
|
return null;
|
|
73
105
|
}
|
|
74
106
|
|
|
107
|
+
function throwIfUnsupportedPackageProfile(sourceKind, packageBasename, profile, packageConfig) {
|
|
108
|
+
const unsupported = packageConfig?.unsupported;
|
|
109
|
+
if (!unsupported || typeof unsupported !== 'object') {
|
|
110
|
+
return;
|
|
111
|
+
}
|
|
112
|
+
const code = normalizeText(unsupported.code) || 'unsupported-package-contract';
|
|
113
|
+
const message = normalizeText(unsupported.message);
|
|
114
|
+
const recommendation = normalizeText(unsupported.recommendation);
|
|
115
|
+
throw new Error(
|
|
116
|
+
`direct-source runtime: ${sourceKind} artifact "${packageBasename}" matches package profile ` +
|
|
117
|
+
`"${profile.id}" but is not supported by Doppler direct-source import (${code}).` +
|
|
118
|
+
(message ? ` ${message}` : '') +
|
|
119
|
+
(recommendation ? ` ${recommendation}` : '')
|
|
120
|
+
);
|
|
121
|
+
}
|
|
122
|
+
|
|
75
123
|
function findTFLiteMetadataEntry(parsedTFLite, name) {
|
|
76
124
|
const target = normalizeText(name);
|
|
77
125
|
if (!target) {
|
|
@@ -352,6 +400,210 @@ function createLiteRTFloatTensor(rawTensor, sourcePath, canonicalName, role, gro
|
|
|
352
400
|
};
|
|
353
401
|
}
|
|
354
402
|
|
|
403
|
+
function shapeEquals(actualShape, expectedShape) {
|
|
404
|
+
if (!Array.isArray(actualShape) || !Array.isArray(expectedShape)) {
|
|
405
|
+
return false;
|
|
406
|
+
}
|
|
407
|
+
if (actualShape.length !== expectedShape.length) {
|
|
408
|
+
return false;
|
|
409
|
+
}
|
|
410
|
+
for (let index = 0; index < expectedShape.length; index += 1) {
|
|
411
|
+
if (Number(actualShape[index]) !== Number(expectedShape[index])) {
|
|
412
|
+
return false;
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
return true;
|
|
416
|
+
}
|
|
417
|
+
|
|
418
|
+
function isLiteRTTensorDtype(rawTensor, sourceDtype) {
|
|
419
|
+
const dtypeId = Number(rawTensor?.dtypeId);
|
|
420
|
+
if (sourceDtype === 'F32') {
|
|
421
|
+
return dtypeId === 0 || normalizeText(rawTensor?.dtype).toUpperCase() === 'F32';
|
|
422
|
+
}
|
|
423
|
+
if (sourceDtype === 'INT4') {
|
|
424
|
+
return dtypeId === 17 || normalizeText(rawTensor?.sourceDtype).toUpperCase() === 'INT4';
|
|
425
|
+
}
|
|
426
|
+
return false;
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
function createGemma412BSplitCursor(parsedTFLite, sourcePath) {
|
|
430
|
+
return {
|
|
431
|
+
tensors: Array.isArray(parsedTFLite?.tensors) ? parsedTFLite.tensors : [],
|
|
432
|
+
sourcePath,
|
|
433
|
+
index: 0,
|
|
434
|
+
lastHiddenNorm: null,
|
|
435
|
+
};
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
function takeNextLiteRTTensor(cursor, expectedShape, sourceDtype, label, options = {}) {
|
|
439
|
+
const optional = options.optional === true;
|
|
440
|
+
for (let tensorIndex = cursor.index; tensorIndex < cursor.tensors.length; tensorIndex += 1) {
|
|
441
|
+
const tensor = cursor.tensors[tensorIndex];
|
|
442
|
+
if (!isLiteRTTensorDtype(tensor, sourceDtype)) {
|
|
443
|
+
continue;
|
|
444
|
+
}
|
|
445
|
+
if (!shapeEquals(tensor.shape, expectedShape)) {
|
|
446
|
+
continue;
|
|
447
|
+
}
|
|
448
|
+
cursor.index = tensorIndex + 1;
|
|
449
|
+
return tensor;
|
|
450
|
+
}
|
|
451
|
+
if (optional) {
|
|
452
|
+
return null;
|
|
453
|
+
}
|
|
454
|
+
throw new Error(
|
|
455
|
+
`direct-source runtime: Gemma 4 12B split-section adapter could not find ${sourceDtype} tensor ` +
|
|
456
|
+
`${label} with shape ${JSON.stringify(expectedShape)} after tensor index ${cursor.index}.`
|
|
457
|
+
);
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
function takeOptionalConsecutiveLiteRTTensor(cursor, expectedShape, sourceDtype) {
|
|
461
|
+
const tensor = cursor.tensors[cursor.index] ?? null;
|
|
462
|
+
if (!isLiteRTTensorDtype(tensor, sourceDtype) || !shapeEquals(tensor.shape, expectedShape)) {
|
|
463
|
+
return null;
|
|
464
|
+
}
|
|
465
|
+
cursor.index += 1;
|
|
466
|
+
return tensor;
|
|
467
|
+
}
|
|
468
|
+
|
|
469
|
+
function collectGemma412BLayerScalarTensors(prefillTFLite, numLayers, packageConfig) {
|
|
470
|
+
const expectedLayers = normalizeLayerIndexList(
|
|
471
|
+
packageConfig?.layerScalarLayers,
|
|
472
|
+
numLayers,
|
|
473
|
+
'package.litertlm.layerScalarLayers'
|
|
474
|
+
);
|
|
475
|
+
if (expectedLayers.length !== numLayers) {
|
|
476
|
+
const missingValue = Number(packageConfig?.missingLayerScalarValue);
|
|
477
|
+
if (!Number.isFinite(missingValue) || missingValue !== 1) {
|
|
478
|
+
throw new Error(
|
|
479
|
+
'direct-source runtime: package.litertlm.missingLayerScalarValue must be 1 when ' +
|
|
480
|
+
'package.litertlm.layerScalarLayers does not cover every layer.'
|
|
481
|
+
);
|
|
482
|
+
}
|
|
483
|
+
}
|
|
484
|
+
|
|
485
|
+
const expected = new Set(expectedLayers);
|
|
486
|
+
const layerScalars = new Map();
|
|
487
|
+
for (const tensor of prefillTFLite.tensors ?? []) {
|
|
488
|
+
if (!isLiteRTTensorDtype(tensor, 'F32') || !shapeEquals(tensor.shape, [])) {
|
|
489
|
+
continue;
|
|
490
|
+
}
|
|
491
|
+
const match = String(tensor.name ?? '').match(/Gemma4UnifiedTextDecoderLayer_(\d+);?$/);
|
|
492
|
+
if (!match) {
|
|
493
|
+
continue;
|
|
494
|
+
}
|
|
495
|
+
const layerIndex = Number(match[1]);
|
|
496
|
+
if (!Number.isInteger(layerIndex) || layerIndex < 0 || layerIndex >= numLayers) {
|
|
497
|
+
throw new Error(
|
|
498
|
+
`direct-source runtime: Gemma 4 12B layer_scalar tensor "${tensor.name}" ` +
|
|
499
|
+
`has out-of-range layer index ${String(match[1])}.`
|
|
500
|
+
);
|
|
501
|
+
}
|
|
502
|
+
if (!expected.has(layerIndex)) {
|
|
503
|
+
throw new Error(
|
|
504
|
+
`direct-source runtime: Gemma 4 12B layer_scalar tensor for layer ${layerIndex} ` +
|
|
505
|
+
'is present in the LiteRT graph but missing from package.litertlm.layerScalarLayers.'
|
|
506
|
+
);
|
|
507
|
+
}
|
|
508
|
+
if (layerScalars.has(layerIndex)) {
|
|
509
|
+
throw new Error(
|
|
510
|
+
`direct-source runtime: Gemma 4 12B LiteRT graph has duplicate layer_scalar tensors for layer ${layerIndex}.`
|
|
511
|
+
);
|
|
512
|
+
}
|
|
513
|
+
layerScalars.set(layerIndex, tensor);
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
for (const layerIndex of expectedLayers) {
|
|
517
|
+
if (!layerScalars.has(layerIndex)) {
|
|
518
|
+
throw new Error(
|
|
519
|
+
`direct-source runtime: Gemma 4 12B split-section adapter expected layer_scalar for layer ${layerIndex}, ` +
|
|
520
|
+
'but the LiteRT graph did not contain a matching Gemma4UnifiedTextDecoderLayer scalar tensor.'
|
|
521
|
+
);
|
|
522
|
+
}
|
|
523
|
+
}
|
|
524
|
+
return layerScalars;
|
|
525
|
+
}
|
|
526
|
+
|
|
527
|
+
function createLiteRTFixedAffineInt4Tensor(
|
|
528
|
+
rawTensor,
|
|
529
|
+
sourcePath,
|
|
530
|
+
canonicalName,
|
|
531
|
+
role,
|
|
532
|
+
group,
|
|
533
|
+
logicalShape,
|
|
534
|
+
fixedScale,
|
|
535
|
+
storageEncoding
|
|
536
|
+
) {
|
|
537
|
+
if (!rawTensor || typeof rawTensor !== 'object') {
|
|
538
|
+
throw new Error(`direct-source runtime: missing LiteRT tensor "${canonicalName}".`);
|
|
539
|
+
}
|
|
540
|
+
if (!Array.isArray(logicalShape) || logicalShape.length !== 2) {
|
|
541
|
+
throw new Error(
|
|
542
|
+
`direct-source runtime: LiteRT tensor "${canonicalName}" requires an explicit 2D logical shape.`
|
|
543
|
+
);
|
|
544
|
+
}
|
|
545
|
+
if (!shapeEquals(rawTensor.shape, logicalShape)) {
|
|
546
|
+
throw new Error(
|
|
547
|
+
`direct-source runtime: LiteRT tensor "${rawTensor.name}" shape ${JSON.stringify(rawTensor.shape)} ` +
|
|
548
|
+
`does not match expected split-section logical shape ${JSON.stringify(logicalShape)} for "${canonicalName}".`
|
|
549
|
+
);
|
|
550
|
+
}
|
|
551
|
+
const expectedSize = computePackedByteSize(logicalShape, 'INT4', canonicalName);
|
|
552
|
+
if (Number(rawTensor.size) !== expectedSize) {
|
|
553
|
+
throw new Error(
|
|
554
|
+
`direct-source runtime: LiteRT tensor "${rawTensor.name}" size ${rawTensor.size} does not match ` +
|
|
555
|
+
`INT4 packed shape ${JSON.stringify(logicalShape)} for "${canonicalName}".`
|
|
556
|
+
);
|
|
557
|
+
}
|
|
558
|
+
if (!Number.isFinite(fixedScale) || fixedScale <= 0) {
|
|
559
|
+
throw new Error(
|
|
560
|
+
`direct-source runtime: LiteRT tensor "${canonicalName}" requires package.litertlm.fixedInt4Scale > 0.`
|
|
561
|
+
);
|
|
562
|
+
}
|
|
563
|
+
return {
|
|
564
|
+
name: canonicalName,
|
|
565
|
+
shape: [Number(logicalShape[0]), Number(logicalShape[1])],
|
|
566
|
+
dtype: 'F16',
|
|
567
|
+
offset: rawTensor.offset,
|
|
568
|
+
size: rawTensor.size,
|
|
569
|
+
sourcePath,
|
|
570
|
+
role,
|
|
571
|
+
...(group ? { group } : {}),
|
|
572
|
+
sourceTransform: {
|
|
573
|
+
kind: 'affine_dequant',
|
|
574
|
+
scheme: 'per_tensor_affine',
|
|
575
|
+
sourceDtype: 'INT4',
|
|
576
|
+
targetDtype: 'F16',
|
|
577
|
+
storageEncoding,
|
|
578
|
+
scale: fixedScale,
|
|
579
|
+
zeroPoint: 0,
|
|
580
|
+
},
|
|
581
|
+
};
|
|
582
|
+
}
|
|
583
|
+
|
|
584
|
+
function resolveGemma412BLayerTypes(runtimeProfile) {
|
|
585
|
+
const explicitLayerTypes = runtimeProfile?.manifestInference?.layerPattern?.layerTypes;
|
|
586
|
+
if (Array.isArray(explicitLayerTypes) && explicitLayerTypes.length > 0) {
|
|
587
|
+
return explicitLayerTypes.map((value) => normalizeText(value));
|
|
588
|
+
}
|
|
589
|
+
const numLayers = Number(runtimeProfile?.architecture?.numLayers ?? 0);
|
|
590
|
+
if (!Number.isInteger(numLayers) || numLayers <= 0) {
|
|
591
|
+
throw new Error('direct-source runtime: Gemma 4 12B split-section adapter is missing architecture.numLayers.');
|
|
592
|
+
}
|
|
593
|
+
const layerTypes = [];
|
|
594
|
+
for (let layerIndex = 0; layerIndex < numLayers; layerIndex += 1) {
|
|
595
|
+
layerTypes.push(isGemma4GlobalLayer(runtimeProfile, layerIndex) ? 'full_attention' : 'sliding_attention');
|
|
596
|
+
}
|
|
597
|
+
return layerTypes;
|
|
598
|
+
}
|
|
599
|
+
|
|
600
|
+
function addFloatToNormalized(normalized, rawTensor, sourcePath, canonicalName, role, group = null) {
|
|
601
|
+
if (!rawTensor) {
|
|
602
|
+
throw new Error(`direct-source runtime: missing LiteRT tensor "${canonicalName}".`);
|
|
603
|
+
}
|
|
604
|
+
normalized.push(createLiteRTFloatTensor(rawTensor, sourcePath, canonicalName, role, group));
|
|
605
|
+
}
|
|
606
|
+
|
|
355
607
|
function createLiteRTRowwiseTensor(
|
|
356
608
|
rawTensor,
|
|
357
609
|
scaleTensor,
|
|
@@ -773,6 +1025,191 @@ function createLiteRTBlockedAxisTensor(
|
|
|
773
1025
|
};
|
|
774
1026
|
}
|
|
775
1027
|
|
|
1028
|
+
function normalizeGemma412BSplitLiteRTTensors(parsedTFLitesByPath, virtualFiles, runtimeProfile, packageConfig) {
|
|
1029
|
+
const embedderModelType = normalizeText(packageConfig?.embedderTFLiteModelType);
|
|
1030
|
+
const prefillModelType = normalizeText(packageConfig?.tfliteModelType);
|
|
1031
|
+
const embedderSourceFile = virtualFiles.find((entry) => entry.kind === 'tflite_embedder_model') ?? null;
|
|
1032
|
+
const prefillSourceFile = virtualFiles.find((entry) => entry.kind === 'tflite_model') ?? null;
|
|
1033
|
+
if (!embedderModelType || !embedderSourceFile) {
|
|
1034
|
+
throw new Error(
|
|
1035
|
+
'direct-source runtime: Gemma 4 12B split-section adapter requires package.litertlm.embedderTFLiteModelType.'
|
|
1036
|
+
);
|
|
1037
|
+
}
|
|
1038
|
+
if (!prefillModelType || !prefillSourceFile) {
|
|
1039
|
+
throw new Error(
|
|
1040
|
+
'direct-source runtime: Gemma 4 12B split-section adapter requires package.litertlm.tfliteModelType.'
|
|
1041
|
+
);
|
|
1042
|
+
}
|
|
1043
|
+
const embedderTFLite = parsedTFLitesByPath?.get(embedderSourceFile.path) ?? null;
|
|
1044
|
+
const prefillTFLite = parsedTFLitesByPath?.get(prefillSourceFile.path) ?? null;
|
|
1045
|
+
if (!embedderTFLite || !prefillTFLite) {
|
|
1046
|
+
throw new Error('direct-source runtime: Gemma 4 12B split-section adapter did not receive both parsed TFLite sections.');
|
|
1047
|
+
}
|
|
1048
|
+
|
|
1049
|
+
const arch = runtimeProfile?.architecture ?? {};
|
|
1050
|
+
const hiddenSize = Number(arch.hiddenSize ?? 0);
|
|
1051
|
+
const vocabSize = Number(arch.vocabSize ?? 0);
|
|
1052
|
+
const intermediateSize = Number(arch.intermediateSize ?? 0);
|
|
1053
|
+
const numAttentionHeads = Number(arch.numAttentionHeads ?? 0);
|
|
1054
|
+
const numKeyValueHeads = Number(arch.numKeyValueHeads ?? 0);
|
|
1055
|
+
const numGlobalKeyValueHeads = Number(arch.numGlobalKeyValueHeads ?? numKeyValueHeads);
|
|
1056
|
+
const headDim = Number(arch.headDim ?? 0);
|
|
1057
|
+
const globalHeadDim = Number(arch.globalHeadDim ?? headDim);
|
|
1058
|
+
const fixedInt4Scale = Number(packageConfig?.fixedInt4Scale);
|
|
1059
|
+
const fixedInt4StorageEncoding = normalizeLiteRTStorageEncoding(
|
|
1060
|
+
packageConfig?.fixedInt4StorageEncoding,
|
|
1061
|
+
'package.litertlm.fixedInt4StorageEncoding'
|
|
1062
|
+
);
|
|
1063
|
+
if (
|
|
1064
|
+
!Number.isInteger(hiddenSize)
|
|
1065
|
+
|| hiddenSize <= 0
|
|
1066
|
+
|| !Number.isInteger(vocabSize)
|
|
1067
|
+
|| vocabSize <= 0
|
|
1068
|
+
|| !Number.isInteger(intermediateSize)
|
|
1069
|
+
|| intermediateSize <= 0
|
|
1070
|
+
|| !Number.isInteger(numAttentionHeads)
|
|
1071
|
+
|| numAttentionHeads <= 0
|
|
1072
|
+
|| !Number.isInteger(numKeyValueHeads)
|
|
1073
|
+
|| numKeyValueHeads <= 0
|
|
1074
|
+
|| !Number.isInteger(numGlobalKeyValueHeads)
|
|
1075
|
+
|| numGlobalKeyValueHeads <= 0
|
|
1076
|
+
|| !Number.isInteger(headDim)
|
|
1077
|
+
|| headDim <= 0
|
|
1078
|
+
|| !Number.isInteger(globalHeadDim)
|
|
1079
|
+
|| globalHeadDim <= 0
|
|
1080
|
+
|| !Number.isFinite(fixedInt4Scale)
|
|
1081
|
+
|| fixedInt4Scale <= 0
|
|
1082
|
+
) {
|
|
1083
|
+
throw new Error(
|
|
1084
|
+
'direct-source runtime: Gemma 4 12B split-section adapter requires explicit architecture sizes and fixedInt4Scale.'
|
|
1085
|
+
);
|
|
1086
|
+
}
|
|
1087
|
+
|
|
1088
|
+
const normalized = [];
|
|
1089
|
+
const embedderTensor = embedderTFLite.tensors.find((tensor) => (
|
|
1090
|
+
isLiteRTTensorDtype(tensor, 'INT4') && shapeEquals(tensor.shape, [vocabSize, hiddenSize])
|
|
1091
|
+
)) ?? null;
|
|
1092
|
+
normalized.push(
|
|
1093
|
+
createLiteRTFixedAffineInt4Tensor(
|
|
1094
|
+
embedderTensor,
|
|
1095
|
+
embedderSourceFile.path,
|
|
1096
|
+
'model.language_model.embed_tokens.weight',
|
|
1097
|
+
'embedding',
|
|
1098
|
+
'embed',
|
|
1099
|
+
[vocabSize, hiddenSize],
|
|
1100
|
+
fixedInt4Scale,
|
|
1101
|
+
fixedInt4StorageEncoding
|
|
1102
|
+
)
|
|
1103
|
+
);
|
|
1104
|
+
|
|
1105
|
+
const cursor = createGemma412BSplitCursor(prefillTFLite, prefillSourceFile.path);
|
|
1106
|
+
const leadingFinalNorm = takeNextLiteRTTensor(cursor, [hiddenSize], 'F32', 'model.language_model.norm.weight', {
|
|
1107
|
+
optional: true,
|
|
1108
|
+
});
|
|
1109
|
+
if (leadingFinalNorm) {
|
|
1110
|
+
cursor.lastHiddenNorm = leadingFinalNorm;
|
|
1111
|
+
takeOptionalConsecutiveLiteRTTensor(cursor, [numGlobalKeyValueHeads * globalHeadDim, hiddenSize], 'INT4');
|
|
1112
|
+
takeOptionalConsecutiveLiteRTTensor(cursor, [globalHeadDim], 'F32');
|
|
1113
|
+
}
|
|
1114
|
+
const layerTypes = resolveGemma412BLayerTypes(runtimeProfile);
|
|
1115
|
+
const layerScalarTensors = collectGemma412BLayerScalarTensors(
|
|
1116
|
+
prefillTFLite,
|
|
1117
|
+
layerTypes.length,
|
|
1118
|
+
packageConfig
|
|
1119
|
+
);
|
|
1120
|
+
for (let layerIndex = 0; layerIndex < layerTypes.length; layerIndex += 1) {
|
|
1121
|
+
const canonicalLayerPrefix = `model.language_model.layers.${layerIndex}`;
|
|
1122
|
+
const globalLayer = normalizeText(layerTypes[layerIndex]) === 'full_attention';
|
|
1123
|
+
const attentionHeadDim = globalLayer ? globalHeadDim : headDim;
|
|
1124
|
+
const kvHeads = globalLayer ? numGlobalKeyValueHeads : numKeyValueHeads;
|
|
1125
|
+
const qRows = numAttentionHeads * attentionHeadDim;
|
|
1126
|
+
const kvRows = kvHeads * attentionHeadDim;
|
|
1127
|
+
|
|
1128
|
+
const addHiddenNorm = (canonicalName) => {
|
|
1129
|
+
const rawTensor = takeNextLiteRTTensor(cursor, [hiddenSize], 'F32', canonicalName);
|
|
1130
|
+
cursor.lastHiddenNorm = rawTensor;
|
|
1131
|
+
normalized.push(createLiteRTFloatTensor(rawTensor, prefillSourceFile.path, canonicalName, 'norm'));
|
|
1132
|
+
};
|
|
1133
|
+
const addFloat = (rawTensor, canonicalName, role = 'norm') => {
|
|
1134
|
+
if (!rawTensor) return;
|
|
1135
|
+
normalized.push(createLiteRTFloatTensor(rawTensor, prefillSourceFile.path, canonicalName, role));
|
|
1136
|
+
};
|
|
1137
|
+
const addInt4 = (rawTensor, canonicalName, role, logicalShape) => {
|
|
1138
|
+
normalized.push(
|
|
1139
|
+
createLiteRTFixedAffineInt4Tensor(
|
|
1140
|
+
rawTensor,
|
|
1141
|
+
prefillSourceFile.path,
|
|
1142
|
+
canonicalName,
|
|
1143
|
+
role,
|
|
1144
|
+
null,
|
|
1145
|
+
logicalShape,
|
|
1146
|
+
fixedInt4Scale,
|
|
1147
|
+
fixedInt4StorageEncoding
|
|
1148
|
+
)
|
|
1149
|
+
);
|
|
1150
|
+
};
|
|
1151
|
+
|
|
1152
|
+
addFloat(layerScalarTensors.get(layerIndex) ?? null, `${canonicalLayerPrefix}.layer_scalar`, 'other');
|
|
1153
|
+
addHiddenNorm(`${canonicalLayerPrefix}.input_layernorm.weight`);
|
|
1154
|
+
addHiddenNorm(`${canonicalLayerPrefix}.post_attention_layernorm.weight`);
|
|
1155
|
+
addHiddenNorm(`${canonicalLayerPrefix}.pre_feedforward_layernorm.weight`);
|
|
1156
|
+
addHiddenNorm(`${canonicalLayerPrefix}.post_feedforward_layernorm.weight`);
|
|
1157
|
+
|
|
1158
|
+
addInt4(
|
|
1159
|
+
takeNextLiteRTTensor(cursor, [hiddenSize, intermediateSize], 'INT4', `${canonicalLayerPrefix}.mlp.down_proj.weight`),
|
|
1160
|
+
`${canonicalLayerPrefix}.mlp.down_proj.weight`,
|
|
1161
|
+
'matmul',
|
|
1162
|
+
[hiddenSize, intermediateSize]
|
|
1163
|
+
);
|
|
1164
|
+
addInt4(
|
|
1165
|
+
takeNextLiteRTTensor(cursor, [intermediateSize, hiddenSize], 'INT4', `${canonicalLayerPrefix}.mlp.gate_proj.weight`),
|
|
1166
|
+
`${canonicalLayerPrefix}.mlp.gate_proj.weight`,
|
|
1167
|
+
'matmul',
|
|
1168
|
+
[intermediateSize, hiddenSize]
|
|
1169
|
+
);
|
|
1170
|
+
addInt4(
|
|
1171
|
+
takeNextLiteRTTensor(cursor, [intermediateSize, hiddenSize], 'INT4', `${canonicalLayerPrefix}.mlp.up_proj.weight`),
|
|
1172
|
+
`${canonicalLayerPrefix}.mlp.up_proj.weight`,
|
|
1173
|
+
'matmul',
|
|
1174
|
+
[intermediateSize, hiddenSize]
|
|
1175
|
+
);
|
|
1176
|
+
addInt4(
|
|
1177
|
+
takeNextLiteRTTensor(cursor, [hiddenSize, qRows], 'INT4', `${canonicalLayerPrefix}.self_attn.o_proj.weight`),
|
|
1178
|
+
`${canonicalLayerPrefix}.self_attn.o_proj.weight`,
|
|
1179
|
+
'matmul',
|
|
1180
|
+
[hiddenSize, qRows]
|
|
1181
|
+
);
|
|
1182
|
+
const kProj = takeNextLiteRTTensor(cursor, [kvRows, hiddenSize], 'INT4', `${canonicalLayerPrefix}.self_attn.k_proj.weight`);
|
|
1183
|
+
const vProj = takeOptionalConsecutiveLiteRTTensor(cursor, [kvRows, hiddenSize], 'INT4') ?? kProj;
|
|
1184
|
+
addInt4(kProj, `${canonicalLayerPrefix}.self_attn.k_proj.weight`, 'matmul', [kvRows, hiddenSize]);
|
|
1185
|
+
addInt4(vProj, `${canonicalLayerPrefix}.self_attn.v_proj.weight`, 'matmul', [kvRows, hiddenSize]);
|
|
1186
|
+
|
|
1187
|
+
const firstNorm = takeOptionalConsecutiveLiteRTTensor(cursor, [attentionHeadDim], 'F32');
|
|
1188
|
+
const secondNorm = takeOptionalConsecutiveLiteRTTensor(cursor, [attentionHeadDim], 'F32');
|
|
1189
|
+
if (firstNorm) {
|
|
1190
|
+
addFloat(firstNorm, `${canonicalLayerPrefix}.self_attn.q_norm.weight`);
|
|
1191
|
+
addFloat(secondNorm ?? firstNorm, `${canonicalLayerPrefix}.self_attn.k_norm.weight`);
|
|
1192
|
+
}
|
|
1193
|
+
|
|
1194
|
+
addInt4(
|
|
1195
|
+
takeNextLiteRTTensor(cursor, [qRows, hiddenSize], 'INT4', `${canonicalLayerPrefix}.self_attn.q_proj.weight`),
|
|
1196
|
+
`${canonicalLayerPrefix}.self_attn.q_proj.weight`,
|
|
1197
|
+
'matmul',
|
|
1198
|
+
[qRows, hiddenSize]
|
|
1199
|
+
);
|
|
1200
|
+
}
|
|
1201
|
+
|
|
1202
|
+
const finalNorm = takeNextLiteRTTensor(cursor, [hiddenSize], 'F32', 'model.language_model.norm.weight', {
|
|
1203
|
+
optional: true,
|
|
1204
|
+
}) ?? leadingFinalNorm ?? cursor.lastHiddenNorm;
|
|
1205
|
+
addFloatToNormalized(normalized, finalNorm, prefillSourceFile.path, 'model.language_model.norm.weight', 'norm', 'head');
|
|
1206
|
+
|
|
1207
|
+
if (normalized.length === 0) {
|
|
1208
|
+
throw new Error('direct-source runtime: Gemma 4 12B split-section adapter did not produce any normalized tensors.');
|
|
1209
|
+
}
|
|
1210
|
+
return normalized;
|
|
1211
|
+
}
|
|
1212
|
+
|
|
776
1213
|
function normalizeGemma4LiteRTTensors(parsedTFLite, sourcePath, runtimeProfile) {
|
|
777
1214
|
const rawByName = new Map();
|
|
778
1215
|
for (const tensor of parsedTFLite.tensors) {
|
|
@@ -1024,7 +1461,18 @@ function normalizeGemma4LiteRTTensors(parsedTFLite, sourcePath, runtimeProfile)
|
|
|
1024
1461
|
return normalized;
|
|
1025
1462
|
}
|
|
1026
1463
|
|
|
1027
|
-
function buildPackageParsedArtifact(
|
|
1464
|
+
function buildPackageParsedArtifact(
|
|
1465
|
+
sourceKind,
|
|
1466
|
+
sourcePathForModelId,
|
|
1467
|
+
runtimeProfile,
|
|
1468
|
+
parsedTFLite,
|
|
1469
|
+
virtualFiles,
|
|
1470
|
+
packageConfig = null,
|
|
1471
|
+
parsedTFLitesByPath = null
|
|
1472
|
+
) {
|
|
1473
|
+
const tfliteSourceFiles = virtualFiles.filter((entry) => (
|
|
1474
|
+
entry.kind === 'tflite_model' || entry.kind === 'tflite_embedder_model'
|
|
1475
|
+
));
|
|
1028
1476
|
const tfliteSourceFile = virtualFiles.find((entry) => entry.kind === 'tflite_model') ?? null;
|
|
1029
1477
|
if (!tfliteSourceFile) {
|
|
1030
1478
|
throw new Error('direct-source runtime: LiteRT package is missing a TFLite model entry.');
|
|
@@ -1034,18 +1482,45 @@ function buildPackageParsedArtifact(sourceKind, sourcePathForModelId, runtimePro
|
|
|
1034
1482
|
const tokenizerModelFile = virtualFiles.find((entry) => entry.kind === 'tokenizer_model') ?? null;
|
|
1035
1483
|
const metadataFile = virtualFiles.find((entry) => entry.kind === 'litert_metadata') ?? null;
|
|
1036
1484
|
const config = cloneJsonValue(runtimeProfile.rawConfig ?? {});
|
|
1037
|
-
const
|
|
1038
|
-
|
|
1039
|
-
|
|
1485
|
+
const graphAdapter = normalizeText(packageConfig?.graphAdapter);
|
|
1486
|
+
const tensors = graphAdapter === 'gemma4_unified_12b_split_int4'
|
|
1487
|
+
? normalizeGemma412BSplitLiteRTTensors(parsedTFLitesByPath, virtualFiles, runtimeProfile, packageConfig)
|
|
1488
|
+
: (normalizeText(runtimeProfile.modelType) === 'gemma4'
|
|
1489
|
+
? normalizeGemma4LiteRTTensors(parsedTFLite, tfliteSourceFile.path, runtimeProfile)
|
|
1490
|
+
: parsedTFLite.tensors.map((tensor) => ({
|
|
1040
1491
|
...tensor,
|
|
1041
1492
|
sourcePath: tfliteSourceFile.path,
|
|
1042
|
-
|
|
1493
|
+
})));
|
|
1494
|
+
const manifestInference = cloneJsonValue(runtimeProfile.manifestInference ?? {});
|
|
1495
|
+
if (graphAdapter === 'gemma4_unified_12b_split_int4') {
|
|
1496
|
+
const qNormLayers = new Set();
|
|
1497
|
+
const kNormLayers = new Set();
|
|
1498
|
+
for (const tensor of tensors) {
|
|
1499
|
+
const qMatch = String(tensor?.name ?? '').match(/\.layers\.(\d+)\.self_attn\.q_norm\.weight$/);
|
|
1500
|
+
if (qMatch) {
|
|
1501
|
+
qNormLayers.add(Number(qMatch[1]));
|
|
1502
|
+
}
|
|
1503
|
+
const kMatch = String(tensor?.name ?? '').match(/\.layers\.(\d+)\.self_attn\.k_norm\.weight$/);
|
|
1504
|
+
if (kMatch) {
|
|
1505
|
+
kNormLayers.add(Number(kMatch[1]));
|
|
1506
|
+
}
|
|
1507
|
+
}
|
|
1508
|
+
const layers = [...qNormLayers]
|
|
1509
|
+
.filter((layerIdx) => kNormLayers.has(layerIdx))
|
|
1510
|
+
.sort((left, right) => left - right);
|
|
1511
|
+
manifestInference.attention = {
|
|
1512
|
+
...(manifestInference.attention ?? {}),
|
|
1513
|
+
queryKeyNorm: manifestInference.attention?.queryKeyNorm === true || layers.length > 0,
|
|
1514
|
+
queryKeyNormLayers: null,
|
|
1515
|
+
queryKeyNormWeightLayers: layers,
|
|
1516
|
+
};
|
|
1517
|
+
}
|
|
1043
1518
|
return {
|
|
1044
1519
|
sourceKind,
|
|
1045
1520
|
modelType: normalizeText(runtimeProfile.modelType),
|
|
1046
1521
|
config,
|
|
1047
1522
|
manifestConfig: cloneJsonValue(runtimeProfile.manifestConfig ?? {}),
|
|
1048
|
-
manifestInference
|
|
1523
|
+
manifestInference,
|
|
1049
1524
|
architectureHint: normalizeText(runtimeProfile.modelType) || 'transformer',
|
|
1050
1525
|
embeddingPostprocessor: null,
|
|
1051
1526
|
architecture: cloneJsonValue(runtimeProfile.architecture ?? null),
|
|
@@ -1056,12 +1531,10 @@ function buildPackageParsedArtifact(sourceKind, sourcePathForModelId, runtimePro
|
|
|
1056
1531
|
tokenizerJsonPath: tokenizerJsonFile ? tokenizerJsonFile.path : null,
|
|
1057
1532
|
tokenizerConfigPath: tokenizerConfigFile ? tokenizerConfigFile.path : null,
|
|
1058
1533
|
tokenizerModelPath: tokenizerModelFile ? tokenizerModelFile.path : null,
|
|
1059
|
-
sourceFiles:
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
},
|
|
1064
|
-
],
|
|
1534
|
+
sourceFiles: tfliteSourceFiles.map((entry) => ({
|
|
1535
|
+
path: entry.path,
|
|
1536
|
+
size: entry.size,
|
|
1537
|
+
})),
|
|
1065
1538
|
auxiliaryFiles: [
|
|
1066
1539
|
...(tokenizerJsonFile
|
|
1067
1540
|
? [{
|
|
@@ -1110,12 +1583,18 @@ async function isRawTFLiteTaskSource(source) {
|
|
|
1110
1583
|
async function parseLiteRTTaskPackage(source, sourcePathForModelId) {
|
|
1111
1584
|
const packageBasename = normalizeText(source?.name);
|
|
1112
1585
|
const profile = resolveRequiredProfile(LITERT_PACKAGE_SOURCE_KIND_TASK, packageBasename);
|
|
1586
|
+
const taskConfig = profile.package?.task ?? {};
|
|
1587
|
+
throwIfUnsupportedPackageProfile(
|
|
1588
|
+
LITERT_PACKAGE_SOURCE_KIND_TASK,
|
|
1589
|
+
packageBasename,
|
|
1590
|
+
profile,
|
|
1591
|
+
taskConfig
|
|
1592
|
+
);
|
|
1113
1593
|
const runtimeProfile = profile.runtime ?? null;
|
|
1114
1594
|
if (!runtimeProfile) {
|
|
1115
1595
|
throw new Error(`direct-source runtime: package profile "${profile.id}" is missing runtime data.`);
|
|
1116
1596
|
}
|
|
1117
1597
|
|
|
1118
|
-
const taskConfig = profile.package?.task ?? {};
|
|
1119
1598
|
const rawTFLiteTask = await isRawTFLiteTaskSource(source);
|
|
1120
1599
|
const virtualFiles = [];
|
|
1121
1600
|
let parsedTFLite = null;
|
|
@@ -1179,7 +1658,8 @@ async function parseLiteRTTaskPackage(source, sourcePathForModelId) {
|
|
|
1179
1658
|
sourcePathForModelId,
|
|
1180
1659
|
runtimeProfile,
|
|
1181
1660
|
parsedTFLite,
|
|
1182
|
-
virtualFiles
|
|
1661
|
+
virtualFiles,
|
|
1662
|
+
taskConfig
|
|
1183
1663
|
),
|
|
1184
1664
|
virtualFiles,
|
|
1185
1665
|
packageProfile: profile,
|
|
@@ -1189,13 +1669,20 @@ async function parseLiteRTTaskPackage(source, sourcePathForModelId) {
|
|
|
1189
1669
|
async function parseLiteRTLMPackage(source, sourcePathForModelId) {
|
|
1190
1670
|
const packageBasename = normalizeText(source?.name);
|
|
1191
1671
|
const profile = resolveRequiredProfile(LITERT_PACKAGE_SOURCE_KIND_LITERTLM, packageBasename);
|
|
1672
|
+
const litertConfig = profile.package?.litertlm ?? {};
|
|
1673
|
+
throwIfUnsupportedPackageProfile(
|
|
1674
|
+
LITERT_PACKAGE_SOURCE_KIND_LITERTLM,
|
|
1675
|
+
packageBasename,
|
|
1676
|
+
profile,
|
|
1677
|
+
litertConfig
|
|
1678
|
+
);
|
|
1192
1679
|
const runtimeProfile = profile.runtime ?? null;
|
|
1193
1680
|
if (!runtimeProfile) {
|
|
1194
1681
|
throw new Error(`direct-source runtime: package profile "${profile.id}" is missing runtime data.`);
|
|
1195
1682
|
}
|
|
1196
1683
|
|
|
1197
1684
|
const parsedLiteRTLM = await parseLiteRTLMFromSource(source);
|
|
1198
|
-
const
|
|
1685
|
+
const graphAdapter = normalizeText(litertConfig.graphAdapter);
|
|
1199
1686
|
const tfliteModelType = normalizeText(litertConfig.tfliteModelType) || LITERT_TASK_DEFAULT_TFLITE_ENTRY;
|
|
1200
1687
|
const weightsSection = findLiteRTLMTFLiteWeightsSection(parsedLiteRTLM, tfliteModelType);
|
|
1201
1688
|
if (weightsSection) {
|
|
@@ -1217,6 +1704,55 @@ async function parseLiteRTLMPackage(source, sourcePathForModelId) {
|
|
|
1217
1704
|
|
|
1218
1705
|
const tokenizerSection = findLiteRTLMSentencePieceTokenizerSection(parsedLiteRTLM);
|
|
1219
1706
|
const metadataSection = findLiteRTLMMetadataSection(parsedLiteRTLM);
|
|
1707
|
+
if (graphAdapter === 'gemma4_unified_12b_split_int4') {
|
|
1708
|
+
const embedderTFLiteModelType = normalizeText(litertConfig.embedderTFLiteModelType);
|
|
1709
|
+
if (!embedderTFLiteModelType) {
|
|
1710
|
+
throw new Error(
|
|
1711
|
+
`direct-source runtime: LiteRT-LM "${packageBasename}" split-section graph adapter requires ` +
|
|
1712
|
+
'package.litertlm.embedderTFLiteModelType.'
|
|
1713
|
+
);
|
|
1714
|
+
}
|
|
1715
|
+
const prefillSection = findLiteRTLMTFLiteModelSection(parsedLiteRTLM, tfliteModelType);
|
|
1716
|
+
const embedderSection = findLiteRTLMTFLiteModelSection(parsedLiteRTLM, embedderTFLiteModelType);
|
|
1717
|
+
if (!prefillSection || !embedderSection) {
|
|
1718
|
+
throw new Error(
|
|
1719
|
+
`direct-source runtime: LiteRT-LM "${packageBasename}" split-section graph adapter requires ` +
|
|
1720
|
+
`TFLiteModel sections "${embedderTFLiteModelType}" and "${tfliteModelType}".`
|
|
1721
|
+
);
|
|
1722
|
+
}
|
|
1723
|
+
const virtualFiles = [
|
|
1724
|
+
createVirtualFile(embedderTFLiteModelType, embedderSection.beginOffset, embedderSection.size, 'tflite_embedder_model'),
|
|
1725
|
+
createVirtualFile(tfliteModelType, prefillSection.beginOffset, prefillSection.size, 'tflite_model'),
|
|
1726
|
+
];
|
|
1727
|
+
if (tokenizerSection) {
|
|
1728
|
+
virtualFiles.push(createVirtualFile('TOKENIZER_MODEL', tokenizerSection.beginOffset, tokenizerSection.size, 'tokenizer_model'));
|
|
1729
|
+
}
|
|
1730
|
+
if (metadataSection) {
|
|
1731
|
+
virtualFiles.push(createVirtualFile('METADATA', metadataSection.beginOffset, metadataSection.size, 'litert_metadata'));
|
|
1732
|
+
}
|
|
1733
|
+
const parsedTFLitesByPath = new Map();
|
|
1734
|
+
for (const virtualFile of virtualFiles.filter((entry) => entry.kind === 'tflite_model' || entry.kind === 'tflite_embedder_model')) {
|
|
1735
|
+
parsedTFLitesByPath.set(
|
|
1736
|
+
virtualFile.path,
|
|
1737
|
+
await parseTFLiteFromSource(createSectionSource(source, virtualFile), {
|
|
1738
|
+
allowPackedQuantization: true,
|
|
1739
|
+
})
|
|
1740
|
+
);
|
|
1741
|
+
}
|
|
1742
|
+
return {
|
|
1743
|
+
parsedArtifact: buildPackageParsedArtifact(
|
|
1744
|
+
LITERT_PACKAGE_SOURCE_KIND_LITERTLM,
|
|
1745
|
+
sourcePathForModelId,
|
|
1746
|
+
runtimeProfile,
|
|
1747
|
+
parsedTFLitesByPath.get(tfliteModelType),
|
|
1748
|
+
virtualFiles,
|
|
1749
|
+
litertConfig,
|
|
1750
|
+
parsedTFLitesByPath
|
|
1751
|
+
),
|
|
1752
|
+
virtualFiles,
|
|
1753
|
+
packageProfile: profile,
|
|
1754
|
+
};
|
|
1755
|
+
}
|
|
1220
1756
|
const errors = [];
|
|
1221
1757
|
|
|
1222
1758
|
for (let candidateIndex = 0; candidateIndex < modelSections.length; candidateIndex += 1) {
|
|
@@ -1244,7 +1780,8 @@ async function parseLiteRTLMPackage(source, sourcePathForModelId) {
|
|
|
1244
1780
|
sourcePathForModelId,
|
|
1245
1781
|
runtimeProfile,
|
|
1246
1782
|
parsedTFLite,
|
|
1247
|
-
virtualFiles
|
|
1783
|
+
virtualFiles,
|
|
1784
|
+
litertConfig
|
|
1248
1785
|
),
|
|
1249
1786
|
virtualFiles,
|
|
1250
1787
|
packageProfile: profile,
|