doppler-gpu 0.4.3 → 0.4.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (433) hide show
  1. package/CHANGELOG.md +28 -0
  2. package/README.md +54 -33
  3. package/assets/doppler.svg +57 -1601
  4. package/package.json +36 -5
  5. package/src/cli/cli-output.js +34 -1
  6. package/src/cli/doppler-cli.d.ts +18 -0
  7. package/src/cli/doppler-cli.js +68 -35
  8. package/src/cli/doppler-serve.d.ts +210 -0
  9. package/src/cli/doppler-serve.js +468 -42
  10. package/src/client/doppler-registry.json +3 -5
  11. package/src/client/provider.d.ts +1 -1
  12. package/src/client/provider.js +1 -1
  13. package/src/client/runtime/index.js +13 -0
  14. package/src/client/runtime/lora.d.ts +11 -1
  15. package/src/client/runtime/lora.js +28 -7
  16. package/src/client/runtime/model-manager.js +35 -11
  17. package/src/client/runtime/model-session.d.ts +2 -1
  18. package/src/client/runtime/model-session.js +2 -2
  19. package/src/client/runtime/model-source.js +20 -1
  20. package/src/client/runtime/types.d.ts +8 -0
  21. package/src/client/wrap-pipeline-handle.d.ts +62 -0
  22. package/src/client/wrap-pipeline-handle.js +188 -13
  23. package/src/config/conversion/README.md +5 -0
  24. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
  25. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +3 -3
  26. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +4 -4
  27. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +4 -4
  28. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
  29. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
  30. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +17 -9
  31. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +4 -4
  32. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +4 -4
  33. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
  34. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
  35. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
  36. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
  37. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
  38. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
  39. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +14 -12
  40. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +7 -7
  41. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +7 -7
  42. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +3 -3
  43. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +3 -3
  44. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +3 -3
  45. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +3 -3
  46. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +8 -4
  47. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +8 -4
  48. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +9 -5
  49. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +8 -4
  50. package/src/config/goal-completion-matrix.json +552 -0
  51. package/src/config/kernel-path-loader.js +192 -38
  52. package/src/config/kernels/kernel-ref-digests.js +52 -18
  53. package/src/config/kernels/registry.json +3017 -744
  54. package/src/config/merge-contract-check.js +7 -3
  55. package/src/config/merge-helpers.d.ts +2 -0
  56. package/src/config/merge-helpers.js +1 -0
  57. package/src/config/merge.d.ts +1 -0
  58. package/src/config/merge.js +30 -0
  59. package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
  60. package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
  61. package/src/config/param-validator.js +4 -0
  62. package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
  63. package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
  64. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
  65. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
  66. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
  67. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
  68. package/src/config/schema/buffer-pool.schema.js +3 -3
  69. package/src/config/schema/claim-evidence-contract.schema.json +93 -0
  70. package/src/config/schema/command-surface-contract.schema.json +81 -0
  71. package/src/config/schema/converter.schema.d.ts +11 -0
  72. package/src/config/schema/converter.schema.js +8 -0
  73. package/src/config/schema/doppler.schema.d.ts +1 -0
  74. package/src/config/schema/doppler.schema.js +38 -6
  75. package/src/config/schema/execution-v1.schema.d.ts +27 -0
  76. package/src/config/schema/execution-v1.schema.js +28 -19
  77. package/src/config/schema/goal-completion-matrix.schema.json +223 -0
  78. package/src/config/schema/inference.schema.d.ts +11 -0
  79. package/src/config/schema/manifest.schema.d.ts +107 -1
  80. package/src/config/schema/manifest.schema.js +10 -1
  81. package/src/config/schema/model-artifact-contract.schema.json +119 -0
  82. package/src/config/schema/moe.schema.d.ts +13 -0
  83. package/src/config/schema/moe.schema.js +1 -1
  84. package/src/config/schema/policy-schema-registry.json +26 -0
  85. package/src/config/schema/subsystem-support-contract.schema.json +48 -0
  86. package/src/config/schema/training-metrics.schema.d.ts +1 -1
  87. package/src/config/schema/training-metrics.schema.js +4 -3
  88. package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
  89. package/src/config/source-packages/litert/gemma-4-e2b-it.json +7 -7
  90. package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
  91. package/src/config/source-packages/registry.json +25 -0
  92. package/src/config/supported-operations.json +5 -0
  93. package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
  94. package/src/config/transforms/execution-graph-transforms.d.ts +42 -0
  95. package/src/config/transforms/execution-graph-transforms.js +227 -230
  96. package/src/converter/conversion-plan.d.ts +2 -0
  97. package/src/converter/conversion-plan.js +18 -4
  98. package/src/converter/core.d.ts +8 -0
  99. package/src/converter/core.js +574 -12
  100. package/src/converter/quantization-info.js +179 -2
  101. package/src/converter/quantizer.d.ts +5 -4
  102. package/src/converter/quantizer.js +16 -2
  103. package/src/experimental/adapters/lora-loader.js +109 -24
  104. package/src/experimental/diffusion/index.d.ts +24 -0
  105. package/src/experimental/diffusion/index.js +10 -0
  106. package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
  107. package/src/experimental/distribution/descriptor-transport.js +342 -0
  108. package/src/experimental/training/README.md +11 -0
  109. package/src/experimental/training/autograd.js +33 -8
  110. package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
  111. package/src/experimental/training/datasets/gepa-frontier.js +220 -0
  112. package/src/experimental/training/datasets/index.d.ts +13 -1
  113. package/src/experimental/training/datasets/index.js +8 -1
  114. package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
  115. package/src/experimental/training/datasets/teacher-traces.js +279 -0
  116. package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
  117. package/src/experimental/training/datasets/text-pairs.js +82 -5
  118. package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
  119. package/src/experimental/training/distillation/student-fixture.js +290 -32
  120. package/src/experimental/training/export.d.ts +9 -0
  121. package/src/experimental/training/export.js +113 -7
  122. package/src/experimental/training/index.d.ts +13 -1
  123. package/src/experimental/training/index.js +13 -1
  124. package/src/experimental/training/lora-pipeline.d.ts +143 -0
  125. package/src/experimental/training/lora-pipeline.js +1737 -49
  126. package/src/experimental/training/lora.js +23 -1
  127. package/src/experimental/training/loss.js +1 -1
  128. package/src/experimental/training/operator-agent-eval.d.ts +65 -0
  129. package/src/experimental/training/operator-agent-eval.js +326 -0
  130. package/src/experimental/training/operator-command.js +310 -51
  131. package/src/experimental/training/runner.d.ts +1 -0
  132. package/src/experimental/training/runner.js +4 -0
  133. package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
  134. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
  135. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
  136. package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
  137. package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
  138. package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
  139. package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
  140. package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
  141. package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
  142. package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
  143. package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
  144. package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
  145. package/src/experimental/training/workload-packs/registry.json +186 -6
  146. package/src/experimental/training/workloads.d.ts +37 -0
  147. package/src/experimental/training/workloads.js +110 -1
  148. package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
  149. package/src/formats/rdrr/functional-descriptor.js +230 -0
  150. package/src/formats/rdrr/index.d.ts +1 -0
  151. package/src/formats/rdrr/index.js +1 -0
  152. package/src/formats/rdrr/parsing.js +22 -6
  153. package/src/formats/rdrr/source-transform-contract.js +10 -1
  154. package/src/formats/rdrr/storage-descriptor.js +9 -2
  155. package/src/formats/rdrr/types.d.ts +5 -3
  156. package/src/formats/rdrr/validation.js +15 -1
  157. package/src/gpu/command-recorder.d.ts +43 -3
  158. package/src/gpu/command-recorder.js +122 -25
  159. package/src/gpu/device.js +111 -11
  160. package/src/gpu/kernel-tuner/cache.js +1 -5
  161. package/src/gpu/kernels/attention.js +17 -9
  162. package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
  163. package/src/gpu/kernels/attention_streaming_f16.wgsl +7 -5
  164. package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
  165. package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
  166. package/src/gpu/kernels/cast.js +74 -67
  167. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +46 -1
  168. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -20
  169. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
  170. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +59 -44
  171. package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
  172. package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
  173. package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
  174. package/src/gpu/kernels/dispatch.js +13 -3
  175. package/src/gpu/kernels/fused_ffn.js +3 -4
  176. package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
  177. package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
  178. package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
  179. package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
  180. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
  181. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
  182. package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
  183. package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
  184. package/src/gpu/kernels/gather.d.ts +84 -1
  185. package/src/gpu/kernels/gather.js +353 -5
  186. package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
  187. package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
  188. package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
  189. package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
  190. package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
  191. package/src/gpu/kernels/gelu.wgsl +11 -7
  192. package/src/gpu/kernels/gelu_f16.wgsl +11 -7
  193. package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
  194. package/src/gpu/kernels/gemma4-route-expert.js +126 -0
  195. package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
  196. package/src/gpu/kernels/index.d.ts +71 -0
  197. package/src/gpu/kernels/index.js +52 -0
  198. package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
  199. package/src/gpu/kernels/kv-cache-write.js +126 -0
  200. package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
  201. package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
  202. package/src/gpu/kernels/lm-head-argmax.js +218 -0
  203. package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
  204. package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
  205. package/src/gpu/kernels/matmul-dispatch.js +37 -1
  206. package/src/gpu/kernels/matmul-selection.d.ts +5 -5
  207. package/src/gpu/kernels/matmul-selection.js +104 -9
  208. package/src/gpu/kernels/matmul.d.ts +1 -1
  209. package/src/gpu/kernels/matmul.js +151 -43
  210. package/src/gpu/kernels/moe.d.ts +1 -0
  211. package/src/gpu/kernels/moe.js +46 -18
  212. package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
  213. package/src/gpu/kernels/pipeline-cache.js +24 -0
  214. package/src/gpu/kernels/rep-penalty.d.ts +21 -0
  215. package/src/gpu/kernels/rep-penalty.js +17 -9
  216. package/src/gpu/kernels/residual.d.ts +1 -0
  217. package/src/gpu/kernels/residual.js +13 -1
  218. package/src/gpu/kernels/residual.wgsl +3 -3
  219. package/src/gpu/kernels/residual_f16.wgsl +5 -3
  220. package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
  221. package/src/gpu/kernels/residual_vec4.wgsl +6 -6
  222. package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
  223. package/src/gpu/kernels/rmsnorm-pair.js +152 -0
  224. package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
  225. package/src/gpu/kernels/rmsnorm-qk.js +128 -0
  226. package/src/gpu/kernels/rmsnorm.d.ts +31 -0
  227. package/src/gpu/kernels/rmsnorm.js +32 -7
  228. package/src/gpu/kernels/rmsnorm.wgsl +10 -6
  229. package/src/gpu/kernels/rmsnorm_f16.wgsl +7 -3
  230. package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
  231. package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
  232. package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
  233. package/src/gpu/kernels/rope-qk.d.ts +41 -0
  234. package/src/gpu/kernels/rope-qk.js +143 -0
  235. package/src/gpu/kernels/rope_qk.wgsl +95 -0
  236. package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
  237. package/src/gpu/kernels/rule-registry.js +50 -10
  238. package/src/gpu/kernels/sample.js +212 -187
  239. package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
  240. package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
  241. package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
  242. package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
  243. package/src/gpu/kernels/shader-cache.js +1 -21
  244. package/src/gpu/kernels/soft-embedding.d.ts +28 -0
  245. package/src/gpu/kernels/soft-embedding.js +447 -0
  246. package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
  247. package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
  248. package/src/gpu/kernels/softmax.wgsl +5 -80
  249. package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
  250. package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
  251. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
  252. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
  253. package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
  254. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
  255. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
  256. package/src/gpu/kernels/uniform-utils.js +2 -5
  257. package/src/gpu/kernels/utils.d.ts +16 -1
  258. package/src/gpu/kernels/utils.js +42 -13
  259. package/src/gpu/readback-utils.js +10 -2
  260. package/src/gpu/uniform-cache.d.ts +10 -1
  261. package/src/gpu/uniform-cache.js +25 -7
  262. package/src/gpu/weight-buffer.d.ts +76 -7
  263. package/src/gpu/weight-buffer.js +85 -1
  264. package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
  265. package/src/inference/browser-harness-suite-helpers.d.ts +9 -0
  266. package/src/inference/browser-harness-suite-helpers.js +438 -2
  267. package/src/inference/browser-harness-text-helpers.d.ts +77 -0
  268. package/src/inference/browser-harness-text-helpers.js +236 -11
  269. package/src/inference/browser-harness.d.ts +8 -0
  270. package/src/inference/browser-harness.js +292 -8
  271. package/src/inference/decode-ring.d.ts +1 -0
  272. package/src/inference/decode-ring.js +7 -5
  273. package/src/inference/kv-cache/base.d.ts +25 -0
  274. package/src/inference/kv-cache/base.js +109 -1
  275. package/src/inference/kv-cache/mixed-geometry.d.ts +10 -0
  276. package/src/inference/kv-cache/mixed-geometry.js +109 -2
  277. package/src/inference/kv-cache/sliding-window.d.ts +14 -0
  278. package/src/inference/kv-cache/sliding-window.js +88 -0
  279. package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
  280. package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
  281. package/src/inference/moe-router.d.ts +14 -1
  282. package/src/inference/moe-router.js +24 -1
  283. package/src/inference/pipelines/audio/gemma4.js +52 -1
  284. package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
  285. package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
  286. package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
  287. package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
  288. package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
  289. package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
  290. package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
  291. package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
  292. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
  293. package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
  294. package/src/inference/pipelines/text/attention/projections.d.ts +71 -2
  295. package/src/inference/pipelines/text/attention/projections.js +329 -49
  296. package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
  297. package/src/inference/pipelines/text/attention/record.js +446 -73
  298. package/src/inference/pipelines/text/attention/run.js +251 -49
  299. package/src/inference/pipelines/text/attention/types.d.ts +5 -1
  300. package/src/inference/pipelines/text/config.d.ts +22 -1
  301. package/src/inference/pipelines/text/config.js +199 -22
  302. package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
  303. package/src/inference/pipelines/text/dtype-contract.js +4 -1
  304. package/src/inference/pipelines/text/embed.d.ts +4 -3
  305. package/src/inference/pipelines/text/embed.js +16 -6
  306. package/src/inference/pipelines/text/execution-plan.d.ts +8 -0
  307. package/src/inference/pipelines/text/execution-plan.js +6 -0
  308. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -0
  309. package/src/inference/pipelines/text/execution-runtime-builders.js +44 -28
  310. package/src/inference/pipelines/text/execution-v1.d.ts +2 -0
  311. package/src/inference/pipelines/text/execution-v1.js +373 -139
  312. package/src/inference/pipelines/text/ffn/dense.js +44 -13
  313. package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
  314. package/src/inference/pipelines/text/ffn/moe.js +9 -2
  315. package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
  316. package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
  317. package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
  318. package/src/inference/pipelines/text/ffn/standard.js +44 -4
  319. package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
  320. package/src/inference/pipelines/text/generator-decode-policy.js +2 -12
  321. package/src/inference/pipelines/text/generator-helpers.js +5 -2
  322. package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
  323. package/src/inference/pipelines/text/generator-prefill-helpers.js +67 -1
  324. package/src/inference/pipelines/text/generator-runtime.d.ts +1 -0
  325. package/src/inference/pipelines/text/generator-steps.d.ts +38 -3
  326. package/src/inference/pipelines/text/generator-steps.js +268 -77
  327. package/src/inference/pipelines/text/generator.d.ts +39 -0
  328. package/src/inference/pipelines/text/generator.js +1105 -18
  329. package/src/inference/pipelines/text/init.d.ts +8 -1
  330. package/src/inference/pipelines/text/init.js +30 -1
  331. package/src/inference/pipelines/text/layer-plan-gpu.js +62 -14
  332. package/src/inference/pipelines/text/layer.js +272 -25
  333. package/src/inference/pipelines/text/linear-attention.js +6 -4
  334. package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
  335. package/src/inference/pipelines/text/logits/gpu.js +417 -66
  336. package/src/inference/pipelines/text/logits/index.d.ts +2 -2
  337. package/src/inference/pipelines/text/logits/index.js +96 -42
  338. package/src/inference/pipelines/text/logits/types.d.ts +2 -2
  339. package/src/inference/pipelines/text/lora-apply.js +17 -0
  340. package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
  341. package/src/inference/pipelines/text/moe-gpu.js +445 -16
  342. package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
  343. package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
  344. package/src/inference/pipelines/text/ops.d.ts +29 -2
  345. package/src/inference/pipelines/text/ops.js +19 -0
  346. package/src/inference/pipelines/text/per-layer-inputs.js +5 -0
  347. package/src/inference/pipelines/text/probes.js +4 -5
  348. package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
  349. package/src/inference/pipelines/text/sampling-config.js +103 -64
  350. package/src/inference/pipelines/text/sampling.js +13 -9
  351. package/src/inference/pipelines/text/state.js +17 -0
  352. package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
  353. package/src/inference/pipelines/text/tsir-fixture-writer.js +8 -9
  354. package/src/inference/pipelines/text/types.d.ts +84 -3
  355. package/src/inference/pipelines/text/weights.d.ts +4 -2
  356. package/src/inference/pipelines/text/weights.js +4 -5
  357. package/src/inference/pipelines/text.d.ts +24 -0
  358. package/src/inference/pipelines/text.js +162 -38
  359. package/src/inference/pipelines/vision/gemma4.js +20 -9
  360. package/src/inference/tokenizer.d.ts +30 -0
  361. package/src/inference/tokenizer.js +340 -164
  362. package/src/inference/tokenizers/bundled.js +71 -49
  363. package/src/loader/doppler-loader.d.ts +11 -0
  364. package/src/loader/doppler-loader.js +290 -15
  365. package/src/loader/embedding-limit-preflight.d.ts +31 -0
  366. package/src/loader/embedding-limit-preflight.js +216 -0
  367. package/src/loader/embedding-loader.d.ts +8 -2
  368. package/src/loader/embedding-loader.js +263 -8
  369. package/src/loader/experts/expert-loader.d.ts +4 -1
  370. package/src/loader/experts/expert-loader.js +174 -5
  371. package/src/loader/final-weights-loader.d.ts +5 -0
  372. package/src/loader/final-weights-loader.js +167 -4
  373. package/src/loader/layer-loader.d.ts +2 -0
  374. package/src/loader/layer-loader.js +74 -6
  375. package/src/loader/loader-types.d.ts +59 -2
  376. package/src/loader/manifest-config.d.ts +8 -0
  377. package/src/loader/shard-resolver.js +37 -4
  378. package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
  379. package/src/loader/tensors/functional-descriptor-loader.js +377 -0
  380. package/src/loader/tensors/tensor-loader.d.ts +36 -1
  381. package/src/loader/tensors/tensor-loader.js +437 -21
  382. package/src/loader/tensors/tensor-reader.d.ts +4 -1
  383. package/src/loader/tensors/tensor-reader.js +11 -4
  384. package/src/loader/weights.d.ts +3 -1
  385. package/src/memory/buffer-pool.js +12 -9
  386. package/src/models/diffusiongemma.d.ts +7 -0
  387. package/src/models/diffusiongemma.js +25 -0
  388. package/src/models/gemma4.js +24 -0
  389. package/src/models/qwen3.js +2 -2
  390. package/src/rules/execution-rules-contract-check.js +93 -10
  391. package/src/rules/inference/capability-transforms.rules.json +26 -2
  392. package/src/rules/inference/config.rules.json +1 -0
  393. package/src/rules/inference/dtype.rules.json +7 -0
  394. package/src/rules/inference/execution.rules.json +108 -17
  395. package/src/rules/inference/ffn.rules.json +18 -0
  396. package/src/rules/kernels/gather.rules.json +81 -0
  397. package/src/rules/kernels/matmul.rules.json +19 -0
  398. package/src/rules/kernels/moe.rules.json +3 -0
  399. package/src/rules/kernels/rep-penalty.rules.json +6 -0
  400. package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
  401. package/src/rules/kernels/rmsnorm.rules.json +42 -7
  402. package/src/rules/kernels/rope-qk.rules.json +6 -0
  403. package/src/rules/loader/tensor-loader.rules.json +8 -0
  404. package/src/rules/rule-registry.js +6 -0
  405. package/src/storage/artifact-storage-context.d.ts +1 -0
  406. package/src/storage/artifact-storage-context.js +10 -0
  407. package/src/tooling/browser-command-runner.d.ts +0 -3
  408. package/src/tooling/browser-command-runner.js +2 -14
  409. package/src/tooling/command-api-family-normalizers.js +4 -2
  410. package/src/tooling/command-api-helpers.js +2 -1
  411. package/src/tooling/command-api.d.ts +2 -0
  412. package/src/tooling/command-api.js +1 -0
  413. package/src/tooling/command-runner-shared.d.ts +4 -0
  414. package/src/tooling/command-runner-shared.js +2 -1
  415. package/src/tooling/diagnose-runner.js +2 -0
  416. package/src/tooling/litert-package-runtime.js +553 -16
  417. package/src/tooling/node-browser-command-runner.js +10 -4
  418. package/src/tooling/node-command-runner.js +7 -2
  419. package/src/tooling/node-convert-worker-pool.js +3 -0
  420. package/src/tooling/node-convert-worker.js +1 -0
  421. package/src/tooling/node-converter.js +12 -0
  422. package/src/tooling/node-webgpu.d.ts +1 -0
  423. package/src/tooling/node-webgpu.js +23 -11
  424. package/src/tooling/program-bundle.d.ts +52 -1
  425. package/src/tooling/rdrr-integrity-refresh.d.ts +18 -0
  426. package/src/tooling/source-artifact-adapter.js +7 -2
  427. package/src/tooling/source-package-profiles.d.ts +18 -1
  428. package/src/tooling/source-package-profiles.js +109 -1
  429. package/src/tooling/source-runtime-bundle.d.ts +1 -0
  430. package/src/tooling/source-runtime-bundle.js +27 -0
  431. package/src/tooling/source-runtime-converter-config.js +12 -3
  432. package/src/tooling-experimental-exports.shared.d.ts +10 -0
  433. package/src/tooling-experimental-exports.shared.js +10 -0
@@ -21,6 +21,38 @@ function normalizeText(value) {
21
21
  return String(value || '').trim();
22
22
  }
23
23
 
24
+ function normalizeLiteRTStorageEncoding(value, fieldName) {
25
+ const storageEncoding = normalizeText(value).toLowerCase();
26
+ if (storageEncoding !== 'signed' && storageEncoding !== 'offset_binary') {
27
+ throw new Error(
28
+ `direct-source runtime: ${fieldName} must be "signed" or "offset_binary", got "${value ?? 'missing'}".`
29
+ );
30
+ }
31
+ return storageEncoding;
32
+ }
33
+
34
+ function normalizeLayerIndexList(value, numLayers, fieldName) {
35
+ if (!Array.isArray(value)) {
36
+ throw new Error(`direct-source runtime: ${fieldName} must be an array of layer indices.`);
37
+ }
38
+ const indexes = [];
39
+ let previous = -1;
40
+ for (const rawIndex of value) {
41
+ const layerIndex = Number(rawIndex);
42
+ if (!Number.isInteger(layerIndex) || layerIndex < 0 || layerIndex >= numLayers) {
43
+ throw new Error(
44
+ `direct-source runtime: ${fieldName} contains invalid layer index "${String(rawIndex)}".`
45
+ );
46
+ }
47
+ if (layerIndex <= previous) {
48
+ throw new Error(`direct-source runtime: ${fieldName} must be sorted and unique.`);
49
+ }
50
+ indexes.push(layerIndex);
51
+ previous = layerIndex;
52
+ }
53
+ return indexes;
54
+ }
55
+
24
56
  function createVirtualFile(path, offset, size, kind, options = {}) {
25
57
  return {
26
58
  path,
@@ -72,6 +104,22 @@ function resolvePackageTokenizerConfig(sourceKind, runtimeProfile) {
72
104
  return null;
73
105
  }
74
106
 
107
+ function throwIfUnsupportedPackageProfile(sourceKind, packageBasename, profile, packageConfig) {
108
+ const unsupported = packageConfig?.unsupported;
109
+ if (!unsupported || typeof unsupported !== 'object') {
110
+ return;
111
+ }
112
+ const code = normalizeText(unsupported.code) || 'unsupported-package-contract';
113
+ const message = normalizeText(unsupported.message);
114
+ const recommendation = normalizeText(unsupported.recommendation);
115
+ throw new Error(
116
+ `direct-source runtime: ${sourceKind} artifact "${packageBasename}" matches package profile ` +
117
+ `"${profile.id}" but is not supported by Doppler direct-source import (${code}).` +
118
+ (message ? ` ${message}` : '') +
119
+ (recommendation ? ` ${recommendation}` : '')
120
+ );
121
+ }
122
+
75
123
  function findTFLiteMetadataEntry(parsedTFLite, name) {
76
124
  const target = normalizeText(name);
77
125
  if (!target) {
@@ -352,6 +400,210 @@ function createLiteRTFloatTensor(rawTensor, sourcePath, canonicalName, role, gro
352
400
  };
353
401
  }
354
402
 
403
+ function shapeEquals(actualShape, expectedShape) {
404
+ if (!Array.isArray(actualShape) || !Array.isArray(expectedShape)) {
405
+ return false;
406
+ }
407
+ if (actualShape.length !== expectedShape.length) {
408
+ return false;
409
+ }
410
+ for (let index = 0; index < expectedShape.length; index += 1) {
411
+ if (Number(actualShape[index]) !== Number(expectedShape[index])) {
412
+ return false;
413
+ }
414
+ }
415
+ return true;
416
+ }
417
+
418
+ function isLiteRTTensorDtype(rawTensor, sourceDtype) {
419
+ const dtypeId = Number(rawTensor?.dtypeId);
420
+ if (sourceDtype === 'F32') {
421
+ return dtypeId === 0 || normalizeText(rawTensor?.dtype).toUpperCase() === 'F32';
422
+ }
423
+ if (sourceDtype === 'INT4') {
424
+ return dtypeId === 17 || normalizeText(rawTensor?.sourceDtype).toUpperCase() === 'INT4';
425
+ }
426
+ return false;
427
+ }
428
+
429
+ function createGemma412BSplitCursor(parsedTFLite, sourcePath) {
430
+ return {
431
+ tensors: Array.isArray(parsedTFLite?.tensors) ? parsedTFLite.tensors : [],
432
+ sourcePath,
433
+ index: 0,
434
+ lastHiddenNorm: null,
435
+ };
436
+ }
437
+
438
+ function takeNextLiteRTTensor(cursor, expectedShape, sourceDtype, label, options = {}) {
439
+ const optional = options.optional === true;
440
+ for (let tensorIndex = cursor.index; tensorIndex < cursor.tensors.length; tensorIndex += 1) {
441
+ const tensor = cursor.tensors[tensorIndex];
442
+ if (!isLiteRTTensorDtype(tensor, sourceDtype)) {
443
+ continue;
444
+ }
445
+ if (!shapeEquals(tensor.shape, expectedShape)) {
446
+ continue;
447
+ }
448
+ cursor.index = tensorIndex + 1;
449
+ return tensor;
450
+ }
451
+ if (optional) {
452
+ return null;
453
+ }
454
+ throw new Error(
455
+ `direct-source runtime: Gemma 4 12B split-section adapter could not find ${sourceDtype} tensor ` +
456
+ `${label} with shape ${JSON.stringify(expectedShape)} after tensor index ${cursor.index}.`
457
+ );
458
+ }
459
+
460
+ function takeOptionalConsecutiveLiteRTTensor(cursor, expectedShape, sourceDtype) {
461
+ const tensor = cursor.tensors[cursor.index] ?? null;
462
+ if (!isLiteRTTensorDtype(tensor, sourceDtype) || !shapeEquals(tensor.shape, expectedShape)) {
463
+ return null;
464
+ }
465
+ cursor.index += 1;
466
+ return tensor;
467
+ }
468
+
469
+ function collectGemma412BLayerScalarTensors(prefillTFLite, numLayers, packageConfig) {
470
+ const expectedLayers = normalizeLayerIndexList(
471
+ packageConfig?.layerScalarLayers,
472
+ numLayers,
473
+ 'package.litertlm.layerScalarLayers'
474
+ );
475
+ if (expectedLayers.length !== numLayers) {
476
+ const missingValue = Number(packageConfig?.missingLayerScalarValue);
477
+ if (!Number.isFinite(missingValue) || missingValue !== 1) {
478
+ throw new Error(
479
+ 'direct-source runtime: package.litertlm.missingLayerScalarValue must be 1 when ' +
480
+ 'package.litertlm.layerScalarLayers does not cover every layer.'
481
+ );
482
+ }
483
+ }
484
+
485
+ const expected = new Set(expectedLayers);
486
+ const layerScalars = new Map();
487
+ for (const tensor of prefillTFLite.tensors ?? []) {
488
+ if (!isLiteRTTensorDtype(tensor, 'F32') || !shapeEquals(tensor.shape, [])) {
489
+ continue;
490
+ }
491
+ const match = String(tensor.name ?? '').match(/Gemma4UnifiedTextDecoderLayer_(\d+);?$/);
492
+ if (!match) {
493
+ continue;
494
+ }
495
+ const layerIndex = Number(match[1]);
496
+ if (!Number.isInteger(layerIndex) || layerIndex < 0 || layerIndex >= numLayers) {
497
+ throw new Error(
498
+ `direct-source runtime: Gemma 4 12B layer_scalar tensor "${tensor.name}" ` +
499
+ `has out-of-range layer index ${String(match[1])}.`
500
+ );
501
+ }
502
+ if (!expected.has(layerIndex)) {
503
+ throw new Error(
504
+ `direct-source runtime: Gemma 4 12B layer_scalar tensor for layer ${layerIndex} ` +
505
+ 'is present in the LiteRT graph but missing from package.litertlm.layerScalarLayers.'
506
+ );
507
+ }
508
+ if (layerScalars.has(layerIndex)) {
509
+ throw new Error(
510
+ `direct-source runtime: Gemma 4 12B LiteRT graph has duplicate layer_scalar tensors for layer ${layerIndex}.`
511
+ );
512
+ }
513
+ layerScalars.set(layerIndex, tensor);
514
+ }
515
+
516
+ for (const layerIndex of expectedLayers) {
517
+ if (!layerScalars.has(layerIndex)) {
518
+ throw new Error(
519
+ `direct-source runtime: Gemma 4 12B split-section adapter expected layer_scalar for layer ${layerIndex}, ` +
520
+ 'but the LiteRT graph did not contain a matching Gemma4UnifiedTextDecoderLayer scalar tensor.'
521
+ );
522
+ }
523
+ }
524
+ return layerScalars;
525
+ }
526
+
527
+ function createLiteRTFixedAffineInt4Tensor(
528
+ rawTensor,
529
+ sourcePath,
530
+ canonicalName,
531
+ role,
532
+ group,
533
+ logicalShape,
534
+ fixedScale,
535
+ storageEncoding
536
+ ) {
537
+ if (!rawTensor || typeof rawTensor !== 'object') {
538
+ throw new Error(`direct-source runtime: missing LiteRT tensor "${canonicalName}".`);
539
+ }
540
+ if (!Array.isArray(logicalShape) || logicalShape.length !== 2) {
541
+ throw new Error(
542
+ `direct-source runtime: LiteRT tensor "${canonicalName}" requires an explicit 2D logical shape.`
543
+ );
544
+ }
545
+ if (!shapeEquals(rawTensor.shape, logicalShape)) {
546
+ throw new Error(
547
+ `direct-source runtime: LiteRT tensor "${rawTensor.name}" shape ${JSON.stringify(rawTensor.shape)} ` +
548
+ `does not match expected split-section logical shape ${JSON.stringify(logicalShape)} for "${canonicalName}".`
549
+ );
550
+ }
551
+ const expectedSize = computePackedByteSize(logicalShape, 'INT4', canonicalName);
552
+ if (Number(rawTensor.size) !== expectedSize) {
553
+ throw new Error(
554
+ `direct-source runtime: LiteRT tensor "${rawTensor.name}" size ${rawTensor.size} does not match ` +
555
+ `INT4 packed shape ${JSON.stringify(logicalShape)} for "${canonicalName}".`
556
+ );
557
+ }
558
+ if (!Number.isFinite(fixedScale) || fixedScale <= 0) {
559
+ throw new Error(
560
+ `direct-source runtime: LiteRT tensor "${canonicalName}" requires package.litertlm.fixedInt4Scale > 0.`
561
+ );
562
+ }
563
+ return {
564
+ name: canonicalName,
565
+ shape: [Number(logicalShape[0]), Number(logicalShape[1])],
566
+ dtype: 'F16',
567
+ offset: rawTensor.offset,
568
+ size: rawTensor.size,
569
+ sourcePath,
570
+ role,
571
+ ...(group ? { group } : {}),
572
+ sourceTransform: {
573
+ kind: 'affine_dequant',
574
+ scheme: 'per_tensor_affine',
575
+ sourceDtype: 'INT4',
576
+ targetDtype: 'F16',
577
+ storageEncoding,
578
+ scale: fixedScale,
579
+ zeroPoint: 0,
580
+ },
581
+ };
582
+ }
583
+
584
+ function resolveGemma412BLayerTypes(runtimeProfile) {
585
+ const explicitLayerTypes = runtimeProfile?.manifestInference?.layerPattern?.layerTypes;
586
+ if (Array.isArray(explicitLayerTypes) && explicitLayerTypes.length > 0) {
587
+ return explicitLayerTypes.map((value) => normalizeText(value));
588
+ }
589
+ const numLayers = Number(runtimeProfile?.architecture?.numLayers ?? 0);
590
+ if (!Number.isInteger(numLayers) || numLayers <= 0) {
591
+ throw new Error('direct-source runtime: Gemma 4 12B split-section adapter is missing architecture.numLayers.');
592
+ }
593
+ const layerTypes = [];
594
+ for (let layerIndex = 0; layerIndex < numLayers; layerIndex += 1) {
595
+ layerTypes.push(isGemma4GlobalLayer(runtimeProfile, layerIndex) ? 'full_attention' : 'sliding_attention');
596
+ }
597
+ return layerTypes;
598
+ }
599
+
600
+ function addFloatToNormalized(normalized, rawTensor, sourcePath, canonicalName, role, group = null) {
601
+ if (!rawTensor) {
602
+ throw new Error(`direct-source runtime: missing LiteRT tensor "${canonicalName}".`);
603
+ }
604
+ normalized.push(createLiteRTFloatTensor(rawTensor, sourcePath, canonicalName, role, group));
605
+ }
606
+
355
607
  function createLiteRTRowwiseTensor(
356
608
  rawTensor,
357
609
  scaleTensor,
@@ -773,6 +1025,191 @@ function createLiteRTBlockedAxisTensor(
773
1025
  };
774
1026
  }
775
1027
 
1028
+ function normalizeGemma412BSplitLiteRTTensors(parsedTFLitesByPath, virtualFiles, runtimeProfile, packageConfig) {
1029
+ const embedderModelType = normalizeText(packageConfig?.embedderTFLiteModelType);
1030
+ const prefillModelType = normalizeText(packageConfig?.tfliteModelType);
1031
+ const embedderSourceFile = virtualFiles.find((entry) => entry.kind === 'tflite_embedder_model') ?? null;
1032
+ const prefillSourceFile = virtualFiles.find((entry) => entry.kind === 'tflite_model') ?? null;
1033
+ if (!embedderModelType || !embedderSourceFile) {
1034
+ throw new Error(
1035
+ 'direct-source runtime: Gemma 4 12B split-section adapter requires package.litertlm.embedderTFLiteModelType.'
1036
+ );
1037
+ }
1038
+ if (!prefillModelType || !prefillSourceFile) {
1039
+ throw new Error(
1040
+ 'direct-source runtime: Gemma 4 12B split-section adapter requires package.litertlm.tfliteModelType.'
1041
+ );
1042
+ }
1043
+ const embedderTFLite = parsedTFLitesByPath?.get(embedderSourceFile.path) ?? null;
1044
+ const prefillTFLite = parsedTFLitesByPath?.get(prefillSourceFile.path) ?? null;
1045
+ if (!embedderTFLite || !prefillTFLite) {
1046
+ throw new Error('direct-source runtime: Gemma 4 12B split-section adapter did not receive both parsed TFLite sections.');
1047
+ }
1048
+
1049
+ const arch = runtimeProfile?.architecture ?? {};
1050
+ const hiddenSize = Number(arch.hiddenSize ?? 0);
1051
+ const vocabSize = Number(arch.vocabSize ?? 0);
1052
+ const intermediateSize = Number(arch.intermediateSize ?? 0);
1053
+ const numAttentionHeads = Number(arch.numAttentionHeads ?? 0);
1054
+ const numKeyValueHeads = Number(arch.numKeyValueHeads ?? 0);
1055
+ const numGlobalKeyValueHeads = Number(arch.numGlobalKeyValueHeads ?? numKeyValueHeads);
1056
+ const headDim = Number(arch.headDim ?? 0);
1057
+ const globalHeadDim = Number(arch.globalHeadDim ?? headDim);
1058
+ const fixedInt4Scale = Number(packageConfig?.fixedInt4Scale);
1059
+ const fixedInt4StorageEncoding = normalizeLiteRTStorageEncoding(
1060
+ packageConfig?.fixedInt4StorageEncoding,
1061
+ 'package.litertlm.fixedInt4StorageEncoding'
1062
+ );
1063
+ if (
1064
+ !Number.isInteger(hiddenSize)
1065
+ || hiddenSize <= 0
1066
+ || !Number.isInteger(vocabSize)
1067
+ || vocabSize <= 0
1068
+ || !Number.isInteger(intermediateSize)
1069
+ || intermediateSize <= 0
1070
+ || !Number.isInteger(numAttentionHeads)
1071
+ || numAttentionHeads <= 0
1072
+ || !Number.isInteger(numKeyValueHeads)
1073
+ || numKeyValueHeads <= 0
1074
+ || !Number.isInteger(numGlobalKeyValueHeads)
1075
+ || numGlobalKeyValueHeads <= 0
1076
+ || !Number.isInteger(headDim)
1077
+ || headDim <= 0
1078
+ || !Number.isInteger(globalHeadDim)
1079
+ || globalHeadDim <= 0
1080
+ || !Number.isFinite(fixedInt4Scale)
1081
+ || fixedInt4Scale <= 0
1082
+ ) {
1083
+ throw new Error(
1084
+ 'direct-source runtime: Gemma 4 12B split-section adapter requires explicit architecture sizes and fixedInt4Scale.'
1085
+ );
1086
+ }
1087
+
1088
+ const normalized = [];
1089
+ const embedderTensor = embedderTFLite.tensors.find((tensor) => (
1090
+ isLiteRTTensorDtype(tensor, 'INT4') && shapeEquals(tensor.shape, [vocabSize, hiddenSize])
1091
+ )) ?? null;
1092
+ normalized.push(
1093
+ createLiteRTFixedAffineInt4Tensor(
1094
+ embedderTensor,
1095
+ embedderSourceFile.path,
1096
+ 'model.language_model.embed_tokens.weight',
1097
+ 'embedding',
1098
+ 'embed',
1099
+ [vocabSize, hiddenSize],
1100
+ fixedInt4Scale,
1101
+ fixedInt4StorageEncoding
1102
+ )
1103
+ );
1104
+
1105
+ const cursor = createGemma412BSplitCursor(prefillTFLite, prefillSourceFile.path);
1106
+ const leadingFinalNorm = takeNextLiteRTTensor(cursor, [hiddenSize], 'F32', 'model.language_model.norm.weight', {
1107
+ optional: true,
1108
+ });
1109
+ if (leadingFinalNorm) {
1110
+ cursor.lastHiddenNorm = leadingFinalNorm;
1111
+ takeOptionalConsecutiveLiteRTTensor(cursor, [numGlobalKeyValueHeads * globalHeadDim, hiddenSize], 'INT4');
1112
+ takeOptionalConsecutiveLiteRTTensor(cursor, [globalHeadDim], 'F32');
1113
+ }
1114
+ const layerTypes = resolveGemma412BLayerTypes(runtimeProfile);
1115
+ const layerScalarTensors = collectGemma412BLayerScalarTensors(
1116
+ prefillTFLite,
1117
+ layerTypes.length,
1118
+ packageConfig
1119
+ );
1120
+ for (let layerIndex = 0; layerIndex < layerTypes.length; layerIndex += 1) {
1121
+ const canonicalLayerPrefix = `model.language_model.layers.${layerIndex}`;
1122
+ const globalLayer = normalizeText(layerTypes[layerIndex]) === 'full_attention';
1123
+ const attentionHeadDim = globalLayer ? globalHeadDim : headDim;
1124
+ const kvHeads = globalLayer ? numGlobalKeyValueHeads : numKeyValueHeads;
1125
+ const qRows = numAttentionHeads * attentionHeadDim;
1126
+ const kvRows = kvHeads * attentionHeadDim;
1127
+
1128
+ const addHiddenNorm = (canonicalName) => {
1129
+ const rawTensor = takeNextLiteRTTensor(cursor, [hiddenSize], 'F32', canonicalName);
1130
+ cursor.lastHiddenNorm = rawTensor;
1131
+ normalized.push(createLiteRTFloatTensor(rawTensor, prefillSourceFile.path, canonicalName, 'norm'));
1132
+ };
1133
+ const addFloat = (rawTensor, canonicalName, role = 'norm') => {
1134
+ if (!rawTensor) return;
1135
+ normalized.push(createLiteRTFloatTensor(rawTensor, prefillSourceFile.path, canonicalName, role));
1136
+ };
1137
+ const addInt4 = (rawTensor, canonicalName, role, logicalShape) => {
1138
+ normalized.push(
1139
+ createLiteRTFixedAffineInt4Tensor(
1140
+ rawTensor,
1141
+ prefillSourceFile.path,
1142
+ canonicalName,
1143
+ role,
1144
+ null,
1145
+ logicalShape,
1146
+ fixedInt4Scale,
1147
+ fixedInt4StorageEncoding
1148
+ )
1149
+ );
1150
+ };
1151
+
1152
+ addFloat(layerScalarTensors.get(layerIndex) ?? null, `${canonicalLayerPrefix}.layer_scalar`, 'other');
1153
+ addHiddenNorm(`${canonicalLayerPrefix}.input_layernorm.weight`);
1154
+ addHiddenNorm(`${canonicalLayerPrefix}.post_attention_layernorm.weight`);
1155
+ addHiddenNorm(`${canonicalLayerPrefix}.pre_feedforward_layernorm.weight`);
1156
+ addHiddenNorm(`${canonicalLayerPrefix}.post_feedforward_layernorm.weight`);
1157
+
1158
+ addInt4(
1159
+ takeNextLiteRTTensor(cursor, [hiddenSize, intermediateSize], 'INT4', `${canonicalLayerPrefix}.mlp.down_proj.weight`),
1160
+ `${canonicalLayerPrefix}.mlp.down_proj.weight`,
1161
+ 'matmul',
1162
+ [hiddenSize, intermediateSize]
1163
+ );
1164
+ addInt4(
1165
+ takeNextLiteRTTensor(cursor, [intermediateSize, hiddenSize], 'INT4', `${canonicalLayerPrefix}.mlp.gate_proj.weight`),
1166
+ `${canonicalLayerPrefix}.mlp.gate_proj.weight`,
1167
+ 'matmul',
1168
+ [intermediateSize, hiddenSize]
1169
+ );
1170
+ addInt4(
1171
+ takeNextLiteRTTensor(cursor, [intermediateSize, hiddenSize], 'INT4', `${canonicalLayerPrefix}.mlp.up_proj.weight`),
1172
+ `${canonicalLayerPrefix}.mlp.up_proj.weight`,
1173
+ 'matmul',
1174
+ [intermediateSize, hiddenSize]
1175
+ );
1176
+ addInt4(
1177
+ takeNextLiteRTTensor(cursor, [hiddenSize, qRows], 'INT4', `${canonicalLayerPrefix}.self_attn.o_proj.weight`),
1178
+ `${canonicalLayerPrefix}.self_attn.o_proj.weight`,
1179
+ 'matmul',
1180
+ [hiddenSize, qRows]
1181
+ );
1182
+ const kProj = takeNextLiteRTTensor(cursor, [kvRows, hiddenSize], 'INT4', `${canonicalLayerPrefix}.self_attn.k_proj.weight`);
1183
+ const vProj = takeOptionalConsecutiveLiteRTTensor(cursor, [kvRows, hiddenSize], 'INT4') ?? kProj;
1184
+ addInt4(kProj, `${canonicalLayerPrefix}.self_attn.k_proj.weight`, 'matmul', [kvRows, hiddenSize]);
1185
+ addInt4(vProj, `${canonicalLayerPrefix}.self_attn.v_proj.weight`, 'matmul', [kvRows, hiddenSize]);
1186
+
1187
+ const firstNorm = takeOptionalConsecutiveLiteRTTensor(cursor, [attentionHeadDim], 'F32');
1188
+ const secondNorm = takeOptionalConsecutiveLiteRTTensor(cursor, [attentionHeadDim], 'F32');
1189
+ if (firstNorm) {
1190
+ addFloat(firstNorm, `${canonicalLayerPrefix}.self_attn.q_norm.weight`);
1191
+ addFloat(secondNorm ?? firstNorm, `${canonicalLayerPrefix}.self_attn.k_norm.weight`);
1192
+ }
1193
+
1194
+ addInt4(
1195
+ takeNextLiteRTTensor(cursor, [qRows, hiddenSize], 'INT4', `${canonicalLayerPrefix}.self_attn.q_proj.weight`),
1196
+ `${canonicalLayerPrefix}.self_attn.q_proj.weight`,
1197
+ 'matmul',
1198
+ [qRows, hiddenSize]
1199
+ );
1200
+ }
1201
+
1202
+ const finalNorm = takeNextLiteRTTensor(cursor, [hiddenSize], 'F32', 'model.language_model.norm.weight', {
1203
+ optional: true,
1204
+ }) ?? leadingFinalNorm ?? cursor.lastHiddenNorm;
1205
+ addFloatToNormalized(normalized, finalNorm, prefillSourceFile.path, 'model.language_model.norm.weight', 'norm', 'head');
1206
+
1207
+ if (normalized.length === 0) {
1208
+ throw new Error('direct-source runtime: Gemma 4 12B split-section adapter did not produce any normalized tensors.');
1209
+ }
1210
+ return normalized;
1211
+ }
1212
+
776
1213
  function normalizeGemma4LiteRTTensors(parsedTFLite, sourcePath, runtimeProfile) {
777
1214
  const rawByName = new Map();
778
1215
  for (const tensor of parsedTFLite.tensors) {
@@ -1024,7 +1461,18 @@ function normalizeGemma4LiteRTTensors(parsedTFLite, sourcePath, runtimeProfile)
1024
1461
  return normalized;
1025
1462
  }
1026
1463
 
1027
- function buildPackageParsedArtifact(sourceKind, sourcePathForModelId, runtimeProfile, parsedTFLite, virtualFiles) {
1464
+ function buildPackageParsedArtifact(
1465
+ sourceKind,
1466
+ sourcePathForModelId,
1467
+ runtimeProfile,
1468
+ parsedTFLite,
1469
+ virtualFiles,
1470
+ packageConfig = null,
1471
+ parsedTFLitesByPath = null
1472
+ ) {
1473
+ const tfliteSourceFiles = virtualFiles.filter((entry) => (
1474
+ entry.kind === 'tflite_model' || entry.kind === 'tflite_embedder_model'
1475
+ ));
1028
1476
  const tfliteSourceFile = virtualFiles.find((entry) => entry.kind === 'tflite_model') ?? null;
1029
1477
  if (!tfliteSourceFile) {
1030
1478
  throw new Error('direct-source runtime: LiteRT package is missing a TFLite model entry.');
@@ -1034,18 +1482,45 @@ function buildPackageParsedArtifact(sourceKind, sourcePathForModelId, runtimePro
1034
1482
  const tokenizerModelFile = virtualFiles.find((entry) => entry.kind === 'tokenizer_model') ?? null;
1035
1483
  const metadataFile = virtualFiles.find((entry) => entry.kind === 'litert_metadata') ?? null;
1036
1484
  const config = cloneJsonValue(runtimeProfile.rawConfig ?? {});
1037
- const tensors = normalizeText(runtimeProfile.modelType) === 'gemma4'
1038
- ? normalizeGemma4LiteRTTensors(parsedTFLite, tfliteSourceFile.path, runtimeProfile)
1039
- : parsedTFLite.tensors.map((tensor) => ({
1485
+ const graphAdapter = normalizeText(packageConfig?.graphAdapter);
1486
+ const tensors = graphAdapter === 'gemma4_unified_12b_split_int4'
1487
+ ? normalizeGemma412BSplitLiteRTTensors(parsedTFLitesByPath, virtualFiles, runtimeProfile, packageConfig)
1488
+ : (normalizeText(runtimeProfile.modelType) === 'gemma4'
1489
+ ? normalizeGemma4LiteRTTensors(parsedTFLite, tfliteSourceFile.path, runtimeProfile)
1490
+ : parsedTFLite.tensors.map((tensor) => ({
1040
1491
  ...tensor,
1041
1492
  sourcePath: tfliteSourceFile.path,
1042
- }));
1493
+ })));
1494
+ const manifestInference = cloneJsonValue(runtimeProfile.manifestInference ?? {});
1495
+ if (graphAdapter === 'gemma4_unified_12b_split_int4') {
1496
+ const qNormLayers = new Set();
1497
+ const kNormLayers = new Set();
1498
+ for (const tensor of tensors) {
1499
+ const qMatch = String(tensor?.name ?? '').match(/\.layers\.(\d+)\.self_attn\.q_norm\.weight$/);
1500
+ if (qMatch) {
1501
+ qNormLayers.add(Number(qMatch[1]));
1502
+ }
1503
+ const kMatch = String(tensor?.name ?? '').match(/\.layers\.(\d+)\.self_attn\.k_norm\.weight$/);
1504
+ if (kMatch) {
1505
+ kNormLayers.add(Number(kMatch[1]));
1506
+ }
1507
+ }
1508
+ const layers = [...qNormLayers]
1509
+ .filter((layerIdx) => kNormLayers.has(layerIdx))
1510
+ .sort((left, right) => left - right);
1511
+ manifestInference.attention = {
1512
+ ...(manifestInference.attention ?? {}),
1513
+ queryKeyNorm: manifestInference.attention?.queryKeyNorm === true || layers.length > 0,
1514
+ queryKeyNormLayers: null,
1515
+ queryKeyNormWeightLayers: layers,
1516
+ };
1517
+ }
1043
1518
  return {
1044
1519
  sourceKind,
1045
1520
  modelType: normalizeText(runtimeProfile.modelType),
1046
1521
  config,
1047
1522
  manifestConfig: cloneJsonValue(runtimeProfile.manifestConfig ?? {}),
1048
- manifestInference: cloneJsonValue(runtimeProfile.manifestInference ?? {}),
1523
+ manifestInference,
1049
1524
  architectureHint: normalizeText(runtimeProfile.modelType) || 'transformer',
1050
1525
  embeddingPostprocessor: null,
1051
1526
  architecture: cloneJsonValue(runtimeProfile.architecture ?? null),
@@ -1056,12 +1531,10 @@ function buildPackageParsedArtifact(sourceKind, sourcePathForModelId, runtimePro
1056
1531
  tokenizerJsonPath: tokenizerJsonFile ? tokenizerJsonFile.path : null,
1057
1532
  tokenizerConfigPath: tokenizerConfigFile ? tokenizerConfigFile.path : null,
1058
1533
  tokenizerModelPath: tokenizerModelFile ? tokenizerModelFile.path : null,
1059
- sourceFiles: [
1060
- {
1061
- path: tfliteSourceFile.path,
1062
- size: tfliteSourceFile.size,
1063
- },
1064
- ],
1534
+ sourceFiles: tfliteSourceFiles.map((entry) => ({
1535
+ path: entry.path,
1536
+ size: entry.size,
1537
+ })),
1065
1538
  auxiliaryFiles: [
1066
1539
  ...(tokenizerJsonFile
1067
1540
  ? [{
@@ -1110,12 +1583,18 @@ async function isRawTFLiteTaskSource(source) {
1110
1583
  async function parseLiteRTTaskPackage(source, sourcePathForModelId) {
1111
1584
  const packageBasename = normalizeText(source?.name);
1112
1585
  const profile = resolveRequiredProfile(LITERT_PACKAGE_SOURCE_KIND_TASK, packageBasename);
1586
+ const taskConfig = profile.package?.task ?? {};
1587
+ throwIfUnsupportedPackageProfile(
1588
+ LITERT_PACKAGE_SOURCE_KIND_TASK,
1589
+ packageBasename,
1590
+ profile,
1591
+ taskConfig
1592
+ );
1113
1593
  const runtimeProfile = profile.runtime ?? null;
1114
1594
  if (!runtimeProfile) {
1115
1595
  throw new Error(`direct-source runtime: package profile "${profile.id}" is missing runtime data.`);
1116
1596
  }
1117
1597
 
1118
- const taskConfig = profile.package?.task ?? {};
1119
1598
  const rawTFLiteTask = await isRawTFLiteTaskSource(source);
1120
1599
  const virtualFiles = [];
1121
1600
  let parsedTFLite = null;
@@ -1179,7 +1658,8 @@ async function parseLiteRTTaskPackage(source, sourcePathForModelId) {
1179
1658
  sourcePathForModelId,
1180
1659
  runtimeProfile,
1181
1660
  parsedTFLite,
1182
- virtualFiles
1661
+ virtualFiles,
1662
+ taskConfig
1183
1663
  ),
1184
1664
  virtualFiles,
1185
1665
  packageProfile: profile,
@@ -1189,13 +1669,20 @@ async function parseLiteRTTaskPackage(source, sourcePathForModelId) {
1189
1669
  async function parseLiteRTLMPackage(source, sourcePathForModelId) {
1190
1670
  const packageBasename = normalizeText(source?.name);
1191
1671
  const profile = resolveRequiredProfile(LITERT_PACKAGE_SOURCE_KIND_LITERTLM, packageBasename);
1672
+ const litertConfig = profile.package?.litertlm ?? {};
1673
+ throwIfUnsupportedPackageProfile(
1674
+ LITERT_PACKAGE_SOURCE_KIND_LITERTLM,
1675
+ packageBasename,
1676
+ profile,
1677
+ litertConfig
1678
+ );
1192
1679
  const runtimeProfile = profile.runtime ?? null;
1193
1680
  if (!runtimeProfile) {
1194
1681
  throw new Error(`direct-source runtime: package profile "${profile.id}" is missing runtime data.`);
1195
1682
  }
1196
1683
 
1197
1684
  const parsedLiteRTLM = await parseLiteRTLMFromSource(source);
1198
- const litertConfig = profile.package?.litertlm ?? {};
1685
+ const graphAdapter = normalizeText(litertConfig.graphAdapter);
1199
1686
  const tfliteModelType = normalizeText(litertConfig.tfliteModelType) || LITERT_TASK_DEFAULT_TFLITE_ENTRY;
1200
1687
  const weightsSection = findLiteRTLMTFLiteWeightsSection(parsedLiteRTLM, tfliteModelType);
1201
1688
  if (weightsSection) {
@@ -1217,6 +1704,55 @@ async function parseLiteRTLMPackage(source, sourcePathForModelId) {
1217
1704
 
1218
1705
  const tokenizerSection = findLiteRTLMSentencePieceTokenizerSection(parsedLiteRTLM);
1219
1706
  const metadataSection = findLiteRTLMMetadataSection(parsedLiteRTLM);
1707
+ if (graphAdapter === 'gemma4_unified_12b_split_int4') {
1708
+ const embedderTFLiteModelType = normalizeText(litertConfig.embedderTFLiteModelType);
1709
+ if (!embedderTFLiteModelType) {
1710
+ throw new Error(
1711
+ `direct-source runtime: LiteRT-LM "${packageBasename}" split-section graph adapter requires ` +
1712
+ 'package.litertlm.embedderTFLiteModelType.'
1713
+ );
1714
+ }
1715
+ const prefillSection = findLiteRTLMTFLiteModelSection(parsedLiteRTLM, tfliteModelType);
1716
+ const embedderSection = findLiteRTLMTFLiteModelSection(parsedLiteRTLM, embedderTFLiteModelType);
1717
+ if (!prefillSection || !embedderSection) {
1718
+ throw new Error(
1719
+ `direct-source runtime: LiteRT-LM "${packageBasename}" split-section graph adapter requires ` +
1720
+ `TFLiteModel sections "${embedderTFLiteModelType}" and "${tfliteModelType}".`
1721
+ );
1722
+ }
1723
+ const virtualFiles = [
1724
+ createVirtualFile(embedderTFLiteModelType, embedderSection.beginOffset, embedderSection.size, 'tflite_embedder_model'),
1725
+ createVirtualFile(tfliteModelType, prefillSection.beginOffset, prefillSection.size, 'tflite_model'),
1726
+ ];
1727
+ if (tokenizerSection) {
1728
+ virtualFiles.push(createVirtualFile('TOKENIZER_MODEL', tokenizerSection.beginOffset, tokenizerSection.size, 'tokenizer_model'));
1729
+ }
1730
+ if (metadataSection) {
1731
+ virtualFiles.push(createVirtualFile('METADATA', metadataSection.beginOffset, metadataSection.size, 'litert_metadata'));
1732
+ }
1733
+ const parsedTFLitesByPath = new Map();
1734
+ for (const virtualFile of virtualFiles.filter((entry) => entry.kind === 'tflite_model' || entry.kind === 'tflite_embedder_model')) {
1735
+ parsedTFLitesByPath.set(
1736
+ virtualFile.path,
1737
+ await parseTFLiteFromSource(createSectionSource(source, virtualFile), {
1738
+ allowPackedQuantization: true,
1739
+ })
1740
+ );
1741
+ }
1742
+ return {
1743
+ parsedArtifact: buildPackageParsedArtifact(
1744
+ LITERT_PACKAGE_SOURCE_KIND_LITERTLM,
1745
+ sourcePathForModelId,
1746
+ runtimeProfile,
1747
+ parsedTFLitesByPath.get(tfliteModelType),
1748
+ virtualFiles,
1749
+ litertConfig,
1750
+ parsedTFLitesByPath
1751
+ ),
1752
+ virtualFiles,
1753
+ packageProfile: profile,
1754
+ };
1755
+ }
1220
1756
  const errors = [];
1221
1757
 
1222
1758
  for (let candidateIndex = 0; candidateIndex < modelSections.length; candidateIndex += 1) {
@@ -1244,7 +1780,8 @@ async function parseLiteRTLMPackage(source, sourcePathForModelId) {
1244
1780
  sourcePathForModelId,
1245
1781
  runtimeProfile,
1246
1782
  parsedTFLite,
1247
- virtualFiles
1783
+ virtualFiles,
1784
+ litertConfig
1248
1785
  ),
1249
1786
  virtualFiles,
1250
1787
  packageProfile: profile,