doppler-gpu 0.4.3 → 0.4.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (435) hide show
  1. package/CHANGELOG.md +28 -0
  2. package/README.md +67 -34
  3. package/assets/doppler.svg +57 -1601
  4. package/package.json +36 -5
  5. package/src/cli/cli-output.js +34 -1
  6. package/src/cli/doppler-cli.d.ts +18 -0
  7. package/src/cli/doppler-cli.js +68 -35
  8. package/src/cli/doppler-serve.d.ts +210 -0
  9. package/src/cli/doppler-serve.js +468 -42
  10. package/src/client/doppler-registry.json +3 -5
  11. package/src/client/provider.d.ts +1 -1
  12. package/src/client/provider.js +1 -1
  13. package/src/client/runtime/index.js +13 -0
  14. package/src/client/runtime/lora.d.ts +11 -1
  15. package/src/client/runtime/lora.js +28 -7
  16. package/src/client/runtime/model-manager.js +35 -11
  17. package/src/client/runtime/model-session.d.ts +2 -1
  18. package/src/client/runtime/model-session.js +2 -2
  19. package/src/client/runtime/model-source.js +20 -1
  20. package/src/client/runtime/types.d.ts +8 -0
  21. package/src/client/wrap-pipeline-handle.d.ts +62 -0
  22. package/src/client/wrap-pipeline-handle.js +188 -13
  23. package/src/config/conversion/README.md +5 -0
  24. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +646 -0
  25. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +3 -3
  26. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +4 -4
  27. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +4 -4
  28. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +393 -0
  29. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +425 -0
  30. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +17 -9
  31. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +4 -4
  32. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +4 -4
  33. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +737 -0
  34. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +730 -0
  35. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +780 -0
  36. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +716 -0
  37. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +5 -5
  38. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +4 -4
  39. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +14 -12
  40. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +7 -7
  41. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +7 -7
  42. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +3 -3
  43. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +3 -3
  44. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +3 -3
  45. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +3 -3
  46. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +8 -4
  47. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +8 -4
  48. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +9 -5
  49. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +8 -4
  50. package/src/config/goal-completion-matrix.json +552 -0
  51. package/src/config/kernel-path-loader.js +192 -38
  52. package/src/config/kernels/kernel-ref-digests.js +52 -18
  53. package/src/config/kernels/registry.json +3017 -744
  54. package/src/config/merge-contract-check.js +7 -3
  55. package/src/config/merge-helpers.d.ts +2 -0
  56. package/src/config/merge-helpers.js +1 -0
  57. package/src/config/merge.d.ts +1 -0
  58. package/src/config/merge.js +30 -0
  59. package/src/config/model-lanes/catalog-lane-resolver.d.ts +66 -0
  60. package/src/config/model-lanes/catalog-lane-resolver.js +170 -0
  61. package/src/config/param-validator.js +4 -0
  62. package/src/config/runtime/experiments/verify/lfm2-verify.json +3 -3
  63. package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +26 -0
  64. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +42 -0
  65. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +39 -0
  66. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +48 -0
  67. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +46 -0
  68. package/src/config/schema/buffer-pool.schema.js +3 -3
  69. package/src/config/schema/claim-evidence-contract.schema.json +93 -0
  70. package/src/config/schema/command-surface-contract.schema.json +81 -0
  71. package/src/config/schema/converter.schema.d.ts +11 -0
  72. package/src/config/schema/converter.schema.js +8 -0
  73. package/src/config/schema/doppler.schema.d.ts +1 -0
  74. package/src/config/schema/doppler.schema.js +38 -6
  75. package/src/config/schema/execution-v1.schema.d.ts +27 -0
  76. package/src/config/schema/execution-v1.schema.js +28 -19
  77. package/src/config/schema/goal-completion-matrix.schema.json +223 -0
  78. package/src/config/schema/inference.schema.d.ts +11 -0
  79. package/src/config/schema/manifest.schema.d.ts +107 -1
  80. package/src/config/schema/manifest.schema.js +10 -1
  81. package/src/config/schema/model-artifact-contract.schema.json +119 -0
  82. package/src/config/schema/moe.schema.d.ts +13 -0
  83. package/src/config/schema/moe.schema.js +1 -1
  84. package/src/config/schema/policy-schema-registry.json +26 -0
  85. package/src/config/schema/subsystem-support-contract.schema.json +48 -0
  86. package/src/config/schema/training-metrics.schema.d.ts +1 -1
  87. package/src/config/schema/training-metrics.schema.js +4 -3
  88. package/src/config/source-packages/litert/gemma-4-12b-it.json +390 -0
  89. package/src/config/source-packages/litert/gemma-4-e2b-it.json +7 -7
  90. package/src/config/source-packages/litert/gemma-4-e4b-it.json +370 -0
  91. package/src/config/source-packages/registry.json +25 -0
  92. package/src/config/supported-operations.json +5 -0
  93. package/src/config/transforms/capability-transform-resolver.d.ts +13 -2
  94. package/src/config/transforms/execution-graph-transforms.d.ts +42 -0
  95. package/src/config/transforms/execution-graph-transforms.js +227 -230
  96. package/src/converter/conversion-plan.d.ts +2 -0
  97. package/src/converter/conversion-plan.js +18 -4
  98. package/src/converter/core.d.ts +8 -0
  99. package/src/converter/core.js +574 -12
  100. package/src/converter/quantization-info.js +179 -2
  101. package/src/converter/quantizer.d.ts +5 -4
  102. package/src/converter/quantizer.js +16 -2
  103. package/src/experimental/adapters/lora-loader.js +109 -24
  104. package/src/experimental/diffusion/index.d.ts +24 -0
  105. package/src/experimental/diffusion/index.js +10 -0
  106. package/src/experimental/distribution/descriptor-transport.d.ts +117 -0
  107. package/src/experimental/distribution/descriptor-transport.js +342 -0
  108. package/src/experimental/training/README.md +11 -0
  109. package/src/experimental/training/autograd.js +33 -8
  110. package/src/experimental/training/datasets/gepa-frontier.d.ts +41 -0
  111. package/src/experimental/training/datasets/gepa-frontier.js +220 -0
  112. package/src/experimental/training/datasets/index.d.ts +13 -1
  113. package/src/experimental/training/datasets/index.js +8 -1
  114. package/src/experimental/training/datasets/teacher-traces.d.ts +51 -0
  115. package/src/experimental/training/datasets/teacher-traces.js +279 -0
  116. package/src/experimental/training/datasets/text-pairs.d.ts +45 -0
  117. package/src/experimental/training/datasets/text-pairs.js +82 -5
  118. package/src/experimental/training/distillation/student-fixture.d.ts +5 -0
  119. package/src/experimental/training/distillation/student-fixture.js +290 -32
  120. package/src/experimental/training/export.d.ts +9 -0
  121. package/src/experimental/training/export.js +113 -7
  122. package/src/experimental/training/index.d.ts +13 -1
  123. package/src/experimental/training/index.js +13 -1
  124. package/src/experimental/training/lora-pipeline.d.ts +143 -0
  125. package/src/experimental/training/lora-pipeline.js +1737 -49
  126. package/src/experimental/training/lora.js +23 -1
  127. package/src/experimental/training/loss.js +1 -1
  128. package/src/experimental/training/operator-agent-eval.d.ts +65 -0
  129. package/src/experimental/training/operator-agent-eval.js +326 -0
  130. package/src/experimental/training/operator-command.js +310 -51
  131. package/src/experimental/training/runner.d.ts +1 -0
  132. package/src/experimental/training/runner.js +4 -0
  133. package/src/experimental/training/workload-packs/distill-glm52-doppler-agent-harness-gemma270m-f16-sft-tiny.json +160 -0
  134. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-gemma270m-f16-sft-tiny.json +175 -0
  135. package/src/experimental/training/workload-packs/distill-glm52-doppler-code-agent-sft-tiny.json +175 -0
  136. package/src/experimental/training/workload-packs/distill-glm52-doppler-js-json-gemma270m-f16-sft-tiny.json +160 -0
  137. package/src/experimental/training/workload-packs/distill-glm52-doppler-review-gemma270m-f16-sft-tiny.json +162 -0
  138. package/src/experimental/training/workload-packs/distill-glm52-doppler-wgsl-gemma270m-f16-sft-tiny.json +160 -0
  139. package/src/experimental/training/workload-packs/lora-doppler-agent-harness-gemma270m-f16-tiny.json +137 -0
  140. package/src/experimental/training/workload-packs/lora-doppler-code-agent-gemma270m-f16-tiny.json +152 -0
  141. package/src/experimental/training/workload-packs/lora-doppler-code-agent-tiny.json +152 -0
  142. package/src/experimental/training/workload-packs/lora-doppler-js-json-gemma270m-f16-tiny.json +137 -0
  143. package/src/experimental/training/workload-packs/lora-doppler-review-gemma270m-f16-tiny.json +139 -0
  144. package/src/experimental/training/workload-packs/lora-doppler-wgsl-gemma270m-f16-tiny.json +137 -0
  145. package/src/experimental/training/workload-packs/registry.json +186 -6
  146. package/src/experimental/training/workloads.d.ts +37 -0
  147. package/src/experimental/training/workloads.js +110 -1
  148. package/src/formats/rdrr/functional-descriptor.d.ts +73 -0
  149. package/src/formats/rdrr/functional-descriptor.js +230 -0
  150. package/src/formats/rdrr/index.d.ts +1 -0
  151. package/src/formats/rdrr/index.js +1 -0
  152. package/src/formats/rdrr/parsing.js +22 -6
  153. package/src/formats/rdrr/source-transform-contract.js +10 -1
  154. package/src/formats/rdrr/storage-descriptor.js +9 -2
  155. package/src/formats/rdrr/types.d.ts +5 -3
  156. package/src/formats/rdrr/validation.js +15 -1
  157. package/src/gpu/command-recorder.d.ts +43 -3
  158. package/src/gpu/command-recorder.js +122 -25
  159. package/src/gpu/device.js +111 -11
  160. package/src/gpu/kernel-tuner/cache.js +1 -5
  161. package/src/gpu/kernels/attention.js +17 -9
  162. package/src/gpu/kernels/attention_decode_online_head256_f16kv.wgsl +236 -0
  163. package/src/gpu/kernels/attention_streaming_f16.wgsl +7 -5
  164. package/src/gpu/kernels/backward/cross_entropy_backward.js +72 -3
  165. package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +2 -2
  166. package/src/gpu/kernels/cast.js +74 -67
  167. package/src/gpu/kernels/codegen/patches/attention_streaming_f16.from.attention_streaming.diff +46 -1
  168. package/src/gpu/kernels/codegen/patches/gelu_f16.from.gelu.diff +0 -20
  169. package/src/gpu/kernels/codegen/patches/residual_f16.from.residual.diff +15 -12
  170. package/src/gpu/kernels/codegen/patches/rmsnorm_f16.from.rmsnorm.diff +59 -44
  171. package/src/gpu/kernels/diffusion-gemma-sampling.d.ts +19 -0
  172. package/src/gpu/kernels/diffusion-gemma-sampling.js +104 -0
  173. package/src/gpu/kernels/diffusion_gemma_sampling.wgsl +130 -0
  174. package/src/gpu/kernels/dispatch.js +13 -3
  175. package/src/gpu/kernels/fused_ffn.js +3 -4
  176. package/src/gpu/kernels/fused_matmul_litert_int4_f16a.wgsl +78 -0
  177. package/src/gpu/kernels/fused_matmul_litert_int4_f16a_f32out.wgsl +78 -0
  178. package/src/gpu/kernels/fused_matmul_litert_int4_f32a_f32out.wgsl +76 -0
  179. package/src/gpu/kernels/fused_matmul_rmsnorm.js +101 -80
  180. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a.wgsl +116 -0
  181. package/src/gpu/kernels/fused_matmul_w4a16_batched_f16a_f32out.wgsl +116 -0
  182. package/src/gpu/kernels/fused_matmul_w4a16_f16a.wgsl +95 -0
  183. package/src/gpu/kernels/fused_matmul_w4a16_f16a_f32out.wgsl +95 -0
  184. package/src/gpu/kernels/gather.d.ts +84 -1
  185. package/src/gpu/kernels/gather.js +353 -5
  186. package/src/gpu/kernels/gather_litert_int4_vec4_f16_out.wgsl +75 -0
  187. package/src/gpu/kernels/gather_litert_int4_vec4_f32_out.wgsl +72 -0
  188. package/src/gpu/kernels/gather_split4_f16_vec4_f16_out.wgsl +103 -0
  189. package/src/gpu/kernels/gather_split8_f16_vec4_f16_out.wgsl +143 -0
  190. package/src/gpu/kernels/gather_split8_f16_vec4_f32_out.wgsl +143 -0
  191. package/src/gpu/kernels/gelu.wgsl +11 -7
  192. package/src/gpu/kernels/gelu_f16.wgsl +11 -7
  193. package/src/gpu/kernels/gemma4-route-expert.d.ts +35 -0
  194. package/src/gpu/kernels/gemma4-route-expert.js +126 -0
  195. package/src/gpu/kernels/gemma4_route_q4_matmul_f16a.wgsl +210 -0
  196. package/src/gpu/kernels/index.d.ts +71 -0
  197. package/src/gpu/kernels/index.js +52 -0
  198. package/src/gpu/kernels/kv-cache-write.d.ts +22 -0
  199. package/src/gpu/kernels/kv-cache-write.js +126 -0
  200. package/src/gpu/kernels/kv_cache_write_f32_to_f16.wgsl +32 -0
  201. package/src/gpu/kernels/lm-head-argmax.d.ts +19 -0
  202. package/src/gpu/kernels/lm-head-argmax.js +218 -0
  203. package/src/gpu/kernels/lm_head_argmax_f16.wgsl +189 -0
  204. package/src/gpu/kernels/matmul-dispatch.d.ts +11 -0
  205. package/src/gpu/kernels/matmul-dispatch.js +37 -1
  206. package/src/gpu/kernels/matmul-selection.d.ts +5 -5
  207. package/src/gpu/kernels/matmul-selection.js +104 -9
  208. package/src/gpu/kernels/matmul.d.ts +1 -1
  209. package/src/gpu/kernels/matmul.js +151 -43
  210. package/src/gpu/kernels/moe.d.ts +1 -0
  211. package/src/gpu/kernels/moe.js +46 -18
  212. package/src/gpu/kernels/pipeline-cache.d.ts +8 -0
  213. package/src/gpu/kernels/pipeline-cache.js +24 -0
  214. package/src/gpu/kernels/rep-penalty.d.ts +21 -0
  215. package/src/gpu/kernels/rep-penalty.js +17 -9
  216. package/src/gpu/kernels/residual.d.ts +1 -0
  217. package/src/gpu/kernels/residual.js +13 -1
  218. package/src/gpu/kernels/residual.wgsl +3 -3
  219. package/src/gpu/kernels/residual_f16.wgsl +5 -3
  220. package/src/gpu/kernels/residual_f16_vec4.wgsl +6 -6
  221. package/src/gpu/kernels/residual_vec4.wgsl +6 -6
  222. package/src/gpu/kernels/rmsnorm-pair.d.ts +39 -0
  223. package/src/gpu/kernels/rmsnorm-pair.js +152 -0
  224. package/src/gpu/kernels/rmsnorm-qk.d.ts +41 -0
  225. package/src/gpu/kernels/rmsnorm-qk.js +128 -0
  226. package/src/gpu/kernels/rmsnorm.d.ts +31 -0
  227. package/src/gpu/kernels/rmsnorm.js +32 -7
  228. package/src/gpu/kernels/rmsnorm.wgsl +10 -6
  229. package/src/gpu/kernels/rmsnorm_f16.wgsl +7 -3
  230. package/src/gpu/kernels/rmsnorm_pair.wgsl +224 -0
  231. package/src/gpu/kernels/rmsnorm_qk.wgsl +118 -0
  232. package/src/gpu/kernels/rmsnorm_qk_f16.wgsl +143 -0
  233. package/src/gpu/kernels/rope-qk.d.ts +41 -0
  234. package/src/gpu/kernels/rope-qk.js +143 -0
  235. package/src/gpu/kernels/rope_qk.wgsl +95 -0
  236. package/src/gpu/kernels/rope_qk_f16.wgsl +97 -0
  237. package/src/gpu/kernels/rule-registry.js +50 -10
  238. package/src/gpu/kernels/sample.js +212 -187
  239. package/src/gpu/kernels/scatter_add_dynamic_expert_scale.wgsl +47 -0
  240. package/src/gpu/kernels/scatter_add_dynamic_f16_expert_scale.wgsl +49 -0
  241. package/src/gpu/kernels/scatter_add_dynamic_f16_weights_expert_scale.wgsl +49 -0
  242. package/src/gpu/kernels/scatter_add_routes_f16_weights_expert_scale.wgsl +45 -0
  243. package/src/gpu/kernels/shader-cache.js +1 -21
  244. package/src/gpu/kernels/soft-embedding.d.ts +28 -0
  245. package/src/gpu/kernels/soft-embedding.js +447 -0
  246. package/src/gpu/kernels/soft_embedding_logits_f16.wgsl +107 -0
  247. package/src/gpu/kernels/soft_embedding_split_f16.wgsl +46 -0
  248. package/src/gpu/kernels/softmax.wgsl +5 -80
  249. package/src/gpu/kernels/split-qkv-rmsnorm-qk.d.ts +44 -0
  250. package/src/gpu/kernels/split-qkv-rmsnorm-qk.js +154 -0
  251. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.d.ts +59 -0
  252. package/src/gpu/kernels/split-qkv-rmsnorm-rope-qk.js +245 -0
  253. package/src/gpu/kernels/split_qkv_rmsnorm_qk.wgsl +146 -0
  254. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk.wgsl +185 -0
  255. package/src/gpu/kernels/split_qkv_rmsnorm_rope_qk_f16kv_cache.wgsl +187 -0
  256. package/src/gpu/kernels/uniform-utils.js +2 -5
  257. package/src/gpu/kernels/utils.d.ts +16 -1
  258. package/src/gpu/kernels/utils.js +42 -13
  259. package/src/gpu/readback-utils.js +10 -2
  260. package/src/gpu/uniform-cache.d.ts +10 -1
  261. package/src/gpu/uniform-cache.js +25 -7
  262. package/src/gpu/weight-buffer.d.ts +76 -7
  263. package/src/gpu/weight-buffer.js +85 -1
  264. package/src/inference/browser-harness-diffusion-energy-suites.js +269 -1
  265. package/src/inference/browser-harness-suite-helpers.d.ts +9 -0
  266. package/src/inference/browser-harness-suite-helpers.js +438 -2
  267. package/src/inference/browser-harness-text-helpers.d.ts +77 -0
  268. package/src/inference/browser-harness-text-helpers.js +236 -11
  269. package/src/inference/browser-harness.d.ts +8 -0
  270. package/src/inference/browser-harness.js +292 -8
  271. package/src/inference/decode-ring.d.ts +1 -0
  272. package/src/inference/decode-ring.js +7 -5
  273. package/src/inference/kv-cache/base.d.ts +25 -0
  274. package/src/inference/kv-cache/base.js +109 -1
  275. package/src/inference/kv-cache/mixed-geometry.d.ts +10 -0
  276. package/src/inference/kv-cache/mixed-geometry.js +109 -2
  277. package/src/inference/kv-cache/sliding-window.d.ts +14 -0
  278. package/src/inference/kv-cache/sliding-window.js +88 -0
  279. package/src/inference/model-lanes/execution-lane-resolver.d.ts +61 -0
  280. package/src/inference/model-lanes/execution-lane-resolver.js +126 -0
  281. package/src/inference/moe-router.d.ts +14 -1
  282. package/src/inference/moe-router.js +24 -1
  283. package/src/inference/pipelines/audio/gemma4.js +52 -1
  284. package/src/inference/pipelines/diffusion-gemma/config.d.ts +27 -0
  285. package/src/inference/pipelines/diffusion-gemma/config.js +149 -0
  286. package/src/inference/pipelines/diffusion-gemma/index.d.ts +31 -0
  287. package/src/inference/pipelines/diffusion-gemma/index.js +12 -0
  288. package/src/inference/pipelines/diffusion-gemma/pipeline.d.ts +69 -0
  289. package/src/inference/pipelines/diffusion-gemma/pipeline.js +380 -0
  290. package/src/inference/pipelines/diffusion-gemma/sampling.d.ts +128 -0
  291. package/src/inference/pipelines/diffusion-gemma/sampling.js +395 -0
  292. package/src/inference/pipelines/text/attention/dispatch-params.d.ts +28 -4
  293. package/src/inference/pipelines/text/attention/dispatch-params.js +176 -2
  294. package/src/inference/pipelines/text/attention/projections.d.ts +71 -2
  295. package/src/inference/pipelines/text/attention/projections.js +329 -49
  296. package/src/inference/pipelines/text/attention/quant-options.d.ts +38 -0
  297. package/src/inference/pipelines/text/attention/record.js +446 -73
  298. package/src/inference/pipelines/text/attention/run.js +251 -49
  299. package/src/inference/pipelines/text/attention/types.d.ts +5 -1
  300. package/src/inference/pipelines/text/config.d.ts +22 -1
  301. package/src/inference/pipelines/text/config.js +199 -22
  302. package/src/inference/pipelines/text/dtype-contract.d.ts +1 -1
  303. package/src/inference/pipelines/text/dtype-contract.js +4 -1
  304. package/src/inference/pipelines/text/embed.d.ts +4 -3
  305. package/src/inference/pipelines/text/embed.js +16 -6
  306. package/src/inference/pipelines/text/execution-plan.d.ts +8 -0
  307. package/src/inference/pipelines/text/execution-plan.js +6 -0
  308. package/src/inference/pipelines/text/execution-runtime-builders.d.ts +4 -0
  309. package/src/inference/pipelines/text/execution-runtime-builders.js +44 -28
  310. package/src/inference/pipelines/text/execution-v1.d.ts +2 -0
  311. package/src/inference/pipelines/text/execution-v1.js +373 -139
  312. package/src/inference/pipelines/text/ffn/dense.js +44 -13
  313. package/src/inference/pipelines/text/ffn/moe.d.ts +4 -1
  314. package/src/inference/pipelines/text/ffn/moe.js +9 -2
  315. package/src/inference/pipelines/text/ffn/sandwich.d.ts +3 -1
  316. package/src/inference/pipelines/text/ffn/sandwich.js +237 -15
  317. package/src/inference/pipelines/text/ffn/standard.d.ts +3 -1
  318. package/src/inference/pipelines/text/ffn/standard.js +44 -4
  319. package/src/inference/pipelines/text/generator-decode-policy.d.ts +81 -0
  320. package/src/inference/pipelines/text/generator-decode-policy.js +2 -12
  321. package/src/inference/pipelines/text/generator-helpers.js +5 -2
  322. package/src/inference/pipelines/text/generator-prefill-helpers.d.ts +84 -0
  323. package/src/inference/pipelines/text/generator-prefill-helpers.js +67 -1
  324. package/src/inference/pipelines/text/generator-runtime.d.ts +1 -0
  325. package/src/inference/pipelines/text/generator-steps.d.ts +38 -3
  326. package/src/inference/pipelines/text/generator-steps.js +268 -77
  327. package/src/inference/pipelines/text/generator.d.ts +39 -0
  328. package/src/inference/pipelines/text/generator.js +1105 -18
  329. package/src/inference/pipelines/text/init.d.ts +8 -1
  330. package/src/inference/pipelines/text/init.js +30 -1
  331. package/src/inference/pipelines/text/layer-plan-gpu.js +62 -14
  332. package/src/inference/pipelines/text/layer.js +272 -25
  333. package/src/inference/pipelines/text/linear-attention.js +6 -4
  334. package/src/inference/pipelines/text/logits/gpu.d.ts +49 -2
  335. package/src/inference/pipelines/text/logits/gpu.js +417 -66
  336. package/src/inference/pipelines/text/logits/index.d.ts +2 -2
  337. package/src/inference/pipelines/text/logits/index.js +96 -42
  338. package/src/inference/pipelines/text/logits/types.d.ts +2 -2
  339. package/src/inference/pipelines/text/lora-apply.js +17 -0
  340. package/src/inference/pipelines/text/model-load.js +5 -1
  341. package/src/inference/pipelines/text/moe-gpu.d.ts +13 -0
  342. package/src/inference/pipelines/text/moe-gpu.js +445 -16
  343. package/src/inference/pipelines/text/moe-impl.d.ts +13 -3
  344. package/src/inference/pipelines/text/moe-shape-validator.js +13 -1
  345. package/src/inference/pipelines/text/ops.d.ts +29 -2
  346. package/src/inference/pipelines/text/ops.js +19 -0
  347. package/src/inference/pipelines/text/per-layer-inputs.js +5 -0
  348. package/src/inference/pipelines/text/probes.js +4 -5
  349. package/src/inference/pipelines/text/sampling-config.d.ts +4 -10
  350. package/src/inference/pipelines/text/sampling-config.js +103 -64
  351. package/src/inference/pipelines/text/sampling.js +13 -9
  352. package/src/inference/pipelines/text/state.js +17 -0
  353. package/src/inference/pipelines/text/tsir-fixture-writer.d.ts +57 -0
  354. package/src/inference/pipelines/text/tsir-fixture-writer.js +8 -9
  355. package/src/inference/pipelines/text/types.d.ts +84 -3
  356. package/src/inference/pipelines/text/weights.d.ts +4 -2
  357. package/src/inference/pipelines/text/weights.js +4 -5
  358. package/src/inference/pipelines/text.d.ts +24 -0
  359. package/src/inference/pipelines/text.js +162 -38
  360. package/src/inference/pipelines/vision/gemma4.js +20 -9
  361. package/src/inference/tokenizer.d.ts +30 -0
  362. package/src/inference/tokenizer.js +340 -164
  363. package/src/inference/tokenizers/bundled.js +71 -49
  364. package/src/loader/doppler-loader.d.ts +11 -0
  365. package/src/loader/doppler-loader.js +290 -15
  366. package/src/loader/embedding-limit-preflight.d.ts +31 -0
  367. package/src/loader/embedding-limit-preflight.js +216 -0
  368. package/src/loader/embedding-loader.d.ts +8 -2
  369. package/src/loader/embedding-loader.js +263 -8
  370. package/src/loader/experts/expert-loader.d.ts +4 -1
  371. package/src/loader/experts/expert-loader.js +174 -5
  372. package/src/loader/final-weights-loader.d.ts +5 -0
  373. package/src/loader/final-weights-loader.js +167 -4
  374. package/src/loader/layer-loader.d.ts +2 -0
  375. package/src/loader/layer-loader.js +74 -6
  376. package/src/loader/loader-types.d.ts +59 -2
  377. package/src/loader/manifest-config.d.ts +8 -0
  378. package/src/loader/shard-resolver.js +37 -4
  379. package/src/loader/tensors/functional-descriptor-loader.d.ts +9 -0
  380. package/src/loader/tensors/functional-descriptor-loader.js +377 -0
  381. package/src/loader/tensors/tensor-loader.d.ts +36 -1
  382. package/src/loader/tensors/tensor-loader.js +437 -21
  383. package/src/loader/tensors/tensor-reader.d.ts +4 -1
  384. package/src/loader/tensors/tensor-reader.js +11 -4
  385. package/src/loader/weights.d.ts +3 -1
  386. package/src/memory/buffer-pool.js +12 -9
  387. package/src/models/diffusiongemma.d.ts +7 -0
  388. package/src/models/diffusiongemma.js +25 -0
  389. package/src/models/gemma4.js +24 -0
  390. package/src/models/qwen3.js +2 -2
  391. package/src/rules/execution-rules-contract-check.js +93 -10
  392. package/src/rules/inference/capability-transforms.rules.json +26 -2
  393. package/src/rules/inference/config.rules.json +1 -0
  394. package/src/rules/inference/dtype.rules.json +7 -0
  395. package/src/rules/inference/execution.rules.json +108 -17
  396. package/src/rules/inference/ffn.rules.json +18 -0
  397. package/src/rules/kernels/gather.rules.json +81 -0
  398. package/src/rules/kernels/matmul.rules.json +19 -0
  399. package/src/rules/kernels/moe.rules.json +3 -0
  400. package/src/rules/kernels/rep-penalty.rules.json +6 -0
  401. package/src/rules/kernels/rmsnorm-qk.rules.json +6 -0
  402. package/src/rules/kernels/rmsnorm.rules.json +42 -7
  403. package/src/rules/kernels/rope-qk.rules.json +6 -0
  404. package/src/rules/loader/tensor-loader.rules.json +8 -0
  405. package/src/rules/rule-registry.js +6 -0
  406. package/src/storage/artifact-storage-context.d.ts +1 -0
  407. package/src/storage/artifact-storage-context.js +10 -0
  408. package/src/tooling/browser-command-runner.d.ts +0 -3
  409. package/src/tooling/browser-command-runner.js +2 -14
  410. package/src/tooling/command-api-family-normalizers.js +4 -2
  411. package/src/tooling/command-api-helpers.js +2 -1
  412. package/src/tooling/command-api.d.ts +2 -0
  413. package/src/tooling/command-api.js +1 -0
  414. package/src/tooling/command-runner-shared.d.ts +4 -0
  415. package/src/tooling/command-runner-shared.js +2 -1
  416. package/src/tooling/diagnose-runner.js +2 -0
  417. package/src/tooling/litert-package-runtime.js +553 -16
  418. package/src/tooling/node-browser-command-runner.js +10 -4
  419. package/src/tooling/node-command-runner.js +7 -2
  420. package/src/tooling/node-convert-worker-pool.js +3 -0
  421. package/src/tooling/node-convert-worker.js +1 -0
  422. package/src/tooling/node-converter.js +12 -0
  423. package/src/tooling/node-webgpu.d.ts +1 -0
  424. package/src/tooling/node-webgpu.js +23 -11
  425. package/src/tooling/program-bundle.d.ts +52 -1
  426. package/src/tooling/rdrr-integrity-refresh.d.ts +18 -0
  427. package/src/tooling/source-artifact-adapter.js +7 -2
  428. package/src/tooling/source-package-profiles.d.ts +18 -1
  429. package/src/tooling/source-package-profiles.js +109 -1
  430. package/src/tooling/source-runtime-bundle.d.ts +1 -0
  431. package/src/tooling/source-runtime-bundle.js +27 -0
  432. package/src/tooling/source-runtime-converter-config.js +12 -3
  433. package/src/tooling-experimental-exports.shared.d.ts +10 -0
  434. package/src/tooling-experimental-exports.shared.js +10 -0
  435. package/src/version.js +1 -1
@@ -1,4 +1,9 @@
1
1
 
2
+ import { readFile } from 'node:fs/promises';
3
+ import { resolve } from 'node:path';
4
+
5
+ import { parseJsonl } from './jsonl.js';
6
+
2
7
  function concatTokens(tokens, maxLength) {
3
8
  if (!maxLength || tokens.length <= maxLength) {
4
9
  return tokens;
@@ -16,6 +21,72 @@ export function buildCausalPair(tokens) {
16
21
  };
17
22
  }
18
23
 
24
+ function readStringField(record, names, index, label) {
25
+ for (const name of names) {
26
+ if (Object.prototype.hasOwnProperty.call(record, name)) {
27
+ const value = record[name];
28
+ if (typeof value !== 'string') {
29
+ throw new Error(`text pair row ${index + 1} field "${name}" for ${label} must be a string.`);
30
+ }
31
+ return { field: name, value };
32
+ }
33
+ }
34
+ throw new Error(`text pair row ${index + 1} requires ${label}.`);
35
+ }
36
+
37
+ export function normalizeTextPair(record, index = 0) {
38
+ if (!record || typeof record !== 'object' || Array.isArray(record)) {
39
+ throw new Error(`text pair row ${index + 1} must be an object.`);
40
+ }
41
+ const prompt = readStringField(record, ['prompt', 'source', 'input'], index, 'prompt/source/input');
42
+ const completion = readStringField(record, ['completion', 'target', 'output'], index, 'completion/target/output');
43
+ return {
44
+ id: String(record.id ?? record.rowId ?? `pair-${index + 1}`),
45
+ prompt: prompt.value,
46
+ completion: completion.value,
47
+ promptField: prompt.field,
48
+ completionField: completion.field,
49
+ };
50
+ }
51
+
52
+ export function mapTextPairs(records) {
53
+ const rows = Array.isArray(records) ? records : [];
54
+ return rows.map((record, index) => normalizeTextPair(record, index));
55
+ }
56
+
57
+ export function parseTextPairsDataset(text, options = {}) {
58
+ const sourceLabel = String(options.sourceLabel || 'dataset.jsonl');
59
+ const parsed = sourceLabel.endsWith('.json')
60
+ ? JSON.parse(String(text))
61
+ : parseJsonl(String(text));
62
+ if (!Array.isArray(parsed)) {
63
+ throw new Error(`text pairs dataset "${sourceLabel}" must be a JSON array or JSONL records.`);
64
+ }
65
+ const rows = mapTextPairs(parsed);
66
+ return {
67
+ sourceLabel,
68
+ rowCount: rows.length,
69
+ rows,
70
+ };
71
+ }
72
+
73
+ export async function loadTextPairsDataset(datasetPath, options = {}) {
74
+ const source = String(datasetPath || '');
75
+ if (!source) {
76
+ throw new Error('loadTextPairsDataset requires a dataset path.');
77
+ }
78
+ const isUrl = /^https?:\/\//i.test(source);
79
+ const text = isUrl
80
+ ? await (options.fetch ? options.fetch(source) : fetch(source).then((res) => res.text()))
81
+ : await (options.readFile ? options.readFile(resolve(source)) : readFile(resolve(source), 'utf8'));
82
+ const parsed = parseTextPairsDataset(text, { sourceLabel: source });
83
+ return {
84
+ absolutePath: isUrl ? source : resolve(source),
85
+ raw: text,
86
+ ...parsed,
87
+ };
88
+ }
89
+
19
90
  export async function tokenizeTextPairs(tokenizer, pairs, options = {}) {
20
91
  if (!tokenizer || typeof tokenizer.encode !== 'function') {
21
92
  throw new Error('tokenizeTextPairs requires a tokenizer with encode()');
@@ -27,15 +98,21 @@ export async function tokenizeTextPairs(tokenizer, pairs, options = {}) {
27
98
  } = options;
28
99
 
29
100
  const samples = [];
30
- for (const pair of pairs) {
31
- const prompt = pair.prompt ?? '';
32
- const completion = pair.completion ?? '';
33
- const fullText = `${prompt}${joinWith}${completion}`;
101
+ const normalizedPairs = mapTextPairs(pairs);
102
+ for (const pair of normalizedPairs) {
103
+ const fullText = `${pair.prompt}${joinWith}${pair.completion}`;
34
104
  const tokens = tokenizer.encode(fullText);
35
105
  const clipped = concatTokens(tokens, maxLength);
36
106
  const { inputIds, targetIds } = buildCausalPair(clipped);
37
107
  if (inputIds.length > 0) {
38
- samples.push({ inputIds, targetIds, text: fullText });
108
+ samples.push({
109
+ id: pair.id,
110
+ inputIds,
111
+ targetIds,
112
+ text: fullText,
113
+ prompt: pair.prompt,
114
+ completion: pair.completion,
115
+ });
39
116
  }
40
117
  }
41
118
  return samples;
@@ -7,8 +7,13 @@ export interface DistillStudentFixture {
7
7
  tape: unknown,
8
8
  options?: Record<string, unknown>
9
9
  ) => Promise<{ logits: unknown }>;
10
+ forwardCausalLm?: (
11
+ batch: unknown,
12
+ tape: unknown
13
+ ) => Promise<{ logits: unknown }>;
10
14
  cleanupDistillStep?: () => void;
11
15
  loraParams?: () => unknown[];
16
+ loraTensorEntries?: () => Array<{ name: string; tensor: unknown }>;
12
17
  paramGroups?: () => Record<string, unknown[]>;
13
18
  };
14
19
  outputDim?: number;
@@ -14,11 +14,21 @@ import { createTensor } from '../../../gpu/tensor.js';
14
14
  import { acquireBuffer, uploadData, releaseBuffer } from '../../../memory/buffer-pool.js';
15
15
  import { getBufferDtype, getWeightDtype, isCpuWeightBuffer, isWeightBuffer } from '../../../gpu/weight-buffer.js';
16
16
  import { OpType } from '../autograd.js';
17
+ import { LoraAdapter } from '../lora.js';
17
18
  import { normalizeOptionalString } from './suite-data.js';
19
+ import { LORA_MODULE_ALIASES } from '../../../inference/pipelines/text/lora.js';
18
20
 
19
21
  const DISTILL_ADAPTER_TOP_K = 64;
20
22
  const DISTILL_STUDENT_GRAPH_PROJECTION = 'projection_head';
21
23
  const DISTILL_STUDENT_GRAPH_FULL = 'transformer_full';
24
+ const TRANSFORMER_LORA_TARGET_MODULES = Object.freeze([
25
+ 'q_proj',
26
+ 'k_proj',
27
+ 'v_proj',
28
+ 'o_proj',
29
+ 'gate_up_proj',
30
+ 'down_proj',
31
+ ]);
22
32
 
23
33
  function makeTensorFromFloat32(values, shape, label) {
24
34
  const data = values instanceof Float32Array ? values : new Float32Array(values);
@@ -87,6 +97,85 @@ function normalizeDistillStudentGraphMode(value) {
87
97
  return DISTILL_STUDENT_GRAPH_FULL;
88
98
  }
89
99
 
100
+ function normalizeTransformerLoraConfig(value) {
101
+ if (!value || typeof value !== 'object') {
102
+ return null;
103
+ }
104
+ const rank = Math.floor(Number(value.rank));
105
+ const alpha = Number(value.alpha);
106
+ if (!Number.isInteger(rank) || rank < 1) {
107
+ throw new Error('Transformer LoRA config requires rank >= 1.');
108
+ }
109
+ if (!Number.isFinite(alpha) || alpha <= 0) {
110
+ throw new Error('Transformer LoRA config requires alpha > 0.');
111
+ }
112
+ const rawModules = Array.isArray(value.targetModules) ? value.targetModules : [];
113
+ const targetModules = [];
114
+ for (const rawModule of rawModules) {
115
+ const normalized = String(rawModule || '').trim();
116
+ if (!normalized) continue;
117
+ const moduleName = LORA_MODULE_ALIASES[normalized] || normalized;
118
+ if (!TRANSFORMER_LORA_TARGET_MODULES.includes(moduleName)) {
119
+ throw new Error(`Transformer LoRA target module "${normalized}" is not supported.`);
120
+ }
121
+ if (!targetModules.includes(moduleName)) {
122
+ targetModules.push(moduleName);
123
+ }
124
+ }
125
+ if (targetModules.length === 0) {
126
+ throw new Error('Transformer LoRA config requires targetModules.');
127
+ }
128
+ return {
129
+ rank,
130
+ alpha,
131
+ targetModules,
132
+ };
133
+ }
134
+
135
+ function resolveTransformerLoraShape(moduleName, dims) {
136
+ if (moduleName === 'q_proj') {
137
+ return { inDim: dims.hiddenSize, outDim: dims.numHeads * dims.headDim };
138
+ }
139
+ if (moduleName === 'k_proj' || moduleName === 'v_proj') {
140
+ return { inDim: dims.hiddenSize, outDim: dims.numKVHeads * dims.headDim };
141
+ }
142
+ if (moduleName === 'o_proj') {
143
+ return { inDim: dims.hiddenSize, outDim: dims.hiddenSize };
144
+ }
145
+ if (moduleName === 'gate_up_proj') {
146
+ return { inDim: dims.hiddenSize, outDim: dims.intermediateSize * 2 };
147
+ }
148
+ if (moduleName === 'down_proj') {
149
+ return { inDim: dims.intermediateSize, outDim: dims.hiddenSize };
150
+ }
151
+ throw new Error(`Transformer LoRA target module "${moduleName}" is not supported.`);
152
+ }
153
+
154
+ function createTransformerLoraAdapters(config, dims) {
155
+ if (!config) return {};
156
+ const adapters = {};
157
+ for (const moduleName of config.targetModules) {
158
+ const shape = resolveTransformerLoraShape(moduleName, dims);
159
+ adapters[moduleName] = new LoraAdapter({
160
+ inDim: shape.inDim,
161
+ outDim: shape.outDim,
162
+ rank: config.rank,
163
+ alpha: config.alpha,
164
+ });
165
+ }
166
+ return adapters;
167
+ }
168
+
169
+ function disposeTransformerLoraAdapters(layers) {
170
+ for (const layer of layers) {
171
+ for (const adapter of Object.values(layer.lora || {})) {
172
+ if (adapter && typeof adapter.dispose === 'function') {
173
+ adapter.dispose();
174
+ }
175
+ }
176
+ }
177
+ }
178
+
90
179
  function resolveTensorDtype(value, fallback = 'f32') {
91
180
  const dtype = isWeightBuffer(value)
92
181
  ? value.dtype
@@ -188,6 +277,41 @@ function hasTensorPayload(value) {
188
277
  return false;
189
278
  }
190
279
 
280
+ function getTensorRows(value) {
281
+ const shape = Array.isArray(value?.shape) ? value.shape : null;
282
+ const rows = Number(shape?.[0]);
283
+ return Number.isFinite(rows) && rows > 0 ? Math.floor(rows) : null;
284
+ }
285
+
286
+ function getTensorCols(value) {
287
+ const shape = Array.isArray(value?.shape) ? value.shape : null;
288
+ const cols = Number(shape?.[1]);
289
+ return Number.isFinite(cols) && cols > 0 ? Math.floor(cols) : null;
290
+ }
291
+
292
+ function resolveLayerFfnIntermediateSize(layerIdx, weights, fallback) {
293
+ const gateUpRows = getTensorRows(weights.gateUp || weights.ffnGateUp);
294
+ if (gateUpRows !== null && gateUpRows % 2 === 0) {
295
+ return gateUpRows / 2;
296
+ }
297
+ const gateRows = getTensorRows(weights.gate || weights.ffnGate);
298
+ if (gateRows !== null) {
299
+ return gateRows;
300
+ }
301
+ const upRows = getTensorRows(weights.up || weights.ffnUp);
302
+ if (upRows !== null) {
303
+ return upRows;
304
+ }
305
+ const downCols = getTensorCols(weights.down || weights.ffnDown);
306
+ if (downCols !== null) {
307
+ return downCols;
308
+ }
309
+ if (Number.isInteger(fallback) && fallback > 0) {
310
+ return fallback;
311
+ }
312
+ throw new Error(`Distill full-graph student cannot resolve FFN size for layer ${layerIdx}.`);
313
+ }
314
+
191
315
  async function fuseGateUpTensors(gateTensor, upTensor, intermediateSize, hiddenSize, label, ownedTrainables = null) {
192
316
  const device = getDevice();
193
317
  if (!device) {
@@ -398,6 +522,10 @@ async function createDistillStudentTransformerModelFixture(overrides = {}, optio
398
522
  const swigluLimit = Number.isFinite(modelConfig.swigluLimit) ? modelConfig.swigluLimit : 0;
399
523
  const useEmbeddingTranspose = modelConfig.embeddingTranspose === true;
400
524
  const tieWordEmbeddings = modelConfig.useTiedEmbeddings === true;
525
+ const loraConfig = normalizeTransformerLoraConfig(options.loraAdapter || null);
526
+ const freezeBaseGrad = Boolean(loraConfig);
527
+ const stopBaseWeight = freezeBaseGrad ? { stopGradInputs: [1] } : {};
528
+ const stopRopeWeights = freezeBaseGrad ? { stopGradInputs: [1, 2] } : {};
401
529
 
402
530
  const config = createTrainingConfig({
403
531
  ...overrides,
@@ -447,18 +575,27 @@ async function createDistillStudentTransformerModelFixture(overrides = {}, optio
447
575
  );
448
576
 
449
577
  const layerParams = [];
578
+ const loraParams = [];
450
579
  const layers = [];
580
+ const loraDims = {
581
+ hiddenSize,
582
+ intermediateSize,
583
+ numHeads,
584
+ numKVHeads,
585
+ headDim,
586
+ };
451
587
  for (let layerIdx = 0; layerIdx < numLayers; layerIdx += 1) {
452
588
  const layerWeights = studentPipeline.weights.get(`layer_${layerIdx}`);
453
589
  if (!layerWeights) {
454
590
  throw new Error(`Distill full-graph student missing layer_${layerIdx} weights.`);
455
591
  }
592
+ const layerIntermediateSize = resolveLayerFfnIntermediateSize(layerIdx, layerWeights, intermediateSize);
456
593
  const gateUpWeight = layerWeights.gateUp || layerWeights.ffnGateUp || null;
457
594
  let layerGateUp = null;
458
595
  if (hasTensorPayload(gateUpWeight)) {
459
596
  layerGateUp = await ensureTrainableTensor(
460
597
  gateUpWeight,
461
- [intermediateSize * 2, hiddenSize],
598
+ [layerIntermediateSize * 2, hiddenSize],
462
599
  `layer_${layerIdx}.ffn_gate_up`,
463
600
  ownedTrainables
464
601
  );
@@ -472,20 +609,20 @@ async function createDistillStudentTransformerModelFixture(overrides = {}, optio
472
609
  }
473
610
  const gateTensor = await ensureTrainableTensor(
474
611
  gateWeight,
475
- [intermediateSize, hiddenSize],
612
+ [layerIntermediateSize, hiddenSize],
476
613
  `layer_${layerIdx}.ffn_gate`,
477
614
  ownedTrainables
478
615
  );
479
616
  const upTensor = await ensureTrainableTensor(
480
617
  upWeight,
481
- [intermediateSize, hiddenSize],
618
+ [layerIntermediateSize, hiddenSize],
482
619
  `layer_${layerIdx}.ffn_up`,
483
620
  ownedTrainables
484
621
  );
485
622
  layerGateUp = await fuseGateUpTensors(
486
623
  gateTensor,
487
624
  upTensor,
488
- intermediateSize,
625
+ layerIntermediateSize,
489
626
  hiddenSize,
490
627
  `layer_${layerIdx}.ffn_gate_up`,
491
628
  ownedTrainables
@@ -533,13 +670,21 @@ async function createDistillStudentTransformerModelFixture(overrides = {}, optio
533
670
  gateUp: layerGateUp,
534
671
  down: await ensureTrainableTensor(
535
672
  layerWeights.down || layerWeights.ffnDown,
536
- [hiddenSize, intermediateSize],
673
+ [hiddenSize, layerIntermediateSize],
537
674
  `layer_${layerIdx}.ffn_down`,
538
675
  ownedTrainables
539
676
  ),
677
+ intermediateSize: layerIntermediateSize,
678
+ lora: createTransformerLoraAdapters(loraConfig, {
679
+ ...loraDims,
680
+ intermediateSize: layerIntermediateSize,
681
+ }),
540
682
  };
541
683
  layers.push(layer);
542
684
  layerParams.push(layer.inputNorm, layer.qProj, layer.kProj, layer.vProj, layer.oProj, layer.gateUp, layer.down);
685
+ for (const adapter of Object.values(layer.lora)) {
686
+ loraParams.push(adapter.A, adapter.B);
687
+ }
543
688
  if (layer.postAttentionNorm) {
544
689
  layerParams.push(layer.postAttentionNorm);
545
690
  }
@@ -568,8 +713,7 @@ async function createDistillStudentTransformerModelFixture(overrides = {}, optio
568
713
  return { tokenTensor, seqLen: tokenIds.length };
569
714
  }
570
715
 
571
- async function runTransformerPrompt(prompt, tape) {
572
- const { tokenTensor, seqLen } = await buildPromptTokens(prompt);
716
+ async function runTransformerTokenTensor(tokenTensor, seqLen, tape, forwardOptions = {}) {
573
717
  let hidden = await tape.record(
574
718
  OpType.EMBED,
575
719
  (indices, embeddings) => runGather(
@@ -591,11 +735,13 @@ async function createDistillStudentTransformerModelFixture(overrides = {}, optio
591
735
  vocabSize,
592
736
  transpose: useEmbeddingTranspose,
593
737
  indexOffset: 0,
738
+ ...stopBaseWeight,
594
739
  }
595
740
  );
596
741
 
597
742
  for (let layerIdx = 0; layerIdx < layers.length; layerIdx += 1) {
598
743
  const layer = layers[layerIdx];
744
+ const layerIntermediateSize = layer.intermediateSize || intermediateSize;
599
745
  const normed = await tape.record(
600
746
  OpType.RMSNORM,
601
747
  (x, gamma) => runRMSNorm(x, gamma, rmsNormEps, {
@@ -604,36 +750,63 @@ async function createDistillStudentTransformerModelFixture(overrides = {}, optio
604
750
  rmsNormWeightOffset: modelConfig.rmsNormWeightOffset === true,
605
751
  }),
606
752
  [hidden, layer.inputNorm],
607
- { numTokens: seqLen, hiddenSize, eps: rmsNormEps }
753
+ { numTokens: seqLen, hiddenSize, eps: rmsNormEps, ...stopBaseWeight }
608
754
  );
609
755
 
610
- const q2d = await tape.record(
756
+ let q2d = await tape.record(
611
757
  OpType.MATMUL,
612
758
  (x, w) => runMatmul(x, w, seqLen, numHeads * headDim, hiddenSize, {
613
759
  transposeB: 'auto',
614
760
  outputDtype: 'f32',
615
761
  }),
616
762
  [normed, layer.qProj],
617
- { M: seqLen, N: numHeads * headDim, K: hiddenSize, transposeB: 'auto' }
763
+ { M: seqLen, N: numHeads * headDim, K: hiddenSize, transposeB: 'auto', ...stopBaseWeight }
618
764
  );
619
- const k2d = await tape.record(
765
+ let k2d = await tape.record(
620
766
  OpType.MATMUL,
621
767
  (x, w) => runMatmul(x, w, seqLen, numKVHeads * headDim, hiddenSize, {
622
768
  transposeB: 'auto',
623
769
  outputDtype: 'f32',
624
770
  }),
625
771
  [normed, layer.kProj],
626
- { M: seqLen, N: numKVHeads * headDim, K: hiddenSize, transposeB: 'auto' }
772
+ { M: seqLen, N: numKVHeads * headDim, K: hiddenSize, transposeB: 'auto', ...stopBaseWeight }
627
773
  );
628
- const v2d = await tape.record(
774
+ let v2d = await tape.record(
629
775
  OpType.MATMUL,
630
776
  (x, w) => runMatmul(x, w, seqLen, numKVHeads * headDim, hiddenSize, {
631
777
  transposeB: 'auto',
632
778
  outputDtype: 'f32',
633
779
  }),
634
780
  [normed, layer.vProj],
635
- { M: seqLen, N: numKVHeads * headDim, K: hiddenSize, transposeB: 'auto' }
781
+ { M: seqLen, N: numKVHeads * headDim, K: hiddenSize, transposeB: 'auto', ...stopBaseWeight }
636
782
  );
783
+ if (layer.lora.q_proj) {
784
+ const delta = await layer.lora.q_proj.forward(normed, tape);
785
+ q2d = await tape.record(
786
+ OpType.RESIDUAL_ADD,
787
+ (a, b) => runResidualAdd(a, b, seqLen * numHeads * headDim),
788
+ [q2d, delta],
789
+ { size: seqLen * numHeads * headDim }
790
+ );
791
+ }
792
+ if (layer.lora.k_proj) {
793
+ const delta = await layer.lora.k_proj.forward(normed, tape);
794
+ k2d = await tape.record(
795
+ OpType.RESIDUAL_ADD,
796
+ (a, b) => runResidualAdd(a, b, seqLen * numKVHeads * headDim),
797
+ [k2d, delta],
798
+ { size: seqLen * numKVHeads * headDim }
799
+ );
800
+ }
801
+ if (layer.lora.v_proj) {
802
+ const delta = await layer.lora.v_proj.forward(normed, tape);
803
+ v2d = await tape.record(
804
+ OpType.RESIDUAL_ADD,
805
+ (a, b) => runResidualAdd(a, b, seqLen * numKVHeads * headDim),
806
+ [v2d, delta],
807
+ { size: seqLen * numKVHeads * headDim }
808
+ );
809
+ }
637
810
 
638
811
  const q3d = createTensor(q2d.buffer, q2d.dtype, [seqLen, numHeads, headDim], `layer_${layerIdx}_q`);
639
812
  const k3d = createTensor(k2d.buffer, k2d.dtype, [seqLen, numKVHeads, headDim], `layer_${layerIdx}_k`);
@@ -643,13 +816,13 @@ async function createDistillStudentTransformerModelFixture(overrides = {}, optio
643
816
  OpType.ROPE,
644
817
  (q, cos, sin) => runRoPE(q, cos, sin, seqLen, { numHeads, headDim, startPos: 0 }),
645
818
  [q3d, ropeCos, ropeSin],
646
- { seqLen, numHeads, headDim, startPos: 0 }
819
+ { seqLen, numHeads, headDim, startPos: 0, ...stopRopeWeights }
647
820
  );
648
821
  const kRope = await tape.record(
649
822
  OpType.ROPE,
650
823
  (k, cos, sin) => runRoPE(k, cos, sin, seqLen, { numHeads: numKVHeads, headDim, startPos: 0 }),
651
824
  [k3d, ropeCos, ropeSin],
652
- { seqLen, numHeads: numKVHeads, headDim, startPos: 0 }
825
+ { seqLen, numHeads: numKVHeads, headDim, startPos: 0, ...stopRopeWeights }
653
826
  );
654
827
 
655
828
  const attention = await tape.record(
@@ -672,15 +845,24 @@ async function createDistillStudentTransformerModelFixture(overrides = {}, optio
672
845
  `layer_${layerIdx}_attn_2d`
673
846
  );
674
847
 
675
- const attentionOutput = await tape.record(
848
+ let attentionOutput = await tape.record(
676
849
  OpType.MATMUL,
677
850
  (x, w) => runMatmul(x, w, seqLen, hiddenSize, hiddenSize, {
678
851
  transposeB: 'auto',
679
852
  outputDtype: 'f32',
680
853
  }),
681
854
  [attention2d, layer.oProj],
682
- { M: seqLen, N: hiddenSize, K: hiddenSize, transposeB: 'auto' }
855
+ { M: seqLen, N: hiddenSize, K: hiddenSize, transposeB: 'auto', ...stopBaseWeight }
683
856
  );
857
+ if (layer.lora.o_proj) {
858
+ const delta = await layer.lora.o_proj.forward(attention2d, tape);
859
+ attentionOutput = await tape.record(
860
+ OpType.RESIDUAL_ADD,
861
+ (a, b) => runResidualAdd(a, b, seqLen * hiddenSize),
862
+ [attentionOutput, delta],
863
+ { size: seqLen * hiddenSize }
864
+ );
865
+ }
684
866
  const postAttention = await tape.record(
685
867
  OpType.RESIDUAL_ADD,
686
868
  (a, b) => runResidualAdd(a, b, seqLen * hiddenSize),
@@ -697,43 +879,61 @@ async function createDistillStudentTransformerModelFixture(overrides = {}, optio
697
879
  rmsNormWeightOffset: modelConfig.rmsNormWeightOffset === true,
698
880
  }),
699
881
  [postAttention, layer.postAttentionNorm],
700
- { numTokens: seqLen, hiddenSize, eps: rmsNormEps }
882
+ { numTokens: seqLen, hiddenSize, eps: rmsNormEps, ...stopBaseWeight }
701
883
  )
702
884
  : postAttention;
703
- const gateUp = await tape.record(
885
+ let gateUp = await tape.record(
704
886
  OpType.MATMUL,
705
- (x, w) => runMatmul(x, w, seqLen, intermediateSize * 2, hiddenSize, {
887
+ (x, w) => runMatmul(x, w, seqLen, layerIntermediateSize * 2, hiddenSize, {
706
888
  transposeB: 'auto',
707
889
  outputDtype: 'f32',
708
890
  }),
709
891
  [ffnInput, layer.gateUp],
710
- { M: seqLen, N: intermediateSize * 2, K: hiddenSize, transposeB: 'auto' }
892
+ { M: seqLen, N: layerIntermediateSize * 2, K: hiddenSize, transposeB: 'auto', ...stopBaseWeight }
711
893
  );
894
+ if (layer.lora.gate_up_proj) {
895
+ const delta = await layer.lora.gate_up_proj.forward(ffnInput, tape);
896
+ gateUp = await tape.record(
897
+ OpType.RESIDUAL_ADD,
898
+ (a, b) => runResidualAdd(a, b, seqLen * layerIntermediateSize * 2),
899
+ [gateUp, delta],
900
+ { size: seqLen * layerIntermediateSize * 2 }
901
+ );
902
+ }
712
903
  const activated = await tape.record(
713
904
  OpType.SILU_ROWSPLIT,
714
905
  (x) => runSiLURowSplit(x, {
715
906
  numTokens: seqLen,
716
- dim: intermediateSize,
907
+ dim: layerIntermediateSize,
717
908
  activation: hiddenActivation === 'gelu' ? 'gelu' : 'silu',
718
909
  swigluLimit: hiddenActivation === 'gelu' ? null : swigluLimit,
719
910
  }),
720
911
  [gateUp],
721
912
  {
722
913
  numTokens: seqLen,
723
- dim: intermediateSize,
914
+ dim: layerIntermediateSize,
724
915
  activation: hiddenActivation === 'gelu' ? 'gelu' : 'silu',
725
916
  swigluLimit: hiddenActivation === 'gelu' ? 0 : swigluLimit,
726
917
  }
727
918
  );
728
- const ffnOutput = await tape.record(
919
+ let ffnOutput = await tape.record(
729
920
  OpType.MATMUL,
730
- (x, w) => runMatmul(x, w, seqLen, hiddenSize, intermediateSize, {
921
+ (x, w) => runMatmul(x, w, seqLen, hiddenSize, layerIntermediateSize, {
731
922
  transposeB: 'auto',
732
923
  outputDtype: 'f32',
733
924
  }),
734
925
  [activated, layer.down],
735
- { M: seqLen, N: hiddenSize, K: intermediateSize, transposeB: 'auto' }
926
+ { M: seqLen, N: hiddenSize, K: layerIntermediateSize, transposeB: 'auto', ...stopBaseWeight }
736
927
  );
928
+ if (layer.lora.down_proj) {
929
+ const delta = await layer.lora.down_proj.forward(activated, tape);
930
+ ffnOutput = await tape.record(
931
+ OpType.RESIDUAL_ADD,
932
+ (a, b) => runResidualAdd(a, b, seqLen * hiddenSize),
933
+ [ffnOutput, delta],
934
+ { size: seqLen * hiddenSize }
935
+ );
936
+ }
737
937
  hidden = await tape.record(
738
938
  OpType.RESIDUAL_ADD,
739
939
  (a, b) => runResidualAdd(a, b, seqLen * hiddenSize),
@@ -750,8 +950,19 @@ async function createDistillStudentTransformerModelFixture(overrides = {}, optio
750
950
  rmsNormWeightOffset: modelConfig.rmsNormWeightOffset === true,
751
951
  }),
752
952
  [hidden, finalNormWeight],
753
- { numTokens: seqLen, hiddenSize, eps: rmsNormEps }
953
+ { numTokens: seqLen, hiddenSize, eps: rmsNormEps, ...stopBaseWeight }
754
954
  );
955
+ if (forwardOptions.logitsMode === 'all') {
956
+ return tape.record(
957
+ OpType.MATMUL,
958
+ (x, w) => runMatmul(x, w, seqLen, vocabSize, hiddenSize, {
959
+ transposeB: 'auto',
960
+ outputDtype: 'f32',
961
+ }),
962
+ [finalHidden, lmHeadWeight],
963
+ { M: seqLen, N: vocabSize, K: hiddenSize, transposeB: 'auto', ...stopBaseWeight }
964
+ );
965
+ }
755
966
  const lastHidden = await tape.record(
756
967
  OpType.ROW_SLICE,
757
968
  (x) => createRowSliceTensor(x, seqLen, hiddenSize, seqLen - 1, 'distill_last_hidden'),
@@ -765,12 +976,35 @@ async function createDistillStudentTransformerModelFixture(overrides = {}, optio
765
976
  outputDtype: 'f32',
766
977
  }),
767
978
  [lastHidden, lmHeadWeight],
768
- { M: 1, N: vocabSize, K: hiddenSize, transposeB: 'auto' }
979
+ { M: 1, N: vocabSize, K: hiddenSize, transposeB: 'auto', ...stopBaseWeight }
769
980
  );
770
981
  }
771
982
 
983
+ async function runTransformerPrompt(prompt, tape) {
984
+ const { tokenTensor, seqLen } = await buildPromptTokens(prompt);
985
+ return runTransformerTokenTensor(tokenTensor, seqLen, tape);
986
+ }
987
+
988
+ function collectLoraTensorEntries() {
989
+ const entries = [];
990
+ for (let layerIdx = 0; layerIdx < layers.length; layerIdx += 1) {
991
+ const layer = layers[layerIdx];
992
+ for (const moduleName of Object.keys(layer.lora).sort((left, right) => left.localeCompare(right))) {
993
+ const adapter = layer.lora[moduleName];
994
+ entries.push(
995
+ { name: `layers.${layerIdx}.${moduleName}.lora_a`, tensor: adapter.A },
996
+ { name: `layers.${layerIdx}.${moduleName}.lora_b`, tensor: adapter.B }
997
+ );
998
+ }
999
+ }
1000
+ return entries;
1001
+ }
1002
+
772
1003
  const model = {
773
1004
  async forward(inputTensor, tape) {
1005
+ if (Array.isArray(inputTensor?.shape) && inputTensor.shape.length === 1) {
1006
+ return runTransformerTokenTensor(inputTensor, inputTensor.shape[0], tape, { logitsMode: 'all' });
1007
+ }
774
1008
  return tape.record(
775
1009
  OpType.MATMUL,
776
1010
  (x, w) => runMatmul(x, w, 1, vocabSize, hiddenSize, {
@@ -778,9 +1012,20 @@ async function createDistillStudentTransformerModelFixture(overrides = {}, optio
778
1012
  outputDtype: 'f32',
779
1013
  }),
780
1014
  [inputTensor, lmHeadWeight],
781
- { M: 1, N: vocabSize, K: hiddenSize, transposeB: 'auto' }
1015
+ { M: 1, N: vocabSize, K: hiddenSize, transposeB: 'auto', ...stopBaseWeight }
782
1016
  );
783
1017
  },
1018
+ async forwardCausalLm(batch, tape) {
1019
+ const inputTensor = batch?.input || null;
1020
+ const seqLen = Array.isArray(inputTensor?.shape) && inputTensor.shape.length === 1
1021
+ ? inputTensor.shape[0]
1022
+ : 0;
1023
+ if (!Number.isInteger(seqLen) || seqLen < 1) {
1024
+ throw new Error('Transformer LoRA causal-LM batch requires input token tensor shape [seqLen].');
1025
+ }
1026
+ const logits = await runTransformerTokenTensor(inputTensor, seqLen, tape, { logitsMode: 'all' });
1027
+ return { logits };
1028
+ },
784
1029
  async forwardDistill(batch, tape, forwardOptions = {}) {
785
1030
  const requestedPhase = String(forwardOptions?.phase || 'anchor').trim();
786
1031
  const phase = requestedPhase === 'positive'
@@ -802,15 +1047,27 @@ async function createDistillStudentTransformerModelFixture(overrides = {}, optio
802
1047
  temporaryInputs.clear();
803
1048
  },
804
1049
  loraParams() {
805
- return decoderParams;
1050
+ return loraParams.length > 0 ? loraParams : decoderParams;
1051
+ },
1052
+ loraTensorEntries() {
1053
+ return collectLoraTensorEntries();
806
1054
  },
807
1055
  paramGroups() {
1056
+ if (loraParams.length > 0) {
1057
+ return {
1058
+ encoder: [],
1059
+ prior: [],
1060
+ decoder: [],
1061
+ base: baseParams,
1062
+ lora: loraParams,
1063
+ };
1064
+ }
808
1065
  return {
809
1066
  encoder: encoderParams,
810
1067
  prior: [],
811
1068
  decoder: decoderParams,
812
1069
  base: baseParams,
813
- lora: [],
1070
+ lora: loraParams,
814
1071
  };
815
1072
  },
816
1073
  };
@@ -822,6 +1079,7 @@ async function createDistillStudentTransformerModelFixture(overrides = {}, optio
822
1079
  embeddingDim: hiddenSize,
823
1080
  cleanup() {
824
1081
  model.cleanupDistillStep();
1082
+ disposeTransformerLoraAdapters(layers);
825
1083
  for (const tensor of ownedTrainables) {
826
1084
  releaseTensor(tensor);
827
1085
  }