@simulatte/doppler 0.1.6 → 0.1.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (355) hide show
  1. package/CHANGELOG.md +145 -0
  2. package/README.md +16 -23
  3. package/package.json +30 -32
  4. package/src/adapters/adapter-registry.js +12 -1
  5. package/src/adapters/lora-loader.js +23 -6
  6. package/src/bridge/extension-client.d.ts +5 -0
  7. package/src/bridge/extension-client.js +40 -0
  8. package/src/bridge/index.d.ts +2 -1
  9. package/src/bridge/index.js +6 -4
  10. package/src/browser/browser-converter.js +31 -1
  11. package/src/browser/file-picker.js +6 -0
  12. package/src/browser/safetensors-parser-browser.js +84 -1
  13. package/src/browser/shard-io-browser.js +2 -2
  14. package/src/browser/tensor-source-download.js +8 -2
  15. package/src/browser/tensor-source-http.d.ts +1 -0
  16. package/src/browser/tensor-source-http.js +5 -1
  17. package/src/client/doppler-api.browser.js +20 -4
  18. package/src/client/doppler-api.js +19 -3
  19. package/src/client/doppler-provider/generation.js +12 -0
  20. package/src/client/doppler-provider/model-manager.d.ts +10 -0
  21. package/src/client/doppler-provider/model-manager.js +91 -19
  22. package/src/client/doppler-provider/source-runtime.d.ts +2 -1
  23. package/src/client/doppler-provider/source-runtime.js +132 -13
  24. package/src/client/doppler-registry.json +5 -20
  25. package/src/config/backward-registry-loader.js +17 -2
  26. package/src/config/execution-v0-contract-check.js +113 -15
  27. package/src/config/kernel-path-contract-check.js +57 -29
  28. package/src/config/kernel-path-loader.d.ts +5 -0
  29. package/src/config/kernel-path-loader.js +18 -36
  30. package/src/config/kernels/kernel-ref-digests.js +1 -1
  31. package/src/config/kernels/registry.js +14 -1
  32. package/src/config/kernels/registry.json +81 -5
  33. package/src/config/loader.d.ts +1 -1
  34. package/src/config/loader.js +15 -2
  35. package/src/config/merge-contract-check.js +66 -4
  36. package/src/config/merge-helpers.js +128 -7
  37. package/src/config/merge.d.ts +1 -0
  38. package/src/config/merge.js +10 -0
  39. package/src/config/param-validator.js +47 -2
  40. package/src/config/presets/kernel-paths/{gemma2-q4k-dequant-f32a.json → gemma2-q4k-dequant-f32a-nosubgroups.json} +3 -3
  41. package/src/config/presets/kernel-paths/gemma3-f16-fused-f32a-online-streamingprefill.json +223 -0
  42. package/src/config/presets/kernel-paths/{gemma3-q4k-dequant-f32a.json → gemma3-q4k-dequant-f32a-nosubgroups.json} +3 -3
  43. package/src/config/presets/kernel-paths/gemma3-q4k-dequant-f32w-f32a-online.json +56 -0
  44. package/src/config/presets/kernel-paths/lfm2-q4k-dequant-f32a-nosubgroups.json +61 -0
  45. package/src/config/presets/kernel-paths/registry.json +43 -8
  46. package/src/config/presets/models/gemma2.json +3 -2
  47. package/src/config/presets/models/gemma3.json +2 -0
  48. package/src/config/presets/models/qwen3.json +4 -3
  49. package/src/config/presets/models/qwen3_5.json +16 -0
  50. package/src/config/presets/runtime/experiments/bench/gemma3-bench-q4k.json +1 -1
  51. package/src/config/presets/runtime/experiments/debug/gemma3-debug-q4k.json +1 -1
  52. package/src/config/presets/runtime/experiments/verify/gemma3-verify.json +1 -1
  53. package/src/config/presets/runtime/kernels/dequant-f16-q4k.json +6 -13
  54. package/src/config/presets/runtime/kernels/dequant-f32-q4k.json +6 -13
  55. package/src/config/presets/runtime/kernels/embeddinggemma-q4k-dequant-f32a.json +37 -0
  56. package/src/config/presets/runtime/kernels/fused-q4k.json +6 -13
  57. package/src/config/presets/runtime/kernels/gemma2-q4k-dequant-f16a.json +33 -0
  58. package/src/config/presets/runtime/kernels/gemma2-q4k-dequant-f32a-nosubgroups.json +33 -0
  59. package/src/config/presets/runtime/kernels/gemma2-q4k-fused-f32a.json +33 -0
  60. package/src/config/presets/runtime/kernels/safe-q4k.json +6 -13
  61. package/src/config/presets/runtime/model/qwen3-5-layer-probe.json +52 -0
  62. package/src/config/presets/runtime/model/qwen3-5-linear-attn-debug.json +90 -0
  63. package/src/config/presets/runtime/platform/metal-apple-q4k.json +1 -1
  64. package/src/config/runtime.js +6 -1
  65. package/src/config/schema/conversion.schema.d.ts +1 -0
  66. package/src/config/schema/debug.schema.d.ts +5 -0
  67. package/src/config/schema/doppler.schema.js +16 -21
  68. package/src/config/schema/inference-defaults.schema.js +3 -3
  69. package/src/config/schema/kernel-path.schema.d.ts +5 -1
  70. package/src/config/schema/kernel-thresholds.schema.js +12 -4
  71. package/src/config/schema/manifest.schema.d.ts +3 -2
  72. package/src/config/schema/manifest.schema.js +17 -4
  73. package/src/config/schema/storage.schema.js +1 -1
  74. package/src/config/training-defaults.js +30 -22
  75. package/src/converter/conversion-plan.js +104 -11
  76. package/src/converter/core.d.ts +7 -0
  77. package/src/converter/core.js +16 -9
  78. package/src/converter/execution-v0-manifest.js +4 -1
  79. package/src/converter/index.d.ts +1 -0
  80. package/src/converter/index.js +1 -0
  81. package/src/converter/manifest-inference.js +50 -29
  82. package/src/converter/parsers/diffusion.js +0 -3
  83. package/src/converter/parsers/transformer.js +4 -0
  84. package/src/converter/quantization-info.js +40 -16
  85. package/src/converter/quantizer.js +19 -12
  86. package/src/converter/rope-config.js +8 -6
  87. package/src/converter/shard-packer.d.ts +1 -1
  88. package/src/converter/shard-packer.js +4 -1
  89. package/src/converter/tokenizer-utils.d.ts +1 -0
  90. package/src/converter/tokenizer-utils.js +4 -1
  91. package/src/debug/config.js +123 -11
  92. package/src/debug/reference/hf_qwen35_linear_attn_debug.py +268 -0
  93. package/src/debug/signals.js +7 -1
  94. package/src/debug/tensor.d.ts +2 -0
  95. package/src/debug/tensor.js +13 -2
  96. package/src/distribution/p2p-control-plane.js +52 -12
  97. package/src/distribution/p2p-observability.js +43 -7
  98. package/src/distribution/p2p-webrtc-browser.js +20 -0
  99. package/src/distribution/shard-delivery.js +83 -27
  100. package/src/formats/gguf/types.js +33 -16
  101. package/src/formats/rdrr/groups.d.ts +12 -4
  102. package/src/formats/rdrr/groups.js +3 -6
  103. package/src/formats/rdrr/parsing.d.ts +4 -0
  104. package/src/formats/rdrr/parsing.js +53 -3
  105. package/src/formats/rdrr/types.d.ts +2 -1
  106. package/src/gpu/command-recorder.js +86 -61
  107. package/src/gpu/device.d.ts +1 -0
  108. package/src/gpu/device.js +73 -19
  109. package/src/gpu/kernel-tuner/benchmarks.js +326 -316
  110. package/src/gpu/kernel-tuner/cache.js +71 -4
  111. package/src/gpu/kernel-tuner/tuner.js +22 -4
  112. package/src/gpu/kernels/attention.js +15 -34
  113. package/src/gpu/kernels/backward/adam.js +62 -58
  114. package/src/gpu/kernels/backward/attention_backward.js +257 -169
  115. package/src/gpu/kernels/backward/conv2d_backward.js +14 -1
  116. package/src/gpu/kernels/cast.js +191 -149
  117. package/src/gpu/kernels/check-stop.js +33 -44
  118. package/src/gpu/kernels/conv2d.js +27 -17
  119. package/src/gpu/kernels/cross_entropy_loss.js +21 -15
  120. package/src/gpu/kernels/depthwise_conv2d.js +36 -26
  121. package/src/gpu/kernels/dequant.js +178 -126
  122. package/src/gpu/kernels/energy.d.ts +3 -21
  123. package/src/gpu/kernels/energy.js +111 -88
  124. package/src/gpu/kernels/feature-check.js +1 -1
  125. package/src/gpu/kernels/fused_ffn.js +84 -65
  126. package/src/gpu/kernels/fused_matmul_residual.js +56 -33
  127. package/src/gpu/kernels/fused_matmul_rmsnorm.js +62 -45
  128. package/src/gpu/kernels/gather.js +33 -15
  129. package/src/gpu/kernels/gelu.js +19 -11
  130. package/src/gpu/kernels/grouped_pointwise_conv2d.js +33 -23
  131. package/src/gpu/kernels/groupnorm.js +34 -23
  132. package/src/gpu/kernels/index.d.ts +8 -0
  133. package/src/gpu/kernels/index.js +6 -0
  134. package/src/gpu/kernels/kv-quantize.js +5 -2
  135. package/src/gpu/kernels/layernorm.js +35 -19
  136. package/src/gpu/kernels/logit-merge.js +5 -3
  137. package/src/gpu/kernels/matmul-selection.js +47 -4
  138. package/src/gpu/kernels/matmul.d.ts +2 -0
  139. package/src/gpu/kernels/matmul.js +59 -40
  140. package/src/gpu/kernels/modulate.js +23 -15
  141. package/src/gpu/kernels/moe.js +221 -175
  142. package/src/gpu/kernels/pixel_shuffle.js +22 -14
  143. package/src/gpu/kernels/relu.js +18 -10
  144. package/src/gpu/kernels/repeat_channels.js +25 -17
  145. package/src/gpu/kernels/residual.js +37 -27
  146. package/src/gpu/kernels/rmsnorm.js +66 -43
  147. package/src/gpu/kernels/rope.js +3 -0
  148. package/src/gpu/kernels/sample.js +27 -38
  149. package/src/gpu/kernels/sana_linear_attention.js +18 -10
  150. package/src/gpu/kernels/scale.js +18 -11
  151. package/src/gpu/kernels/shader-cache.js +4 -2
  152. package/src/gpu/kernels/silu.js +120 -72
  153. package/src/gpu/kernels/softmax.js +44 -25
  154. package/src/gpu/kernels/split_qg.d.ts +50 -0
  155. package/src/gpu/kernels/split_qg.js +46 -0
  156. package/src/gpu/kernels/split_qg.wgsl +58 -0
  157. package/src/gpu/kernels/split_qg_f16.wgsl +62 -0
  158. package/src/gpu/kernels/split_qkv.js +23 -13
  159. package/src/gpu/kernels/transpose.js +18 -10
  160. package/src/gpu/kernels/transpose.wgsl +5 -3
  161. package/src/gpu/kernels/upsample2d.js +21 -13
  162. package/src/gpu/kernels/utils.js +20 -13
  163. package/src/gpu/partitioned-buffer-pool.js +10 -2
  164. package/src/gpu/perf-guards.js +2 -9
  165. package/src/gpu/profiler.js +27 -22
  166. package/src/gpu/readback-utils.d.ts +16 -0
  167. package/src/gpu/readback-utils.js +41 -0
  168. package/src/gpu/submit-tracker.js +13 -0
  169. package/src/gpu/uniform-cache.d.ts +1 -0
  170. package/src/gpu/uniform-cache.js +30 -9
  171. package/src/gpu/weight-buffer.d.ts +1 -1
  172. package/src/gpu/weight-buffer.js +1 -1
  173. package/src/hotswap/intent-bundle.js +6 -0
  174. package/src/hotswap/manifest.d.ts +10 -1
  175. package/src/hotswap/manifest.js +12 -2
  176. package/src/hotswap/runtime.js +30 -8
  177. package/src/index-browser.d.ts +44 -0
  178. package/src/index-browser.js +14 -0
  179. package/src/inference/browser-harness-contract-helpers.d.ts +5 -0
  180. package/src/inference/browser-harness-contract-helpers.js +28 -0
  181. package/src/inference/browser-harness-diffusion-energy-suites.d.ts +2 -0
  182. package/src/inference/browser-harness-diffusion-energy-suites.js +269 -0
  183. package/src/inference/browser-harness-model-helpers.d.ts +16 -0
  184. package/src/inference/browser-harness-model-helpers.js +217 -0
  185. package/src/inference/browser-harness-report-helpers.d.ts +7 -0
  186. package/src/inference/browser-harness-report-helpers.js +42 -0
  187. package/src/inference/browser-harness-runtime-helpers.d.ts +61 -0
  188. package/src/inference/browser-harness-runtime-helpers.js +415 -0
  189. package/src/inference/browser-harness-suite-helpers.d.ts +28 -0
  190. package/src/inference/browser-harness-suite-helpers.js +268 -0
  191. package/src/inference/browser-harness-text-helpers.d.ts +27 -0
  192. package/src/inference/browser-harness-text-helpers.js +788 -0
  193. package/src/inference/browser-harness.d.ts +8 -0
  194. package/src/inference/browser-harness.js +149 -1996
  195. package/src/inference/kv-cache/base.js +140 -94
  196. package/src/inference/kv-cache/tiered.js +5 -3
  197. package/src/inference/moe-router.js +88 -56
  198. package/src/inference/multi-model-network.js +5 -3
  199. package/src/inference/network-evolution.d.ts +11 -2
  200. package/src/inference/network-evolution.js +20 -21
  201. package/src/inference/pipelines/context.d.ts +3 -0
  202. package/src/inference/pipelines/context.js +142 -2
  203. package/src/inference/pipelines/diffusion/helpers.js +10 -2
  204. package/src/inference/pipelines/diffusion/pipeline.js +2 -1
  205. package/src/inference/pipelines/diffusion/sd3-transformer.js +10 -10
  206. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +8 -2
  207. package/src/inference/pipelines/diffusion/vae.js +3 -7
  208. package/src/inference/pipelines/energy/pipeline.js +27 -21
  209. package/src/inference/pipelines/energy/quintel.d.ts +5 -0
  210. package/src/inference/pipelines/energy/quintel.js +11 -0
  211. package/src/inference/pipelines/energy-head/row-head-pipeline.js +17 -13
  212. package/src/inference/pipelines/structured/json-head-pipeline.js +26 -11
  213. package/src/inference/pipelines/text/attention/output-projection.d.ts +12 -0
  214. package/src/inference/pipelines/text/attention/output-projection.js +8 -0
  215. package/src/inference/pipelines/text/attention/projections.d.ts +10 -1
  216. package/src/inference/pipelines/text/attention/projections.js +192 -112
  217. package/src/inference/pipelines/text/attention/record.js +77 -14
  218. package/src/inference/pipelines/text/attention/run.js +112 -14
  219. package/src/inference/pipelines/text/config.js +17 -4
  220. package/src/inference/pipelines/text/embed.js +2 -8
  221. package/src/inference/pipelines/text/execution-plan.js +46 -23
  222. package/src/inference/pipelines/text/execution-v0-contract-helpers.d.ts +59 -0
  223. package/src/inference/pipelines/text/execution-v0-contract-helpers.js +937 -0
  224. package/src/inference/pipelines/text/execution-v0-runtime-builders.d.ts +15 -0
  225. package/src/inference/pipelines/text/execution-v0-runtime-builders.js +279 -0
  226. package/src/inference/pipelines/text/execution-v0.js +62 -1013
  227. package/src/inference/pipelines/text/generator-runtime.js +5 -0
  228. package/src/inference/pipelines/text/generator-steps.d.ts +52 -0
  229. package/src/inference/pipelines/text/generator-steps.js +340 -221
  230. package/src/inference/pipelines/text/generator.js +56 -40
  231. package/src/inference/pipelines/text/init.d.ts +13 -0
  232. package/src/inference/pipelines/text/init.js +94 -25
  233. package/src/inference/pipelines/text/kernel-path-auto-select.js +2 -0
  234. package/src/inference/pipelines/text/kernel-trace.d.ts +2 -0
  235. package/src/inference/pipelines/text/kernel-trace.js +6 -0
  236. package/src/inference/pipelines/text/layer.js +4 -9
  237. package/src/inference/pipelines/text/linear-attention.d.ts +15 -0
  238. package/src/inference/pipelines/text/linear-attention.js +113 -9
  239. package/src/inference/pipelines/text/logits/gpu.js +12 -7
  240. package/src/inference/pipelines/text/logits/index.d.ts +6 -1
  241. package/src/inference/pipelines/text/logits/index.js +13 -12
  242. package/src/inference/pipelines/text/logits/utils.d.ts +7 -0
  243. package/src/inference/pipelines/text/logits/utils.js +9 -0
  244. package/src/inference/pipelines/text/lora-apply.js +50 -32
  245. package/src/inference/pipelines/text/model-load.js +282 -104
  246. package/src/inference/pipelines/text/moe-cache.js +5 -4
  247. package/src/inference/pipelines/text/moe-cpu-gptoss.js +74 -69
  248. package/src/inference/pipelines/text/moe-cpu.js +42 -38
  249. package/src/inference/pipelines/text/moe-gpu.js +110 -86
  250. package/src/inference/pipelines/text/ops.js +90 -90
  251. package/src/inference/pipelines/text/probes.js +9 -9
  252. package/src/inference/pipelines/text/sampling.js +52 -6
  253. package/src/inference/pipelines/text/weights.js +17 -7
  254. package/src/inference/pipelines/text.js +13 -1
  255. package/src/inference/speculative.d.ts +2 -2
  256. package/src/inference/speculative.js +4 -18
  257. package/src/inference/test-harness.d.ts +1 -1
  258. package/src/inference/test-harness.js +17 -7
  259. package/src/inference/tokenizer.d.ts +0 -5
  260. package/src/inference/tokenizer.js +4 -23
  261. package/src/inference/tokenizers/bpe.js +9 -0
  262. package/src/inference/tokenizers/bundled.js +20 -0
  263. package/src/inference/tokenizers/sentencepiece.js +12 -0
  264. package/src/loader/doppler-loader.js +38 -22
  265. package/src/loader/dtype-utils.js +3 -44
  266. package/src/loader/embedding-loader.js +7 -3
  267. package/src/loader/experts/expert-cache.js +13 -6
  268. package/src/loader/experts/expert-loader.js +10 -6
  269. package/src/loader/final-weights-loader.js +10 -4
  270. package/src/loader/layer-loader.js +2 -1
  271. package/src/loader/loader-state.js +2 -2
  272. package/src/loader/memory-monitor.js +8 -0
  273. package/src/loader/multi-model-loader.d.ts +14 -0
  274. package/src/loader/multi-model-loader.js +70 -24
  275. package/src/loader/shard-cache.js +84 -14
  276. package/src/loader/shard-resolver.js +25 -3
  277. package/src/loader/tensors/tensor-loader.js +214 -144
  278. package/src/loader/tensors/tensor-reader.js +76 -19
  279. package/src/loader/weight-downcast.js +1 -1
  280. package/src/memory/buffer-pool.d.ts +9 -1
  281. package/src/memory/buffer-pool.js +109 -44
  282. package/src/memory/unified-detect.js +1 -1
  283. package/src/rules/inference/dtype.rules.json +5 -0
  284. package/src/rules/inference/kernel-path.rules.json +24 -8
  285. package/src/rules/kernels/split-qg.rules.json +6 -0
  286. package/src/rules/rule-registry.js +27 -1
  287. package/src/storage/backends/opfs-store.js +68 -24
  288. package/src/storage/downloader.js +365 -83
  289. package/src/storage/index.d.ts +3 -0
  290. package/src/storage/index.js +3 -0
  291. package/src/storage/preflight.d.ts +2 -2
  292. package/src/storage/preflight.js +24 -2
  293. package/src/storage/quickstart-downloader.js +11 -5
  294. package/src/storage/registry.js +10 -4
  295. package/src/storage/reports.js +1 -1
  296. package/src/storage/shard-manager.d.ts +15 -1
  297. package/src/storage/shard-manager.js +55 -6
  298. package/src/storage/source-artifact-store.d.ts +52 -0
  299. package/src/storage/source-artifact-store.js +234 -0
  300. package/src/tooling/command-api-constants.d.ts +9 -0
  301. package/src/tooling/command-api-constants.js +9 -0
  302. package/src/tooling/command-api-family-normalizers.d.ts +9 -0
  303. package/src/tooling/command-api-family-normalizers.js +343 -0
  304. package/src/tooling/command-api-helpers.d.ts +25 -0
  305. package/src/tooling/command-api-helpers.js +262 -0
  306. package/src/tooling/command-api.js +16 -602
  307. package/src/tooling/command-envelope.js +4 -1
  308. package/src/tooling/command-runner-shared.js +52 -18
  309. package/src/tooling/conversion-config-materializer.js +3 -5
  310. package/src/tooling/lean-execution-contract.js +150 -3
  311. package/src/tooling/node-browser-command-runner.js +161 -271
  312. package/src/tooling/node-command-runner.js +29 -3
  313. package/src/tooling/node-converter.js +30 -1
  314. package/src/tooling/node-source-runtime.d.ts +1 -1
  315. package/src/tooling/node-source-runtime.js +120 -3
  316. package/src/tooling/node-webgpu.js +24 -21
  317. package/src/tooling/opfs-cache.js +21 -4
  318. package/src/tooling/runtime-input-composition.d.ts +38 -0
  319. package/src/tooling/runtime-input-composition.js +86 -0
  320. package/src/tooling/source-runtime-bundle.d.ts +40 -5
  321. package/src/tooling/source-runtime-bundle.js +261 -34
  322. package/src/tooling/source-runtime-materializer.d.ts +6 -0
  323. package/src/tooling/source-runtime-materializer.js +93 -0
  324. package/src/training/attention-backward.js +32 -17
  325. package/src/training/autograd.js +80 -52
  326. package/src/training/checkpoint-watch.d.ts +2 -1
  327. package/src/training/checkpoint-watch.js +39 -6
  328. package/src/training/checkpoint.js +40 -11
  329. package/src/training/clip.js +2 -1
  330. package/src/training/datasets/token-batch.js +20 -8
  331. package/src/training/distillation/checkpoint-watch.js +1 -0
  332. package/src/training/distillation/student-fixture.d.ts +22 -0
  333. package/src/training/distillation/student-fixture.js +846 -0
  334. package/src/training/distillation/suite-data.d.ts +45 -0
  335. package/src/training/distillation/suite-data.js +189 -0
  336. package/src/training/lora-pipeline.js +4 -7
  337. package/src/training/lora.js +26 -12
  338. package/src/training/loss.js +5 -6
  339. package/src/training/objectives/cross_entropy.js +2 -5
  340. package/src/training/objectives/distill_kd.js +4 -8
  341. package/src/training/objectives/distill_triplet.js +4 -8
  342. package/src/training/objectives/ul_stage2_base.js +4 -8
  343. package/src/training/operator-command.js +2 -0
  344. package/src/training/optimizer.js +19 -7
  345. package/src/training/runner.js +2 -1
  346. package/src/training/suite.js +18 -978
  347. package/src/training/tensor-factory.d.ts +9 -0
  348. package/src/training/tensor-factory.js +13 -0
  349. package/src/training/trainer.js +3 -5
  350. package/src/training/ul_dataset.js +3 -5
  351. package/src/training/workloads.js +70 -79
  352. package/src/types/model.d.ts +5 -0
  353. package/src/version.js +1 -1
  354. package/tools/convert-safetensors-node.js +22 -16
  355. package/tools/doppler-cli.js +50 -26
@@ -20,6 +20,7 @@ export {
20
20
  writeShard,
21
21
  createShardWriter,
22
22
  createConversionShardWriter,
23
+ createFileWriter,
23
24
  loadShard,
24
25
  loadShardRange,
25
26
  streamShardRange,
@@ -32,6 +33,7 @@ export {
32
33
  listModels,
33
34
  listFilesInStore,
34
35
  loadFileFromStore,
36
+ loadFileRangeFromStore,
35
37
  streamFileFromStore,
36
38
  getModelInfo,
37
39
  modelExists,
@@ -46,6 +48,7 @@ export {
46
48
  saveAuxFile,
47
49
  loadAuxFile,
48
50
  loadAuxText,
51
+ deleteFileFromStore,
49
52
  cleanup,
50
53
  } from './shard-manager.js';
51
54
 
@@ -113,8 +113,8 @@ export declare const GEMMA_1B_REQUIREMENTS: ModelRequirements;
113
113
 
114
114
  /**
115
115
  * All available model requirements
116
- * Naming convention: {family}-{version}-{size}-{variant}-{quant}
117
- * Examples: gemma-3-1b-it-wq4k-ef16-hf16, llama-3.2-1b-q4, mistral-7b-v0.3-q4
116
+ * Naming convention: {family}-{version}-{size}-{weights}[-{override-groups}]
117
+ * Examples: gemma-3-1b-it-q4k-ehf16-af32, llama-3.2-1b-q4, mistral-7b-v0.3-q4
118
118
  */
119
119
  export declare const MODEL_REQUIREMENTS: Record<string, ModelRequirements>;
120
120
 
@@ -12,7 +12,7 @@ import { GB, MB } from '../config/schema/index.js';
12
12
 
13
13
 
14
14
  export const GEMMA_1B_REQUIREMENTS = {
15
- modelId: 'gemma-3-1b-it-wq4k-ef16-hf16',
15
+ modelId: 'gemma-3-1b-it-q4k-ehf16-af32',
16
16
  displayName: 'Gemma 3 1B IT (Q4_K_M)',
17
17
  downloadSize: 565 * MB, // ~565MB actual size from manifest shards
18
18
  vramRequired: 1.5 * GB, // ~1.5GB VRAM for inference (weights + KV cache)
@@ -21,9 +21,31 @@ export const GEMMA_1B_REQUIREMENTS = {
21
21
  architecture: 'Gemma3ForCausalLM',
22
22
  };
23
23
 
24
+ const GEMMA_270M_REQUIREMENTS = {
25
+ modelId: 'gemma-3-270m-it-q4k-ehf16-af32',
26
+ displayName: 'Gemma 3 270M IT (Q4K)',
27
+ downloadSize: 418512553,
28
+ vramRequired: 1.0 * GB,
29
+ paramCount: '270M',
30
+ quantization: 'Q4K',
31
+ architecture: 'Gemma3ForCausalLM',
32
+ };
33
+
34
+ const EMBEDDINGGEMMA_300M_REQUIREMENTS = {
35
+ modelId: 'google-embeddinggemma-300m-q4k-ehf16-af32',
36
+ displayName: 'EmbeddingGemma 300M (Q4K)',
37
+ downloadSize: 480562497,
38
+ vramRequired: 1.0 * GB,
39
+ paramCount: '300M',
40
+ quantization: 'Q4K',
41
+ architecture: 'EmbeddingGemmaForEmbedding',
42
+ };
43
+
24
44
 
25
45
  export const MODEL_REQUIREMENTS = {
26
- 'gemma-3-1b-it-wq4k-ef16-hf16': GEMMA_1B_REQUIREMENTS,
46
+ 'gemma-3-270m-it-q4k-ehf16-af32': GEMMA_270M_REQUIREMENTS,
47
+ 'google-embeddinggemma-300m-q4k-ehf16-af32': EMBEDDINGGEMMA_300M_REQUIREMENTS,
48
+ 'gemma-3-1b-it-q4k-ehf16-af32': GEMMA_1B_REQUIREMENTS,
27
49
  };
28
50
 
29
51
  // ============================================================================
@@ -3,7 +3,7 @@
3
3
  import { downloadModel } from './downloader.js';
4
4
  import {
5
5
  runPreflightChecks,
6
- GEMMA_1B_REQUIREMENTS,
6
+ MODEL_REQUIREMENTS,
7
7
  } from './preflight.js';
8
8
  import { formatBytes } from './quota.js';
9
9
  import { getCdnBasePath } from './download-types.js';
@@ -53,11 +53,17 @@ export function getCDNBaseUrl() {
53
53
 
54
54
 
55
55
  export const QUICKSTART_MODELS = {
56
- 'gemma-3-1b-it-wq4k-ef16-hf16': {
57
- modelId: 'gemma-3-1b-it-wq4k-ef16-hf16',
58
- displayName: 'Gemma 3 1B IT (Q4_K_M)',
56
+ 'gemma-3-270m-it-q4k-ehf16-af32': {
57
+ modelId: 'gemma-3-270m-it-q4k-ehf16-af32',
58
+ displayName: 'Gemma 3 270M IT (Q4K)',
59
59
  baseUrl: null,
60
- requirements: GEMMA_1B_REQUIREMENTS,
60
+ requirements: MODEL_REQUIREMENTS['gemma-3-270m-it-q4k-ehf16-af32'],
61
+ },
62
+ 'google-embeddinggemma-300m-q4k-ehf16-af32': {
63
+ modelId: 'google-embeddinggemma-300m-q4k-ehf16-af32',
64
+ displayName: 'EmbeddingGemma 300M (Q4K)',
65
+ baseUrl: null,
66
+ requirements: MODEL_REQUIREMENTS['google-embeddinggemma-300m-q4k-ehf16-af32'],
61
67
  },
62
68
  };
63
69
 
@@ -44,13 +44,16 @@ async function readRegistryFromIdb() {
44
44
  const store = createIdbStore(runtime.loading.storage.backend.indexeddb);
45
45
  await store.openModel(REGISTRY_MODEL_ID, { create: true });
46
46
  try {
47
- const text = await store.readFile(REGISTRY_FILENAME);
47
+ const text = await store.readText(REGISTRY_FILENAME);
48
48
  await store.cleanup();
49
49
  if (!text) return { models: [] };
50
50
  return normalizeRegistry(JSON.parse(text));
51
- } catch {
51
+ } catch (error) {
52
52
  await store.cleanup();
53
- return { models: [] };
53
+ if (error?.message?.includes('not found')) {
54
+ return { models: [] };
55
+ }
56
+ throw error;
54
57
  }
55
58
  }
56
59
 
@@ -58,7 +61,10 @@ async function writeRegistryToIdb(registry) {
58
61
  const runtime = getRuntimeConfig();
59
62
  const store = createIdbStore(runtime.loading.storage.backend.indexeddb);
60
63
  await store.openModel(REGISTRY_MODEL_ID, { create: true });
61
- await store.writeFile(REGISTRY_FILENAME, JSON.stringify(registry, null, 2));
64
+ await store.writeFile(
65
+ REGISTRY_FILENAME,
66
+ new TextEncoder().encode(JSON.stringify(registry, null, 2))
67
+ );
62
68
  await store.cleanup();
63
69
  return { backend: 'indexeddb', path: REGISTRY_FILENAME };
64
70
  }
@@ -48,7 +48,7 @@ async function saveReportToIdb(modelId, filename, payload) {
48
48
  const store = createIdbStore(idbConfig);
49
49
  const reportModelId = `${REPORT_MODEL_PREFIX}${modelId}`;
50
50
  await store.openModel(reportModelId, { create: true });
51
- await store.writeFile(filename, payload);
51
+ await store.writeFile(filename, new TextEncoder().encode(payload));
52
52
  await store.cleanup();
53
53
  return `reports/${modelId}/${filename}`;
54
54
  }
@@ -84,6 +84,11 @@ export function createConversionShardWriter(
84
84
  shardIndex: number
85
85
  ): Promise<ShardWriteStream>;
86
86
 
87
+ export function createFileWriter(
88
+ filename: string,
89
+ options?: ShardWriterOptions
90
+ ): Promise<ShardWriteStream>;
91
+
87
92
  export function loadShard(
88
93
  shardIndex: number,
89
94
  options?: ShardReadOptions
@@ -117,7 +122,15 @@ export function deleteModel(modelId: string): Promise<boolean>;
117
122
  export function listModels(): Promise<string[]>;
118
123
  export function listFilesInStore(): Promise<string[]>;
119
124
  export function loadFileFromStore(filename: string): Promise<ArrayBuffer>;
120
- export function streamFileFromStore(filename: string, options?: { chunkBytes?: number }): AsyncIterable<Uint8Array> | null;
125
+ export function loadFileRangeFromStore(
126
+ filename: string,
127
+ offset?: number,
128
+ length?: number | null
129
+ ): Promise<ArrayBuffer>;
130
+ export function streamFileFromStore(
131
+ filename: string,
132
+ options?: { chunkBytes?: number; offset?: number; length?: number | null }
133
+ ): AsyncIterable<Uint8Array> | null;
121
134
  export function getModelInfo(modelId: string): Promise<ModelInfo>;
122
135
  export function modelExists(modelId: string): Promise<boolean>;
123
136
 
@@ -133,5 +146,6 @@ export function loadTokenizerModelFromStore(): Promise<ArrayBuffer | null>;
133
146
  export function saveAuxFile(filename: string, data: string | ArrayBuffer | Uint8Array): Promise<void>;
134
147
  export function loadAuxFile(filename: string): Promise<ArrayBuffer | null>;
135
148
  export function loadAuxText(filename: string): Promise<string | null>;
149
+ export function deleteFileFromStore(filename: string): Promise<boolean>;
136
150
 
137
151
  export function cleanup(): Promise<void>;
@@ -1,5 +1,6 @@
1
1
  import {
2
2
  getManifest,
3
+ getExpectedShardHash,
3
4
  getShardInfo,
4
5
  getShardCount,
5
6
  generateShardFilename,
@@ -280,7 +281,7 @@ export async function writeShard(shardIndex, data, options = { verify: true }) {
280
281
  const manifest = getManifest();
281
282
  const algorithm = requireManifestHashAlgorithm(manifest, 'shard write');
282
283
  const hash = await computeHash(bytes, algorithm);
283
- const expectedHash = shardInfo.hash;
284
+ const expectedHash = getExpectedShardHash(shardInfo, algorithm);
284
285
  if (!expectedHash) {
285
286
  await backend.deleteFile(shardInfo.filename);
286
287
  throw new Error(`Shard ${shardIndex} is missing hash in manifest`);
@@ -341,6 +342,19 @@ export async function createConversionShardWriter(shardIndex) {
341
342
  return backend.createWriteStream(filename);
342
343
  }
343
344
 
345
+ export async function createFileWriter(filename, options = {}) {
346
+ await ensureBackend();
347
+ requireModel();
348
+ if (!filename || typeof filename !== 'string') {
349
+ throw new Error('createFileWriter requires a filename');
350
+ }
351
+ if (!backend.createWriteStream) {
352
+ throw new Error('Storage backend does not support streaming writes');
353
+ }
354
+ const writerOptions = normalizeShardWriterOptions(options);
355
+ return backend.createWriteStream(filename, writerOptions);
356
+ }
357
+
344
358
  export async function loadShard(shardIndex, options = { verify: false }) {
345
359
  await ensureBackend();
346
360
  requireModel();
@@ -356,7 +370,7 @@ export async function loadShard(shardIndex, options = { verify: false }) {
356
370
  const manifest = getManifest();
357
371
  const algorithm = requireManifestHashAlgorithm(manifest, 'shard load');
358
372
  const hash = await computeHash(buffer, algorithm);
359
- const expectedHash = shardInfo.hash;
373
+ const expectedHash = getExpectedShardHash(shardInfo, algorithm);
360
374
  if (!expectedHash) {
361
375
  throw new Error(`Shard ${shardIndex} is missing hash in manifest`);
362
376
  }
@@ -518,7 +532,7 @@ export async function verifyIntegrity(options = {}) {
518
532
  const buffer = await loadShard(i, { verify: false });
519
533
  const hash = await computeHash(buffer, algorithm);
520
534
  const shardInfo = getShardInfo(i);
521
- const expectedHash = shardInfo?.hash;
535
+ const expectedHash = getExpectedShardHash(shardInfo, algorithm);
522
536
  if (!expectedHash) {
523
537
  corruptShards.push(i);
524
538
  continue;
@@ -581,6 +595,25 @@ export async function loadFileFromStore(filename) {
581
595
  return backend.readFile(filename);
582
596
  }
583
597
 
598
+ export async function loadFileRangeFromStore(filename, offset = 0, length = null) {
599
+ await ensureBackend();
600
+ requireModel();
601
+ if (!filename || typeof filename !== 'string') {
602
+ throw new Error('loadFileRangeFromStore requires a filename');
603
+ }
604
+ const start = Number.isFinite(Number(offset)) ? Math.max(0, Math.floor(Number(offset))) : 0;
605
+ const want = length == null
606
+ ? null
607
+ : (Number.isFinite(Number(length)) ? Math.max(0, Math.floor(Number(length))) : 0);
608
+ if (backend && typeof backend.readFileRange === 'function') {
609
+ return backend.readFileRange(filename, start, want);
610
+ }
611
+ const buffer = await backend.readFile(filename);
612
+ const bytes = new Uint8Array(buffer);
613
+ const end = want == null ? bytes.byteLength : Math.min(bytes.byteLength, start + want);
614
+ return bytes.slice(start, end).buffer;
615
+ }
616
+
584
617
  export function streamFileFromStore(filename, options = {}) {
585
618
  if (!backend || !filename || typeof filename !== 'string') {
586
619
  return null;
@@ -589,8 +622,12 @@ export function streamFileFromStore(filename, options = {}) {
589
622
  const runtimeDefault = runtime?.loading?.storage?.backend?.streaming?.readChunkBytes ?? (4 * 1024 * 1024);
590
623
  const raw = options.chunkBytes ?? runtimeDefault;
591
624
  const chunkBytes = Number.isFinite(raw) && raw > 0 ? Math.floor(raw) : (4 * 1024 * 1024);
625
+ const offset = Number.isFinite(Number(options.offset)) ? Math.max(0, Math.floor(Number(options.offset))) : 0;
626
+ const length = options.length == null
627
+ ? null
628
+ : (Number.isFinite(Number(options.length)) ? Math.max(0, Math.floor(Number(options.length))) : 0);
592
629
  if (typeof backend.readFileRangeStream === 'function') {
593
- return backend.readFileRangeStream(filename, 0, null, { chunkBytes });
630
+ return backend.readFileRangeStream(filename, offset, length, { chunkBytes });
594
631
  }
595
632
  return null;
596
633
  }
@@ -731,8 +768,11 @@ export async function loadTokenizerModelFromStore() {
731
768
  requireModel();
732
769
  try {
733
770
  return await backend.readFile('tokenizer.model');
734
- } catch (_error) {
735
- return null;
771
+ } catch (error) {
772
+ if (error?.name === 'NotFoundError' || error?.message?.includes('not found')) {
773
+ return null;
774
+ }
775
+ throw error;
736
776
  }
737
777
  }
738
778
 
@@ -771,6 +811,15 @@ export async function loadAuxFile(filename) {
771
811
  }
772
812
  }
773
813
 
814
+ export async function deleteFileFromStore(filename) {
815
+ await ensureBackend();
816
+ requireModel();
817
+ if (!filename || typeof filename !== 'string') {
818
+ throw new Error('deleteFileFromStore requires a filename');
819
+ }
820
+ return backend.deleteFile(filename);
821
+ }
822
+
774
823
  export async function loadAuxText(filename) {
775
824
  await ensureBackend();
776
825
  requireModel();
@@ -0,0 +1,52 @@
1
+ import type { RDRRManifest } from '../formats/rdrr/types.js';
2
+ import type {
3
+ SourceRuntimeMetadata,
4
+ SourceStorageContext,
5
+ } from '../tooling/source-runtime-bundle.js';
6
+
7
+ export interface StoredSourceArtifactFile {
8
+ path: string;
9
+ size: number | null;
10
+ hash: string | null;
11
+ hashAlgorithm: string | null;
12
+ kind: string;
13
+ }
14
+
15
+ export interface StoredSourceArtifactSourceFile extends StoredSourceArtifactFile {
16
+ index: number;
17
+ }
18
+
19
+ export interface StoredSourceArtifactDescriptor {
20
+ sourceRuntime: SourceRuntimeMetadata;
21
+ sourceFiles: StoredSourceArtifactSourceFile[];
22
+ auxiliaryFiles: StoredSourceArtifactFile[];
23
+ files: StoredSourceArtifactFile[];
24
+ totalBytes: number;
25
+ fingerprint: string;
26
+ }
27
+
28
+ export interface StoredSourceArtifactIntegrity {
29
+ valid: boolean;
30
+ missingFiles: string[];
31
+ corruptFiles: string[];
32
+ }
33
+
34
+ export declare function normalizeSourceArtifactPath(value: unknown): string;
35
+
36
+ export declare function listSourceArtifactFiles(manifest: RDRRManifest): StoredSourceArtifactFile[];
37
+
38
+ export declare function resolveSourceArtifact(
39
+ manifest: RDRRManifest
40
+ ): StoredSourceArtifactDescriptor | null;
41
+
42
+ export declare function buildSourceArtifactFingerprint(manifest: RDRRManifest): string | null;
43
+
44
+ export declare function verifyStoredSourceArtifact(
45
+ manifest: RDRRManifest,
46
+ options?: { checkHashes?: boolean }
47
+ ): Promise<StoredSourceArtifactIntegrity>;
48
+
49
+ export declare function createStoredSourceArtifactContext(
50
+ manifest: RDRRManifest,
51
+ options?: { verifyHashes?: boolean }
52
+ ): SourceStorageContext;
@@ -0,0 +1,234 @@
1
+ import {
2
+ createSourceStorageContext,
3
+ getSourceRuntimeMetadata,
4
+ } from '../tooling/source-runtime-bundle.js';
5
+ import {
6
+ computeHash,
7
+ loadAuxText,
8
+ loadFileFromStore,
9
+ loadFileRangeFromStore,
10
+ streamFileFromStore,
11
+ } from './shard-manager.js';
12
+
13
+ export function normalizeSourceArtifactPath(value) {
14
+ return String(value || '').trim().replace(/\\/g, '/');
15
+ }
16
+
17
+ function encodeUtf8(value) {
18
+ return new TextEncoder().encode(String(value ?? ''));
19
+ }
20
+
21
+ function normalizeArtifactFile(entry, kind) {
22
+ const path = normalizeSourceArtifactPath(entry?.path);
23
+ if (!path) {
24
+ return null;
25
+ }
26
+ return {
27
+ path,
28
+ size: Number.isFinite(entry?.size) ? Math.max(0, Math.floor(Number(entry.size))) : null,
29
+ hash: typeof entry?.hash === 'string' && entry.hash.trim() ? entry.hash.trim().toLowerCase() : null,
30
+ hashAlgorithm: typeof entry?.hashAlgorithm === 'string' && entry.hashAlgorithm.trim()
31
+ ? entry.hashAlgorithm.trim().toLowerCase()
32
+ : null,
33
+ kind,
34
+ };
35
+ }
36
+
37
+ function pushArtifactFile(files, seen, entry, kind) {
38
+ const normalized = normalizeArtifactFile(entry, kind);
39
+ if (!normalized || seen.has(normalized.path)) {
40
+ return;
41
+ }
42
+ seen.add(normalized.path);
43
+ files.push(normalized);
44
+ }
45
+
46
+ function collectSourceArtifactFiles(sourceRuntime) {
47
+ if (!sourceRuntime) {
48
+ return {
49
+ sourceFiles: [],
50
+ auxiliaryFiles: [],
51
+ files: [],
52
+ };
53
+ }
54
+
55
+ const files = [];
56
+ const seen = new Set();
57
+ const sourceFiles = [];
58
+
59
+ for (let index = 0; index < sourceRuntime.sourceFiles.length; index += 1) {
60
+ const entry = sourceRuntime.sourceFiles[index];
61
+ const normalized = normalizeArtifactFile(entry, 'source');
62
+ if (!normalized) {
63
+ continue;
64
+ }
65
+ sourceFiles.push({
66
+ ...normalized,
67
+ index: Number.isFinite(entry?.index) ? Math.max(0, Math.floor(Number(entry.index))) : index,
68
+ });
69
+ pushArtifactFile(files, seen, entry, 'source');
70
+ }
71
+ for (const entry of sourceRuntime.auxiliaryFiles) {
72
+ pushArtifactFile(files, seen, entry, entry?.kind || 'auxiliary');
73
+ }
74
+
75
+ const tokenizer = sourceRuntime.tokenizer ?? {};
76
+ const auxiliaryByPath = new Map(
77
+ sourceRuntime.auxiliaryFiles.map((entry) => [normalizeSourceArtifactPath(entry?.path), entry])
78
+ );
79
+ for (const [path, kind] of [
80
+ [tokenizer.jsonPath, 'tokenizer_json'],
81
+ [tokenizer.configPath, 'tokenizer_config'],
82
+ [tokenizer.modelPath, 'tokenizer_model'],
83
+ ]) {
84
+ const normalizedPath = normalizeSourceArtifactPath(path);
85
+ if (!normalizedPath) {
86
+ continue;
87
+ }
88
+ pushArtifactFile(files, seen, auxiliaryByPath.get(normalizedPath) ?? { path: normalizedPath }, kind);
89
+ }
90
+
91
+ files.sort((left, right) => left.path.localeCompare(right.path));
92
+ return {
93
+ sourceFiles,
94
+ auxiliaryFiles: files.filter((entry) => entry.kind !== 'source'),
95
+ files,
96
+ };
97
+ }
98
+
99
+ export function listSourceArtifactFiles(manifest) {
100
+ const sourceRuntime = getSourceRuntimeMetadata(manifest);
101
+ return collectSourceArtifactFiles(sourceRuntime).files;
102
+ }
103
+
104
+ export function resolveSourceArtifact(manifest) {
105
+ const sourceRuntime = getSourceRuntimeMetadata(manifest);
106
+ if (!sourceRuntime) {
107
+ return null;
108
+ }
109
+ const { sourceFiles, auxiliaryFiles, files } = collectSourceArtifactFiles(sourceRuntime);
110
+ const totalBytes = files.reduce((sum, entry) => sum + (entry.size || 0), 0);
111
+ return {
112
+ sourceRuntime,
113
+ sourceFiles,
114
+ auxiliaryFiles,
115
+ files,
116
+ totalBytes,
117
+ fingerprint: JSON.stringify({
118
+ mode: sourceRuntime.mode,
119
+ schema: sourceRuntime.schema,
120
+ hashAlgorithm: sourceRuntime.hashAlgorithm,
121
+ pathSemantics: sourceRuntime.pathSemantics,
122
+ sourceKind: sourceRuntime.sourceKind,
123
+ files: files.map((entry) => ({
124
+ path: entry.path,
125
+ size: entry.size,
126
+ hash: entry.hash,
127
+ hashAlgorithm: entry.hashAlgorithm,
128
+ kind: entry.kind,
129
+ })),
130
+ }),
131
+ };
132
+ }
133
+
134
+ export function buildSourceArtifactFingerprint(manifest) {
135
+ return resolveSourceArtifact(manifest)?.fingerprint ?? null;
136
+ }
137
+
138
+ async function loadStoreFile(path) {
139
+ try {
140
+ return await loadFileFromStore(path);
141
+ } catch (error) {
142
+ const message = String(error?.message || '');
143
+ if (error?.name === 'NotFoundError' || message.toLowerCase().includes('not found')) {
144
+ return null;
145
+ }
146
+ throw error;
147
+ }
148
+ }
149
+
150
+ export async function verifyStoredSourceArtifact(manifest, options = {}) {
151
+ const sourceRuntime = getSourceRuntimeMetadata(manifest);
152
+ if (!sourceRuntime) {
153
+ throw new Error('verifyStoredSourceArtifact requires a direct-source manifest.');
154
+ }
155
+
156
+ const checkHashes = options.checkHashes !== false;
157
+ const missingFiles = [];
158
+ const corruptFiles = [];
159
+ const files = listSourceArtifactFiles(manifest);
160
+
161
+ for (const entry of files) {
162
+ const payload = await loadStoreFile(entry.path);
163
+ if (!(payload instanceof ArrayBuffer)) {
164
+ missingFiles.push(entry.path);
165
+ continue;
166
+ }
167
+ if (!checkHashes || !entry.hash) {
168
+ continue;
169
+ }
170
+ const isTextAsset = entry.kind === 'config'
171
+ || entry.kind === 'tokenizer_json'
172
+ || entry.kind === 'tokenizer_config'
173
+ || entry.kind === 'safetensors_index';
174
+ const computedHash = isTextAsset
175
+ ? await computeHash(encodeUtf8(new TextDecoder().decode(payload)), entry.hashAlgorithm || sourceRuntime.hashAlgorithm)
176
+ : await computeHash(new Uint8Array(payload), entry.hashAlgorithm || sourceRuntime.hashAlgorithm);
177
+ if (computedHash !== entry.hash) {
178
+ corruptFiles.push(entry.path);
179
+ }
180
+ }
181
+
182
+ return {
183
+ valid: missingFiles.length === 0 && (!checkHashes || corruptFiles.length === 0),
184
+ missingFiles,
185
+ corruptFiles,
186
+ };
187
+ }
188
+
189
+ export function createStoredSourceArtifactContext(manifest, options = {}) {
190
+ const sourceRuntime = getSourceRuntimeMetadata(manifest);
191
+ if (!sourceRuntime) {
192
+ throw new Error('createStoredSourceArtifactContext requires a direct-source manifest.');
193
+ }
194
+
195
+ const readRange = async (path, offset, length) => loadFileRangeFromStore(path, offset, length);
196
+ const streamRange = (path, offset, length, streamOptions = {}) => {
197
+ const stream = streamFileFromStore(path, {
198
+ chunkBytes: streamOptions?.chunkBytes,
199
+ offset,
200
+ length,
201
+ });
202
+ if (!stream) {
203
+ return null;
204
+ }
205
+ return stream;
206
+ };
207
+ const readText = async (path) => loadAuxText(path);
208
+ const readBinary = async (path) => {
209
+ const payload = await loadStoreFile(path);
210
+ if (!(payload instanceof ArrayBuffer)) {
211
+ throw new Error(`Missing stored source binary file: ${path}`);
212
+ }
213
+ return payload;
214
+ };
215
+
216
+ return createSourceStorageContext({
217
+ manifest,
218
+ readRange,
219
+ streamRange: streamRange ? (async function* (path, offset, length, streamOptions = {}) {
220
+ const stream = streamRange(path, offset, length, streamOptions);
221
+ if (!stream) {
222
+ const payload = await loadFileRangeFromStore(path, offset, length);
223
+ yield new Uint8Array(payload);
224
+ return;
225
+ }
226
+ for await (const chunk of stream) {
227
+ yield chunk;
228
+ }
229
+ }) : null,
230
+ readText,
231
+ readBinary,
232
+ verifyHashes: options.verifyHashes !== false,
233
+ });
234
+ }
@@ -0,0 +1,9 @@
1
+ export declare const TOOLING_COMMAND_SET: readonly string[];
2
+ export declare const TOOLING_SURFACE_SET: readonly string[];
3
+ export declare const TOOLING_SUITE_SET: readonly string[];
4
+ export declare const TOOLING_INTENT_SET: readonly string[];
5
+ export declare const VERIFY_SUITES: readonly string[];
6
+ export declare const TRAINING_STAGE_SET: readonly string[];
7
+ export declare const DISTILL_ACTION_SET: readonly string[];
8
+ export declare const LORA_ACTION_SET: readonly string[];
9
+ export declare const TRAINING_COMMAND_SCHEMA_VERSION: number;
@@ -0,0 +1,9 @@
1
+ export const TOOLING_COMMAND_SET = ['convert', 'debug', 'bench', 'verify', 'lora', 'distill'];
2
+ export const TOOLING_SURFACE_SET = ['browser', 'node'];
3
+ export const TOOLING_SUITE_SET = ['kernels', 'inference', 'training', 'bench', 'debug', 'diffusion', 'energy'];
4
+ export const TOOLING_INTENT_SET = ['verify', 'investigate', 'calibrate'];
5
+ export const VERIFY_SUITES = ['kernels', 'inference', 'training', 'diffusion', 'energy'];
6
+ export const TRAINING_STAGE_SET = ['stage1_joint', 'stage2_base', 'stage_a', 'stage_b'];
7
+ export const DISTILL_ACTION_SET = ['run', 'stage-a', 'stage-b', 'eval', 'watch', 'compare', 'quality-gate', 'subsets'];
8
+ export const LORA_ACTION_SET = ['run', 'eval', 'watch', 'export', 'compare', 'quality-gate', 'activate'];
9
+ export const TRAINING_COMMAND_SCHEMA_VERSION = 1;
@@ -0,0 +1,9 @@
1
+ export declare function normalizeConvert(raw: Record<string, unknown>): Record<string, unknown>;
2
+ export declare function normalizeTrainingOperatorCommand(
3
+ raw: Record<string, unknown>,
4
+ command: string
5
+ ): Record<string, unknown>;
6
+ export declare function normalizeSuiteCommand(
7
+ raw: Record<string, unknown>,
8
+ command: string
9
+ ): Record<string, unknown>;