doppler-gpu 0.6.0 → 0.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. package/package.json +8 -2
  2. package/src/capsule-runtime.d.ts +2 -0
  3. package/src/capsule-runtime.js +1 -0
  4. package/src/client/capsule-host.browser.js +1 -0
  5. package/src/client/capsule-host.d.ts +2 -0
  6. package/src/client/capsule-host.js +1 -0
  7. package/src/client/doppler-api.d.ts +2 -0
  8. package/src/client/doppler-api.js +1 -0
  9. package/src/client/model-host/model-session.d.ts +1 -0
  10. package/src/client/model-host/model-session.js +12 -0
  11. package/src/client/runtime/capsule-operation-adapters.js +15 -14
  12. package/src/client/runtime/capsule-operation-executor.js +3 -2
  13. package/src/client/runtime/capsule-program-adapter.js +2 -0
  14. package/src/client/runtime/capsule-session-execution.js +9 -5
  15. package/src/client/runtime/composition-root.d.ts +6 -3
  16. package/src/client/runtime/composition-root.js +13 -5
  17. package/src/client/runtime/session-controller.d.ts +6 -17
  18. package/src/client/runtime/session-controller.js +42 -120
  19. package/src/client/runtime/verified-capsule-artifact-store.js +38 -9
  20. package/src/config/capsule-loading.d.ts +1 -1
  21. package/src/config/capsule-loading.schema.json +3 -2
  22. package/src/config/capsule-operation.js +6 -1
  23. package/src/config/capsule-operations.json +1 -2
  24. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +3 -3
  25. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +1 -1
  26. package/src/config/conversion/esm/esm2-t12-35m-ur50d-f32-af32.json +32 -14
  27. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +2 -2
  28. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +2 -2
  29. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +2 -2
  30. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +2 -2
  31. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +2 -2
  32. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +2 -2
  33. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +2 -2
  34. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +3 -3
  35. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +3 -3
  36. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +5 -5
  37. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +2 -2
  38. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +3 -3
  39. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +3 -3
  40. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +3 -3
  41. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +2 -2
  42. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +2 -2
  43. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +2 -2
  44. package/src/config/conversion/glmocr/glm-ocr-f16-af32.json +1 -1
  45. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +2 -2
  46. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +2 -2
  47. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +2 -2
  48. package/src/config/conversion/minicpm/minicpm4-0-5b-f16-af32.json +2 -2
  49. package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-bf16-af32.json +1 -1
  50. package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-f16-af32.json +1 -1
  51. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +5 -5
  52. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +5 -5
  53. package/src/config/conversion/qwen3/qwen-3-5-9b-q4k-ehaf16.json +5 -5
  54. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +5 -5
  55. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +5 -5
  56. package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-q4k-ehf16-af32.json +2 -2
  57. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-af32.json +2 -2
  58. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-true-logit-af32.json +1 -1
  59. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +1 -1
  60. package/src/config/conversion/qwen3/qwen3.8-27b-text-f16-af32.json +3 -3
  61. package/src/config/conversion/qwen3/qwen3.8-27b-text-q4k-ehaf16.json +3 -3
  62. package/src/config/generation-contract.d.ts +39 -0
  63. package/src/config/generation-contract.js +66 -0
  64. package/src/config/generation-contract.json +32 -0
  65. package/src/config/kernel-path-loader.d.ts +7 -0
  66. package/src/config/kernel-path-loader.js +19 -0
  67. package/src/config/kernels/kernel-ref-digests.js +16 -16
  68. package/src/config/kernels/registry.json +35 -10
  69. package/src/config/schema/doppler.schema.js +1 -0
  70. package/src/config/source-packages/litert/gemma-4-e2b-it.json +2 -2
  71. package/src/converter/forge-qualification.js +2 -1
  72. package/src/converter/forge-stages.js +6 -4
  73. package/src/converter/quantizer.js +18 -4
  74. package/src/gpu/device-state.js +8 -0
  75. package/src/gpu/device.js +6 -1
  76. package/src/gpu/kernels/gelu.d.ts +4 -0
  77. package/src/gpu/kernels/gelu.js +11 -1
  78. package/src/gpu/kernels/gelu.wgsl +16 -0
  79. package/src/gpu/kernels/gelu_f16.wgsl +16 -0
  80. package/src/gpu/kernels/rep-penalty.js +30 -5
  81. package/src/gpu/kernels/rep_penalty.wgsl +25 -30
  82. package/src/gpu/kernels/rep_penalty_f16.wgsl +24 -25
  83. package/src/gpu/kernels/sample.d.ts +1 -0
  84. package/src/gpu/kernels/sample.js +31 -16
  85. package/src/gpu/kernels/sample.wgsl +112 -157
  86. package/src/gpu/kernels/sample_f16.wgsl +168 -162
  87. package/src/index.d.ts +2 -0
  88. package/src/index.js +1 -0
  89. package/src/inference/pipelines/context.js +6 -2
  90. package/src/inference/pipelines/structured/json-grammar-mask.d.ts +2 -0
  91. package/src/inference/pipelines/structured/json-grammar-mask.js +115 -118
  92. package/src/inference/pipelines/text/ffn/dense-executor.js +5 -0
  93. package/src/inference/pipelines/text/ffn/dense.js +2 -2
  94. package/src/inference/pipelines/text/generator/decode-runtime.js +8 -14
  95. package/src/inference/pipelines/text/generator/decode.js +18 -6
  96. package/src/inference/pipelines/text/generator/training.js +3 -2
  97. package/src/inference/pipelines/text/generator-logits-observation.js +5 -2
  98. package/src/inference/pipelines/text/generator-runtime.js +18 -19
  99. package/src/inference/pipelines/text/generator-steps.js +15 -8
  100. package/src/inference/pipelines/text/generator.js +16 -10
  101. package/src/inference/pipelines/text/linear-attention.js +24 -0
  102. package/src/inference/pipelines/text/logits/output-transform.js +10 -3
  103. package/src/inference/pipelines/text/sampling-config.js +22 -113
  104. package/src/inference/pipelines/text/sampling.d.ts +9 -1
  105. package/src/inference/pipelines/text/sampling.js +18 -375
  106. package/src/inference/pipelines/text/types.d.ts +3 -3
  107. package/src/inference/token-sampling.js +319 -0
  108. package/src/storage/backends/opfs-store.d.ts +41 -0
  109. package/src/tooling/node-browser/static-server.js +1 -0
  110. package/src/tooling-exports/storage.d.ts +2 -0
  111. package/src/tooling-exports/storage.js +1 -0
  112. package/src/version.js +1 -1
  113. package/tests/kernels/browser/test-page.js +1 -0
@@ -172,8 +172,8 @@ export const KERNEL_REF_CONTENT_DIGESTS = Object.freeze({
172
172
  "gather_vec4_f16_out.wgsl#gather_vec4_f16_out": "dd631155d4df58d29a1c444f521a8ebbf28532df3eb37d1b27f8015d3a42d9e6",
173
173
  "gather_vec4.wgsl#gather_vec4": "e8d90b826350a9feb96476fc4a5e568aebfc29948cf07a2d7db0dee4a4216b6a",
174
174
  "gather.wgsl#main": "4b12653c53247b32ebde7f6cf6a989d6248977e3816c761540b990b5f9818cb6",
175
- "gelu_f16.wgsl#main": "01202f6b628ba5b5a1ede8c4c424001c9f59d73a970b41d609a66455cb2b2fea",
176
- "gelu.wgsl#main": "7f8900b69de5107e4cf424cac0d4de1539591ece34e977ff30f6f7364460ab34",
175
+ "gelu_f16.wgsl#main": "1f713a30bf6777270e9386792bde2a13824d37284abd0ece1abb1de7e2807fd2",
176
+ "gelu.wgsl#main": "db864455ec72070d7d8729c8744e2eac377a4939a9bc2ecb26eb086d866e16b0",
177
177
  "gemma4_route_q4_matmul_f16a.wgsl#route_matmul_f16a": "7f42e4bc7d7f398cf826df6c708b3da760557a713f5d208aefcd71c6a2512380",
178
178
  "gptoss_mxfp4_expert_fused.wgsl#main_expert": "3159e8cd81da13f909cf905e6d35307fefe1dcbbdf1b2b8e8ff0ce923bd71180",
179
179
  "gptoss_router_topk.wgsl#softmax_topk": "86e4ea709c0c0084d09c6a4cd07710dc14f380e03f91b8ed9ec871b310be49f1",
@@ -251,8 +251,8 @@ export const KERNEL_REF_CONTENT_DIGESTS = Object.freeze({
251
251
  "probe_shader_f16.wgsl#main": "4a44a579136492fdc579997e7d2bd9c249e117039c9f19d44b1f7ab68c668325",
252
252
  "relu_f16.wgsl#main": "baac3a33693e5a72e60c7ec9b4a4dbcec10a83ca1fac2972988b07a75780f14a",
253
253
  "relu.wgsl#main": "52ffa66f9898a4f291758ae93055000cb1eb488c22a993e6b0568a29d9d3ad28",
254
- "rep_penalty_f16.wgsl#main": "5adf5a410b9fd520620e72fc3ae831096ffcd5d40285a225c3bbe4d1775b0486",
255
- "rep_penalty.wgsl#main": "00d5652fb17ac74c023dd9471d37b55d5a46ba0e6a7f3f16e18f743b8a2f3578",
254
+ "rep_penalty_f16.wgsl#main": "2efd3f8dc87f715548bb131e67dd64942e7873a1bd9302a270bdeacc0ea8ecfb",
255
+ "rep_penalty.wgsl#main": "833d412b5db7c6da33ab980c4125717bbb8186f20c6e2a36a76e97e41fd99b4a",
256
256
  "repeat_channels_f16.wgsl#main": "6eee5f9f74d8c17a71f6f422b503bbfb872350a9486ec0065fd1f67718311883",
257
257
  "repeat_channels.wgsl#main": "bce0cb33beed50ce388b2d069961ae0fbe361dd4930b9344cb066b3390475a51",
258
258
  "residual_f16_vec4.wgsl#add_vec4": "7735198b679f375c0a980f109ea05d4b95af13a00438347ace7a27d36009aa76",
@@ -287,18 +287,18 @@ export const KERNEL_REF_CONTENT_DIGESTS = Object.freeze({
287
287
  "rope.wgsl#rope_ntk_scaled": "01ab3773e2558b8a541c3c27574cb277923ecff72941c9b590acfa3eeb45c6d4",
288
288
  "rope.wgsl#rope_qk": "7ba1041b73527c419f22c4f7f08345d411ff23317e9d9107907b3f69637e5064",
289
289
  "rope.wgsl#rope_yarn": "ed18b619bc67605525dddcb324e5728652ab63ece52b3327d4403649e0abac0c",
290
- "sample_f16.wgsl#argmax": "b877987f1c87ff1bbb8842e2b58e4773c9f50abd95904724d9bfe9382cb7268f",
291
- "sample_f16.wgsl#argmax_reduce": "448d279946f3fa776dbb0de4023b37932f427195956bbfddcaebb13e814dd79f",
292
- "sample_f16.wgsl#find_topk_phase1": "99e81c9dff51ecb9e1e25b2dcfaaf7ed5841b655f2fbcc0dd27c4112393e81b9",
293
- "sample_f16.wgsl#find_topk_phase2": "d21b8f7e35236be07d32840d7eac2f8b382f004959a5c01ef1d8abb70feacc7b",
294
- "sample_f16.wgsl#sample_single_pass": "6a1c24cdd9cd80b8dd17979fb60509c238e2440c1b659332efe4b7b6a8d5c3d6",
295
- "sample_f16.wgsl#softmax_and_sample": "f30a5382648a9c249fc96825e764a68cfa0a861831867e7480dde29594a396dd",
296
- "sample.wgsl#argmax": "28fc0a65069950bf7937e67a6baa157dff1f68b0f80f75c667136b6e62270b37",
297
- "sample.wgsl#argmax_reduce": "c35bf6231c9e2154022f4f2643061c26adbb02c9823b64336567b6bec2a0a0e8",
298
- "sample.wgsl#find_topk_phase1": "76b3f50e83aab9b62f8997f0d9ac724970392f1bc612b9e1cee4fbd398aa554a",
299
- "sample.wgsl#find_topk_phase2": "12da16e41055553c80eafd95ee946a203bbb344640f2aec9b38b173e4f6a82b5",
300
- "sample.wgsl#sample_single_pass": "7ff3dd675b4737d920e0b3889e15c4fedde8410a353bbc6d4651db76ce69eb9b",
301
- "sample.wgsl#softmax_and_sample": "0cdf09b057bc135b31b63df2100522753e68d290ba08bc93426645fc5b9f12f5",
290
+ "sample_f16.wgsl#argmax": "f43383f5596546e38ebdcf318463c24105c0de03f6350125d1724b0f4a8bdf03",
291
+ "sample_f16.wgsl#argmax_reduce": "5d77c0c647eeb9ff039d8e8f9866e439f4de4bae4c84bebb6cb1b6c890ea661d",
292
+ "sample_f16.wgsl#find_topk_phase1": "bb24e6ef6381193e653ac3aff2eeb14a0b8b376185951c0722428c3a2b466415",
293
+ "sample_f16.wgsl#find_topk_phase2": "5f3ede884117303a27f4b146bdc5c1274fe525684c6ca606268ece2fb093839e",
294
+ "sample_f16.wgsl#sample_single_pass": "7ce553d40f31f3fbe3a0d035e885ab363d9863a006c1478e0f7fe4886bb0972f",
295
+ "sample_f16.wgsl#softmax_and_sample": "418f2661255a9db6b83cc8d111897c427985e5602fd59dae6b326591fce9d8c9",
296
+ "sample.wgsl#argmax": "50af13644f36245b6944468a7c0048ffcfdf5a0e4bf842cac7efcf521b9b3148",
297
+ "sample.wgsl#argmax_reduce": "c197677307ebe8c01e717f6d5fa9e37aaf1dfa38e2b4928257983d558ca5dbea",
298
+ "sample.wgsl#find_topk_phase1": "c6e7d0795682926bc3ffa9eeba1eddfb54667efaf4b58c7214d8ee8214780879",
299
+ "sample.wgsl#find_topk_phase2": "a5eb7235fb97b2208aa921455563ac46bf84faf7fc40e389cc8ee74d0d0f351b",
300
+ "sample.wgsl#sample_single_pass": "85b79cfe27c04f8c9ff113e0b5bfa8ae4a9b4da059b7504f51d2f76c56fc604b",
301
+ "sample.wgsl#softmax_and_sample": "5ec8436e2064d003b7e2e59956758e2f64f2b89cc6cdff49a8c0bf759110bc84",
302
302
  "scale_f16.wgsl#main": "4b97fc580eb3f192d89a09986cabf73345a7162957f027024e07589efbb20cd1",
303
303
  "scale_f16.wgsl#main_inplace": "9f49a1964d0b8b35b9fe08ea2fcf2307b6dd56ea48339b1762b643159c71ff0a",
304
304
  "scale.wgsl#main": "8070894b1488c76bdf6685b99243e0fadd79a2511b52b22b91ba1ead9990831c",
@@ -8642,11 +8642,11 @@
8642
8642
  "type": "u32",
8643
8643
  "offset": 16
8644
8644
  },
8645
- "_pad0": {
8646
- "type": "u32",
8645
+ "presence_penalty": {
8646
+ "type": "f32",
8647
8647
  "offset": 20
8648
8648
  },
8649
- "_pad1": {
8649
+ "repetition_penalty_window": {
8650
8650
  "type": "u32",
8651
8651
  "offset": 24
8652
8652
  },
@@ -8683,7 +8683,7 @@
8683
8683
  }
8684
8684
  },
8685
8685
  "sample": {
8686
- "description": "GPU-side sampling kernels",
8686
+ "description": "Exact GPU top-k/top-p sampling; F16 logits use F32 candidate and probability accumulation",
8687
8687
  "baseBindings": [
8688
8688
  {
8689
8689
  "index": 0,
@@ -8699,6 +8699,16 @@
8699
8699
  "index": 2,
8700
8700
  "name": "output",
8701
8701
  "type": "storage"
8702
+ },
8703
+ {
8704
+ "index": 3,
8705
+ "name": "topk_indices",
8706
+ "type": "storage"
8707
+ },
8708
+ {
8709
+ "index": 4,
8710
+ "name": "topk_logits",
8711
+ "type": "storage"
8702
8712
  }
8703
8713
  ],
8704
8714
  "baseUniforms": {
@@ -8710,24 +8720,39 @@
8710
8720
  "offset": 0
8711
8721
  },
8712
8722
  {
8713
- "name": "temperature",
8714
- "type": "f32",
8723
+ "name": "top_k",
8724
+ "type": "u32",
8715
8725
  "offset": 4
8716
8726
  },
8717
8727
  {
8718
- "name": "top_k",
8719
- "type": "u32",
8728
+ "name": "temperature",
8729
+ "type": "f32",
8720
8730
  "offset": 8
8721
8731
  },
8722
8732
  {
8723
- "name": "top_p",
8733
+ "name": "random_value",
8724
8734
  "type": "f32",
8725
8735
  "offset": 12
8726
8736
  },
8727
8737
  {
8728
- "name": "random_seed",
8738
+ "name": "pad_token_id",
8729
8739
  "type": "u32",
8730
8740
  "offset": 16
8741
+ },
8742
+ {
8743
+ "name": "logit_softcap",
8744
+ "type": "f32",
8745
+ "offset": 20
8746
+ },
8747
+ {
8748
+ "name": "output_index",
8749
+ "type": "u32",
8750
+ "offset": 24
8751
+ },
8752
+ {
8753
+ "name": "top_p",
8754
+ "type": "f32",
8755
+ "offset": 28
8731
8756
  }
8732
8757
  ]
8733
8758
  },
@@ -45,6 +45,7 @@ export const DEFAULT_RUNTIME_CONFIG = {
45
45
  topP: 0.95,
46
46
  topK: 50,
47
47
  repetitionPenalty: 1.1,
48
+ presencePenalty: 0.0,
48
49
  greedyThreshold: 0.01,
49
50
  repetitionPenaltyWindow: 100,
50
51
  suppressSpecialTokens: false,
@@ -310,7 +310,7 @@
310
310
  "gelu": {
311
311
  "kernel": "gelu.wgsl",
312
312
  "entry": "main",
313
- "digest": "sha256:7f8900b69de5107e4cf424cac0d4de1539591ece34e977ff30f6f7364460ab34",
313
+ "digest": "sha256:db864455ec72070d7d8729c8744e2eac377a4939a9bc2ecb26eb086d866e16b0",
314
314
  "constants": {
315
315
  "HAS_GATE": true
316
316
  }
@@ -370,7 +370,7 @@
370
370
  "sample": {
371
371
  "kernel": "sample.wgsl",
372
372
  "entry": "sample_single_pass",
373
- "digest": "sha256:7ff3dd675b4737d920e0b3889e15c4fedde8410a353bbc6d4651db76ce69eb9b"
373
+ "digest": "sha256:85b79cfe27c04f8c9ff113e0b5bfa8ae4a9b4da059b7504f51d2f76c56fc604b"
374
374
  }
375
375
  },
376
376
  "preLayer": [
@@ -83,7 +83,8 @@ export function buildQualificationRecords(lowered) {
83
83
  transcriptHash: hashStable(transcript),
84
84
  }, ...normalized.qualificationEvidence.map(({ artifact, ...record }) => record)];
85
85
  }
86
- if (normalized.manifest?.modelType === 'embedding') {
86
+ if (normalized.manifest?.modelType === 'embedding'
87
+ && normalized.manifest?.inference?.supportsSequence !== true) {
87
88
  throw new Error('Forge requires text embedding qualification for an embedding model; other operation evidence is insufficient.');
88
89
  }
89
90
  if (transcript?.operation === 'rerank') {
@@ -337,6 +337,7 @@ export function stageAnalyze(normalized) {
337
337
  assertModelTopologyRepresentable(manifest);
338
338
  const architecture = requireObject(manifest.architecture, 'manifest.architecture');
339
339
  const inference = requireObject(manifest.inference, 'manifest.inference');
340
+ const isSequence = inference.supportsSequence === true;
340
341
  const attention = requireObject(inference.attention, 'manifest.inference.attention');
341
342
  const normalization = requireObject(inference.normalization, 'manifest.inference.normalization');
342
343
  const ffn = requireObject(inference.ffn, 'manifest.inference.ffn');
@@ -399,16 +400,17 @@ export function stageAnalyze(normalized) {
399
400
  intermediateSize: requirePositiveInteger(architecture.intermediateSize, 'manifest.architecture.intermediateSize'),
400
401
  },
401
402
  outputTopology: {
402
- headType: manifest.modelType === 'embedding' ? 'text-embedding'
403
- : inference.supportsSequence === true ? 'sequence-encoder' : 'causal-lm',
403
+ headType: isSequence ? 'sequence-encoder'
404
+ : manifest.modelType === 'embedding' ? 'text-embedding' : 'causal-lm',
404
405
  tieWeights: output.tieWordEmbeddings === true,
405
406
  ...(inference.supportsRerank === true
406
407
  ? { rerank: structuredClone(requireObject(inference.rerank, 'manifest.inference.rerank')) }
407
408
  : {}),
408
- ...(inference.supportsSequence === true
409
+ ...(isSequence
409
410
  ? { sequence: structuredClone(requireObject(inference.sequence, 'manifest.inference.sequence')) }
410
411
  : {}),
411
- ...(manifest.modelType === 'embedding' || inference.supportsEmbedding === true
412
+ ...(!isSequence
413
+ && (manifest.modelType === 'embedding' || inference.supportsEmbedding === true)
412
414
  ? { embedding: resolveCapsuleEmbeddingContract(manifest) }
413
415
  : {}),
414
416
  },
@@ -18,7 +18,7 @@ export function float32ToFloat16(value) {
18
18
 
19
19
  const sign = (f >> 31) & 0x1;
20
20
  let exp = (f >> 23) & 0xff;
21
- let frac = f & 0x7fffff;
21
+ const frac = f & 0x7fffff;
22
22
 
23
23
  if (exp === 0xff) {
24
24
  return (sign << 15) | 0x7c00 | (frac ? 0x200 : 0);
@@ -38,11 +38,25 @@ export function float32ToFloat16(value) {
38
38
  if (exp < -10) {
39
39
  return sign << 15;
40
40
  }
41
- frac = (frac | 0x800000) >> (1 - exp);
42
- return (sign << 15) | (frac >> 13);
41
+ const significand = frac | 0x800000;
42
+ const shift = 14 - exp;
43
+ let rounded = significand >>> shift;
44
+ const remainder = significand & ((1 << shift) - 1);
45
+ const halfway = 1 << (shift - 1);
46
+ if (remainder > halfway || (remainder === halfway && (rounded & 1))) {
47
+ rounded += 1;
48
+ }
49
+ return (sign << 15) | rounded;
43
50
  }
44
51
 
45
- return (sign << 15) | (exp << 10) | (frac >> 13);
52
+ // Round to nearest, ties to even. Carry may advance the exponent or overflow
53
+ // the largest finite half to infinity; truncation biases converted weights.
54
+ let rounded = (exp << 10) | (frac >>> 13);
55
+ const remainder = frac & 0x1fff;
56
+ if (remainder > 0x1000 || (remainder === 0x1000 && (rounded & 1))) {
57
+ rounded += 1;
58
+ }
59
+ return (sign << 15) | rounded;
46
60
  }
47
61
 
48
62
  export function float16ToFloat32(h) {
@@ -6,6 +6,9 @@ export function getSharedDeviceState() {
6
6
  if (!(existing.bufferOwners instanceof WeakMap)) {
7
7
  existing.bufferOwners = new WeakMap();
8
8
  }
9
+ if (!(existing.lostDevices instanceof WeakSet)) {
10
+ existing.lostDevices = new WeakSet();
11
+ }
9
12
  if (!('deviceInitPromise' in existing)) {
10
13
  existing.deviceInitPromise = null;
11
14
  }
@@ -19,6 +22,7 @@ export function getSharedDeviceState() {
19
22
  platformInitialized: false,
20
23
  deviceEpoch: 0,
21
24
  bufferOwners: new WeakMap(),
25
+ lostDevices: new WeakSet(),
22
26
  deviceInitPromise: null,
23
27
  };
24
28
  Object.defineProperty(globalThis, SHARED_DEVICE_STATE_KEY, {
@@ -34,3 +38,7 @@ export function getSharedDeviceEpoch() {
34
38
  const epoch = getSharedDeviceState().deviceEpoch;
35
39
  return Number.isInteger(epoch) ? epoch : 0;
36
40
  }
41
+
42
+ export function isDeviceLost(device) {
43
+ return device != null && getSharedDeviceState().lostDevices.has(device);
44
+ }
package/src/gpu/device.js CHANGED
@@ -5,7 +5,7 @@ import { probeSubmitLatency } from './submit-probe.js';
5
5
  import { log } from '../debug/index.js';
6
6
  import { createDopplerError, ERROR_CODES } from '../errors/doppler-error.js';
7
7
  import { GB } from '../config/schema/index.js';
8
- import { getSharedDeviceState } from './device-state.js';
8
+ import { getSharedDeviceState, isDeviceLost } from './device-state.js';
9
9
 
10
10
  // Re-export submit tracker for convenience
11
11
  export { setTrackSubmits };
@@ -143,6 +143,7 @@ function isValidGPUBuffer(value) {
143
143
  function isUsableGPUDevice(device) {
144
144
  return !!(
145
145
  device
146
+ && !isDeviceLost(device)
146
147
  && typeof device.createBuffer === 'function'
147
148
  && typeof device.createBindGroup === 'function'
148
149
  && typeof device.createCommandEncoder === 'function'
@@ -255,6 +256,7 @@ function registerDeviceLostHandler(device) {
255
256
  if (device.lost && typeof device.lost.then === 'function') {
256
257
  const trackedDevice = device;
257
258
  device.lost.then((info) => {
259
+ sharedDeviceState.lostDevices.add(trackedDevice);
258
260
  if (sharedDeviceState.gpuDevice !== trackedDevice) {
259
261
  return;
260
262
  }
@@ -270,6 +272,7 @@ function registerDeviceLostHandler(device) {
270
272
  clearActiveDeviceState();
271
273
  advanceDeviceEpoch();
272
274
  }).catch((error) => {
275
+ sharedDeviceState.lostDevices.add(trackedDevice);
273
276
  if (sharedDeviceState.gpuDevice !== trackedDevice) {
274
277
  return;
275
278
  }
@@ -600,6 +603,7 @@ export async function initDevice() {
600
603
 
601
604
  export function setDevice(device, options = {}) {
602
605
  hydrateDeviceState();
606
+ if (isDeviceLost(device)) throw new Error('Cannot bind a lost GPU device. Explicitly initialize a replacement device.');
603
607
  if (!device) {
604
608
  clearActiveDeviceState();
605
609
  advanceDeviceEpoch();
@@ -694,6 +698,7 @@ export function resetDeviceState() {
694
698
  export function destroyDevice() {
695
699
  hydrateDeviceState();
696
700
  if (gpuDevice) {
701
+ sharedDeviceState.lostDevices.add(gpuDevice);
697
702
  gpuDevice.destroy();
698
703
  clearActiveDeviceState();
699
704
  advanceDeviceEpoch();
@@ -7,11 +7,15 @@
7
7
  import type { Tensor } from '../tensor.js';
8
8
  import type { CommandRecorder } from '../command-recorder.js';
9
9
  import type { OutputBufferOptions } from './types.js';
10
+ import type { KernelPathSchema } from '../../config/schema/kernel-path.schema.js';
10
11
 
11
12
  /** GeLU kernel options */
12
13
  export interface GeLUOptions extends OutputBufferOptions {
13
14
  size?: number | null;
14
15
  gate?: Tensor | null;
16
+ kernelPath?: KernelPathSchema | null;
17
+ phase?: 'prefill' | 'decode';
18
+ layerIdx?: number;
15
19
  }
16
20
 
17
21
  /**
@@ -4,6 +4,7 @@ import { createTensor, dtypeBytes } from '../tensor.js';
4
4
  import { WORKGROUP_SIZES } from './constants.js';
5
5
  import { unifiedKernelWrapper } from './kernel-execution.js';
6
6
  import { selectRuleValue } from './rule-registry.js';
7
+ import { getKernelPathActivationSpec } from '../../config/kernel-path-loader.js';
7
8
 
8
9
  function selectGeluVariant(isF16) {
9
10
  return selectRuleValue('gelu', 'variant', { isF16 });
@@ -20,7 +21,16 @@ async function _gelu(target, input, options = {}) {
20
21
  const isF16 = input.dtype === 'f16';
21
22
  const bytesPerElement = dtypeBytes(input.dtype);
22
23
  const variant = selectGeluVariant(isF16);
23
- const overrides = resolveOverrides({ hasGate: Boolean(gate), useRowsplit: false });
24
+ const declared = getKernelPathActivationSpec('gelu', options.phase, options.layerIdx, options.kernelPath);
25
+ if (declared && declared.variant !== variant) {
26
+ throw new Error(`GeLU input ${input.dtype} does not match declared kernel ${declared.variant}.`);
27
+ }
28
+ for (const [key, actual] of Object.entries({ HAS_GATE: Boolean(gate), USE_ROWSPLIT: false })) {
29
+ if (declared?.constants?.[key] != null && Number(declared.constants[key]) !== Number(actual)) {
30
+ throw new Error(`GeLU ${key} disagrees with the declared execution step.`);
31
+ }
32
+ }
33
+ const overrides = { ...resolveOverrides({ hasGate: Boolean(gate), useRowsplit: false }), ...declared?.constants };
24
34
 
25
35
  const inferredSize = size || (input.buffer.size / bytesPerElement);
26
36
  const outputSize = inferredSize * bytesPerElement;
@@ -8,6 +8,7 @@
8
8
  override WORKGROUP_SIZE: u32 = 256u;
9
9
  override HAS_GATE: bool = false;
10
10
  override USE_ROWSPLIT: bool = false;
11
+ override GELU_ERF: bool = false;
11
12
 
12
13
  struct Uniforms {
13
14
  size: u32, // Total output elements
@@ -21,7 +22,22 @@ struct Uniforms {
21
22
  @group(0) @binding(2) var<storage, read_write> output: array<f32>;
22
23
  @group(0) @binding(3) var<storage, read> gate: array<f32>;
23
24
 
25
+ // Source erf GELU, evaluated in f32 even for f16 storage. The polynomial
26
+ // approximates erf rather than substituting the distinct tanh GELU function.
27
+ fn erf_gelu(x: f32) -> f32 {
28
+ let z = abs(x) * 0.7071067811865476;
29
+ let t = 1.0 / (1.0 + 0.3275911 * z);
30
+ let polynomial = (((((1.061405429 * t - 1.453152027) * t)
31
+ + 1.421413741) * t - 0.284496736) * t + 0.254829592) * t;
32
+ let complement = polynomial * exp(-z * z);
33
+ let cdf_twice = select(complement, 2.0 - complement, x >= 0.0);
34
+ return 0.5 * x * cdf_twice;
35
+ }
36
+
24
37
  fn gelu(x: f32) -> f32 {
38
+ if (GELU_ERF) {
39
+ return erf_gelu(x);
40
+ }
25
41
  let sqrt_2_over_pi: f32 = 0.7978845608;
26
42
  let c: f32 = 0.044715;
27
43
  let inner = sqrt_2_over_pi * (x + c * x * x * x);
@@ -9,6 +9,7 @@ enable f16;
9
9
  override WORKGROUP_SIZE: u32 = 256u;
10
10
  override HAS_GATE: bool = false;
11
11
  override USE_ROWSPLIT: bool = false;
12
+ override GELU_ERF: bool = false;
12
13
 
13
14
  struct Uniforms {
14
15
  size: u32, // Total output elements
@@ -22,7 +23,22 @@ struct Uniforms {
22
23
  @group(0) @binding(2) var<storage, read_write> output: array<f16>;
23
24
  @group(0) @binding(3) var<storage, read> gate: array<f16>;
24
25
 
26
+ // Source erf GELU, evaluated in f32 even for f16 storage. The polynomial
27
+ // approximates erf rather than substituting the distinct tanh GELU function.
28
+ fn erf_gelu(x: f32) -> f32 {
29
+ let z = abs(x) * 0.7071067811865476;
30
+ let t = 1.0 / (1.0 + 0.3275911 * z);
31
+ let polynomial = (((((1.061405429 * t - 1.453152027) * t)
32
+ + 1.421413741) * t - 0.284496736) * t + 0.254829592) * t;
33
+ let complement = polynomial * exp(-z * z);
34
+ let cdf_twice = select(complement, 2.0 - complement, x >= 0.0);
35
+ return 0.5 * x * cdf_twice;
36
+ }
37
+
25
38
  fn gelu(x: f16) -> f16 {
39
+ if (GELU_ERF) {
40
+ return f16(erf_gelu(f32(x)));
41
+ }
26
42
  let sqrt_2_over_pi = f16(0.7978845608);
27
43
  let c = f16(0.044715);
28
44
  let inner = sqrt_2_over_pi * (x + c * x * x * x);
@@ -3,6 +3,24 @@ import { createPipeline, getOrCreateBindGroupLayout } from './pipeline-cache.js'
3
3
  import { createUniformBufferWithView } from './uniform-utils.js';
4
4
  import { WORKGROUP_SIZES } from './constants.js';
5
5
  import { selectRuleValue } from './rule-registry.js';
6
+ import { validateGenerationField } from '../../config/generation-contract.js';
7
+ import { acquireBuffer, releaseBuffer } from '../../memory/buffer-pool.js';
8
+
9
+ export async function recordHistoryPenalties(recorder, logitsBuffer, tokenIds, options) {
10
+ if (options.repetitionPenalty === 1 && options.presencePenalty === 0) return;
11
+ const history = Uint32Array.from(tokenIds.slice(-options.repetitionPenaltyWindow));
12
+ const buffer = acquireBuffer(Math.max(4, history.byteLength), undefined, 'sampling_penalty_history');
13
+ let transferred = false;
14
+ try {
15
+ recorder.trackTemporaryBuffer(buffer);
16
+ transferred = true;
17
+ if (history.byteLength) recorder.device.queue.writeBuffer(buffer, 0, history);
18
+ await recordRepPenalty(recorder, logitsBuffer, buffer, buffer, {
19
+ ...options, penalty: options.repetitionPenalty, historyCount: history.length,
20
+ batchCount: 0, batchOffset: 0,
21
+ });
22
+ } finally { if (!transferred) releaseBuffer(buffer); }
23
+ }
6
24
 
7
25
 
8
26
  function getRepPenaltyBindGroupLayout(device) {
@@ -32,8 +50,8 @@ function createRepPenaltyUniformBuffer(device, recorder, options) {
32
50
  view.setFloat32(8, options.penalty, true);
33
51
  view.setUint32(12, options.batchCount, true);
34
52
  view.setUint32(16, options.batchOffset, true);
35
- view.setUint32(20, 0, true);
36
- view.setUint32(24, 0, true);
53
+ view.setFloat32(20, options.presencePenalty, true);
54
+ view.setUint32(24, options.repetitionPenaltyWindow, true);
37
55
  view.setUint32(28, 0, true);
38
56
  },
39
57
  recorder,
@@ -52,22 +70,27 @@ export async function recordRepPenalty(
52
70
  vocabSize,
53
71
  historyCount,
54
72
  penalty,
73
+ presencePenalty,
74
+ repetitionPenaltyWindow,
55
75
  batchCount,
56
76
  batchOffset,
57
77
  logitsDtype,
58
78
  } = options;
59
79
 
60
- if (!Number.isFinite(batchCount) || batchCount < 0) {
80
+ validateGenerationField('repetitionPenalty', penalty);
81
+ validateGenerationField('presencePenalty', presencePenalty);
82
+ validateGenerationField('repetitionPenaltyWindow', repetitionPenaltyWindow);
83
+ if (!Number.isSafeInteger(batchCount) || batchCount < 0) {
61
84
  throw new Error('[RepPenalty] batchCount is required and must be non-negative.');
62
85
  }
63
- if (!Number.isFinite(batchOffset) || batchOffset < 0) {
86
+ if (!Number.isSafeInteger(batchOffset) || batchOffset < 0) {
64
87
  throw new Error('[RepPenalty] batchOffset is required and must be non-negative.');
65
88
  }
66
89
  if (logitsDtype !== 'f16' && logitsDtype !== 'f32') {
67
90
  throw new Error('[RepPenalty] logitsDtype must be "f16" or "f32".');
68
91
  }
69
92
 
70
- if (penalty === 1.0 || (historyCount === 0 && batchCount === 0)) {
93
+ if ((penalty === 1.0 && presencePenalty === 0) || (historyCount === 0 && batchCount === 0)) {
71
94
  return;
72
95
  }
73
96
 
@@ -85,6 +108,8 @@ export async function recordRepPenalty(
85
108
  vocabSize,
86
109
  historyCount,
87
110
  penalty,
111
+ presencePenalty,
112
+ repetitionPenaltyWindow,
88
113
  batchCount,
89
114
  batchOffset,
90
115
  });
@@ -1,14 +1,5 @@
1
- // Repetition Penalty Kernel
2
- //
3
- // Applies repetition penalty to logits in-place before sampling.
4
- // Reads unique history token IDs (pre-deduplicated on CPU) and
5
- // current-batch tokens from separate buffers.
6
- //
7
- // For each token ID: logit > 0 ? logit / penalty : logit * penalty
8
- // Matches CPU applyRepetitionPenalty() behavior.
9
- //
10
- // One thread per token in (history + batch). Benign data race when the same
11
- // token appears in both buffers: both threads compute identical results.
1
+ // Repetition followed by presence penalty over the ordered recent context.
2
+ // Only the first occurrence inside the window writes a token's logit.
12
3
 
13
4
  override WORKGROUP_SIZE: u32 = 256u;
14
5
 
@@ -17,9 +8,9 @@ struct Uniforms {
17
8
  history_count: u32,
18
9
  penalty: f32,
19
10
  batch_count: u32,
20
- batch_offset: u32, // Offset into batch_tokens (skip startToken at index 0)
21
- _pad0: u32,
22
- _pad1: u32,
11
+ batch_offset: u32,
12
+ presence_penalty: f32,
13
+ repetition_penalty_window: u32,
23
14
  _pad2: u32,
24
15
  }
25
16
 
@@ -28,30 +19,34 @@ struct Uniforms {
28
19
  @group(0) @binding(2) var<storage, read> history: array<u32>;
29
20
  @group(0) @binding(3) var<storage, read> batch_tokens: array<u32>;
30
21
 
22
+ fn context_token(idx: u32) -> u32 {
23
+ if (idx < u.history_count) {
24
+ return history[idx];
25
+ }
26
+ return batch_tokens[u.batch_offset + idx - u.history_count];
27
+ }
28
+
31
29
  @compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
32
30
  fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
33
31
  let idx = gid.x;
34
32
  let total = u.history_count + u.batch_count;
35
- if (idx >= total) {
36
- return;
33
+ var start = 0u;
34
+ if (u.repetition_penalty_window > 0u) {
35
+ start = total - min(total, u.repetition_penalty_window);
37
36
  }
38
-
39
- var token_id: u32;
40
- if (idx < u.history_count) {
41
- token_id = history[idx];
42
- } else {
43
- token_id = batch_tokens[u.batch_offset + (idx - u.history_count)];
37
+ if (idx < start || idx >= total) {
38
+ return;
44
39
  }
45
-
40
+ let token_id = context_token(idx);
46
41
  if (token_id >= u.vocab_size) {
47
42
  return;
48
43
  }
49
-
50
- let penalty = u.penalty;
51
- let logit = logits[token_id];
52
- if (logit > 0.0) {
53
- logits[token_id] = logit / penalty;
54
- } else {
55
- logits[token_id] = logit * penalty;
44
+ for (var prior = start; prior < idx; prior++) {
45
+ if (context_token(prior) == token_id) {
46
+ return;
47
+ }
56
48
  }
49
+ let logit = f32(logits[token_id]);
50
+ let repeated = select(logit * u.penalty, logit / u.penalty, logit > 0.0);
51
+ logits[token_id] = f32(repeated - u.presence_penalty);
57
52
  }
@@ -1,10 +1,5 @@
1
- // Repetition Penalty Kernel (F16 Logits Variant)
2
- //
3
- // F16 variant for models with f16 activations. Reads/writes f16 logits,
4
- // computes penalty arithmetic in f32 for precision.
5
- //
6
- // For each token ID: logit > 0 ? logit / penalty : logit * penalty
7
-
1
+ // Repetition followed by presence penalty over the ordered recent context.
2
+ // Only the first occurrence inside the window writes a token's logit.
8
3
  enable f16;
9
4
 
10
5
  override WORKGROUP_SIZE: u32 = 256u;
@@ -15,8 +10,8 @@ struct Uniforms {
15
10
  penalty: f32,
16
11
  batch_count: u32,
17
12
  batch_offset: u32,
18
- _pad0: u32,
19
- _pad1: u32,
13
+ presence_penalty: f32,
14
+ repetition_penalty_window: u32,
20
15
  _pad2: u32,
21
16
  }
22
17
 
@@ -25,30 +20,34 @@ struct Uniforms {
25
20
  @group(0) @binding(2) var<storage, read> history: array<u32>;
26
21
  @group(0) @binding(3) var<storage, read> batch_tokens: array<u32>;
27
22
 
23
+ fn context_token(idx: u32) -> u32 {
24
+ if (idx < u.history_count) {
25
+ return history[idx];
26
+ }
27
+ return batch_tokens[u.batch_offset + idx - u.history_count];
28
+ }
29
+
28
30
  @compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
29
31
  fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
30
32
  let idx = gid.x;
31
33
  let total = u.history_count + u.batch_count;
32
- if (idx >= total) {
33
- return;
34
+ var start = 0u;
35
+ if (u.repetition_penalty_window > 0u) {
36
+ start = total - min(total, u.repetition_penalty_window);
34
37
  }
35
-
36
- var token_id: u32;
37
- if (idx < u.history_count) {
38
- token_id = history[idx];
39
- } else {
40
- token_id = batch_tokens[u.batch_offset + (idx - u.history_count)];
38
+ if (idx < start || idx >= total) {
39
+ return;
41
40
  }
42
-
41
+ let token_id = context_token(idx);
43
42
  if (token_id >= u.vocab_size) {
44
43
  return;
45
44
  }
46
-
47
- let penalty = u.penalty;
48
- let logit = f32(logits[token_id]);
49
- if (logit > 0.0) {
50
- logits[token_id] = f16(logit / penalty);
51
- } else {
52
- logits[token_id] = f16(logit * penalty);
45
+ for (var prior = start; prior < idx; prior++) {
46
+ if (context_token(prior) == token_id) {
47
+ return;
48
+ }
53
49
  }
50
+ let logit = f32(logits[token_id]);
51
+ let repeated = select(logit * u.penalty, logit / u.penalty, logit > 0.0);
52
+ logits[token_id] = f16(repeated - u.presence_penalty);
54
53
  }