doppler-gpu 0.6.0 → 0.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. package/package.json +8 -2
  2. package/src/capsule-runtime.d.ts +2 -0
  3. package/src/capsule-runtime.js +1 -0
  4. package/src/client/capsule-host.browser.js +1 -0
  5. package/src/client/capsule-host.d.ts +2 -0
  6. package/src/client/capsule-host.js +1 -0
  7. package/src/client/doppler-api.d.ts +2 -0
  8. package/src/client/doppler-api.js +1 -0
  9. package/src/client/model-host/model-session.d.ts +1 -0
  10. package/src/client/model-host/model-session.js +12 -0
  11. package/src/client/runtime/capsule-operation-adapters.js +15 -14
  12. package/src/client/runtime/capsule-operation-executor.js +3 -2
  13. package/src/client/runtime/capsule-program-adapter.js +2 -0
  14. package/src/client/runtime/capsule-session-execution.js +9 -5
  15. package/src/client/runtime/composition-root.d.ts +6 -3
  16. package/src/client/runtime/composition-root.js +13 -5
  17. package/src/client/runtime/session-controller.d.ts +6 -17
  18. package/src/client/runtime/session-controller.js +42 -120
  19. package/src/client/runtime/verified-capsule-artifact-store.js +38 -9
  20. package/src/config/capsule-loading.d.ts +1 -1
  21. package/src/config/capsule-loading.schema.json +3 -2
  22. package/src/config/capsule-operation.js +6 -1
  23. package/src/config/capsule-operations.json +1 -2
  24. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +3 -3
  25. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +1 -1
  26. package/src/config/conversion/esm/esm2-t12-35m-ur50d-f32-af32.json +32 -14
  27. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +2 -2
  28. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +2 -2
  29. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +2 -2
  30. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +2 -2
  31. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +2 -2
  32. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +2 -2
  33. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +2 -2
  34. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +3 -3
  35. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +3 -3
  36. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +5 -5
  37. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +2 -2
  38. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +3 -3
  39. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +3 -3
  40. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +3 -3
  41. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +2 -2
  42. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +2 -2
  43. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +2 -2
  44. package/src/config/conversion/glmocr/glm-ocr-f16-af32.json +1 -1
  45. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +2 -2
  46. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +2 -2
  47. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +2 -2
  48. package/src/config/conversion/minicpm/minicpm4-0-5b-f16-af32.json +2 -2
  49. package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-bf16-af32.json +1 -1
  50. package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-f16-af32.json +1 -1
  51. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +5 -5
  52. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +5 -5
  53. package/src/config/conversion/qwen3/qwen-3-5-9b-q4k-ehaf16.json +5 -5
  54. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +5 -5
  55. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +5 -5
  56. package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-q4k-ehf16-af32.json +2 -2
  57. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-af32.json +2 -2
  58. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-true-logit-af32.json +1 -1
  59. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +1 -1
  60. package/src/config/conversion/qwen3/qwen3.8-27b-text-f16-af32.json +3 -3
  61. package/src/config/conversion/qwen3/qwen3.8-27b-text-q4k-ehaf16.json +3 -3
  62. package/src/config/generation-contract.d.ts +39 -0
  63. package/src/config/generation-contract.js +66 -0
  64. package/src/config/generation-contract.json +32 -0
  65. package/src/config/kernel-path-loader.d.ts +7 -0
  66. package/src/config/kernel-path-loader.js +19 -0
  67. package/src/config/kernels/kernel-ref-digests.js +16 -16
  68. package/src/config/kernels/registry.json +35 -10
  69. package/src/config/schema/doppler.schema.js +1 -0
  70. package/src/config/source-packages/litert/gemma-4-e2b-it.json +2 -2
  71. package/src/converter/forge-qualification.js +2 -1
  72. package/src/converter/forge-stages.js +6 -4
  73. package/src/converter/quantizer.js +18 -4
  74. package/src/gpu/device-state.js +8 -0
  75. package/src/gpu/device.js +6 -1
  76. package/src/gpu/kernels/gelu.d.ts +4 -0
  77. package/src/gpu/kernels/gelu.js +11 -1
  78. package/src/gpu/kernels/gelu.wgsl +16 -0
  79. package/src/gpu/kernels/gelu_f16.wgsl +16 -0
  80. package/src/gpu/kernels/rep-penalty.js +30 -5
  81. package/src/gpu/kernels/rep_penalty.wgsl +25 -30
  82. package/src/gpu/kernels/rep_penalty_f16.wgsl +24 -25
  83. package/src/gpu/kernels/sample.d.ts +1 -0
  84. package/src/gpu/kernels/sample.js +31 -16
  85. package/src/gpu/kernels/sample.wgsl +112 -157
  86. package/src/gpu/kernels/sample_f16.wgsl +168 -162
  87. package/src/index.d.ts +2 -0
  88. package/src/index.js +1 -0
  89. package/src/inference/pipelines/context.js +6 -2
  90. package/src/inference/pipelines/structured/json-grammar-mask.d.ts +2 -0
  91. package/src/inference/pipelines/structured/json-grammar-mask.js +115 -118
  92. package/src/inference/pipelines/text/ffn/dense-executor.js +5 -0
  93. package/src/inference/pipelines/text/ffn/dense.js +2 -2
  94. package/src/inference/pipelines/text/generator/decode-runtime.js +8 -14
  95. package/src/inference/pipelines/text/generator/decode.js +18 -6
  96. package/src/inference/pipelines/text/generator/training.js +3 -2
  97. package/src/inference/pipelines/text/generator-logits-observation.js +5 -2
  98. package/src/inference/pipelines/text/generator-runtime.js +18 -19
  99. package/src/inference/pipelines/text/generator-steps.js +15 -8
  100. package/src/inference/pipelines/text/generator.js +16 -10
  101. package/src/inference/pipelines/text/linear-attention.js +24 -0
  102. package/src/inference/pipelines/text/logits/output-transform.js +10 -3
  103. package/src/inference/pipelines/text/sampling-config.js +22 -113
  104. package/src/inference/pipelines/text/sampling.d.ts +9 -1
  105. package/src/inference/pipelines/text/sampling.js +18 -375
  106. package/src/inference/pipelines/text/types.d.ts +3 -3
  107. package/src/inference/token-sampling.js +319 -0
  108. package/src/storage/backends/opfs-store.d.ts +41 -0
  109. package/src/tooling/node-browser/static-server.js +1 -0
  110. package/src/tooling-exports/storage.d.ts +2 -0
  111. package/src/tooling-exports/storage.js +1 -0
  112. package/src/version.js +1 -1
  113. package/tests/kernels/browser/test-page.js +1 -0
@@ -1,41 +1,50 @@
1
1
  // AUTO-GENERATED from src/gpu/kernels/sample.wgsl.
2
2
  // Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
3
- // sample_f16.wgsl
3
+ // sample_f16.wgsl: F16 storage, F32 sampling accumulation.
4
+
5
+ enable f16;
4
6
 
5
7
  /**
6
- * GPU-Side Sampling Kernel (f16 logits)
8
+ * GPU-Side Sampling Kernel
9
+ *
10
+ * Performs temperature scaling, top-k selection, softmax, and sampling
11
+ * entirely on GPU. Only reads back the single selected token ID.
7
12
  *
8
- * Same as sample.wgsl but logits are f16.
9
- * The all-f16 lane keeps sampling comparisons and reductions in f16.
13
+ * Reduces readback from 1MB (256K vocab × 4 bytes) to 4 bytes.
14
+ *
15
+ * Algorithm:
16
+ * 1. Temperature scaling: logits = logits / temperature
17
+ * 2. Parallel top-k: Each workgroup finds local top-k, then merge
18
+ * 3. Softmax on top-k candidates
19
+ * 4. Multinomial sampling with provided random value
10
20
  */
11
21
 
12
- enable f16;
13
-
14
22
  // Configuration
15
23
  override WORKGROUP_SIZE: u32 = 256u;
16
24
  const MAX_WORKGROUP_SIZE: u32 = 256u;
17
- const MAX_TOP_K: u32 = 128u;
18
- const NEG_INF = f16(-65504.0);
25
+ const NEG_INF: f32 = -3.402823e+38;
19
26
 
20
27
  struct Uniforms {
21
28
  vocab_size: u32,
22
29
  top_k: u32,
23
30
  temperature: f32,
24
- random_value: f32,
31
+ random_value: f32, // Pre-generated random [0,1) for sampling
25
32
  pad_token_id: u32,
26
- logit_softcap: f32,
27
- output_index: u32,
28
- pad0: u32,
33
+ logit_softcap: f32, // Gemma 2: 30.0, 0.0 = disabled
34
+ output_index: u32, // Index into output token buffer
35
+ top_p: f32,
29
36
  }
30
37
 
31
- fn apply_softcap(x: f16, softcap: f16) -> f16 {
32
- if (softcap <= f16(0.0)) {
38
+ // Apply softcapping: softcap * tanh(x / softcap)
39
+ // Returns x unchanged if softcap <= 0
40
+ fn apply_softcap(x: f32, softcap: f32) -> f32 {
41
+ if (softcap <= 0.0) {
33
42
  return x;
34
43
  }
35
44
  return softcap * tanh(x / softcap);
36
45
  }
37
46
 
38
- fn candidate_beats(candidate_value: f16, candidate_index: u32, best_value: f16, best_index: u32) -> bool {
47
+ fn candidate_beats(candidate_value: f32, candidate_index: u32, best_value: f32, best_index: u32) -> bool {
39
48
  if (candidate_value > best_value) {
40
49
  return true;
41
50
  }
@@ -46,173 +55,153 @@ fn candidate_beats(candidate_value: f16, candidate_index: u32, best_value: f16,
46
55
  }
47
56
 
48
57
  @group(0) @binding(0) var<uniform> u: Uniforms;
49
- @group(0) @binding(1) var<storage, read> logits: array<f16>;
50
- @group(0) @binding(2) var<storage, read_write> output: array<u32>;
51
- @group(0) @binding(3) var<storage, read_write> topk_indices: array<u32>;
52
- @group(0) @binding(4) var<storage, read_write> topk_logits: array<f16>;
58
+ @group(0) @binding(1) var<storage, read> logits: array<f16>; // [vocabSize]
59
+ @group(0) @binding(2) var<storage, read_write> output: array<u32>; // [N] - selected tokens
60
+ @group(0) @binding(3) var<storage, read_write> topk_indices: array<u32>; // [topK] - intermediate
61
+ @group(0) @binding(4) var<storage, read_write> topk_logits: array<f32>; // [topK] - intermediate
53
62
 
54
- var<workgroup> shared_values: array<f16, MAX_WORKGROUP_SIZE>;
63
+ // Shared memory for workgroup-level reduction
64
+ var<workgroup> shared_values: array<f32, MAX_WORKGROUP_SIZE>;
55
65
  var<workgroup> shared_indices: array<u32, MAX_WORKGROUP_SIZE>;
56
66
 
57
- @compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
58
- fn find_topk_phase1(
59
- @builtin(global_invocation_id) gid: vec3<u32>,
60
- @builtin(local_invocation_id) lid: vec3<u32>,
61
- @builtin(workgroup_id) wgid: vec3<u32>,
62
- @builtin(num_workgroups) num_wg: vec3<u32>
63
- ) {
64
- let thread_idx = lid.x;
65
- let global_idx = gid.x;
66
- let vocab_size = u.vocab_size;
67
- let temperature = f16(u.temperature);
68
- let pad_id = u.pad_token_id;
69
- let softcap = f16(u.logit_softcap);
67
+ // Each partition retains its exact top-k in a min-heap. The merge has a
68
+ // separate output region, so retaining a winner cannot overwrite unread input.
69
+ // Scratch holds at most two vocabularies; top_k is resolved to [1, vocab_size].
70
+ fn sample_group_count() -> u32 {
71
+ return max(1u, min(min(WORKGROUP_SIZE, (u.vocab_size + WORKGROUP_SIZE - 1u) / WORKGROUP_SIZE),
72
+ u.vocab_size / u.top_k));
73
+ }
70
74
 
71
- if (num_wg.x == 1u) {
72
- var val: f16 = NEG_INF;
73
- if (global_idx < vocab_size && global_idx != pad_id) {
74
- val = apply_softcap(logits[global_idx], softcap) / temperature;
75
- }
76
- topk_logits[thread_idx] = val;
77
- topk_indices[thread_idx] = global_idx;
78
- return;
79
- }
75
+ fn heap_swap(base: u32, a: u32, b: u32) {
76
+ let value = topk_logits[base + a];
77
+ let token = topk_indices[base + a];
78
+ topk_logits[base + a] = topk_logits[base + b];
79
+ topk_indices[base + a] = topk_indices[base + b];
80
+ topk_logits[base + b] = value;
81
+ topk_indices[base + b] = token;
82
+ }
80
83
 
81
- var local_max: f16 = NEG_INF;
82
- var local_max_idx: u32 = 0u;
84
+ fn heap_better(base: u32, a: u32, b: u32) -> bool {
85
+ return candidate_beats(topk_logits[base + a], topk_indices[base + a],
86
+ topk_logits[base + b], topk_indices[base + b]);
87
+ }
83
88
 
84
- var idx = global_idx;
85
- while (idx < vocab_size) {
86
- if (idx != pad_id) {
87
- let val = apply_softcap(logits[idx], softcap) / temperature;
88
- if (candidate_beats(val, idx, local_max, local_max_idx)) {
89
- local_max = val;
90
- local_max_idx = idx;
91
- }
92
- }
93
- idx = idx + WORKGROUP_SIZE * num_wg.x;
89
+ fn heap_down(base: u32, count: u32) {
90
+ var root = 0u;
91
+ loop {
92
+ let left = root * 2u + 1u;
93
+ if (left >= count) { break; }
94
+ var worst = left;
95
+ let right = left + 1u;
96
+ if (right < count && heap_better(base, left, right)) { worst = right; }
97
+ if (!heap_better(base, root, worst)) { break; }
98
+ heap_swap(base, root, worst);
99
+ root = worst;
94
100
  }
101
+ }
95
102
 
96
- shared_values[thread_idx] = local_max;
97
- shared_indices[thread_idx] = local_max_idx;
98
- workgroupBarrier();
99
-
100
- var stride = WORKGROUP_SIZE / 2u;
101
- while (stride > 0u) {
102
- if (thread_idx < stride) {
103
- if (candidate_beats(
104
- shared_values[thread_idx + stride],
105
- shared_indices[thread_idx + stride],
106
- shared_values[thread_idx],
107
- shared_indices[thread_idx]
108
- )) {
109
- shared_values[thread_idx] = shared_values[thread_idx + stride];
110
- shared_indices[thread_idx] = shared_indices[thread_idx + stride];
111
- }
103
+ fn heap_offer(base: u32, count: u32, value: f32, token: u32) -> u32 {
104
+ if (count < u.top_k) {
105
+ topk_logits[base + count] = value;
106
+ topk_indices[base + count] = token;
107
+ var child = count;
108
+ loop {
109
+ if (child == 0u) { break; }
110
+ let parent = (child - 1u) / 2u;
111
+ if (!heap_better(base, parent, child)) { break; }
112
+ heap_swap(base, parent, child);
113
+ child = parent;
112
114
  }
113
- workgroupBarrier();
114
- stride = stride / 2u;
115
+ return count + 1u;
115
116
  }
116
-
117
- if (thread_idx == 0u) {
118
- topk_logits[wgid.x] = shared_values[0];
119
- topk_indices[wgid.x] = shared_indices[0];
117
+ if (candidate_beats(value, token, topk_logits[base], topk_indices[base])) {
118
+ topk_logits[base] = value;
119
+ topk_indices[base] = token;
120
+ heap_down(base, count);
120
121
  }
122
+ return count;
121
123
  }
122
124
 
123
125
  @compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
124
- fn find_topk_phase2(
125
- @builtin(local_invocation_id) lid: vec3<u32>
126
+ fn find_topk_phase1(
127
+ @builtin(local_invocation_id) lid: vec3<u32>,
128
+ @builtin(workgroup_id) wgid: vec3<u32>,
129
+ @builtin(num_workgroups) num_wg: vec3<u32>
126
130
  ) {
127
- let thread_idx = lid.x;
128
- let top_k = u.top_k;
129
- let num_groups = min(WORKGROUP_SIZE, (u.vocab_size + WORKGROUP_SIZE - 1u) / WORKGROUP_SIZE);
130
- let num_candidates = select(num_groups, min(u.vocab_size, WORKGROUP_SIZE), num_groups == 1u);
131
-
132
- if (thread_idx < WORKGROUP_SIZE) {
133
- if (thread_idx < num_candidates) {
134
- shared_values[thread_idx] = topk_logits[thread_idx];
135
- shared_indices[thread_idx] = topk_indices[thread_idx];
136
- } else {
137
- shared_values[thread_idx] = NEG_INF;
138
- shared_indices[thread_idx] = 0u;
131
+ if (lid.x != 0u) { return; }
132
+ let base = wgid.x * u.top_k;
133
+ for (var k = 0u; k < u.top_k; k++) { topk_indices[base + k] = 0xffffffffu; }
134
+ var count = 0u;
135
+ for (var token = wgid.x; token < u.vocab_size; token += num_wg.x) {
136
+ let raw = f32(logits[token]);
137
+ // Non-finite and padding logits never enter the candidate distribution.
138
+ if (token != u.pad_token_id && abs(raw) <= 3.402823e+38) {
139
+ let value = apply_softcap(raw, u.logit_softcap);
140
+ count = heap_offer(base, count, value, token);
139
141
  }
140
142
  }
141
- workgroupBarrier();
142
-
143
- if (thread_idx == 0u) {
144
- for (var k: u32 = 0u; k < top_k && k < num_candidates; k = k + 1u) {
145
- var max_idx = k;
146
- var max_val = shared_values[k];
147
-
148
- for (var i: u32 = k + 1u; i < num_candidates; i = i + 1u) {
149
- if (candidate_beats(shared_values[i], shared_indices[i], max_val, shared_indices[max_idx])) {
150
- max_val = shared_values[i];
151
- max_idx = i;
152
- }
153
- }
154
-
155
- if (max_idx != k) {
156
- let tmp_val = shared_values[k];
157
- let tmp_idx = shared_indices[k];
158
- shared_values[k] = shared_values[max_idx];
159
- shared_indices[k] = shared_indices[max_idx];
160
- shared_values[max_idx] = tmp_val;
161
- shared_indices[max_idx] = tmp_idx;
162
- }
163
- }
143
+ }
164
144
 
165
- for (var k: u32 = 0u; k < top_k; k = k + 1u) {
166
- topk_logits[k] = shared_values[k];
167
- topk_indices[k] = shared_indices[k];
145
+ @compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
146
+ fn find_topk_phase2(@builtin(local_invocation_id) lid: vec3<u32>) {
147
+ if (lid.x != 0u) { return; }
148
+ let base = sample_group_count() * u.top_k;
149
+ for (var k = 0u; k < u.top_k; k++) { topk_indices[base + k] = 0xffffffffu; }
150
+ var count = 0u;
151
+ for (var i = 0u; i < base; i++) {
152
+ if (topk_indices[i] != 0xffffffffu) {
153
+ count = heap_offer(base, count, topk_logits[i], topk_indices[i]);
168
154
  }
169
155
  }
170
156
  }
171
157
 
158
+ // Sort the retained heap, normalize top-k, retain the minimal top-p prefix,
159
+ // then sample from that renormalized prefix. F16 inputs use F32 accumulation.
172
160
  @compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
173
- fn softmax_and_sample(
174
- @builtin(local_invocation_id) lid: vec3<u32>
175
- ) {
176
- let thread_idx = lid.x;
177
- let top_k = u.top_k;
178
- let random_val = f16(u.random_value);
179
-
180
- if (thread_idx < top_k) {
181
- shared_values[thread_idx] = topk_logits[thread_idx];
182
- shared_indices[thread_idx] = topk_indices[thread_idx];
161
+ fn softmax_and_sample(@builtin(local_invocation_id) lid: vec3<u32>) {
162
+ if (lid.x != 0u) { return; }
163
+ let base = sample_group_count() * u.top_k;
164
+ var count = 0u;
165
+ while (count < u.top_k && topk_indices[base + count] != 0xffffffffu) { count++; }
166
+ if (count == 0u) {
167
+ output[u.output_index] = 0xffffffffu;
168
+ return;
183
169
  }
184
- workgroupBarrier();
185
-
186
- if (thread_idx == 0u) {
187
- var max_val: f16 = shared_values[0];
188
- for (var i: u32 = 1u; i < top_k; i = i + 1u) {
189
- max_val = max(max_val, shared_values[i]);
190
- }
191
-
192
- var exp_sum: f16 = f16(0.0);
193
- for (var i: u32 = 0u; i < top_k; i = i + 1u) {
194
- let exp_val = exp(shared_values[i] - max_val);
195
- shared_values[i] = exp_val;
196
- exp_sum = exp_sum + exp_val;
197
- }
198
-
199
- let inv_sum = f16(1.0) / exp_sum;
200
- var cum_prob: f16 = f16(0.0);
201
- var selected_token: u32 = shared_indices[top_k - 1u];
202
-
203
- for (var i: u32 = 0u; i < top_k; i = i + 1u) {
204
- let prob = shared_values[i] * inv_sum;
205
- cum_prob = cum_prob + prob;
206
- if (cum_prob >= random_val) {
207
- selected_token = shared_indices[i];
208
- break;
209
- }
170
+ var remaining = count;
171
+ while (remaining > 1u) {
172
+ remaining--;
173
+ heap_swap(base, 0u, remaining);
174
+ heap_down(base, remaining);
175
+ }
176
+ let maximum = topk_logits[base];
177
+ var total = 0.0;
178
+ for (var i = 0u; i < count; i++) {
179
+ let weight = exp((topk_logits[base + i] - maximum) / u.temperature);
180
+ topk_logits[base + i] = weight;
181
+ total += weight;
182
+ }
183
+ var retained = 0u;
184
+ var retained_total = 0.0;
185
+ loop {
186
+ retained_total += topk_logits[base + retained];
187
+ retained++;
188
+ if (retained >= count || retained_total >= u.top_p * total) { break; }
189
+ }
190
+ let threshold = u.random_value * retained_total;
191
+ var cumulative = 0.0;
192
+ var selected = topk_indices[base + retained - 1u];
193
+ for (var i = 0u; i < retained; i++) {
194
+ cumulative += topk_logits[base + i];
195
+ if (cumulative >= threshold) {
196
+ selected = topk_indices[base + i];
197
+ break;
210
198
  }
211
-
212
- output[u.output_index] = selected_token;
213
199
  }
200
+ output[u.output_index] = selected;
214
201
  }
215
202
 
203
+ // Combined single-pass version for smaller vocabularies (<= 65536)
204
+ // Uses hierarchical reduction within single kernel
216
205
  @compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
217
206
  fn sample_single_pass(
218
207
  @builtin(global_invocation_id) gid: vec3<u32>,
@@ -221,18 +210,20 @@ fn sample_single_pass(
221
210
  ) {
222
211
  let thread_idx = lid.x;
223
212
  let vocab_size = u.vocab_size;
224
- let top_k = min(u.top_k, MAX_TOP_K);
225
- let temperature = f16(u.temperature);
213
+ let temperature = u.temperature;
214
+ let random_val = u.random_value;
226
215
  let pad_id = u.pad_token_id;
227
- let softcap = f16(u.logit_softcap);
216
+ let softcap = u.logit_softcap;
228
217
 
229
- var local_max: f16 = NEG_INF;
218
+ // Phase 1: Find global max
219
+ var local_max: f32 = NEG_INF;
230
220
  var local_max_idx: u32 = 0u;
231
221
 
232
222
  var idx = gid.x;
233
223
  while (idx < vocab_size) {
234
224
  if (idx != pad_id) {
235
- let val = apply_softcap(logits[idx], softcap) / temperature;
225
+ // Apply softcapping before temperature scaling
226
+ let val = apply_softcap(f32(logits[idx]), softcap) / temperature;
236
227
  if (candidate_beats(val, idx, local_max, local_max_idx)) {
237
228
  local_max = val;
238
229
  local_max_idx = idx;
@@ -245,6 +236,7 @@ fn sample_single_pass(
245
236
  shared_indices[thread_idx] = local_max_idx;
246
237
  workgroupBarrier();
247
238
 
239
+ // Reduce to find workgroup max
248
240
  var stride = WORKGROUP_SIZE / 2u;
249
241
  while (stride > 0u) {
250
242
  if (thread_idx < stride) {
@@ -262,11 +254,18 @@ fn sample_single_pass(
262
254
  stride = stride / 2u;
263
255
  }
264
256
 
257
+ // For single workgroup, thread 0 can do everything
265
258
  if (thread_idx == 0u && num_wg.x == 1u) {
259
+ // We have top-1, but need top-k
260
+ // For small vocab, just do the full selection
261
+ // This simplified version selects top-1 only (greedy)
262
+ // Full top-k sampling requires multi-pass for large vocab
263
+
266
264
  output[u.output_index] = shared_indices[0];
267
265
  }
268
266
  }
269
267
 
268
+ // Greedy argmax for deterministic decoding (temperature=0 equivalent)
270
269
  @compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
271
270
  fn argmax(
272
271
  @builtin(global_invocation_id) gid: vec3<u32>,
@@ -278,15 +277,17 @@ fn argmax(
278
277
  let global_idx = gid.x;
279
278
  let vocab_size = u.vocab_size;
280
279
  let pad_id = u.pad_token_id;
281
- let softcap = f16(u.logit_softcap);
280
+ let softcap = u.logit_softcap;
282
281
 
283
- var local_max: f16 = NEG_INF;
282
+ // Each thread finds max in its chunk
283
+ var local_max: f32 = NEG_INF;
284
284
  var local_max_idx: u32 = 0u;
285
285
 
286
286
  var idx = global_idx;
287
287
  while (idx < vocab_size) {
288
288
  if (idx != pad_id) {
289
- let val = apply_softcap(logits[idx], softcap);
289
+ // Apply softcapping (argmax is greedy, no temperature)
290
+ let val = apply_softcap(f32(logits[idx]), softcap);
290
291
  if (candidate_beats(val, idx, local_max, local_max_idx)) {
291
292
  local_max = val;
292
293
  local_max_idx = idx;
@@ -299,6 +300,7 @@ fn argmax(
299
300
  shared_indices[thread_idx] = local_max_idx;
300
301
  workgroupBarrier();
301
302
 
303
+ // Reduce within workgroup
302
304
  var stride = WORKGROUP_SIZE / 2u;
303
305
  while (stride > 0u) {
304
306
  if (thread_idx < stride) {
@@ -316,12 +318,14 @@ fn argmax(
316
318
  stride = stride / 2u;
317
319
  }
318
320
 
321
+ // Write workgroup result to global memory
319
322
  if (thread_idx == 0u) {
320
323
  topk_logits[wgid.x] = shared_values[0];
321
324
  topk_indices[wgid.x] = shared_indices[0];
322
325
  }
323
326
  }
324
327
 
328
+ // Final reduction for argmax across workgroups
325
329
  @compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
326
330
  fn argmax_reduce(
327
331
  @builtin(local_invocation_id) lid: vec3<u32>
@@ -329,6 +333,7 @@ fn argmax_reduce(
329
333
  let thread_idx = lid.x;
330
334
  let num_groups = min(WORKGROUP_SIZE, (u.vocab_size + WORKGROUP_SIZE - 1u) / WORKGROUP_SIZE);
331
335
 
336
+ // Load workgroup maxes (up to WORKGROUP_SIZE)
332
337
  if (thread_idx < num_groups) {
333
338
  shared_values[thread_idx] = topk_logits[thread_idx];
334
339
  shared_indices[thread_idx] = topk_indices[thread_idx];
@@ -338,6 +343,7 @@ fn argmax_reduce(
338
343
  }
339
344
  workgroupBarrier();
340
345
 
346
+ // Reduce
341
347
  var stride = WORKGROUP_SIZE / 2u;
342
348
  while (stride > 0u) {
343
349
  if (thread_idx < stride) {
package/src/index.d.ts CHANGED
@@ -1,4 +1,6 @@
1
1
  export { DOPPLER_VERSION } from './version.js';
2
+ export { GENERATION_CONTRACT, GenerationError, resolveGenerationOptions, validateGenerationInput } from './client/doppler-api.js';
3
+ export type { GenerationOptions, GenerationInput, GenerationOutput, GenerationCompletion, ResolvedGenerationOptions } from './client/doppler-api.js';
2
4
  export {
3
5
  doppler,
4
6
  doppler as dr,
package/src/index.js CHANGED
@@ -1,4 +1,5 @@
1
1
  export { DOPPLER_VERSION } from './version.js';
2
+ export { GENERATION_CONTRACT, GenerationError, resolveGenerationOptions, validateGenerationInput } from './client/doppler-api.js';
2
3
  export {
3
4
  doppler,
4
5
  doppler as dr,
@@ -4,6 +4,7 @@ import {
4
4
  getPlatformConfig,
5
5
  setDevice,
6
6
  } from '../../gpu/device.js';
7
+ import { isDeviceLost } from '../../gpu/device-state.js';
7
8
  import { applyDebugConfig, setGPUDevice } from '../../debug/index.js';
8
9
  import { getRuntimeConfig, setRuntimeConfig } from '../../config/runtime.js';
9
10
  import { kernelTrace } from './text/kernel-trace.js';
@@ -79,7 +80,7 @@ function restoreDebugState(snapshot) {
79
80
  }
80
81
  }
81
82
 
82
- setGPUDevice(snapshot.gpuDevice ?? null);
83
+ setGPUDevice(isDeviceLost(snapshot.gpuDevice) ? getDevice() : snapshot.gpuDevice ?? null);
83
84
  }
84
85
 
85
86
  export function restorePipelineContexts(target) {
@@ -183,7 +184,10 @@ export function applyPipelineContexts(target, contexts = {}, options = {}) {
183
184
  delete target[RESTORE_PIPELINE_CONTEXTS];
184
185
 
185
186
  setRuntimeConfig(previousRuntimeConfig);
186
- if (previousDevice) {
187
+ if (isDeviceLost(previousDevice)) {
188
+ // A closed scope cannot restore a destroyed device or displace its replacement.
189
+ if (isDeviceLost(getDevice())) setDevice(null);
190
+ } else if (previousDevice) {
187
191
  setDevice(previousDevice, {
188
192
  platformConfig: previousPlatformConfig,
189
193
  adapterInfo: previousAdapterInfo,
@@ -1,10 +1,12 @@
1
1
  export interface JsonGrammarMaskTokenizer {
2
2
  decode(ids: number[], skipSpecial?: boolean, skipBos?: boolean): string;
3
+ getSpecialTokens?(): { eos?: number | number[] };
3
4
  }
4
5
 
5
6
  export interface JsonGrammarMaskOptions {
6
7
  tokenizer?: JsonGrammarMaskTokenizer | null;
7
8
  cacheBudget?: number;
9
+ stopTokenIds?: number[];
8
10
  }
9
11
 
10
12
  export interface JsonGrammarMaskContext {