doppler-gpu 0.6.0 → 0.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (113) hide show
  1. package/package.json +8 -2
  2. package/src/capsule-runtime.d.ts +2 -0
  3. package/src/capsule-runtime.js +1 -0
  4. package/src/client/capsule-host.browser.js +1 -0
  5. package/src/client/capsule-host.d.ts +2 -0
  6. package/src/client/capsule-host.js +1 -0
  7. package/src/client/doppler-api.d.ts +2 -0
  8. package/src/client/doppler-api.js +1 -0
  9. package/src/client/model-host/model-session.d.ts +1 -0
  10. package/src/client/model-host/model-session.js +12 -0
  11. package/src/client/runtime/capsule-operation-adapters.js +15 -14
  12. package/src/client/runtime/capsule-operation-executor.js +3 -2
  13. package/src/client/runtime/capsule-program-adapter.js +2 -0
  14. package/src/client/runtime/capsule-session-execution.js +9 -5
  15. package/src/client/runtime/composition-root.d.ts +6 -3
  16. package/src/client/runtime/composition-root.js +13 -5
  17. package/src/client/runtime/session-controller.d.ts +6 -17
  18. package/src/client/runtime/session-controller.js +42 -120
  19. package/src/client/runtime/verified-capsule-artifact-store.js +38 -9
  20. package/src/config/capsule-loading.d.ts +1 -1
  21. package/src/config/capsule-loading.schema.json +3 -2
  22. package/src/config/capsule-operation.js +6 -1
  23. package/src/config/capsule-operations.json +1 -2
  24. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +3 -3
  25. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +1 -1
  26. package/src/config/conversion/esm/esm2-t12-35m-ur50d-f32-af32.json +32 -14
  27. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +2 -2
  28. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +2 -2
  29. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +2 -2
  30. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +2 -2
  31. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +2 -2
  32. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +2 -2
  33. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +2 -2
  34. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +3 -3
  35. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +3 -3
  36. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +5 -5
  37. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +2 -2
  38. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +3 -3
  39. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +3 -3
  40. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +3 -3
  41. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +2 -2
  42. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +2 -2
  43. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +2 -2
  44. package/src/config/conversion/glmocr/glm-ocr-f16-af32.json +1 -1
  45. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +2 -2
  46. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +2 -2
  47. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +2 -2
  48. package/src/config/conversion/minicpm/minicpm4-0-5b-f16-af32.json +2 -2
  49. package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-bf16-af32.json +1 -1
  50. package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-f16-af32.json +1 -1
  51. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +5 -5
  52. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +5 -5
  53. package/src/config/conversion/qwen3/qwen-3-5-9b-q4k-ehaf16.json +5 -5
  54. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +5 -5
  55. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +5 -5
  56. package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-q4k-ehf16-af32.json +2 -2
  57. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-af32.json +2 -2
  58. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-true-logit-af32.json +1 -1
  59. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +1 -1
  60. package/src/config/conversion/qwen3/qwen3.8-27b-text-f16-af32.json +3 -3
  61. package/src/config/conversion/qwen3/qwen3.8-27b-text-q4k-ehaf16.json +3 -3
  62. package/src/config/generation-contract.d.ts +39 -0
  63. package/src/config/generation-contract.js +66 -0
  64. package/src/config/generation-contract.json +32 -0
  65. package/src/config/kernel-path-loader.d.ts +7 -0
  66. package/src/config/kernel-path-loader.js +19 -0
  67. package/src/config/kernels/kernel-ref-digests.js +16 -16
  68. package/src/config/kernels/registry.json +35 -10
  69. package/src/config/schema/doppler.schema.js +1 -0
  70. package/src/config/source-packages/litert/gemma-4-e2b-it.json +2 -2
  71. package/src/converter/forge-qualification.js +2 -1
  72. package/src/converter/forge-stages.js +6 -4
  73. package/src/converter/quantizer.js +18 -4
  74. package/src/gpu/device-state.js +8 -0
  75. package/src/gpu/device.js +6 -1
  76. package/src/gpu/kernels/gelu.d.ts +4 -0
  77. package/src/gpu/kernels/gelu.js +11 -1
  78. package/src/gpu/kernels/gelu.wgsl +16 -0
  79. package/src/gpu/kernels/gelu_f16.wgsl +16 -0
  80. package/src/gpu/kernels/rep-penalty.js +30 -5
  81. package/src/gpu/kernels/rep_penalty.wgsl +25 -30
  82. package/src/gpu/kernels/rep_penalty_f16.wgsl +24 -25
  83. package/src/gpu/kernels/sample.d.ts +1 -0
  84. package/src/gpu/kernels/sample.js +31 -16
  85. package/src/gpu/kernels/sample.wgsl +112 -157
  86. package/src/gpu/kernels/sample_f16.wgsl +168 -162
  87. package/src/index.d.ts +2 -0
  88. package/src/index.js +1 -0
  89. package/src/inference/pipelines/context.js +6 -2
  90. package/src/inference/pipelines/structured/json-grammar-mask.d.ts +2 -0
  91. package/src/inference/pipelines/structured/json-grammar-mask.js +115 -118
  92. package/src/inference/pipelines/text/ffn/dense-executor.js +5 -0
  93. package/src/inference/pipelines/text/ffn/dense.js +2 -2
  94. package/src/inference/pipelines/text/generator/decode-runtime.js +8 -14
  95. package/src/inference/pipelines/text/generator/decode.js +18 -6
  96. package/src/inference/pipelines/text/generator/training.js +3 -2
  97. package/src/inference/pipelines/text/generator-logits-observation.js +5 -2
  98. package/src/inference/pipelines/text/generator-runtime.js +18 -19
  99. package/src/inference/pipelines/text/generator-steps.js +15 -8
  100. package/src/inference/pipelines/text/generator.js +16 -10
  101. package/src/inference/pipelines/text/linear-attention.js +24 -0
  102. package/src/inference/pipelines/text/logits/output-transform.js +10 -3
  103. package/src/inference/pipelines/text/sampling-config.js +22 -113
  104. package/src/inference/pipelines/text/sampling.d.ts +9 -1
  105. package/src/inference/pipelines/text/sampling.js +18 -375
  106. package/src/inference/pipelines/text/types.d.ts +3 -3
  107. package/src/inference/token-sampling.js +319 -0
  108. package/src/storage/backends/opfs-store.d.ts +41 -0
  109. package/src/tooling/node-browser/static-server.js +1 -0
  110. package/src/tooling-exports/storage.d.ts +2 -0
  111. package/src/tooling-exports/storage.js +1 -0
  112. package/src/version.js +1 -1
  113. package/tests/kernels/browser/test-page.js +1 -0
@@ -17,6 +17,7 @@ export interface ArgmaxOptions {
17
17
  export interface SampleOptions extends ArgmaxOptions {
18
18
  temperature: number;
19
19
  topK: number;
20
+ topP: number;
20
21
  randomSeed?: number;
21
22
  greedyThreshold: number;
22
23
  }
@@ -13,6 +13,7 @@ import { selectRuleValue as selectKernelRuleValue } from './rule-registry.js';
13
13
  import { selectRuleValue as selectSharedRuleValue } from '../../rules/rule-registry.js';
14
14
  import { getKernelThresholds } from '../../config/schema/index.js';
15
15
  import { recordDispatch } from './dispatch.js';
16
+ import { validateGenerationField } from '../../config/generation-contract.js';
16
17
 
17
18
 
18
19
  function getSampleBindGroupLayout(device) {
@@ -74,6 +75,7 @@ function assertArgmaxOptions(options, recordMode = false) {
74
75
 
75
76
  function assertSampleOptions(options, recordMode = false) {
76
77
  const suffix = recordMode ? ' (record)' : '';
78
+ for (const key of ['temperature', 'topK', 'topP']) validateGenerationField(key, options[key]);
77
79
  if (options.temperature == null) {
78
80
  throw new Error(`[Sample] temperature is required for sampling${suffix}.`);
79
81
  }
@@ -138,7 +140,7 @@ function createArgmaxUniformBuffer(device, recorder, vocabSize, options) {
138
140
  );
139
141
  }
140
142
 
141
- function createSampleUniformBuffer(device, recorder, vocabSize, topK, temperature, randomValue, padTokenId, logitSoftcap, outputIndex) {
143
+ function createSampleUniformBuffer(device, recorder, vocabSize, topK, temperature, randomValue, padTokenId, logitSoftcap, outputIndex, topP) {
142
144
  return createUniformBufferWithView(
143
145
  'sample_uniforms',
144
146
  32,
@@ -150,6 +152,7 @@ function createSampleUniformBuffer(device, recorder, vocabSize, topK, temperatur
150
152
  view.setUint32(16, padTokenId == null ? 0xFFFFFFFF : padTokenId, true);
151
153
  view.setFloat32(20, logitSoftcap, true);
152
154
  view.setUint32(24, outputIndex, true);
155
+ view.setFloat32(28, topP, true);
153
156
  },
154
157
  recorder,
155
158
  device
@@ -163,7 +166,17 @@ function ensureOutputBufferSize(outputBuffer, minBytes, label) {
163
166
  }
164
167
 
165
168
  async function readTokenFromOutput(outputBuffer, outputIndex) {
166
- return new Uint32Array(await readBufferSlice(outputBuffer, outputIndex * 4, 4))[0];
169
+ const token = new Uint32Array(await readBufferSlice(outputBuffer, outputIndex * 4, 4))[0];
170
+ if (token === 0xFFFFFFFF) throw new Error('[Sampling] No finite candidate logits.');
171
+ return token;
172
+ }
173
+
174
+ function resolveSampleGeometry(vocabSize, requestedTopK) {
175
+ if (!Number.isSafeInteger(vocabSize) || vocabSize < 1) throw new Error('[Sample] Positive vocabSize required.');
176
+ const topK = requestedTopK === 0 ? vocabSize : Math.min(requestedTopK, vocabSize);
177
+ const numWorkgroups = Math.max(1, Math.min(WORKGROUP_SIZES.DEFAULT,
178
+ Math.ceil(vocabSize / WORKGROUP_SIZES.DEFAULT), Math.floor(vocabSize / topK)));
179
+ return { topK, numWorkgroups, scratchBytes: (numWorkgroups + 1) * topK * Uint32Array.BYTES_PER_ELEMENT };
167
180
  }
168
181
 
169
182
  function cleanupRunResources(uniformBuffer, ownedBuffers) {
@@ -357,7 +370,8 @@ export async function runGPUSample(
357
370
 
358
371
  const {
359
372
  temperature,
360
- topK,
373
+ topK: requestedTopK,
374
+ topP,
361
375
  randomSeed,
362
376
  padTokenId,
363
377
  logitSoftcap,
@@ -367,7 +381,8 @@ export async function runGPUSample(
367
381
  } = options;
368
382
  const logitsDtype = resolveLogitsDtype(options.logitsDtype);
369
383
 
370
- if (temperature < greedyThreshold || topK <= 1) {
384
+ const { topK, numWorkgroups, scratchBytes } = resolveSampleGeometry(vocabSize, requestedTopK);
385
+ if (temperature < greedyThreshold || topK === 1) {
371
386
  return runArgmax(logits, vocabSize, {
372
387
  padTokenId,
373
388
  logitSoftcap,
@@ -389,16 +404,14 @@ export async function runGPUSample(
389
404
  const phase2Pipeline = await createSamplePipeline(device, variants.phase2);
390
405
  const phase3Pipeline = await createSamplePipeline(device, variants.phase3);
391
406
 
392
- const numWorkgroups = Math.min(WORKGROUP_SIZES.DEFAULT, Math.ceil(vocabSize / WORKGROUP_SIZES.DEFAULT));
393
-
394
407
  let topkLogits = null;
395
408
  let topkIndices = null;
396
409
  let outputBuffer = null;
397
410
  let ownsOutputBuffer = false;
398
411
  let uniformBuffer = null;
399
412
  try {
400
- topkLogits = acquireBuffer(WORKGROUP_SIZES.DEFAULT * 4, undefined, 'topk_logits');
401
- topkIndices = acquireBuffer(WORKGROUP_SIZES.DEFAULT * 4, undefined, 'topk_indices');
413
+ topkLogits = acquireBuffer(scratchBytes, undefined, 'topk_logits');
414
+ topkIndices = acquireBuffer(scratchBytes, undefined, 'topk_indices');
402
415
  const minOutputBytes = Math.max(4, (outputIndex + 1) * 4);
403
416
  outputBuffer = outputBufferOverride ?? acquireBuffer(minOutputBytes, undefined, 'sample_output');
404
417
  ownsOutputBuffer = !outputBufferOverride;
@@ -413,7 +426,8 @@ export async function runGPUSample(
413
426
  randomValue,
414
427
  padTokenId,
415
428
  logitSoftcap,
416
- outputIndex
429
+ outputIndex,
430
+ topP
417
431
  );
418
432
 
419
433
  const bindGroupLayout = getSampleBindGroupLayout(device);
@@ -480,7 +494,8 @@ export async function recordGPUSample(
480
494
 
481
495
  const {
482
496
  temperature,
483
- topK,
497
+ topK: requestedTopK,
498
+ topP,
484
499
  randomSeed,
485
500
  padTokenId,
486
501
  logitSoftcap,
@@ -490,7 +505,8 @@ export async function recordGPUSample(
490
505
  } = options;
491
506
  const logitsDtype = resolveLogitsDtype(options.logitsDtype);
492
507
 
493
- if (temperature < greedyThreshold || topK <= 1) {
508
+ const { topK, numWorkgroups, scratchBytes } = resolveSampleGeometry(vocabSize, requestedTopK);
509
+ if (temperature < greedyThreshold || topK === 1) {
494
510
  return recordArgmax(recorder, logits, vocabSize, {
495
511
  padTokenId,
496
512
  logitSoftcap,
@@ -511,16 +527,14 @@ export async function recordGPUSample(
511
527
  const phase2Pipeline = await createSamplePipeline(device, variants.phase2);
512
528
  const phase3Pipeline = await createSamplePipeline(device, variants.phase3);
513
529
 
514
- const numWorkgroups = Math.min(WORKGROUP_SIZES.DEFAULT, Math.ceil(vocabSize / WORKGROUP_SIZES.DEFAULT));
515
-
516
530
  let topkLogits = null;
517
531
  let topkIndices = null;
518
532
  let outputBuffer = null;
519
533
  let ownsOutputBuffer = false;
520
534
  let completed = false;
521
535
  try {
522
- topkLogits = acquireBuffer(WORKGROUP_SIZES.DEFAULT * 4, undefined, 'topk_logits');
523
- topkIndices = acquireBuffer(WORKGROUP_SIZES.DEFAULT * 4, undefined, 'topk_indices');
536
+ topkLogits = acquireBuffer(scratchBytes, undefined, 'topk_logits');
537
+ topkIndices = acquireBuffer(scratchBytes, undefined, 'topk_indices');
524
538
  const minOutputBytes = Math.max(4, (outputIndex + 1) * 4);
525
539
  outputBuffer = outputBufferOverride ?? acquireBuffer(minOutputBytes, undefined, 'sample_output');
526
540
  ownsOutputBuffer = !outputBufferOverride;
@@ -535,7 +549,8 @@ export async function recordGPUSample(
535
549
  randomValue,
536
550
  padTokenId,
537
551
  logitSoftcap,
538
- outputIndex
552
+ outputIndex,
553
+ topP
539
554
  );
540
555
 
541
556
  const bindGroupLayout = getSampleBindGroupLayout(device);
@@ -18,7 +18,6 @@
18
18
  // Configuration
19
19
  override WORKGROUP_SIZE: u32 = 256u;
20
20
  const MAX_WORKGROUP_SIZE: u32 = 256u;
21
- const MAX_TOP_K: u32 = 128u; // Max top-k supported
22
21
  const NEG_INF: f32 = -3.402823e+38;
23
22
 
24
23
  struct Uniforms {
@@ -29,7 +28,7 @@ struct Uniforms {
29
28
  pad_token_id: u32,
30
29
  logit_softcap: f32, // Gemma 2: 30.0, 0.0 = disabled
31
30
  output_index: u32, // Index into output token buffer
32
- pad0: u32,
31
+ top_p: f32,
33
32
  }
34
33
 
35
34
  // Apply softcapping: softcap * tanh(x / softcap)
@@ -61,183 +60,140 @@ fn candidate_beats(candidate_value: f32, candidate_index: u32, best_value: f32,
61
60
  var<workgroup> shared_values: array<f32, MAX_WORKGROUP_SIZE>;
62
61
  var<workgroup> shared_indices: array<u32, MAX_WORKGROUP_SIZE>;
63
62
 
64
- // Phase 1: Find local max in each workgroup for parallel top-k
65
- // Each thread scans a chunk of vocabulary, keeps local top element
66
- @compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
67
- fn find_topk_phase1(
68
- @builtin(global_invocation_id) gid: vec3<u32>,
69
- @builtin(local_invocation_id) lid: vec3<u32>,
70
- @builtin(workgroup_id) wgid: vec3<u32>,
71
- @builtin(num_workgroups) num_wg: vec3<u32>
72
- ) {
73
- let thread_idx = lid.x;
74
- let global_idx = gid.x;
75
- let vocab_size = u.vocab_size;
76
- let temperature = u.temperature;
77
- let pad_id = u.pad_token_id;
78
- let softcap = u.logit_softcap;
63
+ // Each partition retains its exact top-k in a min-heap. The merge has a
64
+ // separate output region, so retaining a winner cannot overwrite unread input.
65
+ // Scratch holds at most two vocabularies; top_k is resolved to [1, vocab_size].
66
+ fn sample_group_count() -> u32 {
67
+ return max(1u, min(min(WORKGROUP_SIZE, (u.vocab_size + WORKGROUP_SIZE - 1u) / WORKGROUP_SIZE),
68
+ u.vocab_size / u.top_k));
69
+ }
79
70
 
80
- // Single workgroup: write all logits directly for exact top-k
81
- if (num_wg.x == 1u) {
82
- var val: f32 = NEG_INF;
83
- if (global_idx < vocab_size && global_idx != pad_id) {
84
- val = apply_softcap(logits[global_idx], softcap) / temperature;
85
- }
86
- topk_logits[thread_idx] = val;
87
- topk_indices[thread_idx] = global_idx;
88
- return;
89
- }
71
+ fn heap_swap(base: u32, a: u32, b: u32) {
72
+ let value = topk_logits[base + a];
73
+ let token = topk_indices[base + a];
74
+ topk_logits[base + a] = topk_logits[base + b];
75
+ topk_indices[base + a] = topk_indices[base + b];
76
+ topk_logits[base + b] = value;
77
+ topk_indices[base + b] = token;
78
+ }
90
79
 
91
- // Each thread finds max in its assigned range
92
- var local_max: f32 = NEG_INF; // -FLT_MAX
93
- var local_max_idx: u32 = 0u;
80
+ fn heap_better(base: u32, a: u32, b: u32) -> bool {
81
+ return candidate_beats(topk_logits[base + a], topk_indices[base + a],
82
+ topk_logits[base + b], topk_indices[base + b]);
83
+ }
94
84
 
95
- // Stride through vocabulary
96
- var idx = global_idx;
97
- while (idx < vocab_size) {
98
- if (idx != pad_id) {
99
- // Apply softcapping before temperature scaling
100
- let val = apply_softcap(logits[idx], softcap) / temperature;
101
- if (candidate_beats(val, idx, local_max, local_max_idx)) {
102
- local_max = val;
103
- local_max_idx = idx;
104
- }
105
- }
106
- idx = idx + WORKGROUP_SIZE * num_wg.x;
85
+ fn heap_down(base: u32, count: u32) {
86
+ var root = 0u;
87
+ loop {
88
+ let left = root * 2u + 1u;
89
+ if (left >= count) { break; }
90
+ var worst = left;
91
+ let right = left + 1u;
92
+ if (right < count && heap_better(base, left, right)) { worst = right; }
93
+ if (!heap_better(base, root, worst)) { break; }
94
+ heap_swap(base, root, worst);
95
+ root = worst;
107
96
  }
97
+ }
108
98
 
109
- shared_values[thread_idx] = local_max;
110
- shared_indices[thread_idx] = local_max_idx;
111
- workgroupBarrier();
112
-
113
- // Reduce within workgroup to find workgroup's top value
114
- var stride = WORKGROUP_SIZE / 2u;
115
- while (stride > 0u) {
116
- if (thread_idx < stride) {
117
- if (candidate_beats(
118
- shared_values[thread_idx + stride],
119
- shared_indices[thread_idx + stride],
120
- shared_values[thread_idx],
121
- shared_indices[thread_idx]
122
- )) {
123
- shared_values[thread_idx] = shared_values[thread_idx + stride];
124
- shared_indices[thread_idx] = shared_indices[thread_idx + stride];
125
- }
99
+ fn heap_offer(base: u32, count: u32, value: f32, token: u32) -> u32 {
100
+ if (count < u.top_k) {
101
+ topk_logits[base + count] = value;
102
+ topk_indices[base + count] = token;
103
+ var child = count;
104
+ loop {
105
+ if (child == 0u) { break; }
106
+ let parent = (child - 1u) / 2u;
107
+ if (!heap_better(base, parent, child)) { break; }
108
+ heap_swap(base, parent, child);
109
+ child = parent;
126
110
  }
127
- workgroupBarrier();
128
- stride = stride / 2u;
111
+ return count + 1u;
129
112
  }
130
-
131
- // Thread 0 writes workgroup result
132
- if (thread_idx == 0u) {
133
- topk_logits[wgid.x] = shared_values[0];
134
- topk_indices[wgid.x] = shared_indices[0];
113
+ if (candidate_beats(value, token, topk_logits[base], topk_indices[base])) {
114
+ topk_logits[base] = value;
115
+ topk_indices[base] = token;
116
+ heap_down(base, count);
135
117
  }
118
+ return count;
136
119
  }
137
120
 
138
- // Phase 2: Merge workgroup results and select final top-k
139
- // Single workgroup sorts and selects top-k from workgroup results
140
121
  @compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
141
- fn find_topk_phase2(
142
- @builtin(local_invocation_id) lid: vec3<u32>
122
+ fn find_topk_phase1(
123
+ @builtin(local_invocation_id) lid: vec3<u32>,
124
+ @builtin(workgroup_id) wgid: vec3<u32>,
125
+ @builtin(num_workgroups) num_wg: vec3<u32>
143
126
  ) {
144
- let thread_idx = lid.x;
145
- let top_k = u.top_k;
146
- let num_groups = min(WORKGROUP_SIZE, (u.vocab_size + WORKGROUP_SIZE - 1u) / WORKGROUP_SIZE);
147
- let num_candidates = select(num_groups, min(u.vocab_size, WORKGROUP_SIZE), num_groups == 1u);
148
-
149
- // Load workgroup results into shared memory
150
- // Assume <= WORKGROUP_SIZE workgroups from phase 1
151
- if (thread_idx < WORKGROUP_SIZE) {
152
- if (thread_idx < num_candidates) {
153
- shared_values[thread_idx] = topk_logits[thread_idx];
154
- shared_indices[thread_idx] = topk_indices[thread_idx];
155
- } else {
156
- shared_values[thread_idx] = NEG_INF;
157
- shared_indices[thread_idx] = 0u;
127
+ if (lid.x != 0u) { return; }
128
+ let base = wgid.x * u.top_k;
129
+ for (var k = 0u; k < u.top_k; k++) { topk_indices[base + k] = 0xffffffffu; }
130
+ var count = 0u;
131
+ for (var token = wgid.x; token < u.vocab_size; token += num_wg.x) {
132
+ let raw = logits[token];
133
+ // Non-finite and padding logits never enter the candidate distribution.
134
+ if (token != u.pad_token_id && abs(raw) <= 3.402823e+38) {
135
+ let value = apply_softcap(raw, u.logit_softcap);
136
+ count = heap_offer(base, count, value, token);
158
137
  }
159
138
  }
160
- workgroupBarrier();
161
-
162
- // Thread 0 does partial selection sort for top-k
163
- if (thread_idx == 0u) {
164
- for (var k: u32 = 0u; k < top_k && k < num_candidates; k = k + 1u) {
165
- var max_idx = k;
166
- var max_val = shared_values[k];
167
-
168
- for (var i: u32 = k + 1u; i < num_candidates; i = i + 1u) {
169
- if (candidate_beats(shared_values[i], shared_indices[i], max_val, shared_indices[max_idx])) {
170
- max_val = shared_values[i];
171
- max_idx = i;
172
- }
173
- }
174
-
175
- if (max_idx != k) {
176
- let tmp_val = shared_values[k];
177
- let tmp_idx = shared_indices[k];
178
- shared_values[k] = shared_values[max_idx];
179
- shared_indices[k] = shared_indices[max_idx];
180
- shared_values[max_idx] = tmp_val;
181
- shared_indices[max_idx] = tmp_idx;
182
- }
183
- }
139
+ }
184
140
 
185
- // Write sorted top-k back
186
- for (var k: u32 = 0u; k < top_k; k = k + 1u) {
187
- topk_logits[k] = shared_values[k];
188
- topk_indices[k] = shared_indices[k];
141
+ @compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
142
+ fn find_topk_phase2(@builtin(local_invocation_id) lid: vec3<u32>) {
143
+ if (lid.x != 0u) { return; }
144
+ let base = sample_group_count() * u.top_k;
145
+ for (var k = 0u; k < u.top_k; k++) { topk_indices[base + k] = 0xffffffffu; }
146
+ var count = 0u;
147
+ for (var i = 0u; i < base; i++) {
148
+ if (topk_indices[i] != 0xffffffffu) {
149
+ count = heap_offer(base, count, topk_logits[i], topk_indices[i]);
189
150
  }
190
151
  }
191
152
  }
192
153
 
193
- // Phase 3: Softmax on top-k and sample
154
+ // Sort the retained heap, normalize top-k, retain the minimal top-p prefix,
155
+ // then sample from that renormalized prefix. F16 inputs use F32 accumulation.
194
156
  @compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
195
- fn softmax_and_sample(
196
- @builtin(local_invocation_id) lid: vec3<u32>
197
- ) {
198
- let thread_idx = lid.x;
199
- let top_k = u.top_k;
200
- let random_val = u.random_value;
201
-
202
- // Load top-k logits
203
- if (thread_idx < top_k) {
204
- shared_values[thread_idx] = topk_logits[thread_idx];
205
- shared_indices[thread_idx] = topk_indices[thread_idx];
157
+ fn softmax_and_sample(@builtin(local_invocation_id) lid: vec3<u32>) {
158
+ if (lid.x != 0u) { return; }
159
+ let base = sample_group_count() * u.top_k;
160
+ var count = 0u;
161
+ while (count < u.top_k && topk_indices[base + count] != 0xffffffffu) { count++; }
162
+ if (count == 0u) {
163
+ output[u.output_index] = 0xffffffffu;
164
+ return;
206
165
  }
207
- workgroupBarrier();
208
-
209
- // Thread 0 does softmax and sampling
210
- if (thread_idx == 0u) {
211
- // Find max for numerical stability
212
- var max_val: f32 = shared_values[0];
213
- for (var i: u32 = 1u; i < top_k; i = i + 1u) {
214
- max_val = max(max_val, shared_values[i]);
215
- }
216
-
217
- // Compute exp and sum
218
- var exp_sum: f32 = 0.0;
219
- for (var i: u32 = 0u; i < top_k; i = i + 1u) {
220
- let exp_val = exp(shared_values[i] - max_val);
221
- shared_values[i] = exp_val;
222
- exp_sum = exp_sum + exp_val;
223
- }
224
-
225
- // Normalize to probabilities and sample
226
- let inv_sum = 1.0 / exp_sum;
227
- var cum_prob: f32 = 0.0;
228
- var selected_token: u32 = shared_indices[top_k - 1u]; // Default to last
229
-
230
- for (var i: u32 = 0u; i < top_k; i = i + 1u) {
231
- let prob = shared_values[i] * inv_sum;
232
- cum_prob = cum_prob + prob;
233
- if (cum_prob >= random_val) {
234
- selected_token = shared_indices[i];
235
- break;
236
- }
166
+ var remaining = count;
167
+ while (remaining > 1u) {
168
+ remaining--;
169
+ heap_swap(base, 0u, remaining);
170
+ heap_down(base, remaining);
171
+ }
172
+ let maximum = topk_logits[base];
173
+ var total = 0.0;
174
+ for (var i = 0u; i < count; i++) {
175
+ let weight = exp((topk_logits[base + i] - maximum) / u.temperature);
176
+ topk_logits[base + i] = weight;
177
+ total += weight;
178
+ }
179
+ var retained = 0u;
180
+ var retained_total = 0.0;
181
+ loop {
182
+ retained_total += topk_logits[base + retained];
183
+ retained++;
184
+ if (retained >= count || retained_total >= u.top_p * total) { break; }
185
+ }
186
+ let threshold = u.random_value * retained_total;
187
+ var cumulative = 0.0;
188
+ var selected = topk_indices[base + retained - 1u];
189
+ for (var i = 0u; i < retained; i++) {
190
+ cumulative += topk_logits[base + i];
191
+ if (cumulative >= threshold) {
192
+ selected = topk_indices[base + i];
193
+ break;
237
194
  }
238
-
239
- output[u.output_index] = selected_token;
240
195
  }
196
+ output[u.output_index] = selected;
241
197
  }
242
198
 
243
199
  // Combined single-pass version for smaller vocabularies (<= 65536)
@@ -250,7 +206,6 @@ fn sample_single_pass(
250
206
  ) {
251
207
  let thread_idx = lid.x;
252
208
  let vocab_size = u.vocab_size;
253
- let top_k = min(u.top_k, MAX_TOP_K);
254
209
  let temperature = u.temperature;
255
210
  let random_val = u.random_value;
256
211
  let pad_id = u.pad_token_id;