doppler-gpu 0.6.0 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +8 -2
- package/src/capsule-runtime.d.ts +2 -0
- package/src/capsule-runtime.js +1 -0
- package/src/client/capsule-host.browser.js +1 -0
- package/src/client/capsule-host.d.ts +2 -0
- package/src/client/capsule-host.js +1 -0
- package/src/client/doppler-api.d.ts +2 -0
- package/src/client/doppler-api.js +1 -0
- package/src/client/model-host/model-session.d.ts +1 -0
- package/src/client/model-host/model-session.js +12 -0
- package/src/client/runtime/capsule-operation-adapters.js +15 -14
- package/src/client/runtime/capsule-operation-executor.js +3 -2
- package/src/client/runtime/capsule-program-adapter.js +2 -0
- package/src/client/runtime/capsule-session-execution.js +9 -5
- package/src/client/runtime/composition-root.d.ts +6 -3
- package/src/client/runtime/composition-root.js +13 -5
- package/src/client/runtime/session-controller.d.ts +6 -17
- package/src/client/runtime/session-controller.js +42 -120
- package/src/client/runtime/verified-capsule-artifact-store.js +38 -9
- package/src/config/capsule-loading.d.ts +1 -1
- package/src/config/capsule-loading.schema.json +3 -2
- package/src/config/capsule-operation.js +6 -1
- package/src/config/capsule-operations.json +1 -2
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +3 -3
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +1 -1
- package/src/config/conversion/esm/esm2-t12-35m-ur50d-f32-af32.json +32 -14
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +2 -2
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +2 -2
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +2 -2
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +3 -3
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +5 -5
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +2 -2
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +3 -3
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +3 -3
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +2 -2
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/glmocr/glm-ocr-f16-af32.json +1 -1
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +2 -2
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +2 -2
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/minicpm/minicpm4-0-5b-f16-af32.json +2 -2
- package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-bf16-af32.json +1 -1
- package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-f16-af32.json +1 -1
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +5 -5
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +5 -5
- package/src/config/conversion/qwen3/qwen-3-5-9b-q4k-ehaf16.json +5 -5
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +5 -5
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +5 -5
- package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-af32.json +2 -2
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-true-logit-af32.json +1 -1
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +1 -1
- package/src/config/conversion/qwen3/qwen3.8-27b-text-f16-af32.json +3 -3
- package/src/config/conversion/qwen3/qwen3.8-27b-text-q4k-ehaf16.json +3 -3
- package/src/config/generation-contract.d.ts +39 -0
- package/src/config/generation-contract.js +66 -0
- package/src/config/generation-contract.json +32 -0
- package/src/config/kernel-path-loader.d.ts +7 -0
- package/src/config/kernel-path-loader.js +19 -0
- package/src/config/kernels/kernel-ref-digests.js +16 -16
- package/src/config/kernels/registry.json +35 -10
- package/src/config/schema/doppler.schema.js +1 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +2 -2
- package/src/converter/forge-qualification.js +2 -1
- package/src/converter/forge-stages.js +6 -4
- package/src/converter/quantizer.js +18 -4
- package/src/gpu/device-state.js +8 -0
- package/src/gpu/device.js +6 -1
- package/src/gpu/kernels/gelu.d.ts +4 -0
- package/src/gpu/kernels/gelu.js +11 -1
- package/src/gpu/kernels/gelu.wgsl +16 -0
- package/src/gpu/kernels/gelu_f16.wgsl +16 -0
- package/src/gpu/kernels/rep-penalty.js +30 -5
- package/src/gpu/kernels/rep_penalty.wgsl +25 -30
- package/src/gpu/kernels/rep_penalty_f16.wgsl +24 -25
- package/src/gpu/kernels/sample.d.ts +1 -0
- package/src/gpu/kernels/sample.js +31 -16
- package/src/gpu/kernels/sample.wgsl +112 -157
- package/src/gpu/kernels/sample_f16.wgsl +168 -162
- package/src/index.d.ts +2 -0
- package/src/index.js +1 -0
- package/src/inference/pipelines/context.js +6 -2
- package/src/inference/pipelines/structured/json-grammar-mask.d.ts +2 -0
- package/src/inference/pipelines/structured/json-grammar-mask.js +115 -118
- package/src/inference/pipelines/text/ffn/dense-executor.js +5 -0
- package/src/inference/pipelines/text/ffn/dense.js +2 -2
- package/src/inference/pipelines/text/generator/decode-runtime.js +8 -14
- package/src/inference/pipelines/text/generator/decode.js +18 -6
- package/src/inference/pipelines/text/generator/training.js +3 -2
- package/src/inference/pipelines/text/generator-logits-observation.js +5 -2
- package/src/inference/pipelines/text/generator-runtime.js +18 -19
- package/src/inference/pipelines/text/generator-steps.js +15 -8
- package/src/inference/pipelines/text/generator.js +16 -10
- package/src/inference/pipelines/text/linear-attention.js +24 -0
- package/src/inference/pipelines/text/logits/output-transform.js +10 -3
- package/src/inference/pipelines/text/sampling-config.js +22 -113
- package/src/inference/pipelines/text/sampling.d.ts +9 -1
- package/src/inference/pipelines/text/sampling.js +18 -375
- package/src/inference/pipelines/text/types.d.ts +3 -3
- package/src/inference/token-sampling.js +319 -0
- package/src/storage/backends/opfs-store.d.ts +41 -0
- package/src/tooling/node-browser/static-server.js +1 -0
- package/src/tooling-exports/storage.d.ts +2 -0
- package/src/tooling-exports/storage.js +1 -0
- package/src/version.js +1 -1
- package/tests/kernels/browser/test-page.js +1 -0
|
@@ -13,6 +13,7 @@ import { selectRuleValue as selectKernelRuleValue } from './rule-registry.js';
|
|
|
13
13
|
import { selectRuleValue as selectSharedRuleValue } from '../../rules/rule-registry.js';
|
|
14
14
|
import { getKernelThresholds } from '../../config/schema/index.js';
|
|
15
15
|
import { recordDispatch } from './dispatch.js';
|
|
16
|
+
import { validateGenerationField } from '../../config/generation-contract.js';
|
|
16
17
|
|
|
17
18
|
|
|
18
19
|
function getSampleBindGroupLayout(device) {
|
|
@@ -74,6 +75,7 @@ function assertArgmaxOptions(options, recordMode = false) {
|
|
|
74
75
|
|
|
75
76
|
function assertSampleOptions(options, recordMode = false) {
|
|
76
77
|
const suffix = recordMode ? ' (record)' : '';
|
|
78
|
+
for (const key of ['temperature', 'topK', 'topP']) validateGenerationField(key, options[key]);
|
|
77
79
|
if (options.temperature == null) {
|
|
78
80
|
throw new Error(`[Sample] temperature is required for sampling${suffix}.`);
|
|
79
81
|
}
|
|
@@ -138,7 +140,7 @@ function createArgmaxUniformBuffer(device, recorder, vocabSize, options) {
|
|
|
138
140
|
);
|
|
139
141
|
}
|
|
140
142
|
|
|
141
|
-
function createSampleUniformBuffer(device, recorder, vocabSize, topK, temperature, randomValue, padTokenId, logitSoftcap, outputIndex) {
|
|
143
|
+
function createSampleUniformBuffer(device, recorder, vocabSize, topK, temperature, randomValue, padTokenId, logitSoftcap, outputIndex, topP) {
|
|
142
144
|
return createUniformBufferWithView(
|
|
143
145
|
'sample_uniforms',
|
|
144
146
|
32,
|
|
@@ -150,6 +152,7 @@ function createSampleUniformBuffer(device, recorder, vocabSize, topK, temperatur
|
|
|
150
152
|
view.setUint32(16, padTokenId == null ? 0xFFFFFFFF : padTokenId, true);
|
|
151
153
|
view.setFloat32(20, logitSoftcap, true);
|
|
152
154
|
view.setUint32(24, outputIndex, true);
|
|
155
|
+
view.setFloat32(28, topP, true);
|
|
153
156
|
},
|
|
154
157
|
recorder,
|
|
155
158
|
device
|
|
@@ -163,7 +166,17 @@ function ensureOutputBufferSize(outputBuffer, minBytes, label) {
|
|
|
163
166
|
}
|
|
164
167
|
|
|
165
168
|
async function readTokenFromOutput(outputBuffer, outputIndex) {
|
|
166
|
-
|
|
169
|
+
const token = new Uint32Array(await readBufferSlice(outputBuffer, outputIndex * 4, 4))[0];
|
|
170
|
+
if (token === 0xFFFFFFFF) throw new Error('[Sampling] No finite candidate logits.');
|
|
171
|
+
return token;
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
function resolveSampleGeometry(vocabSize, requestedTopK) {
|
|
175
|
+
if (!Number.isSafeInteger(vocabSize) || vocabSize < 1) throw new Error('[Sample] Positive vocabSize required.');
|
|
176
|
+
const topK = requestedTopK === 0 ? vocabSize : Math.min(requestedTopK, vocabSize);
|
|
177
|
+
const numWorkgroups = Math.max(1, Math.min(WORKGROUP_SIZES.DEFAULT,
|
|
178
|
+
Math.ceil(vocabSize / WORKGROUP_SIZES.DEFAULT), Math.floor(vocabSize / topK)));
|
|
179
|
+
return { topK, numWorkgroups, scratchBytes: (numWorkgroups + 1) * topK * Uint32Array.BYTES_PER_ELEMENT };
|
|
167
180
|
}
|
|
168
181
|
|
|
169
182
|
function cleanupRunResources(uniformBuffer, ownedBuffers) {
|
|
@@ -357,7 +370,8 @@ export async function runGPUSample(
|
|
|
357
370
|
|
|
358
371
|
const {
|
|
359
372
|
temperature,
|
|
360
|
-
topK,
|
|
373
|
+
topK: requestedTopK,
|
|
374
|
+
topP,
|
|
361
375
|
randomSeed,
|
|
362
376
|
padTokenId,
|
|
363
377
|
logitSoftcap,
|
|
@@ -367,7 +381,8 @@ export async function runGPUSample(
|
|
|
367
381
|
} = options;
|
|
368
382
|
const logitsDtype = resolveLogitsDtype(options.logitsDtype);
|
|
369
383
|
|
|
370
|
-
|
|
384
|
+
const { topK, numWorkgroups, scratchBytes } = resolveSampleGeometry(vocabSize, requestedTopK);
|
|
385
|
+
if (temperature < greedyThreshold || topK === 1) {
|
|
371
386
|
return runArgmax(logits, vocabSize, {
|
|
372
387
|
padTokenId,
|
|
373
388
|
logitSoftcap,
|
|
@@ -389,16 +404,14 @@ export async function runGPUSample(
|
|
|
389
404
|
const phase2Pipeline = await createSamplePipeline(device, variants.phase2);
|
|
390
405
|
const phase3Pipeline = await createSamplePipeline(device, variants.phase3);
|
|
391
406
|
|
|
392
|
-
const numWorkgroups = Math.min(WORKGROUP_SIZES.DEFAULT, Math.ceil(vocabSize / WORKGROUP_SIZES.DEFAULT));
|
|
393
|
-
|
|
394
407
|
let topkLogits = null;
|
|
395
408
|
let topkIndices = null;
|
|
396
409
|
let outputBuffer = null;
|
|
397
410
|
let ownsOutputBuffer = false;
|
|
398
411
|
let uniformBuffer = null;
|
|
399
412
|
try {
|
|
400
|
-
topkLogits = acquireBuffer(
|
|
401
|
-
topkIndices = acquireBuffer(
|
|
413
|
+
topkLogits = acquireBuffer(scratchBytes, undefined, 'topk_logits');
|
|
414
|
+
topkIndices = acquireBuffer(scratchBytes, undefined, 'topk_indices');
|
|
402
415
|
const minOutputBytes = Math.max(4, (outputIndex + 1) * 4);
|
|
403
416
|
outputBuffer = outputBufferOverride ?? acquireBuffer(minOutputBytes, undefined, 'sample_output');
|
|
404
417
|
ownsOutputBuffer = !outputBufferOverride;
|
|
@@ -413,7 +426,8 @@ export async function runGPUSample(
|
|
|
413
426
|
randomValue,
|
|
414
427
|
padTokenId,
|
|
415
428
|
logitSoftcap,
|
|
416
|
-
outputIndex
|
|
429
|
+
outputIndex,
|
|
430
|
+
topP
|
|
417
431
|
);
|
|
418
432
|
|
|
419
433
|
const bindGroupLayout = getSampleBindGroupLayout(device);
|
|
@@ -480,7 +494,8 @@ export async function recordGPUSample(
|
|
|
480
494
|
|
|
481
495
|
const {
|
|
482
496
|
temperature,
|
|
483
|
-
topK,
|
|
497
|
+
topK: requestedTopK,
|
|
498
|
+
topP,
|
|
484
499
|
randomSeed,
|
|
485
500
|
padTokenId,
|
|
486
501
|
logitSoftcap,
|
|
@@ -490,7 +505,8 @@ export async function recordGPUSample(
|
|
|
490
505
|
} = options;
|
|
491
506
|
const logitsDtype = resolveLogitsDtype(options.logitsDtype);
|
|
492
507
|
|
|
493
|
-
|
|
508
|
+
const { topK, numWorkgroups, scratchBytes } = resolveSampleGeometry(vocabSize, requestedTopK);
|
|
509
|
+
if (temperature < greedyThreshold || topK === 1) {
|
|
494
510
|
return recordArgmax(recorder, logits, vocabSize, {
|
|
495
511
|
padTokenId,
|
|
496
512
|
logitSoftcap,
|
|
@@ -511,16 +527,14 @@ export async function recordGPUSample(
|
|
|
511
527
|
const phase2Pipeline = await createSamplePipeline(device, variants.phase2);
|
|
512
528
|
const phase3Pipeline = await createSamplePipeline(device, variants.phase3);
|
|
513
529
|
|
|
514
|
-
const numWorkgroups = Math.min(WORKGROUP_SIZES.DEFAULT, Math.ceil(vocabSize / WORKGROUP_SIZES.DEFAULT));
|
|
515
|
-
|
|
516
530
|
let topkLogits = null;
|
|
517
531
|
let topkIndices = null;
|
|
518
532
|
let outputBuffer = null;
|
|
519
533
|
let ownsOutputBuffer = false;
|
|
520
534
|
let completed = false;
|
|
521
535
|
try {
|
|
522
|
-
topkLogits = acquireBuffer(
|
|
523
|
-
topkIndices = acquireBuffer(
|
|
536
|
+
topkLogits = acquireBuffer(scratchBytes, undefined, 'topk_logits');
|
|
537
|
+
topkIndices = acquireBuffer(scratchBytes, undefined, 'topk_indices');
|
|
524
538
|
const minOutputBytes = Math.max(4, (outputIndex + 1) * 4);
|
|
525
539
|
outputBuffer = outputBufferOverride ?? acquireBuffer(minOutputBytes, undefined, 'sample_output');
|
|
526
540
|
ownsOutputBuffer = !outputBufferOverride;
|
|
@@ -535,7 +549,8 @@ export async function recordGPUSample(
|
|
|
535
549
|
randomValue,
|
|
536
550
|
padTokenId,
|
|
537
551
|
logitSoftcap,
|
|
538
|
-
outputIndex
|
|
552
|
+
outputIndex,
|
|
553
|
+
topP
|
|
539
554
|
);
|
|
540
555
|
|
|
541
556
|
const bindGroupLayout = getSampleBindGroupLayout(device);
|
|
@@ -18,7 +18,6 @@
|
|
|
18
18
|
// Configuration
|
|
19
19
|
override WORKGROUP_SIZE: u32 = 256u;
|
|
20
20
|
const MAX_WORKGROUP_SIZE: u32 = 256u;
|
|
21
|
-
const MAX_TOP_K: u32 = 128u; // Max top-k supported
|
|
22
21
|
const NEG_INF: f32 = -3.402823e+38;
|
|
23
22
|
|
|
24
23
|
struct Uniforms {
|
|
@@ -29,7 +28,7 @@ struct Uniforms {
|
|
|
29
28
|
pad_token_id: u32,
|
|
30
29
|
logit_softcap: f32, // Gemma 2: 30.0, 0.0 = disabled
|
|
31
30
|
output_index: u32, // Index into output token buffer
|
|
32
|
-
|
|
31
|
+
top_p: f32,
|
|
33
32
|
}
|
|
34
33
|
|
|
35
34
|
// Apply softcapping: softcap * tanh(x / softcap)
|
|
@@ -61,183 +60,140 @@ fn candidate_beats(candidate_value: f32, candidate_index: u32, best_value: f32,
|
|
|
61
60
|
var<workgroup> shared_values: array<f32, MAX_WORKGROUP_SIZE>;
|
|
62
61
|
var<workgroup> shared_indices: array<u32, MAX_WORKGROUP_SIZE>;
|
|
63
62
|
|
|
64
|
-
//
|
|
65
|
-
//
|
|
66
|
-
|
|
67
|
-
fn
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
@builtin(num_workgroups) num_wg: vec3<u32>
|
|
72
|
-
) {
|
|
73
|
-
let thread_idx = lid.x;
|
|
74
|
-
let global_idx = gid.x;
|
|
75
|
-
let vocab_size = u.vocab_size;
|
|
76
|
-
let temperature = u.temperature;
|
|
77
|
-
let pad_id = u.pad_token_id;
|
|
78
|
-
let softcap = u.logit_softcap;
|
|
63
|
+
// Each partition retains its exact top-k in a min-heap. The merge has a
|
|
64
|
+
// separate output region, so retaining a winner cannot overwrite unread input.
|
|
65
|
+
// Scratch holds at most two vocabularies; top_k is resolved to [1, vocab_size].
|
|
66
|
+
fn sample_group_count() -> u32 {
|
|
67
|
+
return max(1u, min(min(WORKGROUP_SIZE, (u.vocab_size + WORKGROUP_SIZE - 1u) / WORKGROUP_SIZE),
|
|
68
|
+
u.vocab_size / u.top_k));
|
|
69
|
+
}
|
|
79
70
|
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
return;
|
|
89
|
-
}
|
|
71
|
+
fn heap_swap(base: u32, a: u32, b: u32) {
|
|
72
|
+
let value = topk_logits[base + a];
|
|
73
|
+
let token = topk_indices[base + a];
|
|
74
|
+
topk_logits[base + a] = topk_logits[base + b];
|
|
75
|
+
topk_indices[base + a] = topk_indices[base + b];
|
|
76
|
+
topk_logits[base + b] = value;
|
|
77
|
+
topk_indices[base + b] = token;
|
|
78
|
+
}
|
|
90
79
|
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
80
|
+
fn heap_better(base: u32, a: u32, b: u32) -> bool {
|
|
81
|
+
return candidate_beats(topk_logits[base + a], topk_indices[base + a],
|
|
82
|
+
topk_logits[base + b], topk_indices[base + b]);
|
|
83
|
+
}
|
|
94
84
|
|
|
95
|
-
|
|
96
|
-
var
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
idx = idx + WORKGROUP_SIZE * num_wg.x;
|
|
85
|
+
fn heap_down(base: u32, count: u32) {
|
|
86
|
+
var root = 0u;
|
|
87
|
+
loop {
|
|
88
|
+
let left = root * 2u + 1u;
|
|
89
|
+
if (left >= count) { break; }
|
|
90
|
+
var worst = left;
|
|
91
|
+
let right = left + 1u;
|
|
92
|
+
if (right < count && heap_better(base, left, right)) { worst = right; }
|
|
93
|
+
if (!heap_better(base, root, worst)) { break; }
|
|
94
|
+
heap_swap(base, root, worst);
|
|
95
|
+
root = worst;
|
|
107
96
|
}
|
|
97
|
+
}
|
|
108
98
|
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
if (
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
shared_values[thread_idx],
|
|
121
|
-
shared_indices[thread_idx]
|
|
122
|
-
)) {
|
|
123
|
-
shared_values[thread_idx] = shared_values[thread_idx + stride];
|
|
124
|
-
shared_indices[thread_idx] = shared_indices[thread_idx + stride];
|
|
125
|
-
}
|
|
99
|
+
fn heap_offer(base: u32, count: u32, value: f32, token: u32) -> u32 {
|
|
100
|
+
if (count < u.top_k) {
|
|
101
|
+
topk_logits[base + count] = value;
|
|
102
|
+
topk_indices[base + count] = token;
|
|
103
|
+
var child = count;
|
|
104
|
+
loop {
|
|
105
|
+
if (child == 0u) { break; }
|
|
106
|
+
let parent = (child - 1u) / 2u;
|
|
107
|
+
if (!heap_better(base, parent, child)) { break; }
|
|
108
|
+
heap_swap(base, parent, child);
|
|
109
|
+
child = parent;
|
|
126
110
|
}
|
|
127
|
-
|
|
128
|
-
stride = stride / 2u;
|
|
111
|
+
return count + 1u;
|
|
129
112
|
}
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
topk_indices[wgid.x] = shared_indices[0];
|
|
113
|
+
if (candidate_beats(value, token, topk_logits[base], topk_indices[base])) {
|
|
114
|
+
topk_logits[base] = value;
|
|
115
|
+
topk_indices[base] = token;
|
|
116
|
+
heap_down(base, count);
|
|
135
117
|
}
|
|
118
|
+
return count;
|
|
136
119
|
}
|
|
137
120
|
|
|
138
|
-
// Phase 2: Merge workgroup results and select final top-k
|
|
139
|
-
// Single workgroup sorts and selects top-k from workgroup results
|
|
140
121
|
@compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
|
|
141
|
-
fn
|
|
142
|
-
@builtin(local_invocation_id) lid: vec3<u32
|
|
122
|
+
fn find_topk_phase1(
|
|
123
|
+
@builtin(local_invocation_id) lid: vec3<u32>,
|
|
124
|
+
@builtin(workgroup_id) wgid: vec3<u32>,
|
|
125
|
+
@builtin(num_workgroups) num_wg: vec3<u32>
|
|
143
126
|
) {
|
|
144
|
-
|
|
145
|
-
let
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
shared_indices[thread_idx] = topk_indices[thread_idx];
|
|
155
|
-
} else {
|
|
156
|
-
shared_values[thread_idx] = NEG_INF;
|
|
157
|
-
shared_indices[thread_idx] = 0u;
|
|
127
|
+
if (lid.x != 0u) { return; }
|
|
128
|
+
let base = wgid.x * u.top_k;
|
|
129
|
+
for (var k = 0u; k < u.top_k; k++) { topk_indices[base + k] = 0xffffffffu; }
|
|
130
|
+
var count = 0u;
|
|
131
|
+
for (var token = wgid.x; token < u.vocab_size; token += num_wg.x) {
|
|
132
|
+
let raw = logits[token];
|
|
133
|
+
// Non-finite and padding logits never enter the candidate distribution.
|
|
134
|
+
if (token != u.pad_token_id && abs(raw) <= 3.402823e+38) {
|
|
135
|
+
let value = apply_softcap(raw, u.logit_softcap);
|
|
136
|
+
count = heap_offer(base, count, value, token);
|
|
158
137
|
}
|
|
159
138
|
}
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
// Thread 0 does partial selection sort for top-k
|
|
163
|
-
if (thread_idx == 0u) {
|
|
164
|
-
for (var k: u32 = 0u; k < top_k && k < num_candidates; k = k + 1u) {
|
|
165
|
-
var max_idx = k;
|
|
166
|
-
var max_val = shared_values[k];
|
|
167
|
-
|
|
168
|
-
for (var i: u32 = k + 1u; i < num_candidates; i = i + 1u) {
|
|
169
|
-
if (candidate_beats(shared_values[i], shared_indices[i], max_val, shared_indices[max_idx])) {
|
|
170
|
-
max_val = shared_values[i];
|
|
171
|
-
max_idx = i;
|
|
172
|
-
}
|
|
173
|
-
}
|
|
174
|
-
|
|
175
|
-
if (max_idx != k) {
|
|
176
|
-
let tmp_val = shared_values[k];
|
|
177
|
-
let tmp_idx = shared_indices[k];
|
|
178
|
-
shared_values[k] = shared_values[max_idx];
|
|
179
|
-
shared_indices[k] = shared_indices[max_idx];
|
|
180
|
-
shared_values[max_idx] = tmp_val;
|
|
181
|
-
shared_indices[max_idx] = tmp_idx;
|
|
182
|
-
}
|
|
183
|
-
}
|
|
139
|
+
}
|
|
184
140
|
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
141
|
+
@compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
|
|
142
|
+
fn find_topk_phase2(@builtin(local_invocation_id) lid: vec3<u32>) {
|
|
143
|
+
if (lid.x != 0u) { return; }
|
|
144
|
+
let base = sample_group_count() * u.top_k;
|
|
145
|
+
for (var k = 0u; k < u.top_k; k++) { topk_indices[base + k] = 0xffffffffu; }
|
|
146
|
+
var count = 0u;
|
|
147
|
+
for (var i = 0u; i < base; i++) {
|
|
148
|
+
if (topk_indices[i] != 0xffffffffu) {
|
|
149
|
+
count = heap_offer(base, count, topk_logits[i], topk_indices[i]);
|
|
189
150
|
}
|
|
190
151
|
}
|
|
191
152
|
}
|
|
192
153
|
|
|
193
|
-
//
|
|
154
|
+
// Sort the retained heap, normalize top-k, retain the minimal top-p prefix,
|
|
155
|
+
// then sample from that renormalized prefix. F16 inputs use F32 accumulation.
|
|
194
156
|
@compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
|
|
195
|
-
fn softmax_and_sample(
|
|
196
|
-
|
|
197
|
-
)
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
if (thread_idx < top_k) {
|
|
204
|
-
shared_values[thread_idx] = topk_logits[thread_idx];
|
|
205
|
-
shared_indices[thread_idx] = topk_indices[thread_idx];
|
|
157
|
+
fn softmax_and_sample(@builtin(local_invocation_id) lid: vec3<u32>) {
|
|
158
|
+
if (lid.x != 0u) { return; }
|
|
159
|
+
let base = sample_group_count() * u.top_k;
|
|
160
|
+
var count = 0u;
|
|
161
|
+
while (count < u.top_k && topk_indices[base + count] != 0xffffffffu) { count++; }
|
|
162
|
+
if (count == 0u) {
|
|
163
|
+
output[u.output_index] = 0xffffffffu;
|
|
164
|
+
return;
|
|
206
165
|
}
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
break;
|
|
236
|
-
}
|
|
166
|
+
var remaining = count;
|
|
167
|
+
while (remaining > 1u) {
|
|
168
|
+
remaining--;
|
|
169
|
+
heap_swap(base, 0u, remaining);
|
|
170
|
+
heap_down(base, remaining);
|
|
171
|
+
}
|
|
172
|
+
let maximum = topk_logits[base];
|
|
173
|
+
var total = 0.0;
|
|
174
|
+
for (var i = 0u; i < count; i++) {
|
|
175
|
+
let weight = exp((topk_logits[base + i] - maximum) / u.temperature);
|
|
176
|
+
topk_logits[base + i] = weight;
|
|
177
|
+
total += weight;
|
|
178
|
+
}
|
|
179
|
+
var retained = 0u;
|
|
180
|
+
var retained_total = 0.0;
|
|
181
|
+
loop {
|
|
182
|
+
retained_total += topk_logits[base + retained];
|
|
183
|
+
retained++;
|
|
184
|
+
if (retained >= count || retained_total >= u.top_p * total) { break; }
|
|
185
|
+
}
|
|
186
|
+
let threshold = u.random_value * retained_total;
|
|
187
|
+
var cumulative = 0.0;
|
|
188
|
+
var selected = topk_indices[base + retained - 1u];
|
|
189
|
+
for (var i = 0u; i < retained; i++) {
|
|
190
|
+
cumulative += topk_logits[base + i];
|
|
191
|
+
if (cumulative >= threshold) {
|
|
192
|
+
selected = topk_indices[base + i];
|
|
193
|
+
break;
|
|
237
194
|
}
|
|
238
|
-
|
|
239
|
-
output[u.output_index] = selected_token;
|
|
240
195
|
}
|
|
196
|
+
output[u.output_index] = selected;
|
|
241
197
|
}
|
|
242
198
|
|
|
243
199
|
// Combined single-pass version for smaller vocabularies (<= 65536)
|
|
@@ -250,7 +206,6 @@ fn sample_single_pass(
|
|
|
250
206
|
) {
|
|
251
207
|
let thread_idx = lid.x;
|
|
252
208
|
let vocab_size = u.vocab_size;
|
|
253
|
-
let top_k = min(u.top_k, MAX_TOP_K);
|
|
254
209
|
let temperature = u.temperature;
|
|
255
210
|
let random_val = u.random_value;
|
|
256
211
|
let pad_id = u.pad_token_id;
|