doppler-gpu 0.6.0 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +8 -2
- package/src/capsule-runtime.d.ts +2 -0
- package/src/capsule-runtime.js +1 -0
- package/src/client/capsule-host.browser.js +1 -0
- package/src/client/capsule-host.d.ts +2 -0
- package/src/client/capsule-host.js +1 -0
- package/src/client/doppler-api.d.ts +2 -0
- package/src/client/doppler-api.js +1 -0
- package/src/client/model-host/model-session.d.ts +1 -0
- package/src/client/model-host/model-session.js +12 -0
- package/src/client/runtime/capsule-operation-adapters.js +15 -14
- package/src/client/runtime/capsule-operation-executor.js +3 -2
- package/src/client/runtime/capsule-program-adapter.js +2 -0
- package/src/client/runtime/capsule-session-execution.js +9 -5
- package/src/client/runtime/composition-root.d.ts +6 -3
- package/src/client/runtime/composition-root.js +13 -5
- package/src/client/runtime/session-controller.d.ts +6 -17
- package/src/client/runtime/session-controller.js +42 -120
- package/src/client/runtime/verified-capsule-artifact-store.js +38 -9
- package/src/config/capsule-loading.d.ts +1 -1
- package/src/config/capsule-loading.schema.json +3 -2
- package/src/config/capsule-operation.js +6 -1
- package/src/config/capsule-operations.json +1 -2
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +3 -3
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +1 -1
- package/src/config/conversion/esm/esm2-t12-35m-ur50d-f32-af32.json +32 -14
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +2 -2
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +2 -2
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +2 -2
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +3 -3
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +5 -5
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +2 -2
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +3 -3
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +3 -3
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +3 -3
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +2 -2
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/glmocr/glm-ocr-f16-af32.json +1 -1
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +2 -2
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +2 -2
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/minicpm/minicpm4-0-5b-f16-af32.json +2 -2
- package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-bf16-af32.json +1 -1
- package/src/config/conversion/muse-glimmer/muse-glimmer-30b-text-f16-af32.json +1 -1
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +5 -5
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +5 -5
- package/src/config/conversion/qwen3/qwen-3-5-9b-q4k-ehaf16.json +5 -5
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +5 -5
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +5 -5
- package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-q4k-ehf16-af32.json +2 -2
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-af32.json +2 -2
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-true-logit-af32.json +1 -1
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +1 -1
- package/src/config/conversion/qwen3/qwen3.8-27b-text-f16-af32.json +3 -3
- package/src/config/conversion/qwen3/qwen3.8-27b-text-q4k-ehaf16.json +3 -3
- package/src/config/generation-contract.d.ts +39 -0
- package/src/config/generation-contract.js +66 -0
- package/src/config/generation-contract.json +32 -0
- package/src/config/kernel-path-loader.d.ts +7 -0
- package/src/config/kernel-path-loader.js +19 -0
- package/src/config/kernels/kernel-ref-digests.js +16 -16
- package/src/config/kernels/registry.json +35 -10
- package/src/config/schema/doppler.schema.js +1 -0
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +2 -2
- package/src/converter/forge-qualification.js +2 -1
- package/src/converter/forge-stages.js +6 -4
- package/src/converter/quantizer.js +18 -4
- package/src/gpu/device-state.js +8 -0
- package/src/gpu/device.js +6 -1
- package/src/gpu/kernels/gelu.d.ts +4 -0
- package/src/gpu/kernels/gelu.js +11 -1
- package/src/gpu/kernels/gelu.wgsl +16 -0
- package/src/gpu/kernels/gelu_f16.wgsl +16 -0
- package/src/gpu/kernels/rep-penalty.js +30 -5
- package/src/gpu/kernels/rep_penalty.wgsl +25 -30
- package/src/gpu/kernels/rep_penalty_f16.wgsl +24 -25
- package/src/gpu/kernels/sample.d.ts +1 -0
- package/src/gpu/kernels/sample.js +31 -16
- package/src/gpu/kernels/sample.wgsl +112 -157
- package/src/gpu/kernels/sample_f16.wgsl +168 -162
- package/src/index.d.ts +2 -0
- package/src/index.js +1 -0
- package/src/inference/pipelines/context.js +6 -2
- package/src/inference/pipelines/structured/json-grammar-mask.d.ts +2 -0
- package/src/inference/pipelines/structured/json-grammar-mask.js +115 -118
- package/src/inference/pipelines/text/ffn/dense-executor.js +5 -0
- package/src/inference/pipelines/text/ffn/dense.js +2 -2
- package/src/inference/pipelines/text/generator/decode-runtime.js +8 -14
- package/src/inference/pipelines/text/generator/decode.js +18 -6
- package/src/inference/pipelines/text/generator/training.js +3 -2
- package/src/inference/pipelines/text/generator-logits-observation.js +5 -2
- package/src/inference/pipelines/text/generator-runtime.js +18 -19
- package/src/inference/pipelines/text/generator-steps.js +15 -8
- package/src/inference/pipelines/text/generator.js +16 -10
- package/src/inference/pipelines/text/linear-attention.js +24 -0
- package/src/inference/pipelines/text/logits/output-transform.js +10 -3
- package/src/inference/pipelines/text/sampling-config.js +22 -113
- package/src/inference/pipelines/text/sampling.d.ts +9 -1
- package/src/inference/pipelines/text/sampling.js +18 -375
- package/src/inference/pipelines/text/types.d.ts +3 -3
- package/src/inference/token-sampling.js +319 -0
- package/src/storage/backends/opfs-store.d.ts +41 -0
- package/src/tooling/node-browser/static-server.js +1 -0
- package/src/tooling-exports/storage.d.ts +2 -0
- package/src/tooling-exports/storage.js +1 -0
- package/src/version.js +1 -1
- package/tests/kernels/browser/test-page.js +1 -0
|
@@ -1,41 +1,50 @@
|
|
|
1
1
|
// AUTO-GENERATED from src/gpu/kernels/sample.wgsl.
|
|
2
2
|
// Edit the source kernel and src/gpu/kernels/codegen/wgsl-variants.js, then run `npm run kernels:codegen:sync`.
|
|
3
|
-
// sample_f16.wgsl
|
|
3
|
+
// sample_f16.wgsl: F16 storage, F32 sampling accumulation.
|
|
4
|
+
|
|
5
|
+
enable f16;
|
|
4
6
|
|
|
5
7
|
/**
|
|
6
|
-
* GPU-Side Sampling Kernel
|
|
8
|
+
* GPU-Side Sampling Kernel
|
|
9
|
+
*
|
|
10
|
+
* Performs temperature scaling, top-k selection, softmax, and sampling
|
|
11
|
+
* entirely on GPU. Only reads back the single selected token ID.
|
|
7
12
|
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
13
|
+
* Reduces readback from 1MB (256K vocab × 4 bytes) to 4 bytes.
|
|
14
|
+
*
|
|
15
|
+
* Algorithm:
|
|
16
|
+
* 1. Temperature scaling: logits = logits / temperature
|
|
17
|
+
* 2. Parallel top-k: Each workgroup finds local top-k, then merge
|
|
18
|
+
* 3. Softmax on top-k candidates
|
|
19
|
+
* 4. Multinomial sampling with provided random value
|
|
10
20
|
*/
|
|
11
21
|
|
|
12
|
-
enable f16;
|
|
13
|
-
|
|
14
22
|
// Configuration
|
|
15
23
|
override WORKGROUP_SIZE: u32 = 256u;
|
|
16
24
|
const MAX_WORKGROUP_SIZE: u32 = 256u;
|
|
17
|
-
const
|
|
18
|
-
const NEG_INF = f16(-65504.0);
|
|
25
|
+
const NEG_INF: f32 = -3.402823e+38;
|
|
19
26
|
|
|
20
27
|
struct Uniforms {
|
|
21
28
|
vocab_size: u32,
|
|
22
29
|
top_k: u32,
|
|
23
30
|
temperature: f32,
|
|
24
|
-
random_value: f32,
|
|
31
|
+
random_value: f32, // Pre-generated random [0,1) for sampling
|
|
25
32
|
pad_token_id: u32,
|
|
26
|
-
logit_softcap: f32,
|
|
27
|
-
output_index: u32,
|
|
28
|
-
|
|
33
|
+
logit_softcap: f32, // Gemma 2: 30.0, 0.0 = disabled
|
|
34
|
+
output_index: u32, // Index into output token buffer
|
|
35
|
+
top_p: f32,
|
|
29
36
|
}
|
|
30
37
|
|
|
31
|
-
|
|
32
|
-
|
|
38
|
+
// Apply softcapping: softcap * tanh(x / softcap)
|
|
39
|
+
// Returns x unchanged if softcap <= 0
|
|
40
|
+
fn apply_softcap(x: f32, softcap: f32) -> f32 {
|
|
41
|
+
if (softcap <= 0.0) {
|
|
33
42
|
return x;
|
|
34
43
|
}
|
|
35
44
|
return softcap * tanh(x / softcap);
|
|
36
45
|
}
|
|
37
46
|
|
|
38
|
-
fn candidate_beats(candidate_value:
|
|
47
|
+
fn candidate_beats(candidate_value: f32, candidate_index: u32, best_value: f32, best_index: u32) -> bool {
|
|
39
48
|
if (candidate_value > best_value) {
|
|
40
49
|
return true;
|
|
41
50
|
}
|
|
@@ -46,173 +55,153 @@ fn candidate_beats(candidate_value: f16, candidate_index: u32, best_value: f16,
|
|
|
46
55
|
}
|
|
47
56
|
|
|
48
57
|
@group(0) @binding(0) var<uniform> u: Uniforms;
|
|
49
|
-
@group(0) @binding(1) var<storage, read> logits: array<f16>;
|
|
50
|
-
@group(0) @binding(2) var<storage, read_write> output: array<u32>;
|
|
51
|
-
@group(0) @binding(3) var<storage, read_write> topk_indices: array<u32>;
|
|
52
|
-
@group(0) @binding(4) var<storage, read_write> topk_logits: array<
|
|
58
|
+
@group(0) @binding(1) var<storage, read> logits: array<f16>; // [vocabSize]
|
|
59
|
+
@group(0) @binding(2) var<storage, read_write> output: array<u32>; // [N] - selected tokens
|
|
60
|
+
@group(0) @binding(3) var<storage, read_write> topk_indices: array<u32>; // [topK] - intermediate
|
|
61
|
+
@group(0) @binding(4) var<storage, read_write> topk_logits: array<f32>; // [topK] - intermediate
|
|
53
62
|
|
|
54
|
-
|
|
63
|
+
// Shared memory for workgroup-level reduction
|
|
64
|
+
var<workgroup> shared_values: array<f32, MAX_WORKGROUP_SIZE>;
|
|
55
65
|
var<workgroup> shared_indices: array<u32, MAX_WORKGROUP_SIZE>;
|
|
56
66
|
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
let thread_idx = lid.x;
|
|
65
|
-
let global_idx = gid.x;
|
|
66
|
-
let vocab_size = u.vocab_size;
|
|
67
|
-
let temperature = f16(u.temperature);
|
|
68
|
-
let pad_id = u.pad_token_id;
|
|
69
|
-
let softcap = f16(u.logit_softcap);
|
|
67
|
+
// Each partition retains its exact top-k in a min-heap. The merge has a
|
|
68
|
+
// separate output region, so retaining a winner cannot overwrite unread input.
|
|
69
|
+
// Scratch holds at most two vocabularies; top_k is resolved to [1, vocab_size].
|
|
70
|
+
fn sample_group_count() -> u32 {
|
|
71
|
+
return max(1u, min(min(WORKGROUP_SIZE, (u.vocab_size + WORKGROUP_SIZE - 1u) / WORKGROUP_SIZE),
|
|
72
|
+
u.vocab_size / u.top_k));
|
|
73
|
+
}
|
|
70
74
|
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
}
|
|
75
|
+
fn heap_swap(base: u32, a: u32, b: u32) {
|
|
76
|
+
let value = topk_logits[base + a];
|
|
77
|
+
let token = topk_indices[base + a];
|
|
78
|
+
topk_logits[base + a] = topk_logits[base + b];
|
|
79
|
+
topk_indices[base + a] = topk_indices[base + b];
|
|
80
|
+
topk_logits[base + b] = value;
|
|
81
|
+
topk_indices[base + b] = token;
|
|
82
|
+
}
|
|
80
83
|
|
|
81
|
-
|
|
82
|
-
|
|
84
|
+
fn heap_better(base: u32, a: u32, b: u32) -> bool {
|
|
85
|
+
return candidate_beats(topk_logits[base + a], topk_indices[base + a],
|
|
86
|
+
topk_logits[base + b], topk_indices[base + b]);
|
|
87
|
+
}
|
|
83
88
|
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
}
|
|
93
|
-
|
|
89
|
+
fn heap_down(base: u32, count: u32) {
|
|
90
|
+
var root = 0u;
|
|
91
|
+
loop {
|
|
92
|
+
let left = root * 2u + 1u;
|
|
93
|
+
if (left >= count) { break; }
|
|
94
|
+
var worst = left;
|
|
95
|
+
let right = left + 1u;
|
|
96
|
+
if (right < count && heap_better(base, left, right)) { worst = right; }
|
|
97
|
+
if (!heap_better(base, root, worst)) { break; }
|
|
98
|
+
heap_swap(base, root, worst);
|
|
99
|
+
root = worst;
|
|
94
100
|
}
|
|
101
|
+
}
|
|
95
102
|
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
shared_indices[thread_idx]
|
|
108
|
-
)) {
|
|
109
|
-
shared_values[thread_idx] = shared_values[thread_idx + stride];
|
|
110
|
-
shared_indices[thread_idx] = shared_indices[thread_idx + stride];
|
|
111
|
-
}
|
|
103
|
+
fn heap_offer(base: u32, count: u32, value: f32, token: u32) -> u32 {
|
|
104
|
+
if (count < u.top_k) {
|
|
105
|
+
topk_logits[base + count] = value;
|
|
106
|
+
topk_indices[base + count] = token;
|
|
107
|
+
var child = count;
|
|
108
|
+
loop {
|
|
109
|
+
if (child == 0u) { break; }
|
|
110
|
+
let parent = (child - 1u) / 2u;
|
|
111
|
+
if (!heap_better(base, parent, child)) { break; }
|
|
112
|
+
heap_swap(base, parent, child);
|
|
113
|
+
child = parent;
|
|
112
114
|
}
|
|
113
|
-
|
|
114
|
-
stride = stride / 2u;
|
|
115
|
+
return count + 1u;
|
|
115
116
|
}
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
117
|
+
if (candidate_beats(value, token, topk_logits[base], topk_indices[base])) {
|
|
118
|
+
topk_logits[base] = value;
|
|
119
|
+
topk_indices[base] = token;
|
|
120
|
+
heap_down(base, count);
|
|
120
121
|
}
|
|
122
|
+
return count;
|
|
121
123
|
}
|
|
122
124
|
|
|
123
125
|
@compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
|
|
124
|
-
fn
|
|
125
|
-
@builtin(local_invocation_id) lid: vec3<u32
|
|
126
|
+
fn find_topk_phase1(
|
|
127
|
+
@builtin(local_invocation_id) lid: vec3<u32>,
|
|
128
|
+
@builtin(workgroup_id) wgid: vec3<u32>,
|
|
129
|
+
@builtin(num_workgroups) num_wg: vec3<u32>
|
|
126
130
|
) {
|
|
127
|
-
|
|
128
|
-
let
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
shared_values[thread_idx] = NEG_INF;
|
|
138
|
-
shared_indices[thread_idx] = 0u;
|
|
131
|
+
if (lid.x != 0u) { return; }
|
|
132
|
+
let base = wgid.x * u.top_k;
|
|
133
|
+
for (var k = 0u; k < u.top_k; k++) { topk_indices[base + k] = 0xffffffffu; }
|
|
134
|
+
var count = 0u;
|
|
135
|
+
for (var token = wgid.x; token < u.vocab_size; token += num_wg.x) {
|
|
136
|
+
let raw = f32(logits[token]);
|
|
137
|
+
// Non-finite and padding logits never enter the candidate distribution.
|
|
138
|
+
if (token != u.pad_token_id && abs(raw) <= 3.402823e+38) {
|
|
139
|
+
let value = apply_softcap(raw, u.logit_softcap);
|
|
140
|
+
count = heap_offer(base, count, value, token);
|
|
139
141
|
}
|
|
140
142
|
}
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
if (thread_idx == 0u) {
|
|
144
|
-
for (var k: u32 = 0u; k < top_k && k < num_candidates; k = k + 1u) {
|
|
145
|
-
var max_idx = k;
|
|
146
|
-
var max_val = shared_values[k];
|
|
147
|
-
|
|
148
|
-
for (var i: u32 = k + 1u; i < num_candidates; i = i + 1u) {
|
|
149
|
-
if (candidate_beats(shared_values[i], shared_indices[i], max_val, shared_indices[max_idx])) {
|
|
150
|
-
max_val = shared_values[i];
|
|
151
|
-
max_idx = i;
|
|
152
|
-
}
|
|
153
|
-
}
|
|
154
|
-
|
|
155
|
-
if (max_idx != k) {
|
|
156
|
-
let tmp_val = shared_values[k];
|
|
157
|
-
let tmp_idx = shared_indices[k];
|
|
158
|
-
shared_values[k] = shared_values[max_idx];
|
|
159
|
-
shared_indices[k] = shared_indices[max_idx];
|
|
160
|
-
shared_values[max_idx] = tmp_val;
|
|
161
|
-
shared_indices[max_idx] = tmp_idx;
|
|
162
|
-
}
|
|
163
|
-
}
|
|
143
|
+
}
|
|
164
144
|
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
145
|
+
@compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
|
|
146
|
+
fn find_topk_phase2(@builtin(local_invocation_id) lid: vec3<u32>) {
|
|
147
|
+
if (lid.x != 0u) { return; }
|
|
148
|
+
let base = sample_group_count() * u.top_k;
|
|
149
|
+
for (var k = 0u; k < u.top_k; k++) { topk_indices[base + k] = 0xffffffffu; }
|
|
150
|
+
var count = 0u;
|
|
151
|
+
for (var i = 0u; i < base; i++) {
|
|
152
|
+
if (topk_indices[i] != 0xffffffffu) {
|
|
153
|
+
count = heap_offer(base, count, topk_logits[i], topk_indices[i]);
|
|
168
154
|
}
|
|
169
155
|
}
|
|
170
156
|
}
|
|
171
157
|
|
|
158
|
+
// Sort the retained heap, normalize top-k, retain the minimal top-p prefix,
|
|
159
|
+
// then sample from that renormalized prefix. F16 inputs use F32 accumulation.
|
|
172
160
|
@compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
|
|
173
|
-
fn softmax_and_sample(
|
|
174
|
-
|
|
175
|
-
)
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
shared_values[thread_idx] = topk_logits[thread_idx];
|
|
182
|
-
shared_indices[thread_idx] = topk_indices[thread_idx];
|
|
161
|
+
fn softmax_and_sample(@builtin(local_invocation_id) lid: vec3<u32>) {
|
|
162
|
+
if (lid.x != 0u) { return; }
|
|
163
|
+
let base = sample_group_count() * u.top_k;
|
|
164
|
+
var count = 0u;
|
|
165
|
+
while (count < u.top_k && topk_indices[base + count] != 0xffffffffu) { count++; }
|
|
166
|
+
if (count == 0u) {
|
|
167
|
+
output[u.output_index] = 0xffffffffu;
|
|
168
|
+
return;
|
|
183
169
|
}
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
170
|
+
var remaining = count;
|
|
171
|
+
while (remaining > 1u) {
|
|
172
|
+
remaining--;
|
|
173
|
+
heap_swap(base, 0u, remaining);
|
|
174
|
+
heap_down(base, remaining);
|
|
175
|
+
}
|
|
176
|
+
let maximum = topk_logits[base];
|
|
177
|
+
var total = 0.0;
|
|
178
|
+
for (var i = 0u; i < count; i++) {
|
|
179
|
+
let weight = exp((topk_logits[base + i] - maximum) / u.temperature);
|
|
180
|
+
topk_logits[base + i] = weight;
|
|
181
|
+
total += weight;
|
|
182
|
+
}
|
|
183
|
+
var retained = 0u;
|
|
184
|
+
var retained_total = 0.0;
|
|
185
|
+
loop {
|
|
186
|
+
retained_total += topk_logits[base + retained];
|
|
187
|
+
retained++;
|
|
188
|
+
if (retained >= count || retained_total >= u.top_p * total) { break; }
|
|
189
|
+
}
|
|
190
|
+
let threshold = u.random_value * retained_total;
|
|
191
|
+
var cumulative = 0.0;
|
|
192
|
+
var selected = topk_indices[base + retained - 1u];
|
|
193
|
+
for (var i = 0u; i < retained; i++) {
|
|
194
|
+
cumulative += topk_logits[base + i];
|
|
195
|
+
if (cumulative >= threshold) {
|
|
196
|
+
selected = topk_indices[base + i];
|
|
197
|
+
break;
|
|
210
198
|
}
|
|
211
|
-
|
|
212
|
-
output[u.output_index] = selected_token;
|
|
213
199
|
}
|
|
200
|
+
output[u.output_index] = selected;
|
|
214
201
|
}
|
|
215
202
|
|
|
203
|
+
// Combined single-pass version for smaller vocabularies (<= 65536)
|
|
204
|
+
// Uses hierarchical reduction within single kernel
|
|
216
205
|
@compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
|
|
217
206
|
fn sample_single_pass(
|
|
218
207
|
@builtin(global_invocation_id) gid: vec3<u32>,
|
|
@@ -221,18 +210,20 @@ fn sample_single_pass(
|
|
|
221
210
|
) {
|
|
222
211
|
let thread_idx = lid.x;
|
|
223
212
|
let vocab_size = u.vocab_size;
|
|
224
|
-
let
|
|
225
|
-
let
|
|
213
|
+
let temperature = u.temperature;
|
|
214
|
+
let random_val = u.random_value;
|
|
226
215
|
let pad_id = u.pad_token_id;
|
|
227
|
-
let softcap =
|
|
216
|
+
let softcap = u.logit_softcap;
|
|
228
217
|
|
|
229
|
-
|
|
218
|
+
// Phase 1: Find global max
|
|
219
|
+
var local_max: f32 = NEG_INF;
|
|
230
220
|
var local_max_idx: u32 = 0u;
|
|
231
221
|
|
|
232
222
|
var idx = gid.x;
|
|
233
223
|
while (idx < vocab_size) {
|
|
234
224
|
if (idx != pad_id) {
|
|
235
|
-
|
|
225
|
+
// Apply softcapping before temperature scaling
|
|
226
|
+
let val = apply_softcap(f32(logits[idx]), softcap) / temperature;
|
|
236
227
|
if (candidate_beats(val, idx, local_max, local_max_idx)) {
|
|
237
228
|
local_max = val;
|
|
238
229
|
local_max_idx = idx;
|
|
@@ -245,6 +236,7 @@ fn sample_single_pass(
|
|
|
245
236
|
shared_indices[thread_idx] = local_max_idx;
|
|
246
237
|
workgroupBarrier();
|
|
247
238
|
|
|
239
|
+
// Reduce to find workgroup max
|
|
248
240
|
var stride = WORKGROUP_SIZE / 2u;
|
|
249
241
|
while (stride > 0u) {
|
|
250
242
|
if (thread_idx < stride) {
|
|
@@ -262,11 +254,18 @@ fn sample_single_pass(
|
|
|
262
254
|
stride = stride / 2u;
|
|
263
255
|
}
|
|
264
256
|
|
|
257
|
+
// For single workgroup, thread 0 can do everything
|
|
265
258
|
if (thread_idx == 0u && num_wg.x == 1u) {
|
|
259
|
+
// We have top-1, but need top-k
|
|
260
|
+
// For small vocab, just do the full selection
|
|
261
|
+
// This simplified version selects top-1 only (greedy)
|
|
262
|
+
// Full top-k sampling requires multi-pass for large vocab
|
|
263
|
+
|
|
266
264
|
output[u.output_index] = shared_indices[0];
|
|
267
265
|
}
|
|
268
266
|
}
|
|
269
267
|
|
|
268
|
+
// Greedy argmax for deterministic decoding (temperature=0 equivalent)
|
|
270
269
|
@compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
|
|
271
270
|
fn argmax(
|
|
272
271
|
@builtin(global_invocation_id) gid: vec3<u32>,
|
|
@@ -278,15 +277,17 @@ fn argmax(
|
|
|
278
277
|
let global_idx = gid.x;
|
|
279
278
|
let vocab_size = u.vocab_size;
|
|
280
279
|
let pad_id = u.pad_token_id;
|
|
281
|
-
let softcap =
|
|
280
|
+
let softcap = u.logit_softcap;
|
|
282
281
|
|
|
283
|
-
|
|
282
|
+
// Each thread finds max in its chunk
|
|
283
|
+
var local_max: f32 = NEG_INF;
|
|
284
284
|
var local_max_idx: u32 = 0u;
|
|
285
285
|
|
|
286
286
|
var idx = global_idx;
|
|
287
287
|
while (idx < vocab_size) {
|
|
288
288
|
if (idx != pad_id) {
|
|
289
|
-
|
|
289
|
+
// Apply softcapping (argmax is greedy, no temperature)
|
|
290
|
+
let val = apply_softcap(f32(logits[idx]), softcap);
|
|
290
291
|
if (candidate_beats(val, idx, local_max, local_max_idx)) {
|
|
291
292
|
local_max = val;
|
|
292
293
|
local_max_idx = idx;
|
|
@@ -299,6 +300,7 @@ fn argmax(
|
|
|
299
300
|
shared_indices[thread_idx] = local_max_idx;
|
|
300
301
|
workgroupBarrier();
|
|
301
302
|
|
|
303
|
+
// Reduce within workgroup
|
|
302
304
|
var stride = WORKGROUP_SIZE / 2u;
|
|
303
305
|
while (stride > 0u) {
|
|
304
306
|
if (thread_idx < stride) {
|
|
@@ -316,12 +318,14 @@ fn argmax(
|
|
|
316
318
|
stride = stride / 2u;
|
|
317
319
|
}
|
|
318
320
|
|
|
321
|
+
// Write workgroup result to global memory
|
|
319
322
|
if (thread_idx == 0u) {
|
|
320
323
|
topk_logits[wgid.x] = shared_values[0];
|
|
321
324
|
topk_indices[wgid.x] = shared_indices[0];
|
|
322
325
|
}
|
|
323
326
|
}
|
|
324
327
|
|
|
328
|
+
// Final reduction for argmax across workgroups
|
|
325
329
|
@compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
|
|
326
330
|
fn argmax_reduce(
|
|
327
331
|
@builtin(local_invocation_id) lid: vec3<u32>
|
|
@@ -329,6 +333,7 @@ fn argmax_reduce(
|
|
|
329
333
|
let thread_idx = lid.x;
|
|
330
334
|
let num_groups = min(WORKGROUP_SIZE, (u.vocab_size + WORKGROUP_SIZE - 1u) / WORKGROUP_SIZE);
|
|
331
335
|
|
|
336
|
+
// Load workgroup maxes (up to WORKGROUP_SIZE)
|
|
332
337
|
if (thread_idx < num_groups) {
|
|
333
338
|
shared_values[thread_idx] = topk_logits[thread_idx];
|
|
334
339
|
shared_indices[thread_idx] = topk_indices[thread_idx];
|
|
@@ -338,6 +343,7 @@ fn argmax_reduce(
|
|
|
338
343
|
}
|
|
339
344
|
workgroupBarrier();
|
|
340
345
|
|
|
346
|
+
// Reduce
|
|
341
347
|
var stride = WORKGROUP_SIZE / 2u;
|
|
342
348
|
while (stride > 0u) {
|
|
343
349
|
if (thread_idx < stride) {
|
package/src/index.d.ts
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
export { DOPPLER_VERSION } from './version.js';
|
|
2
|
+
export { GENERATION_CONTRACT, GenerationError, resolveGenerationOptions, validateGenerationInput } from './client/doppler-api.js';
|
|
3
|
+
export type { GenerationOptions, GenerationInput, GenerationOutput, GenerationCompletion, ResolvedGenerationOptions } from './client/doppler-api.js';
|
|
2
4
|
export {
|
|
3
5
|
doppler,
|
|
4
6
|
doppler as dr,
|
package/src/index.js
CHANGED
|
@@ -4,6 +4,7 @@ import {
|
|
|
4
4
|
getPlatformConfig,
|
|
5
5
|
setDevice,
|
|
6
6
|
} from '../../gpu/device.js';
|
|
7
|
+
import { isDeviceLost } from '../../gpu/device-state.js';
|
|
7
8
|
import { applyDebugConfig, setGPUDevice } from '../../debug/index.js';
|
|
8
9
|
import { getRuntimeConfig, setRuntimeConfig } from '../../config/runtime.js';
|
|
9
10
|
import { kernelTrace } from './text/kernel-trace.js';
|
|
@@ -79,7 +80,7 @@ function restoreDebugState(snapshot) {
|
|
|
79
80
|
}
|
|
80
81
|
}
|
|
81
82
|
|
|
82
|
-
setGPUDevice(snapshot.gpuDevice ?? null);
|
|
83
|
+
setGPUDevice(isDeviceLost(snapshot.gpuDevice) ? getDevice() : snapshot.gpuDevice ?? null);
|
|
83
84
|
}
|
|
84
85
|
|
|
85
86
|
export function restorePipelineContexts(target) {
|
|
@@ -183,7 +184,10 @@ export function applyPipelineContexts(target, contexts = {}, options = {}) {
|
|
|
183
184
|
delete target[RESTORE_PIPELINE_CONTEXTS];
|
|
184
185
|
|
|
185
186
|
setRuntimeConfig(previousRuntimeConfig);
|
|
186
|
-
if (previousDevice) {
|
|
187
|
+
if (isDeviceLost(previousDevice)) {
|
|
188
|
+
// A closed scope cannot restore a destroyed device or displace its replacement.
|
|
189
|
+
if (isDeviceLost(getDevice())) setDevice(null);
|
|
190
|
+
} else if (previousDevice) {
|
|
187
191
|
setDevice(previousDevice, {
|
|
188
192
|
platformConfig: previousPlatformConfig,
|
|
189
193
|
adapterInfo: previousAdapterInfo,
|
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
export interface JsonGrammarMaskTokenizer {
|
|
2
2
|
decode(ids: number[], skipSpecial?: boolean, skipBos?: boolean): string;
|
|
3
|
+
getSpecialTokens?(): { eos?: number | number[] };
|
|
3
4
|
}
|
|
4
5
|
|
|
5
6
|
export interface JsonGrammarMaskOptions {
|
|
6
7
|
tokenizer?: JsonGrammarMaskTokenizer | null;
|
|
7
8
|
cacheBudget?: number;
|
|
9
|
+
stopTokenIds?: number[];
|
|
8
10
|
}
|
|
9
11
|
|
|
10
12
|
export interface JsonGrammarMaskContext {
|