doppler-gpu 0.4.8 → 0.4.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/CHANGELOG.md +10 -0
  2. package/models/catalog.json +21 -7
  3. package/package.json +8 -1
  4. package/src/config/conversion/qwen3/qwen-3-5-9b-f16-af32.json +474 -0
  5. package/src/config/conversion/qwen3/qwen-3-5-9b-q4k-ehaf16.json +509 -0
  6. package/src/config/kernels/backward-registry.json +13 -2
  7. package/src/config/kernels/kernel-ref-digests.js +8 -4
  8. package/src/config/kernels/registry.json +1035 -2677
  9. package/src/config/runtime/profiles/qwen-3-5-9b-metal-correctness.json +46 -0
  10. package/src/config/schema/training-metrics.schema.js +4 -0
  11. package/src/experimental/bridge/trainer-artifact-bridge.d.ts +161 -0
  12. package/src/experimental/bridge/trainer-artifact-bridge.js +573 -0
  13. package/src/experimental/training/attention-backward.js +123 -55
  14. package/src/experimental/training/autograd.js +101 -5
  15. package/src/experimental/training/clip.js +6 -1
  16. package/src/experimental/training/datasets/text-pairs.js +80 -8
  17. package/src/experimental/training/distillation/student-fixture.js +470 -114
  18. package/src/experimental/training/lora-pipeline.js +202 -27
  19. package/src/experimental/training/lora.js +10 -4
  20. package/src/experimental/training/optimizer.js +9 -1
  21. package/src/experimental/training/runner.js +18 -4
  22. package/src/experimental/training/workloads.js +11 -0
  23. package/src/gpu/kernels/backward/adam.js +6 -4
  24. package/src/gpu/kernels/backward/causal_conv1d_silu_backward.d.ts +24 -0
  25. package/src/gpu/kernels/backward/causal_conv1d_silu_backward.js +74 -0
  26. package/src/gpu/kernels/backward/causal_conv1d_silu_backward.wgsl +59 -0
  27. package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +4 -0
  28. package/src/gpu/kernels/backward/gated_delta_recurrent_backward.d.ts +39 -0
  29. package/src/gpu/kernels/backward/gated_delta_recurrent_backward.js +124 -0
  30. package/src/gpu/kernels/backward/gated_delta_recurrent_backward.wgsl +177 -0
  31. package/src/gpu/kernels/backward/gated_rmsnorm_backward.d.ts +32 -0
  32. package/src/gpu/kernels/backward/gated_rmsnorm_backward.js +90 -0
  33. package/src/gpu/kernels/backward/gated_rmsnorm_backward.wgsl +86 -0
  34. package/src/gpu/kernels/backward/index.d.ts +3 -0
  35. package/src/gpu/kernels/backward/index.js +3 -0
  36. package/src/gpu/kernels/backward/matmul_backward.wgsl +6 -6
  37. package/src/gpu/kernels/backward/matmul_backward_f16w.wgsl +82 -0
  38. package/src/gpu/kernels/backward/rmsnorm_backward.js +26 -4
  39. package/src/gpu/kernels/backward/rmsnorm_backward.wgsl +6 -3
  40. package/src/gpu/kernels/backward/utils.d.ts +12 -2
  41. package/src/gpu/kernels/backward/utils.js +9 -2
  42. package/src/gpu/kernels/cast_f16_to_f32.wgsl +6 -2
  43. package/src/gpu/kernels/index.d.ts +5 -0
  44. package/src/gpu/kernels/index.js +5 -0
  45. package/src/gpu/kernels/matmul-selection.js +5 -0
  46. package/src/gpu/kernels/silu.d.ts +1 -0
  47. package/src/gpu/kernels/silu.js +19 -4
  48. package/src/inference/pipelines/text/embed.js +1 -1
  49. package/src/inference/pipelines/text/execution-v1.js +8 -2
  50. package/src/inference/pipelines/text/ffn/dense.js +2 -0
  51. package/src/inference/pipelines/text/generator.js +29 -0
  52. package/src/inference/pipelines/text/lora-apply.js +4 -4
  53. package/src/inference/pipelines/text/moe-cpu.js +1 -0
  54. package/src/inference/pipelines/text/moe-gpu.js +1 -0
  55. package/src/inference/tokenizers/bundled.js +105 -32
  56. package/src/rules/kernels/matmul.rules.json +1 -0
  57. package/src/tooling/node-converter.js +20 -5
  58. package/src/tooling-experimental-exports.shared.d.ts +25 -0
  59. package/src/tooling-experimental-exports.shared.js +18 -0
  60. package/src/version.js +1 -1
@@ -30,6 +30,10 @@ fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
30
30
  let token_idx = idx / u.vocab_size;
31
31
  let class_idx = idx % u.vocab_size;
32
32
  let target_idx = targets[token_idx];
33
+ if (target_idx >= u.vocab_size) {
34
+ output[idx] = 0.0;
35
+ return;
36
+ }
33
37
  var grad = softmax[idx];
34
38
  if (class_idx == target_idx) {
35
39
  grad = grad - 1.0;
@@ -0,0 +1,39 @@
1
+ import type { Tensor } from '../../tensor.js';
2
+
3
+ export interface GatedDeltaRecurrentBackwardInputs {
4
+ query: Tensor;
5
+ key: Tensor;
6
+ value: Tensor;
7
+ logDecay: Tensor;
8
+ beta: Tensor;
9
+ stateHistory: Tensor;
10
+ gradOutput: Tensor;
11
+ }
12
+
13
+ export interface GatedDeltaRecurrentBackwardOptions {
14
+ numTokens: number;
15
+ numHeads: number;
16
+ keyDim: number;
17
+ valueDim: number;
18
+ queryScale: number;
19
+ gradQueryBuffer?: GPUBuffer | null;
20
+ gradKeyBuffer?: GPUBuffer | null;
21
+ gradValueBuffer?: GPUBuffer | null;
22
+ gradLogDecayBuffer?: GPUBuffer | null;
23
+ gradBetaBuffer?: GPUBuffer | null;
24
+ gradStateBuffer?: GPUBuffer | null;
25
+ }
26
+
27
+ export interface GatedDeltaRecurrentBackwardResult {
28
+ query: Tensor;
29
+ key: Tensor;
30
+ value: Tensor;
31
+ logDecay: Tensor;
32
+ beta: Tensor;
33
+ initialState: Tensor;
34
+ }
35
+
36
+ export declare function runGatedDeltaRecurrentBackward(
37
+ inputs: GatedDeltaRecurrentBackwardInputs,
38
+ options: GatedDeltaRecurrentBackwardOptions
39
+ ): Promise<GatedDeltaRecurrentBackwardResult>;
@@ -0,0 +1,124 @@
1
+ import { getDevice } from '../../device.js';
2
+ import { acquireBuffer, releaseBuffer } from '../../../memory/buffer-pool.js';
3
+ import { createTensor } from '../../tensor.js';
4
+ import { dispatch } from '../dispatch.js';
5
+ import { createPipeline, createUniformBufferWithView } from '../utils.js';
6
+ import { releaseUniformBuffer } from '../../uniform-cache.js';
7
+
8
+ function validate(inputs, options) {
9
+ const numTokens = Math.floor(Number(options?.numTokens));
10
+ const numHeads = Math.floor(Number(options?.numHeads));
11
+ const keyDim = Math.floor(Number(options?.keyDim));
12
+ const valueDim = Math.floor(Number(options?.valueDim));
13
+ const queryScale = Number(options?.queryScale);
14
+ if (numTokens < 1 || numHeads < 1 || keyDim < 1 || valueDim < 1 || valueDim > 128) {
15
+ throw new Error('gated-delta recurrent backward requires positive dimensions and valueDim <= 128.');
16
+ }
17
+ if (!Number.isFinite(queryScale)) {
18
+ throw new Error('gated-delta recurrent backward requires finite queryScale.');
19
+ }
20
+ for (const [label, tensor] of Object.entries(inputs)) {
21
+ if (tensor?.dtype !== 'f32') {
22
+ throw new Error(`gated-delta recurrent backward requires f32 ${label}.`);
23
+ }
24
+ }
25
+ return { numTokens, numHeads, keyDim, valueDim, queryScale };
26
+ }
27
+
28
+ function allocate(bytes, label, provided) {
29
+ return provided || acquireBuffer(bytes, undefined, label);
30
+ }
31
+
32
+ export async function runGatedDeltaRecurrentBackward(inputs, options = {}) {
33
+ const dims = validate(inputs, options);
34
+ const device = getDevice();
35
+ if (!device) throw new Error('gated-delta recurrent backward requires an active GPU device.');
36
+ const queryBytes = dims.numTokens * dims.numHeads * dims.keyDim * 4;
37
+ const valueBytes = dims.numTokens * dims.numHeads * dims.valueDim * 4;
38
+ const scalarBytes = dims.numTokens * dims.numHeads * 4;
39
+ const stateBytes = dims.numHeads * dims.keyDim * dims.valueDim * 4;
40
+ const packedQueryKeyBuffer = acquireBuffer(queryBytes * 2, undefined, 'gated_delta_packed_query_key');
41
+ const packedDecayBetaBuffer = acquireBuffer(scalarBytes * 2, undefined, 'gated_delta_packed_decay_beta');
42
+ const packedGradQueryKeyBuffer = acquireBuffer(queryBytes * 2, undefined, 'gated_delta_packed_grad_query_key');
43
+ const packedGradDecayBetaBuffer = acquireBuffer(scalarBytes * 2, undefined, 'gated_delta_packed_grad_decay_beta');
44
+ const gradQueryBuffer = allocate(queryBytes, 'gated_delta_grad_query', options.gradQueryBuffer);
45
+ const gradKeyBuffer = allocate(queryBytes, 'gated_delta_grad_key', options.gradKeyBuffer);
46
+ const gradValueBuffer = allocate(valueBytes, 'gated_delta_grad_value', options.gradValueBuffer);
47
+ const gradLogDecayBuffer = allocate(scalarBytes, 'gated_delta_grad_log_decay', options.gradLogDecayBuffer);
48
+ const gradBetaBuffer = allocate(scalarBytes, 'gated_delta_grad_beta', options.gradBetaBuffer);
49
+ const gradStateBuffer = allocate(stateBytes, 'gated_delta_grad_initial_state', options.gradStateBuffer);
50
+ let uniformBuffer = null;
51
+ let completed = false;
52
+ try {
53
+ const packEncoder = device.createCommandEncoder({ label: 'gated_delta_backward_pack_inputs' });
54
+ packEncoder.copyBufferToBuffer(inputs.query.buffer, 0, packedQueryKeyBuffer, 0, queryBytes);
55
+ packEncoder.copyBufferToBuffer(inputs.key.buffer, 0, packedQueryKeyBuffer, queryBytes, queryBytes);
56
+ packEncoder.copyBufferToBuffer(inputs.logDecay.buffer, 0, packedDecayBetaBuffer, 0, scalarBytes);
57
+ packEncoder.copyBufferToBuffer(inputs.beta.buffer, 0, packedDecayBetaBuffer, scalarBytes, scalarBytes);
58
+ device.queue.submit([packEncoder.finish()]);
59
+ device.queue.writeBuffer(gradStateBuffer, 0, new Uint8Array(stateBytes));
60
+
61
+ const pipeline = await createPipeline('gated_delta_recurrent_backward', 'default');
62
+ uniformBuffer = createUniformBufferWithView(
63
+ 'gated_delta_recurrent_backward_uniforms',
64
+ 32,
65
+ (view) => {
66
+ view.setUint32(0, dims.numTokens, true);
67
+ view.setUint32(4, dims.numHeads, true);
68
+ view.setUint32(8, dims.keyDim, true);
69
+ view.setUint32(12, dims.valueDim, true);
70
+ view.setFloat32(16, dims.queryScale, true);
71
+ },
72
+ null,
73
+ device
74
+ );
75
+ const bindGroup = device.createBindGroup({
76
+ label: 'gated_delta_recurrent_backward_bind_group',
77
+ layout: pipeline.getBindGroupLayout(0),
78
+ entries: [
79
+ { binding: 0, resource: { buffer: uniformBuffer } },
80
+ { binding: 1, resource: { buffer: packedQueryKeyBuffer } },
81
+ { binding: 2, resource: { buffer: inputs.value.buffer } },
82
+ { binding: 3, resource: { buffer: packedDecayBetaBuffer } },
83
+ { binding: 4, resource: { buffer: inputs.stateHistory.buffer } },
84
+ { binding: 5, resource: { buffer: inputs.gradOutput.buffer } },
85
+ { binding: 6, resource: { buffer: packedGradQueryKeyBuffer } },
86
+ { binding: 7, resource: { buffer: gradValueBuffer } },
87
+ { binding: 8, resource: { buffer: packedGradDecayBetaBuffer } },
88
+ { binding: 9, resource: { buffer: gradStateBuffer } },
89
+ ],
90
+ });
91
+ dispatch(device, pipeline, bindGroup, dims.numHeads, 'gated_delta_recurrent_backward');
92
+ const unpackEncoder = device.createCommandEncoder({ label: 'gated_delta_backward_unpack_outputs' });
93
+ unpackEncoder.copyBufferToBuffer(packedGradQueryKeyBuffer, 0, gradQueryBuffer, 0, queryBytes);
94
+ unpackEncoder.copyBufferToBuffer(packedGradQueryKeyBuffer, queryBytes, gradKeyBuffer, 0, queryBytes);
95
+ unpackEncoder.copyBufferToBuffer(packedGradDecayBetaBuffer, 0, gradLogDecayBuffer, 0, scalarBytes);
96
+ unpackEncoder.copyBufferToBuffer(packedGradDecayBetaBuffer, scalarBytes, gradBetaBuffer, 0, scalarBytes);
97
+ device.queue.submit([unpackEncoder.finish()]);
98
+ await device.queue.onSubmittedWorkDone();
99
+ completed = true;
100
+ return {
101
+ query: createTensor(gradQueryBuffer, 'f32', [dims.numTokens, dims.numHeads, dims.keyDim], 'gated_delta_grad_query'),
102
+ key: createTensor(gradKeyBuffer, 'f32', [dims.numTokens, dims.numHeads, dims.keyDim], 'gated_delta_grad_key'),
103
+ value: createTensor(gradValueBuffer, 'f32', [dims.numTokens, dims.numHeads, dims.valueDim], 'gated_delta_grad_value'),
104
+ logDecay: createTensor(gradLogDecayBuffer, 'f32', [dims.numTokens, dims.numHeads], 'gated_delta_grad_log_decay'),
105
+ beta: createTensor(gradBetaBuffer, 'f32', [dims.numTokens, dims.numHeads], 'gated_delta_grad_beta'),
106
+ initialState: createTensor(gradStateBuffer, 'f32', [dims.numHeads, dims.keyDim, dims.valueDim], 'gated_delta_grad_initial_state'),
107
+ };
108
+ } finally {
109
+ await device.queue.onSubmittedWorkDone().catch(() => {});
110
+ if (uniformBuffer) releaseUniformBuffer(uniformBuffer);
111
+ releaseBuffer(packedQueryKeyBuffer);
112
+ releaseBuffer(packedDecayBetaBuffer);
113
+ releaseBuffer(packedGradQueryKeyBuffer);
114
+ releaseBuffer(packedGradDecayBetaBuffer);
115
+ if (!completed) {
116
+ if (!options.gradQueryBuffer) releaseBuffer(gradQueryBuffer);
117
+ if (!options.gradKeyBuffer) releaseBuffer(gradKeyBuffer);
118
+ if (!options.gradValueBuffer) releaseBuffer(gradValueBuffer);
119
+ if (!options.gradLogDecayBuffer) releaseBuffer(gradLogDecayBuffer);
120
+ if (!options.gradBetaBuffer) releaseBuffer(gradBetaBuffer);
121
+ if (!options.gradStateBuffer) releaseBuffer(gradStateBuffer);
122
+ }
123
+ }
124
+ }
@@ -0,0 +1,177 @@
1
+ override WORKGROUP_SIZE: u32 = 128u;
2
+ const MAX_WORKGROUP_SIZE: u32 = 128u;
3
+
4
+ struct Uniforms {
5
+ num_tokens: u32,
6
+ num_heads: u32,
7
+ key_dim: u32,
8
+ value_dim: u32,
9
+ query_scale: f32,
10
+ _pad0: u32,
11
+ _pad1: u32,
12
+ _pad2: u32,
13
+ }
14
+
15
+ @group(0) @binding(0) var<uniform> u: Uniforms;
16
+ @group(0) @binding(1) var<storage, read> query_key: array<f32>;
17
+ @group(0) @binding(2) var<storage, read> value: array<f32>;
18
+ @group(0) @binding(3) var<storage, read> decay_beta: array<f32>;
19
+ @group(0) @binding(4) var<storage, read> state_history: array<f32>;
20
+ @group(0) @binding(5) var<storage, read> grad_output: array<f32>;
21
+ @group(0) @binding(6) var<storage, read_write> grad_query_key: array<f32>;
22
+ @group(0) @binding(7) var<storage, read_write> grad_value: array<f32>;
23
+ @group(0) @binding(8) var<storage, read_write> grad_decay_beta: array<f32>;
24
+ @group(0) @binding(9) var<storage, read_write> grad_state: array<f32>;
25
+
26
+ var<workgroup> shared_reduce: array<f32, MAX_WORKGROUP_SIZE>;
27
+
28
+ fn vector_index(token: u32, head: u32, dim: u32, width: u32) -> u32 {
29
+ return ((token * u.num_heads + head) * width) + dim;
30
+ }
31
+
32
+ fn state_history_index(token: u32, head: u32, key_index: u32, value_index: u32) -> u32 {
33
+ return ((((token * u.num_heads + head) * u.key_dim) + key_index) * u.value_dim) + value_index;
34
+ }
35
+
36
+ fn state_gradient_index(head: u32, key_index: u32, value_index: u32) -> u32 {
37
+ return (((head * u.key_dim) + key_index) * u.value_dim) + value_index;
38
+ }
39
+
40
+ @compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
41
+ fn main(
42
+ @builtin(workgroup_id) wid: vec3<u32>,
43
+ @builtin(local_invocation_id) lid: vec3<u32>
44
+ ) {
45
+ let head = wid.x;
46
+ let lane = lid.x;
47
+ if (head >= u.num_heads) {
48
+ return;
49
+ }
50
+ let is_active = lane < u.value_dim;
51
+ let query_elements = u.num_tokens * u.num_heads * u.key_dim;
52
+ let scalar_elements = u.num_tokens * u.num_heads;
53
+
54
+ for (var reverse_token: u32 = 0u; reverse_token < u.num_tokens; reverse_token = reverse_token + 1u) {
55
+ let token = u.num_tokens - 1u - reverse_token;
56
+ let scalar_index = token * u.num_heads + head;
57
+ let decay = exp(decay_beta[scalar_index]);
58
+ var memory: f32 = 0.0;
59
+ if (is_active) {
60
+ for (var key_index: u32 = 0u; key_index < u.key_dim; key_index = key_index + 1u) {
61
+ let previous_state = state_history[
62
+ state_history_index(token, head, key_index, lane)
63
+ ];
64
+ memory = memory
65
+ + previous_state
66
+ * decay
67
+ * query_key[query_elements + vector_index(token, head, key_index, u.key_dim)];
68
+ }
69
+ }
70
+ let value_offset = vector_index(token, head, lane, u.value_dim);
71
+ let value_at_token = select(0.0, value[value_offset], is_active);
72
+ let delta = (value_at_token - memory) * decay_beta[scalar_elements + scalar_index];
73
+ let output_gradient = select(0.0, grad_output[value_offset], is_active);
74
+
75
+ for (var key_index: u32 = 0u; key_index < u.key_dim; key_index = key_index + 1u) {
76
+ let query_offset = vector_index(token, head, key_index, u.key_dim);
77
+ let next_state = select(
78
+ 0.0,
79
+ state_history[state_history_index(token + 1u, head, key_index, lane)],
80
+ is_active
81
+ );
82
+ shared_reduce[lane] = output_gradient * next_state * u.query_scale;
83
+ workgroupBarrier();
84
+ for (var stride: u32 = WORKGROUP_SIZE / 2u; stride > 0u; stride = stride / 2u) {
85
+ if (lane < stride) {
86
+ shared_reduce[lane] = shared_reduce[lane] + shared_reduce[lane + stride];
87
+ }
88
+ workgroupBarrier();
89
+ }
90
+ if (lane == 0u) {
91
+ grad_query_key[query_offset] = shared_reduce[0];
92
+ }
93
+ if (is_active) {
94
+ let gradient_state_offset = state_gradient_index(head, key_index, lane);
95
+ grad_state[gradient_state_offset] = grad_state[gradient_state_offset]
96
+ + output_gradient
97
+ * query_key[query_offset]
98
+ * u.query_scale;
99
+ }
100
+ workgroupBarrier();
101
+ }
102
+
103
+ var grad_delta: f32 = 0.0;
104
+ if (is_active) {
105
+ for (var key_index: u32 = 0u; key_index < u.key_dim; key_index = key_index + 1u) {
106
+ let gradient_state_offset = state_gradient_index(head, key_index, lane);
107
+ grad_delta = grad_delta
108
+ + grad_state[gradient_state_offset]
109
+ * query_key[query_elements + vector_index(token, head, key_index, u.key_dim)];
110
+ }
111
+ grad_value[value_offset] = grad_delta * decay_beta[scalar_elements + scalar_index];
112
+ }
113
+ let grad_memory = -grad_delta * decay_beta[scalar_elements + scalar_index];
114
+ shared_reduce[lane] = select(0.0, grad_delta * (value_at_token - memory), is_active);
115
+ workgroupBarrier();
116
+ for (var stride: u32 = WORKGROUP_SIZE / 2u; stride > 0u; stride = stride / 2u) {
117
+ if (lane < stride) {
118
+ shared_reduce[lane] = shared_reduce[lane] + shared_reduce[lane + stride];
119
+ }
120
+ workgroupBarrier();
121
+ }
122
+ if (lane == 0u) {
123
+ grad_decay_beta[scalar_elements + scalar_index] = shared_reduce[0];
124
+ }
125
+ workgroupBarrier();
126
+
127
+ var local_grad_decay: f32 = 0.0;
128
+ for (var key_index: u32 = 0u; key_index < u.key_dim; key_index = key_index + 1u) {
129
+ let key_offset = vector_index(token, head, key_index, u.key_dim);
130
+ let gradient_state_offset = state_gradient_index(head, key_index, lane);
131
+ let previous_state = select(
132
+ 0.0,
133
+ state_history[state_history_index(token, head, key_index, lane)],
134
+ is_active
135
+ );
136
+ let decayed_state = previous_state * decay;
137
+ let gradient_before_decay_update = select(0.0, grad_state[gradient_state_offset], is_active);
138
+
139
+ shared_reduce[lane] = select(
140
+ 0.0,
141
+ (gradient_before_decay_update * delta) + (grad_memory * decayed_state),
142
+ is_active
143
+ );
144
+ workgroupBarrier();
145
+ for (var stride: u32 = WORKGROUP_SIZE / 2u; stride > 0u; stride = stride / 2u) {
146
+ if (lane < stride) {
147
+ shared_reduce[lane] = shared_reduce[lane] + shared_reduce[lane + stride];
148
+ }
149
+ workgroupBarrier();
150
+ }
151
+ if (lane == 0u) {
152
+ grad_query_key[query_elements + key_offset] = shared_reduce[0];
153
+ }
154
+
155
+ if (is_active) {
156
+ let gradient_decayed_state = gradient_before_decay_update
157
+ + grad_memory * query_key[query_elements + key_offset];
158
+ local_grad_decay = local_grad_decay + gradient_decayed_state * decayed_state;
159
+ grad_state[gradient_state_offset] = gradient_decayed_state * decay;
160
+ }
161
+ workgroupBarrier();
162
+ }
163
+
164
+ shared_reduce[lane] = select(0.0, local_grad_decay, is_active);
165
+ workgroupBarrier();
166
+ for (var stride: u32 = WORKGROUP_SIZE / 2u; stride > 0u; stride = stride / 2u) {
167
+ if (lane < stride) {
168
+ shared_reduce[lane] = shared_reduce[lane] + shared_reduce[lane + stride];
169
+ }
170
+ workgroupBarrier();
171
+ }
172
+ if (lane == 0u) {
173
+ grad_decay_beta[scalar_index] = shared_reduce[0];
174
+ }
175
+ workgroupBarrier();
176
+ }
177
+ }
@@ -0,0 +1,32 @@
1
+ import type { CommandRecorder } from '../../command-recorder.js';
2
+ import type { Tensor } from '../../tensor.js';
3
+
4
+ export interface GatedRmsNormBackwardOptions {
5
+ rows: number;
6
+ width: number;
7
+ eps: number;
8
+ gradInputBuffer?: GPUBuffer | null;
9
+ gradGateBuffer?: GPUBuffer | null;
10
+ }
11
+
12
+ export interface GatedRmsNormBackwardResult {
13
+ gradInput: Tensor;
14
+ gradGate: Tensor;
15
+ }
16
+
17
+ export declare function runGatedRmsNormBackward(
18
+ input: Tensor,
19
+ gate: Tensor,
20
+ weight: Tensor,
21
+ gradOutput: Tensor,
22
+ options: GatedRmsNormBackwardOptions
23
+ ): Promise<GatedRmsNormBackwardResult>;
24
+
25
+ export declare function recordGatedRmsNormBackward(
26
+ recorder: CommandRecorder,
27
+ input: Tensor,
28
+ gate: Tensor,
29
+ weight: Tensor,
30
+ gradOutput: Tensor,
31
+ options: GatedRmsNormBackwardOptions
32
+ ): Promise<GatedRmsNormBackwardResult>;
@@ -0,0 +1,90 @@
1
+ import { getDevice } from '../../device.js';
2
+ import { acquireBuffer } from '../../../memory/buffer-pool.js';
3
+ import { createTensor } from '../../tensor.js';
4
+ import { dispatch, recordDispatch } from '../dispatch.js';
5
+ import { createPipeline, createUniformBufferWithView } from '../utils.js';
6
+ import { releaseUniformBuffer } from '../../uniform-cache.js';
7
+
8
+ function validate(input, gate, weight, gradOutput, options) {
9
+ const rows = Math.floor(Number(options?.rows));
10
+ const width = Math.floor(Number(options?.width));
11
+ const eps = Number(options?.eps);
12
+ if (rows < 1 || width < 1 || !Number.isFinite(eps) || eps <= 0) {
13
+ throw new Error('gated RMSNorm backward requires rows, width, and positive eps.');
14
+ }
15
+ for (const [label, tensor] of [
16
+ ['input', input],
17
+ ['gate', gate],
18
+ ['weight', weight],
19
+ ['gradOutput', gradOutput],
20
+ ]) {
21
+ if (tensor?.dtype !== 'f32') {
22
+ throw new Error(`gated RMSNorm backward requires f32 ${label}.`);
23
+ }
24
+ }
25
+ return { rows, width, eps };
26
+ }
27
+
28
+ async function execute(recorder, input, gate, weight, gradOutput, options = {}) {
29
+ const dims = validate(input, gate, weight, gradOutput, options);
30
+ const device = recorder?.device || getDevice();
31
+ if (!device) throw new Error('gated RMSNorm backward requires an active GPU device.');
32
+ const outputBytes = dims.rows * dims.width * 4;
33
+ const gradInputBuffer = options.gradInputBuffer
34
+ || acquireBuffer(outputBytes, undefined, 'gated_rmsnorm_backward_input');
35
+ const gradGateBuffer = options.gradGateBuffer
36
+ || acquireBuffer(outputBytes, undefined, 'gated_rmsnorm_backward_gate');
37
+ const pipeline = await createPipeline('gated_rmsnorm_backward', 'default');
38
+ const uniformBuffer = createUniformBufferWithView(
39
+ 'gated_rmsnorm_backward_uniforms',
40
+ 16,
41
+ (view) => {
42
+ view.setUint32(0, dims.rows, true);
43
+ view.setUint32(4, dims.width, true);
44
+ view.setFloat32(8, dims.eps, true);
45
+ },
46
+ recorder || null,
47
+ device
48
+ );
49
+ const bindGroup = device.createBindGroup({
50
+ label: 'gated_rmsnorm_backward_bind_group',
51
+ layout: pipeline.getBindGroupLayout(0),
52
+ entries: [
53
+ { binding: 0, resource: { buffer: uniformBuffer } },
54
+ { binding: 1, resource: { buffer: input.buffer } },
55
+ { binding: 2, resource: { buffer: gate.buffer } },
56
+ { binding: 3, resource: { buffer: weight.buffer } },
57
+ { binding: 4, resource: { buffer: gradOutput.buffer } },
58
+ { binding: 5, resource: { buffer: gradInputBuffer } },
59
+ { binding: 6, resource: { buffer: gradGateBuffer } },
60
+ ],
61
+ });
62
+ if (recorder) {
63
+ recordDispatch(recorder, pipeline, bindGroup, dims.rows, 'gated_rmsnorm_backward');
64
+ } else {
65
+ dispatch(device, pipeline, bindGroup, dims.rows, 'gated_rmsnorm_backward');
66
+ releaseUniformBuffer(uniformBuffer);
67
+ }
68
+ return {
69
+ gradInput: createTensor(
70
+ gradInputBuffer,
71
+ 'f32',
72
+ [dims.rows, dims.width],
73
+ 'gated_rmsnorm_backward_input'
74
+ ),
75
+ gradGate: createTensor(
76
+ gradGateBuffer,
77
+ 'f32',
78
+ [dims.rows, dims.width],
79
+ 'gated_rmsnorm_backward_gate'
80
+ ),
81
+ };
82
+ }
83
+
84
+ export function runGatedRmsNormBackward(input, gate, weight, gradOutput, options = {}) {
85
+ return execute(null, input, gate, weight, gradOutput, options);
86
+ }
87
+
88
+ export function recordGatedRmsNormBackward(recorder, input, gate, weight, gradOutput, options = {}) {
89
+ return execute(recorder, input, gate, weight, gradOutput, options);
90
+ }
@@ -0,0 +1,86 @@
1
+ override WORKGROUP_SIZE: u32 = 256u;
2
+ const MAX_WORKGROUP_SIZE: u32 = 256u;
3
+
4
+ struct Uniforms {
5
+ rows: u32,
6
+ width: u32,
7
+ eps: f32,
8
+ _pad0: u32,
9
+ }
10
+
11
+ @group(0) @binding(0) var<uniform> u: Uniforms;
12
+ @group(0) @binding(1) var<storage, read> input: array<f32>;
13
+ @group(0) @binding(2) var<storage, read> gate: array<f32>;
14
+ @group(0) @binding(3) var<storage, read> weight: array<f32>;
15
+ @group(0) @binding(4) var<storage, read> grad_output: array<f32>;
16
+ @group(0) @binding(5) var<storage, read_write> grad_input: array<f32>;
17
+ @group(0) @binding(6) var<storage, read_write> grad_gate: array<f32>;
18
+
19
+ var<workgroup> shared_sum_sq: array<f32, MAX_WORKGROUP_SIZE>;
20
+ var<workgroup> shared_sum_gx: array<f32, MAX_WORKGROUP_SIZE>;
21
+
22
+ fn sigmoid(x: f32) -> f32 {
23
+ if (x >= 0.0) {
24
+ let z = exp(-x);
25
+ return 1.0 / (1.0 + z);
26
+ }
27
+ let z = exp(x);
28
+ return z / (1.0 + z);
29
+ }
30
+
31
+ @compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
32
+ fn main(
33
+ @builtin(workgroup_id) wid: vec3<u32>,
34
+ @builtin(local_invocation_id) lid: vec3<u32>
35
+ ) {
36
+ let row = wid.x;
37
+ if (row >= u.rows) {
38
+ return;
39
+ }
40
+ let local_id = lid.x;
41
+ let base = row * u.width;
42
+ var sum_sq: f32 = 0.0;
43
+ var sum_gx: f32 = 0.0;
44
+
45
+ for (var column = local_id; column < u.width; column = column + WORKGROUP_SIZE) {
46
+ let offset = base + column;
47
+ let x = input[offset];
48
+ let probability = sigmoid(gate[offset]);
49
+ let gate_value = gate[offset] * probability;
50
+ let grad_normalized = grad_output[offset] * weight[column] * gate_value;
51
+ sum_sq = sum_sq + x * x;
52
+ sum_gx = sum_gx + grad_normalized * x;
53
+ }
54
+
55
+ shared_sum_sq[local_id] = sum_sq;
56
+ shared_sum_gx[local_id] = sum_gx;
57
+ workgroupBarrier();
58
+
59
+ var stride = WORKGROUP_SIZE / 2u;
60
+ loop {
61
+ if (stride == 0u) {
62
+ break;
63
+ }
64
+ if (local_id < stride) {
65
+ shared_sum_sq[local_id] = shared_sum_sq[local_id] + shared_sum_sq[local_id + stride];
66
+ shared_sum_gx[local_id] = shared_sum_gx[local_id] + shared_sum_gx[local_id + stride];
67
+ }
68
+ workgroupBarrier();
69
+ stride = stride / 2u;
70
+ }
71
+
72
+ let inverse_rms = inverseSqrt(shared_sum_sq[0] / f32(u.width) + u.eps);
73
+ let correction = (shared_sum_gx[0] * inverse_rms * inverse_rms) / f32(u.width);
74
+ for (var column = local_id; column < u.width; column = column + WORKGROUP_SIZE) {
75
+ let offset = base + column;
76
+ let x = input[offset];
77
+ let gate_input = gate[offset];
78
+ let probability = sigmoid(gate_input);
79
+ let gate_value = gate_input * probability;
80
+ let gate_derivative = probability * (1.0 + gate_input * (1.0 - probability));
81
+ let grad = grad_output[offset];
82
+ let grad_normalized = grad * weight[column] * gate_value;
83
+ grad_input[offset] = inverse_rms * (grad_normalized - x * correction);
84
+ grad_gate[offset] = grad * x * inverse_rms * weight[column] * gate_derivative;
85
+ }
86
+ }
@@ -3,6 +3,9 @@ export { runMatmulBackward, recordMatmulBackward } from './matmul_backward.js';
3
3
  export { runSoftmaxBackward, recordSoftmaxBackward } from './softmax_backward.js';
4
4
  export { runRmsNormBackward, recordRmsNormBackward } from './rmsnorm_backward.js';
5
5
  export { runRmsNormBackward as runRmsnormBackward, recordRmsNormBackward as recordRmsnormBackward } from './rmsnorm_backward.js';
6
+ export { runGatedRmsNormBackward, recordGatedRmsNormBackward } from './gated_rmsnorm_backward.js';
7
+ export { runCausalConv1dSiluBackward, recordCausalConv1dSiluBackward } from './causal_conv1d_silu_backward.js';
8
+ export { runGatedDeltaRecurrentBackward } from './gated_delta_recurrent_backward.js';
6
9
  export { runLayerNormBackward, recordLayerNormBackward } from './layernorm_backward.js';
7
10
  export { runLayerNormBackward as runLayernormBackward, recordLayerNormBackward as recordLayernormBackward } from './layernorm_backward.js';
8
11
  export { runAttentionBackward, recordAttentionBackward } from './attention_backward.js';
@@ -3,6 +3,9 @@ export { runMatmulBackward, recordMatmulBackward } from './matmul_backward.js';
3
3
  export { runSoftmaxBackward, recordSoftmaxBackward } from './softmax_backward.js';
4
4
  export { runRmsNormBackward, recordRmsNormBackward } from './rmsnorm_backward.js';
5
5
  export { runRmsNormBackward as runRmsnormBackward, recordRmsNormBackward as recordRmsnormBackward } from './rmsnorm_backward.js';
6
+ export { runGatedRmsNormBackward, recordGatedRmsNormBackward } from './gated_rmsnorm_backward.js';
7
+ export { runCausalConv1dSiluBackward, recordCausalConv1dSiluBackward } from './causal_conv1d_silu_backward.js';
8
+ export { runGatedDeltaRecurrentBackward } from './gated_delta_recurrent_backward.js';
6
9
  export { runLayerNormBackward, recordLayerNormBackward } from './layernorm_backward.js';
7
10
  export { runLayerNormBackward as runLayernormBackward, recordLayerNormBackward as recordLayernormBackward } from './layernorm_backward.js';
8
11
  export { runAttentionBackward, recordAttentionBackward } from './attention_backward.js';
@@ -2,10 +2,10 @@
2
2
  //
3
3
  // Fused tiled matmul for computing input gradients during backpropagation.
4
4
  //
5
- // When transpose_b = 1 (standard case: forward was Y = X @ W, W stored [K,N]):
5
+ // When transpose_b = 0 (standard case: forward was Y = X @ W, W stored [K,N]):
6
6
  // dX[M,K] = dY[M,N] @ W^T[N,K] (reads W as [K,N], transposes on the fly)
7
7
  //
8
- // When transpose_b = 0 (forward was Y = X @ W^T, W stored [N,K]):
8
+ // When transpose_b = 1 (forward was Y = X @ W^T, W stored [N,K]):
9
9
  // dX[M,K] = dY[M,N] @ W[N,K] (reads W directly as [N,K])
10
10
  //
11
11
  // Uses 16x16 shared memory tiles, matching matmul_transpose_a.wgsl style.
@@ -20,7 +20,7 @@ struct Uniforms {
20
20
  N: u32, // Cols of dY (output dim of forward)
21
21
  K: u32, // Cols of dX (input dim of forward)
22
22
  alpha: f32,
23
- transpose_b: u32, // 0 = W stored [N,K], 1 = W stored [K,N]
23
+ transpose_b: u32, // 0 = W stored [K,N], 1 = W stored [N,K]
24
24
  _pad0: u32,
25
25
  _pad1: u32,
26
26
  _pad2: u32,
@@ -65,11 +65,11 @@ fn main(
65
65
  let wt_row = tile_n + local_row;
66
66
  if (wt_row < u.N && col < u.K) {
67
67
  if (u.transpose_b == 0u) {
68
- // W stored as [N, K]: W^T[wt_row, col] = W[wt_row, col]
69
- tileW[tile_idx] = W[wt_row * u.K + col];
70
- } else {
71
68
  // W stored as [K, N]: W^T[wt_row, col] = W[col, wt_row]
72
69
  tileW[tile_idx] = W[col * u.N + wt_row];
70
+ } else {
71
+ // W stored as [N, K]: W^T[wt_row, col] = W[wt_row, col]
72
+ tileW[tile_idx] = W[wt_row * u.K + col];
73
73
  }
74
74
  } else {
75
75
  tileW[tile_idx] = 0.0;