doppler-gpu 0.4.8 → 0.4.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. package/CHANGELOG.md +29 -0
  2. package/README.md +62 -12
  3. package/models/catalog.json +21 -7
  4. package/package.json +25 -1
  5. package/src/config/conversion/qwen3/qwen-3-5-9b-f16-af32.json +474 -0
  6. package/src/config/conversion/qwen3/qwen-3-5-9b-q4k-ehaf16.json +509 -0
  7. package/src/config/kernels/backward-registry.json +13 -2
  8. package/src/config/kernels/kernel-ref-digests.js +8 -4
  9. package/src/config/kernels/registry.json +1035 -2677
  10. package/src/config/runtime/profiles/qwen-3-5-9b-metal-correctness.json +46 -0
  11. package/src/config/schema/training-metrics.schema.js +4 -0
  12. package/src/experimental/bridge/trainer-artifact-bridge.d.ts +161 -0
  13. package/src/experimental/bridge/trainer-artifact-bridge.js +573 -0
  14. package/src/experimental/training/attention-backward.js +123 -55
  15. package/src/experimental/training/autograd.js +101 -5
  16. package/src/experimental/training/clip.js +6 -1
  17. package/src/experimental/training/datasets/text-pairs.js +80 -8
  18. package/src/experimental/training/distillation/student-fixture.js +470 -114
  19. package/src/experimental/training/lora-pipeline.js +202 -27
  20. package/src/experimental/training/lora.js +10 -4
  21. package/src/experimental/training/optimizer.js +9 -1
  22. package/src/experimental/training/runner.js +18 -4
  23. package/src/experimental/training/workloads.js +11 -0
  24. package/src/gpu/device.js +100 -32
  25. package/src/gpu/kernels/backward/adam.js +6 -4
  26. package/src/gpu/kernels/backward/causal_conv1d_silu_backward.d.ts +24 -0
  27. package/src/gpu/kernels/backward/causal_conv1d_silu_backward.js +74 -0
  28. package/src/gpu/kernels/backward/causal_conv1d_silu_backward.wgsl +59 -0
  29. package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +4 -0
  30. package/src/gpu/kernels/backward/gated_delta_recurrent_backward.d.ts +39 -0
  31. package/src/gpu/kernels/backward/gated_delta_recurrent_backward.js +124 -0
  32. package/src/gpu/kernels/backward/gated_delta_recurrent_backward.wgsl +177 -0
  33. package/src/gpu/kernels/backward/gated_rmsnorm_backward.d.ts +32 -0
  34. package/src/gpu/kernels/backward/gated_rmsnorm_backward.js +90 -0
  35. package/src/gpu/kernels/backward/gated_rmsnorm_backward.wgsl +86 -0
  36. package/src/gpu/kernels/backward/index.d.ts +3 -0
  37. package/src/gpu/kernels/backward/index.js +3 -0
  38. package/src/gpu/kernels/backward/matmul_backward.wgsl +6 -6
  39. package/src/gpu/kernels/backward/matmul_backward_f16w.wgsl +82 -0
  40. package/src/gpu/kernels/backward/rmsnorm_backward.js +26 -4
  41. package/src/gpu/kernels/backward/rmsnorm_backward.wgsl +6 -3
  42. package/src/gpu/kernels/backward/utils.d.ts +12 -2
  43. package/src/gpu/kernels/backward/utils.js +9 -2
  44. package/src/gpu/kernels/cast_f16_to_f32.wgsl +6 -2
  45. package/src/gpu/kernels/index.d.ts +5 -0
  46. package/src/gpu/kernels/index.js +5 -0
  47. package/src/gpu/kernels/matmul-selection.js +5 -0
  48. package/src/gpu/kernels/silu.d.ts +1 -0
  49. package/src/gpu/kernels/silu.js +19 -4
  50. package/src/inference/pipelines/text/embed.js +1 -1
  51. package/src/inference/pipelines/text/execution-v1.js +8 -2
  52. package/src/inference/pipelines/text/ffn/dense.js +2 -0
  53. package/src/inference/pipelines/text/generator.js +29 -0
  54. package/src/inference/pipelines/text/lora-apply.js +4 -4
  55. package/src/inference/pipelines/text/moe-cpu.js +1 -0
  56. package/src/inference/pipelines/text/moe-gpu.js +1 -0
  57. package/src/inference/tokenizers/bundled.js +105 -32
  58. package/src/rules/kernels/matmul.rules.json +1 -0
  59. package/src/tooling/node-converter.js +20 -5
  60. package/src/tooling/runtime-optimization.d.ts +109 -0
  61. package/src/tooling/runtime-optimization.js +872 -0
  62. package/src/tooling-experimental-exports.shared.d.ts +25 -0
  63. package/src/tooling-experimental-exports.shared.js +18 -0
  64. package/src/tooling-exports.shared.d.ts +19 -0
  65. package/src/tooling-exports.shared.js +11 -0
  66. package/src/version.js +1 -1
@@ -0,0 +1,177 @@
1
+ override WORKGROUP_SIZE: u32 = 128u;
2
+ const MAX_WORKGROUP_SIZE: u32 = 128u;
3
+
4
+ struct Uniforms {
5
+ num_tokens: u32,
6
+ num_heads: u32,
7
+ key_dim: u32,
8
+ value_dim: u32,
9
+ query_scale: f32,
10
+ _pad0: u32,
11
+ _pad1: u32,
12
+ _pad2: u32,
13
+ }
14
+
15
+ @group(0) @binding(0) var<uniform> u: Uniforms;
16
+ @group(0) @binding(1) var<storage, read> query_key: array<f32>;
17
+ @group(0) @binding(2) var<storage, read> value: array<f32>;
18
+ @group(0) @binding(3) var<storage, read> decay_beta: array<f32>;
19
+ @group(0) @binding(4) var<storage, read> state_history: array<f32>;
20
+ @group(0) @binding(5) var<storage, read> grad_output: array<f32>;
21
+ @group(0) @binding(6) var<storage, read_write> grad_query_key: array<f32>;
22
+ @group(0) @binding(7) var<storage, read_write> grad_value: array<f32>;
23
+ @group(0) @binding(8) var<storage, read_write> grad_decay_beta: array<f32>;
24
+ @group(0) @binding(9) var<storage, read_write> grad_state: array<f32>;
25
+
26
+ var<workgroup> shared_reduce: array<f32, MAX_WORKGROUP_SIZE>;
27
+
28
+ fn vector_index(token: u32, head: u32, dim: u32, width: u32) -> u32 {
29
+ return ((token * u.num_heads + head) * width) + dim;
30
+ }
31
+
32
+ fn state_history_index(token: u32, head: u32, key_index: u32, value_index: u32) -> u32 {
33
+ return ((((token * u.num_heads + head) * u.key_dim) + key_index) * u.value_dim) + value_index;
34
+ }
35
+
36
+ fn state_gradient_index(head: u32, key_index: u32, value_index: u32) -> u32 {
37
+ return (((head * u.key_dim) + key_index) * u.value_dim) + value_index;
38
+ }
39
+
40
+ @compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
41
+ fn main(
42
+ @builtin(workgroup_id) wid: vec3<u32>,
43
+ @builtin(local_invocation_id) lid: vec3<u32>
44
+ ) {
45
+ let head = wid.x;
46
+ let lane = lid.x;
47
+ if (head >= u.num_heads) {
48
+ return;
49
+ }
50
+ let is_active = lane < u.value_dim;
51
+ let query_elements = u.num_tokens * u.num_heads * u.key_dim;
52
+ let scalar_elements = u.num_tokens * u.num_heads;
53
+
54
+ for (var reverse_token: u32 = 0u; reverse_token < u.num_tokens; reverse_token = reverse_token + 1u) {
55
+ let token = u.num_tokens - 1u - reverse_token;
56
+ let scalar_index = token * u.num_heads + head;
57
+ let decay = exp(decay_beta[scalar_index]);
58
+ var memory: f32 = 0.0;
59
+ if (is_active) {
60
+ for (var key_index: u32 = 0u; key_index < u.key_dim; key_index = key_index + 1u) {
61
+ let previous_state = state_history[
62
+ state_history_index(token, head, key_index, lane)
63
+ ];
64
+ memory = memory
65
+ + previous_state
66
+ * decay
67
+ * query_key[query_elements + vector_index(token, head, key_index, u.key_dim)];
68
+ }
69
+ }
70
+ let value_offset = vector_index(token, head, lane, u.value_dim);
71
+ let value_at_token = select(0.0, value[value_offset], is_active);
72
+ let delta = (value_at_token - memory) * decay_beta[scalar_elements + scalar_index];
73
+ let output_gradient = select(0.0, grad_output[value_offset], is_active);
74
+
75
+ for (var key_index: u32 = 0u; key_index < u.key_dim; key_index = key_index + 1u) {
76
+ let query_offset = vector_index(token, head, key_index, u.key_dim);
77
+ let next_state = select(
78
+ 0.0,
79
+ state_history[state_history_index(token + 1u, head, key_index, lane)],
80
+ is_active
81
+ );
82
+ shared_reduce[lane] = output_gradient * next_state * u.query_scale;
83
+ workgroupBarrier();
84
+ for (var stride: u32 = WORKGROUP_SIZE / 2u; stride > 0u; stride = stride / 2u) {
85
+ if (lane < stride) {
86
+ shared_reduce[lane] = shared_reduce[lane] + shared_reduce[lane + stride];
87
+ }
88
+ workgroupBarrier();
89
+ }
90
+ if (lane == 0u) {
91
+ grad_query_key[query_offset] = shared_reduce[0];
92
+ }
93
+ if (is_active) {
94
+ let gradient_state_offset = state_gradient_index(head, key_index, lane);
95
+ grad_state[gradient_state_offset] = grad_state[gradient_state_offset]
96
+ + output_gradient
97
+ * query_key[query_offset]
98
+ * u.query_scale;
99
+ }
100
+ workgroupBarrier();
101
+ }
102
+
103
+ var grad_delta: f32 = 0.0;
104
+ if (is_active) {
105
+ for (var key_index: u32 = 0u; key_index < u.key_dim; key_index = key_index + 1u) {
106
+ let gradient_state_offset = state_gradient_index(head, key_index, lane);
107
+ grad_delta = grad_delta
108
+ + grad_state[gradient_state_offset]
109
+ * query_key[query_elements + vector_index(token, head, key_index, u.key_dim)];
110
+ }
111
+ grad_value[value_offset] = grad_delta * decay_beta[scalar_elements + scalar_index];
112
+ }
113
+ let grad_memory = -grad_delta * decay_beta[scalar_elements + scalar_index];
114
+ shared_reduce[lane] = select(0.0, grad_delta * (value_at_token - memory), is_active);
115
+ workgroupBarrier();
116
+ for (var stride: u32 = WORKGROUP_SIZE / 2u; stride > 0u; stride = stride / 2u) {
117
+ if (lane < stride) {
118
+ shared_reduce[lane] = shared_reduce[lane] + shared_reduce[lane + stride];
119
+ }
120
+ workgroupBarrier();
121
+ }
122
+ if (lane == 0u) {
123
+ grad_decay_beta[scalar_elements + scalar_index] = shared_reduce[0];
124
+ }
125
+ workgroupBarrier();
126
+
127
+ var local_grad_decay: f32 = 0.0;
128
+ for (var key_index: u32 = 0u; key_index < u.key_dim; key_index = key_index + 1u) {
129
+ let key_offset = vector_index(token, head, key_index, u.key_dim);
130
+ let gradient_state_offset = state_gradient_index(head, key_index, lane);
131
+ let previous_state = select(
132
+ 0.0,
133
+ state_history[state_history_index(token, head, key_index, lane)],
134
+ is_active
135
+ );
136
+ let decayed_state = previous_state * decay;
137
+ let gradient_before_decay_update = select(0.0, grad_state[gradient_state_offset], is_active);
138
+
139
+ shared_reduce[lane] = select(
140
+ 0.0,
141
+ (gradient_before_decay_update * delta) + (grad_memory * decayed_state),
142
+ is_active
143
+ );
144
+ workgroupBarrier();
145
+ for (var stride: u32 = WORKGROUP_SIZE / 2u; stride > 0u; stride = stride / 2u) {
146
+ if (lane < stride) {
147
+ shared_reduce[lane] = shared_reduce[lane] + shared_reduce[lane + stride];
148
+ }
149
+ workgroupBarrier();
150
+ }
151
+ if (lane == 0u) {
152
+ grad_query_key[query_elements + key_offset] = shared_reduce[0];
153
+ }
154
+
155
+ if (is_active) {
156
+ let gradient_decayed_state = gradient_before_decay_update
157
+ + grad_memory * query_key[query_elements + key_offset];
158
+ local_grad_decay = local_grad_decay + gradient_decayed_state * decayed_state;
159
+ grad_state[gradient_state_offset] = gradient_decayed_state * decay;
160
+ }
161
+ workgroupBarrier();
162
+ }
163
+
164
+ shared_reduce[lane] = select(0.0, local_grad_decay, is_active);
165
+ workgroupBarrier();
166
+ for (var stride: u32 = WORKGROUP_SIZE / 2u; stride > 0u; stride = stride / 2u) {
167
+ if (lane < stride) {
168
+ shared_reduce[lane] = shared_reduce[lane] + shared_reduce[lane + stride];
169
+ }
170
+ workgroupBarrier();
171
+ }
172
+ if (lane == 0u) {
173
+ grad_decay_beta[scalar_index] = shared_reduce[0];
174
+ }
175
+ workgroupBarrier();
176
+ }
177
+ }
@@ -0,0 +1,32 @@
1
+ import type { CommandRecorder } from '../../command-recorder.js';
2
+ import type { Tensor } from '../../tensor.js';
3
+
4
+ export interface GatedRmsNormBackwardOptions {
5
+ rows: number;
6
+ width: number;
7
+ eps: number;
8
+ gradInputBuffer?: GPUBuffer | null;
9
+ gradGateBuffer?: GPUBuffer | null;
10
+ }
11
+
12
+ export interface GatedRmsNormBackwardResult {
13
+ gradInput: Tensor;
14
+ gradGate: Tensor;
15
+ }
16
+
17
+ export declare function runGatedRmsNormBackward(
18
+ input: Tensor,
19
+ gate: Tensor,
20
+ weight: Tensor,
21
+ gradOutput: Tensor,
22
+ options: GatedRmsNormBackwardOptions
23
+ ): Promise<GatedRmsNormBackwardResult>;
24
+
25
+ export declare function recordGatedRmsNormBackward(
26
+ recorder: CommandRecorder,
27
+ input: Tensor,
28
+ gate: Tensor,
29
+ weight: Tensor,
30
+ gradOutput: Tensor,
31
+ options: GatedRmsNormBackwardOptions
32
+ ): Promise<GatedRmsNormBackwardResult>;
@@ -0,0 +1,90 @@
1
+ import { getDevice } from '../../device.js';
2
+ import { acquireBuffer } from '../../../memory/buffer-pool.js';
3
+ import { createTensor } from '../../tensor.js';
4
+ import { dispatch, recordDispatch } from '../dispatch.js';
5
+ import { createPipeline, createUniformBufferWithView } from '../utils.js';
6
+ import { releaseUniformBuffer } from '../../uniform-cache.js';
7
+
8
+ function validate(input, gate, weight, gradOutput, options) {
9
+ const rows = Math.floor(Number(options?.rows));
10
+ const width = Math.floor(Number(options?.width));
11
+ const eps = Number(options?.eps);
12
+ if (rows < 1 || width < 1 || !Number.isFinite(eps) || eps <= 0) {
13
+ throw new Error('gated RMSNorm backward requires rows, width, and positive eps.');
14
+ }
15
+ for (const [label, tensor] of [
16
+ ['input', input],
17
+ ['gate', gate],
18
+ ['weight', weight],
19
+ ['gradOutput', gradOutput],
20
+ ]) {
21
+ if (tensor?.dtype !== 'f32') {
22
+ throw new Error(`gated RMSNorm backward requires f32 ${label}.`);
23
+ }
24
+ }
25
+ return { rows, width, eps };
26
+ }
27
+
28
+ async function execute(recorder, input, gate, weight, gradOutput, options = {}) {
29
+ const dims = validate(input, gate, weight, gradOutput, options);
30
+ const device = recorder?.device || getDevice();
31
+ if (!device) throw new Error('gated RMSNorm backward requires an active GPU device.');
32
+ const outputBytes = dims.rows * dims.width * 4;
33
+ const gradInputBuffer = options.gradInputBuffer
34
+ || acquireBuffer(outputBytes, undefined, 'gated_rmsnorm_backward_input');
35
+ const gradGateBuffer = options.gradGateBuffer
36
+ || acquireBuffer(outputBytes, undefined, 'gated_rmsnorm_backward_gate');
37
+ const pipeline = await createPipeline('gated_rmsnorm_backward', 'default');
38
+ const uniformBuffer = createUniformBufferWithView(
39
+ 'gated_rmsnorm_backward_uniforms',
40
+ 16,
41
+ (view) => {
42
+ view.setUint32(0, dims.rows, true);
43
+ view.setUint32(4, dims.width, true);
44
+ view.setFloat32(8, dims.eps, true);
45
+ },
46
+ recorder || null,
47
+ device
48
+ );
49
+ const bindGroup = device.createBindGroup({
50
+ label: 'gated_rmsnorm_backward_bind_group',
51
+ layout: pipeline.getBindGroupLayout(0),
52
+ entries: [
53
+ { binding: 0, resource: { buffer: uniformBuffer } },
54
+ { binding: 1, resource: { buffer: input.buffer } },
55
+ { binding: 2, resource: { buffer: gate.buffer } },
56
+ { binding: 3, resource: { buffer: weight.buffer } },
57
+ { binding: 4, resource: { buffer: gradOutput.buffer } },
58
+ { binding: 5, resource: { buffer: gradInputBuffer } },
59
+ { binding: 6, resource: { buffer: gradGateBuffer } },
60
+ ],
61
+ });
62
+ if (recorder) {
63
+ recordDispatch(recorder, pipeline, bindGroup, dims.rows, 'gated_rmsnorm_backward');
64
+ } else {
65
+ dispatch(device, pipeline, bindGroup, dims.rows, 'gated_rmsnorm_backward');
66
+ releaseUniformBuffer(uniformBuffer);
67
+ }
68
+ return {
69
+ gradInput: createTensor(
70
+ gradInputBuffer,
71
+ 'f32',
72
+ [dims.rows, dims.width],
73
+ 'gated_rmsnorm_backward_input'
74
+ ),
75
+ gradGate: createTensor(
76
+ gradGateBuffer,
77
+ 'f32',
78
+ [dims.rows, dims.width],
79
+ 'gated_rmsnorm_backward_gate'
80
+ ),
81
+ };
82
+ }
83
+
84
+ export function runGatedRmsNormBackward(input, gate, weight, gradOutput, options = {}) {
85
+ return execute(null, input, gate, weight, gradOutput, options);
86
+ }
87
+
88
+ export function recordGatedRmsNormBackward(recorder, input, gate, weight, gradOutput, options = {}) {
89
+ return execute(recorder, input, gate, weight, gradOutput, options);
90
+ }
@@ -0,0 +1,86 @@
1
+ override WORKGROUP_SIZE: u32 = 256u;
2
+ const MAX_WORKGROUP_SIZE: u32 = 256u;
3
+
4
+ struct Uniforms {
5
+ rows: u32,
6
+ width: u32,
7
+ eps: f32,
8
+ _pad0: u32,
9
+ }
10
+
11
+ @group(0) @binding(0) var<uniform> u: Uniforms;
12
+ @group(0) @binding(1) var<storage, read> input: array<f32>;
13
+ @group(0) @binding(2) var<storage, read> gate: array<f32>;
14
+ @group(0) @binding(3) var<storage, read> weight: array<f32>;
15
+ @group(0) @binding(4) var<storage, read> grad_output: array<f32>;
16
+ @group(0) @binding(5) var<storage, read_write> grad_input: array<f32>;
17
+ @group(0) @binding(6) var<storage, read_write> grad_gate: array<f32>;
18
+
19
+ var<workgroup> shared_sum_sq: array<f32, MAX_WORKGROUP_SIZE>;
20
+ var<workgroup> shared_sum_gx: array<f32, MAX_WORKGROUP_SIZE>;
21
+
22
+ fn sigmoid(x: f32) -> f32 {
23
+ if (x >= 0.0) {
24
+ let z = exp(-x);
25
+ return 1.0 / (1.0 + z);
26
+ }
27
+ let z = exp(x);
28
+ return z / (1.0 + z);
29
+ }
30
+
31
+ @compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
32
+ fn main(
33
+ @builtin(workgroup_id) wid: vec3<u32>,
34
+ @builtin(local_invocation_id) lid: vec3<u32>
35
+ ) {
36
+ let row = wid.x;
37
+ if (row >= u.rows) {
38
+ return;
39
+ }
40
+ let local_id = lid.x;
41
+ let base = row * u.width;
42
+ var sum_sq: f32 = 0.0;
43
+ var sum_gx: f32 = 0.0;
44
+
45
+ for (var column = local_id; column < u.width; column = column + WORKGROUP_SIZE) {
46
+ let offset = base + column;
47
+ let x = input[offset];
48
+ let probability = sigmoid(gate[offset]);
49
+ let gate_value = gate[offset] * probability;
50
+ let grad_normalized = grad_output[offset] * weight[column] * gate_value;
51
+ sum_sq = sum_sq + x * x;
52
+ sum_gx = sum_gx + grad_normalized * x;
53
+ }
54
+
55
+ shared_sum_sq[local_id] = sum_sq;
56
+ shared_sum_gx[local_id] = sum_gx;
57
+ workgroupBarrier();
58
+
59
+ var stride = WORKGROUP_SIZE / 2u;
60
+ loop {
61
+ if (stride == 0u) {
62
+ break;
63
+ }
64
+ if (local_id < stride) {
65
+ shared_sum_sq[local_id] = shared_sum_sq[local_id] + shared_sum_sq[local_id + stride];
66
+ shared_sum_gx[local_id] = shared_sum_gx[local_id] + shared_sum_gx[local_id + stride];
67
+ }
68
+ workgroupBarrier();
69
+ stride = stride / 2u;
70
+ }
71
+
72
+ let inverse_rms = inverseSqrt(shared_sum_sq[0] / f32(u.width) + u.eps);
73
+ let correction = (shared_sum_gx[0] * inverse_rms * inverse_rms) / f32(u.width);
74
+ for (var column = local_id; column < u.width; column = column + WORKGROUP_SIZE) {
75
+ let offset = base + column;
76
+ let x = input[offset];
77
+ let gate_input = gate[offset];
78
+ let probability = sigmoid(gate_input);
79
+ let gate_value = gate_input * probability;
80
+ let gate_derivative = probability * (1.0 + gate_input * (1.0 - probability));
81
+ let grad = grad_output[offset];
82
+ let grad_normalized = grad * weight[column] * gate_value;
83
+ grad_input[offset] = inverse_rms * (grad_normalized - x * correction);
84
+ grad_gate[offset] = grad * x * inverse_rms * weight[column] * gate_derivative;
85
+ }
86
+ }
@@ -3,6 +3,9 @@ export { runMatmulBackward, recordMatmulBackward } from './matmul_backward.js';
3
3
  export { runSoftmaxBackward, recordSoftmaxBackward } from './softmax_backward.js';
4
4
  export { runRmsNormBackward, recordRmsNormBackward } from './rmsnorm_backward.js';
5
5
  export { runRmsNormBackward as runRmsnormBackward, recordRmsNormBackward as recordRmsnormBackward } from './rmsnorm_backward.js';
6
+ export { runGatedRmsNormBackward, recordGatedRmsNormBackward } from './gated_rmsnorm_backward.js';
7
+ export { runCausalConv1dSiluBackward, recordCausalConv1dSiluBackward } from './causal_conv1d_silu_backward.js';
8
+ export { runGatedDeltaRecurrentBackward } from './gated_delta_recurrent_backward.js';
6
9
  export { runLayerNormBackward, recordLayerNormBackward } from './layernorm_backward.js';
7
10
  export { runLayerNormBackward as runLayernormBackward, recordLayerNormBackward as recordLayernormBackward } from './layernorm_backward.js';
8
11
  export { runAttentionBackward, recordAttentionBackward } from './attention_backward.js';
@@ -3,6 +3,9 @@ export { runMatmulBackward, recordMatmulBackward } from './matmul_backward.js';
3
3
  export { runSoftmaxBackward, recordSoftmaxBackward } from './softmax_backward.js';
4
4
  export { runRmsNormBackward, recordRmsNormBackward } from './rmsnorm_backward.js';
5
5
  export { runRmsNormBackward as runRmsnormBackward, recordRmsNormBackward as recordRmsnormBackward } from './rmsnorm_backward.js';
6
+ export { runGatedRmsNormBackward, recordGatedRmsNormBackward } from './gated_rmsnorm_backward.js';
7
+ export { runCausalConv1dSiluBackward, recordCausalConv1dSiluBackward } from './causal_conv1d_silu_backward.js';
8
+ export { runGatedDeltaRecurrentBackward } from './gated_delta_recurrent_backward.js';
6
9
  export { runLayerNormBackward, recordLayerNormBackward } from './layernorm_backward.js';
7
10
  export { runLayerNormBackward as runLayernormBackward, recordLayerNormBackward as recordLayernormBackward } from './layernorm_backward.js';
8
11
  export { runAttentionBackward, recordAttentionBackward } from './attention_backward.js';
@@ -2,10 +2,10 @@
2
2
  //
3
3
  // Fused tiled matmul for computing input gradients during backpropagation.
4
4
  //
5
- // When transpose_b = 1 (standard case: forward was Y = X @ W, W stored [K,N]):
5
+ // When transpose_b = 0 (standard case: forward was Y = X @ W, W stored [K,N]):
6
6
  // dX[M,K] = dY[M,N] @ W^T[N,K] (reads W as [K,N], transposes on the fly)
7
7
  //
8
- // When transpose_b = 0 (forward was Y = X @ W^T, W stored [N,K]):
8
+ // When transpose_b = 1 (forward was Y = X @ W^T, W stored [N,K]):
9
9
  // dX[M,K] = dY[M,N] @ W[N,K] (reads W directly as [N,K])
10
10
  //
11
11
  // Uses 16x16 shared memory tiles, matching matmul_transpose_a.wgsl style.
@@ -20,7 +20,7 @@ struct Uniforms {
20
20
  N: u32, // Cols of dY (output dim of forward)
21
21
  K: u32, // Cols of dX (input dim of forward)
22
22
  alpha: f32,
23
- transpose_b: u32, // 0 = W stored [N,K], 1 = W stored [K,N]
23
+ transpose_b: u32, // 0 = W stored [K,N], 1 = W stored [N,K]
24
24
  _pad0: u32,
25
25
  _pad1: u32,
26
26
  _pad2: u32,
@@ -65,11 +65,11 @@ fn main(
65
65
  let wt_row = tile_n + local_row;
66
66
  if (wt_row < u.N && col < u.K) {
67
67
  if (u.transpose_b == 0u) {
68
- // W stored as [N, K]: W^T[wt_row, col] = W[wt_row, col]
69
- tileW[tile_idx] = W[wt_row * u.K + col];
70
- } else {
71
68
  // W stored as [K, N]: W^T[wt_row, col] = W[col, wt_row]
72
69
  tileW[tile_idx] = W[col * u.N + wt_row];
70
+ } else {
71
+ // W stored as [N, K]: W^T[wt_row, col] = W[wt_row, col]
72
+ tileW[tile_idx] = W[wt_row * u.K + col];
73
73
  }
74
74
  } else {
75
75
  tileW[tile_idx] = 0.0;
@@ -0,0 +1,82 @@
1
+ // Matrix Multiplication Backward Kernel - dX = dY @ W^T with packed F16 W.
2
+ //
3
+ // dY and dX remain F32. W is stored as consecutive IEEE-754 binary16 values,
4
+ // packed two values per u32 storage word. unpack2x16float keeps this path
5
+ // available without requiring native shader-f16 arithmetic.
6
+
7
+ const MAX_TILE_SIZE: u32 = 16u;
8
+ const TILE_AREA: u32 = MAX_TILE_SIZE * MAX_TILE_SIZE;
9
+
10
+ override TILE_SIZE: u32 = 16u;
11
+
12
+ struct Uniforms {
13
+ M: u32,
14
+ N: u32,
15
+ K: u32,
16
+ alpha: f32,
17
+ transpose_b: u32,
18
+ _pad0: u32,
19
+ _pad1: u32,
20
+ _pad2: u32,
21
+ }
22
+
23
+ @group(0) @binding(0) var<uniform> u: Uniforms;
24
+ @group(0) @binding(1) var<storage, read> dY: array<f32>;
25
+ @group(0) @binding(2) var<storage, read> W: array<u32>;
26
+ @group(0) @binding(3) var<storage, read_write> dX: array<f32>;
27
+
28
+ var<workgroup> tileDY: array<f32, TILE_AREA>;
29
+ var<workgroup> tileW: array<f32, TILE_AREA>;
30
+
31
+ fn load_f16(index: u32) -> f32 {
32
+ let pair = unpack2x16float(W[index >> 1u]);
33
+ return select(pair.x, pair.y, (index & 1u) == 1u);
34
+ }
35
+
36
+ @compute @workgroup_size(TILE_SIZE, TILE_SIZE, 1)
37
+ fn main(
38
+ @builtin(global_invocation_id) global_id: vec3<u32>,
39
+ @builtin(local_invocation_id) local_id: vec3<u32>
40
+ ) {
41
+ let row = global_id.x;
42
+ let col = global_id.y;
43
+ let local_row = local_id.x;
44
+ let local_col = local_id.y;
45
+
46
+ var sum: f32 = 0.0;
47
+ let num_tiles = (u.N + TILE_SIZE - 1u) / TILE_SIZE;
48
+
49
+ for (var t: u32 = 0u; t < num_tiles; t = t + 1u) {
50
+ let tile_n = t * TILE_SIZE;
51
+ let tile_idx = local_row * TILE_SIZE + local_col;
52
+ let dy_col = tile_n + local_col;
53
+ if (row < u.M && dy_col < u.N) {
54
+ tileDY[tile_idx] = dY[row * u.N + dy_col];
55
+ } else {
56
+ tileDY[tile_idx] = 0.0;
57
+ }
58
+
59
+ let wt_row = tile_n + local_row;
60
+ if (wt_row < u.N && col < u.K) {
61
+ if (u.transpose_b == 0u) {
62
+ tileW[tile_idx] = load_f16(col * u.N + wt_row);
63
+ } else {
64
+ tileW[tile_idx] = load_f16(wt_row * u.K + col);
65
+ }
66
+ } else {
67
+ tileW[tile_idx] = 0.0;
68
+ }
69
+
70
+ workgroupBarrier();
71
+
72
+ for (var k: u32 = 0u; k < TILE_SIZE; k = k + 1u) {
73
+ sum += tileDY[local_row * TILE_SIZE + k] * tileW[k * TILE_SIZE + local_col];
74
+ }
75
+
76
+ workgroupBarrier();
77
+ }
78
+
79
+ if (row < u.M && col < u.K) {
80
+ dX[row * u.K + col] = sum * u.alpha;
81
+ }
82
+ }
@@ -9,7 +9,13 @@ import { DEFAULT_HIGH_PRECISION_EPS } from '../../../config/schema/index.js';
9
9
 
10
10
  export async function runRmsNormBackward(input, weight, gradOutput, options = {}) {
11
11
  const device = getDevice();
12
- const { numTokens, hiddenSize, eps = DEFAULT_HIGH_PRECISION_EPS, outputBuffer = null } = options;
12
+ const {
13
+ numTokens,
14
+ hiddenSize,
15
+ eps = DEFAULT_HIGH_PRECISION_EPS,
16
+ rmsNormWeightOffset = false,
17
+ outputBuffer = null,
18
+ } = options;
13
19
 
14
20
  if (!numTokens || !hiddenSize) {
15
21
  throw new Error('rmsnorm backward requires numTokens and hiddenSize');
@@ -22,7 +28,12 @@ export async function runRmsNormBackward(input, weight, gradOutput, options = {}
22
28
  const outputSize = numTokens * hiddenSize * 4;
23
29
  const outputBuf = outputBuffer || acquireBuffer(outputSize, undefined, 'rmsnorm_backward_output');
24
30
 
25
- const pipeline = await createPipeline('rmsnorm_backward', 'default');
31
+ const pipeline = await createPipeline(
32
+ 'rmsnorm_backward',
33
+ 'default',
34
+ null,
35
+ { RMS_NORM_OFFSET: rmsNormWeightOffset }
36
+ );
26
37
  const uniformBuffer = createUniformBufferWithView(
27
38
  'rmsnorm_backward_uniforms',
28
39
  16,
@@ -57,7 +68,13 @@ export async function runRmsNormBackward(input, weight, gradOutput, options = {}
57
68
 
58
69
  export async function recordRmsNormBackward(recorder, input, weight, gradOutput, options = {}) {
59
70
  const device = recorder.device;
60
- const { numTokens, hiddenSize, eps = DEFAULT_HIGH_PRECISION_EPS, outputBuffer = null } = options;
71
+ const {
72
+ numTokens,
73
+ hiddenSize,
74
+ eps = DEFAULT_HIGH_PRECISION_EPS,
75
+ rmsNormWeightOffset = false,
76
+ outputBuffer = null,
77
+ } = options;
61
78
 
62
79
  if (!numTokens || !hiddenSize) {
63
80
  throw new Error('rmsnorm backward requires numTokens and hiddenSize');
@@ -70,7 +87,12 @@ export async function recordRmsNormBackward(recorder, input, weight, gradOutput,
70
87
  const outputSize = numTokens * hiddenSize * 4;
71
88
  const outputBuf = outputBuffer || acquireBuffer(outputSize, undefined, 'rmsnorm_backward_output');
72
89
 
73
- const pipeline = await createPipeline('rmsnorm_backward', 'default');
90
+ const pipeline = await createPipeline(
91
+ 'rmsnorm_backward',
92
+ 'default',
93
+ null,
94
+ { RMS_NORM_OFFSET: rmsNormWeightOffset }
95
+ );
74
96
  const uniformBuffer = createUniformBufferWithView(
75
97
  'rmsnorm_backward_uniforms',
76
98
  16,
@@ -7,6 +7,7 @@
7
7
  * Uses shared-memory reduction for per-token stats.
8
8
  */
9
9
  override WORKGROUP_SIZE: u32 = 256u;
10
+ override RMS_NORM_OFFSET: bool = false;
10
11
  const MAX_WORKGROUP_SIZE: u32 = 256u;
11
12
 
12
13
  struct Uniforms {
@@ -44,7 +45,8 @@ fn main(
44
45
 
45
46
  for (var i: u32 = local_id; i < hidden_size; i = i + WORKGROUP_SIZE) {
46
47
  let x = input[base + i];
47
- let g = grad_output[base + i] * weight[i];
48
+ let effective_weight = select(weight[i], 1.0 + weight[i], RMS_NORM_OFFSET);
49
+ let g = grad_output[base + i] * effective_weight;
48
50
  sum_sq = sum_sq + x * x;
49
51
  sum_gx = sum_gx + g * x;
50
52
  }
@@ -72,7 +74,8 @@ fn main(
72
74
 
73
75
  for (var i: u32 = local_id; i < hidden_size; i = i + WORKGROUP_SIZE) {
74
76
  let x = input[base + i];
75
- let g = grad_output[base + i] * weight[i];
77
+ let effective_weight = select(weight[i], 1.0 + weight[i], RMS_NORM_OFFSET);
78
+ let g = grad_output[base + i] * effective_weight;
76
79
  output[base + i] = g * inv_rms - x * coeff;
77
80
  }
78
- }
81
+ }
@@ -56,6 +56,16 @@ export declare function createBackwardKernel(
56
56
  record: (recorder: CommandRecorder, ...args: unknown[]) => Promise<Tensor>;
57
57
  };
58
58
 
59
+ export declare function resolveMatmulBackwardDxVariant(
60
+ weight: { dtype?: string | null } | null
61
+ ): 'default' | 'f16_weight';
62
+
63
+ export interface MatmulBackwardDxOptions {
64
+ alpha?: number;
65
+ transposeB?: boolean;
66
+ outputBuffer?: GPUBuffer | null;
67
+ }
68
+
59
69
  /**
60
70
  * Matmul-backward gradient w.r.t. input (dX = dY · Wᵀ). Submit + wait.
61
71
  */
@@ -65,7 +75,7 @@ export declare function runMatmulBackwardDx(
65
75
  M: number,
66
76
  K: number,
67
77
  N: number,
68
- options?: { outputBuffer?: GPUBuffer | null }
78
+ options?: MatmulBackwardDxOptions
69
79
  ): Promise<Tensor>;
70
80
 
71
81
  /** Record runMatmulBackwardDx onto a CommandRecorder. */
@@ -76,5 +86,5 @@ export declare function recordMatmulBackwardDx(
76
86
  M: number,
77
87
  K: number,
78
88
  N: number,
79
- options?: { outputBuffer?: GPUBuffer | null }
89
+ options?: MatmulBackwardDxOptions
80
90
  ): Promise<Tensor>;
@@ -192,13 +192,20 @@ export function createBackwardKernel(opName, spec) {
192
192
  return { run, record };
193
193
  }
194
194
 
195
+ export function resolveMatmulBackwardDxVariant(weight) {
196
+ const dtype = String(weight?.dtype || 'f32').toLowerCase();
197
+ if (dtype === 'f16') return 'f16_weight';
198
+ if (dtype === 'f32') return 'default';
199
+ throw new Error(`matmul backward dX does not support weight dtype "${dtype}".`);
200
+ }
201
+
195
202
  export async function runMatmulBackwardDx(dY, W, M, K, N, options = {}) {
196
203
  const { alpha = 1.0, transposeB = false, outputBuffer = null } = options;
197
204
  const device = getDevice();
198
205
  const outputSize = M * K * 4;
199
206
  const outputBuf = outputBuffer || acquireBuffer(outputSize, undefined, 'matmul_backward_dx_output');
200
207
 
201
- const pipeline = await createPipeline('matmul_backward', 'default');
208
+ const pipeline = await createPipeline('matmul_backward', resolveMatmulBackwardDxVariant(W));
202
209
 
203
210
  const uniformBuffer = createUniformBufferWithView(
204
211
  'matmul_backward_uniforms',
@@ -244,7 +251,7 @@ export async function recordMatmulBackwardDx(recorder, dY, W, M, K, N, options =
244
251
  const outputSize = M * K * 4;
245
252
  const outputBuf = outputBuffer || acquireBuffer(outputSize, undefined, 'matmul_backward_dx_output');
246
253
 
247
- const pipeline = await createPipeline('matmul_backward', 'default');
254
+ const pipeline = await createPipeline('matmul_backward', resolveMatmulBackwardDxVariant(W));
248
255
 
249
256
  const uniformBuffer = createUniformBufferWithView(
250
257
  'matmul_backward_uniforms',