doppler-gpu 0.4.8 → 0.4.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +29 -0
- package/README.md +62 -12
- package/models/catalog.json +21 -7
- package/package.json +25 -1
- package/src/config/conversion/qwen3/qwen-3-5-9b-f16-af32.json +474 -0
- package/src/config/conversion/qwen3/qwen-3-5-9b-q4k-ehaf16.json +509 -0
- package/src/config/kernels/backward-registry.json +13 -2
- package/src/config/kernels/kernel-ref-digests.js +8 -4
- package/src/config/kernels/registry.json +1035 -2677
- package/src/config/runtime/profiles/qwen-3-5-9b-metal-correctness.json +46 -0
- package/src/config/schema/training-metrics.schema.js +4 -0
- package/src/experimental/bridge/trainer-artifact-bridge.d.ts +161 -0
- package/src/experimental/bridge/trainer-artifact-bridge.js +573 -0
- package/src/experimental/training/attention-backward.js +123 -55
- package/src/experimental/training/autograd.js +101 -5
- package/src/experimental/training/clip.js +6 -1
- package/src/experimental/training/datasets/text-pairs.js +80 -8
- package/src/experimental/training/distillation/student-fixture.js +470 -114
- package/src/experimental/training/lora-pipeline.js +202 -27
- package/src/experimental/training/lora.js +10 -4
- package/src/experimental/training/optimizer.js +9 -1
- package/src/experimental/training/runner.js +18 -4
- package/src/experimental/training/workloads.js +11 -0
- package/src/gpu/device.js +100 -32
- package/src/gpu/kernels/backward/adam.js +6 -4
- package/src/gpu/kernels/backward/causal_conv1d_silu_backward.d.ts +24 -0
- package/src/gpu/kernels/backward/causal_conv1d_silu_backward.js +74 -0
- package/src/gpu/kernels/backward/causal_conv1d_silu_backward.wgsl +59 -0
- package/src/gpu/kernels/backward/cross_entropy_backward.wgsl +4 -0
- package/src/gpu/kernels/backward/gated_delta_recurrent_backward.d.ts +39 -0
- package/src/gpu/kernels/backward/gated_delta_recurrent_backward.js +124 -0
- package/src/gpu/kernels/backward/gated_delta_recurrent_backward.wgsl +177 -0
- package/src/gpu/kernels/backward/gated_rmsnorm_backward.d.ts +32 -0
- package/src/gpu/kernels/backward/gated_rmsnorm_backward.js +90 -0
- package/src/gpu/kernels/backward/gated_rmsnorm_backward.wgsl +86 -0
- package/src/gpu/kernels/backward/index.d.ts +3 -0
- package/src/gpu/kernels/backward/index.js +3 -0
- package/src/gpu/kernels/backward/matmul_backward.wgsl +6 -6
- package/src/gpu/kernels/backward/matmul_backward_f16w.wgsl +82 -0
- package/src/gpu/kernels/backward/rmsnorm_backward.js +26 -4
- package/src/gpu/kernels/backward/rmsnorm_backward.wgsl +6 -3
- package/src/gpu/kernels/backward/utils.d.ts +12 -2
- package/src/gpu/kernels/backward/utils.js +9 -2
- package/src/gpu/kernels/cast_f16_to_f32.wgsl +6 -2
- package/src/gpu/kernels/index.d.ts +5 -0
- package/src/gpu/kernels/index.js +5 -0
- package/src/gpu/kernels/matmul-selection.js +5 -0
- package/src/gpu/kernels/silu.d.ts +1 -0
- package/src/gpu/kernels/silu.js +19 -4
- package/src/inference/pipelines/text/embed.js +1 -1
- package/src/inference/pipelines/text/execution-v1.js +8 -2
- package/src/inference/pipelines/text/ffn/dense.js +2 -0
- package/src/inference/pipelines/text/generator.js +29 -0
- package/src/inference/pipelines/text/lora-apply.js +4 -4
- package/src/inference/pipelines/text/moe-cpu.js +1 -0
- package/src/inference/pipelines/text/moe-gpu.js +1 -0
- package/src/inference/tokenizers/bundled.js +105 -32
- package/src/rules/kernels/matmul.rules.json +1 -0
- package/src/tooling/node-converter.js +20 -5
- package/src/tooling/runtime-optimization.d.ts +109 -0
- package/src/tooling/runtime-optimization.js +872 -0
- package/src/tooling-experimental-exports.shared.d.ts +25 -0
- package/src/tooling-experimental-exports.shared.js +18 -0
- package/src/tooling-exports.shared.d.ts +19 -0
- package/src/tooling-exports.shared.js +11 -0
- package/src/version.js +1 -1
package/src/gpu/device.js
CHANGED
|
@@ -14,6 +14,12 @@ const SHARED_DEVICE_STATE_KEY = '__dopplerGpuDeviceState';
|
|
|
14
14
|
function getSharedDeviceState() {
|
|
15
15
|
const existing = globalThis[SHARED_DEVICE_STATE_KEY];
|
|
16
16
|
if (existing && typeof existing === 'object') {
|
|
17
|
+
if (!(existing.bufferOwners instanceof WeakMap)) {
|
|
18
|
+
existing.bufferOwners = new WeakMap();
|
|
19
|
+
}
|
|
20
|
+
if (!('deviceInitPromise' in existing)) {
|
|
21
|
+
existing.deviceInitPromise = null;
|
|
22
|
+
}
|
|
17
23
|
return existing;
|
|
18
24
|
}
|
|
19
25
|
const created = {
|
|
@@ -23,6 +29,8 @@ function getSharedDeviceState() {
|
|
|
23
29
|
lastDeviceLossInfo: null,
|
|
24
30
|
platformInitialized: false,
|
|
25
31
|
deviceEpoch: 0,
|
|
32
|
+
bufferOwners: new WeakMap(),
|
|
33
|
+
deviceInitPromise: null,
|
|
26
34
|
};
|
|
27
35
|
Object.defineProperty(globalThis, SHARED_DEVICE_STATE_KEY, {
|
|
28
36
|
value: created,
|
|
@@ -58,14 +66,14 @@ function syncSharedDeviceState() {
|
|
|
58
66
|
}
|
|
59
67
|
|
|
60
68
|
function hydrateDeviceState() {
|
|
61
|
-
gpuDevice =
|
|
62
|
-
kernelCapabilities =
|
|
63
|
-
resolvedPlatformConfig =
|
|
64
|
-
lastDeviceLossInfo =
|
|
65
|
-
platformInitialized =
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
+
gpuDevice = sharedDeviceState.gpuDevice ?? null;
|
|
70
|
+
kernelCapabilities = sharedDeviceState.kernelCapabilities ?? null;
|
|
71
|
+
resolvedPlatformConfig = sharedDeviceState.resolvedPlatformConfig ?? null;
|
|
72
|
+
lastDeviceLossInfo = sharedDeviceState.lastDeviceLossInfo ?? null;
|
|
73
|
+
platformInitialized = sharedDeviceState.platformInitialized === true;
|
|
74
|
+
deviceEpoch = Number.isInteger(sharedDeviceState.deviceEpoch)
|
|
75
|
+
? sharedDeviceState.deviceEpoch
|
|
76
|
+
: 0;
|
|
69
77
|
}
|
|
70
78
|
|
|
71
79
|
function advanceDeviceEpoch() {
|
|
@@ -185,7 +193,7 @@ function describeBindGroupBufferValue(value) {
|
|
|
185
193
|
return typeof value;
|
|
186
194
|
}
|
|
187
195
|
|
|
188
|
-
function validateBindGroupDescriptor(descriptor) {
|
|
196
|
+
function validateBindGroupDescriptor(descriptor, device) {
|
|
189
197
|
const label = descriptor?.label || 'unlabeled_bind_group';
|
|
190
198
|
const entries = Array.isArray(descriptor?.entries) ? descriptor.entries : [];
|
|
191
199
|
for (const entry of entries) {
|
|
@@ -193,13 +201,18 @@ function validateBindGroupDescriptor(descriptor) {
|
|
|
193
201
|
if (!resource || typeof resource !== 'object' || !('buffer' in resource)) {
|
|
194
202
|
continue;
|
|
195
203
|
}
|
|
196
|
-
if (isValidGPUBuffer(resource.buffer)) {
|
|
197
|
-
|
|
204
|
+
if (!isValidGPUBuffer(resource.buffer)) {
|
|
205
|
+
throw new Error(
|
|
206
|
+
`[${label}] binding ${entry.binding} requires a GPUBuffer; ` +
|
|
207
|
+
`got ${describeBindGroupBufferValue(resource.buffer)}.`
|
|
208
|
+
);
|
|
209
|
+
}
|
|
210
|
+
const owner = sharedDeviceState.bufferOwners.get(resource.buffer);
|
|
211
|
+
if (owner && owner !== device) {
|
|
212
|
+
throw new Error(
|
|
213
|
+
`[${label}] binding ${entry.binding} uses a GPUBuffer created by a different GPUDevice.`
|
|
214
|
+
);
|
|
198
215
|
}
|
|
199
|
-
throw new Error(
|
|
200
|
-
`[${label}] binding ${entry.binding} requires a GPUBuffer; ` +
|
|
201
|
-
`got ${describeBindGroupBufferValue(resource.buffer)}.`
|
|
202
|
-
);
|
|
203
216
|
}
|
|
204
217
|
}
|
|
205
218
|
|
|
@@ -211,16 +224,49 @@ function shouldValidateBindGroupDescriptors() {
|
|
|
211
224
|
return false;
|
|
212
225
|
}
|
|
213
226
|
|
|
214
|
-
function
|
|
215
|
-
if (
|
|
227
|
+
function wrapDeviceCreateBuffer(device) {
|
|
228
|
+
if (
|
|
229
|
+
!device
|
|
230
|
+
|| device.__dopplerBufferOwnershipWrapped
|
|
231
|
+
|| typeof device.createBuffer !== 'function'
|
|
232
|
+
) {
|
|
216
233
|
return device;
|
|
217
234
|
}
|
|
218
|
-
|
|
235
|
+
const originalCreateBuffer = device.createBuffer.bind(device);
|
|
236
|
+
device.createBuffer = (descriptor) => {
|
|
237
|
+
const buffer = originalCreateBuffer(descriptor);
|
|
238
|
+
if (buffer && (typeof buffer === 'object' || typeof buffer === 'function')) {
|
|
239
|
+
sharedDeviceState.bufferOwners.set(buffer, device);
|
|
240
|
+
}
|
|
241
|
+
return buffer;
|
|
242
|
+
};
|
|
243
|
+
Object.defineProperty(device, '__dopplerBufferOwnershipWrapped', {
|
|
244
|
+
value: true,
|
|
245
|
+
configurable: true,
|
|
246
|
+
enumerable: false,
|
|
247
|
+
writable: false,
|
|
248
|
+
});
|
|
249
|
+
return device;
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
function wrapDeviceCreateBindGroup(device) {
|
|
253
|
+
if (!device || device.__dopplerBindGroupValidationWrapped) {
|
|
219
254
|
return device;
|
|
220
255
|
}
|
|
221
256
|
const originalCreateBindGroup = device.createBindGroup.bind(device);
|
|
222
257
|
device.createBindGroup = (descriptor) => {
|
|
223
|
-
|
|
258
|
+
if (shouldValidateBindGroupDescriptors()) {
|
|
259
|
+
validateBindGroupDescriptor(descriptor, device);
|
|
260
|
+
} else {
|
|
261
|
+
const entries = Array.isArray(descriptor?.entries) ? descriptor.entries : [];
|
|
262
|
+
for (const entry of entries) {
|
|
263
|
+
const buffer = entry?.resource?.buffer;
|
|
264
|
+
if (buffer && sharedDeviceState.bufferOwners.get(buffer) !== undefined) {
|
|
265
|
+
validateBindGroupDescriptor(descriptor, device);
|
|
266
|
+
break;
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
}
|
|
224
270
|
return originalCreateBindGroup(descriptor);
|
|
225
271
|
};
|
|
226
272
|
Object.defineProperty(device, '__dopplerBindGroupValidationWrapped', {
|
|
@@ -240,9 +286,10 @@ function registerDeviceLostHandler(device) {
|
|
|
240
286
|
if (device.lost && typeof device.lost.then === 'function') {
|
|
241
287
|
const trackedDevice = device;
|
|
242
288
|
device.lost.then((info) => {
|
|
243
|
-
if (gpuDevice !== trackedDevice) {
|
|
289
|
+
if (sharedDeviceState.gpuDevice !== trackedDevice) {
|
|
244
290
|
return;
|
|
245
291
|
}
|
|
292
|
+
hydrateDeviceState();
|
|
246
293
|
lastDeviceLossInfo = {
|
|
247
294
|
message: info?.message ?? '',
|
|
248
295
|
reason: info?.reason ?? 'unknown',
|
|
@@ -254,9 +301,10 @@ function registerDeviceLostHandler(device) {
|
|
|
254
301
|
clearActiveDeviceState();
|
|
255
302
|
advanceDeviceEpoch();
|
|
256
303
|
}).catch((error) => {
|
|
257
|
-
if (gpuDevice !== trackedDevice) {
|
|
304
|
+
if (sharedDeviceState.gpuDevice !== trackedDevice) {
|
|
258
305
|
return;
|
|
259
306
|
}
|
|
307
|
+
hydrateDeviceState();
|
|
260
308
|
lastDeviceLossInfo = {
|
|
261
309
|
message: error?.message ?? String(error),
|
|
262
310
|
reason: 'device_lost_handler_failed',
|
|
@@ -478,17 +526,7 @@ async function initializePlatformAndRegistry(adapter) {
|
|
|
478
526
|
}
|
|
479
527
|
|
|
480
528
|
|
|
481
|
-
|
|
482
|
-
hydrateDeviceState();
|
|
483
|
-
// Return cached device if available
|
|
484
|
-
if (gpuDevice) {
|
|
485
|
-
if (isUsableGPUDevice(gpuDevice)) {
|
|
486
|
-
return gpuDevice;
|
|
487
|
-
}
|
|
488
|
-
clearActiveDeviceState();
|
|
489
|
-
advanceDeviceEpoch();
|
|
490
|
-
}
|
|
491
|
-
|
|
529
|
+
async function initializeDevice() {
|
|
492
530
|
if (!isWebGPUAvailable()) {
|
|
493
531
|
throw createDopplerError(ERROR_CODES.GPU_UNAVAILABLE, 'WebGPU is not available in this browser');
|
|
494
532
|
}
|
|
@@ -517,6 +555,7 @@ export async function initDevice() {
|
|
|
517
555
|
}
|
|
518
556
|
lastDeviceLossInfo = null;
|
|
519
557
|
ensureGpuBufferConstructor(gpuDevice);
|
|
558
|
+
wrapDeviceCreateBuffer(gpuDevice);
|
|
520
559
|
wrapDeviceCreateBindGroup(gpuDevice);
|
|
521
560
|
registerDeviceLostHandler(gpuDevice);
|
|
522
561
|
advanceDeviceEpoch();
|
|
@@ -563,6 +602,34 @@ export async function initDevice() {
|
|
|
563
602
|
return gpuDevice;
|
|
564
603
|
}
|
|
565
604
|
|
|
605
|
+
export async function initDevice() {
|
|
606
|
+
hydrateDeviceState();
|
|
607
|
+
// Return cached device if available
|
|
608
|
+
if (gpuDevice) {
|
|
609
|
+
if (isUsableGPUDevice(gpuDevice)) {
|
|
610
|
+
return gpuDevice;
|
|
611
|
+
}
|
|
612
|
+
clearActiveDeviceState();
|
|
613
|
+
advanceDeviceEpoch();
|
|
614
|
+
}
|
|
615
|
+
if (sharedDeviceState.deviceInitPromise) {
|
|
616
|
+
const device = await sharedDeviceState.deviceInitPromise;
|
|
617
|
+
hydrateDeviceState();
|
|
618
|
+
return device;
|
|
619
|
+
}
|
|
620
|
+
const initialization = initializeDevice();
|
|
621
|
+
sharedDeviceState.deviceInitPromise = initialization;
|
|
622
|
+
try {
|
|
623
|
+
const device = await initialization;
|
|
624
|
+
hydrateDeviceState();
|
|
625
|
+
return device;
|
|
626
|
+
} finally {
|
|
627
|
+
if (sharedDeviceState.deviceInitPromise === initialization) {
|
|
628
|
+
sharedDeviceState.deviceInitPromise = null;
|
|
629
|
+
}
|
|
630
|
+
}
|
|
631
|
+
}
|
|
632
|
+
|
|
566
633
|
export function setDevice(device, options = {}) {
|
|
567
634
|
hydrateDeviceState();
|
|
568
635
|
if (!device) {
|
|
@@ -574,6 +641,7 @@ export function setDevice(device, options = {}) {
|
|
|
574
641
|
gpuDevice = device;
|
|
575
642
|
lastDeviceLossInfo = null;
|
|
576
643
|
ensureGpuBufferConstructor(gpuDevice);
|
|
644
|
+
wrapDeviceCreateBuffer(gpuDevice);
|
|
577
645
|
wrapDeviceCreateBindGroup(gpuDevice);
|
|
578
646
|
registerDeviceLostHandler(gpuDevice);
|
|
579
647
|
advanceDeviceEpoch();
|
|
@@ -7,6 +7,10 @@ import { acquireBuffer, releaseBuffer } from '../../../memory/buffer-pool.js';
|
|
|
7
7
|
|
|
8
8
|
const MAX_ADAM_ELEMENTS_PER_DISPATCH = 65535 * WORKGROUP_SIZES.DEFAULT;
|
|
9
9
|
|
|
10
|
+
function tensorElementCount(tensor) {
|
|
11
|
+
return tensor.shape.reduce((product, value) => product * value, 1);
|
|
12
|
+
}
|
|
13
|
+
|
|
10
14
|
async function runAdamChunked(device, pipeline, params, grads, moment1, moment2, options, inferredCount) {
|
|
11
15
|
if (params.dtype !== 'f32' || grads.dtype !== 'f32' || moment1.dtype !== 'f32' || moment2.dtype !== 'f32') {
|
|
12
16
|
throw new Error(
|
|
@@ -94,8 +98,7 @@ export async function runAdam(
|
|
|
94
98
|
const device = getDevice();
|
|
95
99
|
const { count, step = 1, lr, beta1, beta2, eps } = options;
|
|
96
100
|
|
|
97
|
-
const
|
|
98
|
-
const inferredCount = count ?? Math.floor(params.buffer.size / bytesPerElement);
|
|
101
|
+
const inferredCount = count ?? tensorElementCount(params);
|
|
99
102
|
const pipeline = await createPipeline('adam', 'default');
|
|
100
103
|
|
|
101
104
|
if (inferredCount > MAX_ADAM_ELEMENTS_PER_DISPATCH) {
|
|
@@ -166,8 +169,7 @@ export async function recordAdam(
|
|
|
166
169
|
const device = recorder.device;
|
|
167
170
|
const { count, step = 1, lr, beta1, beta2, eps } = options;
|
|
168
171
|
|
|
169
|
-
const
|
|
170
|
-
const inferredCount = count ?? Math.floor(params.buffer.size / bytesPerElement);
|
|
172
|
+
const inferredCount = count ?? tensorElementCount(params);
|
|
171
173
|
const pipeline = await createPipeline('adam', 'default');
|
|
172
174
|
|
|
173
175
|
const uniformBuffer = createUniformBufferWithView(
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import type { CommandRecorder } from '../../command-recorder.js';
|
|
2
|
+
import type { Tensor } from '../../tensor.js';
|
|
3
|
+
|
|
4
|
+
export interface CausalConv1dSiluBackwardOptions {
|
|
5
|
+
numTokens: number;
|
|
6
|
+
channels: number;
|
|
7
|
+
kernelSize: number;
|
|
8
|
+
outputBuffer?: GPUBuffer | null;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export declare function runCausalConv1dSiluBackward(
|
|
12
|
+
input: Tensor,
|
|
13
|
+
weight: Tensor,
|
|
14
|
+
gradOutput: Tensor,
|
|
15
|
+
options: CausalConv1dSiluBackwardOptions
|
|
16
|
+
): Promise<Tensor>;
|
|
17
|
+
|
|
18
|
+
export declare function recordCausalConv1dSiluBackward(
|
|
19
|
+
recorder: CommandRecorder,
|
|
20
|
+
input: Tensor,
|
|
21
|
+
weight: Tensor,
|
|
22
|
+
gradOutput: Tensor,
|
|
23
|
+
options: CausalConv1dSiluBackwardOptions
|
|
24
|
+
): Promise<Tensor>;
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import { getDevice } from '../../device.js';
|
|
2
|
+
import { acquireBuffer } from '../../../memory/buffer-pool.js';
|
|
3
|
+
import { createTensor } from '../../tensor.js';
|
|
4
|
+
import { WORKGROUP_SIZES } from '../constants.js';
|
|
5
|
+
import { dispatch, recordDispatch } from '../dispatch.js';
|
|
6
|
+
import { createPipeline, createUniformBufferWithView } from '../utils.js';
|
|
7
|
+
import { releaseUniformBuffer } from '../../uniform-cache.js';
|
|
8
|
+
|
|
9
|
+
function validate(input, weight, gradOutput, options) {
|
|
10
|
+
const numTokens = Math.floor(Number(options?.numTokens));
|
|
11
|
+
const channels = Math.floor(Number(options?.channels));
|
|
12
|
+
const kernelSize = Math.floor(Number(options?.kernelSize));
|
|
13
|
+
if (numTokens < 1 || channels < 1 || kernelSize < 1) {
|
|
14
|
+
throw new Error('causal conv1d SiLU backward requires numTokens, channels, and kernelSize.');
|
|
15
|
+
}
|
|
16
|
+
for (const [label, tensor] of [['input', input], ['weight', weight], ['gradOutput', gradOutput]]) {
|
|
17
|
+
if (tensor?.dtype !== 'f32') {
|
|
18
|
+
throw new Error(`causal conv1d SiLU backward requires f32 ${label}.`);
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
return { numTokens, channels, kernelSize };
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
async function execute(recorder, input, weight, gradOutput, options = {}) {
|
|
25
|
+
const dims = validate(input, weight, gradOutput, options);
|
|
26
|
+
const device = recorder?.device || getDevice();
|
|
27
|
+
if (!device) throw new Error('causal conv1d SiLU backward requires an active GPU device.');
|
|
28
|
+
const outputBuffer = options.outputBuffer
|
|
29
|
+
|| acquireBuffer(dims.numTokens * dims.channels * 4, undefined, 'causal_conv1d_silu_backward_output');
|
|
30
|
+
const pipeline = await createPipeline('causal_conv1d_silu_backward', 'default');
|
|
31
|
+
const uniformBuffer = createUniformBufferWithView(
|
|
32
|
+
'causal_conv1d_silu_backward_uniforms',
|
|
33
|
+
16,
|
|
34
|
+
(view) => {
|
|
35
|
+
view.setUint32(0, dims.numTokens, true);
|
|
36
|
+
view.setUint32(4, dims.channels, true);
|
|
37
|
+
view.setUint32(8, dims.kernelSize, true);
|
|
38
|
+
},
|
|
39
|
+
recorder || null,
|
|
40
|
+
device
|
|
41
|
+
);
|
|
42
|
+
const bindGroup = device.createBindGroup({
|
|
43
|
+
label: 'causal_conv1d_silu_backward_bind_group',
|
|
44
|
+
layout: pipeline.getBindGroupLayout(0),
|
|
45
|
+
entries: [
|
|
46
|
+
{ binding: 0, resource: { buffer: uniformBuffer } },
|
|
47
|
+
{ binding: 1, resource: { buffer: input.buffer } },
|
|
48
|
+
{ binding: 2, resource: { buffer: weight.buffer } },
|
|
49
|
+
{ binding: 3, resource: { buffer: gradOutput.buffer } },
|
|
50
|
+
{ binding: 4, resource: { buffer: outputBuffer } },
|
|
51
|
+
],
|
|
52
|
+
});
|
|
53
|
+
const workgroups = Math.ceil((dims.numTokens * dims.channels) / WORKGROUP_SIZES.DEFAULT);
|
|
54
|
+
if (recorder) {
|
|
55
|
+
recordDispatch(recorder, pipeline, bindGroup, workgroups, 'causal_conv1d_silu_backward');
|
|
56
|
+
} else {
|
|
57
|
+
dispatch(device, pipeline, bindGroup, workgroups, 'causal_conv1d_silu_backward');
|
|
58
|
+
releaseUniformBuffer(uniformBuffer);
|
|
59
|
+
}
|
|
60
|
+
return createTensor(
|
|
61
|
+
outputBuffer,
|
|
62
|
+
'f32',
|
|
63
|
+
[dims.numTokens, dims.channels],
|
|
64
|
+
'causal_conv1d_silu_backward_output'
|
|
65
|
+
);
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
export function runCausalConv1dSiluBackward(input, weight, gradOutput, options = {}) {
|
|
69
|
+
return execute(null, input, weight, gradOutput, options);
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
export function recordCausalConv1dSiluBackward(recorder, input, weight, gradOutput, options = {}) {
|
|
73
|
+
return execute(recorder, input, weight, gradOutput, options);
|
|
74
|
+
}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
override WORKGROUP_SIZE: u32 = 256u;
|
|
2
|
+
|
|
3
|
+
struct Uniforms {
|
|
4
|
+
num_tokens: u32,
|
|
5
|
+
channels: u32,
|
|
6
|
+
kernel_size: u32,
|
|
7
|
+
_pad0: u32,
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
@group(0) @binding(0) var<uniform> u: Uniforms;
|
|
11
|
+
@group(0) @binding(1) var<storage, read> input: array<f32>;
|
|
12
|
+
@group(0) @binding(2) var<storage, read> weight: array<f32>;
|
|
13
|
+
@group(0) @binding(3) var<storage, read> grad_output: array<f32>;
|
|
14
|
+
@group(0) @binding(4) var<storage, read_write> grad_input: array<f32>;
|
|
15
|
+
|
|
16
|
+
fn sigmoid(x: f32) -> f32 {
|
|
17
|
+
if (x >= 0.0) {
|
|
18
|
+
let z = exp(-x);
|
|
19
|
+
return 1.0 / (1.0 + z);
|
|
20
|
+
}
|
|
21
|
+
let z = exp(x);
|
|
22
|
+
return z / (1.0 + z);
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
@compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
|
|
26
|
+
fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
|
|
27
|
+
let element = gid.x;
|
|
28
|
+
let element_count = u.num_tokens * u.channels;
|
|
29
|
+
if (element >= element_count) {
|
|
30
|
+
return;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
let token = element / u.channels;
|
|
34
|
+
let channel = element % u.channels;
|
|
35
|
+
let last_output_token = min(u.num_tokens - 1u, token + u.kernel_size - 1u);
|
|
36
|
+
var gradient: f32 = 0.0;
|
|
37
|
+
|
|
38
|
+
for (var output_token = token; output_token <= last_output_token; output_token = output_token + 1u) {
|
|
39
|
+
let kernel_index = token + u.kernel_size - 1u - output_token;
|
|
40
|
+
var raw: f32 = 0.0;
|
|
41
|
+
for (var kernel: u32 = 0u; kernel < u.kernel_size; kernel = kernel + 1u) {
|
|
42
|
+
let padded_source = i32(output_token) + i32(kernel) - i32(u.kernel_size) + 1;
|
|
43
|
+
if (padded_source < 0) {
|
|
44
|
+
continue;
|
|
45
|
+
}
|
|
46
|
+
let source = u32(padded_source) * u.channels + channel;
|
|
47
|
+
raw = raw + input[source] * weight[channel * u.kernel_size + kernel];
|
|
48
|
+
}
|
|
49
|
+
let probability = sigmoid(raw);
|
|
50
|
+
let silu_derivative = probability * (1.0 + raw * (1.0 - probability));
|
|
51
|
+
let output_offset = output_token * u.channels + channel;
|
|
52
|
+
gradient = gradient
|
|
53
|
+
+ grad_output[output_offset]
|
|
54
|
+
* silu_derivative
|
|
55
|
+
* weight[channel * u.kernel_size + kernel_index];
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
grad_input[element] = gradient;
|
|
59
|
+
}
|
|
@@ -30,6 +30,10 @@ fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
|
|
|
30
30
|
let token_idx = idx / u.vocab_size;
|
|
31
31
|
let class_idx = idx % u.vocab_size;
|
|
32
32
|
let target_idx = targets[token_idx];
|
|
33
|
+
if (target_idx >= u.vocab_size) {
|
|
34
|
+
output[idx] = 0.0;
|
|
35
|
+
return;
|
|
36
|
+
}
|
|
33
37
|
var grad = softmax[idx];
|
|
34
38
|
if (class_idx == target_idx) {
|
|
35
39
|
grad = grad - 1.0;
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import type { Tensor } from '../../tensor.js';
|
|
2
|
+
|
|
3
|
+
export interface GatedDeltaRecurrentBackwardInputs {
|
|
4
|
+
query: Tensor;
|
|
5
|
+
key: Tensor;
|
|
6
|
+
value: Tensor;
|
|
7
|
+
logDecay: Tensor;
|
|
8
|
+
beta: Tensor;
|
|
9
|
+
stateHistory: Tensor;
|
|
10
|
+
gradOutput: Tensor;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
export interface GatedDeltaRecurrentBackwardOptions {
|
|
14
|
+
numTokens: number;
|
|
15
|
+
numHeads: number;
|
|
16
|
+
keyDim: number;
|
|
17
|
+
valueDim: number;
|
|
18
|
+
queryScale: number;
|
|
19
|
+
gradQueryBuffer?: GPUBuffer | null;
|
|
20
|
+
gradKeyBuffer?: GPUBuffer | null;
|
|
21
|
+
gradValueBuffer?: GPUBuffer | null;
|
|
22
|
+
gradLogDecayBuffer?: GPUBuffer | null;
|
|
23
|
+
gradBetaBuffer?: GPUBuffer | null;
|
|
24
|
+
gradStateBuffer?: GPUBuffer | null;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export interface GatedDeltaRecurrentBackwardResult {
|
|
28
|
+
query: Tensor;
|
|
29
|
+
key: Tensor;
|
|
30
|
+
value: Tensor;
|
|
31
|
+
logDecay: Tensor;
|
|
32
|
+
beta: Tensor;
|
|
33
|
+
initialState: Tensor;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export declare function runGatedDeltaRecurrentBackward(
|
|
37
|
+
inputs: GatedDeltaRecurrentBackwardInputs,
|
|
38
|
+
options: GatedDeltaRecurrentBackwardOptions
|
|
39
|
+
): Promise<GatedDeltaRecurrentBackwardResult>;
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
import { getDevice } from '../../device.js';
|
|
2
|
+
import { acquireBuffer, releaseBuffer } from '../../../memory/buffer-pool.js';
|
|
3
|
+
import { createTensor } from '../../tensor.js';
|
|
4
|
+
import { dispatch } from '../dispatch.js';
|
|
5
|
+
import { createPipeline, createUniformBufferWithView } from '../utils.js';
|
|
6
|
+
import { releaseUniformBuffer } from '../../uniform-cache.js';
|
|
7
|
+
|
|
8
|
+
function validate(inputs, options) {
|
|
9
|
+
const numTokens = Math.floor(Number(options?.numTokens));
|
|
10
|
+
const numHeads = Math.floor(Number(options?.numHeads));
|
|
11
|
+
const keyDim = Math.floor(Number(options?.keyDim));
|
|
12
|
+
const valueDim = Math.floor(Number(options?.valueDim));
|
|
13
|
+
const queryScale = Number(options?.queryScale);
|
|
14
|
+
if (numTokens < 1 || numHeads < 1 || keyDim < 1 || valueDim < 1 || valueDim > 128) {
|
|
15
|
+
throw new Error('gated-delta recurrent backward requires positive dimensions and valueDim <= 128.');
|
|
16
|
+
}
|
|
17
|
+
if (!Number.isFinite(queryScale)) {
|
|
18
|
+
throw new Error('gated-delta recurrent backward requires finite queryScale.');
|
|
19
|
+
}
|
|
20
|
+
for (const [label, tensor] of Object.entries(inputs)) {
|
|
21
|
+
if (tensor?.dtype !== 'f32') {
|
|
22
|
+
throw new Error(`gated-delta recurrent backward requires f32 ${label}.`);
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
return { numTokens, numHeads, keyDim, valueDim, queryScale };
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
function allocate(bytes, label, provided) {
|
|
29
|
+
return provided || acquireBuffer(bytes, undefined, label);
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export async function runGatedDeltaRecurrentBackward(inputs, options = {}) {
|
|
33
|
+
const dims = validate(inputs, options);
|
|
34
|
+
const device = getDevice();
|
|
35
|
+
if (!device) throw new Error('gated-delta recurrent backward requires an active GPU device.');
|
|
36
|
+
const queryBytes = dims.numTokens * dims.numHeads * dims.keyDim * 4;
|
|
37
|
+
const valueBytes = dims.numTokens * dims.numHeads * dims.valueDim * 4;
|
|
38
|
+
const scalarBytes = dims.numTokens * dims.numHeads * 4;
|
|
39
|
+
const stateBytes = dims.numHeads * dims.keyDim * dims.valueDim * 4;
|
|
40
|
+
const packedQueryKeyBuffer = acquireBuffer(queryBytes * 2, undefined, 'gated_delta_packed_query_key');
|
|
41
|
+
const packedDecayBetaBuffer = acquireBuffer(scalarBytes * 2, undefined, 'gated_delta_packed_decay_beta');
|
|
42
|
+
const packedGradQueryKeyBuffer = acquireBuffer(queryBytes * 2, undefined, 'gated_delta_packed_grad_query_key');
|
|
43
|
+
const packedGradDecayBetaBuffer = acquireBuffer(scalarBytes * 2, undefined, 'gated_delta_packed_grad_decay_beta');
|
|
44
|
+
const gradQueryBuffer = allocate(queryBytes, 'gated_delta_grad_query', options.gradQueryBuffer);
|
|
45
|
+
const gradKeyBuffer = allocate(queryBytes, 'gated_delta_grad_key', options.gradKeyBuffer);
|
|
46
|
+
const gradValueBuffer = allocate(valueBytes, 'gated_delta_grad_value', options.gradValueBuffer);
|
|
47
|
+
const gradLogDecayBuffer = allocate(scalarBytes, 'gated_delta_grad_log_decay', options.gradLogDecayBuffer);
|
|
48
|
+
const gradBetaBuffer = allocate(scalarBytes, 'gated_delta_grad_beta', options.gradBetaBuffer);
|
|
49
|
+
const gradStateBuffer = allocate(stateBytes, 'gated_delta_grad_initial_state', options.gradStateBuffer);
|
|
50
|
+
let uniformBuffer = null;
|
|
51
|
+
let completed = false;
|
|
52
|
+
try {
|
|
53
|
+
const packEncoder = device.createCommandEncoder({ label: 'gated_delta_backward_pack_inputs' });
|
|
54
|
+
packEncoder.copyBufferToBuffer(inputs.query.buffer, 0, packedQueryKeyBuffer, 0, queryBytes);
|
|
55
|
+
packEncoder.copyBufferToBuffer(inputs.key.buffer, 0, packedQueryKeyBuffer, queryBytes, queryBytes);
|
|
56
|
+
packEncoder.copyBufferToBuffer(inputs.logDecay.buffer, 0, packedDecayBetaBuffer, 0, scalarBytes);
|
|
57
|
+
packEncoder.copyBufferToBuffer(inputs.beta.buffer, 0, packedDecayBetaBuffer, scalarBytes, scalarBytes);
|
|
58
|
+
device.queue.submit([packEncoder.finish()]);
|
|
59
|
+
device.queue.writeBuffer(gradStateBuffer, 0, new Uint8Array(stateBytes));
|
|
60
|
+
|
|
61
|
+
const pipeline = await createPipeline('gated_delta_recurrent_backward', 'default');
|
|
62
|
+
uniformBuffer = createUniformBufferWithView(
|
|
63
|
+
'gated_delta_recurrent_backward_uniforms',
|
|
64
|
+
32,
|
|
65
|
+
(view) => {
|
|
66
|
+
view.setUint32(0, dims.numTokens, true);
|
|
67
|
+
view.setUint32(4, dims.numHeads, true);
|
|
68
|
+
view.setUint32(8, dims.keyDim, true);
|
|
69
|
+
view.setUint32(12, dims.valueDim, true);
|
|
70
|
+
view.setFloat32(16, dims.queryScale, true);
|
|
71
|
+
},
|
|
72
|
+
null,
|
|
73
|
+
device
|
|
74
|
+
);
|
|
75
|
+
const bindGroup = device.createBindGroup({
|
|
76
|
+
label: 'gated_delta_recurrent_backward_bind_group',
|
|
77
|
+
layout: pipeline.getBindGroupLayout(0),
|
|
78
|
+
entries: [
|
|
79
|
+
{ binding: 0, resource: { buffer: uniformBuffer } },
|
|
80
|
+
{ binding: 1, resource: { buffer: packedQueryKeyBuffer } },
|
|
81
|
+
{ binding: 2, resource: { buffer: inputs.value.buffer } },
|
|
82
|
+
{ binding: 3, resource: { buffer: packedDecayBetaBuffer } },
|
|
83
|
+
{ binding: 4, resource: { buffer: inputs.stateHistory.buffer } },
|
|
84
|
+
{ binding: 5, resource: { buffer: inputs.gradOutput.buffer } },
|
|
85
|
+
{ binding: 6, resource: { buffer: packedGradQueryKeyBuffer } },
|
|
86
|
+
{ binding: 7, resource: { buffer: gradValueBuffer } },
|
|
87
|
+
{ binding: 8, resource: { buffer: packedGradDecayBetaBuffer } },
|
|
88
|
+
{ binding: 9, resource: { buffer: gradStateBuffer } },
|
|
89
|
+
],
|
|
90
|
+
});
|
|
91
|
+
dispatch(device, pipeline, bindGroup, dims.numHeads, 'gated_delta_recurrent_backward');
|
|
92
|
+
const unpackEncoder = device.createCommandEncoder({ label: 'gated_delta_backward_unpack_outputs' });
|
|
93
|
+
unpackEncoder.copyBufferToBuffer(packedGradQueryKeyBuffer, 0, gradQueryBuffer, 0, queryBytes);
|
|
94
|
+
unpackEncoder.copyBufferToBuffer(packedGradQueryKeyBuffer, queryBytes, gradKeyBuffer, 0, queryBytes);
|
|
95
|
+
unpackEncoder.copyBufferToBuffer(packedGradDecayBetaBuffer, 0, gradLogDecayBuffer, 0, scalarBytes);
|
|
96
|
+
unpackEncoder.copyBufferToBuffer(packedGradDecayBetaBuffer, scalarBytes, gradBetaBuffer, 0, scalarBytes);
|
|
97
|
+
device.queue.submit([unpackEncoder.finish()]);
|
|
98
|
+
await device.queue.onSubmittedWorkDone();
|
|
99
|
+
completed = true;
|
|
100
|
+
return {
|
|
101
|
+
query: createTensor(gradQueryBuffer, 'f32', [dims.numTokens, dims.numHeads, dims.keyDim], 'gated_delta_grad_query'),
|
|
102
|
+
key: createTensor(gradKeyBuffer, 'f32', [dims.numTokens, dims.numHeads, dims.keyDim], 'gated_delta_grad_key'),
|
|
103
|
+
value: createTensor(gradValueBuffer, 'f32', [dims.numTokens, dims.numHeads, dims.valueDim], 'gated_delta_grad_value'),
|
|
104
|
+
logDecay: createTensor(gradLogDecayBuffer, 'f32', [dims.numTokens, dims.numHeads], 'gated_delta_grad_log_decay'),
|
|
105
|
+
beta: createTensor(gradBetaBuffer, 'f32', [dims.numTokens, dims.numHeads], 'gated_delta_grad_beta'),
|
|
106
|
+
initialState: createTensor(gradStateBuffer, 'f32', [dims.numHeads, dims.keyDim, dims.valueDim], 'gated_delta_grad_initial_state'),
|
|
107
|
+
};
|
|
108
|
+
} finally {
|
|
109
|
+
await device.queue.onSubmittedWorkDone().catch(() => {});
|
|
110
|
+
if (uniformBuffer) releaseUniformBuffer(uniformBuffer);
|
|
111
|
+
releaseBuffer(packedQueryKeyBuffer);
|
|
112
|
+
releaseBuffer(packedDecayBetaBuffer);
|
|
113
|
+
releaseBuffer(packedGradQueryKeyBuffer);
|
|
114
|
+
releaseBuffer(packedGradDecayBetaBuffer);
|
|
115
|
+
if (!completed) {
|
|
116
|
+
if (!options.gradQueryBuffer) releaseBuffer(gradQueryBuffer);
|
|
117
|
+
if (!options.gradKeyBuffer) releaseBuffer(gradKeyBuffer);
|
|
118
|
+
if (!options.gradValueBuffer) releaseBuffer(gradValueBuffer);
|
|
119
|
+
if (!options.gradLogDecayBuffer) releaseBuffer(gradLogDecayBuffer);
|
|
120
|
+
if (!options.gradBetaBuffer) releaseBuffer(gradBetaBuffer);
|
|
121
|
+
if (!options.gradStateBuffer) releaseBuffer(gradStateBuffer);
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
}
|