doppler-gpu 0.4.14 → 0.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (222) hide show
  1. package/CHANGELOG.md +9 -0
  2. package/README.md +83 -52
  3. package/package.json +25 -2
  4. package/src/cli/doppler-cli.js +253 -94
  5. package/src/cli/runtime-profiles.js +3 -7
  6. package/src/client/doppler-api.d.ts +28 -0
  7. package/src/client/doppler-api.js +2 -0
  8. package/src/client/inspection.d.ts +122 -0
  9. package/src/client/inspection.js +343 -0
  10. package/src/client/runtime/index.d.ts +54 -0
  11. package/src/client/runtime/index.js +190 -18
  12. package/src/client/runtime/model-manager.d.ts +0 -4
  13. package/src/client/runtime/model-manager.js +0 -28
  14. package/src/client/runtime/model-session.d.ts +116 -1
  15. package/src/client/runtime/model-session.js +351 -1
  16. package/src/client/runtime/model-source.d.ts +10 -1
  17. package/src/client/runtime/scoped-session.d.ts +117 -0
  18. package/src/client/runtime/scoped-session.js +257 -0
  19. package/src/client/runtime/types.d.ts +0 -1
  20. package/src/client/runtime/types.js +0 -1
  21. package/src/config/evidence/boundary-evidence-policy.json +54 -0
  22. package/src/config/evidence/source-intake-policy.json +122 -0
  23. package/src/config/inspection/observation-policies.json +80 -0
  24. package/src/config/param-validator.js +21 -36
  25. package/src/config/runtime/compute/f16-activations.json +1 -0
  26. package/src/config/runtime/compute/f16-batched.json +1 -0
  27. package/src/config/runtime/default.json +1 -0
  28. package/src/config/runtime/diagnostics/debug-logits.json +1 -0
  29. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-alt.json +1 -0
  30. package/src/config/runtime/experiments/bench/gemma3-bench-q4k-batched.json +1 -0
  31. package/src/config/runtime/experiments/bench/gemma3-bench-q4k.json +1 -0
  32. package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +1 -0
  33. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-alt.json +1 -0
  34. package/src/config/runtime/experiments/bench/qwen3-bench-q4k-batched.json +1 -0
  35. package/src/config/runtime/experiments/debug/gemma3-browser-diagnose-q4k.json +1 -1
  36. package/src/config/runtime/experiments/debug/gemma3-debug-q4k.json +1 -3
  37. package/src/config/runtime/experiments/debug/gemma4-e2b-browser-diagnose-q4k.json +1 -1
  38. package/src/config/runtime/experiments/debug/qwen35-browser-diagnose-q4k.json +1 -1
  39. package/src/config/runtime/experiments/verify/gemma3-verify.json +1 -3
  40. package/src/config/runtime/experiments/verify/gemma4-e2b-verify.json +1 -3
  41. package/src/config/runtime/experiments/verify/lfm2-verify.json +1 -3
  42. package/src/config/runtime/experiments/verify/translategemma-verify.json +1 -3
  43. package/src/config/runtime/model/gemma2-debug.json +1 -0
  44. package/src/config/runtime/model/gemma2-pipeline-debug.json +1 -0
  45. package/src/config/runtime/model/gemma2-pipeline.json +1 -0
  46. package/src/config/runtime/model/gemma3-layer-probe.json +1 -0
  47. package/src/config/runtime/model/qwen3-5-layer-probe.json +1 -0
  48. package/src/config/runtime/model/qwen3-5-layer3-boundary-probe.json +1 -0
  49. package/src/config/runtime/model/qwen3-5-linear-attn-debug.json +1 -0
  50. package/src/config/runtime/optimization-candidates/registry.json +5 -0
  51. package/src/config/runtime/platform/metal-apple-q4k.json +1 -0
  52. package/src/config/runtime/profiles/default.json +1 -0
  53. package/src/config/runtime/profiles/diffusiongemma-26b-a4b-throughput.json +1 -0
  54. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-overlapped-probe.json +1 -0
  55. package/src/config/runtime/profiles/gemma3-270m-q4k-throughput-probe.json +1 -0
  56. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-b4.json +1 -3
  57. package/src/config/runtime/profiles/gemma4-12b-rdrr-safe-single.json +1 -3
  58. package/src/config/runtime/profiles/gemma4-31b-f16-activations-probe.json +1 -5
  59. package/src/config/runtime/profiles/gemma4-e2b-baseline-probe.json +1 -3
  60. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-probe.json +1 -3
  61. package/src/config/runtime/profiles/gemma4-e2b-fused-rmsnorm-test.json +1 -0
  62. package/src/config/runtime/profiles/gemma4-e2b-int4ple-baseline-probe.json +1 -5
  63. package/src/config/runtime/profiles/gemma4-e2b-int4ple-f16a-probe.json +1 -5
  64. package/src/config/runtime/profiles/gemma4-e2b-int4ple-reference-export.json +1 -3
  65. package/src/config/runtime/profiles/gemma4-e2b-int4ple-retain-q4k-probe.json +1 -5
  66. package/src/config/runtime/profiles/gemma4-e2b-low-memory.json +1 -0
  67. package/src/config/runtime/profiles/gemma4-e2b-prefill-profile.json +1 -3
  68. package/src/config/runtime/profiles/gemma4-e2b-throughput-probe.json +1 -0
  69. package/src/config/runtime/profiles/low-memory.json +1 -0
  70. package/src/config/runtime/profiles/production.json +1 -3
  71. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-attn128-candidate.json +1 -0
  72. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-maxbatch64-candidate.json +1 -0
  73. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-normed-ffn-candidate.json +1 -0
  74. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-overlap-ringtokens1-candidate.json +1 -0
  75. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-parity.json +1 -0
  76. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-sequential-candidate.json +1 -0
  77. package/src/config/runtime/profiles/qwen-3-5-0-8b-metal-throughput.json +1 -0
  78. package/src/config/runtime/profiles/qwen-3-5-0-8b-prefill-profile.json +1 -3
  79. package/src/config/runtime/profiles/qwen-3-5-0-8b-throughput-probe.json +1 -0
  80. package/src/config/runtime/profiles/qwen-3-5-2b-f16-primary-throughput.json +1 -5
  81. package/src/config/runtime/profiles/qwen-3-5-2b-metal-parity.json +1 -0
  82. package/src/config/runtime/profiles/qwen-3-5-2b-metal-throughput.json +1 -0
  83. package/src/config/runtime/profiles/qwen-3-5-9b-metal-correctness.json +1 -0
  84. package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-stability.json +1 -3
  85. package/src/config/runtime/profiles/qwen-3-embedding-0-6b-metal-throughput.json +1 -0
  86. package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-stability.json +1 -0
  87. package/src/config/runtime/profiles/qwen-3-reranker-0-6b-metal-throughput.json +1 -0
  88. package/src/config/runtime/profiles/qwen3-6-27b-f16-activations-probe.json +1 -5
  89. package/src/config/runtime/profiles/rerank-stability.json +1 -3
  90. package/src/config/runtime/profiles/rerank-throughput.json +1 -3
  91. package/src/config/runtime/profiles/simulation.json +1 -3
  92. package/src/config/runtime/profiles/throughput.json +1 -3
  93. package/src/config/runtime/profiles/trace-layers.json +1 -3
  94. package/src/config/runtime/profiles/translategemma-1b-nativekd2-q4k-rdna3-throughput-probe.json +1 -0
  95. package/src/config/runtime/profiles/turboquant-contiguous-prod.json +1 -0
  96. package/src/config/runtime/profiles/turboquant-contiguous.json +1 -0
  97. package/src/config/runtime/profiles/turboquant.json +1 -0
  98. package/src/config/runtime/profiles/vector-stability.json +1 -3
  99. package/src/config/runtime/profiles/vector-throughput.json +1 -0
  100. package/src/config/runtime/profiles/verbose-trace.json +1 -3
  101. package/src/config/runtime/tiers/gemma4-128gb.json +1 -0
  102. package/src/config/runtime/tiers/gemma4-16gb.json +1 -0
  103. package/src/config/runtime/tiers/gemma4-24gb.json +1 -0
  104. package/src/config/runtime/tiers/gemma4-32gb.json +1 -0
  105. package/src/config/schema/benchmark.schema.d.ts +7 -0
  106. package/src/config/schema/benchmark.schema.js +5 -0
  107. package/src/config/schema/index.d.ts +8 -16
  108. package/src/config/schema/index.js +5 -8
  109. package/src/config/schema/kernel-thresholds.schema.d.ts +0 -12
  110. package/src/config/schema/kernel-thresholds.schema.js +0 -12
  111. package/src/config/schema/kernel-warmup.schema.d.ts +1 -3
  112. package/src/config/schema/kernel-warmup.schema.js +0 -1
  113. package/src/config/schema/runtime-profile.schema.d.ts +24 -0
  114. package/src/config/schema/runtime-profile.schema.js +92 -0
  115. package/src/config/schema/shared-runtime.schema.d.ts +1 -4
  116. package/src/config/schema/shared-runtime.schema.js +0 -2
  117. package/src/config/schema/tooling.schema.d.ts +8 -5
  118. package/src/config/schema/tooling.schema.js +3 -2
  119. package/src/debug/tensor.d.ts +2 -0
  120. package/src/debug/tensor.js +21 -2
  121. package/src/gpu/command-recorder.d.ts +5 -0
  122. package/src/gpu/command-recorder.js +17 -0
  123. package/src/gpu/kernel-runtime.js +2 -10
  124. package/src/gpu/kernels/index.d.ts +0 -2
  125. package/src/gpu/kernels/index.js +0 -2
  126. package/src/gpu/kernels/kernel-prewarm.d.ts +3 -0
  127. package/src/gpu/kernels/kernel-prewarm.js +60 -0
  128. package/src/gpu/kernels/utils.d.ts +2 -4
  129. package/src/gpu/kernels/utils.js +2 -4
  130. package/src/index.d.ts +13 -1
  131. package/src/index.js +7 -1
  132. package/src/inference/browser-harness-model-helpers.js +4 -0
  133. package/src/inference/browser-harness-runtime-helpers.js +25 -10
  134. package/src/inference/browser-harness-text-helpers.js +4 -1
  135. package/src/inference/browser-harness.d.ts +4 -0
  136. package/src/inference/browser-harness.js +91 -4
  137. package/src/inference/observation-context.d.ts +22 -0
  138. package/src/inference/observation-context.js +41 -0
  139. package/src/inference/pipelines/text/attention/attn-config.js +1 -1
  140. package/src/inference/pipelines/text/attention/executor-immediate.js +42 -0
  141. package/src/inference/pipelines/text/attention/executor-recorded.js +29 -0
  142. package/src/inference/pipelines/text/attention/index.js +7 -0
  143. package/src/inference/pipelines/text/attention/interpreter.js +1334 -0
  144. package/src/inference/pipelines/text/attention/plan.d.ts +36 -0
  145. package/src/inference/pipelines/text/attention/plan.js +328 -0
  146. package/src/inference/pipelines/text/attention/receipt.js +60 -0
  147. package/src/inference/pipelines/text/attention/record.js +8 -1278
  148. package/src/inference/pipelines/text/attention/run.js +8 -1386
  149. package/src/inference/pipelines/text/generator-decode-policy.js +4 -1
  150. package/src/inference/pipelines/text/generator-helpers.js +2 -0
  151. package/src/inference/pipelines/text/generator-steps.js +32 -15
  152. package/src/inference/pipelines/text/generator.js +10 -5
  153. package/src/inference/pipelines/text/init.d.ts +3 -0
  154. package/src/inference/pipelines/text/layer-plan-gpu.js +2 -0
  155. package/src/inference/pipelines/text/layer.js +2 -0
  156. package/src/inference/pipelines/text/model-load.d.ts +0 -1
  157. package/src/inference/pipelines/text/model-load.js +2 -11
  158. package/src/inference/pipelines/text/probes.js +10 -0
  159. package/src/inference/pipelines/text/resolved-runtime-session.d.ts +56 -0
  160. package/src/inference/pipelines/text/resolved-runtime-session.js +175 -0
  161. package/src/inference/pipelines/text/state.d.ts +6 -0
  162. package/src/inference/pipelines/text/state.js +3 -0
  163. package/src/inference/pipelines/text/types.d.ts +6 -0
  164. package/src/inference/pipelines/text.js +16 -1
  165. package/src/inference/refactor-receipt.d.ts +49 -0
  166. package/src/inference/refactor-receipt.js +188 -0
  167. package/src/inference/resource-scope.d.ts +33 -0
  168. package/src/inference/resource-scope.js +168 -0
  169. package/src/inference/test-harness.d.ts +3 -0
  170. package/src/inference/test-harness.js +1 -0
  171. package/src/storage/model-cache.d.ts +53 -0
  172. package/src/storage/model-cache.js +550 -0
  173. package/src/tooling/boundary-evidence.d.ts +51 -0
  174. package/src/tooling/boundary-evidence.js +408 -0
  175. package/src/tooling/browser-command-runner.js +4 -4
  176. package/src/tooling/command-context.d.ts +25 -0
  177. package/src/tooling/command-context.js +53 -0
  178. package/src/tooling/command-runner-shared.js +22 -5
  179. package/src/tooling/demo-receipts.d.ts +56 -0
  180. package/src/tooling/demo-receipts.js +113 -0
  181. package/src/tooling/diagnose-runner.js +2 -2
  182. package/src/tooling/execution-cost-ledger.d.ts +21 -0
  183. package/src/tooling/execution-cost-ledger.js +199 -0
  184. package/src/tooling/node-command-runner.js +2 -2
  185. package/src/tooling/opfs-cache.d.ts +11 -49
  186. package/src/tooling/opfs-cache.js +4 -510
  187. package/src/tooling/precision-replay-math.d.ts +42 -0
  188. package/src/tooling/precision-replay-math.js +174 -0
  189. package/src/tooling/program-bundle-parity.d.ts +3 -0
  190. package/src/tooling/program-bundle-parity.js +8 -0
  191. package/src/tooling/program-bundle.d.ts +1 -52
  192. package/src/tooling/program-bundle.js +0 -139
  193. package/src/tooling/registered-variant-calibration.d.ts +24 -0
  194. package/src/tooling/registered-variant-calibration.js +286 -0
  195. package/src/tooling/runtime-input-composition.js +105 -53
  196. package/src/tooling/runtime-intent.js +41 -0
  197. package/src/tooling/runtime-optimization-index.d.ts +6 -0
  198. package/src/tooling/runtime-optimization-index.js +72 -0
  199. package/src/tooling/runtime-optimization.d.ts +47 -6
  200. package/src/tooling/runtime-optimization.js +559 -40
  201. package/src/tooling/source-intake.d.ts +49 -0
  202. package/src/tooling/source-intake.js +414 -0
  203. package/src/tooling-exports/evidence.d.ts +34 -0
  204. package/src/tooling-exports/evidence.js +34 -0
  205. package/src/tooling-exports/runtime.d.ts +7 -0
  206. package/src/tooling-exports/runtime.js +7 -0
  207. package/src/tooling-exports.d.ts +11 -0
  208. package/src/tooling-exports.js +6 -3
  209. package/src/tooling-exports.shared.d.ts +32 -0
  210. package/src/tooling-exports.shared.js +31 -0
  211. package/src/version.js +1 -1
  212. package/src/config/schema/tuner.schema.d.ts +0 -39
  213. package/src/config/schema/tuner.schema.js +0 -13
  214. package/src/gpu/kernel-tuner/benchmarks.js +0 -902
  215. package/src/gpu/kernel-tuner/cache.js +0 -145
  216. package/src/gpu/kernel-tuner/index.js +0 -38
  217. package/src/gpu/kernel-tuner/tuner.js +0 -247
  218. package/src/gpu/kernel-tuner.js +0 -12
  219. package/src/gpu/kernels/kernel-tuning.d.ts +0 -42
  220. package/src/gpu/kernels/kernel-tuning.js +0 -158
  221. package/src/gpu/profiler.js +0 -382
  222. package/src/inference/pipelines/text/attention/output-gate-fusion.js +0 -37
@@ -1,902 +0,0 @@
1
-
2
-
3
-
4
- import { DEFAULT_RMS_NORM_EPS } from '../../config/schema/index.js';
5
-
6
- function createRng(seed = 0x9e3779b9) {
7
- let state = seed >>> 0;
8
- if (!state) state = 0x6d2b79f5;
9
- return () => {
10
- state |= 0;
11
- state = (state + 0x6d2b79f5) | 0;
12
- let t = Math.imul(state ^ (state >>> 15), 1 | state);
13
- t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
14
- return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
15
- };
16
- }
17
-
18
- function fillRandom(data, rng) {
19
- for (let i = 0; i < data.length; i++) data[i] = rng();
20
- }
21
-
22
- function destroyBuffer(buffer) {
23
- if (buffer && typeof buffer.destroy === 'function') {
24
- buffer.destroy();
25
- }
26
- }
27
-
28
- function destroyBuffers(...buffers) {
29
- for (const buffer of buffers) {
30
- destroyBuffer(buffer);
31
- }
32
- }
33
-
34
-
35
- export async function benchmarkPipeline(
36
- device,
37
- pipeline,
38
- bindGroup,
39
- workgroups,
40
- warmup,
41
- iterations
42
- ) {
43
- const [wgX, wgY, wgZ] = workgroups;
44
- if (wgX === 0 || wgY === 0 || wgZ === 0) {
45
- return Infinity;
46
- }
47
-
48
- for (let i = 0; i < warmup; i++) {
49
- const encoder = device.createCommandEncoder();
50
- const pass = encoder.beginComputePass();
51
- pass.setPipeline(pipeline);
52
- pass.setBindGroup(0, bindGroup);
53
- pass.dispatchWorkgroups(wgX, wgY, wgZ);
54
- pass.end();
55
- device.queue.submit([encoder.finish()]);
56
- }
57
- await device.queue.onSubmittedWorkDone();
58
-
59
-
60
- const times = [];
61
- for (let i = 0; i < iterations; i++) {
62
- const start = performance.now();
63
- const encoder = device.createCommandEncoder();
64
- const pass = encoder.beginComputePass();
65
- pass.setPipeline(pipeline);
66
- pass.setBindGroup(0, bindGroup);
67
- pass.dispatchWorkgroups(wgX, wgY, wgZ);
68
- pass.end();
69
- device.queue.submit([encoder.finish()]);
70
- await device.queue.onSubmittedWorkDone();
71
- times.push(performance.now() - start);
72
- }
73
-
74
- return times.reduce((a, b) => a + b, 0) / times.length;
75
- }
76
-
77
-
78
- export async function createComputePipeline(
79
- device,
80
- shaderSource,
81
- entryPoint
82
- ) {
83
- const module = device.createShaderModule({ code: shaderSource });
84
- return await device.createComputePipelineAsync({
85
- layout: 'auto',
86
- compute: { module, entryPoint },
87
- });
88
- }
89
-
90
-
91
- export async function tuneMatmul(
92
- device,
93
- inputSizes,
94
- candidates,
95
- warmup,
96
- iterations,
97
- capabilities
98
- ) {
99
- const { M = 1024, N = 1024, K = 1024 } = inputSizes;
100
-
101
- // Filter to 2D candidates for matmul
102
- const matmulCandidates = candidates.filter(c => c[1] > 1);
103
-
104
-
105
- let best = {
106
- optimalWorkgroupSize: [16, 16, 1],
107
- optimalTileSize: 16,
108
- throughput: 0,
109
- timeMs: Infinity,
110
- deviceInfo: capabilities?.adapterInfo,
111
- };
112
-
113
- const bufferA = device.createBuffer({
114
- size: M * K * 4,
115
- usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST,
116
- });
117
- const bufferB = device.createBuffer({
118
- size: K * N * 4,
119
- usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST,
120
- });
121
- const bufferC = device.createBuffer({
122
- size: M * N * 4,
123
- usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_SRC,
124
- });
125
-
126
- try {
127
- const dataA = new Float32Array(M * K);
128
- const dataB = new Float32Array(K * N);
129
- const matmulRng = createRng(0x13579bdf);
130
- fillRandom(dataA, matmulRng);
131
- fillRandom(dataB, matmulRng);
132
- device.queue.writeBuffer(bufferA, 0, dataA);
133
- device.queue.writeBuffer(bufferB, 0, dataB);
134
-
135
- for (const [wgX, wgY] of matmulCandidates) {
136
- let uniformBuffer = null;
137
- try {
138
- const shader = createMatmulShader(wgX, wgY);
139
- const pipeline = await createComputePipeline(device, shader, 'main');
140
-
141
- uniformBuffer = device.createBuffer({
142
- size: 16,
143
- usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST,
144
- });
145
- const uniformData = new Uint32Array([M, N, K, 0]);
146
- device.queue.writeBuffer(uniformBuffer, 0, uniformData);
147
-
148
- const bindGroup = device.createBindGroup({
149
- layout: pipeline.getBindGroupLayout(0),
150
- entries: [
151
- { binding: 0, resource: { buffer: uniformBuffer } },
152
- { binding: 1, resource: { buffer: bufferA } },
153
- { binding: 2, resource: { buffer: bufferB } },
154
- { binding: 3, resource: { buffer: bufferC } },
155
- ],
156
- });
157
-
158
- for (let i = 0; i < warmup; i++) {
159
- const encoder = device.createCommandEncoder();
160
- const pass = encoder.beginComputePass();
161
- pass.setPipeline(pipeline);
162
- pass.setBindGroup(0, bindGroup);
163
- pass.dispatchWorkgroups(Math.ceil(M / wgX), Math.ceil(N / wgY));
164
- pass.end();
165
- device.queue.submit([encoder.finish()]);
166
- }
167
- await device.queue.onSubmittedWorkDone();
168
-
169
- const times = [];
170
- for (let i = 0; i < iterations; i++) {
171
- const start = performance.now();
172
- const encoder = device.createCommandEncoder();
173
- const pass = encoder.beginComputePass();
174
- pass.setPipeline(pipeline);
175
- pass.setBindGroup(0, bindGroup);
176
- pass.dispatchWorkgroups(Math.ceil(M / wgX), Math.ceil(N / wgY));
177
- pass.end();
178
- device.queue.submit([encoder.finish()]);
179
- await device.queue.onSubmittedWorkDone();
180
- times.push(performance.now() - start);
181
- }
182
-
183
- const avgTime = times.reduce((a, b) => a + b, 0) / times.length;
184
- const flops = 2 * M * N * K;
185
- const gflops = (flops / avgTime) / 1e6;
186
-
187
- if (avgTime < best.timeMs) {
188
- best = {
189
- optimalWorkgroupSize: [wgX, wgY, 1],
190
- optimalTileSize: wgX,
191
- throughput: gflops,
192
- timeMs: avgTime,
193
- deviceInfo: capabilities?.adapterInfo,
194
- };
195
- }
196
- } catch (e) {
197
- continue;
198
- } finally {
199
- destroyBuffer(uniformBuffer);
200
- }
201
- }
202
- } finally {
203
- destroyBuffers(bufferA, bufferB, bufferC);
204
- }
205
-
206
- return best;
207
- }
208
-
209
-
210
- export function createMatmulShader(wgX, wgY) {
211
- return `
212
- struct Uniforms {
213
- M: u32, N: u32, K: u32, _pad: u32,
214
- }
215
- @group(0) @binding(0) var<uniform> uniforms: Uniforms;
216
- @group(0) @binding(1) var<storage, read> A: array<f32>;
217
- @group(0) @binding(2) var<storage, read> B: array<f32>;
218
- @group(0) @binding(3) var<storage, read_write> C: array<f32>;
219
-
220
- @compute @workgroup_size(${wgX}, ${wgY}, 1)
221
- fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
222
- let row = gid.x;
223
- let col = gid.y;
224
- if (row >= uniforms.M || col >= uniforms.N) { return; }
225
-
226
- var sum: f32 = 0.0;
227
- for (var k: u32 = 0u; k < uniforms.K; k = k + 1u) {
228
- sum = sum + A[row * uniforms.K + k] * B[k * uniforms.N + col];
229
- }
230
- C[row * uniforms.N + col] = sum;
231
- }`;
232
- }
233
-
234
-
235
- export async function tuneAttention(
236
- device,
237
- inputSizes,
238
- candidates,
239
- warmup,
240
- iterations,
241
- capabilities
242
- ) {
243
- const { seqLen = 2048, numHeads = 32, headDim = 128 } = inputSizes;
244
-
245
-
246
- let best = {
247
- optimalWorkgroupSize: [64, 1, 1],
248
- optimalTileSize: 64,
249
- throughput: 0,
250
- timeMs: Infinity,
251
- deviceInfo: capabilities?.adapterInfo,
252
- };
253
-
254
- const attentionCandidates = candidates.filter(c => c[1] === 1);
255
- if (attentionCandidates.length === 0) {
256
- return best;
257
- }
258
-
259
- const maxElements = 2_000_000;
260
- const totalHeadsRaw = Math.max(1, seqLen * numHeads);
261
- let benchSeqLen = seqLen;
262
- let totalHeads = totalHeadsRaw;
263
- let totalElements = totalHeads * headDim;
264
-
265
- if (totalElements > maxElements) {
266
- benchSeqLen = Math.max(1, Math.floor(maxElements / (numHeads * headDim)));
267
- totalHeads = Math.max(1, benchSeqLen * numHeads);
268
- totalElements = totalHeads * headDim;
269
- }
270
-
271
- const bufferQ = device.createBuffer({
272
- size: totalElements * 4,
273
- usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST,
274
- });
275
- const bufferK = device.createBuffer({
276
- size: totalElements * 4,
277
- usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST,
278
- });
279
- const bufferOut = device.createBuffer({
280
- size: totalHeads * 4,
281
- usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_SRC,
282
- });
283
-
284
- try {
285
- const dataQ = new Float32Array(totalElements);
286
- const dataK = new Float32Array(totalElements);
287
- const attentionRng = createRng(0x2468ace1);
288
- fillRandom(dataQ, attentionRng);
289
- fillRandom(dataK, attentionRng);
290
- device.queue.writeBuffer(bufferQ, 0, dataQ);
291
- device.queue.writeBuffer(bufferK, 0, dataK);
292
-
293
- for (const [wgX] of attentionCandidates) {
294
- let uniformBuffer = null;
295
- try {
296
- const shader = createAttentionShader(wgX);
297
- const pipeline = await createComputePipeline(device, shader, 'main');
298
-
299
- uniformBuffer = device.createBuffer({
300
- size: 16,
301
- usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST,
302
- });
303
- const uniformData = new Uint32Array([headDim, numHeads, benchSeqLen, 0]);
304
- device.queue.writeBuffer(uniformBuffer, 0, uniformData);
305
-
306
- const bindGroup = device.createBindGroup({
307
- layout: pipeline.getBindGroupLayout(0),
308
- entries: [
309
- { binding: 0, resource: { buffer: uniformBuffer } },
310
- { binding: 1, resource: { buffer: bufferQ } },
311
- { binding: 2, resource: { buffer: bufferK } },
312
- { binding: 3, resource: { buffer: bufferOut } },
313
- ],
314
- });
315
-
316
- const avgTime = await benchmarkPipeline(
317
- device,
318
- pipeline,
319
- bindGroup,
320
- [totalHeads, 1, 1],
321
- warmup,
322
- iterations
323
- );
324
-
325
- const flops = 2 * totalHeads * headDim;
326
- const gflops = avgTime > 0 ? (flops / avgTime) / 1e6 : 0;
327
-
328
- if (avgTime < best.timeMs) {
329
- best = {
330
- optimalWorkgroupSize: [wgX, 1, 1],
331
- optimalTileSize: wgX,
332
- throughput: gflops,
333
- timeMs: avgTime,
334
- deviceInfo: capabilities?.adapterInfo,
335
- };
336
- }
337
- } catch (e) {
338
- continue;
339
- } finally {
340
- destroyBuffer(uniformBuffer);
341
- }
342
- }
343
- } finally {
344
- destroyBuffers(bufferQ, bufferK, bufferOut);
345
- }
346
-
347
- return best;
348
- }
349
-
350
-
351
- export function createAttentionShader(wgSize) {
352
- return `
353
- const WG_SIZE: u32 = ${wgSize}u;
354
-
355
- struct Uniforms {
356
- headDim: u32,
357
- numHeads: u32,
358
- seqLen: u32,
359
- _pad: u32,
360
- }
361
-
362
- @group(0) @binding(0) var<uniform> uniforms: Uniforms;
363
- @group(0) @binding(1) var<storage, read> Q: array<f32>;
364
- @group(0) @binding(2) var<storage, read> K: array<f32>;
365
- @group(0) @binding(3) var<storage, read_write> Out: array<f32>;
366
-
367
- var<workgroup> shared: array<f32, WG_SIZE>;
368
-
369
- @compute @workgroup_size(${wgSize}, 1, 1)
370
- fn main(
371
- @builtin(workgroup_id) wg_id: vec3<u32>,
372
- @builtin(local_invocation_id) local_id: vec3<u32>
373
- ) {
374
- let totalHeads = uniforms.numHeads * uniforms.seqLen;
375
- let idx = wg_id.x;
376
- if (idx >= totalHeads) { return; }
377
-
378
- let headDim = uniforms.headDim;
379
- let offset = idx * headDim;
380
- let lane = local_id.x;
381
-
382
- var sum: f32 = 0.0;
383
- var i: u32 = lane;
384
- loop {
385
- if (i >= headDim) { break; }
386
- sum = sum + Q[offset + i] * K[offset + i];
387
- i = i + WG_SIZE;
388
- }
389
-
390
- shared[lane] = sum;
391
- workgroupBarrier();
392
-
393
- var stride: u32 = WG_SIZE / 2u;
394
- loop {
395
- if (stride == 0u) { break; }
396
- if (lane < stride) {
397
- shared[lane] = shared[lane] + shared[lane + stride];
398
- }
399
- workgroupBarrier();
400
- stride = stride / 2u;
401
- }
402
-
403
- if (lane == 0u) {
404
- Out[idx] = shared[0];
405
- }
406
- }`;
407
- }
408
-
409
-
410
- export async function tuneSoftmax(
411
- device,
412
- inputSizes,
413
- candidates,
414
- warmup,
415
- iterations,
416
- capabilities
417
- ) {
418
- const { innerSize = 32000, outerSize = 1 } = inputSizes;
419
-
420
-
421
- let best = {
422
- optimalWorkgroupSize: [256, 1, 1],
423
- optimalTileSize: 256,
424
- throughput: 0,
425
- timeMs: Infinity,
426
- deviceInfo: capabilities?.adapterInfo,
427
- };
428
-
429
- const softmaxCandidates = candidates.filter(c => c[1] === 1);
430
- if (softmaxCandidates.length === 0) {
431
- return best;
432
- }
433
-
434
- const totalElements = Math.max(1, innerSize * outerSize);
435
-
436
- const bufferIn = device.createBuffer({
437
- size: totalElements * 4,
438
- usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST,
439
- });
440
- const bufferOut = device.createBuffer({
441
- size: totalElements * 4,
442
- usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_SRC,
443
- });
444
-
445
- try {
446
- const dataIn = new Float32Array(totalElements);
447
- const softmaxRng = createRng(0x31415926);
448
- fillRandom(dataIn, softmaxRng);
449
- device.queue.writeBuffer(bufferIn, 0, dataIn);
450
-
451
- for (const [wgX] of softmaxCandidates) {
452
- let uniformBuffer = null;
453
- try {
454
- const shader = createSoftmaxShader(wgX);
455
- const pipeline = await createComputePipeline(device, shader, 'main');
456
-
457
- uniformBuffer = device.createBuffer({
458
- size: 16,
459
- usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST,
460
- });
461
- const uniformData = new Uint32Array([innerSize, outerSize, 0, 0]);
462
- device.queue.writeBuffer(uniformBuffer, 0, uniformData);
463
-
464
- const bindGroup = device.createBindGroup({
465
- layout: pipeline.getBindGroupLayout(0),
466
- entries: [
467
- { binding: 0, resource: { buffer: uniformBuffer } },
468
- { binding: 1, resource: { buffer: bufferIn } },
469
- { binding: 2, resource: { buffer: bufferOut } },
470
- ],
471
- });
472
-
473
- const avgTime = await benchmarkPipeline(
474
- device,
475
- pipeline,
476
- bindGroup,
477
- [outerSize, 1, 1],
478
- warmup,
479
- iterations
480
- );
481
-
482
- const ops = 2 * totalElements;
483
- const gops = avgTime > 0 ? (ops / avgTime) / 1e6 : 0;
484
-
485
- if (avgTime < best.timeMs) {
486
- best = {
487
- optimalWorkgroupSize: [wgX, 1, 1],
488
- optimalTileSize: wgX,
489
- throughput: gops,
490
- timeMs: avgTime,
491
- deviceInfo: capabilities?.adapterInfo,
492
- };
493
- }
494
- } catch (e) {
495
- continue;
496
- } finally {
497
- destroyBuffer(uniformBuffer);
498
- }
499
- }
500
- } finally {
501
- destroyBuffers(bufferIn, bufferOut);
502
- }
503
-
504
- return best;
505
- }
506
-
507
-
508
- export function createSoftmaxShader(wgSize) {
509
- return `
510
- const WG_SIZE: u32 = ${wgSize}u;
511
-
512
- struct Uniforms {
513
- innerSize: u32,
514
- outerSize: u32,
515
- _pad0: u32,
516
- _pad1: u32,
517
- }
518
-
519
- @group(0) @binding(0) var<uniform> uniforms: Uniforms;
520
- @group(0) @binding(1) var<storage, read> input: array<f32>;
521
- @group(0) @binding(2) var<storage, read_write> output: array<f32>;
522
-
523
- var<workgroup> shared: array<f32, WG_SIZE>;
524
-
525
- @compute @workgroup_size(${wgSize}, 1, 1)
526
- fn main(
527
- @builtin(workgroup_id) wg_id: vec3<u32>,
528
- @builtin(local_invocation_id) local_id: vec3<u32>
529
- ) {
530
- let row = wg_id.x;
531
- if (row >= uniforms.outerSize) { return; }
532
-
533
- let inner = uniforms.innerSize;
534
- let lane = local_id.x;
535
- let offset = row * inner;
536
-
537
- var localMax: f32 = -3.402823e+38;
538
- var i: u32 = lane;
539
- loop {
540
- if (i >= inner) { break; }
541
- localMax = max(localMax, input[offset + i]);
542
- i = i + WG_SIZE;
543
- }
544
-
545
- shared[lane] = localMax;
546
- workgroupBarrier();
547
-
548
- var stride: u32 = WG_SIZE / 2u;
549
- loop {
550
- if (stride == 0u) { break; }
551
- if (lane < stride) {
552
- shared[lane] = max(shared[lane], shared[lane + stride]);
553
- }
554
- workgroupBarrier();
555
- stride = stride / 2u;
556
- }
557
-
558
- let rowMax = shared[0];
559
- var localSum: f32 = 0.0;
560
- i = lane;
561
- loop {
562
- if (i >= inner) { break; }
563
- localSum = localSum + exp(input[offset + i] - rowMax);
564
- i = i + WG_SIZE;
565
- }
566
-
567
- shared[lane] = localSum;
568
- workgroupBarrier();
569
-
570
- stride = WG_SIZE / 2u;
571
- loop {
572
- if (stride == 0u) { break; }
573
- if (lane < stride) {
574
- shared[lane] = shared[lane] + shared[lane + stride];
575
- }
576
- workgroupBarrier();
577
- stride = stride / 2u;
578
- }
579
-
580
- let denom = shared[0];
581
- i = lane;
582
- loop {
583
- if (i >= inner) { break; }
584
- output[offset + i] = exp(input[offset + i] - rowMax) / denom;
585
- i = i + WG_SIZE;
586
- }
587
- }`;
588
- }
589
-
590
-
591
- export async function tuneRMSNorm(
592
- device,
593
- inputSizes,
594
- candidates,
595
- warmup,
596
- iterations,
597
- capabilities
598
- ) {
599
- const { hiddenSize = 4096, numTokens = 1 } = inputSizes;
600
-
601
-
602
- let best = {
603
- optimalWorkgroupSize: [256, 1, 1],
604
- optimalTileSize: 256,
605
- throughput: 0,
606
- timeMs: Infinity,
607
- deviceInfo: capabilities?.adapterInfo,
608
- };
609
-
610
- const rmsCandidates = candidates.filter(c => c[1] === 1);
611
- if (rmsCandidates.length === 0) {
612
- return best;
613
- }
614
-
615
- const totalElements = Math.max(1, hiddenSize * numTokens);
616
-
617
- const bufferIn = device.createBuffer({
618
- size: totalElements * 4,
619
- usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST,
620
- });
621
- const bufferWeight = device.createBuffer({
622
- size: hiddenSize * 4,
623
- usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST,
624
- });
625
- const bufferOut = device.createBuffer({
626
- size: totalElements * 4,
627
- usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_SRC,
628
- });
629
-
630
- try {
631
- const dataIn = new Float32Array(totalElements);
632
- const dataWeight = new Float32Array(hiddenSize);
633
- const rmsRng = createRng(0x27182818);
634
- fillRandom(dataIn, rmsRng);
635
- fillRandom(dataWeight, rmsRng);
636
- device.queue.writeBuffer(bufferIn, 0, dataIn);
637
- device.queue.writeBuffer(bufferWeight, 0, dataWeight);
638
-
639
- for (const [wgX] of rmsCandidates) {
640
- let uniformBuffer = null;
641
- try {
642
- const shader = createRMSNormShader(wgX);
643
- const pipeline = await createComputePipeline(device, shader, 'main');
644
-
645
- uniformBuffer = device.createBuffer({
646
- size: 16,
647
- usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST,
648
- });
649
- const uniformData = new ArrayBuffer(16);
650
- const uniformView = new DataView(uniformData);
651
- uniformView.setUint32(0, hiddenSize, true);
652
- uniformView.setUint32(4, numTokens, true);
653
- uniformView.setFloat32(8, DEFAULT_RMS_NORM_EPS, true);
654
- uniformView.setUint32(12, 0, true);
655
- device.queue.writeBuffer(uniformBuffer, 0, uniformData);
656
-
657
- const bindGroup = device.createBindGroup({
658
- layout: pipeline.getBindGroupLayout(0),
659
- entries: [
660
- { binding: 0, resource: { buffer: uniformBuffer } },
661
- { binding: 1, resource: { buffer: bufferIn } },
662
- { binding: 2, resource: { buffer: bufferWeight } },
663
- { binding: 3, resource: { buffer: bufferOut } },
664
- ],
665
- });
666
-
667
- const avgTime = await benchmarkPipeline(
668
- device,
669
- pipeline,
670
- bindGroup,
671
- [numTokens, 1, 1],
672
- warmup,
673
- iterations
674
- );
675
-
676
- const ops = 2 * totalElements;
677
- const gops = avgTime > 0 ? (ops / avgTime) / 1e6 : 0;
678
-
679
- if (avgTime < best.timeMs) {
680
- best = {
681
- optimalWorkgroupSize: [wgX, 1, 1],
682
- optimalTileSize: wgX,
683
- throughput: gops,
684
- timeMs: avgTime,
685
- deviceInfo: capabilities?.adapterInfo,
686
- };
687
- }
688
- } catch (e) {
689
- continue;
690
- } finally {
691
- destroyBuffer(uniformBuffer);
692
- }
693
- }
694
- } finally {
695
- destroyBuffers(bufferIn, bufferWeight, bufferOut);
696
- }
697
-
698
- return best;
699
- }
700
-
701
-
702
- export function createRMSNormShader(wgSize) {
703
- return `
704
- const WG_SIZE: u32 = ${wgSize}u;
705
-
706
- struct Uniforms {
707
- hiddenSize: u32,
708
- numTokens: u32,
709
- eps: f32,
710
- _pad0: u32,
711
- }
712
-
713
- @group(0) @binding(0) var<uniform> uniforms: Uniforms;
714
- @group(0) @binding(1) var<storage, read> input: array<f32>;
715
- @group(0) @binding(2) var<storage, read> weight: array<f32>;
716
- @group(0) @binding(3) var<storage, read_write> output: array<f32>;
717
-
718
- var<workgroup> shared: array<f32, WG_SIZE>;
719
-
720
- @compute @workgroup_size(${wgSize}, 1, 1)
721
- fn main(
722
- @builtin(workgroup_id) wg_id: vec3<u32>,
723
- @builtin(local_invocation_id) local_id: vec3<u32>
724
- ) {
725
- let tokenIdx = wg_id.x;
726
- if (tokenIdx >= uniforms.numTokens) { return; }
727
-
728
- let size = uniforms.hiddenSize;
729
- let base = tokenIdx * size;
730
- let lane = local_id.x;
731
-
732
- var localSumSq: f32 = 0.0;
733
- var i: u32 = lane;
734
- loop {
735
- if (i >= size) { break; }
736
- let x = input[base + i];
737
- localSumSq = localSumSq + x * x;
738
- i = i + WG_SIZE;
739
- }
740
-
741
- shared[lane] = localSumSq;
742
- workgroupBarrier();
743
-
744
- var stride: u32 = WG_SIZE / 2u;
745
- loop {
746
- if (stride == 0u) { break; }
747
- if (lane < stride) {
748
- shared[lane] = shared[lane] + shared[lane + stride];
749
- }
750
- workgroupBarrier();
751
- stride = stride / 2u;
752
- }
753
-
754
- let invRms = 1.0 / sqrt(shared[0] / f32(size) + uniforms.eps);
755
- i = lane;
756
- loop {
757
- if (i >= size) { break; }
758
- output[base + i] = input[base + i] * invRms * weight[i];
759
- i = i + WG_SIZE;
760
- }
761
- }`;
762
- }
763
-
764
-
765
- export async function tuneDequant(
766
- device,
767
- inputSizes,
768
- candidates,
769
- warmup,
770
- iterations,
771
- capabilities
772
- ) {
773
- const { numBlocks = 1000 } = inputSizes;
774
-
775
-
776
- let best = {
777
- optimalWorkgroupSize: [64, 1, 1],
778
- optimalTileSize: 64,
779
- throughput: 0,
780
- timeMs: Infinity,
781
- deviceInfo: capabilities?.adapterInfo,
782
- };
783
-
784
- const dequantCandidates = candidates.filter(c => c[1] === 1);
785
- if (dequantCandidates.length === 0) {
786
- return best;
787
- }
788
-
789
- const numElements = Math.max(1, numBlocks * 256);
790
-
791
- const bufferIn = device.createBuffer({
792
- size: numElements * 4,
793
- usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST,
794
- });
795
- const bufferOut = device.createBuffer({
796
- size: numElements * 4,
797
- usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_SRC,
798
- });
799
-
800
- try {
801
- const dataIn = new Uint32Array(numElements);
802
- for (let i = 0; i < numElements; i++) {
803
- dataIn[i] = i & 0xffff;
804
- }
805
- device.queue.writeBuffer(bufferIn, 0, dataIn);
806
-
807
- for (const [wgX] of dequantCandidates) {
808
- let uniformBuffer = null;
809
- try {
810
- const shader = createDequantShader(wgX);
811
- const pipeline = await createComputePipeline(device, shader, 'main');
812
-
813
- uniformBuffer = device.createBuffer({
814
- size: 16,
815
- usage: GPUBufferUsage.UNIFORM | GPUBufferUsage.COPY_DST,
816
- });
817
- const uniformData = new ArrayBuffer(16);
818
- const uniformView = new DataView(uniformData);
819
- uniformView.setUint32(0, numElements, true);
820
- uniformView.setFloat32(4, 0.01, true);
821
- uniformView.setUint32(8, 0, true);
822
- uniformView.setUint32(12, 0, true);
823
- device.queue.writeBuffer(uniformBuffer, 0, uniformData);
824
-
825
- const bindGroup = device.createBindGroup({
826
- layout: pipeline.getBindGroupLayout(0),
827
- entries: [
828
- { binding: 0, resource: { buffer: uniformBuffer } },
829
- { binding: 1, resource: { buffer: bufferIn } },
830
- { binding: 2, resource: { buffer: bufferOut } },
831
- ],
832
- });
833
-
834
- const workgroups = Math.ceil(numElements / wgX);
835
- const avgTime = await benchmarkPipeline(
836
- device,
837
- pipeline,
838
- bindGroup,
839
- [workgroups, 1, 1],
840
- warmup,
841
- iterations
842
- );
843
-
844
- const ops = numElements;
845
- const gops = avgTime > 0 ? (ops / avgTime) / 1e6 : 0;
846
-
847
- if (avgTime < best.timeMs) {
848
- best = {
849
- optimalWorkgroupSize: [wgX, 1, 1],
850
- optimalTileSize: wgX,
851
- throughput: gops,
852
- timeMs: avgTime,
853
- deviceInfo: capabilities?.adapterInfo,
854
- };
855
- }
856
- } catch (e) {
857
- continue;
858
- } finally {
859
- destroyBuffer(uniformBuffer);
860
- }
861
- }
862
- } finally {
863
- destroyBuffers(bufferIn, bufferOut);
864
- }
865
-
866
- return best;
867
- }
868
-
869
-
870
- export function createDequantShader(wgSize) {
871
- return `
872
- const WG_SIZE: u32 = ${wgSize}u;
873
-
874
- struct Uniforms {
875
- count: u32,
876
- scale: f32,
877
- _pad0: u32,
878
- _pad1: u32,
879
- }
880
-
881
- @group(0) @binding(0) var<uniform> uniforms: Uniforms;
882
- @group(0) @binding(1) var<storage, read> input: array<u32>;
883
- @group(0) @binding(2) var<storage, read_write> output: array<f32>;
884
-
885
- @compute @workgroup_size(${wgSize}, 1, 1)
886
- fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
887
- let idx = gid.x;
888
- if (idx >= uniforms.count) { return; }
889
- output[idx] = f32(input[idx]) * uniforms.scale;
890
- }`;
891
- }
892
-
893
-
894
- export function tuneGeneric(capabilities) {
895
- return {
896
- optimalWorkgroupSize: [256, 1, 1],
897
- optimalTileSize: 256,
898
- throughput: 0,
899
- timeMs: 0,
900
- deviceInfo: capabilities?.adapterInfo,
901
- };
902
- }