doppler-gpu 0.2.2 → 0.4.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. package/README.md +8 -0
  2. package/assets/doppler.svg +1601 -0
  3. package/package.json +6 -5
  4. package/src/cli/doppler-quickstart.js +16 -2
  5. package/src/cli/doppler-serve.js +0 -0
  6. package/src/client/doppler-registry.json +17 -1
  7. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +1 -1
  8. package/src/config/kernels/registry.json +73 -0
  9. package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +32 -0
  10. package/src/config/schema/benchmark.schema.d.ts +2 -0
  11. package/src/config/schema/benchmark.schema.js +1 -0
  12. package/src/config/schema/doppler.schema.d.ts +9 -1
  13. package/src/config/schema/doppler.schema.js +8 -1
  14. package/src/config/schema/index.d.ts +2 -0
  15. package/src/config/schema/index.js +1 -0
  16. package/src/config/schema/manifest.schema.d.ts +5 -0
  17. package/src/converter/quantizer.js +5 -4
  18. package/src/formats/rdrr/source-transform-contract.js +390 -0
  19. package/src/formats/rdrr/types.d.ts +8 -1
  20. package/src/formats/tflite/types.d.ts +8 -0
  21. package/src/formats/tflite/types.js +45 -0
  22. package/src/gpu/command-recorder.d.ts +7 -1
  23. package/src/gpu/command-recorder.js +41 -2
  24. package/src/gpu/kernels/activation-static-qdq.d.ts +22 -0
  25. package/src/gpu/kernels/activation-static-qdq.js +69 -0
  26. package/src/gpu/kernels/activation_static_qdq.wgsl +23 -0
  27. package/src/gpu/kernels/index.d.ts +7 -0
  28. package/src/gpu/kernels/index.js +6 -0
  29. package/src/inference/browser-harness-text-helpers.d.ts +1 -0
  30. package/src/inference/browser-harness-text-helpers.js +44 -4
  31. package/src/inference/browser-harness.js +7 -0
  32. package/src/inference/pipelines/context.js +4 -2
  33. package/src/inference/pipelines/text/attention/types.js +9 -4
  34. package/src/inference/pipelines/text/embed.d.ts +1 -0
  35. package/src/inference/pipelines/text/embed.js +10 -4
  36. package/src/inference/pipelines/text/generator-helpers.d.ts +6 -0
  37. package/src/inference/pipelines/text/generator-helpers.js +20 -1
  38. package/src/inference/pipelines/text/generator-steps.js +17 -8
  39. package/src/inference/pipelines/text/generator.js +51 -3
  40. package/src/inference/pipelines/text/init.d.ts +3 -0
  41. package/src/inference/pipelines/text/init.js +24 -3
  42. package/src/inference/pipelines/text/logits/gpu.d.ts +5 -4
  43. package/src/inference/pipelines/text/logits/gpu.js +146 -11
  44. package/src/inference/pipelines/text/per-layer-inputs.js +320 -57
  45. package/src/inference/pipelines/text/probes.js +79 -1
  46. package/src/inference/pipelines/text/stage-names.js +9 -0
  47. package/src/inference/pipelines/text.js +10 -2
  48. package/src/loader/embedding-loader.js +20 -5
  49. package/src/loader/loader-types.d.ts +2 -0
  50. package/src/loader/manifest-config.d.ts +2 -7
  51. package/src/loader/per-layer-input-loader.js +114 -1
  52. package/src/loader/tensors/source-transform.d.ts +15 -0
  53. package/src/loader/tensors/source-transform.js +312 -34
  54. package/src/loader/tensors/tensor-reader.js +28 -2
  55. package/src/storage/artifact-storage-context.d.ts +1 -0
  56. package/src/storage/artifact-storage-context.js +158 -72
  57. package/src/storage/preflight.js +11 -0
  58. package/src/storage/quickstart-downloader.js +15 -4
  59. package/src/tooling/litert-package-runtime.js +369 -14
  60. package/src/tooling/node-source-runtime.js +256 -140
  61. package/src/tooling/source-runtime-bundle.d.ts +25 -0
  62. package/src/tooling/source-runtime-bundle.js +53 -1
  63. package/src/tooling/source-runtime-materializer.js +3 -1
@@ -8,7 +8,7 @@
8
8
  */
9
9
 
10
10
  import type { Tensor } from '../../../../gpu/tensor.js';
11
- import type { CpuWeightBuffer } from '../../../../gpu/weight-buffer.js';
11
+ import type { CpuWeightBuffer, CpuTensorRangeSource } from '../../../../gpu/weight-buffer.js';
12
12
  import type { CommandRecorder } from '../../../../gpu/command-recorder.js';
13
13
  import type { LargeWeightConfigSchema, ProbeConfigSchema, KernelPathSchema } from '../../../../config/schema/index.js';
14
14
  import type { LogitsConfig, LogitsWeights, LogitsDebugFlags } from './types.js';
@@ -32,13 +32,14 @@ export function resolveLmHeadChunkRows(
32
32
  * Extract a chunk of the LM head weight matrix.
33
33
  */
34
34
  export function extractLmHeadChunk(
35
- data: Float32Array,
35
+ data: Float32Array | Uint16Array | CpuTensorRangeSource,
36
36
  layout: 'row' | 'column',
37
37
  hiddenSize: number,
38
38
  vocabSize: number,
39
39
  rowOffset: number,
40
- rowCount: number
41
- ): Float32Array;
40
+ rowCount: number,
41
+ sourceDtype?: 'f16' | 'f32' | 'bf16'
42
+ ): Promise<Float32Array>;
42
43
 
43
44
  /**
44
45
  * Write chunk logits to the full logits buffer.
@@ -21,6 +21,7 @@ import { runProbes } from '../probes.js';
21
21
  import { assertImplicitDtypeTransitionAllowed } from '../dtype-contract.js';
22
22
  import { f16BufferToF32 } from './cpu.js';
23
23
  import { readBufferWithCleanup } from './utils.js';
24
+ import { f16ToF32 } from '../../../../loader/dtype-utils.js';
24
25
 
25
26
  function shouldForceStableF32Logits(config, inputDtype) {
26
27
  if (inputDtype !== 'f16') {
@@ -140,6 +141,101 @@ function createStableF32LogitsKernelPath(kernelPath) {
140
141
  };
141
142
  }
142
143
 
144
+ const bf16ScratchU32 = new Uint32Array(1);
145
+ const bf16ScratchF32 = new Float32Array(bf16ScratchU32.buffer);
146
+
147
+ function bf16ToF32(value) {
148
+ bf16ScratchU32[0] = (value & 0xffff) << 16;
149
+ return bf16ScratchF32[0];
150
+ }
151
+
152
+ function isRangeBackedCpuWeightSource(value) {
153
+ return (
154
+ typeof value === 'object'
155
+ && value !== null
156
+ && value.kind === 'tensor_range_source'
157
+ && typeof value.loadRange === 'function'
158
+ );
159
+ }
160
+
161
+ function normalizeRangeBytes(value, label) {
162
+ if (value instanceof Uint8Array) return value;
163
+ if (value instanceof ArrayBuffer) return new Uint8Array(value);
164
+ if (ArrayBuffer.isView(value)) {
165
+ return new Uint8Array(value.buffer, value.byteOffset, value.byteLength);
166
+ }
167
+ throw new Error(
168
+ `[Logits] ${label} returned unsupported byte payload type "${value?.constructor?.name ?? typeof value}".`
169
+ );
170
+ }
171
+
172
+ function decodeChunkIntoOutput(bytes, sourceDtype, output, dstOffset, valueCount) {
173
+ if (sourceDtype === 'f16') {
174
+ const values = new Uint16Array(bytes.buffer, bytes.byteOffset, valueCount);
175
+ for (let index = 0; index < valueCount; index += 1) {
176
+ output[dstOffset + index] = f16ToF32(values[index]);
177
+ }
178
+ return;
179
+ }
180
+ if (sourceDtype === 'bf16') {
181
+ const values = new Uint16Array(bytes.buffer, bytes.byteOffset, valueCount);
182
+ for (let index = 0; index < valueCount; index += 1) {
183
+ output[dstOffset + index] = bf16ToF32(values[index]);
184
+ }
185
+ return;
186
+ }
187
+ if (((bytes.byteOffset % 4) === 0) && ((bytes.byteLength % 4) === 0)) {
188
+ output.set(new Float32Array(bytes.buffer, bytes.byteOffset, valueCount), dstOffset);
189
+ return;
190
+ }
191
+ const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
192
+ for (let index = 0; index < valueCount; index += 1) {
193
+ output[dstOffset + index] = view.getFloat32(index * 4, true);
194
+ }
195
+ }
196
+
197
+ function readTypedLmHeadChunk(data, layout, hiddenSize, vocabSize, rowOffset, rowCount, sourceDtype) {
198
+ if (data instanceof Float32Array) {
199
+ if (layout === 'row') {
200
+ const start = rowOffset * hiddenSize;
201
+ return data.subarray(start, start + rowCount * hiddenSize);
202
+ }
203
+ const chunk = new Float32Array(hiddenSize * rowCount);
204
+ for (let k = 0; k < hiddenSize; k++) {
205
+ const srcOffset = k * vocabSize + rowOffset;
206
+ const dstOffset = k * rowCount;
207
+ chunk.set(data.subarray(srcOffset, srcOffset + rowCount), dstOffset);
208
+ }
209
+ return chunk;
210
+ }
211
+
212
+ if (!(data instanceof Uint16Array)) {
213
+ throw new Error(
214
+ `[Logits] Unsupported CPU LM head chunk source type "${data?.constructor?.name ?? typeof data}".`
215
+ );
216
+ }
217
+
218
+ const chunk = new Float32Array(hiddenSize * rowCount);
219
+ if (layout === 'row') {
220
+ const start = rowOffset * hiddenSize;
221
+ for (let index = 0; index < rowCount * hiddenSize; index += 1) {
222
+ const raw = data[start + index];
223
+ chunk[index] = sourceDtype === 'bf16' ? bf16ToF32(raw) : f16ToF32(raw);
224
+ }
225
+ return chunk;
226
+ }
227
+
228
+ for (let k = 0; k < hiddenSize; k += 1) {
229
+ const srcOffset = k * vocabSize + rowOffset;
230
+ const dstOffset = k * rowCount;
231
+ for (let index = 0; index < rowCount; index += 1) {
232
+ const raw = data[srcOffset + index];
233
+ chunk[dstOffset + index] = sourceDtype === 'bf16' ? bf16ToF32(raw) : f16ToF32(raw);
234
+ }
235
+ }
236
+ return chunk;
237
+ }
238
+
143
239
 
144
240
  export function resolveCpuWeightDims(lmHead) {
145
241
  if (lmHead.shape.length !== 2) {
@@ -184,24 +280,62 @@ export function resolveLmHeadChunkRows(
184
280
  }
185
281
 
186
282
 
187
- export function extractLmHeadChunk(
283
+ export async function extractLmHeadChunk(
188
284
  data,
189
285
  layout,
190
286
  hiddenSize,
191
287
  vocabSize,
192
288
  rowOffset,
193
- rowCount
289
+ rowCount,
290
+ sourceDtype = 'f32'
194
291
  ) {
195
- if (layout === 'row') {
196
- const start = rowOffset * hiddenSize;
197
- return data.subarray(start, start + rowCount * hiddenSize);
292
+ const normalizedSourceDtype = String(sourceDtype ?? 'f32').toLowerCase();
293
+ if (normalizedSourceDtype !== 'f32' && normalizedSourceDtype !== 'f16' && normalizedSourceDtype !== 'bf16') {
294
+ throw new Error(`[Logits] Unsupported CPU LM head source dtype "${sourceDtype}".`);
295
+ }
296
+ if (!isRangeBackedCpuWeightSource(data)) {
297
+ return readTypedLmHeadChunk(
298
+ data,
299
+ layout,
300
+ hiddenSize,
301
+ vocabSize,
302
+ rowOffset,
303
+ rowCount,
304
+ normalizedSourceDtype
305
+ );
198
306
  }
199
307
 
308
+ const bytesPerElement = normalizedSourceDtype === 'f32' ? 4 : 2;
200
309
  const chunk = new Float32Array(hiddenSize * rowCount);
201
- for (let k = 0; k < hiddenSize; k++) {
202
- const srcOffset = k * vocabSize + rowOffset;
203
- const dstOffset = k * rowCount;
204
- chunk.set(data.subarray(srcOffset, srcOffset + rowCount), dstOffset);
310
+ if (layout === 'row') {
311
+ const byteOffset = rowOffset * hiddenSize * bytesPerElement;
312
+ const byteLength = rowCount * hiddenSize * bytesPerElement;
313
+ const bytes = normalizeRangeBytes(
314
+ await data.loadRange(byteOffset, byteLength),
315
+ 'CPU LM head range source'
316
+ );
317
+ if (bytes.byteLength !== byteLength) {
318
+ throw new Error(
319
+ `[Logits] CPU LM head range source returned ${bytes.byteLength} bytes, expected ${byteLength}.`
320
+ );
321
+ }
322
+ decodeChunkIntoOutput(bytes, normalizedSourceDtype, chunk, 0, rowCount * hiddenSize);
323
+ return chunk;
324
+ }
325
+
326
+ for (let k = 0; k < hiddenSize; k += 1) {
327
+ const byteOffset = (k * vocabSize + rowOffset) * bytesPerElement;
328
+ const byteLength = rowCount * bytesPerElement;
329
+ const bytes = normalizeRangeBytes(
330
+ await data.loadRange(byteOffset, byteLength),
331
+ `CPU LM head range source column ${k}`
332
+ );
333
+ if (bytes.byteLength !== byteLength) {
334
+ throw new Error(
335
+ `[Logits] CPU LM head range source returned ${bytes.byteLength} bytes for column ${k}, expected ${byteLength}.`
336
+ );
337
+ }
338
+ decodeChunkIntoOutput(bytes, normalizedSourceDtype, chunk, k * rowCount, rowCount);
205
339
  }
206
340
  return chunk;
207
341
  }
@@ -271,13 +405,14 @@ export async function computeChunkedLogitsGPU(
271
405
 
272
406
  for (let rowOffset = 0; rowOffset < vocabSize; rowOffset += chunkRows) {
273
407
  const rowCount = Math.min(chunkRows, vocabSize - rowOffset);
274
- const chunkData = extractLmHeadChunk(
408
+ const chunkData = await extractLmHeadChunk(
275
409
  lmHead.data,
276
410
  lmHead.layout,
277
411
  hiddenSize,
278
412
  weightVocabSize,
279
413
  rowOffset,
280
- rowCount
414
+ rowCount,
415
+ lmHead.dtype
281
416
  );
282
417
 
283
418
  const f32Buffer = acquireBuffer(chunkData.byteLength, undefined, 'lm_head_chunk_f32');