doppler-gpu 0.2.2 → 0.4.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. package/README.md +8 -0
  2. package/assets/doppler.svg +1601 -0
  3. package/package.json +6 -5
  4. package/src/cli/doppler-quickstart.js +16 -2
  5. package/src/cli/doppler-serve.js +0 -0
  6. package/src/client/doppler-registry.json +17 -1
  7. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +1 -1
  8. package/src/config/kernels/registry.json +73 -0
  9. package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +32 -0
  10. package/src/config/schema/benchmark.schema.d.ts +2 -0
  11. package/src/config/schema/benchmark.schema.js +1 -0
  12. package/src/config/schema/doppler.schema.d.ts +9 -1
  13. package/src/config/schema/doppler.schema.js +8 -1
  14. package/src/config/schema/index.d.ts +2 -0
  15. package/src/config/schema/index.js +1 -0
  16. package/src/config/schema/manifest.schema.d.ts +5 -0
  17. package/src/converter/quantizer.js +5 -4
  18. package/src/formats/rdrr/source-transform-contract.js +390 -0
  19. package/src/formats/rdrr/types.d.ts +8 -1
  20. package/src/formats/tflite/types.d.ts +8 -0
  21. package/src/formats/tflite/types.js +45 -0
  22. package/src/gpu/command-recorder.d.ts +7 -1
  23. package/src/gpu/command-recorder.js +41 -2
  24. package/src/gpu/kernels/activation-static-qdq.d.ts +22 -0
  25. package/src/gpu/kernels/activation-static-qdq.js +69 -0
  26. package/src/gpu/kernels/activation_static_qdq.wgsl +23 -0
  27. package/src/gpu/kernels/index.d.ts +7 -0
  28. package/src/gpu/kernels/index.js +6 -0
  29. package/src/inference/browser-harness-text-helpers.d.ts +1 -0
  30. package/src/inference/browser-harness-text-helpers.js +44 -4
  31. package/src/inference/browser-harness.js +7 -0
  32. package/src/inference/pipelines/context.js +4 -2
  33. package/src/inference/pipelines/text/attention/types.js +9 -4
  34. package/src/inference/pipelines/text/embed.d.ts +1 -0
  35. package/src/inference/pipelines/text/embed.js +10 -4
  36. package/src/inference/pipelines/text/generator-helpers.d.ts +6 -0
  37. package/src/inference/pipelines/text/generator-helpers.js +20 -1
  38. package/src/inference/pipelines/text/generator-steps.js +17 -8
  39. package/src/inference/pipelines/text/generator.js +51 -3
  40. package/src/inference/pipelines/text/init.d.ts +3 -0
  41. package/src/inference/pipelines/text/init.js +24 -3
  42. package/src/inference/pipelines/text/logits/gpu.d.ts +5 -4
  43. package/src/inference/pipelines/text/logits/gpu.js +146 -11
  44. package/src/inference/pipelines/text/per-layer-inputs.js +320 -57
  45. package/src/inference/pipelines/text/probes.js +79 -1
  46. package/src/inference/pipelines/text/stage-names.js +9 -0
  47. package/src/inference/pipelines/text.js +10 -2
  48. package/src/loader/embedding-loader.js +20 -5
  49. package/src/loader/loader-types.d.ts +2 -0
  50. package/src/loader/manifest-config.d.ts +2 -7
  51. package/src/loader/per-layer-input-loader.js +114 -1
  52. package/src/loader/tensors/source-transform.d.ts +15 -0
  53. package/src/loader/tensors/source-transform.js +312 -34
  54. package/src/loader/tensors/tensor-reader.js +28 -2
  55. package/src/storage/artifact-storage-context.d.ts +1 -0
  56. package/src/storage/artifact-storage-context.js +158 -72
  57. package/src/storage/preflight.js +11 -0
  58. package/src/storage/quickstart-downloader.js +15 -4
  59. package/src/tooling/litert-package-runtime.js +369 -14
  60. package/src/tooling/node-source-runtime.js +256 -140
  61. package/src/tooling/source-runtime-bundle.d.ts +25 -0
  62. package/src/tooling/source-runtime-bundle.js +53 -1
  63. package/src/tooling/source-runtime-materializer.js +3 -1
@@ -0,0 +1,390 @@
1
+ const SUPPORTED_SOURCE_DTYPES = new Set(['INT8', 'UINT8', 'INT4', 'INT2']);
2
+ const SUPPORTED_LITERT_STORAGE_ENCODINGS = new Set(['signed', 'offset_binary']);
3
+ const SUPPORTED_LITERT_SCALE_SEMANTICS = new Set(['step', 'qmax_abs']);
4
+
5
+ function fail(errorPrefix, message) {
6
+ throw new Error(`${errorPrefix} ${message}`);
7
+ }
8
+
9
+ function normalizeSourceDtype(sourceDtype, tensorName, errorPrefix) {
10
+ const normalized = String(sourceDtype || '').trim().toUpperCase();
11
+ if (!SUPPORTED_SOURCE_DTYPES.has(normalized)) {
12
+ fail(
13
+ errorPrefix,
14
+ `Tensor "${tensorName}" has unsupported sourceTransform.sourceDtype "${sourceDtype}".`
15
+ );
16
+ }
17
+ return normalized;
18
+ }
19
+
20
+ function normalizeTargetDtype(location, targetDtype, tensorName, errorPrefix, label = 'sourceTransform') {
21
+ const normalizedTargetDtype = String(targetDtype || '').trim().toUpperCase();
22
+ if (normalizedTargetDtype !== 'F16') {
23
+ fail(
24
+ errorPrefix,
25
+ `Tensor "${tensorName}" has unsupported ${label}.targetDtype "${targetDtype}".`
26
+ );
27
+ }
28
+ const normalizedLocationDtype = String(location?.dtype || '').trim().toUpperCase();
29
+ if (normalizedLocationDtype !== normalizedTargetDtype) {
30
+ fail(
31
+ errorPrefix,
32
+ `Tensor "${tensorName}" ${label} targetDtype "${normalizedTargetDtype}" ` +
33
+ `does not match location.dtype "${location?.dtype}".`
34
+ );
35
+ }
36
+ return normalizedTargetDtype;
37
+ }
38
+
39
+ function normalizePositiveNumber(value, tensorName, errorPrefix, field) {
40
+ const normalized = Number(value);
41
+ if (!Number.isFinite(normalized) || normalized <= 0) {
42
+ fail(
43
+ errorPrefix,
44
+ `Tensor "${tensorName}" has invalid ${field} ${value}.`
45
+ );
46
+ }
47
+ return normalized;
48
+ }
49
+
50
+ function normalizePositiveInteger(value, tensorName, errorPrefix, field) {
51
+ const normalized = Number(value);
52
+ if (!Number.isInteger(normalized) || normalized <= 0) {
53
+ fail(
54
+ errorPrefix,
55
+ `Tensor "${tensorName}" has invalid ${field} ${value}.`
56
+ );
57
+ }
58
+ return normalized;
59
+ }
60
+
61
+ function normalizeSafeInteger(value, tensorName, errorPrefix, field) {
62
+ const normalized = Number(value);
63
+ if (!Number.isSafeInteger(normalized)) {
64
+ fail(
65
+ errorPrefix,
66
+ `Tensor "${tensorName}" has invalid ${field} ${value}.`
67
+ );
68
+ }
69
+ return normalized;
70
+ }
71
+
72
+ function normalizeNonNegativeInteger(value, tensorName, errorPrefix, field) {
73
+ const normalized = Number(value);
74
+ if (!Number.isInteger(normalized) || normalized < 0) {
75
+ fail(
76
+ errorPrefix,
77
+ `Tensor "${tensorName}" has invalid ${field} ${value}.`
78
+ );
79
+ }
80
+ return normalized;
81
+ }
82
+
83
+ function normalizeStorageEncoding(storageEncoding, tensorName, errorPrefix, label) {
84
+ const normalized = String(storageEncoding || '').trim();
85
+ if (!SUPPORTED_LITERT_STORAGE_ENCODINGS.has(normalized)) {
86
+ fail(
87
+ errorPrefix,
88
+ `Tensor "${tensorName}" has unsupported ${label} "${storageEncoding}".`
89
+ );
90
+ }
91
+ return normalized;
92
+ }
93
+
94
+ function normalizeScaleSemantics(transform, tensorName, errorPrefix, label) {
95
+ const scaleSemantics = String(transform?.scaleSemantics || '').trim().toLowerCase();
96
+ if (!SUPPORTED_LITERT_SCALE_SEMANTICS.has(scaleSemantics)) {
97
+ fail(
98
+ errorPrefix,
99
+ `Tensor "${tensorName}" ${label} has unsupported scaleSemantics "${transform?.scaleSemantics}".`
100
+ );
101
+ }
102
+ const scaleDivisor = scaleSemantics === 'qmax_abs'
103
+ ? normalizePositiveNumber(transform?.scaleDivisor, tensorName, errorPrefix, `${label} scaleDivisor`)
104
+ : (transform?.scaleDivisor == null ? undefined : Number(transform.scaleDivisor));
105
+ return {
106
+ scaleSemantics,
107
+ ...(scaleDivisor === undefined ? {} : { scaleDivisor }),
108
+ };
109
+ }
110
+
111
+ function normalizeSourceRef(ref, tensorName, errorPrefix, label, required = false) {
112
+ if (ref == null) {
113
+ if (required) {
114
+ fail(
115
+ errorPrefix,
116
+ `Tensor "${tensorName}" ${label} is missing.`
117
+ );
118
+ }
119
+ return null;
120
+ }
121
+ if (typeof ref !== 'object') {
122
+ fail(
123
+ errorPrefix,
124
+ `Tensor "${tensorName}" ${label} has invalid value.`
125
+ );
126
+ }
127
+ const shard = ref.shardIndex ?? ref.shard;
128
+ return {
129
+ ...ref,
130
+ shard: normalizeNonNegativeInteger(shard, tensorName, errorPrefix, `${label}.shard`),
131
+ offset: normalizeNonNegativeInteger(ref.offset, tensorName, errorPrefix, `${label}.offset`),
132
+ size: normalizePositiveInteger(ref.size, tensorName, errorPrefix, `${label}.size`),
133
+ };
134
+ }
135
+
136
+ function normalizeStorageShape(storageShape, tensorName, errorPrefix, label) {
137
+ if (!Array.isArray(storageShape) || storageShape.length !== 2) {
138
+ fail(
139
+ errorPrefix,
140
+ `Tensor "${tensorName}" ${label} requires storageShape=[rows, cols].`
141
+ );
142
+ }
143
+ const rows = normalizePositiveInteger(storageShape[0], tensorName, errorPrefix, `${label} storageShape[0]`);
144
+ const cols = normalizePositiveInteger(storageShape[1], tensorName, errorPrefix, `${label} storageShape[1]`);
145
+ return [rows, cols];
146
+ }
147
+
148
+ function normalizeLaneOrder(storageLaneOrder, storageBlockSize, tensorName, errorPrefix) {
149
+ if (!Array.isArray(storageLaneOrder) || storageLaneOrder.length !== storageBlockSize) {
150
+ fail(
151
+ errorPrefix,
152
+ `Tensor "${tensorName}" LiteRT blocked axis transform requires storageLaneOrder ` +
153
+ `with ${storageBlockSize} entries.`
154
+ );
155
+ }
156
+ const seen = new Set();
157
+ return storageLaneOrder.map((value) => {
158
+ const normalized = Number(value);
159
+ if (
160
+ !Number.isInteger(normalized)
161
+ || normalized < 0
162
+ || normalized >= storageBlockSize
163
+ || seen.has(normalized)
164
+ ) {
165
+ fail(
166
+ errorPrefix,
167
+ `Tensor "${tensorName}" has invalid LiteRT storageLaneOrder ${JSON.stringify(storageLaneOrder)}.`
168
+ );
169
+ }
170
+ seen.add(normalized);
171
+ return normalized;
172
+ });
173
+ }
174
+
175
+ function normalizeScaleCompanion(transform, tensorName, errorPrefix) {
176
+ const scaleCompanionDtype = String(transform?.scaleCompanionDtype || '').trim().toUpperCase();
177
+ if (!scaleCompanionDtype) {
178
+ return {};
179
+ }
180
+ if (scaleCompanionDtype !== 'UINT8') {
181
+ fail(
182
+ errorPrefix,
183
+ `Tensor "${tensorName}" LiteRT axis transform has unsupported scaleCompanionDtype "${transform?.scaleCompanionDtype}".`
184
+ );
185
+ }
186
+ if (!transform?.scaleCompanionDequant || typeof transform.scaleCompanionDequant !== 'object') {
187
+ fail(
188
+ errorPrefix,
189
+ `Tensor "${tensorName}" LiteRT axis transform has UINT8 scale companion without scaleCompanionDequant metadata.`
190
+ );
191
+ }
192
+ return {
193
+ scaleCompanionDtype,
194
+ scaleCompanionDequant: {
195
+ scale: normalizePositiveNumber(
196
+ transform.scaleCompanionDequant.scale,
197
+ tensorName,
198
+ errorPrefix,
199
+ 'scaleCompanionDequant.scale'
200
+ ),
201
+ zeroPoint: normalizeSafeInteger(
202
+ transform.scaleCompanionDequant.zeroPoint,
203
+ tensorName,
204
+ errorPrefix,
205
+ 'scaleCompanionDequant.zeroPoint'
206
+ ),
207
+ },
208
+ };
209
+ }
210
+
211
+ export function normalizeTensorSourceTransform(location, tensorName, options = {}) {
212
+ const transform = location?.sourceTransform;
213
+ if (!transform || typeof transform !== 'object') {
214
+ return null;
215
+ }
216
+ const errorPrefix = options.errorPrefix ?? '[DopplerLoader]';
217
+ const kind = String(transform.kind || '').trim();
218
+ const sourceDtype = normalizeSourceDtype(transform.sourceDtype, tensorName, errorPrefix);
219
+
220
+ if (kind === 'affine_dequant') {
221
+ if (transform.scheme !== 'per_tensor_affine') {
222
+ fail(
223
+ errorPrefix,
224
+ `Tensor "${tensorName}" has unsupported sourceTransform.scheme "${transform.scheme}".`
225
+ );
226
+ }
227
+ return {
228
+ kind,
229
+ scheme: transform.scheme,
230
+ sourceDtype,
231
+ targetDtype: normalizeTargetDtype(location, transform.targetDtype, tensorName, errorPrefix),
232
+ scale: normalizePositiveNumber(transform.scale, tensorName, errorPrefix, 'sourceTransform.scale'),
233
+ zeroPoint: normalizeSafeInteger(transform.zeroPoint, tensorName, errorPrefix, 'sourceTransform.zeroPoint'),
234
+ };
235
+ }
236
+
237
+ if (kind === 'litert_rowwise_dequant') {
238
+ if (transform.scheme !== 'per_row_affine') {
239
+ fail(
240
+ errorPrefix,
241
+ `Tensor "${tensorName}" has unsupported sourceTransform.scheme "${transform.scheme}".`
242
+ );
243
+ }
244
+ return {
245
+ kind,
246
+ scheme: transform.scheme,
247
+ sourceDtype,
248
+ targetDtype: normalizeTargetDtype(
249
+ location,
250
+ transform.targetDtype,
251
+ tensorName,
252
+ errorPrefix,
253
+ 'LiteRT row-wise sourceTransform'
254
+ ),
255
+ storageEncoding: normalizeStorageEncoding(
256
+ transform.storageEncoding,
257
+ tensorName,
258
+ errorPrefix,
259
+ 'LiteRT storageEncoding'
260
+ ),
261
+ ...normalizeScaleSemantics(transform, tensorName, errorPrefix, 'LiteRT row-wise transform'),
262
+ scaleSource: normalizeSourceRef(transform.scaleSource, tensorName, errorPrefix, 'LiteRT row-wise scaleSource', true),
263
+ ...(transform.rowSumSource == null
264
+ ? {}
265
+ : {
266
+ rowSumSource: normalizeSourceRef(
267
+ transform.rowSumSource,
268
+ tensorName,
269
+ errorPrefix,
270
+ 'LiteRT row-wise rowSumSource'
271
+ ),
272
+ }),
273
+ };
274
+ }
275
+
276
+ if (kind === 'litert_axis_dequant') {
277
+ const quantAxis = Number(transform.quantAxis);
278
+ if (quantAxis !== 0 && quantAxis !== 1) {
279
+ fail(
280
+ errorPrefix,
281
+ `Tensor "${tensorName}" has invalid LiteRT quantAxis ${transform.quantAxis}.`
282
+ );
283
+ }
284
+ return {
285
+ kind,
286
+ scheme: transform.scheme === 'per_axis_affine'
287
+ ? transform.scheme
288
+ : fail(
289
+ errorPrefix,
290
+ `Tensor "${tensorName}" has unsupported sourceTransform.scheme "${transform.scheme}".`
291
+ ),
292
+ sourceDtype,
293
+ targetDtype: normalizeTargetDtype(
294
+ location,
295
+ transform.targetDtype,
296
+ tensorName,
297
+ errorPrefix,
298
+ 'LiteRT axis sourceTransform'
299
+ ),
300
+ storageEncoding: normalizeStorageEncoding(
301
+ transform.storageEncoding,
302
+ tensorName,
303
+ errorPrefix,
304
+ 'LiteRT storageEncoding'
305
+ ),
306
+ ...normalizeScaleSemantics(transform, tensorName, errorPrefix, 'LiteRT axis transform'),
307
+ storageShape: normalizeStorageShape(transform.storageShape, tensorName, errorPrefix, 'LiteRT axis transform'),
308
+ quantAxis,
309
+ scaleSource: normalizeSourceRef(transform.scaleSource, tensorName, errorPrefix, 'LiteRT axis scaleSource', true),
310
+ ...normalizeScaleCompanion(transform, tensorName, errorPrefix),
311
+ ...(transform.sumSource == null
312
+ ? {}
313
+ : {
314
+ sumSource: normalizeSourceRef(
315
+ transform.sumSource,
316
+ tensorName,
317
+ errorPrefix,
318
+ 'LiteRT axis sumSource'
319
+ ),
320
+ }),
321
+ };
322
+ }
323
+
324
+ if (kind === 'litert_axis_blocked_dequant') {
325
+ if (transform.scheme !== 'per_axis_affine') {
326
+ fail(
327
+ errorPrefix,
328
+ `Tensor "${tensorName}" has unsupported sourceTransform.scheme "${transform.scheme}".`
329
+ );
330
+ }
331
+ const quantAxis = Number(transform.quantAxis);
332
+ if (quantAxis !== 0) {
333
+ fail(
334
+ errorPrefix,
335
+ `Tensor "${tensorName}" LiteRT blocked axis transform only supports quantAxis=0.`
336
+ );
337
+ }
338
+ const storageBlockSize = normalizePositiveInteger(
339
+ transform.storageBlockSize,
340
+ tensorName,
341
+ errorPrefix,
342
+ 'LiteRT blocked axis storageBlockSize'
343
+ );
344
+ return {
345
+ kind,
346
+ scheme: transform.scheme,
347
+ sourceDtype,
348
+ targetDtype: normalizeTargetDtype(
349
+ location,
350
+ transform.targetDtype,
351
+ tensorName,
352
+ errorPrefix,
353
+ 'LiteRT blocked axis sourceTransform'
354
+ ),
355
+ storageEncoding: normalizeStorageEncoding(
356
+ transform.storageEncoding,
357
+ tensorName,
358
+ errorPrefix,
359
+ 'LiteRT storageEncoding'
360
+ ),
361
+ ...normalizeScaleSemantics(transform, tensorName, errorPrefix, 'LiteRT blocked axis transform'),
362
+ storageShape: normalizeStorageShape(transform.storageShape, tensorName, errorPrefix, 'LiteRT blocked axis transform'),
363
+ quantAxis,
364
+ storageBlockSize,
365
+ storageLaneOrder: normalizeLaneOrder(transform.storageLaneOrder, storageBlockSize, tensorName, errorPrefix),
366
+ scaleSource: normalizeSourceRef(
367
+ transform.scaleSource,
368
+ tensorName,
369
+ errorPrefix,
370
+ 'LiteRT blocked axis scaleSource',
371
+ true
372
+ ),
373
+ ...(transform.sumSource == null
374
+ ? {}
375
+ : {
376
+ sumSource: normalizeSourceRef(
377
+ transform.sumSource,
378
+ tensorName,
379
+ errorPrefix,
380
+ 'LiteRT blocked axis sumSource'
381
+ ),
382
+ }),
383
+ };
384
+ }
385
+
386
+ fail(
387
+ errorPrefix,
388
+ `Tensor "${tensorName}" has unsupported sourceTransform.kind "${transform.kind}".`
389
+ );
390
+ }
@@ -52,7 +52,7 @@ export interface TensorSourceLocationRef {
52
52
  }
53
53
 
54
54
  export interface TensorSourceTransform {
55
- kind: 'affine_dequant' | 'litert_rowwise_dequant' | 'litert_axis_dequant';
55
+ kind: 'affine_dequant' | 'litert_rowwise_dequant' | 'litert_axis_dequant' | 'litert_axis_blocked_dequant';
56
56
  scheme: 'per_tensor_affine' | 'per_row_affine' | 'per_axis_affine';
57
57
  sourceDtype: 'INT8' | 'UINT8' | 'INT4' | 'INT2';
58
58
  targetDtype: 'F16';
@@ -63,9 +63,16 @@ export interface TensorSourceTransform {
63
63
  scaleDivisor?: number;
64
64
  storageShape?: number[];
65
65
  quantAxis?: 0 | 1;
66
+ storageBlockSize?: number;
67
+ storageLaneOrder?: number[];
66
68
  scaleSource?: TensorSourceLocationRef;
67
69
  rowSumSource?: TensorSourceLocationRef;
68
70
  sumSource?: TensorSourceLocationRef;
71
+ scaleCompanionDtype?: 'UINT8';
72
+ scaleCompanionDequant?: {
73
+ scale: number;
74
+ zeroPoint: number;
75
+ };
69
76
  }
70
77
 
71
78
  // =============================================================================
@@ -54,12 +54,20 @@ export interface TFLiteTensor {
54
54
  sourceTransform?: TFLiteSourceTransform;
55
55
  }
56
56
 
57
+ export interface TFLiteMetadataEntry {
58
+ name: string;
59
+ buffer: number;
60
+ offset: number;
61
+ size: number;
62
+ }
63
+
57
64
  export interface ParsedTFLite {
58
65
  schemaVersion: number;
59
66
  description: string | null;
60
67
  subgraphCount: number;
61
68
  mainSubgraphName: string | null;
62
69
  tensors: TFLiteTensor[];
70
+ metadataEntries: TFLiteMetadataEntry[];
63
71
  sourceQuantization: 'F32' | 'F16' | 'BF16' | null;
64
72
  }
65
73
 
@@ -469,6 +469,49 @@ async function readBufferLocation(reader, bufferPos, tensorName) {
469
469
  return size > 0 ? { offset, size } : null;
470
470
  }
471
471
 
472
+ async function readMetadataEntries(reader, modelPos, bufferCount) {
473
+ const metadataCount = await reader.readTableVectorLength(modelPos, 6);
474
+ const metadataEntries = [];
475
+ for (let index = 0; index < metadataCount; index++) {
476
+ const metadataPos = await reader.readTableVectorEntry(modelPos, 6, index);
477
+ if (metadataPos == null) {
478
+ continue;
479
+ }
480
+ const name = await reader.readStringAtField(metadataPos, 0);
481
+ if (!name) {
482
+ continue;
483
+ }
484
+ const bufferFieldPos = await reader.getTableFieldPos(metadataPos, 1);
485
+ const bufferIndex = bufferFieldPos == null ? 0 : await reader.readUint32(bufferFieldPos);
486
+ if (bufferIndex === 0) {
487
+ continue;
488
+ }
489
+ if (bufferIndex >= bufferCount) {
490
+ throw new Error(
491
+ `TFLite parser: metadata "${name}" references buffer ${bufferIndex}, ` +
492
+ `but the model only has ${bufferCount} buffers.`
493
+ );
494
+ }
495
+ const bufferPos = await reader.readTableVectorEntry(modelPos, 4, bufferIndex);
496
+ if (bufferPos == null) {
497
+ throw new Error(
498
+ `TFLite parser: missing buffer table entry ${bufferIndex} for metadata "${name}".`
499
+ );
500
+ }
501
+ const dataLocation = await readBufferLocation(reader, bufferPos, `metadata "${name}"`);
502
+ if (!dataLocation || dataLocation.size <= 0) {
503
+ continue;
504
+ }
505
+ metadataEntries.push({
506
+ name,
507
+ buffer: bufferIndex,
508
+ offset: dataLocation.offset,
509
+ size: dataLocation.size,
510
+ });
511
+ }
512
+ return metadataEntries;
513
+ }
514
+
472
515
  async function parseTFLiteReader(reader, options = {}) {
473
516
  const parseOptions = normalizeParseOptions(options);
474
517
  const rootOffset = await reader.readInt32(0);
@@ -610,6 +653,7 @@ async function parseTFLiteReader(reader, options = {}) {
610
653
  'TFLite parser: no supported constant tensors were found in the main subgraph.'
611
654
  );
612
655
  }
656
+ const metadataEntries = await readMetadataEntries(reader, modelPos, bufferCount);
613
657
 
614
658
  return {
615
659
  schemaVersion,
@@ -617,6 +661,7 @@ async function parseTFLiteReader(reader, options = {}) {
617
661
  subgraphCount,
618
662
  mainSubgraphName: mainSubgraphName || null,
619
663
  tensors,
664
+ metadataEntries,
620
665
  sourceQuantization: resolveDirectSourceQuantization(tensors),
621
666
  };
622
667
  }
@@ -113,6 +113,12 @@ export declare class CommandRecorder {
113
113
  */
114
114
  trackTemporaryBuffer(buffer: GPUBuffer): void;
115
115
 
116
+ /**
117
+ * Schedule async work that must run after GPU submission completes but before
118
+ * recorder-owned cleanup makes the captured data unavailable.
119
+ */
120
+ enqueueCompletionTask(task: () => Promise<void> | void): void;
121
+
116
122
  /**
117
123
  * Submit all recorded commands and clean up temporary buffers.
118
124
  * After calling this, the recorder cannot be reused.
@@ -123,7 +129,7 @@ export declare class CommandRecorder {
123
129
  * Finalize a deferred-cleanup submit after another wait condition
124
130
  * has already established GPU completion.
125
131
  */
126
- completeDeferredCleanup(options?: RecorderDeferredCleanupOptions): void;
132
+ completeDeferredCleanup(options?: RecorderDeferredCleanupOptions): Promise<void>;
127
133
 
128
134
  /**
129
135
  * Submit and wait for GPU to complete (useful for debugging/profiling).
@@ -36,6 +36,8 @@ export class CommandRecorder {
36
36
 
37
37
  #deferredCleanup = null;
38
38
 
39
+ #completionTasks;
40
+
39
41
 
40
42
  #submitted;
41
43
 
@@ -80,6 +82,7 @@ export class CommandRecorder {
80
82
  this.#pooledBufferSet = new Set();
81
83
  this.#cleanupPromise = null;
82
84
  this.#deferredCleanup = null;
85
+ this.#completionTasks = [];
83
86
 
84
87
  // Track if already submitted
85
88
  this.#submitted = false;
@@ -292,6 +295,31 @@ export class CommandRecorder {
292
295
  // Treating them as temporary would destroy weights at submit time.
293
296
  }
294
297
 
298
+ enqueueCompletionTask(task) {
299
+ if (this.#submitted) {
300
+ throw new Error('[CommandRecorder] Cannot enqueue completion tasks after submit');
301
+ }
302
+ if (typeof task !== 'function') {
303
+ throw new Error('[CommandRecorder] completion task must be a function');
304
+ }
305
+ this.#completionTasks.push(task);
306
+ }
307
+
308
+ async #runCompletionTasks() {
309
+ if (this.#completionTasks.length === 0) {
310
+ return;
311
+ }
312
+ const tasks = this.#completionTasks;
313
+ this.#completionTasks = [];
314
+ for (const task of tasks) {
315
+ try {
316
+ await task();
317
+ } catch (error) {
318
+ log.warn('CommandRecorder', `Completion task failed: ${error instanceof Error ? error.message : String(error)}`);
319
+ }
320
+ }
321
+ }
322
+
295
323
  #finalizeTrackedBuffers(buffersToDestroy, buffersToRelease, discardPooled) {
296
324
  for (const buffer of buffersToDestroy) {
297
325
  buffer.destroy();
@@ -353,6 +381,13 @@ export class CommandRecorder {
353
381
  this.#cleanupPromise = null;
354
382
 
355
383
  if (cleanup === 'deferred') {
384
+ if (this.#completionTasks.length > 0) {
385
+ this.#cleanupPromise = this.device.queue.onSubmittedWorkDone().then(async () => {
386
+ await this.#runCompletionTasks();
387
+ }).catch((err) => {
388
+ log.warn('CommandRecorder', `Completion tasks failed: ${ (err).message}`);
389
+ });
390
+ }
356
391
  this.#deferredCleanup = {
357
392
  buffersToDestroy,
358
393
  buffersToRelease,
@@ -361,7 +396,8 @@ export class CommandRecorder {
361
396
  return;
362
397
  }
363
398
 
364
- this.#cleanupPromise = this.device.queue.onSubmittedWorkDone().then(() => {
399
+ this.#cleanupPromise = this.device.queue.onSubmittedWorkDone().then(async () => {
400
+ await this.#runCompletionTasks();
365
401
  this.#submitLatencyMs = performance.now() - submitStart;
366
402
  this.#finalizeTrackedBuffers(buffersToDestroy, buffersToRelease, false);
367
403
  }).catch((err) => {
@@ -370,8 +406,11 @@ export class CommandRecorder {
370
406
  });
371
407
  }
372
408
 
373
- completeDeferredCleanup(options = {}) {
409
+ async completeDeferredCleanup(options = {}) {
374
410
  const discardPooled = options.discardPooled === true;
411
+ if (this.#cleanupPromise) {
412
+ await this.#cleanupPromise;
413
+ }
375
414
  this.#finalizeDeferredCleanup(discardPooled);
376
415
  }
377
416
 
@@ -0,0 +1,22 @@
1
+ import type { CommandRecorder } from '../command-recorder.js';
2
+ import type { Tensor } from '../tensor.js';
3
+
4
+ export interface ActivationStaticQdqOptions {
5
+ count?: number;
6
+ outputBuffer?: GPUBuffer | null;
7
+ qmin?: number;
8
+ qmax?: number;
9
+ }
10
+
11
+ export declare function runActivationStaticQdq(
12
+ input: Tensor,
13
+ scale: number,
14
+ options?: ActivationStaticQdqOptions
15
+ ): Promise<Tensor>;
16
+
17
+ export declare function recordActivationStaticQdq(
18
+ recorder: CommandRecorder,
19
+ input: Tensor,
20
+ scale: number,
21
+ options?: ActivationStaticQdqOptions
22
+ ): Promise<Tensor>;
@@ -0,0 +1,69 @@
1
+ import { acquireBuffer, releaseBuffer } from '../../memory/buffer-pool.js';
2
+ import { createTensor, dtypeBytes } from '../tensor.js';
3
+ import { WORKGROUP_SIZES } from './constants.js';
4
+ import { unifiedKernelWrapper } from './utils.js';
5
+
6
+ function resolveCount(tensor, countOverride) {
7
+ if (Number.isFinite(countOverride) && countOverride > 0) {
8
+ return Math.floor(countOverride);
9
+ }
10
+ if (Array.isArray(tensor.shape) && tensor.shape.length > 0) {
11
+ return tensor.shape.reduce((acc, value) => acc * value, 1);
12
+ }
13
+ return Math.floor(tensor.buffer.size / dtypeBytes(tensor.dtype));
14
+ }
15
+
16
+ async function executeActivationStaticQdq(target, input, scale, options = {}) {
17
+ if (input.dtype !== 'f32') {
18
+ throw new Error(`activation_static_qdq: unsupported dtype ${input.dtype}.`);
19
+ }
20
+
21
+ const { count, outputBuffer = null, qmin = -128, qmax = 127 } = options;
22
+ const resolvedScale = Number(scale);
23
+ const resolvedQmin = Number(qmin);
24
+ const resolvedQmax = Number(qmax);
25
+ if (!Number.isFinite(resolvedScale) || resolvedScale <= 0) {
26
+ throw new Error(`activation_static_qdq: scale must be > 0. Got ${String(scale)}.`);
27
+ }
28
+ if (!Number.isFinite(resolvedQmin) || !Number.isFinite(resolvedQmax) || resolvedQmin > resolvedQmax) {
29
+ throw new Error(
30
+ `activation_static_qdq: invalid qmin/qmax range (${String(qmin)}, ${String(qmax)}).`
31
+ );
32
+ }
33
+
34
+ const inferredCount = resolveCount(input, count);
35
+ const outputSize = inferredCount * dtypeBytes(input.dtype);
36
+ const ownsOutput = outputBuffer == null;
37
+ const output = outputBuffer || acquireBuffer(outputSize, undefined, 'activation_static_qdq_output');
38
+
39
+ try {
40
+ await unifiedKernelWrapper(
41
+ 'activation_static_qdq',
42
+ target,
43
+ 'default',
44
+ [input, output],
45
+ {
46
+ size: inferredCount,
47
+ scale: resolvedScale,
48
+ invScale: 1 / resolvedScale,
49
+ qmin: resolvedQmin,
50
+ qmax: resolvedQmax,
51
+ },
52
+ Math.ceil(inferredCount / WORKGROUP_SIZES.DEFAULT)
53
+ );
54
+ return createTensor(output, input.dtype, [...input.shape], 'activation_static_qdq_output');
55
+ } catch (error) {
56
+ if (ownsOutput) {
57
+ releaseBuffer(output);
58
+ }
59
+ throw error;
60
+ }
61
+ }
62
+
63
+ export async function runActivationStaticQdq(input, scale, options = {}) {
64
+ return executeActivationStaticQdq(null, input, scale, options);
65
+ }
66
+
67
+ export async function recordActivationStaticQdq(recorder, input, scale, options = {}) {
68
+ return executeActivationStaticQdq(recorder, input, scale, options);
69
+ }