doppler-gpu 0.2.1 → 0.4.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. package/CHANGELOG.md +8 -0
  2. package/README.md +9 -1
  3. package/assets/doppler.svg +1601 -0
  4. package/package.json +6 -5
  5. package/src/cli/doppler-quickstart.js +16 -2
  6. package/src/cli/doppler-serve.js +0 -0
  7. package/src/client/doppler-registry.json +17 -1
  8. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +1 -1
  9. package/src/config/kernels/registry.json +73 -0
  10. package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +32 -0
  11. package/src/config/schema/benchmark.schema.d.ts +2 -0
  12. package/src/config/schema/benchmark.schema.js +1 -0
  13. package/src/config/schema/doppler.schema.d.ts +9 -1
  14. package/src/config/schema/doppler.schema.js +8 -1
  15. package/src/config/schema/index.d.ts +2 -0
  16. package/src/config/schema/index.js +1 -0
  17. package/src/config/schema/manifest.schema.d.ts +5 -0
  18. package/src/converter/quantizer.js +5 -4
  19. package/src/formats/rdrr/source-transform-contract.js +390 -0
  20. package/src/formats/rdrr/types.d.ts +8 -1
  21. package/src/formats/tflite/types.d.ts +8 -0
  22. package/src/formats/tflite/types.js +45 -0
  23. package/src/gpu/command-recorder.d.ts +7 -1
  24. package/src/gpu/command-recorder.js +41 -2
  25. package/src/gpu/kernels/activation-static-qdq.d.ts +22 -0
  26. package/src/gpu/kernels/activation-static-qdq.js +69 -0
  27. package/src/gpu/kernels/activation_static_qdq.wgsl +23 -0
  28. package/src/gpu/kernels/index.d.ts +7 -0
  29. package/src/gpu/kernels/index.js +6 -0
  30. package/src/inference/browser-harness-text-helpers.d.ts +1 -0
  31. package/src/inference/browser-harness-text-helpers.js +44 -4
  32. package/src/inference/browser-harness.js +7 -0
  33. package/src/inference/pipelines/context.js +4 -2
  34. package/src/inference/pipelines/text/attention/types.js +9 -4
  35. package/src/inference/pipelines/text/embed.d.ts +1 -0
  36. package/src/inference/pipelines/text/embed.js +10 -4
  37. package/src/inference/pipelines/text/generator-helpers.d.ts +6 -0
  38. package/src/inference/pipelines/text/generator-helpers.js +20 -1
  39. package/src/inference/pipelines/text/generator-steps.js +17 -8
  40. package/src/inference/pipelines/text/generator.js +51 -3
  41. package/src/inference/pipelines/text/init.d.ts +3 -0
  42. package/src/inference/pipelines/text/init.js +24 -3
  43. package/src/inference/pipelines/text/logits/gpu.d.ts +5 -4
  44. package/src/inference/pipelines/text/logits/gpu.js +146 -11
  45. package/src/inference/pipelines/text/per-layer-inputs.js +320 -57
  46. package/src/inference/pipelines/text/probes.js +79 -1
  47. package/src/inference/pipelines/text/stage-names.js +9 -0
  48. package/src/inference/pipelines/text.js +10 -2
  49. package/src/loader/embedding-loader.js +20 -5
  50. package/src/loader/loader-types.d.ts +2 -0
  51. package/src/loader/manifest-config.d.ts +2 -7
  52. package/src/loader/per-layer-input-loader.js +114 -1
  53. package/src/loader/tensors/source-transform.d.ts +15 -0
  54. package/src/loader/tensors/source-transform.js +312 -34
  55. package/src/loader/tensors/tensor-reader.js +28 -2
  56. package/src/storage/artifact-storage-context.d.ts +1 -0
  57. package/src/storage/artifact-storage-context.js +158 -72
  58. package/src/storage/preflight.js +11 -0
  59. package/src/storage/quickstart-downloader.js +15 -4
  60. package/src/tooling/litert-package-runtime.js +369 -14
  61. package/src/tooling/node-source-runtime.js +256 -140
  62. package/src/tooling/source-runtime-bundle.d.ts +25 -0
  63. package/src/tooling/source-runtime-bundle.js +53 -1
  64. package/src/tooling/source-runtime-materializer.js +3 -1
  65. package/src/version.js +1 -1
@@ -1,4 +1,5 @@
1
1
  import { float32ToFloat16 } from '../../converter/quantizer.js';
2
+ import { normalizeTensorSourceTransform } from '../../formats/rdrr/source-transform-contract.js';
2
3
 
3
4
  function computeElementCount(shape, tensorName) {
4
5
  if (!Array.isArray(shape)) {
@@ -211,6 +212,26 @@ function validateLiteRTStorageEncoding(storageEncoding, tensorName) {
211
212
  }
212
213
  }
213
214
 
215
+ function validateLiteRTStorageLaneOrder(storageLaneOrder, storageBlockSize, tensorName) {
216
+ if (!Array.isArray(storageLaneOrder) || storageLaneOrder.length !== storageBlockSize) {
217
+ throw new Error(
218
+ `[DopplerLoader] Tensor "${tensorName}" LiteRT blocked axis transform requires storageLaneOrder ` +
219
+ `with ${storageBlockSize} entries.`
220
+ );
221
+ }
222
+ const seen = new Set();
223
+ for (let index = 0; index < storageLaneOrder.length; index++) {
224
+ const value = Number(storageLaneOrder[index]);
225
+ if (!Number.isInteger(value) || value < 0 || value >= storageBlockSize || seen.has(value)) {
226
+ throw new Error(
227
+ `[DopplerLoader] Tensor "${tensorName}" has invalid LiteRT storageLaneOrder ` +
228
+ `${JSON.stringify(storageLaneOrder)}.`
229
+ );
230
+ }
231
+ seen.add(value);
232
+ }
233
+ }
234
+
214
235
  function resolveLiteRTScaleValue(storedScale, transform, tensorName, rowLabel) {
215
236
  const scaleSemantics = String(transform?.scaleSemantics || '').trim().toLowerCase();
216
237
  if (scaleSemantics === 'step') {
@@ -245,13 +266,14 @@ function getLiteRTCompanionByteLength(companionSource, tensorName, label, expect
245
266
  }
246
267
 
247
268
  export function getSourceTransformSpec(location, tensorName) {
248
- if (!hasSourceTransform(location)) {
269
+ const transform = normalizeTensorSourceTransform(location, tensorName);
270
+ if (!transform) {
249
271
  return null;
250
272
  }
251
- const transform = location.sourceTransform;
252
273
  if (transform.kind === 'affine_dequant') {
253
274
  const elementCount = computeElementCount(location.shape, tensorName);
254
275
  return {
276
+ transform,
255
277
  kind: 'affine_dequant',
256
278
  elementCount,
257
279
  outputByteLength: elementCount * 2,
@@ -314,6 +336,7 @@ export function getSourceTransformSpec(location, tensorName) {
314
336
  }
315
337
  }
316
338
  return {
339
+ transform,
317
340
  kind: 'litert_rowwise_dequant',
318
341
  rows,
319
342
  cols,
@@ -397,6 +420,33 @@ export function getSourceTransformSpec(location, tensorName) {
397
420
  );
398
421
  }
399
422
  }
423
+ const scaleCompanionDtype = String(transform?.scaleCompanionDtype || '').toUpperCase();
424
+ const scaleCompanionDequant = scaleCompanionDtype === 'UINT8'
425
+ ? transform.scaleCompanionDequant
426
+ : null;
427
+ if (scaleCompanionDtype === 'UINT8') {
428
+ if (!scaleCompanionDequant || typeof scaleCompanionDequant !== 'object') {
429
+ throw new Error(
430
+ `[DopplerLoader] Tensor "${tensorName}" LiteRT axis transform has UINT8 scale companion without scaleCompanionDequant metadata.`
431
+ );
432
+ }
433
+ const companionScale = Number(scaleCompanionDequant.scale);
434
+ const companionZeroPoint = Number(scaleCompanionDequant.zeroPoint);
435
+ if (!Number.isFinite(companionScale) || companionScale <= 0) {
436
+ throw new Error(
437
+ `[DopplerLoader] Tensor "${tensorName}" LiteRT axis transform has invalid scaleCompanionDequant.scale ${scaleCompanionDequant.scale}.`
438
+ );
439
+ }
440
+ if (!Number.isSafeInteger(companionZeroPoint)) {
441
+ throw new Error(
442
+ `[DopplerLoader] Tensor "${tensorName}" LiteRT axis transform has invalid scaleCompanionDequant.zeroPoint ${scaleCompanionDequant.zeroPoint}.`
443
+ );
444
+ }
445
+ } else if (scaleCompanionDtype !== '') {
446
+ throw new Error(
447
+ `[DopplerLoader] Tensor "${tensorName}" LiteRT axis transform has unsupported scaleCompanionDtype "${scaleCompanionDtype}".`
448
+ );
449
+ }
400
450
  const rawStorageRowBytes = computeSourceByteLength(storageCols, transform.sourceDtype, tensorName);
401
451
  const sourceByteLength = computeSourceByteLength(
402
452
  storageRows * storageCols,
@@ -404,13 +454,14 @@ export function getSourceTransformSpec(location, tensorName) {
404
454
  tensorName
405
455
  );
406
456
  return {
457
+ transform,
407
458
  kind: 'litert_axis_dequant',
408
459
  rows,
409
460
  cols,
410
461
  storageRows,
411
462
  storageCols,
412
463
  rawStorageRowBytes,
413
- scaleRowBytes: 4,
464
+ scaleRowBytes: scaleCompanionDtype === 'UINT8' ? 4 : 4,
414
465
  targetRowBytes: cols * 2,
415
466
  outputByteLength: rows * cols * 2,
416
467
  sourceByteLength,
@@ -419,6 +470,96 @@ export function getSourceTransformSpec(location, tensorName) {
419
470
  };
420
471
  }
421
472
 
473
+ if (transform.kind === 'litert_axis_blocked_dequant') {
474
+ if (transform.scheme !== 'per_axis_affine') {
475
+ throw new Error(
476
+ `[DopplerLoader] Tensor "${tensorName}" has unsupported sourceTransform.scheme "${transform.scheme}".`
477
+ );
478
+ }
479
+ validateLiteRTTransformTarget(location, tensorName, transform, 'LiteRT blocked axis sourceTransform');
480
+ validateLiteRTStorageEncoding(transform.storageEncoding, tensorName);
481
+ if (!Array.isArray(transform.storageShape) || transform.storageShape.length !== 2) {
482
+ throw new Error(
483
+ `[DopplerLoader] Tensor "${tensorName}" LiteRT blocked axis transform requires storageShape=[rows, cols].`
484
+ );
485
+ }
486
+ const storageRows = Number(transform.storageShape[0]);
487
+ const storageCols = Number(transform.storageShape[1]);
488
+ const storageBlockSize = Number(transform.storageBlockSize);
489
+ if (
490
+ !Number.isInteger(storageRows)
491
+ || storageRows <= 0
492
+ || !Number.isInteger(storageCols)
493
+ || storageCols <= 0
494
+ || !Number.isInteger(storageBlockSize)
495
+ || storageBlockSize <= 0
496
+ ) {
497
+ throw new Error(
498
+ `[DopplerLoader] Tensor "${tensorName}" has invalid LiteRT blocked storage shape ` +
499
+ `${JSON.stringify({ storageShape: transform.storageShape, storageBlockSize: transform.storageBlockSize })}.`
500
+ );
501
+ }
502
+ if (transform.quantAxis !== 0) {
503
+ throw new Error(
504
+ `[DopplerLoader] Tensor "${tensorName}" LiteRT blocked axis transform only supports quantAxis=0.`
505
+ );
506
+ }
507
+ validateLiteRTStorageLaneOrder(transform.storageLaneOrder, storageBlockSize, tensorName);
508
+ if (storageCols !== rows || storageRows * storageBlockSize !== cols) {
509
+ throw new Error(
510
+ `[DopplerLoader] Tensor "${tensorName}" LiteRT blocked axis transform expects storageShape ` +
511
+ `[${cols / storageBlockSize}, ${rows}] for blockSize=${storageBlockSize}, got [${storageRows}, ${storageCols}].`
512
+ );
513
+ }
514
+ const scaleSource = transform.scaleSource;
515
+ if (!scaleSource || typeof scaleSource !== 'object') {
516
+ throw new Error(
517
+ `[DopplerLoader] Tensor "${tensorName}" LiteRT blocked axis transform is missing scaleSource.`
518
+ );
519
+ }
520
+ getLiteRTCompanionByteLength(scaleSource, tensorName, 'blocked axis scale', rows * 4);
521
+ if (transform.sumSource != null && typeof transform.sumSource !== 'object') {
522
+ throw new Error(
523
+ `[DopplerLoader] Tensor "${tensorName}" LiteRT blocked axis transform has invalid sumSource.`
524
+ );
525
+ }
526
+ if (transform.sumSource != null) {
527
+ getLiteRTCompanionByteLength(transform.sumSource, tensorName, 'blocked axis sum', rows * 4);
528
+ }
529
+ if (transform.scaleSemantics !== 'step' && transform.scaleSemantics !== 'qmax_abs') {
530
+ throw new Error(
531
+ `[DopplerLoader] Tensor "${tensorName}" LiteRT blocked axis transform has unsupported scaleSemantics "${transform.scaleSemantics}".`
532
+ );
533
+ }
534
+ if (transform.scaleSemantics === 'qmax_abs') {
535
+ const scaleDivisor = Number(transform.scaleDivisor);
536
+ if (!Number.isFinite(scaleDivisor) || scaleDivisor <= 0) {
537
+ throw new Error(
538
+ `[DopplerLoader] Tensor "${tensorName}" LiteRT blocked axis transform requires scaleDivisor > 0 for scaleSemantics="qmax_abs".`
539
+ );
540
+ }
541
+ }
542
+ const storageElementBytes = computeSourceByteLength(storageBlockSize, transform.sourceDtype, tensorName);
543
+ const rawStorageRowBytes = storageCols * storageElementBytes;
544
+ return {
545
+ transform,
546
+ kind: 'litert_axis_blocked_dequant',
547
+ rows,
548
+ cols,
549
+ storageRows,
550
+ storageCols,
551
+ rawStorageRowBytes,
552
+ storageElementBytes,
553
+ scaleRowBytes: 4,
554
+ targetRowBytes: cols * 2,
555
+ outputByteLength: rows * cols * 2,
556
+ sourceByteLength: storageRows * rawStorageRowBytes,
557
+ quantAxis: transform.quantAxis,
558
+ storageBlockSize,
559
+ storageLaneOrder: transform.storageLaneOrder.map((value) => Number(value)),
560
+ };
561
+ }
562
+
422
563
  throw new Error(
423
564
  `[DopplerLoader] Tensor "${tensorName}" has unsupported sourceTransform.kind "${transform.kind}".`
424
565
  );
@@ -438,9 +579,9 @@ export function materializeTensorSourceTransform(rawBytes, location, tensorName,
438
579
  return rawBytes;
439
580
  }
440
581
 
441
- const transform = location.sourceTransform;
582
+ const spec = getSourceTransformSpec(location, tensorName);
583
+ const transform = spec.transform;
442
584
  if (transform.kind === 'litert_rowwise_dequant') {
443
- const spec = getSourceTransformSpec(location, tensorName);
444
585
  const scaleBytes = options.scaleBytes;
445
586
  if (!(scaleBytes instanceof Uint8Array)) {
446
587
  throw new Error(
@@ -549,7 +690,6 @@ export function materializeTensorSourceTransform(rawBytes, location, tensorName,
549
690
  }
550
691
 
551
692
  if (transform.kind === 'litert_axis_dequant') {
552
- const spec = getSourceTransformSpec(location, tensorName);
553
693
  const scaleBytes = options.scaleBytes;
554
694
  if (!(scaleBytes instanceof Uint8Array)) {
555
695
  throw new Error(
@@ -598,6 +738,46 @@ export function materializeTensorSourceTransform(rawBytes, location, tensorName,
598
738
  ? new DataView(sumBytes.buffer, sumBytes.byteOffset, sumBytes.byteLength)
599
739
  : null;
600
740
  const out = new Uint16Array(rowCount * spec.cols);
741
+ const scaleCompanionDtype = String(transform.scaleCompanionDtype || '').toUpperCase();
742
+ const scaleCompanion = scaleCompanionDtype === 'UINT8'
743
+ ? {
744
+ scale: Number(transform.scaleCompanionDequant?.scale),
745
+ zeroPoint: Number(transform.scaleCompanionDequant?.zeroPoint),
746
+ }
747
+ : null;
748
+ const scaleSubCount = scaleCompanion ? spec.scaleRowBytes : 1;
749
+ const scaleSubBlockWidth = scaleCompanion
750
+ ? Math.floor(spec.cols / scaleSubCount)
751
+ : spec.cols;
752
+ if (!Number.isInteger(scaleSubCount) || scaleSubCount <= 0) {
753
+ throw new Error(
754
+ `[DopplerLoader] Tensor "${tensorName}" has invalid LiteRT axis uint8 companion block count ${scaleSubCount}.`
755
+ );
756
+ }
757
+ if (!Number.isInteger(scaleSubBlockWidth) || scaleSubBlockWidth <= 0) {
758
+ throw new Error(
759
+ `[DopplerLoader] Tensor "${tensorName}" has invalid LiteRT axis uint8 companion layout for width ${spec.cols} and sub-count ${scaleSubCount}.`
760
+ );
761
+ }
762
+
763
+ const resolveScale = (logicalRow, col = 0) => {
764
+ const rowScaleOffset = logicalRow * spec.scaleRowBytes;
765
+ const storedScale = scaleCompanion
766
+ ? (scaleView.getUint8(rowScaleOffset + Math.min(Math.floor(col / scaleSubBlockWidth), scaleSubCount - 1))
767
+ - scaleCompanion.zeroPoint) * scaleCompanion.scale
768
+ : scaleView.getFloat32(rowScaleOffset, true);
769
+ if (!Number.isFinite(storedScale) || storedScale <= 0) {
770
+ throw new Error(
771
+ `[DopplerLoader] Tensor "${tensorName}" LiteRT axis row ${logicalRow} has invalid scale ${storedScale}.`
772
+ );
773
+ }
774
+ return resolveLiteRTScaleValue(
775
+ storedScale,
776
+ transform,
777
+ tensorName,
778
+ `LiteRT axis row ${logicalRow}`
779
+ );
780
+ };
601
781
 
602
782
  if (spec.quantAxis === 1) {
603
783
  const expectedRawBytes = rowCount * spec.rawStorageRowBytes;
@@ -608,18 +788,6 @@ export function materializeTensorSourceTransform(rawBytes, location, tensorName,
608
788
  );
609
789
  }
610
790
  for (let row = 0; row < rowCount; row++) {
611
- const scale = scaleView.getFloat32(row * spec.scaleRowBytes, true);
612
- if (!Number.isFinite(scale) || scale <= 0) {
613
- throw new Error(
614
- `[DopplerLoader] Tensor "${tensorName}" LiteRT axis row ${rowStart + row} has invalid scale ${scale}.`
615
- );
616
- }
617
- const resolvedScale = resolveLiteRTScaleValue(
618
- scale,
619
- transform,
620
- tensorName,
621
- `LiteRT axis row ${rowStart + row}`
622
- );
623
791
  const rawRowOffset = row * spec.rawStorageRowBytes;
624
792
  const rowBytes = rawBytes.subarray(rawRowOffset, rawRowOffset + spec.rawStorageRowBytes);
625
793
  const storedRowSum = hasSumSource
@@ -641,6 +809,7 @@ export function materializeTensorSourceTransform(rawBytes, location, tensorName,
641
809
  }
642
810
  const outRowOffset = row * spec.cols;
643
811
  for (let col = 0; col < spec.cols; col++) {
812
+ const scaleForCol = resolveScale(rowStart + row, col);
644
813
  const quantized = rowZeroPoint == null
645
814
  ? readQuantizedValue(
646
815
  rowBytes,
@@ -656,7 +825,7 @@ export function materializeTensorSourceTransform(rawBytes, location, tensorName,
656
825
  transform.storageEncoding
657
826
  ) - rowZeroPoint
658
827
  );
659
- out[outRowOffset + col] = float32ToFloat16(quantized * resolvedScale);
828
+ out[outRowOffset + col] = float32ToFloat16(quantized * scaleForCol);
660
829
  }
661
830
  }
662
831
  return new Uint8Array(out.buffer, out.byteOffset, out.byteLength);
@@ -697,19 +866,6 @@ export function materializeTensorSourceTransform(rawBytes, location, tensorName,
697
866
  for (let row = 0; row < rowCount; row++) {
698
867
  const logicalRow = rowStart + row;
699
868
  const localColumnIndex = localColumnBase + row;
700
- const scale = scaleView.getFloat32(row * spec.scaleRowBytes, true);
701
- if (!Number.isFinite(scale) || scale <= 0) {
702
- throw new Error(
703
- `[DopplerLoader] Tensor "${tensorName}" LiteRT axis row ${logicalRow} has invalid scale ${scale}.`
704
- );
705
- }
706
- const resolvedScale = resolveLiteRTScaleValue(
707
- scale,
708
- transform,
709
- tensorName,
710
- `LiteRT axis row ${logicalRow}`
711
- );
712
-
713
869
  const storedValues = new Array(spec.cols);
714
870
  let storedSum = 0;
715
871
  for (let col = 0; col < spec.cols; col++) {
@@ -737,6 +893,7 @@ export function materializeTensorSourceTransform(rawBytes, location, tensorName,
737
893
 
738
894
  const outRowOffset = row * spec.cols;
739
895
  for (let col = 0; col < spec.cols; col++) {
896
+ const scaleForCol = resolveScale(logicalRow, col);
740
897
  const quantized = rowZeroPoint == null
741
898
  ? readQuantizedValue(
742
899
  rowSlices[col],
@@ -745,14 +902,135 @@ export function materializeTensorSourceTransform(rawBytes, location, tensorName,
745
902
  transform.storageEncoding
746
903
  )
747
904
  : (storedValues[col] - rowZeroPoint);
748
- out[outRowOffset + col] = float32ToFloat16(quantized * resolvedScale);
905
+ out[outRowOffset + col] = float32ToFloat16(quantized * scaleForCol);
749
906
  }
750
907
  }
751
908
  return new Uint8Array(out.buffer, out.byteOffset, out.byteLength);
752
909
  }
753
910
 
754
- const affineTransform = resolveAffineDequantTransform(location, tensorName);
911
+ if (transform.kind === 'litert_axis_blocked_dequant') {
912
+ const scaleBytes = options.scaleBytes;
913
+ if (!(scaleBytes instanceof Uint8Array)) {
914
+ throw new Error(
915
+ `[DopplerLoader] Tensor "${tensorName}" LiteRT blocked axis sourceTransform requires scaleBytes.`
916
+ );
917
+ }
918
+ const sumBytes = options.sumBytes ?? null;
919
+ const hasSumSource = transform.sumSource != null;
920
+ if (hasSumSource && !(sumBytes instanceof Uint8Array)) {
921
+ throw new Error(
922
+ `[DopplerLoader] Tensor "${tensorName}" LiteRT blocked axis sourceTransform requires sumBytes.`
923
+ );
924
+ }
925
+ const rowStart = options.rowStart == null ? 0 : Number(options.rowStart);
926
+ const rowCount = options.rowCount == null ? spec.rows : Number(options.rowCount);
927
+ if (!Number.isInteger(rowStart) || rowStart < 0) {
928
+ throw new Error(
929
+ `[DopplerLoader] Tensor "${tensorName}" has invalid LiteRT blocked axis rowStart ${options.rowStart}.`
930
+ );
931
+ }
932
+ if (!Number.isInteger(rowCount) || rowCount <= 0) {
933
+ throw new Error(
934
+ `[DopplerLoader] Tensor "${tensorName}" has invalid LiteRT blocked axis rowCount ${options.rowCount}.`
935
+ );
936
+ }
937
+ if (rowStart + rowCount > spec.rows) {
938
+ throw new Error(
939
+ `[DopplerLoader] Tensor "${tensorName}" LiteRT blocked axis row range (${rowStart}..${rowStart + rowCount}) exceeds ${spec.rows} rows.`
940
+ );
941
+ }
942
+ if (scaleBytes.byteLength !== rowCount * spec.scaleRowBytes) {
943
+ throw new Error(
944
+ `[DopplerLoader] Tensor "${tensorName}" LiteRT blocked axis scale byte size mismatch. ` +
945
+ `Expected ${rowCount * spec.scaleRowBytes}, got ${scaleBytes.byteLength}.`
946
+ );
947
+ }
948
+ if (hasSumSource && sumBytes.byteLength !== rowCount * 4) {
949
+ throw new Error(
950
+ `[DopplerLoader] Tensor "${tensorName}" LiteRT blocked axis sum byte size mismatch. ` +
951
+ `Expected ${rowCount * 4}, got ${sumBytes.byteLength}.`
952
+ );
953
+ }
954
+ const expectedRawBytes = spec.storageRows * rowCount * spec.storageElementBytes;
955
+ if (rawBytes.byteLength !== expectedRawBytes) {
956
+ throw new Error(
957
+ `[DopplerLoader] Tensor "${tensorName}" LiteRT blocked axis raw byte size mismatch. ` +
958
+ `Expected ${expectedRawBytes}, got ${rawBytes.byteLength}.`
959
+ );
960
+ }
961
+ const scaleView = new DataView(scaleBytes.buffer, scaleBytes.byteOffset, scaleBytes.byteLength);
962
+ const sumView = hasSumSource
963
+ ? new DataView(sumBytes.buffer, sumBytes.byteOffset, sumBytes.byteLength)
964
+ : null;
965
+ const out = new Uint16Array(rowCount * spec.cols);
966
+ for (let row = 0; row < rowCount; row++) {
967
+ const logicalRow = rowStart + row;
968
+ const scale = scaleView.getFloat32(row * spec.scaleRowBytes, true);
969
+ if (!Number.isFinite(scale) || scale <= 0) {
970
+ throw new Error(
971
+ `[DopplerLoader] Tensor "${tensorName}" LiteRT blocked axis row ${logicalRow} has invalid scale ${scale}.`
972
+ );
973
+ }
974
+ const resolvedScale = resolveLiteRTScaleValue(
975
+ scale,
976
+ transform,
977
+ tensorName,
978
+ `LiteRT blocked axis row ${logicalRow}`
979
+ );
980
+ const storedRowSum = hasSumSource
981
+ ? sumView.getInt32(row * 4, true)
982
+ : null;
983
+ let storedSum = 0;
984
+ if (storedRowSum != null) {
985
+ for (let storageRow = 0; storageRow < spec.storageRows; storageRow++) {
986
+ const baseOffset = storageRow * rowCount * spec.storageElementBytes + row * spec.storageElementBytes;
987
+ const blockBytes = rawBytes.subarray(baseOffset, baseOffset + spec.storageElementBytes);
988
+ for (let lane = 0; lane < spec.storageBlockSize; lane++) {
989
+ storedSum += readStoredQuantizedValue(
990
+ blockBytes,
991
+ lane,
992
+ transform.sourceDtype,
993
+ transform.storageEncoding
994
+ );
995
+ }
996
+ }
997
+ }
998
+ const rowZeroPoint = storedRowSum == null
999
+ ? null
1000
+ : (storedSum - storedRowSum) / spec.cols;
1001
+ if (rowZeroPoint != null && !Number.isFinite(rowZeroPoint)) {
1002
+ throw new Error(
1003
+ `[DopplerLoader] Tensor "${tensorName}" LiteRT blocked axis row ${logicalRow} has invalid derived rowZeroPoint ${rowZeroPoint}.`
1004
+ );
1005
+ }
1006
+ const outRowOffset = row * spec.cols;
1007
+ for (let storageRow = 0; storageRow < spec.storageRows; storageRow++) {
1008
+ const baseOffset = storageRow * rowCount * spec.storageElementBytes + row * spec.storageElementBytes;
1009
+ const blockBytes = rawBytes.subarray(baseOffset, baseOffset + spec.storageElementBytes);
1010
+ for (let lane = 0; lane < spec.storageBlockSize; lane++) {
1011
+ const stored = readStoredQuantizedValue(
1012
+ blockBytes,
1013
+ lane,
1014
+ transform.sourceDtype,
1015
+ transform.storageEncoding
1016
+ );
1017
+ const quantized = rowZeroPoint == null
1018
+ ? readQuantizedValue(
1019
+ blockBytes,
1020
+ lane,
1021
+ transform.sourceDtype,
1022
+ transform.storageEncoding
1023
+ )
1024
+ : (stored - rowZeroPoint);
1025
+ const logicalCol = storageRow * spec.storageBlockSize + spec.storageLaneOrder[lane];
1026
+ out[outRowOffset + logicalCol] = float32ToFloat16(quantized * resolvedScale);
1027
+ }
1028
+ }
1029
+ }
1030
+ return new Uint8Array(out.buffer, out.byteOffset, out.byteLength);
1031
+ }
755
1032
 
1033
+ const affineTransform = transform;
756
1034
  const elementCount = computeElementCount(location.shape, tensorName);
757
1035
  const expectedBytes = computeSourceByteLength(
758
1036
  elementCount,
@@ -161,7 +161,7 @@ function getSourceTransformScaleLocation(location, name) {
161
161
 
162
162
  function getSourceTransformSumLocation(location) {
163
163
  const transform = location?.sourceTransform;
164
- const sumLocation = transform?.kind === 'litert_axis_dequant'
164
+ const sumLocation = transform?.kind === 'litert_axis_dequant' || transform?.kind === 'litert_axis_blocked_dequant'
165
165
  ? transform?.sumSource
166
166
  : transform?.rowSumSource;
167
167
  if (!sumLocation || typeof sumLocation !== 'object') {
@@ -177,6 +177,7 @@ async function materializeLocationBytes(rawBytes, location, name, loadShard, loa
177
177
  if (
178
178
  location.sourceTransform.kind === 'litert_rowwise_dequant'
179
179
  || location.sourceTransform.kind === 'litert_axis_dequant'
180
+ || location.sourceTransform.kind === 'litert_axis_blocked_dequant'
180
181
  ) {
181
182
  const scaleLocation = getSourceTransformScaleLocation(location, name);
182
183
  const scaleBytes = options.scaleBytes instanceof Uint8Array
@@ -284,7 +285,11 @@ export async function loadTensorRange(location, name, byteOffset, byteLength, lo
284
285
  if (hasSourceTransform(location)) {
285
286
  const spec = getSourceTransformSpec(location, name);
286
287
  if (
287
- (spec?.kind === 'litert_rowwise_dequant' || spec?.kind === 'litert_axis_dequant')
288
+ (
289
+ spec?.kind === 'litert_rowwise_dequant'
290
+ || spec?.kind === 'litert_axis_dequant'
291
+ || spec?.kind === 'litert_axis_blocked_dequant'
292
+ )
288
293
  && (byteOffset % spec.targetRowBytes) === 0
289
294
  && (byteLength % spec.targetRowBytes) === 0
290
295
  ) {
@@ -343,6 +348,27 @@ export async function loadTensorRange(location, name, byteOffset, byteLength, lo
343
348
  });
344
349
  }
345
350
 
351
+ if (spec.kind === 'litert_axis_blocked_dequant') {
352
+ const raw = new Uint8Array(spec.storageRows * rowCount * spec.storageElementBytes);
353
+ for (let storageRow = 0; storageRow < spec.storageRows; storageRow++) {
354
+ const rowBytes = await loadLocationRange(
355
+ location,
356
+ name,
357
+ `LiteRT blocked axis raw storage row ${storageRow}`,
358
+ storageRow * spec.rawStorageRowBytes + rowStart * spec.storageElementBytes,
359
+ rowCount * spec.storageElementBytes,
360
+ loadShardRange
361
+ );
362
+ raw.set(rowBytes, storageRow * rowCount * spec.storageElementBytes);
363
+ }
364
+ return materializeTensorSourceTransform(raw, location, name, {
365
+ scaleBytes,
366
+ sumBytes,
367
+ rowStart,
368
+ rowCount,
369
+ });
370
+ }
371
+
346
372
  const storageColumnStart = Math.floor(rowStart / spec.storageValuesPerByte) * spec.storageValuesPerByte;
347
373
  const storageColumnEnd = Math.ceil((rowStart + rowCount) / spec.storageValuesPerByte) * spec.storageValuesPerByte;
348
374
  const rawSliceRowBytes = (storageColumnEnd - storageColumnStart) / spec.storageValuesPerByte;
@@ -23,6 +23,7 @@ export interface CreateArtifactStorageContextOptions {
23
23
  ) => AsyncIterable<ArrayBuffer | Uint8Array>;
24
24
  readText?: (path: string) => Promise<string | Record<string, unknown> | null | undefined>;
25
25
  readBinary?: (path: string) => Promise<ArrayBuffer | Uint8Array | null | undefined>;
26
+ close?: (() => Promise<void>) | null;
26
27
  tokenizerJsonPath?: string | null;
27
28
  tokenizerModelPath?: string | null;
28
29
  verifyHashes?: boolean;