doppler-gpu 0.2.2 → 0.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -0
- package/assets/doppler.svg +1601 -0
- package/package.json +6 -5
- package/src/cli/doppler-quickstart.js +16 -2
- package/src/cli/doppler-serve.js +0 -0
- package/src/client/doppler-registry.json +17 -1
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +1 -1
- package/src/config/kernels/registry.json +73 -0
- package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +32 -0
- package/src/config/schema/benchmark.schema.d.ts +2 -0
- package/src/config/schema/benchmark.schema.js +1 -0
- package/src/config/schema/doppler.schema.d.ts +9 -1
- package/src/config/schema/doppler.schema.js +8 -1
- package/src/config/schema/index.d.ts +2 -0
- package/src/config/schema/index.js +1 -0
- package/src/config/schema/manifest.schema.d.ts +5 -0
- package/src/converter/quantizer.js +5 -4
- package/src/formats/rdrr/source-transform-contract.js +390 -0
- package/src/formats/rdrr/types.d.ts +8 -1
- package/src/formats/tflite/types.d.ts +8 -0
- package/src/formats/tflite/types.js +45 -0
- package/src/gpu/command-recorder.d.ts +7 -1
- package/src/gpu/command-recorder.js +41 -2
- package/src/gpu/kernels/activation-static-qdq.d.ts +22 -0
- package/src/gpu/kernels/activation-static-qdq.js +69 -0
- package/src/gpu/kernels/activation_static_qdq.wgsl +23 -0
- package/src/gpu/kernels/index.d.ts +7 -0
- package/src/gpu/kernels/index.js +6 -0
- package/src/inference/browser-harness-text-helpers.d.ts +1 -0
- package/src/inference/browser-harness-text-helpers.js +44 -4
- package/src/inference/browser-harness.js +7 -0
- package/src/inference/pipelines/context.js +4 -2
- package/src/inference/pipelines/text/attention/types.js +9 -4
- package/src/inference/pipelines/text/embed.d.ts +1 -0
- package/src/inference/pipelines/text/embed.js +10 -4
- package/src/inference/pipelines/text/generator-helpers.d.ts +6 -0
- package/src/inference/pipelines/text/generator-helpers.js +20 -1
- package/src/inference/pipelines/text/generator-steps.js +17 -8
- package/src/inference/pipelines/text/generator.js +51 -3
- package/src/inference/pipelines/text/init.d.ts +3 -0
- package/src/inference/pipelines/text/init.js +24 -3
- package/src/inference/pipelines/text/logits/gpu.d.ts +5 -4
- package/src/inference/pipelines/text/logits/gpu.js +146 -11
- package/src/inference/pipelines/text/per-layer-inputs.js +320 -57
- package/src/inference/pipelines/text/probes.js +79 -1
- package/src/inference/pipelines/text/stage-names.js +9 -0
- package/src/inference/pipelines/text.js +10 -2
- package/src/loader/embedding-loader.js +20 -5
- package/src/loader/loader-types.d.ts +2 -0
- package/src/loader/manifest-config.d.ts +2 -7
- package/src/loader/per-layer-input-loader.js +114 -1
- package/src/loader/tensors/source-transform.d.ts +15 -0
- package/src/loader/tensors/source-transform.js +312 -34
- package/src/loader/tensors/tensor-reader.js +28 -2
- package/src/storage/artifact-storage-context.d.ts +1 -0
- package/src/storage/artifact-storage-context.js +158 -72
- package/src/storage/preflight.js +11 -0
- package/src/storage/quickstart-downloader.js +15 -4
- package/src/tooling/litert-package-runtime.js +369 -14
- package/src/tooling/node-source-runtime.js +256 -140
- package/src/tooling/source-runtime-bundle.d.ts +25 -0
- package/src/tooling/source-runtime-bundle.js +53 -1
- package/src/tooling/source-runtime-materializer.js +3 -1
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { float32ToFloat16 } from '../../converter/quantizer.js';
|
|
2
|
+
import { normalizeTensorSourceTransform } from '../../formats/rdrr/source-transform-contract.js';
|
|
2
3
|
|
|
3
4
|
function computeElementCount(shape, tensorName) {
|
|
4
5
|
if (!Array.isArray(shape)) {
|
|
@@ -211,6 +212,26 @@ function validateLiteRTStorageEncoding(storageEncoding, tensorName) {
|
|
|
211
212
|
}
|
|
212
213
|
}
|
|
213
214
|
|
|
215
|
+
function validateLiteRTStorageLaneOrder(storageLaneOrder, storageBlockSize, tensorName) {
|
|
216
|
+
if (!Array.isArray(storageLaneOrder) || storageLaneOrder.length !== storageBlockSize) {
|
|
217
|
+
throw new Error(
|
|
218
|
+
`[DopplerLoader] Tensor "${tensorName}" LiteRT blocked axis transform requires storageLaneOrder ` +
|
|
219
|
+
`with ${storageBlockSize} entries.`
|
|
220
|
+
);
|
|
221
|
+
}
|
|
222
|
+
const seen = new Set();
|
|
223
|
+
for (let index = 0; index < storageLaneOrder.length; index++) {
|
|
224
|
+
const value = Number(storageLaneOrder[index]);
|
|
225
|
+
if (!Number.isInteger(value) || value < 0 || value >= storageBlockSize || seen.has(value)) {
|
|
226
|
+
throw new Error(
|
|
227
|
+
`[DopplerLoader] Tensor "${tensorName}" has invalid LiteRT storageLaneOrder ` +
|
|
228
|
+
`${JSON.stringify(storageLaneOrder)}.`
|
|
229
|
+
);
|
|
230
|
+
}
|
|
231
|
+
seen.add(value);
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
|
|
214
235
|
function resolveLiteRTScaleValue(storedScale, transform, tensorName, rowLabel) {
|
|
215
236
|
const scaleSemantics = String(transform?.scaleSemantics || '').trim().toLowerCase();
|
|
216
237
|
if (scaleSemantics === 'step') {
|
|
@@ -245,13 +266,14 @@ function getLiteRTCompanionByteLength(companionSource, tensorName, label, expect
|
|
|
245
266
|
}
|
|
246
267
|
|
|
247
268
|
export function getSourceTransformSpec(location, tensorName) {
|
|
248
|
-
|
|
269
|
+
const transform = normalizeTensorSourceTransform(location, tensorName);
|
|
270
|
+
if (!transform) {
|
|
249
271
|
return null;
|
|
250
272
|
}
|
|
251
|
-
const transform = location.sourceTransform;
|
|
252
273
|
if (transform.kind === 'affine_dequant') {
|
|
253
274
|
const elementCount = computeElementCount(location.shape, tensorName);
|
|
254
275
|
return {
|
|
276
|
+
transform,
|
|
255
277
|
kind: 'affine_dequant',
|
|
256
278
|
elementCount,
|
|
257
279
|
outputByteLength: elementCount * 2,
|
|
@@ -314,6 +336,7 @@ export function getSourceTransformSpec(location, tensorName) {
|
|
|
314
336
|
}
|
|
315
337
|
}
|
|
316
338
|
return {
|
|
339
|
+
transform,
|
|
317
340
|
kind: 'litert_rowwise_dequant',
|
|
318
341
|
rows,
|
|
319
342
|
cols,
|
|
@@ -397,6 +420,33 @@ export function getSourceTransformSpec(location, tensorName) {
|
|
|
397
420
|
);
|
|
398
421
|
}
|
|
399
422
|
}
|
|
423
|
+
const scaleCompanionDtype = String(transform?.scaleCompanionDtype || '').toUpperCase();
|
|
424
|
+
const scaleCompanionDequant = scaleCompanionDtype === 'UINT8'
|
|
425
|
+
? transform.scaleCompanionDequant
|
|
426
|
+
: null;
|
|
427
|
+
if (scaleCompanionDtype === 'UINT8') {
|
|
428
|
+
if (!scaleCompanionDequant || typeof scaleCompanionDequant !== 'object') {
|
|
429
|
+
throw new Error(
|
|
430
|
+
`[DopplerLoader] Tensor "${tensorName}" LiteRT axis transform has UINT8 scale companion without scaleCompanionDequant metadata.`
|
|
431
|
+
);
|
|
432
|
+
}
|
|
433
|
+
const companionScale = Number(scaleCompanionDequant.scale);
|
|
434
|
+
const companionZeroPoint = Number(scaleCompanionDequant.zeroPoint);
|
|
435
|
+
if (!Number.isFinite(companionScale) || companionScale <= 0) {
|
|
436
|
+
throw new Error(
|
|
437
|
+
`[DopplerLoader] Tensor "${tensorName}" LiteRT axis transform has invalid scaleCompanionDequant.scale ${scaleCompanionDequant.scale}.`
|
|
438
|
+
);
|
|
439
|
+
}
|
|
440
|
+
if (!Number.isSafeInteger(companionZeroPoint)) {
|
|
441
|
+
throw new Error(
|
|
442
|
+
`[DopplerLoader] Tensor "${tensorName}" LiteRT axis transform has invalid scaleCompanionDequant.zeroPoint ${scaleCompanionDequant.zeroPoint}.`
|
|
443
|
+
);
|
|
444
|
+
}
|
|
445
|
+
} else if (scaleCompanionDtype !== '') {
|
|
446
|
+
throw new Error(
|
|
447
|
+
`[DopplerLoader] Tensor "${tensorName}" LiteRT axis transform has unsupported scaleCompanionDtype "${scaleCompanionDtype}".`
|
|
448
|
+
);
|
|
449
|
+
}
|
|
400
450
|
const rawStorageRowBytes = computeSourceByteLength(storageCols, transform.sourceDtype, tensorName);
|
|
401
451
|
const sourceByteLength = computeSourceByteLength(
|
|
402
452
|
storageRows * storageCols,
|
|
@@ -404,13 +454,14 @@ export function getSourceTransformSpec(location, tensorName) {
|
|
|
404
454
|
tensorName
|
|
405
455
|
);
|
|
406
456
|
return {
|
|
457
|
+
transform,
|
|
407
458
|
kind: 'litert_axis_dequant',
|
|
408
459
|
rows,
|
|
409
460
|
cols,
|
|
410
461
|
storageRows,
|
|
411
462
|
storageCols,
|
|
412
463
|
rawStorageRowBytes,
|
|
413
|
-
scaleRowBytes: 4,
|
|
464
|
+
scaleRowBytes: scaleCompanionDtype === 'UINT8' ? 4 : 4,
|
|
414
465
|
targetRowBytes: cols * 2,
|
|
415
466
|
outputByteLength: rows * cols * 2,
|
|
416
467
|
sourceByteLength,
|
|
@@ -419,6 +470,96 @@ export function getSourceTransformSpec(location, tensorName) {
|
|
|
419
470
|
};
|
|
420
471
|
}
|
|
421
472
|
|
|
473
|
+
if (transform.kind === 'litert_axis_blocked_dequant') {
|
|
474
|
+
if (transform.scheme !== 'per_axis_affine') {
|
|
475
|
+
throw new Error(
|
|
476
|
+
`[DopplerLoader] Tensor "${tensorName}" has unsupported sourceTransform.scheme "${transform.scheme}".`
|
|
477
|
+
);
|
|
478
|
+
}
|
|
479
|
+
validateLiteRTTransformTarget(location, tensorName, transform, 'LiteRT blocked axis sourceTransform');
|
|
480
|
+
validateLiteRTStorageEncoding(transform.storageEncoding, tensorName);
|
|
481
|
+
if (!Array.isArray(transform.storageShape) || transform.storageShape.length !== 2) {
|
|
482
|
+
throw new Error(
|
|
483
|
+
`[DopplerLoader] Tensor "${tensorName}" LiteRT blocked axis transform requires storageShape=[rows, cols].`
|
|
484
|
+
);
|
|
485
|
+
}
|
|
486
|
+
const storageRows = Number(transform.storageShape[0]);
|
|
487
|
+
const storageCols = Number(transform.storageShape[1]);
|
|
488
|
+
const storageBlockSize = Number(transform.storageBlockSize);
|
|
489
|
+
if (
|
|
490
|
+
!Number.isInteger(storageRows)
|
|
491
|
+
|| storageRows <= 0
|
|
492
|
+
|| !Number.isInteger(storageCols)
|
|
493
|
+
|| storageCols <= 0
|
|
494
|
+
|| !Number.isInteger(storageBlockSize)
|
|
495
|
+
|| storageBlockSize <= 0
|
|
496
|
+
) {
|
|
497
|
+
throw new Error(
|
|
498
|
+
`[DopplerLoader] Tensor "${tensorName}" has invalid LiteRT blocked storage shape ` +
|
|
499
|
+
`${JSON.stringify({ storageShape: transform.storageShape, storageBlockSize: transform.storageBlockSize })}.`
|
|
500
|
+
);
|
|
501
|
+
}
|
|
502
|
+
if (transform.quantAxis !== 0) {
|
|
503
|
+
throw new Error(
|
|
504
|
+
`[DopplerLoader] Tensor "${tensorName}" LiteRT blocked axis transform only supports quantAxis=0.`
|
|
505
|
+
);
|
|
506
|
+
}
|
|
507
|
+
validateLiteRTStorageLaneOrder(transform.storageLaneOrder, storageBlockSize, tensorName);
|
|
508
|
+
if (storageCols !== rows || storageRows * storageBlockSize !== cols) {
|
|
509
|
+
throw new Error(
|
|
510
|
+
`[DopplerLoader] Tensor "${tensorName}" LiteRT blocked axis transform expects storageShape ` +
|
|
511
|
+
`[${cols / storageBlockSize}, ${rows}] for blockSize=${storageBlockSize}, got [${storageRows}, ${storageCols}].`
|
|
512
|
+
);
|
|
513
|
+
}
|
|
514
|
+
const scaleSource = transform.scaleSource;
|
|
515
|
+
if (!scaleSource || typeof scaleSource !== 'object') {
|
|
516
|
+
throw new Error(
|
|
517
|
+
`[DopplerLoader] Tensor "${tensorName}" LiteRT blocked axis transform is missing scaleSource.`
|
|
518
|
+
);
|
|
519
|
+
}
|
|
520
|
+
getLiteRTCompanionByteLength(scaleSource, tensorName, 'blocked axis scale', rows * 4);
|
|
521
|
+
if (transform.sumSource != null && typeof transform.sumSource !== 'object') {
|
|
522
|
+
throw new Error(
|
|
523
|
+
`[DopplerLoader] Tensor "${tensorName}" LiteRT blocked axis transform has invalid sumSource.`
|
|
524
|
+
);
|
|
525
|
+
}
|
|
526
|
+
if (transform.sumSource != null) {
|
|
527
|
+
getLiteRTCompanionByteLength(transform.sumSource, tensorName, 'blocked axis sum', rows * 4);
|
|
528
|
+
}
|
|
529
|
+
if (transform.scaleSemantics !== 'step' && transform.scaleSemantics !== 'qmax_abs') {
|
|
530
|
+
throw new Error(
|
|
531
|
+
`[DopplerLoader] Tensor "${tensorName}" LiteRT blocked axis transform has unsupported scaleSemantics "${transform.scaleSemantics}".`
|
|
532
|
+
);
|
|
533
|
+
}
|
|
534
|
+
if (transform.scaleSemantics === 'qmax_abs') {
|
|
535
|
+
const scaleDivisor = Number(transform.scaleDivisor);
|
|
536
|
+
if (!Number.isFinite(scaleDivisor) || scaleDivisor <= 0) {
|
|
537
|
+
throw new Error(
|
|
538
|
+
`[DopplerLoader] Tensor "${tensorName}" LiteRT blocked axis transform requires scaleDivisor > 0 for scaleSemantics="qmax_abs".`
|
|
539
|
+
);
|
|
540
|
+
}
|
|
541
|
+
}
|
|
542
|
+
const storageElementBytes = computeSourceByteLength(storageBlockSize, transform.sourceDtype, tensorName);
|
|
543
|
+
const rawStorageRowBytes = storageCols * storageElementBytes;
|
|
544
|
+
return {
|
|
545
|
+
transform,
|
|
546
|
+
kind: 'litert_axis_blocked_dequant',
|
|
547
|
+
rows,
|
|
548
|
+
cols,
|
|
549
|
+
storageRows,
|
|
550
|
+
storageCols,
|
|
551
|
+
rawStorageRowBytes,
|
|
552
|
+
storageElementBytes,
|
|
553
|
+
scaleRowBytes: 4,
|
|
554
|
+
targetRowBytes: cols * 2,
|
|
555
|
+
outputByteLength: rows * cols * 2,
|
|
556
|
+
sourceByteLength: storageRows * rawStorageRowBytes,
|
|
557
|
+
quantAxis: transform.quantAxis,
|
|
558
|
+
storageBlockSize,
|
|
559
|
+
storageLaneOrder: transform.storageLaneOrder.map((value) => Number(value)),
|
|
560
|
+
};
|
|
561
|
+
}
|
|
562
|
+
|
|
422
563
|
throw new Error(
|
|
423
564
|
`[DopplerLoader] Tensor "${tensorName}" has unsupported sourceTransform.kind "${transform.kind}".`
|
|
424
565
|
);
|
|
@@ -438,9 +579,9 @@ export function materializeTensorSourceTransform(rawBytes, location, tensorName,
|
|
|
438
579
|
return rawBytes;
|
|
439
580
|
}
|
|
440
581
|
|
|
441
|
-
const
|
|
582
|
+
const spec = getSourceTransformSpec(location, tensorName);
|
|
583
|
+
const transform = spec.transform;
|
|
442
584
|
if (transform.kind === 'litert_rowwise_dequant') {
|
|
443
|
-
const spec = getSourceTransformSpec(location, tensorName);
|
|
444
585
|
const scaleBytes = options.scaleBytes;
|
|
445
586
|
if (!(scaleBytes instanceof Uint8Array)) {
|
|
446
587
|
throw new Error(
|
|
@@ -549,7 +690,6 @@ export function materializeTensorSourceTransform(rawBytes, location, tensorName,
|
|
|
549
690
|
}
|
|
550
691
|
|
|
551
692
|
if (transform.kind === 'litert_axis_dequant') {
|
|
552
|
-
const spec = getSourceTransformSpec(location, tensorName);
|
|
553
693
|
const scaleBytes = options.scaleBytes;
|
|
554
694
|
if (!(scaleBytes instanceof Uint8Array)) {
|
|
555
695
|
throw new Error(
|
|
@@ -598,6 +738,46 @@ export function materializeTensorSourceTransform(rawBytes, location, tensorName,
|
|
|
598
738
|
? new DataView(sumBytes.buffer, sumBytes.byteOffset, sumBytes.byteLength)
|
|
599
739
|
: null;
|
|
600
740
|
const out = new Uint16Array(rowCount * spec.cols);
|
|
741
|
+
const scaleCompanionDtype = String(transform.scaleCompanionDtype || '').toUpperCase();
|
|
742
|
+
const scaleCompanion = scaleCompanionDtype === 'UINT8'
|
|
743
|
+
? {
|
|
744
|
+
scale: Number(transform.scaleCompanionDequant?.scale),
|
|
745
|
+
zeroPoint: Number(transform.scaleCompanionDequant?.zeroPoint),
|
|
746
|
+
}
|
|
747
|
+
: null;
|
|
748
|
+
const scaleSubCount = scaleCompanion ? spec.scaleRowBytes : 1;
|
|
749
|
+
const scaleSubBlockWidth = scaleCompanion
|
|
750
|
+
? Math.floor(spec.cols / scaleSubCount)
|
|
751
|
+
: spec.cols;
|
|
752
|
+
if (!Number.isInteger(scaleSubCount) || scaleSubCount <= 0) {
|
|
753
|
+
throw new Error(
|
|
754
|
+
`[DopplerLoader] Tensor "${tensorName}" has invalid LiteRT axis uint8 companion block count ${scaleSubCount}.`
|
|
755
|
+
);
|
|
756
|
+
}
|
|
757
|
+
if (!Number.isInteger(scaleSubBlockWidth) || scaleSubBlockWidth <= 0) {
|
|
758
|
+
throw new Error(
|
|
759
|
+
`[DopplerLoader] Tensor "${tensorName}" has invalid LiteRT axis uint8 companion layout for width ${spec.cols} and sub-count ${scaleSubCount}.`
|
|
760
|
+
);
|
|
761
|
+
}
|
|
762
|
+
|
|
763
|
+
const resolveScale = (logicalRow, col = 0) => {
|
|
764
|
+
const rowScaleOffset = logicalRow * spec.scaleRowBytes;
|
|
765
|
+
const storedScale = scaleCompanion
|
|
766
|
+
? (scaleView.getUint8(rowScaleOffset + Math.min(Math.floor(col / scaleSubBlockWidth), scaleSubCount - 1))
|
|
767
|
+
- scaleCompanion.zeroPoint) * scaleCompanion.scale
|
|
768
|
+
: scaleView.getFloat32(rowScaleOffset, true);
|
|
769
|
+
if (!Number.isFinite(storedScale) || storedScale <= 0) {
|
|
770
|
+
throw new Error(
|
|
771
|
+
`[DopplerLoader] Tensor "${tensorName}" LiteRT axis row ${logicalRow} has invalid scale ${storedScale}.`
|
|
772
|
+
);
|
|
773
|
+
}
|
|
774
|
+
return resolveLiteRTScaleValue(
|
|
775
|
+
storedScale,
|
|
776
|
+
transform,
|
|
777
|
+
tensorName,
|
|
778
|
+
`LiteRT axis row ${logicalRow}`
|
|
779
|
+
);
|
|
780
|
+
};
|
|
601
781
|
|
|
602
782
|
if (spec.quantAxis === 1) {
|
|
603
783
|
const expectedRawBytes = rowCount * spec.rawStorageRowBytes;
|
|
@@ -608,18 +788,6 @@ export function materializeTensorSourceTransform(rawBytes, location, tensorName,
|
|
|
608
788
|
);
|
|
609
789
|
}
|
|
610
790
|
for (let row = 0; row < rowCount; row++) {
|
|
611
|
-
const scale = scaleView.getFloat32(row * spec.scaleRowBytes, true);
|
|
612
|
-
if (!Number.isFinite(scale) || scale <= 0) {
|
|
613
|
-
throw new Error(
|
|
614
|
-
`[DopplerLoader] Tensor "${tensorName}" LiteRT axis row ${rowStart + row} has invalid scale ${scale}.`
|
|
615
|
-
);
|
|
616
|
-
}
|
|
617
|
-
const resolvedScale = resolveLiteRTScaleValue(
|
|
618
|
-
scale,
|
|
619
|
-
transform,
|
|
620
|
-
tensorName,
|
|
621
|
-
`LiteRT axis row ${rowStart + row}`
|
|
622
|
-
);
|
|
623
791
|
const rawRowOffset = row * spec.rawStorageRowBytes;
|
|
624
792
|
const rowBytes = rawBytes.subarray(rawRowOffset, rawRowOffset + spec.rawStorageRowBytes);
|
|
625
793
|
const storedRowSum = hasSumSource
|
|
@@ -641,6 +809,7 @@ export function materializeTensorSourceTransform(rawBytes, location, tensorName,
|
|
|
641
809
|
}
|
|
642
810
|
const outRowOffset = row * spec.cols;
|
|
643
811
|
for (let col = 0; col < spec.cols; col++) {
|
|
812
|
+
const scaleForCol = resolveScale(rowStart + row, col);
|
|
644
813
|
const quantized = rowZeroPoint == null
|
|
645
814
|
? readQuantizedValue(
|
|
646
815
|
rowBytes,
|
|
@@ -656,7 +825,7 @@ export function materializeTensorSourceTransform(rawBytes, location, tensorName,
|
|
|
656
825
|
transform.storageEncoding
|
|
657
826
|
) - rowZeroPoint
|
|
658
827
|
);
|
|
659
|
-
out[outRowOffset + col] = float32ToFloat16(quantized *
|
|
828
|
+
out[outRowOffset + col] = float32ToFloat16(quantized * scaleForCol);
|
|
660
829
|
}
|
|
661
830
|
}
|
|
662
831
|
return new Uint8Array(out.buffer, out.byteOffset, out.byteLength);
|
|
@@ -697,19 +866,6 @@ export function materializeTensorSourceTransform(rawBytes, location, tensorName,
|
|
|
697
866
|
for (let row = 0; row < rowCount; row++) {
|
|
698
867
|
const logicalRow = rowStart + row;
|
|
699
868
|
const localColumnIndex = localColumnBase + row;
|
|
700
|
-
const scale = scaleView.getFloat32(row * spec.scaleRowBytes, true);
|
|
701
|
-
if (!Number.isFinite(scale) || scale <= 0) {
|
|
702
|
-
throw new Error(
|
|
703
|
-
`[DopplerLoader] Tensor "${tensorName}" LiteRT axis row ${logicalRow} has invalid scale ${scale}.`
|
|
704
|
-
);
|
|
705
|
-
}
|
|
706
|
-
const resolvedScale = resolveLiteRTScaleValue(
|
|
707
|
-
scale,
|
|
708
|
-
transform,
|
|
709
|
-
tensorName,
|
|
710
|
-
`LiteRT axis row ${logicalRow}`
|
|
711
|
-
);
|
|
712
|
-
|
|
713
869
|
const storedValues = new Array(spec.cols);
|
|
714
870
|
let storedSum = 0;
|
|
715
871
|
for (let col = 0; col < spec.cols; col++) {
|
|
@@ -737,6 +893,7 @@ export function materializeTensorSourceTransform(rawBytes, location, tensorName,
|
|
|
737
893
|
|
|
738
894
|
const outRowOffset = row * spec.cols;
|
|
739
895
|
for (let col = 0; col < spec.cols; col++) {
|
|
896
|
+
const scaleForCol = resolveScale(logicalRow, col);
|
|
740
897
|
const quantized = rowZeroPoint == null
|
|
741
898
|
? readQuantizedValue(
|
|
742
899
|
rowSlices[col],
|
|
@@ -745,14 +902,135 @@ export function materializeTensorSourceTransform(rawBytes, location, tensorName,
|
|
|
745
902
|
transform.storageEncoding
|
|
746
903
|
)
|
|
747
904
|
: (storedValues[col] - rowZeroPoint);
|
|
748
|
-
out[outRowOffset + col] = float32ToFloat16(quantized *
|
|
905
|
+
out[outRowOffset + col] = float32ToFloat16(quantized * scaleForCol);
|
|
749
906
|
}
|
|
750
907
|
}
|
|
751
908
|
return new Uint8Array(out.buffer, out.byteOffset, out.byteLength);
|
|
752
909
|
}
|
|
753
910
|
|
|
754
|
-
|
|
911
|
+
if (transform.kind === 'litert_axis_blocked_dequant') {
|
|
912
|
+
const scaleBytes = options.scaleBytes;
|
|
913
|
+
if (!(scaleBytes instanceof Uint8Array)) {
|
|
914
|
+
throw new Error(
|
|
915
|
+
`[DopplerLoader] Tensor "${tensorName}" LiteRT blocked axis sourceTransform requires scaleBytes.`
|
|
916
|
+
);
|
|
917
|
+
}
|
|
918
|
+
const sumBytes = options.sumBytes ?? null;
|
|
919
|
+
const hasSumSource = transform.sumSource != null;
|
|
920
|
+
if (hasSumSource && !(sumBytes instanceof Uint8Array)) {
|
|
921
|
+
throw new Error(
|
|
922
|
+
`[DopplerLoader] Tensor "${tensorName}" LiteRT blocked axis sourceTransform requires sumBytes.`
|
|
923
|
+
);
|
|
924
|
+
}
|
|
925
|
+
const rowStart = options.rowStart == null ? 0 : Number(options.rowStart);
|
|
926
|
+
const rowCount = options.rowCount == null ? spec.rows : Number(options.rowCount);
|
|
927
|
+
if (!Number.isInteger(rowStart) || rowStart < 0) {
|
|
928
|
+
throw new Error(
|
|
929
|
+
`[DopplerLoader] Tensor "${tensorName}" has invalid LiteRT blocked axis rowStart ${options.rowStart}.`
|
|
930
|
+
);
|
|
931
|
+
}
|
|
932
|
+
if (!Number.isInteger(rowCount) || rowCount <= 0) {
|
|
933
|
+
throw new Error(
|
|
934
|
+
`[DopplerLoader] Tensor "${tensorName}" has invalid LiteRT blocked axis rowCount ${options.rowCount}.`
|
|
935
|
+
);
|
|
936
|
+
}
|
|
937
|
+
if (rowStart + rowCount > spec.rows) {
|
|
938
|
+
throw new Error(
|
|
939
|
+
`[DopplerLoader] Tensor "${tensorName}" LiteRT blocked axis row range (${rowStart}..${rowStart + rowCount}) exceeds ${spec.rows} rows.`
|
|
940
|
+
);
|
|
941
|
+
}
|
|
942
|
+
if (scaleBytes.byteLength !== rowCount * spec.scaleRowBytes) {
|
|
943
|
+
throw new Error(
|
|
944
|
+
`[DopplerLoader] Tensor "${tensorName}" LiteRT blocked axis scale byte size mismatch. ` +
|
|
945
|
+
`Expected ${rowCount * spec.scaleRowBytes}, got ${scaleBytes.byteLength}.`
|
|
946
|
+
);
|
|
947
|
+
}
|
|
948
|
+
if (hasSumSource && sumBytes.byteLength !== rowCount * 4) {
|
|
949
|
+
throw new Error(
|
|
950
|
+
`[DopplerLoader] Tensor "${tensorName}" LiteRT blocked axis sum byte size mismatch. ` +
|
|
951
|
+
`Expected ${rowCount * 4}, got ${sumBytes.byteLength}.`
|
|
952
|
+
);
|
|
953
|
+
}
|
|
954
|
+
const expectedRawBytes = spec.storageRows * rowCount * spec.storageElementBytes;
|
|
955
|
+
if (rawBytes.byteLength !== expectedRawBytes) {
|
|
956
|
+
throw new Error(
|
|
957
|
+
`[DopplerLoader] Tensor "${tensorName}" LiteRT blocked axis raw byte size mismatch. ` +
|
|
958
|
+
`Expected ${expectedRawBytes}, got ${rawBytes.byteLength}.`
|
|
959
|
+
);
|
|
960
|
+
}
|
|
961
|
+
const scaleView = new DataView(scaleBytes.buffer, scaleBytes.byteOffset, scaleBytes.byteLength);
|
|
962
|
+
const sumView = hasSumSource
|
|
963
|
+
? new DataView(sumBytes.buffer, sumBytes.byteOffset, sumBytes.byteLength)
|
|
964
|
+
: null;
|
|
965
|
+
const out = new Uint16Array(rowCount * spec.cols);
|
|
966
|
+
for (let row = 0; row < rowCount; row++) {
|
|
967
|
+
const logicalRow = rowStart + row;
|
|
968
|
+
const scale = scaleView.getFloat32(row * spec.scaleRowBytes, true);
|
|
969
|
+
if (!Number.isFinite(scale) || scale <= 0) {
|
|
970
|
+
throw new Error(
|
|
971
|
+
`[DopplerLoader] Tensor "${tensorName}" LiteRT blocked axis row ${logicalRow} has invalid scale ${scale}.`
|
|
972
|
+
);
|
|
973
|
+
}
|
|
974
|
+
const resolvedScale = resolveLiteRTScaleValue(
|
|
975
|
+
scale,
|
|
976
|
+
transform,
|
|
977
|
+
tensorName,
|
|
978
|
+
`LiteRT blocked axis row ${logicalRow}`
|
|
979
|
+
);
|
|
980
|
+
const storedRowSum = hasSumSource
|
|
981
|
+
? sumView.getInt32(row * 4, true)
|
|
982
|
+
: null;
|
|
983
|
+
let storedSum = 0;
|
|
984
|
+
if (storedRowSum != null) {
|
|
985
|
+
for (let storageRow = 0; storageRow < spec.storageRows; storageRow++) {
|
|
986
|
+
const baseOffset = storageRow * rowCount * spec.storageElementBytes + row * spec.storageElementBytes;
|
|
987
|
+
const blockBytes = rawBytes.subarray(baseOffset, baseOffset + spec.storageElementBytes);
|
|
988
|
+
for (let lane = 0; lane < spec.storageBlockSize; lane++) {
|
|
989
|
+
storedSum += readStoredQuantizedValue(
|
|
990
|
+
blockBytes,
|
|
991
|
+
lane,
|
|
992
|
+
transform.sourceDtype,
|
|
993
|
+
transform.storageEncoding
|
|
994
|
+
);
|
|
995
|
+
}
|
|
996
|
+
}
|
|
997
|
+
}
|
|
998
|
+
const rowZeroPoint = storedRowSum == null
|
|
999
|
+
? null
|
|
1000
|
+
: (storedSum - storedRowSum) / spec.cols;
|
|
1001
|
+
if (rowZeroPoint != null && !Number.isFinite(rowZeroPoint)) {
|
|
1002
|
+
throw new Error(
|
|
1003
|
+
`[DopplerLoader] Tensor "${tensorName}" LiteRT blocked axis row ${logicalRow} has invalid derived rowZeroPoint ${rowZeroPoint}.`
|
|
1004
|
+
);
|
|
1005
|
+
}
|
|
1006
|
+
const outRowOffset = row * spec.cols;
|
|
1007
|
+
for (let storageRow = 0; storageRow < spec.storageRows; storageRow++) {
|
|
1008
|
+
const baseOffset = storageRow * rowCount * spec.storageElementBytes + row * spec.storageElementBytes;
|
|
1009
|
+
const blockBytes = rawBytes.subarray(baseOffset, baseOffset + spec.storageElementBytes);
|
|
1010
|
+
for (let lane = 0; lane < spec.storageBlockSize; lane++) {
|
|
1011
|
+
const stored = readStoredQuantizedValue(
|
|
1012
|
+
blockBytes,
|
|
1013
|
+
lane,
|
|
1014
|
+
transform.sourceDtype,
|
|
1015
|
+
transform.storageEncoding
|
|
1016
|
+
);
|
|
1017
|
+
const quantized = rowZeroPoint == null
|
|
1018
|
+
? readQuantizedValue(
|
|
1019
|
+
blockBytes,
|
|
1020
|
+
lane,
|
|
1021
|
+
transform.sourceDtype,
|
|
1022
|
+
transform.storageEncoding
|
|
1023
|
+
)
|
|
1024
|
+
: (stored - rowZeroPoint);
|
|
1025
|
+
const logicalCol = storageRow * spec.storageBlockSize + spec.storageLaneOrder[lane];
|
|
1026
|
+
out[outRowOffset + logicalCol] = float32ToFloat16(quantized * resolvedScale);
|
|
1027
|
+
}
|
|
1028
|
+
}
|
|
1029
|
+
}
|
|
1030
|
+
return new Uint8Array(out.buffer, out.byteOffset, out.byteLength);
|
|
1031
|
+
}
|
|
755
1032
|
|
|
1033
|
+
const affineTransform = transform;
|
|
756
1034
|
const elementCount = computeElementCount(location.shape, tensorName);
|
|
757
1035
|
const expectedBytes = computeSourceByteLength(
|
|
758
1036
|
elementCount,
|
|
@@ -161,7 +161,7 @@ function getSourceTransformScaleLocation(location, name) {
|
|
|
161
161
|
|
|
162
162
|
function getSourceTransformSumLocation(location) {
|
|
163
163
|
const transform = location?.sourceTransform;
|
|
164
|
-
const sumLocation = transform?.kind === 'litert_axis_dequant'
|
|
164
|
+
const sumLocation = transform?.kind === 'litert_axis_dequant' || transform?.kind === 'litert_axis_blocked_dequant'
|
|
165
165
|
? transform?.sumSource
|
|
166
166
|
: transform?.rowSumSource;
|
|
167
167
|
if (!sumLocation || typeof sumLocation !== 'object') {
|
|
@@ -177,6 +177,7 @@ async function materializeLocationBytes(rawBytes, location, name, loadShard, loa
|
|
|
177
177
|
if (
|
|
178
178
|
location.sourceTransform.kind === 'litert_rowwise_dequant'
|
|
179
179
|
|| location.sourceTransform.kind === 'litert_axis_dequant'
|
|
180
|
+
|| location.sourceTransform.kind === 'litert_axis_blocked_dequant'
|
|
180
181
|
) {
|
|
181
182
|
const scaleLocation = getSourceTransformScaleLocation(location, name);
|
|
182
183
|
const scaleBytes = options.scaleBytes instanceof Uint8Array
|
|
@@ -284,7 +285,11 @@ export async function loadTensorRange(location, name, byteOffset, byteLength, lo
|
|
|
284
285
|
if (hasSourceTransform(location)) {
|
|
285
286
|
const spec = getSourceTransformSpec(location, name);
|
|
286
287
|
if (
|
|
287
|
-
(
|
|
288
|
+
(
|
|
289
|
+
spec?.kind === 'litert_rowwise_dequant'
|
|
290
|
+
|| spec?.kind === 'litert_axis_dequant'
|
|
291
|
+
|| spec?.kind === 'litert_axis_blocked_dequant'
|
|
292
|
+
)
|
|
288
293
|
&& (byteOffset % spec.targetRowBytes) === 0
|
|
289
294
|
&& (byteLength % spec.targetRowBytes) === 0
|
|
290
295
|
) {
|
|
@@ -343,6 +348,27 @@ export async function loadTensorRange(location, name, byteOffset, byteLength, lo
|
|
|
343
348
|
});
|
|
344
349
|
}
|
|
345
350
|
|
|
351
|
+
if (spec.kind === 'litert_axis_blocked_dequant') {
|
|
352
|
+
const raw = new Uint8Array(spec.storageRows * rowCount * spec.storageElementBytes);
|
|
353
|
+
for (let storageRow = 0; storageRow < spec.storageRows; storageRow++) {
|
|
354
|
+
const rowBytes = await loadLocationRange(
|
|
355
|
+
location,
|
|
356
|
+
name,
|
|
357
|
+
`LiteRT blocked axis raw storage row ${storageRow}`,
|
|
358
|
+
storageRow * spec.rawStorageRowBytes + rowStart * spec.storageElementBytes,
|
|
359
|
+
rowCount * spec.storageElementBytes,
|
|
360
|
+
loadShardRange
|
|
361
|
+
);
|
|
362
|
+
raw.set(rowBytes, storageRow * rowCount * spec.storageElementBytes);
|
|
363
|
+
}
|
|
364
|
+
return materializeTensorSourceTransform(raw, location, name, {
|
|
365
|
+
scaleBytes,
|
|
366
|
+
sumBytes,
|
|
367
|
+
rowStart,
|
|
368
|
+
rowCount,
|
|
369
|
+
});
|
|
370
|
+
}
|
|
371
|
+
|
|
346
372
|
const storageColumnStart = Math.floor(rowStart / spec.storageValuesPerByte) * spec.storageValuesPerByte;
|
|
347
373
|
const storageColumnEnd = Math.ceil((rowStart + rowCount) / spec.storageValuesPerByte) * spec.storageValuesPerByte;
|
|
348
374
|
const rawSliceRowBytes = (storageColumnEnd - storageColumnStart) / spec.storageValuesPerByte;
|
|
@@ -23,6 +23,7 @@ export interface CreateArtifactStorageContextOptions {
|
|
|
23
23
|
) => AsyncIterable<ArrayBuffer | Uint8Array>;
|
|
24
24
|
readText?: (path: string) => Promise<string | Record<string, unknown> | null | undefined>;
|
|
25
25
|
readBinary?: (path: string) => Promise<ArrayBuffer | Uint8Array | null | undefined>;
|
|
26
|
+
close?: (() => Promise<void>) | null;
|
|
26
27
|
tokenizerJsonPath?: string | null;
|
|
27
28
|
tokenizerModelPath?: string | null;
|
|
28
29
|
verifyHashes?: boolean;
|