doppler-gpu 0.2.2 → 0.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -0
- package/assets/doppler.svg +1601 -0
- package/package.json +6 -5
- package/src/cli/doppler-quickstart.js +16 -2
- package/src/cli/doppler-serve.js +0 -0
- package/src/client/doppler-registry.json +17 -1
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +1 -1
- package/src/config/kernels/registry.json +73 -0
- package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +32 -0
- package/src/config/schema/benchmark.schema.d.ts +2 -0
- package/src/config/schema/benchmark.schema.js +1 -0
- package/src/config/schema/doppler.schema.d.ts +9 -1
- package/src/config/schema/doppler.schema.js +8 -1
- package/src/config/schema/index.d.ts +2 -0
- package/src/config/schema/index.js +1 -0
- package/src/config/schema/manifest.schema.d.ts +5 -0
- package/src/converter/quantizer.js +5 -4
- package/src/formats/rdrr/source-transform-contract.js +390 -0
- package/src/formats/rdrr/types.d.ts +8 -1
- package/src/formats/tflite/types.d.ts +8 -0
- package/src/formats/tflite/types.js +45 -0
- package/src/gpu/command-recorder.d.ts +7 -1
- package/src/gpu/command-recorder.js +41 -2
- package/src/gpu/kernels/activation-static-qdq.d.ts +22 -0
- package/src/gpu/kernels/activation-static-qdq.js +69 -0
- package/src/gpu/kernels/activation_static_qdq.wgsl +23 -0
- package/src/gpu/kernels/index.d.ts +7 -0
- package/src/gpu/kernels/index.js +6 -0
- package/src/inference/browser-harness-text-helpers.d.ts +1 -0
- package/src/inference/browser-harness-text-helpers.js +44 -4
- package/src/inference/browser-harness.js +7 -0
- package/src/inference/pipelines/context.js +4 -2
- package/src/inference/pipelines/text/attention/types.js +9 -4
- package/src/inference/pipelines/text/embed.d.ts +1 -0
- package/src/inference/pipelines/text/embed.js +10 -4
- package/src/inference/pipelines/text/generator-helpers.d.ts +6 -0
- package/src/inference/pipelines/text/generator-helpers.js +20 -1
- package/src/inference/pipelines/text/generator-steps.js +17 -8
- package/src/inference/pipelines/text/generator.js +51 -3
- package/src/inference/pipelines/text/init.d.ts +3 -0
- package/src/inference/pipelines/text/init.js +24 -3
- package/src/inference/pipelines/text/logits/gpu.d.ts +5 -4
- package/src/inference/pipelines/text/logits/gpu.js +146 -11
- package/src/inference/pipelines/text/per-layer-inputs.js +320 -57
- package/src/inference/pipelines/text/probes.js +79 -1
- package/src/inference/pipelines/text/stage-names.js +9 -0
- package/src/inference/pipelines/text.js +10 -2
- package/src/loader/embedding-loader.js +20 -5
- package/src/loader/loader-types.d.ts +2 -0
- package/src/loader/manifest-config.d.ts +2 -7
- package/src/loader/per-layer-input-loader.js +114 -1
- package/src/loader/tensors/source-transform.d.ts +15 -0
- package/src/loader/tensors/source-transform.js +312 -34
- package/src/loader/tensors/tensor-reader.js +28 -2
- package/src/storage/artifact-storage-context.d.ts +1 -0
- package/src/storage/artifact-storage-context.js +158 -72
- package/src/storage/preflight.js +11 -0
- package/src/storage/quickstart-downloader.js +15 -4
- package/src/tooling/litert-package-runtime.js +369 -14
- package/src/tooling/node-source-runtime.js +256 -140
- package/src/tooling/source-runtime-bundle.d.ts +25 -0
- package/src/tooling/source-runtime-bundle.js +53 -1
- package/src/tooling/source-runtime-materializer.js +3 -1
|
@@ -78,6 +78,17 @@ function resolvePackageTokenizerConfig(sourceKind, runtimeProfile) {
|
|
|
78
78
|
return null;
|
|
79
79
|
}
|
|
80
80
|
|
|
81
|
+
function findTFLiteMetadataEntry(parsedTFLite, name) {
|
|
82
|
+
const target = normalizeText(name);
|
|
83
|
+
if (!target) {
|
|
84
|
+
return null;
|
|
85
|
+
}
|
|
86
|
+
const metadataEntries = Array.isArray(parsedTFLite?.metadataEntries)
|
|
87
|
+
? parsedTFLite.metadataEntries
|
|
88
|
+
: [];
|
|
89
|
+
return metadataEntries.find((entry) => normalizeText(entry?.name) === target) ?? null;
|
|
90
|
+
}
|
|
91
|
+
|
|
81
92
|
function computePackedByteSize(shape, sourceDtype, tensorName) {
|
|
82
93
|
if (!Array.isArray(shape) || shape.length !== 2) {
|
|
83
94
|
throw new Error(
|
|
@@ -107,6 +118,47 @@ function computePackedByteSize(shape, sourceDtype, tensorName) {
|
|
|
107
118
|
}
|
|
108
119
|
|
|
109
120
|
function resolveLiteRTScaleContract(sourceDtype, tensorName, options = {}) {
|
|
121
|
+
const explicitScaleSemantics = normalizeText(options.scaleSemantics).toLowerCase();
|
|
122
|
+
if (explicitScaleSemantics) {
|
|
123
|
+
if (explicitScaleSemantics === 'step') {
|
|
124
|
+
return {
|
|
125
|
+
scaleSemantics: 'step',
|
|
126
|
+
};
|
|
127
|
+
}
|
|
128
|
+
if (explicitScaleSemantics === 'qmax_abs') {
|
|
129
|
+
const explicitScaleDivisor = Number(options.scaleDivisor);
|
|
130
|
+
if (Number.isFinite(explicitScaleDivisor) && explicitScaleDivisor > 0) {
|
|
131
|
+
return {
|
|
132
|
+
scaleSemantics: 'qmax_abs',
|
|
133
|
+
scaleDivisor: explicitScaleDivisor,
|
|
134
|
+
};
|
|
135
|
+
}
|
|
136
|
+
if (sourceDtype === 'INT8' || sourceDtype === 'UINT8') {
|
|
137
|
+
return {
|
|
138
|
+
scaleSemantics: 'qmax_abs',
|
|
139
|
+
scaleDivisor: 128,
|
|
140
|
+
};
|
|
141
|
+
}
|
|
142
|
+
if (sourceDtype === 'INT4') {
|
|
143
|
+
return {
|
|
144
|
+
scaleSemantics: 'qmax_abs',
|
|
145
|
+
scaleDivisor: 8,
|
|
146
|
+
};
|
|
147
|
+
}
|
|
148
|
+
if (sourceDtype === 'INT2') {
|
|
149
|
+
return {
|
|
150
|
+
scaleSemantics: 'qmax_abs',
|
|
151
|
+
scaleDivisor: 2,
|
|
152
|
+
};
|
|
153
|
+
}
|
|
154
|
+
throw new Error(
|
|
155
|
+
`direct-source runtime: unsupported LiteRT scale contract source dtype "${sourceDtype}" for "${tensorName}".`
|
|
156
|
+
);
|
|
157
|
+
}
|
|
158
|
+
throw new Error(
|
|
159
|
+
`direct-source runtime: unsupported LiteRT scaleSemantics "${options.scaleSemantics}" for "${tensorName}".`
|
|
160
|
+
);
|
|
161
|
+
}
|
|
110
162
|
if (options.hasSumCompanion === true) {
|
|
111
163
|
return {
|
|
112
164
|
scaleSemantics: 'step',
|
|
@@ -135,6 +187,70 @@ function resolveLiteRTScaleContract(sourceDtype, tensorName, options = {}) {
|
|
|
135
187
|
);
|
|
136
188
|
}
|
|
137
189
|
|
|
190
|
+
function computeBlockedAxisPackedByteSize(storageShape, storageBlockSize, sourceDtype, tensorName) {
|
|
191
|
+
if (!Array.isArray(storageShape) || storageShape.length !== 2) {
|
|
192
|
+
throw new Error(
|
|
193
|
+
`direct-source runtime: LiteRT tensor "${tensorName}" requires an explicit blocked storageShape=[rows, cols].`
|
|
194
|
+
);
|
|
195
|
+
}
|
|
196
|
+
const storageRows = Number(storageShape[0]);
|
|
197
|
+
const storageCols = Number(storageShape[1]);
|
|
198
|
+
const blockSize = Number(storageBlockSize);
|
|
199
|
+
if (
|
|
200
|
+
!Number.isInteger(storageRows)
|
|
201
|
+
|| storageRows <= 0
|
|
202
|
+
|| !Number.isInteger(storageCols)
|
|
203
|
+
|| storageCols <= 0
|
|
204
|
+
|| !Number.isInteger(blockSize)
|
|
205
|
+
|| blockSize <= 0
|
|
206
|
+
) {
|
|
207
|
+
throw new Error(
|
|
208
|
+
`direct-source runtime: LiteRT tensor "${tensorName}" has invalid blocked storage shape ` +
|
|
209
|
+
`${JSON.stringify({ storageShape, storageBlockSize })}.`
|
|
210
|
+
);
|
|
211
|
+
}
|
|
212
|
+
return computePackedByteSize([storageRows, storageCols * blockSize], sourceDtype, tensorName);
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
function inferLiteRTBlockedAxisLayout(
|
|
216
|
+
rawTensor,
|
|
217
|
+
storageShape,
|
|
218
|
+
storageBlockSize,
|
|
219
|
+
tensorName = rawTensor?.name ?? 'unknown',
|
|
220
|
+
options = {}
|
|
221
|
+
) {
|
|
222
|
+
const dtypeId = Number(rawTensor?.dtypeId);
|
|
223
|
+
const candidates = [];
|
|
224
|
+
if (dtypeId === 17) {
|
|
225
|
+
candidates.push('INT4');
|
|
226
|
+
} else if (dtypeId === 9) {
|
|
227
|
+
candidates.push('INT8', 'INT4', 'INT2');
|
|
228
|
+
} else if (dtypeId === 3) {
|
|
229
|
+
candidates.push('UINT8', 'INT4', 'INT2');
|
|
230
|
+
} else {
|
|
231
|
+
throw new Error(
|
|
232
|
+
`direct-source runtime: unsupported LiteRT blocked tensor dtype for "${tensorName}" (dtypeId=${dtypeId}).`
|
|
233
|
+
);
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
for (const sourceDtype of candidates) {
|
|
237
|
+
if (rawTensor.size === computeBlockedAxisPackedByteSize(storageShape, storageBlockSize, sourceDtype, tensorName)) {
|
|
238
|
+
const preferSignedPacked = options.preferSignedPacked === true;
|
|
239
|
+
return {
|
|
240
|
+
sourceDtype,
|
|
241
|
+
storageEncoding: sourceDtype === 'INT8' || sourceDtype === 'UINT8'
|
|
242
|
+
? 'signed'
|
|
243
|
+
: (preferSignedPacked ? 'signed' : 'offset_binary'),
|
|
244
|
+
};
|
|
245
|
+
}
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
throw new Error(
|
|
249
|
+
`direct-source runtime: LiteRT tensor "${tensorName}" size ${rawTensor?.size} does not match any supported ` +
|
|
250
|
+
`blocked packed layout for storage shape ${JSON.stringify(storageShape)} and blockSize=${storageBlockSize}.`
|
|
251
|
+
);
|
|
252
|
+
}
|
|
253
|
+
|
|
138
254
|
export function inferLiteRTRowwiseLayout(
|
|
139
255
|
rawTensor,
|
|
140
256
|
expectedShape,
|
|
@@ -349,7 +465,8 @@ function createLiteRTAxisTensor(
|
|
|
349
465
|
group = null,
|
|
350
466
|
logicalShape = null,
|
|
351
467
|
storageShape = null,
|
|
352
|
-
quantAxis = 1
|
|
468
|
+
quantAxis = 1,
|
|
469
|
+
scaleContractOptions = null
|
|
353
470
|
) {
|
|
354
471
|
if (!rawTensor || typeof rawTensor !== 'object') {
|
|
355
472
|
throw new Error(`direct-source runtime: missing LiteRT tensor "${canonicalName}".`);
|
|
@@ -381,6 +498,34 @@ function createLiteRTAxisTensor(
|
|
|
381
498
|
);
|
|
382
499
|
}
|
|
383
500
|
|
|
501
|
+
const scaleTensorSourceTransform = scaleTensor.sourceTransform;
|
|
502
|
+
const scaleCompanionDtype = String(scaleTensor.sourceDtype || '').toUpperCase();
|
|
503
|
+
const hasUint8ScaleCompanion = scaleCompanionDtype === 'UINT8';
|
|
504
|
+
const scaleCompanionDequant = hasUint8ScaleCompanion
|
|
505
|
+
? {
|
|
506
|
+
scale: Number(scaleTensorSourceTransform?.scale),
|
|
507
|
+
zeroPoint: Number(scaleTensorSourceTransform?.zeroPoint),
|
|
508
|
+
}
|
|
509
|
+
: null;
|
|
510
|
+
if (hasUint8ScaleCompanion) {
|
|
511
|
+
if (!scaleTensorSourceTransform || scaleTensorSourceTransform.kind !== 'affine_dequant') {
|
|
512
|
+
throw new Error(
|
|
513
|
+
`direct-source runtime: LiteRT tensor "${rawTensor.name}" has UINT8 scale companion "${rawTensor.name}_quantized_scale" ` +
|
|
514
|
+
'without affine_dequant metadata.'
|
|
515
|
+
);
|
|
516
|
+
}
|
|
517
|
+
if (!Number.isFinite(scaleCompanionDequant.scale) || scaleCompanionDequant.scale <= 0) {
|
|
518
|
+
throw new Error(
|
|
519
|
+
`direct-source runtime: LiteRT tensor "${rawTensor.name}" has invalid scale companion affine_dequant scale ${scaleTensorSourceTransform.scale}.`
|
|
520
|
+
);
|
|
521
|
+
}
|
|
522
|
+
if (!Number.isSafeInteger(scaleCompanionDequant.zeroPoint)) {
|
|
523
|
+
throw new Error(
|
|
524
|
+
`direct-source runtime: LiteRT tensor "${rawTensor.name}" has invalid scale companion affine_dequant zeroPoint ${scaleTensorSourceTransform.zeroPoint}.`
|
|
525
|
+
);
|
|
526
|
+
}
|
|
527
|
+
}
|
|
528
|
+
|
|
384
529
|
const logicalRows = Number(logicalShape[0]);
|
|
385
530
|
const logicalCols = Number(logicalShape[1]);
|
|
386
531
|
const storageRows = Number(storageShape[0]);
|
|
@@ -406,6 +551,9 @@ function createLiteRTAxisTensor(
|
|
|
406
551
|
});
|
|
407
552
|
const scaleContract = resolveLiteRTScaleContract(layout.sourceDtype, canonicalName, {
|
|
408
553
|
hasSumCompanion: Boolean(sumTensor && typeof sumTensor === 'object'),
|
|
554
|
+
...(scaleContractOptions && typeof scaleContractOptions === 'object'
|
|
555
|
+
? scaleContractOptions
|
|
556
|
+
: {}),
|
|
409
557
|
});
|
|
410
558
|
const expectedScaleCount = quantAxis === 0 ? storageCols : storageRows;
|
|
411
559
|
const scaleCount = scaleTensor.size / 4;
|
|
@@ -453,6 +601,167 @@ function createLiteRTAxisTensor(
|
|
|
453
601
|
scaleSourcePath: sourcePath,
|
|
454
602
|
scaleOffset: scaleTensor.offset,
|
|
455
603
|
scaleSize: scaleTensor.size,
|
|
604
|
+
...(hasUint8ScaleCompanion
|
|
605
|
+
? {
|
|
606
|
+
scaleCompanionDtype,
|
|
607
|
+
scaleCompanionDequant,
|
|
608
|
+
}
|
|
609
|
+
: {}),
|
|
610
|
+
...(sumTensor && typeof sumTensor === 'object'
|
|
611
|
+
? {
|
|
612
|
+
sumSourcePath: sourcePath,
|
|
613
|
+
sumOffset: sumTensor.offset,
|
|
614
|
+
sumSize: sumTensor.size,
|
|
615
|
+
}
|
|
616
|
+
: {}),
|
|
617
|
+
},
|
|
618
|
+
};
|
|
619
|
+
}
|
|
620
|
+
|
|
621
|
+
function createLiteRTBlockedAxisTensor(
|
|
622
|
+
rawTensor,
|
|
623
|
+
scaleTensor,
|
|
624
|
+
sumTensor,
|
|
625
|
+
sourcePath,
|
|
626
|
+
canonicalName,
|
|
627
|
+
role,
|
|
628
|
+
group = null,
|
|
629
|
+
logicalShape = null,
|
|
630
|
+
storageShape = null,
|
|
631
|
+
quantAxis = 0,
|
|
632
|
+
storageBlockSize = 4,
|
|
633
|
+
storageLaneOrder = null,
|
|
634
|
+
scaleContractOptions = null
|
|
635
|
+
) {
|
|
636
|
+
if (!rawTensor || typeof rawTensor !== 'object') {
|
|
637
|
+
throw new Error(`direct-source runtime: missing LiteRT tensor "${canonicalName}".`);
|
|
638
|
+
}
|
|
639
|
+
if (!scaleTensor || typeof scaleTensor !== 'object') {
|
|
640
|
+
throw new Error(
|
|
641
|
+
`direct-source runtime: LiteRT tensor "${rawTensor.name}" is missing scale companion ` +
|
|
642
|
+
`"${rawTensor.name}_quantized_scale".`
|
|
643
|
+
);
|
|
644
|
+
}
|
|
645
|
+
if (!Array.isArray(logicalShape) || logicalShape.length !== 2) {
|
|
646
|
+
throw new Error(
|
|
647
|
+
`direct-source runtime: LiteRT tensor "${canonicalName}" requires an explicit 2D logical shape.`
|
|
648
|
+
);
|
|
649
|
+
}
|
|
650
|
+
if (!Array.isArray(storageShape) || storageShape.length !== 2) {
|
|
651
|
+
throw new Error(
|
|
652
|
+
`direct-source runtime: LiteRT tensor "${canonicalName}" requires an explicit 2D blocked storage shape.`
|
|
653
|
+
);
|
|
654
|
+
}
|
|
655
|
+
if (quantAxis !== 0) {
|
|
656
|
+
throw new Error(
|
|
657
|
+
`direct-source runtime: LiteRT blocked tensor "${canonicalName}" only supports quantAxis=0.`
|
|
658
|
+
);
|
|
659
|
+
}
|
|
660
|
+
if (scaleTensor.size % 4 !== 0) {
|
|
661
|
+
throw new Error(
|
|
662
|
+
`direct-source runtime: LiteRT tensor "${rawTensor.name}" has invalid blocked scale size ${scaleTensor.size}.`
|
|
663
|
+
);
|
|
664
|
+
}
|
|
665
|
+
|
|
666
|
+
const logicalRows = Number(logicalShape[0]);
|
|
667
|
+
const logicalCols = Number(logicalShape[1]);
|
|
668
|
+
const storageRows = Number(storageShape[0]);
|
|
669
|
+
const storageCols = Number(storageShape[1]);
|
|
670
|
+
const blockSize = Number(storageBlockSize);
|
|
671
|
+
if (
|
|
672
|
+
!Number.isInteger(logicalRows)
|
|
673
|
+
|| logicalRows <= 0
|
|
674
|
+
|| !Number.isInteger(logicalCols)
|
|
675
|
+
|| logicalCols <= 0
|
|
676
|
+
|| !Number.isInteger(storageRows)
|
|
677
|
+
|| storageRows <= 0
|
|
678
|
+
|| !Number.isInteger(storageCols)
|
|
679
|
+
|| storageCols <= 0
|
|
680
|
+
|| !Number.isInteger(blockSize)
|
|
681
|
+
|| blockSize <= 0
|
|
682
|
+
) {
|
|
683
|
+
throw new Error(
|
|
684
|
+
`direct-source runtime: LiteRT tensor "${canonicalName}" has invalid logical/blocked storage shapes ` +
|
|
685
|
+
`${JSON.stringify({ logicalShape, storageShape, storageBlockSize })}.`
|
|
686
|
+
);
|
|
687
|
+
}
|
|
688
|
+
if (storageCols !== logicalRows || storageRows * blockSize !== logicalCols) {
|
|
689
|
+
throw new Error(
|
|
690
|
+
`direct-source runtime: LiteRT blocked tensor "${canonicalName}" expects storageShape ` +
|
|
691
|
+
`[${logicalCols / blockSize}, ${logicalRows}] for logical shape [${logicalRows}, ${logicalCols}] and blockSize=${blockSize}. ` +
|
|
692
|
+
`Got [${storageRows}, ${storageCols}].`
|
|
693
|
+
);
|
|
694
|
+
}
|
|
695
|
+
|
|
696
|
+
const resolvedLaneOrder = Array.isArray(storageLaneOrder) && storageLaneOrder.length > 0
|
|
697
|
+
? storageLaneOrder.map((value) => Number(value))
|
|
698
|
+
: Array.from({ length: blockSize }, (_value, index) => index);
|
|
699
|
+
if (
|
|
700
|
+
resolvedLaneOrder.length !== blockSize
|
|
701
|
+
|| resolvedLaneOrder.some((value) => !Number.isInteger(value) || value < 0 || value >= blockSize)
|
|
702
|
+
|| new Set(resolvedLaneOrder).size !== blockSize
|
|
703
|
+
) {
|
|
704
|
+
throw new Error(
|
|
705
|
+
`direct-source runtime: LiteRT blocked tensor "${canonicalName}" has invalid storageLaneOrder ${JSON.stringify(storageLaneOrder)}.`
|
|
706
|
+
);
|
|
707
|
+
}
|
|
708
|
+
|
|
709
|
+
const layout = inferLiteRTBlockedAxisLayout(rawTensor, storageShape, blockSize, canonicalName, {
|
|
710
|
+
preferSignedPacked: !(sumTensor && typeof sumTensor === 'object'),
|
|
711
|
+
});
|
|
712
|
+
const scaleContract = resolveLiteRTScaleContract(layout.sourceDtype, canonicalName, {
|
|
713
|
+
hasSumCompanion: Boolean(sumTensor && typeof sumTensor === 'object'),
|
|
714
|
+
...(scaleContractOptions && typeof scaleContractOptions === 'object'
|
|
715
|
+
? scaleContractOptions
|
|
716
|
+
: {}),
|
|
717
|
+
});
|
|
718
|
+
const scaleCount = scaleTensor.size / 4;
|
|
719
|
+
if (scaleCount !== logicalRows) {
|
|
720
|
+
throw new Error(
|
|
721
|
+
`direct-source runtime: LiteRT tensor "${rawTensor.name}" blocked scale count ${scaleCount} ` +
|
|
722
|
+
`does not match logical rows ${logicalRows}.`
|
|
723
|
+
);
|
|
724
|
+
}
|
|
725
|
+
|
|
726
|
+
if (sumTensor && typeof sumTensor === 'object') {
|
|
727
|
+
if (sumTensor.size % 4 !== 0) {
|
|
728
|
+
throw new Error(
|
|
729
|
+
`direct-source runtime: LiteRT tensor "${rawTensor.name}" has invalid blocked sum size ${sumTensor.size}.`
|
|
730
|
+
);
|
|
731
|
+
}
|
|
732
|
+
const sumCount = sumTensor.size / 4;
|
|
733
|
+
if (sumCount !== logicalRows) {
|
|
734
|
+
throw new Error(
|
|
735
|
+
`direct-source runtime: LiteRT tensor "${rawTensor.name}" blocked sum count ${sumCount} ` +
|
|
736
|
+
`does not match logical rows ${logicalRows}.`
|
|
737
|
+
);
|
|
738
|
+
}
|
|
739
|
+
}
|
|
740
|
+
|
|
741
|
+
return {
|
|
742
|
+
name: canonicalName,
|
|
743
|
+
shape: [logicalRows, logicalCols],
|
|
744
|
+
dtype: 'F16',
|
|
745
|
+
offset: rawTensor.offset,
|
|
746
|
+
size: rawTensor.size,
|
|
747
|
+
sourcePath,
|
|
748
|
+
role,
|
|
749
|
+
...(group ? { group } : {}),
|
|
750
|
+
sourceTransform: {
|
|
751
|
+
kind: 'litert_axis_blocked_dequant',
|
|
752
|
+
scheme: 'per_axis_affine',
|
|
753
|
+
sourceDtype: layout.sourceDtype,
|
|
754
|
+
targetDtype: 'F16',
|
|
755
|
+
storageEncoding: layout.storageEncoding,
|
|
756
|
+
scaleSemantics: scaleContract.scaleSemantics,
|
|
757
|
+
scaleDivisor: scaleContract.scaleDivisor,
|
|
758
|
+
storageShape: [storageRows, storageCols],
|
|
759
|
+
quantAxis,
|
|
760
|
+
storageBlockSize: blockSize,
|
|
761
|
+
storageLaneOrder: resolvedLaneOrder,
|
|
762
|
+
scaleSourcePath: sourcePath,
|
|
763
|
+
scaleOffset: scaleTensor.offset,
|
|
764
|
+
scaleSize: scaleTensor.size,
|
|
456
765
|
...(sumTensor && typeof sumTensor === 'object'
|
|
457
766
|
? {
|
|
458
767
|
sumSourcePath: sourcePath,
|
|
@@ -521,21 +830,35 @@ function normalizeGemma4LiteRTTensors(parsedTFLite, sourcePath, runtimeProfile)
|
|
|
521
830
|
group,
|
|
522
831
|
resolvedLogicalShape,
|
|
523
832
|
resolvedStorageShape,
|
|
524
|
-
resolvedQuantAxis
|
|
833
|
+
resolvedQuantAxis,
|
|
834
|
+
options.scaleSemantics
|
|
835
|
+
? {
|
|
836
|
+
scaleSemantics: options.scaleSemantics,
|
|
837
|
+
}
|
|
838
|
+
: null
|
|
525
839
|
)
|
|
526
840
|
);
|
|
527
841
|
};
|
|
528
842
|
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
843
|
+
normalized.push(
|
|
844
|
+
createLiteRTBlockedAxisTensor(
|
|
845
|
+
rawByName.get('transformer.embedder.input_embedding.w') ?? null,
|
|
846
|
+
rawByName.get('transformer.embedder.input_embedding.w_quantized_scale') ?? null,
|
|
847
|
+
rawByName.get('transformer.embedder.input_embedding.w.sum_i') ?? null,
|
|
848
|
+
sourcePath,
|
|
849
|
+
'model.language_model.embed_tokens.weight',
|
|
850
|
+
'embedding',
|
|
851
|
+
'embed',
|
|
852
|
+
[vocabSize, hiddenSize],
|
|
853
|
+
[hiddenSize / 4, vocabSize],
|
|
854
|
+
0,
|
|
855
|
+
4,
|
|
856
|
+
[0, 1, 2, 3],
|
|
857
|
+
{
|
|
858
|
+
scaleSemantics: 'qmax_abs',
|
|
859
|
+
scaleDivisor: 3,
|
|
860
|
+
}
|
|
861
|
+
)
|
|
539
862
|
);
|
|
540
863
|
addAxisQuantized(
|
|
541
864
|
'transformer.embedder.per_layer_model_projection.w',
|
|
@@ -548,6 +871,16 @@ function normalizeGemma4LiteRTTensors(parsedTFLite, sourcePath, runtimeProfile)
|
|
|
548
871
|
quantAxis: 0,
|
|
549
872
|
}
|
|
550
873
|
);
|
|
874
|
+
addFloat(
|
|
875
|
+
'transformer.embedder.per_layer_model_projection.input_activation_static_scale',
|
|
876
|
+
'model.language_model.per_layer_model_projection.input_activation_static_scale',
|
|
877
|
+
'other'
|
|
878
|
+
);
|
|
879
|
+
addFloat(
|
|
880
|
+
'transformer.embedder.per_layer_model_projection.output_activation_static_scale',
|
|
881
|
+
'model.language_model.per_layer_model_projection.output_activation_static_scale',
|
|
882
|
+
'other'
|
|
883
|
+
);
|
|
551
884
|
addFloat(
|
|
552
885
|
'transformer.embedder.per_layer_projection_norm.scale',
|
|
553
886
|
'model.language_model.per_layer_projection_norm.weight',
|
|
@@ -681,8 +1014,8 @@ function normalizeGemma4LiteRTTensors(parsedTFLite, sourcePath, runtimeProfile)
|
|
|
681
1014
|
'per_layer_input',
|
|
682
1015
|
[vocabSizePerLayerInput, hiddenSizePerLayerInput],
|
|
683
1016
|
{
|
|
684
|
-
|
|
685
|
-
|
|
1017
|
+
quantAxis: 1,
|
|
1018
|
+
scaleSemantics: 'step',
|
|
686
1019
|
}
|
|
687
1020
|
);
|
|
688
1021
|
}
|
|
@@ -795,6 +1128,28 @@ async function parseLiteRTTaskPackage(source, sourcePathForModelId) {
|
|
|
795
1128
|
parsedTFLite = await parseTFLiteFromSource(source, {
|
|
796
1129
|
allowPackedQuantization: true,
|
|
797
1130
|
});
|
|
1131
|
+
const tokenizerMetadataEntry = findTFLiteMetadataEntry(parsedTFLite, 'spm_vocab_model');
|
|
1132
|
+
if (tokenizerMetadataEntry) {
|
|
1133
|
+
virtualFiles.push(
|
|
1134
|
+
createVirtualFile(
|
|
1135
|
+
'TOKENIZER_MODEL',
|
|
1136
|
+
tokenizerMetadataEntry.offset,
|
|
1137
|
+
tokenizerMetadataEntry.size,
|
|
1138
|
+
'tokenizer_model'
|
|
1139
|
+
)
|
|
1140
|
+
);
|
|
1141
|
+
}
|
|
1142
|
+
const llmParametersEntry = findTFLiteMetadataEntry(parsedTFLite, 'odml.infra.proto.LlmParameters');
|
|
1143
|
+
if (llmParametersEntry) {
|
|
1144
|
+
virtualFiles.push(
|
|
1145
|
+
createVirtualFile(
|
|
1146
|
+
'METADATA',
|
|
1147
|
+
llmParametersEntry.offset,
|
|
1148
|
+
llmParametersEntry.size,
|
|
1149
|
+
'litert_metadata'
|
|
1150
|
+
)
|
|
1151
|
+
);
|
|
1152
|
+
}
|
|
798
1153
|
} else {
|
|
799
1154
|
const parsedTask = await parseLiteRTTaskFromSource(source);
|
|
800
1155
|
const tfliteEntryName = normalizeText(taskConfig.tfliteEntry) || LITERT_TASK_DEFAULT_TFLITE_ENTRY;
|