doppler-gpu 0.2.2 → 0.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -0
- package/assets/doppler.svg +1601 -0
- package/package.json +6 -5
- package/src/cli/doppler-quickstart.js +16 -2
- package/src/cli/doppler-serve.js +0 -0
- package/src/client/doppler-registry.json +17 -1
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +1 -1
- package/src/config/kernels/registry.json +73 -0
- package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +32 -0
- package/src/config/schema/benchmark.schema.d.ts +2 -0
- package/src/config/schema/benchmark.schema.js +1 -0
- package/src/config/schema/doppler.schema.d.ts +9 -1
- package/src/config/schema/doppler.schema.js +8 -1
- package/src/config/schema/index.d.ts +2 -0
- package/src/config/schema/index.js +1 -0
- package/src/config/schema/manifest.schema.d.ts +5 -0
- package/src/converter/quantizer.js +5 -4
- package/src/formats/rdrr/source-transform-contract.js +390 -0
- package/src/formats/rdrr/types.d.ts +8 -1
- package/src/formats/tflite/types.d.ts +8 -0
- package/src/formats/tflite/types.js +45 -0
- package/src/gpu/command-recorder.d.ts +7 -1
- package/src/gpu/command-recorder.js +41 -2
- package/src/gpu/kernels/activation-static-qdq.d.ts +22 -0
- package/src/gpu/kernels/activation-static-qdq.js +69 -0
- package/src/gpu/kernels/activation_static_qdq.wgsl +23 -0
- package/src/gpu/kernels/index.d.ts +7 -0
- package/src/gpu/kernels/index.js +6 -0
- package/src/inference/browser-harness-text-helpers.d.ts +1 -0
- package/src/inference/browser-harness-text-helpers.js +44 -4
- package/src/inference/browser-harness.js +7 -0
- package/src/inference/pipelines/context.js +4 -2
- package/src/inference/pipelines/text/attention/types.js +9 -4
- package/src/inference/pipelines/text/embed.d.ts +1 -0
- package/src/inference/pipelines/text/embed.js +10 -4
- package/src/inference/pipelines/text/generator-helpers.d.ts +6 -0
- package/src/inference/pipelines/text/generator-helpers.js +20 -1
- package/src/inference/pipelines/text/generator-steps.js +17 -8
- package/src/inference/pipelines/text/generator.js +51 -3
- package/src/inference/pipelines/text/init.d.ts +3 -0
- package/src/inference/pipelines/text/init.js +24 -3
- package/src/inference/pipelines/text/logits/gpu.d.ts +5 -4
- package/src/inference/pipelines/text/logits/gpu.js +146 -11
- package/src/inference/pipelines/text/per-layer-inputs.js +320 -57
- package/src/inference/pipelines/text/probes.js +79 -1
- package/src/inference/pipelines/text/stage-names.js +9 -0
- package/src/inference/pipelines/text.js +10 -2
- package/src/loader/embedding-loader.js +20 -5
- package/src/loader/loader-types.d.ts +2 -0
- package/src/loader/manifest-config.d.ts +2 -7
- package/src/loader/per-layer-input-loader.js +114 -1
- package/src/loader/tensors/source-transform.d.ts +15 -0
- package/src/loader/tensors/source-transform.js +312 -34
- package/src/loader/tensors/tensor-reader.js +28 -2
- package/src/storage/artifact-storage-context.d.ts +1 -0
- package/src/storage/artifact-storage-context.js +158 -72
- package/src/storage/preflight.js +11 -0
- package/src/storage/quickstart-downloader.js +15 -4
- package/src/tooling/litert-package-runtime.js +369 -14
- package/src/tooling/node-source-runtime.js +256 -140
- package/src/tooling/source-runtime-bundle.d.ts +25 -0
- package/src/tooling/source-runtime-bundle.js +53 -1
- package/src/tooling/source-runtime-materializer.js +3 -1
|
@@ -0,0 +1,390 @@
|
|
|
1
|
+
const SUPPORTED_SOURCE_DTYPES = new Set(['INT8', 'UINT8', 'INT4', 'INT2']);
|
|
2
|
+
const SUPPORTED_LITERT_STORAGE_ENCODINGS = new Set(['signed', 'offset_binary']);
|
|
3
|
+
const SUPPORTED_LITERT_SCALE_SEMANTICS = new Set(['step', 'qmax_abs']);
|
|
4
|
+
|
|
5
|
+
function fail(errorPrefix, message) {
|
|
6
|
+
throw new Error(`${errorPrefix} ${message}`);
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
function normalizeSourceDtype(sourceDtype, tensorName, errorPrefix) {
|
|
10
|
+
const normalized = String(sourceDtype || '').trim().toUpperCase();
|
|
11
|
+
if (!SUPPORTED_SOURCE_DTYPES.has(normalized)) {
|
|
12
|
+
fail(
|
|
13
|
+
errorPrefix,
|
|
14
|
+
`Tensor "${tensorName}" has unsupported sourceTransform.sourceDtype "${sourceDtype}".`
|
|
15
|
+
);
|
|
16
|
+
}
|
|
17
|
+
return normalized;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
function normalizeTargetDtype(location, targetDtype, tensorName, errorPrefix, label = 'sourceTransform') {
|
|
21
|
+
const normalizedTargetDtype = String(targetDtype || '').trim().toUpperCase();
|
|
22
|
+
if (normalizedTargetDtype !== 'F16') {
|
|
23
|
+
fail(
|
|
24
|
+
errorPrefix,
|
|
25
|
+
`Tensor "${tensorName}" has unsupported ${label}.targetDtype "${targetDtype}".`
|
|
26
|
+
);
|
|
27
|
+
}
|
|
28
|
+
const normalizedLocationDtype = String(location?.dtype || '').trim().toUpperCase();
|
|
29
|
+
if (normalizedLocationDtype !== normalizedTargetDtype) {
|
|
30
|
+
fail(
|
|
31
|
+
errorPrefix,
|
|
32
|
+
`Tensor "${tensorName}" ${label} targetDtype "${normalizedTargetDtype}" ` +
|
|
33
|
+
`does not match location.dtype "${location?.dtype}".`
|
|
34
|
+
);
|
|
35
|
+
}
|
|
36
|
+
return normalizedTargetDtype;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function normalizePositiveNumber(value, tensorName, errorPrefix, field) {
|
|
40
|
+
const normalized = Number(value);
|
|
41
|
+
if (!Number.isFinite(normalized) || normalized <= 0) {
|
|
42
|
+
fail(
|
|
43
|
+
errorPrefix,
|
|
44
|
+
`Tensor "${tensorName}" has invalid ${field} ${value}.`
|
|
45
|
+
);
|
|
46
|
+
}
|
|
47
|
+
return normalized;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function normalizePositiveInteger(value, tensorName, errorPrefix, field) {
|
|
51
|
+
const normalized = Number(value);
|
|
52
|
+
if (!Number.isInteger(normalized) || normalized <= 0) {
|
|
53
|
+
fail(
|
|
54
|
+
errorPrefix,
|
|
55
|
+
`Tensor "${tensorName}" has invalid ${field} ${value}.`
|
|
56
|
+
);
|
|
57
|
+
}
|
|
58
|
+
return normalized;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
function normalizeSafeInteger(value, tensorName, errorPrefix, field) {
|
|
62
|
+
const normalized = Number(value);
|
|
63
|
+
if (!Number.isSafeInteger(normalized)) {
|
|
64
|
+
fail(
|
|
65
|
+
errorPrefix,
|
|
66
|
+
`Tensor "${tensorName}" has invalid ${field} ${value}.`
|
|
67
|
+
);
|
|
68
|
+
}
|
|
69
|
+
return normalized;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
function normalizeNonNegativeInteger(value, tensorName, errorPrefix, field) {
|
|
73
|
+
const normalized = Number(value);
|
|
74
|
+
if (!Number.isInteger(normalized) || normalized < 0) {
|
|
75
|
+
fail(
|
|
76
|
+
errorPrefix,
|
|
77
|
+
`Tensor "${tensorName}" has invalid ${field} ${value}.`
|
|
78
|
+
);
|
|
79
|
+
}
|
|
80
|
+
return normalized;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
function normalizeStorageEncoding(storageEncoding, tensorName, errorPrefix, label) {
|
|
84
|
+
const normalized = String(storageEncoding || '').trim();
|
|
85
|
+
if (!SUPPORTED_LITERT_STORAGE_ENCODINGS.has(normalized)) {
|
|
86
|
+
fail(
|
|
87
|
+
errorPrefix,
|
|
88
|
+
`Tensor "${tensorName}" has unsupported ${label} "${storageEncoding}".`
|
|
89
|
+
);
|
|
90
|
+
}
|
|
91
|
+
return normalized;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
function normalizeScaleSemantics(transform, tensorName, errorPrefix, label) {
|
|
95
|
+
const scaleSemantics = String(transform?.scaleSemantics || '').trim().toLowerCase();
|
|
96
|
+
if (!SUPPORTED_LITERT_SCALE_SEMANTICS.has(scaleSemantics)) {
|
|
97
|
+
fail(
|
|
98
|
+
errorPrefix,
|
|
99
|
+
`Tensor "${tensorName}" ${label} has unsupported scaleSemantics "${transform?.scaleSemantics}".`
|
|
100
|
+
);
|
|
101
|
+
}
|
|
102
|
+
const scaleDivisor = scaleSemantics === 'qmax_abs'
|
|
103
|
+
? normalizePositiveNumber(transform?.scaleDivisor, tensorName, errorPrefix, `${label} scaleDivisor`)
|
|
104
|
+
: (transform?.scaleDivisor == null ? undefined : Number(transform.scaleDivisor));
|
|
105
|
+
return {
|
|
106
|
+
scaleSemantics,
|
|
107
|
+
...(scaleDivisor === undefined ? {} : { scaleDivisor }),
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
function normalizeSourceRef(ref, tensorName, errorPrefix, label, required = false) {
|
|
112
|
+
if (ref == null) {
|
|
113
|
+
if (required) {
|
|
114
|
+
fail(
|
|
115
|
+
errorPrefix,
|
|
116
|
+
`Tensor "${tensorName}" ${label} is missing.`
|
|
117
|
+
);
|
|
118
|
+
}
|
|
119
|
+
return null;
|
|
120
|
+
}
|
|
121
|
+
if (typeof ref !== 'object') {
|
|
122
|
+
fail(
|
|
123
|
+
errorPrefix,
|
|
124
|
+
`Tensor "${tensorName}" ${label} has invalid value.`
|
|
125
|
+
);
|
|
126
|
+
}
|
|
127
|
+
const shard = ref.shardIndex ?? ref.shard;
|
|
128
|
+
return {
|
|
129
|
+
...ref,
|
|
130
|
+
shard: normalizeNonNegativeInteger(shard, tensorName, errorPrefix, `${label}.shard`),
|
|
131
|
+
offset: normalizeNonNegativeInteger(ref.offset, tensorName, errorPrefix, `${label}.offset`),
|
|
132
|
+
size: normalizePositiveInteger(ref.size, tensorName, errorPrefix, `${label}.size`),
|
|
133
|
+
};
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
function normalizeStorageShape(storageShape, tensorName, errorPrefix, label) {
|
|
137
|
+
if (!Array.isArray(storageShape) || storageShape.length !== 2) {
|
|
138
|
+
fail(
|
|
139
|
+
errorPrefix,
|
|
140
|
+
`Tensor "${tensorName}" ${label} requires storageShape=[rows, cols].`
|
|
141
|
+
);
|
|
142
|
+
}
|
|
143
|
+
const rows = normalizePositiveInteger(storageShape[0], tensorName, errorPrefix, `${label} storageShape[0]`);
|
|
144
|
+
const cols = normalizePositiveInteger(storageShape[1], tensorName, errorPrefix, `${label} storageShape[1]`);
|
|
145
|
+
return [rows, cols];
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
function normalizeLaneOrder(storageLaneOrder, storageBlockSize, tensorName, errorPrefix) {
|
|
149
|
+
if (!Array.isArray(storageLaneOrder) || storageLaneOrder.length !== storageBlockSize) {
|
|
150
|
+
fail(
|
|
151
|
+
errorPrefix,
|
|
152
|
+
`Tensor "${tensorName}" LiteRT blocked axis transform requires storageLaneOrder ` +
|
|
153
|
+
`with ${storageBlockSize} entries.`
|
|
154
|
+
);
|
|
155
|
+
}
|
|
156
|
+
const seen = new Set();
|
|
157
|
+
return storageLaneOrder.map((value) => {
|
|
158
|
+
const normalized = Number(value);
|
|
159
|
+
if (
|
|
160
|
+
!Number.isInteger(normalized)
|
|
161
|
+
|| normalized < 0
|
|
162
|
+
|| normalized >= storageBlockSize
|
|
163
|
+
|| seen.has(normalized)
|
|
164
|
+
) {
|
|
165
|
+
fail(
|
|
166
|
+
errorPrefix,
|
|
167
|
+
`Tensor "${tensorName}" has invalid LiteRT storageLaneOrder ${JSON.stringify(storageLaneOrder)}.`
|
|
168
|
+
);
|
|
169
|
+
}
|
|
170
|
+
seen.add(normalized);
|
|
171
|
+
return normalized;
|
|
172
|
+
});
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
function normalizeScaleCompanion(transform, tensorName, errorPrefix) {
|
|
176
|
+
const scaleCompanionDtype = String(transform?.scaleCompanionDtype || '').trim().toUpperCase();
|
|
177
|
+
if (!scaleCompanionDtype) {
|
|
178
|
+
return {};
|
|
179
|
+
}
|
|
180
|
+
if (scaleCompanionDtype !== 'UINT8') {
|
|
181
|
+
fail(
|
|
182
|
+
errorPrefix,
|
|
183
|
+
`Tensor "${tensorName}" LiteRT axis transform has unsupported scaleCompanionDtype "${transform?.scaleCompanionDtype}".`
|
|
184
|
+
);
|
|
185
|
+
}
|
|
186
|
+
if (!transform?.scaleCompanionDequant || typeof transform.scaleCompanionDequant !== 'object') {
|
|
187
|
+
fail(
|
|
188
|
+
errorPrefix,
|
|
189
|
+
`Tensor "${tensorName}" LiteRT axis transform has UINT8 scale companion without scaleCompanionDequant metadata.`
|
|
190
|
+
);
|
|
191
|
+
}
|
|
192
|
+
return {
|
|
193
|
+
scaleCompanionDtype,
|
|
194
|
+
scaleCompanionDequant: {
|
|
195
|
+
scale: normalizePositiveNumber(
|
|
196
|
+
transform.scaleCompanionDequant.scale,
|
|
197
|
+
tensorName,
|
|
198
|
+
errorPrefix,
|
|
199
|
+
'scaleCompanionDequant.scale'
|
|
200
|
+
),
|
|
201
|
+
zeroPoint: normalizeSafeInteger(
|
|
202
|
+
transform.scaleCompanionDequant.zeroPoint,
|
|
203
|
+
tensorName,
|
|
204
|
+
errorPrefix,
|
|
205
|
+
'scaleCompanionDequant.zeroPoint'
|
|
206
|
+
),
|
|
207
|
+
},
|
|
208
|
+
};
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
export function normalizeTensorSourceTransform(location, tensorName, options = {}) {
|
|
212
|
+
const transform = location?.sourceTransform;
|
|
213
|
+
if (!transform || typeof transform !== 'object') {
|
|
214
|
+
return null;
|
|
215
|
+
}
|
|
216
|
+
const errorPrefix = options.errorPrefix ?? '[DopplerLoader]';
|
|
217
|
+
const kind = String(transform.kind || '').trim();
|
|
218
|
+
const sourceDtype = normalizeSourceDtype(transform.sourceDtype, tensorName, errorPrefix);
|
|
219
|
+
|
|
220
|
+
if (kind === 'affine_dequant') {
|
|
221
|
+
if (transform.scheme !== 'per_tensor_affine') {
|
|
222
|
+
fail(
|
|
223
|
+
errorPrefix,
|
|
224
|
+
`Tensor "${tensorName}" has unsupported sourceTransform.scheme "${transform.scheme}".`
|
|
225
|
+
);
|
|
226
|
+
}
|
|
227
|
+
return {
|
|
228
|
+
kind,
|
|
229
|
+
scheme: transform.scheme,
|
|
230
|
+
sourceDtype,
|
|
231
|
+
targetDtype: normalizeTargetDtype(location, transform.targetDtype, tensorName, errorPrefix),
|
|
232
|
+
scale: normalizePositiveNumber(transform.scale, tensorName, errorPrefix, 'sourceTransform.scale'),
|
|
233
|
+
zeroPoint: normalizeSafeInteger(transform.zeroPoint, tensorName, errorPrefix, 'sourceTransform.zeroPoint'),
|
|
234
|
+
};
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
if (kind === 'litert_rowwise_dequant') {
|
|
238
|
+
if (transform.scheme !== 'per_row_affine') {
|
|
239
|
+
fail(
|
|
240
|
+
errorPrefix,
|
|
241
|
+
`Tensor "${tensorName}" has unsupported sourceTransform.scheme "${transform.scheme}".`
|
|
242
|
+
);
|
|
243
|
+
}
|
|
244
|
+
return {
|
|
245
|
+
kind,
|
|
246
|
+
scheme: transform.scheme,
|
|
247
|
+
sourceDtype,
|
|
248
|
+
targetDtype: normalizeTargetDtype(
|
|
249
|
+
location,
|
|
250
|
+
transform.targetDtype,
|
|
251
|
+
tensorName,
|
|
252
|
+
errorPrefix,
|
|
253
|
+
'LiteRT row-wise sourceTransform'
|
|
254
|
+
),
|
|
255
|
+
storageEncoding: normalizeStorageEncoding(
|
|
256
|
+
transform.storageEncoding,
|
|
257
|
+
tensorName,
|
|
258
|
+
errorPrefix,
|
|
259
|
+
'LiteRT storageEncoding'
|
|
260
|
+
),
|
|
261
|
+
...normalizeScaleSemantics(transform, tensorName, errorPrefix, 'LiteRT row-wise transform'),
|
|
262
|
+
scaleSource: normalizeSourceRef(transform.scaleSource, tensorName, errorPrefix, 'LiteRT row-wise scaleSource', true),
|
|
263
|
+
...(transform.rowSumSource == null
|
|
264
|
+
? {}
|
|
265
|
+
: {
|
|
266
|
+
rowSumSource: normalizeSourceRef(
|
|
267
|
+
transform.rowSumSource,
|
|
268
|
+
tensorName,
|
|
269
|
+
errorPrefix,
|
|
270
|
+
'LiteRT row-wise rowSumSource'
|
|
271
|
+
),
|
|
272
|
+
}),
|
|
273
|
+
};
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
if (kind === 'litert_axis_dequant') {
|
|
277
|
+
const quantAxis = Number(transform.quantAxis);
|
|
278
|
+
if (quantAxis !== 0 && quantAxis !== 1) {
|
|
279
|
+
fail(
|
|
280
|
+
errorPrefix,
|
|
281
|
+
`Tensor "${tensorName}" has invalid LiteRT quantAxis ${transform.quantAxis}.`
|
|
282
|
+
);
|
|
283
|
+
}
|
|
284
|
+
return {
|
|
285
|
+
kind,
|
|
286
|
+
scheme: transform.scheme === 'per_axis_affine'
|
|
287
|
+
? transform.scheme
|
|
288
|
+
: fail(
|
|
289
|
+
errorPrefix,
|
|
290
|
+
`Tensor "${tensorName}" has unsupported sourceTransform.scheme "${transform.scheme}".`
|
|
291
|
+
),
|
|
292
|
+
sourceDtype,
|
|
293
|
+
targetDtype: normalizeTargetDtype(
|
|
294
|
+
location,
|
|
295
|
+
transform.targetDtype,
|
|
296
|
+
tensorName,
|
|
297
|
+
errorPrefix,
|
|
298
|
+
'LiteRT axis sourceTransform'
|
|
299
|
+
),
|
|
300
|
+
storageEncoding: normalizeStorageEncoding(
|
|
301
|
+
transform.storageEncoding,
|
|
302
|
+
tensorName,
|
|
303
|
+
errorPrefix,
|
|
304
|
+
'LiteRT storageEncoding'
|
|
305
|
+
),
|
|
306
|
+
...normalizeScaleSemantics(transform, tensorName, errorPrefix, 'LiteRT axis transform'),
|
|
307
|
+
storageShape: normalizeStorageShape(transform.storageShape, tensorName, errorPrefix, 'LiteRT axis transform'),
|
|
308
|
+
quantAxis,
|
|
309
|
+
scaleSource: normalizeSourceRef(transform.scaleSource, tensorName, errorPrefix, 'LiteRT axis scaleSource', true),
|
|
310
|
+
...normalizeScaleCompanion(transform, tensorName, errorPrefix),
|
|
311
|
+
...(transform.sumSource == null
|
|
312
|
+
? {}
|
|
313
|
+
: {
|
|
314
|
+
sumSource: normalizeSourceRef(
|
|
315
|
+
transform.sumSource,
|
|
316
|
+
tensorName,
|
|
317
|
+
errorPrefix,
|
|
318
|
+
'LiteRT axis sumSource'
|
|
319
|
+
),
|
|
320
|
+
}),
|
|
321
|
+
};
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
if (kind === 'litert_axis_blocked_dequant') {
|
|
325
|
+
if (transform.scheme !== 'per_axis_affine') {
|
|
326
|
+
fail(
|
|
327
|
+
errorPrefix,
|
|
328
|
+
`Tensor "${tensorName}" has unsupported sourceTransform.scheme "${transform.scheme}".`
|
|
329
|
+
);
|
|
330
|
+
}
|
|
331
|
+
const quantAxis = Number(transform.quantAxis);
|
|
332
|
+
if (quantAxis !== 0) {
|
|
333
|
+
fail(
|
|
334
|
+
errorPrefix,
|
|
335
|
+
`Tensor "${tensorName}" LiteRT blocked axis transform only supports quantAxis=0.`
|
|
336
|
+
);
|
|
337
|
+
}
|
|
338
|
+
const storageBlockSize = normalizePositiveInteger(
|
|
339
|
+
transform.storageBlockSize,
|
|
340
|
+
tensorName,
|
|
341
|
+
errorPrefix,
|
|
342
|
+
'LiteRT blocked axis storageBlockSize'
|
|
343
|
+
);
|
|
344
|
+
return {
|
|
345
|
+
kind,
|
|
346
|
+
scheme: transform.scheme,
|
|
347
|
+
sourceDtype,
|
|
348
|
+
targetDtype: normalizeTargetDtype(
|
|
349
|
+
location,
|
|
350
|
+
transform.targetDtype,
|
|
351
|
+
tensorName,
|
|
352
|
+
errorPrefix,
|
|
353
|
+
'LiteRT blocked axis sourceTransform'
|
|
354
|
+
),
|
|
355
|
+
storageEncoding: normalizeStorageEncoding(
|
|
356
|
+
transform.storageEncoding,
|
|
357
|
+
tensorName,
|
|
358
|
+
errorPrefix,
|
|
359
|
+
'LiteRT storageEncoding'
|
|
360
|
+
),
|
|
361
|
+
...normalizeScaleSemantics(transform, tensorName, errorPrefix, 'LiteRT blocked axis transform'),
|
|
362
|
+
storageShape: normalizeStorageShape(transform.storageShape, tensorName, errorPrefix, 'LiteRT blocked axis transform'),
|
|
363
|
+
quantAxis,
|
|
364
|
+
storageBlockSize,
|
|
365
|
+
storageLaneOrder: normalizeLaneOrder(transform.storageLaneOrder, storageBlockSize, tensorName, errorPrefix),
|
|
366
|
+
scaleSource: normalizeSourceRef(
|
|
367
|
+
transform.scaleSource,
|
|
368
|
+
tensorName,
|
|
369
|
+
errorPrefix,
|
|
370
|
+
'LiteRT blocked axis scaleSource',
|
|
371
|
+
true
|
|
372
|
+
),
|
|
373
|
+
...(transform.sumSource == null
|
|
374
|
+
? {}
|
|
375
|
+
: {
|
|
376
|
+
sumSource: normalizeSourceRef(
|
|
377
|
+
transform.sumSource,
|
|
378
|
+
tensorName,
|
|
379
|
+
errorPrefix,
|
|
380
|
+
'LiteRT blocked axis sumSource'
|
|
381
|
+
),
|
|
382
|
+
}),
|
|
383
|
+
};
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
fail(
|
|
387
|
+
errorPrefix,
|
|
388
|
+
`Tensor "${tensorName}" has unsupported sourceTransform.kind "${transform.kind}".`
|
|
389
|
+
);
|
|
390
|
+
}
|
|
@@ -52,7 +52,7 @@ export interface TensorSourceLocationRef {
|
|
|
52
52
|
}
|
|
53
53
|
|
|
54
54
|
export interface TensorSourceTransform {
|
|
55
|
-
kind: 'affine_dequant' | 'litert_rowwise_dequant' | 'litert_axis_dequant';
|
|
55
|
+
kind: 'affine_dequant' | 'litert_rowwise_dequant' | 'litert_axis_dequant' | 'litert_axis_blocked_dequant';
|
|
56
56
|
scheme: 'per_tensor_affine' | 'per_row_affine' | 'per_axis_affine';
|
|
57
57
|
sourceDtype: 'INT8' | 'UINT8' | 'INT4' | 'INT2';
|
|
58
58
|
targetDtype: 'F16';
|
|
@@ -63,9 +63,16 @@ export interface TensorSourceTransform {
|
|
|
63
63
|
scaleDivisor?: number;
|
|
64
64
|
storageShape?: number[];
|
|
65
65
|
quantAxis?: 0 | 1;
|
|
66
|
+
storageBlockSize?: number;
|
|
67
|
+
storageLaneOrder?: number[];
|
|
66
68
|
scaleSource?: TensorSourceLocationRef;
|
|
67
69
|
rowSumSource?: TensorSourceLocationRef;
|
|
68
70
|
sumSource?: TensorSourceLocationRef;
|
|
71
|
+
scaleCompanionDtype?: 'UINT8';
|
|
72
|
+
scaleCompanionDequant?: {
|
|
73
|
+
scale: number;
|
|
74
|
+
zeroPoint: number;
|
|
75
|
+
};
|
|
69
76
|
}
|
|
70
77
|
|
|
71
78
|
// =============================================================================
|
|
@@ -54,12 +54,20 @@ export interface TFLiteTensor {
|
|
|
54
54
|
sourceTransform?: TFLiteSourceTransform;
|
|
55
55
|
}
|
|
56
56
|
|
|
57
|
+
export interface TFLiteMetadataEntry {
|
|
58
|
+
name: string;
|
|
59
|
+
buffer: number;
|
|
60
|
+
offset: number;
|
|
61
|
+
size: number;
|
|
62
|
+
}
|
|
63
|
+
|
|
57
64
|
export interface ParsedTFLite {
|
|
58
65
|
schemaVersion: number;
|
|
59
66
|
description: string | null;
|
|
60
67
|
subgraphCount: number;
|
|
61
68
|
mainSubgraphName: string | null;
|
|
62
69
|
tensors: TFLiteTensor[];
|
|
70
|
+
metadataEntries: TFLiteMetadataEntry[];
|
|
63
71
|
sourceQuantization: 'F32' | 'F16' | 'BF16' | null;
|
|
64
72
|
}
|
|
65
73
|
|
|
@@ -469,6 +469,49 @@ async function readBufferLocation(reader, bufferPos, tensorName) {
|
|
|
469
469
|
return size > 0 ? { offset, size } : null;
|
|
470
470
|
}
|
|
471
471
|
|
|
472
|
+
async function readMetadataEntries(reader, modelPos, bufferCount) {
|
|
473
|
+
const metadataCount = await reader.readTableVectorLength(modelPos, 6);
|
|
474
|
+
const metadataEntries = [];
|
|
475
|
+
for (let index = 0; index < metadataCount; index++) {
|
|
476
|
+
const metadataPos = await reader.readTableVectorEntry(modelPos, 6, index);
|
|
477
|
+
if (metadataPos == null) {
|
|
478
|
+
continue;
|
|
479
|
+
}
|
|
480
|
+
const name = await reader.readStringAtField(metadataPos, 0);
|
|
481
|
+
if (!name) {
|
|
482
|
+
continue;
|
|
483
|
+
}
|
|
484
|
+
const bufferFieldPos = await reader.getTableFieldPos(metadataPos, 1);
|
|
485
|
+
const bufferIndex = bufferFieldPos == null ? 0 : await reader.readUint32(bufferFieldPos);
|
|
486
|
+
if (bufferIndex === 0) {
|
|
487
|
+
continue;
|
|
488
|
+
}
|
|
489
|
+
if (bufferIndex >= bufferCount) {
|
|
490
|
+
throw new Error(
|
|
491
|
+
`TFLite parser: metadata "${name}" references buffer ${bufferIndex}, ` +
|
|
492
|
+
`but the model only has ${bufferCount} buffers.`
|
|
493
|
+
);
|
|
494
|
+
}
|
|
495
|
+
const bufferPos = await reader.readTableVectorEntry(modelPos, 4, bufferIndex);
|
|
496
|
+
if (bufferPos == null) {
|
|
497
|
+
throw new Error(
|
|
498
|
+
`TFLite parser: missing buffer table entry ${bufferIndex} for metadata "${name}".`
|
|
499
|
+
);
|
|
500
|
+
}
|
|
501
|
+
const dataLocation = await readBufferLocation(reader, bufferPos, `metadata "${name}"`);
|
|
502
|
+
if (!dataLocation || dataLocation.size <= 0) {
|
|
503
|
+
continue;
|
|
504
|
+
}
|
|
505
|
+
metadataEntries.push({
|
|
506
|
+
name,
|
|
507
|
+
buffer: bufferIndex,
|
|
508
|
+
offset: dataLocation.offset,
|
|
509
|
+
size: dataLocation.size,
|
|
510
|
+
});
|
|
511
|
+
}
|
|
512
|
+
return metadataEntries;
|
|
513
|
+
}
|
|
514
|
+
|
|
472
515
|
async function parseTFLiteReader(reader, options = {}) {
|
|
473
516
|
const parseOptions = normalizeParseOptions(options);
|
|
474
517
|
const rootOffset = await reader.readInt32(0);
|
|
@@ -610,6 +653,7 @@ async function parseTFLiteReader(reader, options = {}) {
|
|
|
610
653
|
'TFLite parser: no supported constant tensors were found in the main subgraph.'
|
|
611
654
|
);
|
|
612
655
|
}
|
|
656
|
+
const metadataEntries = await readMetadataEntries(reader, modelPos, bufferCount);
|
|
613
657
|
|
|
614
658
|
return {
|
|
615
659
|
schemaVersion,
|
|
@@ -617,6 +661,7 @@ async function parseTFLiteReader(reader, options = {}) {
|
|
|
617
661
|
subgraphCount,
|
|
618
662
|
mainSubgraphName: mainSubgraphName || null,
|
|
619
663
|
tensors,
|
|
664
|
+
metadataEntries,
|
|
620
665
|
sourceQuantization: resolveDirectSourceQuantization(tensors),
|
|
621
666
|
};
|
|
622
667
|
}
|
|
@@ -113,6 +113,12 @@ export declare class CommandRecorder {
|
|
|
113
113
|
*/
|
|
114
114
|
trackTemporaryBuffer(buffer: GPUBuffer): void;
|
|
115
115
|
|
|
116
|
+
/**
|
|
117
|
+
* Schedule async work that must run after GPU submission completes but before
|
|
118
|
+
* recorder-owned cleanup makes the captured data unavailable.
|
|
119
|
+
*/
|
|
120
|
+
enqueueCompletionTask(task: () => Promise<void> | void): void;
|
|
121
|
+
|
|
116
122
|
/**
|
|
117
123
|
* Submit all recorded commands and clean up temporary buffers.
|
|
118
124
|
* After calling this, the recorder cannot be reused.
|
|
@@ -123,7 +129,7 @@ export declare class CommandRecorder {
|
|
|
123
129
|
* Finalize a deferred-cleanup submit after another wait condition
|
|
124
130
|
* has already established GPU completion.
|
|
125
131
|
*/
|
|
126
|
-
completeDeferredCleanup(options?: RecorderDeferredCleanupOptions): void
|
|
132
|
+
completeDeferredCleanup(options?: RecorderDeferredCleanupOptions): Promise<void>;
|
|
127
133
|
|
|
128
134
|
/**
|
|
129
135
|
* Submit and wait for GPU to complete (useful for debugging/profiling).
|
|
@@ -36,6 +36,8 @@ export class CommandRecorder {
|
|
|
36
36
|
|
|
37
37
|
#deferredCleanup = null;
|
|
38
38
|
|
|
39
|
+
#completionTasks;
|
|
40
|
+
|
|
39
41
|
|
|
40
42
|
#submitted;
|
|
41
43
|
|
|
@@ -80,6 +82,7 @@ export class CommandRecorder {
|
|
|
80
82
|
this.#pooledBufferSet = new Set();
|
|
81
83
|
this.#cleanupPromise = null;
|
|
82
84
|
this.#deferredCleanup = null;
|
|
85
|
+
this.#completionTasks = [];
|
|
83
86
|
|
|
84
87
|
// Track if already submitted
|
|
85
88
|
this.#submitted = false;
|
|
@@ -292,6 +295,31 @@ export class CommandRecorder {
|
|
|
292
295
|
// Treating them as temporary would destroy weights at submit time.
|
|
293
296
|
}
|
|
294
297
|
|
|
298
|
+
enqueueCompletionTask(task) {
|
|
299
|
+
if (this.#submitted) {
|
|
300
|
+
throw new Error('[CommandRecorder] Cannot enqueue completion tasks after submit');
|
|
301
|
+
}
|
|
302
|
+
if (typeof task !== 'function') {
|
|
303
|
+
throw new Error('[CommandRecorder] completion task must be a function');
|
|
304
|
+
}
|
|
305
|
+
this.#completionTasks.push(task);
|
|
306
|
+
}
|
|
307
|
+
|
|
308
|
+
async #runCompletionTasks() {
|
|
309
|
+
if (this.#completionTasks.length === 0) {
|
|
310
|
+
return;
|
|
311
|
+
}
|
|
312
|
+
const tasks = this.#completionTasks;
|
|
313
|
+
this.#completionTasks = [];
|
|
314
|
+
for (const task of tasks) {
|
|
315
|
+
try {
|
|
316
|
+
await task();
|
|
317
|
+
} catch (error) {
|
|
318
|
+
log.warn('CommandRecorder', `Completion task failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
|
|
295
323
|
#finalizeTrackedBuffers(buffersToDestroy, buffersToRelease, discardPooled) {
|
|
296
324
|
for (const buffer of buffersToDestroy) {
|
|
297
325
|
buffer.destroy();
|
|
@@ -353,6 +381,13 @@ export class CommandRecorder {
|
|
|
353
381
|
this.#cleanupPromise = null;
|
|
354
382
|
|
|
355
383
|
if (cleanup === 'deferred') {
|
|
384
|
+
if (this.#completionTasks.length > 0) {
|
|
385
|
+
this.#cleanupPromise = this.device.queue.onSubmittedWorkDone().then(async () => {
|
|
386
|
+
await this.#runCompletionTasks();
|
|
387
|
+
}).catch((err) => {
|
|
388
|
+
log.warn('CommandRecorder', `Completion tasks failed: ${ (err).message}`);
|
|
389
|
+
});
|
|
390
|
+
}
|
|
356
391
|
this.#deferredCleanup = {
|
|
357
392
|
buffersToDestroy,
|
|
358
393
|
buffersToRelease,
|
|
@@ -361,7 +396,8 @@ export class CommandRecorder {
|
|
|
361
396
|
return;
|
|
362
397
|
}
|
|
363
398
|
|
|
364
|
-
this.#cleanupPromise = this.device.queue.onSubmittedWorkDone().then(() => {
|
|
399
|
+
this.#cleanupPromise = this.device.queue.onSubmittedWorkDone().then(async () => {
|
|
400
|
+
await this.#runCompletionTasks();
|
|
365
401
|
this.#submitLatencyMs = performance.now() - submitStart;
|
|
366
402
|
this.#finalizeTrackedBuffers(buffersToDestroy, buffersToRelease, false);
|
|
367
403
|
}).catch((err) => {
|
|
@@ -370,8 +406,11 @@ export class CommandRecorder {
|
|
|
370
406
|
});
|
|
371
407
|
}
|
|
372
408
|
|
|
373
|
-
completeDeferredCleanup(options = {}) {
|
|
409
|
+
async completeDeferredCleanup(options = {}) {
|
|
374
410
|
const discardPooled = options.discardPooled === true;
|
|
411
|
+
if (this.#cleanupPromise) {
|
|
412
|
+
await this.#cleanupPromise;
|
|
413
|
+
}
|
|
375
414
|
this.#finalizeDeferredCleanup(discardPooled);
|
|
376
415
|
}
|
|
377
416
|
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import type { CommandRecorder } from '../command-recorder.js';
|
|
2
|
+
import type { Tensor } from '../tensor.js';
|
|
3
|
+
|
|
4
|
+
export interface ActivationStaticQdqOptions {
|
|
5
|
+
count?: number;
|
|
6
|
+
outputBuffer?: GPUBuffer | null;
|
|
7
|
+
qmin?: number;
|
|
8
|
+
qmax?: number;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export declare function runActivationStaticQdq(
|
|
12
|
+
input: Tensor,
|
|
13
|
+
scale: number,
|
|
14
|
+
options?: ActivationStaticQdqOptions
|
|
15
|
+
): Promise<Tensor>;
|
|
16
|
+
|
|
17
|
+
export declare function recordActivationStaticQdq(
|
|
18
|
+
recorder: CommandRecorder,
|
|
19
|
+
input: Tensor,
|
|
20
|
+
scale: number,
|
|
21
|
+
options?: ActivationStaticQdqOptions
|
|
22
|
+
): Promise<Tensor>;
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import { acquireBuffer, releaseBuffer } from '../../memory/buffer-pool.js';
|
|
2
|
+
import { createTensor, dtypeBytes } from '../tensor.js';
|
|
3
|
+
import { WORKGROUP_SIZES } from './constants.js';
|
|
4
|
+
import { unifiedKernelWrapper } from './utils.js';
|
|
5
|
+
|
|
6
|
+
function resolveCount(tensor, countOverride) {
|
|
7
|
+
if (Number.isFinite(countOverride) && countOverride > 0) {
|
|
8
|
+
return Math.floor(countOverride);
|
|
9
|
+
}
|
|
10
|
+
if (Array.isArray(tensor.shape) && tensor.shape.length > 0) {
|
|
11
|
+
return tensor.shape.reduce((acc, value) => acc * value, 1);
|
|
12
|
+
}
|
|
13
|
+
return Math.floor(tensor.buffer.size / dtypeBytes(tensor.dtype));
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
async function executeActivationStaticQdq(target, input, scale, options = {}) {
|
|
17
|
+
if (input.dtype !== 'f32') {
|
|
18
|
+
throw new Error(`activation_static_qdq: unsupported dtype ${input.dtype}.`);
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
const { count, outputBuffer = null, qmin = -128, qmax = 127 } = options;
|
|
22
|
+
const resolvedScale = Number(scale);
|
|
23
|
+
const resolvedQmin = Number(qmin);
|
|
24
|
+
const resolvedQmax = Number(qmax);
|
|
25
|
+
if (!Number.isFinite(resolvedScale) || resolvedScale <= 0) {
|
|
26
|
+
throw new Error(`activation_static_qdq: scale must be > 0. Got ${String(scale)}.`);
|
|
27
|
+
}
|
|
28
|
+
if (!Number.isFinite(resolvedQmin) || !Number.isFinite(resolvedQmax) || resolvedQmin > resolvedQmax) {
|
|
29
|
+
throw new Error(
|
|
30
|
+
`activation_static_qdq: invalid qmin/qmax range (${String(qmin)}, ${String(qmax)}).`
|
|
31
|
+
);
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
const inferredCount = resolveCount(input, count);
|
|
35
|
+
const outputSize = inferredCount * dtypeBytes(input.dtype);
|
|
36
|
+
const ownsOutput = outputBuffer == null;
|
|
37
|
+
const output = outputBuffer || acquireBuffer(outputSize, undefined, 'activation_static_qdq_output');
|
|
38
|
+
|
|
39
|
+
try {
|
|
40
|
+
await unifiedKernelWrapper(
|
|
41
|
+
'activation_static_qdq',
|
|
42
|
+
target,
|
|
43
|
+
'default',
|
|
44
|
+
[input, output],
|
|
45
|
+
{
|
|
46
|
+
size: inferredCount,
|
|
47
|
+
scale: resolvedScale,
|
|
48
|
+
invScale: 1 / resolvedScale,
|
|
49
|
+
qmin: resolvedQmin,
|
|
50
|
+
qmax: resolvedQmax,
|
|
51
|
+
},
|
|
52
|
+
Math.ceil(inferredCount / WORKGROUP_SIZES.DEFAULT)
|
|
53
|
+
);
|
|
54
|
+
return createTensor(output, input.dtype, [...input.shape], 'activation_static_qdq_output');
|
|
55
|
+
} catch (error) {
|
|
56
|
+
if (ownsOutput) {
|
|
57
|
+
releaseBuffer(output);
|
|
58
|
+
}
|
|
59
|
+
throw error;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
export async function runActivationStaticQdq(input, scale, options = {}) {
|
|
64
|
+
return executeActivationStaticQdq(null, input, scale, options);
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
export async function recordActivationStaticQdq(recorder, input, scale, options = {}) {
|
|
68
|
+
return executeActivationStaticQdq(recorder, input, scale, options);
|
|
69
|
+
}
|