doppler-gpu 0.2.2 → 0.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -0
- package/assets/doppler.svg +1601 -0
- package/package.json +6 -5
- package/src/cli/doppler-quickstart.js +16 -2
- package/src/cli/doppler-serve.js +0 -0
- package/src/client/doppler-registry.json +17 -1
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +1 -1
- package/src/config/kernels/registry.json +73 -0
- package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +32 -0
- package/src/config/schema/benchmark.schema.d.ts +2 -0
- package/src/config/schema/benchmark.schema.js +1 -0
- package/src/config/schema/doppler.schema.d.ts +9 -1
- package/src/config/schema/doppler.schema.js +8 -1
- package/src/config/schema/index.d.ts +2 -0
- package/src/config/schema/index.js +1 -0
- package/src/config/schema/manifest.schema.d.ts +5 -0
- package/src/converter/quantizer.js +5 -4
- package/src/formats/rdrr/source-transform-contract.js +390 -0
- package/src/formats/rdrr/types.d.ts +8 -1
- package/src/formats/tflite/types.d.ts +8 -0
- package/src/formats/tflite/types.js +45 -0
- package/src/gpu/command-recorder.d.ts +7 -1
- package/src/gpu/command-recorder.js +41 -2
- package/src/gpu/kernels/activation-static-qdq.d.ts +22 -0
- package/src/gpu/kernels/activation-static-qdq.js +69 -0
- package/src/gpu/kernels/activation_static_qdq.wgsl +23 -0
- package/src/gpu/kernels/index.d.ts +7 -0
- package/src/gpu/kernels/index.js +6 -0
- package/src/inference/browser-harness-text-helpers.d.ts +1 -0
- package/src/inference/browser-harness-text-helpers.js +44 -4
- package/src/inference/browser-harness.js +7 -0
- package/src/inference/pipelines/context.js +4 -2
- package/src/inference/pipelines/text/attention/types.js +9 -4
- package/src/inference/pipelines/text/embed.d.ts +1 -0
- package/src/inference/pipelines/text/embed.js +10 -4
- package/src/inference/pipelines/text/generator-helpers.d.ts +6 -0
- package/src/inference/pipelines/text/generator-helpers.js +20 -1
- package/src/inference/pipelines/text/generator-steps.js +17 -8
- package/src/inference/pipelines/text/generator.js +51 -3
- package/src/inference/pipelines/text/init.d.ts +3 -0
- package/src/inference/pipelines/text/init.js +24 -3
- package/src/inference/pipelines/text/logits/gpu.d.ts +5 -4
- package/src/inference/pipelines/text/logits/gpu.js +146 -11
- package/src/inference/pipelines/text/per-layer-inputs.js +320 -57
- package/src/inference/pipelines/text/probes.js +79 -1
- package/src/inference/pipelines/text/stage-names.js +9 -0
- package/src/inference/pipelines/text.js +10 -2
- package/src/loader/embedding-loader.js +20 -5
- package/src/loader/loader-types.d.ts +2 -0
- package/src/loader/manifest-config.d.ts +2 -7
- package/src/loader/per-layer-input-loader.js +114 -1
- package/src/loader/tensors/source-transform.d.ts +15 -0
- package/src/loader/tensors/source-transform.js +312 -34
- package/src/loader/tensors/tensor-reader.js +28 -2
- package/src/storage/artifact-storage-context.d.ts +1 -0
- package/src/storage/artifact-storage-context.js +158 -72
- package/src/storage/preflight.js +11 -0
- package/src/storage/quickstart-downloader.js +15 -4
- package/src/tooling/litert-package-runtime.js +369 -14
- package/src/tooling/node-source-runtime.js +256 -140
- package/src/tooling/source-runtime-bundle.d.ts +25 -0
- package/src/tooling/source-runtime-bundle.js +53 -1
- package/src/tooling/source-runtime-materializer.js +3 -1
|
@@ -20,6 +20,7 @@ import {
|
|
|
20
20
|
resolveTensorRole,
|
|
21
21
|
sortGroupIds,
|
|
22
22
|
} from '../formats/rdrr/index.js';
|
|
23
|
+
import { normalizeTensorSourceTransform } from '../formats/rdrr/source-transform-contract.js';
|
|
23
24
|
import { createRuntimeModelContract } from '../inference/runtime-model.js';
|
|
24
25
|
import { computeHash, createStreamingHasher } from '../storage/shard-manager.js';
|
|
25
26
|
|
|
@@ -278,6 +279,49 @@ function buildSourceTensorTransform(tensor, shardIndexByPath, tensorName) {
|
|
|
278
279
|
storageShape: transform.storageShape,
|
|
279
280
|
quantAxis: transform.quantAxis,
|
|
280
281
|
scaleSource: buildTransformScaleSource(transform, shardIndexByPath, tensorName),
|
|
282
|
+
...(typeof transform.scaleCompanionDtype === 'string'
|
|
283
|
+
? {
|
|
284
|
+
scaleCompanionDtype: transform.scaleCompanionDtype,
|
|
285
|
+
}
|
|
286
|
+
: {}),
|
|
287
|
+
...(transform.scaleCompanionDequant && typeof transform.scaleCompanionDequant === 'object'
|
|
288
|
+
? {
|
|
289
|
+
scaleCompanionDequant: {
|
|
290
|
+
scale: transform.scaleCompanionDequant.scale,
|
|
291
|
+
zeroPoint: transform.scaleCompanionDequant.zeroPoint,
|
|
292
|
+
},
|
|
293
|
+
}
|
|
294
|
+
: {}),
|
|
295
|
+
...(sumSource
|
|
296
|
+
? {
|
|
297
|
+
sumSource,
|
|
298
|
+
}
|
|
299
|
+
: {}),
|
|
300
|
+
};
|
|
301
|
+
}
|
|
302
|
+
if (transform.kind === 'litert_axis_blocked_dequant') {
|
|
303
|
+
const sumSource = buildTransformCompanionSource(
|
|
304
|
+
transform,
|
|
305
|
+
shardIndexByPath,
|
|
306
|
+
tensorName,
|
|
307
|
+
'sumSourcePath',
|
|
308
|
+
'sumOffset',
|
|
309
|
+
'sumSize',
|
|
310
|
+
'sum'
|
|
311
|
+
);
|
|
312
|
+
return {
|
|
313
|
+
kind: transform.kind,
|
|
314
|
+
scheme: transform.scheme,
|
|
315
|
+
sourceDtype: transform.sourceDtype,
|
|
316
|
+
targetDtype: transform.targetDtype,
|
|
317
|
+
storageEncoding: transform.storageEncoding,
|
|
318
|
+
scaleSemantics: transform.scaleSemantics,
|
|
319
|
+
scaleDivisor: transform.scaleDivisor,
|
|
320
|
+
storageShape: transform.storageShape,
|
|
321
|
+
quantAxis: transform.quantAxis,
|
|
322
|
+
storageBlockSize: transform.storageBlockSize,
|
|
323
|
+
storageLaneOrder: transform.storageLaneOrder,
|
|
324
|
+
scaleSource: buildTransformScaleSource(transform, shardIndexByPath, tensorName),
|
|
281
325
|
...(sumSource
|
|
282
326
|
? {
|
|
283
327
|
sumSource,
|
|
@@ -327,7 +371,7 @@ function buildSourceTensorLocations(tensors, shardIndexByPath, modelType) {
|
|
|
327
371
|
: null;
|
|
328
372
|
const sourceTransform = buildSourceTensorTransform(tensor, shardIndexByPath, name);
|
|
329
373
|
|
|
330
|
-
|
|
374
|
+
const location = {
|
|
331
375
|
shard,
|
|
332
376
|
offset,
|
|
333
377
|
size,
|
|
@@ -338,6 +382,10 @@ function buildSourceTensorLocations(tensors, shardIndexByPath, modelType) {
|
|
|
338
382
|
...(layout ? { layout } : {}),
|
|
339
383
|
...(sourceTransform ? { sourceTransform } : {}),
|
|
340
384
|
};
|
|
385
|
+
if (sourceTransform) {
|
|
386
|
+
normalizeTensorSourceTransform(location, name, { errorPrefix: '[SourceRuntime]' });
|
|
387
|
+
}
|
|
388
|
+
locations[name] = location;
|
|
341
389
|
}
|
|
342
390
|
|
|
343
391
|
return locations;
|
|
@@ -822,6 +870,9 @@ export function createSourceStorageContext(options = {}) {
|
|
|
822
870
|
const readBinary = typeof options.readBinary === 'function'
|
|
823
871
|
? options.readBinary
|
|
824
872
|
: null;
|
|
873
|
+
const close = typeof options.close === 'function'
|
|
874
|
+
? options.close
|
|
875
|
+
: null;
|
|
825
876
|
const sourceFileMap = new Map(
|
|
826
877
|
shardSources.map((entry) => [entry.path, entry])
|
|
827
878
|
);
|
|
@@ -1019,5 +1070,6 @@ export function createSourceStorageContext(options = {}) {
|
|
|
1019
1070
|
loadTokenizerJson,
|
|
1020
1071
|
loadTokenizerModel,
|
|
1021
1072
|
verifyHashes,
|
|
1073
|
+
close,
|
|
1022
1074
|
};
|
|
1023
1075
|
}
|
|
@@ -21,7 +21,9 @@ function toRelativeArtifactPath(value, artifactDir, label) {
|
|
|
21
21
|
throw new Error(`${label} path is required.`);
|
|
22
22
|
}
|
|
23
23
|
const resolvedArtifactDir = path.resolve(artifactDir);
|
|
24
|
-
const resolvedTarget = path.
|
|
24
|
+
const resolvedTarget = path.isAbsolute(raw)
|
|
25
|
+
? path.resolve(raw)
|
|
26
|
+
: path.resolve(resolvedArtifactDir, raw);
|
|
25
27
|
const relativePath = path.relative(resolvedArtifactDir, resolvedTarget).replace(/\\/g, '/');
|
|
26
28
|
if (!relativePath || relativePath.startsWith('../') || relativePath === '..') {
|
|
27
29
|
throw new Error(
|