doppler-gpu 0.2.1 → 0.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +8 -0
- package/README.md +9 -1
- package/assets/doppler.svg +1601 -0
- package/package.json +6 -5
- package/src/cli/doppler-quickstart.js +16 -2
- package/src/cli/doppler-serve.js +0 -0
- package/src/client/doppler-registry.json +17 -1
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +1 -1
- package/src/config/kernels/registry.json +73 -0
- package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +32 -0
- package/src/config/schema/benchmark.schema.d.ts +2 -0
- package/src/config/schema/benchmark.schema.js +1 -0
- package/src/config/schema/doppler.schema.d.ts +9 -1
- package/src/config/schema/doppler.schema.js +8 -1
- package/src/config/schema/index.d.ts +2 -0
- package/src/config/schema/index.js +1 -0
- package/src/config/schema/manifest.schema.d.ts +5 -0
- package/src/converter/quantizer.js +5 -4
- package/src/formats/rdrr/source-transform-contract.js +390 -0
- package/src/formats/rdrr/types.d.ts +8 -1
- package/src/formats/tflite/types.d.ts +8 -0
- package/src/formats/tflite/types.js +45 -0
- package/src/gpu/command-recorder.d.ts +7 -1
- package/src/gpu/command-recorder.js +41 -2
- package/src/gpu/kernels/activation-static-qdq.d.ts +22 -0
- package/src/gpu/kernels/activation-static-qdq.js +69 -0
- package/src/gpu/kernels/activation_static_qdq.wgsl +23 -0
- package/src/gpu/kernels/index.d.ts +7 -0
- package/src/gpu/kernels/index.js +6 -0
- package/src/inference/browser-harness-text-helpers.d.ts +1 -0
- package/src/inference/browser-harness-text-helpers.js +44 -4
- package/src/inference/browser-harness.js +7 -0
- package/src/inference/pipelines/context.js +4 -2
- package/src/inference/pipelines/text/attention/types.js +9 -4
- package/src/inference/pipelines/text/embed.d.ts +1 -0
- package/src/inference/pipelines/text/embed.js +10 -4
- package/src/inference/pipelines/text/generator-helpers.d.ts +6 -0
- package/src/inference/pipelines/text/generator-helpers.js +20 -1
- package/src/inference/pipelines/text/generator-steps.js +17 -8
- package/src/inference/pipelines/text/generator.js +51 -3
- package/src/inference/pipelines/text/init.d.ts +3 -0
- package/src/inference/pipelines/text/init.js +24 -3
- package/src/inference/pipelines/text/logits/gpu.d.ts +5 -4
- package/src/inference/pipelines/text/logits/gpu.js +146 -11
- package/src/inference/pipelines/text/per-layer-inputs.js +320 -57
- package/src/inference/pipelines/text/probes.js +79 -1
- package/src/inference/pipelines/text/stage-names.js +9 -0
- package/src/inference/pipelines/text.js +10 -2
- package/src/loader/embedding-loader.js +20 -5
- package/src/loader/loader-types.d.ts +2 -0
- package/src/loader/manifest-config.d.ts +2 -7
- package/src/loader/per-layer-input-loader.js +114 -1
- package/src/loader/tensors/source-transform.d.ts +15 -0
- package/src/loader/tensors/source-transform.js +312 -34
- package/src/loader/tensors/tensor-reader.js +28 -2
- package/src/storage/artifact-storage-context.d.ts +1 -0
- package/src/storage/artifact-storage-context.js +158 -72
- package/src/storage/preflight.js +11 -0
- package/src/storage/quickstart-downloader.js +15 -4
- package/src/tooling/litert-package-runtime.js +369 -14
- package/src/tooling/node-source-runtime.js +256 -140
- package/src/tooling/source-runtime-bundle.d.ts +25 -0
- package/src/tooling/source-runtime-bundle.js +53 -1
- package/src/tooling/source-runtime-materializer.js +3 -1
- package/src/version.js +1 -1
|
@@ -16,6 +16,7 @@ import { createTensor } from '../gpu/tensor.js';
|
|
|
16
16
|
import { castF16ToF32 } from '../gpu/kernel-selector.js';
|
|
17
17
|
import { releaseBuffer } from '../memory/buffer-pool.js';
|
|
18
18
|
import { loadTensorRange } from './tensors/tensor-reader.js';
|
|
19
|
+
import { hasSourceTransform } from './tensors/source-transform.js';
|
|
19
20
|
|
|
20
21
|
// ============================================================================
|
|
21
22
|
// Constants
|
|
@@ -61,6 +62,24 @@ function createRangeBackedWeightBuffer(ctx, name, location) {
|
|
|
61
62
|
return createCpuWeightBuffer(source, dtype, layout, location.shape, name);
|
|
62
63
|
}
|
|
63
64
|
|
|
65
|
+
function shouldUseRangeBackedEmbeddingSource(ctx, name, location) {
|
|
66
|
+
if (!location) {
|
|
67
|
+
return false;
|
|
68
|
+
}
|
|
69
|
+
if (String(location.dtype ?? '').toUpperCase() === 'F16' && ctx.hostHasShaderF16 === false) {
|
|
70
|
+
return true;
|
|
71
|
+
}
|
|
72
|
+
if (hasSourceTransform(location) && typeof ctx.loadShardRange === 'function') {
|
|
73
|
+
log.info(
|
|
74
|
+
'Loader',
|
|
75
|
+
`Embedding "${name}" uses sourceTransform.kind="${location.sourceTransform.kind}". ` +
|
|
76
|
+
'Using range-backed CPU source to avoid eager full materialization.'
|
|
77
|
+
);
|
|
78
|
+
return true;
|
|
79
|
+
}
|
|
80
|
+
return ctx.shouldStreamLargeWeight(name, location, 'Embedding');
|
|
81
|
+
}
|
|
82
|
+
|
|
64
83
|
// ============================================================================
|
|
65
84
|
// Main Function
|
|
66
85
|
// ============================================================================
|
|
@@ -83,11 +102,7 @@ export async function loadEmbeddings(ctx) {
|
|
|
83
102
|
|
|
84
103
|
for (const name of candidates) {
|
|
85
104
|
const loc = ctx.tensorLocations.get(name);
|
|
86
|
-
const
|
|
87
|
-
&& String(loc.dtype ?? '').toUpperCase() === 'F16'
|
|
88
|
-
&& ctx.hostHasShaderF16 === false;
|
|
89
|
-
const shouldStream = requiresCpuFloatFallback
|
|
90
|
-
|| (loc ? ctx.shouldStreamLargeWeight(name, loc, 'Embedding') : false);
|
|
105
|
+
const shouldStream = shouldUseRangeBackedEmbeddingSource(ctx, name, loc);
|
|
91
106
|
|
|
92
107
|
// Load tensor (to CPU if streaming, to GPU otherwise)
|
|
93
108
|
const tensor = shouldStream
|
|
@@ -84,6 +84,8 @@ export interface PerLayerInputWeights {
|
|
|
84
84
|
embedTokensPerLayerSplit?: (GPUBuffer | WeightBuffer | CpuWeightBuffer | Float32Array | null)[] | null;
|
|
85
85
|
perLayerModelProjection: GPUBuffer | WeightBuffer | Float32Array | null;
|
|
86
86
|
perLayerProjectionNorm: GPUBuffer | Float32Array | null;
|
|
87
|
+
perLayerModelProjectionInputActivationStaticScale?: number | null;
|
|
88
|
+
perLayerModelProjectionOutputActivationStaticScale?: number | null;
|
|
87
89
|
}
|
|
88
90
|
|
|
89
91
|
/**
|
|
@@ -12,6 +12,7 @@
|
|
|
12
12
|
import type { RDRRManifest } from '../formats/rdrr/index.js';
|
|
13
13
|
import type { TensorLocation, KernelCapabilities } from './loader-types.js';
|
|
14
14
|
import type { WeightLayout, WeightDtype } from '../gpu/weight-buffer.js';
|
|
15
|
+
import type { LargeWeightConfigSchema } from '../config/schema/index.js';
|
|
15
16
|
|
|
16
17
|
/**
|
|
17
18
|
* Check if model requires (1 + weight) offset for RMSNorm weights.
|
|
@@ -24,16 +25,10 @@ import type { WeightLayout, WeightDtype } from '../gpu/weight-buffer.js';
|
|
|
24
25
|
*/
|
|
25
26
|
export declare function needsNormWeightOffset(manifest: RDRRManifest | null): boolean;
|
|
26
27
|
|
|
27
|
-
export interface LargeWeightConfig {
|
|
28
|
-
enabled: boolean;
|
|
29
|
-
safetyRatio: number;
|
|
30
|
-
preferF16: boolean;
|
|
31
|
-
}
|
|
32
|
-
|
|
33
28
|
/**
|
|
34
29
|
* Get large weight handling configuration from runtime config.
|
|
35
30
|
*/
|
|
36
|
-
export declare function getLargeWeightConfig():
|
|
31
|
+
export declare function getLargeWeightConfig(): LargeWeightConfigSchema;
|
|
37
32
|
|
|
38
33
|
/**
|
|
39
34
|
* Get maximum bytes for a single GPU buffer binding.
|
|
@@ -45,6 +45,22 @@ const PROJECTION_NORM_TENSOR_CANDIDATES = [
|
|
|
45
45
|
'per_layer_projection_norm.weight',
|
|
46
46
|
];
|
|
47
47
|
|
|
48
|
+
const PROJECTION_INPUT_ACTIVATION_STATIC_SCALE_CANDIDATES = [
|
|
49
|
+
'model.language_model.per_layer_model_projection.input_activation_static_scale',
|
|
50
|
+
'language_model.per_layer_model_projection.input_activation_static_scale',
|
|
51
|
+
'language_model.model.per_layer_model_projection.input_activation_static_scale',
|
|
52
|
+
'model.per_layer_model_projection.input_activation_static_scale',
|
|
53
|
+
'per_layer_model_projection.input_activation_static_scale',
|
|
54
|
+
];
|
|
55
|
+
|
|
56
|
+
const PROJECTION_OUTPUT_ACTIVATION_STATIC_SCALE_CANDIDATES = [
|
|
57
|
+
'model.language_model.per_layer_model_projection.output_activation_static_scale',
|
|
58
|
+
'language_model.per_layer_model_projection.output_activation_static_scale',
|
|
59
|
+
'language_model.model.per_layer_model_projection.output_activation_static_scale',
|
|
60
|
+
'model.per_layer_model_projection.output_activation_static_scale',
|
|
61
|
+
'per_layer_model_projection.output_activation_static_scale',
|
|
62
|
+
];
|
|
63
|
+
|
|
48
64
|
function wrapRawTensorAsWeightBuffer(ctx, tensor, name) {
|
|
49
65
|
if (tensor == null || isWeightBuffer(tensor)) {
|
|
50
66
|
return tensor;
|
|
@@ -420,6 +436,80 @@ async function loadOptionalTensor(ctx, candidates, label) {
|
|
|
420
436
|
return null;
|
|
421
437
|
}
|
|
422
438
|
|
|
439
|
+
function extractScalarTensorValue(data, name, label) {
|
|
440
|
+
if (data instanceof Float32Array) {
|
|
441
|
+
if (data.length !== 1) {
|
|
442
|
+
throw new Error(`${label} "${name}" must resolve to exactly one float32 value.`);
|
|
443
|
+
}
|
|
444
|
+
return data[0];
|
|
445
|
+
}
|
|
446
|
+
if (ArrayBuffer.isView(data)) {
|
|
447
|
+
if (data.byteLength !== Float32Array.BYTES_PER_ELEMENT) {
|
|
448
|
+
throw new Error(`${label} "${name}" must resolve to exactly one float32 value.`);
|
|
449
|
+
}
|
|
450
|
+
return new DataView(data.buffer, data.byteOffset, data.byteLength).getFloat32(0, true);
|
|
451
|
+
}
|
|
452
|
+
if (data instanceof ArrayBuffer) {
|
|
453
|
+
if (data.byteLength !== Float32Array.BYTES_PER_ELEMENT) {
|
|
454
|
+
throw new Error(`${label} "${name}" must resolve to exactly one float32 value.`);
|
|
455
|
+
}
|
|
456
|
+
return new DataView(data).getFloat32(0, true);
|
|
457
|
+
}
|
|
458
|
+
throw new Error(
|
|
459
|
+
`${label} "${name}" must load on CPU as a Float32Array, ArrayBufferView, or ArrayBuffer. ` +
|
|
460
|
+
`Got "${data?.constructor?.name ?? typeof data}".`
|
|
461
|
+
);
|
|
462
|
+
}
|
|
463
|
+
|
|
464
|
+
async function loadNamedScalarTensor(ctx, name, label) {
|
|
465
|
+
const location = ctx.tensorLocations.get(name) ?? null;
|
|
466
|
+
if (!location) {
|
|
467
|
+
return null;
|
|
468
|
+
}
|
|
469
|
+
const shape = Array.isArray(location.shape) ? location.shape : null;
|
|
470
|
+
const elementCount = shape
|
|
471
|
+
? shape.reduce((total, dimension) => total * Number(dimension), 1)
|
|
472
|
+
: NaN;
|
|
473
|
+
if (!Number.isFinite(elementCount) || elementCount !== 1) {
|
|
474
|
+
throw new Error(
|
|
475
|
+
`${label} "${name}" must have exactly one element. ` +
|
|
476
|
+
`Got shape ${JSON.stringify(location.shape)}.`
|
|
477
|
+
);
|
|
478
|
+
}
|
|
479
|
+
if (String(location.dtype ?? '').toUpperCase() !== 'F32') {
|
|
480
|
+
throw new Error(
|
|
481
|
+
`${label} "${name}" must use dtype F32. ` +
|
|
482
|
+
`Got "${String(location.dtype)}".`
|
|
483
|
+
);
|
|
484
|
+
}
|
|
485
|
+
const tensor = await ctx.loadTensor(name, false, true);
|
|
486
|
+
if (tensor == null) {
|
|
487
|
+
return null;
|
|
488
|
+
}
|
|
489
|
+
const value = extractScalarTensorValue(tensor, name, label);
|
|
490
|
+
if (!Number.isFinite(value) || value <= 0) {
|
|
491
|
+
throw new Error(
|
|
492
|
+
`${label} "${name}" must be a finite positive float32 scalar. ` +
|
|
493
|
+
`Got ${String(value)}.`
|
|
494
|
+
);
|
|
495
|
+
}
|
|
496
|
+
log.info('Loader', `Per-layer input tensor loaded: ${label} <- ${name} (cpu scalar)`);
|
|
497
|
+
return {
|
|
498
|
+
name,
|
|
499
|
+
value,
|
|
500
|
+
};
|
|
501
|
+
}
|
|
502
|
+
|
|
503
|
+
async function loadOptionalScalarTensor(ctx, candidates, label) {
|
|
504
|
+
for (const name of candidates) {
|
|
505
|
+
const entry = await loadNamedScalarTensor(ctx, name, label);
|
|
506
|
+
if (entry) {
|
|
507
|
+
return entry;
|
|
508
|
+
}
|
|
509
|
+
}
|
|
510
|
+
return null;
|
|
511
|
+
}
|
|
512
|
+
|
|
423
513
|
async function loadSplitEmbedTensors(ctx, numLayers) {
|
|
424
514
|
const tensorNames = resolveSplitPerLayerEmbedTensorNames(ctx.tensorLocations, numLayers);
|
|
425
515
|
if (!tensorNames) {
|
|
@@ -446,9 +536,19 @@ export async function loadPerLayerInputWeights(ctx, architecture) {
|
|
|
446
536
|
const numLayers = Number(architecture?.numLayers ?? 0);
|
|
447
537
|
|
|
448
538
|
const splitEmbedEntries = await loadSplitEmbedTensors(ctx, numLayers);
|
|
449
|
-
const [projectionEntry, projectionNormEntry] = await Promise.all([
|
|
539
|
+
const [projectionEntry, projectionNormEntry, projectionInputScaleEntry, projectionOutputScaleEntry] = await Promise.all([
|
|
450
540
|
loadOptionalTensor(ctx, PROJECTION_TENSOR_CANDIDATES, 'perLayerModelProjection'),
|
|
451
541
|
loadOptionalTensor(ctx, PROJECTION_NORM_TENSOR_CANDIDATES, 'perLayerProjectionNorm'),
|
|
542
|
+
loadOptionalScalarTensor(
|
|
543
|
+
ctx,
|
|
544
|
+
PROJECTION_INPUT_ACTIVATION_STATIC_SCALE_CANDIDATES,
|
|
545
|
+
'perLayerModelProjectionInputActivationStaticScale'
|
|
546
|
+
),
|
|
547
|
+
loadOptionalScalarTensor(
|
|
548
|
+
ctx,
|
|
549
|
+
PROJECTION_OUTPUT_ACTIVATION_STATIC_SCALE_CANDIDATES,
|
|
550
|
+
'perLayerModelProjectionOutputActivationStaticScale'
|
|
551
|
+
),
|
|
452
552
|
]);
|
|
453
553
|
const embedEntry = splitEmbedEntries?.[0]
|
|
454
554
|
?? await loadOptionalTensor(ctx, EMBED_TENSOR_CANDIDATES, 'embedTokensPerLayer');
|
|
@@ -464,6 +564,13 @@ export async function loadPerLayerInputWeights(ctx, architecture) {
|
|
|
464
564
|
`but the loader could not resolve: ${missing.join(', ')}.`
|
|
465
565
|
);
|
|
466
566
|
}
|
|
567
|
+
if ((projectionInputScaleEntry == null) !== (projectionOutputScaleEntry == null)) {
|
|
568
|
+
throw new Error(
|
|
569
|
+
`Manifest "${ctx.modelId ?? 'unknown'}" must resolve both per-layer projection activation static scales together. ` +
|
|
570
|
+
`Got input=${projectionInputScaleEntry?.name ?? 'missing'}, ` +
|
|
571
|
+
`output=${projectionOutputScaleEntry?.name ?? 'missing'}.`
|
|
572
|
+
);
|
|
573
|
+
}
|
|
467
574
|
|
|
468
575
|
return {
|
|
469
576
|
embedTokensPerLayer: embedEntry.tensor,
|
|
@@ -474,5 +581,11 @@ export async function loadPerLayerInputWeights(ctx, architecture) {
|
|
|
474
581
|
: {}),
|
|
475
582
|
perLayerModelProjection: projectionEntry.tensor,
|
|
476
583
|
perLayerProjectionNorm: projectionNormEntry.tensor,
|
|
584
|
+
...(projectionInputScaleEntry && projectionOutputScaleEntry
|
|
585
|
+
? {
|
|
586
|
+
perLayerModelProjectionInputActivationStaticScale: projectionInputScaleEntry.value,
|
|
587
|
+
perLayerModelProjectionOutputActivationStaticScale: projectionOutputScaleEntry.value,
|
|
588
|
+
}
|
|
589
|
+
: {}),
|
|
477
590
|
};
|
|
478
591
|
}
|
|
@@ -33,6 +33,21 @@ export declare function getSourceTransformSpec(
|
|
|
33
33
|
sourceByteLength: number;
|
|
34
34
|
quantAxis: 0 | 1;
|
|
35
35
|
storageValuesPerByte: number;
|
|
36
|
+
} | {
|
|
37
|
+
kind: 'litert_axis_blocked_dequant';
|
|
38
|
+
rows: number;
|
|
39
|
+
cols: number;
|
|
40
|
+
storageRows: number;
|
|
41
|
+
storageCols: number;
|
|
42
|
+
rawStorageRowBytes: number;
|
|
43
|
+
storageElementBytes: number;
|
|
44
|
+
scaleRowBytes: number;
|
|
45
|
+
targetRowBytes: number;
|
|
46
|
+
outputByteLength: number;
|
|
47
|
+
sourceByteLength: number;
|
|
48
|
+
quantAxis: 0;
|
|
49
|
+
storageBlockSize: number;
|
|
50
|
+
storageLaneOrder: number[];
|
|
36
51
|
} | null);
|
|
37
52
|
|
|
38
53
|
export declare function materializeTensorSourceTransform(
|