doppler-gpu 0.2.2 → 0.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -0
- package/assets/doppler.svg +1601 -0
- package/package.json +6 -5
- package/src/cli/doppler-quickstart.js +16 -2
- package/src/cli/doppler-serve.js +0 -0
- package/src/client/doppler-registry.json +17 -1
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +1 -1
- package/src/config/kernels/registry.json +73 -0
- package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +32 -0
- package/src/config/schema/benchmark.schema.d.ts +2 -0
- package/src/config/schema/benchmark.schema.js +1 -0
- package/src/config/schema/doppler.schema.d.ts +9 -1
- package/src/config/schema/doppler.schema.js +8 -1
- package/src/config/schema/index.d.ts +2 -0
- package/src/config/schema/index.js +1 -0
- package/src/config/schema/manifest.schema.d.ts +5 -0
- package/src/converter/quantizer.js +5 -4
- package/src/formats/rdrr/source-transform-contract.js +390 -0
- package/src/formats/rdrr/types.d.ts +8 -1
- package/src/formats/tflite/types.d.ts +8 -0
- package/src/formats/tflite/types.js +45 -0
- package/src/gpu/command-recorder.d.ts +7 -1
- package/src/gpu/command-recorder.js +41 -2
- package/src/gpu/kernels/activation-static-qdq.d.ts +22 -0
- package/src/gpu/kernels/activation-static-qdq.js +69 -0
- package/src/gpu/kernels/activation_static_qdq.wgsl +23 -0
- package/src/gpu/kernels/index.d.ts +7 -0
- package/src/gpu/kernels/index.js +6 -0
- package/src/inference/browser-harness-text-helpers.d.ts +1 -0
- package/src/inference/browser-harness-text-helpers.js +44 -4
- package/src/inference/browser-harness.js +7 -0
- package/src/inference/pipelines/context.js +4 -2
- package/src/inference/pipelines/text/attention/types.js +9 -4
- package/src/inference/pipelines/text/embed.d.ts +1 -0
- package/src/inference/pipelines/text/embed.js +10 -4
- package/src/inference/pipelines/text/generator-helpers.d.ts +6 -0
- package/src/inference/pipelines/text/generator-helpers.js +20 -1
- package/src/inference/pipelines/text/generator-steps.js +17 -8
- package/src/inference/pipelines/text/generator.js +51 -3
- package/src/inference/pipelines/text/init.d.ts +3 -0
- package/src/inference/pipelines/text/init.js +24 -3
- package/src/inference/pipelines/text/logits/gpu.d.ts +5 -4
- package/src/inference/pipelines/text/logits/gpu.js +146 -11
- package/src/inference/pipelines/text/per-layer-inputs.js +320 -57
- package/src/inference/pipelines/text/probes.js +79 -1
- package/src/inference/pipelines/text/stage-names.js +9 -0
- package/src/inference/pipelines/text.js +10 -2
- package/src/loader/embedding-loader.js +20 -5
- package/src/loader/loader-types.d.ts +2 -0
- package/src/loader/manifest-config.d.ts +2 -7
- package/src/loader/per-layer-input-loader.js +114 -1
- package/src/loader/tensors/source-transform.d.ts +15 -0
- package/src/loader/tensors/source-transform.js +312 -34
- package/src/loader/tensors/tensor-reader.js +28 -2
- package/src/storage/artifact-storage-context.d.ts +1 -0
- package/src/storage/artifact-storage-context.js +158 -72
- package/src/storage/preflight.js +11 -0
- package/src/storage/quickstart-downloader.js +15 -4
- package/src/tooling/litert-package-runtime.js +369 -14
- package/src/tooling/node-source-runtime.js +256 -140
- package/src/tooling/source-runtime-bundle.d.ts +25 -0
- package/src/tooling/source-runtime-bundle.js +53 -1
- package/src/tooling/source-runtime-materializer.js +3 -1
|
@@ -8,7 +8,16 @@ import {
|
|
|
8
8
|
isWeightBuffer,
|
|
9
9
|
} from '../../../gpu/weight-buffer.js';
|
|
10
10
|
import { createTensor } from '../../../gpu/tensor.js';
|
|
11
|
-
import {
|
|
11
|
+
import {
|
|
12
|
+
castF16ToF32,
|
|
13
|
+
castF32ToF16,
|
|
14
|
+
recordActivationStaticQdq,
|
|
15
|
+
recordCastF16ToF32,
|
|
16
|
+
recordCastF32ToF16,
|
|
17
|
+
recordScale,
|
|
18
|
+
runActivationStaticQdq,
|
|
19
|
+
runScale,
|
|
20
|
+
} from '../../../gpu/kernel-selector.js';
|
|
12
21
|
import { getNormWeightBuffer, getWeightBuffer } from './weights.js';
|
|
13
22
|
import { doMatmul, doRMSNorm, releaseOrTrack } from './ops.js';
|
|
14
23
|
import { runProbes } from './probes.js';
|
|
@@ -19,6 +28,8 @@ import { acquireBuffer, releaseBuffer } from '../../../memory/buffer-pool.js';
|
|
|
19
28
|
|
|
20
29
|
const pleRuntimeCache = new WeakMap();
|
|
21
30
|
const pleRangeRowCache = new WeakMap();
|
|
31
|
+
const PLE_ACTIVATION_STATIC_QMIN = -128;
|
|
32
|
+
const PLE_ACTIVATION_STATIC_QMAX = 127;
|
|
22
33
|
|
|
23
34
|
function getPerLayerInputWeights(context) {
|
|
24
35
|
const weights = context.weights.get('per_layer_inputs');
|
|
@@ -63,6 +74,106 @@ function isDensePleProjectionDtype(dtype) {
|
|
|
63
74
|
return dtype === 'f16' || dtype === 'f32';
|
|
64
75
|
}
|
|
65
76
|
|
|
77
|
+
function resolvePleActivationStaticScaleValue(value, label) {
|
|
78
|
+
if (value == null) {
|
|
79
|
+
return null;
|
|
80
|
+
}
|
|
81
|
+
const parsed = Number(value);
|
|
82
|
+
if (!Number.isFinite(parsed) || parsed <= 0) {
|
|
83
|
+
throw new Error(
|
|
84
|
+
`Gemma 4 per-layer projection activation static scale "${label}" must be > 0. ` +
|
|
85
|
+
`Got "${String(value)}".`
|
|
86
|
+
);
|
|
87
|
+
}
|
|
88
|
+
return parsed;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
function resolvePleActivationStaticQdqConfig(perLayerInputWeights) {
|
|
92
|
+
const inputScale = resolvePleActivationStaticScaleValue(
|
|
93
|
+
perLayerInputWeights?.perLayerModelProjectionInputActivationStaticScale,
|
|
94
|
+
'input'
|
|
95
|
+
);
|
|
96
|
+
const outputScale = resolvePleActivationStaticScaleValue(
|
|
97
|
+
perLayerInputWeights?.perLayerModelProjectionOutputActivationStaticScale,
|
|
98
|
+
'output'
|
|
99
|
+
);
|
|
100
|
+
if (inputScale == null && outputScale == null) {
|
|
101
|
+
return null;
|
|
102
|
+
}
|
|
103
|
+
if (inputScale == null || outputScale == null) {
|
|
104
|
+
throw new Error(
|
|
105
|
+
'Gemma 4 per-layer projection static activation quantization requires both input and output scales.'
|
|
106
|
+
);
|
|
107
|
+
}
|
|
108
|
+
return {
|
|
109
|
+
inputScale,
|
|
110
|
+
outputScale,
|
|
111
|
+
};
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
function getTensorElementCount(tensor, label) {
|
|
115
|
+
if (!Array.isArray(tensor?.shape) || tensor.shape.length === 0) {
|
|
116
|
+
throw new Error(`${label} requires a tensor with an explicit shape.`);
|
|
117
|
+
}
|
|
118
|
+
const count = tensor.shape.reduce((total, dimension) => total * Number(dimension), 1);
|
|
119
|
+
if (!Number.isFinite(count) || count <= 0) {
|
|
120
|
+
throw new Error(`${label} resolved an invalid tensor element count (${String(count)}).`);
|
|
121
|
+
}
|
|
122
|
+
return count;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
async function applyPleActivationStaticQdq(tensor, scale, recorder, decodeBuffers, label) {
|
|
126
|
+
const count = getTensorElementCount(tensor, label);
|
|
127
|
+
if (tensor.dtype === 'f32') {
|
|
128
|
+
return recorder
|
|
129
|
+
? await recordActivationStaticQdq(recorder, tensor, scale, {
|
|
130
|
+
count,
|
|
131
|
+
qmin: PLE_ACTIVATION_STATIC_QMIN,
|
|
132
|
+
qmax: PLE_ACTIVATION_STATIC_QMAX,
|
|
133
|
+
})
|
|
134
|
+
: await runActivationStaticQdq(tensor, scale, {
|
|
135
|
+
count,
|
|
136
|
+
qmin: PLE_ACTIVATION_STATIC_QMIN,
|
|
137
|
+
qmax: PLE_ACTIVATION_STATIC_QMAX,
|
|
138
|
+
});
|
|
139
|
+
}
|
|
140
|
+
if (tensor.dtype !== 'f16') {
|
|
141
|
+
throw new Error(
|
|
142
|
+
`${label} requires f16/f32 activations for static activation quantize/dequantize. ` +
|
|
143
|
+
`Got "${String(tensor.dtype)}".`
|
|
144
|
+
);
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
let widenedTensor = null;
|
|
148
|
+
let qdqTensor = null;
|
|
149
|
+
try {
|
|
150
|
+
widenedTensor = recorder
|
|
151
|
+
? await recordCastF16ToF32(recorder, tensor)
|
|
152
|
+
: await castF16ToF32(tensor);
|
|
153
|
+
qdqTensor = recorder
|
|
154
|
+
? await recordActivationStaticQdq(recorder, widenedTensor, scale, {
|
|
155
|
+
count,
|
|
156
|
+
qmin: PLE_ACTIVATION_STATIC_QMIN,
|
|
157
|
+
qmax: PLE_ACTIVATION_STATIC_QMAX,
|
|
158
|
+
})
|
|
159
|
+
: await runActivationStaticQdq(widenedTensor, scale, {
|
|
160
|
+
count,
|
|
161
|
+
qmin: PLE_ACTIVATION_STATIC_QMIN,
|
|
162
|
+
qmax: PLE_ACTIVATION_STATIC_QMAX,
|
|
163
|
+
});
|
|
164
|
+
return recorder
|
|
165
|
+
? await recordCastF32ToF16(recorder, qdqTensor)
|
|
166
|
+
: await castF32ToF16(qdqTensor);
|
|
167
|
+
} finally {
|
|
168
|
+
if (qdqTensor) {
|
|
169
|
+
releaseOrTrack(recorder, qdqTensor.buffer, decodeBuffers);
|
|
170
|
+
}
|
|
171
|
+
if (widenedTensor) {
|
|
172
|
+
releaseOrTrack(recorder, widenedTensor.buffer, decodeBuffers);
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
|
|
66
177
|
export function resolveDensePleProjectionWeight(
|
|
67
178
|
weight,
|
|
68
179
|
label = 'per_layer_model_projection'
|
|
@@ -1229,6 +1340,8 @@ export async function preparePerLayerInputs(tokenIds, inputEmbedsTensor, context
|
|
|
1229
1340
|
if (projectionDevice == null) {
|
|
1230
1341
|
throw new Error('Gemma 4 per-layer input projection requires an initialized GPU device.');
|
|
1231
1342
|
}
|
|
1343
|
+
const pleActivationStaticQdq = resolvePleActivationStaticQdqConfig(perLayerInputWeights);
|
|
1344
|
+
let projectionInputTensor = inputEmbedsTensor;
|
|
1232
1345
|
|
|
1233
1346
|
// Decode-path optimizations: coalesced PLE read + fused projection matmul.
|
|
1234
1347
|
// Gated on numTokens === 1 (decode) and row-major embeddings (non-transpose).
|
|
@@ -1237,8 +1350,12 @@ export async function preparePerLayerInputs(tokenIds, inputEmbedsTensor, context
|
|
|
1237
1350
|
const embedTranspose = (hasSplitEmbeddingTables || useHotVocabularyTables) ? false : getEmbeddingTranspose(embedTokensPerLayer);
|
|
1238
1351
|
const canFuseDecodeOps = numTokens === 1 && !embedTranspose && !hasRangeBackedProjection;
|
|
1239
1352
|
const tokenIdsAreGpuBuffer = isGpuBufferInstance(tokenIds);
|
|
1240
|
-
|
|
1241
|
-
|
|
1353
|
+
|
|
1354
|
+
const tokenIdHint = Number.isInteger(options.tokenIdHint) ? Number(options.tokenIdHint) : null;
|
|
1355
|
+
const decodeTokenId = canFuseDecodeOps
|
|
1356
|
+
? (tokenIdsAreGpuBuffer
|
|
1357
|
+
? (Number.isInteger(tokenIdHint) ? tokenIdHint : null)
|
|
1358
|
+
: Number(tokenIds[0]))
|
|
1242
1359
|
: null;
|
|
1243
1360
|
|
|
1244
1361
|
if (decodeTokenId != null) {
|
|
@@ -1310,55 +1427,95 @@ export async function preparePerLayerInputs(tokenIds, inputEmbedsTensor, context
|
|
|
1310
1427
|
// Produces [numTokens × totalPerLayerHiddenSize], then scales the full output and
|
|
1311
1428
|
// extracts per-layer slices via GPU buffer copies.
|
|
1312
1429
|
let fusedProjectionSlices = null;
|
|
1313
|
-
if (canFuseDecodeOps) {
|
|
1314
|
-
const fusedProjection = await doMatmul(
|
|
1315
|
-
inputEmbedsTensor,
|
|
1316
|
-
projectionWeight,
|
|
1317
|
-
numTokens,
|
|
1318
|
-
totalPerLayerHiddenSize,
|
|
1319
|
-
config.hiddenSize,
|
|
1320
|
-
{
|
|
1321
|
-
transposeB: 'auto',
|
|
1322
|
-
label: 'per_layer_fused_projection',
|
|
1323
|
-
kernelPath: context.kernelPath ?? null,
|
|
1324
|
-
role: 'per_layer_model_projection',
|
|
1325
|
-
outputDtype: activationDtype,
|
|
1326
|
-
},
|
|
1327
|
-
recorder
|
|
1328
|
-
);
|
|
1329
|
-
|
|
1330
|
-
// One scale dispatch instead of numLayers
|
|
1331
|
-
const scaledFused = recorder
|
|
1332
|
-
? await recordScale(recorder, fusedProjection, projectionScale, {
|
|
1333
|
-
count: numTokens * totalPerLayerHiddenSize,
|
|
1334
|
-
})
|
|
1335
|
-
: await runScale(fusedProjection, projectionScale, {
|
|
1336
|
-
count: numTokens * totalPerLayerHiddenSize,
|
|
1337
|
-
});
|
|
1338
|
-
releaseOrTrack(recorder, fusedProjection.buffer, decodeBuffers);
|
|
1339
|
-
|
|
1340
|
-
// Step 4: Extract per-layer slices via GPU buffer copies (one encoder, one submit).
|
|
1341
|
-
// Reuse cached slice buffers when available to avoid per-step pool churn.
|
|
1342
|
-
const device = getDevice();
|
|
1343
|
-
const sliceBytes = hiddenSizePerLayerInput * activationBytesPerElement;
|
|
1344
|
-
const encoder = recorder ? recorder.getEncoder() : device.createCommandEncoder();
|
|
1345
|
-
fusedProjectionSlices = new Array(numLayers);
|
|
1346
|
-
for (let l = 0; l < numLayers; l++) {
|
|
1347
|
-
const sliceBuf = pleCache?.sliceBuffers?.[l] ?? acquireBuffer(sliceBytes, undefined, `L${l}.per_layer_proj_slice`);
|
|
1348
|
-
encoder.copyBufferToBuffer(scaledFused.buffer, l * sliceBytes, sliceBuf, 0, sliceBytes);
|
|
1349
|
-
fusedProjectionSlices[l] = sliceBuf;
|
|
1350
|
-
}
|
|
1351
|
-
if (!recorder) {
|
|
1352
|
-
device.queue.submit([encoder.finish()]);
|
|
1353
|
-
}
|
|
1354
|
-
releaseOrTrack(recorder, scaledFused.buffer, decodeBuffers);
|
|
1355
|
-
}
|
|
1356
|
-
|
|
1357
1430
|
const embedDtypeResolved = selectRuleValue('inference', 'dtype', 'f16OrF32FromDtype', {
|
|
1358
1431
|
dtype: perLayerEmbeddingDtype,
|
|
1359
1432
|
});
|
|
1360
1433
|
|
|
1361
1434
|
try {
|
|
1435
|
+
if (pleActivationStaticQdq) {
|
|
1436
|
+
projectionInputTensor = await applyPleActivationStaticQdq(
|
|
1437
|
+
inputEmbedsTensor,
|
|
1438
|
+
pleActivationStaticQdq.inputScale,
|
|
1439
|
+
recorder,
|
|
1440
|
+
decodeBuffers,
|
|
1441
|
+
'Gemma 4 per-layer projection input'
|
|
1442
|
+
);
|
|
1443
|
+
}
|
|
1444
|
+
|
|
1445
|
+
if (canFuseDecodeOps) {
|
|
1446
|
+
let fusedProjection = null;
|
|
1447
|
+
let fusedProjectionForScale = null;
|
|
1448
|
+
let scaledFused = null;
|
|
1449
|
+
try {
|
|
1450
|
+
fusedProjection = await doMatmul(
|
|
1451
|
+
projectionInputTensor,
|
|
1452
|
+
projectionWeight,
|
|
1453
|
+
numTokens,
|
|
1454
|
+
totalPerLayerHiddenSize,
|
|
1455
|
+
config.hiddenSize,
|
|
1456
|
+
{
|
|
1457
|
+
transposeB: 'auto',
|
|
1458
|
+
label: 'per_layer_fused_projection',
|
|
1459
|
+
kernelPath: context.kernelPath ?? null,
|
|
1460
|
+
role: 'per_layer_model_projection',
|
|
1461
|
+
outputDtype: activationDtype,
|
|
1462
|
+
},
|
|
1463
|
+
recorder
|
|
1464
|
+
);
|
|
1465
|
+
fusedProjectionForScale = pleActivationStaticQdq
|
|
1466
|
+
? await applyPleActivationStaticQdq(
|
|
1467
|
+
fusedProjection,
|
|
1468
|
+
pleActivationStaticQdq.outputScale,
|
|
1469
|
+
recorder,
|
|
1470
|
+
decodeBuffers,
|
|
1471
|
+
'Gemma 4 per-layer projection output'
|
|
1472
|
+
)
|
|
1473
|
+
: fusedProjection;
|
|
1474
|
+
if (fusedProjectionForScale !== fusedProjection) {
|
|
1475
|
+
releaseOrTrack(recorder, fusedProjection.buffer, decodeBuffers);
|
|
1476
|
+
fusedProjection = null;
|
|
1477
|
+
}
|
|
1478
|
+
|
|
1479
|
+
scaledFused = recorder
|
|
1480
|
+
? await recordScale(recorder, fusedProjectionForScale, projectionScale, {
|
|
1481
|
+
count: numTokens * totalPerLayerHiddenSize,
|
|
1482
|
+
})
|
|
1483
|
+
: await runScale(fusedProjectionForScale, projectionScale, {
|
|
1484
|
+
count: numTokens * totalPerLayerHiddenSize,
|
|
1485
|
+
});
|
|
1486
|
+
releaseOrTrack(recorder, fusedProjectionForScale.buffer, decodeBuffers);
|
|
1487
|
+
fusedProjectionForScale = null;
|
|
1488
|
+
|
|
1489
|
+
// Step 4: Extract per-layer slices via GPU buffer copies (one encoder, one submit).
|
|
1490
|
+
// Reuse cached slice buffers when available to avoid per-step pool churn.
|
|
1491
|
+
const device = getDevice();
|
|
1492
|
+
const sliceBytes = hiddenSizePerLayerInput * activationBytesPerElement;
|
|
1493
|
+
const encoder = recorder ? recorder.getEncoder() : device.createCommandEncoder();
|
|
1494
|
+
fusedProjectionSlices = new Array(numLayers);
|
|
1495
|
+
for (let l = 0; l < numLayers; l++) {
|
|
1496
|
+
const sliceBuf = pleCache?.sliceBuffers?.[l] ?? acquireBuffer(sliceBytes, undefined, `L${l}.per_layer_proj_slice`);
|
|
1497
|
+
encoder.copyBufferToBuffer(scaledFused.buffer, l * sliceBytes, sliceBuf, 0, sliceBytes);
|
|
1498
|
+
fusedProjectionSlices[l] = sliceBuf;
|
|
1499
|
+
}
|
|
1500
|
+
if (!recorder) {
|
|
1501
|
+
device.queue.submit([encoder.finish()]);
|
|
1502
|
+
}
|
|
1503
|
+
releaseOrTrack(recorder, scaledFused.buffer, decodeBuffers);
|
|
1504
|
+
scaledFused = null;
|
|
1505
|
+
} catch (error) {
|
|
1506
|
+
if (scaledFused) {
|
|
1507
|
+
releaseOrTrack(recorder, scaledFused.buffer, decodeBuffers);
|
|
1508
|
+
}
|
|
1509
|
+
if (fusedProjectionForScale && fusedProjectionForScale !== fusedProjection) {
|
|
1510
|
+
releaseOrTrack(recorder, fusedProjectionForScale.buffer, decodeBuffers);
|
|
1511
|
+
}
|
|
1512
|
+
if (fusedProjection) {
|
|
1513
|
+
releaseOrTrack(recorder, fusedProjection.buffer, decodeBuffers);
|
|
1514
|
+
}
|
|
1515
|
+
throw error;
|
|
1516
|
+
}
|
|
1517
|
+
}
|
|
1518
|
+
|
|
1362
1519
|
for (let layerIdx = 0; layerIdx < numLayers; layerIdx++) {
|
|
1363
1520
|
const hiddenOffset = layerIdx * hiddenSizePerLayerInput;
|
|
1364
1521
|
const layerEmbedSource = useHotVocabularyTables
|
|
@@ -1374,6 +1531,7 @@ export async function preparePerLayerInputs(tokenIds, inputEmbedsTensor, context
|
|
|
1374
1531
|
hiddenSize: hiddenSizePerLayerInput,
|
|
1375
1532
|
vocabSize: useHotVocabularyTables ? (hotVocabularyRuntime.sentinelIndex + 1) : vocabSizePerLayerInput,
|
|
1376
1533
|
scaleEmbeddings: true,
|
|
1534
|
+
probeStage: 'per_layer_embed_out',
|
|
1377
1535
|
recorder,
|
|
1378
1536
|
numTokens,
|
|
1379
1537
|
indexOffset: perLayerTokenIds ? perLayerIndexOffset : indexOffset,
|
|
@@ -1390,6 +1548,7 @@ export async function preparePerLayerInputs(tokenIds, inputEmbedsTensor, context
|
|
|
1390
1548
|
outputBuffer: canFuseDecodeOps
|
|
1391
1549
|
? (pleCache?.gatherSliceBuffers?.[layerIdx] ?? undefined)
|
|
1392
1550
|
: undefined,
|
|
1551
|
+
stats: context.stats ?? null,
|
|
1393
1552
|
});
|
|
1394
1553
|
|
|
1395
1554
|
if (fusedProjectionSlices) {
|
|
@@ -1406,6 +1565,7 @@ export async function preparePerLayerInputs(tokenIds, inputEmbedsTensor, context
|
|
|
1406
1565
|
let projectionWeightBufferForLayer = null;
|
|
1407
1566
|
let projectionWeightOffset = 0;
|
|
1408
1567
|
let projectedTensor = null;
|
|
1568
|
+
let projectionTensorForScale = null;
|
|
1409
1569
|
try {
|
|
1410
1570
|
const rangeBackedProjection = await loadRangeBackedPleProjectionSliceBytes(
|
|
1411
1571
|
perLayerModelProjection,
|
|
@@ -1449,7 +1609,7 @@ export async function preparePerLayerInputs(tokenIds, inputEmbedsTensor, context
|
|
|
1449
1609
|
projectionWeightOffset = hiddenOffset * config.hiddenSize * projectionWeightBytes;
|
|
1450
1610
|
}
|
|
1451
1611
|
projectedTensor = await doMatmul(
|
|
1452
|
-
|
|
1612
|
+
projectionInputTensor,
|
|
1453
1613
|
projectionWeightForLayer,
|
|
1454
1614
|
numTokens,
|
|
1455
1615
|
hiddenSizePerLayerInput,
|
|
@@ -1465,20 +1625,60 @@ export async function preparePerLayerInputs(tokenIds, inputEmbedsTensor, context
|
|
|
1465
1625
|
},
|
|
1466
1626
|
recorder
|
|
1467
1627
|
);
|
|
1628
|
+
await runProbes('per_layer_projection_in', projectedTensor.buffer, {
|
|
1629
|
+
layerIdx,
|
|
1630
|
+
numTokens,
|
|
1631
|
+
hiddenSize: hiddenSizePerLayerInput,
|
|
1632
|
+
probes: context.debugProbes,
|
|
1633
|
+
recorder,
|
|
1634
|
+
operatorDiagnostics: context.operatorDiagnostics,
|
|
1635
|
+
dtype: activationDtype,
|
|
1636
|
+
});
|
|
1637
|
+
projectionTensorForScale = pleActivationStaticQdq
|
|
1638
|
+
? await applyPleActivationStaticQdq(
|
|
1639
|
+
projectedTensor,
|
|
1640
|
+
pleActivationStaticQdq.outputScale,
|
|
1641
|
+
recorder,
|
|
1642
|
+
decodeBuffers,
|
|
1643
|
+
`Gemma 4 per-layer projection output L${layerIdx}`
|
|
1644
|
+
)
|
|
1645
|
+
: projectedTensor;
|
|
1646
|
+
if (projectionTensorForScale !== projectedTensor) {
|
|
1647
|
+
releaseOrTrack(recorder, projectedTensor.buffer, decodeBuffers);
|
|
1648
|
+
projectedTensor = null;
|
|
1649
|
+
}
|
|
1650
|
+
scaledProjectionTensor = recorder
|
|
1651
|
+
? await recordScale(recorder, projectionTensorForScale, projectionScale, {
|
|
1652
|
+
count: numTokens * hiddenSizePerLayerInput,
|
|
1653
|
+
})
|
|
1654
|
+
: await runScale(projectionTensorForScale, projectionScale, {
|
|
1655
|
+
count: numTokens * hiddenSizePerLayerInput,
|
|
1656
|
+
});
|
|
1657
|
+
if (projectionTensorForScale === projectedTensor) {
|
|
1658
|
+
projectedTensor = null;
|
|
1659
|
+
}
|
|
1660
|
+
releaseOrTrack(recorder, projectionTensorForScale.buffer, decodeBuffers);
|
|
1661
|
+
projectionTensorForScale = null;
|
|
1468
1662
|
} finally {
|
|
1663
|
+
if (projectionTensorForScale && projectionTensorForScale !== projectedTensor) {
|
|
1664
|
+
releaseOrTrack(recorder, projectionTensorForScale.buffer, decodeBuffers);
|
|
1665
|
+
}
|
|
1666
|
+
if (projectedTensor) {
|
|
1667
|
+
releaseOrTrack(recorder, projectedTensor.buffer, decodeBuffers);
|
|
1668
|
+
}
|
|
1469
1669
|
if (projectionWeightBufferForLayer) {
|
|
1470
1670
|
releaseOrTrack(recorder, projectionWeightBufferForLayer, decodeBuffers);
|
|
1471
1671
|
}
|
|
1472
1672
|
}
|
|
1473
|
-
scaledProjectionTensor
|
|
1474
|
-
|
|
1475
|
-
|
|
1476
|
-
|
|
1477
|
-
:
|
|
1478
|
-
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
|
|
1673
|
+
await runProbes('per_layer_projection_scaled', scaledProjectionTensor.buffer, {
|
|
1674
|
+
layerIdx,
|
|
1675
|
+
numTokens,
|
|
1676
|
+
hiddenSize: hiddenSizePerLayerInput,
|
|
1677
|
+
probes: context.debugProbes,
|
|
1678
|
+
recorder,
|
|
1679
|
+
operatorDiagnostics: context.operatorDiagnostics,
|
|
1680
|
+
dtype: activationDtype,
|
|
1681
|
+
});
|
|
1482
1682
|
}
|
|
1483
1683
|
|
|
1484
1684
|
// Fuse the residual add into RMSNorm so Gemma 4 PLE decode avoids an
|
|
@@ -1552,6 +1752,9 @@ export async function preparePerLayerInputs(tokenIds, inputEmbedsTensor, context
|
|
|
1552
1752
|
}
|
|
1553
1753
|
throw error;
|
|
1554
1754
|
} finally {
|
|
1755
|
+
if (projectionInputTensor !== inputEmbedsTensor) {
|
|
1756
|
+
releaseOrTrack(recorder, projectionInputTensor.buffer, decodeBuffers);
|
|
1757
|
+
}
|
|
1555
1758
|
if (!usesCachedScaledProjectionNormWeight && !isGpuBufferInstance(perLayerProjectionNorm)) {
|
|
1556
1759
|
releaseOrTrack(recorder, projectionNormWeight, decodeBuffers);
|
|
1557
1760
|
}
|
|
@@ -1568,6 +1771,66 @@ export async function preparePerLayerInputs(tokenIds, inputEmbedsTensor, context
|
|
|
1568
1771
|
);
|
|
1569
1772
|
}
|
|
1570
1773
|
|
|
1774
|
+
// Prefill hot-cache seeding: when we computed a multi-token batch and pleCache is
|
|
1775
|
+
// provided, extract per-token row slices and store unique token IDs in the cache.
|
|
1776
|
+
// Copies are recorded via the active recorder (if any) so they execute after the
|
|
1777
|
+
// batch computation and before the first decode step reads from the cache.
|
|
1778
|
+
// Without a recorder, the batch buffers are already populated so copies are safe
|
|
1779
|
+
// to submit immediately.
|
|
1780
|
+
if (pleCache) {
|
|
1781
|
+
const plePolicy = getPleHotCachePolicy(context.perLayerInputsSession ?? null);
|
|
1782
|
+
if (plePolicy && plePolicy.mode === 'prepared_tokens'
|
|
1783
|
+
&& pleCache.preparedTokenEntries instanceof Map
|
|
1784
|
+
&& !tokenIdsAreGpuBuffer
|
|
1785
|
+
&& numTokens > 0
|
|
1786
|
+
) {
|
|
1787
|
+
const tokenIdArray = Array.isArray(tokenIds) ? tokenIds : Array.from(tokenIds);
|
|
1788
|
+
const sliceBytes = hiddenSizePerLayerInput * activationBytesPerElement;
|
|
1789
|
+
const seen = new Set();
|
|
1790
|
+
const device = getDevice();
|
|
1791
|
+
let pendingEncoder = null;
|
|
1792
|
+
for (let tokenPos = 0; tokenPos < tokenIdArray.length; tokenPos++) {
|
|
1793
|
+
const tid = tokenIdArray[tokenPos];
|
|
1794
|
+
if (seen.has(tid) || pleCache.preparedTokenEntries.has(tid)) continue;
|
|
1795
|
+
if (pleCache.preparedTokenEntries.size >= plePolicy.maxTokens) break;
|
|
1796
|
+
seen.add(tid);
|
|
1797
|
+
const srcOffset = tokenPos * sliceBytes;
|
|
1798
|
+
const sliceBuffers = new Array(numLayers).fill(null);
|
|
1799
|
+
let allValid = true;
|
|
1800
|
+
for (let layerIdx = 0; layerIdx < numLayers; layerIdx++) {
|
|
1801
|
+
const srcBuffer = perLayerBuffers[layerIdx];
|
|
1802
|
+
if (!srcBuffer || srcBuffer.size < srcOffset + sliceBytes) {
|
|
1803
|
+
allValid = false;
|
|
1804
|
+
break;
|
|
1805
|
+
}
|
|
1806
|
+
const dstBuffer = acquireBuffer(sliceBytes, undefined, `ple_seed_L${layerIdx}`);
|
|
1807
|
+
if (recorder) {
|
|
1808
|
+
recorder.getEncoder().copyBufferToBuffer(srcBuffer, srcOffset, dstBuffer, 0, sliceBytes);
|
|
1809
|
+
} else {
|
|
1810
|
+
if (!pendingEncoder) pendingEncoder = device.createCommandEncoder();
|
|
1811
|
+
pendingEncoder.copyBufferToBuffer(srcBuffer, srcOffset, dstBuffer, 0, sliceBytes);
|
|
1812
|
+
}
|
|
1813
|
+
sliceBuffers[layerIdx] = dstBuffer;
|
|
1814
|
+
}
|
|
1815
|
+
if (!allValid) {
|
|
1816
|
+
for (const buf of sliceBuffers) {
|
|
1817
|
+
if (buf) releaseBuffer(buf);
|
|
1818
|
+
}
|
|
1819
|
+
continue;
|
|
1820
|
+
}
|
|
1821
|
+
storePreparedTokenEntry(
|
|
1822
|
+
pleCache, tid, sliceBuffers,
|
|
1823
|
+
context.perLayerInputsSession ?? null,
|
|
1824
|
+
activationDtype,
|
|
1825
|
+
context.stats ?? null
|
|
1826
|
+
);
|
|
1827
|
+
}
|
|
1828
|
+
if (pendingEncoder && device) {
|
|
1829
|
+
device.queue.submit([pendingEncoder.finish()]);
|
|
1830
|
+
}
|
|
1831
|
+
}
|
|
1832
|
+
}
|
|
1833
|
+
|
|
1571
1834
|
return perLayerBuffers;
|
|
1572
1835
|
}
|
|
1573
1836
|
|
|
@@ -15,6 +15,7 @@ import { getDriftPolicyId } from './drift-policy.js';
|
|
|
15
15
|
|
|
16
16
|
const STAGE_DEFAULT_CATEGORY = {
|
|
17
17
|
embed_out: 'embed',
|
|
18
|
+
per_layer_embed_out: 'embed',
|
|
18
19
|
// Attention stages (per-layer)
|
|
19
20
|
attn_input: 'attn',
|
|
20
21
|
post_input_norm: 'attn',
|
|
@@ -36,6 +37,8 @@ const STAGE_DEFAULT_CATEGORY = {
|
|
|
36
37
|
o_proj: 'attn',
|
|
37
38
|
post_attn: 'attn',
|
|
38
39
|
// FFN stages (per-layer)
|
|
40
|
+
per_layer_projection_in: 'ffn',
|
|
41
|
+
per_layer_projection_scaled: 'ffn',
|
|
39
42
|
per_layer_input: 'ffn',
|
|
40
43
|
per_layer_input_gate: 'ffn',
|
|
41
44
|
per_layer_input_activation: 'ffn',
|
|
@@ -127,6 +130,7 @@ export async function runProbes(stage, buffer, options) {
|
|
|
127
130
|
isCpuBuffer,
|
|
128
131
|
shape: [numTokens, hiddenSize],
|
|
129
132
|
dtype,
|
|
133
|
+
recorder,
|
|
130
134
|
});
|
|
131
135
|
diagnostics.emitter.emitRecord(canonicalStage, {
|
|
132
136
|
layerIdx,
|
|
@@ -219,7 +223,14 @@ export function getCanonicalStageName(probeStageName) {
|
|
|
219
223
|
async function buildDiagnosticCapture(level, buffer, options) {
|
|
220
224
|
if (level === 'none') return null;
|
|
221
225
|
|
|
222
|
-
const { isCpuBuffer, shape, dtype } = options;
|
|
226
|
+
const { isCpuBuffer, shape, dtype, recorder } = options;
|
|
227
|
+
if (!isCpuBuffer && recorder) {
|
|
228
|
+
return createDeferredDiagnosticCapture(level, buffer, {
|
|
229
|
+
recorder,
|
|
230
|
+
shape,
|
|
231
|
+
dtype,
|
|
232
|
+
});
|
|
233
|
+
}
|
|
223
234
|
const snapshot = isCpuBuffer
|
|
224
235
|
? snapshotFromArray(buffer, shape, dtype)
|
|
225
236
|
: await snapshotTensor(buffer, shape, dtype);
|
|
@@ -244,3 +255,70 @@ async function buildDiagnosticCapture(level, buffer, options) {
|
|
|
244
255
|
hasInf: snapshot.hasInf === true,
|
|
245
256
|
};
|
|
246
257
|
}
|
|
258
|
+
|
|
259
|
+
function createDeferredDiagnosticCapture(level, buffer, options) {
|
|
260
|
+
const { recorder, shape, dtype } = options;
|
|
261
|
+
const elementSize = dtype === 'f16' ? 2 : 4;
|
|
262
|
+
const numElements = (shape ?? []).reduce((a, b) => a * b, 1);
|
|
263
|
+
const readSize = numElements > 0
|
|
264
|
+
? Math.min(buffer.size, numElements * elementSize)
|
|
265
|
+
: buffer.size;
|
|
266
|
+
const staging = recorder.device.createBuffer({
|
|
267
|
+
label: `${recorder.label}_diagnostic_capture`,
|
|
268
|
+
size: readSize,
|
|
269
|
+
usage: GPUBufferUsage.COPY_DST | GPUBufferUsage.MAP_READ,
|
|
270
|
+
});
|
|
271
|
+
recorder.getEncoder().copyBufferToBuffer(buffer, 0, staging, 0, readSize);
|
|
272
|
+
|
|
273
|
+
const capture = {
|
|
274
|
+
level,
|
|
275
|
+
shape,
|
|
276
|
+
dtype,
|
|
277
|
+
sample: null,
|
|
278
|
+
stats: null,
|
|
279
|
+
hasNaN: false,
|
|
280
|
+
hasInf: false,
|
|
281
|
+
};
|
|
282
|
+
|
|
283
|
+
recorder.enqueueCompletionTask(async () => {
|
|
284
|
+
let mapped = false;
|
|
285
|
+
try {
|
|
286
|
+
await staging.mapAsync(GPUMapMode.READ);
|
|
287
|
+
mapped = true;
|
|
288
|
+
const snapshot = snapshotFromArray(
|
|
289
|
+
decodeSnapshotBytes(staging.getMappedRange().slice(0), dtype),
|
|
290
|
+
shape,
|
|
291
|
+
dtype
|
|
292
|
+
);
|
|
293
|
+
capture.shape = snapshot.shape;
|
|
294
|
+
capture.dtype = snapshot.dtype;
|
|
295
|
+
capture.sample = Array.isArray(snapshot.sample) ? snapshot.sample : null;
|
|
296
|
+
capture.stats = snapshot.stats ?? null;
|
|
297
|
+
capture.hasNaN = snapshot.hasNaN === true;
|
|
298
|
+
capture.hasInf = snapshot.hasInf === true;
|
|
299
|
+
} catch (error) {
|
|
300
|
+
capture.error = error instanceof Error ? error.message : String(error);
|
|
301
|
+
capture.sample = null;
|
|
302
|
+
capture.stats = null;
|
|
303
|
+
} finally {
|
|
304
|
+
if (mapped) {
|
|
305
|
+
staging.unmap();
|
|
306
|
+
}
|
|
307
|
+
staging.destroy();
|
|
308
|
+
}
|
|
309
|
+
});
|
|
310
|
+
|
|
311
|
+
return capture;
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
function decodeSnapshotBytes(data, dtype) {
|
|
315
|
+
if (dtype === 'f16') {
|
|
316
|
+
const src = new Uint16Array(data);
|
|
317
|
+
const decoded = new Float32Array(src.length);
|
|
318
|
+
for (let i = 0; i < src.length; i++) {
|
|
319
|
+
decoded[i] = f16ToF32(src[i]);
|
|
320
|
+
}
|
|
321
|
+
return decoded;
|
|
322
|
+
}
|
|
323
|
+
return new Float32Array(data);
|
|
324
|
+
}
|
|
@@ -36,6 +36,7 @@ export const OPERATOR_CLASSES = Object.freeze({
|
|
|
36
36
|
export const STAGES = Object.freeze({
|
|
37
37
|
// Embedding
|
|
38
38
|
EMBED_OUT: 'embed.out',
|
|
39
|
+
PER_LAYER_EMBED_OUT: 'per_layer_embed.out',
|
|
39
40
|
|
|
40
41
|
// Attention stages (per-layer, prefixed with attn.)
|
|
41
42
|
ATTN_INPUT: 'attn.input',
|
|
@@ -68,6 +69,8 @@ export const STAGES = Object.freeze({
|
|
|
68
69
|
CONV_OUT_PROJ: 'conv.out_proj',
|
|
69
70
|
|
|
70
71
|
// FFN stages (per-layer)
|
|
72
|
+
PER_LAYER_PROJECTION_IN: 'per_layer_projection.in',
|
|
73
|
+
PER_LAYER_PROJECTION_SCALED: 'per_layer_projection.scaled',
|
|
71
74
|
PER_LAYER_INPUT_OUT: 'per_layer_input.out',
|
|
72
75
|
FFN_NORMED: 'ffn.normed',
|
|
73
76
|
FFN_IN: 'ffn.in',
|
|
@@ -100,6 +103,7 @@ export const STAGES = Object.freeze({
|
|
|
100
103
|
|
|
101
104
|
const STAGE_TO_CLASS_MAP = Object.freeze({
|
|
102
105
|
[STAGES.EMBED_OUT]: OPERATOR_CLASSES.EMBEDDING,
|
|
106
|
+
[STAGES.PER_LAYER_EMBED_OUT]: OPERATOR_CLASSES.EMBEDDING,
|
|
103
107
|
|
|
104
108
|
[STAGES.ATTN_INPUT]: OPERATOR_CLASSES.NORMALIZATION,
|
|
105
109
|
[STAGES.ATTN_POST_INPUT_NORM]: OPERATOR_CLASSES.NORMALIZATION,
|
|
@@ -128,6 +132,8 @@ const STAGE_TO_CLASS_MAP = Object.freeze({
|
|
|
128
132
|
[STAGES.CONV_KERNEL]: OPERATOR_CLASSES.CONV,
|
|
129
133
|
[STAGES.CONV_OUT_PROJ]: OPERATOR_CLASSES.CONV,
|
|
130
134
|
|
|
135
|
+
[STAGES.PER_LAYER_PROJECTION_IN]: OPERATOR_CLASSES.PROJECTION,
|
|
136
|
+
[STAGES.PER_LAYER_PROJECTION_SCALED]: OPERATOR_CLASSES.PROJECTION,
|
|
131
137
|
[STAGES.PER_LAYER_INPUT_OUT]: OPERATOR_CLASSES.RESIDUAL,
|
|
132
138
|
[STAGES.FFN_NORMED]: OPERATOR_CLASSES.NORMALIZATION,
|
|
133
139
|
[STAGES.FFN_IN]: OPERATOR_CLASSES.FFN,
|
|
@@ -155,6 +161,7 @@ const STAGE_TO_CLASS_MAP = Object.freeze({
|
|
|
155
161
|
|
|
156
162
|
export const PROBE_TO_CANONICAL = Object.freeze({
|
|
157
163
|
embed_out: STAGES.EMBED_OUT,
|
|
164
|
+
per_layer_embed_out: STAGES.PER_LAYER_EMBED_OUT,
|
|
158
165
|
attn_input: STAGES.ATTN_INPUT,
|
|
159
166
|
post_input_norm: STAGES.ATTN_POST_INPUT_NORM,
|
|
160
167
|
attn_normed: STAGES.ATTN_NORMED,
|
|
@@ -174,6 +181,8 @@ export const PROBE_TO_CANONICAL = Object.freeze({
|
|
|
174
181
|
attn_out: STAGES.ATTN_OUT,
|
|
175
182
|
o_proj: STAGES.ATTN_O_PROJ,
|
|
176
183
|
post_attn: STAGES.ATTN_POST_ATTN,
|
|
184
|
+
per_layer_projection_in: STAGES.PER_LAYER_PROJECTION_IN,
|
|
185
|
+
per_layer_projection_scaled: STAGES.PER_LAYER_PROJECTION_SCALED,
|
|
177
186
|
per_layer_input: STAGES.PER_LAYER_INPUT_OUT,
|
|
178
187
|
ffn_normed: STAGES.FFN_NORMED,
|
|
179
188
|
ffn_in: STAGES.FFN_IN,
|
|
@@ -30,6 +30,7 @@ import {
|
|
|
30
30
|
resolveKernelPathState,
|
|
31
31
|
initTokenizerFromManifest,
|
|
32
32
|
} from './text/model-load.js';
|
|
33
|
+
import { resolvePerLayerInputsSession } from './text/generator-helpers.js';
|
|
33
34
|
import { getKernelPathActivationDtype } from '../../config/kernel-path-loader.js';
|
|
34
35
|
import { applyPipelineDebugConfig } from './text/debug-utils.js';
|
|
35
36
|
import { resolveLayerPipeline } from './text/layer-plan.js';
|
|
@@ -491,8 +492,10 @@ export class InferencePipeline extends PipelineState {
|
|
|
491
492
|
kernelPathSource: this.kernelPathSource,
|
|
492
493
|
keepF32Weights: this.runtimeConfig.inference.compute.keepF32Weights === true,
|
|
493
494
|
loaderDebug: this.runtimeConfig?.shared?.debug?.loader ?? null,
|
|
494
|
-
perLayerInputSession:
|
|
495
|
-
this.
|
|
495
|
+
perLayerInputSession: resolvePerLayerInputsSession(
|
|
496
|
+
this.modelConfig.perLayerInputsSession ?? null,
|
|
497
|
+
this.runtimeConfig?.inference?.session?.perLayerInputs ?? null
|
|
498
|
+
),
|
|
496
499
|
onProgress: (info) => {
|
|
497
500
|
if (info.stage !== 'layers' && info.stage !== 'shards') {
|
|
498
501
|
log.verbose('Loader', `${info.stage}: ${Math.round(info.progress * 100)}%${info.message ? ` - ${info.message}` : ''}`);
|
|
@@ -1548,6 +1551,8 @@ export class InferencePipeline extends PipelineState {
|
|
|
1548
1551
|
|
|
1549
1552
|
|
|
1550
1553
|
async unload() {
|
|
1554
|
+
const storageContext = this.storageContext;
|
|
1555
|
+
this.storageContext = null;
|
|
1551
1556
|
await destroyEmulation(this.emulation);
|
|
1552
1557
|
this.emulation = null;
|
|
1553
1558
|
this.decodeRing?.release();
|
|
@@ -1570,6 +1575,9 @@ export class InferencePipeline extends PipelineState {
|
|
|
1570
1575
|
this.sampleReadbackBuffer.destroy();
|
|
1571
1576
|
this.sampleReadbackBuffer = null;
|
|
1572
1577
|
}
|
|
1578
|
+
if (typeof storageContext?.close === 'function') {
|
|
1579
|
+
await storageContext.close();
|
|
1580
|
+
}
|
|
1573
1581
|
this.isLoaded = false;
|
|
1574
1582
|
this.currentSeqLen = 0;
|
|
1575
1583
|
restorePipelineContexts(this);
|