doppler-gpu 0.2.2 → 0.4.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. package/README.md +8 -0
  2. package/assets/doppler.svg +1601 -0
  3. package/package.json +6 -5
  4. package/src/cli/doppler-quickstart.js +16 -2
  5. package/src/cli/doppler-serve.js +0 -0
  6. package/src/client/doppler-registry.json +17 -1
  7. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +1 -1
  8. package/src/config/kernels/registry.json +73 -0
  9. package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +32 -0
  10. package/src/config/schema/benchmark.schema.d.ts +2 -0
  11. package/src/config/schema/benchmark.schema.js +1 -0
  12. package/src/config/schema/doppler.schema.d.ts +9 -1
  13. package/src/config/schema/doppler.schema.js +8 -1
  14. package/src/config/schema/index.d.ts +2 -0
  15. package/src/config/schema/index.js +1 -0
  16. package/src/config/schema/manifest.schema.d.ts +5 -0
  17. package/src/converter/quantizer.js +5 -4
  18. package/src/formats/rdrr/source-transform-contract.js +390 -0
  19. package/src/formats/rdrr/types.d.ts +8 -1
  20. package/src/formats/tflite/types.d.ts +8 -0
  21. package/src/formats/tflite/types.js +45 -0
  22. package/src/gpu/command-recorder.d.ts +7 -1
  23. package/src/gpu/command-recorder.js +41 -2
  24. package/src/gpu/kernels/activation-static-qdq.d.ts +22 -0
  25. package/src/gpu/kernels/activation-static-qdq.js +69 -0
  26. package/src/gpu/kernels/activation_static_qdq.wgsl +23 -0
  27. package/src/gpu/kernels/index.d.ts +7 -0
  28. package/src/gpu/kernels/index.js +6 -0
  29. package/src/inference/browser-harness-text-helpers.d.ts +1 -0
  30. package/src/inference/browser-harness-text-helpers.js +44 -4
  31. package/src/inference/browser-harness.js +7 -0
  32. package/src/inference/pipelines/context.js +4 -2
  33. package/src/inference/pipelines/text/attention/types.js +9 -4
  34. package/src/inference/pipelines/text/embed.d.ts +1 -0
  35. package/src/inference/pipelines/text/embed.js +10 -4
  36. package/src/inference/pipelines/text/generator-helpers.d.ts +6 -0
  37. package/src/inference/pipelines/text/generator-helpers.js +20 -1
  38. package/src/inference/pipelines/text/generator-steps.js +17 -8
  39. package/src/inference/pipelines/text/generator.js +51 -3
  40. package/src/inference/pipelines/text/init.d.ts +3 -0
  41. package/src/inference/pipelines/text/init.js +24 -3
  42. package/src/inference/pipelines/text/logits/gpu.d.ts +5 -4
  43. package/src/inference/pipelines/text/logits/gpu.js +146 -11
  44. package/src/inference/pipelines/text/per-layer-inputs.js +320 -57
  45. package/src/inference/pipelines/text/probes.js +79 -1
  46. package/src/inference/pipelines/text/stage-names.js +9 -0
  47. package/src/inference/pipelines/text.js +10 -2
  48. package/src/loader/embedding-loader.js +20 -5
  49. package/src/loader/loader-types.d.ts +2 -0
  50. package/src/loader/manifest-config.d.ts +2 -7
  51. package/src/loader/per-layer-input-loader.js +114 -1
  52. package/src/loader/tensors/source-transform.d.ts +15 -0
  53. package/src/loader/tensors/source-transform.js +312 -34
  54. package/src/loader/tensors/tensor-reader.js +28 -2
  55. package/src/storage/artifact-storage-context.d.ts +1 -0
  56. package/src/storage/artifact-storage-context.js +158 -72
  57. package/src/storage/preflight.js +11 -0
  58. package/src/storage/quickstart-downloader.js +15 -4
  59. package/src/tooling/litert-package-runtime.js +369 -14
  60. package/src/tooling/node-source-runtime.js +256 -140
  61. package/src/tooling/source-runtime-bundle.d.ts +25 -0
  62. package/src/tooling/source-runtime-bundle.js +53 -1
  63. package/src/tooling/source-runtime-materializer.js +3 -1
@@ -8,7 +8,16 @@ import {
8
8
  isWeightBuffer,
9
9
  } from '../../../gpu/weight-buffer.js';
10
10
  import { createTensor } from '../../../gpu/tensor.js';
11
- import { recordScale, runScale } from '../../../gpu/kernel-selector.js';
11
+ import {
12
+ castF16ToF32,
13
+ castF32ToF16,
14
+ recordActivationStaticQdq,
15
+ recordCastF16ToF32,
16
+ recordCastF32ToF16,
17
+ recordScale,
18
+ runActivationStaticQdq,
19
+ runScale,
20
+ } from '../../../gpu/kernel-selector.js';
12
21
  import { getNormWeightBuffer, getWeightBuffer } from './weights.js';
13
22
  import { doMatmul, doRMSNorm, releaseOrTrack } from './ops.js';
14
23
  import { runProbes } from './probes.js';
@@ -19,6 +28,8 @@ import { acquireBuffer, releaseBuffer } from '../../../memory/buffer-pool.js';
19
28
 
20
29
  const pleRuntimeCache = new WeakMap();
21
30
  const pleRangeRowCache = new WeakMap();
31
+ const PLE_ACTIVATION_STATIC_QMIN = -128;
32
+ const PLE_ACTIVATION_STATIC_QMAX = 127;
22
33
 
23
34
  function getPerLayerInputWeights(context) {
24
35
  const weights = context.weights.get('per_layer_inputs');
@@ -63,6 +74,106 @@ function isDensePleProjectionDtype(dtype) {
63
74
  return dtype === 'f16' || dtype === 'f32';
64
75
  }
65
76
 
77
+ function resolvePleActivationStaticScaleValue(value, label) {
78
+ if (value == null) {
79
+ return null;
80
+ }
81
+ const parsed = Number(value);
82
+ if (!Number.isFinite(parsed) || parsed <= 0) {
83
+ throw new Error(
84
+ `Gemma 4 per-layer projection activation static scale "${label}" must be > 0. ` +
85
+ `Got "${String(value)}".`
86
+ );
87
+ }
88
+ return parsed;
89
+ }
90
+
91
+ function resolvePleActivationStaticQdqConfig(perLayerInputWeights) {
92
+ const inputScale = resolvePleActivationStaticScaleValue(
93
+ perLayerInputWeights?.perLayerModelProjectionInputActivationStaticScale,
94
+ 'input'
95
+ );
96
+ const outputScale = resolvePleActivationStaticScaleValue(
97
+ perLayerInputWeights?.perLayerModelProjectionOutputActivationStaticScale,
98
+ 'output'
99
+ );
100
+ if (inputScale == null && outputScale == null) {
101
+ return null;
102
+ }
103
+ if (inputScale == null || outputScale == null) {
104
+ throw new Error(
105
+ 'Gemma 4 per-layer projection static activation quantization requires both input and output scales.'
106
+ );
107
+ }
108
+ return {
109
+ inputScale,
110
+ outputScale,
111
+ };
112
+ }
113
+
114
+ function getTensorElementCount(tensor, label) {
115
+ if (!Array.isArray(tensor?.shape) || tensor.shape.length === 0) {
116
+ throw new Error(`${label} requires a tensor with an explicit shape.`);
117
+ }
118
+ const count = tensor.shape.reduce((total, dimension) => total * Number(dimension), 1);
119
+ if (!Number.isFinite(count) || count <= 0) {
120
+ throw new Error(`${label} resolved an invalid tensor element count (${String(count)}).`);
121
+ }
122
+ return count;
123
+ }
124
+
125
+ async function applyPleActivationStaticQdq(tensor, scale, recorder, decodeBuffers, label) {
126
+ const count = getTensorElementCount(tensor, label);
127
+ if (tensor.dtype === 'f32') {
128
+ return recorder
129
+ ? await recordActivationStaticQdq(recorder, tensor, scale, {
130
+ count,
131
+ qmin: PLE_ACTIVATION_STATIC_QMIN,
132
+ qmax: PLE_ACTIVATION_STATIC_QMAX,
133
+ })
134
+ : await runActivationStaticQdq(tensor, scale, {
135
+ count,
136
+ qmin: PLE_ACTIVATION_STATIC_QMIN,
137
+ qmax: PLE_ACTIVATION_STATIC_QMAX,
138
+ });
139
+ }
140
+ if (tensor.dtype !== 'f16') {
141
+ throw new Error(
142
+ `${label} requires f16/f32 activations for static activation quantize/dequantize. ` +
143
+ `Got "${String(tensor.dtype)}".`
144
+ );
145
+ }
146
+
147
+ let widenedTensor = null;
148
+ let qdqTensor = null;
149
+ try {
150
+ widenedTensor = recorder
151
+ ? await recordCastF16ToF32(recorder, tensor)
152
+ : await castF16ToF32(tensor);
153
+ qdqTensor = recorder
154
+ ? await recordActivationStaticQdq(recorder, widenedTensor, scale, {
155
+ count,
156
+ qmin: PLE_ACTIVATION_STATIC_QMIN,
157
+ qmax: PLE_ACTIVATION_STATIC_QMAX,
158
+ })
159
+ : await runActivationStaticQdq(widenedTensor, scale, {
160
+ count,
161
+ qmin: PLE_ACTIVATION_STATIC_QMIN,
162
+ qmax: PLE_ACTIVATION_STATIC_QMAX,
163
+ });
164
+ return recorder
165
+ ? await recordCastF32ToF16(recorder, qdqTensor)
166
+ : await castF32ToF16(qdqTensor);
167
+ } finally {
168
+ if (qdqTensor) {
169
+ releaseOrTrack(recorder, qdqTensor.buffer, decodeBuffers);
170
+ }
171
+ if (widenedTensor) {
172
+ releaseOrTrack(recorder, widenedTensor.buffer, decodeBuffers);
173
+ }
174
+ }
175
+ }
176
+
66
177
  export function resolveDensePleProjectionWeight(
67
178
  weight,
68
179
  label = 'per_layer_model_projection'
@@ -1229,6 +1340,8 @@ export async function preparePerLayerInputs(tokenIds, inputEmbedsTensor, context
1229
1340
  if (projectionDevice == null) {
1230
1341
  throw new Error('Gemma 4 per-layer input projection requires an initialized GPU device.');
1231
1342
  }
1343
+ const pleActivationStaticQdq = resolvePleActivationStaticQdqConfig(perLayerInputWeights);
1344
+ let projectionInputTensor = inputEmbedsTensor;
1232
1345
 
1233
1346
  // Decode-path optimizations: coalesced PLE read + fused projection matmul.
1234
1347
  // Gated on numTokens === 1 (decode) and row-major embeddings (non-transpose).
@@ -1237,8 +1350,12 @@ export async function preparePerLayerInputs(tokenIds, inputEmbedsTensor, context
1237
1350
  const embedTranspose = (hasSplitEmbeddingTables || useHotVocabularyTables) ? false : getEmbeddingTranspose(embedTokensPerLayer);
1238
1351
  const canFuseDecodeOps = numTokens === 1 && !embedTranspose && !hasRangeBackedProjection;
1239
1352
  const tokenIdsAreGpuBuffer = isGpuBufferInstance(tokenIds);
1240
- const decodeTokenId = canFuseDecodeOps && !tokenIdsAreGpuBuffer
1241
- ? Number(tokenIds[0])
1353
+
1354
+ const tokenIdHint = Number.isInteger(options.tokenIdHint) ? Number(options.tokenIdHint) : null;
1355
+ const decodeTokenId = canFuseDecodeOps
1356
+ ? (tokenIdsAreGpuBuffer
1357
+ ? (Number.isInteger(tokenIdHint) ? tokenIdHint : null)
1358
+ : Number(tokenIds[0]))
1242
1359
  : null;
1243
1360
 
1244
1361
  if (decodeTokenId != null) {
@@ -1310,55 +1427,95 @@ export async function preparePerLayerInputs(tokenIds, inputEmbedsTensor, context
1310
1427
  // Produces [numTokens × totalPerLayerHiddenSize], then scales the full output and
1311
1428
  // extracts per-layer slices via GPU buffer copies.
1312
1429
  let fusedProjectionSlices = null;
1313
- if (canFuseDecodeOps) {
1314
- const fusedProjection = await doMatmul(
1315
- inputEmbedsTensor,
1316
- projectionWeight,
1317
- numTokens,
1318
- totalPerLayerHiddenSize,
1319
- config.hiddenSize,
1320
- {
1321
- transposeB: 'auto',
1322
- label: 'per_layer_fused_projection',
1323
- kernelPath: context.kernelPath ?? null,
1324
- role: 'per_layer_model_projection',
1325
- outputDtype: activationDtype,
1326
- },
1327
- recorder
1328
- );
1329
-
1330
- // One scale dispatch instead of numLayers
1331
- const scaledFused = recorder
1332
- ? await recordScale(recorder, fusedProjection, projectionScale, {
1333
- count: numTokens * totalPerLayerHiddenSize,
1334
- })
1335
- : await runScale(fusedProjection, projectionScale, {
1336
- count: numTokens * totalPerLayerHiddenSize,
1337
- });
1338
- releaseOrTrack(recorder, fusedProjection.buffer, decodeBuffers);
1339
-
1340
- // Step 4: Extract per-layer slices via GPU buffer copies (one encoder, one submit).
1341
- // Reuse cached slice buffers when available to avoid per-step pool churn.
1342
- const device = getDevice();
1343
- const sliceBytes = hiddenSizePerLayerInput * activationBytesPerElement;
1344
- const encoder = recorder ? recorder.getEncoder() : device.createCommandEncoder();
1345
- fusedProjectionSlices = new Array(numLayers);
1346
- for (let l = 0; l < numLayers; l++) {
1347
- const sliceBuf = pleCache?.sliceBuffers?.[l] ?? acquireBuffer(sliceBytes, undefined, `L${l}.per_layer_proj_slice`);
1348
- encoder.copyBufferToBuffer(scaledFused.buffer, l * sliceBytes, sliceBuf, 0, sliceBytes);
1349
- fusedProjectionSlices[l] = sliceBuf;
1350
- }
1351
- if (!recorder) {
1352
- device.queue.submit([encoder.finish()]);
1353
- }
1354
- releaseOrTrack(recorder, scaledFused.buffer, decodeBuffers);
1355
- }
1356
-
1357
1430
  const embedDtypeResolved = selectRuleValue('inference', 'dtype', 'f16OrF32FromDtype', {
1358
1431
  dtype: perLayerEmbeddingDtype,
1359
1432
  });
1360
1433
 
1361
1434
  try {
1435
+ if (pleActivationStaticQdq) {
1436
+ projectionInputTensor = await applyPleActivationStaticQdq(
1437
+ inputEmbedsTensor,
1438
+ pleActivationStaticQdq.inputScale,
1439
+ recorder,
1440
+ decodeBuffers,
1441
+ 'Gemma 4 per-layer projection input'
1442
+ );
1443
+ }
1444
+
1445
+ if (canFuseDecodeOps) {
1446
+ let fusedProjection = null;
1447
+ let fusedProjectionForScale = null;
1448
+ let scaledFused = null;
1449
+ try {
1450
+ fusedProjection = await doMatmul(
1451
+ projectionInputTensor,
1452
+ projectionWeight,
1453
+ numTokens,
1454
+ totalPerLayerHiddenSize,
1455
+ config.hiddenSize,
1456
+ {
1457
+ transposeB: 'auto',
1458
+ label: 'per_layer_fused_projection',
1459
+ kernelPath: context.kernelPath ?? null,
1460
+ role: 'per_layer_model_projection',
1461
+ outputDtype: activationDtype,
1462
+ },
1463
+ recorder
1464
+ );
1465
+ fusedProjectionForScale = pleActivationStaticQdq
1466
+ ? await applyPleActivationStaticQdq(
1467
+ fusedProjection,
1468
+ pleActivationStaticQdq.outputScale,
1469
+ recorder,
1470
+ decodeBuffers,
1471
+ 'Gemma 4 per-layer projection output'
1472
+ )
1473
+ : fusedProjection;
1474
+ if (fusedProjectionForScale !== fusedProjection) {
1475
+ releaseOrTrack(recorder, fusedProjection.buffer, decodeBuffers);
1476
+ fusedProjection = null;
1477
+ }
1478
+
1479
+ scaledFused = recorder
1480
+ ? await recordScale(recorder, fusedProjectionForScale, projectionScale, {
1481
+ count: numTokens * totalPerLayerHiddenSize,
1482
+ })
1483
+ : await runScale(fusedProjectionForScale, projectionScale, {
1484
+ count: numTokens * totalPerLayerHiddenSize,
1485
+ });
1486
+ releaseOrTrack(recorder, fusedProjectionForScale.buffer, decodeBuffers);
1487
+ fusedProjectionForScale = null;
1488
+
1489
+ // Step 4: Extract per-layer slices via GPU buffer copies (one encoder, one submit).
1490
+ // Reuse cached slice buffers when available to avoid per-step pool churn.
1491
+ const device = getDevice();
1492
+ const sliceBytes = hiddenSizePerLayerInput * activationBytesPerElement;
1493
+ const encoder = recorder ? recorder.getEncoder() : device.createCommandEncoder();
1494
+ fusedProjectionSlices = new Array(numLayers);
1495
+ for (let l = 0; l < numLayers; l++) {
1496
+ const sliceBuf = pleCache?.sliceBuffers?.[l] ?? acquireBuffer(sliceBytes, undefined, `L${l}.per_layer_proj_slice`);
1497
+ encoder.copyBufferToBuffer(scaledFused.buffer, l * sliceBytes, sliceBuf, 0, sliceBytes);
1498
+ fusedProjectionSlices[l] = sliceBuf;
1499
+ }
1500
+ if (!recorder) {
1501
+ device.queue.submit([encoder.finish()]);
1502
+ }
1503
+ releaseOrTrack(recorder, scaledFused.buffer, decodeBuffers);
1504
+ scaledFused = null;
1505
+ } catch (error) {
1506
+ if (scaledFused) {
1507
+ releaseOrTrack(recorder, scaledFused.buffer, decodeBuffers);
1508
+ }
1509
+ if (fusedProjectionForScale && fusedProjectionForScale !== fusedProjection) {
1510
+ releaseOrTrack(recorder, fusedProjectionForScale.buffer, decodeBuffers);
1511
+ }
1512
+ if (fusedProjection) {
1513
+ releaseOrTrack(recorder, fusedProjection.buffer, decodeBuffers);
1514
+ }
1515
+ throw error;
1516
+ }
1517
+ }
1518
+
1362
1519
  for (let layerIdx = 0; layerIdx < numLayers; layerIdx++) {
1363
1520
  const hiddenOffset = layerIdx * hiddenSizePerLayerInput;
1364
1521
  const layerEmbedSource = useHotVocabularyTables
@@ -1374,6 +1531,7 @@ export async function preparePerLayerInputs(tokenIds, inputEmbedsTensor, context
1374
1531
  hiddenSize: hiddenSizePerLayerInput,
1375
1532
  vocabSize: useHotVocabularyTables ? (hotVocabularyRuntime.sentinelIndex + 1) : vocabSizePerLayerInput,
1376
1533
  scaleEmbeddings: true,
1534
+ probeStage: 'per_layer_embed_out',
1377
1535
  recorder,
1378
1536
  numTokens,
1379
1537
  indexOffset: perLayerTokenIds ? perLayerIndexOffset : indexOffset,
@@ -1390,6 +1548,7 @@ export async function preparePerLayerInputs(tokenIds, inputEmbedsTensor, context
1390
1548
  outputBuffer: canFuseDecodeOps
1391
1549
  ? (pleCache?.gatherSliceBuffers?.[layerIdx] ?? undefined)
1392
1550
  : undefined,
1551
+ stats: context.stats ?? null,
1393
1552
  });
1394
1553
 
1395
1554
  if (fusedProjectionSlices) {
@@ -1406,6 +1565,7 @@ export async function preparePerLayerInputs(tokenIds, inputEmbedsTensor, context
1406
1565
  let projectionWeightBufferForLayer = null;
1407
1566
  let projectionWeightOffset = 0;
1408
1567
  let projectedTensor = null;
1568
+ let projectionTensorForScale = null;
1409
1569
  try {
1410
1570
  const rangeBackedProjection = await loadRangeBackedPleProjectionSliceBytes(
1411
1571
  perLayerModelProjection,
@@ -1449,7 +1609,7 @@ export async function preparePerLayerInputs(tokenIds, inputEmbedsTensor, context
1449
1609
  projectionWeightOffset = hiddenOffset * config.hiddenSize * projectionWeightBytes;
1450
1610
  }
1451
1611
  projectedTensor = await doMatmul(
1452
- inputEmbedsTensor,
1612
+ projectionInputTensor,
1453
1613
  projectionWeightForLayer,
1454
1614
  numTokens,
1455
1615
  hiddenSizePerLayerInput,
@@ -1465,20 +1625,60 @@ export async function preparePerLayerInputs(tokenIds, inputEmbedsTensor, context
1465
1625
  },
1466
1626
  recorder
1467
1627
  );
1628
+ await runProbes('per_layer_projection_in', projectedTensor.buffer, {
1629
+ layerIdx,
1630
+ numTokens,
1631
+ hiddenSize: hiddenSizePerLayerInput,
1632
+ probes: context.debugProbes,
1633
+ recorder,
1634
+ operatorDiagnostics: context.operatorDiagnostics,
1635
+ dtype: activationDtype,
1636
+ });
1637
+ projectionTensorForScale = pleActivationStaticQdq
1638
+ ? await applyPleActivationStaticQdq(
1639
+ projectedTensor,
1640
+ pleActivationStaticQdq.outputScale,
1641
+ recorder,
1642
+ decodeBuffers,
1643
+ `Gemma 4 per-layer projection output L${layerIdx}`
1644
+ )
1645
+ : projectedTensor;
1646
+ if (projectionTensorForScale !== projectedTensor) {
1647
+ releaseOrTrack(recorder, projectedTensor.buffer, decodeBuffers);
1648
+ projectedTensor = null;
1649
+ }
1650
+ scaledProjectionTensor = recorder
1651
+ ? await recordScale(recorder, projectionTensorForScale, projectionScale, {
1652
+ count: numTokens * hiddenSizePerLayerInput,
1653
+ })
1654
+ : await runScale(projectionTensorForScale, projectionScale, {
1655
+ count: numTokens * hiddenSizePerLayerInput,
1656
+ });
1657
+ if (projectionTensorForScale === projectedTensor) {
1658
+ projectedTensor = null;
1659
+ }
1660
+ releaseOrTrack(recorder, projectionTensorForScale.buffer, decodeBuffers);
1661
+ projectionTensorForScale = null;
1468
1662
  } finally {
1663
+ if (projectionTensorForScale && projectionTensorForScale !== projectedTensor) {
1664
+ releaseOrTrack(recorder, projectionTensorForScale.buffer, decodeBuffers);
1665
+ }
1666
+ if (projectedTensor) {
1667
+ releaseOrTrack(recorder, projectedTensor.buffer, decodeBuffers);
1668
+ }
1469
1669
  if (projectionWeightBufferForLayer) {
1470
1670
  releaseOrTrack(recorder, projectionWeightBufferForLayer, decodeBuffers);
1471
1671
  }
1472
1672
  }
1473
- scaledProjectionTensor = recorder
1474
- ? await recordScale(recorder, projectedTensor, projectionScale, {
1475
- count: numTokens * hiddenSizePerLayerInput,
1476
- })
1477
- : await runScale(projectedTensor, projectionScale, {
1478
- count: numTokens * hiddenSizePerLayerInput,
1479
- });
1480
- releaseOrTrack(recorder, projectedTensor.buffer, decodeBuffers);
1481
- projectedTensor = null;
1673
+ await runProbes('per_layer_projection_scaled', scaledProjectionTensor.buffer, {
1674
+ layerIdx,
1675
+ numTokens,
1676
+ hiddenSize: hiddenSizePerLayerInput,
1677
+ probes: context.debugProbes,
1678
+ recorder,
1679
+ operatorDiagnostics: context.operatorDiagnostics,
1680
+ dtype: activationDtype,
1681
+ });
1482
1682
  }
1483
1683
 
1484
1684
  // Fuse the residual add into RMSNorm so Gemma 4 PLE decode avoids an
@@ -1552,6 +1752,9 @@ export async function preparePerLayerInputs(tokenIds, inputEmbedsTensor, context
1552
1752
  }
1553
1753
  throw error;
1554
1754
  } finally {
1755
+ if (projectionInputTensor !== inputEmbedsTensor) {
1756
+ releaseOrTrack(recorder, projectionInputTensor.buffer, decodeBuffers);
1757
+ }
1555
1758
  if (!usesCachedScaledProjectionNormWeight && !isGpuBufferInstance(perLayerProjectionNorm)) {
1556
1759
  releaseOrTrack(recorder, projectionNormWeight, decodeBuffers);
1557
1760
  }
@@ -1568,6 +1771,66 @@ export async function preparePerLayerInputs(tokenIds, inputEmbedsTensor, context
1568
1771
  );
1569
1772
  }
1570
1773
 
1774
+ // Prefill hot-cache seeding: when we computed a multi-token batch and pleCache is
1775
+ // provided, extract per-token row slices and store unique token IDs in the cache.
1776
+ // Copies are recorded via the active recorder (if any) so they execute after the
1777
+ // batch computation and before the first decode step reads from the cache.
1778
+ // Without a recorder, the batch buffers are already populated so copies are safe
1779
+ // to submit immediately.
1780
+ if (pleCache) {
1781
+ const plePolicy = getPleHotCachePolicy(context.perLayerInputsSession ?? null);
1782
+ if (plePolicy && plePolicy.mode === 'prepared_tokens'
1783
+ && pleCache.preparedTokenEntries instanceof Map
1784
+ && !tokenIdsAreGpuBuffer
1785
+ && numTokens > 0
1786
+ ) {
1787
+ const tokenIdArray = Array.isArray(tokenIds) ? tokenIds : Array.from(tokenIds);
1788
+ const sliceBytes = hiddenSizePerLayerInput * activationBytesPerElement;
1789
+ const seen = new Set();
1790
+ const device = getDevice();
1791
+ let pendingEncoder = null;
1792
+ for (let tokenPos = 0; tokenPos < tokenIdArray.length; tokenPos++) {
1793
+ const tid = tokenIdArray[tokenPos];
1794
+ if (seen.has(tid) || pleCache.preparedTokenEntries.has(tid)) continue;
1795
+ if (pleCache.preparedTokenEntries.size >= plePolicy.maxTokens) break;
1796
+ seen.add(tid);
1797
+ const srcOffset = tokenPos * sliceBytes;
1798
+ const sliceBuffers = new Array(numLayers).fill(null);
1799
+ let allValid = true;
1800
+ for (let layerIdx = 0; layerIdx < numLayers; layerIdx++) {
1801
+ const srcBuffer = perLayerBuffers[layerIdx];
1802
+ if (!srcBuffer || srcBuffer.size < srcOffset + sliceBytes) {
1803
+ allValid = false;
1804
+ break;
1805
+ }
1806
+ const dstBuffer = acquireBuffer(sliceBytes, undefined, `ple_seed_L${layerIdx}`);
1807
+ if (recorder) {
1808
+ recorder.getEncoder().copyBufferToBuffer(srcBuffer, srcOffset, dstBuffer, 0, sliceBytes);
1809
+ } else {
1810
+ if (!pendingEncoder) pendingEncoder = device.createCommandEncoder();
1811
+ pendingEncoder.copyBufferToBuffer(srcBuffer, srcOffset, dstBuffer, 0, sliceBytes);
1812
+ }
1813
+ sliceBuffers[layerIdx] = dstBuffer;
1814
+ }
1815
+ if (!allValid) {
1816
+ for (const buf of sliceBuffers) {
1817
+ if (buf) releaseBuffer(buf);
1818
+ }
1819
+ continue;
1820
+ }
1821
+ storePreparedTokenEntry(
1822
+ pleCache, tid, sliceBuffers,
1823
+ context.perLayerInputsSession ?? null,
1824
+ activationDtype,
1825
+ context.stats ?? null
1826
+ );
1827
+ }
1828
+ if (pendingEncoder && device) {
1829
+ device.queue.submit([pendingEncoder.finish()]);
1830
+ }
1831
+ }
1832
+ }
1833
+
1571
1834
  return perLayerBuffers;
1572
1835
  }
1573
1836
 
@@ -15,6 +15,7 @@ import { getDriftPolicyId } from './drift-policy.js';
15
15
 
16
16
  const STAGE_DEFAULT_CATEGORY = {
17
17
  embed_out: 'embed',
18
+ per_layer_embed_out: 'embed',
18
19
  // Attention stages (per-layer)
19
20
  attn_input: 'attn',
20
21
  post_input_norm: 'attn',
@@ -36,6 +37,8 @@ const STAGE_DEFAULT_CATEGORY = {
36
37
  o_proj: 'attn',
37
38
  post_attn: 'attn',
38
39
  // FFN stages (per-layer)
40
+ per_layer_projection_in: 'ffn',
41
+ per_layer_projection_scaled: 'ffn',
39
42
  per_layer_input: 'ffn',
40
43
  per_layer_input_gate: 'ffn',
41
44
  per_layer_input_activation: 'ffn',
@@ -127,6 +130,7 @@ export async function runProbes(stage, buffer, options) {
127
130
  isCpuBuffer,
128
131
  shape: [numTokens, hiddenSize],
129
132
  dtype,
133
+ recorder,
130
134
  });
131
135
  diagnostics.emitter.emitRecord(canonicalStage, {
132
136
  layerIdx,
@@ -219,7 +223,14 @@ export function getCanonicalStageName(probeStageName) {
219
223
  async function buildDiagnosticCapture(level, buffer, options) {
220
224
  if (level === 'none') return null;
221
225
 
222
- const { isCpuBuffer, shape, dtype } = options;
226
+ const { isCpuBuffer, shape, dtype, recorder } = options;
227
+ if (!isCpuBuffer && recorder) {
228
+ return createDeferredDiagnosticCapture(level, buffer, {
229
+ recorder,
230
+ shape,
231
+ dtype,
232
+ });
233
+ }
223
234
  const snapshot = isCpuBuffer
224
235
  ? snapshotFromArray(buffer, shape, dtype)
225
236
  : await snapshotTensor(buffer, shape, dtype);
@@ -244,3 +255,70 @@ async function buildDiagnosticCapture(level, buffer, options) {
244
255
  hasInf: snapshot.hasInf === true,
245
256
  };
246
257
  }
258
+
259
+ function createDeferredDiagnosticCapture(level, buffer, options) {
260
+ const { recorder, shape, dtype } = options;
261
+ const elementSize = dtype === 'f16' ? 2 : 4;
262
+ const numElements = (shape ?? []).reduce((a, b) => a * b, 1);
263
+ const readSize = numElements > 0
264
+ ? Math.min(buffer.size, numElements * elementSize)
265
+ : buffer.size;
266
+ const staging = recorder.device.createBuffer({
267
+ label: `${recorder.label}_diagnostic_capture`,
268
+ size: readSize,
269
+ usage: GPUBufferUsage.COPY_DST | GPUBufferUsage.MAP_READ,
270
+ });
271
+ recorder.getEncoder().copyBufferToBuffer(buffer, 0, staging, 0, readSize);
272
+
273
+ const capture = {
274
+ level,
275
+ shape,
276
+ dtype,
277
+ sample: null,
278
+ stats: null,
279
+ hasNaN: false,
280
+ hasInf: false,
281
+ };
282
+
283
+ recorder.enqueueCompletionTask(async () => {
284
+ let mapped = false;
285
+ try {
286
+ await staging.mapAsync(GPUMapMode.READ);
287
+ mapped = true;
288
+ const snapshot = snapshotFromArray(
289
+ decodeSnapshotBytes(staging.getMappedRange().slice(0), dtype),
290
+ shape,
291
+ dtype
292
+ );
293
+ capture.shape = snapshot.shape;
294
+ capture.dtype = snapshot.dtype;
295
+ capture.sample = Array.isArray(snapshot.sample) ? snapshot.sample : null;
296
+ capture.stats = snapshot.stats ?? null;
297
+ capture.hasNaN = snapshot.hasNaN === true;
298
+ capture.hasInf = snapshot.hasInf === true;
299
+ } catch (error) {
300
+ capture.error = error instanceof Error ? error.message : String(error);
301
+ capture.sample = null;
302
+ capture.stats = null;
303
+ } finally {
304
+ if (mapped) {
305
+ staging.unmap();
306
+ }
307
+ staging.destroy();
308
+ }
309
+ });
310
+
311
+ return capture;
312
+ }
313
+
314
+ function decodeSnapshotBytes(data, dtype) {
315
+ if (dtype === 'f16') {
316
+ const src = new Uint16Array(data);
317
+ const decoded = new Float32Array(src.length);
318
+ for (let i = 0; i < src.length; i++) {
319
+ decoded[i] = f16ToF32(src[i]);
320
+ }
321
+ return decoded;
322
+ }
323
+ return new Float32Array(data);
324
+ }
@@ -36,6 +36,7 @@ export const OPERATOR_CLASSES = Object.freeze({
36
36
  export const STAGES = Object.freeze({
37
37
  // Embedding
38
38
  EMBED_OUT: 'embed.out',
39
+ PER_LAYER_EMBED_OUT: 'per_layer_embed.out',
39
40
 
40
41
  // Attention stages (per-layer, prefixed with attn.)
41
42
  ATTN_INPUT: 'attn.input',
@@ -68,6 +69,8 @@ export const STAGES = Object.freeze({
68
69
  CONV_OUT_PROJ: 'conv.out_proj',
69
70
 
70
71
  // FFN stages (per-layer)
72
+ PER_LAYER_PROJECTION_IN: 'per_layer_projection.in',
73
+ PER_LAYER_PROJECTION_SCALED: 'per_layer_projection.scaled',
71
74
  PER_LAYER_INPUT_OUT: 'per_layer_input.out',
72
75
  FFN_NORMED: 'ffn.normed',
73
76
  FFN_IN: 'ffn.in',
@@ -100,6 +103,7 @@ export const STAGES = Object.freeze({
100
103
 
101
104
  const STAGE_TO_CLASS_MAP = Object.freeze({
102
105
  [STAGES.EMBED_OUT]: OPERATOR_CLASSES.EMBEDDING,
106
+ [STAGES.PER_LAYER_EMBED_OUT]: OPERATOR_CLASSES.EMBEDDING,
103
107
 
104
108
  [STAGES.ATTN_INPUT]: OPERATOR_CLASSES.NORMALIZATION,
105
109
  [STAGES.ATTN_POST_INPUT_NORM]: OPERATOR_CLASSES.NORMALIZATION,
@@ -128,6 +132,8 @@ const STAGE_TO_CLASS_MAP = Object.freeze({
128
132
  [STAGES.CONV_KERNEL]: OPERATOR_CLASSES.CONV,
129
133
  [STAGES.CONV_OUT_PROJ]: OPERATOR_CLASSES.CONV,
130
134
 
135
+ [STAGES.PER_LAYER_PROJECTION_IN]: OPERATOR_CLASSES.PROJECTION,
136
+ [STAGES.PER_LAYER_PROJECTION_SCALED]: OPERATOR_CLASSES.PROJECTION,
131
137
  [STAGES.PER_LAYER_INPUT_OUT]: OPERATOR_CLASSES.RESIDUAL,
132
138
  [STAGES.FFN_NORMED]: OPERATOR_CLASSES.NORMALIZATION,
133
139
  [STAGES.FFN_IN]: OPERATOR_CLASSES.FFN,
@@ -155,6 +161,7 @@ const STAGE_TO_CLASS_MAP = Object.freeze({
155
161
 
156
162
  export const PROBE_TO_CANONICAL = Object.freeze({
157
163
  embed_out: STAGES.EMBED_OUT,
164
+ per_layer_embed_out: STAGES.PER_LAYER_EMBED_OUT,
158
165
  attn_input: STAGES.ATTN_INPUT,
159
166
  post_input_norm: STAGES.ATTN_POST_INPUT_NORM,
160
167
  attn_normed: STAGES.ATTN_NORMED,
@@ -174,6 +181,8 @@ export const PROBE_TO_CANONICAL = Object.freeze({
174
181
  attn_out: STAGES.ATTN_OUT,
175
182
  o_proj: STAGES.ATTN_O_PROJ,
176
183
  post_attn: STAGES.ATTN_POST_ATTN,
184
+ per_layer_projection_in: STAGES.PER_LAYER_PROJECTION_IN,
185
+ per_layer_projection_scaled: STAGES.PER_LAYER_PROJECTION_SCALED,
177
186
  per_layer_input: STAGES.PER_LAYER_INPUT_OUT,
178
187
  ffn_normed: STAGES.FFN_NORMED,
179
188
  ffn_in: STAGES.FFN_IN,
@@ -30,6 +30,7 @@ import {
30
30
  resolveKernelPathState,
31
31
  initTokenizerFromManifest,
32
32
  } from './text/model-load.js';
33
+ import { resolvePerLayerInputsSession } from './text/generator-helpers.js';
33
34
  import { getKernelPathActivationDtype } from '../../config/kernel-path-loader.js';
34
35
  import { applyPipelineDebugConfig } from './text/debug-utils.js';
35
36
  import { resolveLayerPipeline } from './text/layer-plan.js';
@@ -491,8 +492,10 @@ export class InferencePipeline extends PipelineState {
491
492
  kernelPathSource: this.kernelPathSource,
492
493
  keepF32Weights: this.runtimeConfig.inference.compute.keepF32Weights === true,
493
494
  loaderDebug: this.runtimeConfig?.shared?.debug?.loader ?? null,
494
- perLayerInputSession:
495
- this.runtimeConfig.inference.session?.perLayerInputs ?? this.modelConfig.perLayerInputsSession,
495
+ perLayerInputSession: resolvePerLayerInputsSession(
496
+ this.modelConfig.perLayerInputsSession ?? null,
497
+ this.runtimeConfig?.inference?.session?.perLayerInputs ?? null
498
+ ),
496
499
  onProgress: (info) => {
497
500
  if (info.stage !== 'layers' && info.stage !== 'shards') {
498
501
  log.verbose('Loader', `${info.stage}: ${Math.round(info.progress * 100)}%${info.message ? ` - ${info.message}` : ''}`);
@@ -1548,6 +1551,8 @@ export class InferencePipeline extends PipelineState {
1548
1551
 
1549
1552
 
1550
1553
  async unload() {
1554
+ const storageContext = this.storageContext;
1555
+ this.storageContext = null;
1551
1556
  await destroyEmulation(this.emulation);
1552
1557
  this.emulation = null;
1553
1558
  this.decodeRing?.release();
@@ -1570,6 +1575,9 @@ export class InferencePipeline extends PipelineState {
1570
1575
  this.sampleReadbackBuffer.destroy();
1571
1576
  this.sampleReadbackBuffer = null;
1572
1577
  }
1578
+ if (typeof storageContext?.close === 'function') {
1579
+ await storageContext.close();
1580
+ }
1573
1581
  this.isLoaded = false;
1574
1582
  this.currentSeqLen = 0;
1575
1583
  restorePipelineContexts(this);