doppler-gpu 0.2.2 → 0.4.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. package/README.md +8 -0
  2. package/assets/doppler.svg +1601 -0
  3. package/package.json +6 -5
  4. package/src/cli/doppler-quickstart.js +16 -2
  5. package/src/cli/doppler-serve.js +0 -0
  6. package/src/client/doppler-registry.json +17 -1
  7. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +1 -1
  8. package/src/config/kernels/registry.json +73 -0
  9. package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +32 -0
  10. package/src/config/schema/benchmark.schema.d.ts +2 -0
  11. package/src/config/schema/benchmark.schema.js +1 -0
  12. package/src/config/schema/doppler.schema.d.ts +9 -1
  13. package/src/config/schema/doppler.schema.js +8 -1
  14. package/src/config/schema/index.d.ts +2 -0
  15. package/src/config/schema/index.js +1 -0
  16. package/src/config/schema/manifest.schema.d.ts +5 -0
  17. package/src/converter/quantizer.js +5 -4
  18. package/src/formats/rdrr/source-transform-contract.js +390 -0
  19. package/src/formats/rdrr/types.d.ts +8 -1
  20. package/src/formats/tflite/types.d.ts +8 -0
  21. package/src/formats/tflite/types.js +45 -0
  22. package/src/gpu/command-recorder.d.ts +7 -1
  23. package/src/gpu/command-recorder.js +41 -2
  24. package/src/gpu/kernels/activation-static-qdq.d.ts +22 -0
  25. package/src/gpu/kernels/activation-static-qdq.js +69 -0
  26. package/src/gpu/kernels/activation_static_qdq.wgsl +23 -0
  27. package/src/gpu/kernels/index.d.ts +7 -0
  28. package/src/gpu/kernels/index.js +6 -0
  29. package/src/inference/browser-harness-text-helpers.d.ts +1 -0
  30. package/src/inference/browser-harness-text-helpers.js +44 -4
  31. package/src/inference/browser-harness.js +7 -0
  32. package/src/inference/pipelines/context.js +4 -2
  33. package/src/inference/pipelines/text/attention/types.js +9 -4
  34. package/src/inference/pipelines/text/embed.d.ts +1 -0
  35. package/src/inference/pipelines/text/embed.js +10 -4
  36. package/src/inference/pipelines/text/generator-helpers.d.ts +6 -0
  37. package/src/inference/pipelines/text/generator-helpers.js +20 -1
  38. package/src/inference/pipelines/text/generator-steps.js +17 -8
  39. package/src/inference/pipelines/text/generator.js +51 -3
  40. package/src/inference/pipelines/text/init.d.ts +3 -0
  41. package/src/inference/pipelines/text/init.js +24 -3
  42. package/src/inference/pipelines/text/logits/gpu.d.ts +5 -4
  43. package/src/inference/pipelines/text/logits/gpu.js +146 -11
  44. package/src/inference/pipelines/text/per-layer-inputs.js +320 -57
  45. package/src/inference/pipelines/text/probes.js +79 -1
  46. package/src/inference/pipelines/text/stage-names.js +9 -0
  47. package/src/inference/pipelines/text.js +10 -2
  48. package/src/loader/embedding-loader.js +20 -5
  49. package/src/loader/loader-types.d.ts +2 -0
  50. package/src/loader/manifest-config.d.ts +2 -7
  51. package/src/loader/per-layer-input-loader.js +114 -1
  52. package/src/loader/tensors/source-transform.d.ts +15 -0
  53. package/src/loader/tensors/source-transform.js +312 -34
  54. package/src/loader/tensors/tensor-reader.js +28 -2
  55. package/src/storage/artifact-storage-context.d.ts +1 -0
  56. package/src/storage/artifact-storage-context.js +158 -72
  57. package/src/storage/preflight.js +11 -0
  58. package/src/storage/quickstart-downloader.js +15 -4
  59. package/src/tooling/litert-package-runtime.js +369 -14
  60. package/src/tooling/node-source-runtime.js +256 -140
  61. package/src/tooling/source-runtime-bundle.d.ts +25 -0
  62. package/src/tooling/source-runtime-bundle.js +53 -1
  63. package/src/tooling/source-runtime-materializer.js +3 -1
@@ -0,0 +1,23 @@
1
+ struct Uniforms {
2
+ size: u32,
3
+ _pad0: u32,
4
+ scale: f32,
5
+ invScale: f32,
6
+ qmin: f32,
7
+ qmax: f32,
8
+ }
9
+
10
+ @group(0) @binding(0) var<uniform> u: Uniforms;
11
+ @group(0) @binding(1) var<storage, read> input: array<f32>;
12
+ @group(0) @binding(2) var<storage, read_write> output: array<f32>;
13
+
14
+ @compute @workgroup_size(256, 1, 1)
15
+ fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
16
+ let idx = gid.x;
17
+ if (idx >= u.size) {
18
+ return;
19
+ }
20
+
21
+ let quantized = clamp(round(input[idx] * u.invScale), u.qmin, u.qmax);
22
+ output[idx] = quantized * u.scale;
23
+ }
@@ -149,6 +149,13 @@ export {
149
149
  type ClampOptions,
150
150
  } from './clamp.js';
151
151
 
152
+ // Static activation quantize/dequantize
153
+ export {
154
+ runActivationStaticQdq,
155
+ recordActivationStaticQdq,
156
+ type ActivationStaticQdqOptions,
157
+ } from './activation-static-qdq.js';
158
+
152
159
  // Energy (EBM helpers)
153
160
  export {
154
161
  runEnergyEval,
@@ -120,6 +120,12 @@ export {
120
120
  recordClamp,
121
121
  } from './clamp.js';
122
122
 
123
+ // Static activation quantize/dequantize
124
+ export {
125
+ runActivationStaticQdq,
126
+ recordActivationStaticQdq,
127
+ } from './activation-static-qdq.js';
128
+
123
129
  // Energy (EBM helpers)
124
130
  export {
125
131
  runEnergyEval,
@@ -16,6 +16,7 @@ export declare function resolveBenchmarkRunSettings(
16
16
  promptLabel: string;
17
17
  maxTokens: number;
18
18
  sampling: Record<string, unknown>;
19
+ seed?: number;
19
20
  };
20
21
  export declare function runEmbeddingSemanticChecks(
21
22
  pipeline: Record<string, unknown>,
@@ -662,23 +662,42 @@ export function resolveBenchmarkRunSettings(runtimeConfig, source = null) {
662
662
  const benchSampling = isPlainObject(benchConfig?.sampling)
663
663
  ? benchConfig.sampling
664
664
  : {};
665
+ const runSeed = Number.isFinite(benchConfig.seed)
666
+ ? Math.max(0, Math.floor(benchConfig.seed))
667
+ : null;
668
+ const runtimeSeed = Number.isFinite(runtimeSampling.seed)
669
+ ? Math.max(0, Math.floor(runtimeSampling.seed))
670
+ : null;
671
+ const benchSeed = Number.isFinite(benchSampling.seed)
672
+ ? Math.max(0, Math.floor(benchSampling.seed))
673
+ : null;
674
+ const mergedSeed = runSeed != null
675
+ ? runSeed
676
+ : benchSeed != null
677
+ ? benchSeed
678
+ : runtimeSeed;
665
679
  const promptInput = typeof benchConfig.customPrompt === 'string' && benchConfig.customPrompt.trim()
666
680
  ? benchConfig.customPrompt.trim()
667
681
  : resolveGenerationPromptInput(runtimeConfig, null, source);
668
682
  const maxTokens = Number.isFinite(benchConfig.maxNewTokens)
669
683
  ? Math.max(1, Math.floor(benchConfig.maxNewTokens))
670
684
  : resolveMaxTokens(runtimeConfig);
685
+ const sampling = {
686
+ ...runtimeSampling,
687
+ ...benchSampling,
688
+ };
689
+ if (Number.isFinite(mergedSeed)) {
690
+ sampling.seed = mergedSeed;
691
+ }
671
692
 
672
693
  return {
673
694
  warmupRuns: Math.max(0, Math.floor(benchConfig.warmupRuns ?? 0)),
674
695
  timedRuns: Math.max(1, Math.floor(benchConfig.timedRuns ?? 1)),
696
+ ...(Number.isFinite(mergedSeed) ? { seed: mergedSeed } : {}),
675
697
  prompt: promptInput,
676
698
  promptLabel: describePromptInput(promptInput),
677
699
  maxTokens,
678
- sampling: {
679
- ...runtimeSampling,
680
- ...benchSampling,
681
- },
700
+ sampling,
682
701
  };
683
702
  }
684
703
 
@@ -1015,6 +1034,14 @@ function buildGenerationPhaseFromStats(pipeline, durationMs, tokenCount) {
1015
1034
  if (Number.isFinite(stats.decodeRecordMs)) gpu.decodeRecordMs = stats.decodeRecordMs;
1016
1035
  if (Number.isFinite(stats.decodeSubmitWaitMs)) gpu.decodeSubmitWaitMs = stats.decodeSubmitWaitMs;
1017
1036
  if (Number.isFinite(stats.decodeReadbackWaitMs)) gpu.decodeReadbackWaitMs = stats.decodeReadbackWaitMs;
1037
+ if (
1038
+ Number.isFinite(decodeMs) &&
1039
+ Number.isFinite(stats.decodeRecordMs) &&
1040
+ Number.isFinite(stats.decodeSubmitWaitMs) &&
1041
+ Number.isFinite(stats.decodeReadbackWaitMs)
1042
+ ) {
1043
+ gpu.decodeOrchestrationMs = decodeMs - stats.decodeRecordMs - stats.decodeSubmitWaitMs - stats.decodeReadbackWaitMs;
1044
+ }
1018
1045
  if (Number.isFinite(stats.singleTokenSubmitWaitMs)) gpu.singleTokenSubmitWaitMs = stats.singleTokenSubmitWaitMs;
1019
1046
  if (Number.isFinite(stats.singleTokenReadbackWaitMs)) gpu.singleTokenReadbackWaitMs = stats.singleTokenReadbackWaitMs;
1020
1047
  if (Number.isFinite(stats.singleTokenOrchestrationMs)) gpu.singleTokenOrchestrationMs = stats.singleTokenOrchestrationMs;
@@ -1049,6 +1076,12 @@ function buildGenerationPhaseFromStats(pipeline, durationMs, tokenCount) {
1049
1076
  if (Number.isFinite(stats.plePreparedTokenCacheBytes)) {
1050
1077
  plePreparedTokenCache.bytes = stats.plePreparedTokenCacheBytes;
1051
1078
  }
1079
+ if (Number.isFinite(stats.pleWriteBufferCount)) {
1080
+ plePreparedTokenCache.writeBufferCount = stats.pleWriteBufferCount;
1081
+ }
1082
+ if (Number.isFinite(stats.pleWriteBufferBytes)) {
1083
+ plePreparedTokenCache.writeBufferBytes = stats.pleWriteBufferBytes;
1084
+ }
1052
1085
  const plePreparedTokenCachePhase = Object.keys(plePreparedTokenCache).length > 0
1053
1086
  ? plePreparedTokenCache
1054
1087
  : null;
@@ -1112,6 +1145,11 @@ export async function runGeneration(pipeline, runtimeConfig, runOverrides = null
1112
1145
  ...(runtimeConfig.inference?.sampling || {}),
1113
1146
  ...(isPlainObject(runOverrides?.sampling) ? runOverrides.sampling : {}),
1114
1147
  };
1148
+ const seed = Number.isFinite(runOverrides?.seed)
1149
+ ? Math.max(0, Math.floor(runOverrides.seed))
1150
+ : Number.isFinite(sampling.seed)
1151
+ ? Math.max(0, Math.floor(sampling.seed))
1152
+ : null;
1115
1153
  const debugProbes = runtimeConfig.shared?.debug?.probes || [];
1116
1154
  const profile = runtimeConfig.shared?.debug?.profiler?.enabled === true;
1117
1155
  const explicitDiagnosticsEnabled = runtimeConfig.shared?.harness?.mode === 'diagnose'
@@ -1123,6 +1161,7 @@ export async function runGeneration(pipeline, runtimeConfig, runOverrides = null
1123
1161
 
1124
1162
  for await (const tokenText of pipeline.generate(promptInput, {
1125
1163
  maxTokens,
1164
+ ...(Number.isFinite(seed) ? { seed } : {}),
1126
1165
  temperature: sampling.temperature,
1127
1166
  topP: sampling.topP,
1128
1167
  topK: sampling.topK,
@@ -1151,6 +1190,7 @@ export async function runGeneration(pipeline, runtimeConfig, runOverrides = null
1151
1190
  const { phase } = buildGenerationPhaseFromStats(pipeline, durationMs, tokenIds.length);
1152
1191
 
1153
1192
  return {
1193
+ ...(Number.isFinite(seed) ? { seed } : {}),
1154
1194
  prompt: promptLabel,
1155
1195
  promptInput,
1156
1196
  maxTokens,
@@ -775,6 +775,7 @@ async function runBenchSuite(options = {}) {
775
775
  const gpuDecodeRecordMs = [];
776
776
  const gpuDecodeSubmitWaitMs = [];
777
777
  const gpuDecodeReadbackWaitMs = [];
778
+ const gpuDecodeOrchestrationMs = [];
778
779
  const singleTokenSubmitWaitMs = [];
779
780
  const singleTokenReadbackWaitMs = [];
780
781
  const singleTokenOrchestrationMs = [];
@@ -829,6 +830,9 @@ async function runBenchSuite(options = {}) {
829
830
  if (Number.isFinite(phaseGpu?.decodeRecordMs)) gpuDecodeRecordMs.push(phaseGpu.decodeRecordMs);
830
831
  if (Number.isFinite(phaseGpu?.decodeSubmitWaitMs)) gpuDecodeSubmitWaitMs.push(phaseGpu.decodeSubmitWaitMs);
831
832
  if (Number.isFinite(phaseGpu?.decodeReadbackWaitMs)) gpuDecodeReadbackWaitMs.push(phaseGpu.decodeReadbackWaitMs);
833
+ if (Number.isFinite(phaseGpu?.decodeOrchestrationMs)) {
834
+ gpuDecodeOrchestrationMs.push(phaseGpu.decodeOrchestrationMs);
835
+ }
832
836
  if (Number.isFinite(phaseGpu?.singleTokenSubmitWaitMs)) singleTokenSubmitWaitMs.push(phaseGpu.singleTokenSubmitWaitMs);
833
837
  if (Number.isFinite(phaseGpu?.singleTokenReadbackWaitMs)) singleTokenReadbackWaitMs.push(phaseGpu.singleTokenReadbackWaitMs);
834
838
  if (Number.isFinite(phaseGpu?.singleTokenOrchestrationMs)) singleTokenOrchestrationMs.push(phaseGpu.singleTokenOrchestrationMs);
@@ -858,6 +862,7 @@ async function runBenchSuite(options = {}) {
858
862
  const decodeTokensStats = computeSampleStats(decodeTokens);
859
863
  const hasGpuStats = gpuPrefillMs.length > 0 || gpuDecodeMs.length > 0 || gpuDecodeRecordMs.length > 0
860
864
  || gpuDecodeSubmitWaitMs.length > 0 || gpuDecodeReadbackWaitMs.length > 0
865
+ || gpuDecodeOrchestrationMs.length > 0
861
866
  || singleTokenSubmitWaitMs.length > 0 || singleTokenReadbackWaitMs.length > 0
862
867
  || singleTokenOrchestrationMs.length > 0;
863
868
  const gpuPhaseStats = hasGpuStats
@@ -867,6 +872,7 @@ async function runBenchSuite(options = {}) {
867
872
  decodeRecordMs: computeSampleStats(gpuDecodeRecordMs),
868
873
  decodeSubmitWaitMs: computeSampleStats(gpuDecodeSubmitWaitMs),
869
874
  decodeReadbackWaitMs: computeSampleStats(gpuDecodeReadbackWaitMs),
875
+ decodeOrchestrationMs: computeSampleStats(gpuDecodeOrchestrationMs),
870
876
  singleTokenSubmitWaitMs: computeSampleStats(singleTokenSubmitWaitMs),
871
877
  singleTokenReadbackWaitMs: computeSampleStats(singleTokenReadbackWaitMs),
872
878
  singleTokenOrchestrationMs: computeSampleStats(singleTokenOrchestrationMs),
@@ -913,6 +919,7 @@ async function runBenchSuite(options = {}) {
913
919
  metrics = {
914
920
  warmupRuns,
915
921
  timedRuns,
922
+ ...(Number.isFinite(benchRun?.seed) ? { seed: benchRun.seed } : {}),
916
923
  prompt: benchRun.promptLabel,
917
924
  maxTokens: benchRun.maxTokens,
918
925
  decodeTokensPerSec: sampleTimingNumber(decodeTokensPerSecStats, 'median'),
@@ -145,8 +145,10 @@ export function applyPipelineContexts(target, contexts = {}, options = {}) {
145
145
  if (options.assignMemoryContext && contexts.memory) {
146
146
  target.memoryContext = contexts.memory;
147
147
  }
148
- if (options.assignStorageContext && contexts.storage) {
149
- target.storageContext = contexts.storage;
148
+
149
+ const requestedStorageContext = contexts.storage ?? contexts.storageContext;
150
+ if (options.assignStorageContext && requestedStorageContext) {
151
+ target.storageContext = requestedStorageContext;
150
152
  }
151
153
  if (options.assignBaseUrl !== false && contexts.baseUrl) {
152
154
  target.baseUrl = contexts.baseUrl;
@@ -44,16 +44,21 @@ export function releaseOrTrack(recorder, buffer) {
44
44
  // ============================================================================
45
45
 
46
46
 
47
- const qkNormOnesCache = new Map();
47
+ const qkNormOnesCache = new WeakMap();
48
48
 
49
49
 
50
50
  export function getQKNormOnesBuffer(headDim) {
51
- const cached = qkNormOnesCache.get(headDim);
52
- if (cached) return cached;
53
51
  const device = getDevice();
54
52
  if (!device) {
55
53
  throw new Error('No GPU device available for Q/K norm buffer');
56
54
  }
55
+ let perDeviceCache = qkNormOnesCache.get(device);
56
+ if (!perDeviceCache) {
57
+ perDeviceCache = new Map();
58
+ qkNormOnesCache.set(device, perDeviceCache);
59
+ }
60
+ const cached = perDeviceCache.get(headDim);
61
+ if (cached) return cached;
57
62
  const data = new Float32Array(headDim);
58
63
  data.fill(1);
59
64
  const buffer = device.createBuffer({
@@ -62,6 +67,6 @@ export function getQKNormOnesBuffer(headDim) {
62
67
  usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST,
63
68
  });
64
69
  device.queue.writeBuffer(buffer, 0, data);
65
- qkNormOnesCache.set(headDim, buffer);
70
+ perDeviceCache.set(headDim, buffer);
66
71
  return buffer;
67
72
  }
@@ -22,6 +22,7 @@ export interface EmbedConfig {
22
22
  activationDtype?: 'f16' | 'f32';
23
23
  embeddingDtype?: 'f16' | 'f32';
24
24
  executionPolicies?: ExecutionV1PoliciesSchema | null;
25
+ probeStage?: string;
25
26
  inputHiddenSize?: number;
26
27
  hiddenOffset?: number;
27
28
  }
@@ -84,8 +84,10 @@ export async function embed(tokenIds, embedBuffer, config) {
84
84
  activationDtype,
85
85
  embeddingDtype,
86
86
  operatorDiagnostics,
87
+ probeStage = 'embed_out',
87
88
  inputHiddenSize = hiddenSize,
88
89
  hiddenOffset = 0,
90
+ stats = null,
89
91
  } = config;
90
92
  const device = getDevice();
91
93
  const tokenBufferInput = isGpuBufferInstance(tokenIds);
@@ -274,7 +276,7 @@ export async function embed(tokenIds, embedBuffer, config) {
274
276
  } else {
275
277
  releaseBuffer(f32Buffer);
276
278
  }
277
- await runProbes('embed_out', f16Tensor.buffer, {
279
+ await runProbes(probeStage, f16Tensor.buffer, {
278
280
  numTokens,
279
281
  hiddenSize,
280
282
  probes: config.debugProbes,
@@ -290,7 +292,11 @@ export async function embed(tokenIds, embedBuffer, config) {
290
292
  ? preAllocatedOutput
291
293
  : acquireBuffer(outputBytes, undefined, 'embed_cpu_f32_out');
292
294
  device.queue.writeBuffer(outputBuffer, 0, output);
293
- await runProbes('embed_out', outputBuffer, {
295
+ if (stats) {
296
+ stats.pleWriteBufferCount = (stats.pleWriteBufferCount ?? 0) + 1;
297
+ stats.pleWriteBufferBytes = (stats.pleWriteBufferBytes ?? 0) + outputBytes;
298
+ }
299
+ await runProbes(probeStage, outputBuffer, {
294
300
  numTokens,
295
301
  hiddenSize,
296
302
  probes: config.debugProbes,
@@ -360,7 +366,7 @@ export async function embed(tokenIds, embedBuffer, config) {
360
366
  }
361
367
 
362
368
  if (!scaleEmbeddings) {
363
- await runProbes('embed_out', gatherOutput.buffer, {
369
+ await runProbes(probeStage, gatherOutput.buffer, {
364
370
  numTokens,
365
371
  hiddenSize,
366
372
  probes: config.debugProbes,
@@ -431,7 +437,7 @@ export async function embed(tokenIds, embedBuffer, config) {
431
437
  throw new Error('[Embed] Scaled embedding contains NaN/Inf');
432
438
  }
433
439
  }
434
- await runProbes('embed_out', scaledBuffer, {
440
+ await runProbes(probeStage, scaledBuffer, {
435
441
  numTokens,
436
442
  hiddenSize,
437
443
  probes: config.debugProbes,
@@ -9,6 +9,7 @@ import type { LayerContext } from './types.js';
9
9
  import type { LogitsConfig, LogitsWeights } from './logits/index.js';
10
10
  import type { WeightBufferConfig } from './weights.js';
11
11
  import type { ExecutionSessionPlan } from './execution-plan.js';
12
+ import type { ExecutionV1PerLayerInputsSessionSchema } from '../../../config/schema/execution-v1.schema.js';
12
13
 
13
14
  export declare function debugCheckBuffer(
14
15
  state: PipelineState,
@@ -27,6 +28,11 @@ export declare function buildLayerContext(
27
28
  executionPlan?: ExecutionSessionPlan | null
28
29
  ): LayerContext;
29
30
 
31
+ export declare function resolvePerLayerInputsSession(
32
+ manifestSession: ExecutionV1PerLayerInputsSessionSchema | null | undefined,
33
+ runtimeSession: ExecutionV1PerLayerInputsSessionSchema | Record<string, unknown> | null | undefined
34
+ ): ExecutionV1PerLayerInputsSessionSchema | null;
35
+
30
36
  export declare function releaseSharedAttentionState(
31
37
  sharedAttentionState: LayerContext['sharedAttentionState'],
32
38
  recorder?: {
@@ -2,11 +2,26 @@
2
2
  import { readBuffer, releaseBuffer } from '../../../memory/buffer-pool.js';
3
3
  import { allowReadback } from '../../../gpu/perf-guards.js';
4
4
  import { log } from '../../../debug/index.js';
5
+ import { mergeRuntimeValues } from '../../../config/runtime-merge.js';
5
6
  import { selectRuleValue } from '../../../rules/rule-registry.js';
6
7
  import { decodeReadback } from './debug-utils/index.js';
7
8
  import { isWeightBuffer, isCpuWeightBuffer, isGpuBufferInstance } from '../../../gpu/weight-buffer.js';
8
9
  import { resolveRangeAwareSelectiveWideningConfig } from './finiteness-policy.js';
9
10
  import { resolveActiveExecutionPlan } from './execution-plan.js';
11
+
12
+ function isPlainObject(value) {
13
+ return value != null && typeof value === 'object' && !Array.isArray(value);
14
+ }
15
+
16
+ export function resolvePerLayerInputsSession(manifestSession, runtimeSession) {
17
+ if (!isPlainObject(runtimeSession)) {
18
+ return runtimeSession ?? (manifestSession ?? null);
19
+ }
20
+ if (!isPlainObject(manifestSession)) {
21
+ return runtimeSession;
22
+ }
23
+ return mergeRuntimeValues(manifestSession, runtimeSession);
24
+ }
10
25
  export async function debugCheckBuffer(state, buffer, label, numTokens, expectedDim) {
11
26
  if (!allowReadback(`pipeline.debug.${label}`)) return;
12
27
 
@@ -83,6 +98,10 @@ export function buildLayerContext(state, recorder, isDecodeMode, debugLayers, de
83
98
  const activeExecutionPlan = executionPlan ?? resolveActiveExecutionPlan(state);
84
99
  const activeKernelPath = activeExecutionPlan.kernelPath ?? state.resolvedKernelPath ?? null;
85
100
  const effectiveActivationDtype = activeExecutionPlan.activationDtype;
101
+ const perLayerInputsSession = resolvePerLayerInputsSession(
102
+ config.perLayerInputsSession ?? null,
103
+ state.runtimeConfig?.inference?.session?.perLayerInputs ?? null
104
+ );
86
105
  const effectiveComputeConfig = {
87
106
  ...computeConfig,
88
107
  activationDtype: effectiveActivationDtype,
@@ -122,7 +141,7 @@ export function buildLayerContext(state, recorder, isDecodeMode, debugLayers, de
122
141
  debugProbes: state.runtimeConfig.shared.debug.probes,
123
142
  debugCheckBuffer: debugCheckBufferFn,
124
143
  perLayerInputBuffer: null,
125
- perLayerInputsSession: state.runtimeConfig.inference.session?.perLayerInputs ?? config.perLayerInputsSession,
144
+ perLayerInputsSession,
126
145
  pipelinePlan: state.layerPipelinePlan,
127
146
  expertWeights: state.expertWeights,
128
147
  expertLoader: state.dopplerLoader,
@@ -19,6 +19,7 @@ import {
19
19
  import { sample, applyRepetitionPenalty, logitsSanity, getTopK } from './sampling.js';
20
20
  import { isStopToken } from './init.js';
21
21
  import { embed } from './embed.js';
22
+ import { resolvePerLayerInputsSession } from './generator-helpers.js';
22
23
  import { processLayer } from './layer.js';
23
24
  import { computeLogits, computeLogitsGPU, recordLogitsGPU, extractLastPositionLogits, finalizeLogits, applySoftcapping } from './logits/index.js';
24
25
  import { isWeightBuffer, isCpuWeightBuffer, isGpuBufferInstance, getWeightDtype } from '../../../gpu/weight-buffer.js';
@@ -111,12 +112,15 @@ function schedulePlePrefetchForToken(state, tokenId) {
111
112
  if (!pleWeights?.embedTokensPerLayer) {
112
113
  return;
113
114
  }
114
- const perLayerInputsSession = state.runtimeConfig.inference.session?.perLayerInputs ?? config.perLayerInputsSession;
115
+ const resolvedPerLayerInputsSession = resolvePerLayerInputsSession(
116
+ config.perLayerInputsSession ?? null,
117
+ state.runtimeConfig?.inference?.session?.perLayerInputs ?? null
118
+ );
115
119
  state.plePrefetchPending = prefetchPerLayerRow(
116
120
  tokenId,
117
121
  pleWeights.embedTokensPerLayer,
118
122
  config.numLayers * pleHiddenSize,
119
- perLayerInputsSession
123
+ resolvedPerLayerInputsSession
120
124
  );
121
125
  }
122
126
 
@@ -217,7 +221,7 @@ export async function readSampledTokenFromStagingBuffer(stagingBuffer, options =
217
221
  try {
218
222
  await stagingBuffer.mapAsync(GPUMapMode.READ);
219
223
  mapped = true;
220
- cleanupRecorder?.completeDeferredCleanup();
224
+ await cleanupRecorder?.completeDeferredCleanup();
221
225
  cleanupCompleted = true;
222
226
  const mappedWords = new Uint32Array(stagingBuffer.getMappedRange());
223
227
  return {
@@ -231,7 +235,7 @@ export async function readSampledTokenFromStagingBuffer(stagingBuffer, options =
231
235
  stagingBuffer.unmap();
232
236
  }
233
237
  if (!cleanupCompleted) {
234
- cleanupRecorder?.completeDeferredCleanup({ discardPooled: true });
238
+ await cleanupRecorder?.completeDeferredCleanup({ discardPooled: true });
235
239
  }
236
240
  if (ownsStagingBuffer) {
237
241
  stagingBuffer.destroy();
@@ -293,7 +297,7 @@ export async function readBatchTokensFromStagingBuffers(options) {
293
297
  if (mapFailure) {
294
298
  throw mapFailure.reason;
295
299
  }
296
- cleanupRecorder?.completeDeferredCleanup();
300
+ await cleanupRecorder?.completeDeferredCleanup();
297
301
  cleanupCompleted = true;
298
302
 
299
303
  const tokens = Array.from(
@@ -322,7 +326,7 @@ export async function readBatchTokensFromStagingBuffers(options) {
322
326
  stopStagingBuffer.unmap();
323
327
  }
324
328
  if (!cleanupCompleted) {
325
- cleanupRecorder?.completeDeferredCleanup({ discardPooled: true });
329
+ await cleanupRecorder?.completeDeferredCleanup({ discardPooled: true });
326
330
  }
327
331
  if (ownsFinitenessStaging) {
328
332
  finitenessStagingBuffer.destroy();
@@ -1135,7 +1139,10 @@ async function generateNTokensGPUStepwiseRangeBackedPle(state, N, currentIds, op
1135
1139
  ? config.numLayers * pleHiddenSize
1136
1140
  : 0;
1137
1141
  const pleWeights = state.weights.get('per_layer_inputs');
1138
- const pleSession = state.runtimeConfig.inference.session?.perLayerInputs ?? config.perLayerInputsSession;
1142
+ const resolvedPleSession = resolvePerLayerInputsSession(
1143
+ config.perLayerInputsSession ?? null,
1144
+ state.runtimeConfig?.inference?.session?.perLayerInputs ?? null
1145
+ );
1139
1146
  const generatedTokens = [];
1140
1147
  const rollingIds = Array.isArray(currentIds) ? currentIds.slice() : Array.from(currentIds ?? []);
1141
1148
  let gpuSubmissions = 0;
@@ -1152,7 +1159,7 @@ async function generateNTokensGPUStepwiseRangeBackedPle(state, N, currentIds, op
1152
1159
  rollingIds[rollingIds.length - 1],
1153
1160
  pleWeights.embedTokensPerLayer,
1154
1161
  totalPerLayerHiddenSize,
1155
- pleSession
1162
+ resolvedPleSession
1156
1163
  );
1157
1164
  }
1158
1165
 
@@ -1371,6 +1378,7 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
1371
1378
  const currentTokenIdsArray = [startToken];
1372
1379
 
1373
1380
  for (let i = 0; i < N; i++) {
1381
+ currentTokenIdsArray[0] = rollingIds[i];
1374
1382
  const currentPos = state.currentSeqLen + i;
1375
1383
  context.currentSeqLen = currentPos;
1376
1384
  context.currentTokenIds = currentTokenIdsArray;
@@ -1397,6 +1405,7 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
1397
1405
  indexOffset: i,
1398
1406
  perLayerTokenIds: pleInputTokensBuffer,
1399
1407
  perLayerIndexOffset: i,
1408
+ tokenIdHint: Number.isInteger(rollingIds[i]) ? rollingIds[i] : null,
1400
1409
  pleCache: state.pleCache ?? null,
1401
1410
  });
1402
1411
  try {
@@ -43,6 +43,7 @@ import {
43
43
  debugCheckBuffer as debugCheckBufferHelper,
44
44
  getLogitsConfig,
45
45
  getLogitsWeights,
46
+ resolvePerLayerInputsSession,
46
47
  releaseSharedAttentionState,
47
48
  } from './generator-helpers.js';
48
49
  import {
@@ -135,10 +136,24 @@ export function resolveHotVocabularyBatchDecodeAvailability({
135
136
  }
136
137
 
137
138
  async function primePleDecodeRuntimeCache(state, seedTokenIds = null) {
139
+ const perLayerInputsSession = resolvePerLayerInputsSession(
140
+ state.modelConfig?.perLayerInputsSession ?? null,
141
+ state.runtimeConfig?.inference?.session?.perLayerInputs ?? null
142
+ );
143
+ const materialization = perLayerInputsSession?.materialization;
144
+ if (state.debug) {
145
+ log.debug(
146
+ 'Pipeline',
147
+ `PLE materialization session=${materialization ?? 'null'} ` +
148
+ `modelId=${state.modelConfig?.modelId ?? 'unknown'} ` +
149
+ `runtimePerLayerInputs=${Boolean(state.runtimeConfig.inference.session?.perLayerInputs)} ` +
150
+ `manifestPerLayerInputs=${Boolean(state.modelConfig?.perLayerInputsSession)}`
151
+ );
152
+ }
138
153
  await ensurePleGpuHotVocabularyRuntime({
139
154
  config: state.modelConfig,
140
155
  weights: state.weights,
141
- perLayerInputsSession: state.runtimeConfig.inference.session?.perLayerInputs ?? state.modelConfig?.perLayerInputsSession ?? null,
156
+ perLayerInputsSession,
142
157
  debugFlags: state.debugFlags,
143
158
  tokenizer: state.tokenizer ?? null,
144
159
  seedTokenIds: Array.isArray(seedTokenIds) ? seedTokenIds : null,
@@ -146,7 +161,7 @@ async function primePleDecodeRuntimeCache(state, seedTokenIds = null) {
146
161
  await ensurePleGpuSplitTablesRuntime({
147
162
  config: state.modelConfig,
148
163
  weights: state.weights,
149
- perLayerInputsSession: state.runtimeConfig.inference.session?.perLayerInputs ?? state.modelConfig?.perLayerInputsSession ?? null,
164
+ perLayerInputsSession,
150
165
  debugFlags: state.debugFlags,
151
166
  });
152
167
  await ensurePleScaledProjectionNormWeight({
@@ -1304,6 +1319,26 @@ export class PipelineGenerator {
1304
1319
  opts.topP = samplingConfig.topP;
1305
1320
  opts.topK = samplingConfig.topK;
1306
1321
  opts.repetitionPenalty = samplingConfig.repetitionPenalty;
1322
+ const diagnosticsEnabled = options?.diagnostics?.enabled === true
1323
+ || this.#state.runtimeConfig?.shared?.harness?.mode === 'diagnose';
1324
+ if (diagnosticsEnabled) {
1325
+ const captureConfig = {
1326
+ ...createDefaultCaptureConfig(),
1327
+ enabled: true,
1328
+ defaultLevel: CAPTURE_LEVELS.SLICE,
1329
+ ...(options?.diagnostics?.captureConfig ?? {}),
1330
+ };
1331
+ validateCaptureConfig(captureConfig);
1332
+ this.#state.operatorDiagnostics = {
1333
+ enabled: true,
1334
+ captureConfig,
1335
+ emitter: new OperatorEventEmitter({
1336
+ modelHash: this.#state.manifest?.modelId ?? null,
1337
+ runtimeConfigHash: this.#state.resolvedKernelPath?.id ?? null,
1338
+ executionPlanHash: opts.executionPlan?.id ?? null,
1339
+ }),
1340
+ };
1341
+ }
1307
1342
 
1308
1343
  try {
1309
1344
  const prefillStartSeqLen = this.#state.currentSeqLen;
@@ -1370,6 +1405,14 @@ export class PipelineGenerator {
1370
1405
  this.#state.stats.executionPlan.finalActivePlanId = this.#state.executionPlanState?.activePlanId ?? null;
1371
1406
  }
1372
1407
  resetActiveExecutionPlan(this.#state);
1408
+ this.#state.stats.operatorDiagnostics = this.#state.operatorDiagnostics?.emitter
1409
+ ? {
1410
+ enabled: true,
1411
+ timeline: this.#state.operatorDiagnostics.emitter.getTimeline(),
1412
+ recordCount: this.#state.operatorDiagnostics.emitter.length,
1413
+ }
1414
+ : null;
1415
+ this.#state.operatorDiagnostics = null;
1373
1416
  this.#state.isGenerating = false;
1374
1417
  }
1375
1418
  }
@@ -1729,6 +1772,10 @@ export class PipelineGenerator {
1729
1772
  }
1730
1773
 
1731
1774
  const decodeStart = performance.now();
1775
+ const resolvedPerLayerInputsSession = resolvePerLayerInputsSession(
1776
+ this.#state.modelConfig.perLayerInputsSession ?? null,
1777
+ this.#state.runtimeConfig?.inference?.session?.perLayerInputs ?? null
1778
+ );
1732
1779
  const lmHead = this.#state.weights.get('lm_head');
1733
1780
  const embedBuffer = this.#state.weights.get('embed');
1734
1781
  const hasCpuWeights = isCpuWeightBuffer(lmHead)
@@ -2020,7 +2067,7 @@ export class PipelineGenerator {
2020
2067
  nextToken,
2021
2068
  pleWeights.embedTokensPerLayer,
2022
2069
  this.#state.modelConfig.numLayers * pleHiddenSize,
2023
- this.#state.runtimeConfig.inference.session?.perLayerInputs ?? this.#state.modelConfig.perLayerInputsSession,
2070
+ resolvedPerLayerInputsSession,
2024
2071
  );
2025
2072
  }
2026
2073
  }
@@ -2214,6 +2261,7 @@ export class PipelineGenerator {
2214
2261
  context,
2215
2262
  {
2216
2263
  numTokens,
2264
+ pleCache: this.#state.pleCache ?? null,
2217
2265
  }
2218
2266
  );
2219
2267
  } catch (error) {
@@ -57,6 +57,7 @@ export interface PipelineStorageContext {
57
57
  loadTokenizerJson?: () => Promise<Record<string, unknown> | string | null | undefined>;
58
58
  loadTokenizerModel?: (path?: string) => Promise<ArrayBuffer | Uint8Array | null | undefined>;
59
59
  verifyHashes?: boolean;
60
+ close?: () => Promise<void>;
60
61
  }
61
62
 
62
63
  export function createNodeFileShardStorageContext(
@@ -74,6 +75,8 @@ export interface PipelineContexts {
74
75
  memory?: Record<string, unknown>;
75
76
  /** Storage context for custom shard loading */
76
77
  storage?: PipelineStorageContext;
78
+ /** Storage context alias used by several public call sites */
79
+ storageContext?: PipelineStorageContext;
77
80
  /** Base URL for loading model files */
78
81
  baseUrl?: string;
79
82
  /** Full runtime config (merged with defaults) */
@@ -2,7 +2,7 @@
2
2
 
3
3
  import { parseModelConfig } from './config.js';
4
4
  import { getDevice, getDeviceLimits, getKernelCapabilities } from '../../../gpu/device.js';
5
- import { acquireBuffer, releaseBuffer } from '../../../memory/buffer-pool.js';
5
+ import { acquireBuffer, isBufferActive, releaseBuffer } from '../../../memory/buffer-pool.js';
6
6
  import {
7
7
  KVCache,
8
8
  SlidingWindowKVCache,
@@ -29,6 +29,7 @@ import {
29
29
  } from '../../../config/kernel-path-loader.js';
30
30
  import { createWeightBuffer, getWeightDtype, isGpuBufferInstance, isWeightBuffer } from '../../../gpu/weight-buffer.js';
31
31
  import { selectRuleValue } from '../../../rules/rule-registry.js';
32
+ import { resolvePerLayerInputsSession } from './generator-helpers.js';
32
33
  import {
33
34
  createHttpArtifactStorageContext,
34
35
  createNodeFileArtifactStorageContext,
@@ -351,6 +352,18 @@ function isSameRoPEScalingConfig(
351
352
  const GPU_ROPE_BUFFER_CACHE = new WeakMap();
352
353
  const CPU_ROPE_BUFFER_CACHE = new Map();
353
354
 
355
+ function isLiveCachedRopeBuffer(buffer) {
356
+ return buffer == null || isBufferActive(buffer);
357
+ }
358
+
359
+ function hasLiveCachedGpuRopeBuffers(buffers) {
360
+ return !!buffers
361
+ && isLiveCachedRopeBuffer(buffers.cos)
362
+ && isLiveCachedRopeBuffer(buffers.sin)
363
+ && isLiveCachedRopeBuffer(buffers.localCos)
364
+ && isLiveCachedRopeBuffer(buffers.localSin);
365
+ }
366
+
354
367
  function buildRoPECacheKey(config) {
355
368
  return JSON.stringify({
356
369
  headDim: config.headDim,
@@ -556,7 +569,10 @@ export async function initRoPEFrequencies(config, useGPU) {
556
569
  }
557
570
  const cachedBuffers = perDeviceCache.get(cacheKey);
558
571
  if (cachedBuffers) {
559
- return cachedBuffers;
572
+ if (hasLiveCachedGpuRopeBuffers(cachedBuffers)) {
573
+ return cachedBuffers;
574
+ }
575
+ perDeviceCache.delete(cacheKey);
560
576
  }
561
577
  let cosBuffer = null;
562
578
  let sinBuffer = null;
@@ -1090,7 +1106,12 @@ export async function loadWeights(manifest, modelConfig, options = {}) {
1090
1106
  )
1091
1107
  );
1092
1108
  dopplerLoader.setLoaderDebugConfig(loaderDebug ?? null);
1093
- dopplerLoader.setPerLayerInputSession(perLayerInputSession ?? modelConfig.perLayerInputsSession);
1109
+ dopplerLoader.setPerLayerInputSession(
1110
+ resolvePerLayerInputsSession(
1111
+ modelConfig.perLayerInputsSession ?? null,
1112
+ perLayerInputSession ?? null
1113
+ )
1114
+ );
1094
1115
 
1095
1116
  const tensorsFile = isRDRRManifest(manifest) ? manifest.tensorsFile : null;
1096
1117
  if (baseUrl && tensorsFile) {