doppler-gpu 0.2.2 → 0.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -0
- package/assets/doppler.svg +1601 -0
- package/package.json +6 -5
- package/src/cli/doppler-quickstart.js +16 -2
- package/src/cli/doppler-serve.js +0 -0
- package/src/client/doppler-registry.json +17 -1
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +1 -1
- package/src/config/kernels/registry.json +73 -0
- package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +32 -0
- package/src/config/schema/benchmark.schema.d.ts +2 -0
- package/src/config/schema/benchmark.schema.js +1 -0
- package/src/config/schema/doppler.schema.d.ts +9 -1
- package/src/config/schema/doppler.schema.js +8 -1
- package/src/config/schema/index.d.ts +2 -0
- package/src/config/schema/index.js +1 -0
- package/src/config/schema/manifest.schema.d.ts +5 -0
- package/src/converter/quantizer.js +5 -4
- package/src/formats/rdrr/source-transform-contract.js +390 -0
- package/src/formats/rdrr/types.d.ts +8 -1
- package/src/formats/tflite/types.d.ts +8 -0
- package/src/formats/tflite/types.js +45 -0
- package/src/gpu/command-recorder.d.ts +7 -1
- package/src/gpu/command-recorder.js +41 -2
- package/src/gpu/kernels/activation-static-qdq.d.ts +22 -0
- package/src/gpu/kernels/activation-static-qdq.js +69 -0
- package/src/gpu/kernels/activation_static_qdq.wgsl +23 -0
- package/src/gpu/kernels/index.d.ts +7 -0
- package/src/gpu/kernels/index.js +6 -0
- package/src/inference/browser-harness-text-helpers.d.ts +1 -0
- package/src/inference/browser-harness-text-helpers.js +44 -4
- package/src/inference/browser-harness.js +7 -0
- package/src/inference/pipelines/context.js +4 -2
- package/src/inference/pipelines/text/attention/types.js +9 -4
- package/src/inference/pipelines/text/embed.d.ts +1 -0
- package/src/inference/pipelines/text/embed.js +10 -4
- package/src/inference/pipelines/text/generator-helpers.d.ts +6 -0
- package/src/inference/pipelines/text/generator-helpers.js +20 -1
- package/src/inference/pipelines/text/generator-steps.js +17 -8
- package/src/inference/pipelines/text/generator.js +51 -3
- package/src/inference/pipelines/text/init.d.ts +3 -0
- package/src/inference/pipelines/text/init.js +24 -3
- package/src/inference/pipelines/text/logits/gpu.d.ts +5 -4
- package/src/inference/pipelines/text/logits/gpu.js +146 -11
- package/src/inference/pipelines/text/per-layer-inputs.js +320 -57
- package/src/inference/pipelines/text/probes.js +79 -1
- package/src/inference/pipelines/text/stage-names.js +9 -0
- package/src/inference/pipelines/text.js +10 -2
- package/src/loader/embedding-loader.js +20 -5
- package/src/loader/loader-types.d.ts +2 -0
- package/src/loader/manifest-config.d.ts +2 -7
- package/src/loader/per-layer-input-loader.js +114 -1
- package/src/loader/tensors/source-transform.d.ts +15 -0
- package/src/loader/tensors/source-transform.js +312 -34
- package/src/loader/tensors/tensor-reader.js +28 -2
- package/src/storage/artifact-storage-context.d.ts +1 -0
- package/src/storage/artifact-storage-context.js +158 -72
- package/src/storage/preflight.js +11 -0
- package/src/storage/quickstart-downloader.js +15 -4
- package/src/tooling/litert-package-runtime.js +369 -14
- package/src/tooling/node-source-runtime.js +256 -140
- package/src/tooling/source-runtime-bundle.d.ts +25 -0
- package/src/tooling/source-runtime-bundle.js +53 -1
- package/src/tooling/source-runtime-materializer.js +3 -1
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
struct Uniforms {
|
|
2
|
+
size: u32,
|
|
3
|
+
_pad0: u32,
|
|
4
|
+
scale: f32,
|
|
5
|
+
invScale: f32,
|
|
6
|
+
qmin: f32,
|
|
7
|
+
qmax: f32,
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
@group(0) @binding(0) var<uniform> u: Uniforms;
|
|
11
|
+
@group(0) @binding(1) var<storage, read> input: array<f32>;
|
|
12
|
+
@group(0) @binding(2) var<storage, read_write> output: array<f32>;
|
|
13
|
+
|
|
14
|
+
@compute @workgroup_size(256, 1, 1)
|
|
15
|
+
fn main(@builtin(global_invocation_id) gid: vec3<u32>) {
|
|
16
|
+
let idx = gid.x;
|
|
17
|
+
if (idx >= u.size) {
|
|
18
|
+
return;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
let quantized = clamp(round(input[idx] * u.invScale), u.qmin, u.qmax);
|
|
22
|
+
output[idx] = quantized * u.scale;
|
|
23
|
+
}
|
|
@@ -149,6 +149,13 @@ export {
|
|
|
149
149
|
type ClampOptions,
|
|
150
150
|
} from './clamp.js';
|
|
151
151
|
|
|
152
|
+
// Static activation quantize/dequantize
|
|
153
|
+
export {
|
|
154
|
+
runActivationStaticQdq,
|
|
155
|
+
recordActivationStaticQdq,
|
|
156
|
+
type ActivationStaticQdqOptions,
|
|
157
|
+
} from './activation-static-qdq.js';
|
|
158
|
+
|
|
152
159
|
// Energy (EBM helpers)
|
|
153
160
|
export {
|
|
154
161
|
runEnergyEval,
|
package/src/gpu/kernels/index.js
CHANGED
|
@@ -120,6 +120,12 @@ export {
|
|
|
120
120
|
recordClamp,
|
|
121
121
|
} from './clamp.js';
|
|
122
122
|
|
|
123
|
+
// Static activation quantize/dequantize
|
|
124
|
+
export {
|
|
125
|
+
runActivationStaticQdq,
|
|
126
|
+
recordActivationStaticQdq,
|
|
127
|
+
} from './activation-static-qdq.js';
|
|
128
|
+
|
|
123
129
|
// Energy (EBM helpers)
|
|
124
130
|
export {
|
|
125
131
|
runEnergyEval,
|
|
@@ -662,23 +662,42 @@ export function resolveBenchmarkRunSettings(runtimeConfig, source = null) {
|
|
|
662
662
|
const benchSampling = isPlainObject(benchConfig?.sampling)
|
|
663
663
|
? benchConfig.sampling
|
|
664
664
|
: {};
|
|
665
|
+
const runSeed = Number.isFinite(benchConfig.seed)
|
|
666
|
+
? Math.max(0, Math.floor(benchConfig.seed))
|
|
667
|
+
: null;
|
|
668
|
+
const runtimeSeed = Number.isFinite(runtimeSampling.seed)
|
|
669
|
+
? Math.max(0, Math.floor(runtimeSampling.seed))
|
|
670
|
+
: null;
|
|
671
|
+
const benchSeed = Number.isFinite(benchSampling.seed)
|
|
672
|
+
? Math.max(0, Math.floor(benchSampling.seed))
|
|
673
|
+
: null;
|
|
674
|
+
const mergedSeed = runSeed != null
|
|
675
|
+
? runSeed
|
|
676
|
+
: benchSeed != null
|
|
677
|
+
? benchSeed
|
|
678
|
+
: runtimeSeed;
|
|
665
679
|
const promptInput = typeof benchConfig.customPrompt === 'string' && benchConfig.customPrompt.trim()
|
|
666
680
|
? benchConfig.customPrompt.trim()
|
|
667
681
|
: resolveGenerationPromptInput(runtimeConfig, null, source);
|
|
668
682
|
const maxTokens = Number.isFinite(benchConfig.maxNewTokens)
|
|
669
683
|
? Math.max(1, Math.floor(benchConfig.maxNewTokens))
|
|
670
684
|
: resolveMaxTokens(runtimeConfig);
|
|
685
|
+
const sampling = {
|
|
686
|
+
...runtimeSampling,
|
|
687
|
+
...benchSampling,
|
|
688
|
+
};
|
|
689
|
+
if (Number.isFinite(mergedSeed)) {
|
|
690
|
+
sampling.seed = mergedSeed;
|
|
691
|
+
}
|
|
671
692
|
|
|
672
693
|
return {
|
|
673
694
|
warmupRuns: Math.max(0, Math.floor(benchConfig.warmupRuns ?? 0)),
|
|
674
695
|
timedRuns: Math.max(1, Math.floor(benchConfig.timedRuns ?? 1)),
|
|
696
|
+
...(Number.isFinite(mergedSeed) ? { seed: mergedSeed } : {}),
|
|
675
697
|
prompt: promptInput,
|
|
676
698
|
promptLabel: describePromptInput(promptInput),
|
|
677
699
|
maxTokens,
|
|
678
|
-
sampling
|
|
679
|
-
...runtimeSampling,
|
|
680
|
-
...benchSampling,
|
|
681
|
-
},
|
|
700
|
+
sampling,
|
|
682
701
|
};
|
|
683
702
|
}
|
|
684
703
|
|
|
@@ -1015,6 +1034,14 @@ function buildGenerationPhaseFromStats(pipeline, durationMs, tokenCount) {
|
|
|
1015
1034
|
if (Number.isFinite(stats.decodeRecordMs)) gpu.decodeRecordMs = stats.decodeRecordMs;
|
|
1016
1035
|
if (Number.isFinite(stats.decodeSubmitWaitMs)) gpu.decodeSubmitWaitMs = stats.decodeSubmitWaitMs;
|
|
1017
1036
|
if (Number.isFinite(stats.decodeReadbackWaitMs)) gpu.decodeReadbackWaitMs = stats.decodeReadbackWaitMs;
|
|
1037
|
+
if (
|
|
1038
|
+
Number.isFinite(decodeMs) &&
|
|
1039
|
+
Number.isFinite(stats.decodeRecordMs) &&
|
|
1040
|
+
Number.isFinite(stats.decodeSubmitWaitMs) &&
|
|
1041
|
+
Number.isFinite(stats.decodeReadbackWaitMs)
|
|
1042
|
+
) {
|
|
1043
|
+
gpu.decodeOrchestrationMs = decodeMs - stats.decodeRecordMs - stats.decodeSubmitWaitMs - stats.decodeReadbackWaitMs;
|
|
1044
|
+
}
|
|
1018
1045
|
if (Number.isFinite(stats.singleTokenSubmitWaitMs)) gpu.singleTokenSubmitWaitMs = stats.singleTokenSubmitWaitMs;
|
|
1019
1046
|
if (Number.isFinite(stats.singleTokenReadbackWaitMs)) gpu.singleTokenReadbackWaitMs = stats.singleTokenReadbackWaitMs;
|
|
1020
1047
|
if (Number.isFinite(stats.singleTokenOrchestrationMs)) gpu.singleTokenOrchestrationMs = stats.singleTokenOrchestrationMs;
|
|
@@ -1049,6 +1076,12 @@ function buildGenerationPhaseFromStats(pipeline, durationMs, tokenCount) {
|
|
|
1049
1076
|
if (Number.isFinite(stats.plePreparedTokenCacheBytes)) {
|
|
1050
1077
|
plePreparedTokenCache.bytes = stats.plePreparedTokenCacheBytes;
|
|
1051
1078
|
}
|
|
1079
|
+
if (Number.isFinite(stats.pleWriteBufferCount)) {
|
|
1080
|
+
plePreparedTokenCache.writeBufferCount = stats.pleWriteBufferCount;
|
|
1081
|
+
}
|
|
1082
|
+
if (Number.isFinite(stats.pleWriteBufferBytes)) {
|
|
1083
|
+
plePreparedTokenCache.writeBufferBytes = stats.pleWriteBufferBytes;
|
|
1084
|
+
}
|
|
1052
1085
|
const plePreparedTokenCachePhase = Object.keys(plePreparedTokenCache).length > 0
|
|
1053
1086
|
? plePreparedTokenCache
|
|
1054
1087
|
: null;
|
|
@@ -1112,6 +1145,11 @@ export async function runGeneration(pipeline, runtimeConfig, runOverrides = null
|
|
|
1112
1145
|
...(runtimeConfig.inference?.sampling || {}),
|
|
1113
1146
|
...(isPlainObject(runOverrides?.sampling) ? runOverrides.sampling : {}),
|
|
1114
1147
|
};
|
|
1148
|
+
const seed = Number.isFinite(runOverrides?.seed)
|
|
1149
|
+
? Math.max(0, Math.floor(runOverrides.seed))
|
|
1150
|
+
: Number.isFinite(sampling.seed)
|
|
1151
|
+
? Math.max(0, Math.floor(sampling.seed))
|
|
1152
|
+
: null;
|
|
1115
1153
|
const debugProbes = runtimeConfig.shared?.debug?.probes || [];
|
|
1116
1154
|
const profile = runtimeConfig.shared?.debug?.profiler?.enabled === true;
|
|
1117
1155
|
const explicitDiagnosticsEnabled = runtimeConfig.shared?.harness?.mode === 'diagnose'
|
|
@@ -1123,6 +1161,7 @@ export async function runGeneration(pipeline, runtimeConfig, runOverrides = null
|
|
|
1123
1161
|
|
|
1124
1162
|
for await (const tokenText of pipeline.generate(promptInput, {
|
|
1125
1163
|
maxTokens,
|
|
1164
|
+
...(Number.isFinite(seed) ? { seed } : {}),
|
|
1126
1165
|
temperature: sampling.temperature,
|
|
1127
1166
|
topP: sampling.topP,
|
|
1128
1167
|
topK: sampling.topK,
|
|
@@ -1151,6 +1190,7 @@ export async function runGeneration(pipeline, runtimeConfig, runOverrides = null
|
|
|
1151
1190
|
const { phase } = buildGenerationPhaseFromStats(pipeline, durationMs, tokenIds.length);
|
|
1152
1191
|
|
|
1153
1192
|
return {
|
|
1193
|
+
...(Number.isFinite(seed) ? { seed } : {}),
|
|
1154
1194
|
prompt: promptLabel,
|
|
1155
1195
|
promptInput,
|
|
1156
1196
|
maxTokens,
|
|
@@ -775,6 +775,7 @@ async function runBenchSuite(options = {}) {
|
|
|
775
775
|
const gpuDecodeRecordMs = [];
|
|
776
776
|
const gpuDecodeSubmitWaitMs = [];
|
|
777
777
|
const gpuDecodeReadbackWaitMs = [];
|
|
778
|
+
const gpuDecodeOrchestrationMs = [];
|
|
778
779
|
const singleTokenSubmitWaitMs = [];
|
|
779
780
|
const singleTokenReadbackWaitMs = [];
|
|
780
781
|
const singleTokenOrchestrationMs = [];
|
|
@@ -829,6 +830,9 @@ async function runBenchSuite(options = {}) {
|
|
|
829
830
|
if (Number.isFinite(phaseGpu?.decodeRecordMs)) gpuDecodeRecordMs.push(phaseGpu.decodeRecordMs);
|
|
830
831
|
if (Number.isFinite(phaseGpu?.decodeSubmitWaitMs)) gpuDecodeSubmitWaitMs.push(phaseGpu.decodeSubmitWaitMs);
|
|
831
832
|
if (Number.isFinite(phaseGpu?.decodeReadbackWaitMs)) gpuDecodeReadbackWaitMs.push(phaseGpu.decodeReadbackWaitMs);
|
|
833
|
+
if (Number.isFinite(phaseGpu?.decodeOrchestrationMs)) {
|
|
834
|
+
gpuDecodeOrchestrationMs.push(phaseGpu.decodeOrchestrationMs);
|
|
835
|
+
}
|
|
832
836
|
if (Number.isFinite(phaseGpu?.singleTokenSubmitWaitMs)) singleTokenSubmitWaitMs.push(phaseGpu.singleTokenSubmitWaitMs);
|
|
833
837
|
if (Number.isFinite(phaseGpu?.singleTokenReadbackWaitMs)) singleTokenReadbackWaitMs.push(phaseGpu.singleTokenReadbackWaitMs);
|
|
834
838
|
if (Number.isFinite(phaseGpu?.singleTokenOrchestrationMs)) singleTokenOrchestrationMs.push(phaseGpu.singleTokenOrchestrationMs);
|
|
@@ -858,6 +862,7 @@ async function runBenchSuite(options = {}) {
|
|
|
858
862
|
const decodeTokensStats = computeSampleStats(decodeTokens);
|
|
859
863
|
const hasGpuStats = gpuPrefillMs.length > 0 || gpuDecodeMs.length > 0 || gpuDecodeRecordMs.length > 0
|
|
860
864
|
|| gpuDecodeSubmitWaitMs.length > 0 || gpuDecodeReadbackWaitMs.length > 0
|
|
865
|
+
|| gpuDecodeOrchestrationMs.length > 0
|
|
861
866
|
|| singleTokenSubmitWaitMs.length > 0 || singleTokenReadbackWaitMs.length > 0
|
|
862
867
|
|| singleTokenOrchestrationMs.length > 0;
|
|
863
868
|
const gpuPhaseStats = hasGpuStats
|
|
@@ -867,6 +872,7 @@ async function runBenchSuite(options = {}) {
|
|
|
867
872
|
decodeRecordMs: computeSampleStats(gpuDecodeRecordMs),
|
|
868
873
|
decodeSubmitWaitMs: computeSampleStats(gpuDecodeSubmitWaitMs),
|
|
869
874
|
decodeReadbackWaitMs: computeSampleStats(gpuDecodeReadbackWaitMs),
|
|
875
|
+
decodeOrchestrationMs: computeSampleStats(gpuDecodeOrchestrationMs),
|
|
870
876
|
singleTokenSubmitWaitMs: computeSampleStats(singleTokenSubmitWaitMs),
|
|
871
877
|
singleTokenReadbackWaitMs: computeSampleStats(singleTokenReadbackWaitMs),
|
|
872
878
|
singleTokenOrchestrationMs: computeSampleStats(singleTokenOrchestrationMs),
|
|
@@ -913,6 +919,7 @@ async function runBenchSuite(options = {}) {
|
|
|
913
919
|
metrics = {
|
|
914
920
|
warmupRuns,
|
|
915
921
|
timedRuns,
|
|
922
|
+
...(Number.isFinite(benchRun?.seed) ? { seed: benchRun.seed } : {}),
|
|
916
923
|
prompt: benchRun.promptLabel,
|
|
917
924
|
maxTokens: benchRun.maxTokens,
|
|
918
925
|
decodeTokensPerSec: sampleTimingNumber(decodeTokensPerSecStats, 'median'),
|
|
@@ -145,8 +145,10 @@ export function applyPipelineContexts(target, contexts = {}, options = {}) {
|
|
|
145
145
|
if (options.assignMemoryContext && contexts.memory) {
|
|
146
146
|
target.memoryContext = contexts.memory;
|
|
147
147
|
}
|
|
148
|
-
|
|
149
|
-
|
|
148
|
+
|
|
149
|
+
const requestedStorageContext = contexts.storage ?? contexts.storageContext;
|
|
150
|
+
if (options.assignStorageContext && requestedStorageContext) {
|
|
151
|
+
target.storageContext = requestedStorageContext;
|
|
150
152
|
}
|
|
151
153
|
if (options.assignBaseUrl !== false && contexts.baseUrl) {
|
|
152
154
|
target.baseUrl = contexts.baseUrl;
|
|
@@ -44,16 +44,21 @@ export function releaseOrTrack(recorder, buffer) {
|
|
|
44
44
|
// ============================================================================
|
|
45
45
|
|
|
46
46
|
|
|
47
|
-
const qkNormOnesCache = new
|
|
47
|
+
const qkNormOnesCache = new WeakMap();
|
|
48
48
|
|
|
49
49
|
|
|
50
50
|
export function getQKNormOnesBuffer(headDim) {
|
|
51
|
-
const cached = qkNormOnesCache.get(headDim);
|
|
52
|
-
if (cached) return cached;
|
|
53
51
|
const device = getDevice();
|
|
54
52
|
if (!device) {
|
|
55
53
|
throw new Error('No GPU device available for Q/K norm buffer');
|
|
56
54
|
}
|
|
55
|
+
let perDeviceCache = qkNormOnesCache.get(device);
|
|
56
|
+
if (!perDeviceCache) {
|
|
57
|
+
perDeviceCache = new Map();
|
|
58
|
+
qkNormOnesCache.set(device, perDeviceCache);
|
|
59
|
+
}
|
|
60
|
+
const cached = perDeviceCache.get(headDim);
|
|
61
|
+
if (cached) return cached;
|
|
57
62
|
const data = new Float32Array(headDim);
|
|
58
63
|
data.fill(1);
|
|
59
64
|
const buffer = device.createBuffer({
|
|
@@ -62,6 +67,6 @@ export function getQKNormOnesBuffer(headDim) {
|
|
|
62
67
|
usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST,
|
|
63
68
|
});
|
|
64
69
|
device.queue.writeBuffer(buffer, 0, data);
|
|
65
|
-
|
|
70
|
+
perDeviceCache.set(headDim, buffer);
|
|
66
71
|
return buffer;
|
|
67
72
|
}
|
|
@@ -84,8 +84,10 @@ export async function embed(tokenIds, embedBuffer, config) {
|
|
|
84
84
|
activationDtype,
|
|
85
85
|
embeddingDtype,
|
|
86
86
|
operatorDiagnostics,
|
|
87
|
+
probeStage = 'embed_out',
|
|
87
88
|
inputHiddenSize = hiddenSize,
|
|
88
89
|
hiddenOffset = 0,
|
|
90
|
+
stats = null,
|
|
89
91
|
} = config;
|
|
90
92
|
const device = getDevice();
|
|
91
93
|
const tokenBufferInput = isGpuBufferInstance(tokenIds);
|
|
@@ -274,7 +276,7 @@ export async function embed(tokenIds, embedBuffer, config) {
|
|
|
274
276
|
} else {
|
|
275
277
|
releaseBuffer(f32Buffer);
|
|
276
278
|
}
|
|
277
|
-
await runProbes(
|
|
279
|
+
await runProbes(probeStage, f16Tensor.buffer, {
|
|
278
280
|
numTokens,
|
|
279
281
|
hiddenSize,
|
|
280
282
|
probes: config.debugProbes,
|
|
@@ -290,7 +292,11 @@ export async function embed(tokenIds, embedBuffer, config) {
|
|
|
290
292
|
? preAllocatedOutput
|
|
291
293
|
: acquireBuffer(outputBytes, undefined, 'embed_cpu_f32_out');
|
|
292
294
|
device.queue.writeBuffer(outputBuffer, 0, output);
|
|
293
|
-
|
|
295
|
+
if (stats) {
|
|
296
|
+
stats.pleWriteBufferCount = (stats.pleWriteBufferCount ?? 0) + 1;
|
|
297
|
+
stats.pleWriteBufferBytes = (stats.pleWriteBufferBytes ?? 0) + outputBytes;
|
|
298
|
+
}
|
|
299
|
+
await runProbes(probeStage, outputBuffer, {
|
|
294
300
|
numTokens,
|
|
295
301
|
hiddenSize,
|
|
296
302
|
probes: config.debugProbes,
|
|
@@ -360,7 +366,7 @@ export async function embed(tokenIds, embedBuffer, config) {
|
|
|
360
366
|
}
|
|
361
367
|
|
|
362
368
|
if (!scaleEmbeddings) {
|
|
363
|
-
await runProbes(
|
|
369
|
+
await runProbes(probeStage, gatherOutput.buffer, {
|
|
364
370
|
numTokens,
|
|
365
371
|
hiddenSize,
|
|
366
372
|
probes: config.debugProbes,
|
|
@@ -431,7 +437,7 @@ export async function embed(tokenIds, embedBuffer, config) {
|
|
|
431
437
|
throw new Error('[Embed] Scaled embedding contains NaN/Inf');
|
|
432
438
|
}
|
|
433
439
|
}
|
|
434
|
-
await runProbes(
|
|
440
|
+
await runProbes(probeStage, scaledBuffer, {
|
|
435
441
|
numTokens,
|
|
436
442
|
hiddenSize,
|
|
437
443
|
probes: config.debugProbes,
|
|
@@ -9,6 +9,7 @@ import type { LayerContext } from './types.js';
|
|
|
9
9
|
import type { LogitsConfig, LogitsWeights } from './logits/index.js';
|
|
10
10
|
import type { WeightBufferConfig } from './weights.js';
|
|
11
11
|
import type { ExecutionSessionPlan } from './execution-plan.js';
|
|
12
|
+
import type { ExecutionV1PerLayerInputsSessionSchema } from '../../../config/schema/execution-v1.schema.js';
|
|
12
13
|
|
|
13
14
|
export declare function debugCheckBuffer(
|
|
14
15
|
state: PipelineState,
|
|
@@ -27,6 +28,11 @@ export declare function buildLayerContext(
|
|
|
27
28
|
executionPlan?: ExecutionSessionPlan | null
|
|
28
29
|
): LayerContext;
|
|
29
30
|
|
|
31
|
+
export declare function resolvePerLayerInputsSession(
|
|
32
|
+
manifestSession: ExecutionV1PerLayerInputsSessionSchema | null | undefined,
|
|
33
|
+
runtimeSession: ExecutionV1PerLayerInputsSessionSchema | Record<string, unknown> | null | undefined
|
|
34
|
+
): ExecutionV1PerLayerInputsSessionSchema | null;
|
|
35
|
+
|
|
30
36
|
export declare function releaseSharedAttentionState(
|
|
31
37
|
sharedAttentionState: LayerContext['sharedAttentionState'],
|
|
32
38
|
recorder?: {
|
|
@@ -2,11 +2,26 @@
|
|
|
2
2
|
import { readBuffer, releaseBuffer } from '../../../memory/buffer-pool.js';
|
|
3
3
|
import { allowReadback } from '../../../gpu/perf-guards.js';
|
|
4
4
|
import { log } from '../../../debug/index.js';
|
|
5
|
+
import { mergeRuntimeValues } from '../../../config/runtime-merge.js';
|
|
5
6
|
import { selectRuleValue } from '../../../rules/rule-registry.js';
|
|
6
7
|
import { decodeReadback } from './debug-utils/index.js';
|
|
7
8
|
import { isWeightBuffer, isCpuWeightBuffer, isGpuBufferInstance } from '../../../gpu/weight-buffer.js';
|
|
8
9
|
import { resolveRangeAwareSelectiveWideningConfig } from './finiteness-policy.js';
|
|
9
10
|
import { resolveActiveExecutionPlan } from './execution-plan.js';
|
|
11
|
+
|
|
12
|
+
function isPlainObject(value) {
|
|
13
|
+
return value != null && typeof value === 'object' && !Array.isArray(value);
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export function resolvePerLayerInputsSession(manifestSession, runtimeSession) {
|
|
17
|
+
if (!isPlainObject(runtimeSession)) {
|
|
18
|
+
return runtimeSession ?? (manifestSession ?? null);
|
|
19
|
+
}
|
|
20
|
+
if (!isPlainObject(manifestSession)) {
|
|
21
|
+
return runtimeSession;
|
|
22
|
+
}
|
|
23
|
+
return mergeRuntimeValues(manifestSession, runtimeSession);
|
|
24
|
+
}
|
|
10
25
|
export async function debugCheckBuffer(state, buffer, label, numTokens, expectedDim) {
|
|
11
26
|
if (!allowReadback(`pipeline.debug.${label}`)) return;
|
|
12
27
|
|
|
@@ -83,6 +98,10 @@ export function buildLayerContext(state, recorder, isDecodeMode, debugLayers, de
|
|
|
83
98
|
const activeExecutionPlan = executionPlan ?? resolveActiveExecutionPlan(state);
|
|
84
99
|
const activeKernelPath = activeExecutionPlan.kernelPath ?? state.resolvedKernelPath ?? null;
|
|
85
100
|
const effectiveActivationDtype = activeExecutionPlan.activationDtype;
|
|
101
|
+
const perLayerInputsSession = resolvePerLayerInputsSession(
|
|
102
|
+
config.perLayerInputsSession ?? null,
|
|
103
|
+
state.runtimeConfig?.inference?.session?.perLayerInputs ?? null
|
|
104
|
+
);
|
|
86
105
|
const effectiveComputeConfig = {
|
|
87
106
|
...computeConfig,
|
|
88
107
|
activationDtype: effectiveActivationDtype,
|
|
@@ -122,7 +141,7 @@ export function buildLayerContext(state, recorder, isDecodeMode, debugLayers, de
|
|
|
122
141
|
debugProbes: state.runtimeConfig.shared.debug.probes,
|
|
123
142
|
debugCheckBuffer: debugCheckBufferFn,
|
|
124
143
|
perLayerInputBuffer: null,
|
|
125
|
-
perLayerInputsSession
|
|
144
|
+
perLayerInputsSession,
|
|
126
145
|
pipelinePlan: state.layerPipelinePlan,
|
|
127
146
|
expertWeights: state.expertWeights,
|
|
128
147
|
expertLoader: state.dopplerLoader,
|
|
@@ -19,6 +19,7 @@ import {
|
|
|
19
19
|
import { sample, applyRepetitionPenalty, logitsSanity, getTopK } from './sampling.js';
|
|
20
20
|
import { isStopToken } from './init.js';
|
|
21
21
|
import { embed } from './embed.js';
|
|
22
|
+
import { resolvePerLayerInputsSession } from './generator-helpers.js';
|
|
22
23
|
import { processLayer } from './layer.js';
|
|
23
24
|
import { computeLogits, computeLogitsGPU, recordLogitsGPU, extractLastPositionLogits, finalizeLogits, applySoftcapping } from './logits/index.js';
|
|
24
25
|
import { isWeightBuffer, isCpuWeightBuffer, isGpuBufferInstance, getWeightDtype } from '../../../gpu/weight-buffer.js';
|
|
@@ -111,12 +112,15 @@ function schedulePlePrefetchForToken(state, tokenId) {
|
|
|
111
112
|
if (!pleWeights?.embedTokensPerLayer) {
|
|
112
113
|
return;
|
|
113
114
|
}
|
|
114
|
-
const
|
|
115
|
+
const resolvedPerLayerInputsSession = resolvePerLayerInputsSession(
|
|
116
|
+
config.perLayerInputsSession ?? null,
|
|
117
|
+
state.runtimeConfig?.inference?.session?.perLayerInputs ?? null
|
|
118
|
+
);
|
|
115
119
|
state.plePrefetchPending = prefetchPerLayerRow(
|
|
116
120
|
tokenId,
|
|
117
121
|
pleWeights.embedTokensPerLayer,
|
|
118
122
|
config.numLayers * pleHiddenSize,
|
|
119
|
-
|
|
123
|
+
resolvedPerLayerInputsSession
|
|
120
124
|
);
|
|
121
125
|
}
|
|
122
126
|
|
|
@@ -217,7 +221,7 @@ export async function readSampledTokenFromStagingBuffer(stagingBuffer, options =
|
|
|
217
221
|
try {
|
|
218
222
|
await stagingBuffer.mapAsync(GPUMapMode.READ);
|
|
219
223
|
mapped = true;
|
|
220
|
-
cleanupRecorder?.completeDeferredCleanup();
|
|
224
|
+
await cleanupRecorder?.completeDeferredCleanup();
|
|
221
225
|
cleanupCompleted = true;
|
|
222
226
|
const mappedWords = new Uint32Array(stagingBuffer.getMappedRange());
|
|
223
227
|
return {
|
|
@@ -231,7 +235,7 @@ export async function readSampledTokenFromStagingBuffer(stagingBuffer, options =
|
|
|
231
235
|
stagingBuffer.unmap();
|
|
232
236
|
}
|
|
233
237
|
if (!cleanupCompleted) {
|
|
234
|
-
cleanupRecorder?.completeDeferredCleanup({ discardPooled: true });
|
|
238
|
+
await cleanupRecorder?.completeDeferredCleanup({ discardPooled: true });
|
|
235
239
|
}
|
|
236
240
|
if (ownsStagingBuffer) {
|
|
237
241
|
stagingBuffer.destroy();
|
|
@@ -293,7 +297,7 @@ export async function readBatchTokensFromStagingBuffers(options) {
|
|
|
293
297
|
if (mapFailure) {
|
|
294
298
|
throw mapFailure.reason;
|
|
295
299
|
}
|
|
296
|
-
cleanupRecorder?.completeDeferredCleanup();
|
|
300
|
+
await cleanupRecorder?.completeDeferredCleanup();
|
|
297
301
|
cleanupCompleted = true;
|
|
298
302
|
|
|
299
303
|
const tokens = Array.from(
|
|
@@ -322,7 +326,7 @@ export async function readBatchTokensFromStagingBuffers(options) {
|
|
|
322
326
|
stopStagingBuffer.unmap();
|
|
323
327
|
}
|
|
324
328
|
if (!cleanupCompleted) {
|
|
325
|
-
cleanupRecorder?.completeDeferredCleanup({ discardPooled: true });
|
|
329
|
+
await cleanupRecorder?.completeDeferredCleanup({ discardPooled: true });
|
|
326
330
|
}
|
|
327
331
|
if (ownsFinitenessStaging) {
|
|
328
332
|
finitenessStagingBuffer.destroy();
|
|
@@ -1135,7 +1139,10 @@ async function generateNTokensGPUStepwiseRangeBackedPle(state, N, currentIds, op
|
|
|
1135
1139
|
? config.numLayers * pleHiddenSize
|
|
1136
1140
|
: 0;
|
|
1137
1141
|
const pleWeights = state.weights.get('per_layer_inputs');
|
|
1138
|
-
const
|
|
1142
|
+
const resolvedPleSession = resolvePerLayerInputsSession(
|
|
1143
|
+
config.perLayerInputsSession ?? null,
|
|
1144
|
+
state.runtimeConfig?.inference?.session?.perLayerInputs ?? null
|
|
1145
|
+
);
|
|
1139
1146
|
const generatedTokens = [];
|
|
1140
1147
|
const rollingIds = Array.isArray(currentIds) ? currentIds.slice() : Array.from(currentIds ?? []);
|
|
1141
1148
|
let gpuSubmissions = 0;
|
|
@@ -1152,7 +1159,7 @@ async function generateNTokensGPUStepwiseRangeBackedPle(state, N, currentIds, op
|
|
|
1152
1159
|
rollingIds[rollingIds.length - 1],
|
|
1153
1160
|
pleWeights.embedTokensPerLayer,
|
|
1154
1161
|
totalPerLayerHiddenSize,
|
|
1155
|
-
|
|
1162
|
+
resolvedPleSession
|
|
1156
1163
|
);
|
|
1157
1164
|
}
|
|
1158
1165
|
|
|
@@ -1371,6 +1378,7 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
|
|
|
1371
1378
|
const currentTokenIdsArray = [startToken];
|
|
1372
1379
|
|
|
1373
1380
|
for (let i = 0; i < N; i++) {
|
|
1381
|
+
currentTokenIdsArray[0] = rollingIds[i];
|
|
1374
1382
|
const currentPos = state.currentSeqLen + i;
|
|
1375
1383
|
context.currentSeqLen = currentPos;
|
|
1376
1384
|
context.currentTokenIds = currentTokenIdsArray;
|
|
@@ -1397,6 +1405,7 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
|
|
|
1397
1405
|
indexOffset: i,
|
|
1398
1406
|
perLayerTokenIds: pleInputTokensBuffer,
|
|
1399
1407
|
perLayerIndexOffset: i,
|
|
1408
|
+
tokenIdHint: Number.isInteger(rollingIds[i]) ? rollingIds[i] : null,
|
|
1400
1409
|
pleCache: state.pleCache ?? null,
|
|
1401
1410
|
});
|
|
1402
1411
|
try {
|
|
@@ -43,6 +43,7 @@ import {
|
|
|
43
43
|
debugCheckBuffer as debugCheckBufferHelper,
|
|
44
44
|
getLogitsConfig,
|
|
45
45
|
getLogitsWeights,
|
|
46
|
+
resolvePerLayerInputsSession,
|
|
46
47
|
releaseSharedAttentionState,
|
|
47
48
|
} from './generator-helpers.js';
|
|
48
49
|
import {
|
|
@@ -135,10 +136,24 @@ export function resolveHotVocabularyBatchDecodeAvailability({
|
|
|
135
136
|
}
|
|
136
137
|
|
|
137
138
|
async function primePleDecodeRuntimeCache(state, seedTokenIds = null) {
|
|
139
|
+
const perLayerInputsSession = resolvePerLayerInputsSession(
|
|
140
|
+
state.modelConfig?.perLayerInputsSession ?? null,
|
|
141
|
+
state.runtimeConfig?.inference?.session?.perLayerInputs ?? null
|
|
142
|
+
);
|
|
143
|
+
const materialization = perLayerInputsSession?.materialization;
|
|
144
|
+
if (state.debug) {
|
|
145
|
+
log.debug(
|
|
146
|
+
'Pipeline',
|
|
147
|
+
`PLE materialization session=${materialization ?? 'null'} ` +
|
|
148
|
+
`modelId=${state.modelConfig?.modelId ?? 'unknown'} ` +
|
|
149
|
+
`runtimePerLayerInputs=${Boolean(state.runtimeConfig.inference.session?.perLayerInputs)} ` +
|
|
150
|
+
`manifestPerLayerInputs=${Boolean(state.modelConfig?.perLayerInputsSession)}`
|
|
151
|
+
);
|
|
152
|
+
}
|
|
138
153
|
await ensurePleGpuHotVocabularyRuntime({
|
|
139
154
|
config: state.modelConfig,
|
|
140
155
|
weights: state.weights,
|
|
141
|
-
perLayerInputsSession
|
|
156
|
+
perLayerInputsSession,
|
|
142
157
|
debugFlags: state.debugFlags,
|
|
143
158
|
tokenizer: state.tokenizer ?? null,
|
|
144
159
|
seedTokenIds: Array.isArray(seedTokenIds) ? seedTokenIds : null,
|
|
@@ -146,7 +161,7 @@ async function primePleDecodeRuntimeCache(state, seedTokenIds = null) {
|
|
|
146
161
|
await ensurePleGpuSplitTablesRuntime({
|
|
147
162
|
config: state.modelConfig,
|
|
148
163
|
weights: state.weights,
|
|
149
|
-
perLayerInputsSession
|
|
164
|
+
perLayerInputsSession,
|
|
150
165
|
debugFlags: state.debugFlags,
|
|
151
166
|
});
|
|
152
167
|
await ensurePleScaledProjectionNormWeight({
|
|
@@ -1304,6 +1319,26 @@ export class PipelineGenerator {
|
|
|
1304
1319
|
opts.topP = samplingConfig.topP;
|
|
1305
1320
|
opts.topK = samplingConfig.topK;
|
|
1306
1321
|
opts.repetitionPenalty = samplingConfig.repetitionPenalty;
|
|
1322
|
+
const diagnosticsEnabled = options?.diagnostics?.enabled === true
|
|
1323
|
+
|| this.#state.runtimeConfig?.shared?.harness?.mode === 'diagnose';
|
|
1324
|
+
if (diagnosticsEnabled) {
|
|
1325
|
+
const captureConfig = {
|
|
1326
|
+
...createDefaultCaptureConfig(),
|
|
1327
|
+
enabled: true,
|
|
1328
|
+
defaultLevel: CAPTURE_LEVELS.SLICE,
|
|
1329
|
+
...(options?.diagnostics?.captureConfig ?? {}),
|
|
1330
|
+
};
|
|
1331
|
+
validateCaptureConfig(captureConfig);
|
|
1332
|
+
this.#state.operatorDiagnostics = {
|
|
1333
|
+
enabled: true,
|
|
1334
|
+
captureConfig,
|
|
1335
|
+
emitter: new OperatorEventEmitter({
|
|
1336
|
+
modelHash: this.#state.manifest?.modelId ?? null,
|
|
1337
|
+
runtimeConfigHash: this.#state.resolvedKernelPath?.id ?? null,
|
|
1338
|
+
executionPlanHash: opts.executionPlan?.id ?? null,
|
|
1339
|
+
}),
|
|
1340
|
+
};
|
|
1341
|
+
}
|
|
1307
1342
|
|
|
1308
1343
|
try {
|
|
1309
1344
|
const prefillStartSeqLen = this.#state.currentSeqLen;
|
|
@@ -1370,6 +1405,14 @@ export class PipelineGenerator {
|
|
|
1370
1405
|
this.#state.stats.executionPlan.finalActivePlanId = this.#state.executionPlanState?.activePlanId ?? null;
|
|
1371
1406
|
}
|
|
1372
1407
|
resetActiveExecutionPlan(this.#state);
|
|
1408
|
+
this.#state.stats.operatorDiagnostics = this.#state.operatorDiagnostics?.emitter
|
|
1409
|
+
? {
|
|
1410
|
+
enabled: true,
|
|
1411
|
+
timeline: this.#state.operatorDiagnostics.emitter.getTimeline(),
|
|
1412
|
+
recordCount: this.#state.operatorDiagnostics.emitter.length,
|
|
1413
|
+
}
|
|
1414
|
+
: null;
|
|
1415
|
+
this.#state.operatorDiagnostics = null;
|
|
1373
1416
|
this.#state.isGenerating = false;
|
|
1374
1417
|
}
|
|
1375
1418
|
}
|
|
@@ -1729,6 +1772,10 @@ export class PipelineGenerator {
|
|
|
1729
1772
|
}
|
|
1730
1773
|
|
|
1731
1774
|
const decodeStart = performance.now();
|
|
1775
|
+
const resolvedPerLayerInputsSession = resolvePerLayerInputsSession(
|
|
1776
|
+
this.#state.modelConfig.perLayerInputsSession ?? null,
|
|
1777
|
+
this.#state.runtimeConfig?.inference?.session?.perLayerInputs ?? null
|
|
1778
|
+
);
|
|
1732
1779
|
const lmHead = this.#state.weights.get('lm_head');
|
|
1733
1780
|
const embedBuffer = this.#state.weights.get('embed');
|
|
1734
1781
|
const hasCpuWeights = isCpuWeightBuffer(lmHead)
|
|
@@ -2020,7 +2067,7 @@ export class PipelineGenerator {
|
|
|
2020
2067
|
nextToken,
|
|
2021
2068
|
pleWeights.embedTokensPerLayer,
|
|
2022
2069
|
this.#state.modelConfig.numLayers * pleHiddenSize,
|
|
2023
|
-
|
|
2070
|
+
resolvedPerLayerInputsSession,
|
|
2024
2071
|
);
|
|
2025
2072
|
}
|
|
2026
2073
|
}
|
|
@@ -2214,6 +2261,7 @@ export class PipelineGenerator {
|
|
|
2214
2261
|
context,
|
|
2215
2262
|
{
|
|
2216
2263
|
numTokens,
|
|
2264
|
+
pleCache: this.#state.pleCache ?? null,
|
|
2217
2265
|
}
|
|
2218
2266
|
);
|
|
2219
2267
|
} catch (error) {
|
|
@@ -57,6 +57,7 @@ export interface PipelineStorageContext {
|
|
|
57
57
|
loadTokenizerJson?: () => Promise<Record<string, unknown> | string | null | undefined>;
|
|
58
58
|
loadTokenizerModel?: (path?: string) => Promise<ArrayBuffer | Uint8Array | null | undefined>;
|
|
59
59
|
verifyHashes?: boolean;
|
|
60
|
+
close?: () => Promise<void>;
|
|
60
61
|
}
|
|
61
62
|
|
|
62
63
|
export function createNodeFileShardStorageContext(
|
|
@@ -74,6 +75,8 @@ export interface PipelineContexts {
|
|
|
74
75
|
memory?: Record<string, unknown>;
|
|
75
76
|
/** Storage context for custom shard loading */
|
|
76
77
|
storage?: PipelineStorageContext;
|
|
78
|
+
/** Storage context alias used by several public call sites */
|
|
79
|
+
storageContext?: PipelineStorageContext;
|
|
77
80
|
/** Base URL for loading model files */
|
|
78
81
|
baseUrl?: string;
|
|
79
82
|
/** Full runtime config (merged with defaults) */
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
import { parseModelConfig } from './config.js';
|
|
4
4
|
import { getDevice, getDeviceLimits, getKernelCapabilities } from '../../../gpu/device.js';
|
|
5
|
-
import { acquireBuffer, releaseBuffer } from '../../../memory/buffer-pool.js';
|
|
5
|
+
import { acquireBuffer, isBufferActive, releaseBuffer } from '../../../memory/buffer-pool.js';
|
|
6
6
|
import {
|
|
7
7
|
KVCache,
|
|
8
8
|
SlidingWindowKVCache,
|
|
@@ -29,6 +29,7 @@ import {
|
|
|
29
29
|
} from '../../../config/kernel-path-loader.js';
|
|
30
30
|
import { createWeightBuffer, getWeightDtype, isGpuBufferInstance, isWeightBuffer } from '../../../gpu/weight-buffer.js';
|
|
31
31
|
import { selectRuleValue } from '../../../rules/rule-registry.js';
|
|
32
|
+
import { resolvePerLayerInputsSession } from './generator-helpers.js';
|
|
32
33
|
import {
|
|
33
34
|
createHttpArtifactStorageContext,
|
|
34
35
|
createNodeFileArtifactStorageContext,
|
|
@@ -351,6 +352,18 @@ function isSameRoPEScalingConfig(
|
|
|
351
352
|
const GPU_ROPE_BUFFER_CACHE = new WeakMap();
|
|
352
353
|
const CPU_ROPE_BUFFER_CACHE = new Map();
|
|
353
354
|
|
|
355
|
+
function isLiveCachedRopeBuffer(buffer) {
|
|
356
|
+
return buffer == null || isBufferActive(buffer);
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
function hasLiveCachedGpuRopeBuffers(buffers) {
|
|
360
|
+
return !!buffers
|
|
361
|
+
&& isLiveCachedRopeBuffer(buffers.cos)
|
|
362
|
+
&& isLiveCachedRopeBuffer(buffers.sin)
|
|
363
|
+
&& isLiveCachedRopeBuffer(buffers.localCos)
|
|
364
|
+
&& isLiveCachedRopeBuffer(buffers.localSin);
|
|
365
|
+
}
|
|
366
|
+
|
|
354
367
|
function buildRoPECacheKey(config) {
|
|
355
368
|
return JSON.stringify({
|
|
356
369
|
headDim: config.headDim,
|
|
@@ -556,7 +569,10 @@ export async function initRoPEFrequencies(config, useGPU) {
|
|
|
556
569
|
}
|
|
557
570
|
const cachedBuffers = perDeviceCache.get(cacheKey);
|
|
558
571
|
if (cachedBuffers) {
|
|
559
|
-
|
|
572
|
+
if (hasLiveCachedGpuRopeBuffers(cachedBuffers)) {
|
|
573
|
+
return cachedBuffers;
|
|
574
|
+
}
|
|
575
|
+
perDeviceCache.delete(cacheKey);
|
|
560
576
|
}
|
|
561
577
|
let cosBuffer = null;
|
|
562
578
|
let sinBuffer = null;
|
|
@@ -1090,7 +1106,12 @@ export async function loadWeights(manifest, modelConfig, options = {}) {
|
|
|
1090
1106
|
)
|
|
1091
1107
|
);
|
|
1092
1108
|
dopplerLoader.setLoaderDebugConfig(loaderDebug ?? null);
|
|
1093
|
-
dopplerLoader.setPerLayerInputSession(
|
|
1109
|
+
dopplerLoader.setPerLayerInputSession(
|
|
1110
|
+
resolvePerLayerInputsSession(
|
|
1111
|
+
modelConfig.perLayerInputsSession ?? null,
|
|
1112
|
+
perLayerInputSession ?? null
|
|
1113
|
+
)
|
|
1114
|
+
);
|
|
1094
1115
|
|
|
1095
1116
|
const tensorsFile = isRDRRManifest(manifest) ? manifest.tensorsFile : null;
|
|
1096
1117
|
if (baseUrl && tensorsFile) {
|