doppler-gpu 0.4.9 → 0.4.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +53 -0
- package/README.md +89 -12
- package/models/catalog.json +612 -7
- package/package.json +30 -1
- package/src/client/doppler-registry.js +6 -0
- package/src/client/doppler-registry.json +126 -4
- package/src/client/runtime/lora.d.ts +10 -0
- package/src/client/runtime/lora.js +64 -0
- package/src/client/runtime/model-manager.js +6 -167
- package/src/client/runtime/model-session.d.ts +3 -1
- package/src/client/runtime/model-session.js +6 -0
- package/src/client/wrap-pipeline-handle.js +7 -0
- package/src/config/conversion/amplify/amplify-120m-f16-af32.json +516 -0
- package/src/config/conversion/esm/esm2-t12-35m-ur50d-f32-af32.json +571 -0
- package/src/config/conversion/esmc/esmc-300m-f32-af32.json +342 -0
- package/src/config/conversion/nucleotide-transformer/nucleotide-transformer-v2-50m-f32-af32.json +388 -0
- package/src/config/kernels/registry.json +52 -9
- package/src/config/merge.js +24 -0
- package/src/config/required-inference-fields-contract-check.js +2 -0
- package/src/config/schema/converter.schema.d.ts +7 -0
- package/src/config/schema/converter.schema.js +15 -0
- package/src/config/schema/inference.schema.d.ts +8 -0
- package/src/config/schema/manifest.schema.d.ts +32 -1
- package/src/config/schema/manifest.schema.js +26 -0
- package/src/converter/conversion-plan.js +15 -1
- package/src/converter/core.js +5 -4
- package/src/converter/source-tensor-rules.d.ts +50 -0
- package/src/converter/source-tensor-rules.js +180 -0
- package/src/converter/source-tokenizer.d.ts +45 -0
- package/src/converter/source-tokenizer.js +155 -0
- package/src/experimental/adapters/lora-loader.js +15 -0
- package/src/experimental/bridge/trainer-artifact-bridge.d.ts +2 -1
- package/src/experimental/bridge/trainer-artifact-bridge.js +35 -13
- package/src/gpu/device.js +100 -32
- package/src/inference/pipelines/text/attention/attn-config.js +2 -0
- package/src/inference/pipelines/text/attention/projections.js +161 -24
- package/src/inference/pipelines/text/attention/record.js +67 -9
- package/src/inference/pipelines/text/attention/run.js +60 -7
- package/src/inference/pipelines/text/attention/types.js +18 -9
- package/src/inference/pipelines/text/config.d.ts +5 -0
- package/src/inference/pipelines/text/config.js +91 -0
- package/src/inference/pipelines/text/ffn/dense.js +189 -2
- package/src/inference/pipelines/text/ffn/standard.js +50 -3
- package/src/inference/pipelines/text/generator-helpers.js +11 -1
- package/src/inference/pipelines/text/generator-runtime.js +63 -15
- package/src/inference/pipelines/text/generator-steps.js +3 -1
- package/src/inference/pipelines/text/generator.js +90 -3
- package/src/inference/pipelines/text/init-qkv-fusion.js +85 -1
- package/src/inference/pipelines/text/init.d.ts +2 -0
- package/src/inference/pipelines/text/init.js +2 -0
- package/src/inference/pipelines/text/layer.js +6 -0
- package/src/inference/pipelines/text/logits/cpu.d.ts +6 -0
- package/src/inference/pipelines/text/logits/cpu.js +22 -0
- package/src/inference/pipelines/text/logits/index.d.ts +1 -1
- package/src/inference/pipelines/text/logits/index.js +85 -11
- package/src/inference/pipelines/text/logits/types.d.ts +4 -0
- package/src/inference/pipelines/text/ops.js +32 -0
- package/src/inference/pipelines/text/types.d.ts +38 -0
- package/src/inference/pipelines/text/weights.d.ts +10 -0
- package/src/inference/pipelines/text/weights.js +33 -1
- package/src/inference/pipelines/text.d.ts +4 -2
- package/src/inference/pipelines/text.js +112 -0
- package/src/inference/tokenizers/behavior-flags.js +5 -1
- package/src/inference/tokenizers/bundled.js +99 -8
- package/src/loader/doppler-loader.d.ts +2 -0
- package/src/loader/doppler-loader.js +9 -0
- package/src/loader/final-weights-loader.d.ts +8 -0
- package/src/loader/final-weights-loader.js +42 -0
- package/src/loader/layer-loader.d.ts +6 -0
- package/src/loader/layer-loader.js +102 -3
- package/src/loader/loader-types.d.ts +15 -2
- package/src/tooling/node-converter.js +33 -1
- package/src/tooling/node-webgpu.js +85 -2
- package/src/tooling/runtime-optimization.d.ts +109 -0
- package/src/tooling/runtime-optimization.js +872 -0
- package/src/tooling-exports.shared.d.ts +19 -0
- package/src/tooling-exports.shared.js +11 -0
- package/src/version.js +1 -1
|
@@ -63,6 +63,7 @@ import {
|
|
|
63
63
|
resolveAdvanceEmbeddingMode,
|
|
64
64
|
getFinalNormWeights,
|
|
65
65
|
extractEmbeddingFromHidden,
|
|
66
|
+
extractTokenEmbeddingsFromHidden,
|
|
66
67
|
} from './generator-runtime.js';
|
|
67
68
|
|
|
68
69
|
import { resolveSamplingConfig } from './sampling-config.js';
|
|
@@ -642,7 +643,20 @@ export class PipelineGenerator {
|
|
|
642
643
|
embeddingMode,
|
|
643
644
|
finalNormWeights,
|
|
644
645
|
config,
|
|
645
|
-
this.#state.embeddingPostprocessor
|
|
646
|
+
this.#state.embeddingPostprocessor,
|
|
647
|
+
null,
|
|
648
|
+
this.#state.weights.get('final_norm_bias') ?? null
|
|
649
|
+
);
|
|
650
|
+
}
|
|
651
|
+
|
|
652
|
+
_extractTokenEmbeddingsFromHidden(hiddenStates, numTokens, hiddenSize, finalNormWeights, config) {
|
|
653
|
+
return extractTokenEmbeddingsFromHidden(
|
|
654
|
+
hiddenStates,
|
|
655
|
+
numTokens,
|
|
656
|
+
hiddenSize,
|
|
657
|
+
finalNormWeights,
|
|
658
|
+
config,
|
|
659
|
+
this.#state.weights.get('final_norm_bias') ?? null
|
|
646
660
|
);
|
|
647
661
|
}
|
|
648
662
|
|
|
@@ -2112,6 +2126,34 @@ export class PipelineGenerator {
|
|
|
2112
2126
|
this.#state.stats.gpuTimePrefillMs = undefined;
|
|
2113
2127
|
this.#state.stats.prefillProfileSteps = [];
|
|
2114
2128
|
const opts = resolvePrefillEmbeddingOptions(this.#state, options);
|
|
2129
|
+
const diagnosticsEnabled = options?.diagnostics?.enabled === true
|
|
2130
|
+
|| this.#state.runtimeConfig?.shared?.harness?.mode === 'diagnose';
|
|
2131
|
+
const tsirFixtureCfg = this.#state.runtimeConfig?.shared?.harness?.tsirFixture ?? null;
|
|
2132
|
+
if (diagnosticsEnabled || tsirFixtureCfg) {
|
|
2133
|
+
const captureConfig = {
|
|
2134
|
+
...createDefaultCaptureConfig(),
|
|
2135
|
+
enabled: true,
|
|
2136
|
+
defaultLevel: CAPTURE_LEVELS.SLICE,
|
|
2137
|
+
...(options?.diagnostics?.captureConfig ?? {}),
|
|
2138
|
+
};
|
|
2139
|
+
validateCaptureConfig(captureConfig);
|
|
2140
|
+
this.#state.operatorDiagnostics = {
|
|
2141
|
+
enabled: diagnosticsEnabled === true,
|
|
2142
|
+
captureConfig,
|
|
2143
|
+
emitter: diagnosticsEnabled ? new OperatorEventEmitter({
|
|
2144
|
+
modelHash: this.#state.manifest?.modelId ?? null,
|
|
2145
|
+
runtimeConfigHash: this.#state.resolvedKernelPath?.id ?? null,
|
|
2146
|
+
executionPlanHash: opts.executionPlan?.id ?? null,
|
|
2147
|
+
}) : null,
|
|
2148
|
+
tsirFixture: tsirFixtureCfg ? {
|
|
2149
|
+
dir: tsirFixtureCfg.dir,
|
|
2150
|
+
layerFilter: Array.isArray(tsirFixtureCfg.layerFilter)
|
|
2151
|
+
? tsirFixtureCfg.layerFilter
|
|
2152
|
+
: null,
|
|
2153
|
+
records: [],
|
|
2154
|
+
} : null,
|
|
2155
|
+
};
|
|
2156
|
+
}
|
|
2115
2157
|
const prefillStartSeqLen = this.#state.currentSeqLen;
|
|
2116
2158
|
const inputStart = performance.now();
|
|
2117
2159
|
const inputIds = this._resolvePromptOrInputIds(prompt, opts.useChatTemplate, 'prefillWithEmbedding', opts.inputIds);
|
|
@@ -2178,12 +2220,32 @@ export class PipelineGenerator {
|
|
|
2178
2220
|
}
|
|
2179
2221
|
|
|
2180
2222
|
let embedding;
|
|
2223
|
+
let tokenEmbeddings = null;
|
|
2224
|
+
let logits = null;
|
|
2181
2225
|
let hiddenBytes = 0;
|
|
2182
2226
|
let readbackMs = 0;
|
|
2183
2227
|
let decodeHiddenMs = 0;
|
|
2184
2228
|
let finalNormMs = 0;
|
|
2185
2229
|
let extractMs = 0;
|
|
2230
|
+
let logitsMs = 0;
|
|
2186
2231
|
try {
|
|
2232
|
+
if (options.__returnSequenceLogits === true) {
|
|
2233
|
+
const logitsStart = performance.now();
|
|
2234
|
+
logits = await computeLogits(
|
|
2235
|
+
currentHiddenBuffer,
|
|
2236
|
+
numTokens,
|
|
2237
|
+
getLogitsWeights(this.#state),
|
|
2238
|
+
getLogitsConfig(this.#state),
|
|
2239
|
+
this.#state.useGPU,
|
|
2240
|
+
this.#state.debugFlags,
|
|
2241
|
+
undefined,
|
|
2242
|
+
undefined,
|
|
2243
|
+
this.#state.runtimeConfig.shared.debug.probes,
|
|
2244
|
+
{ lastPositionOnly: false },
|
|
2245
|
+
this.#state.operatorDiagnostics
|
|
2246
|
+
);
|
|
2247
|
+
logitsMs = performance.now() - logitsStart;
|
|
2248
|
+
}
|
|
2187
2249
|
const hiddenSize = config.hiddenSize;
|
|
2188
2250
|
hiddenBytes = numTokens * hiddenSize * activationBytes;
|
|
2189
2251
|
const readbackStart = performance.now();
|
|
@@ -2199,13 +2261,25 @@ export class PipelineGenerator {
|
|
|
2199
2261
|
const finalNormWeights = await this._getFinalNormWeights();
|
|
2200
2262
|
finalNormMs = performance.now() - finalNormStart;
|
|
2201
2263
|
const extractStart = performance.now();
|
|
2202
|
-
|
|
2264
|
+
if (options.__returnTokenEmbeddings === true) {
|
|
2265
|
+
tokenEmbeddings = this._extractTokenEmbeddingsFromHidden(
|
|
2266
|
+
hiddenStates,
|
|
2267
|
+
numTokens,
|
|
2268
|
+
hiddenSize,
|
|
2269
|
+
finalNormWeights,
|
|
2270
|
+
config
|
|
2271
|
+
);
|
|
2272
|
+
}
|
|
2273
|
+
embedding = extractEmbeddingFromHidden(
|
|
2203
2274
|
hiddenStates,
|
|
2204
2275
|
numTokens,
|
|
2205
2276
|
hiddenSize,
|
|
2206
2277
|
opts.embeddingMode,
|
|
2207
2278
|
finalNormWeights,
|
|
2208
|
-
config
|
|
2279
|
+
config,
|
|
2280
|
+
this.#state.embeddingPostprocessor,
|
|
2281
|
+
tokenEmbeddings,
|
|
2282
|
+
this.#state.weights.get('final_norm_bias') ?? null
|
|
2209
2283
|
);
|
|
2210
2284
|
extractMs = performance.now() - extractStart;
|
|
2211
2285
|
} finally {
|
|
@@ -2222,6 +2296,7 @@ export class PipelineGenerator {
|
|
|
2222
2296
|
decodeHiddenMs,
|
|
2223
2297
|
finalNormMs,
|
|
2224
2298
|
extractMs,
|
|
2299
|
+
logitsMs,
|
|
2225
2300
|
hiddenBytes,
|
|
2226
2301
|
tokens: numTokens,
|
|
2227
2302
|
activationDtype,
|
|
@@ -2241,6 +2316,8 @@ export class PipelineGenerator {
|
|
|
2241
2316
|
seqLen: this.#state.currentSeqLen,
|
|
2242
2317
|
tokens: inputIds,
|
|
2243
2318
|
embedding,
|
|
2319
|
+
tokenEmbeddings,
|
|
2320
|
+
logits,
|
|
2244
2321
|
embeddingMode: opts.embeddingMode,
|
|
2245
2322
|
phase,
|
|
2246
2323
|
linearAttention: null,
|
|
@@ -2257,12 +2334,22 @@ export class PipelineGenerator {
|
|
|
2257
2334
|
seqLen: this.#state.currentSeqLen,
|
|
2258
2335
|
tokens: inputIds,
|
|
2259
2336
|
embedding,
|
|
2337
|
+
tokenEmbeddings,
|
|
2338
|
+
logits,
|
|
2260
2339
|
embeddingMode: opts.embeddingMode,
|
|
2261
2340
|
phase,
|
|
2262
2341
|
linearAttention: await cloneLinearAttentionRuntime(this.#state.linearAttentionRuntime),
|
|
2263
2342
|
};
|
|
2264
2343
|
} finally {
|
|
2265
2344
|
this._closeFinitenessFallbackWindow(opts);
|
|
2345
|
+
this.#state.stats.operatorDiagnostics = this.#state.operatorDiagnostics?.emitter
|
|
2346
|
+
? {
|
|
2347
|
+
enabled: true,
|
|
2348
|
+
timeline: this.#state.operatorDiagnostics.emitter.getTimeline(),
|
|
2349
|
+
recordCount: this.#state.operatorDiagnostics.emitter.length,
|
|
2350
|
+
}
|
|
2351
|
+
: null;
|
|
2352
|
+
this.#state.operatorDiagnostics = null;
|
|
2266
2353
|
}
|
|
2267
2354
|
}
|
|
2268
2355
|
|
|
@@ -2,7 +2,13 @@ import { getDevice } from '../../../gpu/device.js';
|
|
|
2
2
|
import { Q4K_BLOCK_BYTES, q4kBlockCount } from '../../../config/schema/index.js';
|
|
3
3
|
import { getKernelConfig } from '../../../gpu/kernels/kernel-configs.js';
|
|
4
4
|
import { getKernelPathMatmulVariant } from '../../../config/kernel-path-loader.js';
|
|
5
|
-
import {
|
|
5
|
+
import {
|
|
6
|
+
createWeightBuffer,
|
|
7
|
+
getWeightDtype,
|
|
8
|
+
isGpuBufferInstance,
|
|
9
|
+
isWeightBuffer,
|
|
10
|
+
tagBufferDtype,
|
|
11
|
+
} from '../../../gpu/weight-buffer.js';
|
|
6
12
|
import { selectRuleValue } from '../../../rules/rule-registry.js';
|
|
7
13
|
import { log } from '../../../debug/index.js';
|
|
8
14
|
|
|
@@ -112,6 +118,26 @@ function formatsMatch(a, b) {
|
|
|
112
118
|
&& a?.bytesPerRow === b?.bytesPerRow;
|
|
113
119
|
}
|
|
114
120
|
|
|
121
|
+
function resolveBiasStorageFormat(weight, expectedElements) {
|
|
122
|
+
if (!weight || !Number.isFinite(expectedElements) || expectedElements <= 0) {
|
|
123
|
+
return null;
|
|
124
|
+
}
|
|
125
|
+
const buffer = isWeightBuffer(weight) ? weight.buffer : weight;
|
|
126
|
+
if (!isGpuBufferInstance(buffer)) {
|
|
127
|
+
return null;
|
|
128
|
+
}
|
|
129
|
+
const dtype = String(getWeightDtype(weight) ?? getWeightDtype(buffer) ?? '').toLowerCase();
|
|
130
|
+
if (dtype !== 'f16' && dtype !== 'f32') {
|
|
131
|
+
return null;
|
|
132
|
+
}
|
|
133
|
+
const bytesPerElement = dtype === 'f16' ? 2 : 4;
|
|
134
|
+
const byteLength = expectedElements * bytesPerElement;
|
|
135
|
+
if (buffer.size < byteLength) {
|
|
136
|
+
return null;
|
|
137
|
+
}
|
|
138
|
+
return { buffer, dtype, bytesPerElement, byteLength };
|
|
139
|
+
}
|
|
140
|
+
|
|
115
141
|
export function fuseQKVWeights(layerWeights, modelConfig, kernelPath = null, options = {}) {
|
|
116
142
|
const device = getDevice();
|
|
117
143
|
if (!device) {
|
|
@@ -174,6 +200,17 @@ export function fuseQKVWeights(layerWeights, modelConfig, kernelPath = null, opt
|
|
|
174
200
|
continue;
|
|
175
201
|
}
|
|
176
202
|
|
|
203
|
+
const projectionBiases = [weights.qProjBias, weights.kProjBias, weights.vProjBias];
|
|
204
|
+
const presentProjectionBiases = projectionBiases.filter(Boolean).length;
|
|
205
|
+
if (presentProjectionBiases !== 0 && presentProjectionBiases !== projectionBiases.length) {
|
|
206
|
+
log.debug('QKV Fusion', `Layer ${l}: partial Q/K/V projection biases require split projections`);
|
|
207
|
+
continue;
|
|
208
|
+
}
|
|
209
|
+
if (hasAttentionOutputGate && presentProjectionBiases > 0) {
|
|
210
|
+
log.debug('QKV Fusion', `Layer ${l}: gated Q projection biases require split projections`);
|
|
211
|
+
continue;
|
|
212
|
+
}
|
|
213
|
+
|
|
177
214
|
const qFormat = resolveProjectionStorageFormat(qProj, qProjSize, hiddenSize);
|
|
178
215
|
const kFormat = resolveProjectionStorageFormat(kProj, kSize, hiddenSize);
|
|
179
216
|
const vFormat = resolveProjectionStorageFormat(vProj, vSize, hiddenSize);
|
|
@@ -228,6 +265,32 @@ export function fuseQKVWeights(layerWeights, modelConfig, kernelPath = null, opt
|
|
|
228
265
|
})
|
|
229
266
|
: null;
|
|
230
267
|
|
|
268
|
+
let qBiasFormat = null;
|
|
269
|
+
let kBiasFormat = null;
|
|
270
|
+
let vBiasFormat = null;
|
|
271
|
+
let qkvBiasBuffer = null;
|
|
272
|
+
if (presentProjectionBiases === projectionBiases.length) {
|
|
273
|
+
qBiasFormat = resolveBiasStorageFormat(weights.qProjBias, qSize);
|
|
274
|
+
kBiasFormat = resolveBiasStorageFormat(weights.kProjBias, kSize);
|
|
275
|
+
vBiasFormat = resolveBiasStorageFormat(weights.vProjBias, vSize);
|
|
276
|
+
const biasFormatsMatch = qBiasFormat
|
|
277
|
+
&& kBiasFormat
|
|
278
|
+
&& vBiasFormat
|
|
279
|
+
&& qBiasFormat.dtype === kBiasFormat.dtype
|
|
280
|
+
&& qBiasFormat.dtype === vBiasFormat.dtype;
|
|
281
|
+
if (!biasFormatsMatch) {
|
|
282
|
+
log.debug('QKV Fusion', `Layer ${l}: inconsistent Q/K/V projection bias formats, skipping`);
|
|
283
|
+
qkvBuffer.destroy();
|
|
284
|
+
qGateBuffer?.destroy();
|
|
285
|
+
continue;
|
|
286
|
+
}
|
|
287
|
+
qkvBiasBuffer = device.createBuffer({
|
|
288
|
+
label: `layer_${l}_qkv_proj_bias`,
|
|
289
|
+
size: qBiasFormat.byteLength + kBiasFormat.byteLength + vBiasFormat.byteLength,
|
|
290
|
+
usage: GPUBufferUsage.STORAGE | GPUBufferUsage.COPY_DST,
|
|
291
|
+
});
|
|
292
|
+
}
|
|
293
|
+
|
|
231
294
|
const encoder = device.createCommandEncoder({ label: 'qkv_fusion' });
|
|
232
295
|
if (hasAttentionOutputGate) {
|
|
233
296
|
for (let head = 0; head < numHeads; head++) {
|
|
@@ -266,6 +329,23 @@ export function fuseQKVWeights(layerWeights, modelConfig, kernelPath = null, opt
|
|
|
266
329
|
qkvBuffer, qBytes + kFormat.byteLength,
|
|
267
330
|
vFormat.byteLength
|
|
268
331
|
);
|
|
332
|
+
if (qkvBiasBuffer) {
|
|
333
|
+
encoder.copyBufferToBuffer(qBiasFormat.buffer, 0, qkvBiasBuffer, 0, qBiasFormat.byteLength);
|
|
334
|
+
encoder.copyBufferToBuffer(
|
|
335
|
+
kBiasFormat.buffer,
|
|
336
|
+
0,
|
|
337
|
+
qkvBiasBuffer,
|
|
338
|
+
qBiasFormat.byteLength,
|
|
339
|
+
kBiasFormat.byteLength
|
|
340
|
+
);
|
|
341
|
+
encoder.copyBufferToBuffer(
|
|
342
|
+
vBiasFormat.buffer,
|
|
343
|
+
0,
|
|
344
|
+
qkvBiasBuffer,
|
|
345
|
+
qBiasFormat.byteLength + kBiasFormat.byteLength,
|
|
346
|
+
vBiasFormat.byteLength
|
|
347
|
+
);
|
|
348
|
+
}
|
|
269
349
|
device.queue.submit([encoder.finish()]);
|
|
270
350
|
|
|
271
351
|
// Store fused buffer, sizes, and dtype
|
|
@@ -278,6 +358,10 @@ export function fuseQKVWeights(layerWeights, modelConfig, kernelPath = null, opt
|
|
|
278
358
|
);
|
|
279
359
|
weights.qkvSizes = [qSize, kSize, vSize];
|
|
280
360
|
weights.qkvDtype = dtype;
|
|
361
|
+
if (qkvBiasBuffer) {
|
|
362
|
+
tagBufferDtype(qkvBiasBuffer, qBiasFormat.dtype);
|
|
363
|
+
weights.qkvProjBias = qkvBiasBuffer;
|
|
364
|
+
}
|
|
281
365
|
if (qGateBuffer) {
|
|
282
366
|
weights.qGateProj = createWeightBuffer(
|
|
283
367
|
qGateBuffer,
|
|
@@ -237,7 +237,9 @@ export interface WeightLoadResult {
|
|
|
237
237
|
layerWeights: Map<string, LayerWeights>;
|
|
238
238
|
embeddings: GPUBuffer | WeightBuffer | CpuWeightBuffer | Float32Array | null;
|
|
239
239
|
lmHead: GPUBuffer | WeightBuffer | CpuWeightBuffer | Float32Array | null;
|
|
240
|
+
lmHeadBias: Float32Array | null;
|
|
240
241
|
finalNorm: GPUBuffer | Float32Array | null;
|
|
242
|
+
finalNormBias: Float32Array | null;
|
|
241
243
|
embeddingPostprocessor: LoadedEmbeddingPostprocessor | null;
|
|
242
244
|
diffusionGemmaSelfConditioning: DiffusionGemmaSelfConditioningWeights | null;
|
|
243
245
|
perLayerInputWeights: PerLayerInputWeights | null;
|
|
@@ -1020,7 +1020,9 @@ export async function loadWeights(manifest, modelConfig, options = {}) {
|
|
|
1020
1020
|
layerWeights,
|
|
1021
1021
|
embeddings: dopplerLoader.embeddings,
|
|
1022
1022
|
lmHead: dopplerLoader.lmHead,
|
|
1023
|
+
lmHeadBias: dopplerLoader.lmHeadBias,
|
|
1023
1024
|
finalNorm: dopplerLoader.finalNorm,
|
|
1025
|
+
finalNormBias: dopplerLoader.finalNormBias,
|
|
1024
1026
|
embeddingPostprocessor: dopplerLoader.embeddingPostprocessor,
|
|
1025
1027
|
diffusionGemmaSelfConditioning: dopplerLoader.diffusionGemmaSelfConditioning,
|
|
1026
1028
|
perLayerInputWeights: dopplerLoader.perLayerInputWeights,
|
|
@@ -57,6 +57,7 @@ function shouldUseSandwichRMSNormPairFusion({
|
|
|
57
57
|
attnOutput,
|
|
58
58
|
inputTensor,
|
|
59
59
|
}) {
|
|
60
|
+
if (context.config?.normalizationType === 'layernorm') return false;
|
|
60
61
|
const mergedSession = getRuntimeConfig()?.inference?.session;
|
|
61
62
|
if (mergedSession?.useSandwichRMSNormPairFusion !== true) {
|
|
62
63
|
return false;
|
|
@@ -123,6 +124,7 @@ function shouldUsePostFfnNextInputRMSNormPairFusion({
|
|
|
123
124
|
activationDtype,
|
|
124
125
|
layerScalar,
|
|
125
126
|
}) {
|
|
127
|
+
if (config.normalizationType === 'layernorm') return false;
|
|
126
128
|
const mergedSession = getRuntimeConfig()?.inference?.session;
|
|
127
129
|
if (mergedSession?.usePostFfnNextInputRMSNormPairFusion !== true) {
|
|
128
130
|
return false;
|
|
@@ -176,6 +178,7 @@ function shouldUseStandardPostFfnNextInputRMSNormPairFusion({
|
|
|
176
178
|
layerScalar,
|
|
177
179
|
residualBranchScale,
|
|
178
180
|
}) {
|
|
181
|
+
if (config.normalizationType === 'layernorm') return false;
|
|
179
182
|
const mergedSession = getRuntimeConfig()?.inference?.session;
|
|
180
183
|
if (mergedSession?.usePostFfnNextInputRMSNormPairFusion !== true) {
|
|
181
184
|
return false;
|
|
@@ -851,6 +854,8 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
|
|
|
851
854
|
attnSoftcap: config.attnLogitSoftcapping === null ? 0 : config.attnLogitSoftcapping,
|
|
852
855
|
queryPreAttnScalar: config.queryPreAttnScalar,
|
|
853
856
|
queryKeyNorm,
|
|
857
|
+
queryKeyNormType: config.queryKeyNormType,
|
|
858
|
+
queryKeyNormAxis: config.queryKeyNormAxis,
|
|
854
859
|
queryKeyNormWeightLayers: config.queryKeyNormWeightLayers,
|
|
855
860
|
valueNorm: config.valueNorm,
|
|
856
861
|
attentionOutputGate: config.attentionOutputGate,
|
|
@@ -860,6 +865,7 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
|
|
|
860
865
|
? (context.multimodalBidirectionalSpan ?? null)
|
|
861
866
|
: null,
|
|
862
867
|
rmsNormWeightOffset: config.rmsNormWeightOffset,
|
|
868
|
+
normalizationType: config.normalizationType,
|
|
863
869
|
ropeRotaryDim: resolveAttentionRotaryDim(config, layerType),
|
|
864
870
|
ropeFrequencyBaseDim: resolveAttentionFrequencyBaseDim(config, layerType),
|
|
865
871
|
ropeInterleaved: config.ropeInterleaved,
|
|
@@ -25,6 +25,12 @@ export function rmsNormCPU(
|
|
|
25
25
|
eps?: number,
|
|
26
26
|
rmsNormWeightOffset?: boolean
|
|
27
27
|
): Float32Array;
|
|
28
|
+
export function layerNormCPU(
|
|
29
|
+
x: Float32Array,
|
|
30
|
+
weight: Float32Array,
|
|
31
|
+
bias: Float32Array,
|
|
32
|
+
eps?: number
|
|
33
|
+
): Float32Array;
|
|
28
34
|
|
|
29
35
|
/**
|
|
30
36
|
* Convert a single float16 value to float32.
|
|
@@ -25,6 +25,28 @@ export function rmsNormCPU(
|
|
|
25
25
|
return result;
|
|
26
26
|
}
|
|
27
27
|
|
|
28
|
+
export function layerNormCPU(x, weight, bias, eps = 1e-5) {
|
|
29
|
+
if (weight.length !== x.length || bias.length !== x.length) {
|
|
30
|
+
throw new Error(
|
|
31
|
+
`LayerNorm parameter length mismatch: input=${x.length}, weight=${weight.length}, bias=${bias.length}.`
|
|
32
|
+
);
|
|
33
|
+
}
|
|
34
|
+
let sum = 0;
|
|
35
|
+
for (let i = 0; i < x.length; i++) sum += x[i];
|
|
36
|
+
const mean = sum / x.length;
|
|
37
|
+
let squaredDeviation = 0;
|
|
38
|
+
for (let i = 0; i < x.length; i++) {
|
|
39
|
+
const centered = x[i] - mean;
|
|
40
|
+
squaredDeviation += centered * centered;
|
|
41
|
+
}
|
|
42
|
+
const invStd = 1 / Math.sqrt(squaredDeviation / x.length + eps);
|
|
43
|
+
const result = new Float32Array(x.length);
|
|
44
|
+
for (let i = 0; i < x.length; i++) {
|
|
45
|
+
result[i] = (x[i] - mean) * invStd * weight[i] + bias[i];
|
|
46
|
+
}
|
|
47
|
+
return result;
|
|
48
|
+
}
|
|
49
|
+
|
|
28
50
|
|
|
29
51
|
export function f16BufferToF32(data) {
|
|
30
52
|
const u16 = new Uint16Array(data);
|
|
@@ -17,7 +17,7 @@ import type { LogitsConfig, LogitsWeights, LogitsDebugFlags } from './types.js';
|
|
|
17
17
|
export type { LogitsConfig, LogitsWeights, LogitsDebugFlags } from './types.js';
|
|
18
18
|
|
|
19
19
|
// Re-export CPU functions
|
|
20
|
-
export { rmsNormCPU, matmulCPU, applySoftcapping, f16ToF32, f16BufferToF32 } from './cpu.js';
|
|
20
|
+
export { layerNormCPU, rmsNormCPU, matmulCPU, applySoftcapping, f16ToF32, f16BufferToF32 } from './cpu.js';
|
|
21
21
|
|
|
22
22
|
// Re-export GPU functions
|
|
23
23
|
export { computeLogitsGPU, recordLogitsGPU, recordGreedyLmHeadArgmaxGPU, computeChunkedLogitsGPU, resolveCpuWeightDims, resolveLmHeadChunkRows, extractLmHeadChunk, writeChunkLogits } from './gpu.js';
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
|
|
2
2
|
|
|
3
3
|
// Re-export CPU functions
|
|
4
|
-
export { rmsNormCPU, matmulCPU, applySoftcapping, f16ToF32, f16BufferToF32 } from './cpu.js';
|
|
4
|
+
export { layerNormCPU, rmsNormCPU, matmulCPU, applySoftcapping, f16ToF32, f16BufferToF32 } from './cpu.js';
|
|
5
5
|
|
|
6
6
|
// Re-export GPU functions
|
|
7
7
|
export { computeLogitsGPU, recordLogitsGPU, recordGreedyLmHeadArgmaxGPU, computeChunkedLogitsGPU, resolveCpuWeightDims, resolveLmHeadChunkRows, extractLmHeadChunk, writeChunkLogits } from './gpu.js';
|
|
@@ -12,7 +12,7 @@ export { extractLastPositionLogits, finalizeLogits, readBufferWithCleanup } from
|
|
|
12
12
|
// Imports for computeLogits orchestrator
|
|
13
13
|
import { getDevice } from '../../../../gpu/device.js';
|
|
14
14
|
import { acquireBuffer, releaseBuffer, readBuffer } from '../../../../memory/buffer-pool.js';
|
|
15
|
-
import { runMatmul, runRMSNorm, runScale, castF16ToF32, castF32ToF16, runLmHeadSelectLogitsF16 } from '../../../../gpu/kernel-selector.js';
|
|
15
|
+
import { runLayerNorm, runMatmul, runRMSNorm, runScale, castF16ToF32, castF32ToF16, runLmHeadSelectLogitsF16 } from '../../../../gpu/kernel-selector.js';
|
|
16
16
|
import { createTensor } from '../../../../gpu/tensor.js';
|
|
17
17
|
import { isWeightBuffer, isCpuWeightBuffer, isGpuBufferInstance, isSplitWeightBuffer, getWeightDtype } from '../../../../gpu/weight-buffer.js';
|
|
18
18
|
import { kernelTrace, traceStep } from '../kernel-trace.js';
|
|
@@ -72,6 +72,45 @@ function normalizeSelectedLogitTokenIds(value, vocabSize) {
|
|
|
72
72
|
return tokenIds;
|
|
73
73
|
}
|
|
74
74
|
|
|
75
|
+
function applyLmHeadBias(logits, bias, rowCount, rowSize, selectedTokenIds = null) {
|
|
76
|
+
if (bias == null) {
|
|
77
|
+
return logits;
|
|
78
|
+
}
|
|
79
|
+
if (!(bias instanceof Float32Array)) {
|
|
80
|
+
throw new Error('[Logits] LM head bias must be a Float32Array.');
|
|
81
|
+
}
|
|
82
|
+
if (selectedTokenIds !== null) {
|
|
83
|
+
if (logits.length !== selectedTokenIds.length) {
|
|
84
|
+
throw new Error('[Logits] Selected-logit bias application length mismatch.');
|
|
85
|
+
}
|
|
86
|
+
for (let index = 0; index < selectedTokenIds.length; index += 1) {
|
|
87
|
+
const tokenId = selectedTokenIds[index];
|
|
88
|
+
if (tokenId >= bias.length) {
|
|
89
|
+
throw new Error(`[Logits] LM head bias is missing token ${tokenId}.`);
|
|
90
|
+
}
|
|
91
|
+
logits[index] += bias[tokenId];
|
|
92
|
+
}
|
|
93
|
+
return logits;
|
|
94
|
+
}
|
|
95
|
+
if (bias.length < rowSize) {
|
|
96
|
+
throw new Error(
|
|
97
|
+
`[Logits] LM head bias length mismatch: expected at least ${rowSize}, got ${bias.length}.`
|
|
98
|
+
);
|
|
99
|
+
}
|
|
100
|
+
if (logits.length !== rowCount * rowSize) {
|
|
101
|
+
throw new Error(
|
|
102
|
+
`[Logits] LM head bias matrix mismatch: expected ${rowCount * rowSize} logits, got ${logits.length}.`
|
|
103
|
+
);
|
|
104
|
+
}
|
|
105
|
+
for (let row = 0; row < rowCount; row += 1) {
|
|
106
|
+
const offset = row * rowSize;
|
|
107
|
+
for (let column = 0; column < rowSize; column += 1) {
|
|
108
|
+
logits[offset + column] += bias[column];
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
return logits;
|
|
112
|
+
}
|
|
113
|
+
|
|
75
114
|
async function coerceTensorDtype(tensor, targetDtype, options = {}) {
|
|
76
115
|
if (!targetDtype || tensor.dtype === targetDtype) {
|
|
77
116
|
return tensor;
|
|
@@ -164,7 +203,7 @@ export async function computeLogits(
|
|
|
164
203
|
activationDtype: activationDtypeOverride,
|
|
165
204
|
} = config;
|
|
166
205
|
const activationDtype = activationDtypeOverride ?? getRuntimeConfig().inference.compute.activationDtype;
|
|
167
|
-
const { finalNorm, lmHead } = weights;
|
|
206
|
+
const { finalNorm, finalNormBias = null, lmHead, lmHeadBias = null } = weights;
|
|
168
207
|
const device = getDevice();
|
|
169
208
|
|
|
170
209
|
// Consistency check: warn if lmHead weight dtype disagrees with layer activation dtype.
|
|
@@ -269,6 +308,7 @@ export async function computeLogits(
|
|
|
269
308
|
cpuWeightLayout === 'column' ? cpuWeightVocabSize : null
|
|
270
309
|
)
|
|
271
310
|
: matmulCPU(normed, (lmHead), numTokens, matmulVocabSize, hiddenSize);
|
|
311
|
+
applyLmHeadBias(rawLogits, lmHeadBias, numTokens, matmulVocabSize);
|
|
272
312
|
return finalizeLogits(rawLogits, numTokens, matmulVocabSize, vocabSize, config, debugProbes, operatorDiagnostics);
|
|
273
313
|
}
|
|
274
314
|
|
|
@@ -293,7 +333,7 @@ export async function computeLogits(
|
|
|
293
333
|
dtype: inputDtype,
|
|
294
334
|
});
|
|
295
335
|
|
|
296
|
-
// 2. Apply final
|
|
336
|
+
// 2. Apply the manifest-owned final normalization.
|
|
297
337
|
|
|
298
338
|
let normWeightBuffer;
|
|
299
339
|
let normWeightBufferOwned = false;
|
|
@@ -319,7 +359,10 @@ export async function computeLogits(
|
|
|
319
359
|
const kernelPath = config.kernelPath ?? null;
|
|
320
360
|
const finalNormPrecision = getKernelPathStepPrecision('final_norm', 'postLayer', phase, 0, kernelPath);
|
|
321
361
|
const hasExplicitFinalNormPrecision = finalNormPrecision?.inputDtype != null || finalNormPrecision?.outputDtype != null;
|
|
322
|
-
const forceStableF32Logits = !hasExplicitFinalNormPrecision &&
|
|
362
|
+
const forceStableF32Logits = !hasExplicitFinalNormPrecision && (
|
|
363
|
+
config.normalizationType === 'layernorm'
|
|
364
|
+
|| shouldForceStableF32Logits(config, inputDtype)
|
|
365
|
+
);
|
|
323
366
|
const stableKernelPath = forceStableF32Logits
|
|
324
367
|
? createStableF32LogitsKernelPath(kernelPath)
|
|
325
368
|
: kernelPath;
|
|
@@ -346,11 +389,39 @@ export async function computeLogits(
|
|
|
346
389
|
: inputTensor;
|
|
347
390
|
normInputBufferOwned = normInputTensor !== inputTensor;
|
|
348
391
|
}
|
|
349
|
-
let
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
392
|
+
let finalNormBiasBuffer = null;
|
|
393
|
+
let normedTensor;
|
|
394
|
+
if (config.normalizationType === 'layernorm') {
|
|
395
|
+
if (config.finalNormBiasTensor !== null && !(finalNormBias instanceof Float32Array)) {
|
|
396
|
+
throw new Error(
|
|
397
|
+
`[Logits] LayerNorm declares bias tensor "${config.finalNormBiasTensor}" but it was not loaded.`
|
|
398
|
+
);
|
|
399
|
+
}
|
|
400
|
+
const effectiveBias = finalNormBias instanceof Float32Array
|
|
401
|
+
? finalNormBias
|
|
402
|
+
: new Float32Array(hiddenSize);
|
|
403
|
+
if (effectiveBias.length !== hiddenSize) {
|
|
404
|
+
throw new Error(
|
|
405
|
+
`[Logits] LayerNorm final bias length must be ${hiddenSize}; got ${effectiveBias.length}.`
|
|
406
|
+
);
|
|
407
|
+
}
|
|
408
|
+
finalNormBiasBuffer = acquireBuffer(effectiveBias.byteLength, undefined, 'final_norm_bias');
|
|
409
|
+
device.queue.writeBuffer(finalNormBiasBuffer, 0, effectiveBias);
|
|
410
|
+
normedTensor = await runLayerNorm(
|
|
411
|
+
normInputTensor,
|
|
412
|
+
normWeightBuffer,
|
|
413
|
+
finalNormBiasBuffer,
|
|
414
|
+
rmsNormEps,
|
|
415
|
+
{ batchSize: numTokens, hiddenSize }
|
|
416
|
+
);
|
|
417
|
+
} else {
|
|
418
|
+
normedTensor = await runRMSNorm(normInputTensor, normWeightBuffer, rmsNormEps, {
|
|
419
|
+
batchSize: numTokens,
|
|
420
|
+
hiddenSize,
|
|
421
|
+
rmsNormWeightOffset: config.rmsNormWeightOffset,
|
|
422
|
+
});
|
|
423
|
+
}
|
|
424
|
+
if (finalNormBiasBuffer) releaseBuffer(finalNormBiasBuffer);
|
|
354
425
|
if (normInputBufferOwned) {
|
|
355
426
|
releaseBuffer(normInputTensor.buffer);
|
|
356
427
|
}
|
|
@@ -386,7 +457,7 @@ export async function computeLogits(
|
|
|
386
457
|
|
|
387
458
|
// Trace final norm output
|
|
388
459
|
if (kernelTrace.enabled) {
|
|
389
|
-
await traceStep(
|
|
460
|
+
await traceStep(config.normalizationType, 'final_norm', -1, finalNormTensor.buffer, [numTokens, hiddenSize]);
|
|
390
461
|
}
|
|
391
462
|
|
|
392
463
|
// Debug: Check hidden state after final norm
|
|
@@ -461,6 +532,7 @@ export async function computeLogits(
|
|
|
461
532
|
if (matmulInputOwned) releaseBuffer(matmulInputTensor.buffer);
|
|
462
533
|
if (normWeightBufferOwned) releaseBuffer(normWeightBuffer);
|
|
463
534
|
|
|
535
|
+
applyLmHeadBias(rawLogits, lmHeadBias, matmulRows, matmulVocabSize);
|
|
464
536
|
return finalizeLogits(rawLogits, matmulRows, matmulVocabSize, vocabSize, config, debugProbes, operatorDiagnostics);
|
|
465
537
|
}
|
|
466
538
|
|
|
@@ -533,6 +605,7 @@ export async function computeLogits(
|
|
|
533
605
|
if (lmHeadBufferOwned) releaseBuffer(lmHeadGPU);
|
|
534
606
|
});
|
|
535
607
|
const selectedLogits = new Float32Array(selectedData);
|
|
608
|
+
applyLmHeadBias(selectedLogits, lmHeadBias, 1, selectedTokenIds.length, selectedTokenIds);
|
|
536
609
|
await runProbes('logits_final', selectedLogits, {
|
|
537
610
|
numTokens: 1,
|
|
538
611
|
hiddenSize: selectedTokenIds.length,
|
|
@@ -579,6 +652,7 @@ export async function computeLogits(
|
|
|
579
652
|
const rawLogits = logitsTensor.dtype === 'f16'
|
|
580
653
|
? f16BufferToF32(logitsData)
|
|
581
654
|
: new Float32Array(logitsData);
|
|
655
|
+
applyLmHeadBias(rawLogits, lmHeadBias, matmulRows, matmulVocabSize);
|
|
582
656
|
if (isTraceEnabled('logits')) {
|
|
583
657
|
trace.logits('LM_HEAD_RAW_LOGITS_HEALTH', getLogitsHealth(rawLogits));
|
|
584
658
|
}
|
|
@@ -17,6 +17,8 @@ import type { WeightBuffer, CpuWeightBuffer, SplitWeightBuffer } from '../../../
|
|
|
17
17
|
export interface LogitsConfig {
|
|
18
18
|
hiddenSize: number;
|
|
19
19
|
vocabSize: number;
|
|
20
|
+
normalizationType: 'rmsnorm' | 'layernorm';
|
|
21
|
+
finalNormBiasTensor: string | null;
|
|
20
22
|
rmsNormEps: number;
|
|
21
23
|
useTiedEmbeddings: boolean;
|
|
22
24
|
embeddingVocabSize: number | null;
|
|
@@ -38,7 +40,9 @@ export interface LogitsConfig {
|
|
|
38
40
|
*/
|
|
39
41
|
export interface LogitsWeights {
|
|
40
42
|
finalNorm: GPUBuffer | Float32Array;
|
|
43
|
+
finalNormBias?: Float32Array | null;
|
|
41
44
|
lmHead: GPUBuffer | Float32Array | WeightBuffer | CpuWeightBuffer | SplitWeightBuffer;
|
|
45
|
+
lmHeadBias?: Float32Array | null;
|
|
42
46
|
}
|
|
43
47
|
|
|
44
48
|
/**
|
|
@@ -2,7 +2,9 @@
|
|
|
2
2
|
|
|
3
3
|
import {
|
|
4
4
|
runRMSNorm, runResidualAdd, runMatmul, runSiLU, runGeLU,
|
|
5
|
+
runLayerNorm, runBiasAdd,
|
|
5
6
|
recordRMSNorm, recordResidualAdd, recordMatmul, recordSiLU, recordGeLU,
|
|
7
|
+
recordLayerNorm, recordBiasAdd,
|
|
6
8
|
runSiLURowSplit, recordSiLURowSplit,
|
|
7
9
|
runMatmulRMSNormFused, recordMatmulRMSNormFused,
|
|
8
10
|
runRMSNormStats, recordRMSNormStats,
|
|
@@ -60,6 +62,36 @@ export async function doRMSNorm(input, weight, eps, options, recorder) {
|
|
|
60
62
|
return result;
|
|
61
63
|
}
|
|
62
64
|
|
|
65
|
+
export async function doLayerNorm(input, weight, bias, eps, options, recorder) {
|
|
66
|
+
const result = recorder
|
|
67
|
+
? await recordLayerNorm(recorder, input, weight, bias, eps, options)
|
|
68
|
+
: await runLayerNorm(input, weight, bias, eps, options);
|
|
69
|
+
|
|
70
|
+
if (kernelTrace.enabled && !recorder) {
|
|
71
|
+
const layer = options.layerIdx ?? -1;
|
|
72
|
+
const label = options.label ?? 'layernorm';
|
|
73
|
+
await traceStep('layernorm', label, layer, result.buffer, [options.batchSize, options.hiddenSize]);
|
|
74
|
+
}
|
|
75
|
+
return result;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
export async function doBiasAdd(data, bias, numTokens, dim, options, recorder) {
|
|
79
|
+
const result = recorder
|
|
80
|
+
? await recordBiasAdd(recorder, data, bias, numTokens, dim, options)
|
|
81
|
+
: await runBiasAdd(data, bias, numTokens, dim, options);
|
|
82
|
+
|
|
83
|
+
if (kernelTrace.enabled && !recorder) {
|
|
84
|
+
await traceStep(
|
|
85
|
+
'bias_add',
|
|
86
|
+
options?.label ?? 'bias_add',
|
|
87
|
+
options?.layerIdx ?? -1,
|
|
88
|
+
result.buffer,
|
|
89
|
+
[numTokens, dim]
|
|
90
|
+
);
|
|
91
|
+
}
|
|
92
|
+
return result;
|
|
93
|
+
}
|
|
94
|
+
|
|
63
95
|
export async function doRMSNormStats(input, residual, eps, options, recorder) {
|
|
64
96
|
return recorder
|
|
65
97
|
? await recordRMSNormStats(recorder, input, residual, eps, options)
|