doppler-gpu 0.4.6 → 0.4.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +43 -31
- package/assets/doppler-runtime-map.svg +1 -3
- package/assets/doppler.svg +1 -3
- package/package.json +25 -17
- package/src/cli/doppler-quickstart.js +1 -1
- package/src/cli/doppler-serve.js +2 -2
- package/src/client/doppler-registry.d.ts +19 -0
- package/src/client/doppler-registry.js +12 -0
- package/src/client/doppler-registry.json +50 -1
- package/src/config/conversion/README.md +4 -0
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +28 -4
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +31 -5
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +30 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +30 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +30 -4
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +30 -5
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +30 -5
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +29 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +29 -4
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +29 -4
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/minicpm/minicpm4-0-5b-f16-af32.json +481 -0
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +29 -4
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +29 -4
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +30 -5
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +29 -4
- package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-q4k-ehf16-af32.json +395 -0
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-af32.json +429 -0
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +540 -0
- package/src/config/execution-contract-check.js +14 -10
- package/src/config/merge-contract-check.js +1 -0
- package/src/config/merge.js +30 -0
- package/src/config/param-validator.d.ts +5 -0
- package/src/config/param-validator.js +42 -8
- package/src/config/required-inference-fields-contract-check.js +14 -0
- package/src/config/schema/debug.schema.d.ts +2 -0
- package/src/config/schema/debug.schema.js +1 -0
- package/src/config/schema/doppler.schema.d.ts +5 -1
- package/src/config/schema/doppler.schema.js +82 -292
- package/src/config/schema/inference.schema.d.ts +17 -3
- package/src/config/schema/kvcache.schema.d.ts +19 -3
- package/src/config/schema/kvcache.schema.js +2 -1
- package/src/config/schema/manifest.schema.d.ts +38 -1
- package/src/config/schema/manifest.schema.js +18 -0
- package/src/config/schema/policy-schema-registry.json +5 -0
- package/src/config/source-packages/litert/gemma-4-12b-it.json +20 -1
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +20 -1
- package/src/config/source-packages/litert/gemma-4-e4b-it.json +21 -1
- package/src/config/transforms/execution-graph-transforms.d.ts +6 -0
- package/src/config/transforms/execution-graph-transforms.js +8 -0
- package/src/converter/conversion-plan.js +2 -0
- package/src/converter/parsers/transformer.js +27 -1
- package/src/converter/quantization-info.js +16 -4
- package/src/converter/rope-config.js +55 -3
- package/src/experimental/training/distillation/student-fixture.js +1 -1
- package/src/experimental/training/lora-pipeline.js +1 -1
- package/src/experimental/training/lora.js +2 -2
- package/src/experimental/training/suite.js +1 -1
- package/src/gpu/kernels/README.md +8 -1
- package/src/gpu/kernels/backward/attention_backward.js +8 -0
- package/src/gpu/kernels/dequant.d.ts +6 -6
- package/src/gpu/kernels/dequant.js +17 -6
- package/src/gpu/kernels/linear-attention-core.js +1 -6
- package/src/gpu/kernels/matmul-selection.d.ts +2 -2
- package/src/gpu/kernels/matmul-selection.js +4 -1
- package/src/gpu/kernels/matmul.d.ts +1 -1
- package/src/gpu/kernels/matmul.js +24 -6
- package/src/gpu/kernels/moe.d.ts +5 -1
- package/src/gpu/kernels/moe.js +5 -2
- package/src/gpu/kernels/softmax.d.ts +3 -3
- package/src/gpu/kernels/softmax.js +9 -2
- package/src/gpu/kernels/turboquant-codebook.js +1 -1
- package/src/gpu/kernels/types.d.ts +1 -1
- package/src/gpu/profiler.js +11 -1
- package/src/inference/browser-harness-model-helpers.js +4 -1
- package/src/inference/browser-harness-text-helpers.d.ts +28 -0
- package/src/inference/browser-harness-text-helpers.js +357 -116
- package/src/inference/browser-harness.d.ts +3 -3
- package/src/inference/browser-harness.js +266 -11
- package/src/inference/fixtures/rerank-semantic-fixtures.json +38 -0
- package/src/inference/kv-cache/quantized.js +16 -2
- package/src/inference/kv-cache/tiered.js +50 -15
- package/src/inference/moe-router.js +38 -7
- package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +1 -1
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +8 -6
- package/src/inference/pipelines/text/config.d.ts +5 -1
- package/src/inference/pipelines/text/config.js +135 -112
- package/src/inference/pipelines/text/embed.d.ts +1 -0
- package/src/inference/pipelines/text/embed.js +39 -12
- package/src/inference/pipelines/text/execution-plan.js +8 -2
- package/src/inference/pipelines/text/ffn/dense.js +15 -4
- package/src/inference/pipelines/text/ffn/moe.js +4 -3
- package/src/inference/pipelines/text/ffn/standard.d.ts +2 -1
- package/src/inference/pipelines/text/ffn/standard.js +25 -2
- package/src/inference/pipelines/text/generator-helpers.js +2 -1
- package/src/inference/pipelines/text/generator-runtime.d.ts +6 -2
- package/src/inference/pipelines/text/generator-runtime.js +53 -20
- package/src/inference/pipelines/text/generator-steps.js +3 -0
- package/src/inference/pipelines/text/generator.js +6 -1
- package/src/inference/pipelines/text/init-rope.js +54 -6
- package/src/inference/pipelines/text/init.js +63 -15
- package/src/inference/pipelines/text/layer.js +32 -5
- package/src/inference/pipelines/text/linear-attention.js +12 -8
- package/src/inference/pipelines/text/logits/gpu.d.ts +1 -1
- package/src/inference/pipelines/text/logits/gpu.js +76 -113
- package/src/inference/pipelines/text/logits/index.js +44 -87
- package/src/inference/pipelines/text/logits/precision-policy.d.ts +11 -0
- package/src/inference/pipelines/text/logits/precision-policy.js +103 -0
- package/src/inference/pipelines/text/logits/types.d.ts +1 -0
- package/src/inference/pipelines/text/logits/utils.d.ts +2 -0
- package/src/inference/pipelines/text/logits/utils.js +11 -0
- package/src/inference/pipelines/text/lora-apply.js +6 -4
- package/src/inference/pipelines/text/moe-cpu.js +25 -0
- package/src/inference/pipelines/text/moe-gpu.js +148 -97
- package/src/inference/pipelines/text/moe-shape-validator.d.ts +33 -3
- package/src/inference/pipelines/text/moe-shape-validator.js +149 -31
- package/src/inference/pipelines/text/ops.d.ts +3 -0
- package/src/inference/pipelines/text/ops.js +26 -1
- package/src/inference/pipelines/text/per-layer-inputs.js +27 -12
- package/src/inference/pipelines/text/weights.js +7 -2
- package/src/inference/pipelines/text.js +24 -4
- package/src/inference/tokenizers/behavior-flags.js +23 -2
- package/src/loader/final-weights-loader.js +12 -1
- package/src/rules/execution-rules-contract-check.js +12 -3
- package/src/rules/inference/capability-transforms.rules.json +14 -2
- package/src/rules/inference/config.rules.json +26 -2
- package/src/rules/inference/dtype.rules.json +12 -5
- package/src/rules/inference/execution.rules.json +5 -19
- package/src/rules/inference/moe.rules.json +97 -0
- package/src/tooling/command-api-constants.js +2 -2
- package/src/tooling/command-api-family-normalizers.js +2 -1
- package/src/tooling/command-api.d.ts +1 -1
- package/src/tooling/command-runner-shared.d.ts +1 -1
- package/src/tooling/command-runner-shared.js +3 -1
- package/src/tooling/node-converter.js +20 -0
- package/src/tooling/node-source-runtime.js +5 -1
- package/src/version.js +1 -1
- package/assets/doppler-readme-hero.svg +0 -84
|
@@ -7,6 +7,7 @@ import { isMoELayerLocal } from './types.js';
|
|
|
7
7
|
import { runDenseFFNGPU } from './dense.js';
|
|
8
8
|
import { runMoEFFNGPU } from './moe.js';
|
|
9
9
|
import { acquireBuffer, readBuffer } from '../../../../memory/buffer-pool.js';
|
|
10
|
+
import { runScale, recordScale } from '../../../../gpu/kernel-selector.js';
|
|
10
11
|
import { isGpuBufferInstance, isWeightBuffer } from '../../../../gpu/weight-buffer.js';
|
|
11
12
|
import { shouldDebugLayerOutput, decodeReadback, getLogitsHealth } from '../debug-utils/index.js';
|
|
12
13
|
import { trace, isTraceEnabled } from '../../../../debug/index.js';
|
|
@@ -44,7 +45,8 @@ export async function processFFNStandard(
|
|
|
44
45
|
context,
|
|
45
46
|
layerWeights,
|
|
46
47
|
fusedResidualInput,
|
|
47
|
-
finalOutputScale = 1
|
|
48
|
+
finalOutputScale = 1,
|
|
49
|
+
residualBranchScale = 1
|
|
48
50
|
) {
|
|
49
51
|
const { config, weightConfig, debugFlags, recorder, decodeBuffers } = context;
|
|
50
52
|
const { hiddenSize, rmsNormEps } = config;
|
|
@@ -52,6 +54,13 @@ export async function processFFNStandard(
|
|
|
52
54
|
if (!Number.isFinite(requestedFinalOutputScale)) {
|
|
53
55
|
throw new Error(`Layer ${layerIdx} finalOutputScale must be finite; got "${String(finalOutputScale)}".`);
|
|
54
56
|
}
|
|
57
|
+
const requestedResidualBranchScale = Number(residualBranchScale);
|
|
58
|
+
if (!Number.isFinite(requestedResidualBranchScale) || requestedResidualBranchScale <= 0) {
|
|
59
|
+
throw new Error(
|
|
60
|
+
`Layer ${layerIdx} residualBranchScale must be a positive finite number; ` +
|
|
61
|
+
`got "${String(residualBranchScale)}".`
|
|
62
|
+
);
|
|
63
|
+
}
|
|
55
64
|
context.__layerScalarFusedFired = false;
|
|
56
65
|
|
|
57
66
|
const decodeOutputBuffer = numTokens === 1 && decodeBuffers
|
|
@@ -112,7 +121,9 @@ export async function processFFNStandard(
|
|
|
112
121
|
const pendingResidualForFfn = prenormSumBuffer
|
|
113
122
|
? { buffer: prenormSumBuffer, dtype: postAttn.dtype, shape: postAttn.shape }
|
|
114
123
|
: postAttn;
|
|
115
|
-
context.__pendingFfnResidualTensor =
|
|
124
|
+
context.__pendingFfnResidualTensor = requestedResidualBranchScale === 1
|
|
125
|
+
? pendingResidualForFfn
|
|
126
|
+
: null;
|
|
116
127
|
context.__ffnResidualFusedFired = false;
|
|
117
128
|
|
|
118
129
|
let ffnOutput;
|
|
@@ -156,6 +167,18 @@ export async function processFFNStandard(
|
|
|
156
167
|
if (residualInputOwned) {
|
|
157
168
|
residualInput = await doCast(ffnOutput, residualTensor.dtype, recorder);
|
|
158
169
|
}
|
|
170
|
+
if (requestedResidualBranchScale !== 1) {
|
|
171
|
+
const unscaledResidualInput = residualInput;
|
|
172
|
+
residualInput = recorder
|
|
173
|
+
? await recordScale(recorder, unscaledResidualInput, requestedResidualBranchScale, { count: size })
|
|
174
|
+
: await runScale(unscaledResidualInput, requestedResidualBranchScale, { count: size });
|
|
175
|
+
if (residualInputOwned) {
|
|
176
|
+
releaseOrTrack(recorder, unscaledResidualInput.buffer, decodeBuffers);
|
|
177
|
+
} else if (unscaledResidualInput.buffer !== ffnOutput.buffer) {
|
|
178
|
+
releaseOrTrack(recorder, unscaledResidualInput.buffer, decodeBuffers);
|
|
179
|
+
}
|
|
180
|
+
residualInputOwned = true;
|
|
181
|
+
}
|
|
159
182
|
const residualOutputScale = requestedFinalOutputScale !== 1
|
|
160
183
|
&& residualTensor.dtype === context.activationDtype
|
|
161
184
|
&& !context.debugProbes?.length
|
|
@@ -202,7 +202,7 @@ function getWeightBufferConfig(state) {
|
|
|
202
202
|
export function getLogitsWeights(state) {
|
|
203
203
|
const finalNorm = state.weights.get('final_norm');
|
|
204
204
|
const lmHead = state.weights.get('lm_head');
|
|
205
|
-
if (!finalNorm || !(isGpuBufferInstance(finalNorm) || finalNorm instanceof Float32Array)) {
|
|
205
|
+
if (!finalNorm || !(isGpuBufferInstance(finalNorm) || finalNorm instanceof Float32Array || isWeightBuffer(finalNorm))) {
|
|
206
206
|
throw new Error('Final norm not found or invalid type');
|
|
207
207
|
}
|
|
208
208
|
if (!lmHead || !(isGpuBufferInstance(lmHead) || lmHead instanceof Float32Array || isWeightBuffer(lmHead) || isCpuWeightBuffer(lmHead) || isSplitWeightBuffer(lmHead))) {
|
|
@@ -225,6 +225,7 @@ export function getLogitsConfig(state) {
|
|
|
225
225
|
useTiedEmbeddings: state.useTiedEmbeddings,
|
|
226
226
|
embeddingVocabSize: state.embeddingVocabSize,
|
|
227
227
|
finalLogitSoftcapping: config.finalLogitSoftcapping,
|
|
228
|
+
logitInputScale: config.logitInputScale,
|
|
228
229
|
largeWeights: state.runtimeConfig.inference.largeWeights,
|
|
229
230
|
activationDtype: effectiveActivationDtype,
|
|
230
231
|
kernelPath: activeKernelPath,
|
|
@@ -72,8 +72,12 @@ export function resolveAdvanceEmbeddingMode(
|
|
|
72
72
|
|
|
73
73
|
export function resolveFloatDtypeFromByteSize(
|
|
74
74
|
totalBytes: number,
|
|
75
|
-
expectedLength: number
|
|
76
|
-
|
|
75
|
+
expectedLength: number
|
|
76
|
+
): 'f16' | 'f32';
|
|
77
|
+
|
|
78
|
+
export function resolveFloatDtypeFromBufferMetadata(
|
|
79
|
+
buffer: GPUBuffer,
|
|
80
|
+
expectedLength: number
|
|
77
81
|
): 'f16' | 'f32';
|
|
78
82
|
|
|
79
83
|
export function getFinalNormWeights(state: PipelineState): Promise<Float32Array>;
|
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
import { readBuffer } from '../../../memory/buffer-pool.js';
|
|
2
2
|
import { matmulCPU, rmsNormCPU } from './logits/index.js';
|
|
3
|
-
import { isGpuBufferInstance, isWeightBuffer, isCpuWeightBuffer } from '../../../gpu/weight-buffer.js';
|
|
3
|
+
import { isGpuBufferInstance, isWeightBuffer, isCpuWeightBuffer, getBufferDtype } from '../../../gpu/weight-buffer.js';
|
|
4
4
|
import { decodeReadback } from './debug-utils/index.js';
|
|
5
5
|
import { resolveExecutionSessionPlan } from './execution-plan.js';
|
|
6
6
|
import { selectRuleValue } from '../../../rules/rule-registry.js';
|
|
7
|
+
import { validateSelfSpeculationConfig } from '../../../config/schema/speculation-self.schema.js';
|
|
7
8
|
|
|
8
9
|
const UNKNOWN_TOKENIZER_VOCAB_SIZE = 'unknown';
|
|
9
|
-
const
|
|
10
|
+
const INVALID_DTYPE_SENTINEL = '__invalid_dtype__';
|
|
10
11
|
|
|
11
12
|
function resolveConfiguredValue(value, defaultValue, context, validate) {
|
|
12
13
|
if (value === undefined) {
|
|
@@ -242,16 +243,28 @@ export function resolveGenerateOptions(state, options = {}) {
|
|
|
242
243
|
}
|
|
243
244
|
|
|
244
245
|
function resolveSpeculationConfig(state, options) {
|
|
245
|
-
const sessionSpeculation = state.runtimeConfig?.inference?.session?.speculation
|
|
246
|
+
const sessionSpeculation = state.runtimeConfig?.inference?.session?.speculation;
|
|
246
247
|
const callSpeculation = options.speculation ?? null;
|
|
247
|
-
if (
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
248
|
+
if (sessionSpeculation === undefined) {
|
|
249
|
+
throw new Error('[Pipeline] runtime.inference.session.speculation is required.');
|
|
250
|
+
}
|
|
251
|
+
if (callSpeculation === null) {
|
|
252
|
+
if (sessionSpeculation === null) return null;
|
|
253
|
+
validateSelfSpeculationConfig(sessionSpeculation);
|
|
254
|
+
return sessionSpeculation.mode === 'none' ? null : sessionSpeculation;
|
|
255
|
+
}
|
|
256
|
+
if (!sessionSpeculation || typeof sessionSpeculation !== 'object') {
|
|
257
|
+
throw new Error('[Pipeline] options.speculation requires runtime.inference.session.speculation defaults.');
|
|
258
|
+
}
|
|
259
|
+
if (!callSpeculation || typeof callSpeculation !== 'object') {
|
|
260
|
+
throw new Error('[Pipeline] options.speculation must be an object.');
|
|
261
|
+
}
|
|
262
|
+
const merged = {
|
|
263
|
+
...sessionSpeculation,
|
|
264
|
+
...callSpeculation,
|
|
254
265
|
};
|
|
266
|
+
validateSelfSpeculationConfig(merged);
|
|
267
|
+
return merged.mode === 'none' ? null : merged;
|
|
255
268
|
}
|
|
256
269
|
|
|
257
270
|
export function resolvePrefillOptions(state, options = {}) {
|
|
@@ -333,22 +346,39 @@ export function resolveAdvanceEmbeddingMode(state, options = {}) {
|
|
|
333
346
|
|
|
334
347
|
function resolveFloatDtypeFromAlias(dtype) {
|
|
335
348
|
const normalized = typeof dtype === 'string' ? dtype.trim().toLowerCase() : '';
|
|
336
|
-
if (!normalized)
|
|
337
|
-
|
|
349
|
+
if (!normalized) {
|
|
350
|
+
throw new Error('[Pipeline] float weight dtype is required.');
|
|
351
|
+
}
|
|
352
|
+
const resolved = selectRuleValue('inference', 'dtype', 'dtypeFromAlias', {
|
|
338
353
|
dtype: normalized,
|
|
339
|
-
fallback:
|
|
354
|
+
fallback: INVALID_DTYPE_SENTINEL,
|
|
340
355
|
});
|
|
356
|
+
if (resolved === INVALID_DTYPE_SENTINEL) {
|
|
357
|
+
throw new Error(`[Pipeline] Unsupported float weight dtype "${dtype}".`);
|
|
358
|
+
}
|
|
359
|
+
return resolved;
|
|
341
360
|
}
|
|
342
361
|
|
|
343
362
|
export function resolveFloatDtypeFromByteSize(totalBytes, expectedLength) {
|
|
344
363
|
if (!Number.isFinite(totalBytes) || totalBytes <= 0 || !Number.isFinite(expectedLength) || expectedLength <= 0) {
|
|
345
|
-
|
|
364
|
+
throw new Error(
|
|
365
|
+
`[Pipeline] Cannot infer float dtype from invalid size metadata (totalBytes=${totalBytes}, expectedLength=${expectedLength}).`
|
|
366
|
+
);
|
|
346
367
|
}
|
|
347
368
|
const bytesPerElement = totalBytes / expectedLength;
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
369
|
+
if (bytesPerElement !== 2 && bytesPerElement !== 4) {
|
|
370
|
+
throw new Error(
|
|
371
|
+
`[Pipeline] Cannot infer float dtype from bytesPerElement=${bytesPerElement}; expected 2 or 4.`
|
|
372
|
+
);
|
|
373
|
+
}
|
|
374
|
+
return selectRuleValue('inference', 'dtype', 'f16OrF32FromBytes', { bytesPerElement });
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
export function resolveFloatDtypeFromBufferMetadata(buffer, expectedLength) {
|
|
378
|
+
const taggedDtype = getBufferDtype(buffer);
|
|
379
|
+
return taggedDtype
|
|
380
|
+
? resolveFloatDtypeFromAlias(taggedDtype)
|
|
381
|
+
: resolveFloatDtypeFromByteSize(buffer?.size, expectedLength);
|
|
352
382
|
}
|
|
353
383
|
|
|
354
384
|
function decodeFloatWeights(data, dtype, expectedLength, label) {
|
|
@@ -385,8 +415,11 @@ export async function getFinalNormWeights(state) {
|
|
|
385
415
|
const dtypeValue = typeof finalNorm.dtype === 'string' ? finalNorm.dtype.trim().toLowerCase() : '';
|
|
386
416
|
const dtype = selectRuleValue('inference', 'dtype', 'f16OrF32FromDtypeAlias', {
|
|
387
417
|
dtype: dtypeValue === '' ? undefined : dtypeValue,
|
|
388
|
-
fallback:
|
|
418
|
+
fallback: INVALID_DTYPE_SENTINEL,
|
|
389
419
|
});
|
|
420
|
+
if (dtype === INVALID_DTYPE_SENTINEL) {
|
|
421
|
+
throw new Error(`[Pipeline] Unsupported final_norm dtype "${finalNorm.dtype}".`);
|
|
422
|
+
}
|
|
390
423
|
const bytesPerElement = selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype });
|
|
391
424
|
const readSize = hiddenSize * bytesPerElement;
|
|
392
425
|
const data = await readBuffer(finalNorm.buffer, readSize);
|
|
@@ -395,7 +428,7 @@ export async function getFinalNormWeights(state) {
|
|
|
395
428
|
}
|
|
396
429
|
weights = decodeFloatWeights(data, dtype, hiddenSize, 'final_norm');
|
|
397
430
|
} else if (isGpuBufferInstance(finalNorm)) {
|
|
398
|
-
const dtype =
|
|
431
|
+
const dtype = resolveFloatDtypeFromBufferMetadata(finalNorm, hiddenSize);
|
|
399
432
|
const bytesPerElement = selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype });
|
|
400
433
|
const readSize = hiddenSize * bytesPerElement;
|
|
401
434
|
const data = await readBuffer(finalNorm, readSize);
|
|
@@ -467,6 +467,7 @@ async function runDecodeLayers(state, tokenId, opts, helpers) {
|
|
|
467
467
|
hiddenSize: config.hiddenSize,
|
|
468
468
|
vocabSize: config.vocabSize,
|
|
469
469
|
scaleEmbeddings: config.scaleEmbeddings,
|
|
470
|
+
embeddingScale: config.embeddingScale,
|
|
470
471
|
outputBuffer: decodeHiddenBuffer ?? undefined,
|
|
471
472
|
transpose: state.embeddingTranspose,
|
|
472
473
|
debugProbes: state.runtimeConfig.shared.debug.probes,
|
|
@@ -619,6 +620,7 @@ export async function decodeStep(state, currentIds, opts, helpers) {
|
|
|
619
620
|
hiddenSize: config.hiddenSize,
|
|
620
621
|
vocabSize: config.vocabSize,
|
|
621
622
|
scaleEmbeddings: config.scaleEmbeddings,
|
|
623
|
+
embeddingScale: config.embeddingScale,
|
|
622
624
|
recorder,
|
|
623
625
|
outputBuffer: decodeHiddenBuffer ?? undefined,
|
|
624
626
|
transpose: state.embeddingTranspose,
|
|
@@ -1527,6 +1529,7 @@ export async function generateNTokensGPU(state, startToken, N, currentIds, opts,
|
|
|
1527
1529
|
hiddenSize: config.hiddenSize,
|
|
1528
1530
|
vocabSize: config.vocabSize,
|
|
1529
1531
|
scaleEmbeddings: config.scaleEmbeddings,
|
|
1532
|
+
embeddingScale: config.embeddingScale,
|
|
1530
1533
|
recorder,
|
|
1531
1534
|
outputBuffer: context.decodeBuffers?.getHiddenBuffer() ?? undefined,
|
|
1532
1535
|
transpose: state.embeddingTranspose,
|
|
@@ -1298,6 +1298,7 @@ export class PipelineGenerator {
|
|
|
1298
1298
|
hiddenSize,
|
|
1299
1299
|
vocabSize,
|
|
1300
1300
|
scaleEmbeddings: config.scaleEmbeddings,
|
|
1301
|
+
embeddingScale: config.embeddingScale,
|
|
1301
1302
|
debug: opts.debug,
|
|
1302
1303
|
recorder: null,
|
|
1303
1304
|
transpose: this.#state.embeddingTranspose,
|
|
@@ -2537,7 +2538,10 @@ export class PipelineGenerator {
|
|
|
2537
2538
|
// and speculative use the same weights and state. The benefit is
|
|
2538
2539
|
// amortizing per-iteration overhead for models where batch decode is
|
|
2539
2540
|
// disabled (e.g., linear attention).
|
|
2540
|
-
const speculativeBurstTokens =
|
|
2541
|
+
const speculativeBurstTokens = opts.speculation.tokens;
|
|
2542
|
+
if (!Number.isInteger(speculativeBurstTokens) || speculativeBurstTokens < 1) {
|
|
2543
|
+
throw new Error('[Pipeline] resolved self-speculation tokens must be a positive integer.');
|
|
2544
|
+
}
|
|
2541
2545
|
const doSpecDecode = hasLinearLayers
|
|
2542
2546
|
? () => this._decodeStep(generatedIds, opts)
|
|
2543
2547
|
: decodeSingleTokenViaLogits;
|
|
@@ -2863,6 +2867,7 @@ export class PipelineGenerator {
|
|
|
2863
2867
|
hiddenSize: config.hiddenSize,
|
|
2864
2868
|
vocabSize: config.vocabSize,
|
|
2865
2869
|
scaleEmbeddings: config.scaleEmbeddings,
|
|
2870
|
+
embeddingScale: config.embeddingScale,
|
|
2866
2871
|
debug: opts.debug,
|
|
2867
2872
|
recorder,
|
|
2868
2873
|
transpose: this.#state.embeddingTranspose,
|
|
@@ -28,6 +28,8 @@ function computeRoPEFreqsForTheta(
|
|
|
28
28
|
// Compute per-dimension scaling factors
|
|
29
29
|
const scales = new Float32Array(halfDim);
|
|
30
30
|
const isYarn = ropeScalingType === 'yarn';
|
|
31
|
+
const isLongRoPE = ropeScalingType === 'longrope';
|
|
32
|
+
let magnitudeScale = 1;
|
|
31
33
|
if (isYarn) {
|
|
32
34
|
// YARN scaling - validate ALL required params (fail fast on incomplete manifest)
|
|
33
35
|
if (ropeScaling?.factor == null ||
|
|
@@ -61,13 +63,53 @@ function computeRoPEFreqsForTheta(
|
|
|
61
63
|
scales[i] = 1.0 + (yarnFactor - 1.0) * t;
|
|
62
64
|
}
|
|
63
65
|
}
|
|
66
|
+
} else if (isLongRoPE) {
|
|
67
|
+
if (
|
|
68
|
+
!Array.isArray(ropeScaling?.short_factor)
|
|
69
|
+
|| !Array.isArray(ropeScaling?.long_factor)
|
|
70
|
+
|| ropeScaling?.original_max_position_embeddings == null
|
|
71
|
+
) {
|
|
72
|
+
throw new Error(
|
|
73
|
+
`RoPE scaling type is 'longrope' but LongRoPE params are missing. ` +
|
|
74
|
+
`Manifest must provide short_factor, long_factor, and original_max_position_embeddings. ` +
|
|
75
|
+
`Got: short_factor=${Array.isArray(ropeScaling?.short_factor) ? ropeScaling.short_factor.length : ropeScaling?.short_factor}, ` +
|
|
76
|
+
`long_factor=${Array.isArray(ropeScaling?.long_factor) ? ropeScaling.long_factor.length : ropeScaling?.long_factor}, ` +
|
|
77
|
+
`original_max_position_embeddings=${ropeScaling?.original_max_position_embeddings}`
|
|
78
|
+
);
|
|
79
|
+
}
|
|
80
|
+
if (ropeScaling.short_factor.length !== halfDim || ropeScaling.long_factor.length !== halfDim) {
|
|
81
|
+
throw new Error(
|
|
82
|
+
`LongRoPE factor length mismatch: expected ${halfDim}, ` +
|
|
83
|
+
`got short_factor=${ropeScaling.short_factor.length}, long_factor=${ropeScaling.long_factor.length}.`
|
|
84
|
+
);
|
|
85
|
+
}
|
|
86
|
+
const originalMaxPos = Number(ropeScaling.original_max_position_embeddings);
|
|
87
|
+
if (!Number.isFinite(originalMaxPos) || originalMaxPos <= 0) {
|
|
88
|
+
throw new Error(`LongRoPE original_max_position_embeddings must be positive; got "${String(originalMaxPos)}".`);
|
|
89
|
+
}
|
|
90
|
+
const selectedFactors = maxSeqLen > originalMaxPos
|
|
91
|
+
? ropeScaling.long_factor
|
|
92
|
+
: ropeScaling.short_factor;
|
|
93
|
+
for (let i = 0; i < halfDim; i++) {
|
|
94
|
+
const factor = Number(selectedFactors[i]);
|
|
95
|
+
if (!Number.isFinite(factor) || factor <= 0) {
|
|
96
|
+
throw new Error(`LongRoPE factor[${i}] must be a positive finite number; got "${String(selectedFactors[i])}".`);
|
|
97
|
+
}
|
|
98
|
+
scales[i] = factor;
|
|
99
|
+
}
|
|
100
|
+
magnitudeScale = Math.sqrt(1 + Math.log(maxSeqLen / originalMaxPos) / Math.log(originalMaxPos));
|
|
101
|
+
if (!Number.isFinite(magnitudeScale) || magnitudeScale <= 0) {
|
|
102
|
+
throw new Error(
|
|
103
|
+
`LongRoPE magnitude scale is invalid for maxSeqLen=${maxSeqLen}, ` +
|
|
104
|
+
`original_max_position_embeddings=${originalMaxPos}.`
|
|
105
|
+
);
|
|
106
|
+
}
|
|
64
107
|
} else {
|
|
65
108
|
// Linear scaling: uniform across all dimensions
|
|
66
109
|
if (ropeScalingType != null && ropeScalingType !== 'linear') {
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
'Known types: "linear", "yarn".'
|
|
110
|
+
throw new Error(
|
|
111
|
+
`Unsupported RoPE scaling type "${ropeScalingType}". ` +
|
|
112
|
+
'Supported types: null, "linear", "yarn", "longrope".'
|
|
71
113
|
);
|
|
72
114
|
}
|
|
73
115
|
for (let i = 0; i < halfDim; i++) {
|
|
@@ -83,8 +125,8 @@ function computeRoPEFreqsForTheta(
|
|
|
83
125
|
for (let i = 0; i < halfDim; i++) {
|
|
84
126
|
const scaledPos = pos / scales[i];
|
|
85
127
|
const angle = scaledPos * freqs[i];
|
|
86
|
-
cosValues[pos * halfDim + i] = Math.cos(angle);
|
|
87
|
-
sinValues[pos * halfDim + i] = Math.sin(angle);
|
|
128
|
+
cosValues[pos * halfDim + i] = Math.cos(angle) * magnitudeScale;
|
|
129
|
+
sinValues[pos * halfDim + i] = Math.sin(angle) * magnitudeScale;
|
|
88
130
|
}
|
|
89
131
|
}
|
|
90
132
|
|
|
@@ -101,6 +143,12 @@ function isSameRoPEScalingConfig(
|
|
|
101
143
|
) {
|
|
102
144
|
if (leftType !== rightType) return false;
|
|
103
145
|
if (leftScale !== rightScale) return false;
|
|
146
|
+
if (leftType === 'longrope') {
|
|
147
|
+
return JSON.stringify(leftScaling?.short_factor ?? null) === JSON.stringify(rightScaling?.short_factor ?? null)
|
|
148
|
+
&& JSON.stringify(leftScaling?.long_factor ?? null) === JSON.stringify(rightScaling?.long_factor ?? null)
|
|
149
|
+
&& (leftScaling?.original_max_position_embeddings ?? null)
|
|
150
|
+
=== (rightScaling?.original_max_position_embeddings ?? null);
|
|
151
|
+
}
|
|
104
152
|
if (leftType !== 'yarn') return true;
|
|
105
153
|
return (leftScaling?.beta_fast ?? null) === (rightScaling?.beta_fast ?? null)
|
|
106
154
|
&& (leftScaling?.beta_slow ?? null) === (rightScaling?.beta_slow ?? null)
|
|
@@ -442,22 +442,35 @@ function getRequiredKernelMaxKVLen(operation, variant, label) {
|
|
|
442
442
|
}
|
|
443
443
|
|
|
444
444
|
function resolveTieredRequestedQuantMode(runtimeKV) {
|
|
445
|
-
const
|
|
445
|
+
const tiering = requirePlainObject(
|
|
446
|
+
runtimeKV?.tiering,
|
|
447
|
+
'runtime.inference.session.kvcache.tiering'
|
|
448
|
+
);
|
|
449
|
+
const tieringMode = requireNonEmptyString(
|
|
450
|
+
tiering.mode,
|
|
451
|
+
'runtime.inference.session.kvcache.tiering.mode'
|
|
452
|
+
).toLowerCase();
|
|
446
453
|
assertSupportedTurboQuantMode(tieringMode, 'runtime.inference.session.kvcache.tiering.mode');
|
|
447
|
-
const
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
).trim().toLowerCase();
|
|
454
|
+
const compression = requirePlainObject(
|
|
455
|
+
tiering.compression,
|
|
456
|
+
'runtime.inference.session.kvcache.tiering.compression'
|
|
457
|
+
);
|
|
458
|
+
const compressionMode = requireNonEmptyString(
|
|
459
|
+
compression.mode,
|
|
460
|
+
'runtime.inference.session.kvcache.tiering.compression.mode'
|
|
461
|
+
).toLowerCase();
|
|
456
462
|
assertSupportedTurboQuantMode(
|
|
457
463
|
compressionMode,
|
|
458
464
|
'runtime.inference.session.kvcache.tiering.compression.mode'
|
|
459
465
|
);
|
|
460
|
-
const
|
|
466
|
+
const gating = requirePlainObject(
|
|
467
|
+
tiering.gating,
|
|
468
|
+
'runtime.inference.session.kvcache.tiering.gating'
|
|
469
|
+
);
|
|
470
|
+
const gatingMode = requireNonEmptyString(
|
|
471
|
+
gating.mode,
|
|
472
|
+
'runtime.inference.session.kvcache.tiering.gating.mode'
|
|
473
|
+
).toLowerCase();
|
|
461
474
|
if (gatingMode === 'force_off') {
|
|
462
475
|
return 'none';
|
|
463
476
|
}
|
|
@@ -465,7 +478,14 @@ function resolveTieredRequestedQuantMode(runtimeKV) {
|
|
|
465
478
|
}
|
|
466
479
|
|
|
467
480
|
function resolveContiguousRequestedQuantMode(runtimeKV) {
|
|
468
|
-
const
|
|
481
|
+
const quantization = requirePlainObject(
|
|
482
|
+
runtimeKV?.quantization,
|
|
483
|
+
'runtime.inference.session.kvcache.quantization'
|
|
484
|
+
);
|
|
485
|
+
const quantMode = requireNonEmptyString(
|
|
486
|
+
quantization.mode,
|
|
487
|
+
'runtime.inference.session.kvcache.quantization.mode'
|
|
488
|
+
).toLowerCase();
|
|
469
489
|
assertSupportedTurboQuantMode(
|
|
470
490
|
quantMode,
|
|
471
491
|
'runtime.inference.session.kvcache.quantization.mode'
|
|
@@ -725,17 +745,24 @@ export function createKVCache(modelConfig, useGPU, debug = false, runtimeConfig)
|
|
|
725
745
|
tiering: runtimeKV.tiering,
|
|
726
746
|
});
|
|
727
747
|
} else if (cacheLayout === 'contiguous_quantized') {
|
|
728
|
-
const quantCfg =
|
|
748
|
+
const quantCfg = requirePlainObject(
|
|
749
|
+
runtimeKV.quantization,
|
|
750
|
+
'runtime.inference.kvcache.quantization'
|
|
751
|
+
);
|
|
752
|
+
const bitWidth = requirePositiveInteger(
|
|
753
|
+
quantCfg.bitWidth,
|
|
754
|
+
'runtime.inference.kvcache.quantization.bitWidth'
|
|
755
|
+
);
|
|
729
756
|
const qCache = new QuantizedKVCache({
|
|
730
757
|
...cacheConfig,
|
|
731
758
|
quantMode,
|
|
732
|
-
bitWidth
|
|
759
|
+
bitWidth,
|
|
733
760
|
prodMode: quantCfg.prodMode === true,
|
|
734
761
|
});
|
|
735
762
|
const device = getDevice();
|
|
736
763
|
qCache.setSharedBuffers(retainTurboQuantSharedBuffers(device, {
|
|
737
764
|
headDim: modelConfig.headDim,
|
|
738
|
-
bitWidth
|
|
765
|
+
bitWidth,
|
|
739
766
|
prodMode: quantCfg.prodMode === true,
|
|
740
767
|
}));
|
|
741
768
|
kvCache = qCache;
|
|
@@ -783,6 +810,27 @@ export function createKVCache(modelConfig, useGPU, debug = false, runtimeConfig)
|
|
|
783
810
|
return kvCache;
|
|
784
811
|
}
|
|
785
812
|
|
|
813
|
+
function requirePlainObject(value, label) {
|
|
814
|
+
if (!isPlainObject(value)) {
|
|
815
|
+
throw new Error(`${label} is required.`);
|
|
816
|
+
}
|
|
817
|
+
return value;
|
|
818
|
+
}
|
|
819
|
+
|
|
820
|
+
function requireNonEmptyString(value, label) {
|
|
821
|
+
if (typeof value !== 'string' || value.trim().length === 0) {
|
|
822
|
+
throw new Error(`${label} is required.`);
|
|
823
|
+
}
|
|
824
|
+
return value.trim();
|
|
825
|
+
}
|
|
826
|
+
|
|
827
|
+
function requirePositiveInteger(value, label) {
|
|
828
|
+
if (!Number.isInteger(value) || value <= 0) {
|
|
829
|
+
throw new Error(`${label} must be a positive integer.`);
|
|
830
|
+
}
|
|
831
|
+
return value;
|
|
832
|
+
}
|
|
833
|
+
|
|
786
834
|
// ============================================================================
|
|
787
835
|
// Tokenizer Setup
|
|
788
836
|
// ============================================================================
|
|
@@ -628,6 +628,13 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
|
|
|
628
628
|
|
|
629
629
|
assertSupportedLayerRuntime(layerIdx, config);
|
|
630
630
|
const { hiddenSize, numHeads, numKVHeads, headDim, rmsNormEps } = config;
|
|
631
|
+
const residualBranchScale = Number(config.residualBranchScale);
|
|
632
|
+
if (!Number.isFinite(residualBranchScale) || residualBranchScale <= 0) {
|
|
633
|
+
throw new Error(
|
|
634
|
+
`Layer ${layerIdx} residualBranchScale must be a positive finite number; ` +
|
|
635
|
+
`got "${String(config.residualBranchScale)}".`
|
|
636
|
+
);
|
|
637
|
+
}
|
|
631
638
|
|
|
632
639
|
// Determine activation dtype from context (defaults to f32)
|
|
633
640
|
|
|
@@ -638,6 +645,12 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
|
|
|
638
645
|
|
|
639
646
|
const layerWeights = (weights.get(`layer_${layerIdx}`));
|
|
640
647
|
const sandwichNorm = detectSandwichNorm(config);
|
|
648
|
+
if (sandwichNorm.useSandwichNorm && residualBranchScale !== 1) {
|
|
649
|
+
throw new Error(
|
|
650
|
+
`Layer ${layerIdx} uses sandwich norms with residualBranchScale=${residualBranchScale}. ` +
|
|
651
|
+
'Scaled residual branches for sandwich-norm layers are not implemented.'
|
|
652
|
+
);
|
|
653
|
+
}
|
|
641
654
|
const lastTokenIdx = Math.max(0, numTokens - 1);
|
|
642
655
|
|
|
643
656
|
await runProbes('layer_in', inputBuffer, {
|
|
@@ -651,6 +664,12 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
|
|
|
651
664
|
});
|
|
652
665
|
|
|
653
666
|
if (context.pipelinePlan) {
|
|
667
|
+
if (residualBranchScale !== 1) {
|
|
668
|
+
throw new Error(
|
|
669
|
+
`Layer ${layerIdx} has residualBranchScale=${residualBranchScale}, but pipelinePlan execution ` +
|
|
670
|
+
'does not implement scaled residual branches.'
|
|
671
|
+
);
|
|
672
|
+
}
|
|
654
673
|
return processLayerPlanGPU(layerIdx, inputBuffer, numTokens, isPrefill, size, context, layerWeights, sandwichNorm);
|
|
655
674
|
}
|
|
656
675
|
|
|
@@ -763,7 +782,7 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
|
|
|
763
782
|
activationDtype,
|
|
764
783
|
slidingWindow: config.slidingWindow,
|
|
765
784
|
layerType,
|
|
766
|
-
residualTensor: (numTokens === 1 && !(sandwichNorm.useSandwichNorm && sandwichNorm.hasPostAttentionNorm))
|
|
785
|
+
residualTensor: (numTokens === 1 && !(sandwichNorm.useSandwichNorm && sandwichNorm.hasPostAttentionNorm) && residualBranchScale === 1)
|
|
767
786
|
? inputTensor
|
|
768
787
|
: null,
|
|
769
788
|
attnSoftcap: config.attnLogitSoftcapping === null ? 0 : config.attnLogitSoftcapping,
|
|
@@ -870,6 +889,13 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
|
|
|
870
889
|
operatorDiagnostics: context.operatorDiagnostics,
|
|
871
890
|
dtype: attnOutput.dtype,
|
|
872
891
|
});
|
|
892
|
+
if (!residualFused && residualBranchScale !== 1) {
|
|
893
|
+
const rawAttnOutput = attnOutput;
|
|
894
|
+
attnOutput = recorder
|
|
895
|
+
? await recordScale(recorder, rawAttnOutput, residualBranchScale, { count: size })
|
|
896
|
+
: await runScale(rawAttnOutput, residualBranchScale, { count: size });
|
|
897
|
+
releaseOrTrack(recorder, rawAttnOutput.buffer, context.decodeBuffers);
|
|
898
|
+
}
|
|
873
899
|
|
|
874
900
|
// 2. Handle residual connection based on architecture
|
|
875
901
|
|
|
@@ -1061,10 +1087,11 @@ export async function processLayerGPU(layerIdx, inputBuffer, numTokens, isPrefil
|
|
|
1061
1087
|
numTokens,
|
|
1062
1088
|
size,
|
|
1063
1089
|
context,
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1090
|
+
layerWeights,
|
|
1091
|
+
fusedResidualForFFN,
|
|
1092
|
+
requestFfnLayerScalarFusion ? layerScalar : 1,
|
|
1093
|
+
residualBranchScale
|
|
1094
|
+
);
|
|
1068
1095
|
layerScalarFused = context.__layerScalarFusedFired === true;
|
|
1069
1096
|
context.__layerScalarFusedFired = false;
|
|
1070
1097
|
}
|
|
@@ -34,9 +34,7 @@ function normalizeLinearNormMode(value) {
|
|
|
34
34
|
}
|
|
35
35
|
|
|
36
36
|
function bytesFromDtype(dtype) {
|
|
37
|
-
|
|
38
|
-
if (normalized === 'f16' || normalized === 'bf16') return 2;
|
|
39
|
-
return 4;
|
|
37
|
+
return selectRuleValue('shared', 'dtype', 'bytesFromDtype', { dtype });
|
|
40
38
|
}
|
|
41
39
|
|
|
42
40
|
function resolveMatmulStepDtype(role, phase, layerIdx, kernelPath, fallback, field) {
|
|
@@ -199,13 +197,19 @@ function inferLinearNormModeFromWeight(weight, projectionLayout) {
|
|
|
199
197
|
if (weight instanceof ArrayBuffer) {
|
|
200
198
|
return classify(Math.trunc(weight.byteLength / Float32Array.BYTES_PER_ELEMENT));
|
|
201
199
|
}
|
|
200
|
+
if (!weight || typeof weight !== 'object') {
|
|
201
|
+
return null;
|
|
202
|
+
}
|
|
202
203
|
const explicitDtype = typeof weight?.dtype === 'string' ? weight.dtype.toLowerCase() : null;
|
|
203
204
|
const trackedDtype = isGpuBuffer(weight) ? String(getBufferDtype(weight) ?? '').toLowerCase() : '';
|
|
204
|
-
const
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
205
|
+
const sourceDtype = explicitDtype || trackedDtype;
|
|
206
|
+
if (!sourceDtype || !Number.isFinite(weight?.size)) {
|
|
207
|
+
return null;
|
|
208
|
+
}
|
|
209
|
+
const bytesPerElement = bytesFromDtype(sourceDtype);
|
|
210
|
+
const size = Number(weight.size);
|
|
211
|
+
const sizedElements = Math.trunc(size / bytesPerElement);
|
|
212
|
+
if (sizedElements && size % bytesPerElement === 0) {
|
|
209
213
|
return classify(sizedElements);
|
|
210
214
|
}
|
|
211
215
|
return null;
|