doppler-gpu 0.4.6 → 0.4.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +43 -31
- package/assets/doppler-runtime-map.svg +1 -3
- package/assets/doppler.svg +1 -3
- package/package.json +25 -17
- package/src/cli/doppler-quickstart.js +1 -1
- package/src/cli/doppler-serve.js +2 -2
- package/src/client/doppler-registry.d.ts +19 -0
- package/src/client/doppler-registry.js +12 -0
- package/src/client/doppler-registry.json +50 -1
- package/src/config/conversion/README.md +4 -0
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +28 -4
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +31 -5
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +30 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +30 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +30 -4
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +30 -5
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +30 -5
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +29 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +29 -4
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +29 -4
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/minicpm/minicpm4-0-5b-f16-af32.json +481 -0
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +29 -4
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +29 -4
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +30 -5
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +29 -4
- package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-q4k-ehf16-af32.json +395 -0
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-af32.json +429 -0
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +540 -0
- package/src/config/execution-contract-check.js +14 -10
- package/src/config/merge-contract-check.js +1 -0
- package/src/config/merge.js +30 -0
- package/src/config/param-validator.d.ts +5 -0
- package/src/config/param-validator.js +42 -8
- package/src/config/required-inference-fields-contract-check.js +14 -0
- package/src/config/schema/debug.schema.d.ts +2 -0
- package/src/config/schema/debug.schema.js +1 -0
- package/src/config/schema/doppler.schema.d.ts +5 -1
- package/src/config/schema/doppler.schema.js +82 -292
- package/src/config/schema/inference.schema.d.ts +17 -3
- package/src/config/schema/kvcache.schema.d.ts +19 -3
- package/src/config/schema/kvcache.schema.js +2 -1
- package/src/config/schema/manifest.schema.d.ts +38 -1
- package/src/config/schema/manifest.schema.js +18 -0
- package/src/config/schema/policy-schema-registry.json +5 -0
- package/src/config/source-packages/litert/gemma-4-12b-it.json +20 -1
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +20 -1
- package/src/config/source-packages/litert/gemma-4-e4b-it.json +21 -1
- package/src/config/transforms/execution-graph-transforms.d.ts +6 -0
- package/src/config/transforms/execution-graph-transforms.js +8 -0
- package/src/converter/conversion-plan.js +2 -0
- package/src/converter/parsers/transformer.js +27 -1
- package/src/converter/quantization-info.js +16 -4
- package/src/converter/rope-config.js +55 -3
- package/src/experimental/training/distillation/student-fixture.js +1 -1
- package/src/experimental/training/lora-pipeline.js +1 -1
- package/src/experimental/training/lora.js +2 -2
- package/src/experimental/training/suite.js +1 -1
- package/src/gpu/kernels/README.md +8 -1
- package/src/gpu/kernels/backward/attention_backward.js +8 -0
- package/src/gpu/kernels/dequant.d.ts +6 -6
- package/src/gpu/kernels/dequant.js +17 -6
- package/src/gpu/kernels/linear-attention-core.js +1 -6
- package/src/gpu/kernels/matmul-selection.d.ts +2 -2
- package/src/gpu/kernels/matmul-selection.js +4 -1
- package/src/gpu/kernels/matmul.d.ts +1 -1
- package/src/gpu/kernels/matmul.js +24 -6
- package/src/gpu/kernels/moe.d.ts +5 -1
- package/src/gpu/kernels/moe.js +5 -2
- package/src/gpu/kernels/softmax.d.ts +3 -3
- package/src/gpu/kernels/softmax.js +9 -2
- package/src/gpu/kernels/turboquant-codebook.js +1 -1
- package/src/gpu/kernels/types.d.ts +1 -1
- package/src/gpu/profiler.js +11 -1
- package/src/inference/browser-harness-model-helpers.js +4 -1
- package/src/inference/browser-harness-text-helpers.d.ts +28 -0
- package/src/inference/browser-harness-text-helpers.js +357 -116
- package/src/inference/browser-harness.d.ts +3 -3
- package/src/inference/browser-harness.js +266 -11
- package/src/inference/fixtures/rerank-semantic-fixtures.json +38 -0
- package/src/inference/kv-cache/quantized.js +16 -2
- package/src/inference/kv-cache/tiered.js +50 -15
- package/src/inference/moe-router.js +38 -7
- package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +1 -1
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +8 -6
- package/src/inference/pipelines/text/config.d.ts +5 -1
- package/src/inference/pipelines/text/config.js +135 -112
- package/src/inference/pipelines/text/embed.d.ts +1 -0
- package/src/inference/pipelines/text/embed.js +39 -12
- package/src/inference/pipelines/text/execution-plan.js +8 -2
- package/src/inference/pipelines/text/ffn/dense.js +15 -4
- package/src/inference/pipelines/text/ffn/moe.js +4 -3
- package/src/inference/pipelines/text/ffn/standard.d.ts +2 -1
- package/src/inference/pipelines/text/ffn/standard.js +25 -2
- package/src/inference/pipelines/text/generator-helpers.js +2 -1
- package/src/inference/pipelines/text/generator-runtime.d.ts +6 -2
- package/src/inference/pipelines/text/generator-runtime.js +53 -20
- package/src/inference/pipelines/text/generator-steps.js +3 -0
- package/src/inference/pipelines/text/generator.js +6 -1
- package/src/inference/pipelines/text/init-rope.js +54 -6
- package/src/inference/pipelines/text/init.js +63 -15
- package/src/inference/pipelines/text/layer.js +32 -5
- package/src/inference/pipelines/text/linear-attention.js +12 -8
- package/src/inference/pipelines/text/logits/gpu.d.ts +1 -1
- package/src/inference/pipelines/text/logits/gpu.js +76 -113
- package/src/inference/pipelines/text/logits/index.js +44 -87
- package/src/inference/pipelines/text/logits/precision-policy.d.ts +11 -0
- package/src/inference/pipelines/text/logits/precision-policy.js +103 -0
- package/src/inference/pipelines/text/logits/types.d.ts +1 -0
- package/src/inference/pipelines/text/logits/utils.d.ts +2 -0
- package/src/inference/pipelines/text/logits/utils.js +11 -0
- package/src/inference/pipelines/text/lora-apply.js +6 -4
- package/src/inference/pipelines/text/moe-cpu.js +25 -0
- package/src/inference/pipelines/text/moe-gpu.js +148 -97
- package/src/inference/pipelines/text/moe-shape-validator.d.ts +33 -3
- package/src/inference/pipelines/text/moe-shape-validator.js +149 -31
- package/src/inference/pipelines/text/ops.d.ts +3 -0
- package/src/inference/pipelines/text/ops.js +26 -1
- package/src/inference/pipelines/text/per-layer-inputs.js +27 -12
- package/src/inference/pipelines/text/weights.js +7 -2
- package/src/inference/pipelines/text.js +24 -4
- package/src/inference/tokenizers/behavior-flags.js +23 -2
- package/src/loader/final-weights-loader.js +12 -1
- package/src/rules/execution-rules-contract-check.js +12 -3
- package/src/rules/inference/capability-transforms.rules.json +14 -2
- package/src/rules/inference/config.rules.json +26 -2
- package/src/rules/inference/dtype.rules.json +12 -5
- package/src/rules/inference/execution.rules.json +5 -19
- package/src/rules/inference/moe.rules.json +97 -0
- package/src/tooling/command-api-constants.js +2 -2
- package/src/tooling/command-api-family-normalizers.js +2 -1
- package/src/tooling/command-api.d.ts +1 -1
- package/src/tooling/command-runner-shared.d.ts +1 -1
- package/src/tooling/command-runner-shared.js +3 -1
- package/src/tooling/node-converter.js +20 -0
- package/src/tooling/node-source-runtime.js +5 -1
- package/src/version.js +1 -1
- package/assets/doppler-readme-hero.svg +0 -84
|
@@ -44,8 +44,6 @@ import {
|
|
|
44
44
|
const QUICK_GELU_ALPHA = 1.702;
|
|
45
45
|
const DEFAULT_TIMESTEP_EMBED_DIM = 256;
|
|
46
46
|
const SUPPORTED_CLIP_HIDDEN_ACTIVATIONS = new Set(['gelu', 'quick_gelu']);
|
|
47
|
-
// Standard CLIP hidden activation per OpenAI CLIP specification.
|
|
48
|
-
const DEFAULT_CLIP_HIDDEN_ACT = 'gelu';
|
|
49
47
|
|
|
50
48
|
function padTokens(tokens, maxLength, padTokenId) {
|
|
51
49
|
if (!Number.isFinite(maxLength) || maxLength <= 0) {
|
|
@@ -140,14 +138,18 @@ function createKernelOps(recorder) {
|
|
|
140
138
|
}
|
|
141
139
|
|
|
142
140
|
function resolveClipHiddenActivation(config) {
|
|
143
|
-
const hiddenAct = config?.hidden_act
|
|
144
|
-
if (
|
|
141
|
+
const hiddenAct = config?.hidden_act;
|
|
142
|
+
if (typeof hiddenAct !== 'string' || hiddenAct.trim().length === 0) {
|
|
143
|
+
throw new Error('CLIP text encoder config.hidden_act is required.');
|
|
144
|
+
}
|
|
145
|
+
const normalizedHiddenAct = hiddenAct.trim();
|
|
146
|
+
if (!SUPPORTED_CLIP_HIDDEN_ACTIVATIONS.has(normalizedHiddenAct)) {
|
|
145
147
|
throw new Error(
|
|
146
|
-
`Unsupported CLIP hidden_act "${
|
|
148
|
+
`Unsupported CLIP hidden_act "${normalizedHiddenAct}". ` +
|
|
147
149
|
`Expected one of: ${Array.from(SUPPORTED_CLIP_HIDDEN_ACTIVATIONS).join(', ')}.`
|
|
148
150
|
);
|
|
149
151
|
}
|
|
150
|
-
return
|
|
152
|
+
return normalizedHiddenAct;
|
|
151
153
|
}
|
|
152
154
|
|
|
153
155
|
async function runClipMlpActivation(input, hiddenAct, count, ops, release) {
|
|
@@ -102,6 +102,8 @@ export interface RopeScalingConfig {
|
|
|
102
102
|
beta_fast?: number;
|
|
103
103
|
beta_slow?: number;
|
|
104
104
|
original_max_position_embeddings?: number;
|
|
105
|
+
short_factor?: number[];
|
|
106
|
+
long_factor?: number[];
|
|
105
107
|
}
|
|
106
108
|
|
|
107
109
|
export interface TensorInfo {
|
|
@@ -172,6 +174,9 @@ export interface ParsedModelConfig {
|
|
|
172
174
|
preFeedforwardNorm: boolean;
|
|
173
175
|
postFeedforwardNorm: boolean;
|
|
174
176
|
scaleEmbeddings: boolean;
|
|
177
|
+
embeddingScale: number | null;
|
|
178
|
+
logitInputScale: number;
|
|
179
|
+
residualBranchScale: number;
|
|
175
180
|
useTiedEmbeddings: boolean;
|
|
176
181
|
embeddingTranspose: boolean;
|
|
177
182
|
embeddingVocabSize: number | null;
|
|
@@ -190,7 +195,6 @@ export interface ParsedModelConfig {
|
|
|
190
195
|
linearNormMode: LinearNormMode | null;
|
|
191
196
|
attentionBias: boolean;
|
|
192
197
|
causalAttention: boolean;
|
|
193
|
-
embeddingScale?: number;
|
|
194
198
|
finalLogitSoftcapping: number | null;
|
|
195
199
|
attnLogitSoftcapping: number | null;
|
|
196
200
|
queryKeyNorm: boolean;
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { log } from '../../../debug/index.js';
|
|
2
2
|
import { mergeConfig, dumpConfigSources } from '../../../config/merge.js';
|
|
3
|
+
import { validateModelOverrides } from '../../../config/param-validator.js';
|
|
3
4
|
import { selectRuleValue } from '../../../rules/rule-registry.js';
|
|
4
5
|
import {
|
|
5
6
|
PER_LAYER_INPUT_MATERIALIZATION_MODES,
|
|
@@ -376,90 +377,8 @@ function normalizeFfnTensorShape(value) {
|
|
|
376
377
|
return [Math.trunc(rows), Math.trunc(cols)];
|
|
377
378
|
}
|
|
378
379
|
|
|
379
|
-
function
|
|
380
|
-
|
|
381
|
-
return lower.includes('.experts.') || lower.includes('.expert.') || lower.includes('block_sparse_moe');
|
|
382
|
-
}
|
|
383
|
-
|
|
384
|
-
function inferLfm2IntermediateSizeFromManifest(manifest) {
|
|
385
|
-
const tensors = manifest?.tensors;
|
|
386
|
-
if (!tensors || typeof tensors !== 'object') return null;
|
|
387
|
-
const candidates = [];
|
|
388
|
-
for (const [name, entry] of Object.entries(tensors)) {
|
|
389
|
-
if (!name || isExpertTensorName(name)) continue;
|
|
390
|
-
const shape = normalizeFfnTensorShape(entry?.shape);
|
|
391
|
-
if (!shape) continue;
|
|
392
|
-
const lower = name.toLowerCase();
|
|
393
|
-
if (
|
|
394
|
-
lower.endsWith('.feed_forward.w1.weight')
|
|
395
|
-
|| lower.endsWith('.feed_forward.w3.weight')
|
|
396
|
-
|| lower.endsWith('.ffn_gate.weight')
|
|
397
|
-
|| lower.endsWith('.ffn_up.weight')
|
|
398
|
-
|| lower.endsWith('.ffn.gate_proj.weight')
|
|
399
|
-
|| lower.endsWith('.ffn.up_proj.weight')
|
|
400
|
-
|| lower.endsWith('.mlp.gate_proj.weight')
|
|
401
|
-
|| lower.endsWith('.mlp.up_proj.weight')
|
|
402
|
-
) {
|
|
403
|
-
candidates.push(shape[0]);
|
|
404
|
-
continue;
|
|
405
|
-
}
|
|
406
|
-
if (
|
|
407
|
-
lower.endsWith('.feed_forward.w2.weight')
|
|
408
|
-
|| lower.endsWith('.ffn_down.weight')
|
|
409
|
-
|| lower.endsWith('.ffn.down_proj.weight')
|
|
410
|
-
|| lower.endsWith('.mlp.down_proj.weight')
|
|
411
|
-
) {
|
|
412
|
-
candidates.push(shape[1]);
|
|
413
|
-
continue;
|
|
414
|
-
}
|
|
415
|
-
if (
|
|
416
|
-
lower.endsWith('.feed_forward.w1_w3.weight')
|
|
417
|
-
|| lower.endsWith('.ffn_gate_up.weight')
|
|
418
|
-
|| lower.endsWith('.ffn.gate_up_proj.weight')
|
|
419
|
-
|| lower.endsWith('.mlp.gate_up_proj.weight')
|
|
420
|
-
) {
|
|
421
|
-
if (shape[0] % 2 === 0) {
|
|
422
|
-
candidates.push(Math.trunc(shape[0] / 2));
|
|
423
|
-
}
|
|
424
|
-
}
|
|
425
|
-
}
|
|
426
|
-
if (candidates.length === 0) return null;
|
|
427
|
-
const counts = new Map();
|
|
428
|
-
for (const value of candidates) {
|
|
429
|
-
counts.set(value, (counts.get(value) ?? 0) + 1);
|
|
430
|
-
}
|
|
431
|
-
const result = [...counts.entries()]
|
|
432
|
-
.sort((a, b) => {
|
|
433
|
-
if (b[1] !== a[1]) return b[1] - a[1];
|
|
434
|
-
return a[0] - b[0];
|
|
435
|
-
})[0]?.[0] ?? null;
|
|
436
|
-
if (result != null && (!Number.isInteger(result) || result <= 0)) {
|
|
437
|
-
log.warn(
|
|
438
|
-
'Config',
|
|
439
|
-
`inferLfm2IntermediateSizeFromManifest: inferred intermediateSize ${result} is not a positive integer, discarding`
|
|
440
|
-
);
|
|
441
|
-
return null;
|
|
442
|
-
}
|
|
443
|
-
return result;
|
|
444
|
-
}
|
|
445
|
-
|
|
446
|
-
function resolveIntermediateSizeForRuntime(manifest, inf, arch, modelId) {
|
|
447
|
-
const fromArch = arch?.intermediateSize;
|
|
448
|
-
if (typeof fromArch !== 'number' || !Number.isFinite(fromArch) || fromArch <= 0) {
|
|
449
|
-
return fromArch;
|
|
450
|
-
}
|
|
451
|
-
const normalizedModelId = String(modelId ?? manifest?.modelId ?? '').toLowerCase();
|
|
452
|
-
if (!normalizedModelId.includes('lfm2')) {
|
|
453
|
-
return fromArch;
|
|
454
|
-
}
|
|
455
|
-
const inferred = inferLfm2IntermediateSizeFromManifest(manifest);
|
|
456
|
-
if (inferred == null || inferred === fromArch) {
|
|
457
|
-
return fromArch;
|
|
458
|
-
}
|
|
459
|
-
throw new Error(
|
|
460
|
-
`Manifest "${modelId}" has intermediateSize=${fromArch}, but FFN tensors imply ${inferred}. ` +
|
|
461
|
-
'Re-convert the model so manifest architecture matches the weights.'
|
|
462
|
-
);
|
|
380
|
+
function resolveIntermediateSizeForRuntime(arch) {
|
|
381
|
+
return arch?.intermediateSize;
|
|
463
382
|
}
|
|
464
383
|
|
|
465
384
|
function buildPerLayerIntermediateSizes({
|
|
@@ -555,6 +474,8 @@ function validateLayerIntermediateSizesAgainstManifest(manifest, hiddenSize, lay
|
|
|
555
474
|
`${genericPrefix}.mlp.gate_proj.weight`,
|
|
556
475
|
`${languagePrefix}.ffn.gate_proj.weight`,
|
|
557
476
|
`${languagePrefix}.ffn_gate.weight`,
|
|
477
|
+
`${languagePrefix}.feed_forward.w1.weight`,
|
|
478
|
+
`${genericPrefix}.feed_forward.w1.weight`,
|
|
558
479
|
`layers.${layerIdx}.feed_forward.w1.weight`,
|
|
559
480
|
]);
|
|
560
481
|
const upShape = getDenseFfnTensorShape(tensors, [
|
|
@@ -562,6 +483,8 @@ function validateLayerIntermediateSizesAgainstManifest(manifest, hiddenSize, lay
|
|
|
562
483
|
`${genericPrefix}.mlp.up_proj.weight`,
|
|
563
484
|
`${languagePrefix}.ffn.up_proj.weight`,
|
|
564
485
|
`${languagePrefix}.ffn_up.weight`,
|
|
486
|
+
`${languagePrefix}.feed_forward.w3.weight`,
|
|
487
|
+
`${genericPrefix}.feed_forward.w3.weight`,
|
|
565
488
|
`layers.${layerIdx}.feed_forward.w3.weight`,
|
|
566
489
|
]);
|
|
567
490
|
const downShape = getDenseFfnTensorShape(tensors, [
|
|
@@ -569,6 +492,8 @@ function validateLayerIntermediateSizesAgainstManifest(manifest, hiddenSize, lay
|
|
|
569
492
|
`${genericPrefix}.mlp.down_proj.weight`,
|
|
570
493
|
`${languagePrefix}.ffn.down_proj.weight`,
|
|
571
494
|
`${languagePrefix}.ffn_down.weight`,
|
|
495
|
+
`${languagePrefix}.feed_forward.w2.weight`,
|
|
496
|
+
`${genericPrefix}.feed_forward.w2.weight`,
|
|
572
497
|
`layers.${layerIdx}.feed_forward.w2.weight`,
|
|
573
498
|
]);
|
|
574
499
|
const gateUpShape = getDenseFfnTensorShape(tensors, [
|
|
@@ -576,6 +501,8 @@ function validateLayerIntermediateSizesAgainstManifest(manifest, hiddenSize, lay
|
|
|
576
501
|
`${genericPrefix}.mlp.gate_up_proj.weight`,
|
|
577
502
|
`${languagePrefix}.ffn.gate_up_proj.weight`,
|
|
578
503
|
`${languagePrefix}.ffn_gate_up.weight`,
|
|
504
|
+
`${languagePrefix}.feed_forward.w1_w3.weight`,
|
|
505
|
+
`${genericPrefix}.feed_forward.w1_w3.weight`,
|
|
579
506
|
`layers.${layerIdx}.feed_forward.w1_w3.weight`,
|
|
580
507
|
]);
|
|
581
508
|
|
|
@@ -752,7 +679,9 @@ export function validateRequiredInferenceFields(inf, modelId) {
|
|
|
752
679
|
if (inf.ffn.gatedActivation == null) {
|
|
753
680
|
errors.push('ffn.gatedActivation is required');
|
|
754
681
|
}
|
|
755
|
-
if (inf.ffn.branchMode
|
|
682
|
+
if (inf.ffn.branchMode == null) {
|
|
683
|
+
errors.push('ffn.branchMode is required');
|
|
684
|
+
} else {
|
|
756
685
|
const normalizedBranchMode = typeof inf.ffn.branchMode === 'string'
|
|
757
686
|
? inf.ffn.branchMode.trim().toLowerCase()
|
|
758
687
|
: '';
|
|
@@ -840,6 +769,26 @@ export function validateRequiredInferenceFields(inf, modelId) {
|
|
|
840
769
|
if (inf.output.scaleEmbeddings == null) {
|
|
841
770
|
errors.push('output.scaleEmbeddings is required');
|
|
842
771
|
}
|
|
772
|
+
if (inf.output.embeddingScale === undefined) {
|
|
773
|
+
errors.push('output.embeddingScale must be explicitly set (null for scaleEmbeddings semantics, or number)');
|
|
774
|
+
} else {
|
|
775
|
+
const embeddingScale = inf.output.embeddingScale;
|
|
776
|
+
if (embeddingScale !== null && (typeof embeddingScale !== 'number' || !Number.isFinite(embeddingScale) || embeddingScale <= 0)) {
|
|
777
|
+
errors.push('output.embeddingScale must be a positive finite number or null');
|
|
778
|
+
}
|
|
779
|
+
if (embeddingScale !== null && inf.output.scaleEmbeddings === true) {
|
|
780
|
+
errors.push('output.embeddingScale cannot be set when output.scaleEmbeddings is true');
|
|
781
|
+
}
|
|
782
|
+
}
|
|
783
|
+
if (inf.output.logitInputScale == null) {
|
|
784
|
+
errors.push('output.logitInputScale is required');
|
|
785
|
+
} else if (
|
|
786
|
+
typeof inf.output.logitInputScale !== 'number'
|
|
787
|
+
|| !Number.isFinite(inf.output.logitInputScale)
|
|
788
|
+
|| inf.output.logitInputScale <= 0
|
|
789
|
+
) {
|
|
790
|
+
errors.push('output.logitInputScale must be a positive finite number');
|
|
791
|
+
}
|
|
843
792
|
if (inf.output.embeddingTranspose == null) {
|
|
844
793
|
errors.push('output.embeddingTranspose is required');
|
|
845
794
|
}
|
|
@@ -909,6 +858,15 @@ export function validateRequiredInferenceFields(inf, modelId) {
|
|
|
909
858
|
if (inf.layerPattern?.offset === undefined) {
|
|
910
859
|
errors.push('layerPattern.offset must be explicitly set (null if not applicable)');
|
|
911
860
|
}
|
|
861
|
+
if (inf.layerPattern?.residualBranchScale == null) {
|
|
862
|
+
errors.push('layerPattern.residualBranchScale is required');
|
|
863
|
+
} else if (
|
|
864
|
+
typeof inf.layerPattern.residualBranchScale !== 'number'
|
|
865
|
+
|| !Number.isFinite(inf.layerPattern.residualBranchScale)
|
|
866
|
+
|| inf.layerPattern.residualBranchScale <= 0
|
|
867
|
+
) {
|
|
868
|
+
errors.push('layerPattern.residualBranchScale must be a positive finite number');
|
|
869
|
+
}
|
|
912
870
|
if (inf.layerPattern?.type === 'custom' && inf.layerPattern?.layerTypes === undefined) {
|
|
913
871
|
errors.push('layerPattern.layerTypes must be explicitly set for custom patterns');
|
|
914
872
|
}
|
|
@@ -931,6 +889,36 @@ export function validateRequiredInferenceFields(inf, modelId) {
|
|
|
931
889
|
if (inf.rope.yarnOriginalMaxPos === undefined) {
|
|
932
890
|
errors.push('rope.yarnOriginalMaxPos must be explicitly set (null if not YARN)');
|
|
933
891
|
}
|
|
892
|
+
if (inf.rope.longropeShortFactor === undefined) {
|
|
893
|
+
errors.push('rope.longropeShortFactor must be explicitly set (null if not LongRoPE)');
|
|
894
|
+
}
|
|
895
|
+
if (inf.rope.longropeLongFactor === undefined) {
|
|
896
|
+
errors.push('rope.longropeLongFactor must be explicitly set (null if not LongRoPE)');
|
|
897
|
+
}
|
|
898
|
+
if (inf.rope.longropeOriginalMaxPos === undefined) {
|
|
899
|
+
errors.push('rope.longropeOriginalMaxPos must be explicitly set (null if not LongRoPE)');
|
|
900
|
+
}
|
|
901
|
+
if (inf.rope.ropeScalingType === 'longrope') {
|
|
902
|
+
if (!Array.isArray(inf.rope.longropeShortFactor) || inf.rope.longropeShortFactor.length === 0) {
|
|
903
|
+
errors.push('rope.longropeShortFactor must be a non-empty number array for LongRoPE');
|
|
904
|
+
}
|
|
905
|
+
if (!Array.isArray(inf.rope.longropeLongFactor) || inf.rope.longropeLongFactor.length === 0) {
|
|
906
|
+
errors.push('rope.longropeLongFactor must be a non-empty number array for LongRoPE');
|
|
907
|
+
}
|
|
908
|
+
if (
|
|
909
|
+
typeof inf.rope.longropeOriginalMaxPos !== 'number'
|
|
910
|
+
|| !Number.isFinite(inf.rope.longropeOriginalMaxPos)
|
|
911
|
+
|| inf.rope.longropeOriginalMaxPos <= 0
|
|
912
|
+
) {
|
|
913
|
+
errors.push('rope.longropeOriginalMaxPos must be a positive finite number for LongRoPE');
|
|
914
|
+
}
|
|
915
|
+
} else if (
|
|
916
|
+
inf.rope.longropeShortFactor !== null
|
|
917
|
+
|| inf.rope.longropeLongFactor !== null
|
|
918
|
+
|| inf.rope.longropeOriginalMaxPos !== null
|
|
919
|
+
) {
|
|
920
|
+
errors.push('rope LongRoPE fields must be null unless rope.ropeScalingType is "longrope"');
|
|
921
|
+
}
|
|
934
922
|
if (inf.rope.ropeLocalYarnBetaFast === undefined) {
|
|
935
923
|
errors.push('rope.ropeLocalYarnBetaFast must be explicitly set (null if not local YARN)');
|
|
936
924
|
}
|
|
@@ -1228,6 +1216,25 @@ function resolveAudioConfig(rawConfig, manifest) {
|
|
|
1228
1216
|
}
|
|
1229
1217
|
return number;
|
|
1230
1218
|
};
|
|
1219
|
+
const resolveRequiredFiniteNumber = (value, label) => {
|
|
1220
|
+
const number = Number(value);
|
|
1221
|
+
if (!Number.isFinite(number)) {
|
|
1222
|
+
throw new Error(
|
|
1223
|
+
`Manifest "${modelId}" has invalid audio_config.${label}=${JSON.stringify(value)}. ` +
|
|
1224
|
+
'Expected a finite number.'
|
|
1225
|
+
);
|
|
1226
|
+
}
|
|
1227
|
+
return number;
|
|
1228
|
+
};
|
|
1229
|
+
const resolveRequiredString = (value, label) => {
|
|
1230
|
+
if (typeof value !== 'string' || value.trim().length === 0) {
|
|
1231
|
+
throw new Error(
|
|
1232
|
+
`Manifest "${modelId}" has invalid audio_config.${label}=${JSON.stringify(value)}. ` +
|
|
1233
|
+
'Expected a non-empty string.'
|
|
1234
|
+
);
|
|
1235
|
+
}
|
|
1236
|
+
return value.trim();
|
|
1237
|
+
};
|
|
1231
1238
|
const audioArchitecture = String(ac.audio_architecture ?? '').trim();
|
|
1232
1239
|
if (audioArchitecture !== 'gemma4') {
|
|
1233
1240
|
throw new Error(
|
|
@@ -1240,9 +1247,7 @@ function resolveAudioConfig(rawConfig, manifest) {
|
|
|
1240
1247
|
const isEncoderFree = (depth === 0);
|
|
1241
1248
|
|
|
1242
1249
|
const hiddenSize = resolveRequiredPositiveInteger(ac.hidden_size, 'hidden_size');
|
|
1243
|
-
const numAttentionHeads =
|
|
1244
|
-
? (ac.num_attention_heads !== undefined ? resolveRequiredPositiveInteger(ac.num_attention_heads, 'num_attention_heads') : 1)
|
|
1245
|
-
: resolveRequiredPositiveInteger(ac.num_attention_heads, 'num_attention_heads');
|
|
1250
|
+
const numAttentionHeads = resolveRequiredPositiveInteger(ac.num_attention_heads, 'num_attention_heads');
|
|
1246
1251
|
const headDim = Math.trunc(hiddenSize / numAttentionHeads);
|
|
1247
1252
|
|
|
1248
1253
|
if (!isEncoderFree) {
|
|
@@ -1259,29 +1264,17 @@ function resolveAudioConfig(rawConfig, manifest) {
|
|
|
1259
1264
|
hiddenSize,
|
|
1260
1265
|
numAttentionHeads,
|
|
1261
1266
|
headDim,
|
|
1262
|
-
convKernelSize:
|
|
1263
|
-
|
|
1264
|
-
: resolveRequiredPositiveInteger(ac.conv_kernel_size, 'conv_kernel_size'),
|
|
1265
|
-
subsamplingConvChannels: isEncoderFree
|
|
1266
|
-
? (ac.subsampling_conv_channels ? ac.subsampling_conv_channels.map(Number) : [])
|
|
1267
|
-
: ac.subsampling_conv_channels.map(Number),
|
|
1267
|
+
convKernelSize: resolveRequiredPositiveInteger(ac.conv_kernel_size, 'conv_kernel_size'),
|
|
1268
|
+
subsamplingConvChannels: ac.subsampling_conv_channels.map(Number),
|
|
1268
1269
|
outputProjDims: resolveRequiredPositiveInteger(ac.output_proj_dims, 'output_proj_dims'),
|
|
1269
|
-
attentionContextLeft:
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
? (ac.attention_logit_cap !== undefined ? resolveRequiredPositiveNumber(ac.attention_logit_cap, 'attention_logit_cap') : 1.0)
|
|
1278
|
-
: resolveRequiredPositiveNumber(ac.attention_logit_cap, 'attention_logit_cap'),
|
|
1279
|
-
attentionInvalidLogitsValue: Number(ac.attention_invalid_logits_value ?? -1e9),
|
|
1280
|
-
residualWeight: isEncoderFree
|
|
1281
|
-
? (ac.residual_weight !== undefined ? resolveRequiredPositiveNumber(ac.residual_weight, 'residual_weight') : 1.0)
|
|
1282
|
-
: resolveRequiredPositiveNumber(ac.residual_weight, 'residual_weight'),
|
|
1283
|
-
rmsNormEps: resolveRequiredPositiveNumber(ac.rms_norm_eps ?? 1e-6, 'rms_norm_eps'),
|
|
1284
|
-
hiddenAct: String(ac.hidden_act ?? 'silu').trim(),
|
|
1270
|
+
attentionContextLeft: resolveRequiredPositiveInteger(ac.attention_context_left, 'attention_context_left'),
|
|
1271
|
+
attentionContextRight: resolveRequiredNonNegativeInteger(ac.attention_context_right, 'attention_context_right'),
|
|
1272
|
+
attentionChunkSize: resolveRequiredPositiveInteger(ac.attention_chunk_size, 'attention_chunk_size'),
|
|
1273
|
+
attentionLogitCap: resolveRequiredPositiveNumber(ac.attention_logit_cap, 'attention_logit_cap'),
|
|
1274
|
+
attentionInvalidLogitsValue: resolveRequiredFiniteNumber(ac.attention_invalid_logits_value, 'attention_invalid_logits_value'),
|
|
1275
|
+
residualWeight: resolveRequiredPositiveNumber(ac.residual_weight, 'residual_weight'),
|
|
1276
|
+
rmsNormEps: resolveRequiredPositiveNumber(ac.rms_norm_eps, 'rms_norm_eps'),
|
|
1277
|
+
hiddenAct: resolveRequiredString(ac.hidden_act, 'hidden_act'),
|
|
1285
1278
|
useClippedLinears: ac.use_clipped_linears === true,
|
|
1286
1279
|
audioTokenId: rawConfig?.audio_token_id ?? manifest?.audio_token_id ?? null,
|
|
1287
1280
|
};
|
|
@@ -1364,7 +1357,7 @@ function toParsedConfigFromMerged(merged, manifest) {
|
|
|
1364
1357
|
`Re-convert the model using the latest converter to add manifest.architecture.`
|
|
1365
1358
|
);
|
|
1366
1359
|
}
|
|
1367
|
-
const resolvedIntermediateSize = resolveIntermediateSizeForRuntime(
|
|
1360
|
+
const resolvedIntermediateSize = resolveIntermediateSizeForRuntime(arch);
|
|
1368
1361
|
const archNumHeads = Number(arch.numAttentionHeads ?? arch.numHeads);
|
|
1369
1362
|
const archNumKVHeads = Number(arch.numKeyValueHeads ?? arch.numKVHeads);
|
|
1370
1363
|
const archHeadDim = Number(arch.headDim);
|
|
@@ -1569,6 +1562,24 @@ function toParsedConfigFromMerged(merged, manifest) {
|
|
|
1569
1562
|
);
|
|
1570
1563
|
}
|
|
1571
1564
|
}
|
|
1565
|
+
const validateLongropeFactorArray = (value, label) => {
|
|
1566
|
+
if (!Array.isArray(value)) {
|
|
1567
|
+
throw new Error(`Manifest "${merged.modelId}" ${label} must be an array for LongRoPE.`);
|
|
1568
|
+
}
|
|
1569
|
+
if (value.length !== ropeRotaryDim / 2) {
|
|
1570
|
+
throw new Error(
|
|
1571
|
+
`Manifest "${merged.modelId}" ${label} length ${value.length} does not match ` +
|
|
1572
|
+
`resolved RoPE half dim ${ropeRotaryDim / 2}.`
|
|
1573
|
+
);
|
|
1574
|
+
}
|
|
1575
|
+
for (let i = 0; i < value.length; i += 1) {
|
|
1576
|
+
const entry = Number(value[i]);
|
|
1577
|
+
if (!Number.isFinite(entry) || entry <= 0) {
|
|
1578
|
+
throw new Error(`Manifest "${merged.modelId}" ${label}[${i}] must be a positive finite number.`);
|
|
1579
|
+
}
|
|
1580
|
+
}
|
|
1581
|
+
return value.map((entry) => Number(entry));
|
|
1582
|
+
};
|
|
1572
1583
|
|
|
1573
1584
|
// Build ropeScaling object from manifest values if scaling is enabled
|
|
1574
1585
|
// Include YARN params when present
|
|
@@ -1581,6 +1592,11 @@ function toParsedConfigFromMerged(merged, manifest) {
|
|
|
1581
1592
|
...(ropeScalingType === 'yarn' && inf.rope.yarnOriginalMaxPos != null && {
|
|
1582
1593
|
original_max_position_embeddings: inf.rope.yarnOriginalMaxPos
|
|
1583
1594
|
}),
|
|
1595
|
+
...(ropeScalingType === 'longrope' && {
|
|
1596
|
+
short_factor: validateLongropeFactorArray(inf.rope.longropeShortFactor, 'rope.longropeShortFactor'),
|
|
1597
|
+
long_factor: validateLongropeFactorArray(inf.rope.longropeLongFactor, 'rope.longropeLongFactor'),
|
|
1598
|
+
original_max_position_embeddings: inf.rope.longropeOriginalMaxPos,
|
|
1599
|
+
}),
|
|
1584
1600
|
} : null;
|
|
1585
1601
|
const ropeLocalScaling = ropeLocalScalingType ? {
|
|
1586
1602
|
type: ropeLocalScalingType,
|
|
@@ -1681,6 +1697,9 @@ function toParsedConfigFromMerged(merged, manifest) {
|
|
|
1681
1697
|
const sessionSettings = resolveSessionSettings(inf, merged.modelId);
|
|
1682
1698
|
const largeWeightsConfig = resolveLargeWeightsConfig(inf, merged.modelId);
|
|
1683
1699
|
const diffusionGemma = inf.diffusionGemma ?? null;
|
|
1700
|
+
const embeddingScale = inf.output.embeddingScale;
|
|
1701
|
+
const logitInputScale = inf.output.logitInputScale;
|
|
1702
|
+
const residualBranchScale = inf.layerPattern.residualBranchScale;
|
|
1684
1703
|
|
|
1685
1704
|
return {
|
|
1686
1705
|
modelType: manifest.modelType,
|
|
@@ -1730,12 +1749,15 @@ function toParsedConfigFromMerged(merged, manifest) {
|
|
|
1730
1749
|
preFeedforwardNorm: inf.normalization.preFeedforwardNorm,
|
|
1731
1750
|
postFeedforwardNorm: inf.normalization.postFeedforwardNorm,
|
|
1732
1751
|
scaleEmbeddings: inf.output.scaleEmbeddings,
|
|
1752
|
+
embeddingScale,
|
|
1753
|
+
logitInputScale,
|
|
1754
|
+
residualBranchScale,
|
|
1733
1755
|
useTiedEmbeddings: inf.output.tieWordEmbeddings,
|
|
1734
1756
|
embeddingTranspose: inf.output.embeddingTranspose,
|
|
1735
1757
|
embeddingVocabSize: inf.output.embeddingVocabSize,
|
|
1736
1758
|
embeddingPostprocessor,
|
|
1737
1759
|
hiddenActivation,
|
|
1738
|
-
ffnBranchMode: inf.ffn.branchMode
|
|
1760
|
+
ffnBranchMode: inf.ffn.branchMode,
|
|
1739
1761
|
useDoubleWideMlp: inf.ffn.useDoubleWideMlp,
|
|
1740
1762
|
swigluLimit: inf.ffn.swigluLimit,
|
|
1741
1763
|
stopTokenIds,
|
|
@@ -1775,6 +1797,7 @@ function toParsedConfigFromMerged(merged, manifest) {
|
|
|
1775
1797
|
|
|
1776
1798
|
export function parseModelConfigFromManifest(manifest, runtimeOverrides) {
|
|
1777
1799
|
assertSupportedRuntimeModelType(manifest);
|
|
1800
|
+
validateModelOverrides(runtimeOverrides, 'runtime.inference.modelOverrides');
|
|
1778
1801
|
|
|
1779
1802
|
// Merge manifest inference with runtime overrides
|
|
1780
1803
|
const merged = mergeConfig(
|
|
@@ -63,6 +63,28 @@ export function decodeRangeChunkIntoOutput(bytes, sourceDtype, output, dstOffset
|
|
|
63
63
|
}
|
|
64
64
|
}
|
|
65
65
|
|
|
66
|
+
function resolveEmbeddingScale(config, hiddenSize) {
|
|
67
|
+
const embeddingScale = config.embeddingScale;
|
|
68
|
+
const scaleEmbeddings = config.scaleEmbeddings;
|
|
69
|
+
if (embeddingScale === undefined) {
|
|
70
|
+
throw new Error('[Embed] embeddingScale must be explicitly set (null to use scaleEmbeddings semantics).');
|
|
71
|
+
}
|
|
72
|
+
if (scaleEmbeddings == null) {
|
|
73
|
+
throw new Error('[Embed] scaleEmbeddings is required.');
|
|
74
|
+
}
|
|
75
|
+
if (embeddingScale !== null) {
|
|
76
|
+
const value = Number(embeddingScale);
|
|
77
|
+
if (!Number.isFinite(value) || value <= 0) {
|
|
78
|
+
throw new Error(`[Embed] embeddingScale must be a positive finite number or null; got "${String(embeddingScale)}".`);
|
|
79
|
+
}
|
|
80
|
+
if (scaleEmbeddings === true) {
|
|
81
|
+
throw new Error('[Embed] embeddingScale cannot be set when scaleEmbeddings is true.');
|
|
82
|
+
}
|
|
83
|
+
return value;
|
|
84
|
+
}
|
|
85
|
+
return scaleEmbeddings === true ? Math.sqrt(hiddenSize) : 1;
|
|
86
|
+
}
|
|
87
|
+
|
|
66
88
|
async function readGpuTokenIdsForCpuEmbeddingGather(tokenIds, numTokens, indexOffset) {
|
|
67
89
|
if (numTokens <= 0) {
|
|
68
90
|
throw new Error('[Embed] numTokens must be provided when tokenIds is a GPUBuffer.');
|
|
@@ -77,6 +99,7 @@ export async function embed(tokenIds, embedBuffer, config) {
|
|
|
77
99
|
hiddenSize,
|
|
78
100
|
vocabSize,
|
|
79
101
|
scaleEmbeddings,
|
|
102
|
+
embeddingScale,
|
|
80
103
|
debug = false,
|
|
81
104
|
recorder,
|
|
82
105
|
outputBuffer: preAllocatedOutput,
|
|
@@ -91,6 +114,7 @@ export async function embed(tokenIds, embedBuffer, config) {
|
|
|
91
114
|
embeddingStorageEncoding = null,
|
|
92
115
|
} = config;
|
|
93
116
|
const device = getDevice();
|
|
117
|
+
const resolvedEmbeddingScale = resolveEmbeddingScale({ scaleEmbeddings, embeddingScale }, hiddenSize);
|
|
94
118
|
const tokenBufferInput = isGpuBufferInstance(tokenIds);
|
|
95
119
|
let tokenIdArray = tokenBufferInput ? null : (tokenIds);
|
|
96
120
|
const numTokens = tokenBufferInput
|
|
@@ -193,7 +217,14 @@ export async function embed(tokenIds, embedBuffer, config) {
|
|
|
193
217
|
? cpuEmbeddings
|
|
194
218
|
: null;
|
|
195
219
|
if (rangeBackedSource) {
|
|
196
|
-
const
|
|
220
|
+
const rawSourceDtype = rangeBackedSource.sourceDtype ?? embedBuffer.dtype;
|
|
221
|
+
if (rawSourceDtype == null) {
|
|
222
|
+
throw new Error('[Embed] CPU embedding range source requires sourceDtype or embedding buffer dtype metadata.');
|
|
223
|
+
}
|
|
224
|
+
const sourceDtype = String(rawSourceDtype).toLowerCase();
|
|
225
|
+
if (sourceDtype !== 'f16' && sourceDtype !== 'bf16' && sourceDtype !== 'f32') {
|
|
226
|
+
throw new Error(`[Embed] CPU embedding range source dtype "${sourceDtype}" is unsupported.`);
|
|
227
|
+
}
|
|
197
228
|
const bytesPerElement = sourceDtype === 'f16' || sourceDtype === 'bf16' ? 2 : 4;
|
|
198
229
|
if (!transpose) {
|
|
199
230
|
for (let t = 0; t < numTokens; t++) {
|
|
@@ -256,10 +287,9 @@ export async function embed(tokenIds, embedBuffer, config) {
|
|
|
256
287
|
}
|
|
257
288
|
} // end else (non-preloaded path)
|
|
258
289
|
|
|
259
|
-
if (
|
|
260
|
-
const scaleFactor = Math.sqrt(hiddenSize);
|
|
290
|
+
if (resolvedEmbeddingScale !== 1) {
|
|
261
291
|
for (let i = 0; i < output.length; i++) {
|
|
262
|
-
output[i] *=
|
|
292
|
+
output[i] *= resolvedEmbeddingScale;
|
|
263
293
|
}
|
|
264
294
|
}
|
|
265
295
|
|
|
@@ -375,7 +405,7 @@ export async function embed(tokenIds, embedBuffer, config) {
|
|
|
375
405
|
}
|
|
376
406
|
}
|
|
377
407
|
|
|
378
|
-
if (
|
|
408
|
+
if (resolvedEmbeddingScale === 1) {
|
|
379
409
|
await runProbes(probeStage, gatherOutput.buffer, {
|
|
380
410
|
numTokens,
|
|
381
411
|
hiddenSize,
|
|
@@ -387,9 +417,6 @@ export async function embed(tokenIds, embedBuffer, config) {
|
|
|
387
417
|
return gatherOutput;
|
|
388
418
|
}
|
|
389
419
|
|
|
390
|
-
// Apply embedding scaling: sqrt(hiddenSize)
|
|
391
|
-
const scaleFactor = Math.sqrt(hiddenSize);
|
|
392
|
-
|
|
393
420
|
// Debug: check raw embedding values before scaling
|
|
394
421
|
if (debug && !recorder) {
|
|
395
422
|
const bytesPerElement = gatherOptions.outputDtype === 'f16' ? 2 : 4;
|
|
@@ -401,7 +428,7 @@ export async function embed(tokenIds, embedBuffer, config) {
|
|
|
401
428
|
const abs = Math.abs(f32[i]);
|
|
402
429
|
if (abs > maxAbs) maxAbs = abs;
|
|
403
430
|
}
|
|
404
|
-
trace.embed(`RAW (before scale): maxAbs=${maxAbs.toFixed(4)}, scaleFactor=${
|
|
431
|
+
trace.embed(`RAW (before scale): maxAbs=${maxAbs.toFixed(4)}, scaleFactor=${resolvedEmbeddingScale.toFixed(4)}`);
|
|
405
432
|
}
|
|
406
433
|
|
|
407
434
|
const gatheredTensor = createTensor(
|
|
@@ -411,10 +438,10 @@ export async function embed(tokenIds, embedBuffer, config) {
|
|
|
411
438
|
'embed_gather_output'
|
|
412
439
|
);
|
|
413
440
|
const scaledTensor = recorder
|
|
414
|
-
? await recordScale(recorder, gatheredTensor,
|
|
441
|
+
? await recordScale(recorder, gatheredTensor, resolvedEmbeddingScale, {
|
|
415
442
|
count: numTokens * hiddenSize,
|
|
416
443
|
})
|
|
417
|
-
: await runScale(gatheredTensor,
|
|
444
|
+
: await runScale(gatheredTensor, resolvedEmbeddingScale, {
|
|
418
445
|
count: numTokens * hiddenSize,
|
|
419
446
|
});
|
|
420
447
|
const scaledBuffer = scaledTensor.buffer;
|
|
@@ -441,7 +468,7 @@ export async function embed(tokenIds, embedBuffer, config) {
|
|
|
441
468
|
const abs = Math.abs(f32[i]);
|
|
442
469
|
if (abs > maxAbs) maxAbs = abs;
|
|
443
470
|
}
|
|
444
|
-
trace.embed(`SCALED (after *${
|
|
471
|
+
trace.embed(`SCALED (after *${resolvedEmbeddingScale.toFixed(2)}): maxAbs=${maxAbs.toFixed(4)}, buffer.label=${scaledBuffer.label}, buffer.size=${scaledBuffer.size}`);
|
|
445
472
|
trace.embed(`RETURNING buffer with first8=[${Array.from(f32).slice(0, 8).map(x => x.toFixed(4)).join(', ')}]`);
|
|
446
473
|
if (f32.some(x => !Number.isFinite(x))) {
|
|
447
474
|
throw new Error('[Embed] Scaled embedding contains NaN/Inf');
|
|
@@ -8,7 +8,6 @@ import { log } from '../../../debug/index.js';
|
|
|
8
8
|
|
|
9
9
|
const PRIMARY_EXECUTION_PLAN_ID = 'primary';
|
|
10
10
|
const FINITENESS_FALLBACK_EXECUTION_PLAN_ID = 'finiteness_fallback';
|
|
11
|
-
const DEFAULT_MAX_TOKENS = 256;
|
|
12
11
|
|
|
13
12
|
function assertOptionalBoolean(value, label) {
|
|
14
13
|
if (value === undefined) {
|
|
@@ -30,6 +29,13 @@ function assertOptionalPositiveInt(value, label) {
|
|
|
30
29
|
return value;
|
|
31
30
|
}
|
|
32
31
|
|
|
32
|
+
function assertRequiredPositiveInt(value, label) {
|
|
33
|
+
if (!Number.isInteger(value) || value < 1) {
|
|
34
|
+
throw new Error(`[ExecutionPlan] ${label} must be a positive integer; got ${JSON.stringify(value)}.`);
|
|
35
|
+
}
|
|
36
|
+
return value;
|
|
37
|
+
}
|
|
38
|
+
|
|
33
39
|
function assertOptionalStopCheckMode(value) {
|
|
34
40
|
if (value === undefined) {
|
|
35
41
|
return undefined;
|
|
@@ -215,7 +221,7 @@ export function compileExecutionPlanState(options) {
|
|
|
215
221
|
const batchingConfig = {
|
|
216
222
|
batchSize: decodeLoopConfig.batchSize,
|
|
217
223
|
stopCheckMode: decodeLoopConfig.stopCheckMode,
|
|
218
|
-
maxTokens: generationConfig.maxTokens
|
|
224
|
+
maxTokens: assertRequiredPositiveInt(generationConfig.maxTokens, 'runtime.inference.generation.maxTokens'),
|
|
219
225
|
readbackInterval: decodeLoopConfig.readbackInterval,
|
|
220
226
|
readbackMode: assertReadbackMode(decodeLoopConfig.readbackMode),
|
|
221
227
|
ringTokens: decodeLoopConfig.ringTokens,
|
|
@@ -291,6 +291,13 @@ export function resolveFusedGateUpWeights(layerWeights, options = {}) {
|
|
|
291
291
|
};
|
|
292
292
|
}
|
|
293
293
|
|
|
294
|
+
function requireFusedWeightDtype(dtype, label) {
|
|
295
|
+
if (dtype !== 'f16' && dtype !== 'f32' && dtype !== 'q4k') {
|
|
296
|
+
throw new Error(`[FFN] ${label} dtype metadata is required for fused gate/up planning.`);
|
|
297
|
+
}
|
|
298
|
+
return dtype;
|
|
299
|
+
}
|
|
300
|
+
|
|
294
301
|
async function dispatchActivation(hiddenActivation, input, options, recorder) {
|
|
295
302
|
const op = resolveActivationOp(hiddenActivation);
|
|
296
303
|
const fn = ACTIVATION_FN_MAP[op];
|
|
@@ -591,8 +598,12 @@ export async function runDenseFFNGPU(
|
|
|
591
598
|
phase,
|
|
592
599
|
layerIdx,
|
|
593
600
|
});
|
|
594
|
-
const gateDtype =
|
|
595
|
-
|
|
601
|
+
const gateDtype = hasGate
|
|
602
|
+
? requireFusedWeightDtype(fusedGateUpWeights.gateDtype, 'gate')
|
|
603
|
+
: null;
|
|
604
|
+
const upDtype = hasUp
|
|
605
|
+
? requireFusedWeightDtype(fusedGateUpWeights.upDtype, 'up')
|
|
606
|
+
: null;
|
|
596
607
|
const dtypeMatches = gateDtype != null && upDtype != null && gateDtype === upDtype;
|
|
597
608
|
const q4kFusedAllowed = gateDtype !== 'q4k' || !isFusedQ4KDisabled({ kernelPath });
|
|
598
609
|
const dtypeSupported = gateDtype === 'f16' || gateDtype === 'f32' || (gateDtype === 'q4k' && q4kFusedAllowed);
|
|
@@ -1309,8 +1320,8 @@ export async function runDenseFFNWithFusedPostNormGPU(
|
|
|
1309
1320
|
});
|
|
1310
1321
|
const fusedGateWeight = getWeightBuffer(fusedGateUpWeights.gate ?? layerWeights.gate, 'ffn_gate');
|
|
1311
1322
|
const fusedUpWeight = getWeightBuffer(fusedGateUpWeights.up ?? layerWeights.up, 'ffn_up');
|
|
1312
|
-
const gateDtype = fusedGateUpWeights.gateDtype
|
|
1313
|
-
const upDtype = fusedGateUpWeights.upDtype
|
|
1323
|
+
const gateDtype = requireFusedWeightDtype(fusedGateUpWeights.gateDtype, 'gate');
|
|
1324
|
+
const upDtype = requireFusedWeightDtype(fusedGateUpWeights.upDtype, 'up');
|
|
1314
1325
|
const hasLoRAGate = Boolean(getLoRAModule(lora, layerIdx, 'gate_proj'));
|
|
1315
1326
|
const hasLoRAUp = Boolean(getLoRAModule(lora, layerIdx, 'up_proj'));
|
|
1316
1327
|
const dtypeMatches = gateDtype != null && upDtype != null && gateDtype === upDtype;
|
|
@@ -23,14 +23,15 @@ export async function runMoEFFNGPU(
|
|
|
23
23
|
}
|
|
24
24
|
|
|
25
25
|
const { moeFeedForwardGPU } = await import('../moe-impl.js');
|
|
26
|
+
if (typeof config.modelType !== 'string' || config.modelType.length === 0) {
|
|
27
|
+
throw new Error('MoE config.modelType is required; re-convert the model with an explicit manifest modelType.');
|
|
28
|
+
}
|
|
26
29
|
|
|
27
30
|
const outputBuffer = await moeFeedForwardGPU(
|
|
28
31
|
inputTensor.buffer,
|
|
29
32
|
numTokens,
|
|
30
33
|
{
|
|
31
|
-
modelType: config.modelType
|
|
32
|
-
config.expertFormat === 'gpt-oss' ? 'gpt-oss' : (config.expertFormat === 'mixtral' ? 'mixtral' : 'gemma4')
|
|
33
|
-
),
|
|
34
|
+
modelType: config.modelType,
|
|
34
35
|
hiddenSize: config.hiddenSize,
|
|
35
36
|
intermediateSize: config.intermediateSize,
|
|
36
37
|
rmsNormEps: config.rmsNormEps,
|
|
@@ -21,5 +21,6 @@ export declare function processFFNStandard(
|
|
|
21
21
|
context: LayerContext,
|
|
22
22
|
layerWeights: LayerWeights | undefined,
|
|
23
23
|
fusedResidualInput?: Tensor | null,
|
|
24
|
-
finalOutputScale?: number | null
|
|
24
|
+
finalOutputScale?: number | null,
|
|
25
|
+
residualBranchScale?: number | null
|
|
25
26
|
): Promise<Tensor>;
|