doppler-gpu 0.4.6 → 0.4.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +43 -31
- package/assets/doppler-runtime-map.svg +1 -3
- package/assets/doppler.svg +1 -3
- package/package.json +25 -17
- package/src/cli/doppler-quickstart.js +1 -1
- package/src/cli/doppler-serve.js +2 -2
- package/src/client/doppler-registry.d.ts +19 -0
- package/src/client/doppler-registry.js +12 -0
- package/src/client/doppler-registry.json +50 -1
- package/src/config/conversion/README.md +4 -0
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +28 -4
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +31 -5
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +30 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +30 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +30 -4
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +30 -5
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +30 -5
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +29 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +29 -4
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +29 -4
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/minicpm/minicpm4-0-5b-f16-af32.json +481 -0
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +29 -4
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +29 -4
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +30 -5
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +29 -4
- package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-q4k-ehf16-af32.json +395 -0
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-af32.json +429 -0
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +540 -0
- package/src/config/execution-contract-check.js +14 -10
- package/src/config/merge-contract-check.js +1 -0
- package/src/config/merge.js +30 -0
- package/src/config/param-validator.d.ts +5 -0
- package/src/config/param-validator.js +42 -8
- package/src/config/required-inference-fields-contract-check.js +14 -0
- package/src/config/schema/debug.schema.d.ts +2 -0
- package/src/config/schema/debug.schema.js +1 -0
- package/src/config/schema/doppler.schema.d.ts +5 -1
- package/src/config/schema/doppler.schema.js +82 -292
- package/src/config/schema/inference.schema.d.ts +17 -3
- package/src/config/schema/kvcache.schema.d.ts +19 -3
- package/src/config/schema/kvcache.schema.js +2 -1
- package/src/config/schema/manifest.schema.d.ts +38 -1
- package/src/config/schema/manifest.schema.js +18 -0
- package/src/config/schema/policy-schema-registry.json +5 -0
- package/src/config/source-packages/litert/gemma-4-12b-it.json +20 -1
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +20 -1
- package/src/config/source-packages/litert/gemma-4-e4b-it.json +21 -1
- package/src/config/transforms/execution-graph-transforms.d.ts +6 -0
- package/src/config/transforms/execution-graph-transforms.js +8 -0
- package/src/converter/conversion-plan.js +2 -0
- package/src/converter/parsers/transformer.js +27 -1
- package/src/converter/quantization-info.js +16 -4
- package/src/converter/rope-config.js +55 -3
- package/src/experimental/training/distillation/student-fixture.js +1 -1
- package/src/experimental/training/lora-pipeline.js +1 -1
- package/src/experimental/training/lora.js +2 -2
- package/src/experimental/training/suite.js +1 -1
- package/src/gpu/kernels/README.md +8 -1
- package/src/gpu/kernels/backward/attention_backward.js +8 -0
- package/src/gpu/kernels/dequant.d.ts +6 -6
- package/src/gpu/kernels/dequant.js +17 -6
- package/src/gpu/kernels/linear-attention-core.js +1 -6
- package/src/gpu/kernels/matmul-selection.d.ts +2 -2
- package/src/gpu/kernels/matmul-selection.js +4 -1
- package/src/gpu/kernels/matmul.d.ts +1 -1
- package/src/gpu/kernels/matmul.js +24 -6
- package/src/gpu/kernels/moe.d.ts +5 -1
- package/src/gpu/kernels/moe.js +5 -2
- package/src/gpu/kernels/softmax.d.ts +3 -3
- package/src/gpu/kernels/softmax.js +9 -2
- package/src/gpu/kernels/turboquant-codebook.js +1 -1
- package/src/gpu/kernels/types.d.ts +1 -1
- package/src/gpu/profiler.js +11 -1
- package/src/inference/browser-harness-model-helpers.js +4 -1
- package/src/inference/browser-harness-text-helpers.d.ts +28 -0
- package/src/inference/browser-harness-text-helpers.js +357 -116
- package/src/inference/browser-harness.d.ts +3 -3
- package/src/inference/browser-harness.js +266 -11
- package/src/inference/fixtures/rerank-semantic-fixtures.json +38 -0
- package/src/inference/kv-cache/quantized.js +16 -2
- package/src/inference/kv-cache/tiered.js +50 -15
- package/src/inference/moe-router.js +38 -7
- package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +1 -1
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +8 -6
- package/src/inference/pipelines/text/config.d.ts +5 -1
- package/src/inference/pipelines/text/config.js +135 -112
- package/src/inference/pipelines/text/embed.d.ts +1 -0
- package/src/inference/pipelines/text/embed.js +39 -12
- package/src/inference/pipelines/text/execution-plan.js +8 -2
- package/src/inference/pipelines/text/ffn/dense.js +15 -4
- package/src/inference/pipelines/text/ffn/moe.js +4 -3
- package/src/inference/pipelines/text/ffn/standard.d.ts +2 -1
- package/src/inference/pipelines/text/ffn/standard.js +25 -2
- package/src/inference/pipelines/text/generator-helpers.js +2 -1
- package/src/inference/pipelines/text/generator-runtime.d.ts +6 -2
- package/src/inference/pipelines/text/generator-runtime.js +53 -20
- package/src/inference/pipelines/text/generator-steps.js +3 -0
- package/src/inference/pipelines/text/generator.js +6 -1
- package/src/inference/pipelines/text/init-rope.js +54 -6
- package/src/inference/pipelines/text/init.js +63 -15
- package/src/inference/pipelines/text/layer.js +32 -5
- package/src/inference/pipelines/text/linear-attention.js +12 -8
- package/src/inference/pipelines/text/logits/gpu.d.ts +1 -1
- package/src/inference/pipelines/text/logits/gpu.js +76 -113
- package/src/inference/pipelines/text/logits/index.js +44 -87
- package/src/inference/pipelines/text/logits/precision-policy.d.ts +11 -0
- package/src/inference/pipelines/text/logits/precision-policy.js +103 -0
- package/src/inference/pipelines/text/logits/types.d.ts +1 -0
- package/src/inference/pipelines/text/logits/utils.d.ts +2 -0
- package/src/inference/pipelines/text/logits/utils.js +11 -0
- package/src/inference/pipelines/text/lora-apply.js +6 -4
- package/src/inference/pipelines/text/moe-cpu.js +25 -0
- package/src/inference/pipelines/text/moe-gpu.js +148 -97
- package/src/inference/pipelines/text/moe-shape-validator.d.ts +33 -3
- package/src/inference/pipelines/text/moe-shape-validator.js +149 -31
- package/src/inference/pipelines/text/ops.d.ts +3 -0
- package/src/inference/pipelines/text/ops.js +26 -1
- package/src/inference/pipelines/text/per-layer-inputs.js +27 -12
- package/src/inference/pipelines/text/weights.js +7 -2
- package/src/inference/pipelines/text.js +24 -4
- package/src/inference/tokenizers/behavior-flags.js +23 -2
- package/src/loader/final-weights-loader.js +12 -1
- package/src/rules/execution-rules-contract-check.js +12 -3
- package/src/rules/inference/capability-transforms.rules.json +14 -2
- package/src/rules/inference/config.rules.json +26 -2
- package/src/rules/inference/dtype.rules.json +12 -5
- package/src/rules/inference/execution.rules.json +5 -19
- package/src/rules/inference/moe.rules.json +97 -0
- package/src/tooling/command-api-constants.js +2 -2
- package/src/tooling/command-api-family-normalizers.js +2 -1
- package/src/tooling/command-api.d.ts +1 -1
- package/src/tooling/command-runner-shared.d.ts +1 -1
- package/src/tooling/command-runner-shared.js +3 -1
- package/src/tooling/node-converter.js +20 -0
- package/src/tooling/node-source-runtime.js +5 -1
- package/src/version.js +1 -1
- package/assets/doppler-readme-hero.svg +0 -84
|
@@ -1,39 +1,131 @@
|
|
|
1
1
|
import { getKernelCapabilities } from '../../../gpu/device.js';
|
|
2
2
|
import { selectRuleValue } from '../../../rules/rule-registry.js';
|
|
3
3
|
|
|
4
|
+
const KERNEL_PATH_PROFILE_RESOLVERS = Object.freeze({
|
|
5
|
+
'gpt-oss': resolveGptOssKernelPathProfile,
|
|
6
|
+
mixtral: resolveMixtralKernelPathProfile,
|
|
7
|
+
});
|
|
8
|
+
|
|
4
9
|
function asVendorString(caps) {
|
|
5
10
|
const raw = caps?.adapterInfo?.vendor;
|
|
6
11
|
return typeof raw === 'string' && raw.trim() !== '' ? raw.toLowerCase() : 'unknown';
|
|
7
12
|
}
|
|
8
13
|
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
+
function requireRuleContextValue(value, label) {
|
|
15
|
+
if (value == null || value === '') {
|
|
16
|
+
throw new Error(`[MoE] ${label} is required for rule selection.`);
|
|
17
|
+
}
|
|
18
|
+
return value;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
function requireMoeProfileField(profile, field) {
|
|
22
|
+
const value = profile?.[field];
|
|
23
|
+
if (typeof value !== 'string' || value.length === 0) {
|
|
24
|
+
throw new Error(`[MoE] execution profile is missing ${field}.`);
|
|
25
|
+
}
|
|
26
|
+
return value;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function requireManifestString(value, label) {
|
|
30
|
+
if (typeof value !== 'string' || value.length === 0) {
|
|
31
|
+
throw new Error(`[MoE] ${label} is required for MoE execution profile resolution.`);
|
|
32
|
+
}
|
|
33
|
+
return value;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
function validateVendorProfile(profile) {
|
|
37
|
+
if (!profile || typeof profile !== 'object' || Array.isArray(profile)) {
|
|
38
|
+
throw new Error('[MoE] vendor profile must resolve to an object.');
|
|
39
|
+
}
|
|
40
|
+
if (typeof profile.preferVec4Dequant !== 'boolean') {
|
|
41
|
+
throw new Error('[MoE] vendor profile missing preferVec4Dequant boolean.');
|
|
42
|
+
}
|
|
43
|
+
if (profile.dequantTileShape !== null && profile.dequantTileShape !== 'vec4' && profile.dequantTileShape !== 'scalar') {
|
|
44
|
+
throw new Error('[MoE] vendor profile dequantTileShape must be null, "vec4", or "scalar".');
|
|
45
|
+
}
|
|
46
|
+
if (!Number.isFinite(profile.routerWorkgroupSize) || profile.routerWorkgroupSize <= 0) {
|
|
47
|
+
throw new Error('[MoE] vendor profile missing positive routerWorkgroupSize.');
|
|
48
|
+
}
|
|
49
|
+
if (!Number.isFinite(profile.maxTokensPerExpertScale) || profile.maxTokensPerExpertScale <= 0) {
|
|
50
|
+
throw new Error('[MoE] vendor profile missing positive maxTokensPerExpertScale.');
|
|
51
|
+
}
|
|
52
|
+
return profile;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
function validateMoeExecutionProfile(profile) {
|
|
56
|
+
if (!profile || typeof profile !== 'object' || Array.isArray(profile)) {
|
|
57
|
+
throw new Error('[MoE] executionProfile rule must resolve to an object.');
|
|
58
|
+
}
|
|
59
|
+
requireMoeProfileField(profile, 'id');
|
|
60
|
+
requireMoeProfileField(profile, 'label');
|
|
61
|
+
requireMoeProfileField(profile, 'expertExecutor');
|
|
62
|
+
requireMoeProfileField(profile, 'intermediateSizeSource');
|
|
63
|
+
if (typeof profile.requiresShaderF16 !== 'boolean') {
|
|
64
|
+
throw new Error('[MoE] execution profile is missing requiresShaderF16 boolean.');
|
|
65
|
+
}
|
|
66
|
+
if (profile.routerScaleMode !== 'none' && profile.routerScaleMode !== 'optional' && profile.routerScaleMode !== 'required') {
|
|
67
|
+
throw new Error('[MoE] execution profile routerScaleMode must be none, optional, or required.');
|
|
68
|
+
}
|
|
69
|
+
if (!profile.vendorProfile || typeof profile.vendorProfile !== 'object' || Array.isArray(profile.vendorProfile)) {
|
|
70
|
+
throw new Error('[MoE] execution profile is missing vendorProfile selector.');
|
|
71
|
+
}
|
|
72
|
+
return profile;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
export function resolveMoeExecutionProfile(config, options = {}) {
|
|
76
|
+
const expertFormat = requireManifestString(config?.expertFormat, 'manifest moeConfig.expertFormat');
|
|
77
|
+
const modelType = requireManifestString(
|
|
78
|
+
typeof options?.modelType === 'string' && options.modelType.length > 0
|
|
79
|
+
? options.modelType
|
|
80
|
+
: config?.modelType,
|
|
81
|
+
'manifest modelType'
|
|
82
|
+
);
|
|
83
|
+
const profile = selectRuleValue('inference', 'moe', 'executionProfile', {
|
|
84
|
+
modelType,
|
|
85
|
+
expertFormat,
|
|
86
|
+
});
|
|
87
|
+
return validateMoeExecutionProfile(profile);
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
export function resolveMoeIntermediateSize(config, moeProfile) {
|
|
91
|
+
const source = requireMoeProfileField(moeProfile, 'intermediateSizeSource');
|
|
92
|
+
const resolvers = {
|
|
93
|
+
architecture: () => config?.intermediateSize,
|
|
94
|
+
expert: () => config?.expertIntermediateSize,
|
|
95
|
+
};
|
|
96
|
+
const resolver = resolvers[source];
|
|
97
|
+
if (typeof resolver !== 'function') {
|
|
98
|
+
throw new Error(`[MoE] Unknown intermediateSizeSource "${source}" in execution profile "${moeProfile.id}".`);
|
|
14
99
|
}
|
|
15
|
-
|
|
16
|
-
|
|
100
|
+
const value = Number(resolver());
|
|
101
|
+
if (!Number.isFinite(value) || value <= 0) {
|
|
102
|
+
throw new Error(`[MoE] Invalid ${source} intermediate size for profile "${moeProfile.id}": ${String(value)}.`);
|
|
103
|
+
}
|
|
104
|
+
return value;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
export function resolveMoeVendorProfile(moeProfile) {
|
|
108
|
+
const selector = moeProfile?.vendorProfile;
|
|
109
|
+
if (selector?.kind === 'rule') {
|
|
110
|
+
const caps = getKernelCapabilities();
|
|
111
|
+
const vendor = asVendorString(caps);
|
|
112
|
+
const profile = selectRuleValue(selector.domain, selector.group, selector.name, { vendor });
|
|
113
|
+
return validateVendorProfile(profile);
|
|
17
114
|
}
|
|
18
|
-
if (
|
|
19
|
-
return
|
|
20
|
-
preferVec4Dequant: false,
|
|
21
|
-
dequantTileShape: null,
|
|
22
|
-
routerWorkgroupSize: 256,
|
|
23
|
-
maxTokensPerExpertScale: 1.0,
|
|
24
|
-
};
|
|
115
|
+
if (selector?.kind === 'static') {
|
|
116
|
+
return validateVendorProfile(selector.value);
|
|
25
117
|
}
|
|
26
|
-
throw new Error(`[MoE]
|
|
118
|
+
throw new Error(`[MoE] execution profile "${moeProfile?.id ?? 'unknown'}" has invalid vendorProfile selector.`);
|
|
27
119
|
}
|
|
28
120
|
|
|
29
121
|
function resolveGptOssRuleContext(context) {
|
|
30
122
|
return {
|
|
31
|
-
modelType: '
|
|
123
|
+
modelType: requireRuleContextValue(context?.modelType, 'GPT-OSS kernel rule modelType'),
|
|
32
124
|
hasF16: context?.hasF16,
|
|
33
125
|
hasSubgroups: context?.hasSubgroups,
|
|
34
|
-
inputDtype: context?.routerDtype ?? context?.inputDtype,
|
|
35
|
-
weightsDtype: context?.weightsDtype,
|
|
36
|
-
outputDtype: context?.outputDtype
|
|
126
|
+
inputDtype: requireRuleContextValue(context?.routerDtype ?? context?.inputDtype, 'GPT-OSS router/input dtype'),
|
|
127
|
+
weightsDtype: requireRuleContextValue(context?.weightsDtype, 'GPT-OSS weights dtype'),
|
|
128
|
+
outputDtype: requireRuleContextValue(context?.outputDtype, 'GPT-OSS output dtype'),
|
|
37
129
|
groupSize: context?.groupSize,
|
|
38
130
|
dequantTileShape: context?.tileShape ?? context?.dequantTileShape,
|
|
39
131
|
};
|
|
@@ -49,12 +141,12 @@ export async function resolveGptOssKernelPathProfile(context) {
|
|
|
49
141
|
|
|
50
142
|
function resolveMixtralRuleContext(context) {
|
|
51
143
|
return {
|
|
52
|
-
modelType: '
|
|
144
|
+
modelType: requireRuleContextValue(context?.modelType, 'Mixtral kernel rule modelType'),
|
|
53
145
|
hasF16: context?.hasF16,
|
|
54
146
|
hasSubgroups: context?.hasSubgroups,
|
|
55
|
-
routerDtype: context?.routerDtype
|
|
56
|
-
weightsDtype: context?.weightsDtype,
|
|
57
|
-
outputDtype: context?.outputDtype
|
|
147
|
+
routerDtype: requireRuleContextValue(context?.routerDtype, 'Mixtral router dtype'),
|
|
148
|
+
weightsDtype: requireRuleContextValue(context?.weightsDtype, 'Mixtral weights dtype'),
|
|
149
|
+
outputDtype: requireRuleContextValue(context?.outputDtype, 'Mixtral output dtype'),
|
|
58
150
|
};
|
|
59
151
|
}
|
|
60
152
|
|
|
@@ -66,6 +158,30 @@ export async function resolveMixtralKernelPathProfile(context) {
|
|
|
66
158
|
};
|
|
67
159
|
}
|
|
68
160
|
|
|
161
|
+
export async function resolveMoeKernelPathProfile(moeProfile, context) {
|
|
162
|
+
const resolverId = moeProfile?.kernelPathProfileResolver ?? null;
|
|
163
|
+
if (resolverId === null) {
|
|
164
|
+
return null;
|
|
165
|
+
}
|
|
166
|
+
const resolver = KERNEL_PATH_PROFILE_RESOLVERS[resolverId];
|
|
167
|
+
if (typeof resolver !== 'function') {
|
|
168
|
+
throw new Error(`[MoE] Unknown kernelPathProfileResolver "${resolverId}" in execution profile "${moeProfile.id}".`);
|
|
169
|
+
}
|
|
170
|
+
const modelType = requireRuleContextValue(moeProfile.kernelRuleModelType, `${moeProfile.id} kernel rule modelType`);
|
|
171
|
+
return resolver({ ...context, modelType });
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
function resolveShapePolicy(moeProfile, context) {
|
|
175
|
+
const selector = moeProfile?.shapePolicy ?? null;
|
|
176
|
+
if (selector === null) {
|
|
177
|
+
return null;
|
|
178
|
+
}
|
|
179
|
+
if (selector.kind !== 'rule') {
|
|
180
|
+
throw new Error(`[MoE] execution profile "${moeProfile.id}" has invalid shapePolicy selector.`);
|
|
181
|
+
}
|
|
182
|
+
return selectRuleValue(selector.domain, selector.group, selector.name, context);
|
|
183
|
+
}
|
|
184
|
+
|
|
69
185
|
export function validateMoeShape(config, options = {}) {
|
|
70
186
|
const {
|
|
71
187
|
hiddenSize,
|
|
@@ -74,10 +190,12 @@ export function validateMoeShape(config, options = {}) {
|
|
|
74
190
|
numExperts,
|
|
75
191
|
expertFormat,
|
|
76
192
|
} = config;
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
193
|
+
if (typeof options.modelType !== 'string' || options.modelType.length === 0) {
|
|
194
|
+
throw new Error('[MoE] validateMoeShape requires options.modelType from the manifest.');
|
|
195
|
+
}
|
|
196
|
+
const moeProfile = options.moeProfile ?? resolveMoeExecutionProfile(
|
|
197
|
+
{ expertFormat, modelType: options.modelType },
|
|
198
|
+
{ modelType: options.modelType }
|
|
81
199
|
);
|
|
82
200
|
|
|
83
201
|
if (!Number.isFinite(hiddenSize) || hiddenSize <= 0) {
|
|
@@ -93,16 +211,16 @@ export function validateMoeShape(config, options = {}) {
|
|
|
93
211
|
throw new Error(`[MoE] topK must be in range [1, ${numExperts}], got ${moeTopK}.`);
|
|
94
212
|
}
|
|
95
213
|
|
|
96
|
-
|
|
97
|
-
|
|
214
|
+
const policy = resolveShapePolicy(moeProfile, { modelType: options.modelType });
|
|
215
|
+
if (policy != null) {
|
|
98
216
|
if (policy.hiddenSizeDivisor == null || policy.intermediateSizeDivisor == null) {
|
|
99
|
-
throw new Error(
|
|
217
|
+
throw new Error(`[MoE] ${moeProfile.label} shapePolicy is missing hiddenSizeDivisor or intermediateSizeDivisor.`);
|
|
100
218
|
}
|
|
101
219
|
const hiddenDivisor = policy.hiddenSizeDivisor;
|
|
102
220
|
const intermediateDivisor = policy.intermediateSizeDivisor;
|
|
103
221
|
if (hiddenSize % hiddenDivisor !== 0 || intermediateSize % intermediateDivisor !== 0) {
|
|
104
222
|
throw new Error(
|
|
105
|
-
`[MoE]
|
|
223
|
+
`[MoE] ${moeProfile.label} shape policy violation: hiddenSize (${hiddenSize}) % ${hiddenDivisor} = ${hiddenSize % hiddenDivisor}, ` +
|
|
106
224
|
`intermediateSize (${intermediateSize}) % ${intermediateDivisor} = ${intermediateSize % intermediateDivisor}.`
|
|
107
225
|
);
|
|
108
226
|
}
|
|
@@ -162,6 +162,9 @@ export function doConv(
|
|
|
162
162
|
layerIdx?: number;
|
|
163
163
|
label?: string;
|
|
164
164
|
kernelPath?: unknown;
|
|
165
|
+
weightDtype?: 'f16' | 'f32' | 'q4k';
|
|
166
|
+
convInProjDtype?: 'f16' | 'f32' | 'q4k';
|
|
167
|
+
convOutProjDtype?: 'f16' | 'f32' | 'q4k';
|
|
165
168
|
conv2d?: {
|
|
166
169
|
enabled: boolean;
|
|
167
170
|
inChannels: number;
|
|
@@ -16,7 +16,7 @@ import {
|
|
|
16
16
|
} from '../../../gpu/kernels/cast.js';
|
|
17
17
|
import { createTensor } from '../../../gpu/tensor.js';
|
|
18
18
|
import { releaseBuffer, readBuffer, acquireBuffer, uploadData } from '../../../memory/buffer-pool.js';
|
|
19
|
-
import { isGpuBufferInstance } from '../../../gpu/weight-buffer.js';
|
|
19
|
+
import { getWeightDtype, isGpuBufferInstance, isWeightBuffer } from '../../../gpu/weight-buffer.js';
|
|
20
20
|
import { kernelTrace, traceStep } from './kernel-trace.js';
|
|
21
21
|
import {
|
|
22
22
|
runLayerAttentionGPU,
|
|
@@ -180,6 +180,17 @@ export async function doMatmulRMSNormFused(input, weight, normWeight, options, r
|
|
|
180
180
|
return resultTensor;
|
|
181
181
|
}
|
|
182
182
|
|
|
183
|
+
function requireProjectionWeightDtype(weight, declared, label) {
|
|
184
|
+
if (isWeightBuffer(weight)) {
|
|
185
|
+
return getWeightDtype(weight);
|
|
186
|
+
}
|
|
187
|
+
const dtype = declared ?? weight?.dtype;
|
|
188
|
+
if (dtype === 'f16' || dtype === 'f32' || dtype === 'q4k') {
|
|
189
|
+
return dtype;
|
|
190
|
+
}
|
|
191
|
+
throw new Error(`${label} requires explicit weight dtype.`);
|
|
192
|
+
}
|
|
193
|
+
|
|
183
194
|
export async function doConv(
|
|
184
195
|
inputTensor,
|
|
185
196
|
convInProj,
|
|
@@ -200,6 +211,16 @@ export async function doConv(
|
|
|
200
211
|
if (!Number.isFinite(hiddenSize) || hiddenSize <= 0) {
|
|
201
212
|
throw new Error('doConv requires hiddenSize > 0.');
|
|
202
213
|
}
|
|
214
|
+
const convInProjDtype = requireProjectionWeightDtype(
|
|
215
|
+
convInProj,
|
|
216
|
+
options.convInProjDtype ?? options.weightDtype,
|
|
217
|
+
`${label}.in_proj`
|
|
218
|
+
);
|
|
219
|
+
const convOutProjDtype = requireProjectionWeightDtype(
|
|
220
|
+
convOutProj,
|
|
221
|
+
options.convOutProjDtype ?? options.weightDtype,
|
|
222
|
+
`${label}.out_proj`
|
|
223
|
+
);
|
|
203
224
|
|
|
204
225
|
// LFM2 gated short convolution (GPU-native):
|
|
205
226
|
// in_proj → 3×hidden → GPU kernel: split(B,C,x) + B*x + causal conv1d + C*conv_out → out_proj
|
|
@@ -225,6 +246,8 @@ export async function doConv(
|
|
|
225
246
|
kernelPath,
|
|
226
247
|
role: 'conv_in_proj',
|
|
227
248
|
executionPolicies: options.executionPolicies ?? null,
|
|
249
|
+
bDtype: convInProjDtype,
|
|
250
|
+
outputDtype: inputTensor.dtype,
|
|
228
251
|
},
|
|
229
252
|
recorder
|
|
230
253
|
);
|
|
@@ -265,6 +288,8 @@ export async function doConv(
|
|
|
265
288
|
kernelPath,
|
|
266
289
|
role: 'conv_out_proj',
|
|
267
290
|
executionPolicies: options.executionPolicies ?? null,
|
|
291
|
+
bDtype: convOutProjDtype,
|
|
292
|
+
outputDtype: convOut.dtype,
|
|
268
293
|
},
|
|
269
294
|
recorder
|
|
270
295
|
);
|
|
@@ -431,7 +431,7 @@ export async function ensurePleScaledProjectionNormWeight(context, combineScale
|
|
|
431
431
|
}
|
|
432
432
|
scaledProjectionNormWeight = createPleProjectionNormTensor(
|
|
433
433
|
scaledBuffer,
|
|
434
|
-
|
|
434
|
+
inferPleProjectionNormDtype(projectionNormWeight, hiddenSizePerLayerInput),
|
|
435
435
|
hiddenSizePerLayerInput,
|
|
436
436
|
'per_layer_projection_norm_scaled'
|
|
437
437
|
);
|
|
@@ -474,6 +474,17 @@ function getEmbeddingDtype(weight) {
|
|
|
474
474
|
return getWeightDtype(weight);
|
|
475
475
|
}
|
|
476
476
|
|
|
477
|
+
function requirePleSourceDtype(rawDtype, label, allowed = ['f16', 'f32']) {
|
|
478
|
+
if (rawDtype == null) {
|
|
479
|
+
throw new Error(`${label} requires source dtype metadata.`);
|
|
480
|
+
}
|
|
481
|
+
const sourceDtype = String(rawDtype).toLowerCase();
|
|
482
|
+
if (!allowed.includes(sourceDtype)) {
|
|
483
|
+
throw new Error(`${label} requires ${allowed.join('/')} source rows; got "${sourceDtype}".`);
|
|
484
|
+
}
|
|
485
|
+
return sourceDtype;
|
|
486
|
+
}
|
|
487
|
+
|
|
477
488
|
function getEmbeddingTranspose(weight) {
|
|
478
489
|
if (isWeightBuffer(weight) || isCpuWeightBuffer(weight)) {
|
|
479
490
|
return weight.layout === 'column';
|
|
@@ -721,11 +732,12 @@ function storePreparedTokenEntry(cache, tokenId, buffers, sessionConfig, activat
|
|
|
721
732
|
}
|
|
722
733
|
|
|
723
734
|
function getPleRangeRowLoadConfig(embedTokensPerLayer, totalPerLayerHiddenSize) {
|
|
724
|
-
const sourceDtype =
|
|
735
|
+
const sourceDtype = requirePleSourceDtype(
|
|
725
736
|
(isCpuWeightBuffer(embedTokensPerLayer) ? embedTokensPerLayer.data?.sourceDtype : null)
|
|
726
|
-
?? embedTokensPerLayer?.dtype
|
|
727
|
-
|
|
728
|
-
|
|
737
|
+
?? embedTokensPerLayer?.dtype,
|
|
738
|
+
'Gemma 4 range-backed per-layer input rows',
|
|
739
|
+
['f16', 'bf16', 'f32']
|
|
740
|
+
);
|
|
729
741
|
const bytesPerElement = (sourceDtype === 'f16' || sourceDtype === 'bf16') ? 2 : 4;
|
|
730
742
|
return {
|
|
731
743
|
sourceDtype,
|
|
@@ -916,12 +928,10 @@ export async function ensurePleGpuSplitTablesRuntime(context) {
|
|
|
916
928
|
throw new Error('Gemma 4 gpu_split_tables materialization requires a range-backed CPU embedTokensPerLayer source.');
|
|
917
929
|
}
|
|
918
930
|
|
|
919
|
-
const sourceDtype =
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
);
|
|
924
|
-
}
|
|
931
|
+
const sourceDtype = requirePleSourceDtype(
|
|
932
|
+
embedTokensPerLayer.data?.sourceDtype ?? embedTokensPerLayer.dtype,
|
|
933
|
+
'Gemma 4 gpu_split_tables materialization'
|
|
934
|
+
);
|
|
925
935
|
|
|
926
936
|
const device = getDevice();
|
|
927
937
|
if (!device) {
|
|
@@ -1124,7 +1134,11 @@ export async function ensurePleGpuHotVocabularyRuntime(context) {
|
|
|
1124
1134
|
return null;
|
|
1125
1135
|
}
|
|
1126
1136
|
|
|
1127
|
-
const sourceDtype =
|
|
1137
|
+
const sourceDtype = requirePleSourceDtype(
|
|
1138
|
+
embedTokensPerLayer.data?.sourceDtype ?? embedTokensPerLayer.dtype,
|
|
1139
|
+
'Gemma 4 hot vocabulary cache',
|
|
1140
|
+
[policy.outputDtype]
|
|
1141
|
+
);
|
|
1128
1142
|
if (sourceDtype !== policy.outputDtype) {
|
|
1129
1143
|
throw new Error(
|
|
1130
1144
|
`Gemma 4 hot vocabulary cache requires source dtype "${policy.outputDtype}" for zero-copy row packing; ` +
|
|
@@ -1551,6 +1565,7 @@ export async function preparePerLayerInputs(tokenIds, inputEmbedsTensor, context
|
|
|
1551
1565
|
hiddenSize: hiddenSizePerLayerInput,
|
|
1552
1566
|
vocabSize: useHotVocabularyTables ? (hotVocabularyRuntime.sentinelIndex + 1) : vocabSizePerLayerInput,
|
|
1553
1567
|
scaleEmbeddings: true,
|
|
1568
|
+
embeddingScale: null,
|
|
1554
1569
|
probeStage: 'per_layer_embed_out',
|
|
1555
1570
|
recorder,
|
|
1556
1571
|
numTokens,
|
|
@@ -42,14 +42,19 @@ export function getWeightBuffer(weight, label, deviceOverride = null) {
|
|
|
42
42
|
|
|
43
43
|
|
|
44
44
|
let data;
|
|
45
|
-
let bufferDtype
|
|
45
|
+
let bufferDtype;
|
|
46
46
|
if (isCpuWeightBuffer(weight)) {
|
|
47
47
|
data = weight.data;
|
|
48
|
-
|
|
48
|
+
if (!weight.dtype) {
|
|
49
|
+
throw new Error(`Weight buffer "${label}" is missing dtype metadata.`);
|
|
50
|
+
}
|
|
51
|
+
bufferDtype = weight.dtype;
|
|
49
52
|
} else if (weight instanceof Float32Array) {
|
|
50
53
|
data = weight;
|
|
54
|
+
bufferDtype = 'f32';
|
|
51
55
|
} else {
|
|
52
56
|
data = new Float32Array( (weight));
|
|
57
|
+
bufferDtype = 'f32';
|
|
53
58
|
}
|
|
54
59
|
|
|
55
60
|
const buf = acquireBuffer(data.byteLength, undefined, label);
|
|
@@ -245,6 +245,23 @@ export function buildConservativeMultimodalGenerationOptions(options = {}) {
|
|
|
245
245
|
};
|
|
246
246
|
}
|
|
247
247
|
|
|
248
|
+
function resolveMultimodalMaxTokens(runtimeConfig, requestedMaxTokens) {
|
|
249
|
+
if (requestedMaxTokens !== undefined) {
|
|
250
|
+
return requirePositiveInteger(requestedMaxTokens, 'maxTokens');
|
|
251
|
+
}
|
|
252
|
+
return requirePositiveInteger(
|
|
253
|
+
runtimeConfig?.inference?.generation?.multimodalMaxTokens,
|
|
254
|
+
'runtime.inference.generation.multimodalMaxTokens'
|
|
255
|
+
);
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
function requirePositiveInteger(value, label) {
|
|
259
|
+
if (!Number.isInteger(value) || value <= 0) {
|
|
260
|
+
throw new Error(`${label} must be a positive integer.`);
|
|
261
|
+
}
|
|
262
|
+
return value;
|
|
263
|
+
}
|
|
264
|
+
|
|
248
265
|
// ============================================================================
|
|
249
266
|
// Main Inference Pipeline Class
|
|
250
267
|
// ============================================================================
|
|
@@ -1145,7 +1162,7 @@ export class InferencePipeline extends PipelineState {
|
|
|
1145
1162
|
|
|
1146
1163
|
// Step 3: Generate with embedding override at the image token offset.
|
|
1147
1164
|
const tokens = [];
|
|
1148
|
-
const maxGen = maxTokens
|
|
1165
|
+
const maxGen = resolveMultimodalMaxTokens(this.runtimeConfig, maxTokens);
|
|
1149
1166
|
const stopTokenIds = this.modelConfig.stopTokenIds;
|
|
1150
1167
|
|
|
1151
1168
|
try {
|
|
@@ -1274,7 +1291,7 @@ export class InferencePipeline extends PipelineState {
|
|
|
1274
1291
|
|
|
1275
1292
|
// Step 3: Generate with embedding override at the video token offset
|
|
1276
1293
|
const tokens = [];
|
|
1277
|
-
const maxGen = maxTokens
|
|
1294
|
+
const maxGen = resolveMultimodalMaxTokens(this.runtimeConfig, maxTokens);
|
|
1278
1295
|
const stopTokenIds = this.modelConfig.stopTokenIds;
|
|
1279
1296
|
|
|
1280
1297
|
try {
|
|
@@ -1404,7 +1421,7 @@ export class InferencePipeline extends PipelineState {
|
|
|
1404
1421
|
|
|
1405
1422
|
// Step 4: Generate with embedding override at the audio token offset
|
|
1406
1423
|
const tokens = [];
|
|
1407
|
-
const maxGen = maxTokens
|
|
1424
|
+
const maxGen = resolveMultimodalMaxTokens(this.runtimeConfig, maxTokens);
|
|
1408
1425
|
const stopTokenIds = this.modelConfig.stopTokenIds;
|
|
1409
1426
|
|
|
1410
1427
|
try {
|
|
@@ -1560,7 +1577,10 @@ export class InferencePipeline extends PipelineState {
|
|
|
1560
1577
|
|
|
1561
1578
|
async embed(prompt, options = {}) {
|
|
1562
1579
|
assertNotAborted(options?.signal);
|
|
1563
|
-
const result = await this.prefillWithEmbedding(prompt,
|
|
1580
|
+
const result = await this.prefillWithEmbedding(prompt, {
|
|
1581
|
+
...options,
|
|
1582
|
+
__skipStateSnapshot: true,
|
|
1583
|
+
});
|
|
1564
1584
|
assertNotAborted(options?.signal);
|
|
1565
1585
|
return {
|
|
1566
1586
|
embedding: result.embedding,
|
|
@@ -12,7 +12,10 @@ function isBosLikeLabel(value) {
|
|
|
12
12
|
|
|
13
13
|
function isEosLikeLabel(value) {
|
|
14
14
|
const normalized = normalizeSpecialTokenLabel(value);
|
|
15
|
-
return normalized === '<eos>'
|
|
15
|
+
return normalized === '<eos>'
|
|
16
|
+
|| normalized === '</s>'
|
|
17
|
+
|| normalized.includes('eos')
|
|
18
|
+
|| normalized.includes('endoftext');
|
|
16
19
|
}
|
|
17
20
|
|
|
18
21
|
function classifySpecialTokenReference(ref, specialTokens) {
|
|
@@ -84,7 +87,25 @@ export function inferBundledTokenizerBehaviorFlags(tokenizerJson, specialTokens
|
|
|
84
87
|
return { addBosToken: null, addEosToken: null };
|
|
85
88
|
}
|
|
86
89
|
|
|
87
|
-
const
|
|
90
|
+
const findTemplateProcessing = (postProcessor) => {
|
|
91
|
+
if (!postProcessor || typeof postProcessor !== 'object' || Array.isArray(postProcessor)) {
|
|
92
|
+
return null;
|
|
93
|
+
}
|
|
94
|
+
if (postProcessor.type === 'TemplateProcessing') {
|
|
95
|
+
return postProcessor;
|
|
96
|
+
}
|
|
97
|
+
if (postProcessor.type === 'Sequence' && Array.isArray(postProcessor.processors)) {
|
|
98
|
+
for (const child of postProcessor.processors) {
|
|
99
|
+
const template = findTemplateProcessing(child);
|
|
100
|
+
if (template) {
|
|
101
|
+
return template;
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
return null;
|
|
106
|
+
};
|
|
107
|
+
|
|
108
|
+
const postProcessor = findTemplateProcessing(tokenizerJson.post_processor);
|
|
88
109
|
if (!postProcessor || postProcessor.type !== 'TemplateProcessing') {
|
|
89
110
|
return { addBosToken: null, addEosToken: null };
|
|
90
111
|
}
|
|
@@ -167,6 +167,16 @@ async function loadFirstExistingTensor(ctx, prefixes, suffixes, options = {}) {
|
|
|
167
167
|
return null;
|
|
168
168
|
}
|
|
169
169
|
|
|
170
|
+
function wrapFinalNormTensor(ctx, tensor, name, location) {
|
|
171
|
+
if (!isGpuBufferInstance(tensor) || !location?.shape) {
|
|
172
|
+
return tensor;
|
|
173
|
+
}
|
|
174
|
+
const dtype = selectRuleValue('loader', 'weights', 'floatLocationDtype', {
|
|
175
|
+
locationDtype: location.dtype,
|
|
176
|
+
});
|
|
177
|
+
return createWeightBuffer(tensor, dtype, 'row', location.shape, name);
|
|
178
|
+
}
|
|
179
|
+
|
|
170
180
|
async function loadDiffusionGemmaSelfConditioning(ctx) {
|
|
171
181
|
const preNorm = await loadFirstExistingTensor(
|
|
172
182
|
ctx,
|
|
@@ -284,7 +294,8 @@ async function loadFinalNorm(ctx) {
|
|
|
284
294
|
for (const name of finalNormNames) {
|
|
285
295
|
const location = ctx.tensorLocations.get(name);
|
|
286
296
|
if (location) {
|
|
287
|
-
|
|
297
|
+
const tensor = await ctx.loadTensor(name, true, true);
|
|
298
|
+
finalNorm = wrapFinalNormTensor(ctx, tensor, name, location);
|
|
288
299
|
break;
|
|
289
300
|
}
|
|
290
301
|
}
|
|
@@ -153,8 +153,11 @@ function maxBatchDecodeTokensSemantic(context) {
|
|
|
153
153
|
return 8;
|
|
154
154
|
}
|
|
155
155
|
if (
|
|
156
|
-
|
|
157
|
-
|
|
156
|
+
(
|
|
157
|
+
context.modelId === 'gemma-4-12b-it-text-q4k-ehf16-af16'
|
|
158
|
+
|| context.modelId === 'gemma-4-12b-it-text-q4k-ehf16-hq4k-af16'
|
|
159
|
+
|| context.modelId === 'gemma-4-12b-it-text-w4a16-ct-ehf16-af16'
|
|
160
|
+
)
|
|
158
161
|
&& context.activationDtype === 'f16'
|
|
159
162
|
&& context.numLayers >= 48
|
|
160
163
|
&& context.hiddenSize >= 3840
|
|
@@ -441,7 +444,13 @@ export function buildInferenceExecutionRulesContractArtifact(ruleGroup) {
|
|
|
441
444
|
}
|
|
442
445
|
if (
|
|
443
446
|
!matchesExactObject(gemma412BAf16DenseRule?.match, {
|
|
444
|
-
modelId: {
|
|
447
|
+
modelId: {
|
|
448
|
+
in: [
|
|
449
|
+
'gemma-4-12b-it-text-q4k-ehf16-af16',
|
|
450
|
+
'gemma-4-12b-it-text-q4k-ehf16-hq4k-af16',
|
|
451
|
+
'gemma-4-12b-it-text-w4a16-ct-ehf16-af16',
|
|
452
|
+
],
|
|
453
|
+
},
|
|
445
454
|
activationDtype: 'f16',
|
|
446
455
|
numLayers: { gte: 48 },
|
|
447
456
|
hiddenSize: { gte: 3840 },
|
|
@@ -10,7 +10,13 @@
|
|
|
10
10
|
},
|
|
11
11
|
{
|
|
12
12
|
"match": {
|
|
13
|
-
"modelId": {
|
|
13
|
+
"modelId": {
|
|
14
|
+
"in": [
|
|
15
|
+
"gemma-4-e2b-it-q4k-ehf16-af32",
|
|
16
|
+
"gemma-4-e2b-it-q4k-ehf16-af32-int4ple",
|
|
17
|
+
"gemma-4-e2b-it-q4k-ehf16-af16-int4ple"
|
|
18
|
+
]
|
|
19
|
+
},
|
|
14
20
|
"platformVendor": "apple",
|
|
15
21
|
"retainQ4KMaterialization": true
|
|
16
22
|
},
|
|
@@ -111,7 +117,13 @@
|
|
|
111
117
|
},
|
|
112
118
|
{
|
|
113
119
|
"match": {
|
|
114
|
-
"modelId": {
|
|
120
|
+
"modelId": {
|
|
121
|
+
"in": [
|
|
122
|
+
"gemma-4-e2b-it-q4k-ehf16-af32",
|
|
123
|
+
"gemma-4-e2b-it-q4k-ehf16-af32-int4ple",
|
|
124
|
+
"gemma-4-e2b-it-q4k-ehf16-af16-int4ple"
|
|
125
|
+
]
|
|
126
|
+
},
|
|
115
127
|
"activationDtype": "f32",
|
|
116
128
|
"headDim": 256,
|
|
117
129
|
"kvDtype": "f16",
|
|
@@ -51,8 +51,32 @@
|
|
|
51
51
|
{ "match": {}, "value": [] }
|
|
52
52
|
],
|
|
53
53
|
"embeddingSemanticStyle": [
|
|
54
|
-
{ "match": { "
|
|
55
|
-
{
|
|
54
|
+
{ "match": { "manifestModelType": "embeddinggemma" }, "value": "embeddinggemma" },
|
|
55
|
+
{
|
|
56
|
+
"match": {
|
|
57
|
+
"modelType": "embedding",
|
|
58
|
+
"sourceCheckpointId": "Qwen/Qwen3-Embedding-0.6B"
|
|
59
|
+
},
|
|
60
|
+
"value": "qwen3_embedding"
|
|
61
|
+
},
|
|
56
62
|
{ "match": {}, "value": "default" }
|
|
63
|
+
],
|
|
64
|
+
"stableF32Logits": [
|
|
65
|
+
{
|
|
66
|
+
"match": {
|
|
67
|
+
"inputDtype": "f16",
|
|
68
|
+
"finalLogitSoftcapping": { "gt": 0 }
|
|
69
|
+
},
|
|
70
|
+
"value": true
|
|
71
|
+
},
|
|
72
|
+
{
|
|
73
|
+
"match": {
|
|
74
|
+
"inputDtype": "f16",
|
|
75
|
+
"rmsNormWeightOffset": true,
|
|
76
|
+
"hiddenSize": { "lte": 768 }
|
|
77
|
+
},
|
|
78
|
+
"value": true
|
|
79
|
+
},
|
|
80
|
+
{ "match": {}, "value": false }
|
|
57
81
|
]
|
|
58
82
|
}
|