doppler-gpu 0.4.5 → 0.4.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +45 -29
- package/assets/doppler-runtime-map.svg +82 -0
- package/assets/doppler.svg +1 -3
- package/package.json +25 -17
- package/src/cli/doppler-quickstart.js +1 -1
- package/src/cli/doppler-serve.js +2 -2
- package/src/client/doppler-registry.d.ts +19 -0
- package/src/client/doppler-registry.js +12 -0
- package/src/client/doppler-registry.json +50 -1
- package/src/client/runtime/model-session.d.ts +1 -0
- package/src/client/runtime/model-session.js +9 -0
- package/src/config/conversion/README.md +4 -0
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +28 -4
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +31 -5
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +30 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +30 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +30 -4
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +30 -5
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +30 -5
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +29 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +29 -4
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +29 -4
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/minicpm/minicpm4-0-5b-f16-af32.json +481 -0
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +29 -4
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +29 -4
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +30 -5
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +29 -4
- package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-q4k-ehf16-af32.json +395 -0
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-af32.json +429 -0
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +540 -0
- package/src/config/execution-contract-check.js +14 -10
- package/src/config/merge-contract-check.js +1 -0
- package/src/config/merge.js +30 -0
- package/src/config/param-validator.d.ts +5 -0
- package/src/config/param-validator.js +42 -8
- package/src/config/required-inference-fields-contract-check.js +14 -0
- package/src/config/schema/debug.schema.d.ts +2 -0
- package/src/config/schema/debug.schema.js +1 -0
- package/src/config/schema/doppler.schema.d.ts +5 -1
- package/src/config/schema/doppler.schema.js +82 -292
- package/src/config/schema/inference.schema.d.ts +17 -3
- package/src/config/schema/kvcache.schema.d.ts +19 -3
- package/src/config/schema/kvcache.schema.js +2 -1
- package/src/config/schema/manifest.schema.d.ts +38 -1
- package/src/config/schema/manifest.schema.js +18 -0
- package/src/config/schema/policy-schema-registry.json +5 -0
- package/src/config/source-packages/litert/gemma-4-12b-it.json +20 -1
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +20 -1
- package/src/config/source-packages/litert/gemma-4-e4b-it.json +21 -1
- package/src/config/transforms/execution-graph-transforms.d.ts +6 -0
- package/src/config/transforms/execution-graph-transforms.js +8 -0
- package/src/converter/conversion-plan.js +2 -0
- package/src/converter/parsers/transformer.js +27 -1
- package/src/converter/quantization-info.js +16 -4
- package/src/converter/rope-config.js +55 -3
- package/src/experimental/training/distillation/student-fixture.js +1 -1
- package/src/experimental/training/lora-pipeline.js +1 -1
- package/src/experimental/training/lora.js +2 -2
- package/src/experimental/training/suite.js +1 -1
- package/src/gpu/device.js +54 -11
- package/src/gpu/kernels/README.md +8 -1
- package/src/gpu/kernels/backward/attention_backward.js +8 -0
- package/src/gpu/kernels/dequant.d.ts +6 -6
- package/src/gpu/kernels/dequant.js +17 -6
- package/src/gpu/kernels/linear-attention-core.js +1 -6
- package/src/gpu/kernels/matmul-selection.d.ts +2 -2
- package/src/gpu/kernels/matmul-selection.js +9 -2
- package/src/gpu/kernels/matmul.d.ts +1 -1
- package/src/gpu/kernels/matmul.js +63 -9
- package/src/gpu/kernels/moe.d.ts +5 -1
- package/src/gpu/kernels/moe.js +5 -2
- package/src/gpu/kernels/softmax.d.ts +3 -3
- package/src/gpu/kernels/softmax.js +9 -2
- package/src/gpu/kernels/turboquant-codebook.js +1 -1
- package/src/gpu/kernels/types.d.ts +1 -1
- package/src/gpu/profiler.js +11 -1
- package/src/inference/browser-harness-model-helpers.js +4 -1
- package/src/inference/browser-harness-text-helpers.d.ts +28 -0
- package/src/inference/browser-harness-text-helpers.js +357 -116
- package/src/inference/browser-harness.d.ts +3 -3
- package/src/inference/browser-harness.js +266 -11
- package/src/inference/fixtures/rerank-semantic-fixtures.json +38 -0
- package/src/inference/kv-cache/quantized.js +16 -2
- package/src/inference/kv-cache/tiered.js +50 -15
- package/src/inference/moe-router.js +38 -7
- package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +1 -1
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +8 -6
- package/src/inference/pipelines/text/config.d.ts +5 -1
- package/src/inference/pipelines/text/config.js +135 -112
- package/src/inference/pipelines/text/embed.d.ts +1 -0
- package/src/inference/pipelines/text/embed.js +39 -12
- package/src/inference/pipelines/text/execution-plan.js +8 -2
- package/src/inference/pipelines/text/ffn/dense.js +15 -4
- package/src/inference/pipelines/text/ffn/moe.js +4 -3
- package/src/inference/pipelines/text/ffn/standard.d.ts +2 -1
- package/src/inference/pipelines/text/ffn/standard.js +25 -2
- package/src/inference/pipelines/text/generator-helpers.js +2 -1
- package/src/inference/pipelines/text/generator-runtime.d.ts +6 -2
- package/src/inference/pipelines/text/generator-runtime.js +53 -20
- package/src/inference/pipelines/text/generator-steps.js +3 -0
- package/src/inference/pipelines/text/generator.d.ts +1 -0
- package/src/inference/pipelines/text/generator.js +14 -1
- package/src/inference/pipelines/text/init-rope.js +54 -6
- package/src/inference/pipelines/text/init.js +63 -15
- package/src/inference/pipelines/text/layer.js +32 -5
- package/src/inference/pipelines/text/linear-attention.js +12 -8
- package/src/inference/pipelines/text/logits/gpu.d.ts +1 -1
- package/src/inference/pipelines/text/logits/gpu.js +76 -113
- package/src/inference/pipelines/text/logits/index.js +44 -87
- package/src/inference/pipelines/text/logits/precision-policy.d.ts +11 -0
- package/src/inference/pipelines/text/logits/precision-policy.js +103 -0
- package/src/inference/pipelines/text/logits/types.d.ts +1 -0
- package/src/inference/pipelines/text/logits/utils.d.ts +2 -0
- package/src/inference/pipelines/text/logits/utils.js +11 -0
- package/src/inference/pipelines/text/lora-apply.js +6 -4
- package/src/inference/pipelines/text/moe-cpu.js +25 -0
- package/src/inference/pipelines/text/moe-gpu.js +148 -97
- package/src/inference/pipelines/text/moe-shape-validator.d.ts +33 -3
- package/src/inference/pipelines/text/moe-shape-validator.js +149 -31
- package/src/inference/pipelines/text/ops.d.ts +3 -0
- package/src/inference/pipelines/text/ops.js +26 -1
- package/src/inference/pipelines/text/per-layer-inputs.js +27 -12
- package/src/inference/pipelines/text/sampling.js +52 -10
- package/src/inference/pipelines/text/weights.js +7 -2
- package/src/inference/pipelines/text.js +24 -4
- package/src/inference/tokenizers/behavior-flags.js +23 -2
- package/src/loader/final-weights-loader.js +12 -1
- package/src/loader/tensors/tensor-loader.js +37 -3
- package/src/rules/execution-rules-contract-check.js +12 -3
- package/src/rules/inference/capability-transforms.rules.json +14 -2
- package/src/rules/inference/config.rules.json +26 -2
- package/src/rules/inference/dtype.rules.json +12 -5
- package/src/rules/inference/execution.rules.json +5 -19
- package/src/rules/inference/moe.rules.json +97 -0
- package/src/rules/loader/weights.rules.json +1 -0
- package/src/tooling/command-api-constants.js +2 -2
- package/src/tooling/command-api-family-normalizers.js +2 -1
- package/src/tooling/command-api.d.ts +1 -1
- package/src/tooling/command-runner-shared.d.ts +1 -1
- package/src/tooling/command-runner-shared.js +3 -1
- package/src/tooling/node-converter.js +20 -0
- package/src/tooling/node-source-runtime.js +5 -1
- package/src/version.js +1 -1
|
@@ -1019,14 +1019,48 @@ export async function loadFloat(shardData, location, name, config) {
|
|
|
1019
1019
|
writeBufferAligned(device, buffer, shardData);
|
|
1020
1020
|
}
|
|
1021
1021
|
|
|
1022
|
-
const dtype = selectRuleValue('loader', 'weights', 'floatLocationDtype', {
|
|
1023
|
-
locationDtype: location.dtype,
|
|
1024
|
-
});
|
|
1025
1022
|
const layout = selectRuleValue('loader', 'weights', 'weightLayout', {
|
|
1026
1023
|
layout: location.layout ?? null,
|
|
1027
1024
|
useColumnWise: false,
|
|
1028
1025
|
});
|
|
1029
1026
|
const isMatmulWeight = shouldDequantizeToF16(location);
|
|
1027
|
+
const caps = config.gpuCapabilities || getKernelCapabilities();
|
|
1028
|
+
const dtype = isMatmulWeight
|
|
1029
|
+
? selectRuleValue('loader', 'weights', 'matmulWeightDtype', {
|
|
1030
|
+
locationDtype: location.dtype,
|
|
1031
|
+
hasF16: Boolean(caps?.hasF16),
|
|
1032
|
+
isMatmulWeight,
|
|
1033
|
+
keepF32Weights: Boolean(config.keepF32Weights),
|
|
1034
|
+
})
|
|
1035
|
+
: selectRuleValue('loader', 'weights', 'floatLocationDtype', {
|
|
1036
|
+
locationDtype: location.dtype,
|
|
1037
|
+
});
|
|
1038
|
+
|
|
1039
|
+
if (location.dtype === 'F16' && dtype === 'f32') {
|
|
1040
|
+
if (isGpuBufferInstance(shardData)) {
|
|
1041
|
+
throw new Error(
|
|
1042
|
+
`Tensor "${name}" requires CPU F16->F32 materialization on this runtime, ` +
|
|
1043
|
+
'but the source was already uploaded to a GPU buffer.'
|
|
1044
|
+
);
|
|
1045
|
+
}
|
|
1046
|
+
const f32Values = convertF16ToF32CPU(toUint16View(shardData, `F16 tensor load for ${name}`));
|
|
1047
|
+
const f32Buffer = acquireAlignedBuffer(f32Values.byteLength, `${name}_f32`);
|
|
1048
|
+
resultBuffer = f32Buffer;
|
|
1049
|
+
writeBufferAligned(device, f32Buffer, new Uint8Array(f32Values.buffer, f32Values.byteOffset, f32Values.byteLength));
|
|
1050
|
+
releaseOwnedGpuBuffer(buffer, ownsBuffer);
|
|
1051
|
+
ownsBuffer = false;
|
|
1052
|
+
resultBuffer = null;
|
|
1053
|
+
if (isMatmulWeight) {
|
|
1054
|
+
return {
|
|
1055
|
+
data: createWeightBuffer(f32Buffer, 'f32', layout, location.shape, name),
|
|
1056
|
+
allocatedBuffers: [f32Buffer],
|
|
1057
|
+
};
|
|
1058
|
+
}
|
|
1059
|
+
return {
|
|
1060
|
+
data: applyBufferLayout(f32Buffer, location, 'f32'),
|
|
1061
|
+
allocatedBuffers: [f32Buffer],
|
|
1062
|
+
};
|
|
1063
|
+
}
|
|
1030
1064
|
|
|
1031
1065
|
if (isMatmulWeight) {
|
|
1032
1066
|
ownsBuffer = false;
|
|
@@ -153,8 +153,11 @@ function maxBatchDecodeTokensSemantic(context) {
|
|
|
153
153
|
return 8;
|
|
154
154
|
}
|
|
155
155
|
if (
|
|
156
|
-
|
|
157
|
-
|
|
156
|
+
(
|
|
157
|
+
context.modelId === 'gemma-4-12b-it-text-q4k-ehf16-af16'
|
|
158
|
+
|| context.modelId === 'gemma-4-12b-it-text-q4k-ehf16-hq4k-af16'
|
|
159
|
+
|| context.modelId === 'gemma-4-12b-it-text-w4a16-ct-ehf16-af16'
|
|
160
|
+
)
|
|
158
161
|
&& context.activationDtype === 'f16'
|
|
159
162
|
&& context.numLayers >= 48
|
|
160
163
|
&& context.hiddenSize >= 3840
|
|
@@ -441,7 +444,13 @@ export function buildInferenceExecutionRulesContractArtifact(ruleGroup) {
|
|
|
441
444
|
}
|
|
442
445
|
if (
|
|
443
446
|
!matchesExactObject(gemma412BAf16DenseRule?.match, {
|
|
444
|
-
modelId: {
|
|
447
|
+
modelId: {
|
|
448
|
+
in: [
|
|
449
|
+
'gemma-4-12b-it-text-q4k-ehf16-af16',
|
|
450
|
+
'gemma-4-12b-it-text-q4k-ehf16-hq4k-af16',
|
|
451
|
+
'gemma-4-12b-it-text-w4a16-ct-ehf16-af16',
|
|
452
|
+
],
|
|
453
|
+
},
|
|
445
454
|
activationDtype: 'f16',
|
|
446
455
|
numLayers: { gte: 48 },
|
|
447
456
|
hiddenSize: { gte: 3840 },
|
|
@@ -10,7 +10,13 @@
|
|
|
10
10
|
},
|
|
11
11
|
{
|
|
12
12
|
"match": {
|
|
13
|
-
"modelId": {
|
|
13
|
+
"modelId": {
|
|
14
|
+
"in": [
|
|
15
|
+
"gemma-4-e2b-it-q4k-ehf16-af32",
|
|
16
|
+
"gemma-4-e2b-it-q4k-ehf16-af32-int4ple",
|
|
17
|
+
"gemma-4-e2b-it-q4k-ehf16-af16-int4ple"
|
|
18
|
+
]
|
|
19
|
+
},
|
|
14
20
|
"platformVendor": "apple",
|
|
15
21
|
"retainQ4KMaterialization": true
|
|
16
22
|
},
|
|
@@ -111,7 +117,13 @@
|
|
|
111
117
|
},
|
|
112
118
|
{
|
|
113
119
|
"match": {
|
|
114
|
-
"modelId": {
|
|
120
|
+
"modelId": {
|
|
121
|
+
"in": [
|
|
122
|
+
"gemma-4-e2b-it-q4k-ehf16-af32",
|
|
123
|
+
"gemma-4-e2b-it-q4k-ehf16-af32-int4ple",
|
|
124
|
+
"gemma-4-e2b-it-q4k-ehf16-af16-int4ple"
|
|
125
|
+
]
|
|
126
|
+
},
|
|
115
127
|
"activationDtype": "f32",
|
|
116
128
|
"headDim": 256,
|
|
117
129
|
"kvDtype": "f16",
|
|
@@ -51,8 +51,32 @@
|
|
|
51
51
|
{ "match": {}, "value": [] }
|
|
52
52
|
],
|
|
53
53
|
"embeddingSemanticStyle": [
|
|
54
|
-
{ "match": { "
|
|
55
|
-
{
|
|
54
|
+
{ "match": { "manifestModelType": "embeddinggemma" }, "value": "embeddinggemma" },
|
|
55
|
+
{
|
|
56
|
+
"match": {
|
|
57
|
+
"modelType": "embedding",
|
|
58
|
+
"sourceCheckpointId": "Qwen/Qwen3-Embedding-0.6B"
|
|
59
|
+
},
|
|
60
|
+
"value": "qwen3_embedding"
|
|
61
|
+
},
|
|
56
62
|
{ "match": {}, "value": "default" }
|
|
63
|
+
],
|
|
64
|
+
"stableF32Logits": [
|
|
65
|
+
{
|
|
66
|
+
"match": {
|
|
67
|
+
"inputDtype": "f16",
|
|
68
|
+
"finalLogitSoftcapping": { "gt": 0 }
|
|
69
|
+
},
|
|
70
|
+
"value": true
|
|
71
|
+
},
|
|
72
|
+
{
|
|
73
|
+
"match": {
|
|
74
|
+
"inputDtype": "f16",
|
|
75
|
+
"rmsNormWeightOffset": true,
|
|
76
|
+
"hiddenSize": { "lte": 768 }
|
|
77
|
+
},
|
|
78
|
+
"value": true
|
|
79
|
+
},
|
|
80
|
+
{ "match": {}, "value": false }
|
|
57
81
|
]
|
|
58
82
|
}
|
|
@@ -15,7 +15,8 @@
|
|
|
15
15
|
],
|
|
16
16
|
"f16OrF32FromDtype": [
|
|
17
17
|
{ "match": { "dtype": "f16" }, "value": "f16" },
|
|
18
|
-
{ "match": {}, "value": "f32" }
|
|
18
|
+
{ "match": { "dtype": "f32" }, "value": "f32" },
|
|
19
|
+
{ "match": {}, "value": { "context": "fallback" } }
|
|
19
20
|
],
|
|
20
21
|
"embeddingDtype": [
|
|
21
22
|
{ "match": { "dtype": "litert_int4" }, "value": "litert_int4" },
|
|
@@ -71,22 +72,28 @@
|
|
|
71
72
|
],
|
|
72
73
|
"bytesPerElement": [
|
|
73
74
|
{ "match": { "dtype": "f16" }, "value": 2 },
|
|
74
|
-
{ "match": {}, "value":
|
|
75
|
+
{ "match": { "dtype": "bf16" }, "value": 2 },
|
|
76
|
+
{ "match": { "dtype": "f32" }, "value": 4 },
|
|
77
|
+
{ "match": {}, "value": { "context": "fallback" } }
|
|
75
78
|
],
|
|
76
79
|
"dtypeFromSize": [
|
|
77
80
|
{ "match": { "bytesPerElement": 2 }, "value": "f16" },
|
|
78
|
-
{ "match": {}, "value": "f32" }
|
|
81
|
+
{ "match": { "bytesPerElement": 4 }, "value": "f32" },
|
|
82
|
+
{ "match": {}, "value": { "context": "fallback" } }
|
|
79
83
|
],
|
|
80
84
|
"bytesFromDtype": [
|
|
81
85
|
{ "match": { "dtype": "f16" }, "value": 2 },
|
|
82
|
-
{ "match": {}, "value":
|
|
86
|
+
{ "match": { "dtype": "bf16" }, "value": 2 },
|
|
87
|
+
{ "match": { "dtype": "f32" }, "value": 4 },
|
|
88
|
+
{ "match": {}, "value": { "context": "fallback" } }
|
|
83
89
|
],
|
|
84
90
|
"matmulDtype": [
|
|
85
91
|
{ "match": { "dtype": { "in": ["f16", "bf16"] } }, "value": "f16" },
|
|
92
|
+
{ "match": { "dtype": "f32" }, "value": "f32" },
|
|
86
93
|
{ "match": { "dtype": "q4k" }, "value": "q4k" },
|
|
87
94
|
{ "match": { "dtype": "litert_int4" }, "value": "litert_int4" },
|
|
88
95
|
{ "match": { "dtype": "w4a16" }, "value": "w4a16" },
|
|
89
|
-
{ "match": {}, "value": "
|
|
96
|
+
{ "match": {}, "value": { "context": "fallback" } }
|
|
90
97
|
],
|
|
91
98
|
"logitsDtype": [
|
|
92
99
|
{ "match": { "logitsDtype": "f16" }, "value": "f16" },
|
|
@@ -133,7 +133,11 @@
|
|
|
133
133
|
{
|
|
134
134
|
"match": {
|
|
135
135
|
"modelId": {
|
|
136
|
-
"
|
|
136
|
+
"in": [
|
|
137
|
+
"gemma-4-12b-it-text-q4k-ehf16-af16",
|
|
138
|
+
"gemma-4-12b-it-text-q4k-ehf16-hq4k-af16",
|
|
139
|
+
"gemma-4-12b-it-text-w4a16-ct-ehf16-af16"
|
|
140
|
+
]
|
|
137
141
|
},
|
|
138
142
|
"activationDtype": "f16",
|
|
139
143
|
"numLayers": {
|
|
@@ -187,23 +191,5 @@
|
|
|
187
191
|
"match": {},
|
|
188
192
|
"value": "f32"
|
|
189
193
|
}
|
|
190
|
-
],
|
|
191
|
-
"modelBatchingDefaults": [
|
|
192
|
-
{
|
|
193
|
-
"match": {
|
|
194
|
-
"modelId": {
|
|
195
|
-
"contains": "lfm2"
|
|
196
|
-
}
|
|
197
|
-
},
|
|
198
|
-
"value": {
|
|
199
|
-
"batchSize": 8,
|
|
200
|
-
"stopCheckMode": "batch",
|
|
201
|
-
"readbackInterval": 8
|
|
202
|
-
}
|
|
203
|
-
},
|
|
204
|
-
{
|
|
205
|
-
"match": {},
|
|
206
|
-
"value": null
|
|
207
|
-
}
|
|
208
194
|
]
|
|
209
195
|
}
|
|
@@ -1,4 +1,101 @@
|
|
|
1
1
|
{
|
|
2
|
+
"executionProfile": [
|
|
3
|
+
{
|
|
4
|
+
"match": { "expertFormat": "gpt-oss" },
|
|
5
|
+
"value": {
|
|
6
|
+
"id": "gpt-oss-mxfp4",
|
|
7
|
+
"label": "GPT-OSS MXFP4 experts",
|
|
8
|
+
"expertExecutor": "gpt-oss",
|
|
9
|
+
"intermediateSizeSource": "architecture",
|
|
10
|
+
"requiresShaderF16": true,
|
|
11
|
+
"routerScaleMode": "none",
|
|
12
|
+
"topkRouteExecutor": null,
|
|
13
|
+
"vendorProfile": {
|
|
14
|
+
"kind": "rule",
|
|
15
|
+
"domain": "kernels",
|
|
16
|
+
"group": "moeGptoss",
|
|
17
|
+
"name": "vendorQuirkProfile"
|
|
18
|
+
},
|
|
19
|
+
"kernelPathProfileResolver": "gpt-oss",
|
|
20
|
+
"kernelRuleModelType": "gpt-oss",
|
|
21
|
+
"shapePolicy": {
|
|
22
|
+
"kind": "rule",
|
|
23
|
+
"domain": "kernels",
|
|
24
|
+
"group": "moeGptoss",
|
|
25
|
+
"name": "shapePolicy"
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
},
|
|
29
|
+
{
|
|
30
|
+
"match": { "expertFormat": "mixtral" },
|
|
31
|
+
"value": {
|
|
32
|
+
"id": "mixtral-dense",
|
|
33
|
+
"label": "Mixtral dense experts",
|
|
34
|
+
"expertExecutor": "mixtral",
|
|
35
|
+
"intermediateSizeSource": "architecture",
|
|
36
|
+
"requiresShaderF16": false,
|
|
37
|
+
"routerScaleMode": "none",
|
|
38
|
+
"topkRouteExecutor": null,
|
|
39
|
+
"vendorProfile": {
|
|
40
|
+
"kind": "rule",
|
|
41
|
+
"domain": "kernels",
|
|
42
|
+
"group": "moeMixtral",
|
|
43
|
+
"name": "vendorQuirkProfile"
|
|
44
|
+
},
|
|
45
|
+
"kernelPathProfileResolver": "mixtral",
|
|
46
|
+
"kernelRuleModelType": "mixtral",
|
|
47
|
+
"shapePolicy": null
|
|
48
|
+
}
|
|
49
|
+
},
|
|
50
|
+
{
|
|
51
|
+
"match": { "modelType": "diffusion_gemma", "expertFormat": "gemma4" },
|
|
52
|
+
"value": {
|
|
53
|
+
"id": "diffusion-gemma-packed",
|
|
54
|
+
"label": "DiffusionGemma packed experts",
|
|
55
|
+
"expertExecutor": "gemma4-packed",
|
|
56
|
+
"intermediateSizeSource": "expert",
|
|
57
|
+
"requiresShaderF16": false,
|
|
58
|
+
"routerScaleMode": "required",
|
|
59
|
+
"topkRouteExecutor": "gemma4-route",
|
|
60
|
+
"vendorProfile": {
|
|
61
|
+
"kind": "static",
|
|
62
|
+
"value": {
|
|
63
|
+
"preferVec4Dequant": false,
|
|
64
|
+
"dequantTileShape": null,
|
|
65
|
+
"routerWorkgroupSize": 256,
|
|
66
|
+
"maxTokensPerExpertScale": 1.0
|
|
67
|
+
}
|
|
68
|
+
},
|
|
69
|
+
"kernelPathProfileResolver": null,
|
|
70
|
+
"kernelRuleModelType": null,
|
|
71
|
+
"shapePolicy": null
|
|
72
|
+
}
|
|
73
|
+
},
|
|
74
|
+
{
|
|
75
|
+
"match": { "expertFormat": "gemma4" },
|
|
76
|
+
"value": {
|
|
77
|
+
"id": "gemma4-packed",
|
|
78
|
+
"label": "Gemma packed experts",
|
|
79
|
+
"expertExecutor": "gemma4-packed",
|
|
80
|
+
"intermediateSizeSource": "expert",
|
|
81
|
+
"requiresShaderF16": false,
|
|
82
|
+
"routerScaleMode": "optional",
|
|
83
|
+
"topkRouteExecutor": "gemma4-route",
|
|
84
|
+
"vendorProfile": {
|
|
85
|
+
"kind": "static",
|
|
86
|
+
"value": {
|
|
87
|
+
"preferVec4Dequant": false,
|
|
88
|
+
"dequantTileShape": null,
|
|
89
|
+
"routerWorkgroupSize": 256,
|
|
90
|
+
"maxTokensPerExpertScale": 1.0
|
|
91
|
+
}
|
|
92
|
+
},
|
|
93
|
+
"kernelPathProfileResolver": null,
|
|
94
|
+
"kernelRuleModelType": null,
|
|
95
|
+
"shapePolicy": null
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
],
|
|
2
99
|
"biasAddCode": [
|
|
3
100
|
{
|
|
4
101
|
"match": { "logitsDtype": "f16", "biasDtype": "f16" },
|
|
@@ -10,6 +10,7 @@
|
|
|
10
10
|
{ "match": {}, "value": "f32" }
|
|
11
11
|
],
|
|
12
12
|
"matmulWeightDtype": [
|
|
13
|
+
{ "match": { "locationDtype": "F16", "hasF16": false, "isMatmulWeight": true }, "value": "f32" },
|
|
13
14
|
{ "match": { "locationDtype": "F16" }, "value": "f16" },
|
|
14
15
|
{ "match": { "locationDtype": "BF16", "hasF16": true, "isMatmulWeight": true }, "value": "f16" },
|
|
15
16
|
{ "match": { "locationDtype": "BF16" }, "value": "f32" },
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
export const TOOLING_COMMAND_SET = ['convert', 'refresh-integrity', 'debug', 'bench', 'verify', 'lora', 'distill', 'diagnose'];
|
|
2
2
|
export const TOOLING_SURFACE_SET = ['browser', 'node'];
|
|
3
|
-
export const TOOLING_WORKLOAD_SET = ['kernels', 'inference', 'embedding', 'training', 'diffusion', 'energy'];
|
|
3
|
+
export const TOOLING_WORKLOAD_SET = ['kernels', 'inference', 'embedding', 'rerank', 'training', 'diffusion', 'energy'];
|
|
4
4
|
export const TOOLING_INTENT_SET = ['verify', 'investigate', 'calibrate'];
|
|
5
|
-
export const VERIFY_WORKLOADS = ['kernels', 'inference', 'embedding', 'training', 'diffusion', 'energy'];
|
|
5
|
+
export const VERIFY_WORKLOADS = ['kernels', 'inference', 'embedding', 'rerank', 'training', 'diffusion', 'energy'];
|
|
6
6
|
export const TRAINING_STAGE_SET = ['stage1_joint', 'stage2_base', 'stage_a', 'stage_b'];
|
|
7
7
|
export const DISTILL_ACTION_SET = ['run', 'stage-a', 'stage-b', 'eval', 'watch', 'compare', 'quality-gate', 'subsets'];
|
|
8
8
|
export const LORA_ACTION_SET = ['run', 'eval', 'watch', 'export', 'compare', 'quality-gate', 'activate'];
|
|
@@ -182,11 +182,12 @@ function resolveBenchRequestWorkload(raw) {
|
|
|
182
182
|
if (
|
|
183
183
|
workload !== 'inference'
|
|
184
184
|
&& workload !== 'embedding'
|
|
185
|
+
&& workload !== 'rerank'
|
|
185
186
|
&& workload !== 'training'
|
|
186
187
|
&& workload !== 'diffusion'
|
|
187
188
|
) {
|
|
188
189
|
throw new Error(
|
|
189
|
-
'tooling command: "bench" workload must be "inference", "embedding", "training", or "diffusion".'
|
|
190
|
+
'tooling command: "bench" workload must be "inference", "embedding", "rerank", "training", or "diffusion".'
|
|
190
191
|
);
|
|
191
192
|
}
|
|
192
193
|
return workload;
|
|
@@ -2,7 +2,7 @@ import type { ConverterConfigSchema } from '../config/schema/converter.schema.js
|
|
|
2
2
|
|
|
3
3
|
export type ToolingCommand = 'convert' | 'refresh-integrity' | 'debug' | 'bench' | 'verify' | 'lora' | 'distill' | 'diagnose';
|
|
4
4
|
export type ToolingSurface = 'browser' | 'node';
|
|
5
|
-
export type ToolingWorkload = 'kernels' | 'inference' | 'embedding' | 'training' | 'diffusion' | 'energy';
|
|
5
|
+
export type ToolingWorkload = 'kernels' | 'inference' | 'embedding' | 'rerank' | 'training' | 'diffusion' | 'energy';
|
|
6
6
|
export type ToolingIntent = 'verify' | 'investigate' | 'calibrate' | null;
|
|
7
7
|
export type ToolingTrainingStage = 'stage1_joint' | 'stage2_base' | 'stage_a' | 'stage_b';
|
|
8
8
|
export type ToolingDistillAction = 'run' | 'stage-a' | 'stage-b' | 'eval' | 'watch' | 'compare' | 'quality-gate' | 'subsets';
|
|
@@ -46,7 +46,7 @@ export declare function buildSuiteOptions(
|
|
|
46
46
|
workload: ToolingCommandRequest['workload'];
|
|
47
47
|
command: ToolingCommandRequest['command'];
|
|
48
48
|
surface: string | null;
|
|
49
|
-
expectedModelType?: 'embedding';
|
|
49
|
+
expectedModelType?: 'embedding' | 'rerank';
|
|
50
50
|
modelId?: string;
|
|
51
51
|
trainingTests?: string[];
|
|
52
52
|
trainingStage?: 'stage1_joint' | 'stage2_base' | 'stage_a' | 'stage_b';
|
|
@@ -109,7 +109,9 @@ function resolveExecutionMode(request) {
|
|
|
109
109
|
}
|
|
110
110
|
|
|
111
111
|
function resolveExpectedModelType(request) {
|
|
112
|
-
|
|
112
|
+
if (request.workload === 'embedding') return 'embedding';
|
|
113
|
+
if (request.workload === 'rerank') return 'rerank';
|
|
114
|
+
return undefined;
|
|
113
115
|
}
|
|
114
116
|
|
|
115
117
|
export async function applyRuntimeInputs(request, runtimeBridge, options = {}) {
|
|
@@ -1109,6 +1109,7 @@ export async function convertSafetensorsDirectory(options) {
|
|
|
1109
1109
|
let tokenizerJson = null;
|
|
1110
1110
|
let tokenizerConfig = null;
|
|
1111
1111
|
let generationConfig = null;
|
|
1112
|
+
let rerankScoring = null;
|
|
1112
1113
|
let hasTokenizerModel = false;
|
|
1113
1114
|
let tokenizerModelPath = null;
|
|
1114
1115
|
let diffusionAuxFiles = [];
|
|
@@ -1339,6 +1340,7 @@ export async function convertSafetensorsDirectory(options) {
|
|
|
1339
1340
|
tensors = parsedTransformer.tensors;
|
|
1340
1341
|
architectureHint = parsedTransformer.architectureHint;
|
|
1341
1342
|
embeddingPostprocessor = parsedTransformer.embeddingPostprocessor ?? null;
|
|
1343
|
+
rerankScoring = parsedTransformer.rerankScoring ?? null;
|
|
1342
1344
|
architecture = extractArchitecture(config, null);
|
|
1343
1345
|
const tokenizerJsonPath = path.join(inputDir, 'tokenizer.json');
|
|
1344
1346
|
tokenizerModelPath = path.join(inputDir, 'tokenizer.model');
|
|
@@ -1366,6 +1368,24 @@ export async function convertSafetensorsDirectory(options) {
|
|
|
1366
1368
|
const targetQuantization = plan.manifestQuantization;
|
|
1367
1369
|
const quantizationInfo = plan.quantizationInfo;
|
|
1368
1370
|
const inference = plan.manifestInference;
|
|
1371
|
+
if (rerankScoring) {
|
|
1372
|
+
const rerankConfig = inference?.rerank;
|
|
1373
|
+
if (!rerankConfig || typeof rerankConfig !== 'object' || Array.isArray(rerankConfig)) {
|
|
1374
|
+
throw new Error(
|
|
1375
|
+
'node convert: sentence-transformers LogitScore module requires explicit inference.rerank config.'
|
|
1376
|
+
);
|
|
1377
|
+
}
|
|
1378
|
+
if (Number(rerankConfig.trueTokenId) !== rerankScoring.trueTokenId) {
|
|
1379
|
+
throw new Error(
|
|
1380
|
+
`node convert: inference.rerank.trueTokenId=${rerankConfig.trueTokenId} does not match LogitScore true_token_id=${rerankScoring.trueTokenId}.`
|
|
1381
|
+
);
|
|
1382
|
+
}
|
|
1383
|
+
if (Number(rerankConfig.falseTokenId) !== rerankScoring.falseTokenId) {
|
|
1384
|
+
throw new Error(
|
|
1385
|
+
`node convert: inference.rerank.falseTokenId=${rerankConfig.falseTokenId} does not match LogitScore false_token_id=${rerankScoring.falseTokenId}.`
|
|
1386
|
+
);
|
|
1387
|
+
}
|
|
1388
|
+
}
|
|
1369
1389
|
const explicitModelId = resolveConfiguredModelId(options?.modelId, converterConfig);
|
|
1370
1390
|
if (!explicitModelId) {
|
|
1371
1391
|
throw new Error(
|
|
@@ -304,7 +304,11 @@ function applyPackageTokenizerAssets(parsedArtifact, tokenizerAssets) {
|
|
|
304
304
|
const next = {
|
|
305
305
|
...parsedArtifact,
|
|
306
306
|
};
|
|
307
|
-
|
|
307
|
+
const hasPackageTokenizer = next.tokenizerJson != null
|
|
308
|
+
|| next.tokenizerJsonPath != null
|
|
309
|
+
|| next.tokenizerModelName != null
|
|
310
|
+
|| next.tokenizerModelPath != null;
|
|
311
|
+
if (tokenizerAssets.tokenizerJson && !hasPackageTokenizer) {
|
|
308
312
|
next.tokenizerJson = tokenizerAssets.tokenizerJson;
|
|
309
313
|
next.tokenizerJsonPath = 'tokenizer.json';
|
|
310
314
|
}
|
package/src/version.js
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
export const DOPPLER_VERSION = '0.4.
|
|
1
|
+
export const DOPPLER_VERSION = '0.4.7';
|
|
2
2
|
export const DOPPLER_PROVIDER_VERSION = DOPPLER_VERSION;
|