doppler-gpu 0.4.6 → 0.4.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +43 -31
- package/assets/doppler-runtime-map.svg +1 -3
- package/assets/doppler.svg +1 -3
- package/package.json +25 -17
- package/src/cli/doppler-quickstart.js +1 -1
- package/src/cli/doppler-serve.js +2 -2
- package/src/client/doppler-registry.d.ts +19 -0
- package/src/client/doppler-registry.js +12 -0
- package/src/client/doppler-registry.json +50 -1
- package/src/config/conversion/README.md +4 -0
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +28 -4
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +31 -5
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +30 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +30 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +30 -4
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +30 -5
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +30 -5
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +29 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +29 -4
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +29 -4
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/minicpm/minicpm4-0-5b-f16-af32.json +481 -0
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +29 -4
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +29 -4
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +30 -5
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +29 -4
- package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-q4k-ehf16-af32.json +395 -0
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-af32.json +429 -0
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +540 -0
- package/src/config/execution-contract-check.js +14 -10
- package/src/config/merge-contract-check.js +1 -0
- package/src/config/merge.js +30 -0
- package/src/config/param-validator.d.ts +5 -0
- package/src/config/param-validator.js +42 -8
- package/src/config/required-inference-fields-contract-check.js +14 -0
- package/src/config/schema/debug.schema.d.ts +2 -0
- package/src/config/schema/debug.schema.js +1 -0
- package/src/config/schema/doppler.schema.d.ts +5 -1
- package/src/config/schema/doppler.schema.js +82 -292
- package/src/config/schema/inference.schema.d.ts +17 -3
- package/src/config/schema/kvcache.schema.d.ts +19 -3
- package/src/config/schema/kvcache.schema.js +2 -1
- package/src/config/schema/manifest.schema.d.ts +38 -1
- package/src/config/schema/manifest.schema.js +18 -0
- package/src/config/schema/policy-schema-registry.json +5 -0
- package/src/config/source-packages/litert/gemma-4-12b-it.json +20 -1
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +20 -1
- package/src/config/source-packages/litert/gemma-4-e4b-it.json +21 -1
- package/src/config/transforms/execution-graph-transforms.d.ts +6 -0
- package/src/config/transforms/execution-graph-transforms.js +8 -0
- package/src/converter/conversion-plan.js +2 -0
- package/src/converter/parsers/transformer.js +27 -1
- package/src/converter/quantization-info.js +16 -4
- package/src/converter/rope-config.js +55 -3
- package/src/experimental/training/distillation/student-fixture.js +1 -1
- package/src/experimental/training/lora-pipeline.js +1 -1
- package/src/experimental/training/lora.js +2 -2
- package/src/experimental/training/suite.js +1 -1
- package/src/gpu/kernels/README.md +8 -1
- package/src/gpu/kernels/backward/attention_backward.js +8 -0
- package/src/gpu/kernels/dequant.d.ts +6 -6
- package/src/gpu/kernels/dequant.js +17 -6
- package/src/gpu/kernels/linear-attention-core.js +1 -6
- package/src/gpu/kernels/matmul-selection.d.ts +2 -2
- package/src/gpu/kernels/matmul-selection.js +4 -1
- package/src/gpu/kernels/matmul.d.ts +1 -1
- package/src/gpu/kernels/matmul.js +24 -6
- package/src/gpu/kernels/moe.d.ts +5 -1
- package/src/gpu/kernels/moe.js +5 -2
- package/src/gpu/kernels/softmax.d.ts +3 -3
- package/src/gpu/kernels/softmax.js +9 -2
- package/src/gpu/kernels/turboquant-codebook.js +1 -1
- package/src/gpu/kernels/types.d.ts +1 -1
- package/src/gpu/profiler.js +11 -1
- package/src/inference/browser-harness-model-helpers.js +4 -1
- package/src/inference/browser-harness-text-helpers.d.ts +28 -0
- package/src/inference/browser-harness-text-helpers.js +357 -116
- package/src/inference/browser-harness.d.ts +3 -3
- package/src/inference/browser-harness.js +266 -11
- package/src/inference/fixtures/rerank-semantic-fixtures.json +38 -0
- package/src/inference/kv-cache/quantized.js +16 -2
- package/src/inference/kv-cache/tiered.js +50 -15
- package/src/inference/moe-router.js +38 -7
- package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +1 -1
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +8 -6
- package/src/inference/pipelines/text/config.d.ts +5 -1
- package/src/inference/pipelines/text/config.js +135 -112
- package/src/inference/pipelines/text/embed.d.ts +1 -0
- package/src/inference/pipelines/text/embed.js +39 -12
- package/src/inference/pipelines/text/execution-plan.js +8 -2
- package/src/inference/pipelines/text/ffn/dense.js +15 -4
- package/src/inference/pipelines/text/ffn/moe.js +4 -3
- package/src/inference/pipelines/text/ffn/standard.d.ts +2 -1
- package/src/inference/pipelines/text/ffn/standard.js +25 -2
- package/src/inference/pipelines/text/generator-helpers.js +2 -1
- package/src/inference/pipelines/text/generator-runtime.d.ts +6 -2
- package/src/inference/pipelines/text/generator-runtime.js +53 -20
- package/src/inference/pipelines/text/generator-steps.js +3 -0
- package/src/inference/pipelines/text/generator.js +6 -1
- package/src/inference/pipelines/text/init-rope.js +54 -6
- package/src/inference/pipelines/text/init.js +63 -15
- package/src/inference/pipelines/text/layer.js +32 -5
- package/src/inference/pipelines/text/linear-attention.js +12 -8
- package/src/inference/pipelines/text/logits/gpu.d.ts +1 -1
- package/src/inference/pipelines/text/logits/gpu.js +76 -113
- package/src/inference/pipelines/text/logits/index.js +44 -87
- package/src/inference/pipelines/text/logits/precision-policy.d.ts +11 -0
- package/src/inference/pipelines/text/logits/precision-policy.js +103 -0
- package/src/inference/pipelines/text/logits/types.d.ts +1 -0
- package/src/inference/pipelines/text/logits/utils.d.ts +2 -0
- package/src/inference/pipelines/text/logits/utils.js +11 -0
- package/src/inference/pipelines/text/lora-apply.js +6 -4
- package/src/inference/pipelines/text/moe-cpu.js +25 -0
- package/src/inference/pipelines/text/moe-gpu.js +148 -97
- package/src/inference/pipelines/text/moe-shape-validator.d.ts +33 -3
- package/src/inference/pipelines/text/moe-shape-validator.js +149 -31
- package/src/inference/pipelines/text/ops.d.ts +3 -0
- package/src/inference/pipelines/text/ops.js +26 -1
- package/src/inference/pipelines/text/per-layer-inputs.js +27 -12
- package/src/inference/pipelines/text/weights.js +7 -2
- package/src/inference/pipelines/text.js +24 -4
- package/src/inference/tokenizers/behavior-flags.js +23 -2
- package/src/loader/final-weights-loader.js +12 -1
- package/src/rules/execution-rules-contract-check.js +12 -3
- package/src/rules/inference/capability-transforms.rules.json +14 -2
- package/src/rules/inference/config.rules.json +26 -2
- package/src/rules/inference/dtype.rules.json +12 -5
- package/src/rules/inference/execution.rules.json +5 -19
- package/src/rules/inference/moe.rules.json +97 -0
- package/src/tooling/command-api-constants.js +2 -2
- package/src/tooling/command-api-family-normalizers.js +2 -1
- package/src/tooling/command-api.d.ts +1 -1
- package/src/tooling/command-runner-shared.d.ts +1 -1
- package/src/tooling/command-runner-shared.js +3 -1
- package/src/tooling/node-converter.js +20 -0
- package/src/tooling/node-source-runtime.js +5 -1
- package/src/version.js +1 -1
- package/assets/doppler-readme-hero.svg +0 -84
|
@@ -40,13 +40,13 @@ export interface RoPEConfigSchema {
|
|
|
40
40
|
ropeLocalFrequencyBaseDim?: number | null;
|
|
41
41
|
|
|
42
42
|
/** RoPE scaling type */
|
|
43
|
-
ropeScalingType?: 'linear' | 'dynamic' | 'yarn' | null;
|
|
43
|
+
ropeScalingType?: 'linear' | 'dynamic' | 'yarn' | 'longrope' | null;
|
|
44
44
|
|
|
45
45
|
/** RoPE scaling factor */
|
|
46
46
|
ropeScalingFactor?: number;
|
|
47
47
|
|
|
48
48
|
/** Local RoPE scaling type for sliding window layers */
|
|
49
|
-
ropeLocalScalingType?: 'linear' | 'dynamic' | 'yarn' | null;
|
|
49
|
+
ropeLocalScalingType?: 'linear' | 'dynamic' | 'yarn' | 'longrope' | null;
|
|
50
50
|
|
|
51
51
|
/** Local RoPE scaling factor for sliding window layers */
|
|
52
52
|
ropeLocalScalingFactor?: number;
|
|
@@ -59,6 +59,12 @@ export interface RoPEConfigSchema {
|
|
|
59
59
|
|
|
60
60
|
/** YARN original max position embeddings */
|
|
61
61
|
yarnOriginalMaxPos?: number;
|
|
62
|
+
/** LongRoPE short-context factors */
|
|
63
|
+
longropeShortFactor?: number[] | null;
|
|
64
|
+
/** LongRoPE long-context factors */
|
|
65
|
+
longropeLongFactor?: number[] | null;
|
|
66
|
+
/** LongRoPE original max position embeddings */
|
|
67
|
+
longropeOriginalMaxPos?: number | null;
|
|
62
68
|
|
|
63
69
|
/** Local YARN beta_fast parameter */
|
|
64
70
|
ropeLocalYarnBetaFast?: number;
|
|
@@ -87,7 +93,7 @@ export interface AttentionSchema {
|
|
|
87
93
|
/** Apply sigmoid gate from q_proj split to attention output */
|
|
88
94
|
attentionOutputGate?: boolean;
|
|
89
95
|
/** @deprecated Use RoPEConfigSchema.ropeScalingType instead */
|
|
90
|
-
ropeScalingType?: 'linear' | 'dynamic' | 'yarn' | null;
|
|
96
|
+
ropeScalingType?: 'linear' | 'dynamic' | 'yarn' | 'longrope' | null;
|
|
91
97
|
/** @deprecated Use RoPEConfigSchema.ropeScalingFactor instead */
|
|
92
98
|
ropeScalingFactor?: number;
|
|
93
99
|
}
|
|
@@ -218,6 +224,12 @@ export interface OutputSchema {
|
|
|
218
224
|
finalLogitSoftcapping?: number | null;
|
|
219
225
|
/** Tie embeddings to output */
|
|
220
226
|
tieWordEmbeddings?: boolean;
|
|
227
|
+
/** Scale embeddings by sqrt(hiddenSize). */
|
|
228
|
+
scaleEmbeddings?: boolean;
|
|
229
|
+
/** Explicit embedding multiplier, null to use scaleEmbeddings semantics. */
|
|
230
|
+
embeddingScale?: number | null;
|
|
231
|
+
/** Multiplier applied after final norm before LM head projection. */
|
|
232
|
+
logitInputScale?: number;
|
|
221
233
|
}
|
|
222
234
|
|
|
223
235
|
/** Layer type for hybrid models */
|
|
@@ -255,6 +267,8 @@ export interface LayerPatternSchema {
|
|
|
255
267
|
attentionLayers?: number[];
|
|
256
268
|
/** For 'custom': explicit layer type mapping */
|
|
257
269
|
layerTypes?: LayerType[];
|
|
270
|
+
/** Multiplier applied to attention and FFN branches before residual add. */
|
|
271
|
+
residualBranchScale?: number;
|
|
258
272
|
}
|
|
259
273
|
|
|
260
274
|
/**
|
|
@@ -23,7 +23,7 @@ export type KVDtype = 'f16' | 'f32';
|
|
|
23
23
|
* - 'tiered': Hot ring + cold paged tiers
|
|
24
24
|
* - 'bdpa': Basis-decomposed paged layout (experimental)
|
|
25
25
|
*/
|
|
26
|
-
export type KVLayout = 'contiguous' | 'paged' | 'tiered' | 'bdpa';
|
|
26
|
+
export type KVLayout = 'contiguous' | 'paged' | 'tiered' | 'bdpa' | 'contiguous_quantized';
|
|
27
27
|
|
|
28
28
|
/**
|
|
29
29
|
* Tiered KV cache mode.
|
|
@@ -33,7 +33,7 @@ export type KVLayout = 'contiguous' | 'paged' | 'tiered' | 'bdpa';
|
|
|
33
33
|
* - 'int8': Cold tier compressed to int8
|
|
34
34
|
* - 'int4': Cold tier compressed to int4 (experimental)
|
|
35
35
|
*/
|
|
36
|
-
export type KVTieringMode = 'off' | 'fp16' | 'int8' | 'int4';
|
|
36
|
+
export type KVTieringMode = 'off' | 'fp16' | 'int8' | 'int4' | 'turboquant' | 'turboquant_prod';
|
|
37
37
|
|
|
38
38
|
/**
|
|
39
39
|
* Cold tier compression mode.
|
|
@@ -42,7 +42,7 @@ export type KVTieringMode = 'off' | 'fp16' | 'int8' | 'int4';
|
|
|
42
42
|
* - 'int8': Block-wise int8 compression
|
|
43
43
|
* - 'int4': Block-wise int4 compression
|
|
44
44
|
*/
|
|
45
|
-
export type KVCompressionMode = 'none' | 'int8' | 'int4';
|
|
45
|
+
export type KVCompressionMode = 'none' | 'int8' | 'int4' | 'turboquant' | 'turboquant_prod';
|
|
46
46
|
|
|
47
47
|
/**
|
|
48
48
|
* Gating mode for tiered compression.
|
|
@@ -58,6 +58,19 @@ export interface KVTieringCompressionSchema {
|
|
|
58
58
|
mode: KVCompressionMode;
|
|
59
59
|
/** Compression block size (tokens). Currently only 1 is supported. */
|
|
60
60
|
blockSize: number;
|
|
61
|
+
/** TurboQuant bit width, required for turboquant compression modes */
|
|
62
|
+
bitWidth: number;
|
|
63
|
+
/** Enable TurboQuant production packing */
|
|
64
|
+
prodMode: boolean;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
export interface KVQuantizationConfigSchema {
|
|
68
|
+
/** Contiguous quantized KV cache mode */
|
|
69
|
+
mode: 'none' | 'turboquant' | 'turboquant_prod';
|
|
70
|
+
/** Quantized cache bit width */
|
|
71
|
+
bitWidth: number;
|
|
72
|
+
/** Enable TurboQuant production packing */
|
|
73
|
+
prodMode: boolean;
|
|
61
74
|
}
|
|
62
75
|
|
|
63
76
|
export interface KVTieringGatingSchema {
|
|
@@ -119,6 +132,9 @@ export interface KVCacheConfigSchema {
|
|
|
119
132
|
|
|
120
133
|
/** Tiered cache configuration (hot ring + cold pages) */
|
|
121
134
|
tiering: KVTieringConfigSchema;
|
|
135
|
+
|
|
136
|
+
/** Contiguous quantized KV cache configuration */
|
|
137
|
+
quantization: KVQuantizationConfigSchema;
|
|
122
138
|
}
|
|
123
139
|
|
|
124
140
|
/** Default KV cache configuration */
|
|
@@ -20,6 +20,8 @@ export const DEFAULT_KVCACHE_CONFIG = {
|
|
|
20
20
|
compression: {
|
|
21
21
|
mode: 'none',
|
|
22
22
|
blockSize: 1,
|
|
23
|
+
bitWidth: 4,
|
|
24
|
+
prodMode: false,
|
|
23
25
|
},
|
|
24
26
|
gating: {
|
|
25
27
|
mode: 'auto',
|
|
@@ -36,4 +38,3 @@ export const DEFAULT_KVCACHE_CONFIG = {
|
|
|
36
38
|
};
|
|
37
39
|
|
|
38
40
|
export const PAGED_LAYOUT_SEQ_LEN_THRESHOLD = 8192;
|
|
39
|
-
|
|
@@ -300,7 +300,7 @@ export interface ManifestFFNSchema {
|
|
|
300
300
|
/** Whether activation is gated (e.g., SwiGLU, GeGLU) */
|
|
301
301
|
gatedActivation: boolean;
|
|
302
302
|
/** FFN branch composition. `auto` preserves legacy dense-or-MoE routing. */
|
|
303
|
-
branchMode
|
|
303
|
+
branchMode: 'auto' | 'dense' | 'moe' | 'dense_plus_moe';
|
|
304
304
|
/** Double the FFN intermediate width on KV-shared layers. */
|
|
305
305
|
useDoubleWideMlp: boolean;
|
|
306
306
|
/** Clamp SwiGLU output (null = disabled) */
|
|
@@ -345,6 +345,12 @@ export interface ManifestRoPESchema {
|
|
|
345
345
|
yarnBetaSlow: number | null;
|
|
346
346
|
/** YARN original max position embeddings (null if not YARN scaling) */
|
|
347
347
|
yarnOriginalMaxPos: number | null;
|
|
348
|
+
/** LongRoPE short-context factors (null if not LongRoPE scaling) */
|
|
349
|
+
longropeShortFactor: number[] | null;
|
|
350
|
+
/** LongRoPE long-context factors (null if not LongRoPE scaling) */
|
|
351
|
+
longropeLongFactor: number[] | null;
|
|
352
|
+
/** LongRoPE original max position embeddings (null if not LongRoPE scaling) */
|
|
353
|
+
longropeOriginalMaxPos: number | null;
|
|
348
354
|
/** Local YARN beta_fast parameter (null if not local YARN scaling) */
|
|
349
355
|
ropeLocalYarnBetaFast: number | null;
|
|
350
356
|
/** Local YARN beta_slow parameter (null if not local YARN scaling) */
|
|
@@ -364,6 +370,10 @@ export interface ManifestOutputSchema {
|
|
|
364
370
|
tieWordEmbeddings: boolean;
|
|
365
371
|
/** Scale embeddings by sqrt(hiddenSize) (Gemma models: true) */
|
|
366
372
|
scaleEmbeddings: boolean;
|
|
373
|
+
/** Explicit embedding multiplier, null to use scaleEmbeddings semantics. */
|
|
374
|
+
embeddingScale: number | null;
|
|
375
|
+
/** Multiplier applied after final norm before LM head projection. */
|
|
376
|
+
logitInputScale: number;
|
|
367
377
|
/** Whether embedding weights are stored as [hidden, vocab] (transpose on gather) */
|
|
368
378
|
embeddingTranspose: boolean;
|
|
369
379
|
/** Embedding vocab size from weight tensor (null = use architecture.vocabSize) */
|
|
@@ -402,6 +412,8 @@ export interface ManifestLayerPatternSchema {
|
|
|
402
412
|
offset: number | null;
|
|
403
413
|
/** For custom: explicit per-layer architecture tags, null if not applicable */
|
|
404
414
|
layerTypes: LayerType[] | null;
|
|
415
|
+
/** Multiplier applied to attention and FFN branches before residual add. */
|
|
416
|
+
residualBranchScale: number;
|
|
405
417
|
}
|
|
406
418
|
|
|
407
419
|
/**
|
|
@@ -459,6 +471,20 @@ export interface ManifestDiffusionGemmaSchema {
|
|
|
459
471
|
};
|
|
460
472
|
}
|
|
461
473
|
|
|
474
|
+
export interface ManifestRerankSchema {
|
|
475
|
+
format: 'qwen3_yes_no_logit' | string;
|
|
476
|
+
instruction: string;
|
|
477
|
+
inputTemplate: string;
|
|
478
|
+
prefix: string;
|
|
479
|
+
suffix: string;
|
|
480
|
+
trueToken: string;
|
|
481
|
+
trueTokenId: number;
|
|
482
|
+
falseToken: string;
|
|
483
|
+
falseTokenId: number;
|
|
484
|
+
score: 'logit_difference' | 'true_logit' | string;
|
|
485
|
+
probability: 'sigmoid' | string;
|
|
486
|
+
}
|
|
487
|
+
|
|
462
488
|
/**
|
|
463
489
|
* Complete inference configuration embedded in manifest.
|
|
464
490
|
* All fields are required - converter must populate everything.
|
|
@@ -483,8 +509,14 @@ export interface ManifestInferenceSchema {
|
|
|
483
509
|
layerPattern: ManifestLayerPatternSchema;
|
|
484
510
|
/** Chat template configuration */
|
|
485
511
|
chatTemplate: ManifestChatTemplateSchema;
|
|
512
|
+
/** Whether this artifact exposes embedding workload support through pipeline.embed(). */
|
|
513
|
+
supportsEmbedding: boolean;
|
|
514
|
+
/** Whether this artifact exposes rerank workload support through prefillWithLogits(). */
|
|
515
|
+
supportsRerank: boolean;
|
|
486
516
|
/** DiffusionGemma block-diffusion runtime contract, null for non-DiffusionGemma models. */
|
|
487
517
|
diffusionGemma: ManifestDiffusionGemmaSchema | null;
|
|
518
|
+
/** Manifest-owned rerank scoring contract, null when rerank is unsupported. */
|
|
519
|
+
rerank: ManifestRerankSchema | null;
|
|
488
520
|
/** Layer pipeline override (null = use optimized hardcoded path) */
|
|
489
521
|
pipeline: LayerPipelineSchema | null;
|
|
490
522
|
/** Explicit session policy for execution v1 manifests */
|
|
@@ -761,6 +793,11 @@ export declare function modelSupportsEmbedding(
|
|
|
761
793
|
manifest: Partial<ManifestSchema> | null | undefined
|
|
762
794
|
): boolean;
|
|
763
795
|
|
|
796
|
+
/** Check if a manifest supports rerank workloads. */
|
|
797
|
+
export declare function modelSupportsRerank(
|
|
798
|
+
manifest: Partial<ManifestSchema> | null | undefined
|
|
799
|
+
): boolean;
|
|
800
|
+
|
|
764
801
|
/** Check if a manifest supports audio transcription workloads. */
|
|
765
802
|
export declare function modelSupportsTranscription(
|
|
766
803
|
manifest: Partial<ManifestSchema> | null | undefined
|
|
@@ -49,6 +49,9 @@ export const DEFAULT_MANIFEST_INFERENCE = {
|
|
|
49
49
|
// of modelType: dedicated embedding models (modelType="embedding") always
|
|
50
50
|
// support embedding workloads; text-generation models opt in explicitly.
|
|
51
51
|
supportsEmbedding: false,
|
|
52
|
+
// When true, allows workload="rerank" to dispatch through prefillWithLogits()
|
|
53
|
+
// using manifest-owned scoring metadata in inference.rerank.
|
|
54
|
+
supportsRerank: false,
|
|
52
55
|
// When true, the pipeline exposes pipeline.transcribeAudio({audio,...}) and
|
|
53
56
|
// pipeline.embedAudio({audio}). Requires audio_token_id and audio encoder
|
|
54
57
|
// weights to be present in the manifest. Defaults to false; set to true on
|
|
@@ -106,6 +109,10 @@ export const DEFAULT_MANIFEST_INFERENCE = {
|
|
|
106
109
|
yarnBetaFast: null,
|
|
107
110
|
yarnBetaSlow: null,
|
|
108
111
|
yarnOriginalMaxPos: null,
|
|
112
|
+
// LongRoPE parameters - only relevant when ropeScalingType='longrope'
|
|
113
|
+
longropeShortFactor: null,
|
|
114
|
+
longropeLongFactor: null,
|
|
115
|
+
longropeOriginalMaxPos: null,
|
|
109
116
|
// Local YARN parameters - only relevant when ropeLocalScalingType='yarn'
|
|
110
117
|
ropeLocalYarnBetaFast: null,
|
|
111
118
|
ropeLocalYarnBetaSlow: null,
|
|
@@ -115,6 +122,8 @@ export const DEFAULT_MANIFEST_INFERENCE = {
|
|
|
115
122
|
finalLogitSoftcapping: null, // No softcapping (null = disabled)
|
|
116
123
|
tieWordEmbeddings: false,
|
|
117
124
|
scaleEmbeddings: false,
|
|
125
|
+
embeddingScale: null,
|
|
126
|
+
logitInputScale: 1,
|
|
118
127
|
embeddingTranspose: false,
|
|
119
128
|
embeddingVocabSize: null,
|
|
120
129
|
embeddingPostprocessor: null,
|
|
@@ -125,12 +134,14 @@ export const DEFAULT_MANIFEST_INFERENCE = {
|
|
|
125
134
|
period: null, // No periodic pattern (null = not applicable)
|
|
126
135
|
offset: null, // For every_n: first global layer index modulo period
|
|
127
136
|
layerTypes: null, // For custom: explicit per-layer tags
|
|
137
|
+
residualBranchScale: 1,
|
|
128
138
|
},
|
|
129
139
|
chatTemplate: {
|
|
130
140
|
type: null, // No chat template (null = disabled)
|
|
131
141
|
enabled: false,
|
|
132
142
|
},
|
|
133
143
|
diffusionGemma: null,
|
|
144
|
+
rerank: null,
|
|
134
145
|
pipeline: null,
|
|
135
146
|
session: null,
|
|
136
147
|
execution: null,
|
|
@@ -190,6 +201,13 @@ export function modelSupportsEmbedding(manifest) {
|
|
|
190
201
|
return manifest?.inference?.supportsEmbedding === true;
|
|
191
202
|
}
|
|
192
203
|
|
|
204
|
+
export function modelSupportsRerank(manifest) {
|
|
205
|
+
return manifest?.inference?.supportsRerank === true
|
|
206
|
+
&& manifest?.inference?.rerank != null
|
|
207
|
+
&& typeof manifest.inference.rerank === 'object'
|
|
208
|
+
&& !Array.isArray(manifest.inference.rerank);
|
|
209
|
+
}
|
|
210
|
+
|
|
193
211
|
// Returns true when the manifest declares audio-input capability. Pipelines
|
|
194
212
|
// expose transcribeAudio + embedAudio when this is set. The de-facto runtime
|
|
195
213
|
// gate (audio_token_id + audio encoder weights) still applies; this field is
|
|
@@ -17,6 +17,11 @@
|
|
|
17
17
|
"policyPath": "tools/policies/model-artifact-contract.json",
|
|
18
18
|
"schemaPath": "src/config/schema/model-artifact-contract.schema.json"
|
|
19
19
|
},
|
|
20
|
+
{
|
|
21
|
+
"id": "support-rollout-policy",
|
|
22
|
+
"policyPath": "benchmarks/vendors/support-rollout-policy.json",
|
|
23
|
+
"schemaPath": "benchmarks/vendors/schema/support-rollout-policy.schema.json"
|
|
24
|
+
},
|
|
20
25
|
{
|
|
21
26
|
"id": "subsystem-support-contract",
|
|
22
27
|
"policyPath": "tools/policies/subsystem-support-contract.json",
|
|
@@ -161,6 +161,7 @@
|
|
|
161
161
|
"ffn": {
|
|
162
162
|
"activation": "gelu",
|
|
163
163
|
"gatedActivation": true,
|
|
164
|
+
"branchMode": "auto",
|
|
164
165
|
"useDoubleWideMlp": false,
|
|
165
166
|
"swigluLimit": null
|
|
166
167
|
},
|
|
@@ -267,7 +268,25 @@
|
|
|
267
268
|
"layout": "contiguous",
|
|
268
269
|
"pageSize": 256,
|
|
269
270
|
"tiering": {
|
|
270
|
-
"mode": "off"
|
|
271
|
+
"mode": "off",
|
|
272
|
+
"hotWindow": 1024,
|
|
273
|
+
"coldPageSize": 256,
|
|
274
|
+
"coldDtype": "f16",
|
|
275
|
+
"compression": {
|
|
276
|
+
"mode": "none",
|
|
277
|
+
"blockSize": 1,
|
|
278
|
+
"bitWidth": 4,
|
|
279
|
+
"prodMode": false
|
|
280
|
+
},
|
|
281
|
+
"gating": {
|
|
282
|
+
"mode": "auto",
|
|
283
|
+
"minAluBwRatio": 0
|
|
284
|
+
}
|
|
285
|
+
},
|
|
286
|
+
"quantization": {
|
|
287
|
+
"mode": "none",
|
|
288
|
+
"bitWidth": 4,
|
|
289
|
+
"prodMode": false
|
|
271
290
|
}
|
|
272
291
|
},
|
|
273
292
|
"decodeLoop": {
|
|
@@ -149,6 +149,7 @@
|
|
|
149
149
|
"ffn": {
|
|
150
150
|
"activation": "gelu",
|
|
151
151
|
"gatedActivation": true,
|
|
152
|
+
"branchMode": "auto",
|
|
152
153
|
"useDoubleWideMlp": true,
|
|
153
154
|
"swigluLimit": null
|
|
154
155
|
},
|
|
@@ -207,7 +208,25 @@
|
|
|
207
208
|
"layout": "contiguous",
|
|
208
209
|
"pageSize": 256,
|
|
209
210
|
"tiering": {
|
|
210
|
-
"mode": "off"
|
|
211
|
+
"mode": "off",
|
|
212
|
+
"hotWindow": 1024,
|
|
213
|
+
"coldPageSize": 256,
|
|
214
|
+
"coldDtype": "f16",
|
|
215
|
+
"compression": {
|
|
216
|
+
"mode": "none",
|
|
217
|
+
"blockSize": 1,
|
|
218
|
+
"bitWidth": 4,
|
|
219
|
+
"prodMode": false
|
|
220
|
+
},
|
|
221
|
+
"gating": {
|
|
222
|
+
"mode": "auto",
|
|
223
|
+
"minAluBwRatio": 0
|
|
224
|
+
}
|
|
225
|
+
},
|
|
226
|
+
"quantization": {
|
|
227
|
+
"mode": "none",
|
|
228
|
+
"bitWidth": 4,
|
|
229
|
+
"prodMode": false
|
|
211
230
|
}
|
|
212
231
|
},
|
|
213
232
|
"decodeLoop": {
|
|
@@ -132,6 +132,7 @@
|
|
|
132
132
|
},
|
|
133
133
|
"audio_config": {
|
|
134
134
|
"model_type": "gemma4_audio",
|
|
135
|
+
"audio_architecture": "gemma4",
|
|
135
136
|
"num_hidden_layers": 12,
|
|
136
137
|
"hidden_size": 1024,
|
|
137
138
|
"num_attention_heads": 8,
|
|
@@ -217,6 +218,7 @@
|
|
|
217
218
|
"ffn": {
|
|
218
219
|
"activation": "gelu",
|
|
219
220
|
"gatedActivation": true,
|
|
221
|
+
"branchMode": "auto",
|
|
220
222
|
"useDoubleWideMlp": false,
|
|
221
223
|
"swigluLimit": null
|
|
222
224
|
},
|
|
@@ -275,7 +277,25 @@
|
|
|
275
277
|
"layout": "contiguous",
|
|
276
278
|
"pageSize": 256,
|
|
277
279
|
"tiering": {
|
|
278
|
-
"mode": "off"
|
|
280
|
+
"mode": "off",
|
|
281
|
+
"hotWindow": 1024,
|
|
282
|
+
"coldPageSize": 256,
|
|
283
|
+
"coldDtype": "f16",
|
|
284
|
+
"compression": {
|
|
285
|
+
"mode": "none",
|
|
286
|
+
"blockSize": 1,
|
|
287
|
+
"bitWidth": 4,
|
|
288
|
+
"prodMode": false
|
|
289
|
+
},
|
|
290
|
+
"gating": {
|
|
291
|
+
"mode": "auto",
|
|
292
|
+
"minAluBwRatio": 0
|
|
293
|
+
}
|
|
294
|
+
},
|
|
295
|
+
"quantization": {
|
|
296
|
+
"mode": "none",
|
|
297
|
+
"bitWidth": 4,
|
|
298
|
+
"prodMode": false
|
|
279
299
|
}
|
|
280
300
|
},
|
|
281
301
|
"decodeLoop": {
|
|
@@ -38,6 +38,12 @@ export interface ExecutionKernelPrecision {
|
|
|
38
38
|
outputDtype?: 'f16' | 'f32';
|
|
39
39
|
}
|
|
40
40
|
|
|
41
|
+
export declare function getKernelFilePrecisionPatch(
|
|
42
|
+
kernel: string
|
|
43
|
+
): Pick<ExecutionKernelPrecision, 'inputDtype' | 'outputDtype'> | null;
|
|
44
|
+
|
|
45
|
+
export declare function resolveF16ToF32ActivationKernel(kernel: string): string | null;
|
|
46
|
+
|
|
41
47
|
/**
|
|
42
48
|
* An execution graph kernel entry from manifest.inference.execution.kernels.
|
|
43
49
|
*/
|
|
@@ -49,6 +49,10 @@ const KERNEL_FILE_PRECISION_PATCHES = new Map([
|
|
|
49
49
|
['gather_f16_vec4_f16_out.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
|
|
50
50
|
]);
|
|
51
51
|
|
|
52
|
+
export function getKernelFilePrecisionPatch(kernel) {
|
|
53
|
+
return KERNEL_FILE_PRECISION_PATCHES.get(kernel) ?? null;
|
|
54
|
+
}
|
|
55
|
+
|
|
52
56
|
/*
|
|
53
57
|
* Check whether a kernel entry requires subgroup support.
|
|
54
58
|
*/
|
|
@@ -686,6 +690,10 @@ const F16_TO_F32_ACTIVATION_MAP = new Map([
|
|
|
686
690
|
['attention_head512_f16.wgsl', 'attention_head512_f16kv.wgsl'],
|
|
687
691
|
]);
|
|
688
692
|
|
|
693
|
+
export function resolveF16ToF32ActivationKernel(kernel) {
|
|
694
|
+
return F16_TO_F32_ACTIVATION_MAP.get(kernel) ?? null;
|
|
695
|
+
}
|
|
696
|
+
|
|
689
697
|
/*
|
|
690
698
|
* Activation-only narrowing: f32-activation shaders that still consume f16
|
|
691
699
|
* weights/KV are rewritten onto the matching f16-activation lane.
|
|
@@ -217,8 +217,10 @@ function resolveConversionPlanV1(options) {
|
|
|
217
217
|
layerPattern: inference.layerPattern ?? { type: 'uniform', globalPattern: null, period: null, offset: null, layerTypes: null },
|
|
218
218
|
chatTemplate: inference.chatTemplate,
|
|
219
219
|
supportsEmbedding: inference.supportsEmbedding ?? false,
|
|
220
|
+
supportsRerank: inference.supportsRerank ?? false,
|
|
220
221
|
supportsTranscription: inference.supportsTranscription ?? false,
|
|
221
222
|
supportsVision: inference.supportsVision ?? false,
|
|
223
|
+
rerank: inference.rerank ?? null,
|
|
222
224
|
diffusionGemma: inference.diffusionGemma ?? null,
|
|
223
225
|
pipeline: inference.pipeline ?? null,
|
|
224
226
|
session,
|
|
@@ -54,6 +54,22 @@ function renameDenseTensor(tensor, tensorName) {
|
|
|
54
54
|
};
|
|
55
55
|
}
|
|
56
56
|
|
|
57
|
+
function resolveLogitScoreConfig(config, modulePath) {
|
|
58
|
+
const trueTokenId = Number(config?.true_token_id ?? config?.trueTokenId);
|
|
59
|
+
const falseTokenId = Number(config?.false_token_id ?? config?.falseTokenId);
|
|
60
|
+
if (!Number.isInteger(trueTokenId) || trueTokenId < 0) {
|
|
61
|
+
throw new Error(`transformer parser: ${modulePath}/config.json must declare non-negative integer true_token_id.`);
|
|
62
|
+
}
|
|
63
|
+
if (!Number.isInteger(falseTokenId) || falseTokenId < 0) {
|
|
64
|
+
throw new Error(`transformer parser: ${modulePath}/config.json must declare non-negative integer false_token_id.`);
|
|
65
|
+
}
|
|
66
|
+
return {
|
|
67
|
+
kind: 'logit_score',
|
|
68
|
+
trueTokenId,
|
|
69
|
+
falseTokenId,
|
|
70
|
+
};
|
|
71
|
+
}
|
|
72
|
+
|
|
57
73
|
async function parseSentenceTransformersModules(adapter) {
|
|
58
74
|
const {
|
|
59
75
|
readJson,
|
|
@@ -69,6 +85,7 @@ async function parseSentenceTransformersModules(adapter) {
|
|
|
69
85
|
let normalize = null;
|
|
70
86
|
const projections = [];
|
|
71
87
|
const extraTensors = [];
|
|
88
|
+
let rerankScoring = null;
|
|
72
89
|
|
|
73
90
|
for (const moduleEntry of modules) {
|
|
74
91
|
const type = asNonEmptyString(moduleEntry?.type, 'modules.json type');
|
|
@@ -139,6 +156,12 @@ async function parseSentenceTransformersModules(adapter) {
|
|
|
139
156
|
continue;
|
|
140
157
|
}
|
|
141
158
|
|
|
159
|
+
if (isSentenceTransformerModuleType(type, '.LogitScore')) {
|
|
160
|
+
const config = await readJson(`${modulePath}/config.json`, `${modulePath}/config.json`);
|
|
161
|
+
rerankScoring = resolveLogitScoreConfig(config, modulePath);
|
|
162
|
+
continue;
|
|
163
|
+
}
|
|
164
|
+
|
|
142
165
|
throw new Error(`transformer parser: unsupported sentence-transformers module type "${type}".`);
|
|
143
166
|
}
|
|
144
167
|
|
|
@@ -146,6 +169,7 @@ async function parseSentenceTransformersModules(adapter) {
|
|
|
146
169
|
return {
|
|
147
170
|
extraTensors: [],
|
|
148
171
|
embeddingPostprocessor: null,
|
|
172
|
+
rerankScoring,
|
|
149
173
|
};
|
|
150
174
|
}
|
|
151
175
|
if (poolingMode == null) {
|
|
@@ -160,6 +184,7 @@ async function parseSentenceTransformersModules(adapter) {
|
|
|
160
184
|
projections,
|
|
161
185
|
normalize,
|
|
162
186
|
},
|
|
187
|
+
rerankScoring,
|
|
163
188
|
};
|
|
164
189
|
}
|
|
165
190
|
|
|
@@ -178,7 +203,7 @@ export async function parseTransformerModel(adapter) {
|
|
|
178
203
|
const architectureHint = config.architectures?.[0] ?? config.model_type ?? '';
|
|
179
204
|
const sentenceTransformersModules = await fileExists('modules.json')
|
|
180
205
|
? await parseSentenceTransformersModules(adapter)
|
|
181
|
-
: { extraTensors: [], embeddingPostprocessor: null };
|
|
206
|
+
: { extraTensors: [], embeddingPostprocessor: null, rerankScoring: null };
|
|
182
207
|
|
|
183
208
|
let tensors = null;
|
|
184
209
|
if (await fileExists('model.safetensors.index.json')) {
|
|
@@ -194,5 +219,6 @@ export async function parseTransformerModel(adapter) {
|
|
|
194
219
|
tensors: [...tensors, ...sentenceTransformersModules.extraTensors],
|
|
195
220
|
architectureHint,
|
|
196
221
|
embeddingPostprocessor: sentenceTransformersModules.embeddingPostprocessor,
|
|
222
|
+
rerankScoring: sentenceTransformersModules.rerankScoring ?? null,
|
|
197
223
|
};
|
|
198
224
|
}
|
|
@@ -570,7 +570,7 @@ export function resolveEffectiveQuantizationInfo(baseInfo, tensors) {
|
|
|
570
570
|
const base = (baseInfo && typeof baseInfo === 'object') ? baseInfo : {};
|
|
571
571
|
const entries = Array.isArray(tensors) ? tensors : [];
|
|
572
572
|
|
|
573
|
-
|
|
573
|
+
const detectedWeightDtypes = new Set();
|
|
574
574
|
let detectedEmbeddings = null;
|
|
575
575
|
let detectedLmHead = null;
|
|
576
576
|
let detectedLayout = null;
|
|
@@ -590,15 +590,27 @@ export function resolveEffectiveQuantizationInfo(baseInfo, tensors) {
|
|
|
590
590
|
continue;
|
|
591
591
|
}
|
|
592
592
|
if (role === 'matmul' || role === 'expert' || role === 'router') {
|
|
593
|
-
|
|
593
|
+
detectedWeightDtypes.add(dtype);
|
|
594
594
|
if (detectedLayout == null && (dtype === 'q4k') && tensor?.layout) {
|
|
595
595
|
detectedLayout = normalizeQ4KLayout(tensor.layout);
|
|
596
596
|
}
|
|
597
597
|
}
|
|
598
598
|
}
|
|
599
599
|
|
|
600
|
-
const
|
|
601
|
-
|
|
600
|
+
const baseWeights = normalizeQuantTag(base.weights ?? 'f16');
|
|
601
|
+
let weights = baseWeights;
|
|
602
|
+
if (detectedWeightDtypes.size === 1) {
|
|
603
|
+
weights = [...detectedWeightDtypes][0];
|
|
604
|
+
} else if (detectedWeightDtypes.size > 1) {
|
|
605
|
+
if (detectedWeightDtypes.has(baseWeights)) {
|
|
606
|
+
weights = baseWeights;
|
|
607
|
+
} else {
|
|
608
|
+
throw new Error(
|
|
609
|
+
`Ambiguous converted weight dtypes: ${[...detectedWeightDtypes].sort().join(', ')}. ` +
|
|
610
|
+
`Configured quantizationInfo.weights="${baseWeights}" is not present in converted matmul tensors.`
|
|
611
|
+
);
|
|
612
|
+
}
|
|
613
|
+
}
|
|
602
614
|
const embeddings = detectedEmbeddings
|
|
603
615
|
?? normalizeQuantTag(base.embeddings ?? weights);
|
|
604
616
|
const lmHead = detectedLmHead
|