doppler-gpu 0.4.6 → 0.4.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (149) hide show
  1. package/README.md +43 -31
  2. package/assets/doppler-runtime-map.svg +1 -3
  3. package/assets/doppler.svg +1 -3
  4. package/package.json +25 -17
  5. package/src/cli/doppler-quickstart.js +1 -1
  6. package/src/cli/doppler-serve.js +2 -2
  7. package/src/client/doppler-registry.d.ts +19 -0
  8. package/src/client/doppler-registry.js +12 -0
  9. package/src/client/doppler-registry.json +50 -1
  10. package/src/config/conversion/README.md +4 -0
  11. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +28 -4
  12. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +29 -4
  13. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +29 -4
  14. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +29 -4
  15. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +29 -4
  16. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +29 -4
  17. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +29 -4
  18. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +29 -4
  19. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +29 -4
  20. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +31 -5
  21. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +30 -4
  22. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +30 -4
  23. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +30 -4
  24. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +30 -5
  25. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +29 -4
  26. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +30 -5
  27. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +29 -4
  28. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +29 -4
  29. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +29 -4
  30. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +29 -4
  31. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +29 -4
  32. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +29 -4
  33. package/src/config/conversion/minicpm/minicpm4-0-5b-f16-af32.json +481 -0
  34. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +29 -4
  35. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +29 -4
  36. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +30 -5
  37. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +29 -4
  38. package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-q4k-ehf16-af32.json +395 -0
  39. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-af32.json +429 -0
  40. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +540 -0
  41. package/src/config/execution-contract-check.js +14 -10
  42. package/src/config/merge-contract-check.js +1 -0
  43. package/src/config/merge.js +30 -0
  44. package/src/config/param-validator.d.ts +5 -0
  45. package/src/config/param-validator.js +42 -8
  46. package/src/config/required-inference-fields-contract-check.js +14 -0
  47. package/src/config/schema/debug.schema.d.ts +2 -0
  48. package/src/config/schema/debug.schema.js +1 -0
  49. package/src/config/schema/doppler.schema.d.ts +5 -1
  50. package/src/config/schema/doppler.schema.js +82 -292
  51. package/src/config/schema/inference.schema.d.ts +17 -3
  52. package/src/config/schema/kvcache.schema.d.ts +19 -3
  53. package/src/config/schema/kvcache.schema.js +2 -1
  54. package/src/config/schema/manifest.schema.d.ts +38 -1
  55. package/src/config/schema/manifest.schema.js +18 -0
  56. package/src/config/schema/policy-schema-registry.json +5 -0
  57. package/src/config/source-packages/litert/gemma-4-12b-it.json +20 -1
  58. package/src/config/source-packages/litert/gemma-4-e2b-it.json +20 -1
  59. package/src/config/source-packages/litert/gemma-4-e4b-it.json +21 -1
  60. package/src/config/transforms/execution-graph-transforms.d.ts +6 -0
  61. package/src/config/transforms/execution-graph-transforms.js +8 -0
  62. package/src/converter/conversion-plan.js +2 -0
  63. package/src/converter/parsers/transformer.js +27 -1
  64. package/src/converter/quantization-info.js +16 -4
  65. package/src/converter/rope-config.js +55 -3
  66. package/src/experimental/training/distillation/student-fixture.js +1 -1
  67. package/src/experimental/training/lora-pipeline.js +1 -1
  68. package/src/experimental/training/lora.js +2 -2
  69. package/src/experimental/training/suite.js +1 -1
  70. package/src/gpu/kernels/README.md +8 -1
  71. package/src/gpu/kernels/backward/attention_backward.js +8 -0
  72. package/src/gpu/kernels/dequant.d.ts +6 -6
  73. package/src/gpu/kernels/dequant.js +17 -6
  74. package/src/gpu/kernels/linear-attention-core.js +1 -6
  75. package/src/gpu/kernels/matmul-selection.d.ts +2 -2
  76. package/src/gpu/kernels/matmul-selection.js +4 -1
  77. package/src/gpu/kernels/matmul.d.ts +1 -1
  78. package/src/gpu/kernels/matmul.js +24 -6
  79. package/src/gpu/kernels/moe.d.ts +5 -1
  80. package/src/gpu/kernels/moe.js +5 -2
  81. package/src/gpu/kernels/softmax.d.ts +3 -3
  82. package/src/gpu/kernels/softmax.js +9 -2
  83. package/src/gpu/kernels/turboquant-codebook.js +1 -1
  84. package/src/gpu/kernels/types.d.ts +1 -1
  85. package/src/gpu/profiler.js +11 -1
  86. package/src/inference/browser-harness-model-helpers.js +4 -1
  87. package/src/inference/browser-harness-text-helpers.d.ts +28 -0
  88. package/src/inference/browser-harness-text-helpers.js +357 -116
  89. package/src/inference/browser-harness.d.ts +3 -3
  90. package/src/inference/browser-harness.js +266 -11
  91. package/src/inference/fixtures/rerank-semantic-fixtures.json +38 -0
  92. package/src/inference/kv-cache/quantized.js +16 -2
  93. package/src/inference/kv-cache/tiered.js +50 -15
  94. package/src/inference/moe-router.js +38 -7
  95. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +1 -1
  96. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +8 -6
  97. package/src/inference/pipelines/text/config.d.ts +5 -1
  98. package/src/inference/pipelines/text/config.js +135 -112
  99. package/src/inference/pipelines/text/embed.d.ts +1 -0
  100. package/src/inference/pipelines/text/embed.js +39 -12
  101. package/src/inference/pipelines/text/execution-plan.js +8 -2
  102. package/src/inference/pipelines/text/ffn/dense.js +15 -4
  103. package/src/inference/pipelines/text/ffn/moe.js +4 -3
  104. package/src/inference/pipelines/text/ffn/standard.d.ts +2 -1
  105. package/src/inference/pipelines/text/ffn/standard.js +25 -2
  106. package/src/inference/pipelines/text/generator-helpers.js +2 -1
  107. package/src/inference/pipelines/text/generator-runtime.d.ts +6 -2
  108. package/src/inference/pipelines/text/generator-runtime.js +53 -20
  109. package/src/inference/pipelines/text/generator-steps.js +3 -0
  110. package/src/inference/pipelines/text/generator.js +6 -1
  111. package/src/inference/pipelines/text/init-rope.js +54 -6
  112. package/src/inference/pipelines/text/init.js +63 -15
  113. package/src/inference/pipelines/text/layer.js +32 -5
  114. package/src/inference/pipelines/text/linear-attention.js +12 -8
  115. package/src/inference/pipelines/text/logits/gpu.d.ts +1 -1
  116. package/src/inference/pipelines/text/logits/gpu.js +76 -113
  117. package/src/inference/pipelines/text/logits/index.js +44 -87
  118. package/src/inference/pipelines/text/logits/precision-policy.d.ts +11 -0
  119. package/src/inference/pipelines/text/logits/precision-policy.js +103 -0
  120. package/src/inference/pipelines/text/logits/types.d.ts +1 -0
  121. package/src/inference/pipelines/text/logits/utils.d.ts +2 -0
  122. package/src/inference/pipelines/text/logits/utils.js +11 -0
  123. package/src/inference/pipelines/text/lora-apply.js +6 -4
  124. package/src/inference/pipelines/text/moe-cpu.js +25 -0
  125. package/src/inference/pipelines/text/moe-gpu.js +148 -97
  126. package/src/inference/pipelines/text/moe-shape-validator.d.ts +33 -3
  127. package/src/inference/pipelines/text/moe-shape-validator.js +149 -31
  128. package/src/inference/pipelines/text/ops.d.ts +3 -0
  129. package/src/inference/pipelines/text/ops.js +26 -1
  130. package/src/inference/pipelines/text/per-layer-inputs.js +27 -12
  131. package/src/inference/pipelines/text/weights.js +7 -2
  132. package/src/inference/pipelines/text.js +24 -4
  133. package/src/inference/tokenizers/behavior-flags.js +23 -2
  134. package/src/loader/final-weights-loader.js +12 -1
  135. package/src/rules/execution-rules-contract-check.js +12 -3
  136. package/src/rules/inference/capability-transforms.rules.json +14 -2
  137. package/src/rules/inference/config.rules.json +26 -2
  138. package/src/rules/inference/dtype.rules.json +12 -5
  139. package/src/rules/inference/execution.rules.json +5 -19
  140. package/src/rules/inference/moe.rules.json +97 -0
  141. package/src/tooling/command-api-constants.js +2 -2
  142. package/src/tooling/command-api-family-normalizers.js +2 -1
  143. package/src/tooling/command-api.d.ts +1 -1
  144. package/src/tooling/command-runner-shared.d.ts +1 -1
  145. package/src/tooling/command-runner-shared.js +3 -1
  146. package/src/tooling/node-converter.js +20 -0
  147. package/src/tooling/node-source-runtime.js +5 -1
  148. package/src/version.js +1 -1
  149. package/assets/doppler-readme-hero.svg +0 -84
@@ -40,13 +40,13 @@ export interface RoPEConfigSchema {
40
40
  ropeLocalFrequencyBaseDim?: number | null;
41
41
 
42
42
  /** RoPE scaling type */
43
- ropeScalingType?: 'linear' | 'dynamic' | 'yarn' | null;
43
+ ropeScalingType?: 'linear' | 'dynamic' | 'yarn' | 'longrope' | null;
44
44
 
45
45
  /** RoPE scaling factor */
46
46
  ropeScalingFactor?: number;
47
47
 
48
48
  /** Local RoPE scaling type for sliding window layers */
49
- ropeLocalScalingType?: 'linear' | 'dynamic' | 'yarn' | null;
49
+ ropeLocalScalingType?: 'linear' | 'dynamic' | 'yarn' | 'longrope' | null;
50
50
 
51
51
  /** Local RoPE scaling factor for sliding window layers */
52
52
  ropeLocalScalingFactor?: number;
@@ -59,6 +59,12 @@ export interface RoPEConfigSchema {
59
59
 
60
60
  /** YARN original max position embeddings */
61
61
  yarnOriginalMaxPos?: number;
62
+ /** LongRoPE short-context factors */
63
+ longropeShortFactor?: number[] | null;
64
+ /** LongRoPE long-context factors */
65
+ longropeLongFactor?: number[] | null;
66
+ /** LongRoPE original max position embeddings */
67
+ longropeOriginalMaxPos?: number | null;
62
68
 
63
69
  /** Local YARN beta_fast parameter */
64
70
  ropeLocalYarnBetaFast?: number;
@@ -87,7 +93,7 @@ export interface AttentionSchema {
87
93
  /** Apply sigmoid gate from q_proj split to attention output */
88
94
  attentionOutputGate?: boolean;
89
95
  /** @deprecated Use RoPEConfigSchema.ropeScalingType instead */
90
- ropeScalingType?: 'linear' | 'dynamic' | 'yarn' | null;
96
+ ropeScalingType?: 'linear' | 'dynamic' | 'yarn' | 'longrope' | null;
91
97
  /** @deprecated Use RoPEConfigSchema.ropeScalingFactor instead */
92
98
  ropeScalingFactor?: number;
93
99
  }
@@ -218,6 +224,12 @@ export interface OutputSchema {
218
224
  finalLogitSoftcapping?: number | null;
219
225
  /** Tie embeddings to output */
220
226
  tieWordEmbeddings?: boolean;
227
+ /** Scale embeddings by sqrt(hiddenSize). */
228
+ scaleEmbeddings?: boolean;
229
+ /** Explicit embedding multiplier, null to use scaleEmbeddings semantics. */
230
+ embeddingScale?: number | null;
231
+ /** Multiplier applied after final norm before LM head projection. */
232
+ logitInputScale?: number;
221
233
  }
222
234
 
223
235
  /** Layer type for hybrid models */
@@ -255,6 +267,8 @@ export interface LayerPatternSchema {
255
267
  attentionLayers?: number[];
256
268
  /** For 'custom': explicit layer type mapping */
257
269
  layerTypes?: LayerType[];
270
+ /** Multiplier applied to attention and FFN branches before residual add. */
271
+ residualBranchScale?: number;
258
272
  }
259
273
 
260
274
  /**
@@ -23,7 +23,7 @@ export type KVDtype = 'f16' | 'f32';
23
23
  * - 'tiered': Hot ring + cold paged tiers
24
24
  * - 'bdpa': Basis-decomposed paged layout (experimental)
25
25
  */
26
- export type KVLayout = 'contiguous' | 'paged' | 'tiered' | 'bdpa';
26
+ export type KVLayout = 'contiguous' | 'paged' | 'tiered' | 'bdpa' | 'contiguous_quantized';
27
27
 
28
28
  /**
29
29
  * Tiered KV cache mode.
@@ -33,7 +33,7 @@ export type KVLayout = 'contiguous' | 'paged' | 'tiered' | 'bdpa';
33
33
  * - 'int8': Cold tier compressed to int8
34
34
  * - 'int4': Cold tier compressed to int4 (experimental)
35
35
  */
36
- export type KVTieringMode = 'off' | 'fp16' | 'int8' | 'int4';
36
+ export type KVTieringMode = 'off' | 'fp16' | 'int8' | 'int4' | 'turboquant' | 'turboquant_prod';
37
37
 
38
38
  /**
39
39
  * Cold tier compression mode.
@@ -42,7 +42,7 @@ export type KVTieringMode = 'off' | 'fp16' | 'int8' | 'int4';
42
42
  * - 'int8': Block-wise int8 compression
43
43
  * - 'int4': Block-wise int4 compression
44
44
  */
45
- export type KVCompressionMode = 'none' | 'int8' | 'int4';
45
+ export type KVCompressionMode = 'none' | 'int8' | 'int4' | 'turboquant' | 'turboquant_prod';
46
46
 
47
47
  /**
48
48
  * Gating mode for tiered compression.
@@ -58,6 +58,19 @@ export interface KVTieringCompressionSchema {
58
58
  mode: KVCompressionMode;
59
59
  /** Compression block size (tokens). Currently only 1 is supported. */
60
60
  blockSize: number;
61
+ /** TurboQuant bit width, required for turboquant compression modes */
62
+ bitWidth: number;
63
+ /** Enable TurboQuant production packing */
64
+ prodMode: boolean;
65
+ }
66
+
67
+ export interface KVQuantizationConfigSchema {
68
+ /** Contiguous quantized KV cache mode */
69
+ mode: 'none' | 'turboquant' | 'turboquant_prod';
70
+ /** Quantized cache bit width */
71
+ bitWidth: number;
72
+ /** Enable TurboQuant production packing */
73
+ prodMode: boolean;
61
74
  }
62
75
 
63
76
  export interface KVTieringGatingSchema {
@@ -119,6 +132,9 @@ export interface KVCacheConfigSchema {
119
132
 
120
133
  /** Tiered cache configuration (hot ring + cold pages) */
121
134
  tiering: KVTieringConfigSchema;
135
+
136
+ /** Contiguous quantized KV cache configuration */
137
+ quantization: KVQuantizationConfigSchema;
122
138
  }
123
139
 
124
140
  /** Default KV cache configuration */
@@ -20,6 +20,8 @@ export const DEFAULT_KVCACHE_CONFIG = {
20
20
  compression: {
21
21
  mode: 'none',
22
22
  blockSize: 1,
23
+ bitWidth: 4,
24
+ prodMode: false,
23
25
  },
24
26
  gating: {
25
27
  mode: 'auto',
@@ -36,4 +38,3 @@ export const DEFAULT_KVCACHE_CONFIG = {
36
38
  };
37
39
 
38
40
  export const PAGED_LAYOUT_SEQ_LEN_THRESHOLD = 8192;
39
-
@@ -300,7 +300,7 @@ export interface ManifestFFNSchema {
300
300
  /** Whether activation is gated (e.g., SwiGLU, GeGLU) */
301
301
  gatedActivation: boolean;
302
302
  /** FFN branch composition. `auto` preserves legacy dense-or-MoE routing. */
303
- branchMode?: 'auto' | 'dense' | 'moe' | 'dense_plus_moe';
303
+ branchMode: 'auto' | 'dense' | 'moe' | 'dense_plus_moe';
304
304
  /** Double the FFN intermediate width on KV-shared layers. */
305
305
  useDoubleWideMlp: boolean;
306
306
  /** Clamp SwiGLU output (null = disabled) */
@@ -345,6 +345,12 @@ export interface ManifestRoPESchema {
345
345
  yarnBetaSlow: number | null;
346
346
  /** YARN original max position embeddings (null if not YARN scaling) */
347
347
  yarnOriginalMaxPos: number | null;
348
+ /** LongRoPE short-context factors (null if not LongRoPE scaling) */
349
+ longropeShortFactor: number[] | null;
350
+ /** LongRoPE long-context factors (null if not LongRoPE scaling) */
351
+ longropeLongFactor: number[] | null;
352
+ /** LongRoPE original max position embeddings (null if not LongRoPE scaling) */
353
+ longropeOriginalMaxPos: number | null;
348
354
  /** Local YARN beta_fast parameter (null if not local YARN scaling) */
349
355
  ropeLocalYarnBetaFast: number | null;
350
356
  /** Local YARN beta_slow parameter (null if not local YARN scaling) */
@@ -364,6 +370,10 @@ export interface ManifestOutputSchema {
364
370
  tieWordEmbeddings: boolean;
365
371
  /** Scale embeddings by sqrt(hiddenSize) (Gemma models: true) */
366
372
  scaleEmbeddings: boolean;
373
+ /** Explicit embedding multiplier, null to use scaleEmbeddings semantics. */
374
+ embeddingScale: number | null;
375
+ /** Multiplier applied after final norm before LM head projection. */
376
+ logitInputScale: number;
367
377
  /** Whether embedding weights are stored as [hidden, vocab] (transpose on gather) */
368
378
  embeddingTranspose: boolean;
369
379
  /** Embedding vocab size from weight tensor (null = use architecture.vocabSize) */
@@ -402,6 +412,8 @@ export interface ManifestLayerPatternSchema {
402
412
  offset: number | null;
403
413
  /** For custom: explicit per-layer architecture tags, null if not applicable */
404
414
  layerTypes: LayerType[] | null;
415
+ /** Multiplier applied to attention and FFN branches before residual add. */
416
+ residualBranchScale: number;
405
417
  }
406
418
 
407
419
  /**
@@ -459,6 +471,20 @@ export interface ManifestDiffusionGemmaSchema {
459
471
  };
460
472
  }
461
473
 
474
+ export interface ManifestRerankSchema {
475
+ format: 'qwen3_yes_no_logit' | string;
476
+ instruction: string;
477
+ inputTemplate: string;
478
+ prefix: string;
479
+ suffix: string;
480
+ trueToken: string;
481
+ trueTokenId: number;
482
+ falseToken: string;
483
+ falseTokenId: number;
484
+ score: 'logit_difference' | 'true_logit' | string;
485
+ probability: 'sigmoid' | string;
486
+ }
487
+
462
488
  /**
463
489
  * Complete inference configuration embedded in manifest.
464
490
  * All fields are required - converter must populate everything.
@@ -483,8 +509,14 @@ export interface ManifestInferenceSchema {
483
509
  layerPattern: ManifestLayerPatternSchema;
484
510
  /** Chat template configuration */
485
511
  chatTemplate: ManifestChatTemplateSchema;
512
+ /** Whether this artifact exposes embedding workload support through pipeline.embed(). */
513
+ supportsEmbedding: boolean;
514
+ /** Whether this artifact exposes rerank workload support through prefillWithLogits(). */
515
+ supportsRerank: boolean;
486
516
  /** DiffusionGemma block-diffusion runtime contract, null for non-DiffusionGemma models. */
487
517
  diffusionGemma: ManifestDiffusionGemmaSchema | null;
518
+ /** Manifest-owned rerank scoring contract, null when rerank is unsupported. */
519
+ rerank: ManifestRerankSchema | null;
488
520
  /** Layer pipeline override (null = use optimized hardcoded path) */
489
521
  pipeline: LayerPipelineSchema | null;
490
522
  /** Explicit session policy for execution v1 manifests */
@@ -761,6 +793,11 @@ export declare function modelSupportsEmbedding(
761
793
  manifest: Partial<ManifestSchema> | null | undefined
762
794
  ): boolean;
763
795
 
796
+ /** Check if a manifest supports rerank workloads. */
797
+ export declare function modelSupportsRerank(
798
+ manifest: Partial<ManifestSchema> | null | undefined
799
+ ): boolean;
800
+
764
801
  /** Check if a manifest supports audio transcription workloads. */
765
802
  export declare function modelSupportsTranscription(
766
803
  manifest: Partial<ManifestSchema> | null | undefined
@@ -49,6 +49,9 @@ export const DEFAULT_MANIFEST_INFERENCE = {
49
49
  // of modelType: dedicated embedding models (modelType="embedding") always
50
50
  // support embedding workloads; text-generation models opt in explicitly.
51
51
  supportsEmbedding: false,
52
+ // When true, allows workload="rerank" to dispatch through prefillWithLogits()
53
+ // using manifest-owned scoring metadata in inference.rerank.
54
+ supportsRerank: false,
52
55
  // When true, the pipeline exposes pipeline.transcribeAudio({audio,...}) and
53
56
  // pipeline.embedAudio({audio}). Requires audio_token_id and audio encoder
54
57
  // weights to be present in the manifest. Defaults to false; set to true on
@@ -106,6 +109,10 @@ export const DEFAULT_MANIFEST_INFERENCE = {
106
109
  yarnBetaFast: null,
107
110
  yarnBetaSlow: null,
108
111
  yarnOriginalMaxPos: null,
112
+ // LongRoPE parameters - only relevant when ropeScalingType='longrope'
113
+ longropeShortFactor: null,
114
+ longropeLongFactor: null,
115
+ longropeOriginalMaxPos: null,
109
116
  // Local YARN parameters - only relevant when ropeLocalScalingType='yarn'
110
117
  ropeLocalYarnBetaFast: null,
111
118
  ropeLocalYarnBetaSlow: null,
@@ -115,6 +122,8 @@ export const DEFAULT_MANIFEST_INFERENCE = {
115
122
  finalLogitSoftcapping: null, // No softcapping (null = disabled)
116
123
  tieWordEmbeddings: false,
117
124
  scaleEmbeddings: false,
125
+ embeddingScale: null,
126
+ logitInputScale: 1,
118
127
  embeddingTranspose: false,
119
128
  embeddingVocabSize: null,
120
129
  embeddingPostprocessor: null,
@@ -125,12 +134,14 @@ export const DEFAULT_MANIFEST_INFERENCE = {
125
134
  period: null, // No periodic pattern (null = not applicable)
126
135
  offset: null, // For every_n: first global layer index modulo period
127
136
  layerTypes: null, // For custom: explicit per-layer tags
137
+ residualBranchScale: 1,
128
138
  },
129
139
  chatTemplate: {
130
140
  type: null, // No chat template (null = disabled)
131
141
  enabled: false,
132
142
  },
133
143
  diffusionGemma: null,
144
+ rerank: null,
134
145
  pipeline: null,
135
146
  session: null,
136
147
  execution: null,
@@ -190,6 +201,13 @@ export function modelSupportsEmbedding(manifest) {
190
201
  return manifest?.inference?.supportsEmbedding === true;
191
202
  }
192
203
 
204
+ export function modelSupportsRerank(manifest) {
205
+ return manifest?.inference?.supportsRerank === true
206
+ && manifest?.inference?.rerank != null
207
+ && typeof manifest.inference.rerank === 'object'
208
+ && !Array.isArray(manifest.inference.rerank);
209
+ }
210
+
193
211
  // Returns true when the manifest declares audio-input capability. Pipelines
194
212
  // expose transcribeAudio + embedAudio when this is set. The de-facto runtime
195
213
  // gate (audio_token_id + audio encoder weights) still applies; this field is
@@ -17,6 +17,11 @@
17
17
  "policyPath": "tools/policies/model-artifact-contract.json",
18
18
  "schemaPath": "src/config/schema/model-artifact-contract.schema.json"
19
19
  },
20
+ {
21
+ "id": "support-rollout-policy",
22
+ "policyPath": "benchmarks/vendors/support-rollout-policy.json",
23
+ "schemaPath": "benchmarks/vendors/schema/support-rollout-policy.schema.json"
24
+ },
20
25
  {
21
26
  "id": "subsystem-support-contract",
22
27
  "policyPath": "tools/policies/subsystem-support-contract.json",
@@ -161,6 +161,7 @@
161
161
  "ffn": {
162
162
  "activation": "gelu",
163
163
  "gatedActivation": true,
164
+ "branchMode": "auto",
164
165
  "useDoubleWideMlp": false,
165
166
  "swigluLimit": null
166
167
  },
@@ -267,7 +268,25 @@
267
268
  "layout": "contiguous",
268
269
  "pageSize": 256,
269
270
  "tiering": {
270
- "mode": "off"
271
+ "mode": "off",
272
+ "hotWindow": 1024,
273
+ "coldPageSize": 256,
274
+ "coldDtype": "f16",
275
+ "compression": {
276
+ "mode": "none",
277
+ "blockSize": 1,
278
+ "bitWidth": 4,
279
+ "prodMode": false
280
+ },
281
+ "gating": {
282
+ "mode": "auto",
283
+ "minAluBwRatio": 0
284
+ }
285
+ },
286
+ "quantization": {
287
+ "mode": "none",
288
+ "bitWidth": 4,
289
+ "prodMode": false
271
290
  }
272
291
  },
273
292
  "decodeLoop": {
@@ -149,6 +149,7 @@
149
149
  "ffn": {
150
150
  "activation": "gelu",
151
151
  "gatedActivation": true,
152
+ "branchMode": "auto",
152
153
  "useDoubleWideMlp": true,
153
154
  "swigluLimit": null
154
155
  },
@@ -207,7 +208,25 @@
207
208
  "layout": "contiguous",
208
209
  "pageSize": 256,
209
210
  "tiering": {
210
- "mode": "off"
211
+ "mode": "off",
212
+ "hotWindow": 1024,
213
+ "coldPageSize": 256,
214
+ "coldDtype": "f16",
215
+ "compression": {
216
+ "mode": "none",
217
+ "blockSize": 1,
218
+ "bitWidth": 4,
219
+ "prodMode": false
220
+ },
221
+ "gating": {
222
+ "mode": "auto",
223
+ "minAluBwRatio": 0
224
+ }
225
+ },
226
+ "quantization": {
227
+ "mode": "none",
228
+ "bitWidth": 4,
229
+ "prodMode": false
211
230
  }
212
231
  },
213
232
  "decodeLoop": {
@@ -132,6 +132,7 @@
132
132
  },
133
133
  "audio_config": {
134
134
  "model_type": "gemma4_audio",
135
+ "audio_architecture": "gemma4",
135
136
  "num_hidden_layers": 12,
136
137
  "hidden_size": 1024,
137
138
  "num_attention_heads": 8,
@@ -217,6 +218,7 @@
217
218
  "ffn": {
218
219
  "activation": "gelu",
219
220
  "gatedActivation": true,
221
+ "branchMode": "auto",
220
222
  "useDoubleWideMlp": false,
221
223
  "swigluLimit": null
222
224
  },
@@ -275,7 +277,25 @@
275
277
  "layout": "contiguous",
276
278
  "pageSize": 256,
277
279
  "tiering": {
278
- "mode": "off"
280
+ "mode": "off",
281
+ "hotWindow": 1024,
282
+ "coldPageSize": 256,
283
+ "coldDtype": "f16",
284
+ "compression": {
285
+ "mode": "none",
286
+ "blockSize": 1,
287
+ "bitWidth": 4,
288
+ "prodMode": false
289
+ },
290
+ "gating": {
291
+ "mode": "auto",
292
+ "minAluBwRatio": 0
293
+ }
294
+ },
295
+ "quantization": {
296
+ "mode": "none",
297
+ "bitWidth": 4,
298
+ "prodMode": false
279
299
  }
280
300
  },
281
301
  "decodeLoop": {
@@ -38,6 +38,12 @@ export interface ExecutionKernelPrecision {
38
38
  outputDtype?: 'f16' | 'f32';
39
39
  }
40
40
 
41
+ export declare function getKernelFilePrecisionPatch(
42
+ kernel: string
43
+ ): Pick<ExecutionKernelPrecision, 'inputDtype' | 'outputDtype'> | null;
44
+
45
+ export declare function resolveF16ToF32ActivationKernel(kernel: string): string | null;
46
+
41
47
  /**
42
48
  * An execution graph kernel entry from manifest.inference.execution.kernels.
43
49
  */
@@ -49,6 +49,10 @@ const KERNEL_FILE_PRECISION_PATCHES = new Map([
49
49
  ['gather_f16_vec4_f16_out.wgsl', { inputDtype: 'f16', outputDtype: 'f16' }],
50
50
  ]);
51
51
 
52
+ export function getKernelFilePrecisionPatch(kernel) {
53
+ return KERNEL_FILE_PRECISION_PATCHES.get(kernel) ?? null;
54
+ }
55
+
52
56
  /*
53
57
  * Check whether a kernel entry requires subgroup support.
54
58
  */
@@ -686,6 +690,10 @@ const F16_TO_F32_ACTIVATION_MAP = new Map([
686
690
  ['attention_head512_f16.wgsl', 'attention_head512_f16kv.wgsl'],
687
691
  ]);
688
692
 
693
+ export function resolveF16ToF32ActivationKernel(kernel) {
694
+ return F16_TO_F32_ACTIVATION_MAP.get(kernel) ?? null;
695
+ }
696
+
689
697
  /*
690
698
  * Activation-only narrowing: f32-activation shaders that still consume f16
691
699
  * weights/KV are rewritten onto the matching f16-activation lane.
@@ -217,8 +217,10 @@ function resolveConversionPlanV1(options) {
217
217
  layerPattern: inference.layerPattern ?? { type: 'uniform', globalPattern: null, period: null, offset: null, layerTypes: null },
218
218
  chatTemplate: inference.chatTemplate,
219
219
  supportsEmbedding: inference.supportsEmbedding ?? false,
220
+ supportsRerank: inference.supportsRerank ?? false,
220
221
  supportsTranscription: inference.supportsTranscription ?? false,
221
222
  supportsVision: inference.supportsVision ?? false,
223
+ rerank: inference.rerank ?? null,
222
224
  diffusionGemma: inference.diffusionGemma ?? null,
223
225
  pipeline: inference.pipeline ?? null,
224
226
  session,
@@ -54,6 +54,22 @@ function renameDenseTensor(tensor, tensorName) {
54
54
  };
55
55
  }
56
56
 
57
+ function resolveLogitScoreConfig(config, modulePath) {
58
+ const trueTokenId = Number(config?.true_token_id ?? config?.trueTokenId);
59
+ const falseTokenId = Number(config?.false_token_id ?? config?.falseTokenId);
60
+ if (!Number.isInteger(trueTokenId) || trueTokenId < 0) {
61
+ throw new Error(`transformer parser: ${modulePath}/config.json must declare non-negative integer true_token_id.`);
62
+ }
63
+ if (!Number.isInteger(falseTokenId) || falseTokenId < 0) {
64
+ throw new Error(`transformer parser: ${modulePath}/config.json must declare non-negative integer false_token_id.`);
65
+ }
66
+ return {
67
+ kind: 'logit_score',
68
+ trueTokenId,
69
+ falseTokenId,
70
+ };
71
+ }
72
+
57
73
  async function parseSentenceTransformersModules(adapter) {
58
74
  const {
59
75
  readJson,
@@ -69,6 +85,7 @@ async function parseSentenceTransformersModules(adapter) {
69
85
  let normalize = null;
70
86
  const projections = [];
71
87
  const extraTensors = [];
88
+ let rerankScoring = null;
72
89
 
73
90
  for (const moduleEntry of modules) {
74
91
  const type = asNonEmptyString(moduleEntry?.type, 'modules.json type');
@@ -139,6 +156,12 @@ async function parseSentenceTransformersModules(adapter) {
139
156
  continue;
140
157
  }
141
158
 
159
+ if (isSentenceTransformerModuleType(type, '.LogitScore')) {
160
+ const config = await readJson(`${modulePath}/config.json`, `${modulePath}/config.json`);
161
+ rerankScoring = resolveLogitScoreConfig(config, modulePath);
162
+ continue;
163
+ }
164
+
142
165
  throw new Error(`transformer parser: unsupported sentence-transformers module type "${type}".`);
143
166
  }
144
167
 
@@ -146,6 +169,7 @@ async function parseSentenceTransformersModules(adapter) {
146
169
  return {
147
170
  extraTensors: [],
148
171
  embeddingPostprocessor: null,
172
+ rerankScoring,
149
173
  };
150
174
  }
151
175
  if (poolingMode == null) {
@@ -160,6 +184,7 @@ async function parseSentenceTransformersModules(adapter) {
160
184
  projections,
161
185
  normalize,
162
186
  },
187
+ rerankScoring,
163
188
  };
164
189
  }
165
190
 
@@ -178,7 +203,7 @@ export async function parseTransformerModel(adapter) {
178
203
  const architectureHint = config.architectures?.[0] ?? config.model_type ?? '';
179
204
  const sentenceTransformersModules = await fileExists('modules.json')
180
205
  ? await parseSentenceTransformersModules(adapter)
181
- : { extraTensors: [], embeddingPostprocessor: null };
206
+ : { extraTensors: [], embeddingPostprocessor: null, rerankScoring: null };
182
207
 
183
208
  let tensors = null;
184
209
  if (await fileExists('model.safetensors.index.json')) {
@@ -194,5 +219,6 @@ export async function parseTransformerModel(adapter) {
194
219
  tensors: [...tensors, ...sentenceTransformersModules.extraTensors],
195
220
  architectureHint,
196
221
  embeddingPostprocessor: sentenceTransformersModules.embeddingPostprocessor,
222
+ rerankScoring: sentenceTransformersModules.rerankScoring ?? null,
197
223
  };
198
224
  }
@@ -570,7 +570,7 @@ export function resolveEffectiveQuantizationInfo(baseInfo, tensors) {
570
570
  const base = (baseInfo && typeof baseInfo === 'object') ? baseInfo : {};
571
571
  const entries = Array.isArray(tensors) ? tensors : [];
572
572
 
573
- let detectedWeights = null;
573
+ const detectedWeightDtypes = new Set();
574
574
  let detectedEmbeddings = null;
575
575
  let detectedLmHead = null;
576
576
  let detectedLayout = null;
@@ -590,15 +590,27 @@ export function resolveEffectiveQuantizationInfo(baseInfo, tensors) {
590
590
  continue;
591
591
  }
592
592
  if (role === 'matmul' || role === 'expert' || role === 'router') {
593
- detectedWeights = detectedWeights ?? dtype;
593
+ detectedWeightDtypes.add(dtype);
594
594
  if (detectedLayout == null && (dtype === 'q4k') && tensor?.layout) {
595
595
  detectedLayout = normalizeQ4KLayout(tensor.layout);
596
596
  }
597
597
  }
598
598
  }
599
599
 
600
- const weights = detectedWeights
601
- ?? normalizeQuantTag(base.weights ?? 'f16');
600
+ const baseWeights = normalizeQuantTag(base.weights ?? 'f16');
601
+ let weights = baseWeights;
602
+ if (detectedWeightDtypes.size === 1) {
603
+ weights = [...detectedWeightDtypes][0];
604
+ } else if (detectedWeightDtypes.size > 1) {
605
+ if (detectedWeightDtypes.has(baseWeights)) {
606
+ weights = baseWeights;
607
+ } else {
608
+ throw new Error(
609
+ `Ambiguous converted weight dtypes: ${[...detectedWeightDtypes].sort().join(', ')}. ` +
610
+ `Configured quantizationInfo.weights="${baseWeights}" is not present in converted matmul tensors.`
611
+ );
612
+ }
613
+ }
602
614
  const embeddings = detectedEmbeddings
603
615
  ?? normalizeQuantTag(base.embeddings ?? weights);
604
616
  const lmHead = detectedLmHead