doppler-gpu 0.4.5 → 0.4.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (155) hide show
  1. package/README.md +45 -29
  2. package/assets/doppler-runtime-map.svg +82 -0
  3. package/assets/doppler.svg +1 -3
  4. package/package.json +25 -17
  5. package/src/cli/doppler-quickstart.js +1 -1
  6. package/src/cli/doppler-serve.js +2 -2
  7. package/src/client/doppler-registry.d.ts +19 -0
  8. package/src/client/doppler-registry.js +12 -0
  9. package/src/client/doppler-registry.json +50 -1
  10. package/src/client/runtime/model-session.d.ts +1 -0
  11. package/src/client/runtime/model-session.js +9 -0
  12. package/src/config/conversion/README.md +4 -0
  13. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +28 -4
  14. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +29 -4
  15. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +29 -4
  16. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +29 -4
  17. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +29 -4
  18. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +29 -4
  19. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +29 -4
  20. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +29 -4
  21. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +29 -4
  22. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +31 -5
  23. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +30 -4
  24. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +30 -4
  25. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +30 -4
  26. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +30 -5
  27. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +29 -4
  28. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +30 -5
  29. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +29 -4
  30. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +29 -4
  31. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +29 -4
  32. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +29 -4
  33. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +29 -4
  34. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +29 -4
  35. package/src/config/conversion/minicpm/minicpm4-0-5b-f16-af32.json +481 -0
  36. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +29 -4
  37. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +29 -4
  38. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +30 -5
  39. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +29 -4
  40. package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-q4k-ehf16-af32.json +395 -0
  41. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-af32.json +429 -0
  42. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +540 -0
  43. package/src/config/execution-contract-check.js +14 -10
  44. package/src/config/merge-contract-check.js +1 -0
  45. package/src/config/merge.js +30 -0
  46. package/src/config/param-validator.d.ts +5 -0
  47. package/src/config/param-validator.js +42 -8
  48. package/src/config/required-inference-fields-contract-check.js +14 -0
  49. package/src/config/schema/debug.schema.d.ts +2 -0
  50. package/src/config/schema/debug.schema.js +1 -0
  51. package/src/config/schema/doppler.schema.d.ts +5 -1
  52. package/src/config/schema/doppler.schema.js +82 -292
  53. package/src/config/schema/inference.schema.d.ts +17 -3
  54. package/src/config/schema/kvcache.schema.d.ts +19 -3
  55. package/src/config/schema/kvcache.schema.js +2 -1
  56. package/src/config/schema/manifest.schema.d.ts +38 -1
  57. package/src/config/schema/manifest.schema.js +18 -0
  58. package/src/config/schema/policy-schema-registry.json +5 -0
  59. package/src/config/source-packages/litert/gemma-4-12b-it.json +20 -1
  60. package/src/config/source-packages/litert/gemma-4-e2b-it.json +20 -1
  61. package/src/config/source-packages/litert/gemma-4-e4b-it.json +21 -1
  62. package/src/config/transforms/execution-graph-transforms.d.ts +6 -0
  63. package/src/config/transforms/execution-graph-transforms.js +8 -0
  64. package/src/converter/conversion-plan.js +2 -0
  65. package/src/converter/parsers/transformer.js +27 -1
  66. package/src/converter/quantization-info.js +16 -4
  67. package/src/converter/rope-config.js +55 -3
  68. package/src/experimental/training/distillation/student-fixture.js +1 -1
  69. package/src/experimental/training/lora-pipeline.js +1 -1
  70. package/src/experimental/training/lora.js +2 -2
  71. package/src/experimental/training/suite.js +1 -1
  72. package/src/gpu/device.js +54 -11
  73. package/src/gpu/kernels/README.md +8 -1
  74. package/src/gpu/kernels/backward/attention_backward.js +8 -0
  75. package/src/gpu/kernels/dequant.d.ts +6 -6
  76. package/src/gpu/kernels/dequant.js +17 -6
  77. package/src/gpu/kernels/linear-attention-core.js +1 -6
  78. package/src/gpu/kernels/matmul-selection.d.ts +2 -2
  79. package/src/gpu/kernels/matmul-selection.js +9 -2
  80. package/src/gpu/kernels/matmul.d.ts +1 -1
  81. package/src/gpu/kernels/matmul.js +63 -9
  82. package/src/gpu/kernels/moe.d.ts +5 -1
  83. package/src/gpu/kernels/moe.js +5 -2
  84. package/src/gpu/kernels/softmax.d.ts +3 -3
  85. package/src/gpu/kernels/softmax.js +9 -2
  86. package/src/gpu/kernels/turboquant-codebook.js +1 -1
  87. package/src/gpu/kernels/types.d.ts +1 -1
  88. package/src/gpu/profiler.js +11 -1
  89. package/src/inference/browser-harness-model-helpers.js +4 -1
  90. package/src/inference/browser-harness-text-helpers.d.ts +28 -0
  91. package/src/inference/browser-harness-text-helpers.js +357 -116
  92. package/src/inference/browser-harness.d.ts +3 -3
  93. package/src/inference/browser-harness.js +266 -11
  94. package/src/inference/fixtures/rerank-semantic-fixtures.json +38 -0
  95. package/src/inference/kv-cache/quantized.js +16 -2
  96. package/src/inference/kv-cache/tiered.js +50 -15
  97. package/src/inference/moe-router.js +38 -7
  98. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +1 -1
  99. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +8 -6
  100. package/src/inference/pipelines/text/config.d.ts +5 -1
  101. package/src/inference/pipelines/text/config.js +135 -112
  102. package/src/inference/pipelines/text/embed.d.ts +1 -0
  103. package/src/inference/pipelines/text/embed.js +39 -12
  104. package/src/inference/pipelines/text/execution-plan.js +8 -2
  105. package/src/inference/pipelines/text/ffn/dense.js +15 -4
  106. package/src/inference/pipelines/text/ffn/moe.js +4 -3
  107. package/src/inference/pipelines/text/ffn/standard.d.ts +2 -1
  108. package/src/inference/pipelines/text/ffn/standard.js +25 -2
  109. package/src/inference/pipelines/text/generator-helpers.js +2 -1
  110. package/src/inference/pipelines/text/generator-runtime.d.ts +6 -2
  111. package/src/inference/pipelines/text/generator-runtime.js +53 -20
  112. package/src/inference/pipelines/text/generator-steps.js +3 -0
  113. package/src/inference/pipelines/text/generator.d.ts +1 -0
  114. package/src/inference/pipelines/text/generator.js +14 -1
  115. package/src/inference/pipelines/text/init-rope.js +54 -6
  116. package/src/inference/pipelines/text/init.js +63 -15
  117. package/src/inference/pipelines/text/layer.js +32 -5
  118. package/src/inference/pipelines/text/linear-attention.js +12 -8
  119. package/src/inference/pipelines/text/logits/gpu.d.ts +1 -1
  120. package/src/inference/pipelines/text/logits/gpu.js +76 -113
  121. package/src/inference/pipelines/text/logits/index.js +44 -87
  122. package/src/inference/pipelines/text/logits/precision-policy.d.ts +11 -0
  123. package/src/inference/pipelines/text/logits/precision-policy.js +103 -0
  124. package/src/inference/pipelines/text/logits/types.d.ts +1 -0
  125. package/src/inference/pipelines/text/logits/utils.d.ts +2 -0
  126. package/src/inference/pipelines/text/logits/utils.js +11 -0
  127. package/src/inference/pipelines/text/lora-apply.js +6 -4
  128. package/src/inference/pipelines/text/moe-cpu.js +25 -0
  129. package/src/inference/pipelines/text/moe-gpu.js +148 -97
  130. package/src/inference/pipelines/text/moe-shape-validator.d.ts +33 -3
  131. package/src/inference/pipelines/text/moe-shape-validator.js +149 -31
  132. package/src/inference/pipelines/text/ops.d.ts +3 -0
  133. package/src/inference/pipelines/text/ops.js +26 -1
  134. package/src/inference/pipelines/text/per-layer-inputs.js +27 -12
  135. package/src/inference/pipelines/text/sampling.js +52 -10
  136. package/src/inference/pipelines/text/weights.js +7 -2
  137. package/src/inference/pipelines/text.js +24 -4
  138. package/src/inference/tokenizers/behavior-flags.js +23 -2
  139. package/src/loader/final-weights-loader.js +12 -1
  140. package/src/loader/tensors/tensor-loader.js +37 -3
  141. package/src/rules/execution-rules-contract-check.js +12 -3
  142. package/src/rules/inference/capability-transforms.rules.json +14 -2
  143. package/src/rules/inference/config.rules.json +26 -2
  144. package/src/rules/inference/dtype.rules.json +12 -5
  145. package/src/rules/inference/execution.rules.json +5 -19
  146. package/src/rules/inference/moe.rules.json +97 -0
  147. package/src/rules/loader/weights.rules.json +1 -0
  148. package/src/tooling/command-api-constants.js +2 -2
  149. package/src/tooling/command-api-family-normalizers.js +2 -1
  150. package/src/tooling/command-api.d.ts +1 -1
  151. package/src/tooling/command-runner-shared.d.ts +1 -1
  152. package/src/tooling/command-runner-shared.js +3 -1
  153. package/src/tooling/node-converter.js +20 -0
  154. package/src/tooling/node-source-runtime.js +5 -1
  155. package/src/version.js +1 -1
@@ -470,10 +470,14 @@ export async function recordMoEBuildTokenOffsets(recorder, tokenCounts, tokenMap
470
470
 
471
471
  async function executeScatterAddDynamic(recorder, expertOutputs, indices, weights, tokenOffsets, numTokens, hiddenSize, topK, options = {}) {
472
472
  const execution = resolveExecution(recorder);
473
- const { outputBuffer = null, weightsDtype = 'f32', perExpertScale = null } = options;
473
+ const { outputBuffer = null, perExpertScale = null } = options;
474
+ const weightsDtype = options.weightsDtype;
474
475
  const ownsOutput = outputBuffer == null;
475
476
  const hasExpertScale = perExpertScale != null;
476
477
 
478
+ if (weightsDtype !== 'f16' && weightsDtype !== 'f32') {
479
+ throw new Error(`ScatterAddDynamic requires options.weightsDtype to be "f16" or "f32", got ${String(weightsDtype)}.`);
480
+ }
477
481
  if (weightsDtype === 'f16' && expertOutputs.dtype !== 'f16') {
478
482
  throw new Error('ScatterAddDynamic f16 weights require f16 expert outputs');
479
483
  }
@@ -552,4 +556,3 @@ async function executeScatterAddDynamic(recorder, expertOutputs, indices, weight
552
556
  export async function runScatterAddDynamic(expertOutputs, indices, weights, tokenOffsets, numTokens, hiddenSize, topK, options = {}) {
553
557
  return executeScatterAddDynamic(null, expertOutputs, indices, weights, tokenOffsets, numTokens, hiddenSize, topK, options);
554
558
  }
555
-
@@ -21,8 +21,8 @@ export interface SoftmaxOptions extends OutputBufferOptions {
21
21
 
22
22
  export interface SoftmaxTopKOptions {
23
23
  normalize?: boolean;
24
- inputDtype?: 'f16' | 'f32';
25
- weightsDtype?: 'f16' | 'f32';
24
+ inputDtype: 'f16' | 'f32';
25
+ weightsDtype: 'f16' | 'f32';
26
26
  modelType?: string | null;
27
27
  }
28
28
 
@@ -43,7 +43,7 @@ export declare function runSoftmaxTopK(
43
43
  numTokens: number,
44
44
  numExperts: number,
45
45
  topK: number,
46
- options?: SoftmaxTopKOptions
46
+ options: SoftmaxTopKOptions
47
47
  ): Promise<{ indices: GPUBuffer; weights: GPUBuffer }>;
48
48
 
49
49
  /**
@@ -62,10 +62,17 @@ export async function runSoftmaxTopK(logits, numTokens, numExperts, topK, option
62
62
  const device = getDevice();
63
63
  const {
64
64
  normalize = true,
65
- inputDtype = 'f32',
66
- weightsDtype = 'f32',
67
65
  modelType = null,
68
66
  } = options;
67
+ const inputDtype = options.inputDtype;
68
+ const weightsDtype = options.weightsDtype;
69
+
70
+ if (inputDtype !== 'f16' && inputDtype !== 'f32') {
71
+ throw new Error(`SoftmaxTopK requires options.inputDtype to be "f16" or "f32", got ${String(inputDtype)}.`);
72
+ }
73
+ if (weightsDtype !== 'f16' && weightsDtype !== 'f32') {
74
+ throw new Error(`SoftmaxTopK requires options.weightsDtype to be "f16" or "f32", got ${String(weightsDtype)}.`);
75
+ }
69
76
 
70
77
  if (weightsDtype === 'f16' && inputDtype !== 'f16') {
71
78
  throw new Error('SoftmaxTopK f16 weights require f16 logits');
@@ -435,7 +435,7 @@ function retainSharedBuffer(cache, key, factory) {
435
435
 
436
436
  export function retainTurboQuantSharedBuffers(device, options = {}) {
437
437
  const headDim = Number(options.headDim);
438
- const bitWidth = Number(options.bitWidth ?? 4);
438
+ const bitWidth = Number(options.bitWidth);
439
439
  const prodMode = options.prodMode === true;
440
440
  if (!device) {
441
441
  throw new Error('TurboQuant shared buffers require a GPU device.');
@@ -13,7 +13,7 @@ export interface OutputOffsetOptions {
13
13
  }
14
14
 
15
15
  export interface OutputDtypeOptions {
16
- outputDtype?: 'f16' | 'f32';
16
+ outputDtype: 'f16' | 'f32';
17
17
  }
18
18
 
19
19
  export interface Vec4Options {
@@ -65,7 +65,10 @@ export class GPUProfiler {
65
65
  this.#queryCapacity = runtimeProfiler.queryCapacity;
66
66
  this.#maxSamples = runtimeProfiler.maxSamples;
67
67
  this.#maxDurationMs = runtimeProfiler.maxDurationMs;
68
- this.#maxHistoryLabels = runtimeProfiler.maxHistoryLabels ?? 1024;
68
+ this.#maxHistoryLabels = requirePositiveInteger(
69
+ runtimeProfiler.maxHistoryLabels,
70
+ 'runtime.shared.debug.profiler.maxHistoryLabels'
71
+ );
69
72
 
70
73
  // Initialize query resources if timestamp queries available
71
74
  if (this.#hasTimestampQuery && this.#device) {
@@ -368,5 +371,12 @@ export class GPUProfiler {
368
371
  }
369
372
  }
370
373
 
374
+ function requirePositiveInteger(value, label) {
375
+ if (!Number.isInteger(value) || value <= 0) {
376
+ throw new Error(`${label} must be a positive integer.`);
377
+ }
378
+ return value;
379
+ }
380
+
371
381
 
372
382
  export default GPUProfiler;
@@ -99,7 +99,10 @@ export async function resolveKernelPathForModel(options = {}) {
99
99
 
100
100
  if (!manifest) return null;
101
101
 
102
- const modelConfig = parseModelConfigFromManifest(manifest, runtimeConfig);
102
+ const modelConfig = parseModelConfigFromManifest(
103
+ manifest,
104
+ runtimeConfig?.inference?.modelOverrides ?? null
105
+ );
103
106
  const kernelPathState = resolveKernelPathState({
104
107
  manifest,
105
108
  runtimeConfig,
@@ -6,6 +6,11 @@ export declare function getDefaultEmbeddingSemanticFixtures(): {
6
6
  minPairAcc: number;
7
7
  pairMargin: number;
8
8
  };
9
+ export declare function getDefaultRerankSemanticFixtures(): {
10
+ cases: Array<Record<string, unknown>>;
11
+ minPairAcc: number;
12
+ minScoreMargin: number;
13
+ };
9
14
  export declare function resolveBenchmarkRunSettings(
10
15
  runtimeConfig: Record<string, unknown>,
11
16
  source?: Record<string, unknown> | null
@@ -56,6 +61,29 @@ export declare function runEmbeddingSemanticChecks(
56
61
  pipeline: Record<string, unknown>,
57
62
  options?: Record<string, unknown> | null
58
63
  ): Promise<Record<string, unknown>>;
64
+ export declare function resolveRerankScoringConfig(
65
+ pipeline: Record<string, unknown>
66
+ ): Record<string, unknown>;
67
+ export declare function formatRerankPrompt(
68
+ query: string,
69
+ document: string,
70
+ scoringConfig: Record<string, unknown>
71
+ ): string;
72
+ export declare function scoreRerankDocument(
73
+ pipeline: Record<string, unknown>,
74
+ query: string,
75
+ document: string,
76
+ scoringConfig?: Record<string, unknown> | null
77
+ ): Promise<Record<string, unknown>>;
78
+ export declare function runRerank(
79
+ pipeline: Record<string, unknown>,
80
+ runtimeConfig: Record<string, unknown>,
81
+ runOverrides?: Record<string, unknown> | null
82
+ ): Promise<Record<string, unknown>>;
83
+ export declare function runRerankSemanticChecks(
84
+ pipeline: Record<string, unknown>,
85
+ options?: Record<string, unknown> | null
86
+ ): Promise<Record<string, unknown>>;
59
87
  export declare function isCoherentOutput(tokens: Array<unknown>, output: unknown): boolean;
60
88
 
61
89
  export interface ReferenceLogitsDigest {