doppler-gpu 0.4.5 → 0.4.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (155) hide show
  1. package/README.md +45 -29
  2. package/assets/doppler-runtime-map.svg +82 -0
  3. package/assets/doppler.svg +1 -3
  4. package/package.json +25 -17
  5. package/src/cli/doppler-quickstart.js +1 -1
  6. package/src/cli/doppler-serve.js +2 -2
  7. package/src/client/doppler-registry.d.ts +19 -0
  8. package/src/client/doppler-registry.js +12 -0
  9. package/src/client/doppler-registry.json +50 -1
  10. package/src/client/runtime/model-session.d.ts +1 -0
  11. package/src/client/runtime/model-session.js +9 -0
  12. package/src/config/conversion/README.md +4 -0
  13. package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +28 -4
  14. package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +29 -4
  15. package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +29 -4
  16. package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +29 -4
  17. package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +29 -4
  18. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +29 -4
  19. package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +29 -4
  20. package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +29 -4
  21. package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +29 -4
  22. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +31 -5
  23. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +30 -4
  24. package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +30 -4
  25. package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +30 -4
  26. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +30 -5
  27. package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +29 -4
  28. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +30 -5
  29. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +29 -4
  30. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +29 -4
  31. package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +29 -4
  32. package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +29 -4
  33. package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +29 -4
  34. package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +29 -4
  35. package/src/config/conversion/minicpm/minicpm4-0-5b-f16-af32.json +481 -0
  36. package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +29 -4
  37. package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +29 -4
  38. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +30 -5
  39. package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +29 -4
  40. package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-q4k-ehf16-af32.json +395 -0
  41. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-af32.json +429 -0
  42. package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +540 -0
  43. package/src/config/execution-contract-check.js +14 -10
  44. package/src/config/merge-contract-check.js +1 -0
  45. package/src/config/merge.js +30 -0
  46. package/src/config/param-validator.d.ts +5 -0
  47. package/src/config/param-validator.js +42 -8
  48. package/src/config/required-inference-fields-contract-check.js +14 -0
  49. package/src/config/schema/debug.schema.d.ts +2 -0
  50. package/src/config/schema/debug.schema.js +1 -0
  51. package/src/config/schema/doppler.schema.d.ts +5 -1
  52. package/src/config/schema/doppler.schema.js +82 -292
  53. package/src/config/schema/inference.schema.d.ts +17 -3
  54. package/src/config/schema/kvcache.schema.d.ts +19 -3
  55. package/src/config/schema/kvcache.schema.js +2 -1
  56. package/src/config/schema/manifest.schema.d.ts +38 -1
  57. package/src/config/schema/manifest.schema.js +18 -0
  58. package/src/config/schema/policy-schema-registry.json +5 -0
  59. package/src/config/source-packages/litert/gemma-4-12b-it.json +20 -1
  60. package/src/config/source-packages/litert/gemma-4-e2b-it.json +20 -1
  61. package/src/config/source-packages/litert/gemma-4-e4b-it.json +21 -1
  62. package/src/config/transforms/execution-graph-transforms.d.ts +6 -0
  63. package/src/config/transforms/execution-graph-transforms.js +8 -0
  64. package/src/converter/conversion-plan.js +2 -0
  65. package/src/converter/parsers/transformer.js +27 -1
  66. package/src/converter/quantization-info.js +16 -4
  67. package/src/converter/rope-config.js +55 -3
  68. package/src/experimental/training/distillation/student-fixture.js +1 -1
  69. package/src/experimental/training/lora-pipeline.js +1 -1
  70. package/src/experimental/training/lora.js +2 -2
  71. package/src/experimental/training/suite.js +1 -1
  72. package/src/gpu/device.js +54 -11
  73. package/src/gpu/kernels/README.md +8 -1
  74. package/src/gpu/kernels/backward/attention_backward.js +8 -0
  75. package/src/gpu/kernels/dequant.d.ts +6 -6
  76. package/src/gpu/kernels/dequant.js +17 -6
  77. package/src/gpu/kernels/linear-attention-core.js +1 -6
  78. package/src/gpu/kernels/matmul-selection.d.ts +2 -2
  79. package/src/gpu/kernels/matmul-selection.js +9 -2
  80. package/src/gpu/kernels/matmul.d.ts +1 -1
  81. package/src/gpu/kernels/matmul.js +63 -9
  82. package/src/gpu/kernels/moe.d.ts +5 -1
  83. package/src/gpu/kernels/moe.js +5 -2
  84. package/src/gpu/kernels/softmax.d.ts +3 -3
  85. package/src/gpu/kernels/softmax.js +9 -2
  86. package/src/gpu/kernels/turboquant-codebook.js +1 -1
  87. package/src/gpu/kernels/types.d.ts +1 -1
  88. package/src/gpu/profiler.js +11 -1
  89. package/src/inference/browser-harness-model-helpers.js +4 -1
  90. package/src/inference/browser-harness-text-helpers.d.ts +28 -0
  91. package/src/inference/browser-harness-text-helpers.js +357 -116
  92. package/src/inference/browser-harness.d.ts +3 -3
  93. package/src/inference/browser-harness.js +266 -11
  94. package/src/inference/fixtures/rerank-semantic-fixtures.json +38 -0
  95. package/src/inference/kv-cache/quantized.js +16 -2
  96. package/src/inference/kv-cache/tiered.js +50 -15
  97. package/src/inference/moe-router.js +38 -7
  98. package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +1 -1
  99. package/src/inference/pipelines/diffusion/text-encoder-gpu.js +8 -6
  100. package/src/inference/pipelines/text/config.d.ts +5 -1
  101. package/src/inference/pipelines/text/config.js +135 -112
  102. package/src/inference/pipelines/text/embed.d.ts +1 -0
  103. package/src/inference/pipelines/text/embed.js +39 -12
  104. package/src/inference/pipelines/text/execution-plan.js +8 -2
  105. package/src/inference/pipelines/text/ffn/dense.js +15 -4
  106. package/src/inference/pipelines/text/ffn/moe.js +4 -3
  107. package/src/inference/pipelines/text/ffn/standard.d.ts +2 -1
  108. package/src/inference/pipelines/text/ffn/standard.js +25 -2
  109. package/src/inference/pipelines/text/generator-helpers.js +2 -1
  110. package/src/inference/pipelines/text/generator-runtime.d.ts +6 -2
  111. package/src/inference/pipelines/text/generator-runtime.js +53 -20
  112. package/src/inference/pipelines/text/generator-steps.js +3 -0
  113. package/src/inference/pipelines/text/generator.d.ts +1 -0
  114. package/src/inference/pipelines/text/generator.js +14 -1
  115. package/src/inference/pipelines/text/init-rope.js +54 -6
  116. package/src/inference/pipelines/text/init.js +63 -15
  117. package/src/inference/pipelines/text/layer.js +32 -5
  118. package/src/inference/pipelines/text/linear-attention.js +12 -8
  119. package/src/inference/pipelines/text/logits/gpu.d.ts +1 -1
  120. package/src/inference/pipelines/text/logits/gpu.js +76 -113
  121. package/src/inference/pipelines/text/logits/index.js +44 -87
  122. package/src/inference/pipelines/text/logits/precision-policy.d.ts +11 -0
  123. package/src/inference/pipelines/text/logits/precision-policy.js +103 -0
  124. package/src/inference/pipelines/text/logits/types.d.ts +1 -0
  125. package/src/inference/pipelines/text/logits/utils.d.ts +2 -0
  126. package/src/inference/pipelines/text/logits/utils.js +11 -0
  127. package/src/inference/pipelines/text/lora-apply.js +6 -4
  128. package/src/inference/pipelines/text/moe-cpu.js +25 -0
  129. package/src/inference/pipelines/text/moe-gpu.js +148 -97
  130. package/src/inference/pipelines/text/moe-shape-validator.d.ts +33 -3
  131. package/src/inference/pipelines/text/moe-shape-validator.js +149 -31
  132. package/src/inference/pipelines/text/ops.d.ts +3 -0
  133. package/src/inference/pipelines/text/ops.js +26 -1
  134. package/src/inference/pipelines/text/per-layer-inputs.js +27 -12
  135. package/src/inference/pipelines/text/sampling.js +52 -10
  136. package/src/inference/pipelines/text/weights.js +7 -2
  137. package/src/inference/pipelines/text.js +24 -4
  138. package/src/inference/tokenizers/behavior-flags.js +23 -2
  139. package/src/loader/final-weights-loader.js +12 -1
  140. package/src/loader/tensors/tensor-loader.js +37 -3
  141. package/src/rules/execution-rules-contract-check.js +12 -3
  142. package/src/rules/inference/capability-transforms.rules.json +14 -2
  143. package/src/rules/inference/config.rules.json +26 -2
  144. package/src/rules/inference/dtype.rules.json +12 -5
  145. package/src/rules/inference/execution.rules.json +5 -19
  146. package/src/rules/inference/moe.rules.json +97 -0
  147. package/src/rules/loader/weights.rules.json +1 -0
  148. package/src/tooling/command-api-constants.js +2 -2
  149. package/src/tooling/command-api-family-normalizers.js +2 -1
  150. package/src/tooling/command-api.d.ts +1 -1
  151. package/src/tooling/command-runner-shared.d.ts +1 -1
  152. package/src/tooling/command-runner-shared.js +3 -1
  153. package/src/tooling/node-converter.js +20 -0
  154. package/src/tooling/node-source-runtime.js +5 -1
  155. package/src/version.js +1 -1
@@ -11,7 +11,7 @@ import {
11
11
  getActiveKernelPathPolicy,
12
12
  } from '../config/kernel-path-loader.js';
13
13
  import { validateTrainingMetricsReport } from '../config/schema/training-metrics.schema.js';
14
- import { modelSupportsEmbedding } from '../config/schema/manifest.schema.js';
14
+ import { modelSupportsEmbedding, modelSupportsRerank } from '../config/schema/manifest.schema.js';
15
15
  import {
16
16
  resolveReportTimestamp,
17
17
  resolveRuntime,
@@ -54,6 +54,8 @@ import {
54
54
  buildDecodeRecordTopOps,
55
55
  buildDecodeRecordTopOpGroups,
56
56
  runEmbeddingSemanticChecks,
57
+ runRerank,
58
+ runRerankSemanticChecks,
57
59
  isCoherentOutput,
58
60
  runTextInference,
59
61
  runEmbedding,
@@ -115,6 +117,7 @@ const BROWSER_WORKLOAD_SET = Object.freeze([
115
117
  'kernels',
116
118
  'inference',
117
119
  'embedding',
120
+ 'rerank',
118
121
  'training',
119
122
  'diffusion',
120
123
  'energy',
@@ -127,6 +130,7 @@ const BROWSER_WORKLOAD_DISPATCH_MAP = Object.freeze({
127
130
  kernels: 'runKernelSuite',
128
131
  inference: 'runInferenceSuite',
129
132
  embedding: 'runEmbeddingSuite',
133
+ rerank: 'runRerankSuite',
130
134
  training: 'runTrainingSuite',
131
135
  diffusion: 'runDiffusionSuite',
132
136
  energy: 'runEnergySuite',
@@ -141,6 +145,7 @@ const BROWSER_WORKLOAD_DISPATCH_MAP = Object.freeze({
141
145
  bench: Object.freeze({
142
146
  inference: 'runBenchSuite',
143
147
  embedding: 'runBenchSuite',
148
+ rerank: 'runBenchSuite',
144
149
  training: 'runBenchSuite(training)',
145
150
  diffusion: 'runBenchSuite(diffusion)',
146
151
  }),
@@ -530,19 +535,97 @@ async function runInferenceSuite(options = {}) {
530
535
  const runtimeConfig = getRuntimeConfig();
531
536
  const modelType = harness.manifest?.modelType || 'transformer';
532
537
  const supportsEmbedding = modelSupportsEmbedding(harness.manifest);
538
+ const supportsRerank = modelSupportsRerank(harness.manifest);
533
539
  if (options.expectedModelType === 'embedding' && !supportsEmbedding) {
534
540
  throw new Error(
535
541
  `Expected an embedding-capable model for workload "${options.workload || 'inference'}", got modelType="${modelType}". ` +
536
542
  `Set inference.supportsEmbedding=true in the manifest for text-generation models that should expose pipeline.embed().`
537
543
  );
538
544
  }
545
+ if (options.expectedModelType === 'rerank' && !supportsRerank) {
546
+ throw new Error(
547
+ `Expected a rerank-capable model for workload "${options.workload || 'inference'}", got modelType="${modelType}". ` +
548
+ 'Set inference.supportsRerank=true and inference.rerank in the manifest for models that should expose rerank scoring.'
549
+ );
550
+ }
539
551
  const safeModelLoadMs = toTimingNumber(harness.modelLoadMs, 0);
540
552
 
541
553
  let results;
542
554
  let output = null;
543
555
  let metrics;
544
556
 
545
- if (modelType === 'embedding' || (options.workload === 'embedding' && supportsEmbedding)) {
557
+ if (options.workload === 'rerank' && supportsRerank) {
558
+ const run = await runRerank(harness.pipeline, runtimeConfig);
559
+ const semantic = await runRerankSemanticChecks(harness.pipeline, options);
560
+ const allScoresFinite = run.scores.every((entry) => (
561
+ Number.isFinite(entry.score)
562
+ && Number.isFinite(entry.probability)
563
+ && Number.isFinite(entry.trueLogit)
564
+ && Number.isFinite(entry.falseLogit)
565
+ ));
566
+ const hasRanking = Array.isArray(run.ranking) && run.ranking.length === run.documentCount;
567
+ const isValidRerank = allScoresFinite && hasRanking && run.documentCount > 0;
568
+ const isSemanticValid = semantic.passed;
569
+ output = {
570
+ mode: 'rerank',
571
+ query: run.query,
572
+ documentCount: run.documentCount,
573
+ topDocument: run.topDocument,
574
+ ranking: run.ranking,
575
+ semantic: {
576
+ passed: isSemanticValid,
577
+ pairAcc: Number(semantic.pairAcc.toFixed(4)),
578
+ failedCaseIds: semantic.failedCaseIds,
579
+ details: {
580
+ pairs: semantic.pairs,
581
+ },
582
+ },
583
+ };
584
+ results = [
585
+ {
586
+ name: 'rerank',
587
+ passed: isValidRerank,
588
+ duration: run.durationMs,
589
+ error: isValidRerank
590
+ ? undefined
591
+ : 'Rerank scores must be finite and produce a full ranking.',
592
+ },
593
+ {
594
+ name: 'rerank-semantic',
595
+ passed: isSemanticValid,
596
+ duration: semantic.durationMs,
597
+ error: isSemanticValid
598
+ ? undefined
599
+ : (
600
+ `Rerank semantic checks below threshold: pairs=${(semantic.pairAcc * 100).toFixed(1)}% `
601
+ + `(min ${(semantic.minPairAcc * 100).toFixed(1)}%). `
602
+ + (semantic.failedCaseIds.length > 0 ? `Failed: ${semantic.failedCaseIds.join(', ')}` : '')
603
+ ),
604
+ },
605
+ ];
606
+ metrics = {
607
+ query: run.query,
608
+ documentCount: run.documentCount,
609
+ topDocumentIndex: run.topDocument?.index ?? null,
610
+ topDocumentScore: run.topDocument?.score == null ? null : Number(run.topDocument.score.toFixed(6)),
611
+ topDocumentProbability: run.topDocument?.probability == null ? null : Number(run.topDocument.probability.toFixed(6)),
612
+ rerankMs: Number(run.durationMs.toFixed(2)),
613
+ rerankRanking: run.ranking,
614
+ semanticPassed: isSemanticValid,
615
+ semanticDurationMs: Number(semantic.durationMs.toFixed(2)),
616
+ semanticPairAcc: Number(semantic.pairAcc.toFixed(4)),
617
+ semanticPairPassed: semantic.pairPassed,
618
+ semanticPairTotal: semantic.pairTotal,
619
+ semanticMinPairAcc: Number(semantic.minPairAcc.toFixed(4)),
620
+ semanticMinScoreMargin: Number(semantic.minScoreMargin.toFixed(4)),
621
+ semanticFailedCases: semantic.failedCaseIds,
622
+ semanticDetails: {
623
+ pairs: semantic.pairs,
624
+ },
625
+ modelLoadMs: safeModelLoadMs,
626
+ endToEndMs: safeToFixed(safeModelLoadMs + run.durationMs),
627
+ };
628
+ } else if (modelType === 'embedding' || (options.workload === 'embedding' && supportsEmbedding)) {
546
629
  const run = await runEmbedding(harness.pipeline, runtimeConfig);
547
630
  const semantic = await runEmbeddingSemanticChecks(harness.pipeline, options);
548
631
  const isValidEmbedding = run.embeddingDim > 0 && run.nonFiniteCount === 0;
@@ -772,12 +855,20 @@ async function runInferenceSuite(options = {}) {
772
855
  };
773
856
  }
774
857
 
858
+ function resolveBenchmarkIterationSettings(runtimeConfig) {
859
+ const benchConfig = runtimeConfig?.shared?.benchmark?.run || {};
860
+ return {
861
+ warmupRuns: Math.max(0, Math.floor(benchConfig.warmupRuns ?? 0)),
862
+ timedRuns: Math.max(1, Math.floor(benchConfig.timedRuns ?? 1)),
863
+ };
864
+ }
865
+
775
866
  async function runBenchSuite(options = {}) {
776
867
  const startTime = performance.now();
777
868
  const runtimeConfig = getRuntimeConfig();
778
- const defaultBenchRun = resolveBenchmarkRunSettings(runtimeConfig);
779
- const warmupRuns = defaultBenchRun.warmupRuns;
780
- const timedRuns = defaultBenchRun.timedRuns;
869
+ const iterationSettings = resolveBenchmarkIterationSettings(runtimeConfig);
870
+ const warmupRuns = iterationSettings.warmupRuns;
871
+ const timedRuns = iterationSettings.timedRuns;
781
872
  const cacheMode = normalizeCacheMode(options.cacheMode);
782
873
  const loadMode = normalizeLoadMode(options.loadMode, !!options.modelUrl, options.modelUrl);
783
874
  const workloadType = normalizeWorkloadType(
@@ -792,7 +883,7 @@ async function runBenchSuite(options = {}) {
792
883
  if (workloadType === 'training') {
793
884
  const trainingBench = await runTrainingBenchSuite({
794
885
  ...options,
795
- benchRun: defaultBenchRun,
886
+ benchRun: iterationSettings,
796
887
  workloadType,
797
888
  });
798
889
  const trainingReport = trainingBench?.metrics?.trainingMetricsReport;
@@ -888,15 +979,24 @@ async function runBenchSuite(options = {}) {
888
979
  },
889
980
  () => initializeSuiteModel(options)
890
981
  );
891
- const benchRun = resolveBenchmarkRunSettings(runtimeConfig, harness.pipeline ?? harness);
982
+ const benchRun = options.workload === 'rerank'
983
+ ? iterationSettings
984
+ : resolveBenchmarkRunSettings(runtimeConfig, harness.pipeline ?? harness);
892
985
  const modelType = harness.manifest?.modelType || 'transformer';
893
986
  const supportsEmbedding = modelSupportsEmbedding(harness.manifest);
987
+ const supportsRerank = modelSupportsRerank(harness.manifest);
894
988
  if (options.expectedModelType === 'embedding' && !supportsEmbedding) {
895
989
  throw new Error(
896
990
  `Expected an embedding-capable model for bench workload "${options.workload || 'inference'}", got modelType="${modelType}". ` +
897
991
  `Set inference.supportsEmbedding=true in the manifest for text-generation models that should expose pipeline.embed().`
898
992
  );
899
993
  }
994
+ if (options.expectedModelType === 'rerank' && !supportsRerank) {
995
+ throw new Error(
996
+ `Expected a rerank-capable model for bench workload "${options.workload || 'inference'}", got modelType="${modelType}". ` +
997
+ 'Set inference.supportsRerank=true and inference.rerank in the manifest for models that should expose rerank scoring.'
998
+ );
999
+ }
900
1000
  const safeModelLoadMs = toTimingNumber(harness.modelLoadMs, 0);
901
1001
 
902
1002
  let results;
@@ -904,7 +1004,155 @@ async function runBenchSuite(options = {}) {
904
1004
  let output = null;
905
1005
  let timing;
906
1006
 
907
- if (modelType === 'embedding' || (options.workload === 'embedding' && supportsEmbedding)) {
1007
+ if (options.workload === 'rerank' && supportsRerank) {
1008
+ const durations = [];
1009
+ const timedDurations = [];
1010
+ const documentCounts = [];
1011
+ const topDocumentScores = [];
1012
+ const topDocumentProbabilities = [];
1013
+ let invalidRuns = 0;
1014
+ let nonFiniteScores = 0;
1015
+ let lastRun = null;
1016
+
1017
+ for (let i = 0; i < warmupRuns + timedRuns; i++) {
1018
+ harness.pipeline.reset?.();
1019
+ const run = await runRerank(harness.pipeline, runtimeConfig, benchRun);
1020
+ if (i >= warmupRuns) {
1021
+ timedDurations.push(run.durationMs);
1022
+ const finiteScores = run.scores.filter((entry) => (
1023
+ Number.isFinite(entry.score)
1024
+ && Number.isFinite(entry.probability)
1025
+ && Number.isFinite(entry.trueLogit)
1026
+ && Number.isFinite(entry.falseLogit)
1027
+ ));
1028
+ nonFiniteScores += run.scores.length - finiteScores.length;
1029
+ const hasRanking = Array.isArray(run.ranking) && run.ranking.length === run.documentCount;
1030
+ if (finiteScores.length === run.scores.length && hasRanking && run.documentCount > 0) {
1031
+ durations.push(run.durationMs);
1032
+ documentCounts.push(run.documentCount);
1033
+ if (Number.isFinite(run.topDocument?.score)) {
1034
+ topDocumentScores.push(run.topDocument.score);
1035
+ }
1036
+ if (Number.isFinite(run.topDocument?.probability)) {
1037
+ topDocumentProbabilities.push(run.topDocument.probability);
1038
+ }
1039
+ } else {
1040
+ invalidRuns++;
1041
+ }
1042
+ lastRun = run;
1043
+ }
1044
+ }
1045
+
1046
+ const semantic = await runRerankSemanticChecks(harness.pipeline, options);
1047
+ const rerankMsStats = computeSampleStats(durations);
1048
+ const timedRerankMsStats = computeSampleStats(timedDurations);
1049
+ const documentCountStats = computeSampleStats(documentCounts);
1050
+ const topScoreStats = computeSampleStats(topDocumentScores);
1051
+ const topProbabilityStats = computeSampleStats(topDocumentProbabilities);
1052
+ const avgMs = rerankMsStats.mean;
1053
+ const semanticPassed = semantic.passed;
1054
+
1055
+ results = [
1056
+ {
1057
+ name: 'benchmark-rerank',
1058
+ passed: durations.length > 0 && invalidRuns === 0,
1059
+ duration: durations.reduce((sum, value) => sum + value, 0),
1060
+ error: durations.length > 0
1061
+ ? (
1062
+ invalidRuns === 0
1063
+ ? undefined
1064
+ : `Invalid rerank runs: ${invalidRuns} (non-finite scores or incomplete ranking observed)`
1065
+ )
1066
+ : 'No valid rerank benchmark runs completed',
1067
+ },
1068
+ {
1069
+ name: 'benchmark-rerank-semantic',
1070
+ passed: semanticPassed,
1071
+ duration: semantic.durationMs,
1072
+ error: semanticPassed
1073
+ ? undefined
1074
+ : (
1075
+ `Rerank semantic checks below threshold: pairs=${(semantic.pairAcc * 100).toFixed(1)}% `
1076
+ + `(min ${(semantic.minPairAcc * 100).toFixed(1)}%). `
1077
+ + (semantic.failedCaseIds.length > 0 ? `Failed: ${semantic.failedCaseIds.join(', ')}` : '')
1078
+ ),
1079
+ },
1080
+ ];
1081
+
1082
+ output = {
1083
+ mode: 'rerank',
1084
+ query: lastRun?.query ?? null,
1085
+ documentCount: lastRun?.documentCount ?? null,
1086
+ topDocument: lastRun?.topDocument ?? null,
1087
+ ranking: lastRun?.ranking ?? [],
1088
+ semantic: {
1089
+ passed: semanticPassed,
1090
+ pairAcc: Number(semantic.pairAcc.toFixed(4)),
1091
+ failedCaseIds: semantic.failedCaseIds,
1092
+ details: {
1093
+ pairs: semantic.pairs,
1094
+ },
1095
+ },
1096
+ };
1097
+
1098
+ metrics = {
1099
+ warmupRuns,
1100
+ timedRuns,
1101
+ validRuns: durations.length,
1102
+ invalidRuns,
1103
+ invalidRatePct: Number((timedRuns > 0 ? (invalidRuns / timedRuns) * 100 : 0).toFixed(2)),
1104
+ query: lastRun?.query ?? null,
1105
+ documentCount: Math.round(documentCountStats.mean),
1106
+ topDocumentIndex: lastRun?.topDocument?.index ?? null,
1107
+ topDocumentScore: lastRun?.topDocument?.score == null ? null : Number(lastRun.topDocument.score.toFixed(6)),
1108
+ topDocumentProbability: lastRun?.topDocument?.probability == null ? null : Number(lastRun.topDocument.probability.toFixed(6)),
1109
+ topDocumentScoreStats: topScoreStats,
1110
+ topDocumentProbabilityStats: topProbabilityStats,
1111
+ nonFiniteScores,
1112
+ firstTimedRerankMs: Number((timedDurations[0] ?? 0).toFixed(2)),
1113
+ minRerankMs: Number(rerankMsStats.min.toFixed(2)),
1114
+ medianRerankMs: Number(rerankMsStats.median.toFixed(2)),
1115
+ p95RerankMs: Number(rerankMsStats.p95.toFixed(2)),
1116
+ p99RerankMs: Number(rerankMsStats.p99.toFixed(2)),
1117
+ maxRerankMs: Number(rerankMsStats.max.toFixed(2)),
1118
+ stdDevRerankMs: Number(rerankMsStats.stdDev.toFixed(2)),
1119
+ ci95RerankMs: Number(rerankMsStats.ci95.toFixed(2)),
1120
+ avgRerankMs: Number(avgMs.toFixed(2)),
1121
+ avgReranksPerSec: Number((avgMs > 0 ? (1000 / avgMs) : 0).toFixed(2)),
1122
+ semanticPassed,
1123
+ semanticDurationMs: Number(semantic.durationMs.toFixed(2)),
1124
+ semanticPairAcc: Number(semantic.pairAcc.toFixed(4)),
1125
+ semanticPairPassed: semantic.pairPassed,
1126
+ semanticPairTotal: semantic.pairTotal,
1127
+ semanticMinPairAcc: Number(semantic.minPairAcc.toFixed(4)),
1128
+ semanticMinScoreMargin: Number(semantic.minScoreMargin.toFixed(4)),
1129
+ semanticFailedCases: semantic.failedCaseIds,
1130
+ semanticDetails: {
1131
+ pairs: semantic.pairs,
1132
+ },
1133
+ modelLoadMs: safeModelLoadMs,
1134
+ latency: {
1135
+ timedRerankMs: timedRerankMsStats,
1136
+ rerankMs: rerankMsStats,
1137
+ },
1138
+ documents: {
1139
+ count: documentCountStats,
1140
+ },
1141
+ };
1142
+
1143
+ timing = buildCanonicalTiming({
1144
+ modelLoadMs: safeModelLoadMs,
1145
+ firstTokenMs: null,
1146
+ firstResponseMs: Number.isFinite(timedDurations[0])
1147
+ ? safeModelLoadMs + timedDurations[0]
1148
+ : null,
1149
+ prefillMs: null,
1150
+ decodeMs: null,
1151
+ totalRunMs: rerankMsStats.median,
1152
+ cacheMode,
1153
+ loadMode,
1154
+ });
1155
+ } else if (modelType === 'embedding' || (options.workload === 'embedding' && supportsEmbedding)) {
908
1156
  const durations = [];
909
1157
  const timedDurations = [];
910
1158
  const embeddingDims = [];
@@ -1468,6 +1716,9 @@ async function dispatchBrowserSuite(mode, workload, options) {
1468
1716
  if (mode === 'verify' && workload === 'kernels') {
1469
1717
  return runKernelSuite(options);
1470
1718
  }
1719
+ if (mode === 'bench') {
1720
+ return runBenchSuite(options);
1721
+ }
1471
1722
  if (workload === 'embedding') {
1472
1723
  return runInferenceSuite({
1473
1724
  ...options,
@@ -1475,12 +1726,16 @@ async function dispatchBrowserSuite(mode, workload, options) {
1475
1726
  expectedModelType: options.expectedModelType ?? 'embedding',
1476
1727
  });
1477
1728
  }
1729
+ if (workload === 'rerank') {
1730
+ return runInferenceSuite({
1731
+ ...options,
1732
+ suiteName: 'rerank',
1733
+ expectedModelType: options.expectedModelType ?? 'rerank',
1734
+ });
1735
+ }
1478
1736
  if (mode === 'verify' && workload === 'training') {
1479
1737
  return runTrainingSuite(options);
1480
1738
  }
1481
- if (mode === 'bench') {
1482
- return runBenchSuite(options);
1483
- }
1484
1739
  if (mode === 'verify' && workload === 'diffusion') {
1485
1740
  return runDiffusionSuite(options);
1486
1741
  }
@@ -0,0 +1,38 @@
1
+ {
2
+ "defaults": {
3
+ "minPairAcc": 0.8,
4
+ "minScoreMargin": 0,
5
+ "cases": [
6
+ {
7
+ "id": "china_capital",
8
+ "query": "What is the capital of China?",
9
+ "positive": "The capital of China is Beijing.",
10
+ "negative": "Gravity is a force that attracts two bodies towards each other."
11
+ },
12
+ {
13
+ "id": "password_reset",
14
+ "query": "How do I reset my account password?",
15
+ "positive": "To reset your password, open account settings and choose the forgot-password flow.",
16
+ "negative": "The recipe combines tomatoes, basil, and olive oil."
17
+ },
18
+ {
19
+ "id": "python_sort",
20
+ "query": "How do I sort a list in Python?",
21
+ "positive": "Use sorted() or list.sort() to sort a list in Python.",
22
+ "negative": "The cafe serves coffee and pastries."
23
+ },
24
+ {
25
+ "id": "webgpu_compute",
26
+ "query": "What API exposes GPU hardware for browser compute workloads?",
27
+ "positive": "WebGPU provides low-level access to graphics processors for compute workloads.",
28
+ "negative": "WebSocket provides full-duplex communication over TCP."
29
+ },
30
+ {
31
+ "id": "vaccines",
32
+ "query": "How do vaccines work?",
33
+ "positive": "Vaccines train the immune system to recognize future infections.",
34
+ "negative": "The symphony orchestra has strings, woodwinds, brass, and percussion."
35
+ }
36
+ ]
37
+ }
38
+ }
@@ -25,9 +25,9 @@ export class QuantizedKVCache {
25
25
  this.maxSeqLen = config.maxSeqLen;
26
26
  this.useGPU = config.useGPU;
27
27
  this.layout = 'contiguous_quantized';
28
- this.kvDtype = config.kvDtype ?? 'f16';
28
+ this.kvDtype = requireString(config.kvDtype, 'QuantizedKVCache config.kvDtype');
29
29
  this.quantMode = config.quantMode;
30
- this.bitWidth = config.bitWidth ?? 4;
30
+ this.bitWidth = requirePositiveInteger(config.bitWidth, 'QuantizedKVCache config.bitWidth');
31
31
  this.prodMode = config.prodMode === true;
32
32
  this.currentSeqLen = 0;
33
33
  this.totalTokensSeen = 0;
@@ -383,3 +383,17 @@ export class QuantizedKVCache {
383
383
  }
384
384
  }
385
385
  }
386
+
387
+ function requireString(value, label) {
388
+ if (typeof value !== 'string' || value.length === 0) {
389
+ throw new Error(`${label} is required.`);
390
+ }
391
+ return value;
392
+ }
393
+
394
+ function requirePositiveInteger(value, label) {
395
+ if (!Number.isInteger(value) || value <= 0) {
396
+ throw new Error(`${label} must be a positive integer.`);
397
+ }
398
+ return value;
399
+ }
@@ -59,7 +59,7 @@ export class TieredKVCache {
59
59
 
60
60
  this.layout = 'tiered';
61
61
 
62
- this.kvDtype = config.kvDtype;
62
+ this.kvDtype = requireNonEmptyString(config.kvDtype, 'TieredKVCache config.kvDtype');
63
63
 
64
64
  this.bytesPerElem = this.kvDtype === 'f16' ? 2 : 4;
65
65
 
@@ -69,17 +69,19 @@ export class TieredKVCache {
69
69
 
70
70
  this.coldPageSize = tiering.coldPageSize;
71
71
 
72
- this.coldDtype = tiering.coldDtype ?? this.kvDtype;
72
+ this.coldDtype = requireNonEmptyString(tiering.coldDtype, 'TieredKVCache tiering.coldDtype');
73
73
 
74
- this.tieringMode = tiering.mode;
74
+ this.tieringMode = requireNonEmptyString(tiering.mode, 'TieredKVCache tiering.mode');
75
75
 
76
- const turboQuantModes = ['turboquant', 'turboquant_prod'];
77
- const defaultCompressionMode = turboQuantModes.includes(tiering.mode)
78
- ? tiering.mode
79
- : (tiering.mode === 'int8' ? 'int8' : (tiering.mode === 'int4' ? 'int4' : 'none'));
80
- this.compression = tiering.compression ?? { mode: defaultCompressionMode, blockSize: 1 };
76
+ this.compression = requirePlainObject(
77
+ tiering.compression,
78
+ 'TieredKVCache tiering.compression'
79
+ );
81
80
 
82
- this.gating = tiering.gating ?? { mode: 'force_off', minAluBwRatio: 0.0 };
81
+ this.gating = requirePlainObject(
82
+ tiering.gating,
83
+ 'TieredKVCache tiering.gating'
84
+ );
83
85
 
84
86
  this.currentSeqLen = 0;
85
87
 
@@ -102,7 +104,10 @@ export class TieredKVCache {
102
104
  }
103
105
 
104
106
  assertSupportedTurboQuantMode(this.tieringMode, 'tiering.mode');
105
- assertSupportedTurboQuantMode(this.compression?.mode ?? 'none', 'tiering.compression.mode');
107
+ assertSupportedTurboQuantMode(
108
+ requireNonEmptyString(this.compression.mode, 'TieredKVCache tiering.compression.mode'),
109
+ 'tiering.compression.mode'
110
+ );
106
111
  this.coldQuantMode = this._resolveCompressionMode(this.compression, this.gating);
107
112
  assertSupportedTurboQuantMode(this.coldQuantMode, 'resolved cold quant mode');
108
113
  if (this.coldQuantMode !== 'none' && this.compression.blockSize !== 1) {
@@ -111,7 +116,9 @@ export class TieredKVCache {
111
116
 
112
117
  this.isTurboQuant = isTurboQuantMode(this.coldQuantMode);
113
118
  this.isProdMode = this.coldQuantMode === 'turboquant_prod';
114
- this.turboQuantBitWidth = this.compression.bitWidth ?? 4;
119
+ this.turboQuantBitWidth = this.isTurboQuant
120
+ ? requirePositiveInteger(this.compression.bitWidth, 'TieredKVCache tiering.compression.bitWidth')
121
+ : null;
115
122
 
116
123
  if (this.isTurboQuant) {
117
124
  this.coldPackedStride = computePackedStride(this.headDim, this.turboQuantBitWidth);
@@ -184,10 +191,17 @@ export class TieredKVCache {
184
191
 
185
192
 
186
193
  _resolveCompressionMode(compression, gating) {
187
- const requested = compression?.mode ?? 'none';
188
- if (gating?.mode === 'force_off') return 'none';
189
- if (gating?.mode === 'force_on') return requested;
190
- if (gating?.mode === 'auto' && gating.minAluBwRatio > 0) {
194
+ const requested = requireNonEmptyString(
195
+ compression?.mode,
196
+ 'TieredKVCache tiering.compression.mode'
197
+ );
198
+ const gatingMode = requireNonEmptyString(
199
+ gating?.mode,
200
+ 'TieredKVCache tiering.gating.mode'
201
+ );
202
+ if (gatingMode === 'force_off') return 'none';
203
+ if (gatingMode === 'force_on') return requested;
204
+ if (gatingMode === 'auto' && gating.minAluBwRatio > 0) {
191
205
  throw new Error(
192
206
  'TieredKVCache auto compression gating requires an explicit measured ALU/BW ratio. ' +
193
207
  'Use gating.mode="force_on"/"force_off" or set minAluBwRatio to 0.'
@@ -719,3 +733,24 @@ export class TieredKVCache {
719
733
  return cloned;
720
734
  }
721
735
  }
736
+
737
+ function requirePlainObject(value, label) {
738
+ if (value == null || typeof value !== 'object' || Array.isArray(value)) {
739
+ throw new Error(`${label} is required.`);
740
+ }
741
+ return value;
742
+ }
743
+
744
+ function requireNonEmptyString(value, label) {
745
+ if (typeof value !== 'string' || value.trim().length === 0) {
746
+ throw new Error(`${label} is required.`);
747
+ }
748
+ return value.trim();
749
+ }
750
+
751
+ function requirePositiveInteger(value, label) {
752
+ if (!Number.isInteger(value) || value <= 0) {
753
+ throw new Error(`${label} must be a positive integer.`);
754
+ }
755
+ return value;
756
+ }
@@ -16,6 +16,25 @@ function isRouterVector(value) {
16
16
  return value instanceof Float32Array || isGpuBufferInstance(value) || isWeightBuffer(value);
17
17
  }
18
18
 
19
+ function requireRouterDtype(value, label) {
20
+ if (value !== 'f16' && value !== 'f32') {
21
+ throw new Error(`MoERouter requires ${label} to be "f16" or "f32", got ${value}.`);
22
+ }
23
+ return value;
24
+ }
25
+
26
+ function inferRouterBufferDtypeFromElementCount(buffer, elementCount, label) {
27
+ const byteLength = Number.isFinite(buffer?.size) ? buffer.size : buffer?.byteLength;
28
+ if (!Number.isFinite(byteLength)) {
29
+ throw new Error(`MoERouter requires ${label} byte size to resolve dtype.`);
30
+ }
31
+ const bytesPerElement = byteLength / elementCount;
32
+ if (bytesPerElement !== 2 && bytesPerElement !== 4) {
33
+ throw new Error(`MoERouter cannot infer ${label} dtype from bytesPerElement=${bytesPerElement}.`);
34
+ }
35
+ return selectRuleValue('inference', 'dtype', 'f16OrF32FromBytes', { bytesPerElement });
36
+ }
37
+
19
38
 
20
39
 
21
40
 
@@ -61,8 +80,10 @@ export class MoERouter {
61
80
 
62
81
  _gateWeightGPU = null;
63
82
 
83
+ _gateWeightDtype = null;
84
+
64
85
 
65
- lastLogitsDtype = 'f32';
86
+ lastLogitsDtype = null;
66
87
 
67
88
 
68
89
  constructor(config) {
@@ -122,6 +143,9 @@ export class MoERouter {
122
143
  if (this._gateWeightGPU) {
123
144
  this._gateWeightGPU.destroy();
124
145
  }
146
+ this._gateWeightDtype = isWeightBuffer(weights)
147
+ ? getWeightDtype(weights)
148
+ : inferRouterBufferDtypeFromElementCount(weights, this.numExperts * this.hiddenSize, 'gate weight');
125
149
  this.gateWeight = weights;
126
150
  this.gateBias = bias;
127
151
  this.gateScale = scale;
@@ -140,6 +164,7 @@ export class MoERouter {
140
164
  }
141
165
  this._gateBiasGPU = null;
142
166
  this._gateWeightGPU = null;
167
+ this._gateWeightDtype = null;
143
168
  this.gateWeight = null;
144
169
  this.gateBias = null;
145
170
  this.gateScale = null;
@@ -197,6 +222,8 @@ export class MoERouter {
197
222
  throw new Error('Router gate weights not loaded');
198
223
  }
199
224
  if (!isWeightBuffer(gateWeightBuffer) && !isGpuBufferInstance(gateWeightBuffer)) {
225
+ const sourceWeightDtype = this._gateWeightDtype
226
+ ?? inferRouterBufferDtypeFromElementCount(gateWeightBuffer, this.numExperts * this.hiddenSize, 'gate weight');
200
227
  const uploaded = device.createBuffer({
201
228
  label: 'moe_gate_weight',
202
229
  size: gateWeightBuffer.byteLength,
@@ -209,12 +236,16 @@ export class MoERouter {
209
236
  throw error;
210
237
  }
211
238
  this._gateWeightGPU = uploaded;
239
+ this._gateWeightDtype = sourceWeightDtype;
212
240
  this.gateWeight = uploaded;
213
241
  gateWeightBuffer = uploaded;
214
242
  }
215
243
 
216
- const { inputDtype = 'f32', outputDtype = 'f32' } = options;
217
- const gateWeightDtype = isWeightBuffer(gateWeightBuffer) ? getWeightDtype(gateWeightBuffer) : null;
244
+ const inputDtype = requireRouterDtype(options.inputDtype, 'options.inputDtype');
245
+ const outputDtype = requireRouterDtype(options.outputDtype, 'options.outputDtype');
246
+ const gateWeightDtype = isWeightBuffer(gateWeightBuffer)
247
+ ? getWeightDtype(gateWeightBuffer)
248
+ : this._gateWeightDtype;
218
249
 
219
250
  // Matrix multiply: hidden_states [numTokens, hiddenSize] @ gate_weight [hiddenSize, numExperts]
220
251
  // Result: [numTokens, numExperts]
@@ -274,9 +305,9 @@ export class MoERouter {
274
305
  if (bias instanceof Float32Array) return 'f32';
275
306
  if (isGpuBufferInstance(bias)) {
276
307
  const bytesPerElement = Math.round(bias.size / this.numExperts);
277
- return selectRuleValue('inference', 'dtype', 'f16OrF32FromBytes', { bytesPerElement });
308
+ return inferRouterBufferDtypeFromElementCount(bias, this.numExperts, 'gate bias');
278
309
  }
279
- return 'f32';
310
+ throw new Error('MoERouter cannot infer gate bias dtype.');
280
311
  }
281
312
 
282
313
  _getBiasAddPipeline(logitsDtype, biasDtype, device) {
@@ -368,9 +399,9 @@ export class MoERouter {
368
399
  }
369
400
 
370
401
 
371
- async routeGPU(hiddenStates, numTokens) {
402
+ async routeGPU(hiddenStates, numTokens, options = {}) {
372
403
  // Compute router logits on GPU
373
- const logitsBuffer = await this.computeRouterLogitsGPU(hiddenStates, numTokens);
404
+ const logitsBuffer = await this.computeRouterLogitsGPU(hiddenStates, numTokens, null, options);
374
405
  try {
375
406
  const logitsData = await readBuffer(logitsBuffer);
376
407
  const logits = this.lastLogitsDtype === 'f16'
@@ -79,4 +79,4 @@ export declare function projectContext(
79
79
  options?: { recorder?: CommandRecorder | null }
80
80
  ): Promise<Tensor>;
81
81
 
82
- export declare function assertClipHiddenActivationSupported(config: { hidden_act?: string }): void;
82
+ export declare function assertClipHiddenActivationSupported(config: { hidden_act: string }): void;