doppler-gpu 0.4.6 → 0.4.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +43 -31
- package/assets/doppler-runtime-map.svg +1 -3
- package/assets/doppler.svg +1 -3
- package/package.json +25 -17
- package/src/cli/doppler-quickstart.js +1 -1
- package/src/cli/doppler-serve.js +2 -2
- package/src/client/doppler-registry.d.ts +19 -0
- package/src/client/doppler-registry.js +12 -0
- package/src/client/doppler-registry.json +50 -1
- package/src/config/conversion/README.md +4 -0
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +28 -4
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +31 -5
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +30 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +30 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +30 -4
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +30 -5
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +30 -5
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +29 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +29 -4
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +29 -4
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/minicpm/minicpm4-0-5b-f16-af32.json +481 -0
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +29 -4
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +29 -4
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +30 -5
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +29 -4
- package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-q4k-ehf16-af32.json +395 -0
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-af32.json +429 -0
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +540 -0
- package/src/config/execution-contract-check.js +14 -10
- package/src/config/merge-contract-check.js +1 -0
- package/src/config/merge.js +30 -0
- package/src/config/param-validator.d.ts +5 -0
- package/src/config/param-validator.js +42 -8
- package/src/config/required-inference-fields-contract-check.js +14 -0
- package/src/config/schema/debug.schema.d.ts +2 -0
- package/src/config/schema/debug.schema.js +1 -0
- package/src/config/schema/doppler.schema.d.ts +5 -1
- package/src/config/schema/doppler.schema.js +82 -292
- package/src/config/schema/inference.schema.d.ts +17 -3
- package/src/config/schema/kvcache.schema.d.ts +19 -3
- package/src/config/schema/kvcache.schema.js +2 -1
- package/src/config/schema/manifest.schema.d.ts +38 -1
- package/src/config/schema/manifest.schema.js +18 -0
- package/src/config/schema/policy-schema-registry.json +5 -0
- package/src/config/source-packages/litert/gemma-4-12b-it.json +20 -1
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +20 -1
- package/src/config/source-packages/litert/gemma-4-e4b-it.json +21 -1
- package/src/config/transforms/execution-graph-transforms.d.ts +6 -0
- package/src/config/transforms/execution-graph-transforms.js +8 -0
- package/src/converter/conversion-plan.js +2 -0
- package/src/converter/parsers/transformer.js +27 -1
- package/src/converter/quantization-info.js +16 -4
- package/src/converter/rope-config.js +55 -3
- package/src/experimental/training/distillation/student-fixture.js +1 -1
- package/src/experimental/training/lora-pipeline.js +1 -1
- package/src/experimental/training/lora.js +2 -2
- package/src/experimental/training/suite.js +1 -1
- package/src/gpu/kernels/README.md +8 -1
- package/src/gpu/kernels/backward/attention_backward.js +8 -0
- package/src/gpu/kernels/dequant.d.ts +6 -6
- package/src/gpu/kernels/dequant.js +17 -6
- package/src/gpu/kernels/linear-attention-core.js +1 -6
- package/src/gpu/kernels/matmul-selection.d.ts +2 -2
- package/src/gpu/kernels/matmul-selection.js +4 -1
- package/src/gpu/kernels/matmul.d.ts +1 -1
- package/src/gpu/kernels/matmul.js +24 -6
- package/src/gpu/kernels/moe.d.ts +5 -1
- package/src/gpu/kernels/moe.js +5 -2
- package/src/gpu/kernels/softmax.d.ts +3 -3
- package/src/gpu/kernels/softmax.js +9 -2
- package/src/gpu/kernels/turboquant-codebook.js +1 -1
- package/src/gpu/kernels/types.d.ts +1 -1
- package/src/gpu/profiler.js +11 -1
- package/src/inference/browser-harness-model-helpers.js +4 -1
- package/src/inference/browser-harness-text-helpers.d.ts +28 -0
- package/src/inference/browser-harness-text-helpers.js +357 -116
- package/src/inference/browser-harness.d.ts +3 -3
- package/src/inference/browser-harness.js +266 -11
- package/src/inference/fixtures/rerank-semantic-fixtures.json +38 -0
- package/src/inference/kv-cache/quantized.js +16 -2
- package/src/inference/kv-cache/tiered.js +50 -15
- package/src/inference/moe-router.js +38 -7
- package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +1 -1
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +8 -6
- package/src/inference/pipelines/text/config.d.ts +5 -1
- package/src/inference/pipelines/text/config.js +135 -112
- package/src/inference/pipelines/text/embed.d.ts +1 -0
- package/src/inference/pipelines/text/embed.js +39 -12
- package/src/inference/pipelines/text/execution-plan.js +8 -2
- package/src/inference/pipelines/text/ffn/dense.js +15 -4
- package/src/inference/pipelines/text/ffn/moe.js +4 -3
- package/src/inference/pipelines/text/ffn/standard.d.ts +2 -1
- package/src/inference/pipelines/text/ffn/standard.js +25 -2
- package/src/inference/pipelines/text/generator-helpers.js +2 -1
- package/src/inference/pipelines/text/generator-runtime.d.ts +6 -2
- package/src/inference/pipelines/text/generator-runtime.js +53 -20
- package/src/inference/pipelines/text/generator-steps.js +3 -0
- package/src/inference/pipelines/text/generator.js +6 -1
- package/src/inference/pipelines/text/init-rope.js +54 -6
- package/src/inference/pipelines/text/init.js +63 -15
- package/src/inference/pipelines/text/layer.js +32 -5
- package/src/inference/pipelines/text/linear-attention.js +12 -8
- package/src/inference/pipelines/text/logits/gpu.d.ts +1 -1
- package/src/inference/pipelines/text/logits/gpu.js +76 -113
- package/src/inference/pipelines/text/logits/index.js +44 -87
- package/src/inference/pipelines/text/logits/precision-policy.d.ts +11 -0
- package/src/inference/pipelines/text/logits/precision-policy.js +103 -0
- package/src/inference/pipelines/text/logits/types.d.ts +1 -0
- package/src/inference/pipelines/text/logits/utils.d.ts +2 -0
- package/src/inference/pipelines/text/logits/utils.js +11 -0
- package/src/inference/pipelines/text/lora-apply.js +6 -4
- package/src/inference/pipelines/text/moe-cpu.js +25 -0
- package/src/inference/pipelines/text/moe-gpu.js +148 -97
- package/src/inference/pipelines/text/moe-shape-validator.d.ts +33 -3
- package/src/inference/pipelines/text/moe-shape-validator.js +149 -31
- package/src/inference/pipelines/text/ops.d.ts +3 -0
- package/src/inference/pipelines/text/ops.js +26 -1
- package/src/inference/pipelines/text/per-layer-inputs.js +27 -12
- package/src/inference/pipelines/text/weights.js +7 -2
- package/src/inference/pipelines/text.js +24 -4
- package/src/inference/tokenizers/behavior-flags.js +23 -2
- package/src/loader/final-weights-loader.js +12 -1
- package/src/rules/execution-rules-contract-check.js +12 -3
- package/src/rules/inference/capability-transforms.rules.json +14 -2
- package/src/rules/inference/config.rules.json +26 -2
- package/src/rules/inference/dtype.rules.json +12 -5
- package/src/rules/inference/execution.rules.json +5 -19
- package/src/rules/inference/moe.rules.json +97 -0
- package/src/tooling/command-api-constants.js +2 -2
- package/src/tooling/command-api-family-normalizers.js +2 -1
- package/src/tooling/command-api.d.ts +1 -1
- package/src/tooling/command-runner-shared.d.ts +1 -1
- package/src/tooling/command-runner-shared.js +3 -1
- package/src/tooling/node-converter.js +20 -0
- package/src/tooling/node-source-runtime.js +5 -1
- package/src/version.js +1 -1
- package/assets/doppler-readme-hero.svg +0 -84
|
@@ -11,7 +11,7 @@ import {
|
|
|
11
11
|
getActiveKernelPathPolicy,
|
|
12
12
|
} from '../config/kernel-path-loader.js';
|
|
13
13
|
import { validateTrainingMetricsReport } from '../config/schema/training-metrics.schema.js';
|
|
14
|
-
import { modelSupportsEmbedding } from '../config/schema/manifest.schema.js';
|
|
14
|
+
import { modelSupportsEmbedding, modelSupportsRerank } from '../config/schema/manifest.schema.js';
|
|
15
15
|
import {
|
|
16
16
|
resolveReportTimestamp,
|
|
17
17
|
resolveRuntime,
|
|
@@ -54,6 +54,8 @@ import {
|
|
|
54
54
|
buildDecodeRecordTopOps,
|
|
55
55
|
buildDecodeRecordTopOpGroups,
|
|
56
56
|
runEmbeddingSemanticChecks,
|
|
57
|
+
runRerank,
|
|
58
|
+
runRerankSemanticChecks,
|
|
57
59
|
isCoherentOutput,
|
|
58
60
|
runTextInference,
|
|
59
61
|
runEmbedding,
|
|
@@ -115,6 +117,7 @@ const BROWSER_WORKLOAD_SET = Object.freeze([
|
|
|
115
117
|
'kernels',
|
|
116
118
|
'inference',
|
|
117
119
|
'embedding',
|
|
120
|
+
'rerank',
|
|
118
121
|
'training',
|
|
119
122
|
'diffusion',
|
|
120
123
|
'energy',
|
|
@@ -127,6 +130,7 @@ const BROWSER_WORKLOAD_DISPATCH_MAP = Object.freeze({
|
|
|
127
130
|
kernels: 'runKernelSuite',
|
|
128
131
|
inference: 'runInferenceSuite',
|
|
129
132
|
embedding: 'runEmbeddingSuite',
|
|
133
|
+
rerank: 'runRerankSuite',
|
|
130
134
|
training: 'runTrainingSuite',
|
|
131
135
|
diffusion: 'runDiffusionSuite',
|
|
132
136
|
energy: 'runEnergySuite',
|
|
@@ -141,6 +145,7 @@ const BROWSER_WORKLOAD_DISPATCH_MAP = Object.freeze({
|
|
|
141
145
|
bench: Object.freeze({
|
|
142
146
|
inference: 'runBenchSuite',
|
|
143
147
|
embedding: 'runBenchSuite',
|
|
148
|
+
rerank: 'runBenchSuite',
|
|
144
149
|
training: 'runBenchSuite(training)',
|
|
145
150
|
diffusion: 'runBenchSuite(diffusion)',
|
|
146
151
|
}),
|
|
@@ -530,19 +535,97 @@ async function runInferenceSuite(options = {}) {
|
|
|
530
535
|
const runtimeConfig = getRuntimeConfig();
|
|
531
536
|
const modelType = harness.manifest?.modelType || 'transformer';
|
|
532
537
|
const supportsEmbedding = modelSupportsEmbedding(harness.manifest);
|
|
538
|
+
const supportsRerank = modelSupportsRerank(harness.manifest);
|
|
533
539
|
if (options.expectedModelType === 'embedding' && !supportsEmbedding) {
|
|
534
540
|
throw new Error(
|
|
535
541
|
`Expected an embedding-capable model for workload "${options.workload || 'inference'}", got modelType="${modelType}". ` +
|
|
536
542
|
`Set inference.supportsEmbedding=true in the manifest for text-generation models that should expose pipeline.embed().`
|
|
537
543
|
);
|
|
538
544
|
}
|
|
545
|
+
if (options.expectedModelType === 'rerank' && !supportsRerank) {
|
|
546
|
+
throw new Error(
|
|
547
|
+
`Expected a rerank-capable model for workload "${options.workload || 'inference'}", got modelType="${modelType}". ` +
|
|
548
|
+
'Set inference.supportsRerank=true and inference.rerank in the manifest for models that should expose rerank scoring.'
|
|
549
|
+
);
|
|
550
|
+
}
|
|
539
551
|
const safeModelLoadMs = toTimingNumber(harness.modelLoadMs, 0);
|
|
540
552
|
|
|
541
553
|
let results;
|
|
542
554
|
let output = null;
|
|
543
555
|
let metrics;
|
|
544
556
|
|
|
545
|
-
if (
|
|
557
|
+
if (options.workload === 'rerank' && supportsRerank) {
|
|
558
|
+
const run = await runRerank(harness.pipeline, runtimeConfig);
|
|
559
|
+
const semantic = await runRerankSemanticChecks(harness.pipeline, options);
|
|
560
|
+
const allScoresFinite = run.scores.every((entry) => (
|
|
561
|
+
Number.isFinite(entry.score)
|
|
562
|
+
&& Number.isFinite(entry.probability)
|
|
563
|
+
&& Number.isFinite(entry.trueLogit)
|
|
564
|
+
&& Number.isFinite(entry.falseLogit)
|
|
565
|
+
));
|
|
566
|
+
const hasRanking = Array.isArray(run.ranking) && run.ranking.length === run.documentCount;
|
|
567
|
+
const isValidRerank = allScoresFinite && hasRanking && run.documentCount > 0;
|
|
568
|
+
const isSemanticValid = semantic.passed;
|
|
569
|
+
output = {
|
|
570
|
+
mode: 'rerank',
|
|
571
|
+
query: run.query,
|
|
572
|
+
documentCount: run.documentCount,
|
|
573
|
+
topDocument: run.topDocument,
|
|
574
|
+
ranking: run.ranking,
|
|
575
|
+
semantic: {
|
|
576
|
+
passed: isSemanticValid,
|
|
577
|
+
pairAcc: Number(semantic.pairAcc.toFixed(4)),
|
|
578
|
+
failedCaseIds: semantic.failedCaseIds,
|
|
579
|
+
details: {
|
|
580
|
+
pairs: semantic.pairs,
|
|
581
|
+
},
|
|
582
|
+
},
|
|
583
|
+
};
|
|
584
|
+
results = [
|
|
585
|
+
{
|
|
586
|
+
name: 'rerank',
|
|
587
|
+
passed: isValidRerank,
|
|
588
|
+
duration: run.durationMs,
|
|
589
|
+
error: isValidRerank
|
|
590
|
+
? undefined
|
|
591
|
+
: 'Rerank scores must be finite and produce a full ranking.',
|
|
592
|
+
},
|
|
593
|
+
{
|
|
594
|
+
name: 'rerank-semantic',
|
|
595
|
+
passed: isSemanticValid,
|
|
596
|
+
duration: semantic.durationMs,
|
|
597
|
+
error: isSemanticValid
|
|
598
|
+
? undefined
|
|
599
|
+
: (
|
|
600
|
+
`Rerank semantic checks below threshold: pairs=${(semantic.pairAcc * 100).toFixed(1)}% `
|
|
601
|
+
+ `(min ${(semantic.minPairAcc * 100).toFixed(1)}%). `
|
|
602
|
+
+ (semantic.failedCaseIds.length > 0 ? `Failed: ${semantic.failedCaseIds.join(', ')}` : '')
|
|
603
|
+
),
|
|
604
|
+
},
|
|
605
|
+
];
|
|
606
|
+
metrics = {
|
|
607
|
+
query: run.query,
|
|
608
|
+
documentCount: run.documentCount,
|
|
609
|
+
topDocumentIndex: run.topDocument?.index ?? null,
|
|
610
|
+
topDocumentScore: run.topDocument?.score == null ? null : Number(run.topDocument.score.toFixed(6)),
|
|
611
|
+
topDocumentProbability: run.topDocument?.probability == null ? null : Number(run.topDocument.probability.toFixed(6)),
|
|
612
|
+
rerankMs: Number(run.durationMs.toFixed(2)),
|
|
613
|
+
rerankRanking: run.ranking,
|
|
614
|
+
semanticPassed: isSemanticValid,
|
|
615
|
+
semanticDurationMs: Number(semantic.durationMs.toFixed(2)),
|
|
616
|
+
semanticPairAcc: Number(semantic.pairAcc.toFixed(4)),
|
|
617
|
+
semanticPairPassed: semantic.pairPassed,
|
|
618
|
+
semanticPairTotal: semantic.pairTotal,
|
|
619
|
+
semanticMinPairAcc: Number(semantic.minPairAcc.toFixed(4)),
|
|
620
|
+
semanticMinScoreMargin: Number(semantic.minScoreMargin.toFixed(4)),
|
|
621
|
+
semanticFailedCases: semantic.failedCaseIds,
|
|
622
|
+
semanticDetails: {
|
|
623
|
+
pairs: semantic.pairs,
|
|
624
|
+
},
|
|
625
|
+
modelLoadMs: safeModelLoadMs,
|
|
626
|
+
endToEndMs: safeToFixed(safeModelLoadMs + run.durationMs),
|
|
627
|
+
};
|
|
628
|
+
} else if (modelType === 'embedding' || (options.workload === 'embedding' && supportsEmbedding)) {
|
|
546
629
|
const run = await runEmbedding(harness.pipeline, runtimeConfig);
|
|
547
630
|
const semantic = await runEmbeddingSemanticChecks(harness.pipeline, options);
|
|
548
631
|
const isValidEmbedding = run.embeddingDim > 0 && run.nonFiniteCount === 0;
|
|
@@ -772,12 +855,20 @@ async function runInferenceSuite(options = {}) {
|
|
|
772
855
|
};
|
|
773
856
|
}
|
|
774
857
|
|
|
858
|
+
function resolveBenchmarkIterationSettings(runtimeConfig) {
|
|
859
|
+
const benchConfig = runtimeConfig?.shared?.benchmark?.run || {};
|
|
860
|
+
return {
|
|
861
|
+
warmupRuns: Math.max(0, Math.floor(benchConfig.warmupRuns ?? 0)),
|
|
862
|
+
timedRuns: Math.max(1, Math.floor(benchConfig.timedRuns ?? 1)),
|
|
863
|
+
};
|
|
864
|
+
}
|
|
865
|
+
|
|
775
866
|
async function runBenchSuite(options = {}) {
|
|
776
867
|
const startTime = performance.now();
|
|
777
868
|
const runtimeConfig = getRuntimeConfig();
|
|
778
|
-
const
|
|
779
|
-
const warmupRuns =
|
|
780
|
-
const timedRuns =
|
|
869
|
+
const iterationSettings = resolveBenchmarkIterationSettings(runtimeConfig);
|
|
870
|
+
const warmupRuns = iterationSettings.warmupRuns;
|
|
871
|
+
const timedRuns = iterationSettings.timedRuns;
|
|
781
872
|
const cacheMode = normalizeCacheMode(options.cacheMode);
|
|
782
873
|
const loadMode = normalizeLoadMode(options.loadMode, !!options.modelUrl, options.modelUrl);
|
|
783
874
|
const workloadType = normalizeWorkloadType(
|
|
@@ -792,7 +883,7 @@ async function runBenchSuite(options = {}) {
|
|
|
792
883
|
if (workloadType === 'training') {
|
|
793
884
|
const trainingBench = await runTrainingBenchSuite({
|
|
794
885
|
...options,
|
|
795
|
-
benchRun:
|
|
886
|
+
benchRun: iterationSettings,
|
|
796
887
|
workloadType,
|
|
797
888
|
});
|
|
798
889
|
const trainingReport = trainingBench?.metrics?.trainingMetricsReport;
|
|
@@ -888,15 +979,24 @@ async function runBenchSuite(options = {}) {
|
|
|
888
979
|
},
|
|
889
980
|
() => initializeSuiteModel(options)
|
|
890
981
|
);
|
|
891
|
-
const benchRun =
|
|
982
|
+
const benchRun = options.workload === 'rerank'
|
|
983
|
+
? iterationSettings
|
|
984
|
+
: resolveBenchmarkRunSettings(runtimeConfig, harness.pipeline ?? harness);
|
|
892
985
|
const modelType = harness.manifest?.modelType || 'transformer';
|
|
893
986
|
const supportsEmbedding = modelSupportsEmbedding(harness.manifest);
|
|
987
|
+
const supportsRerank = modelSupportsRerank(harness.manifest);
|
|
894
988
|
if (options.expectedModelType === 'embedding' && !supportsEmbedding) {
|
|
895
989
|
throw new Error(
|
|
896
990
|
`Expected an embedding-capable model for bench workload "${options.workload || 'inference'}", got modelType="${modelType}". ` +
|
|
897
991
|
`Set inference.supportsEmbedding=true in the manifest for text-generation models that should expose pipeline.embed().`
|
|
898
992
|
);
|
|
899
993
|
}
|
|
994
|
+
if (options.expectedModelType === 'rerank' && !supportsRerank) {
|
|
995
|
+
throw new Error(
|
|
996
|
+
`Expected a rerank-capable model for bench workload "${options.workload || 'inference'}", got modelType="${modelType}". ` +
|
|
997
|
+
'Set inference.supportsRerank=true and inference.rerank in the manifest for models that should expose rerank scoring.'
|
|
998
|
+
);
|
|
999
|
+
}
|
|
900
1000
|
const safeModelLoadMs = toTimingNumber(harness.modelLoadMs, 0);
|
|
901
1001
|
|
|
902
1002
|
let results;
|
|
@@ -904,7 +1004,155 @@ async function runBenchSuite(options = {}) {
|
|
|
904
1004
|
let output = null;
|
|
905
1005
|
let timing;
|
|
906
1006
|
|
|
907
|
-
if (
|
|
1007
|
+
if (options.workload === 'rerank' && supportsRerank) {
|
|
1008
|
+
const durations = [];
|
|
1009
|
+
const timedDurations = [];
|
|
1010
|
+
const documentCounts = [];
|
|
1011
|
+
const topDocumentScores = [];
|
|
1012
|
+
const topDocumentProbabilities = [];
|
|
1013
|
+
let invalidRuns = 0;
|
|
1014
|
+
let nonFiniteScores = 0;
|
|
1015
|
+
let lastRun = null;
|
|
1016
|
+
|
|
1017
|
+
for (let i = 0; i < warmupRuns + timedRuns; i++) {
|
|
1018
|
+
harness.pipeline.reset?.();
|
|
1019
|
+
const run = await runRerank(harness.pipeline, runtimeConfig, benchRun);
|
|
1020
|
+
if (i >= warmupRuns) {
|
|
1021
|
+
timedDurations.push(run.durationMs);
|
|
1022
|
+
const finiteScores = run.scores.filter((entry) => (
|
|
1023
|
+
Number.isFinite(entry.score)
|
|
1024
|
+
&& Number.isFinite(entry.probability)
|
|
1025
|
+
&& Number.isFinite(entry.trueLogit)
|
|
1026
|
+
&& Number.isFinite(entry.falseLogit)
|
|
1027
|
+
));
|
|
1028
|
+
nonFiniteScores += run.scores.length - finiteScores.length;
|
|
1029
|
+
const hasRanking = Array.isArray(run.ranking) && run.ranking.length === run.documentCount;
|
|
1030
|
+
if (finiteScores.length === run.scores.length && hasRanking && run.documentCount > 0) {
|
|
1031
|
+
durations.push(run.durationMs);
|
|
1032
|
+
documentCounts.push(run.documentCount);
|
|
1033
|
+
if (Number.isFinite(run.topDocument?.score)) {
|
|
1034
|
+
topDocumentScores.push(run.topDocument.score);
|
|
1035
|
+
}
|
|
1036
|
+
if (Number.isFinite(run.topDocument?.probability)) {
|
|
1037
|
+
topDocumentProbabilities.push(run.topDocument.probability);
|
|
1038
|
+
}
|
|
1039
|
+
} else {
|
|
1040
|
+
invalidRuns++;
|
|
1041
|
+
}
|
|
1042
|
+
lastRun = run;
|
|
1043
|
+
}
|
|
1044
|
+
}
|
|
1045
|
+
|
|
1046
|
+
const semantic = await runRerankSemanticChecks(harness.pipeline, options);
|
|
1047
|
+
const rerankMsStats = computeSampleStats(durations);
|
|
1048
|
+
const timedRerankMsStats = computeSampleStats(timedDurations);
|
|
1049
|
+
const documentCountStats = computeSampleStats(documentCounts);
|
|
1050
|
+
const topScoreStats = computeSampleStats(topDocumentScores);
|
|
1051
|
+
const topProbabilityStats = computeSampleStats(topDocumentProbabilities);
|
|
1052
|
+
const avgMs = rerankMsStats.mean;
|
|
1053
|
+
const semanticPassed = semantic.passed;
|
|
1054
|
+
|
|
1055
|
+
results = [
|
|
1056
|
+
{
|
|
1057
|
+
name: 'benchmark-rerank',
|
|
1058
|
+
passed: durations.length > 0 && invalidRuns === 0,
|
|
1059
|
+
duration: durations.reduce((sum, value) => sum + value, 0),
|
|
1060
|
+
error: durations.length > 0
|
|
1061
|
+
? (
|
|
1062
|
+
invalidRuns === 0
|
|
1063
|
+
? undefined
|
|
1064
|
+
: `Invalid rerank runs: ${invalidRuns} (non-finite scores or incomplete ranking observed)`
|
|
1065
|
+
)
|
|
1066
|
+
: 'No valid rerank benchmark runs completed',
|
|
1067
|
+
},
|
|
1068
|
+
{
|
|
1069
|
+
name: 'benchmark-rerank-semantic',
|
|
1070
|
+
passed: semanticPassed,
|
|
1071
|
+
duration: semantic.durationMs,
|
|
1072
|
+
error: semanticPassed
|
|
1073
|
+
? undefined
|
|
1074
|
+
: (
|
|
1075
|
+
`Rerank semantic checks below threshold: pairs=${(semantic.pairAcc * 100).toFixed(1)}% `
|
|
1076
|
+
+ `(min ${(semantic.minPairAcc * 100).toFixed(1)}%). `
|
|
1077
|
+
+ (semantic.failedCaseIds.length > 0 ? `Failed: ${semantic.failedCaseIds.join(', ')}` : '')
|
|
1078
|
+
),
|
|
1079
|
+
},
|
|
1080
|
+
];
|
|
1081
|
+
|
|
1082
|
+
output = {
|
|
1083
|
+
mode: 'rerank',
|
|
1084
|
+
query: lastRun?.query ?? null,
|
|
1085
|
+
documentCount: lastRun?.documentCount ?? null,
|
|
1086
|
+
topDocument: lastRun?.topDocument ?? null,
|
|
1087
|
+
ranking: lastRun?.ranking ?? [],
|
|
1088
|
+
semantic: {
|
|
1089
|
+
passed: semanticPassed,
|
|
1090
|
+
pairAcc: Number(semantic.pairAcc.toFixed(4)),
|
|
1091
|
+
failedCaseIds: semantic.failedCaseIds,
|
|
1092
|
+
details: {
|
|
1093
|
+
pairs: semantic.pairs,
|
|
1094
|
+
},
|
|
1095
|
+
},
|
|
1096
|
+
};
|
|
1097
|
+
|
|
1098
|
+
metrics = {
|
|
1099
|
+
warmupRuns,
|
|
1100
|
+
timedRuns,
|
|
1101
|
+
validRuns: durations.length,
|
|
1102
|
+
invalidRuns,
|
|
1103
|
+
invalidRatePct: Number((timedRuns > 0 ? (invalidRuns / timedRuns) * 100 : 0).toFixed(2)),
|
|
1104
|
+
query: lastRun?.query ?? null,
|
|
1105
|
+
documentCount: Math.round(documentCountStats.mean),
|
|
1106
|
+
topDocumentIndex: lastRun?.topDocument?.index ?? null,
|
|
1107
|
+
topDocumentScore: lastRun?.topDocument?.score == null ? null : Number(lastRun.topDocument.score.toFixed(6)),
|
|
1108
|
+
topDocumentProbability: lastRun?.topDocument?.probability == null ? null : Number(lastRun.topDocument.probability.toFixed(6)),
|
|
1109
|
+
topDocumentScoreStats: topScoreStats,
|
|
1110
|
+
topDocumentProbabilityStats: topProbabilityStats,
|
|
1111
|
+
nonFiniteScores,
|
|
1112
|
+
firstTimedRerankMs: Number((timedDurations[0] ?? 0).toFixed(2)),
|
|
1113
|
+
minRerankMs: Number(rerankMsStats.min.toFixed(2)),
|
|
1114
|
+
medianRerankMs: Number(rerankMsStats.median.toFixed(2)),
|
|
1115
|
+
p95RerankMs: Number(rerankMsStats.p95.toFixed(2)),
|
|
1116
|
+
p99RerankMs: Number(rerankMsStats.p99.toFixed(2)),
|
|
1117
|
+
maxRerankMs: Number(rerankMsStats.max.toFixed(2)),
|
|
1118
|
+
stdDevRerankMs: Number(rerankMsStats.stdDev.toFixed(2)),
|
|
1119
|
+
ci95RerankMs: Number(rerankMsStats.ci95.toFixed(2)),
|
|
1120
|
+
avgRerankMs: Number(avgMs.toFixed(2)),
|
|
1121
|
+
avgReranksPerSec: Number((avgMs > 0 ? (1000 / avgMs) : 0).toFixed(2)),
|
|
1122
|
+
semanticPassed,
|
|
1123
|
+
semanticDurationMs: Number(semantic.durationMs.toFixed(2)),
|
|
1124
|
+
semanticPairAcc: Number(semantic.pairAcc.toFixed(4)),
|
|
1125
|
+
semanticPairPassed: semantic.pairPassed,
|
|
1126
|
+
semanticPairTotal: semantic.pairTotal,
|
|
1127
|
+
semanticMinPairAcc: Number(semantic.minPairAcc.toFixed(4)),
|
|
1128
|
+
semanticMinScoreMargin: Number(semantic.minScoreMargin.toFixed(4)),
|
|
1129
|
+
semanticFailedCases: semantic.failedCaseIds,
|
|
1130
|
+
semanticDetails: {
|
|
1131
|
+
pairs: semantic.pairs,
|
|
1132
|
+
},
|
|
1133
|
+
modelLoadMs: safeModelLoadMs,
|
|
1134
|
+
latency: {
|
|
1135
|
+
timedRerankMs: timedRerankMsStats,
|
|
1136
|
+
rerankMs: rerankMsStats,
|
|
1137
|
+
},
|
|
1138
|
+
documents: {
|
|
1139
|
+
count: documentCountStats,
|
|
1140
|
+
},
|
|
1141
|
+
};
|
|
1142
|
+
|
|
1143
|
+
timing = buildCanonicalTiming({
|
|
1144
|
+
modelLoadMs: safeModelLoadMs,
|
|
1145
|
+
firstTokenMs: null,
|
|
1146
|
+
firstResponseMs: Number.isFinite(timedDurations[0])
|
|
1147
|
+
? safeModelLoadMs + timedDurations[0]
|
|
1148
|
+
: null,
|
|
1149
|
+
prefillMs: null,
|
|
1150
|
+
decodeMs: null,
|
|
1151
|
+
totalRunMs: rerankMsStats.median,
|
|
1152
|
+
cacheMode,
|
|
1153
|
+
loadMode,
|
|
1154
|
+
});
|
|
1155
|
+
} else if (modelType === 'embedding' || (options.workload === 'embedding' && supportsEmbedding)) {
|
|
908
1156
|
const durations = [];
|
|
909
1157
|
const timedDurations = [];
|
|
910
1158
|
const embeddingDims = [];
|
|
@@ -1468,6 +1716,9 @@ async function dispatchBrowserSuite(mode, workload, options) {
|
|
|
1468
1716
|
if (mode === 'verify' && workload === 'kernels') {
|
|
1469
1717
|
return runKernelSuite(options);
|
|
1470
1718
|
}
|
|
1719
|
+
if (mode === 'bench') {
|
|
1720
|
+
return runBenchSuite(options);
|
|
1721
|
+
}
|
|
1471
1722
|
if (workload === 'embedding') {
|
|
1472
1723
|
return runInferenceSuite({
|
|
1473
1724
|
...options,
|
|
@@ -1475,12 +1726,16 @@ async function dispatchBrowserSuite(mode, workload, options) {
|
|
|
1475
1726
|
expectedModelType: options.expectedModelType ?? 'embedding',
|
|
1476
1727
|
});
|
|
1477
1728
|
}
|
|
1729
|
+
if (workload === 'rerank') {
|
|
1730
|
+
return runInferenceSuite({
|
|
1731
|
+
...options,
|
|
1732
|
+
suiteName: 'rerank',
|
|
1733
|
+
expectedModelType: options.expectedModelType ?? 'rerank',
|
|
1734
|
+
});
|
|
1735
|
+
}
|
|
1478
1736
|
if (mode === 'verify' && workload === 'training') {
|
|
1479
1737
|
return runTrainingSuite(options);
|
|
1480
1738
|
}
|
|
1481
|
-
if (mode === 'bench') {
|
|
1482
|
-
return runBenchSuite(options);
|
|
1483
|
-
}
|
|
1484
1739
|
if (mode === 'verify' && workload === 'diffusion') {
|
|
1485
1740
|
return runDiffusionSuite(options);
|
|
1486
1741
|
}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
{
|
|
2
|
+
"defaults": {
|
|
3
|
+
"minPairAcc": 0.8,
|
|
4
|
+
"minScoreMargin": 0,
|
|
5
|
+
"cases": [
|
|
6
|
+
{
|
|
7
|
+
"id": "china_capital",
|
|
8
|
+
"query": "What is the capital of China?",
|
|
9
|
+
"positive": "The capital of China is Beijing.",
|
|
10
|
+
"negative": "Gravity is a force that attracts two bodies towards each other."
|
|
11
|
+
},
|
|
12
|
+
{
|
|
13
|
+
"id": "password_reset",
|
|
14
|
+
"query": "How do I reset my account password?",
|
|
15
|
+
"positive": "To reset your password, open account settings and choose the forgot-password flow.",
|
|
16
|
+
"negative": "The recipe combines tomatoes, basil, and olive oil."
|
|
17
|
+
},
|
|
18
|
+
{
|
|
19
|
+
"id": "python_sort",
|
|
20
|
+
"query": "How do I sort a list in Python?",
|
|
21
|
+
"positive": "Use sorted() or list.sort() to sort a list in Python.",
|
|
22
|
+
"negative": "The cafe serves coffee and pastries."
|
|
23
|
+
},
|
|
24
|
+
{
|
|
25
|
+
"id": "webgpu_compute",
|
|
26
|
+
"query": "What API exposes GPU hardware for browser compute workloads?",
|
|
27
|
+
"positive": "WebGPU provides low-level access to graphics processors for compute workloads.",
|
|
28
|
+
"negative": "WebSocket provides full-duplex communication over TCP."
|
|
29
|
+
},
|
|
30
|
+
{
|
|
31
|
+
"id": "vaccines",
|
|
32
|
+
"query": "How do vaccines work?",
|
|
33
|
+
"positive": "Vaccines train the immune system to recognize future infections.",
|
|
34
|
+
"negative": "The symphony orchestra has strings, woodwinds, brass, and percussion."
|
|
35
|
+
}
|
|
36
|
+
]
|
|
37
|
+
}
|
|
38
|
+
}
|
|
@@ -25,9 +25,9 @@ export class QuantizedKVCache {
|
|
|
25
25
|
this.maxSeqLen = config.maxSeqLen;
|
|
26
26
|
this.useGPU = config.useGPU;
|
|
27
27
|
this.layout = 'contiguous_quantized';
|
|
28
|
-
this.kvDtype = config.kvDtype
|
|
28
|
+
this.kvDtype = requireString(config.kvDtype, 'QuantizedKVCache config.kvDtype');
|
|
29
29
|
this.quantMode = config.quantMode;
|
|
30
|
-
this.bitWidth = config.bitWidth
|
|
30
|
+
this.bitWidth = requirePositiveInteger(config.bitWidth, 'QuantizedKVCache config.bitWidth');
|
|
31
31
|
this.prodMode = config.prodMode === true;
|
|
32
32
|
this.currentSeqLen = 0;
|
|
33
33
|
this.totalTokensSeen = 0;
|
|
@@ -383,3 +383,17 @@ export class QuantizedKVCache {
|
|
|
383
383
|
}
|
|
384
384
|
}
|
|
385
385
|
}
|
|
386
|
+
|
|
387
|
+
function requireString(value, label) {
|
|
388
|
+
if (typeof value !== 'string' || value.length === 0) {
|
|
389
|
+
throw new Error(`${label} is required.`);
|
|
390
|
+
}
|
|
391
|
+
return value;
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
function requirePositiveInteger(value, label) {
|
|
395
|
+
if (!Number.isInteger(value) || value <= 0) {
|
|
396
|
+
throw new Error(`${label} must be a positive integer.`);
|
|
397
|
+
}
|
|
398
|
+
return value;
|
|
399
|
+
}
|
|
@@ -59,7 +59,7 @@ export class TieredKVCache {
|
|
|
59
59
|
|
|
60
60
|
this.layout = 'tiered';
|
|
61
61
|
|
|
62
|
-
this.kvDtype = config.kvDtype;
|
|
62
|
+
this.kvDtype = requireNonEmptyString(config.kvDtype, 'TieredKVCache config.kvDtype');
|
|
63
63
|
|
|
64
64
|
this.bytesPerElem = this.kvDtype === 'f16' ? 2 : 4;
|
|
65
65
|
|
|
@@ -69,17 +69,19 @@ export class TieredKVCache {
|
|
|
69
69
|
|
|
70
70
|
this.coldPageSize = tiering.coldPageSize;
|
|
71
71
|
|
|
72
|
-
this.coldDtype = tiering.coldDtype
|
|
72
|
+
this.coldDtype = requireNonEmptyString(tiering.coldDtype, 'TieredKVCache tiering.coldDtype');
|
|
73
73
|
|
|
74
|
-
this.tieringMode = tiering.mode;
|
|
74
|
+
this.tieringMode = requireNonEmptyString(tiering.mode, 'TieredKVCache tiering.mode');
|
|
75
75
|
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
this.compression = tiering.compression ?? { mode: defaultCompressionMode, blockSize: 1 };
|
|
76
|
+
this.compression = requirePlainObject(
|
|
77
|
+
tiering.compression,
|
|
78
|
+
'TieredKVCache tiering.compression'
|
|
79
|
+
);
|
|
81
80
|
|
|
82
|
-
this.gating =
|
|
81
|
+
this.gating = requirePlainObject(
|
|
82
|
+
tiering.gating,
|
|
83
|
+
'TieredKVCache tiering.gating'
|
|
84
|
+
);
|
|
83
85
|
|
|
84
86
|
this.currentSeqLen = 0;
|
|
85
87
|
|
|
@@ -102,7 +104,10 @@ export class TieredKVCache {
|
|
|
102
104
|
}
|
|
103
105
|
|
|
104
106
|
assertSupportedTurboQuantMode(this.tieringMode, 'tiering.mode');
|
|
105
|
-
assertSupportedTurboQuantMode(
|
|
107
|
+
assertSupportedTurboQuantMode(
|
|
108
|
+
requireNonEmptyString(this.compression.mode, 'TieredKVCache tiering.compression.mode'),
|
|
109
|
+
'tiering.compression.mode'
|
|
110
|
+
);
|
|
106
111
|
this.coldQuantMode = this._resolveCompressionMode(this.compression, this.gating);
|
|
107
112
|
assertSupportedTurboQuantMode(this.coldQuantMode, 'resolved cold quant mode');
|
|
108
113
|
if (this.coldQuantMode !== 'none' && this.compression.blockSize !== 1) {
|
|
@@ -111,7 +116,9 @@ export class TieredKVCache {
|
|
|
111
116
|
|
|
112
117
|
this.isTurboQuant = isTurboQuantMode(this.coldQuantMode);
|
|
113
118
|
this.isProdMode = this.coldQuantMode === 'turboquant_prod';
|
|
114
|
-
this.turboQuantBitWidth = this.
|
|
119
|
+
this.turboQuantBitWidth = this.isTurboQuant
|
|
120
|
+
? requirePositiveInteger(this.compression.bitWidth, 'TieredKVCache tiering.compression.bitWidth')
|
|
121
|
+
: null;
|
|
115
122
|
|
|
116
123
|
if (this.isTurboQuant) {
|
|
117
124
|
this.coldPackedStride = computePackedStride(this.headDim, this.turboQuantBitWidth);
|
|
@@ -184,10 +191,17 @@ export class TieredKVCache {
|
|
|
184
191
|
|
|
185
192
|
|
|
186
193
|
_resolveCompressionMode(compression, gating) {
|
|
187
|
-
const requested =
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
194
|
+
const requested = requireNonEmptyString(
|
|
195
|
+
compression?.mode,
|
|
196
|
+
'TieredKVCache tiering.compression.mode'
|
|
197
|
+
);
|
|
198
|
+
const gatingMode = requireNonEmptyString(
|
|
199
|
+
gating?.mode,
|
|
200
|
+
'TieredKVCache tiering.gating.mode'
|
|
201
|
+
);
|
|
202
|
+
if (gatingMode === 'force_off') return 'none';
|
|
203
|
+
if (gatingMode === 'force_on') return requested;
|
|
204
|
+
if (gatingMode === 'auto' && gating.minAluBwRatio > 0) {
|
|
191
205
|
throw new Error(
|
|
192
206
|
'TieredKVCache auto compression gating requires an explicit measured ALU/BW ratio. ' +
|
|
193
207
|
'Use gating.mode="force_on"/"force_off" or set minAluBwRatio to 0.'
|
|
@@ -719,3 +733,24 @@ export class TieredKVCache {
|
|
|
719
733
|
return cloned;
|
|
720
734
|
}
|
|
721
735
|
}
|
|
736
|
+
|
|
737
|
+
function requirePlainObject(value, label) {
|
|
738
|
+
if (value == null || typeof value !== 'object' || Array.isArray(value)) {
|
|
739
|
+
throw new Error(`${label} is required.`);
|
|
740
|
+
}
|
|
741
|
+
return value;
|
|
742
|
+
}
|
|
743
|
+
|
|
744
|
+
function requireNonEmptyString(value, label) {
|
|
745
|
+
if (typeof value !== 'string' || value.trim().length === 0) {
|
|
746
|
+
throw new Error(`${label} is required.`);
|
|
747
|
+
}
|
|
748
|
+
return value.trim();
|
|
749
|
+
}
|
|
750
|
+
|
|
751
|
+
function requirePositiveInteger(value, label) {
|
|
752
|
+
if (!Number.isInteger(value) || value <= 0) {
|
|
753
|
+
throw new Error(`${label} must be a positive integer.`);
|
|
754
|
+
}
|
|
755
|
+
return value;
|
|
756
|
+
}
|
|
@@ -16,6 +16,25 @@ function isRouterVector(value) {
|
|
|
16
16
|
return value instanceof Float32Array || isGpuBufferInstance(value) || isWeightBuffer(value);
|
|
17
17
|
}
|
|
18
18
|
|
|
19
|
+
function requireRouterDtype(value, label) {
|
|
20
|
+
if (value !== 'f16' && value !== 'f32') {
|
|
21
|
+
throw new Error(`MoERouter requires ${label} to be "f16" or "f32", got ${value}.`);
|
|
22
|
+
}
|
|
23
|
+
return value;
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
function inferRouterBufferDtypeFromElementCount(buffer, elementCount, label) {
|
|
27
|
+
const byteLength = Number.isFinite(buffer?.size) ? buffer.size : buffer?.byteLength;
|
|
28
|
+
if (!Number.isFinite(byteLength)) {
|
|
29
|
+
throw new Error(`MoERouter requires ${label} byte size to resolve dtype.`);
|
|
30
|
+
}
|
|
31
|
+
const bytesPerElement = byteLength / elementCount;
|
|
32
|
+
if (bytesPerElement !== 2 && bytesPerElement !== 4) {
|
|
33
|
+
throw new Error(`MoERouter cannot infer ${label} dtype from bytesPerElement=${bytesPerElement}.`);
|
|
34
|
+
}
|
|
35
|
+
return selectRuleValue('inference', 'dtype', 'f16OrF32FromBytes', { bytesPerElement });
|
|
36
|
+
}
|
|
37
|
+
|
|
19
38
|
|
|
20
39
|
|
|
21
40
|
|
|
@@ -61,8 +80,10 @@ export class MoERouter {
|
|
|
61
80
|
|
|
62
81
|
_gateWeightGPU = null;
|
|
63
82
|
|
|
83
|
+
_gateWeightDtype = null;
|
|
84
|
+
|
|
64
85
|
|
|
65
|
-
lastLogitsDtype =
|
|
86
|
+
lastLogitsDtype = null;
|
|
66
87
|
|
|
67
88
|
|
|
68
89
|
constructor(config) {
|
|
@@ -122,6 +143,9 @@ export class MoERouter {
|
|
|
122
143
|
if (this._gateWeightGPU) {
|
|
123
144
|
this._gateWeightGPU.destroy();
|
|
124
145
|
}
|
|
146
|
+
this._gateWeightDtype = isWeightBuffer(weights)
|
|
147
|
+
? getWeightDtype(weights)
|
|
148
|
+
: inferRouterBufferDtypeFromElementCount(weights, this.numExperts * this.hiddenSize, 'gate weight');
|
|
125
149
|
this.gateWeight = weights;
|
|
126
150
|
this.gateBias = bias;
|
|
127
151
|
this.gateScale = scale;
|
|
@@ -140,6 +164,7 @@ export class MoERouter {
|
|
|
140
164
|
}
|
|
141
165
|
this._gateBiasGPU = null;
|
|
142
166
|
this._gateWeightGPU = null;
|
|
167
|
+
this._gateWeightDtype = null;
|
|
143
168
|
this.gateWeight = null;
|
|
144
169
|
this.gateBias = null;
|
|
145
170
|
this.gateScale = null;
|
|
@@ -197,6 +222,8 @@ export class MoERouter {
|
|
|
197
222
|
throw new Error('Router gate weights not loaded');
|
|
198
223
|
}
|
|
199
224
|
if (!isWeightBuffer(gateWeightBuffer) && !isGpuBufferInstance(gateWeightBuffer)) {
|
|
225
|
+
const sourceWeightDtype = this._gateWeightDtype
|
|
226
|
+
?? inferRouterBufferDtypeFromElementCount(gateWeightBuffer, this.numExperts * this.hiddenSize, 'gate weight');
|
|
200
227
|
const uploaded = device.createBuffer({
|
|
201
228
|
label: 'moe_gate_weight',
|
|
202
229
|
size: gateWeightBuffer.byteLength,
|
|
@@ -209,12 +236,16 @@ export class MoERouter {
|
|
|
209
236
|
throw error;
|
|
210
237
|
}
|
|
211
238
|
this._gateWeightGPU = uploaded;
|
|
239
|
+
this._gateWeightDtype = sourceWeightDtype;
|
|
212
240
|
this.gateWeight = uploaded;
|
|
213
241
|
gateWeightBuffer = uploaded;
|
|
214
242
|
}
|
|
215
243
|
|
|
216
|
-
const
|
|
217
|
-
const
|
|
244
|
+
const inputDtype = requireRouterDtype(options.inputDtype, 'options.inputDtype');
|
|
245
|
+
const outputDtype = requireRouterDtype(options.outputDtype, 'options.outputDtype');
|
|
246
|
+
const gateWeightDtype = isWeightBuffer(gateWeightBuffer)
|
|
247
|
+
? getWeightDtype(gateWeightBuffer)
|
|
248
|
+
: this._gateWeightDtype;
|
|
218
249
|
|
|
219
250
|
// Matrix multiply: hidden_states [numTokens, hiddenSize] @ gate_weight [hiddenSize, numExperts]
|
|
220
251
|
// Result: [numTokens, numExperts]
|
|
@@ -274,9 +305,9 @@ export class MoERouter {
|
|
|
274
305
|
if (bias instanceof Float32Array) return 'f32';
|
|
275
306
|
if (isGpuBufferInstance(bias)) {
|
|
276
307
|
const bytesPerElement = Math.round(bias.size / this.numExperts);
|
|
277
|
-
return
|
|
308
|
+
return inferRouterBufferDtypeFromElementCount(bias, this.numExperts, 'gate bias');
|
|
278
309
|
}
|
|
279
|
-
|
|
310
|
+
throw new Error('MoERouter cannot infer gate bias dtype.');
|
|
280
311
|
}
|
|
281
312
|
|
|
282
313
|
_getBiasAddPipeline(logitsDtype, biasDtype, device) {
|
|
@@ -368,9 +399,9 @@ export class MoERouter {
|
|
|
368
399
|
}
|
|
369
400
|
|
|
370
401
|
|
|
371
|
-
async routeGPU(hiddenStates, numTokens) {
|
|
402
|
+
async routeGPU(hiddenStates, numTokens, options = {}) {
|
|
372
403
|
// Compute router logits on GPU
|
|
373
|
-
const logitsBuffer = await this.computeRouterLogitsGPU(hiddenStates, numTokens);
|
|
404
|
+
const logitsBuffer = await this.computeRouterLogitsGPU(hiddenStates, numTokens, null, options);
|
|
374
405
|
try {
|
|
375
406
|
const logitsData = await readBuffer(logitsBuffer);
|
|
376
407
|
const logits = this.lastLogitsDtype === 'f16'
|
|
@@ -79,4 +79,4 @@ export declare function projectContext(
|
|
|
79
79
|
options?: { recorder?: CommandRecorder | null }
|
|
80
80
|
): Promise<Tensor>;
|
|
81
81
|
|
|
82
|
-
export declare function assertClipHiddenActivationSupported(config: { hidden_act
|
|
82
|
+
export declare function assertClipHiddenActivationSupported(config: { hidden_act: string }): void;
|