doppler-gpu 0.4.5 → 0.4.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +45 -29
- package/assets/doppler-runtime-map.svg +82 -0
- package/assets/doppler.svg +1 -3
- package/package.json +25 -17
- package/src/cli/doppler-quickstart.js +1 -1
- package/src/cli/doppler-serve.js +2 -2
- package/src/client/doppler-registry.d.ts +19 -0
- package/src/client/doppler-registry.js +12 -0
- package/src/client/doppler-registry.json +50 -1
- package/src/client/runtime/model-session.d.ts +1 -0
- package/src/client/runtime/model-session.js +9 -0
- package/src/config/conversion/README.md +4 -0
- package/src/config/conversion/diffusiongemma/diffusiongemma-26b-a4b-it-q4k-ehf16-af16.json +28 -4
- package/src/config/conversion/embeddinggemma/google-embeddinggemma-300m-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-f16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-1b-it-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-f16-af32.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af16.json +29 -4
- package/src/config/conversion/gemma3/gemma-3-270m-it-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma3/translategemma-4b-1b-enes-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma3/translategemma-4b-it-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af16.json +31 -5
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-af32.json +30 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-q4k-ehf16-hq4k-af16.json +30 -4
- package/src/config/conversion/gemma4/gemma-4-12b-it-text-w4a16-ct-ehf16-af16.json +30 -4
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af16.json +30 -5
- package/src/config/conversion/gemma4/gemma-4-31b-it-text-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af16-int4ple.json +30 -5
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32-int4ple.json +29 -4
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gemma4/gemma-4-moe-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/gpt-oss-20b-f16-xmxfp4.json +29 -4
- package/src/config/conversion/janus/janus-pro-1b-text-q4k-ehaf16.json +29 -4
- package/src/config/conversion/lfm2/lfm2.5-1.2b-instruct-q4k-ehf16-af32.json +29 -4
- package/src/config/conversion/minicpm/minicpm4-0-5b-f16-af32.json +481 -0
- package/src/config/conversion/qwen3/qwen-3-5-0-8b-q4k-ehaf16.json +29 -4
- package/src/config/conversion/qwen3/qwen-3-5-2b-q4k-ehaf16.json +29 -4
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-eaf16.json +30 -5
- package/src/config/conversion/qwen3/qwen-3-6-27b-q4k-ehaf16.json +29 -4
- package/src/config/conversion/qwen3/qwen-3-embedding-0-6b-q4k-ehf16-af32.json +395 -0
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-f16-af32.json +429 -0
- package/src/config/conversion/qwen3/qwen-3-reranker-0-6b-q4k-ehf16-af32.json +540 -0
- package/src/config/execution-contract-check.js +14 -10
- package/src/config/merge-contract-check.js +1 -0
- package/src/config/merge.js +30 -0
- package/src/config/param-validator.d.ts +5 -0
- package/src/config/param-validator.js +42 -8
- package/src/config/required-inference-fields-contract-check.js +14 -0
- package/src/config/schema/debug.schema.d.ts +2 -0
- package/src/config/schema/debug.schema.js +1 -0
- package/src/config/schema/doppler.schema.d.ts +5 -1
- package/src/config/schema/doppler.schema.js +82 -292
- package/src/config/schema/inference.schema.d.ts +17 -3
- package/src/config/schema/kvcache.schema.d.ts +19 -3
- package/src/config/schema/kvcache.schema.js +2 -1
- package/src/config/schema/manifest.schema.d.ts +38 -1
- package/src/config/schema/manifest.schema.js +18 -0
- package/src/config/schema/policy-schema-registry.json +5 -0
- package/src/config/source-packages/litert/gemma-4-12b-it.json +20 -1
- package/src/config/source-packages/litert/gemma-4-e2b-it.json +20 -1
- package/src/config/source-packages/litert/gemma-4-e4b-it.json +21 -1
- package/src/config/transforms/execution-graph-transforms.d.ts +6 -0
- package/src/config/transforms/execution-graph-transforms.js +8 -0
- package/src/converter/conversion-plan.js +2 -0
- package/src/converter/parsers/transformer.js +27 -1
- package/src/converter/quantization-info.js +16 -4
- package/src/converter/rope-config.js +55 -3
- package/src/experimental/training/distillation/student-fixture.js +1 -1
- package/src/experimental/training/lora-pipeline.js +1 -1
- package/src/experimental/training/lora.js +2 -2
- package/src/experimental/training/suite.js +1 -1
- package/src/gpu/device.js +54 -11
- package/src/gpu/kernels/README.md +8 -1
- package/src/gpu/kernels/backward/attention_backward.js +8 -0
- package/src/gpu/kernels/dequant.d.ts +6 -6
- package/src/gpu/kernels/dequant.js +17 -6
- package/src/gpu/kernels/linear-attention-core.js +1 -6
- package/src/gpu/kernels/matmul-selection.d.ts +2 -2
- package/src/gpu/kernels/matmul-selection.js +9 -2
- package/src/gpu/kernels/matmul.d.ts +1 -1
- package/src/gpu/kernels/matmul.js +63 -9
- package/src/gpu/kernels/moe.d.ts +5 -1
- package/src/gpu/kernels/moe.js +5 -2
- package/src/gpu/kernels/softmax.d.ts +3 -3
- package/src/gpu/kernels/softmax.js +9 -2
- package/src/gpu/kernels/turboquant-codebook.js +1 -1
- package/src/gpu/kernels/types.d.ts +1 -1
- package/src/gpu/profiler.js +11 -1
- package/src/inference/browser-harness-model-helpers.js +4 -1
- package/src/inference/browser-harness-text-helpers.d.ts +28 -0
- package/src/inference/browser-harness-text-helpers.js +357 -116
- package/src/inference/browser-harness.d.ts +3 -3
- package/src/inference/browser-harness.js +266 -11
- package/src/inference/fixtures/rerank-semantic-fixtures.json +38 -0
- package/src/inference/kv-cache/quantized.js +16 -2
- package/src/inference/kv-cache/tiered.js +50 -15
- package/src/inference/moe-router.js +38 -7
- package/src/inference/pipelines/diffusion/text-encoder-gpu.d.ts +1 -1
- package/src/inference/pipelines/diffusion/text-encoder-gpu.js +8 -6
- package/src/inference/pipelines/text/config.d.ts +5 -1
- package/src/inference/pipelines/text/config.js +135 -112
- package/src/inference/pipelines/text/embed.d.ts +1 -0
- package/src/inference/pipelines/text/embed.js +39 -12
- package/src/inference/pipelines/text/execution-plan.js +8 -2
- package/src/inference/pipelines/text/ffn/dense.js +15 -4
- package/src/inference/pipelines/text/ffn/moe.js +4 -3
- package/src/inference/pipelines/text/ffn/standard.d.ts +2 -1
- package/src/inference/pipelines/text/ffn/standard.js +25 -2
- package/src/inference/pipelines/text/generator-helpers.js +2 -1
- package/src/inference/pipelines/text/generator-runtime.d.ts +6 -2
- package/src/inference/pipelines/text/generator-runtime.js +53 -20
- package/src/inference/pipelines/text/generator-steps.js +3 -0
- package/src/inference/pipelines/text/generator.d.ts +1 -0
- package/src/inference/pipelines/text/generator.js +14 -1
- package/src/inference/pipelines/text/init-rope.js +54 -6
- package/src/inference/pipelines/text/init.js +63 -15
- package/src/inference/pipelines/text/layer.js +32 -5
- package/src/inference/pipelines/text/linear-attention.js +12 -8
- package/src/inference/pipelines/text/logits/gpu.d.ts +1 -1
- package/src/inference/pipelines/text/logits/gpu.js +76 -113
- package/src/inference/pipelines/text/logits/index.js +44 -87
- package/src/inference/pipelines/text/logits/precision-policy.d.ts +11 -0
- package/src/inference/pipelines/text/logits/precision-policy.js +103 -0
- package/src/inference/pipelines/text/logits/types.d.ts +1 -0
- package/src/inference/pipelines/text/logits/utils.d.ts +2 -0
- package/src/inference/pipelines/text/logits/utils.js +11 -0
- package/src/inference/pipelines/text/lora-apply.js +6 -4
- package/src/inference/pipelines/text/moe-cpu.js +25 -0
- package/src/inference/pipelines/text/moe-gpu.js +148 -97
- package/src/inference/pipelines/text/moe-shape-validator.d.ts +33 -3
- package/src/inference/pipelines/text/moe-shape-validator.js +149 -31
- package/src/inference/pipelines/text/ops.d.ts +3 -0
- package/src/inference/pipelines/text/ops.js +26 -1
- package/src/inference/pipelines/text/per-layer-inputs.js +27 -12
- package/src/inference/pipelines/text/sampling.js +52 -10
- package/src/inference/pipelines/text/weights.js +7 -2
- package/src/inference/pipelines/text.js +24 -4
- package/src/inference/tokenizers/behavior-flags.js +23 -2
- package/src/loader/final-weights-loader.js +12 -1
- package/src/loader/tensors/tensor-loader.js +37 -3
- package/src/rules/execution-rules-contract-check.js +12 -3
- package/src/rules/inference/capability-transforms.rules.json +14 -2
- package/src/rules/inference/config.rules.json +26 -2
- package/src/rules/inference/dtype.rules.json +12 -5
- package/src/rules/inference/execution.rules.json +5 -19
- package/src/rules/inference/moe.rules.json +97 -0
- package/src/rules/loader/weights.rules.json +1 -0
- package/src/tooling/command-api-constants.js +2 -2
- package/src/tooling/command-api-family-normalizers.js +2 -1
- package/src/tooling/command-api.d.ts +1 -1
- package/src/tooling/command-runner-shared.d.ts +1 -1
- package/src/tooling/command-runner-shared.js +3 -1
- package/src/tooling/node-converter.js +20 -0
- package/src/tooling/node-source-runtime.js +5 -1
- package/src/version.js +1 -1
package/src/gpu/kernels/moe.js
CHANGED
|
@@ -470,10 +470,14 @@ export async function recordMoEBuildTokenOffsets(recorder, tokenCounts, tokenMap
|
|
|
470
470
|
|
|
471
471
|
async function executeScatterAddDynamic(recorder, expertOutputs, indices, weights, tokenOffsets, numTokens, hiddenSize, topK, options = {}) {
|
|
472
472
|
const execution = resolveExecution(recorder);
|
|
473
|
-
const { outputBuffer = null,
|
|
473
|
+
const { outputBuffer = null, perExpertScale = null } = options;
|
|
474
|
+
const weightsDtype = options.weightsDtype;
|
|
474
475
|
const ownsOutput = outputBuffer == null;
|
|
475
476
|
const hasExpertScale = perExpertScale != null;
|
|
476
477
|
|
|
478
|
+
if (weightsDtype !== 'f16' && weightsDtype !== 'f32') {
|
|
479
|
+
throw new Error(`ScatterAddDynamic requires options.weightsDtype to be "f16" or "f32", got ${String(weightsDtype)}.`);
|
|
480
|
+
}
|
|
477
481
|
if (weightsDtype === 'f16' && expertOutputs.dtype !== 'f16') {
|
|
478
482
|
throw new Error('ScatterAddDynamic f16 weights require f16 expert outputs');
|
|
479
483
|
}
|
|
@@ -552,4 +556,3 @@ async function executeScatterAddDynamic(recorder, expertOutputs, indices, weight
|
|
|
552
556
|
export async function runScatterAddDynamic(expertOutputs, indices, weights, tokenOffsets, numTokens, hiddenSize, topK, options = {}) {
|
|
553
557
|
return executeScatterAddDynamic(null, expertOutputs, indices, weights, tokenOffsets, numTokens, hiddenSize, topK, options);
|
|
554
558
|
}
|
|
555
|
-
|
|
@@ -21,8 +21,8 @@ export interface SoftmaxOptions extends OutputBufferOptions {
|
|
|
21
21
|
|
|
22
22
|
export interface SoftmaxTopKOptions {
|
|
23
23
|
normalize?: boolean;
|
|
24
|
-
inputDtype
|
|
25
|
-
weightsDtype
|
|
24
|
+
inputDtype: 'f16' | 'f32';
|
|
25
|
+
weightsDtype: 'f16' | 'f32';
|
|
26
26
|
modelType?: string | null;
|
|
27
27
|
}
|
|
28
28
|
|
|
@@ -43,7 +43,7 @@ export declare function runSoftmaxTopK(
|
|
|
43
43
|
numTokens: number,
|
|
44
44
|
numExperts: number,
|
|
45
45
|
topK: number,
|
|
46
|
-
options
|
|
46
|
+
options: SoftmaxTopKOptions
|
|
47
47
|
): Promise<{ indices: GPUBuffer; weights: GPUBuffer }>;
|
|
48
48
|
|
|
49
49
|
/**
|
|
@@ -62,10 +62,17 @@ export async function runSoftmaxTopK(logits, numTokens, numExperts, topK, option
|
|
|
62
62
|
const device = getDevice();
|
|
63
63
|
const {
|
|
64
64
|
normalize = true,
|
|
65
|
-
inputDtype = 'f32',
|
|
66
|
-
weightsDtype = 'f32',
|
|
67
65
|
modelType = null,
|
|
68
66
|
} = options;
|
|
67
|
+
const inputDtype = options.inputDtype;
|
|
68
|
+
const weightsDtype = options.weightsDtype;
|
|
69
|
+
|
|
70
|
+
if (inputDtype !== 'f16' && inputDtype !== 'f32') {
|
|
71
|
+
throw new Error(`SoftmaxTopK requires options.inputDtype to be "f16" or "f32", got ${String(inputDtype)}.`);
|
|
72
|
+
}
|
|
73
|
+
if (weightsDtype !== 'f16' && weightsDtype !== 'f32') {
|
|
74
|
+
throw new Error(`SoftmaxTopK requires options.weightsDtype to be "f16" or "f32", got ${String(weightsDtype)}.`);
|
|
75
|
+
}
|
|
69
76
|
|
|
70
77
|
if (weightsDtype === 'f16' && inputDtype !== 'f16') {
|
|
71
78
|
throw new Error('SoftmaxTopK f16 weights require f16 logits');
|
|
@@ -435,7 +435,7 @@ function retainSharedBuffer(cache, key, factory) {
|
|
|
435
435
|
|
|
436
436
|
export function retainTurboQuantSharedBuffers(device, options = {}) {
|
|
437
437
|
const headDim = Number(options.headDim);
|
|
438
|
-
const bitWidth = Number(options.bitWidth
|
|
438
|
+
const bitWidth = Number(options.bitWidth);
|
|
439
439
|
const prodMode = options.prodMode === true;
|
|
440
440
|
if (!device) {
|
|
441
441
|
throw new Error('TurboQuant shared buffers require a GPU device.');
|
package/src/gpu/profiler.js
CHANGED
|
@@ -65,7 +65,10 @@ export class GPUProfiler {
|
|
|
65
65
|
this.#queryCapacity = runtimeProfiler.queryCapacity;
|
|
66
66
|
this.#maxSamples = runtimeProfiler.maxSamples;
|
|
67
67
|
this.#maxDurationMs = runtimeProfiler.maxDurationMs;
|
|
68
|
-
this.#maxHistoryLabels =
|
|
68
|
+
this.#maxHistoryLabels = requirePositiveInteger(
|
|
69
|
+
runtimeProfiler.maxHistoryLabels,
|
|
70
|
+
'runtime.shared.debug.profiler.maxHistoryLabels'
|
|
71
|
+
);
|
|
69
72
|
|
|
70
73
|
// Initialize query resources if timestamp queries available
|
|
71
74
|
if (this.#hasTimestampQuery && this.#device) {
|
|
@@ -368,5 +371,12 @@ export class GPUProfiler {
|
|
|
368
371
|
}
|
|
369
372
|
}
|
|
370
373
|
|
|
374
|
+
function requirePositiveInteger(value, label) {
|
|
375
|
+
if (!Number.isInteger(value) || value <= 0) {
|
|
376
|
+
throw new Error(`${label} must be a positive integer.`);
|
|
377
|
+
}
|
|
378
|
+
return value;
|
|
379
|
+
}
|
|
380
|
+
|
|
371
381
|
|
|
372
382
|
export default GPUProfiler;
|
|
@@ -99,7 +99,10 @@ export async function resolveKernelPathForModel(options = {}) {
|
|
|
99
99
|
|
|
100
100
|
if (!manifest) return null;
|
|
101
101
|
|
|
102
|
-
const modelConfig = parseModelConfigFromManifest(
|
|
102
|
+
const modelConfig = parseModelConfigFromManifest(
|
|
103
|
+
manifest,
|
|
104
|
+
runtimeConfig?.inference?.modelOverrides ?? null
|
|
105
|
+
);
|
|
103
106
|
const kernelPathState = resolveKernelPathState({
|
|
104
107
|
manifest,
|
|
105
108
|
runtimeConfig,
|
|
@@ -6,6 +6,11 @@ export declare function getDefaultEmbeddingSemanticFixtures(): {
|
|
|
6
6
|
minPairAcc: number;
|
|
7
7
|
pairMargin: number;
|
|
8
8
|
};
|
|
9
|
+
export declare function getDefaultRerankSemanticFixtures(): {
|
|
10
|
+
cases: Array<Record<string, unknown>>;
|
|
11
|
+
minPairAcc: number;
|
|
12
|
+
minScoreMargin: number;
|
|
13
|
+
};
|
|
9
14
|
export declare function resolveBenchmarkRunSettings(
|
|
10
15
|
runtimeConfig: Record<string, unknown>,
|
|
11
16
|
source?: Record<string, unknown> | null
|
|
@@ -56,6 +61,29 @@ export declare function runEmbeddingSemanticChecks(
|
|
|
56
61
|
pipeline: Record<string, unknown>,
|
|
57
62
|
options?: Record<string, unknown> | null
|
|
58
63
|
): Promise<Record<string, unknown>>;
|
|
64
|
+
export declare function resolveRerankScoringConfig(
|
|
65
|
+
pipeline: Record<string, unknown>
|
|
66
|
+
): Record<string, unknown>;
|
|
67
|
+
export declare function formatRerankPrompt(
|
|
68
|
+
query: string,
|
|
69
|
+
document: string,
|
|
70
|
+
scoringConfig: Record<string, unknown>
|
|
71
|
+
): string;
|
|
72
|
+
export declare function scoreRerankDocument(
|
|
73
|
+
pipeline: Record<string, unknown>,
|
|
74
|
+
query: string,
|
|
75
|
+
document: string,
|
|
76
|
+
scoringConfig?: Record<string, unknown> | null
|
|
77
|
+
): Promise<Record<string, unknown>>;
|
|
78
|
+
export declare function runRerank(
|
|
79
|
+
pipeline: Record<string, unknown>,
|
|
80
|
+
runtimeConfig: Record<string, unknown>,
|
|
81
|
+
runOverrides?: Record<string, unknown> | null
|
|
82
|
+
): Promise<Record<string, unknown>>;
|
|
83
|
+
export declare function runRerankSemanticChecks(
|
|
84
|
+
pipeline: Record<string, unknown>,
|
|
85
|
+
options?: Record<string, unknown> | null
|
|
86
|
+
): Promise<Record<string, unknown>>;
|
|
59
87
|
export declare function isCoherentOutput(tokens: Array<unknown>, output: unknown): boolean;
|
|
60
88
|
|
|
61
89
|
export interface ReferenceLogitsDigest {
|