doppler-gpu 0.2.2 → 0.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -0
- package/assets/doppler.svg +1601 -0
- package/package.json +6 -5
- package/src/cli/doppler-quickstart.js +16 -2
- package/src/cli/doppler-serve.js +0 -0
- package/src/client/doppler-registry.json +17 -1
- package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +1 -1
- package/src/config/kernels/registry.json +73 -0
- package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +32 -0
- package/src/config/schema/benchmark.schema.d.ts +2 -0
- package/src/config/schema/benchmark.schema.js +1 -0
- package/src/config/schema/doppler.schema.d.ts +9 -1
- package/src/config/schema/doppler.schema.js +8 -1
- package/src/config/schema/index.d.ts +2 -0
- package/src/config/schema/index.js +1 -0
- package/src/config/schema/manifest.schema.d.ts +5 -0
- package/src/converter/quantizer.js +5 -4
- package/src/formats/rdrr/source-transform-contract.js +390 -0
- package/src/formats/rdrr/types.d.ts +8 -1
- package/src/formats/tflite/types.d.ts +8 -0
- package/src/formats/tflite/types.js +45 -0
- package/src/gpu/command-recorder.d.ts +7 -1
- package/src/gpu/command-recorder.js +41 -2
- package/src/gpu/kernels/activation-static-qdq.d.ts +22 -0
- package/src/gpu/kernels/activation-static-qdq.js +69 -0
- package/src/gpu/kernels/activation_static_qdq.wgsl +23 -0
- package/src/gpu/kernels/index.d.ts +7 -0
- package/src/gpu/kernels/index.js +6 -0
- package/src/inference/browser-harness-text-helpers.d.ts +1 -0
- package/src/inference/browser-harness-text-helpers.js +44 -4
- package/src/inference/browser-harness.js +7 -0
- package/src/inference/pipelines/context.js +4 -2
- package/src/inference/pipelines/text/attention/types.js +9 -4
- package/src/inference/pipelines/text/embed.d.ts +1 -0
- package/src/inference/pipelines/text/embed.js +10 -4
- package/src/inference/pipelines/text/generator-helpers.d.ts +6 -0
- package/src/inference/pipelines/text/generator-helpers.js +20 -1
- package/src/inference/pipelines/text/generator-steps.js +17 -8
- package/src/inference/pipelines/text/generator.js +51 -3
- package/src/inference/pipelines/text/init.d.ts +3 -0
- package/src/inference/pipelines/text/init.js +24 -3
- package/src/inference/pipelines/text/logits/gpu.d.ts +5 -4
- package/src/inference/pipelines/text/logits/gpu.js +146 -11
- package/src/inference/pipelines/text/per-layer-inputs.js +320 -57
- package/src/inference/pipelines/text/probes.js +79 -1
- package/src/inference/pipelines/text/stage-names.js +9 -0
- package/src/inference/pipelines/text.js +10 -2
- package/src/loader/embedding-loader.js +20 -5
- package/src/loader/loader-types.d.ts +2 -0
- package/src/loader/manifest-config.d.ts +2 -7
- package/src/loader/per-layer-input-loader.js +114 -1
- package/src/loader/tensors/source-transform.d.ts +15 -0
- package/src/loader/tensors/source-transform.js +312 -34
- package/src/loader/tensors/tensor-reader.js +28 -2
- package/src/storage/artifact-storage-context.d.ts +1 -0
- package/src/storage/artifact-storage-context.js +158 -72
- package/src/storage/preflight.js +11 -0
- package/src/storage/quickstart-downloader.js +15 -4
- package/src/tooling/litert-package-runtime.js +369 -14
- package/src/tooling/node-source-runtime.js +256 -140
- package/src/tooling/source-runtime-bundle.d.ts +25 -0
- package/src/tooling/source-runtime-bundle.js +53 -1
- package/src/tooling/source-runtime-materializer.js +3 -1
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "doppler-gpu",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.4.2",
|
|
4
4
|
"description": "Browser-native WebGPU inference engine for local intent and inference loops",
|
|
5
5
|
"main": "src/index.js",
|
|
6
6
|
"types": "src/index.d.ts",
|
|
@@ -93,18 +93,17 @@
|
|
|
93
93
|
"verify:gemma-3-270m-it-wq4k-ef16": "node tools/run-registry-verify.js gemma-3-270m-it-wq4k-ef16",
|
|
94
94
|
"verify:gemma-3-270m-it-wq4k-ef16-hf16": "node tools/run-registry-verify.js gemma-3-270m-it-wq4k-ef16-hf16",
|
|
95
95
|
"verify:gemma-3-270m-it-wq4k-ef16-hf16-f32": "node tools/run-registry-verify.js gemma-3-270m-it-wq4k-ef16-hf16-f32",
|
|
96
|
+
"verify:gemma-4-e2b-it-q4k-ehf16-af32": "node tools/run-registry-verify.js gemma-4-e2b-it-q4k-ehf16-af32",
|
|
96
97
|
"verify:gemma3-1b": "node tools/run-registry-verify.js gemma3-1b",
|
|
97
98
|
"verify:gemma3-270m": "node tools/run-registry-verify.js gemma3-270m",
|
|
99
|
+
"verify:gemma4-e2b": "node tools/run-registry-verify.js gemma4-e2b",
|
|
98
100
|
"verify:google-embeddinggemma-300m": "node tools/run-registry-verify.js google-embeddinggemma-300m",
|
|
99
101
|
"verify:google-embeddinggemma-300m-q4k-ehf16-af32": "node tools/run-registry-verify.js google-embeddinggemma-300m-q4k-ehf16-af32",
|
|
100
102
|
"verify:google-embeddinggemma-300m-wq4k-ef16": "node tools/run-registry-verify.js google-embeddinggemma-300m-wq4k-ef16",
|
|
101
103
|
"verify:google-gemma-3-1b-it": "node tools/run-registry-verify.js google-gemma-3-1b-it",
|
|
102
104
|
"verify:google-gemma-3-270m-it": "node tools/run-registry-verify.js google-gemma-3-270m-it",
|
|
105
|
+
"verify:google-gemma-4-e2b-it": "node tools/run-registry-verify.js google-gemma-4-e2b-it",
|
|
103
106
|
"verify:google-translategemma-4b-it": "node tools/run-registry-verify.js google-translategemma-4b-it",
|
|
104
|
-
"verify:lfm2-1.2b": "node tools/run-registry-verify.js lfm2-1.2b",
|
|
105
|
-
"verify:lfm2-5-1-2b-instruct-q4k-ehf16-af32": "node tools/run-registry-verify.js lfm2-5-1-2b-instruct-q4k-ehf16-af32",
|
|
106
|
-
"verify:lfm2.5-1.2b": "node tools/run-registry-verify.js lfm2.5-1.2b",
|
|
107
|
-
"verify:liquidai-lfm2.5-1.2b-instruct": "node tools/run-registry-verify.js liquidai-lfm2.5-1.2b-instruct",
|
|
108
107
|
"verify:qwen-3-5-0-8b": "node tools/run-registry-verify.js qwen-3-5-0-8b",
|
|
109
108
|
"verify:qwen-3-5-0-8b-q4k-ehaf16": "node tools/run-registry-verify.js qwen-3-5-0-8b-q4k-ehaf16",
|
|
110
109
|
"verify:qwen-3-5-0-8b-wq4k-ef16-hf16-f16": "node tools/run-registry-verify.js qwen-3-5-0-8b-wq4k-ef16-hf16-f16",
|
|
@@ -179,6 +178,7 @@
|
|
|
179
178
|
"access": "public"
|
|
180
179
|
},
|
|
181
180
|
"files": [
|
|
181
|
+
"assets",
|
|
182
182
|
"src",
|
|
183
183
|
"src/gpu/kernels/*.wgsl",
|
|
184
184
|
"README.md",
|
|
@@ -203,6 +203,7 @@
|
|
|
203
203
|
"typescript": "^6.0.2"
|
|
204
204
|
},
|
|
205
205
|
"optionalDependencies": {
|
|
206
|
+
"doe-gpu": "^0.4.2",
|
|
206
207
|
"webgpu": "^0.4.0"
|
|
207
208
|
}
|
|
208
209
|
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
|
|
3
|
+
import { realpathSync } from 'node:fs';
|
|
3
4
|
import fs from 'node:fs/promises';
|
|
4
5
|
import path from 'node:path';
|
|
5
6
|
import process from 'node:process';
|
|
@@ -201,6 +202,18 @@ function formatDeviceSummary(deviceInfo) {
|
|
|
201
202
|
return parts.length > 0 ? parts.join(' / ') : 'unknown-device';
|
|
202
203
|
}
|
|
203
204
|
|
|
205
|
+
export function requireQuickstartContent(result) {
|
|
206
|
+
const content = asStringOrNull(result?.content);
|
|
207
|
+
if (content === null) {
|
|
208
|
+
const modelId = asStringOrNull(result?.modelId) ?? 'unknown';
|
|
209
|
+
throw new Error(
|
|
210
|
+
`Quickstart model "${modelId}" returned empty output. `
|
|
211
|
+
+ 'Treat this as a runtime failure and inspect the preceding load/generation error.'
|
|
212
|
+
);
|
|
213
|
+
}
|
|
214
|
+
return content;
|
|
215
|
+
}
|
|
216
|
+
|
|
204
217
|
export async function runQuickstart(settings) {
|
|
205
218
|
const entry = await resolveQuickstartModel(settings.model);
|
|
206
219
|
if (!entry.modes.includes('text')) {
|
|
@@ -257,7 +270,8 @@ async function printModelList(jsonOutput) {
|
|
|
257
270
|
}
|
|
258
271
|
|
|
259
272
|
function printQuickstartResult(result) {
|
|
260
|
-
|
|
273
|
+
const content = requireQuickstartContent(result);
|
|
274
|
+
console.log(content);
|
|
261
275
|
console.error(
|
|
262
276
|
`[doppler-gpu] model=${result.modelId} elapsed=${result.elapsedMs}ms device=${formatDeviceSummary(result.deviceInfo)}`
|
|
263
277
|
);
|
|
@@ -288,7 +302,7 @@ function isMainModule(metaUrl) {
|
|
|
288
302
|
if (!entryPath) {
|
|
289
303
|
return false;
|
|
290
304
|
}
|
|
291
|
-
return path.resolve(fileURLToPath(metaUrl)) === path.resolve(entryPath);
|
|
305
|
+
return realpathSync(path.resolve(fileURLToPath(metaUrl))) === realpathSync(path.resolve(entryPath));
|
|
292
306
|
}
|
|
293
307
|
|
|
294
308
|
if (isMainModule(import.meta.url)) {
|
package/src/cli/doppler-serve.js
CHANGED
|
File without changes
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
],
|
|
18
18
|
"hf": {
|
|
19
19
|
"repoId": "Clocksmith/rdrr",
|
|
20
|
-
"revision": "
|
|
20
|
+
"revision": "732b2bee08cfe315e5c0df5af30e7ad795721b48",
|
|
21
21
|
"path": "models/gemma-3-270m-it-q4k-ehf16-af32"
|
|
22
22
|
}
|
|
23
23
|
},
|
|
@@ -54,6 +54,22 @@
|
|
|
54
54
|
"path": "models/gemma-3-1b-it-q4k-ehf16-af32"
|
|
55
55
|
}
|
|
56
56
|
},
|
|
57
|
+
{
|
|
58
|
+
"modelId": "gemma-4-e2b-it-q4k-ehf16-af32",
|
|
59
|
+
"aliases": [
|
|
60
|
+
"gemma4-e2b",
|
|
61
|
+
"google/gemma-4-e2b-it"
|
|
62
|
+
],
|
|
63
|
+
"modes": [
|
|
64
|
+
"text",
|
|
65
|
+
"vision"
|
|
66
|
+
],
|
|
67
|
+
"hf": {
|
|
68
|
+
"repoId": "Clocksmith/rdrr",
|
|
69
|
+
"revision": "88fa42770918e0c61bfcd67c6a54952e079b5d36",
|
|
70
|
+
"path": "models/gemma-4-e2b-it-q4k-ehf16-af32"
|
|
71
|
+
}
|
|
72
|
+
},
|
|
57
73
|
{
|
|
58
74
|
"modelId": "qwen-3-5-0-8b-q4k-ehaf16",
|
|
59
75
|
"aliases": [
|
|
@@ -4467,6 +4467,79 @@
|
|
|
4467
4467
|
}
|
|
4468
4468
|
}
|
|
4469
4469
|
},
|
|
4470
|
+
"activation_static_qdq": {
|
|
4471
|
+
"description": "Static activation quantize/dequantize kernels",
|
|
4472
|
+
"baseBindings": [
|
|
4473
|
+
{
|
|
4474
|
+
"index": 0,
|
|
4475
|
+
"name": "uniforms",
|
|
4476
|
+
"type": "uniform"
|
|
4477
|
+
},
|
|
4478
|
+
{
|
|
4479
|
+
"index": 1,
|
|
4480
|
+
"name": "input",
|
|
4481
|
+
"type": "read-only-storage"
|
|
4482
|
+
},
|
|
4483
|
+
{
|
|
4484
|
+
"index": 2,
|
|
4485
|
+
"name": "output",
|
|
4486
|
+
"type": "storage"
|
|
4487
|
+
}
|
|
4488
|
+
],
|
|
4489
|
+
"baseUniforms": {
|
|
4490
|
+
"size": 32,
|
|
4491
|
+
"fields": [
|
|
4492
|
+
{
|
|
4493
|
+
"name": "size",
|
|
4494
|
+
"type": "u32",
|
|
4495
|
+
"offset": 0
|
|
4496
|
+
},
|
|
4497
|
+
{
|
|
4498
|
+
"name": "_pad0",
|
|
4499
|
+
"type": "u32",
|
|
4500
|
+
"offset": 4
|
|
4501
|
+
},
|
|
4502
|
+
{
|
|
4503
|
+
"name": "scale",
|
|
4504
|
+
"type": "f32",
|
|
4505
|
+
"offset": 8
|
|
4506
|
+
},
|
|
4507
|
+
{
|
|
4508
|
+
"name": "invScale",
|
|
4509
|
+
"type": "f32",
|
|
4510
|
+
"offset": 12
|
|
4511
|
+
},
|
|
4512
|
+
{
|
|
4513
|
+
"name": "qmin",
|
|
4514
|
+
"type": "f32",
|
|
4515
|
+
"offset": 16
|
|
4516
|
+
},
|
|
4517
|
+
{
|
|
4518
|
+
"name": "qmax",
|
|
4519
|
+
"type": "f32",
|
|
4520
|
+
"offset": 20
|
|
4521
|
+
}
|
|
4522
|
+
]
|
|
4523
|
+
},
|
|
4524
|
+
"variants": {
|
|
4525
|
+
"default": {
|
|
4526
|
+
"wgsl": "activation_static_qdq.wgsl",
|
|
4527
|
+
"entryPoint": "main",
|
|
4528
|
+
"workgroup": [
|
|
4529
|
+
256,
|
|
4530
|
+
1,
|
|
4531
|
+
1
|
|
4532
|
+
],
|
|
4533
|
+
"requires": [],
|
|
4534
|
+
"reachability": {
|
|
4535
|
+
"status": "unused",
|
|
4536
|
+
"inlineConfigs": [],
|
|
4537
|
+
"ruleChains": [],
|
|
4538
|
+
"wgslOnDisk": true
|
|
4539
|
+
}
|
|
4540
|
+
}
|
|
4541
|
+
}
|
|
4542
|
+
},
|
|
4470
4543
|
"energy_eval": {
|
|
4471
4544
|
"description": "Energy evaluation kernels",
|
|
4472
4545
|
"baseBindings": [
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
{
|
|
2
|
+
"id": "experiments/bench/gemma4-bench-q4k",
|
|
3
|
+
"name": "gemma4-bench-q4k",
|
|
4
|
+
"description": "Benchmark run for Gemma 4 E2B Q4K with deterministic generation settings.",
|
|
5
|
+
"intent": "calibrate",
|
|
6
|
+
"stability": "experimental",
|
|
7
|
+
"owner": "doppler-core",
|
|
8
|
+
"createdAtUtc": "2026-04-14T00:00:00Z",
|
|
9
|
+
"model": "gemma-4-e2b-it-q4k-ehf16-af32",
|
|
10
|
+
"extends": "profiles/gemma4-e2b-throughput",
|
|
11
|
+
"runtime": {
|
|
12
|
+
"shared": {
|
|
13
|
+
"benchmark": {
|
|
14
|
+
"run": {
|
|
15
|
+
"promptName": "xs",
|
|
16
|
+
"customPrompt": "The sky is clear and beautiful.",
|
|
17
|
+
"maxNewTokens": 256,
|
|
18
|
+
"warmupRuns": 1,
|
|
19
|
+
"timedRuns": 3,
|
|
20
|
+
"sampling": {
|
|
21
|
+
"temperature": 0,
|
|
22
|
+
"topK": 1,
|
|
23
|
+
"topP": 1,
|
|
24
|
+
"seed": 12345
|
|
25
|
+
},
|
|
26
|
+
"profile": false,
|
|
27
|
+
"captureMemoryTimeSeries": false
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
}
|
|
@@ -6,6 +6,7 @@ export interface BenchmarkRunConfig {
|
|
|
6
6
|
promptName: string;
|
|
7
7
|
customPrompt: string | null;
|
|
8
8
|
maxNewTokens: number;
|
|
9
|
+
seed: number | null;
|
|
9
10
|
runType: string;
|
|
10
11
|
warmupRuns: number;
|
|
11
12
|
timedRuns: number;
|
|
@@ -13,6 +14,7 @@ export interface BenchmarkRunConfig {
|
|
|
13
14
|
temperature: number;
|
|
14
15
|
topK: number;
|
|
15
16
|
topP: number;
|
|
17
|
+
seed?: number;
|
|
16
18
|
};
|
|
17
19
|
debug: boolean;
|
|
18
20
|
profile: boolean;
|
|
@@ -19,6 +19,13 @@ import type { ExecutionV1PatchSchema, ExecutionV1SessionSchema } from './executi
|
|
|
19
19
|
import type { SharedRuntimeConfigSchema } from './shared-runtime.schema.js';
|
|
20
20
|
import type { EmulationConfigSchema } from './emulation.schema.js';
|
|
21
21
|
|
|
22
|
+
export interface LargeWeightConfigSchema {
|
|
23
|
+
enabled: boolean;
|
|
24
|
+
safetyRatio: number;
|
|
25
|
+
preferF16: boolean;
|
|
26
|
+
lmHeadChunkRows: number | null;
|
|
27
|
+
}
|
|
28
|
+
|
|
22
29
|
export interface RuntimeInferenceConfigSchema {
|
|
23
30
|
prompt?: string | null;
|
|
24
31
|
debugTokens?: boolean;
|
|
@@ -26,7 +33,7 @@ export interface RuntimeInferenceConfigSchema {
|
|
|
26
33
|
sampling: Record<string, unknown>;
|
|
27
34
|
compute: Record<string, unknown>;
|
|
28
35
|
tokenizer: Record<string, unknown>;
|
|
29
|
-
largeWeights:
|
|
36
|
+
largeWeights: LargeWeightConfigSchema;
|
|
30
37
|
kvcache: Record<string, unknown>;
|
|
31
38
|
diffusion: Record<string, unknown>;
|
|
32
39
|
energy: Record<string, unknown>;
|
|
@@ -64,6 +71,7 @@ export interface RuntimeConfigSchema {
|
|
|
64
71
|
}
|
|
65
72
|
|
|
66
73
|
/** Default runtime configuration */
|
|
74
|
+
export declare const DEFAULT_LARGE_WEIGHT_CONFIG: LargeWeightConfigSchema;
|
|
67
75
|
export declare const DEFAULT_RUNTIME_CONFIG: RuntimeConfigSchema;
|
|
68
76
|
|
|
69
77
|
/**
|
|
@@ -24,6 +24,13 @@ export const DEFAULT_CHAT_TEMPLATE_CONFIG = Object.freeze({
|
|
|
24
24
|
thinking: false,
|
|
25
25
|
});
|
|
26
26
|
|
|
27
|
+
export const DEFAULT_LARGE_WEIGHT_CONFIG = Object.freeze({
|
|
28
|
+
enabled: true,
|
|
29
|
+
safetyRatio: 0.9,
|
|
30
|
+
preferF16: true,
|
|
31
|
+
lmHeadChunkRows: null,
|
|
32
|
+
});
|
|
33
|
+
|
|
27
34
|
export const DEFAULT_RUNTIME_CONFIG = {
|
|
28
35
|
shared: DEFAULT_SHARED_RUNTIME_CONFIG,
|
|
29
36
|
loading: DEFAULT_LOADING_CONFIG,
|
|
@@ -39,7 +46,7 @@ export const DEFAULT_RUNTIME_CONFIG = {
|
|
|
39
46
|
},
|
|
40
47
|
compute: {},
|
|
41
48
|
tokenizer: {},
|
|
42
|
-
largeWeights:
|
|
49
|
+
largeWeights: DEFAULT_LARGE_WEIGHT_CONFIG,
|
|
43
50
|
kvcache: DEFAULT_KVCACHE_CONFIG,
|
|
44
51
|
diffusion: {},
|
|
45
52
|
energy: {},
|
|
@@ -850,10 +850,12 @@ export {
|
|
|
850
850
|
// =============================================================================
|
|
851
851
|
export {
|
|
852
852
|
// Types
|
|
853
|
+
type LargeWeightConfigSchema,
|
|
853
854
|
type RuntimeConfigSchema,
|
|
854
855
|
type DopplerConfigSchema,
|
|
855
856
|
|
|
856
857
|
// Defaults
|
|
858
|
+
DEFAULT_LARGE_WEIGHT_CONFIG,
|
|
857
859
|
DEFAULT_RUNTIME_CONFIG,
|
|
858
860
|
DEFAULT_DOPPLER_CONFIG,
|
|
859
861
|
|
|
@@ -437,6 +437,11 @@ export interface TensorSchema {
|
|
|
437
437
|
scaleDivisor?: number;
|
|
438
438
|
storageShape: [number, number];
|
|
439
439
|
quantAxis: 0 | 1;
|
|
440
|
+
scaleCompanionDtype?: 'UINT8';
|
|
441
|
+
scaleCompanionDequant?: {
|
|
442
|
+
scale: number;
|
|
443
|
+
zeroPoint: number;
|
|
444
|
+
};
|
|
440
445
|
scaleSource: {
|
|
441
446
|
shard: number;
|
|
442
447
|
shardIndex?: number;
|
|
@@ -9,11 +9,12 @@ import {
|
|
|
9
9
|
// Re-export for backward compatibility
|
|
10
10
|
export { QK_K, K_SCALE_SIZE, QK4_K_BLOCK_SIZE };
|
|
11
11
|
|
|
12
|
+
const f32ToF16ScratchF32 = new Float32Array(1);
|
|
13
|
+
const f32ToF16ScratchU32 = new Uint32Array(f32ToF16ScratchF32.buffer);
|
|
14
|
+
|
|
12
15
|
export function float32ToFloat16(value) {
|
|
13
|
-
|
|
14
|
-
const
|
|
15
|
-
floatView[0] = value;
|
|
16
|
-
const f = int32View[0];
|
|
16
|
+
f32ToF16ScratchF32[0] = value;
|
|
17
|
+
const f = f32ToF16ScratchU32[0];
|
|
17
18
|
|
|
18
19
|
const sign = (f >> 31) & 0x1;
|
|
19
20
|
let exp = (f >> 23) & 0xff;
|