doppler-gpu 0.2.1 → 0.4.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. package/CHANGELOG.md +8 -0
  2. package/README.md +9 -1
  3. package/assets/doppler.svg +1601 -0
  4. package/package.json +6 -5
  5. package/src/cli/doppler-quickstart.js +16 -2
  6. package/src/cli/doppler-serve.js +0 -0
  7. package/src/client/doppler-registry.json +17 -1
  8. package/src/config/conversion/gemma4/gemma-4-e2b-it-q4k-ehf16-af32.json +1 -1
  9. package/src/config/kernels/registry.json +73 -0
  10. package/src/config/runtime/experiments/bench/gemma4-bench-q4k.json +32 -0
  11. package/src/config/schema/benchmark.schema.d.ts +2 -0
  12. package/src/config/schema/benchmark.schema.js +1 -0
  13. package/src/config/schema/doppler.schema.d.ts +9 -1
  14. package/src/config/schema/doppler.schema.js +8 -1
  15. package/src/config/schema/index.d.ts +2 -0
  16. package/src/config/schema/index.js +1 -0
  17. package/src/config/schema/manifest.schema.d.ts +5 -0
  18. package/src/converter/quantizer.js +5 -4
  19. package/src/formats/rdrr/source-transform-contract.js +390 -0
  20. package/src/formats/rdrr/types.d.ts +8 -1
  21. package/src/formats/tflite/types.d.ts +8 -0
  22. package/src/formats/tflite/types.js +45 -0
  23. package/src/gpu/command-recorder.d.ts +7 -1
  24. package/src/gpu/command-recorder.js +41 -2
  25. package/src/gpu/kernels/activation-static-qdq.d.ts +22 -0
  26. package/src/gpu/kernels/activation-static-qdq.js +69 -0
  27. package/src/gpu/kernels/activation_static_qdq.wgsl +23 -0
  28. package/src/gpu/kernels/index.d.ts +7 -0
  29. package/src/gpu/kernels/index.js +6 -0
  30. package/src/inference/browser-harness-text-helpers.d.ts +1 -0
  31. package/src/inference/browser-harness-text-helpers.js +44 -4
  32. package/src/inference/browser-harness.js +7 -0
  33. package/src/inference/pipelines/context.js +4 -2
  34. package/src/inference/pipelines/text/attention/types.js +9 -4
  35. package/src/inference/pipelines/text/embed.d.ts +1 -0
  36. package/src/inference/pipelines/text/embed.js +10 -4
  37. package/src/inference/pipelines/text/generator-helpers.d.ts +6 -0
  38. package/src/inference/pipelines/text/generator-helpers.js +20 -1
  39. package/src/inference/pipelines/text/generator-steps.js +17 -8
  40. package/src/inference/pipelines/text/generator.js +51 -3
  41. package/src/inference/pipelines/text/init.d.ts +3 -0
  42. package/src/inference/pipelines/text/init.js +24 -3
  43. package/src/inference/pipelines/text/logits/gpu.d.ts +5 -4
  44. package/src/inference/pipelines/text/logits/gpu.js +146 -11
  45. package/src/inference/pipelines/text/per-layer-inputs.js +320 -57
  46. package/src/inference/pipelines/text/probes.js +79 -1
  47. package/src/inference/pipelines/text/stage-names.js +9 -0
  48. package/src/inference/pipelines/text.js +10 -2
  49. package/src/loader/embedding-loader.js +20 -5
  50. package/src/loader/loader-types.d.ts +2 -0
  51. package/src/loader/manifest-config.d.ts +2 -7
  52. package/src/loader/per-layer-input-loader.js +114 -1
  53. package/src/loader/tensors/source-transform.d.ts +15 -0
  54. package/src/loader/tensors/source-transform.js +312 -34
  55. package/src/loader/tensors/tensor-reader.js +28 -2
  56. package/src/storage/artifact-storage-context.d.ts +1 -0
  57. package/src/storage/artifact-storage-context.js +158 -72
  58. package/src/storage/preflight.js +11 -0
  59. package/src/storage/quickstart-downloader.js +15 -4
  60. package/src/tooling/litert-package-runtime.js +369 -14
  61. package/src/tooling/node-source-runtime.js +256 -140
  62. package/src/tooling/source-runtime-bundle.d.ts +25 -0
  63. package/src/tooling/source-runtime-bundle.js +53 -1
  64. package/src/tooling/source-runtime-materializer.js +3 -1
  65. package/src/version.js +1 -1
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "doppler-gpu",
3
- "version": "0.2.1",
3
+ "version": "0.4.2",
4
4
  "description": "Browser-native WebGPU inference engine for local intent and inference loops",
5
5
  "main": "src/index.js",
6
6
  "types": "src/index.d.ts",
@@ -93,18 +93,17 @@
93
93
  "verify:gemma-3-270m-it-wq4k-ef16": "node tools/run-registry-verify.js gemma-3-270m-it-wq4k-ef16",
94
94
  "verify:gemma-3-270m-it-wq4k-ef16-hf16": "node tools/run-registry-verify.js gemma-3-270m-it-wq4k-ef16-hf16",
95
95
  "verify:gemma-3-270m-it-wq4k-ef16-hf16-f32": "node tools/run-registry-verify.js gemma-3-270m-it-wq4k-ef16-hf16-f32",
96
+ "verify:gemma-4-e2b-it-q4k-ehf16-af32": "node tools/run-registry-verify.js gemma-4-e2b-it-q4k-ehf16-af32",
96
97
  "verify:gemma3-1b": "node tools/run-registry-verify.js gemma3-1b",
97
98
  "verify:gemma3-270m": "node tools/run-registry-verify.js gemma3-270m",
99
+ "verify:gemma4-e2b": "node tools/run-registry-verify.js gemma4-e2b",
98
100
  "verify:google-embeddinggemma-300m": "node tools/run-registry-verify.js google-embeddinggemma-300m",
99
101
  "verify:google-embeddinggemma-300m-q4k-ehf16-af32": "node tools/run-registry-verify.js google-embeddinggemma-300m-q4k-ehf16-af32",
100
102
  "verify:google-embeddinggemma-300m-wq4k-ef16": "node tools/run-registry-verify.js google-embeddinggemma-300m-wq4k-ef16",
101
103
  "verify:google-gemma-3-1b-it": "node tools/run-registry-verify.js google-gemma-3-1b-it",
102
104
  "verify:google-gemma-3-270m-it": "node tools/run-registry-verify.js google-gemma-3-270m-it",
105
+ "verify:google-gemma-4-e2b-it": "node tools/run-registry-verify.js google-gemma-4-e2b-it",
103
106
  "verify:google-translategemma-4b-it": "node tools/run-registry-verify.js google-translategemma-4b-it",
104
- "verify:lfm2-1.2b": "node tools/run-registry-verify.js lfm2-1.2b",
105
- "verify:lfm2-5-1-2b-instruct-q4k-ehf16-af32": "node tools/run-registry-verify.js lfm2-5-1-2b-instruct-q4k-ehf16-af32",
106
- "verify:lfm2.5-1.2b": "node tools/run-registry-verify.js lfm2.5-1.2b",
107
- "verify:liquidai-lfm2.5-1.2b-instruct": "node tools/run-registry-verify.js liquidai-lfm2.5-1.2b-instruct",
108
107
  "verify:qwen-3-5-0-8b": "node tools/run-registry-verify.js qwen-3-5-0-8b",
109
108
  "verify:qwen-3-5-0-8b-q4k-ehaf16": "node tools/run-registry-verify.js qwen-3-5-0-8b-q4k-ehaf16",
110
109
  "verify:qwen-3-5-0-8b-wq4k-ef16-hf16-f16": "node tools/run-registry-verify.js qwen-3-5-0-8b-wq4k-ef16-hf16-f16",
@@ -179,6 +178,7 @@
179
178
  "access": "public"
180
179
  },
181
180
  "files": [
181
+ "assets",
182
182
  "src",
183
183
  "src/gpu/kernels/*.wgsl",
184
184
  "README.md",
@@ -203,6 +203,7 @@
203
203
  "typescript": "^6.0.2"
204
204
  },
205
205
  "optionalDependencies": {
206
+ "doe-gpu": "^0.4.2",
206
207
  "webgpu": "^0.4.0"
207
208
  }
208
209
  }
@@ -1,5 +1,6 @@
1
1
  #!/usr/bin/env node
2
2
 
3
+ import { realpathSync } from 'node:fs';
3
4
  import fs from 'node:fs/promises';
4
5
  import path from 'node:path';
5
6
  import process from 'node:process';
@@ -201,6 +202,18 @@ function formatDeviceSummary(deviceInfo) {
201
202
  return parts.length > 0 ? parts.join(' / ') : 'unknown-device';
202
203
  }
203
204
 
205
+ export function requireQuickstartContent(result) {
206
+ const content = asStringOrNull(result?.content);
207
+ if (content === null) {
208
+ const modelId = asStringOrNull(result?.modelId) ?? 'unknown';
209
+ throw new Error(
210
+ `Quickstart model "${modelId}" returned empty output. `
211
+ + 'Treat this as a runtime failure and inspect the preceding load/generation error.'
212
+ );
213
+ }
214
+ return content;
215
+ }
216
+
204
217
  export async function runQuickstart(settings) {
205
218
  const entry = await resolveQuickstartModel(settings.model);
206
219
  if (!entry.modes.includes('text')) {
@@ -257,7 +270,8 @@ async function printModelList(jsonOutput) {
257
270
  }
258
271
 
259
272
  function printQuickstartResult(result) {
260
- console.log(result.content);
273
+ const content = requireQuickstartContent(result);
274
+ console.log(content);
261
275
  console.error(
262
276
  `[doppler-gpu] model=${result.modelId} elapsed=${result.elapsedMs}ms device=${formatDeviceSummary(result.deviceInfo)}`
263
277
  );
@@ -288,7 +302,7 @@ function isMainModule(metaUrl) {
288
302
  if (!entryPath) {
289
303
  return false;
290
304
  }
291
- return path.resolve(fileURLToPath(metaUrl)) === path.resolve(entryPath);
305
+ return realpathSync(path.resolve(fileURLToPath(metaUrl))) === realpathSync(path.resolve(entryPath));
292
306
  }
293
307
 
294
308
  if (isMainModule(import.meta.url)) {
File without changes
@@ -17,7 +17,7 @@
17
17
  ],
18
18
  "hf": {
19
19
  "repoId": "Clocksmith/rdrr",
20
- "revision": "f242e44c9a35e72186aa61e4cc0c9873f0596741",
20
+ "revision": "732b2bee08cfe315e5c0df5af30e7ad795721b48",
21
21
  "path": "models/gemma-3-270m-it-q4k-ehf16-af32"
22
22
  }
23
23
  },
@@ -54,6 +54,22 @@
54
54
  "path": "models/gemma-3-1b-it-q4k-ehf16-af32"
55
55
  }
56
56
  },
57
+ {
58
+ "modelId": "gemma-4-e2b-it-q4k-ehf16-af32",
59
+ "aliases": [
60
+ "gemma4-e2b",
61
+ "google/gemma-4-e2b-it"
62
+ ],
63
+ "modes": [
64
+ "text",
65
+ "vision"
66
+ ],
67
+ "hf": {
68
+ "repoId": "Clocksmith/rdrr",
69
+ "revision": "88fa42770918e0c61bfcd67c6a54952e079b5d36",
70
+ "path": "models/gemma-4-e2b-it-q4k-ehf16-af32"
71
+ }
72
+ },
57
73
  {
58
74
  "modelId": "qwen-3-5-0-8b-q4k-ehaf16",
59
75
  "aliases": [
@@ -147,7 +147,7 @@
147
147
  "disableCommandBatching": false
148
148
  },
149
149
  "perLayerInputs": {
150
- "materialization": "range_backed",
150
+ "materialization": "gpu_split_tables",
151
151
  "rowCache": {
152
152
  "mode": "lru",
153
153
  "maxRows": 512,
@@ -4467,6 +4467,79 @@
4467
4467
  }
4468
4468
  }
4469
4469
  },
4470
+ "activation_static_qdq": {
4471
+ "description": "Static activation quantize/dequantize kernels",
4472
+ "baseBindings": [
4473
+ {
4474
+ "index": 0,
4475
+ "name": "uniforms",
4476
+ "type": "uniform"
4477
+ },
4478
+ {
4479
+ "index": 1,
4480
+ "name": "input",
4481
+ "type": "read-only-storage"
4482
+ },
4483
+ {
4484
+ "index": 2,
4485
+ "name": "output",
4486
+ "type": "storage"
4487
+ }
4488
+ ],
4489
+ "baseUniforms": {
4490
+ "size": 32,
4491
+ "fields": [
4492
+ {
4493
+ "name": "size",
4494
+ "type": "u32",
4495
+ "offset": 0
4496
+ },
4497
+ {
4498
+ "name": "_pad0",
4499
+ "type": "u32",
4500
+ "offset": 4
4501
+ },
4502
+ {
4503
+ "name": "scale",
4504
+ "type": "f32",
4505
+ "offset": 8
4506
+ },
4507
+ {
4508
+ "name": "invScale",
4509
+ "type": "f32",
4510
+ "offset": 12
4511
+ },
4512
+ {
4513
+ "name": "qmin",
4514
+ "type": "f32",
4515
+ "offset": 16
4516
+ },
4517
+ {
4518
+ "name": "qmax",
4519
+ "type": "f32",
4520
+ "offset": 20
4521
+ }
4522
+ ]
4523
+ },
4524
+ "variants": {
4525
+ "default": {
4526
+ "wgsl": "activation_static_qdq.wgsl",
4527
+ "entryPoint": "main",
4528
+ "workgroup": [
4529
+ 256,
4530
+ 1,
4531
+ 1
4532
+ ],
4533
+ "requires": [],
4534
+ "reachability": {
4535
+ "status": "unused",
4536
+ "inlineConfigs": [],
4537
+ "ruleChains": [],
4538
+ "wgslOnDisk": true
4539
+ }
4540
+ }
4541
+ }
4542
+ },
4470
4543
  "energy_eval": {
4471
4544
  "description": "Energy evaluation kernels",
4472
4545
  "baseBindings": [
@@ -0,0 +1,32 @@
1
+ {
2
+ "id": "experiments/bench/gemma4-bench-q4k",
3
+ "name": "gemma4-bench-q4k",
4
+ "description": "Benchmark run for Gemma 4 E2B Q4K with deterministic generation settings.",
5
+ "intent": "calibrate",
6
+ "stability": "experimental",
7
+ "owner": "doppler-core",
8
+ "createdAtUtc": "2026-04-14T00:00:00Z",
9
+ "model": "gemma-4-e2b-it-q4k-ehf16-af32",
10
+ "extends": "profiles/gemma4-e2b-throughput",
11
+ "runtime": {
12
+ "shared": {
13
+ "benchmark": {
14
+ "run": {
15
+ "promptName": "xs",
16
+ "customPrompt": "The sky is clear and beautiful.",
17
+ "maxNewTokens": 256,
18
+ "warmupRuns": 1,
19
+ "timedRuns": 3,
20
+ "sampling": {
21
+ "temperature": 0,
22
+ "topK": 1,
23
+ "topP": 1,
24
+ "seed": 12345
25
+ },
26
+ "profile": false,
27
+ "captureMemoryTimeSeries": false
28
+ }
29
+ }
30
+ }
31
+ }
32
+ }
@@ -6,6 +6,7 @@ export interface BenchmarkRunConfig {
6
6
  promptName: string;
7
7
  customPrompt: string | null;
8
8
  maxNewTokens: number;
9
+ seed: number | null;
9
10
  runType: string;
10
11
  warmupRuns: number;
11
12
  timedRuns: number;
@@ -13,6 +14,7 @@ export interface BenchmarkRunConfig {
13
14
  temperature: number;
14
15
  topK: number;
15
16
  topP: number;
17
+ seed?: number;
16
18
  };
17
19
  debug: boolean;
18
20
  profile: boolean;
@@ -14,6 +14,7 @@ export const DEFAULT_BENCHMARK_RUN_CONFIG = {
14
14
  promptName: 'medium',
15
15
  customPrompt: null,
16
16
  maxNewTokens: 128,
17
+ seed: null,
17
18
  runType: 'warm',
18
19
  warmupRuns: 2,
19
20
  timedRuns: 3,
@@ -19,6 +19,13 @@ import type { ExecutionV1PatchSchema, ExecutionV1SessionSchema } from './executi
19
19
  import type { SharedRuntimeConfigSchema } from './shared-runtime.schema.js';
20
20
  import type { EmulationConfigSchema } from './emulation.schema.js';
21
21
 
22
+ export interface LargeWeightConfigSchema {
23
+ enabled: boolean;
24
+ safetyRatio: number;
25
+ preferF16: boolean;
26
+ lmHeadChunkRows: number | null;
27
+ }
28
+
22
29
  export interface RuntimeInferenceConfigSchema {
23
30
  prompt?: string | null;
24
31
  debugTokens?: boolean;
@@ -26,7 +33,7 @@ export interface RuntimeInferenceConfigSchema {
26
33
  sampling: Record<string, unknown>;
27
34
  compute: Record<string, unknown>;
28
35
  tokenizer: Record<string, unknown>;
29
- largeWeights: Record<string, unknown>;
36
+ largeWeights: LargeWeightConfigSchema;
30
37
  kvcache: Record<string, unknown>;
31
38
  diffusion: Record<string, unknown>;
32
39
  energy: Record<string, unknown>;
@@ -64,6 +71,7 @@ export interface RuntimeConfigSchema {
64
71
  }
65
72
 
66
73
  /** Default runtime configuration */
74
+ export declare const DEFAULT_LARGE_WEIGHT_CONFIG: LargeWeightConfigSchema;
67
75
  export declare const DEFAULT_RUNTIME_CONFIG: RuntimeConfigSchema;
68
76
 
69
77
  /**
@@ -24,6 +24,13 @@ export const DEFAULT_CHAT_TEMPLATE_CONFIG = Object.freeze({
24
24
  thinking: false,
25
25
  });
26
26
 
27
+ export const DEFAULT_LARGE_WEIGHT_CONFIG = Object.freeze({
28
+ enabled: true,
29
+ safetyRatio: 0.9,
30
+ preferF16: true,
31
+ lmHeadChunkRows: null,
32
+ });
33
+
27
34
  export const DEFAULT_RUNTIME_CONFIG = {
28
35
  shared: DEFAULT_SHARED_RUNTIME_CONFIG,
29
36
  loading: DEFAULT_LOADING_CONFIG,
@@ -39,7 +46,7 @@ export const DEFAULT_RUNTIME_CONFIG = {
39
46
  },
40
47
  compute: {},
41
48
  tokenizer: {},
42
- largeWeights: {},
49
+ largeWeights: DEFAULT_LARGE_WEIGHT_CONFIG,
43
50
  kvcache: DEFAULT_KVCACHE_CONFIG,
44
51
  diffusion: {},
45
52
  energy: {},
@@ -850,10 +850,12 @@ export {
850
850
  // =============================================================================
851
851
  export {
852
852
  // Types
853
+ type LargeWeightConfigSchema,
853
854
  type RuntimeConfigSchema,
854
855
  type DopplerConfigSchema,
855
856
 
856
857
  // Defaults
858
+ DEFAULT_LARGE_WEIGHT_CONFIG,
857
859
  DEFAULT_RUNTIME_CONFIG,
858
860
  DEFAULT_DOPPLER_CONFIG,
859
861
 
@@ -487,6 +487,7 @@ export {
487
487
  // =============================================================================
488
488
  export {
489
489
  // Defaults
490
+ DEFAULT_LARGE_WEIGHT_CONFIG,
490
491
  DEFAULT_CHAT_TEMPLATE_CONFIG,
491
492
  DEFAULT_RUNTIME_CONFIG,
492
493
  DEFAULT_DOPPLER_CONFIG,
@@ -437,6 +437,11 @@ export interface TensorSchema {
437
437
  scaleDivisor?: number;
438
438
  storageShape: [number, number];
439
439
  quantAxis: 0 | 1;
440
+ scaleCompanionDtype?: 'UINT8';
441
+ scaleCompanionDequant?: {
442
+ scale: number;
443
+ zeroPoint: number;
444
+ };
440
445
  scaleSource: {
441
446
  shard: number;
442
447
  shardIndex?: number;
@@ -9,11 +9,12 @@ import {
9
9
  // Re-export for backward compatibility
10
10
  export { QK_K, K_SCALE_SIZE, QK4_K_BLOCK_SIZE };
11
11
 
12
+ const f32ToF16ScratchF32 = new Float32Array(1);
13
+ const f32ToF16ScratchU32 = new Uint32Array(f32ToF16ScratchF32.buffer);
14
+
12
15
  export function float32ToFloat16(value) {
13
- const floatView = new Float32Array(1);
14
- const int32View = new Int32Array(floatView.buffer);
15
- floatView[0] = value;
16
- const f = int32View[0];
16
+ f32ToF16ScratchF32[0] = value;
17
+ const f = f32ToF16ScratchU32[0];
17
18
 
18
19
  const sign = (f >> 31) & 0x1;
19
20
  let exp = (f >> 23) & 0xff;