doppler-gpu 0.7.7 → 0.7.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. package/CHANGELOG.md +18 -0
  2. package/README.md +9 -17
  3. package/package.json +1 -1
  4. package/src/config/kernels/kernel-ref-digests.js +2 -1
  5. package/src/config/kernels/registry.json +47 -19
  6. package/src/config/schema/execution/kernel-warmup.schema.d.ts +14 -0
  7. package/src/config/schema/execution/kernel-warmup.schema.js +3 -0
  8. package/src/config/schema/platform/gpu-cache.schema.js +1 -1
  9. package/src/config/schema/runtime/buffer-pool.schema.js +2 -2
  10. package/src/config/schema/runtime/loading.schema.d.ts +5 -0
  11. package/src/config/schema/runtime/loading.schema.js +25 -2
  12. package/src/gpu/kernels/moe_route_q4_0_block_f32.wgsl +74 -0
  13. package/src/gpu/kernels/moe_route_q4_0_split_block_f32.wgsl +35 -18
  14. package/src/inference/pipelines/text/partitions/partition-execution.js +28 -1
  15. package/src/inference/pipelines/text/partitions/resident-partition-warmup.js +67 -0
  16. package/src/inference/pipelines/text/partitions/resident-partition.js +8 -2
  17. package/src/loader/experts/compact-expert-cache.js +20 -0
  18. package/src/loader/experts/loader-bindings.js +14 -3
  19. package/src/loader/experts/mapped-expert-staging.js +393 -0
  20. package/src/loader/experts/routed-expert-cache.d.ts +8 -0
  21. package/src/loader/experts/routed-expert-cache.js +12 -11
  22. package/src/loader/experts/routed-expert-prefill.js +77 -1
  23. package/src/loader/experts/routed-expert-source.js +1 -0
  24. package/src/loader/experts/tiered-expert-cache.js +66 -26
  25. package/src/storage/verified-piece-direct.js +153 -0
  26. package/src/storage/verified-piece-gather-lane.js +21 -3
  27. package/src/storage/verified-piece-gather-worker.js +235 -16
  28. package/src/storage/verified-piece-gather.d.ts +47 -0
  29. package/src/storage/verified-piece-gather.js +133 -17
  30. package/src/storage/verified-piece-reader.d.ts +8 -1
  31. package/src/storage/verified-piece-reader.js +16 -3
  32. package/src/storage/verified-piece-state-cache.d.ts +75 -0
  33. package/src/storage/verified-piece-state-cache.js +511 -0
  34. package/src/storage/verified-piece-storage.d.ts +20 -1
  35. package/src/storage/verified-piece-storage.js +207 -9
  36. package/src/version.js +1 -1
package/CHANGELOG.md CHANGED
@@ -4,6 +4,24 @@ Historical package notes are preserved outside the runtime archive in
4
4
  `docs/status/archive/package-changelog-through-0.6.1.md` and the earlier
5
5
  `package-changelog-through-0.4.15.md` snapshot in that directory.
6
6
 
7
+ ## [0.7.8] - 2026-11-19
8
+
9
+ ### Performance
10
+
11
+ - Routed expert weights can be staged through mapped upload buffers (`qwen35Routed.uploadStaging: "mapped"`,
12
+ off by default), with first-time bytes read while worker lanes hash them and one upload's hashing spread
13
+ over idle lanes.
14
+ - `qwen35Routed.uploadVerification: "before-release"` (off by default, mapped staging only) lets a partition
15
+ stage compute while hashes run and holds its output until every hash from that step has passed. The default
16
+ stays `"before-upload"`.
17
+ - Verified gather lanes start together, piece-directory identity checks run synchronously, and a resident
18
+ stage can run an optional warm prompt pass while it opens.
19
+
20
+ ### Correctness
21
+
22
+ - A piece served from the protected cache no longer marks the file's own piece as verified.
23
+ - Q4_0 routed-expert kernels, the file-descriptor-bound piece state cache and tiered preload are included.
24
+
7
25
  ## [0.7.7] - 2026-11-18
8
26
 
9
27
  ### Packaging
package/README.md CHANGED
@@ -1,13 +1,12 @@
1
1
  # Doppler
2
2
 
3
- <a href="assets/readme/doppler-demo-hero.mp4"><img src="assets/readme/doppler-demo-hero.gif" width="100%" alt="Doppler demo: a recorded Qwen 3.5 0.8B answer appears token by token beside per-layer activation changes, then one word's next-token alternatives open with their probabilities." /></a>
3
+ Run open models locally with WebGPU from JavaScript. Doppler generates text, computes embeddings, reranks results, and scores choices on supported models and devices.
4
4
 
5
- <sub>Recorded replay from the <a href="https://d4da.com/doppler/">demo</a>, not a live run. <a href="assets/readme/doppler-demo-hero.mp4">Full-quality video</a>.</sub>
5
+ **[Try the token demo](https://d4da.com/doppler/demo/index.html)** · [Open Observatory](https://d4da.com/observatory/) · [Run locally](#get-started)
6
6
 
7
- A JavaScript library for local model execution through WebGPU: generation,
8
- embeddings, reranking, and choice scoring, qualified per model and host.
7
+ <a href="assets/readme/doppler-demo-hero.mp4"><img src="assets/readme/doppler-demo-hero.gif" width="100%" alt="A recorded Qwen answer appears token by token. Selecting a token reveals its alternatives and their probabilities." /></a>
9
8
 
10
- **[Try the demo](https://d4da.com/doppler/)** · [Get started](docs/getting-started.md) · [Documentation](docs/INDEX.md)
9
+ The demo opens with a recorded answer. Select a token to inspect its alternatives, or load a compatible model to generate on your device. [Watch the full-quality recording](assets/readme/doppler-demo-hero.mp4)
11
10
 
12
11
  <picture>
13
12
  <source media="(max-width: 640px)" srcset="assets/readme/application-mobile.svg" />
@@ -16,18 +15,15 @@ embeddings, reranking, and choice scoring, qualified per model and host.
16
15
 
17
16
  ## Get started
18
17
 
19
- Run the pinned published quickstart with Node 20+ and a working WebGPU provider:
18
+ Use Node 20+ and a working WebGPU provider. This command pins the published CLI and model alias used in the installation guide.
20
19
 
21
20
  ```sh
22
21
  npx --package doppler-gpu@0.7.0 doppler-gpu --model gemma3-270m --prompt "Describe WebGPU briefly"
23
22
  ```
24
23
 
25
- First execution downloads model assets. Browser and Node support are qualified
26
- separately. The [installation guide](docs/getting-started.md) covers provider
27
- requirements and embedding the published compatibility API.
24
+ The first run downloads about 400 MB of weights plus the tokenizer and manifest. See [installation and embedding](docs/getting-started.md) for host requirements.
28
25
 
29
- The [signed Run API](docs/api/root.md) and resident partition exports ship in
30
- 0.7.0 with separate release requirements; qualification stays per model and host.
26
+ [Published versions](https://www.npmjs.com/package/doppler-gpu) · [Source changelog](CHANGELOG.md) · [Signed Run API](docs/api/root.md)
31
27
 
32
28
  ## Benchmarks
33
29
 
@@ -44,11 +40,7 @@ The [signed Run API](docs/api/root.md) and resident partition exports ship in
44
40
 
45
41
  ![Purple session boundaries distinguish signed Capsule opening, persistent model resources, and successive operations through a model program, binder, and executor. The application and WebGPU provider remain external.](assets/readme/technical-architecture.svg)
46
42
 
47
- Rig prepares signed Capsules with declared TargetPlans. Run verifies artifacts
48
- and selects an accepted plan; session operations reuse resident resources through
49
- the model program, binder, and command executor. Applications supply trust and
50
- host ports. JSON declares execution, JavaScript orchestrates, and WGSL computes.
51
- Doe and Reploid remain optional integrations.
43
+ Rig prepares a signed Capsule and its execution plans. Run verifies the artifacts and selects a plan. A session reuses the loaded weights and GPU programs across requests. The application supplies trust and host access.
52
44
 
53
45
  [Architecture and source owners](docs/architecture.md) · [Resident partition lifetimes](docs/architecture.md#resident-partitions-and-resource-lifetime)
54
46
 
@@ -65,7 +57,7 @@ from verified execution and links the lifecycle evidence for each model.
65
57
  The [comparison scoreboard](docs/model-competition-scoreboard.md) reports
66
58
  throughput and loading separately against declared controls. Runtime correctness,
67
59
  task quality, memory, and lifecycle acceptance remain separate checks.
68
- W8A8 remains experimental; it does not qualify distributed chat.
60
+ W8A8 remains experimental.
69
61
 
70
62
  Choice scores are uncalibrated. Unsupported paths fail closed. The retained
71
63
  [search starter](examples/document-search/README.md) needs renewed signed
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "doppler-gpu",
3
- "version": "0.7.7",
3
+ "version": "0.7.8",
4
4
  "description": "Doppler Rig prepares signed model Capsules; Doppler Run executes them locally through WebGPU for JavaScript",
5
5
  "main": "src/capsule-runtime.js",
6
6
  "types": "src/capsule-runtime.d.ts",
@@ -356,8 +356,9 @@ export const KERNEL_REF_CONTENT_DIGESTS = Object.freeze({
356
356
  "moe_norm_router_f32.wgsl#single_pass": "87646d02b567b3c13f66245ac222392df92af863da920ce562caa77dd14f2e93",
357
357
  "moe_offsets.wgsl#build_offsets": "3ea004145fa234659408cdeb0d4d802adff1037c9c5c03af146b3734cc69dd27",
358
358
  "moe_route_gguf_split_f32.wgsl#main": "32a342a5b289797719ab6438f0e7d482ac16cb15a5f6315fbbf4758ac724cebd",
359
+ "moe_route_q4_0_block_f32.wgsl#main": "912d5445e34ce2611356a1a14c6f881630d0a88405035e843a7f676eb9ec9f16",
359
360
  "moe_route_q4_0_f32.wgsl#main": "93701086c9ce454d08a7b22869fc931d186fe3d45e7b6ca95b7b72ee26b1d5be",
360
- "moe_route_q4_0_split_block_f32.wgsl#main": "4d6cda4879b0eb3cf1163e3aa28ed6fc80a7d9e69e567c97bbd85296b9023137",
361
+ "moe_route_q4_0_split_block_f32.wgsl#main": "3b003b37dd66eb24786464a4c553fb57275f5f18de03361ae225d3e3339a5d59",
361
362
  "moe_route_q4_1_block_f32.wgsl#main": "57291511788508e5333e244809e821c6edf71ee1dc6bfcd6bb19c3f4ace78d23",
362
363
  "moe_route_q4_f32.wgsl#main_multicol": "22a02228b1425e2b7354b4e87c275f7b0de166925fd7cf30945d675c37aa86c8",
363
364
  "moe_route_remap.wgsl#remap_routes": "092c11f4603ca96496a000d56f3bbb0211b6f536fd328bd3a9ee668b262353d3",
@@ -7298,7 +7298,10 @@
7298
7298
  "description": "Q4_K GEMV with 1 contiguous subgroup lanes per row.",
7299
7299
  "reachability": {
7300
7300
  "status": "pinned",
7301
- "inlineConfigs": ["runtime/profiles/qwen-3-5-0-8b-vulkan-subgroup8"],
7301
+ "inlineConfigs": [
7302
+ "runtime/profiles/qwen-3-5-0-8b-vulkan-subgroup8",
7303
+ "runtime/profiles/qwen3-8-27b-orin-projections"
7304
+ ],
7302
7305
  "ruleChains": [],
7303
7306
  "wgslOnDisk": true
7304
7307
  },
@@ -7349,7 +7352,10 @@
7349
7352
  "description": "Q4_K GEMV with 2 contiguous subgroup lanes per row.",
7350
7353
  "reachability": {
7351
7354
  "status": "pinned",
7352
- "inlineConfigs": ["runtime/profiles/qwen-3-5-0-8b-vulkan-subgroup8"],
7355
+ "inlineConfigs": [
7356
+ "runtime/profiles/qwen-3-5-0-8b-vulkan-subgroup8",
7357
+ "runtime/profiles/qwen3-8-27b-orin-projections"
7358
+ ],
7353
7359
  "ruleChains": [],
7354
7360
  "wgslOnDisk": true
7355
7361
  },
@@ -7400,7 +7406,10 @@
7400
7406
  "description": "Q4_K GEMV with 4 contiguous subgroup lanes per row.",
7401
7407
  "reachability": {
7402
7408
  "status": "pinned",
7403
- "inlineConfigs": ["runtime/profiles/qwen-3-5-0-8b-vulkan-subgroup8"],
7409
+ "inlineConfigs": [
7410
+ "runtime/profiles/qwen-3-5-0-8b-vulkan-subgroup8",
7411
+ "runtime/profiles/qwen3-8-27b-orin-projections"
7412
+ ],
7404
7413
  "ruleChains": [],
7405
7414
  "wgslOnDisk": true
7406
7415
  },
@@ -7451,7 +7460,10 @@
7451
7460
  "description": "Q4_K GEMV with 8 contiguous subgroup lanes per row.",
7452
7461
  "reachability": {
7453
7462
  "status": "pinned",
7454
- "inlineConfigs": ["runtime/profiles/qwen-3-5-0-8b-vulkan-subgroup8"],
7463
+ "inlineConfigs": [
7464
+ "runtime/profiles/qwen-3-5-0-8b-vulkan-subgroup8",
7465
+ "runtime/profiles/qwen3-8-27b-orin-projections"
7466
+ ],
7455
7467
  "ruleChains": [],
7456
7468
  "wgslOnDisk": true
7457
7469
  },
@@ -7502,7 +7514,10 @@
7502
7514
  "description": "Q4_K GEMV with 16 contiguous subgroup lanes per row.",
7503
7515
  "reachability": {
7504
7516
  "status": "pinned",
7505
- "inlineConfigs": ["runtime/profiles/qwen-3-5-0-8b-vulkan-subgroup8"],
7517
+ "inlineConfigs": [
7518
+ "runtime/profiles/qwen-3-5-0-8b-vulkan-subgroup8",
7519
+ "runtime/profiles/qwen3-8-27b-orin-projections"
7520
+ ],
7506
7521
  "ruleChains": [],
7507
7522
  "wgslOnDisk": true
7508
7523
  },
@@ -7553,7 +7568,10 @@
7553
7568
  "description": "Q4_K GEMV with 32 contiguous subgroup lanes per row.",
7554
7569
  "reachability": {
7555
7570
  "status": "pinned",
7556
- "inlineConfigs": ["runtime/profiles/qwen-3-5-0-8b-vulkan-subgroup8"],
7571
+ "inlineConfigs": [
7572
+ "runtime/profiles/qwen-3-5-0-8b-vulkan-subgroup8",
7573
+ "runtime/profiles/qwen3-8-27b-orin-projections"
7574
+ ],
7557
7575
  "ruleChains": [],
7558
7576
  "wgslOnDisk": true
7559
7577
  },
@@ -8047,7 +8065,11 @@
8047
8065
  ],
8048
8066
  "reachability": {
8049
8067
  "status": "pinned",
8050
- "inlineConfigs": ["runtime/profiles/qwen-3-5-2b-orin-prefill"],
8068
+ "inlineConfigs": [
8069
+ "runtime/profiles/qwen-3-5-2b-orin-prefill",
8070
+ "runtime/profiles/qwen3-8-27b-orin-prefill",
8071
+ "runtime/profiles/qwen3-8-27b-orin-projections"
8072
+ ],
8051
8073
  "ruleChains": [],
8052
8074
  "wgslOnDisk": true
8053
8075
  }
@@ -14750,7 +14772,10 @@
14750
14772
  "description": "Q4_K fused gate/up GEMV with one eight-lane subgroup partition per output row.",
14751
14773
  "reachability": {
14752
14774
  "status": "pinned",
14753
- "inlineConfigs": ["runtime/profiles/qwen-3-5-0-8b-vulkan-subgroup8"],
14775
+ "inlineConfigs": [
14776
+ "runtime/profiles/qwen-3-5-0-8b-vulkan-subgroup8",
14777
+ "runtime/profiles/qwen3-8-27b-orin-decode"
14778
+ ],
14754
14779
  "ruleChains": [],
14755
14780
  "wgslOnDisk": true
14756
14781
  },
@@ -24810,8 +24835,11 @@
24810
24835
  "outputDtype": "f32",
24811
24836
  "description": "Packed F16 bits in u32 split buffers -> F32 output without shader-f16",
24812
24837
  "reachability": {
24813
- "status": "code-dispatched",
24814
- "inlineConfigs": [],
24838
+ "status": "pinned",
24839
+ "inlineConfigs": [
24840
+ "runtime/profiles/qwen3-8-27b-orin-prefill",
24841
+ "runtime/profiles/qwen3-8-27b-orin-projections"
24842
+ ],
24815
24843
  "ruleChains": [],
24816
24844
  "wgslOnDisk": true
24817
24845
  }
@@ -35723,9 +35751,9 @@
35723
35751
  }
35724
35752
  },
35725
35753
  "q4_0_f32": {
35726
- "wgsl": "moe_route_q4_0_f32.wgsl",
35754
+ "wgsl": "moe_route_q4_0_block_f32.wgsl",
35727
35755
  "entryPoint": "main",
35728
- "workgroup": [128,1,1],
35756
+ "workgroup": [256,1,1],
35729
35757
  "requires": [],
35730
35758
  "bindings": [
35731
35759
  {
@@ -35802,15 +35830,15 @@
35802
35830
  "weightDtype": "q4_0",
35803
35831
  "outputDtype": "f32",
35804
35832
  "wgslOverrides": {
35805
- "WORKGROUP_SIZE": 128,
35806
- "COLS_PER_WG": 4,
35833
+ "WORKGROUP_SIZE": 256,
35834
+ "COLS_PER_WG": 8,
35807
35835
  "THREADS_PER_COL_GEMV": 32,
35808
35836
  "TILE_SIZE": 16,
35809
35837
  "BLOCK_VALUES": 32,
35810
35838
  "BLOCK_BYTES": 18,
35811
35839
  "QUANT_FORMAT": 0
35812
35840
  },
35813
- "sharedMemory": 512,
35841
+ "sharedMemory": 1024,
35814
35842
  "variantMetadata": {
35815
35843
  "workgroupConstants": ["WORKGROUP_SIZE",null,null]
35816
35844
  },
@@ -36121,7 +36149,7 @@
36121
36149
  "q4_0_split_f32": {
36122
36150
  "wgsl": "moe_route_q4_0_split_block_f32.wgsl",
36123
36151
  "entryPoint": "main",
36124
- "workgroup": [128,1,1],
36152
+ "workgroup": [256,1,1],
36125
36153
  "requires": [],
36126
36154
  "bindings": [
36127
36155
  {
@@ -36203,15 +36231,15 @@
36203
36231
  "weightDtype": "q4_0",
36204
36232
  "outputDtype": "f32",
36205
36233
  "wgslOverrides": {
36206
- "WORKGROUP_SIZE": 128,
36207
- "COLS_PER_WG": 4,
36234
+ "WORKGROUP_SIZE": 256,
36235
+ "COLS_PER_WG": 8,
36208
36236
  "THREADS_PER_COL_GEMV": 32,
36209
36237
  "TILE_SIZE": 16,
36210
36238
  "BLOCK_VALUES": 32,
36211
36239
  "BLOCK_BYTES": 18,
36212
36240
  "QUANT_FORMAT": 0
36213
36241
  },
36214
- "sharedMemory": 512,
36242
+ "sharedMemory": 1024,
36215
36243
  "variantMetadata": {
36216
36244
  "workgroupConstants": ["WORKGROUP_SIZE",null,null]
36217
36245
  },
@@ -19,6 +19,20 @@ export interface KernelWarmupConfigSchema {
19
19
  * null runs none.
20
20
  */
21
21
  warmGeneration: KernelWarmupGenerationSchema | null;
22
+ /**
23
+ * One prompt pass through a resident partition's own layers while the partition opens: its
24
+ * pipelines are compiled and, on a routed-expert stage, the experts those tokens select are
25
+ * loaded before the first request. Runs in a private attempt that is closed before readiness.
26
+ * Counted in opening time. null runs none.
27
+ */
28
+ warmPartition: KernelWarmupPartitionSchema | null;
29
+ }
30
+
31
+ export interface KernelWarmupPartitionSchema {
32
+ /** Token ids of the warm prompt. A stage that starts from a boundary activation runs zeros of this length. */
33
+ tokenIds: number[];
34
+ /** Single-token steps after the prompt pass (0 to 8). */
35
+ decodeSteps: number;
22
36
  }
23
37
 
24
38
  export interface KernelWarmupGenerationSchema {
@@ -4,4 +4,7 @@ export const DEFAULT_KERNEL_WARMUP_CONFIG = {
4
4
  // { prompt, maxTokens }: one greedy generation at the end of a text pipeline load
5
5
  // (counted in load time), then a sequence reset. null runs none.
6
6
  warmGeneration: null,
7
+ // { tokenIds, decodeSteps }: one prompt pass through a resident partition's own layers while it
8
+ // opens (counted in opening time), in a private attempt that is closed afterwards. null runs none.
9
+ warmPartition: null,
7
10
  };
@@ -3,6 +3,6 @@
3
3
  // =============================================================================
4
4
 
5
5
  export const DEFAULT_GPU_CACHE_CONFIG = {
6
- uniformCacheMaxEntries: 256,
6
+ uniformCacheMaxEntries: 4096,
7
7
  uniformCacheMaxAgeMs: 60000, // 60 seconds
8
8
  };
@@ -15,8 +15,8 @@ export const DEFAULT_BUFFER_POOL_BUCKET_CONFIG = {
15
15
  // =============================================================================
16
16
 
17
17
  export const DEFAULT_BUFFER_POOL_LIMITS_CONFIG = {
18
- maxBuffersPerBucket: 8,
19
- maxTotalPooledBuffers: 64,
18
+ maxBuffersPerBucket: 1024,
19
+ maxTotalPooledBuffers: 4096,
20
20
  };
21
21
 
22
22
  // =============================================================================
@@ -197,6 +197,11 @@ export interface Qwen35RoutedConfig {
197
197
  sourceReadMode: 'plane-serial' | 'expert-parallel' | 'lease-parallel' | 'expert-lease-batch';
198
198
  /** Write authenticated source ranges to their logical working-bank offsets without temporary expert buffers. */
199
199
  prefillUploadMode: 'expert-copy' | 'direct-bank';
200
+ /** 'mapped' stages already verified expert ranges in mapped upload buffers instead of queue.writeBuffer. Omission means 'queue'. */
201
+ uploadStaging?: 'queue' | 'mapped';
202
+ /** When first-time expert bytes are verified: before they reach a model buffer (default), or, with mapped
203
+ * staging in a resident partition step, before that step's output is released. */
204
+ uploadVerification?: 'before-upload' | 'before-release';
200
205
  /** LFU retains causal lifetime counts until unload; LRU remains the default. */
201
206
  evictionPolicy: 'lru' | 'lfu';
202
207
  cacheBytes: number;
@@ -103,6 +103,15 @@ export const DEFAULT_EXPERT_CACHE_CONFIG = {
103
103
  sourceReadMode: 'plane-serial',
104
104
  expertBatchSize: 1,
105
105
  prefillUploadMode: 'expert-copy',
106
+ // 'queue' copies each expert through queue.writeBuffer. 'mapped' reads already verified ranges
107
+ // straight into mapped upload buffers when the storage port can serve them, and otherwise does
108
+ // what 'queue' does.
109
+ uploadStaging: 'queue',
110
+ // 'before-upload': expert bytes are verified before they are copied into a model buffer.
111
+ // 'before-release': with mapped staging inside a resident partition step, first-time bytes are
112
+ // copied and computed with while lanes hash them, and the step's output is withheld until every
113
+ // hash has passed. A failed hash fails the step and refuses all further steps until reopened.
114
+ uploadVerification: 'before-upload',
106
115
  evictionPolicy: 'lru',
107
116
  cacheBytes: 2 * GB,
108
117
  hostCacheBytes: 0,
@@ -135,9 +144,11 @@ export function validateQwen35RoutedConfig(config) {
135
144
  || tiered.mappedProtectedShare < 0 || tiered.mappedProtectedShare > 1
136
145
  || !Number.isSafeInteger(tiered.mappedCacheBytes) || tiered.mappedCacheBytes < 0 || tiered.mappedCacheBytes % 4
137
146
  || config.compactPrefillBytes !== 0 || config.compactPrefillReuse !== 'none'
138
- || config.prefillCompactAdmission !== 'none' || config.sourceReadMode !== 'expert-lease-batch'
147
+ || config.prefillCompactAdmission !== 'none'
148
+ || !['expert-lease-batch', 'expert-parallel'].includes(config.sourceReadMode)
149
+ || (config.sourceReadMode === 'expert-parallel' && (tiered.parkedLayers.length > 0 || tiered.mappedCacheBytes > 0))
139
150
  || config.prefillUploadMode !== 'expert-copy') {
140
- throw new Error('Tiered experts require distinct resident/parked layers, aligned mappedCacheBytes, mappedProtectedShare in [0,1] and batched streaming without full prefill banks.');
151
+ throw new Error('Tiered experts require distinct resident/parked layers, aligned mappedCacheBytes, mappedProtectedShare in [0,1] and batched or parallel streaming without full prefill banks.');
141
152
  }
142
153
  } else if (config.tiered != null) {
143
154
  throw new Error('Tiered placement requires storageMode=tiered-compact.');
@@ -174,6 +185,18 @@ export function validateQwen35RoutedConfig(config) {
174
185
  || (expertBatchSize > 1 && (!compact || config.sourceReadMode !== 'expert-lease-batch'))) {
175
186
  throw new Error('expertBatchSize requires 1..32 and batched compact source leases for multiple experts.');
176
187
  }
188
+ const uploadStaging = config.uploadStaging === undefined
189
+ ? DEFAULT_EXPERT_CACHE_CONFIG.qwen35Routed.uploadStaging : config.uploadStaging;
190
+ if (!['queue', 'mapped'].includes(uploadStaging)
191
+ || (uploadStaging === 'mapped' && (!compact || config.sourceReadMode !== 'expert-lease-batch' || config.hostCacheBytes !== 0))) {
192
+ throw new Error('uploadStaging requires queue, or mapped with batched compact source leases and no host cache.');
193
+ }
194
+ const uploadVerification = config.uploadVerification === undefined
195
+ ? DEFAULT_EXPERT_CACHE_CONFIG.qwen35Routed.uploadVerification : config.uploadVerification;
196
+ if (!['before-upload', 'before-release'].includes(uploadVerification)
197
+ || (uploadVerification === 'before-release' && uploadStaging !== 'mapped')) {
198
+ throw new Error('uploadVerification requires before-upload, or before-release with mapped upload staging.');
199
+ }
177
200
  if (!['expert-copy', 'direct-bank'].includes(config.prefillUploadMode)
178
201
  || (config.prefillUploadMode === 'direct-bank'
179
202
  && (config.storageMode !== 'compact-decode' || config.hostCacheBytes !== 0))) {
@@ -0,0 +1,74 @@
1
+ // Native Q4_0 routed projection. One lane consumes a full 32-value block via whole-word u32 loads.
2
+ // Tuned for 256-thread / 8-column Metal & Vulkan threadgroups while preserving exact block accumulation.
3
+ override WORKGROUP_SIZE: u32 = 256u;
4
+ override COLS_PER_WG: u32 = 8u;
5
+ override THREADS_PER_COL_GEMV: u32 = 32u;
6
+ override TILE_SIZE: u32 = 16u;
7
+ override BLOCK_VALUES: u32 = 32u;
8
+ override BLOCK_BYTES: u32 = 18u;
9
+ override QUANT_FORMAT: u32 = 0u;
10
+ struct Uniforms {
11
+ M: u32, N: u32, K: u32, alpha: f32,
12
+ blocks: u32, num_experts: u32, top_k: u32, input_mode: u32,
13
+ }
14
+ @group(0) @binding(0) var<uniform> u: Uniforms;
15
+ @group(0) @binding(1) var<storage, read> input: array<vec4<f32>>;
16
+ @group(0) @binding(2) var<storage, read> weights: array<u32>;
17
+ @group(0) @binding(3) var<storage, read_write> output: array<f32>;
18
+ @group(0) @binding(4) var<storage, read> route_indices: array<u32>;
19
+ var<workgroup> partial: array<f32, WORKGROUP_SIZE>;
20
+ fn unpack_bytes(word: u32) -> vec4<u32> {
21
+ return vec4<u32>(word, word >> 8u, word >> 16u, word >> 24u) & vec4<u32>(255u);
22
+ }
23
+ @compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
24
+ fn main(@builtin(workgroup_id) group: vec3<u32>, @builtin(num_workgroups) grid: vec3<u32>,
25
+ @builtin(local_invocation_index) lane: u32) {
26
+ let tiles_n = (u.N + COLS_PER_WG - 1u) / COLS_PER_WG;
27
+ let tile = group.x + group.y * grid.x;
28
+ let m = tile / tiles_n;
29
+ if (m >= u.M) { return; }
30
+ let expert = route_indices[m];
31
+ let token = select(m / u.top_k, m, u.input_mode == 1u);
32
+ let column = lane / THREADS_PER_COL_GEMV;
33
+ let thread = lane % THREADS_PER_COL_GEMV;
34
+ let n = (tile % tiles_n) * COLS_PER_WG + column;
35
+ var sum = 0.0f;
36
+ if (n < u.N && expert < u.num_experts) {
37
+ let row = expert * u.N + n;
38
+ for (var block = thread; block < u.blocks; block += THREADS_PER_COL_GEMV) {
39
+ let offset = (row * u.blocks + block) * BLOCK_BYTES;
40
+ let w_base = offset / 4u;
41
+ let half_aligned = (offset & 2u) != 0u;
42
+ let w0 = weights[w_base];
43
+ let w1 = weights[w_base + 1u];
44
+ let w2 = weights[w_base + 2u];
45
+ let w3 = weights[w_base + 3u];
46
+ let w4 = weights[w_base + 4u];
47
+ let scale = select(unpack2x16float(w0).x, unpack2x16float(w0).y, half_aligned);
48
+ let q0 = select((w0 >> 16u) | (w1 << 16u), w1, half_aligned);
49
+ let q1 = select((w1 >> 16u) | (w2 << 16u), w2, half_aligned);
50
+ let q2 = select((w2 >> 16u) | (w3 << 16u), w3, half_aligned);
51
+ let q3 = select((w3 >> 16u) | (w4 << 16u), w4, half_aligned);
52
+ let vec_base = (token * u.K + block * BLOCK_VALUES) / 4u;
53
+ let p0 = unpack_bytes(q0);
54
+ sum += dot(input[vec_base], (vec4<f32>(p0 & vec4<u32>(15u)) - vec4<f32>(8.0)) * scale)
55
+ + dot(input[vec_base + 4u], (vec4<f32>(p0 >> vec4<u32>(4u)) - vec4<f32>(8.0)) * scale);
56
+ let p1 = unpack_bytes(q1);
57
+ sum += dot(input[vec_base + 1u], (vec4<f32>(p1 & vec4<u32>(15u)) - vec4<f32>(8.0)) * scale)
58
+ + dot(input[vec_base + 5u], (vec4<f32>(p1 >> vec4<u32>(4u)) - vec4<f32>(8.0)) * scale);
59
+ let p2 = unpack_bytes(q2);
60
+ sum += dot(input[vec_base + 2u], (vec4<f32>(p2 & vec4<u32>(15u)) - vec4<f32>(8.0)) * scale)
61
+ + dot(input[vec_base + 6u], (vec4<f32>(p2 >> vec4<u32>(4u)) - vec4<f32>(8.0)) * scale);
62
+ let p3 = unpack_bytes(q3);
63
+ sum += dot(input[vec_base + 3u], (vec4<f32>(p3 & vec4<u32>(15u)) - vec4<f32>(8.0)) * scale)
64
+ + dot(input[vec_base + 7u], (vec4<f32>(p3 >> vec4<u32>(4u)) - vec4<f32>(8.0)) * scale);
65
+ }
66
+ }
67
+ partial[lane] = sum;
68
+ workgroupBarrier();
69
+ for (var stride = THREADS_PER_COL_GEMV / 2u; stride > 0u; stride /= 2u) {
70
+ if (thread < stride) { partial[lane] += partial[lane + stride]; }
71
+ workgroupBarrier();
72
+ }
73
+ if (thread == 0u && n < u.N) { output[m * u.N + n] = partial[lane] * u.alpha; }
74
+ }
@@ -1,7 +1,7 @@
1
- // Native Q4_0 separate gate/up projection. One lane consumes a full 32-value block.
1
+ // Native Q4_0 separate gate/up projection. One lane consumes a full 32-value block via whole-word u32 loads.
2
2
  // Preserves packed scales and quantization; output remains [routes, gate then up].
3
- override WORKGROUP_SIZE: u32 = 128u;
4
- override COLS_PER_WG: u32 = 4u;
3
+ override WORKGROUP_SIZE: u32 = 256u;
4
+ override COLS_PER_WG: u32 = 8u;
5
5
  override THREADS_PER_COL_GEMV: u32 = 32u;
6
6
  override TILE_SIZE: u32 = 16u;
7
7
  override BLOCK_VALUES: u32 = 32u;
@@ -12,15 +12,18 @@ struct Uniforms {
12
12
  blocks: u32, num_experts: u32, top_k: u32, input_mode: u32,
13
13
  }
14
14
  @group(0) @binding(0) var<uniform> u: Uniforms;
15
- @group(0) @binding(1) var<storage, read> input: array<f32>;
15
+ @group(0) @binding(1) var<storage, read> input: array<vec4<f32>>;
16
16
  @group(0) @binding(2) var<storage, read> weights: array<u32>;
17
17
  @group(0) @binding(3) var<storage, read_write> output: array<f32>;
18
18
  @group(0) @binding(4) var<storage, read> route_indices: array<u32>;
19
19
  @group(0) @binding(5) var<storage, read> up_weights: array<u32>;
20
20
  var<workgroup> partial: array<f32, WORKGROUP_SIZE>;
21
- fn source_byte(offset: u32, up: bool) -> u32 {
22
- if (up) { return (up_weights[offset / 4u] >> ((offset % 4u) * 8u)) & 255u; }
23
- return (weights[offset / 4u] >> ((offset % 4u) * 8u)) & 255u;
21
+ fn source_word(idx: u32, up: bool) -> u32 {
22
+ if (up) { return up_weights[idx]; }
23
+ return weights[idx];
24
+ }
25
+ fn unpack_bytes(word: u32) -> vec4<u32> {
26
+ return vec4<u32>(word, word >> 8u, word >> 16u, word >> 24u) & vec4<u32>(255u);
24
27
  }
25
28
  @compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
26
29
  fn main(@builtin(workgroup_id) group: vec3<u32>, @builtin(num_workgroups) grid: vec3<u32>,
@@ -40,17 +43,31 @@ fn main(@builtin(workgroup_id) group: vec3<u32>, @builtin(num_workgroups) grid:
40
43
  let up = n >= u.N / 2u;
41
44
  for (var block = thread; block < u.blocks; block += THREADS_PER_COL_GEMV) {
42
45
  let offset = (row * u.blocks + block) * BLOCK_BYTES;
43
- let scale = unpack2x16float(source_byte(offset, up) | (source_byte(offset + 1u, up) << 8u)).x;
44
- let base = token * u.K + block * BLOCK_VALUES;
45
- for (var j = 0u; j < 16u; j += 4u) {
46
- let packed = vec4<u32>(source_byte(offset + 2u + j, up), source_byte(offset + 3u + j, up),
47
- source_byte(offset + 4u + j, up), source_byte(offset + 5u + j, up));
48
- let low = (vec4<f32>(packed & vec4<u32>(15u)) - vec4<f32>(8.0)) * scale;
49
- let high = (vec4<f32>(packed >> vec4<u32>(4u)) - vec4<f32>(8.0)) * scale;
50
- let xLow = vec4<f32>(input[base + j], input[base + j + 1u], input[base + j + 2u], input[base + j + 3u]);
51
- let xHigh = vec4<f32>(input[base + j + 16u], input[base + j + 17u], input[base + j + 18u], input[base + j + 19u]);
52
- sum += dot(xLow, low) + dot(xHigh, high);
53
- }
46
+ let w_base = offset / 4u;
47
+ let half_aligned = (offset & 2u) != 0u;
48
+ let w0 = source_word(w_base, up);
49
+ let w1 = source_word(w_base + 1u, up);
50
+ let w2 = source_word(w_base + 2u, up);
51
+ let w3 = source_word(w_base + 3u, up);
52
+ let w4 = source_word(w_base + 4u, up);
53
+ let scale = select(unpack2x16float(w0).x, unpack2x16float(w0).y, half_aligned);
54
+ let q0 = select((w0 >> 16u) | (w1 << 16u), w1, half_aligned);
55
+ let q1 = select((w1 >> 16u) | (w2 << 16u), w2, half_aligned);
56
+ let q2 = select((w2 >> 16u) | (w3 << 16u), w3, half_aligned);
57
+ let q3 = select((w3 >> 16u) | (w4 << 16u), w4, half_aligned);
58
+ let vec_base = (token * u.K + block * BLOCK_VALUES) / 4u;
59
+ let p0 = unpack_bytes(q0);
60
+ sum += dot(input[vec_base], (vec4<f32>(p0 & vec4<u32>(15u)) - vec4<f32>(8.0)) * scale)
61
+ + dot(input[vec_base + 4u], (vec4<f32>(p0 >> vec4<u32>(4u)) - vec4<f32>(8.0)) * scale);
62
+ let p1 = unpack_bytes(q1);
63
+ sum += dot(input[vec_base + 1u], (vec4<f32>(p1 & vec4<u32>(15u)) - vec4<f32>(8.0)) * scale)
64
+ + dot(input[vec_base + 5u], (vec4<f32>(p1 >> vec4<u32>(4u)) - vec4<f32>(8.0)) * scale);
65
+ let p2 = unpack_bytes(q2);
66
+ sum += dot(input[vec_base + 2u], (vec4<f32>(p2 & vec4<u32>(15u)) - vec4<f32>(8.0)) * scale)
67
+ + dot(input[vec_base + 6u], (vec4<f32>(p2 >> vec4<u32>(4u)) - vec4<f32>(8.0)) * scale);
68
+ let p3 = unpack_bytes(q3);
69
+ sum += dot(input[vec_base + 3u], (vec4<f32>(p3 & vec4<u32>(15u)) - vec4<f32>(8.0)) * scale)
70
+ + dot(input[vec_base + 7u], (vec4<f32>(p3 >> vec4<u32>(4u)) - vec4<f32>(8.0)) * scale);
54
71
  }
55
72
  }
56
73
  partial[lane] = sum;
@@ -20,6 +20,7 @@ import { getLogitsWeights, getLogitsConfig } from "../logits/logits-config.js";
20
20
  import { resolveLayerPartition, getPartitionActivationContract, validateActivationTensorShape } from "../execution/layer-partition-contract.js";
21
21
  import { createHyperConnectionSession } from "../hyper-connection/session.js";
22
22
  import { resolvePrefillTokenChunkSize, shouldDisablePrefillCommandBatching } from "../generator/prefill/prefill-policy.js";
23
+ import { beginStagedRelease, settleStagedRelease } from "../../../../loader/experts/mapped-expert-staging.js";
23
24
 
24
25
  /** @type {import('./partition-execution.js').assertPartitionExecutionSupported} */
25
26
  export function assertPartitionExecutionSupported(state, plan, requireAdapter = true) {
@@ -214,8 +215,34 @@ export function shouldRecordPartitionLayers(state, disableCommandBatching, prefi
214
215
  { startPos: state.currentSeqLen, returnHidden: !finalChunk }));
215
216
  }
216
217
 
217
- /** @type {import('./partition-execution.js').ExecutePartitionChunk} */
218
+ /**
219
+ * With uploadVerification 'before-release', expert bytes loaded for the first time during a chunk are
220
+ * computed with while they are still being hashed. The chunk's activation or logits are handed on only
221
+ * after every one of those hashes has passed; a failure fails the chunk and every later one.
222
+ * @type {import('./partition-execution.js').ExecutePartitionChunk} */
218
223
  async function executePartitionChunk(state, input, signal, prefill, finalChunk) {
224
+ const loader = state.dopplerLoader;
225
+ const port = loader?.manifest?.moeConfig && loader.getRoutedExpertPolicy().uploadVerification === 'before-release'
226
+ ? loader.shardCache.rangeLease ?? null : null;
227
+ if (!port) return runPartitionChunk(state, input, signal, prefill, finalChunk);
228
+ beginStagedRelease(port);
229
+ let result;
230
+ try { result = await runPartitionChunk(state, input, signal, prefill, finalChunk); }
231
+ catch (error) {
232
+ await settleStagedRelease(port, { release: false });
233
+ throw error;
234
+ }
235
+ try { await settleStagedRelease(port); }
236
+ catch (error) {
237
+ // The output was computed with bytes that failed verification. It never leaves this function.
238
+ if (result.logits) releaseBuffer(result.logits.logitsBuffer);
239
+ throw error;
240
+ }
241
+ return result;
242
+ }
243
+
244
+ /** @type {import('./partition-execution.js').ExecutePartitionChunk} */
245
+ async function runPartitionChunk(state, input, signal, prefill, finalChunk) {
219
246
  const { config, partition, executionPlan, dtype, numTokens, byteLength, wireByteLength, wireCast, shape } = resolvePartitionInput(state, input, prefill, true);
220
247
  const chunkLayers = resolvePrefillRecorderChunkLayers({
221
248
  configuredPrefillChunkLayers: state.runtimeConfig.inference.session.prefillChunkLayers,