doppler-gpu 0.7.7 → 0.7.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/README.md +9 -17
- package/package.json +1 -1
- package/src/config/kernels/kernel-ref-digests.js +2 -1
- package/src/config/kernels/registry.json +47 -19
- package/src/config/schema/execution/kernel-warmup.schema.d.ts +14 -0
- package/src/config/schema/execution/kernel-warmup.schema.js +3 -0
- package/src/config/schema/platform/gpu-cache.schema.js +1 -1
- package/src/config/schema/runtime/buffer-pool.schema.js +2 -2
- package/src/config/schema/runtime/loading.schema.d.ts +5 -0
- package/src/config/schema/runtime/loading.schema.js +25 -2
- package/src/gpu/kernels/moe_route_q4_0_block_f32.wgsl +74 -0
- package/src/gpu/kernels/moe_route_q4_0_split_block_f32.wgsl +35 -18
- package/src/inference/pipelines/text/partitions/partition-execution.js +28 -1
- package/src/inference/pipelines/text/partitions/resident-partition-warmup.js +67 -0
- package/src/inference/pipelines/text/partitions/resident-partition.js +8 -2
- package/src/loader/experts/compact-expert-cache.js +20 -0
- package/src/loader/experts/loader-bindings.js +14 -3
- package/src/loader/experts/mapped-expert-staging.js +393 -0
- package/src/loader/experts/routed-expert-cache.d.ts +8 -0
- package/src/loader/experts/routed-expert-cache.js +12 -11
- package/src/loader/experts/routed-expert-prefill.js +77 -1
- package/src/loader/experts/routed-expert-source.js +1 -0
- package/src/loader/experts/tiered-expert-cache.js +66 -26
- package/src/storage/verified-piece-direct.js +153 -0
- package/src/storage/verified-piece-gather-lane.js +21 -3
- package/src/storage/verified-piece-gather-worker.js +235 -16
- package/src/storage/verified-piece-gather.d.ts +47 -0
- package/src/storage/verified-piece-gather.js +133 -17
- package/src/storage/verified-piece-reader.d.ts +8 -1
- package/src/storage/verified-piece-reader.js +16 -3
- package/src/storage/verified-piece-state-cache.d.ts +75 -0
- package/src/storage/verified-piece-state-cache.js +511 -0
- package/src/storage/verified-piece-storage.d.ts +20 -1
- package/src/storage/verified-piece-storage.js +207 -9
- package/src/version.js +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -4,6 +4,24 @@ Historical package notes are preserved outside the runtime archive in
|
|
|
4
4
|
`docs/status/archive/package-changelog-through-0.6.1.md` and the earlier
|
|
5
5
|
`package-changelog-through-0.4.15.md` snapshot in that directory.
|
|
6
6
|
|
|
7
|
+
## [0.7.8] - 2026-11-19
|
|
8
|
+
|
|
9
|
+
### Performance
|
|
10
|
+
|
|
11
|
+
- Routed expert weights can be staged through mapped upload buffers (`qwen35Routed.uploadStaging: "mapped"`,
|
|
12
|
+
off by default), with first-time bytes read while worker lanes hash them and one upload's hashing spread
|
|
13
|
+
over idle lanes.
|
|
14
|
+
- `qwen35Routed.uploadVerification: "before-release"` (off by default, mapped staging only) lets a partition
|
|
15
|
+
stage compute while hashes run and holds its output until every hash from that step has passed. The default
|
|
16
|
+
stays `"before-upload"`.
|
|
17
|
+
- Verified gather lanes start together, piece-directory identity checks run synchronously, and a resident
|
|
18
|
+
stage can run an optional warm prompt pass while it opens.
|
|
19
|
+
|
|
20
|
+
### Correctness
|
|
21
|
+
|
|
22
|
+
- A piece served from the protected cache no longer marks the file's own piece as verified.
|
|
23
|
+
- Q4_0 routed-expert kernels, the file-descriptor-bound piece state cache and tiered preload are included.
|
|
24
|
+
|
|
7
25
|
## [0.7.7] - 2026-11-18
|
|
8
26
|
|
|
9
27
|
### Packaging
|
package/README.md
CHANGED
|
@@ -1,13 +1,12 @@
|
|
|
1
1
|
# Doppler
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
Run open models locally with WebGPU from JavaScript. Doppler generates text, computes embeddings, reranks results, and scores choices on supported models and devices.
|
|
4
4
|
|
|
5
|
-
|
|
5
|
+
**[Try the token demo](https://d4da.com/doppler/demo/index.html)** · [Open Observatory](https://d4da.com/observatory/) · [Run locally](#get-started)
|
|
6
6
|
|
|
7
|
-
A
|
|
8
|
-
embeddings, reranking, and choice scoring, qualified per model and host.
|
|
7
|
+
<a href="assets/readme/doppler-demo-hero.mp4"><img src="assets/readme/doppler-demo-hero.gif" width="100%" alt="A recorded Qwen answer appears token by token. Selecting a token reveals its alternatives and their probabilities." /></a>
|
|
9
8
|
|
|
10
|
-
|
|
9
|
+
The demo opens with a recorded answer. Select a token to inspect its alternatives, or load a compatible model to generate on your device. [Watch the full-quality recording](assets/readme/doppler-demo-hero.mp4)
|
|
11
10
|
|
|
12
11
|
<picture>
|
|
13
12
|
<source media="(max-width: 640px)" srcset="assets/readme/application-mobile.svg" />
|
|
@@ -16,18 +15,15 @@ embeddings, reranking, and choice scoring, qualified per model and host.
|
|
|
16
15
|
|
|
17
16
|
## Get started
|
|
18
17
|
|
|
19
|
-
|
|
18
|
+
Use Node 20+ and a working WebGPU provider. This command pins the published CLI and model alias used in the installation guide.
|
|
20
19
|
|
|
21
20
|
```sh
|
|
22
21
|
npx --package doppler-gpu@0.7.0 doppler-gpu --model gemma3-270m --prompt "Describe WebGPU briefly"
|
|
23
22
|
```
|
|
24
23
|
|
|
25
|
-
|
|
26
|
-
separately. The [installation guide](docs/getting-started.md) covers provider
|
|
27
|
-
requirements and embedding the published compatibility API.
|
|
24
|
+
The first run downloads about 400 MB of weights plus the tokenizer and manifest. See [installation and embedding](docs/getting-started.md) for host requirements.
|
|
28
25
|
|
|
29
|
-
|
|
30
|
-
0.7.0 with separate release requirements; qualification stays per model and host.
|
|
26
|
+
[Published versions](https://www.npmjs.com/package/doppler-gpu) · [Source changelog](CHANGELOG.md) · [Signed Run API](docs/api/root.md)
|
|
31
27
|
|
|
32
28
|
## Benchmarks
|
|
33
29
|
|
|
@@ -44,11 +40,7 @@ The [signed Run API](docs/api/root.md) and resident partition exports ship in
|
|
|
44
40
|
|
|
45
41
|

|
|
46
42
|
|
|
47
|
-
Rig prepares signed
|
|
48
|
-
and selects an accepted plan; session operations reuse resident resources through
|
|
49
|
-
the model program, binder, and command executor. Applications supply trust and
|
|
50
|
-
host ports. JSON declares execution, JavaScript orchestrates, and WGSL computes.
|
|
51
|
-
Doe and Reploid remain optional integrations.
|
|
43
|
+
Rig prepares a signed Capsule and its execution plans. Run verifies the artifacts and selects a plan. A session reuses the loaded weights and GPU programs across requests. The application supplies trust and host access.
|
|
52
44
|
|
|
53
45
|
[Architecture and source owners](docs/architecture.md) · [Resident partition lifetimes](docs/architecture.md#resident-partitions-and-resource-lifetime)
|
|
54
46
|
|
|
@@ -65,7 +57,7 @@ from verified execution and links the lifecycle evidence for each model.
|
|
|
65
57
|
The [comparison scoreboard](docs/model-competition-scoreboard.md) reports
|
|
66
58
|
throughput and loading separately against declared controls. Runtime correctness,
|
|
67
59
|
task quality, memory, and lifecycle acceptance remain separate checks.
|
|
68
|
-
W8A8 remains experimental
|
|
60
|
+
W8A8 remains experimental.
|
|
69
61
|
|
|
70
62
|
Choice scores are uncalibrated. Unsupported paths fail closed. The retained
|
|
71
63
|
[search starter](examples/document-search/README.md) needs renewed signed
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "doppler-gpu",
|
|
3
|
-
"version": "0.7.
|
|
3
|
+
"version": "0.7.8",
|
|
4
4
|
"description": "Doppler Rig prepares signed model Capsules; Doppler Run executes them locally through WebGPU for JavaScript",
|
|
5
5
|
"main": "src/capsule-runtime.js",
|
|
6
6
|
"types": "src/capsule-runtime.d.ts",
|
|
@@ -356,8 +356,9 @@ export const KERNEL_REF_CONTENT_DIGESTS = Object.freeze({
|
|
|
356
356
|
"moe_norm_router_f32.wgsl#single_pass": "87646d02b567b3c13f66245ac222392df92af863da920ce562caa77dd14f2e93",
|
|
357
357
|
"moe_offsets.wgsl#build_offsets": "3ea004145fa234659408cdeb0d4d802adff1037c9c5c03af146b3734cc69dd27",
|
|
358
358
|
"moe_route_gguf_split_f32.wgsl#main": "32a342a5b289797719ab6438f0e7d482ac16cb15a5f6315fbbf4758ac724cebd",
|
|
359
|
+
"moe_route_q4_0_block_f32.wgsl#main": "912d5445e34ce2611356a1a14c6f881630d0a88405035e843a7f676eb9ec9f16",
|
|
359
360
|
"moe_route_q4_0_f32.wgsl#main": "93701086c9ce454d08a7b22869fc931d186fe3d45e7b6ca95b7b72ee26b1d5be",
|
|
360
|
-
"moe_route_q4_0_split_block_f32.wgsl#main": "
|
|
361
|
+
"moe_route_q4_0_split_block_f32.wgsl#main": "3b003b37dd66eb24786464a4c553fb57275f5f18de03361ae225d3e3339a5d59",
|
|
361
362
|
"moe_route_q4_1_block_f32.wgsl#main": "57291511788508e5333e244809e821c6edf71ee1dc6bfcd6bb19c3f4ace78d23",
|
|
362
363
|
"moe_route_q4_f32.wgsl#main_multicol": "22a02228b1425e2b7354b4e87c275f7b0de166925fd7cf30945d675c37aa86c8",
|
|
363
364
|
"moe_route_remap.wgsl#remap_routes": "092c11f4603ca96496a000d56f3bbb0211b6f536fd328bd3a9ee668b262353d3",
|
|
@@ -7298,7 +7298,10 @@
|
|
|
7298
7298
|
"description": "Q4_K GEMV with 1 contiguous subgroup lanes per row.",
|
|
7299
7299
|
"reachability": {
|
|
7300
7300
|
"status": "pinned",
|
|
7301
|
-
"inlineConfigs": [
|
|
7301
|
+
"inlineConfigs": [
|
|
7302
|
+
"runtime/profiles/qwen-3-5-0-8b-vulkan-subgroup8",
|
|
7303
|
+
"runtime/profiles/qwen3-8-27b-orin-projections"
|
|
7304
|
+
],
|
|
7302
7305
|
"ruleChains": [],
|
|
7303
7306
|
"wgslOnDisk": true
|
|
7304
7307
|
},
|
|
@@ -7349,7 +7352,10 @@
|
|
|
7349
7352
|
"description": "Q4_K GEMV with 2 contiguous subgroup lanes per row.",
|
|
7350
7353
|
"reachability": {
|
|
7351
7354
|
"status": "pinned",
|
|
7352
|
-
"inlineConfigs": [
|
|
7355
|
+
"inlineConfigs": [
|
|
7356
|
+
"runtime/profiles/qwen-3-5-0-8b-vulkan-subgroup8",
|
|
7357
|
+
"runtime/profiles/qwen3-8-27b-orin-projections"
|
|
7358
|
+
],
|
|
7353
7359
|
"ruleChains": [],
|
|
7354
7360
|
"wgslOnDisk": true
|
|
7355
7361
|
},
|
|
@@ -7400,7 +7406,10 @@
|
|
|
7400
7406
|
"description": "Q4_K GEMV with 4 contiguous subgroup lanes per row.",
|
|
7401
7407
|
"reachability": {
|
|
7402
7408
|
"status": "pinned",
|
|
7403
|
-
"inlineConfigs": [
|
|
7409
|
+
"inlineConfigs": [
|
|
7410
|
+
"runtime/profiles/qwen-3-5-0-8b-vulkan-subgroup8",
|
|
7411
|
+
"runtime/profiles/qwen3-8-27b-orin-projections"
|
|
7412
|
+
],
|
|
7404
7413
|
"ruleChains": [],
|
|
7405
7414
|
"wgslOnDisk": true
|
|
7406
7415
|
},
|
|
@@ -7451,7 +7460,10 @@
|
|
|
7451
7460
|
"description": "Q4_K GEMV with 8 contiguous subgroup lanes per row.",
|
|
7452
7461
|
"reachability": {
|
|
7453
7462
|
"status": "pinned",
|
|
7454
|
-
"inlineConfigs": [
|
|
7463
|
+
"inlineConfigs": [
|
|
7464
|
+
"runtime/profiles/qwen-3-5-0-8b-vulkan-subgroup8",
|
|
7465
|
+
"runtime/profiles/qwen3-8-27b-orin-projections"
|
|
7466
|
+
],
|
|
7455
7467
|
"ruleChains": [],
|
|
7456
7468
|
"wgslOnDisk": true
|
|
7457
7469
|
},
|
|
@@ -7502,7 +7514,10 @@
|
|
|
7502
7514
|
"description": "Q4_K GEMV with 16 contiguous subgroup lanes per row.",
|
|
7503
7515
|
"reachability": {
|
|
7504
7516
|
"status": "pinned",
|
|
7505
|
-
"inlineConfigs": [
|
|
7517
|
+
"inlineConfigs": [
|
|
7518
|
+
"runtime/profiles/qwen-3-5-0-8b-vulkan-subgroup8",
|
|
7519
|
+
"runtime/profiles/qwen3-8-27b-orin-projections"
|
|
7520
|
+
],
|
|
7506
7521
|
"ruleChains": [],
|
|
7507
7522
|
"wgslOnDisk": true
|
|
7508
7523
|
},
|
|
@@ -7553,7 +7568,10 @@
|
|
|
7553
7568
|
"description": "Q4_K GEMV with 32 contiguous subgroup lanes per row.",
|
|
7554
7569
|
"reachability": {
|
|
7555
7570
|
"status": "pinned",
|
|
7556
|
-
"inlineConfigs": [
|
|
7571
|
+
"inlineConfigs": [
|
|
7572
|
+
"runtime/profiles/qwen-3-5-0-8b-vulkan-subgroup8",
|
|
7573
|
+
"runtime/profiles/qwen3-8-27b-orin-projections"
|
|
7574
|
+
],
|
|
7557
7575
|
"ruleChains": [],
|
|
7558
7576
|
"wgslOnDisk": true
|
|
7559
7577
|
},
|
|
@@ -8047,7 +8065,11 @@
|
|
|
8047
8065
|
],
|
|
8048
8066
|
"reachability": {
|
|
8049
8067
|
"status": "pinned",
|
|
8050
|
-
"inlineConfigs": [
|
|
8068
|
+
"inlineConfigs": [
|
|
8069
|
+
"runtime/profiles/qwen-3-5-2b-orin-prefill",
|
|
8070
|
+
"runtime/profiles/qwen3-8-27b-orin-prefill",
|
|
8071
|
+
"runtime/profiles/qwen3-8-27b-orin-projections"
|
|
8072
|
+
],
|
|
8051
8073
|
"ruleChains": [],
|
|
8052
8074
|
"wgslOnDisk": true
|
|
8053
8075
|
}
|
|
@@ -14750,7 +14772,10 @@
|
|
|
14750
14772
|
"description": "Q4_K fused gate/up GEMV with one eight-lane subgroup partition per output row.",
|
|
14751
14773
|
"reachability": {
|
|
14752
14774
|
"status": "pinned",
|
|
14753
|
-
"inlineConfigs": [
|
|
14775
|
+
"inlineConfigs": [
|
|
14776
|
+
"runtime/profiles/qwen-3-5-0-8b-vulkan-subgroup8",
|
|
14777
|
+
"runtime/profiles/qwen3-8-27b-orin-decode"
|
|
14778
|
+
],
|
|
14754
14779
|
"ruleChains": [],
|
|
14755
14780
|
"wgslOnDisk": true
|
|
14756
14781
|
},
|
|
@@ -24810,8 +24835,11 @@
|
|
|
24810
24835
|
"outputDtype": "f32",
|
|
24811
24836
|
"description": "Packed F16 bits in u32 split buffers -> F32 output without shader-f16",
|
|
24812
24837
|
"reachability": {
|
|
24813
|
-
"status": "
|
|
24814
|
-
"inlineConfigs": [
|
|
24838
|
+
"status": "pinned",
|
|
24839
|
+
"inlineConfigs": [
|
|
24840
|
+
"runtime/profiles/qwen3-8-27b-orin-prefill",
|
|
24841
|
+
"runtime/profiles/qwen3-8-27b-orin-projections"
|
|
24842
|
+
],
|
|
24815
24843
|
"ruleChains": [],
|
|
24816
24844
|
"wgslOnDisk": true
|
|
24817
24845
|
}
|
|
@@ -35723,9 +35751,9 @@
|
|
|
35723
35751
|
}
|
|
35724
35752
|
},
|
|
35725
35753
|
"q4_0_f32": {
|
|
35726
|
-
"wgsl": "
|
|
35754
|
+
"wgsl": "moe_route_q4_0_block_f32.wgsl",
|
|
35727
35755
|
"entryPoint": "main",
|
|
35728
|
-
"workgroup": [
|
|
35756
|
+
"workgroup": [256,1,1],
|
|
35729
35757
|
"requires": [],
|
|
35730
35758
|
"bindings": [
|
|
35731
35759
|
{
|
|
@@ -35802,15 +35830,15 @@
|
|
|
35802
35830
|
"weightDtype": "q4_0",
|
|
35803
35831
|
"outputDtype": "f32",
|
|
35804
35832
|
"wgslOverrides": {
|
|
35805
|
-
"WORKGROUP_SIZE":
|
|
35806
|
-
"COLS_PER_WG":
|
|
35833
|
+
"WORKGROUP_SIZE": 256,
|
|
35834
|
+
"COLS_PER_WG": 8,
|
|
35807
35835
|
"THREADS_PER_COL_GEMV": 32,
|
|
35808
35836
|
"TILE_SIZE": 16,
|
|
35809
35837
|
"BLOCK_VALUES": 32,
|
|
35810
35838
|
"BLOCK_BYTES": 18,
|
|
35811
35839
|
"QUANT_FORMAT": 0
|
|
35812
35840
|
},
|
|
35813
|
-
"sharedMemory":
|
|
35841
|
+
"sharedMemory": 1024,
|
|
35814
35842
|
"variantMetadata": {
|
|
35815
35843
|
"workgroupConstants": ["WORKGROUP_SIZE",null,null]
|
|
35816
35844
|
},
|
|
@@ -36121,7 +36149,7 @@
|
|
|
36121
36149
|
"q4_0_split_f32": {
|
|
36122
36150
|
"wgsl": "moe_route_q4_0_split_block_f32.wgsl",
|
|
36123
36151
|
"entryPoint": "main",
|
|
36124
|
-
"workgroup": [
|
|
36152
|
+
"workgroup": [256,1,1],
|
|
36125
36153
|
"requires": [],
|
|
36126
36154
|
"bindings": [
|
|
36127
36155
|
{
|
|
@@ -36203,15 +36231,15 @@
|
|
|
36203
36231
|
"weightDtype": "q4_0",
|
|
36204
36232
|
"outputDtype": "f32",
|
|
36205
36233
|
"wgslOverrides": {
|
|
36206
|
-
"WORKGROUP_SIZE":
|
|
36207
|
-
"COLS_PER_WG":
|
|
36234
|
+
"WORKGROUP_SIZE": 256,
|
|
36235
|
+
"COLS_PER_WG": 8,
|
|
36208
36236
|
"THREADS_PER_COL_GEMV": 32,
|
|
36209
36237
|
"TILE_SIZE": 16,
|
|
36210
36238
|
"BLOCK_VALUES": 32,
|
|
36211
36239
|
"BLOCK_BYTES": 18,
|
|
36212
36240
|
"QUANT_FORMAT": 0
|
|
36213
36241
|
},
|
|
36214
|
-
"sharedMemory":
|
|
36242
|
+
"sharedMemory": 1024,
|
|
36215
36243
|
"variantMetadata": {
|
|
36216
36244
|
"workgroupConstants": ["WORKGROUP_SIZE",null,null]
|
|
36217
36245
|
},
|
|
@@ -19,6 +19,20 @@ export interface KernelWarmupConfigSchema {
|
|
|
19
19
|
* null runs none.
|
|
20
20
|
*/
|
|
21
21
|
warmGeneration: KernelWarmupGenerationSchema | null;
|
|
22
|
+
/**
|
|
23
|
+
* One prompt pass through a resident partition's own layers while the partition opens: its
|
|
24
|
+
* pipelines are compiled and, on a routed-expert stage, the experts those tokens select are
|
|
25
|
+
* loaded before the first request. Runs in a private attempt that is closed before readiness.
|
|
26
|
+
* Counted in opening time. null runs none.
|
|
27
|
+
*/
|
|
28
|
+
warmPartition: KernelWarmupPartitionSchema | null;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export interface KernelWarmupPartitionSchema {
|
|
32
|
+
/** Token ids of the warm prompt. A stage that starts from a boundary activation runs zeros of this length. */
|
|
33
|
+
tokenIds: number[];
|
|
34
|
+
/** Single-token steps after the prompt pass (0 to 8). */
|
|
35
|
+
decodeSteps: number;
|
|
22
36
|
}
|
|
23
37
|
|
|
24
38
|
export interface KernelWarmupGenerationSchema {
|
|
@@ -4,4 +4,7 @@ export const DEFAULT_KERNEL_WARMUP_CONFIG = {
|
|
|
4
4
|
// { prompt, maxTokens }: one greedy generation at the end of a text pipeline load
|
|
5
5
|
// (counted in load time), then a sequence reset. null runs none.
|
|
6
6
|
warmGeneration: null,
|
|
7
|
+
// { tokenIds, decodeSteps }: one prompt pass through a resident partition's own layers while it
|
|
8
|
+
// opens (counted in opening time), in a private attempt that is closed afterwards. null runs none.
|
|
9
|
+
warmPartition: null,
|
|
7
10
|
};
|
|
@@ -15,8 +15,8 @@ export const DEFAULT_BUFFER_POOL_BUCKET_CONFIG = {
|
|
|
15
15
|
// =============================================================================
|
|
16
16
|
|
|
17
17
|
export const DEFAULT_BUFFER_POOL_LIMITS_CONFIG = {
|
|
18
|
-
maxBuffersPerBucket:
|
|
19
|
-
maxTotalPooledBuffers:
|
|
18
|
+
maxBuffersPerBucket: 1024,
|
|
19
|
+
maxTotalPooledBuffers: 4096,
|
|
20
20
|
};
|
|
21
21
|
|
|
22
22
|
// =============================================================================
|
|
@@ -197,6 +197,11 @@ export interface Qwen35RoutedConfig {
|
|
|
197
197
|
sourceReadMode: 'plane-serial' | 'expert-parallel' | 'lease-parallel' | 'expert-lease-batch';
|
|
198
198
|
/** Write authenticated source ranges to their logical working-bank offsets without temporary expert buffers. */
|
|
199
199
|
prefillUploadMode: 'expert-copy' | 'direct-bank';
|
|
200
|
+
/** 'mapped' stages already verified expert ranges in mapped upload buffers instead of queue.writeBuffer. Omission means 'queue'. */
|
|
201
|
+
uploadStaging?: 'queue' | 'mapped';
|
|
202
|
+
/** When first-time expert bytes are verified: before they reach a model buffer (default), or, with mapped
|
|
203
|
+
* staging in a resident partition step, before that step's output is released. */
|
|
204
|
+
uploadVerification?: 'before-upload' | 'before-release';
|
|
200
205
|
/** LFU retains causal lifetime counts until unload; LRU remains the default. */
|
|
201
206
|
evictionPolicy: 'lru' | 'lfu';
|
|
202
207
|
cacheBytes: number;
|
|
@@ -103,6 +103,15 @@ export const DEFAULT_EXPERT_CACHE_CONFIG = {
|
|
|
103
103
|
sourceReadMode: 'plane-serial',
|
|
104
104
|
expertBatchSize: 1,
|
|
105
105
|
prefillUploadMode: 'expert-copy',
|
|
106
|
+
// 'queue' copies each expert through queue.writeBuffer. 'mapped' reads already verified ranges
|
|
107
|
+
// straight into mapped upload buffers when the storage port can serve them, and otherwise does
|
|
108
|
+
// what 'queue' does.
|
|
109
|
+
uploadStaging: 'queue',
|
|
110
|
+
// 'before-upload': expert bytes are verified before they are copied into a model buffer.
|
|
111
|
+
// 'before-release': with mapped staging inside a resident partition step, first-time bytes are
|
|
112
|
+
// copied and computed with while lanes hash them, and the step's output is withheld until every
|
|
113
|
+
// hash has passed. A failed hash fails the step and refuses all further steps until reopened.
|
|
114
|
+
uploadVerification: 'before-upload',
|
|
106
115
|
evictionPolicy: 'lru',
|
|
107
116
|
cacheBytes: 2 * GB,
|
|
108
117
|
hostCacheBytes: 0,
|
|
@@ -135,9 +144,11 @@ export function validateQwen35RoutedConfig(config) {
|
|
|
135
144
|
|| tiered.mappedProtectedShare < 0 || tiered.mappedProtectedShare > 1
|
|
136
145
|
|| !Number.isSafeInteger(tiered.mappedCacheBytes) || tiered.mappedCacheBytes < 0 || tiered.mappedCacheBytes % 4
|
|
137
146
|
|| config.compactPrefillBytes !== 0 || config.compactPrefillReuse !== 'none'
|
|
138
|
-
|| config.prefillCompactAdmission !== 'none'
|
|
147
|
+
|| config.prefillCompactAdmission !== 'none'
|
|
148
|
+
|| !['expert-lease-batch', 'expert-parallel'].includes(config.sourceReadMode)
|
|
149
|
+
|| (config.sourceReadMode === 'expert-parallel' && (tiered.parkedLayers.length > 0 || tiered.mappedCacheBytes > 0))
|
|
139
150
|
|| config.prefillUploadMode !== 'expert-copy') {
|
|
140
|
-
throw new Error('Tiered experts require distinct resident/parked layers, aligned mappedCacheBytes, mappedProtectedShare in [0,1] and batched streaming without full prefill banks.');
|
|
151
|
+
throw new Error('Tiered experts require distinct resident/parked layers, aligned mappedCacheBytes, mappedProtectedShare in [0,1] and batched or parallel streaming without full prefill banks.');
|
|
141
152
|
}
|
|
142
153
|
} else if (config.tiered != null) {
|
|
143
154
|
throw new Error('Tiered placement requires storageMode=tiered-compact.');
|
|
@@ -174,6 +185,18 @@ export function validateQwen35RoutedConfig(config) {
|
|
|
174
185
|
|| (expertBatchSize > 1 && (!compact || config.sourceReadMode !== 'expert-lease-batch'))) {
|
|
175
186
|
throw new Error('expertBatchSize requires 1..32 and batched compact source leases for multiple experts.');
|
|
176
187
|
}
|
|
188
|
+
const uploadStaging = config.uploadStaging === undefined
|
|
189
|
+
? DEFAULT_EXPERT_CACHE_CONFIG.qwen35Routed.uploadStaging : config.uploadStaging;
|
|
190
|
+
if (!['queue', 'mapped'].includes(uploadStaging)
|
|
191
|
+
|| (uploadStaging === 'mapped' && (!compact || config.sourceReadMode !== 'expert-lease-batch' || config.hostCacheBytes !== 0))) {
|
|
192
|
+
throw new Error('uploadStaging requires queue, or mapped with batched compact source leases and no host cache.');
|
|
193
|
+
}
|
|
194
|
+
const uploadVerification = config.uploadVerification === undefined
|
|
195
|
+
? DEFAULT_EXPERT_CACHE_CONFIG.qwen35Routed.uploadVerification : config.uploadVerification;
|
|
196
|
+
if (!['before-upload', 'before-release'].includes(uploadVerification)
|
|
197
|
+
|| (uploadVerification === 'before-release' && uploadStaging !== 'mapped')) {
|
|
198
|
+
throw new Error('uploadVerification requires before-upload, or before-release with mapped upload staging.');
|
|
199
|
+
}
|
|
177
200
|
if (!['expert-copy', 'direct-bank'].includes(config.prefillUploadMode)
|
|
178
201
|
|| (config.prefillUploadMode === 'direct-bank'
|
|
179
202
|
&& (config.storageMode !== 'compact-decode' || config.hostCacheBytes !== 0))) {
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
// Native Q4_0 routed projection. One lane consumes a full 32-value block via whole-word u32 loads.
|
|
2
|
+
// Tuned for 256-thread / 8-column Metal & Vulkan threadgroups while preserving exact block accumulation.
|
|
3
|
+
override WORKGROUP_SIZE: u32 = 256u;
|
|
4
|
+
override COLS_PER_WG: u32 = 8u;
|
|
5
|
+
override THREADS_PER_COL_GEMV: u32 = 32u;
|
|
6
|
+
override TILE_SIZE: u32 = 16u;
|
|
7
|
+
override BLOCK_VALUES: u32 = 32u;
|
|
8
|
+
override BLOCK_BYTES: u32 = 18u;
|
|
9
|
+
override QUANT_FORMAT: u32 = 0u;
|
|
10
|
+
struct Uniforms {
|
|
11
|
+
M: u32, N: u32, K: u32, alpha: f32,
|
|
12
|
+
blocks: u32, num_experts: u32, top_k: u32, input_mode: u32,
|
|
13
|
+
}
|
|
14
|
+
@group(0) @binding(0) var<uniform> u: Uniforms;
|
|
15
|
+
@group(0) @binding(1) var<storage, read> input: array<vec4<f32>>;
|
|
16
|
+
@group(0) @binding(2) var<storage, read> weights: array<u32>;
|
|
17
|
+
@group(0) @binding(3) var<storage, read_write> output: array<f32>;
|
|
18
|
+
@group(0) @binding(4) var<storage, read> route_indices: array<u32>;
|
|
19
|
+
var<workgroup> partial: array<f32, WORKGROUP_SIZE>;
|
|
20
|
+
fn unpack_bytes(word: u32) -> vec4<u32> {
|
|
21
|
+
return vec4<u32>(word, word >> 8u, word >> 16u, word >> 24u) & vec4<u32>(255u);
|
|
22
|
+
}
|
|
23
|
+
@compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
|
|
24
|
+
fn main(@builtin(workgroup_id) group: vec3<u32>, @builtin(num_workgroups) grid: vec3<u32>,
|
|
25
|
+
@builtin(local_invocation_index) lane: u32) {
|
|
26
|
+
let tiles_n = (u.N + COLS_PER_WG - 1u) / COLS_PER_WG;
|
|
27
|
+
let tile = group.x + group.y * grid.x;
|
|
28
|
+
let m = tile / tiles_n;
|
|
29
|
+
if (m >= u.M) { return; }
|
|
30
|
+
let expert = route_indices[m];
|
|
31
|
+
let token = select(m / u.top_k, m, u.input_mode == 1u);
|
|
32
|
+
let column = lane / THREADS_PER_COL_GEMV;
|
|
33
|
+
let thread = lane % THREADS_PER_COL_GEMV;
|
|
34
|
+
let n = (tile % tiles_n) * COLS_PER_WG + column;
|
|
35
|
+
var sum = 0.0f;
|
|
36
|
+
if (n < u.N && expert < u.num_experts) {
|
|
37
|
+
let row = expert * u.N + n;
|
|
38
|
+
for (var block = thread; block < u.blocks; block += THREADS_PER_COL_GEMV) {
|
|
39
|
+
let offset = (row * u.blocks + block) * BLOCK_BYTES;
|
|
40
|
+
let w_base = offset / 4u;
|
|
41
|
+
let half_aligned = (offset & 2u) != 0u;
|
|
42
|
+
let w0 = weights[w_base];
|
|
43
|
+
let w1 = weights[w_base + 1u];
|
|
44
|
+
let w2 = weights[w_base + 2u];
|
|
45
|
+
let w3 = weights[w_base + 3u];
|
|
46
|
+
let w4 = weights[w_base + 4u];
|
|
47
|
+
let scale = select(unpack2x16float(w0).x, unpack2x16float(w0).y, half_aligned);
|
|
48
|
+
let q0 = select((w0 >> 16u) | (w1 << 16u), w1, half_aligned);
|
|
49
|
+
let q1 = select((w1 >> 16u) | (w2 << 16u), w2, half_aligned);
|
|
50
|
+
let q2 = select((w2 >> 16u) | (w3 << 16u), w3, half_aligned);
|
|
51
|
+
let q3 = select((w3 >> 16u) | (w4 << 16u), w4, half_aligned);
|
|
52
|
+
let vec_base = (token * u.K + block * BLOCK_VALUES) / 4u;
|
|
53
|
+
let p0 = unpack_bytes(q0);
|
|
54
|
+
sum += dot(input[vec_base], (vec4<f32>(p0 & vec4<u32>(15u)) - vec4<f32>(8.0)) * scale)
|
|
55
|
+
+ dot(input[vec_base + 4u], (vec4<f32>(p0 >> vec4<u32>(4u)) - vec4<f32>(8.0)) * scale);
|
|
56
|
+
let p1 = unpack_bytes(q1);
|
|
57
|
+
sum += dot(input[vec_base + 1u], (vec4<f32>(p1 & vec4<u32>(15u)) - vec4<f32>(8.0)) * scale)
|
|
58
|
+
+ dot(input[vec_base + 5u], (vec4<f32>(p1 >> vec4<u32>(4u)) - vec4<f32>(8.0)) * scale);
|
|
59
|
+
let p2 = unpack_bytes(q2);
|
|
60
|
+
sum += dot(input[vec_base + 2u], (vec4<f32>(p2 & vec4<u32>(15u)) - vec4<f32>(8.0)) * scale)
|
|
61
|
+
+ dot(input[vec_base + 6u], (vec4<f32>(p2 >> vec4<u32>(4u)) - vec4<f32>(8.0)) * scale);
|
|
62
|
+
let p3 = unpack_bytes(q3);
|
|
63
|
+
sum += dot(input[vec_base + 3u], (vec4<f32>(p3 & vec4<u32>(15u)) - vec4<f32>(8.0)) * scale)
|
|
64
|
+
+ dot(input[vec_base + 7u], (vec4<f32>(p3 >> vec4<u32>(4u)) - vec4<f32>(8.0)) * scale);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
partial[lane] = sum;
|
|
68
|
+
workgroupBarrier();
|
|
69
|
+
for (var stride = THREADS_PER_COL_GEMV / 2u; stride > 0u; stride /= 2u) {
|
|
70
|
+
if (thread < stride) { partial[lane] += partial[lane + stride]; }
|
|
71
|
+
workgroupBarrier();
|
|
72
|
+
}
|
|
73
|
+
if (thread == 0u && n < u.N) { output[m * u.N + n] = partial[lane] * u.alpha; }
|
|
74
|
+
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
|
-
// Native Q4_0 separate gate/up projection. One lane consumes a full 32-value block.
|
|
1
|
+
// Native Q4_0 separate gate/up projection. One lane consumes a full 32-value block via whole-word u32 loads.
|
|
2
2
|
// Preserves packed scales and quantization; output remains [routes, gate then up].
|
|
3
|
-
override WORKGROUP_SIZE: u32 =
|
|
4
|
-
override COLS_PER_WG: u32 =
|
|
3
|
+
override WORKGROUP_SIZE: u32 = 256u;
|
|
4
|
+
override COLS_PER_WG: u32 = 8u;
|
|
5
5
|
override THREADS_PER_COL_GEMV: u32 = 32u;
|
|
6
6
|
override TILE_SIZE: u32 = 16u;
|
|
7
7
|
override BLOCK_VALUES: u32 = 32u;
|
|
@@ -12,15 +12,18 @@ struct Uniforms {
|
|
|
12
12
|
blocks: u32, num_experts: u32, top_k: u32, input_mode: u32,
|
|
13
13
|
}
|
|
14
14
|
@group(0) @binding(0) var<uniform> u: Uniforms;
|
|
15
|
-
@group(0) @binding(1) var<storage, read> input: array<f32
|
|
15
|
+
@group(0) @binding(1) var<storage, read> input: array<vec4<f32>>;
|
|
16
16
|
@group(0) @binding(2) var<storage, read> weights: array<u32>;
|
|
17
17
|
@group(0) @binding(3) var<storage, read_write> output: array<f32>;
|
|
18
18
|
@group(0) @binding(4) var<storage, read> route_indices: array<u32>;
|
|
19
19
|
@group(0) @binding(5) var<storage, read> up_weights: array<u32>;
|
|
20
20
|
var<workgroup> partial: array<f32, WORKGROUP_SIZE>;
|
|
21
|
-
fn
|
|
22
|
-
if (up) { return
|
|
23
|
-
return
|
|
21
|
+
fn source_word(idx: u32, up: bool) -> u32 {
|
|
22
|
+
if (up) { return up_weights[idx]; }
|
|
23
|
+
return weights[idx];
|
|
24
|
+
}
|
|
25
|
+
fn unpack_bytes(word: u32) -> vec4<u32> {
|
|
26
|
+
return vec4<u32>(word, word >> 8u, word >> 16u, word >> 24u) & vec4<u32>(255u);
|
|
24
27
|
}
|
|
25
28
|
@compute @workgroup_size(WORKGROUP_SIZE, 1, 1)
|
|
26
29
|
fn main(@builtin(workgroup_id) group: vec3<u32>, @builtin(num_workgroups) grid: vec3<u32>,
|
|
@@ -40,17 +43,31 @@ fn main(@builtin(workgroup_id) group: vec3<u32>, @builtin(num_workgroups) grid:
|
|
|
40
43
|
let up = n >= u.N / 2u;
|
|
41
44
|
for (var block = thread; block < u.blocks; block += THREADS_PER_COL_GEMV) {
|
|
42
45
|
let offset = (row * u.blocks + block) * BLOCK_BYTES;
|
|
43
|
-
let
|
|
44
|
-
let
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
46
|
+
let w_base = offset / 4u;
|
|
47
|
+
let half_aligned = (offset & 2u) != 0u;
|
|
48
|
+
let w0 = source_word(w_base, up);
|
|
49
|
+
let w1 = source_word(w_base + 1u, up);
|
|
50
|
+
let w2 = source_word(w_base + 2u, up);
|
|
51
|
+
let w3 = source_word(w_base + 3u, up);
|
|
52
|
+
let w4 = source_word(w_base + 4u, up);
|
|
53
|
+
let scale = select(unpack2x16float(w0).x, unpack2x16float(w0).y, half_aligned);
|
|
54
|
+
let q0 = select((w0 >> 16u) | (w1 << 16u), w1, half_aligned);
|
|
55
|
+
let q1 = select((w1 >> 16u) | (w2 << 16u), w2, half_aligned);
|
|
56
|
+
let q2 = select((w2 >> 16u) | (w3 << 16u), w3, half_aligned);
|
|
57
|
+
let q3 = select((w3 >> 16u) | (w4 << 16u), w4, half_aligned);
|
|
58
|
+
let vec_base = (token * u.K + block * BLOCK_VALUES) / 4u;
|
|
59
|
+
let p0 = unpack_bytes(q0);
|
|
60
|
+
sum += dot(input[vec_base], (vec4<f32>(p0 & vec4<u32>(15u)) - vec4<f32>(8.0)) * scale)
|
|
61
|
+
+ dot(input[vec_base + 4u], (vec4<f32>(p0 >> vec4<u32>(4u)) - vec4<f32>(8.0)) * scale);
|
|
62
|
+
let p1 = unpack_bytes(q1);
|
|
63
|
+
sum += dot(input[vec_base + 1u], (vec4<f32>(p1 & vec4<u32>(15u)) - vec4<f32>(8.0)) * scale)
|
|
64
|
+
+ dot(input[vec_base + 5u], (vec4<f32>(p1 >> vec4<u32>(4u)) - vec4<f32>(8.0)) * scale);
|
|
65
|
+
let p2 = unpack_bytes(q2);
|
|
66
|
+
sum += dot(input[vec_base + 2u], (vec4<f32>(p2 & vec4<u32>(15u)) - vec4<f32>(8.0)) * scale)
|
|
67
|
+
+ dot(input[vec_base + 6u], (vec4<f32>(p2 >> vec4<u32>(4u)) - vec4<f32>(8.0)) * scale);
|
|
68
|
+
let p3 = unpack_bytes(q3);
|
|
69
|
+
sum += dot(input[vec_base + 3u], (vec4<f32>(p3 & vec4<u32>(15u)) - vec4<f32>(8.0)) * scale)
|
|
70
|
+
+ dot(input[vec_base + 7u], (vec4<f32>(p3 >> vec4<u32>(4u)) - vec4<f32>(8.0)) * scale);
|
|
54
71
|
}
|
|
55
72
|
}
|
|
56
73
|
partial[lane] = sum;
|
|
@@ -20,6 +20,7 @@ import { getLogitsWeights, getLogitsConfig } from "../logits/logits-config.js";
|
|
|
20
20
|
import { resolveLayerPartition, getPartitionActivationContract, validateActivationTensorShape } from "../execution/layer-partition-contract.js";
|
|
21
21
|
import { createHyperConnectionSession } from "../hyper-connection/session.js";
|
|
22
22
|
import { resolvePrefillTokenChunkSize, shouldDisablePrefillCommandBatching } from "../generator/prefill/prefill-policy.js";
|
|
23
|
+
import { beginStagedRelease, settleStagedRelease } from "../../../../loader/experts/mapped-expert-staging.js";
|
|
23
24
|
|
|
24
25
|
/** @type {import('./partition-execution.js').assertPartitionExecutionSupported} */
|
|
25
26
|
export function assertPartitionExecutionSupported(state, plan, requireAdapter = true) {
|
|
@@ -214,8 +215,34 @@ export function shouldRecordPartitionLayers(state, disableCommandBatching, prefi
|
|
|
214
215
|
{ startPos: state.currentSeqLen, returnHidden: !finalChunk }));
|
|
215
216
|
}
|
|
216
217
|
|
|
217
|
-
/**
|
|
218
|
+
/**
|
|
219
|
+
* With uploadVerification 'before-release', expert bytes loaded for the first time during a chunk are
|
|
220
|
+
* computed with while they are still being hashed. The chunk's activation or logits are handed on only
|
|
221
|
+
* after every one of those hashes has passed; a failure fails the chunk and every later one.
|
|
222
|
+
* @type {import('./partition-execution.js').ExecutePartitionChunk} */
|
|
218
223
|
async function executePartitionChunk(state, input, signal, prefill, finalChunk) {
|
|
224
|
+
const loader = state.dopplerLoader;
|
|
225
|
+
const port = loader?.manifest?.moeConfig && loader.getRoutedExpertPolicy().uploadVerification === 'before-release'
|
|
226
|
+
? loader.shardCache.rangeLease ?? null : null;
|
|
227
|
+
if (!port) return runPartitionChunk(state, input, signal, prefill, finalChunk);
|
|
228
|
+
beginStagedRelease(port);
|
|
229
|
+
let result;
|
|
230
|
+
try { result = await runPartitionChunk(state, input, signal, prefill, finalChunk); }
|
|
231
|
+
catch (error) {
|
|
232
|
+
await settleStagedRelease(port, { release: false });
|
|
233
|
+
throw error;
|
|
234
|
+
}
|
|
235
|
+
try { await settleStagedRelease(port); }
|
|
236
|
+
catch (error) {
|
|
237
|
+
// The output was computed with bytes that failed verification. It never leaves this function.
|
|
238
|
+
if (result.logits) releaseBuffer(result.logits.logitsBuffer);
|
|
239
|
+
throw error;
|
|
240
|
+
}
|
|
241
|
+
return result;
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
/** @type {import('./partition-execution.js').ExecutePartitionChunk} */
|
|
245
|
+
async function runPartitionChunk(state, input, signal, prefill, finalChunk) {
|
|
219
246
|
const { config, partition, executionPlan, dtype, numTokens, byteLength, wireByteLength, wireCast, shape } = resolvePartitionInput(state, input, prefill, true);
|
|
220
247
|
const chunkLayers = resolvePrefillRecorderChunkLayers({
|
|
221
248
|
configuredPrefillChunkLayers: state.runtimeConfig.inference.session.prefillChunkLayers,
|