doppler-gpu 0.7.7 → 0.7.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. package/CHANGELOG.md +18 -0
  2. package/README.md +9 -17
  3. package/package.json +1 -1
  4. package/src/config/kernels/kernel-ref-digests.js +2 -1
  5. package/src/config/kernels/registry.json +47 -19
  6. package/src/config/schema/execution/kernel-warmup.schema.d.ts +14 -0
  7. package/src/config/schema/execution/kernel-warmup.schema.js +3 -0
  8. package/src/config/schema/platform/gpu-cache.schema.js +1 -1
  9. package/src/config/schema/runtime/buffer-pool.schema.js +2 -2
  10. package/src/config/schema/runtime/loading.schema.d.ts +5 -0
  11. package/src/config/schema/runtime/loading.schema.js +25 -2
  12. package/src/gpu/kernels/moe_route_q4_0_block_f32.wgsl +74 -0
  13. package/src/gpu/kernels/moe_route_q4_0_split_block_f32.wgsl +35 -18
  14. package/src/inference/pipelines/text/partitions/partition-execution.js +28 -1
  15. package/src/inference/pipelines/text/partitions/resident-partition-warmup.js +67 -0
  16. package/src/inference/pipelines/text/partitions/resident-partition.js +8 -2
  17. package/src/loader/experts/compact-expert-cache.js +20 -0
  18. package/src/loader/experts/loader-bindings.js +14 -3
  19. package/src/loader/experts/mapped-expert-staging.js +393 -0
  20. package/src/loader/experts/routed-expert-cache.d.ts +8 -0
  21. package/src/loader/experts/routed-expert-cache.js +12 -11
  22. package/src/loader/experts/routed-expert-prefill.js +77 -1
  23. package/src/loader/experts/routed-expert-source.js +1 -0
  24. package/src/loader/experts/tiered-expert-cache.js +66 -26
  25. package/src/storage/verified-piece-direct.js +153 -0
  26. package/src/storage/verified-piece-gather-lane.js +21 -3
  27. package/src/storage/verified-piece-gather-worker.js +235 -16
  28. package/src/storage/verified-piece-gather.d.ts +47 -0
  29. package/src/storage/verified-piece-gather.js +133 -17
  30. package/src/storage/verified-piece-reader.d.ts +8 -1
  31. package/src/storage/verified-piece-reader.js +16 -3
  32. package/src/storage/verified-piece-state-cache.d.ts +75 -0
  33. package/src/storage/verified-piece-state-cache.js +511 -0
  34. package/src/storage/verified-piece-storage.d.ts +20 -1
  35. package/src/storage/verified-piece-storage.js +207 -9
  36. package/src/version.js +1 -1
@@ -0,0 +1,67 @@
1
+ import { log } from "../../../../debug/index.js";
2
+ import { releaseBuffer } from "../../../../memory/buffer-pool.js";
3
+ import { createPartitionAttempt } from './partition-attempt.js';
4
+ import { executePartitionLayers } from './partition-execution.js';
5
+ import { resolveLayerPartition, getPartitionActivationContract, validateActivationTensorShape } from "../execution/layer-partition-contract.js";
6
+
7
+ // runtime.shared.kernelWarmup.warmPartition: one prompt pass (and optionally a few single-token
8
+ // steps) through this stage's own layers while the stage opens. It compiles the pipelines the
9
+ // stage selects on first use and, for routed-expert stages, loads the experts those tokens
10
+ // select, so the first request finds them resident. It runs in a private attempt that is closed
11
+ // before the stage reports ready: no sequence state survives it. The time counts as opening time.
12
+ // A stage that starts from token ids runs the configured ids; a later stage has no real boundary
13
+ // activation at opening and runs zeros of the declared shape, which only compiles its pipelines.
14
+
15
+ const FIELD = 'runtime.shared.kernelWarmup.warmPartition';
16
+ const MAX_DECODE_STEPS = 8;
17
+
18
+ /** @type {import('./resident-partition-warmup.js').runResidentPartitionWarmup} */
19
+ export async function runResidentPartitionWarmup(owner, allocation, signal) {
20
+ const warm = owner.runtimeConfig?.shared?.kernelWarmup?.warmPartition;
21
+ if (warm == null) return null;
22
+ const { tokenIds, decodeSteps } = warm;
23
+ const vocabSize = allocation.plan.vocabSize;
24
+ if (!Array.isArray(tokenIds) || tokenIds.length < 1 || tokenIds.length > allocation.limits.maxPromptTokens
25
+ || tokenIds.some(id => !Number.isSafeInteger(id) || id < 0 || id >= vocabSize)) {
26
+ throw new Error(`${FIELD}.tokenIds must hold 1 to limits.maxPromptTokens token ids inside the vocabulary.`);
27
+ }
28
+ if (!Number.isSafeInteger(decodeSteps) || decodeSteps < 0 || decodeSteps > MAX_DECODE_STEPS) {
29
+ throw new Error(`${FIELD}.decodeSteps must be an integer from 0 to ${MAX_DECODE_STEPS}.`);
30
+ }
31
+ if (!owner.manifest || !owner.modelPartition) throw new Error('Partition warm-up requires a loaded, assigned model.');
32
+ const partition = resolveLayerPartition(owner.manifest, owner.modelPartition);
33
+ if (!partition) throw new Error('Partition warm-up allocation is missing.');
34
+ const plan = allocation.plan;
35
+ const hiddenSize = getPartitionActivationContract(plan).hiddenSize;
36
+ /** @param {number} count */
37
+ const zeros = count => new Uint8Array(validateActivationTensorShape({ shape: [1, count, hiddenSize],
38
+ dtype: plan.activationDtype, precision: plan.activationPrecision, components: plan.activationComponents,
39
+ elementStrideBytes: plan.activationElementStrideBytes }).expectedBytes);
40
+ const started = performance.now();
41
+ signal?.throwIfAborted();
42
+ const active = signal ?? new AbortController().signal;
43
+ const attempt = createPartitionAttempt(owner, tokenIds.length + decodeSteps);
44
+ const state = attempt.state;
45
+ /** @param {number[]} ids */
46
+ const step = async ids => {
47
+ const result = await executePartitionLayers(state, { numTokens: ids.length, tokenIds: ids,
48
+ ...(partition.hasEmbedding ? {} : { activationBytes: zeros(ids.length) }) }, active);
49
+ if ('logits' in result && result.logits) releaseBuffer(result.logits.logitsBuffer);
50
+ state.currentSeqLen += ids.length;
51
+ state.decodeStepCount++;
52
+ };
53
+ try {
54
+ await step(tokenIds);
55
+ // Single-token steps select their own pipelines; the repeated id is a placeholder, not an answer.
56
+ for (let index = 0; index < decodeSteps; index++) await step([tokenIds[tokenIds.length - 1]]);
57
+ } finally {
58
+ // The private attempt's caches may not be destroyed until its submitted work has settled.
59
+ await owner.gpuContext?.device?.queue.onSubmittedWorkDone();
60
+ await attempt.close();
61
+ }
62
+ const receipt = { promptTokens: tokenIds.length, decodeSteps, tokenInput: partition.hasEmbedding,
63
+ durationMs: performance.now() - started };
64
+ log.info('Pipeline', `Partition warm-up: ${receipt.promptTokens} prompt tokens, ${decodeSteps} single-token steps, `
65
+ + `${receipt.tokenInput ? 'token ids' : 'zero activations'} (${receipt.durationMs.toFixed(0)}ms)`);
66
+ return receipt;
67
+ }
@@ -15,6 +15,7 @@ import { createResidentOutputState } from "../../../resident-output-state.js";
15
15
  import { ORDERED_LAYER_PARTITION_SCHEMA, hashLayerPartitionPlan, getPartitionActivationContract, resolveActivationTensorLayout, validateActivationTensorShape } from "../execution/layer-partition-contract.js";
16
16
  import { markDeviceWeights, getDeviceMemorySnapshot } from "../../../../memory/device-budget.js";
17
17
  import { prepareResidentPartitionPipelines } from './resident-pipeline-preparation.js';
18
+ import { runResidentPartitionWarmup } from './resident-partition-warmup.js';
18
19
 
19
20
  /** @type {import('./resident-partition.js').createResidentPartitionSession} */
20
21
  export async function createResidentPartitionSession(pipeline, allocation, tokens, closeProgram, signal, onLoadProgress) {
@@ -36,6 +37,9 @@ export async function createResidentPartitionSession(pipeline, allocation, token
36
37
  // provider compile failure or crash happens at opening, not mid-answer.
37
38
  const pipelinePreparation = await prepareResidentPartitionPipelines(owner, allocation, signal, onLoadProgress);
38
39
  signal?.throwIfAborted();
40
+ // Optional and counted as opening time: a private prompt pass that leaves no sequence state.
41
+ await runResidentPartitionWarmup(owner, allocation, signal);
42
+ signal?.throwIfAborted();
39
43
  // Resident execution creates one cache per attempt. The opening pipeline's
40
44
  // empty cache is never used by those attempts and must not reserve a second
41
45
  // copy of the participant's entire context allocation.
@@ -137,6 +141,7 @@ export async function createResidentPartitionSession(pipeline, allocation, token
137
141
  }
138
142
  const combined = AbortSignal.any([signal, attempt.controller.signal]);
139
143
  const operation = runPipelineOperation(pipeline, async owner => {
144
+ let queueSettled = false;
140
145
  try {
141
146
  assertOpen(); combined.throwIfAborted();
142
147
  const ids = 'tokenIds' in request ? request.tokenIds : request.inputTokenIds;
@@ -178,6 +183,7 @@ export async function createResidentPartitionSession(pipeline, allocation, token
178
183
  if (ordered && partition.hasLmHead) throw new Error('Final ordered stage must produce logits.');
179
184
  if (canonicalMode) attempt.canonicalAwaiting = true;
180
185
  if (result.activationBytes.byteLength > allocation.limits.maxActivationBytes) throw new Error('Resident activation exceeds its byte allocation.');
186
+ queueSettled = true;
181
187
  return { activationTensor: { shape: [1, request.inputTokenCount, getPartitionActivationContract(allocation.plan).hiddenSize],
182
188
  dtype: allocation.plan.activationDtype,
183
189
  ...(allocation.plan.activationPrecision === 'two-f32' ? { precision: allocation.plan.activationPrecision,
@@ -188,7 +194,7 @@ export async function createResidentPartitionSession(pipeline, allocation, token
188
194
  if (!result.logits) throw new Error('Resident final stage did not produce logits.');
189
195
  let logits;
190
196
  const readbackStarted = performance.now();
191
- try { logits = new Float32Array(await readBuffer(result.logits.logitsBuffer, result.logits.vocabSize * Float32Array.BYTES_PER_ELEMENT)); }
197
+ try { logits = new Float32Array(await readBuffer(result.logits.logitsBuffer, result.logits.vocabSize * Float32Array.BYTES_PER_ELEMENT)); queueSettled = true; }
192
198
  finally { releaseBuffer(result.logits.logitsBuffer); }
193
199
  metrics.logitsReadbackMs = performance.now() - readbackStarted;
194
200
  combined.throwIfAborted();
@@ -236,7 +242,7 @@ export async function createResidentPartitionSession(pipeline, allocation, token
236
242
  } finally {
237
243
  // Attempt caches cannot be destroyed until submitted work has settled,
238
244
  // including cancellation or a failure after recording started.
239
- await owner.gpuContext?.device?.queue.onSubmittedWorkDone();
245
+ if (!queueSettled) await owner.gpuContext?.device?.queue.onSubmittedWorkDone();
240
246
  }
241
247
  }, combined);
242
248
  attempt.pending = operation;
@@ -2,6 +2,7 @@ import { DEFAULT_EXPERT_CACHE_CONFIG, validateQwen35RoutedConfig } from "../../c
2
2
  import { createWeightBuffer } from '../../gpu/weight-buffer.js';
3
3
  import { isDeviceLost } from '../../memory/device-residency.js';
4
4
  import { readRoutedExpertRange, readRoutedExpertPayload, readRoutedExpertsLease, readRoutedExpertPlaneLeases, recordRoutedExpertSourceTimings, assertExpertReadShape } from './routed-expert-source.js';
5
+ import { getMappedExpertStaging, stageRoutedExperts } from './mapped-expert-staging.js';
5
6
 
6
7
  /** @type {typeof import('./compact-expert-cache.js').compactExpertCapacity} */
7
8
  export function compactExpertCapacity(plan, options) {
@@ -117,6 +118,7 @@ export class CompactExpertCache {
117
118
  * @param {import('./compact-expert-cache.js').CompactExpertPool} pool
118
119
  * @param {{expert: number, slot: number}} miss @param {AbortSignal} signal */
119
120
  async #fillSinglePlanes(plan, pool, { expert, slot }, signal) {
121
+ if (await this.#stage(plan, pool, [{ expert, slot }], signal)) return;
120
122
  const started = performance.now();
121
123
  const leases = await readRoutedExpertPlaneLeases(this.#options, this.#stats, plan, expert, signal, pool.stats);
122
124
  let failure;
@@ -250,10 +252,28 @@ export class CompactExpertCache {
250
252
  }
251
253
  }
252
254
 
255
+ /** Mapped staging for experts the storage port serves from verified state; false means use the lease path.
256
+ * @param {import('./routed-expert-plan.js').RoutedExpertBankPlan} plan
257
+ * @param {import('./compact-expert-cache.js').CompactExpertPool} pool
258
+ * @param {Array<{expert: number, slot: number}>} misses @param {AbortSignal} signal */
259
+ async #stage(plan, pool, misses, signal) {
260
+ // A host cache both supplies and remembers expert bytes; staged bytes never reach JavaScript memory.
261
+ if (this.#options.copyCachedExpert || this.#options.rememberExpert) return false;
262
+ const staging = getMappedExpertStaging(this.#options, plan);
263
+ if (!staging) return false;
264
+ const slots = new Map(misses.map(({ expert, slot }) => [expert, slot]));
265
+ return stageRoutedExperts(staging, this.#stats, plan, misses.map(row => row.expert), (expert, plane) => {
266
+ const buffer = pool.planes.get(plane.role), slot = slots.get(expert);
267
+ if (!buffer || slot === undefined) throw new Error('Compact expert plane is missing.');
268
+ return { buffer, offset: slot * plane.expertBytes };
269
+ }, signal, null, pool.stats);
270
+ }
271
+
253
272
  /** @param {import('./routed-expert-plan.js').RoutedExpertBankPlan} plan
254
273
  * @param {import('./compact-expert-cache.js').CompactExpertPool} pool
255
274
  * @param {Array<{expert: number, slot: number}>} misses @param {AbortSignal} signal */
256
275
  async #fillBatch(plan, pool, misses, signal) {
276
+ if (await this.#stage(plan, pool, misses, signal)) return;
257
277
  const pending = misses.filter(({ expert, slot }) => !this.#copyCached(plan, pool, expert, slot, signal));
258
278
  if (!pending.length) return;
259
279
  const started = performance.now();
@@ -6,6 +6,8 @@ import { TieredExpertCache } from './tiered-expert-cache.js';
6
6
  import { RoutedExpertCache } from './routed-expert-cache.js';
7
7
  import { planRoutedExpertBank } from './routed-expert-plan.js';
8
8
  import { prepareQwenExpertLayer } from './qwen-paging.js';
9
+ import { trace, isTraceEnabled } from '../../debug/index.js';
10
+ import { assertStagedVerification } from './mapped-expert-staging.js';
9
11
 
10
12
  /** @type {typeof import('./loader-bindings.js').prepareExpertLayer} */
11
13
  export async function prepareExpertLayer(loader, policy, layer, consumers, recorder, signal, activeSelection) {
@@ -112,12 +114,21 @@ export async function prepareRoutedExperts(loader, layer, expertIds, signal, pha
112
114
  if (!loader.manifest) throw new Error('Routed experts require the loaded manifest.');
113
115
  const plan = planRoutedExpertBank(loader.manifest, loader.tensorLocations, layer);
114
116
  const cache = ensureRoutedCache(loader, plan);
117
+ // After a failed deferred verification, unverified experts are resident in the pools. No caller gets them,
118
+ // whether or not it opens a release gate, and whether it would upload or only hit the cache.
119
+ if (loader.shardCache.rangeLease) assertStagedVerification(loader.shardCache.rangeLease);
120
+ // With the loader trace on, one line when a layer asks for its experts and one when they are ready for
121
+ // its kernels. Together with the staged-upload lines these put read, hash, upload and use on one clock.
122
+ const traced = isTraceEnabled('loader');
123
+ if (traced) trace.loader(`routed-experts request ${JSON.stringify({ layer, phase, experts: expertIds.length, at: Number(performance.now().toFixed(2)) })}`);
124
+ let weights;
115
125
  if (cache instanceof TieredExpertCache) {
116
126
  const admitted = tieredPlans.get(cache)?.get(layer);
117
127
  if (!admitted) throw new Error('Tiered expert request lacks the admitted original layer plan.');
118
- return cache.prepare(admitted, expertIds, signal, phase);
119
- }
120
- return cache.prepare(plan, expertIds, signal, phase);
128
+ weights = await cache.prepare(admitted, expertIds, signal, phase);
129
+ } else weights = await cache.prepare(plan, expertIds, signal, phase);
130
+ if (traced) trace.loader(`routed-experts ready ${JSON.stringify({ layer, phase, at: Number(performance.now().toFixed(2)) })}`);
131
+ return weights;
121
132
  }
122
133
 
123
134
  /** @type {typeof import('./loader-bindings.js').disposeRoutedExperts} */
@@ -0,0 +1,393 @@
1
+ import { recordRoutedExpertSourceTimings } from './routed-expert-source.js';
2
+ import { trace, isTraceEnabled } from '../../debug/index.js';
3
+
4
+ /*
5
+ * Staged expert uploads for policy.uploadStaging === 'mapped'.
6
+ *
7
+ * queue.writeBuffer copies every expert on the JavaScript thread, which made that thread the limit
8
+ * of expert loading. Here a small ring of mapped upload buffers receives the bytes instead: the
9
+ * storage port reads already verified ranges on the I/O pool straight into a mapped slot, the slot
10
+ * is unmapped, and one submitted command buffer copies each plane to its place. The submit happens
11
+ * where the writeBuffer calls happened, so queue order relative to the caller's kernels is the same.
12
+ *
13
+ * A range whose pieces are not verified yet (first-time verification) is read into the slot at the
14
+ * same time as a lane hashes those pieces. The slot is unmapped and its copy submitted only after the
15
+ * lane has finished and the port confirms that every piece is verified for the file state the read
16
+ * saw. Until then the bytes exist only in the mapped slot, which no model buffer and no kernel can
17
+ * see. If the confirmation fails, or the port cannot read ahead, nothing is submitted and the caller
18
+ * uses its lease path. The lane's own copy of the bytes is not used: copying it into the slot would
19
+ * put the copy back on the JavaScript thread.
20
+ *
21
+ * policy.uploadVerification === 'before-release' moves that wait. Inside a release gate (a resident
22
+ * partition step opens one and settles it before its output leaves the stage) first-time bytes are
23
+ * submitted as soon as they are read, the kernels compute with them, and the lanes hash in the
24
+ * background. Settling the gate waits for every hash and confirmation. If one fails, the step fails,
25
+ * its output is never released, and the gate stays failed: no later step may begin until the stage is
26
+ * reopened, because unverified bytes are resident in model buffers. Outside a gate nothing is deferred.
27
+ */
28
+
29
+ const MAX_SLOTS = 16;
30
+ /** @type {WeakMap<object, MappedExpertStaging | null>} */
31
+ const stagings = new WeakMap();
32
+ /** One release gate per storage port.
33
+ * @type {WeakMap<object, { depth: number, tasks: Set<Promise<void>>, failure: Error | null, stats: import('./routed-expert-source.js').RoutedExpertCounters | null }>} */
34
+ const gates = new WeakMap();
35
+
36
+ /** @type {import('./mapped-expert-staging.js').beginStagedRelease} */
37
+ export function beginStagedRelease(port) {
38
+ let gate = gates.get(port);
39
+ if (!gate) { gate = { depth: 0, tasks: new Set(), failure: null, stats: null }; gates.set(port, gate); }
40
+ if (gate.failure) throw gate.failure;
41
+ gate.depth++;
42
+ }
43
+
44
+ /** @type {import('./mapped-expert-staging.js').assertStagedVerification} */
45
+ export function assertStagedVerification(port) {
46
+ const failure = gates.get(port)?.failure;
47
+ if (failure) throw failure;
48
+ }
49
+
50
+ /** @type {import('./mapped-expert-staging.js').settleStagedRelease} */
51
+ export async function settleStagedRelease(port, { release = true } = {}) {
52
+ const gate = gates.get(port);
53
+ if (!gate || gate.depth < 1) throw new Error('Staged release gate is not open.');
54
+ const started = performance.now();
55
+ try {
56
+ // A task may start another only through a new upload, which the caller has stopped making.
57
+ while (gate.tasks.size) await Promise.allSettled([...gate.tasks]);
58
+ } finally { gate.depth--; }
59
+ if (gate.stats) gate.stats.stagedReleaseWaitMs += performance.now() - started;
60
+ if (release && gate.failure) throw gate.failure;
61
+ }
62
+
63
+ class MappedExpertStaging {
64
+ /** @type {GPUDevice} */ #device;
65
+ /** @type {NonNullable<import('../../storage/verified-piece-gather.js').VerifiedRangeLeasePort['readVerifiedInto']>} */ #read;
66
+ /** @type {((ranges: Array<{ shard: number, offset: number, length: number }>, signal: AbortSignal) => Promise<import('../../storage/verified-piece-gather.js').VerifiedRangeLease>) | null} */ #verify;
67
+ /** @type {import('../../storage/verified-piece-gather.js').VerifiedRangeLeasePort['readAheadInto'] | null} */ #readAhead;
68
+ /** @type {import('../../storage/verified-piece-gather.js').VerifiedRangeLeasePort['hasVerified'] | null} */ #hasVerified;
69
+ /** @type {object} */ #port;
70
+ /** @type {import('../../storage/verified-piece-gather.js').VerifiedRangeLeasePort['verifyMany'] | null} */ #verifyQueued;
71
+ /** @type {import('../../storage/verified-piece-gather.js').VerifiedRangeLeasePort['verifyMany'] | null} */ #verifyWaiting;
72
+ #lanes = 1; #active = 0;
73
+ #maxRanges; #maxPartBytes;
74
+ /** @type {Array<{ buffer: GPUBuffer, view: Uint8Array | null }>} */ #slots = [];
75
+ /** @type {Array<{ buffer: GPUBuffer, view: Uint8Array | null }>} */ #free = [];
76
+ /** @type {Array<(slot: { buffer: GPUBuffer, view: Uint8Array | null } | null) => void>} */ #waiters = [];
77
+ /** @type {Set<Promise<void>>} */ #remapping = new Set();
78
+ #closed = false;
79
+
80
+ /** @param {{ device: GPUDevice, port: import('../../storage/verified-piece-gather.js').VerifiedRangeLeasePort, slotBytes: number, slotCount: number, deferVerification: boolean }} options */
81
+ constructor({ device, port, slotBytes, slotCount, deferVerification }) {
82
+ if (typeof port.readVerifiedInto !== 'function') throw new Error('Mapped expert staging requires direct verified reads.');
83
+ this.#device = device; this.#read = port.readVerifiedInto.bind(port);
84
+ this.#verify = typeof port.readMany === 'function' ? port.readMany.bind(port) : null;
85
+ const ahead = typeof port.readAheadInto === 'function' && typeof port.hasVerified === 'function';
86
+ this.#readAhead = ahead ? /** @type {NonNullable<typeof port.readAheadInto>} */ (port.readAheadInto).bind(port) : null;
87
+ this.#hasVerified = ahead ? /** @type {NonNullable<typeof port.hasVerified>} */ (port.hasVerified).bind(port) : null;
88
+ this.#port = port;
89
+ this.#verifyWaiting = typeof port.verifyMany === 'function' ? port.verifyMany.bind(port) : null;
90
+ this.#verifyQueued = deferVerification ? this.#verifyWaiting : null;
91
+ this.#lanes = Number.isSafeInteger(port.concurrency) && Number(port.concurrency) > 0 ? Number(port.concurrency) : 1;
92
+ this.#maxRanges = port.maxRanges ?? 3; this.#maxPartBytes = port.maxPartBytes;
93
+ this.slotBytes = slotBytes; this.slotCount = slotCount;
94
+ }
95
+
96
+ get bytes() { return this.#slots.length * this.slotBytes; }
97
+
98
+ /** @returns {Promise<{ buffer: GPUBuffer, view: Uint8Array | null } | null>} */
99
+ #take() {
100
+ if (this.#closed) return Promise.resolve(null);
101
+ const ready = this.#free.pop();
102
+ if (ready) return Promise.resolve(ready);
103
+ if (this.#slots.length < this.slotCount) {
104
+ const buffer = this.#device.createBuffer({ label: 'expert_staged_upload', size: this.slotBytes,
105
+ usage: GPUBufferUsage.MAP_WRITE | GPUBufferUsage.COPY_SRC, mappedAtCreation: true });
106
+ const slot = { buffer, view: new Uint8Array(buffer.getMappedRange(0, this.slotBytes)) };
107
+ this.#slots.push(slot);
108
+ return Promise.resolve(slot);
109
+ }
110
+ return new Promise(resolve => this.#waiters.push(resolve));
111
+ }
112
+
113
+ /** @param {{ buffer: GPUBuffer, view: Uint8Array | null }} slot */
114
+ #give(slot) {
115
+ const waiter = this.#waiters.shift();
116
+ if (waiter) waiter(slot); else this.#free.push(slot);
117
+ }
118
+
119
+ /** The GPU may still be copying from the slot; it is free again when the map resolves.
120
+ * @param {{ buffer: GPUBuffer, view: Uint8Array | null }} slot */
121
+ #remap(slot) {
122
+ const operation = slot.buffer.mapAsync(GPUMapMode.WRITE).then(() => {
123
+ if (this.#closed) return;
124
+ slot.view = new Uint8Array(slot.buffer.getMappedRange(0, this.slotBytes));
125
+ this.#give(slot);
126
+ }, () => {
127
+ // A slot that cannot be mapped again is retired; waiting callers fall back to their lease path.
128
+ this.#slots = this.#slots.filter(candidate => candidate !== slot);
129
+ slot.buffer.destroy();
130
+ if (this.#slots.length === 0) for (const waiter of this.#waiters.splice(0)) waiter(null);
131
+ });
132
+ this.#remapping.add(operation);
133
+ operation.finally(() => this.#remapping.delete(operation));
134
+ }
135
+
136
+ /**
137
+ * @param {ReadonlyArray<{ shard: number, offset: number, length: number, at: number }>} parts
138
+ * @param {ReadonlyArray<{ at: number, length: number, target: GPUBuffer, targetOffset: number }>} copies
139
+ * @param {AbortSignal} signal @param {(() => void) | null} beforeSubmit
140
+ * @param {import('./routed-expert-source.js').RoutedExpertCounters | null} [stats] receives the timings of background verification
141
+ * @param {number | null} [layer] names the upload in the loader trace
142
+ * @returns {Promise<false | import('./mapped-expert-staging.js').StagedUpload>} false when nothing was staged and nothing was submitted
143
+ */
144
+ async upload(parts, copies, signal, beforeSubmit, stats = null, layer = null) {
145
+ const gate = gates.get(this.#port) ?? null;
146
+ if (gate?.failure) throw gate.failure;
147
+ const slot = await this.#take();
148
+ if (!slot) return false;
149
+ let unmapped = false;
150
+ this.#active++;
151
+ try {
152
+ const view = slot.view;
153
+ if (!view) throw new Error('Staged upload slot is not mapped.');
154
+ const started = performance.now();
155
+ const hasVerified = this.#hasVerified, readAhead = this.#readAhead;
156
+ const pending = hasVerified ? parts.filter(part => !hasVerified(part.shard, part.offset, part.length)) : [];
157
+ if (pending.length && (!this.#verify || !readAhead || pending.some(part => part.length > this.#maxPartBytes))) return false;
158
+ const ahead = new Set(pending);
159
+ const verifyQueued = this.#verifyQueued;
160
+ if (pending.length && verifyQueued && gate && gate.depth > 0) {
161
+ // Verification before release: submit once the bytes are read, and let the gate hold the output.
162
+ const read = await Promise.allSettled(parts.map(part => {
163
+ const target = view.subarray(part.at, part.at + part.length);
164
+ return ahead.has(part) && readAhead ? readAhead(part.shard, part.offset, target, signal)
165
+ : this.#read(part.shard, part.offset, target, signal);
166
+ }));
167
+ const broken = read.find(outcome => outcome.status === 'rejected');
168
+ if (broken) throw /** @type {PromiseRejectedResult} */ (broken).reason;
169
+ const values = read.map(outcome => /** @type {PromiseFulfilledResult<boolean | (() => boolean)>} */ (outcome).value);
170
+ // Only an already verified range (true) or a read with a confirmation to run later may be submitted.
171
+ if (values.some(value => value !== true && typeof value !== 'function')) return false;
172
+ const readWait = performance.now() - started;
173
+ signal.throwIfAborted();
174
+ // The gate must still be open: an upload that outlived its step has nothing left to hold its bytes back.
175
+ if (this.#closed || gate.failure || gate.depth < 1) return false;
176
+ const confirms = values.filter(value => typeof value === 'function');
177
+ const groups = this.#groups(pending);
178
+ const bytes = pending.reduce((sum, part) => sum + part.length, 0);
179
+ slot.view = null; slot.buffer.unmap(); unmapped = true;
180
+ beforeSubmit?.();
181
+ const encoder = this.#device.createCommandEncoder({ label: 'expert_staged_upload' });
182
+ for (const copy of copies) encoder.copyBufferToBuffer(slot.buffer, copy.at, copy.target, copy.targetOffset, copy.length);
183
+ this.#device.queue.submit([encoder.finish()]);
184
+ // From here unverified bytes are in model buffers. The verification task is registered before anything
185
+ // else can throw. Every lane job settles before the outcome is decided, and only a failed hash or a
186
+ // failed confirmation fails the gate; counters and trace lines cannot.
187
+ const task = (async () => {
188
+ const outcomes = await Promise.allSettled(groups.map(group => verifyQueued(group)));
189
+ const failed = outcomes.find(outcome => outcome.status === 'rejected');
190
+ let failure = failed ? /** @type {PromiseRejectedResult} */ (failed).reason : null;
191
+ if (!failure) {
192
+ try {
193
+ for (const confirm of confirms) {
194
+ if (!(/** @type {() => boolean} */ (confirm))()) throw new Error('Expert bytes already in use could not be confirmed verified.');
195
+ }
196
+ } catch (error) { failure = error; }
197
+ }
198
+ if (failure) {
199
+ gate.failure ??= new Error(`Staged expert verification failed after upload: ${failure?.message ?? failure}`, { cause: failure });
200
+ return;
201
+ }
202
+ try {
203
+ const timings = outcomes.map(outcome => /** @type {PromiseFulfilledResult<import('./mapped-expert-staging.js').StagedUpload['laneTimings'][number]>} */ (outcome).value ?? null);
204
+ if (stats) recordRoutedExpertSourceTimings(stats, timings.map(timing => timing ? { timings: timing } : null));
205
+ if (isTraceEnabled('loader')) traceUpload({ layer, bytes, verifiedAfterSubmit: true, startedAt: started, verifyDoneAt: performance.now(), lanes: timings });
206
+ } catch { /* Observation only: the bytes are verified. */ }
207
+ })();
208
+ gate.tasks.add(task);
209
+ task.finally(() => gate.tasks.delete(task));
210
+ if (stats) gate.stats = stats;
211
+ if (isTraceEnabled('loader')) traceUpload({ layer, bytes, firstTime: bytes, deferred: true, startedAt: started, readDoneAt: started + readWait, submittedAt: performance.now() });
212
+ return { laneTimings: [], readAheadBytes: bytes, readWaitMs: readWait, verifyWaitMs: 0, deferred: true };
213
+ }
214
+ // The reads and the lanes run together. Every read settles before the slot is unmapped or handed
215
+ // on, including after a failure, because a native read may still be writing into the slot.
216
+ let readWaitMs = 0, verifyWaitMs = 0;
217
+ const reads = Promise.allSettled(parts.map(part => {
218
+ const target = view.subarray(part.at, part.at + part.length);
219
+ return ahead.has(part) && readAhead ? readAhead(part.shard, part.offset, target, signal)
220
+ : this.#read(part.shard, part.offset, target, signal);
221
+ })).then(outcomes => { readWaitMs = performance.now() - started; return outcomes; });
222
+ /** @type {Promise<{ timings: import('./mapped-expert-staging.js').StagedUpload['laneTimings'], error?: unknown }>} */
223
+ // Lanes that no other upload is using hash this upload's ranges side by side. During decode a layer
224
+ // asks for one or two experts at a time, so most lanes would otherwise sit idle behind one job.
225
+ const verification = pending.length
226
+ ? this.#verifyRanges(pending, signal, Math.floor(this.#lanes / this.#active)).then(timings => { verifyWaitMs = performance.now() - started; return { timings }; },
227
+ error => ({ timings: [], error }))
228
+ : Promise.resolve({ timings: [] });
229
+ const [outcomes, verified] = await Promise.all([reads, verification]);
230
+ const failed = outcomes.find(outcome => outcome.status === 'rejected');
231
+ if (failed) throw /** @type {PromiseRejectedResult} */ (failed).reason;
232
+ if ('error' in verified) throw verified.error;
233
+ // A range read ahead counts only once the port confirms its pieces are now verified.
234
+ for (const outcome of outcomes) {
235
+ const value = /** @type {PromiseFulfilledResult<boolean | (() => boolean)>} */ (outcome).value;
236
+ if (value !== true && (typeof value !== 'function' || !value())) return false;
237
+ }
238
+ signal.throwIfAborted();
239
+ if (this.#closed) return false;
240
+ slot.view = null; slot.buffer.unmap(); unmapped = true;
241
+ beforeSubmit?.();
242
+ const encoder = this.#device.createCommandEncoder({ label: 'expert_staged_upload' });
243
+ for (const copy of copies) encoder.copyBufferToBuffer(slot.buffer, copy.at, copy.target, copy.targetOffset, copy.length);
244
+ this.#device.queue.submit([encoder.finish()]);
245
+ const firstTime = pending.reduce((sum, part) => sum + part.length, 0);
246
+ if (isTraceEnabled('loader')) {
247
+ traceUpload({ layer, bytes: parts.reduce((sum, part) => sum + part.length, 0), firstTime, deferred: false, startedAt: started,
248
+ readDoneAt: started + readWaitMs, ...(firstTime ? { verifyDoneAt: started + verifyWaitMs, lanes: verified.timings } : {}), submittedAt: performance.now() });
249
+ }
250
+ return { laneTimings: verified.timings, readAheadBytes: firstTime, readWaitMs, verifyWaitMs, deferred: false };
251
+ } finally {
252
+ this.#active--;
253
+ if (unmapped) this.#remap(slot);
254
+ else this.#give(slot);
255
+ }
256
+ }
257
+
258
+ /** Have lanes authenticate these ranges; their copies of the bytes are not used. Resolves each job's timings.
259
+ * With `spread` above one and a port that queues verification, the ranges go to up to that many lanes at once.
260
+ * @param {ReadonlyArray<{ shard: number, offset: number, length: number }>} parts @param {AbortSignal} signal
261
+ * @param {number} [spread]
262
+ * @returns {Promise<import('./mapped-expert-staging.js').StagedUpload['laneTimings']>} */
263
+ async #verifyRanges(parts, signal, spread = 1) {
264
+ if (!this.#verify) throw new Error('Staged upload has no lane to verify with.');
265
+ const waiting = this.#verifyWaiting;
266
+ if (waiting && spread > 1 && parts.length > 1) {
267
+ const jobs = Math.min(spread, parts.length), size = Math.min(this.#maxRanges, Math.ceil(parts.length / jobs));
268
+ /** @type {Array<Array<{ shard: number, offset: number, length: number }>>} */ const chunks = [];
269
+ for (let at = 0; at < parts.length; at += size) chunks.push(parts.slice(at, at + size).map(({ shard, offset, length }) => ({ shard, offset, length })));
270
+ // Every job settles before a failure is reported, so no lane is still reading when the caller moves on.
271
+ const outcomes = await Promise.allSettled(chunks.flatMap(chunk => this.#groups(chunk)).map(group => waiting(group, signal)));
272
+ const failed = outcomes.find(outcome => outcome.status === 'rejected');
273
+ if (failed) throw /** @type {PromiseRejectedResult} */ (failed).reason;
274
+ return outcomes.map(outcome => /** @type {PromiseFulfilledResult<import('./mapped-expert-staging.js').StagedUpload['laneTimings'][number]>} */ (outcome).value ?? null);
275
+ }
276
+ /** @type {import('./mapped-expert-staging.js').StagedUpload['laneTimings']} */ const timings = [];
277
+ for (const group of this.#groups(parts)) {
278
+ signal.throwIfAborted();
279
+ const lease = await this.#verify(group, signal);
280
+ timings.push(lease.timings ?? null);
281
+ lease.release();
282
+ }
283
+ return timings;
284
+ }
285
+
286
+ /** Split ranges into lane jobs within the port's range and byte bounds.
287
+ * @param {ReadonlyArray<{ shard: number, offset: number, length: number }>} parts */
288
+ #groups(parts) {
289
+ /** @type {Array<Array<{ shard: number, offset: number, length: number }>>} */ const groups = [];
290
+ let bytes = 0;
291
+ for (const { shard, offset, length } of parts) {
292
+ const group = groups.at(-1);
293
+ if (!group || group.length >= this.#maxRanges || bytes + length > this.#maxPartBytes) { groups.push([{ shard, offset, length }]); bytes = length; }
294
+ else { group.push({ shard, offset, length }); bytes += length; }
295
+ }
296
+ return groups;
297
+ }
298
+
299
+ async destroy() {
300
+ this.#closed = true;
301
+ const gate = gates.get(this.#port);
302
+ if (gate) await Promise.allSettled([...gate.tasks]);
303
+ for (const waiter of this.#waiters.splice(0)) waiter(null);
304
+ await Promise.allSettled([...this.#remapping]);
305
+ await this.#device.queue.onSubmittedWorkDone();
306
+ for (const slot of this.#slots.splice(0)) slot.buffer.destroy();
307
+ this.#free.length = 0;
308
+ }
309
+ }
310
+
311
+ /**
312
+ * One loader-trace line per staged upload, in milliseconds on the process clock: when it started, when its
313
+ * reads into the slot had settled, when its lanes had hashed any first-time pieces (with each lane job's own
314
+ * read, hash and wall time), and when its copy was submitted.
315
+ * @param {{ layer: number | null, bytes: number, firstTime?: number, deferred?: boolean, verifiedAfterSubmit?: boolean, startedAt: number,
316
+ * readDoneAt?: number, verifyDoneAt?: number, submittedAt?: number, lanes?: ReadonlyArray<unknown> }} event */
317
+ function traceUpload(event) {
318
+ const round = /** @param {number | undefined} value */ value => value === undefined ? undefined : Number(value.toFixed(2));
319
+ const lanes = (event.lanes ?? []).filter(Boolean).map(job => {
320
+ const timing = /** @type {{ workerWallMs: number, readWaitMs: number, verificationMs: number, roundTripMs: number }} */ (job);
321
+ return [round(timing.workerWallMs), round(timing.readWaitMs), round(timing.verificationMs), round(timing.roundTripMs)];
322
+ });
323
+ trace.loader(`staged-upload ${JSON.stringify({ layer: event.layer, bytes: event.bytes, firstTime: event.firstTime, deferred: event.deferred,
324
+ verifiedAfterSubmit: event.verifiedAfterSubmit, start: round(event.startedAt), readDone: round(event.readDoneAt),
325
+ verifyDone: round(event.verifyDoneAt), submitted: round(event.submittedAt), ...(lanes.length ? { laneWallReadHashRoundTrip: lanes } : {}) })}`);
326
+ }
327
+
328
+ /** One ring per storage port: prefill and decode share it.
329
+ * @type {import('./mapped-expert-staging.js').getMappedExpertStaging} */
330
+ export function getMappedExpertStaging(options, plan) {
331
+ const port = options.rangeLease;
332
+ if (options.policy.uploadStaging !== 'mapped' || options.policy.sourceReadMode !== 'expert-lease-batch'
333
+ || !port || typeof port.readVerifiedInto !== 'function') return null;
334
+ let staging = stagings.get(port);
335
+ if (staging === undefined) {
336
+ const slotBytes = Math.ceil(plan.expertBytes / 4) * 4;
337
+ const slotCount = Math.min(MAX_SLOTS, Math.floor(options.policy.stagingBytes / slotBytes));
338
+ staging = slotCount >= 1 ? new MappedExpertStaging({ device: options.device, port, slotBytes, slotCount,
339
+ deferVerification: options.policy.uploadVerification === 'before-release' }) : null;
340
+ stagings.set(port, staging);
341
+ }
342
+ return staging;
343
+ }
344
+
345
+ /** @type {import('./mapped-expert-staging.js').destroyMappedExpertStaging} */
346
+ export async function destroyMappedExpertStaging(options) {
347
+ const port = options.rangeLease;
348
+ const staging = port ? stagings.get(port) : null;
349
+ if (!port || !staging) return;
350
+ stagings.delete(port);
351
+ await staging.destroy();
352
+ }
353
+
354
+ /**
355
+ * Stage whole experts (every plane) and copy each plane to the buffer `targetOf` names.
356
+ * @type {import('./mapped-expert-staging.js').stageRoutedExperts}
357
+ */
358
+ export async function stageRoutedExperts(staging, stats, plan, experts, targetOf, signal, beforeSubmit, layerStats) {
359
+ if (!experts.length || experts.length * plan.expertBytes > staging.slotBytes
360
+ || plan.planes.some(plane => plane.location.sourceTransform != null
361
+ || (plane.location.spans && plane.location.spans.length !== 1) || plane.expertBytes % 4)) return false;
362
+ /** @type {Array<{ shard: number, offset: number, length: number, at: number }>} */ const parts = [];
363
+ /** @type {Array<{ at: number, length: number, target: GPUBuffer, targetOffset: number }>} */ const copies = [];
364
+ let at = 0;
365
+ for (const expert of experts) {
366
+ for (const plane of plan.planes) {
367
+ const physical = plane.location.spans?.[0] ?? plane.location;
368
+ const offset = Number(physical.offset) + expert * plane.expertBytes;
369
+ if (!Number.isSafeInteger(physical.shardIndex) || Number(physical.shardIndex) < 0 || !Number.isSafeInteger(offset)
370
+ || offset % 4 || offset + plane.expertBytes > Number(physical.offset) + Number(physical.size)) return false;
371
+ const { buffer, offset: targetOffset } = targetOf(expert, plane);
372
+ parts.push({ shard: Number(physical.shardIndex), offset, length: plane.expertBytes, at });
373
+ copies.push({ at, length: plane.expertBytes, target: buffer, targetOffset });
374
+ at += plane.expertBytes;
375
+ }
376
+ }
377
+ const started = performance.now();
378
+ const staged = await staging.upload(parts, copies, signal, beforeSubmit, stats, plan.layer);
379
+ if (!staged) return false;
380
+ const elapsed = performance.now() - started;
381
+ // Lanes ran only for ranges verified for the first time. With none, the job is counted as untimed. A
382
+ // deferred upload's lane jobs are recorded when they finish.
383
+ if (staged.deferred) stats.stagedDeferredBytes += staged.readAheadBytes;
384
+ else recordRoutedExpertSourceTimings(stats, staged.laneTimings.length ? staged.laneTimings.map(timings => timings ? { timings } : null) : [null]);
385
+ stats.stagedReadAheadBytes += staged.readAheadBytes; stats.stagedReadWaitMs += staged.readWaitMs; stats.stagedVerifyWaitMs += staged.verifyWaitMs;
386
+ stats.sourceReads += parts.length; stats.sourceReadBytes += at; stats.sourceReadIntoBytes += at;
387
+ stats.uploadBytes += at; stats.sourceReadWaitMs += elapsed; stats.ioWaitMs += elapsed;
388
+ stats.peakStagingBytes = Math.max(stats.peakStagingBytes, staging.bytes);
389
+ if (layerStats) {
390
+ layerStats.sourceReadBytes += at; layerStats.uploadBytes += at; layerStats.sourceReadWaitMs += elapsed;
391
+ }
392
+ return true;
393
+ }
@@ -68,6 +68,14 @@ export interface RoutedExpertCacheStats {
68
68
  sourceJobVerificationMs: number;
69
69
  sourceJobCopyMs: number;
70
70
  sourceJobRoundTripMs: number;
71
+ /** Mapped staging, first-time verification: bytes read into a slot while a lane hashed their pieces. */
72
+ stagedReadAheadBytes: number;
73
+ /** Mapped staging: summed per-upload wall time until the slot's reads settled, and until its lanes finished. */
74
+ stagedReadWaitMs: number;
75
+ stagedVerifyWaitMs: number;
76
+ /** Bytes submitted before their verification finished, and the time release gates then waited for it. */
77
+ stagedDeferredBytes: number;
78
+ stagedReleaseWaitMs: number;
71
79
  uploadBytes: number;
72
80
  cacheHits: number;
73
81
  cacheMisses: number;