doppler-gpu 0.7.7 → 0.7.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/README.md +9 -17
- package/package.json +1 -1
- package/src/config/kernels/kernel-ref-digests.js +2 -1
- package/src/config/kernels/registry.json +47 -19
- package/src/config/schema/execution/kernel-warmup.schema.d.ts +14 -0
- package/src/config/schema/execution/kernel-warmup.schema.js +3 -0
- package/src/config/schema/platform/gpu-cache.schema.js +1 -1
- package/src/config/schema/runtime/buffer-pool.schema.js +2 -2
- package/src/config/schema/runtime/loading.schema.d.ts +5 -0
- package/src/config/schema/runtime/loading.schema.js +25 -2
- package/src/gpu/kernels/moe_route_q4_0_block_f32.wgsl +74 -0
- package/src/gpu/kernels/moe_route_q4_0_split_block_f32.wgsl +35 -18
- package/src/inference/pipelines/text/partitions/partition-execution.js +28 -1
- package/src/inference/pipelines/text/partitions/resident-partition-warmup.js +67 -0
- package/src/inference/pipelines/text/partitions/resident-partition.js +8 -2
- package/src/loader/experts/compact-expert-cache.js +20 -0
- package/src/loader/experts/loader-bindings.js +14 -3
- package/src/loader/experts/mapped-expert-staging.js +393 -0
- package/src/loader/experts/routed-expert-cache.d.ts +8 -0
- package/src/loader/experts/routed-expert-cache.js +12 -11
- package/src/loader/experts/routed-expert-prefill.js +77 -1
- package/src/loader/experts/routed-expert-source.js +1 -0
- package/src/loader/experts/tiered-expert-cache.js +66 -26
- package/src/storage/verified-piece-direct.js +153 -0
- package/src/storage/verified-piece-gather-lane.js +21 -3
- package/src/storage/verified-piece-gather-worker.js +235 -16
- package/src/storage/verified-piece-gather.d.ts +47 -0
- package/src/storage/verified-piece-gather.js +133 -17
- package/src/storage/verified-piece-reader.d.ts +8 -1
- package/src/storage/verified-piece-reader.js +16 -3
- package/src/storage/verified-piece-state-cache.d.ts +75 -0
- package/src/storage/verified-piece-state-cache.js +511 -0
- package/src/storage/verified-piece-storage.d.ts +20 -1
- package/src/storage/verified-piece-storage.js +207 -9
- package/src/version.js +1 -1
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
import { log } from "../../../../debug/index.js";
|
|
2
|
+
import { releaseBuffer } from "../../../../memory/buffer-pool.js";
|
|
3
|
+
import { createPartitionAttempt } from './partition-attempt.js';
|
|
4
|
+
import { executePartitionLayers } from './partition-execution.js';
|
|
5
|
+
import { resolveLayerPartition, getPartitionActivationContract, validateActivationTensorShape } from "../execution/layer-partition-contract.js";
|
|
6
|
+
|
|
7
|
+
// runtime.shared.kernelWarmup.warmPartition: one prompt pass (and optionally a few single-token
|
|
8
|
+
// steps) through this stage's own layers while the stage opens. It compiles the pipelines the
|
|
9
|
+
// stage selects on first use and, for routed-expert stages, loads the experts those tokens
|
|
10
|
+
// select, so the first request finds them resident. It runs in a private attempt that is closed
|
|
11
|
+
// before the stage reports ready: no sequence state survives it. The time counts as opening time.
|
|
12
|
+
// A stage that starts from token ids runs the configured ids; a later stage has no real boundary
|
|
13
|
+
// activation at opening and runs zeros of the declared shape, which only compiles its pipelines.
|
|
14
|
+
|
|
15
|
+
const FIELD = 'runtime.shared.kernelWarmup.warmPartition';
|
|
16
|
+
const MAX_DECODE_STEPS = 8;
|
|
17
|
+
|
|
18
|
+
/** @type {import('./resident-partition-warmup.js').runResidentPartitionWarmup} */
|
|
19
|
+
export async function runResidentPartitionWarmup(owner, allocation, signal) {
|
|
20
|
+
const warm = owner.runtimeConfig?.shared?.kernelWarmup?.warmPartition;
|
|
21
|
+
if (warm == null) return null;
|
|
22
|
+
const { tokenIds, decodeSteps } = warm;
|
|
23
|
+
const vocabSize = allocation.plan.vocabSize;
|
|
24
|
+
if (!Array.isArray(tokenIds) || tokenIds.length < 1 || tokenIds.length > allocation.limits.maxPromptTokens
|
|
25
|
+
|| tokenIds.some(id => !Number.isSafeInteger(id) || id < 0 || id >= vocabSize)) {
|
|
26
|
+
throw new Error(`${FIELD}.tokenIds must hold 1 to limits.maxPromptTokens token ids inside the vocabulary.`);
|
|
27
|
+
}
|
|
28
|
+
if (!Number.isSafeInteger(decodeSteps) || decodeSteps < 0 || decodeSteps > MAX_DECODE_STEPS) {
|
|
29
|
+
throw new Error(`${FIELD}.decodeSteps must be an integer from 0 to ${MAX_DECODE_STEPS}.`);
|
|
30
|
+
}
|
|
31
|
+
if (!owner.manifest || !owner.modelPartition) throw new Error('Partition warm-up requires a loaded, assigned model.');
|
|
32
|
+
const partition = resolveLayerPartition(owner.manifest, owner.modelPartition);
|
|
33
|
+
if (!partition) throw new Error('Partition warm-up allocation is missing.');
|
|
34
|
+
const plan = allocation.plan;
|
|
35
|
+
const hiddenSize = getPartitionActivationContract(plan).hiddenSize;
|
|
36
|
+
/** @param {number} count */
|
|
37
|
+
const zeros = count => new Uint8Array(validateActivationTensorShape({ shape: [1, count, hiddenSize],
|
|
38
|
+
dtype: plan.activationDtype, precision: plan.activationPrecision, components: plan.activationComponents,
|
|
39
|
+
elementStrideBytes: plan.activationElementStrideBytes }).expectedBytes);
|
|
40
|
+
const started = performance.now();
|
|
41
|
+
signal?.throwIfAborted();
|
|
42
|
+
const active = signal ?? new AbortController().signal;
|
|
43
|
+
const attempt = createPartitionAttempt(owner, tokenIds.length + decodeSteps);
|
|
44
|
+
const state = attempt.state;
|
|
45
|
+
/** @param {number[]} ids */
|
|
46
|
+
const step = async ids => {
|
|
47
|
+
const result = await executePartitionLayers(state, { numTokens: ids.length, tokenIds: ids,
|
|
48
|
+
...(partition.hasEmbedding ? {} : { activationBytes: zeros(ids.length) }) }, active);
|
|
49
|
+
if ('logits' in result && result.logits) releaseBuffer(result.logits.logitsBuffer);
|
|
50
|
+
state.currentSeqLen += ids.length;
|
|
51
|
+
state.decodeStepCount++;
|
|
52
|
+
};
|
|
53
|
+
try {
|
|
54
|
+
await step(tokenIds);
|
|
55
|
+
// Single-token steps select their own pipelines; the repeated id is a placeholder, not an answer.
|
|
56
|
+
for (let index = 0; index < decodeSteps; index++) await step([tokenIds[tokenIds.length - 1]]);
|
|
57
|
+
} finally {
|
|
58
|
+
// The private attempt's caches may not be destroyed until its submitted work has settled.
|
|
59
|
+
await owner.gpuContext?.device?.queue.onSubmittedWorkDone();
|
|
60
|
+
await attempt.close();
|
|
61
|
+
}
|
|
62
|
+
const receipt = { promptTokens: tokenIds.length, decodeSteps, tokenInput: partition.hasEmbedding,
|
|
63
|
+
durationMs: performance.now() - started };
|
|
64
|
+
log.info('Pipeline', `Partition warm-up: ${receipt.promptTokens} prompt tokens, ${decodeSteps} single-token steps, `
|
|
65
|
+
+ `${receipt.tokenInput ? 'token ids' : 'zero activations'} (${receipt.durationMs.toFixed(0)}ms)`);
|
|
66
|
+
return receipt;
|
|
67
|
+
}
|
|
@@ -15,6 +15,7 @@ import { createResidentOutputState } from "../../../resident-output-state.js";
|
|
|
15
15
|
import { ORDERED_LAYER_PARTITION_SCHEMA, hashLayerPartitionPlan, getPartitionActivationContract, resolveActivationTensorLayout, validateActivationTensorShape } from "../execution/layer-partition-contract.js";
|
|
16
16
|
import { markDeviceWeights, getDeviceMemorySnapshot } from "../../../../memory/device-budget.js";
|
|
17
17
|
import { prepareResidentPartitionPipelines } from './resident-pipeline-preparation.js';
|
|
18
|
+
import { runResidentPartitionWarmup } from './resident-partition-warmup.js';
|
|
18
19
|
|
|
19
20
|
/** @type {import('./resident-partition.js').createResidentPartitionSession} */
|
|
20
21
|
export async function createResidentPartitionSession(pipeline, allocation, tokens, closeProgram, signal, onLoadProgress) {
|
|
@@ -36,6 +37,9 @@ export async function createResidentPartitionSession(pipeline, allocation, token
|
|
|
36
37
|
// provider compile failure or crash happens at opening, not mid-answer.
|
|
37
38
|
const pipelinePreparation = await prepareResidentPartitionPipelines(owner, allocation, signal, onLoadProgress);
|
|
38
39
|
signal?.throwIfAborted();
|
|
40
|
+
// Optional and counted as opening time: a private prompt pass that leaves no sequence state.
|
|
41
|
+
await runResidentPartitionWarmup(owner, allocation, signal);
|
|
42
|
+
signal?.throwIfAborted();
|
|
39
43
|
// Resident execution creates one cache per attempt. The opening pipeline's
|
|
40
44
|
// empty cache is never used by those attempts and must not reserve a second
|
|
41
45
|
// copy of the participant's entire context allocation.
|
|
@@ -137,6 +141,7 @@ export async function createResidentPartitionSession(pipeline, allocation, token
|
|
|
137
141
|
}
|
|
138
142
|
const combined = AbortSignal.any([signal, attempt.controller.signal]);
|
|
139
143
|
const operation = runPipelineOperation(pipeline, async owner => {
|
|
144
|
+
let queueSettled = false;
|
|
140
145
|
try {
|
|
141
146
|
assertOpen(); combined.throwIfAborted();
|
|
142
147
|
const ids = 'tokenIds' in request ? request.tokenIds : request.inputTokenIds;
|
|
@@ -178,6 +183,7 @@ export async function createResidentPartitionSession(pipeline, allocation, token
|
|
|
178
183
|
if (ordered && partition.hasLmHead) throw new Error('Final ordered stage must produce logits.');
|
|
179
184
|
if (canonicalMode) attempt.canonicalAwaiting = true;
|
|
180
185
|
if (result.activationBytes.byteLength > allocation.limits.maxActivationBytes) throw new Error('Resident activation exceeds its byte allocation.');
|
|
186
|
+
queueSettled = true;
|
|
181
187
|
return { activationTensor: { shape: [1, request.inputTokenCount, getPartitionActivationContract(allocation.plan).hiddenSize],
|
|
182
188
|
dtype: allocation.plan.activationDtype,
|
|
183
189
|
...(allocation.plan.activationPrecision === 'two-f32' ? { precision: allocation.plan.activationPrecision,
|
|
@@ -188,7 +194,7 @@ export async function createResidentPartitionSession(pipeline, allocation, token
|
|
|
188
194
|
if (!result.logits) throw new Error('Resident final stage did not produce logits.');
|
|
189
195
|
let logits;
|
|
190
196
|
const readbackStarted = performance.now();
|
|
191
|
-
try { logits = new Float32Array(await readBuffer(result.logits.logitsBuffer, result.logits.vocabSize * Float32Array.BYTES_PER_ELEMENT)); }
|
|
197
|
+
try { logits = new Float32Array(await readBuffer(result.logits.logitsBuffer, result.logits.vocabSize * Float32Array.BYTES_PER_ELEMENT)); queueSettled = true; }
|
|
192
198
|
finally { releaseBuffer(result.logits.logitsBuffer); }
|
|
193
199
|
metrics.logitsReadbackMs = performance.now() - readbackStarted;
|
|
194
200
|
combined.throwIfAborted();
|
|
@@ -236,7 +242,7 @@ export async function createResidentPartitionSession(pipeline, allocation, token
|
|
|
236
242
|
} finally {
|
|
237
243
|
// Attempt caches cannot be destroyed until submitted work has settled,
|
|
238
244
|
// including cancellation or a failure after recording started.
|
|
239
|
-
await owner.gpuContext?.device?.queue.onSubmittedWorkDone();
|
|
245
|
+
if (!queueSettled) await owner.gpuContext?.device?.queue.onSubmittedWorkDone();
|
|
240
246
|
}
|
|
241
247
|
}, combined);
|
|
242
248
|
attempt.pending = operation;
|
|
@@ -2,6 +2,7 @@ import { DEFAULT_EXPERT_CACHE_CONFIG, validateQwen35RoutedConfig } from "../../c
|
|
|
2
2
|
import { createWeightBuffer } from '../../gpu/weight-buffer.js';
|
|
3
3
|
import { isDeviceLost } from '../../memory/device-residency.js';
|
|
4
4
|
import { readRoutedExpertRange, readRoutedExpertPayload, readRoutedExpertsLease, readRoutedExpertPlaneLeases, recordRoutedExpertSourceTimings, assertExpertReadShape } from './routed-expert-source.js';
|
|
5
|
+
import { getMappedExpertStaging, stageRoutedExperts } from './mapped-expert-staging.js';
|
|
5
6
|
|
|
6
7
|
/** @type {typeof import('./compact-expert-cache.js').compactExpertCapacity} */
|
|
7
8
|
export function compactExpertCapacity(plan, options) {
|
|
@@ -117,6 +118,7 @@ export class CompactExpertCache {
|
|
|
117
118
|
* @param {import('./compact-expert-cache.js').CompactExpertPool} pool
|
|
118
119
|
* @param {{expert: number, slot: number}} miss @param {AbortSignal} signal */
|
|
119
120
|
async #fillSinglePlanes(plan, pool, { expert, slot }, signal) {
|
|
121
|
+
if (await this.#stage(plan, pool, [{ expert, slot }], signal)) return;
|
|
120
122
|
const started = performance.now();
|
|
121
123
|
const leases = await readRoutedExpertPlaneLeases(this.#options, this.#stats, plan, expert, signal, pool.stats);
|
|
122
124
|
let failure;
|
|
@@ -250,10 +252,28 @@ export class CompactExpertCache {
|
|
|
250
252
|
}
|
|
251
253
|
}
|
|
252
254
|
|
|
255
|
+
/** Mapped staging for experts the storage port serves from verified state; false means use the lease path.
|
|
256
|
+
* @param {import('./routed-expert-plan.js').RoutedExpertBankPlan} plan
|
|
257
|
+
* @param {import('./compact-expert-cache.js').CompactExpertPool} pool
|
|
258
|
+
* @param {Array<{expert: number, slot: number}>} misses @param {AbortSignal} signal */
|
|
259
|
+
async #stage(plan, pool, misses, signal) {
|
|
260
|
+
// A host cache both supplies and remembers expert bytes; staged bytes never reach JavaScript memory.
|
|
261
|
+
if (this.#options.copyCachedExpert || this.#options.rememberExpert) return false;
|
|
262
|
+
const staging = getMappedExpertStaging(this.#options, plan);
|
|
263
|
+
if (!staging) return false;
|
|
264
|
+
const slots = new Map(misses.map(({ expert, slot }) => [expert, slot]));
|
|
265
|
+
return stageRoutedExperts(staging, this.#stats, plan, misses.map(row => row.expert), (expert, plane) => {
|
|
266
|
+
const buffer = pool.planes.get(plane.role), slot = slots.get(expert);
|
|
267
|
+
if (!buffer || slot === undefined) throw new Error('Compact expert plane is missing.');
|
|
268
|
+
return { buffer, offset: slot * plane.expertBytes };
|
|
269
|
+
}, signal, null, pool.stats);
|
|
270
|
+
}
|
|
271
|
+
|
|
253
272
|
/** @param {import('./routed-expert-plan.js').RoutedExpertBankPlan} plan
|
|
254
273
|
* @param {import('./compact-expert-cache.js').CompactExpertPool} pool
|
|
255
274
|
* @param {Array<{expert: number, slot: number}>} misses @param {AbortSignal} signal */
|
|
256
275
|
async #fillBatch(plan, pool, misses, signal) {
|
|
276
|
+
if (await this.#stage(plan, pool, misses, signal)) return;
|
|
257
277
|
const pending = misses.filter(({ expert, slot }) => !this.#copyCached(plan, pool, expert, slot, signal));
|
|
258
278
|
if (!pending.length) return;
|
|
259
279
|
const started = performance.now();
|
|
@@ -6,6 +6,8 @@ import { TieredExpertCache } from './tiered-expert-cache.js';
|
|
|
6
6
|
import { RoutedExpertCache } from './routed-expert-cache.js';
|
|
7
7
|
import { planRoutedExpertBank } from './routed-expert-plan.js';
|
|
8
8
|
import { prepareQwenExpertLayer } from './qwen-paging.js';
|
|
9
|
+
import { trace, isTraceEnabled } from '../../debug/index.js';
|
|
10
|
+
import { assertStagedVerification } from './mapped-expert-staging.js';
|
|
9
11
|
|
|
10
12
|
/** @type {typeof import('./loader-bindings.js').prepareExpertLayer} */
|
|
11
13
|
export async function prepareExpertLayer(loader, policy, layer, consumers, recorder, signal, activeSelection) {
|
|
@@ -112,12 +114,21 @@ export async function prepareRoutedExperts(loader, layer, expertIds, signal, pha
|
|
|
112
114
|
if (!loader.manifest) throw new Error('Routed experts require the loaded manifest.');
|
|
113
115
|
const plan = planRoutedExpertBank(loader.manifest, loader.tensorLocations, layer);
|
|
114
116
|
const cache = ensureRoutedCache(loader, plan);
|
|
117
|
+
// After a failed deferred verification, unverified experts are resident in the pools. No caller gets them,
|
|
118
|
+
// whether or not it opens a release gate, and whether it would upload or only hit the cache.
|
|
119
|
+
if (loader.shardCache.rangeLease) assertStagedVerification(loader.shardCache.rangeLease);
|
|
120
|
+
// With the loader trace on, one line when a layer asks for its experts and one when they are ready for
|
|
121
|
+
// its kernels. Together with the staged-upload lines these put read, hash, upload and use on one clock.
|
|
122
|
+
const traced = isTraceEnabled('loader');
|
|
123
|
+
if (traced) trace.loader(`routed-experts request ${JSON.stringify({ layer, phase, experts: expertIds.length, at: Number(performance.now().toFixed(2)) })}`);
|
|
124
|
+
let weights;
|
|
115
125
|
if (cache instanceof TieredExpertCache) {
|
|
116
126
|
const admitted = tieredPlans.get(cache)?.get(layer);
|
|
117
127
|
if (!admitted) throw new Error('Tiered expert request lacks the admitted original layer plan.');
|
|
118
|
-
|
|
119
|
-
}
|
|
120
|
-
|
|
128
|
+
weights = await cache.prepare(admitted, expertIds, signal, phase);
|
|
129
|
+
} else weights = await cache.prepare(plan, expertIds, signal, phase);
|
|
130
|
+
if (traced) trace.loader(`routed-experts ready ${JSON.stringify({ layer, phase, at: Number(performance.now().toFixed(2)) })}`);
|
|
131
|
+
return weights;
|
|
121
132
|
}
|
|
122
133
|
|
|
123
134
|
/** @type {typeof import('./loader-bindings.js').disposeRoutedExperts} */
|
|
@@ -0,0 +1,393 @@
|
|
|
1
|
+
import { recordRoutedExpertSourceTimings } from './routed-expert-source.js';
|
|
2
|
+
import { trace, isTraceEnabled } from '../../debug/index.js';
|
|
3
|
+
|
|
4
|
+
/*
|
|
5
|
+
* Staged expert uploads for policy.uploadStaging === 'mapped'.
|
|
6
|
+
*
|
|
7
|
+
* queue.writeBuffer copies every expert on the JavaScript thread, which made that thread the limit
|
|
8
|
+
* of expert loading. Here a small ring of mapped upload buffers receives the bytes instead: the
|
|
9
|
+
* storage port reads already verified ranges on the I/O pool straight into a mapped slot, the slot
|
|
10
|
+
* is unmapped, and one submitted command buffer copies each plane to its place. The submit happens
|
|
11
|
+
* where the writeBuffer calls happened, so queue order relative to the caller's kernels is the same.
|
|
12
|
+
*
|
|
13
|
+
* A range whose pieces are not verified yet (first-time verification) is read into the slot at the
|
|
14
|
+
* same time as a lane hashes those pieces. The slot is unmapped and its copy submitted only after the
|
|
15
|
+
* lane has finished and the port confirms that every piece is verified for the file state the read
|
|
16
|
+
* saw. Until then the bytes exist only in the mapped slot, which no model buffer and no kernel can
|
|
17
|
+
* see. If the confirmation fails, or the port cannot read ahead, nothing is submitted and the caller
|
|
18
|
+
* uses its lease path. The lane's own copy of the bytes is not used: copying it into the slot would
|
|
19
|
+
* put the copy back on the JavaScript thread.
|
|
20
|
+
*
|
|
21
|
+
* policy.uploadVerification === 'before-release' moves that wait. Inside a release gate (a resident
|
|
22
|
+
* partition step opens one and settles it before its output leaves the stage) first-time bytes are
|
|
23
|
+
* submitted as soon as they are read, the kernels compute with them, and the lanes hash in the
|
|
24
|
+
* background. Settling the gate waits for every hash and confirmation. If one fails, the step fails,
|
|
25
|
+
* its output is never released, and the gate stays failed: no later step may begin until the stage is
|
|
26
|
+
* reopened, because unverified bytes are resident in model buffers. Outside a gate nothing is deferred.
|
|
27
|
+
*/
|
|
28
|
+
|
|
29
|
+
const MAX_SLOTS = 16;
|
|
30
|
+
/** @type {WeakMap<object, MappedExpertStaging | null>} */
|
|
31
|
+
const stagings = new WeakMap();
|
|
32
|
+
/** One release gate per storage port.
|
|
33
|
+
* @type {WeakMap<object, { depth: number, tasks: Set<Promise<void>>, failure: Error | null, stats: import('./routed-expert-source.js').RoutedExpertCounters | null }>} */
|
|
34
|
+
const gates = new WeakMap();
|
|
35
|
+
|
|
36
|
+
/** @type {import('./mapped-expert-staging.js').beginStagedRelease} */
|
|
37
|
+
export function beginStagedRelease(port) {
|
|
38
|
+
let gate = gates.get(port);
|
|
39
|
+
if (!gate) { gate = { depth: 0, tasks: new Set(), failure: null, stats: null }; gates.set(port, gate); }
|
|
40
|
+
if (gate.failure) throw gate.failure;
|
|
41
|
+
gate.depth++;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/** @type {import('./mapped-expert-staging.js').assertStagedVerification} */
|
|
45
|
+
export function assertStagedVerification(port) {
|
|
46
|
+
const failure = gates.get(port)?.failure;
|
|
47
|
+
if (failure) throw failure;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/** @type {import('./mapped-expert-staging.js').settleStagedRelease} */
|
|
51
|
+
export async function settleStagedRelease(port, { release = true } = {}) {
|
|
52
|
+
const gate = gates.get(port);
|
|
53
|
+
if (!gate || gate.depth < 1) throw new Error('Staged release gate is not open.');
|
|
54
|
+
const started = performance.now();
|
|
55
|
+
try {
|
|
56
|
+
// A task may start another only through a new upload, which the caller has stopped making.
|
|
57
|
+
while (gate.tasks.size) await Promise.allSettled([...gate.tasks]);
|
|
58
|
+
} finally { gate.depth--; }
|
|
59
|
+
if (gate.stats) gate.stats.stagedReleaseWaitMs += performance.now() - started;
|
|
60
|
+
if (release && gate.failure) throw gate.failure;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
class MappedExpertStaging {
|
|
64
|
+
/** @type {GPUDevice} */ #device;
|
|
65
|
+
/** @type {NonNullable<import('../../storage/verified-piece-gather.js').VerifiedRangeLeasePort['readVerifiedInto']>} */ #read;
|
|
66
|
+
/** @type {((ranges: Array<{ shard: number, offset: number, length: number }>, signal: AbortSignal) => Promise<import('../../storage/verified-piece-gather.js').VerifiedRangeLease>) | null} */ #verify;
|
|
67
|
+
/** @type {import('../../storage/verified-piece-gather.js').VerifiedRangeLeasePort['readAheadInto'] | null} */ #readAhead;
|
|
68
|
+
/** @type {import('../../storage/verified-piece-gather.js').VerifiedRangeLeasePort['hasVerified'] | null} */ #hasVerified;
|
|
69
|
+
/** @type {object} */ #port;
|
|
70
|
+
/** @type {import('../../storage/verified-piece-gather.js').VerifiedRangeLeasePort['verifyMany'] | null} */ #verifyQueued;
|
|
71
|
+
/** @type {import('../../storage/verified-piece-gather.js').VerifiedRangeLeasePort['verifyMany'] | null} */ #verifyWaiting;
|
|
72
|
+
#lanes = 1; #active = 0;
|
|
73
|
+
#maxRanges; #maxPartBytes;
|
|
74
|
+
/** @type {Array<{ buffer: GPUBuffer, view: Uint8Array | null }>} */ #slots = [];
|
|
75
|
+
/** @type {Array<{ buffer: GPUBuffer, view: Uint8Array | null }>} */ #free = [];
|
|
76
|
+
/** @type {Array<(slot: { buffer: GPUBuffer, view: Uint8Array | null } | null) => void>} */ #waiters = [];
|
|
77
|
+
/** @type {Set<Promise<void>>} */ #remapping = new Set();
|
|
78
|
+
#closed = false;
|
|
79
|
+
|
|
80
|
+
/** @param {{ device: GPUDevice, port: import('../../storage/verified-piece-gather.js').VerifiedRangeLeasePort, slotBytes: number, slotCount: number, deferVerification: boolean }} options */
|
|
81
|
+
constructor({ device, port, slotBytes, slotCount, deferVerification }) {
|
|
82
|
+
if (typeof port.readVerifiedInto !== 'function') throw new Error('Mapped expert staging requires direct verified reads.');
|
|
83
|
+
this.#device = device; this.#read = port.readVerifiedInto.bind(port);
|
|
84
|
+
this.#verify = typeof port.readMany === 'function' ? port.readMany.bind(port) : null;
|
|
85
|
+
const ahead = typeof port.readAheadInto === 'function' && typeof port.hasVerified === 'function';
|
|
86
|
+
this.#readAhead = ahead ? /** @type {NonNullable<typeof port.readAheadInto>} */ (port.readAheadInto).bind(port) : null;
|
|
87
|
+
this.#hasVerified = ahead ? /** @type {NonNullable<typeof port.hasVerified>} */ (port.hasVerified).bind(port) : null;
|
|
88
|
+
this.#port = port;
|
|
89
|
+
this.#verifyWaiting = typeof port.verifyMany === 'function' ? port.verifyMany.bind(port) : null;
|
|
90
|
+
this.#verifyQueued = deferVerification ? this.#verifyWaiting : null;
|
|
91
|
+
this.#lanes = Number.isSafeInteger(port.concurrency) && Number(port.concurrency) > 0 ? Number(port.concurrency) : 1;
|
|
92
|
+
this.#maxRanges = port.maxRanges ?? 3; this.#maxPartBytes = port.maxPartBytes;
|
|
93
|
+
this.slotBytes = slotBytes; this.slotCount = slotCount;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
get bytes() { return this.#slots.length * this.slotBytes; }
|
|
97
|
+
|
|
98
|
+
/** @returns {Promise<{ buffer: GPUBuffer, view: Uint8Array | null } | null>} */
|
|
99
|
+
#take() {
|
|
100
|
+
if (this.#closed) return Promise.resolve(null);
|
|
101
|
+
const ready = this.#free.pop();
|
|
102
|
+
if (ready) return Promise.resolve(ready);
|
|
103
|
+
if (this.#slots.length < this.slotCount) {
|
|
104
|
+
const buffer = this.#device.createBuffer({ label: 'expert_staged_upload', size: this.slotBytes,
|
|
105
|
+
usage: GPUBufferUsage.MAP_WRITE | GPUBufferUsage.COPY_SRC, mappedAtCreation: true });
|
|
106
|
+
const slot = { buffer, view: new Uint8Array(buffer.getMappedRange(0, this.slotBytes)) };
|
|
107
|
+
this.#slots.push(slot);
|
|
108
|
+
return Promise.resolve(slot);
|
|
109
|
+
}
|
|
110
|
+
return new Promise(resolve => this.#waiters.push(resolve));
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/** @param {{ buffer: GPUBuffer, view: Uint8Array | null }} slot */
|
|
114
|
+
#give(slot) {
|
|
115
|
+
const waiter = this.#waiters.shift();
|
|
116
|
+
if (waiter) waiter(slot); else this.#free.push(slot);
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/** The GPU may still be copying from the slot; it is free again when the map resolves.
|
|
120
|
+
* @param {{ buffer: GPUBuffer, view: Uint8Array | null }} slot */
|
|
121
|
+
#remap(slot) {
|
|
122
|
+
const operation = slot.buffer.mapAsync(GPUMapMode.WRITE).then(() => {
|
|
123
|
+
if (this.#closed) return;
|
|
124
|
+
slot.view = new Uint8Array(slot.buffer.getMappedRange(0, this.slotBytes));
|
|
125
|
+
this.#give(slot);
|
|
126
|
+
}, () => {
|
|
127
|
+
// A slot that cannot be mapped again is retired; waiting callers fall back to their lease path.
|
|
128
|
+
this.#slots = this.#slots.filter(candidate => candidate !== slot);
|
|
129
|
+
slot.buffer.destroy();
|
|
130
|
+
if (this.#slots.length === 0) for (const waiter of this.#waiters.splice(0)) waiter(null);
|
|
131
|
+
});
|
|
132
|
+
this.#remapping.add(operation);
|
|
133
|
+
operation.finally(() => this.#remapping.delete(operation));
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* @param {ReadonlyArray<{ shard: number, offset: number, length: number, at: number }>} parts
|
|
138
|
+
* @param {ReadonlyArray<{ at: number, length: number, target: GPUBuffer, targetOffset: number }>} copies
|
|
139
|
+
* @param {AbortSignal} signal @param {(() => void) | null} beforeSubmit
|
|
140
|
+
* @param {import('./routed-expert-source.js').RoutedExpertCounters | null} [stats] receives the timings of background verification
|
|
141
|
+
* @param {number | null} [layer] names the upload in the loader trace
|
|
142
|
+
* @returns {Promise<false | import('./mapped-expert-staging.js').StagedUpload>} false when nothing was staged and nothing was submitted
|
|
143
|
+
*/
|
|
144
|
+
async upload(parts, copies, signal, beforeSubmit, stats = null, layer = null) {
|
|
145
|
+
const gate = gates.get(this.#port) ?? null;
|
|
146
|
+
if (gate?.failure) throw gate.failure;
|
|
147
|
+
const slot = await this.#take();
|
|
148
|
+
if (!slot) return false;
|
|
149
|
+
let unmapped = false;
|
|
150
|
+
this.#active++;
|
|
151
|
+
try {
|
|
152
|
+
const view = slot.view;
|
|
153
|
+
if (!view) throw new Error('Staged upload slot is not mapped.');
|
|
154
|
+
const started = performance.now();
|
|
155
|
+
const hasVerified = this.#hasVerified, readAhead = this.#readAhead;
|
|
156
|
+
const pending = hasVerified ? parts.filter(part => !hasVerified(part.shard, part.offset, part.length)) : [];
|
|
157
|
+
if (pending.length && (!this.#verify || !readAhead || pending.some(part => part.length > this.#maxPartBytes))) return false;
|
|
158
|
+
const ahead = new Set(pending);
|
|
159
|
+
const verifyQueued = this.#verifyQueued;
|
|
160
|
+
if (pending.length && verifyQueued && gate && gate.depth > 0) {
|
|
161
|
+
// Verification before release: submit once the bytes are read, and let the gate hold the output.
|
|
162
|
+
const read = await Promise.allSettled(parts.map(part => {
|
|
163
|
+
const target = view.subarray(part.at, part.at + part.length);
|
|
164
|
+
return ahead.has(part) && readAhead ? readAhead(part.shard, part.offset, target, signal)
|
|
165
|
+
: this.#read(part.shard, part.offset, target, signal);
|
|
166
|
+
}));
|
|
167
|
+
const broken = read.find(outcome => outcome.status === 'rejected');
|
|
168
|
+
if (broken) throw /** @type {PromiseRejectedResult} */ (broken).reason;
|
|
169
|
+
const values = read.map(outcome => /** @type {PromiseFulfilledResult<boolean | (() => boolean)>} */ (outcome).value);
|
|
170
|
+
// Only an already verified range (true) or a read with a confirmation to run later may be submitted.
|
|
171
|
+
if (values.some(value => value !== true && typeof value !== 'function')) return false;
|
|
172
|
+
const readWait = performance.now() - started;
|
|
173
|
+
signal.throwIfAborted();
|
|
174
|
+
// The gate must still be open: an upload that outlived its step has nothing left to hold its bytes back.
|
|
175
|
+
if (this.#closed || gate.failure || gate.depth < 1) return false;
|
|
176
|
+
const confirms = values.filter(value => typeof value === 'function');
|
|
177
|
+
const groups = this.#groups(pending);
|
|
178
|
+
const bytes = pending.reduce((sum, part) => sum + part.length, 0);
|
|
179
|
+
slot.view = null; slot.buffer.unmap(); unmapped = true;
|
|
180
|
+
beforeSubmit?.();
|
|
181
|
+
const encoder = this.#device.createCommandEncoder({ label: 'expert_staged_upload' });
|
|
182
|
+
for (const copy of copies) encoder.copyBufferToBuffer(slot.buffer, copy.at, copy.target, copy.targetOffset, copy.length);
|
|
183
|
+
this.#device.queue.submit([encoder.finish()]);
|
|
184
|
+
// From here unverified bytes are in model buffers. The verification task is registered before anything
|
|
185
|
+
// else can throw. Every lane job settles before the outcome is decided, and only a failed hash or a
|
|
186
|
+
// failed confirmation fails the gate; counters and trace lines cannot.
|
|
187
|
+
const task = (async () => {
|
|
188
|
+
const outcomes = await Promise.allSettled(groups.map(group => verifyQueued(group)));
|
|
189
|
+
const failed = outcomes.find(outcome => outcome.status === 'rejected');
|
|
190
|
+
let failure = failed ? /** @type {PromiseRejectedResult} */ (failed).reason : null;
|
|
191
|
+
if (!failure) {
|
|
192
|
+
try {
|
|
193
|
+
for (const confirm of confirms) {
|
|
194
|
+
if (!(/** @type {() => boolean} */ (confirm))()) throw new Error('Expert bytes already in use could not be confirmed verified.');
|
|
195
|
+
}
|
|
196
|
+
} catch (error) { failure = error; }
|
|
197
|
+
}
|
|
198
|
+
if (failure) {
|
|
199
|
+
gate.failure ??= new Error(`Staged expert verification failed after upload: ${failure?.message ?? failure}`, { cause: failure });
|
|
200
|
+
return;
|
|
201
|
+
}
|
|
202
|
+
try {
|
|
203
|
+
const timings = outcomes.map(outcome => /** @type {PromiseFulfilledResult<import('./mapped-expert-staging.js').StagedUpload['laneTimings'][number]>} */ (outcome).value ?? null);
|
|
204
|
+
if (stats) recordRoutedExpertSourceTimings(stats, timings.map(timing => timing ? { timings: timing } : null));
|
|
205
|
+
if (isTraceEnabled('loader')) traceUpload({ layer, bytes, verifiedAfterSubmit: true, startedAt: started, verifyDoneAt: performance.now(), lanes: timings });
|
|
206
|
+
} catch { /* Observation only: the bytes are verified. */ }
|
|
207
|
+
})();
|
|
208
|
+
gate.tasks.add(task);
|
|
209
|
+
task.finally(() => gate.tasks.delete(task));
|
|
210
|
+
if (stats) gate.stats = stats;
|
|
211
|
+
if (isTraceEnabled('loader')) traceUpload({ layer, bytes, firstTime: bytes, deferred: true, startedAt: started, readDoneAt: started + readWait, submittedAt: performance.now() });
|
|
212
|
+
return { laneTimings: [], readAheadBytes: bytes, readWaitMs: readWait, verifyWaitMs: 0, deferred: true };
|
|
213
|
+
}
|
|
214
|
+
// The reads and the lanes run together. Every read settles before the slot is unmapped or handed
|
|
215
|
+
// on, including after a failure, because a native read may still be writing into the slot.
|
|
216
|
+
let readWaitMs = 0, verifyWaitMs = 0;
|
|
217
|
+
const reads = Promise.allSettled(parts.map(part => {
|
|
218
|
+
const target = view.subarray(part.at, part.at + part.length);
|
|
219
|
+
return ahead.has(part) && readAhead ? readAhead(part.shard, part.offset, target, signal)
|
|
220
|
+
: this.#read(part.shard, part.offset, target, signal);
|
|
221
|
+
})).then(outcomes => { readWaitMs = performance.now() - started; return outcomes; });
|
|
222
|
+
/** @type {Promise<{ timings: import('./mapped-expert-staging.js').StagedUpload['laneTimings'], error?: unknown }>} */
|
|
223
|
+
// Lanes that no other upload is using hash this upload's ranges side by side. During decode a layer
|
|
224
|
+
// asks for one or two experts at a time, so most lanes would otherwise sit idle behind one job.
|
|
225
|
+
const verification = pending.length
|
|
226
|
+
? this.#verifyRanges(pending, signal, Math.floor(this.#lanes / this.#active)).then(timings => { verifyWaitMs = performance.now() - started; return { timings }; },
|
|
227
|
+
error => ({ timings: [], error }))
|
|
228
|
+
: Promise.resolve({ timings: [] });
|
|
229
|
+
const [outcomes, verified] = await Promise.all([reads, verification]);
|
|
230
|
+
const failed = outcomes.find(outcome => outcome.status === 'rejected');
|
|
231
|
+
if (failed) throw /** @type {PromiseRejectedResult} */ (failed).reason;
|
|
232
|
+
if ('error' in verified) throw verified.error;
|
|
233
|
+
// A range read ahead counts only once the port confirms its pieces are now verified.
|
|
234
|
+
for (const outcome of outcomes) {
|
|
235
|
+
const value = /** @type {PromiseFulfilledResult<boolean | (() => boolean)>} */ (outcome).value;
|
|
236
|
+
if (value !== true && (typeof value !== 'function' || !value())) return false;
|
|
237
|
+
}
|
|
238
|
+
signal.throwIfAborted();
|
|
239
|
+
if (this.#closed) return false;
|
|
240
|
+
slot.view = null; slot.buffer.unmap(); unmapped = true;
|
|
241
|
+
beforeSubmit?.();
|
|
242
|
+
const encoder = this.#device.createCommandEncoder({ label: 'expert_staged_upload' });
|
|
243
|
+
for (const copy of copies) encoder.copyBufferToBuffer(slot.buffer, copy.at, copy.target, copy.targetOffset, copy.length);
|
|
244
|
+
this.#device.queue.submit([encoder.finish()]);
|
|
245
|
+
const firstTime = pending.reduce((sum, part) => sum + part.length, 0);
|
|
246
|
+
if (isTraceEnabled('loader')) {
|
|
247
|
+
traceUpload({ layer, bytes: parts.reduce((sum, part) => sum + part.length, 0), firstTime, deferred: false, startedAt: started,
|
|
248
|
+
readDoneAt: started + readWaitMs, ...(firstTime ? { verifyDoneAt: started + verifyWaitMs, lanes: verified.timings } : {}), submittedAt: performance.now() });
|
|
249
|
+
}
|
|
250
|
+
return { laneTimings: verified.timings, readAheadBytes: firstTime, readWaitMs, verifyWaitMs, deferred: false };
|
|
251
|
+
} finally {
|
|
252
|
+
this.#active--;
|
|
253
|
+
if (unmapped) this.#remap(slot);
|
|
254
|
+
else this.#give(slot);
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
/** Have lanes authenticate these ranges; their copies of the bytes are not used. Resolves each job's timings.
|
|
259
|
+
* With `spread` above one and a port that queues verification, the ranges go to up to that many lanes at once.
|
|
260
|
+
* @param {ReadonlyArray<{ shard: number, offset: number, length: number }>} parts @param {AbortSignal} signal
|
|
261
|
+
* @param {number} [spread]
|
|
262
|
+
* @returns {Promise<import('./mapped-expert-staging.js').StagedUpload['laneTimings']>} */
|
|
263
|
+
async #verifyRanges(parts, signal, spread = 1) {
|
|
264
|
+
if (!this.#verify) throw new Error('Staged upload has no lane to verify with.');
|
|
265
|
+
const waiting = this.#verifyWaiting;
|
|
266
|
+
if (waiting && spread > 1 && parts.length > 1) {
|
|
267
|
+
const jobs = Math.min(spread, parts.length), size = Math.min(this.#maxRanges, Math.ceil(parts.length / jobs));
|
|
268
|
+
/** @type {Array<Array<{ shard: number, offset: number, length: number }>>} */ const chunks = [];
|
|
269
|
+
for (let at = 0; at < parts.length; at += size) chunks.push(parts.slice(at, at + size).map(({ shard, offset, length }) => ({ shard, offset, length })));
|
|
270
|
+
// Every job settles before a failure is reported, so no lane is still reading when the caller moves on.
|
|
271
|
+
const outcomes = await Promise.allSettled(chunks.flatMap(chunk => this.#groups(chunk)).map(group => waiting(group, signal)));
|
|
272
|
+
const failed = outcomes.find(outcome => outcome.status === 'rejected');
|
|
273
|
+
if (failed) throw /** @type {PromiseRejectedResult} */ (failed).reason;
|
|
274
|
+
return outcomes.map(outcome => /** @type {PromiseFulfilledResult<import('./mapped-expert-staging.js').StagedUpload['laneTimings'][number]>} */ (outcome).value ?? null);
|
|
275
|
+
}
|
|
276
|
+
/** @type {import('./mapped-expert-staging.js').StagedUpload['laneTimings']} */ const timings = [];
|
|
277
|
+
for (const group of this.#groups(parts)) {
|
|
278
|
+
signal.throwIfAborted();
|
|
279
|
+
const lease = await this.#verify(group, signal);
|
|
280
|
+
timings.push(lease.timings ?? null);
|
|
281
|
+
lease.release();
|
|
282
|
+
}
|
|
283
|
+
return timings;
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
/** Split ranges into lane jobs within the port's range and byte bounds.
|
|
287
|
+
* @param {ReadonlyArray<{ shard: number, offset: number, length: number }>} parts */
|
|
288
|
+
#groups(parts) {
|
|
289
|
+
/** @type {Array<Array<{ shard: number, offset: number, length: number }>>} */ const groups = [];
|
|
290
|
+
let bytes = 0;
|
|
291
|
+
for (const { shard, offset, length } of parts) {
|
|
292
|
+
const group = groups.at(-1);
|
|
293
|
+
if (!group || group.length >= this.#maxRanges || bytes + length > this.#maxPartBytes) { groups.push([{ shard, offset, length }]); bytes = length; }
|
|
294
|
+
else { group.push({ shard, offset, length }); bytes += length; }
|
|
295
|
+
}
|
|
296
|
+
return groups;
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
async destroy() {
|
|
300
|
+
this.#closed = true;
|
|
301
|
+
const gate = gates.get(this.#port);
|
|
302
|
+
if (gate) await Promise.allSettled([...gate.tasks]);
|
|
303
|
+
for (const waiter of this.#waiters.splice(0)) waiter(null);
|
|
304
|
+
await Promise.allSettled([...this.#remapping]);
|
|
305
|
+
await this.#device.queue.onSubmittedWorkDone();
|
|
306
|
+
for (const slot of this.#slots.splice(0)) slot.buffer.destroy();
|
|
307
|
+
this.#free.length = 0;
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
/**
|
|
312
|
+
* One loader-trace line per staged upload, in milliseconds on the process clock: when it started, when its
|
|
313
|
+
* reads into the slot had settled, when its lanes had hashed any first-time pieces (with each lane job's own
|
|
314
|
+
* read, hash and wall time), and when its copy was submitted.
|
|
315
|
+
* @param {{ layer: number | null, bytes: number, firstTime?: number, deferred?: boolean, verifiedAfterSubmit?: boolean, startedAt: number,
|
|
316
|
+
* readDoneAt?: number, verifyDoneAt?: number, submittedAt?: number, lanes?: ReadonlyArray<unknown> }} event */
|
|
317
|
+
function traceUpload(event) {
|
|
318
|
+
const round = /** @param {number | undefined} value */ value => value === undefined ? undefined : Number(value.toFixed(2));
|
|
319
|
+
const lanes = (event.lanes ?? []).filter(Boolean).map(job => {
|
|
320
|
+
const timing = /** @type {{ workerWallMs: number, readWaitMs: number, verificationMs: number, roundTripMs: number }} */ (job);
|
|
321
|
+
return [round(timing.workerWallMs), round(timing.readWaitMs), round(timing.verificationMs), round(timing.roundTripMs)];
|
|
322
|
+
});
|
|
323
|
+
trace.loader(`staged-upload ${JSON.stringify({ layer: event.layer, bytes: event.bytes, firstTime: event.firstTime, deferred: event.deferred,
|
|
324
|
+
verifiedAfterSubmit: event.verifiedAfterSubmit, start: round(event.startedAt), readDone: round(event.readDoneAt),
|
|
325
|
+
verifyDone: round(event.verifyDoneAt), submitted: round(event.submittedAt), ...(lanes.length ? { laneWallReadHashRoundTrip: lanes } : {}) })}`);
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
/** One ring per storage port: prefill and decode share it.
|
|
329
|
+
* @type {import('./mapped-expert-staging.js').getMappedExpertStaging} */
|
|
330
|
+
export function getMappedExpertStaging(options, plan) {
|
|
331
|
+
const port = options.rangeLease;
|
|
332
|
+
if (options.policy.uploadStaging !== 'mapped' || options.policy.sourceReadMode !== 'expert-lease-batch'
|
|
333
|
+
|| !port || typeof port.readVerifiedInto !== 'function') return null;
|
|
334
|
+
let staging = stagings.get(port);
|
|
335
|
+
if (staging === undefined) {
|
|
336
|
+
const slotBytes = Math.ceil(plan.expertBytes / 4) * 4;
|
|
337
|
+
const slotCount = Math.min(MAX_SLOTS, Math.floor(options.policy.stagingBytes / slotBytes));
|
|
338
|
+
staging = slotCount >= 1 ? new MappedExpertStaging({ device: options.device, port, slotBytes, slotCount,
|
|
339
|
+
deferVerification: options.policy.uploadVerification === 'before-release' }) : null;
|
|
340
|
+
stagings.set(port, staging);
|
|
341
|
+
}
|
|
342
|
+
return staging;
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
/** @type {import('./mapped-expert-staging.js').destroyMappedExpertStaging} */
|
|
346
|
+
export async function destroyMappedExpertStaging(options) {
|
|
347
|
+
const port = options.rangeLease;
|
|
348
|
+
const staging = port ? stagings.get(port) : null;
|
|
349
|
+
if (!port || !staging) return;
|
|
350
|
+
stagings.delete(port);
|
|
351
|
+
await staging.destroy();
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
/**
|
|
355
|
+
* Stage whole experts (every plane) and copy each plane to the buffer `targetOf` names.
|
|
356
|
+
* @type {import('./mapped-expert-staging.js').stageRoutedExperts}
|
|
357
|
+
*/
|
|
358
|
+
export async function stageRoutedExperts(staging, stats, plan, experts, targetOf, signal, beforeSubmit, layerStats) {
|
|
359
|
+
if (!experts.length || experts.length * plan.expertBytes > staging.slotBytes
|
|
360
|
+
|| plan.planes.some(plane => plane.location.sourceTransform != null
|
|
361
|
+
|| (plane.location.spans && plane.location.spans.length !== 1) || plane.expertBytes % 4)) return false;
|
|
362
|
+
/** @type {Array<{ shard: number, offset: number, length: number, at: number }>} */ const parts = [];
|
|
363
|
+
/** @type {Array<{ at: number, length: number, target: GPUBuffer, targetOffset: number }>} */ const copies = [];
|
|
364
|
+
let at = 0;
|
|
365
|
+
for (const expert of experts) {
|
|
366
|
+
for (const plane of plan.planes) {
|
|
367
|
+
const physical = plane.location.spans?.[0] ?? plane.location;
|
|
368
|
+
const offset = Number(physical.offset) + expert * plane.expertBytes;
|
|
369
|
+
if (!Number.isSafeInteger(physical.shardIndex) || Number(physical.shardIndex) < 0 || !Number.isSafeInteger(offset)
|
|
370
|
+
|| offset % 4 || offset + plane.expertBytes > Number(physical.offset) + Number(physical.size)) return false;
|
|
371
|
+
const { buffer, offset: targetOffset } = targetOf(expert, plane);
|
|
372
|
+
parts.push({ shard: Number(physical.shardIndex), offset, length: plane.expertBytes, at });
|
|
373
|
+
copies.push({ at, length: plane.expertBytes, target: buffer, targetOffset });
|
|
374
|
+
at += plane.expertBytes;
|
|
375
|
+
}
|
|
376
|
+
}
|
|
377
|
+
const started = performance.now();
|
|
378
|
+
const staged = await staging.upload(parts, copies, signal, beforeSubmit, stats, plan.layer);
|
|
379
|
+
if (!staged) return false;
|
|
380
|
+
const elapsed = performance.now() - started;
|
|
381
|
+
// Lanes ran only for ranges verified for the first time. With none, the job is counted as untimed. A
|
|
382
|
+
// deferred upload's lane jobs are recorded when they finish.
|
|
383
|
+
if (staged.deferred) stats.stagedDeferredBytes += staged.readAheadBytes;
|
|
384
|
+
else recordRoutedExpertSourceTimings(stats, staged.laneTimings.length ? staged.laneTimings.map(timings => timings ? { timings } : null) : [null]);
|
|
385
|
+
stats.stagedReadAheadBytes += staged.readAheadBytes; stats.stagedReadWaitMs += staged.readWaitMs; stats.stagedVerifyWaitMs += staged.verifyWaitMs;
|
|
386
|
+
stats.sourceReads += parts.length; stats.sourceReadBytes += at; stats.sourceReadIntoBytes += at;
|
|
387
|
+
stats.uploadBytes += at; stats.sourceReadWaitMs += elapsed; stats.ioWaitMs += elapsed;
|
|
388
|
+
stats.peakStagingBytes = Math.max(stats.peakStagingBytes, staging.bytes);
|
|
389
|
+
if (layerStats) {
|
|
390
|
+
layerStats.sourceReadBytes += at; layerStats.uploadBytes += at; layerStats.sourceReadWaitMs += elapsed;
|
|
391
|
+
}
|
|
392
|
+
return true;
|
|
393
|
+
}
|
|
@@ -68,6 +68,14 @@ export interface RoutedExpertCacheStats {
|
|
|
68
68
|
sourceJobVerificationMs: number;
|
|
69
69
|
sourceJobCopyMs: number;
|
|
70
70
|
sourceJobRoundTripMs: number;
|
|
71
|
+
/** Mapped staging, first-time verification: bytes read into a slot while a lane hashed their pieces. */
|
|
72
|
+
stagedReadAheadBytes: number;
|
|
73
|
+
/** Mapped staging: summed per-upload wall time until the slot's reads settled, and until its lanes finished. */
|
|
74
|
+
stagedReadWaitMs: number;
|
|
75
|
+
stagedVerifyWaitMs: number;
|
|
76
|
+
/** Bytes submitted before their verification finished, and the time release gates then waited for it. */
|
|
77
|
+
stagedDeferredBytes: number;
|
|
78
|
+
stagedReleaseWaitMs: number;
|
|
71
79
|
uploadBytes: number;
|
|
72
80
|
cacheHits: number;
|
|
73
81
|
cacheMisses: number;
|