doppler-gpu 0.7.7 → 0.7.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/README.md +9 -17
- package/package.json +1 -1
- package/src/config/kernels/kernel-ref-digests.js +2 -1
- package/src/config/kernels/registry.json +47 -19
- package/src/config/schema/execution/kernel-warmup.schema.d.ts +14 -0
- package/src/config/schema/execution/kernel-warmup.schema.js +3 -0
- package/src/config/schema/platform/gpu-cache.schema.js +1 -1
- package/src/config/schema/runtime/buffer-pool.schema.js +2 -2
- package/src/config/schema/runtime/loading.schema.d.ts +5 -0
- package/src/config/schema/runtime/loading.schema.js +25 -2
- package/src/gpu/kernels/moe_route_q4_0_block_f32.wgsl +74 -0
- package/src/gpu/kernels/moe_route_q4_0_split_block_f32.wgsl +35 -18
- package/src/inference/pipelines/text/partitions/partition-execution.js +28 -1
- package/src/inference/pipelines/text/partitions/resident-partition-warmup.js +67 -0
- package/src/inference/pipelines/text/partitions/resident-partition.js +8 -2
- package/src/loader/experts/compact-expert-cache.js +20 -0
- package/src/loader/experts/loader-bindings.js +14 -3
- package/src/loader/experts/mapped-expert-staging.js +393 -0
- package/src/loader/experts/routed-expert-cache.d.ts +8 -0
- package/src/loader/experts/routed-expert-cache.js +12 -11
- package/src/loader/experts/routed-expert-prefill.js +77 -1
- package/src/loader/experts/routed-expert-source.js +1 -0
- package/src/loader/experts/tiered-expert-cache.js +66 -26
- package/src/storage/verified-piece-direct.js +153 -0
- package/src/storage/verified-piece-gather-lane.js +21 -3
- package/src/storage/verified-piece-gather-worker.js +235 -16
- package/src/storage/verified-piece-gather.d.ts +47 -0
- package/src/storage/verified-piece-gather.js +133 -17
- package/src/storage/verified-piece-reader.d.ts +8 -1
- package/src/storage/verified-piece-reader.js +16 -3
- package/src/storage/verified-piece-state-cache.d.ts +75 -0
- package/src/storage/verified-piece-state-cache.js +511 -0
- package/src/storage/verified-piece-storage.d.ts +20 -1
- package/src/storage/verified-piece-storage.js +207 -9
- package/src/version.js +1 -1
|
@@ -4,8 +4,8 @@ import { CompactExpertCache, compactExpertCapacity } from './compact-expert-cach
|
|
|
4
4
|
import { createWeightBuffer } from '../../gpu/weight-buffer.js';
|
|
5
5
|
import { validateQwen35RoutedConfig } from "../../config/schema/runtime/loading.schema.js";
|
|
6
6
|
import { createRoutedExpertCounters, readRoutedExpertRange, assertExpertReadShape, readRoutedExpertPayload } from './routed-expert-source.js';
|
|
7
|
-
|
|
8
|
-
import {
|
|
7
|
+
import { createBatchedDirectPrefillState, runBatchedDirectPrefillLane, writeRoutedPrefillExpert, selectPrefillSeedExperts } from './routed-expert-prefill.js';
|
|
8
|
+
import { destroyMappedExpertStaging } from './mapped-expert-staging.js';
|
|
9
9
|
|
|
10
10
|
// The cache owns individual expert payloads. One reusable logical bank preserves
|
|
11
11
|
// the existing kernels' expert IDs, source-format byte offsets and accumulation order.
|
|
@@ -618,21 +618,23 @@ export class RoutedExpertCache {
|
|
|
618
618
|
submitted = true;
|
|
619
619
|
};
|
|
620
620
|
const hitBatch = this.#copyCachedHits(plan, ids, signal, beforeSubmit);
|
|
621
|
+
const laneCount = this.#options.rangeLease && phase === 'prefill' && this.#options.policy.prefillUploadMode === 'direct-bank'
|
|
622
|
+
? Math.min(this.#options.policy.readConcurrency, this.#options.rangeLease.concurrency ?? 1) : this.#options.policy.readConcurrency;
|
|
623
|
+
const batchedPlan = phase === 'prefill' ? hitBatch.then(copied => createBatchedDirectPrefillState(
|
|
624
|
+
this.#options, this.#stats, plan, ids, copied, this.#loaded, laneCount, expert => this.#frequency?.observe(`${plan.layer}:${expert}`))) : null;
|
|
621
625
|
/** @param {number} reader */
|
|
622
626
|
const work = async (reader) => {
|
|
627
|
+
const batched = batchedPlan ? await batchedPlan : null;
|
|
628
|
+
if (batched) return runBatchedDirectPrefillLane(batched, this.#options, this.#stats, plan, signal, this.#working,
|
|
629
|
+
this.#loaded, () => this.#sampleProfileMemory(), () => { beforeSubmit(); ++this.#submittedSerial; });
|
|
623
630
|
const copiedHits = await hitBatch;
|
|
624
631
|
while (cursor < ids.length) {
|
|
625
632
|
signal.throwIfAborted();
|
|
626
633
|
const expert = ids[cursor++];
|
|
627
634
|
if (copiedHits.has(expert)) continue;
|
|
628
|
-
if (this.#loaded.has(expert)) {
|
|
629
|
-
this.#frequency?.observe(`${plan.layer}:${expert}`);
|
|
630
|
-
this.#stats.workingHits++;
|
|
631
|
-
continue;
|
|
632
|
-
}
|
|
635
|
+
if (this.#loaded.has(expert)) { this.#frequency?.observe(`${plan.layer}:${expert}`); this.#stats.workingHits++; continue; }
|
|
633
636
|
if (phase === 'prefill' && this.#options.policy.prefillUploadMode === 'direct-bank') {
|
|
634
|
-
await this.#writeWorking(plan, expert, signal, beforeSubmit, reader);
|
|
635
|
-
continue;
|
|
637
|
+
await this.#writeWorking(plan, expert, signal, beforeSubmit, reader); continue;
|
|
636
638
|
}
|
|
637
639
|
const entry = await this.#get(plan, expert, signal, beforeSubmit, phase, reader);
|
|
638
640
|
try {
|
|
@@ -647,8 +649,6 @@ export class RoutedExpertCache {
|
|
|
647
649
|
} finally { entry.users--; }
|
|
648
650
|
}
|
|
649
651
|
};
|
|
650
|
-
const laneCount = this.#options.rangeLease && phase === 'prefill' && this.#options.policy.prefillUploadMode === 'direct-bank'
|
|
651
|
-
? Math.min(this.#options.policy.readConcurrency, this.#options.rangeLease.concurrency ?? 1) : this.#options.policy.readConcurrency;
|
|
652
652
|
const outcomes = await Promise.allSettled(Array.from({ length: laneCount }, async (_, reader) => {
|
|
653
653
|
try { await work(reader); } catch (error) { controller.abort(error); throw error; }
|
|
654
654
|
}));
|
|
@@ -706,6 +706,7 @@ export class RoutedExpertCache {
|
|
|
706
706
|
this.#clearReaderStaging();
|
|
707
707
|
this.#hostEntries.clear(); this.#hostSeen.clear(); this.#hostBytes = 0; this.#hostReserved = 0;
|
|
708
708
|
const errors = [];
|
|
709
|
+
try { await destroyMappedExpertStaging(this.#options); } catch (error) { errors.push(error); }
|
|
709
710
|
if (this.#compact) {
|
|
710
711
|
try { await this.#compact.destroy(); this.#compact = null; } catch (error) { errors.push(error); }
|
|
711
712
|
}
|
|
@@ -1,4 +1,80 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { DEFAULT_EXPERT_CACHE_CONFIG } from '../../config/schema/runtime/loading.schema.js';
|
|
2
|
+
import { readRoutedExpertPayload, readRoutedExpertRange, readRoutedExpertsLease, recordRoutedExpertSourceTimings } from './routed-expert-source.js';
|
|
3
|
+
import { getMappedExpertStaging, stageRoutedExperts } from './mapped-expert-staging.js';
|
|
4
|
+
|
|
5
|
+
/** @type {import('./routed-expert-prefill.js').createBatchedDirectPrefillState} */
|
|
6
|
+
export function createBatchedDirectPrefillState(options, stats, plan, ids, copiedHits, loaded, laneCount, onWorkingHit) {
|
|
7
|
+
if (options.policy.prefillUploadMode !== 'direct-bank' || options.policy.sourceReadMode !== 'expert-lease-batch'
|
|
8
|
+
|| typeof options.rangeLease?.readMany !== 'function' || plan.expertBytes > Number(options.rangeLease.maxPartBytes)
|
|
9
|
+
|| plan.planes.some(plane => plane.location.sourceTransform != null || (plane.location.spans && plane.location.spans.length > 1))) {
|
|
10
|
+
return null;
|
|
11
|
+
}
|
|
12
|
+
/** @type {number[]} */
|
|
13
|
+
const misses = [];
|
|
14
|
+
for (const expert of ids) {
|
|
15
|
+
if (copiedHits.has(expert)) continue;
|
|
16
|
+
if (loaded.has(expert)) {
|
|
17
|
+
onWorkingHit(expert);
|
|
18
|
+
stats.workingHits++;
|
|
19
|
+
continue;
|
|
20
|
+
}
|
|
21
|
+
misses.push(expert);
|
|
22
|
+
}
|
|
23
|
+
const requestedBatch = options.policy.expertBatchSize ?? DEFAULT_EXPERT_CACHE_CONFIG.qwen35Routed.expertBatchSize;
|
|
24
|
+
const batchSize = Math.max(1, Math.min(requestedBatch, Math.ceil(misses.length / Math.max(1, laneCount)),
|
|
25
|
+
Math.floor(Number(options.rangeLease?.maxPartBytes) / plan.expertBytes)));
|
|
26
|
+
let cursor = 0;
|
|
27
|
+
return { misses, batchSize, taskCount: Math.ceil(misses.length / batchSize), nextTask: () => cursor++ };
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/** @type {import('./routed-expert-prefill.js').runBatchedDirectPrefillLane} */
|
|
31
|
+
export async function runBatchedDirectPrefillLane(state, options, stats, plan, signal, working, loaded, beforeBatch, beforeUpload) {
|
|
32
|
+
while (true) {
|
|
33
|
+
const task = state.nextTask();
|
|
34
|
+
if (task >= state.taskCount) return;
|
|
35
|
+
signal.throwIfAborted();
|
|
36
|
+
const batch = state.misses.slice(task * state.batchSize, (task + 1) * state.batchSize);
|
|
37
|
+
if (!batch.length) continue;
|
|
38
|
+
beforeBatch();
|
|
39
|
+
stats.cacheMisses += batch.length;
|
|
40
|
+
await writeRoutedPrefillExpertsBatch(options, stats, plan, batch, signal, working, beforeUpload);
|
|
41
|
+
for (const expert of batch) loaded.add(expert);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
/** @type {import('./routed-expert-prefill.js').writeRoutedPrefillExpertsBatch} */
|
|
46
|
+
export async function writeRoutedPrefillExpertsBatch(options, stats, plan, experts, signal, working, beforeUpload) {
|
|
47
|
+
if (!experts.length) return;
|
|
48
|
+
const staging = getMappedExpertStaging(options, plan);
|
|
49
|
+
if (staging && await stageRoutedExperts(staging, stats, plan, experts, (expert, plane) => {
|
|
50
|
+
const target = working.get(plane.role);
|
|
51
|
+
if (!target) throw new Error('Direct prefill requires a complete working bank.');
|
|
52
|
+
return { buffer: target, offset: expert * plane.expertBytes };
|
|
53
|
+
}, signal, beforeUpload, null)) return;
|
|
54
|
+
const start = performance.now();
|
|
55
|
+
const lease = await readRoutedExpertsLease(options, stats, plan, experts, signal, null);
|
|
56
|
+
try {
|
|
57
|
+
for (const [index, expert] of experts.entries()) {
|
|
58
|
+
const payload = lease.bytes.subarray(index * plan.expertBytes, (index + 1) * plan.expertBytes);
|
|
59
|
+
let offset = 0;
|
|
60
|
+
for (const plane of plan.planes) {
|
|
61
|
+
signal.throwIfAborted();
|
|
62
|
+
const target = working.get(plane.role);
|
|
63
|
+
if (!target) throw new Error('Direct prefill requires a complete working bank.');
|
|
64
|
+
beforeUpload();
|
|
65
|
+
const uploadStart = performance.now();
|
|
66
|
+
options.device.queue.writeBuffer(target, expert * plane.expertBytes, payload.subarray(offset, offset + plane.expertBytes));
|
|
67
|
+
stats.uploadEnqueueMs += performance.now() - uploadStart;
|
|
68
|
+
stats.uploadBytes += plane.expertBytes;
|
|
69
|
+
offset += plane.expertBytes;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
} finally {
|
|
73
|
+
// writeBuffer copied the view synchronously; no queued command borrows it.
|
|
74
|
+
lease.release();
|
|
75
|
+
stats.ioWaitMs += performance.now() - start;
|
|
76
|
+
}
|
|
77
|
+
}
|
|
2
78
|
|
|
3
79
|
/** @type {import('./routed-expert-prefill.js').writeRoutedPrefillExpert} */
|
|
4
80
|
export async function writeRoutedPrefillExpert(options, stats, plan, expert, signal, working, scratch, beforeUpload) {
|
|
@@ -9,6 +9,7 @@ export function createRoutedExpertCounters() {
|
|
|
9
9
|
ioWaitMs: 0, sourceReadWaitMs: 0, uploadEnqueueMs: 0, initialQueueIdleForWeightsMs: 0,
|
|
10
10
|
sourceTimingJobs: 0, sourceUntimedJobs: 0, sourceJobWallMs: 0, sourceJobReadWaitMs: 0,
|
|
11
11
|
sourceJobVerificationMs: 0, sourceJobCopyMs: 0, sourceJobRoundTripMs: 0,
|
|
12
|
+
stagedReadAheadBytes: 0, stagedReadWaitMs: 0, stagedVerifyWaitMs: 0, stagedDeferredBytes: 0, stagedReleaseWaitMs: 0,
|
|
12
13
|
hostPrefillAdmissionSkips: 0, hostPrefillAdmissions: 0, hostDecodeAdmissions: 0, hostCacheHits: 0, hostCacheMisses: 0, hostEvictions: 0, hostAdmissionSkips: 0, peakHostOwnedOrReservedBytes: 0 };
|
|
13
14
|
}
|
|
14
15
|
|
|
@@ -3,7 +3,7 @@ import { isDeviceLost, observeDeviceLoss } from '../../memory/device-residency.j
|
|
|
3
3
|
import { CompactExpertCache } from './compact-expert-cache.js';
|
|
4
4
|
import { MappedExpertUploadCache } from './mapped-expert-upload-cache.js';
|
|
5
5
|
import { planTieredExpertPlacement } from './tiered-expert-plan.js';
|
|
6
|
-
import { assertExpertReadShape, createRoutedExpertCounters, readRoutedExpertLease } from './routed-expert-source.js';
|
|
6
|
+
import { assertExpertReadShape, createRoutedExpertCounters, readRoutedExpertLease, readRoutedExpertPayload } from './routed-expert-source.js';
|
|
7
7
|
|
|
8
8
|
// Placement is explicit and model-bound. Setup owns resident banks; streamed
|
|
9
9
|
// layers keep compact slots in both phases, never a second full prefill bank.
|
|
@@ -38,16 +38,24 @@ export class TieredExpertCache {
|
|
|
38
38
|
this.#placement = planTieredExpertPlacement(plans, options.policy, options.device.limits, minimumSlots);
|
|
39
39
|
for (const plan of plans) assertExpertReadShape(options, plan);
|
|
40
40
|
const lease = options.rangeLease;
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
41
|
+
if (options.policy.sourceReadMode === 'expert-parallel') {
|
|
42
|
+
if (lease || !options.readRangeInto
|
|
43
|
+
|| options.policy.stagingBytes < Math.max(...plans.map(plan => plan.expertBytes)) * 2 * options.policy.readConcurrency) {
|
|
44
|
+
throw new Error('Tiered parallel expert reads require bounded direct scratch within stagingBytes.');
|
|
45
|
+
}
|
|
46
|
+
} else {
|
|
47
|
+
const mappedLimit = options.policy.stagingBytes - (lease?.memoryBytes ?? 0);
|
|
48
|
+
const lanes = Math.min(options.policy.readConcurrency, lease?.concurrency ?? 1);
|
|
49
|
+
const leaseLanes = lease?.concurrency ?? 1;
|
|
50
|
+
if (!lease || !Number.isSafeInteger(leaseLanes) || leaseLanes < 1
|
|
51
|
+
|| !Number.isSafeInteger(lease.maxPartBytes) || lease.maxPartBytes < 4 || lease.maxPartBytes % 4
|
|
52
|
+
|| !Number.isSafeInteger(lease.memoryBytes) || !Number.isSafeInteger(lease.maxPartBytes * leaseLanes)
|
|
53
|
+
|| lease.memoryBytes < lease.maxPartBytes * leaseLanes
|
|
54
|
+
|| mappedLimit < Math.max(...plans.map(plan => plan.expertBytes)) * lanes) {
|
|
55
|
+
throw new Error('Tiered expert leases and concurrent mapped writers must fit aggregate stagingBytes.');
|
|
56
|
+
}
|
|
50
57
|
}
|
|
58
|
+
const mappedLimit = options.policy.stagingBytes - (lease?.memoryBytes ?? 0);
|
|
51
59
|
if (options.otherGpuBytes() + this.#placement.cacheUpperBytes > options.policy.gpuBudgetBytes - options.policy.reserveBytes) {
|
|
52
60
|
throw new Error('Tiered expert placement and existing weights exceed the GPU reserve.');
|
|
53
61
|
}
|
|
@@ -179,16 +187,23 @@ export class TieredExpertCache {
|
|
|
179
187
|
signal.addEventListener('abort', abort, { once: true });
|
|
180
188
|
if (signal.aborted) abort();
|
|
181
189
|
try {
|
|
182
|
-
const
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
190
|
+
const directParallel = this.#options.policy.sourceReadMode === 'expert-parallel';
|
|
191
|
+
const concurrency = directParallel
|
|
192
|
+
? this.#options.policy.readConcurrency
|
|
193
|
+
: Math.min(this.#options.policy.readConcurrency, this.#options.rangeLease?.concurrency ?? 1);
|
|
194
|
+
const results = await Promise.allSettled(Array.from({ length: concurrency }, async () => {
|
|
195
|
+
const directScratch = directParallel ? new Uint8Array(plan.expertBytes) : null;
|
|
196
|
+
if (directScratch) {
|
|
197
|
+
this.#uploadingBytes += directScratch.byteLength;
|
|
198
|
+
this.#sampleStaging();
|
|
199
|
+
}
|
|
200
|
+
try {
|
|
201
|
+
while (cursor < plan.numExperts) {
|
|
202
|
+
const expert = cursor++;
|
|
188
203
|
try {
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
204
|
+
if (directScratch) {
|
|
205
|
+
await readRoutedExpertPayload(this.#options, this.#stats, plan, expert, controller.signal, directScratch, null);
|
|
206
|
+
controller.signal.throwIfAborted();
|
|
192
207
|
this.#uploadingBytes += plan.expertBytes;
|
|
193
208
|
this.#sampleStaging();
|
|
194
209
|
try {
|
|
@@ -197,20 +212,45 @@ export class TieredExpertCache {
|
|
|
197
212
|
const buffer = planes.get(plane.role);
|
|
198
213
|
if (!buffer) throw new Error('Tiered resident plane is missing.');
|
|
199
214
|
this.#options.device.queue.writeBuffer(buffer, expert * plane.expertBytes,
|
|
200
|
-
|
|
215
|
+
directScratch.subarray(offset, offset + plane.expertBytes));
|
|
201
216
|
this.#stats.uploadBytes += plane.expertBytes;
|
|
202
217
|
offset += plane.expertBytes;
|
|
203
218
|
}
|
|
204
|
-
// writeBuffer copies its source immediately, but its native
|
|
205
|
-
// upload allocation may survive until queue completion. Keep
|
|
206
|
-
// this lane charged and stop further reads until that upload
|
|
207
|
-
// settles; model-sized setup must not queue model-sized staging.
|
|
208
219
|
await this.#options.device.queue.onSubmittedWorkDone();
|
|
209
220
|
controller.signal.throwIfAborted();
|
|
210
221
|
} finally { this.#uploadingBytes -= plan.expertBytes; }
|
|
222
|
+
} else {
|
|
223
|
+
const lease = await readRoutedExpertLease(this.#options, this.#stats, plan, expert, controller.signal, null);
|
|
224
|
+
try {
|
|
225
|
+
controller.signal.throwIfAborted();
|
|
226
|
+
if (mode === 'parked') await this.#remember(plan, expert, lease.bytes, controller.signal);
|
|
227
|
+
else {
|
|
228
|
+
this.#uploadingBytes += plan.expertBytes;
|
|
229
|
+
this.#sampleStaging();
|
|
230
|
+
try {
|
|
231
|
+
let offset = 0;
|
|
232
|
+
for (const plane of plan.planes) {
|
|
233
|
+
const buffer = planes.get(plane.role);
|
|
234
|
+
if (!buffer) throw new Error('Tiered resident plane is missing.');
|
|
235
|
+
this.#options.device.queue.writeBuffer(buffer, expert * plane.expertBytes,
|
|
236
|
+
lease.bytes.subarray(offset, offset + plane.expertBytes));
|
|
237
|
+
this.#stats.uploadBytes += plane.expertBytes;
|
|
238
|
+
offset += plane.expertBytes;
|
|
239
|
+
}
|
|
240
|
+
// writeBuffer copies its source immediately, but its native
|
|
241
|
+
// upload allocation may survive until queue completion. Keep
|
|
242
|
+
// this lane charged and stop further reads until that upload
|
|
243
|
+
// settles; model-sized setup must not queue model-sized staging.
|
|
244
|
+
await this.#options.device.queue.onSubmittedWorkDone();
|
|
245
|
+
controller.signal.throwIfAborted();
|
|
246
|
+
} finally { this.#uploadingBytes -= plan.expertBytes; }
|
|
247
|
+
}
|
|
248
|
+
} finally { lease.release(); }
|
|
211
249
|
}
|
|
212
|
-
}
|
|
213
|
-
}
|
|
250
|
+
} catch (error) { controller.abort(error); throw error; }
|
|
251
|
+
}
|
|
252
|
+
} finally {
|
|
253
|
+
if (directScratch) this.#uploadingBytes -= directScratch.byteLength;
|
|
214
254
|
}
|
|
215
255
|
}));
|
|
216
256
|
const failure = results.find(result => result.status === 'rejected');
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
/*
|
|
2
|
+
* Node-only direct reads of ranges whose every piece is already verified for the file's current
|
|
3
|
+
* state. The bytes go from the approved file into caller-owned memory on the I/O pool, with no
|
|
4
|
+
* worker hop and no intermediate copy; the caller may pass a mapped GPU staging range.
|
|
5
|
+
*
|
|
6
|
+
* It applies the checks a gather lane applies to a state-verified piece: the open descriptor and
|
|
7
|
+
* the approved pathname must both still be the admitted file (device, inode, size) in the exact
|
|
8
|
+
* state its pieces were verified in (also mtime and ctime), before the read and again after it.
|
|
9
|
+
* A range with any unverified piece, or a file that no longer matches, is declined: first-time
|
|
10
|
+
* verification stays with the lanes, which hash before they publish.
|
|
11
|
+
*
|
|
12
|
+
* A read ahead of verification fills caller memory from the admitted file while a lane hashes the same
|
|
13
|
+
* pieces. It returns a check instead of true. The bytes are unauthenticated until that check passes:
|
|
14
|
+
* every covering piece verified, for the file state the read saw before and after it.
|
|
15
|
+
*/
|
|
16
|
+
import { openSync, closeSync, fstatSync, lstatSync, read as readAsync, constants } from 'node:fs';
|
|
17
|
+
|
|
18
|
+
/** @type {import('./verified-piece-direct.js').createVerifiedDirectReader} */
|
|
19
|
+
export function createVerifiedDirectReader({ files, fileIdentities, index, stateCache }) {
|
|
20
|
+
if (!Number.isInteger(constants.O_NOFOLLOW) || !Number.isInteger(constants.O_NONBLOCK)) {
|
|
21
|
+
throw new Error('Direct verified reads require O_NOFOLLOW and O_NONBLOCK support.');
|
|
22
|
+
}
|
|
23
|
+
const approved = { ...files };
|
|
24
|
+
const identities = structuredClone(fileIdentities);
|
|
25
|
+
/** @type {Map<string, ReadonlyArray<{ offset: number, size: number }>>} */
|
|
26
|
+
const pieces = new Map(index.files.map(file => [file.path, file.pieces]));
|
|
27
|
+
/** @type {Map<string, number>} */
|
|
28
|
+
const handles = new Map();
|
|
29
|
+
/** @type {Set<Promise<unknown>>} */
|
|
30
|
+
const operations = new Set();
|
|
31
|
+
const stats = { reads: 0, bytes: 0, declined: 0, aheadReads: 0, aheadBytes: 0, aheadConfirmed: 0 };
|
|
32
|
+
let closed = false;
|
|
33
|
+
/** @type {Promise<void> | null} */
|
|
34
|
+
let closing = null;
|
|
35
|
+
|
|
36
|
+
/** @param {string} path @param {import('node:fs').BigIntStats} stat */
|
|
37
|
+
const admitted = (path, stat) => {
|
|
38
|
+
const expected = identities[path];
|
|
39
|
+
return !!expected && stat.isFile() && stat.dev.toString() === expected.device
|
|
40
|
+
&& stat.ino.toString() === expected.inode && stat.size.toString() === expected.size;
|
|
41
|
+
};
|
|
42
|
+
/** @param {string} path @param {number} handle */
|
|
43
|
+
const current = (path, handle) => {
|
|
44
|
+
const descriptor = fstatSync(handle, { bigint: true });
|
|
45
|
+
const pathname = lstatSync(approved[path], { bigint: true });
|
|
46
|
+
return admitted(path, descriptor) && admitted(path, pathname)
|
|
47
|
+
&& stateCache.checkFileStat(path, descriptor) && stateCache.checkFileStat(path, pathname);
|
|
48
|
+
};
|
|
49
|
+
/** Every byte of the range lies in a piece verified for the current file state.
|
|
50
|
+
* @param {string} path @param {number} offset @param {number} length */
|
|
51
|
+
const verified = (path, offset, length) => {
|
|
52
|
+
const list = pieces.get(path);
|
|
53
|
+
if (!list) return false;
|
|
54
|
+
let low = 0, high = list.length;
|
|
55
|
+
while (low < high) {
|
|
56
|
+
const mid = (low + high) >>> 1;
|
|
57
|
+
if (list[mid].offset + list[mid].size <= offset) low = mid + 1; else high = mid;
|
|
58
|
+
}
|
|
59
|
+
let end = offset;
|
|
60
|
+
for (let at = low; at < list.length && end < offset + length; at++) {
|
|
61
|
+
const piece = list[at];
|
|
62
|
+
if (piece.offset > end || !stateCache.isPieceVerified(path, piece.offset)) return false;
|
|
63
|
+
end = piece.offset + piece.size;
|
|
64
|
+
}
|
|
65
|
+
return end >= offset + length;
|
|
66
|
+
};
|
|
67
|
+
/** @param {number} handle @param {Uint8Array} target @param {number} at @param {number} length @param {number} position
|
|
68
|
+
* @returns {Promise<number>} */
|
|
69
|
+
const readAt = (handle, target, at, length, position) => new Promise((resolve, reject) => {
|
|
70
|
+
readAsync(handle, target, at, length, position, (error, bytesRead) => error ? reject(error) : resolve(bytesRead));
|
|
71
|
+
});
|
|
72
|
+
|
|
73
|
+
/** @param {string} path @param {number} offset @param {Uint8Array} destination @param {AbortSignal | undefined} signal
|
|
74
|
+
* @param {boolean} ahead @returns {Promise<boolean | (() => boolean)>} */
|
|
75
|
+
async function readOwned(path, offset, destination, signal, ahead) {
|
|
76
|
+
signal?.throwIfAborted();
|
|
77
|
+
if (closed) throw new Error('Gather reader closed.');
|
|
78
|
+
if (!(destination instanceof Uint8Array) || destination.byteLength < 1
|
|
79
|
+
|| !Number.isSafeInteger(offset) || offset < 0 || !Number.isSafeInteger(offset + destination.byteLength)) {
|
|
80
|
+
throw new Error('Direct verified read requires a non-empty destination and a safe range.');
|
|
81
|
+
}
|
|
82
|
+
const length = destination.byteLength;
|
|
83
|
+
if (!Object.hasOwn(approved, path)) { stats.declined++; return false; }
|
|
84
|
+
const already = verified(path, offset, length);
|
|
85
|
+
if (!already && !ahead) { stats.declined++; return false; }
|
|
86
|
+
let handle = handles.get(path);
|
|
87
|
+
if (handle === undefined) {
|
|
88
|
+
handle = openSync(approved[path], constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
|
|
89
|
+
handles.set(path, handle);
|
|
90
|
+
}
|
|
91
|
+
if (!current(path, handle)) { stats.declined++; return false; }
|
|
92
|
+
let received = 0;
|
|
93
|
+
while (received < length) {
|
|
94
|
+
const count = await readAt(handle, destination, received, length - received, offset + received);
|
|
95
|
+
if (count <= 0) throw new Error('Direct verified read truncated.');
|
|
96
|
+
received += count;
|
|
97
|
+
}
|
|
98
|
+
// The destination holds authentic bytes only if the file stayed the verified one throughout.
|
|
99
|
+
if (!current(path, handle) || (already && !verified(path, offset, length))) {
|
|
100
|
+
throw new Error('Gather source changed during a direct verified read.');
|
|
101
|
+
}
|
|
102
|
+
if (closed) throw new Error('Gather reader closed.');
|
|
103
|
+
signal?.throwIfAborted();
|
|
104
|
+
if (already) { stats.reads++; stats.bytes += length; return true; }
|
|
105
|
+
// Read ahead of verification. These bytes came from the admitted file in the state the verification
|
|
106
|
+
// cache records, checked before and after the read. A lane marks a piece verified only for that same
|
|
107
|
+
// state, so once every covering piece is marked and the file still matches, they are the hashed bytes.
|
|
108
|
+
stats.aheadReads++; stats.aheadBytes += length;
|
|
109
|
+
const opened = handle;
|
|
110
|
+
let confirmed = false;
|
|
111
|
+
return () => {
|
|
112
|
+
if (confirmed) return true;
|
|
113
|
+
if (closed || handles.get(path) !== opened || !current(path, opened) || !verified(path, offset, length)) return false;
|
|
114
|
+
confirmed = true; stats.aheadConfirmed++;
|
|
115
|
+
return true;
|
|
116
|
+
};
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/** @type {import('./verified-piece-direct.js').VerifiedDirectReader} */
|
|
120
|
+
const reader = {
|
|
121
|
+
readInto(path, offset, destination, signal) {
|
|
122
|
+
const operation = /** @type {Promise<boolean>} */ (readOwned(path, offset, destination, signal, false));
|
|
123
|
+
operations.add(operation);
|
|
124
|
+
operation.finally(() => operations.delete(operation)).catch(() => {});
|
|
125
|
+
return operation;
|
|
126
|
+
},
|
|
127
|
+
readAheadInto(path, offset, destination, signal) {
|
|
128
|
+
const operation = readOwned(path, offset, destination, signal, true);
|
|
129
|
+
operations.add(operation);
|
|
130
|
+
operation.finally(() => operations.delete(operation)).catch(() => {});
|
|
131
|
+
return operation;
|
|
132
|
+
},
|
|
133
|
+
isVerified: (path, offset, length) => !closed && Object.hasOwn(approved, path)
|
|
134
|
+
&& Number.isSafeInteger(offset) && offset >= 0 && Number.isSafeInteger(length) && length > 0 && verified(path, offset, length),
|
|
135
|
+
getStats: () => ({ ...stats }),
|
|
136
|
+
close() {
|
|
137
|
+
if (closing) return closing;
|
|
138
|
+
closed = true;
|
|
139
|
+
closing = (async () => {
|
|
140
|
+
// A descriptor is never closed under a read that still targets caller memory.
|
|
141
|
+
await Promise.allSettled([...operations]);
|
|
142
|
+
const failures = [];
|
|
143
|
+
for (const [path, handle] of handles) {
|
|
144
|
+
try { closeSync(handle); handles.delete(path); } catch (error) { failures.push(error); }
|
|
145
|
+
}
|
|
146
|
+
if (failures.length) throw new AggregateError(failures, 'Direct verified reader cleanup failed.');
|
|
147
|
+
})();
|
|
148
|
+
closing.catch(() => { closing = null; });
|
|
149
|
+
return closing;
|
|
150
|
+
},
|
|
151
|
+
};
|
|
152
|
+
return Object.freeze(reader);
|
|
153
|
+
}
|
|
@@ -4,7 +4,17 @@
|
|
|
4
4
|
* the caller until release; release is safe after queue.writeBuffer returns.
|
|
5
5
|
*/
|
|
6
6
|
import { snapshotVerifiedPieceJobTimings, snapshotVerifiedRangeLeaseTimings } from './verified-piece-timings.js';
|
|
7
|
-
/**
|
|
7
|
+
/**
|
|
8
|
+
* @param {import('./verified-piece-gather.js').VerifiedPieceGatherOptions & {
|
|
9
|
+
* protectedCache?: unknown,
|
|
10
|
+
* stateCache?: Readonly<import('./verified-piece-gather.js').WorkerStateCachePayload>,
|
|
11
|
+
* workerIndex?: number,
|
|
12
|
+
* workerCount?: number,
|
|
13
|
+
* indexMaxPieceBytes?: number,
|
|
14
|
+
* onStateDirty?: () => void,
|
|
15
|
+
* }} options
|
|
16
|
+
* @returns {ReturnType<import('./verified-piece-gather.js').createVerifiedPieceGather>}
|
|
17
|
+
*/
|
|
8
18
|
export async function createVerifiedPieceGatherLane(options) {
|
|
9
19
|
const { maxPartBytes, memoryBudgetBytes, indexIdentity } = options;
|
|
10
20
|
const maxRanges = options.maxRanges === undefined ? 3 : options.maxRanges;
|
|
@@ -14,8 +24,9 @@ export async function createVerifiedPieceGatherLane(options) {
|
|
|
14
24
|
const manifestBytes = new Uint8Array(options.manifestBytes);
|
|
15
25
|
const indexBytes = new Uint8Array(options.indexBytes);
|
|
16
26
|
const files = { ...options.files };
|
|
17
|
-
|
|
18
|
-
const pieceBytes =
|
|
27
|
+
// The pool already parsed the pinned index; a lane started on its own parses it here.
|
|
28
|
+
const pieceBytes = options.indexMaxPieceBytes === undefined
|
|
29
|
+
? JSON.parse(new TextDecoder().decode(indexBytes)).maxPieceBytes : options.indexMaxPieceBytes;
|
|
19
30
|
if (!Number.isSafeInteger(maxPartBytes) || maxPartBytes < 1
|
|
20
31
|
|| !Number.isSafeInteger(pieceBytes) || pieceBytes < 1
|
|
21
32
|
|| !Number.isSafeInteger(maxPartBytes + 2 * pieceBytes) || !Number.isSafeInteger(memoryBudgetBytes) || memoryBudgetBytes < maxPartBytes + 2 * pieceBytes) {
|
|
@@ -33,6 +44,9 @@ export async function createVerifiedPieceGatherLane(options) {
|
|
|
33
44
|
maxRanges,
|
|
34
45
|
fileIdentities: options.fileIdentities,
|
|
35
46
|
protectedCache: options.protectedCache,
|
|
47
|
+
stateCache: options.stateCache,
|
|
48
|
+
workerIndex: options.workerIndex ?? 0,
|
|
49
|
+
workerCount: options.workerCount ?? 1,
|
|
36
50
|
metadata: { manifestBytes, indexBytes, indexIdentity },
|
|
37
51
|
},
|
|
38
52
|
});
|
|
@@ -54,6 +68,10 @@ export async function createVerifiedPieceGatherLane(options) {
|
|
|
54
68
|
receive = message => { if (message.ready) { failPending = null; resolve(); } };
|
|
55
69
|
});
|
|
56
70
|
worker.on('message', message => {
|
|
71
|
+
if (message?.dirtyState === true) {
|
|
72
|
+
options.onStateDirty?.();
|
|
73
|
+
if (!('buffer' in message) && !('ready' in message) && !('receipt' in message) && !('closed' in message)) return;
|
|
74
|
+
}
|
|
57
75
|
if (closed && message.buffer instanceof ArrayBuffer) { clearOwned(message.buffer); return; }
|
|
58
76
|
receive?.(message);
|
|
59
77
|
});
|