doppler-gpu 0.7.7 → 0.7.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. package/CHANGELOG.md +18 -0
  2. package/README.md +9 -17
  3. package/package.json +1 -1
  4. package/src/config/kernels/kernel-ref-digests.js +2 -1
  5. package/src/config/kernels/registry.json +47 -19
  6. package/src/config/schema/execution/kernel-warmup.schema.d.ts +14 -0
  7. package/src/config/schema/execution/kernel-warmup.schema.js +3 -0
  8. package/src/config/schema/platform/gpu-cache.schema.js +1 -1
  9. package/src/config/schema/runtime/buffer-pool.schema.js +2 -2
  10. package/src/config/schema/runtime/loading.schema.d.ts +5 -0
  11. package/src/config/schema/runtime/loading.schema.js +25 -2
  12. package/src/gpu/kernels/moe_route_q4_0_block_f32.wgsl +74 -0
  13. package/src/gpu/kernels/moe_route_q4_0_split_block_f32.wgsl +35 -18
  14. package/src/inference/pipelines/text/partitions/partition-execution.js +28 -1
  15. package/src/inference/pipelines/text/partitions/resident-partition-warmup.js +67 -0
  16. package/src/inference/pipelines/text/partitions/resident-partition.js +8 -2
  17. package/src/loader/experts/compact-expert-cache.js +20 -0
  18. package/src/loader/experts/loader-bindings.js +14 -3
  19. package/src/loader/experts/mapped-expert-staging.js +393 -0
  20. package/src/loader/experts/routed-expert-cache.d.ts +8 -0
  21. package/src/loader/experts/routed-expert-cache.js +12 -11
  22. package/src/loader/experts/routed-expert-prefill.js +77 -1
  23. package/src/loader/experts/routed-expert-source.js +1 -0
  24. package/src/loader/experts/tiered-expert-cache.js +66 -26
  25. package/src/storage/verified-piece-direct.js +153 -0
  26. package/src/storage/verified-piece-gather-lane.js +21 -3
  27. package/src/storage/verified-piece-gather-worker.js +235 -16
  28. package/src/storage/verified-piece-gather.d.ts +47 -0
  29. package/src/storage/verified-piece-gather.js +133 -17
  30. package/src/storage/verified-piece-reader.d.ts +8 -1
  31. package/src/storage/verified-piece-reader.js +16 -3
  32. package/src/storage/verified-piece-state-cache.d.ts +75 -0
  33. package/src/storage/verified-piece-state-cache.js +511 -0
  34. package/src/storage/verified-piece-storage.d.ts +20 -1
  35. package/src/storage/verified-piece-storage.js +207 -9
  36. package/src/version.js +1 -1
@@ -4,8 +4,8 @@ import { CompactExpertCache, compactExpertCapacity } from './compact-expert-cach
4
4
  import { createWeightBuffer } from '../../gpu/weight-buffer.js';
5
5
  import { validateQwen35RoutedConfig } from "../../config/schema/runtime/loading.schema.js";
6
6
  import { createRoutedExpertCounters, readRoutedExpertRange, assertExpertReadShape, readRoutedExpertPayload } from './routed-expert-source.js';
7
-
8
- import { writeRoutedPrefillExpert, selectPrefillSeedExperts } from './routed-expert-prefill.js';
7
+ import { createBatchedDirectPrefillState, runBatchedDirectPrefillLane, writeRoutedPrefillExpert, selectPrefillSeedExperts } from './routed-expert-prefill.js';
8
+ import { destroyMappedExpertStaging } from './mapped-expert-staging.js';
9
9
 
10
10
  // The cache owns individual expert payloads. One reusable logical bank preserves
11
11
  // the existing kernels' expert IDs, source-format byte offsets and accumulation order.
@@ -618,21 +618,23 @@ export class RoutedExpertCache {
618
618
  submitted = true;
619
619
  };
620
620
  const hitBatch = this.#copyCachedHits(plan, ids, signal, beforeSubmit);
621
+ const laneCount = this.#options.rangeLease && phase === 'prefill' && this.#options.policy.prefillUploadMode === 'direct-bank'
622
+ ? Math.min(this.#options.policy.readConcurrency, this.#options.rangeLease.concurrency ?? 1) : this.#options.policy.readConcurrency;
623
+ const batchedPlan = phase === 'prefill' ? hitBatch.then(copied => createBatchedDirectPrefillState(
624
+ this.#options, this.#stats, plan, ids, copied, this.#loaded, laneCount, expert => this.#frequency?.observe(`${plan.layer}:${expert}`))) : null;
621
625
  /** @param {number} reader */
622
626
  const work = async (reader) => {
627
+ const batched = batchedPlan ? await batchedPlan : null;
628
+ if (batched) return runBatchedDirectPrefillLane(batched, this.#options, this.#stats, plan, signal, this.#working,
629
+ this.#loaded, () => this.#sampleProfileMemory(), () => { beforeSubmit(); ++this.#submittedSerial; });
623
630
  const copiedHits = await hitBatch;
624
631
  while (cursor < ids.length) {
625
632
  signal.throwIfAborted();
626
633
  const expert = ids[cursor++];
627
634
  if (copiedHits.has(expert)) continue;
628
- if (this.#loaded.has(expert)) {
629
- this.#frequency?.observe(`${plan.layer}:${expert}`);
630
- this.#stats.workingHits++;
631
- continue;
632
- }
635
+ if (this.#loaded.has(expert)) { this.#frequency?.observe(`${plan.layer}:${expert}`); this.#stats.workingHits++; continue; }
633
636
  if (phase === 'prefill' && this.#options.policy.prefillUploadMode === 'direct-bank') {
634
- await this.#writeWorking(plan, expert, signal, beforeSubmit, reader);
635
- continue;
637
+ await this.#writeWorking(plan, expert, signal, beforeSubmit, reader); continue;
636
638
  }
637
639
  const entry = await this.#get(plan, expert, signal, beforeSubmit, phase, reader);
638
640
  try {
@@ -647,8 +649,6 @@ export class RoutedExpertCache {
647
649
  } finally { entry.users--; }
648
650
  }
649
651
  };
650
- const laneCount = this.#options.rangeLease && phase === 'prefill' && this.#options.policy.prefillUploadMode === 'direct-bank'
651
- ? Math.min(this.#options.policy.readConcurrency, this.#options.rangeLease.concurrency ?? 1) : this.#options.policy.readConcurrency;
652
652
  const outcomes = await Promise.allSettled(Array.from({ length: laneCount }, async (_, reader) => {
653
653
  try { await work(reader); } catch (error) { controller.abort(error); throw error; }
654
654
  }));
@@ -706,6 +706,7 @@ export class RoutedExpertCache {
706
706
  this.#clearReaderStaging();
707
707
  this.#hostEntries.clear(); this.#hostSeen.clear(); this.#hostBytes = 0; this.#hostReserved = 0;
708
708
  const errors = [];
709
+ try { await destroyMappedExpertStaging(this.#options); } catch (error) { errors.push(error); }
709
710
  if (this.#compact) {
710
711
  try { await this.#compact.destroy(); this.#compact = null; } catch (error) { errors.push(error); }
711
712
  }
@@ -1,4 +1,80 @@
1
- import { readRoutedExpertPayload, readRoutedExpertRange, recordRoutedExpertSourceTimings } from './routed-expert-source.js';
1
+ import { DEFAULT_EXPERT_CACHE_CONFIG } from '../../config/schema/runtime/loading.schema.js';
2
+ import { readRoutedExpertPayload, readRoutedExpertRange, readRoutedExpertsLease, recordRoutedExpertSourceTimings } from './routed-expert-source.js';
3
+ import { getMappedExpertStaging, stageRoutedExperts } from './mapped-expert-staging.js';
4
+
5
+ /** @type {import('./routed-expert-prefill.js').createBatchedDirectPrefillState} */
6
+ export function createBatchedDirectPrefillState(options, stats, plan, ids, copiedHits, loaded, laneCount, onWorkingHit) {
7
+ if (options.policy.prefillUploadMode !== 'direct-bank' || options.policy.sourceReadMode !== 'expert-lease-batch'
8
+ || typeof options.rangeLease?.readMany !== 'function' || plan.expertBytes > Number(options.rangeLease.maxPartBytes)
9
+ || plan.planes.some(plane => plane.location.sourceTransform != null || (plane.location.spans && plane.location.spans.length > 1))) {
10
+ return null;
11
+ }
12
+ /** @type {number[]} */
13
+ const misses = [];
14
+ for (const expert of ids) {
15
+ if (copiedHits.has(expert)) continue;
16
+ if (loaded.has(expert)) {
17
+ onWorkingHit(expert);
18
+ stats.workingHits++;
19
+ continue;
20
+ }
21
+ misses.push(expert);
22
+ }
23
+ const requestedBatch = options.policy.expertBatchSize ?? DEFAULT_EXPERT_CACHE_CONFIG.qwen35Routed.expertBatchSize;
24
+ const batchSize = Math.max(1, Math.min(requestedBatch, Math.ceil(misses.length / Math.max(1, laneCount)),
25
+ Math.floor(Number(options.rangeLease?.maxPartBytes) / plan.expertBytes)));
26
+ let cursor = 0;
27
+ return { misses, batchSize, taskCount: Math.ceil(misses.length / batchSize), nextTask: () => cursor++ };
28
+ }
29
+
30
+ /** @type {import('./routed-expert-prefill.js').runBatchedDirectPrefillLane} */
31
+ export async function runBatchedDirectPrefillLane(state, options, stats, plan, signal, working, loaded, beforeBatch, beforeUpload) {
32
+ while (true) {
33
+ const task = state.nextTask();
34
+ if (task >= state.taskCount) return;
35
+ signal.throwIfAborted();
36
+ const batch = state.misses.slice(task * state.batchSize, (task + 1) * state.batchSize);
37
+ if (!batch.length) continue;
38
+ beforeBatch();
39
+ stats.cacheMisses += batch.length;
40
+ await writeRoutedPrefillExpertsBatch(options, stats, plan, batch, signal, working, beforeUpload);
41
+ for (const expert of batch) loaded.add(expert);
42
+ }
43
+ }
44
+
45
+ /** @type {import('./routed-expert-prefill.js').writeRoutedPrefillExpertsBatch} */
46
+ export async function writeRoutedPrefillExpertsBatch(options, stats, plan, experts, signal, working, beforeUpload) {
47
+ if (!experts.length) return;
48
+ const staging = getMappedExpertStaging(options, plan);
49
+ if (staging && await stageRoutedExperts(staging, stats, plan, experts, (expert, plane) => {
50
+ const target = working.get(plane.role);
51
+ if (!target) throw new Error('Direct prefill requires a complete working bank.');
52
+ return { buffer: target, offset: expert * plane.expertBytes };
53
+ }, signal, beforeUpload, null)) return;
54
+ const start = performance.now();
55
+ const lease = await readRoutedExpertsLease(options, stats, plan, experts, signal, null);
56
+ try {
57
+ for (const [index, expert] of experts.entries()) {
58
+ const payload = lease.bytes.subarray(index * plan.expertBytes, (index + 1) * plan.expertBytes);
59
+ let offset = 0;
60
+ for (const plane of plan.planes) {
61
+ signal.throwIfAborted();
62
+ const target = working.get(plane.role);
63
+ if (!target) throw new Error('Direct prefill requires a complete working bank.');
64
+ beforeUpload();
65
+ const uploadStart = performance.now();
66
+ options.device.queue.writeBuffer(target, expert * plane.expertBytes, payload.subarray(offset, offset + plane.expertBytes));
67
+ stats.uploadEnqueueMs += performance.now() - uploadStart;
68
+ stats.uploadBytes += plane.expertBytes;
69
+ offset += plane.expertBytes;
70
+ }
71
+ }
72
+ } finally {
73
+ // writeBuffer copied the view synchronously; no queued command borrows it.
74
+ lease.release();
75
+ stats.ioWaitMs += performance.now() - start;
76
+ }
77
+ }
2
78
 
3
79
  /** @type {import('./routed-expert-prefill.js').writeRoutedPrefillExpert} */
4
80
  export async function writeRoutedPrefillExpert(options, stats, plan, expert, signal, working, scratch, beforeUpload) {
@@ -9,6 +9,7 @@ export function createRoutedExpertCounters() {
9
9
  ioWaitMs: 0, sourceReadWaitMs: 0, uploadEnqueueMs: 0, initialQueueIdleForWeightsMs: 0,
10
10
  sourceTimingJobs: 0, sourceUntimedJobs: 0, sourceJobWallMs: 0, sourceJobReadWaitMs: 0,
11
11
  sourceJobVerificationMs: 0, sourceJobCopyMs: 0, sourceJobRoundTripMs: 0,
12
+ stagedReadAheadBytes: 0, stagedReadWaitMs: 0, stagedVerifyWaitMs: 0, stagedDeferredBytes: 0, stagedReleaseWaitMs: 0,
12
13
  hostPrefillAdmissionSkips: 0, hostPrefillAdmissions: 0, hostDecodeAdmissions: 0, hostCacheHits: 0, hostCacheMisses: 0, hostEvictions: 0, hostAdmissionSkips: 0, peakHostOwnedOrReservedBytes: 0 };
13
14
  }
14
15
 
@@ -3,7 +3,7 @@ import { isDeviceLost, observeDeviceLoss } from '../../memory/device-residency.j
3
3
  import { CompactExpertCache } from './compact-expert-cache.js';
4
4
  import { MappedExpertUploadCache } from './mapped-expert-upload-cache.js';
5
5
  import { planTieredExpertPlacement } from './tiered-expert-plan.js';
6
- import { assertExpertReadShape, createRoutedExpertCounters, readRoutedExpertLease } from './routed-expert-source.js';
6
+ import { assertExpertReadShape, createRoutedExpertCounters, readRoutedExpertLease, readRoutedExpertPayload } from './routed-expert-source.js';
7
7
 
8
8
  // Placement is explicit and model-bound. Setup owns resident banks; streamed
9
9
  // layers keep compact slots in both phases, never a second full prefill bank.
@@ -38,16 +38,24 @@ export class TieredExpertCache {
38
38
  this.#placement = planTieredExpertPlacement(plans, options.policy, options.device.limits, minimumSlots);
39
39
  for (const plan of plans) assertExpertReadShape(options, plan);
40
40
  const lease = options.rangeLease;
41
- const mappedLimit = options.policy.stagingBytes - (lease?.memoryBytes ?? 0);
42
- const lanes = Math.min(options.policy.readConcurrency, lease?.concurrency ?? 1);
43
- const leaseLanes = lease?.concurrency ?? 1;
44
- if (!lease || !Number.isSafeInteger(leaseLanes) || leaseLanes < 1
45
- || !Number.isSafeInteger(lease.maxPartBytes) || lease.maxPartBytes < 4 || lease.maxPartBytes % 4
46
- || !Number.isSafeInteger(lease.memoryBytes) || !Number.isSafeInteger(lease.maxPartBytes * leaseLanes)
47
- || lease.memoryBytes < lease.maxPartBytes * leaseLanes
48
- || mappedLimit < Math.max(...plans.map(plan => plan.expertBytes)) * lanes) {
49
- throw new Error('Tiered expert leases and concurrent mapped writers must fit aggregate stagingBytes.');
41
+ if (options.policy.sourceReadMode === 'expert-parallel') {
42
+ if (lease || !options.readRangeInto
43
+ || options.policy.stagingBytes < Math.max(...plans.map(plan => plan.expertBytes)) * 2 * options.policy.readConcurrency) {
44
+ throw new Error('Tiered parallel expert reads require bounded direct scratch within stagingBytes.');
45
+ }
46
+ } else {
47
+ const mappedLimit = options.policy.stagingBytes - (lease?.memoryBytes ?? 0);
48
+ const lanes = Math.min(options.policy.readConcurrency, lease?.concurrency ?? 1);
49
+ const leaseLanes = lease?.concurrency ?? 1;
50
+ if (!lease || !Number.isSafeInteger(leaseLanes) || leaseLanes < 1
51
+ || !Number.isSafeInteger(lease.maxPartBytes) || lease.maxPartBytes < 4 || lease.maxPartBytes % 4
52
+ || !Number.isSafeInteger(lease.memoryBytes) || !Number.isSafeInteger(lease.maxPartBytes * leaseLanes)
53
+ || lease.memoryBytes < lease.maxPartBytes * leaseLanes
54
+ || mappedLimit < Math.max(...plans.map(plan => plan.expertBytes)) * lanes) {
55
+ throw new Error('Tiered expert leases and concurrent mapped writers must fit aggregate stagingBytes.');
56
+ }
50
57
  }
58
+ const mappedLimit = options.policy.stagingBytes - (lease?.memoryBytes ?? 0);
51
59
  if (options.otherGpuBytes() + this.#placement.cacheUpperBytes > options.policy.gpuBudgetBytes - options.policy.reserveBytes) {
52
60
  throw new Error('Tiered expert placement and existing weights exceed the GPU reserve.');
53
61
  }
@@ -179,16 +187,23 @@ export class TieredExpertCache {
179
187
  signal.addEventListener('abort', abort, { once: true });
180
188
  if (signal.aborted) abort();
181
189
  try {
182
- const results = await Promise.allSettled(Array.from({ length: Math.min(this.#options.policy.readConcurrency,
183
- this.#options.rangeLease?.concurrency ?? 1) }, async () => {
184
- while (cursor < plan.numExperts) {
185
- const expert = cursor++;
186
- try {
187
- const lease = await readRoutedExpertLease(this.#options, this.#stats, plan, expert, controller.signal, null);
190
+ const directParallel = this.#options.policy.sourceReadMode === 'expert-parallel';
191
+ const concurrency = directParallel
192
+ ? this.#options.policy.readConcurrency
193
+ : Math.min(this.#options.policy.readConcurrency, this.#options.rangeLease?.concurrency ?? 1);
194
+ const results = await Promise.allSettled(Array.from({ length: concurrency }, async () => {
195
+ const directScratch = directParallel ? new Uint8Array(plan.expertBytes) : null;
196
+ if (directScratch) {
197
+ this.#uploadingBytes += directScratch.byteLength;
198
+ this.#sampleStaging();
199
+ }
200
+ try {
201
+ while (cursor < plan.numExperts) {
202
+ const expert = cursor++;
188
203
  try {
189
- controller.signal.throwIfAborted();
190
- if (mode === 'parked') await this.#remember(plan, expert, lease.bytes, controller.signal);
191
- else {
204
+ if (directScratch) {
205
+ await readRoutedExpertPayload(this.#options, this.#stats, plan, expert, controller.signal, directScratch, null);
206
+ controller.signal.throwIfAborted();
192
207
  this.#uploadingBytes += plan.expertBytes;
193
208
  this.#sampleStaging();
194
209
  try {
@@ -197,20 +212,45 @@ export class TieredExpertCache {
197
212
  const buffer = planes.get(plane.role);
198
213
  if (!buffer) throw new Error('Tiered resident plane is missing.');
199
214
  this.#options.device.queue.writeBuffer(buffer, expert * plane.expertBytes,
200
- lease.bytes.subarray(offset, offset + plane.expertBytes));
215
+ directScratch.subarray(offset, offset + plane.expertBytes));
201
216
  this.#stats.uploadBytes += plane.expertBytes;
202
217
  offset += plane.expertBytes;
203
218
  }
204
- // writeBuffer copies its source immediately, but its native
205
- // upload allocation may survive until queue completion. Keep
206
- // this lane charged and stop further reads until that upload
207
- // settles; model-sized setup must not queue model-sized staging.
208
219
  await this.#options.device.queue.onSubmittedWorkDone();
209
220
  controller.signal.throwIfAborted();
210
221
  } finally { this.#uploadingBytes -= plan.expertBytes; }
222
+ } else {
223
+ const lease = await readRoutedExpertLease(this.#options, this.#stats, plan, expert, controller.signal, null);
224
+ try {
225
+ controller.signal.throwIfAborted();
226
+ if (mode === 'parked') await this.#remember(plan, expert, lease.bytes, controller.signal);
227
+ else {
228
+ this.#uploadingBytes += plan.expertBytes;
229
+ this.#sampleStaging();
230
+ try {
231
+ let offset = 0;
232
+ for (const plane of plan.planes) {
233
+ const buffer = planes.get(plane.role);
234
+ if (!buffer) throw new Error('Tiered resident plane is missing.');
235
+ this.#options.device.queue.writeBuffer(buffer, expert * plane.expertBytes,
236
+ lease.bytes.subarray(offset, offset + plane.expertBytes));
237
+ this.#stats.uploadBytes += plane.expertBytes;
238
+ offset += plane.expertBytes;
239
+ }
240
+ // writeBuffer copies its source immediately, but its native
241
+ // upload allocation may survive until queue completion. Keep
242
+ // this lane charged and stop further reads until that upload
243
+ // settles; model-sized setup must not queue model-sized staging.
244
+ await this.#options.device.queue.onSubmittedWorkDone();
245
+ controller.signal.throwIfAborted();
246
+ } finally { this.#uploadingBytes -= plan.expertBytes; }
247
+ }
248
+ } finally { lease.release(); }
211
249
  }
212
- } finally { lease.release(); }
213
- } catch (error) { controller.abort(error); throw error; }
250
+ } catch (error) { controller.abort(error); throw error; }
251
+ }
252
+ } finally {
253
+ if (directScratch) this.#uploadingBytes -= directScratch.byteLength;
214
254
  }
215
255
  }));
216
256
  const failure = results.find(result => result.status === 'rejected');
@@ -0,0 +1,153 @@
1
+ /*
2
+ * Node-only direct reads of ranges whose every piece is already verified for the file's current
3
+ * state. The bytes go from the approved file into caller-owned memory on the I/O pool, with no
4
+ * worker hop and no intermediate copy; the caller may pass a mapped GPU staging range.
5
+ *
6
+ * It applies the checks a gather lane applies to a state-verified piece: the open descriptor and
7
+ * the approved pathname must both still be the admitted file (device, inode, size) in the exact
8
+ * state its pieces were verified in (also mtime and ctime), before the read and again after it.
9
+ * A range with any unverified piece, or a file that no longer matches, is declined: first-time
10
+ * verification stays with the lanes, which hash before they publish.
11
+ *
12
+ * A read ahead of verification fills caller memory from the admitted file while a lane hashes the same
13
+ * pieces. It returns a check instead of true. The bytes are unauthenticated until that check passes:
14
+ * every covering piece verified, for the file state the read saw before and after it.
15
+ */
16
+ import { openSync, closeSync, fstatSync, lstatSync, read as readAsync, constants } from 'node:fs';
17
+
18
+ /** @type {import('./verified-piece-direct.js').createVerifiedDirectReader} */
19
+ export function createVerifiedDirectReader({ files, fileIdentities, index, stateCache }) {
20
+ if (!Number.isInteger(constants.O_NOFOLLOW) || !Number.isInteger(constants.O_NONBLOCK)) {
21
+ throw new Error('Direct verified reads require O_NOFOLLOW and O_NONBLOCK support.');
22
+ }
23
+ const approved = { ...files };
24
+ const identities = structuredClone(fileIdentities);
25
+ /** @type {Map<string, ReadonlyArray<{ offset: number, size: number }>>} */
26
+ const pieces = new Map(index.files.map(file => [file.path, file.pieces]));
27
+ /** @type {Map<string, number>} */
28
+ const handles = new Map();
29
+ /** @type {Set<Promise<unknown>>} */
30
+ const operations = new Set();
31
+ const stats = { reads: 0, bytes: 0, declined: 0, aheadReads: 0, aheadBytes: 0, aheadConfirmed: 0 };
32
+ let closed = false;
33
+ /** @type {Promise<void> | null} */
34
+ let closing = null;
35
+
36
+ /** @param {string} path @param {import('node:fs').BigIntStats} stat */
37
+ const admitted = (path, stat) => {
38
+ const expected = identities[path];
39
+ return !!expected && stat.isFile() && stat.dev.toString() === expected.device
40
+ && stat.ino.toString() === expected.inode && stat.size.toString() === expected.size;
41
+ };
42
+ /** @param {string} path @param {number} handle */
43
+ const current = (path, handle) => {
44
+ const descriptor = fstatSync(handle, { bigint: true });
45
+ const pathname = lstatSync(approved[path], { bigint: true });
46
+ return admitted(path, descriptor) && admitted(path, pathname)
47
+ && stateCache.checkFileStat(path, descriptor) && stateCache.checkFileStat(path, pathname);
48
+ };
49
+ /** Every byte of the range lies in a piece verified for the current file state.
50
+ * @param {string} path @param {number} offset @param {number} length */
51
+ const verified = (path, offset, length) => {
52
+ const list = pieces.get(path);
53
+ if (!list) return false;
54
+ let low = 0, high = list.length;
55
+ while (low < high) {
56
+ const mid = (low + high) >>> 1;
57
+ if (list[mid].offset + list[mid].size <= offset) low = mid + 1; else high = mid;
58
+ }
59
+ let end = offset;
60
+ for (let at = low; at < list.length && end < offset + length; at++) {
61
+ const piece = list[at];
62
+ if (piece.offset > end || !stateCache.isPieceVerified(path, piece.offset)) return false;
63
+ end = piece.offset + piece.size;
64
+ }
65
+ return end >= offset + length;
66
+ };
67
+ /** @param {number} handle @param {Uint8Array} target @param {number} at @param {number} length @param {number} position
68
+ * @returns {Promise<number>} */
69
+ const readAt = (handle, target, at, length, position) => new Promise((resolve, reject) => {
70
+ readAsync(handle, target, at, length, position, (error, bytesRead) => error ? reject(error) : resolve(bytesRead));
71
+ });
72
+
73
+ /** @param {string} path @param {number} offset @param {Uint8Array} destination @param {AbortSignal | undefined} signal
74
+ * @param {boolean} ahead @returns {Promise<boolean | (() => boolean)>} */
75
+ async function readOwned(path, offset, destination, signal, ahead) {
76
+ signal?.throwIfAborted();
77
+ if (closed) throw new Error('Gather reader closed.');
78
+ if (!(destination instanceof Uint8Array) || destination.byteLength < 1
79
+ || !Number.isSafeInteger(offset) || offset < 0 || !Number.isSafeInteger(offset + destination.byteLength)) {
80
+ throw new Error('Direct verified read requires a non-empty destination and a safe range.');
81
+ }
82
+ const length = destination.byteLength;
83
+ if (!Object.hasOwn(approved, path)) { stats.declined++; return false; }
84
+ const already = verified(path, offset, length);
85
+ if (!already && !ahead) { stats.declined++; return false; }
86
+ let handle = handles.get(path);
87
+ if (handle === undefined) {
88
+ handle = openSync(approved[path], constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
89
+ handles.set(path, handle);
90
+ }
91
+ if (!current(path, handle)) { stats.declined++; return false; }
92
+ let received = 0;
93
+ while (received < length) {
94
+ const count = await readAt(handle, destination, received, length - received, offset + received);
95
+ if (count <= 0) throw new Error('Direct verified read truncated.');
96
+ received += count;
97
+ }
98
+ // The destination holds authentic bytes only if the file stayed the verified one throughout.
99
+ if (!current(path, handle) || (already && !verified(path, offset, length))) {
100
+ throw new Error('Gather source changed during a direct verified read.');
101
+ }
102
+ if (closed) throw new Error('Gather reader closed.');
103
+ signal?.throwIfAborted();
104
+ if (already) { stats.reads++; stats.bytes += length; return true; }
105
+ // Read ahead of verification. These bytes came from the admitted file in the state the verification
106
+ // cache records, checked before and after the read. A lane marks a piece verified only for that same
107
+ // state, so once every covering piece is marked and the file still matches, they are the hashed bytes.
108
+ stats.aheadReads++; stats.aheadBytes += length;
109
+ const opened = handle;
110
+ let confirmed = false;
111
+ return () => {
112
+ if (confirmed) return true;
113
+ if (closed || handles.get(path) !== opened || !current(path, opened) || !verified(path, offset, length)) return false;
114
+ confirmed = true; stats.aheadConfirmed++;
115
+ return true;
116
+ };
117
+ }
118
+
119
+ /** @type {import('./verified-piece-direct.js').VerifiedDirectReader} */
120
+ const reader = {
121
+ readInto(path, offset, destination, signal) {
122
+ const operation = /** @type {Promise<boolean>} */ (readOwned(path, offset, destination, signal, false));
123
+ operations.add(operation);
124
+ operation.finally(() => operations.delete(operation)).catch(() => {});
125
+ return operation;
126
+ },
127
+ readAheadInto(path, offset, destination, signal) {
128
+ const operation = readOwned(path, offset, destination, signal, true);
129
+ operations.add(operation);
130
+ operation.finally(() => operations.delete(operation)).catch(() => {});
131
+ return operation;
132
+ },
133
+ isVerified: (path, offset, length) => !closed && Object.hasOwn(approved, path)
134
+ && Number.isSafeInteger(offset) && offset >= 0 && Number.isSafeInteger(length) && length > 0 && verified(path, offset, length),
135
+ getStats: () => ({ ...stats }),
136
+ close() {
137
+ if (closing) return closing;
138
+ closed = true;
139
+ closing = (async () => {
140
+ // A descriptor is never closed under a read that still targets caller memory.
141
+ await Promise.allSettled([...operations]);
142
+ const failures = [];
143
+ for (const [path, handle] of handles) {
144
+ try { closeSync(handle); handles.delete(path); } catch (error) { failures.push(error); }
145
+ }
146
+ if (failures.length) throw new AggregateError(failures, 'Direct verified reader cleanup failed.');
147
+ })();
148
+ closing.catch(() => { closing = null; });
149
+ return closing;
150
+ },
151
+ };
152
+ return Object.freeze(reader);
153
+ }
@@ -4,7 +4,17 @@
4
4
  * the caller until release; release is safe after queue.writeBuffer returns.
5
5
  */
6
6
  import { snapshotVerifiedPieceJobTimings, snapshotVerifiedRangeLeaseTimings } from './verified-piece-timings.js';
7
- /** @type {import('./verified-piece-gather.js').createVerifiedPieceGather} */
7
+ /**
8
+ * @param {import('./verified-piece-gather.js').VerifiedPieceGatherOptions & {
9
+ * protectedCache?: unknown,
10
+ * stateCache?: Readonly<import('./verified-piece-gather.js').WorkerStateCachePayload>,
11
+ * workerIndex?: number,
12
+ * workerCount?: number,
13
+ * indexMaxPieceBytes?: number,
14
+ * onStateDirty?: () => void,
15
+ * }} options
16
+ * @returns {ReturnType<import('./verified-piece-gather.js').createVerifiedPieceGather>}
17
+ */
8
18
  export async function createVerifiedPieceGatherLane(options) {
9
19
  const { maxPartBytes, memoryBudgetBytes, indexIdentity } = options;
10
20
  const maxRanges = options.maxRanges === undefined ? 3 : options.maxRanges;
@@ -14,8 +24,9 @@ export async function createVerifiedPieceGatherLane(options) {
14
24
  const manifestBytes = new Uint8Array(options.manifestBytes);
15
25
  const indexBytes = new Uint8Array(options.indexBytes);
16
26
  const files = { ...options.files };
17
- const index = JSON.parse(new TextDecoder().decode(indexBytes));
18
- const pieceBytes = index.maxPieceBytes;
27
+ // The pool already parsed the pinned index; a lane started on its own parses it here.
28
+ const pieceBytes = options.indexMaxPieceBytes === undefined
29
+ ? JSON.parse(new TextDecoder().decode(indexBytes)).maxPieceBytes : options.indexMaxPieceBytes;
19
30
  if (!Number.isSafeInteger(maxPartBytes) || maxPartBytes < 1
20
31
  || !Number.isSafeInteger(pieceBytes) || pieceBytes < 1
21
32
  || !Number.isSafeInteger(maxPartBytes + 2 * pieceBytes) || !Number.isSafeInteger(memoryBudgetBytes) || memoryBudgetBytes < maxPartBytes + 2 * pieceBytes) {
@@ -33,6 +44,9 @@ export async function createVerifiedPieceGatherLane(options) {
33
44
  maxRanges,
34
45
  fileIdentities: options.fileIdentities,
35
46
  protectedCache: options.protectedCache,
47
+ stateCache: options.stateCache,
48
+ workerIndex: options.workerIndex ?? 0,
49
+ workerCount: options.workerCount ?? 1,
36
50
  metadata: { manifestBytes, indexBytes, indexIdentity },
37
51
  },
38
52
  });
@@ -54,6 +68,10 @@ export async function createVerifiedPieceGatherLane(options) {
54
68
  receive = message => { if (message.ready) { failPending = null; resolve(); } };
55
69
  });
56
70
  worker.on('message', message => {
71
+ if (message?.dirtyState === true) {
72
+ options.onStateDirty?.();
73
+ if (!('buffer' in message) && !('ready' in message) && !('receipt' in message) && !('closed' in message)) return;
74
+ }
57
75
  if (closed && message.buffer instanceof ArrayBuffer) { clearOwned(message.buffer); return; }
58
76
  receive?.(message);
59
77
  });