doppler-gpu 0.7.7 → 0.7.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. package/CHANGELOG.md +18 -0
  2. package/README.md +9 -17
  3. package/package.json +1 -1
  4. package/src/config/kernels/kernel-ref-digests.js +2 -1
  5. package/src/config/kernels/registry.json +47 -19
  6. package/src/config/schema/execution/kernel-warmup.schema.d.ts +14 -0
  7. package/src/config/schema/execution/kernel-warmup.schema.js +3 -0
  8. package/src/config/schema/platform/gpu-cache.schema.js +1 -1
  9. package/src/config/schema/runtime/buffer-pool.schema.js +2 -2
  10. package/src/config/schema/runtime/loading.schema.d.ts +5 -0
  11. package/src/config/schema/runtime/loading.schema.js +25 -2
  12. package/src/gpu/kernels/moe_route_q4_0_block_f32.wgsl +74 -0
  13. package/src/gpu/kernels/moe_route_q4_0_split_block_f32.wgsl +35 -18
  14. package/src/inference/pipelines/text/partitions/partition-execution.js +28 -1
  15. package/src/inference/pipelines/text/partitions/resident-partition-warmup.js +67 -0
  16. package/src/inference/pipelines/text/partitions/resident-partition.js +8 -2
  17. package/src/loader/experts/compact-expert-cache.js +20 -0
  18. package/src/loader/experts/loader-bindings.js +14 -3
  19. package/src/loader/experts/mapped-expert-staging.js +393 -0
  20. package/src/loader/experts/routed-expert-cache.d.ts +8 -0
  21. package/src/loader/experts/routed-expert-cache.js +12 -11
  22. package/src/loader/experts/routed-expert-prefill.js +77 -1
  23. package/src/loader/experts/routed-expert-source.js +1 -0
  24. package/src/loader/experts/tiered-expert-cache.js +66 -26
  25. package/src/storage/verified-piece-direct.js +153 -0
  26. package/src/storage/verified-piece-gather-lane.js +21 -3
  27. package/src/storage/verified-piece-gather-worker.js +235 -16
  28. package/src/storage/verified-piece-gather.d.ts +47 -0
  29. package/src/storage/verified-piece-gather.js +133 -17
  30. package/src/storage/verified-piece-reader.d.ts +8 -1
  31. package/src/storage/verified-piece-reader.js +16 -3
  32. package/src/storage/verified-piece-state-cache.d.ts +75 -0
  33. package/src/storage/verified-piece-state-cache.js +511 -0
  34. package/src/storage/verified-piece-storage.d.ts +20 -1
  35. package/src/storage/verified-piece-storage.js +207 -9
  36. package/src/version.js +1 -1
@@ -6,6 +6,7 @@ import {
6
6
  import { createHash } from 'node:crypto';
7
7
  import { join } from 'node:path';
8
8
  import { createVerifiedPieceReader } from './verified-piece-reader.js';
9
+ import { matchesFileStateSnapshot } from './verified-piece-state-cache.js';
9
10
  import { snapshotVerifiedPieceJobTimings } from './verified-piece-timings.js';
10
11
 
11
12
  // This worker owns acquisition, complete-piece authentication and range gathering.
@@ -101,7 +102,84 @@ let activeGenericScratchBytes = 0, peakGenericScratchBytes = 0, genericRequested
101
102
  let protectedCacheHits = 0, protectedCacheHitBytes = 0, protectedCacheMisses = 0;
102
103
  let protectedCacheAdmissions = 0, protectedCacheAdmittedBytes = 0;
103
104
  let protectedCacheEvictions = 0, protectedCacheEvictedBytes = 0, protectedCacheSkips = 0;
105
+ let stateCacheHits = 0, stateCacheHitBytes = 0, stateCacheMisses = 0;
106
+ let stateCacheAdmissions = 0, stateCacheAdmittedBytes = 0;
107
+ let backgroundVerifiedPieces = 0, backgroundVerificationMs = 0;
108
+ let pendingDirtyState = false;
104
109
  const protectedCache = workerData.protectedCache ?? null;
110
+ const stateCacheConfig = workerData.stateCache ?? null;
111
+ const parsedIndex = JSON.parse(new TextDecoder().decode(workerData.metadata.indexBytes));
112
+ /** @type {Map<string, { canonicalPath: string, snapshot: import('./verified-piece-state-cache.js').FileStateSnapshot, wordCount: number, words: Int32Array, pieces: Array<{ offset: number, size: number, identity: string }> }>} */
113
+ const stateFiles = new Map();
114
+ if (stateCacheConfig?.files) {
115
+ for (const file of parsedIndex.files) {
116
+ const desc = stateCacheConfig.files[file.path];
117
+ if (!desc) continue;
118
+ stateFiles.set(file.path, {
119
+ canonicalPath: desc.canonicalPath,
120
+ snapshot: desc.snapshot,
121
+ wordCount: desc.wordCount,
122
+ words: new Int32Array(desc.sharedBuffer),
123
+ pieces: file.pieces,
124
+ });
125
+ }
126
+ }
127
+
128
+ /** @param {Array<{ offset: number, size: number }>} pieces @param {number} offset */
129
+ function findStatePieceIndex(pieces, offset) {
130
+ let low = 0, high = pieces.length;
131
+ while (low < high) {
132
+ const mid = (low + high) >>> 1;
133
+ const p = pieces[mid];
134
+ if (p.offset + p.size <= offset) low = mid + 1;
135
+ else high = mid;
136
+ }
137
+ return low < pieces.length && pieces[low].offset === offset ? low : -1;
138
+ }
139
+
140
+ /** @param {string} path @param {number} handle */
141
+ function checkStateStat(path, handle) {
142
+ const entry = stateFiles.get(path);
143
+ if (!entry || Atomics.load(entry.words, entry.wordCount + 1) !== 0) return false;
144
+ try {
145
+ const fst = fstatSync(handle, { bigint: true });
146
+ const lst = lstatSync(entry.canonicalPath, { bigint: true });
147
+ if (!matchesFileStateSnapshot(fst, entry.snapshot) || !matchesFileStateSnapshot(lst, entry.snapshot)) {
148
+ Atomics.store(entry.words, entry.wordCount + 1, 1);
149
+ return false;
150
+ }
151
+ return true;
152
+ } catch {
153
+ Atomics.store(entry.words, entry.wordCount + 1, 1);
154
+ return false;
155
+ }
156
+ }
157
+
158
+ /** @param {Readonly<import('./verified-piece-reader.js').VerifiedPiece>} piece */
159
+ function isStateVerified(piece) {
160
+ const entry = stateFiles.get(piece.path);
161
+ if (!entry || Atomics.load(entry.words, entry.wordCount + 1) !== 0) return false;
162
+ const idx = findStatePieceIndex(entry.pieces, piece.offset);
163
+ return idx >= 0 && (Atomics.load(entry.words, idx >>> 5) & (1 << (idx & 31))) !== 0;
164
+ }
165
+
166
+ /** @param {Readonly<import('./verified-piece-reader.js').VerifiedPiece>} piece */
167
+ function markStateVerified(piece) {
168
+ const entry = stateFiles.get(piece.path);
169
+ if (!entry || Atomics.load(entry.words, entry.wordCount + 1) !== 0) return false;
170
+ const idx = findStatePieceIndex(entry.pieces, piece.offset);
171
+ if (idx < 0) return false;
172
+ const mask = 1 << (idx & 31);
173
+ const prev = Atomics.or(entry.words, idx >>> 5, mask);
174
+ if ((prev & mask) === 0) {
175
+ Atomics.add(entry.words, entry.wordCount, 1);
176
+ stateCacheAdmissions++;
177
+ stateCacheAdmittedBytes += piece.size;
178
+ pendingDirtyState = true;
179
+ return true;
180
+ }
181
+ return false;
182
+ }
105
183
  const bucketBytesView = protectedCache ? new BigInt64Array(protectedCache.sharedBucketBytes) : null;
106
184
  const expectedUid = typeof process.geteuid === 'function' ? process.geteuid() : null;
107
185
  let tmpSeq = 0, lastMemCheckAt = 0, lastMemAvailableBytes = Infinity;
@@ -309,7 +387,7 @@ function admitProtectedPiece(piece, output) {
309
387
  }
310
388
  /** @type {Map<string, Readonly<import('./verified-piece-reader.js').VerifiedPiece>>} */
311
389
  const verifiedRanges = new Map();
312
- const maxPieceBytes = JSON.parse(new TextDecoder().decode(workerData.metadata.indexBytes)).maxPieceBytes;
390
+ const maxPieceBytes = parsedIndex.maxPieceBytes;
313
391
  /** @param {Readonly<import('./verified-piece-reader.js').VerifiedPiece>} piece
314
392
  * @param {string} identity */
315
393
  function recordPiece(piece, identity) {
@@ -348,17 +426,85 @@ function getReceipt() {
348
426
  activeGenericScratchBytes, peakGenericScratchBytes,
349
427
  protectedCacheHits, protectedCacheHitBytes, protectedCacheMisses,
350
428
  protectedCacheAdmissions, protectedCacheAdmittedBytes,
351
- protectedCacheEvictions, protectedCacheEvictedBytes, protectedCacheSkips };
429
+ protectedCacheEvictions, protectedCacheEvictedBytes, protectedCacheSkips,
430
+ stateCacheHits, stateCacheHitBytes, stateCacheMisses,
431
+ stateCacheAdmissions, stateCacheAdmittedBytes,
432
+ backgroundVerifiedPieces, backgroundVerificationMs };
352
433
  }
353
434
  port.postMessage({ ready: true });
354
435
  /** @type {Set<Promise<void>>} */
355
436
  const operations = new Set();
437
+ /** @type {ReturnType<typeof setTimeout> | null} */
438
+ let bgTimer = null;
439
+ /** @type {Promise<void> | null} */
440
+ let bgActive = null;
441
+ /** @type {Array<Readonly<import('./verified-piece-reader.js').VerifiedPiece>>} */
442
+ const bgQueue = [];
443
+ let bgCursor = 0;
444
+ /** @type {Uint8Array | null} */
445
+ let bgScratch = null;
446
+ if (stateCacheConfig?.backgroundVerify === true && stateFiles.size > 0) {
447
+ const workerIndex = workerData.workerIndex ?? 0;
448
+ const workerCount = Math.max(1, workerData.workerCount ?? 1);
449
+ let seq = 0;
450
+ for (const [piecePath, entry] of stateFiles) {
451
+ for (const piece of entry.pieces) {
452
+ if (seq % workerCount === workerIndex) {
453
+ bgQueue.push(Object.freeze({ ...piece, path: piecePath }));
454
+ }
455
+ seq++;
456
+ }
457
+ }
458
+ }
459
+
460
+ function scheduleBackgroundVerify() {
461
+ if (closing || !bgQueue.length || bgCursor >= bgQueue.length || bgTimer !== null || bgActive !== null) return;
462
+ bgTimer = setTimeout(() => {
463
+ bgTimer = null;
464
+ if (closing || operations.size > 0 || bgActive !== null) return;
465
+ while (bgCursor < bgQueue.length && isStateVerified(bgQueue[bgCursor])) bgCursor++;
466
+ if (bgCursor >= bgQueue.length) { bgScratch = null; return; }
467
+ const piece = bgQueue[bgCursor++];
468
+ const task = (async () => {
469
+ bgScratch ??= new Uint8Array(maxPieceBytes);
470
+ const view = bgScratch.subarray(0, piece.size);
471
+ const handle = admitFile(piece, view);
472
+ if (!checkStateStat(piece.path, handle) || isStateVerified(piece)) return;
473
+ await readFileParallel(piece, view, handle);
474
+ if (closing || operations.size > 0 || !checkStateStat(piece.path, handle)) return;
475
+ const hashStart = performance.now();
476
+ const digest = 'sha256:' + createHash('sha256').update(view).digest('hex');
477
+ backgroundVerificationMs += performance.now() - hashStart;
478
+ if (digest === piece.identity && checkStateStat(piece.path, handle)) {
479
+ markStateVerified(piece);
480
+ backgroundVerifiedPieces++;
481
+ if (pendingDirtyState) {
482
+ pendingDirtyState = false;
483
+ port.postMessage({ dirtyState: true });
484
+ }
485
+ }
486
+ })().catch(() => {}).finally(() => {
487
+ bgActive = null;
488
+ if (!closing && operations.size === 0 && bgCursor < bgQueue.length) scheduleBackgroundVerify();
489
+ });
490
+ bgActive = task;
491
+ }, stateCacheConfig?.backgroundIdleDelayMs ?? 50);
492
+ bgTimer.unref?.();
493
+ }
494
+ if (bgQueue.length > 0) scheduleBackgroundVerify();
495
+
356
496
  /** @type {Promise<void> | null} */
357
497
  let retirement = null;
358
498
  function closeWorker() {
359
499
  closing = true;
500
+ if (bgTimer !== null) {
501
+ clearTimeout(bgTimer);
502
+ bgTimer = null;
503
+ }
360
504
  if (!retirement) {
361
505
  const operation = (async () => {
506
+ if (bgActive) await bgActive;
507
+ bgScratch = null;
362
508
  await Promise.allSettled([...operations]);
363
509
  await reader.close();
364
510
  const errors = [];
@@ -408,23 +554,43 @@ async function gather({ ranges, length, buffer, batch }) {
408
554
  }
409
555
  const reads = pieces.map((piece, index) => {
410
556
  const output = outputs[index];
557
+ const handle = /** @type {number} */ (admitted.get(piece.path));
411
558
  return (async () => {
559
+ const stateOkBefore = checkStateStat(piece.path, handle);
560
+ if (stateOkBefore && isStateVerified(piece)) {
561
+ const readStart = performance.now();
562
+ try {
563
+ if (pieces.length === 1) readFile(piece, output, handle);
564
+ else await readFileParallel(piece, output, handle);
565
+ } finally { acquisitionMs += performance.now() - readStart; }
566
+ if (closing) throw new Error('Gather worker closed.');
567
+ if (checkStateStat(piece.path, handle)) {
568
+ recordDirect(piece, piece.identity);
569
+ stateCacheHits++;
570
+ stateCacheHitBytes += piece.size;
571
+ return;
572
+ }
573
+ }
574
+ if (stateFiles.has(piece.path)) stateCacheMisses++;
412
575
  if (tryReadProtectedPiece(piece, output, true)) {
413
576
  if (closing) throw new Error('Gather worker closed.');
577
+ // The authenticated bytes came from the protected copy. The file's own bytes at this piece were
578
+ // not read, so the file is not marked verified: a direct read of the file must still be refused.
414
579
  return;
415
580
  }
416
581
  if (protectedCache) protectedCacheMisses++;
417
582
  const readStart = performance.now();
418
583
  try {
419
- if (pieces.length === 1) readFile(piece, output, admitted.get(piece.path));
420
- else await readFileParallel(piece, output, admitted.get(piece.path));
584
+ if (pieces.length === 1) readFile(piece, output, handle);
585
+ else await readFileParallel(piece, output, handle);
421
586
  } finally { acquisitionMs += performance.now() - readStart; }
422
587
  if (closing) throw new Error('Gather worker closed.');
423
588
  // This worker exclusively owns the output until transfer. Hash that
424
589
  // storage directly, without WebCrypto's input snapshot and another
425
590
  // shared thread-pool handoff. Other worker lanes still run in parallel.
426
591
  authenticateDirect(piece, output);
427
- admitProtectedPiece(piece, output);
592
+ if (stateOkBefore && checkStateStat(piece.path, handle)) markStateVerified(piece);
593
+ else admitProtectedPiece(piece, output);
428
594
  })();
429
595
  });
430
596
  // A rejected read must not zero or transfer a buffer while a sibling native
@@ -447,13 +613,34 @@ async function gather({ ranges, length, buffer, batch }) {
447
613
  const piece = pieces[index];
448
614
  if (piece) {
449
615
  const admittedHandle = admitFile(piece, output);
450
- if (!tryReadProtectedPiece(piece, output, true)) {
451
- if (protectedCache) protectedCacheMisses++;
616
+ const stateOkBefore = checkStateStat(piece.path, admittedHandle);
617
+ let satisfied = false;
618
+ if (stateOkBefore && isStateVerified(piece)) {
452
619
  const readStart = performance.now();
453
620
  try { readFile(piece, output, admittedHandle); }
454
621
  finally { acquisitionMs += performance.now() - readStart; }
455
- authenticateDirect(piece, output);
456
- admitProtectedPiece(piece, output);
622
+ if (checkStateStat(piece.path, admittedHandle)) {
623
+ recordDirect(piece, piece.identity);
624
+ stateCacheHits++;
625
+ stateCacheHitBytes += piece.size;
626
+ satisfied = true;
627
+ } else {
628
+ authenticateDirect(piece, output);
629
+ admitProtectedPiece(piece, output);
630
+ satisfied = true;
631
+ }
632
+ }
633
+ if (!satisfied) {
634
+ if (stateFiles.has(piece.path)) stateCacheMisses++;
635
+ if (!tryReadProtectedPiece(piece, output, true)) {
636
+ if (protectedCache) protectedCacheMisses++;
637
+ const readStart = performance.now();
638
+ try { readFile(piece, output, admittedHandle); }
639
+ finally { acquisitionMs += performance.now() - readStart; }
640
+ authenticateDirect(piece, output);
641
+ if (stateOkBefore && checkStateStat(piece.path, admittedHandle)) markStateVerified(piece);
642
+ else admitProtectedPiece(piece, output);
643
+ }
457
644
  }
458
645
  } else {
459
646
  const selected = reader.resolveRangePieces(path, offset, rangeLength);
@@ -466,15 +653,39 @@ async function gather({ ranges, length, buffer, batch }) {
466
653
  for (const current of selected) {
467
654
  const bytes = scratch.subarray(0, current.size);
468
655
  const admittedHandle = admitFile(current, bytes);
469
- if (!tryReadProtectedPiece(current, bytes, false)) {
470
- if (protectedCache) protectedCacheMisses++;
656
+ const stateOkBefore = checkStateStat(current.path, admittedHandle);
657
+ let satisfied = false;
658
+ if (stateOkBefore && isStateVerified(current)) {
471
659
  const readStart = performance.now();
472
660
  try { readFile(current, bytes, admittedHandle); }
473
661
  finally { acquisitionMs += performance.now() - readStart; }
474
- const hashStart = performance.now();
475
- const identity = 'sha256:' + createHash('sha256').update(bytes).digest('hex');
476
- verificationMs += performance.now() - hashStart; recordPiece(current, identity);
477
- admitProtectedPiece(current, bytes);
662
+ if (checkStateStat(current.path, admittedHandle)) {
663
+ recordPiece(current, current.identity);
664
+ stateCacheHits++;
665
+ stateCacheHitBytes += current.size;
666
+ satisfied = true;
667
+ } else {
668
+ const hashStart = performance.now();
669
+ const identity = 'sha256:' + createHash('sha256').update(bytes).digest('hex');
670
+ verificationMs += performance.now() - hashStart;
671
+ recordPiece(current, identity);
672
+ admitProtectedPiece(current, bytes);
673
+ satisfied = true;
674
+ }
675
+ }
676
+ if (!satisfied) {
677
+ if (stateFiles.has(current.path)) stateCacheMisses++;
678
+ if (!tryReadProtectedPiece(current, bytes, false)) {
679
+ if (protectedCache) protectedCacheMisses++;
680
+ const readStart = performance.now();
681
+ try { readFile(current, bytes, admittedHandle); }
682
+ finally { acquisitionMs += performance.now() - readStart; }
683
+ const hashStart = performance.now();
684
+ const identity = 'sha256:' + createHash('sha256').update(bytes).digest('hex');
685
+ verificationMs += performance.now() - hashStart; recordPiece(current, identity);
686
+ if (stateOkBefore && checkStateStat(current.path, admittedHandle)) markStateVerified(current);
687
+ else admitProtectedPiece(current, bytes);
688
+ }
478
689
  }
479
690
  const start = Math.max(offset, current.offset), end = Math.min(offset + rangeLength, current.offset + current.size);
480
691
  const copyStart = performance.now();
@@ -496,17 +707,25 @@ async function gather({ ranges, length, buffer, batch }) {
496
707
  }
497
708
  if (closing) throw new Error('Gather worker closed.');
498
709
  if (batch) { batchReads++; batchRanges += ranges.length; batchBytes += length; }
710
+ const dirtyState = pendingDirtyState;
711
+ pendingDirtyState = false;
499
712
  const timings = snapshotVerifiedPieceJobTimings({ workerWallMs: performance.now() - started,
500
713
  readWaitMs: acquisitionMs - before.acquisitionMs, verificationMs: verificationMs - before.verificationMs,
501
714
  copyMs: genericCopyMs - before.genericCopyMs });
502
- port.postMessage({ buffer, length, timings }, [buffer]);
715
+ port.postMessage({ buffer, length, timings, ...(dirtyState ? { dirtyState: true } : {}) }, [buffer]);
503
716
  } catch (error) {
504
717
  // Never publish a verified prefix after a later piece fails verification.
505
718
  new Uint8Array(buffer).fill(0);
506
719
  port.postMessage({ buffer, error: String(error.message ?? error) }, [buffer]);
720
+ } finally {
721
+ if (!closing && bgQueue.length > 0 && bgCursor < bgQueue.length) scheduleBackgroundVerify();
507
722
  }
508
723
  }
509
724
  port.on('message', message => {
725
+ if (bgTimer !== null) {
726
+ clearTimeout(bgTimer);
727
+ bgTimer = null;
728
+ }
510
729
  if (message.close === true) {
511
730
  closeWorker().then(() => port.postMessage({ closed: true, receipt: getReceipt() }),
512
731
  error => port.postMessage({ closed: true, error: String(error.message ?? error) }));
@@ -1,4 +1,6 @@
1
1
  import type { VerifiedPieceReceipt } from './verified-piece-reader.js';
2
+ import type { VerifiedPieceStateCacheOptions, WorkerStateCachePayload, prepareVerifiedPieceStateCache } from './verified-piece-state-cache.js';
3
+ export type { VerifiedPieceStateCacheOptions, WorkerStateCachePayload } from './verified-piece-state-cache.js';
2
4
  export interface VerifiedPieceProtectedCacheOptions {
3
5
  /** Explicit absolute directory (for example on tmpfs /dev/shm) owned by the current user; symlinks are rejected. */
4
6
  directory: string;
@@ -32,6 +34,10 @@ export interface VerifiedPieceGatherOptions {
32
34
  memoryBudgetBytes: number;
33
35
  /** Optional bounded host-protected cache of authenticated pieces; every cache hit is still SHA-256 verified. */
34
36
  protectedCache?: VerifiedPieceProtectedCacheOptions;
37
+ /** Optional per-file-state verification record cache (`(dev, ino, size, mtimeNs, ctimeNs)`) with off-critical-path persistence. */
38
+ verificationCache?: VerifiedPieceStateCacheOptions;
39
+ /** Internal shared state cache coordinator owned by verified-piece-storage. */
40
+ sharedStateCache?: Awaited<ReturnType<typeof prepareVerifiedPieceStateCache>>;
35
41
  }
36
42
  /** Per-job worker observations. Async read waits and different worker jobs can overlap. */
37
43
  export interface VerifiedPieceJobTimings {
@@ -78,6 +84,23 @@ export interface VerifiedPieceGatherReceipt extends VerifiedPieceReceipt {
78
84
  protectedCacheEvictions: number;
79
85
  protectedCacheEvictedBytes: number;
80
86
  protectedCacheSkips: number;
87
+ /** Per-file-state verification cache counters; zero when verificationCache is omitted. */
88
+ stateCacheHits: number;
89
+ stateCacheHitBytes: number;
90
+ stateCacheMisses: number;
91
+ stateCacheAdmissions: number;
92
+ stateCacheAdmittedBytes: number;
93
+ backgroundVerifiedPieces: number;
94
+ backgroundVerificationMs: number;
95
+ /** Present when direct verified reads are bound: reads served to caller memory from already verified pieces. */
96
+ directVerifiedReads?: number;
97
+ directVerifiedBytes?: number;
98
+ /** Direct reads declined because a piece was unverified or its file no longer matched. */
99
+ directDeclinedReads?: number;
100
+ /** Reads made ahead of verification, their bytes, and how many were later confirmed verified. */
101
+ directAheadReads?: number;
102
+ directAheadBytes?: number;
103
+ directAheadConfirmed?: number;
81
104
  }
82
105
  export function createVerifiedPieceGather(options: VerifiedPieceGatherOptions): Promise<Readonly<{
83
106
  /** At most concurrency in-flight reads or outstanding leases. Cancellation is checked before
@@ -99,6 +122,8 @@ export interface VerifiedRangeLeasePort {
99
122
  fileIdentitiesBound?: boolean;
100
123
  /** True when workers read/admit through an explicit bounded protected piece cache. */
101
124
  protectedCacheBound?: boolean;
125
+ /** True when workers record and reuse piece verification per immutable file state. */
126
+ verificationCacheBound?: boolean;
102
127
  /** readMany validates every requested path and open descriptor before reads and after authentication, before publication. */
103
128
  batchFileIdentityChecks?: 'before-read-after-auth/v1';
104
129
  /** Omission denotes a legacy single-lane port. */
@@ -110,4 +135,26 @@ export interface VerifiedRangeLeasePort {
110
135
  read(shard: number, offset: number, length: number, signal: AbortSignal): Promise<VerifiedRangeLease>;
111
136
  /** Optional batch capability. One output contains ranges in caller order, without padding. */
112
137
  readMany?(ranges: ReadonlyArray<{ shard: number; offset: number; length: number }>, signal: AbortSignal): Promise<VerifiedRangeLease>;
138
+ /**
139
+ * Optional: present when file identities and a verification cache are both bound. Fills caller memory
140
+ * with a range whose every piece is already verified for the file's current state and resolves true;
141
+ * resolves false (destination unusable) when a piece is unverified or the file no longer matches;
142
+ * rejects if the file changes during the read.
143
+ */
144
+ readVerifiedInto?(shard: number, offset: number, destination: Uint8Array, signal?: AbortSignal): Promise<boolean>;
145
+ /**
146
+ * Optional, with `readVerifiedInto`. Fills caller memory before the range's pieces are verified, so the
147
+ * read can run while a lane hashes them. Resolves true when the pieces were already verified, false when
148
+ * the file is not the admitted one, and otherwise a check: the bytes are unauthenticated until it
149
+ * returns true (every covering piece verified, file state unchanged), and must not be used before that.
150
+ */
151
+ readAheadInto?(shard: number, offset: number, destination: Uint8Array, signal?: AbortSignal): Promise<boolean | (() => boolean)>;
152
+ /**
153
+ * Optional: have a lane read and hash the pieces covering these ranges, waiting for a free lane instead of
154
+ * failing when all are busy. The lane's copy of the bytes is dropped. Resolves the job's timings, and
155
+ * rejects if a piece does not match its identity. Foreground reads are served ahead of these jobs.
156
+ */
157
+ verifyMany?(ranges: ReadonlyArray<{ shard: number; offset: number; length: number }>, signal?: AbortSignal): Promise<VerifiedRangeLease['timings'] | null>;
158
+ /** Optional, with `readAheadInto`: whether every piece covering the range is already verified. No I/O. */
159
+ hasVerified?(shard: number, offset: number, length: number): boolean;
113
160
  }
@@ -1,4 +1,6 @@
1
1
  import { createVerifiedPieceGatherLane } from './verified-piece-gather-lane.js';
2
+ import { createVerifiedDirectReader } from './verified-piece-direct.js';
3
+ import { prepareVerifiedPieceStateCache } from './verified-piece-state-cache.js';
2
4
 
3
5
  /**
4
6
  * @param {import('./verified-piece-gather.js').VerifiedPieceProtectedCacheOptions | undefined} rawPolicy
@@ -86,7 +88,8 @@ export async function createVerifiedPieceGather(options) {
86
88
  }))) {
87
89
  throw new Error('Gather file identities must bind every approved file device, inode and size.');
88
90
  }
89
- const pieceBytes = JSON.parse(new TextDecoder().decode(pinned.indexBytes)).maxPieceBytes;
91
+ const index = JSON.parse(new TextDecoder().decode(pinned.indexBytes));
92
+ const pieceBytes = index.maxPieceBytes;
90
93
  const laneBytes = pinned.maxPartBytes + 2 * pieceBytes;
91
94
  const memoryBytes = concurrency * laneBytes;
92
95
  if (!Number.isSafeInteger(concurrency) || concurrency < 1
@@ -97,41 +100,112 @@ export async function createVerifiedPieceGather(options) {
97
100
  throw new Error('Gather budget must cover every concurrent output and complete-piece buffer.');
98
101
  }
99
102
  const protectedCache = await prepareProtectedCache(options.protectedCache, pieceBytes);
103
+ const ownedStateCache = options.sharedStateCache === undefined
104
+ ? await prepareVerifiedPieceStateCache({
105
+ rawPolicy: options.verificationCache,
106
+ manifestIdentity: index.manifestIdentity,
107
+ indexIdentity: pinned.indexIdentity,
108
+ index,
109
+ files: pinned.files,
110
+ })
111
+ : undefined;
112
+ const stateCache = options.sharedStateCache ?? ownedStateCache;
100
113
  /** @type {Array<{ reader: Awaited<ReturnType<typeof createVerifiedPieceGatherLane>>, busy: boolean }>} */
101
114
  const lanes = [];
102
115
  try {
103
- for (let index = 0; index < concurrency; index++) {
104
- pinned.startupSignal?.throwIfAborted();
105
- lanes.push({ reader: await createVerifiedPieceGatherLane({ ...pinned, memoryBudgetBytes: laneBytes, protectedCache }), busy: false });
106
- pinned.startupSignal?.throwIfAborted();
116
+ pinned.startupSignal?.throwIfAborted();
117
+ // Every lane authenticates the same pinned index in its own worker. Start them
118
+ // together: admitting them one after another made startup cost one index
119
+ // authentication per lane (about 0.75 s each for a 63 MB index).
120
+ const started = await Promise.allSettled(Array.from({ length: concurrency }, (_, index) =>
121
+ createVerifiedPieceGatherLane({
122
+ ...pinned,
123
+ memoryBudgetBytes: laneBytes,
124
+ indexMaxPieceBytes: pieceBytes,
125
+ protectedCache,
126
+ stateCache: stateCache?.workerPayload,
127
+ workerIndex: index,
128
+ workerCount: concurrency,
129
+ onStateDirty: stateCache ? () => stateCache.scheduleFlush() : undefined,
130
+ })));
131
+ // Register every lane that did start before reporting a failure, so cleanup owns them all.
132
+ for (const outcome of started) {
133
+ if (outcome.status === 'fulfilled') lanes.push({ reader: outcome.value, busy: false });
107
134
  }
135
+ const failed = started.find(outcome => outcome.status === 'rejected');
136
+ if (failed) throw failed.reason;
137
+ pinned.startupSignal?.throwIfAborted();
108
138
  } catch (error) {
109
- const outcomes = await Promise.allSettled(lanes.map(async lane => lane.reader.close()));
139
+ const outcomes = await Promise.allSettled([
140
+ ...lanes.map(async lane => lane.reader.close()),
141
+ ...(ownedStateCache ? [ownedStateCache.close()] : []),
142
+ ]);
110
143
  const failures = outcomes.flatMap(outcome => outcome.status === 'rejected' ? [outcome.reason] : []);
111
144
  if (failures.length) throw new AggregateError([error, ...failures], 'Gather startup and cleanup failed.');
112
145
  throw error;
113
146
  }
147
+ // Ranges whose pieces are already verified for the current file state need no lane: the caller may
148
+ // read them straight into its own memory. Bound file identities and a verification cache are both
149
+ // required, because those are what the direct checks compare against.
150
+ const direct = stateCache && pinned.fileIdentities !== undefined
151
+ ? createVerifiedDirectReader({ files: pinned.files, fileIdentities: pinned.fileIdentities, index, stateCache }) : null;
114
152
  let closed = false, collecting = false;
115
153
  /** @type {Promise<void> | null} */
116
154
  let closing = null;
155
+ // Background verification waits for a lane instead of failing. A foreground read keeps its contract
156
+ // (more concurrent reads than lanes is the caller's error) and waits only when a lane is held by a
157
+ // background job, which it is served ahead of.
158
+ /** @typedef {{ reader: Awaited<ReturnType<typeof createVerifiedPieceGatherLane>>, busy: boolean, background?: boolean }} Lane */
159
+ /** @type {Array<{ resolve: (lane: Lane) => void, reject: (error: unknown) => void, background: boolean }>} */
160
+ const waiters = [];
161
+ /** @param {Lane} lane */
162
+ const releaseLane = lane => {
163
+ let next = collecting ? -1 : waiters.findIndex(waiter => !waiter.background);
164
+ if (next < 0 && !collecting && waiters.length) next = 0;
165
+ if (next < 0) { lane.busy = false; lane.background = false; return; }
166
+ const [waiter] = waiters.splice(next, 1);
167
+ lane.background = waiter.background;
168
+ waiter.resolve(lane);
169
+ };
170
+ /** @param {AbortSignal | undefined} signal @param {boolean} background @returns {Lane | Promise<Lane>} */
171
+ const acquireLane = (signal, background) => {
172
+ /** @type {Lane | undefined} */
173
+ const lane = collecting ? undefined : lanes.find(candidate => !candidate.busy);
174
+ if (lane) { lane.busy = true; lane.background = background; return lane; }
175
+ if (!background && !lanes.some(/** @param {Lane} candidate */ candidate => candidate.background === true)) {
176
+ throw new Error('Release a current gather lease before another read.');
177
+ }
178
+ return new Promise((resolve, reject) => {
179
+ /** @param {Lane} acquired */
180
+ const settle = acquired => { signal?.removeEventListener('abort', abort); resolve(acquired); };
181
+ const waiter = { background, reject, resolve: settle };
182
+ const abort = () => {
183
+ const at = waiters.indexOf(waiter);
184
+ if (at >= 0) { waiters.splice(at, 1); reject(signal?.reason); }
185
+ };
186
+ signal?.addEventListener('abort', abort, { once: true });
187
+ waiters.push(waiter);
188
+ });
189
+ };
117
190
  /** @param {(reader: Awaited<ReturnType<typeof createVerifiedPieceGatherLane>>) => Promise<import('./verified-piece-gather.js').VerifiedRangeLease>} open
118
- * @param {AbortSignal | undefined} signal */
119
- const read = async (open, signal) => {
191
+ * @param {AbortSignal | undefined} signal @param {boolean} [background] */
192
+ const read = async (open, signal, background = false) => {
120
193
  signal?.throwIfAborted();
121
194
  if (closed) throw new Error('Gather reader closed.');
122
- const lane = collecting ? null : lanes.find(candidate => !candidate.busy);
123
- if (!lane) throw new Error('Release a current gather lease before another read.');
124
- lane.busy = true;
195
+ // A free lane is taken without yielding, so the job is posted in the caller's own turn as before.
196
+ const acquired = acquireLane(signal, background);
197
+ const lane = acquired instanceof Promise ? await acquired : acquired;
125
198
  try {
199
+ if (closed) throw new Error('Gather reader closed.');
126
200
  const owned = await open(lane.reader);
127
201
  if (closed) { owned.release(); throw new Error('Gather reader closed.'); }
128
202
  let released = false;
129
203
  return Object.freeze({ bytes: owned.bytes, ...(owned.timings === undefined ? {} : { timings: owned.timings }), release() {
130
204
  if (released) return;
131
205
  released = true;
132
- try { owned.release(); } finally { lane.busy = false; }
206
+ try { owned.release(); } finally { releaseLane(lane); }
133
207
  } });
134
- } catch (error) { lane.busy = false; throw error; }
208
+ } catch (error) { releaseLane(lane); throw error; }
135
209
  };
136
210
  /** @type {Pick<Awaited<ReturnType<import('./verified-piece-gather.js').createVerifiedPieceGather>>, 'readLease' | 'readManyLease' | 'getReceipt' | 'close'>} */
137
211
  const result = {
@@ -149,22 +223,35 @@ export async function createVerifiedPieceGather(options) {
149
223
  if (closed) throw new Error('Gather reader closed.');
150
224
  const receipts = outcomes.flatMap(outcome => outcome.status === 'fulfilled' ? [outcome.value] : []);
151
225
  const merged = { ...receipts[0] };
152
- const fields = /** @type {const} */ (['requestedBytes', 'acquisitionMs', 'verificationMs', 'copyMs', 'activeReadBytes', 'peakReadBytes', 'directPieceReads', 'directPieceBytes', 'batchReads', 'batchRanges', 'batchBytes', 'activeDigestCopyBytes', 'peakDigestCopyBytes', 'activeGenericScratchBytes', 'peakGenericScratchBytes', 'protectedCacheHits', 'protectedCacheHitBytes', 'protectedCacheMisses', 'protectedCacheAdmissions', 'protectedCacheAdmittedBytes', 'protectedCacheEvictions', 'protectedCacheEvictedBytes', 'protectedCacheSkips']);
153
- Object.assign(merged, Object.fromEntries(fields.map(field => [field, receipts.reduce((total, receipt) => total + receipt[field], 0)])));
226
+ const fields = /** @type {const} */ (['requestedBytes', 'acquisitionMs', 'verificationMs', 'copyMs', 'activeReadBytes', 'peakReadBytes', 'directPieceReads', 'directPieceBytes', 'batchReads', 'batchRanges', 'batchBytes', 'activeDigestCopyBytes', 'peakDigestCopyBytes', 'activeGenericScratchBytes', 'peakGenericScratchBytes', 'protectedCacheHits', 'protectedCacheHitBytes', 'protectedCacheMisses', 'protectedCacheAdmissions', 'protectedCacheAdmittedBytes', 'protectedCacheEvictions', 'protectedCacheEvictedBytes', 'protectedCacheSkips', 'stateCacheHits', 'stateCacheHitBytes', 'stateCacheMisses', 'stateCacheAdmissions', 'stateCacheAdmittedBytes', 'backgroundVerifiedPieces', 'backgroundVerificationMs']);
227
+ Object.assign(merged, Object.fromEntries(fields.map(field => [field, receipts.reduce((total, receipt) => total + (receipt[field] ?? 0), 0)])));
154
228
  /** @type {Map<string, import('./verified-piece-reader.js').VerifiedPiece>} */
155
229
  const ranges = new Map(receipts.flatMap(receipt => receipt.ranges.map(range => [`${range.path}:${range.offset}`, range])));
156
230
  merged.ranges = [...ranges.values()];
157
231
  const pieces = new Map(merged.ranges.map(range => [range.identity, range.size]));
158
232
  merged.pieces = [...pieces.keys()];
159
233
  merged.verifiedBytes = [...pieces.values()].reduce((total, bytes) => total + bytes, 0);
234
+ if (direct) {
235
+ // Bytes served from already verified pieces without a lane; not part of ranges or verifiedBytes.
236
+ const served = direct.getStats();
237
+ Object.assign(merged, { directVerifiedReads: served.reads, directVerifiedBytes: served.bytes, directDeclinedReads: served.declined,
238
+ directAheadReads: served.aheadReads, directAheadBytes: served.aheadBytes, directAheadConfirmed: served.aheadConfirmed });
239
+ }
160
240
  return merged;
161
- } finally { collecting = false; }
241
+ } finally {
242
+ collecting = false;
243
+ // Background jobs that arrived meanwhile take the lanes that are free now.
244
+ for (const lane of lanes) if (!lane.busy && waiters.length) { lane.busy = true; releaseLane(lane); }
245
+ }
162
246
  },
163
247
  close() {
164
248
  if (!closing) {
165
249
  closed = true;
250
+ for (const waiter of waiters.splice(0)) waiter.reject(new Error('Gather reader closed.'));
166
251
  const operation = (async () => {
167
- const outcomes = await Promise.allSettled(lanes.map(async lane => lane.reader.close()));
252
+ const outcomes = await Promise.allSettled([...lanes.map(async lane => lane.reader.close()),
253
+ ...(direct ? [direct.close()] : [])]);
254
+ if (ownedStateCache) await ownedStateCache.close();
168
255
  const failed = outcomes.find(outcome => outcome.status === 'rejected');
169
256
  if (failed) throw failed.reason;
170
257
  })();
@@ -179,12 +266,27 @@ export async function createVerifiedPieceGather(options) {
179
266
  const rangeLease = { concurrency, maxRanges, maxPartBytes: pinned.maxPartBytes, memoryBytes,
180
267
  fileIdentitiesBound: pinned.fileIdentities !== undefined,
181
268
  ...(protectedCache ? { protectedCacheBound: true } : {}),
269
+ ...(stateCache ? { verificationCacheBound: true } : {}),
182
270
  ...(pinned.fileIdentities !== undefined ? { batchFileIdentityChecks: /** @type {const} */ ('before-read-after-auth/v1') } : {}),
183
271
  read(shard, offset, length, signal) {
184
272
  const file = manifest.shards[shard];
185
273
  if (!file) return Promise.reject(new Error('Undeclared gather shard.'));
186
274
  return result.readLease(file.filename, offset, length, signal);
187
275
  },
276
+ ...(direct ? { readVerifiedInto(shard, offset, destination, signal) {
277
+ const file = manifest.shards[shard];
278
+ if (!file) return Promise.reject(new Error('Undeclared gather shard.'));
279
+ if (closed) return Promise.reject(new Error('Gather reader closed.'));
280
+ return direct.readInto(file.filename, offset, destination, signal);
281
+ }, readAheadInto(shard, offset, destination, signal) {
282
+ const file = manifest.shards[shard];
283
+ if (!file) return Promise.reject(new Error('Undeclared gather shard.'));
284
+ if (closed) return Promise.reject(new Error('Gather reader closed.'));
285
+ return direct.readAheadInto(file.filename, offset, destination, signal);
286
+ }, hasVerified(shard, offset, length) {
287
+ const file = manifest.shards[shard];
288
+ return !!file && !closed && direct.isVerified(file.filename, offset, length);
289
+ } } : {}),
188
290
  readMany(ranges, signal) {
189
291
  if (!Array.isArray(ranges) || ranges.length < 1 || ranges.length > maxRanges) return Promise.reject(new Error('Gather batch exceeds its admitted ranges bound.'));
190
292
  const selected = [];
@@ -194,6 +296,20 @@ export async function createVerifiedPieceGather(options) {
194
296
  selected.push({ path: file.filename, offset: range.offset, length: range.length });
195
297
  }
196
298
  return result.readManyLease(selected, signal);
299
+ },
300
+ async verifyMany(ranges, signal) {
301
+ if (!Array.isArray(ranges) || ranges.length < 1 || ranges.length > maxRanges) throw new Error('Gather batch exceeds its admitted ranges bound.');
302
+ /** @type {Array<{ path: string, offset: number, length: number }>} */
303
+ const selected = [];
304
+ for (const range of ranges) {
305
+ const file = Number.isSafeInteger(range?.shard) && range.shard >= 0 ? manifest.shards[range.shard] : null;
306
+ if (!file) throw new Error('Undeclared gather shard.');
307
+ selected.push({ path: file.filename, offset: range.offset, length: range.length });
308
+ }
309
+ const lease = await read(reader => reader.readManyLease(selected, signal), signal, true);
310
+ const timings = lease.timings ?? null;
311
+ lease.release();
312
+ return timings;
197
313
  } };
198
314
  return Object.freeze({ ...result, rangeLease: Object.freeze(rangeLease) });
199
315
  }