doppler-gpu 0.7.7 → 0.7.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. package/CHANGELOG.md +18 -0
  2. package/README.md +9 -17
  3. package/package.json +1 -1
  4. package/src/config/kernels/kernel-ref-digests.js +2 -1
  5. package/src/config/kernels/registry.json +47 -19
  6. package/src/config/schema/execution/kernel-warmup.schema.d.ts +14 -0
  7. package/src/config/schema/execution/kernel-warmup.schema.js +3 -0
  8. package/src/config/schema/platform/gpu-cache.schema.js +1 -1
  9. package/src/config/schema/runtime/buffer-pool.schema.js +2 -2
  10. package/src/config/schema/runtime/loading.schema.d.ts +5 -0
  11. package/src/config/schema/runtime/loading.schema.js +25 -2
  12. package/src/gpu/kernels/moe_route_q4_0_block_f32.wgsl +74 -0
  13. package/src/gpu/kernels/moe_route_q4_0_split_block_f32.wgsl +35 -18
  14. package/src/inference/pipelines/text/partitions/partition-execution.js +28 -1
  15. package/src/inference/pipelines/text/partitions/resident-partition-warmup.js +67 -0
  16. package/src/inference/pipelines/text/partitions/resident-partition.js +8 -2
  17. package/src/loader/experts/compact-expert-cache.js +20 -0
  18. package/src/loader/experts/loader-bindings.js +14 -3
  19. package/src/loader/experts/mapped-expert-staging.js +393 -0
  20. package/src/loader/experts/routed-expert-cache.d.ts +8 -0
  21. package/src/loader/experts/routed-expert-cache.js +12 -11
  22. package/src/loader/experts/routed-expert-prefill.js +77 -1
  23. package/src/loader/experts/routed-expert-source.js +1 -0
  24. package/src/loader/experts/tiered-expert-cache.js +66 -26
  25. package/src/storage/verified-piece-direct.js +153 -0
  26. package/src/storage/verified-piece-gather-lane.js +21 -3
  27. package/src/storage/verified-piece-gather-worker.js +235 -16
  28. package/src/storage/verified-piece-gather.d.ts +47 -0
  29. package/src/storage/verified-piece-gather.js +133 -17
  30. package/src/storage/verified-piece-reader.d.ts +8 -1
  31. package/src/storage/verified-piece-reader.js +16 -3
  32. package/src/storage/verified-piece-state-cache.d.ts +75 -0
  33. package/src/storage/verified-piece-state-cache.js +511 -0
  34. package/src/storage/verified-piece-storage.d.ts +20 -1
  35. package/src/storage/verified-piece-storage.js +207 -9
  36. package/src/version.js +1 -1
@@ -7,11 +7,18 @@ export interface VerifiedPieceReceipt {
7
7
  acquisitionMs: number; verificationMs: number; copyMs: number;
8
8
  /** Range output buffers owned by active reads, before transfer to the loader; not process RSS. */
9
9
  activeReadBytes: number; peakReadBytes: number;
10
+ stateCacheHits?: number; stateCacheHitBytes?: number; stateCacheMisses?: number;
11
+ }
12
+ export interface StateVerifiedAcquisition {
13
+ bytes: Uint8Array | ArrayBuffer;
14
+ stateVerified?: boolean;
15
+ onAuthenticated?: () => void;
10
16
  }
11
17
  export interface VerifiedPieceReaderOptions {
12
18
  manifestBytes: ArrayBuffer | Uint8Array<ArrayBuffer>; indexBytes: ArrayBuffer | Uint8Array<ArrayBuffer>; indexIdentity: string;
13
- acquire: (piece: Readonly<VerifiedPiece>, options: { signal?: AbortSignal }) => Promise<Uint8Array | ArrayBuffer>;
19
+ acquire: (piece: Readonly<VerifiedPiece>, options: { signal?: AbortSignal }) => Promise<Uint8Array | ArrayBuffer | StateVerifiedAcquisition>;
14
20
  signal?: AbortSignal;
21
+ allowStateVerifiedAcquire?: boolean;
15
22
  }
16
23
  export function createVerifiedPieceReader(options: VerifiedPieceReaderOptions): Promise<Readonly<{
17
24
  manifest: import('../formats/rdrr/index.js').RDRRManifest;
@@ -7,7 +7,7 @@ const digest = async bytes => 'sha256:' + [...new Uint8Array(await crypto.subtle
7
7
  * This reader authenticates bytes; it does not establish signed Capsule qualification.
8
8
  * @type {import('./verified-piece-reader.js').createVerifiedPieceReader}
9
9
  */
10
- export async function createVerifiedPieceReader({ manifestBytes, indexBytes, indexIdentity, acquire, signal }) {
10
+ export async function createVerifiedPieceReader({ manifestBytes, indexBytes, indexIdentity, acquire, signal, allowStateVerifiedAcquire = false }) {
11
11
  for (const bytes of [manifestBytes, indexBytes]) {
12
12
  if (!(bytes instanceof Uint8Array) && !(bytes instanceof ArrayBuffer)) throw new Error('Piece metadata requires bytes.');
13
13
  }
@@ -121,13 +121,26 @@ export async function createVerifiedPieceReader({ manifestBytes, indexBytes, ind
121
121
  if (piece.offset >= offset + length) break;
122
122
  signal?.throwIfAborted();
123
123
  const acquisitionStarted = performance.now();
124
- const received = await waitActive(acquire(Object.freeze({ ...piece, path }), { signal }));
124
+ const rawAcquired = await waitActive(acquire(Object.freeze({ ...piece, path }), { signal }));
125
125
  receipt.acquisitionMs += performance.now() - acquisitionStarted;
126
126
  const verificationStarted = performance.now();
127
+ const structured = allowStateVerifiedAcquire === true
128
+ && rawAcquired !== null && typeof rawAcquired === 'object'
129
+ && !(rawAcquired instanceof Uint8Array) && !(rawAcquired instanceof ArrayBuffer);
130
+ const received = structured ? rawAcquired.bytes : rawAcquired;
127
131
  if (!(received instanceof Uint8Array) && !(received instanceof ArrayBuffer)
128
132
  || received.byteLength !== piece.size) throw new Error('Acquired piece integrity mismatch.');
129
133
  const bytes = new Uint8Array(received instanceof Uint8Array ? received : new Uint8Array(received));
130
- if (await waitActive(digest(bytes)) !== piece.identity) throw new Error('Acquired piece integrity mismatch.');
134
+ if (structured && rawAcquired.stateVerified === true) {
135
+ receipt.stateCacheHits = (receipt.stateCacheHits ?? 0) + 1;
136
+ receipt.stateCacheHitBytes = (receipt.stateCacheHitBytes ?? 0) + bytes.byteLength;
137
+ } else {
138
+ if (await waitActive(digest(bytes)) !== piece.identity) throw new Error('Acquired piece integrity mismatch.');
139
+ if (structured) {
140
+ receipt.stateCacheMisses = (receipt.stateCacheMisses ?? 0) + 1;
141
+ rawAcquired.onAuthenticated?.();
142
+ }
143
+ }
131
144
  receipt.verificationMs += performance.now() - verificationStarted;
132
145
  signal?.throwIfAborted();
133
146
  if (closed) throw new Error('Piece storage closed.');
@@ -0,0 +1,75 @@
1
+ interface VerifiedFileStat {
2
+ isFile(): boolean;
3
+ dev: bigint;
4
+ ino: bigint;
5
+ size: bigint;
6
+ mtimeNs: bigint;
7
+ ctimeNs: bigint;
8
+ }
9
+
10
+ export interface FileStateSnapshot {
11
+ device: string;
12
+ inode: string;
13
+ size: string;
14
+ mtimeNs: string;
15
+ ctimeNs: string;
16
+ }
17
+
18
+ export interface VerifiedPieceStateCacheOptions {
19
+ /** Optional absolute directory owned by the current user; symlinks are rejected. When omitted, verification bits live in memory for the open storage session only. */
20
+ directory?: string;
21
+ /** Optional idle-only background piece verification on gather workers (default false). */
22
+ backgroundVerify?: boolean;
23
+ /** Minimum idle delay in milliseconds before running a background verification step (default 50). */
24
+ backgroundIdleDelayMs?: number;
25
+ }
26
+
27
+ export interface WorkerFileStateDescriptor {
28
+ canonicalPath: string;
29
+ snapshot: FileStateSnapshot;
30
+ wordCount: number;
31
+ pieceCount: number;
32
+ sharedBuffer: SharedArrayBuffer;
33
+ stateFilePath: string | null;
34
+ }
35
+
36
+ export interface WorkerStateCachePayload {
37
+ directory: string | null;
38
+ backgroundVerify: boolean;
39
+ backgroundIdleDelayMs: number;
40
+ files: Record<string, WorkerFileStateDescriptor>;
41
+ }
42
+
43
+ export function matchesFileStateSnapshot(stat: VerifiedFileStat, snapshot: FileStateSnapshot): boolean;
44
+
45
+ export function prepareVerifiedPieceStateCache(options: {
46
+ rawPolicy: VerifiedPieceStateCacheOptions | undefined;
47
+ manifestIdentity: string;
48
+ indexIdentity: string;
49
+ index: { files: ReadonlyArray<{ path: string; size: number; pieces: Array<{ offset: number; size: number; identity: string }> }> };
50
+ files: Record<string, string>;
51
+ }): Promise<Readonly<{
52
+ directory: string | null;
53
+ backgroundVerify: boolean;
54
+ backgroundIdleDelayMs: number;
55
+ workerPayload: Readonly<WorkerStateCachePayload>;
56
+ checkFileStat(piecePath: string, stat: VerifiedFileStat): boolean;
57
+ isPieceVerified(piecePath: string, offset: number): boolean;
58
+ markPieceVerified(piecePath: string, offset: number): boolean;
59
+ scheduleFlush(): void;
60
+ flushSync(): void;
61
+ close(): Promise<void>;
62
+ }> | undefined>;
63
+
64
+ export function openPieceDirectoryStateCache(options: {
65
+ directory: string;
66
+ pieceDirectory: string;
67
+ manifestIdentity: string;
68
+ indexIdentity: string;
69
+ index: { files: ReadonlyArray<{ path: string; size: number; pieces: Array<{ offset: number; size: number; identity: string }> }> };
70
+ }): Promise<Readonly<{
71
+ directory: string;
72
+ isVerified(piecePath: string, offset: number, stat: VerifiedFileStat): boolean;
73
+ markVerified(piecePath: string, offset: number, stat: VerifiedFileStat): void;
74
+ close(): Promise<void>;
75
+ }>>;
@@ -0,0 +1,511 @@
1
+ import { createHash } from 'node:crypto';
2
+ import {
3
+ closeSync, fstatSync, lstatSync, mkdirSync, openSync, readSync,
4
+ realpathSync, renameSync, unlinkSync, writeSync, constants,
5
+ } from 'node:fs';
6
+ import { open, rename, unlink } from 'node:fs/promises';
7
+ import { isAbsolute, join } from 'node:path';
8
+
9
+ const FILE_STATE_SCHEMA = 'doppler.verified-piece-file-state/v1';
10
+ const PIECE_DIR_STATE_SCHEMA = 'doppler.verified-piece-dir-state/v1';
11
+ const HEADER_BYTES = 32;
12
+
13
+ /** @param {string | Uint8Array} input */
14
+ const sha256Bytes = input => createHash('sha256').update(input).digest();
15
+
16
+ /** @param {string | Uint8Array} input */
17
+ const sha256Hex = input => createHash('sha256').update(input).digest('hex');
18
+
19
+ /**
20
+ * @param {Uint8Array} a
21
+ * @param {Uint8Array} b
22
+ */
23
+ function equalBytes(a, b) {
24
+ if (a.byteLength !== b.byteLength) return false;
25
+ for (let i = 0; i < a.byteLength; i++) {
26
+ if (a[i] !== b[i]) return false;
27
+ }
28
+ return true;
29
+ }
30
+
31
+ /**
32
+ * @param {string} directory
33
+ * @param {number | null} expectedUid
34
+ */
35
+ function ensureOwnedStateDirectory(directory, expectedUid) {
36
+ if (typeof directory !== 'string' || !directory.trim() || !isAbsolute(directory)) {
37
+ throw new Error('Piece state verification cache requires an absolute directory.');
38
+ }
39
+ mkdirSync(directory, { recursive: true, mode: 0o700 });
40
+ const st = lstatSync(directory);
41
+ if (!st.isDirectory() || (st.mode & 0o022) !== 0 || (expectedUid !== null && st.uid !== expectedUid)) {
42
+ throw new Error('Piece state verification cache directory must be an owned non-symlink directory.');
43
+ }
44
+ return realpathSync(directory);
45
+ }
46
+
47
+ /**
48
+ * @param {Array<{ offset: number, size: number }>} pieces
49
+ * @param {number} offset
50
+ */
51
+ function findPieceIndex(pieces, offset) {
52
+ let low = 0;
53
+ let high = pieces.length;
54
+ while (low < high) {
55
+ const mid = (low + high) >>> 1;
56
+ const piece = pieces[mid];
57
+ if (piece.offset + piece.size <= offset) low = mid + 1;
58
+ else high = mid;
59
+ }
60
+ return low < pieces.length && pieces[low].offset === offset ? low : -1;
61
+ }
62
+
63
+ /**
64
+ * @param {string} stateFilePath
65
+ * @param {number} payloadBytes
66
+ * @param {number | null} expectedUid
67
+ */
68
+ function tryLoadVerifiedPayload(stateFilePath, payloadBytes, expectedUid) {
69
+ let fd = -1;
70
+ try {
71
+ fd = openSync(stateFilePath, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
72
+ const st = fstatSync(fd);
73
+ if (!st.isFile() || (st.mode & 0o022) !== 0 || st.size !== HEADER_BYTES + payloadBytes
74
+ || (expectedUid !== null && st.uid !== expectedUid)) {
75
+ try { unlinkSync(stateFilePath); } catch {}
76
+ return null;
77
+ }
78
+ const raw = new Uint8Array(HEADER_BYTES + payloadBytes);
79
+ let read = 0;
80
+ while (read < raw.byteLength) {
81
+ const count = readSync(fd, raw, read, raw.byteLength - read, read);
82
+ if (!count) {
83
+ try { unlinkSync(stateFilePath); } catch {}
84
+ return null;
85
+ }
86
+ read += count;
87
+ }
88
+ const header = raw.subarray(0, HEADER_BYTES);
89
+ const payload = raw.subarray(HEADER_BYTES);
90
+ if (!equalBytes(header, sha256Bytes(payload))) {
91
+ try { unlinkSync(stateFilePath); } catch {}
92
+ return null;
93
+ }
94
+ return payload;
95
+ } catch {
96
+ return null;
97
+ } finally {
98
+ if (fd !== -1) {
99
+ try { closeSync(fd); } catch {}
100
+ }
101
+ }
102
+ }
103
+
104
+ let tempCounter = 0;
105
+
106
+ /**
107
+ * @param {string} stateFilePath
108
+ * @param {Uint8Array} payloadSnapshot
109
+ */
110
+ async function writeVerifiedPayloadAsync(stateFilePath, payloadSnapshot) {
111
+ const tmpPath = `${stateFilePath}.tmp.${process.pid}.${++tempCounter}`;
112
+ const out = new Uint8Array(HEADER_BYTES + payloadSnapshot.byteLength);
113
+ out.set(sha256Bytes(payloadSnapshot), 0);
114
+ out.set(payloadSnapshot, HEADER_BYTES);
115
+ let handle = null;
116
+ try {
117
+ handle = await open(tmpPath, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY | constants.O_NOFOLLOW, 0o600);
118
+ let written = 0;
119
+ while (written < out.byteLength) {
120
+ const res = await handle.write(out, written, out.byteLength - written, written);
121
+ if (!res.bytesWritten) throw new Error('Incomplete state cache write.');
122
+ written += res.bytesWritten;
123
+ }
124
+ await handle.close();
125
+ handle = null;
126
+ await rename(tmpPath, stateFilePath);
127
+ } catch {
128
+ if (handle) {
129
+ try { await handle.close(); } catch {}
130
+ }
131
+ try { await unlink(tmpPath); } catch {}
132
+ }
133
+ }
134
+
135
+ /**
136
+ * @param {string} stateFilePath
137
+ * @param {Uint8Array} payloadSnapshot
138
+ */
139
+ function writeVerifiedPayloadSync(stateFilePath, payloadSnapshot) {
140
+ const tmpPath = `${stateFilePath}.tmp.${process.pid}.${++tempCounter}`;
141
+ const out = new Uint8Array(HEADER_BYTES + payloadSnapshot.byteLength);
142
+ out.set(sha256Bytes(payloadSnapshot), 0);
143
+ out.set(payloadSnapshot, HEADER_BYTES);
144
+ let fd = -1;
145
+ try {
146
+ fd = openSync(tmpPath, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY | constants.O_NOFOLLOW, 0o600);
147
+ let written = 0;
148
+ while (written < out.byteLength) {
149
+ written += writeSync(fd, out, written, out.byteLength - written, written);
150
+ }
151
+ closeSync(fd);
152
+ fd = -1;
153
+ renameSync(tmpPath, stateFilePath);
154
+ } catch {
155
+ if (fd !== -1) {
156
+ try { closeSync(fd); } catch {}
157
+ }
158
+ try { unlinkSync(tmpPath); } catch {}
159
+ }
160
+ }
161
+
162
+ /**
163
+ * @param {import('node:fs').BigIntStats} stat
164
+ * @param {{ device: string, inode: string, size: string, mtimeNs: string, ctimeNs: string }} snapshot
165
+ */
166
+ export function matchesFileStateSnapshot(stat, snapshot) {
167
+ return stat.isFile()
168
+ && stat.dev.toString() === snapshot.device
169
+ && stat.ino.toString() === snapshot.inode
170
+ && stat.size.toString() === snapshot.size
171
+ && stat.mtimeNs.toString() === snapshot.mtimeNs
172
+ && stat.ctimeNs.toString() === snapshot.ctimeNs;
173
+ }
174
+
175
+ /**
176
+ * Prepare shared in-memory + optional persistent per-file-state verification bitsets
177
+ * across main-thread readers and worker gather lanes.
178
+ * @type {typeof import('./verified-piece-state-cache.js').prepareVerifiedPieceStateCache}
179
+ */
180
+ export async function prepareVerifiedPieceStateCache(options) {
181
+ const { rawPolicy, manifestIdentity, indexIdentity, index, files } = options;
182
+ if (rawPolicy === undefined) return undefined;
183
+ if (!rawPolicy || typeof rawPolicy !== 'object' || Array.isArray(rawPolicy)) {
184
+ throw new Error('Piece state verification cache policy must be an explicit object.');
185
+ }
186
+ const backgroundVerify = rawPolicy.backgroundVerify === true;
187
+ const backgroundIdleDelayMs = rawPolicy.backgroundIdleDelayMs ?? 50;
188
+ if (typeof backgroundVerify !== 'boolean'
189
+ || !Number.isSafeInteger(backgroundIdleDelayMs) || backgroundIdleDelayMs < 0 || backgroundIdleDelayMs > 60_000) {
190
+ throw new Error('Piece state verification cache requires valid backgroundVerify and backgroundIdleDelayMs.');
191
+ }
192
+ const expectedUid = typeof process.geteuid === 'function' ? process.geteuid() : null;
193
+ const directory = rawPolicy.directory === undefined
194
+ ? null
195
+ : ensureOwnedStateDirectory(rawPolicy.directory, expectedUid);
196
+
197
+ /** @type {Record<string, import('./verified-piece-state-cache.js').WorkerFileStateDescriptor>} */
198
+ const workerFiles = {};
199
+ /** @type {Map<string, { pieces: Array<{ offset: number, size: number }>, words: Int32Array, wordCount: number, snapshot: import('./verified-piece-state-cache.js').FileStateSnapshot, canonicalPath: string, stateFilePath: string | null, lastSavedSeq: number }>} */
200
+ const entries = new Map();
201
+
202
+ for (const file of index.files) {
203
+ const mapped = files?.[file.path];
204
+ if (typeof mapped !== 'string') continue;
205
+ let st;
206
+ try {
207
+ st = lstatSync(mapped, { bigint: true });
208
+ } catch {
209
+ continue;
210
+ }
211
+ if (!st.isFile() || st.size !== BigInt(file.size)) continue;
212
+ const snapshot = {
213
+ device: st.dev.toString(),
214
+ inode: st.ino.toString(),
215
+ size: st.size.toString(),
216
+ mtimeNs: st.mtimeNs.toString(),
217
+ ctimeNs: st.ctimeNs.toString(),
218
+ };
219
+ const wordCount = Math.max(1, Math.ceil(file.pieces.length / 32));
220
+ const sharedBuffer = new SharedArrayBuffer((wordCount + 2) * 4);
221
+ const words = new Int32Array(sharedBuffer);
222
+ let stateFilePath = null;
223
+ if (directory !== null) {
224
+ const stateKey = sha256Hex(JSON.stringify({
225
+ schema: FILE_STATE_SCHEMA,
226
+ manifestIdentity,
227
+ indexIdentity,
228
+ piecePath: file.path,
229
+ canonicalPath: mapped,
230
+ uid: expectedUid,
231
+ ...snapshot,
232
+ pieceCount: file.pieces.length,
233
+ }));
234
+ stateFilePath = join(directory, `${stateKey}.state`);
235
+ const loaded = tryLoadVerifiedPayload(stateFilePath, wordCount * 4, expectedUid);
236
+ if (loaded) {
237
+ new Uint8Array(sharedBuffer, 0, wordCount * 4).set(loaded);
238
+ }
239
+ }
240
+ workerFiles[file.path] = {
241
+ canonicalPath: mapped,
242
+ snapshot,
243
+ wordCount,
244
+ pieceCount: file.pieces.length,
245
+ sharedBuffer,
246
+ stateFilePath,
247
+ };
248
+ entries.set(file.path, {
249
+ pieces: file.pieces,
250
+ words,
251
+ wordCount,
252
+ snapshot,
253
+ canonicalPath: mapped,
254
+ stateFilePath,
255
+ lastSavedSeq: 0,
256
+ });
257
+ }
258
+
259
+ /** @type {ReturnType<typeof setTimeout> | null} */
260
+ let flushTimer = null;
261
+ /** @type {Promise<void>} */
262
+ let flushChain = Promise.resolve();
263
+ let closed = false;
264
+
265
+ const flushDirty = () => {
266
+ if (directory === null) return Promise.resolve();
267
+ flushChain = flushChain.then(async () => {
268
+ for (const entry of entries.values()) {
269
+ if (!entry.stateFilePath) continue;
270
+ if (Atomics.load(entry.words, entry.wordCount + 1) !== 0) continue;
271
+ const seq = Atomics.load(entry.words, entry.wordCount);
272
+ if (seq === entry.lastSavedSeq) continue;
273
+ const snapshotBytes = new Uint8Array(entry.wordCount * 4);
274
+ for (let w = 0; w < entry.wordCount; w++) {
275
+ const val = Atomics.load(entry.words, w);
276
+ snapshotBytes[w * 4] = val & 0xff;
277
+ snapshotBytes[w * 4 + 1] = (val >>> 8) & 0xff;
278
+ snapshotBytes[w * 4 + 2] = (val >>> 16) & 0xff;
279
+ snapshotBytes[w * 4 + 3] = (val >>> 24) & 0xff;
280
+ }
281
+ await writeVerifiedPayloadAsync(entry.stateFilePath, snapshotBytes);
282
+ entry.lastSavedSeq = seq;
283
+ }
284
+ }).catch(() => {});
285
+ return flushChain;
286
+ };
287
+
288
+ const scheduleFlush = () => {
289
+ if (directory === null || closed || flushTimer !== null) return;
290
+ flushTimer = setTimeout(() => {
291
+ flushTimer = null;
292
+ void flushDirty();
293
+ }, 25);
294
+ flushTimer.unref?.();
295
+ };
296
+
297
+ /**
298
+ * Check whether a mapped full file currently matches its admitted snapshot,
299
+ * invalidating cached bits immediately if any stat field changed.
300
+ * @param {string} piecePath
301
+ * @param {import('node:fs').BigIntStats} stat
302
+ */
303
+ const checkFileStat = (piecePath, stat) => {
304
+ const entry = entries.get(piecePath);
305
+ if (!entry) return false;
306
+ if (!matchesFileStateSnapshot(stat, entry.snapshot)) {
307
+ Atomics.store(entry.words, entry.wordCount + 1, 1);
308
+ return false;
309
+ }
310
+ return Atomics.load(entry.words, entry.wordCount + 1) === 0;
311
+ };
312
+
313
+ /**
314
+ * @param {string} piecePath
315
+ * @param {number} offset
316
+ */
317
+ const isPieceVerified = (piecePath, offset) => {
318
+ const entry = entries.get(piecePath);
319
+ if (!entry || Atomics.load(entry.words, entry.wordCount + 1) !== 0) return false;
320
+ const idx = findPieceIndex(entry.pieces, offset);
321
+ if (idx < 0) return false;
322
+ return (Atomics.load(entry.words, idx >>> 5) & (1 << (idx & 31))) !== 0;
323
+ };
324
+
325
+ /**
326
+ * @param {string} piecePath
327
+ * @param {number} offset
328
+ */
329
+ const markPieceVerified = (piecePath, offset) => {
330
+ const entry = entries.get(piecePath);
331
+ if (!entry || Atomics.load(entry.words, entry.wordCount + 1) !== 0) return false;
332
+ const idx = findPieceIndex(entry.pieces, offset);
333
+ if (idx < 0) return false;
334
+ const mask = 1 << (idx & 31);
335
+ const prev = Atomics.or(entry.words, idx >>> 5, mask);
336
+ if ((prev & mask) === 0) {
337
+ Atomics.add(entry.words, entry.wordCount, 1);
338
+ scheduleFlush();
339
+ return true;
340
+ }
341
+ return false;
342
+ };
343
+
344
+ const close = async () => {
345
+ closed = true;
346
+ if (flushTimer !== null) {
347
+ clearTimeout(flushTimer);
348
+ flushTimer = null;
349
+ }
350
+ await flushDirty();
351
+ };
352
+
353
+ return Object.freeze({
354
+ directory,
355
+ backgroundVerify,
356
+ backgroundIdleDelayMs,
357
+ workerPayload: Object.freeze({
358
+ directory,
359
+ backgroundVerify,
360
+ backgroundIdleDelayMs,
361
+ files: workerFiles,
362
+ }),
363
+ checkFileStat,
364
+ isPieceVerified,
365
+ markPieceVerified,
366
+ scheduleFlush,
367
+ flushSync() {
368
+ if (directory === null) return;
369
+ for (const entry of entries.values()) {
370
+ if (!entry.stateFilePath || Atomics.load(entry.words, entry.wordCount + 1) !== 0) continue;
371
+ const seq = Atomics.load(entry.words, entry.wordCount);
372
+ if (seq === entry.lastSavedSeq) continue;
373
+ const snapshotBytes = new Uint8Array(entry.wordCount * 4);
374
+ for (let w = 0; w < entry.wordCount; w++) {
375
+ const val = Atomics.load(entry.words, w);
376
+ snapshotBytes[w * 4] = val & 0xff;
377
+ snapshotBytes[w * 4 + 1] = (val >>> 8) & 0xff;
378
+ snapshotBytes[w * 4 + 2] = (val >>> 16) & 0xff;
379
+ snapshotBytes[w * 4 + 3] = (val >>> 24) & 0xff;
380
+ }
381
+ writeVerifiedPayloadSync(entry.stateFilePath, snapshotBytes);
382
+ entry.lastSavedSeq = seq;
383
+ }
384
+ },
385
+ close,
386
+ });
387
+ }
388
+
389
+ /**
390
+ * Persistent per-piece-file `(dev, ino, size, mtimeNs, ctimeNs)` verification cache
391
+ * for partial piece-cache directories (`<pieceDirectory>/<path>.<offset>`).
392
+ * @type {typeof import('./verified-piece-state-cache.js').openPieceDirectoryStateCache}
393
+ */
394
+ export async function openPieceDirectoryStateCache(options) {
395
+ const { directory: rawDir, pieceDirectory, manifestIdentity, indexIdentity, index } = options;
396
+ const expectedUid = typeof process.geteuid === 'function' ? process.geteuid() : null;
397
+ const directory = ensureOwnedStateDirectory(rawDir, expectedUid);
398
+ const canonicalPieceDir = realpathSync(pieceDirectory);
399
+ const dirStat = lstatSync(canonicalPieceDir, { bigint: true });
400
+ if (!dirStat.isDirectory()) throw new Error('Piece directory must be a non-symlink directory.');
401
+
402
+ /** @type {Map<string, { baseIndex: number, pieces: Array<{ offset: number, size: number }> }>} */
403
+ const fileLayout = new Map();
404
+ let totalPieces = 0;
405
+ for (const file of index.files) {
406
+ fileLayout.set(file.path, { baseIndex: totalPieces, pieces: file.pieces });
407
+ totalPieces += file.pieces.length;
408
+ }
409
+
410
+ // 4 x BigUint64 per piece: [dev, ino, mtimeNs, ctimeNs]; piece size is fixed in index.
411
+ const payloadBytes = totalPieces * 32;
412
+ const tableBuffer = new ArrayBuffer(payloadBytes);
413
+ const table = new BigUint64Array(tableBuffer);
414
+ const stateKey = sha256Hex(JSON.stringify({
415
+ schema: PIECE_DIR_STATE_SCHEMA,
416
+ manifestIdentity,
417
+ indexIdentity,
418
+ pieceDirectory: canonicalPieceDir,
419
+ uid: expectedUid,
420
+ totalPieces,
421
+ }));
422
+ const stateFilePath = join(directory, `${stateKey}.pstate`);
423
+ const loaded = tryLoadVerifiedPayload(stateFilePath, payloadBytes, expectedUid);
424
+ if (loaded) {
425
+ new Uint8Array(tableBuffer).set(loaded);
426
+ }
427
+
428
+ let dirty = false;
429
+ let savedDirtyGen = 0;
430
+ let dirtyGen = 0;
431
+ /** @type {ReturnType<typeof setTimeout> | null} */
432
+ let flushTimer = null;
433
+ /** @type {Promise<void>} */
434
+ let flushChain = Promise.resolve();
435
+ let closed = false;
436
+
437
+ /**
438
+ * @param {string} piecePath
439
+ * @param {number} offset
440
+ */
441
+ const resolveSlot = (piecePath, offset) => {
442
+ const file = fileLayout.get(piecePath);
443
+ if (!file) return -1;
444
+ const localIdx = findPieceIndex(file.pieces, offset);
445
+ if (localIdx < 0) return -1;
446
+ return (file.baseIndex + localIdx) * 4;
447
+ };
448
+
449
+ const flushDirty = () => {
450
+ if (!dirty || savedDirtyGen === dirtyGen) return flushChain;
451
+ const targetGen = dirtyGen;
452
+ const snapshot = new Uint8Array(tableBuffer).slice();
453
+ flushChain = flushChain.then(async () => {
454
+ await writeVerifiedPayloadAsync(stateFilePath, snapshot);
455
+ savedDirtyGen = targetGen;
456
+ }).catch(() => {});
457
+ return flushChain;
458
+ };
459
+
460
+ const scheduleFlush = () => {
461
+ if (closed || flushTimer !== null) return;
462
+ flushTimer = setTimeout(() => {
463
+ flushTimer = null;
464
+ void flushDirty();
465
+ }, 50);
466
+ flushTimer.unref?.();
467
+ };
468
+
469
+ return Object.freeze({
470
+ directory,
471
+ /**
472
+ * @param {string} piecePath
473
+ * @param {number} offset
474
+ * @param {import('node:fs').BigIntStats} stat
475
+ */
476
+ isVerified(piecePath, offset, stat) {
477
+ const slot = resolveSlot(piecePath, offset);
478
+ if (slot < 0) return false;
479
+ const ino = table[slot + 1];
480
+ if (ino === 0n) return false;
481
+ return table[slot] === BigInt.asUintN(64, stat.dev)
482
+ && ino === BigInt.asUintN(64, stat.ino)
483
+ && table[slot + 2] === BigInt.asUintN(64, stat.mtimeNs)
484
+ && table[slot + 3] === BigInt.asUintN(64, stat.ctimeNs);
485
+ },
486
+ /**
487
+ * @param {string} piecePath
488
+ * @param {number} offset
489
+ * @param {import('node:fs').BigIntStats} stat
490
+ */
491
+ markVerified(piecePath, offset, stat) {
492
+ const slot = resolveSlot(piecePath, offset);
493
+ if (slot < 0) return;
494
+ table[slot] = BigInt.asUintN(64, stat.dev);
495
+ table[slot + 1] = BigInt.asUintN(64, stat.ino);
496
+ table[slot + 2] = BigInt.asUintN(64, stat.mtimeNs);
497
+ table[slot + 3] = BigInt.asUintN(64, stat.ctimeNs);
498
+ dirty = true;
499
+ dirtyGen++;
500
+ scheduleFlush();
501
+ },
502
+ async close() {
503
+ closed = true;
504
+ if (flushTimer !== null) {
505
+ clearTimeout(flushTimer);
506
+ flushTimer = null;
507
+ }
508
+ await flushDirty();
509
+ },
510
+ });
511
+ }
@@ -1,9 +1,28 @@
1
1
  import type { VerifiedPieceReaderOptions, VerifiedPieceReceipt } from './verified-piece-reader.js';
2
2
  export type { VerifiedPiece, VerifiedPieceReceipt } from './verified-piece-reader.js';
3
- export function createVerifiedPieceStorage(options: VerifiedPieceReaderOptions & { nodeGather?: Pick<import('./verified-piece-gather.js').VerifiedPieceGatherOptions, 'files' | 'fileIdentities' | 'maxPartBytes' | 'maxRanges' | 'memoryBudgetBytes' | 'concurrency' | 'protectedCache'> }): Promise<{
3
+ export interface NodePieceDirectoryOptions {
4
+ directory: string;
5
+ maxPieceBytes: number;
6
+ maxActiveReadBytes: number;
7
+ verificationCache?: { directory: string };
8
+ }
9
+ export interface NodePieceDirectoryReceipt {
10
+ reads: number;
11
+ activeReadBytes: number;
12
+ peakReadBytes: number;
13
+ missingPieces: number;
14
+ pendingDescriptorCloses: number;
15
+ }
16
+ export function createVerifiedPieceStorage(options: Omit<VerifiedPieceReaderOptions, 'acquire'> & {
17
+ acquire?: VerifiedPieceReaderOptions['acquire'];
18
+ nodeGather?: Pick<import('./verified-piece-gather.js').VerifiedPieceGatherOptions, 'files' | 'fileIdentities' | 'maxPartBytes' | 'maxRanges' | 'memoryBudgetBytes' | 'concurrency' | 'protectedCache' | 'verificationCache'>;
19
+ nodePieceDirectory?: NodePieceDirectoryOptions;
20
+ pieceDirectoryVerificationCache?: { directory: string; pieceDirectory: string };
21
+ }): Promise<{
4
22
  manifest: import('../formats/rdrr/index.js').RDRRManifest;
5
23
  storage: ReturnType<typeof import('./artifact-storage-context.js').createArtifactStorageContext>;
6
24
  /** Generic reader evidence. Gather evidence is separate, never silently merged. */
7
25
  getReceipt(): VerifiedPieceReceipt;
8
26
  getGatherReceipt: (() => Promise<import('./verified-piece-gather.js').VerifiedPieceGatherReceipt>) | null;
27
+ getPieceDirectoryReceipt?: () => NodePieceDirectoryReceipt;
9
28
  }>;