doppler-gpu 0.7.7 → 0.7.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/README.md +9 -17
- package/package.json +1 -1
- package/src/config/kernels/kernel-ref-digests.js +2 -1
- package/src/config/kernels/registry.json +47 -19
- package/src/config/schema/execution/kernel-warmup.schema.d.ts +14 -0
- package/src/config/schema/execution/kernel-warmup.schema.js +3 -0
- package/src/config/schema/platform/gpu-cache.schema.js +1 -1
- package/src/config/schema/runtime/buffer-pool.schema.js +2 -2
- package/src/config/schema/runtime/loading.schema.d.ts +5 -0
- package/src/config/schema/runtime/loading.schema.js +25 -2
- package/src/gpu/kernels/moe_route_q4_0_block_f32.wgsl +74 -0
- package/src/gpu/kernels/moe_route_q4_0_split_block_f32.wgsl +35 -18
- package/src/inference/pipelines/text/partitions/partition-execution.js +28 -1
- package/src/inference/pipelines/text/partitions/resident-partition-warmup.js +67 -0
- package/src/inference/pipelines/text/partitions/resident-partition.js +8 -2
- package/src/loader/experts/compact-expert-cache.js +20 -0
- package/src/loader/experts/loader-bindings.js +14 -3
- package/src/loader/experts/mapped-expert-staging.js +393 -0
- package/src/loader/experts/routed-expert-cache.d.ts +8 -0
- package/src/loader/experts/routed-expert-cache.js +12 -11
- package/src/loader/experts/routed-expert-prefill.js +77 -1
- package/src/loader/experts/routed-expert-source.js +1 -0
- package/src/loader/experts/tiered-expert-cache.js +66 -26
- package/src/storage/verified-piece-direct.js +153 -0
- package/src/storage/verified-piece-gather-lane.js +21 -3
- package/src/storage/verified-piece-gather-worker.js +235 -16
- package/src/storage/verified-piece-gather.d.ts +47 -0
- package/src/storage/verified-piece-gather.js +133 -17
- package/src/storage/verified-piece-reader.d.ts +8 -1
- package/src/storage/verified-piece-reader.js +16 -3
- package/src/storage/verified-piece-state-cache.d.ts +75 -0
- package/src/storage/verified-piece-state-cache.js +511 -0
- package/src/storage/verified-piece-storage.d.ts +20 -1
- package/src/storage/verified-piece-storage.js +207 -9
- package/src/version.js +1 -1
|
@@ -7,11 +7,18 @@ export interface VerifiedPieceReceipt {
|
|
|
7
7
|
acquisitionMs: number; verificationMs: number; copyMs: number;
|
|
8
8
|
/** Range output buffers owned by active reads, before transfer to the loader; not process RSS. */
|
|
9
9
|
activeReadBytes: number; peakReadBytes: number;
|
|
10
|
+
stateCacheHits?: number; stateCacheHitBytes?: number; stateCacheMisses?: number;
|
|
11
|
+
}
|
|
12
|
+
export interface StateVerifiedAcquisition {
|
|
13
|
+
bytes: Uint8Array | ArrayBuffer;
|
|
14
|
+
stateVerified?: boolean;
|
|
15
|
+
onAuthenticated?: () => void;
|
|
10
16
|
}
|
|
11
17
|
export interface VerifiedPieceReaderOptions {
|
|
12
18
|
manifestBytes: ArrayBuffer | Uint8Array<ArrayBuffer>; indexBytes: ArrayBuffer | Uint8Array<ArrayBuffer>; indexIdentity: string;
|
|
13
|
-
acquire: (piece: Readonly<VerifiedPiece>, options: { signal?: AbortSignal }) => Promise<Uint8Array | ArrayBuffer>;
|
|
19
|
+
acquire: (piece: Readonly<VerifiedPiece>, options: { signal?: AbortSignal }) => Promise<Uint8Array | ArrayBuffer | StateVerifiedAcquisition>;
|
|
14
20
|
signal?: AbortSignal;
|
|
21
|
+
allowStateVerifiedAcquire?: boolean;
|
|
15
22
|
}
|
|
16
23
|
export function createVerifiedPieceReader(options: VerifiedPieceReaderOptions): Promise<Readonly<{
|
|
17
24
|
manifest: import('../formats/rdrr/index.js').RDRRManifest;
|
|
@@ -7,7 +7,7 @@ const digest = async bytes => 'sha256:' + [...new Uint8Array(await crypto.subtle
|
|
|
7
7
|
* This reader authenticates bytes; it does not establish signed Capsule qualification.
|
|
8
8
|
* @type {import('./verified-piece-reader.js').createVerifiedPieceReader}
|
|
9
9
|
*/
|
|
10
|
-
export async function createVerifiedPieceReader({ manifestBytes, indexBytes, indexIdentity, acquire, signal }) {
|
|
10
|
+
export async function createVerifiedPieceReader({ manifestBytes, indexBytes, indexIdentity, acquire, signal, allowStateVerifiedAcquire = false }) {
|
|
11
11
|
for (const bytes of [manifestBytes, indexBytes]) {
|
|
12
12
|
if (!(bytes instanceof Uint8Array) && !(bytes instanceof ArrayBuffer)) throw new Error('Piece metadata requires bytes.');
|
|
13
13
|
}
|
|
@@ -121,13 +121,26 @@ export async function createVerifiedPieceReader({ manifestBytes, indexBytes, ind
|
|
|
121
121
|
if (piece.offset >= offset + length) break;
|
|
122
122
|
signal?.throwIfAborted();
|
|
123
123
|
const acquisitionStarted = performance.now();
|
|
124
|
-
const
|
|
124
|
+
const rawAcquired = await waitActive(acquire(Object.freeze({ ...piece, path }), { signal }));
|
|
125
125
|
receipt.acquisitionMs += performance.now() - acquisitionStarted;
|
|
126
126
|
const verificationStarted = performance.now();
|
|
127
|
+
const structured = allowStateVerifiedAcquire === true
|
|
128
|
+
&& rawAcquired !== null && typeof rawAcquired === 'object'
|
|
129
|
+
&& !(rawAcquired instanceof Uint8Array) && !(rawAcquired instanceof ArrayBuffer);
|
|
130
|
+
const received = structured ? rawAcquired.bytes : rawAcquired;
|
|
127
131
|
if (!(received instanceof Uint8Array) && !(received instanceof ArrayBuffer)
|
|
128
132
|
|| received.byteLength !== piece.size) throw new Error('Acquired piece integrity mismatch.');
|
|
129
133
|
const bytes = new Uint8Array(received instanceof Uint8Array ? received : new Uint8Array(received));
|
|
130
|
-
if (
|
|
134
|
+
if (structured && rawAcquired.stateVerified === true) {
|
|
135
|
+
receipt.stateCacheHits = (receipt.stateCacheHits ?? 0) + 1;
|
|
136
|
+
receipt.stateCacheHitBytes = (receipt.stateCacheHitBytes ?? 0) + bytes.byteLength;
|
|
137
|
+
} else {
|
|
138
|
+
if (await waitActive(digest(bytes)) !== piece.identity) throw new Error('Acquired piece integrity mismatch.');
|
|
139
|
+
if (structured) {
|
|
140
|
+
receipt.stateCacheMisses = (receipt.stateCacheMisses ?? 0) + 1;
|
|
141
|
+
rawAcquired.onAuthenticated?.();
|
|
142
|
+
}
|
|
143
|
+
}
|
|
131
144
|
receipt.verificationMs += performance.now() - verificationStarted;
|
|
132
145
|
signal?.throwIfAborted();
|
|
133
146
|
if (closed) throw new Error('Piece storage closed.');
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
interface VerifiedFileStat {
|
|
2
|
+
isFile(): boolean;
|
|
3
|
+
dev: bigint;
|
|
4
|
+
ino: bigint;
|
|
5
|
+
size: bigint;
|
|
6
|
+
mtimeNs: bigint;
|
|
7
|
+
ctimeNs: bigint;
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
export interface FileStateSnapshot {
|
|
11
|
+
device: string;
|
|
12
|
+
inode: string;
|
|
13
|
+
size: string;
|
|
14
|
+
mtimeNs: string;
|
|
15
|
+
ctimeNs: string;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export interface VerifiedPieceStateCacheOptions {
|
|
19
|
+
/** Optional absolute directory owned by the current user; symlinks are rejected. When omitted, verification bits live in memory for the open storage session only. */
|
|
20
|
+
directory?: string;
|
|
21
|
+
/** Optional idle-only background piece verification on gather workers (default false). */
|
|
22
|
+
backgroundVerify?: boolean;
|
|
23
|
+
/** Minimum idle delay in milliseconds before running a background verification step (default 50). */
|
|
24
|
+
backgroundIdleDelayMs?: number;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export interface WorkerFileStateDescriptor {
|
|
28
|
+
canonicalPath: string;
|
|
29
|
+
snapshot: FileStateSnapshot;
|
|
30
|
+
wordCount: number;
|
|
31
|
+
pieceCount: number;
|
|
32
|
+
sharedBuffer: SharedArrayBuffer;
|
|
33
|
+
stateFilePath: string | null;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export interface WorkerStateCachePayload {
|
|
37
|
+
directory: string | null;
|
|
38
|
+
backgroundVerify: boolean;
|
|
39
|
+
backgroundIdleDelayMs: number;
|
|
40
|
+
files: Record<string, WorkerFileStateDescriptor>;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
export function matchesFileStateSnapshot(stat: VerifiedFileStat, snapshot: FileStateSnapshot): boolean;
|
|
44
|
+
|
|
45
|
+
export function prepareVerifiedPieceStateCache(options: {
|
|
46
|
+
rawPolicy: VerifiedPieceStateCacheOptions | undefined;
|
|
47
|
+
manifestIdentity: string;
|
|
48
|
+
indexIdentity: string;
|
|
49
|
+
index: { files: ReadonlyArray<{ path: string; size: number; pieces: Array<{ offset: number; size: number; identity: string }> }> };
|
|
50
|
+
files: Record<string, string>;
|
|
51
|
+
}): Promise<Readonly<{
|
|
52
|
+
directory: string | null;
|
|
53
|
+
backgroundVerify: boolean;
|
|
54
|
+
backgroundIdleDelayMs: number;
|
|
55
|
+
workerPayload: Readonly<WorkerStateCachePayload>;
|
|
56
|
+
checkFileStat(piecePath: string, stat: VerifiedFileStat): boolean;
|
|
57
|
+
isPieceVerified(piecePath: string, offset: number): boolean;
|
|
58
|
+
markPieceVerified(piecePath: string, offset: number): boolean;
|
|
59
|
+
scheduleFlush(): void;
|
|
60
|
+
flushSync(): void;
|
|
61
|
+
close(): Promise<void>;
|
|
62
|
+
}> | undefined>;
|
|
63
|
+
|
|
64
|
+
export function openPieceDirectoryStateCache(options: {
|
|
65
|
+
directory: string;
|
|
66
|
+
pieceDirectory: string;
|
|
67
|
+
manifestIdentity: string;
|
|
68
|
+
indexIdentity: string;
|
|
69
|
+
index: { files: ReadonlyArray<{ path: string; size: number; pieces: Array<{ offset: number; size: number; identity: string }> }> };
|
|
70
|
+
}): Promise<Readonly<{
|
|
71
|
+
directory: string;
|
|
72
|
+
isVerified(piecePath: string, offset: number, stat: VerifiedFileStat): boolean;
|
|
73
|
+
markVerified(piecePath: string, offset: number, stat: VerifiedFileStat): void;
|
|
74
|
+
close(): Promise<void>;
|
|
75
|
+
}>>;
|
|
@@ -0,0 +1,511 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
import {
|
|
3
|
+
closeSync, fstatSync, lstatSync, mkdirSync, openSync, readSync,
|
|
4
|
+
realpathSync, renameSync, unlinkSync, writeSync, constants,
|
|
5
|
+
} from 'node:fs';
|
|
6
|
+
import { open, rename, unlink } from 'node:fs/promises';
|
|
7
|
+
import { isAbsolute, join } from 'node:path';
|
|
8
|
+
|
|
9
|
+
const FILE_STATE_SCHEMA = 'doppler.verified-piece-file-state/v1';
|
|
10
|
+
const PIECE_DIR_STATE_SCHEMA = 'doppler.verified-piece-dir-state/v1';
|
|
11
|
+
const HEADER_BYTES = 32;
|
|
12
|
+
|
|
13
|
+
/** @param {string | Uint8Array} input */
|
|
14
|
+
const sha256Bytes = input => createHash('sha256').update(input).digest();
|
|
15
|
+
|
|
16
|
+
/** @param {string | Uint8Array} input */
|
|
17
|
+
const sha256Hex = input => createHash('sha256').update(input).digest('hex');
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* @param {Uint8Array} a
|
|
21
|
+
* @param {Uint8Array} b
|
|
22
|
+
*/
|
|
23
|
+
function equalBytes(a, b) {
|
|
24
|
+
if (a.byteLength !== b.byteLength) return false;
|
|
25
|
+
for (let i = 0; i < a.byteLength; i++) {
|
|
26
|
+
if (a[i] !== b[i]) return false;
|
|
27
|
+
}
|
|
28
|
+
return true;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* @param {string} directory
|
|
33
|
+
* @param {number | null} expectedUid
|
|
34
|
+
*/
|
|
35
|
+
function ensureOwnedStateDirectory(directory, expectedUid) {
|
|
36
|
+
if (typeof directory !== 'string' || !directory.trim() || !isAbsolute(directory)) {
|
|
37
|
+
throw new Error('Piece state verification cache requires an absolute directory.');
|
|
38
|
+
}
|
|
39
|
+
mkdirSync(directory, { recursive: true, mode: 0o700 });
|
|
40
|
+
const st = lstatSync(directory);
|
|
41
|
+
if (!st.isDirectory() || (st.mode & 0o022) !== 0 || (expectedUid !== null && st.uid !== expectedUid)) {
|
|
42
|
+
throw new Error('Piece state verification cache directory must be an owned non-symlink directory.');
|
|
43
|
+
}
|
|
44
|
+
return realpathSync(directory);
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
/**
|
|
48
|
+
* @param {Array<{ offset: number, size: number }>} pieces
|
|
49
|
+
* @param {number} offset
|
|
50
|
+
*/
|
|
51
|
+
function findPieceIndex(pieces, offset) {
|
|
52
|
+
let low = 0;
|
|
53
|
+
let high = pieces.length;
|
|
54
|
+
while (low < high) {
|
|
55
|
+
const mid = (low + high) >>> 1;
|
|
56
|
+
const piece = pieces[mid];
|
|
57
|
+
if (piece.offset + piece.size <= offset) low = mid + 1;
|
|
58
|
+
else high = mid;
|
|
59
|
+
}
|
|
60
|
+
return low < pieces.length && pieces[low].offset === offset ? low : -1;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* @param {string} stateFilePath
|
|
65
|
+
* @param {number} payloadBytes
|
|
66
|
+
* @param {number | null} expectedUid
|
|
67
|
+
*/
|
|
68
|
+
function tryLoadVerifiedPayload(stateFilePath, payloadBytes, expectedUid) {
|
|
69
|
+
let fd = -1;
|
|
70
|
+
try {
|
|
71
|
+
fd = openSync(stateFilePath, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
|
|
72
|
+
const st = fstatSync(fd);
|
|
73
|
+
if (!st.isFile() || (st.mode & 0o022) !== 0 || st.size !== HEADER_BYTES + payloadBytes
|
|
74
|
+
|| (expectedUid !== null && st.uid !== expectedUid)) {
|
|
75
|
+
try { unlinkSync(stateFilePath); } catch {}
|
|
76
|
+
return null;
|
|
77
|
+
}
|
|
78
|
+
const raw = new Uint8Array(HEADER_BYTES + payloadBytes);
|
|
79
|
+
let read = 0;
|
|
80
|
+
while (read < raw.byteLength) {
|
|
81
|
+
const count = readSync(fd, raw, read, raw.byteLength - read, read);
|
|
82
|
+
if (!count) {
|
|
83
|
+
try { unlinkSync(stateFilePath); } catch {}
|
|
84
|
+
return null;
|
|
85
|
+
}
|
|
86
|
+
read += count;
|
|
87
|
+
}
|
|
88
|
+
const header = raw.subarray(0, HEADER_BYTES);
|
|
89
|
+
const payload = raw.subarray(HEADER_BYTES);
|
|
90
|
+
if (!equalBytes(header, sha256Bytes(payload))) {
|
|
91
|
+
try { unlinkSync(stateFilePath); } catch {}
|
|
92
|
+
return null;
|
|
93
|
+
}
|
|
94
|
+
return payload;
|
|
95
|
+
} catch {
|
|
96
|
+
return null;
|
|
97
|
+
} finally {
|
|
98
|
+
if (fd !== -1) {
|
|
99
|
+
try { closeSync(fd); } catch {}
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
let tempCounter = 0;
|
|
105
|
+
|
|
106
|
+
/**
|
|
107
|
+
* @param {string} stateFilePath
|
|
108
|
+
* @param {Uint8Array} payloadSnapshot
|
|
109
|
+
*/
|
|
110
|
+
async function writeVerifiedPayloadAsync(stateFilePath, payloadSnapshot) {
|
|
111
|
+
const tmpPath = `${stateFilePath}.tmp.${process.pid}.${++tempCounter}`;
|
|
112
|
+
const out = new Uint8Array(HEADER_BYTES + payloadSnapshot.byteLength);
|
|
113
|
+
out.set(sha256Bytes(payloadSnapshot), 0);
|
|
114
|
+
out.set(payloadSnapshot, HEADER_BYTES);
|
|
115
|
+
let handle = null;
|
|
116
|
+
try {
|
|
117
|
+
handle = await open(tmpPath, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY | constants.O_NOFOLLOW, 0o600);
|
|
118
|
+
let written = 0;
|
|
119
|
+
while (written < out.byteLength) {
|
|
120
|
+
const res = await handle.write(out, written, out.byteLength - written, written);
|
|
121
|
+
if (!res.bytesWritten) throw new Error('Incomplete state cache write.');
|
|
122
|
+
written += res.bytesWritten;
|
|
123
|
+
}
|
|
124
|
+
await handle.close();
|
|
125
|
+
handle = null;
|
|
126
|
+
await rename(tmpPath, stateFilePath);
|
|
127
|
+
} catch {
|
|
128
|
+
if (handle) {
|
|
129
|
+
try { await handle.close(); } catch {}
|
|
130
|
+
}
|
|
131
|
+
try { await unlink(tmpPath); } catch {}
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/**
|
|
136
|
+
* @param {string} stateFilePath
|
|
137
|
+
* @param {Uint8Array} payloadSnapshot
|
|
138
|
+
*/
|
|
139
|
+
function writeVerifiedPayloadSync(stateFilePath, payloadSnapshot) {
|
|
140
|
+
const tmpPath = `${stateFilePath}.tmp.${process.pid}.${++tempCounter}`;
|
|
141
|
+
const out = new Uint8Array(HEADER_BYTES + payloadSnapshot.byteLength);
|
|
142
|
+
out.set(sha256Bytes(payloadSnapshot), 0);
|
|
143
|
+
out.set(payloadSnapshot, HEADER_BYTES);
|
|
144
|
+
let fd = -1;
|
|
145
|
+
try {
|
|
146
|
+
fd = openSync(tmpPath, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY | constants.O_NOFOLLOW, 0o600);
|
|
147
|
+
let written = 0;
|
|
148
|
+
while (written < out.byteLength) {
|
|
149
|
+
written += writeSync(fd, out, written, out.byteLength - written, written);
|
|
150
|
+
}
|
|
151
|
+
closeSync(fd);
|
|
152
|
+
fd = -1;
|
|
153
|
+
renameSync(tmpPath, stateFilePath);
|
|
154
|
+
} catch {
|
|
155
|
+
if (fd !== -1) {
|
|
156
|
+
try { closeSync(fd); } catch {}
|
|
157
|
+
}
|
|
158
|
+
try { unlinkSync(tmpPath); } catch {}
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/**
|
|
163
|
+
* @param {import('node:fs').BigIntStats} stat
|
|
164
|
+
* @param {{ device: string, inode: string, size: string, mtimeNs: string, ctimeNs: string }} snapshot
|
|
165
|
+
*/
|
|
166
|
+
export function matchesFileStateSnapshot(stat, snapshot) {
|
|
167
|
+
return stat.isFile()
|
|
168
|
+
&& stat.dev.toString() === snapshot.device
|
|
169
|
+
&& stat.ino.toString() === snapshot.inode
|
|
170
|
+
&& stat.size.toString() === snapshot.size
|
|
171
|
+
&& stat.mtimeNs.toString() === snapshot.mtimeNs
|
|
172
|
+
&& stat.ctimeNs.toString() === snapshot.ctimeNs;
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
/**
|
|
176
|
+
* Prepare shared in-memory + optional persistent per-file-state verification bitsets
|
|
177
|
+
* across main-thread readers and worker gather lanes.
|
|
178
|
+
* @type {typeof import('./verified-piece-state-cache.js').prepareVerifiedPieceStateCache}
|
|
179
|
+
*/
|
|
180
|
+
export async function prepareVerifiedPieceStateCache(options) {
|
|
181
|
+
const { rawPolicy, manifestIdentity, indexIdentity, index, files } = options;
|
|
182
|
+
if (rawPolicy === undefined) return undefined;
|
|
183
|
+
if (!rawPolicy || typeof rawPolicy !== 'object' || Array.isArray(rawPolicy)) {
|
|
184
|
+
throw new Error('Piece state verification cache policy must be an explicit object.');
|
|
185
|
+
}
|
|
186
|
+
const backgroundVerify = rawPolicy.backgroundVerify === true;
|
|
187
|
+
const backgroundIdleDelayMs = rawPolicy.backgroundIdleDelayMs ?? 50;
|
|
188
|
+
if (typeof backgroundVerify !== 'boolean'
|
|
189
|
+
|| !Number.isSafeInteger(backgroundIdleDelayMs) || backgroundIdleDelayMs < 0 || backgroundIdleDelayMs > 60_000) {
|
|
190
|
+
throw new Error('Piece state verification cache requires valid backgroundVerify and backgroundIdleDelayMs.');
|
|
191
|
+
}
|
|
192
|
+
const expectedUid = typeof process.geteuid === 'function' ? process.geteuid() : null;
|
|
193
|
+
const directory = rawPolicy.directory === undefined
|
|
194
|
+
? null
|
|
195
|
+
: ensureOwnedStateDirectory(rawPolicy.directory, expectedUid);
|
|
196
|
+
|
|
197
|
+
/** @type {Record<string, import('./verified-piece-state-cache.js').WorkerFileStateDescriptor>} */
|
|
198
|
+
const workerFiles = {};
|
|
199
|
+
/** @type {Map<string, { pieces: Array<{ offset: number, size: number }>, words: Int32Array, wordCount: number, snapshot: import('./verified-piece-state-cache.js').FileStateSnapshot, canonicalPath: string, stateFilePath: string | null, lastSavedSeq: number }>} */
|
|
200
|
+
const entries = new Map();
|
|
201
|
+
|
|
202
|
+
for (const file of index.files) {
|
|
203
|
+
const mapped = files?.[file.path];
|
|
204
|
+
if (typeof mapped !== 'string') continue;
|
|
205
|
+
let st;
|
|
206
|
+
try {
|
|
207
|
+
st = lstatSync(mapped, { bigint: true });
|
|
208
|
+
} catch {
|
|
209
|
+
continue;
|
|
210
|
+
}
|
|
211
|
+
if (!st.isFile() || st.size !== BigInt(file.size)) continue;
|
|
212
|
+
const snapshot = {
|
|
213
|
+
device: st.dev.toString(),
|
|
214
|
+
inode: st.ino.toString(),
|
|
215
|
+
size: st.size.toString(),
|
|
216
|
+
mtimeNs: st.mtimeNs.toString(),
|
|
217
|
+
ctimeNs: st.ctimeNs.toString(),
|
|
218
|
+
};
|
|
219
|
+
const wordCount = Math.max(1, Math.ceil(file.pieces.length / 32));
|
|
220
|
+
const sharedBuffer = new SharedArrayBuffer((wordCount + 2) * 4);
|
|
221
|
+
const words = new Int32Array(sharedBuffer);
|
|
222
|
+
let stateFilePath = null;
|
|
223
|
+
if (directory !== null) {
|
|
224
|
+
const stateKey = sha256Hex(JSON.stringify({
|
|
225
|
+
schema: FILE_STATE_SCHEMA,
|
|
226
|
+
manifestIdentity,
|
|
227
|
+
indexIdentity,
|
|
228
|
+
piecePath: file.path,
|
|
229
|
+
canonicalPath: mapped,
|
|
230
|
+
uid: expectedUid,
|
|
231
|
+
...snapshot,
|
|
232
|
+
pieceCount: file.pieces.length,
|
|
233
|
+
}));
|
|
234
|
+
stateFilePath = join(directory, `${stateKey}.state`);
|
|
235
|
+
const loaded = tryLoadVerifiedPayload(stateFilePath, wordCount * 4, expectedUid);
|
|
236
|
+
if (loaded) {
|
|
237
|
+
new Uint8Array(sharedBuffer, 0, wordCount * 4).set(loaded);
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
workerFiles[file.path] = {
|
|
241
|
+
canonicalPath: mapped,
|
|
242
|
+
snapshot,
|
|
243
|
+
wordCount,
|
|
244
|
+
pieceCount: file.pieces.length,
|
|
245
|
+
sharedBuffer,
|
|
246
|
+
stateFilePath,
|
|
247
|
+
};
|
|
248
|
+
entries.set(file.path, {
|
|
249
|
+
pieces: file.pieces,
|
|
250
|
+
words,
|
|
251
|
+
wordCount,
|
|
252
|
+
snapshot,
|
|
253
|
+
canonicalPath: mapped,
|
|
254
|
+
stateFilePath,
|
|
255
|
+
lastSavedSeq: 0,
|
|
256
|
+
});
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
/** @type {ReturnType<typeof setTimeout> | null} */
|
|
260
|
+
let flushTimer = null;
|
|
261
|
+
/** @type {Promise<void>} */
|
|
262
|
+
let flushChain = Promise.resolve();
|
|
263
|
+
let closed = false;
|
|
264
|
+
|
|
265
|
+
const flushDirty = () => {
|
|
266
|
+
if (directory === null) return Promise.resolve();
|
|
267
|
+
flushChain = flushChain.then(async () => {
|
|
268
|
+
for (const entry of entries.values()) {
|
|
269
|
+
if (!entry.stateFilePath) continue;
|
|
270
|
+
if (Atomics.load(entry.words, entry.wordCount + 1) !== 0) continue;
|
|
271
|
+
const seq = Atomics.load(entry.words, entry.wordCount);
|
|
272
|
+
if (seq === entry.lastSavedSeq) continue;
|
|
273
|
+
const snapshotBytes = new Uint8Array(entry.wordCount * 4);
|
|
274
|
+
for (let w = 0; w < entry.wordCount; w++) {
|
|
275
|
+
const val = Atomics.load(entry.words, w);
|
|
276
|
+
snapshotBytes[w * 4] = val & 0xff;
|
|
277
|
+
snapshotBytes[w * 4 + 1] = (val >>> 8) & 0xff;
|
|
278
|
+
snapshotBytes[w * 4 + 2] = (val >>> 16) & 0xff;
|
|
279
|
+
snapshotBytes[w * 4 + 3] = (val >>> 24) & 0xff;
|
|
280
|
+
}
|
|
281
|
+
await writeVerifiedPayloadAsync(entry.stateFilePath, snapshotBytes);
|
|
282
|
+
entry.lastSavedSeq = seq;
|
|
283
|
+
}
|
|
284
|
+
}).catch(() => {});
|
|
285
|
+
return flushChain;
|
|
286
|
+
};
|
|
287
|
+
|
|
288
|
+
const scheduleFlush = () => {
|
|
289
|
+
if (directory === null || closed || flushTimer !== null) return;
|
|
290
|
+
flushTimer = setTimeout(() => {
|
|
291
|
+
flushTimer = null;
|
|
292
|
+
void flushDirty();
|
|
293
|
+
}, 25);
|
|
294
|
+
flushTimer.unref?.();
|
|
295
|
+
};
|
|
296
|
+
|
|
297
|
+
/**
|
|
298
|
+
* Check whether a mapped full file currently matches its admitted snapshot,
|
|
299
|
+
* invalidating cached bits immediately if any stat field changed.
|
|
300
|
+
* @param {string} piecePath
|
|
301
|
+
* @param {import('node:fs').BigIntStats} stat
|
|
302
|
+
*/
|
|
303
|
+
const checkFileStat = (piecePath, stat) => {
|
|
304
|
+
const entry = entries.get(piecePath);
|
|
305
|
+
if (!entry) return false;
|
|
306
|
+
if (!matchesFileStateSnapshot(stat, entry.snapshot)) {
|
|
307
|
+
Atomics.store(entry.words, entry.wordCount + 1, 1);
|
|
308
|
+
return false;
|
|
309
|
+
}
|
|
310
|
+
return Atomics.load(entry.words, entry.wordCount + 1) === 0;
|
|
311
|
+
};
|
|
312
|
+
|
|
313
|
+
/**
|
|
314
|
+
* @param {string} piecePath
|
|
315
|
+
* @param {number} offset
|
|
316
|
+
*/
|
|
317
|
+
const isPieceVerified = (piecePath, offset) => {
|
|
318
|
+
const entry = entries.get(piecePath);
|
|
319
|
+
if (!entry || Atomics.load(entry.words, entry.wordCount + 1) !== 0) return false;
|
|
320
|
+
const idx = findPieceIndex(entry.pieces, offset);
|
|
321
|
+
if (idx < 0) return false;
|
|
322
|
+
return (Atomics.load(entry.words, idx >>> 5) & (1 << (idx & 31))) !== 0;
|
|
323
|
+
};
|
|
324
|
+
|
|
325
|
+
/**
|
|
326
|
+
* @param {string} piecePath
|
|
327
|
+
* @param {number} offset
|
|
328
|
+
*/
|
|
329
|
+
const markPieceVerified = (piecePath, offset) => {
|
|
330
|
+
const entry = entries.get(piecePath);
|
|
331
|
+
if (!entry || Atomics.load(entry.words, entry.wordCount + 1) !== 0) return false;
|
|
332
|
+
const idx = findPieceIndex(entry.pieces, offset);
|
|
333
|
+
if (idx < 0) return false;
|
|
334
|
+
const mask = 1 << (idx & 31);
|
|
335
|
+
const prev = Atomics.or(entry.words, idx >>> 5, mask);
|
|
336
|
+
if ((prev & mask) === 0) {
|
|
337
|
+
Atomics.add(entry.words, entry.wordCount, 1);
|
|
338
|
+
scheduleFlush();
|
|
339
|
+
return true;
|
|
340
|
+
}
|
|
341
|
+
return false;
|
|
342
|
+
};
|
|
343
|
+
|
|
344
|
+
const close = async () => {
|
|
345
|
+
closed = true;
|
|
346
|
+
if (flushTimer !== null) {
|
|
347
|
+
clearTimeout(flushTimer);
|
|
348
|
+
flushTimer = null;
|
|
349
|
+
}
|
|
350
|
+
await flushDirty();
|
|
351
|
+
};
|
|
352
|
+
|
|
353
|
+
return Object.freeze({
|
|
354
|
+
directory,
|
|
355
|
+
backgroundVerify,
|
|
356
|
+
backgroundIdleDelayMs,
|
|
357
|
+
workerPayload: Object.freeze({
|
|
358
|
+
directory,
|
|
359
|
+
backgroundVerify,
|
|
360
|
+
backgroundIdleDelayMs,
|
|
361
|
+
files: workerFiles,
|
|
362
|
+
}),
|
|
363
|
+
checkFileStat,
|
|
364
|
+
isPieceVerified,
|
|
365
|
+
markPieceVerified,
|
|
366
|
+
scheduleFlush,
|
|
367
|
+
flushSync() {
|
|
368
|
+
if (directory === null) return;
|
|
369
|
+
for (const entry of entries.values()) {
|
|
370
|
+
if (!entry.stateFilePath || Atomics.load(entry.words, entry.wordCount + 1) !== 0) continue;
|
|
371
|
+
const seq = Atomics.load(entry.words, entry.wordCount);
|
|
372
|
+
if (seq === entry.lastSavedSeq) continue;
|
|
373
|
+
const snapshotBytes = new Uint8Array(entry.wordCount * 4);
|
|
374
|
+
for (let w = 0; w < entry.wordCount; w++) {
|
|
375
|
+
const val = Atomics.load(entry.words, w);
|
|
376
|
+
snapshotBytes[w * 4] = val & 0xff;
|
|
377
|
+
snapshotBytes[w * 4 + 1] = (val >>> 8) & 0xff;
|
|
378
|
+
snapshotBytes[w * 4 + 2] = (val >>> 16) & 0xff;
|
|
379
|
+
snapshotBytes[w * 4 + 3] = (val >>> 24) & 0xff;
|
|
380
|
+
}
|
|
381
|
+
writeVerifiedPayloadSync(entry.stateFilePath, snapshotBytes);
|
|
382
|
+
entry.lastSavedSeq = seq;
|
|
383
|
+
}
|
|
384
|
+
},
|
|
385
|
+
close,
|
|
386
|
+
});
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
/**
|
|
390
|
+
* Persistent per-piece-file `(dev, ino, size, mtimeNs, ctimeNs)` verification cache
|
|
391
|
+
* for partial piece-cache directories (`<pieceDirectory>/<path>.<offset>`).
|
|
392
|
+
* @type {typeof import('./verified-piece-state-cache.js').openPieceDirectoryStateCache}
|
|
393
|
+
*/
|
|
394
|
+
export async function openPieceDirectoryStateCache(options) {
|
|
395
|
+
const { directory: rawDir, pieceDirectory, manifestIdentity, indexIdentity, index } = options;
|
|
396
|
+
const expectedUid = typeof process.geteuid === 'function' ? process.geteuid() : null;
|
|
397
|
+
const directory = ensureOwnedStateDirectory(rawDir, expectedUid);
|
|
398
|
+
const canonicalPieceDir = realpathSync(pieceDirectory);
|
|
399
|
+
const dirStat = lstatSync(canonicalPieceDir, { bigint: true });
|
|
400
|
+
if (!dirStat.isDirectory()) throw new Error('Piece directory must be a non-symlink directory.');
|
|
401
|
+
|
|
402
|
+
/** @type {Map<string, { baseIndex: number, pieces: Array<{ offset: number, size: number }> }>} */
|
|
403
|
+
const fileLayout = new Map();
|
|
404
|
+
let totalPieces = 0;
|
|
405
|
+
for (const file of index.files) {
|
|
406
|
+
fileLayout.set(file.path, { baseIndex: totalPieces, pieces: file.pieces });
|
|
407
|
+
totalPieces += file.pieces.length;
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
// 4 x BigUint64 per piece: [dev, ino, mtimeNs, ctimeNs]; piece size is fixed in index.
|
|
411
|
+
const payloadBytes = totalPieces * 32;
|
|
412
|
+
const tableBuffer = new ArrayBuffer(payloadBytes);
|
|
413
|
+
const table = new BigUint64Array(tableBuffer);
|
|
414
|
+
const stateKey = sha256Hex(JSON.stringify({
|
|
415
|
+
schema: PIECE_DIR_STATE_SCHEMA,
|
|
416
|
+
manifestIdentity,
|
|
417
|
+
indexIdentity,
|
|
418
|
+
pieceDirectory: canonicalPieceDir,
|
|
419
|
+
uid: expectedUid,
|
|
420
|
+
totalPieces,
|
|
421
|
+
}));
|
|
422
|
+
const stateFilePath = join(directory, `${stateKey}.pstate`);
|
|
423
|
+
const loaded = tryLoadVerifiedPayload(stateFilePath, payloadBytes, expectedUid);
|
|
424
|
+
if (loaded) {
|
|
425
|
+
new Uint8Array(tableBuffer).set(loaded);
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
let dirty = false;
|
|
429
|
+
let savedDirtyGen = 0;
|
|
430
|
+
let dirtyGen = 0;
|
|
431
|
+
/** @type {ReturnType<typeof setTimeout> | null} */
|
|
432
|
+
let flushTimer = null;
|
|
433
|
+
/** @type {Promise<void>} */
|
|
434
|
+
let flushChain = Promise.resolve();
|
|
435
|
+
let closed = false;
|
|
436
|
+
|
|
437
|
+
/**
|
|
438
|
+
* @param {string} piecePath
|
|
439
|
+
* @param {number} offset
|
|
440
|
+
*/
|
|
441
|
+
const resolveSlot = (piecePath, offset) => {
|
|
442
|
+
const file = fileLayout.get(piecePath);
|
|
443
|
+
if (!file) return -1;
|
|
444
|
+
const localIdx = findPieceIndex(file.pieces, offset);
|
|
445
|
+
if (localIdx < 0) return -1;
|
|
446
|
+
return (file.baseIndex + localIdx) * 4;
|
|
447
|
+
};
|
|
448
|
+
|
|
449
|
+
const flushDirty = () => {
|
|
450
|
+
if (!dirty || savedDirtyGen === dirtyGen) return flushChain;
|
|
451
|
+
const targetGen = dirtyGen;
|
|
452
|
+
const snapshot = new Uint8Array(tableBuffer).slice();
|
|
453
|
+
flushChain = flushChain.then(async () => {
|
|
454
|
+
await writeVerifiedPayloadAsync(stateFilePath, snapshot);
|
|
455
|
+
savedDirtyGen = targetGen;
|
|
456
|
+
}).catch(() => {});
|
|
457
|
+
return flushChain;
|
|
458
|
+
};
|
|
459
|
+
|
|
460
|
+
const scheduleFlush = () => {
|
|
461
|
+
if (closed || flushTimer !== null) return;
|
|
462
|
+
flushTimer = setTimeout(() => {
|
|
463
|
+
flushTimer = null;
|
|
464
|
+
void flushDirty();
|
|
465
|
+
}, 50);
|
|
466
|
+
flushTimer.unref?.();
|
|
467
|
+
};
|
|
468
|
+
|
|
469
|
+
return Object.freeze({
|
|
470
|
+
directory,
|
|
471
|
+
/**
|
|
472
|
+
* @param {string} piecePath
|
|
473
|
+
* @param {number} offset
|
|
474
|
+
* @param {import('node:fs').BigIntStats} stat
|
|
475
|
+
*/
|
|
476
|
+
isVerified(piecePath, offset, stat) {
|
|
477
|
+
const slot = resolveSlot(piecePath, offset);
|
|
478
|
+
if (slot < 0) return false;
|
|
479
|
+
const ino = table[slot + 1];
|
|
480
|
+
if (ino === 0n) return false;
|
|
481
|
+
return table[slot] === BigInt.asUintN(64, stat.dev)
|
|
482
|
+
&& ino === BigInt.asUintN(64, stat.ino)
|
|
483
|
+
&& table[slot + 2] === BigInt.asUintN(64, stat.mtimeNs)
|
|
484
|
+
&& table[slot + 3] === BigInt.asUintN(64, stat.ctimeNs);
|
|
485
|
+
},
|
|
486
|
+
/**
|
|
487
|
+
* @param {string} piecePath
|
|
488
|
+
* @param {number} offset
|
|
489
|
+
* @param {import('node:fs').BigIntStats} stat
|
|
490
|
+
*/
|
|
491
|
+
markVerified(piecePath, offset, stat) {
|
|
492
|
+
const slot = resolveSlot(piecePath, offset);
|
|
493
|
+
if (slot < 0) return;
|
|
494
|
+
table[slot] = BigInt.asUintN(64, stat.dev);
|
|
495
|
+
table[slot + 1] = BigInt.asUintN(64, stat.ino);
|
|
496
|
+
table[slot + 2] = BigInt.asUintN(64, stat.mtimeNs);
|
|
497
|
+
table[slot + 3] = BigInt.asUintN(64, stat.ctimeNs);
|
|
498
|
+
dirty = true;
|
|
499
|
+
dirtyGen++;
|
|
500
|
+
scheduleFlush();
|
|
501
|
+
},
|
|
502
|
+
async close() {
|
|
503
|
+
closed = true;
|
|
504
|
+
if (flushTimer !== null) {
|
|
505
|
+
clearTimeout(flushTimer);
|
|
506
|
+
flushTimer = null;
|
|
507
|
+
}
|
|
508
|
+
await flushDirty();
|
|
509
|
+
},
|
|
510
|
+
});
|
|
511
|
+
}
|
|
@@ -1,9 +1,28 @@
|
|
|
1
1
|
import type { VerifiedPieceReaderOptions, VerifiedPieceReceipt } from './verified-piece-reader.js';
|
|
2
2
|
export type { VerifiedPiece, VerifiedPieceReceipt } from './verified-piece-reader.js';
|
|
3
|
-
export
|
|
3
|
+
export interface NodePieceDirectoryOptions {
|
|
4
|
+
directory: string;
|
|
5
|
+
maxPieceBytes: number;
|
|
6
|
+
maxActiveReadBytes: number;
|
|
7
|
+
verificationCache?: { directory: string };
|
|
8
|
+
}
|
|
9
|
+
export interface NodePieceDirectoryReceipt {
|
|
10
|
+
reads: number;
|
|
11
|
+
activeReadBytes: number;
|
|
12
|
+
peakReadBytes: number;
|
|
13
|
+
missingPieces: number;
|
|
14
|
+
pendingDescriptorCloses: number;
|
|
15
|
+
}
|
|
16
|
+
export function createVerifiedPieceStorage(options: Omit<VerifiedPieceReaderOptions, 'acquire'> & {
|
|
17
|
+
acquire?: VerifiedPieceReaderOptions['acquire'];
|
|
18
|
+
nodeGather?: Pick<import('./verified-piece-gather.js').VerifiedPieceGatherOptions, 'files' | 'fileIdentities' | 'maxPartBytes' | 'maxRanges' | 'memoryBudgetBytes' | 'concurrency' | 'protectedCache' | 'verificationCache'>;
|
|
19
|
+
nodePieceDirectory?: NodePieceDirectoryOptions;
|
|
20
|
+
pieceDirectoryVerificationCache?: { directory: string; pieceDirectory: string };
|
|
21
|
+
}): Promise<{
|
|
4
22
|
manifest: import('../formats/rdrr/index.js').RDRRManifest;
|
|
5
23
|
storage: ReturnType<typeof import('./artifact-storage-context.js').createArtifactStorageContext>;
|
|
6
24
|
/** Generic reader evidence. Gather evidence is separate, never silently merged. */
|
|
7
25
|
getReceipt(): VerifiedPieceReceipt;
|
|
8
26
|
getGatherReceipt: (() => Promise<import('./verified-piece-gather.js').VerifiedPieceGatherReceipt>) | null;
|
|
27
|
+
getPieceDirectoryReceipt?: () => NodePieceDirectoryReceipt;
|
|
9
28
|
}>;
|