doppler-gpu 0.7.7 → 0.7.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/README.md +9 -17
- package/package.json +1 -1
- package/src/config/kernels/kernel-ref-digests.js +2 -1
- package/src/config/kernels/registry.json +47 -19
- package/src/config/schema/execution/kernel-warmup.schema.d.ts +14 -0
- package/src/config/schema/execution/kernel-warmup.schema.js +3 -0
- package/src/config/schema/platform/gpu-cache.schema.js +1 -1
- package/src/config/schema/runtime/buffer-pool.schema.js +2 -2
- package/src/config/schema/runtime/loading.schema.d.ts +5 -0
- package/src/config/schema/runtime/loading.schema.js +25 -2
- package/src/gpu/kernels/moe_route_q4_0_block_f32.wgsl +74 -0
- package/src/gpu/kernels/moe_route_q4_0_split_block_f32.wgsl +35 -18
- package/src/inference/pipelines/text/partitions/partition-execution.js +28 -1
- package/src/inference/pipelines/text/partitions/resident-partition-warmup.js +67 -0
- package/src/inference/pipelines/text/partitions/resident-partition.js +8 -2
- package/src/loader/experts/compact-expert-cache.js +20 -0
- package/src/loader/experts/loader-bindings.js +14 -3
- package/src/loader/experts/mapped-expert-staging.js +393 -0
- package/src/loader/experts/routed-expert-cache.d.ts +8 -0
- package/src/loader/experts/routed-expert-cache.js +12 -11
- package/src/loader/experts/routed-expert-prefill.js +77 -1
- package/src/loader/experts/routed-expert-source.js +1 -0
- package/src/loader/experts/tiered-expert-cache.js +66 -26
- package/src/storage/verified-piece-direct.js +153 -0
- package/src/storage/verified-piece-gather-lane.js +21 -3
- package/src/storage/verified-piece-gather-worker.js +235 -16
- package/src/storage/verified-piece-gather.d.ts +47 -0
- package/src/storage/verified-piece-gather.js +133 -17
- package/src/storage/verified-piece-reader.d.ts +8 -1
- package/src/storage/verified-piece-reader.js +16 -3
- package/src/storage/verified-piece-state-cache.d.ts +75 -0
- package/src/storage/verified-piece-state-cache.js +511 -0
- package/src/storage/verified-piece-storage.d.ts +20 -1
- package/src/storage/verified-piece-storage.js +207 -9
- package/src/version.js +1 -1
|
@@ -11,8 +11,191 @@ export async function createVerifiedPieceStorage(options) {
|
|
|
11
11
|
const pinned = { ...options, manifestBytes, indexBytes };
|
|
12
12
|
const nodeGather = options.nodeGather ? { ...options.nodeGather, files: { ...options.nodeGather.files },
|
|
13
13
|
...(options.nodeGather.fileIdentities ? { fileIdentities: structuredClone(options.nodeGather.fileIdentities) } : {}),
|
|
14
|
-
...(options.nodeGather.protectedCache ? { protectedCache: structuredClone(options.nodeGather.protectedCache) } : {})
|
|
15
|
-
|
|
14
|
+
...(options.nodeGather.protectedCache ? { protectedCache: structuredClone(options.nodeGather.protectedCache) } : {}),
|
|
15
|
+
...(options.nodeGather.verificationCache ? { verificationCache: structuredClone(options.nodeGather.verificationCache) } : {}) } : null;
|
|
16
|
+
const nodePieceDirectory = options.nodePieceDirectory ? structuredClone(options.nodePieceDirectory) : null;
|
|
17
|
+
if (nodePieceDirectory !== null && (pinned.acquire !== undefined || nodeGather !== null)) {
|
|
18
|
+
throw new Error('Owned nodePieceDirectory cannot be combined with a generic acquire callback or nodeGather.');
|
|
19
|
+
}
|
|
20
|
+
if (nodePieceDirectory === null && typeof pinned.acquire !== 'function') {
|
|
21
|
+
throw new Error('Verified piece storage requires an acquire callback or nodePieceDirectory.');
|
|
22
|
+
}
|
|
23
|
+
/** @type {Awaited<ReturnType<typeof import('./verified-piece-state-cache.js').prepareVerifiedPieceStateCache>>} */
|
|
24
|
+
let sharedStateCache;
|
|
25
|
+
/** @type {Awaited<ReturnType<typeof import('./verified-piece-state-cache.js').openPieceDirectoryStateCache>> | null} */
|
|
26
|
+
let pieceDirStateCache = null;
|
|
27
|
+
/** @type {import('./verified-piece-reader.js').VerifiedPieceReaderOptions['acquire']} */
|
|
28
|
+
let readerAcquire;
|
|
29
|
+
// Generic caller callbacks never inherit or seed file-bound verification state.
|
|
30
|
+
let allowStateVerifiedAcquire = false;
|
|
31
|
+
/** @type {(() => Promise<void>) | null} */
|
|
32
|
+
let closePieceDirectory = null;
|
|
33
|
+
/** @type {(() => import('./verified-piece-storage.js').NodePieceDirectoryReceipt) | undefined} */
|
|
34
|
+
let getPieceDirectoryReceipt;
|
|
35
|
+
|
|
36
|
+
if (nodeGather?.verificationCache !== undefined) {
|
|
37
|
+
const parsedIndex = JSON.parse(new TextDecoder().decode(indexBytes));
|
|
38
|
+
const { prepareVerifiedPieceStateCache } = await import('./verified-piece-state-cache.js');
|
|
39
|
+
sharedStateCache = await prepareVerifiedPieceStateCache({
|
|
40
|
+
rawPolicy: nodeGather.verificationCache,
|
|
41
|
+
manifestIdentity: parsedIndex.manifestIdentity,
|
|
42
|
+
indexIdentity: pinned.indexIdentity,
|
|
43
|
+
index: parsedIndex,
|
|
44
|
+
files: nodeGather.files,
|
|
45
|
+
});
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
if (nodePieceDirectory !== null) {
|
|
49
|
+
const parsedIndex = JSON.parse(new TextDecoder().decode(indexBytes));
|
|
50
|
+
const [{ openPieceDirectoryStateCache }, fsPromises, fs, path] = await Promise.all([
|
|
51
|
+
import('./verified-piece-state-cache.js'),
|
|
52
|
+
import('node:fs/promises'),
|
|
53
|
+
import('node:fs'),
|
|
54
|
+
import('node:path'),
|
|
55
|
+
]);
|
|
56
|
+
const { constants } = fs;
|
|
57
|
+
if (typeof nodePieceDirectory.directory !== 'string' || !path.isAbsolute(nodePieceDirectory.directory)
|
|
58
|
+
|| !Number.isSafeInteger(nodePieceDirectory.maxPieceBytes) || nodePieceDirectory.maxPieceBytes < parsedIndex.maxPieceBytes
|
|
59
|
+
|| !Number.isSafeInteger(nodePieceDirectory.maxActiveReadBytes) || nodePieceDirectory.maxActiveReadBytes < nodePieceDirectory.maxPieceBytes) {
|
|
60
|
+
throw new Error('Owned nodePieceDirectory requires an absolute directory and explicit piece/active-read bounds.');
|
|
61
|
+
}
|
|
62
|
+
const canonicalDir = await fsPromises.realpath(nodePieceDirectory.directory);
|
|
63
|
+
const directoryStat = await fsPromises.stat(canonicalDir, { bigint: true });
|
|
64
|
+
if (!directoryStat.isDirectory()) throw new Error('Owned nodePieceDirectory must be a directory.');
|
|
65
|
+
if (nodePieceDirectory.verificationCache !== undefined) {
|
|
66
|
+
if (!nodePieceDirectory.verificationCache || typeof nodePieceDirectory.verificationCache !== 'object'
|
|
67
|
+
|| Array.isArray(nodePieceDirectory.verificationCache)
|
|
68
|
+
|| typeof nodePieceDirectory.verificationCache.directory !== 'string'
|
|
69
|
+
|| !path.isAbsolute(nodePieceDirectory.verificationCache.directory)) {
|
|
70
|
+
throw new Error('Owned nodePieceDirectory verificationCache requires an explicit absolute directory.');
|
|
71
|
+
}
|
|
72
|
+
pieceDirStateCache = await openPieceDirectoryStateCache({
|
|
73
|
+
directory: nodePieceDirectory.verificationCache.directory,
|
|
74
|
+
pieceDirectory: canonicalDir,
|
|
75
|
+
manifestIdentity: parsedIndex.manifestIdentity,
|
|
76
|
+
indexIdentity: pinned.indexIdentity,
|
|
77
|
+
index: parsedIndex,
|
|
78
|
+
});
|
|
79
|
+
}
|
|
80
|
+
if (!Number.isInteger(constants.O_NOFOLLOW) || !Number.isInteger(constants.O_NONBLOCK)) {
|
|
81
|
+
throw new Error('Owned piece reads require O_NOFOLLOW and O_NONBLOCK support.');
|
|
82
|
+
}
|
|
83
|
+
const dirLifetime = new AbortController();
|
|
84
|
+
/** @type {Set<Promise<unknown>>} */
|
|
85
|
+
const dirOperations = new Set();
|
|
86
|
+
/** Descriptors whose close failed; retried when the directory closes. @type {Set<number>} */
|
|
87
|
+
const failedHandles = new Set();
|
|
88
|
+
/** @param {number} fd @param {Uint8Array} target @param {number} offset @param {number} length @param {number} position
|
|
89
|
+
* @returns {Promise<number>} */
|
|
90
|
+
const readAt = (fd, target, offset, length, position) => new Promise((resolve, reject) => {
|
|
91
|
+
fs.read(fd, target, offset, length, position, (error, bytesRead) => error ? reject(error) : resolve(bytesRead));
|
|
92
|
+
});
|
|
93
|
+
const dirStats = { reads: 0, activeReadBytes: 0, peakReadBytes: 0, missingPieces: 0 };
|
|
94
|
+
let dirClosed = false;
|
|
95
|
+
const stateCacheRef = pieceDirStateCache;
|
|
96
|
+
allowStateVerifiedAcquire = stateCacheRef !== null;
|
|
97
|
+
readerAcquire = (piece, control) => {
|
|
98
|
+
const op = (async () => {
|
|
99
|
+
const active = AbortSignal.any([
|
|
100
|
+
dirLifetime.signal,
|
|
101
|
+
...(pinned.signal ? [pinned.signal] : []),
|
|
102
|
+
...(control?.signal ? [control.signal] : []),
|
|
103
|
+
]);
|
|
104
|
+
active.throwIfAborted();
|
|
105
|
+
if (dirClosed) throw new Error('Diagnostic piece cache closed');
|
|
106
|
+
if (!/^[a-zA-Z0-9_.-]+$/u.test(piece.path) || ['.', '..'].includes(piece.path)
|
|
107
|
+
|| !Number.isSafeInteger(piece.offset) || piece.offset < 0
|
|
108
|
+
|| !Number.isSafeInteger(piece.size) || piece.size <= 0 || piece.size > nodePieceDirectory.maxPieceBytes) {
|
|
109
|
+
throw new Error('Invalid authenticated cache piece');
|
|
110
|
+
}
|
|
111
|
+
if (dirStats.activeReadBytes + piece.size > nodePieceDirectory.maxActiveReadBytes) {
|
|
112
|
+
throw new Error('Piece cache active-read budget exceeded');
|
|
113
|
+
}
|
|
114
|
+
dirStats.activeReadBytes += piece.size;
|
|
115
|
+
dirStats.peakReadBytes = Math.max(dirStats.peakReadBytes, dirStats.activeReadBytes);
|
|
116
|
+
// The identity checks, open and close are metadata calls on files the OS already holds; they run
|
|
117
|
+
// synchronously so each piece queues one pool task (its read) instead of seven. Queued behind
|
|
118
|
+
// the others, first-read hashing was waiting for the pool rather than for the processor.
|
|
119
|
+
/** @type {number | undefined} */
|
|
120
|
+
let handle;
|
|
121
|
+
try {
|
|
122
|
+
const currentDirectory = fs.lstatSync(canonicalDir, { bigint: true });
|
|
123
|
+
if (!currentDirectory.isDirectory() || currentDirectory.dev !== directoryStat.dev
|
|
124
|
+
|| currentDirectory.ino !== directoryStat.ino) {
|
|
125
|
+
throw new Error('Piece cache directory changed');
|
|
126
|
+
}
|
|
127
|
+
const filename = path.join(canonicalDir, `${piece.path}.${piece.offset}`);
|
|
128
|
+
handle = fs.openSync(filename, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
|
|
129
|
+
const before = fs.fstatSync(handle, { bigint: true });
|
|
130
|
+
const linkBefore = fs.lstatSync(filename, { bigint: true });
|
|
131
|
+
if (!before.isFile() || !linkBefore.isFile() || before.size !== BigInt(piece.size)
|
|
132
|
+
|| linkBefore.dev !== before.dev || linkBefore.ino !== before.ino || linkBefore.size !== before.size
|
|
133
|
+
|| linkBefore.mtimeNs !== before.mtimeNs || linkBefore.ctimeNs !== before.ctimeNs) {
|
|
134
|
+
throw new Error('Cached piece must be a complete regular file');
|
|
135
|
+
}
|
|
136
|
+
const bytes = new Uint8Array(piece.size);
|
|
137
|
+
let received = 0;
|
|
138
|
+
while (received < bytes.length) {
|
|
139
|
+
active.throwIfAborted();
|
|
140
|
+
const bytesRead = await readAt(handle, bytes, received, bytes.length - received, received);
|
|
141
|
+
if (bytesRead <= 0) throw new Error('Cached piece read truncated');
|
|
142
|
+
received += bytesRead;
|
|
143
|
+
}
|
|
144
|
+
const after = fs.fstatSync(handle, { bigint: true });
|
|
145
|
+
const linkAfter = fs.lstatSync(filename, { bigint: true });
|
|
146
|
+
const unchanged = after.isFile() && linkAfter.isFile()
|
|
147
|
+
&& after.dev === before.dev && after.ino === before.ino && after.size === before.size
|
|
148
|
+
&& after.mtimeNs === before.mtimeNs && after.ctimeNs === before.ctimeNs
|
|
149
|
+
&& linkAfter.dev === before.dev && linkAfter.ino === before.ino && linkAfter.size === before.size
|
|
150
|
+
&& linkAfter.mtimeNs === before.mtimeNs && linkAfter.ctimeNs === before.ctimeNs;
|
|
151
|
+
if (!unchanged) throw new Error('Cached piece changed during read');
|
|
152
|
+
active.throwIfAborted();
|
|
153
|
+
dirStats.reads++;
|
|
154
|
+
if (!stateCacheRef) return bytes;
|
|
155
|
+
return {
|
|
156
|
+
bytes,
|
|
157
|
+
stateVerified: stateCacheRef.isVerified(piece.path, piece.offset, after),
|
|
158
|
+
onAuthenticated: () => { stateCacheRef.markVerified(piece.path, piece.offset, after); },
|
|
159
|
+
};
|
|
160
|
+
} catch (error) {
|
|
161
|
+
if (error && typeof error === 'object' && 'code' in error && error.code === 'ENOENT') dirStats.missingPieces++;
|
|
162
|
+
throw error;
|
|
163
|
+
} finally {
|
|
164
|
+
try {
|
|
165
|
+
if (handle !== undefined) {
|
|
166
|
+
try { fs.closeSync(handle); }
|
|
167
|
+
catch (error) { failedHandles.add(handle); dirClosed = true; dirLifetime.abort(error); throw error; }
|
|
168
|
+
}
|
|
169
|
+
} finally { dirStats.activeReadBytes -= piece.size; }
|
|
170
|
+
}
|
|
171
|
+
})();
|
|
172
|
+
dirOperations.add(op);
|
|
173
|
+
op.finally(() => dirOperations.delete(op)).catch(() => {});
|
|
174
|
+
return op;
|
|
175
|
+
};
|
|
176
|
+
closePieceDirectory = async () => {
|
|
177
|
+
dirClosed = true;
|
|
178
|
+
dirLifetime.abort(new Error('Diagnostic piece cache closed'));
|
|
179
|
+
await Promise.allSettled([...dirOperations]);
|
|
180
|
+
const closeFailures = [];
|
|
181
|
+
for (const handle of [...failedHandles]) {
|
|
182
|
+
try { fs.closeSync(handle); failedHandles.delete(handle); }
|
|
183
|
+
catch (error) { closeFailures.push(error); }
|
|
184
|
+
}
|
|
185
|
+
if (closeFailures.length) throw new AggregateError(closeFailures, 'Diagnostic piece cache cleanup failed');
|
|
186
|
+
};
|
|
187
|
+
getPieceDirectoryReceipt = () => ({ ...dirStats, pendingDescriptorCloses: failedHandles.size });
|
|
188
|
+
} else {
|
|
189
|
+
readerAcquire = /** @type {import('./verified-piece-reader.js').VerifiedPieceReaderOptions['acquire']} */ (pinned.acquire);
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
let reader;
|
|
193
|
+
try {
|
|
194
|
+
reader = await createVerifiedPieceReader({ ...pinned, acquire: readerAcquire, allowStateVerifiedAcquire });
|
|
195
|
+
} catch (error) {
|
|
196
|
+
await Promise.allSettled([closePieceDirectory?.(), sharedStateCache?.close(), pieceDirStateCache?.close()]);
|
|
197
|
+
throw error;
|
|
198
|
+
}
|
|
16
199
|
/** @type {Awaited<ReturnType<import('./verified-piece-gather.js').createVerifiedPieceGather>> | null} */
|
|
17
200
|
let gather = null;
|
|
18
201
|
const { manifest, readRange, readInto, readBinary } = reader;
|
|
@@ -20,7 +203,7 @@ export async function createVerifiedPieceStorage(options) {
|
|
|
20
203
|
pinned.signal?.throwIfAborted();
|
|
21
204
|
if (nodeGather) {
|
|
22
205
|
const { createVerifiedPieceGather } = await import('./verified-piece-gather.js');
|
|
23
|
-
gather = await createVerifiedPieceGather({ ...nodeGather, manifestBytes, indexBytes,
|
|
206
|
+
gather = await createVerifiedPieceGather({ ...nodeGather, sharedStateCache, manifestBytes, indexBytes,
|
|
24
207
|
indexIdentity: pinned.indexIdentity, startupSignal: pinned.signal });
|
|
25
208
|
}
|
|
26
209
|
pinned.signal?.throwIfAborted();
|
|
@@ -29,7 +212,13 @@ export async function createVerifiedPieceStorage(options) {
|
|
|
29
212
|
const operations = new Set();
|
|
30
213
|
const close = async () => {
|
|
31
214
|
closed = true;
|
|
32
|
-
const results = await Promise.allSettled([
|
|
215
|
+
const results = await Promise.allSettled([
|
|
216
|
+
reader.close(),
|
|
217
|
+
gather?.close(),
|
|
218
|
+
closePieceDirectory?.(),
|
|
219
|
+
sharedStateCache?.close(),
|
|
220
|
+
pieceDirStateCache?.close(),
|
|
221
|
+
]);
|
|
33
222
|
await Promise.allSettled([...operations]);
|
|
34
223
|
const failed = results.find(result => result.status === 'rejected');
|
|
35
224
|
if (failed) throw failed.reason;
|
|
@@ -60,9 +249,6 @@ export async function createVerifiedPieceStorage(options) {
|
|
|
60
249
|
const storage = createArtifactStorageContext({ manifest, readRange, readInto, readBinary, shardSources,
|
|
61
250
|
tokenizerJsonPath: directSource ? manifest.tokenizer?.file : undefined,
|
|
62
251
|
readText: async path => new TextDecoder().decode(await readBinary(path)),
|
|
63
|
-
// The approved index binds canonical shards to the manifest. Every returned
|
|
64
|
-
// byte still passes owned-copy piece verification; whole-source scans would
|
|
65
|
-
// fetch unrelated partitions. Neither flag bypasses the authenticated reader.
|
|
66
252
|
verifyHashes: true, hashesTrusted: true, sourceHashesTrusted: directSource, close });
|
|
67
253
|
storage.preflight = reader.preflight;
|
|
68
254
|
if (gather) {
|
|
@@ -96,9 +282,21 @@ export async function createVerifiedPieceStorage(options) {
|
|
|
96
282
|
};
|
|
97
283
|
storage.rangeLease = Object.freeze(scopedPort);
|
|
98
284
|
}
|
|
99
|
-
return {
|
|
285
|
+
return {
|
|
286
|
+
manifest,
|
|
287
|
+
storage,
|
|
288
|
+
getReceipt: reader.getReceipt,
|
|
289
|
+
getGatherReceipt: gather ? gather.getReceipt : null,
|
|
290
|
+
...(getPieceDirectoryReceipt ? { getPieceDirectoryReceipt } : {}),
|
|
291
|
+
};
|
|
100
292
|
} catch (error) {
|
|
101
|
-
const outcomes = await Promise.allSettled([
|
|
293
|
+
const outcomes = await Promise.allSettled([
|
|
294
|
+
reader.close(),
|
|
295
|
+
gather?.close(),
|
|
296
|
+
closePieceDirectory?.(),
|
|
297
|
+
sharedStateCache?.close(),
|
|
298
|
+
pieceDirStateCache?.close(),
|
|
299
|
+
]);
|
|
102
300
|
const failures = outcomes.flatMap(outcome => outcome.status === 'rejected' ? [outcome.reason] : []);
|
|
103
301
|
if (failures.length) throw new AggregateError([error, ...failures], 'Verified source opening and cleanup failed.');
|
|
104
302
|
throw error;
|
package/src/version.js
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
export const DOPPLER_VERSION = '0.7.
|
|
1
|
+
export const DOPPLER_VERSION = '0.7.8';
|
|
2
2
|
export const DOPPLER_PROVIDER_VERSION = DOPPLER_VERSION;
|