doppler-gpu 0.7.7 → 0.7.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (36) hide show
  1. package/CHANGELOG.md +18 -0
  2. package/README.md +9 -17
  3. package/package.json +1 -1
  4. package/src/config/kernels/kernel-ref-digests.js +2 -1
  5. package/src/config/kernels/registry.json +47 -19
  6. package/src/config/schema/execution/kernel-warmup.schema.d.ts +14 -0
  7. package/src/config/schema/execution/kernel-warmup.schema.js +3 -0
  8. package/src/config/schema/platform/gpu-cache.schema.js +1 -1
  9. package/src/config/schema/runtime/buffer-pool.schema.js +2 -2
  10. package/src/config/schema/runtime/loading.schema.d.ts +5 -0
  11. package/src/config/schema/runtime/loading.schema.js +25 -2
  12. package/src/gpu/kernels/moe_route_q4_0_block_f32.wgsl +74 -0
  13. package/src/gpu/kernels/moe_route_q4_0_split_block_f32.wgsl +35 -18
  14. package/src/inference/pipelines/text/partitions/partition-execution.js +28 -1
  15. package/src/inference/pipelines/text/partitions/resident-partition-warmup.js +67 -0
  16. package/src/inference/pipelines/text/partitions/resident-partition.js +8 -2
  17. package/src/loader/experts/compact-expert-cache.js +20 -0
  18. package/src/loader/experts/loader-bindings.js +14 -3
  19. package/src/loader/experts/mapped-expert-staging.js +393 -0
  20. package/src/loader/experts/routed-expert-cache.d.ts +8 -0
  21. package/src/loader/experts/routed-expert-cache.js +12 -11
  22. package/src/loader/experts/routed-expert-prefill.js +77 -1
  23. package/src/loader/experts/routed-expert-source.js +1 -0
  24. package/src/loader/experts/tiered-expert-cache.js +66 -26
  25. package/src/storage/verified-piece-direct.js +153 -0
  26. package/src/storage/verified-piece-gather-lane.js +21 -3
  27. package/src/storage/verified-piece-gather-worker.js +235 -16
  28. package/src/storage/verified-piece-gather.d.ts +47 -0
  29. package/src/storage/verified-piece-gather.js +133 -17
  30. package/src/storage/verified-piece-reader.d.ts +8 -1
  31. package/src/storage/verified-piece-reader.js +16 -3
  32. package/src/storage/verified-piece-state-cache.d.ts +75 -0
  33. package/src/storage/verified-piece-state-cache.js +511 -0
  34. package/src/storage/verified-piece-storage.d.ts +20 -1
  35. package/src/storage/verified-piece-storage.js +207 -9
  36. package/src/version.js +1 -1
@@ -11,8 +11,191 @@ export async function createVerifiedPieceStorage(options) {
11
11
  const pinned = { ...options, manifestBytes, indexBytes };
12
12
  const nodeGather = options.nodeGather ? { ...options.nodeGather, files: { ...options.nodeGather.files },
13
13
  ...(options.nodeGather.fileIdentities ? { fileIdentities: structuredClone(options.nodeGather.fileIdentities) } : {}),
14
- ...(options.nodeGather.protectedCache ? { protectedCache: structuredClone(options.nodeGather.protectedCache) } : {}) } : null;
15
- const reader = await createVerifiedPieceReader(pinned);
14
+ ...(options.nodeGather.protectedCache ? { protectedCache: structuredClone(options.nodeGather.protectedCache) } : {}),
15
+ ...(options.nodeGather.verificationCache ? { verificationCache: structuredClone(options.nodeGather.verificationCache) } : {}) } : null;
16
+ const nodePieceDirectory = options.nodePieceDirectory ? structuredClone(options.nodePieceDirectory) : null;
17
+ if (nodePieceDirectory !== null && (pinned.acquire !== undefined || nodeGather !== null)) {
18
+ throw new Error('Owned nodePieceDirectory cannot be combined with a generic acquire callback or nodeGather.');
19
+ }
20
+ if (nodePieceDirectory === null && typeof pinned.acquire !== 'function') {
21
+ throw new Error('Verified piece storage requires an acquire callback or nodePieceDirectory.');
22
+ }
23
+ /** @type {Awaited<ReturnType<typeof import('./verified-piece-state-cache.js').prepareVerifiedPieceStateCache>>} */
24
+ let sharedStateCache;
25
+ /** @type {Awaited<ReturnType<typeof import('./verified-piece-state-cache.js').openPieceDirectoryStateCache>> | null} */
26
+ let pieceDirStateCache = null;
27
+ /** @type {import('./verified-piece-reader.js').VerifiedPieceReaderOptions['acquire']} */
28
+ let readerAcquire;
29
+ // Generic caller callbacks never inherit or seed file-bound verification state.
30
+ let allowStateVerifiedAcquire = false;
31
+ /** @type {(() => Promise<void>) | null} */
32
+ let closePieceDirectory = null;
33
+ /** @type {(() => import('./verified-piece-storage.js').NodePieceDirectoryReceipt) | undefined} */
34
+ let getPieceDirectoryReceipt;
35
+
36
+ if (nodeGather?.verificationCache !== undefined) {
37
+ const parsedIndex = JSON.parse(new TextDecoder().decode(indexBytes));
38
+ const { prepareVerifiedPieceStateCache } = await import('./verified-piece-state-cache.js');
39
+ sharedStateCache = await prepareVerifiedPieceStateCache({
40
+ rawPolicy: nodeGather.verificationCache,
41
+ manifestIdentity: parsedIndex.manifestIdentity,
42
+ indexIdentity: pinned.indexIdentity,
43
+ index: parsedIndex,
44
+ files: nodeGather.files,
45
+ });
46
+ }
47
+
48
+ if (nodePieceDirectory !== null) {
49
+ const parsedIndex = JSON.parse(new TextDecoder().decode(indexBytes));
50
+ const [{ openPieceDirectoryStateCache }, fsPromises, fs, path] = await Promise.all([
51
+ import('./verified-piece-state-cache.js'),
52
+ import('node:fs/promises'),
53
+ import('node:fs'),
54
+ import('node:path'),
55
+ ]);
56
+ const { constants } = fs;
57
+ if (typeof nodePieceDirectory.directory !== 'string' || !path.isAbsolute(nodePieceDirectory.directory)
58
+ || !Number.isSafeInteger(nodePieceDirectory.maxPieceBytes) || nodePieceDirectory.maxPieceBytes < parsedIndex.maxPieceBytes
59
+ || !Number.isSafeInteger(nodePieceDirectory.maxActiveReadBytes) || nodePieceDirectory.maxActiveReadBytes < nodePieceDirectory.maxPieceBytes) {
60
+ throw new Error('Owned nodePieceDirectory requires an absolute directory and explicit piece/active-read bounds.');
61
+ }
62
+ const canonicalDir = await fsPromises.realpath(nodePieceDirectory.directory);
63
+ const directoryStat = await fsPromises.stat(canonicalDir, { bigint: true });
64
+ if (!directoryStat.isDirectory()) throw new Error('Owned nodePieceDirectory must be a directory.');
65
+ if (nodePieceDirectory.verificationCache !== undefined) {
66
+ if (!nodePieceDirectory.verificationCache || typeof nodePieceDirectory.verificationCache !== 'object'
67
+ || Array.isArray(nodePieceDirectory.verificationCache)
68
+ || typeof nodePieceDirectory.verificationCache.directory !== 'string'
69
+ || !path.isAbsolute(nodePieceDirectory.verificationCache.directory)) {
70
+ throw new Error('Owned nodePieceDirectory verificationCache requires an explicit absolute directory.');
71
+ }
72
+ pieceDirStateCache = await openPieceDirectoryStateCache({
73
+ directory: nodePieceDirectory.verificationCache.directory,
74
+ pieceDirectory: canonicalDir,
75
+ manifestIdentity: parsedIndex.manifestIdentity,
76
+ indexIdentity: pinned.indexIdentity,
77
+ index: parsedIndex,
78
+ });
79
+ }
80
+ if (!Number.isInteger(constants.O_NOFOLLOW) || !Number.isInteger(constants.O_NONBLOCK)) {
81
+ throw new Error('Owned piece reads require O_NOFOLLOW and O_NONBLOCK support.');
82
+ }
83
+ const dirLifetime = new AbortController();
84
+ /** @type {Set<Promise<unknown>>} */
85
+ const dirOperations = new Set();
86
+ /** Descriptors whose close failed; retried when the directory closes. @type {Set<number>} */
87
+ const failedHandles = new Set();
88
+ /** @param {number} fd @param {Uint8Array} target @param {number} offset @param {number} length @param {number} position
89
+ * @returns {Promise<number>} */
90
+ const readAt = (fd, target, offset, length, position) => new Promise((resolve, reject) => {
91
+ fs.read(fd, target, offset, length, position, (error, bytesRead) => error ? reject(error) : resolve(bytesRead));
92
+ });
93
+ const dirStats = { reads: 0, activeReadBytes: 0, peakReadBytes: 0, missingPieces: 0 };
94
+ let dirClosed = false;
95
+ const stateCacheRef = pieceDirStateCache;
96
+ allowStateVerifiedAcquire = stateCacheRef !== null;
97
+ readerAcquire = (piece, control) => {
98
+ const op = (async () => {
99
+ const active = AbortSignal.any([
100
+ dirLifetime.signal,
101
+ ...(pinned.signal ? [pinned.signal] : []),
102
+ ...(control?.signal ? [control.signal] : []),
103
+ ]);
104
+ active.throwIfAborted();
105
+ if (dirClosed) throw new Error('Diagnostic piece cache closed');
106
+ if (!/^[a-zA-Z0-9_.-]+$/u.test(piece.path) || ['.', '..'].includes(piece.path)
107
+ || !Number.isSafeInteger(piece.offset) || piece.offset < 0
108
+ || !Number.isSafeInteger(piece.size) || piece.size <= 0 || piece.size > nodePieceDirectory.maxPieceBytes) {
109
+ throw new Error('Invalid authenticated cache piece');
110
+ }
111
+ if (dirStats.activeReadBytes + piece.size > nodePieceDirectory.maxActiveReadBytes) {
112
+ throw new Error('Piece cache active-read budget exceeded');
113
+ }
114
+ dirStats.activeReadBytes += piece.size;
115
+ dirStats.peakReadBytes = Math.max(dirStats.peakReadBytes, dirStats.activeReadBytes);
116
+ // The identity checks, open and close are metadata calls on files the OS already holds; they run
117
+ // synchronously so each piece queues one pool task (its read) instead of seven. Queued behind
118
+ // the others, first-read hashing was waiting for the pool rather than for the processor.
119
+ /** @type {number | undefined} */
120
+ let handle;
121
+ try {
122
+ const currentDirectory = fs.lstatSync(canonicalDir, { bigint: true });
123
+ if (!currentDirectory.isDirectory() || currentDirectory.dev !== directoryStat.dev
124
+ || currentDirectory.ino !== directoryStat.ino) {
125
+ throw new Error('Piece cache directory changed');
126
+ }
127
+ const filename = path.join(canonicalDir, `${piece.path}.${piece.offset}`);
128
+ handle = fs.openSync(filename, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
129
+ const before = fs.fstatSync(handle, { bigint: true });
130
+ const linkBefore = fs.lstatSync(filename, { bigint: true });
131
+ if (!before.isFile() || !linkBefore.isFile() || before.size !== BigInt(piece.size)
132
+ || linkBefore.dev !== before.dev || linkBefore.ino !== before.ino || linkBefore.size !== before.size
133
+ || linkBefore.mtimeNs !== before.mtimeNs || linkBefore.ctimeNs !== before.ctimeNs) {
134
+ throw new Error('Cached piece must be a complete regular file');
135
+ }
136
+ const bytes = new Uint8Array(piece.size);
137
+ let received = 0;
138
+ while (received < bytes.length) {
139
+ active.throwIfAborted();
140
+ const bytesRead = await readAt(handle, bytes, received, bytes.length - received, received);
141
+ if (bytesRead <= 0) throw new Error('Cached piece read truncated');
142
+ received += bytesRead;
143
+ }
144
+ const after = fs.fstatSync(handle, { bigint: true });
145
+ const linkAfter = fs.lstatSync(filename, { bigint: true });
146
+ const unchanged = after.isFile() && linkAfter.isFile()
147
+ && after.dev === before.dev && after.ino === before.ino && after.size === before.size
148
+ && after.mtimeNs === before.mtimeNs && after.ctimeNs === before.ctimeNs
149
+ && linkAfter.dev === before.dev && linkAfter.ino === before.ino && linkAfter.size === before.size
150
+ && linkAfter.mtimeNs === before.mtimeNs && linkAfter.ctimeNs === before.ctimeNs;
151
+ if (!unchanged) throw new Error('Cached piece changed during read');
152
+ active.throwIfAborted();
153
+ dirStats.reads++;
154
+ if (!stateCacheRef) return bytes;
155
+ return {
156
+ bytes,
157
+ stateVerified: stateCacheRef.isVerified(piece.path, piece.offset, after),
158
+ onAuthenticated: () => { stateCacheRef.markVerified(piece.path, piece.offset, after); },
159
+ };
160
+ } catch (error) {
161
+ if (error && typeof error === 'object' && 'code' in error && error.code === 'ENOENT') dirStats.missingPieces++;
162
+ throw error;
163
+ } finally {
164
+ try {
165
+ if (handle !== undefined) {
166
+ try { fs.closeSync(handle); }
167
+ catch (error) { failedHandles.add(handle); dirClosed = true; dirLifetime.abort(error); throw error; }
168
+ }
169
+ } finally { dirStats.activeReadBytes -= piece.size; }
170
+ }
171
+ })();
172
+ dirOperations.add(op);
173
+ op.finally(() => dirOperations.delete(op)).catch(() => {});
174
+ return op;
175
+ };
176
+ closePieceDirectory = async () => {
177
+ dirClosed = true;
178
+ dirLifetime.abort(new Error('Diagnostic piece cache closed'));
179
+ await Promise.allSettled([...dirOperations]);
180
+ const closeFailures = [];
181
+ for (const handle of [...failedHandles]) {
182
+ try { fs.closeSync(handle); failedHandles.delete(handle); }
183
+ catch (error) { closeFailures.push(error); }
184
+ }
185
+ if (closeFailures.length) throw new AggregateError(closeFailures, 'Diagnostic piece cache cleanup failed');
186
+ };
187
+ getPieceDirectoryReceipt = () => ({ ...dirStats, pendingDescriptorCloses: failedHandles.size });
188
+ } else {
189
+ readerAcquire = /** @type {import('./verified-piece-reader.js').VerifiedPieceReaderOptions['acquire']} */ (pinned.acquire);
190
+ }
191
+
192
+ let reader;
193
+ try {
194
+ reader = await createVerifiedPieceReader({ ...pinned, acquire: readerAcquire, allowStateVerifiedAcquire });
195
+ } catch (error) {
196
+ await Promise.allSettled([closePieceDirectory?.(), sharedStateCache?.close(), pieceDirStateCache?.close()]);
197
+ throw error;
198
+ }
16
199
  /** @type {Awaited<ReturnType<import('./verified-piece-gather.js').createVerifiedPieceGather>> | null} */
17
200
  let gather = null;
18
201
  const { manifest, readRange, readInto, readBinary } = reader;
@@ -20,7 +203,7 @@ export async function createVerifiedPieceStorage(options) {
20
203
  pinned.signal?.throwIfAborted();
21
204
  if (nodeGather) {
22
205
  const { createVerifiedPieceGather } = await import('./verified-piece-gather.js');
23
- gather = await createVerifiedPieceGather({ ...nodeGather, manifestBytes, indexBytes,
206
+ gather = await createVerifiedPieceGather({ ...nodeGather, sharedStateCache, manifestBytes, indexBytes,
24
207
  indexIdentity: pinned.indexIdentity, startupSignal: pinned.signal });
25
208
  }
26
209
  pinned.signal?.throwIfAborted();
@@ -29,7 +212,13 @@ export async function createVerifiedPieceStorage(options) {
29
212
  const operations = new Set();
30
213
  const close = async () => {
31
214
  closed = true;
32
- const results = await Promise.allSettled([reader.close(), gather?.close()]);
215
+ const results = await Promise.allSettled([
216
+ reader.close(),
217
+ gather?.close(),
218
+ closePieceDirectory?.(),
219
+ sharedStateCache?.close(),
220
+ pieceDirStateCache?.close(),
221
+ ]);
33
222
  await Promise.allSettled([...operations]);
34
223
  const failed = results.find(result => result.status === 'rejected');
35
224
  if (failed) throw failed.reason;
@@ -60,9 +249,6 @@ export async function createVerifiedPieceStorage(options) {
60
249
  const storage = createArtifactStorageContext({ manifest, readRange, readInto, readBinary, shardSources,
61
250
  tokenizerJsonPath: directSource ? manifest.tokenizer?.file : undefined,
62
251
  readText: async path => new TextDecoder().decode(await readBinary(path)),
63
- // The approved index binds canonical shards to the manifest. Every returned
64
- // byte still passes owned-copy piece verification; whole-source scans would
65
- // fetch unrelated partitions. Neither flag bypasses the authenticated reader.
66
252
  verifyHashes: true, hashesTrusted: true, sourceHashesTrusted: directSource, close });
67
253
  storage.preflight = reader.preflight;
68
254
  if (gather) {
@@ -96,9 +282,21 @@ export async function createVerifiedPieceStorage(options) {
96
282
  };
97
283
  storage.rangeLease = Object.freeze(scopedPort);
98
284
  }
99
- return { manifest, storage, getReceipt: reader.getReceipt, getGatherReceipt: gather ? gather.getReceipt : null };
285
+ return {
286
+ manifest,
287
+ storage,
288
+ getReceipt: reader.getReceipt,
289
+ getGatherReceipt: gather ? gather.getReceipt : null,
290
+ ...(getPieceDirectoryReceipt ? { getPieceDirectoryReceipt } : {}),
291
+ };
100
292
  } catch (error) {
101
- const outcomes = await Promise.allSettled([reader.close(), gather?.close()]);
293
+ const outcomes = await Promise.allSettled([
294
+ reader.close(),
295
+ gather?.close(),
296
+ closePieceDirectory?.(),
297
+ sharedStateCache?.close(),
298
+ pieceDirStateCache?.close(),
299
+ ]);
102
300
  const failures = outcomes.flatMap(outcome => outcome.status === 'rejected' ? [outcome.reason] : []);
103
301
  if (failures.length) throw new AggregateError([error, ...failures], 'Verified source opening and cleanup failed.');
104
302
  throw error;
package/src/version.js CHANGED
@@ -1,2 +1,2 @@
1
- export const DOPPLER_VERSION = '0.7.7';
1
+ export const DOPPLER_VERSION = '0.7.8';
2
2
  export const DOPPLER_PROVIDER_VERSION = DOPPLER_VERSION;