@avantf/dsh-plugin-base 0.3.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +29 -15
- package/dist/bootstrap.d.ts +3 -2
- package/dist/bootstrap.d.ts.map +1 -1
- package/dist/bootstrap.js +3 -2
- package/dist/compat.d.ts +2 -2
- package/dist/compat.js +5 -5
- package/dist/errors.d.ts +6 -1
- package/dist/errors.d.ts.map +1 -1
- package/dist/index.d.ts +23 -9
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +45 -22
- package/dist/interface.d.ts +296 -23
- package/dist/interface.d.ts.map +1 -1
- package/dist/interface.js +206 -8
- package/dist/interface_gate.d.ts +20 -6
- package/dist/interface_gate.d.ts.map +1 -1
- package/dist/interface_gate.js +37 -7
- package/dist/internal.d.ts +26 -1
- package/dist/internal.d.ts.map +1 -1
- package/dist/internal.js +27 -1
- package/dist/kit/family.d.ts +2 -2
- package/dist/kit/family.js +3 -3
- package/dist/kit/index.d.ts +5 -1
- package/dist/kit/index.d.ts.map +1 -1
- package/dist/kit/index.js +5 -1
- package/dist/kit/logger.d.ts +1 -1
- package/dist/kit/logger.js +1 -1
- package/dist/kit/prompt_files.d.ts +37 -3
- package/dist/kit/prompt_files.d.ts.map +1 -1
- package/dist/kit/prompt_files.js +45 -0
- package/dist/kit/typert.d.ts +2 -2
- package/dist/kit/wellformed.d.ts +79 -0
- package/dist/kit/wellformed.d.ts.map +1 -0
- package/dist/kit/wellformed.js +155 -0
- package/dist/layout.d.ts +17 -2
- package/dist/layout.d.ts.map +1 -1
- package/dist/layout.js +15 -5
- package/dist/lint.d.ts.map +1 -1
- package/dist/lint.js +2 -2
- package/dist/lock.d.ts +18 -0
- package/dist/lock.d.ts.map +1 -1
- package/dist/lock.js +77 -10
- package/dist/net.d.ts +106 -3
- package/dist/net.d.ts.map +1 -1
- package/dist/net.js +128 -30
- package/dist/providers/archive.d.ts.map +1 -1
- package/dist/providers/archive.js +108 -20
- package/dist/providers/model.d.ts +26 -3
- package/dist/providers/model.d.ts.map +1 -1
- package/dist/providers/model.js +159 -32
- package/dist/providers/npm.d.ts +5 -0
- package/dist/providers/npm.d.ts.map +1 -1
- package/dist/providers/npm.js +62 -21
- package/dist/provisioner.d.ts.map +1 -1
- package/dist/provisioner.js +128 -31
- package/dist/state.d.ts.map +1 -1
- package/dist/state.js +8 -3
- package/dist/tar.d.ts +7 -0
- package/dist/tar.d.ts.map +1 -1
- package/dist/tar.js +12 -5
- package/dist/types.d.ts +11 -1
- package/dist/types.d.ts.map +1 -1
- package/dist/zip.d.ts.map +1 -1
- package/dist/zip.js +10 -1
- package/package.json +18 -19
- package/dist/artifact.js.map +0 -1
- package/dist/bootstrap.js.map +0 -1
- package/dist/cli.js.map +0 -1
- package/dist/compat.js.map +0 -1
- package/dist/conformance.js.map +0 -1
- package/dist/errors.js.map +0 -1
- package/dist/fs.js.map +0 -1
- package/dist/index.js.map +0 -1
- package/dist/integrity.js.map +0 -1
- package/dist/interface.js.map +0 -1
- package/dist/interface_gate.js.map +0 -1
- package/dist/internal.js.map +0 -1
- package/dist/kit/family.js.map +0 -1
- package/dist/kit/index.js.map +0 -1
- package/dist/kit/logger.js.map +0 -1
- package/dist/kit/prompt_files.js.map +0 -1
- package/dist/kit/typert.js.map +0 -1
- package/dist/layout.js.map +0 -1
- package/dist/lint.js.map +0 -1
- package/dist/lock.js.map +0 -1
- package/dist/manifest.js.map +0 -1
- package/dist/net.js.map +0 -1
- package/dist/package-name.js.map +0 -1
- package/dist/preset.js.map +0 -1
- package/dist/providers/archive.js.map +0 -1
- package/dist/providers/model.js.map +0 -1
- package/dist/providers/npm.js.map +0 -1
- package/dist/provisioner.js.map +0 -1
- package/dist/semver.js.map +0 -1
- package/dist/state.js.map +0 -1
- package/dist/tar.js.map +0 -1
- package/dist/types.js.map +0 -1
- package/dist/zip.js.map +0 -1
package/dist/providers/model.js
CHANGED
|
@@ -3,21 +3,42 @@
|
|
|
3
3
|
* @module providers/model
|
|
4
4
|
*/
|
|
5
5
|
import { createHash } from 'node:crypto';
|
|
6
|
+
import { once } from 'node:events';
|
|
7
|
+
import { createWriteStream } from 'node:fs';
|
|
6
8
|
import { basename, dirname, join, relative, resolve } from 'node:path';
|
|
9
|
+
import { finished } from 'node:stream/promises';
|
|
7
10
|
import { ProvisionError } from '../errors.js';
|
|
8
11
|
import { ATOMIC_TEMP_PREFIX, exists, sweepAtomicTemps } from '../fs.js';
|
|
9
12
|
import { isInside, lockPath } from '../layout.js';
|
|
10
13
|
import { defaultLock } from '../lock.js';
|
|
11
|
-
import { fetchImplOf, readCapped, signalFor } from '../net.js';
|
|
14
|
+
import { METADATA_MAX_BYTES, SPILL_THRESHOLD_BYTES, fetchImplOf, readCapped, readCappedOrSpill, signalFor } from '../net.js';
|
|
12
15
|
export const MODEL_CACHE_KIND = 'model-cache';
|
|
13
16
|
const DEFAULT_ENDPOINT = 'https://huggingface.co';
|
|
14
17
|
const DEFAULT_REVISION = 'main';
|
|
15
|
-
/**
|
|
16
|
-
|
|
18
|
+
/**
|
|
19
|
+
* Maximum bytes read for a single model file.
|
|
20
|
+
*
|
|
21
|
+
* 2 GiB, not the old 4 GiB. The cap is the number of bytes the MEMORY path would have to hold TWICE
|
|
22
|
+
* (the received chunks plus the joined copy), so it bounds the worst-case spike directly; the measured
|
|
23
|
+
* ~2× peak for a body of this size is ~4 GiB, which a 16 GB machine survives and a 4 GiB cap does not.
|
|
24
|
+
* 2 GiB is also still far above what the framework is for: a real `bge-small` is ~90 MB and even a
|
|
25
|
+
* multi-billion-parameter checkpoint is a handful of files well under this. Anything larger wants a
|
|
26
|
+
* different transport, not a bigger heap.
|
|
27
|
+
*
|
|
28
|
+
* (The review's "4 GiB is exactly Node's `kMaxLength`" does NOT reproduce on the target runtime:
|
|
29
|
+
* Node 22.23.2 / linux-x64 reports `buffer.constants.MAX_LENGTH` 2^53−1 and allocates a 4 GiB
|
|
30
|
+
* `Uint8Array` fine. The rejection boundary is a legitimate design choice here, not a crash we are
|
|
31
|
+
* ducking — see the report accompanying this change.)
|
|
32
|
+
*/
|
|
33
|
+
export const MAX_MODEL_BYTES = 2 * 1024 * 1024 * 1024;
|
|
17
34
|
/** How long the placement lock waits before reporting `lock/timeout`. */
|
|
18
35
|
const LOCK_TIMEOUT_MS = 15_000;
|
|
19
|
-
/**
|
|
20
|
-
|
|
36
|
+
/**
|
|
37
|
+
* A placement lock older than this whose pid is gone is reclaimed. It is ALSO the mtime age at which
|
|
38
|
+
* a waiter warns about a LIVE holder, so it must stay BELOW `LOCK_TIMEOUT_MS`; at `60_000` the
|
|
39
|
+
* waiter timed out first and the warning was dead code (P15). Live holders are never reclaimed.
|
|
40
|
+
*/
|
|
41
|
+
const STALE_LOCK_MS = 10_000;
|
|
21
42
|
/** A resolved commit id: 40 or 64 lowercase hex. */
|
|
22
43
|
const SHA_PATTERN = /^(?:[0-9a-f]{40}|[0-9a-f]{64})$/;
|
|
23
44
|
/** A content-addressed blob name: sha256 lowercase hex. */
|
|
@@ -26,6 +47,8 @@ const BLOB_PATTERN = /^[0-9a-f]{64}$/;
|
|
|
26
47
|
const CONTROL_PATTERN = /[\u0000-\u001f\u007f]/;
|
|
27
48
|
/** Monotonic per-process sequence for the same-directory link temps. */
|
|
28
49
|
let linkSequence = 0;
|
|
50
|
+
/** Monotonic per-process sequence for the same-directory spill temps. */
|
|
51
|
+
let spillSequence = 0;
|
|
29
52
|
/** The default layout: the content-addressed snapshot tree. */
|
|
30
53
|
const DEFAULT_LAYOUT = 'hub';
|
|
31
54
|
/** The layout whose files a runtime reads directly at `<root>/<repo>/<file>`. */
|
|
@@ -150,7 +173,9 @@ async function resolveRevision(ctx, spec) {
|
|
|
150
173
|
problems.push(`${url} → HTTP ${String(response.status)}`);
|
|
151
174
|
continue;
|
|
152
175
|
}
|
|
153
|
-
|
|
176
|
+
// Metadata goes through the METADATA byte cap (not the multi-GB model-data cap): it is parsed
|
|
177
|
+
// as JSON, so the cap has to bound the parsed size, not just the wire size.
|
|
178
|
+
const body = JSON.parse(new TextDecoder().decode(await readCapped(response, METADATA_MAX_BYTES)));
|
|
154
179
|
const sha = typeof body === 'object' && body !== null ? body.sha : undefined;
|
|
155
180
|
if (!isSha(sha)) {
|
|
156
181
|
problems.push(`${url} → 响应 sha 不是 40/64 位小写十六进制`);
|
|
@@ -159,6 +184,9 @@ async function resolveRevision(ctx, spec) {
|
|
|
159
184
|
return sha;
|
|
160
185
|
}
|
|
161
186
|
catch (error) {
|
|
187
|
+
// Crossing the byte cap is terminal: every endpoint serves the same oversized object.
|
|
188
|
+
if (error instanceof ProvisionError && error.code === 'fetch/too-large')
|
|
189
|
+
throw error;
|
|
162
190
|
problems.push(`${url} → ${error instanceof Error ? error.message : String(error)}`);
|
|
163
191
|
}
|
|
164
192
|
}
|
|
@@ -185,7 +213,8 @@ async function listFiles(ctx, spec) {
|
|
|
185
213
|
problems.push(`${url} → HTTP ${String(response.status)}`);
|
|
186
214
|
continue;
|
|
187
215
|
}
|
|
188
|
-
|
|
216
|
+
// The siblings list is metadata too: same METADATA cap (parsed as JSON), not the model-data cap.
|
|
217
|
+
const body = JSON.parse(new TextDecoder().decode(await readCapped(response, METADATA_MAX_BYTES)));
|
|
189
218
|
const siblings = typeof body === 'object' && body !== null ? body.siblings : undefined;
|
|
190
219
|
const names = Array.isArray(siblings)
|
|
191
220
|
? siblings.map(row => (typeof row === 'object' && row !== null ? row.rfilename : undefined))
|
|
@@ -198,16 +227,88 @@ async function listFiles(ctx, spec) {
|
|
|
198
227
|
return files;
|
|
199
228
|
}
|
|
200
229
|
catch (error) {
|
|
230
|
+
// Crossing the byte cap is terminal: every endpoint serves the same oversized object.
|
|
231
|
+
if (error instanceof ProvisionError && error.code === 'fetch/too-large')
|
|
232
|
+
throw error;
|
|
201
233
|
problems.push(`${url} → ${error instanceof Error ? error.message : String(error)}`);
|
|
202
234
|
}
|
|
203
235
|
}
|
|
204
236
|
throw new ProvisionError('fetch/failed', `无法列出 ${spec.repo} 的文件:${problems.join('; ')}`);
|
|
205
237
|
}
|
|
206
238
|
/**
|
|
207
|
-
*
|
|
208
|
-
*
|
|
239
|
+
* A {@link BodySpill} that streams one response into `<blobs>/.atomic-spill-*` while hashing it, then
|
|
240
|
+
* renames the temp onto its content-addressed name. Peak memory is a single chunk: the bytes are
|
|
241
|
+
* never all resident at once, on either side of the write.
|
|
242
|
+
*
|
|
243
|
+
* The write itself goes through `node:fs` (`createWriteStream`) because {@link ProvisionFs} has no
|
|
244
|
+
* streaming writer and this change must not move the interface generation; the temp name carries
|
|
245
|
+
* {@link ATOMIC_TEMP_PREFIX}, so {@link sweepAtomicTemps} reaps a crashed writer's leftovers, and the
|
|
246
|
+
* final placement still uses the fs seam (`stat` / `rm` / `rename`).
|
|
247
|
+
*/
|
|
248
|
+
async function openBlobSpill(fs, blobs) {
|
|
249
|
+
const temp = join(blobs, `${ATOMIC_TEMP_PREFIX}spill-${String(process.pid)}-${String((spillSequence += 1))}`);
|
|
250
|
+
await fs.rm(temp).catch(() => undefined);
|
|
251
|
+
const out = createWriteStream(temp);
|
|
252
|
+
const hash = createHash('sha256');
|
|
253
|
+
let failure;
|
|
254
|
+
// A permanent listener: without one, an open/write error would be an unhandled 'error' event.
|
|
255
|
+
out.on('error', (error) => { failure = error; });
|
|
256
|
+
return {
|
|
257
|
+
async write(chunk) {
|
|
258
|
+
if (failure !== undefined)
|
|
259
|
+
throw failure;
|
|
260
|
+
hash.update(chunk);
|
|
261
|
+
if (!out.write(chunk))
|
|
262
|
+
await once(out, 'drain');
|
|
263
|
+
},
|
|
264
|
+
async finish(total) {
|
|
265
|
+
if (failure !== undefined)
|
|
266
|
+
throw failure;
|
|
267
|
+
out.end();
|
|
268
|
+
await finished(out);
|
|
269
|
+
const digest = hash.digest('hex');
|
|
270
|
+
const blob = join(blobs, digest);
|
|
271
|
+
const info = await fs.stat(blob);
|
|
272
|
+
if (info?.isDirectory === true)
|
|
273
|
+
await fs.rm(blob, { recursive: true });
|
|
274
|
+
// Content-addressed: a complete blob of the same size is already the right bytes.
|
|
275
|
+
if (info !== undefined && !info.isDirectory && info.size === total) {
|
|
276
|
+
await fs.rm(temp).catch(() => undefined);
|
|
277
|
+
}
|
|
278
|
+
else {
|
|
279
|
+
await fs.rename(temp, blob);
|
|
280
|
+
}
|
|
281
|
+
return { blob, digest, size: total };
|
|
282
|
+
},
|
|
283
|
+
async abort() {
|
|
284
|
+
out.destroy();
|
|
285
|
+
await fs.rm(temp).catch(() => undefined);
|
|
286
|
+
},
|
|
287
|
+
};
|
|
288
|
+
}
|
|
289
|
+
/**
|
|
290
|
+
* Download one file, trying each endpoint in order, and land it under `blobs`.
|
|
291
|
+
*
|
|
292
|
+
* Below {@link SPILL_THRESHOLD_BYTES} this is the historical path unchanged: read the whole body,
|
|
293
|
+
* digest it, `atomicWrite` the blob. Above it the body streams through {@link openBlobSpill} to a
|
|
294
|
+
* same-directory temp and is renamed into place, so a multi-GB model costs one chunk of heap instead
|
|
295
|
+
* of two copies of itself.
|
|
296
|
+
*
|
|
297
|
+
* `readCappedOrSpill` reports the two fetch failure shapes with different codes precisely so this
|
|
298
|
+
* branch can be exact instead of collapsing them, because they mean different things here:
|
|
299
|
+
*
|
|
300
|
+
* - `fetch/too-large` — the body crosses the hard byte cap. TERMINAL: every mirror serves the same
|
|
301
|
+
* oversized object, so the remaining endpoints are not tried.
|
|
302
|
+
* - `fetch/failed` — a truncated body whose declared length did not match. A TRANSPORT failure that
|
|
303
|
+
* falls through to the next endpoint. This is the case that matters most here: this is the largest
|
|
304
|
+
* model path, the largest thing the framework downloads, and a slow/flaky mirror that truncates
|
|
305
|
+
* must not permanently fail the item.
|
|
306
|
+
*
|
|
307
|
+
* A complete-but-wrong body stays final, but that is a different failure: the caller's `verifyHub` /
|
|
308
|
+
* `verifyFlat` sha256-checks the landed snapshot and raises `verify/failed`, which the provisioner does
|
|
309
|
+
* not retry (another mirror cannot make wrong bytes right).
|
|
209
310
|
*/
|
|
210
|
-
async function
|
|
311
|
+
async function landFile(ctx, endpoints, repo, sha, file, blobs, limits) {
|
|
211
312
|
const problems = [];
|
|
212
313
|
for (const endpoint of endpoints) {
|
|
213
314
|
const url = `${endpoint}/${repo}/resolve/${sha}/${file}`;
|
|
@@ -217,10 +318,20 @@ async function downloadFile(ctx, endpoints, repo, sha, file) {
|
|
|
217
318
|
problems.push(`${url} → HTTP ${String(response.status)}`);
|
|
218
319
|
continue;
|
|
219
320
|
}
|
|
220
|
-
|
|
321
|
+
const landed = await readCappedOrSpill(response, {
|
|
322
|
+
maxBytes: limits.maxBytes,
|
|
323
|
+
spillAtBytes: limits.spillAtBytes,
|
|
324
|
+
spill: () => openBlobSpill(ctx.fs, blobs),
|
|
325
|
+
});
|
|
326
|
+
if (!(landed instanceof Uint8Array))
|
|
327
|
+
return landed;
|
|
328
|
+
const digest = sha256Hex(landed);
|
|
329
|
+
const blob = join(blobs, digest);
|
|
330
|
+
await writeBlob(ctx.fs, blob, landed);
|
|
331
|
+
return { blob, digest, size: landed.byteLength };
|
|
221
332
|
}
|
|
222
333
|
catch (error) {
|
|
223
|
-
if (error instanceof ProvisionError)
|
|
334
|
+
if (error instanceof ProvisionError && error.code === 'fetch/too-large')
|
|
224
335
|
throw error;
|
|
225
336
|
problems.push(`${url} → ${error instanceof Error ? error.message : String(error)}`);
|
|
226
337
|
}
|
|
@@ -276,12 +387,19 @@ function decodeRecord(bytes) {
|
|
|
276
387
|
}
|
|
277
388
|
return { schemaVersion: RECORD_SCHEMA_VERSION, repo: record['repo'], revision: record['revision'], sha: record['sha'], files, digests };
|
|
278
389
|
}
|
|
279
|
-
/** Run `
|
|
280
|
-
async function withLock(ctx,
|
|
281
|
-
const lock = ctx.lock ?? defaultLock(
|
|
390
|
+
/** Run `mission` under the core's placement lock. */
|
|
391
|
+
async function withLock(ctx, mission) {
|
|
392
|
+
const lock = ctx.lock ?? defaultLock({
|
|
393
|
+
// The lock exists for the GiB-scale placement below (`linkOrWrite`); when that outlives
|
|
394
|
+
// `staleMs` the waiter must hear WHY it is not reclaiming rather than only seeing a timeout.
|
|
395
|
+
onSlowHold: info => {
|
|
396
|
+
ctx.logger.warn(`模型放置锁 ${info.path} 的临界区已持有 ${String(info.heldMs)} ms(超过 staleMs ${String(info.staleMs)} ms),` +
|
|
397
|
+
`但 pid ${String(info.pid)} 活着,所以不回收;按 timeoutMs 预算继续等待。`, { path: info.path, pid: info.pid, startedAt: info.startedAt, heldMs: info.heldMs, staleMs: info.staleMs, waitedMs: info.waitedMs });
|
|
398
|
+
},
|
|
399
|
+
});
|
|
282
400
|
const handle = await lock.acquire(lockPath(ctx.home), { timeoutMs: LOCK_TIMEOUT_MS, staleMs: STALE_LOCK_MS });
|
|
283
401
|
try {
|
|
284
|
-
return await
|
|
402
|
+
return await mission();
|
|
285
403
|
}
|
|
286
404
|
finally {
|
|
287
405
|
handle.dispose();
|
|
@@ -290,7 +408,9 @@ async function withLock(ctx, work) {
|
|
|
290
408
|
/**
|
|
291
409
|
* Make `destination` resolve to `blob`: keep an existing link that already resolves to the blob,
|
|
292
410
|
* else place a fresh temp symlink and rename it over the destination. When links are unsupported,
|
|
293
|
-
*
|
|
411
|
+
* copy through a same-directory temp and rename it over the destination — the blob is never held in
|
|
412
|
+
* memory and is never read twice (a GiB-scale model used to be read whole, then written). A
|
|
413
|
+
* destination that is a directory is cleared first.
|
|
294
414
|
*/
|
|
295
415
|
async function linkOrWrite(fs, blob, destination) {
|
|
296
416
|
const parent = dirname(destination);
|
|
@@ -315,7 +435,16 @@ async function linkOrWrite(fs, blob, destination) {
|
|
|
315
435
|
await fs.rm(temp).catch(() => undefined);
|
|
316
436
|
}
|
|
317
437
|
}
|
|
318
|
-
|
|
438
|
+
const temp = join(parent, `${ATOMIC_TEMP_PREFIX}copy-${String(process.pid)}-${String((linkSequence += 1))}`);
|
|
439
|
+
await fs.rm(temp).catch(() => undefined);
|
|
440
|
+
try {
|
|
441
|
+
await fs.copyFile(blob, temp);
|
|
442
|
+
await fs.rename(temp, destination);
|
|
443
|
+
}
|
|
444
|
+
catch (error) {
|
|
445
|
+
await fs.rm(temp).catch(() => undefined);
|
|
446
|
+
throw error;
|
|
447
|
+
}
|
|
319
448
|
}
|
|
320
449
|
/** Write `bytes` to a content-addressed `blob`, repairing a missing, directory or wrong-sized blob. */
|
|
321
450
|
async function writeBlob(fs, blob, bytes) {
|
|
@@ -442,6 +571,10 @@ async function verifyFlat(ctx, item, spec) {
|
|
|
442
571
|
/** Build the built-in model-cache provider. */
|
|
443
572
|
export function modelCacheProvider(options = {}) {
|
|
444
573
|
const id = options.id ?? '@avantf/dsh-plugin-base/model-cache';
|
|
574
|
+
const limits = {
|
|
575
|
+
maxBytes: options.maxBytes ?? MAX_MODEL_BYTES,
|
|
576
|
+
spillAtBytes: options.spillAtBytes ?? SPILL_THRESHOLD_BYTES,
|
|
577
|
+
};
|
|
445
578
|
return {
|
|
446
579
|
id,
|
|
447
580
|
kinds: [MODEL_CACHE_KIND],
|
|
@@ -489,19 +622,16 @@ export function modelCacheProvider(options = {}) {
|
|
|
489
622
|
await ctx.fs.mkdir(blobs);
|
|
490
623
|
const placements = [];
|
|
491
624
|
for (const file of files) {
|
|
492
|
-
const
|
|
493
|
-
ctx.onProgress?.({ key: `${MODEL_CACHE_KIND}+${spec.repo}`, phase: 'download', loaded:
|
|
494
|
-
|
|
495
|
-
const blob = join(blobs, digest);
|
|
496
|
-
// Content-addressed blobs are written atomically.
|
|
497
|
-
await writeBlob(ctx.fs, blob, bytes);
|
|
498
|
-
placements.push({ file, blob, digest });
|
|
625
|
+
const landed = await landFile(ctx, endpoints, spec.repo, sha, file, blobs, limits);
|
|
626
|
+
ctx.onProgress?.({ key: `${MODEL_CACHE_KIND}+${spec.repo}`, phase: 'download', loaded: landed.size, total: landed.size });
|
|
627
|
+
placements.push({ file, blob: landed.blob, digest: landed.digest });
|
|
499
628
|
}
|
|
500
629
|
// Placement and the completion marker are one critical section.
|
|
501
630
|
await withLock(ctx, async () => {
|
|
502
631
|
for (const placement of placements)
|
|
503
632
|
await linkOrWrite(ctx.fs, placement.blob, fileUnder(dataDir, placement.file));
|
|
504
633
|
await sweepAtomicTemps(ctx.fs, sidecar);
|
|
634
|
+
await sweepAtomicTemps(ctx.fs, blobs);
|
|
505
635
|
await ctx.fs.atomicWrite(join(sidecar, RECORD_FILE), encodeRecord({
|
|
506
636
|
schemaVersion: RECORD_SCHEMA_VERSION,
|
|
507
637
|
repo: spec.repo,
|
|
@@ -520,19 +650,16 @@ export function modelCacheProvider(options = {}) {
|
|
|
520
650
|
await ctx.fs.mkdir(snapshot);
|
|
521
651
|
const placements = [];
|
|
522
652
|
for (const file of files) {
|
|
523
|
-
const
|
|
524
|
-
ctx.onProgress?.({ key: `${MODEL_CACHE_KIND}+${spec.repo}`, phase: 'download', loaded:
|
|
525
|
-
|
|
526
|
-
const blob = join(blobs, digest);
|
|
527
|
-
// Content-addressed blobs are written atomically.
|
|
528
|
-
await writeBlob(ctx.fs, blob, bytes);
|
|
529
|
-
placements.push({ file, blob, digest });
|
|
653
|
+
const landed = await landFile(ctx, endpoints, spec.repo, sha, file, blobs, limits);
|
|
654
|
+
ctx.onProgress?.({ key: `${MODEL_CACHE_KIND}+${spec.repo}`, phase: 'download', loaded: landed.size, total: landed.size });
|
|
655
|
+
placements.push({ file, blob: landed.blob, digest: landed.digest });
|
|
530
656
|
}
|
|
531
657
|
// Snapshot links and the ref marker are one critical section.
|
|
532
658
|
await withLock(ctx, async () => {
|
|
533
659
|
for (const placement of placements)
|
|
534
660
|
await linkOrWrite(ctx.fs, placement.blob, fileUnder(snapshot, placement.file));
|
|
535
661
|
await sweepAtomicTemps(ctx.fs, join(root, 'refs'));
|
|
662
|
+
await sweepAtomicTemps(ctx.fs, blobs);
|
|
536
663
|
await ctx.fs.mkdir(join(root, 'refs'));
|
|
537
664
|
await ctx.fs.atomicWrite(join(root, 'refs', revision), new TextEncoder().encode(sha));
|
|
538
665
|
});
|
package/dist/providers/npm.d.ts
CHANGED
|
@@ -14,6 +14,11 @@ export interface NpmPackageSpec {
|
|
|
14
14
|
readonly dir?: string;
|
|
15
15
|
}[];
|
|
16
16
|
}
|
|
17
|
+
/** Walk up from a resolved entry to the owning package directory.
|
|
18
|
+
*
|
|
19
|
+
* The candidate is checked BEFORE stepping up (the entry usually sits directly in the package), and
|
|
20
|
+
* `dirnameImpl` is the platform `dirname` so the win32 separator path is covered from Linux. */
|
|
21
|
+
export declare function packageDirOf(entry: string, name: string, dirnameImpl?: (path: string) => string): string | undefined;
|
|
17
22
|
/** Build the built-in npm provider. */
|
|
18
23
|
export declare function npmPackageProvider(options?: {
|
|
19
24
|
readonly id?: string;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"npm.d.ts","sourceRoot":"","sources":["../../src/providers/npm.ts"],"names":[],"mappings":"AAgBA,OAAO,KAAK,EAA+B,QAAQ,EAAuE,MAAM,aAAa,CAAA;AAE7I,eAAO,MAAM,gBAAgB,gBAAgB,CAAA;AAM7C,2CAA2C;AAC3C,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;IAC1B,0EAA0E;IAC1E,QAAQ,CAAC,iBAAiB,CAAC,EAAE,MAAM,CAAA;IACnC,gGAAgG;IAChG,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS;QAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,OAAO,CAAC;QAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,CAAA;CAC1G;
|
|
1
|
+
{"version":3,"file":"npm.d.ts","sourceRoot":"","sources":["../../src/providers/npm.ts"],"names":[],"mappings":"AAgBA,OAAO,KAAK,EAA+B,QAAQ,EAAuE,MAAM,aAAa,CAAA;AAE7I,eAAO,MAAM,gBAAgB,gBAAgB,CAAA;AAM7C,2CAA2C;AAC3C,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;IACtB,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;IAC1B,0EAA0E;IAC1E,QAAQ,CAAC,iBAAiB,CAAC,EAAE,MAAM,CAAA;IACnC,gGAAgG;IAChG,QAAQ,CAAC,KAAK,CAAC,EAAE,SAAS;QAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,OAAO,CAAC;QAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,CAAA;KAAE,EAAE,CAAA;CAC1G;AA4KD;;;iGAGiG;AACjG,wBAAgB,YAAY,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,GAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAgB,GAAG,MAAM,GAAG,SAAS,CAS7H;AAgID,uCAAuC;AACvC,wBAAgB,kBAAkB,CAAC,OAAO,GAAE;IAAE,QAAQ,CAAC,EAAE,CAAC,EAAE,MAAM,CAAA;CAAO,GAAG,QAAQ,CAqKnF"}
|
package/dist/providers/npm.js
CHANGED
|
@@ -8,9 +8,9 @@ import { pathToFileURL } from 'node:url';
|
|
|
8
8
|
import { ProvisionError } from '../errors.js';
|
|
9
9
|
import { exists, linkOrCopy } from '../fs.js';
|
|
10
10
|
import { verifyIntegrity } from '../integrity.js';
|
|
11
|
-
import { isInside } from '../layout.js';
|
|
11
|
+
import { isInside, assertSafeRelativePath } from '../layout.js';
|
|
12
12
|
import { decodeJson, readInstallManifest } from '../manifest.js';
|
|
13
|
-
import { DEFAULT_MAX_BYTES, candidateUrls, fetchImplOf, readCapped, signalFor } from '../net.js';
|
|
13
|
+
import { DEFAULT_MAX_BYTES, METADATA_MAX_BYTES, candidateUrls, fetchImplOf, readCapped, signalFor } from '../net.js';
|
|
14
14
|
import { assertPackageName } from '../package-name.js';
|
|
15
15
|
import { parseRange, parseVersion, satisfiesRange, selectVersion } from '../semver.js';
|
|
16
16
|
import { extractTarGz } from '../tar.js';
|
|
@@ -45,7 +45,10 @@ async function fetchPackument(ctx, spec) {
|
|
|
45
45
|
}
|
|
46
46
|
if (!response.ok)
|
|
47
47
|
throw new ProvisionError('fetch/failed', `packument ${url} 返回 HTTP ${String(response.status)}`);
|
|
48
|
-
|
|
48
|
+
// Metadata read through a hard byte cap of its own (`METADATA_MAX_BYTES`, not the 256 MiB archive
|
|
49
|
+
// cap): a hostile or broken mirror must not OOM the host with an unbounded `response.json()`, and
|
|
50
|
+
// because a packument is `JSON.parse`d its cap has to bound the PARSED size, not just the wire size.
|
|
51
|
+
const body = JSON.parse(new TextDecoder().decode(await readCapped(response, METADATA_MAX_BYTES)));
|
|
49
52
|
const versions = typeof body === 'object' && body !== null ? body.versions : undefined;
|
|
50
53
|
if (typeof versions !== 'object' || versions === null) {
|
|
51
54
|
throw new ProvisionError('fetch/failed', `packument ${url} 没有 versions`);
|
|
@@ -168,39 +171,66 @@ async function packageVersionOf(ctx, dir) {
|
|
|
168
171
|
return undefined;
|
|
169
172
|
}
|
|
170
173
|
}
|
|
171
|
-
/**
|
|
172
|
-
|
|
173
|
-
|
|
174
|
+
/** Does `dir` end with the package's path segments? Compares segments, not a raw string suffix.
|
|
175
|
+
*
|
|
176
|
+
* `dir.endsWith(name)` never matched a scoped peer on Windows (`…\node_modules\@scope\pkg` does not
|
|
177
|
+
* end with `@scope/pkg`), so peer resolution silently fell through. Comparing the trailing segments
|
|
178
|
+
* on forward slashes works with either separator; no match stays a `peer/unsatisfied` (fail-closed). */
|
|
179
|
+
function dirIsPackage(dir, name) {
|
|
180
|
+
const wanted = name.split('/');
|
|
181
|
+
const got = dir.replaceAll('\\', '/').replace(/\/+$/, '').split('/');
|
|
182
|
+
if (got.length < wanted.length)
|
|
183
|
+
return false;
|
|
184
|
+
const tail = got.slice(got.length - wanted.length);
|
|
185
|
+
return tail.every((segment, index) => segment === wanted[index]);
|
|
186
|
+
}
|
|
187
|
+
/** Walk up from a resolved entry to the owning package directory.
|
|
188
|
+
*
|
|
189
|
+
* The candidate is checked BEFORE stepping up (the entry usually sits directly in the package), and
|
|
190
|
+
* `dirnameImpl` is the platform `dirname` so the win32 separator path is covered from Linux. */
|
|
191
|
+
export function packageDirOf(entry, name, dirnameImpl = dirname) {
|
|
192
|
+
let dir = dirnameImpl(entry);
|
|
174
193
|
for (let depth = 0; depth < 6; depth += 1) {
|
|
175
|
-
if (dir
|
|
176
|
-
return undefined;
|
|
177
|
-
dir = dirname(dir);
|
|
178
|
-
if (dir.endsWith(name))
|
|
194
|
+
if (dirIsPackage(dir, name))
|
|
179
195
|
return dir;
|
|
196
|
+
const parent = dirnameImpl(dir);
|
|
197
|
+
if (parent === dir)
|
|
198
|
+
return undefined;
|
|
199
|
+
dir = parent;
|
|
180
200
|
}
|
|
181
201
|
return undefined;
|
|
182
202
|
}
|
|
183
|
-
/** Resolve a peer directory from
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
203
|
+
/** Resolve a peer directory from this module's tree, falling back to the host CWD with a warning.
|
|
204
|
+
*
|
|
205
|
+
* The order matters: `import.meta.url` is the tree that actually loaded this provider, while the CWD
|
|
206
|
+
* is wherever the host happened to start. A peer resolved from the CWD becomes an ABSOLUTE symlink
|
|
207
|
+
* inside the published version directory (see {@link linkPeers}); it dangles the moment the host
|
|
208
|
+
* starts elsewhere or that tree is removed, and it made "one zod" depend on the launch directory.
|
|
209
|
+
* The CWD stays as a last resort, but never silently. */
|
|
210
|
+
function resolvePeerDir(peer, ctx) {
|
|
211
|
+
const from = (root) => {
|
|
187
212
|
try {
|
|
188
213
|
return dirname(createRequire(root).resolve(`${peer}/package.json`));
|
|
189
214
|
}
|
|
190
215
|
catch {
|
|
191
216
|
// fall through to the entry walk-up
|
|
192
217
|
}
|
|
193
|
-
}
|
|
194
|
-
for (const root of roots) {
|
|
195
218
|
try {
|
|
196
219
|
const entry = createRequire(root).resolve(peer);
|
|
197
|
-
|
|
198
|
-
if (dir !== undefined)
|
|
199
|
-
return dir;
|
|
220
|
+
return packageDirOf(entry, peer);
|
|
200
221
|
}
|
|
201
222
|
catch {
|
|
202
|
-
|
|
223
|
+
return undefined;
|
|
203
224
|
}
|
|
225
|
+
};
|
|
226
|
+
const own = from(import.meta.url);
|
|
227
|
+
if (own !== undefined)
|
|
228
|
+
return own;
|
|
229
|
+
const cwd = join(process.cwd(), 'noop.js');
|
|
230
|
+
const fromCwd = from(cwd);
|
|
231
|
+
if (fromCwd !== undefined) {
|
|
232
|
+
ctx.logger.warn(`peer ${peer} 只能从宿主启动目录解析到(${process.cwd()} → ${fromCwd});发布物里嵌的是该位置的绝对链接,换目录/卸载后会悬空`);
|
|
233
|
+
return fromCwd;
|
|
204
234
|
}
|
|
205
235
|
return undefined;
|
|
206
236
|
}
|
|
@@ -215,7 +245,7 @@ async function linkPeers(pkgDir, spec, staging, ctx) {
|
|
|
215
245
|
assertPackageName(peer, 'peer 名');
|
|
216
246
|
const declared = spec.peers?.find(candidate => candidate.name === peer);
|
|
217
247
|
const optional = meta[peer]?.optional === true || declared?.optional === true;
|
|
218
|
-
const dir = declared?.dir ?? resolvePeerDir(peer);
|
|
248
|
+
const dir = declared?.dir ?? resolvePeerDir(peer, ctx);
|
|
219
249
|
if (dir === undefined) {
|
|
220
250
|
if (optional)
|
|
221
251
|
continue;
|
|
@@ -342,7 +372,18 @@ export function npmPackageProvider(options = {}) {
|
|
|
342
372
|
ctx.logger.warn(`unverifiable: ${versionDir} 的 install.json 没有 integrity;按可用处理`);
|
|
343
373
|
}
|
|
344
374
|
const relativeEntry = manifest.entryDir ?? `node_modules/${spec.name}`;
|
|
375
|
+
// The read-back side of the same check `publish()` runs on write: an `install.json` whose
|
|
376
|
+
// entryDir is absolute or `..`-escaping must not be joined into a path at all.
|
|
377
|
+
try {
|
|
378
|
+
assertSafeRelativePath(relativeEntry);
|
|
379
|
+
}
|
|
380
|
+
catch {
|
|
381
|
+
ctx.logger.warn(`install.json 的 entryDir 不安全,已跳过:${relativeEntry}(${versionDir})`);
|
|
382
|
+
continue;
|
|
383
|
+
}
|
|
345
384
|
const entryDir = join(versionDir, ...relativeEntry.split('/'));
|
|
385
|
+
if (!isInside(versionDir, entryDir))
|
|
386
|
+
continue;
|
|
346
387
|
// A copy whose entry directory is missing is not usable.
|
|
347
388
|
if (!(await exists(ctx.fs, entryDir)))
|
|
348
389
|
continue;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"provisioner.d.ts","sourceRoot":"","sources":["../src/provisioner.ts"],"names":[],"mappings":"AAqCA,OAAO,KAAK,EACV,SAAS,EACT,KAAK,EAeL,aAAa,EAMb,WAAW,EAEX,kBAAkB,EAMnB,MAAM,YAAY,CAAA;AAEnB,yDAAyD;AACzD,eAAO,MAAM,YAAY,EAAE,SAAS,MAAM,EAA6C,CAAA;AAEvF,kDAAkD;AAClD,eAAO,MAAM,mBAAmB,QAAS,CAAA;AAEzC,yDAAyD;AACzD,eAAO,MAAM,mBAAmB,IAAI,CAAA;
|
|
1
|
+
{"version":3,"file":"provisioner.d.ts","sourceRoot":"","sources":["../src/provisioner.ts"],"names":[],"mappings":"AAqCA,OAAO,KAAK,EACV,SAAS,EACT,KAAK,EAeL,aAAa,EAMb,WAAW,EAEX,kBAAkB,EAMnB,MAAM,YAAY,CAAA;AAEnB,yDAAyD;AACzD,eAAO,MAAM,YAAY,EAAE,SAAS,MAAM,EAA6C,CAAA;AAEvF,kDAAkD;AAClD,eAAO,MAAM,mBAAmB,QAAS,CAAA;AAEzC,yDAAyD;AACzD,eAAO,MAAM,mBAAmB,IAAI,CAAA;AAepC,UAAU,iBAAiB;IACzB,QAAQ,CAAC,SAAS,EAAE,SAAS,CAAA;IAC7B,QAAQ,CAAC,KAAK,EAAE,KAAK,CAAA;CACtB;AAED;;;iFAGiF;AACjF,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,aAAa,CAAC,WAAW,CAAC,GAAG,iBAAiB,CAKvF;AA0DD;cACc;AACd,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,kBAAkB,GAAG,WAAW,CAk+B1E"}
|