@intx/tool-packaging 0.2.2 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -12
- package/dist/atomic-apply.d.ts +5 -0
- package/dist/atomic-apply.js +1 -0
- package/dist/cache.d.ts +1 -1
- package/dist/cache.js +14 -19
- package/dist/git-materialize.d.ts +19 -0
- package/dist/git-materialize.js +77 -0
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/loader-internal.d.ts +47 -0
- package/dist/loader-internal.js +66 -0
- package/dist/loader.d.ts +71 -62
- package/dist/loader.js +135 -855
- package/dist/registry-fetch.d.ts +40 -0
- package/dist/registry-fetch.js +231 -0
- package/dist/resolver.d.ts +8 -5
- package/dist/resolver.js +23 -7
- package/dist/store-layout.d.ts +25 -0
- package/dist/store-layout.js +683 -0
- package/package.json +7 -4
|
@@ -0,0 +1,683 @@
|
|
|
1
|
+
// Closure materialization and per-instance store layout for the tool-package
|
|
2
|
+
// loader: fetching + SRI-verifying + extracting each platform-matching entry
|
|
3
|
+
// into the content-addressable cache, resolving dependency ranges, and
|
|
4
|
+
// copying the closure into a per-instance store the loader then imports
|
|
5
|
+
// author code from. Extracted from `loader.ts` so this eval-free
|
|
6
|
+
// materialization concern is isolated from author-code loading. `loader.ts`
|
|
7
|
+
// re-exports `materializeClosure` and `storeEntryDir` for existing consumers.
|
|
8
|
+
import { constants as fsConstants, promises as fs } from "node:fs";
|
|
9
|
+
import path from "node:path";
|
|
10
|
+
import semver from "semver";
|
|
11
|
+
import { getLogger } from "@intx/log";
|
|
12
|
+
import { TarballIntegrityMismatchError } from "./cache.js";
|
|
13
|
+
import { materializeGitEntry } from "./git-materialize.js";
|
|
14
|
+
import { ToolLoaderError, describeError, isEEXIST, isENOENT, platformListMatches, } from "./loader-internal.js";
|
|
15
|
+
const logger = getLogger(["sidecar", "tool-packaging", "store-layout"]);
|
|
16
|
+
/**
|
|
17
|
+
* Lay out a resolved manifest closure into the per-instance store
|
|
18
|
+
* WITHOUT importing any author code. This is phases 1-2 of the loader:
|
|
19
|
+
*
|
|
20
|
+
* 1. Fetch + SRI-verify + extract each platform-matching entry into
|
|
21
|
+
* the content-addressable cache.
|
|
22
|
+
* 2. Resolve the closure's dependency ranges by first arrival, then
|
|
23
|
+
* copy each entry into `<instanceScratchDir>/store/<name>/<version>/`
|
|
24
|
+
* and symlink each direct dep into that entry's `node_modules/` so
|
|
25
|
+
* Node's ancestor walk resolves bare-specifier imports.
|
|
26
|
+
*
|
|
27
|
+
* Returns the `storeDir` the closure was laid out under. The first real
|
|
28
|
+
* `import()` of author code is NOT here — it belongs to `loadManifest`'s
|
|
29
|
+
* phase-3 loop — so a caller (e.g. an install-time probe) can stage the
|
|
30
|
+
* frozen closure into a package directory the loader consumes without
|
|
31
|
+
* executing author code on its host.
|
|
32
|
+
*
|
|
33
|
+
* Only entries whose `os`/`cpu` match `host` are materialized; the rest
|
|
34
|
+
* are skipped with a `platform.mismatch.skipped` debug log. Errors
|
|
35
|
+
* surface as `ToolLoaderError` with a `category` matching the
|
|
36
|
+
* corresponding `DeployApplyErrorCategory`.
|
|
37
|
+
*/
|
|
38
|
+
export async function materializeClosure(args) {
|
|
39
|
+
const filtered = args.manifest.entries.filter((entry) => passesPlatformFilter(entry, args.host));
|
|
40
|
+
const storeDir = path.join(args.instanceScratchDir, "store");
|
|
41
|
+
async function materialize(entry) {
|
|
42
|
+
// Resolve the source's config/mount before any I/O so a cache hit
|
|
43
|
+
// cannot hide a misconfiguration. A source-format asset is a git
|
|
44
|
+
// subtree materialized by checkout and returns early, bypassing the
|
|
45
|
+
// SRI tarball cache; every other entry is a tarball keyed on its SRI.
|
|
46
|
+
const source = entry.source;
|
|
47
|
+
let integrity;
|
|
48
|
+
if (source.kind === "registry") {
|
|
49
|
+
if (!args.registries.has(source.registry)) {
|
|
50
|
+
throw new ToolLoaderError({
|
|
51
|
+
category: "registry.unknown",
|
|
52
|
+
message: `manifest references registry "${source.registry}" which is not in the sidecar config`,
|
|
53
|
+
package: { name: entry.name, version: entry.version },
|
|
54
|
+
});
|
|
55
|
+
}
|
|
56
|
+
integrity = source.integrity;
|
|
57
|
+
}
|
|
58
|
+
else if (source.package.format === "source") {
|
|
59
|
+
const gitDir = args.gitDirs.get(source.assetId);
|
|
60
|
+
if (gitDir === undefined) {
|
|
61
|
+
throw new ToolLoaderError({
|
|
62
|
+
category: "git.materialization.failed",
|
|
63
|
+
message: `manifest entry references git assetId "${source.assetId}" which is not in the deploy pack's git-dirs map`,
|
|
64
|
+
package: { name: entry.name, version: entry.version },
|
|
65
|
+
});
|
|
66
|
+
}
|
|
67
|
+
return materializeGitEntry({
|
|
68
|
+
tree: source.package,
|
|
69
|
+
name: entry.name,
|
|
70
|
+
version: entry.version,
|
|
71
|
+
gitDir,
|
|
72
|
+
instanceScratchDir: args.instanceScratchDir,
|
|
73
|
+
});
|
|
74
|
+
}
|
|
75
|
+
else {
|
|
76
|
+
if (!args.assetMounts.has(source.assetId)) {
|
|
77
|
+
throw new ToolLoaderError({
|
|
78
|
+
category: "asset.mount.missing",
|
|
79
|
+
message: `manifest entry references assetId "${source.assetId}" which is not in the deploy pack's asset-mounts map`,
|
|
80
|
+
package: { name: entry.name, version: entry.version },
|
|
81
|
+
});
|
|
82
|
+
}
|
|
83
|
+
integrity = source.package.integrity;
|
|
84
|
+
}
|
|
85
|
+
// Probe cache presence with `has` rather than `get`: the bytes are
|
|
86
|
+
// only needed when they have to be fetched-then-stored, and
|
|
87
|
+
// `extractTarball` below re-reads them from disk on the way to the
|
|
88
|
+
// per-integrity unpack directory. `has` checks file existence
|
|
89
|
+
// without reading or atime-touching the bytes, so a cache-hit
|
|
90
|
+
// apply avoids the wasted read of a tarball that immediately gets
|
|
91
|
+
// discarded.
|
|
92
|
+
if (!(await args.cache.has(integrity))) {
|
|
93
|
+
const bytes = await args.fetchTarball(entry, {
|
|
94
|
+
registries: args.registries,
|
|
95
|
+
assetRoot: args.assetRoot,
|
|
96
|
+
assetMounts: args.assetMounts,
|
|
97
|
+
});
|
|
98
|
+
try {
|
|
99
|
+
await args.cache.put(integrity, bytes);
|
|
100
|
+
}
|
|
101
|
+
catch (err) {
|
|
102
|
+
if (err instanceof TarballIntegrityMismatchError) {
|
|
103
|
+
throw new ToolLoaderError({
|
|
104
|
+
category: "integrity.mismatch",
|
|
105
|
+
message: `bytes for ${entry.name}@${entry.version} did not match pinned integrity`,
|
|
106
|
+
package: { name: entry.name, version: entry.version },
|
|
107
|
+
});
|
|
108
|
+
}
|
|
109
|
+
throw err;
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
try {
|
|
113
|
+
return await args.cache.extractTarball(integrity);
|
|
114
|
+
}
|
|
115
|
+
catch (err) {
|
|
116
|
+
// Eviction is reserved for the integrity-mismatch path: the bytes
|
|
117
|
+
// on disk no longer match the pinned hash, so the entry is poison
|
|
118
|
+
// and must be re-fetched. Other failures — tar parse errors, FS
|
|
119
|
+
// transients (EIO, ENOSPC) — leave the cached bytes intact. The
|
|
120
|
+
// cache's `evict` defers physical reclaim of the extraction tree
|
|
121
|
+
// until every outstanding `release` from a concurrent
|
|
122
|
+
// `extractTarball` has fired, so a parallel agent's in-flight
|
|
123
|
+
// layout copy against the same extraction will not
|
|
124
|
+
// ENOENT mid-readdir.
|
|
125
|
+
if (err instanceof TarballIntegrityMismatchError) {
|
|
126
|
+
await args.cache.evict(integrity);
|
|
127
|
+
}
|
|
128
|
+
throw new ToolLoaderError({
|
|
129
|
+
category: "tarball.extract.failed",
|
|
130
|
+
message: `tar extraction failed for ${entry.name}@${entry.version}: ${describeError(err)}`,
|
|
131
|
+
package: { name: entry.name, version: entry.version },
|
|
132
|
+
});
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
// 1. Materialize every filtered entry into the cache and capture its
|
|
136
|
+
// extraction directory. This validates the manifest is
|
|
137
|
+
// registry-chain-consistent (each entry resolves end-to-end
|
|
138
|
+
// against its declared source) and primes the cache so the layout
|
|
139
|
+
// step can copy without re-fetching.
|
|
140
|
+
//
|
|
141
|
+
// Each materialize() returns an `{ dir, release }` pair: the cache
|
|
142
|
+
// treats the returned `dir` as held until `release` is called, so a
|
|
143
|
+
// concurrent eviction of the same integrity defers its physical
|
|
144
|
+
// reclaim of the extraction tree until after the buildStoreLayout
|
|
145
|
+
// pass below has finished walking every dir to copy files out.
|
|
146
|
+
// Releases are aggregated and drained in a `finally` so an error
|
|
147
|
+
// mid-layout still hands the cache its references back. They are
|
|
148
|
+
// drained once the layout is built: the phase-3 import loop reads
|
|
149
|
+
// from the copied store, not from these extraction trees, so
|
|
150
|
+
// the extraction handles are a phase-1-2 concern only.
|
|
151
|
+
const extractionByEntry = new Map();
|
|
152
|
+
const entriesByNameVersion = new Map();
|
|
153
|
+
const releases = [];
|
|
154
|
+
try {
|
|
155
|
+
for (const entry of filtered) {
|
|
156
|
+
const handle = await materialize(entry);
|
|
157
|
+
const key = `${entry.name}@${entry.version}`;
|
|
158
|
+
extractionByEntry.set(key, handle.dir);
|
|
159
|
+
entriesByNameVersion.set(key, entry);
|
|
160
|
+
releases.push(handle.release);
|
|
161
|
+
}
|
|
162
|
+
// 2. Build the per-instance store layout. Each filtered entry gets a
|
|
163
|
+
// real directory at `<store>/<name>/<version>/` populated by
|
|
164
|
+
// copies from its cache extraction; the direct-dependency walk
|
|
165
|
+
// then symlinks `node_modules/<dep>` into each layout dir so
|
|
166
|
+
// Node's standard ancestor walk resolves bare-specifier imports
|
|
167
|
+
// from inside the package's body against the closure's pinned
|
|
168
|
+
// versions.
|
|
169
|
+
const rangeResolution = await resolveRangesByFirstArrival({
|
|
170
|
+
topLevel: args.manifest.topLevel,
|
|
171
|
+
filtered,
|
|
172
|
+
extractionByEntry,
|
|
173
|
+
entriesByNameVersion,
|
|
174
|
+
});
|
|
175
|
+
await buildStoreLayout({
|
|
176
|
+
filtered,
|
|
177
|
+
storeDir,
|
|
178
|
+
extractionByEntry,
|
|
179
|
+
rangeResolution,
|
|
180
|
+
});
|
|
181
|
+
}
|
|
182
|
+
finally {
|
|
183
|
+
for (const release of releases) {
|
|
184
|
+
release();
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
return { storeDir, entries: filtered };
|
|
188
|
+
}
|
|
189
|
+
/**
|
|
190
|
+
* True when `entry` may run on `host`: its `os`/`cpu` platform lists (if
|
|
191
|
+
* present) accept the host per npm's allow/block-list semantics. Pure —
|
|
192
|
+
* the single source of truth for the host-match decision, shared by the
|
|
193
|
+
* logging filter `passesPlatformFilter` and `loadManifest`'s phase-3
|
|
194
|
+
* re-selection (which must not re-emit the mismatch debug logs
|
|
195
|
+
* `materializeClosure` already wrote).
|
|
196
|
+
*/
|
|
197
|
+
function entryMatchesHost(entry, host) {
|
|
198
|
+
if (entry.os !== undefined && !platformListMatches(entry.os, host.os)) {
|
|
199
|
+
return false;
|
|
200
|
+
}
|
|
201
|
+
if (entry.cpu !== undefined && !platformListMatches(entry.cpu, host.cpu)) {
|
|
202
|
+
return false;
|
|
203
|
+
}
|
|
204
|
+
return true;
|
|
205
|
+
}
|
|
206
|
+
/**
|
|
207
|
+
* `entryMatchesHost` with the `platform.mismatch.skipped` debug log a
|
|
208
|
+
* rejected entry emits. `materializeClosure` filters with this so the
|
|
209
|
+
* skip is diagnosable; the re-derived `platformListMatches` call on the
|
|
210
|
+
* reject path only runs to pick the os-vs-cpu wording for the message.
|
|
211
|
+
*/
|
|
212
|
+
function passesPlatformFilter(entry, host) {
|
|
213
|
+
if (entryMatchesHost(entry, host))
|
|
214
|
+
return true;
|
|
215
|
+
if (entry.os !== undefined && !platformListMatches(entry.os, host.os)) {
|
|
216
|
+
logger.debug `platform.mismatch.skipped: ${entry.name}@${entry.version} requires os ${entry.os.join(",")} (host is ${host.os})`;
|
|
217
|
+
}
|
|
218
|
+
else if (entry.cpu !== undefined &&
|
|
219
|
+
!platformListMatches(entry.cpu, host.cpu)) {
|
|
220
|
+
logger.debug `platform.mismatch.skipped: ${entry.name}@${entry.version} requires cpu ${entry.cpu.join(",")} (host is ${host.cpu})`;
|
|
221
|
+
}
|
|
222
|
+
return false;
|
|
223
|
+
}
|
|
224
|
+
export function storeEntryDir(storeDir, name, version) {
|
|
225
|
+
// `@scope/name` carries a slash that, taken naively, would push the
|
|
226
|
+
// package's contents one directory deeper than `loadTopLevel`
|
|
227
|
+
// expects. Mirror npm's on-disk shape: `node_modules/@scope/name/`,
|
|
228
|
+
// so a scoped entry's dir is `<store>/@scope/name/<version>/`.
|
|
229
|
+
return path.join(storeDir, name, version);
|
|
230
|
+
}
|
|
231
|
+
/**
|
|
232
|
+
* Build the per-instance `<store>/<name>/<version>/` tree for every
|
|
233
|
+
* filtered manifest entry: copy each entry's source files in from
|
|
234
|
+
* the cache extraction, then symlink each direct dep into the entry's
|
|
235
|
+
* `node_modules/`. Symlinks at the `node_modules/` boundary let Node's
|
|
236
|
+
* realpath-based resolver walk to the dep's own layout dir (with its
|
|
237
|
+
* own `node_modules/`) so transitive resolution composes recursively.
|
|
238
|
+
*/
|
|
239
|
+
async function buildStoreLayout(args) {
|
|
240
|
+
// First materialize every layout dir with its copied contents.
|
|
241
|
+
// node_modules symlinks come after, so a dep's layout dir is already
|
|
242
|
+
// populated when its parent's symlink starts pointing at it.
|
|
243
|
+
for (const entry of args.filtered) {
|
|
244
|
+
const key = `${entry.name}@${entry.version}`;
|
|
245
|
+
const extraction = args.extractionByEntry.get(key);
|
|
246
|
+
if (extraction === undefined) {
|
|
247
|
+
throw new Error(`internal: layout build for ${key} found no cache extraction`);
|
|
248
|
+
}
|
|
249
|
+
const layoutDir = storeEntryDir(args.storeDir, entry.name, entry.version);
|
|
250
|
+
await fs.mkdir(path.dirname(layoutDir), { recursive: true });
|
|
251
|
+
await copyTree(extraction, layoutDir);
|
|
252
|
+
}
|
|
253
|
+
for (const entry of args.filtered) {
|
|
254
|
+
const key = `${entry.name}@${entry.version}`;
|
|
255
|
+
const extraction = args.extractionByEntry.get(key);
|
|
256
|
+
if (extraction === undefined) {
|
|
257
|
+
throw new Error(`internal: layout link pass for ${key} found no cache extraction`);
|
|
258
|
+
}
|
|
259
|
+
const layoutDir = storeEntryDir(args.storeDir, entry.name, entry.version);
|
|
260
|
+
const deps = await readDirectDependencies(extraction, entry);
|
|
261
|
+
if (deps.length === 0)
|
|
262
|
+
continue;
|
|
263
|
+
const modulesDir = path.join(layoutDir, "node_modules");
|
|
264
|
+
await fs.mkdir(modulesDir, { recursive: true });
|
|
265
|
+
for (const dep of deps) {
|
|
266
|
+
const pickedVersion = args.rangeResolution.lookup(dep.name, dep.range);
|
|
267
|
+
if (pickedVersion === null) {
|
|
268
|
+
if (dep.optional) {
|
|
269
|
+
logger.debug `optional.dropped.skipped: ${entry.name}@${entry.version} optional dep ${dep.name}@${dep.range} has no satisfying version in the closure (likely platform-filtered out)`;
|
|
270
|
+
continue;
|
|
271
|
+
}
|
|
272
|
+
throw new ToolLoaderError({
|
|
273
|
+
category: "package.entry.invalid",
|
|
274
|
+
message: `${entry.name}@${entry.version} depends on ${dep.name}@${dep.range} but the manifest closure has no satisfying version; the resolver was expected to include it`,
|
|
275
|
+
package: { name: entry.name, version: entry.version },
|
|
276
|
+
});
|
|
277
|
+
}
|
|
278
|
+
const target = storeEntryDir(args.storeDir, dep.name, pickedVersion);
|
|
279
|
+
const symlinkPath = path.join(modulesDir, dep.name);
|
|
280
|
+
// Scoped deps live one directory deep under `node_modules/`;
|
|
281
|
+
// ensure the scope dir exists before linking.
|
|
282
|
+
await fs.mkdir(path.dirname(symlinkPath), { recursive: true });
|
|
283
|
+
const relativeTarget = path.relative(path.dirname(symlinkPath), target);
|
|
284
|
+
try {
|
|
285
|
+
await fs.symlink(relativeTarget, symlinkPath, "dir");
|
|
286
|
+
}
|
|
287
|
+
catch (err) {
|
|
288
|
+
if (!isEEXIST(err))
|
|
289
|
+
throw err;
|
|
290
|
+
const existing = await fs.readlink(symlinkPath);
|
|
291
|
+
if (existing !== relativeTarget) {
|
|
292
|
+
// A symlink collision inside the loader's per-package
|
|
293
|
+
// layout pass is a loader-layer invariant violation, not an
|
|
294
|
+
// unknown error shape — route it through the same structured
|
|
295
|
+
// envelope every other loader failure uses so atomic-apply
|
|
296
|
+
// surfaces it as `package.entry.invalid` instead of falling
|
|
297
|
+
// back to the unknown-shape catch-all (`factory.construct.
|
|
298
|
+
// failed`).
|
|
299
|
+
throw new ToolLoaderError({
|
|
300
|
+
category: "package.entry.invalid",
|
|
301
|
+
message: `symlink collision at ${symlinkPath}: existing target ${existing} differs from ${relativeTarget}`,
|
|
302
|
+
});
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
/**
|
|
309
|
+
* Walk the closure in BFS order from the top-level pins (in their
|
|
310
|
+
* input order) and record, for each `(name, range)` first encountered,
|
|
311
|
+
* the version chosen out of the closure. Subsequent edges with the
|
|
312
|
+
* same `(name, range)` reuse the recorded pick instead of re-running
|
|
313
|
+
* `semver.maxSatisfying` against the current closure shape.
|
|
314
|
+
*
|
|
315
|
+
* Mirrors the resolver's first-arrival-per-`(name, range)` semantics
|
|
316
|
+
* on the loader side. Without this, two requirers with overlapping
|
|
317
|
+
* ranges of the same dep could each pick a different version of that
|
|
318
|
+
* dep — `maxSatisfying` is deterministic given its candidate set, but
|
|
319
|
+
* the candidate set is the full closure for the name and a transitive
|
|
320
|
+
* addition since the first arrival can shift the answer. Recording
|
|
321
|
+
* the first arrival per range freezes the pick so every requirer in
|
|
322
|
+
* the same equivalence class lands on the same version of the dep.
|
|
323
|
+
*
|
|
324
|
+
* Returns null for a `(name, range)` that has no satisfying entry in
|
|
325
|
+
* the filtered closure; callers decide whether that is fatal (hard
|
|
326
|
+
* dep) or skippable (optional dep).
|
|
327
|
+
*/
|
|
328
|
+
async function resolveRangesByFirstArrival(args) {
|
|
329
|
+
const recorded = new Map();
|
|
330
|
+
const visited = new Set();
|
|
331
|
+
const filteredKeys = new Set(args.filtered.map((e) => `${e.name}@${e.version}`));
|
|
332
|
+
function rangeKey(name, range) {
|
|
333
|
+
return `${name}@${range}`;
|
|
334
|
+
}
|
|
335
|
+
function pickFromClosure(name, range) {
|
|
336
|
+
const candidates = [];
|
|
337
|
+
for (const entry of args.entriesByNameVersion.values()) {
|
|
338
|
+
if (entry.name !== name)
|
|
339
|
+
continue;
|
|
340
|
+
if (!filteredKeys.has(`${entry.name}@${entry.version}`))
|
|
341
|
+
continue;
|
|
342
|
+
candidates.push(entry.version);
|
|
343
|
+
}
|
|
344
|
+
if (candidates.length === 0)
|
|
345
|
+
return null;
|
|
346
|
+
const valid = candidates.filter((v) => semver.valid(v) !== null);
|
|
347
|
+
if (valid.length > 0) {
|
|
348
|
+
const picked = semver.maxSatisfying(valid, range, {
|
|
349
|
+
includePrerelease: true,
|
|
350
|
+
});
|
|
351
|
+
if (picked !== null)
|
|
352
|
+
return picked;
|
|
353
|
+
}
|
|
354
|
+
// Literal-version fallback: when a transitive dep's range is
|
|
355
|
+
// itself a concrete version string (e.g. `'1.0.0'` not
|
|
356
|
+
// `'^1.0.0'`), `maxSatisfying` rejects on prerelease semantics but
|
|
357
|
+
// the literal match is valid.
|
|
358
|
+
if (candidates.includes(range))
|
|
359
|
+
return range;
|
|
360
|
+
return null;
|
|
361
|
+
}
|
|
362
|
+
// BFS frontier carries the entry whose direct deps we are about to
|
|
363
|
+
// fan out on next. Seed with the top-level pins in pin order, mapped
|
|
364
|
+
// through the filtered closure so platform-filtered tops are skipped
|
|
365
|
+
// (their deps would not have layout dirs to link into).
|
|
366
|
+
const queue = [];
|
|
367
|
+
for (const pin of args.topLevel) {
|
|
368
|
+
const key = `${pin.name}@${pin.version}`;
|
|
369
|
+
const entry = args.entriesByNameVersion.get(key);
|
|
370
|
+
if (entry === undefined)
|
|
371
|
+
continue;
|
|
372
|
+
if (!filteredKeys.has(key))
|
|
373
|
+
continue;
|
|
374
|
+
if (visited.has(key))
|
|
375
|
+
continue;
|
|
376
|
+
visited.add(key);
|
|
377
|
+
queue.push(entry);
|
|
378
|
+
}
|
|
379
|
+
while (queue.length > 0) {
|
|
380
|
+
const entry = queue.shift();
|
|
381
|
+
if (entry === undefined)
|
|
382
|
+
break;
|
|
383
|
+
const extraction = args.extractionByEntry.get(`${entry.name}@${entry.version}`);
|
|
384
|
+
if (extraction === undefined)
|
|
385
|
+
continue;
|
|
386
|
+
const deps = await readDirectDependencies(extraction, entry);
|
|
387
|
+
for (const dep of deps) {
|
|
388
|
+
const key = rangeKey(dep.name, dep.range);
|
|
389
|
+
// `recorded.get(key)` returning `null` is the "we picked this
|
|
390
|
+
// range against the closure and got nothing" cached answer.
|
|
391
|
+
// Caching the null is safe only because the closure is static
|
|
392
|
+
// across this loader pass — `entriesByNameVersion` does not
|
|
393
|
+
// grow underneath us. If a future change starts adding entries
|
|
394
|
+
// mid-walk (e.g. lazy fetches during BFS), the cached null
|
|
395
|
+
// would shadow the new candidates and produce a phantom miss;
|
|
396
|
+
// the cache key would need to be invalidated alongside the
|
|
397
|
+
// closure additions.
|
|
398
|
+
let picked = recorded.get(key);
|
|
399
|
+
if (picked === undefined) {
|
|
400
|
+
picked = pickFromClosure(dep.name, dep.range);
|
|
401
|
+
recorded.set(key, picked);
|
|
402
|
+
}
|
|
403
|
+
if (picked === null)
|
|
404
|
+
continue;
|
|
405
|
+
const depKey = `${dep.name}@${picked}`;
|
|
406
|
+
if (visited.has(depKey))
|
|
407
|
+
continue;
|
|
408
|
+
visited.add(depKey);
|
|
409
|
+
const depEntry = args.entriesByNameVersion.get(depKey);
|
|
410
|
+
if (depEntry === undefined)
|
|
411
|
+
continue;
|
|
412
|
+
queue.push(depEntry);
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
return {
|
|
416
|
+
lookup(name, range) {
|
|
417
|
+
const key = rangeKey(name, range);
|
|
418
|
+
if (recorded.has(key)) {
|
|
419
|
+
const picked = recorded.get(key);
|
|
420
|
+
return picked === undefined ? null : picked;
|
|
421
|
+
}
|
|
422
|
+
// The BFS only walks entries reachable from the top-level pins.
|
|
423
|
+
// A dep declared by an entry the BFS did not reach (e.g. a
|
|
424
|
+
// closure entry that no top-level chain ever required) is not
|
|
425
|
+
// pre-recorded; fall through to a fresh pick from the closure
|
|
426
|
+
// so the layout for such entries still resolves deterministically.
|
|
427
|
+
const fallback = pickFromClosure(name, range);
|
|
428
|
+
recorded.set(key, fallback);
|
|
429
|
+
return fallback;
|
|
430
|
+
},
|
|
431
|
+
};
|
|
432
|
+
}
|
|
433
|
+
async function copyTree(srcDir, destDir, extractionRoot = srcDir) {
|
|
434
|
+
await fs.mkdir(destDir, { recursive: true });
|
|
435
|
+
const entries = await fs.readdir(srcDir, { withFileTypes: true });
|
|
436
|
+
for (const entry of entries) {
|
|
437
|
+
const src = path.join(srcDir, entry.name);
|
|
438
|
+
const dest = path.join(destDir, entry.name);
|
|
439
|
+
if (entry.isDirectory()) {
|
|
440
|
+
await copyTree(src, dest, extractionRoot);
|
|
441
|
+
}
|
|
442
|
+
else if (entry.isFile()) {
|
|
443
|
+
try {
|
|
444
|
+
await fs.copyFile(src, dest, fsConstants.COPYFILE_EXCL);
|
|
445
|
+
}
|
|
446
|
+
catch (err) {
|
|
447
|
+
if (!isEEXIST(err))
|
|
448
|
+
throw err;
|
|
449
|
+
}
|
|
450
|
+
}
|
|
451
|
+
else if (entry.isSymbolicLink()) {
|
|
452
|
+
// Preserve symlinks from the tarball verbatim; npm packages
|
|
453
|
+
// occasionally ship them and replacing one with a regular file would
|
|
454
|
+
// change the file's identity.
|
|
455
|
+
//
|
|
456
|
+
// ISOMORPHIC-LAYOUT ASSUMPTION: writing the source-side
|
|
457
|
+
// relative target verbatim into the destination only works
|
|
458
|
+
// because the source extraction tree and the per-instance
|
|
459
|
+
// store tree mirror each other entry-for-entry — the symlink
|
|
460
|
+
// copies into the same shape, so the relative target still
|
|
461
|
+
// resolves to the same sibling in the destination. A future
|
|
462
|
+
// change that flattens, reshapes, or partially copies the
|
|
463
|
+
// extraction tree would invalidate every symlink it touched
|
|
464
|
+
// and would need to rewrite the targets instead of preserving
|
|
465
|
+
// them.
|
|
466
|
+
//
|
|
467
|
+
// Symlink targets originate from the tarball and cross the trust
|
|
468
|
+
// boundary into the sidecar. Resolve each target against the
|
|
469
|
+
// symlink's own directory and verify it lands inside the
|
|
470
|
+
// extraction root; a target that escapes would let a malicious
|
|
471
|
+
// tarball point at arbitrary sidecar-readable files via the
|
|
472
|
+
// layout dir's `node_modules` walk.
|
|
473
|
+
//
|
|
474
|
+
// The `tar` package version we use rejects absolute symlink
|
|
475
|
+
// targets during extraction, so by the time we observe a
|
|
476
|
+
// symlink here it is necessarily relative.
|
|
477
|
+
//
|
|
478
|
+
// The immediate target of `src` may itself be a directory whose
|
|
479
|
+
// own contents include another symlink. Resolving only the
|
|
480
|
+
// first hop with `path.resolve(path.dirname(src), target)`
|
|
481
|
+
// checks containment of the link's literal target — a chain
|
|
482
|
+
// whose first hop lands inside the extraction root but whose
|
|
483
|
+
// realpath ultimately escapes (target is a directory that
|
|
484
|
+
// itself contains an escaping symlink) would slip past.
|
|
485
|
+
// `fs.realpath` walks the full chain and returns the canonical
|
|
486
|
+
// absolute path; verify containment against that.
|
|
487
|
+
const target = await fs.readlink(src);
|
|
488
|
+
// Compare against the realpath of the extraction root so a chain
|
|
489
|
+
// whose canonical path lands under the same logical root, but
|
|
490
|
+
// via a symlinked tmpdir prefix (notably macOS where `/tmp`
|
|
491
|
+
// resolves to `/private/tmp`), is not incorrectly flagged as
|
|
492
|
+
// an escape.
|
|
493
|
+
let realExtractionRoot;
|
|
494
|
+
try {
|
|
495
|
+
realExtractionRoot = await fs.realpath(extractionRoot);
|
|
496
|
+
}
|
|
497
|
+
catch (err) {
|
|
498
|
+
throw new ToolLoaderError({
|
|
499
|
+
category: "package.entry.invalid",
|
|
500
|
+
message: `tarball symlink ${src} → ${target}: extraction-root realpath failed: ${describeError(err)}`,
|
|
501
|
+
});
|
|
502
|
+
}
|
|
503
|
+
// `path.resolve` produces the absolute path the symlink would
|
|
504
|
+
// dereference to without following any links itself; realpath
|
|
505
|
+
// walks the chain. A dangling symlink — one whose target chain
|
|
506
|
+
// ENOENTs before the final inode — is harmless on disk (it
|
|
507
|
+
// points at a name that does not exist), so the containment
|
|
508
|
+
// check falls back to the literal resolved path in that case.
|
|
509
|
+
// Any other realpath error is fatal; we cannot prove containment
|
|
510
|
+
// and the package is rejected.
|
|
511
|
+
//
|
|
512
|
+
// The fallback anchors the literal resolution at `realpath(src
|
|
513
|
+
// dirname)` rather than the as-declared `dirname(src)`. The
|
|
514
|
+
// dirname already exists on disk (extraction wrote it); realpath
|
|
515
|
+
// walks any symlinks in the prefix so the comparison against
|
|
516
|
+
// `realExtractionRoot` is realpath-vs-realpath on both sides.
|
|
517
|
+
// Without this, platforms whose extraction-root prefix contains
|
|
518
|
+
// symlinks (notably macOS, where `/var/folders/...` resolves to
|
|
519
|
+
// `/private/var/folders/...`) would reject a properly-contained
|
|
520
|
+
// dangling link because the literal path keeps the as-declared
|
|
521
|
+
// prefix while the extraction root has been realpath'd.
|
|
522
|
+
let targetAbs;
|
|
523
|
+
try {
|
|
524
|
+
targetAbs = await fs.realpath(path.resolve(path.dirname(src), target));
|
|
525
|
+
}
|
|
526
|
+
catch (err) {
|
|
527
|
+
if (!isENOENT(err)) {
|
|
528
|
+
throw new ToolLoaderError({
|
|
529
|
+
category: "package.entry.invalid",
|
|
530
|
+
message: `tarball contains symlink ${src} → ${target} whose target could not be resolved: ${describeError(err)}`,
|
|
531
|
+
});
|
|
532
|
+
}
|
|
533
|
+
let srcDirReal;
|
|
534
|
+
try {
|
|
535
|
+
srcDirReal = await fs.realpath(path.dirname(src));
|
|
536
|
+
}
|
|
537
|
+
catch (dirErr) {
|
|
538
|
+
throw new ToolLoaderError({
|
|
539
|
+
category: "package.entry.invalid",
|
|
540
|
+
message: `tarball symlink ${src} → ${target}: dirname realpath failed during dangling-link fallback: ${describeError(dirErr)}`,
|
|
541
|
+
});
|
|
542
|
+
}
|
|
543
|
+
targetAbs = path.resolve(srcDirReal, target);
|
|
544
|
+
}
|
|
545
|
+
const realContainmentRoot = realExtractionRoot.endsWith(path.sep)
|
|
546
|
+
? realExtractionRoot
|
|
547
|
+
: realExtractionRoot + path.sep;
|
|
548
|
+
if (targetAbs !== realExtractionRoot &&
|
|
549
|
+
!targetAbs.startsWith(realContainmentRoot)) {
|
|
550
|
+
throw new ToolLoaderError({
|
|
551
|
+
category: "package.entry.invalid",
|
|
552
|
+
message: `tarball contains symlink ${src} → ${target} that escapes the package extraction directory`,
|
|
553
|
+
});
|
|
554
|
+
}
|
|
555
|
+
try {
|
|
556
|
+
await fs.symlink(target, dest);
|
|
557
|
+
}
|
|
558
|
+
catch (err) {
|
|
559
|
+
if (!isEEXIST(err))
|
|
560
|
+
throw err;
|
|
561
|
+
}
|
|
562
|
+
}
|
|
563
|
+
}
|
|
564
|
+
}
|
|
565
|
+
/**
|
|
566
|
+
* Read the package.json at `extractionDir/package.json` and return the
|
|
567
|
+
* union of `dependencies` and `optionalDependencies`. Each entry is
|
|
568
|
+
* tagged with whether it came from the optional field so the layout
|
|
569
|
+
* pass can decide whether a missing closure entry is fatal (hard dep)
|
|
570
|
+
* or skippable (the resolver's platform filter excluded it from the
|
|
571
|
+
* closure for this host).
|
|
572
|
+
*
|
|
573
|
+
* `dependencies` shadows `optionalDependencies` when the same name
|
|
574
|
+
* appears in both — npm treats the dep as required in that case.
|
|
575
|
+
*/
|
|
576
|
+
async function readDirectDependencies(extractionDir, entry) {
|
|
577
|
+
const pkgJsonRaw = await fs.readFile(path.join(extractionDir, "package.json"), "utf8");
|
|
578
|
+
let pkg;
|
|
579
|
+
try {
|
|
580
|
+
pkg = JSON.parse(pkgJsonRaw);
|
|
581
|
+
}
|
|
582
|
+
catch (err) {
|
|
583
|
+
throw new ToolLoaderError({
|
|
584
|
+
category: "package.entry.invalid",
|
|
585
|
+
message: `malformed package.json in ${entry.name}@${entry.version}: ${describeError(err)}`,
|
|
586
|
+
package: { name: entry.name, version: entry.version },
|
|
587
|
+
});
|
|
588
|
+
}
|
|
589
|
+
const byName = new Map();
|
|
590
|
+
if (pkg === null || typeof pkg !== "object")
|
|
591
|
+
return [];
|
|
592
|
+
const record = { ...pkg };
|
|
593
|
+
// A non-string range value (number, null, nested object, array) is
|
|
594
|
+
// a malformed package.json the npm CLI would also reject. Silently
|
|
595
|
+
// dropping it would let the closure resolver later reject the apply
|
|
596
|
+
// with a misleading `package.entry.invalid` for the wrong layer —
|
|
597
|
+
// the malformation is here, not in the closure walk. Surface it as
|
|
598
|
+
// `package.entry.invalid` directly so the operator-facing message
|
|
599
|
+
// points at the bad package.
|
|
600
|
+
//
|
|
601
|
+
// Iteration order matters: write optionalDependencies FIRST, then
|
|
602
|
+
// dependencies. The `dependencies` write overwrites the same key on
|
|
603
|
+
// collision, which is the npm-shadowing rule documented above.
|
|
604
|
+
// Reversing these two blocks would silently make the optional
|
|
605
|
+
// declaration win and demote a hard dependency to optional.
|
|
606
|
+
const optionalDeps = record["optionalDependencies"];
|
|
607
|
+
if (optionalDeps !== undefined) {
|
|
608
|
+
assertDepMapShape(optionalDeps, "optionalDependencies", entry);
|
|
609
|
+
if (optionalDeps !== null && typeof optionalDeps === "object") {
|
|
610
|
+
for (const [name, range] of Object.entries(optionalDeps)) {
|
|
611
|
+
if (typeof range !== "string") {
|
|
612
|
+
throw new ToolLoaderError({
|
|
613
|
+
category: "package.entry.invalid",
|
|
614
|
+
message: `package.json field optionalDependencies["${name}"] in ${entry.name}@${entry.version} is ${typeof range}, expected a string range`,
|
|
615
|
+
package: { name: entry.name, version: entry.version },
|
|
616
|
+
});
|
|
617
|
+
}
|
|
618
|
+
byName.set(name, {
|
|
619
|
+
name,
|
|
620
|
+
range: normalizeDepRange(range),
|
|
621
|
+
optional: true,
|
|
622
|
+
});
|
|
623
|
+
}
|
|
624
|
+
}
|
|
625
|
+
}
|
|
626
|
+
const deps = record["dependencies"];
|
|
627
|
+
if (deps !== undefined) {
|
|
628
|
+
assertDepMapShape(deps, "dependencies", entry);
|
|
629
|
+
if (deps !== null && typeof deps === "object") {
|
|
630
|
+
for (const [name, range] of Object.entries(deps)) {
|
|
631
|
+
if (typeof range !== "string") {
|
|
632
|
+
throw new ToolLoaderError({
|
|
633
|
+
category: "package.entry.invalid",
|
|
634
|
+
message: `package.json field dependencies["${name}"] in ${entry.name}@${entry.version} is ${typeof range}, expected a string range`,
|
|
635
|
+
package: { name: entry.name, version: entry.version },
|
|
636
|
+
});
|
|
637
|
+
}
|
|
638
|
+
byName.set(name, {
|
|
639
|
+
name,
|
|
640
|
+
range: normalizeDepRange(range),
|
|
641
|
+
optional: false,
|
|
642
|
+
});
|
|
643
|
+
}
|
|
644
|
+
}
|
|
645
|
+
}
|
|
646
|
+
return Array.from(byName.values());
|
|
647
|
+
}
|
|
648
|
+
// Protocol prefixes a source-workspace member's on-disk package.json can carry
|
|
649
|
+
// in a dependency range. Both target a package the closure has ALREADY resolved
|
|
650
|
+
// to a single version -- `workspace:` a workspace-local member, `catalog:` a
|
|
651
|
+
// catalog entry the hub expanded at resolve time -- so neither string is a
|
|
652
|
+
// semver range the resolver can pick against.
|
|
653
|
+
const CLOSURE_RESOLVED_RANGE_PREFIXES = ["workspace:", "catalog:"];
|
|
654
|
+
/**
|
|
655
|
+
* Rewrite a `workspace:`/`catalog:` protocol range to `*` when reading a
|
|
656
|
+
* package's dependencies. The closure has one entry of the target's name, so
|
|
657
|
+
* `*` matches it by name; the raw protocol string would otherwise reach
|
|
658
|
+
* `semver.maxSatisfying` and be rejected as unsatisfiable. A registry-published
|
|
659
|
+
* package never carries either protocol (npm rewrites them on publish), so this
|
|
660
|
+
* only affects a source-workspace closure.
|
|
661
|
+
*/
|
|
662
|
+
function normalizeDepRange(range) {
|
|
663
|
+
return CLOSURE_RESOLVED_RANGE_PREFIXES.some((prefix) => range.startsWith(prefix))
|
|
664
|
+
? "*"
|
|
665
|
+
: range;
|
|
666
|
+
}
|
|
667
|
+
/**
|
|
668
|
+
* Reject array-shaped `dependencies` / `optionalDependencies`. The
|
|
669
|
+
* surrounding code narrows with `typeof X === "object"`, which is true
|
|
670
|
+
* for arrays — and `Object.entries(["foo"])` produces `[["0", "foo"]]`,
|
|
671
|
+
* feeding nonsense package names into the closure resolver. Failure
|
|
672
|
+
* downstream is loud but the message points at the wrong layer. Reject
|
|
673
|
+
* at the package-json read with a clear, structured failure instead.
|
|
674
|
+
*/
|
|
675
|
+
function assertDepMapShape(value, field, entry) {
|
|
676
|
+
if (Array.isArray(value)) {
|
|
677
|
+
throw new ToolLoaderError({
|
|
678
|
+
category: "package.entry.invalid",
|
|
679
|
+
message: `package.json#${field} for ${entry.name}@${entry.version} must be an object map of name→range, not an array`,
|
|
680
|
+
package: { name: entry.name, version: entry.version },
|
|
681
|
+
});
|
|
682
|
+
}
|
|
683
|
+
}
|