@exodus/stasis 1.0.0-beta.13 → 1.0.0-beta.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -1
- package/bin/stasis.js +111 -139
- package/package.json +19 -7
- package/src/audit.js +3 -40
- package/src/babel.js +44 -37
- package/src/bundle-reader.js +24 -4
- package/src/cmd/build.js +194 -8
- package/src/cmd/bundle-options.js +21 -18
- package/src/cmd/bundle.js +959 -262
- package/src/cmd/github-bundle.js +17 -8
- package/src/cmd/output.js +10 -3
- package/src/cmd/tokens.js +67 -0
- package/src/diff.js +2 -2
- package/src/flow.js +22 -0
- package/src/installed-commits.js +4 -2
- package/src/loaders/bash.js +2 -27
- package/src/loaders/cargo.js +6 -6
- package/src/loaders/foundry.js +4 -6
- package/src/loaders/paths.js +4 -15
- package/src/loaders/php.js +3 -5
- package/src/loaders/rust.js +19 -39
- package/src/loaders/solidity-ownership.js +4 -15
- package/src/loaders/solidity.js +4 -67
- package/src/metro-assets.js +415 -0
- package/src/metro-resolver.js +5 -3
- package/src/parse.js +4 -3
- package/src/resolve-fields.js +60 -28
- package/src/resolve-node.js +9 -7
- package/src/resolve-typescript.js +3 -3
- package/src/scan.js +137 -42
- package/src/table.js +63 -0
- package/src/validate.js +1241 -0
- package/src/vfs-bundle/entries.js +5 -2
- package/src/vfs-bundle/github.js +67 -20
- package/src/vfs-bundle/glob.js +420 -0
- package/src/vfs-bundle/lazy.js +128 -0
- package/src/vfs-bundle/tree.js +238 -75
- package/src/vfs-bundle/yarnclean.js +100 -0
- package/src/vfs-bundle.js +16 -7
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
import { AsyncLocalStorage } from 'node:async_hooks'
|
|
2
|
+
import { posix } from 'node:path'
|
|
3
|
+
|
|
4
|
+
// A lazy tree (@preventive/deptree's `lazy`): laid out from the lockfile alone, each package's
|
|
5
|
+
// directory left empty until it is unpacked. The host reads synchronously, so a build reading it
|
|
6
|
+
// cannot wait for a tarball: it runs in rounds. A round that reads under a package not yet unpacked
|
|
7
|
+
// is void, whatever it built, and those packages are unpacked for the next; the first round that
|
|
8
|
+
// reads none read nothing but what the full tree holds there, and is the build. What a round's scan
|
|
9
|
+
// reads of a file is kept for the rounds after it where reading it read no such package (Scan's
|
|
10
|
+
// `memo`), so each file is scanned once, but those that read what was not unpacked yet.
|
|
11
|
+
|
|
12
|
+
// The tree's Vfs as tree.js's vfsHost reads it (lstat, readlink, readFile and readdir). A read under
|
|
13
|
+
// the directory of a package of `roots` not in `unpacked`, or a listing of that directory itself,
|
|
14
|
+
// adds it to `reads.asked`, and to `reads.read` but while `reads.probing` (a check of what files the
|
|
15
|
+
// package holds, which reads none of its code), counts in `reads.touches`, and reads what the layout
|
|
16
|
+
// holds there. The directory itself, and those above a package's on the way to it, are as the full
|
|
17
|
+
// tree holds them, so reading them asks for nothing.
|
|
18
|
+
export function lazyView(vfs, { roots, unpacked, reads }) {
|
|
19
|
+
const above = new Set()
|
|
20
|
+
for (const root of roots) {
|
|
21
|
+
for (let dir = posix.dirname(root); dir !== '/' && !above.has(dir); dir = posix.dirname(dir)) above.add(dir)
|
|
22
|
+
}
|
|
23
|
+
const ownerOf = (path) => {
|
|
24
|
+
for (let dir = path; dir !== '/'; dir = posix.dirname(dir)) if (roots.has(dir)) return dir
|
|
25
|
+
return null
|
|
26
|
+
}
|
|
27
|
+
const touch = (path, listing) => {
|
|
28
|
+
const owner = ownerOf(path)
|
|
29
|
+
if (owner === null || unpacked.has(owner)) return
|
|
30
|
+
if (!listing && (path === owner || above.has(path))) return
|
|
31
|
+
reads.asked.add(owner)
|
|
32
|
+
if (reads.probing === 0) reads.read.add(owner)
|
|
33
|
+
reads.touches++
|
|
34
|
+
}
|
|
35
|
+
return {
|
|
36
|
+
lstat(path) {
|
|
37
|
+
touch(path, false)
|
|
38
|
+
return vfs.lstat(path)
|
|
39
|
+
},
|
|
40
|
+
readlink(path) {
|
|
41
|
+
touch(path, false)
|
|
42
|
+
return vfs.readlink(path)
|
|
43
|
+
},
|
|
44
|
+
readFile(path) {
|
|
45
|
+
touch(path, false)
|
|
46
|
+
return vfs.readFile(path)
|
|
47
|
+
},
|
|
48
|
+
readdir(path) {
|
|
49
|
+
touch(path, true)
|
|
50
|
+
return vfs.readdir(path)
|
|
51
|
+
},
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
// A get-or-read cache, `(key, read) => read()`'s value, as for a tree that holds still: but kept only
|
|
56
|
+
// where reading it read nothing a lazy tree does not hold yet (counted by `touches`, a lazy tree's
|
|
57
|
+
// host's), which is read again; and kept nowhere without `cache`. With `retouch`, for a cache that
|
|
58
|
+
// lives no longer than one round of a lazy tree, in which the tree holds still, what such a reading
|
|
59
|
+
// gave is kept too, and each time it is taken it touches again (`retouch()`), as reading it again
|
|
60
|
+
// would: what reads it knows it read what the tree does not hold yet, which a round's scan keeps none
|
|
61
|
+
// of (Scan's `memo`), and it is not read again only to fail again. With `across`, a Map kept across
|
|
62
|
+
// a lazy tree's rounds, what was read with none is kept there, for each round after, as a scan
|
|
63
|
+
// keeps a file it read so.
|
|
64
|
+
export function untouchedMemo(touches = () => 0, { cache = true, retouch, across } = {}) {
|
|
65
|
+
const kept = new Map()
|
|
66
|
+
return (key, read) => {
|
|
67
|
+
if (across?.has(key)) return across.get(key)
|
|
68
|
+
const hit = kept.get(key)
|
|
69
|
+
if (hit !== undefined) {
|
|
70
|
+
if (hit.touched) retouch()
|
|
71
|
+
return hit.value
|
|
72
|
+
}
|
|
73
|
+
const before = touches()
|
|
74
|
+
const value = read()
|
|
75
|
+
const touched = touches() !== before
|
|
76
|
+
if (!cache) return value
|
|
77
|
+
if (!touched && across !== undefined) across.set(key, value)
|
|
78
|
+
else if (!touched || retouch !== undefined) kept.set(key, { value, touched })
|
|
79
|
+
return value
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
// What console.warn gives within a round, held for the round: one that is void says nothing.
|
|
84
|
+
const rounds = new AsyncLocalStorage()
|
|
85
|
+
let holders = 0
|
|
86
|
+
let passed
|
|
87
|
+
function warnHeld(...args) {
|
|
88
|
+
const held = rounds.getStore()
|
|
89
|
+
if (held === undefined) passed.apply(console, args)
|
|
90
|
+
else held.push(args)
|
|
91
|
+
}
|
|
92
|
+
async function holding(held, fn) {
|
|
93
|
+
if (holders++ === 0) {
|
|
94
|
+
passed = console.warn
|
|
95
|
+
console.warn = warnHeld
|
|
96
|
+
}
|
|
97
|
+
try {
|
|
98
|
+
return await rounds.run(held, fn)
|
|
99
|
+
} finally {
|
|
100
|
+
if (--holders === 0 && console.warn === warnHeld) console.warn = passed
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
// `attempt(reads)`'s result, of the first round whose reads ask for no package (`reads.asked`), its
|
|
105
|
+
// warnings given then; after each round that does, `unpack(reads.asked, reads.read)`, `reads.read`
|
|
106
|
+
// those of them it read other than by a probe (lazyView). Each round asks for one package at least
|
|
107
|
+
// that no round before did, so there are no more rounds than packages.
|
|
108
|
+
export async function inRounds(attempt, unpack) {
|
|
109
|
+
for (;;) {
|
|
110
|
+
const reads = { asked: new Set(), read: new Set(), probing: 0, touches: 0 }
|
|
111
|
+
const held = []
|
|
112
|
+
let result
|
|
113
|
+
let failure
|
|
114
|
+
try {
|
|
115
|
+
// eslint-disable-next-line no-await-in-loop -- each round reads what the one before unpacked
|
|
116
|
+
result = await holding(held, () => attempt(reads))
|
|
117
|
+
} catch (error) {
|
|
118
|
+
failure = { error }
|
|
119
|
+
}
|
|
120
|
+
if (reads.asked.size === 0) {
|
|
121
|
+
for (const args of held) console.warn(...args)
|
|
122
|
+
if (failure !== undefined) throw failure.error
|
|
123
|
+
return result
|
|
124
|
+
}
|
|
125
|
+
// eslint-disable-next-line no-await-in-loop -- what the next round reads
|
|
126
|
+
await unpack(reads.asked, reads.read)
|
|
127
|
+
}
|
|
128
|
+
}
|
package/src/vfs-bundle/tree.js
CHANGED
|
@@ -2,7 +2,7 @@ import { constants } from 'node:fs'
|
|
|
2
2
|
import { basename, dirname, join, posix, relative, resolve, sep } from 'node:path'
|
|
3
3
|
|
|
4
4
|
import { isValidRepoField } from '@exodus/stasis-core/bundle'
|
|
5
|
-
import { NO_ENTRY, packageJSONStat, packageJSONText, parseGithubRepository, readJson } from '@exodus/stasis-core/bundle-util'
|
|
5
|
+
import { NO_ENTRY, invalidPackageConfig, lexists, packageJSONStat, packageJSONText, parseGithubRepository, readJson } from '@exodus/stasis-core/bundle-util'
|
|
6
6
|
import { byName } from '@exodus/stasis-core/host'
|
|
7
7
|
import { hasNodeModulesSegment, isPlainObject } from '@exodus/stasis-core/util'
|
|
8
8
|
import { buildNpmTree, findNpmWorkspaces } from '@preventive/deptree/npm.js'
|
|
@@ -11,14 +11,22 @@ import { LockfileError, TomlError, buildSoldeerTree } from '@preventive/deptree/
|
|
|
11
11
|
import { buildYarn1Tree, findYarn1Workspaces } from '@preventive/deptree/yarn1.js'
|
|
12
12
|
import { VfsError } from '@preventive/vfs'
|
|
13
13
|
import { createNodeResolver } from '../resolve-node.js'
|
|
14
|
+
import { globSync, matchesGlob } from './glob.js'
|
|
15
|
+
import { lazyView, untouchedMemo } from './lazy.js'
|
|
14
16
|
import { settleSoldeer } from './soldeer.js'
|
|
17
|
+
import { YARNCLEAN, yarnclean } from './yarnclean.js'
|
|
15
18
|
import { isDir, isFile } from '../resolve-typescript.js'
|
|
16
19
|
|
|
17
20
|
// A project's dependencies laid out in memory from its lockfile by @preventive/deptree, as `pnpm
|
|
18
21
|
// install --frozen-lockfile --ignore-scripts` lays out its node_modules with pnpm 9, 10, 11 or 12,
|
|
19
22
|
// `yarn install --frozen-lockfile --ignore-scripts` with yarn 1.22, `npm ci --ignore-scripts` with
|
|
20
23
|
// npm 10.9 or 11, or `soldeer install` its dependencies folder with Soldeer 0.12, and the host that
|
|
21
|
-
// reads the project through them
|
|
24
|
+
// reads the project through them; without devDependencies, as `pnpm install --prod`, `yarn install
|
|
25
|
+
// --production` and `npm ci --omit=dev` lay it out, where the project's stasis.config.json says so.
|
|
26
|
+
// node_modules is then cleaned as `yarn autoclean` would clean it by each .yarnclean in the
|
|
27
|
+
// directory installed or above it (yarnclean.js), whatever installs it. The project is read through
|
|
28
|
+
// a host it is given, and nothing else. pnpm's and npm's trees may be laid out lazily, each package
|
|
29
|
+
// unpacked once asked for (lazy.js).
|
|
22
30
|
|
|
23
31
|
// cwd or the nearest of its ancestors that `holds`, or null.
|
|
24
32
|
export function nearest(cwd, holds) {
|
|
@@ -188,7 +196,9 @@ const npmHost = (npm, given) => {
|
|
|
188
196
|
// (for the os given, where that changes it); whether a directory holding the lockfile is installed
|
|
189
197
|
// from by itself, by the names in it and in each directory above it (`above()`, as listings); the
|
|
190
198
|
// projects it finds in a view of that directory, for the machine given; the directory it installs
|
|
191
|
-
// in each, and any it hides, as another package manager's;
|
|
199
|
+
// in each, and any it hides, as another package manager's; whether it lays out a tree from the
|
|
200
|
+
// lockfile alone, which may then be lazy; and the tree, without what devDependencies alone reach
|
|
201
|
+
// where `given.production` (Soldeer has no devDependencies, so it leaves out nothing).
|
|
192
202
|
const PACKAGE_MANAGERS = {
|
|
193
203
|
pnpm: {
|
|
194
204
|
kind: 'js',
|
|
@@ -200,7 +210,8 @@ const PACKAGE_MANAGERS = {
|
|
|
200
210
|
alone: async (names, above) => !(await above()).some((dir) => dir.includes('pnpm-workspace.yaml')),
|
|
201
211
|
projects: (view, pnpm) => findPnpmProjects({ project: view, host: { pnpm } }),
|
|
202
212
|
installs: 'node_modules',
|
|
203
|
-
|
|
213
|
+
lazy: true,
|
|
214
|
+
build: (view, pnpm, _file, given) => buildPnpmTree({ project: view, host: { pnpm, ...target(given) }, cache: given?.cache, lazy: given?.lazy, production: given?.production }),
|
|
204
215
|
},
|
|
205
216
|
yarn1: {
|
|
206
217
|
kind: 'js',
|
|
@@ -214,7 +225,7 @@ const PACKAGE_MANAGERS = {
|
|
|
214
225
|
installs: 'node_modules',
|
|
215
226
|
build: (view, yarn, file, given) => {
|
|
216
227
|
const { libc: _, ...host } = target({ ...given, libc: 'unknown' }) // yarn 1 matches no libc
|
|
217
|
-
return naming(file, buildYarn1Tree({ project: view, host: { yarn, ...host }, cache: given?.cache }))
|
|
228
|
+
return naming(file, buildYarn1Tree({ project: view, host: { yarn, ...host }, cache: given?.cache, production: given?.production }))
|
|
218
229
|
},
|
|
219
230
|
},
|
|
220
231
|
npm: {
|
|
@@ -227,7 +238,8 @@ const PACKAGE_MANAGERS = {
|
|
|
227
238
|
alone: async (names, above) => !(await above()).some((dir) => dir.includes('package.json')),
|
|
228
239
|
projects: (view, _npm, given) => findNpmWorkspaces({ project: view, os: target(given).os }),
|
|
229
240
|
installs: 'node_modules',
|
|
230
|
-
|
|
241
|
+
lazy: true,
|
|
242
|
+
build: (view, npm, file, given) => naming(file, buildNpmTree({ project: view, host: npmHost(npm, given), cache: given?.cache, lazy: given?.lazy, production: given?.production })),
|
|
231
243
|
},
|
|
232
244
|
soldeer: {
|
|
233
245
|
kind: 'sol',
|
|
@@ -251,21 +263,46 @@ const PACKAGE_MANAGERS = {
|
|
|
251
263
|
// Their names.
|
|
252
264
|
const PACKAGE_MANAGER_NAMES = Object.keys(PACKAGE_MANAGERS)
|
|
253
265
|
|
|
254
|
-
// layOutTree's tree, with the host reading `project` through it (treeHost)
|
|
266
|
+
// layOutTree's tree, with the host reading `project` through it (treeHost), but a lazy one's, whose
|
|
267
|
+
// hosts are made for each round that reads it.
|
|
255
268
|
export async function loadTree(options) {
|
|
256
269
|
const tree = await layOutTree(options)
|
|
257
|
-
return { ...tree, host: treeHost(tree, options.project) }
|
|
270
|
+
return tree.lazy === undefined ? { ...tree, host: treeHost(tree, options.project) } : tree
|
|
258
271
|
}
|
|
259
272
|
|
|
260
273
|
// The host reading `project`, a host of the project's Vfs, through `tree` (layOutTree's): the tree
|
|
261
|
-
// serves what the package manager installs, and any file it writes beside it at the root.
|
|
262
|
-
|
|
274
|
+
// serves what the package manager installs, and any file it writes beside it at the root. A lazy
|
|
275
|
+
// tree is read as lazyView reads it, into the round's `reads`, caching nothing it reads of a package
|
|
276
|
+
// not unpacked yet: each such read reaches it, so that what a file's scan reads tells whether it
|
|
277
|
+
// touched one. Its `probing(fn)` is `fn()`, its reads taken as probes of what files a package holds
|
|
278
|
+
// (lazyView), which read none of its code.
|
|
279
|
+
export function treeHost(tree, project, reads) {
|
|
263
280
|
const pm = PACKAGE_MANAGERS[tree.packageManager]
|
|
264
281
|
const files = tree.vfs.readdir('/').filter((name) => tree.vfs.lstat(`/${name}`).type === 'file')
|
|
265
282
|
const installs = [...[...tree.projects].map((dir) => join(dir, pm.installs)), ...files]
|
|
266
|
-
return vfsHost(tree.vfs, { root: tree.root, outside: project, installs, hides: pm.hides })
|
|
283
|
+
if (tree.lazy === undefined) return vfsHost(tree.vfs, { root: tree.root, outside: project, installs, hides: pm.hides })
|
|
284
|
+
const host = vfsHost(lazyView(tree.vfs, { ...tree.lazy, reads }), { root: tree.root, outside: project, installs, hides: pm.hides, touches: () => reads.touches, retouch: () => reads.touches++ })
|
|
285
|
+
host.probing = (fn) => {
|
|
286
|
+
reads.probing++
|
|
287
|
+
try {
|
|
288
|
+
return fn()
|
|
289
|
+
} finally {
|
|
290
|
+
reads.probing--
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
return host
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
// Refused where `packageManager` lays out no tree from the lockfile alone, which `lazy` would have.
|
|
298
|
+
export function checkLazy(name, packageManager) {
|
|
299
|
+
if (!PACKAGE_MANAGERS[packageManager].lazy) throw new Error(`${name}: lazy is taken with pnpm and npm alone, which lay out a tree from the lockfile alone; ${packageManager}'s is not`)
|
|
267
300
|
}
|
|
268
301
|
|
|
302
|
+
// Whether `packageManager`'s tree of the project `project` holds at `cwd` can be lazy, as layOutTree
|
|
303
|
+
// lays it out: pnpm's and npm's, but where a .yarnclean in cwd or above it keeps paths with `!`.
|
|
304
|
+
export const lazyTaken = (project, packageManager, cwd) => PACKAGE_MANAGERS[packageManager].lazy === true && !yarnclean(project, cwd)?.keeps
|
|
305
|
+
|
|
269
306
|
// The lockfile `packageManager` installs from, e.g. 'pnpm-lock.yaml'.
|
|
270
307
|
export const lockfileOf = (packageManager) => PACKAGE_MANAGERS[packageManager].lockfile
|
|
271
308
|
|
|
@@ -289,22 +326,25 @@ export const lockfileRoot = (host, packageManager, cwd, os) => rootOf(host, PACK
|
|
|
289
326
|
// The package managers that install node_modules.
|
|
290
327
|
export const NODE_MODULES_MANAGERS = PACKAGE_MANAGER_NAMES.filter((name) => PACKAGE_MANAGERS[name].installs === 'node_modules')
|
|
291
328
|
|
|
329
|
+
// Why the package manager is detected, as a refusal to detect one says: none was given.
|
|
330
|
+
const NOT_GIVEN = 'no packageManager given'
|
|
331
|
+
|
|
292
332
|
// The one of the package managers `names` that installs `cwd` in the project `host` reads from a
|
|
293
333
|
// directory holding its lockfile, as lockfileRoot finds it for `os`: refused where none or more than
|
|
294
|
-
// one does.
|
|
295
|
-
export function detectPackageManager(name, host, cwd, { names = PACKAGE_MANAGER_NAMES, os } = {}) {
|
|
334
|
+
// one does, saying `why` it is detected.
|
|
335
|
+
export function detectPackageManager(name, host, cwd, { names = PACKAGE_MANAGER_NAMES, os, why = NOT_GIVEN } = {}) {
|
|
296
336
|
const real = host.realpath(cwd)
|
|
297
337
|
const found = names.flatMap((pm) => {
|
|
298
338
|
const root = rootOf(host, PACKAGE_MANAGERS[pm], real, os)
|
|
299
339
|
return root === null ? [] : [[pm, join(root, PACKAGE_MANAGERS[pm].lockfile)]]
|
|
300
340
|
})
|
|
301
341
|
if (found.length === 1) return found[0][0]
|
|
302
|
-
if (found.length === 0) throw noLockfile(name, real, names)
|
|
303
|
-
throw new Error(`${name}:
|
|
342
|
+
if (found.length === 0) throw noLockfile(name, real, names, why)
|
|
343
|
+
throw new Error(`${name}: ${why}, and more than one lockfile installs ${real}: ${found.map(([pm, file]) => `${file} (${pm})`).join(', ')}`)
|
|
304
344
|
}
|
|
305
345
|
|
|
306
|
-
// That none of the lockfiles of `names` installs `place
|
|
307
|
-
export const noLockfile = (name, place, names = PACKAGE_MANAGER_NAMES) => new Error(`${name}:
|
|
346
|
+
// That none of the lockfiles of `names` installs `place`, saying `why` one is looked for.
|
|
347
|
+
export const noLockfile = (name, place, names = PACKAGE_MANAGER_NAMES, why = NOT_GIVEN) => new Error(`${name}: ${why}, and none of ${names.map((pm) => PACKAGE_MANAGERS[pm].lockfile).join(', ')} installs ${place}`)
|
|
308
348
|
|
|
309
349
|
// Whether a directory's listing (listRepoDir's) holds `packageManager`'s lockfile.
|
|
310
350
|
const listsLockfile = (listing, packageManager) => listing.some((entry) => entry.path === PACKAGE_MANAGERS[packageManager].lockfile && entry.type === 'blob')
|
|
@@ -361,8 +401,15 @@ export function checkVfs(name, vfs) {
|
|
|
361
401
|
// written through. `client`, a @preventive/upstream/github.js client, is what a Soldeer git
|
|
362
402
|
// dependency is fetched through (deptree's `github`); without one, a lockfile holding one is refused
|
|
363
403
|
// before anything is fetched. `cache` (checkCache) is where npm packages' tarballs and version
|
|
364
|
-
// documents are kept, which deptree fetches for pnpm, yarn1 and npm.
|
|
365
|
-
|
|
404
|
+
// documents are kept, which deptree fetches for pnpm, yarn1 and npm. Each project's node_modules is
|
|
405
|
+
// cleaned by every .yarnclean in cwd and the directories above it (yarnclean.js). With `lazy`, for
|
|
406
|
+
// pnpm and npm (checkLazy), the tree is laid out with nothing fetched, and comes with `unpack` and
|
|
407
|
+
// `lazy` besides (lazyTree). Without `installDevDependencies`, what devDependencies alone reach is
|
|
408
|
+
// left out, as the package manager's production install leaves it out (deptree's `production`).
|
|
409
|
+
// With `refuseInstalled`, refused before anything is fetched where `project` holds anything at all
|
|
410
|
+
// (a link too) where the tree installs a project's dependencies: its node_modules, or Soldeer's
|
|
411
|
+
// dependencies folder, which the tree would serve in place of what is there.
|
|
412
|
+
async function layOutTree({ project, packageManager, cwd, packageManagerVersion, os, cpu, libc, client, cache, lazy = false, installDevDependencies = true, refuseInstalled = false }) {
|
|
366
413
|
const pm = PACKAGE_MANAGERS[packageManager]
|
|
367
414
|
const found = pm.root(project, cwd, os)
|
|
368
415
|
if (found === null) throw new Error(`no ${pm.lockfile} found in ${cwd} or any parent directory`)
|
|
@@ -375,15 +422,109 @@ async function layOutTree({ project, packageManager, cwd, packageManagerVersion,
|
|
|
375
422
|
const projects = new Set(pm.projects(view, version, { os, cpu, libc }))
|
|
376
423
|
const other = pm.kind === 'js' ? outsider(project, found, cwd, projects) : null
|
|
377
424
|
if (other !== null) throw new Error(`${file} does not install ${other}: it is none of the lockfile's projects`)
|
|
378
|
-
|
|
425
|
+
if (refuseInstalled) {
|
|
426
|
+
for (const dir of projects) {
|
|
427
|
+
const at = join(found, dir, pm.installs)
|
|
428
|
+
if (lexists(at, project)) throw new Error(`${at} is there already, and the tree is laid out from ${file} alone, over nothing installed: remove it first`)
|
|
429
|
+
}
|
|
430
|
+
}
|
|
431
|
+
// Read before anything is fetched: matching its patterns needs the optional minimatch.
|
|
432
|
+
const clean = pm.kind === 'js' ? yarnclean(project, cwd) : null
|
|
433
|
+
if (lazy && clean?.keeps) throw new Error(`a ${YARNCLEAN} in ${cwd} or above it keeps paths with \`!\`, which lazy does not take: what it keeps of one package turns on what the others hold, which a lazy tree has not fetched`)
|
|
434
|
+
const built = await pm.build(view, version, file, { os, cpu, libc, client, cache, lazy, production: !installDevDependencies })
|
|
435
|
+
// Each project's node_modules, cleaned as `yarn autoclean` would clean it, whatever installs it.
|
|
436
|
+
const cleanAll = () => {
|
|
437
|
+
if (clean !== null) for (const dir of projects) clean(built.vfs, posix.join('/', dir, pm.installs))
|
|
438
|
+
}
|
|
439
|
+
cleanAll()
|
|
379
440
|
return {
|
|
380
441
|
root: project.realpath(found),
|
|
381
|
-
vfs,
|
|
442
|
+
vfs: built.vfs,
|
|
382
443
|
projects,
|
|
383
|
-
stats,
|
|
384
|
-
installed,
|
|
444
|
+
stats: built.stats,
|
|
445
|
+
installed: built.installed,
|
|
385
446
|
packageManager,
|
|
386
447
|
packageManagerVersion: version ?? (String(pinned).startsWith(`${pm.pin}@`) ? /^[^@]+@([^+]+)/u.exec(pinned)[1] : undefined),
|
|
448
|
+
...lazy ? lazyTree(built, cleanAll) : {},
|
|
449
|
+
}
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
// A package name, as a directory of node_modules holds it: one name, or a scope and one.
|
|
453
|
+
const PACKAGE_NAME = /^(?:@[^/@.][^/]*\/)?[^/@.][^/]*$/u
|
|
454
|
+
|
|
455
|
+
// The directories among `roots` of what the package at `dir` depends on, as its package.json names
|
|
456
|
+
// them (dependencies, optionalDependencies and peerDependencies) and Node finds them from `dir` in
|
|
457
|
+
// `vfs`: what the build reads next, where it reads that package, but none it is held to.
|
|
458
|
+
function dependenciesOf(vfs, dir, roots) {
|
|
459
|
+
const manifest = readJson(`${dir}/package.json`, vfs)
|
|
460
|
+
if (!isPlainObject(manifest)) return []
|
|
461
|
+
const names = new Set(['dependencies', 'optionalDependencies', 'peerDependencies'].flatMap((field) => (isPlainObject(manifest[field]) ? Object.keys(manifest[field]) : [])))
|
|
462
|
+
const found = []
|
|
463
|
+
for (const name of names) {
|
|
464
|
+
if (!PACKAGE_NAME.test(name)) continue
|
|
465
|
+
for (let at = dir; ; at = posix.dirname(at)) {
|
|
466
|
+
if (posix.basename(at) !== 'node_modules') {
|
|
467
|
+
let real
|
|
468
|
+
try {
|
|
469
|
+
real = vfs.realpath(posix.join(at, 'node_modules', name))
|
|
470
|
+
} catch {
|
|
471
|
+
real = null
|
|
472
|
+
}
|
|
473
|
+
if (real !== null) {
|
|
474
|
+
if (roots.has(real)) found.push(real)
|
|
475
|
+
break
|
|
476
|
+
}
|
|
477
|
+
}
|
|
478
|
+
if (at === '/') break
|
|
479
|
+
}
|
|
480
|
+
}
|
|
481
|
+
return found
|
|
482
|
+
}
|
|
483
|
+
|
|
484
|
+
// A lazy tree's `unpack(dirs)`, of its packages' directories from `/` of its Vfs, as lazyView asks
|
|
485
|
+
// for them, which deptree unpacks, cleaned as the rest is; and its `lazy`, those directories, as
|
|
486
|
+
// lazyView takes them: `roots`, each the layout has (a .yarnclean may take some away), and
|
|
487
|
+
// `unpacked`, each unpacked so far. Positive patterns alone clean each path by its own, whatever
|
|
488
|
+
// else the tree holds, so cleaning again cleans each package unpacked as the full tree's is.
|
|
489
|
+
// `installed` takes in what each one's version document says, once it is unpacked. What each
|
|
490
|
+
// package `read` (inRounds') depends on is unpacked with it, as the build would ask for it a round
|
|
491
|
+
// later, each round a build; but not what one only probed for its files depends on, nor what those
|
|
492
|
+
// depend on in turn, which most builds never read: a package's dependencies are often its tools. One
|
|
493
|
+
// deptree refuses is left for the build to ask for, which is then refused, so that one it never
|
|
494
|
+
// reads is never refused, as one never fetched is not.
|
|
495
|
+
function lazyTree({ vfs, installed, unpack }, cleanAll) {
|
|
496
|
+
const roots = new Set(installed.map(({ path }) => `/${path}`).filter((dir) => vfs.isDirectory(dir)))
|
|
497
|
+
const unpacked = new Set()
|
|
498
|
+
const refused = new Set()
|
|
499
|
+
const at = new Map(installed.map(({ path }, i) => [path, i]))
|
|
500
|
+
const unpackOne = async (dir) => {
|
|
501
|
+
const [entry] = await unpack([dir.slice(1)])
|
|
502
|
+
installed[at.get(entry.path)] = entry
|
|
503
|
+
unpacked.add(dir)
|
|
504
|
+
}
|
|
505
|
+
return {
|
|
506
|
+
lazy: { roots, unpacked },
|
|
507
|
+
async unpack(asked, read = asked) {
|
|
508
|
+
// Each asked for, and once one read is unpacked, what it depends on, then and there rather
|
|
509
|
+
// than once every one asked for is: so the fetches of both share the time, not one waiting
|
|
510
|
+
// on the last of the other. The first refusal of one asked for is thrown once every one is
|
|
511
|
+
// done, as deptree throws one.
|
|
512
|
+
const seen = new Set(asked)
|
|
513
|
+
const ahead = []
|
|
514
|
+
const done = await Promise.allSettled([...asked].map(async (dir) => {
|
|
515
|
+
await unpackOne(dir)
|
|
516
|
+
if (!read.has(dir)) return
|
|
517
|
+
for (const dependency of dependenciesOf(vfs, dir, roots)) {
|
|
518
|
+
if (seen.has(dependency) || unpacked.has(dependency) || refused.has(dependency)) continue
|
|
519
|
+
seen.add(dependency)
|
|
520
|
+
ahead.push(unpackOne(dependency).catch(() => refused.add(dependency)))
|
|
521
|
+
}
|
|
522
|
+
}))
|
|
523
|
+
await Promise.all(ahead)
|
|
524
|
+
const failed = done.find(({ status }) => status === 'rejected')
|
|
525
|
+
if (failed !== undefined) throw failed.reason
|
|
526
|
+
cleanAll()
|
|
527
|
+
},
|
|
387
528
|
}
|
|
388
529
|
}
|
|
389
530
|
|
|
@@ -429,10 +570,24 @@ const dirent = (name, type) => ({
|
|
|
429
570
|
// A view over the Vfs's bytes, never a copy.
|
|
430
571
|
const asBuffer = (bytes) => Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength)
|
|
431
572
|
|
|
432
|
-
//
|
|
573
|
+
// `make()`, an error for a name a lookup does not find, made without a stack: a resolver probes
|
|
574
|
+
// many names for each it finds, and its callers take a miss as none, so the stack is never read,
|
|
575
|
+
// and building it is most of what a miss costs.
|
|
576
|
+
function stackless(make) {
|
|
577
|
+
const limit = Error.stackTraceLimit
|
|
578
|
+
Error.stackTraceLimit = 0
|
|
579
|
+
try {
|
|
580
|
+
return make()
|
|
581
|
+
} finally {
|
|
582
|
+
Error.stackTraceLimit = limit
|
|
583
|
+
}
|
|
584
|
+
}
|
|
585
|
+
|
|
586
|
+
// What `vfs` holds at `p` itself (the last link not followed), or null: the Vfs throws for a name
|
|
587
|
+
// it does not hold, which is made without a stack (stackless).
|
|
433
588
|
function lstatOrNull(vfs, p) {
|
|
434
589
|
try {
|
|
435
|
-
return vfs.lstat(p)
|
|
590
|
+
return stackless(() => vfs.lstat(p))
|
|
436
591
|
} catch (err) {
|
|
437
592
|
if (NO_ENTRY.has(err?.code)) return null
|
|
438
593
|
throw err
|
|
@@ -443,8 +598,6 @@ const TREE = 'tree'
|
|
|
443
598
|
const NONE = 'none'
|
|
444
599
|
const OUTSIDE = 'outside'
|
|
445
600
|
|
|
446
|
-
const invalidPackageConfig = (path, cause) => Object.assign(new Error(`Invalid package config ${path}.`, { cause }), { code: 'ERR_INVALID_PACKAGE_CONFIG' })
|
|
447
|
-
|
|
448
601
|
// A `host` (@exodus/stasis-core/host) over `vfs`. Alone, every path is the Vfs's. With `root`, a
|
|
449
602
|
// real path, and `outside`, a host of the rest, the Vfs's `/` stands for `root`, holding the tree
|
|
450
603
|
// the package manager lays out there: each of the paths it `installs` (from `root`) is served from
|
|
@@ -454,8 +607,11 @@ const invalidPackageConfig = (path, cause) => Object.assign(new Error(`Invalid p
|
|
|
454
607
|
// the OS reads one: a `..` after a link leads up from where the link leads, never textually (Node's
|
|
455
608
|
// own resolution, `findPackageJSON` and `resolve`, normalizes its paths first, as Node does). A
|
|
456
609
|
// relative path is from `/`. What it reads is cached, as for a tree that holds still, unless `cache`
|
|
457
|
-
// is false
|
|
458
|
-
|
|
610
|
+
// is false; with `touches`, a count of the reads that read what the tree does not hold yet (a lazy
|
|
611
|
+
// tree's, lazy.js), only what was read with none is cached, so that each of those is read again; but
|
|
612
|
+
// with `retouch` too, for a host of one round of a lazy tree, those are cached too, and each taken
|
|
613
|
+
// from the cache counts as read again (untouchedMemo's `retouch`).
|
|
614
|
+
export function vfsHost(vfs, { root, outside, installs = [], hides, cache = true, touches = () => 0, retouch } = {}) {
|
|
459
615
|
if (sep !== '/') throw new Error('The Vfs host is POSIX-only')
|
|
460
616
|
const disk = outside ?? null
|
|
461
617
|
const installed = new Set(installs)
|
|
@@ -484,26 +640,28 @@ export function vfsHost(vfs, { root, outside, installs = [], hides, cache = true
|
|
|
484
640
|
// `p` from `/`, as spelled: realpath resolves its `..` after the link before it.
|
|
485
641
|
const abs = (p) => (p.startsWith('/') ? p : `/${p}`)
|
|
486
642
|
const normal = (p) => resolve('/', p)
|
|
487
|
-
const memo = () => (
|
|
643
|
+
const memo = () => untouchedMemo(touches, { cache, retouch })
|
|
488
644
|
|
|
489
645
|
// `p` is a real path but for its last name; null when it exists and isn't a link.
|
|
490
646
|
const readlink = (p) => {
|
|
491
647
|
const zone = zoneOf(p)
|
|
492
648
|
if (zone === OUTSIDE) return disk.readlink(p)
|
|
493
|
-
if (zone === NONE) throw new VfsError('ENOENT', p)
|
|
649
|
+
if (zone === NONE) throw stackless(() => new VfsError('ENOENT', p))
|
|
494
650
|
const node = lstatOrNull(vfs, inVfs(p))
|
|
495
|
-
if (node === null) throw new VfsError('ENOENT', p)
|
|
651
|
+
if (node === null) throw stackless(() => new VfsError('ENOENT', p))
|
|
496
652
|
return node.type === 'symlink' ? vfs.readlink(inVfs(p)) : null
|
|
497
653
|
}
|
|
498
654
|
|
|
499
|
-
const realCache =
|
|
655
|
+
const realCache = cache ? new Map() : { get() {}, set() {} }
|
|
500
656
|
const realpath = (p, hops = 0) => {
|
|
501
657
|
if (p === '/') return '/'
|
|
502
658
|
const hit = realCache.get(p)
|
|
503
659
|
if (hit !== undefined) {
|
|
504
|
-
if (hit
|
|
505
|
-
|
|
660
|
+
if (hit.touched) retouch()
|
|
661
|
+
if (hit.error !== undefined) throw hit.error
|
|
662
|
+
return hit.result
|
|
506
663
|
}
|
|
664
|
+
const before = touches()
|
|
507
665
|
let result
|
|
508
666
|
try {
|
|
509
667
|
const parentReal = realpath(dirname(p), hops)
|
|
@@ -512,22 +670,23 @@ export function vfsHost(vfs, { root, outside, installs = [], hides, cache = true
|
|
|
512
670
|
if (name === '.' || name === '..') {
|
|
513
671
|
if (statOf(parentReal)?.isDirectory() !== true) throw new VfsError('ENOTDIR', p)
|
|
514
672
|
result = name === '.' ? parentReal : dirname(parentReal)
|
|
515
|
-
realCache.set(p, result)
|
|
516
|
-
return result
|
|
517
|
-
}
|
|
518
|
-
const candidate = join(parentReal, name)
|
|
519
|
-
const link = readlink(candidate)
|
|
520
|
-
if (link === null) {
|
|
521
|
-
result = candidate
|
|
522
673
|
} else {
|
|
523
|
-
|
|
524
|
-
|
|
674
|
+
const candidate = join(parentReal, name)
|
|
675
|
+
const link = readlink(candidate)
|
|
676
|
+
if (link === null) {
|
|
677
|
+
result = candidate
|
|
678
|
+
} else {
|
|
679
|
+
if (hops >= 40) throw new VfsError('ELOOP', p)
|
|
680
|
+
result = realpath(link.startsWith('/') ? link : `${dirname(candidate)}/${link}`, hops + 1)
|
|
681
|
+
}
|
|
525
682
|
}
|
|
526
683
|
} catch (err) {
|
|
527
|
-
|
|
684
|
+
const touched = touches() !== before
|
|
685
|
+
if (hops === 0 && (err.code === 'ENOENT' || err.code === 'ELOOP') && (!touched || retouch !== undefined)) realCache.set(p, { error: err, touched })
|
|
528
686
|
throw err
|
|
529
687
|
}
|
|
530
|
-
|
|
688
|
+
const touched = touches() !== before
|
|
689
|
+
if (!touched || retouch !== undefined) realCache.set(p, { result, touched })
|
|
531
690
|
return result
|
|
532
691
|
}
|
|
533
692
|
|
|
@@ -538,33 +697,42 @@ export function vfsHost(vfs, { root, outside, installs = [], hides, cache = true
|
|
|
538
697
|
}
|
|
539
698
|
|
|
540
699
|
// Node's findPackageJSON refuses a manifest that is no JSON object.
|
|
541
|
-
const
|
|
700
|
+
const manifestError = memo()
|
|
542
701
|
const checkManifest = (path) => {
|
|
543
|
-
|
|
544
|
-
if (error === undefined) {
|
|
545
|
-
error = null
|
|
702
|
+
const error = manifestError(path, () => {
|
|
546
703
|
try {
|
|
547
|
-
|
|
548
|
-
if (!isPlainObject(data)) error = invalidPackageConfig(path)
|
|
704
|
+
return isPlainObject(JSON.parse(packageJSONText(host.readFile(path)))) ? null : invalidPackageConfig(path)
|
|
549
705
|
} catch (cause) {
|
|
550
|
-
|
|
706
|
+
return invalidPackageConfig(path, cause)
|
|
551
707
|
}
|
|
552
|
-
|
|
553
|
-
}
|
|
708
|
+
})
|
|
554
709
|
if (error !== null) throw error
|
|
555
710
|
}
|
|
556
711
|
|
|
712
|
+
// Of `outside` too, whose realpaths and listings are cached here as well.
|
|
557
713
|
const statCache = memo()
|
|
558
|
-
const statOf = (real) => {
|
|
714
|
+
const statOf = (real) => statCache(real, () => {
|
|
559
715
|
if (zoneOf(real) === OUTSIDE) return disk.stat(real)
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
716
|
+
const node = lstatOrNull(vfs, inVfs(real))
|
|
717
|
+
return node !== null && node.type !== 'symlink' ? statsFor(node) : null
|
|
718
|
+
})
|
|
719
|
+
// The entries of the directory at `real`, `p` as the caller spelled it, sorted by name.
|
|
720
|
+
const listingOf = (real, p) => {
|
|
721
|
+
if (zoneOf(real) === TREE) {
|
|
722
|
+
if (nodeOf(real).type !== 'directory') throw new VfsError('ENOTDIR', p)
|
|
723
|
+
const dir = inVfs(real)
|
|
724
|
+
return vfs.readdir(dir).map((name) => dirent(name, vfs.lstat(join(dir, name)).type)).toSorted(byName)
|
|
725
|
+
}
|
|
726
|
+
// An outside directory shows the tree's installed paths in place of its own, and nothing that
|
|
727
|
+
// is none.
|
|
728
|
+
const out = disk.readdir(real).filter((d) => zoneOf(join(real, d.name)) === OUTSIDE)
|
|
729
|
+
for (const name of installedIn.get(real) ?? []) {
|
|
730
|
+
const node = lstatOrNull(vfs, inVfs(join(real, name)))
|
|
731
|
+
if (node !== null) out.push(dirent(name, node.type))
|
|
565
732
|
}
|
|
566
|
-
return
|
|
733
|
+
return out.toSorted(byName)
|
|
567
734
|
}
|
|
735
|
+
const listings = memo()
|
|
568
736
|
// A path ending in `/` or `/.` names a directory, which path.resolve drops.
|
|
569
737
|
const namesDir = (p) => p.endsWith('/') || p.endsWith('/.')
|
|
570
738
|
const checkDir = (p, real) => {
|
|
@@ -591,19 +759,8 @@ export function vfsHost(vfs, { root, outside, installs = [], hides, cache = true
|
|
|
591
759
|
},
|
|
592
760
|
readdir(p) {
|
|
593
761
|
const real = realpath(abs(p))
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
const dir = inVfs(real)
|
|
597
|
-
return vfs.readdir(dir).map((name) => dirent(name, vfs.lstat(join(dir, name)).type)).toSorted(byName)
|
|
598
|
-
}
|
|
599
|
-
// An outside directory shows the tree's installed paths in place of its own, and nothing that
|
|
600
|
-
// is none.
|
|
601
|
-
const out = disk.readdir(real).filter((d) => zoneOf(join(real, d.name)) === OUTSIDE)
|
|
602
|
-
for (const name of installedIn.get(real) ?? []) {
|
|
603
|
-
const node = lstatOrNull(vfs, inVfs(join(real, name)))
|
|
604
|
-
if (node !== null) out.push(dirent(name, node.type))
|
|
605
|
-
}
|
|
606
|
-
return out.toSorted(byName)
|
|
762
|
+
// A copy, the caller's to sort or add to.
|
|
763
|
+
return [...listings(real, () => listingOf(real, p))]
|
|
607
764
|
},
|
|
608
765
|
readlink(p) {
|
|
609
766
|
if (namesDir(p)) {
|
|
@@ -623,6 +780,9 @@ export function vfsHost(vfs, { root, outside, installs = [], hides, cache = true
|
|
|
623
780
|
checkDir(p, real)
|
|
624
781
|
return real
|
|
625
782
|
},
|
|
783
|
+
// Node's fs.globSync and path.matchesGlob, as the disk host's, over this host (glob.js).
|
|
784
|
+
glob: (pattern, options) => globSync(host, pattern, options),
|
|
785
|
+
matchesGlob,
|
|
626
786
|
// As Node's: the nearest package.json above a file's real path, never out of a node_modules dir;
|
|
627
787
|
// one there that can't be read is refused (packageJSONStat).
|
|
628
788
|
findPackageJSON(p) {
|
|
@@ -641,6 +801,9 @@ export function vfsHost(vfs, { root, outside, installs = [], hides, cache = true
|
|
|
641
801
|
resolve(parentFile, specifier, conditions) {
|
|
642
802
|
return (resolver ?? createNodeResolver(host)).resolve(normal(parentFile), specifier, conditions)
|
|
643
803
|
},
|
|
804
|
+
// What createNodeResolver caches by, as this host does.
|
|
805
|
+
touches,
|
|
806
|
+
retouch,
|
|
644
807
|
}
|
|
645
808
|
const resolver = cache ? createNodeResolver(host) : undefined
|
|
646
809
|
return host
|