@exodus/stasis 1.0.0-beta.13 → 1.0.0-beta.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,128 @@
1
+ import { AsyncLocalStorage } from 'node:async_hooks'
2
+ import { posix } from 'node:path'
3
+
4
+ // A lazy tree (@preventive/deptree's `lazy`): laid out from the lockfile alone, each package's
5
+ // directory left empty until it is unpacked. The host reads synchronously, so a build reading it
6
+ // cannot wait for a tarball: it runs in rounds. A round that reads under a package not yet unpacked
7
+ // is void, whatever it built, and those packages are unpacked for the next; the first round that
8
+ // reads none read nothing but what the full tree holds there, and is the build. What a round's scan
9
+ // reads of a file is kept for the rounds after it where reading it read no such package (Scan's
10
+ // `memo`), so each file is scanned once, but those that read what was not unpacked yet.
11
+
12
+ // The tree's Vfs as tree.js's vfsHost reads it (lstat, readlink, readFile and readdir). A read under
13
+ // the directory of a package of `roots` not in `unpacked`, or a listing of that directory itself,
14
+ // adds it to `reads.asked`, and to `reads.read` but while `reads.probing` (a check of what files the
15
+ // package holds, which reads none of its code), counts in `reads.touches`, and reads what the layout
16
+ // holds there. The directory itself, and those above a package's on the way to it, are as the full
17
+ // tree holds them, so reading them asks for nothing.
18
+ export function lazyView(vfs, { roots, unpacked, reads }) {
19
+ const above = new Set()
20
+ for (const root of roots) {
21
+ for (let dir = posix.dirname(root); dir !== '/' && !above.has(dir); dir = posix.dirname(dir)) above.add(dir)
22
+ }
23
+ const ownerOf = (path) => {
24
+ for (let dir = path; dir !== '/'; dir = posix.dirname(dir)) if (roots.has(dir)) return dir
25
+ return null
26
+ }
27
+ const touch = (path, listing) => {
28
+ const owner = ownerOf(path)
29
+ if (owner === null || unpacked.has(owner)) return
30
+ if (!listing && (path === owner || above.has(path))) return
31
+ reads.asked.add(owner)
32
+ if (reads.probing === 0) reads.read.add(owner)
33
+ reads.touches++
34
+ }
35
+ return {
36
+ lstat(path) {
37
+ touch(path, false)
38
+ return vfs.lstat(path)
39
+ },
40
+ readlink(path) {
41
+ touch(path, false)
42
+ return vfs.readlink(path)
43
+ },
44
+ readFile(path) {
45
+ touch(path, false)
46
+ return vfs.readFile(path)
47
+ },
48
+ readdir(path) {
49
+ touch(path, true)
50
+ return vfs.readdir(path)
51
+ },
52
+ }
53
+ }
54
+
55
+ // A get-or-read cache, `(key, read) => read()`'s value, as for a tree that holds still: but kept only
56
+ // where reading it read nothing a lazy tree does not hold yet (counted by `touches`, a lazy tree's
57
+ // host's), which is read again; and kept nowhere without `cache`. With `retouch`, for a cache that
58
+ // lives no longer than one round of a lazy tree, in which the tree holds still, what such a reading
59
+ // gave is kept too, and each time it is taken it touches again (`retouch()`), as reading it again
60
+ // would: what reads it knows it read what the tree does not hold yet, which a round's scan keeps none
61
+ // of (Scan's `memo`), and it is not read again only to fail again. With `across`, a Map kept across
62
+ // a lazy tree's rounds, what was read with none is kept there, for each round after, as a scan
63
+ // keeps a file it read so.
64
+ export function untouchedMemo(touches = () => 0, { cache = true, retouch, across } = {}) {
65
+ const kept = new Map()
66
+ return (key, read) => {
67
+ if (across?.has(key)) return across.get(key)
68
+ const hit = kept.get(key)
69
+ if (hit !== undefined) {
70
+ if (hit.touched) retouch()
71
+ return hit.value
72
+ }
73
+ const before = touches()
74
+ const value = read()
75
+ const touched = touches() !== before
76
+ if (!cache) return value
77
+ if (!touched && across !== undefined) across.set(key, value)
78
+ else if (!touched || retouch !== undefined) kept.set(key, { value, touched })
79
+ return value
80
+ }
81
+ }
82
+
83
+ // What console.warn gives within a round, held for the round: one that is void says nothing.
84
+ const rounds = new AsyncLocalStorage()
85
+ let holders = 0
86
+ let passed
87
+ function warnHeld(...args) {
88
+ const held = rounds.getStore()
89
+ if (held === undefined) passed.apply(console, args)
90
+ else held.push(args)
91
+ }
92
+ async function holding(held, fn) {
93
+ if (holders++ === 0) {
94
+ passed = console.warn
95
+ console.warn = warnHeld
96
+ }
97
+ try {
98
+ return await rounds.run(held, fn)
99
+ } finally {
100
+ if (--holders === 0 && console.warn === warnHeld) console.warn = passed
101
+ }
102
+ }
103
+
104
+ // `attempt(reads)`'s result, of the first round whose reads ask for no package (`reads.asked`), its
105
+ // warnings given then; after each round that does, `unpack(reads.asked, reads.read)`, `reads.read`
106
+ // those of them it read other than by a probe (lazyView). Each round asks for one package at least
107
+ // that no round before did, so there are no more rounds than packages.
108
+ export async function inRounds(attempt, unpack) {
109
+ for (;;) {
110
+ const reads = { asked: new Set(), read: new Set(), probing: 0, touches: 0 }
111
+ const held = []
112
+ let result
113
+ let failure
114
+ try {
115
+ // eslint-disable-next-line no-await-in-loop -- each round reads what the one before unpacked
116
+ result = await holding(held, () => attempt(reads))
117
+ } catch (error) {
118
+ failure = { error }
119
+ }
120
+ if (reads.asked.size === 0) {
121
+ for (const args of held) console.warn(...args)
122
+ if (failure !== undefined) throw failure.error
123
+ return result
124
+ }
125
+ // eslint-disable-next-line no-await-in-loop -- what the next round reads
126
+ await unpack(reads.asked, reads.read)
127
+ }
128
+ }
@@ -2,7 +2,7 @@ import { constants } from 'node:fs'
2
2
  import { basename, dirname, join, posix, relative, resolve, sep } from 'node:path'
3
3
 
4
4
  import { isValidRepoField } from '@exodus/stasis-core/bundle'
5
- import { NO_ENTRY, packageJSONStat, packageJSONText, parseGithubRepository, readJson } from '@exodus/stasis-core/bundle-util'
5
+ import { NO_ENTRY, invalidPackageConfig, lexists, packageJSONStat, packageJSONText, parseGithubRepository, readJson } from '@exodus/stasis-core/bundle-util'
6
6
  import { byName } from '@exodus/stasis-core/host'
7
7
  import { hasNodeModulesSegment, isPlainObject } from '@exodus/stasis-core/util'
8
8
  import { buildNpmTree, findNpmWorkspaces } from '@preventive/deptree/npm.js'
@@ -11,14 +11,22 @@ import { LockfileError, TomlError, buildSoldeerTree } from '@preventive/deptree/
11
11
  import { buildYarn1Tree, findYarn1Workspaces } from '@preventive/deptree/yarn1.js'
12
12
  import { VfsError } from '@preventive/vfs'
13
13
  import { createNodeResolver } from '../resolve-node.js'
14
+ import { globSync, matchesGlob } from './glob.js'
15
+ import { lazyView, untouchedMemo } from './lazy.js'
14
16
  import { settleSoldeer } from './soldeer.js'
17
+ import { YARNCLEAN, yarnclean } from './yarnclean.js'
15
18
  import { isDir, isFile } from '../resolve-typescript.js'
16
19
 
17
20
  // A project's dependencies laid out in memory from its lockfile by @preventive/deptree, as `pnpm
18
21
  // install --frozen-lockfile --ignore-scripts` lays out its node_modules with pnpm 9, 10, 11 or 12,
19
22
  // `yarn install --frozen-lockfile --ignore-scripts` with yarn 1.22, `npm ci --ignore-scripts` with
20
23
  // npm 10.9 or 11, or `soldeer install` its dependencies folder with Soldeer 0.12, and the host that
21
- // reads the project through them. The project is read through a host it is given, and nothing else.
24
+ // reads the project through them; without devDependencies, as `pnpm install --prod`, `yarn install
25
+ // --production` and `npm ci --omit=dev` lay it out, where the project's stasis.config.json says so.
26
+ // node_modules is then cleaned as `yarn autoclean` would clean it by each .yarnclean in the
27
+ // directory installed or above it (yarnclean.js), whatever installs it. The project is read through
28
+ // a host it is given, and nothing else. pnpm's and npm's trees may be laid out lazily, each package
29
+ // unpacked once asked for (lazy.js).
22
30
 
23
31
  // cwd or the nearest of its ancestors that `holds`, or null.
24
32
  export function nearest(cwd, holds) {
@@ -188,7 +196,9 @@ const npmHost = (npm, given) => {
188
196
  // (for the os given, where that changes it); whether a directory holding the lockfile is installed
189
197
  // from by itself, by the names in it and in each directory above it (`above()`, as listings); the
190
198
  // projects it finds in a view of that directory, for the machine given; the directory it installs
191
- // in each, and any it hides, as another package manager's; and the tree.
199
+ // in each, and any it hides, as another package manager's; whether it lays out a tree from the
200
+ // lockfile alone, which may then be lazy; and the tree, without what devDependencies alone reach
201
+ // where `given.production` (Soldeer has no devDependencies, so it leaves out nothing).
192
202
  const PACKAGE_MANAGERS = {
193
203
  pnpm: {
194
204
  kind: 'js',
@@ -200,7 +210,8 @@ const PACKAGE_MANAGERS = {
200
210
  alone: async (names, above) => !(await above()).some((dir) => dir.includes('pnpm-workspace.yaml')),
201
211
  projects: (view, pnpm) => findPnpmProjects({ project: view, host: { pnpm } }),
202
212
  installs: 'node_modules',
203
- build: (view, pnpm, _file, given) => buildPnpmTree({ project: view, host: { pnpm, ...target(given) }, cache: given?.cache }),
213
+ lazy: true,
214
+ build: (view, pnpm, _file, given) => buildPnpmTree({ project: view, host: { pnpm, ...target(given) }, cache: given?.cache, lazy: given?.lazy, production: given?.production }),
204
215
  },
205
216
  yarn1: {
206
217
  kind: 'js',
@@ -214,7 +225,7 @@ const PACKAGE_MANAGERS = {
214
225
  installs: 'node_modules',
215
226
  build: (view, yarn, file, given) => {
216
227
  const { libc: _, ...host } = target({ ...given, libc: 'unknown' }) // yarn 1 matches no libc
217
- return naming(file, buildYarn1Tree({ project: view, host: { yarn, ...host }, cache: given?.cache }))
228
+ return naming(file, buildYarn1Tree({ project: view, host: { yarn, ...host }, cache: given?.cache, production: given?.production }))
218
229
  },
219
230
  },
220
231
  npm: {
@@ -227,7 +238,8 @@ const PACKAGE_MANAGERS = {
227
238
  alone: async (names, above) => !(await above()).some((dir) => dir.includes('package.json')),
228
239
  projects: (view, _npm, given) => findNpmWorkspaces({ project: view, os: target(given).os }),
229
240
  installs: 'node_modules',
230
- build: (view, npm, file, given) => naming(file, buildNpmTree({ project: view, host: npmHost(npm, given), cache: given?.cache })),
241
+ lazy: true,
242
+ build: (view, npm, file, given) => naming(file, buildNpmTree({ project: view, host: npmHost(npm, given), cache: given?.cache, lazy: given?.lazy, production: given?.production })),
231
243
  },
232
244
  soldeer: {
233
245
  kind: 'sol',
@@ -251,21 +263,46 @@ const PACKAGE_MANAGERS = {
251
263
  // Their names.
252
264
  const PACKAGE_MANAGER_NAMES = Object.keys(PACKAGE_MANAGERS)
253
265
 
254
- // layOutTree's tree, with the host reading `project` through it (treeHost).
266
+ // layOutTree's tree, with the host reading `project` through it (treeHost), but a lazy one's, whose
267
+ // hosts are made for each round that reads it.
255
268
  export async function loadTree(options) {
256
269
  const tree = await layOutTree(options)
257
- return { ...tree, host: treeHost(tree, options.project) }
270
+ return tree.lazy === undefined ? { ...tree, host: treeHost(tree, options.project) } : tree
258
271
  }
259
272
 
260
273
  // The host reading `project`, a host of the project's Vfs, through `tree` (layOutTree's): the tree
261
- // serves what the package manager installs, and any file it writes beside it at the root.
262
- export function treeHost(tree, project) {
274
+ // serves what the package manager installs, and any file it writes beside it at the root. A lazy
275
+ // tree is read as lazyView reads it, into the round's `reads`, caching nothing it reads of a package
276
+ // not unpacked yet: each such read reaches it, so that what a file's scan reads tells whether it
277
+ // touched one. Its `probing(fn)` is `fn()`, its reads taken as probes of what files a package holds
278
+ // (lazyView), which read none of its code.
279
+ export function treeHost(tree, project, reads) {
263
280
  const pm = PACKAGE_MANAGERS[tree.packageManager]
264
281
  const files = tree.vfs.readdir('/').filter((name) => tree.vfs.lstat(`/${name}`).type === 'file')
265
282
  const installs = [...[...tree.projects].map((dir) => join(dir, pm.installs)), ...files]
266
- return vfsHost(tree.vfs, { root: tree.root, outside: project, installs, hides: pm.hides })
283
+ if (tree.lazy === undefined) return vfsHost(tree.vfs, { root: tree.root, outside: project, installs, hides: pm.hides })
284
+ const host = vfsHost(lazyView(tree.vfs, { ...tree.lazy, reads }), { root: tree.root, outside: project, installs, hides: pm.hides, touches: () => reads.touches, retouch: () => reads.touches++ })
285
+ host.probing = (fn) => {
286
+ reads.probing++
287
+ try {
288
+ return fn()
289
+ } finally {
290
+ reads.probing--
291
+ }
292
+ }
293
+ return host
294
+ }
295
+
296
+
297
+ // Refused where `packageManager` lays out no tree from the lockfile alone, which `lazy` would have.
298
+ export function checkLazy(name, packageManager) {
299
+ if (!PACKAGE_MANAGERS[packageManager].lazy) throw new Error(`${name}: lazy is taken with pnpm and npm alone, which lay out a tree from the lockfile alone; ${packageManager}'s is not`)
267
300
  }
268
301
 
302
+ // Whether `packageManager`'s tree of the project `project` holds at `cwd` can be lazy, as layOutTree
303
+ // lays it out: pnpm's and npm's, but where a .yarnclean in cwd or above it keeps paths with `!`.
304
+ export const lazyTaken = (project, packageManager, cwd) => PACKAGE_MANAGERS[packageManager].lazy === true && !yarnclean(project, cwd)?.keeps
305
+
269
306
  // The lockfile `packageManager` installs from, e.g. 'pnpm-lock.yaml'.
270
307
  export const lockfileOf = (packageManager) => PACKAGE_MANAGERS[packageManager].lockfile
271
308
 
@@ -289,22 +326,25 @@ export const lockfileRoot = (host, packageManager, cwd, os) => rootOf(host, PACK
289
326
  // The package managers that install node_modules.
290
327
  export const NODE_MODULES_MANAGERS = PACKAGE_MANAGER_NAMES.filter((name) => PACKAGE_MANAGERS[name].installs === 'node_modules')
291
328
 
329
+ // Why the package manager is detected, as a refusal to detect one says: none was given.
330
+ const NOT_GIVEN = 'no packageManager given'
331
+
292
332
  // The one of the package managers `names` that installs `cwd` in the project `host` reads from a
293
333
  // directory holding its lockfile, as lockfileRoot finds it for `os`: refused where none or more than
294
- // one does.
295
- export function detectPackageManager(name, host, cwd, { names = PACKAGE_MANAGER_NAMES, os } = {}) {
334
+ // one does, saying `why` it is detected.
335
+ export function detectPackageManager(name, host, cwd, { names = PACKAGE_MANAGER_NAMES, os, why = NOT_GIVEN } = {}) {
296
336
  const real = host.realpath(cwd)
297
337
  const found = names.flatMap((pm) => {
298
338
  const root = rootOf(host, PACKAGE_MANAGERS[pm], real, os)
299
339
  return root === null ? [] : [[pm, join(root, PACKAGE_MANAGERS[pm].lockfile)]]
300
340
  })
301
341
  if (found.length === 1) return found[0][0]
302
- if (found.length === 0) throw noLockfile(name, real, names)
303
- throw new Error(`${name}: no packageManager given, and more than one lockfile installs ${real}: ${found.map(([pm, file]) => `${file} (${pm})`).join(', ')}`)
342
+ if (found.length === 0) throw noLockfile(name, real, names, why)
343
+ throw new Error(`${name}: ${why}, and more than one lockfile installs ${real}: ${found.map(([pm, file]) => `${file} (${pm})`).join(', ')}`)
304
344
  }
305
345
 
306
- // That none of the lockfiles of `names` installs `place`.
307
- export const noLockfile = (name, place, names = PACKAGE_MANAGER_NAMES) => new Error(`${name}: no packageManager given, and none of ${names.map((pm) => PACKAGE_MANAGERS[pm].lockfile).join(', ')} installs ${place}`)
346
+ // That none of the lockfiles of `names` installs `place`, saying `why` one is looked for.
347
+ export const noLockfile = (name, place, names = PACKAGE_MANAGER_NAMES, why = NOT_GIVEN) => new Error(`${name}: ${why}, and none of ${names.map((pm) => PACKAGE_MANAGERS[pm].lockfile).join(', ')} installs ${place}`)
308
348
 
309
349
  // Whether a directory's listing (listRepoDir's) holds `packageManager`'s lockfile.
310
350
  const listsLockfile = (listing, packageManager) => listing.some((entry) => entry.path === PACKAGE_MANAGERS[packageManager].lockfile && entry.type === 'blob')
@@ -361,8 +401,15 @@ export function checkVfs(name, vfs) {
361
401
  // written through. `client`, a @preventive/upstream/github.js client, is what a Soldeer git
362
402
  // dependency is fetched through (deptree's `github`); without one, a lockfile holding one is refused
363
403
  // before anything is fetched. `cache` (checkCache) is where npm packages' tarballs and version
364
- // documents are kept, which deptree fetches for pnpm, yarn1 and npm.
365
- async function layOutTree({ project, packageManager, cwd, packageManagerVersion, os, cpu, libc, client, cache }) {
404
+ // documents are kept, which deptree fetches for pnpm, yarn1 and npm. Each project's node_modules is
405
+ // cleaned by every .yarnclean in cwd and the directories above it (yarnclean.js). With `lazy`, for
406
+ // pnpm and npm (checkLazy), the tree is laid out with nothing fetched, and comes with `unpack` and
407
+ // `lazy` besides (lazyTree). Without `installDevDependencies`, what devDependencies alone reach is
408
+ // left out, as the package manager's production install leaves it out (deptree's `production`).
409
+ // With `refuseInstalled`, refused before anything is fetched where `project` holds anything at all
410
+ // (a link too) where the tree installs a project's dependencies: its node_modules, or Soldeer's
411
+ // dependencies folder, which the tree would serve in place of what is there.
412
+ async function layOutTree({ project, packageManager, cwd, packageManagerVersion, os, cpu, libc, client, cache, lazy = false, installDevDependencies = true, refuseInstalled = false }) {
366
413
  const pm = PACKAGE_MANAGERS[packageManager]
367
414
  const found = pm.root(project, cwd, os)
368
415
  if (found === null) throw new Error(`no ${pm.lockfile} found in ${cwd} or any parent directory`)
@@ -375,15 +422,109 @@ async function layOutTree({ project, packageManager, cwd, packageManagerVersion,
375
422
  const projects = new Set(pm.projects(view, version, { os, cpu, libc }))
376
423
  const other = pm.kind === 'js' ? outsider(project, found, cwd, projects) : null
377
424
  if (other !== null) throw new Error(`${file} does not install ${other}: it is none of the lockfile's projects`)
378
- const { vfs, stats, installed } = await pm.build(view, version, file, { os, cpu, libc, client, cache })
425
+ if (refuseInstalled) {
426
+ for (const dir of projects) {
427
+ const at = join(found, dir, pm.installs)
428
+ if (lexists(at, project)) throw new Error(`${at} is there already, and the tree is laid out from ${file} alone, over nothing installed: remove it first`)
429
+ }
430
+ }
431
+ // Read before anything is fetched: matching its patterns needs the optional minimatch.
432
+ const clean = pm.kind === 'js' ? yarnclean(project, cwd) : null
433
+ if (lazy && clean?.keeps) throw new Error(`a ${YARNCLEAN} in ${cwd} or above it keeps paths with \`!\`, which lazy does not take: what it keeps of one package turns on what the others hold, which a lazy tree has not fetched`)
434
+ const built = await pm.build(view, version, file, { os, cpu, libc, client, cache, lazy, production: !installDevDependencies })
435
+ // Each project's node_modules, cleaned as `yarn autoclean` would clean it, whatever installs it.
436
+ const cleanAll = () => {
437
+ if (clean !== null) for (const dir of projects) clean(built.vfs, posix.join('/', dir, pm.installs))
438
+ }
439
+ cleanAll()
379
440
  return {
380
441
  root: project.realpath(found),
381
- vfs,
442
+ vfs: built.vfs,
382
443
  projects,
383
- stats,
384
- installed,
444
+ stats: built.stats,
445
+ installed: built.installed,
385
446
  packageManager,
386
447
  packageManagerVersion: version ?? (String(pinned).startsWith(`${pm.pin}@`) ? /^[^@]+@([^+]+)/u.exec(pinned)[1] : undefined),
448
+ ...lazy ? lazyTree(built, cleanAll) : {},
449
+ }
450
+ }
451
+
452
+ // A package name, as a directory of node_modules holds it: one name, or a scope and one.
453
+ const PACKAGE_NAME = /^(?:@[^/@.][^/]*\/)?[^/@.][^/]*$/u
454
+
455
+ // The directories among `roots` of what the package at `dir` depends on, as its package.json names
456
+ // them (dependencies, optionalDependencies and peerDependencies) and Node finds them from `dir` in
457
+ // `vfs`: what the build reads next, where it reads that package, but none it is held to.
458
+ function dependenciesOf(vfs, dir, roots) {
459
+ const manifest = readJson(`${dir}/package.json`, vfs)
460
+ if (!isPlainObject(manifest)) return []
461
+ const names = new Set(['dependencies', 'optionalDependencies', 'peerDependencies'].flatMap((field) => (isPlainObject(manifest[field]) ? Object.keys(manifest[field]) : [])))
462
+ const found = []
463
+ for (const name of names) {
464
+ if (!PACKAGE_NAME.test(name)) continue
465
+ for (let at = dir; ; at = posix.dirname(at)) {
466
+ if (posix.basename(at) !== 'node_modules') {
467
+ let real
468
+ try {
469
+ real = vfs.realpath(posix.join(at, 'node_modules', name))
470
+ } catch {
471
+ real = null
472
+ }
473
+ if (real !== null) {
474
+ if (roots.has(real)) found.push(real)
475
+ break
476
+ }
477
+ }
478
+ if (at === '/') break
479
+ }
480
+ }
481
+ return found
482
+ }
483
+
484
+ // A lazy tree's `unpack(dirs)`, of its packages' directories from `/` of its Vfs, as lazyView asks
485
+ // for them, which deptree unpacks, cleaned as the rest is; and its `lazy`, those directories, as
486
+ // lazyView takes them: `roots`, each the layout has (a .yarnclean may take some away), and
487
+ // `unpacked`, each unpacked so far. Positive patterns alone clean each path by its own, whatever
488
+ // else the tree holds, so cleaning again cleans each package unpacked as the full tree's is.
489
+ // `installed` takes in what each one's version document says, once it is unpacked. What each
490
+ // package `read` (inRounds') depends on is unpacked with it, as the build would ask for it a round
491
+ // later, each round a build; but not what one only probed for its files depends on, nor what those
492
+ // depend on in turn, which most builds never read: a package's dependencies are often its tools. One
493
+ // deptree refuses is left for the build to ask for, which is then refused, so that one it never
494
+ // reads is never refused, as one never fetched is not.
495
+ function lazyTree({ vfs, installed, unpack }, cleanAll) {
496
+ const roots = new Set(installed.map(({ path }) => `/${path}`).filter((dir) => vfs.isDirectory(dir)))
497
+ const unpacked = new Set()
498
+ const refused = new Set()
499
+ const at = new Map(installed.map(({ path }, i) => [path, i]))
500
+ const unpackOne = async (dir) => {
501
+ const [entry] = await unpack([dir.slice(1)])
502
+ installed[at.get(entry.path)] = entry
503
+ unpacked.add(dir)
504
+ }
505
+ return {
506
+ lazy: { roots, unpacked },
507
+ async unpack(asked, read = asked) {
508
+ // Each asked for, and once one read is unpacked, what it depends on, then and there rather
509
+ // than once every one asked for is: so the fetches of both share the time, not one waiting
510
+ // on the last of the other. The first refusal of one asked for is thrown once every one is
511
+ // done, as deptree throws one.
512
+ const seen = new Set(asked)
513
+ const ahead = []
514
+ const done = await Promise.allSettled([...asked].map(async (dir) => {
515
+ await unpackOne(dir)
516
+ if (!read.has(dir)) return
517
+ for (const dependency of dependenciesOf(vfs, dir, roots)) {
518
+ if (seen.has(dependency) || unpacked.has(dependency) || refused.has(dependency)) continue
519
+ seen.add(dependency)
520
+ ahead.push(unpackOne(dependency).catch(() => refused.add(dependency)))
521
+ }
522
+ }))
523
+ await Promise.all(ahead)
524
+ const failed = done.find(({ status }) => status === 'rejected')
525
+ if (failed !== undefined) throw failed.reason
526
+ cleanAll()
527
+ },
387
528
  }
388
529
  }
389
530
 
@@ -429,10 +570,24 @@ const dirent = (name, type) => ({
429
570
  // A view over the Vfs's bytes, never a copy.
430
571
  const asBuffer = (bytes) => Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength)
431
572
 
432
- // What `vfs` holds at `p` itself (the last link not followed), or null.
573
+ // `make()`, an error for a name a lookup does not find, made without a stack: a resolver probes
574
+ // many names for each it finds, and its callers take a miss as none, so the stack is never read,
575
+ // and building it is most of what a miss costs.
576
+ function stackless(make) {
577
+ const limit = Error.stackTraceLimit
578
+ Error.stackTraceLimit = 0
579
+ try {
580
+ return make()
581
+ } finally {
582
+ Error.stackTraceLimit = limit
583
+ }
584
+ }
585
+
586
+ // What `vfs` holds at `p` itself (the last link not followed), or null: the Vfs throws for a name
587
+ // it does not hold, which is made without a stack (stackless).
433
588
  function lstatOrNull(vfs, p) {
434
589
  try {
435
- return vfs.lstat(p)
590
+ return stackless(() => vfs.lstat(p))
436
591
  } catch (err) {
437
592
  if (NO_ENTRY.has(err?.code)) return null
438
593
  throw err
@@ -443,8 +598,6 @@ const TREE = 'tree'
443
598
  const NONE = 'none'
444
599
  const OUTSIDE = 'outside'
445
600
 
446
- const invalidPackageConfig = (path, cause) => Object.assign(new Error(`Invalid package config ${path}.`, { cause }), { code: 'ERR_INVALID_PACKAGE_CONFIG' })
447
-
448
601
  // A `host` (@exodus/stasis-core/host) over `vfs`. Alone, every path is the Vfs's. With `root`, a
449
602
  // real path, and `outside`, a host of the rest, the Vfs's `/` stands for `root`, holding the tree
450
603
  // the package manager lays out there: each of the paths it `installs` (from `root`) is served from
@@ -454,8 +607,11 @@ const invalidPackageConfig = (path, cause) => Object.assign(new Error(`Invalid p
454
607
  // the OS reads one: a `..` after a link leads up from where the link leads, never textually (Node's
455
608
  // own resolution, `findPackageJSON` and `resolve`, normalizes its paths first, as Node does). A
456
609
  // relative path is from `/`. What it reads is cached, as for a tree that holds still, unless `cache`
457
- // is false.
458
- export function vfsHost(vfs, { root, outside, installs = [], hides, cache = true } = {}) {
610
+ // is false; with `touches`, a count of the reads that read what the tree does not hold yet (a lazy
611
+ // tree's, lazy.js), only what was read with none is cached, so that each of those is read again; but
612
+ // with `retouch` too, for a host of one round of a lazy tree, those are cached too, and each taken
613
+ // from the cache counts as read again (untouchedMemo's `retouch`).
614
+ export function vfsHost(vfs, { root, outside, installs = [], hides, cache = true, touches = () => 0, retouch } = {}) {
459
615
  if (sep !== '/') throw new Error('The Vfs host is POSIX-only')
460
616
  const disk = outside ?? null
461
617
  const installed = new Set(installs)
@@ -484,26 +640,28 @@ export function vfsHost(vfs, { root, outside, installs = [], hides, cache = true
484
640
  // `p` from `/`, as spelled: realpath resolves its `..` after the link before it.
485
641
  const abs = (p) => (p.startsWith('/') ? p : `/${p}`)
486
642
  const normal = (p) => resolve('/', p)
487
- const memo = () => (cache ? new Map() : { get() {}, set() {} })
643
+ const memo = () => untouchedMemo(touches, { cache, retouch })
488
644
 
489
645
  // `p` is a real path but for its last name; null when it exists and isn't a link.
490
646
  const readlink = (p) => {
491
647
  const zone = zoneOf(p)
492
648
  if (zone === OUTSIDE) return disk.readlink(p)
493
- if (zone === NONE) throw new VfsError('ENOENT', p)
649
+ if (zone === NONE) throw stackless(() => new VfsError('ENOENT', p))
494
650
  const node = lstatOrNull(vfs, inVfs(p))
495
- if (node === null) throw new VfsError('ENOENT', p)
651
+ if (node === null) throw stackless(() => new VfsError('ENOENT', p))
496
652
  return node.type === 'symlink' ? vfs.readlink(inVfs(p)) : null
497
653
  }
498
654
 
499
- const realCache = memo()
655
+ const realCache = cache ? new Map() : { get() {}, set() {} }
500
656
  const realpath = (p, hops = 0) => {
501
657
  if (p === '/') return '/'
502
658
  const hit = realCache.get(p)
503
659
  if (hit !== undefined) {
504
- if (hit instanceof Error) throw hit
505
- return hit
660
+ if (hit.touched) retouch()
661
+ if (hit.error !== undefined) throw hit.error
662
+ return hit.result
506
663
  }
664
+ const before = touches()
507
665
  let result
508
666
  try {
509
667
  const parentReal = realpath(dirname(p), hops)
@@ -512,22 +670,23 @@ export function vfsHost(vfs, { root, outside, installs = [], hides, cache = true
512
670
  if (name === '.' || name === '..') {
513
671
  if (statOf(parentReal)?.isDirectory() !== true) throw new VfsError('ENOTDIR', p)
514
672
  result = name === '.' ? parentReal : dirname(parentReal)
515
- realCache.set(p, result)
516
- return result
517
- }
518
- const candidate = join(parentReal, name)
519
- const link = readlink(candidate)
520
- if (link === null) {
521
- result = candidate
522
673
  } else {
523
- if (hops >= 40) throw new VfsError('ELOOP', p)
524
- result = realpath(link.startsWith('/') ? link : `${dirname(candidate)}/${link}`, hops + 1)
674
+ const candidate = join(parentReal, name)
675
+ const link = readlink(candidate)
676
+ if (link === null) {
677
+ result = candidate
678
+ } else {
679
+ if (hops >= 40) throw new VfsError('ELOOP', p)
680
+ result = realpath(link.startsWith('/') ? link : `${dirname(candidate)}/${link}`, hops + 1)
681
+ }
525
682
  }
526
683
  } catch (err) {
527
- if (hops === 0 && (err.code === 'ENOENT' || err.code === 'ELOOP')) realCache.set(p, err)
684
+ const touched = touches() !== before
685
+ if (hops === 0 && (err.code === 'ENOENT' || err.code === 'ELOOP') && (!touched || retouch !== undefined)) realCache.set(p, { error: err, touched })
528
686
  throw err
529
687
  }
530
- realCache.set(p, result)
688
+ const touched = touches() !== before
689
+ if (!touched || retouch !== undefined) realCache.set(p, { result, touched })
531
690
  return result
532
691
  }
533
692
 
@@ -538,33 +697,42 @@ export function vfsHost(vfs, { root, outside, installs = [], hides, cache = true
538
697
  }
539
698
 
540
699
  // Node's findPackageJSON refuses a manifest that is no JSON object.
541
- const manifestErrors = memo()
700
+ const manifestError = memo()
542
701
  const checkManifest = (path) => {
543
- let error = manifestErrors.get(path)
544
- if (error === undefined) {
545
- error = null
702
+ const error = manifestError(path, () => {
546
703
  try {
547
- const data = JSON.parse(packageJSONText(host.readFile(path)))
548
- if (!isPlainObject(data)) error = invalidPackageConfig(path)
704
+ return isPlainObject(JSON.parse(packageJSONText(host.readFile(path)))) ? null : invalidPackageConfig(path)
549
705
  } catch (cause) {
550
- error = invalidPackageConfig(path, cause)
706
+ return invalidPackageConfig(path, cause)
551
707
  }
552
- manifestErrors.set(path, error)
553
- }
708
+ })
554
709
  if (error !== null) throw error
555
710
  }
556
711
 
712
+ // Of `outside` too, whose realpaths and listings are cached here as well.
557
713
  const statCache = memo()
558
- const statOf = (real) => {
714
+ const statOf = (real) => statCache(real, () => {
559
715
  if (zoneOf(real) === OUTSIDE) return disk.stat(real)
560
- let stats = statCache.get(real)
561
- if (stats === undefined) {
562
- const node = lstatOrNull(vfs, inVfs(real))
563
- stats = node !== null && node.type !== 'symlink' ? statsFor(node) : null
564
- statCache.set(real, stats)
716
+ const node = lstatOrNull(vfs, inVfs(real))
717
+ return node !== null && node.type !== 'symlink' ? statsFor(node) : null
718
+ })
719
+ // The entries of the directory at `real`, `p` as the caller spelled it, sorted by name.
720
+ const listingOf = (real, p) => {
721
+ if (zoneOf(real) === TREE) {
722
+ if (nodeOf(real).type !== 'directory') throw new VfsError('ENOTDIR', p)
723
+ const dir = inVfs(real)
724
+ return vfs.readdir(dir).map((name) => dirent(name, vfs.lstat(join(dir, name)).type)).toSorted(byName)
725
+ }
726
+ // An outside directory shows the tree's installed paths in place of its own, and nothing that
727
+ // is none.
728
+ const out = disk.readdir(real).filter((d) => zoneOf(join(real, d.name)) === OUTSIDE)
729
+ for (const name of installedIn.get(real) ?? []) {
730
+ const node = lstatOrNull(vfs, inVfs(join(real, name)))
731
+ if (node !== null) out.push(dirent(name, node.type))
565
732
  }
566
- return stats
733
+ return out.toSorted(byName)
567
734
  }
735
+ const listings = memo()
568
736
  // A path ending in `/` or `/.` names a directory, which path.resolve drops.
569
737
  const namesDir = (p) => p.endsWith('/') || p.endsWith('/.')
570
738
  const checkDir = (p, real) => {
@@ -591,19 +759,8 @@ export function vfsHost(vfs, { root, outside, installs = [], hides, cache = true
591
759
  },
592
760
  readdir(p) {
593
761
  const real = realpath(abs(p))
594
- if (zoneOf(real) === TREE) {
595
- if (nodeOf(real).type !== 'directory') throw new VfsError('ENOTDIR', p)
596
- const dir = inVfs(real)
597
- return vfs.readdir(dir).map((name) => dirent(name, vfs.lstat(join(dir, name)).type)).toSorted(byName)
598
- }
599
- // An outside directory shows the tree's installed paths in place of its own, and nothing that
600
- // is none.
601
- const out = disk.readdir(real).filter((d) => zoneOf(join(real, d.name)) === OUTSIDE)
602
- for (const name of installedIn.get(real) ?? []) {
603
- const node = lstatOrNull(vfs, inVfs(join(real, name)))
604
- if (node !== null) out.push(dirent(name, node.type))
605
- }
606
- return out.toSorted(byName)
762
+ // A copy, the caller's to sort or add to.
763
+ return [...listings(real, () => listingOf(real, p))]
607
764
  },
608
765
  readlink(p) {
609
766
  if (namesDir(p)) {
@@ -623,6 +780,9 @@ export function vfsHost(vfs, { root, outside, installs = [], hides, cache = true
623
780
  checkDir(p, real)
624
781
  return real
625
782
  },
783
+ // Node's fs.globSync and path.matchesGlob, as the disk host's, over this host (glob.js).
784
+ glob: (pattern, options) => globSync(host, pattern, options),
785
+ matchesGlob,
626
786
  // As Node's: the nearest package.json above a file's real path, never out of a node_modules dir;
627
787
  // one there that can't be read is refused (packageJSONStat).
628
788
  findPackageJSON(p) {
@@ -641,6 +801,9 @@ export function vfsHost(vfs, { root, outside, installs = [], hides, cache = true
641
801
  resolve(parentFile, specifier, conditions) {
642
802
  return (resolver ?? createNodeResolver(host)).resolve(normal(parentFile), specifier, conditions)
643
803
  },
804
+ // What createNodeResolver caches by, as this host does.
805
+ touches,
806
+ retouch,
644
807
  }
645
808
  const resolver = cache ? createNodeResolver(host) : undefined
646
809
  return host