@exodus/stasis 1.0.0-beta.0 → 1.0.0-beta.10

package/src/vfs-bundle/entries.js

@@ -0,0 +1,144 @@
+import { join, normalize, relative } from 'node:path'
+import { fileURLToPath } from 'node:url'
+
+import { readJson } from '@exodus/stasis-core/bundle-util'
+import { isAutoExcludedDir, isPlainObject, posixPathEscapes, relativeEscapes } from '@exodus/stasis-core/util'
+import { Vfs } from '@preventive/vfs'
+import { checkVfsOptions, fieldResolverFor } from '../cmd/bundle.js'
+import { foundrySourceDir } from '../loaders/foundry.js'
+import { resolveConditions } from '../resolve-fields.js'
+import { resolveTypescriptFallback, typescriptExportsTarget } from '../resolve-typescript.js'
+import { checkKind, checkVersion, packageManagerOf, vfsHost } from './tree.js'
+
+const JS = /\.(?:[cm]?[jt]s|[jt]sx)$/u
+// Node's own conditions for require() and for import, which the build's are added to; with
+// `mainFields` or `metro`, the field resolver's bundler ones instead (no `node`).
+const NODE_CONDITIONS = [['require', 'node', 'node-addons', 'module-sync'], ['import', 'node', 'node-addons', 'module-sync']]
+const FIELD_CONDITIONS = ['commonjs', 'module'].map((format) => resolveConditions(format))
+
+// The entry points the package.json in `dir` names, as paths from `dir`, resolved as the JS build
+// with the given options resolves: as Node does, `conditions` added, or with `mainFields` or
+// `metro` through the build's field resolver, once per platform of `platforms` under `metro`; what
+// that misses mapped as tsc maps it under `typescript`, as the build's fallback maps it. In
+// that order: its own entry, as the build resolves `./` there (`main`, or the first of the main
+// fields, else index); each subpath `exports` holds (but a pattern), as the package's name resolves
+// for require() and for import, with the conditions the build adds (the RN ones under `metro`) to
+// Node's, or with `mainFields` or `metro` to the field resolver's; and each `bin`. Only JS files in
+// `dir` that are there, named from within it; none without a package.json.
+function packageEntries(host, dir, { conditions = [], mainFields, metro = false, platforms = [], typescript = false } = {}) {
+  const real = host.realpath(dir)
+  const manifest = join(real, 'package.json')
+  const pkg = readJson(manifest, host)
+  if (!isPlainObject(pkg)) return []
+  const found = new Set()
+  const add = (file) => {
+    const rel = file === undefined ? '' : relative(real, file)
+    if (rel !== '' && !relativeEscapes(rel) && JS.test(rel)) found.add(rel)
+  }
+  const passes = metro || mainFields !== undefined
+    ? (metro ? platforms : [null]).map((platform) => fieldResolverFor(platform, { mainFields, metro, conditions, typescript, host }))
+    : [{ extras: conditions, mainFields: ['main'] }]
+  // Node's resolution of `specifier`, else under `typescript` the miss `mapped` as tsc maps it; a
+  // real path either way, or undefined.
+  const viaNode = (specifier, names, extras, mapped) => {
+    const set = new Set([...names, ...extras])
+    try {
+      return host.resolve(manifest, specifier, set)
+    } catch {
+      const hit = typescript ? mapped(set) : null
+      return hit === null ? undefined : host.realpath(hit)
+    }
+  }
+  for (const { extras, mainFields: fields, resolver } of passes) {
+    const entry = fields.map((name) => pkg[name]).find((value) => typeof value === 'string' && value !== '')
+    if (entry !== undefined && posixPathEscapes(entry)) continue
+    if (resolver === undefined) {
+      add(viaNode('./', NODE_CONDITIONS[0], extras, (set) => resolveTypescriptFallback(manifest, './', { conditions: set, host })))
+    } else {
+      // A real path, as Node's resolution gives and the scan takes.
+      const hit = resolver(manifest, './')
+      if (hit?.url !== undefined) add(host.realpath(fileURLToPath(hit.url)))
+    }
+  }
+  const { exports, name } = pkg
+  const keyed = isPlainObject(exports) && Object.keys(exports).some((key) => key.startsWith('.'))
+  const subpaths = keyed ? Object.keys(exports) : exports === undefined || exports === null ? [] : ['.']
+  for (const subpath of typeof name === 'string' ? subpaths : []) {
+    if (!subpath.startsWith('.') || subpath.includes('*') || subpath.endsWith('/')) continue
+    const specifier = subpath === '.' ? name : `${name}${subpath.slice(1)}`
+    for (const { extras, resolver } of passes) {
+      for (const names of resolver === undefined ? NODE_CONDITIONS : FIELD_CONDITIONS) add(viaNode(specifier, names, extras, (set) => typescriptExportsTarget(real, exports, subpath, { conditions: set, host })))
+    }
+  }
+  const bins = typeof pkg.bin === 'string' ? [pkg.bin] : pkg.bin !== null && typeof pkg.bin === 'object' ? Object.values(pkg.bin) : []
+  for (const bin of bins) {
+    if (typeof bin === 'string' && !posixPathEscapes(bin) && host.stat(join(real, bin))?.isFile()) add(host.realpath(join(real, bin)))
+  }
+  return [...found]
+}
+
+// Directories whose .sol files are none of a project's entry points: its tests, scripts and mocks,
+// what it depends on or builds, and what no walk descends into.
+const SKIPPED_DIRS = new Set(['test', 'tests', 'script', 'scripts', 'mock', 'mocks', 'lib', 'node_modules', 'dependencies', 'out', 'cache', 'artifacts', 'build'])
+const isSkippedDir = (name) => SKIPPED_DIRS.has(name.toLowerCase()) || isAutoExcludedDir(name)
+// A .sol file but a test or script by its name.
+const isEntrySol = (entry) => entry.isFile() && entry.name.endsWith('.sol') && !/\.(?:t|s|test|spec)\.sol$/u.test(entry.name)
+
+// The .sol files under the directory `rel` of `real` that are entry points by name, but those in a
+// skipped directory; sorted. Links are not followed.
+function solidityFiles(host, real, rel) {
+  const out = []
+  const walk = (sub) => {
+    for (const entry of host.readdir(join(real, sub))) {
+      const path = sub === '.' ? entry.name : `${sub}/${entry.name}`
+      if (entry.isDirectory()) {
+        if (!isSkippedDir(entry.name)) walk(path)
+      } else if (isEntrySol(entry)) {
+        out.push(path)
+      }
+    }
+  }
+  walk(rel)
+  return out.toSorted()
+}
+
+// The entry points of the Soldeer project in `dir`, by name and layout, as paths from `dir`: its
+// .sol files directly in it, then those under contracts/, then those under its source directory
+// (its foundry.toml's default profile's `src`, else `src`); none of its tests, scripts or mocks,
+// nor any in what it depends on or builds. A link to a directory is none.
+function solidityEntries(host, dir) {
+  const real = host.realpath(dir)
+  const isDirectory = (rel) => !posixPathEscapes(rel) && host.stat(join(real, rel))?.isDirectory() && host.readlink(join(real, rel)) === null
+  const under = (rel) => (isDirectory(rel) ? solidityFiles(host, real, rel) : [])
+  const src = normalize(foundrySourceDir(real, { host })).replace(/\/+$/u, '')
+  const own = host.readdir(real).filter(isEntrySol).map((entry) => entry.name).toSorted()
+  return [...new Set([...own, ...under('contracts'), ...(src === 'contracts' ? [] : under(src))])]
+}
+
+// By the kind of bundle a package manager builds: an entry its options are checked over before
+// there are any, the entries the build takes where none are given, and what having none says.
+export const KINDS = {
+  js: { standIn: 'index.js', entries: packageEntries, none: 'has no package.json naming a JS entry point there' },
+  sol: { standIn: 'index.sol', entries: solidityEntries, none: 'has no .sol entry point directly in it, under contracts/, or under its source directory' },
+}
+
+const EMPTY = vfsHost(new Vfs())
+
+// The build's checks of `options` before anything is fetched, over an empty tree, which never
+// decides them: for the package manager given, else for any; over the entries given, else over one
+// of the kind the default ones are. -> the PACKAGE_MANAGERS entry of the one given
+export function checkAhead(name, packageManager, options) {
+  checkVersion(name, { packageManager, ...options })
+  const pm = packageManager === undefined ? undefined : packageManagerOf(name, packageManager)
+  const refusal = (entries) => {
+    try {
+      checkKind(name, checkVfsOptions(name, { ...options, entries, cwd: '/', host: EMPTY, fetched: false }), packageManager === undefined ? undefined : [packageManager])
+      return null
+    } catch (error) {
+      return error
+    }
+  }
+  const refusals = options.entries === undefined ? (pm === undefined ? Object.values(KINDS) : [KINDS[pm.kind]]).map(({ standIn }) => refusal([standIn])) : [refusal(options.entries)]
+  if (!refusals.includes(null)) throw refusals[0]
+  return pm
+}

package/src/vfs-bundle/github.js

@@ -0,0 +1,202 @@
+import { posix } from 'node:path'
+
+import { isValidRepoField } from '@exodus/stasis-core/bundle'
+import { posixPathEscapes } from '@exodus/stasis-core/util'
+import { decompress } from '@preventive/archive/compression.js'
+import { ArchiveError, unpack } from '@preventive/archive/tar.js'
+import { createClient } from '@preventive/upstream/github.js'
+import { vfsFromEntries } from '@preventive/vfs'
+import { buildVfsBundle } from '../cmd/bundle.js'
+import { KINDS, checkAhead } from './entries.js'
+import { checkCache, checkTarget, detectPackageManager, installedAlone, lockfileOf, lockfilesListed, noLockfile, packageManagerOf, vfsHost } from './tree.js'
+
+// As upstream's tree verification bounds a tarball's unpacked size.
+const MAX_TAR_BYTES = 2 ** 30
+// What a git tree holds (upstream's verification refuses anything else from GitHub).
+const TREE_TYPES = new Set(['file', 'directory', 'symlink'])
+// Lockfile references to a path above the lockfile's directory (conservatively: any importer's):
+// pnpm's and yarn's specs, and package-lock.json's package keys and `resolved` paths, `..` itself
+// (`link:..`, npm's `".."`) or under it.
+const LOCKFILE_ESCAPE = /(?:link:|file:|directory: |["'])\.\.(?![^/"'\s])/u
+const TSCONFIG = /(?:^|\/)[jt]sconfig[^/]*\.json$/u
+
+// Whether `target`, a path relative to the file `from`, resolves outside the tree.
+const resolvesOutside = (from, target) => target.startsWith('/') || posixPathEscapes(posix.join(posix.dirname(from), target))
+
+// The mode a checkout gives an entry under the usual umask 022. Git keeps only a file's executable
+// bit (upstream verifies that much), and GitHub's tarballs are written with git archive's tar.umask
+// 0002, which makes them 664 and 775 besides.
+const checkoutMode = ({ type, mode }) => (type === 'symlink' ? mode : type === 'directory' || mode & 0o100 ? 0o755 : 0o644)
+
+// A GitHub tarball's entries, its one top directory dropped, with the modes a checkout has.
+async function treeEntries(tarball, where) {
+  const entries = []
+  for (const entry of unpack(await decompress(tarball, 'gzip', { limit: MAX_TAR_BYTES }))) {
+    const name = entry.name.slice(entry.name.indexOf('/') + 1)
+    if (!TREE_TYPES.has(entry.type)) throw new Error(`${where}: unexpected ${entry.type} ${JSON.stringify(name)} in the tarball`)
+    if (entry.name.includes('/')) entries.push({ ...entry, name, mode: checkoutMode(entry) })
+  }
+  return entries
+}
+
+// The first symlink pointing outside the tree, which would otherwise resolve within it.
+const escapingLink = (entries) => entries.find((entry) => entry.type === 'symlink' && resolvesOutside(entry.name, entry.linkname))
+
+// Whether a subtree's files refer to a path above it: a lockfile link or a [jt]sconfig path.
+function refersAbove(entries, lockfile) {
+  const text = (entry) => new TextDecoder().decode(entry.data)
+  return entries.some((entry) => entry.type === 'file' && (entry.name === lockfile
+    ? LOCKFILE_ESCAPE.test(text(entry))
+    : TSCONFIG.test(entry.name) && [...text(entry).matchAll(/"(\.\.?\/[^"]*)"/gu)].some(([, path]) => resolvesOutside(entry.name, path))))
+}
+
+// The directories above `directory`, the repo's root (undefined) last.
+function ancestorsOf(directory) {
+  const out = []
+  for (let dir = posix.dirname(directory); dir !== '.'; dir = posix.dirname(dir)) out.push(dir)
+  return [...out, undefined]
+}
+
+// The files of the subtree `tree`, or null where one is a symlink out of it.
+async function treeFiles(client, { github, tree, where }) {
+  let files
+  try {
+    files = await treeEntries(await client.getRepoTreeTarball({ repo: github, tree }), where)
+  } catch (error) {
+    if (error instanceof ArchiveError) return null // e.g. a symlink out of the subtree
+    throw error
+  }
+  return escapingLink(files) ? null : files
+}
+
+// The whole repo's files at `sha`.
+async function repoFiles(client, { github, sha, where }) {
+  const files = await treeEntries(await client.getRepoTarball({ repo: github, sha }), where)
+  const link = escapingLink(files)
+  if (link) throw new Error(`${where}: symlink ${JSON.stringify(link.name)} points outside the repo`)
+  return files
+}
+
+// Checked as the Bundle checks them, before anything is fetched; a tag's name is upstream's to check.
+function checkRepo(name, { github, sha, tag, directory }) {
+  if (github === undefined) throw new Error(`${name}: github is required`)
+  if (tag !== undefined && (typeof tag !== 'string' || tag === '')) throw new TypeError(`${name}: tag must be a non-empty string`)
+  if (sha !== undefined && tag !== undefined) throw new Error(`${name}: sha and tag both name the commit: give one`)
+  for (const [key, value] of Object.entries({ github, commit: sha, directory: directory || undefined })) {
+    if (value !== undefined && !isValidRepoField(key, value)) throw new Error(`${name}: invalid ${key}: ${JSON.stringify(value)}`)
+  }
+}
+
+// listRepoDir's listing of a directory of the repo at `sha`, each listed once.
+function lister(client, { github, sha }) {
+  const listings = new Map()
+  return (directory) => {
+    if (!listings.has(directory)) listings.set(directory, client.listRepoDir({ repo: github, sha, directory }))
+    return listings.get(directory)
+  }
+}
+
+// Of no package manager given, the one whose lockfile alone `directory` (or the repo's root) and
+// the directories above it list, which is then the only one that may install it; refused where
+// none does; undefined where more than one does, or the listings can't tell, for the tree to tell.
+async function listedPackageManager(list, { directory, where }) {
+  let listings
+  try {
+    listings = await Promise.all((directory ? [directory, ...ancestorsOf(directory)] : [undefined]).map(list))
+  } catch {
+    // No plain directory in git (a symlink, or under one); the root's failure is GitHub's.
+    await list(undefined)
+    return undefined
+  }
+  const listed = lockfilesListed(listings)
+  if (listed.length === 0) throw noLockfile(where, posix.resolve('/', directory || '.'))
+  return listed.length === 1 ? listed[0] : undefined
+}
+
+// The files of `directory` alone, when it holds the lockfile, is installed from itself and stands
+// alone; else null.
+async function subtreeEntries(client, list, { github, directory, packageManager, where }) {
+  const names = async (path) => (await list(path)).map((entry) => entry.path)
+  let listing
+  try {
+    listing = await list(directory)
+  } catch {
+    return null // no plain directory in git (a symlink, or under one): the whole repo resolves it
+  }
+  if (!lockfilesListed([listing]).includes(packageManager)) return null
+  const above = ancestorsOf(directory)
+  // Installed from a root above it (a workspace's), it is built from there.
+  if (!(await installedAlone(packageManager, listing.map((entry) => entry.path), () => Promise.all(above.map(names))))) return null
+  // Its tree id, as the listing above it holds it, held to that directory's own.
+  const tree = (await list(above[0])).find((entry) => entry.path === posix.basename(directory) && entry.type === 'tree')?.sha
+  const files = tree === undefined ? null : await treeFiles(client, { github, tree, where })
+  return files === null || refersAbove(files, lockfileOf(packageManager)) ? null : files
+}
+
+// The repo as buildGitHubBundle downloads it, for `name`: at `sha`, else the commit `tag` names (an
+// annotated tag followed to it), else the default branch's head; `directory` alone where it stands
+// alone for the package manager installing it, else the whole repo. That package manager is
+// `packageManager` if given, else the one the listings tell, before the tree is fetched, else the
+// tree; `options` are checked for it once it is known.
+// -> { vfs, host, cwd, subtree, sha, where, packageManager, kind }
+async function repoTree(name, { github, sha, tag, directory, client, packageManager }, options) {
+  checkAhead(name, packageManager, options)
+  checkRepo(name, { github, sha, tag, directory })
+  client ??= createClient({ token: null })
+  sha ??= tag === undefined ? (await client.getRepoHead({ repo: github })).oid : (await client.getRepoTag({ repo: github, tag })).oid
+  const where = `${name}: ${github}@${sha}`
+  const list = lister(client, { github, sha })
+  let found = packageManager
+  if (found === undefined) {
+    found = await listedPackageManager(list, { directory, where })
+    if (found !== undefined) checkAhead(name, found, options)
+  }
+  const subtree = directory && found !== undefined ? await subtreeEntries(client, list, { github, directory, packageManager: found, where }) : null
+  const vfs = vfsFromEntries(subtree ?? await repoFiles(client, { github, sha, where }))
+  const host = vfsHost(vfs)
+  const cwd = posix.resolve('/', subtree ? '.' : directory ?? '.')
+  if (packageManager === undefined) {
+    const detected = detectPackageManager(where, host, cwd, { os: options.os })
+    if (detected !== found) checkAhead(name, detected, options)
+    found = detected
+  }
+  return { vfs, host, cwd, subtree, sha, where, packageManager: found, kind: KINDS[packageManagerOf(name, found).kind] }
+}
+
+// A GitHub repo at a full commit, or the one `tag` names, the default branch's head without either,
+// as buildVfsBundle builds it: `directory` downloaded alone where it stands alone, else the whole
+// repo, and without a `packageManager`, the one whose lockfile installs it, where only one's does
+// (repoTree). Without `entries`, they are the ones suggestedEntries suggests. The bundle's paths are
+// relative to `directory`, or for a JS bundle, to the innermost package directory at or above it
+// holding every file it bundles (buildJsBundle's innermostRoot), which is the project's root where
+// one is outside the others; `repo` names that directory and the commit. Nothing is read from disk:
+// the tree's bytes come from GitHub, or from the cache setCacheDir names, held to the git tree id
+// either way (@preventive/upstream), and are unpacked into a Vfs that buildVfsBundle reads alone.
+// `cache` is buildVfsBundle's, for the npm packages the tree installs: the repo's tree is cached
+// where setCacheDir says, whatever it is.
+export async function buildGitHubBundle({ github, sha, tag, directory, client, packageManager, ...options } = {}) {
+  checkTarget('buildGitHubBundle', options)
+  checkCache('buildGitHubBundle', options.cache)
+  // Its own, for a Soldeer git dependency too.
+  client ??= createClient({ token: null })
+  const tree = await repoTree('buildGitHubBundle', { github, sha, tag, directory, client, packageManager }, options)
+  let { entries } = options
+  if (entries === undefined) {
+    // As suggestedEntries suggests them, from the tree at hand.
+    entries = tree.kind.entries(tree.host, tree.cwd, options)
+    if (entries.length === 0) throw new Error(`${tree.where}: no entries given, and ${directory || 'the repo root'} ${tree.kind.none}`)
+  }
+  const repo = { github, commit: tree.sha }
+  const built = await buildVfsBundle({ ...options, entries, packageManager: tree.packageManager, vfs: tree.vfs, cwd: tree.cwd, innermostRoot: true, repo, client })
+  // The Vfs is the repo's tree, or the subtree at `directory`: a layout known, so its root is `''`.
+  const at = posix.join(tree.subtree ? directory : '.', posix.relative('/', built.root)).replace(/^\.$/u, '')
+  built.bundle.repo = { ...repo, ...(isValidRepoField('directory', at) ? { directory: at } : {}) }
+  return built
+}
+
+// suggestedEntries of a GitHub repo, `{ github, sha, tag, directory, client, packageManager }`, as
+// buildGitHubBundle downloads it.
+export async function suggestedRepoEntries(repo, resolution) {
+  const tree = await repoTree('suggestedEntries', repo, resolution)
+  return tree.kind.entries(tree.host, tree.cwd, resolution)
+}