@exodus/stasis 1.0.0-beta.14 → 1.0.0-beta.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/validate.js CHANGED
@@ -28,7 +28,8 @@
28
28
  // holds, is held to the crate instead: its package checksum, and each file's it lists.
29
29
  import { createHash } from 'node:crypto'
30
30
  import { readFileSync } from 'node:fs'
31
- import { posix } from 'node:path'
31
+ import { readFile, readdir } from 'node:fs/promises'
32
+ import { join, parse as parsePath, posix, relative, sep } from 'node:path'
32
33
 
33
34
  import { toBase32 } from '@exodus/bytes/base32.js'
34
35
  import { EMPTY_MODULE_PATH, dependencyEcosystem, hasNodeModulesSegment, isStatFormat, moduleFileKey } from '@exodus/stasis-core/util'
@@ -46,6 +47,7 @@ import { STORE_HASH, resolutionCommit } from './installed-commits.js'
46
47
  import { findCargoLock } from './loaders/cargo.js'
47
48
  import { parseFileWithKind } from './parse.js'
48
49
  import { DIM, formatTable, painter } from './table.js'
50
+ import { FILE_REASONS, LOCKFILE_ERRORS, OWN_REASONS, noteText } from './validate-notes.js'
49
51
  import { lockfileOf, lockfileRoot, vfsHost } from './vfs-bundle/tree.js'
50
52
 
51
53
  // The files told failing, and the packages told, past which they are counted.
@@ -75,9 +77,10 @@ export class PatchError extends Error {
75
77
  // Why `validate` won't check a file, by its `code`: it is a lockfile (`not-a-bundle`); the bundle's
76
78
  // own code, or its lockfile, has no repository (`no-repository`) or commit (`no-commit`) to be read
77
79
  // of, given or recorded, and neither its dependencies alone (`own: false`, --only-deps) nor another
78
- // lockfile or none (`lockfile`, --lockfile, --no-lockfile) were asked for; or no lockfile it can read
79
- // locks its dependencies (`no-lockfile`, `lockfile-ambiguous`, `lockfile-unsupported`,
80
- // `lockfile-unreadable`).
80
+ // lockfile or none (`lockfile`, --lockfile, --no-lockfile) were asked for; its commit is in none of
81
+ // the repository's own history (`commit-not-in-history`), or that can't be told (`history-unread`,
82
+ // historyOf); or no lockfile it can read locks its dependencies (`no-lockfile`, `lockfile-ambiguous`,
83
+ // `lockfile-unsupported`, `lockfile-unreadable`).
81
84
  export class ValidateError extends Error {
82
85
  constructor(code, message) {
83
86
  super(message)
@@ -514,6 +517,13 @@ async function patchOf(held, pkg, files, locked, verdicts) {
514
517
  }
515
518
  }
516
519
 
520
+ // A path git can have in a tree, which upstream lists: no part empty, `.`, `..` or `.git`, as upstream's
521
+ // isRepoPath (src/args.js) has it, which it doesn't export.
522
+ const inTree = (path) => path.length <= 4096 && path.isWellFormed() && !/\p{Cc}/u.test(path) && path.split('/').every((part) => !['', '.', '..', '.git'].includes(part.toLowerCase()))
523
+
524
+ // `name` in the directory `dir` of the repository, '' its root.
525
+ const pathIn = (dir, name) => (dir ? `${dir}/${name}` : name)
526
+
517
527
  // `items` each run through `run`, `parallel` at a time.
518
528
  async function eachOf(items, parallel, run) {
519
529
  let next = 0
@@ -684,7 +694,8 @@ export async function validateBundle(plan, own, { fetch = fetchPackage, locked =
684
694
  found.set(file.path, kind)
685
695
  }
686
696
  if (entry === undefined || entry.submodule) {
687
- const reason = entry?.submodule || inSubmodule(path) ? 'submodule' : read.cut ? 'tree-cut' : undefined
697
+ // One at a path no tree can have is no file left unlisted.
698
+ const reason = entry?.submodule || inSubmodule(path) ? 'submodule' : read.cut && inTree(path) ? 'tree-cut' : undefined
688
699
  verdict(reason === 'tree-cut' ? 'unverified' : 'unexpected', reason)
689
700
  } else if (entry.symlink !== undefined) verdict('differs', 'symlink')
690
701
  else {
@@ -785,15 +796,21 @@ function readLockfile(name, read) {
785
796
  }
786
797
  const local = (pkg, dir) => locals.set(pkg, new Set([...(locals.get(pkg) ?? []), posix.normalize(dir)]))
787
798
  // Each name a version is installed as in place of its own (an alias, `foo: npm:ms@2.1.3`), as
788
- // `<name it is installed as>\0<its name>@<its version>`.
799
+ // `<name it is installed as>\0<its name>@<its version>`; and each a directory's package is, any version
800
+ // of it, as `<name it is installed as>\0<its name>`.
789
801
  const aliases = new Set()
802
+ const directoryAliases = new Set()
790
803
  if (name === 'pnpm-lock.yaml') {
791
804
  const { lockfile } = parsePnpmLockfile(text)
792
- // pnpm locks an alias's dependency by its name and version (`foo: ms@2.1.3`), a version alone otherwise.
805
+ // pnpm locks an alias's dependency by its name and version (`foo: ms@2.1.3`), a version alone
806
+ // otherwise; one of a directory by its name and the directory (`foo: lib@file:lib`, as a `file:`
807
+ // override installs it), the version the one it has there.
793
808
  for (const node of [...Object.values(lockfile?.importers ?? {}), ...Object.values(lockfile?.packages ?? {})]) {
794
809
  for (const [as, ref] of Object.entries({ ...node.dependencies, ...node.devDependencies, ...node.optionalDependencies })) {
795
810
  const split = typeof ref === 'string' ? ref.indexOf('@', 1) : -1
796
- if (split > 0 && VERSION.test(ref.slice(split + 1).split('(')[0])) aliases.add(`${as}\0${ref.slice(0, split)}@${ref.slice(split + 1).split('(')[0]}`)
811
+ const of = split > 0 ? ref.slice(split + 1) : ''
812
+ if (VERSION.test(of.split('(')[0])) aliases.add(`${as}\0${ref.slice(0, split)}@${of.split('(')[0]}`)
813
+ else if (of.startsWith('file:') && as !== ref.slice(0, split)) directoryAliases.add(`${as}\0${ref.slice(0, split)}`)
797
814
  }
798
815
  }
799
816
  const hashes = new Map()
@@ -811,7 +828,7 @@ function readLockfile(name, read) {
811
828
  for (const [selector, { hash, path }] of Object.entries(lockfile?.patchedDependencies ?? {})) {
812
829
  paths.set(hash, { path: paths.get(hash)?.path ?? path, selectors: [...(paths.get(hash)?.selectors ?? []), selector] })
813
830
  }
814
- return { entries, locals, aliases, patches: { hashes, paths } }
831
+ return { entries, locals, aliases, directoryAliases, patches: { hashes, paths } }
815
832
  }
816
833
  if (name === 'Cargo.lock') {
817
834
  for (const pkg of Object.values(parseCargoLock(text).packages)) add(pkg.name, pkg.version, lockedFrom(pkg.source?.startsWith('git+'), CRATES_IO.has(pkg.source), pkg.checksum))
@@ -924,8 +941,11 @@ export function lockedBy(lockfiles) {
924
941
  const location = npm.root ? `${npm.root}/${dir}` : dir
925
942
  return npm.links.has(location) ? undefined : npm.locations.get(location) ?? null
926
943
  },
927
- // Whether the npm lockfile installs `named` (`<name>@<version>`) under the name `as`; undefined for none read.
928
- aliased: (as, named) => npm?.aliases.has(`${as}\0${named}`),
944
+ // Whether the npm lockfile installs `named` (`<name>@<version>`) under the name `as`: that version,
945
+ // or a directory's package of that name, where it locks no other of that version, so that its
946
+ // files are held to the directory's (locals); undefined for none read.
947
+ aliased: (as, named) => npm && (npm.aliases.has(`${as}\0${named}`)
948
+ || (!npm.entries.has(named) && Boolean(npm.directoryAliases?.has(`${as}\0${named.slice(0, named.lastIndexOf('@'))}`)))),
929
949
  from: npm?.from,
930
950
  tree: npm?.tree,
931
951
  read(path) {
@@ -951,7 +971,6 @@ export async function repositoryLockfiles({ directory = '', ecosystems, list, re
951
971
  if (dir === '') break
952
972
  }
953
973
  const listed = await Promise.all(dirs.map(list))
954
- const at = (dir, name) => (dir ? `${dir}/${name}` : name)
955
974
  const where = directory || 'the repository\'s root'
956
975
  // The manifests a root is found by, read where a lockfile they could pick is listed: package.json
957
976
  // for npm's and yarn's workspaces, Cargo.toml for cargo's.
@@ -963,9 +982,9 @@ export async function repositoryLockfiles({ directory = '', ecosystems, list, re
963
982
  // Each directory there, whatever it holds of those.
964
983
  const files = Object.fromEntries(dirs.filter(Boolean).map((dir) => [dir, { type: 'directory' }]))
965
984
  await Promise.all(dirs.flatMap((dir, index) => listed[index].map(async (name) => {
966
- if (manifests.has(name)) files[at(dir, name)] = await read(at(dir, name))
985
+ if (manifests.has(name)) files[pathIn(dir, name)] = await read(pathIn(dir, name))
967
986
  // npm reads npm-shrinkwrap.json in place of package-lock.json, at the root it finds by either.
968
- else if (LOCKFILES.has(name) || name === 'pnpm-workspace.yaml') files[at(dir, name === 'npm-shrinkwrap.json' ? 'package-lock.json' : name)] ??= ''
987
+ else if (LOCKFILES.has(name) || name === 'pnpm-workspace.yaml') files[pathIn(dir, name === 'npm-shrinkwrap.json' ? 'package-lock.json' : name)] ??= ''
969
988
  })))
970
989
  const host = vfsHost(createVfs(files))
971
990
  const cwd = posix.resolve('/', directory || '.')
@@ -983,7 +1002,7 @@ export async function repositoryLockfiles({ directory = '', ecosystems, list, re
983
1002
  }
984
1003
  if (root === null) continue
985
1004
  const names = listed[dirs.indexOf(fromRoot(root))]
986
- found.push(at(fromRoot(root), pm === 'npm' && names.includes('npm-shrinkwrap.json') ? 'npm-shrinkwrap.json' : lockfileOf(pm)))
1005
+ found.push(pathIn(fromRoot(root), pm === 'npm' && names.includes('npm-shrinkwrap.json') ? 'npm-shrinkwrap.json' : lockfileOf(pm)))
987
1006
  }
988
1007
  if (found.length === 0) {
989
1008
  const bun = ['bun.lock', 'bun.lockb'].find((name) => lists(name))
@@ -1007,6 +1026,42 @@ export async function repositoryLockfiles({ directory = '', ecosystems, list, re
1007
1026
  return Promise.all(chosen.map(async (path) => ({ path, text: await read(path) })))
1008
1027
  }
1009
1028
 
1029
+ // The npm packages the package manager's lockfile installing `dir` (an absolute path) locks of a
1030
+ // registry, each `{ name, version }` once, a version locked from git or a directory left out; and the
1031
+ // lockfile's absolute `path`. Found as repositoryLockfiles finds it, read of the disk at and above `dir`;
1032
+ // null where none is. Refused (ValidateError) where more than one package manager's is, it is bun's, which
1033
+ // isn't read, or it can't be read.
1034
+ export async function installedNpmPackages(dir) {
1035
+ const { root } = parsePath(dir)
1036
+ const at = (path) => join(root, ...path.split('/'))
1037
+ let found
1038
+ try {
1039
+ found = await repositoryLockfiles({
1040
+ directory: relative(root, dir).split(sep).join('/'),
1041
+ ecosystems: new Set(['npm']),
1042
+ list: (path) => readdir(at(path)).catch(() => []),
1043
+ read: (path) => readFile(at(path), 'utf8'),
1044
+ })
1045
+ } catch (error) {
1046
+ if (!(error instanceof ValidateError)) throw error
1047
+ if (error.code === 'no-lockfile') return null
1048
+ // As repositoryLockfiles tells it, but the flags validate takes in its place.
1049
+ throw new ValidateError(error.code, error.message.replace(/[:;] give --lockfile=path.*$/su, '').replace("which validate doesn't read", "which isn't read"))
1050
+ }
1051
+ const [{ path, text }] = found
1052
+ let locks
1053
+ try {
1054
+ locks = readLockfile(posix.basename(path), text)
1055
+ } catch (error) {
1056
+ throw new ValidateError('lockfile-unreadable', `${at(path)}: ${error.message}`)
1057
+ }
1058
+ const packages = [...locks.entries].filter(([, entries]) => entries.some((entry) => entry.from === 'registry')).map(([key]) => {
1059
+ const split = key.lastIndexOf('@')
1060
+ return { name: key.slice(0, split), version: key.slice(split + 1) }
1061
+ })
1062
+ return { path: at(path), packages }
1063
+ }
1064
+
1010
1065
  // GitHub's redirects for a repository renamed or transferred.
1011
1066
  const MOVED = new Set([301, 302, 307, 308])
1012
1067
 
@@ -1036,15 +1091,18 @@ export function followingMoves(client, github) {
1036
1091
  }
1037
1092
 
1038
1093
  // The branches compared at once, where a commit is in neither the default branch's history nor at a
1039
- // tag or a branch's head.
1094
+ // tag or a branch's head; and at most, beside the default branch.
1040
1095
  const BRANCHES_AT_ONCE = 6
1096
+ const MAX_COMPARED = 100
1041
1097
 
1042
1098
  // The ref of `repository` (followingMoves's) that has `commit` in its history, the first of: its
1043
1099
  // default branch, a tag at it, a branch at it, another branch; `{ branch }` or `{ tag }`, null for
1044
1100
  // none. GitHub answers for a commit of any fork of a repository through the repository's own name
1045
1101
  // too, its trees and files read there all the same: that one is the repository's own is that a ref
1046
- // of it holds it (as its compare tells, `ahead` of it or `identical`).
1047
- export async function historyOf(repository, commit) {
1102
+ // of it holds it (as its compare tells, `ahead` of it or `identical`). Other branches are compared
1103
+ // `maxCompared` at most, none past `deadline`: where those left uncompared might hold it, it can't
1104
+ // tell (ValidateError, history-unread).
1105
+ export async function historyOf(repository, commit, { maxCompared = MAX_COMPARED, deadline = Number.POSITIVE_INFINITY, now = Date.now } = {}) {
1048
1106
  const holds = async (ref) => {
1049
1107
  try {
1050
1108
  return ['ahead', 'identical'].includes((await repository.compareCommits({ base: commit, head: ref })).status)
@@ -1062,58 +1120,100 @@ export async function historyOf(repository, commit) {
1062
1120
  const at = branches.find((entry) => entry.oid === commit)
1063
1121
  if (at) return { branch: at.branch }
1064
1122
  let found = null
1065
- await eachOf(branches, BRANCHES_AT_ONCE, async ({ branch }) => {
1066
- if (found === null && (await holds(branch))) found ??= { branch }
1123
+ let compared = 0
1124
+ await eachOf(branches.slice(0, maxCompared), BRANCHES_AT_ONCE, async ({ branch }) => {
1125
+ if (found !== null || now() > deadline) return
1126
+ compared++
1127
+ if (await holds(branch)) found ??= { branch }
1067
1128
  })
1129
+ const left = branches.length - compared
1130
+ if (found === null && left > 0) {
1131
+ throw new ValidateError('history-unread', `${commit} is at no tag or branch head of ${repository.name()} nor in its default branch's history, and ${left} of its ${branches.length} other branches were left uncompared${now() > deadline ? ' past the deadline' : ` (${maxCompared} compared at most)`}: give --sha= a commit of its default branch or at a tag`)
1132
+ }
1068
1133
  return found
1069
1134
  }
1070
1135
 
1071
- // A bundle file validated (validateBundle): its own code against the repository its `repo` names (or
1072
- // `github`), at its commit (or `sha`), under its directory (or `directory`), read with `client`
1073
- // (@preventive/upstream/github.js); its dependencies fetched with upstream, held to the lockfile that
1074
- // installs it there (repositoryLockfiles), or to the lockfiles at the paths `lockfile` gives, or, with
1075
- // `lockfile` false, to their registry alone. Its dependencies alone only where `own` is false. A
1076
- // bundle with no repository or commit to read its own code or its lockfile of is refused
1077
- // (ValidateError), as is one no lockfile it can read locks the dependencies of.
1078
- export async function validate(file, { client, github, sha, directory, own = true, lockfile = true, fetch, deadline } = {}) {
1079
- const { kind, artifact } = parseFileWithKind(file)
1080
- if (kind !== 'bundle') throw new ValidateError('not-a-bundle', `${file} is a lockfile, which holds no file to check; give a bundle`)
1081
- const plan = validationPlan(artifact)
1082
- // The ecosystems of the versions fetched, which a lockfile is to lock.
1083
- const ecosystems = new Set(plan.packages.filter((pkg) => FETCHED.has(pkg.ecosystem) && pkg.name && pkg.version).map((pkg) => pkg.ecosystem))
1136
+ // The ecosystems of the versions fetched, which a lockfile is to lock.
1137
+ const fetchedEcosystems = (plan) => new Set(plan.packages.filter((pkg) => FETCHED.has(pkg.ecosystem) && pkg.name && pkg.version).map((pkg) => pkg.ecosystem))
1138
+
1139
+ // git's entries of a listing (upstream's listRepoDir's or listRepoTree's) as validateBundle takes
1140
+ // them, by their paths under `dir`: a file by its blob's id.
1141
+ function listedEntries(listed, dir = '') {
1142
+ const entries = new Map()
1143
+ for (const entry of listed) {
1144
+ const value = entry.type === 'commit' ? { submodule: true } : entry.mode === '120000' ? { symlink: '' } : entry.type === 'blob' ? { sha: entry.sha } : undefined
1145
+ if (value) entries.set(pathIn(dir, entry.path), value)
1146
+ }
1147
+ return entries
1148
+ }
1149
+
1150
+ // The bundle `plan` (validationPlan) validated (validateBundle) against `repository` (followingMoves's;
1151
+ // null for none, its dependencies alone validated) at `commit`, under `directory`, the one it was built
1152
+ // in, the commit one of the repository's own history (historyOf). Its own code, unless `own` is false,
1153
+ // held to the repository's files there (repositoryFiles); or, given `listTree(paths)`, to git's blob ids
1154
+ // of the commit's tree as upstream's listRepoTree lists it (`{ entries, cut }`), asked for the paths of
1155
+ // the bundle's own files and of `directory`'s files, its lockfile's directories' files named by it too.
1156
+ // Its dependencies held to the lockfile that installs it there (repositoryLockfiles, `lockfile` true),
1157
+ // or to those `lockfile` gives (lockedBy's), or, `lockfile` false, to their registry alone. Refused
1158
+ // (ValidateError): no repository or commit (`no-repository`, `no-commit`) where own code or the
1159
+ // repository's lockfile is to be read, a commit of none of its own history (`commit-not-in-history`,
1160
+ // `history-unread`), and no lockfile to hold the dependencies to. With `strict` false, told instead:
1161
+ // own code unverified for the error's code (`own.reason`), and the dependencies held to their registry
1162
+ // alone, the code told (`lockfileError`) where their lockfile went unread for it.
1163
+ export async function validateRepository(plan, { repository = null, commit, directory = '' }, { own = true, lockfile = true, listTree, strict = true, fetch, deadline } = {}) {
1164
+ const ecosystems = fetchedEcosystems(plan)
1084
1165
  const fromRepository = lockfile === true && ecosystems.size > 0
1085
- const repo = artifact.repo ?? {}
1086
- const where = { github: github ?? repo.github, commit: sha ?? repo.commit, directory: directory ?? repo.directory ?? '' }
1087
- if (own || fromRepository) {
1088
- const or = !own ? 'or --lockfile=path, or --no-lockfile to hold its dependencies to their registry alone'
1089
- : fromRepository ? 'or, to validate its dependencies alone, --only-deps with --lockfile=path or --no-lockfile' : 'or --only-deps to validate its dependencies alone'
1090
- const what = own ? 'its own code' : 'its lockfile'
1091
- if (where.github === undefined) throw new ValidateError('no-repository', `${file} records no repository to read ${what} of: give --github=owner/name and --sha=commit, ${or}`)
1092
- if (where.commit === undefined) throw new ValidateError('no-commit', `${file} records no commit to read ${what} at: give --sha=commit, ${or}`)
1166
+ const github = repository?.name()
1167
+ const told = (error) => {
1168
+ if (strict || !(error instanceof ValidateError)) throw error
1169
+ return error.code
1093
1170
  }
1094
- // The repository read where it has moved too, at a commit of its own history alone: GitHub serves
1095
- // a commit of any fork of it through its name too.
1096
- const repository = where.github === undefined ? null : followingMoves(client, where.github)
1097
- const readsCommit = own || fromRepository || (plan.patches.size > 0 && where.commit !== undefined)
1098
- const ref = repository && readsCommit && where.commit !== undefined ? await historyOf(repository, where.commit) : undefined
1099
- if (ref === null) {
1100
- throw new ValidateError('commit-not-in-history', `${where.commit} is in the history of no branch or tag of ${repository.name()} (a fork's, say, which GitHub serves through its name too): give --sha= a commit of its own`)
1171
+ let ref
1172
+ try {
1173
+ if ((own || fromRepository) && (repository === null || commit === undefined)) {
1174
+ const [code, what] = repository === null ? ['no-repository', 'repository to read it of'] : ['no-commit', 'commit to read it at']
1175
+ throw new ValidateError(code, `no ${what}: ${own ? 'its own code' : 'its lockfile'} is read of the repository`)
1176
+ }
1177
+ // Read at a commit of its own history alone: GitHub serves a commit of any fork of it through its name too.
1178
+ ref = repository && commit !== undefined && (own || fromRepository || plan.patches.size > 0) ? await historyOf(repository, commit, { deadline }) : undefined
1179
+ if (ref === null) {
1180
+ throw new ValidateError('commit-not-in-history', `${commit} is in the history of no branch or tag of ${repository.name()} (a fork's, say, which GitHub serves through its name too): give --sha= a commit of its own`)
1181
+ }
1182
+ } catch (error) {
1183
+ const code = told(error)
1184
+ const report = await validateBundle(plan, { reason: own ? code : 'skipped' }, { fetch, deadline })
1185
+ return fromRepository ? { ...report, lockfileError: code } : report
1101
1186
  }
1102
- const reading = own ? repositoryFiles(repository, where) : null
1103
- // Awaited later, its failure then: none meanwhile.
1187
+ // Own code's files, read whole (the tree's tarball), or by git's blob ids of a listing, with each
1188
+ // directory's files' names; awaited later, its failure then.
1189
+ const reading = own && !listTree ? repositoryFiles(repository, { commit, directory }) : null
1190
+ const listing = own && listTree
1191
+ ? listTree([...new Set([...plan.own.map((file) => pathIn(directory, file.path)), pathIn(directory, 'package.json')])].filter(inTree)).then(({ entries, cut }) => {
1192
+ const listed = listedEntries(entries)
1193
+ // The directories it lists, any entry in each: where it was cut short, those alone.
1194
+ const dirs = new Set(entries.map((entry) => posix.dirname(entry.path)))
1195
+ return { entries: listed, cut, dirs, names: Map.groupBy([...listed.keys()].filter((path) => !listed.get(path).submodule), (path) => posix.dirname(path)) }
1196
+ })
1197
+ : null
1104
1198
  reading?.catch(() => {})
1105
- const list = async (dir) => (await repository.listRepoDir({ sha: where.commit, ...(dir && { directory: dir }) }))
1106
- .filter((entry) => entry.type === 'blob').map((entry) => entry.path)
1199
+ listing?.catch(() => {})
1200
+ const listDir = (dir) => repository.listRepoDir({ sha: commit, ...(dir && { directory: dir }) })
1201
+ // A directory's files' names: of the listing, where it lists the directory (or is whole); else its own.
1202
+ const list = async (dir) => {
1203
+ const listed = await listing
1204
+ if (listed && (!listed.cut || listed.dirs.has(dir || '.'))) return (listed.names.get(dir || '.') ?? []).map((path) => posix.basename(path))
1205
+ return (await listDir(dir)).filter((entry) => entry.type === 'blob').map((entry) => entry.path)
1206
+ }
1107
1207
  // A directory that isn't there, as upstream tells it: one with no files.
1108
1208
  const absent = (error) => /has no directory at/u.test(error?.message ?? '')
1209
+ const under = (path) => reading && (directory === '' || path === directory || path.startsWith(`${directory}/`))
1109
1210
  // A directory's files of the repository by their paths in it: of the tree read for own code, where
1110
1211
  // it is under the bundle's; else its own tree, read once.
1111
1212
  const trees = new Map()
1112
1213
  const tree = (dir) => {
1113
1214
  if (!trees.has(dir)) {
1114
1215
  const prefix = dir ? `${dir}/` : ''
1115
- const under = reading && (where.directory === '' || dir === where.directory || dir.startsWith(`${where.directory}/`))
1116
- trees.set(dir, (under ? reading : repositoryFiles(repository, { ...where, directory: dir }))
1216
+ trees.set(dir, (under(dir) ? reading : repositoryFiles(repository, { commit, directory: dir }))
1117
1217
  .catch((error) => {
1118
1218
  if (absent(error)) return new Map()
1119
1219
  throw error
@@ -1125,28 +1225,62 @@ export async function validate(file, { client, github, sha, directory, own = tru
1125
1225
  // The repository's entries at the patches the bundle holds, by git's blob ids its listings give.
1126
1226
  const patchEntries = async () => {
1127
1227
  const entries = new Map()
1128
- const paths = [...plan.patches.keys()].map((path) => (where.directory ? `${where.directory}/${path}` : path))
1129
- await Promise.all([...new Set(paths.map((path) => posix.dirname(path)))].map(async (dir) => {
1130
- let listed
1228
+ const dirs = new Set([...plan.patches.keys()].map((path) => posix.dirname(pathIn(directory, path))))
1229
+ await Promise.all([...dirs].map(async (dir) => {
1230
+ const of = dir === '.' ? '' : dir
1131
1231
  try {
1132
- listed = await repository.listRepoDir({ sha: where.commit, ...(dir !== '.' && { directory: dir }) })
1232
+ for (const [path, entry] of listedEntries(await listDir(of), of)) entries.set(path, entry)
1133
1233
  } catch (error) {
1134
- if (absent(error)) return
1135
- throw error
1136
- }
1137
- for (const entry of listed) {
1138
- const value = entry.type === 'commit' ? { submodule: true } : entry.mode === '120000' ? { symlink: '' } : entry.type === 'blob' ? { sha: entry.sha } : undefined
1139
- if (value) entries.set(dir === '.' ? entry.path : `${dir}/${entry.path}`, value)
1234
+ if (!absent(error)) throw error
1140
1235
  }
1141
1236
  }))
1142
1237
  return entries
1143
1238
  }
1144
1239
  // A file under the directory read of its tree, where that is being fetched whole; another alone.
1145
1240
  const readRepository = async (path) => {
1146
- const under = where.directory === '' || path.startsWith(`${where.directory}/`)
1147
- const held = reading && under ? (await reading).get(path)?.data : undefined
1148
- return held ? asText.decode(held) : repository.getRepoFile({ path, ref: where.commit })
1241
+ const held = under(path) ? (await reading).get(path)?.data : undefined
1242
+ return held ? asText.decode(held) : repository.getRepoFile({ path, ref: commit })
1243
+ }
1244
+ let locked = null
1245
+ let lockfileError
1246
+ try {
1247
+ const files = lockfile === false ? null : ecosystems.size === 0 ? [] : lockfile !== true ? lockfile
1248
+ : (await repositoryLockfiles({ directory, ecosystems, list, read: readRepository })).map((one) => ({
1249
+ ...one, from: 'repository', read: readRepository, tree, root: posix.relative(posix.dirname(one.path), directory),
1250
+ // A patch the lockfile names by its hash alone, looked for in the project's patches directory, then the root's.
1251
+ patches: [...new Set([pathIn(directory, 'patches'), 'patches'])],
1252
+ }))
1253
+ const held = files && lockedBy(files)
1254
+ for (const ecosystem of held ? ecosystems : []) {
1255
+ if (!held.locks(ecosystem)) throw new ValidateError('no-lockfile', `no lockfile given locks its ${ecosystem} packages: give it with --lockfile=path, or --no-lockfile to hold them to their registry alone`)
1256
+ }
1257
+ locked = held
1258
+ } catch (error) {
1259
+ lockfileError = told(error)
1149
1260
  }
1261
+ // The packages fetched as the repository's tree is read, where it is; own code skipped, the patches
1262
+ // the bundle holds checked all the same, where a repository and commit are given or recorded.
1263
+ const read = reading?.then((entries) => ({ entries })) ?? listing?.then(({ entries, cut }) => ({ entries, cut }))
1264
+ ?? (repository && commit !== undefined && plan.patches.size > 0 ? patchEntries().then((entries) => ({ reason: 'skipped', entries })) : null)
1265
+ // Where it was read, the name it has now where it has moved.
1266
+ const checked = read ? read.then((found) => ({ ...found, repository: repository.name(), ...(repository.name() !== github && { movedFrom: github }), commit, ref, directory })) : { reason: 'skipped' }
1267
+ const report = await validateBundle(plan, checked, { fetch, locked, deadline })
1268
+ return lockfileError ? { ...report, lockfileError } : report
1269
+ }
1270
+
1271
+ // A bundle file validated (validateRepository): its own code against the repository its `repo` names
1272
+ // (or `github`), at its commit (or `sha`), under its directory (or `directory`), read with `client`
1273
+ // (@preventive/upstream/github.js) where it has moved too (followingMoves); its dependencies fetched
1274
+ // with upstream, held to the lockfile that installs it there, or to the lockfiles at the paths
1275
+ // `lockfile` gives, or, with `lockfile` false, to their registry alone. Its dependencies alone only
1276
+ // where `own` is false. A bundle with no repository or commit to read its own code or its lockfile of
1277
+ // is refused (ValidateError), as is one no lockfile it can read locks the dependencies of.
1278
+ export async function validate(file, { client, github, sha, directory, own = true, lockfile = true, fetch, deadline } = {}) {
1279
+ const { kind, artifact } = parseFileWithKind(file)
1280
+ if (kind !== 'bundle') throw new ValidateError('not-a-bundle', `${file} is a lockfile, which holds no file to check; give a bundle`)
1281
+ const plan = validationPlan(artifact)
1282
+ const repo = artifact.repo ?? {}
1283
+ const where = { github: github ?? repo.github, commit: sha ?? repo.commit, directory: directory ?? repo.directory ?? '' }
1150
1284
  const readLocal = (path) => {
1151
1285
  try {
1152
1286
  return readFileSync(path, 'utf8')
@@ -1154,57 +1288,30 @@ export async function validate(file, { client, github, sha, directory, own = tru
1154
1288
  throw new ValidateError('lockfile-unreadable', `${path}: ${error.message}`)
1155
1289
  }
1156
1290
  }
1157
- const lockfiles = lockfile === false ? null : ecosystems.size === 0 ? []
1158
- : Array.isArray(lockfile) ? lockfile.map((path) => ({ path, text: readLocal(path), from: 'local', read: readLocal, root: where.directory }))
1159
- : repositoryLockfiles({ directory: where.directory, ecosystems, list, read: readRepository }).then((files) => files.map((one) => ({
1160
- ...one, from: 'repository', read: readRepository, tree, root: posix.relative(posix.dirname(one.path), where.directory),
1161
- // A patch the lockfile names by its hash alone, looked for in the project's patches directory, then the root's.
1162
- patches: [...new Set([where.directory ? `${where.directory}/patches` : 'patches', 'patches'])],
1163
- })))
1164
- const files = await lockfiles
1165
- const locked = files && lockedBy(files)
1166
- for (const ecosystem of locked ? ecosystems : []) {
1167
- if (!locked.locks(ecosystem)) throw new ValidateError('no-lockfile', `no lockfile given locks its ${ecosystem} packages: give it with --lockfile=path, or --no-lockfile to hold them to their registry alone`)
1291
+ // Each read as it is held to, once the commit is: none where no package needs one.
1292
+ const lockfiles = Array.isArray(lockfile) ? lockfile.map((path) => ({ path, get text() { return readLocal(path) }, from: 'local', read: readLocal, root: where.directory })) : lockfile
1293
+ try {
1294
+ return await validateRepository(plan, { repository: where.github === undefined ? null : followingMoves(client, where.github), commit: where.commit, directory: where.directory },
1295
+ { own, lockfile: lockfiles, fetch, deadline })
1296
+ } catch (error) {
1297
+ if (!(error instanceof ValidateError) || !['no-repository', 'no-commit'].includes(error.code)) throw error
1298
+ // What the file records none of, and what to give in its place.
1299
+ const fromRepository = lockfile === true && fetchedEcosystems(plan).size > 0
1300
+ const or = !own ? 'or --lockfile=path, or --no-lockfile to hold its dependencies to their registry alone'
1301
+ : fromRepository ? 'or, to validate its dependencies alone, --only-deps with --lockfile=path or --no-lockfile' : 'or --only-deps to validate its dependencies alone'
1302
+ const what = own ? 'its own code' : 'its lockfile'
1303
+ throw error.code === 'no-repository'
1304
+ ? new ValidateError('no-repository', `${file} records no repository to read ${what} of: give --github=owner/name and --sha=commit, ${or}`)
1305
+ : new ValidateError('no-commit', `${file} records no commit to read ${what} at: give --sha=commit, ${or}`)
1168
1306
  }
1169
- // The packages fetched as the repository's tree is read, where it is; own code skipped, the patches
1170
- // the bundle holds checked all the same, where a repository and commit are given or recorded.
1171
- // Where it was read, the name it has now where it has moved.
1172
- const at = () => ({ repository: repository.name(), ...(repository.name() !== where.github && { movedFrom: where.github }), commit: where.commit, ...(ref && { ref }), directory: where.directory })
1173
- const checked = own ? reading.then((entries) => ({ entries, ...at() }))
1174
- : where.github !== undefined && where.commit !== undefined && plan.patches.size > 0 ? patchEntries().then((entries) => ({ reason: 'skipped', entries, ...at() }))
1175
- : { reason: 'skipped' }
1176
- return validateBundle(plan, checked, { fetch, locked, deadline })
1177
1307
  }
1178
1308
 
1179
- // Why own code or a package wasn't checked, or a file is as it is, as a report tells it.
1180
- const OWN_UNCHECKED = { skipped: 'skipped (--only-deps): its dependencies alone validated' }
1181
- // A package's reason, unverified (the first) or failing its lockfile (the last three).
1182
- const PACKAGE_REASONS = { local: 'installed from a directory of the repository, which isn\'t read', 'patch-unverified': 'as patched by the bundle\'s patch, which isn\'t verified against the repository', git: 'locked from git, not a registry', 'not-registry': 'locked from elsewhere than its registry', 'no-integrity': 'locked with no sha512 to hold its tarball to', unsupported: 'no upstream package for its ecosystem', unnamed: 'no name or version recorded', 'not-published': 'not published', unreadable: 'its registry refused it', timeout: 'not reached in time', failed: 'could not be fetched',
1183
- 'alias-unverified': 'installed under another name, an alias no lockfile read tells of',
1184
- misplaced: 'installed where its path or its lockfile names another package or version',
1185
- 'not-locked': 'not locked by the lockfile', 'integrity-mismatch': 'its registry\'s tarball is not the one the lockfile locks', 'integrity-conflict': 'locked with two integrities' }
1186
- // Why a patch wasn't applied, told after `its patch` and its path.
1187
- const PATCH_ERRORS = { 'patch-not-in-repository': 'is the bundle\'s, not the repository\'s', 'patch-not-found': 'is in no file of its hash (the bundle\'s, or the patches directory\'s)', 'patch-mismatch': 'is not of the hash its lockfile locks' }
1188
- const FILE_REASONS = { symlink: 'a symlink in the repository', submodule: 'in a submodule', 'tree-cut': 'the repository\'s tree was too large to list whole' }
1189
-
1190
1309
  // Each status's colour: failing red, unverified yellow, valid green.
1191
1310
  const TONES = { differs: 'red', unexpected: 'red', invalid: 'red', unverified: 'yellow', incomplete: 'yellow', valid: 'green' }
1192
1311
 
1193
1312
  // What a package told is of, as a table names it: an npm one by its name and version alone.
1194
1313
  const packageName = (of) => of.replace(/^npm:/u, '')
1195
1314
 
1196
- // What a report tells of a package beside its status: why it is unverified or fails its lockfile,
1197
- // its patch, or the directory it is held to; '' where nothing more.
1198
- function noteOf(pkg) {
1199
- const { patch } = pkg
1200
- if (pkg.reason) return `${PACKAGE_REASONS[pkg.reason] ?? pkg.reason}${pkg.at ? `: ${pkg.at}` : ''}`
1201
- if (patch?.error) return `its patch${patch.path ? ` ${patch.path}` : ''} ${PATCH_ERRORS[patch.error] ?? `could not be applied (${patch.error})`}`
1202
- // A patch the bundle holds, as checked against the repository.
1203
- const whose = patch?.from !== 'bundle' ? '' : patch.verified ? ' (the bundle\'s, as the repository has it)' : ' (the bundle\'s, not verified against the repository)'
1204
- if (patch) return `${patch.applied === false ? 'locked patched, bundled as published' : 'patched'}${patch.path ? ` by ${patch.path}${whose}` : ''}`
1205
- return pkg.directory ? `held to ${pkg.directory} of the repository` : ''
1206
- }
1207
-
1208
1315
  // The report as tables on `out`: the files failing, and own ones unverified, one row for each kind of
1209
1316
  // what they are of (own code, or a package), the worst first; then the packages with something to
1210
1317
  // tell (why they are unverified or fail, a patch, a directory), those valid only with `showValid`.
@@ -1222,10 +1329,10 @@ export function printValidationReport(report, { out = process.stdout, err = proc
1222
1329
  files: group.map((problem) => `${problem.path}${problem.reason ? ` (${FILE_REASONS[problem.reason] ?? problem.reason})` : ''}`).join('\n'),
1223
1330
  }))
1224
1331
  if (files.length > 0) tables.push(formatTable(files, ['status', 'of', 'files'], { multiline: ['files'], paint: status(paintOut) }) + (report.moreProblems > 0 ? `\n... and ${report.moreProblems} more files` : ''))
1225
- const told = report.packages.filter((pkg) => (showValid || pkg.status !== 'valid') && noteOf(pkg) !== '')
1332
+ const told = report.packages.filter((pkg) => (showValid || pkg.status !== 'valid') && noteText(pkg) !== '')
1226
1333
  // The ecosystem a column of its own only where one is not npm's.
1227
1334
  const ecosystems = told.some((pkg) => pkg.ecosystem !== 'npm')
1228
- const packages = told.map((pkg) => ({ status: pkg.status, ...(ecosystems && { ecosystem: pkg.ecosystem }), package: `${pkg.name}@${pkg.version}`, note: noteOf(pkg) }))
1335
+ const packages = told.map((pkg) => ({ status: pkg.status, ...(ecosystems && { ecosystem: pkg.ecosystem }), package: `${pkg.name}@${pkg.version}`, note: noteText(pkg) }))
1229
1336
  if (packages.length > 0) tables.push(formatTable(packages, ['status', ...(ecosystems ? ['ecosystem'] : []), 'package', 'note'], { paint: status(paintOut) }))
1230
1337
  if (tables.length > 0) out.write(`${tables.join('\n\n')}\n`)
1231
1338
 
@@ -1233,9 +1340,10 @@ export function printValidationReport(report, { out = process.stdout, err = proc
1233
1340
  const lines = [
1234
1341
  `${paintErr(['bold', TONES[report.status]], report.status)}: ${counts.matched} of ${counts.files} files as they should be, ${counts.differs} differ, ${counts.unexpected} unexpected, ${counts.unverified} unverified${counts.skipped ? `, ${counts.skipped} skipped` : ''}; ${counts.validPackages} of ${counts.packages} packages as published${counts.lockFailures ? `, ${counts.lockFailures} failing their lockfile` : ''}${counts.misplaced ? `, ${counts.misplaced} installed where another is` : ''}${counts.eol ? `; ${counts.eol} with a checkout's CRLF line endings` : ''}`,
1235
1342
  own.checked ? `Own code held to ${own.repository}${own.movedFrom ? ` (moved from ${own.movedFrom})` : ''} at ${own.commit}${own.ref ? (own.ref.tag ? `, tag ${own.ref.tag}` : ` of ${own.ref.branch}`) : ''}${own.directory ? ` under ${own.directory}` : ''}`
1236
- : paintErr('yellow', `Own code not verified: ${OWN_UNCHECKED[own.reason] ?? own.reason}${own.patches ? `; the ${own.patches === 1 ? 'patch' : `${own.patches} patches`} the bundle holds checked all the same` : ''}`),
1343
+ : paintErr('yellow', `Own code not verified: ${OWN_REASONS[own.reason] ?? own.reason}${own.patches ? `; the ${own.patches === 1 ? 'patch' : `${own.patches} patches`} the bundle holds checked all the same` : ''}`),
1237
1344
  ...(report.lockfiles?.length > 0 ? [`Dependencies held to ${report.lockfiles.join(', ')}`]
1238
- : report.lockfiles === null && counts.packages > 0 ? [paintErr('yellow', 'Dependencies held to no lockfile (--no-lockfile): each to its registry\'s integrity alone')] : []),
1345
+ : report.lockfiles === null && counts.packages > 0
1346
+ ? [paintErr('yellow', `Dependencies held to no lockfile (${report.lockfileError ? LOCKFILE_ERRORS[report.lockfileError] ?? report.lockfileError : '--no-lockfile'}): each to its registry's integrity alone`)] : []),
1239
1347
  ]
1240
1348
  err.write(`${lines.join('\n')}\n`)
1241
1349
  }
@@ -242,7 +242,9 @@ export async function buildGitHubBundle({ github, sha, tag, directory, path, cli
242
242
  // The Vfs is the repo's tree, or the subtree at `directory`: a layout known, so its root is `''`.
243
243
  const at = posix.join(tree.subtree ? tree.directory : '.', posix.relative('/', built.root)).replace(/^\.$/u, '')
244
244
  built.bundle.repo = { ...repo, ...(isValidRepoField('directory', at) ? { directory: at } : {}) }
245
- return { ...built, directory: tree.directory }
245
+ // Onto buildVfsBundle's own, not a copy, which would read its lockfile, built only when read.
246
+ built.directory = tree.directory
247
+ return built
246
248
  }
247
249
 
248
250
  // suggestedEntries of a GitHub repo, `{ github, sha, tag, directory | path, client, packageManager }`,