@preventive/triage 1.0.0-alpha.5 → 1.0.0-alpha.50

package/risk/npm/assess.js

@@ -0,0 +1,285 @@
+// A package version's risk, as the managed server asks it: what calls for a
+// look at it, against the version before it where one is given or risk picks
+// one (old-version.js), each file's stamps, and its dependencies', from npm's
+// registry, kept in the caller's cache (registry.js) as it goes.
+import { npmBaseFilesTell, npmCompareFindings } from './compare.js'
+import { npmCombinedFindings, npmHeldDependencyFindings } from './combined.js'
+import { npmHeldFindings, npmVersionFindings } from './held.js'
+import { npmVersionFileRisk } from './files.js'
+import { npmDependencyChanges, npmFlagDependencyChanges, npmUnusedDependencyCounted } from './dependencies.js'
+import { npmDownloadMonths, npmDownloadTier, npmSettledDownloads, npmTierLevel } from './downloads.js'
+import { NpmRiskError, npmDownloads, npmPackageFacts, npmPackageReleases, npmPublicFacts, npmVersionAdvisories, npmVersionFiles } from './registry.js'
+import { npmMalwareFinding } from './malware.js'
+import { npmKnownScope } from './manifest.js'
+import { DEFAULT_MINIMUM_RELEASE_AGE, npmAccountStanding } from './release.js'
+import { npmAddedDependency, npmUnusedDependencySeverity } from './new-dependency.js'
+import { SEVERITIES } from '../files/stamps.js'
+import { npmRiskScore } from './score.js'
+
+// Which reading of a version an assessment kept (`risk/npm/assessments`) is:
+// one more whenever a heuristic changes what it finds, so one kept by an
+// older reading is read anew.
+const RISK_VERSION = 18
+
+// How long an assessment kept may be as JSON, and what risk reads of either
+// version's files: a package past it, whose files name thousands of hosts or
+// do what they run on thousands of lines each, is refused as too large as its
+// files are read, before more of them is held (filesRisk), and the assessment
+// once told, rather than kept and answered. A package's files tell a few
+// hundred characters each, a few MiB for the most files npm's viewer reads.
+const MAX_ASSESSMENT_LENGTH = 16 * 1024 * 1024
+const tooLarge = what => new NpmRiskError('package-too-large', `what risk tells of ${what} is past ${MAX_ASSESSMENT_LENGTH} characters`)
+
+// A value as JSON holds it: a Map as an object, its keys as strings.
+const plain = value => value instanceof Map ? Object.fromEntries([...value].map(([key, item]) => [key, plain(item)]))
+  : Array.isArray(value) ? value.map(plain)
+  : value && typeof value === 'object' ? Object.fromEntries(Object.entries(value).map(([key, item]) => [key, plain(item)])) : value
+
+// A tier among npm's packages as an assessment says it: `{ tier, level }`, the
+// rank a package is among (null past the last, unpopular) and its tint
+// (downloads.js npmTierLevel).
+const tierOf = tier => ({ tier: Number.isFinite(tier) ? tier : null, level: npmTierLevel(tier) })
+
+// Each of a version's files' stamps and what risk read of it (stamps.js
+// npmFileRisk), by path, as JSON holds them: read a file at a time and given
+// up on, as too large, once they are past MAX_ASSESSMENT_LENGTH as JSON
+// altogether, so that no more of them is held than that.
+function filesRisk(version, what) {
+  let length = 0
+  return Object.fromEntries(version.json.sources.map(path => {
+    const { readability, runsCode, exfiltration, hosts, stamps } = npmVersionFileRisk(version, path)
+    const risk = plain({ stamps, readability, runsCode, exfiltration, hosts })
+    length += path.length + JSON.stringify(risk).length
+    if (length > MAX_ASSESSMENT_LENGTH) throw tooLarge(what)
+    return [path, risk]
+  }))
+}
+
+// What of an assessment holds whatever the time, of the open version
+// (`other`) against the one compared with (`base`), or alone without one,
+// each as compare.js takes a version, as JSON holds it: what it holds,
+// always (held.js npmHeldFindings, and its dependencies against none,
+// combined.js npmHeldDependencyFindings), and against `base` what its files,
+// dependencies, install scripts, publisher and provenance call for anew
+// (compare.js npmCompareFindings), folded into what it holds, never lowering
+// it (combined.js npmCombinedFindings); each file's stamps and what risk read
+// of it (filesRisk); and its dependencies added and changed, each with its
+// flags (dependencies.js), against `base` each it holds too (`held`). Its own findings are told fresh
+// (npmAssessmentOf). The managed server's assessments keep it (`lasting`),
+// and a test can tell what one says of versions it holds. Each publisher's
+// standing on the package is told where `standing` is given (release.js
+// npmAccountStanding, of the open version).
+export const npmLastingRisk = (other, base = null, standing = null) => lastingOf(other, base, filesRisk(other, `${other.npm.name}@${other.npm.version}`), standing)
+
+// npmLastingRisk's, the open version's files read within bounds (`files`,
+// filesRisk's), as the one compared with's are where `base` has them, before
+// the findings read them all.
+function lastingOf(other, base, files, standing) {
+  if (base?.json) filesRisk(base, `${base.npm.name}@${base.npm.version}`)
+  const all = npmFlagDependencyChanges(npmDependencyChanges(undefined, other.npm.manifest), null, other)
+  const held = [...npmHeldFindings(other), ...npmHeldDependencyFindings(all)]
+  const { added, changed } = base ? npmFlagDependencyChanges(npmDependencyChanges(base.npm.manifest, other.npm.manifest), base, other) : all
+  const findings = base ? npmCombinedFindings(held, npmCompareFindings(base, other, { added, changed }, standing)) : held
+  const kept = { findings: plain(findings), files, dependencies: plain({ added, changed, ...base ? { held: all.added } : {} }) }
+  if (JSON.stringify(kept).length > MAX_ASSESSMENT_LENGTH) throw tooLarge(`${other.npm.name}@${other.npm.version}`)
+  return kept
+}
+
+// What of an assessment holds whatever the time (npmLastingRisk), kept once a
+// reading and pair, its versions' files read from npm only where it isn't:
+// the open version's first, then the one compared with's only where they
+// could change what is found (compare.js npmBaseFilesTell), that one else
+// told by what npm's document of it says alone; and where another account
+// published it, each one's standing on the package (release.js
+// npmAccountStanding), by its versions as `facts` read them, or as its
+// document is read again where they were kept. A standing changes as npm
+// removes a version, its publisher then unknown, and never else, so such a
+// pair is kept by how many npm has removed (`~<removals>`), read anew as it
+// removes another.
+async function lasting(name, version, oldVersion, facts, context, fromEmpty) {
+  const old = oldVersion === null ? null : facts.versions.get(oldVersion)
+  const publishers = [old, facts.versions.get(version)].map(fact => fact?.manifest.publisher)
+  const changed = publishers.every(Boolean) && publishers[0] !== publishers[1]
+  const key = `${RISK_VERSION}/${name}@${version}${fromEmpty ? '..(empty)' : oldVersion === null ? '' : `..${oldVersion}`}${changed ? `~${facts.removals}` : ''}`
+  const hit = context.cache ? await context.cache.read('risk/npm/assessments', key) : null
+  if (hit) return hit
+  const other = await npmVersionFiles(name, version, facts.versions.get(version), context)
+  const files = filesRisk(other, `${name}@${version}`)
+  const base = fromEmpty ? { npm: { name, version: '', manifest: {} }, json: { sources: [], sourcesContent: [] } } : old === null ? null : npmBaseFilesTell(other, old.manifest) ? await npmVersionFiles(name, oldVersion, old, context)
+    : { npm: { name, version: oldVersion, manifest: old.manifest, published: old.published }, json: null }
+  const standing = changed ? npmAccountStanding(facts.releases ?? await npmPackageReleases(name, context), other.npm.published) : null
+  const kept = lastingOf(other, base, files, standing)
+  if (context.cache) await context.cache.write('risk/npm/assessments', key, kept)
+  return kept
+}
+
+// How many dependencies added are told of at most, however many a version
+// adds, as each costs npm a request or two: the rest are left as they are,
+// their packages unknown.
+const MAX_TOLD_DEPENDENCIES = 100
+
+// Each dependency `added` that npm installs from its registry, up to
+// MAX_TOLD_DEPENDENCIES of them, but for the caller's own packages (`owned`,
+// as its own scopes take, few downloading them, as new as its releases),
+// with what risk tells of its package
+// (new-dependency.js npmAddedDependency): its tier, its last whole month's
+// downloads by what its tier goes by where it has one (`month`, `{ total,
+// from }`), `uncounted` where npm counts none of them, `created` and `gap`
+// where it is new since the version compared with (`published`, when that
+// and the open version were), and for one nothing imports (flagged so, or
+// among `unusedKeys`) that risk counts (dependencies.js
+// npmUnusedDependencyCounted), the version its spec installs (`installs`,
+// registry.js npmPublicFacts); `missing` where
+// its package was looked up and npm has none of it for anyone, as once
+// unpublished or taken down; four asked at a time, whether each is the
+// caller's own among them, none more once one fails or MAX_TOLD_DEPENDENCIES
+// not the caller's own were taken, however many a version adds.
+async function addedDependencies(added, published, context, owned, unusedKeys = new Set()) {
+  const told = [...added]
+  const asked = told.flatMap((row, at) => row.opens ? [at] : [])
+  let next = 0, taken = 0
+  const tell = async row => {
+    const downloads = await npmDownloads(row.opens, context)
+    const unused = (row.flags.some(flag => flag.check === 'unimported') || unusedKeys.has(keyOf(row))) && npmUnusedDependencyCounted(row)
+    let facts = npmAddedDependency({ downloads, published, unused }, context.now), missing = false
+    if (facts.wanted) {
+      const listed = await npmPublicFacts(row.opens, context, unused ? row.range : null)
+      missing = listed === null
+      facts = npmAddedDependency({ downloads, listed: listed && { ...listed, private: false }, published, unused }, context.now)
+    }
+    const settled = npmSettledDownloads(downloads, context.now)
+    const month = npmDownloadMonths(settled).at(-1)
+    return { ...row, ...facts.tier === null ? {} : { ...tierOf(facts.tier), ...settled ? month && { month: { total: month.total, from: month.from.toISOString().slice(0, 10) } } : { uncounted: true } },
+      ...facts.created ? { created: facts.created, gap: facts.gap } : {}, ...facts.installs ? { installs: facts.installs } : {}, ...missing ? { missing } : {} }
+  }
+  await Promise.all(Array.from({ length: 4 }, async () => {
+    while (next < asked.length && taken < MAX_TOLD_DEPENDENCIES) {
+      const at = asked[next++]
+      try {
+        if (await owned(told[at].opens)) continue
+        if (taken >= MAX_TOLD_DEPENDENCIES) break
+        taken++
+        told[at] = await tell(told[at])
+      } catch (error) { next = asked.length; throw error }
+    }
+  }))
+  return told
+}
+
+// What calls for a look at `version` of package `name`, as JSON holds it:
+// malware first (malware.js npmMalwareFinding: its package taken down by
+// npm, and npm's advisories of malware naming it where `advisories`, true
+// unless given false, has them asked, OSV's too where `osv` is, true unless
+// given false, as a private package's name is never sent it); against `oldVersion` where given,
+// true for the one of its versions most likely free of what an attack
+// injects, as risk picks it by npm's document (old-version.js npmOldVersion;
+// none where none is below it), the one it picked said as `oldVersion`
+// (compare.js npmCompareFindings's order), else what it holds (held.js
+// npmHeldFindings), a deprecation first, and last
+// one younger than `minimumReleaseAge` (two weeks unless given; 0 for none, as
+// a caller's own scopes take), and the package's tier past the top 20,000
+// (`popularity`, held.js npmVersionFindings, unless `popularity` is given
+// false, as a caller's own scopes take); `files`, each file's stamps and what risk read of it; and
+// `dependencies`, those added and changed against it, each with its flags,
+// an added one with its package's tier and age where `downloads` (true unless
+// given false) has them asked, and `dependencies` (true unless given false)
+// too, but where `owned` says its package is the caller's own, as its own scopes
+// take (none unless given). `cache` keeps what is read (registry.js),
+// `packument` asks for a package's whole document and `tarball` for a
+// version's tarball in risk's stead (registry.js), `now` is when it is asked, `signal` gives up on what is
+// still asked of npm. A version npm removed, as it removes every one it finds
+// malware in, is told of by its malware alone (`removed`), nothing of it left
+// to read. Throws an NpmRiskError (registry.js), its `code` saying why, where
+// npm has no such package or version (nor malware explaining one removed),
+// answers past risk's bounds, or can't be asked.
+export async function assessNpm(name, version, { oldVersion = null, fromEmpty = false, minimumReleaseAge = DEFAULT_MINIMUM_RELEASE_AGE, popularity = true, cache, packument, tarball, now = Date.now(), signal, downloads = true, dependencies = true, advisories = true, osv = true, owned = () => false } = {}) {
+  const context = { cache, packument, tarball, now, signal }
+  const facts = await npmPackageFacts(name, version, oldVersion, context)
+  const old = facts.oldVersion
+  if (old !== null && !facts.versions.has(old)) throw new NpmRiskError('version-not-found', `npm has no ${name}@${old}`)
+  // Its advisories, its lasting risk and its downloads need none of each
+  // other, so all are asked at once, and waited on in this order, so that a
+  // failure fails it as the first of them failing would.
+  const listed = facts.versions.has(version)
+  const advised = settledLater(advisories ? npmVersionAdvisories(name, version, osv, context) : Promise.resolve([]))
+  const keeping = listed ? settledLater(lasting(name, version, old, facts, context, fromEmpty)) : null
+  const counting = listed && downloads && popularity ? settledLater(npmDownloads(name, context)) : null
+  const malware = npmMalwareFinding({ takenDown: facts.takenDown, advisories: await advised })
+  if (!listed) {
+    if (!malware) throw new NpmRiskError('version-not-found', `npm has no ${name}@${version}`)
+    return { name, version, oldVersion: null, published: facts.removed.get(version), removed: true, findings: [malware], files: {}, dependencies: { added: [], changed: [] },
+      score: npmRiskScore([malware]) }
+  }
+  const { published, manifest } = facts.versions.get(version)
+  const kept = await keeping
+  const counted = await counting
+  const own = npmVersionFindings({ deprecated: facts.deprecated[version], published, minimumReleaseAge, now, known: Boolean(manifest.knownAccounts), scope: npmKnownScope(name),
+    provenance: Boolean(manifest.provenance), downloads: counted })
+  const was = old === null ? null : facts.versions.get(old).published
+  // Those it holds that nothing imports by another spec than the version
+  // compared with's, told by their new spec as an added one nothing imports
+  // is (`respecified`), and an added one nothing imports however it was
+  // flagged, as one moved from another kind of dependency is.
+  const unusedKeys = unusedKeysOf(kept)
+  const respecified = kept.dependencies.changed.filter(row => unusedKeys.has(keyOf(row))).map(row => ({ ...row, range: row.to }))
+  const told = downloads && dependencies ? await addedDependencies([...kept.dependencies.added, ...respecified], [was, published], context, owned, unusedKeys)
+    : [...kept.dependencies.added, ...respecified]
+  const added = told.slice(0, kept.dependencies.added.length)
+  const assessment = npmAssessmentOf(kept, malware ? [malware, ...own] : own, added, told.slice(added.length))
+  return { name, version, oldVersion: old, published, ...assessment, score: npmRiskScore(assessment.findings, { published, now, tier: counted ? npmDownloadTier(counted) : null }) }
+}
+
+// A dependency's key among a version's, by its kind and name.
+const keyOf = row => `${row.kind ?? ''}\0${row.name}`
+
+// `promise`, waited on later: its failure, if it fails before then, not
+// thrown as unhandled meanwhile.
+const settledLater = promise => { promise.catch(() => {}); return promise }
+
+// The keys of the dependencies a version holds that nothing of it imports
+// (`kept`, npmLastingRisk's: its `held-unused-deps`), those risk counts
+// (dependencies.js npmUnusedDependencyCounted).
+const unusedKeysOf = kept => new Set((kept.findings.find(({ id }) => id === 'held-unused-deps')?.dependencies ?? []).filter(npmUnusedDependencyCounted).map(keyOf))
+
+// The dependencies nothing of a version imports (`held-unused-deps`), each
+// graded by what risk tells of its package fresh (`told`, as
+// addedDependencies tells it; new-dependency.js
+// npmUnusedDependencySeverity), with its tier, when its package was first
+// published and the version its spec installs where told: kept from the
+// version compared with where there was one (`compared`) and it named it as
+// the same kind of dependency by the same spec: not one marked new, nor
+// `added` (its kind and name new, as one moved from another kind is), nor
+// `changed` (each said with the spec it had, `was`); one risk doesn't count
+// (dependencies.js npmUnusedDependencyCounted) a notice, told all the same.
+// The finding as its most severe.
+function unusedGraded(finding, told, compared, { added, changed }) {
+  const facts = new Map(told.map(row => [keyOf(row), row])), was = new Map(changed.map(row => [keyOf(row), row.from]))
+  const anew = new Set(added.map(keyOf))
+  const dependencies = finding.dependencies.map(row => {
+    const of = facts.get(keyOf(row))
+    return { ...row, ...was.has(keyOf(row)) ? { was: was.get(keyOf(row)) } : {}, severity: npmUnusedDependencyCounted(row)
+      ? npmUnusedDependencySeverity(of, compared && !row.added && !anew.has(keyOf(row)) && !was.has(keyOf(row))) : 'notice',
+      ...of?.level ? { tier: of.tier, level: of.level } : {}, ...of?.created ? { created: of.created, gap: of.gap } : {}, ...of?.installs ? { installs: of.installs } : {},
+      ...of?.missing ? { missing: true } : {} }
+  })
+  return { ...finding, severity: SEVERITIES[Math.min(...dependencies.map(row => SEVERITIES.indexOf(row.severity)))], dependencies }
+}
+
+// An assessment of what of it holds whatever the time (`kept`,
+// npmLastingRisk's) and what is told fresh: the open version's own findings
+// (`own`, held.js npmVersionFindings, and its malware), its malware and
+// deprecation first and the rest last, its tier as JSON holds it (tierOf),
+// and its dependencies `added`, as addedDependencies tells them, those it
+// holds by another spec that nothing imports beside them (`respecified`),
+// for its unused dependencies' grades. The managed
+// server's (assessNpm), and a test's of versions it holds.
+const FIRST = new Set(['malware', 'deprecated'])
+export function npmAssessmentOf(kept, own, added = kept.dependencies.added, respecified = []) {
+  return {
+    findings: [...own.filter(({ id }) => FIRST.has(id)), ...kept.findings.map(finding => finding.id === 'held-unused-deps'
+      ? unusedGraded(finding, [...added, ...respecified], Boolean(kept.dependencies.held), kept.dependencies) : finding),
+      ...own.filter(({ id }) => !FIRST.has(id)).map(finding => finding.id === 'popularity' ? { ...finding, ...tierOf(finding.tier) } : finding)],
+    files: kept.files,
+    dependencies: { ...kept.dependencies, added },
+  }
+}

package/risk/npm/combined.js

@@ -0,0 +1,68 @@
+// What calls for a look at a version against the one compared with, as one
+// list: what it holds, always (held.js npmHeldFindings, and its dependencies
+// against none, npmHeldDependencyFindings), as no change hides any of it; and
+// what changed (compare.js npmCompareFindings), each change to what it holds
+// folded into its held finding, never lowering it: its severity the higher of
+// the two, the files, scripts or dependencies the change brought marked
+// (`added`); the rest (a publisher changed, provenance lost, large code
+// added, a file become unreadable, hosts named anew) as they are.
+import { SEVERITIES } from '../files/stamps.js'
+import { HOST_TAGS } from '../files/hosts.js'
+import { npmDependencyFindings } from './compare.js'
+import { npmHostFinding } from './held.js'
+
+// A version's dependencies that call for a look whatever came before it
+// (compare.js npmDependencyFindings against none, `dependencies` flagged
+// against none): `held-non-registry-deps` and `held-unused-deps`.
+export const npmHeldDependencyFindings = dependencies => npmDependencyFindings(dependencies, null).map(finding => ({ ...finding, id: `held-${finding.id}` }))
+
+// The more severe of two severities (stamps.js SEVERITIES).
+const higher = (a, b) => SEVERITIES[Math.min(SEVERITIES.indexOf(a), SEVERITIES.indexOf(b))]
+
+// What a change to what a version holds marks of its held finding's rows,
+// by the change's id: a file's what it runs, the services it names under a
+// hosts' tag, and an obfuscator's fingerprint new to it (`added`); the
+// scripts added or changed and what each version ran (`flagged`, `was`); a
+// dependency taken from elsewhere than npm's registry anew, its spec before
+// (`was`), or added that nothing imports (`added`). Code kept running beyond
+// itself or reaching for credentials (`kept-runs-code`, `kept-credentials`)
+// marks nothing, its files held already.
+const byPath = (rows, key) => new Map(rows.map(row => [row.path, row[key]]))
+const marked = (held, rows) => ({ ...held, files: held.files.map(row => rows.has(row.path) ? { ...row, added: rows.get(row.path) } : row) })
+const FOLDS = {
+  'runs-code': ['held-runs-code', (held, change) => marked(held, byPath(change.files, 'added'))],
+  'kept-runs-code': ['held-runs-code', held => held],
+  credentials: ['held-credentials', (held, change) => marked(held, byPath(change.files, 'added'))],
+  'kept-credentials': ['held-credentials', held => held],
+  obfuscated: ['held-obfuscated', (held, change) => marked(held, byPath(change.files, 'obfuscation'))],
+  'install-scripts': ['held-install-scripts', (held, change) => ({ ...held, flagged: change.flagged, was: [...change.was] })],
+  'non-registry-deps': ['held-non-registry-deps', (held, change) => {
+    const was = new Map(change.dependencies.map(row => [row.name, row.was]))
+    return { ...held, dependencies: held.dependencies.map(row => was.has(row.name) ? { ...row, was: was.get(row.name), added: true } : row) }
+  }],
+  'unused-deps': ['held-unused-deps', (held, change) => {
+    const added = new Set(change.dependencies.map(row => `${row.kind ?? ''}\0${row.name}`))
+    return { ...held, dependencies: held.dependencies.map(row => added.has(`${row.kind ?? ''}\0${row.name}`) ? { ...row, added: true } : row) }
+  }],
+  ...Object.fromEntries([...HOST_TAGS.keys()].map(npmHostFinding).map(id => [id, [`held-${id}`, (held, change) => marked(held, byPath(change.files, 'added'))]])),
+}
+
+// Who published the open version and how, ahead of what it holds, as a
+// reviewer reads them; what else changed after it.
+const FIRST = new Set(['publisher', 'provenance-lost'])
+
+// What a version holds (`held`) with what changed against the one compared
+// with (`changed`) folded in, as one list in the order a reviewer reads it: a
+// publisher changed and provenance lost, what it holds, then the rest of what
+// changed. A change whose held finding is missing is kept as it is.
+export function npmCombinedFindings(held, changed) {
+  const combined = new Map(held.map(finding => [finding.id, finding]))
+  const rest = []
+  for (const change of changed) {
+    const [id, fold] = FOLDS[change.id] ?? []
+    const into = id && combined.get(id)
+    if (into) combined.set(id, { ...fold(into, change), severity: higher(into.severity, change.severity) })
+    else rest.push(change)
+  }
+  return [...rest.filter(({ id }) => FIRST.has(id)), ...combined.values(), ...rest.filter(({ id }) => !FIRST.has(id))]
+}

package/risk/npm/compare.js

@@ -0,0 +1,257 @@
+// What calls for a look at a package version against the one before it, as
+// findings: `{ id, severity, ... }` each (stamps.js SEVERITIES), what it found
+// beside. Each version as risk holds one (`{ kind: 'sourcemap', npm, json,
+// fileSizes? }`: npm's facts, `npm.manifest` as manifest.js reads a version's
+// document, its files as a source map lists them, `json.sources` and
+// `json.sourcesContent`, a text or null each, and their sizes by path where
+// known), and `files` how they differ (diff.js npmFilesDiff): `onlyOther`, the
+// files the open version adds, and `changed`, those it changes, each `{ path,
+// renamedFrom?, modified? }`, by its path in the open version.
+import { bundleFileSizes, bundleFilesAsMap } from '../../common/bundle-sources.js'
+import { npmLoadedScripts, npmVersionFileRisk, npmVersionHosts } from './files.js'
+import { npmHostFinding } from './held.js'
+import { HOST_TAGS, npmHostSeverity, npmHostsTagged } from '../files/hosts.js'
+import { npmBecameUnreadable, npmBinaryCode, npmLargeCode, npmUnreadable } from '../files/readability.js'
+import { npmIsScript } from '../files/script.js'
+import { SEVERITIES } from '../files/stamps.js'
+import { CREDENTIAL_TAGS, npmFoundAdded, npmRunsCodeFlags, npmRunsCodeTagged, npmRunsCodeWithout } from '../files/runs-code.js'
+import { npmDependencySpecs } from './dependencies.js'
+import { npmFilesDiff } from './diff.js'
+import { npmInstallScripts } from './install-scripts.js'
+import { npmInstallScriptsChange, npmProvenanceLost, npmPublisherChange } from './release.js'
+
+// The files the open version (`other`) runs as scripts that the one compared
+// with (`base`) held, changed or not, but didn't run, by their names
+// (script.js npmIsScript, a `.js` renamed `.txt`) or as code loaded them
+// (npm/files.js npmLoadedScripts), as its code now loads them, each a row as npmFilesDiff's
+// `changed` holds one, its own where it has one: what node runs of them is
+// new, as a changed file's is.
+function npmWokenFiles(base, other, files) {
+  const added = new Set(files.onlyOther.map(row => row.path)), rows = new Map(files.changed.map(row => [row.path, row]))
+  const before = bundleFilesAsMap(base), loadedBefore = npmLoadedScripts(base)
+  // A file that isn't text by its name alone (`payload.js`).
+  const ran = path => loadedBefore.has(path) || (before.has(path) && npmIsScript(path, typeof before.get(path) === 'string' ? before.get(path) : ''))
+  return [...npmLoadedScripts(other)].filter(path => !added.has(path)).map(path => rows.get(path) ?? { path })
+    .filter(row => !ran(row.renamedFrom ?? row.path))
+}
+
+// What the open version does to its files that calls for a look, as red
+// findings: code that runs beyond itself (runs-code.js), new to a file it
+// adds or changes, the kinds with their lines (`runs-code`), those reaching
+// for credentials apart (CREDENTIAL_TAGS, `credentials`); a service under
+// one of the hosts' tags new to a file it adds or changes (hosts.js
+// npmExfiltration), a finding a tag a level higher than its own
+// (npmHostSeverity: `exfiltration` and `remote` red, `ip-lookup` orange); a
+// code file of 256 KiB or more it adds, one node runs as a script whatever
+// its name too (npmLargeCode, `large-code`); a file
+// it changes that had no line over 200 characters and now has one over 2,000
+// (npmBecameUnreadable, `now-unreadable`); a file it adds or changes with
+// an obfuscator's fingerprint where the one compared with had none
+// (`obfuscated`); and a code file it adds or changes that isn't text, where
+// the one compared with's was text, or no code, or none (npmBinaryCode,
+// `binary-code`). Then in orange what any of its files runs beyond itself that
+// `runs-code` doesn't name, kept from the version compared with
+// (`kept-runs-code`, `kept-credentials`). A renamed file by its old path; one renamed alone,
+// unchanged, holds what it held. A file become unreadable calls for less
+// where its source map was rebuilt with it (`rebuilt`, rebuiltWithMap), as a
+// build minifies a file and writes its map together, and in a `major`
+// release (npmMajorRelease): `now-unreadable` orange where every such file is
+// either, else red. A file it held, changed or not, that node now runs as a
+// script (npmWokenFiles) counts as one it changes, large code where it is
+// one, and unreadable now where it wasn't as no script, whatever its lines
+// were.
+function npmFileFindings(base, other, files, major) {
+  const added = new Set(files.onlyOther)
+  const after = row => npmVersionFileRisk(other, row.path)
+  const before = row => added.has(row) ? null : npmVersionFileRisk(base, row.renamedFrom ?? row.path)
+  const changed = files.changed.filter(row => row.renamedFrom == null || row.modified)
+  const woken = npmWokenFiles(base, other, files)
+  const touched = [...new Set([...files.onlyOther, ...changed, ...woken])]
+  const sizes = bundleFileSizes(other)
+  // The files that hold what they didn't before, each with that.
+  const gained = key => touched.map(row => ({ path: row.path, added: npmFoundAdded(after(row)[key], before(row)?.[key] ?? null) })).filter(row => row.added)
+  const exfiltration = gained('exfiltration'), runsCode = gained('runsCode')
+  const flagged = new Set(runsCode.flatMap(row => npmRunsCodeFlags(row.added).map(({ name }) => name)))
+  const kept = [...bundleFilesAsMap(other).keys()].map(path => ({ path, found: npmRunsCodeWithout(npmVersionFileRisk(other, path).runsCode, flagged) })).filter(row => row.found)
+  // Code that runs beyond itself, or of the kinds reaching for credentials.
+  const ofKinds = (rows, key, of) => rows.map(row => ({ ...row, [key]: of(row[key], CREDENTIAL_TAGS) })).filter(row => row[key])
+  return [
+    { id: 'runs-code', severity: 'critical', files: ofKinds(runsCode, 'added', npmRunsCodeWithout) },
+    { id: 'credentials', severity: 'critical', files: ofKinds(runsCode, 'added', npmRunsCodeTagged) },
+    ...[...HOST_TAGS.keys()].map(tag => {
+      const rows = exfiltration.map(row => ({ path: row.path, added: npmHostsTagged(row.added, tag) })).filter(row => row.added)
+      return { id: npmHostFinding(tag), severity: rows.length > 0 ? npmHostSeverity(tag, rows[0].added, null) : HOST_TAGS.get(tag), files: rows }
+    }),
+    { id: 'large-code', severity: 'critical', files: [...files.onlyOther, ...woken].filter(row => npmLargeCode(row.path, sizes.get(row.path), npmLoadedScripts(other).has(row.path)))
+      .map(row => ({ path: row.path, bytes: sizes.get(row.path) })) },
+    { id: 'now-unreadable', ...unreadableFinding(major, [...new Set([...changed.filter(row => npmBecameUnreadable(row.path, before(row)?.readability, after(row).readability, npmLoadedScripts(other).has(row.path))),
+      ...woken.filter(row => npmUnreadable(row.path, after(row).readability, true) && !npmUnreadable(row.renamedFrom ?? row.path, before(row).readability))])]
+      .map(row => ({ path: row.path, was: before(row).readability.longest, now: after(row).readability.longest,
+        ...rebuiltWithMap(base, other, row.path, row.renamedFrom ?? row.path) ? { rebuilt: true } : {} }))) },
+    { id: 'obfuscated', severity: 'critical', files: touched.filter(row => after(row).readability.obfuscation && !before(row)?.readability.obfuscation)
+      .map(row => ({ path: row.path, obfuscation: after(row).readability.obfuscation })) },
+    { id: 'binary-code', severity: 'critical', files: touched.filter(row => npmBinaryCode(row.path, after(row).readability, npmLoadedScripts(other).has(row.path))
+      && !npmBinaryCode(row.renamedFrom ?? row.path, before(row)?.readability, npmLoadedScripts(base).has(row.renamedFrom ?? row.path)))
+      .map(row => ({ path: row.path, bytes: sizes.get(row.path) })) },
+    { id: 'kept-runs-code', severity: 'high', files: ofKinds(kept, 'found', npmRunsCodeWithout) },
+    { id: 'kept-credentials', severity: 'high', files: ofKinds(kept, 'found', npmRunsCodeTagged) },
+  ].filter(finding => finding.files.length > 0)
+}
+
+// Hosts whose naming anew calls for no look: where packages keep their code
+// and docs, as their links and error messages name them, the most a version
+// names anew of popular packages a year apart (lit.dev, Lit's), and none
+// whose content anyone may publish (a wiki's raw pages) but github.com's
+// pages, as hosts.js githubNamed names them: a repository's own URL, its
+// files and releases are named by it. Each as it is written, not a host
+// under it, so one an attack sends to or fetches from there
+// (`api.github.com`, `raw.githubusercontent.com`) is still named; and no link
+// shortener (`git.io`, `git.new`), which can lead anywhere.
+const ALLOWED_HOSTS = new Set(['github.com', 'lit.dev', 'developer.mozilla.org', 'json-schema.org', 'nodejs.org', 'babeljs.io', 'tseslint.com',
+  'devblogs.microsoft.com', 'semver.org'])
+
+// A severity one level lower (stamps.js SEVERITIES).
+const lower = severity => SEVERITIES[Math.min(SEVERITIES.indexOf(severity) + 1, SEVERITIES.length - 1)]
+
+// Whether the open version (`now`) is a major release past the one compared
+// with (`was`): its major number higher, or its minor for 0.x. A rewrite
+// changes how a package's files read and who publishes it, and npm's `^`
+// ranges don't take a major release by themselves, as a hijacked release
+// wants them to.
+function npmMajorRelease(was, now) {
+  const [a, b] = [was, now].map(version => /^v?(\d+)\.(\d+)\./u.exec(version ?? '')?.slice(1).map(Number))
+  return Boolean(a && b) && (b[0] > a[0] || (a[0] === 0 && b[0] === 0 && b[1] > a[1]))
+}
+
+// Whether the open version's file at `path` (`basePath` in the one compared
+// with) had its source map rebuilt with it: its last line, a
+// `sourceMappingURL` comment and no more, names a map file of the package by
+// a plain path from the file's own (MAP_NAME), a source map
+// (its `mappings`, or each of an indexed map's `sections`, told), new or
+// changed from the one at its path, or beside the file's own before it was
+// renamed, as a build writes them together. An attack editing a built file
+// leaves its map as it was.
+const SOURCE_MAP_FILE = /^\s*(?:\/\/[#@] sourceMappingURL=(\S+)|\/\*[#@] sourceMappingURL=(\S+?)\s*\*\/)\s*$/u
+// A map name a build writes (`a.js.map`, `../maps/a.map`): segments of
+// letters, digits and `._~@+-`, as a browser reads them alike. One at a URL,
+// from a site's root, or with an escape or a backslash names none.
+const MAP_NAME = /^(?:[\w.~@+-]+\/)*[\w.~@+-]+\.map$/u
+function besideFile(file, name) {
+  const parts = file.split('/').slice(0, -1)
+  for (const part of name.split('/')) {
+    if (part === '..' && parts.length === 0) return null
+    if (part === '..') parts.pop()
+    else if (part !== '.') parts.push(part)
+  }
+  return parts.join('/')
+}
+function rebuiltWithMap(base, other, path, basePath) {
+  const text = bundleFilesAsMap(other).get(path)
+  const directive = typeof text === 'string' ? SOURCE_MAP_FILE.exec(text.trimEnd().split('\n').at(-1)) : null
+  const name = directive?.[1] ?? directive?.[2]
+  if (!name || !MAP_NAME.test(name)) return false
+  const [map, was] = [bundleFilesAsMap(other).get(besideFile(path, name)), bundleFilesAsMap(base)]
+  return isSourceMap(map) && map !== was.get(besideFile(path, name)) && map !== was.get(besideFile(basePath, name))
+}
+const isSourceMap = text => {
+  try {
+    const json = JSON.parse(text)
+    return typeof json?.mappings === 'string'
+      || (Array.isArray(json?.sections) && json.sections.length > 0 && json.sections.every(section => typeof section?.map?.mappings === 'string'))
+  } catch { return false }
+}
+
+// The files become unreadable as a finding says them: red, but orange in a
+// major release, or where each had its source map rebuilt with it.
+const unreadableFinding = (major, files) => ({ severity: major || files.every(row => row.rebuilt) ? 'high' : 'critical', ...major ? { major } : {}, files })
+
+// The hosts the open version's code names in its URLs that the one compared
+// with's named in none of its, ALLOWED_HOSTS aside, each `{ host, path, line,
+// places }`: the file it is first named in and its line, and `places` every
+// file that names it, `{ path, line }` each, the line it is first named on
+// there. A file unchanged names what it named, so the files the open version
+// adds or changes hold all it names anew.
+function npmNewHosts(base, other, files) {
+  const found = new Map(), known = npmVersionHosts(base)
+  for (const path of new Set([...files.onlyOther, ...files.changed, ...npmWokenFiles(base, other, files)].map(row => row.path))) {
+    for (const [host, line] of npmVersionFileRisk(other, path).hosts ?? []) {
+      if (known.has(host) || ALLOWED_HOSTS.has(host)) continue
+      const place = { path, line }
+      if (found.has(host)) found.get(host).places.push(place)
+      else found.set(host, { host, ...place, places: [place] })
+    }
+  }
+  return [...found.values()]
+}
+
+// Whether the files of the version compared with could change what
+// npmCompareFindings finds of the open version (`other`), told before they
+// are read, by the open version's files and the manifest npm's document of
+// the one compared with gives (`manifest`): what its files call for
+// (npmFileFindings, npmNewHosts) is only ever what the open version's own
+// files hold that the one compared with's didn't, so where none of them runs
+// code beyond itself, names a service under one of the hosts' tags or a host
+// past ALLOWED_HOSTS, bears an obfuscator's fingerprint, has a line no one
+// writes (npmUnreadable), is large code or is code that isn't text
+// (npmBinaryCode), nothing is found of them whatever
+// the one compared with holds. Nor do the one compared with's install scripts
+// differ by a binding.gyp among its files, but where its manifest names
+// neither `preinstall` nor `install`, and the open version has some.
+export function npmBaseFilesTell(other, manifest) {
+  if (npmInstallScripts(other.npm.manifest, other.json.sources).length > 0
+    && npmInstallScripts(manifest).length !== npmInstallScripts(manifest, ['binding.gyp']).length) return true
+  const sizes = bundleFileSizes(other)
+  return [...bundleFilesAsMap(other).keys()].some(path => {
+    const { runsCode, exfiltration, readability, hosts } = npmVersionFileRisk(other, path)
+    return Boolean(runsCode || exfiltration || readability.obfuscation) || npmUnreadable(path, readability, npmLoadedScripts(other).has(path)) || npmLargeCode(path, sizes.get(path), npmLoadedScripts(other).has(path))
+      || npmBinaryCode(path, readability, npmLoadedScripts(other).has(path))
+      || [...hosts?.keys() ?? []].some(host => !ALLOWED_HOSTS.has(host))
+  })
+}
+
+// Of the dependencies the open version adds and changes against `base`, or
+// alone, none given, each with its flags (dependencies.js
+// npmFlagDependencyChanges): those it takes from elsewhere than npm's
+// registry, where the one compared with didn't, or by another spec
+// (`non-registry-deps`, each `{ name, was, now }`, its specs); and those it
+// adds that nothing of it imports, which npm installs all the same, its
+// install scripts and all, as a dependency added to run them would be
+// (`unused-deps`, each `{ name, kind, range }`).
+export function npmDependencyFindings({ added, changed }, base) {
+  const flagged = check => [...added, ...changed].filter(row => row.flags.some(flag => flag.check === check))
+  const was = base ? npmDependencySpecs(base.npm.manifest) : null
+  const fetched = flagged('source').map(row => ({ name: row.name, was: was?.get(row.name) ?? null, now: row.to ?? row.range }))
+  const unimported = flagged('unimported').map(({ name, kind, range }) => ({ name, kind, range }))
+  return [
+    ...fetched.length > 0 ? [{ id: 'non-registry-deps', severity: 'high', dependencies: fetched }] : [],
+    ...unimported.length > 0 ? [{ id: 'unused-deps', severity: 'high', dependencies: unimported }] : [],
+  ]
+}
+
+// Everything that calls for a look at the open version (`other`) against the
+// one compared with (`base`), whatever the time, in the order a reviewer reads
+// them: a publisher that changed (release.js npmPublisherChange,
+// `publisher`, each account's standing on the package told where `standing`
+// is given), a level lower in a major release (npmMajorRelease, `major`),
+// as a package changing hands with a rewrite does; provenance lost (`provenance-lost`); install scripts added or
+// changed (`install-scripts`); its `dependencies` added and changed, as
+// flagged (npmDependencyFindings); its files, by how they differ (diff.js
+// npmFilesDiff, npmFileFindings); and the hosts its code names anew
+// (npmNewHosts, `new-hosts`). A `base` without its files (`json` null),
+// where npmBaseFilesTell says they would change nothing, is found nothing of
+// by them.
+export function npmCompareFindings(base, other, dependencies, standing = null) {
+  const major = npmMajorRelease(base.npm.version, other.npm.version)
+  const publisher = npmPublisherChange(base.npm.manifest, other.npm.manifest, standing)
+  const scripts = npmInstallScriptsChange(...[base, other].map(version => ({ manifest: version.npm.manifest, paths: version.json?.sources })))
+  const files = base.json ? npmFilesDiff(base, other) : null
+  const hosts = files ? npmNewHosts(base, other, files) : []
+  return [
+    ...publisher ? [{ id: 'publisher', ...publisher, ...major ? { severity: lower(publisher.severity), major } : {} }] : [],
+    ...npmProvenanceLost(base.npm.manifest, other.npm.manifest) ? [{ id: 'provenance-lost', severity: 'critical' }] : [],
+    ...scripts ? [{ id: 'install-scripts', severity: 'high', ...scripts }] : [],
+    ...npmDependencyFindings(dependencies, base),
+    ...files ? npmFileFindings(base, other, files, major) : [],
+    ...hosts.length > 0 ? [{ id: 'new-hosts', severity: 'high', hosts }] : [],
+  ]
+}