@preventive/triage 1.0.0-alpha.37 → 1.0.0-alpha.39

package/server-managed/npm-kept.ts

@@ -0,0 +1,65 @@
+// What the npm viewer and repository risk scans keep in the database
+// (managed_upstream_cache, upstream-cache.ts), beyond the instance, so that
+// neither asks for it again on every read, nor after a restart or on another
+// instance: public versions' documents, of the fields read of them
+// (npm-manifest.ts npmVersionFields), for a day (readNpmVersion asks npm again
+// then, as a version can be deprecated, its package made private, or itself
+// removed); and, for scans, each package's risk line, for as long as a scan
+// keeps it (repository-risk.ts). Each store removes its own rows kept longer
+// than that as it writes, at most once an hour a database, so that rows no
+// read will take again don't pile up.
+// The database is an optimization: a failure reads as a miss, and writes
+// nothing.
+import { createHash } from 'node:crypto'
+import { plainObject } from './npm-manifest.ts'
+import { type NpmVersionStore, VERSION_KEPT_MS } from './npm-packages.ts'
+import type { UpstreamCacheStore } from './upstream-cache.ts'
+
+// A value kept by key, as JSON, `at` when it was.
+export interface KeptStore<T> {
+  read(key: string): Promise<{ at: number; value: T } | null>
+  write(key: string, value: T, at: number): Promise<void>
+}
+
+// When each database last had each type's old rows removed.
+const PRUNE_EVERY_MS = 60 * 60_000
+const pruned = new WeakMap<UpstreamCacheStore, Map<string, number>>()
+
+// Values of `type`, each read for `keptMs` after it was kept.
+function keptStore<T>(db: UpstreamCacheStore, type: string, keptMs: number, debug: boolean): KeptStore<T> {
+  const warn = (what: string, error: unknown) => { if (debug) console.warn(`managed: kept ${type} ${what} failed:`, error) }
+  const prune = async (at: number) => {
+    const times = pruned.get(db) ?? pruned.set(db, new Map()).get(db)!
+    if (at - (times.get(type) ?? -Infinity) < PRUNE_EVERY_MS) return
+    times.set(type, at)
+    try { await db.deleteUpstreamCacheEntries(`${type}/`, at - keptMs) } catch (error) { warn('prune', error) }
+  }
+  // Keys of any length or character, as the database's: hashed.
+  const keyOf = (key: string) => `${type}/${createHash('sha256').update(key).digest('hex')}`
+  return {
+    async read(key) {
+      try {
+        const kept: unknown = JSON.parse(await db.getUpstreamCacheEntry(keyOf(key)) ?? 'null')
+        return plainObject(kept) && typeof kept['at'] === 'number' && 'value' in kept ? { at: kept['at'], value: kept['value'] as T } : null
+      } catch (error) { warn('read', error); return null }
+    },
+    async write(key, value, at) {
+      try { await db.setUpstreamCacheEntry(keyOf(key), JSON.stringify({ at, value }), at) } catch (error) { warn('write', error) }
+      await prune(at)
+    },
+  }
+}
+
+export function npmVersionStore(db: UpstreamCacheStore, debug = false): NpmVersionStore {
+  const kept = keptStore<Record<string, unknown>>(db, 'npm/version-documents', VERSION_KEPT_MS, debug)
+  return {
+    async read(name, version) {
+      const hit = await kept.read(`${name}@${version}`)
+      return hit && plainObject(hit.value) ? { at: hit.at, json: hit.value } : null
+    },
+    write: (name, version, json) => kept.write(`${name}@${version}`, json, Date.now()),
+  }
+}
+
+// Risk lines, each read for `keptMs`, as long as a scan keeps them.
+export const riskLineStore = <T>(db: UpstreamCacheStore, keptMs: number, debug = false): KeptStore<T> => keptStore<T>(db, 'repository-risk/lines', keptMs, debug)

package/server-managed/npm-loads.ts

@@ -0,0 +1,167 @@
+// Loading a published version for the npm package viewer (npm-packages.ts):
+// its tarball, kept on disk as bundle builds keep theirs, else read from the
+// registry, and the viewer's response built from it, each shared by the
+// readers asking for it meanwhile and held to a few at once per process.
+import { Buffer } from 'node:buffer'
+import { homedir } from 'node:os'
+import { join, resolve } from 'node:path'
+import { promisify } from 'node:util'
+import { gunzip } from 'node:zlib'
+import { encodeBrotli } from './brotli.ts'
+import { MAX_NPM_PACKAGE_BYTES, MAX_NPM_PACKAGE_FILES, NPM_REGISTRY, NpmPackageError, type NpmPackageFile, type NpmVersionDocument, askUpstream,
+  npmFileRows, npmPublishTime, npmToken, readNpmTar } from './npm-packages.ts'
+import { keepFile, readKeptFile, upstreamCachePath } from './upstream-cache.ts'
+
+// The tar stream a package's files may come in, headers and padding included.
+// A tarball larger than that cannot unpack within it, so it is refused as it
+// arrives.
+export const MAX_TAR_BYTES = 96 * 1024 * 1024
+const TARBALL_TIMEOUT_MS = 120_000
+// Loads in flight per process; each can hold a tarball, its tar, its files
+// and their encoded JSON.
+const MAX_ACTIVE_LOADS = 4
+
+const gunzipAsync = promisify(gunzip)
+
+// A load's result, shared by its readers, and a reader's release of it.
+export interface NpmLoad<T> { result: Promise<T>; release: () => void }
+
+interface Load { job: Promise<unknown>; readers: number; settled: boolean }
+
+// Loads by what they read; readers asking for the same share one. A load
+// keeps its place among the active ones until its work is done and each
+// reader has released it, as a reader does once its response is written or
+// abandoned, so what responses still hold counts too.
+const loads = new Map<string, Load>()
+
+const refused = <T>(status: number, code: string): NpmLoad<T> => {
+  const result = Promise.reject(new NpmPackageError(status, code))
+  result.catch(() => {})
+  return { result, release() {} }
+}
+
+function shared<T>(key: string, doc: NpmVersionDocument, work: () => Promise<T>): NpmLoad<T> {
+  const { unpackedSize, fileCount } = doc.dist
+  if ((unpackedSize ?? 0) > MAX_NPM_PACKAGE_BYTES || (fileCount ?? 0) > MAX_NPM_PACKAGE_FILES) return refused(413, 'package-too-large')
+  let load = loads.get(key)
+  if (!load) {
+    if (loads.size >= MAX_ACTIVE_LOADS) return refused(429, 'npm-busy')
+    const created: Load = { job: work(), readers: 0, settled: false }
+    const settle = () => {
+      created.settled = true
+      if (created.readers === 0 && loads.get(key) === created) loads.delete(key)
+    }
+    created.job.then(settle, settle)
+    loads.set(key, created)
+    load = created
+  }
+  const held = load
+  held.readers++
+  let released = false
+  return {
+    result: held.job as Promise<T>,
+    release() {
+      if (released) return
+      released = true
+      if (--held.readers === 0 && held.settled && loads.get(key) === held) loads.delete(key)
+    },
+  }
+}
+
+// The sha512 an integrity names, in base64, or null where it names none.
+const sha512Of = (integrity: string) => /(?:^|\s)sha512-([\d+/A-Za-z]{86}==)(?=\s|$)/u.exec(integrity)?.[1] ?? null
+// Hashed off the event loop: a tarball runs to tens of MiB.
+const sha512 = async (bytes: Uint8Array) => Buffer.from(await crypto.subtle.digest('SHA-512', bytes as Uint8Array<ArrayBuffer>)).toString('base64')
+
+// Upstream's disk cache (setCacheDir), where tarballs are kept between
+// loads, or null to keep none and read none.
+let tarballCache: string | null = null
+
+export function setNpmTarballCache(dir: string | null) {
+  tarballCache = dir === null ? null : resolve(dir)
+}
+
+// Where npm's own cache files content by its sha512, short of an .npmrc
+// moving the cache.
+function npmCachePath(hash: string): string | null {
+  const configured = process.env['npm_config_cache'] || process.env['NPM_CONFIG_CACHE']
+  let root
+  try {
+    root = configured ? resolve(configured.replace(/^~(?=[/\\])/u, homedir()))
+      : process.platform === 'win32' ? join(process.env['LOCALAPPDATA'] || homedir(), 'npm-cache') : join(homedir(), '.npm')
+  } catch { return null }
+  const hex = Buffer.from(hash, 'base64').toString('hex')
+  return join(root, '_cacache', 'content-v2', 'sha512', hex.slice(0, 2), hex.slice(2, 4), hex.slice(4))
+}
+
+// The version's tarball from the registry, at the package's own path: with
+// the token only for a private version, and its size bounded as it arrives.
+async function downloadTarball(url: string, isPrivate: boolean): Promise<Buffer> {
+  const token = isPrivate ? npmToken() : null
+  const bytes = await askUpstream(url, { headers: token ? { authorization: `Bearer ${token}` } : {}, limit: MAX_TAR_BYTES, ms: TARBALL_TIMEOUT_MS, absent: [401, 403, 404] })
+    .catch((err: unknown) => { throw err instanceof NpmPackageError && err.message === 'upstream-too-large' ? new NpmPackageError(413, 'package-too-large') : err })
+  if (bytes === null) throw new NpmPackageError(404, 'package-not-found')
+  return bytes
+}
+
+// The version's tarball, its bytes held to the document's sha512 wherever
+// they come from: kept in the cache, where bundle builds keep what they
+// fetch with the token, or in npm's; else downloaded, and then kept. Bytes
+// matching the sha512 the reader's own document names are the tarball that
+// reader may download, wherever they were kept and whoever fetched them.
+async function readTarball(doc: NpmVersionDocument): Promise<Uint8Array> {
+  const expected = sha512Of(doc.dist.integrity), url = doc.dist.tarball
+  if (!url.startsWith(`${NPM_REGISTRY}/${doc.name}/-/`) || URL.parse(url)?.href !== url || expected === null) throw new NpmPackageError(502, 'upstream-invalid')
+  const kept = tarballCache === null ? null : upstreamCachePath(tarballCache, 'npm/tarballs', `${doc.name}@${doc.version}.tgz`)
+  if (kept !== null) {
+    for (const path of [kept, npmCachePath(expected)]) {
+      const bytes = path === null ? null : await readKeptFile(path, MAX_TAR_BYTES)
+      if (bytes && await sha512(bytes) === expected) return bytes
+    }
+  }
+  const bytes = await downloadTarball(url, doc.private)
+  if (await sha512(bytes) !== expected) throw new NpmPackageError(502, 'upstream-invalid')
+  if (kept !== null) await keepFile(kept, bytes)
+  return bytes
+}
+
+async function readFiles(tarball: Uint8Array): Promise<NpmPackageFile[]> {
+  let tar: Buffer
+  try { tar = await gunzipAsync(tarball, { maxOutputLength: MAX_TAR_BYTES }) }
+  catch (err) {
+    throw new NpmPackageError((err as { code?: string }).code === 'ERR_BUFFER_TOO_LARGE' ? 413 : 422,
+      (err as { code?: string }).code === 'ERR_BUFFER_TOO_LARGE' ? 'package-too-large' : 'bad-tarball')
+  }
+  return readNpmTar(tar)
+}
+
+// The version's tarball, for download.
+export function loadNpmTarball(doc: NpmVersionDocument): NpmLoad<Uint8Array> {
+  return shared(`tarball ${doc.dist.integrity} ${doc.dist.tarball}`, doc, () => readTarball(doc))
+}
+
+// The version's files, for one of them to be pretty-printed (pretty-print.ts).
+export function loadNpmFiles(doc: NpmVersionDocument): NpmLoad<NpmPackageFile[]> {
+  return shared(`files ${doc.dist.integrity} ${doc.dist.tarball}`, doc, async () => readFiles(await readTarball(doc)))
+}
+
+// The version as the viewer reads it, as brotli-encoded JSON: `{ name,
+// version, private, integrity, tarballSize, published, manifest, files,
+// known }`, its files as npmFileRows gives them, and with them, as its files
+// are encoded, when it was published where that is kept (npmPublishTime,
+// else null) and what `known` gives: what is kept about its package
+// (npm-insights.ts npmKnown), which the viewer need not then ask for. Encoding
+// is where a load holds the most, so it takes its place among the active
+// loads until its body is done, and readers asking meanwhile share that body,
+// the first one's manifest and `known` in it; nothing is kept once its readers
+// are done.
+export function loadNpmPackageBody(doc: NpmVersionDocument, known: () => object = () => ({})): NpmLoad<Buffer> {
+  return shared(`package ${doc.dist.integrity} ${doc.dist.tarball} ${doc.private}`, doc, async () => {
+    const tarball = await readTarball(doc)
+    const files = await readFiles(tarball)
+    return encodeBrotli(Buffer.from(JSON.stringify({
+      name: doc.name, version: doc.version, private: doc.private, integrity: doc.dist.integrity, tarballSize: tarball.byteLength,
+      published: npmPublishTime(doc.name, doc.version), manifest: doc.manifest, files: npmFileRows(files), known: known(),
+    })))
+  })
+}

package/server-managed/npm-manifest.ts

@@ -0,0 +1,115 @@
+// What a published version's document tells of it as the npm viewer's
+// Overview shows it (npmManifest), and the fields of the document read for it
+// and for the version itself (npm-packages.ts readNpmVersion), the rest
+// stripped where it is kept (npmVersionFields).
+import publishers from '@preventive/publishers/npm.js'
+import { getRepo } from '@preventive/upstream/package.js'
+
+export const plainObject = (value: unknown): value is Record<string, unknown> => Boolean(value) && typeof value === 'object' && !Array.isArray(value)
+export const stringRecord = (value: unknown) => plainObject(value)
+  ? Object.fromEntries(Object.entries(value).filter(([, item]) => typeof item === 'string')) as Record<string, string> : undefined
+
+interface NpmPerson { name: string | null; email: string | null }
+const NPM_ACCOUNT = /^[\w.-]{1,214}$/u
+// The npm accounts known to publish: those with standing on one of npm's
+// most downloaded packages (@preventive/publishers).
+const KNOWN_PUBLISHERS = new Set(publishers.map(name => name.toLowerCase()))
+
+// One of package.json's people: `{ name, email, url }`, or `Name <email>
+// (url)` in one string.
+function npmPerson(value: unknown): NpmPerson | null {
+  const text = (part: unknown) => typeof part === 'string' && part.trim() !== '' ? part.trim() : null
+  if (plainObject(value)) return { name: text(value['name']), email: text(value['email']) }
+  if (typeof value !== 'string') return null
+  const parts = /^([^<(]*)(?:<([^>]*)>)?\s*(?:\([^)]*\))?\s*$/u.exec(value)
+  return parts ? { name: text(parts[1]), email: text(parts[2]) } : { name: text(value), email: null }
+}
+
+// The GitLab project (`group/project`, subgroups and all) a package names as
+// its repository, in any of the forms npm takes (`gitlab:group/project`,
+// `git@gitlab.com:…`, `git+https://gitlab.com/….git`); null for another.
+// GitLab's are only on gitlab.com, as npm's provenance has them.
+function gitlabProject(repository: unknown): string | null {
+  const url = typeof repository === 'string' ? repository : plainObject(repository) && typeof repository['url'] === 'string' ? repository['url'] : ''
+  if (url.length > 2048) return null
+  const project = /^(?:gitlab:|git@gitlab\.com:|(?:git\+)?(?:https|ssh|git):\/\/(?:git@)?gitlab\.com\/)([\w.-]+(?:\/[\w.-]+)+)\/?$/u
+    .exec(url.split('/-/')[0]!)?.[1]?.replace(/\.git$/u, '')
+  return project && !project.split('/').some(part => /^\.+$/u.test(part)) ? project : null
+}
+
+// The fields the Overview shows, in the shapes package.json gives them; each
+// left out where the document has none, or has another shape. A homepage that
+// leads only where its repository does (the repository, its directory there,
+// or its readme, npm's homepage where the package names none) is left out.
+export function npmManifest(json: Record<string, unknown>): Record<string, unknown> {
+  const pick: Record<string, unknown> = {}
+  for (const key of ['description', 'license', 'homepage', 'main', 'module', 'types', 'type', 'deprecated', 'gitHead']) {
+    if (typeof json[key] === 'string') pick[key] = json[key]
+  }
+  // Its author's name, and their npm account where it can be told: the
+  // publisher's or a maintainer's whose email or name is the author's. Emails
+  // only match people up; none is kept.
+  const author = npmPerson(json['author'])
+  if (author?.name) pick['author'] = author.name
+  const accounts = [json['_npmUser'], ...Array.isArray(json['maintainers']) ? json['maintainers'] : []].map(npmPerson)
+    .filter((person): person is NpmPerson & { name: string } => person?.name != null && NPM_ACCOUNT.test(person.name))
+  // The npm account that published it, for its profile; or, published by
+  // a CI npm trusts (npm's trusted publishing), the CI, as npm names it
+  // (`GitHub Actions`).
+  const publisher = npmPerson(json['_npmUser'])?.name
+  if (publisher && NPM_ACCOUNT.test(publisher)) pick['publisher'] = publisher
+  else if (publisher && publisher.length <= 64 && plainObject(json['_npmUser']) && plainObject(json['_npmUser']['trustedPublisher'])) pick['trustedPublisher'] = publisher
+  const same = (a: string | null, b: string | null) => a !== null && b !== null && a.toLowerCase() === b.toLowerCase()
+  const account = author && accounts.find(person => same(person.email, author.email) || same(person.name, author.name))
+  if (account) pick['authorAccount'] = account.name
+  // Which of those accounts are known to publish, for the Overview to tell
+  // them from the rest.
+  const known = [...new Set([publisher, account?.name])].filter(name => name && KNOWN_PUBLISHERS.has(name.toLowerCase()))
+  if (known.length > 0) pick['knownAccounts'] = known
+  if (Array.isArray(json['keywords'])) pick['keywords'] = json['keywords'].filter(item => typeof item === 'string').slice(0, 50)
+  if (typeof json['bin'] === 'string') pick['bin'] = { [String(json['name'])]: json['bin'] }
+  else if (stringRecord(json['bin'])) pick['bin'] = stringRecord(json['bin'])
+  for (const key of ['dependencies', 'peerDependencies', 'optionalDependencies', 'engines']) {
+    const record = stringRecord(json[key])
+    if (record && Object.keys(record).length > 0) pick[key] = record
+  }
+  if (json['hasInstallScript'] === true) pick['hasInstallScript'] = true
+  // Whether npm keeps a provenance statement for it (npm-provenance.ts).
+  if (plainObject(json['dist']) && plainObject(json['dist']['attestations']) && plainObject(json['dist']['attestations']['provenance'])) pick['provenance'] = true
+  // The scripts npm runs installing it from the registry; not `prepare`,
+  // which runs only for a local or git install.
+  const scripts = stringRecord(json['scripts'])
+  const install = scripts ? ['preinstall', 'install', 'postinstall'].filter(script => script in scripts) : []
+  if (install.length > 0) pick['installScripts'] = Object.fromEntries(install.map(script => [script, scripts![script]]))
+  // A GitLab project it names, as its provenance's source is matched with
+  // the repository it names, GitHub's below.
+  const gitlab = gitlabProject(json['repository'])
+  if (gitlab) pick['gitlab'] = gitlab
+  try {
+    const repo = getRepo(json)
+    if (repo.github) {
+      pick['github'] = { github: repo.github, ...(repo.directory === undefined ? {} : { directory: repo.directory }) }
+      const home = typeof pick['homepage'] === 'string' ? getRepo({ homepage: pick['homepage'] }) : {}
+      if (home.github?.toLowerCase() === repo.github.toLowerCase() && (home.directory ?? '') === (repo.directory ?? '')) delete pick['homepage']
+    }
+  } catch {}
+  return pick
+}
+
+// Of a version's document, the fields readNpmVersion and npmManifest read,
+// of `dist` and `scripts` only theirs, and of a provenance statement only that
+// there is one: what is kept of it (npm-kept.ts), so that both read a kept one
+// as they read npm's.
+const VERSION_FIELDS = ['name', 'version', 'description', 'license', 'homepage', 'main', 'module', 'types', 'type', 'deprecated', 'gitHead', 'author', '_npmUser',
+  'maintainers', 'keywords', 'bin', 'dependencies', 'peerDependencies', 'optionalDependencies', 'engines', 'hasInstallScript', 'repository', 'bugs']
+const DIST_FIELDS = ['tarball', 'integrity', 'unpackedSize', 'fileCount']
+const INSTALL_SCRIPTS = ['preinstall', 'install', 'postinstall']
+const picked = (json: Record<string, unknown>, keys: string[]) => Object.fromEntries(keys.filter(key => Object.hasOwn(json, key)).map(key => [key, json[key]]))
+
+export function npmVersionFields(json: Record<string, unknown>): Record<string, unknown> {
+  const dist = plainObject(json['dist']) ? json['dist'] : {}
+  const provenance = plainObject(dist['attestations']) && plainObject(dist['attestations']['provenance'])
+  const scripts = picked(stringRecord(json['scripts']) ?? {}, INSTALL_SCRIPTS)
+  return { ...picked(json, VERSION_FIELDS), dist: { ...picked(dist, DIST_FIELDS), ...provenance && { attestations: { provenance: {} } } },
+    ...Object.keys(scripts).length > 0 && { scripts } }
+}