@preventive/triage 1.0.0-alpha.41 → 1.0.0-alpha.43

package/server-managed/npm-risk.ts

@@ -9,8 +9,13 @@
 // - `risk/npm/versions`, what a version's document says, which npm never
 //   changes once published: a public package's in the database's upstream
 //   cache table, shared by every instance; a private one's in memory.
-// - `risk/npm/packages`, `risk/npm/public`, `risk/npm/advisories` and
-//   `risk/npm/downloads`, kept an hour or a day by risk itself: in memory.
+// - `risk/npm/downloads`, a package's downloads a day, which risk tells as
+//   kept for a week: a public package's in the database too, so that another
+//   instance, or this one restarted, asks npm's downloads API, which answers
+//   some 40 lookups a minute, none of them again that week, each removed a
+//   day past it (npm-kept.ts pruneKept); a private one's in memory.
+// - `risk/npm/packages`, `risk/npm/public` and `risk/npm/advisories`, kept an
+//   hour or a day by risk itself: in memory.
 // - `npm/tarballs`: none asked, as tarballs are read as the viewer's loads
 //   read them, kept where they keep them (npm-loads.ts loadNpmTarball), and
 //   an assessment kept needs none.
@@ -28,11 +33,12 @@ import { Buffer } from 'node:buffer'
 import { createHash } from 'node:crypto'
 import { buffer } from 'node:stream/consumers'
 import type { CacheStore } from '@preventive/upstream/npm.js'
-import { DEFAULT_MINIMUM_RELEASE_AGE, type NpmAssessment, assessNpm } from '../risk/index.js'
+import { DEFAULT_MINIMUM_RELEASE_AGE, type NpmAssessment, assessNpm, npmReadyDownload, npmReadyDownloads } from '../risk/index.js'
 import { CacheMissError, type CacheStorage } from './cache-storage.ts'
 import { loadNpmTarball } from './npm-loads.ts'
-import { NpmPackageError, type NpmVersionDocument, readNpmPackument, readNpmVersion } from './npm-packages.ts'
+import { NpmPackageError, type NpmVersionDocument, npmTokenReads, readNpmPackument, readNpmVersion } from './npm-packages.ts'
 import { npmPackageScope } from '../common/managed/npm-packages.js'
+import { pruneKept } from './npm-kept.ts'
 import type { UpstreamCacheStore } from './upstream-cache.ts'
 
 const MEMORY_ENTRIES = 2000
@@ -63,6 +69,10 @@ function remember(key: string, value: unknown) {
 }
 
 const hashed = (type: string, key: string) => `risk/${createHash('sha256').update(`${type}\0${key}`).digest('hex')}`
+// What of a public package is kept in the database, and for how long its
+// downloads are: risk's week, and a day more.
+const IN_DATABASE = new Set(['risk/npm/versions', 'risk/npm/downloads'])
+const DOWNLOADS_KEPT_MS = 8 * 24 * 60 * 60_000
 
 export interface NpmRiskStores {
   db: UpstreamCacheStore
@@ -70,16 +80,38 @@ export interface NpmRiskStores {
   debug?: boolean | undefined
 }
 
+// A cache that reads and writes many of a type at once too, as risk's
+// readying of downloads asks (npmReadyDownloads): one read and one write of
+// the database each, not one a key, as a request's queries take turns on its
+// connection (db-neon.ts).
+export interface NpmRiskCache extends CacheStore {
+  readMany(type: string, keys: string[]): Promise<Map<string, unknown>>
+  writeMany(type: string, entries: [key: string, value: unknown][]): Promise<void>
+}
+
 // Risk's cache for one package: `isPublic` whether anyone can read it, so
 // that what is kept of it can be shared.
-export function npmRiskCache({ db, npmCache, debug }: NpmRiskStores, isPublic: boolean): CacheStore {
+export function npmRiskCache({ db, npmCache, debug }: NpmRiskStores, isPublic: boolean): NpmRiskCache {
   const warn = (what: string, error: unknown) => { if (debug) console.warn(`managed: risk cache ${what} failed:`, error) }
-  return {
+  const cache: NpmRiskCache = {
+    async readMany(type, keys) {
+      if (!isPublic || !IN_DATABASE.has(type)) return new Map(await Promise.all(keys.map(async key => [key, await cache.read(type, key)] as const)))
+      try {
+        const found = await db.getUpstreamCacheEntries(keys.map(key => `${type}/${key}`))
+        return new Map(keys.map(key => { const value = found.get(`${type}/${key}`); return [key, value === undefined ? null : JSON.parse(value)] }))
+      } catch (error) { warn('read', error); return new Map() }
+    },
+    async writeMany(type, entries) {
+      if (!isPublic || !IN_DATABASE.has(type)) { for (const [key, value] of entries) await cache.write(type, key, value); return }
+      const at = Date.now()
+      try { await db.setUpstreamCacheEntries(entries.map(([key, value]) => [`${type}/${key}`, JSON.stringify(value)]), at) } catch (error) { warn('write', error); return }
+      if (type === 'risk/npm/downloads') await pruneKept(db, `${type}/`, DOWNLOADS_KEPT_MS, at, error => { warn('prune', error) })
+    },
     async read(type, key) {
       if (type === 'npm/tarballs') return null
       try {
         if (isPublic && type === 'risk/npm/assessments' && npmCache) return JSON.parse((await buffer((await npmCache.open(hashed(type, key))).stream)).toString('utf8'))
-        if (isPublic && type === 'risk/npm/versions') { const value = await db.getUpstreamCacheEntry(`${type}/${key}`); return value === null ? null : JSON.parse(value) }
+        if (isPublic && IN_DATABASE.has(type)) { const value = await db.getUpstreamCacheEntry(`${type}/${key}`); return value === null ? null : JSON.parse(value) }
       } catch (error) { if (!(error instanceof CacheMissError)) warn('read', error); return null }
       return memory.get(`${isPublic ? '' : 'private\0'}${type}\0${key}`)?.value ?? null
     },
@@ -87,11 +119,17 @@ export function npmRiskCache({ db, npmCache, debug }: NpmRiskStores, isPublic: b
       if (type === 'npm/tarballs') return
       try {
         if (isPublic && type === 'risk/npm/assessments' && npmCache) { await npmCache.put(hashed(type, key), Buffer.from(JSON.stringify(value))); return }
-        if (isPublic && type === 'risk/npm/versions') { await db.setUpstreamCacheEntry(`${type}/${key}`, JSON.stringify(value), Date.now()); return }
+        if (isPublic && IN_DATABASE.has(type)) {
+          const at = Date.now()
+          await db.setUpstreamCacheEntry(`${type}/${key}`, JSON.stringify(value), at)
+          if (type === 'risk/npm/downloads') await pruneKept(db, `${type}/`, DOWNLOADS_KEPT_MS, at, error => { warn('prune', error) })
+          return
+        }
       } catch (error) { warn('write', error); return }
       remember(`${isPublic ? '' : 'private\0'}${type}\0${key}`, value)
     },
   }
+  return cache
 }
 
 // A version's tarball as the viewer's loads read it (npm-loads.ts), its bytes
@@ -160,8 +198,9 @@ async function bounded(spare: boolean, assess: () => Promise<NpmAssessment>): Pr
 // by npm, as a scan's (`spare`), and its advisories of malware, npm's and,
 // where anyone can read its package, OSV's, which alone has some of those
 // npm removed (nx's), a private package's name never sent to OSV. Its
-// package's document read anonymously, else with npm's token where the
-// reader may have it (`privileged`), and kept as a private package's is; no
+// package's document read anonymously, else with npm's token alone where
+// that reads more for this reader (`privileged`, npmTokenReads), and kept as
+// a private package's is; no
 // tarball read, nothing of the version being left. Spare, within
 // MAX_SPARE_ASSESSMENTS as a scan's others are. An NpmRiskError where
 // nothing tells of it.
@@ -171,7 +210,26 @@ export function npmRiskRemoved(name: string, version: string,
     const open = await readNpmPackument(name, false, signal)
     return assessNpm(name, version, { minimumReleaseAge: teamScoped ? 0 : DEFAULT_MINIMUM_RELEASE_AGE, popularity: false, downloads: false, dependencies: false,
       osv: open !== null, cache: npmRiskCache(stores, false), signal,
-      packument: (read, { authorized }) => read === name && open ? Promise.resolve(open) : readNpmPackument(read, privileged && authorized, signal),
+      packument: (read, { authorized }) => read === name
+        ? open || !npmTokenReads(name, privileged && authorized) ? Promise.resolve(open) : readNpmPackument(name, true, signal, { anonymous: false })
+        : readNpmPackument(read, privileged && authorized, signal),
       tarball: () => Promise.reject(new NpmPackageError(502, 'upstream-invalid')) })
   })
 }
+
+// What a scan readies before assessing its packages (repository-risk.ts):
+// their downloads, as risk asks them, kept where a public package's are
+// (npmRiskCache), so that their assessments wait on npm's downloads API for
+// none. At once, those kept and unscoped ones, in bulk, all public, as only a
+// scoped package can be private; each other one on its own
+// (npmReadyScanDownload), kept only where npm counts some, which it does of
+// public packages alone: a private one's none, as of one it has no such
+// package of, isn't kept where public ones are, for its assessment to ask
+// its own where private ones are.
+export const npmReadyScanDownloads = (names: string[], ready: (name: string) => void, stores: NpmRiskStores, signal: AbortSignal) =>
+  npmReadyDownloads(names, ready, { cache: npmRiskCache(stores, true), signal })
+export function npmReadyScanDownload(name: string, stores: NpmRiskStores, signal: AbortSignal) {
+  const cache = npmRiskCache(stores, true)
+  const counted = (value: unknown) => !(value && typeof value === 'object' && 'none' in value)
+  return npmReadyDownload(name, { cache: { read: cache.read, write: (type, key, value) => counted(value) ? cache.write(type, key, value) : Promise.resolve() }, signal })
+}

package/server-managed/pretty-print.ts

@@ -28,7 +28,7 @@ import { type NpmVersionDocument, npmFileText } from './npm-packages.ts'
 // Part of every kept copy's name; bumped when what the formatter writes for
 // the same file changes, or what its check lets through, so copies written
 // before are no longer read.
-const PRETTY_VERSION = 3
+const PRETTY_VERSION = 4
 // Formats running at once per process, each holding its file's syntax tree,
 // and those waiting their turn past them (a version's files are formatted one
 // after another, beside other readers'), any more refused.

package/server-managed/queue.ts

@@ -0,0 +1,23 @@
+// Items in the order they are added, each taken once by whichever reader asks
+// first: none once closed and all taken, or once `signal` aborts, but those
+// added by then where `drain`, none waited for.
+export interface Queue<T> { add(item: T): void; close(): void; take(): Promise<T | undefined> }
+export function queue<T>(signal: AbortSignal, { drain = false } = {}): Queue<T> {
+  const items: T[] = []
+  const waiting: (() => void)[] = []
+  let closed = false
+  const wake = () => { for (const resolve of waiting.splice(0)) resolve() }
+  signal.addEventListener('abort', wake, { once: true })
+  return {
+    add(item) { items.push(item); wake() },
+    close() { closed = true; wake() },
+    async take() {
+      for (;;) {
+        if (signal.aborted && !drain) return undefined
+        const item = items.shift()
+        if (item !== undefined || closed || signal.aborted) return item
+        await new Promise<void>(resolve => { waiting.push(resolve) })
+      }
+    },
+  }
+}

package/server-managed/report-sources.ts

@@ -29,7 +29,7 @@ function formatDirectory(bundleId: string, sha256: string, name: string) {
 }
 function sourceCacheFilename(report: ReportRecord, bundle: ManagedBundle, permissions: ViewerPermissions, repo: { github: string | null }, sourcePaths?: Set<string>) {
   const key = createHash('sha256').update(JSON.stringify([
-    'finding-access-v9', sourcePaths ? [...sourcePaths].toSorted() : null, bundle.integrity, bundle.kind, permissions.dependencies, permissions.security, repo,
+    'finding-access-v10', sourcePaths ? [...sourcePaths].toSorted() : null, bundle.integrity, bundle.kind, permissions.dependencies, permissions.security, repo,
   ])).digest('hex')
   return `${formatDirectory(bundle.id, report.sha256, report.filename)}/${key}.json.gz`
 }

package/server-managed/repository-risk-assess.ts

@@ -0,0 +1,105 @@
+// A package's risk line, as a repository risk scan tells it
+// (repository-risk.ts): its version's document read and assessed, as risk
+// tells one npm removed where npm has none; and how a failure is told.
+import { setTimeout as delay } from 'node:timers/promises'
+import { NpmRiskError, SEVERITIES, type Severity } from '../risk/index.js'
+import { GithubApiError } from './github-errors.ts'
+import { NpmPackageError, type NpmVersionDocument } from './npm-packages.ts'
+import type { RepositoryRiskLine } from './repository-risk.ts'
+
+// An assessment refused as busy (`npm-busy`, by npm-risk.ts or the npm reads
+// beneath it), or that npm asked to ask later (risk's `upstream-busy`, as its
+// downloads API answers past a burst of requests), is asked again after a
+// wait, up to this many times.
+export const BUSY_RETRIES = 20
+export const BUSY_WAIT_MS = 1000
+
+// How long a package's risk line, or a lockfile's advisories, is told as it
+// is: after it, a risk line is told stale and assessed again, as risk tells
+// again what of a package changes in time, and advisories asked again, as npm
+// publishes them (repository-risk.ts).
+export const RISK_KEPT_MS = 60 * 60_000
+
+// The most severe of `findings`, or null for none.
+export function highestSeverity(findings: readonly { severity: Severity }[]): Severity | null {
+  return SEVERITIES.find(severity => findings.some(finding => finding.severity === severity)) ?? null
+}
+
+// Why a step failed, as a line says it: an error's code, and for a lockfile
+// refused, what of it its reader refused.
+export type RiskFailure = { error: string; detail?: string }
+type RiskLine = Extract<RepositoryRiskLine, { type: 'risk' }>
+
+// What an error is known by: a GitHub or npm failure's code, or `failed`;
+// GitHub not reached with what failed (githubUnreachable), while `doing` what
+// of a repository, and risk's with what it asked of npm and how that failed,
+// where they say.
+export function failureOf(error: unknown, doing?: string): RiskFailure {
+  if (error instanceof GithubApiError && error.detail) return { error: error.message, detail: doing ? `${doing}: ${error.detail}` : error.detail }
+  if (error instanceof GithubApiError || error instanceof NpmPackageError) return { error: error.message }
+  if (!(error instanceof NpmRiskError)) return { error: 'failed' }
+  return error.message === error.code ? { error: error.code } : { error: error.code, detail: error.message }
+}
+
+export const isBusy = (error: unknown) => (error instanceof NpmPackageError && error.status === 429) || (error instanceof NpmRiskError && error.code === 'upstream-busy')
+
+// A package to assess: npm's of its name and version, or, with an
+// `integrity`, a mirror's held to it.
+export interface AssessedPackage { name: string; version: string; integrity?: string }
+
+export type Assessment = { published: string | null; findings: { id: string; severity: Severity; takenDown?: unknown }[]; score: { score: number } }
+
+// What of a scan's source assessing a package asks (RepositoryRiskSource).
+export interface AssessingSource {
+  version(name: string, version: string, signal: AbortSignal): Promise<NpmVersionDocument>
+  assess(doc: NpmVersionDocument, teamScoped: boolean, signal: AbortSignal): Promise<Assessment>
+  removed(name: string, version: string, teamScoped: boolean, signal: AbortSignal): Promise<Assessment>
+}
+
+// A package's risk line, its version's document kept and the assessment asked
+// again after a wait while the server is busy; null once the scan stops. A
+// version npm has no document of is told as risk tells one npm removed
+// (removedOf), its malware keeping whether its package was taken down.
+export async function assessPackage(source: AssessingSource, signal: AbortSignal, pkg: AssessedPackage, teamScoped: boolean): Promise<RiskLine | null> {
+  const at = { type: 'risk', ...pkg } as const
+  let doc: NpmVersionDocument | NpmPackageError | undefined
+  for (let attempt = 0; ; attempt++) {
+    try {
+      const read = doc ??= await documentOf(source, pkg, signal)
+      if (!(read instanceof NpmPackageError) && pkg.integrity !== undefined && !pkg.integrity.split(/\s+/u).includes(read.dist.integrity)) return { ...at, error: 'integrity-mismatch' }
+      const { published, findings, score } = read instanceof NpmPackageError ? await removedOf(source, pkg, teamScoped, signal, read) : await assessedOf(source, pkg, teamScoped, signal, read)
+      const told = findings.map(({ id, severity, takenDown }) => ({ id, severity, ...takenDown === true ? { takenDown: true as const } : {} }))
+      return { ...at, published, severity: highestSeverity(told), score: score.score, findings: told }
+    } catch (error) {
+      if (signal.aborted) return null
+      if (!isBusy(error) || attempt >= BUSY_RETRIES) return { ...at, ...failureOf(error) }
+      try { await delay(BUSY_WAIT_MS * Math.min(attempt + 1, 5), undefined, { signal }) } catch { return null }
+    }
+  }
+}
+
+// npm's document of a version, or its answer that it has none (a 404).
+async function documentOf(source: AssessingSource, pkg: AssessedPackage, signal: AbortSignal): Promise<NpmVersionDocument | NpmPackageError> {
+  try { return await source.version(pkg.name, pkg.version, signal) } catch (error) {
+    if (error instanceof NpmPackageError && error.status === 404) return error
+    throw error
+  }
+}
+
+// A version's assessment; where risk finds npm no longer has it, as a document
+// read from where versions' are kept doesn't tell, as one npm removed.
+async function assessedOf(source: AssessingSource, pkg: AssessedPackage, teamScoped: boolean, signal: AbortSignal, doc: NpmVersionDocument): Promise<Assessment> {
+  try { return await source.assess(doc, teamScoped, signal) } catch (error) {
+    if (!(error instanceof NpmRiskError && error.code === 'version-not-found')) throw error
+    return removedOf(source, pkg, teamScoped, signal, new NpmPackageError(404, 'package-not-found'))
+  }
+}
+
+// A version npm has no document of, as risk tells one npm removed, as it
+// removes every one it finds malware in (`removed`): `gone`, npm's answer of
+// none, thrown where nothing tells of it.
+async function removedOf(source: AssessingSource, pkg: AssessedPackage, teamScoped: boolean, signal: AbortSignal, gone: NpmPackageError): Promise<Assessment> {
+  try { return await source.removed(pkg.name, pkg.version, teamScoped, signal) } catch (error) {
+    throw error instanceof NpmRiskError && (error.code === 'version-not-found' || error.code === 'package-not-found') ? gone : error
+  }
+}

package/server-managed/repository-risk-audit.ts

@@ -0,0 +1,66 @@
+// A lockfile's advisories, as a repository risk scan asks them
+// (repository-risk.ts): npm's registry's of its packages of the registry or
+// a mirror, asked at once, as `npm audit` asks, each told with whether it is
+// of malware (risk's npmMalwareAdvisory); kept by the lockfile's blob, in the
+// instance's memory and the source's `audits`, for RISK_KEPT_MS, as npm
+// publishes advisories, or why they couldn't be had for AUDIT_FAILED_KEPT_MS.
+import type { Advisory } from '@preventive/upstream/advisories.js'
+import { isExactVersion } from '@preventive/upstream/semver.js'
+import { npmMalwareAdvisory } from '../risk/index.js'
+import { isNpmPackageName } from '../common/managed/npm-packages.js'
+import type { LockedPackage } from './locked-packages.ts'
+import type { Lru } from './lru.ts'
+import type { KeptStore } from './npm-kept.ts'
+import { RISK_KEPT_MS, type RiskFailure, failureOf } from './repository-risk-assess.ts'
+
+export type AdvisoryRow = Advisory & { malware: boolean }
+// A lockfile's advisories, or why they couldn't be had.
+export type Audit = AdvisoryRow[] | RiskFailure
+
+export interface AuditSource {
+  // npm's registry's advisories of these packages' versions, asked at once.
+  advisories(packages: { name: string; versions: string[] }[], signal: AbortSignal): Promise<Advisory[]>
+  // Where audits are kept beyond this instance, by the lockfile's blob.
+  audits?: KeptStore<Audit>
+}
+
+// How long a lockfile's advisories that couldn't be had are told so, rather
+// than asked again: a scan's next parts don't wait on an audit that failed,
+// as by timing out, again and again.
+const AUDIT_FAILED_KEPT_MS = 10 * 60_000
+
+// Whether an audit kept `at` is told as it is: its advisories within
+// RISK_KEPT_MS, why they couldn't be had within AUDIT_FAILED_KEPT_MS.
+const isFresh = ({ at, audit }: { at: number; audit: Audit }) => Date.now() - at < (Array.isArray(audit) ? RISK_KEPT_MS : AUDIT_FAILED_KEPT_MS)
+
+// The audit of a lockfile's `packages`, by its `key` as kept (`memory`, then
+// the source's), else asked and kept; null where it locks none to ask of, or
+// once `signal` aborts.
+export async function auditLockfile(source: AuditSource, memory: Lru<{ at: number; audit: Audit }>, signal: AbortSignal, key: string,
+  packages: readonly LockedPackage[]): Promise<Audit | null> {
+  let kept = memory.get(key)
+  if (!kept || !isFresh(kept)) {
+    const stored = await source.audits?.read(key)
+    kept = stored ? { at: stored.at, audit: stored.value } : undefined
+  }
+  if (kept && isFresh(kept)) { memory.set(key, kept); return kept.audit }
+  const asked = new Map<string, Set<string>>()
+  for (const { name, version, source: from } of packages) {
+    if ((from !== 'registry' && from !== 'mirror') || !isNpmPackageName(name) || !isExactVersion(version)) continue
+    if (!asked.has(name)) asked.set(name, new Set())
+    asked.get(name)!.add(version)
+  }
+  if (asked.size === 0) return null
+  let audit: Audit
+  try {
+    audit = (await source.advisories([...asked].map(([name, versions]) => ({ name, versions: [...versions] })), signal))
+      .map(advisory => ({ ...advisory, malware: npmMalwareAdvisory(advisory) }))
+  } catch (error) {
+    if (signal.aborted) return null
+    audit = failureOf(error)
+  }
+  const now = Date.now()
+  memory.set(key, { at: now, audit })
+  await source.audits?.write(key, audit, now)
+  return audit
+}