@preventive/triage 1.0.0-alpha.42 → 1.0.0-alpha.44

package/server-managed/npm-packages.ts

@@ -43,7 +43,7 @@ export class NpmPackageError extends Error {
 }
 
 // Who reads, as the session has it now: the role, and the scopes of the
-// visible teams they are a member of.
+// visible teams they are a member of, where those count (npmScopesCount).
 export interface NpmReader { role: Role; scopes: ReadonlySet<string>; userId: string }
 
 // Whether a reader may have a package read with the server's token.
@@ -53,8 +53,19 @@ export function canReadPrivateNpm(reader: NpmReader, name: string): boolean {
   return scope !== null && reader.scopes.has(scope)
 }
 
+// Whether a reader's team scopes count for any of `names`, as
+// canReadPrivateNpm reads them: a scoped one, read by someone below manager.
+export const npmScopesCount = (role: Role, names: readonly string[]): boolean =>
+  role !== 'admin' && role !== 'manage' && names.some(name => npmPackageScope(name) !== null)
+
 export const npmToken = () => process.env['NPM_TOKEN'] || null
 
+// Whether a read of `name` that npm answers none anonymously is asked again
+// with the server's token: by a reader with private access (`privileged`),
+// where there is a token and the package is scoped, as npm's private packages
+// always are. Else the anonymous answer is the reader's.
+export const npmTokenReads = (name: string, privileged: boolean): boolean => privileged && npmToken() !== null && npmPackageScope(name) !== null
+
 function registryUrl(name: string, spec?: string): string {
   return `${NPM_REGISTRY}/${name}${spec === undefined ? '' : `/${encodeURIComponent(spec)}`}`
 }
@@ -138,16 +149,17 @@ async function publicWholeDocument(name: string, url: string, { limit, signal }:
   return json
 }
 
-// Asked anonymously first: a reader without private access stops there. One
-// with it asks again with the token, where there is one and the package is
-// scoped (npm's private packages always are), and the answer is private. A
-// whole document (`whole`) anyone may read is kept (publicWholeDocument).
-async function readDocument(name: string, url: string, privileged: boolean, { whole = false, ...options }: { accept: string; limit: number; signal: AbortSignal; whole?: boolean }) {
-  const open = whole ? await publicWholeDocument(name, url, options) : await registryDocument(url, { ...options, token: null })
+// Asked anonymously first, unless its caller knows npm has none for anyone
+// (`anonymous` false): a reader without private access stops there. One
+// with it asks again with the token where npmTokenReads says so, and the
+// answer is private. A whole document (`whole`) anyone may read is kept
+// (publicWholeDocument).
+async function readDocument(name: string, url: string, privileged: boolean,
+  { whole = false, anonymous = true, ...options }: { accept: string; limit: number; signal: AbortSignal; whole?: boolean; anonymous?: boolean }) {
+  const open = anonymous ? whole ? await publicWholeDocument(name, url, options) : await registryDocument(url, { ...options, token: null }) : null
   if (open) return { json: open, private: false }
-  const token = npmToken()
-  if (!privileged || !token || npmPackageScope(name) === null) return null
-  const json = await registryDocument(url, { ...options, token })
+  if (!npmTokenReads(name, privileged)) return null
+  const json = await registryDocument(url, { ...options, token: npmToken() })
   return json ? { json, private: true } : null
 }
 
@@ -244,10 +256,12 @@ export function readNpmVersions(name: string, privileged: boolean, signal: Abort
 // A package's whole document as npm has it for this reader, for risk
 // (npm-risk.ts), within the bounds and budget every document read here is
 // held to; null where npm has none for this reader, or the name is none npm
-// takes.
-export function readNpmPackument(name: string, privileged: boolean, signal: AbortSignal): Promise<Record<string, unknown> | null> {
+// takes. With `anonymous` false, where npm is known to have none for anyone,
+// it is asked only with the token (readDocument).
+export function readNpmPackument(name: string, privileged: boolean, signal: AbortSignal, { anonymous = true } = {}): Promise<Record<string, unknown> | null> {
   if (!isNpmPackageName(name)) return Promise.resolve(null)
-  return withDocumentBudget(PACKUMENT_BYTES, async () => (await readDocument(name, registryUrl(name), privileged, { accept: 'application/json', limit: PACKUMENT_BYTES, signal, whole: true }))?.json ?? null)
+  return withDocumentBudget(PACKUMENT_BYTES, async () => (await readDocument(name, registryUrl(name), privileged,
+    { accept: 'application/json', limit: PACKUMENT_BYTES, signal, whole: true, anonymous }))?.json ?? null)
 }
 
 const isPublishTime = (value: string | undefined) => typeof value === 'string' && value.length <= 40 && !Number.isNaN(Date.parse(value))

package/server-managed/npm-risk.ts

@@ -9,8 +9,13 @@
 // - `risk/npm/versions`, what a version's document says, which npm never
 //   changes once published: a public package's in the database's upstream
 //   cache table, shared by every instance; a private one's in memory.
-// - `risk/npm/packages`, `risk/npm/public`, `risk/npm/advisories` and
-//   `risk/npm/downloads`, kept an hour or a day by risk itself: in memory.
+// - `risk/npm/downloads`, a package's downloads a day, which risk tells as
+//   kept for a week: a public package's in the database too, so that another
+//   instance, or this one restarted, asks npm's downloads API, which answers
+//   some 40 lookups a minute, none of them again that week, each removed a
+//   day past it (npm-kept.ts pruneKept); a private one's in memory.
+// - `risk/npm/packages`, `risk/npm/public` and `risk/npm/advisories`, kept an
+//   hour or a day by risk itself: in memory.
 // - `npm/tarballs`: none asked, as tarballs are read as the viewer's loads
 //   read them, kept where they keep them (npm-loads.ts loadNpmTarball), and
 //   an assessment kept needs none.
@@ -28,11 +33,12 @@ import { Buffer } from 'node:buffer'
 import { createHash } from 'node:crypto'
 import { buffer } from 'node:stream/consumers'
 import type { CacheStore } from '@preventive/upstream/npm.js'
-import { DEFAULT_MINIMUM_RELEASE_AGE, type NpmAssessment, assessNpm } from '../risk/index.js'
+import { DEFAULT_MINIMUM_RELEASE_AGE, type NpmAssessment, assessNpm, npmReadyDownload, npmReadyDownloads } from '../risk/index.js'
 import { CacheMissError, type CacheStorage } from './cache-storage.ts'
 import { loadNpmTarball } from './npm-loads.ts'
-import { NpmPackageError, type NpmVersionDocument, readNpmPackument, readNpmVersion } from './npm-packages.ts'
+import { NpmPackageError, type NpmVersionDocument, npmTokenReads, readNpmPackument, readNpmVersion } from './npm-packages.ts'
 import { npmPackageScope } from '../common/managed/npm-packages.js'
+import { pruneKept } from './npm-kept.ts'
 import type { UpstreamCacheStore } from './upstream-cache.ts'
 
 const MEMORY_ENTRIES = 2000
@@ -63,6 +69,10 @@ function remember(key: string, value: unknown) {
 }
 
 const hashed = (type: string, key: string) => `risk/${createHash('sha256').update(`${type}\0${key}`).digest('hex')}`
+// What of a public package is kept in the database, and for how long its
+// downloads are: risk's week, and a day more.
+const IN_DATABASE = new Set(['risk/npm/versions', 'risk/npm/downloads'])
+const DOWNLOADS_KEPT_MS = 8 * 24 * 60 * 60_000
 
 export interface NpmRiskStores {
   db: UpstreamCacheStore
@@ -70,16 +80,38 @@ export interface NpmRiskStores {
   debug?: boolean | undefined
 }
 
+// A cache that reads and writes many of a type at once too, as risk's
+// readying of downloads asks (npmReadyDownloads): one read and one write of
+// the database each, not one a key, as a request's queries take turns on its
+// connection (db-neon.ts).
+export interface NpmRiskCache extends CacheStore {
+  readMany(type: string, keys: string[]): Promise<Map<string, unknown>>
+  writeMany(type: string, entries: [key: string, value: unknown][]): Promise<void>
+}
+
 // Risk's cache for one package: `isPublic` whether anyone can read it, so
 // that what is kept of it can be shared.
-export function npmRiskCache({ db, npmCache, debug }: NpmRiskStores, isPublic: boolean): CacheStore {
+export function npmRiskCache({ db, npmCache, debug }: NpmRiskStores, isPublic: boolean): NpmRiskCache {
   const warn = (what: string, error: unknown) => { if (debug) console.warn(`managed: risk cache ${what} failed:`, error) }
-  return {
+  const cache: NpmRiskCache = {
+    async readMany(type, keys) {
+      if (!isPublic || !IN_DATABASE.has(type)) return new Map(await Promise.all(keys.map(async key => [key, await cache.read(type, key)] as const)))
+      try {
+        const found = await db.getUpstreamCacheEntries(keys.map(key => `${type}/${key}`))
+        return new Map(keys.map(key => { const value = found.get(`${type}/${key}`); return [key, value === undefined ? null : JSON.parse(value)] }))
+      } catch (error) { warn('read', error); return new Map() }
+    },
+    async writeMany(type, entries) {
+      if (!isPublic || !IN_DATABASE.has(type)) { for (const [key, value] of entries) await cache.write(type, key, value); return }
+      const at = Date.now()
+      try { await db.setUpstreamCacheEntries(entries.map(([key, value]) => [`${type}/${key}`, JSON.stringify(value)]), at) } catch (error) { warn('write', error); return }
+      if (type === 'risk/npm/downloads') await pruneKept(db, `${type}/`, DOWNLOADS_KEPT_MS, at, error => { warn('prune', error) })
+    },
     async read(type, key) {
       if (type === 'npm/tarballs') return null
       try {
         if (isPublic && type === 'risk/npm/assessments' && npmCache) return JSON.parse((await buffer((await npmCache.open(hashed(type, key))).stream)).toString('utf8'))
-        if (isPublic && type === 'risk/npm/versions') { const value = await db.getUpstreamCacheEntry(`${type}/${key}`); return value === null ? null : JSON.parse(value) }
+        if (isPublic && IN_DATABASE.has(type)) { const value = await db.getUpstreamCacheEntry(`${type}/${key}`); return value === null ? null : JSON.parse(value) }
       } catch (error) { if (!(error instanceof CacheMissError)) warn('read', error); return null }
       return memory.get(`${isPublic ? '' : 'private\0'}${type}\0${key}`)?.value ?? null
     },
@@ -87,11 +119,17 @@ export function npmRiskCache({ db, npmCache, debug }: NpmRiskStores, isPublic: b
       if (type === 'npm/tarballs') return
       try {
         if (isPublic && type === 'risk/npm/assessments' && npmCache) { await npmCache.put(hashed(type, key), Buffer.from(JSON.stringify(value))); return }
-        if (isPublic && type === 'risk/npm/versions') { await db.setUpstreamCacheEntry(`${type}/${key}`, JSON.stringify(value), Date.now()); return }
+        if (isPublic && IN_DATABASE.has(type)) {
+          const at = Date.now()
+          await db.setUpstreamCacheEntry(`${type}/${key}`, JSON.stringify(value), at)
+          if (type === 'risk/npm/downloads') await pruneKept(db, `${type}/`, DOWNLOADS_KEPT_MS, at, error => { warn('prune', error) })
+          return
+        }
       } catch (error) { warn('write', error); return }
       remember(`${isPublic ? '' : 'private\0'}${type}\0${key}`, value)
     },
   }
+  return cache
 }
 
 // A version's tarball as the viewer's loads read it (npm-loads.ts), its bytes
@@ -160,8 +198,9 @@ async function bounded(spare: boolean, assess: () => Promise<NpmAssessment>): Pr
 // by npm, as a scan's (`spare`), and its advisories of malware, npm's and,
 // where anyone can read its package, OSV's, which alone has some of those
 // npm removed (nx's), a private package's name never sent to OSV. Its
-// package's document read anonymously, else with npm's token where the
-// reader may have it (`privileged`), and kept as a private package's is; no
+// package's document read anonymously, else with npm's token alone where
+// that reads more for this reader (`privileged`, npmTokenReads), and kept as
+// a private package's is; no
 // tarball read, nothing of the version being left. Spare, within
 // MAX_SPARE_ASSESSMENTS as a scan's others are. An NpmRiskError where
 // nothing tells of it.
@@ -171,7 +210,26 @@ export function npmRiskRemoved(name: string, version: string,
     const open = await readNpmPackument(name, false, signal)
     return assessNpm(name, version, { minimumReleaseAge: teamScoped ? 0 : DEFAULT_MINIMUM_RELEASE_AGE, popularity: false, downloads: false, dependencies: false,
       osv: open !== null, cache: npmRiskCache(stores, false), signal,
-      packument: (read, { authorized }) => read === name && open ? Promise.resolve(open) : readNpmPackument(read, privileged && authorized, signal),
+      packument: (read, { authorized }) => read === name
+        ? open || !npmTokenReads(name, privileged && authorized) ? Promise.resolve(open) : readNpmPackument(name, true, signal, { anonymous: false })
+        : readNpmPackument(read, privileged && authorized, signal),
       tarball: () => Promise.reject(new NpmPackageError(502, 'upstream-invalid')) })
   })
 }
+
+// What a scan readies before assessing its packages (repository-risk.ts):
+// their downloads, as risk asks them, kept where a public package's are
+// (npmRiskCache), so that their assessments wait on npm's downloads API for
+// none. At once, those kept and unscoped ones, in bulk, all public, as only a
+// scoped package can be private; each other one on its own
+// (npmReadyScanDownload), kept only where npm counts some, which it does of
+// public packages alone: a private one's none, as of one it has no such
+// package of, isn't kept where public ones are, for its assessment to ask
+// its own where private ones are.
+export const npmReadyScanDownloads = (names: string[], ready: (name: string) => void, stores: NpmRiskStores, signal: AbortSignal) =>
+  npmReadyDownloads(names, ready, { cache: npmRiskCache(stores, true), signal })
+export function npmReadyScanDownload(name: string, stores: NpmRiskStores, signal: AbortSignal) {
+  const cache = npmRiskCache(stores, true)
+  const counted = (value: unknown) => !(value && typeof value === 'object' && 'none' in value)
+  return npmReadyDownload(name, { cache: { read: cache.read, write: (type, key, value) => counted(value) ? cache.write(type, key, value) : Promise.resolve() }, signal })
+}

package/server-managed/pretty-print.ts

@@ -250,18 +250,21 @@ export function stalePrettyRemover(npm: CacheStorage): (signal?: AbortSignal) =>
   }
 }
 
+// A bundle's details, from its bytes in `store`: bundles are read one at a
+// time, as reading one decodes all of it.
+function detailsReader(store: BundleStore) {
+  const enqueue = serial()
+  return (record: ManagedBundle) => enqueue(async () => {
+    const details = await readBundleDetails(record, store)
+    if (!details) throw new PrettyError(422, 'bundle-unavailable')
+    return details
+  })
+}
+
 export function createPrettyCache(bundles: BundleCacheStorage, npm: CacheStorage, db: Pick<ManagedDb, 'getBundle'>, store: BundleStore) {
   // Copies being made; readers asking for one meanwhile share it.
   const shared = sharing<Buffer>()
-  // Bundles are read one at a time: reading one decodes all of it.
-  const enqueue = serial()
-  function bundleDetails(record: ManagedBundle) {
-    return enqueue(async () => {
-      const details = await readBundleDetails(record, store)
-      if (!details) throw new PrettyError(422, 'bundle-unavailable')
-      return details
-    })
-  }
+  const bundleDetails = detailsReader(store)
   // A bundle's file pretty-printed from the text `read` reads, and kept in
   // its cache.
   function bundleCopy(record: ManagedBundle, request: PrettyRequest, read: () => Promise<string>): Promise<Buffer> {
@@ -341,6 +344,8 @@ export function createPrettyCache(bundles: BundleCacheStorage, npm: CacheStorage
         made: async (request, text) => opened(await bundleCopy(record, request, () => Promise.resolve(text))) })
       return [await sideOf(base, 'base'), await sideOf(other, 'other')]
     },
+    // A bundle's details, read in its turn, as Compare pairs them (comparison.ts).
+    bundleDetails,
     async npmSides(base: NpmVersionDocument, other: NpmVersionDocument): Promise<[ComparedSide, ComparedSide]> {
       const sideOf = async (doc: NpmVersionDocument, side: Side): Promise<ComparedSide> => ({ side, ...await sideFiles(await npmFiles(doc)), packageDirs: null,
         kept: request => (doc.private ? Promise.resolve(null) : kept(() => npm.open(prettyFile(request)))),

package/server-managed/public-workspace.ts

@@ -17,7 +17,8 @@ import { serveTeamFeed } from './team-feed.ts'
 import { type ComparedSide, PrettyError, compared, prettyRequest, prettyText } from './pretty-print.ts'
 import { renameDiffOf, renameDiffRequest, renamedPairs } from './rename-diff.ts'
 import { typeOnlyFiles } from './type-only.ts'
-import { sharedTeamApp } from './team-app.ts'
+import { comparisonBody } from './comparison.ts'
+import { sharedTeamApp, withWorkspaceSummary } from './team-app.ts'
 
 function json(res: ServerResponse, status: number, body: unknown): void {
   sendJson(res, status, body, { 'cache-control': 'private, no-store', 'referrer-policy': 'no-referrer' })
@@ -35,7 +36,7 @@ export async function handlePublicWorkspace(req: IncomingMessage, res: ServerRes
   const shareRoute = /^\/api\/shares\/([A-Za-z0-9_-]+)\/workspace$/u.exec(url.pathname)
   const teamRoute = /^\/api\/teams\/([^/]+)\/(shared|reports|feed|annotations)$/u.exec(url.pathname)
   const reportRoute = /^\/api\/reports\/([^/]+)\/(triage|comments|sources)$/u.exec(url.pathname)
-  const bundleRoute = /^\/api\/bundles\/([^/]+)\/(metadata|contents|download|advisories|pretty|rename-diff|type-only)$/u.exec(url.pathname)
+  const bundleRoute = /^\/api\/bundles\/([^/]+)\/(metadata|contents|download|advisories|pretty|rename-diff|type-only|comparison)$/u.exec(url.pathname)
   if (!shareRoute && !teamRoute && !reportRoute && !bundleRoute) { json(res, 403, { error: 'share-scope-required' }); return }
   const tokenHash = hashToken(token)
   const snapshot = await deps.db.getWorkspaceShare(tokenHash)
@@ -46,8 +47,8 @@ export async function handlePublicWorkspace(req: IncomingMessage, res: ServerRes
   }
   const send = async (body: unknown) => { await recheck(); json(res, 200, body) }
   const sendTeam = async () => {
-    const [summaries, app] = await Promise.all([bundleSummaries(snapshot.team.bundles, deps.bundleCache), sharedTeamApp(deps.db, deps.reportStore, snapshot)])
-    await send({ user: snapshot.user, team: { ...snapshot.team, app,
+    const [summaries, workspace] = await Promise.all([bundleSummaries(snapshot.team.bundles, deps.bundleCache), sharedTeamApp(deps.db, deps.reportStore, snapshot)])
+    await send({ user: snapshot.user, team: { ...withWorkspaceSummary(snapshot.team, workspace),
       bundles: snapshot.team.bundles.map(bundle => ({ ...bundle, ...(summaries.get(bundle.integrity) ?? { summary: null, summaryRetryAt: null }) })),
     } })
     await backfillBundleSummaries(snapshot.team.bundles, deps.bundleCache)
@@ -101,7 +102,7 @@ export async function handlePublicWorkspace(req: IncomingMessage, res: ServerRes
   const bundle = snapshot.bundles.find(item => item.id === bundleRoute![1])
   if (!bundle) { json(res, 404, { error: 'no-bundle' }); return }
   if (bundleRoute![2] === 'advisories' && !snapshot.permissions.security) { json(res, 403, { error: 'security-access-required' }); return }
-  if (bundleRoute![2] === 'rename-diff' || bundleRoute![2] === 'type-only' || (bundleRoute![2] === 'pretty' && url.searchParams.has('other'))) {
+  if (bundleRoute![2] === 'rename-diff' || bundleRoute![2] === 'type-only' || bundleRoute![2] === 'comparison' || (bundleRoute![2] === 'pretty' && url.searchParams.has('other'))) {
     if (method !== 'GET') { json(res, 405, { error: 'method-not-allowed' }); return }
     await serveCompared(res, deps, bundle, snapshot.bundles.find(item => item.id === url.searchParams.get('other')), bundleRoute![2], url, recheck, stream); return
   }
@@ -109,11 +110,12 @@ export async function handlePublicWorkspace(req: IncomingMessage, res: ServerRes
 }
 
 // What a bundle is compared with `other`, another of the link's bundles, by:
-// one of their JavaScript files diffed with renamed names hidden, or the
-// files that changed between them, or the pairs of those, or those that
-// changed in types alone, a line each.
+// their files paired, as one comparison; one of their JavaScript files diffed
+// with renamed names hidden; or the files that changed between them, or the
+// pairs of those, or those that changed in types alone, a line each.
 function serveCompared(res: ServerResponse, deps: ManagedHttpDeps, bundle: ManagedBundle, other: ManagedBundle | undefined, part: string, url: URL,
   recheck: () => Promise<void>, stream: (stored: OpenedBlob, encoding: string | null, contentType?: string) => Promise<void>) {
+  if (part === 'comparison') return serveComparison(res, deps, bundle, other, url, stream)
   if (part === 'rename-diff' && url.searchParams.has('path')) return serveRenameDiff(res, deps, bundle, other, url, stream)
   return serveChanges(res, deps, bundle, other, recheck, part === 'pretty' ? compared : part === 'type-only' ? typeOnlyFiles : renamedPairs,
     part === 'pretty' ? '' : url.searchParams.get('scope') ?? '')
@@ -139,6 +141,22 @@ async function serveRenameDiff(res: ServerResponse, deps: ManagedHttpDeps, bundl
   await stream(diff, 'br')
 }
 
+// A bundle's files paired with `other`'s, another of the link's bundles, as
+// Compare pairs them (comparison.ts), within a reason `scope` where given.
+async function serveComparison(res: ServerResponse, deps: ManagedHttpDeps, bundle: ManagedBundle, other: ManagedBundle | undefined, url: URL,
+  stream: (stored: OpenedBlob, encoding: string | null, contentType?: string) => Promise<void>) {
+  if (!other) { json(res, 404, { error: 'no-bundle' }); return }
+  const { prettyCache } = deps
+  if (!prettyCache) { json(res, 503, { error: 'unavailable' }); return }
+  let body
+  try { body = await comparisonBody(await prettyCache.bundleDetails(bundle), await prettyCache.bundleDetails(other), url.searchParams.get('scope') ?? '') }
+  catch (err) {
+    if (err instanceof PrettyError) { json(res, err.status, { error: err.message }); return }
+    throw err
+  }
+  await stream(body, 'br')
+}
+
 // The files that changed between a bundle and `other`, another of the link's
 // bundles, both sides' pretty-printed (pretty-print.ts compared), or the pairs
 // of them its viewer diffs with renamed names hidden (rename-diff.ts

package/server-managed/queue.ts

@@ -0,0 +1,23 @@
+// Items in the order they are added, each taken once by whichever reader asks
+// first: none once closed and all taken, or once `signal` aborts, but those
+// added by then where `drain`, none waited for.
+export interface Queue<T> { add(item: T): void; close(): void; take(): Promise<T | undefined> }
+export function queue<T>(signal: AbortSignal, { drain = false } = {}): Queue<T> {
+  const items: T[] = []
+  const waiting: (() => void)[] = []
+  let closed = false
+  const wake = () => { for (const resolve of waiting.splice(0)) resolve() }
+  signal.addEventListener('abort', wake, { once: true })
+  return {
+    add(item) { items.push(item); wake() },
+    close() { closed = true; wake() },
+    async take() {
+      for (;;) {
+        if (signal.aborted && !drain) return undefined
+        const item = items.shift()
+        if (item !== undefined || closed || signal.aborted) return item
+        await new Promise<void>(resolve => { waiting.push(resolve) })
+      }
+    },
+  }
+}

package/server-managed/report-sources.ts

@@ -29,7 +29,7 @@ function formatDirectory(bundleId: string, sha256: string, name: string) {
 }
 function sourceCacheFilename(report: ReportRecord, bundle: ManagedBundle, permissions: ViewerPermissions, repo: { github: string | null }, sourcePaths?: Set<string>) {
   const key = createHash('sha256').update(JSON.stringify([
-    'finding-access-v9', sourcePaths ? [...sourcePaths].toSorted() : null, bundle.integrity, bundle.kind, permissions.dependencies, permissions.security, repo,
+    'finding-access-v10', sourcePaths ? [...sourcePaths].toSorted() : null, bundle.integrity, bundle.kind, permissions.dependencies, permissions.security, repo,
   ])).digest('hex')
   return `${formatDirectory(bundle.id, report.sha256, report.filename)}/${key}.json.gz`
 }

package/server-managed/repository-risk-assess.ts

@@ -0,0 +1,105 @@
+// A package's risk line, as a repository risk scan tells it
+// (repository-risk.ts): its version's document read and assessed, as risk
+// tells one npm removed where npm has none; and how a failure is told.
+import { setTimeout as delay } from 'node:timers/promises'
+import { NpmRiskError, SEVERITIES, type Severity } from '../risk/index.js'
+import { GithubApiError } from './github-errors.ts'
+import { NpmPackageError, type NpmVersionDocument } from './npm-packages.ts'
+import type { RepositoryRiskLine } from './repository-risk.ts'
+
+// An assessment refused as busy (`npm-busy`, by npm-risk.ts or the npm reads
+// beneath it), or that npm asked to ask later (risk's `upstream-busy`, as its
+// downloads API answers past a burst of requests), is asked again after a
+// wait, up to this many times.
+export const BUSY_RETRIES = 20
+export const BUSY_WAIT_MS = 1000
+
+// How long a package's risk line, or a lockfile's advisories, is told as it
+// is: after it, a risk line is told stale and assessed again, as risk tells
+// again what of a package changes in time, and advisories asked again, as npm
+// publishes them (repository-risk.ts).
+export const RISK_KEPT_MS = 60 * 60_000
+
+// The most severe of `findings`, or null for none.
+export function highestSeverity(findings: readonly { severity: Severity }[]): Severity | null {
+  return SEVERITIES.find(severity => findings.some(finding => finding.severity === severity)) ?? null
+}
+
+// Why a step failed, as a line says it: an error's code, and for a lockfile
+// refused, what of it its reader refused.
+export type RiskFailure = { error: string; detail?: string }
+type RiskLine = Extract<RepositoryRiskLine, { type: 'risk' }>
+
+// What an error is known by: a GitHub or npm failure's code, or `failed`;
+// GitHub not reached with what failed (githubUnreachable), while `doing` what
+// of a repository, and risk's with what it asked of npm and how that failed,
+// where they say.
+export function failureOf(error: unknown, doing?: string): RiskFailure {
+  if (error instanceof GithubApiError && error.detail) return { error: error.message, detail: doing ? `${doing}: ${error.detail}` : error.detail }
+  if (error instanceof GithubApiError || error instanceof NpmPackageError) return { error: error.message }
+  if (!(error instanceof NpmRiskError)) return { error: 'failed' }
+  return error.message === error.code ? { error: error.code } : { error: error.code, detail: error.message }
+}
+
+export const isBusy = (error: unknown) => (error instanceof NpmPackageError && error.status === 429) || (error instanceof NpmRiskError && error.code === 'upstream-busy')
+
+// A package to assess: npm's of its name and version, or, with an
+// `integrity`, a mirror's held to it.
+export interface AssessedPackage { name: string; version: string; integrity?: string }
+
+export type Assessment = { published: string | null; findings: { id: string; severity: Severity; takenDown?: unknown }[]; score: { score: number } }
+
+// What of a scan's source assessing a package asks (RepositoryRiskSource).
+export interface AssessingSource {
+  version(name: string, version: string, signal: AbortSignal): Promise<NpmVersionDocument>
+  assess(doc: NpmVersionDocument, teamScoped: boolean, signal: AbortSignal): Promise<Assessment>
+  removed(name: string, version: string, teamScoped: boolean, signal: AbortSignal): Promise<Assessment>
+}
+
+// A package's risk line, its version's document kept and the assessment asked
+// again after a wait while the server is busy; null once the scan stops. A
+// version npm has no document of is told as risk tells one npm removed
+// (removedOf), its malware keeping whether its package was taken down.
+export async function assessPackage(source: AssessingSource, signal: AbortSignal, pkg: AssessedPackage, teamScoped: boolean): Promise<RiskLine | null> {
+  const at = { type: 'risk', ...pkg } as const
+  let doc: NpmVersionDocument | NpmPackageError | undefined
+  for (let attempt = 0; ; attempt++) {
+    try {
+      const read = doc ??= await documentOf(source, pkg, signal)
+      if (!(read instanceof NpmPackageError) && pkg.integrity !== undefined && !pkg.integrity.split(/\s+/u).includes(read.dist.integrity)) return { ...at, error: 'integrity-mismatch' }
+      const { published, findings, score } = read instanceof NpmPackageError ? await removedOf(source, pkg, teamScoped, signal, read) : await assessedOf(source, pkg, teamScoped, signal, read)
+      const told = findings.map(({ id, severity, takenDown }) => ({ id, severity, ...takenDown === true ? { takenDown: true as const } : {} }))
+      return { ...at, published, severity: highestSeverity(told), score: score.score, findings: told }
+    } catch (error) {
+      if (signal.aborted) return null
+      if (!isBusy(error) || attempt >= BUSY_RETRIES) return { ...at, ...failureOf(error) }
+      try { await delay(BUSY_WAIT_MS * Math.min(attempt + 1, 5), undefined, { signal }) } catch { return null }
+    }
+  }
+}
+
+// npm's document of a version, or its answer that it has none (a 404).
+async function documentOf(source: AssessingSource, pkg: AssessedPackage, signal: AbortSignal): Promise<NpmVersionDocument | NpmPackageError> {
+  try { return await source.version(pkg.name, pkg.version, signal) } catch (error) {
+    if (error instanceof NpmPackageError && error.status === 404) return error
+    throw error
+  }
+}
+
+// A version's assessment; where risk finds npm no longer has it, as a document
+// read from where versions' are kept doesn't tell, as one npm removed.
+async function assessedOf(source: AssessingSource, pkg: AssessedPackage, teamScoped: boolean, signal: AbortSignal, doc: NpmVersionDocument): Promise<Assessment> {
+  try { return await source.assess(doc, teamScoped, signal) } catch (error) {
+    if (!(error instanceof NpmRiskError && error.code === 'version-not-found')) throw error
+    return removedOf(source, pkg, teamScoped, signal, new NpmPackageError(404, 'package-not-found'))
+  }
+}
+
+// A version npm has no document of, as risk tells one npm removed, as it
+// removes every one it finds malware in (`removed`): `gone`, npm's answer of
+// none, thrown where nothing tells of it.
+async function removedOf(source: AssessingSource, pkg: AssessedPackage, teamScoped: boolean, signal: AbortSignal, gone: NpmPackageError): Promise<Assessment> {
+  try { return await source.removed(pkg.name, pkg.version, teamScoped, signal) } catch (error) {
+    throw error instanceof NpmRiskError && (error.code === 'version-not-found' || error.code === 'package-not-found') ? gone : error
+  }
+}

package/server-managed/repository-risk-audit.ts

@@ -0,0 +1,66 @@
+// A lockfile's advisories, as a repository risk scan asks them
+// (repository-risk.ts): npm's registry's of its packages of the registry or
+// a mirror, asked at once, as `npm audit` asks, each told with whether it is
+// of malware (risk's npmMalwareAdvisory); kept by the lockfile's blob, in the
+// instance's memory and the source's `audits`, for RISK_KEPT_MS, as npm
+// publishes advisories, or why they couldn't be had for AUDIT_FAILED_KEPT_MS.
+import type { Advisory } from '@preventive/upstream/advisories.js'
+import { isExactVersion } from '@preventive/upstream/semver.js'
+import { npmMalwareAdvisory } from '../risk/index.js'
+import { isNpmPackageName } from '../common/managed/npm-packages.js'
+import type { LockedPackage } from './locked-packages.ts'
+import type { Lru } from './lru.ts'
+import type { KeptStore } from './npm-kept.ts'
+import { RISK_KEPT_MS, type RiskFailure, failureOf } from './repository-risk-assess.ts'
+
+export type AdvisoryRow = Advisory & { malware: boolean }
+// A lockfile's advisories, or why they couldn't be had.
+export type Audit = AdvisoryRow[] | RiskFailure
+
+export interface AuditSource {
+  // npm's registry's advisories of these packages' versions, asked at once.
+  advisories(packages: { name: string; versions: string[] }[], signal: AbortSignal): Promise<Advisory[]>
+  // Where audits are kept beyond this instance, by the lockfile's blob.
+  audits?: KeptStore<Audit>
+}
+
+// How long a lockfile's advisories that couldn't be had are told so, rather
+// than asked again: a scan's next parts don't wait on an audit that failed,
+// as by timing out, again and again.
+const AUDIT_FAILED_KEPT_MS = 10 * 60_000
+
+// Whether an audit kept `at` is told as it is: its advisories within
+// RISK_KEPT_MS, why they couldn't be had within AUDIT_FAILED_KEPT_MS.
+const isFresh = ({ at, audit }: { at: number; audit: Audit }) => Date.now() - at < (Array.isArray(audit) ? RISK_KEPT_MS : AUDIT_FAILED_KEPT_MS)
+
+// The audit of a lockfile's `packages`, by its `key` as kept (`memory`, then
+// the source's), else asked and kept; null where it locks none to ask of, or
+// once `signal` aborts.
+export async function auditLockfile(source: AuditSource, memory: Lru<{ at: number; audit: Audit }>, signal: AbortSignal, key: string,
+  packages: readonly LockedPackage[]): Promise<Audit | null> {
+  let kept = memory.get(key)
+  if (!kept || !isFresh(kept)) {
+    const stored = await source.audits?.read(key)
+    kept = stored ? { at: stored.at, audit: stored.value } : undefined
+  }
+  if (kept && isFresh(kept)) { memory.set(key, kept); return kept.audit }
+  const asked = new Map<string, Set<string>>()
+  for (const { name, version, source: from } of packages) {
+    if ((from !== 'registry' && from !== 'mirror') || !isNpmPackageName(name) || !isExactVersion(version)) continue
+    if (!asked.has(name)) asked.set(name, new Set())
+    asked.get(name)!.add(version)
+  }
+  if (asked.size === 0) return null
+  let audit: Audit
+  try {
+    audit = (await source.advisories([...asked].map(([name, versions]) => ({ name, versions: [...versions] })), signal))
+      .map(advisory => ({ ...advisory, malware: npmMalwareAdvisory(advisory) }))
+  } catch (error) {
+    if (signal.aborted) return null
+    audit = failureOf(error)
+  }
+  const now = Date.now()
+  memory.set(key, { at: now, audit })
+  await source.audits?.write(key, audit, now)
+  return audit
+}