@preventive/triage 1.0.0-alpha.42 → 1.0.0-alpha.43

package/server-managed/http.ts

@@ -89,7 +89,7 @@ import { mergeLinkGroups } from './link-reports.ts'
 import { backfillFindingIds, reportEntries } from '@preventive/report'
 import type { IncomingMessage, ServerResponse } from 'node:http'
 import { pipeline } from 'node:stream/promises'
-import { npmVersionStore, riskLineStore } from './npm-kept.ts'
+import { npmVersionStore, riskLineStore, scanReadStore } from './npm-kept.ts'
 import { ADVISORIES_TIMEOUT_MS, fetchBundleAdvisories } from './bundle-advisories.ts'
 import { auditCache, auditedRepos, lastListing } from './upstream-cache.ts'
 import type { BundleAdvisoryInventory } from './bundle-advisory-inventory.ts'
@@ -128,7 +128,7 @@ import { createClient } from '@preventive/upstream/github.js'
 import type { ConnectedRepo } from './github-app.ts'
 import { canAddAnyPublicRepository, canAddRepositories, passesPublicRepositorySafeguard } from './repository-policy.ts'
 import { RepositoryDiscovery } from './repository-discovery.ts'
-import { RISK_KEPT_MS, repositoryRiskLines } from './repository-risk.ts'
+import { RISK_KEPT_MS, SCAN_READ_KEPT_MS, type ScanProgress, repositoryRiskLines } from './repository-risk.ts'
 import { type RepositoryEntry, createRepositoryBrowser, scopedDirectory } from './repository-browser.ts'
 import { CALLBACK_PATH, LOGIN_PATH, OAuthError, buildLoginRedirect, ensureUserAccessToken, handleCallback } from './github-oauth.ts'
 import { clearCookie, endSession, readSession } from './session.ts'
@@ -151,14 +151,14 @@ import { hashToken, randomToken, safeEqual } from './crypto.ts'
 import { canDeleteComment, parseCommentBody } from '../common/managed/comments.ts'
 import { ManagedMutationError, reportReferenceSnapshot } from './management.ts'
 import { BundleBuildError, buildRepositoryBundle, parseBundleBuild, withBundleBuildLease } from './bundle-build.ts'
-import { NpmPackageError, type NpmReader, type NpmVersionDocument, canReadPrivateNpm, npmFileRows, npmTarballFilename, readNpmVersion, readNpmVersions } from './npm-packages.ts'
+import { NpmPackageError, type NpmReader, type NpmVersionDocument, canReadPrivateNpm, npmFileRows, npmScopesCount, npmTarballFilename, readNpmVersion, readNpmVersions } from './npm-packages.ts'
 import { loadNpmFiles, loadNpmPackageBody, loadNpmTarball } from './npm-loads.ts'
 import { computeBundleDiff } from '../common/bundle-compare-diff.js'
 import type { ReviewChange, ReviewContext, ReviewInput } from './dependency-reviews.ts'
 import { allowedReviewVerdicts, isReviewableFile, npmReviewVerdictLimit, npmTakenDown } from '../common/dependency-reviews.js'
 import { type NpmAdvisoryOptions, npmAdvisories, npmCommitTags, npmDownloads, npmGithubStats, npmKnown, npmLatestRepo, npmSocketReport } from './npm-insights.ts'
 import { npmProvenance } from './npm-provenance.ts'
-import { npmRisk, npmRiskRemoved } from './npm-risk.ts'
+import { npmReadyScanDownload, npmReadyScanDownloads, npmRisk, npmRiskRemoved } from './npm-risk.ts'
 import { NpmRiskError } from '../risk/index.js'
 import { type PrettyCache, PrettyError, type PrettyRequest, compared, prettyRequest, prettyText } from './pretty-print.ts'
 import { renameDiffOf, renameDiffRequest, renamedPairs } from './rename-diff.ts'
@@ -1351,6 +1351,10 @@ async function sendEncoded(req: IncomingMessage, res: ServerResponse, blob: Open
 // go idle, unread or with no line to send, before the scan is stopped.
 const REPO_RISK_RECHECK_MS = 10_000
 const REPO_RISK_IDLE_MS = 5 * 60_000
+// A scan's part on a serverless deployment, within its function's 300s
+// (vercel.managed.json): past it the scan stops, its last line `continue`,
+// and the page asks again, from where it stopped (repository-risk.ts).
+const REPO_RISK_PART_MS = 270_000
 
 // The repositories a scan of risk reads, by its query: every connected one;
 // a team's (`team`, its id), each in the team's directories of it; or one
@@ -1379,6 +1383,14 @@ async function riskRepositories(db: ManagedDb, query: URLSearchParams): Promise<
   })
 }
 
+// Where a scan in parts keeps its progress (repository-risk.ts
+// ScanProgress) under `key`, for a day; none where the page gave no scan id.
+function scanProgress(deps: ManagedHttpDeps, key: string, scan: string | null) {
+  if (scan === null || !/^[\w-]{1,64}$/u.test(scan)) return undefined
+  const kept = scanReadStore<ScanProgress>(deps.db, 'progress', SCAN_READ_KEPT_MS, deps.config.debug)
+  return { read: () => kept.read(key), write: (progress: ScanProgress) => kept.write(key, progress, Date.now()) }
+}
+
 // GET /api/admin/repositories/risk: every connected repository's npm
 // packages, or a team's or one repository's (riskRepositories), as its
 // lockfiles lock them at its default branch's head, each assessed by risk
@@ -1404,8 +1416,19 @@ async function handleRepositoryRisk(res: ServerResponse, deps: ManagedHttpDeps,
   res.setTimeout(REPO_RISK_IDLE_MS, () => res.destroy())
   const browser = createRepositoryBrowser(deps.config, await repositoryBrowserUser(deps, s.user.id))
   const scopes = new Map<string, Promise<boolean>>()
+  const teamScoped = (name: string) => {
+    const scope = npmPackageScope(name)
+    if (scope === null) return Promise.resolve(false)
+    if (!scopes.has(scope)) scopes.set(scope, deps.db.hasTeamNpmScope(scope))
+    return scopes.get(scope)!
+  }
   const versions = npmVersionStore(deps.db, deps.config.debug)
+  const stores = { db: deps.db, npmCache: deps.npmCacheStorage, debug: deps.config.debug }
   const lines = repositoryRiskLines({
+    // Which repositories a scan in parts told whole, by the admin, the id the
+    // page gives the scan (`scan`) and its scope, for the next part to go on
+    // past them.
+    progress: scanProgress(deps, `${s.user.id}\0${query.get('scan')}\0${query.get('team') ?? ''}\0${query.get('repo') ?? ''}`, query.get('scan')),
     repos: () => Promise.resolve(repos),
     async open(repo) {
       const reader = await browser.reader(repo)
@@ -1418,21 +1441,32 @@ async function handleRepositoryRisk(res: ServerResponse, deps: ManagedHttpDeps,
         recheck: async () => { await reader.recheckAccess() },
       }
     },
-    teamScoped(name) {
-      const scope = npmPackageScope(name)
-      if (scope === null) return Promise.resolve(false)
-      if (!scopes.has(scope)) scopes.set(scope, deps.db.hasTeamNpmScope(scope))
-      return scopes.get(scope)!
+    teamScoped,
+    // Each package's downloads readied before it is assessed
+    // (scan-downloads.ts), so that none waits on npm's downloads API; a
+    // team's own not asked, as risk asks none of theirs.
+    downloads: {
+      async atOnce(names, ready, signal) {
+        const asked: string[] = []
+        for (const name of names) if (await teamScoped(name)) ready(name); else asked.push(name)
+        await npmReadyScanDownloads(asked, ready, stores, signal)
+      },
+      one: (name, signal) => npmReadyScanDownload(name, stores, signal),
     },
     // Public versions' documents kept in the database (npm-kept.ts), as the
     // viewer keeps them: whether npm still has one, its whole document tells
     // risk.
     version: async (name, version, signal) => found(await readNpmVersion(name, version, true, signal, versions)),
-    assess: (doc, teamScoped, signal) => npmRisk(doc, 'pick', { db: deps.db, npmCache: deps.npmCacheStorage, debug: deps.config.debug,
-      teamScoped, privileged: true, signal, spare: true }),
-    removed: (name, version, teamScoped, signal) => npmRiskRemoved(name, version, { db: deps.db, npmCache: deps.npmCacheStorage, debug: deps.config.debug,
-      teamScoped, privileged: true, signal }),
-    risks: riskLineStore(deps.db, RISK_KEPT_MS, deps.config.debug),
+    assess: (doc, ownScope, signal) => npmRisk(doc, 'pick', { ...stores, teamScoped: ownScope, privileged: true, signal, spare: true }),
+    removed: (name, version, ownScope, signal) => npmRiskRemoved(name, version, { ...stores, teamScoped: ownScope, privileged: true, signal }),
+    // Kept for good, a scan telling one past RISK_KEPT_MS at once, as stale.
+    risks: riskLineStore(deps.db, Infinity, deps.config.debug),
+    // What it read of GitHub kept a day, for a scan answered in parts to go
+    // on from where the last stopped, on whichever instance answers the next.
+    listings: scanReadStore(deps.db, 'listings', SCAN_READ_KEPT_MS, deps.config.debug),
+    lockfiles: scanReadStore(deps.db, 'lockfiles', SCAN_READ_KEPT_MS, deps.config.debug),
+    // And each lockfile's advisories, for as long as a scan tells them.
+    audits: scanReadStore(deps.db, 'advisories', RISK_KEPT_MS, deps.config.debug),
     // As a bundle's advisories are asked (bundle-advisories.ts), npm's registry
     // and, for packages anyone may read, OSV: no repository's. Each lockfile's
     // audit is bounded on its own, for one npm doesn't answer to fail alone and
@@ -1444,7 +1478,7 @@ async function handleRepositoryRisk(res: ServerResponse, deps: ManagedHttpDeps,
       if (audited.status !== 200) throw new NpmPackageError(502, audited.body.error)
       return audited.body
     },
-  }, controller.signal)
+  }, controller.signal, undefined, { partMs: deps.config.serverless ? REPO_RISK_PART_MS : null })
   let checked = Date.now()
   await sendLines(res, lines, async () => {
     if (deps.isShuttingDown()) return false
@@ -2614,12 +2648,15 @@ async function handleSetTeamNpmScopes(req: IncomingMessage, res: ServerResponse,
   sendJson(res, 200, { ok: true, scopes: (await deps.db.listTeamNpmScopes())[teamId] ?? [] })
 }
 
-// Who reads npm packages now: a session with workspace access, its role and
-// the npm scopes of its visible teams. Null for none.
-async function npmReader(deps: ManagedHttpDeps, cookie: string | undefined): Promise<NpmReader | null> {
+// Who reads npm packages now: a session with workspace access, its role and,
+// where they count for any of `names` it is asked about (npmScopesCount), the
+// npm scopes of its visible teams, else none: none are read for an unscoped
+// package, an admin or a manager, or an answer checked again that is public.
+// Null for none.
+async function npmReader(deps: ManagedHttpDeps, cookie: string | undefined, names: readonly string[]): Promise<NpmReader | null> {
   const s = await readSession(deps.config, deps.db, cookie, Date.now())
   if (s == null || !roleAtLeast(s.user.role, 'view')) return null
-  return { role: s.user.role, scopes: new Set(await deps.db.listUserNpmScopes(s.user.id)), userId: s.user.id }
+  return { role: s.user.role, scopes: new Set(npmScopesCount(s.user.role, names) ? await deps.db.listUserNpmScopes(s.user.id) : []), userId: s.user.id }
 }
 
 // What a registry read found, or a 404 for the package (handleNpm's catch
@@ -2636,7 +2673,7 @@ async function readableNpmReviews<T extends ReviewContext>(res: ServerResponse,
   const onClose = () => { if (!res.writableEnded) controller.abort() }
   res.on('close', onClose)
   try {
-    const reader = await npmReader(deps, cookie)
+    const reader = await npmReader(deps, cookie, reviews.flatMap(review => [review.package, ...review.basePackage ? [review.basePackage] : []]))
     if (!reader || reader.userId !== user) { sendJson(res, 401, { error: 'unauthenticated' }); return null }
     const documents = new Map<string, Promise<NpmVersionDocument | null>>()
     // Public versions' documents as the viewer keeps them (npm-kept.ts), a
@@ -2650,7 +2687,7 @@ async function readableNpmReviews<T extends ReviewContext>(res: ServerResponse,
     const packages = await Promise.all(reviews.map(review => Promise.all([
       read(review.package, review.version), ...(review.basePackage ? [read(review.basePackage, review.baseVersion!)] : []),
     ])))
-    const current = await npmReader(deps, cookie)
+    const current = await npmReader(deps, cookie, packages.flat().flatMap(doc => doc?.private ? [doc.name] : []))
     if (!current || current.userId !== user) { sendJson(res, 401, { error: 'unauthenticated' }); return null }
     return reviews.filter((_, index) => packages[index]!.every(doc => doc && (!doc.private || canReadPrivateNpm(current, doc.name))))
   } catch (error) {
@@ -2760,7 +2797,7 @@ async function handleNpmReviews(req: IncomingMessage, res: ServerResponse, deps:
   const onClose = () => { if (!res.writableEnded) controller.abort() }
   res.on('close', onClose)
   try {
-    const reader = await npmReader(deps, cookie)
+    const reader = await npmReader(deps, cookie, [context.package, ...context.basePackage ? [context.basePackage] : []])
     if (!reader || reader.userId !== session.user.id) { sendJson(res, 401, { error: 'unauthenticated' }); return }
     const kept = npmVersionStore(deps.db, deps.config.debug)
     const read = (pkg: string, spec: string) => readNpmVersion(pkg, spec, canReadPrivateNpm(reader, pkg), controller.signal, kept)
@@ -2791,7 +2828,7 @@ async function handleNpmReviews(req: IncomingMessage, res: ServerResponse, deps:
     }
     const limit = change?.action === 'publish' && change.final && change.final.status !== 'reject'
       ? await npmReviewPublicationLimit(deps, reader, doc, controller.signal) : 'accept'
-    const current = await npmReader(deps, cookie)
+    const current = await npmReader(deps, cookie, [doc, base].flatMap(item => item?.private ? [item.name] : []))
     if (!current || current.userId !== session.user.id) { sendJson(res, 401, { error: 'unauthenticated' }); return }
     if ([doc, base].some(item => item?.private && !canReadPrivateNpm(current, item.name))) { sendJson(res, 404, { error: 'package-not-found' }); return }
     if (!change) {
@@ -2834,7 +2871,7 @@ async function handleNpm(req: IncomingMessage, res: ServerResponse, deps: Manage
   }
   const diffed = path === NPM_RENAME_DIFF_PATH && params.has('path') ? renameDiffRequest(params) : null
   if (path === NPM_RENAME_DIFF_PATH && params.has('path') && !diffed) { sendJson(res, 400, { error: 'bad-file' }); return }
-  const reader = await npmReader(deps, cookie)
+  const reader = await npmReader(deps, cookie, [name])
   if (reader == null) { sendJson(res, 401, { error: 'unauthenticated' }); return }
   const controller = new AbortController()
   const onClose = () => { if (!res.writableEnded) controller.abort() }
@@ -2842,7 +2879,7 @@ async function handleNpm(req: IncomingMessage, res: ServerResponse, deps: Manage
   // A reader who lost their session, or their private access to an answer
   // that needed it, while the registry was asked gets nothing.
   const recheck = async (isPrivate: boolean) => {
-    const current = await npmReader(deps, cookie)
+    const current = await npmReader(deps, cookie, isPrivate ? [name] : [])
     if (current == null) { sendJson(res, 401, { error: 'unauthenticated' }); return false }
     if (isPrivate && !canReadPrivateNpm(current, name)) { sendJson(res, 404, { error: 'package-not-found' }); return false }
     return !res.destroyed
@@ -2963,7 +3000,7 @@ async function handleNpm(req: IncomingMessage, res: ServerResponse, deps: Manage
       if (!(await recheck(isPrivate))) return
       res.setTimeout(NPM_RESPONSE_IDLE_MS, () => res.destroy())
       await sendLines(res, lines, async () => {
-        const current = await npmReader(deps, cookie)
+        const current = await npmReader(deps, cookie, isPrivate ? [name] : [])
         return current != null && (!isPrivate || canReadPrivateNpm(current, name))
       })
       return

package/server-managed/locked-packages.ts

@@ -1,7 +1,8 @@
 // The packages a lockfile of npm's ecosystem locks, as @preventive/lockfile
 // reads package-lock.json and npm-shrinkwrap.json of `lockfileVersion: 3`,
 // pnpm-lock.yaml of pnpm 9 to 12 and Yarn 1's yarn.lock, each with where it
-// is installed from (repository-risk.ts).
+// is installed from and whether only devDependencies need it
+// (repository-risk.ts).
 import { LockfileError, parseNpmLockfile } from '@preventive/lockfile/npm.js'
 import { parsePnpmLockfile } from '@preventive/lockfile/pnpm.js'
 import { parseYarn1Lockfile } from '@preventive/lockfile/yarn1.js'
@@ -21,6 +22,11 @@ export interface LockedPackage {
   // A mirror's tarball's, as the lockfile has it: npm's package of its name
   // and version only where npm's integrity is among it.
   integrity?: string
+  // Where only the importers' devDependencies reach it, as npm's `dev` flag
+  // says, and an install of no devDependencies leaves it out: in npm's and
+  // pnpm's lockfiles. Yarn 1's says nothing of it, which only the
+  // package.json files do, and none of its are.
+  dev?: true
 }
 
 // A package as each reader resolves one, its name the one it is published
@@ -29,7 +35,7 @@ type Resolution =
   | { type: 'tarball'; tarball: string | undefined; integrity: string | undefined; gitHosted?: boolean }
   | { type: 'git'; repo: string }
   | { type: 'directory' }
-interface ReadPackage { name: string; version: string | undefined; resolution: Resolution | undefined }
+interface ReadPackage { name: string; version: string | undefined; resolution: Resolution | undefined; dev?: boolean | undefined }
 
 // The packages of a lockfile of `format`, as its reader resolves them.
 // Versions go unchecked against ranges, as only what is locked is asked for.
@@ -72,14 +78,20 @@ function locked({ name, version, resolution }: ReadPackage): LockedPackage | nul
 }
 
 // The packages a lockfile of `format` (npm, pnpm or yarn, as
-// findRepoLockfiles names them) locks, each once, in the lockfile's order.
-// Throws the reader's LockfileError or YamlError for one it refuses.
+// findRepoLockfiles names them) locks, each once, in the lockfile's order:
+// `dev` only where it is wherever it is installed, at each path of npm's
+// tree, and with each of pnpm's peers. Throws the reader's LockfileError or
+// YamlError for one it refuses.
 export function lockedPackages(format: LockfileFormat, text: string): LockedPackage[] {
   const found = new Map<string, LockedPackage>()
   for (const pkg of packagesOf(format, text)) {
     const one = locked(pkg)
-    const key = one && `${one.source}\0${one.name}@${one.version ?? ''}\0${one.from ?? ''}\0${one.integrity ?? ''}`
-    if (one && key && !found.has(key)) found.set(key, one)
+    if (!one) continue
+    if (pkg.dev === true) one.dev = true
+    const key = `${one.source}\0${one.name}@${one.version ?? ''}\0${one.from ?? ''}\0${one.integrity ?? ''}`
+    const had = found.get(key)
+    if (!had) found.set(key, one)
+    else if (had.dev && !one.dev) delete had.dev
   }
   return [...found.values()]
 }

package/server-managed/npm-kept.ts

@@ -5,9 +5,10 @@
 // (npm-manifest.ts npmVersionFields), for a day (readNpmVersion asks npm again
 // then, as a version can be deprecated, its package made private, or itself
 // removed); and, for scans, each package's risk line, for as long as a scan
-// keeps it (repository-risk.ts). Each store removes its own rows kept longer
-// than that as it writes, at most once an hour a database, so that rows no
-// read will take again don't pile up.
+// keeps it, and what a scan read of GitHub, for a day, and of npm's
+// advisories, for an hour (repository-risk.ts). Each store removes its own rows kept longer
+// than that as it writes, at most once an hour a database (pruneKept), so
+// that rows no read will take again don't pile up.
 // The database is an optimization: a failure reads as a miss, and writes
 // nothing.
 import { createHash } from 'node:crypto'
@@ -18,30 +19,45 @@ import type { UpstreamCacheStore } from './upstream-cache.ts'
 // A value kept by key, as JSON, `at` when it was.
 export interface KeptStore<T> {
   read(key: string): Promise<{ at: number; value: T } | null>
+  // Those of `keys` kept, by key, in one read of the database.
+  readMany(keys: string[]): Promise<Map<string, { at: number; value: T }>>
   write(key: string, value: T, at: number): Promise<void>
 }
 
-// When each database last had each type's old rows removed.
+// When each database last had the rows under each prefix removed.
 const PRUNE_EVERY_MS = 60 * 60_000
 const pruned = new WeakMap<UpstreamCacheStore, Map<string, number>>()
 
+// Removes the rows under `prefix` kept `keptMs` before `at`, at most once an
+// hour a database, as rows under it are written: here, and risk's downloads
+// (npm-risk.ts). A failure is `warn`ed of, and removes nothing.
+export async function pruneKept(db: UpstreamCacheStore, prefix: string, keptMs: number, at: number, warn: (error: unknown) => void) {
+  const times = pruned.get(db) ?? pruned.set(db, new Map()).get(db)!
+  if (at - (times.get(prefix) ?? -Infinity) < PRUNE_EVERY_MS) return
+  times.set(prefix, at)
+  try { await db.deleteUpstreamCacheEntries(prefix, at - keptMs) } catch (error) { warn(error) }
+}
+
 // Values of `type`, each read for `keptMs` after it was kept.
 function keptStore<T>(db: UpstreamCacheStore, type: string, keptMs: number, debug: boolean): KeptStore<T> {
   const warn = (what: string, error: unknown) => { if (debug) console.warn(`managed: kept ${type} ${what} failed:`, error) }
-  const prune = async (at: number) => {
-    const times = pruned.get(db) ?? pruned.set(db, new Map()).get(db)!
-    if (at - (times.get(type) ?? -Infinity) < PRUNE_EVERY_MS) return
-    times.set(type, at)
-    try { await db.deleteUpstreamCacheEntries(`${type}/`, at - keptMs) } catch (error) { warn('prune', error) }
-  }
+  // Kept for good (Infinity): none removed.
+  const prune = (at: number) => keptMs === Infinity ? Promise.resolve() : pruneKept(db, `${type}/`, keptMs, at, error => { warn('prune', error) })
   // Keys of any length or character, as the database's: hashed.
   const keyOf = (key: string) => `${type}/${createHash('sha256').update(key).digest('hex')}`
+  const valueOf = (row: string | null | undefined) => {
+    const kept: unknown = JSON.parse(row ?? 'null')
+    return plainObject(kept) && typeof kept['at'] === 'number' && 'value' in kept ? { at: kept['at'], value: kept['value'] as T } : null
+  }
   return {
     async read(key) {
+      try { return valueOf(await db.getUpstreamCacheEntry(keyOf(key))) } catch (error) { warn('read', error); return null }
+    },
+    async readMany(keys) {
       try {
-        const kept: unknown = JSON.parse(await db.getUpstreamCacheEntry(keyOf(key)) ?? 'null')
-        return plainObject(kept) && typeof kept['at'] === 'number' && 'value' in kept ? { at: kept['at'], value: kept['value'] as T } : null
-      } catch (error) { warn('read', error); return null }
+        const rows = await db.getUpstreamCacheEntries(keys.map(keyOf))
+        return new Map(keys.flatMap(key => { const kept = valueOf(rows.get(keyOf(key))); return kept ? [[key, kept] as const] : [] }))
+      } catch (error) { warn('read', error); return new Map() }
     },
     async write(key, value, at) {
       try { await db.setUpstreamCacheEntry(keyOf(key), JSON.stringify({ at, value }), at) } catch (error) { warn('write', error) }
@@ -61,5 +77,13 @@ export function npmVersionStore(db: UpstreamCacheStore, debug = false): NpmVersi
   }
 }
 
-// Risk lines, each read for `keptMs`, as long as a scan keeps them.
+// Risk lines, each kept `keptMs`, for good where Infinity, a scan telling one
+// past RISK_KEPT_MS as stale (repository-risk.ts).
 export const riskLineStore = <T>(db: UpstreamCacheStore, keptMs: number, debug = false): KeptStore<T> => keptStore<T>(db, 'repository-risk/lines', keptMs, debug)
+// What a scan read, each read for keptMs (repository-risk.ts): of GitHub, a
+// repository's lockfiles as listed at a commit (`listings`) and a lockfile's
+// packages by its blob (`lockfiles`), for SCAN_READ_KEPT_MS; of npm, a
+// lockfile's advisories by its blob (`advisories`), for RISK_KEPT_MS; and the
+// repositories a scan in parts told whole (`progress`).
+export const scanReadStore = <T>(db: UpstreamCacheStore, type: 'listings' | 'lockfiles' | 'advisories' | 'progress', keptMs: number, debug = false): KeptStore<T> =>
+  keptStore<T>(db, `repository-risk/${type}`, keptMs, debug)

package/server-managed/npm-packages.ts

@@ -43,7 +43,7 @@ export class NpmPackageError extends Error {
 }
 
 // Who reads, as the session has it now: the role, and the scopes of the
-// visible teams they are a member of.
+// visible teams they are a member of, where those count (npmScopesCount).
 export interface NpmReader { role: Role; scopes: ReadonlySet<string>; userId: string }
 
 // Whether a reader may have a package read with the server's token.
@@ -53,8 +53,19 @@ export function canReadPrivateNpm(reader: NpmReader, name: string): boolean {
   return scope !== null && reader.scopes.has(scope)
 }
 
+// Whether a reader's team scopes count for any of `names`, as
+// canReadPrivateNpm reads them: a scoped one, read by someone below manager.
+export const npmScopesCount = (role: Role, names: readonly string[]): boolean =>
+  role !== 'admin' && role !== 'manage' && names.some(name => npmPackageScope(name) !== null)
+
 export const npmToken = () => process.env['NPM_TOKEN'] || null
 
+// Whether a read of `name` that npm answers none anonymously is asked again
+// with the server's token: by a reader with private access (`privileged`),
+// where there is a token and the package is scoped, as npm's private packages
+// always are. Else the anonymous answer is the reader's.
+export const npmTokenReads = (name: string, privileged: boolean): boolean => privileged && npmToken() !== null && npmPackageScope(name) !== null
+
 function registryUrl(name: string, spec?: string): string {
   return `${NPM_REGISTRY}/${name}${spec === undefined ? '' : `/${encodeURIComponent(spec)}`}`
 }
@@ -138,16 +149,17 @@ async function publicWholeDocument(name: string, url: string, { limit, signal }:
   return json
 }
 
-// Asked anonymously first: a reader without private access stops there. One
-// with it asks again with the token, where there is one and the package is
-// scoped (npm's private packages always are), and the answer is private. A
-// whole document (`whole`) anyone may read is kept (publicWholeDocument).
-async function readDocument(name: string, url: string, privileged: boolean, { whole = false, ...options }: { accept: string; limit: number; signal: AbortSignal; whole?: boolean }) {
-  const open = whole ? await publicWholeDocument(name, url, options) : await registryDocument(url, { ...options, token: null })
+// Asked anonymously first, unless its caller knows npm has none for anyone
+// (`anonymous` false): a reader without private access stops there. One
+// with it asks again with the token where npmTokenReads says so, and the
+// answer is private. A whole document (`whole`) anyone may read is kept
+// (publicWholeDocument).
+async function readDocument(name: string, url: string, privileged: boolean,
+  { whole = false, anonymous = true, ...options }: { accept: string; limit: number; signal: AbortSignal; whole?: boolean; anonymous?: boolean }) {
+  const open = anonymous ? whole ? await publicWholeDocument(name, url, options) : await registryDocument(url, { ...options, token: null }) : null
   if (open) return { json: open, private: false }
-  const token = npmToken()
-  if (!privileged || !token || npmPackageScope(name) === null) return null
-  const json = await registryDocument(url, { ...options, token })
+  if (!npmTokenReads(name, privileged)) return null
+  const json = await registryDocument(url, { ...options, token: npmToken() })
   return json ? { json, private: true } : null
 }
 
@@ -244,10 +256,12 @@ export function readNpmVersions(name: string, privileged: boolean, signal: Abort
 // A package's whole document as npm has it for this reader, for risk
 // (npm-risk.ts), within the bounds and budget every document read here is
 // held to; null where npm has none for this reader, or the name is none npm
-// takes.
-export function readNpmPackument(name: string, privileged: boolean, signal: AbortSignal): Promise<Record<string, unknown> | null> {
+// takes. With `anonymous` false, where npm is known to have none for anyone,
+// it is asked only with the token (readDocument).
+export function readNpmPackument(name: string, privileged: boolean, signal: AbortSignal, { anonymous = true } = {}): Promise<Record<string, unknown> | null> {
   if (!isNpmPackageName(name)) return Promise.resolve(null)
-  return withDocumentBudget(PACKUMENT_BYTES, async () => (await readDocument(name, registryUrl(name), privileged, { accept: 'application/json', limit: PACKUMENT_BYTES, signal, whole: true }))?.json ?? null)
+  return withDocumentBudget(PACKUMENT_BYTES, async () => (await readDocument(name, registryUrl(name), privileged,
+    { accept: 'application/json', limit: PACKUMENT_BYTES, signal, whole: true, anonymous }))?.json ?? null)
 }
 
 const isPublishTime = (value: string | undefined) => typeof value === 'string' && value.length <= 40 && !Number.isNaN(Date.parse(value))

package/server-managed/npm-risk.ts

@@ -9,8 +9,13 @@
 // - `risk/npm/versions`, what a version's document says, which npm never
 //   changes once published: a public package's in the database's upstream
 //   cache table, shared by every instance; a private one's in memory.
-// - `risk/npm/packages`, `risk/npm/public`, `risk/npm/advisories` and
-//   `risk/npm/downloads`, kept an hour or a day by risk itself: in memory.
+// - `risk/npm/downloads`, a package's downloads a day, which risk tells as
+//   kept for a week: a public package's in the database too, so that another
+//   instance, or this one restarted, asks npm's downloads API, which answers
+//   some 40 lookups a minute, none of them again that week, each removed a
+//   day past it (npm-kept.ts pruneKept); a private one's in memory.
+// - `risk/npm/packages`, `risk/npm/public` and `risk/npm/advisories`, kept an
+//   hour or a day by risk itself: in memory.
 // - `npm/tarballs`: none asked, as tarballs are read as the viewer's loads
 //   read them, kept where they keep them (npm-loads.ts loadNpmTarball), and
 //   an assessment kept needs none.
@@ -28,11 +33,12 @@ import { Buffer } from 'node:buffer'
 import { createHash } from 'node:crypto'
 import { buffer } from 'node:stream/consumers'
 import type { CacheStore } from '@preventive/upstream/npm.js'
-import { DEFAULT_MINIMUM_RELEASE_AGE, type NpmAssessment, assessNpm } from '../risk/index.js'
+import { DEFAULT_MINIMUM_RELEASE_AGE, type NpmAssessment, assessNpm, npmReadyDownload, npmReadyDownloads } from '../risk/index.js'
 import { CacheMissError, type CacheStorage } from './cache-storage.ts'
 import { loadNpmTarball } from './npm-loads.ts'
-import { NpmPackageError, type NpmVersionDocument, readNpmPackument, readNpmVersion } from './npm-packages.ts'
+import { NpmPackageError, type NpmVersionDocument, npmTokenReads, readNpmPackument, readNpmVersion } from './npm-packages.ts'
 import { npmPackageScope } from '../common/managed/npm-packages.js'
+import { pruneKept } from './npm-kept.ts'
 import type { UpstreamCacheStore } from './upstream-cache.ts'
 
 const MEMORY_ENTRIES = 2000
@@ -63,6 +69,10 @@ function remember(key: string, value: unknown) {
 }
 
 const hashed = (type: string, key: string) => `risk/${createHash('sha256').update(`${type}\0${key}`).digest('hex')}`
+// What of a public package is kept in the database, and for how long its
+// downloads are: risk's week, and a day more.
+const IN_DATABASE = new Set(['risk/npm/versions', 'risk/npm/downloads'])
+const DOWNLOADS_KEPT_MS = 8 * 24 * 60 * 60_000
 
 export interface NpmRiskStores {
   db: UpstreamCacheStore
@@ -70,16 +80,38 @@ export interface NpmRiskStores {
   debug?: boolean | undefined
 }
 
+// A cache that reads and writes many of a type at once too, as risk's
+// readying of downloads asks (npmReadyDownloads): one read and one write of
+// the database each, not one a key, as a request's queries take turns on its
+// connection (db-neon.ts).
+export interface NpmRiskCache extends CacheStore {
+  readMany(type: string, keys: string[]): Promise<Map<string, unknown>>
+  writeMany(type: string, entries: [key: string, value: unknown][]): Promise<void>
+}
+
 // Risk's cache for one package: `isPublic` whether anyone can read it, so
 // that what is kept of it can be shared.
-export function npmRiskCache({ db, npmCache, debug }: NpmRiskStores, isPublic: boolean): CacheStore {
+export function npmRiskCache({ db, npmCache, debug }: NpmRiskStores, isPublic: boolean): NpmRiskCache {
   const warn = (what: string, error: unknown) => { if (debug) console.warn(`managed: risk cache ${what} failed:`, error) }
-  return {
+  const cache: NpmRiskCache = {
+    async readMany(type, keys) {
+      if (!isPublic || !IN_DATABASE.has(type)) return new Map(await Promise.all(keys.map(async key => [key, await cache.read(type, key)] as const)))
+      try {
+        const found = await db.getUpstreamCacheEntries(keys.map(key => `${type}/${key}`))
+        return new Map(keys.map(key => { const value = found.get(`${type}/${key}`); return [key, value === undefined ? null : JSON.parse(value)] }))
+      } catch (error) { warn('read', error); return new Map() }
+    },
+    async writeMany(type, entries) {
+      if (!isPublic || !IN_DATABASE.has(type)) { for (const [key, value] of entries) await cache.write(type, key, value); return }
+      const at = Date.now()
+      try { await db.setUpstreamCacheEntries(entries.map(([key, value]) => [`${type}/${key}`, JSON.stringify(value)]), at) } catch (error) { warn('write', error); return }
+      if (type === 'risk/npm/downloads') await pruneKept(db, `${type}/`, DOWNLOADS_KEPT_MS, at, error => { warn('prune', error) })
+    },
     async read(type, key) {
       if (type === 'npm/tarballs') return null
       try {
         if (isPublic && type === 'risk/npm/assessments' && npmCache) return JSON.parse((await buffer((await npmCache.open(hashed(type, key))).stream)).toString('utf8'))
-        if (isPublic && type === 'risk/npm/versions') { const value = await db.getUpstreamCacheEntry(`${type}/${key}`); return value === null ? null : JSON.parse(value) }
+        if (isPublic && IN_DATABASE.has(type)) { const value = await db.getUpstreamCacheEntry(`${type}/${key}`); return value === null ? null : JSON.parse(value) }
       } catch (error) { if (!(error instanceof CacheMissError)) warn('read', error); return null }
       return memory.get(`${isPublic ? '' : 'private\0'}${type}\0${key}`)?.value ?? null
     },
@@ -87,11 +119,17 @@ export function npmRiskCache({ db, npmCache, debug }: NpmRiskStores, isPublic: b
       if (type === 'npm/tarballs') return
       try {
         if (isPublic && type === 'risk/npm/assessments' && npmCache) { await npmCache.put(hashed(type, key), Buffer.from(JSON.stringify(value))); return }
-        if (isPublic && type === 'risk/npm/versions') { await db.setUpstreamCacheEntry(`${type}/${key}`, JSON.stringify(value), Date.now()); return }
+        if (isPublic && IN_DATABASE.has(type)) {
+          const at = Date.now()
+          await db.setUpstreamCacheEntry(`${type}/${key}`, JSON.stringify(value), at)
+          if (type === 'risk/npm/downloads') await pruneKept(db, `${type}/`, DOWNLOADS_KEPT_MS, at, error => { warn('prune', error) })
+          return
+        }
       } catch (error) { warn('write', error); return }
       remember(`${isPublic ? '' : 'private\0'}${type}\0${key}`, value)
     },
   }
+  return cache
 }
 
 // A version's tarball as the viewer's loads read it (npm-loads.ts), its bytes
@@ -160,8 +198,9 @@ async function bounded(spare: boolean, assess: () => Promise<NpmAssessment>): Pr
 // by npm, as a scan's (`spare`), and its advisories of malware, npm's and,
 // where anyone can read its package, OSV's, which alone has some of those
 // npm removed (nx's), a private package's name never sent to OSV. Its
-// package's document read anonymously, else with npm's token where the
-// reader may have it (`privileged`), and kept as a private package's is; no
+// package's document read anonymously, else with npm's token alone where
+// that reads more for this reader (`privileged`, npmTokenReads), and kept as
+// a private package's is; no
 // tarball read, nothing of the version being left. Spare, within
 // MAX_SPARE_ASSESSMENTS as a scan's others are. An NpmRiskError where
 // nothing tells of it.
@@ -171,7 +210,26 @@ export function npmRiskRemoved(name: string, version: string,
     const open = await readNpmPackument(name, false, signal)
     return assessNpm(name, version, { minimumReleaseAge: teamScoped ? 0 : DEFAULT_MINIMUM_RELEASE_AGE, popularity: false, downloads: false, dependencies: false,
       osv: open !== null, cache: npmRiskCache(stores, false), signal,
-      packument: (read, { authorized }) => read === name && open ? Promise.resolve(open) : readNpmPackument(read, privileged && authorized, signal),
+      packument: (read, { authorized }) => read === name
+        ? open || !npmTokenReads(name, privileged && authorized) ? Promise.resolve(open) : readNpmPackument(name, true, signal, { anonymous: false })
+        : readNpmPackument(read, privileged && authorized, signal),
       tarball: () => Promise.reject(new NpmPackageError(502, 'upstream-invalid')) })
   })
 }
+
+// What a scan readies before assessing its packages (repository-risk.ts):
+// their downloads, as risk asks them, kept where a public package's are
+// (npmRiskCache), so that their assessments wait on npm's downloads API for
+// none. At once, those kept and unscoped ones, in bulk, all public, as only a
+// scoped package can be private; each other one on its own
+// (npmReadyScanDownload), kept only where npm counts some, which it does of
+// public packages alone: a private one's none, as of one it has no such
+// package of, isn't kept where public ones are, for its assessment to ask
+// its own where private ones are.
+export const npmReadyScanDownloads = (names: string[], ready: (name: string) => void, stores: NpmRiskStores, signal: AbortSignal) =>
+  npmReadyDownloads(names, ready, { cache: npmRiskCache(stores, true), signal })
+export function npmReadyScanDownload(name: string, stores: NpmRiskStores, signal: AbortSignal) {
+  const cache = npmRiskCache(stores, true)
+  const counted = (value: unknown) => !(value && typeof value === 'object' && 'none' in value)
+  return npmReadyDownload(name, { cache: { read: cache.read, write: (type, key, value) => counted(value) ? cache.write(type, key, value) : Promise.resolve() }, signal })
+}

package/server-managed/queue.ts

@@ -0,0 +1,23 @@
+// Items in the order they are added, each taken once by whichever reader asks
+// first: none once closed and all taken, or once `signal` aborts, but those
+// added by then where `drain`, none waited for.
+export interface Queue<T> { add(item: T): void; close(): void; take(): Promise<T | undefined> }
+export function queue<T>(signal: AbortSignal, { drain = false } = {}): Queue<T> {
+  const items: T[] = []
+  const waiting: (() => void)[] = []
+  let closed = false
+  const wake = () => { for (const resolve of waiting.splice(0)) resolve() }
+  signal.addEventListener('abort', wake, { once: true })
+  return {
+    add(item) { items.push(item); wake() },
+    close() { closed = true; wake() },
+    async take() {
+      for (;;) {
+        if (signal.aborted && !drain) return undefined
+        const item = items.shift()
+        if (item !== undefined || closed || signal.aborted) return item
+        await new Promise<void>(resolve => { waiting.push(resolve) })
+      }
+    },
+  }
+}

package/server-managed/report-sources.ts

@@ -29,7 +29,7 @@ function formatDirectory(bundleId: string, sha256: string, name: string) {
 }
 function sourceCacheFilename(report: ReportRecord, bundle: ManagedBundle, permissions: ViewerPermissions, repo: { github: string | null }, sourcePaths?: Set<string>) {
   const key = createHash('sha256').update(JSON.stringify([
-    'finding-access-v9', sourcePaths ? [...sourcePaths].toSorted() : null, bundle.integrity, bundle.kind, permissions.dependencies, permissions.security, repo,
+    'finding-access-v10', sourcePaths ? [...sourcePaths].toSorted() : null, bundle.integrity, bundle.kind, permissions.dependencies, permissions.security, repo,
   ])).digest('hex')
   return `${formatDirectory(bundle.id, report.sha256, report.filename)}/${key}.json.gz`
 }