@preventive/triage 1.0.0-alpha.41 → 1.0.0-alpha.43

package/risk/npm/registry.js

@@ -33,7 +33,9 @@
 // tarball unpacking past tarball.js's, `package-too-large`.
 import { advisories } from '@preventive/upstream/advisories.js'
 import { getTarball } from '@preventive/upstream/npm.js'
+import { compareVersions, satisfies, validRange } from '@preventive/upstream/semver.js'
 import { npmVersionManifest } from './manifest.js'
+import { npmAlias } from './specs.js'
 import { npmPlaceholder } from './malware.js'
 import { npmOldVersion } from './old-version.js'
 import { npmTarballVersion } from './tarball.js'
@@ -50,6 +52,14 @@ const ADVISORIES_MS = 60 * 60_000
 // months' downloads a day.
 const PACKUMENT_BYTES = 64 * 1024 * 1024
 const DOWNLOADS_BYTES = 1024 * 1024
+// How long a package's downloads are kept: a week, as the tier they put it in,
+// by the most of its latest six weeks' (downloads.js npmDownloadTier), hardly
+// moves in one, and npm's downloads API answers only some 40 lookups a minute.
+const DOWNLOADS_MS = 7 * DAY_MS
+// How many packages npm's downloads API answers in one lookup, unscoped ones
+// alone (it refuses a scoped one in a bulk lookup), and how much of it is read.
+const BULK_DOWNLOADS = 128
+const BULK_DOWNLOADS_BYTES = 8 * 1024 * 1024
 // How long npm is waited on for an answer, as the managed server waits on its
 // upstreams (server-managed/npm-packages.ts askUpstream).
 const REQUEST_MS = 30_000
@@ -59,6 +69,9 @@ export class NpmRiskError extends Error {
 }
 
 const isTime = value => typeof value === 'string' && value.length <= 40 && !Number.isNaN(Date.parse(value))
+// An object as JSON holds one, not an array (manifest.js plainObject).
+const plainObject = value => Boolean(value) && typeof value === 'object' && !Array.isArray(value)
+
 // A scoped name in a downloads API path, its slash escaped.
 const encoded = name => name.replace('/', '%2f')
 const token = () => typeof process === 'undefined' ? null
@@ -245,13 +258,42 @@ export async function npmVersionFiles(name, version, { manifest, published, dist
   }
 }
 
+// The version of a package's document npm installs for a dependency's spec
+// (`spec`, an alias's by its own range, `npm:name@^1`, as specs.js npmAlias
+// reads one), as npm picks one:
+// the tag it names, else its `latest` where that satisfies it, or any range
+// does (`*`, a prerelease latest too), and isn't deprecated, else the highest
+// that satisfies it, one not deprecated first; null for none, or for a spec
+// that is no range nor tag (a URL, a repository).
+function installedVersion(doc, spec) {
+  const alias = npmAlias(spec)
+  const range = (alias ? alias.spec : String(spec)).trim()
+  const tags = plainObject(doc['dist-tags']) ? doc['dist-tags'] : {}, versions = plainObject(doc.versions) ? doc.versions : {}
+  if (Object.hasOwn(tags, range) && Object.hasOwn(versions, tags[range])) return tags[range]
+  if (validRange(range || '*') === null) return null
+  if (Object.hasOwn(versions, tags.latest) && typeof versions[tags.latest]?.deprecated !== 'string' && ((range || '*') === '*' || satisfies(tags.latest, range))) return tags.latest
+  const matching = Object.keys(versions).filter(version => satisfies(version, range || '*'))
+  const current = matching.filter(version => typeof versions[version]?.deprecated !== 'string')
+  return (current.length > 0 ? current : matching).reduce((best, version) => best === null || compareVersions(version, best) > 0 ? version : best, null)
+}
+
 // When a package anyone can read was first published, `{ created }`, or null
-// for one npm has none of for anyone.
-export async function npmPublicFacts(name, context) {
-  const facts = await kept(context, 'risk/npm/public', name,
-    async () => { const doc = await packageDocument(name, context, false); return doc ? { at: context.now, created: firstPublished(doc) } : { at: context.now, missing: true } },
-    hit => context.now - hit.at < PACKAGE_MS)
-  return facts.missing ? null : { created: facts.created }
+// for one npm has none of for anyone; and for a dependency's spec (`spec`),
+// the version of it npm installs (installedVersion), `installs`: `{ version,
+// latest, published, scripts }`, its package's `latest` beside it, when it
+// was published and whether it has install scripts: its manifest's, or
+// node-gyp's for a `binding.gyp` (`gypfile`), or as the registry says
+// (`hasInstallScript`); null for none.
+export async function npmPublicFacts(name, context, spec = null) {
+  const facts = await kept(context, 'risk/npm/public', spec === null ? name : `${name}@${spec}`, async () => {
+    const doc = await packageDocument(name, context, false)
+    if (!doc) return { at: context.now, missing: true }
+    const version = spec === null ? null : installedVersion(doc, spec)
+    return { at: context.now, created: firstPublished(doc), ...spec === null ? {} : { installs: version === null ? null
+      : { version, latest: typeof doc['dist-tags']?.latest === 'string' ? doc['dist-tags'].latest : null, published: isTime(doc.time?.[version]) ? doc.time[version] : null,
+        scripts: Boolean(npmVersionManifest(doc.versions[version]).installScripts) || doc.versions[version]?.gypfile === true || doc.versions[version]?.hasInstallScript === true } } }
+  }, hit => context.now - hit.at < PACKAGE_MS)
+  return facts.missing ? null : { created: facts.created, ...spec === null ? {} : { installs: facts.installs } }
 }
 
 // The days a package's downloads are asked over at `now`: from the first of
@@ -262,21 +304,80 @@ function rangeAt(now) {
   return `${day(Date.UTC(today.getUTCFullYear(), today.getUTCMonth() - 2, 1))}:${day(now - DAY_MS)}`
 }
 
-// A package's downloads a day over rangeAt's days, `{ start, end, days }`,
-// null where npm counts none.
+// A package's downloads as kept, from npm's answer of them over a range of
+// days `asked` then: `{ at, start, end, days }`, a count a day from
+// `start`, or `{ at, none }` where npm counts none (null).
+function downloadsOf(json, asked) {
+  if (json === null) return { at: asked, none: true }
+  const start = Date.parse(json.start)
+  const length = Math.round((Date.parse(json.end) - start) / DAY_MS) + 1
+  const days = Array.from({ length: Number.isSafeInteger(length) && length > 0 && length <= 400 ? length : 0 }, () => 0)
+  for (const row of json.downloads ?? []) {
+    const at = Math.round((Date.parse(row.day) - start) / DAY_MS)
+    if (at >= 0 && at < days.length && Number.isSafeInteger(row.downloads)) days[at] = row.downloads
+  }
+  return { at: asked, start: json.start, end: json.end, days }
+}
+// Whether downloads kept are told as they are: kept within DOWNLOADS_MS.
+const freshDownloads = (held, now) => held != null && now - held.at < DOWNLOADS_MS
+
+// A package's downloads a day over rangeAt's days when asked, `{ start, end,
+// days }`, kept a week (DOWNLOADS_MS), null where npm counts none.
 export async function npmDownloads(name, context) {
-  const range = rangeAt(context.now)
-  const answer = await kept(context, 'risk/npm/downloads', `${name}@${range}`, async () => {
-    const json = await askNpm(`${DOWNLOADS}/${range}/${encoded(name)}`, {}, DOWNLOADS_BYTES, `${name}'s downloads`, context)
-    if (json === null) return { none: true }
-    const start = Date.parse(json.start)
-    const length = Math.round((Date.parse(json.end) - start) / DAY_MS) + 1
-    const days = Array.from({ length: Number.isSafeInteger(length) && length > 0 && length <= 400 ? length : 0 }, () => 0)
-    for (const row of json.downloads ?? []) {
-      const at = Math.round((Date.parse(row.day) - start) / DAY_MS)
-      if (at >= 0 && at < days.length && Number.isSafeInteger(row.downloads)) days[at] = row.downloads
-    }
-    return { start: json.start, end: json.end, days }
-  })
+  const answer = await kept(context, 'risk/npm/downloads', name,
+    async () => downloadsOf(await askNpm(`${DOWNLOADS}/${rangeAt(context.now)}/${encoded(name)}`, {}, DOWNLOADS_BYTES, `${name}'s downloads`, context), context.now),
+    hit => freshDownloads(hit, context.now))
   return answer.none ? null : answer
 }
+
+// Readies, at once, the downloads assessNpm asks of each of these packages
+// (`names`) that can be, kept in `cache` as npmDownloads keeps them, so that
+// their assessments ask npm none: those kept within a week (DOWNLOADS_MS), as
+// they are, and unscoped ones, asked of npm BULK_DOWNLOADS at a time, as its downloads API
+// answers them; `ready` is told each once its downloads are kept. The rest,
+// scoped ones (it answers those only one by one) and any a bulk lookup
+// didn't answer, are left for npmReadyDownload, one at a time. npm asking to
+// be asked later is thrown (`upstream-busy`), as is any other failure, those
+// readied by then told.
+export async function npmReadyDownloads(names, ready, { cache, signal, now = Date.now() }) {
+  const context = { cache, signal, now }
+  const range = rangeAt(now)
+  const unique = [...new Set(names)]
+  const unscoped = []
+  const ask = async part => {
+    const json = await askNpm(`${DOWNLOADS}/${range}/${part.join(',')}`, {}, BULK_DOWNLOADS_BYTES, `${part.length} packages' downloads`, context)
+    const answered = part.filter(name => json && Object.hasOwn(json, name))
+    await keepMany(cache, answered.map(name => [name, downloadsOf(json[name], now)]))
+    for (const name of answered) ready(name)
+  }
+  // Those kept read BULK_DOWNLOADS at a time, the rest asked as they fill a
+  // lookup.
+  for (let at = 0; at < unique.length; at += BULK_DOWNLOADS) {
+    const part = unique.slice(at, at + BULK_DOWNLOADS)
+    const held = await keptMany(cache, part)
+    for (const name of part) {
+      if (freshDownloads(held.get(name), now)) ready(name)
+      else if (!name.startsWith('@')) unscoped.push(name)
+    }
+    while (unscoped.length >= BULK_DOWNLOADS) await ask(unscoped.splice(0, BULK_DOWNLOADS))
+  }
+  // A lookup of one is answered as that package's own, not by its name.
+  if (unscoped.length > 1) await ask(unscoped)
+}
+
+// Downloads kept under each of `keys`, by key, and many kept at once: in one
+// read and one write where the cache reads and writes many at once
+// (`readMany`, `writeMany`), else a key at a time.
+const keptMany = async (cache, keys) => cache.readMany ? await cache.readMany('risk/npm/downloads', keys)
+  : new Map(await Promise.all(keys.map(async key => [key, await cache.read('risk/npm/downloads', key)])))
+const keepMany = async (cache, entries) => {
+  if (cache.writeMany) await cache.writeMany('risk/npm/downloads', entries)
+  else for (const [key, value] of entries) await cache.write('risk/npm/downloads', key, value)
+}
+
+// Readies the downloads assessNpm asks of a package, asked of npm on their
+// own where they aren't kept (npmDownloads), as npmReadyDownloads leaves a
+// scoped one's: npm asking to be asked later thrown (`upstream-busy`).
+export async function npmReadyDownload(name, { cache, signal, now = Date.now() }) {
+  await npmDownloads(name, { cache, signal, now })
+}

package/server-managed/http.ts

@@ -89,7 +89,7 @@ import { mergeLinkGroups } from './link-reports.ts'
 import { backfillFindingIds, reportEntries } from '@preventive/report'
 import type { IncomingMessage, ServerResponse } from 'node:http'
 import { pipeline } from 'node:stream/promises'
-import { npmVersionStore, riskLineStore } from './npm-kept.ts'
+import { npmVersionStore, riskLineStore, scanReadStore } from './npm-kept.ts'
 import { ADVISORIES_TIMEOUT_MS, fetchBundleAdvisories } from './bundle-advisories.ts'
 import { auditCache, auditedRepos, lastListing } from './upstream-cache.ts'
 import type { BundleAdvisoryInventory } from './bundle-advisory-inventory.ts'
@@ -128,7 +128,7 @@ import { createClient } from '@preventive/upstream/github.js'
 import type { ConnectedRepo } from './github-app.ts'
 import { canAddAnyPublicRepository, canAddRepositories, passesPublicRepositorySafeguard } from './repository-policy.ts'
 import { RepositoryDiscovery } from './repository-discovery.ts'
-import { RISK_KEPT_MS, repositoryRiskLines } from './repository-risk.ts'
+import { RISK_KEPT_MS, SCAN_READ_KEPT_MS, type ScanProgress, repositoryRiskLines } from './repository-risk.ts'
 import { type RepositoryEntry, createRepositoryBrowser, scopedDirectory } from './repository-browser.ts'
 import { CALLBACK_PATH, LOGIN_PATH, OAuthError, buildLoginRedirect, ensureUserAccessToken, handleCallback } from './github-oauth.ts'
 import { clearCookie, endSession, readSession } from './session.ts'
@@ -151,14 +151,14 @@ import { hashToken, randomToken, safeEqual } from './crypto.ts'
 import { canDeleteComment, parseCommentBody } from '../common/managed/comments.ts'
 import { ManagedMutationError, reportReferenceSnapshot } from './management.ts'
 import { BundleBuildError, buildRepositoryBundle, parseBundleBuild, withBundleBuildLease } from './bundle-build.ts'
-import { NpmPackageError, type NpmReader, type NpmVersionDocument, canReadPrivateNpm, npmFileRows, npmTarballFilename, readNpmVersion, readNpmVersions } from './npm-packages.ts'
+import { NpmPackageError, type NpmReader, type NpmVersionDocument, canReadPrivateNpm, npmFileRows, npmScopesCount, npmTarballFilename, readNpmVersion, readNpmVersions } from './npm-packages.ts'
 import { loadNpmFiles, loadNpmPackageBody, loadNpmTarball } from './npm-loads.ts'
 import { computeBundleDiff } from '../common/bundle-compare-diff.js'
 import type { ReviewChange, ReviewContext, ReviewInput } from './dependency-reviews.ts'
 import { allowedReviewVerdicts, isReviewableFile, npmReviewVerdictLimit, npmTakenDown } from '../common/dependency-reviews.js'
 import { type NpmAdvisoryOptions, npmAdvisories, npmCommitTags, npmDownloads, npmGithubStats, npmKnown, npmLatestRepo, npmSocketReport } from './npm-insights.ts'
 import { npmProvenance } from './npm-provenance.ts'
-import { npmRisk, npmRiskRemoved } from './npm-risk.ts'
+import { npmReadyScanDownload, npmReadyScanDownloads, npmRisk, npmRiskRemoved } from './npm-risk.ts'
 import { NpmRiskError } from '../risk/index.js'
 import { type PrettyCache, PrettyError, type PrettyRequest, compared, prettyRequest, prettyText } from './pretty-print.ts'
 import { renameDiffOf, renameDiffRequest, renamedPairs } from './rename-diff.ts'
@@ -1351,6 +1351,10 @@ async function sendEncoded(req: IncomingMessage, res: ServerResponse, blob: Open
 // go idle, unread or with no line to send, before the scan is stopped.
 const REPO_RISK_RECHECK_MS = 10_000
 const REPO_RISK_IDLE_MS = 5 * 60_000
+// A scan's part on a serverless deployment, within its function's 300s
+// (vercel.managed.json): past it the scan stops, its last line `continue`,
+// and the page asks again, from where it stopped (repository-risk.ts).
+const REPO_RISK_PART_MS = 270_000
 
 // The repositories a scan of risk reads, by its query: every connected one;
 // a team's (`team`, its id), each in the team's directories of it; or one
@@ -1379,6 +1383,14 @@ async function riskRepositories(db: ManagedDb, query: URLSearchParams): Promise<
   })
 }
 
+// Where a scan in parts keeps its progress (repository-risk.ts
+// ScanProgress) under `key`, for a day; none where the page gave no scan id.
+function scanProgress(deps: ManagedHttpDeps, key: string, scan: string | null) {
+  if (scan === null || !/^[\w-]{1,64}$/u.test(scan)) return undefined
+  const kept = scanReadStore<ScanProgress>(deps.db, 'progress', SCAN_READ_KEPT_MS, deps.config.debug)
+  return { read: () => kept.read(key), write: (progress: ScanProgress) => kept.write(key, progress, Date.now()) }
+}
+
 // GET /api/admin/repositories/risk: every connected repository's npm
 // packages, or a team's or one repository's (riskRepositories), as its
 // lockfiles lock them at its default branch's head, each assessed by risk
@@ -1404,8 +1416,19 @@ async function handleRepositoryRisk(res: ServerResponse, deps: ManagedHttpDeps,
   res.setTimeout(REPO_RISK_IDLE_MS, () => res.destroy())
   const browser = createRepositoryBrowser(deps.config, await repositoryBrowserUser(deps, s.user.id))
   const scopes = new Map<string, Promise<boolean>>()
+  const teamScoped = (name: string) => {
+    const scope = npmPackageScope(name)
+    if (scope === null) return Promise.resolve(false)
+    if (!scopes.has(scope)) scopes.set(scope, deps.db.hasTeamNpmScope(scope))
+    return scopes.get(scope)!
+  }
   const versions = npmVersionStore(deps.db, deps.config.debug)
+  const stores = { db: deps.db, npmCache: deps.npmCacheStorage, debug: deps.config.debug }
   const lines = repositoryRiskLines({
+    // Which repositories a scan in parts told whole, by the admin, the id the
+    // page gives the scan (`scan`) and its scope, for the next part to go on
+    // past them.
+    progress: scanProgress(deps, `${s.user.id}\0${query.get('scan')}\0${query.get('team') ?? ''}\0${query.get('repo') ?? ''}`, query.get('scan')),
     repos: () => Promise.resolve(repos),
     async open(repo) {
       const reader = await browser.reader(repo)
@@ -1418,21 +1441,32 @@ async function handleRepositoryRisk(res: ServerResponse, deps: ManagedHttpDeps,
         recheck: async () => { await reader.recheckAccess() },
       }
     },
-    teamScoped(name) {
-      const scope = npmPackageScope(name)
-      if (scope === null) return Promise.resolve(false)
-      if (!scopes.has(scope)) scopes.set(scope, deps.db.hasTeamNpmScope(scope))
-      return scopes.get(scope)!
+    teamScoped,
+    // Each package's downloads readied before it is assessed
+    // (scan-downloads.ts), so that none waits on npm's downloads API; a
+    // team's own not asked, as risk asks none of theirs.
+    downloads: {
+      async atOnce(names, ready, signal) {
+        const asked: string[] = []
+        for (const name of names) if (await teamScoped(name)) ready(name); else asked.push(name)
+        await npmReadyScanDownloads(asked, ready, stores, signal)
+      },
+      one: (name, signal) => npmReadyScanDownload(name, stores, signal),
     },
     // Public versions' documents kept in the database (npm-kept.ts), as the
     // viewer keeps them: whether npm still has one, its whole document tells
     // risk.
     version: async (name, version, signal) => found(await readNpmVersion(name, version, true, signal, versions)),
-    assess: (doc, teamScoped, signal) => npmRisk(doc, 'pick', { db: deps.db, npmCache: deps.npmCacheStorage, debug: deps.config.debug,
-      teamScoped, privileged: true, signal, spare: true }),
-    removed: (name, version, teamScoped, signal) => npmRiskRemoved(name, version, { db: deps.db, npmCache: deps.npmCacheStorage, debug: deps.config.debug,
-      teamScoped, privileged: true, signal }),
-    risks: riskLineStore(deps.db, RISK_KEPT_MS, deps.config.debug),
+    assess: (doc, ownScope, signal) => npmRisk(doc, 'pick', { ...stores, teamScoped: ownScope, privileged: true, signal, spare: true }),
+    removed: (name, version, ownScope, signal) => npmRiskRemoved(name, version, { ...stores, teamScoped: ownScope, privileged: true, signal }),
+    // Kept for good, a scan telling one past RISK_KEPT_MS at once, as stale.
+    risks: riskLineStore(deps.db, Infinity, deps.config.debug),
+    // What it read of GitHub kept a day, for a scan answered in parts to go
+    // on from where the last stopped, on whichever instance answers the next.
+    listings: scanReadStore(deps.db, 'listings', SCAN_READ_KEPT_MS, deps.config.debug),
+    lockfiles: scanReadStore(deps.db, 'lockfiles', SCAN_READ_KEPT_MS, deps.config.debug),
+    // And each lockfile's advisories, for as long as a scan tells them.
+    audits: scanReadStore(deps.db, 'advisories', RISK_KEPT_MS, deps.config.debug),
     // As a bundle's advisories are asked (bundle-advisories.ts), npm's registry
     // and, for packages anyone may read, OSV: no repository's. Each lockfile's
     // audit is bounded on its own, for one npm doesn't answer to fail alone and
@@ -1444,7 +1478,7 @@ async function handleRepositoryRisk(res: ServerResponse, deps: ManagedHttpDeps,
       if (audited.status !== 200) throw new NpmPackageError(502, audited.body.error)
       return audited.body
     },
-  }, controller.signal)
+  }, controller.signal, undefined, { partMs: deps.config.serverless ? REPO_RISK_PART_MS : null })
   let checked = Date.now()
   await sendLines(res, lines, async () => {
     if (deps.isShuttingDown()) return false
@@ -2614,12 +2648,15 @@ async function handleSetTeamNpmScopes(req: IncomingMessage, res: ServerResponse,
   sendJson(res, 200, { ok: true, scopes: (await deps.db.listTeamNpmScopes())[teamId] ?? [] })
 }
 
-// Who reads npm packages now: a session with workspace access, its role and
-// the npm scopes of its visible teams. Null for none.
-async function npmReader(deps: ManagedHttpDeps, cookie: string | undefined): Promise<NpmReader | null> {
+// Who reads npm packages now: a session with workspace access, its role and,
+// where they count for any of `names` it is asked about (npmScopesCount), the
+// npm scopes of its visible teams, else none: none are read for an unscoped
+// package, an admin or a manager, or an answer checked again that is public.
+// Null for none.
+async function npmReader(deps: ManagedHttpDeps, cookie: string | undefined, names: readonly string[]): Promise<NpmReader | null> {
   const s = await readSession(deps.config, deps.db, cookie, Date.now())
   if (s == null || !roleAtLeast(s.user.role, 'view')) return null
-  return { role: s.user.role, scopes: new Set(await deps.db.listUserNpmScopes(s.user.id)), userId: s.user.id }
+  return { role: s.user.role, scopes: new Set(npmScopesCount(s.user.role, names) ? await deps.db.listUserNpmScopes(s.user.id) : []), userId: s.user.id }
 }
 
 // What a registry read found, or a 404 for the package (handleNpm's catch
@@ -2636,7 +2673,7 @@ async function readableNpmReviews<T extends ReviewContext>(res: ServerResponse,
   const onClose = () => { if (!res.writableEnded) controller.abort() }
   res.on('close', onClose)
   try {
-    const reader = await npmReader(deps, cookie)
+    const reader = await npmReader(deps, cookie, reviews.flatMap(review => [review.package, ...review.basePackage ? [review.basePackage] : []]))
     if (!reader || reader.userId !== user) { sendJson(res, 401, { error: 'unauthenticated' }); return null }
     const documents = new Map<string, Promise<NpmVersionDocument | null>>()
     // Public versions' documents as the viewer keeps them (npm-kept.ts), a
@@ -2650,7 +2687,7 @@ async function readableNpmReviews<T extends ReviewContext>(res: ServerResponse,
     const packages = await Promise.all(reviews.map(review => Promise.all([
       read(review.package, review.version), ...(review.basePackage ? [read(review.basePackage, review.baseVersion!)] : []),
     ])))
-    const current = await npmReader(deps, cookie)
+    const current = await npmReader(deps, cookie, packages.flat().flatMap(doc => doc?.private ? [doc.name] : []))
     if (!current || current.userId !== user) { sendJson(res, 401, { error: 'unauthenticated' }); return null }
     return reviews.filter((_, index) => packages[index]!.every(doc => doc && (!doc.private || canReadPrivateNpm(current, doc.name))))
   } catch (error) {
@@ -2760,7 +2797,7 @@ async function handleNpmReviews(req: IncomingMessage, res: ServerResponse, deps:
   const onClose = () => { if (!res.writableEnded) controller.abort() }
   res.on('close', onClose)
   try {
-    const reader = await npmReader(deps, cookie)
+    const reader = await npmReader(deps, cookie, [context.package, ...context.basePackage ? [context.basePackage] : []])
     if (!reader || reader.userId !== session.user.id) { sendJson(res, 401, { error: 'unauthenticated' }); return }
     const kept = npmVersionStore(deps.db, deps.config.debug)
     const read = (pkg: string, spec: string) => readNpmVersion(pkg, spec, canReadPrivateNpm(reader, pkg), controller.signal, kept)
@@ -2791,7 +2828,7 @@ async function handleNpmReviews(req: IncomingMessage, res: ServerResponse, deps:
     }
     const limit = change?.action === 'publish' && change.final && change.final.status !== 'reject'
       ? await npmReviewPublicationLimit(deps, reader, doc, controller.signal) : 'accept'
-    const current = await npmReader(deps, cookie)
+    const current = await npmReader(deps, cookie, [doc, base].flatMap(item => item?.private ? [item.name] : []))
     if (!current || current.userId !== session.user.id) { sendJson(res, 401, { error: 'unauthenticated' }); return }
     if ([doc, base].some(item => item?.private && !canReadPrivateNpm(current, item.name))) { sendJson(res, 404, { error: 'package-not-found' }); return }
     if (!change) {
@@ -2834,7 +2871,7 @@ async function handleNpm(req: IncomingMessage, res: ServerResponse, deps: Manage
   }
   const diffed = path === NPM_RENAME_DIFF_PATH && params.has('path') ? renameDiffRequest(params) : null
   if (path === NPM_RENAME_DIFF_PATH && params.has('path') && !diffed) { sendJson(res, 400, { error: 'bad-file' }); return }
-  const reader = await npmReader(deps, cookie)
+  const reader = await npmReader(deps, cookie, [name])
   if (reader == null) { sendJson(res, 401, { error: 'unauthenticated' }); return }
   const controller = new AbortController()
   const onClose = () => { if (!res.writableEnded) controller.abort() }
@@ -2842,7 +2879,7 @@ async function handleNpm(req: IncomingMessage, res: ServerResponse, deps: Manage
   // A reader who lost their session, or their private access to an answer
   // that needed it, while the registry was asked gets nothing.
   const recheck = async (isPrivate: boolean) => {
-    const current = await npmReader(deps, cookie)
+    const current = await npmReader(deps, cookie, isPrivate ? [name] : [])
     if (current == null) { sendJson(res, 401, { error: 'unauthenticated' }); return false }
     if (isPrivate && !canReadPrivateNpm(current, name)) { sendJson(res, 404, { error: 'package-not-found' }); return false }
     return !res.destroyed
@@ -2963,7 +3000,7 @@ async function handleNpm(req: IncomingMessage, res: ServerResponse, deps: Manage
       if (!(await recheck(isPrivate))) return
       res.setTimeout(NPM_RESPONSE_IDLE_MS, () => res.destroy())
       await sendLines(res, lines, async () => {
-        const current = await npmReader(deps, cookie)
+        const current = await npmReader(deps, cookie, isPrivate ? [name] : [])
         return current != null && (!isPrivate || canReadPrivateNpm(current, name))
       })
       return

package/server-managed/locked-packages.ts

@@ -1,7 +1,8 @@
 // The packages a lockfile of npm's ecosystem locks, as @preventive/lockfile
 // reads package-lock.json and npm-shrinkwrap.json of `lockfileVersion: 3`,
 // pnpm-lock.yaml of pnpm 9 to 12 and Yarn 1's yarn.lock, each with where it
-// is installed from (repository-risk.ts).
+// is installed from and whether only devDependencies need it
+// (repository-risk.ts).
 import { LockfileError, parseNpmLockfile } from '@preventive/lockfile/npm.js'
 import { parsePnpmLockfile } from '@preventive/lockfile/pnpm.js'
 import { parseYarn1Lockfile } from '@preventive/lockfile/yarn1.js'
@@ -21,6 +22,11 @@ export interface LockedPackage {
   // A mirror's tarball's, as the lockfile has it: npm's package of its name
   // and version only where npm's integrity is among it.
   integrity?: string
+  // Where only the importers' devDependencies reach it, as npm's `dev` flag
+  // says, and an install of no devDependencies leaves it out: in npm's and
+  // pnpm's lockfiles. Yarn 1's says nothing of it, which only the
+  // package.json files do, and none of its are.
+  dev?: true
 }
 
 // A package as each reader resolves one, its name the one it is published
@@ -29,7 +35,7 @@ type Resolution =
   | { type: 'tarball'; tarball: string | undefined; integrity: string | undefined; gitHosted?: boolean }
   | { type: 'git'; repo: string }
   | { type: 'directory' }
-interface ReadPackage { name: string; version: string | undefined; resolution: Resolution | undefined }
+interface ReadPackage { name: string; version: string | undefined; resolution: Resolution | undefined; dev?: boolean | undefined }
 
 // The packages of a lockfile of `format`, as its reader resolves them.
 // Versions go unchecked against ranges, as only what is locked is asked for.
@@ -72,14 +78,20 @@ function locked({ name, version, resolution }: ReadPackage): LockedPackage | nul
 }
 
 // The packages a lockfile of `format` (npm, pnpm or yarn, as
-// findRepoLockfiles names them) locks, each once, in the lockfile's order.
-// Throws the reader's LockfileError or YamlError for one it refuses.
+// findRepoLockfiles names them) locks, each once, in the lockfile's order:
+// `dev` only where it is wherever it is installed, at each path of npm's
+// tree, and with each of pnpm's peers. Throws the reader's LockfileError or
+// YamlError for one it refuses.
 export function lockedPackages(format: LockfileFormat, text: string): LockedPackage[] {
   const found = new Map<string, LockedPackage>()
   for (const pkg of packagesOf(format, text)) {
     const one = locked(pkg)
-    const key = one && `${one.source}\0${one.name}@${one.version ?? ''}\0${one.from ?? ''}\0${one.integrity ?? ''}`
-    if (one && key && !found.has(key)) found.set(key, one)
+    if (!one) continue
+    if (pkg.dev === true) one.dev = true
+    const key = `${one.source}\0${one.name}@${one.version ?? ''}\0${one.from ?? ''}\0${one.integrity ?? ''}`
+    const had = found.get(key)
+    if (!had) found.set(key, one)
+    else if (had.dev && !one.dev) delete had.dev
   }
   return [...found.values()]
 }

package/server-managed/npm-kept.ts

@@ -5,9 +5,10 @@
 // (npm-manifest.ts npmVersionFields), for a day (readNpmVersion asks npm again
 // then, as a version can be deprecated, its package made private, or itself
 // removed); and, for scans, each package's risk line, for as long as a scan
-// keeps it (repository-risk.ts). Each store removes its own rows kept longer
-// than that as it writes, at most once an hour a database, so that rows no
-// read will take again don't pile up.
+// keeps it, and what a scan read of GitHub, for a day, and of npm's
+// advisories, for an hour (repository-risk.ts). Each store removes its own rows kept longer
+// than that as it writes, at most once an hour a database (pruneKept), so
+// that rows no read will take again don't pile up.
 // The database is an optimization: a failure reads as a miss, and writes
 // nothing.
 import { createHash } from 'node:crypto'
@@ -18,30 +19,45 @@ import type { UpstreamCacheStore } from './upstream-cache.ts'
 // A value kept by key, as JSON, `at` when it was.
 export interface KeptStore<T> {
   read(key: string): Promise<{ at: number; value: T } | null>
+  // Those of `keys` kept, by key, in one read of the database.
+  readMany(keys: string[]): Promise<Map<string, { at: number; value: T }>>
   write(key: string, value: T, at: number): Promise<void>
 }
 
-// When each database last had each type's old rows removed.
+// When each database last had the rows under each prefix removed.
 const PRUNE_EVERY_MS = 60 * 60_000
 const pruned = new WeakMap<UpstreamCacheStore, Map<string, number>>()
 
+// Removes the rows under `prefix` kept `keptMs` before `at`, at most once an
+// hour a database, as rows under it are written: here, and risk's downloads
+// (npm-risk.ts). A failure is `warn`ed of, and removes nothing.
+export async function pruneKept(db: UpstreamCacheStore, prefix: string, keptMs: number, at: number, warn: (error: unknown) => void) {
+  const times = pruned.get(db) ?? pruned.set(db, new Map()).get(db)!
+  if (at - (times.get(prefix) ?? -Infinity) < PRUNE_EVERY_MS) return
+  times.set(prefix, at)
+  try { await db.deleteUpstreamCacheEntries(prefix, at - keptMs) } catch (error) { warn(error) }
+}
+
 // Values of `type`, each read for `keptMs` after it was kept.
 function keptStore<T>(db: UpstreamCacheStore, type: string, keptMs: number, debug: boolean): KeptStore<T> {
   const warn = (what: string, error: unknown) => { if (debug) console.warn(`managed: kept ${type} ${what} failed:`, error) }
-  const prune = async (at: number) => {
-    const times = pruned.get(db) ?? pruned.set(db, new Map()).get(db)!
-    if (at - (times.get(type) ?? -Infinity) < PRUNE_EVERY_MS) return
-    times.set(type, at)
-    try { await db.deleteUpstreamCacheEntries(`${type}/`, at - keptMs) } catch (error) { warn('prune', error) }
-  }
+  // Kept for good (Infinity): none removed.
+  const prune = (at: number) => keptMs === Infinity ? Promise.resolve() : pruneKept(db, `${type}/`, keptMs, at, error => { warn('prune', error) })
   // Keys of any length or character, as the database's: hashed.
   const keyOf = (key: string) => `${type}/${createHash('sha256').update(key).digest('hex')}`
+  const valueOf = (row: string | null | undefined) => {
+    const kept: unknown = JSON.parse(row ?? 'null')
+    return plainObject(kept) && typeof kept['at'] === 'number' && 'value' in kept ? { at: kept['at'], value: kept['value'] as T } : null
+  }
   return {
     async read(key) {
+      try { return valueOf(await db.getUpstreamCacheEntry(keyOf(key))) } catch (error) { warn('read', error); return null }
+    },
+    async readMany(keys) {
       try {
-        const kept: unknown = JSON.parse(await db.getUpstreamCacheEntry(keyOf(key)) ?? 'null')
-        return plainObject(kept) && typeof kept['at'] === 'number' && 'value' in kept ? { at: kept['at'], value: kept['value'] as T } : null
-      } catch (error) { warn('read', error); return null }
+        const rows = await db.getUpstreamCacheEntries(keys.map(keyOf))
+        return new Map(keys.flatMap(key => { const kept = valueOf(rows.get(keyOf(key))); return kept ? [[key, kept] as const] : [] }))
+      } catch (error) { warn('read', error); return new Map() }
     },
     async write(key, value, at) {
       try { await db.setUpstreamCacheEntry(keyOf(key), JSON.stringify({ at, value }), at) } catch (error) { warn('write', error) }
@@ -61,5 +77,13 @@ export function npmVersionStore(db: UpstreamCacheStore, debug = false): NpmVersi
   }
 }
 
-// Risk lines, each read for `keptMs`, as long as a scan keeps them.
+// Risk lines, each kept `keptMs`, for good where Infinity, a scan telling one
+// past RISK_KEPT_MS as stale (repository-risk.ts).
 export const riskLineStore = <T>(db: UpstreamCacheStore, keptMs: number, debug = false): KeptStore<T> => keptStore<T>(db, 'repository-risk/lines', keptMs, debug)
+// What a scan read, each read for keptMs (repository-risk.ts): of GitHub, a
+// repository's lockfiles as listed at a commit (`listings`) and a lockfile's
+// packages by its blob (`lockfiles`), for SCAN_READ_KEPT_MS; of npm, a
+// lockfile's advisories by its blob (`advisories`), for RISK_KEPT_MS; and the
+// repositories a scan in parts told whole (`progress`).
+export const scanReadStore = <T>(db: UpstreamCacheStore, type: 'listings' | 'lockfiles' | 'advisories' | 'progress', keptMs: number, debug = false): KeptStore<T> =>
+  keptStore<T>(db, `repository-risk/${type}`, keptMs, debug)

package/server-managed/npm-packages.ts

@@ -43,7 +43,7 @@ export class NpmPackageError extends Error {
 }
 
 // Who reads, as the session has it now: the role, and the scopes of the
-// visible teams they are a member of.
+// visible teams they are a member of, where those count (npmScopesCount).
 export interface NpmReader { role: Role; scopes: ReadonlySet<string>; userId: string }
 
 // Whether a reader may have a package read with the server's token.
@@ -53,8 +53,19 @@ export function canReadPrivateNpm(reader: NpmReader, name: string): boolean {
   return scope !== null && reader.scopes.has(scope)
 }
 
+// Whether a reader's team scopes count for any of `names`, as
+// canReadPrivateNpm reads them: a scoped one, read by someone below manager.
+export const npmScopesCount = (role: Role, names: readonly string[]): boolean =>
+  role !== 'admin' && role !== 'manage' && names.some(name => npmPackageScope(name) !== null)
+
 export const npmToken = () => process.env['NPM_TOKEN'] || null
 
+// Whether a read of `name` that npm answers none anonymously is asked again
+// with the server's token: by a reader with private access (`privileged`),
+// where there is a token and the package is scoped, as npm's private packages
+// always are. Else the anonymous answer is the reader's.
+export const npmTokenReads = (name: string, privileged: boolean): boolean => privileged && npmToken() !== null && npmPackageScope(name) !== null
+
 function registryUrl(name: string, spec?: string): string {
   return `${NPM_REGISTRY}/${name}${spec === undefined ? '' : `/${encodeURIComponent(spec)}`}`
 }
@@ -138,16 +149,17 @@ async function publicWholeDocument(name: string, url: string, { limit, signal }:
   return json
 }
 
-// Asked anonymously first: a reader without private access stops there. One
-// with it asks again with the token, where there is one and the package is
-// scoped (npm's private packages always are), and the answer is private. A
-// whole document (`whole`) anyone may read is kept (publicWholeDocument).
-async function readDocument(name: string, url: string, privileged: boolean, { whole = false, ...options }: { accept: string; limit: number; signal: AbortSignal; whole?: boolean }) {
-  const open = whole ? await publicWholeDocument(name, url, options) : await registryDocument(url, { ...options, token: null })
+// Asked anonymously first, unless its caller knows npm has none for anyone
+// (`anonymous` false): a reader without private access stops there. One
+// with it asks again with the token where npmTokenReads says so, and the
+// answer is private. A whole document (`whole`) anyone may read is kept
+// (publicWholeDocument).
+async function readDocument(name: string, url: string, privileged: boolean,
+  { whole = false, anonymous = true, ...options }: { accept: string; limit: number; signal: AbortSignal; whole?: boolean; anonymous?: boolean }) {
+  const open = anonymous ? whole ? await publicWholeDocument(name, url, options) : await registryDocument(url, { ...options, token: null }) : null
   if (open) return { json: open, private: false }
-  const token = npmToken()
-  if (!privileged || !token || npmPackageScope(name) === null) return null
-  const json = await registryDocument(url, { ...options, token })
+  if (!npmTokenReads(name, privileged)) return null
+  const json = await registryDocument(url, { ...options, token: npmToken() })
   return json ? { json, private: true } : null
 }
 
@@ -244,10 +256,12 @@ export function readNpmVersions(name: string, privileged: boolean, signal: Abort
 // A package's whole document as npm has it for this reader, for risk
 // (npm-risk.ts), within the bounds and budget every document read here is
 // held to; null where npm has none for this reader, or the name is none npm
-// takes.
-export function readNpmPackument(name: string, privileged: boolean, signal: AbortSignal): Promise<Record<string, unknown> | null> {
+// takes. With `anonymous` false, where npm is known to have none for anyone,
+// it is asked only with the token (readDocument).
+export function readNpmPackument(name: string, privileged: boolean, signal: AbortSignal, { anonymous = true } = {}): Promise<Record<string, unknown> | null> {
   if (!isNpmPackageName(name)) return Promise.resolve(null)
-  return withDocumentBudget(PACKUMENT_BYTES, async () => (await readDocument(name, registryUrl(name), privileged, { accept: 'application/json', limit: PACKUMENT_BYTES, signal, whole: true }))?.json ?? null)
+  return withDocumentBudget(PACKUMENT_BYTES, async () => (await readDocument(name, registryUrl(name), privileged,
+    { accept: 'application/json', limit: PACKUMENT_BYTES, signal, whole: true, anonymous }))?.json ?? null)
 }
 
 const isPublishTime = (value: string | undefined) => typeof value === 'string' && value.length <= 40 && !Number.isNaN(Date.parse(value))