@preventive/triage 1.0.0-alpha.46 → 1.0.0-alpha.47

package/server-managed/db-neon.ts

@@ -35,11 +35,18 @@ export type PgConnect = () => Promise<PgConnection>
 // presence updates, and maintenance leases do not take it. Also fences migrations.
 const LOCK = 'SELECT pg_advisory_xact_lock(1937006964, 1835101793)'
 
+// Whether every version up to the latest is recorded, in one round trip at
+// a cold start: a database with no versions table yet (undefined_table) has
+// none. Outside a transaction, so the failed read aborts nothing.
+const SCHEMA_VERSIONS = 31
 async function currentSchema(db: PgConnection): Promise<boolean> {
-  const exists = (await db.query("SELECT to_regclass('managed_schema_version') AS name")).rows[0]?.['name']
-  if (!exists) return false
-  const versions = new Set((await db.query('SELECT version FROM managed_schema_version')).rows.map(row => Number(row['version'])))
-  return Array.from({ length: 31 }, (_, i) => i + 1).every(version => versions.has(version))
+  try {
+    const { rows } = await db.query(`SELECT count(*) AS n FROM managed_schema_version WHERE version BETWEEN 1 AND ${SCHEMA_VERSIONS}`)
+    return Number(rows[0]?.['n']) === SCHEMA_VERSIONS
+  } catch (error) {
+    if ((error as { code?: unknown })?.code === '42P01') return false
+    throw error
+  }
 }
 
 async function migrateRepositoryDefaultCache(db: PgConnection): Promise<void> {

package/server-managed/http.ts

@@ -135,7 +135,7 @@ import { mergeLinkGroups } from './link-reports.ts'
 import { backfillFindingIds, reportEntries } from '@preventive/report'
 import type { IncomingMessage, ServerResponse } from 'node:http'
 import { pipeline } from 'node:stream/promises'
-import { REPOSITORY_READ_KEPT_MS, historyChangeStore, historyPageStore, historySquashStore, historyTreeStore, npmDeltaStore, npmPublishTimeStore, npmVersionStore, pullDependencyStore,
+import { REPOSITORY_READ_KEPT_MS, historyChangeStore, historyPageStore, historySquashStore, historyTreeStore, npmDeltaStore, npmPublishTimeStore, npmVersionStore, ownKeptPrunes, pullDependencyStore,
   repositoryReadStore, reviewBaselineStore, riskLineStore, scanReadStore } from './npm-kept.ts'
 import { type NpmDelta, npmDelta } from './npm-delta.ts'
 import { ADVISORIES_TIMEOUT_MS, fetchBundleAdvisories } from './bundle-advisories.ts'
@@ -151,7 +151,7 @@ import { contentAccess } from './content-access.ts'
 import type { BundleStore } from './bundle-store.ts'
 import { Buffer } from 'node:buffer'
 import { createHash, randomUUID } from 'node:crypto'
-import type { AvatarStore } from './avatar-store.ts'
+import type { AvatarStore, CachedAvatar } from './avatar-store.ts'
 import type { BlobStore, OpenedBlob } from './blob-store.ts'
 import { bundleFilePrefix, bundleIntegrity, bundleKind, bundleRepo, reportBundleHashes } from './bundle.ts'
 import { resolveRepositoryImportLocation } from './repository-aliases.ts'
@@ -181,6 +181,7 @@ import { RISK_KEPT_MS, SCAN_READ_KEPT_MS, type ScanProgress, repositoryRiskLines
 import { WORKFLOWS_DIR } from './repository-actions.ts'
 import { githubActionsSource } from './github-actions.ts'
 import { type RepositoryEntry, createRepositoryBrowser, repositoryTreeFiles, scopedDirectory } from './repository-browser.ts'
+import { type Lru, lru } from './lru.ts'
 import { CALLBACK_PATH, LOGIN_PATH, OAuthError, buildLoginRedirect, cacheAvatar, ensureUserAccessToken, handleCallback } from './github-oauth.ts'
 import { clearCookie, endSession, readSession, runRequestPresence } from './session.ts'
 import { VIEW_AS_PATH, VIEW_ONLY_ERROR, clearViewCookie, endViewSession, runViewing, startViewSession, viewToken } from './view-as.ts'
@@ -468,14 +469,18 @@ function firstHeader(v: string | string[] | undefined): string | null {
 }
 
 // `key` is the image chosen for `id`; one not cached falls back to `id`'s own.
-async function serveAvatar(res: ServerResponse, avatarStore: AvatarStore, id: string, key = id): Promise<void> {
-  const avatar = await avatarStore.get(key) ?? (key === id ? null : await avatarStore.get(id))
+// A user's is never kept by the browser, as it changes with their profile.
+async function serveAvatar(res: ServerResponse, avatarStore: AvatarStore, id: string, key = id, cacheControl = 'no-store'): Promise<void> {
+  sendAvatar(res, await avatarStore.get(key) ?? (key === id ? null : await avatarStore.get(id)), cacheControl)
+}
+
+function sendAvatar(res: ServerResponse, avatar: CachedAvatar | null, cacheControl = 'no-store'): void {
   if (avatar == null) { sendJson(res, 404, { error: 'no-avatar' }); return }
   res.writeHead(200, {
     'content-type': avatar.contentType,
     'content-length': String(avatar.bytes.length),
     'x-content-type-options': 'nosniff',
-    'cache-control': 'no-store',
+    'cache-control': cacheControl,
   })
   res.end(avatar.bytes)
 }
@@ -489,15 +494,20 @@ async function serveAvatar(res: ServerResponse, avatarStore: AvatarStore, id: st
 const AGENT_AVATARS: ReadonlySet<string> = new Set([...APP_AGENTS, ...Object.values(GITHUB_AGENTS)]
   .flatMap(agent => 'avatar' in agent && agent.avatar ? [agent.avatar] : []))
 const AGENT_AVATAR = /^(app|user)-([1-9]\d{0,11})$/u
+// An agent's mark is a public GitHub image that never changes with anyone's
+// profile, drawn on every tile and comment by it: kept by the browser a day,
+// for it alone.
+const AGENT_AVATAR_CACHE = 'private, max-age=86400'
 
 async function serveAgentAvatar(res: ServerResponse, avatarStore: AvatarStore, avatar: string): Promise<void> {
   const [, kind, id] = AGENT_AVATAR.exec(avatar)!
   const key = `00000000-0000-0000-000${kind === 'app' ? 0 : 1}-${id!.padStart(12, '0')}`
-  if (await avatarStore.get(key) == null) {
-    await cacheAvatar(avatarStore, key, `https://avatars.githubusercontent.com/${kind === 'app' ? 'in' : 'u'}/${id}`, globalThis.fetch)
-      .catch((err) => { console.warn('managed: agent avatar caching failed:', err) })
-  }
-  await serveAvatar(res, avatarStore, key)
+  // Read once where it is kept; read again only past fetching it.
+  const kept = await avatarStore.get(key)
+  if (kept != null) { sendAvatar(res, kept, AGENT_AVATAR_CACHE); return }
+  await cacheAvatar(avatarStore, key, `https://avatars.githubusercontent.com/${kind === 'app' ? 'in' : 'u'}/${id}`, globalThis.fetch)
+    .catch((err) => { console.warn('managed: agent avatar caching failed:', err) })
+  await serveAvatar(res, avatarStore, key, key, AGENT_AVATAR_CACHE)
 }
 
 const MAX_JSON_BODY_BYTES = 4096
@@ -518,14 +528,17 @@ async function readBodyBytes(req: IncomingMessage, maxBytes: number): Promise<Bu
 }
 
 // A file upload's content, opening a body the browser sealed to this session
-// (common/managed/upload-seal.ts). Link reports are never sent in parts.
+// (common/managed/upload-seal.ts). Link reports are never sent in parts; the
+// parts of one sent in them are deleted behind its answer.
 async function readUploadBody(req: IncomingMessage, deps: ManagedHttpDeps, session: ManagedSession, kind: UploadKind | 'links'): Promise<Buffer> {
   const maxBytes = kind === 'bundles' ? deps.config.maxBundleBytes : deps.config.maxReportBytes
   const sealed = req.headers[UPLOAD_SEAL_HEADER] !== undefined
   const limit = sealed ? maxSealedBytes(maxBytes) : maxBytes
   let bytes
   if (kind === 'links' || req.headers['x-upload-id'] == null) bytes = await readBodyBytes(req, limit)
-  else if (deps.uploadStore) bytes = await readUpload(deps.uploadStore, req, session.id, kind, limit)
+  else if (deps.uploadStore) {
+    bytes = await readUpload(deps.uploadStore, req, session.id, kind, limit, cleanup => { deps.track(cleanup); deps.waitUntil?.(cleanup) })
+  }
   else throw new Error('bad-upload')
   return sealed ? await openSessionUpload(session.uploadKey, bytes, maxBytes) : bytes
 }
@@ -640,7 +653,7 @@ async function handleViewAs(req: IncomingMessage, res: ServerResponse, deps: Man
   const userId = (body as { userId?: unknown } | null)?.userId
   if (typeof userId !== 'string') { sendJson(res, 400, { error: 'bad-request' }); return }
   if (userId === s.user.id) { sendJson(res, 400, { error: 'cannot-view-as-self' }); return }
-  const target = (await deps.db.listUsers()).find(user => user.id === userId)
+  const target = await deps.db.getUser(userId)
   if (!target) { sendJson(res, 404, { error: 'not-found' }); return }
   // The insert itself rechecks the admin session and role.
   const setCookie = await startViewSession(deps.config, deps.db, s.session, userId, Date.now())
@@ -723,7 +736,7 @@ async function handleSetRole(req: IncomingMessage, res: ServerResponse, deps: Ma
   const role = (body as { role?: unknown } | null)?.role
   if (typeof userId !== 'string' || !isRole(role)) { sendJson(res, 400, { error: 'bad-request' }); return }
   if (userId === s.user.id) { sendJson(res, 403, { error: 'cannot-change-own-role' }); return }
-  const target = (await deps.db.listUsers()).find(user => user.id === userId)
+  const target = await deps.db.getUser(userId)
   // A caller can hold the body open while their admin access is revoked.
   if (await readAdminSession(res, deps, cookie) == null) return
   const ok = await deps.db.setUserRole(userId, role)
@@ -1260,8 +1273,8 @@ async function handleListReports(res: ServerResponse, deps: ManagedHttpDeps, ses
 // declared integrity that has a stored bundle wins (bundleId set). When none is
 // stored yet, keep the first declared integrity so a later bundle upload of it
 // re-links (see linkReportsToBundle). Returns the (bundleId, integrity) to store.
-async function resolveReportBundle(deps: ManagedHttpDeps, user: StoredUser, bytes: Buffer): Promise<{ bundleId: string | null; integrity: string | null }> {
-  const hashes = reportBundleHashes(bytes)
+async function resolveReportBundle(deps: ManagedHttpDeps, user: StoredUser, json: unknown): Promise<{ bundleId: string | null; integrity: string | null }> {
+  const hashes = reportBundleHashes(json)
   for (const h of hashes) {
     const b = await deps.db.getBundleByIntegrity(h)
     if (b != null && await canAccessBundle(deps, user, b.id)) return { bundleId: b.id, integrity: h }
@@ -1371,7 +1384,8 @@ async function handleUploadReport(req: IncomingMessage, res: ServerResponse, dep
   if (existing) { await sendUploadedReport(req, res, deps, cookie, existing, true); return }
   const id = randomUUID()
   const contentType = (firstHeader(req.headers['content-type']) ?? '').split(';', 1)[0]!.trim() || 'application/json'
-  const { bundleId, integrity } = await resolveReportBundle(deps, s.user, bytes)
+  // A JSON report's parse already in hand, not parsed again for its hashes.
+  const { bundleId, integrity } = await resolveReportBundle(deps, s.user, parsed.format === 'json' ? parsed.data : null)
   const dataKey = await deps.reportStore.put(id, bytes)
   let report: ReportRecord
   try {
@@ -1461,8 +1475,10 @@ async function repositoryBrowserUser(deps: ManagedHttpDeps, userId: string) {
 // user's repository access, for the next catalog to send.
 async function backfillCatalogCommits(deps: ManagedHttpDeps, userId: string, missing: readonly CatalogCommit[]): Promise<void> {
   let browser: ReturnType<typeof createRepositoryBrowser> | undefined
+  let repos: ReturnType<ManagedDb['listAllRepos']> | undefined
   await backfillBundleCommits(deps.db, userId, missing, async repoId => {
-    const repo = (await deps.db.listAllRepos()).find(item => item.repoId === repoId)
+    // Read once for the repositories the backfill reaches, not once each.
+    const repo = (await (repos ??= deps.db.listAllRepos())).find(item => item.repoId === repoId)
     if (!repo) return null
     browser ??= createRepositoryBrowser(deps.config, await repositoryBrowserUser(deps, userId))
     return browser.reader(repo)
@@ -1534,9 +1550,11 @@ function repositoryQuery(res: ServerResponse, query: URLSearchParams) {
 async function authorizeRepository(res: ServerResponse, deps: ManagedHttpDeps, cookie: string | undefined, repoId: number, path: string) {
   const s = await readWorkspaceSession(res, deps, cookie)
   if (!s) return null
-  const repo = (await bundleRepos(deps, s.user)).find(item => item.repoId === repoId)
+  // bundleRepos's reads, the reader's scopes read once for both.
+  const access = await contentAccess(deps.db, s.user)
+  const repo = (await deps.db.listSelectedRepos()).filter(access.bundle).find(item => item.repoId === repoId)
   if (!repo) { sendJson(res, 404, { error: 'no-repository' }); return null }
-  const scopes = s.user.role === 'admin' ? [null] : (await deps.db.listRepoScopesForUser(s.user.id)).filter(scope => scope.repoId === repoId).map(scope => scope.path)
+  const scopes = access.scopes == null ? [null] : access.scopes.filter(scope => scope.repoId === repoId).map(scope => scope.path)
   return { repo, virtualEntries: scopedDirectory(path, scopes), user: s.user }
 }
 
@@ -1602,6 +1620,22 @@ async function handleRepositoryBrowser(res: ServerResponse, deps: ManagedHttpDep
 // read, before any of it is sent. The load stays held until this response is
 // written or abandoned, and a reader who stops reading it gives it up after
 // a while.
+// What GitHub answered of a repository's trees for the code viewer, which a
+// commit or tree it names never changes: the tree at a directory of a commit
+// (getRepoTreeId reads the commit, then a listing a level down to it), and
+// how many files a tree holds under it and their bytes (repositoryTreeFiles),
+// by the repository's id, for the next to open it. Asked of them only once
+// the reader's own access has named the commit, as GitHub would have
+// answered them; by the database, as each instance has one.
+const REPOSITORY_TREES_KEPT = 4096
+type TreeFiles = Awaited<ReturnType<typeof repositoryTreeFiles>>
+const repositoryTrees = new WeakMap<ManagedDb, { ids: Lru<string>; files: Lru<TreeFiles> }>()
+function repositoryTreesOf(db: ManagedDb) {
+  let kept = repositoryTrees.get(db)
+  if (!kept) repositoryTrees.set(db, kept = { ids: lru(REPOSITORY_TREES_KEPT), files: lru(REPOSITORY_TREES_KEPT) })
+  return kept
+}
+
 async function handleRepositoryCode(res: ServerResponse, deps: ManagedHttpDeps, cookie: string | undefined, query: URLSearchParams): Promise<void> {
   const asked = repositoryQuery(res, query)
   if (!asked) return
@@ -1618,10 +1652,12 @@ async function handleRepositoryCode(res: ServerResponse, deps: ManagedHttpDeps,
     const commit = await reader.commit(ref)
     const repo = access.repo.fullName
     const client = createClient({ token: reader.readToken() })
+    const treeKey = `${access.repo.repoId}\0${commit}\0${path}`, trees = repositoryTreesOf(deps.db)
     // A directory the commit has none at is none to read, as browsing it says.
-    const tree = await upstreamGithub(() => client.getRepoTreeId({ repo, sha: commit, ...(path ? { directory: path } : {}) }).catch((err: unknown) => {
+    const tree = trees.ids.get(treeKey) ?? await upstreamGithub(() => client.getRepoTreeId({ repo, sha: commit, ...(path ? { directory: path } : {}) }).catch((err: unknown) => {
       throw err instanceof Error && err.name === 'AssertionError' && err.message.includes(' has no directory at ') ? new GithubApiError(404, 'no-directory') : err
     }))
+    trees.ids.set(treeKey, tree)
     // Access still held, and the same repository, before anything read is sent.
     const fenced = async () => {
       await reader.recheckAccess()
@@ -1631,12 +1667,15 @@ async function handleRepositoryCode(res: ServerResponse, deps: ManagedHttpDeps,
       return true
     }
     // Too large names the commit it was read at, for History to read.
-    if (!repositoryCodeFits(await repositoryTreeFiles(repo, reader.readToken(), tree))) {
+    const filesKey = `${access.repo.repoId}\0${tree}`
+    let listed = trees.files.get(filesKey)
+    if (listed === undefined) trees.files.set(filesKey, listed = await repositoryTreeFiles(repo, reader.readToken(), tree))
+    if (!repositoryCodeFits(listed)) {
       if (await fenced()) sendJson(res, 413, { error: 'directory-too-large', commit })
       return
     }
     if (res.destroyed) return
-    const load = loadRepositoryCodeBody({ repoId, repo, commit, path, tree }, () => upstreamGithub(() => client.getRepoTreeTarball({ repo, tree })))
+    const load = loadRepositoryCodeBody({ repoId, repo, commit, path, tree, listed }, () => upstreamGithub(() => client.getRepoTreeTarball({ repo, tree })))
     res.once('close', load.release)
     const bytes = await load.result
     if (!(await fenced()) || res.destroyed) return
@@ -2763,9 +2802,8 @@ async function handleCreateBundle(req: IncomingMessage, res: ServerResponse, dep
         throw new BundleBuildError(409, 'repository-changed')
       }
       signal.throwIfAborted()
-      const bytes = Buffer.from(built.bytes)
-      await storeUploadedBundle(req, res, deps, cookie, current, bytes, input.repoId, built.directory, built.filename, 'build',
-        bundleIntegrity(bytes), input.conditions)
+      await storeUploadedBundle(req, res, deps, cookie, current, built.bytes, input.repoId, built.directory, built.filename, 'build',
+        bundleIntegrity(built.bytes), input.conditions)
     })
   } catch (error) {
     if (res.destroyed || controller.signal.aborted) return
@@ -2846,15 +2884,17 @@ async function handleUploadBundle(req: IncomingMessage, res: ServerResponse, dep
     repo.repoId = null
     directory = ''
   }
-  await storeUploadedBundle(req, res, deps, cookie, s, bytes, repo.repoId, directory, filename, 'upload', integrity)
+  await storeUploadedBundle(req, res, deps, cookie, s, bytes, repo.repoId, directory, filename, 'upload', integrity, null, existing === null)
 }
 
+// `absent` where the caller has just found no bundle of `integrity`: not read
+// again, one inserted since meeting the insert's UNIQUE below as a race does.
 async function storeUploadedBundle(req: IncomingMessage, res: ServerResponse, deps: ManagedHttpDeps, cookie: string | undefined,
   s: { session: ManagedSession; user: StoredUser }, bytes: Buffer, repoId: number | null, directory: string, filename: string,
-  provenance: BundleProvenance, integrity = bundleIntegrity(bytes), buildConditions: BundleBuildConditions | null = null): Promise<void> {
+  provenance: BundleProvenance, integrity = bundleIntegrity(bytes), buildConditions: BundleBuildConditions | null = null, absent = false): Promise<void> {
   const kind = bundleKind(filename)
   const reuse = { provenance, filename, kind, buildConditions }
-  const existing = await deps.db.getBundleByIntegrity(integrity)
+  const existing = absent ? null : await deps.db.getBundleByIntegrity(integrity)
   if (existing) { await sendUploadedBundle(req, res, deps, cookie, existing, reuse); return }
   const id = randomUUID()
   const dataKey = await deps.bundleStore.put(id, bytes, kind)
@@ -2984,12 +3024,11 @@ async function handleMyTeams(res: ServerResponse, deps: ManagedHttpDeps, session
 // need a published report in their teams; ownership never overrides role none.
 async function canViewReport(deps: ManagedHttpDeps, user: StoredUser, reportId: string): Promise<boolean> {
   if (!roleAtLeast(user.role, 'view')) return false
-  const report = await deps.db.getReport(reportId)
+  const report = await deps.db.getReportView(user.id, reportId)
   if (report == null) return false
   if (user.role === 'admin') return true
-  if (user.role === 'manage') return report.uploadedBy === user.id || deps.db.userCanReadReport(user.id, reportId)
-  if (!report.visible) return false
-  return deps.db.userCanReadReport(user.id, reportId)
+  if (user.role === 'manage') return report.uploadedBy === user.id || report.readable
+  return report.visible && report.readable
 }
 
 // GET/HEAD /api/reports/:id/sources. Only the linked bundle's files cited by
@@ -4230,6 +4269,9 @@ async function handleWorkspaceShare(req: IncomingMessage, res: ServerResponse, d
 export function createManagedRequestHandler(deps: ManagedHttpDeps): Handler {
   const { config, db, avatarStore, isShuttingDown, track } = deps
   const repositoryDiscovery = new RepositoryDiscovery(config)
+  // Kept reads' prunes run behind their writes: tracked, so the database
+  // outlives them, and on serverless kept alive past the answer.
+  ownKeptPrunes(db, prune => { track(prune); deps.waitUntil?.(prune) })
 
   async function route(req: IncomingMessage, res: ServerResponse): Promise<void> {
     const url = new URL(req.url ?? '/', 'http://localhost')

package/server-managed/link-reports.ts

@@ -39,7 +39,11 @@ function decryptGroups(key: StorageKey | null, id: string, payload: string): str
   try { return JSON.parse(plain.toString('utf8')) as string[][] } finally { plain.fill(0) }
 }
 export async function linkRevision(db: ManagedSql): Promise<string> {
-  const rows = await db.prepare('SELECT id, sha256 FROM managed_link_report WHERE enabled = 1 ORDER BY id').all()
+  return linkRevisionOf(await db.prepare('SELECT id, sha256 FROM managed_link_report WHERE enabled = 1 ORDER BY id').all())
+}
+// The revision of the enabled link reports' `{ id, sha256 }` rows, by id,
+// however they were read.
+export function linkRevisionOf(rows: unknown[]): string {
   return rows.length > 0 ? createHash('sha256').update(JSON.stringify(rows)).digest('base64url') : ''
 }
 function parseLinkReport(content: string) {

package/server-managed/npm-kept.ts

@@ -38,15 +38,25 @@ export interface KeptStore<T> {
 // When each database last had the rows under each prefix removed.
 const PRUNE_EVERY_MS = 60 * 60_000
 const pruned = new WeakMap<UpstreamCacheStore, Map<string, number>>()
+// Who sees each database's prunes through (index.ts: its app, which waits
+// for them before closing it, and on serverless keeps its invocation alive
+// till they settle), as a write no longer waits for one.
+const pruneOwners = new WeakMap<UpstreamCacheStore, (prune: Promise<unknown>) => void>()
+export function ownKeptPrunes(db: UpstreamCacheStore, own: (prune: Promise<unknown>) => void) {
+  pruneOwners.set(db, own)
+}
 
 // Removes the rows under `prefix` kept `keptMs` before `at`, at most once an
 // hour a database, as rows under it are written: here, and risk's downloads
-// (npm-risk.ts). A failure is `warn`ed of, and removes nothing.
-export async function pruneKept(db: UpstreamCacheStore, prefix: string, keptMs: number, at: number, warn: (error: unknown) => void) {
+// (npm-risk.ts), each one run handed to the database's owner. A failure is
+// `warn`ed of, and removes nothing.
+export function pruneKept(db: UpstreamCacheStore, prefix: string, keptMs: number, at: number, warn: (error: unknown) => void): Promise<void> {
   const times = pruned.get(db) ?? pruned.set(db, new Map()).get(db)!
-  if (at - (times.get(prefix) ?? -Infinity) < PRUNE_EVERY_MS) return
+  if (at - (times.get(prefix) ?? -Infinity) < PRUNE_EVERY_MS) return Promise.resolve()
   times.set(prefix, at)
-  try { await db.deleteUpstreamCacheEntries(prefix, at - keptMs) } catch (error) { warn(error) }
+  const prune = (async () => { try { await db.deleteUpstreamCacheEntries(prefix, at - keptMs) } catch (error) { warn(error) } })()
+  pruneOwners.get(db)?.(prune)
+  return prune
 }
 
 // Values of `type`, each read for `keptMs` after it was kept.
@@ -79,7 +89,11 @@ function keptStore<T>(db: UpstreamCacheStore, type: string, keptMs: number, debu
     },
     async write(key, value, at) {
       try { await db.setUpstreamCacheEntry(keyOf(key), JSON.stringify({ at, value }), at) } catch (error) { warn('write', error) }
-      await prune(at)
+      // Behind the write, not before it is done: the prune reads the whole
+      // table (no index serves its prefix), which no writer need wait on.
+      // Its hour is marked before it runs, so writes meanwhile start none,
+      // and its database's owner sees it through (ownKeptPrunes).
+      void prune(at)
     },
     async remove(key) {
       try { await db.deleteUpstreamCacheEntries(keyOf(key), Number.MAX_SAFE_INTEGER) } catch (error) { warn('remove', error) }

package/server-managed/npm-loads.ts

@@ -9,8 +9,8 @@ import { join, resolve } from 'node:path'
 import { CompressionError, decompress } from '@preventive/archive/compression.js'
 import { encodeBrotli } from './brotli.ts'
 import { keptBriefly } from './kept-briefly.ts'
-import { MAX_NPM_PACKAGE_BYTES, MAX_NPM_PACKAGE_FILES, NPM_REGISTRY, NpmPackageError, type NpmPackageFile, type NpmVersionDocument, askUpstream,
-  npmFileRows, npmPublishTime, npmToken, readNpmTar } from './npm-packages.ts'
+import { MAX_NPM_PACKAGE_BYTES, MAX_NPM_PACKAGE_FILES, NPM_REGISTRY, NpmPackageError, type NpmPackageFile, type NpmVersionDocument, type TarReading,
+  askUpstream, npmFileRows, npmPublishTime, npmToken, readNpmTar } from './npm-packages.ts'
 import { keepFile, readKeptFile, upstreamCachePath } from './upstream-cache.ts'
 
 // The tar stream a package's files may come in, headers and padding included.
@@ -39,9 +39,10 @@ interface Load { job: Promise<unknown>; readers: number; settled: boolean; slot:
 // keeps its place among the active ones until its work is done and each
 // reader has released it, as a reader does once its response is written or
 // abandoned, so what responses still hold counts too. Places by the slot
-// their loads share, with how many hold each.
+// their loads share, with how many hold each and how many places it takes.
 const loads = new Map<string, Load>()
-const slots = new Map<string, number>()
+const slots = new Map<string, { holding: number; places: number }>()
+const placesTaken = () => [...slots.values()].reduce((sum, slot) => sum + slot.places, 0)
 
 const refused = <T>(status: number, code: string): NpmLoad<T> => {
   const result = Promise.reject(new NpmPackageError(status, code))
@@ -57,22 +58,32 @@ function shared<T>(what: string, doc: NpmVersionDocument, work: () => Promise<T>
   const { unpackedSize, fileCount } = doc.dist
   if ((unpackedSize ?? 0) > MAX_NPM_PACKAGE_BYTES || (fileCount ?? 0) > MAX_NPM_PACKAGE_FILES) return refused(413, 'package-too-large')
   const slot = tarballKey(doc)
-  return sharedLoad(`${what} ${slot}`, work, 'npm-busy', slot)
+  return sharedLoad(`${what} ${slot}`, work, 'npm-busy', { slot })
+}
+
+// The places among the active ones a load takes whose files hold `files` and
+// `bytes`: as many as packages they would fill, up to every one, each place
+// holding what a package may.
+export function loadPlaces({ files, bytes }: { files: number; bytes: number }): number {
+  return Math.min(MAX_ACTIVE_LOADS, Math.max(1, Math.ceil(files / MAX_NPM_PACKAGE_FILES), Math.ceil(bytes / MAX_NPM_PACKAGE_BYTES)))
 }
 
 // A load among the active ones, by `key`, refused as `busy` past them: a
 // repository directory's (repository-code.ts) as a package's, each holding
 // alike a tarball, its tar, its files and their encoded JSON. Loads of one
-// `slot` take one place between them.
-export function sharedLoad<T>(key: string, work: () => Promise<T>, busy: string, slot = key): NpmLoad<T> {
+// `slot` take one place between them, or `places` (loadPlaces) where they
+// hold more.
+export function sharedLoad<T>(key: string, work: () => Promise<T>, busy: string,
+  { slot = key, places = 1 }: { slot?: string; places?: number } = {}): NpmLoad<T> {
   let load = loads.get(key)
   if (!load) {
-    if (!slots.has(slot) && slots.size >= MAX_ACTIVE_LOADS) return refused(429, busy)
+    if (!slots.has(slot) && placesTaken() + places > MAX_ACTIVE_LOADS) return refused(429, busy)
     const created: Load = { job: work(), readers: 0, settled: false, slot }
     const finish = () => { settle(key, created) }
     created.job.then(finish, finish)
     loads.set(key, created)
-    slots.set(slot, (slots.get(slot) ?? 0) + 1)
+    const taken = slots.get(slot)
+    slots.set(slot, { holding: (taken?.holding ?? 0) + 1, places: taken?.places ?? places })
     load = created
   }
   const held = load
@@ -99,9 +110,9 @@ function settle(key: string, load: Load) {
 function done(key: string, load: Load) {
   if (load.readers !== 0 || !load.settled || loads.get(key) !== load) return
   loads.delete(key)
-  const holding = slots.get(load.slot)! - 1
-  if (holding === 0) slots.delete(load.slot)
-  else slots.set(load.slot, holding)
+  const taken = slots.get(load.slot)!
+  if (taken.holding === 1) slots.delete(load.slot)
+  else taken.holding--
 }
 
 // The sha512 an integrity names, in base64, or null where it names none.
@@ -201,10 +212,11 @@ function unpackedOf(doc: NpmVersionDocument): Promise<Unpacked> {
 
 // A gzipped tarball's files, its top directory left out, as npm extracts
 // a package's: a repository tree's as GitHub archives it, too, its symlinks
-// let climb as readNpmTar's `symlinksAbove` says.
-export async function readTarballFiles(tarball: Uint8Array, options: { symlinksAbove?: number } = {}): Promise<NpmPackageFile[]> {
+// let climb as readNpmTar's `symlinksAbove` says, and its tar stream and
+// files bounded as `options` give, a package's bounds unless given.
+export async function readTarballFiles(tarball: Uint8Array, { maxTarBytes = MAX_TAR_BYTES, ...options }: TarReading & { maxTarBytes?: number } = {}): Promise<NpmPackageFile[]> {
   let tar: Uint8Array
-  try { tar = await decompress(tarball, 'gzip', { limit: MAX_TAR_BYTES }) }
+  try { tar = await decompress(tarball, 'gzip', { limit: maxTarBytes }) }
   catch (err) {
     const limited = err instanceof CompressionError && err.limited
     throw new NpmPackageError(limited ? 413 : 422, limited ? 'package-too-large' : 'bad-tarball')

package/server-managed/npm-packages.ts

@@ -31,11 +31,12 @@ const PACKUMENT_BYTES = 64 * 1024 * 1024
 // limit until it is read and parsed: four packuments, or 32 version documents.
 const MAX_DOCUMENT_BYTES = 4 * PACKUMENT_BYTES
 // What a package may unpack to: the files' bytes and their count; the tar
-// stream holding them is bounded in npm-loads.ts.
+// stream holding them is bounded in npm-loads.ts. A repository directory's
+// are its own (repository-code.ts).
 export const MAX_NPM_PACKAGE_BYTES = 64 * 1024 * 1024
 export const MAX_NPM_PACKAGE_FILES = 20_000
 // The entries its tar stream may hold, read before any is counted as a file:
-// a directory's too, as a GitHub tree's tarball has one for each, npm's none.
+// a directory's too, though npm packs none.
 export const MAX_NPM_TAR_ENTRIES = 2 * MAX_NPM_PACKAGE_FILES
 // What the files may take as the viewer's JSON. Escaping grows a text up to
 // six times (a control character becomes `\u0001`), so it is counted before
@@ -367,18 +368,23 @@ export function npmPublishTime(doc: Pick<NpmVersionDocument, 'name' | 'version'
 
 export interface NpmPackageFile { path: string; bytes: Uint8Array }
 
+// How a tar stream is read (readNpmTar): how far its symlinks may climb above
+// it, and the entries, files and bytes it may hold, a package's unless given.
+export interface TarReading { symlinksAbove?: number; maxEntries?: number; maxFiles?: number; maxBytes?: number }
+
 // The regular files of a tar stream, by package path: npm drops the first
 // component (`package/`, or whatever the archive used), and a file at the top
 // is not extracted. The stream is read by @preventive/archive, which refuses
 // one malformed, a name or a symlink that would climb out of it -- a symlink
 // past `symlinksAbove` levels above it, none unless given -- and a name
-// repeated as another entry, and gives up past MAX_NPM_TAR_ENTRIES before
-// reading on; a file under another top directory at the same path replaces
-// an earlier one, as extraction would. Directories, links and devices hold no
+// repeated as another entry, and gives up past `maxEntries` before reading
+// on; a file under another top directory at the same path replaces an
+// earlier one, as extraction would. Directories, links and devices hold no
 // file content and are skipped.
-export function readNpmTar(tar: Uint8Array, { symlinksAbove = 0 }: { symlinksAbove?: number } = {}): NpmPackageFile[] {
+export function readNpmTar(tar: Uint8Array, { symlinksAbove = 0, maxEntries = MAX_NPM_TAR_ENTRIES, maxFiles = MAX_NPM_PACKAGE_FILES,
+  maxBytes = MAX_NPM_PACKAGE_BYTES }: TarReading = {}): NpmPackageFile[] {
   let entries: Entry[]
-  try { entries = unpack(tar, { maxEntries: MAX_NPM_TAR_ENTRIES, symlinksAbove }) }
+  try { entries = unpack(tar, { maxEntries, symlinksAbove }) }
   catch (err) {
     if (err instanceof ArchiveError) throw err.limited ? new NpmPackageError(413, 'package-too-large') : new NpmPackageError(422, 'bad-tarball')
     throw err
@@ -391,7 +397,7 @@ export function readNpmTar(tar: Uint8Array, { symlinksAbove = 0 }: { symlinksAbo
     const path = name.slice(slash + 1)
     total += data.length - (files.get(path)?.length ?? 0)
     files.set(path, data)
-    if (files.size > MAX_NPM_PACKAGE_FILES || total > MAX_NPM_PACKAGE_BYTES) throw new NpmPackageError(413, 'package-too-large')
+    if (files.size > maxFiles || total > maxBytes) throw new NpmPackageError(413, 'package-too-large')
   }
   return [...files].map(([path, bytes]) => ({ path, bytes })).toSorted((a, b) => a.path < b.path ? -1 : a.path > b.path ? 1 : 0)
 }
@@ -421,13 +427,14 @@ const ROW_LENGTH = 80
 
 // The files as the viewer gets them: `[path, bytes, text]`, text null for a
 // file that is not UTF-8, which instead carries its bytes' sha256, so a
-// comparison of two versions tells a changed one from an unchanged one.
-export function npmFileRows(files: NpmPackageFile[]): ([string, number, string] | [string, number, null, string])[] {
+// comparison of two versions tells a changed one from an unchanged one. They
+// may take `maxLength` characters of JSON, a package's unless given.
+export function npmFileRows(files: NpmPackageFile[], maxLength = MAX_NPM_JSON_LENGTH): ([string, number, string] | [string, number, null, string])[] {
   let length = 0
   return files.map(({ path, bytes }) => {
     const text = npmFileText(bytes)
     length += ROW_LENGTH + jsonLength(path) + (text === null ? 0 : jsonLength(text))
-    if (length > MAX_NPM_JSON_LENGTH) throw new NpmPackageError(413, 'package-too-large')
+    if (length > maxLength) throw new NpmPackageError(413, 'package-too-large')
     return text === null ? [path, bytes.byteLength, null, `sha256-${createHash('sha256').update(bytes).digest('base64')}`] : [path, bytes.byteLength, text]
   })
 }