@preventive/triage 1.0.0-alpha.36 → 1.0.0-alpha.38

package/server-managed/repository-risk.ts

@@ -0,0 +1,413 @@
+// Every connected repository's npm packages, as its lockfiles lock them, each
+// assessed by risk (npm-risk.ts), for admins: GET /api/admin/repositories/risk
+// answers a line at a time as each is found and assessed, and stops once its
+// reader leaves (`signal`).
+//
+// A repository's lockfiles are listed at its default branch's head by
+// @preventive/upstream's findRepoLockfiles; of them, those of npm's ecosystem
+// that @preventive/lockfile reads (package-lock.json and npm-shrinkwrap.json
+// of `lockfileVersion: 3`, pnpm-lock.yaml of pnpm 9 to 12, and Yarn 1's
+// yarn.lock) are read for their packages. A package from npm's registry is
+// assessed once a scan, however many lockfiles lock it; one from another
+// host with an integrity, such as a company's mirror of npm's, as npm's
+// package of its name and version where npm's integrity is among it; one
+// from elsewhere (git, a tarball with no integrity, a file) is listed, not
+// assessed. Repositories are read one at a time, as GitHub's limits are the
+// caller's; packages are assessed meanwhile, as many at a time as npm-risk.ts
+// makes spare assessments. A lockfile's packages of npm's registry or a
+// mirror are asked of npm's advisories all at once, as `npm audit` asks, each
+// advisory told with whether it is of malware (risk's npmMalwareAdvisory),
+// which alone is a supply chain's risk; the rest are vulnerabilities, told
+// beside.
+import { setTimeout as delay } from 'node:timers/promises'
+import type { Advisory } from '@preventive/upstream/advisories.js'
+import type { LockfileFormat, RepoLockfile } from '@preventive/upstream/github.js'
+import { isExactVersion } from '@preventive/upstream/semver.js'
+import { NpmRiskError, SEVERITIES, type Severity, npmMalwareAdvisory } from '../risk/index.js'
+import { isNpmPackageName, isNpmPackageSpec } from '../common/managed/npm-packages.js'
+import { GithubApiError } from './github-errors.ts'
+import { NpmPackageError, type NpmVersionDocument } from './npm-packages.ts'
+import { type Lru, lru } from './lru.ts'
+import type { KeptStore } from './npm-kept.ts'
+import { MAX_SPARE_ASSESSMENTS } from './npm-risk.ts'
+import { type LockedPackage, lockedPackages } from './locked-packages.ts'
+
+// The formats of npm's ecosystem, as findRepoLockfiles names them, and those
+// of them read here.
+const NPM_LOCKFILE_FORMATS: ReadonlySet<LockfileFormat> = new Set(['npm', 'pnpm', 'yarn', 'bun', 'deno'])
+const READ_FORMATS: ReadonlySet<LockfileFormat> = new Set(['npm', 'pnpm', 'yarn'])
+
+// An assessment refused as busy (`npm-busy`, by npm-risk.ts or the npm reads
+// beneath it), or that npm asked to ask later (risk's `upstream-busy`, as its
+// downloads API answers past a burst of requests), is asked again after a
+// wait, up to this many times.
+const BUSY_RETRIES = 20
+const BUSY_WAIT_MS = 1000
+
+// The most severe of `findings`, or null for none.
+export function highestSeverity(findings: readonly { severity: Severity }[]): Severity | null {
+  return SEVERITIES.find(severity => findings.some(finding => finding.severity === severity)) ?? null
+}
+
+// A repository to read: where `paths` are given, a team's directories in it,
+// only the lockfiles in them or above them, as a workspace's root lockfile
+// locks them too.
+export interface RepositoryRiskRepo { repoId: number; fullName: string; paths?: readonly string[] }
+
+// Whether a lockfile at `path` is of `paths` (RepositoryRiskRepo).
+function lockfileOf(path: string, paths: readonly string[] | undefined) {
+  const dir = path.slice(0, Math.max(path.lastIndexOf('/'), 0))
+  return !paths || paths.some(scope => dir === '' || dir === scope || dir.startsWith(`${scope}/`) || scope.startsWith(`${dir}/`))
+}
+
+type ListedLockfile = Pick<RepoLockfile, 'path' | 'format' | 'sha'>
+
+// A repository opened at its default branch's head (`commit`): its lockfiles,
+// as findRepoLockfiles lists them, each with its blob's id, and a file's
+// text; `recheck` checks access to it again, throwing where it is gone.
+export interface OpenedRepository {
+  commit: string
+  lockfiles(): Promise<ListedLockfile[]>
+  file(path: string): Promise<string>
+  recheck(): Promise<void>
+}
+
+type Assessment = { published: string | null; findings: { id: string; severity: Severity; takenDown?: unknown }[] }
+
+export interface RepositoryRiskSource<R extends RepositoryRiskRepo> {
+  repos(): Promise<R[]>
+  open(repo: R): Promise<OpenedRepository>
+  // Whether a team lists a package's scope, its own packages.
+  teamScoped(name: string): Promise<boolean>
+  // npm's document of a version; an NpmPackageError where npm has none.
+  version(name: string, version: string, signal: AbortSignal): Promise<NpmVersionDocument>
+  assess(doc: NpmVersionDocument, teamScoped: boolean, signal: AbortSignal): Promise<Assessment>
+  // A version npm has no document of, as risk tells one npm removed: its
+  // malware alone, its package taken down by npm; an NpmRiskError where
+  // nothing tells of it (npm-risk.ts npmRiskRemoved).
+  removed(name: string, version: string, teamScoped: boolean, signal: AbortSignal): Promise<Assessment>
+  // npm's registry's advisories of these packages' versions, asked at once.
+  advisories(packages: { name: string; versions: string[] }[], signal: AbortSignal): Promise<Advisory[]>
+  // Where risk lines are kept beyond this instance, as RiskCaches keeps them
+  // in it (npm-kept.ts riskLineStore, the database's).
+  risks?: KeptStore<RiskLine>
+}
+
+type AdvisoryRow = Advisory & { malware: boolean }
+
+// Why a step failed, as a line says it: an error's code, and for a lockfile
+// refused, what of it its reader refused.
+type RiskFailure = { error: string; detail?: string }
+
+// A risk line is of npm's package of its name and version, or, with an
+// `integrity`, of a mirror's that its lockfiles hold to it.
+export type RepositoryRiskLine =
+  | { type: 'repos'; repos: { id: number; fullName: string }[] }
+  | ({ type: 'repo'; id: number } & ({ commit: string; lockfiles: { path: string; format: LockfileFormat }[] } | RiskFailure))
+  | ({ type: 'lockfile'; repo: number; path: string; format: LockfileFormat } & ({ packages: LockedPackage[] } | RiskFailure))
+  | ({ type: 'risk'; name: string; version: string; integrity?: string }
+    & ({ published: string | null; severity: Severity | null; findings: { id: string; severity: Severity; takenDown?: true }[] } | RiskFailure))
+  | ({ type: 'advisories'; repo: number; path: string } & ({ advisories: AdvisoryRow[] } | RiskFailure))
+  | { type: 'done' }
+
+// What an error is known by: a GitHub or npm failure's code, or `failed`;
+// risk's with what it asked of npm and how that failed, where it says.
+function failureOf(error: unknown): RiskFailure {
+  if (error instanceof GithubApiError || error instanceof NpmPackageError) return { error: error.message }
+  if (!(error instanceof NpmRiskError)) return { error: 'failed' }
+  return error.message === error.code ? { error: error.code } : { error: error.code, detail: error.message }
+}
+
+const isBusy = (error: unknown) => (error instanceof NpmPackageError && error.status === 429) || (error instanceof NpmRiskError && error.code === 'upstream-busy')
+
+type RiskLine = Extract<RepositoryRiskLine, { type: 'risk' }>
+type LockfileRead = { packages: LockedPackage[] } | RiskFailure
+
+// What scans keep between them, in this instance's memory: a repository's
+// lockfiles as listed at a commit, and a lockfile's packages by its blob,
+// neither of which changes; and a package's risk line and a lockfile's
+// advisories, asked again after RISK_KEPT_MS, as risk tells again what of a
+// package changes in time (a deprecation, its age, its downloads) and npm
+// publishes advisories. Each is told only once access to its repository is
+// checked as when it was read.
+export interface RiskCaches {
+  listings: Lru<ListedLockfile[]>
+  lockfiles: Lru<LockfileRead>
+  risks: Lru<{ at: number; line: RiskLine }>
+  advisories: Lru<{ at: number; rows: AdvisoryRow[] }>
+}
+export const riskCaches = (): RiskCaches => ({
+  listings: lru(1000),
+  // By the packages they hold, a few hundred bytes each.
+  lockfiles: lru(200_000, read => 'packages' in read ? read.packages.length + 1 : 1),
+  risks: lru(50_000),
+  advisories: lru(50_000, kept => kept.rows.length + 1),
+})
+export const RISK_KEPT_MS = 60 * 60_000
+const shared = riskCaches()
+
+// Items in the order they are added, each taken once by whichever reader asks
+// first: none once closed and all taken, or once `signal` aborts.
+interface Queue<T> { add(item: T): void; close(): void; take(): Promise<T | undefined> }
+function queue<T>(signal: AbortSignal): Queue<T> {
+  const items: T[] = []
+  const waiting: (() => void)[] = []
+  let closed = false
+  const wake = () => { for (const resolve of waiting.splice(0)) resolve() }
+  signal.addEventListener('abort', wake, { once: true })
+  return {
+    add(item) { items.push(item); wake() },
+    close() { closed = true; wake() },
+    async take() {
+      for (;;) {
+        if (signal.aborted) return undefined
+        const item = items.shift()
+        if (item !== undefined || closed) return item
+        await new Promise<void>(resolve => { waiting.push(resolve) })
+      }
+    },
+  }
+}
+
+// A package to assess: npm's of its name and version, or, with an
+// `integrity`, a mirror's held to it.
+interface AssessedPackage { name: string; version: string; integrity?: string }
+
+interface Scan {
+  source: RepositoryRiskSource<RepositoryRiskRepo>
+  caches: RiskCaches
+  signal: AbortSignal
+  lines: Queue<RepositoryRiskLine>
+  // Packages to assess, and those already asked for, by name, version and a
+  // mirror's integrity.
+  pending: Queue<AssessedPackage>
+  seen: Set<string>
+}
+
+// A package's risk line, its version's document kept and the assessment asked
+// again after a wait while the server is busy; null once the scan stops. A
+// version npm has no document of is told as risk tells one npm removed
+// (removedOf), its malware keeping whether its package was taken down.
+async function assess({ source, signal }: Scan, pkg: AssessedPackage, teamScoped: boolean): Promise<RiskLine | null> {
+  const at = { type: 'risk', ...pkg } as const
+  let doc: NpmVersionDocument | NpmPackageError | undefined
+  for (let attempt = 0; ; attempt++) {
+    try {
+      const read = doc ??= await documentOf(source, pkg, signal)
+      if (!(read instanceof NpmPackageError) && pkg.integrity !== undefined && !pkg.integrity.split(/\s+/u).includes(read.dist.integrity)) return { ...at, error: 'integrity-mismatch' }
+      const { published, findings } = read instanceof NpmPackageError ? await removedOf(source, pkg, teamScoped, signal, read) : await assessedOf(source, pkg, teamScoped, signal, read)
+      const told = findings.map(({ id, severity, takenDown }) => ({ id, severity, ...takenDown === true ? { takenDown: true as const } : {} }))
+      return { ...at, published, severity: highestSeverity(told), findings: told }
+    } catch (error) {
+      if (signal.aborted) return null
+      if (!isBusy(error) || attempt >= BUSY_RETRIES) return { ...at, ...failureOf(error) }
+      try { await delay(BUSY_WAIT_MS * Math.min(attempt + 1, 5), undefined, { signal }) } catch { return null }
+    }
+  }
+}
+
+// npm's document of a version, or its answer that it has none (a 404).
+async function documentOf(source: Scan['source'], pkg: AssessedPackage, signal: AbortSignal): Promise<NpmVersionDocument | NpmPackageError> {
+  try { return await source.version(pkg.name, pkg.version, signal) } catch (error) {
+    if (error instanceof NpmPackageError && error.status === 404) return error
+    throw error
+  }
+}
+
+// A version's assessment; where risk finds npm no longer has it, as a document
+// read from where versions' are kept doesn't tell, as one npm removed.
+async function assessedOf(source: Scan['source'], pkg: AssessedPackage, teamScoped: boolean, signal: AbortSignal, doc: NpmVersionDocument): Promise<Assessment> {
+  try { return await source.assess(doc, teamScoped, signal) } catch (error) {
+    if (!(error instanceof NpmRiskError && error.code === 'version-not-found')) throw error
+    return removedOf(source, pkg, teamScoped, signal, new NpmPackageError(404, 'package-not-found'))
+  }
+}
+
+// A version npm has no document of, as risk tells one npm removed, as it
+// removes every one it finds malware in (`removed`): `gone`, npm's answer of
+// none, thrown where nothing tells of it.
+async function removedOf(source: Scan['source'], pkg: AssessedPackage, teamScoped: boolean, signal: AbortSignal, gone: NpmPackageError): Promise<Assessment> {
+  try { return await source.removed(pkg.name, pkg.version, teamScoped, signal) } catch (error) {
+    throw error instanceof NpmRiskError && (error.code === 'version-not-found' || error.code === 'package-not-found') ? gone : error
+  }
+}
+
+// A package's risk line as kept (RiskCaches, else the source's `risks`), else
+// assessed and kept, in both, where it tells what risk found, or that a
+// mirror's copy isn't npm's.
+async function assessed(scan: Scan, pkg: AssessedPackage): Promise<RiskLine | null> {
+  let teamScoped
+  try { teamScoped = await scan.source.teamScoped(pkg.name) } catch (error) { return { type: 'risk', ...pkg, ...failureOf(error) } }
+  const key = `${pkg.name}@${pkg.version}\0${pkg.integrity ?? ''}\0${teamScoped}`
+  const kept = scan.caches.risks.get(key) ?? await keptLine(scan, key)
+  if (kept && Date.now() - kept.at < RISK_KEPT_MS) return kept.line
+  const line = await assess(scan, pkg, teamScoped)
+  if (line && (!('error' in line) || line.error === 'integrity-mismatch')) {
+    const at = Date.now()
+    scan.caches.risks.set(key, { at, line })
+    await scan.source.risks?.write(key, line, at)
+  }
+  return line
+}
+
+async function keptLine({ source, caches }: Scan, key: string) {
+  const hit = await source.risks?.read(key)
+  if (!hit) return null
+  caches.risks.set(key, { at: hit.at, line: hit.value })
+  return { at: hit.at, line: hit.value }
+}
+
+async function assessor(scan: Scan) {
+  for (let next = await scan.pending.take(); next !== undefined; next = await scan.pending.take()) {
+    const line = await assessed(scan, next)
+    if (line) scan.lines.add(line)
+  }
+}
+
+type LockfileLine = Extract<RepositoryRiskLine, { type: 'lockfile' }>
+
+// A lockfile's line, its packages or why they couldn't be had: as kept by its
+// blob, else read from GitHub and kept.
+async function readLockfile({ caches }: Scan, repo: number, opened: OpenedRepository, { path, format, sha }: ListedLockfile): Promise<LockfileLine> {
+  const at = { type: 'lockfile', repo, path, format } as const
+  if (!READ_FORMATS.has(format)) return { ...at, error: 'unsupported-format' }
+  const key = `${format}\0${sha}`
+  let read = caches.lockfiles.get(key)
+  if (!read) {
+    let text
+    try { text = await opened.file(path) } catch (error) { return { ...at, ...failureOf(error) } }
+    try { read = { packages: lockedPackages(format, text) } } catch (error) {
+      read = { error: 'lockfile-unreadable', ...error instanceof Error ? { detail: error.message.slice(0, 500) } : {} }
+    }
+    caches.lockfiles.set(key, read)
+  }
+  return { ...at, ...read }
+}
+
+// A lockfile's packages of npm's registry or a mirror not yet seen this scan,
+// given to the assessors.
+function assessLater(scan: Scan, packages: LockedPackage[]) {
+  for (const { name, version, source, integrity } of packages) {
+    const key = `${name}@${version}\0${integrity ?? ''}`
+    if ((source !== 'registry' && source !== 'mirror') || version === null || scan.seen.has(key) || !isNpmPackageName(name) || !isNpmPackageSpec(version)) continue
+    scan.seen.add(key)
+    scan.pending.add({ name, version, ...integrity === undefined ? {} : { integrity } })
+  }
+}
+
+// A repository opened and its lockfiles of npm's ecosystem listed, as kept
+// for its commit or else from GitHub, those of its `paths` alone; null, with
+// its line telling why, where either failed, or where the scan stopped.
+async function listLockfiles(scan: Scan, repo: RepositoryRiskRepo) {
+  try {
+    const opened = await scan.source.open(repo)
+    if (scan.signal.aborted) return null
+    const key = `${repo.repoId}@${opened.commit}`
+    let listed = scan.caches.listings.get(key)
+    if (!listed) scan.caches.listings.set(key, listed = await opened.lockfiles())
+    const lockfiles = listed.filter(({ path, format }) => NPM_LOCKFILE_FORMATS.has(format) && lockfileOf(path, repo.paths))
+    return { opened, lockfiles }
+  } catch (error) {
+    if (!scan.signal.aborted) scan.lines.add({ type: 'repo', id: repo.repoId, ...failureOf(error) })
+    return null
+  }
+}
+
+type AdvisoriesLine = Extract<RepositoryRiskLine, { type: 'advisories' }>
+
+// A lockfile's advisories line: npm's registry's advisories of its packages
+// of the registry or a mirror, by its blob as kept, else asked at once and
+// kept; null where it locks none, or once the scan stops.
+async function readAdvisories(scan: Scan, { repo, path, format, packages }: LockfileLine & { packages: LockedPackage[] }, sha: string): Promise<AdvisoriesLine | null> {
+  const at = { type: 'advisories', repo, path } as const
+  const key = `${format}\0${sha}`
+  const kept = scan.caches.advisories.get(key)
+  if (kept && Date.now() - kept.at < RISK_KEPT_MS) return { ...at, advisories: kept.rows }
+  const asked = new Map<string, Set<string>>()
+  for (const { name, version, source } of packages) {
+    if ((source !== 'registry' && source !== 'mirror') || !isNpmPackageName(name) || !isExactVersion(version)) continue
+    if (!asked.has(name)) asked.set(name, new Set())
+    asked.get(name)!.add(version)
+  }
+  if (asked.size === 0) return null
+  try {
+    const rows = (await scan.source.advisories([...asked].map(([name, versions]) => ({ name, versions: [...versions] })), scan.signal))
+      .map(advisory => ({ ...advisory, malware: npmMalwareAdvisory(advisory) }))
+    scan.caches.advisories.set(key, { at: Date.now(), rows })
+    return { ...at, advisories: rows }
+  } catch (error) {
+    return scan.signal.aborted ? null : { ...at, ...failureOf(error) }
+  }
+}
+
+// A repository's line, its lockfiles or why they couldn't be listed, and
+// then its lockfiles', and their advisories: told once they are all read
+// and access to the repository is checked again, as GitHub may have
+// withdrawn it meanwhile.
+async function readRepository(scan: Scan, repo: RepositoryRiskRepo) {
+  const listed = await listLockfiles(scan, repo)
+  if (!listed) return
+  const lockfiles: LockfileLine[] = []
+  for (const lockfile of listed.lockfiles) {
+    if (scan.signal.aborted) return
+    lockfiles.push(await readLockfile(scan, repo.repoId, listed.opened, lockfile))
+  }
+  try { await listed.opened.recheck() } catch (error) {
+    if (!scan.signal.aborted) scan.lines.add({ type: 'repo', id: repo.repoId, ...failureOf(error) })
+    return
+  }
+  if (scan.signal.aborted) return
+  scan.lines.add({ type: 'repo', id: repo.repoId, commit: listed.opened.commit, lockfiles: listed.lockfiles.map(({ path, format }) => ({ path, format })) })
+  for (const line of lockfiles) {
+    scan.lines.add(line)
+    if ('packages' in line) assessLater(scan, line.packages)
+  }
+  for (const [at, line] of lockfiles.entries()) {
+    if (scan.signal.aborted) return
+    const advisories = 'packages' in line ? await readAdvisories(scan, line, listed.lockfiles[at]!.sha) : null
+    if (advisories) scan.lines.add(advisories)
+  }
+}
+
+// Each repository, a repository at a time.
+async function readRepositories(scan: Scan) {
+  const repos = await scan.source.repos()
+  scan.lines.add({ type: 'repos', repos: repos.map(repo => ({ id: repo.repoId, fullName: repo.fullName })) })
+  for (const repo of repos) {
+    if (scan.signal.aborted) return
+    await readRepository(scan, repo)
+  }
+}
+
+// The scan's lines, in the order they are found: the repositories first,
+// then each repository's lockfiles and their packages once read and access
+// is checked again, and each package's risk once assessed, while the
+// repositories are still being read; `done` last. Ends at once, the work
+// under way given up on, where `signal` aborts or the lines are no longer read. A failure of
+// a repository, a lockfile or a package is told in its line; one of the scan
+// itself is thrown, the lines before it told. What it reads is kept in
+// `caches` (RiskCaches), the instance's own unless given.
+export async function* repositoryRiskLines<R extends RepositoryRiskRepo>(source: RepositoryRiskSource<R>, signal: AbortSignal, caches = shared): AsyncGenerator<RepositoryRiskLine> {
+  const stopped = new AbortController()
+  const running = AbortSignal.any([signal, stopped.signal])
+  const scan: Scan = { source, caches, signal: running, lines: queue(signal), pending: queue(running), seen: new Set() }
+  const failed: unknown[] = []
+  void (async () => {
+    const assessors = Array.from({ length: MAX_SPARE_ASSESSMENTS }, () => assessor(scan))
+    try {
+      try { await readRepositories(scan) } finally { scan.pending.close() }
+      await Promise.all(assessors)
+      if (!running.aborted) scan.lines.add({ type: 'done' })
+    } catch (error) {
+      failed.push(error)
+      stopped.abort()
+    } finally { scan.lines.close() }
+  })()
+  try {
+    for (let line = await scan.lines.take(); line !== undefined; line = await scan.lines.take()) yield line
+    if (failed.length > 0 && !signal.aborted) throw failed[0]
+  } finally {
+    // A GitHub request under way isn't waited on: it settles on its own.
+    stopped.abort()
+  }
+}

package/server-managed/session.ts

@@ -13,6 +13,14 @@ import { hashToken, randomToken } from './crypto.ts'
 // `cookieName` when cookies are Secure.
 export const STATE_COOKIE = 'dvstate'
 
+// Base names for GitHub issue authorization state (github-issue-oauth.ts) and
+// an administrator's read-only view as another user (view-as.ts).
+export const ISSUE_STATE_COOKIE = 'dvissuestate'
+export const VIEW_COOKIE = 'dvview'
+// The session cookie cannot share a name with these, or one would overwrite
+// or be read as the other.
+export const RESERVED_COOKIES: readonly string[] = [STATE_COOKIE, ISSUE_STATE_COOKIE, VIEW_COOKIE]
+
 // `__Host-`-prefix the base name when cookies are Secure (the prefix mandates
 // Secure + Path=/ + no Domain, which `buildCookie` already emits).
 export function cookieName(config: ManagedConfig, base: string): string {
@@ -62,11 +70,17 @@ export async function createSession(
 }
 
 // Resolve the current session (+ user) from the request's cookies, or null.
+// A view cookie resolves only to that admin's view as another user: a view
+// that has ended never falls back to the admin's own session. `own` skips the
+// view and reads the signed-in session itself.
 export function readSession(
-  config: ManagedConfig, db: ManagedDb, cookieHeader: string | undefined, now: number,
+  config: ManagedConfig, db: ManagedDb, cookieHeader: string | undefined, now: number, { own = false } = {},
 ): Promise<{ session: ManagedSession; user: StoredUser } | null> {
-  const token = parseCookies(cookieHeader).get(config.sessionCookieName)
+  const cookies = parseCookies(cookieHeader)
+  const token = cookies.get(config.sessionCookieName)
   if (token == null || token === '') return Promise.resolve(null)
+  const view = own ? undefined : cookies.get(cookieName(config, VIEW_COOKIE))
+  if (view != null && view !== '') return db.viewSessionWithUser(hashToken(view), hashToken(token), now)
   return db.sessionWithUser(hashToken(token), now)
 }
 

package/server-managed/sql.ts

@@ -22,7 +22,10 @@ export interface ManagedSqlDriver extends ManagedSql {
 const SINGLE_STATEMENTS = new Set(['getReport', 'getBundle', 'getStorageRow', 'getStorageEncryption',
   'listReports', 'listBundles', 'listReadableBundleIds', 'listAllRepos', 'listSelectedRepos', 'listRepoScopesForUser', 'listTriage',
   'getFeedState', 'getWorkspaceShareFeedState', 'claimMaintenanceLease', 'finishMaintenanceLease', 'cacheRepoDefaultBranch', 'releaseBundleBuildLease',
-  'getUpstreamCacheEntry', 'setUpstreamCacheEntry'])
+  'getUpstreamCacheEntry', 'getUpstreamCacheEntries', 'setUpstreamCacheEntry', 'ensureSessionUploadKey',
+  'listGithubCommits', 'listGithubCommitTags', 'setGithubCommits'])
+// Session presence and upload keys change only their own rows.
+const UNLOCKED_WRITES = new Set(['sessionWithUser', 'viewSessionWithUser', 'ensureSessionUploadKey'])
 
 export function scopeManagedMethods(methods: ManagedDb, driver: ManagedSqlDriver): ManagedDb {
   const entries = Object.entries(methods).map(([name, method]) => {
@@ -31,7 +34,7 @@ export function scopeManagedMethods(methods: ManagedDb, driver: ManagedSqlDriver
     return [name, (...args: unknown[]) => driver.scope(write, () => method(...args), {
       // Presence writes only one user's timestamp; they need no application
       // writer lock. Keep the session/user read and timestamp in one transaction.
-      lock: write && name !== 'sessionWithUser', statement: SINGLE_STATEMENTS.has(name),
+      lock: write && !UNLOCKED_WRITES.has(name), statement: SINGLE_STATEMENTS.has(name),
     })]
   })
   return Object.fromEntries(entries) as ManagedDb

package/server-managed/storage-encryption.ts

@@ -4,7 +4,7 @@ import { Readable } from 'node:stream'
 import { type StorageKey, decryptStorageStream, encryptStorageStream, wrapStorageValue } from '../server-common/storage-crypto.ts'
 import { ENCRYPTED_CACHE_PREFIX, type ObjectStorage, type RawObject, type RawObjectStorage, deleteObjects, objectPath, openObjectVersion } from './object-storage.ts'
 import { type StorageDb, type StorageEncryptionState, type StorageRow, type StorageRowKind, dataKeyIdentity, unwrapDataKey } from './storage-db.ts'
-import { inspectStorageObject, storageOwner, storageRowPaths, verifyStoragePayload } from './storage-payload.ts'
+import { PUBLIC_CACHE_PREFIX, inspectStorageObject, storageOwner, storageRowPaths, verifyStoragePayload } from './storage-payload.ts'
 
 const STORAGE_WRITE_MS = 180_000
 export const STORAGE_UPLOAD_TTL_MS = 86_400_000
@@ -22,6 +22,8 @@ function logicalKey(identity: string) {
 }
 const encryptedCachePath = (identity: string) => `${ENCRYPTED_CACHE_PREFIX}${identity.slice('cache/'.length)}`
 const deletionPaths = (identity: string) => identity.startsWith('cache/') ? [identity, encryptedCachePath(identity)] : [identity]
+// Public bytes, stored as they are whether or not storage is encrypted.
+const publicPath = (identity: string) => identity.startsWith('avatars/') || identity.startsWith(PUBLIC_CACHE_PREFIX)
 
 // A PUT can commit and still throw. Remove only this attempted plaintext,
 // comparing bytes rather than magic: arbitrary uploads can share the header.
@@ -199,7 +201,7 @@ export async function createEncryptedObjectStorage(raw: RawObjectStorage, db: St
   async function open(identity: string): ReturnType<ObjectStorage['open']> {
     logicalKey(identity)
     const state = await mode()
-    if (identity.startsWith('avatars/')) return raw.open(identity)
+    if (publicPath(identity)) return raw.open(identity)
     if (state) return openEncrypted(raw, db, key!, identity, state)
     const stored = await raw.open(identity)
     if (!stored) return null
@@ -231,7 +233,7 @@ export async function createEncryptedObjectStorage(raw: RawObjectStorage, db: St
       if (replaceReport) return convertStoredReport(raw, db, key, identity, bytes, mode)
       const state = await mode()
       const owner = storageOwner(identity)
-      if (identity.startsWith('avatars/')) { await raw.put(identity, bytes); return null }
+      if (publicPath(identity)) { await raw.put(identity, bytes); return null }
       if (!state) return putPlaintext(raw, identity, bytes, () => mode(true))
       let dataKey: Buffer, wrapped: string | null = null
       if (owner?.cache) {

package/server-managed/storage-payload.ts

@@ -15,6 +15,10 @@ export function storageRowPaths(type: StorageRowKind, row: Pick<StorageRow, 'id'
   return type === 'report' ? [`reports/${row.id}.br`, `reports/${row.id}`]
     : [`bundles/${row.id}${row.kind === 'sourcemap' ? '.map.br' : ''}`]
 }
+// A cache of public content, which no row owns, kept unencrypted as avatars
+// are: public npm versions' pretty-printed files (pretty-print.ts).
+export const PUBLIC_CACHE = 'npm'
+export const PUBLIC_CACHE_PREFIX = `cache/${PUBLIC_CACHE}/`
 export function storageOwner(identity: string): { type: StorageRowKind; id: string; cache: boolean } | null {
   const match = /^(reports|bundles)\/([^/]+)$/u.exec(identity)
   if (match) {

package/server-managed/storage-stores.ts

@@ -8,6 +8,7 @@ import { createBundleStore } from './bundle-store.ts'
 import { createReportStore } from './report-store.ts'
 import { CacheMissError, type CacheStorage, validateCacheKey } from './cache-storage.ts'
 import { type ObjectStorage, isBlobId } from './object-storage.ts'
+import { PUBLIC_CACHE } from './storage-payload.ts'
 
 function id(value: string): string {
   if (!isBlobId(value)) throw new Error('Invalid managed blob id')
@@ -79,5 +80,6 @@ export function createManagedStores(objects: ObjectStorage, avatarSidecar: boole
     avatarStore: avatarStore(objects, avatarSidecar),
     cacheStorage,
     reportSourcesStorage: cacheStore(objects, 'report-sources'),
+    npmCacheStorage: cacheStore(objects, PUBLIC_CACHE),
   }
 }

package/server-managed/storage.ts

@@ -2,6 +2,7 @@
 import { dirname } from 'node:path'
 import { parseStorageKey } from '../server-common/storage-crypto.ts'
 import { createBundleCache } from './bundle-cache.ts'
+import { createPrettyCache, stalePrettyRemover } from './pretty-print.ts'
 import { createReportSourcesCache } from './report-sources.ts'
 import { openNeonManagedDb } from './db-neon.ts'
 import type { ManagedConfig } from './config.ts'
@@ -51,9 +52,11 @@ export async function openManagedStorage(config: ManagedConfig) {
         await storage.reportStore.delete(report.id).catch(error => console.warn('managed: legacy link cleanup failed:', error))
       }
     }
-    return { ...storage, db, uploadStore: config.neonUrl ? storage.uploadStore : undefined,
-      bundleCache: createBundleCache(storage.cacheStorage, db, storage.bundleStore),
-      reportSourcesCache: createReportSourcesCache(storage.reportSourcesStorage, db, storage.reportStore, storage.bundleStore),
+    const bundleCache = createBundleCache(storage.cacheStorage, db, storage.bundleStore)
+    return { ...storage, db, uploadStore: config.neonUrl ? storage.uploadStore : undefined, bundleCache,
+      reportSourcesCache: createReportSourcesCache(storage.reportSourcesStorage, db, storage.reportStore, storage.bundleStore, bundle => bundleCache.sourcemapEdges(bundle)),
+      prettyCache: createPrettyCache(storage.cacheStorage, storage.npmCacheStorage, db, storage.bundleStore),
+      reapPrettyCopies: stalePrettyRemover(storage.npmCacheStorage),
       async reapStorage(signal?: AbortSignal) {
         if (!config.storageEncryptionMigrate || signal?.aborted) return
         const state = await db.getStorageEncryption()

package/server-managed/team-app.ts

@@ -0,0 +1,94 @@
+// Whether each team's sidebar entry opens collapsed to its App findings,
+// decided before the catalog is sent so the first paint is already final. A
+// team is classified from the same filtered, published workspace its view
+// loads. Results are keyed by the catalog's own content keys: an unchanged
+// catalog reuses them, and every change that can move a result also changes
+// the catalog revision that clients refresh on.
+import { type TeamAppMetadata, managedTeamAppMetadata } from '../common/managed/team-app.js'
+import type { BlobStore } from './blob-store.ts'
+import type { ManagedDb, TeamReportAccessSnapshot, UserTeam } from './db.ts'
+import type { WorkspaceShareSnapshot } from './workspace-shares.ts'
+import { TeamReportsError, loadTeamReportsResponse } from './team-reports.ts'
+
+export type { TeamAppMetadata }
+
+const CACHE_MAX_ENTRIES = 4096
+// Each workspace build already reads four reports at a time.
+const CLASSIFY_CONCURRENCY = 2
+const caches = new WeakMap<ManagedDb, Map<string, TeamAppMetadata>>()
+const pendingJobs = new WeakMap<ManagedDb, Map<string, Promise<TeamAppMetadata | null>>>()
+
+// Published reports with their content keys, and the team's own key (access,
+// scopes and links). Equal keys describe the same filtered workspace, whoever
+// reads it, so accounts with the same access share one classification.
+export function teamAppKey(team: UserTeam): string {
+  return JSON.stringify([team.cacheKey ?? null, published(team).map(report => [report.id, report.cacheKey]).toSorted()])
+}
+
+function published(team: UserTeam) {
+  return team.reports.filter(report => report.visible !== false)
+}
+
+function remember(db: ManagedDb, key: string, app: TeamAppMetadata): TeamAppMetadata {
+  let cache = caches.get(db)
+  if (!cache) { cache = new Map(); caches.set(db, cache) }
+  cache.delete(key)
+  cache.set(key, app)
+  for (const oldest of cache.keys()) {
+    if (cache.size <= CACHE_MAX_ENTRIES) break
+    cache.delete(oldest)
+  }
+  return app
+}
+
+// Null when the team cannot be classified right now: its catalog went stale,
+// the access snapshot is gone, or storage failed. The sidebar then keeps the
+// team expanded, and a later catalog asks again. Oversized and unreadable
+// workspaces never have an App view, so they are remembered as such.
+async function classify(db: ManagedDb, store: BlobStore, key: string, team: UserTeam,
+  snapshot: () => Promise<TeamReportAccessSnapshot | null>): Promise<TeamAppMetadata | null> {
+  const cached = caches.get(db)?.get(key)
+  if (cached) return remember(db, key, cached)
+  const ids = new Set(published(team).map(report => report.id))
+  if (ids.size === 0) return remember(db, key, { appMode: false })
+  let jobs = pendingJobs.get(db)
+  if (!jobs) { jobs = new Map(); pendingJobs.set(db, jobs) }
+  const existing = jobs.get(key)
+  if (existing) return existing
+  const job = (async () => {
+    const access = await snapshot()
+    // A catalog read before a concurrent publication names another report set.
+    if (!access?.teamId || access.reports.length !== ids.size || access.reports.some(report => !ids.has(report.id))) return null
+    const body = JSON.parse((await loadTeamReportsResponse(db, store, access)).toString('utf8')) as {
+      reports: { filename: string; data: unknown }[]; links?: string[][]
+    }
+    return remember(db, key, managedTeamAppMetadata(Object.assign(body.reports, { links: body.links ?? [] })))
+  })().catch((error: unknown) => {
+    if (error instanceof TeamReportsError && [413, 422].includes(error.status)) return remember(db, key, { appMode: false })
+    if (!(error instanceof TeamReportsError)) console.warn('managed: team App classification failed:', error)
+    return null
+  })
+  jobs.set(key, job)
+  try { return await job } finally { jobs.delete(key) }
+}
+
+// Classify a user's catalog, keyed by teamAppKey. `known` carries results from
+// an earlier read of the same request, so only changed teams are classified.
+export async function teamAppStates(db: ManagedDb, store: BlobStore, sessionId: string, teams: readonly UserTeam[],
+  known: ReadonlyMap<string, TeamAppMetadata | null> = new Map()): Promise<Map<string, TeamAppMetadata | null>> {
+  const states = new Map(known)
+  const missing = [...new Map(teams.map(team => [teamAppKey(team), team])).entries()].filter(([key]) => !states.has(key))
+  let next = 0
+  await Promise.all(Array.from({ length: Math.min(CLASSIFY_CONCURRENCY, missing.length) }, async () => {
+    while (next < missing.length) {
+      const [key, team] = missing[next++]!
+      states.set(key, await classify(db, store, key, team, () => db.getTeamReportAccessSnapshot(sessionId, Date.now(), team.id)))
+    }
+  }))
+  return states
+}
+
+// A public link's workspace is its own access snapshot.
+export function sharedTeamApp(db: ManagedDb, store: BlobStore, snapshot: WorkspaceShareSnapshot): Promise<TeamAppMetadata | null> {
+  return classify(db, store, JSON.stringify(['share', teamAppKey(snapshot.team)]), snapshot.team, () => Promise.resolve(snapshot))
+}