@preventive/triage 1.0.0-alpha.52 → 1.0.0-alpha.54

package/server-managed/github-pulls.ts

@@ -6,6 +6,9 @@ import { ensureUserAccessToken } from './github-oauth.ts'
 
 type Metadata = Pick<GithubMetadata, 'title' | 'description' | 'status' | 'stateReason'>
 const MAX_GITHUB_LOOKUPS = 200
+// A load reading this many open or unknown items anew also rechecks one closed
+// item, rounded up (lookupFixes).
+const CLOSED_RECHECK_EVERY = 10
 const GITHUB_LOOKUP_TIMEOUT_MS = 10_000
 const GITHUB_METADATA_TTL_MS = 60_000
 const GITHUB_PUBLIC_VISIBILITY_TTL_MS = 60_000
@@ -132,6 +135,32 @@ async function fetchMetadata(ref: GithubFixRef, token: string | null, fetchImpl:
   } catch { return null }
 }
 
+// What a load reads of its `entries`, by what is cached of them at `now`:
+// those missing or stale, the least recently checked first, as many as
+// MAX_GITHUB_LOOKUPS. Closed items are kept once fetched, but GitHub can
+// reopen them (merged ones are final): a load reading open or unknown items
+// anew also rechecks a tenth as many closed ones, the least recently checked
+// first, after them. None where every item is closed or fresh, and a busy
+// workspace cycles through its closed items, a recheck rotating its item
+// behind the others whether or not GitHub answers.
+function readsOf<Entry extends [string, GithubFixRef]>(entries: Entry[], metadata: Map<string, GithubMetadata>, now: number): Entry[] {
+  const lastCheckedAt = ([key]: Entry) => Math.max(metadata.get(key)?.fetchedAt ?? 0, metadata.get(key)?.attemptedAt ?? 0)
+  const oldestFirst = (a: Entry, b: Entry) => lastCheckedAt(a) - lastCheckedAt(b)
+  const stale = ([key, ref]: Entry) => {
+    const cached = metadata.get(key)
+    // Backfill closed issues cached before closure reasons existed, within the
+    // same authorized/capped queue. Even an unknown reason completes backfill.
+    return !cached || (ref.kind === 'issue' && cached.status === 'closed' && cached.stateReason == null)
+      || (['open', 'draft'].includes(cached.status) && cached.fetchedAt + GITHUB_METADATA_TTL_MS <= now)
+  }
+  const pending = entries.filter(stale).toSorted(oldestFirst).slice(0, MAX_GITHUB_LOOKUPS)
+  // Stale ones are missing, open, draft, or closed issues awaiting backfill.
+  const reread = pending.filter(([key]) => metadata.get(key)?.status !== 'closed').length
+  const rechecks = entries.filter(entry => metadata.get(entry[0])?.status === 'closed' && !stale(entry))
+    .toSorted(oldestFirst).slice(0, Math.ceil(reread / CLOSED_RECHECK_EVERY))
+  return [...pending, ...rechecks]
+}
+
 // The workspace handler checks user -> team membership and finding visibility.
 // Shared private metadata also requires this viewer's live GitHub access,
 // including for admins. Only recently verified public repos skip that check.
@@ -169,27 +198,17 @@ export async function lookupFixes(config: ManagedConfig, db: ManagedDb, snapshot
   const readable = await readableRepositories(repositories, context)
   const authorized = [...jobs.entries()].filter(([, ref]) => readable.has(ref.repo))
   const metadata = new Map((await db.listGithubMetadata(authorized.map(([key]) => key))).map(entry => [entry.key, entry]))
-  const lastCheckedAt = (key: string) => {
-    const cached = metadata.get(key)
-    return Math.max(cached?.fetchedAt ?? 0, cached?.attemptedAt ?? 0)
-  }
-  const pending = authorized.filter(([key, ref]) => {
-    const cached = metadata.get(key)
-    // Backfill closed issues cached before closure reasons existed, within the
-    // same authorized/capped queue. Even an unknown reason completes backfill.
-    return !cached || (ref.kind === 'issue' && cached.status === 'closed' && cached.stateReason == null)
-      || (['open', 'draft'].includes(cached.status) && cached.fetchedAt + GITHUB_METADATA_TTL_MS <= now)
-  }).toSorted(([a], [b]) => lastCheckedAt(a) - lastCheckedAt(b)).slice(0, MAX_GITHUB_LOOKUPS)
+  const reads = readsOf(authorized, metadata, now)
   // Only stale/missing, distinct, authorized entries consume the request budget.
   // Prioritize missing/oldest entries so a large workspace makes progress.
-  if (pending.length > 0) {
-    const publicRefreshes = new Set(pending.filter(([, ref]) => publicRepos.has(ref.repo) && !checks.verified.has(ref.repo)).map(([, ref]) => ref.repo))
+  if (reads.length > 0) {
+    const publicRefreshes = new Set(reads.filter(([, ref]) => publicRepos.has(ref.repo) && !checks.verified.has(ref.repo)).map(([, ref]) => ref.repo))
     if (publicRefreshes.size > 0) token = await verifyPublicRefreshes(config, snapshot.user.id, repositories.filter(repo => publicRefreshes.has(repo.github)), context)
     const fresh: Omit<GithubMetadata, 'attemptedAt'>[] = []
     const attempted: string[] = []
     // All waves and access checks share one deadline and four-call limit.
-    for (let i = 0; i < pending.length && !signal.aborted; i += 4) {
-      await Promise.all(pending.slice(i, i + 4).map(async ([key, ref]) => {
+    for (let i = 0; i < reads.length && !signal.aborted; i += 4) {
+      await Promise.all(reads.slice(i, i + 4).map(async ([key, ref]) => {
         if (signal.aborted) return
         if (metadata.has(key)) attempted.push(key)
         const result = await fetchMetadata(ref, publicRepos.has(ref.repo) && !checks.verified.has(ref.repo) ? null : token, fetchWithinDeadline)

package/server-managed/history-windows.ts

@@ -0,0 +1,172 @@
+// A default branch's history read by when its commits were made, for the
+// Pulse (repository-pulse.ts yearCommits): where the pages kept from a head
+// run out, the rest of the time it needs read at once, in windows of time
+// (GitHub's `history(since:, until:)`), not a page after another, each page
+// waiting on the last; and the pages GitHub would list from the commits read
+// (git log) kept by their tops, as reading a page at a time keeps them, so a
+// walk later, from a head moved on or in the Commits tab, reads them kept.
+import { linkNoreplyUsers } from './github-accounts.ts'
+import { githubGraphql } from './github-graphql.ts'
+import { COMMIT_FRAGMENT, type GraphCommit, HISTORY_PAGE, type HistoryCommit, type HistoryPages, PAGES_PARALLEL, PAGES_PER_QUERY, commitActors, graphCommit,
+  inBatches, variablesOf } from './repository-history.ts'
+
+// Windows a read is split into at most, each meant to hold as many commits
+// as WINDOW_COMMITS, a page's worth with room to spare, as often as the
+// commits read before it were made, as far back as they reach, the rest left
+// to the next read; pages read of them at most a read, those over going on
+// where a window holds more, as many times as MAX_WAVES, each waiting on the
+// last; and pages kept of what was read at most.
+const MAX_WINDOWS = 40
+const WINDOW_COMMITS = 70
+const WINDOW_PAGES = 50
+const MAX_WAVES = 5
+const MAX_KEPT = 500
+const KEPT_AT_ONCE = 10
+const WINDOWS_AFTER = 2
+const HOUR_MS = 60 * 60_000
+
+// Each window's commits from `$head`, `w<n>` the `n`th's: those made since
+// `$s<n>` (none, any time) and until `$u<n>`, after `$a<n>`.
+const windowsQuery = (count: number) => `query($owner: String!, $name: String!, $head: GitObjectID!, ${variablesOf('s', count, 'GitTimestamp')}, ${
+  variablesOf('u', count, 'GitTimestamp!')}, ${variablesOf('a', count, 'String')}) {
+  repository(owner: $owner, name: $name) {
+    object(oid: $head) { ... on Commit {
+      ${Array.from({ length: count }, (_, at) => `w${at}: history(first: ${HISTORY_PAGE}, since: $s${at}, until: $u${at}, after: $a${at}) {
+        pageInfo { hasNextPage endCursor } nodes { ...historyCommit } }`).join('\n      ')}
+    } }
+  }
+}
+${COMMIT_FRAGMENT}`
+
+type Window = { since: number | null; until: number; after: string | null; oldest?: number }
+type WindowPage = { pageInfo?: { hasNextPage?: unknown; endCursor?: unknown }; nodes?: GraphCommit[] } | null
+type Windows = { repository?: { object?: Record<string, WindowPage> | null } | null } | null
+// A time as GitHub takes it, to the second: a window's start rounded down,
+// its end up, so that those beside each other overlap rather than part.
+const stamp = (at: number, up: boolean) => new Date((up ? Math.ceil : Math.floor)(at / 1000) * 1000).toISOString()
+
+// The commits from `head` made from `low` to `high`, as often as `perMs` (commits
+// a millisecond) says they are made, as many windows as MAX_WINDOWS reach
+// down from `high`, read at once (inBatches), the latest first, each paged on
+// as far as it goes, as many pages as WINDOW_PAGES; and, those reaching
+// `low`, a page of those made before it besides, where a walk reading them
+// goes on from. Since when every one is read (`through`: where the windows
+// reach, each read whole, else, of the latest not, just after the earliest
+// made of those it read, as GitHub lists them the latest made first, those
+// made together left out; or its end, none read), and how many pages were
+// read (`read`).
+export async function readWindows(fullName: string, token: string, head: string, low: number, high: number, perMs: number, fetchImpl: typeof fetch) {
+  const [owner, name] = fullName.split('/') as [string, string]
+  const bottom = Math.max(low, high - MAX_WINDOWS * WINDOW_COMMITS / perMs)
+  const count = Math.min(MAX_WINDOWS, Math.max(1, Math.ceil((high - bottom) * perMs / WINDOW_COMMITS)))
+  const width = (high - bottom) / count
+  let open: Window[] = [...Array.from({ length: count }, (_, at) => ({ since: high - (at + 1) * width, until: high - at * width, after: null })),
+    ...bottom > low ? [] : [{ since: null, until: low - 1000, after: null }]]
+  const commits = new Map<string, HistoryCommit>()
+  let [read, waves] = [0, 0]
+  while (open.length > 0 && read < WINDOW_PAGES && waves++ < MAX_WAVES) {
+    const wave = open.slice(0, WINDOW_PAGES - read)
+    const next: Window[] = []
+    read += wave.length
+    await inBatches(wave, PAGES_PER_QUERY, PAGES_PARALLEL, async batch => {
+      const data = await githubGraphql(windowsQuery(batch.length), { owner, name, head, ...Object.fromEntries(batch.flatMap((window, at) => [
+        [`s${at}`, window.since === null ? null : stamp(window.since, false)], [`u${at}`, stamp(window.until, true)], [`a${at}`, window.after]])) }, token, fetchImpl) as Windows
+      const pages = batch.map((_, at) => data?.repository?.object?.[`w${at}`] ?? null)
+      await linkNoreplyUsers(pages.flatMap(page => page?.nodes ?? []).flatMap(commitActors), token, fetchImpl)
+      for (const [at, window] of batch.entries()) {
+        let oldest = window.oldest ?? Number.POSITIVE_INFINITY
+        for (const commit of (pages[at]?.nodes ?? []).flatMap(node => graphCommit(node) ?? [])) {
+          if (!commits.has(commit.sha)) commits.set(commit.sha, commit)
+          oldest = Math.min(oldest, commit.committedAt ?? Number.POSITIVE_INFINITY)
+        }
+        const info = pages[at]?.pageInfo
+        if (window.since !== null && info?.hasNextPage === true && typeof info.endCursor === 'string') next.push({ ...window, after: info.endCursor, oldest })
+      }
+    })
+    open = [...next, ...open.slice(wave.length)]
+  }
+  const through = Math.max(bottom, ...open.flatMap(window => window.since === null ? [] : [Number.isFinite(window.oldest) ? window.oldest! + 1 : window.until]))
+  return { commits: [...commits.values()], through, read }
+}
+
+// The page GitHub would list from `top` (git log: of those reached, the
+// latest made first, of those made together the first reached) of `known`
+// commits, every one made since `low` known; none where it would list one not
+// known, or one made before `low` while one not known waits beside it.
+export function gitLogPage(top: string, known: Map<string, HistoryCommit>, low: number): HistoryCommit[] | null {
+  if (!known.has(top)) return null
+  const page: HistoryCommit[] = []
+  const reached = new Set<string>()
+  const waiting: HistoryCommit[] = []
+  let unknown = 0
+  const reach = (sha: string) => {
+    if (reached.has(sha)) return
+    reached.add(sha)
+    const commit = known.get(sha)
+    if (!commit || commit.committedAt === null) { unknown++; return }
+    const at = waiting.findIndex(other => other.committedAt! < commit.committedAt!)
+    waiting.splice(at < 0 ? waiting.length : at, 0, commit)
+  }
+  reach(top)
+  while (page.length < HISTORY_PAGE && waiting.length > 0) {
+    const commit = waiting.shift()!
+    if (commit.committedAt! < low && unknown > 0) return null
+    page.push(commit)
+    for (const parent of commit.parents) reach(parent)
+  }
+  return page.length < HISTORY_PAGE && unknown > 0 ? null : page
+}
+
+// The pages from each of `tops` GitHub would list of `known` commits
+// (gitLogPage), kept by their tops, KEPT_AT_ONCE at a time, and from the
+// parents of those they list that none of `listed` is nor any kept lists, on
+// till none is left, as many as MAX_KEPT: as a walk reading pages goes on to
+// them.
+export async function keepPages(tops: string[], known: Map<string, HistoryCommit>, listed: Set<string>, low: number, pages: HistoryPages) {
+  const covered = new Set(listed)
+  const kept: [string, HistoryCommit[]][] = []
+  const waiting = [...tops]
+  while (waiting.length > 0 && kept.length < MAX_KEPT) {
+    const top = waiting.shift()!
+    if (covered.has(top)) continue
+    const page = gitLogPage(top, known, low)
+    if (!page) continue
+    kept.push([top, page])
+    for (const commit of page) covered.add(commit.sha)
+    for (const commit of page) for (const parent of commit.parents) if (!covered.has(parent) && known.has(parent)) waiting.push(parent)
+  }
+  for (let at = 0; at < kept.length; at += KEPT_AT_ONCE) await Promise.all(kept.slice(at, at + KEPT_AT_ONCE).map(([top, page]) => pages.write(top, page)))
+}
+
+// What a walk from `head` (yearCommits) reads next of `starts`: those a page
+// is kept from (`batch`), where any is, or all of them, a page at a time, as
+// long as it has read GitHub fewer times than WINDOWS_AFTER (`queries`), a
+// head moved on soon reaching the pages kept from where it was; else, none
+// kept, those made from
+// `low` to `high` read at once (readWindows), as often as those `found`
+// were made, told it (`found`), and the pages from `starts` on kept of them,
+// as far down as every one is read (keepPages), the queries that took
+// (`queries`, PAGES_PER_QUERY pages each, as many at once as PAGES_PARALLEL
+// a query of its) and none to read (`windowed`); where the windows can't be
+// read, all of them, a page at a time (`windowed` all the same).
+export async function windowsOn(fullName: string, token: string, head: string, starts: string[], found: Map<string, HistoryCommit>, low: number, high: number,
+  pages: HistoryPages, fetchImpl: typeof fetch, queries: number): Promise<{ batch: string[]; queries: number; windowed: boolean }> {
+  const kept = await pages.read(starts)
+  if (kept.size > 0 || queries < WINDOWS_AFTER || !(high > low) || !Number.isFinite(high)) {
+    return { batch: kept.size > 0 ? starts.filter(sha => kept.has(sha)) : starts, queries: 0, windowed: false }
+  }
+  let [newest, oldest, dated] = [Number.NEGATIVE_INFINITY, Number.POSITIVE_INFINITY, 0]
+  for (const { committedAt } of found.values()) {
+    if (committedAt === null) continue
+    [newest, oldest, dated] = [Math.max(newest, committedAt), Math.min(oldest, committedAt), dated + 1]
+  }
+  // From the one commit the walk goes on from, where there is one, none of
+  // those read listed again; else from the head.
+  const read = await readWindows(fullName, token, starts.length === 1 ? starts[0]! : head, low, high, dated / Math.max(newest - oldest, HOUR_MS), fetchImpl)
+    .catch((err: unknown) => { console.warn('managed: history windows read failed:', err); return null })
+  if (!read) return { batch: starts, queries: 0, windowed: true }
+  const listed = new Set(found.keys())
+  for (const commit of read.commits) if (!found.has(commit.sha)) found.set(commit.sha, commit)
+  await keepPages(starts, found, listed, read.through, pages)
+  return { batch: [], queries: Math.ceil(read.read / (PAGES_PER_QUERY * PAGES_PARALLEL)), windowed: true }
+}

package/server-managed/http.ts

@@ -76,6 +76,10 @@
 //   GET  /api/admin/reports/<id>/location → admin|manage live location suggestion | 401/403/404/503
 //   GET  /api/bundles/<id>/{metadata,contents} → authorized encoded bytes | 401/404/422
 //   GET  /api/bundles/<id>/advisories → published dependency advisories (security access) | 401/403/404
+//   GET  /api/bundles/<id>/dependency-review → what the npm versions it bundles want of review, as the lockfiles installing it lock
+//        them at the commit it was built from, where its repository's review is in effect for a reader of all of it | 401/404/409/413/422/429/502
+//   GET  /api/bundles/<id>/validation → its last validation the reader may see, else none; POST validates it anew: its own code
+//        as GitHub has it at its commit, its dependencies as upstream's packages have them, those covering every file | 401/403/404/409/422/429/502
 //   GET  /api/bundles/<id>/download → authorized original upload | 401/404
 //   GET  /api/bundles/<id>/pretty?path=&hash= → one of its files pretty-printed, Brotli-encoded | 400/401/404/409/413/422/429
 //   GET  /api/bundles/<id>/pretty?other= → the files that changed between it and another bundle, both sides' pretty-printed,
@@ -89,6 +93,7 @@
 //   GET  /api/admin/bundles      → admin all bundles; managers own/team bundles | 401/403
 //   POST /api/admin/bundles      → admin|manage uploads a bundle (raw body) | 401/403/413
 //   GET  /api/admin/bundles/<id> → admin|manage downloads a stored bundle | 401/403/404
+//   GET  /api/admin/bundles/<id>/usage → admin|manage the reports linked to it or declaring it, asked before a delete | 401/403/404
 //   DELETE /api/admin/bundles/<id> → admin|manage deletes a bundle | 401/403/404
 //   POST /api/admin/bundles/set-repo → admin|manage attaches/detaches a bundle's repo | 401/403/404
 //   GET  /api/admin/uploads/key  → admin|manage public key for sealed upload bodies | 401/403
@@ -139,17 +144,18 @@ import { backfillFindingIds, reportEntries } from '@preventive/report'
 import type { IncomingMessage, ServerResponse } from 'node:http'
 import { pipeline } from 'node:stream/promises'
 import { setTimeout as delay } from 'node:timers/promises'
-import { type KeptStore, REPOSITORY_READ_KEPT_MS, authoredCommitStore, branchTipStore, codeHeadStore, codeTreeStore, commitPullStore, githubTagStore, historyChangeStore, historyPageStore, historySquashStore, historyTreeStore,
+import { type KeptStore, REPOSITORY_READ_KEPT_MS, authoredCommitStore, branchTipStore, bundleValidationStore, codeHeadStore, codeTreeStore, commitPullStore, githubTagStore, historyChangeStore, historyPageStore, historySquashStore, historyTreeStore,
   issueListStore, lockfileReadStore,
   npmDeltaStore, npmPublishTimeStore, npmVersionStore, ownKeptPrunes, pullDependencyStore, pullListStore, repositoryReadStore, reviewBaselineStore, riskLineStore, scanReadStore } from './npm-kept.ts'
 import { plainObject } from './npm-manifest.ts'
 import { type NpmDelta, keptNpmDeltas, npmDelta } from './npm-delta.ts'
 import { ADVISORIES_TIMEOUT_MS, fetchBundleAdvisories } from './bundle-advisories.ts'
-import { auditCache, auditedRepos, lastListing } from './upstream-cache.ts'
+import { auditCache, auditedKeys, lastListing } from './upstream-cache.ts'
 import type { BundleAdvisoryInventory } from './bundle-advisory-inventory.ts'
 import { UPLOAD_CHUNK_BYTES, type UploadKind, deleteUpload, newUploadKey, openSessionUpload, putUploadPart, readUpload, uploadPublicKey, validUpload, validUploadPart } from './uploads.ts'
 import { UPLOAD_SEAL_HEADER, maxSealedBytes } from '../common/managed/upload-seal.ts'
-import { type BundleCache, type BundleCachePart, MAX_PACKAGE_INVENTORY_BYTES } from './bundle-cache.ts'
+import { type BundleCache, type BundleCachePart, MAX_PACKAGE_INVENTORY_BYTES, readBundleDetails } from './bundle-cache.ts'
+import { BUNDLE_VALIDATION_READ, type BundleValidation, type PlannedBundle, validateAtCommit, validateDependencies, validationPlan } from './bundle-validation.ts'
 import type { CacheStorage } from './cache-storage.ts'
 import { backfillBundleSummaries, bundleSummaries } from './bundle-catalog.ts'
 import { type CatalogCommit, backfillBundleCommits, bundleCommits, cacheCommitDetails } from './bundle-commits.ts'
@@ -170,7 +176,7 @@ import { filterReportData } from '../common/managed/report-filter.ts'
 import type { TriageEntryPatch } from '../common/managed/triage.ts'
 import { GITHUB_AGENTS } from '../common/managed/github-agents.ts'
 import { MAX_FINDING_ID, MAX_TRIAGE_BODY_BYTES, MAX_TRIAGE_ENTRIES, MAX_TRIAGE_HISTORY, parseTriageEntryPatch } from '../common/managed/triage.ts'
-import { reportRepoGithub } from '@preventive/report'
+import { reportRepoGithub, repoDirectory as stampedDirectory } from '@preventive/report'
 import { findingsRepository, loadManagedFindings, ownFileDirectory, readManagedReport } from '../common/managed/report-content.ts'
 import type { ReportSourcesCache } from './report-sources.ts'
 import { normalizeTeamPath } from './repo-path.ts'
@@ -231,7 +237,8 @@ import { type GithubAsked, type ListedLockfile, type LockfileRead, type Lockfile
   type PullDependencyStore, eachOf, lockfileFormat, pullDependenciesAt } from './pull-dependencies.ts'
 import { lockedAt, lockedIn, lockfilesAt } from './commit-lockfiles.ts'
 import { MAX_REQUIRED_REVIEWS, type RepoDependencyReview, SINCE_COMMIT } from './repository-dependency-reviews.ts'
-import { EVERY_LOCKFILE, heldTo, scopeOf } from './lockfile-scope.ts'
+import { EVERY_LOCKFILE, heldTo, reviewedIn, scopeOf } from './lockfile-scope.ts'
+import { type BundledVersion, bundleDependencies, installsBundle } from './bundle-dependency-reviews.ts'
 import { type ReviewBaselineStore, baselineFor, heldDependencies, reviewedDependencies, reviewedPackages } from './pull-dependency-reviews.ts'
 import { formattedPull, withFormatted } from './dependency-review-comments.ts'
 import type { ReviewedDependencies } from './reviewed-dependencies.ts'
@@ -359,7 +366,9 @@ async function handleWorkspaceFixes(res: ServerResponse, deps: ManagedHttpDeps,
   const storedIssues = await deps.db.listManagedIssues(ids)
   const issues = visibleManagedIssues(storedIssues, snapshot)
   const visibleIds = new Set(issues.map(issue => issue.findingId))
-  const urls = [...new Set([...manualUrls, ...issues.flatMap(issue => [issue.issueUrl!, ...(issue.autoFixUrl ? [issue.autoFixUrl] : [])])])]
+  // A saved issue's duplicate is read with it, as a kept redirect is followed:
+  // its status even where only cached metadata can be shown.
+  const urls = [...new Set([...manualUrls, ...issues.flatMap(issue => [issue.issueUrl, issue.autoFixUrl, issue.duplicateOfUrl].filter(url => url !== null))])]
   await recheckTeam(deps.db, s.session.id, snapshot)
   const result = issues.length > 0
     ? await lookupIssueLinks(deps.config, deps.db, snapshot, urls, storedIssues.filter(issue => visibleIds.has(issue.findingId)))
@@ -1566,7 +1575,7 @@ async function readRepositoryRefs(db: ManagedDb, reader: RepositoryReader, repo:
 
 function scopeRepositoryContents(contents: RepositoryContents | null, virtualEntries: RepositoryEntry[] | null) {
   return contents && { ...contents, ...(virtualEntries ? {
-    entries: virtualEntries, limited: false, packageEntryPoints: [], solidityEntryPoints: [], soliditySuggestionsLimited: false,
+    entries: virtualEntries, limited: false, configEntryPoints: [], packageEntryPoints: [], solidityEntryPoints: [], soliditySuggestionsLimited: false,
   } : {}) }
 }
 
@@ -3258,7 +3267,17 @@ async function handleRepositoryRisk(res: ServerResponse, deps: ManagedHttpDeps,
           (error: unknown) => { if (error instanceof GithubApiError && error.status === 404) return []; throw error }),
       }
     },
-    actions: githubActionsSource(user.token),
+    // References resolved with the admin's token, and each one's advisories
+    // asked as a bundle's `github` package's are (bundle-advisories.ts), one
+    // `internal` of its repository alone, each bounded on its own.
+    actions: { ...githubActionsSource(user.token), async advisories(name, asked, internal) {
+      const signal = AbortSignal.any([controller.signal, AbortSignal.timeout(ADVISORIES_TIMEOUT_MS)])
+      const packages = [{ ecosystem: 'github' as const, name, versions: asked, internal }]
+      const audited = await fetchBundleAdvisories(packages, signal, { debug: deps.config.debug, githubToken: user.token,
+        cache: auditCache(deps.config.upstreamCacheDir, deps.db, signal, deps.config.debug, auditedKeys(packages, false)) })
+      if (audited.status !== 200) throw new GithubApiError(502, audited.body.error)
+      return audited.body
+    } },
     actionFiles: scanReadStore(deps.db, 'action-files', SCAN_READ_KEPT_MS, deps.config.debug),
     actionLines: scanReadStore(deps.db, 'actions', RISK_KEPT_MS, deps.config.debug),
     teamScoped,
@@ -3291,10 +3310,11 @@ async function handleRepositoryRisk(res: ServerResponse, deps: ManagedHttpDeps,
     // and, for packages anyone may read, OSV: no repository's. Each lockfile's
     // audit is bounded on its own, for one npm doesn't answer to fail alone and
     // the scan to carry on.
-    async advisories(packages, scanning) {
+    async advisories(asked, scanning) {
       const signal = AbortSignal.any([scanning, AbortSignal.timeout(ADVISORIES_TIMEOUT_MS)])
-      const audited = await fetchBundleAdvisories(packages.map(pkg => ({ ecosystem: 'npm' as const, ...pkg })), signal,
-        { debug: deps.config.debug, cache: auditCache(deps.config.upstreamCacheDir, deps.db, signal, deps.config.debug) })
+      const packages = asked.map(pkg => ({ ecosystem: 'npm' as const, ...pkg }))
+      const audited = await fetchBundleAdvisories(packages, signal,
+        { debug: deps.config.debug, cache: auditCache(deps.config.upstreamCacheDir, deps.db, signal, deps.config.debug, auditedKeys(packages, false)) })
       if (audited.status !== 200) throw new NpmPackageError(502, audited.body.error)
       return audited.body
     },
@@ -3446,7 +3466,7 @@ async function handleBundleAdvisories(res: ServerResponse, deps: ManagedHttpDeps
     if (res.destroyed || (needsGithub && !(await authorize()))) return
     const result = await fetchBundleAdvisories(inventory.packages, controller.signal, {
       debug: deps.config.debug, repoAdvisories, details, githubToken,
-      cache: auditCache(deps.config.upstreamCacheDir, deps.db, controller.signal, deps.config.debug, auditedRepos(inventory.packages, repoAdvisories)),
+      cache: auditCache(deps.config.upstreamCacheDir, deps.db, controller.signal, deps.config.debug, auditedKeys(inventory.packages, repoAdvisories)),
     })
     if (res.destroyed || !(await authorize())) return
     // Return just inventory and public advisories, never source or report data.
@@ -3457,6 +3477,163 @@ async function handleBundleAdvisories(res: ServerResponse, deps: ManagedHttpDeps
   }
 }
 
+// Bundle `id` as `session` may read it now, else null, its refusal sent.
+async function readableBundle(res: ServerResponse, deps: ManagedHttpDeps, session: ManagedSession, id: string) {
+  const access = await deps.db.getBundleAccessSnapshot(session.id, Date.now(), id)
+  if (!access) { sendJson(res, 401, { error: 'unauthenticated' }); return null }
+  if (!access.bundle) { sendJson(res, 404, { error: 'no-bundle' }); return null }
+  return access.bundle
+}
+
+// `body` told of bundle `record` where `session` may still read it, and it is
+// still where it was read: else refused, as one of another repository.
+async function tellOfBundle(res: ServerResponse, deps: ManagedHttpDeps, session: ManagedSession, id: string, record: ManagedBundle, body: unknown) {
+  const now = await readableBundle(res, deps, session, id)
+  if (!now) return
+  if (now.repoId !== record.repoId || now.repoDirectory !== record.repoDirectory) sendJson(res, 409, { error: 'repository-changed' })
+  else sendJson(res, 200, body)
+}
+
+// GET /api/bundles/:id/dependency-review: what the npm package versions a
+// Stasis bundle holds (its advisory inventory's, bundle-advisory-inventory.ts)
+// want of review, where the repository it is attached to reviews
+// dependencies, as its Overview tells it (bundle-dependency-reviews.ts):
+// read as a default branch's are (branchDependencyAnswer), at the commit its
+// stamp names, of the lockfiles installing it there. None (`review: null`),
+// GitHub not asked, for a bundle of no repository, commit or npm versions,
+// one whose repository reviews none, and for a reader who may read only some
+// of it, as a team's Dependencies tab leaves such a one out; the bundle's
+// access checked again before it is told, and its repository not moved.
+async function handleBundleDependencyReview(res: ServerResponse, deps: ManagedHttpDeps, cookie: string | undefined, session: ManagedSession, id: string) {
+  const record = await readableBundle(res, deps, session, id)
+  if (!record) return
+  const none = () => { sendJson(res, 200, { review: null }) }
+  const repoId = record.repoId
+  if (record.kind !== 'stasis' || repoId === null || !deps.bundleCache) { none(); return }
+  const setting = await deps.db.repoDependencyReview(repoId)
+  if (!setting?.enabled) { none(); return }
+  const access = await wholeRepositoryAccess(deps, cookie, repoId)
+  if ('error' in access) {
+    if (PARTIAL_ACCESS.has(access.error)) none()
+    else sendRefusal(res, access)
+    return
+  }
+  // Its inventory first: building it, where it isn't yet, keeps its summary too.
+  let commit: string | undefined, inventory: BundleAdvisoryInventory | null | undefined
+  try {
+    inventory = await deps.bundleCache.advisoryInventory(record)
+    commit = (await deps.bundleCache.summary(record))?.commit
+  } catch { sendJson(res, 422, { error: 'bundle-unavailable' }); return }
+  if (inventory === null) { sendJson(res, 413, { error: 'payload-too-large' }); return }
+  const bundled = (inventory?.packages ?? []).filter(pkg => pkg.ecosystem === 'npm').flatMap(pkg => pkg.versions.map(version => ({ name: pkg.name, version })))
+  // Its commit as GitHub names one, of SHA-1.
+  if (!commit || !/^[a-f\d]{40}$/u.test(commit) || bundled.length === 0) { none(); return }
+  const read = await readWholeRepositoryOf(deps, access, (fullName, token, user) => bundleDependencyAnswer(deps, fullName, token, user, repoId, setting,
+    { commit: commit!, directory: record.repoDirectory, bundled }), () => wholeRepositoryAccess(deps, cookie, repoId))
+  if ('error' in read) { sendRefusal(res, read); return }
+  await tellOfBundle(res, deps, session, id, record, read.value)
+}
+
+// A bundle's answer, as GET /api/bundles/:id/dependency-review gives it, and
+// whether all of it was kept, GitHub asked for none of it: the versions
+// `bundled` (those of the bundle in `directory`, built from `commit`), as the
+// lockfiles there its review holds to lock them, judged by its review
+// (pull-dependency-reviews.ts), with how many it bundles (`versions`) and
+// those none of them lock (`unlocked`); none where its review holds to none
+// of the lockfiles installing it, there being some.
+async function bundleDependencyAnswer(deps: ManagedHttpDeps, fullName: string, token: string, user: StoredUser, repoId: number, setting: RepoDependencyReview,
+  { commit, directory, bundled }: { commit: string; directory: string; bundled: BundledVersion[] }): Promise<{ value: object; kept: boolean }> {
+  const asked: GithubAsked = { github: false }
+  const reads = lockfileKept(deps, repoId)
+  const installing = (await lockfilesAt(fullName, token, commit, reads, asked)).map(({ path }) => path).filter(path => installsBundle(path, directory))
+  if (!reviewedIn(setting.lockfiles, installing)) return { value: { review: null }, kept: !asked.github }
+  const read = await lockedAt(fullName, token, commit, reads, path => installsBundle(path, directory) && heldTo(setting.lockfiles, path), asked)
+  const { told, ...bundle } = bundleDependencies(bundled, read, commit)
+  const now = await deps.db.repoDependencyReviewState(repoId, user.id, reviewedPackages(told), setting)
+  if (!now) return { value: { review: null }, kept: !asked.github }
+  const state = { ...now, lockfiles: setting.lockfiles }
+  const reviewed = deps.reviewedDependencies ?? null
+  const { baseline, unread: baselineUnread, error } = reviewed ? { baseline: null, unread: [], error: null }
+    : await baselineFor(fullName, token, repoId, state, baselineKept(deps), reads, asked)
+  const { base: _base, ...answer } = await reviewedDependencies(told, state, { baseline, baselineUnread, baselineError: error, reviewed })
+  return { value: { ...answer, ...bundle }, kept: !asked.github }
+}
+
+// GET /api/bundles/:id/validation: the Stasis bundle's last validation its
+// reader may see, `{ validation }`; POST (`run`) validates it anew and keeps
+// it. Own code is checked only for a reader of all of its repository, and
+// told only while they still are; see doc/bundles.md#validation.
+const VALIDATIONS_AT_ONCE = 2
+const VALIDATION_MS = 240_000
+const validating = new Map<string, Promise<BundleValidation | Refusal>>()
+async function handleBundleValidation(req: IncomingMessage, res: ServerResponse, deps: ManagedHttpDeps, cookie: string | undefined, session: ManagedSession, id: string,
+  run: boolean) {
+  if (run && !await checkMutation(req, res, deps, cookie, { handed: true })) return
+  const record = await readableBundle(res, deps, session, id)
+  if (!record) return
+  if (record.kind !== 'stasis') { sendJson(res, 422, { error: 'unsupported-bundle' }); return }
+  const whole = record.repoId === null ? null : await wholeRepositoryAccess(deps, cookie, record.repoId)
+  if (whole && 'error' in whole && !PARTIAL_ACCESS.has(whole.error)) { sendRefusal(res, whole); return }
+  const access = whole && !('error' in whole) ? whole : null
+  const store = bundleValidationStore<BundleValidation>(deps.db, deps.config.debug)
+  const keyOf = (own: boolean) => `${record.id}\0${record.integrity}\0${record.repoId ?? ''}\0${record.repoDirectory}\0${own ? 'own' : 'dependencies'}\0${BUNDLE_VALIDATION_READ}`
+  // Why the reader of all of the repository may no longer be told its own
+  // code, asked once that is read; null where they still may.
+  const lostWhole = async () => {
+    if (!access) return null
+    const current = await wholeRepositoryAccess(deps, cookie, access.repo.repoId)
+    return 'error' in current ? current : sameRepository(current.repo, access.repo) ? null : refusal(409, 'repository-changed')
+  }
+  if (!run) {
+    const read = access ? await store.readFirst([keyOf(true)], [keyOf(false)]) : null
+    let kept = read ? [...read.found.values()][0]?.value : (await store.read(keyOf(false)))?.value
+    // The one that checked own code told only while its reader still reads all of the repository.
+    if (read?.first && await lostWhole()) kept = (await store.read(keyOf(false)))?.value
+    await tellOfBundle(res, deps, session, id, record, { validation: plainObject(kept) && typeof kept.status === 'string' ? kept : null })
+    return
+  }
+  const key = keyOf(access !== null)
+  const shared = access ? `${key}\0${access.user.id}` : key
+  let job = validating.get(shared)
+  if (!job) {
+    if (validating.size >= VALIDATIONS_AT_ONCE) { sendJson(res, 429, { error: 'validation-busy' }); return }
+    // Kept before it is told, for a reader asking next.
+    job = bundleValidation(deps, cookie, record, access).then(async validation => {
+      if (!('error' in validation)) await store.write(key, validation, validation.validatedAt)
+      return validation
+    }).finally(() => validating.delete(shared))
+    validating.set(shared, job)
+  }
+  let validation: BundleValidation | Refusal
+  try { validation = await job } catch (err) {
+    if (deps.config.debug) console.warn('managed: bundle validation failed:', err)
+    sendJson(res, 502, { error: 'validation-failed' }); return
+  }
+  if ('error' in validation) { sendRefusal(res, validation); return }
+  const lost = validation.own.checked ? await lostWhole() : null
+  if (lost) { sendRefusal(res, lost); return }
+  await tellOfBundle(res, deps, session, id, record, { validation })
+}
+
+// Bundle `record` validated (bundle-validation.ts): at the commit its stamp
+// names, with the GitHub access of `access`'s reader of all of its repository,
+// checked again after; else its dependencies alone, and why.
+async function bundleValidation(deps: ManagedHttpDeps, cookie: string | undefined, record: ManagedBundle, access: WholeAccess | null) {
+  let details
+  try { details = await readBundleDetails(record, deps.bundleStore) } catch { return refusal(422, 'bundle-unavailable') }
+  const plan = details?.bundle ? validationPlan(details.bundle as PlannedBundle) : null
+  if (!plan) return refusal(422, 'bundle-unavailable')
+  const stamp = (details!.bundle as { repo?: { commit?: unknown; directory?: unknown } } | undefined)?.repo
+  const commit = typeof stamp?.commit === 'string' && /^[a-f\d]{40}$/u.test(stamp.commit) ? stamp.commit : null
+  // Where it was built, as its stamp says, else where it is kept.
+  const directory = stampedDirectory(typeof stamp?.directory === 'string' ? stamp : { directory: record.repoDirectory })
+  const options = { deadline: Date.now() + VALIDATION_MS }
+  if (!access || !commit) return validateDependencies(plan, access ? 'no-commit' : record.repoId === null ? 'no-repository' : 'no-access', options)
+  const read = await readWholeRepositoryOf(deps, access, async (fullName, token) => ({ value: await validateAtCommit(plan, { fullName, token, commit, directory }, options), kept: false }),
+    () => wholeRepositoryAccess(deps, cookie, access.repo.repoId))
+  return 'error' in read ? read : read.value
+}
+
 // GET /api/admin/bundles — the uploaded bundles for the "Manage bundles" page.
 // admin|manage, read-only (no CSRF). `maxBytes` is the upload cap; `repos` feeds
 // the upload repo picker.
@@ -3686,6 +3863,27 @@ async function handleGetBundle(req: IncomingMessage, res: ServerResponse, deps:
   try { await pipeline(stored.stream, res) } catch { res.destroy() }
 }
 
+// GET /api/admin/bundles/<id>/usage — the reports that name a bundle, read
+// before deleting it (admin|manage, as the delete): linked to it, so their
+// findings show its code, or declaring it in their bundleHashes without the
+// link. Reports the caller can't read are counted, never named. Read-only, so
+// no CSRF. 404 when the caller can't manage the bundle.
+async function handleBundleUsage(res: ServerResponse, deps: ManagedHttpDeps, cookie: string | undefined, id: string): Promise<void> {
+  const s = await readManageSession(res, deps, cookie, { handed: true })
+  if (s == null) return
+  const bundle = await deps.db.getBundle(id)
+  if (!bundle || (s.user.role !== 'admin' && !await deps.db.userCanReadBundle(s.user.id, id))) { sendJson(res, 404, { error: 'no-bundle' }); return }
+  // Each report's readability is read with it, as canViewReport tells a
+  // manager's: one statement however many reports use the bundle.
+  const reports: { id: string; filename: string; linked: boolean }[] = []
+  let otherReports = 0
+  for (const report of await deps.db.listReportsForBundle(id, bundle.integrity, s.user.role === 'admin' ? undefined : s.user.id)) {
+    if (report.readable) reports.push({ id: report.id, filename: report.filename, linked: report.bundleId === id })
+    else otherReports++
+  }
+  sendJson(res, 200, { bundleId: id, reports, otherReports })
+}
+
 // DELETE /api/admin/bundles/<id> — remove a bundle (admin|manage). Mutation:
 // same-origin + CSRF. Drops the row (referencing reports' bundle_id null out via
 // the FK; their bundle_integrity stays, so a re-upload re-links), then the bytes
@@ -5397,6 +5595,18 @@ export function createManagedRequestHandler(deps: ManagedHttpDeps): Handler {
       await handleBundleAdvisories(res, deps, workspaceSession!.session, bundleAdvisories[1]!, url.searchParams.get('team'), url.searchParams.get('reason') ?? '', url.searchParams.get('repoAdvisories') === 'true', url.searchParams.get('details') === 'true')
       return
     }
+    const bundleValidationRoute = /^\/api\/bundles\/([a-f\d-]{36})\/validation$/iu.exec(path)
+    if (bundleValidationRoute) {
+      if (method !== 'GET' && method !== 'POST') { send405(res, 'GET, POST'); return }
+      await handleBundleValidation(req, res, deps, cookie, workspaceSession!.session, bundleValidationRoute[1]!, method === 'POST')
+      return
+    }
+    const bundleReview = /^\/api\/bundles\/([a-f\d-]{36})\/dependency-review$/iu.exec(path)
+    if (bundleReview) {
+      if (method !== 'GET') { send405(res, 'GET'); return }
+      await handleBundleDependencyReview(res, deps, cookie, workspaceSession!.session, bundleReview[1]!)
+      return
+    }
     const bundleRead = /^\/api\/bundles\/([a-f\d-]{36})\/(metadata|contents|download|pretty|rename-diff|type-only|comparison)$/iu.exec(path)
     if (bundleRead) {
       const id = bundleRead[1]!, part = bundleRead[2]!, { session } = workspaceSession!, { bundleCache, prettyCache } = deps
@@ -5436,6 +5646,11 @@ export function createManagedRequestHandler(deps: ManagedHttpDeps): Handler {
       if (method !== 'POST') { send405(res, 'POST'); return }
       await handleSetBundleVisible(req, res, deps, cookie); return
     }
+    const bundleUsage = /^\/api\/admin\/bundles\/([a-f\d-]{36})\/usage$/iu.exec(path)
+    if (bundleUsage) {
+      if (method !== 'GET') { send405(res, 'GET'); return }
+      await handleBundleUsage(res, deps, cookie, bundleUsage[1]!); return
+    }
     if (path.startsWith(BUNDLE_PREFIX)) {
       const id = path.slice(BUNDLE_PREFIX.length)
       if (method === 'GET') { await handleGetBundle(req, res, deps, workspaceSession!.session, id); return }

package/server-managed/index.ts

@@ -9,6 +9,7 @@ import { runReapers, withReap } from '../server-common/reap.ts'
 import { startServer } from '../server-common/standalone.ts'
 import { initializeApp } from '../server-common/initialize.ts'
 import { type ManagedConfig, loadManagedConfig } from './config.ts'
+import type { ManagedDb } from './db.ts'
 import { type ManagedHttpDeps, createManagedRequestHandler } from './http.ts'
 import { setNpmTarballCache } from './npm-loads.ts'
 import { assembledReviewedDependencies } from './reviewed-dependencies.ts'
@@ -21,6 +22,17 @@ import { setCacheDir } from '@preventive/upstream/npm.js'
 // without a timer or cron. Failed sweeps retry on later traffic after a minute.
 const REAP_INTERVAL_MS = 3_600_000
 const REAP_RETRY_MS = 60_000
+// A team's stored catalog summary (team-app-summaries.ts) not stored again in
+// this long: one still asked for is worked out again once.
+const TEAM_APP_SUMMARY_KEPT_MS = 30 * 86_400_000
+
+// Rows kept past their time: expired sessions, how many, and teams' catalog
+// summaries stored too long ago.
+async function reapRows(db: Pick<ManagedDb, 'deleteExpiredSessions' | 'deleteTeamAppSummaries'>): Promise<number> {
+  const now = Date.now()
+  const [sessions] = await Promise.all([db.deleteExpiredSessions(now), db.deleteTeamAppSummaries(now - TEAM_APP_SUMMARY_KEPT_MS)])
+  return sessions
+}
 
 // `ui: false` serves only the API: `out/` is never read.
 type ManagedAppOptions = Partial<Pick<ManagedHttpDeps, 'next' | 'serverInfo' | 'isShuttingDown' | 'reviewedDependencies'>> & { ui?: boolean }
@@ -79,7 +91,7 @@ async function assembleManagedApp(config: ManagedConfig, { ui = true, ...options
       let succeeded = false
       try {
         await runReapers({
-          sessions: async () => { sessions = await db.deleteExpiredSessions(Date.now()) },
+          sessions: async () => { sessions = await reapRows(db) },
           ...(storage.reapUploads ? { uploads: async () => { uploads = await storage.reapUploads!(maintenance.signal) } } : {}),
           ...('reapPrettyCopies' in storage ? { prettyCopies: () => storage.reapPrettyCopies(maintenance.signal) } : {}),
           ...('reapStorage' in storage ? { storage: () => storage.reapStorage(maintenance.signal) } : {}),