@preventive/triage 1.0.0-alpha.47 → 1.0.0-alpha.49

package/server-managed/index.ts

@@ -44,7 +44,7 @@ export async function createManagedApp(config: ManagedConfig, options: ManagedAp
 
 async function assembleManagedApp(config: ManagedConfig, { ui = true, ...options }: ManagedAppOptions, rollback: AsyncDisposableStack, waitUntil?: (promise: Promise<unknown>) => void) {
   const storage = await openManagedStorage(config)
-  const { db, avatarStore, reportStore, bundleStore, bundleCache, reportSourcesCache, prettyCache, npmCacheStorage } = storage
+  const { db, avatarStore, reportStore, bundleStore, bundleCache, reportSourcesCache, prettyCache, npmCacheStorage, repositoryCodeStorage } = storage
   rollback.defer(() => db.close())
   const originGate = createOriginGate(config.host, config.trustProxyEnv)
 
@@ -60,7 +60,7 @@ async function assembleManagedApp(config: ManagedConfig, { ui = true, ...options
     indexOnly: options.next != null, scanServer: options.serverInfo?.deepviewScanServer ?? null,
   }) : () => false
   const routeRequest = createManagedRequestHandler({
-    ...options, config, db, avatarStore, reportStore, bundleStore, bundleCache, reportSourcesCache, prettyCache, npmCacheStorage, originGate, serveStatic,
+    ...options, config, db, avatarStore, reportStore, bundleStore, bundleCache, reportSourcesCache, prettyCache, npmCacheStorage, repositoryCodeStorage, originGate, serveStatic,
     ...(storage.uploadStore ? { uploadStore: storage.uploadStore } : {}), ...(waitUntil ? { waitUntil } : {}),
     isShuttingDown, track,
   })

package/server-managed/managed-issues.ts

@@ -53,6 +53,9 @@ export function managedIssueMethods(db: ManagedSql): ManagedIssueStore {
     issue_url AS issueUrl, auto_fix_url AS autoFixUrl, auto_fix_checked_at AS autoFixCheckedAt, created_by AS createdBy, created_at AS createdAt`
   const get = db.prepare(`SELECT ${fields} FROM managed_finding_issue WHERE finding_id = ?`)
   const list = db.prepare(`SELECT ${fields} FROM managed_finding_issue WHERE finding_id IN (SELECT value FROM json_each(?)) ORDER BY finding_id`)
+  // A lookup's derived Fix replaces the one it read, unless one started later
+  // changed it since: the check time, which only changes stamp
+  // (github-issue-links.ts issueUpdates), orders them.
   const autoFix = db.prepare(`UPDATE managed_finding_issue SET auto_fix_url = ?, auto_fix_checked_at = ?
     WHERE finding_id = ? AND issue_url = ? AND COALESCE(auto_fix_url, '') = COALESCE(?, '')
       AND (auto_fix_checked_at IS NULL OR auto_fix_checked_at <= ?)`)

package/server-managed/npm-delta.ts

@@ -154,6 +154,22 @@ export function npmDelta(base: Side | null, other: Side, kept: KeptStore<NpmDelt
   return work
 }
 
+// The deltas of those of `sides` (each the integrities of the version it is
+// against, or none, and of the version) held or kept, by their keys
+// (npmDelta's), in one read of `kept` for those not held: none worked out.
+export async function keptNpmDeltas(sides: [string | null, string][], kept: KeptStore<NpmDelta>): Promise<Map<string, NpmDelta>> {
+  const keys = [...new Set(sides.map(([base, other]) => `${base ?? '(empty)'}\0${other}`))]
+  const found = new Map(keys.flatMap(key => { const known = deltas.get(key); return known ? [[key, known] as const] : [] }))
+  const unheld = keys.filter(key => !found.has(key))
+  for (const [key, row] of unheld.length > 0 ? await kept.readMany(unheld) : []) {
+    const known = deltaOf(row.value)
+    if (!known) continue
+    deltas.set(key, known)
+    found.set(key, known)
+  }
+  return found
+}
+
 // As kept, else its turn awaited, then both sides' files read, counted and kept.
 async function worked(key: string, base: Side | null, other: Side, kept: KeptStore<NpmDelta> | null): Promise<NpmDelta> {
   const known = deltaOf((await kept?.read(key))?.value)

package/server-managed/npm-kept.ts

@@ -11,10 +11,14 @@
 // of a repository's GitHub, its issues, pull requests and agents' work, for a
 // day (http.ts keptAnswer), and its history, a page from a commit at a time,
 // for a month (repository-history.ts historyFrom), as are the files each of
-// its commits changes and holds (repository-changes.ts), and the packages a
-// pull request's lockfiles add (pull-dependencies.ts). Each store removes its own rows kept longer
-// than that as it writes, at most once an hour a database (pruneKept), so
-// that rows no read will take again don't pile up.
+// its commits changes and holds (repository-changes.ts), its trees as the
+// code viewer reads them (codeTreeStore), the packages a pull request's
+// lockfiles add (pull-dependencies.ts), and the lockfiles a commit lists and
+// each one's packages, as its dependency review reads them
+// (commit-lockfiles.ts); and the commit the code viewer keeps its
+// directories at, for as long as it does (codeHeadStore). Each store removes
+// its own rows kept longer than that as it writes, at most once an hour a
+// database (pruneKept), so that rows no read will take again don't pile up.
 // The database is an optimization: a failure reads as a miss, and writes
 // nothing.
 import { createHash } from 'node:crypto'
@@ -155,7 +159,7 @@ export const repositoryReadStore = (db: UpstreamCacheStore, debug = false): Kept
 export const HISTORY_PAGE_KEPT_MS = 30 * 24 * 60 * 60_000
 export const historyPageStore = <T>(db: UpstreamCacheStore, debug = false): KeptStore<T> => keptStore<T>(db, 'repository/history', HISTORY_PAGE_KEPT_MS, debug)
 // The pull requests squashed into a repository's commits, by the commit
-// (repository-history.ts squashedPulls): kept as its history's pages are.
+// (repository-squashes.ts squashedPulls): kept as its history's pages are.
 export const historySquashStore = <T>(db: UpstreamCacheStore, debug = false): KeptStore<T> => keptStore<T>(db, 'repository/squashes', HISTORY_PAGE_KEPT_MS, debug)
 // The files each of a repository's commits changes (repository-changes.ts
 // repositoryChanges), by the commit, which never change: kept as its
@@ -165,6 +169,21 @@ export const historyChangeStore = <T>(db: UpstreamCacheStore, debug = false): Ke
 // repositoryTree), by the commit, which never change: kept as its history's
 // pages are.
 export const historyTreeStore = <T>(db: UpstreamCacheStore, debug = false): KeptStore<T> => keptStore<T>(db, 'repository/trees', HISTORY_PAGE_KEPT_MS, debug)
+// What the code viewer read of a repository's trees (http.ts
+// repositoryTreesOf), which never changes once a commit or tree names it: the
+// tree at a directory of a commit, by the repository's id, the commit and the
+// directory (`trees`), and how many files a tree holds and their bytes, by
+// the repository's id and the tree's (`tree-files`): kept as its history's
+// pages are.
+export const codeTreeStore = <T>(db: UpstreamCacheStore, type: 'trees' | 'tree-files', debug = false): KeptStore<T> =>
+  keptStore<T>(db, `repository/code-${type}`, HISTORY_PAGE_KEPT_MS, debug)
+// The commit the code viewer keeps a repository's directories at in byte
+// storage, its default branch's head as last read, and the repository's
+// name they are kept under (repository-code.ts repositoryCodeKept), by the
+// repository's id: kept until the head moves, the repository is renamed or
+// it is removed, one for each repository.
+export const codeHeadStore = (db: UpstreamCacheStore, debug = false): KeptStore<{ commit: string; repo: string }> =>
+  keptStore<{ commit: string; repo: string }>(db, 'repository/code-heads', Infinity, debug)
 // The packages a repository's pull request's lockfiles add
 // (pull-dependencies.ts), by the commits of its base and head, which never
 // change: kept as its history's pages are.
@@ -178,3 +197,11 @@ export const pullDependencyStore = <T>(db: UpstreamCacheStore, debug = false): K
 export const npmDeltaStore = <T>(db: UpstreamCacheStore, debug = false): KeptStore<T> => keptStore<T>(db, 'npm/deltas', HISTORY_PAGE_KEPT_MS, debug)
 export const reviewBaselineStore = (db: UpstreamCacheStore, debug = false): KeptStore<{ versions: string[]; unread: string[] }> =>
   keptStore<{ versions: string[]; unread: string[] }>(db, 'repository/review-baselines', HISTORY_PAGE_KEPT_MS, debug)
+// What a repository's dependency review read of GitHub (pull-dependencies.ts
+// LockfileStore): the lockfiles a commit lists (`listings`), by the commit,
+// and a lockfile's packages with what their reviews are held to
+// (`lockfiles`), by its blob, which never change: kept as its history's
+// pages are, apart from a scan's (scanReadStore), which reads them without
+// what their reviews are held to.
+export const lockfileReadStore = <T>(db: UpstreamCacheStore, type: 'listings' | 'lockfiles', debug = false): KeptStore<T> =>
+  keptStore<T>(db, `repository/review-${type}`, HISTORY_PAGE_KEPT_MS, debug)

package/server-managed/npm-loads.ts

@@ -50,9 +50,9 @@ const refused = <T>(status: number, code: string): NpmLoad<T> => {
   return { result, release() {} }
 }
 
-// What names a version's tarball, and so its files: its integrity and URL,
-// and whether it is read with the token.
-const tarballKey = (doc: NpmVersionDocument) => `${doc.dist.integrity} ${doc.dist.tarball} ${doc.private}`
+// What names a version's tarball, and so its files: its integrity, its
+// shasum and URL, and whether it is read with the token.
+const tarballKey = (doc: NpmVersionDocument) => `${doc.dist.integrity} ${doc.dist.shasum ?? ''} ${doc.dist.tarball} ${doc.private}`
 
 function shared<T>(what: string, doc: NpmVersionDocument, work: () => Promise<T>): NpmLoad<T> {
   const { unpackedSize, fileCount } = doc.dist
@@ -118,7 +118,13 @@ function done(key: string, load: Load) {
 // The sha512 an integrity names, in base64, or null where it names none.
 const sha512Of = (integrity: string) => /(?:^|\s)sha512-([\d+/A-Za-z]{86}==)(?=\s|$)/u.exec(integrity)?.[1] ?? null
 // Hashed off the event loop: a tarball runs to tens of MiB.
-const sha512 = async (bytes: Uint8Array) => Buffer.from(await crypto.subtle.digest('SHA-512', bytes as Uint8Array<ArrayBuffer>)).toString('base64')
+const digest = async (algorithm: string, bytes: Uint8Array) => Buffer.from(await crypto.subtle.digest(algorithm, bytes as Uint8Array<ArrayBuffer>))
+// Whether `bytes` are those of the sha512 an integrity names and, where there
+// is one, of the shasum (a sha1, in hex).
+async function hashedAs(bytes: Uint8Array, sha512: string, shasum: string | null): Promise<boolean> {
+  const [own512, own1] = await Promise.all([digest('SHA-512', bytes), shasum === null ? null : digest('SHA-1', bytes)])
+  return own512.toString('base64') === sha512 && (own1 === null || own1.toString('hex') === shasum)
+}
 
 // Upstream's disk cache (setCacheDir), where tarballs are kept between
 // loads, or null to keep none and read none.
@@ -151,23 +157,25 @@ async function downloadTarball(url: string, isPrivate: boolean): Promise<Buffer>
   return bytes
 }
 
-// The version's tarball, its bytes held to the document's sha512 wherever
-// they come from: kept in the cache, where bundle builds keep what they
-// fetch with the token, or in npm's; else downloaded, and then kept. Bytes
-// matching the sha512 the reader's own document names are the tarball that
-// reader may download, wherever they were kept and whoever fetched them.
+// The version's tarball, its bytes held to the document's sha512, and to its
+// shasum where it tells one, wherever they come from: kept in the cache,
+// where bundle builds keep what they fetch with the token, or in npm's; else
+// downloaded, and then kept. Bytes matching the hashes the reader's own
+// document names are the tarball that reader may download, wherever they
+// were kept and whoever fetched them.
 async function readTarball(doc: NpmVersionDocument): Promise<Uint8Array> {
-  const expected = sha512Of(doc.dist.integrity), url = doc.dist.tarball
-  if (!url.startsWith(`${NPM_REGISTRY}/${doc.name}/-/`) || URL.parse(url)?.href !== url || expected === null) throw new NpmPackageError(502, 'upstream-invalid')
+  const expected = sha512Of(doc.dist.integrity), shasum = doc.dist.shasum ?? null, url = doc.dist.tarball
+  if (!url.startsWith(`${NPM_REGISTRY}/${doc.name}/-/`) || URL.parse(url)?.href !== url || expected === null
+    || (shasum !== null && !/^[\da-f]{40}$/u.test(shasum))) throw new NpmPackageError(502, 'upstream-invalid')
   const kept = tarballCache === null ? null : upstreamCachePath(tarballCache, 'npm/tarballs', `${doc.name}@${doc.version}.tgz`)
   if (kept !== null) {
     for (const path of [kept, npmCachePath(expected)]) {
       const bytes = path === null ? null : await readKeptFile(path, MAX_TAR_BYTES)
-      if (bytes && await sha512(bytes) === expected) return bytes
+      if (bytes && await hashedAs(bytes, expected, shasum)) return bytes
     }
   }
   const bytes = await downloadTarball(url, doc.private)
-  if (await sha512(bytes) !== expected) throw new NpmPackageError(502, 'upstream-invalid')
+  if (!await hashedAs(bytes, expected, shasum)) throw new NpmPackageError(502, 'upstream-invalid')
   if (kept !== null) await keepFile(kept, bytes)
   return bytes
 }

package/server-managed/npm-manifest.ts

@@ -102,7 +102,7 @@ export function npmManifest(json: Record<string, unknown>): Record<string, unkno
 // as they read npm's.
 const VERSION_FIELDS = ['name', 'version', 'description', 'license', 'homepage', 'main', 'module', 'types', 'type', 'deprecated', 'gitHead', 'author', '_npmUser',
   'maintainers', 'keywords', 'bin', 'dependencies', 'peerDependencies', 'optionalDependencies', 'engines', 'hasInstallScript', 'repository', 'bugs']
-const DIST_FIELDS = ['tarball', 'integrity', 'unpackedSize', 'fileCount']
+const DIST_FIELDS = ['tarball', 'integrity', 'shasum', 'unpackedSize', 'fileCount']
 const INSTALL_SCRIPTS = ['preinstall', 'install', 'postinstall']
 const picked = (json: Record<string, unknown>, keys: string[]) => Object.fromEntries(keys.filter(key => Object.hasOwn(json, key)).map(key => [key, json[key]]))
 

package/server-managed/npm-packages.ts

@@ -228,7 +228,9 @@ export interface NpmVersionDocument {
   name: string
   version: string
   private: boolean
-  dist: { tarball: string; integrity: string; unpackedSize: number | null; fileCount: number | null }
+  // `shasum`, the tarball's sha1 in hex, where npm tells it: held, with the
+  // integrity, to the tarball its files are read from (npm-loads.ts).
+  dist: { tarball: string; integrity: string; shasum: string | null; unpackedSize: number | null; fileCount: number | null }
   manifest: Record<string, unknown>
 }
 
@@ -238,10 +240,12 @@ const count = (value: unknown) => Number.isSafeInteger(value) && (value as numbe
 function versionDocument(name: string, json: Record<string, unknown>, isPrivate: boolean): NpmVersionDocument {
   const dist = json['dist']
   if (json['name'] !== name || typeof json['version'] !== 'string' || !isNpmPackageSpec(json['version']) || !plainObject(dist)
-      || typeof dist['tarball'] !== 'string' || typeof dist['integrity'] !== 'string') throw new NpmPackageError(502, 'upstream-invalid')
+      || typeof dist['tarball'] !== 'string' || typeof dist['integrity'] !== 'string'
+      || (dist['shasum'] !== undefined && typeof dist['shasum'] !== 'string')) throw new NpmPackageError(502, 'upstream-invalid')
   return {
     name, version: json['version'], private: isPrivate,
-    dist: { tarball: dist['tarball'], integrity: dist['integrity'], unpackedSize: count(dist['unpackedSize']), fileCount: count(dist['fileCount']) },
+    dist: { tarball: dist['tarball'], integrity: dist['integrity'], shasum: typeof dist['shasum'] === 'string' ? dist['shasum'] : null,
+      unpackedSize: count(dist['unpackedSize']), fileCount: count(dist['fileCount']) },
     manifest: npmManifest(json),
   }
 }
@@ -262,13 +266,18 @@ export interface NpmVersionStore {
 // changes a version's files, but its document can change: a deprecation, the
 // package made private (a scoped one can be), the version removed (which whoever
 // reads it from a store learns elsewhere too: risk does). A day bounds each.
+// One kept with no shasum, as none was before it was kept (npmVersionFields),
+// is asked of npm again, which tells one of each version.
 export const VERSION_KEPT_MS = 24 * 60 * 60_000
 export function readNpmVersion(name: string, spec: string, privileged: boolean, signal: AbortSignal, store?: NpmVersionStore): Promise<NpmVersionDocument | null> {
   if (!isNpmPackageName(name) || !isNpmPackageSpec(spec)) return Promise.reject(new NpmPackageError(400, 'bad-package'))
   // A kept one is read within the same budget as npm's answer.
   return withDocumentBudget(VERSION_DOCUMENT_BYTES, async () => {
     const kept = store && isExactVersion(spec) ? await store.read(name, spec) : null
-    if (kept && Date.now() - kept.at < VERSION_KEPT_MS) try { return versionDocument(name, kept.json, false) } catch {}
+    const keptDist = kept?.json['dist']
+    if (kept && Date.now() - kept.at < VERSION_KEPT_MS && plainObject(keptDist) && typeof keptDist['shasum'] === 'string') {
+      try { return versionDocument(name, kept.json, false) } catch {}
+    }
     const found = await readDocument(name, registryUrl(name, spec), privileged, { accept: 'application/json', limit: VERSION_DOCUMENT_BYTES, signal })
     if (!found) return null
     const doc = versionDocument(name, found.json, found.private)

package/server-managed/object-storage.ts

@@ -30,7 +30,8 @@ export interface ObjectStorage {
   exists(key: string): Promise<boolean>
   put(key: string, bytes: Buffer, options?: { replaceReport?: boolean }): Promise<string | null>
   delete(key: string): Promise<void>
-  deletePrefix(prefix: string): Promise<void>
+  // Every object under `prefix`, but those under `keep`, a prefix within it.
+  deletePrefix(prefix: string, keep?: string): Promise<void>
 }
 
 export function isBlobId(value: string): boolean {
@@ -52,14 +53,14 @@ export async function openObjectVersion(raw: RawObjectStorage, key: string, vers
   return null
 }
 
-export async function deleteObjects(raw: RawObjectStorage, prefix: string, signal: AbortSignal, before?: number): Promise<number> {
+export async function deleteObjects(raw: RawObjectStorage, prefix: string, signal: AbortSignal, before?: number, keep?: string): Promise<number> {
   const eligible = (modifiedAt: number) => before === undefined || modifiedAt < before
   // Finish listing before deleting: provider cursors may be offset-based.
   const cursors = new Set<string>(), keys = new Set<string>()
   let cursor: string | null = null
   do {
     const page = await raw.list(prefix, cursor, 100, signal)
-    for (const object of page.objects) if (eligible(object.modifiedAt)) keys.add(object.key)
+    for (const object of page.objects) if (eligible(object.modifiedAt) && (keep === undefined || !object.key.startsWith(keep))) keys.add(object.key)
     cursor = page.cursor
     if (cursor !== null && cursors.has(cursor)) throw new Error('Invalid blob pagination')
     if (cursor !== null) cursors.add(cursor)

package/server-managed/public-workspace.ts

@@ -7,7 +7,7 @@ import type { OpenedBlob } from './blob-store.ts'
 import type { WorkspaceShareSnapshot } from './workspace-shares.ts'
 import { sendJson, sendLines, writeResponse } from './http-response.ts'
 import { hashToken } from './crypto.ts'
-import { TeamReportsError, loadTeamAnnotations, loadTeamReportsResponse, teamReportVisibility } from './team-reports.ts'
+import { TeamReportsError, loadTeamAnnotations, loadTeamReportsResponse, loadTeamTreesResponse, teamReportVisibility } from './team-reports.ts'
 import { triageWireEntry } from './triage-response.ts'
 import { MAX_PACKAGE_INVENTORY_BYTES } from './bundle-cache.ts'
 import { backfillBundleSummaries, bundleSummaries } from './bundle-catalog.ts'
@@ -34,7 +34,7 @@ export async function handlePublicWorkspace(req: IncomingMessage, res: ServerRes
   const method = req.method ?? 'GET'
   if (method !== 'GET' && method !== 'HEAD') { json(res, 403, { error: 'share-read-only' }); return }
   const shareRoute = /^\/api\/shares\/([A-Za-z0-9_-]+)\/workspace$/u.exec(url.pathname)
-  const teamRoute = /^\/api\/teams\/([^/]+)\/(shared|reports|feed|annotations)$/u.exec(url.pathname)
+  const teamRoute = /^\/api\/teams\/([^/]+)\/(shared|reports|trees|feed|annotations)$/u.exec(url.pathname)
   const reportRoute = /^\/api\/reports\/([^/]+)\/(triage|comments|sources)$/u.exec(url.pathname)
   const bundleRoute = /^\/api\/bundles\/([^/]+)\/(metadata|contents|download|advisories|pretty|rename-diff|type-only|comparison)$/u.exec(url.pathname)
   if (!shareRoute && !teamRoute && !reportRoute && !bundleRoute) { json(res, 403, { error: 'share-scope-required' }); return }
@@ -84,7 +84,7 @@ export async function handlePublicWorkspace(req: IncomingMessage, res: ServerRes
       await send(await loadTeamAnnotations(deps.db, deps.reportStore, snapshot, url.searchParams.get('reportId'))); return
     }
     if (teamRoute[2] === 'shared') { await sendTeam(); return }
-    const body = await loadTeamReportsResponse(deps.db, deps.reportStore, snapshot)
+    const body = await (teamRoute[2] === 'trees' ? loadTeamTreesResponse : loadTeamReportsResponse)(deps.db, deps.reportStore, snapshot)
     await recheck()
     res.writeHead(200, { 'content-type': 'application/json', 'cache-control': 'private, no-store', 'referrer-policy': 'no-referrer' })
     writeResponse(res, body); return

package/server-managed/pull-comment-reviews.ts

@@ -132,7 +132,7 @@ function counted(answer: CommentReview[], read: Authored[], reviewers: Map<strin
       const { name, version, integrity, patchHash, status } = dependency ?? {}
       if (typeof name !== 'string' || typeof version !== 'string' || typeof integrity !== 'string' || !VERDICTS.has(status)) return []
       return integrity.split(/\s+/u).filter(Boolean).map(hash => ({ package: name, version, integrity: hash, ...typeof patchHash === 'string' && patchHash ? { patchHash } : {},
-        basePackage: null, baseVersion: null, reviewer, status: status as ReviewVerdict, full: true, time }))
+        basePackage: null, baseVersion: null, reviewer, login: review.user as string, status: status as ReviewVerdict, full: true, time }))
     })
   })
 }

package/server-managed/pull-dependencies.ts

@@ -22,7 +22,10 @@
 // changes, so what is told of one is kept by them (`kept`), and each
 // lockfile read is kept in this instance's memory by its commit
 // (`lockfileReads`): the pull requests branching off the same commit, as a
-// bot's do, read their base's once.
+// bot's do, read their base's once. One read as a commit lists it, with its
+// blob's id (commit-lockfiles.ts), is kept by its blob instead, and beyond
+// this instance where its reader keeps it (LockfileStore): what a blob holds
+// never changes either, and a push changing no lockfile changes none.
 import { createClient } from '@preventive/upstream/github.js'
 import { GithubApiError, githubJson } from './github-app.ts'
 import { upstreamGithub } from './github-errors.ts'
@@ -46,7 +49,8 @@ export const MAX_LOCKED = 20_000
 // Lockfiles read at once.
 const READ_PARALLEL = 4
 // The rules what is told was told by, kept with it (http.ts
-// pullDependencyKept): one told by others is told again.
+// pullDependencyKept), and the lockfiles read for it (lockfileKept): one
+// told or read by others is told or read again.
 export const PULL_DEPENDENCIES_READ = 3
 const SHA = /^[a-f\d]{40}$/u
 
@@ -104,20 +108,42 @@ export interface PullDependencyStore {
   write(key: string, value: PullDependencies): Promise<void>
 }
 
-// A lockfile's packages, or why its reader refused it, by its repository,
-// commit and path, read in this instance; by the packages each holds, as the
-// scan's are (repository-risk.ts riskCaches).
-type LockfileRead = { packages: LockedPackage[] } | { error: string; detail?: string }
+// A lockfile's packages, or why its reader refused it, by its repository and
+// readKey, read in this instance; by the packages each holds, as the scan's
+// are (repository-risk.ts riskCaches).
+export type LockfileRead = { packages: LockedPackage[] } | { error: string; detail?: string }
 const lockfileReads: Lru<LockfileRead> = lru(200_000, read => 'packages' in read ? read.packages.length + 1 : 1)
-// Those being read, by the token reading them too, lest each asking at once read them.
-const lockfilesReading = new Map<string, Promise<LockfileRead>>()
+// Those being read, by the token reading them too, lest each asking at once
+// read them, with whether GitHub was asked.
+const lockfilesReading = new Map<string, Promise<{ read: LockfileRead; github: boolean }>>()
+// A lockfile at a commit as findRepoLockfiles lists it, by its path, with its
+// format (lockfileFormat) and its blob's id (`sha`).
+export interface ListedLockfile { path: string; format: Format; sha: string }
+// What a repository's lockfiles read for its dependency review keep beyond
+// this instance (http.ts lockfileKept), by what never changes: a commit's
+// lockfiles as listed (`listings`, by the commit), and a lockfile's read
+// (`lockfiles`, by its readKey). A read GitHub failed is kept nowhere.
+export interface LockfileStore {
+  listings: { read(commit: string): Promise<ListedLockfile[] | null>; write(commit: string, listed: ListedLockfile[]): Promise<void> }
+  lockfiles: { read(key: string): Promise<LockfileRead | null>; write(key: string, read: LockfileRead): Promise<void> }
+}
+// Set where GitHub was asked for what a read reads (`github`), not all of it
+// held in this instance or kept beyond it.
+export interface GithubAsked { github: boolean }
+
+// A lockfile's read, by its blob where it is listed with its id (`sha`),
+// which with its format alone tell how it reads; else by the commit and its
+// path.
+const readKey = (ref: string, { path, format, sha }: { path: string; format: Format; sha?: string }) =>
+  sha ? `blob\0${format}\0${sha}` : `commit\0${ref}\0${path}`
 
 const api = (fullName: string) => `https://api.github.com/repos/${fullName.split('/').map(encodeURIComponent).join('/')}`
 
 // The pull request's base's and head's commits, as GitHub has them now: its
 // base's moving on as it does, its head's with each push, both kept once it
 // is merged or closed.
-async function pullCommits(fullName: string, token: string, number: number, fetchImpl: typeof fetch) {
+export interface PullCommits { base: string; head: string }
+async function pullCommits(fullName: string, token: string, number: number, fetchImpl: typeof fetch): Promise<PullCommits> {
   const pull = await githubJson(`${api(fullName)}/pulls/${number}`, token, fetchImpl) as
     { number?: unknown; base?: { sha?: unknown; repo?: { full_name?: unknown } | null }; head?: { sha?: unknown } } | null
   const base = pull?.base?.sha, head = pull?.head?.sha, repo = pull?.base?.repo?.full_name
@@ -129,13 +155,16 @@ async function pullCommits(fullName: string, token: string, number: number, fetc
 // The pull request's merge base and the files it changes from it: as GitHub
 // compares its base with its head, else, where that lists as many as it lists
 // at most, from the pull request's own list, a page at a time; `cut` where
-// that lists as many as it lists at most, too.
-async function pullFiles(fullName: string, token: string, number: number, base: string, head: string, fetchImpl: typeof fetch) {
+// that lists as many as it lists at most, too. That list is of its commits
+// now: null where they are no longer `base` and `head` (`current`).
+async function pullFiles(fullName: string, token: string, number: number, base: string, head: string, fetchImpl: typeof fetch,
+  current: () => Promise<boolean>) {
   const compared = await githubJson(`${api(fullName)}/compare/${base}...${head}?per_page=1`, token, fetchImpl) as { merge_base_commit?: { sha?: unknown } } | null
   const mergeBase = compared?.merge_base_commit?.sha
   if (typeof mergeBase !== 'string' || !SHA.test(mergeBase)) throw new GithubApiError(502, 'github-malformed')
   const { files, more } = commitChanges(compared)
   if (!more) return { mergeBase, files, cut: false }
+  if (!await current()) return null
   const listed: ChangedFile[] = []
   for (let page = 1; listed.length < PULL_FILES; page++) {
     const data = await githubJson(`${api(fullName)}/pulls/${number}/files?per_page=${FILES_PAGE}&page=${page}`, token, fetchImpl)
@@ -171,39 +200,49 @@ function lockfilesChanged(files: ChangedFile[]) {
     omitted: all.slice(MAX_DIRECTORIES).flatMap(([, directory]) => [...directory.paths]).toSorted() }
 }
 
-// Whether `lockfile` at `ref` is held in this instance, read without asking
-// GitHub: Bun's, never read, too.
-export const lockfileHeld = (fullName: string, ref: string, { path, format }: { path: string; format: Format }) =>
-  format === 'bun' || lockfileReads.get(`${fullName}\0${ref}\0${path}`) !== undefined
-
-// A lockfile's packages at `ref`, as read in this instance, else shared with
-// a read of it with the same token still under way, else of GitHub; a
-// lockfile its reader refuses, why. Any failure to read it of GitHub but its
-// having none there is the whole answer's, as it would be every file's.
-export function readLockfile(fullName: string, token: string, ref: string, lockfile: { path: string; format: Format }): Promise<LockfileRead> {
-  if (lockfile.format === 'bun') return Promise.resolve({ error: 'lockfile-unsupported' })
-  const key = `${fullName}\0${ref}\0${lockfile.path}`
-  const known = lockfileReads.get(key)
-  if (known) return Promise.resolve(known)
-  const shared = `${token}\0${key}`
-  const reading = lockfilesReading.get(shared) ?? readOf(fullName, token, ref, lockfile, key).finally(() => lockfilesReading.delete(shared))
+// A lockfile's packages at `ref`, as read in this instance, else as kept
+// beyond it (`kept`), else shared with a read of it with the same token still
+// under way, else of GitHub, and kept; a lockfile its reader refuses, why,
+// kept as well, its reader refusing it again; `asked` told where GitHub was.
+// Any failure to read it of GitHub but its having none there is the whole
+// answer's, as it would be every file's; neither is kept.
+export async function readLockfile(fullName: string, token: string, ref: string, lockfile: { path: string; format: Format; sha?: string },
+  kept: LockfileStore | null = null, asked?: GithubAsked): Promise<LockfileRead> {
+  if (lockfile.format === 'bun') return { error: 'lockfile-unsupported' }
+  const key = readKey(ref, lockfile)
+  const known = lockfileReads.get(`${fullName}\0${key}`)
+  if (known) return known
+  const shared = `${token}\0${fullName}\0${key}`
+  const reading = lockfilesReading.get(shared) ?? readOf(fullName, token, ref, lockfile, key, kept).finally(() => lockfilesReading.delete(shared))
   lockfilesReading.set(shared, reading)
-  return reading
+  // One failing was asked of GitHub too, though a caller swallows its
+  // failure, as a baseline's does: its access is asked again all the same.
+  let told
+  try { told = await reading } catch (err) {
+    if (asked) asked.github = true
+    throw err
+  }
+  if (told.github && asked) asked.github = true
+  return told.read
 }
-async function readOf(fullName: string, token: string, ref: string, { path, format }: { path: string; format: Format }, key: string): Promise<LockfileRead> {
+async function readOf(fullName: string, token: string, ref: string, { path, format }: { path: string; format: Format }, key: string,
+  kept: LockfileStore | null): Promise<{ read: LockfileRead; github: boolean }> {
+  const stored = await kept?.lockfiles.read(key)
+  if (stored) { lockfileReads.set(`${fullName}\0${key}`, stored); return { read: stored, github: false } }
   let text: string
   try {
     text = await upstreamGithub(() => createClient({ token, userAgent: 'deepview-triage' }).getRepoFile({ repo: fullName, path, ref }))
   } catch (err) {
-    if (err instanceof GithubApiError && err.message === 'github-not-found') return { error: 'github-not-found' }
+    if (err instanceof GithubApiError && err.message === 'github-not-found') return { read: { error: 'github-not-found' }, github: true }
     throw err
   }
   let read: LockfileRead
   try { read = { packages: lockedPackages(format, text, { review: true }) } } catch (error) {
     read = { error: 'lockfile-unreadable', ...error instanceof Error ? { detail: error.message.slice(0, 500) } : {} }
   }
-  lockfileReads.set(key, read)
-  return read
+  lockfileReads.set(`${fullName}\0${key}`, read)
+  await kept?.lockfiles.write(key, read)
+  return { read, github: true }
 }
 
 // Runs `run` for each of `items`, as many at once as `parallel`, each's
@@ -278,14 +317,23 @@ export async function pullDependencies(fullName: string, token: string, number:
   return (await pullDependenciesAt(fullName, token, number, kept, fetchImpl)).told
 }
 // The same, with the commits it is kept by (`commits`), its base's and
-// head's, by which `kept` reads it.
+// head's, by which `kept` reads it: those `given`, as a page of its
+// repository's open pull requests read them (repository-issues.ts), else as
+// GitHub has them now. Commits given that its own list of its files is no
+// longer of are read as GitHub has them now.
 export async function pullDependenciesAt(fullName: string, token: string, number: number, kept: PullDependencyStore,
-  fetchImpl: typeof fetch = globalThis.fetch): Promise<{ commits: string; told: PullDependencies }> {
-  const { base, head } = await pullCommits(fullName, token, number, fetchImpl)
+  fetchImpl: typeof fetch = globalThis.fetch, given?: PullCommits): Promise<{ commits: string; told: PullDependencies }> {
+  const { base, head } = given ?? await pullCommits(fullName, token, number, fetchImpl)
   const key = `${base}...${head}`
   const held = await kept.read(key)
   if (held) return { commits: key, told: held }
-  const { mergeBase, files, cut: filesCut } = await pullFiles(fullName, token, number, base, head, fetchImpl)
+  const current = async () => {
+    const now = given && await pullCommits(fullName, token, number, fetchImpl)
+    return !now || (now.base === base && now.head === head)
+  }
+  const pulled = await pullFiles(fullName, token, number, base, head, fetchImpl, current)
+  if (!pulled) return pullDependenciesAt(fullName, token, number, kept, fetchImpl)
+  const { mergeBase, files, cut: filesCut } = pulled
   const { directories, cut, omitted } = lockfilesChanged(files)
   const reads = directories.flatMap(({ before, after }) => [...before.map(lockfile => [mergeBase, lockfile] as const), ...after.map(lockfile => [head, lockfile] as const)])
   const read = await eachOf(reads, READ_PARALLEL, ([ref, lockfile]) => readLockfile(fullName, token, ref, lockfile))

package/server-managed/pull-dependency-reviews.ts

@@ -24,12 +24,12 @@
 // changes: it is read once, every lockfile there of those read here
 // (pull-dependencies.ts), and kept by the commit (`baselines` in this
 // instance, `kept` beyond it), never stored as reviews.
-import { type GithubAsked, lockedAt, lockedIn } from './commit-lockfiles.ts'
+import { lockedAt, lockedIn } from './commit-lockfiles.ts'
 import { GithubApiError } from './github-app.ts'
 import type { LockedPackage } from './locked-packages.ts'
 import { EVERY_LOCKFILE, type LockfileScope, heldTo, isEvery, reviewedIn } from './lockfile-scope.ts'
 import { type Lru, lru } from './lru.ts'
-import { type DirectoryDependencies, MAX_ADDED, type PullDependencies, lockedAs, lockedWith } from './pull-dependencies.ts'
+import { type DirectoryDependencies, type GithubAsked, type LockfileStore, MAX_ADDED, type PullDependencies, lockedAs, lockedWith } from './pull-dependencies.ts'
 import type { PublishedReview, RepoReviewState } from './repository-dependency-reviews.ts'
 import { type ReviewedDependencies, type ReviewedDependency, askReviewed } from './reviewed-dependencies.ts'
 
@@ -64,17 +64,19 @@ const hashesOf = (pkg: LockedPackage) => (pkg.integrity ?? '').split(/\s+/u).fil
 const baselineKeys = (pkg: LockedPackage) => hashesOf(pkg).map(hash => `${pkg.name}@${pkg.version ?? ''}\0${hash}\0${pkg.patchHash ?? ''}`)
 
 // The versions the repository's lockfiles locked at commit `since` that are
-// reviewed (reviewedAs), as kept, else read of GitHub (commit-lockfiles.ts)
-// and kept; with the lockfiles there its reader refuses, isn't one of, or
-// past those read (`unread`). GitHub having none of a lockfile it lists
-// there is a failure of the whole, as of any file, never kept.
-export async function reviewBaseline(fullName: string, token: string, repoId: number, since: string, kept: ReviewBaselineStore, asked?: GithubAsked): Promise<Baseline> {
+// reviewed (reviewedAs), as kept, else read of GitHub (commit-lockfiles.ts,
+// its lockfiles as `reads` keeps them) and kept; with the lockfiles there its
+// reader refuses, isn't one of, or past those read (`unread`). GitHub having
+// none of a lockfile it lists there is a failure of the whole, as of any
+// file, never kept.
+export async function reviewBaseline(fullName: string, token: string, repoId: number, since: string, kept: ReviewBaselineStore, reads: LockfileStore | null = null,
+  asked?: GithubAsked): Promise<Baseline> {
   const key = `${repoId}\0${since}`
   const known = baselines.get(key)
   if (known) return known
   const stored = await kept.read(key)
   if (stored) { const baseline = { versions: new Set(stored.versions), unread: stored.unread }; baselines.set(key, baseline); return baseline }
-  const read = await lockedAt(fullName, token, since, () => true, asked)
+  const read = await lockedAt(fullName, token, since, reads, () => true, asked)
   const { packages } = lockedIn(read), { unread } = read
   const versions = new Set<string>()
   for (const pkg of packages) if (reviewedAs(pkg) === 'version') for (const held of baselineKeys(pkg)) versions.add(held)
@@ -105,7 +107,9 @@ const ACCEPTING = new Set(['accept', 'accept-with-nits'])
 // review of theirs, whole or compared with one of `bases` of its package
 // (any, for `any`); and how many review it in its pull request's comments
 // alone (`commented`), which count for it alone: whose reviews passing it,
-// else whose latest, are its comments'.
+// else whose latest, are its comments'. Each reviewer too (`reviewers`), by
+// their login, their latest verdict, whether they pass it, and whether that
+// verdict is its comments' (`comment`), which counts for it alone.
 function stancesOf(pkg: LockedPackage, reviews: ReviewsOf, bases: string[] | 'any') {
   const hashes = hashesOf(pkg)
   const by = new Map<string, PublishedReview[]>()
@@ -116,17 +120,21 @@ function stancesOf(pkg: LockedPackage, reviews: ReviewsOf, bases: string[] | 'an
   const qualifies = (review: PublishedReview) => review.full && ACCEPTING.has(review.status)
     && (review.basePackage === null || (review.basePackage === pkg.name && (bases === 'any' || bases.includes(review.baseVersion ?? ''))))
   let commented = 0, passing = 0, rejected = false
+  const reviewers: Reviewer[] = []
   for (const theirs of by.values()) {
     const latest = theirs.reduce((one, other) => other.time > one.time ? other : one)
+    const passes = ACCEPTING.has(latest.status) && theirs.some(qualifies)
     // In its comments alone where those telling where they stand are: those
     // passing it, where they pass it, else their latest.
-    const telling = ACCEPTING.has(latest.status) && theirs.some(qualifies) ? theirs.filter(qualifies) : [latest]
-    if (telling.every(review => review.comment)) commented++
+    const comment = (passes ? theirs.filter(qualifies) : [latest]).every(review => review.comment)
+    if (comment) commented++
     if (latest.status === 'reject') rejected = true
-    else if (ACCEPTING.has(latest.status) && theirs.some(qualifies)) passing++
+    else if (passes) passing++
+    reviewers.push({ login: latest.login, status: latest.status, ...passes ? { passes: true as const } : {}, ...latest.comment ? { comment: true as const } : {} })
   }
-  return { passing, rejected, commented }
+  return { passing, rejected, commented, reviewers }
 }
+type Reviewer = { login: string; status: PublishedReview['status']; passes?: true; comment?: true }
 
 // A version as the client is told it: what its reviews are held to left
 // out, but whether it is patched.
@@ -146,7 +154,8 @@ function askedOf(pkg: LockedPackage): [string, ReviewedDependency] {
 
 // A version's review, and whether it wants one (`wants`), neither passing
 // nor needing none, as only one that can be reviewed may.
-type Review = { unsupported: Unsupported } | { baseline: true } | { reviewed: true } | { reviews: number; rejected?: true; commented?: number } | Record<string, never>
+type Review = { unsupported: Unsupported } | { baseline: true } | { reviewed: true } | { reviews: number; rejected?: true; commented?: number; reviewers?: Reviewer[] }
+  | Record<string, never>
 interface Judged { review: Review; wants: boolean }
 type Judge = (pkg: LockedPackage, bases: string[] | 'any') => Judged
 interface Context { state: RepoReviewState; reviews: ReviewsOf; baseline: Set<string> | null }
@@ -155,8 +164,8 @@ function judge(pkg: LockedPackage, bases: string[] | 'any', { state, reviews, ba
   if (as === 'skip') return { review: {}, wants: false }
   if (as !== 'version') return { review: { unsupported: as }, wants: false }
   if (baselineKeys(pkg).some(key => baseline?.has(key))) return { review: { baseline: true }, wants: false }
-  const { passing, rejected, commented } = stancesOf(pkg, reviews, bases)
-  const told = { reviews: passing, ...commented > 0 ? { commented } : {} }
+  const { passing, rejected, commented, reviewers } = stancesOf(pkg, reviews, bases)
+  const told = { reviews: passing, ...commented > 0 ? { commented } : {}, ...reviewers.length > 0 ? { reviewers } : {} }
   if (rejected) return { review: { ...told, rejected: true }, wants: true }
   return { review: told, wants: passing < state.required }
 }
@@ -306,10 +315,11 @@ export const reviewedPackages = (told: PullDependencies) => told.lockfiles
 // A repository's baseline as read for a pull request: its versions and the
 // lockfiles there left unread; none where it reviews since no commit; and
 // where GitHub doesn't read it, none and why.
-export async function baselineFor(fullName: string, token: string, repoId: number, state: RepoReviewState, kept: ReviewBaselineStore, asked?: GithubAsked) {
+export async function baselineFor(fullName: string, token: string, repoId: number, state: RepoReviewState, kept: ReviewBaselineStore, reads: LockfileStore | null,
+  asked?: GithubAsked) {
   if (!state.since) return { baseline: null, unread: [], error: null }
   try {
-    const { versions, unread } = await reviewBaseline(fullName, token, repoId, state.since, kept, asked)
+    const { versions, unread } = await reviewBaseline(fullName, token, repoId, state.since, kept, reads, asked)
     return { baseline: versions, unread, error: null }
   } catch (err) {
     if (err instanceof GithubApiError && err.status !== 429 && err.status !== 401) return { baseline: null, unread: [], error: err.message }