@preventive/triage 1.0.0-alpha.45 → 1.0.0-alpha.46

package/server-managed/repository-browser.ts

@@ -1,6 +1,7 @@
+import { createHash } from 'node:crypto'
 import type { ManagedConfig } from './config.ts'
 import type { SelectedRepo } from './db.ts'
-import { GithubApiError, type GithubIdentity, READ_PERMISSIONS, githubJson, githubRepoPermission, githubUserIdentity, repoAccessToken } from './github-app.ts'
+import { GithubApiError, type GithubIdentity, READ_PERMISSIONS, githubJson, githubRepoPermission, githubUserIdentity, onTokenRejected, repoAccessToken } from './github-app.ts'
 import { MAX_PACKAGE_BYTES, readPackageEntryPoints } from './package-entry-points.ts'
 import { readSolidityEntryPoints } from './solidity-entry-points.ts'
 import { parseGithubCommit } from './bundle-commits.ts'
@@ -40,14 +41,63 @@ function credentialFailure(err: unknown): err is GithubApiError {
   return err instanceof GithubApiError && ([401, 404].includes(err.status) || err.message === 'github-status-403')
 }
 
-// Create once per HTTP request. Coalesce concurrent credential lookups without
-// caching repository visibility or permissions, or sharing credentials across users.
+// GitHub's answer on a user's access to a repository, as checkAccess reads it:
+// the repository's metadata, which credential reads it (the user's token, the
+// App installation's, or none, as for a public repository read anonymously)
+// and the permission level GitHub told, where it was asked. Kept per user, by
+// their GitHub id and token, and per repository, by its id, name and
+// installation, for config.repositoryAccessCacheMs; only access granted is kept.
+// One answered for a user with a token is kept no longer once GitHub rejects
+// that token (`userToken`, hashed), whichever credential reads it: a private
+// repository's was granted on who that token told GitHub they are. The next
+// check asks GitHub, falling back to the installation's or none as it does.
+type RepoMetadata = { id?: unknown; full_name?: unknown; private?: unknown; visibility?: unknown; default_branch?: unknown }
+type AccessVerdict = { until: number; data: RepoMetadata; credential: 'user' | 'app' | 'none'; level: string | null; userToken: string | null }
+type VerdictCache = { get(): AccessVerdict | undefined; set(verdict: Omit<AccessVerdict, 'until' | 'userToken'>): void }
+const MAX_ACCESS_VERDICTS = 1000
+// By the fetch reading GitHub, as installation tokens are: a test's own.
+const accessVerdicts = new WeakMap<typeof fetch, Map<string, AccessVerdict>>()
+const hashed = (token: string) => createHash('sha256').update(token).digest('hex')
+onTokenRejected((token, fetchImpl) => {
+  const kept = accessVerdicts.get(fetchImpl)
+  if (!kept) return
+  const rejected = hashed(token)
+  for (const [key, verdict] of kept) if (verdict.userToken === rejected) kept.delete(key)
+})
+
+function verdictCache(config: ManagedConfig, user: RepositoryBrowserUser, repo: SelectedRepo, fetchImpl: typeof fetch): VerdictCache | null {
+  const keepMs = config.repositoryAccessCacheMs
+  if (!(keepMs > 0)) return null
+  let verdicts = accessVerdicts.get(fetchImpl)
+  if (!verdicts) { verdicts = new Map(); accessVerdicts.set(fetchImpl, verdicts) }
+  const kept = verdicts
+  // The token hashed with the rest, never kept as it is.
+  const key = createHash('sha256').update(JSON.stringify([user.githubUserId, user.token, repo.repoId, repo.fullName.toLowerCase(), repo.installationId, config.githubAppId])).digest('hex')
+  return {
+    get() {
+      const verdict = kept.get(key)
+      if (verdict && verdict.until > Date.now()) return verdict
+      kept.delete(key)
+      return undefined
+    },
+    set(verdict) {
+      kept.delete(key)
+      kept.set(key, { ...verdict, until: Date.now() + keepMs, userToken: user.token ? hashed(user.token) : null })
+      while (kept.size > MAX_ACCESS_VERDICTS) kept.delete(kept.keys().next().value!)
+    },
+  }
+}
+
+// Create once per HTTP request. Coalesce concurrent credential lookups, and
+// keep GitHub's answer on each user's access to each repository a while
+// (verdictCache), never sharing credentials or answers across users.
 export function createRepositoryBrowser(config: ManagedConfig, user: RepositoryBrowserUser, fetchImpl: typeof fetch = globalThis.fetch) {
   const tokens = new Map<number, Promise<string | null>>()
   let identity: Promise<GithubIdentity> | undefined
-  const getIdentity = (fresh = false): Promise<GithubIdentity> => {
+  // The user as GitHub tells them, asked once a request.
+  const getIdentity = (): Promise<GithubIdentity> => {
     if (!user.token || user.githubUserId == null) return Promise.reject(new GithubApiError(404, 'no-repository'))
-    if (!identity || fresh) {
+    if (!identity) {
       identity = githubUserIdentity(user.token, fetchImpl).then(value => {
         if (value.id !== user.githubUserId) throw new GithubApiError(404, 'no-repository')
         return value
@@ -71,7 +121,7 @@ export function createRepositoryBrowser(config: ManagedConfig, user: RepositoryB
       // Its Dependabot alerts, with the token of the installation GitHub tells
       // covers it, where the user's own access lets them see them
       // (dependabot-access.ts).
-      return repositoryReader(repo, user.token, installationToken, getIdentity, fetchImpl).then(reader => ({ ...reader,
+      return repositoryReader(repo, user.token, installationToken, getIdentity, fetchImpl, verdictCache(config, user, repo, fetchImpl)).then(reader => ({ ...reader,
         dependabotAlerts: () => appDependabotAlerts(config, repo, { identity: getIdentity, permission: reader.permission }, fetchImpl) }))
     },
   }
@@ -96,48 +146,86 @@ export async function repositoryTreeFiles(fullName: string, token: string | null
   return { files, bytes }
 }
 
-async function repositoryReader(repo: SelectedRepo, userToken: string | null, installationToken: () => Promise<string | null>, getIdentity: (fresh?: boolean) => Promise<GithubIdentity>, fetchImpl: typeof fetch) {
+type Access = { data: RepoMetadata; token: string | null; level: string | null }
+
+// GitHub asked whether the user may read `repo`: its metadata, read with
+// `token` (the reader's credential so far), else the installation's, else
+// none; and of one not public, the user's permission, read with the
+// installation's. What it told, the token reading it, and the level asked.
+async function askAccess(repo: SelectedRepo, start: string | null, userToken: string | null, installationToken: () => Promise<string | null>,
+  getIdentity: () => Promise<GithubIdentity>, fetchImpl: typeof fetch): Promise<Access> {
   const base = `https://api.github.com/repos/${repo.fullName.split('/').map(encodeURIComponent).join('/')}`
-  // Public reads can use the signed-in user's quota without repository grants.
-  // Nonpublic reads still require the repository App and the effective-user gate,
-  // whose permission level is kept (`level`).
-  let level: string | null = null, token = userToken ?? await installationToken()
-  const read = (suffix: string) => githubJson(base + suffix, token, fetchImpl)
+  let token = start
   const metadata = async () => {
-    try { return await read('') }
+    try { return await githubJson(base, token, fetchImpl) }
     catch (err) {
       if (!token || !credentialFailure(err)) throw err
       const fallback = await installationToken()
       if (fallback && fallback !== token) {
         token = fallback
-        try { return await read('') } catch (fallbackError) { if (!credentialFailure(fallbackError)) throw fallbackError }
+        try { return await githubJson(base, token, fetchImpl) } catch (fallbackError) { if (!credentialFailure(fallbackError)) throw fallbackError }
       }
       // Anonymous fallback is only for unavailable credentials, never rate
       // limits, network errors, or GitHub outages.
       token = null
-      return read('')
+      return githubJson(base, token, fetchImpl)
     }
   }
-  const checkAccess = async (freshIdentity = false) => {
-    // Stored visibility can be stale (or internal with private=false). Require
-    // current, explicit public visibility or the caller's effective permission.
-    const data = await metadata() as { id?: unknown; full_name?: unknown; private?: unknown; visibility?: unknown; default_branch?: unknown }
-    if (data?.id !== repo.repoId || typeof data.full_name !== 'string' || data.full_name.toLowerCase() !== repo.fullName.toLowerCase()) {
-      throw new GithubApiError(404, 'no-repository')
-    }
-    if (data.private === false && data.visibility === 'public') return data
-    if (!token) throw new GithubApiError(404, 'no-repository')
-    const appToken = await installationToken()
-    if (!appToken) throw new GithubApiError(404, 'no-repository')
-    level = await githubRepoPermission(repo.fullName, appToken, await getIdentity(freshIdentity), fetchImpl)
-    if (!READ_PERMISSIONS.has(level)) throw new GithubApiError(404, 'no-repository')
-    token = appToken
-    return data
+  // A repository last known private asks who the user is as it asks for its
+  // metadata, not after: its permission is asked of them.
+  if (repo.private && userToken) getIdentity().catch(() => {})
+  // Stored visibility can be stale (or internal with private=false). Require
+  // current, explicit public visibility or the caller's effective permission.
+  const data = await metadata() as RepoMetadata
+  if (data?.id !== repo.repoId || typeof data.full_name !== 'string' || data.full_name.toLowerCase() !== repo.fullName.toLowerCase()) {
+    throw new GithubApiError(404, 'no-repository')
+  }
+  if (data.private === false && data.visibility === 'public') return { data, token, level: null }
+  if (!token) throw new GithubApiError(404, 'no-repository')
+  const appToken = await installationToken()
+  if (!appToken) throw new GithubApiError(404, 'no-repository')
+  const level = await githubRepoPermission(repo.fullName, appToken, await getIdentity(), fetchImpl)
+  if (!READ_PERMISSIONS.has(level)) throw new GithubApiError(404, 'no-repository')
+  return { data, token: appToken, level }
+}
+
+// GitHub's answer as kept (verdictCache), with the credential it chose,
+// where that credential is still had; else asked (`ask`), and kept where it
+// grants access.
+async function keptAccess(verdicts: VerdictCache | null, userToken: string | null, installationToken: () => Promise<string | null>,
+  ask: () => Promise<Access>): Promise<Access> {
+  const kept = verdicts?.get()
+  if (kept) {
+    const credential = kept.credential === 'user' ? userToken : kept.credential === 'app' ? await installationToken() : null
+    if (kept.credential === 'none' || credential) return { data: kept.data, token: credential, level: kept.level }
+  }
+  const asked = await ask()
+  const { id, full_name, private: hidden, visibility, default_branch } = asked.data
+  const data = { id, full_name, private: hidden, visibility, default_branch }
+  verdicts?.set({ data, credential: asked.token === null ? 'none' : asked.token === userToken ? 'user' : 'app', level: asked.level })
+  return { ...asked, data }
+}
+
+async function repositoryReader(repo: SelectedRepo, userToken: string | null, installationToken: () => Promise<string | null>, getIdentity: (fresh?: boolean) => Promise<GithubIdentity>, fetchImpl: typeof fetch,
+  verdicts: VerdictCache | null) {
+  const base = `https://api.github.com/repos/${repo.fullName.split('/').map(encodeURIComponent).join('/')}`
+  // Public reads can use the signed-in user's quota without repository grants.
+  // Nonpublic reads still require the repository App and the effective-user gate,
+  // whose permission level is kept (`level`).
+  let level: string | null = null, token = userToken ?? await installationToken()
+  const read = (suffix: string) => githubJson(base + suffix, token, fetchImpl)
+  const checkAccess = async (): Promise<RepoMetadata> => {
+    const access = await keptAccess(verdicts, userToken, installationToken, () => askAccess(repo, token, userToken, installationToken, getIdentity, fetchImpl))
+    token = access.token
+    level = access.level
+    return access.data
   }
   const current = await checkAccess()
   const defaultBranch = typeof current.default_branch === 'string' ? current.default_branch : ''
   return {
-    recheckAccess: () => checkAccess(true),
+    // Its visibility and the user's permission asked again, the user as
+    // GitHub told them earlier in the request, not asked again.
+    recheckAccess: () => checkAccess(),
     // Server-only credential, selected by the same live visibility/permission
     // gate as directory browsing. Never included in an HTTP response.
     readToken: () => token,

package/server-managed/repository-dependency-reviews.ts

@@ -38,8 +38,13 @@ export interface PublishedReview {
 // published reviews of the packages asked.
 export interface RepoReviewState extends Omit<RepoDependencyReview, 'enabled'> { canReview: boolean; reviews: PublishedReview[] }
 
+// A package version whose published reviews are asked.
+export interface ReviewedVersionKey { name: string; version: string }
+
 export interface RepositoryDependencyReviewStore {
   listRepoDependencyReviews(): Promise<Map<number, RepoDependencyReview>>
+  // One repository's, none where it isn't connected.
+  repoDependencyReview(repoId: number): Promise<RepoDependencyReview | null>
   // The teams whose published reviews count for each repository (COUNTING_TEAM),
   // by name.
   listRepoReviewTeams(): Promise<Map<number, { id: string; name: string }[]>>
@@ -47,7 +52,9 @@ export interface RepositoryDependencyReviewStore {
   // `since` or `lockfiles` is undefined, which keeps it; the setting then, or
   // null for no repository.
   setRepoDependencyReview(repoId: number, setting: Omit<RepoDependencyReview, 'since' | 'lockfiles'> & { since?: string | null; lockfiles?: LockfileScope }): Promise<RepoDependencyReview | null>
-  repoDependencyReviewState(repoId: number, user: string, packages: string[]): Promise<RepoReviewState | null>
+  // Its review in effect for `user`, as `setting` tells it where read
+  // already, with the published reviews of `versions`.
+  repoDependencyReviewState(repoId: number, user: string, versions: ReviewedVersionKey[], setting?: RepoDependencyReview | null): Promise<RepoReviewState | null>
   // Which of `users` publish reviews to a team counting for repository
   // `repoId`, whose reviews its pull requests' comments make count.
   listRepoReviewers(repoId: number, users: string[]): Promise<Set<string>>
@@ -59,33 +66,42 @@ const setting = (row: SettingRow): RepoDependencyReview => ({ enabled: row.enabl
 // dependencies and have it, not hidden.
 const COUNTING_TEAM = `t.hidden = 0 AND t.dependency_review = 1 AND EXISTS (
   SELECT 1 FROM managed_team_repo tr WHERE tr.team_id = t.id AND tr.repo_id = ?)`
-const PACKAGES_AT_ONCE = 500
+const VERSIONS_AT_ONCE = 250
 
-// The published and unrevoked reviews of `packages` in teams counting for
-// repository `repoId`, each with whether it is full, a few hundred packages
-// a query.
-async function publishedReviews(db: ManagedSql, repoId: number, packages: string[]): Promise<PublishedReview[]> {
+// The published and unrevoked reviews of `versions` in teams counting for
+// repository `repoId`, each with whether it is full, a couple of hundred
+// versions a query: those of their packages' and versions' found, the
+// versions asked alone kept, their files' reviews read of those alone.
+async function publishedReviews(db: ManagedSql, repoId: number, versions: ReviewedVersionKey[]): Promise<PublishedReview[]> {
   const found: PublishedReview[] = []
-  for (let at = 0; at < packages.length; at += PACKAGES_AT_ONCE) {
-    const names = packages.slice(at, at + PACKAGES_AT_ONCE)
-    const rows = await db.prepare(`SELECT r.id, r.user_id AS userId, r.user_login AS userLogin, r.package, r.version, r.integrity,
+  for (let at = 0; at < versions.length; at += VERSIONS_AT_ONCE) {
+    const asked = versions.slice(at, at + VERSIONS_AT_ONCE)
+    const keys = new Set(asked.map(({ name, version }) => `${name}@${version}`))
+    const [names, numbers] = [[...new Set(asked.map(({ name }) => name))], [...new Set(asked.map(({ version }) => version))]]
+    const rows = (await db.prepare(`SELECT r.id, r.user_id AS userId, r.user_login AS userLogin, r.package, r.version, r.integrity,
         NULLIF(r.base_package, '') AS basePackage, NULLIF(r.base_version, '') AS baseVersion, r.files, r.status, r.published_at AS publishedAt
       FROM managed_dependency_review r
       WHERE r.ecosystem = 'npm' AND r.published_at IS NOT NULL AND r.revoked_at IS NULL AND r.status IS NOT NULL
-        AND r.package IN (${names.map(() => '?').join(',')})
+        AND r.package IN (${names.map(() => '?').join(',')}) AND r.version IN (${numbers.map(() => '?').join(',')})
         AND EXISTS (SELECT 1 FROM managed_dependency_review_team rt JOIN managed_team t ON t.id = rt.team_id
-          WHERE rt.review_id = r.id AND ${COUNTING_TEAM})`).all(...names, repoId) as
+          WHERE rt.review_id = r.id AND ${COUNTING_TEAM})`).all(...names, ...numbers, repoId) as
       { id: string; userId: string | null; userLogin: string; package: string; version: string; integrity: string
-        basePackage: string | null; baseVersion: string | null; files: string | null; status: ReviewVerdict; publishedAt: number }[]
+        basePackage: string | null; baseVersion: string | null; files: string | null; status: ReviewVerdict; publishedAt: number }[])
+      .filter(row => keys.has(`${row.package}@${row.version}`))
     if (rows.length === 0) continue
-    const entries = await db.prepare(`SELECT review_id AS reviewId, file, status FROM managed_dependency_file_review
-      WHERE review_id IN (${rows.map(() => '?').join(',')})`).all(...rows.map(row => row.id)) as { reviewId: string; file: string; status: ReviewVerdict | null }[]
+    const entries = new Map<string, { file: string; status: ReviewVerdict | null }[]>()
+    for (const { reviewId, ...entry } of await db.prepare(`SELECT review_id AS reviewId, file, status FROM managed_dependency_file_review
+      WHERE review_id IN (${rows.map(() => '?').join(',')})`).all(...rows.map(row => row.id)) as { reviewId: string; file: string; status: ReviewVerdict | null }[]) {
+      const of = entries.get(reviewId)
+      if (of) of.push(entry)
+      else entries.set(reviewId, [entry])
+    }
     for (const row of rows) {
       const snapshot = row.files === null ? null : JSON.parse(row.files) as string[] | { paths: string[]; nonTextFiles: string[] }
       // A snapshot of paths alone, before non-text files were told, can't be full.
       const files = Array.isArray(snapshot) ? snapshot : snapshot?.paths ?? null
       const nonText = Array.isArray(snapshot) ? null : snapshot?.nonTextFiles ?? []
-      const full = summarizeReview(entries.filter(entry => entry.reviewId === row.id), files, nonText).coverage === 'full'
+      const full = summarizeReview(entries.get(row.id) ?? [], files, nonText).coverage === 'full'
       found.push({ id: row.id, package: row.package, version: row.version, integrity: row.integrity, basePackage: row.basePackage, baseVersion: row.baseVersion,
         reviewer: row.userId ?? `login:${row.userLogin.toLowerCase()}`, status: row.status, full, time: Number(row.publishedAt) })
     }
@@ -96,11 +112,16 @@ async function publishedReviews(db: ManagedSql, repoId: number, packages: string
 export function repositoryDependencyReviewMethods(db: ManagedSql): RepositoryDependencyReviewStore {
   const columns = `repo_id AS repoId, dependency_review AS enabled, dependency_reviews_required AS required, dependency_review_since AS since,
     dependency_review_lockfiles AS lockfiles`
+  const readSetting = async (repoId: number) => {
+    const row = await db.prepare(`SELECT ${columns} FROM managed_selected_repo WHERE repo_id = ?`).get(repoId) as SettingRow | undefined
+    return row ? setting(row) : null
+  }
   return {
     async listRepoDependencyReviews() {
       const rows = await db.prepare(`SELECT ${columns} FROM managed_selected_repo`).all() as SettingRow[]
       return new Map(rows.map(row => [row.repoId, setting(row)]))
     },
+    repoDependencyReview: readSetting,
     async listRepoReviewTeams() {
       const rows = await db.prepare(`SELECT DISTINCT tr.repo_id AS repoId, t.id AS id, t.name AS name FROM managed_team_repo tr
         JOIN managed_team t ON t.id = tr.team_id WHERE t.hidden = 0 AND t.dependency_review = 1 ORDER BY t.name, t.id`).all() as { repoId: number; id: string; name: string }[]
@@ -123,13 +144,14 @@ export function repositoryDependencyReviewMethods(db: ManagedSql): RepositoryDep
         WHERE u.id IN (${users.map(() => '?').join(',')}) AND ${reviewsIn(true)} AND ${COUNTING_TEAM}`).all(...users, repoId) as { id: string }[]
       return new Set(rows.map(row => row.id))
     },
-    async repoDependencyReviewState(repoId, user, packages) {
-      const row = await db.prepare(`SELECT ${columns} FROM managed_selected_repo WHERE repo_id = ?`).get(repoId) as SettingRow | undefined
-      if (!row || row.enabled !== 1) return null
+    async repoDependencyReviewState(repoId, user, versions, read) {
+      const now = read === undefined ? await readSetting(repoId) : read
+      if (!now?.enabled) return null
       if (!await db.prepare(`SELECT 1 FROM managed_team t WHERE ${COUNTING_TEAM}`).get(repoId)) return null
       const canReview = await db.prepare(`SELECT 1 FROM managed_team t JOIN managed_user u ON u.id = ?
         WHERE ${reviewsIn(true)} AND ${COUNTING_TEAM}`).get(user, repoId) != null
-      return { required: row.required, since: row.since, lockfiles: storedScope(row.lockfiles), canReview, reviews: await publishedReviews(db, repoId, [...new Set(packages)]) }
+      const asked = [...new Map(versions.map(version => [`${version.name}@${version.version}`, version])).values()]
+      return { required: now.required, since: now.since, lockfiles: now.lockfiles, canReview, reviews: await publishedReviews(db, repoId, asked) }
     },
   }
 }

package/server-managed/repository-history.ts

@@ -122,7 +122,7 @@ const TIMED_OUT = /\btimeout\b/iu
 // are logged, those read past too.
 export async function githubGraphql(query: string, variables: Record<string, unknown>, token: string, fetchImpl: typeof fetch,
   tolerated: (error: GraphqlError) => boolean = () => false) {
-  const body = await githubJson('https://api.github.com/graphql', token, fetchImpl, 'POST', { query, variables }) as { data?: unknown; errors?: GraphqlError[] } | null
+  const body = await githubJson('https://api.github.com/graphql', token, fetchImpl, 'POST', { query, variables }, true) as { data?: unknown; errors?: GraphqlError[] } | null
   const all = Array.isArray(body?.errors) ? body.errors : []
   const unnamed = all.filter(error => !NAMED_ERRORS.has(String(error?.type)))
   if (unnamed.length > 0) {
@@ -172,6 +172,9 @@ const GLUE_LOOKUP = 50
 // changes.
 export interface HistoryPages {
   read(tops: string[]): Promise<Map<string, HistoryCommit[]>>
+  // The pages kept of `first`; where none is, those of `then`, in one read
+  // (`first` whether they are `first`'s). Without it, both are read together.
+  readFirst?(first: string[], then: string[]): Promise<{ first: boolean; pages: Map<string, HistoryCommit[]> }>
   write(top: string, commits: HistoryCommit[]): Promise<void>
 }
 
@@ -241,8 +244,11 @@ const graphCommit = (node: GraphCommit | null | undefined) => historyCommit(node
 
 // The history from each of `starts`, a page of each, kept (`pages`), or read
 // and then kept by its top; glued on to it, the pages kept that start at a
-// parent of a commit listed not listed, or at a commit listed, the oldest
-// first (where each goes on furthest), GLUE_LOOKUP looked up at a time, and
+// parent of a commit listed not listed, each adding commits none listed has,
+// and, where any of those has none kept and readable, at a commit listed, the
+// oldest first (where each goes on furthest; a page at a commit listed lists
+// mostly commits listed already), GLUE_LOOKUP of each looked up at a time, in
+// one read where none of those parents' is kept, else two, and
 // those that start where those go on, till HISTORY_GLUED are listed. The
 // pages from `starts` are listed whole, however many that is (all branches'
 // first, as many as ALL_BRANCHES of them): the cap is gluing's. Each commit
@@ -259,11 +265,21 @@ export async function historyFrom(fullName: string, token: string | null, starts
   for (const top of starts) add(kept.get(top) ?? read.get(top) ?? [])
   const looked = new Set(starts)
   for (let round = 0; round < MAX_GLUE_ROUNDS && listed.size < HISTORY_GLUED; round++) {
-    const parents = [...listed.values()].flatMap(commit => commit.parents.filter(parent => !listed.has(parent)))
-    const tops = [...new Set([...parents, ...[...listed.keys()].toReversed()])].filter(sha => !looked.has(sha)).slice(0, GLUE_LOOKUP)
-    if (tops.length === 0) break
-    for (const sha of tops) looked.add(sha)
-    const glued = await pages.read(tops)
+    const parents = [...new Set([...listed.values()].flatMap(commit => commit.parents.filter(parent => !listed.has(parent) && !looked.has(parent))))].slice(0, GLUE_LOOKUP)
+    const within = [...listed.keys()].toReversed().filter(sha => !looked.has(sha)).slice(0, GLUE_LOOKUP)
+    if (parents.length === 0 && within.length === 0) break
+    // Without readFirst, GLUE_LOOKUP of both at once, the parents first.
+    const asked = pages.readFirst ? null : [...parents, ...within].slice(0, GLUE_LOOKUP)
+    let { first, pages: glued } = asked ? { first: false, pages: await pages.read(asked) } : await pages.readFirst!(parents, within)
+    // A parent with no page kept, or none readable (one GitHub's REST API
+    // listed, read with GraphQL), its line going on only by a page at a
+    // commit listed: those read too, apart, the others' lines notwithstanding.
+    if (first && glued.size < parents.length && within.length > 0) {
+      glued = new Map([...glued, ...await pages.read(within)])
+      first = false
+    }
+    // Those at commits listed, unread where a parent's answered, are looked up again.
+    for (const sha of asked ?? (first ? parents : [...parents, ...within])) looked.add(sha)
     for (const commit of [...glued.values()].flat()) {
       if (listed.size >= HISTORY_GLUED) break
       if (!listed.has(commit.sha)) listed.set(commit.sha, commit)

package/server-managed/repository-pulse.ts

@@ -14,6 +14,7 @@
 // issues closed and opened, how many and the latest of each.
 import { agentOperated, githubAgent } from '../common/managed/github-agents.ts'
 import { GithubApiError, githubJson } from './github-app.ts'
+import { shareGithubRead } from './github-requests.ts'
 import { ACTOR, type GraphAccount, type HistoryCommit, type HistoryPages, accountOf, actorOf, githubGraphql, historyFrom } from './repository-history.ts'
 
 export const PULSE_WEEKS = [1, 4, 13, 52]
@@ -99,7 +100,13 @@ const emptyPunch = () => Array.from({ length: 7 }, () => Array.from({ length: 24
 // One statistic: its rows once GitHub has computed it, 'pending' while it
 // computes them (202), none for a repository with no commits (204, no body),
 // 'too-large' where it counts none (lines past 10,000 commits, 422).
-async function statistic(fullName: string, token: string | null, kind: string, fetchImpl: typeof fetch): Promise<unknown[] | 'pending' | 'too-large'> {
+function statistic(fullName: string, token: string | null, kind: string, fetchImpl: typeof fetch) {
+  // Stats and lines both need contributors. Share by the original transport;
+  // the per-call status wrapper below would otherwise give each its own read.
+  return shareGithubRead(fetchImpl, JSON.stringify(['statistic', fullName, token, kind]), () => readStatistic(fullName, token, kind, fetchImpl))
+}
+
+async function readStatistic(fullName: string, token: string | null, kind: string, fetchImpl: typeof fetch): Promise<unknown[] | 'pending' | 'too-large'> {
   let status = 0
   const answered: typeof fetch = async (input, init) => {
     const res = await fetchImpl(input, init)

package/server-managed/repository-risk-audit.ts

@@ -33,6 +33,19 @@ const AUDIT_FAILED_KEPT_MS = 10 * 60_000
 // RISK_KEPT_MS, why they couldn't be had within AUDIT_FAILED_KEPT_MS.
 const isFresh = ({ at, audit }: { at: number; audit: Audit }) => Date.now() - at < (Array.isArray(audit) ? RISK_KEPT_MS : AUDIT_FAILED_KEPT_MS)
 
+// The audits of lockfiles, each of its `packages` by its `key` (auditLockfile),
+// `inTurn` with others, asked at once: each null where it has none to ask
+// of, or once `signal` aborts. Each is the caller's to tell once it may.
+export function auditLockfiles(source: AuditSource, memory: Lru<{ at: number; audit: Audit }>, signal: AbortSignal,
+  lockfiles: { key: string; packages: readonly LockedPackage[] | null }[], inTurn: <T>(work: () => Promise<T>) => Promise<T>): Promise<Audit | null>[] {
+  return lockfiles.map(({ key, packages }) => {
+    const audit = packages ? inTurn(() => (signal.aborted ? Promise.resolve(null) : auditLockfile(source, memory, signal, key, packages))) : Promise.resolve(null)
+    // One no longer waited on, as the scan stopped, fails alone.
+    audit.catch(() => {})
+    return audit
+  })
+}
+
 // The audit of a lockfile's `packages`, by its `key` as kept (`memory`, then
 // the source's), else asked and kept; null where it locks none to ask of, or
 // once `signal` aborts.

package/server-managed/repository-risk-read.ts

@@ -16,6 +16,7 @@ import type { DependabotAlert } from './dependabot-alerts.ts'
 import { type LockedPackage, isTestFixture, lockedPackages } from './locked-packages.ts'
 import type { Lru } from './lru.ts'
 import type { KeptStore } from './npm-kept.ts'
+import { inTurns } from './queue.ts'
 import { type WorkflowsRead, lastingRead, readActions } from './repository-actions.ts'
 import { type RiskFailure, failureOf } from './repository-risk-assess.ts'
 import type { RepositoryRiskLine, RepositoryRiskRepo } from './repository-risk.ts'
@@ -28,6 +29,9 @@ const READ_FORMATS: ReadonlySet<LockfileFormat> = new Set(['npm', 'pnpm', 'yarn'
 // How long the source's `listings` and `lockfiles` are read for: what
 // GitHub has at a commit or a blob doesn't change, how it is read here may.
 export const SCAN_READ_KEPT_MS = 24 * 60 * 60_000
+// How many of a repository's lockfiles are read at once: each one GitHub is
+// asked for where it isn't kept, then parsed.
+const LOCKFILES_AT_ONCE = 4
 
 export type ListedLockfile = Pick<RepoLockfile, 'path' | 'format' | 'sha'>
 export type LockfileRead = { packages: LockedPackage[] } | RiskFailure
@@ -140,8 +144,8 @@ async function readDependabot(opened: OpenedRepository, paths: readonly string[]
   }
 }
 
-// A repository read: its lockfiles listed and read, its Dependabot alerts
-// asked and its Actions read meanwhile; its line telling why where listing
+// A repository read: its lockfiles listed and read, a few at once, its
+// Dependabot alerts asked and its Actions read meanwhile; its line telling why where listing
 // failed; null where the scan stopped. A test's
 // fixture or example its reader refuses (isTestFixture) is left out, as a
 // test may keep one so on purpose.
@@ -153,11 +157,12 @@ export async function readRepository<R extends RepositoryRiskRepo>(reading: Repo
   const acting = workflows ? keptRead(reading.caches.actionFiles, reading.source.actionFiles, `${repo.repoId}@${opened.commit}`,
     () => readActions({ workflows, file: path => opened.file(path) }), lastingRead) : Promise.resolve(null)
   acting.catch(() => {})
+  const inTurn = inTurns(LOCKFILES_AT_ONCE)
+  const lines = await Promise.all(listed.lockfiles.map(lockfile => inTurn(() => (reading.signal.aborted ? Promise.resolve(null) : readLockfile(reading, repo.repoId, opened, lockfile)))))
   const lockfiles: RepositoryRead['lockfiles'] = []
-  for (const lockfile of listed.lockfiles) {
-    if (reading.signal.aborted) return null
-    const line = await readLockfile(reading, repo.repoId, listed.opened, lockfile)
-    if (!('error' in line && line.error === 'lockfile-unreadable' && isTestFixture(line.path))) lockfiles.push({ sha: lockfile.sha, line })
+  for (const [at, line] of lines.entries()) {
+    if (!line) return null
+    if (!('error' in line && line.error === 'lockfile-unreadable' && isTestFixture(line.path))) lockfiles.push({ sha: listed.lockfiles[at]!.sha, line })
   }
   const dependabot = await alerting
   const actions = await acting

package/server-managed/repository-risk.ts

@@ -34,10 +34,10 @@ import { type Lru, lru } from './lru.ts'
 import type { KeptStore } from './npm-kept.ts'
 import { MAX_SPARE_ASSESSMENTS } from './npm-risk.ts'
 import type { LockedPackage } from './locked-packages.ts'
-import { type Queue, queue } from './queue.ts'
+import { type Queue, inTurns, queue } from './queue.ts'
 import { type DownloadsSource, readyingDownloads } from './scan-downloads.ts'
 import { type AssessedPackage, type Assessment, BUSY_RETRIES, BUSY_WAIT_MS, RISK_KEPT_MS, type RiskFailure, assessPackage, failureOf, isBusy } from './repository-risk-assess.ts'
-import { type AdvisoryRow, type Audit, type AuditSource, auditLockfile } from './repository-risk-audit.ts'
+import { type AdvisoryRow, type Audit, type AuditSource, auditLockfiles } from './repository-risk-audit.ts'
 import { type DependabotRead, type ListedLockfile, type LockfileRead, type OpenedRepository, type RepositoryFailure, type RepositoryRead, SCAN_READ_KEPT_MS,
   readRepository } from './repository-risk-read.ts'
 import { type DependabotAlert, alertVersions } from './dependabot-alerts.ts'
@@ -157,6 +157,9 @@ export const riskCaches = (): RiskCaches => ({
 // the one told: each asks GitHub one thing after another (opening it, and
 // its head) even where all else is kept.
 const READ_AHEAD = 6
+// How many lockfiles' advisories are asked of npm at once, those of the
+// repositories read ahead as they are read, each told in its turn.
+const AUDITS_AT_ONCE = 4
 // How long a part's progress, written as it ends (`progress`), is waited on.
 const PROGRESS_WRITE_MS = 5000
 // How many kept risk lines are read of the source's `risks` at once.
@@ -180,6 +183,7 @@ interface Scan {
   // The repositories told whole, this part or one before (ScanProgress).
   told: Map<number, ToldRepository>
   actions: ReturnType<typeof resolvingActions> | null
+  audits: ReturnType<typeof inTurns>
 }
 
 // A package as a scan meets it: by its name, version and a mirror's integrity.
@@ -285,16 +289,16 @@ function assessLater(scan: Scan, packages: LockedPackage[]) {
   if (fresh.length > 0) scan.lookups.add(fresh)
 }
 
-type AdvisoriesLine = Extract<RepositoryRiskLine, { type: 'advisories' }>
 type LockfileLine = Extract<RepositoryRiskLine, { type: 'lockfile' }>
 
-// A lockfile's advisories line (repository-risk-audit.ts auditLockfile);
-// null where it locks none, or once the scan stops.
-async function readAdvisories(scan: Scan, { repo, path, format, packages }: LockfileLine & { packages: LockedPackage[] }, sha: string): Promise<AdvisoriesLine | null> {
-  const audit = await auditLockfile(scan.source, scan.caches.advisories, scan.signal, `${format}\0${sha}`, packages)
-  const at = { type: 'advisories', repo, path } as const
-  return audit === null ? null : Array.isArray(audit) ? { ...at, advisories: audit } : { ...at, ...audit }
-}
+// A lockfile's advisories line, of its audit (repository-risk-audit.ts auditLockfile).
+const advisoriesLine = ({ repo, path }: LockfileLine, audit: Audit) => ({ type: 'advisories', repo, path, ...Array.isArray(audit) ? { advisories: audit } : audit } as const)
+
+// A repository read, its lockfiles' audits asked meanwhile, AUDITS_AT_ONCE
+// across the scan (auditLockfiles), to tell in turn once access is checked again.
+type Audited = (RepositoryRead & { audits: Promise<Audit | null>[] }) | RepositoryFailure
+const audited = (scan: Scan, read: RepositoryRead | RepositoryFailure | null): Audited | null => (read === null || 'type' in read ? read : { ...read, audits: auditLockfiles(scan.source, scan.caches.advisories,
+  scan.signal, read.lockfiles.map(({ sha, line }) => ({ key: `${line.format}\0${sha}`, packages: 'packages' in line ? line.packages : null })), scan.audits) })
 
 type DependabotLine = Extract<RepositoryRiskLine, { type: 'dependabot' }>
 
@@ -319,10 +323,11 @@ async function rechecked(read: DependabotRead): Promise<DependabotRead> {
 
 // A repository read (repository-risk-read.ts readRepository) told: its
 // line, or why it couldn't be read, then its lockfiles', and their
-// advisories, its packages assessed meanwhile, then its Dependabot alerts.
+// advisories, asked as it was read, its packages assessed meanwhile, then
+// its Dependabot alerts.
 // Access to it is checked again
 // just before, however long ago it was read, as GitHub may have withdrawn it.
-async function tellRepository(scan: Scan, repo: RepositoryRiskRepo, read: RepositoryRead | RepositoryFailure | null) {
+async function tellRepository(scan: Scan, repo: RepositoryRiskRepo, read: Audited | null) {
   if (!read || scan.signal.aborted) return
   if ('type' in read) { scan.lines.add(read); return }
   try { await read.recheck() } catch (error) {
@@ -337,10 +342,10 @@ async function tellRepository(scan: Scan, repo: RepositoryRiskRepo, read: Reposi
     if ('packages' in line) assessLater(scan, line.packages)
   }
   const refs = read.actions ? tellActions(scan, repo, read.actions, lockfiles) : []
-  for (const { sha, line } of lockfiles) {
+  for (const [at, { line }] of lockfiles.entries()) {
     if (scan.signal.aborted) return
-    const advisories = 'packages' in line ? await readAdvisories(scan, line, sha) : null
-    if (advisories) scan.lines.add(advisories)
+    const audit = await read.audits[at]
+    if (audit) scan.lines.add(advisoriesLine(line, audit))
   }
   if (scan.signal.aborted) return
   const dependabot = read.dependabot && await rechecked(read.dependabot)
@@ -366,9 +371,10 @@ function tellActions(scan: Scan, repo: RepositoryRiskRepo, read: WorkflowsRead,
 async function toldBefore(scan: Scan, repo: RepositoryRiskRepo) {
   const told = scan.told.get(repo.repoId)
   if (!told || told.name !== repo.fullName || JSON.stringify(told.paths) !== JSON.stringify(repo.paths ?? null)) return false
-  const reads: LockedPackage[][] = []
+  const reads: LockedPackage[][] = [], unknown = told.lockfiles.filter(key => !scan.caches.lockfiles.get(key))
+  const stored = unknown.length > 0 && scan.source.lockfiles ? await scan.source.lockfiles.readMany(unknown) : null
   for (const key of told.lockfiles) {
-    const read = scan.caches.lockfiles.get(key) ?? (await scan.source.lockfiles?.read(key))?.value
+    const read = scan.caches.lockfiles.get(key) ?? stored?.get(key)?.value
     if (!read || !('packages' in read)) return false
     reads.push(read.packages)
   }
@@ -388,11 +394,11 @@ async function readRepositories(scan: Scan) {
   const kept = await scan.source.progress?.read()
   const progress = kept && Date.now() - kept.at < SCAN_READ_KEPT_MS ? kept.value.repos : []
   for (const told of progress) if (listed.has(told.id)) scan.told.set(told.id, told)
-  const reads: Promise<RepositoryRead | RepositoryFailure | null>[] = []
+  const reads: Promise<Audited | null>[] = []
   const read = (at: number) => {
     if (reads[at]) return reads[at]
     const repo = repos[at]!
-    const started = reads[at] = toldBefore(scan, repo).then(told => told ? null : readRepository(scan, repo))
+    const started = reads[at] = toldBefore(scan, repo).then(async told => (told ? null : audited(scan, await readRepository(scan, repo))))
     // One read ahead and no longer waited on, as the scan stopped, fails alone.
     started.catch(() => {})
     return started
@@ -438,7 +444,8 @@ export async function* repositoryRiskLines<R extends RepositoryRiskRepo>(source:
   const lines = queue<RepositoryRiskLine>(AbortSignal.any([signal, part.signal]), { drain: true })
   const actions = source.actions ? resolvingActions({ source: source.actions, signal: running, tell: line => { lines.add(line) }, memory: caches.actions,
     answers: caches.actionAnswers, store: source.actionLines }) : null
-  const scan: Scan = { source, caches, signal: running, lines, lookups: queue(running), seen: new Map(), dev: new Map(), stale: [], told: new Map(), actions }
+  const scan: Scan = { source, caches, signal: running, lines, lookups: queue(running), seen: new Map(), dev: new Map(), stale: [], told: new Map(), actions,
+    audits: inTurns(AUDITS_AT_ONCE) }
   const failed: unknown[] = []
   void (async () => {
     try {