@preventive/triage 1.0.0-alpha.46 → 1.0.0-alpha.48

package/server-managed/repository-code.ts

@@ -4,27 +4,44 @@
 // bundle builds keep theirs (getRepoTreeTarball), and its files as the npm
 // viewer reads a package's (npm-packages.ts npmFileRows). Readers asking for
 // the same directory meanwhile share its body, a load among the npm viewer's
-// active ones (npm-loads.ts sharedLoad).
+// active ones (npm-loads.ts sharedLoad). The default branch's head's bodies
+// are kept in byte storage, for the next to open them (repositoryCodeKept).
 import { Buffer } from 'node:buffer'
+import { createHash } from 'node:crypto'
 import { encodeBrotli } from './brotli.ts'
-import { MAX_NPM_PACKAGE_BYTES, MAX_NPM_PACKAGE_FILES, NpmPackageError, npmFileRows } from './npm-packages.ts'
-import { type NpmLoad, readTarballFiles, sharedLoad } from './npm-loads.ts'
+import { CacheMissError, type CacheStorage } from './cache-storage.ts'
+import type { KeptStore } from './npm-kept.ts'
+import { plainObject } from './npm-manifest.ts'
+import { MAX_NPM_JSON_LENGTH, NpmPackageError, npmFileRows } from './npm-packages.ts'
+import { type NpmLoad, loadPlaces, readTarballFiles, sharedLoad } from './npm-loads.ts'
 
-// As much as a package may hold: the viewer holds every file in memory.
-export const MAX_REPOSITORY_CODE_FILES = MAX_NPM_PACKAGE_FILES
-export const MAX_REPOSITORY_CODE_BYTES = MAX_NPM_PACKAGE_BYTES
+// What a directory may hold, bounded apart from a package's (npm-packages.ts),
+// as the viewer holds every file in memory: its files' count and bytes.
+export const MAX_REPOSITORY_CODE_FILES = 80_000
+export const MAX_REPOSITORY_CODE_BYTES = 256 * 1024 * 1024
+// The tarball they come in, bounded in proportion as a package's is: its tar
+// stream, headers and padding included, and its entries, as many again as
+// files, as GitHub's has one for each directory too.
+const REPOSITORY_TAR = { maxFiles: MAX_REPOSITORY_CODE_FILES, maxBytes: MAX_REPOSITORY_CODE_BYTES, maxEntries: 2 * MAX_REPOSITORY_CODE_FILES, maxTarBytes: 384 * 1024 * 1024 }
+// What the files may take as the viewer's JSON, written as one string: short
+// of the longest V8 holds (2^29 - 24 characters).
+const MAX_REPOSITORY_CODE_JSON_LENGTH = 384 * 1024 * 1024
 
 // Whether a directory's files fit, by GitHub's listing of its tree and every
 // tree under it (repository-browser.ts repositoryTreeFiles): asked before
 // its tarball is, which may run to hundreds of MiB. One GitHub cut short
 // (null) has too many.
-export function repositoryCodeFits(files: { files: number; bytes: number } | null): boolean {
+export function repositoryCodeFits(files: RepositoryCodeFiles | null): files is RepositoryCodeFiles {
   return files !== null && files.files <= MAX_REPOSITORY_CODE_FILES && files.bytes <= MAX_REPOSITORY_CODE_BYTES
 }
 
+// A directory's files' count and bytes, as GitHub lists its tree.
+export interface RepositoryCodeFiles { files: number; bytes: number }
+
 // The directory read: the repository by its id and name, the commit, the
-// directory (`''` for the root) and its tree's id.
-export interface RepositoryCodeInput { repoId: number; repo: string; commit: string; path: string; tree: string }
+// directory (`''` for the root), its tree's id, and its files as listed,
+// which weigh its load among the active ones.
+export interface RepositoryCodeInput { repoId: number; repo: string; commit: string; path: string; tree: string; listed: RepositoryCodeFiles }
 
 // A package's refusals, as a directory's: past the bounds, or an archive
 // @preventive/archive refuses, as it does one holding a symlink that climbs
@@ -37,16 +54,162 @@ const directoryError = (err: unknown) => {
 
 // The directory as the viewer reads it, as brotli-encoded JSON: `{ repoId,
 // repo, commit, path, tree, files }`, its files as npmFileRows gives them,
-// by their paths in the directory, from `tarball`, the tree's.
-export function loadRepositoryCodeBody(input: RepositoryCodeInput, tarball: () => Promise<Uint8Array>): NpmLoad<Buffer> {
-  const { repoId, repo, commit, path, tree } = input
+// by their paths in the directory, from `tarball`, the tree's, as upstream
+// verifies it with its symlinks let climb `symlinksAbove` levels.
+export function loadRepositoryCodeBody(input: RepositoryCodeInput, tarball: (symlinksAbove: number) => Promise<Uint8Array>): NpmLoad<Buffer> {
+  const { repoId, repo, commit, path, tree, listed } = input
+  // A symlink may reach the repository's root, as a monorepo package's
+  // LICENSE links the root's, and no further: the directory's depth, less
+  // the level the archive's root already stands above its top directory.
+  const symlinksAbove = path.split('/').length - 1
+  // Its places among the active loads, by its files as listed, hold their
+  // JSON too, which the listing cannot tell, as escaping may grow a text
+  // sixfold: as much as a package's may take for each.
+  const places = loadPlaces(listed)
+  const maxLength = Math.min(MAX_REPOSITORY_CODE_JSON_LENGTH, places * MAX_NPM_JSON_LENGTH)
   return sharedLoad(`repository ${repoId} ${repo} ${commit} ${path} ${tree}`, async () => {
     let files
-    // A symlink may reach the repository's root, as a monorepo package's
-    // LICENSE links the root's, and no further: the directory's depth, less
-    // the level the archive's root already stands above its top directory.
-    try { files = npmFileRows(await readTarballFiles(await tarball(), { symlinksAbove: path.split('/').length - 1 })) }
+    try { files = npmFileRows(await readTarballFiles(await tarball(symlinksAbove), { ...REPOSITORY_TAR, symlinksAbove }), maxLength) }
     catch (err) { throw directoryError(err) }
     return encodeBrotli(Buffer.from(JSON.stringify({ repoId, repo, commit, path, tree, files })))
-  }, 'repository-busy')
+  }, 'repository-busy', { places })
+}
+
+// The form of the bodies kept, bumped as what a body holds changes
+// (npmFileRows): one kept in another is none to read.
+const KEPT_CODE_VERSION = 1
+// The largest body kept. Source text runs, as brotli's JSON, to a third of
+// its bytes or less, and a directory's binary files are named, not held, so
+// this keeps one of some 200 MiB of text: nearly every directory that opens
+// at all (MAX_REPOSITORY_CODE_BYTES). One past it would take as much byte
+// storage as hundreds of ordinary directories, and hold a serverless
+// invocation a while uploading it: it is sent as ever, and read again the
+// next time.
+export const MAX_KEPT_CODE_BYTES = 64 * 1024 * 1024
+const COMMIT = /^[a-f\d]{40}$/u
+
+// A directory as kept: the repository by its id and name, the commit and the
+// directory, as its body names them; and the head they are kept at.
+export type KeptCodeAt = Pick<RepositoryCodeInput, 'repoId' | 'repo' | 'commit' | 'path'>
+type KeptCodeHead = Pick<KeptCodeAt, 'commit' | 'repo'>
+// Under the repository, then the commit, so either goes by its prefix; the
+// repository's name and the directory hashed, as a key's parts are names.
+const keptKey = ({ repoId, repo, commit, path }: KeptCodeAt) =>
+  `${repoId}/${commit}/v${KEPT_CODE_VERSION}-${createHash('sha256').update(JSON.stringify([repo, path])).digest('hex')}.json.br`
+
+// Bodies being kept, by their storage and key, each kept once; and each
+// storage's own name among the active loads, as its reads are one of them.
+const keeping = new WeakMap<CacheStorage, Set<string>>()
+const storageNames = new WeakMap<CacheStorage, number>()
+let storagesNamed = 0
+
+// A body kept, read whole, where it is one MAX_KEPT_CODE_BYTES keeps.
+async function readKept(storage: CacheStorage, key: string): Promise<Buffer | null> {
+  const { size, stream } = await storage.open(key)
+  try {
+    if (size !== null && size > MAX_KEPT_CODE_BYTES) return null
+    const parts: Buffer[] = []
+    let length = 0
+    for await (const part of stream) {
+      length += (part as Buffer).byteLength
+      if (length > MAX_KEPT_CODE_BYTES) return null
+      parts.push(Buffer.from(part as Buffer))
+    }
+    return Buffer.concat(parts, length)
+  } finally { stream.destroy() }
+}
+
+// The directories of repositories' default branches' heads, as
+// loadRepositoryCodeBody encodes them, kept in the managed byte storage
+// (`storage`: Vercel Blob, else the filesystem, encrypted at rest where
+// storage is: storage-payload.ts REPOSITORY_CODE_CACHE), so that one opened
+// again, on any instance, asks GitHub for no tarball, and is neither unpacked
+// nor encoded again; by the commit `heads` keeps for each repository, the
+// head as last read (npm-kept.ts codeHeadStore). Most opens read the head:
+// `/repo/<repo id>` opens it, and its route then names its commit, as do its
+// directories opened from it and a reload. A read naming the default branch
+// (`''`, or `heads/` and its name) takes its head as GitHub resolves it for
+// the commit kept, the directories kept at any other removed (`own`ed,
+// behind its answer); any other read, of a commit, a tag or another branch,
+// reads and keeps a directory only where what it resolves to is the commit
+// kept. Nothing kept is per reader, and each is read only once the reader's
+// own access has named its commit. A body names what it was read of, so
+// instances disagreeing on the head meanwhile, as at a push, only cost a
+// directory read again: one kept for a head since moved on removes itself.
+// The head is kept with the repository's name, which each body's key holds,
+// so a repository renamed has what was kept under its name before removed
+// as a read next names its head. A head is written only for a repository
+// still `active` after it, as one removed meanwhile has its head removed
+// after (remove). Byte storage, as the database, is an optimization: its
+// failures read as a directory kept nowhere, and keep nothing.
+export function repositoryCodeKept(storage: CacheStorage, heads: KeptStore<KeptCodeHead>, active: (repoId: number) => Promise<boolean>,
+  own: (work: Promise<unknown>) => void, debug = false) {
+  const warn = (what: string) => (error: unknown) => { if (debug) console.warn(`managed: kept repository code ${what} failed:`, error) }
+  const headOf = async (repoId: number): Promise<KeptCodeHead | null> => {
+    const kept: unknown = (await heads.read(String(repoId)))?.value
+    return plainObject(kept) && typeof kept['commit'] === 'string' && typeof kept['repo'] === 'string' ? { commit: kept['commit'], repo: kept['repo'] } : null
+  }
+  // What is kept of repository `repoId`: its head first, so that a body being
+  // kept meanwhile removes itself.
+  const remove = async (repoId: number) => {
+    await heads.remove(String(repoId))
+    await storage.delete(String(repoId))
+  }
+  return {
+    // Whether a read of `ref` at repository `repoId`, by its name `repo`,
+    // resolved to `commit` with `defaultBranch` GitHub's, reads and keeps
+    // its directories.
+    async at({ repoId, repo, commit }: Omit<KeptCodeAt, 'path'>, ref: string, defaultBranch: string): Promise<boolean> {
+      if (!COMMIT.test(commit)) return false
+      const kept = await headOf(repoId)
+      const atKept = kept?.commit === commit && kept.repo === repo
+      if (ref !== '' && (defaultBranch === '' || ref !== `heads/${defaultBranch}`)) return atKept
+      if (!atKept) {
+        await heads.write(String(repoId), { commit, repo }, Date.now())
+        // One removed as its commit was read, its head removed before this
+        // wrote it again, keeps nothing, and has what is kept removed again.
+        if (!await active(repoId).catch(() => false)) {
+          await remove(repoId).catch(warn('removal'))
+          return false
+        }
+        // Those kept at the head before, or, renamed, every one kept under
+        // its name before.
+        own(storage.delete(String(repoId), kept?.repo === repo ? commit : undefined).catch(warn('removal')))
+      }
+      return true
+    },
+    // The body kept of the directory `at` names, or null: a load among the
+    // active ones (npm-loads.ts sharedLoad), as one read from GitHub is, a
+    // place of them, as it holds no more than a package may
+    // (MAX_KEPT_CODE_BYTES), till each reader has released it; readers
+    // asking for it meanwhile share it, and none is read where every place
+    // is taken (`repository-busy`).
+    read(at: KeptCodeAt): NpmLoad<Buffer | null> {
+      const key = keptKey(at)
+      const name = storageNames.get(storage) ?? storageNames.set(storage, ++storagesNamed).get(storage)!
+      return sharedLoad(`repository kept ${name} ${key}`, () => readKept(storage, key).catch((error: unknown) => {
+        if (!(error instanceof CacheMissError)) warn('read')(error)
+        return null
+      }), 'repository-busy')
+    },
+    // The directory's body kept, behind its answer, where it is no larger
+    // than MAX_KEPT_CODE_BYTES; and removed again, with the rest of its
+    // commit's, where the head kept is another by then, as when it moved on
+    // or the repository was renamed or removed meanwhile.
+    keep(at: KeptCodeAt, body: Buffer): void {
+      const key = keptKey(at)
+      const kept = keeping.get(storage) ?? keeping.set(storage, new Set()).get(storage)!
+      if (body.byteLength > MAX_KEPT_CODE_BYTES || kept.has(key)) return
+      kept.add(key)
+      own((async () => {
+        await storage.put(key, body)
+        const head = await headOf(at.repoId)
+        if (head?.commit !== at.commit || head.repo !== at.repo) await storage.delete(`${at.repoId}/${at.commit}`)
+      })().catch(warn('write')).finally(() => kept.delete(key)))
+    },
+    // What is kept of repository `repoId`, removed once the repository is
+    // no longer active.
+    remove,
+  }
 }
+export type RepositoryCodeKept = ReturnType<typeof repositoryCodeKept>

package/server-managed/repository-dependency-reviews.ts

@@ -45,6 +45,8 @@ export interface RepositoryDependencyReviewStore {
   listRepoDependencyReviews(): Promise<Map<number, RepoDependencyReview>>
   // One repository's, none where it isn't connected.
   repoDependencyReview(repoId: number): Promise<RepoDependencyReview | null>
+  // Those of `repoIds` connected, by their ids, in one statement.
+  getRepoDependencyReviews(repoIds: number[]): Promise<Map<number, RepoDependencyReview>>
   // The teams whose published reviews count for each repository (COUNTING_TEAM),
   // by name.
   listRepoReviewTeams(): Promise<Map<number, { id: string; name: string }[]>>
@@ -122,6 +124,11 @@ export function repositoryDependencyReviewMethods(db: ManagedSql): RepositoryDep
       return new Map(rows.map(row => [row.repoId, setting(row)]))
     },
     repoDependencyReview: readSetting,
+    async getRepoDependencyReviews(repoIds) {
+      if (repoIds.length === 0) return new Map()
+      const rows = await db.prepare(`SELECT ${columns} FROM managed_selected_repo WHERE repo_id IN (${repoIds.map(() => '?').join(',')})`).all(...repoIds) as SettingRow[]
+      return new Map(rows.map(row => [row.repoId, setting(row)]))
+    },
     async listRepoReviewTeams() {
       const rows = await db.prepare(`SELECT DISTINCT tr.repo_id AS repoId, t.id AS id, t.name AS name FROM managed_team_repo tr
         JOIN managed_team t ON t.id = tr.team_id WHERE t.hidden = 0 AND t.dependency_review = 1 ORDER BY t.name, t.id`).all() as { repoId: number; id: string; name: string }[]

package/server-managed/repository-history.ts

@@ -9,11 +9,13 @@
 // read a page from a commit at a time, each page kept by the commit it starts
 // at (historyFrom): a commit's history never changes, and where a commit read
 // starts a page kept, that page glues on, and those after it.
-import { type Coauthor, MAX_COAUTHORS, type Maker, coauthorsOf } from './commit-coauthors.ts'
+import { type Coauthor, MAX_COAUTHORS, type Maker, coauthorTrailers, coauthorsOf } from './commit-coauthors.ts'
 import { GithubApiError, githubJson } from './github-app.ts'
-import { type GraphActor, accountOf, commitAccount } from './github-accounts.ts'
+import { type GraphActor, accountOf, commitAccount, linkNoreplyUsers } from './github-accounts.ts'
+import { githubGraphql } from './github-graphql.ts'
 
-export { ACTOR, type GraphAccount, type GraphActor, type NamedAccount, accountOf, actorOf, commitAccount } from './github-accounts.ts'
+export { ACTOR, type GraphAccount, type GraphActor, type NamedAccount, accountOf, actorOf, commitAccount, linkNoreplyUsers } from './github-accounts.ts'
+export { type GraphqlError, githubGraphql } from './github-graphql.ts'
 
 export const HISTORY_PAGE = 100
 // The deepest page read, as far as Older commits goes.
@@ -79,19 +81,27 @@ function historyCommit(sha: unknown, parents: unknown[], message: unknown, autho
 
 type RestCommit = { sha?: unknown; parents?: { sha?: unknown }[]; author?: { login?: unknown; id?: unknown } | null
   commit?: { message?: unknown; author?: { name?: unknown; email?: unknown } | null; committer?: { date?: unknown } | null } }
-const restCommit = (item: RestCommit) => historyCommit(item?.sha, Array.isArray(item?.parents) ? item.parents.map(parent => parent?.sha) : [],
-  item?.commit?.message, { name: item?.commit?.author?.name, email: item?.commit?.author?.email, account: accountOf(item?.author) }, item?.commit?.committer?.date)
+// Its co-authors (`listed`) by its trailers, with the accounts of their addresses.
+const restCommit = (item: RestCommit, listed: Maker[] | null = null) => historyCommit(item?.sha, Array.isArray(item?.parents) ? item.parents.map(parent => parent?.sha) : [],
+  item?.commit?.message, { name: item?.commit?.author?.name, email: item?.commit?.author?.email, account: accountOf(item?.author) }, item?.commit?.committer?.date,
+  null, listed)
 
 const repoApi = (fullName: string) => `https://api.github.com/repos/${fullName.split('/').map(encodeURIComponent).join('/')}`
 
 // A page of the commits `commit` reaches, newest first as GitHub lists them,
 // those touching `path` alone where it names a directory; `more` where a
-// full page leaves some for the next, one that may be read.
+// full page leaves some for the next, one that may be read. GitHub links
+// each's author to its account; with a token, its co-authors by GitHub's older
+// no-reply address too (linkNoreplyUsers).
 export async function repositoryHistory(fullName: string, token: string | null, commit: string, path: string, page: number, fetchImpl: typeof fetch = globalThis.fetch) {
   const params = new URLSearchParams({ sha: commit, per_page: String(HISTORY_PAGE), page: String(page), ...path ? { path } : {} })
   const data = await githubJson(`${repoApi(fullName)}/commits?${params}`, token, fetchImpl)
   if (!Array.isArray(data)) throw new GithubApiError(502, 'github-malformed')
-  return { commits: data.flatMap(item => restCommit(item as RestCommit) ?? []), more: data.length >= HISTORY_PAGE && page < MAX_HISTORY_PAGE }
+  const items = data as RestCommit[]
+  const trailers = items.map(item => coauthorTrailers(item?.commit?.message).map(({ name, email }): GraphActor & { name: string } => ({ name, email, user: null })))
+  if (token) await linkNoreplyUsers(trailers.flat(), token, fetchImpl)
+  return { commits: items.flatMap((item, at) => restCommit(item, token ? trailers[at]!.map(actor => ({ name: actor.name, email: actor.email, account: commitAccount(actor) })) : null) ?? []),
+    more: data.length >= HISTORY_PAGE && page < MAX_HISTORY_PAGE }
 }
 
 // The commits the repository's branches and tags point at, as GitHub's
@@ -104,41 +114,8 @@ export async function repositoryRefTips(fullName: string, token: string | null,
   return { branches: tips(branches), tags: tips(tags) }
 }
 
-export type GraphqlError = { type?: unknown; path?: unknown; message?: unknown }
-
-const NAMED_ERRORS = new Set(['RATE_LIMITED', 'NOT_FOUND', 'FORBIDDEN'])
-// GitHub's own words for a query it stopped past its time (ten seconds):
-// "Something went wrong while executing your query. This may be the result
-// of a timeout, …", with no type.
-const TIMED_OUT = /\btimeout\b/iu
-
-// One GraphQL query with `token`, answered with its `data`: one GitHub answers
-// with errors, but for those the caller reads past (`tolerated`, the fields
-// they name then null), is rate limited, names what isn't there, is refused,
-// as one a GitHub App lacks the permission for is, or a token the scope for
-// (`INSUFFICIENT_SCOPES`, a sign-in's having none), took GitHub too long, or
-// else is malformed, as one GitHub rejects unread is (a field or type its API
-// doesn't know). GitHub's words for errors of none of those first three types
-// are logged, those read past too.
-export async function githubGraphql(query: string, variables: Record<string, unknown>, token: string, fetchImpl: typeof fetch,
-  tolerated: (error: GraphqlError) => boolean = () => false) {
-  const body = await githubJson('https://api.github.com/graphql', token, fetchImpl, 'POST', { query, variables }, true) as { data?: unknown; errors?: GraphqlError[] } | null
-  const all = Array.isArray(body?.errors) ? body.errors : []
-  const unnamed = all.filter(error => !NAMED_ERRORS.has(String(error?.type)))
-  if (unnamed.length > 0) {
-    console.warn('managed: GitHub answered a query with errors:', JSON.stringify(unnamed.slice(0, 3)
-      .map(error => ({ type: error?.type, message: String(error?.message).slice(0, 300), path: error?.path }))))
-  }
-  const errors = all.filter(error => !tolerated(error))
-  if (errors.length > 0) {
-    if (errors.some(error => error?.type === 'RATE_LIMITED')) throw new GithubApiError(429, 'github-rate-limited', 60)
-    if (errors.some(error => error?.type === 'NOT_FOUND')) throw new GithubApiError(404, 'github-not-found')
-    if (errors.some(error => error?.type === 'FORBIDDEN' || error?.type === 'INSUFFICIENT_SCOPES')) throw new GithubApiError(502, 'github-status-403')
-    if (errors.some(error => typeof error?.message === 'string' && TIMED_OUT.test(error.message))) throw new GithubApiError(502, 'github-timeout')
-    throw new GithubApiError(502, 'github-malformed')
-  }
-  return body?.data
-}
+// A commit's author and those GitHub lists beside it.
+export const commitActors = (node: GraphCommit | null | undefined) => [node?.author, ...node?.authors?.nodes ?? []]
 
 // A tag's target, an annotated tag's own target in turn, through as many as
 // MAX_TAG_DEPTH annotated tags, each object's type naming what it is.
@@ -179,10 +156,10 @@ export interface HistoryPages {
 }
 
 // What a commit is read by in GraphQL.
-const COMMIT_FRAGMENT = `fragment historyCommit on Commit { oid message committedDate additions deletions author { name email date user { login databaseId } }
+export const COMMIT_FRAGMENT = `fragment historyCommit on Commit { oid message committedDate additions deletions author { name email date user { login databaseId } }
   authors(first: ${MAX_COAUTHORS + 1}) { nodes { name email user { login databaseId } } } parents(first: ${MAX_PARENTS}) { nodes { oid } } }`
 // A query's variables `$<prefix><n>` of `type`, as many as `count`.
-const variablesOf = (prefix: string, count: number, type: string) => Array.from({ length: count }, (_, at) => `$${prefix}${at}: ${type}`).join(', ')
+export const variablesOf = (prefix: string, count: number, type: string) => Array.from({ length: count }, (_, at) => `$${prefix}${at}: ${type}`).join(', ')
 
 // The latest commits of each tip, by its id, `b<n>` the `n`th's.
 const historiesQuery = (count: number) => `query($owner: String!, $name: String!, ${variablesOf('o', count, 'GitObjectID!')}) {
@@ -192,7 +169,7 @@ const historiesQuery = (count: number) => `query($owner: String!, $name: String!
 }
 ${COMMIT_FRAGMENT}`
 
-type GraphCommit = { oid?: unknown; message?: unknown; committedDate?: unknown; additions?: unknown; deletions?: unknown
+export type GraphCommit = { oid?: unknown; message?: unknown; committedDate?: unknown; additions?: unknown; deletions?: unknown
   author?: ({ name?: unknown; date?: unknown } & GraphActor) | null; authors?: { nodes?: (({ name?: unknown } & GraphActor) | null)[] } | null
   parents?: { nodes?: { oid?: unknown }[] } }
 type GraphTarget = { __typename?: unknown; oid?: unknown; committedDate?: unknown; target?: GraphTarget | null }
@@ -224,20 +201,37 @@ async function repositoryHeads(owner: string, name: string, token: string, fetch
   return { heads, repository: first, cut: true }
 }
 
-// The first page of the history from each of `tops`, by its top: with a
-// token, in one GraphQL query; without, a REST list each.
-async function readPages(fullName: string, token: string | null, tops: string[], fetchImpl: typeof fetch) {
+// Histories read in a GraphQL query, and queries at once: each counts its
+// commits' lines, which GitHub works out as it answers, stopping a query
+// past ten seconds.
+const PAGES_PER_QUERY = 5
+const PAGES_PARALLEL = 5
+
+// The first page of the history from each of `tops`, by its top, each told
+// to `keep` as it is read, so those read stay kept where another fails: with
+// a token, in GraphQL queries of PAGES_PER_QUERY (inBatches); without, a
+// REST list each.
+async function readPages(fullName: string, token: string | null, tops: string[], fetchImpl: typeof fetch,
+  keep: (top: string, commits: HistoryCommit[]) => Promise<void>) {
+  const read = new Map<string, HistoryCommit[]>()
+  const got = (top: string, commits: HistoryCommit[]) => { read.set(top, commits); return keep(top, commits) }
   if (!token) {
-    return new Map(await Promise.all(tops.map(async top => [top, (await repositoryHistory(fullName, null, top, '', 1, fetchImpl)).commits] as const)))
+    await Promise.all(tops.map(async top => got(top, (await repositoryHistory(fullName, null, top, '', 1, fetchImpl)).commits)))
+    return read
   }
   const [owner, name] = fullName.split('/') as [string, string]
-  const data = await githubGraphql(historiesQuery(tops.length), { owner, name, ...Object.fromEntries(tops.map((sha, at) => [`o${at}`, sha])) }, token, fetchImpl) as Histories
-  return new Map(tops.map((top, at) => [top, (data?.repository?.[`b${at}`]?.history?.nodes ?? []).flatMap(node => graphCommit(node) ?? [])]))
+  await inBatches(tops, PAGES_PER_QUERY, PAGES_PARALLEL, async batch => {
+    const data = await githubGraphql(historiesQuery(batch.length), { owner, name, ...Object.fromEntries(batch.map((sha, at) => [`o${at}`, sha])) }, token, fetchImpl) as Histories
+    const nodes = batch.map((_, at) => data?.repository?.[`b${at}`]?.history?.nodes ?? [])
+    await linkNoreplyUsers(nodes.flat().flatMap(commitActors), token, fetchImpl)
+    await Promise.all(batch.map((top, at) => got(top, nodes[at]!.flatMap(node => graphCommit(node) ?? []))))
+  })
+  return read
 }
 
 // Its authors past the first, the git author, GitHub lists from its
 // `Co-authored-by:` trailers.
-const graphCommit = (node: GraphCommit | null | undefined) => historyCommit(node?.oid, (node?.parents?.nodes ?? []).map(parent => parent?.oid),
+export const graphCommit = (node: GraphCommit | null | undefined) => historyCommit(node?.oid, (node?.parents?.nodes ?? []).map(parent => parent?.oid),
   node?.message, { name: node?.author?.name, email: node?.author?.email, account: commitAccount(node?.author) }, node?.committedDate,
   { additions: node?.additions, deletions: node?.deletions, authoredAt: node?.author?.date },
   Array.isArray(node?.authors?.nodes) ? node.authors.nodes.slice(1).map(actor => ({ name: actor?.name, email: actor?.email, account: commitAccount(actor) })) : null)
@@ -257,9 +251,9 @@ const graphCommit = (node: GraphCommit | null | undefined) => historyCommit(node
 export async function historyFrom(fullName: string, token: string | null, starts: string[], pages: HistoryPages, fetchImpl: typeof fetch = globalThis.fetch) {
   const kept = await pages.read(starts)
   const unread = starts.filter(top => !kept.has(top))
-  const read = unread.length > 0 ? await readPages(fullName, token, unread, fetchImpl) : new Map<string, HistoryCommit[]>()
   // Kept only as GitHub listed it from its top: none for a commit it named nothing of.
-  await Promise.all([...read].flatMap(([top, commits]) => commits[0]?.sha === top ? [pages.write(top, commits)] : []))
+  const keep = (top: string, commits: HistoryCommit[]) => commits[0]?.sha === top ? pages.write(top, commits) : Promise.resolve()
+  const read = unread.length > 0 ? await readPages(fullName, token, unread, fetchImpl, keep) : new Map<string, HistoryCommit[]>()
   const listed = new Map<string, HistoryCommit>()
   const add = (commits: HistoryCommit[]) => { for (const commit of commits) if (!listed.has(commit.sha)) listed.set(commit.sha, commit) }
   for (const top of starts) add(kept.get(top) ?? read.get(top) ?? [])
@@ -334,106 +328,45 @@ export async function repositoryAllHistory(fullName: string, token: string, { he
   return { commits: read.commits, branches, tags, defaultBranch, unread, cut, more }
 }
 
-// A pull request squashed into a commit of the default branch's history
-// (`merge`): its number, the commit its head was at when merged (`head`,
-// GitHub's own of it, which a branch reused or pushed over since never
-// moves), and its own commits, the latest MAX_SQUASHED of them, `more`
-// where it had more. Drawn as the merge it was, its commits on a lane of
-// their own (repository-history.js).
-export interface SquashedPull { merge: string; number: number; head: string; commits: HistoryCommit[]; more: boolean }
-// The pull requests squashed into commits, kept by each commit (a merged pull
-// request's commits never change); null for a commit none was.
-export interface Squashes {
-  read(shas: string[]): Promise<Map<string, SquashedPull | null>>
-  write(sha: string, pull: SquashedPull | null): Promise<void>
-}
-
-const MAX_SQUASHED = 100
-// Looked up of an answer's commits not yet kept, at most, the newest first;
-// in queries of as many commits, or pull requests, as many at once.
-export const MAX_SQUASH_LOOKUPS = 500
-const SQUASH_COMMITS = 100
-const SQUASH_PULLS = 20
-const SQUASH_PARALLEL = 4
-
-// The merged pull request that brought each commit in, `c<n>` the `n`th's.
-const mergedPullsQuery = (count: number) => `query($owner: String!, $name: String!, ${variablesOf('o', count, 'GitObjectID!')}) {
-  repository(owner: $owner, name: $name) {
-    ${Array.from({ length: count }, (_, at) => `c${at}: object(oid: $o${at}) { ... on Commit { associatedPullRequests(first: 1) { nodes { number mergeCommit { oid } } } } }`).join('\n    ')}
+const chunked = <T>(items: T[], size: number) => Array.from({ length: Math.ceil(items.length / size) }, (_, at) => items.slice(at * size, (at + 1) * size))
+// GitHub stopping a query past its time: in its answer's errors, or as a 502
+// or 504 of its own.
+const SLOW = new Set(['github-timeout', 'github-status-502', 'github-status-504'])
+// `run` for `items` in batches of `size`, as many at once as `parallel`; a
+// batch GitHub stops past its time run again in halves, down to one alone,
+// so that one too slow to read fails alone, the others read. The halves
+// wait their turn with the rest, and as many batches are halved as run at
+// once twice over at most, GitHub slow throughout past that. Where one
+// fails, none more is run, those running done before its error is thrown,
+// so that what they read is kept.
+export async function inBatches<T>(items: T[], size: number, parallel: number, run: (batch: T[]) => Promise<void>) {
+  const failed: unknown[] = []
+  const waiting: (() => void)[] = []
+  let halved = 0, running = 0
+  const turn = () => {
+    if (running < parallel) { running++; return Promise.resolve() }
+    return new Promise<void>(resolve => { waiting.push(resolve) })
   }
-}`
-// Each pull request's head when merged and its latest commits, `p<n>` the `n`th's.
-const squashedQuery = (count: number) => `query($owner: String!, $name: String!, ${variablesOf('n', count, 'Int!')}) {
-  repository(owner: $owner, name: $name) {
-    ${Array.from({ length: count }, (_, at) => `p${at}: pullRequest(number: $n${at}) { headRefOid commits(last: ${MAX_SQUASHED}) { totalCount nodes { commit { ...historyCommit } } } }`).join('\n    ')}
+  // A turn done, handed to the batch waiting longest.
+  const done = () => {
+    const next = waiting.shift()
+    if (next) next()
+    else running--
   }
-}
-${COMMIT_FRAGMENT}`
-
-type MergedPulls = { repository?: Record<string, { associatedPullRequests?: { nodes?: ({ number?: unknown; mergeCommit?: { oid?: unknown } | null } | null)[] } } | null> | null }
-type SquashedPage = { repository?: Record<string, { headRefOid?: unknown; commits?: { totalCount?: unknown; nodes?: ({ commit?: GraphCommit | null } | null)[] } } | null> | null }
-
-const chunked = <T>(items: T[], size: number) => Array.from({ length: Math.ceil(items.length / size) }, (_, at) => items.slice(at * size, (at + 1) * size))
-// `run` for each of `items`, as many at once as `parallel`.
-async function eachOf<T>(items: T[], parallel: number, run: (item: T) => Promise<void>) {
-  let next = 0
-  await Promise.all(Array.from({ length: Math.min(parallel, items.length) }, async () => { while (next < items.length) await run(items[next++]!) }))
-}
-// A pull request or commit GitHub has none of, as one deleted: its field null.
-const notFound = (error: GraphqlError) => error?.type === 'NOT_FOUND'
-
-// The pull requests squashed into `commits`, a merged pull request's merge
-// commit of one parent, not one rebased (its commits copied onto the branch,
-// told by their subjects), kept by each commit (`squashes`), those not yet
-// kept looked up with GraphQL, as many as MAX_SQUASH_LOOKUPS an answer.
-export async function squashedPulls(fullName: string, token: string, commits: HistoryCommit[], squashes: Squashes,
-  fetchImpl: typeof fetch = globalThis.fetch): Promise<SquashedPull[]> {
-  const [owner, name] = fullName.split('/') as [string, string]
-  const single = commits.filter(commit => commit.parents.length === 1)
-  const listed = new Map(commits.map(commit => [commit.sha, commit]))
-  const kept = await squashes.read(single.map(commit => commit.sha))
-  const unknown = single.filter(commit => !kept.has(commit.sha)).slice(0, MAX_SQUASH_LOOKUPS)
-  const merged: { commit: HistoryCommit; number: number }[] = []
-  const found = new Map<string, SquashedPull | null>()
-  await eachOf(chunked(unknown, SQUASH_COMMITS), SQUASH_PARALLEL, async batch => {
-    const data = await githubGraphql(mergedPullsQuery(batch.length), { owner, name, ...Object.fromEntries(batch.map((commit, at) => [`o${at}`, commit.sha])) },
-      token, fetchImpl, notFound) as MergedPulls
-    for (const [at, commit] of batch.entries()) {
-      const node = data?.repository?.[`c${at}`]
-      if (!node) continue
-      const pull = node.associatedPullRequests?.nodes?.[0]
-      const number = pull?.mergeCommit?.oid === commit.sha && Number.isSafeInteger(pull.number) && (pull.number as number) > 0 ? pull.number as number : null
-      if (number === null) { found.set(commit.sha, null); await squashes.write(commit.sha, null) }
-      else merged.push({ commit, number })
-    }
-  })
-  await eachOf(chunked(merged, SQUASH_PULLS), SQUASH_PARALLEL, async batch => {
-    const data = await githubGraphql(squashedQuery(batch.length), { owner, name, ...Object.fromEntries(batch.map(({ number }, at) => [`n${at}`, number])) },
-      token, fetchImpl, notFound) as SquashedPage
-    for (const [at, { commit, number }] of batch.entries()) {
-      const node = data?.repository?.[`p${at}`]
-      if (!node) continue
-      const own = (node.commits?.nodes ?? []).flatMap(item => graphCommit(item?.commit) ?? [])
-      const head = typeof node.headRefOid === 'string' && SHA.test(node.headRefOid) ? node.headRefOid : own.at(-1)?.sha
-      // Fast-forwarded, its last commit the merge's own; or rebased, its
-      // commits, more than one, copied onto the branch: the merge and those
-      // below it on their first parents, by their subjects, its commits'
-      // (one squashed is told by its commit's title too, by default, and
-      // drawn as it was either way).
-      const below = [commit]
-      for (let next = listed.get(commit.parents[0] ?? ''); next && below.length < own.length; next = listed.get(next.parents[0] ?? '')) below.push(next)
-      const copies = own.toReversed().slice(0, below.length).every((item, step) => below[step]!.subject === item.subject)
-      // Its commits' copies as far as those listed go, but not all of them
-      // listed: told, and kept, once they are.
-      if (own.length > 1 && copies && below.length < own.length) continue
-      const rebased = own.length > 1 && copies
-      const squashed = head !== undefined && head !== commit.sha && own.length > 0 && !own.some(item => item.sha === commit.sha) && !rebased
-      const pull = squashed ? { merge: commit.sha, number, head, commits: own, more: Number(node.commits?.totalCount) > own.length } : null
-      found.set(commit.sha, pull)
-      await squashes.write(commit.sha, pull)
-    }
-  })
-  return single.flatMap(commit => kept.get(commit.sha) ?? found.get(commit.sha) ?? [])
+  const attempt = async (batch: T[]): Promise<void> => {
+    await turn()
+    try {
+      if (failed.length === 0) await run(batch)
+      return
+    } catch (err) {
+      if (batch.length === 1 || halved >= 2 * parallel || !(err instanceof GithubApiError && SLOW.has(err.message))) { failed.push(err); return }
+      halved++
+    } finally { done() }
+    const half = Math.ceil(batch.length / 2)
+    await Promise.all([attempt(batch.slice(0, half)), attempt(batch.slice(half))])
+  }
+  await Promise.all(chunked(items, size).map(attempt))
+  if (failed.length > 0) throw failed[0]
 }
 
 // The open pull requests most recently updated, each by the commit its head