@preventive/triage 1.0.0-alpha.44 → 1.0.0-alpha.46

package/server-managed/pretty-print.ts

@@ -12,16 +12,17 @@ import { promisify } from 'node:util'
 import { brotliDecompress } from 'node:zlib'
 import { format } from '@preventive/compare/pretty.js'
 import { computeFileHash } from '@preventive/report'
-import { FILE_HASH } from '../common/bundle-metadata.js'
+import { FILE_HASH, computeBundleFileHashes } from '../common/bundle-metadata.js'
 import { bundleCompareFiles } from '../common/bundle-compare-inputs.js'
 import { bundlePackageDirs, bundleSourcesAsMap } from '../common/bundle-sources.js'
 import { MAX_PRETTY_BYTES, needsPrettyPrint, prettyExtension } from '../common/pretty-print.js'
 import type { OpenedBlob } from './blob-store.ts'
-import { type BundleCacheStorage, readBundleDetails } from './bundle-cache.ts'
+import { type BundleCacheStorage, readSizedBundleDetails } from './bundle-cache.ts'
 import type { BundleStore } from './bundle-store.ts'
 import { encodeBrotli } from './brotli.ts'
 import { CacheMissError, type CacheStorage } from './cache-storage.ts'
 import type { ManagedBundle, ManagedDb } from './db.ts'
+import { keptBriefly } from './kept-briefly.ts'
 import { loadNpmFiles } from './npm-loads.ts'
 import { type NpmVersionDocument, npmFileText } from './npm-packages.ts'
 
@@ -34,6 +35,16 @@ const PRETTY_VERSION = 4
 // after another, beside other readers'), any more refused.
 const MAX_ACTIVE_FORMATS = 2
 const MAX_WAITING_FORMATS = 16
+// How long a bundle's details are kept once read, and what the two last read
+// may have decoded from between them, the least lately read going first past
+// it: Compare's requests for a pair of bundles come one after another, each
+// reading both, and the Code tab's for one, each reading it.
+const DETAILS_KEPT_MS = 60_000
+const MAX_KEPT_DETAILS = 2
+const MAX_KEPT_DETAILS_BYTES = 256 * 1024 * 1024
+// Files hashed at once: one at a time, each waits its own round trip to the
+// thread pool; all at once, every one's job is held together.
+const HASHES_AT_ONCE = 32
 
 export class PrettyError extends Error {
   status: number
@@ -108,6 +119,18 @@ export async function prettyBody(text: string, extension: string): Promise<Buffe
 
 const opened = (body: Buffer): OpenedBlob => ({ size: body.byteLength, stream: Readable.from([body]) })
 
+// A version's file's hash, by its bytes: those of a version's files loaded
+// a little while since are the same bytes (npm-loads.ts), hashed once.
+const fileHashes = new WeakMap<Uint8Array, Promise<string>>()
+function npmFileHash(bytes: Uint8Array): Promise<string> {
+  let hash = fileHashes.get(bytes)
+  if (!hash) {
+    fileHashes.set(bytes, hash = computeFileHash(bytes))
+    hash.catch(() => { fileHashes.delete(bytes) })
+  }
+  return hash
+}
+
 const decompress = promisify(brotliDecompress)
 
 // A pretty-printed copy's text, from its Brotli-encoded bytes.
@@ -127,7 +150,7 @@ async function npmPrintable(doc: NpmVersionDocument): Promise<Array<PrettyReques
       const extension = prettyExtension(path)
       if (extension === null || bytes.byteLength > MAX_PRETTY_BYTES) continue
       const text = npmFileText(bytes)
-      if (text !== null && needsPrettyPrint(path, text)) files.push({ path, hash: await computeFileHash(bytes), extension, text })
+      if (text !== null && needsPrettyPrint(path, text)) files.push({ path, hash: await npmFileHash(bytes), extension, text })
     }
   } finally { load.release() }
   return files.toSorted((a, b) => (a.path < b.path ? -1 : 1))
@@ -206,26 +229,35 @@ async function kept(open: () => Promise<OpenedBlob>): Promise<OpenedBlob | null>
 
 // A side's files that can be formatted, with their texts, and every one of
 // its files' hashes by path, from their paths and contents: texts, or bytes,
-// read as text where they are.
-async function sideFiles(entries: Iterable<[string, string | Uint8Array]>): Promise<Pick<ComparedSide, 'files' | 'hashes'>> {
+// read as text where they are; hashed by `hashOf`, a few at once.
+async function sideFiles<T extends string | Uint8Array>(entries: Array<[string, T]>, hashOf: (file: string, content: T) => Promise<string>): Promise<Pick<ComparedSide, 'files' | 'hashes'>> {
   const files: ComparedSide['files'] = [], hashes = new Map<string, string>()
+  for (let i = 0; i < entries.length; i += HASHES_AT_ONCE) {
+    const chunk = entries.slice(i, i + HASHES_AT_ONCE)
+    const hashed = await Promise.all(chunk.map(async ([path, content]) => [path, await hashOf(path, content)] as const))
+    for (const [path, hash] of hashed) hashes.set(path, hash)
+  }
   for (const [path, content] of entries) {
-    const hash = await computeFileHash(content)
-    hashes.set(path, hash)
     const extension = prettyExtension(path)
     if (extension === null || (typeof content === 'string' ? Buffer.byteLength(content) : content.byteLength) > MAX_PRETTY_BYTES) continue
     const text = typeof content === 'string' ? content : npmFileText(content)
-    if (text !== null) files.push({ path, hash, extension, text })
+    if (text !== null) files.push({ path, hash: hashes.get(path)!, extension, text })
   }
   return { files, hashes }
 }
 
 // A bundle's side, from its details: its files, as sideFiles reads them,
 // within a reason `scope` as Compare reads them there (bundleCompareFiles),
-// and its packages' directories.
+// and its packages' directories; hashed once for its details, as long as
+// they are kept (computeBundleFileHashes).
 async function bundleSide(details: Parameters<typeof bundlePackageDirs>[0], scope: string): Promise<Pick<ComparedSide, 'files' | 'hashes' | 'packageDirs'>> {
   const scoped = scope ? new Set(bundleCompareFiles(details, scope).keys()) : null, sources = bundleSourcesAsMap(details)
-  return { ...await sideFiles(scoped ? [...sources].filter(([path]) => scoped.has(path)) : sources), packageDirs: bundlePackageDirs(details) }
+  const hashed: Map<string, string> = await computeBundleFileHashes(details)
+  return { ...await sideFiles(scoped ? [...sources].filter(([path]) => scoped.has(path)) : [...sources], (path, content) => {
+    const hash = hashed.get(path)
+    return hash === undefined ? computeFileHash(content) : Promise.resolve(hash)
+  }),
+    packageDirs: bundlePackageDirs(details) }
 }
 
 // A version's files by path, from one load of them.
@@ -251,14 +283,25 @@ export function stalePrettyRemover(npm: CacheStorage): (signal?: AbortSignal) =>
 }
 
 // A bundle's details, from its bytes in `store`: bundles are read one at a
-// time, as reading one decodes all of it.
+// time, as reading one decodes all of it, readers asking for one waiting its
+// turn or being read sharing it; and kept a little while once read
+// (DETAILS_KEPT_MS), each reader's access to the bundle the caller's to check
+// as ever.
 function detailsReader(store: BundleStore) {
-  const enqueue = serial()
-  return (record: ManagedBundle) => enqueue(async () => {
-    const details = await readBundleDetails(record, store)
-    if (!details) throw new PrettyError(422, 'bundle-unavailable')
-    return details
-  })
+  type Details = NonNullable<Awaited<ReturnType<typeof readSizedBundleDetails>>>['details']
+  const enqueue = serial(), share = sharing<Details>()
+  const held = keptBriefly<Details>({ keptMs: DETAILS_KEPT_MS, maxBytes: MAX_KEPT_DETAILS_BYTES, maxEntries: MAX_KEPT_DETAILS })
+  return (record: ManagedBundle): Promise<Details> => {
+    const key = `${record.id} ${record.integrity}`
+    const hit = held.get(key)
+    if (hit) return Promise.resolve(hit)
+    return share(key, () => enqueue(async () => {
+      const read = await readSizedBundleDetails(record, store)
+      if (!read) throw new PrettyError(422, 'bundle-unavailable')
+      held.keep(key, read.details, read.decodedBytes)
+      return read.details
+    }))
+  }
 }
 
 export function createPrettyCache(bundles: BundleCacheStorage, npm: CacheStorage, db: Pick<ManagedDb, 'getBundle'>, store: BundleStore) {
@@ -317,7 +360,7 @@ export function createPrettyCache(bundles: BundleCacheStorage, npm: CacheStorage
         try {
           const bytes = (await load.result).find(entry => entry.path === request.path)?.bytes
           if (!bytes) throw new PrettyError(404, 'no-file')
-          if (await computeFileHash(bytes) !== request.hash) throw new PrettyError(409, 'hash-mismatch')
+          if (await npmFileHash(bytes) !== request.hash) throw new PrettyError(409, 'hash-mismatch')
           text = npmFileText(bytes)
         } finally { load.release() }
         if (text === null) throw new PrettyError(422, 'unformattable')
@@ -338,19 +381,20 @@ export function createPrettyCache(bundles: BundleCacheStorage, npm: CacheStorage
     // pretty-printed (compared), and their diffs with renamed names hidden
     // (rename-diff.ts renamedPairs); a bundle's within a reason `scope`
     // where one is asked for, as Compare reads it then.
-    async bundleSides(base: ManagedBundle, other: ManagedBundle, scope = ''): Promise<[ComparedSide, ComparedSide]> {
+    bundleSides(base: ManagedBundle, other: ManagedBundle, scope = ''): Promise<[ComparedSide, ComparedSide]> {
       const sideOf = async (record: ManagedBundle, side: Side): Promise<ComparedSide> => ({ side, ...await bundleSide(await bundleDetails(record), scope),
         kept: request => kept(() => bundles.open(record.id, prettyFile(request))),
         made: async (request, text) => opened(await bundleCopy(record, request, () => Promise.resolve(text))) })
-      return [await sideOf(base, 'base'), await sideOf(other, 'other')]
+      return Promise.all([sideOf(base, 'base'), sideOf(other, 'other')])
     },
     // A bundle's details, read in its turn, as Compare pairs them (comparison.ts).
     bundleDetails,
-    async npmSides(base: NpmVersionDocument, other: NpmVersionDocument): Promise<[ComparedSide, ComparedSide]> {
-      const sideOf = async (doc: NpmVersionDocument, side: Side): Promise<ComparedSide> => ({ side, ...await sideFiles(await npmFiles(doc)), packageDirs: null,
+    npmSides(base: NpmVersionDocument, other: NpmVersionDocument): Promise<[ComparedSide, ComparedSide]> {
+      const sideOf = async (doc: NpmVersionDocument, side: Side): Promise<ComparedSide> => ({ side, ...await sideFiles(await npmFiles(doc), (_, bytes) => npmFileHash(bytes)),
+        packageDirs: null,
         kept: request => (doc.private ? Promise.resolve(null) : kept(() => npm.open(prettyFile(request)))),
         made: (request, text) => npmCopy(doc, request, () => Promise.resolve(text)) })
-      return [await sideOf(base, 'base'), await sideOf(other, 'other')]
+      return Promise.all([sideOf(base, 'base'), sideOf(other, 'other')])
     },
   }
 }

package/server-managed/pull-comment-reviews.ts

@@ -0,0 +1,161 @@
+// The reviews of dependencies an open pull request's comments make, as the
+// host's parser reads them (reviewed-dependencies.ts `parseComments`): its
+// comments, its reviews' own, whatever their state, and those on lines of
+// its diff, read of GitHub's GraphQL API, as many as MAX_COMMENTS, with
+// whether it is open, kept in this instance KEPT_MS; none of one not open;
+// the parser's answer to the same kept PARSED_KEPT_MS.
+// A review counts only where its user is one of their authors and the app's
+// user that account is (by its numeric id, else its login, its linked
+// accounts' too) publishes reviews to a team counting for the repository
+// (`reviewers`): by that user, as their reviews here are, each once among
+// them (pull-dependency-reviews.ts), as of its time, else their latest
+// comment's. They count for the pull request alone, never stored.
+import { createHash } from 'node:crypto'
+import { ACTOR, type GraphAccount, actorOf } from './github-accounts.ts'
+import { type Lru, lru } from './lru.ts'
+import type { CommentReviews } from './pull-dependency-reviews.ts'
+import type { PublishedReview } from './repository-dependency-reviews.ts'
+import { githubGraphql } from './repository-history.ts'
+import type { CommentReview, ParseCommentsInput, ReviewComment, ReviewVerdict, ReviewedDependencies } from './reviewed-dependencies.ts'
+
+const MAX_COMMENTS = 1000
+const KEPT_MS = 60_000
+const PARSED_KEPT_MS = 10 * 60_000
+const VERDICTS = new Set<string>(['accept', 'accept-with-nits', 'request-changes', 'reject'])
+
+// A comment as read: when it was written, and last edited, where it was.
+const COMMENT = `id author { ${ACTOR} } body createdAt lastEditedAt url`
+// Its comments and its reviews, each a page on from its cursor, each only
+// while it has more (`readComments`, `readReviews`).
+const QUERY = `query($owner: String!, $name: String!, $number: Int!, $comments: String, $reviews: String, $readComments: Boolean!, $readReviews: Boolean!) {
+  repository(owner: $owner, name: $name) { pullRequest(number: $number) { state
+    comments(first: 100, after: $comments) @include(if: $readComments) { pageInfo { hasNextPage endCursor } nodes { ${COMMENT} } }
+    reviews(first: 50, after: $reviews) @include(if: $readReviews) { pageInfo { hasNextPage endCursor }
+      nodes { ${COMMENT} state comments(first: 100) { pageInfo { hasNextPage endCursor } nodes { ${COMMENT} } } } }
+  } } }`
+// A review's comments on lines of its diff, a page on from its cursor.
+const REVIEW_COMMENTS = `query($id: ID!, $after: String) { node(id: $id) { ... on PullRequestReview {
+  comments(first: 100, after: $after) { pageInfo { hasNextPage endCursor } nodes { ${COMMENT} } } } } }`
+
+// A comment as read, and its author's account.
+interface Authored { comment: ReviewComment; account: { login: string | null; id: number | null } }
+type Page = { pageInfo?: { hasNextPage?: unknown; endCursor?: unknown }; nodes?: unknown }
+type Node = { id?: unknown; author?: GraphAccount; body?: unknown; state?: unknown; createdAt?: unknown; lastEditedAt?: unknown; url?: unknown; comments?: Page }
+
+// A pull request's comments as read, and whether it is open.
+interface PullComments { open: boolean; read: Authored[] }
+const comments: Lru<{ at: number } & PullComments> = lru(200)
+const answers: Lru<{ at: number; reviews: CommentReview[] }> = lru(2000)
+
+function authored(node: Node, kind: ReviewComment['kind']): Authored[] {
+  if (typeof node?.id !== 'string' || typeof node.body !== 'string' || typeof node.createdAt !== 'string') return []
+  const account = actorOf(node.author)
+  return [{ account, comment: { kind, id: node.id, author: account.login, body: node.body, ...typeof node.state === 'string' ? { state: node.state } : {},
+    createdAt: node.createdAt, ...typeof node.lastEditedAt === 'string' ? { editedAt: node.lastEditedAt } : {}, url: typeof node.url === 'string' ? node.url : '' } }]
+}
+const nodes = (list: unknown): Node[] => Array.isArray(list) ? list as Node[] : []
+// The cursor a page ends at, where more follow it.
+const next = (page: Page | undefined) => page?.pageInfo?.hasNextPage === true && typeof page.pageInfo.endCursor === 'string' ? page.pageInfo.endCursor : null
+
+// Review `review`'s comments on lines of its diff: the page read with it,
+// then those after it read of GitHub, till as many as `room` are.
+async function inlineComments(review: Node, room: number, token: string, fetchImpl: typeof fetch): Promise<Authored[]> {
+  const read = nodes(review.comments?.nodes).flatMap(inline => authored(inline, 'review-comment'))
+  for (let after = typeof review.id === 'string' ? next(review.comments) : null; after !== null && read.length < room;) {
+    const data = await githubGraphql(REVIEW_COMMENTS, { id: review.id, after }, token, fetchImpl) as { node?: { comments?: Page } | null } | null
+    read.push(...nodes(data?.node?.comments?.nodes).flatMap(inline => authored(inline, 'review-comment')))
+    after = next(data?.node?.comments)
+  }
+  return read
+}
+
+// Pull request `number`'s comments, its reviews' and those on its diff, each
+// review's all of them, and whether it is open, as kept, else read of GitHub
+// a page at a time.
+export async function pullComments(fullName: string, token: string, number: number, fetchImpl: typeof fetch = globalThis.fetch): Promise<PullComments> {
+  const key = `${fullName}\0${number}`
+  const known = comments.get(key)
+  if (known && Date.now() - known.at < KEPT_MS) return known
+  const [owner, name] = fullName.split('/') as [string, string]
+  const read: Authored[] = []
+  let open: boolean | null = null
+  // Each connection read on from its cursor till it has no more, the other
+  // read on without it, never again from its first page.
+  let cursors: { comments: string | null; reviews: string | null } = { comments: null, reviews: null }
+  let reading = { comments: true, reviews: true }
+  while ((reading.comments || reading.reviews) && read.length < MAX_COMMENTS) {
+    const data = await githubGraphql(QUERY, { owner, name, number, ...cursors, readComments: reading.comments, readReviews: reading.reviews }, token, fetchImpl) as
+      { repository?: { pullRequest?: { state?: unknown; comments?: Page; reviews?: Page } | null } | null } | null
+    const pull = data?.repository?.pullRequest
+    if (!pull) break
+    open ??= pull.state === 'OPEN'
+    for (const node of nodes(pull.comments?.nodes)) read.push(...authored(node, 'comment'))
+    for (const node of nodes(pull.reviews?.nodes)) {
+      read.push(...authored(node, 'review'))
+      read.push(...await inlineComments(node, MAX_COMMENTS - read.length, token, fetchImpl))
+    }
+    cursors = { comments: reading.comments ? next(pull.comments) : null, reviews: reading.reviews ? next(pull.reviews) : null }
+    reading = { comments: cursors.comments !== null, reviews: cursors.reviews !== null }
+  }
+  const kept = { open: open === true, read: read.slice(0, MAX_COMMENTS) }
+  comments.set(key, { at: Date.now(), ...kept })
+  return kept
+}
+
+// The parser's reviews of `input`, as kept for the same, else asked.
+async function parsed(parse: NonNullable<ReviewedDependencies['parseComments']>, input: ParseCommentsInput): Promise<CommentReview[]> {
+  const key = createHash('sha256').update(JSON.stringify(input)).digest('base64url')
+  const known = answers.get(key)
+  if (known && Date.now() - known.at < PARSED_KEPT_MS) return known.reviews
+  const reviews = await parse(structuredClone(input))
+  if (!Array.isArray(reviews)) throw new TypeError('parseComments answered with no list')
+  answers.set(key, { at: Date.now(), reviews })
+  return reviews
+}
+
+const when = (time: unknown) => typeof time === 'number' ? time : typeof time === 'string' ? Date.parse(time) : Number.NaN
+// The reviews `answer` makes that count, by the users `reviewers` names,
+// each dependency's of each hash its integrity names, as of the time it
+// tells, else its user's latest comment's, as last written or edited.
+function counted(answer: CommentReview[], read: Authored[], reviewers: Map<string, string>): PublishedReview[] {
+  const latest = new Map<string, number>()
+  for (const { comment } of read) {
+    const login = comment.author?.toLowerCase()
+    if (login) latest.set(login, Math.max(latest.get(login) ?? 0, when(comment.createdAt) || 0, when(comment.editedAt) || 0))
+  }
+  return answer.flatMap(review => {
+    const login = typeof review?.user === 'string' ? review.user.toLowerCase() : ''
+    const reviewer = reviewers.get(login)
+    if (!reviewer || !latest.has(login) || !Array.isArray(review.dependencies)) return []
+    const time = Number.isFinite(when(review.time)) ? when(review.time) : latest.get(login)!
+    return review.dependencies.flatMap(dependency => {
+      const { name, version, integrity, patchHash, status } = dependency ?? {}
+      if (typeof name !== 'string' || typeof version !== 'string' || typeof integrity !== 'string' || !VERDICTS.has(status)) return []
+      return integrity.split(/\s+/u).filter(Boolean).map(hash => ({ package: name, version, integrity: hash, ...typeof patchHash === 'string' && patchHash ? { patchHash } : {},
+        basePackage: null, baseVersion: null, reviewer, status: status as ReviewVerdict, full: true, time }))
+    })
+  })
+}
+
+// The reviews pull request `number`'s comments make, as the host's parser
+// reads them, of the versions still wanting one, none where it isn't open
+// (the parser then not asked); `reviewers` the app's users
+// the accounts are, by their logins in lower case, of those whose reviews
+// count for the repository.
+export function commentReviews({ fullName, number, token, parse, reviewers, fetchImpl = globalThis.fetch }: {
+  fullName: string; number: number; token: string; parse: NonNullable<ReviewedDependencies['parseComments']>
+  reviewers: (accounts: { login: string; githubId: number | null }[]) => Promise<Map<string, string>>; fetchImpl?: typeof fetch
+}): CommentReviews {
+  return async wanting => {
+    try {
+      const { open, read } = await pullComments(fullName, token, number, fetchImpl)
+      if (!open) return { reviews: [], failed: false }
+      const accounts = new Map(read.flatMap(({ account }) => account.login ? [[account.login.toLowerCase(), { login: account.login, githubId: account.id }]] : []))
+      const answer = await parsed(parse, { repository: fullName, number, comments: read.map(({ comment }) => comment), dependencies: wanting })
+      return { reviews: counted(answer, read, await reviewers([...accounts.values()])), failed: false }
+    } catch (error) {
+      console.warn('reviewed dependencies: comments of', `${fullName}#${number}`, 'not read:', error instanceof Error ? error.message : error)
+      return { reviews: [], failed: true }
+    }
+  }
+}

package/server-managed/pull-dependencies.ts

@@ -0,0 +1,300 @@
+// The package versions a pull request's lockfiles lock that they didn't
+// before it, as the code viewer's Pull requests tab shows them on an open
+// one's tile and its popup on any (GET
+// /api/admin/repositories/pull-dependencies): read only where it changes a
+// lockfile of npm's ecosystem that @preventive/lockfile reads
+// (package-lock.json and npm-shrinkwrap.json of `lockfileVersion: 3`,
+// pnpm-lock.yaml of pnpm 9 to 12 and Yarn 1's yarn.lock, locked-packages.ts),
+// and Bun's, which it doesn't (bun.lock and bun.lockb, its directory told
+// `lockfile-unsupported`); none under a node_modules nor a test's fixtures
+// (isTestFixture), which no install takes. Its files are GitHub's comparison of its base with its head
+// (GET /repos/:owner/:repo/compare/:base...:head), which names where its
+// branch left its base, its merge base, as GitHub's own Files changed
+// compares from it, and lists as many as a commit's files are read
+// (repository-changes.ts MAX_FILES, 300, the most it lists); one changing as
+// many, from its own list of them, a page at a time, as many as PULL_FILES.
+// Each directory's lockfiles it changes are read at its merge base and at its
+// head, a lockfile replaced by another, Yarn's by pnpm's say, read as the
+// one it replaced; a version new to a directory is one its lockfiles lock at
+// the head that none of them locked at the merge base, from where it is
+// installed: a package new to it, or a version of one they locked others of,
+// a bump's, told with those (`was`). What a pair of commits holds never
+// changes, so what is told of one is kept by them (`kept`), and each
+// lockfile read is kept in this instance's memory by its commit
+// (`lockfileReads`): the pull requests branching off the same commit, as a
+// bot's do, read their base's once.
+import { createClient } from '@preventive/upstream/github.js'
+import { GithubApiError, githubJson } from './github-app.ts'
+import { upstreamGithub } from './github-errors.ts'
+import { compareVersions, isExactVersion } from '@preventive/upstream/semver.js'
+import { type LockedPackage, isTestFixture, lockedPackages } from './locked-packages.ts'
+import { type Lru, lru } from './lru.ts'
+import { type ChangedFile, commitChanges } from './repository-changes.ts'
+
+// The files of a pull request read from its own list past those compared,
+// FILES_PAGE a page, as many as GitHub lists (3,000).
+const FILES_PAGE = 100
+export const PULL_FILES = 3000
+// The directories whose lockfiles are read, those past them left out
+// (`cut`), and the packages told new of each, those past them counted.
+export const MAX_DIRECTORIES = 20
+export const MAX_ADDED = 200
+// The versions a directory's lockfiles lock at the head kept with what they
+// add, as many as MAX_LOCKED, those past them counted, for those not new to
+// be reviewed as well (pull-dependency-reviews.ts).
+export const MAX_LOCKED = 20_000
+// Lockfiles read at once.
+const READ_PARALLEL = 4
+// The rules what is told was told by, kept with it (http.ts
+// pullDependencyKept): one told by others is told again.
+export const PULL_DEPENDENCIES_READ = 3
+const SHA = /^[a-f\d]{40}$/u
+
+type Format = 'npm' | 'pnpm' | 'yarn' | 'bun'
+
+// The format of the lockfile of npm's ecosystem at `path`, as
+// findRepoLockfiles names it (with pnpm's of a branch,
+// pnpm-lock.<branch>.yaml, and of before pnpm 3, shrinkwrap.yaml, which its
+// reader refuses); null for another file, and for one under a node_modules or
+// of a test's fixtures. Bun's alone isn't read.
+export function lockfileFormat(path: string): Format | null {
+  const dirs = path.split('/')
+  const name = dirs.pop() ?? ''
+  if (dirs.includes('node_modules') || isTestFixture(path)) return null
+  if (name === 'package-lock.json' || name === 'npm-shrinkwrap.json') return 'npm'
+  if (/^pnpm-lock(?:\..+)?\.yaml$/u.test(name) || name === 'shrinkwrap.yaml') return 'pnpm'
+  if (name === 'bun.lock' || name === 'bun.lockb') return 'bun'
+  return name === 'yarn.lock' ? 'yarn' : null
+}
+
+// Whether a pull request changing `paths` changes a lockfile read here: null
+// where none of them is one but they are fewer than it changes
+// (`changed`), which it may yet.
+export function touchesLockfile(paths: string[], changed: number): boolean | null {
+  if (paths.some(path => lockfileFormat(path) !== null)) return true
+  return changed > paths.length ? null : false
+}
+
+// A package version new to a directory, as its lockfile locks it, with its
+// integrity, which its reviews are held to; of a package its lockfiles locked
+// before, the versions of it they did (`was`, the highest first), none of one
+// new to it.
+export type AddedPackage = LockedPackage & { was?: string[] }
+// A directory's lockfiles the pull request changes (`paths`, those it adds,
+// modifies, removes or renames, by either name): the package versions they
+// lock new to it, by name and version (as many as MAX_LOCKED, told as many
+// as MAX_ADDED, `more` past them), and all they lock (`locked`, as many as MAX_LOCKED, how many more
+// `moreLocked`); or why they couldn't be told, a lockfile's reader refusing one
+// (`lockfile-unreadable`, with what it said), one of a format not read
+// (`lockfile-unsupported`, Bun's) or GitHub having none at a commit compared.
+export type DirectoryDependencies = { paths: string[] } & ({ added: AddedPackage[]; more: number; locked: LockedPackage[]; moreLocked: number }
+  | { error: string; detail?: string })
+// What a pull request's lockfiles add: the commits compared (`base`, its
+// merge base, and `head`); each directory's, of those it changes a lockfile
+// in; and `cut` where it changes more files than GitHub lists, or lockfiles
+// in more directories than are read, with, where only those cut it and they
+// are few enough, the paths of those directories' lockfiles (`omitted`), for
+// a review holding to none of them to fail in nothing.
+export interface PullDependencies { base: string; head: string; lockfiles: DirectoryDependencies[]; cut: boolean; omitted?: string[] }
+const MAX_OMITTED = 1000
+
+// What is told of a pull request, by the commits it is of, `${base}...${head}`.
+export interface PullDependencyStore {
+  read(key: string): Promise<PullDependencies | null>
+  write(key: string, value: PullDependencies): Promise<void>
+}
+
+// A lockfile's packages, or why its reader refused it, by its repository,
+// commit and path, read in this instance; by the packages each holds, as the
+// scan's are (repository-risk.ts riskCaches).
+type LockfileRead = { packages: LockedPackage[] } | { error: string; detail?: string }
+const lockfileReads: Lru<LockfileRead> = lru(200_000, read => 'packages' in read ? read.packages.length + 1 : 1)
+// Those being read, by the token reading them too, lest each asking at once read them.
+const lockfilesReading = new Map<string, Promise<LockfileRead>>()
+
+const api = (fullName: string) => `https://api.github.com/repos/${fullName.split('/').map(encodeURIComponent).join('/')}`
+
+// The pull request's base's and head's commits, as GitHub has them now: its
+// base's moving on as it does, its head's with each push, both kept once it
+// is merged or closed.
+async function pullCommits(fullName: string, token: string, number: number, fetchImpl: typeof fetch) {
+  const pull = await githubJson(`${api(fullName)}/pulls/${number}`, token, fetchImpl) as
+    { number?: unknown; base?: { sha?: unknown; repo?: { full_name?: unknown } | null }; head?: { sha?: unknown } } | null
+  const base = pull?.base?.sha, head = pull?.head?.sha, repo = pull?.base?.repo?.full_name
+  if (pull?.number !== number || typeof base !== 'string' || !SHA.test(base) || typeof head !== 'string' || !SHA.test(head)
+    || typeof repo !== 'string' || repo.toLowerCase() !== fullName.toLowerCase()) throw new GithubApiError(502, 'github-malformed')
+  return { base, head }
+}
+
+// The pull request's merge base and the files it changes from it: as GitHub
+// compares its base with its head, else, where that lists as many as it lists
+// at most, from the pull request's own list, a page at a time; `cut` where
+// that lists as many as it lists at most, too.
+async function pullFiles(fullName: string, token: string, number: number, base: string, head: string, fetchImpl: typeof fetch) {
+  const compared = await githubJson(`${api(fullName)}/compare/${base}...${head}?per_page=1`, token, fetchImpl) as { merge_base_commit?: { sha?: unknown } } | null
+  const mergeBase = compared?.merge_base_commit?.sha
+  if (typeof mergeBase !== 'string' || !SHA.test(mergeBase)) throw new GithubApiError(502, 'github-malformed')
+  const { files, more } = commitChanges(compared)
+  if (!more) return { mergeBase, files, cut: false }
+  const listed: ChangedFile[] = []
+  for (let page = 1; listed.length < PULL_FILES; page++) {
+    const data = await githubJson(`${api(fullName)}/pulls/${number}/files?per_page=${FILES_PAGE}&page=${page}`, token, fetchImpl)
+    if (!Array.isArray(data)) throw new GithubApiError(502, 'github-malformed')
+    listed.push(...commitChanges({ files: data }).files)
+    if (data.length < FILES_PAGE) return { mergeBase, files: listed, cut: false }
+  }
+  return { mergeBase, files: listed, cut: true }
+}
+
+// The lockfiles read here a pull request changes (`files`), by directory:
+// each as it was (`before`, one modified or removed, or a rename's path
+// before) and as it is (`after`, one added or modified, or a rename's path
+// now), and every path of them (`paths`), as many directories as
+// MAX_DIRECTORIES.
+function lockfilesChanged(files: ChangedFile[]) {
+  type Directory = { before: { path: string; format: Format }[]; after: { path: string; format: Format }[]; paths: Set<string> }
+  const directories = new Map<string, Directory>()
+  const take = (side: 'before' | 'after', path: string | undefined) => {
+    const format = path ? lockfileFormat(path) : null
+    if (!path || !format) return
+    const dir = path.slice(0, Math.max(path.lastIndexOf('/'), 0))
+    const directory = directories.get(dir) ?? directories.set(dir, { before: [], after: [], paths: new Set() }).get(dir)!
+    directory[side].push({ path, format })
+    directory.paths.add(path)
+  }
+  for (const [path, status, from] of files) {
+    take('before', status === 'R' ? from : status === 'A' ? undefined : path)
+    take('after', status === 'D' ? undefined : path)
+  }
+  const all = [...directories.entries()].toSorted(([a], [b]) => a < b ? -1 : a > b ? 1 : 0)
+  return { directories: all.slice(0, MAX_DIRECTORIES).map(([, directory]) => directory), cut: all.length > MAX_DIRECTORIES,
+    omitted: all.slice(MAX_DIRECTORIES).flatMap(([, directory]) => [...directory.paths]).toSorted() }
+}
+
+// Whether `lockfile` at `ref` is held in this instance, read without asking
+// GitHub: Bun's, never read, too.
+export const lockfileHeld = (fullName: string, ref: string, { path, format }: { path: string; format: Format }) =>
+  format === 'bun' || lockfileReads.get(`${fullName}\0${ref}\0${path}`) !== undefined
+
+// A lockfile's packages at `ref`, as read in this instance, else shared with
+// a read of it with the same token still under way, else of GitHub; a
+// lockfile its reader refuses, why. Any failure to read it of GitHub but its
+// having none there is the whole answer's, as it would be every file's.
+export function readLockfile(fullName: string, token: string, ref: string, lockfile: { path: string; format: Format }): Promise<LockfileRead> {
+  if (lockfile.format === 'bun') return Promise.resolve({ error: 'lockfile-unsupported' })
+  const key = `${fullName}\0${ref}\0${lockfile.path}`
+  const known = lockfileReads.get(key)
+  if (known) return Promise.resolve(known)
+  const shared = `${token}\0${key}`
+  const reading = lockfilesReading.get(shared) ?? readOf(fullName, token, ref, lockfile, key).finally(() => lockfilesReading.delete(shared))
+  lockfilesReading.set(shared, reading)
+  return reading
+}
+async function readOf(fullName: string, token: string, ref: string, { path, format }: { path: string; format: Format }, key: string): Promise<LockfileRead> {
+  let text: string
+  try {
+    text = await upstreamGithub(() => createClient({ token, userAgent: 'deepview-triage' }).getRepoFile({ repo: fullName, path, ref }))
+  } catch (err) {
+    if (err instanceof GithubApiError && err.message === 'github-not-found') return { error: 'github-not-found' }
+    throw err
+  }
+  let read: LockfileRead
+  try { read = { packages: lockedPackages(format, text, { review: true }) } } catch (error) {
+    read = { error: 'lockfile-unreadable', ...error instanceof Error ? { detail: error.message.slice(0, 500) } : {} }
+  }
+  lockfileReads.set(key, read)
+  return read
+}
+
+// Runs `run` for each of `items`, as many at once as `parallel`, each's
+// result in its place.
+export async function eachOf<T, R>(items: T[], parallel: number, run: (item: T) => Promise<R>): Promise<R[]> {
+  const results: R[] = []
+  let next = 0
+  await Promise.all(Array.from({ length: Math.min(parallel, items.length) }, async () => {
+    while (next < items.length) {
+      const at = next++
+      results[at] = await run(items[at]!)
+    }
+  }))
+  return results
+}
+
+// Versions by semver, the highest first, any semver can't read after them.
+function byVersion(a: string, b: string) {
+  const [exactA, exactB] = [isExactVersion(a), isExactVersion(b)]
+  if (exactA && exactB) return compareVersions(b, a)
+  if (exactA === exactB) return a < b ? -1 : a > b ? 1 : 0
+  return exactA ? -1 : 1
+}
+const byName = (a: AddedPackage, b: AddedPackage) => a.name < b.name ? -1 : a.name > b.name ? 1 : -byVersion(a.version ?? '', b.version ?? '')
+// A package as it is installed: its name and version, where from, and its
+// patch, a patched version's new where its patch is; and as it is locked,
+// with what its reviews are held to too.
+export const lockedAs = (pkg: LockedPackage) => `${pkg.source}\0${pkg.name}@${pkg.version ?? ''}\0${pkg.from ?? ''}\0${pkg.patchHash ?? ''}`
+export const lockedWith = (pkg: LockedPackage) => `${lockedAs(pkg)}\0${pkg.integrity ?? ''}\0${pkg.shasum ?? ''}\0${pkg.lockfile ?? ''}`
+
+// What a directory's lockfiles add, from each read before and after: the
+// package versions after none before has, each once, by name and version,
+// each with the versions of its name before, those past MAX_ADDED counted
+// (`more`), not told but reviewed as they are; and every one after, each
+// integrity it is locked with (`locked`), as many as MAX_LOCKED; or the first
+// failure among them.
+function addedIn(paths: string[], before: LockfileRead[], after: LockfileRead[]): DirectoryDependencies {
+  const failed = [...before, ...after].find(read => 'error' in read)
+  if (failed && 'error' in failed) return { paths, error: failed.error, ...failed.detail ? { detail: failed.detail } : {} }
+  const known = new Set<string>()
+  const versions = new Map<string, Set<string>>()
+  for (const pkg of before.flatMap(read => 'packages' in read ? read.packages : [])) {
+    known.add(lockedAs(pkg))
+    const of = versions.get(pkg.name) ?? versions.set(pkg.name, new Set()).get(pkg.name)!
+    if (pkg.version) of.add(pkg.version)
+  }
+  const added = new Map<string, AddedPackage>()
+  const locked = new Map<string, LockedPackage>()
+  for (const read of after) {
+    for (const pkg of 'packages' in read ? read.packages : []) {
+      const key = lockedAs(pkg)
+      const held = locked.get(lockedWith(pkg))
+      if (!held) locked.set(lockedWith(pkg), { ...pkg })
+      else if (held.dev && !pkg.dev) delete held.dev
+      if (known.has(key)) continue
+      const had = added.get(key)
+      const was = versions.get(pkg.name)
+      if (!had) added.set(key, { ...pkg, ...was ? { was: [...was].toSorted(byVersion) } : {} })
+      else if (had.dev && !pkg.dev) delete had.dev
+    }
+  }
+  const all = [...added.values()].toSorted(byName)
+  return { paths, added: all.slice(0, MAX_LOCKED), more: Math.max(0, all.length - MAX_ADDED),
+    locked: [...locked.values()].slice(0, MAX_LOCKED), moreLocked: Math.max(0, locked.size - MAX_LOCKED) }
+}
+
+// What pull request `number` adds to its lockfiles' packages (PullDependencies),
+// kept by its base's and head's commits, else read of GitHub and kept; none
+// where it changes no lockfile read here.
+export async function pullDependencies(fullName: string, token: string, number: number, kept: PullDependencyStore,
+  fetchImpl: typeof fetch = globalThis.fetch): Promise<PullDependencies> {
+  return (await pullDependenciesAt(fullName, token, number, kept, fetchImpl)).told
+}
+// The same, with the commits it is kept by (`commits`), its base's and
+// head's, by which `kept` reads it.
+export async function pullDependenciesAt(fullName: string, token: string, number: number, kept: PullDependencyStore,
+  fetchImpl: typeof fetch = globalThis.fetch): Promise<{ commits: string; told: PullDependencies }> {
+  const { base, head } = await pullCommits(fullName, token, number, fetchImpl)
+  const key = `${base}...${head}`
+  const held = await kept.read(key)
+  if (held) return { commits: key, told: held }
+  const { mergeBase, files, cut: filesCut } = await pullFiles(fullName, token, number, base, head, fetchImpl)
+  const { directories, cut, omitted } = lockfilesChanged(files)
+  const reads = directories.flatMap(({ before, after }) => [...before.map(lockfile => [mergeBase, lockfile] as const), ...after.map(lockfile => [head, lockfile] as const)])
+  const read = await eachOf(reads, READ_PARALLEL, ([ref, lockfile]) => readLockfile(fullName, token, ref, lockfile))
+  let at = 0
+  const lockfiles = directories.map(({ before, after, paths }) => {
+    const readBefore = read.slice(at, at += before.length)
+    return addedIn([...paths].toSorted(), readBefore, read.slice(at, at += after.length))
+  })
+  const told = { base: mergeBase, head, lockfiles, cut: filesCut || cut, ...cut && !filesCut && omitted.length <= MAX_OMITTED ? { omitted } : {} }
+  await kept.write(key, told)
+  return { commits: key, told }
+}