@preventive/triage 1.0.0-alpha.47 → 1.0.0-alpha.49

package/server-managed/rename-diff-worker.js

@@ -1,6 +1,6 @@
 // Compare's diff with renamed names hidden (rename-diff.ts), worked out off the
 // server's event loop by @preventive/compare's renamedDiffs: the old side given
-// the new side's names, formatted again where that changes its layout alone,
+// the new side's names, not formatted again (each line where the old side's was),
 // diffed with whitespace changes shown and hidden, and the blocks checked to
 // take it to the new side. `{ before, blocks, reason, reads, marks,
 // ignoringWhitespace }`: the old side as diffed, the change blocks, why the

package/server-managed/repository-agents.ts

@@ -11,13 +11,12 @@
 // an agent, and those alone are answered, never the texts. GraphQL alone
 // gives them, so it needs a token.
 import { type AgentAuthor, type AgentBranch, type AgentCommit, type AgentIssue, type AgentPull, type AppAgent, agentSignals } from '../common/managed/agent-tasks.ts'
+import { GITHUB_AGENTS, type GithubAgentAccount, type GithubAgentId } from '../common/managed/github-agents.ts'
 import { AGENT_BRANCH_PREFIXES, APP_AGENTS } from './app-agents.ts'
 import { GithubApiError } from './github-app.ts'
-import { ACTOR, type NamedAccount, actorOf, commitAccount, githubGraphql } from './repository-history.ts'
+import { ACTOR, type NamedAccount, actorOf, commitAccount, githubGraphql, linkNoreplyUsers } from './repository-history.ts'
 import { ISSUES_CURSOR } from './repository-issues.ts'
 
-export { APP_AGENTS }
-
 export const AGENT_PULLS_PAGE = 50
 export const AGENT_COMMITS_PAGE = 100
 const AGENT_ISSUES = 100
@@ -35,7 +34,8 @@ const SHA = /^[a-f\d]{40}$/u
 const CLAUDE_EMAIL = 'noreply@anthropic.com'
 
 // The app agents' accounts' no-reply addresses, by each login of theirs.
-const AGENT_EMAILS = APP_AGENTS.flatMap(({ account }) => account?.logins.map(login => `${account.id}+${login}@users.noreply.github.com`) ?? [])
+const AGENT_EMAILS = APP_AGENTS.flatMap(({ id }) => GITHUB_AGENTS[id as GithubAgentId].accounts
+  .flatMap(({ id: account, logins }: GithubAgentAccount) => account === undefined ? [] : logins.map(login => `${account}+${login}@users.noreply.github.com`)))
 
 // An agent's branches, a page at a time, as GitHub lists them, by name;
 // each its tip commit's time, message and author.
@@ -164,6 +164,7 @@ async function agentBranches(owner: string, name: string, prefix: string, token:
   for (let page = 0; page < MAX_BRANCH_PAGES; page++) {
     const refs: Connection<RefNode> | undefined = (await githubGraphql(BRANCHES_QUERY, { owner, name, prefix: `refs/heads/${prefix}`, after }, token, fetchImpl) as RefsPage)
       ?.repository?.refs
+    await linkNoreplyUsers((refs?.nodes ?? []).map(node => node?.target?.author), token, fetchImpl)
     found.push(...(refs?.nodes ?? []).flatMap(node => branch(prefix)(node) ?? []))
     const next: Connection<RefNode>['pageInfo'] = refs?.pageInfo
     if (next?.hasNextPage !== true || typeof next.endCursor !== 'string') break
@@ -175,7 +176,7 @@ async function agentBranches(owner: string, name: string, prefix: string, token:
 // What its answers are kept by, with their cursors (http.ts
 // handleRepositoryAgents): bumped as what a page reads, or how it tells an
 // agent, changes, so that no answer an older server kept is sent for a day.
-export const AGENTS_READ_VERSION = 6
+export const AGENTS_READ_VERSION = 7
 
 // A cursor of the default branch's commits: GitHub names one by a commit
 // and how many on from it, `<oid> <offset>`, a space between.
@@ -206,7 +207,9 @@ export async function repositoryAgents(fullName: string, token: string, cursors:
   if (!repository) throw new GithubApiError(404, 'github-not-found')
   const history = repository.defaultBranchRef?.target?.history
   const after = { pulls: nextCursor(repository.pullRequests), commits: nextCursor(history, COMMITS_CURSOR) }
-  const commits = [...history?.nodes ?? [], ...authored?.repository?.defaultBranchRef?.target?.history?.nodes ?? []].flatMap(node => directCommit(node) ?? [])
+  const nodes = [...history?.nodes ?? [], ...authored?.repository?.defaultBranchRef?.target?.history?.nodes ?? []]
+  await linkNoreplyUsers(nodes.map(node => node?.author), token, fetchImpl)
+  const commits = nodes.flatMap(node => directCommit(node) ?? [])
   return {
     pulls: (repository.pullRequests?.nodes ?? []).flatMap(node => pull(node) ?? []),
     commits: commits.filter((commit, index) => commits.findIndex(other => other.oid === commit.oid) === index),

package/server-managed/repository-browser.ts

@@ -171,9 +171,16 @@ async function askAccess(repo: SelectedRepo, start: string | null, userToken: st
       return githubJson(base, token, fetchImpl)
     }
   }
-  // A repository last known private asks who the user is as it asks for its
-  // metadata, not after: its permission is asked of them.
-  if (repo.private && userToken) getIdentity().catch(() => {})
+  // A repository last known private asks who the user is, and mints the
+  // installation token its permission is asked with, as it asks for its
+  // metadata, not after. Each is asked once a request (the token once for
+  // concurrent requests too, github-app.ts), so a failure is answered where
+  // it is awaited. One made public since mints a token it never waits for,
+  // kept for later requests.
+  if (repo.private && userToken) {
+    getIdentity().catch(() => {})
+    if (repo.installationId != null) installationToken().catch(() => {})
+  }
   // Stored visibility can be stale (or internal with private=false). Require
   // current, explicit public visibility or the caller's effective permission.
   const data = await metadata() as RepoMetadata
@@ -232,6 +239,9 @@ async function repositoryReader(repo: SelectedRepo, userToken: string | null, in
     // The user's permission level on it where the gate asked GitHub of it, as
     // for a private repository; null for a public one.
     permission: () => level,
+    // Its default branch as GitHub named it when access was first checked,
+    // `''` where it named none: the one whose head `commit('')` reads.
+    defaultBranch,
     async refs() {
       // Suggestions are bounded; the input also accepts any branch or tag name.
       const list = async (kind: string) => {

package/server-managed/repository-code.ts

@@ -4,9 +4,14 @@
 // bundle builds keep theirs (getRepoTreeTarball), and its files as the npm
 // viewer reads a package's (npm-packages.ts npmFileRows). Readers asking for
 // the same directory meanwhile share its body, a load among the npm viewer's
-// active ones (npm-loads.ts sharedLoad).
+// active ones (npm-loads.ts sharedLoad). The default branch's head's bodies
+// are kept in byte storage, for the next to open them (repositoryCodeKept).
 import { Buffer } from 'node:buffer'
+import { createHash } from 'node:crypto'
 import { encodeBrotli } from './brotli.ts'
+import { CacheMissError, type CacheStorage } from './cache-storage.ts'
+import type { KeptStore } from './npm-kept.ts'
+import { plainObject } from './npm-manifest.ts'
 import { MAX_NPM_JSON_LENGTH, NpmPackageError, npmFileRows } from './npm-packages.ts'
 import { type NpmLoad, loadPlaces, readTarballFiles, sharedLoad } from './npm-loads.ts'
 
@@ -49,9 +54,14 @@ const directoryError = (err: unknown) => {
 
 // The directory as the viewer reads it, as brotli-encoded JSON: `{ repoId,
 // repo, commit, path, tree, files }`, its files as npmFileRows gives them,
-// by their paths in the directory, from `tarball`, the tree's.
-export function loadRepositoryCodeBody(input: RepositoryCodeInput, tarball: () => Promise<Uint8Array>): NpmLoad<Buffer> {
+// by their paths in the directory, from `tarball`, the tree's, as upstream
+// verifies it with its symlinks let climb `symlinksAbove` levels.
+export function loadRepositoryCodeBody(input: RepositoryCodeInput, tarball: (symlinksAbove: number) => Promise<Uint8Array>): NpmLoad<Buffer> {
   const { repoId, repo, commit, path, tree, listed } = input
+  // A symlink may reach the repository's root, as a monorepo package's
+  // LICENSE links the root's, and no further: the directory's depth, less
+  // the level the archive's root already stands above its top directory.
+  const symlinksAbove = path.split('/').length - 1
   // Its places among the active loads, by its files as listed, hold their
   // JSON too, which the listing cannot tell, as escaping may grow a text
   // sixfold: as much as a package's may take for each.
@@ -59,11 +69,147 @@ export function loadRepositoryCodeBody(input: RepositoryCodeInput, tarball: () =
   const maxLength = Math.min(MAX_REPOSITORY_CODE_JSON_LENGTH, places * MAX_NPM_JSON_LENGTH)
   return sharedLoad(`repository ${repoId} ${repo} ${commit} ${path} ${tree}`, async () => {
     let files
-    // A symlink may reach the repository's root, as a monorepo package's
-    // LICENSE links the root's, and no further: the directory's depth, less
-    // the level the archive's root already stands above its top directory.
-    try { files = npmFileRows(await readTarballFiles(await tarball(), { ...REPOSITORY_TAR, symlinksAbove: path.split('/').length - 1 }), maxLength) }
+    try { files = npmFileRows(await readTarballFiles(await tarball(symlinksAbove), { ...REPOSITORY_TAR, symlinksAbove }), maxLength) }
     catch (err) { throw directoryError(err) }
     return encodeBrotli(Buffer.from(JSON.stringify({ repoId, repo, commit, path, tree, files })))
   }, 'repository-busy', { places })
 }
+
+// The form of the bodies kept, bumped as what a body holds changes
+// (npmFileRows): one kept in another is none to read.
+const KEPT_CODE_VERSION = 1
+// The largest body kept. Source text runs, as brotli's JSON, to a third of
+// its bytes or less, and a directory's binary files are named, not held, so
+// this keeps one of some 200 MiB of text: nearly every directory that opens
+// at all (MAX_REPOSITORY_CODE_BYTES). One past it would take as much byte
+// storage as hundreds of ordinary directories, and hold a serverless
+// invocation a while uploading it: it is sent as ever, and read again the
+// next time.
+export const MAX_KEPT_CODE_BYTES = 64 * 1024 * 1024
+const COMMIT = /^[a-f\d]{40}$/u
+
+// A directory as kept: the repository by its id and name, the commit and the
+// directory, as its body names them; and the head they are kept at.
+export type KeptCodeAt = Pick<RepositoryCodeInput, 'repoId' | 'repo' | 'commit' | 'path'>
+type KeptCodeHead = Pick<KeptCodeAt, 'commit' | 'repo'>
+// Under the repository, then the commit, so either goes by its prefix; the
+// repository's name and the directory hashed, as a key's parts are names.
+const keptKey = ({ repoId, repo, commit, path }: KeptCodeAt) =>
+  `${repoId}/${commit}/v${KEPT_CODE_VERSION}-${createHash('sha256').update(JSON.stringify([repo, path])).digest('hex')}.json.br`
+
+// Bodies being kept, by their storage and key, each kept once; and each
+// storage's own name among the active loads, as its reads are one of them.
+const keeping = new WeakMap<CacheStorage, Set<string>>()
+const storageNames = new WeakMap<CacheStorage, number>()
+let storagesNamed = 0
+
+// A body kept, read whole, where it is one MAX_KEPT_CODE_BYTES keeps.
+async function readKept(storage: CacheStorage, key: string): Promise<Buffer | null> {
+  const { size, stream } = await storage.open(key)
+  try {
+    if (size !== null && size > MAX_KEPT_CODE_BYTES) return null
+    const parts: Buffer[] = []
+    let length = 0
+    for await (const part of stream) {
+      length += (part as Buffer).byteLength
+      if (length > MAX_KEPT_CODE_BYTES) return null
+      parts.push(Buffer.from(part as Buffer))
+    }
+    return Buffer.concat(parts, length)
+  } finally { stream.destroy() }
+}
+
+// The directories of repositories' default branches' heads, as
+// loadRepositoryCodeBody encodes them, kept in the managed byte storage
+// (`storage`: Vercel Blob, else the filesystem, encrypted at rest where
+// storage is: storage-payload.ts REPOSITORY_CODE_CACHE), so that one opened
+// again, on any instance, asks GitHub for no tarball, and is neither unpacked
+// nor encoded again; by the commit `heads` keeps for each repository, the
+// head as last read (npm-kept.ts codeHeadStore). Most opens read the head:
+// `/repo/<repo id>` opens it, and its route then names its commit, as do its
+// directories opened from it and a reload. A read naming the default branch
+// (`''`, or `heads/` and its name) takes its head as GitHub resolves it for
+// the commit kept, the directories kept at any other removed (`own`ed,
+// behind its answer); any other read, of a commit, a tag or another branch,
+// reads and keeps a directory only where what it resolves to is the commit
+// kept. Nothing kept is per reader, and each is read only once the reader's
+// own access has named its commit. A body names what it was read of, so
+// instances disagreeing on the head meanwhile, as at a push, only cost a
+// directory read again: one kept for a head since moved on removes itself.
+// The head is kept with the repository's name, which each body's key holds,
+// so a repository renamed has what was kept under its name before removed
+// as a read next names its head. A head is written only for a repository
+// still `active` after it, as one removed meanwhile has its head removed
+// after (remove). Byte storage, as the database, is an optimization: its
+// failures read as a directory kept nowhere, and keep nothing.
+export function repositoryCodeKept(storage: CacheStorage, heads: KeptStore<KeptCodeHead>, active: (repoId: number) => Promise<boolean>,
+  own: (work: Promise<unknown>) => void, debug = false) {
+  const warn = (what: string) => (error: unknown) => { if (debug) console.warn(`managed: kept repository code ${what} failed:`, error) }
+  const headOf = async (repoId: number): Promise<KeptCodeHead | null> => {
+    const kept: unknown = (await heads.read(String(repoId)))?.value
+    return plainObject(kept) && typeof kept['commit'] === 'string' && typeof kept['repo'] === 'string' ? { commit: kept['commit'], repo: kept['repo'] } : null
+  }
+  // What is kept of repository `repoId`: its head first, so that a body being
+  // kept meanwhile removes itself.
+  const remove = async (repoId: number) => {
+    await heads.remove(String(repoId))
+    await storage.delete(String(repoId))
+  }
+  return {
+    // Whether a read of `ref` at repository `repoId`, by its name `repo`,
+    // resolved to `commit` with `defaultBranch` GitHub's, reads and keeps
+    // its directories.
+    async at({ repoId, repo, commit }: Omit<KeptCodeAt, 'path'>, ref: string, defaultBranch: string): Promise<boolean> {
+      if (!COMMIT.test(commit)) return false
+      const kept = await headOf(repoId)
+      const atKept = kept?.commit === commit && kept.repo === repo
+      if (ref !== '' && (defaultBranch === '' || ref !== `heads/${defaultBranch}`)) return atKept
+      if (!atKept) {
+        await heads.write(String(repoId), { commit, repo }, Date.now())
+        // One removed as its commit was read, its head removed before this
+        // wrote it again, keeps nothing, and has what is kept removed again.
+        if (!await active(repoId).catch(() => false)) {
+          await remove(repoId).catch(warn('removal'))
+          return false
+        }
+        // Those kept at the head before, or, renamed, every one kept under
+        // its name before.
+        own(storage.delete(String(repoId), kept?.repo === repo ? commit : undefined).catch(warn('removal')))
+      }
+      return true
+    },
+    // The body kept of the directory `at` names, or null: a load among the
+    // active ones (npm-loads.ts sharedLoad), as one read from GitHub is, a
+    // place of them, as it holds no more than a package may
+    // (MAX_KEPT_CODE_BYTES), till each reader has released it; readers
+    // asking for it meanwhile share it, and none is read where every place
+    // is taken (`repository-busy`).
+    read(at: KeptCodeAt): NpmLoad<Buffer | null> {
+      const key = keptKey(at)
+      const name = storageNames.get(storage) ?? storageNames.set(storage, ++storagesNamed).get(storage)!
+      return sharedLoad(`repository kept ${name} ${key}`, () => readKept(storage, key).catch((error: unknown) => {
+        if (!(error instanceof CacheMissError)) warn('read')(error)
+        return null
+      }), 'repository-busy')
+    },
+    // The directory's body kept, behind its answer, where it is no larger
+    // than MAX_KEPT_CODE_BYTES; and removed again, with the rest of its
+    // commit's, where the head kept is another by then, as when it moved on
+    // or the repository was renamed or removed meanwhile.
+    keep(at: KeptCodeAt, body: Buffer): void {
+      const key = keptKey(at)
+      const kept = keeping.get(storage) ?? keeping.set(storage, new Set()).get(storage)!
+      if (body.byteLength > MAX_KEPT_CODE_BYTES || kept.has(key)) return
+      kept.add(key)
+      own((async () => {
+        await storage.put(key, body)
+        const head = await headOf(at.repoId)
+        if (head?.commit !== at.commit || head.repo !== at.repo) await storage.delete(`${at.repoId}/${at.commit}`)
+      })().catch(warn('write')).finally(() => kept.delete(key)))
+    },
+    // What is kept of repository `repoId`, removed once the repository is
+    // no longer active.
+    remove,
+  }
+}
+export type RepositoryCodeKept = ReturnType<typeof repositoryCodeKept>

package/server-managed/repository-dependency-reviews.ts

@@ -25,13 +25,14 @@ export interface RepoDependencyReview { enabled: boolean; required: number; sinc
 // A review published of the version it is of: its package, version and
 // npm's integrity of it, the patch it is of (none here, the app reviewing no
 // patch yet), the version it was compared with (none for a whole package's),
-// who wrote it (`reviewer`, their id, else their login then), its verdict,
-// whether it checked every file (`full`), and when it was published.
+// who wrote it (`reviewer`, their id, else their login then), their login
+// as it was then (`login`), its verdict, whether it checked every file
+// (`full`), and when it was published.
 // One made in a pull request's comments (`comment`) counts for it alone;
 // one published here has its `id`.
 export interface PublishedReview {
   id?: string; package: string; version: string; integrity: string; patchHash?: string; basePackage: string | null; baseVersion: string | null
-  reviewer: string; status: ReviewVerdict; full: boolean; time: number; comment?: true
+  reviewer: string; login: string; status: ReviewVerdict; full: boolean; time: number; comment?: true
 }
 // A repository's review in effect for a reader: its settings, whether they
 // publish reviews to one of the teams that count (`canReview`), and the
@@ -45,6 +46,8 @@ export interface RepositoryDependencyReviewStore {
   listRepoDependencyReviews(): Promise<Map<number, RepoDependencyReview>>
   // One repository's, none where it isn't connected.
   repoDependencyReview(repoId: number): Promise<RepoDependencyReview | null>
+  // Those of `repoIds` connected, by their ids, in one statement.
+  getRepoDependencyReviews(repoIds: number[]): Promise<Map<number, RepoDependencyReview>>
   // The teams whose published reviews count for each repository (COUNTING_TEAM),
   // by name.
   listRepoReviewTeams(): Promise<Map<number, { id: string; name: string }[]>>
@@ -103,7 +106,7 @@ async function publishedReviews(db: ManagedSql, repoId: number, versions: Review
       const nonText = Array.isArray(snapshot) ? null : snapshot?.nonTextFiles ?? []
       const full = summarizeReview(entries.get(row.id) ?? [], files, nonText).coverage === 'full'
       found.push({ id: row.id, package: row.package, version: row.version, integrity: row.integrity, basePackage: row.basePackage, baseVersion: row.baseVersion,
-        reviewer: row.userId ?? `login:${row.userLogin.toLowerCase()}`, status: row.status, full, time: Number(row.publishedAt) })
+        reviewer: row.userId ?? `login:${row.userLogin.toLowerCase()}`, login: row.userLogin, status: row.status, full, time: Number(row.publishedAt) })
     }
   }
   return found
@@ -122,6 +125,11 @@ export function repositoryDependencyReviewMethods(db: ManagedSql): RepositoryDep
       return new Map(rows.map(row => [row.repoId, setting(row)]))
     },
     repoDependencyReview: readSetting,
+    async getRepoDependencyReviews(repoIds) {
+      if (repoIds.length === 0) return new Map()
+      const rows = await db.prepare(`SELECT ${columns} FROM managed_selected_repo WHERE repo_id IN (${repoIds.map(() => '?').join(',')})`).all(...repoIds) as SettingRow[]
+      return new Map(rows.map(row => [row.repoId, setting(row)]))
+    },
     async listRepoReviewTeams() {
       const rows = await db.prepare(`SELECT DISTINCT tr.repo_id AS repoId, t.id AS id, t.name AS name FROM managed_team_repo tr
         JOIN managed_team t ON t.id = tr.team_id WHERE t.hidden = 0 AND t.dependency_review = 1 ORDER BY t.name, t.id`).all() as { repoId: number; id: string; name: string }[]