@preventive/triage 1.0.0-alpha.51 → 1.0.0-alpha.53

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@preventive/triage",
-  "version": "1.0.0-alpha.51",
+  "version": "1.0.0-alpha.53",
   "description": "Client & relay server for triaging of automated reports",
   "license": "MIT OR Apache-2.0",
   "author": {
@@ -141,6 +141,7 @@
     "server-managed/github-oauth.ts",
     "server-managed/github-pulls.ts",
     "server-managed/github-requests.ts",
+    "server-managed/history-windows.ts",
     "server-managed/http-response.ts",
     "server-managed/http.ts",
     "server-managed/link-reports.ts",
@@ -198,6 +199,7 @@
     "server-managed/repository-pulse.ts",
     "server-managed/repository-pulls.ts",
     "server-managed/listed-activity.ts",
+    "server-managed/repository-stamp.ts",
     "server-managed/repository-aliases.ts",
     "server-managed/repository-discovery.ts",
     "server-managed/repository-policy.ts",
@@ -285,6 +287,10 @@
     "common/managed/comments.ts",
     "common/managed/finding-detail.d.ts",
     "common/managed/finding-detail.js",
+    "common/managed/finding-values.d.ts",
+    "common/managed/finding-values.js",
+    "common/managed/lite-name.d.ts",
+    "common/managed/lite-name.js",
     "common/managed/github-agents.ts",
     "common/managed/github-logins.ts",
     "common/managed/npm-packages.d.ts",

package/server-managed/github-requests.ts

@@ -44,15 +44,21 @@ export async function shareGithubRead<T>(fetchImpl: typeof fetch, request: strin
 }
 
 // GitHub's answers to GETs that came with an ETag, by the request (its URL
-// and credential, hashed as outstanding reads are): the next asked again
-// If-None-Match, GitHub checking the credential as ever, and a 304, which
-// it doesn't count against the rate limit, answered by the body kept. Each
-// transport's own, as many as MAX_TAGGED_READS and MAX_TAGGED_BYTES hold,
-// the least recently used first to go; none larger than MAX_TAGGED_READ_BYTES.
+// and credential, hashed as outstanding reads are; an installation's token,
+// which the App mints anew every few minutes on every instance, as the
+// installations' one credential, GitHub telling apart which may read what):
+// the next asked again If-None-Match, GitHub checking the credential as
+// ever, and a 304, which it doesn't count against the rate limit, answered
+// by the body kept. Each transport's own, as many as MAX_TAGGED_READS and
+// MAX_TAGGED_BYTES hold, the least recently used first to go; none larger
+// than MAX_TAGGED_READ_BYTES. Those one this process doesn't hold, read of
+// the store kept beyond it (keepTaggedReadsIn), as a serverless instance
+// starting anew holds none, as many as fit it (MAX_STORED_READ_BYTES).
 const MAX_TAGGED_READS = 1000
 const MAX_TAGGED_BYTES = 32 * 1024 * 1024
 const MAX_TAGGED_READ_BYTES = 1024 * 1024
-type TaggedRead = { etag: string; text: string }
+const MAX_STORED_READ_BYTES = 256 * 1024
+export type TaggedRead = { etag: string; text: string }
 const taggedReads = new WeakMap<typeof fetch, { reads: Map<string, TaggedRead>; bytes: number }>()
 const taggedOf = (fetchImpl: typeof fetch) => {
   let tagged = taggedReads.get(fetchImpl)
@@ -60,29 +66,50 @@ const taggedOf = (fetchImpl: typeof fetch) => {
   return tagged
 }
 const requestKey = (request: string) => createHash('sha256').update(request).digest('hex')
+// A GET's request as its answer is kept by: its URL and credential.
+const taggedRequest = (url: string, token: string | null) => JSON.stringify([url, token?.startsWith('ghs_') ? { installation: true } : token])
 
-// The answer kept to `request`, its ETag and body as GitHub wrote it.
-export function taggedRead(fetchImpl: typeof fetch, request: string): TaggedRead | undefined {
+// Where GitHub's tagged answers are kept beyond this process (http.ts
+// registers the database's), by the request's key; its failures are none
+// kept, or none written.
+export interface TaggedStore { read(key: string): Promise<TaggedRead | null>; write(key: string, read: TaggedRead): Promise<void> }
+let taggedStore: TaggedStore | null = null
+export function keepTaggedReadsIn(store: TaggedStore | null): void { taggedStore = store }
+
+// The answer kept to `request`, its ETag and body as GitHub wrote it: as
+// this process holds it, else, where a store is kept beyond it, as that
+// keeps it.
+export function taggedRead(fetchImpl: typeof fetch, request: string): TaggedRead | undefined | Promise<TaggedRead | undefined> {
   const { reads } = taggedOf(fetchImpl)
   const key = requestKey(request)
   const read = reads.get(key)
-  if (read) { reads.delete(key); reads.set(key, read) }
-  return read
+  if (read) { reads.delete(key); reads.set(key, read); return read }
+  return taggedStore ? storedTaggedRead(fetchImpl, key, taggedStore) : undefined
+}
+async function storedTaggedRead(fetchImpl: typeof fetch, key: string, store: TaggedStore) {
+  const stored = await store.read(key).catch(() => null)
+  if (!stored || typeof stored.etag !== 'string' || typeof stored.text !== 'string') return undefined
+  holdTaggedRead(fetchImpl, key, stored)
+  return stored
 }
 
 // `text` kept as GitHub's answer to `request`, by its ETag.
-export function keepTaggedRead(fetchImpl: typeof fetch, request: string, etag: string, text: string): void {
-  const tagged = taggedOf(fetchImpl)
+export async function keepTaggedRead(fetchImpl: typeof fetch, request: string, etag: string, text: string): Promise<void> {
   const key = requestKey(request)
+  holdTaggedRead(fetchImpl, key, { etag, text })
+  if (text.length <= MAX_STORED_READ_BYTES) await taggedStore?.write(key, { etag, text }).catch(() => {})
+}
+function holdTaggedRead(fetchImpl: typeof fetch, key: string, read: TaggedRead) {
+  const tagged = taggedOf(fetchImpl)
   const before = tagged.reads.get(key)
   if (before) { tagged.reads.delete(key); tagged.bytes -= before.text.length }
-  if (text.length > MAX_TAGGED_READ_BYTES) return
-  tagged.reads.set(key, { etag, text })
-  tagged.bytes += text.length
-  for (const [oldest, read] of tagged.reads) {
+  if (read.text.length > MAX_TAGGED_READ_BYTES) return
+  tagged.reads.set(key, read)
+  tagged.bytes += read.text.length
+  for (const [oldest, held] of tagged.reads) {
     if (tagged.reads.size <= MAX_TAGGED_READS && tagged.bytes <= MAX_TAGGED_BYTES) break
     tagged.reads.delete(oldest)
-    tagged.bytes -= read.text.length
+    tagged.bytes -= held.text.length
   }
 }
 
@@ -97,10 +124,11 @@ export function keepTaggedRead(fetchImpl: typeof fetch, request: string, etag: s
 export function githubJson(url: string, token: string | null, fetchImpl: typeof fetch, method: 'GET' | 'POST' = 'GET', body?: unknown,
   readOnly = method === 'GET'): Promise<unknown> {
   if (readOnly) return shareGithubRead(fetchImpl, JSON.stringify([url, token, method, body]), () => githubJson(url, token, fetchImpl, method, body, false))
-  const request = method === 'GET' && body === undefined ? JSON.stringify([url, token]) : null
-  const tagged = request === null ? undefined : taggedRead(fetchImpl, request)
+  const request = method === 'GET' && body === undefined ? taggedRequest(url, token) : null
   // Each call told in its request's Server-Timing (request-timing.ts).
   return timed('gh', async () => {
+    const held = request === null ? undefined : taggedRead(fetchImpl, request)
+    const tagged = held instanceof Promise ? await held : held
     let res: Response
     const started = Date.now()
     try {
@@ -122,15 +150,17 @@ export function githubJson(url: string, token: string | null, fetchImpl: typeof
     // The answer's body is read within the same timeout: one not read whole is
     // GitHub not reached, as one not answered is; only one that isn't JSON is
     // malformed.
-    const etag = request === null ? null : res.headers?.get('etag') ?? null
+    // Kept only whole: never a 202's, GitHub still working out its answer.
+    const etag = request === null || res.status !== 200 ? null : res.headers?.get('etag') ?? null
+    let text: string | null = null, value: unknown
     try {
       if (etag === null) return await res.json()
-      const text = await res.text()
-      const value = JSON.parse(text) as unknown
-      keepTaggedRead(fetchImpl, request!, etag, text)
-      return value
+      text = await res.text()
+      value = JSON.parse(text)
     } catch (error) {
       throw error instanceof SyntaxError ? new GithubApiError(502, 'github-malformed') : githubUnreachable(error, started)
     }
+    await keepTaggedRead(fetchImpl, request!, etag, text)
+    return value
   })
 }

package/server-managed/history-windows.ts

@@ -0,0 +1,172 @@
+// A default branch's history read by when its commits were made, for the
+// Pulse (repository-pulse.ts yearCommits): where the pages kept from a head
+// run out, the rest of the time it needs read at once, in windows of time
+// (GitHub's `history(since:, until:)`), not a page after another, each page
+// waiting on the last; and the pages GitHub would list from the commits read
+// (git log) kept by their tops, as reading a page at a time keeps them, so a
+// walk later, from a head moved on or in the Commits tab, reads them kept.
+import { linkNoreplyUsers } from './github-accounts.ts'
+import { githubGraphql } from './github-graphql.ts'
+import { COMMIT_FRAGMENT, type GraphCommit, HISTORY_PAGE, type HistoryCommit, type HistoryPages, PAGES_PARALLEL, PAGES_PER_QUERY, commitActors, graphCommit,
+  inBatches, variablesOf } from './repository-history.ts'
+
+// Windows a read is split into at most, each meant to hold as many commits
+// as WINDOW_COMMITS, a page's worth with room to spare, as often as the
+// commits read before it were made, as far back as they reach, the rest left
+// to the next read; pages read of them at most a read, those over going on
+// where a window holds more, as many times as MAX_WAVES, each waiting on the
+// last; and pages kept of what was read at most.
+const MAX_WINDOWS = 40
+const WINDOW_COMMITS = 70
+const WINDOW_PAGES = 50
+const MAX_WAVES = 5
+const MAX_KEPT = 500
+const KEPT_AT_ONCE = 10
+const WINDOWS_AFTER = 2
+const HOUR_MS = 60 * 60_000
+
+// Each window's commits from `$head`, `w<n>` the `n`th's: those made since
+// `$s<n>` (none, any time) and until `$u<n>`, after `$a<n>`.
+const windowsQuery = (count: number) => `query($owner: String!, $name: String!, $head: GitObjectID!, ${variablesOf('s', count, 'GitTimestamp')}, ${
+  variablesOf('u', count, 'GitTimestamp!')}, ${variablesOf('a', count, 'String')}) {
+  repository(owner: $owner, name: $name) {
+    object(oid: $head) { ... on Commit {
+      ${Array.from({ length: count }, (_, at) => `w${at}: history(first: ${HISTORY_PAGE}, since: $s${at}, until: $u${at}, after: $a${at}) {
+        pageInfo { hasNextPage endCursor } nodes { ...historyCommit } }`).join('\n      ')}
+    } }
+  }
+}
+${COMMIT_FRAGMENT}`
+
+type Window = { since: number | null; until: number; after: string | null; oldest?: number }
+type WindowPage = { pageInfo?: { hasNextPage?: unknown; endCursor?: unknown }; nodes?: GraphCommit[] } | null
+type Windows = { repository?: { object?: Record<string, WindowPage> | null } | null } | null
+// A time as GitHub takes it, to the second: a window's start rounded down,
+// its end up, so that those beside each other overlap rather than part.
+const stamp = (at: number, up: boolean) => new Date((up ? Math.ceil : Math.floor)(at / 1000) * 1000).toISOString()
+
+// The commits from `head` made from `low` to `high`, as often as `perMs` (commits
+// a millisecond) says they are made, as many windows as MAX_WINDOWS reach
+// down from `high`, read at once (inBatches), the latest first, each paged on
+// as far as it goes, as many pages as WINDOW_PAGES; and, those reaching
+// `low`, a page of those made before it besides, where a walk reading them
+// goes on from. Since when every one is read (`through`: where the windows
+// reach, each read whole, else, of the latest not, just after the earliest
+// made of those it read, as GitHub lists them the latest made first, those
+// made together left out; or its end, none read), and how many pages were
+// read (`read`).
+export async function readWindows(fullName: string, token: string, head: string, low: number, high: number, perMs: number, fetchImpl: typeof fetch) {
+  const [owner, name] = fullName.split('/') as [string, string]
+  const bottom = Math.max(low, high - MAX_WINDOWS * WINDOW_COMMITS / perMs)
+  const count = Math.min(MAX_WINDOWS, Math.max(1, Math.ceil((high - bottom) * perMs / WINDOW_COMMITS)))
+  const width = (high - bottom) / count
+  let open: Window[] = [...Array.from({ length: count }, (_, at) => ({ since: high - (at + 1) * width, until: high - at * width, after: null })),
+    ...bottom > low ? [] : [{ since: null, until: low - 1000, after: null }]]
+  const commits = new Map<string, HistoryCommit>()
+  let [read, waves] = [0, 0]
+  while (open.length > 0 && read < WINDOW_PAGES && waves++ < MAX_WAVES) {
+    const wave = open.slice(0, WINDOW_PAGES - read)
+    const next: Window[] = []
+    read += wave.length
+    await inBatches(wave, PAGES_PER_QUERY, PAGES_PARALLEL, async batch => {
+      const data = await githubGraphql(windowsQuery(batch.length), { owner, name, head, ...Object.fromEntries(batch.flatMap((window, at) => [
+        [`s${at}`, window.since === null ? null : stamp(window.since, false)], [`u${at}`, stamp(window.until, true)], [`a${at}`, window.after]])) }, token, fetchImpl) as Windows
+      const pages = batch.map((_, at) => data?.repository?.object?.[`w${at}`] ?? null)
+      await linkNoreplyUsers(pages.flatMap(page => page?.nodes ?? []).flatMap(commitActors), token, fetchImpl)
+      for (const [at, window] of batch.entries()) {
+        let oldest = window.oldest ?? Number.POSITIVE_INFINITY
+        for (const commit of (pages[at]?.nodes ?? []).flatMap(node => graphCommit(node) ?? [])) {
+          if (!commits.has(commit.sha)) commits.set(commit.sha, commit)
+          oldest = Math.min(oldest, commit.committedAt ?? Number.POSITIVE_INFINITY)
+        }
+        const info = pages[at]?.pageInfo
+        if (window.since !== null && info?.hasNextPage === true && typeof info.endCursor === 'string') next.push({ ...window, after: info.endCursor, oldest })
+      }
+    })
+    open = [...next, ...open.slice(wave.length)]
+  }
+  const through = Math.max(bottom, ...open.flatMap(window => window.since === null ? [] : [Number.isFinite(window.oldest) ? window.oldest! + 1 : window.until]))
+  return { commits: [...commits.values()], through, read }
+}
+
+// The page GitHub would list from `top` (git log: of those reached, the
+// latest made first, of those made together the first reached) of `known`
+// commits, every one made since `low` known; none where it would list one not
+// known, or one made before `low` while one not known waits beside it.
+export function gitLogPage(top: string, known: Map<string, HistoryCommit>, low: number): HistoryCommit[] | null {
+  if (!known.has(top)) return null
+  const page: HistoryCommit[] = []
+  const reached = new Set<string>()
+  const waiting: HistoryCommit[] = []
+  let unknown = 0
+  const reach = (sha: string) => {
+    if (reached.has(sha)) return
+    reached.add(sha)
+    const commit = known.get(sha)
+    if (!commit || commit.committedAt === null) { unknown++; return }
+    const at = waiting.findIndex(other => other.committedAt! < commit.committedAt!)
+    waiting.splice(at < 0 ? waiting.length : at, 0, commit)
+  }
+  reach(top)
+  while (page.length < HISTORY_PAGE && waiting.length > 0) {
+    const commit = waiting.shift()!
+    if (commit.committedAt! < low && unknown > 0) return null
+    page.push(commit)
+    for (const parent of commit.parents) reach(parent)
+  }
+  return page.length < HISTORY_PAGE && unknown > 0 ? null : page
+}
+
+// The pages from each of `tops` GitHub would list of `known` commits
+// (gitLogPage), kept by their tops, KEPT_AT_ONCE at a time, and from the
+// parents of those they list that none of `listed` is nor any kept lists, on
+// till none is left, as many as MAX_KEPT: as a walk reading pages goes on to
+// them.
+export async function keepPages(tops: string[], known: Map<string, HistoryCommit>, listed: Set<string>, low: number, pages: HistoryPages) {
+  const covered = new Set(listed)
+  const kept: [string, HistoryCommit[]][] = []
+  const waiting = [...tops]
+  while (waiting.length > 0 && kept.length < MAX_KEPT) {
+    const top = waiting.shift()!
+    if (covered.has(top)) continue
+    const page = gitLogPage(top, known, low)
+    if (!page) continue
+    kept.push([top, page])
+    for (const commit of page) covered.add(commit.sha)
+    for (const commit of page) for (const parent of commit.parents) if (!covered.has(parent) && known.has(parent)) waiting.push(parent)
+  }
+  for (let at = 0; at < kept.length; at += KEPT_AT_ONCE) await Promise.all(kept.slice(at, at + KEPT_AT_ONCE).map(([top, page]) => pages.write(top, page)))
+}
+
+// What a walk from `head` (yearCommits) reads next of `starts`: those a page
+// is kept from (`batch`), where any is, or all of them, a page at a time, as
+// long as it has read GitHub fewer times than WINDOWS_AFTER (`queries`), a
+// head moved on soon reaching the pages kept from where it was; else, none
+// kept, those made from
+// `low` to `high` read at once (readWindows), as often as those `found`
+// were made, told it (`found`), and the pages from `starts` on kept of them,
+// as far down as every one is read (keepPages), the queries that took
+// (`queries`, PAGES_PER_QUERY pages each, as many at once as PAGES_PARALLEL
+// a query of its) and none to read (`windowed`); where the windows can't be
+// read, all of them, a page at a time (`windowed` all the same).
+export async function windowsOn(fullName: string, token: string, head: string, starts: string[], found: Map<string, HistoryCommit>, low: number, high: number,
+  pages: HistoryPages, fetchImpl: typeof fetch, queries: number): Promise<{ batch: string[]; queries: number; windowed: boolean }> {
+  const kept = await pages.read(starts)
+  if (kept.size > 0 || queries < WINDOWS_AFTER || !(high > low) || !Number.isFinite(high)) {
+    return { batch: kept.size > 0 ? starts.filter(sha => kept.has(sha)) : starts, queries: 0, windowed: false }
+  }
+  let [newest, oldest, dated] = [Number.NEGATIVE_INFINITY, Number.POSITIVE_INFINITY, 0]
+  for (const { committedAt } of found.values()) {
+    if (committedAt === null) continue
+    [newest, oldest, dated] = [Math.max(newest, committedAt), Math.min(oldest, committedAt), dated + 1]
+  }
+  // From the one commit the walk goes on from, where there is one, none of
+  // those read listed again; else from the head.
+  const read = await readWindows(fullName, token, starts.length === 1 ? starts[0]! : head, low, high, dated / Math.max(newest - oldest, HOUR_MS), fetchImpl)
+    .catch((err: unknown) => { console.warn('managed: history windows read failed:', err); return null })
+  if (!read) return { batch: starts, queries: 0, windowed: true }
+  const listed = new Set(found.keys())
+  for (const commit of read.commits) if (!found.has(commit.sha)) found.set(commit.sha, commit)
+  await keepPages(starts, found, listed, read.through, pages)
+  return { batch: [], queries: Math.ceil(read.read / (PAGES_PER_QUERY * PAGES_PARALLEL)), windowed: true }
+}