@preventive/triage 1.0.0-alpha.51 → 1.0.0-alpha.53

package/server-managed/http.ts

@@ -138,7 +138,8 @@ import { mergeLinkGroups } from './link-reports.ts'
 import { backfillFindingIds, reportEntries } from '@preventive/report'
 import type { IncomingMessage, ServerResponse } from 'node:http'
 import { pipeline } from 'node:stream/promises'
-import { type KeptStore, REPOSITORY_READ_KEPT_MS, authoredCommitStore, branchTipStore, codeHeadStore, codeTreeStore, commitPullStore, historyChangeStore, historyPageStore, historySquashStore, historyTreeStore,
+import { setTimeout as delay } from 'node:timers/promises'
+import { type KeptStore, REPOSITORY_READ_KEPT_MS, authoredCommitStore, branchTipStore, codeHeadStore, codeTreeStore, commitPullStore, githubTagStore, historyChangeStore, historyPageStore, historySquashStore, historyTreeStore,
   issueListStore, lockfileReadStore,
   npmDeltaStore, npmPublishTimeStore, npmVersionStore, ownKeptPrunes, pullDependencyStore, pullListStore, repositoryReadStore, reviewBaselineStore, riskLineStore, scanReadStore } from './npm-kept.ts'
 import { plainObject } from './npm-manifest.ts'
@@ -177,6 +178,7 @@ import { DEFAULT_MANAGED_SCAN_MODEL, MANAGED_SCAN_MODELS } from '../common/manag
 import { CONFIG_PATH, type ManagedSessionInfo, type ServerInfo } from '../common/server-info.ts'
 import { bundleCommitHash } from '../common/bundle-commit.js'
 import { GithubApiError, collectRepos, fetchPublicRepository, installUrl, publicRepositoryName, repositoryInstallation } from './github-app.ts'
+import { type TaggedRead, keepTaggedReadsIn } from './github-requests.ts'
 import { upstreamGithub } from './github-errors.ts'
 import { createClient } from '@preventive/upstream/github.js'
 import type { ConnectedRepo } from './github-app.ts'
@@ -239,6 +241,7 @@ import { PULSE_WEEKS, PULSE_YEAR_VERSION, type PulseNarrowing, type PulseStats,
   repositoryStats, withLines, yearCommits } from './repository-pulse.ts'
 import { type AgentMerges, agentCommits, agentMerges, directCommits, sameMerges } from './agent-merges.ts'
 import { listedActivity } from './listed-activity.ts'
+import { STAMP_READ, repositoryStamp } from './repository-stamp.ts'
 import { ISSUES_LIST_READ_VERSION, type IssueList, type Listed, PULLS_READ_VERSION, type PullList, pullsReadTo, readIssues, readPulls } from './repository-pulls.ts'
 import { AGENT_MARKS_READ } from './app-agents.ts'
 import { releaseCommits } from './release-commits.ts'
@@ -1869,7 +1872,8 @@ async function handleRepositoryHistory(res: ServerResponse, deps: ManagedHttpDep
     // Open pull heads do not depend on the commit history. Start them with
     // it, handling failures immediately even if the history fails first.
     if (token && path === '' && page === 1) {
-      readingPulls = keptAnswer(deps, `${repoId}\0openPulls`, () => openPullHeads(repo, token), undefined, fresh).then(answer => answer.value.pulls, quiet(null))
+      readingPulls = keptAnswer(deps, `${repoId}\0openPulls`, () => openPullHeads(repo, token), undefined, fresh, fresh ? null : stampOf(deps, repoId, repo, token))
+        .then(answer => answer.value.pulls, quiet(null))
     }
     let body: object
     if (all) {
@@ -1878,7 +1882,8 @@ async function handleRepositoryHistory(res: ServerResponse, deps: ManagedHttpDep
       // follow, the deepest page says.
       // Asked for afresh, those kept answered as just read: how long till
       // they may be read again (`retryIn`), for the client to ask then.
-      const heads = starts ? null : await keptAnswer<AllHeads>(deps, `${repoId}\0heads`, () => repositoryAllHeads(repo, token), undefined, fresh)
+      const heads = starts ? null : await keptAnswer<AllHeads>(deps, `${repoId}\0heads`, () => repositoryAllHeads(repo, token), undefined, fresh,
+        fresh ? null : stampOf(deps, repoId, repo, token))
       const retryIn = heads && fresh && heads.kept ? heads.at + REPOSITORY_READ_FLOOR_MS - Date.now() : 0
       body = starts ? { all: true, page, commits: (await historyFrom(repo, token, starts, pages)).commits, more: page < MAX_HISTORY_PAGE }
         : { all: true, ...await repositoryAllHistory(repo, token, heads!.value, pages), ...retryIn > 0 ? { retryIn } : {} }
@@ -1888,7 +1893,7 @@ async function handleRepositoryHistory(res: ServerResponse, deps: ManagedHttpDep
       // branches reads and the latest tags, kept for both; without, GitHub's
       // first page of each.
       const naming = () => token
-        ? keptAnswer<AllHeads>(deps, `${repoId}\0heads`, () => repositoryAllHeads(repo, token)).then(({ value }) => ({
+        ? keptAnswer<AllHeads>(deps, `${repoId}\0heads`, () => repositoryAllHeads(repo, token), undefined, false, stampOf(deps, repoId, repo, token)).then(({ value }) => ({
           branches: value.heads.map(({ name, sha }) => ({ name, sha })), tags: value.tags }))
         : keptAnswer(deps, `${repoId}\0refs`, () => repositoryRefTips(repo, token)).then(answer => answer.value)
       const tips = page === 1 ? naming() : null
@@ -2052,33 +2057,58 @@ const refusedRead = (err: unknown) => err instanceof GithubApiError
 // answered where that read fails. Those asking for one not kept while it is
 // read, as a tab's first readers do together, are answered by that read. One GitHub refuses to read again is no
 // longer kept: the next to ask is refused as GitHub refused it. `at` is when
-// what it answers was read.
+// what it answers was read. With `stamp`, the repository's change stamp
+// (stampOf), one kept is read again, behind it or afresh, only where the
+// stamp moved since it was read whole again (the stamp read before it
+// was), or that was STAMP_TRUSTED_MS ago or longer; else it is kept as read
+// now, as GitHub would answer it. One read the first time, none's stamp
+// asked for, is read whole once more before its stamp keeps it.
+const STAMP_TRUSTED_MS = 60 * 60_000
 async function keptAnswer<T extends object>(deps: ManagedHttpDeps, key: string, read: () => Promise<T>,
-  keepThrough: (err: unknown) => boolean = () => false, fresh = false): Promise<{ value: T; kept: boolean; at: number }> {
+  keepThrough: (err: unknown) => boolean = () => false, fresh = false, stamp: Stamp | null = null): Promise<{ value: T; kept: boolean; at: number }> {
   const store = repositoryReadStore(deps.db, deps.config.debug)
   // One kept longer than a day, not yet removed (pruneKept removes them as
   // others are written), is none.
   const stored = await store.read(key)
   const held = stored && Date.now() - stored.at < REPOSITORY_READ_KEPT_MS ? stored : null
-  // Read, and kept as read when it was, which it is said read at.
-  const readKept = async () => {
+  // The stamp what is kept was read whole at, and when (`read`).
+  const stampKey = `${key}\0stamp`
+  // Read, and kept as read when it was, which it is said read at, with the
+  // stamp read before it (`began`), where one was.
+  const readKept = async (began: string | null = null) => {
     try {
       const value = await read()
       const at = Date.now()
       await store.write(key, value, at)
-      return { value, at }
+      if (began !== null) await store.write(stampKey, { stamp: began, read: at }, at)
+      return { value, at, kept: false }
     } catch (err) {
       if (refusedRead(err)) await store.remove(key)
       throw err
     }
   }
+  // Read whole again, or kept as read now where the repository's stamp
+  // hasn't moved since it was.
+  const refresh = async () => {
+    if (held && stamp) {
+      const [told, current] = await Promise.all([store.read(stampKey), stamp().catch(() => null)])
+      const was = told?.value as { stamp?: unknown; read?: unknown } | undefined
+      if (current !== null && was?.stamp === current && typeof was.read === 'number' && Date.now() - was.read < STAMP_TRUSTED_MS) {
+        const at = Date.now()
+        await store.write(key, held.value, at)
+        return { value: held.value as T, at, kept: true }
+      }
+      return readKept(current)
+    }
+    return readKept()
+  }
   const reread = () => {
     let reading = repositoryRereads.get(key)
     if (!reading) {
-      reading = readKept().finally(() => repositoryRereads.delete(key))
+      reading = refresh().finally(() => repositoryRereads.delete(key))
       repositoryRereads.set(key, reading)
     }
-    return reading as Promise<{ value: T; at: number }>
+    return reading as Promise<{ value: T; at: number; kept: boolean }>
   }
   const age = held ? Date.now() - held.at : Infinity
   const now = fresh && age >= REPOSITORY_READ_FLOOR_MS
@@ -2092,13 +2122,23 @@ async function keptAnswer<T extends object>(deps: ManagedHttpDeps, key: string,
     return { value: held.value as T, kept: true, at: held.at }
   }
   try {
-    return { ...await reread(), kept: false }
+    return await reread()
   } catch (err) {
     if (held && ((now && !refusedRead(err)) || keepThrough(err))) return { value: held.value as T, kept: true, at: held.at }
     throw err
   }
 }
 
+// Repository `repoId`'s change stamp (repository-stamp.ts), read with `token`
+// at most every REPOSITORY_READ_FLOOR_MS by every reader together, for its
+// kept answers to be read again only where it moved (keptAnswer); `also`
+// what else such an answer moves with, as a period's days do.
+type Stamp = () => Promise<string>
+const stampOf = (deps: ManagedHttpDeps, repoId: number, fullName: string, token: string, also = ''): Stamp => async () =>
+  `${(await keptAnswer(deps, `${repoId}\0stamp\0${STAMP_READ}`, () => repositoryStamp(fullName, token), undefined, true)).value.stamp}${also}`
+// The day it is (UTC), as a period's answer moves with it.
+const today = () => `\0${Math.floor(Date.now() / (24 * 60 * 60_000))}`
+
 // The pages of repository `repoId`'s history kept (historyPageStore), each by
 // the commit it starts at (repository-history.ts historyFrom) and the rules
 // its co-authors were read by (commit-coauthors.ts COAUTHORS_READ), those
@@ -2221,9 +2261,9 @@ const keptIssues = (deps: ManagedHttpDeps, repoId: number, fullName: string, tok
 // What is kept by `key` (keptAnswer), read now where it was read
 // REPOSITORY_READ_FRESH_MS ago or longer, not behind: for an answer that is
 // kept as fresh itself, as told of it (the Agents tab's).
-async function currentAnswer<T extends object>(deps: ManagedHttpDeps, key: string, read: () => Promise<T>) {
+async function currentAnswer<T extends object>(deps: ManagedHttpDeps, key: string, read: () => Promise<T>, stamp: Stamp | null = null) {
   const stored = await repositoryReadStore(deps.db, deps.config.debug).read(key)
-  return keptAnswer(deps, key, read, undefined, !stored || Date.now() - stored.at >= REPOSITORY_READ_FRESH_MS)
+  return keptAnswer(deps, key, read, undefined, !stored || Date.now() - stored.at >= REPOSITORY_READ_FRESH_MS, stamp)
 }
 
 // Repository `repoId`'s default branch's head, as its tabs read it: as the
@@ -2237,7 +2277,8 @@ async function defaultTip(deps: ManagedHttpDeps, repoId: number, fullName: strin
   const tip = all?.heads.find(head => head.name === all.defaultBranch)?.sha
   if (typeof tip === 'string' && /^[a-f\d]{40}$/u.test(tip)) return { sha: tip, kept: true }
   const head = async () => ({ sha: await defaultHead(fullName, token) })
-  const read = await (current ? currentAnswer(deps, `${repoId}\0pulse-head`, head) : keptAnswer(deps, `${repoId}\0pulse-head`, head))
+  const stamp = stampOf(deps, repoId, fullName, token)
+  const read = await (current ? currentAnswer(deps, `${repoId}\0pulse-head`, head, stamp) : keptAnswer(deps, `${repoId}\0pulse-head`, head, undefined, false, stamp))
   return { sha: read.value.sha, kept: read.kept }
 }
 
@@ -2287,10 +2328,10 @@ function historySquashes(deps: ManagedHttpDeps, repoId: number): Squashes {
 // `readAt`, when it was read from GitHub, one kept as old as it was kept, so
 // that a reader holding two reads of the same can tell the later.
 async function readWholeRepository(res: ServerResponse, deps: ManagedHttpDeps, cookie: string | undefined, repoId: number,
-  read: (fullName: string, token: string) => Promise<object>, key: string | null = null, readAt = false): Promise<void> {
+  read: (fullName: string, token: string) => Promise<object>, key: string | null = null, readAt = false, stamped = false): Promise<void> {
   await answerWholeRepository(res, deps, cookie, repoId, async (fullName, token) => {
     const { value, kept, at } = key === null ? { value: await read(fullName, token), kept: false, at: Date.now() }
-      : await keptAnswer(deps, `${repoId}\0${key}`, () => read(fullName, token))
+      : await keptAnswer(deps, `${repoId}\0${key}`, () => read(fullName, token), undefined, false, stamped ? stampOf(deps, repoId, fullName, token) : null)
     return { value: readAt ? { ...value, readAt: at } : value, kept }
   })
 }
@@ -2387,7 +2428,10 @@ async function handleRepositoryIssues(res: ServerResponse, deps: ManagedHttpDeps
     || (after !== null && !ISSUES_CURSOR.test(after))) { sendJson(res, 400, { error: 'bad-request' }); return }
   await answerWholeRepository(res, deps, cookie, repoId, async (fullName, token) => {
     const { value: { commits: _commits, ...page }, kept, at } = await keptAnswer(deps, `${repoId}\0issues\0${ISSUES_READ_VERSION}\0${kind}\0${state}\0${after ?? ''}`,
-      async () => ({ kind, state, ...await repositoryIssues(fullName, token, kind, state, after) }))
+      async () => ({ kind, state, ...await repositoryIssues(fullName, token, kind, state, after) }), undefined, false,
+      // An issue's tile tells nothing its update time doesn't move with; a
+      // pull request's, its checks.
+      kind === 'issues' ? stampOf(deps, repoId, fullName, token) : null)
     return { value: { ...page, readAt: at }, kept }
   })
 }
@@ -2408,9 +2452,9 @@ async function handleRepositoryAgents(res: ServerResponse, deps: ManagedHttpDeps
   await readWholeRepository(res, deps, cookie, repoId, (fullName, token) => repositoryAgents(fullName, token, cursors, historyPages(deps, repoId, true, true),
     commitPulls(deps, repoId), { pulls: () => keptPulls(deps, repoId, fullName, token, null), issues: () => keptIssues(deps, repoId, fullName, token, null),
       head: async () => (await defaultTip(deps, repoId, fullName, token, true)).sha,
-      heads: async () => (await currentAnswer<AllHeads>(deps, `${repoId}\0heads`, () => repositoryAllHeads(fullName, token))).value,
+      heads: async () => (await currentAnswer<AllHeads>(deps, `${repoId}\0heads`, () => repositoryAllHeads(fullName, token), stampOf(deps, repoId, fullName, token))).value,
       tips: branchTips(deps, repoId), authored: authoredCommits(deps, repoId) }),
-    `agents\0${AGENTS_READ_VERSION}\0${cursors.pulls ?? ''}\0${cursors.commits ?? ''}`)
+    `agents\0${AGENTS_READ_VERSION}\0${cursors.pulls ?? ''}\0${cursors.commits ?? ''}`, false, true)
 }
 
 // GET /api/admin/repositories/item: the issue or pull request numbered
@@ -2449,13 +2493,19 @@ const PULLS_AT_ONCE = 3
 async function handleRepositoryPullDependencies(res: ServerResponse, deps: ManagedHttpDeps, cookie: string | undefined, query: URLSearchParams): Promise<void> {
   const repoId = Number(query.get('repoId'))
   const review = query.get('review') === '1'
+  // As an open one's tile on the Pull requests tab asks (`listed=1`): each
+  // by the commits the page it is of names, where one kept names them
+  // (keptPullCommits); else by its number, kept a while, as a popup's.
+  const tiles = query.get('listed') === '1'
   const listed = query.get('numbers')
   const numbers = listed === null ? [itemNumber(query)] : listed.split(',').map(text => itemNumber(new URLSearchParams({ number: text })))
   if (!Number.isSafeInteger(repoId) || repoId <= 0 || numbers.length > MAX_ASKED_PULLS || numbers.some(number => number === null) || new Set(numbers).size < numbers.length
-    || !['1', null].includes(query.get('review'))) { sendJson(res, 400, { error: 'bad-request' }); return }
+    || !['1', null].includes(query.get('review')) || !['1', null].includes(query.get('listed'))) { sendJson(res, 400, { error: 'bad-request' }); return }
   await answerWholeRepository(res, deps, cookie, repoId, async (fullName, token, user) => {
-    if (listed === null) return pullDependencyAnswer(deps, fullName, token, user, repoId, numbers[0]!, review)
-    const answers = await eachOf(numbers as number[], PULLS_AT_ONCE, number => pullDependencyAnswer(deps, fullName, token, user, repoId, number, review).catch((err: unknown) => {
+    const commits = tiles ? await keptPullCommits(deps, repoId, numbers as number[]) : new Map<number, PullCommits>()
+    if (listed === null) return pullDependencyAnswer(deps, fullName, token, user, repoId, numbers[0]!, review, commits.get(numbers[0]!))
+    const answers = await eachOf(numbers as number[], PULLS_AT_ONCE, number => pullDependencyAnswer(deps, fullName, token, user, repoId, number, review, commits.get(number))
+      .catch((err: unknown) => {
       // GitHub asked, though it failed: its access asked again after.
       if (err instanceof GithubApiError) return { value: { number, error: err.message }, kept: false }
       throw err
@@ -2477,7 +2527,7 @@ async function pullDependencyAnswer(deps: ManagedHttpDeps, fullName: string, tok
       const read = await pullDependenciesAt(fullName, token, number, kept)
       reading.told = read.told
       return { commits: read.commits }
-    })
+    }, undefined, false, stampOf(deps, repoId, fullName, token))
   // What its commits tell, as read, else as kept by them, else, no longer
   // kept, read again: by those given, without asking GitHub for them.
   const held = reading.told ?? (typeof answered.value.commits === 'string' ? await kept.read(answered.value.commits) : null)
@@ -2696,22 +2746,48 @@ async function teamOpenPulls(deps: ManagedHttpDeps, repoId: number, fullName: st
   let after: string | null = null, kept = true
   for (let page = 0; page < TEAM_PULL_PAGES; page++) {
     const cursor = after
-    const answer = await keptAnswer(deps, `${repoId}\0issues\0${ISSUES_READ_VERSION}\0pulls\0open\0${cursor ?? ''}`,
-      async () => ({ kind: 'pulls', state: 'open', ...await repositoryIssues(fullName, token, 'pulls', 'open', cursor) }))
+    const answer = await keptAnswer(deps, openPullsKey(repoId, cursor), async () => ({ kind: 'pulls', state: 'open', ...await repositoryIssues(fullName, token, 'pulls', 'open', cursor) }))
     kept &&= answer.kept
-    const value = answer.value as { items?: unknown; more?: unknown; after?: unknown; commits?: unknown }
+    const value = answer.value as OpenPullsPage
     items.push(...(Array.isArray(value.items) ? value.items : []).filter(item => Number.isSafeInteger(item?.number) && item.number > 0))
-    for (const pull of Array.isArray(value.commits) ? value.commits as { number?: unknown; base?: unknown; head?: unknown }[] : []) {
-      const { number, base, head } = pull ?? {}
-      if (Number.isSafeInteger(number) && typeof base === 'string' && OBJECT_ID.test(base) && typeof head === 'string' && OBJECT_ID.test(head)) {
-        commits.set(number as number, { base, head })
-      }
-    }
+    pageCommits(value, commits)
     if (value.more !== true || typeof value.after !== 'string') return { items, commits, more: false, kept }
     after = value.after
   }
   return { items, commits, more: true, kept }
 }
+type OpenPullsPage = { items?: unknown; more?: unknown; after?: unknown; commits?: unknown }
+const openPullsKey = (repoId: number, cursor: string | null) => `${repoId}\0issues\0${ISSUES_READ_VERSION}\0pulls\0open\0${cursor ?? ''}`
+// Each open pull request's base's and head's commits a page of them names,
+// into `commits` by its number.
+function pageCommits(page: OpenPullsPage, commits: Map<number, PullCommits>) {
+  for (const pull of Array.isArray(page.commits) ? page.commits as { number?: unknown; base?: unknown; head?: unknown }[] : []) {
+    const { number, base, head } = pull ?? {}
+    if (Number.isSafeInteger(number) && typeof base === 'string' && OBJECT_ID.test(base) && typeof head === 'string' && OBJECT_ID.test(head)) {
+      commits.set(number as number, { base, head })
+    }
+  }
+}
+
+// Open pull requests `numbers`' base's and head's commits, as the pages of
+// them the Pull requests tab read were kept with them (GET issues), as many
+// of those as TEAM_PULL_PAGES on from its first, while all are kept: GitHub
+// not asked, nor pages read where none is kept. A tile of a page kept, even
+// one read again behind it, is of the commits it names.
+async function keptPullCommits(deps: ManagedHttpDeps, repoId: number, numbers: number[]) {
+  const store = repositoryReadStore(deps.db, deps.config.debug)
+  const commits = new Map<number, PullCommits>()
+  let after: string | null = null
+  for (let page = 0; page < TEAM_PULL_PAGES && numbers.some(number => !commits.has(number)); page++) {
+    const stored = await store.read(openPullsKey(repoId, after))
+    if (!stored || Date.now() - stored.at >= REPOSITORY_READ_KEPT_MS) break
+    const value = stored.value as OpenPullsPage
+    pageCommits(value, commits)
+    if (value.more !== true || typeof value.after !== 'string') break
+    after = value.after
+  }
+  return commits
+}
 
 // The lines the versions wanting reviews of `answers` change, as their rows
 // show them (npm-delta.js deltaBase), a patched one's none, where they are
@@ -2830,15 +2906,55 @@ async function handleRepositoryPulse(res: ServerResponse, deps: ManagedHttpDeps,
   const weeks = Number(query.get('weeks') ?? '4')
   if (!Number.isSafeInteger(repoId) || repoId <= 0 || !PULSE_WEEKS.includes(weeks)) { sendJson(res, 400, { error: 'bad-request' }); return }
   await answerWholeRepository(res, deps, cookie, repoId, async (fullName, token) => {
-    const [period, year] = await Promise.all([
-      keptAnswer(deps, `${repoId}\0pulse\0${weeks}`, () => pulseActivity(deps, repoId, fullName, token, weeks)),
-      pulseCommits(deps, repoId, fullName, token, weeks),
+    const year = pulseYearRead(deps, repoId, fullName, token, weeks)
+    const [period, counted] = await Promise.all([
+      keptAnswer(deps, `${repoId}\0pulse\0${weeks}`, () => pulseActivity(deps, repoId, fullName, token, weeks), undefined, false, stampOf(deps, repoId, fullName, token, today())),
+      Promise.race([year.reading, delay(PULSE_YEAR_WAIT_MS, null, { ref: false })]),
     ])
-    return { value: { weeks, ...period.value, stats: year.stats && periodStats(year.stats, weeks), reading: year.reading, pulls: year.pulls,
-      pending: year.pending }, kept: period.kept && year.kept }
+    const told = counted ?? { stats: await lastPulseYear(deps, repoId, pulseSince(52, Date.now())), reading: year.progress.commits, pulls: null, pending: false,
+      kept: false }
+    return { value: { weeks, ...period.value, stats: told.stats && periodStats(told.stats, weeks), reading: told.reading, pulls: told.pulls,
+      pending: told.pending, ...counted ? {} : { behind: true } }, kept: period.kept && told.kept }
   })
 }
 
+// How long a Pulse answer waits on its year (pulseCommits) before it is
+// sent without it: the period's pull requests and issues told, and how many
+// of the year's commits are read so far (`reading`, beside the year last
+// counted), the year read on behind it (`behind`, for the client to ask
+// again till it is, however long a page of it takes to read); those
+// asking meanwhile, by the database, the repository and the period, waiting
+// on that read, not one of their own, unless it began PULSE_YEAR_HELD_MS ago
+// or longer.
+const PULSE_YEAR_WAIT_MS = 1500
+const PULSE_YEAR_HELD_MS = 60_000
+type PulseYear = Awaited<ReturnType<typeof pulseCommits>>
+type PulseYearRead = { reading: Promise<PulseYear>; progress: { commits: number }; began: number }
+const pulseYearReads = new WeakMap<ManagedDb, Map<string, PulseYearRead>>()
+function pulseYearRead(deps: ManagedHttpDeps, repoId: number, fullName: string, token: string, weeks: number) {
+  let reads = pulseYearReads.get(deps.db)
+  if (!reads) pulseYearReads.set(deps.db, reads = new Map())
+  const key = `${repoId}\0${weeks}`
+  const held = reads.get(key)
+  if (held && Date.now() - held.began < PULSE_YEAR_HELD_MS) return held
+  const progress = { commits: 0 }
+  const year = { reading: pulseCommits(deps, repoId, fullName, token, weeks, progress), progress, began: Date.now() }
+  reads.set(key, year)
+  const settled = year.reading.catch((err: unknown) => { console.warn('managed: pulse year read failed:', err) })
+    .finally(() => { if (reads.get(key) === year) reads.delete(key) })
+  deps.track(settled)
+  deps.waitUntil?.(settled)
+  return year
+}
+
+// The year last counted of repository `repoId` (pulseCommits), where it is
+// no more than PULSE_LAST_MS behind the year from `first`; else none.
+const pulseLastKey = (repoId: number) => `${repoId}\0pulse-year-last`
+async function lastPulseYear(deps: ManagedHttpDeps, repoId: number, first: number) {
+  const kept = (await repositoryReadStore(deps.db, deps.config.debug).read(pulseLastKey(repoId)))?.value as { first: number; stats: PulseStats } | undefined
+  return kept && first - kept.first <= PULSE_LAST_MS ? kept.stats : null
+}
+
 // The default branch's year (repository-pulse.ts commitStats), counted from
 // its commits as the Commits tab keeps them, with their lines
 // (yearCommits): kept by the head it was read from (repositoryReadStore),
@@ -2908,12 +3024,12 @@ async function handleRepositoryPulseItems(res: ServerResponse, deps: ManagedHttp
   await answerWholeRepository(res, deps, cookie, repoId, async (fullName, token) => {
     const narrowing = { from, to, authors }
     const items = await keptAnswer(deps, `${repoId}\0pulse-items\0${weeks}\0${from}\0${to ?? ''}\0${narrowing.authors.join(',')}`,
-      () => pulseActivity(deps, repoId, fullName, token, weeks, narrowing))
+      () => pulseActivity(deps, repoId, fullName, token, weeks, narrowing), undefined, false, stampOf(deps, repoId, fullName, token, today()))
     return { value: { weeks, ...items.value }, kept: items.kept }
   })
 }
 
-async function pulseCommits(deps: ManagedHttpDeps, repoId: number, fullName: string, token: string, weeks = 52) {
+async function pulseCommits(deps: ManagedHttpDeps, repoId: number, fullName: string, token: string, weeks = 52, progress: { commits: number } | null = null) {
   const now = Date.now()
   const first = pulseSince(52, now)
   // As far back as a period of `weeks` weeks needs the year read whole: its
@@ -2924,22 +3040,21 @@ async function pulseCommits(deps: ManagedHttpDeps, repoId: number, fullName: str
   if (!head.sha) return { stats: commitStats([], first), reading: null, pulls: null, pending: false, kept: head.kept }
   const store = repositoryReadStore(deps.db, deps.config.debug)
   const key = `${repoId}\0pulse-year\0${PULSE_YEAR_VERSION}\0${head.sha}\0${first}`
-  const lastKey = `${repoId}\0pulse-year-last`
+  const lastKey = pulseLastKey(repoId)
   const mergesKey = pulseMergesKey(repoId)
-  const last = async () => {
-    const kept = (await store.read(lastKey))?.value as { first: number; stats: PulseStats } | undefined
-    return kept && first - kept.first <= PULSE_LAST_MS ? kept.stats : null
-  }
+  const last = () => lastPulseYear(deps, repoId, first)
   const sha = head.sha
   // What tells whose the year's commits are, kept as read: the agents' pull
   // requests merged (agentMerges), as the pull requests kept tell, read as
   // far back as `whole` (keptPulls), and the commits that release the
-  // repository (`releases`, release-commits.ts releaseCommits).
+  // repository since then (`releases`, release-commits.ts releaseCommits),
+  // as far back as what is counted whole, not the year, a week's reading a
+  // week's releases' commits.
   type Attribution = AgentMerges & { releases?: string[] }
   const keptAttribution = async () => ((await store.read(mergesKey))?.value as Attribution | undefined) ?? null
   const attribution = async (whole: number): Promise<Attribution> => {
     const [pulls, releases] = await Promise.all([keptPulls(deps, repoId, fullName, token, whole),
-      releaseCommits(fullName, token, first, historyChanges(deps, repoId))])
+      releaseCommits(fullName, token, whole, historyChanges(deps, repoId))])
     return { ...agentMerges(pulls, first), releases }
   }
   // Since when every pull request merged is read (repository-pulls.ts
@@ -2947,12 +3062,13 @@ async function pulseCommits(deps: ManagedHttpDeps, repoId: number, fullName: str
   const readTo = (read: Attribution) => pullsReadTo(read)
   // The year counted from its `commits` as `read` tells whose they are and
   // which were pushed directly, the branch followed through those it was
-  // read on past (`passed`, yearCommits) too, whole from when both the
-  // commits and the pull requests merged are read (`from`, none where all
-  // are); kept by its head and as the year last counted.
-  const count = async ({ commits, passed, from }: { commits: HistoryCommit[]; passed: HistoryCommit[]; from: number | null }, read: Attribution) => {
+  // read on past (`passed`, yearCommits) too, whole from when the commits,
+  // the pull requests merged and, where it read them since later than the
+  // year, the releases (`since`) are all read (`from`, none where all are);
+  // kept by its head and as the year last counted.
+  const count = async ({ commits, passed, from }: { commits: HistoryCommit[]; passed: HistoryCommit[]; from: number | null }, read: Attribution, since: number) => {
     const walked = [...commits, ...passed]
-    const whole = Math.max(from ?? Number.NEGATIVE_INFINITY, readTo(read))
+    const whole = Math.max(from ?? Number.NEGATIVE_INFINITY, readTo(read), since > first ? since : Number.NEGATIVE_INFINITY)
     const stats = { ...commitStats(commits, first, agentCommits(read, walked), new Set(read.releases), directCommits(read, walked, sha)),
       ...Number.isFinite(whole) ? { from: whole } : {} }
     await Promise.all([store.write(key, stats, Date.now()), store.write(lastKey, { first, stats }, Date.now())])
@@ -2973,28 +3089,38 @@ async function pulseCommits(deps: ManagedHttpDeps, repoId: number, fullName: str
       const same = kept && sameMerges(read, kept) && read.releases!.join() === (kept.releases ?? []).join() && readTo(kept) <= readTo(read)
       if (readTo(read) >= whole || same) return read
       const year = await yearCommits(fullName, token, sha, first, historyPages(deps, repoId, true), globalThis.fetch, whole)
-      if (year.complete) recounted.stats = await count(year, read)
+      if (year.complete) recounted.stats = await count(year, read, whole)
       return read
-    }, () => true).catch((err: unknown) => { console.warn('managed: pulse attribution read failed:', err) })
+    }, () => true, false, stampOf(deps, repoId, fullName, token, today())).catch((err: unknown) => { console.warn('managed: pulse attribution read failed:', err) })
     return recounted.stats ?? stats
   }
   // A year kept, read whole as far back as the period needs, as it was;
   // else its commits and the pull requests merged read on together as far
-  // as it needs, and counted once both are.
-  const held = (await store.read(key))?.value as PulseStats | { tooMany: true } | undefined
-  if (held && !('tooMany' in held) && (held.from === undefined || held.from < need)) {
+  // as it needs, and counted once both are, how many of its commits are read
+  // told as they are (`progress`); but where they are too many to read
+  // since no later than it needs (kept apart, `within`, one kept before
+  // with the year's own the year's), GitHub's statistics, a week's commits
+  // read where the year's are too many.
+  const tooManyKey = `${key}\0too-many`
+  const [held, tooMany] = await Promise.all([store.read(key), store.read(tooManyKey)]).then(([year, marked]) => [
+    year?.value as PulseStats | { tooMany: true } | undefined, marked?.value as { within?: unknown } | undefined] as const)
+  if (held && !('tooMany' in held) && (held.from === undefined || held.from <= need)) {
     return { stats: await relabelled(held), reading: null, pulls: null, pending: false, kept: head.kept }
   }
-  if (!held || !('tooMany' in held)) {
-    const [year, read] = await Promise.all([yearCommits(fullName, token, sha, first, historyPages(deps, repoId, true), globalThis.fetch, need),
-      attribution(need)])
+  const tooManyWithin = typeof tooMany?.within === 'number' ? tooMany.within : held && 'tooMany' in held ? first : Number.NEGATIVE_INFINITY
+  // One kept with the year's own before, kept apart now, before a shorter
+  // period's count takes its place.
+  if (!tooMany && held && 'tooMany' in held) await store.write(tooManyKey, { within: first }, Date.now())
+  if (need > tooManyWithin) {
+    const [year, read] = await Promise.all([yearCommits(fullName, token, sha, first, historyPages(deps, repoId, true), globalThis.fetch, need,
+      commits => { if (progress) progress.commits = commits }), attribution(need)])
     await store.write(mergesKey, read, Date.now())
     if (year.complete) {
       if (readTo(read) >= need) return { stats: await last(), reading: null, pulls: read.next?.read ?? 0, pending: false, kept: false }
-      return { stats: await count(year, read), reading: null, pulls: null, pending: false, kept: false }
+      return { stats: await count(year, read, need), reading: null, pulls: null, pending: false, kept: false }
     }
     if (!year.tooMany) return { stats: await last(), reading: year.read, pulls: null, pending: false, kept: false }
-    await store.write(key, { tooMany: true }, Date.now())
+    await store.write(tooManyKey, { within: need }, Date.now())
   }
   // Its commits and lines read together, kept as one: never the one GitHub
   // has counted since a push beside the other kept from before it.
@@ -5000,6 +5126,10 @@ export function createManagedRequestHandler(deps: ManagedHttpDeps): Handler {
   // Kept reads' prunes run behind their writes: tracked, so the database
   // outlives them, and on serverless kept alive past the answer.
   ownKeptPrunes(db, prune => { track(prune); deps.waitUntil?.(prune) })
+  // GitHub's tagged answers kept beyond this process, for an instance
+  // starting anew to ask again If-None-Match (github-requests.ts).
+  const tags = githubTagStore<TaggedRead>(db, config.debug)
+  keepTaggedReadsIn({ read: async key => (await tags.read(key))?.value ?? null, write: (key, read) => tags.write(key, read, Date.now()) })
 
   async function route(req: IncomingMessage, res: ServerResponse): Promise<void> {
     const url = new URL(req.url ?? '/', 'http://localhost')

package/server-managed/npm-kept.ts

@@ -153,6 +153,10 @@ export const scanReadStore = <T>(db: UpstreamCacheStore, type: 'listings' | 'loc
 export const REPOSITORY_READ_KEPT_MS = 24 * 60 * 60_000
 export const repositoryReadStore = (db: UpstreamCacheStore, debug = false): KeptStore<object> =>
   keptStore<object>(db, 'repository/reads', REPOSITORY_READ_KEPT_MS, debug)
+// GitHub's answers to GETs that came with an ETag (github-requests.ts
+// taggedRead), by their request's key, for any instance to ask again
+// If-None-Match, kept as long as what was read of a repository is.
+export const githubTagStore = <T>(db: UpstreamCacheStore, debug = false): KeptStore<T> => keptStore<T>(db, 'github/tags', REPOSITORY_READ_KEPT_MS, debug)
 // Pages of a repository's history (repository-history.ts historyFrom), by its
 // id and the commit each starts at, which never change: kept a month after
 // they were last read from GitHub.

package/server-managed/repository-history.ts

@@ -242,8 +242,8 @@ async function repositoryHeads(owner: string, name: string, token: string, fetch
 // Histories read in a GraphQL query, and queries at once: each counts its
 // commits' lines, which GitHub works out as it answers, stopping a query
 // past ten seconds.
-const PAGES_PER_QUERY = 5
-const PAGES_PARALLEL = 5
+export const PAGES_PER_QUERY = 5
+export const PAGES_PARALLEL = 5
 
 // The first page of the history from each of `tops`, by its top, each told
 // to `keep` as it is read, so those read stay kept where another fails: with

package/server-managed/repository-pulse.ts

@@ -16,6 +16,7 @@ import { agentOperated, githubAgent } from '../common/managed/github-agents.ts'
 import { GithubApiError, githubJson } from './github-app.ts'
 import { shareGithubRead } from './github-requests.ts'
 import { coauthorsOf } from './commit-coauthors.ts'
+import { windowsOn } from './history-windows.ts'
 import { ACTOR, type GraphAccount, type HistoryCommit, type HistoryPages, accountOf, actorOf, githubGraphql, historyFrom } from './repository-history.ts'
 
 export const PULSE_WEEKS = [1, 4, 13, 52]
@@ -211,9 +212,10 @@ export async function defaultHead(fullName: string, token: string, fetchImpl: ty
 // before it, a clock behind, the branch's commits since beneath them. Read
 // no further than `need` asks, those since it read whole (`complete`, `from`
 // the newest of those read whose parents are not, every commit after it
-// read; null, the year read whole).
+// read; null, the year read whole). How many are read so far `told` as they
+// are.
 export async function yearCommits(fullName: string, token: string, head: string, since: number, pages: HistoryPages,
-  fetchImpl: typeof fetch = globalThis.fetch, need = since): Promise<YearCommits> {
+  fetchImpl: typeof fetch = globalThis.fetch, need = since, told: (read: number) => void = () => {}): Promise<YearCommits> {
   const found = new Map<string, HistoryCommit>()
   const tried = new Set<string>()
   const within = (commit: HistoryCommit) => commit.committedAt === null || commit.committedAt >= since - YEAR_SKEW_MS
@@ -221,6 +223,7 @@ export async function yearCommits(fullName: string, token: string, head: string,
   const read = () => [...found.values()].filter(dated).length
   const passed = () => [...found.values()].filter(commit => !dated(commit) && within(commit))
   let queries = 0
+  let windowed = false
   let starts = [head]
   while (starts.length > 0) {
     // Those not read yet, the head before any, else the parents of those
@@ -231,11 +234,19 @@ export async function yearCommits(fullName: string, token: string, head: string,
       .filter(commit => commit.parents.some(sha => unread.has(sha))).map(commit => commit.committedAt ?? Number.POSITIVE_INFINITY))
     if (need > since && from < need) return { complete: true, commits: [...found.values()].filter(dated), passed: passed(), from }
     if (queries >= YEAR_QUERIES) return { complete: false, read: read(), tooMany: false }
-    const batch = starts.slice(0, YEAR_STARTS)
+    // The pages kept read first; once past them, the rest read at once by
+    // when its commits were made (history-windows.ts windowsOn), as far back
+    // as it needs, the year's as far as its commits are followed.
+    const next: Awaited<ReturnType<typeof windowsOn>> | null = windowed || found.size === 0 ? null
+      : await windowsOn(fullName, token, head, starts, found, need > since ? need : since - YEAR_SKEW_MS, from, pages, fetchImpl, queries)
+    windowed ||= next?.windowed === true
+    queries += next?.queries ?? 0
+    const batch = (next?.batch ?? starts).slice(0, YEAR_STARTS)
     for (const sha of batch) tried.add(sha)
-    const got = await historyFrom(fullName, token, batch, pages, fetchImpl)
+    const got = batch.length > 0 ? await historyFrom(fullName, token, batch, pages, fetchImpl) : { commits: [], read: 0 }
     if (got.read > 0) queries++
     for (const commit of got.commits) if (!found.has(commit.sha)) found.set(commit.sha, commit)
+    told(read())
     if (read() > YEAR_COMMITS) return { complete: false, read: read(), tooMany: true }
     starts = [...new Set([...found.values()].filter(within).flatMap(commit => commit.parents))].filter(sha => !found.has(sha) && !tried.has(sha))
   }

package/server-managed/repository-stamp.ts

@@ -0,0 +1,41 @@
+// A repository's change stamp: what moves as anything its tabs read of it
+// changes, but the checks its pull requests run, which move no pull
+// request's update time, read in one cheap GraphQL query: when it was last
+// pushed to, which branch is its default, how many branches and tags it
+// has, and how many pull requests and issues it has and when the one
+// updated last was. An answer kept of it
+// is read again behind its readers only where its stamp moved since the
+// answer was read (http.ts keptAnswer).
+import { GithubApiError } from './github-errors.ts'
+import { githubGraphql } from './github-graphql.ts'
+
+// The rules a stamp is read by: its key's, so that a stamp read by others is
+// none's (http.ts stampOf).
+export const STAMP_READ = 2
+
+const STAMP_QUERY = `query($owner: String!, $name: String!) {
+  repository(owner: $owner, name: $name) {
+    pushedAt
+    defaultBranchRef { name }
+    heads: refs(refPrefix: "refs/heads/", first: 1) { totalCount }
+    tags: refs(refPrefix: "refs/tags/", first: 1) { totalCount }
+    pullRequests(first: 1, orderBy: { field: UPDATED_AT, direction: DESC }) { totalCount nodes { updatedAt } }
+    issues(first: 1, orderBy: { field: UPDATED_AT, direction: DESC }) { totalCount nodes { updatedAt } }
+  }
+}`
+type Counted = { totalCount?: unknown; nodes?: ({ updatedAt?: unknown } | null)[] | null } | null
+type Stamped = { repository?: { pushedAt?: unknown; defaultBranchRef?: { name?: unknown } | null; heads?: Counted; tags?: Counted; pullRequests?: Counted; issues?: Counted } | null }
+  | null
+
+// Repository `fullName`'s stamp, read with `token`, as text to compare.
+export async function repositoryStamp(fullName: string, token: string, fetchImpl: typeof fetch = globalThis.fetch): Promise<{ stamp: string }> {
+  const [owner, name] = fullName.split('/') as [string, string]
+  const repository = (await githubGraphql(STAMP_QUERY, { owner, name }, token, fetchImpl) as Stamped)?.repository
+  if (!repository) throw new GithubApiError(404, 'github-not-found')
+  // One missing a count tells nothing: none, never one that would move with nothing.
+  const { heads, tags, pullRequests, issues } = repository
+  if (![heads, tags, pullRequests, issues].every(connection => Number.isSafeInteger(connection?.totalCount))) throw new GithubApiError(502, 'github-malformed')
+  const latest = (connection: Counted | undefined) => [connection?.totalCount, connection?.nodes?.[0]?.updatedAt ?? null]
+  return { stamp: JSON.stringify([repository.pushedAt ?? null, repository.defaultBranchRef?.name ?? null, heads?.totalCount, tags?.totalCount, ...latest(pullRequests),
+    ...latest(issues)]) }
+}

package/server-managed/team-app.ts

@@ -17,6 +17,7 @@
 // findings, and only expanded does it show that they fail to combine.
 import { reportEntries } from '@preventive/report'
 import { type TeamAppMetadata, managedTeamAppMetadata } from '../common/managed/team-app.js'
+import { expandFindingValues } from '../common/managed/finding-values.js'
 import type { BlobStore } from './blob-store.ts'
 import type { ManagedDb, TeamReportAccessSnapshot, UserTeam } from './db.ts'
 import type { WorkspaceShareSnapshot } from './workspace-shares.ts'
@@ -86,8 +87,9 @@ async function classify(db: ManagedDb, store: BlobStore, key: string, team: User
     // A catalog read before a concurrent publication names another report set.
     if (!access?.teamId || access.reports.length !== ids.size || access.reports.some(report => !ids.has(report.id))) return null
     const body = JSON.parse((await loadTeamReportsResponse(db, store, access)).toString('utf8')) as {
-      reports: { id: string; filename: string; data: unknown }[]; links?: string[][]
+      reports: { id: string; filename: string; data: unknown }[]; links?: string[][]; values?: unknown[]
     }
+    for (const report of body.reports) expandFindingValues(report.data, body.values)
     const restricted = access.reports.some(report => !report.permissions.dependencies || !report.permissions.security)
     const empty = restricted ? body.reports.filter(report => !hasContent(report.data)).map(report => report.id) : []
     const links = body.links ?? []