@preventive/triage 1.0.0-alpha.45 → 1.0.0-alpha.47

package/server-managed/db-neon.ts

@@ -8,7 +8,7 @@ import { AsyncLocalStorage } from 'node:async_hooks'
 import { WebSocket } from 'ws'
 import { type ManagedDb, type ManagedDbOptions, createManagedMethods } from './db-methods.ts'
 import { revisionSchema } from './revisions.ts'
-import { MANAGED_SCHEMA, VIEW_SESSION_INDEX } from './db-schema.ts'
+import { MANAGED_SCHEMA, REPO_UPLOAD_INDEXES, VIEW_SESSION_INDEX } from './db-schema.ts'
 import { STORAGE_SCHEMA } from './storage-db.ts'
 import { GITHUB_COMMIT_SCHEMA, GITHUB_METADATA_SCHEMA, GITHUB_STATE_REASON_COLUMN } from './github-metadata.ts'
 import { MANAGED_ISSUE_SCHEMA } from './managed-issues.ts'
@@ -27,18 +27,26 @@ export interface PgConnection {
     rows: Record<string, unknown>[]; rowCount?: number | null
     fields?: { name: string; dataTypeID: number }[]
   }>
-  release(): Promise<void>
+  // After a failure the connection's state is unknown: discard it, never reuse it.
+  release(discard?: boolean): Promise<void>
 }
 export type PgConnect = () => Promise<PgConnection>
 // Content/access mutations cooperate on this transaction-scoped lock. Reads,
 // presence updates, and maintenance leases do not take it. Also fences migrations.
 const LOCK = 'SELECT pg_advisory_xact_lock(1937006964, 1835101793)'
 
+// Whether every version up to the latest is recorded, in one round trip at
+// a cold start: a database with no versions table yet (undefined_table) has
+// none. Outside a transaction, so the failed read aborts nothing.
+const SCHEMA_VERSIONS = 31
 async function currentSchema(db: PgConnection): Promise<boolean> {
-  const exists = (await db.query("SELECT to_regclass('managed_schema_version') AS name")).rows[0]?.['name']
-  if (!exists) return false
-  const versions = new Set((await db.query('SELECT version FROM managed_schema_version')).rows.map(row => Number(row['version'])))
-  return Array.from({ length: 30 }, (_, i) => i + 1).every(version => versions.has(version))
+  try {
+    const { rows } = await db.query(`SELECT count(*) AS n FROM managed_schema_version WHERE version BETWEEN 1 AND ${SCHEMA_VERSIONS}`)
+    return Number(rows[0]?.['n']) === SCHEMA_VERSIONS
+  } catch (error) {
+    if ((error as { code?: unknown })?.code === '42P01') return false
+    throw error
+  }
 }
 
 async function migrateRepositoryDefaultCache(db: PgConnection): Promise<void> {
@@ -169,6 +177,12 @@ async function migrateRepositoryDependencyReviewLockfiles(db: PgConnection): Pro
   await db.query('INSERT INTO managed_schema_version VALUES (30)')
 }
 
+async function migrateRepoUploadIndexes(db: PgConnection): Promise<void> {
+  if ((await db.query('SELECT version FROM managed_schema_version WHERE version = 31')).rows.length > 0) return
+  await db.query(REPO_UPLOAD_INDEXES)
+  await db.query('INSERT INTO managed_schema_version VALUES (31)')
+}
+
 async function initialize(db: PgConnection): Promise<void> {
   if (await currentSchema(db)) return
   await db.query('BEGIN')
@@ -243,7 +257,7 @@ async function initialize(db: PgConnection): Promise<void> {
       await db.query(postgresSchema(revisionSchema(true)) + '; INSERT INTO managed_schema_version VALUES (11)')
     }
     for (const migrate of [migrateRepositoryDefaultCache, migrateBundleBuildLeases, migrateBundleVisibility, migrateLinkReports, migrateRepositoryAliases, migrateHiddenTeams, migrateIssueFixes, migrateBundleProvenance, migrateUpstreamCache, migrateBundleBuildConditions, migrateSessionUploadKeys, migrateViewSessions, migrateGithubCommits, migrateTeamNpmScopes, migrateDependencyReviews, migrateGithubLinks, migrateRepositoryDependencyReviews,
-      migrateRepositoryDependencyReviewLockfiles]) await migrate(db)
+      migrateRepositoryDependencyReviewLockfiles, migrateRepoUploadIndexes]) await migrate(db)
     await db.query('COMMIT')
   } catch (err) {
     await db.query('ROLLBACK')
@@ -276,7 +290,7 @@ function requestConnections(connect: PgConnect) {
       const connection = await connect()
       let result: T
       try { result = await work(connection) }
-      catch (error) { await connection.release().catch(() => {}); throw error }
+      catch (error) { await connection.release(true).catch(() => {}); throw error }
       await connection.release()
       return result
     }
@@ -287,7 +301,7 @@ function requestConnections(connect: PgConnect) {
       try { return await work(connection) }
       catch (error) {
         delete request.connection
-        await connection.release().catch(() => {})
+        await connection.release(true).catch(() => {})
         throw error
       }
     })
@@ -303,7 +317,7 @@ function requestConnections(connect: PgConnect) {
     finally {
       request.ended = true
       await request.queue
-      if (failed) await request.connection?.release().catch(() => {})
+      if (failed) await request.connection?.release(true).catch(() => {})
       else await request.connection?.release()
     }
   }
@@ -313,7 +327,8 @@ function requestConnections(connect: PgConnect) {
 // Exposed for parity tests against real PostgreSQL semantics via PGlite.
 export async function openPostgresManagedDb(connect: PgConnect, options: ManagedDbOptions = {}, reuseConnections = false): Promise<ManagedDb> {
   const initial = await connect()
-  try { await initialize(initial) } finally { await initial.release() }
+  try { await initialize(initial) } catch (err) { await initial.release(true).catch(() => {}); throw err }
+  await initial.release()
   const { withConnection, withRequest } = requestConnections(connect)
   const context = new AsyncLocalStorage<PgConnection>()
   let closed = false
@@ -376,20 +391,52 @@ export async function openPostgresManagedDb(connect: PgConnect, options: Managed
   return methods
 }
 
-export async function openNeonManagedDb(url: string, options: ManagedDbOptions = {}): Promise<ManagedDb> {
-  // Reuse the e2e optional-driver boundary. Requests lease one connection and
-  // close it before returning; no socket must survive a frozen invocation.
-  const { Client, neonConfig } = await import('../server-e2e/neon-driver.ts') as unknown as {
-    Client: new (url: string) => { connect(): Promise<void>; end(): Promise<void>; query: PgConnection['query']; on?(event: 'error', listener: (error: Error) => void): void }
+// Idle connections stay open this long between leases, so a warm instance's
+// requests and feed polls (three seconds apart) skip the TLS, WebSocket and
+// Postgres handshake a new connection repeats.
+const NEON_POOL_IDLE_MS = 5_000
+
+type NeonPoolClient = {
+  query: PgConnection['query']
+  release(discard?: boolean): void
+  on(event: string, listener: (...args: unknown[]) => void): void
+}
+export type NeonPool = {
+  connect(): Promise<NeonPoolClient>
+  end(): Promise<void>
+  on(event: string, listener: (...args: unknown[]) => void): void
+}
+
+// `attachPool` is Vercel's attachDatabasePool on serverless instances: each
+// release keeps the invocation alive until idle connections have closed, so
+// no socket survives a frozen instance.
+export async function openNeonManagedDb(url: string, options: ManagedDbOptions = {}, attachPool?: (pool: NeonPool) => void): Promise<ManagedDb> {
+  // Reuse the e2e optional-driver boundary.
+  const { Pool, neonConfig } = await import('../server-e2e/neon-driver.ts') as unknown as {
+    Pool: new (config: { connectionString: string; max: number; idleTimeoutMillis: number }) => NeonPool
     neonConfig: { webSocketConstructor: typeof WebSocket }
   }
   neonConfig.webSocketConstructor = WebSocket
-  return openPostgresManagedDb(async () => {
-    const client = new Client(url)
-    // A remote disconnect between operations must reject the next query, not
-    // become an unhandled EventEmitter error while the request is reading Blob.
-    client.on?.('error', () => {})
-    try { await client.connect() } catch (err) { await client.end().catch(() => {}); throw err }
-    return { query: (sql, params) => client.query(sql, params), release: () => client.end() }
-  }, options, true)
+  // Leases stay unbounded, one per request or operation as before: a cap would
+  // stall a request that holds one while it awaits work needing another.
+  const pool = new Pool({ connectionString: url, max: Infinity, idleTimeoutMillis: NEON_POOL_IDLE_MS })
+  // The pool evicts an idle connection the server closed. Neither that nor a
+  // remote disconnect while leased may become an unhandled EventEmitter error;
+  // a leased one must instead reject its next query, and is then discarded.
+  pool.on('error', () => {})
+  attachPool?.(pool)
+  const failed = new WeakSet<NeonPoolClient>(), watched = new WeakSet<NeonPoolClient>()
+  let ending: Promise<void> | undefined
+  const end = () => ending ??= pool.end()
+  let db: ManagedDb
+  try {
+    db = await openPostgresManagedDb(async () => {
+      const client = await pool.connect()
+      if (!watched.has(client)) { watched.add(client); client.on('error', () => { failed.add(client) }) }
+      return { query: (sql, params) => client.query(sql, params),
+        release: discard => Promise.try(() => client.release(discard === true || failed.has(client))) }
+    }, options, true)
+  } catch (err) { await end().catch(() => {}); throw err }
+  const close = db.close
+  return { ...db, async close() { try { await close() } finally { await end() } } }
 }

package/server-managed/db-schema.ts

@@ -7,6 +7,10 @@ import { GITHUB_LINK_SCHEMA } from './github-links.ts'
 
 // Created after the column migration, which predates it on older databases.
 export const VIEW_SESSION_INDEX = 'CREATE INDEX IF NOT EXISTS managed_session_viewer_idx ON managed_session(viewer_session);'
+// What a team's scopes join reports and bundles on: their repository, and the
+// directory in it. Created after the column migrations, as above.
+export const REPO_UPLOAD_INDEXES = `CREATE INDEX IF NOT EXISTS managed_report_repo_idx ON managed_report(repo_id, repo_directory);
+CREATE INDEX IF NOT EXISTS managed_bundle_repo_idx ON managed_bundle(repo_id, repo_directory);`
 
 export const MANAGED_SCHEMA = `
 CREATE TABLE IF NOT EXISTS managed_user (

package/server-managed/db.ts

@@ -7,7 +7,7 @@ import { migrateReportLocations } from './report-migration.ts'
 import { initCommentMethods } from './comments.ts'
 import { initActivityMethods } from './activity.ts'
 import { revisionSchema } from './revisions.ts'
-import { MANAGED_SCHEMA, VIEW_SESSION_INDEX } from './db-schema.ts'
+import { MANAGED_SCHEMA, REPO_UPLOAD_INDEXES, VIEW_SESSION_INDEX } from './db-schema.ts'
 import { GITHUB_LINK_USER_COLUMNS } from './github-links.ts'
 import { REPOSITORY_DEPENDENCY_REVIEW_COLUMNS } from './repository-dependency-reviews.ts'
 import { STORAGE_SCHEMA } from './storage-db.ts'
@@ -113,6 +113,7 @@ export function openSqliteManagedDb(path: string, options: ManagedDbOptions = {}
     if (addedVisible) db.prepare('UPDATE managed_report SET visible = 1 WHERE visible = 0').run()
     ensureColumn(db, 'managed_bundle', 'uploaded_by_login', 'TEXT')
     ensureColumn(db, 'managed_bundle', 'repo_directory', "TEXT NOT NULL DEFAULT ''")
+    db.exec(REPO_UPLOAD_INDEXES)
     ensureColumn(db, 'managed_bundle', 'visible', 'INTEGER NOT NULL DEFAULT 1')
     // Existing rows stay NULL (unknown): some may have been built on the server.
     ensureColumn(db, 'managed_bundle', 'provenance', 'TEXT')

package/server-managed/dependency-reviews.ts

@@ -134,13 +134,23 @@ async function expandReviews(db: ManagedSql, rows: unknown[], user: string): Pro
     JOIN managed_dependency_review r ON r.id = rt.review_id JOIN managed_team t ON t.id = rt.team_id JOIN managed_user u ON u.id = ?
     WHERE rt.review_id IN (${placeholders}) AND (r.user_id = u.id OR (${reviewsIn(false)}))
     ORDER BY t.name, t.id`).all(user, ...ids) as (ReviewTeam & { reviewId: string })[]
+  // Each review's own, in their order, by its id.
+  const byReview = <T extends { reviewId: string }>(list: T[]) => {
+    const of = new Map<string, Omit<T, 'reviewId'>[]>()
+    for (const { reviewId, ...row } of list) {
+      const own = of.get(reviewId)
+      if (own) own.push(row)
+      else of.set(reviewId, [row])
+    }
+    return of
+  }
+  const [entriesOf, teamsOf] = [byReview(entries), byReview(linked)]
   return reviews.map(review => {
     const snapshot = review.files === null ? null : JSON.parse(review.files) as string[] | { paths: string[]; nonTextFiles: string[] }
     const files = Array.isArray(snapshot) ? snapshot : snapshot?.paths ?? null
     const nonTextFiles = Array.isArray(snapshot) ? null : snapshot?.nonTextFiles ?? []
-    const ownEntries = fileEntries(entries.filter(entry => entry.reviewId === review.id).map(({ reviewId: _reviewId, ...entry }) => entry))
-    return { ...review, files, nonTextFiles, entries: ownEntries, ...summarizeReview(ownEntries, files, nonTextFiles),
-      teams: linked.filter(team => team.reviewId === review.id).map(({ reviewId: _reviewId, ...team }) => team) }
+    const ownEntries = fileEntries((entriesOf.get(review.id) ?? []) as ReviewEntry[])
+    return { ...review, files, nonTextFiles, entries: ownEntries, ...summarizeReview(ownEntries, files, nonTextFiles), teams: (teamsOf.get(review.id) ?? []) as ReviewTeam[] }
   })
 }
 

package/server-managed/github-app.ts

@@ -7,8 +7,10 @@ import { createHash, createSign } from 'node:crypto'
 import { createClient } from '@preventive/upstream/github.js'
 import type { ManagedConfig } from './config.ts'
 import { GithubApiError, githubUnreachable, throwGithubResponseError, upstreamGithub } from './github-errors.ts'
+import { onTokenRejected, shareGithubRead, tokenRejected } from './github-requests.ts'
 
 export { GithubApiError } from './github-errors.ts'
+export { onTokenRejected } from './github-requests.ts'
 
 const GITHUB_API = 'https://api.github.com'
 const API_VERSION = '2022-11-28'
@@ -22,12 +24,12 @@ interface InstallationTokenEntry { token: string | null; expiresAt: number; pend
 // cached here. Bound memory per transport and separate App/key/install scopes.
 const installationTokens = new WeakMap<typeof fetch, Map<string, InstallationTokenEntry>>()
 
-function discardInstallationToken(token: string | null, fetchImpl: typeof fetch): void {
-  if (!token) return
+// A token GitHub rejected (401): an installation's no longer reused.
+onTokenRejected((token, fetchImpl) => {
   const cache = installationTokens.get(fetchImpl)
   if (!cache) return
   for (const [key, entry] of cache) if (entry.token === token) cache.delete(key)
-}
+})
 
 // One listed repository. Carries the context to read its contents later:
 // `installationId` mints an App installation token (Contents: Read) for a repo
@@ -69,8 +71,12 @@ export function mergeRepos(...lists: ConnectedRepo[][]): ConnectedRepo[] {
 // One GitHub API call returning parsed JSON, optionally Bearer-authed by a
 // user token, App JWT, or installation token, `body` sent as JSON (a GraphQL
 // query's, given longer to answer). Network / non-2xx / malformed fold into a
-// GithubApiError (401 passes through for the caller to handle).
-export async function githubJson(url: string, token: string | null, fetchImpl: typeof fetch, method: 'GET' | 'POST' = 'GET', body?: unknown): Promise<unknown> {
+// GithubApiError (401 passes through for the caller to handle). Identical GETs
+// share an outstanding read through its body. POSTs only opt in for read-only
+// GraphQL queries (repository-history.ts), never for mutations or token mints.
+export async function githubJson(url: string, token: string | null, fetchImpl: typeof fetch, method: 'GET' | 'POST' = 'GET', body?: unknown,
+  readOnly = method === 'GET'): Promise<unknown> {
+  if (readOnly) return shareGithubRead(fetchImpl, JSON.stringify([url, token, method, body]), () => githubJson(url, token, fetchImpl, method, body, false))
   let res: Response
   const started = Date.now()
   try {
@@ -85,7 +91,7 @@ export async function githubJson(url: string, token: string | null, fetchImpl: t
       ...(body === undefined ? {} : { body: JSON.stringify(body) }),
     })
   } catch (error) { throw githubUnreachable(error, started) }
-  if (res.status === 401) discardInstallationToken(token, fetchImpl)
+  if (res.status === 401 && token) tokenRejected(token, fetchImpl)
   if (!res.ok) await throwGithubResponseError(res)
   // The answer's body is read within the same timeout: one not read whole is
   // GitHub not reached, as one not answered is; only one that isn't JSON is

package/server-managed/github-issue-links.ts

@@ -52,6 +52,15 @@ function readPage(job: Job, item: Issue | Pull, metadata: Map<string, Metadata>)
 
 function issueUpdates(jobs: Job[], issues: ManagedIssue[], allowed: Map<string, Repository>, checkedAt: number, metadata: Map<string, Metadata>) {
   const updates: IssueFixUpdate[] = []
+  // The issues each job's issue is, by repository and number, read once.
+  const byIssue = new Map<string, ManagedIssue[]>()
+  for (const issue of issues) {
+    const ref = parseGithubFixUrl(issue.issueUrl)
+    if (ref?.kind !== 'issue') continue
+    const key = `${issue.repoId}:${ref.number}`, same = byIssue.get(key)
+    if (same) same.push(issue)
+    else byIssue.set(key, [issue])
+  }
   for (const job of jobs) {
     if (!job.linked || !job.complete) continue
     // "Last" means the newest PR by creation time, never its latest comment.
@@ -62,11 +71,8 @@ function issueUpdates(jobs: Job[], issues: ManagedIssue[], allowed: Map<string,
     if (latest && (!repo || repo.repoId !== latest.repository.databaseId)) continue
     const next = latest ? `https://github.com/${repo!.github}/pull/${latest.number}` : null
     if (latest && next) metadata.set(next, { url: next, ...itemMetadata(latest, 'pull')! })
-    for (const issue of issues) {
-      const ref = parseGithubFixUrl(issue.issueUrl)
-      if (ref?.kind === 'issue' && ref.number === job.number && issue.repoId === job.repo.repoId) {
-        updates.push({ findingId: issue.findingId, issueUrl: issue.issueUrl!, previous: issue.autoFixUrl, next, checkedAt })
-      }
+    for (const issue of byIssue.get(`${job.repo.repoId}:${job.number}`) ?? []) {
+      updates.push({ findingId: issue.findingId, issueUrl: issue.issueUrl!, previous: issue.autoFixUrl, next, checkedAt })
     }
   }
   return updates

package/server-managed/github-pulls.ts

@@ -222,6 +222,6 @@ export async function lookupFixes(config: ManagedConfig, db: ManagedDb, snapshot
   })
 }
 
-export function storedFixUrls(entries: { fix: string | null }[]): string[] {
-  return [...new Set(entries.flatMap(entry => entry.fix && parseGithubFixUrl(entry.fix) ? [entry.fix] : []))].toSorted()
+export function storedFixUrls(fixes: string[]): string[] {
+  return [...new Set(fixes.filter(fix => parseGithubFixUrl(fix)))].toSorted()
 }

package/server-managed/github-requests.ts

@@ -0,0 +1,39 @@
+import { createHash } from 'node:crypto'
+
+// Only outstanding reads: a completed answer (or failure) is never reused.
+// Keep transports and credentials separate, including anonymous reads. When
+// full, new reads proceed normally rather than waiting for an unrelated one.
+const MAX_PENDING_READS = 1000
+type PendingRead = { promise: Promise<unknown>; shared: boolean }
+const pendingReads = new WeakMap<typeof fetch, Map<string, PendingRead>>()
+
+// Invalidate installation credentials and access verdicts when GitHub rejects
+// a token, including when several callers share the rejected request.
+const rejectedTokenListeners = new Set<(token: string, fetchImpl: typeof fetch) => void>()
+export function onTokenRejected(listener: (token: string, fetchImpl: typeof fetch) => void): void {
+  rejectedTokenListeners.add(listener)
+}
+export function tokenRejected(token: string, fetchImpl: typeof fetch): void {
+  for (const listener of rejectedTokenListeners) listener(token, fetchImpl)
+}
+
+export async function shareGithubRead<T>(fetchImpl: typeof fetch, request: string, read: () => Promise<T>): Promise<T> {
+  let pending = pendingReads.get(fetchImpl)
+  if (!pending) { pending = new Map(); pendingReads.set(fetchImpl, pending) }
+  // The request includes credentials and query variables. Retain only its hash.
+  const key = createHash('sha256').update(request).digest('hex')
+  const held = pending.get(key)
+  if (held) {
+    held.shared = true
+    return structuredClone(await held.promise) as T
+  }
+  if (pending.size >= MAX_PENDING_READS) return read()
+  const entry: PendingRead = { promise: read(), shared: false }
+  pending.set(key, entry)
+  try {
+    const value = await entry.promise
+    // Independent objects for coalesced callers, without copying large trees
+    // or histories when nobody joined the read.
+    return (entry.shared ? structuredClone(value) : value) as T
+  } finally { pending.delete(key) }
+}

package/server-managed/http-response.ts

@@ -15,8 +15,16 @@ export function attachmentDisposition(filename: string): string {
 // already materialized by the authorization/filtering layer before we get here.
 export function writeResponse(res: ServerResponse, body: string | Buffer): void {
   const bytes = typeof body === 'string' ? Buffer.from(body) : body
-  if (bytes.length <= UPLOAD_CHUNK_BYTES) { res.end(body); return }
-  for (let offset = 0; offset < bytes.length; offset += 64 * 1024) res.write(bytes.subarray(offset, offset + 64 * 1024))
+  writeResponseParts(res, [bytes], bytes.length)
+}
+
+// A body of `length` bytes held in parts, written as writeResponse writes
+// one without first copying them into one buffer.
+export function writeResponseParts(res: ServerResponse, parts: Buffer[], length: number): void {
+  if (length <= UPLOAD_CHUNK_BYTES) { res.end(parts.length === 1 ? parts[0] : Buffer.concat(parts, length)); return }
+  for (const part of parts) {
+    for (let offset = 0; offset < part.length; offset += 64 * 1024) res.write(part.subarray(offset, offset + 64 * 1024))
+  }
   res.end()
 }