@preventive/triage 1.0.0-alpha.52 → 1.0.0-alpha.54

package/server-managed/bundle-validation.ts

@@ -0,0 +1,32 @@
+// A Stasis bundle validated (GET and POST /api/bundles/:id/validation) by
+// Stasis's own validation (`@exodus/stasis/validate`; its rules are in
+// Stasis's doc/validate.md), the repository read the app's way: with its
+// reader's GitHub access, own code by git's blob ids of the commit's listing.
+import { type BundleValidation, type ValidationPlan, followingMoves, validateBundle, validateRepository } from '@exodus/stasis/validate'
+import { createClient } from '@preventive/upstream/github.js'
+import { upstreamGithub } from './github-errors.ts'
+
+export { type BundleValidation, type PlannedBundle, validationPlan } from '@exodus/stasis/validate'
+
+// The rules a validation was made by, kept with it: one made by others is made again.
+export const BUNDLE_VALIDATION_READ = 3
+
+type Options = { deadline?: number }
+
+// The bundle `plan`'s dependencies alone, held to their registry, own code unverified for `reason`.
+export const validateDependencies = (plan: ValidationPlan, reason: string, options: Options = {}): Promise<BundleValidation> =>
+  validateBundle(plan, { reason }, options)
+
+// The bundle `plan` validated at `commit` of `fullName`, read with `token`,
+// built in `directory`: own code held to git's blob ids of the commit's tree
+// as upstream lists it (each listing held to its tree's id), the commit to the
+// repository's own history, the dependencies to the lockfiles installing it;
+// what can't be held to told in the report (own code's `reason`,
+// `lockfileError`), not thrown.
+export function validateAtCommit(plan: ValidationPlan, { fullName, token, commit, directory }: { fullName: string; token: string; commit: string; directory: string },
+  options: Options = {}): Promise<BundleValidation> {
+  const client = createClient({ token, userAgent: 'deepview-triage' })
+  const repository = followingMoves(client, fullName)
+  return upstreamGithub(() => validateRepository(plan, { repository, commit, directory },
+    { ...options, strict: false, listTree: paths => client.listRepoTree({ repo: repository.name(), sha: commit, paths }) }))
+}

package/server-managed/db-methods.ts

@@ -17,6 +17,7 @@ import { type ManagedIssue, type ManagedIssueStore, managedIssueMethods } from '
 import type { IssueFixUpdate } from './github-issue-links.ts'
 import { type BundleBuildLeaseStore, bundleBuildLeaseMethods } from './bundle-build-leases.ts'
 import { type UpstreamCacheStore, upstreamCacheMethods } from './upstream-cache.ts'
+import { type TeamAppSummaryStore, teamAppSummaryMethods } from './team-app-summaries.ts'
 import { type ImportTriageStore, importTriageMethods } from './import-triage.ts'
 import type { ManagedSql } from './sql.ts'
 import { type WorkspaceShareStore, workspaceShareMethods } from './workspace-shares.ts'
@@ -147,6 +148,9 @@ export interface TeamReportAccessSnapshot extends ReportAccessSnapshot {
   reportId?: string | null
   hiddenReportId?: string | null
   repositories: { repoId: number; github: string; path: string | null }[]
+  // By report ID, the content key its catalog entry has (UserTeamReport
+  // cacheKey) as read with the rest.
+  catalogKeys?: Readonly<Record<string, string>>
 }
 export interface ReportAccessSnapshot {
   user: StoredUser
@@ -206,6 +210,11 @@ export interface RepoDataItem {
 
 export type RepoReportItem = Pick<ReportRecord, 'id' | 'filename' | 'sha256' | 'bundleId'>
 
+// A report that names a bundle: linked to it (bundleId is that bundle's), or
+// declaring its integrity as the bundle it was made from without the link.
+// `readable`: whether the manager asking may read it (as getReportView tells).
+export type BundleReportItem = Pick<ReportRecord, 'id' | 'filename' | 'bundleId'> & { readable: boolean }
+
 // A stored per-finding triage row, with the last writer's login resolved like
 // listReports (live login, falling back to the durable snapshot). `flagged`
 // maps the tri-state column: null unset, true/false set. Every field null =
@@ -368,7 +377,7 @@ export interface UserTeam {
 }
 
 // Backend-agnostic store surface (SQLite + PostgreSQL implementations).
-export interface ManagedDb extends DependencyReviewStore, RepositoryDependencyReviewStore, ActivityStore, CommentStore, GithubMetadataStore, GithubCommitStore, ManagedIssueStore, BundleBuildLeaseStore, UpstreamCacheStore, WorkspaceShareStore, ImportTriageStore, ManagementStore, ManagementCatalogStore, StorageDb, LinkReportStore, RepositoryAliasStore, TeamNpmScopeStore, GithubLinkStore {
+export interface ManagedDb extends DependencyReviewStore, RepositoryDependencyReviewStore, ActivityStore, CommentStore, GithubMetadataStore, GithubCommitStore, ManagedIssueStore, BundleBuildLeaseStore, UpstreamCacheStore, TeamAppSummaryStore, WorkspaceShareStore, ImportTriageStore, ManagementStore, ManagementCatalogStore, StorageDb, LinkReportStore, RepositoryAliasStore, TeamNpmScopeStore, GithubLinkStore {
   claimMaintenanceLease(owner: string, now: number, until: number, migration?: boolean): Promise<boolean>
   finishMaintenanceLease(owner: string, until: number): Promise<void>
   getFeedState(sessionId: string, now: number): Promise<{ user: Pick<StoredUser, 'id' | 'role'>; catalog: number; annotations: number } | null>
@@ -451,6 +460,10 @@ export interface ManagedDb extends DependencyReviewStore, RepositoryDependencyRe
   getTeamReportAccessSnapshot(sessionId: string, now: number, teamId: string, reportId?: string | null): Promise<TeamReportAccessSnapshot | null>
   getReportAccessSnapshot(sessionId: string, now: number, ids: readonly string[]): Promise<ReportAccessSnapshot | null>
   listReportFilenamesWithBundleHash(bundleId: string, sha256: string): Promise<string[]>
+  // The reports linked to a bundle, or whose bundleHashes declared its integrity
+  // (bundle_integrity), newest first: what deleting it leaves without code.
+  // Each says whether the manager `userId` (none: an admin) may read it.
+  listReportsForBundle(bundleId: string, integrity: string, userId?: string): Promise<BundleReportItem[]>
   deleteReport(id: string): Promise<boolean>
   // Attach / detach a report's repo + directory link (repoId null = detach);
   // resolves true iff the report exists. The caller validates repoId and the
@@ -641,15 +654,23 @@ type UserTeamRows = {
   bundles: (Omit<UserTeamBundle, 'visible'> & { teamId: string; visible: number })[]
 }
 
+// A catalog report's content key: its content, where it is, and what its
+// reader may see of it (their grants and role) under the link reports'
+// revision `links`. A team snapshot names it of each report it reads.
+type CatalogReportRow = Pick<UserTeamRows['reports'][number], 'sha256' | 'bundleId' | 'filename' | 'repoId' | 'repoDirectory' | 'repoFullName' | 'dependencies' | 'security'>
+function reportCatalogKey(r: CatalogReportRow, role: string, links: string): string {
+  return createHash('sha256').update(JSON.stringify([
+    r.sha256, r.bundleId, r.filename, r.repoId, r.repoDirectory, r.repoFullName, r.dependencies, r.security, role, links,
+  ])).digest('base64url')
+}
+
 // A user's team catalog from its rows, `links` the link reports' revision.
 function userTeams(teams: UserTeamRows['teams'], scopes: UserTeamRows['scopes'], links: string, reports: UserTeamRows['reports'], bundles: UserTeamRows['bundles']): UserTeam[] {
   const reportsByTeam = new Map<string, UserTeamReport[]>()
   for (const r of reports) {
     const list = reportsByTeam.get(r.teamId) ?? []
-    const cacheKey = createHash('sha256').update(JSON.stringify([
-      r.sha256, r.bundleId, r.filename, r.repoId, r.repoDirectory, r.repoFullName, r.dependencies, r.security, r.role, links,
-    ])).digest('base64url')
-    list.push({ id: r.id, slug: r.slug, filename: r.filename, visible: r.visible === 1, analyzer: r.analyzer, repoFullName: r.repoFullName, repoDirectory: r.repoDirectory, cacheKey })
+    list.push({ id: r.id, slug: r.slug, filename: r.filename, visible: r.visible === 1, analyzer: r.analyzer, repoFullName: r.repoFullName, repoDirectory: r.repoDirectory,
+      cacheKey: reportCatalogKey(r, r.role, links) })
     reportsByTeam.set(r.teamId, list)
   }
   const bundlesByTeam = new Map<string, UserTeamBundle[]>()
@@ -899,10 +920,10 @@ function prepareStatements(db: ManagedSql) {
               EXISTS(SELECT 1 FROM managed_team_user tu JOIN managed_team t ON t.id = tu.team_id
                       WHERE tu.user_id = v.uid AND tu.team_id = ? AND t.hidden = 0) AS member,
               (SELECT json_group_array(json_object('id', x.id, 'filename', x.filename, 'byteSize', x.byteSize, 'sha256', x.sha256,
-                        'visible', x.visible, 'repoDirectory', x.repoDirectory, 'repoFullName', x.repoFullName,
+                        'visible', x.visible, 'bundleId', x.bundleId, 'repoId', x.repoId, 'repoDirectory', x.repoDirectory, 'repoFullName', x.repoFullName,
                         'dependencies', x.dependencies, 'security', x.security) ORDER BY x.id)
                  FROM (SELECT DISTINCT r.id AS id, r.filename AS filename, r.byte_size AS byteSize, r.sha256 AS sha256, r.visible AS visible,
-                              r.repo_directory AS repoDirectory, sr.full_name AS repoFullName,
+                              r.bundle_id AS bundleId, r.repo_id AS repoId, r.repo_directory AS repoDirectory, sr.full_name AS repoFullName,
                               tu.view_dependencies AS dependencies, tu.view_security AS security
                          FROM managed_team_user tu
                          JOIN managed_team t ON t.id = tu.team_id AND t.hidden = 0
@@ -917,6 +938,15 @@ function prepareStatements(db: ManagedSql) {
          FROM viewer v`,
     ),
     reportFilenamesWithBundleHashStmt: db.prepare(`SELECT DISTINCT filename FROM managed_report WHERE bundle_id = ? AND sha256 = ?`),
+    reportsForBundleStmt: db.prepare(
+      `SELECT r.id AS id, r.filename AS filename, r.bundle_id AS bundleId,
+              (? IS NULL OR r.uploaded_by = ? OR EXISTS(SELECT 1 FROM managed_team_repo tr
+                 JOIN managed_team_user tu ON tu.team_id = tr.team_id
+                 JOIN managed_team t ON t.id = tu.team_id AND t.hidden = 0
+                WHERE tr.repo_id = r.repo_id AND ${REPORT_IN_TEAM_PATH_SQL} AND tu.user_id = ?)) AS readable
+         FROM managed_report r WHERE r.bundle_id = ? OR r.bundle_integrity = ?
+        ORDER BY r.uploaded_at DESC, r.filename ASC, r.id ASC`,
+    ),
     deleteReportStmt: db.prepare(`DELETE FROM managed_report WHERE id = ?`),
     setReportRepoStmt: db.prepare(`UPDATE managed_report SET repo_id = ?, repo_directory = ? WHERE id = ?`),
     setReportVisibleStmt: db.prepare(`UPDATE managed_report SET visible = ? WHERE id = ?`),
@@ -982,7 +1012,7 @@ function prepareStatements(db: ManagedSql) {
                 WHERE c.finding_id IN (SELECT id FROM requested)) AS comments,
               (SELECT json_group_array(json_object('findingId', i.finding_id, 'repoId', i.repo_id, 'repository', i.repository, 'requestId', i.request_id,
                         'issueUrl', i.issue_url, 'autoFixUrl', i.auto_fix_url, 'autoFixCheckedAt', i.auto_fix_checked_at,
-                        'createdBy', i.created_by, 'createdAt', i.created_at) ORDER BY i.finding_id)
+                        'duplicateOfUrl', i.duplicate_of_url, 'createdBy', i.created_by, 'createdAt', i.created_at) ORDER BY i.finding_id)
                  FROM managed_finding_issue i WHERE i.finding_id IN (SELECT id FROM requested)) AS issues`,
     ),
     selectTriageFixesStmt: db.prepare(`SELECT DISTINCT fix FROM managed_finding_triage
@@ -996,7 +1026,8 @@ function prepareStatements(db: ManagedSql) {
                  FROM managed_finding_triage t WHERE t.finding_id IN (SELECT id FROM requested)) AS triage,
               (SELECT json_group_array(json_object('id', c.id, 'version', c.version) ORDER BY c.id)
                  FROM managed_finding_comment c WHERE c.finding_id IN (SELECT id FROM requested)) AS comments,
-              (SELECT json_group_array(json_object('findingId', i.finding_id, 'issueUrl', i.issue_url, 'autoFixUrl', i.auto_fix_url) ORDER BY i.finding_id)
+              (SELECT json_group_array(json_object('findingId', i.finding_id, 'issueUrl', i.issue_url, 'autoFixUrl', i.auto_fix_url,
+                        'duplicateOfUrl', i.duplicate_of_url) ORDER BY i.finding_id)
                  FROM managed_finding_issue i WHERE i.finding_id IN (SELECT id FROM requested) AND i.issue_url IS NOT NULL) AS issues`,
     ),
     insertBundleStmt: db.prepare(
@@ -1368,21 +1399,19 @@ function reportMethods(stmts: ReturnType<typeof prepareStatements>, db: ManagedS
       const user: StoredUser = { id: session.uid, login: session.login, name: session.name, avatarUrl: session.avatar, role: session.role }
       if (user.role === 'none' || !session.member) return { user, teamId: null, reports: [], repositories: [] }
       const whole = user.role === 'admin' || user.role === 'manage'
-      const rows = jsonRows<{
-        id: string; filename: string; byteSize: number; sha256: string; repoDirectory: string
-        repoFullName: string | null; dependencies: number; security: number; visible: number
-      }>(session.reports)
+      const rows = jsonRows<CatalogReportRow & { id: string; byteSize: number; visible: number }>(session.reports)
       // Only privileged callers can receive a hidden row from this query. Give
       // that individual preview its own capacity; published report selections
       // still use the complete workspace for cross-report classification.
       const hidden = rows.find(row => row.id === reportId && row.visible === 0)
       const selected = hidden ? [hidden] : rows
       const repositories = jsonRows<TeamReportAccessSnapshot['repositories'][number]>(session.repositories)
-      return { user, teamId, reportId, hiddenReportId: hidden?.id ?? null, repositories, linkRevision: linkRevisionOf(jsonRows(session.links)), reports: selected.map(row => ({
+      const links = linkRevisionOf(jsonRows(session.links))
+      return { user, teamId, reportId, hiddenReportId: hidden?.id ?? null, repositories, linkRevision: links, reports: selected.map(row => ({
         id: row.id, filename: row.filename, byteSize: row.byteSize, sha256: row.sha256,
         repo: { github: row.repoFullName, directory: row.repoDirectory },
         permissions: { dependencies: whole || row.dependencies === 1, security: whole || row.security === 1 },
-      })) }
+      })), catalogKeys: Object.fromEntries(selected.map(row => [row.id, reportCatalogKey(row, user.role, links)])) }
     },
     async getReportAccessSnapshot(sessionId: string, now: number, ids: readonly string[]): Promise<ReportAccessSnapshot | null> {
       const session = await stmts.selectReportSnapshotStmt.get(sessionId, now, JSON.stringify([...new Set(ids)])) as
@@ -1405,6 +1434,11 @@ function reportMethods(stmts: ReturnType<typeof prepareStatements>, db: ManagedS
       const rows = (await stmts.reportFilenamesWithBundleHashStmt.all(bundleId, sha256)) as { filename: string }[]
       return rows.map(row => row.filename)
     },
+    async listReportsForBundle(bundleId: string, integrity: string, userId?: string): Promise<BundleReportItem[]> {
+      const rows = await stmts.reportsForBundleStmt.all(userId ?? null, userId ?? null, userId ?? null, bundleId, integrity) as
+        (Omit<BundleReportItem, 'readable'> & { readable: unknown })[]
+      return rows.map(row => ({ id: row.id, filename: row.filename, bundleId: row.bundleId, readable: Boolean(row.readable) }))
+    },
     async deleteReport(id: string): Promise<boolean> {
       return Number((await deleteReportStmt.run(id)).changes) > 0
     },
@@ -1800,6 +1834,7 @@ export function createManagedMethods(db: ManagedSql, options: ManagedDbOptions =
     ...repositoryDependencyReviewMethods(db),
     ...bundleBuildLeaseMethods(db),
     ...upstreamCacheMethods(db),
+    ...teamAppSummaryMethods(db),
     async claimMaintenanceLease(owner, now, until, migration = false) {
       return !!await claimLeaseStmt.get(migration ? 2 : 1, owner, until, now)
     },
@@ -1906,7 +1941,7 @@ export function createManagedMethods(db: ManagedSql, options: ManagedDbOptions =
       // Recheck access inside the same write transaction as the derived field.
       const current = await methods.getTeamReportAccessSnapshot(sessionId, Date.now(), snapshot.teamId!, snapshot.reportId)
       if (!current?.teamId || JSON.stringify(current) !== JSON.stringify(snapshot)) return false
-      for (const update of updates) await methods.setManagedIssueAutoFix(update.findingId, update.issueUrl, update.previous, update.next, update.checkedAt)
+      for (const update of updates) await methods.setManagedIssueTracking(update.findingId, update.issueUrl, update.previous, update.next, update.checkedAt)
       return true
     },
     ...workspaceShareMethods(db),

package/server-managed/db-neon.ts

@@ -14,6 +14,7 @@ import { GITHUB_COMMIT_SCHEMA, GITHUB_METADATA_SCHEMA, GITHUB_STATE_REASON_COLUM
 import { MANAGED_ISSUE_SCHEMA } from './managed-issues.ts'
 import { BUNDLE_BUILD_LEASE_SCHEMA } from './bundle-build-leases.ts'
 import { UPSTREAM_CACHE_SCHEMA } from './upstream-cache.ts'
+import { TEAM_APP_SUMMARY_SCHEMA } from './team-app-summaries.ts'
 import { COMMENT_SCHEMA } from './comments.ts'
 import { ACTIVITY_SCHEMA, uploadAction } from './activity.ts'
 import { type ManagedSqlDriver, scopeManagedMethods } from './sql.ts'
@@ -39,7 +40,7 @@ const LOCK = 'SELECT pg_advisory_xact_lock(1937006964, 1835101793)'
 // Whether every version up to the latest is recorded, in one round trip at
 // a cold start: a database with no versions table yet (undefined_table) has
 // none. Outside a transaction, so the failed read aborts nothing.
-const SCHEMA_VERSIONS = 33
+const SCHEMA_VERSIONS = 35
 async function currentSchema(db: PgConnection): Promise<boolean> {
   try {
     const { rows } = await db.query(`SELECT count(*) AS n FROM managed_schema_version WHERE version BETWEEN 1 AND ${SCHEMA_VERSIONS}`)
@@ -202,6 +203,20 @@ async function migrateDependencyReviewPatches(db: PgConnection): Promise<void> {
   await db.query('INSERT INTO managed_schema_version VALUES (33)')
 }
 
+async function migrateTeamAppSummaries(db: PgConnection): Promise<void> {
+  if ((await db.query('SELECT version FROM managed_schema_version WHERE version = 34')).rows.length > 0) return
+  await db.query(postgresSchema(TEAM_APP_SUMMARY_SCHEMA))
+  await db.query('INSERT INTO managed_schema_version VALUES (34)')
+}
+
+// The issue a saved one was closed as a duplicate of, none known of those
+// checked before.
+async function migrateIssueDuplicates(db: PgConnection): Promise<void> {
+  if ((await db.query('SELECT version FROM managed_schema_version WHERE version = 35')).rows.length > 0) return
+  await db.query('ALTER TABLE managed_finding_issue ADD COLUMN IF NOT EXISTS duplicate_of_url TEXT')
+  await db.query('INSERT INTO managed_schema_version VALUES (35)')
+}
+
 async function initialize(db: PgConnection): Promise<void> {
   if (await currentSchema(db)) return
   await db.query('BEGIN')
@@ -276,7 +291,8 @@ async function initialize(db: PgConnection): Promise<void> {
       await db.query(postgresSchema(revisionSchema(true)) + '; INSERT INTO managed_schema_version VALUES (11)')
     }
     for (const migrate of [migrateRepositoryDefaultCache, migrateBundleBuildLeases, migrateBundleVisibility, migrateLinkReports, migrateRepositoryAliases, migrateHiddenTeams, migrateIssueFixes, migrateBundleProvenance, migrateUpstreamCache, migrateBundleBuildConditions, migrateSessionUploadKeys, migrateViewSessions, migrateGithubCommits, migrateTeamNpmScopes, migrateDependencyReviews, migrateGithubLinks, migrateRepositoryDependencyReviews,
-      migrateRepositoryDependencyReviewLockfiles, migrateRepoUploadIndexes, migrateDependencyReviewShasums, migrateDependencyReviewPatches]) await migrate(db)
+      migrateRepositoryDependencyReviewLockfiles, migrateRepoUploadIndexes, migrateDependencyReviewShasums, migrateDependencyReviewPatches, migrateTeamAppSummaries,
+      migrateIssueDuplicates]) await migrate(db)
     await db.query('COMMIT')
   } catch (err) {
     await db.query('ROLLBACK')

package/server-managed/db.ts

@@ -16,6 +16,7 @@ import { GITHUB_COMMIT_SCHEMA, GITHUB_METADATA_SCHEMA, GITHUB_STATE_REASON_COLUM
 import { MANAGED_ISSUE_SCHEMA } from './managed-issues.ts'
 import { BUNDLE_BUILD_LEASE_SCHEMA } from './bundle-build-leases.ts'
 import { UPSTREAM_CACHE_SCHEMA } from './upstream-cache.ts'
+import { TEAM_APP_SUMMARY_SCHEMA } from './team-app-summaries.ts'
 import { managedTableRenames } from './db-table-names.ts'
 import { type ManagedDb, type ManagedDbOptions, createManagedMethods } from './db-methods.ts'
 import { createSqliteDriver, scopeManagedMethods } from './sql.ts'
@@ -69,10 +70,16 @@ export function openSqliteManagedDb(path: string, options: ManagedDbOptions = {}
   const db = new DatabaseSync(path)
   try {
     db.exec('PRAGMA journal_mode = WAL;')
-    db.exec('PRAGMA synchronous = FULL;')
+    // The bootstrap below commits each schema statement on its own: under
+    // FULL, ~120 fsyncs on a new file. Each step checks the schema before
+    // changing it and only reads outside the database, so a step lost to a
+    // power failure reruns on the next open, and WAL stays consistent under
+    // NORMAL. FULL is restored after it; the next commit's fsync of the WAL
+    // persists the bootstrap's frames too.
+    db.exec('PRAGMA synchronous = NORMAL;')
     db.exec('PRAGMA foreign_keys = ON;')
     migrateManagedTableNames(db)
-    db.exec(MANAGED_SCHEMA + GITHUB_METADATA_SCHEMA + GITHUB_COMMIT_SCHEMA + MANAGED_ISSUE_SCHEMA + STORAGE_SCHEMA + BUNDLE_BUILD_LEASE_SCHEMA + UPSTREAM_CACHE_SCHEMA)
+    db.exec(MANAGED_SCHEMA + GITHUB_METADATA_SCHEMA + GITHUB_COMMIT_SCHEMA + MANAGED_ISSUE_SCHEMA + STORAGE_SCHEMA + BUNDLE_BUILD_LEASE_SCHEMA + UPSTREAM_CACHE_SCHEMA + TEAM_APP_SUMMARY_SCHEMA)
     ensureColumn(db, 'managed_selected_repo', 'cached_default_branch', 'TEXT')
     ensureColumn(db, 'managed_session', 'upload_key', 'TEXT')
     ensureColumn(db, 'managed_session', 'viewer_session', 'TEXT REFERENCES managed_session(id) ON DELETE CASCADE')
@@ -123,6 +130,7 @@ export function openSqliteManagedDb(path: string, options: ManagedDbOptions = {}
     ensureColumn(db, 'managed_team', 'dependency_review', 'INTEGER NOT NULL DEFAULT 0')
     ensureColumn(db, 'managed_finding_issue', 'auto_fix_url', 'TEXT')
     ensureColumn(db, 'managed_finding_issue', 'auto_fix_checked_at', 'INTEGER')
+    ensureColumn(db, 'managed_finding_issue', 'duplicate_of_url', 'TEXT')
     ensureColumn(db, 'managed_selected_repo', 'active', 'INTEGER NOT NULL DEFAULT 1')
     for (const [column, type] of REPOSITORY_DEPENDENCY_REVIEW_COLUMNS) ensureColumn(db, 'managed_selected_repo', column, type)
     for (const [column, type] of DEPENDENCY_REVIEW_COLUMNS) ensureColumn(db, 'managed_dependency_review', column, type)
@@ -133,6 +141,7 @@ export function openSqliteManagedDb(path: string, options: ManagedDbOptions = {}
     initCommentMethods(db)
     initActivityMethods(db)
     db.exec(revisionSchema())
+    db.exec('PRAGMA synchronous = FULL;')
   } catch (err) {
     try { db.close() } catch {}
     throw err

package/server-managed/github-issue-links.ts

@@ -1,26 +1,52 @@
-import { type GithubFixResult, githubIssueClosedReason, isGithubRepoName, parseGithubFixUrl } from '../common/github-pr.ts'
+import { type GithubFixResult, githubIssueClosedReason, isGithubRepoName, parseGithubFixUrl, parseGithubIssueUrl } from '../common/github-pr.ts'
 import type { ManagedConfig } from './config.ts'
 import type { ManagedDb, TeamReportAccessSnapshot } from './db.ts'
-import type { ManagedIssue } from './managed-issues.ts'
+import type { IssueTracking, ManagedIssue } from './managed-issues.ts'
 import { ensureUserAccessToken } from './github-oauth.ts'
 import { lookupFixes } from './github-pulls.ts'
 
-export type IssueFixUpdate = { findingId: string; issueUrl: string; previous: string | null; next: string | null; checkedAt: number }
+export type IssueFixUpdate = { findingId: string; issueUrl: string; previous: IssueTracking; next: IssueTracking; checkedAt: number }
 type Pull = { number: number; title: string; body: string | null; state: string; isDraft: boolean; createdAt: string;
   repository: { databaseId: number; nameWithOwner: string } }
 type Connection = { nodes: Pull[]; pageInfo: { hasNextPage: boolean; endCursor: string | null } }
-type Issue = { number: number; title: string; body: string | null; state: string; stateReason: string | null; closedByPullRequestsReferences?: Connection }
+type RepositoryReference = { databaseId?: unknown; nameWithOwner?: unknown } | null | undefined
+type IssueReference = { number?: unknown; repository?: RepositoryReference }
+type Issue = { number: number; title: string; body: string | null; state: string; stateReason: string | null; closedByPullRequestsReferences?: Connection
+  duplicateOf?: IssueReference | null }
 type Repository = TeamReportAccessSnapshot['repositories'][number]
+// The issue a linked one was closed as a duplicate of, in a workspace
+// repository: null for none (or one outside them, never followed or
+// disclosed, as a pull request outside them isn't).
+type Duplicate = { repo: Repository; number: number } | null
+// `duplicateOf` is a linked issue's, from its last page read: undefined while
+// none is, or where GitHub refused the reference.
 type Job = { repo: Repository; kind: 'pull' | 'issue'; number: number; aliases: string[];
-  linked: boolean; cursor: string | null; pulls: Pull[]; complete: boolean }
+  linked: boolean; cursor: string | null; pulls: Pull[]; complete: boolean; duplicateOf?: Duplicate | undefined }
 type Metadata = Exclude<GithubFixResult, { error: string }>
 const pullFields = 'number title body state isDraft createdAt repository { databaseId nameWithOwner }'
+// A linked issue's: the issue it was closed as a duplicate of, and a page of
+// the pull requests linked to it.
+const linkedFields = (cursor: string | null) => 'duplicateOf { number repository { databaseId nameWithOwner } } '
+  + `closedByPullRequestsReferences(first: 100, includeClosedPrs: true, after: ${JSON.stringify(cursor)}) { nodes { ${pullFields} } pageInfo { hasNextPage endCursor } }`
+// How many issues a saved one is followed through, each closed as a
+// duplicate of the next: each is another request after the last.
+const MAX_DUPLICATE_HOPS = 5
+const jobKey = (repoId: number, kind: Job['kind'], number: number) => `${repoId}:${kind}:${number}`
+const issueUrlOf = (repo: Repository, number: number) => `https://github.com/${repo.github}/issues/${number}`
+// Whether GitHub named a repository as one is: by its stable ID and a name.
+const isRepositoryReference = (ref: RepositoryReference): ref is { databaseId: number; nameWithOwner: string } =>
+  Number.isSafeInteger(ref?.databaseId) && typeof ref?.nameWithOwner === 'string' && isGithubRepoName(ref.nameWithOwner)
+// The workspace repository GitHub named, matching its name and stable ID
+// both; none for one outside the workspace repository grants.
+const workspaceRepository = (ref: { databaseId: number; nameWithOwner: string }, allowed: Map<string, Repository>) => {
+  const repo = allowed.get(ref.nameWithOwner.toLowerCase())
+  return repo?.repoId === ref.databaseId ? repo : null
+}
 
 function metadataQuery(jobs: Job[]) {
   return `{ ${jobs.map((job, index) => {
     const [owner, name] = job.repo.github.split('/')
-    const fields = job.kind === 'pull' ? pullFields : `number title body state stateReason ${job.linked
-      ? `closedByPullRequestsReferences(first: 100, includeClosedPrs: true, after: ${JSON.stringify(job.cursor)}) { nodes { ${pullFields} } pageInfo { hasNextPage endCursor } }` : ''}`
+    const fields = job.kind === 'pull' ? pullFields : `number title body state stateReason ${job.linked ? linkedFields(job.cursor) : ''}`
     return `r${index}: repository(owner: ${JSON.stringify(owner)}, name: ${JSON.stringify(name)}) { databaseId nameWithOwner item: ${job.kind === 'pull' ? 'pullRequest' : 'issue'}(number: ${job.number}) { ${fields} } }`
   }).join('\n')} }`
 }
@@ -33,16 +59,32 @@ function itemMetadata(item: Issue | Pull, kind: Job['kind']): Omit<Metadata, 'ur
   return { title: item.title, description: item.body ?? null, status, stateReason }
 }
 
-function readPage(job: Job, item: Issue | Pull, metadata: Map<string, Metadata>): boolean {
+// The issue `issue` was closed as a duplicate of, where GitHub names one
+// (Duplicate); undefined for a reference it gave malformed.
+function duplicateOf(issue: Issue, allowed: Map<string, Repository>): Duplicate | undefined {
+  if (issue.state !== 'CLOSED' || issue.stateReason !== 'DUPLICATE' || issue.duplicateOf == null) return null
+  const { number, repository } = issue.duplicateOf
+  if (!Number.isSafeInteger(number) || (number as number) <= 0 || !isRepositoryReference(repository)) return undefined
+  const repo = workspaceRepository(repository, allowed)
+  return repo ? { repo, number: number as number } : null
+}
+
+// `refused`: GitHub answered with an error on the issue's duplicate alone,
+// which, like a malformed one, leaves it shown but what it redirects to
+// unknown.
+function readPage(job: Job, item: Issue | Pull, metadata: Map<string, Metadata>, allowed: Map<string, Repository>, refused: boolean): boolean {
   const details = itemMetadata(item, job.kind)
   if (!details || item.number !== job.number) return false
   for (const url of job.aliases) metadata.set(url, { url, ...details })
   if (!job.linked) return false
+  const duplicate = refused ? undefined : duplicateOf(item as Issue, allowed)
+  if (duplicate === undefined) return false
   const connection = (item as Issue).closedByPullRequestsReferences
   if (!connection || !Array.isArray(connection.nodes) || typeof connection.pageInfo?.hasNextPage !== 'boolean'
     || connection.nodes.some(pr => !pr || !Number.isSafeInteger(pr.number) || pr.number <= 0 || typeof pr.createdAt !== 'string'
       || !Number.isFinite(Date.parse(pr.createdAt)) || !itemMetadata(pr, 'pull') || typeof pr.isDraft !== 'boolean'
-      || !Number.isSafeInteger(pr.repository?.databaseId) || typeof pr.repository?.nameWithOwner !== 'string' || !isGithubRepoName(pr.repository.nameWithOwner))) return false
+      || !isRepositoryReference(pr.repository))) return false
+  job.duplicateOf = duplicate
   job.pulls.push(...connection.nodes)
   if (!connection.pageInfo.hasNextPage) { job.complete = true; return false }
   if (typeof connection.pageInfo.endCursor !== 'string' || !connection.pageInfo.endCursor || connection.pageInfo.endCursor === job.cursor) return false
@@ -50,43 +92,89 @@ function readPage(job: Job, item: Issue | Pull, metadata: Map<string, Metadata>)
   return true
 }
 
-function issueUpdates(jobs: Job[], issues: ManagedIssue[], allowed: Map<string, Repository>, checkedAt: number, metadata: Map<string, Metadata>) {
-  const updates: IssueFixUpdate[] = []
-  // The issues each job's issue is, by repository and number, read once.
-  const byIssue = new Map<string, ManagedIssue[]>()
-  for (const issue of issues) {
-    const ref = parseGithubFixUrl(issue.issueUrl)
-    if (ref?.kind !== 'issue') continue
-    const key = `${issue.repoId}:${ref.number}`, same = byIssue.get(key)
-    if (same) same.push(issue)
-    else byIssue.set(key, [issue])
+// A saved issue's job, then those of the issues it was closed as a duplicate
+// of in turn, as far as this lookup has read them, a redirect followed rather
+// than the saved issue replaced. `next` is null where the last names none
+// further to follow (or one already in the chain, or past
+// MAX_DUPLICATE_HOPS); the one it names where this lookup has no linked job
+// for it; undefined where one is yet unread, or what it names unknown.
+function duplicateChain(root: Job, jobs: Map<string, Job>): { chain: Job[]; next: Duplicate | undefined } {
+  const chain = [root]
+  for (let job = root; ;) {
+    const target = job.duplicateOf
+    if (!target || chain.length > MAX_DUPLICATE_HOPS) return { chain, next: target && null }
+    const next = jobs.get(jobKey(target.repo.repoId, 'issue', target.number))
+    if (next && chain.includes(next)) return { chain, next: null }
+    if (!next?.linked) return { chain, next: target }
+    chain.push(job = next)
   }
-  for (const job of jobs) {
-    if (!job.linked || !job.complete) continue
+}
+
+// The issues saved ones were closed as duplicates of, which this lookup has
+// yet to read with their linked pull requests: each made a linked job, as a
+// saved issue's own is (one read without them never read further).
+function duplicateJobs(jobs: Map<string, Job>, roots: Iterable<string>): Job[] {
+  const found: Job[] = []
+  for (const key of roots) {
+    const root = jobs.get(key)
+    const next = root && duplicateChain(root, jobs).next
+    if (!next) continue
+    const nextKey = jobKey(next.repo.repoId, 'issue', next.number), url = issueUrlOf(next.repo, next.number)
+    let job = jobs.get(nextKey)
+    if (!job) jobs.set(nextKey, job = { repo: next.repo, kind: 'issue', number: next.number, aliases: [], linked: true, cursor: null, pulls: [], complete: false })
+    job.linked = true
+    if (!job.aliases.includes(url)) job.aliases.push(url)
+    found.push(job)
+  }
+  return found
+}
+
+// `byIssue`: the saved issues each job's issue is, by its key.
+function issueUpdates(jobs: Map<string, Job>, byIssue: Map<string, ManagedIssue[]>, allowed: Map<string, Repository>, checkedAt: number, metadata: Map<string, Metadata>) {
+  const updates: IssueFixUpdate[] = []
+  for (const [key, saved] of byIssue) {
+    const root = jobs.get(key)
+    if (!root?.linked) continue
+    // The saved issue and every one it was closed as a duplicate of in turn,
+    // each read whole: the last is tracked, and the pull requests of all.
+    const { chain, next } = duplicateChain(root, jobs)
+    if (next !== null || !chain.every(job => job.complete)) continue
     // "Last" means the newest PR by creation time, never its latest comment.
-    const latest = job.pulls.filter(pr => pr.state !== 'CLOSED').toSorted((a, b) => Date.parse(b.createdAt) - Date.parse(a.createdAt) || b.number - a.number)[0]
-    const repo = latest && allowed.get(latest.repository.nameWithOwner.toLowerCase())
+    const latest = chain.flatMap(job => job.pulls).filter(pr => pr.state !== 'CLOSED')
+      .toSorted((a, b) => Date.parse(b.createdAt) - Date.parse(a.createdAt) || b.number - a.number)[0]
+    const repo = latest && workspaceRepository(latest.repository, allowed)
     // The viewer's token can have broader access than this workspace. Never
-    // persist or disclose a PR from outside the workspace repository grants.
-    if (latest && (!repo || repo.repoId !== latest.repository.databaseId)) continue
-    const next = latest ? `https://github.com/${repo!.github}/pull/${latest.number}` : null
-    if (latest && next) metadata.set(next, { url: next, ...itemMetadata(latest, 'pull')! })
-    // Only a changed derived Fix is written: an unchanged one's check time
-    // alone would bump the annotations revision that every open team feed
-    // polls, for nothing they show. Changes still stamp it, so an older lookup
-    // never replaces a newer change (managed-issues.ts). An older lookup that
-    // differs can now land after a newer unchanged one, as it could already
-    // land before it (whose compare-and-set then fails); either way the next
-    // lookup finds the stored Fix differs, and writes the current one.
-    for (const issue of byIssue.get(`${job.repo.repoId}:${job.number}`) ?? []) {
-      if (issue.autoFixUrl !== next) updates.push({ findingId: issue.findingId, issueUrl: issue.issueUrl!, previous: issue.autoFixUrl, next, checkedAt })
+    // persist or disclose a PR from outside the workspace repository grants:
+    // where the newest is one, the saved Fix stays. The duplicate, checked
+    // against the grants itself, is saved all the same.
+    const foreign = Boolean(latest && !repo)
+    const autoFix = latest && repo ? `https://github.com/${repo.github}/pull/${latest.number}` : null
+    if (latest && autoFix) metadata.set(autoFix, { url: autoFix, ...itemMetadata(latest, 'pull')! })
+    const last = chain.at(-1)!
+    const duplicate = chain.length > 1 ? issueUrlOf(last.repo, last.number) : null
+    // Only a changed derived Fix or duplicate is written: an unchanged one's
+    // check time alone would bump the annotations revision that every open
+    // team feed polls, for nothing they show. Changes still stamp it, so an
+    // older lookup never replaces a newer change (managed-issues.ts). An older
+    // lookup that differs can now land after a newer unchanged one, as it
+    // could already land before it (whose compare-and-set then fails); either
+    // way the next lookup finds the stored values differ, and writes the
+    // current ones.
+    for (const issue of saved) {
+      const tracking = { autoFix: foreign ? issue.autoFixUrl : autoFix, duplicateOf: duplicate }
+      if (issue.autoFixUrl !== tracking.autoFix || issue.duplicateOfUrl !== tracking.duplicateOf) {
+        updates.push({ findingId: issue.findingId, issueUrl: issue.issueUrl!, checkedAt,
+          previous: { autoFix: issue.autoFixUrl, duplicateOf: issue.duplicateOfUrl }, next: tracking })
+      }
     }
   }
   return updates
 }
 
 // Fetch issue details, linked PR states and manual Fix metadata in one batch.
-// Pagination and workspaces with over 200 distinct links need further batches.
+// Pagination, workspaces with over 200 distinct links, and issues closed as a
+// duplicate of one not yet read need further batches. A duplicate saved last
+// time is read in the first, as a redirect kept is: its URL among `urls`.
 // Live GraphQL results prove this viewer's access; never publish them into the
 // REST shared cache under an older public-visibility grant.
 export async function lookupIssueLinks(config: ManagedConfig, db: ManagedDb, snapshot: TeamReportAccessSnapshot,
@@ -94,14 +182,22 @@ export async function lookupIssueLinks(config: ManagedConfig, db: ManagedDb, sna
   const checkedAt = Date.now(), signal = AbortSignal.timeout(10_000)
   const allowed = new Map(snapshot.repositories.filter(repo => isGithubRepoName(repo.github)).map(repo => [repo.github.toLowerCase(), repo]))
   const jobs = new Map<string, Job>()
-  const linked = new Set(issues.filter(issue => issue.issueUrl).map(issue => {
-    const ref = parseGithubFixUrl(issue.issueUrl)
-    return ref?.kind === 'issue' ? `${issue.repoId}:issue:${ref.number}` : ''
-  }))
+  // The saved issues by their issue's key, and the issues read with their
+  // linked pull requests: theirs, and the duplicates they were saved with.
+  const byIssue = new Map<string, ManagedIssue[]>(), linked = new Set<string>()
+  for (const issue of issues) {
+    const ref = parseGithubIssueUrl(issue.issueUrl)
+    if (!ref) continue
+    const key = jobKey(issue.repoId, 'issue', ref.number)
+    byIssue.set(key, [...byIssue.get(key) ?? [], issue])
+    linked.add(key)
+    const duplicate = parseGithubIssueUrl(issue.duplicateOfUrl), repo = duplicate && allowed.get(duplicate.repo.toLowerCase())
+    if (duplicate && repo) linked.add(jobKey(repo.repoId, 'issue', duplicate.number))
+  }
   for (const url of urls) {
     const ref = parseGithubFixUrl(url), repo = ref && allowed.get(ref.repo.toLowerCase())
     if (!ref || !repo) continue
-    const key = `${repo.repoId}:${ref.kind}:${ref.number}`, previous = jobs.get(key)
+    const key = jobKey(repo.repoId, ref.kind, ref.number), previous = jobs.get(key)
     if (previous) { previous.aliases.push(url); continue }
     jobs.set(key, { repo, kind: ref.kind, number: ref.number, aliases: [url], linked: linked.has(key), cursor: null, pulls: [], complete: false })
   }
@@ -109,11 +205,12 @@ export async function lookupIssueLinks(config: ManagedConfig, db: ManagedDb, sna
   const withinDeadline: typeof fetch = (input, init) => { signal.throwIfAborted(); return fetchImpl(input, { ...init, signal }) }
   const token = await ensureUserAccessToken(config, db, snapshot.user.id, checkedAt, withinDeadline)
   if (!token) return { fixes: await lookupFixes(config, db, snapshot, urls, withinDeadline), updates: [] }
-  const metadata = new Map<string, Metadata>(), selected = [...jobs.values()]
-  let pending = selected
+  const metadata = new Map<string, Metadata>()
+  let pending = [...jobs.values()]
   for (let page = 0; pending.length > 0 && page < 100 && !signal.aborted; page++) {
     const batch = pending.slice(0, 200), query = metadataQuery(batch)
     let data: Record<string, { databaseId: number; nameWithOwner: string; item: Issue | Pull } | null>
+    const refused = new Set<string>()
     try {
       const response = await withinDeadline('https://api.github.com/graphql', {
         method: 'POST', redirect: 'error',
@@ -123,18 +220,25 @@ export async function lookupIssueLinks(config: ManagedConfig, db: ManagedDb, sna
       const body = await response.json() as { data?: typeof data; errors?: { path?: unknown[] }[] }
       // Errors attached to an alias invalidate that whole job, including its
       // connection. Other aliases can succeed even if one issue was deleted.
+      // One on the issue it duplicates alone, which the viewer may be unable
+      // to read, leaves its status, and only what it redirects to unknown.
       if (!body?.data || body.errors != null && (!Array.isArray(body.errors)
         || body.errors.some(error => typeof error?.path?.[0] !== 'string' || !/^r\d+$/u.test(error.path[0])))) break
       data = body.data
-      for (const error of body.errors ?? []) data[error.path![0] as string] = null
+      for (const error of body.errors ?? []) {
+        const [alias, field, reference] = error.path! as string[]
+        if (field === 'item' && reference === 'duplicateOf') refused.add(alias!)
+        else data[alias!] = null
+      }
     } catch { break }
     pending = pending.slice(batch.length)
     for (const [index, job] of batch.entries()) {
       const repository = data[`r${index}`]
       if (repository?.databaseId !== job.repo.repoId || typeof repository.nameWithOwner !== 'string' || repository.nameWithOwner.toLowerCase() !== job.repo.github.toLowerCase()) continue
-      if (readPage(job, repository.item, metadata)) pending.push(job)
+      if (readPage(job, repository.item, metadata, allowed, refused.has(`r${index}`))) pending.push(job)
     }
+    for (const job of duplicateJobs(jobs, byIssue.keys())) if (!pending.includes(job)) pending.push(job)
   }
-  const updates = issueUpdates(selected, issues, allowed, checkedAt, metadata)
+  const updates = issueUpdates(jobs, byIssue, allowed, checkedAt, metadata)
   return { fixes: [...metadata.values()], updates }
 }