@preventive/triage 1.0.0-alpha.27 → 1.0.0-alpha.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE-APACHE +202 -0
- package/client/linked-findings.js +1 -1
- package/common/bundle-metadata.js +12 -10
- package/common/code-language.js +17 -2
- package/common/default-scan-models.ts +3 -3
- package/common/finding-id.js +1 -1
- package/common/managed/report-content.ts +1 -1
- package/common/managed/report-filter.ts +1 -1
- package/out/client-managed-import.js +16 -16
- package/out/client-managed.js +93 -68
- package/out/graph.js +11 -11
- package/out/index.html +1 -1
- package/out/prism.js +2 -2
- package/out/stasis.svg +0 -25
- package/out/terminal.js +163 -163
- package/out/view.css +1 -1
- package/out/view.js +223 -190
- package/package.json +18 -28
- package/server-e2e/static.ts +3 -4
- package/server-managed/bundle-advisories.ts +53 -0
- package/server-managed/bundle-advisory-inventory.ts +75 -0
- package/server-managed/bundle-build-leases.ts +41 -0
- package/server-managed/bundle-build-worker.js +44 -0
- package/server-managed/bundle-build.ts +141 -0
- package/server-managed/bundle-cache.ts +22 -13
- package/server-managed/bundle-summary-cache.ts +27 -9
- package/server-managed/bundle.ts +66 -1
- package/server-managed/cache-storage.ts +4 -0
- package/server-managed/content-access.ts +1 -1
- package/server-managed/db-methods.ts +122 -20
- package/server-managed/db-neon.ts +121 -32
- package/server-managed/db-schema.ts +2 -0
- package/server-managed/db.ts +5 -1
- package/server-managed/github-app.ts +10 -15
- package/server-managed/github-errors.ts +21 -0
- package/server-managed/github-issues.ts +1 -1
- package/server-managed/github-metadata.ts +34 -1
- package/server-managed/github-pulls.ts +142 -30
- package/server-managed/http.ts +244 -131
- package/server-managed/index.ts +25 -9
- package/server-managed/management-catalog.ts +45 -0
- package/server-managed/object-storage-vercel.ts +1 -1
- package/server-managed/public-workspace.ts +20 -14
- package/server-managed/report-migration.ts +1 -1
- package/server-managed/report-sources.ts +14 -4
- package/server-managed/repository-browser.ts +3 -0
- package/server-managed/repository-discovery.ts +1 -1
- package/server-managed/revisions.ts +35 -0
- package/server-managed/sql-postgres.ts +1 -0
- package/server-managed/sql.ts +14 -3
- package/server-managed/storage-db.ts +12 -5
- package/server-managed/storage-maintenance.ts +37 -22
- package/server-managed/storage-payload.ts +16 -4
- package/server-managed/storage-stores.ts +2 -2
- package/server-managed/storage.ts +6 -4
- package/server-managed/team-feed.ts +50 -5
- package/server-managed/team-reports.ts +85 -22
- package/server-managed/workspace-shares.ts +15 -3
- package/report/index.js +0 -260
- package/report/src/finding-id.js +0 -80
- package/report/src/finding.js +0 -314
- package/report/src/labels.js +0 -35
- package/report/src/md-structure.js +0 -471
- package/report/src/md-text.js +0 -167
- package/report/src/meta.js +0 -76
- package/report/src/parse-codex.js +0 -149
- package/report/src/parse-deepsec.js +0 -199
- package/report/src/parse-deepview-fields.js +0 -375
- package/report/src/parse-deepview-md.js +0 -186
- package/report/src/parse-md-id.js +0 -137
- package/report/src/parse-md.js +0 -323
- package/report/src/parse-piolium-id.js +0 -79
- package/report/src/parse-piolium-rows.js +0 -126
- package/report/src/parse-piolium-tokens.js +0 -185
- package/report/src/parse-piolium.js +0 -399
- package/report/src/security.js +0 -63
- package/report/src/severity.js +0 -18
- package/report/src/utf8.js +0 -21
- package/report/src/write-md-finding.js +0 -273
- package/report/src/write-md.js +0 -291
- /package/{LICENSE → LICENSE-MIT} +0 -0
|
@@ -45,6 +45,8 @@ CREATE TABLE IF NOT EXISTS managed_selected_repo (
|
|
|
45
45
|
is_private INTEGER NOT NULL,
|
|
46
46
|
installation_id INTEGER,
|
|
47
47
|
default_branch TEXT NOT NULL,
|
|
48
|
+
-- Live browser hint; independent of the default discovered at selection.
|
|
49
|
+
cached_default_branch TEXT,
|
|
48
50
|
html_url TEXT NOT NULL,
|
|
49
51
|
added_by TEXT REFERENCES managed_user(id) ON DELETE SET NULL,
|
|
50
52
|
added_at INTEGER NOT NULL,
|
package/server-managed/db.ts
CHANGED
|
@@ -6,10 +6,12 @@ import { migrateSlugs } from './slugs.ts'
|
|
|
6
6
|
import { migrateReportLocations } from './report-migration.ts'
|
|
7
7
|
import { initCommentMethods } from './comments.ts'
|
|
8
8
|
import { initActivityMethods } from './activity.ts'
|
|
9
|
+
import { revisionSchema } from './revisions.ts'
|
|
9
10
|
import { MANAGED_SCHEMA } from './db-schema.ts'
|
|
10
11
|
import { STORAGE_SCHEMA } from './storage-db.ts'
|
|
11
12
|
import { GITHUB_METADATA_SCHEMA, GITHUB_STATE_REASON_COLUMN } from './github-metadata.ts'
|
|
12
13
|
import { MANAGED_ISSUE_SCHEMA } from './managed-issues.ts'
|
|
14
|
+
import { BUNDLE_BUILD_LEASE_SCHEMA } from './bundle-build-leases.ts'
|
|
13
15
|
import { managedTableRenames } from './db-table-names.ts'
|
|
14
16
|
import { type ManagedDb, type ManagedDbOptions, createManagedMethods } from './db-methods.ts'
|
|
15
17
|
import { createSqliteDriver, scopeManagedMethods } from './sql.ts'
|
|
@@ -66,7 +68,8 @@ export function openSqliteManagedDb(path: string, options: ManagedDbOptions = {}
|
|
|
66
68
|
db.exec('PRAGMA synchronous = FULL;')
|
|
67
69
|
db.exec('PRAGMA foreign_keys = ON;')
|
|
68
70
|
migrateManagedTableNames(db)
|
|
69
|
-
db.exec(MANAGED_SCHEMA + GITHUB_METADATA_SCHEMA + MANAGED_ISSUE_SCHEMA + STORAGE_SCHEMA)
|
|
71
|
+
db.exec(MANAGED_SCHEMA + GITHUB_METADATA_SCHEMA + MANAGED_ISSUE_SCHEMA + STORAGE_SCHEMA + BUNDLE_BUILD_LEASE_SCHEMA)
|
|
72
|
+
ensureColumn(db, 'managed_selected_repo', 'cached_default_branch', 'TEXT')
|
|
70
73
|
for (const table of ['managed_report', 'managed_bundle']) {
|
|
71
74
|
ensureColumn(db, table, 'data_key', 'TEXT')
|
|
72
75
|
ensureColumn(db, table, 'storage_encrypted', 'INTEGER NOT NULL DEFAULT 0')
|
|
@@ -106,6 +109,7 @@ export function openSqliteManagedDb(path: string, options: ManagedDbOptions = {}
|
|
|
106
109
|
ensureColumn(db, 'managed_finding_triage_event', 'repo', 'TEXT')
|
|
107
110
|
initCommentMethods(db)
|
|
108
111
|
initActivityMethods(db)
|
|
112
|
+
db.exec(revisionSchema())
|
|
109
113
|
} catch (err) {
|
|
110
114
|
try { db.close() } catch {}
|
|
111
115
|
throw err
|
|
@@ -4,8 +4,9 @@
|
|
|
4
4
|
// permissions. Discovery itself is read-only and skips archived repositories.
|
|
5
5
|
import { Buffer } from 'node:buffer'
|
|
6
6
|
import { createHash, createSign } from 'node:crypto'
|
|
7
|
+
import { createClient } from '@preventive/upstream/github.js'
|
|
7
8
|
import type { ManagedConfig } from './config.ts'
|
|
8
|
-
import { GithubApiError, throwGithubResponseError } from './github-errors.ts'
|
|
9
|
+
import { GithubApiError, throwGithubResponseError, upstreamGithub } from './github-errors.ts'
|
|
9
10
|
|
|
10
11
|
export { GithubApiError } from './github-errors.ts'
|
|
11
12
|
|
|
@@ -124,10 +125,9 @@ export function publicRepositoryName(raw: unknown): string | null {
|
|
|
124
125
|
// Public additions must be readable by the server without the acting user's
|
|
125
126
|
// credentials. Require explicit public visibility and use GitHub's canonical
|
|
126
127
|
// metadata, never client-supplied ids, installation ids, or branches.
|
|
127
|
-
export async function fetchPublicRepository(fullName: string
|
|
128
|
+
export async function fetchPublicRepository(fullName: string): Promise<ConnectedRepo> {
|
|
128
129
|
if (publicRepositoryName(fullName) !== fullName) throw new GithubApiError(400, 'bad-repository')
|
|
129
|
-
const
|
|
130
|
-
const repo = parseRepo(await githubJson(`${GITHUB_API}/repos/${path}`, null, fetchImpl), null)
|
|
130
|
+
const repo = parseRepo(await upstreamGithub(() => createClient({ token: null, userAgent: USER_AGENT }).getRepo({ repo: fullName })), null)
|
|
131
131
|
if (repo == null || repo.private || repo.visibility !== 'public') throw new GithubApiError(409, 'repo-not-public')
|
|
132
132
|
if (publicRepositoryName(repo.fullName) !== repo.fullName || repo.fullName.toLowerCase() !== fullName.toLowerCase()) throw new GithubApiError(502, 'github-malformed')
|
|
133
133
|
return { ...repo, htmlUrl: `https://github.com/${repo.fullName}` }
|
|
@@ -138,17 +138,12 @@ export async function fetchPublicRepository(fullName: string, fetchImpl: typeof
|
|
|
138
138
|
// List the authenticated user's repositories (GET /user/repos), paginated until
|
|
139
139
|
// a short page, deduped + sorted. With an identity-only token this returns the
|
|
140
140
|
// user's PUBLIC repos. READ-ONLY.
|
|
141
|
-
export async function listUserRepos(accessToken: string
|
|
141
|
+
export async function listUserRepos(accessToken: string): Promise<ConnectedRepo[]> {
|
|
142
142
|
const byName = new Map<string, ConnectedRepo>()
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
const
|
|
146
|
-
if (
|
|
147
|
-
for (const r of body) {
|
|
148
|
-
const repo = parseRepo(r, null)
|
|
149
|
-
if (repo != null) byName.set(repo.fullName, repo)
|
|
150
|
-
}
|
|
151
|
-
if (body.length < PER_PAGE) break
|
|
143
|
+
const body = await upstreamGithub(() => createClient({ token: accessToken, userAgent: USER_AGENT }).listUserRepos())
|
|
144
|
+
for (const r of body) {
|
|
145
|
+
const repo = parseRepo(r, null)
|
|
146
|
+
if (repo != null) byName.set(repo.fullName, repo)
|
|
152
147
|
}
|
|
153
148
|
return [...byName.values()].toSorted((a, b) => a.fullName.localeCompare(b.fullName))
|
|
154
149
|
}
|
|
@@ -373,7 +368,7 @@ export async function collectRepos(config: ManagedConfig, userToken: string | nu
|
|
|
373
368
|
const lists: ConnectedRepo[][] = []
|
|
374
369
|
if (userToken != null) {
|
|
375
370
|
try {
|
|
376
|
-
lists.push(await listUserRepos(userToken
|
|
371
|
+
lists.push(await listUserRepos(userToken))
|
|
377
372
|
} catch (err) {
|
|
378
373
|
if (err instanceof GithubApiError && err.status === 401) tokenMissing = true
|
|
379
374
|
else console.warn('managed: public repo list failed:', err)
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { HttpError } from '@preventive/upstream/github.js'
|
|
2
|
+
|
|
1
3
|
// A failure carrying the HTTP status the router should surface. 401 passes
|
|
2
4
|
// through so the user-token path can map it to "log in again".
|
|
3
5
|
export class GithubApiError extends Error {
|
|
@@ -26,3 +28,22 @@ export async function throwGithubResponseError(res: Response): Promise<never> {
|
|
|
26
28
|
if (res.status === 404) throw new GithubApiError(404, 'github-not-found')
|
|
27
29
|
throw new GithubApiError(502, `github-status-${res.status}`)
|
|
28
30
|
}
|
|
31
|
+
|
|
32
|
+
export async function upstreamGithub<T>(request: () => Promise<T>): Promise<T> {
|
|
33
|
+
try { return await request() } catch (error) {
|
|
34
|
+
if (error instanceof HttpError) {
|
|
35
|
+
// The upstream error includes GitHub's response text. Retain managed
|
|
36
|
+
// status handling, including rate limits, without exposing that text.
|
|
37
|
+
if (error.status === 401) throw new GithubApiError(401, 'github-unauthorized')
|
|
38
|
+
if (error.status === 404) throw new GithubApiError(404, 'github-not-found')
|
|
39
|
+
if (error.status === 429 || (error.status === 403 && /rate limit|abuse detection/iu.test(error.message))) {
|
|
40
|
+
throw new GithubApiError(429, 'github-rate-limited', 60)
|
|
41
|
+
}
|
|
42
|
+
throw new GithubApiError(502, `github-status-${error.status}`)
|
|
43
|
+
}
|
|
44
|
+
if (error instanceof Error && (error.name === 'AssertionError' || error.cause instanceof SyntaxError)) {
|
|
45
|
+
throw new GithubApiError(502, 'github-malformed')
|
|
46
|
+
}
|
|
47
|
+
throw new GithubApiError(502, 'github-unreachable')
|
|
48
|
+
}
|
|
49
|
+
}
|
|
@@ -4,7 +4,7 @@ import type { BlobStore } from './blob-store.ts'
|
|
|
4
4
|
import type { ManagedIssue } from './managed-issues.ts'
|
|
5
5
|
import { randomToken } from './crypto.ts'
|
|
6
6
|
import { githubIssueClosedReason } from '../common/github-pr.ts'
|
|
7
|
-
import { reportEntries, reportRepoGithub } from '
|
|
7
|
+
import { reportEntries, reportRepoGithub } from '@preventive/report'
|
|
8
8
|
import { newIssueLabels } from '../common/github-issue-labels.js'
|
|
9
9
|
import { appJwt, githubAppConfigured, installUrl, publicRepositoryName } from './github-app.ts'
|
|
10
10
|
import { ISSUE_LOGIN_PATH, issueUserToken } from './github-issue-oauth.ts'
|
|
@@ -12,15 +12,24 @@ export interface GithubMetadata {
|
|
|
12
12
|
fetchedAt: number
|
|
13
13
|
attemptedAt: number | null
|
|
14
14
|
}
|
|
15
|
+
export interface GithubRepositoryVisibility {
|
|
16
|
+
repoId: number
|
|
17
|
+
github: string
|
|
18
|
+
public: boolean
|
|
19
|
+
checkedAt: number
|
|
20
|
+
}
|
|
15
21
|
export interface GithubMetadataStore {
|
|
16
22
|
listGithubMetadata(keys: readonly string[]): Promise<GithubMetadata[]>
|
|
17
23
|
setGithubMetadata(entries: readonly Omit<GithubMetadata, 'attemptedAt'>[]): Promise<void>
|
|
18
24
|
recordGithubMetadataAttempts(keys: readonly string[], attemptedAt: number): Promise<void>
|
|
25
|
+
listGithubRepositoryVisibility(repoIds: readonly number[]): Promise<GithubRepositoryVisibility[]>
|
|
26
|
+
setGithubRepositoryVisibility(entries: readonly GithubRepositoryVisibility[]): Promise<void>
|
|
19
27
|
}
|
|
20
28
|
|
|
21
29
|
// Successful GitHub reads are shared and retained without eviction. The stable
|
|
22
30
|
// repository ID in the key survives renames, but is never an access grant.
|
|
23
|
-
// Callers must
|
|
31
|
+
// Callers must verify current user -> team -> repo grants plus either recently
|
|
32
|
+
// verified public visibility or the viewer's live GitHub access with their token.
|
|
24
33
|
export const GITHUB_STATE_REASON_COLUMN = "TEXT CHECK (state_reason IN ('completed', 'not_planned', 'duplicate', 'unknown'))"
|
|
25
34
|
export const GITHUB_METADATA_SCHEMA = `
|
|
26
35
|
CREATE TABLE IF NOT EXISTS managed_github_metadata (
|
|
@@ -32,6 +41,12 @@ CREATE TABLE IF NOT EXISTS managed_github_metadata (
|
|
|
32
41
|
fetched_at INTEGER NOT NULL,
|
|
33
42
|
attempted_at INTEGER
|
|
34
43
|
) STRICT;
|
|
44
|
+
CREATE TABLE IF NOT EXISTS managed_github_repository_visibility (
|
|
45
|
+
repo_id INTEGER PRIMARY KEY,
|
|
46
|
+
github TEXT NOT NULL,
|
|
47
|
+
is_public INTEGER NOT NULL CHECK (is_public IN (0, 1)),
|
|
48
|
+
checked_at INTEGER NOT NULL
|
|
49
|
+
) STRICT;
|
|
35
50
|
`
|
|
36
51
|
|
|
37
52
|
export function githubMetadataMethods(db: ManagedSql): GithubMetadataStore {
|
|
@@ -49,6 +64,16 @@ export function githubMetadataMethods(db: ManagedSql): GithubMetadataStore {
|
|
|
49
64
|
// The monotonic guard preserves a newer attempt recorded by another reader.
|
|
50
65
|
const attempt = db.prepare(`UPDATE managed_github_metadata SET attempted_at = ?
|
|
51
66
|
WHERE cache_key IN (SELECT value FROM json_each(?)) AND (attempted_at IS NULL OR attempted_at < ?)`)
|
|
67
|
+
const visibility = db.prepare(`SELECT repo_id AS repoId, github, is_public AS public, checked_at AS checkedAt
|
|
68
|
+
FROM managed_github_repository_visibility WHERE repo_id IN (SELECT CAST(value AS BIGINT) FROM json_each(?))`)
|
|
69
|
+
const setVisibility = db.prepare(`INSERT INTO managed_github_repository_visibility (repo_id, github, is_public, checked_at)
|
|
70
|
+
SELECT CAST(json_extract(e.value, '$.repoId') AS BIGINT), json_extract(e.value, '$.github'),
|
|
71
|
+
CAST(json_extract(e.value, '$.public') AS BIGINT), CAST(json_extract(e.value, '$.checkedAt') AS BIGINT)
|
|
72
|
+
FROM json_each(:contexts) e WHERE 1 = 1
|
|
73
|
+
ON CONFLICT (repo_id) DO UPDATE SET github = excluded.github, is_public = excluded.is_public, checked_at = excluded.checked_at
|
|
74
|
+
WHERE managed_github_repository_visibility.checked_at < excluded.checked_at
|
|
75
|
+
OR (managed_github_repository_visibility.checked_at = excluded.checked_at
|
|
76
|
+
AND (managed_github_repository_visibility.is_public = 1 OR excluded.is_public = 0))`)
|
|
52
77
|
return {
|
|
53
78
|
async listGithubMetadata(keys) {
|
|
54
79
|
if (keys.length === 0) return []
|
|
@@ -61,5 +86,13 @@ export function githubMetadataMethods(db: ManagedSql): GithubMetadataStore {
|
|
|
61
86
|
async recordGithubMetadataAttempts(keys, attemptedAt) {
|
|
62
87
|
if (keys.length > 0) await attempt.run(attemptedAt, JSON.stringify(keys), attemptedAt)
|
|
63
88
|
},
|
|
89
|
+
async listGithubRepositoryVisibility(repoIds) {
|
|
90
|
+
if (repoIds.length === 0) return []
|
|
91
|
+
const rows = await visibility.all(JSON.stringify(repoIds)) as (Omit<GithubRepositoryVisibility, 'public'> & { public: number })[]
|
|
92
|
+
return rows.map(row => ({ ...row, public: row.public === 1 }))
|
|
93
|
+
},
|
|
94
|
+
async setGithubRepositoryVisibility(entries) {
|
|
95
|
+
if (entries.length > 0) await setVisibility.run({ contexts: JSON.stringify(entries.map(entry => ({ ...entry, public: entry.public ? 1 : 0 }))) })
|
|
96
|
+
},
|
|
64
97
|
}
|
|
65
98
|
}
|
|
@@ -1,25 +1,115 @@
|
|
|
1
1
|
import { type GithubFixRef, type GithubFixResult, type GithubFixStatus, githubIssueClosedReason, isGithubRepoName, parseGithubFixUrl } from '../common/github-pr.ts'
|
|
2
2
|
import type { ManagedConfig } from './config.ts'
|
|
3
3
|
import type { ManagedDb, TeamReportAccessSnapshot } from './db.ts'
|
|
4
|
-
import type { GithubMetadata } from './github-metadata.ts'
|
|
4
|
+
import type { GithubMetadata, GithubRepositoryVisibility } from './github-metadata.ts'
|
|
5
5
|
import { ensureUserAccessToken } from './github-oauth.ts'
|
|
6
6
|
|
|
7
7
|
type Metadata = Pick<GithubMetadata, 'title' | 'description' | 'status' | 'stateReason'>
|
|
8
8
|
const MAX_GITHUB_LOOKUPS = 200
|
|
9
9
|
const GITHUB_LOOKUP_TIMEOUT_MS = 10_000
|
|
10
10
|
const GITHUB_METADATA_TTL_MS = 60_000
|
|
11
|
+
const GITHUB_PUBLIC_VISIBILITY_TTL_MS = 60_000
|
|
12
|
+
const GITHUB_PRIVATE_REPOSITORIES_URL = 'https://api.github.com/user/repos?visibility=private&per_page=100&page=1'
|
|
13
|
+
|
|
14
|
+
type Repository = TeamReportAccessSnapshot['repositories'][number]
|
|
15
|
+
type RepositoryPayload = { id?: unknown; full_name?: unknown; private?: unknown; visibility?: unknown; permissions?: { pull?: unknown } }
|
|
16
|
+
type RepositoryChecks = { remaining: number; verified: Set<string> }
|
|
17
|
+
type RepositoryAccessContext = {
|
|
18
|
+
db: ManagedDb
|
|
19
|
+
visibility: GithubRepositoryVisibility[]
|
|
20
|
+
publicRepos: Set<string>
|
|
21
|
+
token: string | null
|
|
22
|
+
signal: AbortSignal
|
|
23
|
+
fetchImpl: typeof fetch
|
|
24
|
+
checks: RepositoryChecks
|
|
25
|
+
includePrivateList?: boolean
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
function githubRequest(token: string | null): RequestInit {
|
|
29
|
+
return {
|
|
30
|
+
headers: {
|
|
31
|
+
...(token ? { authorization: `Bearer ${token}` } : {}), accept: 'application/vnd.github+json',
|
|
32
|
+
'user-agent': 'deepview-triage', 'x-github-api-version': '2022-11-28',
|
|
33
|
+
},
|
|
34
|
+
redirect: 'error',
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
function repositoryMatches(repo: Repository, body: RepositoryPayload | null): boolean {
|
|
39
|
+
return body?.id === repo.repoId && typeof body.full_name === 'string' && body.full_name.toLowerCase() === repo.github.toLowerCase()
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
async function listPrivateRepositories(token: string, signal: AbortSignal, fetchImpl: typeof fetch): Promise<(RepositoryPayload | null)[]> {
|
|
43
|
+
try {
|
|
44
|
+
const response = await fetchImpl(GITHUB_PRIVATE_REPOSITORIES_URL, githubRequest(token))
|
|
45
|
+
if (!response.ok) return []
|
|
46
|
+
const body = await response.json() as (RepositoryPayload | null)[] | null
|
|
47
|
+
return Array.isArray(body) && body.length <= 100 && !signal.aborted ? body : []
|
|
48
|
+
} catch { return [] } // A failed list is unknown access; direct checks can still verify it.
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
// Public visibility can be reused briefly across users. Private access stays
|
|
52
|
+
// request-local: one page of the viewer's own repo list grants matching repos,
|
|
53
|
+
// then direct checks cover omissions, pagination and failed list requests.
|
|
54
|
+
async function readableRepositories(repositories: Repository[], { db, visibility, publicRepos, token, signal, fetchImpl, checks, includePrivateList = true }: RepositoryAccessContext): Promise<Set<string>> {
|
|
55
|
+
const readable = new Set(publicRepos)
|
|
56
|
+
const byId = new Map(repositories.map(repo => [repo.repoId, repo]))
|
|
57
|
+
const unverified = repositories.filter(repo => !readable.has(repo.github))
|
|
58
|
+
const updates = new Map<number, GithubRepositoryVisibility>()
|
|
59
|
+
const grant = (repo: Repository, body: RepositoryPayload, checkedAt: number) => {
|
|
60
|
+
const isPublic = body.private === false && body.visibility === 'public'
|
|
61
|
+
readable.add(repo.github)
|
|
62
|
+
checks.verified.add(repo.github)
|
|
63
|
+
if (isPublic) publicRepos.add(repo.github)
|
|
64
|
+
updates.set(repo.repoId, { repoId: repo.repoId, github: repo.github, public: isPublic, checkedAt })
|
|
65
|
+
}
|
|
66
|
+
if (includePrivateList && token && unverified.length > 0 && !signal.aborted) {
|
|
67
|
+
const checkedAt = Date.now()
|
|
68
|
+
for (const entry of await listPrivateRepositories(token, signal, fetchImpl)) {
|
|
69
|
+
const repo = typeof entry?.id === 'number' ? byId.get(entry.id) : undefined
|
|
70
|
+
if (repo && !readable.has(repo.github) && repositoryMatches(repo, entry) && entry?.permissions?.pull === true) grant(repo, entry, checkedAt)
|
|
71
|
+
}
|
|
72
|
+
}
|
|
73
|
+
const formerlyPublic = new Set(visibility.filter(entry => entry.public
|
|
74
|
+
&& byId.get(entry.repoId)?.github.toLowerCase() === entry.github.toLowerCase()).map(entry => entry.repoId))
|
|
75
|
+
const pending = unverified.filter(repo => !readable.has(repo.github) && (token || formerlyPublic.has(repo.repoId))).slice(0, checks.remaining)
|
|
76
|
+
checks.remaining -= pending.length
|
|
77
|
+
for (let i = 0; i < pending.length && !signal.aborted; i += 4) {
|
|
78
|
+
await Promise.all(pending.slice(i, i + 4).map(async repo => {
|
|
79
|
+
if (signal.aborted) return
|
|
80
|
+
const checkedAt = Date.now()
|
|
81
|
+
try {
|
|
82
|
+
const response = await fetchImpl(`https://api.github.com/repos/${repo.github}`, githubRequest(token))
|
|
83
|
+
if (!response.ok) return
|
|
84
|
+
const body = await response.json() as RepositoryPayload | null
|
|
85
|
+
if (!signal.aborted && body && repositoryMatches(repo, body)
|
|
86
|
+
&& (token || (body.private === false && body.visibility === 'public'))) grant(repo, body, checkedAt)
|
|
87
|
+
} catch { /* Unverified access must never expose cached metadata. */ }
|
|
88
|
+
}))
|
|
89
|
+
}
|
|
90
|
+
if (updates.size > 0) await db.setGithubRepositoryVisibility([...updates.values()])
|
|
91
|
+
return readable
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
async function verifyPublicRefreshes(config: ManagedConfig, userId: string, repositories: Repository[], context: RepositoryAccessContext): Promise<string | null> {
|
|
95
|
+
// Cached public metadata needs no credentials. New content uses a viewer
|
|
96
|
+
// token only after a live check, so privatization cannot publish private data
|
|
97
|
+
// under an old public grant. Anonymous reads remain available without a token.
|
|
98
|
+
const { db, publicRepos } = context
|
|
99
|
+
const token = context.token ?? await ensureUserAccessToken(config, db, userId, Date.now(), context.fetchImpl)
|
|
100
|
+
if (token) {
|
|
101
|
+
const currentPublic = new Set<string>()
|
|
102
|
+
const readable = await readableRepositories(repositories, { ...context, publicRepos: currentPublic, token, includePrivateList: false })
|
|
103
|
+
for (const repo of repositories) if (readable.has(repo.github) && !currentPublic.has(repo.github)) publicRepos.delete(repo.github)
|
|
104
|
+
}
|
|
105
|
+
return token
|
|
106
|
+
}
|
|
11
107
|
|
|
12
108
|
// ref.repo is always the canonical name from managed_selected_repo, never an
|
|
13
109
|
// owner/repo/path taken from a Fix URL. Redirects cannot move the user token.
|
|
14
|
-
async function fetchMetadata(ref: GithubFixRef, token: string, fetchImpl: typeof fetch): Promise<Metadata | null> {
|
|
110
|
+
async function fetchMetadata(ref: GithubFixRef, token: string | null, fetchImpl: typeof fetch): Promise<Metadata | null> {
|
|
15
111
|
try {
|
|
16
|
-
const response = await fetchImpl(`https://api.github.com/repos/${ref.repo}/${ref.kind === 'pull' ? 'pulls' : 'issues'}/${ref.number}`,
|
|
17
|
-
headers: {
|
|
18
|
-
authorization: `Bearer ${token}`, accept: 'application/vnd.github+json',
|
|
19
|
-
'user-agent': 'deepview-triage', 'x-github-api-version': '2022-11-28',
|
|
20
|
-
},
|
|
21
|
-
redirect: 'error',
|
|
22
|
-
})
|
|
112
|
+
const response = await fetchImpl(`https://api.github.com/repos/${ref.repo}/${ref.kind === 'pull' ? 'pulls' : 'issues'}/${ref.number}`, githubRequest(token))
|
|
23
113
|
if (!response.ok) return null
|
|
24
114
|
const body = await response.json() as { number?: unknown; title?: unknown; body?: unknown; state?: unknown; merged?: unknown;
|
|
25
115
|
draft?: unknown; base?: { repo?: { full_name?: unknown } }; repository_url?: unknown; pull_request?: unknown; state_reason?: unknown } | null
|
|
@@ -43,8 +133,8 @@ async function fetchMetadata(ref: GithubFixRef, token: string, fetchImpl: typeof
|
|
|
43
133
|
}
|
|
44
134
|
|
|
45
135
|
// The workspace handler checks user -> team membership and finding visibility.
|
|
46
|
-
//
|
|
47
|
-
// including for admins
|
|
136
|
+
// Shared private metadata also requires this viewer's live GitHub access,
|
|
137
|
+
// including for admins. Only recently verified public repos skip that check.
|
|
48
138
|
export async function lookupFixes(config: ManagedConfig, db: ManagedDb, snapshot: TeamReportAccessSnapshot, urls: string[], fetchImpl: typeof fetch = globalThis.fetch): Promise<GithubFixResult[]> {
|
|
49
139
|
const allowed = new Map(snapshot.repositories.filter(repo => isGithubRepoName(repo.github)).map(repo => [repo.github.toLowerCase(), repo]))
|
|
50
140
|
const parsed = urls.map(parseGithubFixUrl)
|
|
@@ -56,13 +146,34 @@ export async function lookupFixes(config: ManagedConfig, db: ManagedDb, snapshot
|
|
|
56
146
|
jobs.set(key, { ...ref, repo: repo.github })
|
|
57
147
|
return key
|
|
58
148
|
})
|
|
59
|
-
|
|
149
|
+
if (jobs.size === 0) return []
|
|
60
150
|
const now = Date.now()
|
|
151
|
+
const signal = AbortSignal.timeout(GITHUB_LOOKUP_TIMEOUT_MS)
|
|
152
|
+
const fetchWithinDeadline: typeof fetch = (input, init) => {
|
|
153
|
+
signal.throwIfAborted()
|
|
154
|
+
return fetchImpl(input, { ...init, signal })
|
|
155
|
+
}
|
|
156
|
+
const repositories = [...new Map([...jobs.values()].map(ref => {
|
|
157
|
+
const repo = allowed.get(ref.repo.toLowerCase())!
|
|
158
|
+
return [repo.repoId, repo] as const
|
|
159
|
+
})).values()]
|
|
160
|
+
const visibility = await db.listGithubRepositoryVisibility(repositories.map(repo => repo.repoId))
|
|
161
|
+
const byId = new Map(repositories.map(repo => [repo.repoId, repo]))
|
|
162
|
+
const publicRepos = new Set(visibility.filter(entry => entry.public && entry.checkedAt <= now
|
|
163
|
+
&& entry.checkedAt + GITHUB_PUBLIC_VISIBILITY_TTL_MS > now
|
|
164
|
+
&& byId.get(entry.repoId)?.github.toLowerCase() === entry.github.toLowerCase()).map(entry => byId.get(entry.repoId)!.github))
|
|
165
|
+
let token = repositories.some(repo => !publicRepos.has(repo.github))
|
|
166
|
+
? await ensureUserAccessToken(config, db, snapshot.user.id, now, fetchWithinDeadline) : null
|
|
167
|
+
const checks: RepositoryChecks = { remaining: MAX_GITHUB_LOOKUPS, verified: new Set() }
|
|
168
|
+
const context = { db, visibility, publicRepos, token, signal, fetchImpl: fetchWithinDeadline, checks }
|
|
169
|
+
const readable = await readableRepositories(repositories, context)
|
|
170
|
+
const authorized = [...jobs.entries()].filter(([, ref]) => readable.has(ref.repo))
|
|
171
|
+
const metadata = new Map((await db.listGithubMetadata(authorized.map(([key]) => key))).map(entry => [entry.key, entry]))
|
|
61
172
|
const lastCheckedAt = (key: string) => {
|
|
62
173
|
const cached = metadata.get(key)
|
|
63
174
|
return Math.max(cached?.fetchedAt ?? 0, cached?.attemptedAt ?? 0)
|
|
64
175
|
}
|
|
65
|
-
const pending =
|
|
176
|
+
const pending = authorized.filter(([key, ref]) => {
|
|
66
177
|
const cached = metadata.get(key)
|
|
67
178
|
// Backfill closed issues cached before closure reasons existed, within the
|
|
68
179
|
// same authorized/capped queue. Even an unknown reason completes backfill.
|
|
@@ -72,25 +183,18 @@ export async function lookupFixes(config: ManagedConfig, db: ManagedDb, snapshot
|
|
|
72
183
|
// Only stale/missing, distinct, authorized entries consume the request budget.
|
|
73
184
|
// Prioritize missing/oldest entries so a large workspace makes progress.
|
|
74
185
|
if (pending.length > 0) {
|
|
75
|
-
const
|
|
76
|
-
|
|
77
|
-
signal.throwIfAborted()
|
|
78
|
-
return fetchImpl(input, { ...init, signal })
|
|
79
|
-
}
|
|
80
|
-
const token = await ensureUserAccessToken(config, db, snapshot.user.id, now, fetchWithinDeadline)
|
|
186
|
+
const publicRefreshes = new Set(pending.filter(([, ref]) => publicRepos.has(ref.repo) && !checks.verified.has(ref.repo)).map(([, ref]) => ref.repo))
|
|
187
|
+
if (publicRefreshes.size > 0) token = await verifyPublicRefreshes(config, snapshot.user.id, repositories.filter(repo => publicRefreshes.has(repo.github)), context)
|
|
81
188
|
const fresh: Omit<GithubMetadata, 'attemptedAt'>[] = []
|
|
82
189
|
const attempted: string[] = []
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
if (result) fresh.push({ key, ...result, fetchedAt: Date.now() })
|
|
92
|
-
}))
|
|
93
|
-
}
|
|
190
|
+
// All waves and access checks share one deadline and four-call limit.
|
|
191
|
+
for (let i = 0; i < pending.length && !signal.aborted; i += 4) {
|
|
192
|
+
await Promise.all(pending.slice(i, i + 4).map(async ([key, ref]) => {
|
|
193
|
+
if (signal.aborted) return
|
|
194
|
+
if (metadata.has(key)) attempted.push(key)
|
|
195
|
+
const result = await fetchMetadata(ref, publicRepos.has(ref.repo) && !checks.verified.has(ref.repo) ? null : token, fetchWithinDeadline)
|
|
196
|
+
if (result) fresh.push({ key, ...result, fetchedAt: Date.now() })
|
|
197
|
+
}))
|
|
94
198
|
}
|
|
95
199
|
// Only started cached reads rotate. Missing entries retain priority, and
|
|
96
200
|
// jobs skipped by the cap, deadline or absent token keep their place.
|
|
@@ -101,6 +205,14 @@ export async function lookupFixes(config: ManagedConfig, db: ManagedDb, snapshot
|
|
|
101
205
|
for (const entry of await db.listGithubMetadata(fresh.map(row => row.key))) metadata.set(entry.key, entry)
|
|
102
206
|
}
|
|
103
207
|
}
|
|
208
|
+
if (publicRepos.size > 0) {
|
|
209
|
+
// Another request may already have discovered a public -> private change
|
|
210
|
+
// and written new private metadata. Honor that newer visibility observation.
|
|
211
|
+
const current = await db.listGithubRepositoryVisibility(repositories.filter(repo => publicRepos.has(repo.github)).map(repo => repo.repoId))
|
|
212
|
+
const stillPublic = new Set(current.filter(entry => entry.public
|
|
213
|
+
&& byId.get(entry.repoId)?.github.toLowerCase() === entry.github.toLowerCase()).map(entry => byId.get(entry.repoId)!.github))
|
|
214
|
+
for (const [key, ref] of authorized) if (publicRepos.has(ref.repo) && !stillPublic.has(ref.repo)) metadata.delete(key)
|
|
215
|
+
}
|
|
104
216
|
// The handler rechecks workspace access and saved Fix links before returning.
|
|
105
217
|
return urls.flatMap((url, index): GithubFixResult[] => {
|
|
106
218
|
const key = keys[index]
|