@preventive/triage 1.0.0-alpha.27 → 1.0.0-alpha.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE-APACHE +202 -0
- package/client/linked-findings.js +1 -1
- package/common/bundle-metadata.js +12 -10
- package/common/code-language.js +17 -2
- package/common/default-scan-models.ts +3 -3
- package/common/finding-id.js +1 -1
- package/common/managed/report-content.ts +1 -1
- package/common/managed/report-filter.ts +1 -1
- package/out/client-managed-import.js +16 -16
- package/out/client-managed.js +93 -68
- package/out/graph.js +11 -11
- package/out/index.html +1 -1
- package/out/prism.js +2 -2
- package/out/stasis.svg +0 -25
- package/out/terminal.js +163 -163
- package/out/view.css +1 -1
- package/out/view.js +223 -190
- package/package.json +21 -28
- package/server-e2e/static.ts +3 -4
- package/server-managed/bundle-advisories.ts +53 -0
- package/server-managed/bundle-advisory-inventory.ts +75 -0
- package/server-managed/bundle-build-diagnostics.d.ts +10 -0
- package/server-managed/bundle-build-diagnostics.js +23 -0
- package/server-managed/bundle-build-leases.ts +41 -0
- package/server-managed/bundle-build-worker.js +54 -0
- package/server-managed/bundle-build.ts +168 -0
- package/server-managed/bundle-cache.ts +22 -13
- package/server-managed/bundle-summary-cache.ts +27 -9
- package/server-managed/bundle.ts +66 -1
- package/server-managed/cache-storage.ts +4 -0
- package/server-managed/content-access.ts +1 -1
- package/server-managed/db-methods.ts +122 -20
- package/server-managed/db-neon.ts +121 -32
- package/server-managed/db-schema.ts +2 -0
- package/server-managed/db.ts +5 -1
- package/server-managed/github-app.ts +10 -15
- package/server-managed/github-errors.ts +21 -0
- package/server-managed/github-issues.ts +1 -1
- package/server-managed/github-metadata.ts +34 -1
- package/server-managed/github-pulls.ts +142 -30
- package/server-managed/http.ts +244 -131
- package/server-managed/index.ts +28 -11
- package/server-managed/management-catalog.ts +45 -0
- package/server-managed/object-storage-vercel.ts +1 -1
- package/server-managed/public-workspace.ts +20 -14
- package/server-managed/report-migration.ts +1 -1
- package/server-managed/report-sources.ts +14 -4
- package/server-managed/repository-browser.ts +3 -0
- package/server-managed/repository-discovery.ts +1 -1
- package/server-managed/revisions.ts +35 -0
- package/server-managed/sql-postgres.ts +1 -0
- package/server-managed/sql.ts +14 -3
- package/server-managed/storage-db.ts +12 -5
- package/server-managed/storage-maintenance.ts +47 -22
- package/server-managed/storage-payload.ts +16 -4
- package/server-managed/storage-stores.ts +2 -2
- package/server-managed/storage.ts +6 -4
- package/server-managed/team-feed.ts +50 -5
- package/server-managed/team-reports.ts +85 -22
- package/server-managed/workspace-shares.ts +15 -3
- package/report/index.js +0 -260
- package/report/src/finding-id.js +0 -80
- package/report/src/finding.js +0 -314
- package/report/src/labels.js +0 -35
- package/report/src/md-structure.js +0 -471
- package/report/src/md-text.js +0 -167
- package/report/src/meta.js +0 -76
- package/report/src/parse-codex.js +0 -149
- package/report/src/parse-deepsec.js +0 -199
- package/report/src/parse-deepview-fields.js +0 -375
- package/report/src/parse-deepview-md.js +0 -186
- package/report/src/parse-md-id.js +0 -137
- package/report/src/parse-md.js +0 -323
- package/report/src/parse-piolium-id.js +0 -79
- package/report/src/parse-piolium-rows.js +0 -126
- package/report/src/parse-piolium-tokens.js +0 -185
- package/report/src/parse-piolium.js +0 -399
- package/report/src/security.js +0 -63
- package/report/src/severity.js +0 -18
- package/report/src/utf8.js +0 -21
- package/report/src/write-md-finding.js +0 -273
- package/report/src/write-md.js +0 -291
- /package/{LICENSE → LICENSE-MIT} +0 -0
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
import { AsyncLocalStorage } from 'node:async_hooks'
|
|
2
2
|
import { WebSocket } from 'ws'
|
|
3
3
|
import { type ManagedDb, type ManagedDbOptions, createManagedMethods } from './db-methods.ts'
|
|
4
|
+
import { revisionSchema } from './revisions.ts'
|
|
4
5
|
import { MANAGED_SCHEMA } from './db-schema.ts'
|
|
5
6
|
import { STORAGE_SCHEMA } from './storage-db.ts'
|
|
6
7
|
import { GITHUB_METADATA_SCHEMA, GITHUB_STATE_REASON_COLUMN } from './github-metadata.ts'
|
|
7
8
|
import { MANAGED_ISSUE_SCHEMA } from './managed-issues.ts'
|
|
9
|
+
import { BUNDLE_BUILD_LEASE_SCHEMA } from './bundle-build-leases.ts'
|
|
8
10
|
import { COMMENT_SCHEMA } from './comments.ts'
|
|
9
11
|
import { ACTIVITY_SCHEMA } from './activity.ts'
|
|
10
12
|
import { type ManagedSqlDriver, scopeManagedMethods } from './sql.ts'
|
|
@@ -21,11 +23,38 @@ export interface PgConnection {
|
|
|
21
23
|
release(): Promise<void>
|
|
22
24
|
}
|
|
23
25
|
export type PgConnect = () => Promise<PgConnection>
|
|
24
|
-
//
|
|
25
|
-
//
|
|
26
|
+
// Content/access mutations cooperate on this transaction-scoped lock. Reads,
|
|
27
|
+
// presence updates, and maintenance leases do not take it. Also fences migrations.
|
|
26
28
|
const LOCK = 'SELECT pg_advisory_xact_lock(1937006964, 1835101793)'
|
|
27
29
|
|
|
30
|
+
async function currentSchema(db: PgConnection): Promise<boolean> {
|
|
31
|
+
const exists = (await db.query("SELECT to_regclass('managed_schema_version') AS name")).rows[0]?.['name']
|
|
32
|
+
if (!exists) return false
|
|
33
|
+
const versions = new Set((await db.query('SELECT version FROM managed_schema_version')).rows.map(row => Number(row['version'])))
|
|
34
|
+
return Array.from({ length: 14 }, (_, i) => i + 1).every(version => versions.has(version))
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
async function migrateRepositoryDefaultCache(db: PgConnection): Promise<void> {
|
|
38
|
+
if ((await db.query('SELECT version FROM managed_schema_version WHERE version = 12')).rows.length > 0) return
|
|
39
|
+
await db.query('ALTER TABLE managed_selected_repo ADD COLUMN IF NOT EXISTS cached_default_branch TEXT')
|
|
40
|
+
await db.query('INSERT INTO managed_schema_version VALUES (12)')
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
async function migrateBundleBuildLeases(db: PgConnection): Promise<void> {
|
|
44
|
+
if ((await db.query('SELECT version FROM managed_schema_version WHERE version = 13')).rows.length > 0) return
|
|
45
|
+
await db.query(postgresSchema(BUNDLE_BUILD_LEASE_SCHEMA))
|
|
46
|
+
await db.query('INSERT INTO managed_schema_version VALUES (13)')
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
async function migrateGithubMetadata(db: PgConnection): Promise<void> {
|
|
50
|
+
await db.query(postgresSchema(GITHUB_METADATA_SCHEMA + MANAGED_ISSUE_SCHEMA))
|
|
51
|
+
await db.query(`ALTER TABLE managed_github_metadata ADD COLUMN IF NOT EXISTS state_reason ${GITHUB_STATE_REASON_COLUMN}`)
|
|
52
|
+
await db.query('ALTER TABLE managed_github_metadata ADD COLUMN IF NOT EXISTS attempted_at BIGINT')
|
|
53
|
+
await db.query('INSERT INTO managed_schema_version VALUES (14) ON CONFLICT DO NOTHING')
|
|
54
|
+
}
|
|
55
|
+
|
|
28
56
|
async function initialize(db: PgConnection): Promise<void> {
|
|
57
|
+
if (await currentSchema(db)) return
|
|
29
58
|
await db.query('BEGIN')
|
|
30
59
|
try {
|
|
31
60
|
await db.query(LOCK)
|
|
@@ -58,9 +87,7 @@ async function initialize(db: PgConnection): Promise<void> {
|
|
|
58
87
|
await db.query(postgresSchema(WORKSPACE_SHARE_SCHEMA))
|
|
59
88
|
await db.query('INSERT INTO managed_schema_version VALUES (4)')
|
|
60
89
|
}
|
|
61
|
-
await db
|
|
62
|
-
await db.query(`ALTER TABLE managed_github_metadata ADD COLUMN IF NOT EXISTS state_reason ${GITHUB_STATE_REASON_COLUMN}`)
|
|
63
|
-
await db.query('ALTER TABLE managed_github_metadata ADD COLUMN IF NOT EXISTS attempted_at BIGINT')
|
|
90
|
+
await migrateGithubMetadata(db)
|
|
64
91
|
if ((await db.query('SELECT version FROM managed_schema_version WHERE version = 5')).rows.length === 0) {
|
|
65
92
|
await db.query(`ALTER TABLE managed_workspace_share ADD COLUMN IF NOT EXISTS dependencies INTEGER NOT NULL DEFAULT 0,
|
|
66
93
|
ADD COLUMN IF NOT EXISTS security INTEGER NOT NULL DEFAULT 0`)
|
|
@@ -84,10 +111,21 @@ async function initialize(db: PgConnection): Promise<void> {
|
|
|
84
111
|
await db.query('CREATE INDEX IF NOT EXISTS managed_report_hash_idx ON managed_report(sha256, uploaded_at, id)')
|
|
85
112
|
await db.query('INSERT INTO managed_schema_version VALUES (8)')
|
|
86
113
|
}
|
|
87
|
-
|
|
88
|
-
await db.query(
|
|
114
|
+
if ((await db.query('SELECT version FROM managed_schema_version WHERE version = 9')).rows.length === 0) {
|
|
115
|
+
await db.query(postgresSchema(STORAGE_SCHEMA))
|
|
116
|
+
for (const table of ['managed_report', 'managed_bundle']) {
|
|
117
|
+
await db.query(`ALTER TABLE ${table} ADD COLUMN IF NOT EXISTS data_key TEXT, ADD COLUMN IF NOT EXISTS storage_encrypted INTEGER NOT NULL DEFAULT 0`)
|
|
118
|
+
}
|
|
119
|
+
await db.query('ALTER TABLE managed_user ADD COLUMN IF NOT EXISTS gh_tokens_encrypted INTEGER NOT NULL DEFAULT 0')
|
|
120
|
+
await db.query('INSERT INTO managed_schema_version VALUES (9)')
|
|
121
|
+
}
|
|
122
|
+
if ((await db.query('SELECT version FROM managed_schema_version WHERE version = 10')).rows.length === 0) {
|
|
123
|
+
await db.query(postgresSchema(revisionSchema(true)) + '; INSERT INTO managed_schema_version VALUES (10)')
|
|
124
|
+
}
|
|
125
|
+
if ((await db.query('SELECT version FROM managed_schema_version WHERE version = 11')).rows.length === 0) {
|
|
126
|
+
await db.query(postgresSchema(revisionSchema(true)) + '; INSERT INTO managed_schema_version VALUES (11)')
|
|
89
127
|
}
|
|
90
|
-
|
|
128
|
+
for (const migrate of [migrateRepositoryDefaultCache, migrateBundleBuildLeases]) await migrate(db)
|
|
91
129
|
await db.query('COMMIT')
|
|
92
130
|
} catch (err) {
|
|
93
131
|
await db.query('ROLLBACK')
|
|
@@ -112,10 +150,53 @@ async function createUploadTrigger(db: PgConnection, type: string, createTrigger
|
|
|
112
150
|
}
|
|
113
151
|
}
|
|
114
152
|
|
|
153
|
+
function requestConnections(connect: PgConnect) {
|
|
154
|
+
const requests = new AsyncLocalStorage<{ connection?: PgConnection; queue: Promise<unknown>; ended: boolean }>()
|
|
155
|
+
async function withConnection<T>(work: (db: PgConnection) => Promise<T>): Promise<T> {
|
|
156
|
+
const request = requests.getStore()
|
|
157
|
+
if (!request || request.ended) {
|
|
158
|
+
const connection = await connect()
|
|
159
|
+
let result: T
|
|
160
|
+
try { result = await work(connection) }
|
|
161
|
+
catch (error) { await connection.release().catch(() => {}); throw error }
|
|
162
|
+
await connection.release()
|
|
163
|
+
return result
|
|
164
|
+
}
|
|
165
|
+
// Serialize transactions on the leased connection, never interleave BEGINs
|
|
166
|
+
// from concurrent Blob/catalog tasks. Remote Blob reads remain concurrent.
|
|
167
|
+
const result = request.queue.then(async () => {
|
|
168
|
+
const connection = request.connection ??= await connect()
|
|
169
|
+
try { return await work(connection) }
|
|
170
|
+
catch (error) {
|
|
171
|
+
delete request.connection
|
|
172
|
+
await connection.release().catch(() => {})
|
|
173
|
+
throw error
|
|
174
|
+
}
|
|
175
|
+
})
|
|
176
|
+
request.queue = result.catch(() => {})
|
|
177
|
+
return result
|
|
178
|
+
}
|
|
179
|
+
async function withRequest<T>(work: () => Promise<T>): Promise<T> {
|
|
180
|
+
if (requests.getStore()) return work()
|
|
181
|
+
const request: { connection?: PgConnection; queue: Promise<unknown>; ended: boolean } = { queue: Promise.resolve(), ended: false }
|
|
182
|
+
let failed = false
|
|
183
|
+
try { return await requests.run(request, work) }
|
|
184
|
+
catch (error) { failed = true; throw error }
|
|
185
|
+
finally {
|
|
186
|
+
request.ended = true
|
|
187
|
+
await request.queue
|
|
188
|
+
if (failed) await request.connection?.release().catch(() => {})
|
|
189
|
+
else await request.connection?.release()
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
return { withConnection, withRequest }
|
|
193
|
+
}
|
|
194
|
+
|
|
115
195
|
// Exposed for parity tests against real PostgreSQL semantics via PGlite.
|
|
116
|
-
export async function openPostgresManagedDb(connect: PgConnect, options: ManagedDbOptions = {}): Promise<ManagedDb> {
|
|
196
|
+
export async function openPostgresManagedDb(connect: PgConnect, options: ManagedDbOptions = {}, reuseConnections = false): Promise<ManagedDb> {
|
|
117
197
|
const initial = await connect()
|
|
118
198
|
try { await initialize(initial) } finally { await initial.release() }
|
|
199
|
+
const { withConnection, withRequest } = requestConnections(connect)
|
|
119
200
|
const context = new AsyncLocalStorage<PgConnection>()
|
|
120
201
|
let closed = false
|
|
121
202
|
const driver: ManagedSqlDriver = {
|
|
@@ -146,43 +227,51 @@ export async function openPostgresManagedDb(connect: PgConnect, options: Managed
|
|
|
146
227
|
run: async (...args) => ({ changes: (await query(args)).rowCount ?? 0 }),
|
|
147
228
|
}
|
|
148
229
|
},
|
|
149
|
-
async scope(write, work) {
|
|
230
|
+
async scope(write, work, { lock = write, statement = false } = {}) {
|
|
150
231
|
if (closed) throw new Error('Managed database is closed')
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
232
|
+
let committed = false
|
|
233
|
+
try { return await withConnection(async db => {
|
|
234
|
+
if (statement) return context.run(db, work)
|
|
235
|
+
let commitAttempted = false
|
|
236
|
+
try {
|
|
237
|
+
await db.query(write ? 'BEGIN' : 'BEGIN ISOLATION LEVEL REPEATABLE READ READ ONLY')
|
|
238
|
+
if (lock) await db.query(LOCK)
|
|
239
|
+
const result = await context.run(db, work)
|
|
240
|
+
commitAttempted = true
|
|
241
|
+
await db.query('COMMIT')
|
|
242
|
+
committed = true
|
|
243
|
+
return result
|
|
244
|
+
} catch (err) {
|
|
245
|
+
await db.query('ROLLBACK').catch(() => {})
|
|
246
|
+
if (write && commitAttempted) throw new Error('Managed commit outcome is uncertain', { cause: err })
|
|
247
|
+
throw err
|
|
248
|
+
}
|
|
249
|
+
}) } catch (error) {
|
|
250
|
+
if (write && committed) throw new Error('Managed commit outcome is uncertain', { cause: error })
|
|
251
|
+
throw error
|
|
165
252
|
}
|
|
166
|
-
try { await db.release() }
|
|
167
|
-
catch (err) { if (write) throw new Error('Managed commit outcome is uncertain', { cause: err }); throw err }
|
|
168
|
-
return result
|
|
169
253
|
},
|
|
170
254
|
close() { closed = true },
|
|
171
255
|
}
|
|
172
|
-
|
|
256
|
+
const methods = scopeManagedMethods(createManagedMethods(driver, options), driver)
|
|
257
|
+
if (reuseConnections) methods.withRequest = withRequest
|
|
258
|
+
return methods
|
|
173
259
|
}
|
|
174
260
|
|
|
175
261
|
export async function openNeonManagedDb(url: string, options: ManagedDbOptions = {}): Promise<ManagedDb> {
|
|
176
|
-
// Reuse the e2e optional-driver boundary.
|
|
177
|
-
// before returning; no
|
|
262
|
+
// Reuse the e2e optional-driver boundary. Requests lease one connection and
|
|
263
|
+
// close it before returning; no socket must survive a frozen invocation.
|
|
178
264
|
const { Client, neonConfig } = await import('../server-e2e/neon-driver.ts') as unknown as {
|
|
179
|
-
Client: new (url: string) => { connect(): Promise<void>; end(): Promise<void>; query: PgConnection['query'] }
|
|
265
|
+
Client: new (url: string) => { connect(): Promise<void>; end(): Promise<void>; query: PgConnection['query']; on?(event: 'error', listener: (error: Error) => void): void }
|
|
180
266
|
neonConfig: { webSocketConstructor: typeof WebSocket }
|
|
181
267
|
}
|
|
182
268
|
neonConfig.webSocketConstructor = WebSocket
|
|
183
269
|
return openPostgresManagedDb(async () => {
|
|
184
270
|
const client = new Client(url)
|
|
271
|
+
// A remote disconnect between operations must reject the next query, not
|
|
272
|
+
// become an unhandled EventEmitter error while the request is reading Blob.
|
|
273
|
+
client.on?.('error', () => {})
|
|
185
274
|
try { await client.connect() } catch (err) { await client.end().catch(() => {}); throw err }
|
|
186
275
|
return { query: (sql, params) => client.query(sql, params), release: () => client.end() }
|
|
187
|
-
}, options)
|
|
276
|
+
}, options, true)
|
|
188
277
|
}
|
|
@@ -45,6 +45,8 @@ CREATE TABLE IF NOT EXISTS managed_selected_repo (
|
|
|
45
45
|
is_private INTEGER NOT NULL,
|
|
46
46
|
installation_id INTEGER,
|
|
47
47
|
default_branch TEXT NOT NULL,
|
|
48
|
+
-- Live browser hint; independent of the default discovered at selection.
|
|
49
|
+
cached_default_branch TEXT,
|
|
48
50
|
html_url TEXT NOT NULL,
|
|
49
51
|
added_by TEXT REFERENCES managed_user(id) ON DELETE SET NULL,
|
|
50
52
|
added_at INTEGER NOT NULL,
|
package/server-managed/db.ts
CHANGED
|
@@ -6,10 +6,12 @@ import { migrateSlugs } from './slugs.ts'
|
|
|
6
6
|
import { migrateReportLocations } from './report-migration.ts'
|
|
7
7
|
import { initCommentMethods } from './comments.ts'
|
|
8
8
|
import { initActivityMethods } from './activity.ts'
|
|
9
|
+
import { revisionSchema } from './revisions.ts'
|
|
9
10
|
import { MANAGED_SCHEMA } from './db-schema.ts'
|
|
10
11
|
import { STORAGE_SCHEMA } from './storage-db.ts'
|
|
11
12
|
import { GITHUB_METADATA_SCHEMA, GITHUB_STATE_REASON_COLUMN } from './github-metadata.ts'
|
|
12
13
|
import { MANAGED_ISSUE_SCHEMA } from './managed-issues.ts'
|
|
14
|
+
import { BUNDLE_BUILD_LEASE_SCHEMA } from './bundle-build-leases.ts'
|
|
13
15
|
import { managedTableRenames } from './db-table-names.ts'
|
|
14
16
|
import { type ManagedDb, type ManagedDbOptions, createManagedMethods } from './db-methods.ts'
|
|
15
17
|
import { createSqliteDriver, scopeManagedMethods } from './sql.ts'
|
|
@@ -66,7 +68,8 @@ export function openSqliteManagedDb(path: string, options: ManagedDbOptions = {}
|
|
|
66
68
|
db.exec('PRAGMA synchronous = FULL;')
|
|
67
69
|
db.exec('PRAGMA foreign_keys = ON;')
|
|
68
70
|
migrateManagedTableNames(db)
|
|
69
|
-
db.exec(MANAGED_SCHEMA + GITHUB_METADATA_SCHEMA + MANAGED_ISSUE_SCHEMA + STORAGE_SCHEMA)
|
|
71
|
+
db.exec(MANAGED_SCHEMA + GITHUB_METADATA_SCHEMA + MANAGED_ISSUE_SCHEMA + STORAGE_SCHEMA + BUNDLE_BUILD_LEASE_SCHEMA)
|
|
72
|
+
ensureColumn(db, 'managed_selected_repo', 'cached_default_branch', 'TEXT')
|
|
70
73
|
for (const table of ['managed_report', 'managed_bundle']) {
|
|
71
74
|
ensureColumn(db, table, 'data_key', 'TEXT')
|
|
72
75
|
ensureColumn(db, table, 'storage_encrypted', 'INTEGER NOT NULL DEFAULT 0')
|
|
@@ -106,6 +109,7 @@ export function openSqliteManagedDb(path: string, options: ManagedDbOptions = {}
|
|
|
106
109
|
ensureColumn(db, 'managed_finding_triage_event', 'repo', 'TEXT')
|
|
107
110
|
initCommentMethods(db)
|
|
108
111
|
initActivityMethods(db)
|
|
112
|
+
db.exec(revisionSchema())
|
|
109
113
|
} catch (err) {
|
|
110
114
|
try { db.close() } catch {}
|
|
111
115
|
throw err
|
|
@@ -4,8 +4,9 @@
|
|
|
4
4
|
// permissions. Discovery itself is read-only and skips archived repositories.
|
|
5
5
|
import { Buffer } from 'node:buffer'
|
|
6
6
|
import { createHash, createSign } from 'node:crypto'
|
|
7
|
+
import { createClient } from '@preventive/upstream/github.js'
|
|
7
8
|
import type { ManagedConfig } from './config.ts'
|
|
8
|
-
import { GithubApiError, throwGithubResponseError } from './github-errors.ts'
|
|
9
|
+
import { GithubApiError, throwGithubResponseError, upstreamGithub } from './github-errors.ts'
|
|
9
10
|
|
|
10
11
|
export { GithubApiError } from './github-errors.ts'
|
|
11
12
|
|
|
@@ -124,10 +125,9 @@ export function publicRepositoryName(raw: unknown): string | null {
|
|
|
124
125
|
// Public additions must be readable by the server without the acting user's
|
|
125
126
|
// credentials. Require explicit public visibility and use GitHub's canonical
|
|
126
127
|
// metadata, never client-supplied ids, installation ids, or branches.
|
|
127
|
-
export async function fetchPublicRepository(fullName: string
|
|
128
|
+
export async function fetchPublicRepository(fullName: string): Promise<ConnectedRepo> {
|
|
128
129
|
if (publicRepositoryName(fullName) !== fullName) throw new GithubApiError(400, 'bad-repository')
|
|
129
|
-
const
|
|
130
|
-
const repo = parseRepo(await githubJson(`${GITHUB_API}/repos/${path}`, null, fetchImpl), null)
|
|
130
|
+
const repo = parseRepo(await upstreamGithub(() => createClient({ token: null, userAgent: USER_AGENT }).getRepo({ repo: fullName })), null)
|
|
131
131
|
if (repo == null || repo.private || repo.visibility !== 'public') throw new GithubApiError(409, 'repo-not-public')
|
|
132
132
|
if (publicRepositoryName(repo.fullName) !== repo.fullName || repo.fullName.toLowerCase() !== fullName.toLowerCase()) throw new GithubApiError(502, 'github-malformed')
|
|
133
133
|
return { ...repo, htmlUrl: `https://github.com/${repo.fullName}` }
|
|
@@ -138,17 +138,12 @@ export async function fetchPublicRepository(fullName: string, fetchImpl: typeof
|
|
|
138
138
|
// List the authenticated user's repositories (GET /user/repos), paginated until
|
|
139
139
|
// a short page, deduped + sorted. With an identity-only token this returns the
|
|
140
140
|
// user's PUBLIC repos. READ-ONLY.
|
|
141
|
-
export async function listUserRepos(accessToken: string
|
|
141
|
+
export async function listUserRepos(accessToken: string): Promise<ConnectedRepo[]> {
|
|
142
142
|
const byName = new Map<string, ConnectedRepo>()
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
const
|
|
146
|
-
if (
|
|
147
|
-
for (const r of body) {
|
|
148
|
-
const repo = parseRepo(r, null)
|
|
149
|
-
if (repo != null) byName.set(repo.fullName, repo)
|
|
150
|
-
}
|
|
151
|
-
if (body.length < PER_PAGE) break
|
|
143
|
+
const body = await upstreamGithub(() => createClient({ token: accessToken, userAgent: USER_AGENT }).listUserRepos())
|
|
144
|
+
for (const r of body) {
|
|
145
|
+
const repo = parseRepo(r, null)
|
|
146
|
+
if (repo != null) byName.set(repo.fullName, repo)
|
|
152
147
|
}
|
|
153
148
|
return [...byName.values()].toSorted((a, b) => a.fullName.localeCompare(b.fullName))
|
|
154
149
|
}
|
|
@@ -373,7 +368,7 @@ export async function collectRepos(config: ManagedConfig, userToken: string | nu
|
|
|
373
368
|
const lists: ConnectedRepo[][] = []
|
|
374
369
|
if (userToken != null) {
|
|
375
370
|
try {
|
|
376
|
-
lists.push(await listUserRepos(userToken
|
|
371
|
+
lists.push(await listUserRepos(userToken))
|
|
377
372
|
} catch (err) {
|
|
378
373
|
if (err instanceof GithubApiError && err.status === 401) tokenMissing = true
|
|
379
374
|
else console.warn('managed: public repo list failed:', err)
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { HttpError } from '@preventive/upstream/github.js'
|
|
2
|
+
|
|
1
3
|
// A failure carrying the HTTP status the router should surface. 401 passes
|
|
2
4
|
// through so the user-token path can map it to "log in again".
|
|
3
5
|
export class GithubApiError extends Error {
|
|
@@ -26,3 +28,22 @@ export async function throwGithubResponseError(res: Response): Promise<never> {
|
|
|
26
28
|
if (res.status === 404) throw new GithubApiError(404, 'github-not-found')
|
|
27
29
|
throw new GithubApiError(502, `github-status-${res.status}`)
|
|
28
30
|
}
|
|
31
|
+
|
|
32
|
+
export async function upstreamGithub<T>(request: () => Promise<T>): Promise<T> {
|
|
33
|
+
try { return await request() } catch (error) {
|
|
34
|
+
if (error instanceof HttpError) {
|
|
35
|
+
// The upstream error includes GitHub's response text. Retain managed
|
|
36
|
+
// status handling, including rate limits, without exposing that text.
|
|
37
|
+
if (error.status === 401) throw new GithubApiError(401, 'github-unauthorized')
|
|
38
|
+
if (error.status === 404) throw new GithubApiError(404, 'github-not-found')
|
|
39
|
+
if (error.status === 429 || (error.status === 403 && /rate limit|abuse detection/iu.test(error.message))) {
|
|
40
|
+
throw new GithubApiError(429, 'github-rate-limited', 60)
|
|
41
|
+
}
|
|
42
|
+
throw new GithubApiError(502, `github-status-${error.status}`)
|
|
43
|
+
}
|
|
44
|
+
if (error instanceof Error && (error.name === 'AssertionError' || error.cause instanceof SyntaxError)) {
|
|
45
|
+
throw new GithubApiError(502, 'github-malformed')
|
|
46
|
+
}
|
|
47
|
+
throw new GithubApiError(502, 'github-unreachable')
|
|
48
|
+
}
|
|
49
|
+
}
|
|
@@ -4,7 +4,7 @@ import type { BlobStore } from './blob-store.ts'
|
|
|
4
4
|
import type { ManagedIssue } from './managed-issues.ts'
|
|
5
5
|
import { randomToken } from './crypto.ts'
|
|
6
6
|
import { githubIssueClosedReason } from '../common/github-pr.ts'
|
|
7
|
-
import { reportEntries, reportRepoGithub } from '
|
|
7
|
+
import { reportEntries, reportRepoGithub } from '@preventive/report'
|
|
8
8
|
import { newIssueLabels } from '../common/github-issue-labels.js'
|
|
9
9
|
import { appJwt, githubAppConfigured, installUrl, publicRepositoryName } from './github-app.ts'
|
|
10
10
|
import { ISSUE_LOGIN_PATH, issueUserToken } from './github-issue-oauth.ts'
|
|
@@ -12,15 +12,24 @@ export interface GithubMetadata {
|
|
|
12
12
|
fetchedAt: number
|
|
13
13
|
attemptedAt: number | null
|
|
14
14
|
}
|
|
15
|
+
export interface GithubRepositoryVisibility {
|
|
16
|
+
repoId: number
|
|
17
|
+
github: string
|
|
18
|
+
public: boolean
|
|
19
|
+
checkedAt: number
|
|
20
|
+
}
|
|
15
21
|
export interface GithubMetadataStore {
|
|
16
22
|
listGithubMetadata(keys: readonly string[]): Promise<GithubMetadata[]>
|
|
17
23
|
setGithubMetadata(entries: readonly Omit<GithubMetadata, 'attemptedAt'>[]): Promise<void>
|
|
18
24
|
recordGithubMetadataAttempts(keys: readonly string[], attemptedAt: number): Promise<void>
|
|
25
|
+
listGithubRepositoryVisibility(repoIds: readonly number[]): Promise<GithubRepositoryVisibility[]>
|
|
26
|
+
setGithubRepositoryVisibility(entries: readonly GithubRepositoryVisibility[]): Promise<void>
|
|
19
27
|
}
|
|
20
28
|
|
|
21
29
|
// Successful GitHub reads are shared and retained without eviction. The stable
|
|
22
30
|
// repository ID in the key survives renames, but is never an access grant.
|
|
23
|
-
// Callers must
|
|
31
|
+
// Callers must verify current user -> team -> repo grants plus either recently
|
|
32
|
+
// verified public visibility or the viewer's live GitHub access with their token.
|
|
24
33
|
export const GITHUB_STATE_REASON_COLUMN = "TEXT CHECK (state_reason IN ('completed', 'not_planned', 'duplicate', 'unknown'))"
|
|
25
34
|
export const GITHUB_METADATA_SCHEMA = `
|
|
26
35
|
CREATE TABLE IF NOT EXISTS managed_github_metadata (
|
|
@@ -32,6 +41,12 @@ CREATE TABLE IF NOT EXISTS managed_github_metadata (
|
|
|
32
41
|
fetched_at INTEGER NOT NULL,
|
|
33
42
|
attempted_at INTEGER
|
|
34
43
|
) STRICT;
|
|
44
|
+
CREATE TABLE IF NOT EXISTS managed_github_repository_visibility (
|
|
45
|
+
repo_id INTEGER PRIMARY KEY,
|
|
46
|
+
github TEXT NOT NULL,
|
|
47
|
+
is_public INTEGER NOT NULL CHECK (is_public IN (0, 1)),
|
|
48
|
+
checked_at INTEGER NOT NULL
|
|
49
|
+
) STRICT;
|
|
35
50
|
`
|
|
36
51
|
|
|
37
52
|
export function githubMetadataMethods(db: ManagedSql): GithubMetadataStore {
|
|
@@ -49,6 +64,16 @@ export function githubMetadataMethods(db: ManagedSql): GithubMetadataStore {
|
|
|
49
64
|
// The monotonic guard preserves a newer attempt recorded by another reader.
|
|
50
65
|
const attempt = db.prepare(`UPDATE managed_github_metadata SET attempted_at = ?
|
|
51
66
|
WHERE cache_key IN (SELECT value FROM json_each(?)) AND (attempted_at IS NULL OR attempted_at < ?)`)
|
|
67
|
+
const visibility = db.prepare(`SELECT repo_id AS repoId, github, is_public AS public, checked_at AS checkedAt
|
|
68
|
+
FROM managed_github_repository_visibility WHERE repo_id IN (SELECT CAST(value AS BIGINT) FROM json_each(?))`)
|
|
69
|
+
const setVisibility = db.prepare(`INSERT INTO managed_github_repository_visibility (repo_id, github, is_public, checked_at)
|
|
70
|
+
SELECT CAST(json_extract(e.value, '$.repoId') AS BIGINT), json_extract(e.value, '$.github'),
|
|
71
|
+
CAST(json_extract(e.value, '$.public') AS BIGINT), CAST(json_extract(e.value, '$.checkedAt') AS BIGINT)
|
|
72
|
+
FROM json_each(:contexts) e WHERE 1 = 1
|
|
73
|
+
ON CONFLICT (repo_id) DO UPDATE SET github = excluded.github, is_public = excluded.is_public, checked_at = excluded.checked_at
|
|
74
|
+
WHERE managed_github_repository_visibility.checked_at < excluded.checked_at
|
|
75
|
+
OR (managed_github_repository_visibility.checked_at = excluded.checked_at
|
|
76
|
+
AND (managed_github_repository_visibility.is_public = 1 OR excluded.is_public = 0))`)
|
|
52
77
|
return {
|
|
53
78
|
async listGithubMetadata(keys) {
|
|
54
79
|
if (keys.length === 0) return []
|
|
@@ -61,5 +86,13 @@ export function githubMetadataMethods(db: ManagedSql): GithubMetadataStore {
|
|
|
61
86
|
async recordGithubMetadataAttempts(keys, attemptedAt) {
|
|
62
87
|
if (keys.length > 0) await attempt.run(attemptedAt, JSON.stringify(keys), attemptedAt)
|
|
63
88
|
},
|
|
89
|
+
async listGithubRepositoryVisibility(repoIds) {
|
|
90
|
+
if (repoIds.length === 0) return []
|
|
91
|
+
const rows = await visibility.all(JSON.stringify(repoIds)) as (Omit<GithubRepositoryVisibility, 'public'> & { public: number })[]
|
|
92
|
+
return rows.map(row => ({ ...row, public: row.public === 1 }))
|
|
93
|
+
},
|
|
94
|
+
async setGithubRepositoryVisibility(entries) {
|
|
95
|
+
if (entries.length > 0) await setVisibility.run({ contexts: JSON.stringify(entries.map(entry => ({ ...entry, public: entry.public ? 1 : 0 }))) })
|
|
96
|
+
},
|
|
64
97
|
}
|
|
65
98
|
}
|