@preventive/triage 1.0.0-alpha.45 → 1.0.0-alpha.47
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/client/linked-findings.js +3 -0
- package/common/bundle-metadata.d.ts +1 -0
- package/common/bundle-metadata.js +1 -1
- package/common/bundle-sources.js +5 -1
- package/common/line-diff.js +5 -4
- package/common/managed/agent-tasks.ts +5 -3
- package/common/managed/github-agents.ts +28 -10
- package/out/activity.js +4 -0
- package/out/{chunk-NPW52BM2.js → chunk-4HI2MSAV.js} +13 -13
- package/out/chunk-5AHIQFFG.js +36 -0
- package/out/chunk-6ANSHRFQ.js +6 -0
- package/out/chunk-6WF3KNTL.js +3 -0
- package/out/chunk-7BKKYVWR.js +7 -0
- package/out/{chunk-HMAJDVXN.js → chunk-FH24VHSJ.js} +1 -1
- package/out/chunk-NOZJULWS.js +52 -0
- package/out/{chunk-OKRLLE5I.js → chunk-PWFUOLZK.js} +1 -1
- package/out/chunk-S6U2J62U.js +232 -0
- package/out/chunk-SPEFPML6.js +2 -0
- package/out/chunk-UENXT2HK.js +3 -0
- package/out/chunk-VL43BTSO.js +1 -0
- package/out/{chunk-3PS4KWD2.js → chunk-WCPJDAYA.js} +1 -1
- package/out/chunk-ZCS4UYHI.js +1 -0
- package/out/chunk-ZGNMDGSB.js +1 -0
- package/out/client-managed-import.js +1 -1
- package/out/client-managed.js +15 -15
- package/out/client-sync.js +2 -2
- package/out/compare.js +12 -12
- package/out/dialogs.js +28 -28
- package/out/graph.js +11 -11
- package/out/index.html +11 -11
- package/out/scan.js +2 -2
- package/out/terminal.js +6 -6
- package/out/view.css +1 -1
- package/out/view.js +15 -15
- package/package.json +4 -1
- package/risk/actions/findings.js +14 -9
- package/risk/actions/scripts/shell.js +5 -1
- package/risk/index.d.ts +9 -4
- package/risk/npm/assess.js +15 -4
- package/risk/npm/registry.js +6 -1
- package/server-e2e/app.ts +5 -4
- package/server-e2e/http.ts +5 -2
- package/server-managed/activity.ts +24 -11
- package/server-managed/app-agents.ts +10 -21
- package/server-managed/bundle-build-worker.js +7 -3
- package/server-managed/bundle-build.ts +5 -3
- package/server-managed/bundle-cache.ts +16 -4
- package/server-managed/bundle.ts +3 -4
- package/server-managed/combined.ts +3 -2
- package/server-managed/commit-coauthors.ts +5 -9
- package/server-managed/commit-lockfiles.ts +21 -11
- package/server-managed/config.ts +6 -0
- package/server-managed/db-methods.ts +321 -156
- package/server-managed/db-neon.ts +71 -24
- package/server-managed/db-schema.ts +4 -0
- package/server-managed/db.ts +2 -1
- package/server-managed/dependency-reviews.ts +13 -3
- package/server-managed/github-app.ts +12 -6
- package/server-managed/github-issue-links.ts +11 -5
- package/server-managed/github-pulls.ts +2 -2
- package/server-managed/github-requests.ts +39 -0
- package/server-managed/http-response.ts +10 -2
- package/server-managed/http.ts +269 -143
- package/server-managed/index.ts +8 -7
- package/server-managed/kept-briefly.ts +37 -0
- package/server-managed/link-reports.ts +5 -1
- package/server-managed/npm-delta.ts +11 -6
- package/server-managed/npm-kept.ts +51 -7
- package/server-managed/npm-loads.ts +117 -35
- package/server-managed/npm-packages.ts +107 -14
- package/server-managed/npm-risk.ts +68 -18
- package/server-managed/pretty-print.ts +112 -35
- package/server-managed/pull-dependencies.ts +29 -9
- package/server-managed/pull-dependency-reviews.ts +8 -8
- package/server-managed/queue.ts +16 -0
- package/server-managed/rename-diff.ts +22 -13
- package/server-managed/report-sources.ts +1 -1
- package/server-managed/report-store.ts +50 -12
- package/server-managed/repository-actions.ts +15 -6
- package/server-managed/repository-agents.ts +1 -1
- package/server-managed/repository-browser.ts +123 -33
- package/server-managed/repository-code.ts +28 -11
- package/server-managed/repository-dependency-reviews.ts +41 -19
- package/server-managed/repository-history.ts +24 -8
- package/server-managed/repository-pulse.ts +9 -2
- package/server-managed/repository-risk-audit.ts +13 -0
- package/server-managed/repository-risk-read.ts +11 -6
- package/server-managed/repository-risk.ts +27 -20
- package/server-managed/session.ts +18 -4
- package/server-managed/solidity-entry-points.ts +5 -3
- package/server-managed/sql-postgres.ts +3 -0
- package/server-managed/sql.ts +6 -2
- package/server-managed/storage-db.ts +11 -3
- package/server-managed/storage-encryption.ts +7 -4
- package/server-managed/storage.ts +8 -4
- package/server-managed/team-feed.ts +3 -1
- package/server-managed/team-reports.ts +30 -13
- package/server-managed/uploads.ts +29 -9
- package/server-managed/upstream-cache.ts +12 -0
- package/server.ts +14 -5
- package/out/chunk-5DHL5NG3.js +0 -2
- package/out/chunk-6HNXNIQY.js +0 -52
- package/out/chunk-AFLZH7WG.js +0 -1
- package/out/chunk-NV7ES4EW.js +0 -1
- package/out/chunk-PTGDYLNS.js +0 -36
- package/out/chunk-Q6MVYGYV.js +0 -3
- package/out/chunk-QRD6GIG5.js +0 -229
- package/out/chunk-WI7Z5A2T.js +0 -6
- package/out/chunk-WQ74ZBZF.js +0 -7
- package/out/chunk-XVBEOGOS.js +0 -3
|
@@ -109,6 +109,8 @@ export interface PullDependencyStore {
|
|
|
109
109
|
// scan's are (repository-risk.ts riskCaches).
|
|
110
110
|
type LockfileRead = { packages: LockedPackage[] } | { error: string; detail?: string }
|
|
111
111
|
const lockfileReads: Lru<LockfileRead> = lru(200_000, read => 'packages' in read ? read.packages.length + 1 : 1)
|
|
112
|
+
// Those being read, by the token reading them too, lest each asking at once read them.
|
|
113
|
+
const lockfilesReading = new Map<string, Promise<LockfileRead>>()
|
|
112
114
|
|
|
113
115
|
const api = (fullName: string) => `https://api.github.com/repos/${fullName.split('/').map(encodeURIComponent).join('/')}`
|
|
114
116
|
|
|
@@ -169,14 +171,26 @@ function lockfilesChanged(files: ChangedFile[]) {
|
|
|
169
171
|
omitted: all.slice(MAX_DIRECTORIES).flatMap(([, directory]) => [...directory.paths]).toSorted() }
|
|
170
172
|
}
|
|
171
173
|
|
|
172
|
-
//
|
|
173
|
-
//
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
174
|
+
// Whether `lockfile` at `ref` is held in this instance, read without asking
|
|
175
|
+
// GitHub: Bun's, never read, too.
|
|
176
|
+
export const lockfileHeld = (fullName: string, ref: string, { path, format }: { path: string; format: Format }) =>
|
|
177
|
+
format === 'bun' || lockfileReads.get(`${fullName}\0${ref}\0${path}`) !== undefined
|
|
178
|
+
|
|
179
|
+
// A lockfile's packages at `ref`, as read in this instance, else shared with
|
|
180
|
+
// a read of it with the same token still under way, else of GitHub; a
|
|
181
|
+
// lockfile its reader refuses, why. Any failure to read it of GitHub but its
|
|
182
|
+
// having none there is the whole answer's, as it would be every file's.
|
|
183
|
+
export function readLockfile(fullName: string, token: string, ref: string, lockfile: { path: string; format: Format }): Promise<LockfileRead> {
|
|
184
|
+
if (lockfile.format === 'bun') return Promise.resolve({ error: 'lockfile-unsupported' })
|
|
185
|
+
const key = `${fullName}\0${ref}\0${lockfile.path}`
|
|
178
186
|
const known = lockfileReads.get(key)
|
|
179
|
-
if (known) return known
|
|
187
|
+
if (known) return Promise.resolve(known)
|
|
188
|
+
const shared = `${token}\0${key}`
|
|
189
|
+
const reading = lockfilesReading.get(shared) ?? readOf(fullName, token, ref, lockfile, key).finally(() => lockfilesReading.delete(shared))
|
|
190
|
+
lockfilesReading.set(shared, reading)
|
|
191
|
+
return reading
|
|
192
|
+
}
|
|
193
|
+
async function readOf(fullName: string, token: string, ref: string, { path, format }: { path: string; format: Format }, key: string): Promise<LockfileRead> {
|
|
180
194
|
let text: string
|
|
181
195
|
try {
|
|
182
196
|
text = await upstreamGithub(() => createClient({ token, userAgent: 'deepview-triage' }).getRepoFile({ repo: fullName, path, ref }))
|
|
@@ -261,10 +275,16 @@ function addedIn(paths: string[], before: LockfileRead[], after: LockfileRead[])
|
|
|
261
275
|
// where it changes no lockfile read here.
|
|
262
276
|
export async function pullDependencies(fullName: string, token: string, number: number, kept: PullDependencyStore,
|
|
263
277
|
fetchImpl: typeof fetch = globalThis.fetch): Promise<PullDependencies> {
|
|
278
|
+
return (await pullDependenciesAt(fullName, token, number, kept, fetchImpl)).told
|
|
279
|
+
}
|
|
280
|
+
// The same, with the commits it is kept by (`commits`), its base's and
|
|
281
|
+
// head's, by which `kept` reads it.
|
|
282
|
+
export async function pullDependenciesAt(fullName: string, token: string, number: number, kept: PullDependencyStore,
|
|
283
|
+
fetchImpl: typeof fetch = globalThis.fetch): Promise<{ commits: string; told: PullDependencies }> {
|
|
264
284
|
const { base, head } = await pullCommits(fullName, token, number, fetchImpl)
|
|
265
285
|
const key = `${base}...${head}`
|
|
266
286
|
const held = await kept.read(key)
|
|
267
|
-
if (held) return held
|
|
287
|
+
if (held) return { commits: key, told: held }
|
|
268
288
|
const { mergeBase, files, cut: filesCut } = await pullFiles(fullName, token, number, base, head, fetchImpl)
|
|
269
289
|
const { directories, cut, omitted } = lockfilesChanged(files)
|
|
270
290
|
const reads = directories.flatMap(({ before, after }) => [...before.map(lockfile => [mergeBase, lockfile] as const), ...after.map(lockfile => [head, lockfile] as const)])
|
|
@@ -276,5 +296,5 @@ export async function pullDependencies(fullName: string, token: string, number:
|
|
|
276
296
|
})
|
|
277
297
|
const told = { base: mergeBase, head, lockfiles, cut: filesCut || cut, ...cut && !filesCut && omitted.length <= MAX_OMITTED ? { omitted } : {} }
|
|
278
298
|
await kept.write(key, told)
|
|
279
|
-
return told
|
|
299
|
+
return { commits: key, told }
|
|
280
300
|
}
|
|
@@ -24,7 +24,7 @@
|
|
|
24
24
|
// changes: it is read once, every lockfile there of those read here
|
|
25
25
|
// (pull-dependencies.ts), and kept by the commit (`baselines` in this
|
|
26
26
|
// instance, `kept` beyond it), never stored as reviews.
|
|
27
|
-
import { lockedAt, lockedIn } from './commit-lockfiles.ts'
|
|
27
|
+
import { type GithubAsked, lockedAt, lockedIn } from './commit-lockfiles.ts'
|
|
28
28
|
import { GithubApiError } from './github-app.ts'
|
|
29
29
|
import type { LockedPackage } from './locked-packages.ts'
|
|
30
30
|
import { EVERY_LOCKFILE, type LockfileScope, heldTo, isEvery, reviewedIn } from './lockfile-scope.ts'
|
|
@@ -68,13 +68,13 @@ const baselineKeys = (pkg: LockedPackage) => hashesOf(pkg).map(hash => `${pkg.na
|
|
|
68
68
|
// and kept; with the lockfiles there its reader refuses, isn't one of, or
|
|
69
69
|
// past those read (`unread`). GitHub having none of a lockfile it lists
|
|
70
70
|
// there is a failure of the whole, as of any file, never kept.
|
|
71
|
-
export async function reviewBaseline(fullName: string, token: string, repoId: number, since: string, kept: ReviewBaselineStore): Promise<Baseline> {
|
|
71
|
+
export async function reviewBaseline(fullName: string, token: string, repoId: number, since: string, kept: ReviewBaselineStore, asked?: GithubAsked): Promise<Baseline> {
|
|
72
72
|
const key = `${repoId}\0${since}`
|
|
73
73
|
const known = baselines.get(key)
|
|
74
74
|
if (known) return known
|
|
75
75
|
const stored = await kept.read(key)
|
|
76
76
|
if (stored) { const baseline = { versions: new Set(stored.versions), unread: stored.unread }; baselines.set(key, baseline); return baseline }
|
|
77
|
-
const read = await lockedAt(fullName, token, since)
|
|
77
|
+
const read = await lockedAt(fullName, token, since, () => true, asked)
|
|
78
78
|
const { packages } = lockedIn(read), { unread } = read
|
|
79
79
|
const versions = new Set<string>()
|
|
80
80
|
for (const pkg of packages) if (reviewedAs(pkg) === 'version') for (const held of baselineKeys(pkg)) versions.add(held)
|
|
@@ -298,18 +298,18 @@ export async function reviewedDependencies(told: PullDependencies, state: RepoRe
|
|
|
298
298
|
...unread.length > 0 && !reviewed ? { baselineUnread: unread.slice(0, MAX_UNREVIEWED) } : {}, ...isEvery(scope) ? {} : { lockfiles: scope } } }
|
|
299
299
|
}
|
|
300
300
|
|
|
301
|
-
// The
|
|
302
|
-
// pass, whose reviews are asked: those that can be reviewed, unpatched.
|
|
301
|
+
// The package versions a pull request's lockfiles lock that its reviews here
|
|
302
|
+
// may pass, whose reviews are asked: those that can be reviewed, unpatched.
|
|
303
303
|
export const reviewedPackages = (told: PullDependencies) => told.lockfiles
|
|
304
|
-
.flatMap(dir => 'error' in dir ? [] : dir.locked.filter(pkg => reviewedAs(pkg) === 'version' && !pkg.patchHash).map(pkg => pkg.name))
|
|
304
|
+
.flatMap(dir => 'error' in dir ? [] : dir.locked.filter(pkg => reviewedAs(pkg) === 'version' && !pkg.patchHash).map(pkg => ({ name: pkg.name, version: pkg.version! })))
|
|
305
305
|
|
|
306
306
|
// A repository's baseline as read for a pull request: its versions and the
|
|
307
307
|
// lockfiles there left unread; none where it reviews since no commit; and
|
|
308
308
|
// where GitHub doesn't read it, none and why.
|
|
309
|
-
export async function baselineFor(fullName: string, token: string, repoId: number, state: RepoReviewState, kept: ReviewBaselineStore) {
|
|
309
|
+
export async function baselineFor(fullName: string, token: string, repoId: number, state: RepoReviewState, kept: ReviewBaselineStore, asked?: GithubAsked) {
|
|
310
310
|
if (!state.since) return { baseline: null, unread: [], error: null }
|
|
311
311
|
try {
|
|
312
|
-
const { versions, unread } = await reviewBaseline(fullName, token, repoId, state.since, kept)
|
|
312
|
+
const { versions, unread } = await reviewBaseline(fullName, token, repoId, state.since, kept, asked)
|
|
313
313
|
return { baseline: versions, unread, error: null }
|
|
314
314
|
} catch (err) {
|
|
315
315
|
if (err instanceof GithubApiError && err.status !== 429 && err.status !== 401) return { baseline: null, unread: [], error: err.message }
|
package/server-managed/queue.ts
CHANGED
|
@@ -21,3 +21,19 @@ export function queue<T>(signal: AbortSignal, { drain = false } = {}): Queue<T>
|
|
|
21
21
|
},
|
|
22
22
|
}
|
|
23
23
|
}
|
|
24
|
+
|
|
25
|
+
// Work done `count` at a time, in the order asked, the rest waiting their
|
|
26
|
+
// turn.
|
|
27
|
+
export function inTurns(count: number): <T>(work: () => Promise<T>) => Promise<T> {
|
|
28
|
+
let active = 0
|
|
29
|
+
const waiting: (() => void)[] = []
|
|
30
|
+
return async work => {
|
|
31
|
+
if (active < count) active++
|
|
32
|
+
else await new Promise<void>(resolve => { waiting.push(resolve) })
|
|
33
|
+
try { return await work() } finally {
|
|
34
|
+
const next = waiting.shift()
|
|
35
|
+
if (next) next()
|
|
36
|
+
else active--
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
}
|
|
@@ -179,7 +179,10 @@ export async function renameDiffOf(request: RenameDiffRequest, texts: () => Prom
|
|
|
179
179
|
return opened(await answerOf(request, texts, keep))
|
|
180
180
|
}
|
|
181
181
|
|
|
182
|
-
|
|
182
|
+
// Sides `verified` to be the files the request names (renamedPairs) are read
|
|
183
|
+
// only where no answer is kept or on its way; others are read first, as
|
|
184
|
+
// reading them checks they are.
|
|
185
|
+
async function answerOf(request: RenameDiffRequest, texts: () => Promise<[string, string]>, keep: boolean, verified = false): Promise<Buffer> {
|
|
183
186
|
// Sides not kept are formatted again for each diff (pretty-print.ts), so
|
|
184
187
|
// they are read in its turn: the files a Diff view asks for at once wait
|
|
185
188
|
// here rather than being refused by the formats running. Within its minute
|
|
@@ -193,21 +196,27 @@ async function answerOf(request: RenameDiffRequest, texts: () => Promise<[string
|
|
|
193
196
|
})
|
|
194
197
|
return encoded(diff, request, sides)
|
|
195
198
|
}
|
|
196
|
-
const [before, after] = await texts()
|
|
197
199
|
// A file's type is part of how it is read: the same text as a script and as a module differ.
|
|
198
200
|
const key = `${request.base.hash}\0${request.base.extension}\0${request.other.hash}\0${request.other.extension}`
|
|
199
|
-
const
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
201
|
+
const held = () => {
|
|
202
|
+
const found = kept.get(key)
|
|
203
|
+
if (found) {
|
|
204
|
+
kept.delete(key)
|
|
205
|
+
kept.set(key, found)
|
|
206
|
+
}
|
|
207
|
+
return found ?? pending.get(key)
|
|
204
208
|
}
|
|
205
|
-
let
|
|
206
|
-
if (!
|
|
207
|
-
const
|
|
208
|
-
|
|
209
|
+
let answer = verified ? held() : undefined
|
|
210
|
+
if (!answer) {
|
|
211
|
+
const [before, after] = await texts()
|
|
212
|
+
answer = held()
|
|
213
|
+
if (!answer) {
|
|
214
|
+
const deadline = Date.now() + DIFF_TIMEOUT_MS
|
|
215
|
+
const job = inTurn(deadline, () => inWorker(before, after, request, deadline)).then(diff => encoded(diff, request, [before, after])).finally(() => pending.delete(key))
|
|
216
|
+
pending.set(key, answer = job)
|
|
217
|
+
}
|
|
209
218
|
}
|
|
210
|
-
const body = await
|
|
219
|
+
const body = await answer
|
|
211
220
|
kept.set(key, body)
|
|
212
221
|
if (kept.size > KEPT_DIFFS) kept.delete(kept.keys().next().value!)
|
|
213
222
|
return body
|
|
@@ -258,7 +267,7 @@ export function renamedPairs([base, other]: readonly [ComparedSide, ComparedSide
|
|
|
258
267
|
const named = { path: from.path, hash: from.hash, otherPath: to.path, otherHash: to.hash }
|
|
259
268
|
const copy = async (side: ComparedSide, { text, ...file }: typeof from) => prettyText(await side.kept(file) ?? await side.made(file, text))
|
|
260
269
|
let body
|
|
261
|
-
try { body = await answerOf(request, async () => [await copy(base, from), await copy(other, to)], keep) }
|
|
270
|
+
try { body = await answerOf(request, async () => [await copy(base, from), await copy(other, to)], keep, true) }
|
|
262
271
|
catch (err) {
|
|
263
272
|
if (!(err instanceof PrettyError)) throw err
|
|
264
273
|
yield { ...named, error: err.message, status: err.status }
|
|
@@ -146,7 +146,7 @@ export function createReportSourcesCache(storage: CacheStorage, db: ManagedDb, r
|
|
|
146
146
|
}
|
|
147
147
|
return {
|
|
148
148
|
async open(report: ReportRecord, bundle: ManagedBundle, permissions: ViewerPermissions, sourcePaths?: Set<string>) {
|
|
149
|
-
const repo = { github:
|
|
149
|
+
const repo = { github: report.repoId == null ? null : await db.getRepoFullName(report.repoId) }
|
|
150
150
|
const target = sourceCacheFilename(report, bundle, permissions, repo, sourcePaths)
|
|
151
151
|
try { return { ...await storage.open(target), repo } }
|
|
152
152
|
catch (error) { if (!(error instanceof CacheMissError)) throw error }
|
|
@@ -15,33 +15,71 @@ async function readBytes(stored: OpenedBlob): Promise<Buffer> {
|
|
|
15
15
|
} finally { stored.stream.destroy() }
|
|
16
16
|
}
|
|
17
17
|
|
|
18
|
+
// Ids whose original copy is known gone, at most this many.
|
|
19
|
+
const COMPRESSED_ONLY_MAX = 16_384
|
|
20
|
+
|
|
18
21
|
export function createReportStore(legacy: BlobStore, compressed: BlobStore,
|
|
19
22
|
replaceLegacy: (id: string, bytes: Buffer) => Promise<unknown>): BlobStore {
|
|
23
|
+
// Reports this store wrote, converted, or found with only the compressed
|
|
24
|
+
// copy. Uploads write only that copy and nothing writes an original again,
|
|
25
|
+
// so their reads stop asking storage for one, each a miss (and, encrypted,
|
|
26
|
+
// a row read too). A compressed copy missing there asks for the original
|
|
27
|
+
// after all. An original left by a conversion that failed is never in it:
|
|
28
|
+
// the next read finds that original and resumes the conversion.
|
|
29
|
+
const compressedOnly = new Set<string>()
|
|
30
|
+
function remember(id: string): void {
|
|
31
|
+
compressedOnly.delete(id)
|
|
32
|
+
if (compressedOnly.size >= COMPRESSED_ONLY_MAX) compressedOnly.delete(compressedOnly.values().next().value!)
|
|
33
|
+
compressedOnly.add(id)
|
|
34
|
+
}
|
|
35
|
+
function decoded(stored: OpenedBlob): OpenedBlob {
|
|
36
|
+
const stream = createBrotliDecompress()
|
|
37
|
+
// pipeline carries storage/authentication errors into the decoded stream
|
|
38
|
+
// and closes the stored source when a reader leaves before EOF.
|
|
39
|
+
void pipeline(stored.stream, stream).catch(() => {})
|
|
40
|
+
return { size: null, stream }
|
|
41
|
+
}
|
|
20
42
|
async function open(id: string): Promise<OpenedBlob | null> {
|
|
43
|
+
if (compressedOnly.has(id)) {
|
|
44
|
+
const stored = await compressed.open(id)
|
|
45
|
+
if (stored) return decoded(stored)
|
|
46
|
+
compressedOnly.delete(id)
|
|
47
|
+
}
|
|
21
48
|
// Keep the original available until conversion succeeds. Concurrent reads
|
|
22
49
|
// can finish using an already-open stream while another reader converts it.
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
50
|
+
// Both copies are asked for at once, the original answering where it is
|
|
51
|
+
// still there: one round trip of storage for a report not known yet, as
|
|
52
|
+
// a cold instance knows none.
|
|
53
|
+
const [original, stored] = await Promise.allSettled([legacy.open(id), compressed.open(id)])
|
|
54
|
+
const unused = (copy: PromiseSettledResult<OpenedBlob | null>) => { if (copy.status === 'fulfilled') copy.value?.stream.destroy() }
|
|
55
|
+
if (original.status === 'rejected') { unused(stored); throw original.reason }
|
|
56
|
+
if (original.value) {
|
|
57
|
+
unused(stored)
|
|
58
|
+
const bytes = await readBytes(original.value)
|
|
26
59
|
await replaceLegacy(id, await encodeBrotli(bytes, 9))
|
|
60
|
+
remember(id)
|
|
27
61
|
return { size: bytes.length, stream: Readable.from([bytes]) }
|
|
28
62
|
}
|
|
29
|
-
|
|
30
|
-
if (!stored) return null
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
// and closes the stored source when a reader leaves before EOF.
|
|
34
|
-
void pipeline(stored.stream, stream).catch(() => {})
|
|
35
|
-
return { size: null, stream }
|
|
63
|
+
if (stored.status === 'rejected') throw stored.reason
|
|
64
|
+
if (!stored.value) return null
|
|
65
|
+
remember(id)
|
|
66
|
+
return decoded(stored.value)
|
|
36
67
|
}
|
|
37
68
|
return {
|
|
38
|
-
|
|
69
|
+
async put(id, bytes) {
|
|
70
|
+
const wrapped = await compressed.put(id, await encodeBrotli(bytes, 9))
|
|
71
|
+
remember(id)
|
|
72
|
+
return wrapped
|
|
73
|
+
},
|
|
39
74
|
open,
|
|
40
75
|
async get(id) {
|
|
41
76
|
const stored = await open(id)
|
|
42
77
|
if (!stored) return null
|
|
43
78
|
return readBytes(stored)
|
|
44
79
|
},
|
|
45
|
-
async delete(id) {
|
|
80
|
+
async delete(id) {
|
|
81
|
+
compressedOnly.delete(id)
|
|
82
|
+
await Promise.all([compressed.delete(id), legacy.delete(id)])
|
|
83
|
+
},
|
|
46
84
|
}
|
|
47
85
|
}
|
|
@@ -92,7 +92,8 @@ export function actionsLines(repo: number, owner: string, read: WorkflowsRead, l
|
|
|
92
92
|
if ('error' in read) return { lines: [{ type: 'workflows', repo, ...read } as WorkflowsLine], refs: [] as ActionRef[] }
|
|
93
93
|
const lines: (WorkflowsLine | WorkflowLine)[] = [{ type: 'workflows', repo, paths: read.files.filter(file => file.kind === 'workflow').map(file => file.path) }]
|
|
94
94
|
const refs = new Map<string, ActionRef>()
|
|
95
|
-
|
|
95
|
+
// Its own actions by their directories and workflows by their paths, as a step or a job uses one.
|
|
96
|
+
const actions = new Map(read.files.flatMap(file => file.kind !== 'dockerfile' && 'text' in file ? [[file.kind === 'action' ? actionDir(file.path) : file.path, file.text] as const] : []))
|
|
96
97
|
for (const file of read.files) {
|
|
97
98
|
const scan = 'error' in file ? file : file.kind === 'dockerfile' ? { uses: [], findings: dockerfileFindings(file.text) } : scanWorkflow(file.kind, file.text, { owner, lockfiles, actions })
|
|
98
99
|
lines.push({ type: 'workflow', repo, path: file.path, kind: file.kind, ...scan })
|
|
@@ -151,10 +152,12 @@ export interface ActionsResolving {
|
|
|
151
152
|
store?: KeptStore<ActionLine> | undefined
|
|
152
153
|
}
|
|
153
154
|
|
|
154
|
-
// A reference's line: kept
|
|
155
|
-
// repository's; why not, where
|
|
156
|
-
|
|
157
|
-
|
|
155
|
+
// A reference's line: kept (`stored`, as read of the store with others),
|
|
156
|
+
// else resolved, and kept where it is a public repository's; why not, where
|
|
157
|
+
// it couldn't be.
|
|
158
|
+
async function resolved(resolving: ActionsResolving, source: ActionsSource, { key, owner }: ActionRef,
|
|
159
|
+
stored = resolving.store?.read(key)): Promise<ActionLine | null> {
|
|
160
|
+
const kept = resolving.memory.get(key) ?? await stored?.then(row => row && { at: row.at, line: row.value })
|
|
158
161
|
if (kept && Date.now() - kept.at < RISK_KEPT_MS) { resolving.memory.set(key, kept); return kept.line }
|
|
159
162
|
const uses = usesOfKey(key)
|
|
160
163
|
if (!uses) return { type: 'action', key, error: 'invalid-uses' }
|
|
@@ -180,6 +183,9 @@ export function resolvingActions(resolving: ActionsResolving) {
|
|
|
180
183
|
const source = keptSource(sharedSource(resolving.source, resolving.answers))
|
|
181
184
|
const pending: Queue<ActionRef> = queue(resolving.signal)
|
|
182
185
|
const latest = new Map<string, ActionRef>()
|
|
186
|
+
// Those of the references added at once kept beyond this instance, read of
|
|
187
|
+
// the store together as they are added.
|
|
188
|
+
const stored = new Map<string, Promise<{ at: number; value: ActionLine } | null>>()
|
|
183
189
|
let closed = false, waiting = 0
|
|
184
190
|
const settle = () => { if (closed && waiting === 0) pending.close() }
|
|
185
191
|
const add = (refs: ActionRef[]) => {
|
|
@@ -191,11 +197,14 @@ export function resolvingActions(resolving: ActionsResolving) {
|
|
|
191
197
|
waiting++
|
|
192
198
|
pending.add(kept)
|
|
193
199
|
}
|
|
200
|
+
const unread = [...new Set(refs.map(ref => ref.key))].filter(key => latest.has(key) && !stored.has(key) && !resolving.memory.get(key))
|
|
201
|
+
const read = unread.length > 0 ? resolving.store?.readMany(unread) : undefined
|
|
202
|
+
for (const key of read ? unread : []) stored.set(key, read!.then(found => found.get(key) ?? null))
|
|
194
203
|
}
|
|
195
204
|
const work = async () => {
|
|
196
205
|
for (let ref = await pending.take(); ref !== undefined; ref = await pending.take()) {
|
|
197
206
|
try {
|
|
198
|
-
const line = await resolved(resolving, source, ref)
|
|
207
|
+
const line = await resolved(resolving, source, ref, stored.get(ref.key))
|
|
199
208
|
if (line && latest.get(ref.key) === ref) {
|
|
200
209
|
resolving.tell(line)
|
|
201
210
|
const owner = 'repository' in line ? (line.repository?.fullName ?? line.repo).split('/')[0]! : ref.owner
|
|
@@ -175,7 +175,7 @@ async function agentBranches(owner: string, name: string, prefix: string, token:
|
|
|
175
175
|
// What its answers are kept by, with their cursors (http.ts
|
|
176
176
|
// handleRepositoryAgents): bumped as what a page reads, or how it tells an
|
|
177
177
|
// agent, changes, so that no answer an older server kept is sent for a day.
|
|
178
|
-
export const AGENTS_READ_VERSION =
|
|
178
|
+
export const AGENTS_READ_VERSION = 6
|
|
179
179
|
|
|
180
180
|
// A cursor of the default branch's commits: GitHub names one by a commit
|
|
181
181
|
// and how many on from it, `<oid> <offset>`, a space between.
|
|
@@ -1,6 +1,7 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto'
|
|
1
2
|
import type { ManagedConfig } from './config.ts'
|
|
2
3
|
import type { SelectedRepo } from './db.ts'
|
|
3
|
-
import { GithubApiError, type GithubIdentity, READ_PERMISSIONS, githubJson, githubRepoPermission, githubUserIdentity, repoAccessToken } from './github-app.ts'
|
|
4
|
+
import { GithubApiError, type GithubIdentity, READ_PERMISSIONS, githubJson, githubRepoPermission, githubUserIdentity, onTokenRejected, repoAccessToken } from './github-app.ts'
|
|
4
5
|
import { MAX_PACKAGE_BYTES, readPackageEntryPoints } from './package-entry-points.ts'
|
|
5
6
|
import { readSolidityEntryPoints } from './solidity-entry-points.ts'
|
|
6
7
|
import { parseGithubCommit } from './bundle-commits.ts'
|
|
@@ -40,14 +41,63 @@ function credentialFailure(err: unknown): err is GithubApiError {
|
|
|
40
41
|
return err instanceof GithubApiError && ([401, 404].includes(err.status) || err.message === 'github-status-403')
|
|
41
42
|
}
|
|
42
43
|
|
|
43
|
-
//
|
|
44
|
-
//
|
|
44
|
+
// GitHub's answer on a user's access to a repository, as checkAccess reads it:
|
|
45
|
+
// the repository's metadata, which credential reads it (the user's token, the
|
|
46
|
+
// App installation's, or none, as for a public repository read anonymously)
|
|
47
|
+
// and the permission level GitHub told, where it was asked. Kept per user, by
|
|
48
|
+
// their GitHub id and token, and per repository, by its id, name and
|
|
49
|
+
// installation, for config.repositoryAccessCacheMs; only access granted is kept.
|
|
50
|
+
// One answered for a user with a token is kept no longer once GitHub rejects
|
|
51
|
+
// that token (`userToken`, hashed), whichever credential reads it: a private
|
|
52
|
+
// repository's was granted on who that token told GitHub they are. The next
|
|
53
|
+
// check asks GitHub, falling back to the installation's or none as it does.
|
|
54
|
+
type RepoMetadata = { id?: unknown; full_name?: unknown; private?: unknown; visibility?: unknown; default_branch?: unknown }
|
|
55
|
+
type AccessVerdict = { until: number; data: RepoMetadata; credential: 'user' | 'app' | 'none'; level: string | null; userToken: string | null }
|
|
56
|
+
type VerdictCache = { get(): AccessVerdict | undefined; set(verdict: Omit<AccessVerdict, 'until' | 'userToken'>): void }
|
|
57
|
+
const MAX_ACCESS_VERDICTS = 1000
|
|
58
|
+
// By the fetch reading GitHub, as installation tokens are: a test's own.
|
|
59
|
+
const accessVerdicts = new WeakMap<typeof fetch, Map<string, AccessVerdict>>()
|
|
60
|
+
const hashed = (token: string) => createHash('sha256').update(token).digest('hex')
|
|
61
|
+
onTokenRejected((token, fetchImpl) => {
|
|
62
|
+
const kept = accessVerdicts.get(fetchImpl)
|
|
63
|
+
if (!kept) return
|
|
64
|
+
const rejected = hashed(token)
|
|
65
|
+
for (const [key, verdict] of kept) if (verdict.userToken === rejected) kept.delete(key)
|
|
66
|
+
})
|
|
67
|
+
|
|
68
|
+
function verdictCache(config: ManagedConfig, user: RepositoryBrowserUser, repo: SelectedRepo, fetchImpl: typeof fetch): VerdictCache | null {
|
|
69
|
+
const keepMs = config.repositoryAccessCacheMs
|
|
70
|
+
if (!(keepMs > 0)) return null
|
|
71
|
+
let verdicts = accessVerdicts.get(fetchImpl)
|
|
72
|
+
if (!verdicts) { verdicts = new Map(); accessVerdicts.set(fetchImpl, verdicts) }
|
|
73
|
+
const kept = verdicts
|
|
74
|
+
// The token hashed with the rest, never kept as it is.
|
|
75
|
+
const key = createHash('sha256').update(JSON.stringify([user.githubUserId, user.token, repo.repoId, repo.fullName.toLowerCase(), repo.installationId, config.githubAppId])).digest('hex')
|
|
76
|
+
return {
|
|
77
|
+
get() {
|
|
78
|
+
const verdict = kept.get(key)
|
|
79
|
+
if (verdict && verdict.until > Date.now()) return verdict
|
|
80
|
+
kept.delete(key)
|
|
81
|
+
return undefined
|
|
82
|
+
},
|
|
83
|
+
set(verdict) {
|
|
84
|
+
kept.delete(key)
|
|
85
|
+
kept.set(key, { ...verdict, until: Date.now() + keepMs, userToken: user.token ? hashed(user.token) : null })
|
|
86
|
+
while (kept.size > MAX_ACCESS_VERDICTS) kept.delete(kept.keys().next().value!)
|
|
87
|
+
},
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
// Create once per HTTP request. Coalesce concurrent credential lookups, and
|
|
92
|
+
// keep GitHub's answer on each user's access to each repository a while
|
|
93
|
+
// (verdictCache), never sharing credentials or answers across users.
|
|
45
94
|
export function createRepositoryBrowser(config: ManagedConfig, user: RepositoryBrowserUser, fetchImpl: typeof fetch = globalThis.fetch) {
|
|
46
95
|
const tokens = new Map<number, Promise<string | null>>()
|
|
47
96
|
let identity: Promise<GithubIdentity> | undefined
|
|
48
|
-
|
|
97
|
+
// The user as GitHub tells them, asked once a request.
|
|
98
|
+
const getIdentity = (): Promise<GithubIdentity> => {
|
|
49
99
|
if (!user.token || user.githubUserId == null) return Promise.reject(new GithubApiError(404, 'no-repository'))
|
|
50
|
-
if (!identity
|
|
100
|
+
if (!identity) {
|
|
51
101
|
identity = githubUserIdentity(user.token, fetchImpl).then(value => {
|
|
52
102
|
if (value.id !== user.githubUserId) throw new GithubApiError(404, 'no-repository')
|
|
53
103
|
return value
|
|
@@ -71,7 +121,7 @@ export function createRepositoryBrowser(config: ManagedConfig, user: RepositoryB
|
|
|
71
121
|
// Its Dependabot alerts, with the token of the installation GitHub tells
|
|
72
122
|
// covers it, where the user's own access lets them see them
|
|
73
123
|
// (dependabot-access.ts).
|
|
74
|
-
return repositoryReader(repo, user.token, installationToken, getIdentity, fetchImpl).then(reader => ({ ...reader,
|
|
124
|
+
return repositoryReader(repo, user.token, installationToken, getIdentity, fetchImpl, verdictCache(config, user, repo, fetchImpl)).then(reader => ({ ...reader,
|
|
75
125
|
dependabotAlerts: () => appDependabotAlerts(config, repo, { identity: getIdentity, permission: reader.permission }, fetchImpl) }))
|
|
76
126
|
},
|
|
77
127
|
}
|
|
@@ -96,48 +146,86 @@ export async function repositoryTreeFiles(fullName: string, token: string | null
|
|
|
96
146
|
return { files, bytes }
|
|
97
147
|
}
|
|
98
148
|
|
|
99
|
-
|
|
149
|
+
type Access = { data: RepoMetadata; token: string | null; level: string | null }
|
|
150
|
+
|
|
151
|
+
// GitHub asked whether the user may read `repo`: its metadata, read with
|
|
152
|
+
// `token` (the reader's credential so far), else the installation's, else
|
|
153
|
+
// none; and of one not public, the user's permission, read with the
|
|
154
|
+
// installation's. What it told, the token reading it, and the level asked.
|
|
155
|
+
async function askAccess(repo: SelectedRepo, start: string | null, userToken: string | null, installationToken: () => Promise<string | null>,
|
|
156
|
+
getIdentity: () => Promise<GithubIdentity>, fetchImpl: typeof fetch): Promise<Access> {
|
|
100
157
|
const base = `https://api.github.com/repos/${repo.fullName.split('/').map(encodeURIComponent).join('/')}`
|
|
101
|
-
|
|
102
|
-
// Nonpublic reads still require the repository App and the effective-user gate,
|
|
103
|
-
// whose permission level is kept (`level`).
|
|
104
|
-
let level: string | null = null, token = userToken ?? await installationToken()
|
|
105
|
-
const read = (suffix: string) => githubJson(base + suffix, token, fetchImpl)
|
|
158
|
+
let token = start
|
|
106
159
|
const metadata = async () => {
|
|
107
|
-
try { return await
|
|
160
|
+
try { return await githubJson(base, token, fetchImpl) }
|
|
108
161
|
catch (err) {
|
|
109
162
|
if (!token || !credentialFailure(err)) throw err
|
|
110
163
|
const fallback = await installationToken()
|
|
111
164
|
if (fallback && fallback !== token) {
|
|
112
165
|
token = fallback
|
|
113
|
-
try { return await
|
|
166
|
+
try { return await githubJson(base, token, fetchImpl) } catch (fallbackError) { if (!credentialFailure(fallbackError)) throw fallbackError }
|
|
114
167
|
}
|
|
115
168
|
// Anonymous fallback is only for unavailable credentials, never rate
|
|
116
169
|
// limits, network errors, or GitHub outages.
|
|
117
170
|
token = null
|
|
118
|
-
return
|
|
171
|
+
return githubJson(base, token, fetchImpl)
|
|
119
172
|
}
|
|
120
173
|
}
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
174
|
+
// A repository last known private asks who the user is as it asks for its
|
|
175
|
+
// metadata, not after: its permission is asked of them.
|
|
176
|
+
if (repo.private && userToken) getIdentity().catch(() => {})
|
|
177
|
+
// Stored visibility can be stale (or internal with private=false). Require
|
|
178
|
+
// current, explicit public visibility or the caller's effective permission.
|
|
179
|
+
const data = await metadata() as RepoMetadata
|
|
180
|
+
if (data?.id !== repo.repoId || typeof data.full_name !== 'string' || data.full_name.toLowerCase() !== repo.fullName.toLowerCase()) {
|
|
181
|
+
throw new GithubApiError(404, 'no-repository')
|
|
182
|
+
}
|
|
183
|
+
if (data.private === false && data.visibility === 'public') return { data, token, level: null }
|
|
184
|
+
if (!token) throw new GithubApiError(404, 'no-repository')
|
|
185
|
+
const appToken = await installationToken()
|
|
186
|
+
if (!appToken) throw new GithubApiError(404, 'no-repository')
|
|
187
|
+
const level = await githubRepoPermission(repo.fullName, appToken, await getIdentity(), fetchImpl)
|
|
188
|
+
if (!READ_PERMISSIONS.has(level)) throw new GithubApiError(404, 'no-repository')
|
|
189
|
+
return { data, token: appToken, level }
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
// GitHub's answer as kept (verdictCache), with the credential it chose,
|
|
193
|
+
// where that credential is still had; else asked (`ask`), and kept where it
|
|
194
|
+
// grants access.
|
|
195
|
+
async function keptAccess(verdicts: VerdictCache | null, userToken: string | null, installationToken: () => Promise<string | null>,
|
|
196
|
+
ask: () => Promise<Access>): Promise<Access> {
|
|
197
|
+
const kept = verdicts?.get()
|
|
198
|
+
if (kept) {
|
|
199
|
+
const credential = kept.credential === 'user' ? userToken : kept.credential === 'app' ? await installationToken() : null
|
|
200
|
+
if (kept.credential === 'none' || credential) return { data: kept.data, token: credential, level: kept.level }
|
|
201
|
+
}
|
|
202
|
+
const asked = await ask()
|
|
203
|
+
const { id, full_name, private: hidden, visibility, default_branch } = asked.data
|
|
204
|
+
const data = { id, full_name, private: hidden, visibility, default_branch }
|
|
205
|
+
verdicts?.set({ data, credential: asked.token === null ? 'none' : asked.token === userToken ? 'user' : 'app', level: asked.level })
|
|
206
|
+
return { ...asked, data }
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
async function repositoryReader(repo: SelectedRepo, userToken: string | null, installationToken: () => Promise<string | null>, getIdentity: (fresh?: boolean) => Promise<GithubIdentity>, fetchImpl: typeof fetch,
|
|
210
|
+
verdicts: VerdictCache | null) {
|
|
211
|
+
const base = `https://api.github.com/repos/${repo.fullName.split('/').map(encodeURIComponent).join('/')}`
|
|
212
|
+
// Public reads can use the signed-in user's quota without repository grants.
|
|
213
|
+
// Nonpublic reads still require the repository App and the effective-user gate,
|
|
214
|
+
// whose permission level is kept (`level`).
|
|
215
|
+
let level: string | null = null, token = userToken ?? await installationToken()
|
|
216
|
+
const read = (suffix: string) => githubJson(base + suffix, token, fetchImpl)
|
|
217
|
+
const checkAccess = async (): Promise<RepoMetadata> => {
|
|
218
|
+
const access = await keptAccess(verdicts, userToken, installationToken, () => askAccess(repo, token, userToken, installationToken, getIdentity, fetchImpl))
|
|
219
|
+
token = access.token
|
|
220
|
+
level = access.level
|
|
221
|
+
return access.data
|
|
136
222
|
}
|
|
137
223
|
const current = await checkAccess()
|
|
138
224
|
const defaultBranch = typeof current.default_branch === 'string' ? current.default_branch : ''
|
|
139
225
|
return {
|
|
140
|
-
|
|
226
|
+
// Its visibility and the user's permission asked again, the user as
|
|
227
|
+
// GitHub told them earlier in the request, not asked again.
|
|
228
|
+
recheckAccess: () => checkAccess(),
|
|
141
229
|
// Server-only credential, selected by the same live visibility/permission
|
|
142
230
|
// gate as directory browsing. Never included in an HTTP response.
|
|
143
231
|
readToken: () => token,
|
|
@@ -184,8 +272,10 @@ async function repositoryReader(repo: SelectedRepo, userToken: string | null, in
|
|
|
184
272
|
const manifestPath = path ? `${path}/package.json` : 'package.json'
|
|
185
273
|
const manifest = data.find(item => item?.path === manifestPath && item.name === 'package.json' && item.type === 'file' && !item.submodule_git_url
|
|
186
274
|
&& typeof item.size === 'number' && item.size >= 0 && item.size <= MAX_PACKAGE_BYTES && typeof item.sha === 'string' && /^[a-f\d]{40}$/iu.test(item.sha))
|
|
187
|
-
|
|
188
|
-
|
|
275
|
+
// Both optional suggestions, read together: neither fails, each
|
|
276
|
+
// answering none where its reads do.
|
|
277
|
+
const [packageEntryPoints, solidity] = await Promise.all([manifest ? readPackageEntryPoints(manifest.sha, path, read) : undefined,
|
|
278
|
+
readSolidityEntryPoints(path, data, read)])
|
|
189
279
|
return { entries, limited: data.length >= 1000, ...(packageEntryPoints ? { packageEntryPoints } : {}),
|
|
190
280
|
...(solidity.paths.length > 0 || solidity.limited ? { solidityEntryPoints: solidity.paths, soliditySuggestionsLimited: solidity.limited } : {}),
|
|
191
281
|
}
|