@preventive/triage 1.0.0-alpha.45 → 1.0.0-alpha.47

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. package/client/linked-findings.js +3 -0
  2. package/common/bundle-metadata.d.ts +1 -0
  3. package/common/bundle-metadata.js +1 -1
  4. package/common/bundle-sources.js +5 -1
  5. package/common/line-diff.js +5 -4
  6. package/common/managed/agent-tasks.ts +5 -3
  7. package/common/managed/github-agents.ts +28 -10
  8. package/out/activity.js +4 -0
  9. package/out/{chunk-NPW52BM2.js → chunk-4HI2MSAV.js} +13 -13
  10. package/out/chunk-5AHIQFFG.js +36 -0
  11. package/out/chunk-6ANSHRFQ.js +6 -0
  12. package/out/chunk-6WF3KNTL.js +3 -0
  13. package/out/chunk-7BKKYVWR.js +7 -0
  14. package/out/{chunk-HMAJDVXN.js → chunk-FH24VHSJ.js} +1 -1
  15. package/out/chunk-NOZJULWS.js +52 -0
  16. package/out/{chunk-OKRLLE5I.js → chunk-PWFUOLZK.js} +1 -1
  17. package/out/chunk-S6U2J62U.js +232 -0
  18. package/out/chunk-SPEFPML6.js +2 -0
  19. package/out/chunk-UENXT2HK.js +3 -0
  20. package/out/chunk-VL43BTSO.js +1 -0
  21. package/out/{chunk-3PS4KWD2.js → chunk-WCPJDAYA.js} +1 -1
  22. package/out/chunk-ZCS4UYHI.js +1 -0
  23. package/out/chunk-ZGNMDGSB.js +1 -0
  24. package/out/client-managed-import.js +1 -1
  25. package/out/client-managed.js +15 -15
  26. package/out/client-sync.js +2 -2
  27. package/out/compare.js +12 -12
  28. package/out/dialogs.js +28 -28
  29. package/out/graph.js +11 -11
  30. package/out/index.html +11 -11
  31. package/out/scan.js +2 -2
  32. package/out/terminal.js +6 -6
  33. package/out/view.css +1 -1
  34. package/out/view.js +15 -15
  35. package/package.json +4 -1
  36. package/risk/actions/findings.js +14 -9
  37. package/risk/actions/scripts/shell.js +5 -1
  38. package/risk/index.d.ts +9 -4
  39. package/risk/npm/assess.js +15 -4
  40. package/risk/npm/registry.js +6 -1
  41. package/server-e2e/app.ts +5 -4
  42. package/server-e2e/http.ts +5 -2
  43. package/server-managed/activity.ts +24 -11
  44. package/server-managed/app-agents.ts +10 -21
  45. package/server-managed/bundle-build-worker.js +7 -3
  46. package/server-managed/bundle-build.ts +5 -3
  47. package/server-managed/bundle-cache.ts +16 -4
  48. package/server-managed/bundle.ts +3 -4
  49. package/server-managed/combined.ts +3 -2
  50. package/server-managed/commit-coauthors.ts +5 -9
  51. package/server-managed/commit-lockfiles.ts +21 -11
  52. package/server-managed/config.ts +6 -0
  53. package/server-managed/db-methods.ts +321 -156
  54. package/server-managed/db-neon.ts +71 -24
  55. package/server-managed/db-schema.ts +4 -0
  56. package/server-managed/db.ts +2 -1
  57. package/server-managed/dependency-reviews.ts +13 -3
  58. package/server-managed/github-app.ts +12 -6
  59. package/server-managed/github-issue-links.ts +11 -5
  60. package/server-managed/github-pulls.ts +2 -2
  61. package/server-managed/github-requests.ts +39 -0
  62. package/server-managed/http-response.ts +10 -2
  63. package/server-managed/http.ts +269 -143
  64. package/server-managed/index.ts +8 -7
  65. package/server-managed/kept-briefly.ts +37 -0
  66. package/server-managed/link-reports.ts +5 -1
  67. package/server-managed/npm-delta.ts +11 -6
  68. package/server-managed/npm-kept.ts +51 -7
  69. package/server-managed/npm-loads.ts +117 -35
  70. package/server-managed/npm-packages.ts +107 -14
  71. package/server-managed/npm-risk.ts +68 -18
  72. package/server-managed/pretty-print.ts +112 -35
  73. package/server-managed/pull-dependencies.ts +29 -9
  74. package/server-managed/pull-dependency-reviews.ts +8 -8
  75. package/server-managed/queue.ts +16 -0
  76. package/server-managed/rename-diff.ts +22 -13
  77. package/server-managed/report-sources.ts +1 -1
  78. package/server-managed/report-store.ts +50 -12
  79. package/server-managed/repository-actions.ts +15 -6
  80. package/server-managed/repository-agents.ts +1 -1
  81. package/server-managed/repository-browser.ts +123 -33
  82. package/server-managed/repository-code.ts +28 -11
  83. package/server-managed/repository-dependency-reviews.ts +41 -19
  84. package/server-managed/repository-history.ts +24 -8
  85. package/server-managed/repository-pulse.ts +9 -2
  86. package/server-managed/repository-risk-audit.ts +13 -0
  87. package/server-managed/repository-risk-read.ts +11 -6
  88. package/server-managed/repository-risk.ts +27 -20
  89. package/server-managed/session.ts +18 -4
  90. package/server-managed/solidity-entry-points.ts +5 -3
  91. package/server-managed/sql-postgres.ts +3 -0
  92. package/server-managed/sql.ts +6 -2
  93. package/server-managed/storage-db.ts +11 -3
  94. package/server-managed/storage-encryption.ts +7 -4
  95. package/server-managed/storage.ts +8 -4
  96. package/server-managed/team-feed.ts +3 -1
  97. package/server-managed/team-reports.ts +30 -13
  98. package/server-managed/uploads.ts +29 -9
  99. package/server-managed/upstream-cache.ts +12 -0
  100. package/server.ts +14 -5
  101. package/out/chunk-5DHL5NG3.js +0 -2
  102. package/out/chunk-6HNXNIQY.js +0 -52
  103. package/out/chunk-AFLZH7WG.js +0 -1
  104. package/out/chunk-NV7ES4EW.js +0 -1
  105. package/out/chunk-PTGDYLNS.js +0 -36
  106. package/out/chunk-Q6MVYGYV.js +0 -3
  107. package/out/chunk-QRD6GIG5.js +0 -229
  108. package/out/chunk-WI7Z5A2T.js +0 -6
  109. package/out/chunk-WQ74ZBZF.js +0 -7
  110. package/out/chunk-XVBEOGOS.js +0 -3
@@ -109,6 +109,8 @@ export interface PullDependencyStore {
109
109
  // scan's are (repository-risk.ts riskCaches).
110
110
  type LockfileRead = { packages: LockedPackage[] } | { error: string; detail?: string }
111
111
  const lockfileReads: Lru<LockfileRead> = lru(200_000, read => 'packages' in read ? read.packages.length + 1 : 1)
112
+ // Those being read, by the token reading them too, lest each asking at once read them.
113
+ const lockfilesReading = new Map<string, Promise<LockfileRead>>()
112
114
 
113
115
  const api = (fullName: string) => `https://api.github.com/repos/${fullName.split('/').map(encodeURIComponent).join('/')}`
114
116
 
@@ -169,14 +171,26 @@ function lockfilesChanged(files: ChangedFile[]) {
169
171
  omitted: all.slice(MAX_DIRECTORIES).flatMap(([, directory]) => [...directory.paths]).toSorted() }
170
172
  }
171
173
 
172
- // A lockfile's packages at `ref`, as read in this instance, else of GitHub;
173
- // a lockfile its reader refuses, why. Any failure to read it of GitHub but
174
- // its having none there is the whole answer's, as it would be every file's.
175
- export async function readLockfile(fullName: string, token: string, ref: string, { path, format }: { path: string; format: Format }): Promise<LockfileRead> {
176
- if (format === 'bun') return { error: 'lockfile-unsupported' }
177
- const key = `${fullName}\0${ref}\0${path}`
174
+ // Whether `lockfile` at `ref` is held in this instance, read without asking
175
+ // GitHub: Bun's, never read, too.
176
+ export const lockfileHeld = (fullName: string, ref: string, { path, format }: { path: string; format: Format }) =>
177
+ format === 'bun' || lockfileReads.get(`${fullName}\0${ref}\0${path}`) !== undefined
178
+
179
+ // A lockfile's packages at `ref`, as read in this instance, else shared with
180
+ // a read of it with the same token still under way, else of GitHub; a
181
+ // lockfile its reader refuses, why. Any failure to read it of GitHub but its
182
+ // having none there is the whole answer's, as it would be every file's.
183
+ export function readLockfile(fullName: string, token: string, ref: string, lockfile: { path: string; format: Format }): Promise<LockfileRead> {
184
+ if (lockfile.format === 'bun') return Promise.resolve({ error: 'lockfile-unsupported' })
185
+ const key = `${fullName}\0${ref}\0${lockfile.path}`
178
186
  const known = lockfileReads.get(key)
179
- if (known) return known
187
+ if (known) return Promise.resolve(known)
188
+ const shared = `${token}\0${key}`
189
+ const reading = lockfilesReading.get(shared) ?? readOf(fullName, token, ref, lockfile, key).finally(() => lockfilesReading.delete(shared))
190
+ lockfilesReading.set(shared, reading)
191
+ return reading
192
+ }
193
+ async function readOf(fullName: string, token: string, ref: string, { path, format }: { path: string; format: Format }, key: string): Promise<LockfileRead> {
180
194
  let text: string
181
195
  try {
182
196
  text = await upstreamGithub(() => createClient({ token, userAgent: 'deepview-triage' }).getRepoFile({ repo: fullName, path, ref }))
@@ -261,10 +275,16 @@ function addedIn(paths: string[], before: LockfileRead[], after: LockfileRead[])
261
275
  // where it changes no lockfile read here.
262
276
  export async function pullDependencies(fullName: string, token: string, number: number, kept: PullDependencyStore,
263
277
  fetchImpl: typeof fetch = globalThis.fetch): Promise<PullDependencies> {
278
+ return (await pullDependenciesAt(fullName, token, number, kept, fetchImpl)).told
279
+ }
280
+ // The same, with the commits it is kept by (`commits`), its base's and
281
+ // head's, by which `kept` reads it.
282
+ export async function pullDependenciesAt(fullName: string, token: string, number: number, kept: PullDependencyStore,
283
+ fetchImpl: typeof fetch = globalThis.fetch): Promise<{ commits: string; told: PullDependencies }> {
264
284
  const { base, head } = await pullCommits(fullName, token, number, fetchImpl)
265
285
  const key = `${base}...${head}`
266
286
  const held = await kept.read(key)
267
- if (held) return held
287
+ if (held) return { commits: key, told: held }
268
288
  const { mergeBase, files, cut: filesCut } = await pullFiles(fullName, token, number, base, head, fetchImpl)
269
289
  const { directories, cut, omitted } = lockfilesChanged(files)
270
290
  const reads = directories.flatMap(({ before, after }) => [...before.map(lockfile => [mergeBase, lockfile] as const), ...after.map(lockfile => [head, lockfile] as const)])
@@ -276,5 +296,5 @@ export async function pullDependencies(fullName: string, token: string, number:
276
296
  })
277
297
  const told = { base: mergeBase, head, lockfiles, cut: filesCut || cut, ...cut && !filesCut && omitted.length <= MAX_OMITTED ? { omitted } : {} }
278
298
  await kept.write(key, told)
279
- return told
299
+ return { commits: key, told }
280
300
  }
@@ -24,7 +24,7 @@
24
24
  // changes: it is read once, every lockfile there of those read here
25
25
  // (pull-dependencies.ts), and kept by the commit (`baselines` in this
26
26
  // instance, `kept` beyond it), never stored as reviews.
27
- import { lockedAt, lockedIn } from './commit-lockfiles.ts'
27
+ import { type GithubAsked, lockedAt, lockedIn } from './commit-lockfiles.ts'
28
28
  import { GithubApiError } from './github-app.ts'
29
29
  import type { LockedPackage } from './locked-packages.ts'
30
30
  import { EVERY_LOCKFILE, type LockfileScope, heldTo, isEvery, reviewedIn } from './lockfile-scope.ts'
@@ -68,13 +68,13 @@ const baselineKeys = (pkg: LockedPackage) => hashesOf(pkg).map(hash => `${pkg.na
68
68
  // and kept; with the lockfiles there its reader refuses, isn't one of, or
69
69
  // past those read (`unread`). GitHub having none of a lockfile it lists
70
70
  // there is a failure of the whole, as of any file, never kept.
71
- export async function reviewBaseline(fullName: string, token: string, repoId: number, since: string, kept: ReviewBaselineStore): Promise<Baseline> {
71
+ export async function reviewBaseline(fullName: string, token: string, repoId: number, since: string, kept: ReviewBaselineStore, asked?: GithubAsked): Promise<Baseline> {
72
72
  const key = `${repoId}\0${since}`
73
73
  const known = baselines.get(key)
74
74
  if (known) return known
75
75
  const stored = await kept.read(key)
76
76
  if (stored) { const baseline = { versions: new Set(stored.versions), unread: stored.unread }; baselines.set(key, baseline); return baseline }
77
- const read = await lockedAt(fullName, token, since)
77
+ const read = await lockedAt(fullName, token, since, () => true, asked)
78
78
  const { packages } = lockedIn(read), { unread } = read
79
79
  const versions = new Set<string>()
80
80
  for (const pkg of packages) if (reviewedAs(pkg) === 'version') for (const held of baselineKeys(pkg)) versions.add(held)
@@ -298,18 +298,18 @@ export async function reviewedDependencies(told: PullDependencies, state: RepoRe
298
298
  ...unread.length > 0 && !reviewed ? { baselineUnread: unread.slice(0, MAX_UNREVIEWED) } : {}, ...isEvery(scope) ? {} : { lockfiles: scope } } }
299
299
  }
300
300
 
301
- // The packages a pull request's lockfiles lock that its reviews here may
302
- // pass, whose reviews are asked: those that can be reviewed, unpatched.
301
+ // The package versions a pull request's lockfiles lock that its reviews here
302
+ // may pass, whose reviews are asked: those that can be reviewed, unpatched.
303
303
  export const reviewedPackages = (told: PullDependencies) => told.lockfiles
304
- .flatMap(dir => 'error' in dir ? [] : dir.locked.filter(pkg => reviewedAs(pkg) === 'version' && !pkg.patchHash).map(pkg => pkg.name))
304
+ .flatMap(dir => 'error' in dir ? [] : dir.locked.filter(pkg => reviewedAs(pkg) === 'version' && !pkg.patchHash).map(pkg => ({ name: pkg.name, version: pkg.version! })))
305
305
 
306
306
  // A repository's baseline as read for a pull request: its versions and the
307
307
  // lockfiles there left unread; none where it reviews since no commit; and
308
308
  // where GitHub doesn't read it, none and why.
309
- export async function baselineFor(fullName: string, token: string, repoId: number, state: RepoReviewState, kept: ReviewBaselineStore) {
309
+ export async function baselineFor(fullName: string, token: string, repoId: number, state: RepoReviewState, kept: ReviewBaselineStore, asked?: GithubAsked) {
310
310
  if (!state.since) return { baseline: null, unread: [], error: null }
311
311
  try {
312
- const { versions, unread } = await reviewBaseline(fullName, token, repoId, state.since, kept)
312
+ const { versions, unread } = await reviewBaseline(fullName, token, repoId, state.since, kept, asked)
313
313
  return { baseline: versions, unread, error: null }
314
314
  } catch (err) {
315
315
  if (err instanceof GithubApiError && err.status !== 429 && err.status !== 401) return { baseline: null, unread: [], error: err.message }
@@ -21,3 +21,19 @@ export function queue<T>(signal: AbortSignal, { drain = false } = {}): Queue<T>
21
21
  },
22
22
  }
23
23
  }
24
+
25
+ // Work done `count` at a time, in the order asked, the rest waiting their
26
+ // turn.
27
+ export function inTurns(count: number): <T>(work: () => Promise<T>) => Promise<T> {
28
+ let active = 0
29
+ const waiting: (() => void)[] = []
30
+ return async work => {
31
+ if (active < count) active++
32
+ else await new Promise<void>(resolve => { waiting.push(resolve) })
33
+ try { return await work() } finally {
34
+ const next = waiting.shift()
35
+ if (next) next()
36
+ else active--
37
+ }
38
+ }
39
+ }
@@ -179,7 +179,10 @@ export async function renameDiffOf(request: RenameDiffRequest, texts: () => Prom
179
179
  return opened(await answerOf(request, texts, keep))
180
180
  }
181
181
 
182
- async function answerOf(request: RenameDiffRequest, texts: () => Promise<[string, string]>, keep: boolean): Promise<Buffer> {
182
+ // Sides `verified` to be the files the request names (renamedPairs) are read
183
+ // only where no answer is kept or on its way; others are read first, as
184
+ // reading them checks they are.
185
+ async function answerOf(request: RenameDiffRequest, texts: () => Promise<[string, string]>, keep: boolean, verified = false): Promise<Buffer> {
183
186
  // Sides not kept are formatted again for each diff (pretty-print.ts), so
184
187
  // they are read in its turn: the files a Diff view asks for at once wait
185
188
  // here rather than being refused by the formats running. Within its minute
@@ -193,21 +196,27 @@ async function answerOf(request: RenameDiffRequest, texts: () => Promise<[string
193
196
  })
194
197
  return encoded(diff, request, sides)
195
198
  }
196
- const [before, after] = await texts()
197
199
  // A file's type is part of how it is read: the same text as a script and as a module differ.
198
200
  const key = `${request.base.hash}\0${request.base.extension}\0${request.other.hash}\0${request.other.extension}`
199
- const found = kept.get(key)
200
- if (found) {
201
- kept.delete(key)
202
- kept.set(key, found)
203
- return found
201
+ const held = () => {
202
+ const found = kept.get(key)
203
+ if (found) {
204
+ kept.delete(key)
205
+ kept.set(key, found)
206
+ }
207
+ return found ?? pending.get(key)
204
208
  }
205
- let job = pending.get(key)
206
- if (!job) {
207
- const deadline = Date.now() + DIFF_TIMEOUT_MS
208
- pending.set(key, job = inTurn(deadline, () => inWorker(before, after, request, deadline)).then(diff => encoded(diff, request, [before, after])).finally(() => pending.delete(key)))
209
+ let answer = verified ? held() : undefined
210
+ if (!answer) {
211
+ const [before, after] = await texts()
212
+ answer = held()
213
+ if (!answer) {
214
+ const deadline = Date.now() + DIFF_TIMEOUT_MS
215
+ const job = inTurn(deadline, () => inWorker(before, after, request, deadline)).then(diff => encoded(diff, request, [before, after])).finally(() => pending.delete(key))
216
+ pending.set(key, answer = job)
217
+ }
209
218
  }
210
- const body = await job
219
+ const body = await answer
211
220
  kept.set(key, body)
212
221
  if (kept.size > KEPT_DIFFS) kept.delete(kept.keys().next().value!)
213
222
  return body
@@ -258,7 +267,7 @@ export function renamedPairs([base, other]: readonly [ComparedSide, ComparedSide
258
267
  const named = { path: from.path, hash: from.hash, otherPath: to.path, otherHash: to.hash }
259
268
  const copy = async (side: ComparedSide, { text, ...file }: typeof from) => prettyText(await side.kept(file) ?? await side.made(file, text))
260
269
  let body
261
- try { body = await answerOf(request, async () => [await copy(base, from), await copy(other, to)], keep) }
270
+ try { body = await answerOf(request, async () => [await copy(base, from), await copy(other, to)], keep, true) }
262
271
  catch (err) {
263
272
  if (!(err instanceof PrettyError)) throw err
264
273
  yield { ...named, error: err.message, status: err.status }
@@ -146,7 +146,7 @@ export function createReportSourcesCache(storage: CacheStorage, db: ManagedDb, r
146
146
  }
147
147
  return {
148
148
  async open(report: ReportRecord, bundle: ManagedBundle, permissions: ViewerPermissions, sourcePaths?: Set<string>) {
149
- const repo = { github: (await db.listAllRepos()).find(entry => entry.repoId === report.repoId)?.fullName ?? null }
149
+ const repo = { github: report.repoId == null ? null : await db.getRepoFullName(report.repoId) }
150
150
  const target = sourceCacheFilename(report, bundle, permissions, repo, sourcePaths)
151
151
  try { return { ...await storage.open(target), repo } }
152
152
  catch (error) { if (!(error instanceof CacheMissError)) throw error }
@@ -15,33 +15,71 @@ async function readBytes(stored: OpenedBlob): Promise<Buffer> {
15
15
  } finally { stored.stream.destroy() }
16
16
  }
17
17
 
18
+ // Ids whose original copy is known gone, at most this many.
19
+ const COMPRESSED_ONLY_MAX = 16_384
20
+
18
21
  export function createReportStore(legacy: BlobStore, compressed: BlobStore,
19
22
  replaceLegacy: (id: string, bytes: Buffer) => Promise<unknown>): BlobStore {
23
+ // Reports this store wrote, converted, or found with only the compressed
24
+ // copy. Uploads write only that copy and nothing writes an original again,
25
+ // so their reads stop asking storage for one, each a miss (and, encrypted,
26
+ // a row read too). A compressed copy missing there asks for the original
27
+ // after all. An original left by a conversion that failed is never in it:
28
+ // the next read finds that original and resumes the conversion.
29
+ const compressedOnly = new Set<string>()
30
+ function remember(id: string): void {
31
+ compressedOnly.delete(id)
32
+ if (compressedOnly.size >= COMPRESSED_ONLY_MAX) compressedOnly.delete(compressedOnly.values().next().value!)
33
+ compressedOnly.add(id)
34
+ }
35
+ function decoded(stored: OpenedBlob): OpenedBlob {
36
+ const stream = createBrotliDecompress()
37
+ // pipeline carries storage/authentication errors into the decoded stream
38
+ // and closes the stored source when a reader leaves before EOF.
39
+ void pipeline(stored.stream, stream).catch(() => {})
40
+ return { size: null, stream }
41
+ }
20
42
  async function open(id: string): Promise<OpenedBlob | null> {
43
+ if (compressedOnly.has(id)) {
44
+ const stored = await compressed.open(id)
45
+ if (stored) return decoded(stored)
46
+ compressedOnly.delete(id)
47
+ }
21
48
  // Keep the original available until conversion succeeds. Concurrent reads
22
49
  // can finish using an already-open stream while another reader converts it.
23
- const original = await legacy.open(id)
24
- if (original) {
25
- const bytes = await readBytes(original)
50
+ // Both copies are asked for at once, the original answering where it is
51
+ // still there: one round trip of storage for a report not known yet, as
52
+ // a cold instance knows none.
53
+ const [original, stored] = await Promise.allSettled([legacy.open(id), compressed.open(id)])
54
+ const unused = (copy: PromiseSettledResult<OpenedBlob | null>) => { if (copy.status === 'fulfilled') copy.value?.stream.destroy() }
55
+ if (original.status === 'rejected') { unused(stored); throw original.reason }
56
+ if (original.value) {
57
+ unused(stored)
58
+ const bytes = await readBytes(original.value)
26
59
  await replaceLegacy(id, await encodeBrotli(bytes, 9))
60
+ remember(id)
27
61
  return { size: bytes.length, stream: Readable.from([bytes]) }
28
62
  }
29
- const stored = await compressed.open(id)
30
- if (!stored) return null
31
- const stream = createBrotliDecompress()
32
- // pipeline carries storage/authentication errors into the decoded stream
33
- // and closes the stored source when a reader leaves before EOF.
34
- void pipeline(stored.stream, stream).catch(() => {})
35
- return { size: null, stream }
63
+ if (stored.status === 'rejected') throw stored.reason
64
+ if (!stored.value) return null
65
+ remember(id)
66
+ return decoded(stored.value)
36
67
  }
37
68
  return {
38
- put: async (id, bytes) => compressed.put(id, await encodeBrotli(bytes, 9)),
69
+ async put(id, bytes) {
70
+ const wrapped = await compressed.put(id, await encodeBrotli(bytes, 9))
71
+ remember(id)
72
+ return wrapped
73
+ },
39
74
  open,
40
75
  async get(id) {
41
76
  const stored = await open(id)
42
77
  if (!stored) return null
43
78
  return readBytes(stored)
44
79
  },
45
- async delete(id) { await Promise.all([compressed.delete(id), legacy.delete(id)]) },
80
+ async delete(id) {
81
+ compressedOnly.delete(id)
82
+ await Promise.all([compressed.delete(id), legacy.delete(id)])
83
+ },
46
84
  }
47
85
  }
@@ -92,7 +92,8 @@ export function actionsLines(repo: number, owner: string, read: WorkflowsRead, l
92
92
  if ('error' in read) return { lines: [{ type: 'workflows', repo, ...read } as WorkflowsLine], refs: [] as ActionRef[] }
93
93
  const lines: (WorkflowsLine | WorkflowLine)[] = [{ type: 'workflows', repo, paths: read.files.filter(file => file.kind === 'workflow').map(file => file.path) }]
94
94
  const refs = new Map<string, ActionRef>()
95
- const actions = new Map(read.files.flatMap(file => file.kind === 'action' && 'text' in file ? [[actionDir(file.path), file.text] as const] : []))
95
+ // Its own actions by their directories and workflows by their paths, as a step or a job uses one.
96
+ const actions = new Map(read.files.flatMap(file => file.kind !== 'dockerfile' && 'text' in file ? [[file.kind === 'action' ? actionDir(file.path) : file.path, file.text] as const] : []))
96
97
  for (const file of read.files) {
97
98
  const scan = 'error' in file ? file : file.kind === 'dockerfile' ? { uses: [], findings: dockerfileFindings(file.text) } : scanWorkflow(file.kind, file.text, { owner, lockfiles, actions })
98
99
  lines.push({ type: 'workflow', repo, path: file.path, kind: file.kind, ...scan })
@@ -151,10 +152,12 @@ export interface ActionsResolving {
151
152
  store?: KeptStore<ActionLine> | undefined
152
153
  }
153
154
 
154
- // A reference's line: kept, else resolved, and kept where it is a public
155
- // repository's; why not, where it couldn't be.
156
- async function resolved(resolving: ActionsResolving, source: ActionsSource, { key, owner }: ActionRef): Promise<ActionLine | null> {
157
- const kept = resolving.memory.get(key) ?? await resolving.store?.read(key).then(row => row && { at: row.at, line: row.value })
155
+ // A reference's line: kept (`stored`, as read of the store with others),
156
+ // else resolved, and kept where it is a public repository's; why not, where
157
+ // it couldn't be.
158
+ async function resolved(resolving: ActionsResolving, source: ActionsSource, { key, owner }: ActionRef,
159
+ stored = resolving.store?.read(key)): Promise<ActionLine | null> {
160
+ const kept = resolving.memory.get(key) ?? await stored?.then(row => row && { at: row.at, line: row.value })
158
161
  if (kept && Date.now() - kept.at < RISK_KEPT_MS) { resolving.memory.set(key, kept); return kept.line }
159
162
  const uses = usesOfKey(key)
160
163
  if (!uses) return { type: 'action', key, error: 'invalid-uses' }
@@ -180,6 +183,9 @@ export function resolvingActions(resolving: ActionsResolving) {
180
183
  const source = keptSource(sharedSource(resolving.source, resolving.answers))
181
184
  const pending: Queue<ActionRef> = queue(resolving.signal)
182
185
  const latest = new Map<string, ActionRef>()
186
+ // Those of the references added at once kept beyond this instance, read of
187
+ // the store together as they are added.
188
+ const stored = new Map<string, Promise<{ at: number; value: ActionLine } | null>>()
183
189
  let closed = false, waiting = 0
184
190
  const settle = () => { if (closed && waiting === 0) pending.close() }
185
191
  const add = (refs: ActionRef[]) => {
@@ -191,11 +197,14 @@ export function resolvingActions(resolving: ActionsResolving) {
191
197
  waiting++
192
198
  pending.add(kept)
193
199
  }
200
+ const unread = [...new Set(refs.map(ref => ref.key))].filter(key => latest.has(key) && !stored.has(key) && !resolving.memory.get(key))
201
+ const read = unread.length > 0 ? resolving.store?.readMany(unread) : undefined
202
+ for (const key of read ? unread : []) stored.set(key, read!.then(found => found.get(key) ?? null))
194
203
  }
195
204
  const work = async () => {
196
205
  for (let ref = await pending.take(); ref !== undefined; ref = await pending.take()) {
197
206
  try {
198
- const line = await resolved(resolving, source, ref)
207
+ const line = await resolved(resolving, source, ref, stored.get(ref.key))
199
208
  if (line && latest.get(ref.key) === ref) {
200
209
  resolving.tell(line)
201
210
  const owner = 'repository' in line ? (line.repository?.fullName ?? line.repo).split('/')[0]! : ref.owner
@@ -175,7 +175,7 @@ async function agentBranches(owner: string, name: string, prefix: string, token:
175
175
  // What its answers are kept by, with their cursors (http.ts
176
176
  // handleRepositoryAgents): bumped as what a page reads, or how it tells an
177
177
  // agent, changes, so that no answer an older server kept is sent for a day.
178
- export const AGENTS_READ_VERSION = 5
178
+ export const AGENTS_READ_VERSION = 6
179
179
 
180
180
  // A cursor of the default branch's commits: GitHub names one by a commit
181
181
  // and how many on from it, `<oid> <offset>`, a space between.
@@ -1,6 +1,7 @@
1
+ import { createHash } from 'node:crypto'
1
2
  import type { ManagedConfig } from './config.ts'
2
3
  import type { SelectedRepo } from './db.ts'
3
- import { GithubApiError, type GithubIdentity, READ_PERMISSIONS, githubJson, githubRepoPermission, githubUserIdentity, repoAccessToken } from './github-app.ts'
4
+ import { GithubApiError, type GithubIdentity, READ_PERMISSIONS, githubJson, githubRepoPermission, githubUserIdentity, onTokenRejected, repoAccessToken } from './github-app.ts'
4
5
  import { MAX_PACKAGE_BYTES, readPackageEntryPoints } from './package-entry-points.ts'
5
6
  import { readSolidityEntryPoints } from './solidity-entry-points.ts'
6
7
  import { parseGithubCommit } from './bundle-commits.ts'
@@ -40,14 +41,63 @@ function credentialFailure(err: unknown): err is GithubApiError {
40
41
  return err instanceof GithubApiError && ([401, 404].includes(err.status) || err.message === 'github-status-403')
41
42
  }
42
43
 
43
- // Create once per HTTP request. Coalesce concurrent credential lookups without
44
- // caching repository visibility or permissions, or sharing credentials across users.
44
+ // GitHub's answer on a user's access to a repository, as checkAccess reads it:
45
+ // the repository's metadata, which credential reads it (the user's token, the
46
+ // App installation's, or none, as for a public repository read anonymously)
47
+ // and the permission level GitHub told, where it was asked. Kept per user, by
48
+ // their GitHub id and token, and per repository, by its id, name and
49
+ // installation, for config.repositoryAccessCacheMs; only access granted is kept.
50
+ // One answered for a user with a token is kept no longer once GitHub rejects
51
+ // that token (`userToken`, hashed), whichever credential reads it: a private
52
+ // repository's was granted on who that token told GitHub they are. The next
53
+ // check asks GitHub, falling back to the installation's or none as it does.
54
+ type RepoMetadata = { id?: unknown; full_name?: unknown; private?: unknown; visibility?: unknown; default_branch?: unknown }
55
+ type AccessVerdict = { until: number; data: RepoMetadata; credential: 'user' | 'app' | 'none'; level: string | null; userToken: string | null }
56
+ type VerdictCache = { get(): AccessVerdict | undefined; set(verdict: Omit<AccessVerdict, 'until' | 'userToken'>): void }
57
+ const MAX_ACCESS_VERDICTS = 1000
58
+ // By the fetch reading GitHub, as installation tokens are: a test's own.
59
+ const accessVerdicts = new WeakMap<typeof fetch, Map<string, AccessVerdict>>()
60
+ const hashed = (token: string) => createHash('sha256').update(token).digest('hex')
61
+ onTokenRejected((token, fetchImpl) => {
62
+ const kept = accessVerdicts.get(fetchImpl)
63
+ if (!kept) return
64
+ const rejected = hashed(token)
65
+ for (const [key, verdict] of kept) if (verdict.userToken === rejected) kept.delete(key)
66
+ })
67
+
68
+ function verdictCache(config: ManagedConfig, user: RepositoryBrowserUser, repo: SelectedRepo, fetchImpl: typeof fetch): VerdictCache | null {
69
+ const keepMs = config.repositoryAccessCacheMs
70
+ if (!(keepMs > 0)) return null
71
+ let verdicts = accessVerdicts.get(fetchImpl)
72
+ if (!verdicts) { verdicts = new Map(); accessVerdicts.set(fetchImpl, verdicts) }
73
+ const kept = verdicts
74
+ // The token hashed with the rest, never kept as it is.
75
+ const key = createHash('sha256').update(JSON.stringify([user.githubUserId, user.token, repo.repoId, repo.fullName.toLowerCase(), repo.installationId, config.githubAppId])).digest('hex')
76
+ return {
77
+ get() {
78
+ const verdict = kept.get(key)
79
+ if (verdict && verdict.until > Date.now()) return verdict
80
+ kept.delete(key)
81
+ return undefined
82
+ },
83
+ set(verdict) {
84
+ kept.delete(key)
85
+ kept.set(key, { ...verdict, until: Date.now() + keepMs, userToken: user.token ? hashed(user.token) : null })
86
+ while (kept.size > MAX_ACCESS_VERDICTS) kept.delete(kept.keys().next().value!)
87
+ },
88
+ }
89
+ }
90
+
91
+ // Create once per HTTP request. Coalesce concurrent credential lookups, and
92
+ // keep GitHub's answer on each user's access to each repository a while
93
+ // (verdictCache), never sharing credentials or answers across users.
45
94
  export function createRepositoryBrowser(config: ManagedConfig, user: RepositoryBrowserUser, fetchImpl: typeof fetch = globalThis.fetch) {
46
95
  const tokens = new Map<number, Promise<string | null>>()
47
96
  let identity: Promise<GithubIdentity> | undefined
48
- const getIdentity = (fresh = false): Promise<GithubIdentity> => {
97
+ // The user as GitHub tells them, asked once a request.
98
+ const getIdentity = (): Promise<GithubIdentity> => {
49
99
  if (!user.token || user.githubUserId == null) return Promise.reject(new GithubApiError(404, 'no-repository'))
50
- if (!identity || fresh) {
100
+ if (!identity) {
51
101
  identity = githubUserIdentity(user.token, fetchImpl).then(value => {
52
102
  if (value.id !== user.githubUserId) throw new GithubApiError(404, 'no-repository')
53
103
  return value
@@ -71,7 +121,7 @@ export function createRepositoryBrowser(config: ManagedConfig, user: RepositoryB
71
121
  // Its Dependabot alerts, with the token of the installation GitHub tells
72
122
  // covers it, where the user's own access lets them see them
73
123
  // (dependabot-access.ts).
74
- return repositoryReader(repo, user.token, installationToken, getIdentity, fetchImpl).then(reader => ({ ...reader,
124
+ return repositoryReader(repo, user.token, installationToken, getIdentity, fetchImpl, verdictCache(config, user, repo, fetchImpl)).then(reader => ({ ...reader,
75
125
  dependabotAlerts: () => appDependabotAlerts(config, repo, { identity: getIdentity, permission: reader.permission }, fetchImpl) }))
76
126
  },
77
127
  }
@@ -96,48 +146,86 @@ export async function repositoryTreeFiles(fullName: string, token: string | null
96
146
  return { files, bytes }
97
147
  }
98
148
 
99
- async function repositoryReader(repo: SelectedRepo, userToken: string | null, installationToken: () => Promise<string | null>, getIdentity: (fresh?: boolean) => Promise<GithubIdentity>, fetchImpl: typeof fetch) {
149
+ type Access = { data: RepoMetadata; token: string | null; level: string | null }
150
+
151
+ // GitHub asked whether the user may read `repo`: its metadata, read with
152
+ // `token` (the reader's credential so far), else the installation's, else
153
+ // none; and of one not public, the user's permission, read with the
154
+ // installation's. What it told, the token reading it, and the level asked.
155
+ async function askAccess(repo: SelectedRepo, start: string | null, userToken: string | null, installationToken: () => Promise<string | null>,
156
+ getIdentity: () => Promise<GithubIdentity>, fetchImpl: typeof fetch): Promise<Access> {
100
157
  const base = `https://api.github.com/repos/${repo.fullName.split('/').map(encodeURIComponent).join('/')}`
101
- // Public reads can use the signed-in user's quota without repository grants.
102
- // Nonpublic reads still require the repository App and the effective-user gate,
103
- // whose permission level is kept (`level`).
104
- let level: string | null = null, token = userToken ?? await installationToken()
105
- const read = (suffix: string) => githubJson(base + suffix, token, fetchImpl)
158
+ let token = start
106
159
  const metadata = async () => {
107
- try { return await read('') }
160
+ try { return await githubJson(base, token, fetchImpl) }
108
161
  catch (err) {
109
162
  if (!token || !credentialFailure(err)) throw err
110
163
  const fallback = await installationToken()
111
164
  if (fallback && fallback !== token) {
112
165
  token = fallback
113
- try { return await read('') } catch (fallbackError) { if (!credentialFailure(fallbackError)) throw fallbackError }
166
+ try { return await githubJson(base, token, fetchImpl) } catch (fallbackError) { if (!credentialFailure(fallbackError)) throw fallbackError }
114
167
  }
115
168
  // Anonymous fallback is only for unavailable credentials, never rate
116
169
  // limits, network errors, or GitHub outages.
117
170
  token = null
118
- return read('')
171
+ return githubJson(base, token, fetchImpl)
119
172
  }
120
173
  }
121
- const checkAccess = async (freshIdentity = false) => {
122
- // Stored visibility can be stale (or internal with private=false). Require
123
- // current, explicit public visibility or the caller's effective permission.
124
- const data = await metadata() as { id?: unknown; full_name?: unknown; private?: unknown; visibility?: unknown; default_branch?: unknown }
125
- if (data?.id !== repo.repoId || typeof data.full_name !== 'string' || data.full_name.toLowerCase() !== repo.fullName.toLowerCase()) {
126
- throw new GithubApiError(404, 'no-repository')
127
- }
128
- if (data.private === false && data.visibility === 'public') return data
129
- if (!token) throw new GithubApiError(404, 'no-repository')
130
- const appToken = await installationToken()
131
- if (!appToken) throw new GithubApiError(404, 'no-repository')
132
- level = await githubRepoPermission(repo.fullName, appToken, await getIdentity(freshIdentity), fetchImpl)
133
- if (!READ_PERMISSIONS.has(level)) throw new GithubApiError(404, 'no-repository')
134
- token = appToken
135
- return data
174
+ // A repository last known private asks who the user is as it asks for its
175
+ // metadata, not after: its permission is asked of them.
176
+ if (repo.private && userToken) getIdentity().catch(() => {})
177
+ // Stored visibility can be stale (or internal with private=false). Require
178
+ // current, explicit public visibility or the caller's effective permission.
179
+ const data = await metadata() as RepoMetadata
180
+ if (data?.id !== repo.repoId || typeof data.full_name !== 'string' || data.full_name.toLowerCase() !== repo.fullName.toLowerCase()) {
181
+ throw new GithubApiError(404, 'no-repository')
182
+ }
183
+ if (data.private === false && data.visibility === 'public') return { data, token, level: null }
184
+ if (!token) throw new GithubApiError(404, 'no-repository')
185
+ const appToken = await installationToken()
186
+ if (!appToken) throw new GithubApiError(404, 'no-repository')
187
+ const level = await githubRepoPermission(repo.fullName, appToken, await getIdentity(), fetchImpl)
188
+ if (!READ_PERMISSIONS.has(level)) throw new GithubApiError(404, 'no-repository')
189
+ return { data, token: appToken, level }
190
+ }
191
+
192
+ // GitHub's answer as kept (verdictCache), with the credential it chose,
193
+ // where that credential is still had; else asked (`ask`), and kept where it
194
+ // grants access.
195
+ async function keptAccess(verdicts: VerdictCache | null, userToken: string | null, installationToken: () => Promise<string | null>,
196
+ ask: () => Promise<Access>): Promise<Access> {
197
+ const kept = verdicts?.get()
198
+ if (kept) {
199
+ const credential = kept.credential === 'user' ? userToken : kept.credential === 'app' ? await installationToken() : null
200
+ if (kept.credential === 'none' || credential) return { data: kept.data, token: credential, level: kept.level }
201
+ }
202
+ const asked = await ask()
203
+ const { id, full_name, private: hidden, visibility, default_branch } = asked.data
204
+ const data = { id, full_name, private: hidden, visibility, default_branch }
205
+ verdicts?.set({ data, credential: asked.token === null ? 'none' : asked.token === userToken ? 'user' : 'app', level: asked.level })
206
+ return { ...asked, data }
207
+ }
208
+
209
+ async function repositoryReader(repo: SelectedRepo, userToken: string | null, installationToken: () => Promise<string | null>, getIdentity: (fresh?: boolean) => Promise<GithubIdentity>, fetchImpl: typeof fetch,
210
+ verdicts: VerdictCache | null) {
211
+ const base = `https://api.github.com/repos/${repo.fullName.split('/').map(encodeURIComponent).join('/')}`
212
+ // Public reads can use the signed-in user's quota without repository grants.
213
+ // Nonpublic reads still require the repository App and the effective-user gate,
214
+ // whose permission level is kept (`level`).
215
+ let level: string | null = null, token = userToken ?? await installationToken()
216
+ const read = (suffix: string) => githubJson(base + suffix, token, fetchImpl)
217
+ const checkAccess = async (): Promise<RepoMetadata> => {
218
+ const access = await keptAccess(verdicts, userToken, installationToken, () => askAccess(repo, token, userToken, installationToken, getIdentity, fetchImpl))
219
+ token = access.token
220
+ level = access.level
221
+ return access.data
136
222
  }
137
223
  const current = await checkAccess()
138
224
  const defaultBranch = typeof current.default_branch === 'string' ? current.default_branch : ''
139
225
  return {
140
- recheckAccess: () => checkAccess(true),
226
+ // Its visibility and the user's permission asked again, the user as
227
+ // GitHub told them earlier in the request, not asked again.
228
+ recheckAccess: () => checkAccess(),
141
229
  // Server-only credential, selected by the same live visibility/permission
142
230
  // gate as directory browsing. Never included in an HTTP response.
143
231
  readToken: () => token,
@@ -184,8 +272,10 @@ async function repositoryReader(repo: SelectedRepo, userToken: string | null, in
184
272
  const manifestPath = path ? `${path}/package.json` : 'package.json'
185
273
  const manifest = data.find(item => item?.path === manifestPath && item.name === 'package.json' && item.type === 'file' && !item.submodule_git_url
186
274
  && typeof item.size === 'number' && item.size >= 0 && item.size <= MAX_PACKAGE_BYTES && typeof item.sha === 'string' && /^[a-f\d]{40}$/iu.test(item.sha))
187
- const packageEntryPoints = manifest ? await readPackageEntryPoints(manifest.sha, path, read) : undefined
188
- const solidity = await readSolidityEntryPoints(path, data, read)
275
+ // Both optional suggestions, read together: neither fails, each
276
+ // answering none where its reads do.
277
+ const [packageEntryPoints, solidity] = await Promise.all([manifest ? readPackageEntryPoints(manifest.sha, path, read) : undefined,
278
+ readSolidityEntryPoints(path, data, read)])
189
279
  return { entries, limited: data.length >= 1000, ...(packageEntryPoints ? { packageEntryPoints } : {}),
190
280
  ...(solidity.paths.length > 0 || solidity.limited ? { solidityEntryPoints: solidity.paths, soliditySuggestionsLimited: solidity.limited } : {}),
191
281
  }