@preventive/triage 1.0.0-alpha.43 → 1.0.0-alpha.45

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (186) hide show
  1. package/common/bundle-compare-diff.js +2 -167
  2. package/common/bundle-compare-versions.js +172 -0
  3. package/common/bundle-comparison.d.ts +8 -0
  4. package/common/bundle-comparison.js +62 -0
  5. package/common/bundle-sources.js +20 -1
  6. package/common/default-scan-models.ts +1 -1
  7. package/common/dependency-reviews.d.ts +0 -5
  8. package/common/dependency-reviews.js +0 -17
  9. package/common/managed/agent-tasks.ts +292 -0
  10. package/common/managed/github-agents.ts +92 -0
  11. package/common/managed/github-logins.ts +49 -0
  12. package/common/managed/roles.ts +7 -0
  13. package/common/managed/routes.js +91 -10
  14. package/common/managed/team-app.d.ts +5 -2
  15. package/common/managed/team-app.js +2 -2
  16. package/common/server-info.ts +11 -0
  17. package/common/workspace-app.js +19 -3
  18. package/out/brotli-fallback.js +3 -3
  19. package/out/chunk-2XUEZHIP.js +18 -18
  20. package/out/chunk-2ZDZWLFY.js +1 -0
  21. package/out/chunk-3PS4KWD2.js +10 -0
  22. package/out/chunk-5DHL5NG3.js +2 -0
  23. package/out/chunk-6HNXNIQY.js +52 -0
  24. package/out/chunk-7C7DWL7I.js +1 -0
  25. package/out/chunk-AFLZH7WG.js +1 -0
  26. package/out/chunk-BC2UCLFJ.js +1 -0
  27. package/out/chunk-C4Z6ZBXV.js +6 -6
  28. package/out/chunk-GDPCLSOS.js +2 -0
  29. package/out/chunk-GO7FT4OD.js +1 -0
  30. package/out/chunk-HMAJDVXN.js +1 -0
  31. package/out/chunk-I2MDGSQX.js +3 -0
  32. package/out/chunk-IYPY47MC.js +1 -0
  33. package/out/chunk-IZ7POWI3.js +18 -0
  34. package/out/chunk-JCIIBLKR.js +1 -0
  35. package/out/chunk-NPW52BM2.js +17 -0
  36. package/out/chunk-NRPC74AM.js +1 -1
  37. package/out/chunk-NV7ES4EW.js +1 -0
  38. package/out/chunk-OKRLLE5I.js +17 -0
  39. package/out/chunk-OOJM4CTU.js +1 -1
  40. package/out/chunk-PNHBZGDD.js +1 -1
  41. package/out/chunk-PTGDYLNS.js +36 -0
  42. package/out/chunk-PXK7EU4V.js +1 -0
  43. package/out/chunk-Q6MVYGYV.js +3 -0
  44. package/out/chunk-QB4CBOX2.js +1 -0
  45. package/out/chunk-QRD6GIG5.js +229 -0
  46. package/out/chunk-R3NZFXVR.js +1 -0
  47. package/out/chunk-R5O6FAW4.js +1 -1
  48. package/out/chunk-SXV2T5C4.js +80 -0
  49. package/out/chunk-TUII34K2.js +1 -1
  50. package/out/chunk-VFWMQ3LG.js +2 -0
  51. package/out/chunk-VH4XMZIW.js +2 -0
  52. package/out/chunk-WFYLMF6Z.js +10 -0
  53. package/out/chunk-WI7Z5A2T.js +6 -0
  54. package/out/chunk-WQ74ZBZF.js +7 -0
  55. package/out/chunk-XVBEOGOS.js +3 -0
  56. package/out/client-managed-import.js +3 -3
  57. package/out/client-managed.js +31 -35
  58. package/out/client-sync.js +11 -11
  59. package/out/compare.js +35 -0
  60. package/out/dialogs.js +51 -0
  61. package/out/github-issue-authorized.html +1 -1
  62. package/out/github-issue-authorized.js +1 -1
  63. package/out/graph.js +17 -17
  64. package/out/index.html +31 -31
  65. package/out/prism.js +14 -14
  66. package/out/scan.js +2 -0
  67. package/out/terminal.js +294 -293
  68. package/out/view.css +1 -1
  69. package/out/view.js +16 -107
  70. package/package.json +55 -6
  71. package/risk/actions/findings.js +156 -0
  72. package/risk/actions/resolve.js +219 -0
  73. package/risk/actions/scripts/commands.js +182 -0
  74. package/risk/actions/scripts/grade.js +113 -0
  75. package/risk/actions/scripts/packages.js +123 -0
  76. package/risk/actions/scripts/shell.js +165 -0
  77. package/risk/actions/uses.js +76 -0
  78. package/risk/actions/workflow.js +110 -0
  79. package/risk/files/hosts.js +6 -6
  80. package/risk/files/readability.js +23 -17
  81. package/risk/files/runs-code.js +18 -6
  82. package/risk/files/script.js +7 -3
  83. package/risk/files/stamps.js +8 -7
  84. package/risk/index.d.ts +68 -1
  85. package/risk/index.js +6 -2
  86. package/risk/npm/assess.js +13 -8
  87. package/risk/npm/compare.js +34 -10
  88. package/risk/npm/dependencies.js +11 -0
  89. package/risk/npm/files.js +413 -2
  90. package/risk/npm/tarball.js +56 -73
  91. package/server-managed/agent-merges.ts +198 -0
  92. package/server-managed/app-agents.ts +30 -0
  93. package/server-managed/avatar-store.ts +1 -0
  94. package/server-managed/commit-coauthors.ts +83 -0
  95. package/server-managed/commit-lockfiles.ts +62 -0
  96. package/server-managed/comparison.ts +41 -0
  97. package/server-managed/db-methods.ts +29 -10
  98. package/server-managed/db-neon.ts +30 -2
  99. package/server-managed/db-schema.ts +9 -2
  100. package/server-managed/db.ts +7 -0
  101. package/server-managed/dependabot-access.ts +60 -0
  102. package/server-managed/dependabot-alerts.ts +185 -0
  103. package/server-managed/dependency-review-comments.ts +97 -0
  104. package/server-managed/dependency-reviews.ts +65 -15
  105. package/server-managed/github-accounts.ts +33 -0
  106. package/server-managed/github-actions.ts +71 -0
  107. package/server-managed/github-app.ts +23 -9
  108. package/server-managed/github-errors.ts +2 -0
  109. package/server-managed/github-link-oauth.ts +69 -0
  110. package/server-managed/github-links.ts +204 -0
  111. package/server-managed/github-oauth.ts +5 -2
  112. package/server-managed/github-users.ts +30 -0
  113. package/server-managed/http.ts +1219 -81
  114. package/server-managed/index.ts +6 -2
  115. package/server-managed/locked-packages.ts +48 -16
  116. package/server-managed/lockfile-scope.ts +49 -0
  117. package/server-managed/npm-delta-worker.js +23 -0
  118. package/server-managed/npm-delta.ts +173 -0
  119. package/server-managed/npm-insights.ts +20 -11
  120. package/server-managed/npm-kept.ts +51 -4
  121. package/server-managed/npm-loads.ts +20 -13
  122. package/server-managed/npm-packages.ts +24 -64
  123. package/server-managed/pretty-print.ts +14 -9
  124. package/server-managed/public-workspace.ts +26 -8
  125. package/server-managed/pull-comment-reviews.ts +161 -0
  126. package/server-managed/pull-dependencies.ts +280 -0
  127. package/server-managed/pull-dependency-reviews.ts +318 -0
  128. package/server-managed/pull-reviews.ts +205 -0
  129. package/server-managed/release-commits.ts +84 -0
  130. package/server-managed/repository-actions.ts +209 -0
  131. package/server-managed/repository-agents.ts +220 -0
  132. package/server-managed/repository-browser.ts +54 -13
  133. package/server-managed/repository-changes.ts +284 -0
  134. package/server-managed/repository-code.ts +52 -0
  135. package/server-managed/repository-dependency-reviews.ts +135 -0
  136. package/server-managed/repository-history.ts +445 -0
  137. package/server-managed/repository-issues.ts +181 -0
  138. package/server-managed/repository-pulse.ts +432 -0
  139. package/server-managed/repository-risk-read.ts +165 -0
  140. package/server-managed/repository-risk.ts +116 -106
  141. package/server-managed/reviewed-dependencies.ts +149 -0
  142. package/server-managed/session.ts +9 -5
  143. package/server-managed/sql.ts +1 -1
  144. package/server-managed/storage-stores.ts +5 -0
  145. package/server-managed/team-app.ts +54 -14
  146. package/server-managed/workspace-shares.ts +4 -1
  147. package/server.ts +11 -0
  148. package/out/chunk-2KBF5TZ7.js +0 -2
  149. package/out/chunk-35VE6CDH.js +0 -2
  150. package/out/chunk-3BZ7JERB.js +0 -1
  151. package/out/chunk-4F6JSF57.js +0 -3
  152. package/out/chunk-4XCDFGDP.js +0 -10
  153. package/out/chunk-7RU5EDRO.js +0 -226
  154. package/out/chunk-A7M4GV7M.js +0 -1
  155. package/out/chunk-CLPKNNC7.js +0 -51
  156. package/out/chunk-CMQA4DM6.js +0 -1
  157. package/out/chunk-CVVJAXVY.js +0 -2
  158. package/out/chunk-DIJJNTRH.js +0 -10
  159. package/out/chunk-GJ4IUCNS.js +0 -1
  160. package/out/chunk-GNXOKIGP.js +0 -80
  161. package/out/chunk-GZ4MS3TE.js +0 -13
  162. package/out/chunk-IU3CCINR.js +0 -3
  163. package/out/chunk-K3X5EJBQ.js +0 -2
  164. package/out/chunk-KEX2ATEW.js +0 -2
  165. package/out/chunk-KYCNJVB7.js +0 -1
  166. package/out/chunk-LAAB5SZT.js +0 -2
  167. package/out/chunk-NWZ4PVPN.js +0 -1
  168. package/out/chunk-OSIMGFUF.js +0 -1
  169. package/out/chunk-RFZQQXSG.js +0 -17
  170. package/out/chunk-RSHMVYNS.js +0 -1
  171. package/out/chunk-TZ5KB2Y5.js +0 -10
  172. package/out/chunk-UNTRGE26.js +0 -2
  173. package/out/chunk-UXNUAIIC.js +0 -33
  174. package/out/chunk-UYAXVRVL.js +0 -1
  175. package/out/chunk-V3ME6FI5.js +0 -2
  176. package/out/chunk-V5O7M2YT.js +0 -1
  177. package/out/chunk-VIHGO77Y.js +0 -1
  178. package/out/chunk-VQJCXGIY.js +0 -7
  179. package/out/chunk-W4J3MGLB.js +0 -1
  180. package/out/chunk-WGG7KQUH.js +0 -1
  181. package/out/chunk-WZDTZE64.js +0 -1
  182. package/out/chunk-XOCAPDCM.js +0 -1
  183. package/out/chunk-XYFXXB2G.js +0 -10
  184. package/out/chunk-YVZH5SHP.js +0 -1
  185. package/out/chunk-ZBBOBQ43.js +0 -1
  186. package/out/chunk-ZMQQCGF6.js +0 -2
@@ -0,0 +1,165 @@
1
+ // A repository as a repository risk scan reads it of GitHub
2
+ // (repository-risk.ts): opened at its default branch's head, its lockfiles
3
+ // of npm's ecosystem listed (findRepoLockfiles), those of its `paths` alone,
4
+ // each read for its packages (@preventive/lockfile), and access to it checked
5
+ // again before they are told (`recheck`); and its Dependabot alerts, open or dismissed, of
6
+ // npm's, where the source asks them (dependabot-alerts.ts), asked meanwhile
7
+ // and never kept, a failure to have them its own, and its GitHub Actions
8
+ // (repository-actions.ts). A listing is kept by the commit it was listed at,
9
+ // its Actions' files too, and a lockfile's packages by its blob: in the
10
+ // instance's memory, and the source's `listings`, `actionFiles` and
11
+ // `lockfiles` for SCAN_READ_KEPT_MS, so that only opening a repository and
12
+ // checking access to it again ask GitHub where all else is kept.
13
+ import type { LockfileFormat, RepoLockfile } from '@preventive/upstream/github.js'
14
+ import type { AlertsRead } from './dependabot-access.ts'
15
+ import type { DependabotAlert } from './dependabot-alerts.ts'
16
+ import { type LockedPackage, isTestFixture, lockedPackages } from './locked-packages.ts'
17
+ import type { Lru } from './lru.ts'
18
+ import type { KeptStore } from './npm-kept.ts'
19
+ import { type WorkflowsRead, lastingRead, readActions } from './repository-actions.ts'
20
+ import { type RiskFailure, failureOf } from './repository-risk-assess.ts'
21
+ import type { RepositoryRiskLine, RepositoryRiskRepo } from './repository-risk.ts'
22
+
23
+ // The formats of npm's ecosystem, as findRepoLockfiles names them, and those
24
+ // of them read here.
25
+ const NPM_LOCKFILE_FORMATS: ReadonlySet<LockfileFormat> = new Set(['npm', 'pnpm', 'yarn', 'bun', 'deno'])
26
+ const READ_FORMATS: ReadonlySet<LockfileFormat> = new Set(['npm', 'pnpm', 'yarn'])
27
+
28
+ // How long the source's `listings` and `lockfiles` are read for: what
29
+ // GitHub has at a commit or a blob doesn't change, how it is read here may.
30
+ export const SCAN_READ_KEPT_MS = 24 * 60 * 60_000
31
+
32
+ export type ListedLockfile = Pick<RepoLockfile, 'path' | 'format' | 'sha'>
33
+ export type LockfileRead = { packages: LockedPackage[] } | RiskFailure
34
+ // A repository's Dependabot alerts, those of its `paths` alone, with access to
35
+ // them to check again before they are told (`recheck`), or why they couldn't
36
+ // be had.
37
+ export type DependabotRead = { alerts: DependabotAlert[]; recheck?(): Promise<void> } | RiskFailure
38
+ type LockfileLine = Extract<RepositoryRiskLine, { type: 'lockfile' }>
39
+ export type RepositoryFailure = Extract<RepositoryRiskLine, { type: 'repo'; error: string }>
40
+
41
+ // A repository opened at its default branch's head (`commit`): its lockfiles,
42
+ // as findRepoLockfiles lists them, each with its blob's id, and a file's
43
+ // text; `recheck` checks access to it again, throwing where it is gone; and
44
+ // `dependabot`, where asked, its Dependabot alerts of npm's, open or dismissed, as GitHub
45
+ // has them now, with access to them to check again before they are told.
46
+ export interface OpenedRepository {
47
+ commit: string
48
+ lockfiles(): Promise<ListedLockfile[]>
49
+ file(path: string): Promise<string>
50
+ recheck(): Promise<void>
51
+ dependabot?(): Promise<AlertsRead>
52
+ // Its workflows' paths, none where it has none.
53
+ workflows?(): Promise<string[]>
54
+ }
55
+
56
+ // What a repository is read with: the scan's source and caches, and its
57
+ // signal.
58
+ export interface RepositoryReading<R extends RepositoryRiskRepo> {
59
+ source: { open(repo: R): Promise<OpenedRepository>; listings?: KeptStore<ListedLockfile[]>; lockfiles?: KeptStore<LockfileRead>; actionFiles?: KeptStore<WorkflowsRead> }
60
+ caches: { listings: Lru<ListedLockfile[]>; lockfiles: Lru<LockfileRead>; actionFiles: Lru<WorkflowsRead> }
61
+ signal: AbortSignal
62
+ }
63
+
64
+ // A repository read: its commit, its lockfiles' lines, each with its blob's
65
+ // id, its Dependabot alerts where asked, and access to it to check again just
66
+ // before they are told (OpenedRepository `recheck`), as GitHub may have
67
+ // withdrawn it meanwhile.
68
+ export interface RepositoryRead {
69
+ commit: string
70
+ lockfiles: { sha: string; line: LockfileLine }[]
71
+ dependabot: DependabotRead | null
72
+ // Its Actions' files, where it lists its workflows.
73
+ actions: WorkflowsRead | null
74
+ recheck(): Promise<void>
75
+ }
76
+
77
+ // Whether a lockfile at `path` is of `paths` (RepositoryRiskRepo).
78
+ function lockfileOf(path: string, paths: readonly string[] | undefined) {
79
+ const dir = path.slice(0, Math.max(path.lastIndexOf('/'), 0))
80
+ return !paths || paths.some(scope => dir === '' || dir === scope || dir.startsWith(`${scope}/`) || scope.startsWith(`${dir}/`))
81
+ }
82
+
83
+ // What is kept by `key`: in this instance's memory, else in `store` within
84
+ // SCAN_READ_KEPT_MS, else `read` and kept in both, where it is to be.
85
+ async function keptRead<T>(memory: Lru<T>, store: KeptStore<T> | undefined, key: string, read: () => Promise<T>, keep = (_value: T) => true) {
86
+ const known = memory.get(key)
87
+ if (known !== undefined) return known
88
+ const kept = await store?.read(key)
89
+ if (kept && Date.now() - kept.at < SCAN_READ_KEPT_MS) { memory.set(key, kept.value); return kept.value }
90
+ const value = await read()
91
+ if (!keep(value)) return value
92
+ memory.set(key, value)
93
+ await store?.write(key, value, Date.now())
94
+ return value
95
+ }
96
+
97
+ // A lockfile's line, its packages or why they couldn't be had: as kept by its
98
+ // blob, else read from GitHub and kept.
99
+ async function readLockfile({ source, caches }: RepositoryReading<RepositoryRiskRepo>, repo: number, opened: OpenedRepository,
100
+ { path, format, sha }: ListedLockfile): Promise<LockfileLine> {
101
+ const at = { type: 'lockfile', repo, path, format } as const
102
+ if (!READ_FORMATS.has(format)) return { ...at, error: 'unsupported-format' }
103
+ try {
104
+ return { ...at, ...await keptRead(caches.lockfiles, source.lockfiles, `${format}\0${sha}`, async (): Promise<LockfileRead> => {
105
+ const text = await opened.file(path)
106
+ try { return { packages: lockedPackages(format, text) } } catch (error) {
107
+ return { error: 'lockfile-unreadable', ...error instanceof Error ? { detail: error.message.slice(0, 500) } : {} }
108
+ }
109
+ }) }
110
+ } catch (error) { return { ...at, ...failureOf(error) } }
111
+ }
112
+
113
+ // A repository opened and its lockfiles of npm's ecosystem listed, as kept
114
+ // for its commit or else from GitHub, those of its `paths` alone; its line
115
+ // telling why where either failed; null where the scan stopped.
116
+ async function listLockfiles<R extends RepositoryRiskRepo>(reading: RepositoryReading<R>, repo: R) {
117
+ let doing = 'opening it'
118
+ try {
119
+ const opened = await reading.source.open(repo)
120
+ if (reading.signal.aborted) return null
121
+ doing = 'listing its lockfiles'
122
+ const listed = await keptRead(reading.caches.listings, reading.source.listings, `${repo.repoId}@${opened.commit}`, () => opened.lockfiles())
123
+ const lockfiles = listed.filter(({ path, format }) => NPM_LOCKFILE_FORMATS.has(format) && lockfileOf(path, repo.paths))
124
+ return { opened, lockfiles }
125
+ } catch (error) {
126
+ return reading.signal.aborted ? null : { type: 'repo', id: repo.repoId, ...failureOf(error, doing) } as const
127
+ }
128
+ }
129
+
130
+ // A repository's Dependabot alerts of its `paths` (RepositoryRiskRepo), as
131
+ // its lockfiles are, or why they couldn't be had; null where they aren't
132
+ // asked.
133
+ async function readDependabot(opened: OpenedRepository, paths: readonly string[] | undefined): Promise<DependabotRead | null> {
134
+ if (!opened.dependabot) return null
135
+ try {
136
+ const { alerts, recheck } = await opened.dependabot()
137
+ return { alerts: alerts.filter(alert => lockfileOf(alert.manifest, paths)), recheck }
138
+ } catch (error) {
139
+ return failureOf(error, 'asking its Dependabot alerts')
140
+ }
141
+ }
142
+
143
+ // A repository read: its lockfiles listed and read, its Dependabot alerts
144
+ // asked and its Actions read meanwhile; its line telling why where listing
145
+ // failed; null where the scan stopped. A test's
146
+ // fixture or example its reader refuses (isTestFixture) is left out, as a
147
+ // test may keep one so on purpose.
148
+ export async function readRepository<R extends RepositoryRiskRepo>(reading: RepositoryReading<R>, repo: R): Promise<RepositoryRead | RepositoryFailure | null> {
149
+ const listed = await listLockfiles(reading, repo)
150
+ if (!listed || 'type' in listed) return listed
151
+ const alerting = readDependabot(listed.opened, repo.paths)
152
+ const { opened } = listed, { workflows } = opened
153
+ const acting = workflows ? keptRead(reading.caches.actionFiles, reading.source.actionFiles, `${repo.repoId}@${opened.commit}`,
154
+ () => readActions({ workflows, file: path => opened.file(path) }), lastingRead) : Promise.resolve(null)
155
+ acting.catch(() => {})
156
+ const lockfiles: RepositoryRead['lockfiles'] = []
157
+ for (const lockfile of listed.lockfiles) {
158
+ if (reading.signal.aborted) return null
159
+ const line = await readLockfile(reading, repo.repoId, listed.opened, lockfile)
160
+ if (!('error' in line && line.error === 'lockfile-unreadable' && isTestFixture(line.path))) lockfiles.push({ sha: lockfile.sha, line })
161
+ }
162
+ const dependabot = await alerting
163
+ const actions = await acting
164
+ return reading.signal.aborted ? null : { commit: listed.opened.commit, lockfiles, dependabot, actions, recheck: () => listed.opened.recheck() }
165
+ }
@@ -12,58 +12,47 @@
12
12
  // host with an integrity, such as a company's mirror of npm's, as npm's
13
13
  // package of its name and version where npm's integrity is among it; one
14
14
  // from elsewhere (git, a tarball with no integrity, a file) is listed, not
15
- // assessed. Repositories are read one at a time, as GitHub's limits are the
16
- // caller's; packages are assessed meanwhile, as many at a time as npm-risk.ts
15
+ // assessed. Repositories are read a few at a time (READ_AHEAD), each told in
16
+ // turn, as GitHub's limits are the caller's; packages are assessed meanwhile, as many at a time as npm-risk.ts
17
17
  // makes spare assessments, those only devDependencies need in every lockfile
18
18
  // that locks them (LockedPackage `dev`) once all the rest are, as an install
19
19
  // of no devDependencies leaves them out. A lockfile's packages of npm's
20
20
  // registry or a mirror are asked of npm's advisories all at once, as `npm
21
21
  // audit` asks, each advisory told with whether it is of malware (risk's
22
22
  // npmMalwareAdvisory), which alone is a supply chain's risk; the rest are
23
- // vulnerabilities, told beside.
24
- import type { LockfileFormat, RepoLockfile } from '@preventive/upstream/github.js'
25
- import type { Severity } from '../risk/index.js'
23
+ // vulnerabilities, told beside. A repository's Dependabot alerts of npm's, open or dismissed,
24
+ // are told once its lockfiles' advisories are, each with the versions its
25
+ // lockfile locks that it covers, for the page to blend them into npm's; one
26
+ // whose alerts couldn't be had tells why, and the scan goes on. Its GitHub
27
+ // Actions are told once its lockfiles are, the references they make
28
+ // resolved as the scan goes on (repository-actions.ts).
29
+ import type { LockfileFormat } from '@preventive/upstream/github.js'
30
+ import { type Severity, npmMalwareAdvisory } from '../risk/index.js'
26
31
  import { isNpmPackageName, isNpmPackageSpec } from '../common/managed/npm-packages.js'
27
32
  import type { NpmVersionDocument } from './npm-packages.ts'
28
33
  import { type Lru, lru } from './lru.ts'
29
34
  import type { KeptStore } from './npm-kept.ts'
30
35
  import { MAX_SPARE_ASSESSMENTS } from './npm-risk.ts'
31
- import { type LockedPackage, lockedPackages } from './locked-packages.ts'
36
+ import type { LockedPackage } from './locked-packages.ts'
32
37
  import { type Queue, queue } from './queue.ts'
33
38
  import { type DownloadsSource, readyingDownloads } from './scan-downloads.ts'
34
39
  import { type AssessedPackage, type Assessment, BUSY_RETRIES, BUSY_WAIT_MS, RISK_KEPT_MS, type RiskFailure, assessPackage, failureOf, isBusy } from './repository-risk-assess.ts'
35
40
  import { type AdvisoryRow, type Audit, type AuditSource, auditLockfile } from './repository-risk-audit.ts'
41
+ import { type DependabotRead, type ListedLockfile, type LockfileRead, type OpenedRepository, type RepositoryFailure, type RepositoryRead, SCAN_READ_KEPT_MS,
42
+ readRepository } from './repository-risk-read.ts'
43
+ import { type DependabotAlert, alertVersions } from './dependabot-alerts.ts'
44
+ import type { ActionsSource } from '../risk/index.js'
45
+ import { type ActionAnswers, type ActionLine, type ActionRef, type WorkflowLine, type WorkflowsLine, type WorkflowsRead, actionsLines, answerWeight,
46
+ lockedNames, resolvingActions } from './repository-actions.ts'
36
47
 
37
48
  export { RISK_KEPT_MS, highestSeverity } from './repository-risk-assess.ts'
38
-
39
- // The formats of npm's ecosystem, as findRepoLockfiles names them, and those
40
- // of them read here.
41
- const NPM_LOCKFILE_FORMATS: ReadonlySet<LockfileFormat> = new Set(['npm', 'pnpm', 'yarn', 'bun', 'deno'])
42
- const READ_FORMATS: ReadonlySet<LockfileFormat> = new Set(['npm', 'pnpm', 'yarn'])
49
+ export { type OpenedRepository, SCAN_READ_KEPT_MS } from './repository-risk-read.ts'
43
50
 
44
51
  // A repository to read: where `paths` are given, a team's directories in it,
45
52
  // only the lockfiles in them or above them, as a workspace's root lockfile
46
53
  // locks them too.
47
54
  export interface RepositoryRiskRepo { repoId: number; fullName: string; paths?: readonly string[] }
48
55
 
49
- // Whether a lockfile at `path` is of `paths` (RepositoryRiskRepo).
50
- function lockfileOf(path: string, paths: readonly string[] | undefined) {
51
- const dir = path.slice(0, Math.max(path.lastIndexOf('/'), 0))
52
- return !paths || paths.some(scope => dir === '' || dir === scope || dir.startsWith(`${scope}/`) || scope.startsWith(`${dir}/`))
53
- }
54
-
55
- type ListedLockfile = Pick<RepoLockfile, 'path' | 'format' | 'sha'>
56
-
57
- // A repository opened at its default branch's head (`commit`): its lockfiles,
58
- // as findRepoLockfiles lists them, each with its blob's id, and a file's
59
- // text; `recheck` checks access to it again, throwing where it is gone.
60
- export interface OpenedRepository {
61
- commit: string
62
- lockfiles(): Promise<ListedLockfile[]>
63
- file(path: string): Promise<string>
64
- recheck(): Promise<void>
65
- }
66
-
67
56
  export interface RepositoryRiskSource<R extends RepositoryRiskRepo> extends AuditSource {
68
57
  repos(): Promise<R[]>
69
58
  open(repo: R): Promise<OpenedRepository>
@@ -94,10 +83,21 @@ export interface RepositoryRiskSource<R extends RepositoryRiskRepo> extends Audi
94
83
  // Readies npm's downloads of packages before they are assessed
95
84
  // (scan-downloads.ts); without it, each assessment asks its own.
96
85
  downloads?: DownloadsSource
86
+ // GitHub as references are resolved with; none without it.
87
+ actions?: ActionsSource
88
+ // Where Actions' files read and references resolved are kept.
89
+ actionFiles?: KeptStore<WorkflowsRead>
90
+ actionLines?: KeptStore<ActionLine>
97
91
  }
98
92
 
99
93
 
100
94
 
95
+ // A Dependabot alert as a scan tells it: with the versions of its package
96
+ // the lockfile it is of locks that it covers, none where it is of none read
97
+ // (a package.json, bun's lockfile), and whether it is of malware, as an
98
+ // advisory's is told.
99
+ export type DependabotAlertRow = DependabotAlert & { versions: string[]; malware: boolean }
100
+
101
101
  // A risk line is of npm's package of its name and version, or, with an
102
102
  // `integrity`, of a mirror's that its lockfiles hold to it; its `score` risk's
103
103
  // of it, out of 100 (assessNpm `score`, risk/npm/score.js).
@@ -108,6 +108,8 @@ export type RepositoryRiskLine =
108
108
  | ({ type: 'risk'; name: string; version: string; integrity?: string; stale?: { assessedAt: number } }
109
109
  & ({ published: string | null; severity: Severity | null; score: number; findings: { id: string; severity: Severity; takenDown?: true }[] } | RiskFailure))
110
110
  | ({ type: 'advisories'; repo: number; path: string } & ({ advisories: AdvisoryRow[] } | RiskFailure))
111
+ | ({ type: 'dependabot'; repo: number } & ({ alerts: DependabotAlertRow[] } | RiskFailure))
112
+ | WorkflowsLine | WorkflowLine | ActionLine
111
113
  | { type: 'done' }
112
114
  // The scan stopped for its part's time (repositoryRiskLines `partMs`), what
113
115
  // it told kept: to be asked again, from where it stopped.
@@ -115,12 +117,12 @@ export type RepositoryRiskLine =
115
117
 
116
118
  // The repositories a scan answered in parts told whole by a part's end: each
117
119
  // by its id, its name and a team's directories in it then, and its
118
- // lockfiles' keys (`${format}\0${sha}`) of those read.
120
+ // lockfiles' keys (`${format}\0${sha}`) of those read, and the references
121
+ // its Actions make, to resolve.
119
122
  export interface ScanProgress { repos: ToldRepository[] }
120
- interface ToldRepository { id: number; name: string; paths: readonly string[] | null; lockfiles: string[] }
123
+ interface ToldRepository { id: number; name: string; paths: readonly string[] | null; lockfiles: string[]; actions?: ActionRef[] }
121
124
 
122
125
  type RiskLine = Extract<RepositoryRiskLine, { type: 'risk' }>
123
- type LockfileRead = { packages: LockedPackage[] } | RiskFailure
124
126
 
125
127
  // What scans keep between them, in this instance's memory: a repository's
126
128
  // lockfiles as listed at a commit, and a lockfile's packages by its blob,
@@ -135,6 +137,9 @@ export interface RiskCaches {
135
137
  lockfiles: Lru<LockfileRead>
136
138
  risks: Lru<{ at: number; line: RiskLine }>
137
139
  advisories: Lru<{ at: number; audit: Audit }>
140
+ actionFiles: Lru<WorkflowsRead>
141
+ actions: Lru<{ at: number; line: ActionLine }>
142
+ actionAnswers: ActionAnswers
138
143
  }
139
144
  export const riskCaches = (): RiskCaches => ({
140
145
  listings: lru(1000),
@@ -142,10 +147,16 @@ export const riskCaches = (): RiskCaches => ({
142
147
  lockfiles: lru(200_000, read => 'packages' in read ? read.packages.length + 1 : 1),
143
148
  risks: lru(50_000),
144
149
  advisories: lru(50_000, kept => Array.isArray(kept.audit) ? kept.audit.length + 1 : 1),
150
+ // By their text, a kilobyte at a time.
151
+ actionFiles: lru(64 * 1024, read => 'files' in read ? read.files.reduce((sum, file) => sum + ('text' in file ? Math.ceil(file.text.length / 1024) : 1), 1) : 1),
152
+ actions: lru(10_000),
153
+ // What GitHub answered of public repositories' actions, by answerWeight.
154
+ actionAnswers: lru(200_000, answerWeight),
145
155
  })
146
- // How long the source's `listings` and `lockfiles` are read for: what
147
- // GitHub has at a commit or a blob doesn't change, how it is read here may.
148
- export const SCAN_READ_KEPT_MS = 24 * 60 * 60_000
156
+ // How many repositories are read at once (repository-risk-read.ts), ahead of
157
+ // the one told: each asks GitHub one thing after another (opening it, and
158
+ // its head) even where all else is kept.
159
+ const READ_AHEAD = 6
149
160
  // How long a part's progress, written as it ends (`progress`), is waited on.
150
161
  const PROGRESS_WRITE_MS = 5000
151
162
  // How many kept risk lines are read of the source's `risks` at once.
@@ -168,6 +179,7 @@ interface Scan {
168
179
  stale: AssessedPackage[]
169
180
  // The repositories told whole, this part or one before (ScanProgress).
170
181
  told: Map<number, ToldRepository>
182
+ actions: ReturnType<typeof resolvingActions> | null
171
183
  }
172
184
 
173
185
  // A package as a scan meets it: by its name, version and a mirror's integrity.
@@ -255,36 +267,6 @@ async function assessing(scan: Scan, feed: (assess: (packages: AssessedPackage[]
255
267
  await Promise.all(assessors)
256
268
  }
257
269
 
258
- type LockfileLine = Extract<RepositoryRiskLine, { type: 'lockfile' }>
259
-
260
- // What is kept by `key`: in this instance's memory, else in `store` within
261
- // SCAN_READ_KEPT_MS, else `read` and kept in both.
262
- async function keptRead<T>(memory: Lru<T>, store: KeptStore<T> | undefined, key: string, read: () => Promise<T>) {
263
- const known = memory.get(key)
264
- if (known !== undefined) return known
265
- const kept = await store?.read(key)
266
- if (kept && Date.now() - kept.at < SCAN_READ_KEPT_MS) { memory.set(key, kept.value); return kept.value }
267
- const value = await read()
268
- memory.set(key, value)
269
- await store?.write(key, value, Date.now())
270
- return value
271
- }
272
-
273
- // A lockfile's line, its packages or why they couldn't be had: as kept by its
274
- // blob, else read from GitHub and kept.
275
- async function readLockfile({ source, caches }: Scan, repo: number, opened: OpenedRepository, { path, format, sha }: ListedLockfile): Promise<LockfileLine> {
276
- const at = { type: 'lockfile', repo, path, format } as const
277
- if (!READ_FORMATS.has(format)) return { ...at, error: 'unsupported-format' }
278
- try {
279
- return { ...at, ...await keptRead(caches.lockfiles, source.lockfiles, `${format}\0${sha}`, async (): Promise<LockfileRead> => {
280
- const text = await opened.file(path)
281
- try { return { packages: lockedPackages(format, text) } } catch (error) {
282
- return { error: 'lockfile-unreadable', ...error instanceof Error ? { detail: error.message.slice(0, 500) } : {} }
283
- }
284
- }) }
285
- } catch (error) { return { ...at, ...failureOf(error) } }
286
- }
287
-
288
270
  // A lockfile's packages of npm's registry or a mirror not yet seen this scan,
289
271
  // to look up as kept (lookUp), and assess where none is; and one seen as only
290
272
  // devDependencies need it, and put off, that this lockfile needs otherwise,
@@ -303,25 +285,8 @@ function assessLater(scan: Scan, packages: LockedPackage[]) {
303
285
  if (fresh.length > 0) scan.lookups.add(fresh)
304
286
  }
305
287
 
306
- // A repository opened and its lockfiles of npm's ecosystem listed, as kept
307
- // for its commit or else from GitHub, those of its `paths` alone; null, with
308
- // its line telling why, where either failed, or where the scan stopped.
309
- async function listLockfiles(scan: Scan, repo: RepositoryRiskRepo) {
310
- let doing = 'opening it'
311
- try {
312
- const opened = await scan.source.open(repo)
313
- if (scan.signal.aborted) return null
314
- doing = 'listing its lockfiles'
315
- const listed = await keptRead(scan.caches.listings, scan.source.listings, `${repo.repoId}@${opened.commit}`, () => opened.lockfiles())
316
- const lockfiles = listed.filter(({ path, format }) => NPM_LOCKFILE_FORMATS.has(format) && lockfileOf(path, repo.paths))
317
- return { opened, lockfiles }
318
- } catch (error) {
319
- if (!scan.signal.aborted) scan.lines.add({ type: 'repo', id: repo.repoId, ...failureOf(error, doing) })
320
- return null
321
- }
322
- }
323
-
324
288
  type AdvisoriesLine = Extract<RepositoryRiskLine, { type: 'advisories' }>
289
+ type LockfileLine = Extract<RepositoryRiskLine, { type: 'lockfile' }>
325
290
 
326
291
  // A lockfile's advisories line (repository-risk-audit.ts auditLockfile);
327
292
  // null where it locks none, or once the scan stops.
@@ -331,36 +296,67 @@ async function readAdvisories(scan: Scan, { repo, path, format, packages }: Lock
331
296
  return audit === null ? null : Array.isArray(audit) ? { ...at, advisories: audit } : { ...at, ...audit }
332
297
  }
333
298
 
334
- // A repository's line, its lockfiles or why they couldn't be listed, and
335
- // then its lockfiles', and their advisories: told once they are all read
336
- // and access to the repository is checked again, as GitHub may have
337
- // withdrawn it meanwhile.
338
- async function readRepository(scan: Scan, repo: RepositoryRiskRepo) {
339
- const listed = await listLockfiles(scan, repo)
340
- if (!listed) return
341
- const lockfiles: LockfileLine[] = []
342
- for (const lockfile of listed.lockfiles) {
343
- if (scan.signal.aborted) return
344
- lockfiles.push(await readLockfile(scan, repo.repoId, listed.opened, lockfile))
345
- }
346
- try { await listed.opened.recheck() } catch (error) {
299
+ type DependabotLine = Extract<RepositoryRiskLine, { type: 'dependabot' }>
300
+
301
+ // A repository's Dependabot alerts' line, each alert with the versions the
302
+ // lockfile it is of locks that it covers (`lockfiles`, a repository read's),
303
+ // or why they couldn't be had.
304
+ function dependabotLine(repo: number, read: DependabotRead, lockfiles: RepositoryRead['lockfiles']): DependabotLine {
305
+ if ('error' in read) return { type: 'dependabot', repo, ...read }
306
+ const locked = new Map(lockfiles.flatMap(({ line }) => 'packages' in line ? [[line.path, line.packages] as const] : []))
307
+ return { type: 'dependabot', repo, alerts: read.alerts.map(alert => ({ ...alert, versions: alertVersions(alert, locked.get(alert.manifest)),
308
+ malware: npmMalwareAdvisory({ id: alert.ghsa, aliases: alert.aliases, cwe: alert.cwe, title: alert.title }) })) }
309
+ }
310
+
311
+ // A repository's alerts checked again just before they are told, as access to
312
+ // the repository is (DependabotRead `recheck`): why not, where the user's
313
+ // access to them is withdrawn since they were read.
314
+ async function rechecked(read: DependabotRead): Promise<DependabotRead> {
315
+ if (!('alerts' in read) || !read.recheck) return read
316
+ try { await read.recheck() } catch (error) { return failureOf(error, 'checking access to its Dependabot alerts again') }
317
+ return read
318
+ }
319
+
320
+ // A repository read (repository-risk-read.ts readRepository) told: its
321
+ // line, or why it couldn't be read, then its lockfiles', and their
322
+ // advisories, its packages assessed meanwhile, then its Dependabot alerts.
323
+ // Access to it is checked again
324
+ // just before, however long ago it was read, as GitHub may have withdrawn it.
325
+ async function tellRepository(scan: Scan, repo: RepositoryRiskRepo, read: RepositoryRead | RepositoryFailure | null) {
326
+ if (!read || scan.signal.aborted) return
327
+ if ('type' in read) { scan.lines.add(read); return }
328
+ try { await read.recheck() } catch (error) {
347
329
  if (!scan.signal.aborted) scan.lines.add({ type: 'repo', id: repo.repoId, ...failureOf(error, 'checking access to it again') })
348
330
  return
349
331
  }
350
332
  if (scan.signal.aborted) return
351
- scan.lines.add({ type: 'repo', id: repo.repoId, commit: listed.opened.commit, lockfiles: listed.lockfiles.map(({ path, format }) => ({ path, format })) })
352
- for (const line of lockfiles) {
333
+ const { commit, lockfiles } = read
334
+ scan.lines.add({ type: 'repo', id: repo.repoId, commit, lockfiles: lockfiles.map(({ line: { path, format } }) => ({ path, format })) })
335
+ for (const { line } of lockfiles) {
353
336
  scan.lines.add(line)
354
337
  if ('packages' in line) assessLater(scan, line.packages)
355
338
  }
356
- for (const [at, line] of lockfiles.entries()) {
339
+ const refs = read.actions ? tellActions(scan, repo, read.actions, lockfiles) : []
340
+ for (const { sha, line } of lockfiles) {
357
341
  if (scan.signal.aborted) return
358
- const advisories = 'packages' in line ? await readAdvisories(scan, line, listed.lockfiles[at]!.sha) : null
342
+ const advisories = 'packages' in line ? await readAdvisories(scan, line, sha) : null
359
343
  if (advisories) scan.lines.add(advisories)
360
344
  }
361
345
  if (scan.signal.aborted) return
346
+ const dependabot = read.dependabot && await rechecked(read.dependabot)
347
+ if (scan.signal.aborted) return
348
+ if (dependabot) scan.lines.add(dependabotLine(repo.repoId, dependabot, lockfiles))
362
349
  scan.told.set(repo.repoId, { id: repo.repoId, name: repo.fullName, paths: repo.paths ?? null,
363
- lockfiles: lockfiles.flatMap((line, at) => 'packages' in line ? [`${line.format}\0${listed.lockfiles[at]!.sha}`] : []) })
350
+ lockfiles: lockfiles.flatMap(({ sha, line }) => 'packages' in line ? [`${line.format}\0${sha}`] : []), ...refs.length > 0 ? { actions: refs } : {} })
351
+ }
352
+
353
+ // A repository's Actions told, and the references they make to resolve.
354
+ function tellActions(scan: Scan, repo: RepositoryRiskRepo, read: WorkflowsRead, lockfiles: RepositoryRead['lockfiles']) {
355
+ const locked = lockfiles.flatMap(({ line }) => 'packages' in line ? [lockedNames(line.path, line.packages)] : [])
356
+ const { lines, refs } = actionsLines(repo.repoId, repo.fullName.split('/')[0]!, read, locked)
357
+ for (const line of lines) scan.lines.add(line)
358
+ scan.actions?.add(refs)
359
+ return refs
364
360
  }
365
361
 
366
362
  // Whether an earlier part told `repo` whole, by its name and a team's
@@ -377,11 +373,12 @@ async function toldBefore(scan: Scan, repo: RepositoryRiskRepo) {
377
373
  reads.push(read.packages)
378
374
  }
379
375
  for (const packages of reads) assessLater(scan, packages)
376
+ scan.actions?.add(told.actions ?? [])
380
377
  return true
381
378
  }
382
379
 
383
- // Each repository, a repository at a time, but those an earlier part told
384
- // whole (ScanProgress).
380
+ // Each repository, READ_AHEAD read at a time, each told in turn, but those an
381
+ // earlier part told whole (ScanProgress), read of none.
385
382
  async function readRepositories(scan: Scan) {
386
383
  const repos = await scan.source.repos()
387
384
  scan.lines.add({ type: 'repos', repos: repos.map(repo => ({ id: repo.repoId, fullName: repo.fullName })) })
@@ -391,9 +388,19 @@ async function readRepositories(scan: Scan) {
391
388
  const kept = await scan.source.progress?.read()
392
389
  const progress = kept && Date.now() - kept.at < SCAN_READ_KEPT_MS ? kept.value.repos : []
393
390
  for (const told of progress) if (listed.has(told.id)) scan.told.set(told.id, told)
394
- for (const repo of repos) {
391
+ const reads: Promise<RepositoryRead | RepositoryFailure | null>[] = []
392
+ const read = (at: number) => {
393
+ if (reads[at]) return reads[at]
394
+ const repo = repos[at]!
395
+ const started = reads[at] = toldBefore(scan, repo).then(told => told ? null : readRepository(scan, repo))
396
+ // One read ahead and no longer waited on, as the scan stopped, fails alone.
397
+ started.catch(() => {})
398
+ return started
399
+ }
400
+ for (const [at, repo] of repos.entries()) {
395
401
  if (scan.signal.aborted) return
396
- if (!await toldBefore(scan, repo)) await readRepository(scan, repo)
402
+ for (let ahead = at; ahead < Math.min(at + READ_AHEAD, repos.length); ahead++) void read(ahead)
403
+ await tellRepository(scan, repo, await read(at))
397
404
  }
398
405
  }
399
406
 
@@ -429,7 +436,9 @@ export async function* repositoryRiskLines<R extends RepositoryRiskRepo>(source:
429
436
  // none waited for: the work under way, such as a request to GitHub, which
430
437
  // isn't given up on at once, outlasting it.
431
438
  const lines = queue<RepositoryRiskLine>(AbortSignal.any([signal, part.signal]), { drain: true })
432
- const scan: Scan = { source, caches, signal: running, lines, lookups: queue(running), seen: new Map(), dev: new Map(), stale: [], told: new Map() }
439
+ const actions = source.actions ? resolvingActions({ source: source.actions, signal: running, tell: line => { lines.add(line) }, memory: caches.actions,
440
+ answers: caches.actionAnswers, store: source.actionLines }) : null
441
+ const scan: Scan = { source, caches, signal: running, lines, lookups: queue(running), seen: new Map(), dev: new Map(), stale: [], told: new Map(), actions }
433
442
  const failed: unknown[] = []
434
443
  void (async () => {
435
444
  try {
@@ -444,6 +453,7 @@ export async function* repositoryRiskLines<R extends RepositoryRiskRepo>(source:
444
453
  for (const left of leftOver(scan)) {
445
454
  if (!running.aborted && left.length > 0) await assessing(scan, assess => { assess(left); return Promise.resolve() })
446
455
  }
456
+ await actions?.close()
447
457
  if (!running.aborted) scan.lines.add({ type: 'done' })
448
458
  } catch (error) {
449
459
  failed.push(error)