@preventive/triage 1.0.0-alpha.44 → 1.0.0-alpha.46
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/client/linked-findings.js +3 -0
- package/common/bundle-metadata.d.ts +1 -0
- package/common/bundle-metadata.js +1 -1
- package/common/bundle-sources.js +25 -2
- package/common/default-scan-models.ts +1 -1
- package/common/dependency-reviews.d.ts +0 -5
- package/common/dependency-reviews.js +0 -17
- package/common/line-diff.js +5 -4
- package/common/managed/agent-tasks.ts +292 -0
- package/common/managed/github-agents.ts +92 -0
- package/common/managed/github-logins.ts +49 -0
- package/common/managed/roles.ts +7 -0
- package/common/managed/routes.js +88 -7
- package/common/managed/team-app.d.ts +5 -2
- package/common/managed/team-app.js +2 -2
- package/common/server-info.ts +3 -0
- package/common/workspace-app.js +6 -3
- package/out/brotli-fallback.js +3 -3
- package/out/chunk-2XUEZHIP.js +18 -18
- package/out/chunk-2ZDZWLFY.js +1 -0
- package/out/chunk-3PS4KWD2.js +10 -0
- package/out/chunk-4HI2MSAV.js +17 -0
- package/out/chunk-4UFAMJAX.js +1 -0
- package/out/chunk-7C7DWL7I.js +1 -0
- package/out/chunk-7XRR4OFU.js +1 -0
- package/out/chunk-BC2UCLFJ.js +1 -0
- package/out/chunk-BOIXMUU5.js +3 -0
- package/out/chunk-C4Z6ZBXV.js +6 -6
- package/out/chunk-CC4OM6TZ.js +3 -0
- package/out/chunk-G24A42V6.js +2 -0
- package/out/chunk-GDPCLSOS.js +2 -0
- package/out/chunk-GO7FT4OD.js +1 -0
- package/out/chunk-I2MDGSQX.js +3 -0
- package/out/chunk-IYPY47MC.js +1 -0
- package/out/chunk-IZ7POWI3.js +18 -0
- package/out/chunk-JCIIBLKR.js +1 -1
- package/out/chunk-JKGQ3O7O.js +6 -0
- package/out/chunk-LE5ZWLG4.js +235 -0
- package/out/chunk-MSION3G4.js +36 -0
- package/out/chunk-NRPC74AM.js +1 -1
- package/out/chunk-OKRLLE5I.js +17 -0
- package/out/chunk-OOJM4CTU.js +1 -1
- package/out/chunk-PNHBZGDD.js +1 -1
- package/out/chunk-PXK7EU4V.js +1 -0
- package/out/chunk-QB4CBOX2.js +1 -0
- package/out/chunk-R3NZFXVR.js +1 -0
- package/out/chunk-R5O6FAW4.js +1 -1
- package/out/chunk-S6GIXWQW.js +52 -0
- package/out/chunk-SXV2T5C4.js +80 -0
- package/out/chunk-TUII34K2.js +1 -1
- package/out/chunk-V3KWNNMY.js +1 -0
- package/out/chunk-VFWMQ3LG.js +2 -0
- package/out/chunk-VH4XMZIW.js +2 -0
- package/out/chunk-WFYLMF6Z.js +10 -0
- package/out/chunk-WQ74ZBZF.js +7 -0
- package/out/client-managed-import.js +3 -3
- package/out/client-managed.js +31 -35
- package/out/client-sync.js +11 -11
- package/out/compare.js +35 -0
- package/out/dialogs.js +51 -0
- package/out/github-issue-authorized.html +1 -1
- package/out/github-issue-authorized.js +1 -1
- package/out/graph.js +17 -17
- package/out/index.html +30 -34
- package/out/prism.js +14 -14
- package/out/scan.js +2 -0
- package/out/terminal.js +294 -293
- package/out/view.css +1 -1
- package/out/view.js +16 -107
- package/package.json +53 -6
- package/risk/actions/findings.js +161 -0
- package/risk/actions/resolve.js +219 -0
- package/risk/actions/scripts/commands.js +182 -0
- package/risk/actions/scripts/grade.js +113 -0
- package/risk/actions/scripts/packages.js +123 -0
- package/risk/actions/scripts/shell.js +169 -0
- package/risk/actions/uses.js +76 -0
- package/risk/actions/workflow.js +110 -0
- package/risk/files/hosts.js +6 -6
- package/risk/files/readability.js +23 -17
- package/risk/files/runs-code.js +18 -6
- package/risk/files/script.js +7 -3
- package/risk/files/stamps.js +8 -7
- package/risk/index.d.ts +71 -1
- package/risk/index.js +6 -2
- package/risk/npm/assess.js +1 -1
- package/risk/npm/compare.js +34 -10
- package/risk/npm/files.js +413 -2
- package/risk/npm/tarball.js +56 -73
- package/server-e2e/app.ts +5 -4
- package/server-e2e/http.ts +5 -2
- package/server-managed/agent-merges.ts +198 -0
- package/server-managed/app-agents.ts +30 -0
- package/server-managed/avatar-store.ts +1 -0
- package/server-managed/bundle-cache.ts +7 -1
- package/server-managed/combined.ts +3 -2
- package/server-managed/commit-coauthors.ts +83 -0
- package/server-managed/commit-lockfiles.ts +72 -0
- package/server-managed/config.ts +6 -0
- package/server-managed/db-methods.ts +66 -19
- package/server-managed/db-neon.ts +89 -21
- package/server-managed/db-schema.ts +13 -2
- package/server-managed/db.ts +9 -1
- package/server-managed/dependabot-access.ts +60 -0
- package/server-managed/dependabot-alerts.ts +185 -0
- package/server-managed/dependency-review-comments.ts +97 -0
- package/server-managed/dependency-reviews.ts +78 -18
- package/server-managed/github-accounts.ts +33 -0
- package/server-managed/github-actions.ts +71 -0
- package/server-managed/github-app.ts +33 -13
- package/server-managed/github-errors.ts +2 -0
- package/server-managed/github-issue-links.ts +11 -5
- package/server-managed/github-link-oauth.ts +69 -0
- package/server-managed/github-links.ts +204 -0
- package/server-managed/github-oauth.ts +5 -2
- package/server-managed/github-pulls.ts +2 -2
- package/server-managed/github-requests.ts +39 -0
- package/server-managed/github-users.ts +30 -0
- package/server-managed/http-response.ts +10 -2
- package/server-managed/http.ts +1251 -97
- package/server-managed/index.ts +13 -8
- package/server-managed/kept-briefly.ts +37 -0
- package/server-managed/locked-packages.ts +37 -16
- package/server-managed/lockfile-scope.ts +49 -0
- package/server-managed/npm-delta-worker.js +23 -0
- package/server-managed/npm-delta.ts +178 -0
- package/server-managed/npm-insights.ts +20 -11
- package/server-managed/npm-kept.ts +83 -6
- package/server-managed/npm-loads.ts +114 -37
- package/server-managed/npm-packages.ts +110 -64
- package/server-managed/npm-risk.ts +4 -1
- package/server-managed/pretty-print.ts +67 -23
- package/server-managed/pull-comment-reviews.ts +161 -0
- package/server-managed/pull-dependencies.ts +300 -0
- package/server-managed/pull-dependency-reviews.ts +318 -0
- package/server-managed/pull-reviews.ts +205 -0
- package/server-managed/queue.ts +16 -0
- package/server-managed/release-commits.ts +84 -0
- package/server-managed/rename-diff.ts +22 -13
- package/server-managed/report-sources.ts +1 -1
- package/server-managed/report-store.ts +42 -9
- package/server-managed/repository-actions.ts +218 -0
- package/server-managed/repository-agents.ts +220 -0
- package/server-managed/repository-browser.ts +166 -37
- package/server-managed/repository-changes.ts +284 -0
- package/server-managed/repository-code.ts +52 -0
- package/server-managed/repository-dependency-reviews.ts +157 -0
- package/server-managed/repository-history.ts +461 -0
- package/server-managed/repository-issues.ts +181 -0
- package/server-managed/repository-pulse.ts +439 -0
- package/server-managed/repository-risk-audit.ts +13 -0
- package/server-managed/repository-risk-read.ts +170 -0
- package/server-managed/repository-risk.ts +133 -118
- package/server-managed/reviewed-dependencies.ts +149 -0
- package/server-managed/session.ts +27 -9
- package/server-managed/sql.ts +6 -3
- package/server-managed/storage-stores.ts +5 -0
- package/server-managed/storage.ts +4 -2
- package/server-managed/team-app.ts +19 -3
- package/server-managed/team-reports.ts +24 -12
- package/server-managed/upstream-cache.ts +12 -0
- package/server-managed/workspace-shares.ts +4 -1
- package/server.ts +25 -5
- package/out/chunk-2ABLKCOY.js +0 -46
- package/out/chunk-2KBF5TZ7.js +0 -2
- package/out/chunk-35VE6CDH.js +0 -2
- package/out/chunk-3BZ7JERB.js +0 -1
- package/out/chunk-4XCDFGDP.js +0 -10
- package/out/chunk-5LSXOLUS.js +0 -1
- package/out/chunk-5RCM2J63.js +0 -1
- package/out/chunk-7B6HB4D7.js +0 -1
- package/out/chunk-AJ4GTATZ.js +0 -3
- package/out/chunk-CH4UN6WK.js +0 -225
- package/out/chunk-CMQA4DM6.js +0 -1
- package/out/chunk-CVVJAXVY.js +0 -2
- package/out/chunk-DIJJNTRH.js +0 -10
- package/out/chunk-EDKLBQQN.js +0 -17
- package/out/chunk-F47DXCEV.js +0 -33
- package/out/chunk-FK3DZEKL.js +0 -1
- package/out/chunk-G6AJWENO.js +0 -1
- package/out/chunk-GJ4IUCNS.js +0 -1
- package/out/chunk-GNXOKIGP.js +0 -80
- package/out/chunk-GZ4MS3TE.js +0 -13
- package/out/chunk-GZWM7UQM.js +0 -2
- package/out/chunk-IU3CCINR.js +0 -3
- package/out/chunk-K3X5EJBQ.js +0 -2
- package/out/chunk-KEX2ATEW.js +0 -2
- package/out/chunk-KYCNJVB7.js +0 -1
- package/out/chunk-LPCTJYNA.js +0 -7
- package/out/chunk-O4RJY5TP.js +0 -2
- package/out/chunk-OD4LPN3H.js +0 -1
- package/out/chunk-QWDWTBM7.js +0 -1
- package/out/chunk-QYWKC557.js +0 -1
- package/out/chunk-R75XOKUP.js +0 -1
- package/out/chunk-SWTLADW5.js +0 -1
- package/out/chunk-T7OKH4VW.js +0 -1
- package/out/chunk-TZ5KB2Y5.js +0 -10
- package/out/chunk-UNTRGE26.js +0 -2
- package/out/chunk-UTR2LVME.js +0 -2
- package/out/chunk-UYAXVRVL.js +0 -1
- package/out/chunk-V3ME6FI5.js +0 -2
- package/out/chunk-V5O7M2YT.js +0 -1
- package/out/chunk-VIHGO77Y.js +0 -1
- package/out/chunk-W4J3MGLB.js +0 -1
- package/out/chunk-WGG7KQUH.js +0 -1
- package/out/chunk-XYFXXB2G.js +0 -10
- package/out/chunk-ZESLQCTF.js +0 -6
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
// A repository as a repository risk scan reads it of GitHub
|
|
2
|
+
// (repository-risk.ts): opened at its default branch's head, its lockfiles
|
|
3
|
+
// of npm's ecosystem listed (findRepoLockfiles), those of its `paths` alone,
|
|
4
|
+
// each read for its packages (@preventive/lockfile), and access to it checked
|
|
5
|
+
// again before they are told (`recheck`); and its Dependabot alerts, open or dismissed, of
|
|
6
|
+
// npm's, where the source asks them (dependabot-alerts.ts), asked meanwhile
|
|
7
|
+
// and never kept, a failure to have them its own, and its GitHub Actions
|
|
8
|
+
// (repository-actions.ts). A listing is kept by the commit it was listed at,
|
|
9
|
+
// its Actions' files too, and a lockfile's packages by its blob: in the
|
|
10
|
+
// instance's memory, and the source's `listings`, `actionFiles` and
|
|
11
|
+
// `lockfiles` for SCAN_READ_KEPT_MS, so that only opening a repository and
|
|
12
|
+
// checking access to it again ask GitHub where all else is kept.
|
|
13
|
+
import type { LockfileFormat, RepoLockfile } from '@preventive/upstream/github.js'
|
|
14
|
+
import type { AlertsRead } from './dependabot-access.ts'
|
|
15
|
+
import type { DependabotAlert } from './dependabot-alerts.ts'
|
|
16
|
+
import { type LockedPackage, isTestFixture, lockedPackages } from './locked-packages.ts'
|
|
17
|
+
import type { Lru } from './lru.ts'
|
|
18
|
+
import type { KeptStore } from './npm-kept.ts'
|
|
19
|
+
import { inTurns } from './queue.ts'
|
|
20
|
+
import { type WorkflowsRead, lastingRead, readActions } from './repository-actions.ts'
|
|
21
|
+
import { type RiskFailure, failureOf } from './repository-risk-assess.ts'
|
|
22
|
+
import type { RepositoryRiskLine, RepositoryRiskRepo } from './repository-risk.ts'
|
|
23
|
+
|
|
24
|
+
// The formats of npm's ecosystem, as findRepoLockfiles names them, and those
|
|
25
|
+
// of them read here.
|
|
26
|
+
const NPM_LOCKFILE_FORMATS: ReadonlySet<LockfileFormat> = new Set(['npm', 'pnpm', 'yarn', 'bun', 'deno'])
|
|
27
|
+
const READ_FORMATS: ReadonlySet<LockfileFormat> = new Set(['npm', 'pnpm', 'yarn'])
|
|
28
|
+
|
|
29
|
+
// How long the source's `listings` and `lockfiles` are read for: what
|
|
30
|
+
// GitHub has at a commit or a blob doesn't change, how it is read here may.
|
|
31
|
+
export const SCAN_READ_KEPT_MS = 24 * 60 * 60_000
|
|
32
|
+
// How many of a repository's lockfiles are read at once: each one GitHub is
|
|
33
|
+
// asked for where it isn't kept, then parsed.
|
|
34
|
+
const LOCKFILES_AT_ONCE = 4
|
|
35
|
+
|
|
36
|
+
export type ListedLockfile = Pick<RepoLockfile, 'path' | 'format' | 'sha'>
|
|
37
|
+
export type LockfileRead = { packages: LockedPackage[] } | RiskFailure
|
|
38
|
+
// A repository's Dependabot alerts, those of its `paths` alone, with access to
|
|
39
|
+
// them to check again before they are told (`recheck`), or why they couldn't
|
|
40
|
+
// be had.
|
|
41
|
+
export type DependabotRead = { alerts: DependabotAlert[]; recheck?(): Promise<void> } | RiskFailure
|
|
42
|
+
type LockfileLine = Extract<RepositoryRiskLine, { type: 'lockfile' }>
|
|
43
|
+
export type RepositoryFailure = Extract<RepositoryRiskLine, { type: 'repo'; error: string }>
|
|
44
|
+
|
|
45
|
+
// A repository opened at its default branch's head (`commit`): its lockfiles,
|
|
46
|
+
// as findRepoLockfiles lists them, each with its blob's id, and a file's
|
|
47
|
+
// text; `recheck` checks access to it again, throwing where it is gone; and
|
|
48
|
+
// `dependabot`, where asked, its Dependabot alerts of npm's, open or dismissed, as GitHub
|
|
49
|
+
// has them now, with access to them to check again before they are told.
|
|
50
|
+
export interface OpenedRepository {
|
|
51
|
+
commit: string
|
|
52
|
+
lockfiles(): Promise<ListedLockfile[]>
|
|
53
|
+
file(path: string): Promise<string>
|
|
54
|
+
recheck(): Promise<void>
|
|
55
|
+
dependabot?(): Promise<AlertsRead>
|
|
56
|
+
// Its workflows' paths, none where it has none.
|
|
57
|
+
workflows?(): Promise<string[]>
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
// What a repository is read with: the scan's source and caches, and its
|
|
61
|
+
// signal.
|
|
62
|
+
export interface RepositoryReading<R extends RepositoryRiskRepo> {
|
|
63
|
+
source: { open(repo: R): Promise<OpenedRepository>; listings?: KeptStore<ListedLockfile[]>; lockfiles?: KeptStore<LockfileRead>; actionFiles?: KeptStore<WorkflowsRead> }
|
|
64
|
+
caches: { listings: Lru<ListedLockfile[]>; lockfiles: Lru<LockfileRead>; actionFiles: Lru<WorkflowsRead> }
|
|
65
|
+
signal: AbortSignal
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
// A repository read: its commit, its lockfiles' lines, each with its blob's
|
|
69
|
+
// id, its Dependabot alerts where asked, and access to it to check again just
|
|
70
|
+
// before they are told (OpenedRepository `recheck`), as GitHub may have
|
|
71
|
+
// withdrawn it meanwhile.
|
|
72
|
+
export interface RepositoryRead {
|
|
73
|
+
commit: string
|
|
74
|
+
lockfiles: { sha: string; line: LockfileLine }[]
|
|
75
|
+
dependabot: DependabotRead | null
|
|
76
|
+
// Its Actions' files, where it lists its workflows.
|
|
77
|
+
actions: WorkflowsRead | null
|
|
78
|
+
recheck(): Promise<void>
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
// Whether a lockfile at `path` is of `paths` (RepositoryRiskRepo).
|
|
82
|
+
function lockfileOf(path: string, paths: readonly string[] | undefined) {
|
|
83
|
+
const dir = path.slice(0, Math.max(path.lastIndexOf('/'), 0))
|
|
84
|
+
return !paths || paths.some(scope => dir === '' || dir === scope || dir.startsWith(`${scope}/`) || scope.startsWith(`${dir}/`))
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
// What is kept by `key`: in this instance's memory, else in `store` within
|
|
88
|
+
// SCAN_READ_KEPT_MS, else `read` and kept in both, where it is to be.
|
|
89
|
+
async function keptRead<T>(memory: Lru<T>, store: KeptStore<T> | undefined, key: string, read: () => Promise<T>, keep = (_value: T) => true) {
|
|
90
|
+
const known = memory.get(key)
|
|
91
|
+
if (known !== undefined) return known
|
|
92
|
+
const kept = await store?.read(key)
|
|
93
|
+
if (kept && Date.now() - kept.at < SCAN_READ_KEPT_MS) { memory.set(key, kept.value); return kept.value }
|
|
94
|
+
const value = await read()
|
|
95
|
+
if (!keep(value)) return value
|
|
96
|
+
memory.set(key, value)
|
|
97
|
+
await store?.write(key, value, Date.now())
|
|
98
|
+
return value
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
// A lockfile's line, its packages or why they couldn't be had: as kept by its
|
|
102
|
+
// blob, else read from GitHub and kept.
|
|
103
|
+
async function readLockfile({ source, caches }: RepositoryReading<RepositoryRiskRepo>, repo: number, opened: OpenedRepository,
|
|
104
|
+
{ path, format, sha }: ListedLockfile): Promise<LockfileLine> {
|
|
105
|
+
const at = { type: 'lockfile', repo, path, format } as const
|
|
106
|
+
if (!READ_FORMATS.has(format)) return { ...at, error: 'unsupported-format' }
|
|
107
|
+
try {
|
|
108
|
+
return { ...at, ...await keptRead(caches.lockfiles, source.lockfiles, `${format}\0${sha}`, async (): Promise<LockfileRead> => {
|
|
109
|
+
const text = await opened.file(path)
|
|
110
|
+
try { return { packages: lockedPackages(format, text) } } catch (error) {
|
|
111
|
+
return { error: 'lockfile-unreadable', ...error instanceof Error ? { detail: error.message.slice(0, 500) } : {} }
|
|
112
|
+
}
|
|
113
|
+
}) }
|
|
114
|
+
} catch (error) { return { ...at, ...failureOf(error) } }
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
// A repository opened and its lockfiles of npm's ecosystem listed, as kept
|
|
118
|
+
// for its commit or else from GitHub, those of its `paths` alone; its line
|
|
119
|
+
// telling why where either failed; null where the scan stopped.
|
|
120
|
+
async function listLockfiles<R extends RepositoryRiskRepo>(reading: RepositoryReading<R>, repo: R) {
|
|
121
|
+
let doing = 'opening it'
|
|
122
|
+
try {
|
|
123
|
+
const opened = await reading.source.open(repo)
|
|
124
|
+
if (reading.signal.aborted) return null
|
|
125
|
+
doing = 'listing its lockfiles'
|
|
126
|
+
const listed = await keptRead(reading.caches.listings, reading.source.listings, `${repo.repoId}@${opened.commit}`, () => opened.lockfiles())
|
|
127
|
+
const lockfiles = listed.filter(({ path, format }) => NPM_LOCKFILE_FORMATS.has(format) && lockfileOf(path, repo.paths))
|
|
128
|
+
return { opened, lockfiles }
|
|
129
|
+
} catch (error) {
|
|
130
|
+
return reading.signal.aborted ? null : { type: 'repo', id: repo.repoId, ...failureOf(error, doing) } as const
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
// A repository's Dependabot alerts of its `paths` (RepositoryRiskRepo), as
|
|
135
|
+
// its lockfiles are, or why they couldn't be had; null where they aren't
|
|
136
|
+
// asked.
|
|
137
|
+
async function readDependabot(opened: OpenedRepository, paths: readonly string[] | undefined): Promise<DependabotRead | null> {
|
|
138
|
+
if (!opened.dependabot) return null
|
|
139
|
+
try {
|
|
140
|
+
const { alerts, recheck } = await opened.dependabot()
|
|
141
|
+
return { alerts: alerts.filter(alert => lockfileOf(alert.manifest, paths)), recheck }
|
|
142
|
+
} catch (error) {
|
|
143
|
+
return failureOf(error, 'asking its Dependabot alerts')
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
// A repository read: its lockfiles listed and read, a few at once, its
|
|
148
|
+
// Dependabot alerts asked and its Actions read meanwhile; its line telling why where listing
|
|
149
|
+
// failed; null where the scan stopped. A test's
|
|
150
|
+
// fixture or example its reader refuses (isTestFixture) is left out, as a
|
|
151
|
+
// test may keep one so on purpose.
|
|
152
|
+
export async function readRepository<R extends RepositoryRiskRepo>(reading: RepositoryReading<R>, repo: R): Promise<RepositoryRead | RepositoryFailure | null> {
|
|
153
|
+
const listed = await listLockfiles(reading, repo)
|
|
154
|
+
if (!listed || 'type' in listed) return listed
|
|
155
|
+
const alerting = readDependabot(listed.opened, repo.paths)
|
|
156
|
+
const { opened } = listed, { workflows } = opened
|
|
157
|
+
const acting = workflows ? keptRead(reading.caches.actionFiles, reading.source.actionFiles, `${repo.repoId}@${opened.commit}`,
|
|
158
|
+
() => readActions({ workflows, file: path => opened.file(path) }), lastingRead) : Promise.resolve(null)
|
|
159
|
+
acting.catch(() => {})
|
|
160
|
+
const inTurn = inTurns(LOCKFILES_AT_ONCE)
|
|
161
|
+
const lines = await Promise.all(listed.lockfiles.map(lockfile => inTurn(() => (reading.signal.aborted ? Promise.resolve(null) : readLockfile(reading, repo.repoId, opened, lockfile)))))
|
|
162
|
+
const lockfiles: RepositoryRead['lockfiles'] = []
|
|
163
|
+
for (const [at, line] of lines.entries()) {
|
|
164
|
+
if (!line) return null
|
|
165
|
+
if (!('error' in line && line.error === 'lockfile-unreadable' && isTestFixture(line.path))) lockfiles.push({ sha: listed.lockfiles[at]!.sha, line })
|
|
166
|
+
}
|
|
167
|
+
const dependabot = await alerting
|
|
168
|
+
const actions = await acting
|
|
169
|
+
return reading.signal.aborted ? null : { commit: listed.opened.commit, lockfiles, dependabot, actions, recheck: () => listed.opened.recheck() }
|
|
170
|
+
}
|
|
@@ -12,58 +12,47 @@
|
|
|
12
12
|
// host with an integrity, such as a company's mirror of npm's, as npm's
|
|
13
13
|
// package of its name and version where npm's integrity is among it; one
|
|
14
14
|
// from elsewhere (git, a tarball with no integrity, a file) is listed, not
|
|
15
|
-
// assessed. Repositories are read
|
|
16
|
-
// caller's; packages are assessed meanwhile, as many at a time as npm-risk.ts
|
|
15
|
+
// assessed. Repositories are read a few at a time (READ_AHEAD), each told in
|
|
16
|
+
// turn, as GitHub's limits are the caller's; packages are assessed meanwhile, as many at a time as npm-risk.ts
|
|
17
17
|
// makes spare assessments, those only devDependencies need in every lockfile
|
|
18
18
|
// that locks them (LockedPackage `dev`) once all the rest are, as an install
|
|
19
19
|
// of no devDependencies leaves them out. A lockfile's packages of npm's
|
|
20
20
|
// registry or a mirror are asked of npm's advisories all at once, as `npm
|
|
21
21
|
// audit` asks, each advisory told with whether it is of malware (risk's
|
|
22
22
|
// npmMalwareAdvisory), which alone is a supply chain's risk; the rest are
|
|
23
|
-
// vulnerabilities, told beside.
|
|
24
|
-
|
|
25
|
-
|
|
23
|
+
// vulnerabilities, told beside. A repository's Dependabot alerts of npm's, open or dismissed,
|
|
24
|
+
// are told once its lockfiles' advisories are, each with the versions its
|
|
25
|
+
// lockfile locks that it covers, for the page to blend them into npm's; one
|
|
26
|
+
// whose alerts couldn't be had tells why, and the scan goes on. Its GitHub
|
|
27
|
+
// Actions are told once its lockfiles are, the references they make
|
|
28
|
+
// resolved as the scan goes on (repository-actions.ts).
|
|
29
|
+
import type { LockfileFormat } from '@preventive/upstream/github.js'
|
|
30
|
+
import { type Severity, npmMalwareAdvisory } from '../risk/index.js'
|
|
26
31
|
import { isNpmPackageName, isNpmPackageSpec } from '../common/managed/npm-packages.js'
|
|
27
32
|
import type { NpmVersionDocument } from './npm-packages.ts'
|
|
28
33
|
import { type Lru, lru } from './lru.ts'
|
|
29
34
|
import type { KeptStore } from './npm-kept.ts'
|
|
30
35
|
import { MAX_SPARE_ASSESSMENTS } from './npm-risk.ts'
|
|
31
|
-
import {
|
|
32
|
-
import { type Queue, queue } from './queue.ts'
|
|
36
|
+
import type { LockedPackage } from './locked-packages.ts'
|
|
37
|
+
import { type Queue, inTurns, queue } from './queue.ts'
|
|
33
38
|
import { type DownloadsSource, readyingDownloads } from './scan-downloads.ts'
|
|
34
39
|
import { type AssessedPackage, type Assessment, BUSY_RETRIES, BUSY_WAIT_MS, RISK_KEPT_MS, type RiskFailure, assessPackage, failureOf, isBusy } from './repository-risk-assess.ts'
|
|
35
|
-
import { type AdvisoryRow, type Audit, type AuditSource,
|
|
40
|
+
import { type AdvisoryRow, type Audit, type AuditSource, auditLockfiles } from './repository-risk-audit.ts'
|
|
41
|
+
import { type DependabotRead, type ListedLockfile, type LockfileRead, type OpenedRepository, type RepositoryFailure, type RepositoryRead, SCAN_READ_KEPT_MS,
|
|
42
|
+
readRepository } from './repository-risk-read.ts'
|
|
43
|
+
import { type DependabotAlert, alertVersions } from './dependabot-alerts.ts'
|
|
44
|
+
import type { ActionsSource } from '../risk/index.js'
|
|
45
|
+
import { type ActionAnswers, type ActionLine, type ActionRef, type WorkflowLine, type WorkflowsLine, type WorkflowsRead, actionsLines, answerWeight,
|
|
46
|
+
lockedNames, resolvingActions } from './repository-actions.ts'
|
|
36
47
|
|
|
37
48
|
export { RISK_KEPT_MS, highestSeverity } from './repository-risk-assess.ts'
|
|
38
|
-
|
|
39
|
-
// The formats of npm's ecosystem, as findRepoLockfiles names them, and those
|
|
40
|
-
// of them read here.
|
|
41
|
-
const NPM_LOCKFILE_FORMATS: ReadonlySet<LockfileFormat> = new Set(['npm', 'pnpm', 'yarn', 'bun', 'deno'])
|
|
42
|
-
const READ_FORMATS: ReadonlySet<LockfileFormat> = new Set(['npm', 'pnpm', 'yarn'])
|
|
49
|
+
export { type OpenedRepository, SCAN_READ_KEPT_MS } from './repository-risk-read.ts'
|
|
43
50
|
|
|
44
51
|
// A repository to read: where `paths` are given, a team's directories in it,
|
|
45
52
|
// only the lockfiles in them or above them, as a workspace's root lockfile
|
|
46
53
|
// locks them too.
|
|
47
54
|
export interface RepositoryRiskRepo { repoId: number; fullName: string; paths?: readonly string[] }
|
|
48
55
|
|
|
49
|
-
// Whether a lockfile at `path` is of `paths` (RepositoryRiskRepo).
|
|
50
|
-
function lockfileOf(path: string, paths: readonly string[] | undefined) {
|
|
51
|
-
const dir = path.slice(0, Math.max(path.lastIndexOf('/'), 0))
|
|
52
|
-
return !paths || paths.some(scope => dir === '' || dir === scope || dir.startsWith(`${scope}/`) || scope.startsWith(`${dir}/`))
|
|
53
|
-
}
|
|
54
|
-
|
|
55
|
-
type ListedLockfile = Pick<RepoLockfile, 'path' | 'format' | 'sha'>
|
|
56
|
-
|
|
57
|
-
// A repository opened at its default branch's head (`commit`): its lockfiles,
|
|
58
|
-
// as findRepoLockfiles lists them, each with its blob's id, and a file's
|
|
59
|
-
// text; `recheck` checks access to it again, throwing where it is gone.
|
|
60
|
-
export interface OpenedRepository {
|
|
61
|
-
commit: string
|
|
62
|
-
lockfiles(): Promise<ListedLockfile[]>
|
|
63
|
-
file(path: string): Promise<string>
|
|
64
|
-
recheck(): Promise<void>
|
|
65
|
-
}
|
|
66
|
-
|
|
67
56
|
export interface RepositoryRiskSource<R extends RepositoryRiskRepo> extends AuditSource {
|
|
68
57
|
repos(): Promise<R[]>
|
|
69
58
|
open(repo: R): Promise<OpenedRepository>
|
|
@@ -94,10 +83,21 @@ export interface RepositoryRiskSource<R extends RepositoryRiskRepo> extends Audi
|
|
|
94
83
|
// Readies npm's downloads of packages before they are assessed
|
|
95
84
|
// (scan-downloads.ts); without it, each assessment asks its own.
|
|
96
85
|
downloads?: DownloadsSource
|
|
86
|
+
// GitHub as references are resolved with; none without it.
|
|
87
|
+
actions?: ActionsSource
|
|
88
|
+
// Where Actions' files read and references resolved are kept.
|
|
89
|
+
actionFiles?: KeptStore<WorkflowsRead>
|
|
90
|
+
actionLines?: KeptStore<ActionLine>
|
|
97
91
|
}
|
|
98
92
|
|
|
99
93
|
|
|
100
94
|
|
|
95
|
+
// A Dependabot alert as a scan tells it: with the versions of its package
|
|
96
|
+
// the lockfile it is of locks that it covers, none where it is of none read
|
|
97
|
+
// (a package.json, bun's lockfile), and whether it is of malware, as an
|
|
98
|
+
// advisory's is told.
|
|
99
|
+
export type DependabotAlertRow = DependabotAlert & { versions: string[]; malware: boolean }
|
|
100
|
+
|
|
101
101
|
// A risk line is of npm's package of its name and version, or, with an
|
|
102
102
|
// `integrity`, of a mirror's that its lockfiles hold to it; its `score` risk's
|
|
103
103
|
// of it, out of 100 (assessNpm `score`, risk/npm/score.js).
|
|
@@ -108,6 +108,8 @@ export type RepositoryRiskLine =
|
|
|
108
108
|
| ({ type: 'risk'; name: string; version: string; integrity?: string; stale?: { assessedAt: number } }
|
|
109
109
|
& ({ published: string | null; severity: Severity | null; score: number; findings: { id: string; severity: Severity; takenDown?: true }[] } | RiskFailure))
|
|
110
110
|
| ({ type: 'advisories'; repo: number; path: string } & ({ advisories: AdvisoryRow[] } | RiskFailure))
|
|
111
|
+
| ({ type: 'dependabot'; repo: number } & ({ alerts: DependabotAlertRow[] } | RiskFailure))
|
|
112
|
+
| WorkflowsLine | WorkflowLine | ActionLine
|
|
111
113
|
| { type: 'done' }
|
|
112
114
|
// The scan stopped for its part's time (repositoryRiskLines `partMs`), what
|
|
113
115
|
// it told kept: to be asked again, from where it stopped.
|
|
@@ -115,12 +117,12 @@ export type RepositoryRiskLine =
|
|
|
115
117
|
|
|
116
118
|
// The repositories a scan answered in parts told whole by a part's end: each
|
|
117
119
|
// by its id, its name and a team's directories in it then, and its
|
|
118
|
-
// lockfiles' keys (`${format}\0${sha}`) of those read
|
|
120
|
+
// lockfiles' keys (`${format}\0${sha}`) of those read, and the references
|
|
121
|
+
// its Actions make, to resolve.
|
|
119
122
|
export interface ScanProgress { repos: ToldRepository[] }
|
|
120
|
-
interface ToldRepository { id: number; name: string; paths: readonly string[] | null; lockfiles: string[] }
|
|
123
|
+
interface ToldRepository { id: number; name: string; paths: readonly string[] | null; lockfiles: string[]; actions?: ActionRef[] }
|
|
121
124
|
|
|
122
125
|
type RiskLine = Extract<RepositoryRiskLine, { type: 'risk' }>
|
|
123
|
-
type LockfileRead = { packages: LockedPackage[] } | RiskFailure
|
|
124
126
|
|
|
125
127
|
// What scans keep between them, in this instance's memory: a repository's
|
|
126
128
|
// lockfiles as listed at a commit, and a lockfile's packages by its blob,
|
|
@@ -135,6 +137,9 @@ export interface RiskCaches {
|
|
|
135
137
|
lockfiles: Lru<LockfileRead>
|
|
136
138
|
risks: Lru<{ at: number; line: RiskLine }>
|
|
137
139
|
advisories: Lru<{ at: number; audit: Audit }>
|
|
140
|
+
actionFiles: Lru<WorkflowsRead>
|
|
141
|
+
actions: Lru<{ at: number; line: ActionLine }>
|
|
142
|
+
actionAnswers: ActionAnswers
|
|
138
143
|
}
|
|
139
144
|
export const riskCaches = (): RiskCaches => ({
|
|
140
145
|
listings: lru(1000),
|
|
@@ -142,10 +147,19 @@ export const riskCaches = (): RiskCaches => ({
|
|
|
142
147
|
lockfiles: lru(200_000, read => 'packages' in read ? read.packages.length + 1 : 1),
|
|
143
148
|
risks: lru(50_000),
|
|
144
149
|
advisories: lru(50_000, kept => Array.isArray(kept.audit) ? kept.audit.length + 1 : 1),
|
|
150
|
+
// By their text, a kilobyte at a time.
|
|
151
|
+
actionFiles: lru(64 * 1024, read => 'files' in read ? read.files.reduce((sum, file) => sum + ('text' in file ? Math.ceil(file.text.length / 1024) : 1), 1) : 1),
|
|
152
|
+
actions: lru(10_000),
|
|
153
|
+
// What GitHub answered of public repositories' actions, by answerWeight.
|
|
154
|
+
actionAnswers: lru(200_000, answerWeight),
|
|
145
155
|
})
|
|
146
|
-
// How
|
|
147
|
-
//
|
|
148
|
-
|
|
156
|
+
// How many repositories are read at once (repository-risk-read.ts), ahead of
|
|
157
|
+
// the one told: each asks GitHub one thing after another (opening it, and
|
|
158
|
+
// its head) even where all else is kept.
|
|
159
|
+
const READ_AHEAD = 6
|
|
160
|
+
// How many lockfiles' advisories are asked of npm at once, those of the
|
|
161
|
+
// repositories read ahead as they are read, each told in its turn.
|
|
162
|
+
const AUDITS_AT_ONCE = 4
|
|
149
163
|
// How long a part's progress, written as it ends (`progress`), is waited on.
|
|
150
164
|
const PROGRESS_WRITE_MS = 5000
|
|
151
165
|
// How many kept risk lines are read of the source's `risks` at once.
|
|
@@ -168,6 +182,8 @@ interface Scan {
|
|
|
168
182
|
stale: AssessedPackage[]
|
|
169
183
|
// The repositories told whole, this part or one before (ScanProgress).
|
|
170
184
|
told: Map<number, ToldRepository>
|
|
185
|
+
actions: ReturnType<typeof resolvingActions> | null
|
|
186
|
+
audits: ReturnType<typeof inTurns>
|
|
171
187
|
}
|
|
172
188
|
|
|
173
189
|
// A package as a scan meets it: by its name, version and a mirror's integrity.
|
|
@@ -255,36 +271,6 @@ async function assessing(scan: Scan, feed: (assess: (packages: AssessedPackage[]
|
|
|
255
271
|
await Promise.all(assessors)
|
|
256
272
|
}
|
|
257
273
|
|
|
258
|
-
type LockfileLine = Extract<RepositoryRiskLine, { type: 'lockfile' }>
|
|
259
|
-
|
|
260
|
-
// What is kept by `key`: in this instance's memory, else in `store` within
|
|
261
|
-
// SCAN_READ_KEPT_MS, else `read` and kept in both.
|
|
262
|
-
async function keptRead<T>(memory: Lru<T>, store: KeptStore<T> | undefined, key: string, read: () => Promise<T>) {
|
|
263
|
-
const known = memory.get(key)
|
|
264
|
-
if (known !== undefined) return known
|
|
265
|
-
const kept = await store?.read(key)
|
|
266
|
-
if (kept && Date.now() - kept.at < SCAN_READ_KEPT_MS) { memory.set(key, kept.value); return kept.value }
|
|
267
|
-
const value = await read()
|
|
268
|
-
memory.set(key, value)
|
|
269
|
-
await store?.write(key, value, Date.now())
|
|
270
|
-
return value
|
|
271
|
-
}
|
|
272
|
-
|
|
273
|
-
// A lockfile's line, its packages or why they couldn't be had: as kept by its
|
|
274
|
-
// blob, else read from GitHub and kept.
|
|
275
|
-
async function readLockfile({ source, caches }: Scan, repo: number, opened: OpenedRepository, { path, format, sha }: ListedLockfile): Promise<LockfileLine> {
|
|
276
|
-
const at = { type: 'lockfile', repo, path, format } as const
|
|
277
|
-
if (!READ_FORMATS.has(format)) return { ...at, error: 'unsupported-format' }
|
|
278
|
-
try {
|
|
279
|
-
return { ...at, ...await keptRead(caches.lockfiles, source.lockfiles, `${format}\0${sha}`, async (): Promise<LockfileRead> => {
|
|
280
|
-
const text = await opened.file(path)
|
|
281
|
-
try { return { packages: lockedPackages(format, text) } } catch (error) {
|
|
282
|
-
return { error: 'lockfile-unreadable', ...error instanceof Error ? { detail: error.message.slice(0, 500) } : {} }
|
|
283
|
-
}
|
|
284
|
-
}) }
|
|
285
|
-
} catch (error) { return { ...at, ...failureOf(error) } }
|
|
286
|
-
}
|
|
287
|
-
|
|
288
274
|
// A lockfile's packages of npm's registry or a mirror not yet seen this scan,
|
|
289
275
|
// to look up as kept (lookUp), and assess where none is; and one seen as only
|
|
290
276
|
// devDependencies need it, and put off, that this lockfile needs otherwise,
|
|
@@ -303,66 +289,79 @@ function assessLater(scan: Scan, packages: LockedPackage[]) {
|
|
|
303
289
|
if (fresh.length > 0) scan.lookups.add(fresh)
|
|
304
290
|
}
|
|
305
291
|
|
|
306
|
-
|
|
307
|
-
// for its commit or else from GitHub, those of its `paths` alone; null, with
|
|
308
|
-
// its line telling why, where either failed, or where the scan stopped.
|
|
309
|
-
async function listLockfiles(scan: Scan, repo: RepositoryRiskRepo) {
|
|
310
|
-
let doing = 'opening it'
|
|
311
|
-
try {
|
|
312
|
-
const opened = await scan.source.open(repo)
|
|
313
|
-
if (scan.signal.aborted) return null
|
|
314
|
-
doing = 'listing its lockfiles'
|
|
315
|
-
const listed = await keptRead(scan.caches.listings, scan.source.listings, `${repo.repoId}@${opened.commit}`, () => opened.lockfiles())
|
|
316
|
-
const lockfiles = listed.filter(({ path, format }) => NPM_LOCKFILE_FORMATS.has(format) && lockfileOf(path, repo.paths))
|
|
317
|
-
return { opened, lockfiles }
|
|
318
|
-
} catch (error) {
|
|
319
|
-
if (!scan.signal.aborted) scan.lines.add({ type: 'repo', id: repo.repoId, ...failureOf(error, doing) })
|
|
320
|
-
return null
|
|
321
|
-
}
|
|
322
|
-
}
|
|
292
|
+
type LockfileLine = Extract<RepositoryRiskLine, { type: 'lockfile' }>
|
|
323
293
|
|
|
324
|
-
|
|
294
|
+
// A lockfile's advisories line, of its audit (repository-risk-audit.ts auditLockfile).
|
|
295
|
+
const advisoriesLine = ({ repo, path }: LockfileLine, audit: Audit) => ({ type: 'advisories', repo, path, ...Array.isArray(audit) ? { advisories: audit } : audit } as const)
|
|
296
|
+
|
|
297
|
+
// A repository read, its lockfiles' audits asked meanwhile, AUDITS_AT_ONCE
|
|
298
|
+
// across the scan (auditLockfiles), to tell in turn once access is checked again.
|
|
299
|
+
type Audited = (RepositoryRead & { audits: Promise<Audit | null>[] }) | RepositoryFailure
|
|
300
|
+
const audited = (scan: Scan, read: RepositoryRead | RepositoryFailure | null): Audited | null => (read === null || 'type' in read ? read : { ...read, audits: auditLockfiles(scan.source, scan.caches.advisories,
|
|
301
|
+
scan.signal, read.lockfiles.map(({ sha, line }) => ({ key: `${line.format}\0${sha}`, packages: 'packages' in line ? line.packages : null })), scan.audits) })
|
|
302
|
+
|
|
303
|
+
type DependabotLine = Extract<RepositoryRiskLine, { type: 'dependabot' }>
|
|
304
|
+
|
|
305
|
+
// A repository's Dependabot alerts' line, each alert with the versions the
|
|
306
|
+
// lockfile it is of locks that it covers (`lockfiles`, a repository read's),
|
|
307
|
+
// or why they couldn't be had.
|
|
308
|
+
function dependabotLine(repo: number, read: DependabotRead, lockfiles: RepositoryRead['lockfiles']): DependabotLine {
|
|
309
|
+
if ('error' in read) return { type: 'dependabot', repo, ...read }
|
|
310
|
+
const locked = new Map(lockfiles.flatMap(({ line }) => 'packages' in line ? [[line.path, line.packages] as const] : []))
|
|
311
|
+
return { type: 'dependabot', repo, alerts: read.alerts.map(alert => ({ ...alert, versions: alertVersions(alert, locked.get(alert.manifest)),
|
|
312
|
+
malware: npmMalwareAdvisory({ id: alert.ghsa, aliases: alert.aliases, cwe: alert.cwe, title: alert.title }) })) }
|
|
313
|
+
}
|
|
325
314
|
|
|
326
|
-
// A
|
|
327
|
-
//
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
315
|
+
// A repository's alerts checked again just before they are told, as access to
|
|
316
|
+
// the repository is (DependabotRead `recheck`): why not, where the user's
|
|
317
|
+
// access to them is withdrawn since they were read.
|
|
318
|
+
async function rechecked(read: DependabotRead): Promise<DependabotRead> {
|
|
319
|
+
if (!('alerts' in read) || !read.recheck) return read
|
|
320
|
+
try { await read.recheck() } catch (error) { return failureOf(error, 'checking access to its Dependabot alerts again') }
|
|
321
|
+
return read
|
|
332
322
|
}
|
|
333
323
|
|
|
334
|
-
// A repository
|
|
335
|
-
// then its lockfiles', and their
|
|
336
|
-
//
|
|
337
|
-
//
|
|
338
|
-
//
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
if (!
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
if (scan.signal.aborted) return
|
|
345
|
-
const line = await readLockfile(scan, repo.repoId, listed.opened, lockfile)
|
|
346
|
-
if (!('error' in line && line.error === 'lockfile-unreadable' && isTestFixture(line.path))) lockfiles.push({ sha: lockfile.sha, line })
|
|
347
|
-
}
|
|
348
|
-
try { await listed.opened.recheck() } catch (error) {
|
|
324
|
+
// A repository read (repository-risk-read.ts readRepository) told: its
|
|
325
|
+
// line, or why it couldn't be read, then its lockfiles', and their
|
|
326
|
+
// advisories, asked as it was read, its packages assessed meanwhile, then
|
|
327
|
+
// its Dependabot alerts.
|
|
328
|
+
// Access to it is checked again
|
|
329
|
+
// just before, however long ago it was read, as GitHub may have withdrawn it.
|
|
330
|
+
async function tellRepository(scan: Scan, repo: RepositoryRiskRepo, read: Audited | null) {
|
|
331
|
+
if (!read || scan.signal.aborted) return
|
|
332
|
+
if ('type' in read) { scan.lines.add(read); return }
|
|
333
|
+
try { await read.recheck() } catch (error) {
|
|
349
334
|
if (!scan.signal.aborted) scan.lines.add({ type: 'repo', id: repo.repoId, ...failureOf(error, 'checking access to it again') })
|
|
350
335
|
return
|
|
351
336
|
}
|
|
352
337
|
if (scan.signal.aborted) return
|
|
353
|
-
|
|
338
|
+
const { commit, lockfiles } = read
|
|
339
|
+
scan.lines.add({ type: 'repo', id: repo.repoId, commit, lockfiles: lockfiles.map(({ line: { path, format } }) => ({ path, format })) })
|
|
354
340
|
for (const { line } of lockfiles) {
|
|
355
341
|
scan.lines.add(line)
|
|
356
342
|
if ('packages' in line) assessLater(scan, line.packages)
|
|
357
343
|
}
|
|
358
|
-
|
|
344
|
+
const refs = read.actions ? tellActions(scan, repo, read.actions, lockfiles) : []
|
|
345
|
+
for (const [at, { line }] of lockfiles.entries()) {
|
|
359
346
|
if (scan.signal.aborted) return
|
|
360
|
-
const
|
|
361
|
-
if (
|
|
347
|
+
const audit = await read.audits[at]
|
|
348
|
+
if (audit) scan.lines.add(advisoriesLine(line, audit))
|
|
362
349
|
}
|
|
363
350
|
if (scan.signal.aborted) return
|
|
351
|
+
const dependabot = read.dependabot && await rechecked(read.dependabot)
|
|
352
|
+
if (scan.signal.aborted) return
|
|
353
|
+
if (dependabot) scan.lines.add(dependabotLine(repo.repoId, dependabot, lockfiles))
|
|
364
354
|
scan.told.set(repo.repoId, { id: repo.repoId, name: repo.fullName, paths: repo.paths ?? null,
|
|
365
|
-
lockfiles: lockfiles.flatMap(({ sha, line }) => 'packages' in line ? [`${line.format}\0${sha}`] : []) })
|
|
355
|
+
lockfiles: lockfiles.flatMap(({ sha, line }) => 'packages' in line ? [`${line.format}\0${sha}`] : []), ...refs.length > 0 ? { actions: refs } : {} })
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
// A repository's Actions told, and the references they make to resolve.
|
|
359
|
+
function tellActions(scan: Scan, repo: RepositoryRiskRepo, read: WorkflowsRead, lockfiles: RepositoryRead['lockfiles']) {
|
|
360
|
+
const locked = lockfiles.flatMap(({ line }) => 'packages' in line ? [lockedNames(line.path, line.packages)] : [])
|
|
361
|
+
const { lines, refs } = actionsLines(repo.repoId, repo.fullName.split('/')[0]!, read, locked)
|
|
362
|
+
for (const line of lines) scan.lines.add(line)
|
|
363
|
+
scan.actions?.add(refs)
|
|
364
|
+
return refs
|
|
366
365
|
}
|
|
367
366
|
|
|
368
367
|
// Whether an earlier part told `repo` whole, by its name and a team's
|
|
@@ -372,18 +371,20 @@ async function readRepository(scan: Scan, repo: RepositoryRiskRepo) {
|
|
|
372
371
|
async function toldBefore(scan: Scan, repo: RepositoryRiskRepo) {
|
|
373
372
|
const told = scan.told.get(repo.repoId)
|
|
374
373
|
if (!told || told.name !== repo.fullName || JSON.stringify(told.paths) !== JSON.stringify(repo.paths ?? null)) return false
|
|
375
|
-
const reads: LockedPackage[][] = []
|
|
374
|
+
const reads: LockedPackage[][] = [], unknown = told.lockfiles.filter(key => !scan.caches.lockfiles.get(key))
|
|
375
|
+
const stored = unknown.length > 0 && scan.source.lockfiles ? await scan.source.lockfiles.readMany(unknown) : null
|
|
376
376
|
for (const key of told.lockfiles) {
|
|
377
|
-
const read = scan.caches.lockfiles.get(key) ??
|
|
377
|
+
const read = scan.caches.lockfiles.get(key) ?? stored?.get(key)?.value
|
|
378
378
|
if (!read || !('packages' in read)) return false
|
|
379
379
|
reads.push(read.packages)
|
|
380
380
|
}
|
|
381
381
|
for (const packages of reads) assessLater(scan, packages)
|
|
382
|
+
scan.actions?.add(told.actions ?? [])
|
|
382
383
|
return true
|
|
383
384
|
}
|
|
384
385
|
|
|
385
|
-
// Each repository,
|
|
386
|
-
// whole (ScanProgress).
|
|
386
|
+
// Each repository, READ_AHEAD read at a time, each told in turn, but those an
|
|
387
|
+
// earlier part told whole (ScanProgress), read of none.
|
|
387
388
|
async function readRepositories(scan: Scan) {
|
|
388
389
|
const repos = await scan.source.repos()
|
|
389
390
|
scan.lines.add({ type: 'repos', repos: repos.map(repo => ({ id: repo.repoId, fullName: repo.fullName })) })
|
|
@@ -393,9 +394,19 @@ async function readRepositories(scan: Scan) {
|
|
|
393
394
|
const kept = await scan.source.progress?.read()
|
|
394
395
|
const progress = kept && Date.now() - kept.at < SCAN_READ_KEPT_MS ? kept.value.repos : []
|
|
395
396
|
for (const told of progress) if (listed.has(told.id)) scan.told.set(told.id, told)
|
|
396
|
-
|
|
397
|
+
const reads: Promise<Audited | null>[] = []
|
|
398
|
+
const read = (at: number) => {
|
|
399
|
+
if (reads[at]) return reads[at]
|
|
400
|
+
const repo = repos[at]!
|
|
401
|
+
const started = reads[at] = toldBefore(scan, repo).then(async told => (told ? null : audited(scan, await readRepository(scan, repo))))
|
|
402
|
+
// One read ahead and no longer waited on, as the scan stopped, fails alone.
|
|
403
|
+
started.catch(() => {})
|
|
404
|
+
return started
|
|
405
|
+
}
|
|
406
|
+
for (const [at, repo] of repos.entries()) {
|
|
397
407
|
if (scan.signal.aborted) return
|
|
398
|
-
|
|
408
|
+
for (let ahead = at; ahead < Math.min(at + READ_AHEAD, repos.length); ahead++) void read(ahead)
|
|
409
|
+
await tellRepository(scan, repo, await read(at))
|
|
399
410
|
}
|
|
400
411
|
}
|
|
401
412
|
|
|
@@ -431,7 +442,10 @@ export async function* repositoryRiskLines<R extends RepositoryRiskRepo>(source:
|
|
|
431
442
|
// none waited for: the work under way, such as a request to GitHub, which
|
|
432
443
|
// isn't given up on at once, outlasting it.
|
|
433
444
|
const lines = queue<RepositoryRiskLine>(AbortSignal.any([signal, part.signal]), { drain: true })
|
|
434
|
-
const
|
|
445
|
+
const actions = source.actions ? resolvingActions({ source: source.actions, signal: running, tell: line => { lines.add(line) }, memory: caches.actions,
|
|
446
|
+
answers: caches.actionAnswers, store: source.actionLines }) : null
|
|
447
|
+
const scan: Scan = { source, caches, signal: running, lines, lookups: queue(running), seen: new Map(), dev: new Map(), stale: [], told: new Map(), actions,
|
|
448
|
+
audits: inTurns(AUDITS_AT_ONCE) }
|
|
435
449
|
const failed: unknown[] = []
|
|
436
450
|
void (async () => {
|
|
437
451
|
try {
|
|
@@ -446,6 +460,7 @@ export async function* repositoryRiskLines<R extends RepositoryRiskRepo>(source:
|
|
|
446
460
|
for (const left of leftOver(scan)) {
|
|
447
461
|
if (!running.aborted && left.length > 0) await assessing(scan, assess => { assess(left); return Promise.resolve() })
|
|
448
462
|
}
|
|
463
|
+
await actions?.close()
|
|
449
464
|
if (!running.aborted) scan.lines.add({ type: 'done' })
|
|
450
465
|
} catch (error) {
|
|
451
466
|
failed.push(error)
|