@preventive/triage 1.0.0-alpha.44 → 1.0.0-alpha.45
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/common/bundle-sources.js +20 -1
- package/common/default-scan-models.ts +1 -1
- package/common/dependency-reviews.d.ts +0 -5
- package/common/dependency-reviews.js +0 -17
- package/common/managed/agent-tasks.ts +292 -0
- package/common/managed/github-agents.ts +92 -0
- package/common/managed/github-logins.ts +49 -0
- package/common/managed/roles.ts +7 -0
- package/common/managed/routes.js +88 -7
- package/common/managed/team-app.d.ts +5 -2
- package/common/managed/team-app.js +2 -2
- package/common/server-info.ts +3 -0
- package/common/workspace-app.js +6 -3
- package/out/brotli-fallback.js +3 -3
- package/out/chunk-2XUEZHIP.js +18 -18
- package/out/chunk-2ZDZWLFY.js +1 -0
- package/out/chunk-3PS4KWD2.js +10 -0
- package/out/chunk-5DHL5NG3.js +2 -0
- package/out/chunk-6HNXNIQY.js +52 -0
- package/out/chunk-7C7DWL7I.js +1 -0
- package/out/chunk-AFLZH7WG.js +1 -0
- package/out/chunk-BC2UCLFJ.js +1 -0
- package/out/chunk-C4Z6ZBXV.js +6 -6
- package/out/chunk-GDPCLSOS.js +2 -0
- package/out/chunk-GO7FT4OD.js +1 -0
- package/out/chunk-HMAJDVXN.js +1 -0
- package/out/chunk-I2MDGSQX.js +3 -0
- package/out/chunk-IYPY47MC.js +1 -0
- package/out/chunk-IZ7POWI3.js +18 -0
- package/out/chunk-JCIIBLKR.js +1 -1
- package/out/chunk-NPW52BM2.js +17 -0
- package/out/chunk-NRPC74AM.js +1 -1
- package/out/chunk-NV7ES4EW.js +1 -0
- package/out/chunk-OKRLLE5I.js +17 -0
- package/out/chunk-OOJM4CTU.js +1 -1
- package/out/chunk-PNHBZGDD.js +1 -1
- package/out/chunk-PTGDYLNS.js +36 -0
- package/out/chunk-PXK7EU4V.js +1 -0
- package/out/chunk-Q6MVYGYV.js +3 -0
- package/out/chunk-QB4CBOX2.js +1 -0
- package/out/chunk-QRD6GIG5.js +229 -0
- package/out/chunk-R3NZFXVR.js +1 -0
- package/out/chunk-R5O6FAW4.js +1 -1
- package/out/chunk-SXV2T5C4.js +80 -0
- package/out/chunk-TUII34K2.js +1 -1
- package/out/chunk-VFWMQ3LG.js +2 -0
- package/out/chunk-VH4XMZIW.js +2 -0
- package/out/chunk-WFYLMF6Z.js +10 -0
- package/out/chunk-WI7Z5A2T.js +6 -0
- package/out/chunk-WQ74ZBZF.js +7 -0
- package/out/chunk-XVBEOGOS.js +3 -0
- package/out/client-managed-import.js +3 -3
- package/out/client-managed.js +31 -35
- package/out/client-sync.js +11 -11
- package/out/compare.js +35 -0
- package/out/dialogs.js +51 -0
- package/out/github-issue-authorized.html +1 -1
- package/out/github-issue-authorized.js +1 -1
- package/out/graph.js +17 -17
- package/out/index.html +30 -34
- package/out/prism.js +14 -14
- package/out/scan.js +2 -0
- package/out/terminal.js +294 -293
- package/out/view.css +1 -1
- package/out/view.js +16 -107
- package/package.json +51 -6
- package/risk/actions/findings.js +156 -0
- package/risk/actions/resolve.js +219 -0
- package/risk/actions/scripts/commands.js +182 -0
- package/risk/actions/scripts/grade.js +113 -0
- package/risk/actions/scripts/packages.js +123 -0
- package/risk/actions/scripts/shell.js +165 -0
- package/risk/actions/uses.js +76 -0
- package/risk/actions/workflow.js +110 -0
- package/risk/files/hosts.js +6 -6
- package/risk/files/readability.js +23 -17
- package/risk/files/runs-code.js +18 -6
- package/risk/files/script.js +7 -3
- package/risk/files/stamps.js +8 -7
- package/risk/index.d.ts +68 -1
- package/risk/index.js +6 -2
- package/risk/npm/assess.js +1 -1
- package/risk/npm/compare.js +34 -10
- package/risk/npm/files.js +413 -2
- package/risk/npm/tarball.js +56 -73
- package/server-managed/agent-merges.ts +198 -0
- package/server-managed/app-agents.ts +30 -0
- package/server-managed/avatar-store.ts +1 -0
- package/server-managed/commit-coauthors.ts +83 -0
- package/server-managed/commit-lockfiles.ts +62 -0
- package/server-managed/db-methods.ts +29 -10
- package/server-managed/db-neon.ts +30 -2
- package/server-managed/db-schema.ts +9 -2
- package/server-managed/db.ts +7 -0
- package/server-managed/dependabot-access.ts +60 -0
- package/server-managed/dependabot-alerts.ts +185 -0
- package/server-managed/dependency-review-comments.ts +97 -0
- package/server-managed/dependency-reviews.ts +65 -15
- package/server-managed/github-accounts.ts +33 -0
- package/server-managed/github-actions.ts +71 -0
- package/server-managed/github-app.ts +23 -9
- package/server-managed/github-errors.ts +2 -0
- package/server-managed/github-link-oauth.ts +69 -0
- package/server-managed/github-links.ts +204 -0
- package/server-managed/github-oauth.ts +5 -2
- package/server-managed/github-users.ts +30 -0
- package/server-managed/http.ts +1122 -52
- package/server-managed/index.ts +6 -2
- package/server-managed/locked-packages.ts +37 -16
- package/server-managed/lockfile-scope.ts +49 -0
- package/server-managed/npm-delta-worker.js +23 -0
- package/server-managed/npm-delta.ts +173 -0
- package/server-managed/npm-insights.ts +20 -11
- package/server-managed/npm-kept.ts +51 -4
- package/server-managed/npm-loads.ts +20 -13
- package/server-managed/npm-packages.ts +24 -64
- package/server-managed/pull-comment-reviews.ts +161 -0
- package/server-managed/pull-dependencies.ts +280 -0
- package/server-managed/pull-dependency-reviews.ts +318 -0
- package/server-managed/pull-reviews.ts +205 -0
- package/server-managed/release-commits.ts +84 -0
- package/server-managed/repository-actions.ts +209 -0
- package/server-managed/repository-agents.ts +220 -0
- package/server-managed/repository-browser.ts +54 -13
- package/server-managed/repository-changes.ts +284 -0
- package/server-managed/repository-code.ts +52 -0
- package/server-managed/repository-dependency-reviews.ts +135 -0
- package/server-managed/repository-history.ts +445 -0
- package/server-managed/repository-issues.ts +181 -0
- package/server-managed/repository-pulse.ts +432 -0
- package/server-managed/repository-risk-read.ts +165 -0
- package/server-managed/repository-risk.ts +113 -105
- package/server-managed/reviewed-dependencies.ts +149 -0
- package/server-managed/session.ts +9 -5
- package/server-managed/sql.ts +1 -1
- package/server-managed/storage-stores.ts +5 -0
- package/server-managed/team-app.ts +19 -3
- package/server-managed/workspace-shares.ts +4 -1
- package/server.ts +11 -0
- package/out/chunk-2ABLKCOY.js +0 -46
- package/out/chunk-2KBF5TZ7.js +0 -2
- package/out/chunk-35VE6CDH.js +0 -2
- package/out/chunk-3BZ7JERB.js +0 -1
- package/out/chunk-4XCDFGDP.js +0 -10
- package/out/chunk-5LSXOLUS.js +0 -1
- package/out/chunk-5RCM2J63.js +0 -1
- package/out/chunk-7B6HB4D7.js +0 -1
- package/out/chunk-AJ4GTATZ.js +0 -3
- package/out/chunk-CH4UN6WK.js +0 -225
- package/out/chunk-CMQA4DM6.js +0 -1
- package/out/chunk-CVVJAXVY.js +0 -2
- package/out/chunk-DIJJNTRH.js +0 -10
- package/out/chunk-EDKLBQQN.js +0 -17
- package/out/chunk-F47DXCEV.js +0 -33
- package/out/chunk-FK3DZEKL.js +0 -1
- package/out/chunk-G6AJWENO.js +0 -1
- package/out/chunk-GJ4IUCNS.js +0 -1
- package/out/chunk-GNXOKIGP.js +0 -80
- package/out/chunk-GZ4MS3TE.js +0 -13
- package/out/chunk-GZWM7UQM.js +0 -2
- package/out/chunk-IU3CCINR.js +0 -3
- package/out/chunk-K3X5EJBQ.js +0 -2
- package/out/chunk-KEX2ATEW.js +0 -2
- package/out/chunk-KYCNJVB7.js +0 -1
- package/out/chunk-LPCTJYNA.js +0 -7
- package/out/chunk-O4RJY5TP.js +0 -2
- package/out/chunk-OD4LPN3H.js +0 -1
- package/out/chunk-QWDWTBM7.js +0 -1
- package/out/chunk-QYWKC557.js +0 -1
- package/out/chunk-R75XOKUP.js +0 -1
- package/out/chunk-SWTLADW5.js +0 -1
- package/out/chunk-T7OKH4VW.js +0 -1
- package/out/chunk-TZ5KB2Y5.js +0 -10
- package/out/chunk-UNTRGE26.js +0 -2
- package/out/chunk-UTR2LVME.js +0 -2
- package/out/chunk-UYAXVRVL.js +0 -1
- package/out/chunk-V3ME6FI5.js +0 -2
- package/out/chunk-V5O7M2YT.js +0 -1
- package/out/chunk-VIHGO77Y.js +0 -1
- package/out/chunk-W4J3MGLB.js +0 -1
- package/out/chunk-WGG7KQUH.js +0 -1
- package/out/chunk-XYFXXB2G.js +0 -10
- package/out/chunk-ZESLQCTF.js +0 -6
|
@@ -0,0 +1,284 @@
|
|
|
1
|
+
// The files each commit changes, as the code viewer's Replay plays them
|
|
2
|
+
// (GET /api/admin/repositories/changes), as `git log --name-status` lists
|
|
3
|
+
// them: each file's path and whether the commit added (`A`), modified (`M`),
|
|
4
|
+
// removed (`D`) or renamed (`R`, with the path it had) it, as GitHub's REST
|
|
5
|
+
// API lists a commit's files (GET /repos/:owner/:repo/commits/:sha), a
|
|
6
|
+
// page of FILES_PAGE at a time, as many as MAX_FILES, `more` where it had
|
|
7
|
+
// more. What a commit changes never changes, so each is kept by the commit
|
|
8
|
+
// (`kept`), whole, read of GitHub once; what a reader is answered, only the
|
|
9
|
+
// files within the directory they asked of (changesWithin). And the files a
|
|
10
|
+
// commit holds, which Replay starts from (GET /api/admin/repositories/tree),
|
|
11
|
+
// as GitHub lists its tree at once (repositoryTree), kept as well.
|
|
12
|
+
|
|
13
|
+
import { GithubApiError, githubJson } from './github-app.ts'
|
|
14
|
+
|
|
15
|
+
// Commits an answer tells at most, read of GitHub as many at once as
|
|
16
|
+
// CHANGES_PARALLEL.
|
|
17
|
+
export const CHANGES_ASKED = 50
|
|
18
|
+
const CHANGES_PARALLEL = 6
|
|
19
|
+
// A commit's files read, as many as fill MAX_FILES, FILES_PAGE a page (the
|
|
20
|
+
// most GitHub lists a page; it lists fewer, unasked).
|
|
21
|
+
export const MAX_FILES = 300
|
|
22
|
+
export const FILES_PAGE = 100
|
|
23
|
+
const MAX_PATH = 4096
|
|
24
|
+
const SHA = /^[a-f\d]{40}$/u
|
|
25
|
+
|
|
26
|
+
export type ChangeStatus = 'A' | 'M' | 'D' | 'R'
|
|
27
|
+
// A file a commit changes: its path, its status, and a rename's path before.
|
|
28
|
+
export type ChangedFile = [path: string, status: Exclude<ChangeStatus, 'R'>] | [path: string, status: 'R', from: string]
|
|
29
|
+
export interface CommitChanges { files: ChangedFile[]; more: boolean }
|
|
30
|
+
|
|
31
|
+
// What each commit changes kept, by the commit.
|
|
32
|
+
export interface ChangesStore {
|
|
33
|
+
read(shas: string[]): Promise<Map<string, CommitChanges>>
|
|
34
|
+
write(sha: string, changes: CommitChanges): Promise<void>
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
const pathOf = (value: unknown) => typeof value === 'string' && value !== '' && value.length <= MAX_PATH && !/\p{Cc}/u.test(value) ? value : null
|
|
38
|
+
|
|
39
|
+
// A file's status as git names it: a copy is a file added; one GitHub
|
|
40
|
+
// lists unchanged (its mode alone changed, say) changes none.
|
|
41
|
+
const STATUSES: Record<string, ChangeStatus> = { added: 'A', copied: 'A', modified: 'M', changed: 'M', removed: 'D', renamed: 'R' }
|
|
42
|
+
|
|
43
|
+
type GithubFile = { filename?: unknown; status?: unknown; previous_filename?: unknown }
|
|
44
|
+
|
|
45
|
+
// A commit's files as GitHub lists them; a rename it names no path before
|
|
46
|
+
// for, a file added.
|
|
47
|
+
export function commitChanges(data: unknown): CommitChanges {
|
|
48
|
+
const listed = (data as { files?: unknown } | null)?.files
|
|
49
|
+
if (!Array.isArray(listed)) throw new GithubApiError(502, 'github-malformed')
|
|
50
|
+
const files = (listed as GithubFile[]).slice(0, MAX_FILES).flatMap((file): ChangedFile[] => {
|
|
51
|
+
const path = pathOf(file?.filename)
|
|
52
|
+
const status = typeof file?.status === 'string' ? STATUSES[file.status] : undefined
|
|
53
|
+
if (!path || !status) return []
|
|
54
|
+
const from = status === 'R' ? pathOf(file.previous_filename) : null
|
|
55
|
+
return [status === 'R' ? from ? [path, 'R', from] : [path, 'A'] : [path, status]]
|
|
56
|
+
})
|
|
57
|
+
return { files, more: listed.length >= MAX_FILES }
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
// What a commit changes within the directory `path` ('' the whole
|
|
61
|
+
// repository): its files there, a rename across its edge the file added
|
|
62
|
+
// where it came in, removed where it went out.
|
|
63
|
+
export function changesWithin({ files, more }: CommitChanges, path: string): CommitChanges {
|
|
64
|
+
if (path === '') return { files, more }
|
|
65
|
+
const within = (file: string) => file.startsWith(`${path}/`)
|
|
66
|
+
return { more, files: files.flatMap((file): ChangedFile[] => {
|
|
67
|
+
if (file[1] !== 'R') return within(file[0]) ? [file] : []
|
|
68
|
+
const [to, , from] = file
|
|
69
|
+
if (within(to)) return [within(from) ? file : [to, 'A']]
|
|
70
|
+
return within(from) ? [[from, 'D']] : []
|
|
71
|
+
}) }
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
// Runs `run` for each of `items`, as many at once as `parallel`.
|
|
75
|
+
async function eachOf<T>(items: T[], parallel: number, run: (item: T) => Promise<void>) {
|
|
76
|
+
let next = 0
|
|
77
|
+
await Promise.all(Array.from({ length: Math.min(parallel, items.length) }, async () => { while (next < items.length) await run(items[next++]!) }))
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
// A commit's files as GitHub lists them, read a page at a time while each
|
|
81
|
+
// is full, till MAX_FILES are.
|
|
82
|
+
async function commitFiles(url: string, token: string | null, fetchImpl: typeof fetch) {
|
|
83
|
+
const files: unknown[] = []
|
|
84
|
+
for (let page = 1; files.length < MAX_FILES; page++) {
|
|
85
|
+
const data = await githubJson(`${url}?per_page=${FILES_PAGE}&page=${page}`, token, fetchImpl) as { files?: unknown } | null
|
|
86
|
+
if (!Array.isArray(data?.files)) throw new GithubApiError(502, 'github-malformed')
|
|
87
|
+
files.push(...data.files)
|
|
88
|
+
if (data.files.length < FILES_PAGE) break
|
|
89
|
+
}
|
|
90
|
+
return { files }
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
// What each of `shas` changes, kept, else read of GitHub and then kept; null
|
|
94
|
+
// for one GitHub has no commit of (as one of another repository is).
|
|
95
|
+
export async function repositoryChanges(fullName: string, token: string | null, shas: string[], kept: ChangesStore,
|
|
96
|
+
fetchImpl: typeof fetch = globalThis.fetch): Promise<Map<string, CommitChanges | null>> {
|
|
97
|
+
const asked = [...new Set(shas)].filter(sha => SHA.test(sha))
|
|
98
|
+
const found = new Map<string, CommitChanges | null>(await kept.read(asked))
|
|
99
|
+
const repo = `https://api.github.com/repos/${fullName.split('/').map(encodeURIComponent).join('/')}`
|
|
100
|
+
await eachOf(asked.filter(sha => !found.has(sha)), CHANGES_PARALLEL, async sha => {
|
|
101
|
+
try {
|
|
102
|
+
const changes = commitChanges(await commitFiles(`${repo}/commits/${sha}`, token, fetchImpl))
|
|
103
|
+
found.set(sha, changes)
|
|
104
|
+
await kept.write(sha, changes)
|
|
105
|
+
} catch (err) {
|
|
106
|
+
// No such commit: GitHub's 404, or its 422 for an id naming none.
|
|
107
|
+
if (err instanceof GithubApiError && (err.message === 'github-not-found' || err.message === 'github-status-422')) found.set(sha, null)
|
|
108
|
+
else throw err
|
|
109
|
+
}
|
|
110
|
+
})
|
|
111
|
+
return new Map(asked.map(sha => [sha, found.get(sha) ?? null]))
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
// The files a commit (or a tree) holds, as many as MAX_TREE_FILES, each by its path and
|
|
115
|
+
// its bytes (`sizes`, in the same order), as
|
|
116
|
+
// GitHub lists its tree and every tree under it at once
|
|
117
|
+
// (GET /repos/:owner/:repo/git/trees/:tree?recursive=1); where GitHub cuts
|
|
118
|
+
// that listing short (past 100,000 entries or 7 MB, as a large repository's
|
|
119
|
+
// is), as GitHub advises: the tree's own entries listed alone, and each tree
|
|
120
|
+
// in it read so in turn, as many at once as TREES_PARALLEL (treeByTrees).
|
|
121
|
+
// Its blobs, not its directories nor submodules, sorted by path; past
|
|
122
|
+
// MAX_TREE_FILES those nearest its root of those read, `cut` where some are
|
|
123
|
+
// left out: more files than MAX_TREE_FILES, a tree's own entries cut short,
|
|
124
|
+
// or more trees to read than MAX_TREE_READS.
|
|
125
|
+
// Null for a commit GitHub has none of. Kept by the commit and the rules it
|
|
126
|
+
// is read by (TREES_READ), a tree kept by older ones (without sizes, the
|
|
127
|
+
// last) read again.
|
|
128
|
+
export const MAX_TREE_FILES = 75_000
|
|
129
|
+
export const TREES_READ = 3
|
|
130
|
+
const MAX_TREE_READS = 400
|
|
131
|
+
const TREES_PARALLEL = 6
|
|
132
|
+
export interface CommitTree { files: string[]; sizes: number[]; cut: boolean }
|
|
133
|
+
export interface TreeStore {
|
|
134
|
+
read(commit: string): Promise<CommitTree | null>
|
|
135
|
+
write(commit: string, tree: CommitTree): Promise<void>
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
type TreeEntry = { type?: unknown; path?: unknown; sha?: unknown; size?: unknown }
|
|
139
|
+
// A file read: its path and its bytes.
|
|
140
|
+
type Blob = [path: string, size: number]
|
|
141
|
+
type TreeListing = (tree: string, recursive: boolean) => Promise<{ entries: TreeEntry[]; truncated: boolean }>
|
|
142
|
+
|
|
143
|
+
export async function repositoryTree(fullName: string, token: string | null, commit: string, kept: TreeStore,
|
|
144
|
+
fetchImpl: typeof fetch = globalThis.fetch): Promise<CommitTree | null> {
|
|
145
|
+
if (!SHA.test(commit)) return null
|
|
146
|
+
const held = await kept.read(commit)
|
|
147
|
+
if (held) return held
|
|
148
|
+
const repo = `https://api.github.com/repos/${fullName.split('/').map(encodeURIComponent).join('/')}`
|
|
149
|
+
const listing: TreeListing = async (tree, recursive) => {
|
|
150
|
+
const data = await githubJson(`${repo}/git/trees/${tree}${recursive ? '?recursive=1' : ''}`, token, fetchImpl) as { tree?: unknown; truncated?: unknown }
|
|
151
|
+
if (!Array.isArray(data?.tree)) throw new GithubApiError(502, 'github-malformed')
|
|
152
|
+
return { entries: data.tree as TreeEntry[], truncated: data.truncated === true }
|
|
153
|
+
}
|
|
154
|
+
let whole: Awaited<ReturnType<TreeListing>>
|
|
155
|
+
try {
|
|
156
|
+
whole = await listing(commit, true)
|
|
157
|
+
} catch (err) {
|
|
158
|
+
if (err instanceof GithubApiError && (err.message === 'github-not-found' || err.message === 'github-status-422')) return null
|
|
159
|
+
throw err
|
|
160
|
+
}
|
|
161
|
+
const files: Blob[] = []
|
|
162
|
+
let cut = false
|
|
163
|
+
if (whole.truncated) cut = await treeByTrees(commit, listing, files)
|
|
164
|
+
else blobsOf(whole.entries, '', files)
|
|
165
|
+
const depth = (path: string) => path.split('/').length
|
|
166
|
+
const byPath = ([a]: Blob, [b]: Blob) => a < b ? -1 : a > b ? 1 : 0
|
|
167
|
+
const nearest = files.length > MAX_TREE_FILES ? files.toSorted((a, b) => depth(a[0]) - depth(b[0]) || byPath(a, b)).slice(0, MAX_TREE_FILES) : files
|
|
168
|
+
const sorted = nearest.toSorted(byPath)
|
|
169
|
+
const tree = { files: sorted.map(([path]) => path), sizes: sorted.map(([, size]) => size), cut: cut || files.length > MAX_TREE_FILES }
|
|
170
|
+
await kept.write(commit, tree)
|
|
171
|
+
return tree
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
// The blobs of a tree's entries, each its path under `prefix` and its bytes
|
|
175
|
+
// (none, as GitHub names none), onto `files`.
|
|
176
|
+
function blobsOf(entries: TreeEntry[], prefix: string, files: Blob[]) {
|
|
177
|
+
for (const item of entries) {
|
|
178
|
+
if (item?.type !== 'blob' || !pathOf(item.path)) continue
|
|
179
|
+
files.push([`${prefix}${item.path as string}`, Number.isSafeInteger(item.size) && (item.size as number) > 0 ? item.size as number : 0])
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
// The files under the commit's tree, `commit`, onto `files`, its trees read
|
|
184
|
+
// a round at a time, those nearest the root first: its own entries, then
|
|
185
|
+
// each tree in it whole, or, where that is cut short, alone in turn, and so
|
|
186
|
+
// on. Each round read whole, whatever answers first, but for trees no file
|
|
187
|
+
// of which could be kept: as deep as, or deeper than, the files nearest the
|
|
188
|
+
// root, as many as MAX_TREE_FILES, of those read (files of a tree are at
|
|
189
|
+
// least a level deeper than it is). As many trees read in all as
|
|
190
|
+
// MAX_TREE_READS, those of a round first in its order. Whether some were
|
|
191
|
+
// left out.
|
|
192
|
+
async function treeByTrees(commit: string, listing: TreeListing, files: Blob[]): Promise<boolean> {
|
|
193
|
+
let cut = false, reads = 1, round = [{ tree: commit, prefix: '', recursive: false }]
|
|
194
|
+
const atDepth: number[] = []
|
|
195
|
+
let counted = 0
|
|
196
|
+
while (round.length > 0) {
|
|
197
|
+
for (; counted < files.length; counted++) {
|
|
198
|
+
const depth = files[counted]![0].split('/').length
|
|
199
|
+
atDepth[depth] = (atDepth[depth] ?? 0) + 1
|
|
200
|
+
}
|
|
201
|
+
// The depth no file deeper than is kept.
|
|
202
|
+
let kept = Number.POSITIVE_INFINITY
|
|
203
|
+
for (let depth = 0, sum = 0; depth < atDepth.length; depth++) {
|
|
204
|
+
sum += atDepth[depth] ?? 0
|
|
205
|
+
if (sum >= MAX_TREE_FILES) {
|
|
206
|
+
kept = depth
|
|
207
|
+
break
|
|
208
|
+
}
|
|
209
|
+
}
|
|
210
|
+
const asked = round.filter(({ prefix }) => prefix.split('/').length - 1 < kept).slice(0, Math.max(0, MAX_TREE_READS - reads))
|
|
211
|
+
cut ||= asked.length < round.length
|
|
212
|
+
reads += asked.length
|
|
213
|
+
const listed: Awaited<ReturnType<TreeListing>>[] = []
|
|
214
|
+
await eachOf([...asked.keys()], TREES_PARALLEL, async at => { listed[at] = await listing(asked[at]!.tree, asked[at]!.recursive) })
|
|
215
|
+
const next: typeof round = []
|
|
216
|
+
for (const [at, { tree, prefix, recursive }] of asked.entries()) {
|
|
217
|
+
const { entries, truncated } = listed[at]!
|
|
218
|
+
if (truncated && recursive) {
|
|
219
|
+
next.push({ tree, prefix, recursive: false })
|
|
220
|
+
continue
|
|
221
|
+
}
|
|
222
|
+
cut ||= truncated
|
|
223
|
+
blobsOf(entries, prefix, files)
|
|
224
|
+
if (recursive) continue
|
|
225
|
+
for (const item of entries) {
|
|
226
|
+
const path = pathOf(item?.path)
|
|
227
|
+
if (item?.type === 'tree' && path && typeof item.sha === 'string' && SHA.test(item.sha)) next.push({ tree: item.sha, prefix: `${prefix}${path}/`, recursive: true })
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
round = next
|
|
231
|
+
}
|
|
232
|
+
return cut
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
// The files commit `commit` holds within the directory `path` ('' the whole
|
|
236
|
+
// repository), as repositoryTree reads them: its whole tree's within it; where
|
|
237
|
+
// that is cut short, some of the directory's perhaps among those left out,
|
|
238
|
+
// the directory's own tree read so (directoryTree), kept by its id, the files
|
|
239
|
+
// nearest it kept. Null for a commit GitHub has none of.
|
|
240
|
+
export async function directoryFiles(fullName: string, token: string | null, commit: string, path: string, kept: TreeStore,
|
|
241
|
+
fetchImpl: typeof fetch = globalThis.fetch): Promise<CommitTree | null> {
|
|
242
|
+
const whole = await repositoryTree(fullName, token, commit, kept, fetchImpl)
|
|
243
|
+
if (!whole || path === '' || !whole.cut) return whole && treeWithin(whole, path)
|
|
244
|
+
const id = await directoryTree(fullName, token, commit, path, fetchImpl)
|
|
245
|
+
const own = id ? await repositoryTree(fullName, token, id, kept, fetchImpl) : null
|
|
246
|
+
return own ? { ...own, files: own.files.map(file => `${path}/${file}`) } : treeWithin(whole, path)
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
// The id of the tree at `path` in commit `commit`, read a level at a time
|
|
250
|
+
// (GET /repos/:owner/:repo/git/trees/:tree, its own entries alone); null
|
|
251
|
+
// where it has none there, or GitHub has no such commit.
|
|
252
|
+
async function directoryTree(fullName: string, token: string | null, commit: string, path: string, fetchImpl: typeof fetch): Promise<string | null> {
|
|
253
|
+
const repo = `https://api.github.com/repos/${fullName.split('/').map(encodeURIComponent).join('/')}`
|
|
254
|
+
let tree = commit
|
|
255
|
+
for (const name of path.split('/')) {
|
|
256
|
+
let data: { tree?: unknown }
|
|
257
|
+
try {
|
|
258
|
+
data = await githubJson(`${repo}/git/trees/${tree}`, token, fetchImpl) as typeof data
|
|
259
|
+
} catch (err) {
|
|
260
|
+
if (err instanceof GithubApiError && (err.message === 'github-not-found' || err.message === 'github-status-422')) return null
|
|
261
|
+
throw err
|
|
262
|
+
}
|
|
263
|
+
if (!Array.isArray(data?.tree)) throw new GithubApiError(502, 'github-malformed')
|
|
264
|
+
const entry = (data.tree as TreeEntry[]).find(item => item?.path === name && item.type === 'tree')
|
|
265
|
+
if (typeof entry?.sha !== 'string' || !SHA.test(entry.sha)) return null
|
|
266
|
+
tree = entry.sha
|
|
267
|
+
}
|
|
268
|
+
return tree
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
// The files of a tree within the directory `path` ('' the whole repository).
|
|
272
|
+
export function treeWithin({ files, sizes, cut }: CommitTree, path: string): CommitTree {
|
|
273
|
+
if (path === '') return { files, sizes, cut }
|
|
274
|
+
const within = [...files.keys()].filter(at => files[at]!.startsWith(`${path}/`))
|
|
275
|
+
return { files: within.map(at => files[at]!), sizes: within.map(at => sizes[at]!), cut }
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
// The commits a request names (`shas`), each once, as many as
|
|
279
|
+
// CHANGES_ASKED; null for none, or for one that isn't a commit id.
|
|
280
|
+
export function changesAsked(shas: string | null): string[] | null {
|
|
281
|
+
if (!shas) return null
|
|
282
|
+
const asked = [...new Set(shas.split(','))]
|
|
283
|
+
return asked.length <= CHANGES_ASKED && asked.every(sha => SHA.test(sha)) ? asked : null
|
|
284
|
+
}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
// Reading a directory of a repository at a commit for the managed code viewer
|
|
2
|
+
// (GET /api/admin/repositories/code): its tree's tarball as GitHub archives
|
|
3
|
+
// it, verified against the tree's id and kept in upstream's disk cache as
|
|
4
|
+
// bundle builds keep theirs (getRepoTreeTarball), and its files as the npm
|
|
5
|
+
// viewer reads a package's (npm-packages.ts npmFileRows). Readers asking for
|
|
6
|
+
// the same directory meanwhile share its body, a load among the npm viewer's
|
|
7
|
+
// active ones (npm-loads.ts sharedLoad).
|
|
8
|
+
import { Buffer } from 'node:buffer'
|
|
9
|
+
import { encodeBrotli } from './brotli.ts'
|
|
10
|
+
import { MAX_NPM_PACKAGE_BYTES, MAX_NPM_PACKAGE_FILES, NpmPackageError, npmFileRows } from './npm-packages.ts'
|
|
11
|
+
import { type NpmLoad, readTarballFiles, sharedLoad } from './npm-loads.ts'
|
|
12
|
+
|
|
13
|
+
// As much as a package may hold: the viewer holds every file in memory.
|
|
14
|
+
export const MAX_REPOSITORY_CODE_FILES = MAX_NPM_PACKAGE_FILES
|
|
15
|
+
export const MAX_REPOSITORY_CODE_BYTES = MAX_NPM_PACKAGE_BYTES
|
|
16
|
+
|
|
17
|
+
// Whether a directory's files fit, by GitHub's listing of its tree and every
|
|
18
|
+
// tree under it (repository-browser.ts repositoryTreeFiles): asked before
|
|
19
|
+
// its tarball is, which may run to hundreds of MiB. One GitHub cut short
|
|
20
|
+
// (null) has too many.
|
|
21
|
+
export function repositoryCodeFits(files: { files: number; bytes: number } | null): boolean {
|
|
22
|
+
return files !== null && files.files <= MAX_REPOSITORY_CODE_FILES && files.bytes <= MAX_REPOSITORY_CODE_BYTES
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
// The directory read: the repository by its id and name, the commit, the
|
|
26
|
+
// directory (`''` for the root) and its tree's id.
|
|
27
|
+
export interface RepositoryCodeInput { repoId: number; repo: string; commit: string; path: string; tree: string }
|
|
28
|
+
|
|
29
|
+
// A package's refusals, as a directory's: past the bounds, or an archive
|
|
30
|
+
// @preventive/archive refuses, as it does one holding a symlink that climbs
|
|
31
|
+
// past the repository's root, or an absolute one.
|
|
32
|
+
const DIRECTORY_ERRORS: Record<string, [number, string]> = { 'package-too-large': [413, 'directory-too-large'], 'bad-tarball': [422, 'directory-unreadable'] }
|
|
33
|
+
const directoryError = (err: unknown) => {
|
|
34
|
+
const as = err instanceof NpmPackageError ? DIRECTORY_ERRORS[err.message] : undefined
|
|
35
|
+
return as ? new NpmPackageError(...as) : err
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
// The directory as the viewer reads it, as brotli-encoded JSON: `{ repoId,
|
|
39
|
+
// repo, commit, path, tree, files }`, its files as npmFileRows gives them,
|
|
40
|
+
// by their paths in the directory, from `tarball`, the tree's.
|
|
41
|
+
export function loadRepositoryCodeBody(input: RepositoryCodeInput, tarball: () => Promise<Uint8Array>): NpmLoad<Buffer> {
|
|
42
|
+
const { repoId, repo, commit, path, tree } = input
|
|
43
|
+
return sharedLoad(`repository ${repoId} ${repo} ${commit} ${path} ${tree}`, async () => {
|
|
44
|
+
let files
|
|
45
|
+
// A symlink may reach the repository's root, as a monorepo package's
|
|
46
|
+
// LICENSE links the root's, and no further: the directory's depth, less
|
|
47
|
+
// the level the archive's root already stands above its top directory.
|
|
48
|
+
try { files = npmFileRows(await readTarballFiles(await tarball(), { symlinksAbove: path.split('/').length - 1 })) }
|
|
49
|
+
catch (err) { throw directoryError(err) }
|
|
50
|
+
return encodeBrotli(Buffer.from(JSON.stringify({ repoId, repo, commit, path, tree, files })))
|
|
51
|
+
}, 'repository-busy')
|
|
52
|
+
}
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
// A connected repository's dependency review, as an admin sets it, and the
|
|
2
|
+
// reviews that pass its pull requests' dependencies (pull-dependency-reviews.ts).
|
|
3
|
+
// Enabled, each package version its open pull requests' lockfiles lock needs
|
|
4
|
+
// as many published reviews as `required`; those its lockfiles locked at
|
|
5
|
+
// `since`, a commit (null for none), none; of the lockfiles it holds to
|
|
6
|
+
// (`lockfiles`, lockfile-scope.ts), every one by default. It is in effect only where a team
|
|
7
|
+
// that reviews dependencies has the repository, which is where reviews that
|
|
8
|
+
// count are published, and not revoked: each reviewer's latest of a version
|
|
9
|
+
// tells where they stand on it, a reject holding it unreviewed whatever
|
|
10
|
+
// others passed it till they accept it again; and a reviewer standing on
|
|
11
|
+
// accepting it passes it by a review of theirs that is full (every file
|
|
12
|
+
// checked; a partial review counts for nothing yet) and accepting (Accept,
|
|
13
|
+
// or Accept with nits).
|
|
14
|
+
import type { ManagedSql } from './sql.ts'
|
|
15
|
+
import { type LockfileScope, scopeText, storedScope } from './lockfile-scope.ts'
|
|
16
|
+
import { reviewsIn } from './dependency-reviews.ts'
|
|
17
|
+
import { type ReviewVerdict, summarizeReview } from '../common/dependency-reviews.js'
|
|
18
|
+
|
|
19
|
+
export const REPOSITORY_DEPENDENCY_REVIEW_COLUMNS = [['dependency_review', 'INTEGER NOT NULL DEFAULT 0'],
|
|
20
|
+
['dependency_reviews_required', 'INTEGER NOT NULL DEFAULT 1'], ['dependency_review_since', 'TEXT'], ['dependency_review_lockfiles', 'TEXT']] as const
|
|
21
|
+
export const MAX_REQUIRED_REVIEWS = 10
|
|
22
|
+
export const SINCE_COMMIT = /^[a-f\d]{40}$/u
|
|
23
|
+
|
|
24
|
+
export interface RepoDependencyReview { enabled: boolean; required: number; since: string | null; lockfiles: LockfileScope }
|
|
25
|
+
// A review published of the version it is of: its package, version and
|
|
26
|
+
// npm's integrity of it, the patch it is of (none here, the app reviewing no
|
|
27
|
+
// patch yet), the version it was compared with (none for a whole package's),
|
|
28
|
+
// who wrote it (`reviewer`, their id, else their login then), its verdict,
|
|
29
|
+
// whether it checked every file (`full`), and when it was published.
|
|
30
|
+
// One made in a pull request's comments (`comment`) counts for it alone;
|
|
31
|
+
// one published here has its `id`.
|
|
32
|
+
export interface PublishedReview {
|
|
33
|
+
id?: string; package: string; version: string; integrity: string; patchHash?: string; basePackage: string | null; baseVersion: string | null
|
|
34
|
+
reviewer: string; status: ReviewVerdict; full: boolean; time: number; comment?: true
|
|
35
|
+
}
|
|
36
|
+
// A repository's review in effect for a reader: its settings, whether they
|
|
37
|
+
// publish reviews to one of the teams that count (`canReview`), and the
|
|
38
|
+
// published reviews of the packages asked.
|
|
39
|
+
export interface RepoReviewState extends Omit<RepoDependencyReview, 'enabled'> { canReview: boolean; reviews: PublishedReview[] }
|
|
40
|
+
|
|
41
|
+
export interface RepositoryDependencyReviewStore {
|
|
42
|
+
listRepoDependencyReviews(): Promise<Map<number, RepoDependencyReview>>
|
|
43
|
+
// The teams whose published reviews count for each repository (COUNTING_TEAM),
|
|
44
|
+
// by name.
|
|
45
|
+
listRepoReviewTeams(): Promise<Map<number, { id: string; name: string }[]>>
|
|
46
|
+
// Sets a repository's review, its commit and lockfiles too but where
|
|
47
|
+
// `since` or `lockfiles` is undefined, which keeps it; the setting then, or
|
|
48
|
+
// null for no repository.
|
|
49
|
+
setRepoDependencyReview(repoId: number, setting: Omit<RepoDependencyReview, 'since' | 'lockfiles'> & { since?: string | null; lockfiles?: LockfileScope }): Promise<RepoDependencyReview | null>
|
|
50
|
+
repoDependencyReviewState(repoId: number, user: string, packages: string[]): Promise<RepoReviewState | null>
|
|
51
|
+
// Which of `users` publish reviews to a team counting for repository
|
|
52
|
+
// `repoId`, whose reviews its pull requests' comments make count.
|
|
53
|
+
listRepoReviewers(repoId: number, users: string[]): Promise<Set<string>>
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
type SettingRow = { repoId: number; enabled: number; required: number; since: string | null; lockfiles: string | null }
|
|
57
|
+
const setting = (row: SettingRow): RepoDependencyReview => ({ enabled: row.enabled === 1, required: row.required, since: row.since, lockfiles: storedScope(row.lockfiles) })
|
|
58
|
+
// Teams whose published reviews count for repository `?`: that review
|
|
59
|
+
// dependencies and have it, not hidden.
|
|
60
|
+
const COUNTING_TEAM = `t.hidden = 0 AND t.dependency_review = 1 AND EXISTS (
|
|
61
|
+
SELECT 1 FROM managed_team_repo tr WHERE tr.team_id = t.id AND tr.repo_id = ?)`
|
|
62
|
+
const PACKAGES_AT_ONCE = 500
|
|
63
|
+
|
|
64
|
+
// The published and unrevoked reviews of `packages` in teams counting for
|
|
65
|
+
// repository `repoId`, each with whether it is full, a few hundred packages
|
|
66
|
+
// a query.
|
|
67
|
+
async function publishedReviews(db: ManagedSql, repoId: number, packages: string[]): Promise<PublishedReview[]> {
|
|
68
|
+
const found: PublishedReview[] = []
|
|
69
|
+
for (let at = 0; at < packages.length; at += PACKAGES_AT_ONCE) {
|
|
70
|
+
const names = packages.slice(at, at + PACKAGES_AT_ONCE)
|
|
71
|
+
const rows = await db.prepare(`SELECT r.id, r.user_id AS userId, r.user_login AS userLogin, r.package, r.version, r.integrity,
|
|
72
|
+
NULLIF(r.base_package, '') AS basePackage, NULLIF(r.base_version, '') AS baseVersion, r.files, r.status, r.published_at AS publishedAt
|
|
73
|
+
FROM managed_dependency_review r
|
|
74
|
+
WHERE r.ecosystem = 'npm' AND r.published_at IS NOT NULL AND r.revoked_at IS NULL AND r.status IS NOT NULL
|
|
75
|
+
AND r.package IN (${names.map(() => '?').join(',')})
|
|
76
|
+
AND EXISTS (SELECT 1 FROM managed_dependency_review_team rt JOIN managed_team t ON t.id = rt.team_id
|
|
77
|
+
WHERE rt.review_id = r.id AND ${COUNTING_TEAM})`).all(...names, repoId) as
|
|
78
|
+
{ id: string; userId: string | null; userLogin: string; package: string; version: string; integrity: string
|
|
79
|
+
basePackage: string | null; baseVersion: string | null; files: string | null; status: ReviewVerdict; publishedAt: number }[]
|
|
80
|
+
if (rows.length === 0) continue
|
|
81
|
+
const entries = await db.prepare(`SELECT review_id AS reviewId, file, status FROM managed_dependency_file_review
|
|
82
|
+
WHERE review_id IN (${rows.map(() => '?').join(',')})`).all(...rows.map(row => row.id)) as { reviewId: string; file: string; status: ReviewVerdict | null }[]
|
|
83
|
+
for (const row of rows) {
|
|
84
|
+
const snapshot = row.files === null ? null : JSON.parse(row.files) as string[] | { paths: string[]; nonTextFiles: string[] }
|
|
85
|
+
// A snapshot of paths alone, before non-text files were told, can't be full.
|
|
86
|
+
const files = Array.isArray(snapshot) ? snapshot : snapshot?.paths ?? null
|
|
87
|
+
const nonText = Array.isArray(snapshot) ? null : snapshot?.nonTextFiles ?? []
|
|
88
|
+
const full = summarizeReview(entries.filter(entry => entry.reviewId === row.id), files, nonText).coverage === 'full'
|
|
89
|
+
found.push({ id: row.id, package: row.package, version: row.version, integrity: row.integrity, basePackage: row.basePackage, baseVersion: row.baseVersion,
|
|
90
|
+
reviewer: row.userId ?? `login:${row.userLogin.toLowerCase()}`, status: row.status, full, time: Number(row.publishedAt) })
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
return found
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
export function repositoryDependencyReviewMethods(db: ManagedSql): RepositoryDependencyReviewStore {
|
|
97
|
+
const columns = `repo_id AS repoId, dependency_review AS enabled, dependency_reviews_required AS required, dependency_review_since AS since,
|
|
98
|
+
dependency_review_lockfiles AS lockfiles`
|
|
99
|
+
return {
|
|
100
|
+
async listRepoDependencyReviews() {
|
|
101
|
+
const rows = await db.prepare(`SELECT ${columns} FROM managed_selected_repo`).all() as SettingRow[]
|
|
102
|
+
return new Map(rows.map(row => [row.repoId, setting(row)]))
|
|
103
|
+
},
|
|
104
|
+
async listRepoReviewTeams() {
|
|
105
|
+
const rows = await db.prepare(`SELECT DISTINCT tr.repo_id AS repoId, t.id AS id, t.name AS name FROM managed_team_repo tr
|
|
106
|
+
JOIN managed_team t ON t.id = tr.team_id WHERE t.hidden = 0 AND t.dependency_review = 1 ORDER BY t.name, t.id`).all() as { repoId: number; id: string; name: string }[]
|
|
107
|
+
const teams = new Map<number, { id: string; name: string }[]>()
|
|
108
|
+
for (const { repoId, id, name } of rows) teams.set(repoId, [...teams.get(repoId) ?? [], { id, name }])
|
|
109
|
+
return teams
|
|
110
|
+
},
|
|
111
|
+
async setRepoDependencyReview(repoId, { enabled, required, since, lockfiles }) {
|
|
112
|
+
const set: [string, unknown][] = [['dependency_review', enabled ? 1 : 0], ['dependency_reviews_required', required],
|
|
113
|
+
...since === undefined ? [] : [['dependency_review_since', since] as [string, unknown]],
|
|
114
|
+
...lockfiles === undefined ? [] : [['dependency_review_lockfiles', scopeText(lockfiles)] as [string, unknown]]]
|
|
115
|
+
const { changes } = await db.prepare(`UPDATE managed_selected_repo SET ${set.map(([column]) => `${column} = ?`).join(', ')} WHERE repo_id = ?`)
|
|
116
|
+
.run(...set.map(([, value]) => value), repoId)
|
|
117
|
+
if (Number(changes) === 0) return null
|
|
118
|
+
return setting(await db.prepare(`SELECT ${columns} FROM managed_selected_repo WHERE repo_id = ?`).get(repoId) as SettingRow)
|
|
119
|
+
},
|
|
120
|
+
async listRepoReviewers(repoId, users) {
|
|
121
|
+
if (users.length === 0) return new Set()
|
|
122
|
+
const rows = await db.prepare(`SELECT DISTINCT u.id FROM managed_user u JOIN managed_team t ON t.hidden = 0
|
|
123
|
+
WHERE u.id IN (${users.map(() => '?').join(',')}) AND ${reviewsIn(true)} AND ${COUNTING_TEAM}`).all(...users, repoId) as { id: string }[]
|
|
124
|
+
return new Set(rows.map(row => row.id))
|
|
125
|
+
},
|
|
126
|
+
async repoDependencyReviewState(repoId, user, packages) {
|
|
127
|
+
const row = await db.prepare(`SELECT ${columns} FROM managed_selected_repo WHERE repo_id = ?`).get(repoId) as SettingRow | undefined
|
|
128
|
+
if (!row || row.enabled !== 1) return null
|
|
129
|
+
if (!await db.prepare(`SELECT 1 FROM managed_team t WHERE ${COUNTING_TEAM}`).get(repoId)) return null
|
|
130
|
+
const canReview = await db.prepare(`SELECT 1 FROM managed_team t JOIN managed_user u ON u.id = ?
|
|
131
|
+
WHERE ${reviewsIn(true)} AND ${COUNTING_TEAM}`).get(user, repoId) != null
|
|
132
|
+
return { required: row.required, since: row.since, lockfiles: storedScope(row.lockfiles), canReview, reviews: await publishedReviews(db, repoId, [...new Set(packages)]) }
|
|
133
|
+
},
|
|
134
|
+
}
|
|
135
|
+
}
|