@preventive/triage 1.0.0-alpha.47 → 1.0.0-alpha.48
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/client/dependency-paths.d.ts +2 -1
- package/client/dependency-paths.js +16 -4
- package/common/managed/agent-tasks.ts +12 -14
- package/common/managed/github-agents.ts +7 -6
- package/out/activity.js +4 -4
- package/out/{chunk-NOZJULWS.js → chunk-6LNQTKAV.js} +1 -1
- package/out/chunk-7VAWINQF.js +6 -0
- package/out/{chunk-5AHIQFFG.js → chunk-C2AVODGZ.js} +1 -1
- package/out/{chunk-I2MDGSQX.js → chunk-IO7JTZ6O.js} +1 -1
- package/out/{chunk-UENXT2HK.js → chunk-MOACFRJ4.js} +1 -1
- package/out/chunk-O6M574JI.js +1 -0
- package/out/{chunk-ZCS4UYHI.js → chunk-PBZPYIG7.js} +1 -1
- package/out/{chunk-S6U2J62U.js → chunk-PENGGKX7.js} +42 -41
- package/out/{chunk-6WF3KNTL.js → chunk-SZD5CW5R.js} +2 -2
- package/out/{chunk-SPEFPML6.js → chunk-TQJDYHVE.js} +1 -1
- package/out/{chunk-FH24VHSJ.js → chunk-WYLQWYLA.js} +1 -1
- package/out/{chunk-VL43BTSO.js → chunk-X6PJAK3T.js} +1 -1
- package/out/client-managed-import.js +1 -1
- package/out/client-managed.js +26 -26
- package/out/compare.js +13 -13
- package/out/dialogs.js +17 -17
- package/out/graph.js +1 -1
- package/out/index.html +7 -7
- package/out/scan.js +1 -1
- package/out/terminal.js +214 -214
- package/out/view.css +1 -1
- package/out/view.js +15 -15
- package/package.json +8 -5
- package/risk/npm/score.js +1 -1
- package/server-managed/agent-merges.ts +10 -8
- package/server-managed/app-agents.ts +6 -8
- package/server-managed/cache-storage.ts +3 -2
- package/server-managed/commit-coauthors.ts +7 -2
- package/server-managed/commit-lockfiles.ts +22 -21
- package/server-managed/db-methods.ts +52 -38
- package/server-managed/db-neon.ts +21 -5
- package/server-managed/db.ts +2 -0
- package/server-managed/dependabot-access.ts +1 -1
- package/server-managed/dependency-review-comments.ts +7 -4
- package/server-managed/dependency-reviews.ts +71 -33
- package/server-managed/github-accounts.ts +28 -1
- package/server-managed/github-app.ts +27 -23
- package/server-managed/github-graphql.ts +39 -0
- package/server-managed/github-issue-links.ts +8 -1
- package/server-managed/github-metadata.ts +2 -2
- package/server-managed/http-response.ts +28 -10
- package/server-managed/http.ts +677 -230
- package/server-managed/index.ts +2 -2
- package/server-managed/managed-issues.ts +3 -0
- package/server-managed/npm-delta.ts +16 -0
- package/server-managed/npm-kept.ts +32 -5
- package/server-managed/npm-loads.ts +21 -13
- package/server-managed/npm-manifest.ts +1 -1
- package/server-managed/npm-packages.ts +13 -4
- package/server-managed/object-storage.ts +4 -3
- package/server-managed/public-workspace.ts +3 -3
- package/server-managed/pull-dependencies.ts +84 -36
- package/server-managed/pull-dependency-reviews.ts +13 -10
- package/server-managed/rename-diff-worker.js +1 -1
- package/server-managed/repository-agents.ts +9 -6
- package/server-managed/repository-browser.ts +13 -3
- package/server-managed/repository-code.ts +153 -7
- package/server-managed/repository-dependency-reviews.ts +7 -0
- package/server-managed/repository-history.ts +85 -152
- package/server-managed/repository-issues.ts +27 -13
- package/server-managed/repository-pulse.ts +9 -3
- package/server-managed/repository-squashes.ts +145 -0
- package/server-managed/request-timing.ts +63 -0
- package/server-managed/reviewed-dependencies.ts +3 -2
- package/server-managed/session.ts +34 -12
- package/server-managed/sql.ts +16 -8
- package/server-managed/storage-encryption.ts +16 -8
- package/server-managed/storage-maintenance.ts +5 -2
- package/server-managed/storage-payload.ts +18 -0
- package/server-managed/storage-stores.ts +3 -2
- package/server-managed/team-feed.ts +16 -11
- package/server-managed/team-npm-scopes.ts +29 -0
- package/server-managed/team-reports.ts +68 -16
- package/out/chunk-6ANSHRFQ.js +0 -6
- package/out/chunk-ZGNMDGSB.js +0 -1
|
@@ -6,9 +6,11 @@
|
|
|
6
6
|
// review and its checks, which GraphQL alone gives, so it needs a token; an
|
|
7
7
|
// open one's, and its popup's, whether it changes a lockfile whose new
|
|
8
8
|
// packages are told (pull-dependencies.ts touchesLockfile), and its reviews
|
|
9
|
-
// (pull-reviews.ts), each reviewer's
|
|
10
|
-
// its
|
|
11
|
-
//
|
|
9
|
+
// (pull-reviews.ts), each reviewer's; and an open one's base's and head's
|
|
10
|
+
// commits, kept with its page but never told, by which what its lockfiles
|
|
11
|
+
// add is read (http.ts teamRepositoryDependencies). A nonpublic repository's
|
|
12
|
+
// are read with its GitHub App's installation, which reads pull requests only
|
|
13
|
+
// with their own permission (doc/code-viewer.md).
|
|
12
14
|
import { GithubApiError } from './github-app.ts'
|
|
13
15
|
import { touchesLockfile } from './pull-dependencies.ts'
|
|
14
16
|
import { ACTOR, type GraphAccount, type GraphqlError, actorOf, githubGraphql } from './repository-history.ts'
|
|
@@ -16,11 +18,12 @@ import { ITEM_THREADS, type ReviewNode, TILE_THREADS, pullReviews, reviewFields,
|
|
|
16
18
|
|
|
17
19
|
export const ISSUES_PAGE = 50
|
|
18
20
|
// What a kept page or item was read as: kept ones read before differ.
|
|
19
|
-
export const ISSUES_READ_VERSION =
|
|
21
|
+
export const ISSUES_READ_VERSION = 4
|
|
20
22
|
const MAX_LABELS = 20
|
|
21
23
|
const MAX_TITLE = 1000
|
|
22
24
|
const MAX_NAME = 256
|
|
23
25
|
const COLOR = /^[a-f\d]{6}$/iu
|
|
26
|
+
const SHA = /^[a-f\d]{40}$/u
|
|
24
27
|
// A page's cursor as GitHub gives it, to read on from: opaque, base64 or
|
|
25
28
|
// base64url as it is now (`…rOrK_u7Q==`), passed back to GitHub alone, so any
|
|
26
29
|
// printable one is, but none with spaces or control characters.
|
|
@@ -43,21 +46,22 @@ const connection = { issues: 'issues', pulls: 'pullRequests' }
|
|
|
43
46
|
const states = { issues: { open: '[OPEN]', closed: '[CLOSED]' }, pulls: { open: '[OPEN]', closed: '[CLOSED, MERGED]' } }
|
|
44
47
|
|
|
45
48
|
// A page of `kind`, those in `state`, after the page `after` ends, and how
|
|
46
|
-
// many of each state there are; open pull requests with their
|
|
49
|
+
// many of each state there are; open pull requests with their commits and
|
|
50
|
+
// their reviews.
|
|
47
51
|
const pageQuery = (kind: IssuesKind, state: IssuesState) => `query($owner: String!, $name: String!, $after: String) {
|
|
48
52
|
repository(owner: $owner, name: $name) {
|
|
49
53
|
open: ${connection[kind]}(states: ${states[kind].open}) { totalCount }
|
|
50
54
|
closed: ${connection[kind]}(states: ${states[kind].closed}) { totalCount }
|
|
51
55
|
page: ${connection[kind]}(states: ${states[kind][state]}, first: ${ISSUES_PAGE}, after: $after, orderBy: { field: UPDATED_AT, direction: DESC }) {
|
|
52
56
|
pageInfo { hasNextPage endCursor }
|
|
53
|
-
nodes { ${fields[kind]}${kind === 'pulls' && state === 'open' ? ` ${files} ${reviewFields(TILE_THREADS, false)}` : ''} }
|
|
57
|
+
nodes { ${fields[kind]}${kind === 'pulls' && state === 'open' ? ` baseRefOid headRefOid ${files} ${reviewFields(TILE_THREADS, false)}` : ''} }
|
|
54
58
|
}
|
|
55
59
|
}
|
|
56
60
|
}`
|
|
57
61
|
|
|
58
62
|
type Node = ReviewNode & { number?: unknown; title?: unknown; createdAt?: unknown; updatedAt?: unknown; closedAt?: unknown; mergedAt?: unknown; author?: GraphAccount
|
|
59
63
|
comments?: { totalCount?: unknown }; labels?: { totalCount?: unknown; nodes?: { name?: unknown; color?: unknown }[] }; state?: unknown; stateReason?: unknown
|
|
60
|
-
isDraft?: unknown; reviewDecision?: unknown; additions?: unknown; deletions?: unknown; headRefName?: unknown; baseRefName?: unknown
|
|
64
|
+
isDraft?: unknown; reviewDecision?: unknown; additions?: unknown; deletions?: unknown; headRefName?: unknown; baseRefName?: unknown; baseRefOid?: unknown
|
|
61
65
|
commits?: { nodes?: { commit?: { statusCheckRollup?: { state?: unknown } | null } | null }[] }
|
|
62
66
|
files?: { totalCount?: unknown; nodes?: ({ path?: unknown } | null)[] } | null }
|
|
63
67
|
type Page = { repository?: { open?: { totalCount?: unknown }; closed?: { totalCount?: unknown }
|
|
@@ -159,9 +163,17 @@ export async function repositoryItem(fullName: string, token: string, number: nu
|
|
|
159
163
|
mergedBy: merger.login, mergedById: merger.id, reviews: pullReviews(node, tile.author, true) }
|
|
160
164
|
}
|
|
161
165
|
|
|
166
|
+
// An open pull request's base's and head's commits, by its number, as GitHub
|
|
167
|
+
// has them now: those its REST API names `base.sha` and `head.sha`
|
|
168
|
+
// (pull-dependencies.ts pullCommits). None where either isn't a commit's id.
|
|
169
|
+
const commitsOf = (number: number, { baseRefOid: base, headRefOid: head }: Node) =>
|
|
170
|
+
typeof base === 'string' && SHA.test(base) && typeof head === 'string' && SHA.test(head) ? [{ number, base, head }] : []
|
|
171
|
+
|
|
162
172
|
// A page of the repository's issues or pull requests in `state`, after the
|
|
163
173
|
// page `after` ended (null for the first): its `items`, how many are open and
|
|
164
|
-
// closed (`counts`), and, where more follow, the cursor reading them
|
|
174
|
+
// closed (`counts`), and, where more follow, the cursor reading them
|
|
175
|
+
// (`after`); open pull requests' commits (`commits`, commitsOf) too, which
|
|
176
|
+
// the code viewer is never told (http.ts handleRepositoryIssues).
|
|
165
177
|
export async function repositoryIssues(fullName: string, token: string, kind: IssuesKind, state: IssuesState, after: string | null, fetchImpl: typeof fetch = globalThis.fetch) {
|
|
166
178
|
const [owner, name] = fullName.split('/') as [string, string]
|
|
167
179
|
const data = await githubGraphql(pageQuery(kind, state), { owner, name, after }, token, fetchImpl, partRefused) as Page
|
|
@@ -169,13 +181,15 @@ export async function repositoryIssues(fullName: string, token: string, kind: Is
|
|
|
169
181
|
if (!repository) throw new GithubApiError(404, 'github-not-found')
|
|
170
182
|
const info = repository.page?.pageInfo
|
|
171
183
|
const next = info?.hasNextPage === true && typeof info.endCursor === 'string' && ISSUES_CURSOR.test(info.endCursor) ? info.endCursor : null
|
|
184
|
+
const open = kind === 'pulls' && state === 'open'
|
|
185
|
+
const tiles = (repository.page?.nodes ?? []).flatMap(node => {
|
|
186
|
+
const tile = item(kind, node)
|
|
187
|
+
return tile && node ? [{ node, tile }] : []
|
|
188
|
+
})
|
|
172
189
|
return {
|
|
173
|
-
items: (
|
|
174
|
-
const tile = item(kind, node)
|
|
175
|
-
if (!tile) return []
|
|
176
|
-
return kind === 'pulls' && state === 'open' && node ? [{ ...tile, reviews: pullReviews(node, tile.author) }] : [tile]
|
|
177
|
-
}),
|
|
190
|
+
items: tiles.map(({ node, tile }) => open ? { ...tile, reviews: pullReviews(node, tile.author) } : tile),
|
|
178
191
|
counts: { open: count(repository.open?.totalCount), closed: count(repository.closed?.totalCount) },
|
|
179
192
|
more: next !== null, after: next,
|
|
193
|
+
...open ? { commits: tiles.flatMap(({ node, tile }) => commitsOf(tile.number, node)) } : {},
|
|
180
194
|
}
|
|
181
195
|
}
|
|
@@ -20,7 +20,7 @@ import { ACTOR, type GraphAccount, type HistoryCommit, type HistoryPages, accoun
|
|
|
20
20
|
export const PULSE_WEEKS = [1, 4, 13, 52]
|
|
21
21
|
// What a year counted from its commits holds, its version: kept years of
|
|
22
22
|
// another are counted again.
|
|
23
|
-
export const PULSE_YEAR_VERSION =
|
|
23
|
+
export const PULSE_YEAR_VERSION = 11
|
|
24
24
|
const WEEK_MS = 7 * 24 * 60 * 60_000
|
|
25
25
|
const DAY_MS = 24 * 60 * 60_000
|
|
26
26
|
const MAX_CONTRIBUTORS = 100
|
|
@@ -227,15 +227,21 @@ export async function yearCommits(fullName: string, token: string, head: string,
|
|
|
227
227
|
return { complete: true, commits: [...found.values()].filter(dated), passed: [...found.values()].filter(commit => !dated(commit) && within(commit)) }
|
|
228
228
|
}
|
|
229
229
|
|
|
230
|
+
// Whether an author is a GitHub App's bot: by its login (`<app>[bot]`, as a
|
|
231
|
+
// commit's no-reply address names it, github-accounts.ts commitAccount),
|
|
232
|
+
// else, for one GitHub names no account for, by the name it commits as.
|
|
233
|
+
const bot = (login: string | null | undefined, name: string | null | undefined) => /\[bot\]$/iu.test(login ?? name ?? '')
|
|
234
|
+
|
|
230
235
|
// Whether an agent made `commit` or co-authored it: its author one of
|
|
231
236
|
// GitHub's agents (common/managed/github-agents.ts githubAgent), or any of
|
|
232
237
|
// its co-authors an agent (commit-coauthors.ts coauthorsOf), as Commits
|
|
233
|
-
// marks them; or one an agent's pull request brought (`merges`,
|
|
238
|
+
// marks them, or any bot, whichever GitHub App's; or one an agent's pull request brought (`merges`,
|
|
234
239
|
// agent-merges.ts agentCommits); or one an account agents work as pushed to
|
|
235
240
|
// the branch directly (`direct`, github-agents.ts agentOperated).
|
|
236
241
|
const withAgent = (commit: HistoryCommit, merges: ReadonlySet<string>, direct: ReadonlySet<string>) => merges.has(commit.sha)
|
|
237
242
|
|| (direct.has(commit.sha) && agentOperated(commit.authorLogin, commit.authorId ?? null))
|
|
238
|
-
|| githubAgent(commit.authorLogin, commit.authorId ?? null, commit.authorName) !== null || (commit.
|
|
243
|
+
|| githubAgent(commit.authorLogin, commit.authorId ?? null, commit.authorName) !== null || bot(commit.authorLogin, commit.authorName)
|
|
244
|
+
|| (commit.coauthors ?? []).some(coauthor => Boolean(coauthor.agent) || bot(coauthor.login, coauthor.name))
|
|
239
245
|
|
|
240
246
|
// The year from `first` (pulseSince(52, …)) of `commits`, as GitHub's
|
|
241
247
|
// statistics count them, merges left out: a week a row, its commits each day
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
// The pull requests squashed into a repository's commits, as the code
|
|
2
|
+
// viewer's Commits tab draws them (repository-history.ts), each looked up
|
|
3
|
+
// with GraphQL once and kept by its commit.
|
|
4
|
+
import { linkNoreplyUsers } from './github-accounts.ts'
|
|
5
|
+
import { type GraphqlError, githubGraphql } from './github-graphql.ts'
|
|
6
|
+
import { COMMIT_FRAGMENT, type GraphCommit, type HistoryCommit, commitActors, graphCommit, inBatches, variablesOf } from './repository-history.ts'
|
|
7
|
+
|
|
8
|
+
const SHA = /^[a-f\d]{40}$/u
|
|
9
|
+
|
|
10
|
+
// A pull request squashed into a commit of the default branch's history
|
|
11
|
+
// (`merge`): its number, the commit its head was at when merged (`head`,
|
|
12
|
+
// GitHub's own of it, which a branch reused or pushed over since never
|
|
13
|
+
// moves), and its own commits, the latest MAX_SQUASHED of them, `more`
|
|
14
|
+
// where it had more. Drawn as the merge it was, its commits on a lane of
|
|
15
|
+
// their own (repository-history.js).
|
|
16
|
+
export interface SquashedPull { merge: string; number: number; head: string; commits: HistoryCommit[]; more: boolean }
|
|
17
|
+
// The pull requests squashed into commits, kept by each commit (a merged pull
|
|
18
|
+
// request's commits never change); null for a commit none was.
|
|
19
|
+
export interface Squashes {
|
|
20
|
+
read(shas: string[]): Promise<Map<string, SquashedPull | null>>
|
|
21
|
+
write(sha: string, pull: SquashedPull | null): Promise<void>
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
const MAX_SQUASHED = 100
|
|
25
|
+
// Looked up of an answer's commits not yet kept, at most, the newest first;
|
|
26
|
+
// in queries of as many commits, or pull requests (their commits' lines
|
|
27
|
+
// counted, as a history's are), as many at once.
|
|
28
|
+
export const MAX_SQUASH_LOOKUPS = 500
|
|
29
|
+
const SQUASH_COMMITS = 100
|
|
30
|
+
const SQUASH_PULLS = 5
|
|
31
|
+
const SQUASH_PARALLEL = 4
|
|
32
|
+
|
|
33
|
+
// The merged pull request that brought each commit in, `c<n>` the `n`th's.
|
|
34
|
+
const mergedPullsQuery = (count: number) => `query($owner: String!, $name: String!, ${variablesOf('o', count, 'GitObjectID!')}) {
|
|
35
|
+
repository(owner: $owner, name: $name) {
|
|
36
|
+
${Array.from({ length: count }, (_, at) => `c${at}: object(oid: $o${at}) { ... on Commit { associatedPullRequests(first: 1) { nodes { number mergeCommit { oid } } } } }`).join('\n ')}
|
|
37
|
+
}
|
|
38
|
+
}`
|
|
39
|
+
// Each pull request's head when merged and its latest commits, `p<n>` the `n`th's.
|
|
40
|
+
const squashedQuery = (count: number) => `query($owner: String!, $name: String!, ${variablesOf('n', count, 'Int!')}) {
|
|
41
|
+
repository(owner: $owner, name: $name) {
|
|
42
|
+
${Array.from({ length: count }, (_, at) => `p${at}: pullRequest(number: $n${at}) { headRefOid commits(last: ${MAX_SQUASHED}) { totalCount nodes { commit { ...historyCommit } } } }`).join('\n ')}
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
${COMMIT_FRAGMENT}`
|
|
46
|
+
|
|
47
|
+
type MergedPulls = { repository?: Record<string, { associatedPullRequests?: { nodes?: ({ number?: unknown; mergeCommit?: { oid?: unknown } | null } | null)[] } } | null> | null }
|
|
48
|
+
type SquashedPage = { repository?: Record<string, { headRefOid?: unknown; commits?: { totalCount?: unknown; nodes?: ({ commit?: GraphCommit | null } | null)[] } } | null> | null }
|
|
49
|
+
|
|
50
|
+
// A pull request or commit GitHub has none of, as one deleted: its field null.
|
|
51
|
+
const notFound = (error: GraphqlError) => error?.type === 'NOT_FOUND'
|
|
52
|
+
|
|
53
|
+
// Each repository's lookup of squashed pull requests going on, by its full
|
|
54
|
+
// name: one at a time, those asking meanwhile waiting on it, and their
|
|
55
|
+
// commits it doesn't look up looked up once it is done.
|
|
56
|
+
const lookingUp = new Map<string, Promise<void>>()
|
|
57
|
+
|
|
58
|
+
// Whether `promise` settles within `ms`, waited on that long at most.
|
|
59
|
+
const within = (promise: Promise<unknown>, ms: number) => ms === Number.POSITIVE_INFINITY ? promise
|
|
60
|
+
: new Promise<void>(resolve => {
|
|
61
|
+
const timer = setTimeout(resolve, ms)
|
|
62
|
+
void promise.finally(() => { clearTimeout(timer); resolve() })
|
|
63
|
+
})
|
|
64
|
+
|
|
65
|
+
// The pull requests squashed into `commits`, a merged pull request's merge
|
|
66
|
+
// commit of one parent, not one rebased (its commits copied onto the branch,
|
|
67
|
+
// told by their subjects), kept by each commit (`squashes`), those not yet
|
|
68
|
+
// kept looked up with GraphQL, as many as MAX_SQUASH_LOOKUPS an answer, each
|
|
69
|
+
// kept as it is found (lookUpSquashes). Answered with those kept once the
|
|
70
|
+
// repository's lookup is done, or `waitMs` on, that lookup going on behind:
|
|
71
|
+
// `behind`, one this started, for the caller to keep alive, and `pending`
|
|
72
|
+
// while one does. Another's lookup done within it, but not of all of these,
|
|
73
|
+
// those it left are looked up as this one's, within what is left of it.
|
|
74
|
+
export async function squashedPulls(fullName: string, token: string, commits: HistoryCommit[], squashes: Squashes,
|
|
75
|
+
waitMs = Number.POSITIVE_INFINITY, fetchImpl: typeof fetch = globalThis.fetch): Promise<{ pulls: SquashedPull[]; pending: boolean; behind: Promise<void> | null }> {
|
|
76
|
+
const single = commits.filter(commit => commit.parents.length === 1)
|
|
77
|
+
const kept = await squashes.read(single.map(commit => commit.sha))
|
|
78
|
+
let unknown = single.filter(commit => !kept.has(commit.sha))
|
|
79
|
+
if (unknown.length === 0) return { pulls: single.flatMap(commit => kept.get(commit.sha) ?? []), pending: false, behind: null }
|
|
80
|
+
const listed = new Map(commits.map(commit => [commit.sha, commit]))
|
|
81
|
+
const deadline = Date.now() + waitMs
|
|
82
|
+
const found = new Map<string, SquashedPull | null>()
|
|
83
|
+
// Waited on, as long as is left, and those of `unknown` it kept found.
|
|
84
|
+
const settle = async (looking: Promise<void>) => {
|
|
85
|
+
await within(looking, Math.max(0, deadline - Date.now()))
|
|
86
|
+
for (const [sha, pull] of await squashes.read(unknown.map(commit => commit.sha))) found.set(sha, pull)
|
|
87
|
+
unknown = unknown.filter(commit => !found.has(commit.sha))
|
|
88
|
+
}
|
|
89
|
+
const another = lookingUp.get(fullName)
|
|
90
|
+
if (another) await settle(another)
|
|
91
|
+
let behind: Promise<void> | null = null
|
|
92
|
+
if (!another || (unknown.length > 0 && !lookingUp.has(fullName))) {
|
|
93
|
+
const lookup = lookUpSquashes(fullName, token, unknown.slice(0, MAX_SQUASH_LOOKUPS), listed, squashes, fetchImpl)
|
|
94
|
+
const done: Promise<void> = lookup.catch(() => {}).finally(() => { if (lookingUp.get(fullName) === done) lookingUp.delete(fullName) })
|
|
95
|
+
lookingUp.set(fullName, done)
|
|
96
|
+
behind = lookup
|
|
97
|
+
await settle(done)
|
|
98
|
+
}
|
|
99
|
+
return { pulls: single.flatMap(commit => kept.get(commit.sha) ?? found.get(commit.sha) ?? []), pending: lookingUp.has(fullName), behind }
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
// The pull requests squashed into `unknown`, of the commits `listed`, looked
|
|
103
|
+
// up, each kept as it is found (squashedPulls).
|
|
104
|
+
async function lookUpSquashes(fullName: string, token: string, unknown: HistoryCommit[], listed: Map<string, HistoryCommit>, squashes: Squashes,
|
|
105
|
+
fetchImpl: typeof fetch) {
|
|
106
|
+
const [owner, name] = fullName.split('/') as [string, string]
|
|
107
|
+
const merged: { commit: HistoryCommit; number: number }[] = []
|
|
108
|
+
await inBatches(unknown, SQUASH_COMMITS, SQUASH_PARALLEL, async batch => {
|
|
109
|
+
const data = await githubGraphql(mergedPullsQuery(batch.length), { owner, name, ...Object.fromEntries(batch.map((commit, at) => [`o${at}`, commit.sha])) },
|
|
110
|
+
token, fetchImpl, notFound) as MergedPulls
|
|
111
|
+
for (const [at, commit] of batch.entries()) {
|
|
112
|
+
const node = data?.repository?.[`c${at}`]
|
|
113
|
+
if (!node) continue
|
|
114
|
+
const pull = node.associatedPullRequests?.nodes?.[0]
|
|
115
|
+
const number = pull?.mergeCommit?.oid === commit.sha && Number.isSafeInteger(pull.number) && (pull.number as number) > 0 ? pull.number as number : null
|
|
116
|
+
if (number === null) await squashes.write(commit.sha, null)
|
|
117
|
+
else merged.push({ commit, number })
|
|
118
|
+
}
|
|
119
|
+
})
|
|
120
|
+
await inBatches(merged, SQUASH_PULLS, SQUASH_PARALLEL, async batch => {
|
|
121
|
+
const data = await githubGraphql(squashedQuery(batch.length), { owner, name, ...Object.fromEntries(batch.map(({ number }, at) => [`n${at}`, number])) },
|
|
122
|
+
token, fetchImpl, notFound) as SquashedPage
|
|
123
|
+
await linkNoreplyUsers(Object.values(data?.repository ?? {}).flatMap(pull => (pull?.commits?.nodes ?? []).flatMap(item => commitActors(item?.commit))), token, fetchImpl)
|
|
124
|
+
for (const [at, { commit, number }] of batch.entries()) {
|
|
125
|
+
const node = data?.repository?.[`p${at}`]
|
|
126
|
+
if (!node) continue
|
|
127
|
+
const own = (node.commits?.nodes ?? []).flatMap(item => graphCommit(item?.commit) ?? [])
|
|
128
|
+
const head = typeof node.headRefOid === 'string' && SHA.test(node.headRefOid) ? node.headRefOid : own.at(-1)?.sha
|
|
129
|
+
// Fast-forwarded, its last commit the merge's own; or rebased, its
|
|
130
|
+
// commits, more than one, copied onto the branch: the merge and those
|
|
131
|
+
// below it on their first parents, by their subjects, its commits'
|
|
132
|
+
// (one squashed is told by its commit's title too, by default, and
|
|
133
|
+
// drawn as it was either way).
|
|
134
|
+
const below = [commit]
|
|
135
|
+
for (let next = listed.get(commit.parents[0] ?? ''); next && below.length < own.length; next = listed.get(next.parents[0] ?? '')) below.push(next)
|
|
136
|
+
const copies = own.toReversed().slice(0, below.length).every((item, step) => below[step]!.subject === item.subject)
|
|
137
|
+
// Its commits' copies as far as those listed go, but not all of them
|
|
138
|
+
// listed: told, and kept, once they are.
|
|
139
|
+
if (own.length > 1 && copies && below.length < own.length) continue
|
|
140
|
+
const rebased = own.length > 1 && copies
|
|
141
|
+
const squashed = head !== undefined && head !== commit.sha && own.length > 0 && !own.some(item => item.sha === commit.sha) && !rebased
|
|
142
|
+
await squashes.write(commit.sha, squashed ? { merge: commit.sha, number, head, commits: own, more: Number(node.commits?.totalCount) > own.length } : null)
|
|
143
|
+
}
|
|
144
|
+
})
|
|
145
|
+
}
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
// Where a request's time goes, told back in its Server-Timing header, which
|
|
2
|
+
// DevTools shows beside the request: the database's statements and the time
|
|
3
|
+
// spent in its operations (sql.ts), new pooled connections and their
|
|
4
|
+
// handshakes (db-neon.ts), GitHub's calls (github-app.ts githubJson), and
|
|
5
|
+
// the whole, from the handler's start to its answer's head; an instance's
|
|
6
|
+
// first request marked cold. Counts and durations alone, nothing of what was
|
|
7
|
+
// asked; a streamed answer's head tells what was known as it was written.
|
|
8
|
+
import { AsyncLocalStorage } from 'node:async_hooks'
|
|
9
|
+
import { ServerResponse } from 'node:http'
|
|
10
|
+
|
|
11
|
+
interface Timing {
|
|
12
|
+
started: number
|
|
13
|
+
cold: boolean
|
|
14
|
+
statements: number
|
|
15
|
+
db: number; dbMs: number
|
|
16
|
+
conn: number; connMs: number
|
|
17
|
+
gh: number; ghMs: number
|
|
18
|
+
}
|
|
19
|
+
const requests = new AsyncLocalStorage<Timing>()
|
|
20
|
+
let served = false
|
|
21
|
+
|
|
22
|
+
// `work` timed, its response's head carrying what it spent where; a test's
|
|
23
|
+
// stand-in for a response, not a ServerResponse, is left as it is.
|
|
24
|
+
export function runTimed<T>(res: ServerResponse, work: () => T): T {
|
|
25
|
+
const timing: Timing = { started: performance.now(), cold: !served, statements: 0, db: 0, dbMs: 0, conn: 0, connMs: 0, gh: 0, ghMs: 0 }
|
|
26
|
+
served = true
|
|
27
|
+
if (res instanceof ServerResponse) {
|
|
28
|
+
const writeHead = res.writeHead
|
|
29
|
+
res.writeHead = function (this: ServerResponse, ...args: Parameters<ServerResponse['writeHead']>) {
|
|
30
|
+
if (!this.headersSent) this.setHeader('server-timing', header(timing))
|
|
31
|
+
return writeHead.apply(this, args)
|
|
32
|
+
} as ServerResponse['writeHead']
|
|
33
|
+
}
|
|
34
|
+
return requests.run(timing, work)
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
// `work`, its time counted as `kind`'s where a request is timed.
|
|
38
|
+
export async function timed<T>(kind: 'db' | 'conn' | 'gh', work: () => Promise<T>): Promise<T> {
|
|
39
|
+
const timing = requests.getStore()
|
|
40
|
+
if (!timing) return work()
|
|
41
|
+
const start = performance.now()
|
|
42
|
+
try { return await work() } finally {
|
|
43
|
+
timing[kind]++
|
|
44
|
+
timing[`${kind}Ms`] += performance.now() - start
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
// A statement sent to the database, where a request is timed.
|
|
49
|
+
export function countStatement(): void {
|
|
50
|
+
const timing = requests.getStore()
|
|
51
|
+
if (timing) timing.statements++
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
function header(timing: Timing): string {
|
|
55
|
+
const ms = (value: number) => value.toFixed(1)
|
|
56
|
+
return [
|
|
57
|
+
...timing.db > 0 || timing.statements > 0 ? [`db;dur=${ms(timing.dbMs)};desc="${timing.statements} statements"`] : [],
|
|
58
|
+
...timing.conn > 0 ? [`conn;dur=${ms(timing.connMs)}`] : [],
|
|
59
|
+
...timing.gh > 0 ? [`gh;dur=${ms(timing.ghMs)};desc="${timing.gh} calls"`] : [],
|
|
60
|
+
...timing.cold ? ['cold;desc="first request of this instance"'] : [],
|
|
61
|
+
`app;dur=${ms(performance.now() - timing.started)}`,
|
|
62
|
+
].join(', ')
|
|
63
|
+
}
|
|
@@ -64,8 +64,9 @@ export interface CommentReview {
|
|
|
64
64
|
dependencies: { ecosystem?: 'npm'; name: string; version: string; integrity: string; patchHash?: string; status: ReviewVerdict }[]
|
|
65
65
|
}
|
|
66
66
|
// A version a dependency review is of, or compared it with: its package's
|
|
67
|
-
// name, its version
|
|
68
|
-
//
|
|
67
|
+
// name, its version, the integrity and, where npm tells one, the shasum of
|
|
68
|
+
// its tarball, each held to the tarball reviewed, and its patch's hash where
|
|
69
|
+
// it is known (pnpm's patched dependencies').
|
|
69
70
|
export interface ReviewedVersion { name: string; version: string; integrity: string; shasum?: string; patchHash?: string }
|
|
70
71
|
// One of one's own published dependency reviews, as `formatDependencyReviews`
|
|
71
72
|
// is given it: the version it is of (`new`) and the one it compared it with
|
|
@@ -77,29 +77,51 @@ export async function createSession(
|
|
|
77
77
|
// A request's sign-in, whose presence its first read to find it records.
|
|
78
78
|
// Every read after it in that request (the rechecks after its reads of
|
|
79
79
|
// storage, GitHub or npm among them) only reads: last_seen_at was written a
|
|
80
|
-
// moment before, by the read that let the request in.
|
|
81
|
-
|
|
80
|
+
// moment before, by the read that let the request in. What the router's
|
|
81
|
+
// gate read is handed to the read after it that says nothing was read
|
|
82
|
+
// between them (`handed`), asking nothing again; once only, and to none
|
|
83
|
+
// once another read came first.
|
|
84
|
+
type SessionRead = { session: ManagedSession; user: StoredUser }
|
|
85
|
+
const requests = new AsyncLocalStorage<{ seen: boolean; handed: { cookieHeader: string | undefined; read: SessionRead } | null }>()
|
|
82
86
|
export function runRequestPresence<T>(work: () => T): T {
|
|
83
|
-
return requests.run({ seen: false }, work)
|
|
87
|
+
return requests.run({ seen: false, handed: null }, work)
|
|
88
|
+
}
|
|
89
|
+
export function handOverSession(cookieHeader: string | undefined, read: SessionRead): void {
|
|
90
|
+
const request = requests.getStore()
|
|
91
|
+
if (request) request.handed = { cookieHeader, read }
|
|
84
92
|
}
|
|
85
93
|
|
|
86
94
|
// Resolve the current session (+ user) from the request's cookies, or null.
|
|
87
95
|
// A view cookie resolves only to that admin's view as another user: a view
|
|
88
96
|
// that has ended never falls back to the admin's own session. `own` skips the
|
|
89
|
-
// view and reads the signed-in session itself.
|
|
97
|
+
// view and reads the signed-in session itself. `handed` takes the gate's
|
|
98
|
+
// read, where it is this read's first since (handOverSession).
|
|
90
99
|
export async function readSession(
|
|
91
|
-
config: ManagedConfig, db: ManagedDb, cookieHeader: string | undefined, now: number, { own = false } = {},
|
|
92
|
-
): Promise<
|
|
100
|
+
config: ManagedConfig, db: ManagedDb, cookieHeader: string | undefined, now: number, { own = false, handed = false } = {},
|
|
101
|
+
): Promise<SessionRead | null> {
|
|
102
|
+
const request = requests.getStore()
|
|
103
|
+
const given = request?.handed
|
|
104
|
+
if (request) request.handed = null
|
|
105
|
+
if (handed && !own && given && given.cookieHeader === cookieHeader) return given.read
|
|
106
|
+
const ids = sessionIds(config, cookieHeader, { own })
|
|
107
|
+
if (!ids) return null
|
|
108
|
+
const seen = request?.seen === true
|
|
109
|
+
const s = ids.viewer === null
|
|
110
|
+
? await (seen ? db.getSessionWithUser : db.sessionWithUser)(ids.id, now)
|
|
111
|
+
: await (seen ? db.getViewSessionWithUser : db.viewSessionWithUser)(ids.id, ids.viewer, now)
|
|
112
|
+
if (s && request) request.seen = true
|
|
113
|
+
return s
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
// The sessions the request's cookies name, as readSession reads them: the
|
|
117
|
+
// one acting (`id`), a view where an admin views as another user, with the
|
|
118
|
+
// admin's own it is a view of (`viewer`); none signed out.
|
|
119
|
+
export function sessionIds(config: ManagedConfig, cookieHeader: string | undefined, { own = false } = {}): { id: string; viewer: string | null } | null {
|
|
93
120
|
const cookies = parseCookies(cookieHeader)
|
|
94
121
|
const token = cookies.get(config.sessionCookieName)
|
|
95
122
|
if (token == null || token === '') return null
|
|
96
123
|
const view = own ? undefined : cookies.get(cookieName(config, VIEW_COOKIE))
|
|
97
|
-
|
|
98
|
-
const s = view != null && view !== ''
|
|
99
|
-
? await (seen ? db.getViewSessionWithUser : db.viewSessionWithUser)(hashToken(view), hashToken(token), now)
|
|
100
|
-
: await (seen ? db.getSessionWithUser : db.sessionWithUser)(hashToken(token), now)
|
|
101
|
-
if (s && request) request.seen = true
|
|
102
|
-
return s
|
|
124
|
+
return view != null && view !== '' ? { id: hashToken(view), viewer: hashToken(token) } : { id: hashToken(token), viewer: null }
|
|
103
125
|
}
|
|
104
126
|
|
|
105
127
|
// Drop the session named by the request's cookie (logout). The caller also
|
package/server-managed/sql.ts
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
// its single connection across awaits; Neon can reuse a request-owned connection.
|
|
3
3
|
import type { DatabaseSync, SQLInputValue } from 'node:sqlite'
|
|
4
4
|
import type { ManagedDb } from './db-methods.ts'
|
|
5
|
+
import { countStatement, timed } from './request-timing.ts'
|
|
5
6
|
|
|
6
7
|
type MaybePromise<T> = T | Promise<T>
|
|
7
8
|
export interface ManagedSql {
|
|
@@ -22,8 +23,12 @@ export interface ManagedSqlDriver extends ManagedSql {
|
|
|
22
23
|
const SINGLE_STATEMENTS = new Set(['getReport', 'getBundle', 'getStorageRow', 'getStorageEncryption', 'getTeam', 'getComment',
|
|
23
24
|
'userCanReadReport', 'userCanReadRepo', 'userCanReadRepoPath', 'userCanReadBundle', 'reportPermissionsFor', 'listComments', 'listManagedIssues',
|
|
24
25
|
'listTriageHistory', 'listUsers', 'listUserOptions', 'listActivityReports', 'getRepoFullName', 'getUser', 'getSessionWithUser', 'getViewSessionWithUser',
|
|
25
|
-
'listTriageFixes',
|
|
26
|
-
'
|
|
26
|
+
'listTriageFixes', 'getRepoDependencyReviews', 'listTeamDependencyReviews', 'getNpmReviewReader',
|
|
27
|
+
'listDependencyReviewTeams', 'listDependencyReviewEnabledTeams', 'listUserNpmScopes', 'listTeamNpmScopes', 'hasTeamNpmScope',
|
|
28
|
+
'listGithubMetadata', 'listGithubRepositoryVisibility', 'setGithubMetadata', 'recordGithubMetadataAttempts', 'setGithubRepositoryVisibility',
|
|
29
|
+
'listLinkReports', 'listLegacyLinkReports', 'listReportFilenamesWithBundleHash', 'listReportsForRepo', 'listBundlesForRepo',
|
|
30
|
+
'listFindingCatalogReports', 'userCanReadBundleAdvisories',
|
|
31
|
+
'sessionWithUser', 'getTeamReportAccessSnapshot', 'getReportAccessSnapshot', 'getBundleAccessSnapshot', 'getReportView', 'getUserGithubId', 'getUserTeamFeedSnapshot', 'getAnnotations', 'getAnnotationRevision', 'getUserTokens',
|
|
27
32
|
'listReports', 'listBundles', 'listReadableBundleIds', 'listAllRepos', 'listSelectedRepos', 'listRepoScopesForUser', 'listTriage',
|
|
28
33
|
'getFeedState', 'getWorkspaceShareFeedState', 'claimMaintenanceLease', 'finishMaintenanceLease', 'cacheRepoDefaultBranch', 'releaseBundleBuildLease',
|
|
29
34
|
'getUpstreamCacheEntry', 'getUpstreamCacheEntries', 'getFirstUpstreamCacheEntries', 'setUpstreamCacheEntry', 'setUpstreamCacheEntries', 'ensureSessionUploadKey',
|
|
@@ -34,12 +39,13 @@ const UNLOCKED_WRITES = new Set(['sessionWithUser', 'viewSessionWithUser', 'ensu
|
|
|
34
39
|
export function scopeManagedMethods(methods: ManagedDb, driver: ManagedSqlDriver): ManagedDb {
|
|
35
40
|
const entries = Object.entries(methods).map(([name, method]) => {
|
|
36
41
|
if (name === 'close') return [name, async () => { await driver.close() }]
|
|
37
|
-
const write = !/^(?:get|list|userCanRead|reportPermissionsFor)/u.test(name)
|
|
38
|
-
|
|
42
|
+
const write = !/^(?:get|list|has|userCanRead|reportPermissionsFor)/u.test(name)
|
|
43
|
+
// Each operation's time told in its request's Server-Timing (request-timing.ts).
|
|
44
|
+
return [name, (...args: unknown[]) => timed('db', () => driver.scope(write, () => method(...args), {
|
|
39
45
|
// Presence writes only one user's timestamp; they need no application
|
|
40
46
|
// writer lock. Keep the session/user read and timestamp in one transaction.
|
|
41
47
|
lock: write && !UNLOCKED_WRITES.has(name), statement: SINGLE_STATEMENTS.has(name),
|
|
42
|
-
})]
|
|
48
|
+
}))]
|
|
43
49
|
})
|
|
44
50
|
return Object.fromEntries(entries) as ManagedDb
|
|
45
51
|
}
|
|
@@ -56,17 +62,19 @@ export function createSqliteDriver(db: DatabaseSync): ManagedSqlDriver {
|
|
|
56
62
|
prepare(sql) {
|
|
57
63
|
const stmt = db.prepare(sql)
|
|
58
64
|
return {
|
|
59
|
-
get: (...args) => stmt.get(...args as SQLInputValue[]),
|
|
60
|
-
all: (...args) => stmt.all(...args as SQLInputValue[]),
|
|
61
|
-
run: (...args) => stmt.run(...args as SQLInputValue[]),
|
|
65
|
+
get: (...args) => { countStatement(); return stmt.get(...args as SQLInputValue[]) },
|
|
66
|
+
all: (...args) => { countStatement(); return stmt.all(...args as SQLInputValue[]) },
|
|
67
|
+
run: (...args) => { countStatement(); return stmt.run(...args as SQLInputValue[]) },
|
|
62
68
|
}
|
|
63
69
|
},
|
|
64
70
|
scope(write, work) {
|
|
65
71
|
if (closed) return Promise.reject(new Error('Managed database is closed'))
|
|
66
72
|
return enqueue(async () => {
|
|
73
|
+
countStatement()
|
|
67
74
|
db.exec(write ? 'BEGIN IMMEDIATE' : 'BEGIN')
|
|
68
75
|
try {
|
|
69
76
|
const result = await work()
|
|
77
|
+
countStatement()
|
|
70
78
|
db.exec('COMMIT')
|
|
71
79
|
return result
|
|
72
80
|
} catch (err) {
|
|
@@ -4,7 +4,7 @@ import { Readable } from 'node:stream'
|
|
|
4
4
|
import { type StorageKey, decryptStorageStream, encryptStorageStream, wrapStorageValue } from '../server-common/storage-crypto.ts'
|
|
5
5
|
import { ENCRYPTED_CACHE_PREFIX, type ObjectStorage, type RawObject, type RawObjectStorage, deleteObjects, objectPath, openObjectVersion } from './object-storage.ts'
|
|
6
6
|
import { type StorageDb, type StorageEncryptionState, type StorageRow, type StorageRowKind, dataKeyIdentity, unwrapDataKey } from './storage-db.ts'
|
|
7
|
-
import { PUBLIC_CACHE_PREFIX, inspectStorageObject, storageOwner, storageRowPaths, verifyStoragePayload } from './storage-payload.ts'
|
|
7
|
+
import { PUBLIC_CACHE_PREFIX, inspectStorageObject, masterKeyCache, storageOwner, storageRowPaths, verifyStoragePayload } from './storage-payload.ts'
|
|
8
8
|
|
|
9
9
|
const STORAGE_WRITE_MS = 180_000
|
|
10
10
|
export const STORAGE_UPLOAD_TTL_MS = 86_400_000
|
|
@@ -68,6 +68,12 @@ async function decryptOwned(stored: RawObject, key: StorageKey, type: StorageRow
|
|
|
68
68
|
async function openEncrypted(raw: RawObjectStorage, db: StorageDb, key: StorageKey, identity: string, state: StorageEncryptionState): ReturnType<ObjectStorage['open']> {
|
|
69
69
|
const owner = storageOwner(identity)
|
|
70
70
|
if (!owner) {
|
|
71
|
+
// A cache no row owns: its ciphertext alone, what was kept as plaintext
|
|
72
|
+
// before activation never read.
|
|
73
|
+
if (masterKeyCache(identity)) {
|
|
74
|
+
const stored = await raw.open(encryptedCachePath(identity))
|
|
75
|
+
return stored && decryptStorageStream(stored.stream, key.bytes, identity)
|
|
76
|
+
}
|
|
71
77
|
if (!identity.startsWith('uploads/')) throw new Error('Unknown managed storage owner')
|
|
72
78
|
const stored = await raw.open(identity)
|
|
73
79
|
if (!stored) return null
|
|
@@ -229,7 +235,7 @@ export async function createEncryptedObjectStorage(raw: RawObjectStorage, db: St
|
|
|
229
235
|
if (!row || (owner.cache ? !row.dataKey : !storageRowPaths(owner.type, row).includes(identity))) return false
|
|
230
236
|
}
|
|
231
237
|
// Existence is metadata only; open() authenticates the actual bytes.
|
|
232
|
-
return await raw.head(state && owner?.cache ? encryptedCachePath(identity) : identity) !== null
|
|
238
|
+
return await raw.head(state && (owner?.cache || masterKeyCache(identity)) ? encryptedCachePath(identity) : identity) !== null
|
|
233
239
|
},
|
|
234
240
|
async put(identity, bytes, { replaceReport = false } = {}) {
|
|
235
241
|
logicalKey(identity)
|
|
@@ -248,10 +254,10 @@ export async function createEncryptedObjectStorage(raw: RawObjectStorage, db: St
|
|
|
248
254
|
dataKey = randomBytes(32)
|
|
249
255
|
wrapped = wrapStorageValue(key!, dataKeyIdentity(owner.type, owner.id), dataKey)
|
|
250
256
|
} else {
|
|
251
|
-
if (!identity.startsWith('uploads/')) throw new Error('Unknown managed storage owner')
|
|
257
|
+
if (!identity.startsWith('uploads/') && !masterKeyCache(identity)) throw new Error('Unknown managed storage owner')
|
|
252
258
|
dataKey = Buffer.from(key!.bytes)
|
|
253
259
|
}
|
|
254
|
-
const target = owner?.cache ? encryptedCachePath(identity) : identity
|
|
260
|
+
const target = owner?.cache || masterKeyCache(identity) ? encryptedCachePath(identity) : identity
|
|
255
261
|
const encrypted = encryptStorageStream(Readable.from([bytes]), dataKey, identity, bytes.length)
|
|
256
262
|
try {
|
|
257
263
|
await raw.put(target, encrypted, AbortSignal.timeout(STORAGE_WRITE_MS), undefined, bytes.length)
|
|
@@ -268,13 +274,15 @@ export async function createEncryptedObjectStorage(raw: RawObjectStorage, db: St
|
|
|
268
274
|
// when migration is disabled or another instance has just enabled it.
|
|
269
275
|
for (const target of deletionPaths(identity)) await raw.delete(target)
|
|
270
276
|
},
|
|
271
|
-
async deletePrefix(prefix) {
|
|
277
|
+
async deletePrefix(prefix, keep) {
|
|
272
278
|
logicalKey(`${prefix}validate`)
|
|
273
|
-
if (!prefix.endsWith('/')) throw new Error('Invalid storage prefix')
|
|
279
|
+
if (!prefix.endsWith('/') || (keep !== undefined && (!keep.startsWith(prefix) || !keep.endsWith('/')))) throw new Error('Invalid storage prefix')
|
|
274
280
|
await mode()
|
|
275
281
|
const signal = AbortSignal.timeout(STORAGE_WRITE_MS)
|
|
276
|
-
|
|
277
|
-
|
|
282
|
+
// What is kept, in the same generation as each removed.
|
|
283
|
+
const kept = keep === undefined ? [] : deletionPaths(keep)
|
|
284
|
+
for (const [i, target] of deletionPaths(prefix).entries()) {
|
|
285
|
+
await deleteObjects(raw, target, signal, undefined, kept[i])
|
|
278
286
|
await raw.prune?.(target, signal)
|
|
279
287
|
}
|
|
280
288
|
},
|
|
@@ -3,7 +3,7 @@ import { type StorageKey, decryptStorageStream, encryptStorageStream } from '../
|
|
|
3
3
|
import { ENCRYPTED_CACHE_PREFIX, type RawObjectStorage, deleteObjects, isBlobId, openObjectVersion } from './object-storage.ts'
|
|
4
4
|
import { type StorageDb, type StorageEncryptionState, type StorageMigrationRow, type StorageRow, type StorageRowKind, unwrapDataKey } from './storage-db.ts'
|
|
5
5
|
import { STORAGE_UPLOAD_TTL_MS } from './storage-encryption.ts'
|
|
6
|
-
import { StoragePayloadError, inspectStorageObject, storageOwner, storageRowPaths, verifiedStoragePayload, verifyStoragePayload } from './storage-payload.ts'
|
|
6
|
+
import { StoragePayloadError, inspectStorageObject, masterKeyCache, storageOwner, storageRowPaths, verifiedStoragePayload, verifyStoragePayload } from './storage-payload.ts'
|
|
7
7
|
|
|
8
8
|
async function replacementFailure(raw: RawObjectStorage, identity: string, version: string, signal: AbortSignal): Promise<string> {
|
|
9
9
|
const format = (value: string) => value.startsWith('W/') ? 'weak' : value.startsWith('"') && value.endsWith('"') ? 'quoted' : 'unquoted'
|
|
@@ -113,7 +113,10 @@ function cleanupTarget(identity: string) {
|
|
|
113
113
|
const path = temporary ? temp![1]! : identity
|
|
114
114
|
const encryptedCache = path.startsWith(ENCRYPTED_CACHE_PREFIX)
|
|
115
115
|
if (encryptedCache && !temporary) return null
|
|
116
|
-
const
|
|
116
|
+
const logical = encryptedCache ? `cache/${path.slice(ENCRYPTED_CACHE_PREFIX.length)}` : path
|
|
117
|
+
// A cache no row owns is a cache all the same: its plaintext legacy, and
|
|
118
|
+
// its temporary files removed past the staging window.
|
|
119
|
+
const owner = storageOwner(logical) ?? (masterKeyCache(logical) ? { cache: true as const } : null)
|
|
117
120
|
return owner ? { owner, temporary } : null
|
|
118
121
|
}
|
|
119
122
|
|
|
@@ -19,6 +19,24 @@ export function storageRowPaths(type: StorageRowKind, row: Pick<StorageRow, 'id'
|
|
|
19
19
|
// are: public npm versions' pretty-printed files (pretty-print.ts).
|
|
20
20
|
export const PUBLIC_CACHE = 'npm'
|
|
21
21
|
export const PUBLIC_CACHE_PREFIX = `cache/${PUBLIC_CACHE}/`
|
|
22
|
+
// A cache of private content no row owns: the code viewer's directories at
|
|
23
|
+
// repositories' default branches' heads (repository-code.ts
|
|
24
|
+
// repositoryCodeKept), by repository. Plaintext while storage is not
|
|
25
|
+
// encrypted, as other caches are; once it is, kept apart under
|
|
26
|
+
// ENCRYPTED_CACHE_PREFIX as a bundle's caches are, so that what was kept
|
|
27
|
+
// before is never read and legacy cleanup removes it, and encrypted with the
|
|
28
|
+
// master key, as upload parts are, each write's key derived from its path,
|
|
29
|
+
// so that one path's ciphertext fails to authenticate at another. No storage
|
|
30
|
+
// row holds a repository's code for a data key of its own to be wrapped in:
|
|
31
|
+
// its selection is none, and stays as it is deactivated. A key of its own
|
|
32
|
+
// would add a column and its migration only for its deletion to make what is
|
|
33
|
+
// left unreadable at once, as a deleted bundle's caches are; here deletion
|
|
34
|
+
// is physical, by the repository's prefix, at best, as an upload part's is,
|
|
35
|
+
// and what a failed one leaves is a directory as GitHub has it at a commit,
|
|
36
|
+
// read only by a reader GitHub lets read that repository.
|
|
37
|
+
export const REPOSITORY_CODE_CACHE = 'repository-code'
|
|
38
|
+
const REPOSITORY_CODE_CACHE_PREFIX = `cache/${REPOSITORY_CODE_CACHE}/`
|
|
39
|
+
export const masterKeyCache = (identity: string) => identity.startsWith(REPOSITORY_CODE_CACHE_PREFIX)
|
|
22
40
|
export function storageOwner(identity: string): { type: StorageRowKind; id: string; cache: boolean } | null {
|
|
23
41
|
const match = /^(reports|bundles)\/([^/]+)$/u.exec(identity)
|
|
24
42
|
if (match) {
|
|
@@ -8,7 +8,7 @@ import { createBundleStore } from './bundle-store.ts'
|
|
|
8
8
|
import { createReportStore } from './report-store.ts'
|
|
9
9
|
import { CacheMissError, type CacheStorage, validateCacheKey } from './cache-storage.ts'
|
|
10
10
|
import { type ObjectStorage, isBlobId } from './object-storage.ts'
|
|
11
|
-
import { PUBLIC_CACHE } from './storage-payload.ts'
|
|
11
|
+
import { PUBLIC_CACHE, REPOSITORY_CODE_CACHE } from './storage-payload.ts'
|
|
12
12
|
|
|
13
13
|
function id(value: string): string {
|
|
14
14
|
if (!isBlobId(value)) throw new Error('Invalid managed blob id')
|
|
@@ -35,7 +35,7 @@ function cacheStore(objects: ObjectStorage, name: string): CacheStorage {
|
|
|
35
35
|
if (!value) throw new CacheMissError()
|
|
36
36
|
return value
|
|
37
37
|
},
|
|
38
|
-
delete: prefix => objects.deletePrefix(`${path(prefix)}/`),
|
|
38
|
+
delete: (prefix, keep) => objects.deletePrefix(`${path(prefix)}/`, keep === undefined ? undefined : `${path(`${prefix}/${keep}`)}/`),
|
|
39
39
|
}
|
|
40
40
|
}
|
|
41
41
|
|
|
@@ -86,5 +86,6 @@ export function createManagedStores(objects: ObjectStorage, avatarSidecar: boole
|
|
|
86
86
|
cacheStorage,
|
|
87
87
|
reportSourcesStorage: cacheStore(objects, 'report-sources'),
|
|
88
88
|
npmCacheStorage: cacheStore(objects, PUBLIC_CACHE),
|
|
89
|
+
repositoryCodeStorage: cacheStore(objects, REPOSITORY_CODE_CACHE),
|
|
89
90
|
}
|
|
90
91
|
}
|