@preventive/triage 1.0.0-alpha.46 → 1.0.0-alpha.48
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/client/dependency-paths.d.ts +2 -1
- package/client/dependency-paths.js +16 -4
- package/common/managed/agent-tasks.ts +12 -12
- package/common/managed/github-agents.ts +29 -10
- package/out/activity.js +4 -0
- package/out/chunk-6LNQTKAV.js +52 -0
- package/out/chunk-7BKKYVWR.js +7 -0
- package/out/chunk-7VAWINQF.js +6 -0
- package/out/chunk-C2AVODGZ.js +36 -0
- package/out/{chunk-I2MDGSQX.js → chunk-IO7JTZ6O.js} +1 -1
- package/out/chunk-MOACFRJ4.js +3 -0
- package/out/chunk-O6M574JI.js +1 -0
- package/out/chunk-PBZPYIG7.js +1 -0
- package/out/chunk-PENGGKX7.js +233 -0
- package/out/{chunk-OKRLLE5I.js → chunk-PWFUOLZK.js} +1 -1
- package/out/chunk-SZD5CW5R.js +3 -0
- package/out/{chunk-G24A42V6.js → chunk-TQJDYHVE.js} +1 -1
- package/out/{chunk-3PS4KWD2.js → chunk-WCPJDAYA.js} +1 -1
- package/out/{chunk-7XRR4OFU.js → chunk-WYLQWYLA.js} +1 -1
- package/out/chunk-X6PJAK3T.js +1 -0
- package/out/client-managed-import.js +1 -1
- package/out/client-managed.js +26 -26
- package/out/client-sync.js +2 -2
- package/out/compare.js +14 -14
- package/out/dialogs.js +40 -40
- package/out/graph.js +11 -11
- package/out/index.html +12 -12
- package/out/scan.js +2 -2
- package/out/terminal.js +215 -215
- package/out/view.css +1 -1
- package/out/view.js +15 -15
- package/package.json +9 -5
- package/risk/index.d.ts +4 -2
- package/risk/npm/assess.js +15 -4
- package/risk/npm/registry.js +6 -1
- package/risk/npm/score.js +1 -1
- package/server-managed/activity.ts +24 -11
- package/server-managed/agent-merges.ts +10 -8
- package/server-managed/app-agents.ts +8 -21
- package/server-managed/bundle-build-worker.js +7 -3
- package/server-managed/bundle-build.ts +5 -3
- package/server-managed/bundle-cache.ts +9 -3
- package/server-managed/bundle.ts +3 -4
- package/server-managed/cache-storage.ts +3 -2
- package/server-managed/commit-coauthors.ts +11 -10
- package/server-managed/commit-lockfiles.ts +22 -21
- package/server-managed/db-methods.ts +338 -187
- package/server-managed/db-neon.ts +31 -8
- package/server-managed/db.ts +2 -0
- package/server-managed/dependabot-access.ts +1 -1
- package/server-managed/dependency-review-comments.ts +7 -4
- package/server-managed/dependency-reviews.ts +71 -33
- package/server-managed/github-accounts.ts +28 -1
- package/server-managed/github-app.ts +27 -23
- package/server-managed/github-graphql.ts +39 -0
- package/server-managed/github-issue-links.ts +8 -1
- package/server-managed/github-metadata.ts +2 -2
- package/server-managed/http-response.ts +28 -10
- package/server-managed/http.ts +736 -247
- package/server-managed/index.ts +2 -2
- package/server-managed/link-reports.ts +5 -1
- package/server-managed/managed-issues.ts +3 -0
- package/server-managed/npm-delta.ts +16 -0
- package/server-managed/npm-kept.ts +51 -10
- package/server-managed/npm-loads.ts +48 -28
- package/server-managed/npm-manifest.ts +1 -1
- package/server-managed/npm-packages.ts +31 -15
- package/server-managed/npm-risk.ts +64 -17
- package/server-managed/object-storage.ts +4 -3
- package/server-managed/pretty-print.ts +45 -12
- package/server-managed/public-workspace.ts +3 -3
- package/server-managed/pull-dependencies.ts +84 -36
- package/server-managed/pull-dependency-reviews.ts +13 -10
- package/server-managed/rename-diff-worker.js +1 -1
- package/server-managed/report-store.ts +16 -11
- package/server-managed/repository-agents.ts +9 -6
- package/server-managed/repository-browser.ts +17 -5
- package/server-managed/repository-code.ts +180 -17
- package/server-managed/repository-dependency-reviews.ts +7 -0
- package/server-managed/repository-history.ts +85 -152
- package/server-managed/repository-issues.ts +27 -13
- package/server-managed/repository-pulse.ts +9 -3
- package/server-managed/repository-squashes.ts +145 -0
- package/server-managed/request-timing.ts +63 -0
- package/server-managed/reviewed-dependencies.ts +3 -2
- package/server-managed/session.ts +34 -12
- package/server-managed/solidity-entry-points.ts +5 -3
- package/server-managed/sql-postgres.ts +3 -0
- package/server-managed/sql.ts +17 -8
- package/server-managed/storage-db.ts +11 -3
- package/server-managed/storage-encryption.ts +23 -12
- package/server-managed/storage-maintenance.ts +5 -2
- package/server-managed/storage-payload.ts +18 -0
- package/server-managed/storage-stores.ts +3 -2
- package/server-managed/storage.ts +4 -2
- package/server-managed/team-feed.ts +19 -12
- package/server-managed/team-npm-scopes.ts +29 -0
- package/server-managed/team-reports.ts +74 -17
- package/server-managed/uploads.ts +29 -9
- package/out/chunk-4UFAMJAX.js +0 -1
- package/out/chunk-BOIXMUU5.js +0 -3
- package/out/chunk-CC4OM6TZ.js +0 -3
- package/out/chunk-JKGQ3O7O.js +0 -6
- package/out/chunk-LE5ZWLG4.js +0 -235
- package/out/chunk-MSION3G4.js +0 -36
- package/out/chunk-S6GIXWQW.js +0 -52
- package/out/chunk-V3KWNNMY.js +0 -1
- package/out/chunk-WQ74ZBZF.js +0 -7
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@preventive/triage",
|
|
3
|
-
"version": "1.0.0-alpha.
|
|
3
|
+
"version": "1.0.0-alpha.48",
|
|
4
4
|
"description": "Client & relay server for triaging of automated reports",
|
|
5
5
|
"license": "MIT OR Apache-2.0",
|
|
6
6
|
"author": {
|
|
@@ -130,6 +130,7 @@
|
|
|
130
130
|
"server-managed/github-actions.ts",
|
|
131
131
|
"server-managed/github-app.ts",
|
|
132
132
|
"server-managed/github-errors.ts",
|
|
133
|
+
"server-managed/github-graphql.ts",
|
|
133
134
|
"server-managed/github-metadata.ts",
|
|
134
135
|
"server-managed/github-issue-oauth.ts",
|
|
135
136
|
"server-managed/github-issues.ts",
|
|
@@ -187,9 +188,11 @@
|
|
|
187
188
|
"server-managed/repository-changes.ts",
|
|
188
189
|
"server-managed/repository-code.ts",
|
|
189
190
|
"server-managed/repository-dependency-reviews.ts",
|
|
191
|
+
"server-managed/request-timing.ts",
|
|
190
192
|
"server-managed/reviewed-dependencies.ts",
|
|
191
193
|
"server-managed/repository-agents.ts",
|
|
192
194
|
"server-managed/repository-history.ts",
|
|
195
|
+
"server-managed/repository-squashes.ts",
|
|
193
196
|
"server-managed/repository-issues.ts",
|
|
194
197
|
"server-managed/repository-pulse.ts",
|
|
195
198
|
"server-managed/repository-aliases.ts",
|
|
@@ -343,6 +346,7 @@
|
|
|
343
346
|
"client/finding-link.js",
|
|
344
347
|
"client/linked-findings.d.ts",
|
|
345
348
|
"client/linked-findings.js",
|
|
349
|
+
"out/activity.js",
|
|
346
350
|
"out/brotli-fallback.js",
|
|
347
351
|
"out/chunk-*.js",
|
|
348
352
|
"out/client-managed.js",
|
|
@@ -400,16 +404,16 @@
|
|
|
400
404
|
"typescript": "^7.0.2"
|
|
401
405
|
},
|
|
402
406
|
"dependencies": {
|
|
403
|
-
"@exodus/stasis": "1.0.0-beta.
|
|
404
|
-
"@exodus/stasis-core": "1.0.0-beta.
|
|
405
|
-
"@preventive/archive": "^1.
|
|
407
|
+
"@exodus/stasis": "1.0.0-beta.11",
|
|
408
|
+
"@exodus/stasis-core": "1.0.0-beta.11",
|
|
409
|
+
"@preventive/archive": "^1.2.0",
|
|
406
410
|
"@preventive/compare": "^1.2.11",
|
|
407
411
|
"@preventive/diff": "^1.1.0",
|
|
408
412
|
"@preventive/lockfile": "^1.2.0",
|
|
409
413
|
"@preventive/publishers": "^1.1.0",
|
|
410
414
|
"@preventive/report": "^1.0.0",
|
|
411
415
|
"@preventive/sourcemap": "1.0.0-alpha.1",
|
|
412
|
-
"@preventive/upstream": "^1.
|
|
416
|
+
"@preventive/upstream": "^1.3.0",
|
|
413
417
|
"css-tree": "^3.2.1",
|
|
414
418
|
"oxc-minify": "^0.152.0",
|
|
415
419
|
"oxc-parser": "^0.152.0",
|
package/risk/index.d.ts
CHANGED
|
@@ -40,8 +40,10 @@ export interface AssessNpmOptions {
|
|
|
40
40
|
// Where what is read is kept, by type (risk/npm/registry.js and assess.js
|
|
41
41
|
// name each): `npm/tarballs`, `risk/npm/versions`, `risk/npm/packages`,
|
|
42
42
|
// `risk/npm/old-versions`, `risk/npm/public`, `risk/npm/downloads`,
|
|
43
|
-
// `risk/npm/advisories` and `risk/npm/assessments`.
|
|
44
|
-
|
|
43
|
+
// `risk/npm/advisories` and `risk/npm/assessments`. One that reads records
|
|
44
|
+
// of any types at once (`readEach`, each null where none is kept) is read
|
|
45
|
+
// so where risk reads several together.
|
|
46
|
+
cache?: (CacheStore & { readEach?(entries: [type: string, key: string][]): Promise<unknown[]> }) | false
|
|
45
47
|
// Asks for a package's whole document in risk's stead, as the managed
|
|
46
48
|
// server does within its bounds: null where npm has none for this asker.
|
|
47
49
|
// `authorized` whether it may be asked with the caller's npm token: the
|
package/risk/npm/assess.js
CHANGED
|
@@ -197,15 +197,22 @@ export async function assessNpm(name, version, { oldVersion = null, fromEmpty =
|
|
|
197
197
|
const facts = await npmPackageFacts(name, version, oldVersion, context)
|
|
198
198
|
const old = facts.oldVersion
|
|
199
199
|
if (old !== null && !facts.versions.has(old)) throw new NpmRiskError('version-not-found', `npm has no ${name}@${old}`)
|
|
200
|
-
|
|
201
|
-
|
|
200
|
+
// Its advisories, its lasting risk and its downloads need none of each
|
|
201
|
+
// other, so all are asked at once, and waited on in this order, so that a
|
|
202
|
+
// failure fails it as the first of them failing would.
|
|
203
|
+
const listed = facts.versions.has(version)
|
|
204
|
+
const advised = settledLater(advisories ? npmVersionAdvisories(name, version, osv, context) : Promise.resolve([]))
|
|
205
|
+
const keeping = listed ? settledLater(lasting(name, version, old, facts, context, fromEmpty)) : null
|
|
206
|
+
const counting = listed && downloads && popularity ? settledLater(npmDownloads(name, context)) : null
|
|
207
|
+
const malware = npmMalwareFinding({ takenDown: facts.takenDown, advisories: await advised })
|
|
208
|
+
if (!listed) {
|
|
202
209
|
if (!malware) throw new NpmRiskError('version-not-found', `npm has no ${name}@${version}`)
|
|
203
210
|
return { name, version, oldVersion: null, published: facts.removed.get(version), removed: true, findings: [malware], files: {}, dependencies: { added: [], changed: [] },
|
|
204
211
|
score: npmRiskScore([malware]) }
|
|
205
212
|
}
|
|
206
213
|
const { published, manifest } = facts.versions.get(version)
|
|
207
|
-
const kept = await
|
|
208
|
-
const counted =
|
|
214
|
+
const kept = await keeping
|
|
215
|
+
const counted = await counting
|
|
209
216
|
const own = npmVersionFindings({ deprecated: facts.deprecated[version], published, minimumReleaseAge, now, known: Boolean(manifest.knownAccounts), scope: npmKnownScope(name),
|
|
210
217
|
provenance: Boolean(manifest.provenance), downloads: counted })
|
|
211
218
|
const was = old === null ? null : facts.versions.get(old).published
|
|
@@ -225,6 +232,10 @@ export async function assessNpm(name, version, { oldVersion = null, fromEmpty =
|
|
|
225
232
|
// A dependency's key among a version's, by its kind and name.
|
|
226
233
|
const keyOf = row => `${row.kind ?? ''}\0${row.name}`
|
|
227
234
|
|
|
235
|
+
// `promise`, waited on later: its failure, if it fails before then, not
|
|
236
|
+
// thrown as unhandled meanwhile.
|
|
237
|
+
const settledLater = promise => { promise.catch(() => {}); return promise }
|
|
238
|
+
|
|
228
239
|
// The keys of the dependencies a version holds that nothing of it imports
|
|
229
240
|
// (`kept`, npmLastingRisk's: its `held-unused-deps`), those risk counts
|
|
230
241
|
// (dependencies.js npmUnusedDependencyCounted).
|
package/risk/npm/registry.js
CHANGED
|
@@ -87,6 +87,11 @@ async function kept({ cache }, type, key, ask, fresh = () => true) {
|
|
|
87
87
|
return value
|
|
88
88
|
}
|
|
89
89
|
|
|
90
|
+
// What is kept under each `[type, key]` of `entries`, in their order, null
|
|
91
|
+
// where nothing is: in one read where the cache reads records of any types at
|
|
92
|
+
// once (`readEach`), else one a key.
|
|
93
|
+
const readEach = (cache, entries) => cache.readEach ? cache.readEach(entries) : Promise.all(entries.map(([type, key]) => cache.read(type, key)))
|
|
94
|
+
|
|
90
95
|
// An answer's body as JSON, read no further than `limit` bytes.
|
|
91
96
|
async function jsonOf(res, limit, what) {
|
|
92
97
|
const tooLarge = () => new NpmRiskError('upstream-too-large', `npm's answer for ${what} is over ${limit} bytes`)
|
|
@@ -169,7 +174,7 @@ export async function npmPackageFacts(name, version, oldVersion, context) {
|
|
|
169
174
|
const given = oldVersion === true ? fresh(picked) ? picked.oldVersion : undefined : oldVersion
|
|
170
175
|
if (cache && given !== undefined) {
|
|
171
176
|
const wanted = given === null ? [version] : [version, given]
|
|
172
|
-
const [
|
|
177
|
+
const [pkg, ...facts] = await readEach(cache, [['risk/npm/packages', name], ...wanted.map(at => ['risk/npm/versions', `${name}@${at}`])])
|
|
173
178
|
const gone = at => Object.hasOwn(pkg.removed, at)
|
|
174
179
|
if (fresh(pkg) && 'removed' in pkg && wanted.every((at, i) => facts[i] || gone(at))) {
|
|
175
180
|
return { versions: new Map(wanted.flatMap((at, i) => gone(at) ? [] : [[at, facts[i]]])), removed: new Map(wanted.filter(gone).map(at => [at, pkg.removed[at]])),
|
package/risk/npm/score.js
CHANGED
|
@@ -7,7 +7,7 @@ import { DAY_MS } from './downloads.js'
|
|
|
7
7
|
// How much each finding raises the score, by its severity: a version's
|
|
8
8
|
// findings combined as independent chances are (1 − Π(1 − weight)), so that
|
|
9
9
|
// each adds less the more there are, none past 100. One red finding alone is
|
|
10
|
-
// 50; two,
|
|
10
|
+
// 50; two are 75, and one with two orange 68, as attacks show.
|
|
11
11
|
const WEIGHTS = { critical: 0.5, high: 0.2, moderate: 0.08, low: 0.03, notice: 0 }
|
|
12
12
|
// What the version's age tells instead (`too-new`), as exposure does below.
|
|
13
13
|
const TOLD_BY_AGE = new Set(['too-new'])
|
|
@@ -176,12 +176,21 @@ export function activityMethods(db: ManagedSql): ActivityStore {
|
|
|
176
176
|
const stmts = contexts == null ? admin : manager
|
|
177
177
|
const scope = contexts == null ? {} : { contexts: JSON.stringify(contexts), userId: userId ?? '' }
|
|
178
178
|
const params = { ...scope, kind, query, repo, actor }
|
|
179
|
-
const
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
179
|
+
const read = async (at: number) => (await stmts.rows.all({ ...params, limit, offset: (at - 1) * limit })) as (ActivityEntry & { total: number | string })[]
|
|
180
|
+
// A page's rows carry the total. Only a page past the last has none to
|
|
181
|
+
// carry it: it is counted, and the last page read in its place.
|
|
182
|
+
let currentPage = page, rows = await read(page)
|
|
183
|
+
let total = rows.length > 0 ? Number(rows[0]!.total) : 0
|
|
184
|
+
if (rows.length === 0 && page > 1) {
|
|
185
|
+
total = Number(((await stmts.count.get(params)) as { total: number | string }).total)
|
|
186
|
+
currentPage = Math.min(page, Math.max(1, Math.ceil(total / limit)))
|
|
187
|
+
if (currentPage !== page) rows = total === 0 ? [] : await read(currentPage)
|
|
188
|
+
}
|
|
189
|
+
const history = rows.map(({ total: _total, ...entry }) => entry)
|
|
190
|
+
const filters: ActivityFilters = { repos: [], users: [] }
|
|
191
|
+
for (const row of (await stmts.filters.all(scope)) as { facet: string; id: string; login: string; detail: string | null }[]) {
|
|
192
|
+
if (row.facet === 'repo') filters.repos.push(row.id)
|
|
193
|
+
else filters.users.push({ id: row.id, login: row.login, detail: row.detail })
|
|
185
194
|
}
|
|
186
195
|
return { history, total, page: currentPage, limit, filters }
|
|
187
196
|
},
|
|
@@ -199,17 +208,21 @@ function activityStatements(db: ManagedSql, source: string) {
|
|
|
199
208
|
AND (:repo = '' OR repo = :repo) AND (:actor = '' OR actorKey = :actor)
|
|
200
209
|
AND (:query = '' OR instr(lower(coalesce(actor, '') || ' ' || action || ' ' ||
|
|
201
210
|
coalesce(repo, '') || ' ' || coalesce(report, '') || ' ' || coalesce(finding, '')), lower(:query)) > 0)`
|
|
211
|
+
// A page's rows each carry the total, so the source is read once for both;
|
|
212
|
+
// the selector choices, repositories then users, once between them.
|
|
202
213
|
return {
|
|
203
214
|
count: db.prepare(`SELECT count(*) AS total FROM (${filtered}) AS filtered`),
|
|
204
|
-
rows: db.prepare(
|
|
215
|
+
rows: db.prepare(`SELECT filtered.*, count(*) OVER () AS total FROM (${filtered}) AS filtered ORDER BY at DESC,
|
|
205
216
|
CASE WHEN kind = 'triage' THEN CAST(substr(id, instr(id, ':') + 1) AS INTEGER) ELSE 0 END DESC,
|
|
206
217
|
id DESC LIMIT :limit OFFSET :offset`),
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
218
|
+
filters: db.prepare(`${activity} SELECT 'repo' AS facet, repo AS id, repo AS login, NULL AS detail, repo AS sort, '' AS tiebreak
|
|
219
|
+
FROM (SELECT DISTINCT repo FROM activity WHERE repo IS NOT NULL AND repo <> '') AS repos
|
|
220
|
+
UNION ALL SELECT 'user' AS facet, a.actorKey AS id, COALESCE(u.login, a.actor, a.actorId) AS login,
|
|
221
|
+
CASE WHEN a.actorId IS NULL THEN 'Legacy actor' ELSE NULL END AS detail, COALESCE(u.login, a.actor, a.actorId) AS sort, a.actorKey AS tiebreak FROM (
|
|
210
222
|
SELECT actorKey, actorId, actor, ROW_NUMBER() OVER (PARTITION BY actorKey ORDER BY at DESC, id DESC) AS position
|
|
211
223
|
FROM activity WHERE actorKey IS NOT NULL AND actorKey <> 'legacy:'
|
|
212
|
-
) a LEFT JOIN managed_user u ON u.id = a.actorId WHERE a.position = 1
|
|
224
|
+
) a LEFT JOIN managed_user u ON u.id = a.actorId WHERE a.position = 1
|
|
225
|
+
ORDER BY facet, sort, tiebreak`),
|
|
213
226
|
}
|
|
214
227
|
}
|
|
215
228
|
|
|
@@ -4,10 +4,11 @@
|
|
|
4
4
|
// pull request's branch alone naming it (app-agents.ts
|
|
5
5
|
// AGENT_BRANCH_PREFIXES), or a label one who ran it gave it naming the agent
|
|
6
6
|
// (agent-tasks.ts labelAgents), as the Agents tab tells an agent's work; or
|
|
7
|
-
// its author, an account agents work as (github-agents.ts AGENT_OPERATED)
|
|
7
|
+
// its author, an account agents work as (github-agents.ts AGENT_OPERATED),
|
|
8
|
+
// or any GitHub App's bot.
|
|
8
9
|
import { labelAgents } from '../common/managed/agent-tasks.ts'
|
|
9
10
|
import { GITHUB_AGENTS, agentOperated } from '../common/managed/github-agents.ts'
|
|
10
|
-
import { AGENT_BRANCH_PREFIXES
|
|
11
|
+
import { AGENT_BRANCH_PREFIXES } from './app-agents.ts'
|
|
11
12
|
import { type HistoryCommit, githubGraphql } from './repository-history.ts'
|
|
12
13
|
|
|
13
14
|
// The merged pull requests read an answer, a page a query, as many queries
|
|
@@ -22,7 +23,7 @@ const MERGED_LABELS = 100
|
|
|
22
23
|
// How long before or after a pull request rebased onto the branch was merged
|
|
23
24
|
// its copies are committed.
|
|
24
25
|
const REBASE_SKEW_MS = 10 * 60_000
|
|
25
|
-
const LABEL_AGENTS =
|
|
26
|
+
const LABEL_AGENTS = Object.keys(GITHUB_AGENTS)
|
|
26
27
|
const SHA = /^[a-f\d]{40}$/u
|
|
27
28
|
|
|
28
29
|
const count = (value: unknown) => Number.isSafeInteger(value) && (value as number) >= 0 ? value as number : 0
|
|
@@ -53,14 +54,14 @@ const MERGED_QUERY = `query($owner: String!, $name: String!, $after: String) {
|
|
|
53
54
|
repository(owner: $owner, name: $name) {
|
|
54
55
|
pullRequests(states: MERGED, first: ${MERGED_PAGE}, after: $after, orderBy: { field: UPDATED_AT, direction: DESC }) {
|
|
55
56
|
pageInfo { hasNextPage endCursor }
|
|
56
|
-
nodes { updatedAt mergedAt headRefName author { login ... on User { databaseId } } labels(first: ${MERGED_LABELS}) { nodes { name } }
|
|
57
|
+
nodes { updatedAt mergedAt headRefName author { __typename login ... on User { databaseId } } labels(first: ${MERGED_LABELS}) { nodes { name } }
|
|
57
58
|
mergeCommit { oid parents { totalCount } }
|
|
58
59
|
commits(last: ${MERGED_COMMITS}) { totalCount nodes { commit { oid messageHeadline changedFilesIfAvailable } } } }
|
|
59
60
|
}
|
|
60
61
|
}
|
|
61
62
|
}`
|
|
62
63
|
type MergedPage = { repository?: { pullRequests?: { pageInfo?: { hasNextPage?: unknown; endCursor?: unknown }
|
|
63
|
-
nodes?: ({ updatedAt?: unknown; mergedAt?: unknown; headRefName?: unknown; author?: { login?: unknown; databaseId?: unknown } | null
|
|
64
|
+
nodes?: ({ updatedAt?: unknown; mergedAt?: unknown; headRefName?: unknown; author?: { __typename?: unknown; login?: unknown; databaseId?: unknown } | null
|
|
64
65
|
labels?: { nodes?: ({ name?: unknown } | null)[] } | null
|
|
65
66
|
mergeCommit?: { oid?: unknown; parents?: { totalCount?: unknown } | null } | null
|
|
66
67
|
commits?: { totalCount?: unknown; nodes?: ({ commit?: { oid?: unknown; messageHeadline?: unknown; changedFilesIfAvailable?: unknown } | null } | null)[] } | null
|
|
@@ -68,9 +69,10 @@ type MergedPage = { repository?: { pullRequests?: { pageInfo?: { hasNextPage?: u
|
|
|
68
69
|
const agentBranch = (name: unknown) => typeof name === 'string' && AGENT_BRANCH_PREFIXES.some(prefix => name.startsWith(prefix))
|
|
69
70
|
type MergedNode = NonNullable<NonNullable<NonNullable<NonNullable<MergedPage['repository']>['pullRequests']>['nodes']>[number]>
|
|
70
71
|
// Whether a pull request is an agent's: by its branch, a label naming one,
|
|
71
|
-
// or its author, an account agents work as
|
|
72
|
+
// or its author, an account agents work as or a bot (GitHub's GraphQL API
|
|
73
|
+
// naming a bot by its app's login alone, its type telling it).
|
|
72
74
|
const agentPull = ({ headRefName, labels, author }: MergedNode) => agentBranch(headRefName)
|
|
73
|
-
|| labelAgents((labels?.nodes ?? []).map(label => label?.name), LABEL_AGENTS).length > 0
|
|
75
|
+
|| labelAgents((labels?.nodes ?? []).map(label => label?.name), LABEL_AGENTS).length > 0 || author?.__typename === 'Bot'
|
|
74
76
|
|| agentOperated(typeof author?.login === 'string' ? author.login : null, Number.isSafeInteger(author?.databaseId) ? author!.databaseId as number : null)
|
|
75
77
|
|
|
76
78
|
// The commits the pull requests merged since `since` from an agent's branch,
|
|
@@ -152,7 +154,7 @@ const headed = (subject: string, headline: string) => headline.endsWith('…') ?
|
|
|
152
154
|
// (`read`, agentMerges): those it names, and of one rebased, its commits'
|
|
153
155
|
// copies, the last and those below it on their first parents, as long as
|
|
154
156
|
// each one's subject is that of one of its commits older than the last
|
|
155
|
-
// matched, as Commits tells a rebase (repository-
|
|
157
|
+
// matched, as Commits tells a rebase (repository-squashes.ts squashedPulls),
|
|
156
158
|
// past those of its commits the rebase left none of (an empty one GitHub
|
|
157
159
|
// counted no files of), and, past them all, as many more as it had before
|
|
158
160
|
// its latest read; a squash merge's one commit alone, its subject the pull
|
|
@@ -2,29 +2,16 @@
|
|
|
2
2
|
// tab's, server-managed/repository-agents.ts, whose marks the Commits tab
|
|
3
3
|
// reads on commits too, repository-history.ts).
|
|
4
4
|
import type { AppAgent, AppAgentMarks } from '../common/managed/agent-tasks.ts'
|
|
5
|
-
import { GITHUB_AGENTS } from '../common/managed/github-agents.ts'
|
|
5
|
+
import { GITHUB_AGENTS, type GithubAgent } from '../common/managed/github-agents.ts'
|
|
6
6
|
|
|
7
7
|
// The agents known by their GitHub Apps or accounts, beside Claude and
|
|
8
|
-
// Codex:
|
|
9
|
-
// /
|
|
10
|
-
//
|
|
11
|
-
//
|
|
12
|
-
//
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
// requests and authors its commits as its account, Copilot (198982749), on
|
|
16
|
-
// `copilot/` branches; Dependabot, its GitHub App (29110), opens its pull
|
|
17
|
-
// requests as its account, `dependabot` (49699333, `dependabot[bot]` to
|
|
18
|
-
// GitHub's REST API and in its address), on `dependabot/<ecosystem>/…`
|
|
19
|
-
// branches. Their accounts and marks are those the app draws their work by
|
|
20
|
-
// wherever it shows it (common/managed/github-agents.ts): Copilot's its own,
|
|
21
|
-
// the app's to draw, no avatar.
|
|
22
|
-
const { copilot, dependabot } = GITHUB_AGENTS
|
|
23
|
-
export const APP_AGENTS: readonly (AppAgent & AppAgentMarks)[] = [
|
|
24
|
-
{ id: 'neo', name: 'Neo', avatar: 'app-3115942', fingerprint: '🤖 Generated with [Neo](', prefix: 'neo/' },
|
|
25
|
-
{ id: 'copilot', name: copilot.name, account: copilot.accounts[0], prefix: 'copilot/' },
|
|
26
|
-
{ id: 'dependabot', name: dependabot.name, avatar: dependabot.avatar, account: dependabot.accounts[0], prefix: 'dependabot/' },
|
|
27
|
-
]
|
|
8
|
+
// Codex: those of GITHUB_AGENTS with branches of their own, in its order
|
|
9
|
+
// (common/managed/github-agents.ts, where each is told, its accounts among
|
|
10
|
+
// it), each its id and name, its avatar (its mark, served by
|
|
11
|
+
// /api/avatar/<avatar>, a GitHub App's, `app-<id>`), the fingerprint it
|
|
12
|
+
// leaves, and its branches' prefix.
|
|
13
|
+
export const APP_AGENTS: readonly (AppAgent & AppAgentMarks)[] = Object.entries<GithubAgent>(GITHUB_AGENTS)
|
|
14
|
+
.flatMap(([id, { name, avatar, prefix, fingerprint }]) => prefix ? [{ id, name, prefix, ...avatar ? { avatar } : {}, ...fingerprint ? { fingerprint } : {} }] : [])
|
|
28
15
|
// The prefixes of the branches agents work on: Claude Code's and Codex's,
|
|
29
16
|
// and the app agents'.
|
|
30
17
|
export const AGENT_BRANCH_PREFIXES = ['claude/', 'codex/', ...APP_AGENTS.map(agent => agent.prefix)]
|
|
@@ -96,13 +96,17 @@ export function buildErrorCode(error, message) {
|
|
|
96
96
|
if (parentPort && workerData?.type === 'managed-bundle-build') {
|
|
97
97
|
// Send diagnostics to the request's parent thread, which writes them before
|
|
98
98
|
// returning the HTTP response. Worker stdout can be lost on termination.
|
|
99
|
-
const send = message => {
|
|
99
|
+
const send = (message, transfer = []) => {
|
|
100
100
|
// Node worker messages have no browser target origin.
|
|
101
101
|
// oxlint-disable-next-line unicorn/require-post-message-target-origin
|
|
102
|
-
parentPort.postMessage(message)
|
|
102
|
+
parentPort.postMessage(message, transfer)
|
|
103
103
|
}
|
|
104
104
|
try {
|
|
105
|
-
|
|
105
|
+
const built = await buildStasisBundle(workerData, undefined, stage => send({ type: 'progress', stage }))
|
|
106
|
+
// The bytes handed over rather than copied, where they alone fill their
|
|
107
|
+
// buffer (a small one may share Node's pool, which can't be moved).
|
|
108
|
+
const { bytes } = built
|
|
109
|
+
send(built, bytes.byteOffset === 0 && bytes.byteLength === bytes.buffer.byteLength ? [bytes.buffer] : [])
|
|
106
110
|
} catch (error) {
|
|
107
111
|
const diagnostic = bundleBuildDiagnostic(error, workerData.token)
|
|
108
112
|
const code = buildErrorCode(error, diagnostic.message)
|
|
@@ -95,7 +95,7 @@ export function githubBundleFilename(github: string, directory: string, commit:
|
|
|
95
95
|
return `${repo}.${dir}${tail}`
|
|
96
96
|
}
|
|
97
97
|
|
|
98
|
-
export interface BuiltBundle { bytes:
|
|
98
|
+
export interface BuiltBundle { bytes: Buffer; directory: string; filename: string }
|
|
99
99
|
export interface BuildRequest {
|
|
100
100
|
input: BundleBuildInput; github: string; token: string | null; maxBytes: number; scopes: (string | null)[]
|
|
101
101
|
// Upstream's disk cache for what a build fetches (setCacheDir), or null for
|
|
@@ -174,14 +174,16 @@ export async function buildRepositoryBundle(userId: string, request: BuildReques
|
|
|
174
174
|
const abort = () => fail(signal.reason instanceof BundleBuildError ? signal.reason.status : 499,
|
|
175
175
|
signal.reason instanceof BundleBuildError ? signal.reason.code : 'build-cancelled')
|
|
176
176
|
signal.addEventListener('abort', abort, { once: true })
|
|
177
|
-
running.on('message', (message: BuiltBundle & { type?: string; stage?: BundleBuildStage; error?: string; status?: number; diagnostic?: BundleBuildDiagnostic }) => {
|
|
177
|
+
running.on('message', (message: Omit<BuiltBundle, 'bytes'> & { bytes: Uint8Array; type?: string; stage?: BundleBuildStage; error?: string; status?: number; diagnostic?: BundleBuildDiagnostic }) => {
|
|
178
178
|
if (settled) return
|
|
179
179
|
if (message.type === 'progress' && message.stage) { stage = message.stage; log('progress'); return }
|
|
180
180
|
if (message.error) { fail(message.status ?? 422, message.error, { diagnostic: message.diagnostic }); return }
|
|
181
181
|
settled = true
|
|
182
182
|
cleanup()
|
|
183
183
|
log('completed', { byteSize: message.bytes.length })
|
|
184
|
-
|
|
184
|
+
// A view of the bytes the worker handed over, not a copy of them.
|
|
185
|
+
const { bytes } = message
|
|
186
|
+
resolve({ bytes: Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength), directory: message.directory, filename: message.filename })
|
|
185
187
|
})
|
|
186
188
|
running.once('error', error => fail(422, 'build-failed', { diagnostic: bundleBuildDiagnostic(error, request.token) }))
|
|
187
189
|
running.once('exit', exitCode => fail(422, 'build-failed', { exitCode }))
|
|
@@ -111,16 +111,22 @@ async function build(record: BundleCacheRecord, storage: BundleCacheStorage, db:
|
|
|
111
111
|
const metadata = { ...await createBundleMetadata(details), id: record.id, filename: record.filename }
|
|
112
112
|
const body = await encodeBrotli(Buffer.from(JSON.stringify(metadata)))
|
|
113
113
|
if (!(await db.getBundle(record.id))) throw new Error('Bundle deleted')
|
|
114
|
-
await storage.put(record.id, filename, body)
|
|
115
|
-
if (record.kind === 'stasis') await storage.put(record.id, packagesFilename, encodePackageInventory(details))
|
|
116
114
|
const summary = bundleSummary(details, metadata)
|
|
117
|
-
|
|
115
|
+
// Written together, each settled before the row is looked at again: one
|
|
116
|
+
// failing leaves the others, which a later open builds over.
|
|
117
|
+
const written = await Promise.allSettled([
|
|
118
|
+
storage.put(record.id, filename, body),
|
|
119
|
+
...record.kind === 'stasis' ? [storage.put(record.id, packagesFilename, encodePackageInventory(details))] : [],
|
|
120
|
+
storage.put(record.id, SUMMARY_FILENAME, Buffer.from(JSON.stringify(summary))),
|
|
121
|
+
])
|
|
118
122
|
// A different instance may have deleted the row while these writes ran.
|
|
119
123
|
// Reconcile after publishing so its cleanup cannot be undone by us.
|
|
120
124
|
if (!(await db.getBundle(record.id))) {
|
|
121
125
|
await storage.delete(record.id)
|
|
122
126
|
throw new Error('Bundle deleted')
|
|
123
127
|
}
|
|
128
|
+
const failed = written.find(result => result.status === 'rejected')
|
|
129
|
+
if (failed) throw failed.reason
|
|
124
130
|
return summary
|
|
125
131
|
}
|
|
126
132
|
|
package/server-managed/bundle.ts
CHANGED
|
@@ -109,10 +109,9 @@ export async function bundleFilePrefix(bytes: Buffer): Promise<string> {
|
|
|
109
109
|
}
|
|
110
110
|
|
|
111
111
|
// Extract the bundle integrities a report declares (its top-level
|
|
112
|
-
// `bundleHashes`)
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
try { data = JSON.parse(bytes.toString('utf8')) } catch { return [] }
|
|
112
|
+
// `bundleHashes`), from the JSON it parsed to (readManagedReport's, as is).
|
|
113
|
+
// Non-JSON reports (markdown / CSV, given as null) or a missing field → [].
|
|
114
|
+
export function reportBundleHashes(data: unknown): string[] {
|
|
116
115
|
if (data == null || typeof data !== 'object') return []
|
|
117
116
|
const hashes = (data as { bundleHashes?: unknown }).bundleHashes
|
|
118
117
|
if (!Array.isArray(hashes)) return []
|
|
@@ -5,12 +5,13 @@ export class CacheMissError extends Error {
|
|
|
5
5
|
constructor() { super('Managed cache unavailable') }
|
|
6
6
|
}
|
|
7
7
|
|
|
8
|
-
// Keys are internal relative paths; delete removes a whole directory of
|
|
8
|
+
// Keys are internal relative paths; delete removes a whole directory of
|
|
9
|
+
// variants, but for one directory within it where `keep` names one.
|
|
9
10
|
export interface CacheStorage {
|
|
10
11
|
exists(key: string): Promise<boolean>
|
|
11
12
|
put(key: string, bytes: Buffer): Promise<void>
|
|
12
13
|
open(key: string): Promise<OpenedBlob>
|
|
13
|
-
delete(prefix: string): Promise<void>
|
|
14
|
+
delete(prefix: string, keep?: string): Promise<void>
|
|
14
15
|
}
|
|
15
16
|
|
|
16
17
|
export function validateCacheKey(key: string): string {
|
|
@@ -10,15 +10,14 @@ export const MAX_COAUTHORS = 3
|
|
|
10
10
|
// The rules co-authors are read by, the agents' addresses and names among
|
|
11
11
|
// them: what was kept read by others (history pages, squashed pull requests)
|
|
12
12
|
// read again by these, once raised.
|
|
13
|
-
export const COAUTHORS_READ =
|
|
13
|
+
export const COAUTHORS_READ = 4
|
|
14
14
|
const MAX_NAME = 256
|
|
15
15
|
const textOf = (value: unknown, max: number) => typeof value === 'string' && value.trim() ? value.trim().slice(0, max) : null
|
|
16
16
|
|
|
17
17
|
// One who made a commit beside its author: by name and account, as its
|
|
18
|
-
// author is, and the agent it is where known,
|
|
19
|
-
//
|
|
20
|
-
|
|
21
|
-
export interface Coauthor { name: string | null; login: string | null; id: number | null; agent?: string; avatar?: string }
|
|
18
|
+
// author is, and the agent it is where known (common/managed/github-agents.ts),
|
|
19
|
+
// by its account, its commits' address or name, or the marks it leaves.
|
|
20
|
+
export interface Coauthor { name: string | null; login: string | null; id: number | null; agent?: string }
|
|
22
21
|
|
|
23
22
|
// `Co-authored-by: <name> <address>` trailers, GitHub matching the key
|
|
24
23
|
// regardless of case; Cursor's agents' links, as their work carries them;
|
|
@@ -31,6 +30,11 @@ const FINGERPRINTED = APP_AGENTS.filter(agent => agent.fingerprint)
|
|
|
31
30
|
// commit gives (where it is read), and account.
|
|
32
31
|
export type Maker = { name: unknown; email?: unknown; account: NamedAccount }
|
|
33
32
|
|
|
33
|
+
// Those a commit's message names in its `Co-authored-by:` trailers
|
|
34
|
+
// (agent-tasks.ts trailerBlock), each by name and address.
|
|
35
|
+
export const coauthorTrailers = (message: unknown) => typeof message === 'string'
|
|
36
|
+
? [...trailerBlock(message.replaceAll('\r\n', '\n')).matchAll(COAUTHOR_TRAILER)].map(([, name, email]) => ({ name: name!, email: email! })) : []
|
|
37
|
+
|
|
34
38
|
// The agent one who made a commit is: by their account, else the address
|
|
35
39
|
// their commit names them by; else, GitHub naming no account for them, by
|
|
36
40
|
// their name.
|
|
@@ -51,7 +55,7 @@ const makerAgent = ({ name, email, account }: Maker): GithubAgentId | null => gi
|
|
|
51
55
|
// two people of one name with accounts or addresses of their own two.
|
|
52
56
|
export function coauthorsOf(message: unknown, author: Maker, listed: Maker[] | null = null): Coauthor[] {
|
|
53
57
|
const text = typeof message === 'string' ? message.replaceAll('\r\n', '\n') : ''
|
|
54
|
-
const named = listed ??
|
|
58
|
+
const named = listed ?? coauthorTrailers(text).map(({ name, email }) => ({ name, email, account: commitAccount({ email }) }))
|
|
55
59
|
const signals = agentSignals([text], FINGERPRINTED)
|
|
56
60
|
const marked = [...signals.claude || signals.sessions.length > 0 ? ['claude'] : [], ...signals.tasks.length > 0 || signals.codex ? ['codex'] : [],
|
|
57
61
|
...CURSOR_AGENT.test(text) ? ['cursor'] : [], ...signals.agents]
|
|
@@ -75,9 +79,6 @@ export function coauthorsOf(message: unknown, author: Maker, listed: Maker[] | n
|
|
|
75
79
|
const { login, id } = agent && maker.account.login && githubAgent(maker.account.login, maker.account.id) !== agent ? { login: null, id: null } : maker.account
|
|
76
80
|
add({ name: textOf(maker.name, MAX_NAME), login, id, ...agent ? { agent } : {} }, maker.email)
|
|
77
81
|
}
|
|
78
|
-
for (const agent of marked) {
|
|
79
|
-
const app = APP_AGENTS.find(known => known.id === agent && !Object.hasOwn(GITHUB_AGENTS, known.id))
|
|
80
|
-
add({ name: app?.name ?? GITHUB_AGENTS[agent as GithubAgentId].name, login: null, id: null, agent, ...app?.avatar ? { avatar: app.avatar } : {} })
|
|
81
|
-
}
|
|
82
|
+
for (const agent of marked) add({ name: GITHUB_AGENTS[agent as GithubAgentId].name, login: null, id: null, agent })
|
|
82
83
|
return coauthors
|
|
83
84
|
}
|
|
@@ -1,51 +1,52 @@
|
|
|
1
1
|
// The versions a repository's lockfiles of npm's ecosystem lock at a commit,
|
|
2
2
|
// as their review reads them (pull-dependency-reviews.ts): every lockfile
|
|
3
3
|
// there (findRepoLockfiles) a review holds to (`held`, lockfile-scope.ts),
|
|
4
|
-
// read as a pull request's are (pull-dependencies.ts readLockfile, kept
|
|
5
|
-
// this instance
|
|
6
|
-
// each's versions (`lockfiles`), and those its reader
|
|
7
|
-
// past MAX_LOCKFILES (`unread`), as many read at once
|
|
8
|
-
// those of some of them (lockedIn), each version once by
|
|
9
|
-
// are held to (lockedWith), dev only where it is wherever it
|
|
10
|
-
// GitHub having none of a lockfile it lists there fails the
|
|
11
|
-
// would for any file.
|
|
4
|
+
// read as a pull request's are (pull-dependencies.ts readLockfile), but kept
|
|
5
|
+
// by its blob, in this instance and beyond it (`kept`), as many as
|
|
6
|
+
// MAX_LOCKFILES of those, each's versions (`lockfiles`), and those its reader
|
|
7
|
+
// refuses, of Bun's, or past MAX_LOCKFILES (`unread`), as many read at once
|
|
8
|
+
// as READ_AT_ONCE; and those of some of them (lockedIn), each version once by
|
|
9
|
+
// what its reviews are held to (lockedWith), dev only where it is wherever it
|
|
10
|
+
// is locked. GitHub having none of a lockfile it lists there fails the
|
|
11
|
+
// whole, as it would for any file.
|
|
12
12
|
import { createClient } from '@preventive/upstream/github.js'
|
|
13
13
|
import { upstreamGithub } from './github-errors.ts'
|
|
14
14
|
import { GithubApiError } from './github-app.ts'
|
|
15
15
|
import type { LockedPackage } from './locked-packages.ts'
|
|
16
16
|
import { type Lru, lru } from './lru.ts'
|
|
17
|
-
import { eachOf, lockedWith, lockfileFormat,
|
|
17
|
+
import { type GithubAsked, type ListedLockfile, type LockfileStore, eachOf, lockedWith, lockfileFormat, readLockfile } from './pull-dependencies.ts'
|
|
18
18
|
|
|
19
19
|
export const MAX_LOCKFILES = 100
|
|
20
20
|
const READ_AT_ONCE = 4
|
|
21
21
|
|
|
22
22
|
export interface CommitLockfiles { lockfiles: { path: string; packages: LockedPackage[] }[]; unread: string[] }
|
|
23
|
-
// Set where GitHub was asked for what a read reads (`github`), not all of it
|
|
24
|
-
// held in this instance.
|
|
25
|
-
export interface GithubAsked { github: boolean }
|
|
26
23
|
|
|
27
24
|
// The lockfiles at commit `sha` its review reads, by their paths, each with
|
|
28
|
-
// its format (pull-dependencies.ts lockfileFormat), kept by
|
|
29
|
-
// `asked` told where
|
|
30
|
-
|
|
31
|
-
|
|
25
|
+
// its format (pull-dependencies.ts lockfileFormat) and its blob's id, kept by
|
|
26
|
+
// the commit, in this instance and beyond it (`kept`); `asked` told where
|
|
27
|
+
// GitHub was.
|
|
28
|
+
const listings: Lru<ListedLockfile[]> = lru(20_000, listed => listed.length + 1)
|
|
29
|
+
export async function lockfilesAt(fullName: string, token: string, sha: string, kept: LockfileStore | null, asked?: GithubAsked) {
|
|
32
30
|
const key = `${fullName}\0${sha}`
|
|
33
31
|
const known = listings.get(key)
|
|
34
32
|
if (known) return known
|
|
33
|
+
const stored = await kept?.listings.read(sha)
|
|
34
|
+
if (stored) { listings.set(key, stored); return stored }
|
|
35
35
|
if (asked) asked.github = true
|
|
36
36
|
const { lockfiles } = await upstreamGithub(() => createClient({ token, userAgent: 'deepview-triage' }).findRepoLockfiles({ repo: fullName, sha }))
|
|
37
|
-
const listed = lockfiles.flatMap(({ path }) => { const format = lockfileFormat(path); return format ? [{ path, format }] : [] })
|
|
37
|
+
const listed = lockfiles.flatMap(({ path, sha: blob }) => { const format = lockfileFormat(path); return format ? [{ path, format, sha: blob }] : [] })
|
|
38
38
|
listings.set(key, listed)
|
|
39
|
+
await kept?.listings.write(sha, listed)
|
|
39
40
|
return listed
|
|
40
41
|
}
|
|
41
42
|
|
|
42
|
-
export async function lockedAt(fullName: string, token: string, sha: string, held: (path: string) => boolean = () => true,
|
|
43
|
-
|
|
43
|
+
export async function lockedAt(fullName: string, token: string, sha: string, kept: LockfileStore | null, held: (path: string) => boolean = () => true,
|
|
44
|
+
asked?: GithubAsked): Promise<CommitLockfiles> {
|
|
45
|
+
const listed = (await lockfilesAt(fullName, token, sha, kept, asked)).filter(({ path }) => held(path))
|
|
44
46
|
const unread = listed.slice(MAX_LOCKFILES).map(({ path }) => path)
|
|
45
47
|
const lockfiles: CommitLockfiles['lockfiles'] = []
|
|
46
48
|
const reading = listed.slice(0, MAX_LOCKFILES)
|
|
47
|
-
|
|
48
|
-
const reads = await eachOf(reading, READ_AT_ONCE, lockfile => readLockfile(fullName, token, sha, lockfile))
|
|
49
|
+
const reads = await eachOf(reading, READ_AT_ONCE, lockfile => readLockfile(fullName, token, sha, lockfile, kept, asked))
|
|
49
50
|
for (const [at, lockfile] of reading.entries()) {
|
|
50
51
|
const read = reads[at]!
|
|
51
52
|
if ('error' in read && read.error === 'github-not-found') throw new GithubApiError(404, 'github-not-found')
|