@preventive/triage 1.0.0-alpha.45 → 1.0.0-alpha.47

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. package/client/linked-findings.js +3 -0
  2. package/common/bundle-metadata.d.ts +1 -0
  3. package/common/bundle-metadata.js +1 -1
  4. package/common/bundle-sources.js +5 -1
  5. package/common/line-diff.js +5 -4
  6. package/common/managed/agent-tasks.ts +5 -3
  7. package/common/managed/github-agents.ts +28 -10
  8. package/out/activity.js +4 -0
  9. package/out/{chunk-NPW52BM2.js → chunk-4HI2MSAV.js} +13 -13
  10. package/out/chunk-5AHIQFFG.js +36 -0
  11. package/out/chunk-6ANSHRFQ.js +6 -0
  12. package/out/chunk-6WF3KNTL.js +3 -0
  13. package/out/chunk-7BKKYVWR.js +7 -0
  14. package/out/{chunk-HMAJDVXN.js → chunk-FH24VHSJ.js} +1 -1
  15. package/out/chunk-NOZJULWS.js +52 -0
  16. package/out/{chunk-OKRLLE5I.js → chunk-PWFUOLZK.js} +1 -1
  17. package/out/chunk-S6U2J62U.js +232 -0
  18. package/out/chunk-SPEFPML6.js +2 -0
  19. package/out/chunk-UENXT2HK.js +3 -0
  20. package/out/chunk-VL43BTSO.js +1 -0
  21. package/out/{chunk-3PS4KWD2.js → chunk-WCPJDAYA.js} +1 -1
  22. package/out/chunk-ZCS4UYHI.js +1 -0
  23. package/out/chunk-ZGNMDGSB.js +1 -0
  24. package/out/client-managed-import.js +1 -1
  25. package/out/client-managed.js +15 -15
  26. package/out/client-sync.js +2 -2
  27. package/out/compare.js +12 -12
  28. package/out/dialogs.js +28 -28
  29. package/out/graph.js +11 -11
  30. package/out/index.html +11 -11
  31. package/out/scan.js +2 -2
  32. package/out/terminal.js +6 -6
  33. package/out/view.css +1 -1
  34. package/out/view.js +15 -15
  35. package/package.json +4 -1
  36. package/risk/actions/findings.js +14 -9
  37. package/risk/actions/scripts/shell.js +5 -1
  38. package/risk/index.d.ts +9 -4
  39. package/risk/npm/assess.js +15 -4
  40. package/risk/npm/registry.js +6 -1
  41. package/server-e2e/app.ts +5 -4
  42. package/server-e2e/http.ts +5 -2
  43. package/server-managed/activity.ts +24 -11
  44. package/server-managed/app-agents.ts +10 -21
  45. package/server-managed/bundle-build-worker.js +7 -3
  46. package/server-managed/bundle-build.ts +5 -3
  47. package/server-managed/bundle-cache.ts +16 -4
  48. package/server-managed/bundle.ts +3 -4
  49. package/server-managed/combined.ts +3 -2
  50. package/server-managed/commit-coauthors.ts +5 -9
  51. package/server-managed/commit-lockfiles.ts +21 -11
  52. package/server-managed/config.ts +6 -0
  53. package/server-managed/db-methods.ts +321 -156
  54. package/server-managed/db-neon.ts +71 -24
  55. package/server-managed/db-schema.ts +4 -0
  56. package/server-managed/db.ts +2 -1
  57. package/server-managed/dependency-reviews.ts +13 -3
  58. package/server-managed/github-app.ts +12 -6
  59. package/server-managed/github-issue-links.ts +11 -5
  60. package/server-managed/github-pulls.ts +2 -2
  61. package/server-managed/github-requests.ts +39 -0
  62. package/server-managed/http-response.ts +10 -2
  63. package/server-managed/http.ts +269 -143
  64. package/server-managed/index.ts +8 -7
  65. package/server-managed/kept-briefly.ts +37 -0
  66. package/server-managed/link-reports.ts +5 -1
  67. package/server-managed/npm-delta.ts +11 -6
  68. package/server-managed/npm-kept.ts +51 -7
  69. package/server-managed/npm-loads.ts +117 -35
  70. package/server-managed/npm-packages.ts +107 -14
  71. package/server-managed/npm-risk.ts +68 -18
  72. package/server-managed/pretty-print.ts +112 -35
  73. package/server-managed/pull-dependencies.ts +29 -9
  74. package/server-managed/pull-dependency-reviews.ts +8 -8
  75. package/server-managed/queue.ts +16 -0
  76. package/server-managed/rename-diff.ts +22 -13
  77. package/server-managed/report-sources.ts +1 -1
  78. package/server-managed/report-store.ts +50 -12
  79. package/server-managed/repository-actions.ts +15 -6
  80. package/server-managed/repository-agents.ts +1 -1
  81. package/server-managed/repository-browser.ts +123 -33
  82. package/server-managed/repository-code.ts +28 -11
  83. package/server-managed/repository-dependency-reviews.ts +41 -19
  84. package/server-managed/repository-history.ts +24 -8
  85. package/server-managed/repository-pulse.ts +9 -2
  86. package/server-managed/repository-risk-audit.ts +13 -0
  87. package/server-managed/repository-risk-read.ts +11 -6
  88. package/server-managed/repository-risk.ts +27 -20
  89. package/server-managed/session.ts +18 -4
  90. package/server-managed/solidity-entry-points.ts +5 -3
  91. package/server-managed/sql-postgres.ts +3 -0
  92. package/server-managed/sql.ts +6 -2
  93. package/server-managed/storage-db.ts +11 -3
  94. package/server-managed/storage-encryption.ts +7 -4
  95. package/server-managed/storage.ts +8 -4
  96. package/server-managed/team-feed.ts +3 -1
  97. package/server-managed/team-reports.ts +30 -13
  98. package/server-managed/uploads.ts +29 -9
  99. package/server-managed/upstream-cache.ts +12 -0
  100. package/server.ts +14 -5
  101. package/out/chunk-5DHL5NG3.js +0 -2
  102. package/out/chunk-6HNXNIQY.js +0 -52
  103. package/out/chunk-AFLZH7WG.js +0 -1
  104. package/out/chunk-NV7ES4EW.js +0 -1
  105. package/out/chunk-PTGDYLNS.js +0 -36
  106. package/out/chunk-Q6MVYGYV.js +0 -3
  107. package/out/chunk-QRD6GIG5.js +0 -229
  108. package/out/chunk-WI7Z5A2T.js +0 -6
  109. package/out/chunk-WQ74ZBZF.js +0 -7
  110. package/out/chunk-XVBEOGOS.js +0 -3
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@preventive/triage",
3
- "version": "1.0.0-alpha.45",
3
+ "version": "1.0.0-alpha.47",
4
4
  "description": "Client & relay server for triaging of automated reports",
5
5
  "license": "MIT OR Apache-2.0",
6
6
  "author": {
@@ -139,6 +139,7 @@
139
139
  "server-managed/github-users.ts",
140
140
  "server-managed/github-oauth.ts",
141
141
  "server-managed/github-pulls.ts",
142
+ "server-managed/github-requests.ts",
142
143
  "server-managed/http-response.ts",
143
144
  "server-managed/http.ts",
144
145
  "server-managed/link-reports.ts",
@@ -156,6 +157,7 @@
156
157
  "server-managed/npm-risk.ts",
157
158
  "server-managed/npm-provenance.ts",
158
159
  "server-managed/kept.ts",
160
+ "server-managed/kept-briefly.ts",
159
161
  "server-managed/npm-kept.ts",
160
162
  "server-managed/npm-loads.ts",
161
163
  "server-managed/npm-delta.ts",
@@ -341,6 +343,7 @@
341
343
  "client/finding-link.js",
342
344
  "client/linked-findings.d.ts",
343
345
  "client/linked-findings.js",
346
+ "out/activity.js",
344
347
  "out/brotli-fallback.js",
345
348
  "out/chunk-*.js",
346
349
  "out/client-managed.js",
@@ -64,16 +64,21 @@ function lockedIn(lockfiles, directory, name) {
64
64
  // The inputs a text names (`${{ inputs.title }}`).
65
65
  const inputsIn = text => typeof text === 'string' ? [...dotted(text).matchAll(/\$\{\{[^}]*?\binputs\s*\.\s*([\w-]+)/gu)].map(match => match[1]) : []
66
66
 
67
- // The inputs a step's action writes into a script: github-script's
68
- // `script`, and those an action of the repository's own (ScanOptions
69
- // `actions`) names in its `run:`s or hands to such an input of a step's.
67
+ // The inputs a step's action, or a job's reusable workflow, writes into a
68
+ // script: github-script's `script`; those one of the repository's own
69
+ // (ScanOptions `actions`) names in its `run:`s or its jobs' images, or hands
70
+ // to such an input of a step's or a job's; and each of another reusable
71
+ // workflow's, or one whose text isn't told, as it may.
70
72
  function scriptInputs(step, actions, seen = new Set()) {
71
73
  if (step.uses?.repo?.toLowerCase() === 'actions/github-script') return ['script']
72
- const dir = step.uses?.kind === 'local' && actions?.has(step.uses.path) && !seen.has(step.uses.path) ? step.uses.path : null
73
- const read = dir === null ? null : readWorkflow('action', actions.get(dir))
74
+ if (step.uses?.kind === 'workflow' || (step.uses?.job && !actions?.has(step.uses.path))) return Object.keys(step.with ?? {})
75
+ const path = step.uses?.kind === 'local' && actions?.has(step.uses.path) && !seen.has(step.uses.path) ? step.uses.path : null
76
+ const read = path === null ? null : readWorkflow(step.uses.job ? 'workflow' : 'action', actions.get(path))
74
77
  if (!read || read instanceof YamlError) return []
75
- return [...new Set(read.jobs.flatMap(job => job.steps.flatMap(each => [...inputsIn(each.run?.script),
76
- ...scriptInputs(each, actions, new Set(seen).add(dir)).flatMap(input => inputsIn(each.with?.[input]))])))]
78
+ const inner = new Set(seen).add(path)
79
+ const handed = each => scriptInputs(each, actions, inner).flatMap(input => inputsIn(each.with?.[input]))
80
+ return [...new Set(read.jobs.flatMap(job => [...job.uses ? handed(job) : [], ...job.images.flatMap(image => inputsIn(image.text)),
81
+ ...job.steps.flatMap(each => [...inputsIn(each.run?.script), ...handed(each)])]))]
77
82
  }
78
83
 
79
84
  // Injections through the inputs an action writes into its scripts, and a
@@ -98,8 +103,8 @@ function jobFindings(job, triggers, { owner, lockfiles, external, actions }) {
98
103
  const finding = usesFinding(job.uses, owner)
99
104
  if (finding) found.push({ ...finding, line: job.uses.line })
100
105
  if (job.secrets === 'inherit' && job.uses.repo && ownerKind(job.uses.repo, owner) === 'third-party') found.push({ id: 'secrets-inherited', severity: 'high', uses: job.uses.uses, key: usesKey(job.uses), line: job.uses.line })
101
- // An input of a reusable workflow, which one writes into its scripts.
102
- for (const [input, value] of Object.entries(job.with ?? {})) for (const { index: _index, ...injected } of scriptInjections(value, triggers)) found.push({ ...injected, line: job.withLines[input] })
106
+ // An input a reusable workflow writes into its scripts.
107
+ for (const input of scriptInputs(job, actions)) for (const { index: _index, ...injected } of scriptInjections(job.with?.[input], triggers)) found.push({ ...injected, line: job.withLines[input] })
103
108
  }
104
109
  for (const image of job.images) {
105
110
  if (!image.read?.digest && !image.text.includes('${{')) found.push({ ...imageFinding(image.text, image.read), line: image.line })
@@ -12,7 +12,11 @@ const SHELL = new RegExp(String.raw`^(?:.*[\/\\])?${SHELLS}(?:\.exe)?$`, 'iu')
12
12
  // pipe, `&&` or `||` the next line goes on from), and the line it starts
13
13
  // on, counting from 0.
14
14
  export function scriptLines(script) {
15
- const lines = script.split('\n')
15
+ // An expression (`${{ … }}`), which GitHub writes in before the shell reads
16
+ // the script, read as a variable is: a value unknown here, each line it
17
+ // spans past its first continuing the one it is on; a `}}` in one of its
18
+ // strings (`format('{{0}}')`) not its end.
19
+ const lines = script.replaceAll(/\$\{\{(?:'[^']*'|[^'])*?\}\}/gu, expression => `$EXPRESSION${' \\\n'.repeat(expression.split('\n').length - 1)}`).split('\n')
16
20
  const joined = []
17
21
  for (let at = 0; at < lines.length; at++) {
18
22
  const start = at
package/risk/index.d.ts CHANGED
@@ -40,8 +40,10 @@ export interface AssessNpmOptions {
40
40
  // Where what is read is kept, by type (risk/npm/registry.js and assess.js
41
41
  // name each): `npm/tarballs`, `risk/npm/versions`, `risk/npm/packages`,
42
42
  // `risk/npm/old-versions`, `risk/npm/public`, `risk/npm/downloads`,
43
- // `risk/npm/advisories` and `risk/npm/assessments`.
44
- cache?: CacheStore | false
43
+ // `risk/npm/advisories` and `risk/npm/assessments`. One that reads records
44
+ // of any types at once (`readEach`, each null where none is kept) is read
45
+ // so where risk reads several together.
46
+ cache?: (CacheStore & { readEach?(entries: [type: string, key: string][]): Promise<unknown[]> }) | false
45
47
  // Asks for a package's whole document in risk's stead, as the managed
46
48
  // server does within its bounds: null where npm has none for this asker.
47
49
  // `authorized` whether it may be asked with the caller's npm token: the
@@ -200,8 +202,11 @@ export interface ScanOptions {
200
202
  // Its lockfiles' paths, the packages each locks, as npx runs those, by
201
203
  // name and by `name@version`, and those only devDependencies need.
202
204
  lockfiles?: { path: string; names: ReadonlySet<string>; dev?: ReadonlySet<string> }[]
203
- // Its own actions' metadata by their directories, for where they install
204
- // its packages; else each counts as installing them anywhere.
205
+ // Its own actions' metadata by their directories, and its workflows by
206
+ // their paths: for where its actions install its packages, else each
207
+ // counts as installing them anywhere; and for the inputs its actions and
208
+ // reusable workflows write into their scripts, else each of a reusable
209
+ // workflow's counts as one.
205
210
  actions?: ReadonlyMap<string, string> | null
206
211
  }
207
212
 
@@ -197,15 +197,22 @@ export async function assessNpm(name, version, { oldVersion = null, fromEmpty =
197
197
  const facts = await npmPackageFacts(name, version, oldVersion, context)
198
198
  const old = facts.oldVersion
199
199
  if (old !== null && !facts.versions.has(old)) throw new NpmRiskError('version-not-found', `npm has no ${name}@${old}`)
200
- const malware = npmMalwareFinding({ takenDown: facts.takenDown, advisories: advisories ? await npmVersionAdvisories(name, version, osv, context) : [] })
201
- if (!facts.versions.has(version)) {
200
+ // Its advisories, its lasting risk and its downloads need none of each
201
+ // other, so all are asked at once, and waited on in this order, so that a
202
+ // failure fails it as the first of them failing would.
203
+ const listed = facts.versions.has(version)
204
+ const advised = settledLater(advisories ? npmVersionAdvisories(name, version, osv, context) : Promise.resolve([]))
205
+ const keeping = listed ? settledLater(lasting(name, version, old, facts, context, fromEmpty)) : null
206
+ const counting = listed && downloads && popularity ? settledLater(npmDownloads(name, context)) : null
207
+ const malware = npmMalwareFinding({ takenDown: facts.takenDown, advisories: await advised })
208
+ if (!listed) {
202
209
  if (!malware) throw new NpmRiskError('version-not-found', `npm has no ${name}@${version}`)
203
210
  return { name, version, oldVersion: null, published: facts.removed.get(version), removed: true, findings: [malware], files: {}, dependencies: { added: [], changed: [] },
204
211
  score: npmRiskScore([malware]) }
205
212
  }
206
213
  const { published, manifest } = facts.versions.get(version)
207
- const kept = await lasting(name, version, old, facts, context, fromEmpty)
208
- const counted = downloads && popularity ? await npmDownloads(name, context) : null
214
+ const kept = await keeping
215
+ const counted = await counting
209
216
  const own = npmVersionFindings({ deprecated: facts.deprecated[version], published, minimumReleaseAge, now, known: Boolean(manifest.knownAccounts), scope: npmKnownScope(name),
210
217
  provenance: Boolean(manifest.provenance), downloads: counted })
211
218
  const was = old === null ? null : facts.versions.get(old).published
@@ -225,6 +232,10 @@ export async function assessNpm(name, version, { oldVersion = null, fromEmpty =
225
232
  // A dependency's key among a version's, by its kind and name.
226
233
  const keyOf = row => `${row.kind ?? ''}\0${row.name}`
227
234
 
235
+ // `promise`, waited on later: its failure, if it fails before then, not
236
+ // thrown as unhandled meanwhile.
237
+ const settledLater = promise => { promise.catch(() => {}); return promise }
238
+
228
239
  // The keys of the dependencies a version holds that nothing of it imports
229
240
  // (`kept`, npmLastingRisk's: its `held-unused-deps`), those risk counts
230
241
  // (dependencies.js npmUnusedDependencyCounted).
@@ -87,6 +87,11 @@ async function kept({ cache }, type, key, ask, fresh = () => true) {
87
87
  return value
88
88
  }
89
89
 
90
+ // What is kept under each `[type, key]` of `entries`, in their order, null
91
+ // where nothing is: in one read where the cache reads records of any types at
92
+ // once (`readEach`), else one a key.
93
+ const readEach = (cache, entries) => cache.readEach ? cache.readEach(entries) : Promise.all(entries.map(([type, key]) => cache.read(type, key)))
94
+
90
95
  // An answer's body as JSON, read no further than `limit` bytes.
91
96
  async function jsonOf(res, limit, what) {
92
97
  const tooLarge = () => new NpmRiskError('upstream-too-large', `npm's answer for ${what} is over ${limit} bytes`)
@@ -169,7 +174,7 @@ export async function npmPackageFacts(name, version, oldVersion, context) {
169
174
  const given = oldVersion === true ? fresh(picked) ? picked.oldVersion : undefined : oldVersion
170
175
  if (cache && given !== undefined) {
171
176
  const wanted = given === null ? [version] : [version, given]
172
- const [facts, pkg] = await Promise.all([Promise.all(wanted.map(at => cache.read('risk/npm/versions', `${name}@${at}`))), cache.read('risk/npm/packages', name)])
177
+ const [pkg, ...facts] = await readEach(cache, [['risk/npm/packages', name], ...wanted.map(at => ['risk/npm/versions', `${name}@${at}`])])
173
178
  const gone = at => Object.hasOwn(pkg.removed, at)
174
179
  if (fresh(pkg) && 'removed' in pkg && wanted.every((at, i) => facts[i] || gone(at))) {
175
180
  return { versions: new Map(wanted.flatMap((at, i) => gone(at) ? [] : [[at, facts[i]]])), removed: new Map(wanted.filter(gone).map(at => [at, pkg.removed[at]])),
package/server-e2e/app.ts CHANGED
@@ -102,11 +102,12 @@ import { createBusReceiver } from './bus-receiver.ts'
102
102
  import { e2eStorageLines } from '../server-common/storage-log.ts'
103
103
 
104
104
  // Each embedding gets its own stores, transports, timers and shutdown state.
105
- export async function createE2eApp(config = loadConfig()) {
106
- return await initializeApp(rollback => assembleE2eApp(config, rollback))
105
+ // `ui: false` serves only the API (see ./http.ts).
106
+ export async function createE2eApp(config = loadConfig(), { ui = true }: { ui?: boolean } = {}) {
107
+ return await initializeApp(rollback => assembleE2eApp(config, ui, rollback))
107
108
  }
108
109
 
109
- async function assembleE2eApp(config: Config, rollback: AsyncDisposableStack) {
110
+ async function assembleE2eApp(config: Config, ui: boolean, rollback: AsyncDisposableStack) {
110
111
  // All external inputs (env vars + optional config.json) are parsed
111
112
  // and validated in ./config.ts; destructure into the uppercase names
112
113
  // the rest of this module uses.
@@ -388,7 +389,7 @@ async function assembleE2eApp(config: Config, rollback: AsyncDisposableStack) {
388
389
  // below.
389
390
  const httpServer = createHttpServer({
390
391
  wss, restDeps: objstoreRestDeps, sseServer, serverInfo: SERVER_INFO, isOriginAllowed,
391
- isShuttingDown, track, handleSaveRest,
392
+ isShuttingDown, track, handleSaveRest, ui,
392
393
  restPutIdleTimeoutMs: REST_PUT_IDLE_TIMEOUT_MS, debug: DEBUG,
393
394
  })
394
395
  const requestHandler = httpServer.listeners('request')[0]!
@@ -59,6 +59,9 @@ export type HttpServerDeps = {
59
59
  handleSaveRest: (req: HttpRequest, res: ServerResponse) => Promise<void>
60
60
  restPutIdleTimeoutMs: number
61
61
  debug: boolean
62
+ // Serve the static UI bundle (default). False serves only the API and
63
+ // never reads `out/`, for hosts that serve the UI themselves.
64
+ ui?: boolean
62
65
  }
63
66
 
64
67
  // `POST /api/sync/save` dispatch. Returns the in-flight handler promise when
@@ -114,12 +117,12 @@ function dispatchSaveRest(
114
117
  }
115
118
 
116
119
  export function createHttpServer(deps: HttpServerDeps): Server {
117
- const { wss, restDeps, sseServer, serverInfo, isOriginAllowed, isShuttingDown, track, handleSaveRest, restPutIdleTimeoutMs, debug } = deps
120
+ const { wss, restDeps, sseServer, serverInfo, isOriginAllowed, isShuttingDown, track, handleSaveRest, restPutIdleTimeoutMs, debug, ui = true } = deps
118
121
  // Static-file plane (see ./static.ts). The directory is the
119
122
  // `build.js build` output sibling to this file; the loader handles
120
123
  // enumeration, pre-compression, and ETag derivation. Plugged in after
121
124
  // the `/api/objstore/...` REST branch.
122
- const handleStatic = loadStatic(fileURLToPath(new URL('../out', import.meta.url)), serverInfo.deepviewScanServer ?? null)
125
+ const handleStatic = ui ? loadStatic(fileURLToPath(new URL('../out', import.meta.url)), serverInfo.deepviewScanServer ?? null) : () => false
123
126
 
124
127
  const httpServer = createServer((req: HttpRequest, res: ServerResponse) => {
125
128
  // Static mode probe: GET /api/config → this server's `server-info` as JSON.
@@ -176,12 +176,21 @@ export function activityMethods(db: ManagedSql): ActivityStore {
176
176
  const stmts = contexts == null ? admin : manager
177
177
  const scope = contexts == null ? {} : { contexts: JSON.stringify(contexts), userId: userId ?? '' }
178
178
  const params = { ...scope, kind, query, repo, actor }
179
- const { total } = (await stmts.count.get(params)) as { total: number }
180
- const currentPage = Math.min(page, Math.max(1, Math.ceil(total / limit)))
181
- const history = (await stmts.rows.all({ ...params, limit, offset: (currentPage - 1) * limit })) as ActivityEntry[]
182
- const filters: ActivityFilters = {
183
- repos: ((await stmts.repos.all(scope)) as { repo: string }[]).map(row => row.repo),
184
- users: ((await stmts.users.all(scope)) as ActivityFilters['users']).map(row => ({ id: row.id, login: row.login, detail: row.detail })),
179
+ const read = async (at: number) => (await stmts.rows.all({ ...params, limit, offset: (at - 1) * limit })) as (ActivityEntry & { total: number | string })[]
180
+ // A page's rows carry the total. Only a page past the last has none to
181
+ // carry it: it is counted, and the last page read in its place.
182
+ let currentPage = page, rows = await read(page)
183
+ let total = rows.length > 0 ? Number(rows[0]!.total) : 0
184
+ if (rows.length === 0 && page > 1) {
185
+ total = Number(((await stmts.count.get(params)) as { total: number | string }).total)
186
+ currentPage = Math.min(page, Math.max(1, Math.ceil(total / limit)))
187
+ if (currentPage !== page) rows = total === 0 ? [] : await read(currentPage)
188
+ }
189
+ const history = rows.map(({ total: _total, ...entry }) => entry)
190
+ const filters: ActivityFilters = { repos: [], users: [] }
191
+ for (const row of (await stmts.filters.all(scope)) as { facet: string; id: string; login: string; detail: string | null }[]) {
192
+ if (row.facet === 'repo') filters.repos.push(row.id)
193
+ else filters.users.push({ id: row.id, login: row.login, detail: row.detail })
185
194
  }
186
195
  return { history, total, page: currentPage, limit, filters }
187
196
  },
@@ -199,17 +208,21 @@ function activityStatements(db: ManagedSql, source: string) {
199
208
  AND (:repo = '' OR repo = :repo) AND (:actor = '' OR actorKey = :actor)
200
209
  AND (:query = '' OR instr(lower(coalesce(actor, '') || ' ' || action || ' ' ||
201
210
  coalesce(repo, '') || ' ' || coalesce(report, '') || ' ' || coalesce(finding, '')), lower(:query)) > 0)`
211
+ // A page's rows each carry the total, so the source is read once for both;
212
+ // the selector choices, repositories then users, once between them.
202
213
  return {
203
214
  count: db.prepare(`SELECT count(*) AS total FROM (${filtered}) AS filtered`),
204
- rows: db.prepare(`${filtered} ORDER BY at DESC,
215
+ rows: db.prepare(`SELECT filtered.*, count(*) OVER () AS total FROM (${filtered}) AS filtered ORDER BY at DESC,
205
216
  CASE WHEN kind = 'triage' THEN CAST(substr(id, instr(id, ':') + 1) AS INTEGER) ELSE 0 END DESC,
206
217
  id DESC LIMIT :limit OFFSET :offset`),
207
- repos: db.prepare(`${activity} SELECT DISTINCT repo FROM activity WHERE repo IS NOT NULL AND repo <> '' ORDER BY repo`),
208
- users: db.prepare(`${activity} SELECT a.actorKey AS id, COALESCE(u.login, a.actor, a.actorId) AS login,
209
- CASE WHEN a.actorId IS NULL THEN 'Legacy actor' ELSE NULL END AS detail FROM (
218
+ filters: db.prepare(`${activity} SELECT 'repo' AS facet, repo AS id, repo AS login, NULL AS detail, repo AS sort, '' AS tiebreak
219
+ FROM (SELECT DISTINCT repo FROM activity WHERE repo IS NOT NULL AND repo <> '') AS repos
220
+ UNION ALL SELECT 'user' AS facet, a.actorKey AS id, COALESCE(u.login, a.actor, a.actorId) AS login,
221
+ CASE WHEN a.actorId IS NULL THEN 'Legacy actor' ELSE NULL END AS detail, COALESCE(u.login, a.actor, a.actorId) AS sort, a.actorKey AS tiebreak FROM (
210
222
  SELECT actorKey, actorId, actor, ROW_NUMBER() OVER (PARTITION BY actorKey ORDER BY at DESC, id DESC) AS position
211
223
  FROM activity WHERE actorKey IS NOT NULL AND actorKey <> 'legacy:'
212
- ) a LEFT JOIN managed_user u ON u.id = a.actorId WHERE a.position = 1 ORDER BY login, a.actorKey`),
224
+ ) a LEFT JOIN managed_user u ON u.id = a.actorId WHERE a.position = 1
225
+ ORDER BY facet, sort, tiebreak`),
213
226
  }
214
227
  }
215
228
 
@@ -2,29 +2,18 @@
2
2
  // tab's, server-managed/repository-agents.ts, whose marks the Commits tab
3
3
  // reads on commits too, repository-history.ts).
4
4
  import type { AppAgent, AppAgentMarks } from '../common/managed/agent-tasks.ts'
5
- import { GITHUB_AGENTS } from '../common/managed/github-agents.ts'
5
+ import { GITHUB_AGENTS, type GithubAgent } from '../common/managed/github-agents.ts'
6
6
 
7
7
  // The agents known by their GitHub Apps or accounts, beside Claude and
8
- // Codex: each its id and name, its avatar (its mark, served by
9
- // /api/avatar/<avatar>, for these alone: a GitHub App's, `app-<id>`, or
10
- // account's, `user-<id>`), the fingerprint it leaves as a line of its pull
11
- // requests' bodies and its commits' messages, the account its work is
12
- // authored by (its id and the logins GitHub names it by, told by both), and
13
- // its branches' prefix. Their work may carry Claude's or
14
- // Codex's marks too, those its tools. Copilot's coding agent opens its pull
15
- // requests and authors its commits as its account, Copilot (198982749), on
16
- // `copilot/` branches; Dependabot, its GitHub App (29110), opens its pull
17
- // requests as its account, `dependabot` (49699333, `dependabot[bot]` to
18
- // GitHub's REST API and in its address), on `dependabot/<ecosystem>/…`
19
- // branches. Their accounts and marks are those the app draws their work by
20
- // wherever it shows it (common/managed/github-agents.ts): Copilot's its own,
21
- // the app's to draw, no avatar.
22
- const { copilot, dependabot } = GITHUB_AGENTS
23
- export const APP_AGENTS: readonly (AppAgent & AppAgentMarks)[] = [
24
- { id: 'neo', name: 'Neo', avatar: 'app-3115942', fingerprint: '🤖 Generated with [Neo](', prefix: 'neo/' },
25
- { id: 'copilot', name: copilot.name, account: copilot.accounts[0], prefix: 'copilot/' },
26
- { id: 'dependabot', name: dependabot.name, avatar: dependabot.avatar, account: dependabot.accounts[0], prefix: 'dependabot/' },
27
- ]
8
+ // Codex: those of GITHUB_AGENTS with branches of their own, in its order
9
+ // (common/managed/github-agents.ts, where each is told), each its id and
10
+ // name, its avatar (its mark, served by /api/avatar/<avatar>, a GitHub
11
+ // App's, `app-<id>`), the fingerprint it leaves, the account its work is
12
+ // authored by (its id and the logins GitHub names it by, told by both, or by
13
+ // its id alone where it names none), and its branches' prefix.
14
+ export const APP_AGENTS: readonly (AppAgent & AppAgentMarks)[] = (Object.entries(GITHUB_AGENTS) as [string, GithubAgent][])
15
+ .flatMap(([id, { name, avatar, accounts: [account], prefix, fingerprint }]) => prefix ? [{ id, name, prefix, ...avatar ? { avatar } : {},
16
+ ...account?.id === undefined ? {} : { account: { id: account.id, logins: account.logins } }, ...fingerprint ? { fingerprint } : {} }] : [])
28
17
  // The prefixes of the branches agents work on: Claude Code's and Codex's,
29
18
  // and the app agents'.
30
19
  export const AGENT_BRANCH_PREFIXES = ['claude/', 'codex/', ...APP_AGENTS.map(agent => agent.prefix)]
@@ -96,13 +96,17 @@ export function buildErrorCode(error, message) {
96
96
  if (parentPort && workerData?.type === 'managed-bundle-build') {
97
97
  // Send diagnostics to the request's parent thread, which writes them before
98
98
  // returning the HTTP response. Worker stdout can be lost on termination.
99
- const send = message => {
99
+ const send = (message, transfer = []) => {
100
100
  // Node worker messages have no browser target origin.
101
101
  // oxlint-disable-next-line unicorn/require-post-message-target-origin
102
- parentPort.postMessage(message)
102
+ parentPort.postMessage(message, transfer)
103
103
  }
104
104
  try {
105
- send(await buildStasisBundle(workerData, undefined, stage => send({ type: 'progress', stage })))
105
+ const built = await buildStasisBundle(workerData, undefined, stage => send({ type: 'progress', stage }))
106
+ // The bytes handed over rather than copied, where they alone fill their
107
+ // buffer (a small one may share Node's pool, which can't be moved).
108
+ const { bytes } = built
109
+ send(built, bytes.byteOffset === 0 && bytes.byteLength === bytes.buffer.byteLength ? [bytes.buffer] : [])
106
110
  } catch (error) {
107
111
  const diagnostic = bundleBuildDiagnostic(error, workerData.token)
108
112
  const code = buildErrorCode(error, diagnostic.message)
@@ -95,7 +95,7 @@ export function githubBundleFilename(github: string, directory: string, commit:
95
95
  return `${repo}.${dir}${tail}`
96
96
  }
97
97
 
98
- export interface BuiltBundle { bytes: Uint8Array; directory: string; filename: string }
98
+ export interface BuiltBundle { bytes: Buffer; directory: string; filename: string }
99
99
  export interface BuildRequest {
100
100
  input: BundleBuildInput; github: string; token: string | null; maxBytes: number; scopes: (string | null)[]
101
101
  // Upstream's disk cache for what a build fetches (setCacheDir), or null for
@@ -174,14 +174,16 @@ export async function buildRepositoryBundle(userId: string, request: BuildReques
174
174
  const abort = () => fail(signal.reason instanceof BundleBuildError ? signal.reason.status : 499,
175
175
  signal.reason instanceof BundleBuildError ? signal.reason.code : 'build-cancelled')
176
176
  signal.addEventListener('abort', abort, { once: true })
177
- running.on('message', (message: BuiltBundle & { type?: string; stage?: BundleBuildStage; error?: string; status?: number; diagnostic?: BundleBuildDiagnostic }) => {
177
+ running.on('message', (message: Omit<BuiltBundle, 'bytes'> & { bytes: Uint8Array; type?: string; stage?: BundleBuildStage; error?: string; status?: number; diagnostic?: BundleBuildDiagnostic }) => {
178
178
  if (settled) return
179
179
  if (message.type === 'progress' && message.stage) { stage = message.stage; log('progress'); return }
180
180
  if (message.error) { fail(message.status ?? 422, message.error, { diagnostic: message.diagnostic }); return }
181
181
  settled = true
182
182
  cleanup()
183
183
  log('completed', { byteSize: message.bytes.length })
184
- resolve({ bytes: Buffer.from(message.bytes), directory: message.directory, filename: message.filename })
184
+ // A view of the bytes the worker handed over, not a copy of them.
185
+ const { bytes } = message
186
+ resolve({ bytes: Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength), directory: message.directory, filename: message.filename })
185
187
  })
186
188
  running.once('error', error => fail(422, 'build-failed', { diagnostic: bundleBuildDiagnostic(error, request.token) }))
187
189
  running.once('exit', exitCode => fail(422, 'build-failed', { exitCode }))
@@ -24,11 +24,17 @@ export type BundleCacheRecord = Pick<ManagedBundle, 'id' | 'integrity' | 'filena
24
24
  export interface BundleSummary { files: number; codeFiles: number; lines: number; commit?: string; stasisVersion?: number; versionedPackages?: number }
25
25
 
26
26
  export async function readBundleDetails(record: BundleCacheRecord, store: BundleStore) {
27
+ return (await readSizedBundleDetails(record, store))?.details ?? null
28
+ }
29
+
30
+ // As readBundleDetails, with the bytes the bundle decoded to: about what
31
+ // holding its details takes.
32
+ export async function readSizedBundleDetails(record: BundleCacheRecord, store: BundleStore) {
27
33
  const bytes = await store.get(record.id, record.kind)
28
34
  if (!bytes) return null
29
35
  const decoded = record.kind === 'stasis' || record.kind === 'sourcemap' ? await decompress(bytes, { maxOutputLength: MAX_DECODED_BYTES }) : bytes
30
36
  if (decoded.length > MAX_DECODED_BYTES) throw new Error('Decoded bundle too large')
31
- return parseBundleContents(decodeUtf8(decoded), { integrity: record.integrity, kind: record.kind, size: record.byteSize })
37
+ return { details: parseBundleContents(decodeUtf8(decoded), { integrity: record.integrity, kind: record.kind, size: record.byteSize }), decodedBytes: decoded.length }
32
38
  }
33
39
 
34
40
  export interface BundleCacheStorage {
@@ -105,16 +111,22 @@ async function build(record: BundleCacheRecord, storage: BundleCacheStorage, db:
105
111
  const metadata = { ...await createBundleMetadata(details), id: record.id, filename: record.filename }
106
112
  const body = await encodeBrotli(Buffer.from(JSON.stringify(metadata)))
107
113
  if (!(await db.getBundle(record.id))) throw new Error('Bundle deleted')
108
- await storage.put(record.id, filename, body)
109
- if (record.kind === 'stasis') await storage.put(record.id, packagesFilename, encodePackageInventory(details))
110
114
  const summary = bundleSummary(details, metadata)
111
- await storage.put(record.id, SUMMARY_FILENAME, Buffer.from(JSON.stringify(summary)))
115
+ // Written together, each settled before the row is looked at again: one
116
+ // failing leaves the others, which a later open builds over.
117
+ const written = await Promise.allSettled([
118
+ storage.put(record.id, filename, body),
119
+ ...record.kind === 'stasis' ? [storage.put(record.id, packagesFilename, encodePackageInventory(details))] : [],
120
+ storage.put(record.id, SUMMARY_FILENAME, Buffer.from(JSON.stringify(summary))),
121
+ ])
112
122
  // A different instance may have deleted the row while these writes ran.
113
123
  // Reconcile after publishing so its cleanup cannot be undone by us.
114
124
  if (!(await db.getBundle(record.id))) {
115
125
  await storage.delete(record.id)
116
126
  throw new Error('Bundle deleted')
117
127
  }
128
+ const failed = written.find(result => result.status === 'rejected')
129
+ if (failed) throw failed.reason
118
130
  return summary
119
131
  }
120
132
 
@@ -109,10 +109,9 @@ export async function bundleFilePrefix(bytes: Buffer): Promise<string> {
109
109
  }
110
110
 
111
111
  // Extract the bundle integrities a report declares (its top-level
112
- // `bundleHashes`). Non-JSON reports (markdown / CSV) or a missing field → [].
113
- export function reportBundleHashes(bytes: Buffer): string[] {
114
- let data: unknown
115
- try { data = JSON.parse(bytes.toString('utf8')) } catch { return [] }
112
+ // `bundleHashes`), from the JSON it parsed to (readManagedReport's, as is).
113
+ // Non-JSON reports (markdown / CSV, given as null) or a missing field → [].
114
+ export function reportBundleHashes(data: unknown): string[] {
116
115
  if (data == null || typeof data !== 'object') return []
117
116
  const hashes = (data as { bundleHashes?: unknown }).bundleHashes
118
117
  if (!Array.isArray(hashes)) return []
@@ -10,13 +10,13 @@ import { loadManagedConfig } from './config.ts'
10
10
  import { LOGIN_PATH } from './github-oauth.ts'
11
11
  import { createManagedApp, logManagedStartup } from './index.ts'
12
12
 
13
- export async function init(mode: 'managed+e2e' | 'e2e+managed'): Promise<Server> {
13
+ export async function init(mode: 'managed+e2e' | 'e2e+managed', { ui = true }: { ui?: boolean } = {}): Promise<Server> {
14
14
  const config = loadManagedConfig({ combined: true })
15
15
  const e2eConfig = loadConfig()
16
16
  if (!e2eConfig.neonUrl && !config.neonUrl && config.dbPath !== ':memory:' && resolve(config.dbPath) === resolve(e2eConfig.dbPath)) {
17
17
  throw new Error('MANAGED_DB_PATH must differ from e2e DB_PATH in combined mode.')
18
18
  }
19
- const e2e = await createE2eApp(e2eConfig)
19
+ const e2e = await createE2eApp(e2eConfig, { ui })
20
20
  // Replace the assembled server's single
21
21
  // request listener; adding a second would write two responses per request.
22
22
  const handlers = e2e.httpServer.listeners('request')
@@ -25,6 +25,7 @@ export async function init(mode: 'managed+e2e' | 'e2e+managed'): Promise<Server>
25
25
  const managed = await createManagedApp(config, {
26
26
  next: e2e.handleRequest,
27
27
  isShuttingDown: e2e.isShuttingDown,
28
+ ui,
28
29
  serverInfo: {
29
30
  mode, managed: { loginPath: LOGIN_PATH, cookieName: config.sessionCookieName },
30
31
  ...(e2eConfig.deepviewScanServer ? { deepviewScanServer: e2eConfig.deepviewScanServer } : {}),
@@ -10,15 +10,14 @@ export const MAX_COAUTHORS = 3
10
10
  // The rules co-authors are read by, the agents' addresses and names among
11
11
  // them: what was kept read by others (history pages, squashed pull requests)
12
12
  // read again by these, once raised.
13
- export const COAUTHORS_READ = 2
13
+ export const COAUTHORS_READ = 3
14
14
  const MAX_NAME = 256
15
15
  const textOf = (value: unknown, max: number) => typeof value === 'string' && value.trim() ? value.trim().slice(0, max) : null
16
16
 
17
17
  // One who made a commit beside its author: by name and account, as its
18
- // author is, and the agent it is where known, one GitHub knows by its account
19
- // or its commits' address or name (common/managed/github-agents.ts) or one
20
- // the server knows by its app (app-agents.ts), with its avatar.
21
- export interface Coauthor { name: string | null; login: string | null; id: number | null; agent?: string; avatar?: string }
18
+ // author is, and the agent it is where known (common/managed/github-agents.ts),
19
+ // by its account, its commits' address or name, or the marks it leaves.
20
+ export interface Coauthor { name: string | null; login: string | null; id: number | null; agent?: string }
22
21
 
23
22
  // `Co-authored-by: <name> <address>` trailers, GitHub matching the key
24
23
  // regardless of case; Cursor's agents' links, as their work carries them;
@@ -75,9 +74,6 @@ export function coauthorsOf(message: unknown, author: Maker, listed: Maker[] | n
75
74
  const { login, id } = agent && maker.account.login && githubAgent(maker.account.login, maker.account.id) !== agent ? { login: null, id: null } : maker.account
76
75
  add({ name: textOf(maker.name, MAX_NAME), login, id, ...agent ? { agent } : {} }, maker.email)
77
76
  }
78
- for (const agent of marked) {
79
- const app = APP_AGENTS.find(known => known.id === agent && !Object.hasOwn(GITHUB_AGENTS, known.id))
80
- add({ name: app?.name ?? GITHUB_AGENTS[agent as GithubAgentId].name, login: null, id: null, agent, ...app?.avatar ? { avatar: app.avatar } : {} })
81
- }
77
+ for (const agent of marked) add({ name: GITHUB_AGENTS[agent as GithubAgentId].name, login: null, id: null, agent })
82
78
  return coauthors
83
79
  }
@@ -4,40 +4,50 @@
4
4
  // read as a pull request's are (pull-dependencies.ts readLockfile, kept in
5
5
  // this instance by the commit and path), as many as MAX_LOCKFILES of those,
6
6
  // each's versions (`lockfiles`), and those its reader refuses, of Bun's, or
7
- // past MAX_LOCKFILES (`unread`); and those of some of them (lockedIn), each
8
- // version once by what its reviews are held to (lockedWith), dev only where
9
- // it is wherever it is locked. GitHub having none of a lockfile it lists
10
- // there fails the whole, as it would for any file.
7
+ // past MAX_LOCKFILES (`unread`), as many read at once as READ_AT_ONCE; and
8
+ // those of some of them (lockedIn), each version once by what its reviews
9
+ // are held to (lockedWith), dev only where it is wherever it is locked.
10
+ // GitHub having none of a lockfile it lists there fails the whole, as it
11
+ // would for any file.
11
12
  import { createClient } from '@preventive/upstream/github.js'
12
13
  import { upstreamGithub } from './github-errors.ts'
13
14
  import { GithubApiError } from './github-app.ts'
14
15
  import type { LockedPackage } from './locked-packages.ts'
15
16
  import { type Lru, lru } from './lru.ts'
16
- import { lockedWith, lockfileFormat, readLockfile } from './pull-dependencies.ts'
17
+ import { eachOf, lockedWith, lockfileFormat, lockfileHeld, readLockfile } from './pull-dependencies.ts'
17
18
 
18
19
  export const MAX_LOCKFILES = 100
20
+ const READ_AT_ONCE = 4
19
21
 
20
22
  export interface CommitLockfiles { lockfiles: { path: string; packages: LockedPackage[] }[]; unread: string[] }
23
+ // Set where GitHub was asked for what a read reads (`github`), not all of it
24
+ // held in this instance.
25
+ export interface GithubAsked { github: boolean }
21
26
 
22
27
  // The lockfiles at commit `sha` its review reads, by their paths, each with
23
- // its format (pull-dependencies.ts lockfileFormat), kept by the commit.
28
+ // its format (pull-dependencies.ts lockfileFormat), kept by the commit;
29
+ // `asked` told where GitHub was.
24
30
  const listings: Lru<{ path: string; format: NonNullable<ReturnType<typeof lockfileFormat>> }[]> = lru(20_000, listed => listed.length + 1)
25
- export async function lockfilesAt(fullName: string, token: string, sha: string) {
31
+ export async function lockfilesAt(fullName: string, token: string, sha: string, asked?: GithubAsked) {
26
32
  const key = `${fullName}\0${sha}`
27
33
  const known = listings.get(key)
28
34
  if (known) return known
35
+ if (asked) asked.github = true
29
36
  const { lockfiles } = await upstreamGithub(() => createClient({ token, userAgent: 'deepview-triage' }).findRepoLockfiles({ repo: fullName, sha }))
30
37
  const listed = lockfiles.flatMap(({ path }) => { const format = lockfileFormat(path); return format ? [{ path, format }] : [] })
31
38
  listings.set(key, listed)
32
39
  return listed
33
40
  }
34
41
 
35
- export async function lockedAt(fullName: string, token: string, sha: string, held: (path: string) => boolean = () => true): Promise<CommitLockfiles> {
36
- const listed = (await lockfilesAt(fullName, token, sha)).filter(({ path }) => held(path))
42
+ export async function lockedAt(fullName: string, token: string, sha: string, held: (path: string) => boolean = () => true, asked?: GithubAsked): Promise<CommitLockfiles> {
43
+ const listed = (await lockfilesAt(fullName, token, sha, asked)).filter(({ path }) => held(path))
37
44
  const unread = listed.slice(MAX_LOCKFILES).map(({ path }) => path)
38
45
  const lockfiles: CommitLockfiles['lockfiles'] = []
39
- for (const lockfile of listed.slice(0, MAX_LOCKFILES)) {
40
- const read = await readLockfile(fullName, token, sha, lockfile)
46
+ const reading = listed.slice(0, MAX_LOCKFILES)
47
+ if (asked && reading.some(lockfile => !lockfileHeld(fullName, sha, lockfile))) asked.github = true
48
+ const reads = await eachOf(reading, READ_AT_ONCE, lockfile => readLockfile(fullName, token, sha, lockfile))
49
+ for (const [at, lockfile] of reading.entries()) {
50
+ const read = reads[at]!
41
51
  if ('error' in read && read.error === 'github-not-found') throw new GithubApiError(404, 'github-not-found')
42
52
  if ('error' in read) unread.push(lockfile.path)
43
53
  else lockfiles.push({ path: lockfile.path, packages: read.packages })
@@ -42,6 +42,11 @@ export interface ManagedConfig {
42
42
  cookieSecure: boolean
43
43
  sessionCookieName: string
44
44
  sessionTtlMs: number
45
+ // How long GitHub's answer on a user's access to a repository (its
46
+ // visibility, their permission) is kept for that user and repository
47
+ // (repository-browser.ts), asked again past it; 0 asks on every request
48
+ // (MANAGED_REPOSITORY_ACCESS_CACHE_MS, five minutes by default).
49
+ repositoryAccessCacheMs: number
45
50
  // Optional installation credentials for the same GitHub App as login.
46
51
  // Repository permissions are approved when connecting repositories.
47
52
  githubAppId: string | null
@@ -140,6 +145,7 @@ export function loadManagedConfig({ combined = false } = {}): ManagedConfig {
140
145
  cookieSecure,
141
146
  sessionCookieName,
142
147
  sessionTtlMs: intEnv('SESSION_TTL_MS', 1_209_600_000, 60_000, 7_776_000_000),
148
+ repositoryAccessCacheMs: intEnv('MANAGED_REPOSITORY_ACCESS_CACHE_MS', 300_000, 0, 3_600_000),
143
149
  githubAppId: env['GITHUB_APP_ID'] ?? null,
144
150
  githubAppPrivateKey: normalizePem(env['GITHUB_APP_PRIVATE_KEY']),
145
151
  githubAppSlug: env['GITHUB_APP_SLUG'] ?? null,