@preventive/triage 1.0.0-alpha.32 → 1.0.0-alpha.34
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/client/linked-findings.js +5 -4
- package/common/bundle-metadata.js +4 -3
- package/common/bundle-source-links.d.ts +2 -1
- package/common/bundle-source-links.js +133 -6
- package/common/bundle-sources.d.ts +2 -0
- package/common/bundle-sources.js +2 -4
- package/common/managed/repository-alias.ts +65 -0
- package/common/managed/routes.js +20 -1
- package/common/managed/triage.ts +2 -2
- package/out/client-managed-import.js +51 -46
- package/out/client-managed.js +252 -72
- package/out/client-sync.js +14 -14
- package/out/graph.js +11 -11
- package/out/index.html +1 -0
- package/out/terminal.js +169 -169
- package/out/view.css +1 -1
- package/out/view.js +417 -201
- package/package.json +23 -17
- package/server-managed/activity.ts +15 -1
- package/server-managed/brotli.ts +3 -3
- package/server-managed/bundle-advisories.ts +8 -3
- package/server-managed/bundle-advisory-inventory.ts +27 -25
- package/server-managed/bundle-build-worker.js +45 -9
- package/server-managed/bundle-build.ts +35 -11
- package/server-managed/bundle-cache.ts +1 -1
- package/server-managed/bundle.ts +17 -1
- package/server-managed/config.ts +7 -2
- package/server-managed/db-methods.ts +170 -49
- package/server-managed/db-neon.ts +59 -4
- package/server-managed/db-schema.ts +12 -3
- package/server-managed/db.ts +9 -1
- package/server-managed/github-issue-links.ts +127 -0
- package/server-managed/github-issues.ts +1 -1
- package/server-managed/http-response.ts +7 -1
- package/server-managed/http.ts +248 -80
- package/server-managed/index.ts +4 -0
- package/server-managed/link-reports.ts +158 -0
- package/server-managed/managed-issues.ts +31 -3
- package/server-managed/management.ts +17 -4
- package/server-managed/object-storage-disk.ts +7 -3
- package/server-managed/object-storage-vercel.ts +6 -2
- package/server-managed/object-storage.ts +3 -2
- package/server-managed/public-workspace.ts +5 -2
- package/server-managed/report-sources.ts +6 -5
- package/server-managed/report-store.ts +47 -0
- package/server-managed/repository-aliases.ts +91 -0
- package/server-managed/revisions.ts +2 -2
- package/server-managed/sql.ts +3 -2
- package/server-managed/storage-encryption.ts +72 -6
- package/server-managed/storage-maintenance.ts +64 -49
- package/server-managed/storage-payload.ts +11 -10
- package/server-managed/storage-stores.ts +3 -1
- package/server-managed/storage.ts +17 -0
- package/server-managed/team-feed.ts +3 -3
- package/server-managed/team-reports.ts +24 -15
- package/server-managed/upstream-cache.ts +73 -0
- package/server-managed/workspace-shares.ts +16 -12
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@preventive/triage",
|
|
3
|
-
"version": "1.0.0-alpha.
|
|
3
|
+
"version": "1.0.0-alpha.34",
|
|
4
4
|
"description": "Client & relay server for triaging of automated reports",
|
|
5
5
|
"license": "MIT OR Apache-2.0",
|
|
6
6
|
"author": {
|
|
@@ -121,10 +121,12 @@
|
|
|
121
121
|
"server-managed/github-metadata.ts",
|
|
122
122
|
"server-managed/github-issue-oauth.ts",
|
|
123
123
|
"server-managed/github-issues.ts",
|
|
124
|
+
"server-managed/github-issue-links.ts",
|
|
124
125
|
"server-managed/github-oauth.ts",
|
|
125
126
|
"server-managed/github-pulls.ts",
|
|
126
127
|
"server-managed/http-response.ts",
|
|
127
128
|
"server-managed/http.ts",
|
|
129
|
+
"server-managed/link-reports.ts",
|
|
128
130
|
"server-managed/import-triage.ts",
|
|
129
131
|
"server-managed/managed-issues.ts",
|
|
130
132
|
"server-managed/management.ts",
|
|
@@ -139,9 +141,11 @@
|
|
|
139
141
|
"server-managed/report-migration.ts",
|
|
140
142
|
"server-managed/report-query.ts",
|
|
141
143
|
"server-managed/report-response.ts",
|
|
144
|
+
"server-managed/report-store.ts",
|
|
142
145
|
"server-managed/report-sources.ts",
|
|
143
146
|
"server-managed/revisions.ts",
|
|
144
147
|
"server-managed/repository-browser.ts",
|
|
148
|
+
"server-managed/repository-aliases.ts",
|
|
145
149
|
"server-managed/repository-discovery.ts",
|
|
146
150
|
"server-managed/repository-policy.ts",
|
|
147
151
|
"server-managed/session.ts",
|
|
@@ -162,6 +166,7 @@
|
|
|
162
166
|
"server-managed/team-reports.ts",
|
|
163
167
|
"server-managed/triage-response.ts",
|
|
164
168
|
"server-managed/uploads.ts",
|
|
169
|
+
"server-managed/upstream-cache.ts",
|
|
165
170
|
"server-managed/workspace-shares.ts",
|
|
166
171
|
"server-common/database-config.ts",
|
|
167
172
|
"server-common/initialize.ts",
|
|
@@ -194,6 +199,7 @@
|
|
|
194
199
|
"common/managed/comments.ts",
|
|
195
200
|
"common/managed/permissions.ts",
|
|
196
201
|
"common/managed/report-content.ts",
|
|
202
|
+
"common/managed/repository-alias.ts",
|
|
197
203
|
"common/managed/report-filter.ts",
|
|
198
204
|
"common/managed/roles.ts",
|
|
199
205
|
"common/managed/routes.d.ts",
|
|
@@ -239,35 +245,35 @@
|
|
|
239
245
|
"server-e2e": "node server-e2e/index.ts",
|
|
240
246
|
"server-managed": "node server-managed/index.ts",
|
|
241
247
|
"server-managed-test": "node server-managed/test-server.ts",
|
|
242
|
-
"test": "node --js-base-64 --
|
|
248
|
+
"test": "node --js-base-64 --require=./tests/_resource-loader.js --experimental-test-module-mocks --test-timeout=120000 --test ./tests/*.test.js"
|
|
243
249
|
},
|
|
244
250
|
"devDependencies": {
|
|
245
|
-
"@electric-sql/pglite": "^0.5.
|
|
246
|
-
"@noble/ciphers": "^2.
|
|
247
|
-
"@preventive/diff": "^1.0.
|
|
248
|
-
"@preventive/lockfile": "1.0.0-alpha.
|
|
249
|
-
"@preventive/terminal": "2.0.0-beta.
|
|
251
|
+
"@electric-sql/pglite": "^0.5.8",
|
|
252
|
+
"@noble/ciphers": "^2.4.0",
|
|
253
|
+
"@preventive/diff": "^1.0.1",
|
|
254
|
+
"@preventive/lockfile": "1.0.0-alpha.7",
|
|
255
|
+
"@preventive/terminal": "2.0.0-beta.2",
|
|
250
256
|
"@rray/frontend": "^1.0.0",
|
|
251
|
-
"@stylistic/stylelint-plugin": "^5.3.
|
|
252
|
-
"@types/node": "^26.
|
|
253
|
-
"@types/ws": "^8.18.
|
|
257
|
+
"@stylistic/stylelint-plugin": "^5.3.1",
|
|
258
|
+
"@types/node": "^26.6.4",
|
|
259
|
+
"@types/ws": "^8.18.2",
|
|
254
260
|
"@vercel/functions": "^3.9.11",
|
|
255
261
|
"brotli": "^1.3.3",
|
|
256
262
|
"esbuild": "^0.28.2",
|
|
257
263
|
"lit": "^3.3.3",
|
|
258
264
|
"minify-html-literals": "^1.3.5",
|
|
259
|
-
"oxlint": "^1.
|
|
265
|
+
"oxlint": "^1.87.0",
|
|
260
266
|
"prismjs": "^1.30.0",
|
|
261
|
-
"stylelint": "^17.
|
|
267
|
+
"stylelint": "^17.16.0",
|
|
262
268
|
"stylelint-config-recommended": "^18.0.0",
|
|
263
269
|
"typescript": "^7.0.2"
|
|
264
270
|
},
|
|
265
271
|
"dependencies": {
|
|
266
|
-
"@exodus/stasis": "1.0.0-beta.
|
|
267
|
-
"@exodus/stasis-core": "1.0.0-beta.
|
|
268
|
-
"@preventive/report": "1.0.0-alpha.
|
|
269
|
-
"@preventive/upstream": "1.0.0-alpha.
|
|
270
|
-
"ws": "^8.
|
|
272
|
+
"@exodus/stasis": "1.0.0-beta.9",
|
|
273
|
+
"@exodus/stasis-core": "1.0.0-beta.9",
|
|
274
|
+
"@preventive/report": "1.0.0-alpha.9",
|
|
275
|
+
"@preventive/upstream": "1.0.0-alpha.12",
|
|
276
|
+
"ws": "^8.22.0"
|
|
271
277
|
},
|
|
272
278
|
"peerDependencies": {
|
|
273
279
|
"@neondatabase/serverless": "^1.0.2",
|
|
@@ -89,6 +89,7 @@ const managerSource = `SELECT ${triageFields},
|
|
|
89
89
|
LEFT JOIN managed_selected_repo p ON p.repo_id = r.repo_id
|
|
90
90
|
WHERE EXISTS (
|
|
91
91
|
SELECT 1 FROM managed_team_repo tr JOIN managed_team_user tu ON tu.team_id = tr.team_id
|
|
92
|
+
JOIN managed_team t ON t.id = tu.team_id AND t.hidden = 0
|
|
92
93
|
WHERE tu.user_id = :userId AND tr.repo_id = r.repo_id AND ${withinTeamPath('r.repo_directory')}
|
|
93
94
|
)
|
|
94
95
|
UNION ALL SELECT a.id, a.kind, a.actor, a.actor_id AS actorId,
|
|
@@ -98,6 +99,7 @@ const managerSource = `SELECT ${triageFields},
|
|
|
98
99
|
LEFT JOIN managed_selected_repo p ON p.repo_id = b.repo_id
|
|
99
100
|
WHERE a.report_id IS NULL AND EXISTS (
|
|
100
101
|
SELECT 1 FROM managed_team_repo tr JOIN managed_team_user tu ON tu.team_id = tr.team_id
|
|
102
|
+
JOIN managed_team t ON t.id = tu.team_id AND t.hidden = 0
|
|
101
103
|
WHERE tu.user_id = :userId AND tr.repo_id = b.repo_id AND ${withinTeamPath('b.repo_directory')}
|
|
102
104
|
)
|
|
103
105
|
UNION ALL SELECT a.id, a.kind, a.actor, a.actor_id AS actorId, a.action, a.repo, a.report_id AS reportId,
|
|
@@ -106,6 +108,7 @@ const managerSource = `SELECT ${triageFields},
|
|
|
106
108
|
AND NOT EXISTS (SELECT 1 FROM managed_report WHERE id = a.report_id)
|
|
107
109
|
AND NOT EXISTS (SELECT 1 FROM managed_bundle WHERE id = a.bundle_id)
|
|
108
110
|
AND EXISTS (SELECT 1 FROM managed_team_repo tr JOIN managed_team_user tu ON tu.team_id = tr.team_id
|
|
111
|
+
JOIN managed_team t ON t.id = tu.team_id AND t.hidden = 0
|
|
109
112
|
WHERE tu.user_id = :userId AND tr.repo_id = a.repo_id
|
|
110
113
|
AND ${withinTeamPath('a.repo_directory')})`
|
|
111
114
|
|
|
@@ -125,10 +128,14 @@ export function initActivityMethods(db: DatabaseSync): void {
|
|
|
125
128
|
db.exec('CREATE INDEX IF NOT EXISTS managed_activity_actor_at_idx ON managed_activity(actor_id, at)')
|
|
126
129
|
// Stable upload IDs make backfill idempotent, including after restarts.
|
|
127
130
|
// Snapshots deliberately have no cascading FKs: deletion is itself activity.
|
|
131
|
+
// CREATE TRIGGER IF NOT EXISTS keeps an older body; replace the one that
|
|
132
|
+
// predates server-built bundles so builds are not recorded as uploads.
|
|
133
|
+
const bundleTrigger = db.prepare("SELECT sql FROM sqlite_schema WHERE type = 'trigger' AND name = 'managed_bundle_activity'").get() as { sql: string } | undefined
|
|
134
|
+
if (bundleTrigger && !bundleTrigger.sql.includes('provenance')) db.exec('DROP TRIGGER managed_bundle_activity')
|
|
128
135
|
for (const type of ['report', 'bundle']) {
|
|
129
136
|
const columns = `id, kind, actor, action, repo, report_id, report, at, bundle_id, actor_id`
|
|
130
137
|
const values = (alias: string) => `'${type}-upload:' || ${alias}.id, 'upload',
|
|
131
|
-
COALESCE(${alias}.uploaded_by_login, (SELECT login FROM managed_user WHERE id = ${alias}.uploaded_by)),
|
|
138
|
+
COALESCE(${alias}.uploaded_by_login, (SELECT login FROM managed_user WHERE id = ${alias}.uploaded_by)), ${uploadAction(type, alias)},
|
|
132
139
|
(SELECT full_name FROM managed_selected_repo WHERE repo_id = ${alias}.repo_id),
|
|
133
140
|
${type === 'report' ? `${alias}.id` : 'NULL'}, ${alias}.filename, ${alias}.uploaded_at,
|
|
134
141
|
${type === 'bundle' ? `${alias}.id` : 'NULL'}, ${alias}.uploaded_by`
|
|
@@ -140,6 +147,12 @@ export function initActivityMethods(db: DatabaseSync): void {
|
|
|
140
147
|
}
|
|
141
148
|
}
|
|
142
149
|
|
|
150
|
+
// The creation entry for a stored report or bundle. Shared with the Postgres
|
|
151
|
+
// trigger function; `alias` is the trigger row (NEW) or a backfill alias.
|
|
152
|
+
export function uploadAction(type: string, alias: string): string {
|
|
153
|
+
return type === 'bundle' ? `CASE WHEN ${alias}.provenance = 'build' THEN 'built a bundle' ELSE 'uploaded a bundle' END` : `'uploaded a ${type}'`
|
|
154
|
+
}
|
|
155
|
+
|
|
143
156
|
export function activityMethods(db: ManagedSql): ActivityStore {
|
|
144
157
|
const insert = db.prepare(`INSERT INTO managed_activity (id, kind, actor, action, repo, report_id, report, at, bundle_id, repo_id, repo_directory, actor_id)
|
|
145
158
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
|
|
@@ -148,6 +161,7 @@ export function activityMethods(db: ManagedSql): ActivityStore {
|
|
|
148
161
|
const reports = db.prepare(`SELECT r.id AS reportId, r.filename AS report, p.full_name AS repo
|
|
149
162
|
FROM managed_report r JOIN managed_selected_repo p ON p.repo_id = r.repo_id
|
|
150
163
|
WHERE EXISTS (SELECT 1 FROM managed_team_repo tr JOIN managed_team_user tu ON tu.team_id = tr.team_id
|
|
164
|
+
JOIN managed_team t ON t.id = tu.team_id AND t.hidden = 0
|
|
151
165
|
WHERE tu.user_id = ? AND tr.repo_id = r.repo_id AND ${withinTeamPath('r.repo_directory')})
|
|
152
166
|
ORDER BY r.uploaded_at DESC, r.id`)
|
|
153
167
|
return {
|
package/server-managed/brotli.ts
CHANGED
|
@@ -4,7 +4,7 @@ import type { Buffer } from 'node:buffer'
|
|
|
4
4
|
|
|
5
5
|
const compress = promisify(brotliCompress)
|
|
6
6
|
|
|
7
|
-
//
|
|
8
|
-
export function encodeBrotli(bytes: Uint8Array): Promise<Buffer> {
|
|
9
|
-
return compress(bytes, { params: { [constants.BROTLI_PARAM_QUALITY]:
|
|
7
|
+
// Bundles and cold caches use quality 4; stored reports use quality 9.
|
|
8
|
+
export function encodeBrotli(bytes: Uint8Array, quality = 4): Promise<Buffer> {
|
|
9
|
+
return compress(bytes, { params: { [constants.BROTLI_PARAM_QUALITY]: quality } })
|
|
10
10
|
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type Advisory, type Package, advisories } from '@preventive/upstream/advisories.js'
|
|
1
|
+
import { type Advisory, type CacheStore, type Package, advisories } from '@preventive/upstream/advisories.js'
|
|
2
2
|
import { type Client, HttpError, createClient } from '@preventive/upstream/github.js'
|
|
3
3
|
|
|
4
4
|
export const ADVISORIES_TIMEOUT_MS = 30_000
|
|
@@ -26,8 +26,11 @@ function advisoryGithubClient(token: string | null, signal: AbortSignal): Client
|
|
|
26
26
|
// The managed caller supplies its user's token when available; public shares
|
|
27
27
|
// remain anonymous. Upstream sends this credential only to GitHub and returns
|
|
28
28
|
// published advisories, matching versions across npm, OSV and repository rows.
|
|
29
|
+
// `cache` keeps each repository's listing (upstream-cache.ts) across audits.
|
|
29
30
|
export async function fetchBundleAdvisories(packages: Package[], signal: AbortSignal,
|
|
30
|
-
{ debug = false, repoAdvisories = false,
|
|
31
|
+
{ debug = false, repoAdvisories = false, details = false, githubToken = null, cache }: {
|
|
32
|
+
debug?: boolean; repoAdvisories?: boolean; details?: boolean; githubToken?: string | null; cache?: CacheStore | undefined
|
|
33
|
+
} = {}): Promise<
|
|
31
34
|
{ status: 200; body: Advisory[] } | { status: 502; body: { error: string } }
|
|
32
35
|
> {
|
|
33
36
|
let onAbort: (() => void) | undefined
|
|
@@ -40,8 +43,10 @@ export async function fetchBundleAdvisories(packages: Package[], signal: AbortSi
|
|
|
40
43
|
onAbort = () => reject(signal.reason)
|
|
41
44
|
signal.addEventListener('abort', onAbort, { once: true })
|
|
42
45
|
})
|
|
46
|
+
// Kept in the caller's store, or without one in upstream's disk cache,
|
|
47
|
+
// where the server set one (auditCache).
|
|
43
48
|
const result = await Promise.race([
|
|
44
|
-
advisories(packages, { github: advisoryGithubClient(githubToken, signal), repoAdvisories }), deadline,
|
|
49
|
+
advisories(packages, { github: advisoryGithubClient(githubToken, signal), repoAdvisories, details, ...(cache && { cache }) }), deadline,
|
|
45
50
|
])
|
|
46
51
|
return { status: 200, body: result }
|
|
47
52
|
} catch (error) {
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
+
import { isEvidenceFile } from '@exodus/stasis/audit-corrections'
|
|
2
|
+
import { packageRepo } from '@exodus/stasis-core/bundle-util'
|
|
1
3
|
import type { Package } from '@preventive/upstream/advisories.js'
|
|
2
|
-
import { satisfies, valid } from '@preventive/upstream/semver.js'
|
|
3
4
|
import type { BundleDetails } from '../common/bundle-metadata.js'
|
|
4
5
|
|
|
5
6
|
export interface SkippedAdvisoryPackage {
|
|
@@ -9,25 +10,14 @@ export interface SkippedAdvisoryPackage {
|
|
|
9
10
|
because: string
|
|
10
11
|
}
|
|
11
12
|
export interface BundleAdvisoryInventory { packages: Package[]; skipped: SkippedAdvisoryPackage[] }
|
|
12
|
-
interface BundleModule { ecosystem?: string; name?: string; version?: string; files: Record<string, unknown> }
|
|
13
|
+
interface BundleModule { ecosystem?: string; name?: string; version?: string; repo?: { github?: Package['github'] }; files: Record<string, unknown> }
|
|
13
14
|
|
|
14
|
-
//
|
|
15
|
-
//
|
|
16
|
-
//
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
['npm', ['package.json']],
|
|
21
|
-
['cargo', cargoManifests], ['cargo-git', cargoManifests], ['cargo-unknown', cargoManifests],
|
|
22
|
-
['composer', ['composer.json', 'composer.lock']],
|
|
23
|
-
['soldeer', solidityManifests], ['github', solidityManifests],
|
|
24
|
-
])
|
|
25
|
-
const browserStubRanges = new Map([['ws', '<=8.21.1'], ['node-fetch', '<=2.7.0']])
|
|
26
|
-
|
|
27
|
-
function isEvidence(ecosystem: string, name: string, version: string, file: string): boolean {
|
|
28
|
-
if (manifests.get(ecosystem)?.includes(file.slice(file.lastIndexOf('/') + 1))) return false
|
|
29
|
-
const stubRange = ecosystem === 'npm' && file === 'browser.js' ? browserStubRanges.get(name) : undefined
|
|
30
|
-
return !stubRange || !valid(version) || !satisfies(version, stubRange)
|
|
15
|
+
// The GitHub repository a bundled package.json's text names, read as Stasis
|
|
16
|
+
// records a dependency's (packageRepo) and as Node reads the file, past a byte
|
|
17
|
+
// order mark the bundle keeps. Undefined where it names none or doesn't parse.
|
|
18
|
+
function manifestRepo(text: unknown): Package['github'] {
|
|
19
|
+
if (typeof text !== 'string') return undefined
|
|
20
|
+
try { return packageRepo(JSON.parse(text.replace(/^\uFEFF/u, '')))?.github } catch { return undefined }
|
|
31
21
|
}
|
|
32
22
|
|
|
33
23
|
function supported(ecosystem: string): ecosystem is Package['ecosystem'] {
|
|
@@ -41,20 +31,21 @@ function skipReason(ecosystem: string, version: string): string | undefined {
|
|
|
41
31
|
return undefined
|
|
42
32
|
}
|
|
43
33
|
|
|
44
|
-
// Package and reason presence both require a recorded evidence file
|
|
45
|
-
//
|
|
34
|
+
// Package and reason presence both require a recorded evidence file, by Stasis's
|
|
35
|
+
// audit rule (server-only, as its semver uses Node's npm). Entry and manually
|
|
36
|
+
// added files count even when they have no incoming import edge.
|
|
46
37
|
export function bundleAdvisoryInventory(details: BundleDetails, paths: Iterable<string> | null = null): BundleAdvisoryInventory {
|
|
47
38
|
const modules = (details.bundle as { modules?: ReadonlyMap<string, BundleModule> } | undefined)?.modules
|
|
48
39
|
if (details.kind !== 'stasis' || !modules) return { packages: [], skipped: [] }
|
|
49
40
|
const selected = paths === null ? null : new Set(paths)
|
|
50
|
-
const packages = new Map<string, { ecosystem: Package['ecosystem']; name: string; versions: Set<string
|
|
41
|
+
const packages = new Map<string, { ecosystem: Package['ecosystem']; name: string; versions: Set<string>; github?: Package['github'] | null }>()
|
|
51
42
|
const skipped = new Map<string, SkippedAdvisoryPackage>()
|
|
52
43
|
for (const [dir, info] of modules) {
|
|
53
44
|
if (dir === '.') continue
|
|
54
45
|
const ecosystem = info.ecosystem ?? (/(?:^|\/)node_modules\//u.test(dir) ? 'npm' : undefined)
|
|
55
46
|
const { name, version } = info
|
|
56
47
|
if (ecosystem === undefined || typeof name !== 'string' || !name || typeof version !== 'string' || !version) continue
|
|
57
|
-
if (!Object.keys(info.files).some(file => (!selected || selected.has(`${dir}/${file}`)) &&
|
|
48
|
+
if (!Object.keys(info.files).some(file => (!selected || selected.has(`${dir}/${file}`)) && isEvidenceFile(name, version, file, ecosystem))) continue
|
|
58
49
|
const because = skipReason(ecosystem, version)
|
|
59
50
|
if (!supported(ecosystem) || because !== undefined) {
|
|
60
51
|
skipped.set(JSON.stringify([ecosystem, name, version]), { ecosystem, name, version, because: because ?? `No advisory source is supported for ${ecosystem}.` })
|
|
@@ -65,11 +56,22 @@ export function bundleAdvisoryInventory(details: BundleDetails, paths: Iterable<
|
|
|
65
56
|
const auditedVersion = ecosystem === 'github' && version === '.' ? '0.0.0' : version
|
|
66
57
|
const key = JSON.stringify([ecosystem, ecosystem === 'github' ? name.toLowerCase() : name])
|
|
67
58
|
if (!packages.has(key)) packages.set(key, { ecosystem, name, versions: new Set() })
|
|
68
|
-
packages.get(key)
|
|
59
|
+
const pkg = packages.get(key)!
|
|
60
|
+
pkg.versions.add(auditedVersion)
|
|
61
|
+
// A bundle from before Stasis recorded `repo` may still carry the npm
|
|
62
|
+
// package's package.json; ask the repository it names, as Stasis does.
|
|
63
|
+
let github = ecosystem === 'github' ? undefined : info.repo?.github
|
|
64
|
+
if (github === undefined && ecosystem === 'npm') github = manifestRepo(info.files['package.json'])
|
|
65
|
+
if (github !== undefined) {
|
|
66
|
+
// Upstream accepts one repository per package. Conflicting hints (for
|
|
67
|
+
// example, versions from before and after a move) use registry discovery.
|
|
68
|
+
if (pkg.github === undefined) pkg.github = github
|
|
69
|
+
else if (pkg.github !== null && pkg.github.toLowerCase() !== github.toLowerCase()) pkg.github = null
|
|
70
|
+
}
|
|
69
71
|
}
|
|
70
72
|
const byPackage = (a: { ecosystem: string; name: string }, b: { ecosystem: string; name: string }) => a.ecosystem.localeCompare(b.ecosystem) || a.name.localeCompare(b.name)
|
|
71
73
|
return {
|
|
72
|
-
packages: [...packages.values()].map(pkg => ({ ...pkg, versions: [...pkg.versions].toSorted() })).toSorted(byPackage),
|
|
74
|
+
packages: [...packages.values()].map(({ github, ...pkg }) => ({ ...pkg, versions: [...pkg.versions].toSorted(), ...(github ? { github } : {}) })).toSorted(byPackage),
|
|
73
75
|
skipped: [...skipped.values()].toSorted((a, b) => byPackage(a, b) || a.version.localeCompare(b.version)),
|
|
74
76
|
}
|
|
75
77
|
}
|
|
@@ -2,25 +2,60 @@ import { parentPort, workerData } from 'node:worker_threads'
|
|
|
2
2
|
import { Buffer } from 'node:buffer'
|
|
3
3
|
import { posix } from 'node:path'
|
|
4
4
|
import { brotliCompressSync } from 'node:zlib'
|
|
5
|
-
import { buildGitHubBundle } from '@exodus/stasis/vfs-bundle'
|
|
5
|
+
import { buildGitHubBundle, setCacheDir } from '@exodus/stasis/vfs-bundle'
|
|
6
6
|
import { brotliOptions } from '@exodus/stasis-core/brotli'
|
|
7
7
|
import { HttpError, createClient } from '@preventive/upstream/github.js'
|
|
8
8
|
import { githubBundleFilename } from './bundle-build.ts'
|
|
9
9
|
import { bundleBuildDiagnostic } from './bundle-build-diagnostics.js'
|
|
10
10
|
|
|
11
|
-
|
|
11
|
+
const regularFile = entry => entry.type === 'blob' && ['100644', '100755'].includes(entry.mode)
|
|
12
|
+
|
|
13
|
+
async function projectDirectory(input, github, client) {
|
|
14
|
+
// Input validation permits either JS/TS entries or Solidity entries, never a mix.
|
|
15
|
+
const manifests = input.entries[0].endsWith('.sol') ? ['foundry.toml', 'soldeer.toml', 'soldeer.lock'] : ['package.json']
|
|
16
|
+
if (!input.directory) return ''
|
|
17
|
+
for (let directory = input.directory; ; directory = posix.dirname(directory).replace(/^\.$/u, '')) {
|
|
18
|
+
const entries = await client.listRepoDir({ repo: github, sha: input.commit, directory: directory || undefined })
|
|
19
|
+
if (entries.some(entry => manifests.includes(entry.path) && regularFile(entry))) return directory
|
|
20
|
+
if (!directory) return input.directory
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
// The name Stasis read from the package.json in the project directory we
|
|
25
|
+
// provide, where its module holds bundled files. JS builds only: a Solidity
|
|
26
|
+
// build records `solidity-bundle` for a package.json without name and version.
|
|
27
|
+
async function packageName(input, github, project, bundle, client) {
|
|
28
|
+
if (!project || input.entries[0].endsWith('.sol')) return null
|
|
29
|
+
const entries = await client.listRepoDir({ repo: github, sha: input.commit, directory: project })
|
|
30
|
+
if (!entries.some(entry => entry.path === 'package.json' && regularFile(entry))) return null
|
|
31
|
+
const name = bundle.modules.get(posix.relative(bundle.repo?.directory ?? '', project) || '.')?.name
|
|
32
|
+
return typeof name === 'string' ? name : null
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export async function buildStasisBundle({ input, github, token, maxBytes, scopes, cacheDir = null }, client = createClient({ token }), progress = () => {}) {
|
|
12
36
|
const allowed = path => scopes.some(scope => !scope || path === scope || path.startsWith(scope + '/'))
|
|
13
|
-
//
|
|
14
|
-
|
|
37
|
+
// Reuse these immutable listings when Stasis discovers the lockfile root.
|
|
38
|
+
const listings = new Map()
|
|
39
|
+
const buildClient = { ...client, listRepoDir(options) {
|
|
40
|
+
const key = JSON.stringify([options.repo, options.sha, options.directory ?? ''])
|
|
41
|
+
if (!listings.has(key)) listings.set(key, client.listRepoDir(options))
|
|
42
|
+
return listings.get(key)
|
|
43
|
+
} }
|
|
44
|
+
// Upstream's disk cache keeps the npm tarballs and version documents, and
|
|
45
|
+
// the repo's tree, in `cacheDir`; without one (Vercel), nothing is kept on
|
|
46
|
+
// disk. This fresh worker inherits no application credentials.
|
|
47
|
+
setCacheDir(cacheDir ?? false)
|
|
15
48
|
progress('build')
|
|
49
|
+
const project = await projectDirectory(input, github, buildClient)
|
|
50
|
+
if (!allowed(project)) throw new Error('build-scope')
|
|
16
51
|
const { bundle } = await buildGitHubBundle({
|
|
17
|
-
github, sha: input.commit, directory:
|
|
18
|
-
entries: input.entries.map(entry => posix.relative(
|
|
19
|
-
...input.options, client,
|
|
52
|
+
github, sha: input.commit, directory: project || undefined,
|
|
53
|
+
entries: input.entries.map(entry => posix.relative(project || '.', entry)),
|
|
54
|
+
...input.options, client: buildClient, ...(cacheDir === null && { cache: false }),
|
|
20
55
|
})
|
|
21
56
|
const directory = bundle.repo?.directory ?? ''
|
|
22
57
|
progress('scope')
|
|
23
|
-
//
|
|
58
|
+
// Imports outside the selected package may widen the root. Never publish it under a
|
|
24
59
|
// narrower team grant: the storage location must cover the actual bundle.
|
|
25
60
|
if (!allowed(directory)) throw new Error('build-scope')
|
|
26
61
|
progress('serialize')
|
|
@@ -29,7 +64,8 @@ export async function buildStasisBundle({ input, github, token, maxBytes, scopes
|
|
|
29
64
|
progress('compress')
|
|
30
65
|
const bytes = brotliCompressSync(serialized, brotliOptions())
|
|
31
66
|
if (bytes.length > maxBytes) throw new Error('too-large')
|
|
32
|
-
|
|
67
|
+
const name = await packageName(input, github, project, bundle, buildClient)
|
|
68
|
+
return { bytes, directory, filename: githubBundleFilename(github, project, input.commit, name) }
|
|
33
69
|
}
|
|
34
70
|
|
|
35
71
|
if (parentPort && workerData?.type === 'managed-bundle-build') {
|
|
@@ -4,6 +4,7 @@ import { posix } from 'node:path'
|
|
|
4
4
|
import { Worker } from 'node:worker_threads'
|
|
5
5
|
import { BUNDLE_BUILD_TIMEOUT_MS, type BundleBuildLeaseStore } from './bundle-build-leases.ts'
|
|
6
6
|
import { type BundleBuildDiagnostic, type BundleBuildStage, bundleBuildDiagnostic } from './bundle-build-diagnostics.js'
|
|
7
|
+
import type { BundleBuildConditions } from './db-methods.ts'
|
|
7
8
|
|
|
8
9
|
export class BundleBuildError extends Error {
|
|
9
10
|
status: number
|
|
@@ -16,7 +17,9 @@ export interface BundleBuildInput {
|
|
|
16
17
|
commit: string
|
|
17
18
|
entries: string[]
|
|
18
19
|
directory: string
|
|
19
|
-
options: { conditions?: string[]; mainFields?: string[]; metro?: boolean; platforms?: string[];
|
|
20
|
+
options: { conditions?: string[]; mainFields?: string[]; metro?: boolean; platforms?: string[]; jsx?: boolean }
|
|
21
|
+
// What the stored bundle records; null for Solidity, which takes no conditions.
|
|
22
|
+
conditions: BundleBuildConditions | null
|
|
20
23
|
}
|
|
21
24
|
|
|
22
25
|
function path(value: unknown): value is string {
|
|
@@ -32,7 +35,7 @@ export function parseBundleBuild(value: unknown): BundleBuildInput {
|
|
|
32
35
|
|| typeof commit !== 'string' || !/^[a-f\d]{40}$/u.test(commit)
|
|
33
36
|
|| !Array.isArray(entries) || entries.length === 0 || entries.length > 100 || !entries.every(path)) return fail()
|
|
34
37
|
const selected = [...new Set(entries as string[])]
|
|
35
|
-
const scripts = selected.every(entry => /\.[
|
|
38
|
+
const scripts = selected.every(entry => /\.[cm]?[tj]sx?$/iu.test(entry))
|
|
36
39
|
if (!scripts && !selected.every(entry => entry.endsWith('.sol'))) return fail('unsupported-entries')
|
|
37
40
|
const parts = posix.dirname(selected[0]!).split('/').filter(part => part !== '.')
|
|
38
41
|
for (const entry of selected) {
|
|
@@ -40,33 +43,49 @@ export function parseBundleBuild(value: unknown): BundleBuildInput {
|
|
|
40
43
|
while (parts.some((part, i) => parent[i] !== part)) parts.pop()
|
|
41
44
|
}
|
|
42
45
|
const options: BundleBuildInput['options'] = {}
|
|
46
|
+
let recorded: BundleBuildConditions | null = null
|
|
43
47
|
if (scripts) {
|
|
44
48
|
if (conditions == null || typeof conditions !== 'object' || Array.isArray(conditions)) return fail('bad-conditions')
|
|
45
49
|
const { preset, conditions: names, platforms } = conditions as Record<string, unknown>
|
|
46
|
-
if (!['node', 'browser', 'metro'].includes(
|
|
50
|
+
if (typeof preset !== 'string' || !['node', 'browser', 'metro'].includes(preset) || !Array.isArray(names) || names.length > 16
|
|
47
51
|
|| !names.every(name => typeof name === 'string' && name.length > 0 && name.length <= 64
|
|
48
52
|
&& !/^\.|^\d+$|[\s,/\\\p{Cc}]/u.test(name) && !['default', 'import', 'require'].includes(name))) return fail('bad-conditions')
|
|
49
53
|
if (preset === 'metro') {
|
|
50
54
|
if (names.length !== 1 || names[0] !== 'react-native') return fail('metro-conditions')
|
|
51
55
|
if (!Array.isArray(platforms) || platforms.length === 0 || platforms.length > 2
|
|
52
56
|
|| !platforms.every(platform => ['ios', 'android'].includes(platform))) return fail('bad-conditions')
|
|
53
|
-
|
|
57
|
+
// React Native's convention puts JSX in .js files, which Stasis parses as
|
|
58
|
+
// JSX only on request. TypeScript is left to Stasis to detect.
|
|
59
|
+
Object.assign(options, { metro: true, platforms: [...new Set(platforms)], jsx: true })
|
|
54
60
|
} else {
|
|
55
61
|
options.conditions = [...new Set(names)]
|
|
56
62
|
if (preset === 'browser') options.mainFields = ['browser', 'module', 'main']
|
|
57
63
|
}
|
|
58
|
-
|
|
59
|
-
|
|
64
|
+
recorded = { preset: preset as BundleBuildConditions['preset'], conditions: [...new Set(names as string[])],
|
|
65
|
+
platforms: options.platforms ?? [] }
|
|
60
66
|
}
|
|
61
|
-
return { repoId, commit, entries: selected, directory: parts.join('/'), options }
|
|
67
|
+
return { repoId, commit, entries: selected, directory: parts.join('/'), options, conditions: recorded }
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
// A package name as a filename base: `@scope/name` as `scope-name`, or as
|
|
71
|
+
// `name` alone where it is `scope` or already starts with `scope-`. Null for
|
|
72
|
+
// a name npm could not publish by its length or leading dot.
|
|
73
|
+
function packageBase(name: string): string | null {
|
|
74
|
+
const match = name.length <= 214 ? /^(?:@([^/.][^/]*)\/)?([^/.][^/]*)$/u.exec(name) : null
|
|
75
|
+
if (!match) return null
|
|
76
|
+
const base = match[2]!, scope = match[1]
|
|
77
|
+
return scope === undefined || base === scope || base.startsWith(`${scope}-`) ? base : `${scope}-${base}`
|
|
62
78
|
}
|
|
63
79
|
|
|
64
80
|
// Same portable name and 255-character limit as Stasis's githubBundleFile
|
|
65
|
-
// (src/cmd/github-bundle.js), which is not a public package export
|
|
66
|
-
|
|
81
|
+
// (src/cmd/github-bundle.js), which is not a public package export, except
|
|
82
|
+
// that a usable `packageName` replaces both the repo and the directory.
|
|
83
|
+
export function githubBundleFilename(github: string, directory: string, commit: string, packageName: string | null = null): string {
|
|
67
84
|
const portable = (value: string) => value.replaceAll(/[^\w.-]/gu, '_')
|
|
68
|
-
const repo = portable(github.replace('/', '-'))
|
|
69
85
|
const tail = `.${portable(commit.slice(0, 7))}.stasis.code.br`
|
|
86
|
+
const named = packageName === null ? null : packageBase(packageName)
|
|
87
|
+
if (named !== null) return `${portable(named)}${tail}`
|
|
88
|
+
const repo = portable(github.replace('/', '-'))
|
|
70
89
|
if (!directory) return `${repo}${tail}`
|
|
71
90
|
const room = 255 - `${repo}.${tail}`.length
|
|
72
91
|
let dir = portable(directory.replaceAll('/', '-'))
|
|
@@ -75,7 +94,12 @@ export function githubBundleFilename(github: string, directory: string, commit:
|
|
|
75
94
|
}
|
|
76
95
|
|
|
77
96
|
export interface BuiltBundle { bytes: Uint8Array; directory: string; filename: string }
|
|
78
|
-
export interface BuildRequest {
|
|
97
|
+
export interface BuildRequest {
|
|
98
|
+
input: BundleBuildInput; github: string; token: string | null; maxBytes: number; scopes: (string | null)[]
|
|
99
|
+
// Upstream's disk cache for what a build fetches (setCacheDir), or null for
|
|
100
|
+
// none, as on Vercel. Resolved here: the worker runs with an empty env.
|
|
101
|
+
cacheDir: string | null
|
|
102
|
+
}
|
|
79
103
|
// An extra per-process ceiling; admission must also hold the shared DB lease.
|
|
80
104
|
const active = new Set<string>()
|
|
81
105
|
|
|
@@ -38,7 +38,7 @@ export interface BundleCacheStorage {
|
|
|
38
38
|
}
|
|
39
39
|
|
|
40
40
|
const filename = `v${BUNDLE_METADATA_VERSION}-metadata.json.br`
|
|
41
|
-
const packagesFilename = '
|
|
41
|
+
const packagesFilename = 'v6-advisory-inventory.json'
|
|
42
42
|
|
|
43
43
|
// All scopes share one bounded derivative. Never decode full bundle metadata
|
|
44
44
|
// on advisory requests, including when selecting a reason. Persist null when
|
package/server-managed/bundle.ts
CHANGED
|
@@ -8,7 +8,12 @@
|
|
|
8
8
|
import { createHash } from 'node:crypto'
|
|
9
9
|
import type { Buffer } from 'node:buffer'
|
|
10
10
|
import { StringDecoder } from 'node:string_decoder'
|
|
11
|
-
import { createBrotliDecompress } from 'node:zlib'
|
|
11
|
+
import { brotliDecompress, createBrotliDecompress } from 'node:zlib'
|
|
12
|
+
import { promisify } from 'node:util'
|
|
13
|
+
import { parseBundleContents } from '../common/bundle-metadata.js'
|
|
14
|
+
import { bundleFilesAsMap } from '../common/bundle-sources.js'
|
|
15
|
+
import { commonFileDirectory } from '../common/managed/repository-alias.ts'
|
|
16
|
+
import { decodeUtf8 } from '../common/utf8.js'
|
|
12
17
|
|
|
13
18
|
// `sha512-<base64>` identity for a bundle's bytes. MUST stay byte-identical to
|
|
14
19
|
// the client's common/integrity.js (SHA-512 → standard base64 WITH padding) so
|
|
@@ -92,6 +97,17 @@ export async function bundleRepo(bytes: Buffer): Promise<BundleRepo | null> {
|
|
|
92
97
|
finally { stream.destroy() }
|
|
93
98
|
}
|
|
94
99
|
|
|
100
|
+
const decompress = promisify(brotliDecompress)
|
|
101
|
+
// Same decoded size bound as the bundle metadata cache; only reached when an
|
|
102
|
+
// alias can use a shared file suffix. Invalid/opaque bundles keep header defaults.
|
|
103
|
+
export async function bundleFilePrefix(bytes: Buffer): Promise<string> {
|
|
104
|
+
try {
|
|
105
|
+
const decoded = await decompress(bytes, { maxOutputLength: 512 * 1024 * 1024 })
|
|
106
|
+
const details = parseBundleContents(decodeUtf8(decoded), { integrity: '', kind: 'stasis', size: bytes.length })
|
|
107
|
+
return commonFileDirectory(bundleFilesAsMap(details).keys())
|
|
108
|
+
} catch { return '' }
|
|
109
|
+
}
|
|
110
|
+
|
|
95
111
|
// Extract the bundle integrities a report declares (its top-level
|
|
96
112
|
// `bundleHashes`). Non-JSON reports (markdown / CSV) or a missing field → [].
|
|
97
113
|
export function reportBundleHashes(bytes: Buffer): string[] {
|
package/server-managed/config.ts
CHANGED
|
@@ -5,6 +5,7 @@ import { env } from 'node:process'
|
|
|
5
5
|
import { databaseUrls } from '../server-common/database-config.ts'
|
|
6
6
|
import { MAX_UPLOAD_BYTES } from './uploads.ts'
|
|
7
7
|
import { parseStorageKey } from '../server-common/storage-crypto.ts'
|
|
8
|
+
import { defaultCacheDir } from '@preventive/upstream/npm.js'
|
|
8
9
|
|
|
9
10
|
const LOOPBACK_HOSTS = new Set(['127.0.0.1', '::1', 'localhost'])
|
|
10
11
|
|
|
@@ -19,6 +20,9 @@ export interface ManagedConfig {
|
|
|
19
20
|
storageEncryptionMigrateMaxMs?: number
|
|
20
21
|
vercelPreview?: boolean
|
|
21
22
|
serverless?: boolean
|
|
23
|
+
// Upstream's disk cache (setCacheDir) for audits and bundle builds: its default
|
|
24
|
+
// location, or null for none, as on Vercel, where the database keeps listings.
|
|
25
|
+
upstreamCacheDir?: string | null
|
|
22
26
|
debug: boolean
|
|
23
27
|
allowShare: boolean
|
|
24
28
|
trustProxyEnv: string | undefined
|
|
@@ -44,7 +48,7 @@ export interface ManagedConfig {
|
|
|
44
48
|
githubAppSlug: string | null
|
|
45
49
|
githubNewIssueLabels?: string
|
|
46
50
|
// Max accepted size (bytes) for an uploaded report on the "Manage reports"
|
|
47
|
-
// page
|
|
51
|
+
// page, before storage compression (default 25 MiB).
|
|
48
52
|
maxReportBytes: number
|
|
49
53
|
// Max accepted size (bytes) for an uploaded bundle (sourcemap / stasis
|
|
50
54
|
// archive). Bundles run larger than reports, so a higher cap (default 200 MiB).
|
|
@@ -117,6 +121,7 @@ export function loadManagedConfig({ combined = false } = {}): ManagedConfig {
|
|
|
117
121
|
neonUrl, blobToken, serverless, storageEncryptionKey, storageEncryptionMigrate,
|
|
118
122
|
storageEncryptionMigrateMaxMs: intEnv('MANAGED_STORAGE_ENCRYPTION_MIGRATE_MAX_MS', 150_000, 1, serverless ? 240_000 : 3_600_000),
|
|
119
123
|
vercelPreview: env['VERCEL_ENV'] === 'preview',
|
|
124
|
+
upstreamCacheDir: serverless ? null : defaultCacheDir ?? null,
|
|
120
125
|
port: intEnv('PORT', 8765, 0, 65535),
|
|
121
126
|
host,
|
|
122
127
|
// DB_PATH belongs to e2e in a combined process. Keep the two stores apart.
|
|
@@ -135,7 +140,7 @@ export function loadManagedConfig({ combined = false } = {}): ManagedConfig {
|
|
|
135
140
|
githubAppPrivateKey: normalizePem(env['GITHUB_APP_PRIVATE_KEY']),
|
|
136
141
|
githubAppSlug: env['GITHUB_APP_SLUG'] ?? null,
|
|
137
142
|
githubNewIssueLabels: env['GITHUB_NEW_ISSUE_LABELS'] ?? '',
|
|
138
|
-
maxReportBytes: intEnv('MAX_REPORT_BYTES',
|
|
143
|
+
maxReportBytes: intEnv('MAX_REPORT_BYTES', 26_214_400, 1, 104_857_600),
|
|
139
144
|
maxBundleBytes: intEnv('MAX_BUNDLE_BYTES', 209_715_200, 1, MAX_UPLOAD_BYTES),
|
|
140
145
|
triageHistoryLimit: intEnv('TRIAGE_HISTORY_LIMIT', 0, 0, 1_000_000_000),
|
|
141
146
|
}
|