@preventive/triage 1.0.0-alpha.2 → 1.0.0-alpha.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/api/reap.ts +17 -0
- package/cli.js +6 -0
- package/client/finding-link.js +305 -0
- package/client/linked-findings.d.ts +1 -0
- package/client/linked-findings.js +111 -0
- package/common/bundle-metadata.d.ts +10 -0
- package/common/bundle-metadata.js +177 -0
- package/common/bundle-reasons.d.ts +2 -0
- package/common/bundle-reasons.js +21 -0
- package/common/bundle-sources.d.ts +3 -0
- package/common/bundle-sources.js +284 -0
- package/common/bundle-stats.js +41 -0
- package/common/bundle-tabs.js +1 -0
- package/common/code-language.js +36 -0
- package/common/default-scan-models.ts +30 -0
- package/common/finding-id.js +47 -0
- package/common/github-pr.ts +56 -0
- package/common/managed/comments.ts +34 -0
- package/common/managed/permissions.ts +35 -0
- package/common/managed/report-content.ts +42 -0
- package/common/managed/report-filter.ts +108 -0
- package/common/managed/roles.ts +28 -0
- package/common/managed/routes.d.ts +2 -0
- package/common/managed/routes.js +121 -0
- package/common/managed/scan-models.ts +6 -0
- package/common/managed/triage.ts +83 -0
- package/common/save-error-reason.ts +20 -7
- package/common/scan-server.ts +13 -0
- package/common/server-info.ts +33 -0
- package/common/utf8.d.ts +3 -0
- package/common/utf8.js +45 -0
- package/out/brotli-fallback.js +3 -3
- package/out/client-managed-import.js +81 -0
- package/out/client-managed.js +110 -0
- package/out/client-sync.js +16 -13
- package/out/graph.js +30 -4
- package/out/index.html +55 -8
- package/out/prism.js +2 -2
- package/out/stasis.svg +45 -0
- package/out/terminal.js +273 -39
- package/out/view.css +1 -1
- package/out/view.js +198 -62
- package/package.json +179 -55
- package/report/index.js +254 -0
- package/report/src/finding-id.js +80 -0
- package/report/src/finding.js +312 -0
- package/report/src/labels.js +33 -0
- package/report/src/md-structure.js +471 -0
- package/report/src/md-text.js +167 -0
- package/report/src/meta.js +76 -0
- package/report/src/parse-codex.js +147 -0
- package/report/src/parse-deepsec.js +197 -0
- package/report/src/parse-deepview-fields.js +375 -0
- package/report/src/parse-deepview-md.js +185 -0
- package/report/src/parse-md-id.js +137 -0
- package/report/src/parse-md.js +322 -0
- package/report/src/parse-piolium-id.js +79 -0
- package/report/src/parse-piolium-rows.js +131 -0
- package/report/src/parse-piolium-tokens.js +175 -0
- package/report/src/parse-piolium.js +400 -0
- package/report/src/security.js +63 -0
- package/report/src/utf8.js +21 -0
- package/report/src/write-md-finding.js +273 -0
- package/report/src/write-md.js +291 -0
- package/server-common/database-config.ts +16 -0
- package/server-common/initialize.ts +18 -0
- package/server-common/npm-advisories.ts +101 -0
- package/{server → server-common}/origin.ts +5 -5
- package/server-common/reap.ts +48 -0
- package/server-common/scan-config.ts +19 -0
- package/server-common/standalone.ts +29 -0
- package/server-common/storage-log.ts +34 -0
- package/server-common/vercel-blob.ts +110 -0
- package/server-e2e/app.ts +485 -0
- package/{server → server-e2e}/auth.ts +5 -1
- package/{server → server-e2e}/bus-receiver.ts +9 -8
- package/{server → server-e2e}/cli.js +9 -4
- package/{server → server-e2e}/config.ts +54 -39
- package/{server → server-e2e}/db-neon.ts +2 -2
- package/{server → server-e2e}/db-revision-sql.ts +7 -10
- package/{server → server-e2e}/db-stmt.ts +2 -2
- package/{server → server-e2e}/db.ts +96 -135
- package/{server → server-e2e}/http.ts +110 -12
- package/{server → server-e2e}/hub.ts +44 -14
- package/server-e2e/index.ts +17 -0
- package/server-e2e/lifecycle.ts +95 -0
- package/{server → server-e2e}/neon-driver.ts +2 -2
- package/{server → server-e2e}/npm-proxy.ts +11 -144
- package/{server → server-e2e}/objstore/blob-fs.ts +6 -8
- package/{server → server-e2e}/objstore/blob-vercel.ts +49 -141
- package/{server → server-e2e}/objstore/blob.ts +24 -9
- package/server-e2e/objstore/fetch-mint-guard.ts +74 -0
- package/{server → server-e2e}/objstore/handlers.ts +19 -20
- package/{server → server-e2e}/objstore/init.ts +53 -27
- package/{server → server-e2e}/objstore/reaper.ts +31 -11
- package/server-e2e/objstore/rest-deny.ts +28 -0
- package/server-e2e/objstore/rest-mint.ts +224 -0
- package/{server → server-e2e}/objstore/rest.ts +110 -93
- package/{server → server-e2e}/objstore/sign.ts +105 -0
- package/{server → server-e2e}/objstore/store-neon.ts +10 -14
- package/{server → server-e2e}/objstore/store.ts +98 -118
- package/{server → server-e2e}/objstore/tokens.ts +9 -12
- package/{server → server-e2e}/peer.ts +7 -9
- package/{server → server-e2e}/pubsub.ts +29 -36
- package/{server → server-e2e}/sign.ts +12 -14
- package/{server → server-e2e}/sse-server.ts +105 -73
- package/{server → server-e2e}/sse-session.ts +30 -16
- package/{server → server-e2e}/static.ts +36 -29
- package/server-e2e/sync-handlers.ts +408 -0
- package/{server → server-e2e}/util.ts +9 -0
- package/{server → server-e2e}/ws-server.ts +29 -23
- package/server-managed/activity.ts +231 -0
- package/server-managed/avatar-store.ts +51 -0
- package/server-managed/blob-store.ts +66 -0
- package/server-managed/blob-vercel.ts +125 -0
- package/server-managed/brotli.ts +10 -0
- package/server-managed/bundle-cache.ts +185 -0
- package/server-managed/bundle-catalog.ts +29 -0
- package/server-managed/bundle-store.ts +28 -0
- package/server-managed/bundle-summary-cache.ts +97 -0
- package/server-managed/bundle.ts +39 -0
- package/server-managed/cache-storage.ts +40 -0
- package/server-managed/cli.js +13 -0
- package/server-managed/combined.ts +46 -0
- package/server-managed/comments.ts +151 -0
- package/server-managed/config.ts +144 -0
- package/server-managed/content-access.ts +15 -0
- package/server-managed/crypto.ts +25 -0
- package/server-managed/db-methods.ts +1330 -0
- package/server-managed/db-neon.ts +171 -0
- package/server-managed/db-schema.ts +203 -0
- package/server-managed/db-table-names.ts +22 -0
- package/server-managed/db.ts +109 -0
- package/server-managed/github-app.ts +332 -0
- package/server-managed/github-metadata.ts +65 -0
- package/server-managed/github-oauth.ts +215 -0
- package/server-managed/github-pulls.ts +115 -0
- package/server-managed/http-response.ts +18 -0
- package/server-managed/http.ts +2043 -0
- package/server-managed/import-triage.ts +48 -0
- package/server-managed/index.ts +135 -0
- package/server-managed/public-workspace.ts +150 -0
- package/server-managed/repo-path.ts +21 -0
- package/server-managed/report-migration.ts +35 -0
- package/server-managed/report-query.ts +4 -0
- package/server-managed/report-response.ts +16 -0
- package/server-managed/report-sources.ts +154 -0
- package/server-managed/repository-discovery.ts +82 -0
- package/server-managed/repository-policy.ts +25 -0
- package/server-managed/session.ts +78 -0
- package/server-managed/slugs.ts +38 -0
- package/server-managed/sql-postgres.ts +30 -0
- package/server-managed/sql.ts +61 -0
- package/server-managed/static.ts +28 -0
- package/server-managed/storage.ts +34 -0
- package/server-managed/team-catalog.ts +7 -0
- package/server-managed/team-feed.ts +128 -0
- package/server-managed/team-reports.ts +156 -0
- package/server-managed/triage-response.ts +16 -0
- package/server-managed/uploads.ts +47 -0
- package/server-managed/workspace-shares.ts +150 -0
- package/server.ts +50 -0
- package/server/index.ts +0 -481
- package/server/lifecycle.ts +0 -204
- package/server/sync-handlers.ts +0 -327
- /package/{server → server-e2e}/config.example.json +0 -0
- /package/{server → server-e2e}/objstore/fs.ts +0 -0
- /package/{server → server-e2e}/validation.ts +0 -0
|
@@ -0,0 +1,231 @@
|
|
|
1
|
+
// Persistent workspace activity. Uploads are recorded in the same transaction
|
|
2
|
+
// as their metadata; triage reuses its existing immutable trail, including
|
|
3
|
+
// legacy entries. Never copy annotation bodies or credentials into this feed.
|
|
4
|
+
import { randomUUID } from 'node:crypto'
|
|
5
|
+
import type { DatabaseSync } from 'node:sqlite'
|
|
6
|
+
import type { ManagedSql } from './sql.ts'
|
|
7
|
+
|
|
8
|
+
export interface ActivityInput {
|
|
9
|
+
kind: 'access' | 'repository' | 'visibility' | 'delete'
|
|
10
|
+
actor: string
|
|
11
|
+
actorId?: string | null
|
|
12
|
+
action: string
|
|
13
|
+
repo?: string | null
|
|
14
|
+
reportId?: string | null
|
|
15
|
+
bundleId?: string | null
|
|
16
|
+
repoId?: number | null
|
|
17
|
+
repoDirectory?: string | null
|
|
18
|
+
report?: string | null
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export interface ActivityContext {
|
|
22
|
+
finding: string
|
|
23
|
+
reportId: string
|
|
24
|
+
report: string
|
|
25
|
+
repo: string | null
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export type ActivityEntry = {
|
|
29
|
+
id: string; kind: string; actor: string | null; action: string
|
|
30
|
+
repo: string | null; reportId: string | null; report: string | null
|
|
31
|
+
finding: string | null; at: number
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export interface ActivityQuery {
|
|
35
|
+
page: number
|
|
36
|
+
limit: number
|
|
37
|
+
kind: string
|
|
38
|
+
query: string
|
|
39
|
+
repo?: string
|
|
40
|
+
actor?: string
|
|
41
|
+
// null = admin; [] = no accessible findings. Context is supplied by the
|
|
42
|
+
// server, never by the client. It also replaces potentially private context
|
|
43
|
+
// from an old report carrying the same globally shared finding.
|
|
44
|
+
contexts: ActivityContext[] | null
|
|
45
|
+
userId?: string
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export interface ActivityFilters {
|
|
49
|
+
repos: string[]
|
|
50
|
+
users: { id: string; login: string; detail: string | null }[]
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export interface ActivityStore {
|
|
54
|
+
recordActivity(entry: ActivityInput, at: number): Promise<void>
|
|
55
|
+
listActivityReports(userId: string): Promise<Omit<ActivityContext, 'finding'>[]>
|
|
56
|
+
listActivity(query: ActivityQuery): Promise<{ history: ActivityEntry[]; total: number; page: number; limit: number; filters: ActivityFilters }>
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
const triageFields = `'triage:' || e.seq AS id, 'triage' AS kind,
|
|
60
|
+
COALESCE(u.login, e.actor_login) AS actor, e.actor_id AS actorId,
|
|
61
|
+
CASE WHEN e.color IS NULL AND e.triage IS NULL AND e.comment IS NULL AND e.fix IS NULL AND e.flagged IS NULL
|
|
62
|
+
THEN 'cleared triage' ELSE 'updated triage' END AS action`
|
|
63
|
+
const triage = `SELECT ${triageFields}, e.repo, e.report_id AS reportId, e.report, e.finding_id AS finding, e.at
|
|
64
|
+
FROM managed_finding_triage_event e LEFT JOIN managed_user u ON u.id = e.actor_id`
|
|
65
|
+
const commentFields = `'comment:' || e.seq AS id, 'triage' AS kind,
|
|
66
|
+
COALESCE(u.login, e.actor_login) AS actor, e.actor_id AS actorId, e.action`
|
|
67
|
+
const comments = `SELECT ${commentFields}, e.repo, e.report_id AS reportId, e.report, e.finding_id AS finding, e.at
|
|
68
|
+
FROM managed_finding_comment_event e LEFT JOIN managed_user u ON u.id = e.actor_id`
|
|
69
|
+
const adminSource = `${triage} UNION ALL ${comments} UNION ALL
|
|
70
|
+
SELECT id, kind, actor, actor_id AS actorId, action, repo, report_id AS reportId, report, NULL AS finding, at FROM managed_activity`
|
|
71
|
+
// Keep the finite set of accessible findings outside the indexed event
|
|
72
|
+
// lookup. Scanning all events against json_each for every row is quadratic.
|
|
73
|
+
const managerSource = `SELECT ${triageFields},
|
|
74
|
+
json_extract(c.value, '$.repo') AS repo, json_extract(c.value, '$.reportId') AS reportId,
|
|
75
|
+
json_extract(c.value, '$.report') AS report, e.finding_id AS finding, e.at
|
|
76
|
+
FROM json_each(:contexts) c
|
|
77
|
+
CROSS JOIN managed_finding_triage_event e ON e.finding_id = json_extract(c.value, '$.finding')
|
|
78
|
+
LEFT JOIN managed_user u ON u.id = e.actor_id
|
|
79
|
+
UNION ALL SELECT ${commentFields},
|
|
80
|
+
json_extract(c.value, '$.repo') AS repo, json_extract(c.value, '$.reportId') AS reportId,
|
|
81
|
+
json_extract(c.value, '$.report') AS report, e.finding_id AS finding, e.at
|
|
82
|
+
FROM json_each(:contexts) c
|
|
83
|
+
CROSS JOIN managed_finding_comment_event e ON e.finding_id = json_extract(c.value, '$.finding')
|
|
84
|
+
LEFT JOIN managed_user u ON u.id = e.actor_id
|
|
85
|
+
UNION ALL SELECT a.id, a.kind, a.actor, a.actor_id AS actorId,
|
|
86
|
+
CASE WHEN a.kind = 'repository' THEN 'changed a report repository assignment' ELSE a.action END AS action,
|
|
87
|
+
p.full_name AS repo, r.id AS reportId, r.filename AS report, NULL AS finding, a.at
|
|
88
|
+
FROM managed_activity a JOIN managed_report r ON r.id = a.report_id
|
|
89
|
+
LEFT JOIN managed_selected_repo p ON p.repo_id = r.repo_id
|
|
90
|
+
WHERE EXISTS (
|
|
91
|
+
SELECT 1 FROM managed_team_repo tr JOIN managed_team_user tu ON tu.team_id = tr.team_id
|
|
92
|
+
WHERE tu.user_id = :userId AND tr.repo_id = r.repo_id AND ${withinTeamPath('r.repo_directory')}
|
|
93
|
+
)
|
|
94
|
+
UNION ALL SELECT a.id, a.kind, a.actor, a.actor_id AS actorId,
|
|
95
|
+
CASE WHEN a.kind = 'repository' THEN 'changed a bundle repository assignment' ELSE a.action END AS action,
|
|
96
|
+
p.full_name AS repo, NULL AS reportId, b.filename AS report, NULL AS finding, a.at
|
|
97
|
+
FROM managed_activity a JOIN managed_bundle b ON b.id = a.bundle_id
|
|
98
|
+
LEFT JOIN managed_selected_repo p ON p.repo_id = b.repo_id
|
|
99
|
+
WHERE a.report_id IS NULL AND EXISTS (
|
|
100
|
+
SELECT 1 FROM managed_team_repo tr JOIN managed_team_user tu ON tu.team_id = tr.team_id
|
|
101
|
+
WHERE tu.user_id = :userId AND tr.repo_id = b.repo_id AND ${withinTeamPath('b.repo_directory')}
|
|
102
|
+
)
|
|
103
|
+
UNION ALL SELECT a.id, a.kind, a.actor, a.actor_id AS actorId, a.action, a.repo, a.report_id AS reportId,
|
|
104
|
+
a.report, NULL AS finding, a.at FROM managed_activity a
|
|
105
|
+
WHERE a.kind = 'delete' AND (a.report_id IS NOT NULL OR a.bundle_id IS NOT NULL)
|
|
106
|
+
AND NOT EXISTS (SELECT 1 FROM managed_report WHERE id = a.report_id)
|
|
107
|
+
AND NOT EXISTS (SELECT 1 FROM managed_bundle WHERE id = a.bundle_id)
|
|
108
|
+
AND EXISTS (SELECT 1 FROM managed_team_repo tr JOIN managed_team_user tu ON tu.team_id = tr.team_id
|
|
109
|
+
WHERE tu.user_id = :userId AND tr.repo_id = a.repo_id
|
|
110
|
+
AND ${withinTeamPath('a.repo_directory')})`
|
|
111
|
+
|
|
112
|
+
export const ACTIVITY_SCHEMA = `
|
|
113
|
+
CREATE TABLE IF NOT EXISTS managed_activity (
|
|
114
|
+
id TEXT PRIMARY KEY, kind TEXT NOT NULL, actor TEXT, action TEXT NOT NULL,
|
|
115
|
+
repo TEXT, report_id TEXT, report TEXT, at INTEGER NOT NULL,
|
|
116
|
+
bundle_id TEXT, repo_id INTEGER, repo_directory TEXT, actor_id TEXT
|
|
117
|
+
) STRICT;
|
|
118
|
+
CREATE INDEX IF NOT EXISTS managed_activity_at_idx ON managed_activity(at, id);
|
|
119
|
+
CREATE INDEX IF NOT EXISTS finding_triage_event_at_idx ON managed_finding_triage_event(at, seq);
|
|
120
|
+
`
|
|
121
|
+
|
|
122
|
+
export function initActivityMethods(db: DatabaseSync): void {
|
|
123
|
+
db.exec(ACTIVITY_SCHEMA)
|
|
124
|
+
migrateActivityScope(db)
|
|
125
|
+
db.exec('CREATE INDEX IF NOT EXISTS managed_activity_actor_at_idx ON managed_activity(actor_id, at)')
|
|
126
|
+
// Stable upload IDs make backfill idempotent, including after restarts.
|
|
127
|
+
// Snapshots deliberately have no cascading FKs: deletion is itself activity.
|
|
128
|
+
for (const type of ['report', 'bundle']) {
|
|
129
|
+
const columns = `id, kind, actor, action, repo, report_id, report, at, bundle_id, actor_id`
|
|
130
|
+
const values = (alias: string) => `'${type}-upload:' || ${alias}.id, 'upload',
|
|
131
|
+
COALESCE(${alias}.uploaded_by_login, (SELECT login FROM managed_user WHERE id = ${alias}.uploaded_by)), 'uploaded a ${type}',
|
|
132
|
+
(SELECT full_name FROM managed_selected_repo WHERE repo_id = ${alias}.repo_id),
|
|
133
|
+
${type === 'report' ? `${alias}.id` : 'NULL'}, ${alias}.filename, ${alias}.uploaded_at,
|
|
134
|
+
${type === 'bundle' ? `${alias}.id` : 'NULL'}, ${alias}.uploaded_by`
|
|
135
|
+
db.exec(`
|
|
136
|
+
CREATE TRIGGER IF NOT EXISTS managed_${type}_activity AFTER INSERT ON managed_${type}
|
|
137
|
+
BEGIN INSERT INTO managed_activity (${columns}) SELECT ${values('NEW')}; END;
|
|
138
|
+
INSERT OR IGNORE INTO managed_activity (${columns}) SELECT ${values('source')} FROM managed_${type} source;
|
|
139
|
+
`)
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
export function activityMethods(db: ManagedSql): ActivityStore {
|
|
144
|
+
const insert = db.prepare(`INSERT INTO managed_activity (id, kind, actor, action, repo, report_id, report, at, bundle_id, repo_id, repo_directory, actor_id)
|
|
145
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
|
|
146
|
+
const admin = activityStatements(db, adminSource)
|
|
147
|
+
const manager = activityStatements(db, managerSource)
|
|
148
|
+
const reports = db.prepare(`SELECT r.id AS reportId, r.filename AS report, p.full_name AS repo
|
|
149
|
+
FROM managed_report r JOIN managed_selected_repo p ON p.repo_id = r.repo_id
|
|
150
|
+
WHERE EXISTS (SELECT 1 FROM managed_team_repo tr JOIN managed_team_user tu ON tu.team_id = tr.team_id
|
|
151
|
+
WHERE tu.user_id = ? AND tr.repo_id = r.repo_id AND ${withinTeamPath('r.repo_directory')})
|
|
152
|
+
ORDER BY r.uploaded_at DESC, r.id`)
|
|
153
|
+
return {
|
|
154
|
+
async recordActivity(entry, at) {
|
|
155
|
+
await insert.run(randomUUID(), entry.kind, entry.actor, entry.action, entry.repo ?? null, entry.reportId ?? null, entry.report ?? null, at,
|
|
156
|
+
entry.bundleId ?? null, entry.repoId ?? null, entry.repoDirectory ?? null, entry.actorId ?? null)
|
|
157
|
+
},
|
|
158
|
+
async listActivityReports(userId) {
|
|
159
|
+
return (await reports.all(userId)) as Omit<ActivityContext, 'finding'>[]
|
|
160
|
+
},
|
|
161
|
+
async listActivity({ page, limit, kind, query, repo = '', actor = '', contexts, userId }) {
|
|
162
|
+
const stmts = contexts == null ? admin : manager
|
|
163
|
+
const scope = contexts == null ? {} : { contexts: JSON.stringify(contexts), userId: userId ?? '' }
|
|
164
|
+
const params = { ...scope, kind, query, repo, actor }
|
|
165
|
+
const { total } = (await stmts.count.get(params)) as { total: number }
|
|
166
|
+
const currentPage = Math.min(page, Math.max(1, Math.ceil(total / limit)))
|
|
167
|
+
const history = (await stmts.rows.all({ ...params, limit, offset: (currentPage - 1) * limit })) as ActivityEntry[]
|
|
168
|
+
const filters: ActivityFilters = {
|
|
169
|
+
repos: ((await stmts.repos.all(scope)) as { repo: string }[]).map(row => row.repo),
|
|
170
|
+
users: ((await stmts.users.all(scope)) as ActivityFilters['users']).map(row => ({ id: row.id, login: row.login, detail: row.detail })),
|
|
171
|
+
}
|
|
172
|
+
return { history, total, page: currentPage, limit, filters }
|
|
173
|
+
},
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
// Both rows and selector choices come from the authorized source. Keep legacy
|
|
178
|
+
// login-only actors separate: a historical login is not proof of user identity.
|
|
179
|
+
function activityStatements(db: ManagedSql, source: string) {
|
|
180
|
+
const activity = `WITH events AS (${source}), activity AS (
|
|
181
|
+
SELECT events.*, COALESCE('user:' || actorId, 'legacy:' || actor) AS actorKey FROM events
|
|
182
|
+
)`
|
|
183
|
+
const filtered = `${activity} SELECT id, kind, actor, action, repo, reportId, report, finding, at FROM activity
|
|
184
|
+
WHERE (:kind = 'all' OR kind = :kind)
|
|
185
|
+
AND (:repo = '' OR repo = :repo) AND (:actor = '' OR actorKey = :actor)
|
|
186
|
+
AND (:query = '' OR instr(lower(coalesce(actor, '') || ' ' || action || ' ' ||
|
|
187
|
+
coalesce(repo, '') || ' ' || coalesce(report, '') || ' ' || coalesce(finding, '')), lower(:query)) > 0)`
|
|
188
|
+
return {
|
|
189
|
+
count: db.prepare(`SELECT count(*) AS total FROM (${filtered}) AS filtered`),
|
|
190
|
+
rows: db.prepare(`${filtered} ORDER BY at DESC,
|
|
191
|
+
CASE WHEN kind = 'triage' THEN CAST(substr(id, instr(id, ':') + 1) AS INTEGER) ELSE 0 END DESC,
|
|
192
|
+
id DESC LIMIT :limit OFFSET :offset`),
|
|
193
|
+
repos: db.prepare(`${activity} SELECT DISTINCT repo FROM activity WHERE repo IS NOT NULL AND repo <> '' ORDER BY repo`),
|
|
194
|
+
users: db.prepare(`${activity} SELECT a.actorKey AS id, COALESCE(u.login, a.actor, a.actorId) AS login,
|
|
195
|
+
CASE WHEN a.actorId IS NULL THEN 'Legacy actor' ELSE NULL END AS detail FROM (
|
|
196
|
+
SELECT actorKey, actorId, actor, ROW_NUMBER() OVER (PARTITION BY actorKey ORDER BY at DESC, id DESC) AS position
|
|
197
|
+
FROM activity WHERE actorKey IS NOT NULL AND actorKey <> 'legacy:'
|
|
198
|
+
) a LEFT JOIN managed_user u ON u.id = a.actorId WHERE a.position = 1 ORDER BY login, a.actorKey`),
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
// Literal path comparison preserves separators, case and SQL wildcard chars.
|
|
203
|
+
function withinTeamPath(path: string): string {
|
|
204
|
+
return `(tr.path IS NULL OR tr.path = '' OR ${path} = tr.path OR substr(${path}, 1, length(tr.path) + 1) = tr.path || '/')`
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
function migrateActivityScope(db: DatabaseSync): void {
|
|
208
|
+
const columns = new Set((db.prepare('PRAGMA table_info(managed_activity)').all() as { name: string }[]).map(column => column.name))
|
|
209
|
+
const missing = [['bundle_id', 'TEXT'], ['repo_id', 'INTEGER'], ['repo_directory', 'TEXT'], ['actor_id', 'TEXT']].filter(([name]) => !columns.has(name!))
|
|
210
|
+
if (missing.length === 0) return
|
|
211
|
+
db.exec('BEGIN')
|
|
212
|
+
try {
|
|
213
|
+
for (const [name, type] of missing) db.exec(`ALTER TABLE managed_activity ADD COLUMN ${name} ${type}`)
|
|
214
|
+
if (!columns.has('bundle_id') || !columns.has('actor_id')) {
|
|
215
|
+
db.exec('DROP TRIGGER IF EXISTS managed_report_activity; DROP TRIGGER IF EXISTS managed_bundle_activity;')
|
|
216
|
+
}
|
|
217
|
+
if (!columns.has('bundle_id')) db.exec("UPDATE managed_activity SET bundle_id = substr(id, 15) WHERE id LIKE 'bundle-upload:%'")
|
|
218
|
+
if (!columns.has('actor_id')) {
|
|
219
|
+
// Existing upload targets retain a reliable uploader identity. Do not
|
|
220
|
+
// guess identities from historical logins: names can change or be reused.
|
|
221
|
+
for (const type of ['report', 'bundle']) {
|
|
222
|
+
db.exec(`UPDATE managed_activity SET actor_id = (
|
|
223
|
+
SELECT uploaded_by FROM managed_${type} WHERE id = substr(managed_activity.id, 15)
|
|
224
|
+
) WHERE id LIKE '${type}-upload:%'`)
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
// Legacy deletions/assignments without a durable target/scope remain
|
|
228
|
+
// admin-only; neither filenames nor repository names authorize access.
|
|
229
|
+
db.exec('COMMIT')
|
|
230
|
+
} catch (err) { db.exec('ROLLBACK'); throw err }
|
|
231
|
+
}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
// Avatar cache for the managed server. The user's GitHub avatar is fetched once
|
|
2
|
+
// at login (see github-oauth.ts) and served same-origin from GET
|
|
3
|
+
// /api/auth/avatar — the app's CSP is `img-src 'self'`, so the GitHub-hosted
|
|
4
|
+
// original can't be loaded by the page directly.
|
|
5
|
+
//
|
|
6
|
+
// The disk backend uses a dir beside SQLite; blob-vercel.ts implements the
|
|
7
|
+
// same AvatarStore interface for remote storage. Keyed by the user's opaque `id`.
|
|
8
|
+
import { mkdir, readFile, writeFile } from 'node:fs/promises'
|
|
9
|
+
import { dirname, join } from 'node:path'
|
|
10
|
+
import type { Buffer } from 'node:buffer'
|
|
11
|
+
|
|
12
|
+
export interface CachedAvatar {
|
|
13
|
+
contentType: string
|
|
14
|
+
bytes: Buffer
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export interface AvatarStore {
|
|
18
|
+
put(id: string, contentType: string, bytes: Buffer): Promise<void>
|
|
19
|
+
get(id: string): Promise<CachedAvatar | null>
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
// The user id is a v4 uuid (crypto.randomUUID) — lowercase hex + dashes, no
|
|
23
|
+
// path separators. Validated anyway so a crafted id can't escape the store dir.
|
|
24
|
+
const ID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/u
|
|
25
|
+
|
|
26
|
+
export function createDiskAvatarStore(dir: string): AvatarStore {
|
|
27
|
+
function pathFor(id: string): string {
|
|
28
|
+
if (!ID_RE.test(id)) throw new Error('invalid avatar id')
|
|
29
|
+
return join(dir, id)
|
|
30
|
+
}
|
|
31
|
+
return {
|
|
32
|
+
async put(id, contentType, bytes) {
|
|
33
|
+
const p = pathFor(id)
|
|
34
|
+
await mkdir(dirname(p), { recursive: true })
|
|
35
|
+
// Bytes first, then the content-type sidecar — a crash between the two
|
|
36
|
+
// leaves a typeless blob that `get` treats as missing (re-fetched next
|
|
37
|
+
// login) rather than serving with a wrong/absent type.
|
|
38
|
+
await writeFile(p, bytes)
|
|
39
|
+
await writeFile(`${p}.type`, contentType, 'utf8')
|
|
40
|
+
},
|
|
41
|
+
async get(id) {
|
|
42
|
+
try {
|
|
43
|
+
const p = pathFor(id)
|
|
44
|
+
const [bytes, contentType] = await Promise.all([readFile(p), readFile(`${p}.type`, 'utf8')])
|
|
45
|
+
return { bytes, contentType: contentType.trim() || 'application/octet-stream' }
|
|
46
|
+
} catch {
|
|
47
|
+
return null
|
|
48
|
+
}
|
|
49
|
+
},
|
|
50
|
+
}
|
|
51
|
+
}
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
// Disk byte-store for the managed server's stored blobs (uploaded reports and
|
|
2
|
+
// bundles). Unlike the e2e relay (which only ever holds opaque client-encrypted
|
|
3
|
+
// blobs), a managed server is TRUSTED and stores the bytes in the clear so it
|
|
4
|
+
// can operate on them — the metadata + attribution live in the DB, the bytes
|
|
5
|
+
// live here, keyed by the blob's opaque `id`.
|
|
6
|
+
//
|
|
7
|
+
// The disk backend uses a dir beside SQLite (data/reports/<uuid> or
|
|
8
|
+
// data/bundles/<uuid>); blob-vercel.ts implements the same interface remotely. Bytes only — the
|
|
9
|
+
// content-type / filename / integrity ride the DB row, so there's no sidecar.
|
|
10
|
+
import { mkdir, open, readFile, rm, writeFile } from 'node:fs/promises'
|
|
11
|
+
import { dirname, join } from 'node:path'
|
|
12
|
+
import type { Buffer } from 'node:buffer'
|
|
13
|
+
import type { Readable } from 'node:stream'
|
|
14
|
+
|
|
15
|
+
export interface OpenedBlob {
|
|
16
|
+
// Remote streams may omit their size; callers then omit Content-Length.
|
|
17
|
+
size: number | null
|
|
18
|
+
stream: Readable
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export interface BlobStore {
|
|
22
|
+
put(id: string, bytes: Buffer): Promise<void>
|
|
23
|
+
get(id: string): Promise<Buffer | null>
|
|
24
|
+
open(id: string): Promise<OpenedBlob | null>
|
|
25
|
+
delete(id: string): Promise<void>
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
// The blob id is a v4 uuid (crypto.randomUUID) — lowercase hex + dashes, no
|
|
29
|
+
// path separators. Validated anyway so a crafted id can't escape the store dir.
|
|
30
|
+
const ID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/u
|
|
31
|
+
|
|
32
|
+
export function createDiskBlobStore(dir: string, suffix = ''): BlobStore {
|
|
33
|
+
function pathFor(id: string): string {
|
|
34
|
+
if (!ID_RE.test(id)) throw new Error('invalid blob id')
|
|
35
|
+
return join(dir, id + suffix)
|
|
36
|
+
}
|
|
37
|
+
return {
|
|
38
|
+
async put(id, bytes) {
|
|
39
|
+
const p = pathFor(id)
|
|
40
|
+
await mkdir(dirname(p), { recursive: true })
|
|
41
|
+
await writeFile(p, bytes)
|
|
42
|
+
},
|
|
43
|
+
async get(id) {
|
|
44
|
+
try {
|
|
45
|
+
return await readFile(pathFor(id))
|
|
46
|
+
} catch {
|
|
47
|
+
return null
|
|
48
|
+
}
|
|
49
|
+
},
|
|
50
|
+
async open(id) {
|
|
51
|
+
let file
|
|
52
|
+
try { file = await open(pathFor(id), 'r') }
|
|
53
|
+
catch (err) { if ((err as NodeJS.ErrnoException).code === 'ENOENT') return null; throw err }
|
|
54
|
+
try {
|
|
55
|
+
const info = await file.stat()
|
|
56
|
+
// Remain paused until piped. HEAD can close without reading the body.
|
|
57
|
+
return { size: info.size, stream: file.createReadStream() }
|
|
58
|
+
} catch (err) { await file.close(); throw err }
|
|
59
|
+
},
|
|
60
|
+
async delete(id) {
|
|
61
|
+
// `force` so a missing file (already gone) is a no-op, not a throw — the
|
|
62
|
+
// metadata row is the source of truth for whether a blob "exists".
|
|
63
|
+
await rm(pathFor(id), { force: true })
|
|
64
|
+
},
|
|
65
|
+
}
|
|
66
|
+
}
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
import { Buffer } from 'node:buffer'
|
|
2
|
+
import { Readable } from 'node:stream'
|
|
3
|
+
import { type VercelBlobSdk, isNotFound, loadVercelBlobSdk } from '../server-common/vercel-blob.ts'
|
|
4
|
+
import type { AvatarStore } from './avatar-store.ts'
|
|
5
|
+
import type { BlobStore } from './blob-store.ts'
|
|
6
|
+
import { type CacheStorage, validateCacheKey } from './cache-storage.ts'
|
|
7
|
+
import type { BundleCacheStorage } from './bundle-cache.ts'
|
|
8
|
+
import { createBundleStore } from './bundle-store.ts'
|
|
9
|
+
|
|
10
|
+
const ID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/u
|
|
11
|
+
function validate(id: string): string {
|
|
12
|
+
if (!ID.test(id)) throw new Error('Invalid managed blob id')
|
|
13
|
+
return id
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
type ListedBlob = Awaited<ReturnType<VercelBlobSdk['list']>>['blobs'][number]
|
|
17
|
+
|
|
18
|
+
async function deletePrefix(blobs: VercelBlobSdk, token: string, prefix: string, remove: (path: string) => Promise<void>, matches = (_blob: ListedBlob) => true): Promise<void> {
|
|
19
|
+
const cursors = new Set<string>(), paths = new Set<string>()
|
|
20
|
+
let cursor: string | undefined
|
|
21
|
+
// Finish listing before deleting so pagination cannot skip objects as the
|
|
22
|
+
// listed set shrinks. Callers supply a trailing slash to isolate a namespace.
|
|
23
|
+
do {
|
|
24
|
+
const page = await blobs.list({ token, prefix, ...(cursor ? { cursor } : {}) })
|
|
25
|
+
for (const blob of page.blobs) if (blob.pathname.startsWith(prefix) && matches(blob)) paths.add(blob.pathname)
|
|
26
|
+
if (!page.hasMore) break
|
|
27
|
+
if (!page.cursor || cursors.has(page.cursor)) throw new Error('Invalid blob pagination')
|
|
28
|
+
cursor = page.cursor
|
|
29
|
+
cursors.add(cursor)
|
|
30
|
+
} while (cursor)
|
|
31
|
+
for (const path of paths) await remove(path)
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function createBlobAvatarStore(avatarBlobs: BlobStore): AvatarStore {
|
|
35
|
+
return {
|
|
36
|
+
put(id, contentType, bytes) {
|
|
37
|
+
const type = contentType.split(';', 1)[0]!.trim()
|
|
38
|
+
if (!/^image\/[a-z0-9.+-]+$/iu.test(type)) throw new Error('Invalid avatar type')
|
|
39
|
+
return avatarBlobs.put(id, Buffer.concat([Buffer.from(`${type}\n`), bytes]))
|
|
40
|
+
},
|
|
41
|
+
async get(id) {
|
|
42
|
+
const bytes = await avatarBlobs.get(id)
|
|
43
|
+
if (!bytes) return null
|
|
44
|
+
const split = bytes.indexOf(10)
|
|
45
|
+
if (split < 0 || split > 100) throw new Error('Invalid cached avatar')
|
|
46
|
+
return { contentType: bytes.subarray(0, split).toString(), bytes: bytes.subarray(split + 1) }
|
|
47
|
+
},
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export async function openManagedVercelStorage(token: string, sdk?: VercelBlobSdk) {
|
|
52
|
+
const blobs = sdk ?? await loadVercelBlobSdk()
|
|
53
|
+
const options = { token, access: 'private' as const, useCache: false }
|
|
54
|
+
async function put(path: string, bytes: Buffer) {
|
|
55
|
+
await blobs.put(path, bytes, { token, access: 'private', addRandomSuffix: false,
|
|
56
|
+
allowOverwrite: true, contentType: 'application/octet-stream', cacheControlMaxAge: 60 })
|
|
57
|
+
}
|
|
58
|
+
async function open(path: string) {
|
|
59
|
+
try {
|
|
60
|
+
const result = await blobs.get(path, options)
|
|
61
|
+
if (!result) return null
|
|
62
|
+
if (result.statusCode !== 200 || !result.stream) throw new Error('Unexpected blob response')
|
|
63
|
+
// Private SDK GETs may report zero for a nonempty body. Stream without
|
|
64
|
+
// Content-Length in that case so HTTP clients do not truncate the body.
|
|
65
|
+
const size = result.blob.size === 0 ? null : result.blob.size
|
|
66
|
+
return { size, stream: Readable.fromWeb(result.stream as Parameters<typeof Readable.fromWeb>[0]) }
|
|
67
|
+
} catch (err) { if (isNotFound(err)) return null; throw err }
|
|
68
|
+
}
|
|
69
|
+
async function get(path: string) {
|
|
70
|
+
const result = await open(path)
|
|
71
|
+
if (!result) return null
|
|
72
|
+
const parts: Buffer[] = []
|
|
73
|
+
for await (const chunk of result.stream) parts.push(Buffer.from(chunk))
|
|
74
|
+
return Buffer.concat(parts)
|
|
75
|
+
}
|
|
76
|
+
async function remove(path: string) {
|
|
77
|
+
try { await blobs.del(path, { token }) } catch (err) { if (!isNotFound(err)) throw err }
|
|
78
|
+
}
|
|
79
|
+
function store(name: string, suffix = ''): BlobStore {
|
|
80
|
+
const path = (id: string) => `.managed/${name}/${validate(id)}${suffix}`
|
|
81
|
+
return {
|
|
82
|
+
async put(id, bytes) { await put(path(id), bytes) },
|
|
83
|
+
async get(id) { return await get(path(id)) },
|
|
84
|
+
open: id => open(path(id)),
|
|
85
|
+
async delete(id) { await remove(path(id)) },
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
const avatarStore = createBlobAvatarStore(store('avatars'))
|
|
89
|
+
const cachePath = (id: string, file: string) => `.managed/cache/bundles/${validate(id)}/${file}`
|
|
90
|
+
const cacheStorage: BundleCacheStorage = {
|
|
91
|
+
async exists(id, file) {
|
|
92
|
+
try { await blobs.head(cachePath(id, file), { token }); return true }
|
|
93
|
+
catch (err) { if (isNotFound(err)) return false; throw err }
|
|
94
|
+
},
|
|
95
|
+
put: (id, file, bytes) => put(cachePath(id, file), bytes),
|
|
96
|
+
async open(id, file) {
|
|
97
|
+
const result = await open(cachePath(id, file))
|
|
98
|
+
if (!result) throw new Error('Bundle cache unavailable')
|
|
99
|
+
return result
|
|
100
|
+
},
|
|
101
|
+
delete: id => deletePrefix(blobs, token, cachePath(id, ''), remove),
|
|
102
|
+
}
|
|
103
|
+
// Staging uploads are never published. An interrupted browser leaves only
|
|
104
|
+
// parts here; the authenticated cron removes them after a full day.
|
|
105
|
+
function reapUploads(now = Date.now()) {
|
|
106
|
+
return deletePrefix(blobs, token, '.managed/uploads/', remove,
|
|
107
|
+
blob => new Date(blob.uploadedAt ?? now).getTime() < now - 86_400_000)
|
|
108
|
+
}
|
|
109
|
+
const sourcesPath = (key: string) => `.managed/cache/report-sources/${validateCacheKey(key)}`
|
|
110
|
+
const reportSourcesStorage: CacheStorage = {
|
|
111
|
+
async exists(key) {
|
|
112
|
+
try { await blobs.head(sourcesPath(key), { token }); return true }
|
|
113
|
+
catch (err) { if (isNotFound(err)) return false; throw err }
|
|
114
|
+
},
|
|
115
|
+
put: (key, bytes) => put(sourcesPath(key), bytes),
|
|
116
|
+
async open(key) {
|
|
117
|
+
const result = await open(sourcesPath(key))
|
|
118
|
+
if (!result) throw new Error('Report sources cache unavailable')
|
|
119
|
+
return result
|
|
120
|
+
},
|
|
121
|
+
delete: prefix => deletePrefix(blobs, token, `${sourcesPath(prefix)}/`, remove),
|
|
122
|
+
}
|
|
123
|
+
const bundleStore = createBundleStore(store('bundles'), store('bundles', '.map.br'))
|
|
124
|
+
return { reportStore: store('reports'), bundleStore, uploadStore: store('uploads'), avatarStore, cacheStorage, reportSourcesStorage, reapUploads }
|
|
125
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { brotliCompress, constants } from 'node:zlib'
|
|
2
|
+
import { promisify } from 'node:util'
|
|
3
|
+
import type { Buffer } from 'node:buffer'
|
|
4
|
+
|
|
5
|
+
const compress = promisify(brotliCompress)
|
|
6
|
+
|
|
7
|
+
// Quality 4 avoids Brotli's expensive default (11) for uploads and cold caches.
|
|
8
|
+
export function encodeBrotli(bytes: Uint8Array): Promise<Buffer> {
|
|
9
|
+
return compress(bytes, { params: { [constants.BROTLI_PARAM_QUALITY]: 4 } })
|
|
10
|
+
}
|
|
@@ -0,0 +1,185 @@
|
|
|
1
|
+
// Metadata is cached as Brotli. Contents use the stored Brotli bytes directly:
|
|
2
|
+
// unchanged Stasis uploads or sourcemaps compressed once at upload.
|
|
3
|
+
import { Buffer } from 'node:buffer'
|
|
4
|
+
import { mkdir, open, rename, rm, stat, writeFile } from 'node:fs/promises'
|
|
5
|
+
import { join } from 'node:path'
|
|
6
|
+
import { randomUUID } from 'node:crypto'
|
|
7
|
+
import { brotliDecompress } from 'node:zlib'
|
|
8
|
+
import { promisify } from 'node:util'
|
|
9
|
+
import { BUNDLE_METADATA_VERSION, type BundleDetails, createBundleMetadata, createBundleSummary, parseBundleContents } from '../common/bundle-metadata.js'
|
|
10
|
+
import { bundleReasons } from '../common/bundle-reasons.js'
|
|
11
|
+
import { bundlePackageVersions } from '../common/bundle-sources.js'
|
|
12
|
+
import { decodeUtf8 } from '../common/utf8.js'
|
|
13
|
+
import type { OpenedBlob } from './blob-store.ts'
|
|
14
|
+
import type { BundleStore } from './bundle-store.ts'
|
|
15
|
+
import { encodeBrotli } from './brotli.ts'
|
|
16
|
+
import type { ManagedBundle, ManagedDb } from './db.ts'
|
|
17
|
+
import { SUMMARY_FILENAME, createBundleSummaryCache } from './bundle-summary-cache.ts'
|
|
18
|
+
|
|
19
|
+
const decompress = promisify(brotliDecompress)
|
|
20
|
+
const MAX_DECODED_BYTES = 512 * 1024 * 1024
|
|
21
|
+
export const MAX_PACKAGE_INVENTORY_BYTES = 1024 * 1024
|
|
22
|
+
export type BundleCachePart = 'metadata' | 'contents'
|
|
23
|
+
export type BundleCacheRecord = Pick<ManagedBundle, 'id' | 'integrity' | 'filename' | 'kind' | 'byteSize'>
|
|
24
|
+
export interface BundleSummary { files: number; codeFiles: number; lines: number }
|
|
25
|
+
|
|
26
|
+
export async function readBundleDetails(record: BundleCacheRecord, store: BundleStore) {
|
|
27
|
+
const bytes = await store.get(record.id, record.kind)
|
|
28
|
+
if (!bytes) return null
|
|
29
|
+
const decoded = record.kind === 'stasis' || record.kind === 'sourcemap' ? await decompress(bytes, { maxOutputLength: MAX_DECODED_BYTES }) : bytes
|
|
30
|
+
if (decoded.length > MAX_DECODED_BYTES) throw new Error('Decoded bundle too large')
|
|
31
|
+
return parseBundleContents(decodeUtf8(decoded), { integrity: record.integrity, kind: record.kind, size: record.byteSize })
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export interface BundleCacheStorage {
|
|
35
|
+
exists(id: string, file: string): Promise<boolean>
|
|
36
|
+
put(id: string, file: string, bytes: Buffer): Promise<void>
|
|
37
|
+
open(id: string, file: string): Promise<OpenedBlob>
|
|
38
|
+
// Remove all cached versions for this bundle, including legacy derivatives.
|
|
39
|
+
delete(id: string): Promise<void>
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
const filename = `v${BUNDLE_METADATA_VERSION}-metadata.json.br`
|
|
43
|
+
const packagesFilename = 'v2-package-versions.json'
|
|
44
|
+
|
|
45
|
+
// All scopes share one bounded derivative. Never decode full bundle metadata
|
|
46
|
+
// on advisory requests, including when selecting a reason. Persist null when
|
|
47
|
+
// the combined inventory exceeds the budget, without disabling metadata.
|
|
48
|
+
function encodePackageInventory(details: BundleDetails): Buffer {
|
|
49
|
+
const parts: string[] = []
|
|
50
|
+
let size = 0
|
|
51
|
+
function append(part: string) {
|
|
52
|
+
size += Buffer.byteLength(part)
|
|
53
|
+
if (size > MAX_PACKAGE_INVENTORY_BYTES) return false
|
|
54
|
+
parts.push(part)
|
|
55
|
+
return true
|
|
56
|
+
}
|
|
57
|
+
function inventory(paths: Set<string> | null) {
|
|
58
|
+
if (!append('{')) return false
|
|
59
|
+
let first = true
|
|
60
|
+
for (const [name, versions] of bundlePackageVersions(details, paths)) {
|
|
61
|
+
if (!append(`${first ? '' : ','}${JSON.stringify(name)}:${JSON.stringify([...versions].toSorted())}`)) return false
|
|
62
|
+
first = false
|
|
63
|
+
}
|
|
64
|
+
return append('}')
|
|
65
|
+
}
|
|
66
|
+
append('{"all":')
|
|
67
|
+
if (!inventory(null) || !append(',"reasons":{')) return Buffer.from('null')
|
|
68
|
+
let first = true
|
|
69
|
+
for (const [reason, paths] of bundleReasons(details)) {
|
|
70
|
+
if (!append(`${first ? '' : ','}${JSON.stringify(reason)}:`) || !inventory(paths)) return Buffer.from('null')
|
|
71
|
+
first = false
|
|
72
|
+
}
|
|
73
|
+
if (!append('}}')) return Buffer.from('null')
|
|
74
|
+
return Buffer.from(parts.join(''))
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
async function build(record: BundleCacheRecord, storage: BundleCacheStorage, db: ManagedDb, store: BundleStore) {
|
|
78
|
+
const details = await readBundleDetails(record, store)
|
|
79
|
+
if (!details) throw new Error('Bundle bytes unavailable')
|
|
80
|
+
const metadata = { ...await createBundleMetadata(details), id: record.id, filename: record.filename }
|
|
81
|
+
const body = await encodeBrotli(Buffer.from(JSON.stringify(metadata)))
|
|
82
|
+
if (!(await db.getBundle(record.id))) throw new Error('Bundle deleted')
|
|
83
|
+
await storage.put(record.id, filename, body)
|
|
84
|
+
if (record.kind === 'stasis') await storage.put(record.id, packagesFilename, encodePackageInventory(details))
|
|
85
|
+
const summary = createBundleSummary(details, metadata)
|
|
86
|
+
await storage.put(record.id, SUMMARY_FILENAME, Buffer.from(JSON.stringify(summary)))
|
|
87
|
+
// A different instance may have deleted the row while these writes ran.
|
|
88
|
+
// Reconcile after publishing so its cleanup cannot be undone by us.
|
|
89
|
+
if (!(await db.getBundle(record.id))) {
|
|
90
|
+
await storage.delete(record.id)
|
|
91
|
+
throw new Error('Bundle deleted')
|
|
92
|
+
}
|
|
93
|
+
return summary
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
export function createBundleCache(storage: BundleCacheStorage, db: ManagedDb, store: BundleStore) {
|
|
97
|
+
const pending = new Map<string, Promise<void>>()
|
|
98
|
+
// The database deduplicates by integrity, so each hash has one persistent
|
|
99
|
+
// bundle/cache directory, shared across teams and repeated uploads.
|
|
100
|
+
const summaries = createBundleSummaryCache(storage, async record => {
|
|
101
|
+
const details = await readBundleDetails(record, store)
|
|
102
|
+
if (!details) throw new Error('Bundle bytes unavailable')
|
|
103
|
+
return createBundleSummary(details)
|
|
104
|
+
}, id => db.getBundle(id))
|
|
105
|
+
let queue = Promise.resolve()
|
|
106
|
+
async function ensure(record: BundleCacheRecord): Promise<void> {
|
|
107
|
+
const existing = pending.get(record.id)
|
|
108
|
+
if (existing) return existing
|
|
109
|
+
const job = (async () => {
|
|
110
|
+
if (await storage.exists(record.id, filename) && (record.kind !== 'stasis' || await storage.exists(record.id, packagesFilename))) return
|
|
111
|
+
const work = queue.then(async () => summaries.remember(record, await build(record, storage, db, store)))
|
|
112
|
+
queue = work.catch(() => {})
|
|
113
|
+
await work
|
|
114
|
+
})()
|
|
115
|
+
pending.set(record.id, job)
|
|
116
|
+
try { await job } finally { if (pending.get(record.id) === job) pending.delete(record.id) }
|
|
117
|
+
}
|
|
118
|
+
return {
|
|
119
|
+
prebuild: ensure,
|
|
120
|
+
summary: summaries.summary,
|
|
121
|
+
summaryStatus: summaries.summaryStatus,
|
|
122
|
+
backfillSummaries: summaries.backfill,
|
|
123
|
+
async open(record: ManagedBundle, part: BundleCachePart) {
|
|
124
|
+
if (part === 'contents') {
|
|
125
|
+
if (record.kind !== 'stasis' && record.kind !== 'sourcemap') throw new Error('Unsupported bundle')
|
|
126
|
+
const stored = await store.open(record.id, record.kind)
|
|
127
|
+
if (!stored) throw new Error('Bundle bytes unavailable')
|
|
128
|
+
return stored
|
|
129
|
+
}
|
|
130
|
+
await ensure(record)
|
|
131
|
+
return storage.open(record.id, filename)
|
|
132
|
+
},
|
|
133
|
+
async packageVersions(record: ManagedBundle, reason = ''): Promise<Record<string, string[]> | null | undefined> {
|
|
134
|
+
if (record.kind !== 'stasis') return {}
|
|
135
|
+
await ensure(record)
|
|
136
|
+
const cached = await storage.open(record.id, packagesFilename)
|
|
137
|
+
try {
|
|
138
|
+
if (cached.size != null && cached.size > MAX_PACKAGE_INVENTORY_BYTES) return null
|
|
139
|
+
const chunks: Buffer[] = []
|
|
140
|
+
let size = 0
|
|
141
|
+
for await (const chunk of cached.stream) {
|
|
142
|
+
size += chunk.length
|
|
143
|
+
if (size > MAX_PACKAGE_INVENTORY_BYTES) return null
|
|
144
|
+
chunks.push(Buffer.from(chunk))
|
|
145
|
+
}
|
|
146
|
+
const inventory = JSON.parse(decodeUtf8(Buffer.concat(chunks)))
|
|
147
|
+
if (inventory === null) return null
|
|
148
|
+
return reason ? (Object.hasOwn(inventory.reasons, reason) ? inventory.reasons[reason] : undefined) : inventory.all
|
|
149
|
+
} finally { cached.stream.destroy() }
|
|
150
|
+
},
|
|
151
|
+
async delete(id: string) {
|
|
152
|
+
// Call after deleting the row. Waiting prevents an in-flight builder
|
|
153
|
+
// from recreating its files after deletion has completed.
|
|
154
|
+
await pending.get(id)?.catch(() => {})
|
|
155
|
+
await summaries.forget(id)
|
|
156
|
+
await storage.delete(id)
|
|
157
|
+
},
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
export type BundleCache = ReturnType<typeof createBundleCache>
|
|
161
|
+
|
|
162
|
+
export function createDiskBundleCache(dir: string, db: ManagedDb, store: BundleStore): BundleCache {
|
|
163
|
+
function directory(id: string) {
|
|
164
|
+
if (!/^[a-f\d-]{36}$/iu.test(id)) throw new Error('Invalid bundle id')
|
|
165
|
+
return join(dir, id)
|
|
166
|
+
}
|
|
167
|
+
return createBundleCache({
|
|
168
|
+
async exists(id, file) {
|
|
169
|
+
try { await stat(join(directory(id), file)); return true }
|
|
170
|
+
catch (err) { if ((err as NodeJS.ErrnoException).code === 'ENOENT') return false; throw err }
|
|
171
|
+
},
|
|
172
|
+
async put(id, file, bytes) {
|
|
173
|
+
await mkdir(directory(id), { recursive: true })
|
|
174
|
+
const target = join(directory(id), file), temp = `${target}.${randomUUID()}.tmp`
|
|
175
|
+
try { await writeFile(temp, bytes); await rename(temp, target) }
|
|
176
|
+
finally { await rm(temp, { force: true }) }
|
|
177
|
+
},
|
|
178
|
+
async open(id, name) {
|
|
179
|
+
const file = await open(join(directory(id), name), 'r')
|
|
180
|
+
try { return { size: (await file.stat()).size, stream: file.createReadStream() } }
|
|
181
|
+
catch (err) { await file.close(); throw err }
|
|
182
|
+
},
|
|
183
|
+
async delete(id) { await rm(directory(id), { recursive: true, force: true }) },
|
|
184
|
+
}, db, store)
|
|
185
|
+
}
|