@preventive/triage 1.0.0-alpha.3 → 1.0.0-alpha.30
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE-APACHE +202 -0
- package/api/reap.ts +17 -0
- package/cli.js +6 -0
- package/client/finding-link.js +305 -0
- package/client/linked-findings.d.ts +1 -0
- package/client/linked-findings.js +111 -0
- package/common/bundle-commit.d.ts +1 -0
- package/common/bundle-commit.js +4 -0
- package/common/bundle-metadata.d.ts +10 -0
- package/common/bundle-metadata.js +182 -0
- package/common/bundle-reasons.d.ts +2 -0
- package/common/bundle-reasons.js +21 -0
- package/common/bundle-source-links.d.ts +5 -0
- package/common/bundle-source-links.js +82 -0
- package/common/bundle-sources.d.ts +3 -0
- package/common/bundle-sources.js +284 -0
- package/common/bundle-stats.js +41 -0
- package/common/bundle-tabs.js +1 -0
- package/common/code-language.js +51 -0
- package/common/default-scan-models.ts +30 -0
- package/common/finding-id.js +47 -0
- package/common/github-issue-labels.d.ts +1 -0
- package/common/github-issue-labels.js +11 -0
- package/common/github-pr.ts +56 -0
- package/common/managed/comments.ts +34 -0
- package/common/managed/permissions.ts +35 -0
- package/common/managed/report-content.ts +42 -0
- package/common/managed/report-filter.ts +108 -0
- package/common/managed/roles.ts +28 -0
- package/common/managed/routes.d.ts +2 -0
- package/common/managed/routes.js +124 -0
- package/common/managed/scan-models.ts +6 -0
- package/common/managed/triage.ts +83 -0
- package/common/save-error-reason.ts +20 -7
- package/common/scan-server.ts +13 -0
- package/common/server-info.ts +35 -0
- package/common/utf8.d.ts +3 -0
- package/common/utf8.js +45 -0
- package/out/brotli-fallback.js +3 -3
- package/out/client-managed-import.js +81 -0
- package/out/client-managed.js +167 -0
- package/out/client-sync.js +16 -13
- package/out/github-issue-authorized.html +3 -0
- package/out/github-issue-authorized.js +1 -0
- package/out/graph.js +30 -4
- package/out/index.html +55 -8
- package/out/prism.js +2 -2
- package/out/stasis.svg +20 -0
- package/out/terminal.js +273 -39
- package/out/view.css +1 -1
- package/out/view.js +259 -63
- package/package.json +205 -57
- package/server-common/database-config.ts +16 -0
- package/server-common/initialize.ts +18 -0
- package/server-common/npm-advisories.ts +101 -0
- package/{server → server-common}/origin.ts +5 -5
- package/server-common/reap.ts +48 -0
- package/server-common/scan-config.ts +19 -0
- package/server-common/standalone.ts +29 -0
- package/server-common/storage-crypto.ts +182 -0
- package/server-common/storage-log.ts +38 -0
- package/server-common/vercel-blob.ts +102 -0
- package/server-e2e/app.ts +486 -0
- package/{server → server-e2e}/auth.ts +5 -1
- package/{server → server-e2e}/bus-receiver.ts +2 -1
- package/{server → server-e2e}/cli.js +9 -4
- package/{server → server-e2e}/config.ts +56 -35
- package/{server → server-e2e}/db-neon.ts +2 -2
- package/{server → server-e2e}/db-revision-sql.ts +1 -1
- package/{server → server-e2e}/db-stmt.ts +2 -2
- package/{server → server-e2e}/db.ts +7 -7
- package/{server → server-e2e}/http.ts +110 -12
- package/{server → server-e2e}/hub.ts +40 -9
- package/server-e2e/index.ts +17 -0
- package/server-e2e/lifecycle.ts +95 -0
- package/{server → server-e2e}/neon-driver.ts +2 -2
- package/{server → server-e2e}/npm-proxy.ts +11 -144
- package/{server → server-e2e}/objstore/blob-fs.ts +2 -2
- package/{server → server-e2e}/objstore/blob-vercel.ts +13 -117
- package/{server → server-e2e}/objstore/blob.ts +9 -2
- package/server-e2e/objstore/fetch-mint-guard.ts +74 -0
- package/{server → server-e2e}/objstore/handlers.ts +9 -8
- package/{server → server-e2e}/objstore/init.ts +35 -18
- package/{server → server-e2e}/objstore/reaper.ts +31 -11
- package/server-e2e/objstore/rest-deny.ts +28 -0
- package/server-e2e/objstore/rest-mint.ts +224 -0
- package/{server → server-e2e}/objstore/rest.ts +58 -33
- package/{server → server-e2e}/objstore/sign.ts +105 -0
- package/{server → server-e2e}/objstore/store-neon.ts +6 -10
- package/{server → server-e2e}/objstore/store.ts +7 -7
- package/{server → server-e2e}/objstore/tokens.ts +1 -1
- package/{server → server-e2e}/pubsub.ts +12 -9
- package/{server → server-e2e}/sign.ts +4 -4
- package/{server → server-e2e}/sse-server.ts +102 -68
- package/{server → server-e2e}/sse-session.ts +28 -8
- package/{server → server-e2e}/static.ts +31 -21
- package/server-e2e/sync-handlers.ts +408 -0
- package/{server → server-e2e}/util.ts +9 -0
- package/{server → server-e2e}/ws-server.ts +20 -9
- package/server-managed/activity.ts +231 -0
- package/server-managed/avatar-store.ts +12 -0
- package/server-managed/blob-store.ts +17 -0
- package/server-managed/brotli.ts +10 -0
- package/server-managed/bundle-advisories.ts +53 -0
- package/server-managed/bundle-advisory-inventory.ts +75 -0
- package/server-managed/bundle-build-diagnostics.d.ts +10 -0
- package/server-managed/bundle-build-diagnostics.js +23 -0
- package/server-managed/bundle-build-leases.ts +41 -0
- package/server-managed/bundle-build-worker.js +54 -0
- package/server-managed/bundle-build.ts +168 -0
- package/server-managed/bundle-cache.ts +166 -0
- package/server-managed/bundle-catalog.ts +29 -0
- package/server-managed/bundle-store.ts +24 -0
- package/server-managed/bundle-summary-cache.ts +116 -0
- package/server-managed/bundle.ts +104 -0
- package/server-managed/cache-storage.ts +19 -0
- package/server-managed/cli.js +21 -0
- package/server-managed/combined.ts +46 -0
- package/server-managed/comments.ts +151 -0
- package/server-managed/config.ts +149 -0
- package/server-managed/content-access.ts +15 -0
- package/server-managed/crypto.ts +25 -0
- package/server-managed/db-methods.ts +1517 -0
- package/server-managed/db-neon.ts +277 -0
- package/server-managed/db-schema.ts +206 -0
- package/server-managed/db-table-names.ts +22 -0
- package/server-managed/db.ts +120 -0
- package/server-managed/github-app.ts +385 -0
- package/server-managed/github-errors.ts +49 -0
- package/server-managed/github-issue-oauth.ts +48 -0
- package/server-managed/github-issues.ts +152 -0
- package/server-managed/github-metadata.ts +98 -0
- package/server-managed/github-oauth.ts +199 -0
- package/server-managed/github-pulls.ts +227 -0
- package/server-managed/http-response.ts +18 -0
- package/server-managed/http.ts +2350 -0
- package/server-managed/import-triage.ts +51 -0
- package/server-managed/index.ts +175 -0
- package/server-managed/managed-issues.ts +47 -0
- package/server-managed/management-catalog.ts +45 -0
- package/server-managed/management.ts +86 -0
- package/server-managed/object-storage-disk.ts +144 -0
- package/server-managed/object-storage-vercel.ts +60 -0
- package/server-managed/object-storage.ts +71 -0
- package/server-managed/package-entry-points.ts +51 -0
- package/server-managed/public-workspace.ts +146 -0
- package/server-managed/repo-path.ts +21 -0
- package/server-managed/report-migration.ts +35 -0
- package/server-managed/report-query.ts +10 -0
- package/server-managed/report-response.ts +16 -0
- package/server-managed/report-sources.ts +166 -0
- package/server-managed/repository-browser.ts +139 -0
- package/server-managed/repository-discovery.ts +82 -0
- package/server-managed/repository-policy.ts +27 -0
- package/server-managed/revisions.ts +35 -0
- package/server-managed/session.ts +78 -0
- package/server-managed/slugs.ts +38 -0
- package/server-managed/solidity-entry-points.ts +69 -0
- package/server-managed/sql-postgres.ts +31 -0
- package/server-managed/sql.ts +72 -0
- package/server-managed/static.ts +28 -0
- package/server-managed/storage-command.ts +16 -0
- package/server-managed/storage-db.ts +175 -0
- package/server-managed/storage-encryption.ts +211 -0
- package/server-managed/storage-maintenance.ts +223 -0
- package/server-managed/storage-payload.ts +72 -0
- package/server-managed/storage-stores.ts +81 -0
- package/server-managed/storage.ts +58 -0
- package/server-managed/team-catalog.ts +7 -0
- package/server-managed/team-feed.ts +173 -0
- package/server-managed/team-reports.ts +219 -0
- package/server-managed/triage-response.ts +16 -0
- package/server-managed/uploads.ts +65 -0
- package/server-managed/workspace-shares.ts +162 -0
- package/server.ts +50 -0
- package/server/index.ts +0 -477
- package/server/lifecycle.ts +0 -204
- package/server/sync-handlers.ts +0 -320
- /package/{LICENSE → LICENSE-MIT} +0 -0
- /package/{server → server-e2e}/config.example.json +0 -0
- /package/{server → server-e2e}/objstore/fs.ts +0 -0
- /package/{server → server-e2e}/peer.ts +0 -0
- /package/{server → server-e2e}/validation.ts +0 -0
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto'
|
|
2
|
+
import type { ManagedDb } from './db-methods.ts'
|
|
3
|
+
import type { ManagedComment } from '../common/managed/comments.ts'
|
|
4
|
+
import type { TriageEntryPatch } from '../common/managed/triage.ts'
|
|
5
|
+
import { triageWireEntry } from './triage-response.ts'
|
|
6
|
+
|
|
7
|
+
export interface ImportTriageSnapshot {
|
|
8
|
+
entry: TriageEntryPatch | null
|
|
9
|
+
comments: ManagedComment[]
|
|
10
|
+
version: string
|
|
11
|
+
}
|
|
12
|
+
export interface ImportTriageStore {
|
|
13
|
+
getImportTriage(ids: readonly string[]): Promise<Record<string, ImportTriageSnapshot>>
|
|
14
|
+
importTriage(entries: [string, TriageEntryPatch | null][], expected: Record<string, string>, actor: { id: string; login: string }, reportId: string, now: number): Promise<boolean>
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
// Raw store methods share one transaction for comparison, triage, and comments.
|
|
18
|
+
export function importTriageMethods(store: Pick<ManagedDb, 'getReport' | 'listTriage' | 'listComments' | 'setTriageEntries' | 'createComment'>): ImportTriageStore {
|
|
19
|
+
async function snapshot(ids: readonly string[]) {
|
|
20
|
+
const rows = new Map((await store.listTriage(ids)).map(row => [row.findingId, row]))
|
|
21
|
+
const comments = await store.listComments(ids)
|
|
22
|
+
return Object.fromEntries(ids.map(id => {
|
|
23
|
+
const row = rows.get(id)
|
|
24
|
+
const entry = row ? triageWireEntry(row) : null
|
|
25
|
+
const own = comments.filter(comment => comment.findingId === id)
|
|
26
|
+
const version = createHash('sha256').update(JSON.stringify([entry, row?.updatedAt, own])).digest('hex')
|
|
27
|
+
return [id, { entry, comments: own, version }]
|
|
28
|
+
}))
|
|
29
|
+
}
|
|
30
|
+
return {
|
|
31
|
+
getImportTriage: snapshot,
|
|
32
|
+
async importTriage(entries, expected, actor, reportId, now) {
|
|
33
|
+
// A report deleted after the HTTP membership check must not leave newly
|
|
34
|
+
// imported annotations behind. Check existence in the write transaction.
|
|
35
|
+
if (!reportId || !(await store.getReport(reportId))) return false
|
|
36
|
+
const current = await snapshot(entries.map(([id]) => id))
|
|
37
|
+
if (entries.some(([id]) => current[id]!.version !== expected[id])) return false
|
|
38
|
+
const changes: [string, TriageEntryPatch | null][] = []
|
|
39
|
+
for (const [id, value] of entries) {
|
|
40
|
+
const { comment, ...entry } = value ?? {}
|
|
41
|
+
changes.push([id, Object.keys(entry).length > 0 ? entry : null])
|
|
42
|
+
if (comment && !current[id]!.comments.some(existing => existing.body === comment)) {
|
|
43
|
+
await store.createComment({ findingId: id, body: comment, authorId: null, authorLogin: null,
|
|
44
|
+
actor, createdAt: null, updatedAt: null, reportId }, now)
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
await store.setTriageEntries(changes, actor.id, actor.login, now, reportId)
|
|
48
|
+
return true
|
|
49
|
+
},
|
|
50
|
+
}
|
|
51
|
+
}
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
// Managed HTTP app and standalone boot. The combined launcher mounts this
|
|
2
|
+
// same app on e2e's listener; storage, routing and cleanup stay here.
|
|
3
|
+
import { randomUUID } from 'node:crypto'
|
|
4
|
+
import { type Server, createServer } from 'node:http'
|
|
5
|
+
import { fileURLToPath } from 'node:url'
|
|
6
|
+
import { createOriginGate } from '../server-common/origin.ts'
|
|
7
|
+
import { managedStorageLines } from '../server-common/storage-log.ts'
|
|
8
|
+
import { runReapers, withReap } from '../server-common/reap.ts'
|
|
9
|
+
import { startServer } from '../server-common/standalone.ts'
|
|
10
|
+
import { initializeApp } from '../server-common/initialize.ts'
|
|
11
|
+
import { type ManagedConfig, loadManagedConfig } from './config.ts'
|
|
12
|
+
import { type ManagedHttpDeps, createManagedRequestHandler } from './http.ts'
|
|
13
|
+
import { loadManagedStatic } from './static.ts'
|
|
14
|
+
import { openManagedStorage } from './storage.ts'
|
|
15
|
+
|
|
16
|
+
// Housekeeping runs on ordinary traffic too, including serverless instances
|
|
17
|
+
// without a timer or cron. Failed sweeps retry on later traffic after a minute.
|
|
18
|
+
const REAP_INTERVAL_MS = 3_600_000
|
|
19
|
+
const REAP_RETRY_MS = 60_000
|
|
20
|
+
|
|
21
|
+
type ManagedAppOptions = Partial<Pick<ManagedHttpDeps, 'next' | 'serverInfo' | 'isShuttingDown'>>
|
|
22
|
+
|
|
23
|
+
export async function createManagedApp(config: ManagedConfig, options: ManagedAppOptions = {}) {
|
|
24
|
+
return await initializeApp(rollback => assembleManagedApp(config, options, rollback))
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
async function assembleManagedApp(config: ManagedConfig, options: ManagedAppOptions, rollback: AsyncDisposableStack) {
|
|
28
|
+
const storage = await openManagedStorage(config)
|
|
29
|
+
const { db, avatarStore, reportStore, bundleStore, bundleCache, reportSourcesCache } = storage
|
|
30
|
+
rollback.defer(() => db.close())
|
|
31
|
+
const originGate = createOriginGate(config.host, config.trustProxyEnv)
|
|
32
|
+
|
|
33
|
+
let shuttingDown = false
|
|
34
|
+
const isShuttingDown = () => shuttingDown || options.isShuttingDown?.() === true
|
|
35
|
+
const inFlight = new Set<Promise<unknown>>()
|
|
36
|
+
function track(p: Promise<unknown>): void {
|
|
37
|
+
inFlight.add(p)
|
|
38
|
+
p.finally(() => inFlight.delete(p)).catch(() => {})
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
const serveStatic = loadManagedStatic(fileURLToPath(new URL('../out/', import.meta.url)), {
|
|
42
|
+
indexOnly: options.next != null, scanServer: options.serverInfo?.deepviewScanServer ?? null,
|
|
43
|
+
})
|
|
44
|
+
const routeRequest = createManagedRequestHandler({
|
|
45
|
+
...options, config, db, avatarStore, reportStore, bundleStore, bundleCache, reportSourcesCache, originGate, serveStatic,
|
|
46
|
+
...(storage.uploadStore ? { uploadStore: storage.uploadStore } : {}),
|
|
47
|
+
isShuttingDown, track,
|
|
48
|
+
})
|
|
49
|
+
|
|
50
|
+
let cleanup: Promise<void> | undefined
|
|
51
|
+
let nextCleanupAt = 0
|
|
52
|
+
const maintenance = new AbortController()
|
|
53
|
+
function reap(automatic = false): Promise<void> {
|
|
54
|
+
if (cleanup) return cleanup
|
|
55
|
+
if (isShuttingDown()) return Promise.resolve()
|
|
56
|
+
const startedAt = Date.now()
|
|
57
|
+
nextCleanupAt = startedAt + REAP_INTERVAL_MS
|
|
58
|
+
let sessions = 0, uploads = 0
|
|
59
|
+
cleanup = (async () => {
|
|
60
|
+
const owner = automatic && config.serverless ? randomUUID() : null
|
|
61
|
+
if (owner && !await db.claimMaintenanceLease(owner, startedAt, startedAt + Math.max(300_000, (config.storageEncryptionMigrateMaxMs ?? 0) + 60_000), config.storageEncryptionMigrate)) return false
|
|
62
|
+
let succeeded = false
|
|
63
|
+
try {
|
|
64
|
+
await runReapers({
|
|
65
|
+
sessions: async () => { sessions = await db.deleteExpiredSessions(Date.now()) },
|
|
66
|
+
...(storage.reapUploads ? { uploads: async () => { uploads = await storage.reapUploads!(maintenance.signal) } } : {}),
|
|
67
|
+
...('reapStorage' in storage ? { storage: () => storage.reapStorage(maintenance.signal) } : {}),
|
|
68
|
+
})
|
|
69
|
+
succeeded = true
|
|
70
|
+
return true
|
|
71
|
+
} finally {
|
|
72
|
+
if (owner) await db.finishMaintenanceLease(owner, Date.now() + (succeeded ? REAP_INTERVAL_MS : REAP_RETRY_MS))
|
|
73
|
+
}
|
|
74
|
+
})().then(ran => {
|
|
75
|
+
nextCleanupAt = Date.now() + REAP_INTERVAL_MS
|
|
76
|
+
if (!ran) return
|
|
77
|
+
return console.info(`managed-reaper: removed ${sessions} expired session(s), ${uploads} stale upload part(s) in ${Date.now() - startedAt}ms`)
|
|
78
|
+
}, err => {
|
|
79
|
+
nextCleanupAt = Date.now() + REAP_RETRY_MS
|
|
80
|
+
throw err
|
|
81
|
+
}).finally(() => { cleanup = undefined })
|
|
82
|
+
track(cleanup)
|
|
83
|
+
return cleanup
|
|
84
|
+
}
|
|
85
|
+
function automaticReap(): Promise<void> {
|
|
86
|
+
if (isShuttingDown() || cleanup || Date.now() < nextCleanupAt) return Promise.resolve()
|
|
87
|
+
return reap(true).catch(err => console.warn('managed-reaper: cleanup failed:', err))
|
|
88
|
+
}
|
|
89
|
+
async function handleRequest(req: Parameters<typeof routeRequest>[0], res: Parameters<typeof routeRequest>[1]): Promise<void> {
|
|
90
|
+
const pendingMaintenance = automaticReap()
|
|
91
|
+
// Send the normal response without waiting for storage housekeeping. The
|
|
92
|
+
// Vercel entrypoint registers this whole promise with waitUntil; awaiting
|
|
93
|
+
// work after res.end() alone cannot extend a serverless invocation.
|
|
94
|
+
try { await routeRequest(req, res) }
|
|
95
|
+
finally { await pendingMaintenance }
|
|
96
|
+
}
|
|
97
|
+
const gcTimer = config.serverless ? null : setInterval(() => {
|
|
98
|
+
void automaticReap()
|
|
99
|
+
}, REAP_INTERVAL_MS)
|
|
100
|
+
rollback.defer(stop)
|
|
101
|
+
|
|
102
|
+
function stop(): void {
|
|
103
|
+
shuttingDown = true
|
|
104
|
+
maintenance.abort()
|
|
105
|
+
if (gcTimer) clearInterval(gcTimer)
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
async function close(): Promise<void> {
|
|
109
|
+
stop()
|
|
110
|
+
if (inFlight.size > 0) await Promise.allSettled([...inFlight])
|
|
111
|
+
await db.close()
|
|
112
|
+
}
|
|
113
|
+
return { handleRequest, reap, isShuttingDown, stop, close }
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
export function logManagedStartup(config: ManagedConfig, port: number, mode = 'managed'): void {
|
|
117
|
+
console.log([
|
|
118
|
+
`DeepView managed server (${mode}):`,
|
|
119
|
+
` HTTP: http://${config.host}:${port}/`,
|
|
120
|
+
...managedStorageLines(config),
|
|
121
|
+
].join('\n'))
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
export async function init(config = loadManagedConfig()): Promise<Server> {
|
|
125
|
+
const app = await createManagedApp(config)
|
|
126
|
+
const server = createServer(withReap(app.handleRequest, { managed: app.reap }, { isShuttingDown: app.isShuttingDown }))
|
|
127
|
+
let disposal: Promise<void> | undefined
|
|
128
|
+
function dispose(): Promise<void> {
|
|
129
|
+
if (!disposal) {
|
|
130
|
+
app.stop()
|
|
131
|
+
disposal = (async () => {
|
|
132
|
+
try {
|
|
133
|
+
if (server.listening) {
|
|
134
|
+
await new Promise<void>((resolve, reject) => {
|
|
135
|
+
server.close(err => { if (err) reject(err); else resolve() })
|
|
136
|
+
})
|
|
137
|
+
}
|
|
138
|
+
} finally {
|
|
139
|
+
try { await app.close() }
|
|
140
|
+
finally {
|
|
141
|
+
server.off('error', onError)
|
|
142
|
+
server.off('close', onClose)
|
|
143
|
+
server.off('listening', onListening)
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
})()
|
|
147
|
+
}
|
|
148
|
+
return disposal
|
|
149
|
+
}
|
|
150
|
+
// Embedding hosts can await full cleanup even when they only copied the
|
|
151
|
+
// request listener and never bound this server. Repeated disposal is safe.
|
|
152
|
+
server[Symbol.asyncDispose] = dispose
|
|
153
|
+
|
|
154
|
+
function onError(): void { onClose() }
|
|
155
|
+
function onClose(): void {
|
|
156
|
+
if (!disposal) void dispose().catch(err => { console.error('Managed server cleanup failed:', err) })
|
|
157
|
+
}
|
|
158
|
+
function onListening(): void {
|
|
159
|
+
const address = server.address()
|
|
160
|
+
const port = typeof address === 'object' && address ? address.port : config.port
|
|
161
|
+
logManagedStartup(config, port)
|
|
162
|
+
}
|
|
163
|
+
server.on('error', onError)
|
|
164
|
+
server.once('close', onClose)
|
|
165
|
+
server.on('listening', onListening)
|
|
166
|
+
return server
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
export async function start(): Promise<void> {
|
|
170
|
+
const config = loadManagedConfig()
|
|
171
|
+
const server = await init(config)
|
|
172
|
+
startServer(server, config)
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
if (import.meta.main) await start()
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import type { ManagedSql } from './sql.ts'
|
|
2
|
+
|
|
3
|
+
// One immutable reference per finding, shared across reports and teams. Keep
|
|
4
|
+
// reservations across crashes/uncertain GitHub responses to prevent duplicates.
|
|
5
|
+
// No triage/import/admin patch accepts these fields.
|
|
6
|
+
export const MANAGED_ISSUE_SCHEMA = `
|
|
7
|
+
CREATE TABLE IF NOT EXISTS managed_finding_issue (
|
|
8
|
+
finding_id TEXT PRIMARY KEY,
|
|
9
|
+
repo_id INTEGER NOT NULL,
|
|
10
|
+
repository TEXT NOT NULL,
|
|
11
|
+
request_id TEXT NOT NULL,
|
|
12
|
+
issue_url TEXT,
|
|
13
|
+
created_by TEXT REFERENCES managed_user(id) ON DELETE SET NULL,
|
|
14
|
+
created_at INTEGER NOT NULL
|
|
15
|
+
) STRICT;
|
|
16
|
+
`
|
|
17
|
+
export interface ManagedIssue {
|
|
18
|
+
findingId: string
|
|
19
|
+
repoId: number
|
|
20
|
+
repository: string
|
|
21
|
+
requestId: string
|
|
22
|
+
issueUrl: string | null
|
|
23
|
+
createdBy: string | null
|
|
24
|
+
createdAt: number
|
|
25
|
+
}
|
|
26
|
+
export interface ManagedIssueStore {
|
|
27
|
+
getManagedIssue(findingId: string): Promise<ManagedIssue | null>
|
|
28
|
+
claimManagedIssue(issue: Omit<ManagedIssue, 'issueUrl'>): Promise<boolean>
|
|
29
|
+
finishManagedIssue(findingId: string, requestId: string, url: string): Promise<boolean>
|
|
30
|
+
releaseManagedIssue(findingId: string, requestId: string): Promise<void>
|
|
31
|
+
}
|
|
32
|
+
export function managedIssueMethods(db: ManagedSql): ManagedIssueStore {
|
|
33
|
+
const get = db.prepare(`SELECT finding_id AS findingId, repo_id AS repoId, repository, request_id AS requestId,
|
|
34
|
+
issue_url AS issueUrl, created_by AS createdBy, created_at AS createdAt FROM managed_finding_issue WHERE finding_id = ?`)
|
|
35
|
+
const claim = db.prepare(`INSERT INTO managed_finding_issue (finding_id, repo_id, repository, request_id, created_by, created_at)
|
|
36
|
+
VALUES (?, ?, ?, ?, ?, ?) ON CONFLICT (finding_id) DO NOTHING`)
|
|
37
|
+
const finish = db.prepare(`UPDATE managed_finding_issue SET issue_url = ? WHERE finding_id = ? AND request_id = ? AND issue_url IS NULL`)
|
|
38
|
+
const release = db.prepare(`DELETE FROM managed_finding_issue WHERE finding_id = ? AND request_id = ? AND issue_url IS NULL`)
|
|
39
|
+
return {
|
|
40
|
+
async getManagedIssue(findingId) { return (await get.get(findingId) as ManagedIssue | undefined) ?? null },
|
|
41
|
+
async claimManagedIssue(issue) {
|
|
42
|
+
return Number((await claim.run(issue.findingId, issue.repoId, issue.repository, issue.requestId, issue.createdBy, issue.createdAt)).changes) > 0
|
|
43
|
+
},
|
|
44
|
+
async finishManagedIssue(findingId, requestId, url) { return Number((await finish.run(url, findingId, requestId)).changes) > 0 },
|
|
45
|
+
async releaseManagedIssue(findingId, requestId) { await release.run(findingId, requestId) },
|
|
46
|
+
}
|
|
47
|
+
}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { roleAtLeast } from '../common/managed/roles.ts'
|
|
2
|
+
import { type RepoScope, contentAccess } from './content-access.ts'
|
|
3
|
+
import type { AdminBundle, AdminReport, ManagedDb } from './db-methods.ts'
|
|
4
|
+
import { ManagedMutationError } from './management.ts'
|
|
5
|
+
|
|
6
|
+
type CatalogAccess = { repos: { repoId: number; fullName: string }[]; repoScopes: RepoScope[] | null }
|
|
7
|
+
export interface ManagementCatalogStore {
|
|
8
|
+
getReportCatalog(sessionId: string, now: number): Promise<CatalogAccess & { reports: (AdminReport & { canChangeRepo: boolean })[] }>
|
|
9
|
+
getBundleCatalog(sessionId: string, now: number): Promise<CatalogAccess & { bundles: (AdminBundle & { canChangeRepo: boolean })[] }>
|
|
10
|
+
}
|
|
11
|
+
type CatalogDb = Pick<ManagedDb, 'getReportAccessSnapshot' | 'listRepoScopesForUser' | 'listSelectedRepos'
|
|
12
|
+
| 'listReports' | 'listBundles' | 'listReadableBundleIds'>
|
|
13
|
+
|
|
14
|
+
// The outer store method owns one read snapshot. Use raw methods here so the
|
|
15
|
+
// session, grants, rows and action flags agree without per-row transactions.
|
|
16
|
+
export function managementCatalogMethods(db: CatalogDb): ManagementCatalogStore {
|
|
17
|
+
async function access(sessionId: string, now: number) {
|
|
18
|
+
const snapshot = await db.getReportAccessSnapshot(sessionId, now, [])
|
|
19
|
+
if (!snapshot) throw new ManagedMutationError(401, 'unauthenticated')
|
|
20
|
+
if (!roleAtLeast(snapshot.user.role, 'manage')) throw new ManagedMutationError(403, 'forbidden')
|
|
21
|
+
const scope = await contentAccess(db, snapshot.user)
|
|
22
|
+
const repos = (await db.listSelectedRepos()).filter(scope.bundle).map(({ repoId, fullName }) => ({ repoId, fullName }))
|
|
23
|
+
return { userId: snapshot.user.role === 'admin' ? undefined : snapshot.user.id, scope, repos }
|
|
24
|
+
}
|
|
25
|
+
return {
|
|
26
|
+
async getReportCatalog(sessionId, now) {
|
|
27
|
+
const { userId, scope, repos } = await access(sessionId, now)
|
|
28
|
+
const reports = await db.listReports(userId)
|
|
29
|
+
const linked = reports.flatMap(report => report.bundleId ? [report.bundleId] : [])
|
|
30
|
+
const readable = userId === undefined ? new Set(linked) : new Set(await db.listReadableBundleIds(userId, linked))
|
|
31
|
+
return { repos, repoScopes: scope.scopes, reports: reports.map(report => ({ ...report,
|
|
32
|
+
canChangeRepo: report.repoId === null || scope.report(report),
|
|
33
|
+
bundleId: report.bundleId && readable.has(report.bundleId) ? report.bundleId : null,
|
|
34
|
+
bundleFilename: report.bundleId && readable.has(report.bundleId) ? report.bundleFilename : null,
|
|
35
|
+
})) }
|
|
36
|
+
},
|
|
37
|
+
async getBundleCatalog(sessionId, now) {
|
|
38
|
+
const { userId, scope, repos } = await access(sessionId, now)
|
|
39
|
+
const bundles = await db.listBundles(userId)
|
|
40
|
+
return { repos, repoScopes: scope.scopes, bundles: bundles.map(bundle => ({ ...bundle,
|
|
41
|
+
canChangeRepo: bundle.repoId === null || scope.report(bundle),
|
|
42
|
+
})) }
|
|
43
|
+
},
|
|
44
|
+
}
|
|
45
|
+
}
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import type { AdminReport, ManagedBundle, ManagedDb, ManagedRepo, RepoDataItem, RepoReportItem, ReportRecord, StoredUser } from './db-methods.ts'
|
|
2
|
+
import { roleAtLeast } from '../common/managed/roles.ts'
|
|
3
|
+
|
|
4
|
+
export class ManagedMutationError extends Error {
|
|
5
|
+
status: number
|
|
6
|
+
constructor(status: number, message: string) { super(message); this.status = status }
|
|
7
|
+
}
|
|
8
|
+
|
|
9
|
+
export type ContentMutation = { type: 'delete' } | { type: 'repo'; repoId: number | null; directory: string }
|
|
10
|
+
export type ReportMutation = ContentMutation | { type: 'visibility'; visible: boolean }
|
|
11
|
+
type RemovalRepo = Pick<ManagedRepo, 'repoId' | 'fullName' | 'addedAt'>
|
|
12
|
+
type RemovalAnnotations = { reports: string; ids: string[] }
|
|
13
|
+
export interface ManagementStore {
|
|
14
|
+
mutateReport(sessionId: string, id: string, change: ReportMutation): Promise<{ report: ReportRecord; user: StoredUser }>
|
|
15
|
+
mutateBundle(sessionId: string, id: string, change: ContentMutation): Promise<{ bundle: ManagedBundle; user: StoredUser }>
|
|
16
|
+
removeRepository(sessionId: string, expected: RemovalRepo, annotations: RemovalAnnotations | null): Promise<{
|
|
17
|
+
reports: RepoReportItem[]; bundles: RepoDataItem[]; deletedReports: number; deletedBundles: number; deletedTriage: number
|
|
18
|
+
}>
|
|
19
|
+
}
|
|
20
|
+
type ManagementDb = Pick<ManagedDb, 'sessionWithUser' | 'getReport' | 'getBundle' | 'listReports' | 'listReportsForRepo'
|
|
21
|
+
| 'listBundlesForRepo' | 'listAllRepos' | 'listSelectedRepos' | 'userCanReadReport' | 'userCanReadBundle' | 'userCanReadRepoPath'
|
|
22
|
+
| 'deleteReport' | 'setReportRepo' | 'setReportVisible' | 'deleteBundle' | 'setBundleRepo'
|
|
23
|
+
| 'deleteReportsForRepo' | 'deleteBundlesForRepo' | 'deleteRepo' | 'deleteTriage' | 'recordActivity'>
|
|
24
|
+
|
|
25
|
+
// Only immutable bytes/format and repository membership affect the overlap
|
|
26
|
+
// scan. Ignore publication, user display names, and unrelated bundle links.
|
|
27
|
+
export function reportReferenceSnapshot(reports: readonly Pick<AdminReport, 'id' | 'sha256' | 'filename' | 'repoId'>[]): string {
|
|
28
|
+
return JSON.stringify(reports.map(r => [r.id, r.sha256, r.filename, r.repoId]).toSorted((a, b) => String(a[0]).localeCompare(String(b[0]))))
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
// These methods are wrapped by scopeManagedMethods: authorization, comparison,
|
|
32
|
+
// metadata changes (and repository-removal audit) share the writer transaction. Blob reads
|
|
33
|
+
// and deletes stay outside it. Use the raw store here, never nested scopes.
|
|
34
|
+
export function managementMethods(db: ManagementDb): ManagementStore {
|
|
35
|
+
async function authorize(sessionId: string, role: 'admin' | 'manage') {
|
|
36
|
+
const s = await db.sessionWithUser(sessionId, Date.now())
|
|
37
|
+
if (!s) throw new ManagedMutationError(401, 'unauthenticated')
|
|
38
|
+
if (!roleAtLeast(s.user.role, role)) throw new ManagedMutationError(403, 'forbidden')
|
|
39
|
+
return s.user
|
|
40
|
+
}
|
|
41
|
+
async function destination(user: { id: string; role: string }, change: ContentMutation) {
|
|
42
|
+
if (change.type !== 'repo' || change.repoId === null) return
|
|
43
|
+
if (!(await db.listSelectedRepos()).some(repo => repo.repoId === change.repoId)) throw new ManagedMutationError(400, 'bad-repo')
|
|
44
|
+
if (user.role !== 'admin' && !await db.userCanReadRepoPath(user.id, change.repoId, change.directory)) throw new ManagedMutationError(403, 'repo-forbidden')
|
|
45
|
+
}
|
|
46
|
+
return {
|
|
47
|
+
async mutateReport(sessionId: string, id: string, change: ReportMutation) {
|
|
48
|
+
const user = await authorize(sessionId, 'manage')
|
|
49
|
+
const report = await db.getReport(id)
|
|
50
|
+
if (!report || (user.role !== 'admin' && report.uploadedBy !== user.id && !await db.userCanReadReport(user.id, id))) throw new ManagedMutationError(404, 'no-report')
|
|
51
|
+
if (user.role !== 'admin' && report.repoId !== null && !await db.userCanReadReport(user.id, id)) throw new ManagedMutationError(403, 'repo-forbidden')
|
|
52
|
+
if (change.type === 'repo' && report.repoEmbedded) throw new ManagedMutationError(409, 'repo-in-report')
|
|
53
|
+
if (change.type !== 'visibility') await destination(user, change)
|
|
54
|
+
if (change.type === 'delete') await db.deleteReport(id)
|
|
55
|
+
else if (change.type === 'visibility') await db.setReportVisible(id, change.visible)
|
|
56
|
+
else await db.setReportRepo(id, change.repoId, change.directory)
|
|
57
|
+
return { report, user }
|
|
58
|
+
},
|
|
59
|
+
async mutateBundle(sessionId: string, id: string, change: ContentMutation) {
|
|
60
|
+
const user = await authorize(sessionId, 'manage')
|
|
61
|
+
const bundle = await db.getBundle(id)
|
|
62
|
+
if (!bundle || (user.role !== 'admin' && !await db.userCanReadBundle(user.id, id))) throw new ManagedMutationError(404, 'no-bundle')
|
|
63
|
+
if (user.role !== 'admin' && bundle.repoId !== null && !await db.userCanReadRepoPath(user.id, bundle.repoId, bundle.repoDirectory)) throw new ManagedMutationError(403, 'repo-forbidden')
|
|
64
|
+
await destination(user, change)
|
|
65
|
+
if (change.type === 'delete') await db.deleteBundle(id)
|
|
66
|
+
else await db.setBundleRepo(id, change.repoId, change.directory)
|
|
67
|
+
return { bundle, user }
|
|
68
|
+
},
|
|
69
|
+
async removeRepository(sessionId: string, expected: RemovalRepo, annotations: RemovalAnnotations | null) {
|
|
70
|
+
const user = await authorize(sessionId, 'admin')
|
|
71
|
+
const repo = (await db.listAllRepos()).find(r => r.repoId === expected.repoId)
|
|
72
|
+
if (!repo) throw new ManagedMutationError(404, 'no-repo')
|
|
73
|
+
if (repo.fullName !== expected.fullName || repo.addedAt !== expected.addedAt) throw new ManagedMutationError(409, 'repository-changed')
|
|
74
|
+
if (annotations && reportReferenceSnapshot(await db.listReports()) !== annotations.reports) throw new ManagedMutationError(409, 'repository-changed')
|
|
75
|
+
const reports = await db.listReportsForRepo(repo.repoId)
|
|
76
|
+
const bundles = await db.listBundlesForRepo(repo.repoId)
|
|
77
|
+
const deletedReports = await db.deleteReportsForRepo(repo.repoId)
|
|
78
|
+
const deletedBundles = await db.deleteBundlesForRepo(repo.repoId)
|
|
79
|
+
const deletedTriage = await db.deleteTriage(annotations?.ids ?? [])
|
|
80
|
+
await db.deleteRepo(repo.repoId)
|
|
81
|
+
await db.recordActivity({ kind: 'delete', actor: user.login, actorId: user.id, repo: repo.fullName,
|
|
82
|
+
action: `removed a repository (${deletedReports} reports, ${deletedBundles} bundles, ${deletedTriage} triage entries)` }, Date.now())
|
|
83
|
+
return { reports, bundles, deletedReports, deletedBundles, deletedTriage }
|
|
84
|
+
},
|
|
85
|
+
}
|
|
86
|
+
}
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto'
|
|
2
|
+
import { lstat, mkdir, open, readdir, rename, rm, rmdir, stat } from 'node:fs/promises'
|
|
3
|
+
import { dirname, join } from 'node:path'
|
|
4
|
+
import { type ObjectPage, type RawObjectStorage, objectPath } from './object-storage.ts'
|
|
5
|
+
|
|
6
|
+
function missing(err: unknown): boolean { return (err as NodeJS.ErrnoException).code === 'ENOENT' }
|
|
7
|
+
function version(info: { dev: number; ino: number; size: number; mtimeMs: number; ctimeMs: number }): string {
|
|
8
|
+
return JSON.stringify([info.dev, info.ino, info.size, info.mtimeMs, info.ctimeMs])
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
async function syncDirectory(dir: string): Promise<void> {
|
|
12
|
+
try {
|
|
13
|
+
const directory = await open(dir, 'r')
|
|
14
|
+
try { await directory.sync() } finally { await directory.close() }
|
|
15
|
+
} catch (err) {
|
|
16
|
+
// Windows and some mounted filesystems cannot open/fsync directories.
|
|
17
|
+
// Preserve real I/O failures; file fsync remains mandatory on every write.
|
|
18
|
+
if (!['EINVAL', 'ENOTSUP', 'EOPNOTSUPP', 'ENOSYS', 'EISDIR', 'EBADF', 'EPERM'].includes((err as NodeJS.ErrnoException).code ?? '')) throw err
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
// Keyset pagination survives concurrent file creation/deletion. A final sweep
|
|
23
|
+
// catches objects inserted before the cursor during migration.
|
|
24
|
+
async function listFiles(dir: string, prefix: string, cursor: string | null, limit: number, signal?: AbortSignal): Promise<ObjectPage> {
|
|
25
|
+
const objects: ObjectPage['objects'] = []
|
|
26
|
+
async function visit(key: string): Promise<void> {
|
|
27
|
+
signal?.throwIfAborted()
|
|
28
|
+
let entries
|
|
29
|
+
try {
|
|
30
|
+
if (!(await lstat(join(dir, key))).isDirectory()) {
|
|
31
|
+
console.warn('managed-storage: skipping unsupported directory', JSON.stringify(key))
|
|
32
|
+
return
|
|
33
|
+
}
|
|
34
|
+
entries = await readdir(join(dir, key), { withFileTypes: true })
|
|
35
|
+
}
|
|
36
|
+
catch (err) { if (missing(err)) return; throw err }
|
|
37
|
+
const order = (entry: typeof entries[number]) => `${entry.name}${entry.isDirectory() ? '/' : ''}`
|
|
38
|
+
entries.sort((a, b) => order(a) < order(b) ? -1 : order(a) > order(b) ? 1 : 0)
|
|
39
|
+
for (const entry of entries) {
|
|
40
|
+
signal?.throwIfAborted()
|
|
41
|
+
const child = `${key}${entry.name}`
|
|
42
|
+
if (entry.isSymbolicLink() || (!entry.isDirectory() && !entry.isFile())) {
|
|
43
|
+
console.warn('managed-storage: skipping unsupported entry', JSON.stringify(child))
|
|
44
|
+
continue
|
|
45
|
+
}
|
|
46
|
+
try { objectPath(child) }
|
|
47
|
+
catch { console.warn('managed-storage: skipping invalid entry', JSON.stringify(child)); continue }
|
|
48
|
+
if (entry.isDirectory()) {
|
|
49
|
+
const folder = `${child}/`
|
|
50
|
+
if (cursor && folder < cursor && !cursor.startsWith(folder)) continue
|
|
51
|
+
await visit(folder)
|
|
52
|
+
if (objects.length > limit) return
|
|
53
|
+
}
|
|
54
|
+
else if (entry.isFile() && (!cursor || child > cursor)) {
|
|
55
|
+
try { objects.push({ key: objectPath(child), modifiedAt: (await stat(join(dir, child))).mtimeMs }) }
|
|
56
|
+
catch (err) { if (!missing(err)) throw err }
|
|
57
|
+
if (objects.length > limit) return
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
// Another migration worker can observe an unlink before its writer has
|
|
61
|
+
// synced the directory. Persist those observed removals before an empty
|
|
62
|
+
// inventory can mark the shared migration complete.
|
|
63
|
+
try { await syncDirectory(join(dir, key)) } catch (err) { if (!missing(err)) throw err }
|
|
64
|
+
}
|
|
65
|
+
if (!prefix.endsWith('/') || !Number.isInteger(limit) || limit < 1) throw new Error('Invalid storage page')
|
|
66
|
+
objectPath(`${prefix}validate`)
|
|
67
|
+
await visit(prefix)
|
|
68
|
+
return { objects: objects.slice(0, limit), cursor: objects.length > limit ? objects[limit - 1]!.key : null }
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
export function createDiskObjectStorage(dir: string): RawObjectStorage {
|
|
72
|
+
const path = (key: string) => join(dir, objectPath(key))
|
|
73
|
+
return {
|
|
74
|
+
async head(key, signal) {
|
|
75
|
+
signal?.throwIfAborted()
|
|
76
|
+
try {
|
|
77
|
+
const info = await stat(path(key))
|
|
78
|
+
return { size: info.size, version: version(info), modifiedAt: info.mtimeMs }
|
|
79
|
+
} catch (err) { if (missing(err)) return null; throw err }
|
|
80
|
+
},
|
|
81
|
+
async open(key, signal) {
|
|
82
|
+
signal?.throwIfAborted()
|
|
83
|
+
let file
|
|
84
|
+
try { file = await open(path(key), 'r') }
|
|
85
|
+
catch (err) { if (missing(err)) return null; throw err }
|
|
86
|
+
try {
|
|
87
|
+
const info = await file.stat()
|
|
88
|
+
return { size: info.size, version: version(info), modifiedAt: info.mtimeMs,
|
|
89
|
+
stream: file.createReadStream(signal ? { signal } : {}) }
|
|
90
|
+
} catch (err) { await file.close(); throw err }
|
|
91
|
+
},
|
|
92
|
+
async put(key, bytes, signal, expected) {
|
|
93
|
+
const target = path(key)
|
|
94
|
+
// Temp-file + rename makes replacement atomic to readers. Concurrent
|
|
95
|
+
// migrations use the same persisted data key and immutable plaintext.
|
|
96
|
+
const temp = `${target}.${randomUUID()}.tmp`
|
|
97
|
+
await mkdir(dirname(target), { recursive: true })
|
|
98
|
+
const file = await open(temp, 'wx', 0o600)
|
|
99
|
+
try {
|
|
100
|
+
await file.writeFile(bytes, signal ? { signal } : {})
|
|
101
|
+
await file.sync(); await file.close()
|
|
102
|
+
signal?.throwIfAborted()
|
|
103
|
+
if (expected !== undefined) {
|
|
104
|
+
try { if (version(await stat(target)) !== expected) return false }
|
|
105
|
+
catch (err) { if (missing(err)) return false; throw err }
|
|
106
|
+
}
|
|
107
|
+
await rename(temp, target)
|
|
108
|
+
await syncDirectory(dirname(target))
|
|
109
|
+
return true
|
|
110
|
+
} finally { await file.close(); await rm(temp, { force: true }) }
|
|
111
|
+
},
|
|
112
|
+
async delete(key, expected, signal) {
|
|
113
|
+
signal?.throwIfAborted()
|
|
114
|
+
try {
|
|
115
|
+
if (expected !== undefined && version(await stat(path(key))) !== expected) return false
|
|
116
|
+
await rm(path(key), { force: true })
|
|
117
|
+
// Persist plaintext removal before migration checkpoints completion.
|
|
118
|
+
// Otherwise a crash could restore the directory entry after SQL has
|
|
119
|
+
// durably disabled further plaintext inventory sweeps.
|
|
120
|
+
await syncDirectory(dirname(path(key)))
|
|
121
|
+
return true
|
|
122
|
+
} catch (err) { if (missing(err)) return true; throw err }
|
|
123
|
+
},
|
|
124
|
+
async sync(key, signal) { signal?.throwIfAborted(); await syncDirectory(dirname(path(key))) },
|
|
125
|
+
async prune(prefix, signal) {
|
|
126
|
+
if (!prefix.endsWith('/')) throw new Error('Invalid storage prefix')
|
|
127
|
+
objectPath(`${prefix}validate`)
|
|
128
|
+
async function visit(folder: string): Promise<void> {
|
|
129
|
+
signal?.throwIfAborted()
|
|
130
|
+
let entries
|
|
131
|
+
try {
|
|
132
|
+
if (!(await lstat(folder)).isDirectory()) return
|
|
133
|
+
entries = await readdir(folder, { withFileTypes: true })
|
|
134
|
+
}
|
|
135
|
+
catch (err) { if (missing(err)) return; throw err }
|
|
136
|
+
for (const entry of entries) if (entry.isDirectory()) await visit(join(folder, entry.name))
|
|
137
|
+
try { await rmdir(folder); await syncDirectory(dirname(folder)) }
|
|
138
|
+
catch (err) { if (!['ENOENT', 'ENOTEMPTY', 'EEXIST'].includes((err as NodeJS.ErrnoException).code ?? '')) throw err }
|
|
139
|
+
}
|
|
140
|
+
await visit(join(dir, prefix))
|
|
141
|
+
},
|
|
142
|
+
list: (prefix, cursor, limit, signal) => listFiles(dir, prefix, cursor, limit, signal),
|
|
143
|
+
}
|
|
144
|
+
}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import { Readable } from 'node:stream'
|
|
2
|
+
import { Buffer } from 'node:buffer'
|
|
3
|
+
import { type VercelBlobSdk, isNotFound, loadVercelBlobSdk } from '../server-common/vercel-blob.ts'
|
|
4
|
+
import { type RawObjectStorage, objectPath } from './object-storage.ts'
|
|
5
|
+
|
|
6
|
+
export async function openVercelObjectStorage(token: string, sdk?: VercelBlobSdk): Promise<RawObjectStorage> {
|
|
7
|
+
const blobs = sdk ?? await loadVercelBlobSdk()
|
|
8
|
+
const path = (key: string) => `.managed/${objectPath(key)}`
|
|
9
|
+
return {
|
|
10
|
+
async head(key, signal) {
|
|
11
|
+
try {
|
|
12
|
+
const meta = await blobs.head(path(key), { token, ...(signal ? { abortSignal: signal } : {}) })
|
|
13
|
+
if (!meta.etag) throw new Error('Missing object version')
|
|
14
|
+
return { size: meta.size, version: meta.etag, modifiedAt: new Date(meta.uploadedAt ?? Date.now()).getTime() }
|
|
15
|
+
} catch (err) { if (isNotFound(err, blobs)) return null; throw err }
|
|
16
|
+
},
|
|
17
|
+
async open(key, signal) {
|
|
18
|
+
try {
|
|
19
|
+
const result = await blobs.get(path(key), { token, access: 'private', useCache: false, ...(signal ? { abortSignal: signal } : {}) })
|
|
20
|
+
if (!result) return null
|
|
21
|
+
if (result.statusCode !== 200 || !result.stream || !result.blob.etag) {
|
|
22
|
+
await result.stream?.cancel()
|
|
23
|
+
throw new Error('Unexpected blob response or missing object version')
|
|
24
|
+
}
|
|
25
|
+
return { size: result.blob.size === 0 ? null : result.blob.size, version: result.blob.etag,
|
|
26
|
+
modifiedAt: new Date(result.blob.uploadedAt ?? Date.now()).getTime(),
|
|
27
|
+
stream: Readable.fromWeb(result.stream as Parameters<typeof Readable.fromWeb>[0], signal ? { signal } : {}) }
|
|
28
|
+
} catch (err) { if (isNotFound(err, blobs)) return null; throw err }
|
|
29
|
+
},
|
|
30
|
+
async put(key, bytes, signal, expected, sizeHint) {
|
|
31
|
+
const multipart = (sizeHint ?? (Buffer.isBuffer(bytes) ? bytes.length : Infinity)) >= 5 * 1024 * 1024
|
|
32
|
+
try { await blobs.put(path(key), bytes, { token, access: 'private', addRandomSuffix: false, allowOverwrite: true,
|
|
33
|
+
...(expected ? { ifMatch: expected } : {}), multipart, contentType: 'application/octet-stream', cacheControlMaxAge: 60, ...(signal ? { abortSignal: signal } : {}) })
|
|
34
|
+
return true
|
|
35
|
+
} catch (err) {
|
|
36
|
+
if (blobs.BlobPreconditionFailedError && err instanceof blobs.BlobPreconditionFailedError) return false
|
|
37
|
+
throw err
|
|
38
|
+
}
|
|
39
|
+
},
|
|
40
|
+
async delete(key, version, signal) {
|
|
41
|
+
signal?.throwIfAborted()
|
|
42
|
+
try { await blobs.del(path(key), { token, ...(version ? { ifMatch: version } : {}), ...(signal ? { abortSignal: signal } : {}) }); return true }
|
|
43
|
+
catch (err) {
|
|
44
|
+
if (isNotFound(err, blobs)) return true
|
|
45
|
+
if (blobs.BlobPreconditionFailedError && err instanceof blobs.BlobPreconditionFailedError) return false
|
|
46
|
+
throw err
|
|
47
|
+
}
|
|
48
|
+
},
|
|
49
|
+
async list(prefix, cursor, limit, signal) {
|
|
50
|
+
signal?.throwIfAborted()
|
|
51
|
+
if (!prefix.endsWith('/') || !Number.isInteger(limit) || limit < 1) throw new Error('Invalid storage page')
|
|
52
|
+
objectPath(`${prefix}validate`)
|
|
53
|
+
const page = await blobs.list({ prefix: `.managed/${prefix}`, token, limit, ...(cursor ? { cursor } : {}), ...(signal ? { abortSignal: signal } : {}) })
|
|
54
|
+
if (page.hasMore && (!page.cursor || page.cursor === cursor)) throw new Error('Invalid blob pagination')
|
|
55
|
+
return { objects: page.blobs.filter(blob => blob.pathname.startsWith(`.managed/${prefix}`)).map(blob => ({ key: blob.pathname.slice('.managed/'.length),
|
|
56
|
+
modifiedAt: new Date(blob.uploadedAt ?? Date.now()).getTime() })),
|
|
57
|
+
cursor: page.hasMore ? page.cursor! : null }
|
|
58
|
+
},
|
|
59
|
+
}
|
|
60
|
+
}
|