@preventive/triage 1.0.0-alpha.25 → 1.0.0-alpha.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/common/server-info.ts +1 -0
- package/out/client-managed-import.js +18 -18
- package/out/client-managed.js +66 -65
- package/out/prism.js +2 -2
- package/out/view.css +1 -1
- package/out/view.js +182 -182
- package/package.json +12 -2
- package/server-common/storage-crypto.ts +182 -0
- package/server-common/storage-log.ts +5 -1
- package/server-common/vercel-blob.ts +6 -3
- package/server-e2e/config.ts +4 -0
- package/server-managed/avatar-store.ts +1 -40
- package/server-managed/blob-store.ts +3 -52
- package/server-managed/bundle-cache.ts +0 -28
- package/server-managed/bundle-store.ts +3 -7
- package/server-managed/cache-storage.ts +0 -25
- package/server-managed/cli.js +10 -2
- package/server-managed/config.ts +14 -2
- package/server-managed/db-methods.ts +56 -25
- package/server-managed/db-neon.ts +9 -4
- package/server-managed/db.ts +7 -1
- package/server-managed/http.ts +93 -62
- package/server-managed/import-triage.ts +6 -3
- package/server-managed/index.ts +32 -9
- package/server-managed/object-storage-disk.ts +144 -0
- package/server-managed/object-storage-vercel.ts +60 -0
- package/server-managed/object-storage.ts +71 -0
- package/server-managed/report-query.ts +6 -0
- package/server-managed/repository-browser.ts +5 -1
- package/server-managed/solidity-entry-points.ts +69 -0
- package/server-managed/sql.ts +0 -5
- package/server-managed/storage-command.ts +16 -0
- package/server-managed/storage-db.ts +168 -0
- package/server-managed/storage-encryption.ts +211 -0
- package/server-managed/storage-maintenance.ts +182 -0
- package/server-managed/storage-payload.ts +60 -0
- package/server-managed/storage-stores.ts +81 -0
- package/server-managed/storage.ts +46 -24
- package/server-managed/uploads.ts +30 -12
- package/server-managed/blob-vercel.ts +0 -125
|
@@ -11,11 +11,11 @@ export interface ImportTriageSnapshot {
|
|
|
11
11
|
}
|
|
12
12
|
export interface ImportTriageStore {
|
|
13
13
|
getImportTriage(ids: readonly string[]): Promise<Record<string, ImportTriageSnapshot>>
|
|
14
|
-
importTriage(entries: [string, TriageEntryPatch | null][], expected: Record<string, string>, actor: { id: string; login: string }, reportId: string
|
|
14
|
+
importTriage(entries: [string, TriageEntryPatch | null][], expected: Record<string, string>, actor: { id: string; login: string }, reportId: string, now: number): Promise<boolean>
|
|
15
15
|
}
|
|
16
16
|
|
|
17
17
|
// Raw store methods share one transaction for comparison, triage, and comments.
|
|
18
|
-
export function importTriageMethods(store: Pick<ManagedDb, 'listTriage' | 'listComments' | 'setTriageEntries' | 'createComment'>): ImportTriageStore {
|
|
18
|
+
export function importTriageMethods(store: Pick<ManagedDb, 'getReport' | 'listTriage' | 'listComments' | 'setTriageEntries' | 'createComment'>): ImportTriageStore {
|
|
19
19
|
async function snapshot(ids: readonly string[]) {
|
|
20
20
|
const rows = new Map((await store.listTriage(ids)).map(row => [row.findingId, row]))
|
|
21
21
|
const comments = await store.listComments(ids)
|
|
@@ -30,6 +30,9 @@ export function importTriageMethods(store: Pick<ManagedDb, 'listTriage' | 'listC
|
|
|
30
30
|
return {
|
|
31
31
|
getImportTriage: snapshot,
|
|
32
32
|
async importTriage(entries, expected, actor, reportId, now) {
|
|
33
|
+
// A report deleted after the HTTP membership check must not leave newly
|
|
34
|
+
// imported annotations behind. Check existence in the write transaction.
|
|
35
|
+
if (!reportId || !(await store.getReport(reportId))) return false
|
|
33
36
|
const current = await snapshot(entries.map(([id]) => id))
|
|
34
37
|
if (entries.some(([id]) => current[id]!.version !== expected[id])) return false
|
|
35
38
|
const changes: [string, TriageEntryPatch | null][] = []
|
|
@@ -38,7 +41,7 @@ export function importTriageMethods(store: Pick<ManagedDb, 'listTriage' | 'listC
|
|
|
38
41
|
changes.push([id, Object.keys(entry).length > 0 ? entry : null])
|
|
39
42
|
if (comment && !current[id]!.comments.some(existing => existing.body === comment)) {
|
|
40
43
|
await store.createComment({ findingId: id, body: comment, authorId: null, authorLogin: null,
|
|
41
|
-
actor, createdAt: null, updatedAt: null, reportId
|
|
44
|
+
actor, createdAt: null, updatedAt: null, reportId }, now)
|
|
42
45
|
}
|
|
43
46
|
}
|
|
44
47
|
await store.setTriageEntries(changes, actor.id, actor.login, now, reportId)
|
package/server-managed/index.ts
CHANGED
|
@@ -12,9 +12,10 @@ import { type ManagedHttpDeps, createManagedRequestHandler } from './http.ts'
|
|
|
12
12
|
import { loadManagedStatic } from './static.ts'
|
|
13
13
|
import { openManagedStorage } from './storage.ts'
|
|
14
14
|
|
|
15
|
-
//
|
|
16
|
-
//
|
|
17
|
-
const
|
|
15
|
+
// Housekeeping runs on ordinary traffic too, including serverless instances
|
|
16
|
+
// without a timer or cron. Failed sweeps retry on later traffic after a minute.
|
|
17
|
+
const REAP_INTERVAL_MS = 3_600_000
|
|
18
|
+
const REAP_RETRY_MS = 60_000
|
|
18
19
|
|
|
19
20
|
type ManagedAppOptions = Partial<Pick<ManagedHttpDeps, 'next' | 'serverInfo' | 'isShuttingDown'>>
|
|
20
21
|
|
|
@@ -39,25 +40,47 @@ async function assembleManagedApp(config: ManagedConfig, options: ManagedAppOpti
|
|
|
39
40
|
const serveStatic = loadManagedStatic(fileURLToPath(new URL('../out/', import.meta.url)), {
|
|
40
41
|
indexOnly: options.next != null, scanServer: options.serverInfo?.deepviewScanServer ?? null,
|
|
41
42
|
})
|
|
42
|
-
const
|
|
43
|
+
const routeRequest = createManagedRequestHandler({
|
|
43
44
|
...options, config, db, avatarStore, reportStore, bundleStore, bundleCache, reportSourcesCache, originGate, serveStatic,
|
|
44
|
-
...(
|
|
45
|
+
...(storage.uploadStore ? { uploadStore: storage.uploadStore } : {}),
|
|
45
46
|
isShuttingDown, track,
|
|
46
47
|
})
|
|
47
48
|
|
|
48
49
|
let cleanup: Promise<void> | undefined
|
|
50
|
+
let nextCleanupAt = 0
|
|
49
51
|
function reap(): Promise<void> {
|
|
50
52
|
if (cleanup) return cleanup
|
|
53
|
+
const startedAt = Date.now()
|
|
54
|
+
nextCleanupAt = startedAt + REAP_INTERVAL_MS
|
|
55
|
+
let sessions = 0, uploads = 0
|
|
51
56
|
cleanup = runReapers({
|
|
52
|
-
sessions: () => db.deleteExpiredSessions(Date.now()),
|
|
53
|
-
...(
|
|
57
|
+
sessions: async () => { sessions = await db.deleteExpiredSessions(Date.now()) },
|
|
58
|
+
...(storage.reapUploads ? { uploads: async () => { uploads = await storage.reapUploads!() } } : {}),
|
|
59
|
+
...('reapStorage' in storage ? { storage: () => storage.reapStorage() } : {}),
|
|
60
|
+
}).then(() => {
|
|
61
|
+
nextCleanupAt = Date.now() + REAP_INTERVAL_MS
|
|
62
|
+
return console.info(`managed-reaper: removed ${sessions} expired session(s), ${uploads} stale upload part(s) in ${Date.now() - startedAt}ms`)
|
|
63
|
+
}, err => {
|
|
64
|
+
nextCleanupAt = Date.now() + REAP_RETRY_MS
|
|
65
|
+
throw err
|
|
54
66
|
}).finally(() => { cleanup = undefined })
|
|
55
67
|
track(cleanup)
|
|
56
68
|
return cleanup
|
|
57
69
|
}
|
|
70
|
+
function automaticReap(): Promise<void> {
|
|
71
|
+
if (isShuttingDown() || cleanup || Date.now() < nextCleanupAt) return Promise.resolve()
|
|
72
|
+
return reap().catch(err => console.warn('managed-reaper: cleanup failed:', err))
|
|
73
|
+
}
|
|
74
|
+
async function handleRequest(req: Parameters<typeof routeRequest>[0], res: Parameters<typeof routeRequest>[1]): Promise<void> {
|
|
75
|
+
const maintenance = automaticReap()
|
|
76
|
+
// Send the normal response without waiting for storage housekeeping, but
|
|
77
|
+
// keep the invocation alive until both finish. No detached serverless work.
|
|
78
|
+
try { await routeRequest(req, res) }
|
|
79
|
+
finally { await maintenance }
|
|
80
|
+
}
|
|
58
81
|
const gcTimer = config.serverless ? null : setInterval(() => {
|
|
59
|
-
void
|
|
60
|
-
},
|
|
82
|
+
void automaticReap()
|
|
83
|
+
}, REAP_INTERVAL_MS)
|
|
61
84
|
rollback.defer(stop)
|
|
62
85
|
|
|
63
86
|
function stop(): void {
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto'
|
|
2
|
+
import { lstat, mkdir, open, readdir, rename, rm, rmdir, stat } from 'node:fs/promises'
|
|
3
|
+
import { dirname, join } from 'node:path'
|
|
4
|
+
import { type ObjectPage, type RawObjectStorage, objectPath } from './object-storage.ts'
|
|
5
|
+
|
|
6
|
+
function missing(err: unknown): boolean { return (err as NodeJS.ErrnoException).code === 'ENOENT' }
|
|
7
|
+
function version(info: { dev: number; ino: number; size: number; mtimeMs: number; ctimeMs: number }): string {
|
|
8
|
+
return JSON.stringify([info.dev, info.ino, info.size, info.mtimeMs, info.ctimeMs])
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
async function syncDirectory(dir: string): Promise<void> {
|
|
12
|
+
try {
|
|
13
|
+
const directory = await open(dir, 'r')
|
|
14
|
+
try { await directory.sync() } finally { await directory.close() }
|
|
15
|
+
} catch (err) {
|
|
16
|
+
// Windows and some mounted filesystems cannot open/fsync directories.
|
|
17
|
+
// Preserve real I/O failures; file fsync remains mandatory on every write.
|
|
18
|
+
if (!['EINVAL', 'ENOTSUP', 'EOPNOTSUPP', 'ENOSYS', 'EISDIR', 'EBADF', 'EPERM'].includes((err as NodeJS.ErrnoException).code ?? '')) throw err
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
// Keyset pagination survives concurrent file creation/deletion. A final sweep
|
|
23
|
+
// catches objects inserted before the cursor during migration.
|
|
24
|
+
async function listFiles(dir: string, prefix: string, cursor: string | null, limit: number, signal?: AbortSignal): Promise<ObjectPage> {
|
|
25
|
+
const objects: ObjectPage['objects'] = []
|
|
26
|
+
async function visit(key: string): Promise<void> {
|
|
27
|
+
signal?.throwIfAborted()
|
|
28
|
+
let entries
|
|
29
|
+
try {
|
|
30
|
+
if (!(await lstat(join(dir, key))).isDirectory()) {
|
|
31
|
+
console.warn('managed-storage: skipping unsupported directory', JSON.stringify(key))
|
|
32
|
+
return
|
|
33
|
+
}
|
|
34
|
+
entries = await readdir(join(dir, key), { withFileTypes: true })
|
|
35
|
+
}
|
|
36
|
+
catch (err) { if (missing(err)) return; throw err }
|
|
37
|
+
const order = (entry: typeof entries[number]) => `${entry.name}${entry.isDirectory() ? '/' : ''}`
|
|
38
|
+
entries.sort((a, b) => order(a) < order(b) ? -1 : order(a) > order(b) ? 1 : 0)
|
|
39
|
+
for (const entry of entries) {
|
|
40
|
+
signal?.throwIfAborted()
|
|
41
|
+
const child = `${key}${entry.name}`
|
|
42
|
+
if (entry.isSymbolicLink() || (!entry.isDirectory() && !entry.isFile())) {
|
|
43
|
+
console.warn('managed-storage: skipping unsupported entry', JSON.stringify(child))
|
|
44
|
+
continue
|
|
45
|
+
}
|
|
46
|
+
try { objectPath(child) }
|
|
47
|
+
catch { console.warn('managed-storage: skipping invalid entry', JSON.stringify(child)); continue }
|
|
48
|
+
if (entry.isDirectory()) {
|
|
49
|
+
const folder = `${child}/`
|
|
50
|
+
if (cursor && folder < cursor && !cursor.startsWith(folder)) continue
|
|
51
|
+
await visit(folder)
|
|
52
|
+
if (objects.length > limit) return
|
|
53
|
+
}
|
|
54
|
+
else if (entry.isFile() && (!cursor || child > cursor)) {
|
|
55
|
+
try { objects.push({ key: objectPath(child), modifiedAt: (await stat(join(dir, child))).mtimeMs }) }
|
|
56
|
+
catch (err) { if (!missing(err)) throw err }
|
|
57
|
+
if (objects.length > limit) return
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
// Another migration worker can observe an unlink before its writer has
|
|
61
|
+
// synced the directory. Persist those observed removals before an empty
|
|
62
|
+
// inventory can mark the shared migration complete.
|
|
63
|
+
try { await syncDirectory(join(dir, key)) } catch (err) { if (!missing(err)) throw err }
|
|
64
|
+
}
|
|
65
|
+
if (!prefix.endsWith('/') || !Number.isInteger(limit) || limit < 1) throw new Error('Invalid storage page')
|
|
66
|
+
objectPath(`${prefix}validate`)
|
|
67
|
+
await visit(prefix)
|
|
68
|
+
return { objects: objects.slice(0, limit), cursor: objects.length > limit ? objects[limit - 1]!.key : null }
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
export function createDiskObjectStorage(dir: string): RawObjectStorage {
|
|
72
|
+
const path = (key: string) => join(dir, objectPath(key))
|
|
73
|
+
return {
|
|
74
|
+
async head(key, signal) {
|
|
75
|
+
signal?.throwIfAborted()
|
|
76
|
+
try {
|
|
77
|
+
const info = await stat(path(key))
|
|
78
|
+
return { size: info.size, version: version(info), modifiedAt: info.mtimeMs }
|
|
79
|
+
} catch (err) { if (missing(err)) return null; throw err }
|
|
80
|
+
},
|
|
81
|
+
async open(key, signal) {
|
|
82
|
+
signal?.throwIfAborted()
|
|
83
|
+
let file
|
|
84
|
+
try { file = await open(path(key), 'r') }
|
|
85
|
+
catch (err) { if (missing(err)) return null; throw err }
|
|
86
|
+
try {
|
|
87
|
+
const info = await file.stat()
|
|
88
|
+
return { size: info.size, version: version(info), modifiedAt: info.mtimeMs,
|
|
89
|
+
stream: file.createReadStream(signal ? { signal } : {}) }
|
|
90
|
+
} catch (err) { await file.close(); throw err }
|
|
91
|
+
},
|
|
92
|
+
async put(key, bytes, signal, expected) {
|
|
93
|
+
const target = path(key)
|
|
94
|
+
// Temp-file + rename makes replacement atomic to readers. Concurrent
|
|
95
|
+
// migrations use the same persisted data key and immutable plaintext.
|
|
96
|
+
const temp = `${target}.${randomUUID()}.tmp`
|
|
97
|
+
await mkdir(dirname(target), { recursive: true })
|
|
98
|
+
const file = await open(temp, 'wx', 0o600)
|
|
99
|
+
try {
|
|
100
|
+
await file.writeFile(bytes, signal ? { signal } : {})
|
|
101
|
+
await file.sync(); await file.close()
|
|
102
|
+
signal?.throwIfAborted()
|
|
103
|
+
if (expected !== undefined) {
|
|
104
|
+
try { if (version(await stat(target)) !== expected) return false }
|
|
105
|
+
catch (err) { if (missing(err)) return false; throw err }
|
|
106
|
+
}
|
|
107
|
+
await rename(temp, target)
|
|
108
|
+
await syncDirectory(dirname(target))
|
|
109
|
+
return true
|
|
110
|
+
} finally { await file.close(); await rm(temp, { force: true }) }
|
|
111
|
+
},
|
|
112
|
+
async delete(key, expected, signal) {
|
|
113
|
+
signal?.throwIfAborted()
|
|
114
|
+
try {
|
|
115
|
+
if (expected !== undefined && version(await stat(path(key))) !== expected) return false
|
|
116
|
+
await rm(path(key), { force: true })
|
|
117
|
+
// Persist plaintext removal before migration checkpoints completion.
|
|
118
|
+
// Otherwise a crash could restore the directory entry after SQL has
|
|
119
|
+
// durably disabled further plaintext inventory sweeps.
|
|
120
|
+
await syncDirectory(dirname(path(key)))
|
|
121
|
+
return true
|
|
122
|
+
} catch (err) { if (missing(err)) return true; throw err }
|
|
123
|
+
},
|
|
124
|
+
async sync(key, signal) { signal?.throwIfAborted(); await syncDirectory(dirname(path(key))) },
|
|
125
|
+
async prune(prefix, signal) {
|
|
126
|
+
if (!prefix.endsWith('/')) throw new Error('Invalid storage prefix')
|
|
127
|
+
objectPath(`${prefix}validate`)
|
|
128
|
+
async function visit(folder: string): Promise<void> {
|
|
129
|
+
signal?.throwIfAborted()
|
|
130
|
+
let entries
|
|
131
|
+
try {
|
|
132
|
+
if (!(await lstat(folder)).isDirectory()) return
|
|
133
|
+
entries = await readdir(folder, { withFileTypes: true })
|
|
134
|
+
}
|
|
135
|
+
catch (err) { if (missing(err)) return; throw err }
|
|
136
|
+
for (const entry of entries) if (entry.isDirectory()) await visit(join(folder, entry.name))
|
|
137
|
+
try { await rmdir(folder); await syncDirectory(dirname(folder)) }
|
|
138
|
+
catch (err) { if (!['ENOENT', 'ENOTEMPTY', 'EEXIST'].includes((err as NodeJS.ErrnoException).code ?? '')) throw err }
|
|
139
|
+
}
|
|
140
|
+
await visit(join(dir, prefix))
|
|
141
|
+
},
|
|
142
|
+
list: (prefix, cursor, limit, signal) => listFiles(dir, prefix, cursor, limit, signal),
|
|
143
|
+
}
|
|
144
|
+
}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import { Readable } from 'node:stream'
|
|
2
|
+
import { Buffer } from 'node:buffer'
|
|
3
|
+
import { type VercelBlobSdk, isNotFound, loadVercelBlobSdk } from '../server-common/vercel-blob.ts'
|
|
4
|
+
import { type RawObjectStorage, objectPath } from './object-storage.ts'
|
|
5
|
+
|
|
6
|
+
export async function openVercelObjectStorage(token: string, sdk?: VercelBlobSdk): Promise<RawObjectStorage> {
|
|
7
|
+
const blobs = sdk ?? await loadVercelBlobSdk()
|
|
8
|
+
const path = (key: string) => `.managed/${objectPath(key)}`
|
|
9
|
+
return {
|
|
10
|
+
async head(key, signal) {
|
|
11
|
+
try {
|
|
12
|
+
const meta = await blobs.head(path(key), { token, ...(signal ? { abortSignal: signal } : {}) })
|
|
13
|
+
if (!meta.etag) throw new Error('Missing object version')
|
|
14
|
+
return { size: meta.size, version: meta.etag, modifiedAt: new Date(meta.uploadedAt ?? Date.now()).getTime() }
|
|
15
|
+
} catch (err) { if (isNotFound(err, blobs)) return null; throw err }
|
|
16
|
+
},
|
|
17
|
+
async open(key, signal) {
|
|
18
|
+
try {
|
|
19
|
+
const result = await blobs.get(path(key), { token, access: 'private', useCache: false, ...(signal ? { abortSignal: signal } : {}) })
|
|
20
|
+
if (!result) return null
|
|
21
|
+
if (result.statusCode !== 200 || !result.stream || !result.blob.etag) {
|
|
22
|
+
await result.stream?.cancel()
|
|
23
|
+
throw new Error('Unexpected blob response or missing object version')
|
|
24
|
+
}
|
|
25
|
+
return { size: result.blob.size === 0 ? null : result.blob.size, version: result.blob.etag,
|
|
26
|
+
modifiedAt: new Date(result.blob.uploadedAt ?? Date.now()).getTime(),
|
|
27
|
+
stream: Readable.fromWeb(result.stream as Parameters<typeof Readable.fromWeb>[0]) }
|
|
28
|
+
} catch (err) { if (isNotFound(err, blobs)) return null; throw err }
|
|
29
|
+
},
|
|
30
|
+
async put(key, bytes, signal, expected, sizeHint) {
|
|
31
|
+
const multipart = (sizeHint ?? (Buffer.isBuffer(bytes) ? bytes.length : Infinity)) >= 5 * 1024 * 1024
|
|
32
|
+
try { await blobs.put(path(key), bytes, { token, access: 'private', addRandomSuffix: false, allowOverwrite: true,
|
|
33
|
+
...(expected ? { ifMatch: expected } : {}), multipart, contentType: 'application/octet-stream', cacheControlMaxAge: 60, ...(signal ? { abortSignal: signal } : {}) })
|
|
34
|
+
return true
|
|
35
|
+
} catch (err) {
|
|
36
|
+
if (blobs.BlobPreconditionFailedError && err instanceof blobs.BlobPreconditionFailedError) return false
|
|
37
|
+
throw err
|
|
38
|
+
}
|
|
39
|
+
},
|
|
40
|
+
async delete(key, version, signal) {
|
|
41
|
+
signal?.throwIfAborted()
|
|
42
|
+
try { await blobs.del(path(key), { token, ...(version ? { ifMatch: version } : {}), ...(signal ? { abortSignal: signal } : {}) }); return true }
|
|
43
|
+
catch (err) {
|
|
44
|
+
if (isNotFound(err, blobs)) return true
|
|
45
|
+
if (blobs.BlobPreconditionFailedError && err instanceof blobs.BlobPreconditionFailedError) return false
|
|
46
|
+
throw err
|
|
47
|
+
}
|
|
48
|
+
},
|
|
49
|
+
async list(prefix, cursor, limit, signal) {
|
|
50
|
+
signal?.throwIfAborted()
|
|
51
|
+
if (!prefix.endsWith('/') || !Number.isInteger(limit) || limit < 1) throw new Error('Invalid storage page')
|
|
52
|
+
objectPath(`${prefix}validate`)
|
|
53
|
+
const page = await blobs.list({ prefix: `.managed/${prefix}`, token, limit, ...(cursor ? { cursor } : {}), ...(signal ? { abortSignal: signal } : {}) })
|
|
54
|
+
if (page.hasMore && (!page.cursor || page.cursor === cursor)) throw new Error('Invalid blob pagination')
|
|
55
|
+
return { objects: page.blobs.filter(blob => blob.pathname.startsWith(`.managed/${prefix}`)).map(blob => ({ key: blob.pathname.slice('.managed/'.length),
|
|
56
|
+
modifiedAt: new Date(blob.uploadedAt ?? Date.now()).getTime() })),
|
|
57
|
+
cursor: page.hasMore ? page.cursor! : null }
|
|
58
|
+
},
|
|
59
|
+
}
|
|
60
|
+
}
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
import type { Buffer } from 'node:buffer'
|
|
2
|
+
import type { Readable } from 'node:stream'
|
|
3
|
+
import type { OpenedBlob } from './blob-store.ts'
|
|
4
|
+
import { validateCacheKey } from './cache-storage.ts'
|
|
5
|
+
|
|
6
|
+
export const ENCRYPTED_CACHE_PREFIX = 'cache-encrypted-v1/'
|
|
7
|
+
const STORAGE_PREFIXES = ['reports/', 'bundles/', 'avatars/', 'uploads/', 'cache/', ENCRYPTED_CACHE_PREFIX]
|
|
8
|
+
export interface RawObject extends OpenedBlob { version: string; modifiedAt: number }
|
|
9
|
+
type ObjectMetadata = Omit<RawObject, 'stream'>
|
|
10
|
+
interface ListedObject { key: string; modifiedAt: number }
|
|
11
|
+
export interface ObjectPage { objects: ListedObject[]; cursor: string | null }
|
|
12
|
+
export interface RawObjectStorage {
|
|
13
|
+
open(key: string, signal?: AbortSignal): Promise<RawObject | null>
|
|
14
|
+
head(key: string, signal?: AbortSignal): Promise<ObjectMetadata | null>
|
|
15
|
+
prune?(prefix: string, signal?: AbortSignal): Promise<void>
|
|
16
|
+
// Persist an observed disk replacement before its SQL migration checkpoint.
|
|
17
|
+
sync?(key: string, signal?: AbortSignal): Promise<void>
|
|
18
|
+
// A plaintext size hint selects multipart uploads without buffering streams.
|
|
19
|
+
put(key: string, bytes: Buffer | Readable, signal?: AbortSignal, expected?: string, sizeHint?: number): Promise<boolean>
|
|
20
|
+
// Return false if the original version changed; do not delete its replacement.
|
|
21
|
+
delete(key: string, version?: string, signal?: AbortSignal): Promise<boolean>
|
|
22
|
+
list(prefix: string, cursor: string | null, limit: number, signal?: AbortSignal): Promise<ObjectPage>
|
|
23
|
+
}
|
|
24
|
+
export interface ObjectStorage {
|
|
25
|
+
open(key: string): Promise<OpenedBlob | null>
|
|
26
|
+
get(key: string): Promise<Buffer | null>
|
|
27
|
+
exists(key: string): Promise<boolean>
|
|
28
|
+
put(key: string, bytes: Buffer): Promise<string | null>
|
|
29
|
+
delete(key: string): Promise<void>
|
|
30
|
+
deletePrefix(prefix: string): Promise<void>
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export function isBlobId(value: string): boolean {
|
|
34
|
+
return /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/u.test(value)
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export function objectPath(key: string): string {
|
|
38
|
+
validateCacheKey(key)
|
|
39
|
+
if (!STORAGE_PREFIXES.some(prefix => key.startsWith(prefix))) throw new Error('Invalid managed storage namespace')
|
|
40
|
+
return key
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
// Hash verification and migration may consume a stream before it can be used.
|
|
44
|
+
// Reopen only the same object version, closing any concurrent replacement.
|
|
45
|
+
export async function openObjectVersion(raw: RawObjectStorage, key: string, version: string, signal?: AbortSignal): Promise<RawObject | null> {
|
|
46
|
+
const current = await raw.open(key, signal)
|
|
47
|
+
if (current?.version === version) return current
|
|
48
|
+
current?.stream.destroy()
|
|
49
|
+
return null
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
export async function deleteObjects(raw: RawObjectStorage, prefix: string, signal: AbortSignal, before?: number): Promise<number> {
|
|
53
|
+
const eligible = (modifiedAt: number) => before === undefined || modifiedAt < before
|
|
54
|
+
// Finish listing before deleting: provider cursors may be offset-based.
|
|
55
|
+
const cursors = new Set<string>(), keys = new Set<string>()
|
|
56
|
+
let cursor: string | null = null
|
|
57
|
+
do {
|
|
58
|
+
const page = await raw.list(prefix, cursor, 100, signal)
|
|
59
|
+
for (const object of page.objects) if (eligible(object.modifiedAt)) keys.add(object.key)
|
|
60
|
+
cursor = page.cursor
|
|
61
|
+
if (cursor !== null && cursors.has(cursor)) throw new Error('Invalid blob pagination')
|
|
62
|
+
if (cursor !== null) cursors.add(cursor)
|
|
63
|
+
} while (cursor !== null)
|
|
64
|
+
let removed = 0
|
|
65
|
+
for (const key of keys) {
|
|
66
|
+
const stored = await raw.head(key, signal)
|
|
67
|
+
// Ignore stale listings and preserve concurrent replacements.
|
|
68
|
+
if (stored && eligible(stored.modifiedAt) && await raw.delete(key, stored.version, signal)) removed++
|
|
69
|
+
}
|
|
70
|
+
return removed
|
|
71
|
+
}
|
|
@@ -2,3 +2,9 @@
|
|
|
2
2
|
// the encoded response, independently of the per-upload size limit.
|
|
3
3
|
export const MAX_REPORT_QUERY_COUNT = 4096
|
|
4
4
|
export const MAX_REPORT_QUERY_BYTES = 1024 * 1024 * 1024
|
|
5
|
+
|
|
6
|
+
// The import catalog scans a bounded page, not the whole installation. A
|
|
7
|
+
// report larger than the byte target occupies a page by itself; upload limits
|
|
8
|
+
// still bound that single report and the cursor must always make progress.
|
|
9
|
+
export const FINDING_CATALOG_PAGE_COUNT = 128
|
|
10
|
+
export const FINDING_CATALOG_PAGE_BYTES = 32 * 1024 * 1024
|
|
@@ -2,6 +2,7 @@ import type { ManagedConfig } from './config.ts'
|
|
|
2
2
|
import type { SelectedRepo } from './db.ts'
|
|
3
3
|
import { GithubApiError, githubJson, githubRepoReadPermission, githubUserIdentity, repoAccessToken } from './github-app.ts'
|
|
4
4
|
import { MAX_PACKAGE_BYTES, readPackageEntryPoints } from './package-entry-points.ts'
|
|
5
|
+
import { readSolidityEntryPoints } from './solidity-entry-points.ts'
|
|
5
6
|
|
|
6
7
|
export interface RepositoryEntry { name: string; path: string; type: 'dir' | 'file' | 'symlink' | 'submodule' }
|
|
7
8
|
|
|
@@ -126,7 +127,10 @@ async function repositoryReader(repo: SelectedRepo, userToken: string | null, in
|
|
|
126
127
|
const manifest = data.find(item => item?.path === manifestPath && item.name === 'package.json' && item.type === 'file' && !item.submodule_git_url
|
|
127
128
|
&& typeof item.size === 'number' && item.size >= 0 && item.size <= MAX_PACKAGE_BYTES && typeof item.sha === 'string' && /^[a-f\d]{40}$/iu.test(item.sha))
|
|
128
129
|
const packageEntryPoints = manifest ? await readPackageEntryPoints(manifest.sha, path, read) : undefined
|
|
129
|
-
|
|
130
|
+
const solidity = await readSolidityEntryPoints(path, data, read)
|
|
131
|
+
return { entries, limited: data.length >= 1000, ...(packageEntryPoints ? { packageEntryPoints } : {}),
|
|
132
|
+
...(solidity.paths.length > 0 || solidity.limited ? { solidityEntryPoints: solidity.paths, soliditySuggestionsLimited: solidity.limited } : {}),
|
|
133
|
+
}
|
|
130
134
|
},
|
|
131
135
|
}
|
|
132
136
|
}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import { normalizeTeamPath } from './repo-path.ts'
|
|
2
|
+
|
|
3
|
+
const MAX_SUGGESTIONS = 100
|
|
4
|
+
const EXCLUDED_DIRECTORIES = new Set(['test', 'tests', 'script', 'scripts', 'mock', 'mocks', 'node_modules', 'vendor', 'out', 'artifacts', 'cache', 'build', 'dist', 'broadcast'])
|
|
5
|
+
|
|
6
|
+
function sourcePath(value: unknown): value is string {
|
|
7
|
+
if (typeof value !== 'string' || !value.endsWith('.sol') || /\.(?:t|s|test|spec)\.sol$/iu.test(value)) return false
|
|
8
|
+
const normalized = normalizeTeamPath(value)
|
|
9
|
+
const directories = value.split('/').slice(0, -1).map(part => part.toLowerCase())
|
|
10
|
+
const sourceRoot = directories.findIndex(part => part === 'src' || part === 'contracts')
|
|
11
|
+
return normalized.ok && normalized.path === value && !directories.some((part, index) =>
|
|
12
|
+
EXCLUDED_DIRECTORIES.has(part) || (part === 'lib' && (sourceRoot < 0 || index < sourceRoot)))
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
type ListingEntry = { name?: unknown; path?: unknown; type?: unknown; sha?: unknown; submodule_git_url?: unknown }
|
|
16
|
+
|
|
17
|
+
// Maintain the alphabetically first 100 unique candidates even when GitHub's
|
|
18
|
+
// input is unordered. Return whether a unique candidate exceeds the capacity.
|
|
19
|
+
export function addSoliditySuggestion(paths: string[], path: string): boolean {
|
|
20
|
+
let start = 0
|
|
21
|
+
let end = paths.length
|
|
22
|
+
while (start < end) {
|
|
23
|
+
const middle = Math.floor((start + end) / 2)
|
|
24
|
+
if (paths[middle]! < path) start = middle + 1
|
|
25
|
+
else end = middle
|
|
26
|
+
}
|
|
27
|
+
if (paths[start] === path) return false
|
|
28
|
+
const full = paths.length === MAX_SUGGESTIONS
|
|
29
|
+
if (start < MAX_SUGGESTIONS) {
|
|
30
|
+
// Evict before inserting, so retained candidates never exceed the cap.
|
|
31
|
+
if (full) paths.pop()
|
|
32
|
+
paths.splice(start, 0, path)
|
|
33
|
+
}
|
|
34
|
+
return full
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
// Suggestions from filenames, not deployability or import analysis. Read at
|
|
38
|
+
// most two source trees, using SHAs from the authorized pinned directory.
|
|
39
|
+
export async function readSolidityEntryPoints(directory: string, listing: unknown[], read: (suffix: string) => Promise<unknown>) {
|
|
40
|
+
const prefix = directory ? directory + '/' : ''
|
|
41
|
+
const entries = listing.filter((entry): entry is ListingEntry => !!entry && typeof entry === 'object').filter(entry =>
|
|
42
|
+
typeof entry.name === 'string' && !entry.name.includes('/') && entry.path === prefix + entry.name && !entry.submodule_git_url)
|
|
43
|
+
const hasFramework = entries.some(entry => entry.type === 'file' && typeof entry.name === 'string'
|
|
44
|
+
&& (entry.name === 'foundry.toml' || /^hardhat\.config\.(?:[cm]?js|ts)$/u.test(entry.name)))
|
|
45
|
+
const roots = entries.filter(entry => entry.type === 'dir' && (entry.name === 'contracts' || (hasFramework && entry.name === 'src'))
|
|
46
|
+
&& typeof entry.sha === 'string' && /^[a-f\d]{40}$/iu.test(entry.sha)).slice(0, 2)
|
|
47
|
+
const paths: string[] = []
|
|
48
|
+
let limited = false
|
|
49
|
+
for (const entry of entries) {
|
|
50
|
+
if (entry.type === 'file' && sourcePath(entry.path) && addSoliditySuggestion(paths, entry.path)) limited = true
|
|
51
|
+
}
|
|
52
|
+
for (const root of roots) {
|
|
53
|
+
if (!sourcePath(`${root.path}/Contract.sol`)) continue
|
|
54
|
+
try {
|
|
55
|
+
const data = await read(`/git/trees/${root.sha}?recursive=1`) as { tree?: unknown; truncated?: unknown }
|
|
56
|
+
if (!Array.isArray(data?.tree)) continue
|
|
57
|
+
limited ||= data.truncated === true
|
|
58
|
+
for (const entry of data.tree) {
|
|
59
|
+
if (entry?.type !== 'blob' || !['100644', '100755'].includes(entry.mode) || typeof entry.path !== 'string') continue
|
|
60
|
+
const path = `${root.path}/${entry.path}`
|
|
61
|
+
if (sourcePath(path) && addSoliditySuggestion(paths, path)) limited = true
|
|
62
|
+
}
|
|
63
|
+
} catch {
|
|
64
|
+
// Optional suggestions must not block browsing. The request handler
|
|
65
|
+
// rechecks GitHub access and local path grants after all reads.
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
return { paths, limited }
|
|
69
|
+
}
|
package/server-managed/sql.ts
CHANGED
|
@@ -4,11 +4,6 @@ import type { DatabaseSync, SQLInputValue } from 'node:sqlite'
|
|
|
4
4
|
import type { ManagedDb } from './db-methods.ts'
|
|
5
5
|
|
|
6
6
|
type MaybePromise<T> = T | Promise<T>
|
|
7
|
-
// A failed COMMIT acknowledgement (or connection close after COMMIT) does not
|
|
8
|
-
// establish rollback. Callers must retain primary bytes until reconciled.
|
|
9
|
-
export class ManagedCommitError extends Error {
|
|
10
|
-
constructor(cause: unknown) { super('Managed commit outcome is uncertain', { cause }) }
|
|
11
|
-
}
|
|
12
7
|
export interface ManagedSql {
|
|
13
8
|
prepare(sql: string): {
|
|
14
9
|
get(...params: unknown[]): MaybePromise<unknown>
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { loadManagedConfig } from './config.ts'
|
|
2
|
+
import { openManagedStorageDb } from './storage.ts'
|
|
3
|
+
|
|
4
|
+
// Require the deployment mode: DB_PATH belongs to e2e in combined deployments.
|
|
5
|
+
// Read the database requirement without opening bytes or enabling encryption.
|
|
6
|
+
export async function storageCommand(mode: string | undefined): Promise<void> {
|
|
7
|
+
if (mode !== 'managed' && mode !== 'managed-e2e' && mode !== 'e2e-managed') throw new Error('Storage status requires a deployment mode: --storage-encryption-status managed|managed-e2e|e2e-managed')
|
|
8
|
+
const db = await openManagedStorageDb(loadManagedConfig({ combined: mode !== 'managed' }))
|
|
9
|
+
try {
|
|
10
|
+
const status = await db.getStorageEncryption()
|
|
11
|
+
console.log(JSON.stringify(status ? {
|
|
12
|
+
encryption: 'chacha20-poly1305', complete: status.complete === 1, cleanupComplete: status.cleanupComplete === 1,
|
|
13
|
+
migrated: status.migrated, cursor: status.cursor,
|
|
14
|
+
} : { encryption: 'disabled' }))
|
|
15
|
+
} finally { await db.close() }
|
|
16
|
+
}
|