@preventive/triage 1.0.0-alpha.25 → 1.0.0-alpha.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/common/server-info.ts +1 -0
- package/out/client-managed-import.js +18 -18
- package/out/client-managed.js +66 -65
- package/out/prism.js +2 -2
- package/out/view.css +1 -1
- package/out/view.js +182 -182
- package/package.json +12 -2
- package/server-common/storage-crypto.ts +182 -0
- package/server-common/storage-log.ts +5 -1
- package/server-common/vercel-blob.ts +6 -3
- package/server-e2e/config.ts +4 -0
- package/server-managed/avatar-store.ts +1 -40
- package/server-managed/blob-store.ts +3 -52
- package/server-managed/bundle-cache.ts +0 -28
- package/server-managed/bundle-store.ts +3 -7
- package/server-managed/cache-storage.ts +0 -25
- package/server-managed/cli.js +10 -2
- package/server-managed/config.ts +14 -2
- package/server-managed/db-methods.ts +56 -25
- package/server-managed/db-neon.ts +9 -4
- package/server-managed/db.ts +7 -1
- package/server-managed/http.ts +93 -62
- package/server-managed/import-triage.ts +6 -3
- package/server-managed/index.ts +32 -9
- package/server-managed/object-storage-disk.ts +144 -0
- package/server-managed/object-storage-vercel.ts +60 -0
- package/server-managed/object-storage.ts +71 -0
- package/server-managed/report-query.ts +6 -0
- package/server-managed/repository-browser.ts +5 -1
- package/server-managed/solidity-entry-points.ts +69 -0
- package/server-managed/sql.ts +0 -5
- package/server-managed/storage-command.ts +16 -0
- package/server-managed/storage-db.ts +168 -0
- package/server-managed/storage-encryption.ts +211 -0
- package/server-managed/storage-maintenance.ts +182 -0
- package/server-managed/storage-payload.ts +60 -0
- package/server-managed/storage-stores.ts +81 -0
- package/server-managed/storage.ts +46 -24
- package/server-managed/uploads.ts +30 -12
- package/server-managed/blob-vercel.ts +0 -125
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@preventive/triage",
|
|
3
|
-
"version": "1.0.0-alpha.
|
|
3
|
+
"version": "1.0.0-alpha.27",
|
|
4
4
|
"description": "Client & relay server for triaging of automated reports",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": {
|
|
@@ -89,7 +89,6 @@
|
|
|
89
89
|
"server-managed/activity.ts",
|
|
90
90
|
"server-managed/avatar-store.ts",
|
|
91
91
|
"server-managed/blob-store.ts",
|
|
92
|
-
"server-managed/blob-vercel.ts",
|
|
93
92
|
"server-managed/brotli.ts",
|
|
94
93
|
"server-managed/bundle-cache.ts",
|
|
95
94
|
"server-managed/bundle-catalog.ts",
|
|
@@ -120,6 +119,9 @@
|
|
|
120
119
|
"server-managed/import-triage.ts",
|
|
121
120
|
"server-managed/managed-issues.ts",
|
|
122
121
|
"server-managed/management.ts",
|
|
122
|
+
"server-managed/object-storage.ts",
|
|
123
|
+
"server-managed/object-storage-disk.ts",
|
|
124
|
+
"server-managed/object-storage-vercel.ts",
|
|
123
125
|
"server-managed/index.ts",
|
|
124
126
|
"server-managed/package-entry-points.ts",
|
|
125
127
|
"server-managed/public-workspace.ts",
|
|
@@ -133,10 +135,17 @@
|
|
|
133
135
|
"server-managed/repository-policy.ts",
|
|
134
136
|
"server-managed/session.ts",
|
|
135
137
|
"server-managed/slugs.ts",
|
|
138
|
+
"server-managed/solidity-entry-points.ts",
|
|
136
139
|
"server-managed/sql-postgres.ts",
|
|
137
140
|
"server-managed/sql.ts",
|
|
138
141
|
"server-managed/static.ts",
|
|
139
142
|
"server-managed/storage.ts",
|
|
143
|
+
"server-managed/storage-command.ts",
|
|
144
|
+
"server-managed/storage-db.ts",
|
|
145
|
+
"server-managed/storage-encryption.ts",
|
|
146
|
+
"server-managed/storage-maintenance.ts",
|
|
147
|
+
"server-managed/storage-payload.ts",
|
|
148
|
+
"server-managed/storage-stores.ts",
|
|
140
149
|
"server-managed/team-catalog.ts",
|
|
141
150
|
"server-managed/team-feed.ts",
|
|
142
151
|
"server-managed/team-reports.ts",
|
|
@@ -150,6 +159,7 @@
|
|
|
150
159
|
"server-common/origin.ts",
|
|
151
160
|
"server-common/standalone.ts",
|
|
152
161
|
"server-common/storage-log.ts",
|
|
162
|
+
"server-common/storage-crypto.ts",
|
|
153
163
|
"server-common/vercel-blob.ts",
|
|
154
164
|
"server-common/scan-config.ts",
|
|
155
165
|
"report/index.js",
|
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
// Storage envelope v1: header, then 64 KiB plaintext chunks with full AEAD tags.
|
|
2
|
+
// The nonce layout follows age's STREAM construction (counter || final flag).
|
|
3
|
+
// Every write derives a new key, including rewrites of the same logical object.
|
|
4
|
+
import { Buffer } from 'node:buffer'
|
|
5
|
+
import { createCipheriv, createDecipheriv, createHash, hkdfSync, randomBytes } from 'node:crypto'
|
|
6
|
+
import { Readable } from 'node:stream'
|
|
7
|
+
|
|
8
|
+
export const STORAGE_MAGIC = Buffer.from('DeepView.storage')
|
|
9
|
+
const VERSION = 1
|
|
10
|
+
export const STORAGE_CHUNK_BYTES = 65_536
|
|
11
|
+
export const STORAGE_HEADER_BYTES = 57
|
|
12
|
+
const TAG_BYTES = 16
|
|
13
|
+
const UNKNOWN_SIZE = 0xffff_ffff_ffff_ffffn
|
|
14
|
+
|
|
15
|
+
export interface StorageKey { bytes: Buffer; id: string }
|
|
16
|
+
|
|
17
|
+
export function parseStorageKey(value: string | null | undefined): StorageKey | null {
|
|
18
|
+
if (value == null || value === '') return null
|
|
19
|
+
const bytes = Buffer.from(value, 'base64')
|
|
20
|
+
if (bytes.length !== 32 || bytes.toString('base64') !== value) {
|
|
21
|
+
throw new Error('MANAGED_STORAGE_ENCRYPTION_KEY must be canonical base64 encoding of 32 random bytes')
|
|
22
|
+
}
|
|
23
|
+
const id = createHash('sha256').update('deepview.storage.key.v1\0').update(bytes).digest('hex').slice(0, 32)
|
|
24
|
+
return { bytes, id }
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
// Re-chunk without accumulating a file, regardless of the input's read sizes.
|
|
28
|
+
async function* chunks(source: AsyncIterable<Uint8Array>, size: number): AsyncGenerator<Buffer> {
|
|
29
|
+
let length = 0, pending: Buffer[] = []
|
|
30
|
+
for await (const value of source) {
|
|
31
|
+
const input = Buffer.from(value.buffer, value.byteOffset, value.byteLength)
|
|
32
|
+
for (let offset = 0; offset < input.length;) {
|
|
33
|
+
const n = Math.min(size - length, input.length - offset)
|
|
34
|
+
pending.push(input.subarray(offset, offset + n)); length += n; offset += n
|
|
35
|
+
if (length === size) { yield Buffer.concat(pending, length); pending = []; length = 0 }
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
if (length) yield Buffer.concat(pending, length)
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
async function* frames(source: AsyncIterable<Uint8Array>, size: number) {
|
|
42
|
+
const iterator = chunks(source, size)[Symbol.asyncIterator]()
|
|
43
|
+
try {
|
|
44
|
+
let current = await iterator.next()
|
|
45
|
+
if (current.done) { yield { bytes: Buffer.alloc(0), last: true }; return }
|
|
46
|
+
while (!current.done) {
|
|
47
|
+
const next = await iterator.next()
|
|
48
|
+
yield { bytes: current.value, last: next.done === true }
|
|
49
|
+
current = next
|
|
50
|
+
}
|
|
51
|
+
} finally { await iterator.return(undefined) }
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
function nonce(index: number, last: boolean): Buffer {
|
|
55
|
+
if (!Number.isSafeInteger(index) || index < 0 || index > 0xffff_ffff) throw new Error('Storage object has too many chunks')
|
|
56
|
+
const value = Buffer.alloc(12)
|
|
57
|
+
value.writeUInt32BE(index, 7)
|
|
58
|
+
value[11] = last ? 1 : 0
|
|
59
|
+
return value
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
function objectKey(key: Uint8Array, header: Buffer, identity: string): Buffer {
|
|
63
|
+
return Buffer.from(hkdfSync('sha256', key, header.subarray(17, 49),
|
|
64
|
+
Buffer.from(JSON.stringify(['deepview.storage.v1', identity])), 32))
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
function makeHeader(size: number | null): Buffer {
|
|
68
|
+
if (size !== null && (!Number.isSafeInteger(size) || size < 0)) throw new Error('Invalid storage size')
|
|
69
|
+
const header = Buffer.alloc(STORAGE_HEADER_BYTES)
|
|
70
|
+
STORAGE_MAGIC.copy(header); header[16] = VERSION
|
|
71
|
+
randomBytes(32).copy(header, 17)
|
|
72
|
+
header.writeBigUInt64BE(size === null ? UNKNOWN_SIZE : BigInt(size), 49)
|
|
73
|
+
return header
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
export function encryptStorageStream(source: Readable, key: Uint8Array, identity: string, size: number | null): Readable {
|
|
77
|
+
async function* encrypt() {
|
|
78
|
+
const header = makeHeader(size)
|
|
79
|
+
const derived = objectKey(key, header, identity)
|
|
80
|
+
let index = 0, length = 0
|
|
81
|
+
try {
|
|
82
|
+
yield header
|
|
83
|
+
for await (const { bytes, last } of frames(source, STORAGE_CHUNK_BYTES)) {
|
|
84
|
+
length += bytes.length
|
|
85
|
+
if (last && size !== null && length !== size) throw new Error('Storage size changed while encrypting')
|
|
86
|
+
const cipher = createCipheriv('chacha20-poly1305', derived, nonce(index++, last), { authTagLength: TAG_BYTES })
|
|
87
|
+
cipher.setAAD(header)
|
|
88
|
+
yield Buffer.concat([cipher.update(bytes), cipher.final(), cipher.getAuthTag()])
|
|
89
|
+
}
|
|
90
|
+
} finally { derived.fill(0); source.destroy() }
|
|
91
|
+
}
|
|
92
|
+
const stream = Readable.from(encrypt(), { objectMode: false })
|
|
93
|
+
stream.once('close', () => source.destroy())
|
|
94
|
+
return stream
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
// Read just the envelope and authenticate the first frame before returning.
|
|
98
|
+
// In particular, never trust a corrupted size or release update() output
|
|
99
|
+
// before final() authenticates the complete chunk.
|
|
100
|
+
export async function decryptStorageStream(source: Readable, key: Uint8Array, identity: string) {
|
|
101
|
+
const iterator = source[Symbol.asyncIterator]()
|
|
102
|
+
let done = false
|
|
103
|
+
let peek: Buffer = Buffer.alloc(0), remainder: Buffer = Buffer.alloc(0)
|
|
104
|
+
async function read(size: number): Promise<Buffer> {
|
|
105
|
+
const parts: Buffer[] = peek.length > 0 ? [peek] : []
|
|
106
|
+
let length = peek.length
|
|
107
|
+
peek = Buffer.alloc(0)
|
|
108
|
+
while (length < size) {
|
|
109
|
+
if (remainder.length === 0 && !done) {
|
|
110
|
+
const next = await iterator.next(); done = next.done === true
|
|
111
|
+
remainder = done ? Buffer.alloc(0) : next.value
|
|
112
|
+
}
|
|
113
|
+
if (remainder.length === 0 && done) break
|
|
114
|
+
const n = Math.min(size - length, remainder.length)
|
|
115
|
+
parts.push(remainder.subarray(0, n)); remainder = remainder.subarray(n); length += n
|
|
116
|
+
}
|
|
117
|
+
return Buffer.concat(parts, length)
|
|
118
|
+
}
|
|
119
|
+
let derived: Buffer | undefined
|
|
120
|
+
try {
|
|
121
|
+
const header = await read(STORAGE_HEADER_BYTES)
|
|
122
|
+
if (header.length !== STORAGE_HEADER_BYTES || !header.subarray(0, 16).equals(STORAGE_MAGIC) || header[16] !== VERSION) {
|
|
123
|
+
throw new Error('Invalid encrypted storage envelope')
|
|
124
|
+
}
|
|
125
|
+
const rawSize = header.readBigUInt64BE(49)
|
|
126
|
+
if (rawSize !== UNKNOWN_SIZE && rawSize > BigInt(Number.MAX_SAFE_INTEGER)) throw new Error('Invalid encrypted storage size')
|
|
127
|
+
const size = rawSize === UNKNOWN_SIZE ? null : Number(rawSize)
|
|
128
|
+
derived = objectKey(key, header, identity)
|
|
129
|
+
const derivedKey = derived
|
|
130
|
+
let index = 0, length = 0
|
|
131
|
+
const nextFrame = async () => {
|
|
132
|
+
const bytes = await read(STORAGE_CHUNK_BYTES + TAG_BYTES)
|
|
133
|
+
// Peek beyond the frame: EOF authenticates the final flag, including
|
|
134
|
+
// exact chunk boundaries. Keep the byte for the next read.
|
|
135
|
+
peek = await read(1)
|
|
136
|
+
const last = peek.length === 0
|
|
137
|
+
if (bytes.length < TAG_BYTES || (index > 0 && bytes.length === TAG_BYTES)) throw new Error('Truncated encrypted storage object')
|
|
138
|
+
const decipher = createDecipheriv('chacha20-poly1305', derivedKey, nonce(index++, last), { authTagLength: TAG_BYTES })
|
|
139
|
+
decipher.setAAD(header); decipher.setAuthTag(bytes.subarray(-TAG_BYTES))
|
|
140
|
+
const plain = Buffer.concat([decipher.update(bytes.subarray(0, -TAG_BYTES)), decipher.final()])
|
|
141
|
+
length += plain.length
|
|
142
|
+
if (size !== null && (length > size || (last && length !== size))) throw new Error('Encrypted storage size mismatch')
|
|
143
|
+
return { plain, last }
|
|
144
|
+
}
|
|
145
|
+
const first = await nextFrame()
|
|
146
|
+
const decrypt = async function* () {
|
|
147
|
+
try {
|
|
148
|
+
yield first.plain
|
|
149
|
+
let last = first.last
|
|
150
|
+
while (!last) { const frame = await nextFrame(); last = frame.last; yield frame.plain }
|
|
151
|
+
} finally { derivedKey.fill(0); await iterator.return?.(); source.destroy() }
|
|
152
|
+
}
|
|
153
|
+
const stream = Readable.from(decrypt(), { objectMode: false })
|
|
154
|
+
stream.once('close', () => { derivedKey.fill(0); source.destroy() })
|
|
155
|
+
return { size, stream }
|
|
156
|
+
} catch (err) { derived?.fill(0); source.destroy(); throw err }
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
// Small SQL values (data keys, OAuth tokens and the installation sentinel).
|
|
160
|
+
// Fresh salt derives a one-use wrapping key, so the nonce can be all zeroes.
|
|
161
|
+
// The version and master-key ID are authenticated along with the row/field.
|
|
162
|
+
export function wrapStorageValue(key: StorageKey, identity: string, value: Uint8Array): string {
|
|
163
|
+
const header = Buffer.concat([Buffer.from([1]), Buffer.from(key.id, 'hex'), randomBytes(32)])
|
|
164
|
+
const derived = Buffer.from(hkdfSync('sha256', key.bytes, header.subarray(17), 'deepview.wrap.v1', 32))
|
|
165
|
+
try {
|
|
166
|
+
const cipher = createCipheriv('chacha20-poly1305', derived, Buffer.alloc(12), { authTagLength: TAG_BYTES })
|
|
167
|
+
cipher.setAAD(Buffer.concat([header, Buffer.from(identity)]))
|
|
168
|
+
return Buffer.concat([header, cipher.update(value), cipher.final(), cipher.getAuthTag()]).toString('base64')
|
|
169
|
+
} finally { derived.fill(0) }
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
export function unwrapStorageValue(key: StorageKey, identity: string, value: string): Buffer {
|
|
173
|
+
const bytes = Buffer.from(value, 'base64'), header = bytes.subarray(0, 49)
|
|
174
|
+
if (bytes.length < 65 || bytes.toString('base64') !== value || header[0] !== 1
|
|
175
|
+
|| header.subarray(1, 17).toString('hex') !== key.id) throw new Error('Invalid wrapped storage value or encryption key')
|
|
176
|
+
const derived = Buffer.from(hkdfSync('sha256', key.bytes, header.subarray(17), 'deepview.wrap.v1', 32))
|
|
177
|
+
try {
|
|
178
|
+
const decipher = createDecipheriv('chacha20-poly1305', derived, Buffer.alloc(12), { authTagLength: TAG_BYTES })
|
|
179
|
+
decipher.setAAD(Buffer.concat([header, Buffer.from(identity)])); decipher.setAuthTag(bytes.subarray(-TAG_BYTES))
|
|
180
|
+
return Buffer.concat([decipher.update(bytes.subarray(49, -TAG_BYTES)), decipher.final()])
|
|
181
|
+
} finally { derived.fill(0) }
|
|
182
|
+
}
|
|
@@ -19,11 +19,15 @@ export function e2eStorageLines(config: DatabaseConfig & { objstoreDir: string }
|
|
|
19
19
|
]
|
|
20
20
|
}
|
|
21
21
|
|
|
22
|
-
export function managedStorageLines(config: DatabaseConfig): string[] {
|
|
22
|
+
export function managedStorageLines(config: DatabaseConfig & { storageEncryptionKey?: string | null }): string[] {
|
|
23
23
|
const dir = dirname(config.dbPath)
|
|
24
24
|
const location = (path: string) => config.neonUrl ? `Vercel Blob (private), .managed/${path}/` : resolve(dir, path)
|
|
25
25
|
return [
|
|
26
26
|
` Managed database: ${databaseLocation(config)}`,
|
|
27
|
+
...(config.storageEncryptionKey ? [
|
|
28
|
+
' Managed storage encryption: ChaCha20-Poly1305',
|
|
29
|
+
` Managed encrypted caches: ${location('cache-encrypted-v1')}`,
|
|
30
|
+
] : []),
|
|
27
31
|
` Managed reports: ${location('reports')}`,
|
|
28
32
|
` Managed bundles: ${location('bundles')}`,
|
|
29
33
|
` Managed avatars: ${location('avatars')}`,
|
|
@@ -18,6 +18,7 @@ import type { Buffer } from 'node:buffer'
|
|
|
18
18
|
type VercelBlobBody = Readable | Buffer | string | Blob | ArrayBuffer | ReadableStream<Uint8Array>
|
|
19
19
|
export type VercelBlobSdk = {
|
|
20
20
|
BlobNotFoundError: new () => Error
|
|
21
|
+
BlobPreconditionFailedError?: new () => Error
|
|
21
22
|
put: (
|
|
22
23
|
pathname: string,
|
|
23
24
|
body: VercelBlobBody,
|
|
@@ -27,6 +28,7 @@ export type VercelBlobSdk = {
|
|
|
27
28
|
allowOverwrite?: boolean
|
|
28
29
|
contentType?: string
|
|
29
30
|
token?: string
|
|
31
|
+
ifMatch?: string
|
|
30
32
|
multipart?: boolean
|
|
31
33
|
abortSignal?: AbortSignal
|
|
32
34
|
cacheControlMaxAge?: number
|
|
@@ -35,7 +37,7 @@ export type VercelBlobSdk = {
|
|
|
35
37
|
head: (
|
|
36
38
|
pathname: string,
|
|
37
39
|
options?: { token?: string; abortSignal?: AbortSignal },
|
|
38
|
-
) => Promise<{ size: number; pathname: string; url: string }>
|
|
40
|
+
) => Promise<{ size: number; pathname: string; url: string; etag?: string; uploadedAt?: Date | string | number }>
|
|
39
41
|
get: (
|
|
40
42
|
pathname: string,
|
|
41
43
|
options: {
|
|
@@ -47,7 +49,7 @@ export type VercelBlobSdk = {
|
|
|
47
49
|
) => Promise<{
|
|
48
50
|
statusCode: 200 | 304
|
|
49
51
|
stream: ReadableStream<Uint8Array> | null
|
|
50
|
-
blob: { size: number | null }
|
|
52
|
+
blob: { size: number | null; etag?: string; uploadedAt?: Date | string | number }
|
|
51
53
|
} | null>
|
|
52
54
|
copy: (
|
|
53
55
|
fromPathname: string,
|
|
@@ -63,7 +65,7 @@ export type VercelBlobSdk = {
|
|
|
63
65
|
) => Promise<{ url: string; pathname: string }>
|
|
64
66
|
del: (
|
|
65
67
|
urlOrPathname: string | string[],
|
|
66
|
-
options?: { token?: string; abortSignal?: AbortSignal },
|
|
68
|
+
options?: { token?: string; abortSignal?: AbortSignal; ifMatch?: string },
|
|
67
69
|
) => Promise<void>
|
|
68
70
|
list: (options: {
|
|
69
71
|
prefix?: string
|
|
@@ -71,6 +73,7 @@ export type VercelBlobSdk = {
|
|
|
71
73
|
limit?: number
|
|
72
74
|
mode?: 'expanded' | 'folded'
|
|
73
75
|
token?: string
|
|
76
|
+
abortSignal?: AbortSignal
|
|
74
77
|
}) => Promise<{
|
|
75
78
|
// `uploadedAt` (a Date per the SDK v2 surface) is the blob's
|
|
76
79
|
// creation time — used by the reaper's GC grace window. Optional
|
package/server-e2e/config.ts
CHANGED
|
@@ -54,6 +54,10 @@ Environment:
|
|
|
54
54
|
DEEPVIEW_SCAN_SERVER optional HTTP(S) scan-service URL; advertised
|
|
55
55
|
in /api/config and allowed by the UI CSP.
|
|
56
56
|
Unset by default (no external scan service).
|
|
57
|
+
GITHUB_NEW_ISSUE_LABELS optional comma-separated labels for the client's
|
|
58
|
+
prefilled GitHub issue form, in addition to
|
|
59
|
+
deepview and security (for security findings).
|
|
60
|
+
Advertised in /api/config and sync connections.
|
|
57
61
|
DB_PATH sqlite file (default: server-e2e/data/data.db);
|
|
58
62
|
ignored when an e2e database URL is set
|
|
59
63
|
DATABASE_URL shared Neon Postgres URL for enabled modes.
|
|
@@ -1,12 +1,4 @@
|
|
|
1
|
-
//
|
|
2
|
-
// at login (see github-oauth.ts) and served same-origin from GET
|
|
3
|
-
// /api/auth/avatar — the app's CSP is `img-src 'self'`, so the GitHub-hosted
|
|
4
|
-
// original can't be loaded by the page directly.
|
|
5
|
-
//
|
|
6
|
-
// The disk backend uses a dir beside SQLite; blob-vercel.ts implements the
|
|
7
|
-
// same AvatarStore interface for remote storage. Keyed by the user's opaque `id`.
|
|
8
|
-
import { mkdir, readFile, writeFile } from 'node:fs/promises'
|
|
9
|
-
import { dirname, join } from 'node:path'
|
|
1
|
+
// Shared avatar cache interface, implemented by storage-stores.ts.
|
|
10
2
|
import type { Buffer } from 'node:buffer'
|
|
11
3
|
|
|
12
4
|
export interface CachedAvatar {
|
|
@@ -18,34 +10,3 @@ export interface AvatarStore {
|
|
|
18
10
|
put(id: string, contentType: string, bytes: Buffer): Promise<void>
|
|
19
11
|
get(id: string): Promise<CachedAvatar | null>
|
|
20
12
|
}
|
|
21
|
-
|
|
22
|
-
// The user id is a v4 uuid (crypto.randomUUID) — lowercase hex + dashes, no
|
|
23
|
-
// path separators. Validated anyway so a crafted id can't escape the store dir.
|
|
24
|
-
const ID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/u
|
|
25
|
-
|
|
26
|
-
export function createDiskAvatarStore(dir: string): AvatarStore {
|
|
27
|
-
function pathFor(id: string): string {
|
|
28
|
-
if (!ID_RE.test(id)) throw new Error('invalid avatar id')
|
|
29
|
-
return join(dir, id)
|
|
30
|
-
}
|
|
31
|
-
return {
|
|
32
|
-
async put(id, contentType, bytes) {
|
|
33
|
-
const p = pathFor(id)
|
|
34
|
-
await mkdir(dirname(p), { recursive: true })
|
|
35
|
-
// Bytes first, then the content-type sidecar — a crash between the two
|
|
36
|
-
// leaves a typeless blob that `get` treats as missing (re-fetched next
|
|
37
|
-
// login) rather than serving with a wrong/absent type.
|
|
38
|
-
await writeFile(p, bytes)
|
|
39
|
-
await writeFile(`${p}.type`, contentType, 'utf8')
|
|
40
|
-
},
|
|
41
|
-
async get(id) {
|
|
42
|
-
try {
|
|
43
|
-
const p = pathFor(id)
|
|
44
|
-
const [bytes, contentType] = await Promise.all([readFile(p), readFile(`${p}.type`, 'utf8')])
|
|
45
|
-
return { bytes, contentType: contentType.trim() || 'application/octet-stream' }
|
|
46
|
-
} catch {
|
|
47
|
-
return null
|
|
48
|
-
}
|
|
49
|
-
},
|
|
50
|
-
}
|
|
51
|
-
}
|
|
@@ -1,14 +1,4 @@
|
|
|
1
|
-
//
|
|
2
|
-
// bundles). Unlike the e2e relay (which only ever holds opaque client-encrypted
|
|
3
|
-
// blobs), a managed server is TRUSTED and stores the bytes in the clear so it
|
|
4
|
-
// can operate on them — the metadata + attribution live in the DB, the bytes
|
|
5
|
-
// live here, keyed by the blob's opaque `id`.
|
|
6
|
-
//
|
|
7
|
-
// The disk backend uses a dir beside SQLite (data/reports/<uuid> or
|
|
8
|
-
// data/bundles/<uuid>); blob-vercel.ts implements the same interface remotely. Bytes only — the
|
|
9
|
-
// content-type / filename / integrity ride the DB row, so there's no sidecar.
|
|
10
|
-
import { mkdir, open, readFile, rm, writeFile } from 'node:fs/promises'
|
|
11
|
-
import { dirname, join } from 'node:path'
|
|
1
|
+
// Logical upload-store interfaces shared by disk and Vercel storage.
|
|
12
2
|
import type { Buffer } from 'node:buffer'
|
|
13
3
|
import type { Readable } from 'node:stream'
|
|
14
4
|
|
|
@@ -19,48 +9,9 @@ export interface OpenedBlob {
|
|
|
19
9
|
}
|
|
20
10
|
|
|
21
11
|
export interface BlobStore {
|
|
22
|
-
|
|
12
|
+
// Encrypted upload stores return the wrapped data key for the SQL insert.
|
|
13
|
+
put(id: string, bytes: Buffer): Promise<string | null>
|
|
23
14
|
get(id: string): Promise<Buffer | null>
|
|
24
15
|
open(id: string): Promise<OpenedBlob | null>
|
|
25
16
|
delete(id: string): Promise<void>
|
|
26
17
|
}
|
|
27
|
-
|
|
28
|
-
// The blob id is a v4 uuid (crypto.randomUUID) — lowercase hex + dashes, no
|
|
29
|
-
// path separators. Validated anyway so a crafted id can't escape the store dir.
|
|
30
|
-
const ID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/u
|
|
31
|
-
|
|
32
|
-
export function createDiskBlobStore(dir: string, suffix = ''): BlobStore {
|
|
33
|
-
function pathFor(id: string): string {
|
|
34
|
-
if (!ID_RE.test(id)) throw new Error('invalid blob id')
|
|
35
|
-
return join(dir, id + suffix)
|
|
36
|
-
}
|
|
37
|
-
return {
|
|
38
|
-
async put(id, bytes) {
|
|
39
|
-
const p = pathFor(id)
|
|
40
|
-
await mkdir(dirname(p), { recursive: true })
|
|
41
|
-
await writeFile(p, bytes)
|
|
42
|
-
},
|
|
43
|
-
async get(id) {
|
|
44
|
-
try {
|
|
45
|
-
return await readFile(pathFor(id))
|
|
46
|
-
} catch {
|
|
47
|
-
return null
|
|
48
|
-
}
|
|
49
|
-
},
|
|
50
|
-
async open(id) {
|
|
51
|
-
let file
|
|
52
|
-
try { file = await open(pathFor(id), 'r') }
|
|
53
|
-
catch (err) { if ((err as NodeJS.ErrnoException).code === 'ENOENT') return null; throw err }
|
|
54
|
-
try {
|
|
55
|
-
const info = await file.stat()
|
|
56
|
-
// Remain paused until piped. HEAD can close without reading the body.
|
|
57
|
-
return { size: info.size, stream: file.createReadStream() }
|
|
58
|
-
} catch (err) { await file.close(); throw err }
|
|
59
|
-
},
|
|
60
|
-
async delete(id) {
|
|
61
|
-
// `force` so a missing file (already gone) is a no-op, not a throw — the
|
|
62
|
-
// metadata row is the source of truth for whether a blob "exists".
|
|
63
|
-
await rm(pathFor(id), { force: true })
|
|
64
|
-
},
|
|
65
|
-
}
|
|
66
|
-
}
|
|
@@ -1,9 +1,6 @@
|
|
|
1
1
|
// Metadata is cached as Brotli. Contents use the stored Brotli bytes directly:
|
|
2
2
|
// unchanged Stasis uploads or sourcemaps compressed once at upload.
|
|
3
3
|
import { Buffer } from 'node:buffer'
|
|
4
|
-
import { mkdir, open, rename, rm, stat, writeFile } from 'node:fs/promises'
|
|
5
|
-
import { join } from 'node:path'
|
|
6
|
-
import { randomUUID } from 'node:crypto'
|
|
7
4
|
import { brotliDecompress } from 'node:zlib'
|
|
8
5
|
import { promisify } from 'node:util'
|
|
9
6
|
import { BUNDLE_METADATA_VERSION, type BundleDetails, createBundleMetadata, createBundleSummary, parseBundleContents } from '../common/bundle-metadata.js'
|
|
@@ -158,28 +155,3 @@ export function createBundleCache(storage: BundleCacheStorage, db: ManagedDb, st
|
|
|
158
155
|
}
|
|
159
156
|
}
|
|
160
157
|
export type BundleCache = ReturnType<typeof createBundleCache>
|
|
161
|
-
|
|
162
|
-
export function createDiskBundleCache(dir: string, db: ManagedDb, store: BundleStore): BundleCache {
|
|
163
|
-
function directory(id: string) {
|
|
164
|
-
if (!/^[a-f\d-]{36}$/iu.test(id)) throw new Error('Invalid bundle id')
|
|
165
|
-
return join(dir, id)
|
|
166
|
-
}
|
|
167
|
-
return createBundleCache({
|
|
168
|
-
async exists(id, file) {
|
|
169
|
-
try { await stat(join(directory(id), file)); return true }
|
|
170
|
-
catch (err) { if ((err as NodeJS.ErrnoException).code === 'ENOENT') return false; throw err }
|
|
171
|
-
},
|
|
172
|
-
async put(id, file, bytes) {
|
|
173
|
-
await mkdir(directory(id), { recursive: true })
|
|
174
|
-
const target = join(directory(id), file), temp = `${target}.${randomUUID()}.tmp`
|
|
175
|
-
try { await writeFile(temp, bytes); await rename(temp, target) }
|
|
176
|
-
finally { await rm(temp, { force: true }) }
|
|
177
|
-
},
|
|
178
|
-
async open(id, name) {
|
|
179
|
-
const file = await open(join(directory(id), name), 'r')
|
|
180
|
-
try { return { size: (await file.stat()).size, stream: file.createReadStream() } }
|
|
181
|
-
catch (err) { await file.close(); throw err }
|
|
182
|
-
},
|
|
183
|
-
async delete(id) { await rm(directory(id), { recursive: true, force: true }) },
|
|
184
|
-
}, db, store)
|
|
185
|
-
}
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
// Sourcemaps are stored once as Brotli. The DB retains the uploaded filename,
|
|
2
2
|
// byte size and integrity so compression does not change report links/dedup.
|
|
3
3
|
import type { Buffer } from 'node:buffer'
|
|
4
|
-
import {
|
|
4
|
+
import type { BlobStore, OpenedBlob } from './blob-store.ts'
|
|
5
5
|
import { encodeBrotli } from './brotli.ts'
|
|
6
6
|
|
|
7
7
|
export interface BundleStore {
|
|
8
|
-
put(id: string, bytes: Buffer, kind: string | null): Promise<
|
|
8
|
+
put(id: string, bytes: Buffer, kind: string | null): Promise<string | null>
|
|
9
9
|
get(id: string, kind: string | null): Promise<Buffer | null>
|
|
10
10
|
open(id: string, kind: string | null): Promise<OpenedBlob | null>
|
|
11
11
|
delete(id: string): Promise<void>
|
|
@@ -15,14 +15,10 @@ export function createBundleStore(archives: BlobStore, sourcemaps: BlobStore): B
|
|
|
15
15
|
const storage = (kind: string | null) => kind === 'sourcemap' ? sourcemaps : archives
|
|
16
16
|
return {
|
|
17
17
|
async put(id, bytes, kind) {
|
|
18
|
-
|
|
18
|
+
return storage(kind).put(id, kind === 'sourcemap' ? await encodeBrotli(bytes) : bytes)
|
|
19
19
|
},
|
|
20
20
|
get: (id, kind) => storage(kind).get(id),
|
|
21
21
|
open: (id, kind) => storage(kind).open(id),
|
|
22
22
|
async delete(id) { await Promise.all([archives.delete(id), sourcemaps.delete(id)]) },
|
|
23
23
|
}
|
|
24
24
|
}
|
|
25
|
-
|
|
26
|
-
export function createDiskBundleStore(dir: string): BundleStore {
|
|
27
|
-
return createBundleStore(createDiskBlobStore(dir), createDiskBlobStore(dir, '.map.br'))
|
|
28
|
-
}
|
|
@@ -1,7 +1,4 @@
|
|
|
1
1
|
import type { Buffer } from 'node:buffer'
|
|
2
|
-
import { randomUUID } from 'node:crypto'
|
|
3
|
-
import { mkdir, open, rename, rm, stat, writeFile } from 'node:fs/promises'
|
|
4
|
-
import { dirname, join } from 'node:path'
|
|
5
2
|
import type { OpenedBlob } from './blob-store.ts'
|
|
6
3
|
|
|
7
4
|
// Keys are internal relative paths; delete removes a whole directory of variants.
|
|
@@ -16,25 +13,3 @@ export function validateCacheKey(key: string): string {
|
|
|
16
13
|
if (!key.split('/').every(part => /^[a-z0-9][a-z0-9.-]*$/iu.test(part))) throw new Error('Invalid cache key')
|
|
17
14
|
return key
|
|
18
15
|
}
|
|
19
|
-
|
|
20
|
-
export function createDiskCacheStorage(dir: string): CacheStorage {
|
|
21
|
-
const path = (key: string) => join(dir, validateCacheKey(key))
|
|
22
|
-
return {
|
|
23
|
-
async exists(key) {
|
|
24
|
-
try { await stat(path(key)); return true }
|
|
25
|
-
catch (err) { if ((err as NodeJS.ErrnoException).code === 'ENOENT') return false; throw err }
|
|
26
|
-
},
|
|
27
|
-
async put(key, bytes) {
|
|
28
|
-
const target = path(key), temp = `${target}.${randomUUID()}.tmp`
|
|
29
|
-
await mkdir(dirname(target), { recursive: true })
|
|
30
|
-
try { await writeFile(temp, bytes); await rename(temp, target) }
|
|
31
|
-
finally { await rm(temp, { force: true }) }
|
|
32
|
-
},
|
|
33
|
-
async open(key) {
|
|
34
|
-
const file = await open(path(key), 'r')
|
|
35
|
-
try { return { size: (await file.stat()).size, stream: file.createReadStream() } }
|
|
36
|
-
catch (err) { await file.close(); throw err }
|
|
37
|
-
},
|
|
38
|
-
async delete(prefix) { await rm(path(prefix), { recursive: true, force: true }) },
|
|
39
|
-
}
|
|
40
|
-
}
|
package/server-managed/cli.js
CHANGED
|
@@ -9,5 +9,13 @@
|
|
|
9
9
|
// Imported here rather than run as the process entry, so index.ts's
|
|
10
10
|
// `import.meta.main` auto-start gate stays off and we call `start()` ourselves.
|
|
11
11
|
import '../strip-types-loader.js'
|
|
12
|
-
const
|
|
13
|
-
|
|
12
|
+
const command = process.argv[2]
|
|
13
|
+
if (command === '--storage-encryption-status') {
|
|
14
|
+
const { storageCommand } = await import('./storage-command.ts')
|
|
15
|
+
await storageCommand(process.argv[3])
|
|
16
|
+
} else if (command) {
|
|
17
|
+
throw new Error(`Unknown command: ${command}`)
|
|
18
|
+
} else {
|
|
19
|
+
const { start } = await import('./index.ts')
|
|
20
|
+
await start()
|
|
21
|
+
}
|
package/server-managed/config.ts
CHANGED
|
@@ -3,6 +3,8 @@
|
|
|
3
3
|
// GitHub App can provide both flows, with repository grants at installation.
|
|
4
4
|
import { env } from 'node:process'
|
|
5
5
|
import { databaseUrls } from '../server-common/database-config.ts'
|
|
6
|
+
import { MAX_UPLOAD_BYTES } from './uploads.ts'
|
|
7
|
+
import { parseStorageKey } from '../server-common/storage-crypto.ts'
|
|
6
8
|
|
|
7
9
|
const LOOPBACK_HOSTS = new Set(['127.0.0.1', '::1', 'localhost'])
|
|
8
10
|
|
|
@@ -12,6 +14,10 @@ export interface ManagedConfig {
|
|
|
12
14
|
dbPath: string
|
|
13
15
|
neonUrl?: string | null
|
|
14
16
|
blobToken?: string | null
|
|
17
|
+
storageEncryptionKey?: string | null
|
|
18
|
+
storageEncryptionMigrate?: boolean
|
|
19
|
+
storageEncryptionMigrateMaxMs?: number
|
|
20
|
+
vercelPreview?: boolean
|
|
15
21
|
serverless?: boolean
|
|
16
22
|
debug: boolean
|
|
17
23
|
allowShare: boolean
|
|
@@ -87,6 +93,10 @@ export function loadManagedConfig({ combined = false } = {}): ManagedConfig {
|
|
|
87
93
|
const serverless = env['VERCEL'] === '1'
|
|
88
94
|
const neonUrl = databaseUrls({ combined }).managed
|
|
89
95
|
const blobToken = env['BLOB_READ_WRITE_TOKEN'] || null
|
|
96
|
+
const storageEncryptionKey = env['MANAGED_STORAGE_ENCRYPTION_KEY'] || null
|
|
97
|
+
parseStorageKey(storageEncryptionKey)
|
|
98
|
+
const storageEncryptionMigrate = env['MANAGED_STORAGE_ENCRYPTION_MIGRATE'] === '1'
|
|
99
|
+
if (storageEncryptionMigrate && !storageEncryptionKey) fail('MANAGED_STORAGE_ENCRYPTION_MIGRATE requires MANAGED_STORAGE_ENCRYPTION_KEY.')
|
|
90
100
|
if (serverless && !neonUrl) fail('Vercel managed mode requires DATABASE_URL or MANAGED_DATABASE_URL.')
|
|
91
101
|
if (neonUrl && !blobToken) fail('Managed Neon mode requires BLOB_READ_WRITE_TOKEN.')
|
|
92
102
|
const host = env['HOST'] ?? '127.0.0.1'
|
|
@@ -104,7 +114,9 @@ export function loadManagedConfig({ combined = false } = {}): ManagedConfig {
|
|
|
104
114
|
fail(`SESSION_COOKIE_NAME=${sessionCookieName} uses the __Host- prefix but the callback is not https. Use a non-prefixed name for loopback http dev.`)
|
|
105
115
|
}
|
|
106
116
|
return {
|
|
107
|
-
neonUrl, blobToken, serverless,
|
|
117
|
+
neonUrl, blobToken, serverless, storageEncryptionKey, storageEncryptionMigrate,
|
|
118
|
+
storageEncryptionMigrateMaxMs: intEnv('MANAGED_STORAGE_ENCRYPTION_MIGRATE_MAX_MS', 150_000, 1, serverless ? 240_000 : 3_600_000),
|
|
119
|
+
vercelPreview: env['VERCEL_ENV'] === 'preview',
|
|
108
120
|
port: intEnv('PORT', 8765, 0, 65535),
|
|
109
121
|
host,
|
|
110
122
|
// DB_PATH belongs to e2e in a combined process. Keep the two stores apart.
|
|
@@ -124,7 +136,7 @@ export function loadManagedConfig({ combined = false } = {}): ManagedConfig {
|
|
|
124
136
|
githubAppSlug: env['GITHUB_APP_SLUG'] ?? null,
|
|
125
137
|
githubNewIssueLabels: env['GITHUB_NEW_ISSUE_LABELS'] ?? '',
|
|
126
138
|
maxReportBytes: intEnv('MAX_REPORT_BYTES', 10_485_760, 1, 104_857_600),
|
|
127
|
-
maxBundleBytes: intEnv('MAX_BUNDLE_BYTES', 209_715_200, 1,
|
|
139
|
+
maxBundleBytes: intEnv('MAX_BUNDLE_BYTES', 209_715_200, 1, MAX_UPLOAD_BYTES),
|
|
128
140
|
triageHistoryLimit: intEnv('TRIAGE_HISTORY_LIMIT', 0, 0, 1_000_000_000),
|
|
129
141
|
}
|
|
130
142
|
}
|