@preventive/triage 1.0.0-alpha.26 → 1.0.0-alpha.27

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. package/out/client-managed-import.js +18 -18
  2. package/out/client-managed.js +47 -47
  3. package/out/view.css +1 -1
  4. package/out/view.js +159 -159
  5. package/package.json +12 -2
  6. package/server-common/storage-crypto.ts +182 -0
  7. package/server-common/storage-log.ts +5 -1
  8. package/server-common/vercel-blob.ts +6 -3
  9. package/server-managed/avatar-store.ts +1 -40
  10. package/server-managed/blob-store.ts +3 -52
  11. package/server-managed/bundle-cache.ts +0 -28
  12. package/server-managed/bundle-store.ts +3 -7
  13. package/server-managed/cache-storage.ts +0 -25
  14. package/server-managed/cli.js +10 -2
  15. package/server-managed/config.ts +12 -1
  16. package/server-managed/db-methods.ts +42 -21
  17. package/server-managed/db-neon.ts +9 -4
  18. package/server-managed/db.ts +7 -1
  19. package/server-managed/http.ts +37 -39
  20. package/server-managed/index.ts +3 -2
  21. package/server-managed/object-storage-disk.ts +144 -0
  22. package/server-managed/object-storage-vercel.ts +60 -0
  23. package/server-managed/object-storage.ts +71 -0
  24. package/server-managed/repository-browser.ts +5 -1
  25. package/server-managed/solidity-entry-points.ts +69 -0
  26. package/server-managed/sql.ts +0 -5
  27. package/server-managed/storage-command.ts +16 -0
  28. package/server-managed/storage-db.ts +168 -0
  29. package/server-managed/storage-encryption.ts +211 -0
  30. package/server-managed/storage-maintenance.ts +182 -0
  31. package/server-managed/storage-payload.ts +60 -0
  32. package/server-managed/storage-stores.ts +81 -0
  33. package/server-managed/storage.ts +46 -24
  34. package/server-managed/uploads.ts +2 -2
  35. package/server-managed/blob-vercel.ts +0 -127
@@ -0,0 +1,71 @@
1
+ import type { Buffer } from 'node:buffer'
2
+ import type { Readable } from 'node:stream'
3
+ import type { OpenedBlob } from './blob-store.ts'
4
+ import { validateCacheKey } from './cache-storage.ts'
5
+
6
+ export const ENCRYPTED_CACHE_PREFIX = 'cache-encrypted-v1/'
7
+ const STORAGE_PREFIXES = ['reports/', 'bundles/', 'avatars/', 'uploads/', 'cache/', ENCRYPTED_CACHE_PREFIX]
8
+ export interface RawObject extends OpenedBlob { version: string; modifiedAt: number }
9
+ type ObjectMetadata = Omit<RawObject, 'stream'>
10
+ interface ListedObject { key: string; modifiedAt: number }
11
+ export interface ObjectPage { objects: ListedObject[]; cursor: string | null }
12
+ export interface RawObjectStorage {
13
+ open(key: string, signal?: AbortSignal): Promise<RawObject | null>
14
+ head(key: string, signal?: AbortSignal): Promise<ObjectMetadata | null>
15
+ prune?(prefix: string, signal?: AbortSignal): Promise<void>
16
+ // Persist an observed disk replacement before its SQL migration checkpoint.
17
+ sync?(key: string, signal?: AbortSignal): Promise<void>
18
+ // A plaintext size hint selects multipart uploads without buffering streams.
19
+ put(key: string, bytes: Buffer | Readable, signal?: AbortSignal, expected?: string, sizeHint?: number): Promise<boolean>
20
+ // Return false if the original version changed; do not delete its replacement.
21
+ delete(key: string, version?: string, signal?: AbortSignal): Promise<boolean>
22
+ list(prefix: string, cursor: string | null, limit: number, signal?: AbortSignal): Promise<ObjectPage>
23
+ }
24
+ export interface ObjectStorage {
25
+ open(key: string): Promise<OpenedBlob | null>
26
+ get(key: string): Promise<Buffer | null>
27
+ exists(key: string): Promise<boolean>
28
+ put(key: string, bytes: Buffer): Promise<string | null>
29
+ delete(key: string): Promise<void>
30
+ deletePrefix(prefix: string): Promise<void>
31
+ }
32
+
33
+ export function isBlobId(value: string): boolean {
34
+ return /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/u.test(value)
35
+ }
36
+
37
+ export function objectPath(key: string): string {
38
+ validateCacheKey(key)
39
+ if (!STORAGE_PREFIXES.some(prefix => key.startsWith(prefix))) throw new Error('Invalid managed storage namespace')
40
+ return key
41
+ }
42
+
43
+ // Hash verification and migration may consume a stream before it can be used.
44
+ // Reopen only the same object version, closing any concurrent replacement.
45
+ export async function openObjectVersion(raw: RawObjectStorage, key: string, version: string, signal?: AbortSignal): Promise<RawObject | null> {
46
+ const current = await raw.open(key, signal)
47
+ if (current?.version === version) return current
48
+ current?.stream.destroy()
49
+ return null
50
+ }
51
+
52
+ export async function deleteObjects(raw: RawObjectStorage, prefix: string, signal: AbortSignal, before?: number): Promise<number> {
53
+ const eligible = (modifiedAt: number) => before === undefined || modifiedAt < before
54
+ // Finish listing before deleting: provider cursors may be offset-based.
55
+ const cursors = new Set<string>(), keys = new Set<string>()
56
+ let cursor: string | null = null
57
+ do {
58
+ const page = await raw.list(prefix, cursor, 100, signal)
59
+ for (const object of page.objects) if (eligible(object.modifiedAt)) keys.add(object.key)
60
+ cursor = page.cursor
61
+ if (cursor !== null && cursors.has(cursor)) throw new Error('Invalid blob pagination')
62
+ if (cursor !== null) cursors.add(cursor)
63
+ } while (cursor !== null)
64
+ let removed = 0
65
+ for (const key of keys) {
66
+ const stored = await raw.head(key, signal)
67
+ // Ignore stale listings and preserve concurrent replacements.
68
+ if (stored && eligible(stored.modifiedAt) && await raw.delete(key, stored.version, signal)) removed++
69
+ }
70
+ return removed
71
+ }
@@ -2,6 +2,7 @@ import type { ManagedConfig } from './config.ts'
2
2
  import type { SelectedRepo } from './db.ts'
3
3
  import { GithubApiError, githubJson, githubRepoReadPermission, githubUserIdentity, repoAccessToken } from './github-app.ts'
4
4
  import { MAX_PACKAGE_BYTES, readPackageEntryPoints } from './package-entry-points.ts'
5
+ import { readSolidityEntryPoints } from './solidity-entry-points.ts'
5
6
 
6
7
  export interface RepositoryEntry { name: string; path: string; type: 'dir' | 'file' | 'symlink' | 'submodule' }
7
8
 
@@ -126,7 +127,10 @@ async function repositoryReader(repo: SelectedRepo, userToken: string | null, in
126
127
  const manifest = data.find(item => item?.path === manifestPath && item.name === 'package.json' && item.type === 'file' && !item.submodule_git_url
127
128
  && typeof item.size === 'number' && item.size >= 0 && item.size <= MAX_PACKAGE_BYTES && typeof item.sha === 'string' && /^[a-f\d]{40}$/iu.test(item.sha))
128
129
  const packageEntryPoints = manifest ? await readPackageEntryPoints(manifest.sha, path, read) : undefined
129
- return { entries, limited: data.length >= 1000, ...(packageEntryPoints ? { packageEntryPoints } : {}) }
130
+ const solidity = await readSolidityEntryPoints(path, data, read)
131
+ return { entries, limited: data.length >= 1000, ...(packageEntryPoints ? { packageEntryPoints } : {}),
132
+ ...(solidity.paths.length > 0 || solidity.limited ? { solidityEntryPoints: solidity.paths, soliditySuggestionsLimited: solidity.limited } : {}),
133
+ }
130
134
  },
131
135
  }
132
136
  }
@@ -0,0 +1,69 @@
1
+ import { normalizeTeamPath } from './repo-path.ts'
2
+
3
+ const MAX_SUGGESTIONS = 100
4
+ const EXCLUDED_DIRECTORIES = new Set(['test', 'tests', 'script', 'scripts', 'mock', 'mocks', 'node_modules', 'vendor', 'out', 'artifacts', 'cache', 'build', 'dist', 'broadcast'])
5
+
6
+ function sourcePath(value: unknown): value is string {
7
+ if (typeof value !== 'string' || !value.endsWith('.sol') || /\.(?:t|s|test|spec)\.sol$/iu.test(value)) return false
8
+ const normalized = normalizeTeamPath(value)
9
+ const directories = value.split('/').slice(0, -1).map(part => part.toLowerCase())
10
+ const sourceRoot = directories.findIndex(part => part === 'src' || part === 'contracts')
11
+ return normalized.ok && normalized.path === value && !directories.some((part, index) =>
12
+ EXCLUDED_DIRECTORIES.has(part) || (part === 'lib' && (sourceRoot < 0 || index < sourceRoot)))
13
+ }
14
+
15
+ type ListingEntry = { name?: unknown; path?: unknown; type?: unknown; sha?: unknown; submodule_git_url?: unknown }
16
+
17
+ // Maintain the alphabetically first 100 unique candidates even when GitHub's
18
+ // input is unordered. Return whether a unique candidate exceeds the capacity.
19
+ export function addSoliditySuggestion(paths: string[], path: string): boolean {
20
+ let start = 0
21
+ let end = paths.length
22
+ while (start < end) {
23
+ const middle = Math.floor((start + end) / 2)
24
+ if (paths[middle]! < path) start = middle + 1
25
+ else end = middle
26
+ }
27
+ if (paths[start] === path) return false
28
+ const full = paths.length === MAX_SUGGESTIONS
29
+ if (start < MAX_SUGGESTIONS) {
30
+ // Evict before inserting, so retained candidates never exceed the cap.
31
+ if (full) paths.pop()
32
+ paths.splice(start, 0, path)
33
+ }
34
+ return full
35
+ }
36
+
37
+ // Suggestions from filenames, not deployability or import analysis. Read at
38
+ // most two source trees, using SHAs from the authorized pinned directory.
39
+ export async function readSolidityEntryPoints(directory: string, listing: unknown[], read: (suffix: string) => Promise<unknown>) {
40
+ const prefix = directory ? directory + '/' : ''
41
+ const entries = listing.filter((entry): entry is ListingEntry => !!entry && typeof entry === 'object').filter(entry =>
42
+ typeof entry.name === 'string' && !entry.name.includes('/') && entry.path === prefix + entry.name && !entry.submodule_git_url)
43
+ const hasFramework = entries.some(entry => entry.type === 'file' && typeof entry.name === 'string'
44
+ && (entry.name === 'foundry.toml' || /^hardhat\.config\.(?:[cm]?js|ts)$/u.test(entry.name)))
45
+ const roots = entries.filter(entry => entry.type === 'dir' && (entry.name === 'contracts' || (hasFramework && entry.name === 'src'))
46
+ && typeof entry.sha === 'string' && /^[a-f\d]{40}$/iu.test(entry.sha)).slice(0, 2)
47
+ const paths: string[] = []
48
+ let limited = false
49
+ for (const entry of entries) {
50
+ if (entry.type === 'file' && sourcePath(entry.path) && addSoliditySuggestion(paths, entry.path)) limited = true
51
+ }
52
+ for (const root of roots) {
53
+ if (!sourcePath(`${root.path}/Contract.sol`)) continue
54
+ try {
55
+ const data = await read(`/git/trees/${root.sha}?recursive=1`) as { tree?: unknown; truncated?: unknown }
56
+ if (!Array.isArray(data?.tree)) continue
57
+ limited ||= data.truncated === true
58
+ for (const entry of data.tree) {
59
+ if (entry?.type !== 'blob' || !['100644', '100755'].includes(entry.mode) || typeof entry.path !== 'string') continue
60
+ const path = `${root.path}/${entry.path}`
61
+ if (sourcePath(path) && addSoliditySuggestion(paths, path)) limited = true
62
+ }
63
+ } catch {
64
+ // Optional suggestions must not block browsing. The request handler
65
+ // rechecks GitHub access and local path grants after all reads.
66
+ }
67
+ }
68
+ return { paths, limited }
69
+ }
@@ -4,11 +4,6 @@ import type { DatabaseSync, SQLInputValue } from 'node:sqlite'
4
4
  import type { ManagedDb } from './db-methods.ts'
5
5
 
6
6
  type MaybePromise<T> = T | Promise<T>
7
- // A failed COMMIT acknowledgement (or connection close after COMMIT) does not
8
- // establish rollback. Callers must retain primary bytes until reconciled.
9
- export class ManagedCommitError extends Error {
10
- constructor(cause: unknown) { super('Managed commit outcome is uncertain', { cause }) }
11
- }
12
7
  export interface ManagedSql {
13
8
  prepare(sql: string): {
14
9
  get(...params: unknown[]): MaybePromise<unknown>
@@ -0,0 +1,16 @@
1
+ import { loadManagedConfig } from './config.ts'
2
+ import { openManagedStorageDb } from './storage.ts'
3
+
4
+ // Require the deployment mode: DB_PATH belongs to e2e in combined deployments.
5
+ // Read the database requirement without opening bytes or enabling encryption.
6
+ export async function storageCommand(mode: string | undefined): Promise<void> {
7
+ if (mode !== 'managed' && mode !== 'managed-e2e' && mode !== 'e2e-managed') throw new Error('Storage status requires a deployment mode: --storage-encryption-status managed|managed-e2e|e2e-managed')
8
+ const db = await openManagedStorageDb(loadManagedConfig({ combined: mode !== 'managed' }))
9
+ try {
10
+ const status = await db.getStorageEncryption()
11
+ console.log(JSON.stringify(status ? {
12
+ encryption: 'chacha20-poly1305', complete: status.complete === 1, cleanupComplete: status.cleanupComplete === 1,
13
+ migrated: status.migrated, cursor: status.cursor,
14
+ } : { encryption: 'disabled' }))
15
+ } finally { await db.close() }
16
+ }
@@ -0,0 +1,168 @@
1
+ // Keys belong to the existing upload rows. Only installation state and bounded
2
+ // migration progress need a separate table; payload paths never move.
3
+ import { Buffer } from 'node:buffer'
4
+ import { randomBytes } from 'node:crypto'
5
+ import { type StorageKey, unwrapStorageValue, wrapStorageValue } from '../server-common/storage-crypto.ts'
6
+ import type { ManagedSql } from './sql.ts'
7
+
8
+ export const STORAGE_SCHEMA = `
9
+ CREATE TABLE IF NOT EXISTS managed_storage_encryption (
10
+ id INTEGER PRIMARY KEY CHECK (id = 1), test_value TEXT NOT NULL,
11
+ enabled_at INTEGER NOT NULL, cursor TEXT, complete INTEGER NOT NULL DEFAULT 0,
12
+ migrated INTEGER NOT NULL DEFAULT 0, cleanup_cursor TEXT,
13
+ cleanup_found INTEGER NOT NULL DEFAULT 0, cleanup_complete INTEGER NOT NULL DEFAULT 0
14
+ ) STRICT;
15
+ `
16
+ const SENTINEL = Buffer.from('deepview.storage.required.v1')
17
+ export type StorageRowKind = 'report' | 'bundle'
18
+ export interface StorageEncryptionState {
19
+ enabledAt: number; cursor: string | null; complete: number; migrated: number
20
+ cleanupCursor: string | null; cleanupFound: number; cleanupComplete: number
21
+ }
22
+ export interface StorageRow {
23
+ id: string; dataKey: string | null; encrypted: number; hash: string; kind: string | null
24
+ }
25
+ export interface StorageMigrationRow { position: string; id: string; type: StorageRowKind | 'user' }
26
+ export interface StorageDb {
27
+ enableStorageEncryption(): Promise<StorageEncryptionState>
28
+ getStorageEncryption(): Promise<StorageEncryptionState | null>
29
+ getStorageRow(type: StorageRowKind, id: string): Promise<StorageRow | null>
30
+ ensureStorageDataKey(type: StorageRowKind, id: string): Promise<StorageRow | null>
31
+ markStorageEncrypted(type: StorageRowKind, id: string, dataKey: string): Promise<void>
32
+ listStorageMigrationRows(after: string | null, limit: number): Promise<StorageMigrationRow[]>
33
+ advanceStorageMigration(expected: string | null, next: string | null): Promise<void>
34
+ advanceStorageCleanup(expected: string | null, next: string | null, removed: number): Promise<void>
35
+ migrateStorageUserTokens(id: string): Promise<void>
36
+ }
37
+
38
+ function table(type: StorageRowKind): string {
39
+ if (type !== 'report' && type !== 'bundle') throw new Error('Invalid storage row type')
40
+ return `managed_${type}`
41
+ }
42
+ export const dataKeyIdentity = (type: StorageRowKind, id: string): string => `${table(type)}:${id}`
43
+ export function unwrapDataKey(key: StorageKey, type: StorageRowKind, row: Pick<StorageRow, 'id' | 'dataKey'>): Buffer {
44
+ if (!row.dataKey) throw new Error('Missing storage data key')
45
+ const value = unwrapStorageValue(key, dataKeyIdentity(type, row.id), row.dataKey)
46
+ if (value.length !== 32) { value.fill(0); throw new Error('Invalid storage data key') }
47
+ return value
48
+ }
49
+
50
+ export async function storageState(db: ManagedSql, key: StorageKey | null): Promise<StorageEncryptionState | null> {
51
+ const row = await db.prepare(`SELECT test_value AS testValue, enabled_at AS enabledAt, cursor, complete, migrated,
52
+ cleanup_cursor AS cleanupCursor, cleanup_found AS cleanupFound, cleanup_complete AS cleanupComplete
53
+ FROM managed_storage_encryption WHERE id = 1`).get() as (StorageEncryptionState & { testValue: string }) | undefined
54
+ if (!row) return null
55
+ if (!key) throw new Error('Managed storage requires its configured encryption key')
56
+ const plain = unwrapStorageValue(key, 'managed_storage_encryption:1', row.testValue)
57
+ try { if (!plain.equals(SENTINEL)) throw new Error('Invalid storage encryption test value') }
58
+ finally { plain.fill(0) }
59
+ const { testValue: _testValue, ...state } = row
60
+ return state
61
+ }
62
+
63
+ // Called inside the insert transaction, fencing an old/plain upload that raced
64
+ // activation. The wrapped key never enters public report/bundle DTOs.
65
+ export async function validateStorageUpload(db: ManagedSql, key: StorageKey | null, type: StorageRowKind,
66
+ id: string, dataKey: string | null | undefined): Promise<void> {
67
+ const state = await storageState(db, key)
68
+ if (state) unwrapDataKey(key!, type, { id, dataKey: dataKey ?? null }).fill(0)
69
+ else if (dataKey) throw new Error('Storage encryption is not enabled')
70
+ }
71
+
72
+ export interface StoredTokens { access: string | null; refresh: string | null; exp: number | null; encrypted: number }
73
+ const tokenIdentity = (id: string, field: string) => `managed_user.gh_${field}_token:${id}`
74
+ export function encodeStorageTokens(key: StorageKey, id: string, access: string | null, refresh: string | null) {
75
+ const encode = (field: string, value: string | null) => value === null ? null : wrapStorageValue(key, tokenIdentity(id, field), Buffer.from(value))
76
+ return { access: encode('access', access), refresh: encode('refresh', refresh) }
77
+ }
78
+ export function decodeStorageTokens(key: StorageKey, id: string, row: StoredTokens) {
79
+ const decode = (field: string, value: string | null) => {
80
+ if (value === null) return null
81
+ const plain = unwrapStorageValue(key, tokenIdentity(id, field), value)
82
+ try { return plain.toString('utf8') } finally { plain.fill(0) }
83
+ }
84
+ return { access: decode('access', row.access), refresh: decode('refresh', row.refresh) }
85
+ }
86
+
87
+ const PENDING = `SELECT 'bundle:' || id AS position, id, 'bundle' AS type FROM managed_bundle WHERE storage_encrypted = 0
88
+ UNION ALL SELECT 'report:' || id AS position, id, 'report' AS type FROM managed_report WHERE storage_encrypted = 0
89
+ UNION ALL SELECT 'user:' || id AS position, id, 'user' AS type FROM managed_user
90
+ WHERE gh_tokens_encrypted = 0 AND (gh_access_token IS NOT NULL OR gh_refresh_token IS NOT NULL)`
91
+
92
+ export function storageMethods(db: ManagedSql, key: StorageKey | null): StorageDb {
93
+ async function requireEnabled() {
94
+ const state = await storageState(db, key)
95
+ if (!state) throw new Error('Storage encryption is not enabled; configure MANAGED_STORAGE_ENCRYPTION_KEY')
96
+ return state
97
+ }
98
+ async function getRow(type: StorageRowKind, id: string): Promise<StorageRow | null> {
99
+ const row = await db.prepare(`SELECT id, data_key AS dataKey, storage_encrypted AS encrypted,
100
+ ${type === 'report' ? 'sha256 AS hash, NULL AS kind' : 'integrity AS hash, kind'} FROM ${table(type)} WHERE id = ?`).get(id) as StorageRow | undefined ?? null
101
+ if (row && row.encrypted !== 0 && row.encrypted !== 1) throw new Error('Invalid storage encryption state')
102
+ if (row?.encrypted && !row.dataKey) throw new Error('Encrypted storage row is missing its data key')
103
+ return row
104
+ }
105
+ return {
106
+ async enableStorageEncryption() {
107
+ if (!key) throw new Error('Set MANAGED_STORAGE_ENCRYPTION_KEY before enabling storage encryption')
108
+ // Called during startup in a writer transaction. Concurrent instances
109
+ // must reuse the marker and validate its key before serving requests.
110
+ const existing = await storageState(db, key)
111
+ if (existing) return existing
112
+ await db.prepare('INSERT INTO managed_storage_encryption (id, test_value, enabled_at) VALUES (1, ?, ?)')
113
+ .run(wrapStorageValue(key, 'managed_storage_encryption:1', SENTINEL), Date.now())
114
+ return (await storageState(db, key))!
115
+ },
116
+ getStorageEncryption: () => storageState(db, key),
117
+ getStorageRow: getRow,
118
+ async ensureStorageDataKey(type, id) {
119
+ await requireEnabled()
120
+ const current = await getRow(type, id)
121
+ if (!current || current.dataKey) return current
122
+ const bytes = randomBytes(32)
123
+ let wrapped: string
124
+ try { wrapped = wrapStorageValue(key!, dataKeyIdentity(type, id), bytes) } finally { bytes.fill(0) }
125
+ await db.prepare(`UPDATE ${table(type)} SET data_key = ? WHERE id = ? AND data_key IS NULL`).run(wrapped, id)
126
+ // Return the persisted winner, never a worker's uncommitted candidate.
127
+ return getRow(type, id)
128
+ },
129
+ async markStorageEncrypted(type, id, wrapped) {
130
+ await requireEnabled()
131
+ const result = await db.prepare(`UPDATE ${table(type)} SET storage_encrypted = 1
132
+ WHERE id = ? AND data_key = ? AND storage_encrypted = 0`).run(id, wrapped)
133
+ if (Number(result.changes)) await db.prepare('UPDATE managed_storage_encryption SET migrated = migrated + 1 WHERE id = 1').run()
134
+ },
135
+ async listStorageMigrationRows(after, limit) {
136
+ await requireEnabled()
137
+ return await db.prepare(`SELECT position, id, type FROM (${PENDING}) AS pending
138
+ WHERE position > ? ORDER BY position LIMIT ?`).all(after ?? '', limit) as StorageMigrationRow[]
139
+ },
140
+ async advanceStorageMigration(expected, next) {
141
+ await requireEnabled()
142
+ if (expected !== null && next !== null && next <= expected) return
143
+ // End-of-pass completion checks every pending row, including failed rows
144
+ // before the cursor. A failure cannot be mistaken for a finished pass.
145
+ await db.prepare(`UPDATE managed_storage_encryption SET cursor = ?, complete =
146
+ CASE WHEN ? IS NULL AND NOT EXISTS (SELECT 1 FROM (${PENDING}) AS pending) THEN 1 ELSE 0 END
147
+ WHERE id = 1 AND (cursor = ? OR (cursor IS NULL AND ? IS NULL))`).run(next, next, expected, expected)
148
+ },
149
+ async advanceStorageCleanup(expected, next, removed) {
150
+ const state = await requireEnabled()
151
+ const found = state.cleanupFound + removed
152
+ await db.prepare(`UPDATE managed_storage_encryption SET cleanup_cursor = ?, cleanup_found = ?, cleanup_complete = ?
153
+ WHERE id = 1 AND (cleanup_cursor = ? OR (cleanup_cursor IS NULL AND ? IS NULL))`).run(next, next === null ? 0 : found, next === null && found === 0 ? 1 : 0, expected, expected)
154
+ },
155
+ async migrateStorageUserTokens(id) {
156
+ await requireEnabled()
157
+ const row = await db.prepare(`SELECT gh_access_token AS access, gh_refresh_token AS refresh, gh_token_expires_at AS exp,
158
+ gh_tokens_encrypted AS encrypted FROM managed_user WHERE id = ?`).get(id) as StoredTokens | undefined
159
+ if (!row || row.encrypted || (row.access === null && row.refresh === null)) return
160
+ const value = encodeStorageTokens(key!, id, row.access, row.refresh)
161
+ // This operation holds the writer transaction; token refresh cannot be
162
+ // overwritten by a migration that read an earlier token value.
163
+ await db.prepare('UPDATE managed_user SET gh_access_token = ?, gh_refresh_token = ?, gh_tokens_encrypted = 1 WHERE id = ?')
164
+ .run(value.access, value.refresh, id)
165
+ await db.prepare('UPDATE managed_storage_encryption SET migrated = migrated + 1 WHERE id = 1').run()
166
+ },
167
+ }
168
+ }
@@ -0,0 +1,211 @@
1
+ import { Buffer } from 'node:buffer'
2
+ import { randomBytes } from 'node:crypto'
3
+ import { Readable } from 'node:stream'
4
+ import { type StorageKey, decryptStorageStream, encryptStorageStream, wrapStorageValue } from '../server-common/storage-crypto.ts'
5
+ import { ENCRYPTED_CACHE_PREFIX, type ObjectStorage, type RawObject, type RawObjectStorage, deleteObjects, objectPath, openObjectVersion } from './object-storage.ts'
6
+ import { type StorageDb, type StorageEncryptionState, type StorageRow, type StorageRowKind, dataKeyIdentity, unwrapDataKey } from './storage-db.ts'
7
+ import { inspectStorageObject, storageOwner, storageRowPath, verifyStoragePayload } from './storage-payload.ts'
8
+
9
+ const STORAGE_WRITE_MS = 180_000
10
+ export const STORAGE_UPLOAD_TTL_MS = 86_400_000
11
+ export const STORAGE_DISABLED_TTL_MS = 5_000
12
+
13
+ async function readBytes(stored: Awaited<ReturnType<ObjectStorage['open']>>): Promise<Buffer | null> {
14
+ if (!stored) return null
15
+ const parts: Buffer[] = []
16
+ try { for await (const part of stored.stream) parts.push(Buffer.from(part)); return Buffer.concat(parts) }
17
+ finally { stored.stream.destroy() }
18
+ }
19
+ function logicalKey(identity: string) {
20
+ objectPath(identity)
21
+ if (identity.startsWith(ENCRYPTED_CACHE_PREFIX)) throw new Error('Encrypted cache paths are internal')
22
+ }
23
+ const encryptedCachePath = (identity: string) => `${ENCRYPTED_CACHE_PREFIX}${identity.slice('cache/'.length)}`
24
+ const deletionPaths = (identity: string) => identity.startsWith('cache/') ? [identity, encryptedCachePath(identity)] : [identity]
25
+
26
+ // A PUT can commit and still throw. Remove only this attempted plaintext,
27
+ // comparing bytes rather than magic: arbitrary uploads can share the header.
28
+ // A concurrent migration's ciphertext must survive this reconciliation.
29
+ async function discardPlaintextWrite(raw: RawObjectStorage, identity: string, bytes: Buffer): Promise<void> {
30
+ // Leave time to report the original failure inside a serverless invocation.
31
+ const signal = AbortSignal.timeout(30_000)
32
+ const current = await raw.open(identity, signal)
33
+ if (!current) return
34
+ try {
35
+ let offset = 0
36
+ for await (const part of current.stream) {
37
+ const chunk = Buffer.from(part)
38
+ if (!chunk.equals(bytes.subarray(offset, offset + chunk.length))) return
39
+ offset += chunk.length
40
+ }
41
+ if (offset === bytes.length) await raw.delete(identity, current.version, signal)
42
+ } finally { current.stream.destroy() }
43
+ }
44
+
45
+ async function putPlaintext(raw: RawObjectStorage, identity: string, bytes: Buffer, mode: () => Promise<StorageEncryptionState | null>): Promise<null> {
46
+ try {
47
+ await raw.put(identity, bytes, AbortSignal.timeout(STORAGE_WRITE_MS))
48
+ if (!await mode()) return null
49
+ throw new Error('Storage encryption was enabled during upload; retry the upload')
50
+ } catch (err) {
51
+ try { await discardPlaintextWrite(raw, identity, bytes) }
52
+ catch (cleanup) { throw new AggregateError([err, cleanup], 'Storage write and plaintext cleanup failed', { cause: cleanup }) }
53
+ throw err
54
+ }
55
+ }
56
+
57
+ async function decryptOwned(stored: RawObject, key: StorageKey, type: StorageRowKind, row: StorageRow, identity: string) {
58
+ let dataKey: Buffer | undefined
59
+ try {
60
+ dataKey = unwrapDataKey(key, type, row)
61
+ return await decryptStorageStream(stored.stream, dataKey, identity)
62
+ } catch (err) { stored.stream.destroy(); throw err }
63
+ finally { dataKey?.fill(0) }
64
+ }
65
+
66
+ async function openEncrypted(raw: RawObjectStorage, db: StorageDb, key: StorageKey, identity: string, state: StorageEncryptionState): ReturnType<ObjectStorage['open']> {
67
+ const owner = storageOwner(identity)
68
+ if (!owner) {
69
+ if (!identity.startsWith('uploads/')) throw new Error('Unknown managed storage owner')
70
+ const stored = await raw.open(identity)
71
+ if (!stored) return null
72
+ const inspected = await inspectStorageObject(stored)
73
+ if (inspected.encrypted) return decryptStorageStream(inspected.stream, key.bytes, identity)
74
+ if (stored.modifiedAt <= state.enabledAt && Date.now() < state.enabledAt + STORAGE_UPLOAD_TTL_MS) return inspected
75
+ inspected.stream.destroy()
76
+ throw new Error('Unexpected plaintext upload part')
77
+ }
78
+ for (let attempt = 0; attempt < 3; attempt++) {
79
+ const row = await db.getStorageRow(owner.type, owner.id)
80
+ if (!row) return null
81
+ if (!owner.cache && storageRowPath(owner.type, row) !== identity) return null
82
+ if (owner.cache && !row.dataKey) return null
83
+ if (!owner.cache && !row.encrypted) state = (await db.getStorageEncryption())!
84
+ const stored = await raw.open(owner.cache ? encryptedCachePath(identity) : identity)
85
+ if (!stored) return null
86
+ // Completed uploads and all new caches are always encrypted. Only legacy
87
+ // pending rows need header inspection and hash-verified plaintext fallback.
88
+ if (row.encrypted || owner.cache || state.complete) {
89
+ return decryptOwned(stored, key, owner.type, row, identity)
90
+ }
91
+ let inspected = await inspectStorageObject(stored)
92
+ if (inspected.encrypted && !row.dataKey && (await db.getStorageRow(owner.type, owner.id))?.dataKey) {
93
+ inspected.stream.destroy(); continue
94
+ }
95
+ if (inspected.encrypted && row.dataKey) {
96
+ try { return await decryptOwned(inspected, key, owner.type, row, identity) }
97
+ catch {
98
+ // An arbitrary legacy upload can begin with our magic bytes. Only
99
+ // its original SQL hash can authorize that plaintext interpretation.
100
+ const legacy = await openObjectVersion(raw, identity, stored.version)
101
+ if (!legacy) continue
102
+ inspected = { ...legacy, encrypted: false }
103
+ }
104
+ }
105
+ // Only a pending row may read plaintext, and only the original upload.
106
+ // Verify first, then reopen that exact version before exposing bytes.
107
+ await verifyStoragePayload(owner.type, row, inspected.stream)
108
+ const current = await openObjectVersion(raw, identity, stored.version)
109
+ if (current) return current
110
+ }
111
+ throw new Error('Storage changed repeatedly during read')
112
+ }
113
+
114
+ // Startup enables encryption before constructing stores; validate the persisted
115
+ // requirement even when this adapter is opened without a key.
116
+ async function storagePolicy(db: StorageDb, key: StorageKey | null) {
117
+ let enabled = await db.getStorageEncryption()
118
+ let checkedAt = Date.now()
119
+ return async (fresh = false) => {
120
+ // Enabled is irreversible. Keyless instances cache "off" briefly, while
121
+ // writes and detected ciphertext always recheck the activation fence.
122
+ if (!enabled && (fresh || key || Date.now() - checkedAt >= STORAGE_DISABLED_TTL_MS)) {
123
+ const state = await db.getStorageEncryption()
124
+ if (state) enabled = state // A delayed disabled read cannot undo "on".
125
+ checkedAt = Date.now()
126
+ }
127
+ if (enabled && !key) throw new Error('Managed storage requires its configured encryption key')
128
+ return enabled
129
+ }
130
+ }
131
+
132
+ export async function createEncryptedObjectStorage(raw: RawObjectStorage, db: StorageDb, key: StorageKey | null): Promise<ObjectStorage> {
133
+ const mode = await storagePolicy(db, key)
134
+ async function open(identity: string): ReturnType<ObjectStorage['open']> {
135
+ logicalKey(identity)
136
+ const state = await mode()
137
+ if (identity.startsWith('avatars/')) return raw.open(identity)
138
+ if (state) return openEncrypted(raw, db, key!, identity, state)
139
+ const stored = await raw.open(identity)
140
+ if (!stored) return null
141
+ const inspected = await inspectStorageObject(stored)
142
+ // Activation/migration can win after the disabled-state read. Never send
143
+ // that ciphertext to a caller expecting the original plaintext bytes.
144
+ if (inspected.encrypted) {
145
+ let current: StorageEncryptionState | null
146
+ try { current = await mode(true) } catch (err) { inspected.stream.destroy(); throw err }
147
+ if (current) { inspected.stream.destroy(); return openEncrypted(raw, db, key!, identity, current) }
148
+ }
149
+ return inspected
150
+ }
151
+ return {
152
+ open,
153
+ get: async identity => readBytes(await open(identity)),
154
+ async exists(identity) {
155
+ logicalKey(identity)
156
+ const owner = storageOwner(identity), state = await mode()
157
+ if (state && owner) {
158
+ const row = await db.getStorageRow(owner.type, owner.id)
159
+ if (!row || (owner.cache ? !row.dataKey : storageRowPath(owner.type, row) !== identity)) return false
160
+ }
161
+ // Existence is metadata only; open() authenticates the actual bytes.
162
+ return await raw.head(state && owner?.cache ? encryptedCachePath(identity) : identity) !== null
163
+ },
164
+ async put(identity, bytes) {
165
+ logicalKey(identity)
166
+ const state = await mode()
167
+ if (identity.startsWith('avatars/')) { await raw.put(identity, bytes); return null }
168
+ if (!state) return putPlaintext(raw, identity, bytes, () => mode(true))
169
+ const owner = storageOwner(identity)
170
+ let dataKey: Buffer, wrapped: string | null = null
171
+ if (owner?.cache) {
172
+ const row = await db.ensureStorageDataKey(owner.type, owner.id)
173
+ if (!row) throw new Error('Cache owner unavailable')
174
+ dataKey = unwrapDataKey(key!, owner.type, row)
175
+ } else if (owner) {
176
+ if (await db.getStorageRow(owner.type, owner.id)) throw new Error('Stored payloads are immutable')
177
+ dataKey = randomBytes(32)
178
+ wrapped = wrapStorageValue(key!, dataKeyIdentity(owner.type, owner.id), dataKey)
179
+ } else {
180
+ if (!identity.startsWith('uploads/')) throw new Error('Unknown managed storage owner')
181
+ dataKey = Buffer.from(key!.bytes)
182
+ }
183
+ const target = owner?.cache ? encryptedCachePath(identity) : identity
184
+ const encrypted = encryptStorageStream(Readable.from([bytes]), dataKey, identity, bytes.length)
185
+ try {
186
+ await raw.put(target, encrypted, AbortSignal.timeout(STORAGE_WRITE_MS), undefined, bytes.length)
187
+ // A late cache builder must not leave a readable cache for a deleted
188
+ // bundle. The missing SQL key also makes failed cleanup unreadable.
189
+ if (owner?.cache && !await db.getStorageRow(owner.type, owner.id)) await raw.delete(target)
190
+ return wrapped
191
+ } finally { encrypted.destroy(); dataKey.fill(0) }
192
+ },
193
+ async delete(identity) {
194
+ logicalKey(identity)
195
+ await mode()
196
+ // Always remove both generations, including caches predating activation
197
+ // when migration is disabled or another instance has just enabled it.
198
+ for (const target of deletionPaths(identity)) await raw.delete(target)
199
+ },
200
+ async deletePrefix(prefix) {
201
+ logicalKey(`${prefix}validate`)
202
+ if (!prefix.endsWith('/')) throw new Error('Invalid storage prefix')
203
+ await mode()
204
+ const signal = AbortSignal.timeout(STORAGE_WRITE_MS)
205
+ for (const target of deletionPaths(prefix)) {
206
+ await deleteObjects(raw, target, signal)
207
+ await raw.prune?.(target, signal)
208
+ }
209
+ },
210
+ }
211
+ }