@preventive/triage 1.0.0-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/common/save-error-reason.ts +53 -0
- package/common/utf8.d.ts +13 -0
- package/common/utf8.js +57 -0
- package/out/brotli-fallback.js +3 -0
- package/out/client-sync.js +15 -0
- package/out/graph.js +4 -0
- package/out/icon-maskable.svg +5 -0
- package/out/icon.svg +5 -0
- package/out/index.html +78 -0
- package/out/manifest.webmanifest +30 -0
- package/out/prism.js +14 -0
- package/out/terminal.js +39 -0
- package/out/view.css +1 -0
- package/out/view.html +12 -0
- package/out/view.js +138 -0
- package/package.json +129 -0
- package/server/auth.ts +99 -0
- package/server/config.example.json +3 -0
- package/server/config.ts +196 -0
- package/server/db-neon.ts +374 -0
- package/server/db-revision-sql.ts +152 -0
- package/server/db-stmt.ts +53 -0
- package/server/db.ts +577 -0
- package/server/http.ts +142 -0
- package/server/hub.ts +98 -0
- package/server/index.ts +353 -0
- package/server/lifecycle.ts +177 -0
- package/server/neon-driver.ts +26 -0
- package/server/objstore/blob-fs.ts +164 -0
- package/server/objstore/blob-vercel.ts +508 -0
- package/server/objstore/blob.ts +169 -0
- package/server/objstore/fs.ts +67 -0
- package/server/objstore/handlers.ts +235 -0
- package/server/objstore/init.ts +118 -0
- package/server/objstore/reaper.ts +199 -0
- package/server/objstore/rest.ts +484 -0
- package/server/objstore/sign.ts +164 -0
- package/server/objstore/store-neon.ts +351 -0
- package/server/objstore/store.ts +799 -0
- package/server/objstore/tokens.ts +168 -0
- package/server/origin.ts +68 -0
- package/server/peer.ts +38 -0
- package/server/sign.ts +231 -0
- package/server/static.ts +374 -0
- package/server/sync-handlers.ts +311 -0
- package/server/util.ts +27 -0
- package/server/validation.ts +36 -0
- package/server/ws-server.ts +245 -0
package/server/http.ts
ADDED
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
// HTTP plane: the REST byte-transfer routing (`/api/objstore/...`), the
|
|
2
|
+
// static UI bundle, and the WebSocket upgrade gate. Built once at boot
|
|
3
|
+
// with the WS server plus the lifecycle hooks it needs (`track` to
|
|
4
|
+
// drain in-flight requests on shutdown, `isShuttingDown` to gate new
|
|
5
|
+
// ones). The WS *connection* handler is wired on `wss` separately in
|
|
6
|
+
// index.ts; this module only owns the upgrade handshake.
|
|
7
|
+
|
|
8
|
+
import { type IncomingMessage as HttpRequest, type Server, type ServerResponse, createServer } from 'node:http'
|
|
9
|
+
import { Buffer } from 'node:buffer'
|
|
10
|
+
import { fileURLToPath } from 'node:url'
|
|
11
|
+
import type { WebSocketServer } from 'ws'
|
|
12
|
+
import { type ObjstoreRestDeps, handleRest, matchRoute } from './objstore/rest.ts'
|
|
13
|
+
import { loadStatic } from './static.ts'
|
|
14
|
+
import { errStack } from './util.ts'
|
|
15
|
+
|
|
16
|
+
// `/api/*` is reserved for backend traffic so a fronting nginx (or
|
|
17
|
+
// similar) can route `/api/*` → this process and `/*` → the static UI
|
|
18
|
+
// bundle with a single location block.
|
|
19
|
+
export const WS_UPGRADE_PATH = '/api/sync'
|
|
20
|
+
function isUpgradePath(url: string | undefined): boolean {
|
|
21
|
+
if (typeof url !== 'string') return false
|
|
22
|
+
// Strip `?…` so clients can carry build / debug tags. Exact match
|
|
23
|
+
// otherwise — `/api/sync/` (trailing slash) doesn't pass.
|
|
24
|
+
return url.split('?', 1)[0] === WS_UPGRADE_PATH
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
const NOT_FOUND_BODY = JSON.stringify({ error: 'not-found' })
|
|
28
|
+
|
|
29
|
+
type HasHeaders = { headers: HttpRequest['headers'] }
|
|
30
|
+
|
|
31
|
+
export type HttpServerDeps = {
|
|
32
|
+
wss: WebSocketServer
|
|
33
|
+
restDeps: ObjstoreRestDeps
|
|
34
|
+
isOriginAllowed: (req: HasHeaders) => boolean
|
|
35
|
+
isShuttingDown: () => boolean
|
|
36
|
+
track: (promise: Promise<unknown>) => void
|
|
37
|
+
restPutIdleTimeoutMs: number
|
|
38
|
+
debug: boolean
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export function createHttpServer(deps: HttpServerDeps): Server {
|
|
42
|
+
const { wss, restDeps, isOriginAllowed, isShuttingDown, track, restPutIdleTimeoutMs, debug } = deps
|
|
43
|
+
// Static-file plane (see ./static.ts). The directory is the
|
|
44
|
+
// `build.js build` output sibling to this file; the loader handles
|
|
45
|
+
// enumeration, pre-compression, and ETag derivation. Plugged in after
|
|
46
|
+
// the `/api/objstore/...` REST branch.
|
|
47
|
+
const handleStatic = loadStatic(fileURLToPath(new URL('../out', import.meta.url)))
|
|
48
|
+
|
|
49
|
+
const httpServer = createServer((req: HttpRequest, res: ServerResponse) => {
|
|
50
|
+
if (matchRoute(req.url) != null) {
|
|
51
|
+
// Shutdown gate. The WS plane gates new messages on `shuttingDown`;
|
|
52
|
+
// REST handlers go through a separate path and must mirror it.
|
|
53
|
+
// Without this, a REST PUT arriving on an existing keep-alive
|
|
54
|
+
// socket AFTER SIGTERM but BEFORE `httpServer.close()` finishes
|
|
55
|
+
// draining could land in `withCommitLock`, acquire a lease, and
|
|
56
|
+
// finish its `finally { release() }` AFTER the shutdown's
|
|
57
|
+
// `heldLeaseCount` snapshot — leaving an orphan lock row that pins
|
|
58
|
+
// the key until TTL expiry. The 503 + `shutting-down` reason tells
|
|
59
|
+
// the client to retry against a different replica. Transport
|
|
60
|
+
// audit + multi-replica shutdown ordering review.
|
|
61
|
+
if (isShuttingDown()) {
|
|
62
|
+
res.writeHead(503, { 'content-type': 'application/json', 'connection': 'close' })
|
|
63
|
+
res.end(JSON.stringify({ error: 'shutting-down' }))
|
|
64
|
+
return
|
|
65
|
+
}
|
|
66
|
+
// Same-origin gate. Token IS the auth on REST, but a hostile origin
|
|
67
|
+
// that holds a valid token (e.g. via XSS that read a freshly-minted
|
|
68
|
+
// one) would PUT with its own Origin header — caught here.
|
|
69
|
+
// Same-origin XHR/fetch may omit Origin; that path is allowed (see
|
|
70
|
+
// `isOriginAllowed`). Transport audit `server/objstore/rest.ts:103`.
|
|
71
|
+
if (!isOriginAllowed(req)) {
|
|
72
|
+
res.writeHead(403, { 'content-type': 'application/json' })
|
|
73
|
+
res.end(JSON.stringify({ error: 'origin-denied' }))
|
|
74
|
+
return
|
|
75
|
+
}
|
|
76
|
+
// PUT idle-body timeout — a slow-loris client trickling bytes
|
|
77
|
+
// within the declared Content-Length holds the staging fd + an
|
|
78
|
+
// inFlightSids slot indefinitely. `req.setTimeout` fires on
|
|
79
|
+
// inactivity; we destroy the request, aborting the body pipeline.
|
|
80
|
+
// Transport audit `server/objstore/rest.ts:218`.
|
|
81
|
+
if (req.method === 'PUT') {
|
|
82
|
+
req.setTimeout(restPutIdleTimeoutMs, () => {
|
|
83
|
+
if (debug) console.warn(`REST PUT idle ${restPutIdleTimeoutMs}ms → abort`)
|
|
84
|
+
try { req.destroy(new Error('idle-timeout')) } catch {}
|
|
85
|
+
})
|
|
86
|
+
}
|
|
87
|
+
// Track so SIGTERM mid-upload/download awaits handleRest before the
|
|
88
|
+
// DB close. The outer `.catch` is the unhandled-rejection guard for
|
|
89
|
+
// a stray throw OUTSIDE handleRest's internal try/catch blocks —
|
|
90
|
+
// Node 20+ defaults `--unhandled-rejections=throw`, which would
|
|
91
|
+
// crash the server. Logs and terminates the response so the TCP
|
|
92
|
+
// socket doesn't dangle.
|
|
93
|
+
const p = handleRest(restDeps, req, res).catch((err) => {
|
|
94
|
+
console.warn('REST handler error:', errStack(err))
|
|
95
|
+
if (res.headersSent) { try { res.destroy() } catch {} }
|
|
96
|
+
else { try { res.writeHead(500, { 'content-type': 'application/json' }); res.end(JSON.stringify({ error: 'internal' })) } catch {} }
|
|
97
|
+
})
|
|
98
|
+
track(p)
|
|
99
|
+
return
|
|
100
|
+
}
|
|
101
|
+
if (handleStatic(req, res)) return
|
|
102
|
+
// `Connection: close` so an HTTP/1.1 keep-alive client doesn't hold
|
|
103
|
+
// the socket open expecting more requests on a server that only
|
|
104
|
+
// serves a small REST surface.
|
|
105
|
+
res.writeHead(404, { 'content-type': 'application/json', 'connection': 'close' })
|
|
106
|
+
res.end(NOT_FOUND_BODY)
|
|
107
|
+
})
|
|
108
|
+
|
|
109
|
+
httpServer.on('upgrade', (req, socket, head) => {
|
|
110
|
+
// RFC 6455: the WS upgrade IS an HTTP request; reject with a normal
|
|
111
|
+
// HTTP response so a misconfigured client sees the JSON body instead
|
|
112
|
+
// of ECONNRESET. `socket.end(body)` flushes before sending FIN.
|
|
113
|
+
if (!isUpgradePath(req.url)) {
|
|
114
|
+
socket.end(
|
|
115
|
+
'HTTP/1.1 404 Not Found\r\n' +
|
|
116
|
+
'Content-Type: application/json\r\n' +
|
|
117
|
+
`Content-Length: ${Buffer.byteLength(NOT_FOUND_BODY)}\r\n` +
|
|
118
|
+
'Connection: close\r\n\r\n' +
|
|
119
|
+
NOT_FOUND_BODY,
|
|
120
|
+
)
|
|
121
|
+
return
|
|
122
|
+
}
|
|
123
|
+
// Same-origin gate. The WS upgrade IS a cross-origin-reachable
|
|
124
|
+
// surface in the browser; without this any tab can open a session to
|
|
125
|
+
// a 127.0.0.1 relay and probe handler shape / burn verify CPU.
|
|
126
|
+
// Browser WS handshakes always carry Origin (RFC 6455); non-browser
|
|
127
|
+
// clients omit it and are allowed (network is their trust boundary).
|
|
128
|
+
if (!isOriginAllowed(req)) {
|
|
129
|
+
socket.end(
|
|
130
|
+
'HTTP/1.1 403 Forbidden\r\n' +
|
|
131
|
+
'Content-Type: application/json\r\n' +
|
|
132
|
+
'Content-Length: 26\r\n' +
|
|
133
|
+
'Connection: close\r\n\r\n' +
|
|
134
|
+
'{"error":"origin-denied"}\n',
|
|
135
|
+
)
|
|
136
|
+
return
|
|
137
|
+
}
|
|
138
|
+
wss.handleUpgrade(req, socket, head, (ws) => { wss.emit('connection', ws, req) })
|
|
139
|
+
})
|
|
140
|
+
|
|
141
|
+
return httpServer
|
|
142
|
+
}
|
package/server/hub.ts
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
// WS fan-out hub: the subscriber registry plus the backpressure-aware
|
|
2
|
+
// send / broadcast primitives. Created once at boot with the shared
|
|
3
|
+
// `peers` registry; the connection handler, the triage-sync handlers,
|
|
4
|
+
// the objstore plane, and the REST PUT broadcast all go through the
|
|
5
|
+
// returned methods. Kept separate from the protocol handlers so the
|
|
6
|
+
// transport concern (who's subscribed, backpressure, fan-out) is
|
|
7
|
+
// testable and reasoned about on its own.
|
|
8
|
+
|
|
9
|
+
import type { WebSocket } from 'ws'
|
|
10
|
+
import type { PeerRegistry } from './peer.ts'
|
|
11
|
+
|
|
12
|
+
export type Hub = {
|
|
13
|
+
subscribe(socket: WebSocket, tag: string): void
|
|
14
|
+
unsubscribeAll(socket: WebSocket): void
|
|
15
|
+
send(socket: WebSocket, msg: object): void
|
|
16
|
+
// Lower-level send for an already-serialised payload (broadcast
|
|
17
|
+
// fan-out stringifies once, sends N times).
|
|
18
|
+
sendRaw(socket: WebSocket, payload: string): void
|
|
19
|
+
// `except: null` is the REST-originated path — byte transfer landed
|
|
20
|
+
// via HTTP, not a particular WS socket, so it hits every subscriber.
|
|
21
|
+
// WS-originated broadcasts pass the originator so it doesn't see its
|
|
22
|
+
// own message echoed back.
|
|
23
|
+
broadcast(tag: string, msg: object, except: WebSocket | null): void
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export function createHub(deps: { peers: PeerRegistry; maxBufferedBytes: number; debug: boolean }): Hub {
|
|
27
|
+
const { peers, maxBufferedBytes, debug } = deps
|
|
28
|
+
// workspaceTag → Set<WebSocket>. The per-socket reverse index lives on
|
|
29
|
+
// `Peer.tags` (see ./peer.ts) and is read by `unsubscribeAll` on close.
|
|
30
|
+
const subscribers = new Map<string, Set<WebSocket>>()
|
|
31
|
+
|
|
32
|
+
function subscribe(socket: WebSocket, tag: string): void {
|
|
33
|
+
let set = subscribers.get(tag)
|
|
34
|
+
if (!set) {
|
|
35
|
+
set = new Set()
|
|
36
|
+
subscribers.set(tag, set)
|
|
37
|
+
}
|
|
38
|
+
set.add(socket)
|
|
39
|
+
peers.get(socket)?.tags.add(tag)
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function unsubscribeAll(socket: WebSocket): void {
|
|
43
|
+
const tags = peers.get(socket)?.tags
|
|
44
|
+
if (!tags) return
|
|
45
|
+
for (const tag of tags) {
|
|
46
|
+
const set = subscribers.get(tag)
|
|
47
|
+
if (!set) continue
|
|
48
|
+
set.delete(socket)
|
|
49
|
+
if (set.size === 0) subscribers.delete(tag)
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
function send(socket: WebSocket, msg: object): void {
|
|
54
|
+
sendRaw(socket, JSON.stringify(msg))
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
function sendRaw(socket: WebSocket, payload: string): void {
|
|
58
|
+
if (socket.readyState !== socket.OPEN) return
|
|
59
|
+
// Backpressure cap. `socket.bufferedAmount` is the count of bytes
|
|
60
|
+
// queued in the `ws` send pipeline that haven't drained to the
|
|
61
|
+
// kernel yet — a slow / blackholed peer accumulates them unboundedly
|
|
62
|
+
// during fan-out broadcasts. Drop above the cap and terminate the
|
|
63
|
+
// socket so the heartbeat doesn't keep it alive on ping/pong while
|
|
64
|
+
// every broadcast piles up. Transport audit `server/index.ts:225`.
|
|
65
|
+
if (socket.bufferedAmount > maxBufferedBytes) {
|
|
66
|
+
if (debug) console.warn(`drop broadcast: socket buffered ${socket.bufferedAmount}B > cap`)
|
|
67
|
+
try { socket.terminate() } catch {}
|
|
68
|
+
return
|
|
69
|
+
}
|
|
70
|
+
// Wrap send() in try/catch — readyState can transition from OPEN to
|
|
71
|
+
// CLOSING between the check above and the send() call (TOCTOU window
|
|
72
|
+
// in `ws`'s event loop). Without this, a socket dying mid-broadcast
|
|
73
|
+
// would throw and abort the broadcast loop, skipping every
|
|
74
|
+
// subscriber after the dead one. Audit M4.
|
|
75
|
+
try { socket.send(payload) } catch {}
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
function broadcast(tag: string, msg: object, except: WebSocket | null): void {
|
|
79
|
+
const set = subscribers.get(tag)
|
|
80
|
+
if (!set) return
|
|
81
|
+
// Stringify ONCE outside the fan-out loop. For a workspace-state
|
|
82
|
+
// catch-up with a multi-MB ciphertext × N subscribers, per-recipient
|
|
83
|
+
// JSON.stringify would dominate CPU; this is the cheap win.
|
|
84
|
+
const payload = JSON.stringify(msg)
|
|
85
|
+
// Snapshot before iterating — `send`'s try/catch swallows
|
|
86
|
+
// socket.send errors, but a socket transitioning to CLOSED
|
|
87
|
+
// mid-broadcast triggers `unsubscribeAll` from the 'close' handler,
|
|
88
|
+
// which mutates `set` while we're walking it. The snapshot keeps a
|
|
89
|
+
// future refactor (different collection, async send) from silently
|
|
90
|
+
// skipping subscribers. Audit M4 round-3.
|
|
91
|
+
for (const s of [...set]) {
|
|
92
|
+
if (s === except) continue
|
|
93
|
+
sendRaw(s, payload)
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
return { subscribe, unsubscribeAll, send, sendRaw, broadcast }
|
|
98
|
+
}
|
package/server/index.ts
ADDED
|
@@ -0,0 +1,353 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// DeepView triage-sync relay server. WebSocket front-end, SQLite
|
|
3
|
+
// backing store. Implements the protocol described in
|
|
4
|
+
// `client/triage-sync.js` (and `server/sign.ts` for the canonical
|
|
5
|
+
// signature payloads):
|
|
6
|
+
//
|
|
7
|
+
// server → client challenge { nonce } — emitted on every
|
|
8
|
+
// accept, BEFORE any client
|
|
9
|
+
// frame; per-socket random
|
|
10
|
+
// 128-bit value the client
|
|
11
|
+
// must bind into every
|
|
12
|
+
// `workspace-subscribe`
|
|
13
|
+
// signature (round-9 H2)
|
|
14
|
+
// client → server workspace-save { workspaceTag, base,
|
|
15
|
+
// keyframe, nonce, ciphertext,
|
|
16
|
+
// signature } — `keyframe` is
|
|
17
|
+
// a boolean (`true` exactly,
|
|
18
|
+
// else falsy), bound into the
|
|
19
|
+
// signed canonical
|
|
20
|
+
// client → server workspace-subscribe { workspaceTag, from,
|
|
21
|
+
// signature } — `from` is the
|
|
22
|
+
// last revision id the client
|
|
23
|
+
// claims to have applied (or
|
|
24
|
+
// null for fresh)
|
|
25
|
+
// client → server ping — heartbeat
|
|
26
|
+
// server → client pong — heartbeat reply
|
|
27
|
+
// server → client workspace-save-ack { workspaceTag, base, id }
|
|
28
|
+
// server → client workspace-save-error { workspaceTag, base, reason }
|
|
29
|
+
// — explicit failure surface for
|
|
30
|
+
// the legit-signer case where
|
|
31
|
+
// the server rejects a signed
|
|
32
|
+
// save (e.g. `too-large` past
|
|
33
|
+
// MAX_CIPHERTEXT_LEN). Sent
|
|
34
|
+
// AFTER sig verify so the
|
|
35
|
+
// response only reaches a
|
|
36
|
+
// legitimate seed holder; shape
|
|
37
|
+
// attacks still drop silently.
|
|
38
|
+
// server → client workspace-subscribed { workspaceTag } — explicit
|
|
39
|
+
// handshake-complete ack so
|
|
40
|
+
// the client can flip its
|
|
41
|
+
// status from `connecting` to
|
|
42
|
+
// `online` only after the
|
|
43
|
+
// server registered it as a
|
|
44
|
+
// peer (not just on socket
|
|
45
|
+
// open)
|
|
46
|
+
// server → client workspace-state { workspaceTag, revisions:
|
|
47
|
+
// [{ base, id, keyframe,
|
|
48
|
+
// nonce, ciphertext,
|
|
49
|
+
// signature }, ...] }
|
|
50
|
+
//
|
|
51
|
+
// Authentication: every signed message is checked against the
|
|
52
|
+
// `workspaceTag` (= base64url Ed25519 public key) before any
|
|
53
|
+
// state mutation. Unsigned / bad-sig messages are dropped silently
|
|
54
|
+
// — the legitimate signer will retry, and an attacker who learns
|
|
55
|
+
// the tag without holding the seed can't get past the verify.
|
|
56
|
+
//
|
|
57
|
+
// Content opacity: `nonce` and `ciphertext` are opaque to the
|
|
58
|
+
// server. We store and forward them; we never inspect.
|
|
59
|
+
//
|
|
60
|
+
// Subscriber tracking: `subscribers: Map<workspaceTag, Set<socket>>`.
|
|
61
|
+
// A socket joins the set ONLY via an explicit, signature-verified
|
|
62
|
+
// `workspace-subscribe` (sole call site of `subscribe()` is in
|
|
63
|
+
// `handleSubscribe`, gated by the per-connection challenge nonce
|
|
64
|
+
// bound into the signature and a post-await `readyState` check).
|
|
65
|
+
// It leaves on disconnect. Broadcasts go to every subscriber for
|
|
66
|
+
// the workspaceTag except the originator.
|
|
67
|
+
//
|
|
68
|
+
// `workspace-save` deliberately does NOT auto-attach the sender,
|
|
69
|
+
// even on a valid signature — see the audit note in `handleSave`
|
|
70
|
+
// (round-9 H1): auto-subscribe-on-save let a passive observer
|
|
71
|
+
// replay any captured save frame from any TCP connection to attach
|
|
72
|
+
// as a silent mirror, since the duplicate-id path returns ack-only
|
|
73
|
+
// and would not reject the attaching socket.
|
|
74
|
+
|
|
75
|
+
import { type WebSocket, WebSocketServer } from 'ws'
|
|
76
|
+
import { errMsg } from './util.ts'
|
|
77
|
+
import type { PeerRegistry } from './peer.ts'
|
|
78
|
+
import { LOOPBACK_HOSTS, createOriginGate } from './origin.ts'
|
|
79
|
+
import { createHub } from './hub.ts'
|
|
80
|
+
import { createAuth } from './auth.ts'
|
|
81
|
+
import { createSyncHandlers } from './sync-handlers.ts'
|
|
82
|
+
import { WS_UPGRADE_PATH, createHttpServer } from './http.ts'
|
|
83
|
+
import { installWsServer } from './ws-server.ts'
|
|
84
|
+
import { createLifecycle } from './lifecycle.ts'
|
|
85
|
+
import { loadConfig } from './config.ts'
|
|
86
|
+
import { type Handle, openDb } from './db.ts'
|
|
87
|
+
import { openNeonDb } from './db-neon.ts'
|
|
88
|
+
import { initObjstore } from './objstore/init.ts'
|
|
89
|
+
import { type Handle as ObjstoreHandle, listLive, objectMetaWire, openObjstore } from './objstore/store.ts'
|
|
90
|
+
import { openNeonObjstore } from './objstore/store-neon.ts'
|
|
91
|
+
import { openVercelBlobBackend } from './objstore/blob-vercel.ts'
|
|
92
|
+
|
|
93
|
+
// All external inputs (env vars + optional config.json) are parsed
|
|
94
|
+
// and validated in ./config.ts. Destructure into the existing
|
|
95
|
+
// uppercase names so the rest of this module reads unchanged.
|
|
96
|
+
const config = loadConfig()
|
|
97
|
+
const {
|
|
98
|
+
port: PORT, host: HOST, dbPath: DB_PATH, objstoreDir: OBJSTORE_DIR,
|
|
99
|
+
reapIntervalMs: OBJSTORE_REAP_INTERVAL_MS,
|
|
100
|
+
maxInflightPerSocket: MAX_INFLIGHT_PER_SOCKET, debug: DEBUG,
|
|
101
|
+
neonUrl: NEON_URL, blobToken: BLOB_TOKEN, tokenSecret: TOKEN_SECRET,
|
|
102
|
+
password: CONFIG_PASSWORD, trustProxyEnv: TRUST_PROXY_ENV,
|
|
103
|
+
} = config
|
|
104
|
+
|
|
105
|
+
// Same-origin gate for the WS upgrade and REST data plane (see
|
|
106
|
+
// ./origin.ts). `TRUST_PROXY_ENV` (from config) also feeds the
|
|
107
|
+
// boot-time misconfiguration fail-fast below.
|
|
108
|
+
const { trustProxy: TRUST_PROXY, isOriginAllowed } = createOriginGate(HOST, TRUST_PROXY_ENV)
|
|
109
|
+
|
|
110
|
+
// Per-socket buffered-bytes cap. `socket.send` returns synchronously
|
|
111
|
+
// even when the kernel/ws library can't drain to the wire fast
|
|
112
|
+
// enough; the unsent payload accumulates in `bufferedAmount`. A
|
|
113
|
+
// slow / blackholed peer on a high-volume workspace can hold many
|
|
114
|
+
// MB of fan-out broadcasts in this buffer with no backpressure on
|
|
115
|
+
// the broadcast loop. Drop the message when the buffer crosses the
|
|
116
|
+
// cap; the heartbeat will eventually close a peer that never
|
|
117
|
+
// drains. Transport audit `server/index.ts:225`.
|
|
118
|
+
const MAX_BUFFERED_BYTES = 16 * 1024 * 1024
|
|
119
|
+
// Per-socket in-flight async-handler cap (MAX_INFLIGHT_PER_SOCKET,
|
|
120
|
+
// env-validated in config). Each inbound text frame spawns a
|
|
121
|
+
// `track(handler)` IIFE; an authorised peer firing valid frames could
|
|
122
|
+
// otherwise grow the set without bound, stretching SIGTERM drain time.
|
|
123
|
+
// Saves dropped at the cap surface as a typed `busy` NACK. Transport
|
|
124
|
+
// audit `server/index.ts:590`.
|
|
125
|
+
|
|
126
|
+
// Per-connection state registry. One `Peer` per accepted socket holds
|
|
127
|
+
// the challenge nonce, auth flag, heartbeat liveness, in-flight count,
|
|
128
|
+
// and subscribed tags (see ./peer.ts) — replacing what were five
|
|
129
|
+
// parallel per-socket WeakMaps. The connection handler holds the Peer
|
|
130
|
+
// in a closure for the hot paths; cross-function call sites resolve it
|
|
131
|
+
// via `peers.get(socket)`.
|
|
132
|
+
const peers: PeerRegistry = new WeakMap()
|
|
133
|
+
|
|
134
|
+
// REST PUT idle-body timeout. A slow-loris client trickling bytes
|
|
135
|
+
// within the declared Content-Length holds the staging fd and an
|
|
136
|
+
// inFlightSids slot until the global staging TTL reaps it. Aborting
|
|
137
|
+
// the per-chunk-idle period closes that window. Transport audit
|
|
138
|
+
// `server/objstore/rest.ts:218`.
|
|
139
|
+
const REST_PUT_IDLE_TIMEOUT_MS = 30_000
|
|
140
|
+
|
|
141
|
+
// Server-driven WS heartbeat. Every `HEARTBEAT_INTERVAL_MS` we walk
|
|
142
|
+
// `wss.clients`, terminate anyone who didn't pong since the last
|
|
143
|
+
// tick, and ping the rest. The client-initiated `{type:'ping'}` /
|
|
144
|
+
// `{type:'pong'}` JSON heartbeat the protocol already had only
|
|
145
|
+
// catches the case where the CLIENT notices the socket's gone — it
|
|
146
|
+
// can't recover an FD when the client itself has wandered off
|
|
147
|
+
// (battery-killed background tab, mid-transfer NAT timeout, a
|
|
148
|
+
// hostile non-browser client that opens the socket and never
|
|
149
|
+
// speaks again). The same-origin upgrade gate allows missing
|
|
150
|
+
// Origin headers through (legitimate non-browser callers), so a
|
|
151
|
+
// hostile CLI can stack arbitrarily many idle sockets without it.
|
|
152
|
+
// Kernel TCP keepalive is hours by default; without this interval
|
|
153
|
+
// each abandoned socket pins its `wss.clients` Set entry, its `Peer`
|
|
154
|
+
// state, and an FD until the kernel reclaims it. Two ticks max
|
|
155
|
+
// from silence to termination, so the longest a dead socket
|
|
156
|
+
// survives is ~2 × HEARTBEAT_INTERVAL_MS.
|
|
157
|
+
const HEARTBEAT_INTERVAL_MS = 30_000
|
|
158
|
+
|
|
159
|
+
// Backend selection. Both planes (workspace_revision DB + the
|
|
160
|
+
// v1.objstore byte store) are picked from config at boot. Two supported
|
|
161
|
+
// pairings:
|
|
162
|
+
// 1. DATABASE_URL set → Neon (workspace_revision + objstore
|
|
163
|
+
// tables) + Vercel Blob Private Storage (bytes). Requires
|
|
164
|
+
// BLOB_READ_WRITE_TOKEN — fail fast at boot if missing, since
|
|
165
|
+
// a local-FS byte plane can't back a multi-replica deployment
|
|
166
|
+
// (one replica's writes wouldn't be visible to another).
|
|
167
|
+
// 2. DATABASE_URL absent → SQLite + local FS bytes. Single-
|
|
168
|
+
// process; the only pairing the SQLite plane supports.
|
|
169
|
+
// The Neon / Vercel files import their peer deps lazily inside the
|
|
170
|
+
// open functions, so static imports here are safe even on a SQLite-
|
|
171
|
+
// only install where the optional peer deps aren't present. Branch
|
|
172
|
+
// out explicitly (rather than via a ternary) so the SQLite path
|
|
173
|
+
// keeps its `SqliteHandle` narrowing — `sqliteHandle.db` is typed
|
|
174
|
+
// as a non-optional `DatabaseSync` and `openObjstore` accepts it
|
|
175
|
+
// without a non-null assertion.
|
|
176
|
+
let handle: Handle
|
|
177
|
+
let objstoreHandle: ObjstoreHandle
|
|
178
|
+
let objstoreBanner: string
|
|
179
|
+
if (NEON_URL) {
|
|
180
|
+
if (!BLOB_TOKEN) {
|
|
181
|
+
console.error('DATABASE_URL is set but BLOB_READ_WRITE_TOKEN is not.')
|
|
182
|
+
console.error('The Neon DB plane requires the Vercel Blob byte plane (local-FS bytes cannot back a multi-replica deployment).')
|
|
183
|
+
console.error('Set BLOB_READ_WRITE_TOKEN to your Vercel Blob R/W token, or unset DATABASE_URL to fall back to SQLite + local FS.')
|
|
184
|
+
process.exit(1)
|
|
185
|
+
}
|
|
186
|
+
if (!TOKEN_SECRET) {
|
|
187
|
+
console.error('DATABASE_URL is set but OBJSTORE_TOKEN_SECRET is not.')
|
|
188
|
+
console.error('Multi-replica deployments need a shared HMAC secret so REST bearer tokens minted on one replica validate on any other.')
|
|
189
|
+
console.error('Generate one with: node -e \'console.log(require("crypto").randomBytes(32).toString("base64"))\'')
|
|
190
|
+
process.exit(1)
|
|
191
|
+
}
|
|
192
|
+
handle = await openNeonDb(NEON_URL)
|
|
193
|
+
const blob = await openVercelBlobBackend({ token: BLOB_TOKEN })
|
|
194
|
+
objstoreHandle = await openNeonObjstore(NEON_URL, blob)
|
|
195
|
+
objstoreBanner = 'objstore: vercel-blob (private)'
|
|
196
|
+
} else {
|
|
197
|
+
const sqliteHandle = openDb(DB_PATH)
|
|
198
|
+
handle = sqliteHandle
|
|
199
|
+
objstoreHandle = openObjstore(sqliteHandle.db, OBJSTORE_DIR)
|
|
200
|
+
objstoreBanner = `objstore: ${OBJSTORE_DIR}`
|
|
201
|
+
}
|
|
202
|
+
// Multi-replica deployments behind a load balancer / TLS terminator
|
|
203
|
+
// (the typical Vercel + Neon shape) need TRUST_PROXY=1 to honour
|
|
204
|
+
// X-Forwarded-Host when computing the same-origin gate's expected
|
|
205
|
+
// origin. Otherwise the gate derives the origin from the internal
|
|
206
|
+
// container hostname and rejects every browser request as a
|
|
207
|
+
// cross-origin attempt — silently from the operator's perspective
|
|
208
|
+
// until users report 403s. Fail fast (parallels the
|
|
209
|
+
// BLOB_READ_WRITE_TOKEN / OBJSTORE_TOKEN_SECRET checks above) so
|
|
210
|
+
// a misconfigured deploy doesn't ship a 100%-403 fleet. An
|
|
211
|
+
// operator who genuinely terminates TLS in the container without
|
|
212
|
+
// X-Forwarded-* (rare) can set `TRUST_PROXY=0` to acknowledge.
|
|
213
|
+
if (NEON_URL && !TRUST_PROXY && !LOOPBACK_HOSTS.has(HOST) && TRUST_PROXY_ENV !== '0' && TRUST_PROXY_ENV !== 'false') {
|
|
214
|
+
console.error(`DATABASE_URL is set and HOST=${HOST} is not loopback, but TRUST_PROXY is not enabled.`)
|
|
215
|
+
console.error('Browser requests through a load balancer / TLS terminator will be rejected by the same-origin gate (all 403).')
|
|
216
|
+
console.error('Set TRUST_PROXY=1 to honour X-Forwarded-Host / X-Forwarded-Proto from the upstream proxy.')
|
|
217
|
+
console.error('Set TRUST_PROXY=0 if you really terminate TLS in the container without X-Forwarded-* headers (no proxy).')
|
|
218
|
+
process.exit(1)
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
// "Workspace exists on the server" gate. The auth requirement only
|
|
222
|
+
// kicks in for the FIRST action against a never-before-seen tag —
|
|
223
|
+
// once any row lands (triage revision or objstore object), the
|
|
224
|
+
// workspace is considered established and signature-gated. Checks
|
|
225
|
+
// both planes so the gate stays consistent regardless of which
|
|
226
|
+
// action the user picks first (workspace-save in the common case;
|
|
227
|
+
// objstore-put-begin for a bundle-first flow).
|
|
228
|
+
async function workspaceExists(tag: string): Promise<boolean> {
|
|
229
|
+
if (await handle.headFor.get(tag)) return true
|
|
230
|
+
const c = await objstoreHandle.countLive.get(tag)
|
|
231
|
+
return (c?.c ?? 0) > 0
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
// WS fan-out hub: subscriber registry + backpressure-aware send /
|
|
235
|
+
// broadcast (see ./hub.ts). Destructure into the existing names so the
|
|
236
|
+
// handlers / dispatcher / objstore wiring below read unchanged.
|
|
237
|
+
const hub = createHub({ peers, maxBufferedBytes: MAX_BUFFERED_BYTES, debug: DEBUG })
|
|
238
|
+
const { send, broadcast, subscribe, unsubscribeAll } = hub
|
|
239
|
+
|
|
240
|
+
// Password gate (see ./auth.ts) — HMAC derivation + the `authenticate`
|
|
241
|
+
// handshake. Destructure into the existing names for the wiring below.
|
|
242
|
+
const auth = createAuth({ peers, password: CONFIG_PASSWORD, send, debug: DEBUG })
|
|
243
|
+
const { requiresAuth, handleAuthenticate, sendUnauthorized } = auth
|
|
244
|
+
|
|
245
|
+
// Triage-sync protocol handlers (see ./sync-handlers.ts). `getNonce`
|
|
246
|
+
// resolves a socket's challenge nonce and is shared with the objstore
|
|
247
|
+
// wiring below; `sendSaveError` is reused by the dispatcher's `busy`
|
|
248
|
+
// inflight-cap NACK path.
|
|
249
|
+
const getNonce = (socket: WebSocket): string | undefined => peers.get(socket)?.challenge
|
|
250
|
+
const { handleSave, handleSubscribe, sendSaveError } = createSyncHandlers({
|
|
251
|
+
handle, send, broadcast, subscribe, getNonce,
|
|
252
|
+
requiresAuth, sendUnauthorized, workspaceExists,
|
|
253
|
+
// Folds the objstore inventory into the `workspace-subscribed` ack.
|
|
254
|
+
// The objstore store keeps its own richer `Handle`, so we wire the
|
|
255
|
+
// query here where both handles exist rather than coupling
|
|
256
|
+
// sync-handlers to the store type.
|
|
257
|
+
objstoreResources: async (tag) => (await listLive(objstoreHandle, tag)).map(objectMetaWire),
|
|
258
|
+
debug: DEBUG,
|
|
259
|
+
})
|
|
260
|
+
|
|
261
|
+
const { handlers: objstore, restDeps: objstoreRestDeps, startupReap, stopReaper } = initObjstore({
|
|
262
|
+
handle: objstoreHandle, reapIntervalMs: OBJSTORE_REAP_INTERVAL_MS,
|
|
263
|
+
send, broadcast, getNonce, debug: DEBUG,
|
|
264
|
+
// Auth gate for the FIRST objstore-put-begin against a workspace
|
|
265
|
+
// that doesn't yet exist on the server. Mirrors handleSave's gate
|
|
266
|
+
// below; handlers.ts calls this AFTER sig verify so the
|
|
267
|
+
// `unauthorized` frame only reaches a legitimate signer. Returns
|
|
268
|
+
// `false` to allow, `true` to deny — handlers.ts emits the
|
|
269
|
+
// `unauthorized` frame and bails on `true`.
|
|
270
|
+
authGate: async (socket, tag) => requiresAuth(socket) && !await workspaceExists(tag),
|
|
271
|
+
sendUnauthorized,
|
|
272
|
+
// `tokenSecret` is set only when OBJSTORE_TOKEN_SECRET was
|
|
273
|
+
// provided in env (see TOKEN_SECRET resolution above). Omitted
|
|
274
|
+
// → initObjstore mints a fresh per-process secret (the pre-PR
|
|
275
|
+
// behaviour, fine for single-replica).
|
|
276
|
+
...(TOKEN_SECRET ? { tokenSecret: TOKEN_SECRET } : {}),
|
|
277
|
+
})
|
|
278
|
+
|
|
279
|
+
// 4 MiB cap leaves headroom above MAX_CIPHERTEXT_LEN (2 MiB) for
|
|
280
|
+
// the JSON envelope + base64 overhead. `ws` defaults to 100 MiB
|
|
281
|
+
// which any unauthenticated peer could spam — every connection
|
|
282
|
+
// accepts and JSON.parses up to that before the signature-fail drops
|
|
283
|
+
// the frame.
|
|
284
|
+
const wss = new WebSocketServer({ noServer: true, maxPayload: 4 * 1024 * 1024 })
|
|
285
|
+
|
|
286
|
+
// Process lifecycle (see ./lifecycle.ts): `track` (in-flight request
|
|
287
|
+
// drain) and `isShuttingDown` (the new-work gate) are consumed by the
|
|
288
|
+
// HTTP + WS planes below; the teardown is `installLifecycle`d once
|
|
289
|
+
// every server object exists.
|
|
290
|
+
const { track, isShuttingDown, install: installLifecycle } = createLifecycle()
|
|
291
|
+
|
|
292
|
+
// HTTP plane: REST byte-transfer routing + the WS upgrade gate (see
|
|
293
|
+
// ./http.ts). Built after the lifecycle state above because the REST
|
|
294
|
+
// shutdown gate reads `shuttingDown` and the request drain uses
|
|
295
|
+
// `track`. The WS connection handler is wired on `wss` below.
|
|
296
|
+
const httpServer = createHttpServer({
|
|
297
|
+
wss, restDeps: objstoreRestDeps, isOriginAllowed,
|
|
298
|
+
isShuttingDown, track,
|
|
299
|
+
restPutIdleTimeoutMs: REST_PUT_IDLE_TIMEOUT_MS, debug: DEBUG,
|
|
300
|
+
})
|
|
301
|
+
|
|
302
|
+
// WS runtime: per-connection handler + message dispatch + heartbeat
|
|
303
|
+
// sweep (see ./ws-server.ts). Returns the heartbeat timer so shutdown
|
|
304
|
+
// can clear it.
|
|
305
|
+
const { heartbeatTimer } = installWsServer({
|
|
306
|
+
wss, peers, send, unsubscribeAll,
|
|
307
|
+
handleSave, handleSubscribe, handleAuthenticate, sendSaveError, objstore,
|
|
308
|
+
track, isShuttingDown,
|
|
309
|
+
maxInflightPerSocket: MAX_INFLIGHT_PER_SOCKET, heartbeatIntervalMs: HEARTBEAT_INTERVAL_MS,
|
|
310
|
+
debug: DEBUG,
|
|
311
|
+
})
|
|
312
|
+
|
|
313
|
+
httpServer.on('listening', () => {
|
|
314
|
+
// Read the actual bound port from `httpServer.address()` rather
|
|
315
|
+
// than the `PORT` env constant. Operators (and the test harness)
|
|
316
|
+
// can boot with `PORT=0` to get an OS-assigned ephemeral port;
|
|
317
|
+
// the log line then carries the real bound number, not `0`.
|
|
318
|
+
// Server-side bind failure took the error path above, so
|
|
319
|
+
// `address()` is always a populated AddressInfo here.
|
|
320
|
+
const addr = httpServer.address()
|
|
321
|
+
const boundPort = typeof addr === 'object' && addr ? addr.port : PORT
|
|
322
|
+
// Differentiate the storage banner by backend so the log line
|
|
323
|
+
// doesn't claim a misleading DB_PATH under Neon, or a misleading
|
|
324
|
+
// OBJSTORE_DIR under Vercel Blob.
|
|
325
|
+
const dbBanner = NEON_URL ? 'db: neon-postgres' : `db: ${DB_PATH}`
|
|
326
|
+
console.log(`DeepView triage-sync server: ws://${HOST}:${boundPort}${WS_UPGRADE_PATH} http://${HOST}:${boundPort}/api/objstore/{workspaceTag}/{resourceTag} (${dbBanner}, ${objstoreBanner})`)
|
|
327
|
+
})
|
|
328
|
+
|
|
329
|
+
// App-specific shutdown step (run after the in-flight drain), wired
|
|
330
|
+
// into the lifecycle teardown below.
|
|
331
|
+
const closeDb = async (): Promise<void> => {
|
|
332
|
+
// objstoreHandle has no close(): SQLite shares this DatabaseSync and
|
|
333
|
+
// Neon has no persistent connection; `handle.close()` covers both.
|
|
334
|
+
try { await handle.close() } catch (err) { console.warn('DB close error:', errMsg(err)) }
|
|
335
|
+
}
|
|
336
|
+
// Wire graceful shutdown + the signal / error / process-catchall
|
|
337
|
+
// handlers (see ./lifecycle.ts). Installed last, once httpServer, wss,
|
|
338
|
+
// and the heartbeat timer all exist.
|
|
339
|
+
installLifecycle({
|
|
340
|
+
httpServer, wss, heartbeatTimer, stopReaper,
|
|
341
|
+
closeDb,
|
|
342
|
+
})
|
|
343
|
+
|
|
344
|
+
// Bind only after the startup orphan sweep finishes — otherwise a
|
|
345
|
+
// fresh boot could serve traffic against tags whose on-disk state
|
|
346
|
+
// still has residue from a prior crash. Top-level await is fine
|
|
347
|
+
// for an entry-point ESM module (no other module imports this for
|
|
348
|
+
// its exports — the side effect IS the program). `startupReap`
|
|
349
|
+
// already resolves on any error (the reaper's own catch logs the
|
|
350
|
+
// failure unconditionally and returns void), so no outer `.catch`
|
|
351
|
+
// is needed here.
|
|
352
|
+
await startupReap
|
|
353
|
+
httpServer.listen(PORT, HOST)
|