@preventive/triage 1.0.0-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/common/save-error-reason.ts +53 -0
- package/common/utf8.d.ts +13 -0
- package/common/utf8.js +57 -0
- package/out/brotli-fallback.js +3 -0
- package/out/client-sync.js +15 -0
- package/out/graph.js +4 -0
- package/out/icon-maskable.svg +5 -0
- package/out/icon.svg +5 -0
- package/out/index.html +78 -0
- package/out/manifest.webmanifest +30 -0
- package/out/prism.js +14 -0
- package/out/terminal.js +39 -0
- package/out/view.css +1 -0
- package/out/view.html +12 -0
- package/out/view.js +138 -0
- package/package.json +129 -0
- package/server/auth.ts +99 -0
- package/server/config.example.json +3 -0
- package/server/config.ts +196 -0
- package/server/db-neon.ts +374 -0
- package/server/db-revision-sql.ts +152 -0
- package/server/db-stmt.ts +53 -0
- package/server/db.ts +577 -0
- package/server/http.ts +142 -0
- package/server/hub.ts +98 -0
- package/server/index.ts +353 -0
- package/server/lifecycle.ts +177 -0
- package/server/neon-driver.ts +26 -0
- package/server/objstore/blob-fs.ts +164 -0
- package/server/objstore/blob-vercel.ts +508 -0
- package/server/objstore/blob.ts +169 -0
- package/server/objstore/fs.ts +67 -0
- package/server/objstore/handlers.ts +235 -0
- package/server/objstore/init.ts +118 -0
- package/server/objstore/reaper.ts +199 -0
- package/server/objstore/rest.ts +484 -0
- package/server/objstore/sign.ts +164 -0
- package/server/objstore/store-neon.ts +351 -0
- package/server/objstore/store.ts +799 -0
- package/server/objstore/tokens.ts +168 -0
- package/server/origin.ts +68 -0
- package/server/peer.ts +38 -0
- package/server/sign.ts +231 -0
- package/server/static.ts +374 -0
- package/server/sync-handlers.ts +311 -0
- package/server/util.ts +27 -0
- package/server/validation.ts +36 -0
- package/server/ws-server.ts +245 -0
|
@@ -0,0 +1,484 @@
|
|
|
1
|
+
// HTTP plane for the v1.objstore extension.
|
|
2
|
+
//
|
|
3
|
+
// Two routes mounted under `/api/objstore/{workspaceTag}/{resourceTag}`:
|
|
4
|
+
//
|
|
5
|
+
// PUT — body is the raw ciphertext blob; `Authorization: Bearer
|
|
6
|
+
// <put-token>` carries the WS-issued capability that binds
|
|
7
|
+
// (workspaceTag, resourceTag, stagingId, expectedLength).
|
|
8
|
+
// Server streams the body to the staging file, runs
|
|
9
|
+
// `commitPut` (whose version-CAS arbitrates concurrent
|
|
10
|
+
// commits — no lock), broadcasts `objstore-put` to subscribed
|
|
11
|
+
// peers, and replies 200 with `{ version, contentHash }`.
|
|
12
|
+
//
|
|
13
|
+
// GET — `Authorization: Bearer <get-token>` carries a capability
|
|
14
|
+
// bound to (workspaceTag, resourceTag, version). Server
|
|
15
|
+
// opens the live file and pipes it as the response body.
|
|
16
|
+
// A token whose `version` no longer matches the live row
|
|
17
|
+
// (resource was overwritten or deleted post-issuance) is
|
|
18
|
+
// a 404 — the issued capability was for a specific snapshot
|
|
19
|
+
// that no longer exists.
|
|
20
|
+
//
|
|
21
|
+
// Same-origin deployment means no CORS preflight machinery.
|
|
22
|
+
// 4xx responses carry a tiny JSON body so a developer staring at
|
|
23
|
+
// devtools sees something more useful than an empty status code,
|
|
24
|
+
// but the body is not load-bearing for the protocol.
|
|
25
|
+
|
|
26
|
+
import type { IncomingMessage, ServerResponse } from 'node:http'
|
|
27
|
+
import { pipeline } from 'node:stream/promises'
|
|
28
|
+
import { Transform } from 'node:stream'
|
|
29
|
+
import { Buffer } from 'node:buffer'
|
|
30
|
+
import type { WebSocket } from 'ws'
|
|
31
|
+
import {
|
|
32
|
+
type CommitPutResult,
|
|
33
|
+
type Handle,
|
|
34
|
+
MAX_CONTENT_LENGTH,
|
|
35
|
+
abortPut,
|
|
36
|
+
commitPut,
|
|
37
|
+
getLive,
|
|
38
|
+
isValidStagingId,
|
|
39
|
+
isValidTag,
|
|
40
|
+
objectMetaWire,
|
|
41
|
+
} from './store.ts'
|
|
42
|
+
import type { LiveReader } from './blob.ts'
|
|
43
|
+
import { type TokenSecret, extractBearer, verifyToken } from './tokens.ts'
|
|
44
|
+
import { errStack } from '../util.ts'
|
|
45
|
+
|
|
46
|
+
// Server-side fault codes that should surface as 500 `io-error`
|
|
47
|
+
// rather than 400 `aborted`. `pipeline(req, ws)` rejects with the
|
|
48
|
+
// first stream error; the client-side codes (socket close, the
|
|
49
|
+
// manual `overrun`) are everything else. ENOENT is included because
|
|
50
|
+
// `createWriteStream` will reject with it if the `${tag}/.staging`
|
|
51
|
+
// dir was removed out from under us (operator action / external
|
|
52
|
+
// fs activity) — that's a server-side state, not a client-fixable
|
|
53
|
+
// abort. PR #4 review. The Vercel-blob backend surfaces failures
|
|
54
|
+
// through other paths (rejected put promise → caught by the
|
|
55
|
+
// pipeline catch as a plain Error without a `code`); those land in
|
|
56
|
+
// the default `aborted` branch and the operator-facing log line
|
|
57
|
+
// carries the SDK's error message.
|
|
58
|
+
const IO_FAULT_CODES = new Set(['ENOSPC', 'EACCES', 'EROFS', 'EIO', 'EMFILE', 'ENFILE', 'EDQUOT', 'EPERM', 'ENOENT'])
|
|
59
|
+
|
|
60
|
+
export type ObjstoreRestDeps = {
|
|
61
|
+
handle: Handle
|
|
62
|
+
secret: TokenSecret
|
|
63
|
+
broadcast: (tag: string, msg: object, except: WebSocket | null) => void
|
|
64
|
+
debug: boolean
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
// Concurrent commits need no lock at all: the live blob is content-
|
|
68
|
+
// addressed (`${tag}/${contentHash}.bin`) so two racing commits write
|
|
69
|
+
// to DIFFERENT immutable addresses, and the commit itself is an atomic
|
|
70
|
+
// version compare-and-set on the live row (see commitPut in store.ts).
|
|
71
|
+
// Exactly one racer wins the CAS; the loser gets a 409 `conflict` and
|
|
72
|
+
// rebases. This holds within a single process and across replicas —
|
|
73
|
+
// there is no in-process mutex serialising commits. (The PUT *body*
|
|
74
|
+
// does take a per-process single-writer reservation — `inFlightSids`
|
|
75
|
+
// below — but that only rejects a duplicate upload of one staging
|
|
76
|
+
// slot; it never makes distinct commits wait.)
|
|
77
|
+
|
|
78
|
+
// Per-process single-writer guard for the REST PUT body. A put-token is
|
|
79
|
+
// a REUSABLE bearer capability (tokens.ts) valid for its whole TTL, so a
|
|
80
|
+
// client replaying it on overlapping PUTs — a retry that doesn't cancel
|
|
81
|
+
// the in-flight request, a proxy re-issuing the PUT, a double-submit —
|
|
82
|
+
// would otherwise have two requests stream into the SAME staging file
|
|
83
|
+
// (same sid). On the FS backend `createWriteStream(…, { flags: 'w' })`
|
|
84
|
+
// truncates, so the two writers clobber each other's bytes/size and BOTH
|
|
85
|
+
// can fail (size-mismatch 400 + promote-race 500) with neither
|
|
86
|
+
// committing. `inFlightSids` admits exactly ONE in-flight upload per
|
|
87
|
+
// staging id; a concurrent same-token PUT is rejected (409) before it
|
|
88
|
+
// can open a second writer. The slot is held only across the body +
|
|
89
|
+
// commit and released in a `finally` (and bounded by the REST idle-body
|
|
90
|
+
// timeout in server/http.ts so a slow-loris can't pin it). It is NOT a
|
|
91
|
+
// commit lock — commits stay lock-free on the version-CAS above. It's
|
|
92
|
+
// per-process: the Vercel multi-replica path leans on `allowOverwrite:
|
|
93
|
+
// true` + the commit CAS + content-addressing for cross-replica
|
|
94
|
+
// correctness (a duplicate that lands on another replica can't clobber a
|
|
95
|
+
// shared local file and still loses the CAS), not on this set.
|
|
96
|
+
const inFlightSids = new Set<string>()
|
|
97
|
+
|
|
98
|
+
// `/api/objstore/${workspaceTag}/${resourceTag}` — base64url
|
|
99
|
+
// alphabet, case-sensitive. The `?…` query is permitted but ignored
|
|
100
|
+
// (token rides the Authorization header, not the URL — querystring
|
|
101
|
+
// tokens leak via access logs and referer).
|
|
102
|
+
const ROUTE_RE = /^\/api\/objstore\/([\w-]+)\/([\w-]+)(?:\?.*)?$/u
|
|
103
|
+
|
|
104
|
+
export type RouteMatch = { tag: string; resourceTag: string }
|
|
105
|
+
|
|
106
|
+
export function matchRoute(url: string | undefined): RouteMatch | null {
|
|
107
|
+
if (typeof url !== 'string') return null
|
|
108
|
+
const m = ROUTE_RE.exec(url)
|
|
109
|
+
if (!m) return null
|
|
110
|
+
const [, tag, resourceTag] = m
|
|
111
|
+
if (!isValidTag(tag) || !isValidTag(resourceTag)) return null
|
|
112
|
+
return { tag: tag!, resourceTag: resourceTag! }
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
function deny(res: ServerResponse, status: number, body: string): void {
|
|
116
|
+
// Uniform `{ error: <reason> }` JSON envelope for every failure so
|
|
117
|
+
// clients have one shape to parse. Status + reason are NOT
|
|
118
|
+
// intentionally indistinguishable across causes — 401, 404, 405,
|
|
119
|
+
// 410, 411, 500 each map to a documented reason in server/README.md
|
|
120
|
+
// and the client decides recovery from the code. Defense against
|
|
121
|
+
// probe-driven distinguishing isn't a property the relay aims for;
|
|
122
|
+
// every error reason is reachable only after the route + bearer-
|
|
123
|
+
// token check passes (or as 401/404 from the public surface), so
|
|
124
|
+
// there's no signal here a probe couldn't otherwise enumerate.
|
|
125
|
+
res.writeHead(status, { 'content-type': 'application/json' })
|
|
126
|
+
res.end(JSON.stringify({ error: body }))
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
// Variant of `deny` that augments the JSON envelope with the live
|
|
130
|
+
// row's `currentVersion` + `currentIncarnation` so a REST PUT 409 lets
|
|
131
|
+
// the caller rebase onto the right precondition token. Without this the
|
|
132
|
+
// client only learns the slot is occupied — not at what (version,
|
|
133
|
+
// incarnation) — and retries blindly against a non-empty slot, looping
|
|
134
|
+
// indefinitely against a live row. Symmetric with the WS plane's
|
|
135
|
+
// `objstore-conflict` envelope.
|
|
136
|
+
function denyConflict(res: ServerResponse, currentVersion: number | null, currentIncarnation: string | null): void {
|
|
137
|
+
res.writeHead(409, { 'content-type': 'application/json' })
|
|
138
|
+
res.end(JSON.stringify({ error: 'conflict', currentVersion, currentIncarnation }))
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
export async function handleRest(deps: ObjstoreRestDeps, req: IncomingMessage, res: ServerResponse): Promise<void> {
|
|
142
|
+
const route = matchRoute(req.url)
|
|
143
|
+
if (!route) { deny(res, 404, 'not-found'); return }
|
|
144
|
+
const token = extractBearer(req.headers['authorization'])
|
|
145
|
+
if (!token) { deny(res, 401, 'unauthorized'); return }
|
|
146
|
+
const payload = verifyToken(deps.secret, token)
|
|
147
|
+
if (!payload) { deny(res, 401, 'unauthorized'); return }
|
|
148
|
+
if (payload.tag !== route.tag || payload.res !== route.resourceTag) {
|
|
149
|
+
deny(res, 401, 'unauthorized'); return
|
|
150
|
+
}
|
|
151
|
+
// Defense in depth: `verifyToken` only checked `typeof sid ===
|
|
152
|
+
// 'string'`. A path-bearing sid (`../../etc/passwd`) inside an
|
|
153
|
+
// HMAC-valid payload would otherwise reach `stagingFilePath` and
|
|
154
|
+
// escape `OBJSTORE_DIR`. The HMAC binds the payload bytes, so this
|
|
155
|
+
// is reachable only under a secret compromise — match the
|
|
156
|
+
// reaper's row-field validation (PR #4 review #15) at the wire
|
|
157
|
+
// boundary too.
|
|
158
|
+
if (payload.op === 'put' && !isValidStagingId(payload.sid)) {
|
|
159
|
+
deny(res, 401, 'unauthorized'); return
|
|
160
|
+
}
|
|
161
|
+
if (req.method === 'PUT' && payload.op === 'put') {
|
|
162
|
+
try { await handleRestPut(deps, req, res, route, payload) }
|
|
163
|
+
catch (err: unknown) {
|
|
164
|
+
// Outer catch is the forensic safety net — handleRestPut has
|
|
165
|
+
// its own internal pipeline catch. Log `.stack` so a post-
|
|
166
|
+
// mortem has the throw site.
|
|
167
|
+
if (deps.debug) console.warn('objstore PUT error:', errStack(err))
|
|
168
|
+
if (res.headersSent) res.destroy()
|
|
169
|
+
else deny(res, 500, 'internal')
|
|
170
|
+
}
|
|
171
|
+
return
|
|
172
|
+
}
|
|
173
|
+
if (req.method === 'GET' && payload.op === 'get') {
|
|
174
|
+
try { await handleRestGet(deps, res, route, payload) }
|
|
175
|
+
catch (err: unknown) {
|
|
176
|
+
if (deps.debug) console.warn('objstore GET error:', errStack(err))
|
|
177
|
+
if (res.headersSent) res.destroy()
|
|
178
|
+
else deny(res, 500, 'internal')
|
|
179
|
+
}
|
|
180
|
+
return
|
|
181
|
+
}
|
|
182
|
+
deny(res, 405, 'method-not-allowed')
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
async function handleRestPut(
|
|
186
|
+
deps: ObjstoreRestDeps,
|
|
187
|
+
req: IncomingMessage,
|
|
188
|
+
res: ServerResponse,
|
|
189
|
+
route: RouteMatch,
|
|
190
|
+
payload: { sid: string; len: number },
|
|
191
|
+
): Promise<void> {
|
|
192
|
+
// Require an explicit Content-Length so a mismatch is rejected
|
|
193
|
+
// before opening the file. Chunked transfer-encoding without a
|
|
194
|
+
// length header is rejected — the client always knows the size
|
|
195
|
+
// up front (signed into put-begin).
|
|
196
|
+
const lenHeader = req.headers['content-length']
|
|
197
|
+
const declared = typeof lenHeader === 'string' ? Number(lenHeader) : NaN
|
|
198
|
+
if (!Number.isSafeInteger(declared) || declared < 0 || declared > MAX_CONTENT_LENGTH) {
|
|
199
|
+
deny(res, 411, 'length-required'); return
|
|
200
|
+
}
|
|
201
|
+
if (declared !== payload.len) { deny(res, 400, 'length-mismatch'); return }
|
|
202
|
+
// No lock: the commit's version-CAS arbitrates concurrent commits
|
|
203
|
+
// (the live blob is content-addressed, so racers can't desync
|
|
204
|
+
// metadata vs bytes — the loser surfaces a 409 `conflict`). The
|
|
205
|
+
// staging row is protected from the reaper not by a lock but by its
|
|
206
|
+
// `begun_at`: an upload under the staging TTL stays fresh through the
|
|
207
|
+
// body, and the after-body `refreshStagingBegunAt` re-extends the
|
|
208
|
+
// TTL across the commit. (An upload exceeding the TTL during the
|
|
209
|
+
// body can be reaped mid-flight → commit 410s; documented accepted
|
|
210
|
+
// tradeoff — see commitPut in store.ts.)
|
|
211
|
+
await handleRestPutBody(deps, req, res, route, payload, declared)
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
// Map a `commitPut` failure to its wire response. Extracted from
|
|
215
|
+
// `handleRestPutBody` to keep it under the per-function line cap
|
|
216
|
+
// and to make the wire-mapping ladder its own audit surface — the
|
|
217
|
+
// exhaustiveness `never` guard at the bottom catches a forward-
|
|
218
|
+
// compat hazard where a new `CommitPutResult` reason lands without
|
|
219
|
+
// updating this dispatch.
|
|
220
|
+
function denyCommitFailure(res: ServerResponse, result: Exclude<CommitPutResult, { ok: true }>): void {
|
|
221
|
+
if (result.reason === 'conflict') {
|
|
222
|
+
denyConflict(res, result.conflict?.version ?? null, result.conflict?.incarnation ?? null)
|
|
223
|
+
return
|
|
224
|
+
}
|
|
225
|
+
if (result.reason === 'no-staging') { deny(res, 410, 'gone'); return }
|
|
226
|
+
// `io-error` = FS/disk fault (EACCES/ENOSPC/EIO/racing abort);
|
|
227
|
+
// server-side, not client-fixable. `size-mismatch` is the
|
|
228
|
+
// remaining client-data fault — wire-rename to the documented
|
|
229
|
+
// `length-mismatch` shape so the README enumeration stays
|
|
230
|
+
// exhaustive.
|
|
231
|
+
if (result.reason === 'io-error') { deny(res, 500, 'io-error'); return }
|
|
232
|
+
if (result.reason === 'size-mismatch') { deny(res, 400, 'length-mismatch'); return }
|
|
233
|
+
// Exhaustiveness guard — a new `CommitPutResult` reason added
|
|
234
|
+
// without updating this ladder trips the `never` cast at compile
|
|
235
|
+
// time. Forward-compat hazard called out in audit round-10.
|
|
236
|
+
const _exhaustive: never = result.reason
|
|
237
|
+
void _exhaustive
|
|
238
|
+
deny(res, 500, 'internal')
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
async function handleRestPutBody(
|
|
242
|
+
deps: ObjstoreRestDeps,
|
|
243
|
+
req: IncomingMessage,
|
|
244
|
+
res: ServerResponse,
|
|
245
|
+
route: RouteMatch,
|
|
246
|
+
payload: { sid: string; len: number },
|
|
247
|
+
declared: number,
|
|
248
|
+
): Promise<void> {
|
|
249
|
+
// Cheap staging-row precheck — bail before accepting up to
|
|
250
|
+
// MAX_CONTENT_LENGTH bytes for a replayed / expired token. The
|
|
251
|
+
// row could still vanish before commit (reaper / abort) but the
|
|
252
|
+
// commit recheck catches that. PR #4 review.
|
|
253
|
+
if (!await deps.handle.selectStaging.get(route.tag, route.resourceTag, payload.sid)) {
|
|
254
|
+
deny(res, 410, 'gone'); return
|
|
255
|
+
}
|
|
256
|
+
// Single-writer guard (see `inFlightSids`): reject a concurrent PUT
|
|
257
|
+
// replaying this same token before it can open a second writer on the
|
|
258
|
+
// shared staging file. `has` + `add` run with no `await` between them,
|
|
259
|
+
// so they're atomic on Node's single thread — exactly one concurrent
|
|
260
|
+
// PUT passes. The 409 echoes the live version like a commit-time
|
|
261
|
+
// conflict so the client rebases / re-handshakes.
|
|
262
|
+
if (inFlightSids.has(payload.sid)) {
|
|
263
|
+
const cur = await getLive(deps.handle, route.tag, route.resourceTag)
|
|
264
|
+
denyConflict(res, cur?.version ?? null, cur?.incarnation ?? null)
|
|
265
|
+
return
|
|
266
|
+
}
|
|
267
|
+
inFlightSids.add(payload.sid)
|
|
268
|
+
try {
|
|
269
|
+
// Stream the upload + commit, no lock. The commit's version-CAS
|
|
270
|
+
// arbitrates concurrent commits; the staging row is kept fresh for
|
|
271
|
+
// the reaper by its `begun_at` (+ the after-body refresh), not by a
|
|
272
|
+
// mutex. See the rationale in handleRestPut above.
|
|
273
|
+
const result = await runUploadAndCommit(deps, route, payload, declared, req, res)
|
|
274
|
+
if (result.handled) return
|
|
275
|
+
if (!result.commit.ok) { denyCommitFailure(res, result.commit); return }
|
|
276
|
+
const row = result.commit.row
|
|
277
|
+
res.writeHead(200, { 'content-type': 'application/json' })
|
|
278
|
+
res.end(JSON.stringify({
|
|
279
|
+
version: row.version,
|
|
280
|
+
incarnation: row.incarnation,
|
|
281
|
+
contentHash: row.contentHash,
|
|
282
|
+
contentLength: row.contentLength,
|
|
283
|
+
}))
|
|
284
|
+
deps.broadcast(route.tag, {
|
|
285
|
+
type: 'objstore-put',
|
|
286
|
+
workspaceTag: route.tag,
|
|
287
|
+
...objectMetaWire(row),
|
|
288
|
+
}, null)
|
|
289
|
+
if (deps.debug) console.log(`objstore put → ${route.tag.slice(0, 12)}…/${route.resourceTag.slice(0, 8)}… v${row.version}`)
|
|
290
|
+
} finally {
|
|
291
|
+
// Release the slot on every exit (success, commit failure, pipeline
|
|
292
|
+
// error, idle-timeout abort) so a later legitimate retry isn't
|
|
293
|
+
// wrongly rejected.
|
|
294
|
+
inFlightSids.delete(payload.sid)
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
// The body of a PUT: stream the upload into the staging slot, verify
|
|
299
|
+
// size, refresh the staging row's begun_at, and commit. No lock — the
|
|
300
|
+
// commit's version-CAS arbitrates concurrent commits. Returns
|
|
301
|
+
// `{ handled: true }` when it already wrote the (error) response
|
|
302
|
+
// itself; `{ handled: false, commit }` when it reached commitPut
|
|
303
|
+
// (caller maps the result to the success / failure response, keeping
|
|
304
|
+
// the broadcast + body write out of this helper).
|
|
305
|
+
async function runUploadAndCommit(
|
|
306
|
+
deps: ObjstoreRestDeps,
|
|
307
|
+
route: RouteMatch,
|
|
308
|
+
payload: { sid: string; len: number },
|
|
309
|
+
declared: number,
|
|
310
|
+
req: IncomingMessage,
|
|
311
|
+
res: ServerResponse,
|
|
312
|
+
): Promise<{ handled: true } | { handled: false; commit: CommitPutResult }> {
|
|
313
|
+
// Open the backend's staging writer. For the FS backend this is a
|
|
314
|
+
// `createWriteStream` to the canonical staging path; for the
|
|
315
|
+
// Vercel backend it's a PassThrough whose other end feeds the
|
|
316
|
+
// SDK's `put`. Both surfaces are awaited via `finalize()` after
|
|
317
|
+
// the pipeline below.
|
|
318
|
+
const writer = await deps.handle.blob.openStagingWriter(route.tag, payload.sid)
|
|
319
|
+
// Byte counter as a Transform in the pipeline. EventEmitter delivers
|
|
320
|
+
// chunks to every attached listener, so `req.on('data', count)` +
|
|
321
|
+
// `pipeline(req, ws)` would also work — but the dual-listener
|
|
322
|
+
// pattern is brittle: a future stream-semantics change (or an
|
|
323
|
+
// upstream resuming synchronously on listener attach) could let
|
|
324
|
+
// chunks bypass the pipe. Sitting in the pipeline removes the
|
|
325
|
+
// ambiguity — we count exactly what gets written. Overrun aborts
|
|
326
|
+
// by callback-error, which pipeline propagates to tear down req +
|
|
327
|
+
// ws together (no manual `req.destroy()` / `ws.destroy()` dance).
|
|
328
|
+
//
|
|
329
|
+
// Defensive overrun cap — Node truncates at Content-Length, but a
|
|
330
|
+
// buggy proxy / hostile client sending more shouldn't end up on
|
|
331
|
+
// disk.
|
|
332
|
+
let received = 0
|
|
333
|
+
const counter = new Transform({
|
|
334
|
+
transform(chunk: Buffer, _encoding, cb): void {
|
|
335
|
+
received += chunk.byteLength
|
|
336
|
+
if (received > declared) cb(new Error('overrun'))
|
|
337
|
+
else cb(null, chunk)
|
|
338
|
+
},
|
|
339
|
+
})
|
|
340
|
+
try {
|
|
341
|
+
await pipeline(req, counter, writer.writable)
|
|
342
|
+
// For the FS backend `finalize` is a no-op (pipeline already
|
|
343
|
+
// awaited the WriteStream's 'finish'); for the Vercel backend
|
|
344
|
+
// it awaits the SDK's put-promise so we know the bytes are
|
|
345
|
+
// durable at the remote before commitPut runs its size check.
|
|
346
|
+
await writer.finalize()
|
|
347
|
+
} catch (err) {
|
|
348
|
+
// Await writer.abort() so a Vercel-backed upload's in-flight
|
|
349
|
+
// HTTP request has time to settle (rejected with
|
|
350
|
+
// BlobRequestAbortedError) BEFORE abortPut → unlinkStaging runs.
|
|
351
|
+
// Otherwise a late-arriving upload chunk recreates the staging
|
|
352
|
+
// blob after we've cleaned it. FS backend's abort is an immediate
|
|
353
|
+
// microtask — no real wait.
|
|
354
|
+
await writer.abort(err)
|
|
355
|
+
await abortPut(deps.handle, route.tag, route.resourceTag, payload.sid)
|
|
356
|
+
// Branch on `err.code` so a write-side fault (ENOSPC / EACCES /
|
|
357
|
+
// EIO …) surfaces as a 5xx per the README contract, separate
|
|
358
|
+
// from a client-side abort / overrun which stays 400. PR #4
|
|
359
|
+
// review. Vercel-blob upload failures land here without a
|
|
360
|
+
// `code` field and route through the 400 `aborted` branch; the
|
|
361
|
+
// DEBUG=1 log line carries the SDK's error.message for
|
|
362
|
+
// operator triage.
|
|
363
|
+
const code = (err as NodeJS.ErrnoException)?.code
|
|
364
|
+
if (code !== undefined && IO_FAULT_CODES.has(code)) deny(res, 500, 'io-error')
|
|
365
|
+
else deny(res, 400, 'aborted')
|
|
366
|
+
// Log `code` only; Node `fs` error messages interpolate the
|
|
367
|
+
// full path, which includes the raw (un-truncated) workspaceTag
|
|
368
|
+
// (= Ed25519 public key). Operator logs shouldn't carry it
|
|
369
|
+
// verbatim — `debugTag` is the convention everywhere else. If
|
|
370
|
+
// `code` is missing (non-errno throw), log a placeholder so the
|
|
371
|
+
// count is still visible at DEBUG=1.
|
|
372
|
+
if (deps.debug) console.warn('objstore PUT aborted mid-body:', code ?? '<no-code>')
|
|
373
|
+
return { handled: true }
|
|
374
|
+
}
|
|
375
|
+
if (received !== declared) {
|
|
376
|
+
await abortPut(deps.handle, route.tag, route.resourceTag, payload.sid)
|
|
377
|
+
deny(res, 400, 'length-mismatch'); return { handled: true }
|
|
378
|
+
}
|
|
379
|
+
// Belt-and-braces: confirm storage-side size (catches a writer
|
|
380
|
+
// that silently absorbed less, e.g. ENOSPC near the end on FS,
|
|
381
|
+
// or a partial multipart upload that the SDK didn't propagate as
|
|
382
|
+
// a reject). A null result here means the staging slot is gone
|
|
383
|
+
// entirely (racing reaper / abort), which is an FS-side / backend
|
|
384
|
+
// fault — wire string is `io-error`, mapped to HTTP 500, matching
|
|
385
|
+
// the README contract.
|
|
386
|
+
let onDisk: number | null
|
|
387
|
+
try { onDisk = await deps.handle.blob.statStaging(route.tag, payload.sid) } catch {
|
|
388
|
+
await abortPut(deps.handle, route.tag, route.resourceTag, payload.sid)
|
|
389
|
+
deny(res, 500, 'io-error'); return { handled: true }
|
|
390
|
+
}
|
|
391
|
+
if (onDisk == null) {
|
|
392
|
+
await abortPut(deps.handle, route.tag, route.resourceTag, payload.sid)
|
|
393
|
+
deny(res, 500, 'io-error'); return { handled: true }
|
|
394
|
+
}
|
|
395
|
+
if (onDisk !== payload.len) {
|
|
396
|
+
await abortPut(deps.handle, route.tag, route.resourceTag, payload.sid)
|
|
397
|
+
deny(res, 400, 'length-mismatch'); return { handled: true }
|
|
398
|
+
}
|
|
399
|
+
// Refresh begun_at AFTER the body lands so the TTL effectively
|
|
400
|
+
// counts from upload-done. This is what keeps the reaper's atomic
|
|
401
|
+
// conditional delete (`deleteStagingIfStale`, predicate
|
|
402
|
+
// `begun_at < staleBefore`) from matching this row at the commit
|
|
403
|
+
// step: a sub-TTL upload's begun_at is bumped fresh here, well
|
|
404
|
+
// inside the window, so a concurrent reaper sweep can't drop it.
|
|
405
|
+
// (No lock — the conditional delete IS the F1 protection now.)
|
|
406
|
+
await deps.handle.refreshStagingBegunAt.run(Date.now(), route.tag, route.resourceTag, payload.sid)
|
|
407
|
+
// Commit: precondition recheck + content-addressed promote + version
|
|
408
|
+
// CAS (the CAS arbitrates concurrent commits — no lock). Thread the
|
|
409
|
+
// post-upload `onDisk` size as `observedSize` so commitPut skips its
|
|
410
|
+
// own redundant statStaging round-trip (one fewer Vercel HEAD per
|
|
411
|
+
// PUT) — safe because staging ids are random, so nothing else writes
|
|
412
|
+
// this blob, and the upload pipeline already finished above.
|
|
413
|
+
const commit = await commitPut(deps.handle, {
|
|
414
|
+
workspaceTag: route.tag, resourceTag: route.resourceTag, stagingId: payload.sid,
|
|
415
|
+
observedSize: onDisk,
|
|
416
|
+
})
|
|
417
|
+
if (!commit.ok) await abortPut(deps.handle, route.tag, route.resourceTag, payload.sid)
|
|
418
|
+
return { handled: false, commit }
|
|
419
|
+
}
|
|
420
|
+
|
|
421
|
+
type GetOpened =
|
|
422
|
+
| { reason: 'ok'; reader: LiveReader }
|
|
423
|
+
| { reason: 'not-found' }
|
|
424
|
+
| { reason: 'unavailable' }
|
|
425
|
+
|
|
426
|
+
async function openLiveSnapshot(
|
|
427
|
+
deps: ObjstoreRestDeps, route: RouteMatch, payload: { ver: number; inc: string },
|
|
428
|
+
): Promise<GetOpened> {
|
|
429
|
+
// Validate row version + open the reader (by the row's content hash).
|
|
430
|
+
// No lock is needed because the live blob is CONTENT-ADDRESSED and
|
|
431
|
+
// therefore IMMUTABLE: a concurrent re-upload writes a DIFFERENT hash
|
|
432
|
+
// (a new address), leaving the hash this row names untouched. So the
|
|
433
|
+
// bytes behind `live.content_hash` can never change underneath us —
|
|
434
|
+
// the worst a race can do is have the reaper GC an already-superseded
|
|
435
|
+
// hash just before we open it, which surfaces as openLiveReader
|
|
436
|
+
// not-found → `unavailable` → 503, and the client refetches. We can
|
|
437
|
+
// never serve torn or wrong bytes. (For the FS backend the open also
|
|
438
|
+
// returns a pinned fd; for the Vercel backend a fetch-backed stream.)
|
|
439
|
+
const live = await deps.handle.selectLiveOne.get(route.tag, route.resourceTag)
|
|
440
|
+
if (!live || live.version !== payload.ver || live.incarnation !== payload.inc) return { reason: 'not-found' }
|
|
441
|
+
let opened
|
|
442
|
+
try { opened = await deps.handle.blob.openLiveReader(route.tag, live.content_hash) }
|
|
443
|
+
catch { return { reason: 'unavailable' } }
|
|
444
|
+
if (!opened.ok) return { reason: opened.reason }
|
|
445
|
+
// Size mismatch between the live row and the on-storage bytes
|
|
446
|
+
// is a transient inconsistency — reaper will reconcile. Close
|
|
447
|
+
// the reader before returning so we don't leak the fd / fetch
|
|
448
|
+
// reader. PR #4 review H8.
|
|
449
|
+
if (opened.reader.size !== live.content_length) {
|
|
450
|
+
await opened.reader.close().catch(() => {})
|
|
451
|
+
return { reason: 'unavailable' }
|
|
452
|
+
}
|
|
453
|
+
return { reason: 'ok', reader: opened.reader }
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
async function handleRestGet(
|
|
457
|
+
deps: ObjstoreRestDeps,
|
|
458
|
+
res: ServerResponse,
|
|
459
|
+
route: RouteMatch,
|
|
460
|
+
payload: { ver: number; inc: string },
|
|
461
|
+
): Promise<void> {
|
|
462
|
+
// Token's `ver` is the live row's version at issuance. A later
|
|
463
|
+
// PUT/DELETE invalidates the capability — new version (or missing
|
|
464
|
+
// row) means this snapshot is gone. 404 keeps the response shape
|
|
465
|
+
// uniform with "never existed" so a probe can't distinguish.
|
|
466
|
+
const opened = await openLiveSnapshot(deps, route, payload)
|
|
467
|
+
if (opened.reason === 'not-found') { deny(res, 404, 'not-found'); return }
|
|
468
|
+
// If the live row is there but the bytes are missing / wrong size,
|
|
469
|
+
// it's a transient inconsistency the reaper will sort out — 503
|
|
470
|
+
// (vs 404) tells the client this is a server-side state, not a
|
|
471
|
+
// "the resource truly isn't there" answer.
|
|
472
|
+
if (opened.reason === 'unavailable') { deny(res, 503, 'unavailable'); return }
|
|
473
|
+
res.writeHead(200, {
|
|
474
|
+
'content-type': 'application/octet-stream',
|
|
475
|
+
'content-length': String(opened.reader.size),
|
|
476
|
+
})
|
|
477
|
+
// `pipeline` (vs `stream.pipe(res)`) destroys the source when the
|
|
478
|
+
// destination errors — a client that aborts mid-download would
|
|
479
|
+
// otherwise leak the read-stream fd / fetch reader until GC. The
|
|
480
|
+
// backend's reader auto-closes its underlying resource when the
|
|
481
|
+
// stream finishes or is destroyed; `pipeline` rejecting bubbles
|
|
482
|
+
// to handleRest's outer catch for `res.destroy()`.
|
|
483
|
+
await pipeline(opened.reader.stream, res)
|
|
484
|
+
}
|
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
// Canonical bytes + Ed25519 verifiers for the v1.objstore extension.
|
|
2
|
+
// Domain prefixes are distinct from triage-sync's so a captured save
|
|
3
|
+
// / subscribe signature can't be replayed as a PUT / DELETE / LIST
|
|
4
|
+
// / FETCH and vice versa.
|
|
5
|
+
|
|
6
|
+
import { encodeUtf8 } from '../../common/utf8.js'
|
|
7
|
+
import { verifyEd25519 } from '../sign.ts'
|
|
8
|
+
import { isValidIncarnation } from './store.ts'
|
|
9
|
+
|
|
10
|
+
const OBJSTORE_PUT_DOMAIN = 'deepview-objstore.v1.put'
|
|
11
|
+
const OBJSTORE_DELETE_DOMAIN = 'deepview-objstore.v1.delete'
|
|
12
|
+
const OBJSTORE_FETCH_DOMAIN = 'deepview-objstore.v1.fetch'
|
|
13
|
+
|
|
14
|
+
// Wire shapes the verifiers accept. Fields land here post-
|
|
15
|
+
// `JSON.parse`, so every value starts life as `unknown` — strict
|
|
16
|
+
// type checks inside each verifier are the trust boundary.
|
|
17
|
+
export type ObjstorePutBeginMsg = {
|
|
18
|
+
workspaceTag?: unknown
|
|
19
|
+
resourceTag?: unknown
|
|
20
|
+
prevVersion?: unknown
|
|
21
|
+
prevIncarnation?: unknown
|
|
22
|
+
expectedLength?: unknown
|
|
23
|
+
contentHash?: unknown
|
|
24
|
+
signature?: unknown
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export type ObjstoreDeleteMsg = {
|
|
28
|
+
workspaceTag?: unknown
|
|
29
|
+
resourceTag?: unknown
|
|
30
|
+
prevVersion?: unknown
|
|
31
|
+
prevIncarnation?: unknown
|
|
32
|
+
signature?: unknown
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export type ObjstoreFetchMsg = {
|
|
36
|
+
workspaceTag?: unknown
|
|
37
|
+
resourceTag?: unknown
|
|
38
|
+
signature?: unknown
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function intOrEmpty(v: unknown): string {
|
|
42
|
+
return v == null ? '' : String(v)
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
// `prevIncarnation` → '' when null, the base64url id otherwise. The
|
|
46
|
+
// client mirror (`incOrEmpty`) passes the string through verbatim;
|
|
47
|
+
// `String(v)` here is a no-op on the validated string and matches it
|
|
48
|
+
// byte-for-byte.
|
|
49
|
+
function strOrEmpty(v: unknown): string {
|
|
50
|
+
return v == null ? '' : String(v)
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
// Newline-joined fields after the domain prefix — same construction
|
|
54
|
+
// as triage-sync's canonicalSave. Newlines can't appear in base64url
|
|
55
|
+
// tokens or in the bare integer fields, so framing is unambiguous
|
|
56
|
+
// without length-prefixes. `prevVersion` is `''` when null, decimal
|
|
57
|
+
// otherwise — matches the server's storage convention.
|
|
58
|
+
//
|
|
59
|
+
// EVERY canonical (put / delete / list / fetch) binds the per-
|
|
60
|
+
// connection challenge nonce so a captured frame can't be replayed
|
|
61
|
+
// across connections. Without this, a passive observer of past wire
|
|
62
|
+
// traffic could replay a `objstore-delete` whenever the live version
|
|
63
|
+
// happens to match (versions restart at 1 after each delete, so
|
|
64
|
+
// `prevVersion=1` is a recurring alignment window). PR #4 review H2.
|
|
65
|
+
function canonicalObjstorePut(msg: ObjstorePutBeginMsg, connectionNonce: string): Uint8Array<ArrayBuffer> {
|
|
66
|
+
return encodeUtf8([
|
|
67
|
+
OBJSTORE_PUT_DOMAIN,
|
|
68
|
+
msg.workspaceTag as string,
|
|
69
|
+
msg.resourceTag as string,
|
|
70
|
+
intOrEmpty(msg.prevVersion),
|
|
71
|
+
strOrEmpty(msg.prevIncarnation),
|
|
72
|
+
msg.contentHash as string,
|
|
73
|
+
String(msg.expectedLength),
|
|
74
|
+
connectionNonce,
|
|
75
|
+
].join('\n'))
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
function canonicalObjstoreDelete(msg: ObjstoreDeleteMsg, connectionNonce: string): Uint8Array<ArrayBuffer> {
|
|
79
|
+
return encodeUtf8([
|
|
80
|
+
OBJSTORE_DELETE_DOMAIN,
|
|
81
|
+
msg.workspaceTag as string,
|
|
82
|
+
msg.resourceTag as string,
|
|
83
|
+
intOrEmpty(msg.prevVersion),
|
|
84
|
+
strOrEmpty(msg.prevIncarnation),
|
|
85
|
+
connectionNonce,
|
|
86
|
+
].join('\n'))
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
function canonicalObjstoreFetch(msg: ObjstoreFetchMsg, connectionNonce: string): Uint8Array<ArrayBuffer> {
|
|
90
|
+
return encodeUtf8([
|
|
91
|
+
OBJSTORE_FETCH_DOMAIN,
|
|
92
|
+
msg.workspaceTag as string,
|
|
93
|
+
msg.resourceTag as string,
|
|
94
|
+
connectionNonce,
|
|
95
|
+
].join('\n'))
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
// `Number.isSafeInteger` rather than `Number.isInteger`: JSON numbers
|
|
99
|
+
// are IEEE-754 and integers above 2^53-1 aren't precisely
|
|
100
|
+
// representable. Accepting non-safe integers would let a signed
|
|
101
|
+
// `expectedLength = 9_007_199_254_740_993` round-trip to a different
|
|
102
|
+
// value on receivers, fail size comparisons silently, and cascade
|
|
103
|
+
// into mismatched-canonical-bytes / failed verifies. PR #4 review.
|
|
104
|
+
function isSafeNonNegativeInt(v: unknown): v is number {
|
|
105
|
+
return typeof v === 'number' && Number.isSafeInteger(v) && v >= 0
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
function isSafeIntOrNull(v: unknown): boolean {
|
|
109
|
+
return v == null || (typeof v === 'number' && Number.isSafeInteger(v))
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
// Shared tail for the four verifiers. The universal trust-boundary
|
|
113
|
+
// checks (workspaceTag / signature / connectionNonce must be strings)
|
|
114
|
+
// live here; each verifier adds only its own message-specific field
|
|
115
|
+
// gates before delegating. `build` is a thunk over the already-
|
|
116
|
+
// validated message + the (now-narrowed) nonce — a throw from it
|
|
117
|
+
// (lone surrogate, etc.) is treated as a verify failure, never an
|
|
118
|
+
// escaping exception. Centralising the build→verify step keeps the
|
|
119
|
+
// four message types from drifting on the canonical-bytes / Ed25519
|
|
120
|
+
// plumbing.
|
|
121
|
+
async function verifyObjstoreSig(
|
|
122
|
+
msg: { workspaceTag?: unknown; signature?: unknown },
|
|
123
|
+
connectionNonce: unknown,
|
|
124
|
+
build: (connectionNonce: string) => Uint8Array<ArrayBuffer>,
|
|
125
|
+
): Promise<boolean> {
|
|
126
|
+
if (typeof msg.workspaceTag !== 'string') return false
|
|
127
|
+
if (typeof msg.signature !== 'string') return false
|
|
128
|
+
if (typeof connectionNonce !== 'string') return false
|
|
129
|
+
let payload: Uint8Array<ArrayBuffer>
|
|
130
|
+
try { payload = build(connectionNonce) } catch { return false }
|
|
131
|
+
return await verifyEd25519(msg.workspaceTag, payload, msg.signature)
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
// prevIncarnation must travel as an inseparable pair with prevVersion:
|
|
135
|
+
// a numeric prevVersion carries a valid base64url incarnation id; a
|
|
136
|
+
// null prevVersion carries null. Reject the mixed combos (a half-pair)
|
|
137
|
+
// so a forged or stale precondition can't slip a version match past the
|
|
138
|
+
// CAS without the matching incarnation. The shape gate mirrors the
|
|
139
|
+
// staging-id check — a malformed id can't reach the CAS predicate.
|
|
140
|
+
function validPrevPair(prevVersion: unknown, prevIncarnation: unknown): boolean {
|
|
141
|
+
if (prevVersion == null) return prevIncarnation == null
|
|
142
|
+
return isValidIncarnation(prevIncarnation)
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
export function verifyObjstorePutSig(msg: ObjstorePutBeginMsg, connectionNonce: unknown): Promise<boolean> {
|
|
146
|
+
if (typeof msg.resourceTag !== 'string') return Promise.resolve(false)
|
|
147
|
+
if (typeof msg.contentHash !== 'string') return Promise.resolve(false)
|
|
148
|
+
if (!isSafeIntOrNull(msg.prevVersion)) return Promise.resolve(false)
|
|
149
|
+
if (!validPrevPair(msg.prevVersion, msg.prevIncarnation)) return Promise.resolve(false)
|
|
150
|
+
if (!isSafeNonNegativeInt(msg.expectedLength)) return Promise.resolve(false)
|
|
151
|
+
return verifyObjstoreSig(msg, connectionNonce, (nonce) => canonicalObjstorePut(msg, nonce))
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
export function verifyObjstoreDeleteSig(msg: ObjstoreDeleteMsg, connectionNonce: unknown): Promise<boolean> {
|
|
155
|
+
if (typeof msg.resourceTag !== 'string') return Promise.resolve(false)
|
|
156
|
+
if (!isSafeIntOrNull(msg.prevVersion)) return Promise.resolve(false)
|
|
157
|
+
if (!validPrevPair(msg.prevVersion, msg.prevIncarnation)) return Promise.resolve(false)
|
|
158
|
+
return verifyObjstoreSig(msg, connectionNonce, (nonce) => canonicalObjstoreDelete(msg, nonce))
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
export function verifyObjstoreFetchSig(msg: ObjstoreFetchMsg, connectionNonce: unknown): Promise<boolean> {
|
|
162
|
+
if (typeof msg.resourceTag !== 'string') return Promise.resolve(false)
|
|
163
|
+
return verifyObjstoreSig(msg, connectionNonce, (nonce) => canonicalObjstoreFetch(msg, nonce))
|
|
164
|
+
}
|