@preventive/triage 1.0.0-alpha.5 → 1.0.0-alpha.50

package/{server → server-e2e}/objstore/init.ts

@@ -15,12 +15,12 @@ import { errStack } from '../util.ts'
 
 export type ObjstoreInitDeps = {
   // Pre-opened storage handle, sharing the SQLite connection with
-  // the workspace_revision handle in server/db.ts.
+  // the workspace_revision handle in server-e2e/db.ts.
   handle: Handle
   reapIntervalMs: number
-  // Hard off-switch (OBJSTORE_REAP_DISABLED). When true, NEITHER the boot
+  // Automatic-sweep switch (OBJSTORE_REAP_DISABLED). When true, NEITHER the boot
   // sweep nor the periodic timer runs: `startupReap` resolves immediately
-  // and `stopReaper` is a no-op. Orphaned/superseded blobs and stale
+  // and no timer is installed. Orphaned/superseded blobs and stale
   // staging rows then accumulate unbounded — only safe if an external job
   // handles GC. Omitted/false → reaper runs (the default). See config.ts.
   reapDisabled?: boolean
@@ -28,9 +28,9 @@ export type ObjstoreInitDeps = {
   broadcast: (tag: string, msg: object, except: WebSocket | null) => void
   // Cross-instance pub/sub publishers. SQLite mode passes no-ops; Neon
   // mode passes Postgres LISTEN/NOTIFY-backed implementations. See
-  // server/pubsub.ts for the bus design.
+  // server-e2e/pubsub.ts for the bus design.
   publishObjPut: (tag: string, resourceTag: string) => void
-  publishObjDeleted: (tag: string, resourceTag: string, version: number) => void
+  publishObjDeleted: (tag: string, resourceTag: string, version: number, incarnation: string) => void
   getNonce: (socket: WebSocket) => string | undefined
   debug: boolean
   // Auth gate for the FIRST objstore-put-begin against a workspace
@@ -45,7 +45,7 @@ export type ObjstoreInitDeps = {
   // process behaviour). Multi-replica deployments MUST supply a
   // shared secret here so a token minted on one replica's WS plane
   // validates on another replica's REST plane (the WS-to-REST hop
-  // is not load-balancer-pinned). See server/index.ts boot logic
+  // is not load-balancer-pinned). See server-e2e/index.ts boot logic
   // for the env var (`OBJSTORE_TOKEN_SECRET`).
   tokenSecret?: TokenSecret
   // New-workspace operator gate for the REST put-begin mint — returns
@@ -59,6 +59,7 @@ export type ObjstoreInitDeps = {
 export type ObjstoreInit = {
   handlers: ObjstoreHandlers
   restDeps: ObjstoreRestDeps
+  reap: () => Promise<void>
   startupReap: Promise<void>
   // Cancels the periodic timer AND resolves only after any
   // currently-in-flight sweep has finished, so shutdown can safely
@@ -102,31 +103,22 @@ export function initObjstore(deps: ObjstoreInitDeps): ObjstoreInit {
   let inFlight: Promise<void> | null = null
   function enqueueSweep(): Promise<void> {
     if (inFlight) return inFlight
-    // Log reaper failures unconditionally (not gated on `debug`) — a
-    // failed sweep means stranded files / staging rows that never get
-    // cleaned, and operators need to see it. Inner catch makes the
-    // promise resolve so callers (startupReap awaiter + setInterval)
-    // don't have to handle rejection.
-    const p = reapOrphans(handle).catch((err) => {
-      console.warn('objstore reaper error:', errStack(err))
-    }).finally(() => { if (inFlight === p) inFlight = null })
+    // Keep failures visible to explicit callers; automatic sweeps log them.
+    const p = reapOrphans(handle).finally(() => { if (inFlight === p) inFlight = null })
     inFlight = p
     return p
   }
-  // Hard off-switch (OBJSTORE_REAP_DISABLED). Skip the boot sweep AND the
-  // periodic timer entirely: `startupReap` resolves immediately so the
-  // index.ts `await startupReap` gate is a no-op, and `stopReaper` has
-  // nothing to clear or drain. Loud, unconditional warning — with GC off,
-  // orphaned/superseded blobs and stale staging rows are never reclaimed
-  // (they accumulate until an external job, if any, collects them).
+  const drain = async () => { if (inFlight) await inFlight.catch(() => {}) }
+  const automaticSweep = () => enqueueSweep().catch(err => console.warn('objstore reaper error:', errStack(err)))
+  // Explicit cleanup remains available when automatic sweeps are disabled.
   if (deps.reapDisabled) {
-    console.warn('objstore: reaper DISABLED (OBJSTORE_REAP_DISABLED) — orphaned/superseded blobs and stale staging will NOT be reclaimed')
-    return { handlers, restDeps, startupReap: Promise.resolve(), stopReaper: async () => {} }
+    console.warn('objstore: automatic reaper DISABLED (OBJSTORE_REAP_DISABLED) — use /api/reap or an external cleanup job')
+    return { handlers, restDeps, reap: enqueueSweep, startupReap: Promise.resolve(), stopReaper: drain }
   }
   // Caller awaits this before accepting traffic so a fresh boot
   // can't hand out list / fetch / put-begin against a tag whose
   // on-disk state still has stranded files from a prior crash.
-  const startupReap = enqueueSweep()
+  const startupReap = automaticSweep()
   // Jittered start of the periodic timer. Multi-replica deploys
   // (Neon + Vercel Blob) commonly boot N replicas in tight lock-step
   // (deploy rollout, cluster restart), which would otherwise sync every
@@ -139,20 +131,20 @@ export function initObjstore(deps: ObjstoreInitDeps): ObjstoreInit {
   let reapTimer: ReturnType<typeof setInterval> | null = null
   const jitterMs = Math.floor(Math.random() * deps.reapIntervalMs)
   const firstTimer = setTimeout(() => {
-    enqueueSweep()
-    reapTimer = setInterval(enqueueSweep, deps.reapIntervalMs)
+    void automaticSweep()
+    reapTimer = setInterval(automaticSweep, deps.reapIntervalMs)
     reapTimer.unref?.()
   }, deps.reapIntervalMs + jitterMs)
   firstTimer.unref?.()
   return {
-    handlers, restDeps, startupReap,
+    handlers, restDeps, reap: enqueueSweep, startupReap,
     stopReaper: async () => {
       clearTimeout(firstTimer)
       if (reapTimer) clearInterval(reapTimer)
       // Drain whichever sweep is currently running (startup or
       // periodic) — either could be mid-readdir/unlink at SIGTERM
       // time and would otherwise outlive `handle.close()`.
-      if (inFlight) await inFlight.catch(() => {})
+      await drain()
     },
   }
 }

package/{server → server-e2e}/objstore/reaper.ts

@@ -121,7 +121,7 @@ async function reapStaleStagingRows(handle: Handle, now: number, stagingTtlMs: n
   // Push the staleness filter into SQL so the
   // `workspace_object_staging_begun_at_idx` index handles the scan.
   // The snapshot is O(stale-rows) cluster-wide. DB-layout audit
-  // `server/objstore/store.ts`.
+  // `server-e2e/objstore/store.ts`.
   const staleBefore = now - stagingTtlMs
   const staging = await handle.listAllStaging.all(staleBefore) as StagingRow[]
   for (const s of staging) {

package/server-e2e/objstore/rest-deny.ts

@@ -0,0 +1,28 @@
+// Shared failure-response helpers for the objstore REST plane
+// (./rest.ts + ./rest-mint.ts). A uniform `{ error: <reason> }` JSON
+// envelope so clients parse one shape; the 409 variant additionally
+// carries the live version/incarnation so a PUT / put-begin can rebase.
+// Reasons (400/401/403/404/405/409/410/411/500/503) are documented in
+// server-e2e/README.md and the client decides recovery from the code —
+// probe-distinguishing defense isn't a goal: the richer PUT/GET reasons are
+// reachable only after the bearer-token check, while the public surfaces (an
+// unauthed GET/PUT, or the signature-authed POST mint) expose only
+// 400/401/404 — none of which gives a probe signal it couldn't already
+// enumerate.
+
+import type { ServerResponse } from 'node:http'
+
+export function deny(res: ServerResponse, status: number, body: string): void {
+  res.writeHead(status, { 'content-type': 'application/json' })
+  res.end(JSON.stringify({ error: body }))
+}
+
+// Variant of `deny` that augments the envelope with the live row's
+// `currentVersion` + `currentIncarnation` so a 409'd PUT / put-begin can
+// rebase onto the right precondition token rather than retry blindly
+// against an occupied slot. Symmetric with the WS plane's
+// `objstore-conflict` frame.
+export function denyConflict(res: ServerResponse, currentVersion: number | null, currentIncarnation: string | null): void {
+  res.writeHead(409, { 'content-type': 'application/json' })
+  res.end(JSON.stringify({ error: 'conflict', currentVersion, currentIncarnation }))
+}

package/server-e2e/objstore/rest-mint.ts

@@ -0,0 +1,224 @@
+// REST endpoints for the objstore plane — the single-round-trip,
+// SSE-session-independent alternative to the WS `objstore-fetch` /
+// `objstore-put-begin` / `objstore-delete` handshakes (so these ops can't
+// be interrupted by an SSE replica hop). One route, `POST
+// /api/objstore/{tag}/{res}`, with a JSON body `{ op, ts, signature, ... }`:
+// the workspace signs the request (the workspaceTag IS the Ed25519 pubkey,
+// so verification is self-contained — no socket, no stored key), the server
+// verifies it + freshness/replay-guards (the connection-nonce stand-in).
+// fetch/put return the SAME token shape the WS path sends, for the UNCHANGED
+// bearer-token GET / PUT byte transfers; delete mutates in place and returns
+// `{ deletedVersion }` (no token — there are no bytes to move). See
+// server-e2e/README.md "REST endpoints & tokens".
+
+import type { IncomingMessage, ServerResponse } from 'node:http'
+import { Buffer } from 'node:buffer'
+
+import type { ObjstoreRestDeps, RouteMatch } from './rest.ts'
+import { deny, denyConflict } from './rest-deny.ts'
+import { MAX_CONTENT_LENGTH, beginPut, deleteObject, getLive, isValidContentHash, isValidIncarnation, isValidSignature, objectMetaWire } from './store.ts'
+import { mintGetToken, mintPutToken } from './tokens.ts'
+import { verifyObjstoreDeleteRestSig, verifyObjstoreFetchRestSig, verifyObjstorePutBeginRestSig } from './sign.ts'
+import { createFetchMintGuard } from './fetch-mint-guard.ts'
+
+// Per-process freshness + replay guard for the REST POSTs (fetch, put-begin,
+// delete). They have no connection nonce to bind, so they bind a client
+// timestamp — see ./fetch-mint-guard.ts. The three ops' signatures are
+// globally unique (distinct canonical domains), so one guard dedups all three.
+const mintGuard = createFetchMintGuard()
+
+// Hard cap on a mint JSON body. The put-begin body (op, ts, signature,
+// prevVersion, prevIncarnation, expectedLength, contentHash) is the larger
+// of the two and still only a few hundred bytes; 4 KiB is comfortable
+// headroom. Bounds the read so a hostile client can't stream an unbounded
+// body into memory before the parse.
+const MINT_BODY_MAX = 4096
+
+// Read a small JSON request body up to `maxBytes`, returning the parsed
+// value or null on overflow / parse failure / read error. Used only by
+// the mint POST; the PUT body is the raw blob and streams to disk via a
+// different path.
+async function readJsonBody(req: IncomingMessage, maxBytes: number): Promise<unknown> {
+  const chunks: Buffer[] = []
+  let total = 0
+  try {
+    for await (const chunk of req) {
+      const buf = chunk as Buffer
+      total += buf.length
+      if (total > maxBytes) return null
+      chunks.push(buf)
+    }
+  } catch { return null }
+  try { return JSON.parse(Buffer.concat(chunks).toString('utf8')) }
+  catch { return null }
+}
+
+// Extract + range-check the auth fields common to both mint ops. Gates the
+// signature to the wire shape (`isValidSignature`, matching the WS plane) so
+// an obviously-malformed sig is rejected up front rather than burning an
+// Ed25519 verify — and, since this runs before `mintGuard.admit`, a bad sig
+// never reaches the replay cache regardless.
+function parseMintAuth(body: object): { ts: number; signature: string } | null {
+  const ts = (body as { ts?: unknown }).ts
+  const signature = (body as { signature?: unknown }).signature
+  if (!Number.isSafeInteger(ts) || (ts as number) < 0) return null
+  if (!isValidSignature(signature)) return null
+  return { ts: ts as number, signature }
+}
+
+// Dispatch the mint by body `op`. Auth-proxy friendly: the signature rides
+// the JSON body (not a header or cookie, so it never collides with a
+// cookie-based proxy), and the response is JSON — no redirect, no token in
+// any URL. Client note: a replayed signature is rejected, so a retry MUST
+// re-sign with a fresh `ts` rather than resend the same body.
+export async function handleRestMint(
+  deps: ObjstoreRestDeps, req: IncomingMessage, res: ServerResponse, route: RouteMatch,
+): Promise<void> {
+  const body = await readJsonBody(req, MINT_BODY_MAX)
+  if (!body || typeof body !== 'object') { deny(res, 400, 'bad-request'); return }
+  const op = (body as { op?: unknown }).op
+  if (op === 'fetch') { await handleRestFetchMint(deps, res, route, body); return }
+  if (op === 'put') { await handleRestPutBegin(deps, res, route, body); return }
+  if (op === 'delete') { await handleRestDelete(deps, res, route, body); return }
+  deny(res, 400, 'bad-request')
+}
+
+// op:'fetch' — mirrors the WS `objstore-fetch` → `objstore-fetch-token`
+// handshake; returns `{ ...meta, urlPath, token, expiresAt }` for the
+// UNCHANGED bearer-token GET.
+async function handleRestFetchMint(
+  deps: ObjstoreRestDeps, res: ServerResponse, route: RouteMatch, body: object,
+): Promise<void> {
+  const auth = parseMintAuth(body)
+  if (!auth) { deny(res, 400, 'bad-request'); return }
+  // Verify the signature BEFORE touching the replay guard so a bad-sig
+  // request can't consume cache space. The signature commits to THIS `ts`.
+  if (!await verifyObjstoreFetchRestSig(route.tag, route.resourceTag, auth.ts, auth.signature)) {
+    deny(res, 401, 'unauthorized'); return
+  }
+  // Freshness window + single-use dedup (the connection-nonce stand-in).
+  // 'stale'/'replay' are both opaque 401s — the client re-signs with a
+  // fresh `ts` and retries.
+  if (mintGuard.admit(auth.signature, auth.ts) !== 'ok') { deny(res, 401, 'unauthorized'); return }
+  const row = await getLive(deps.handle, route.tag, route.resourceTag)
+  if (!row) { deny(res, 404, 'not-found'); return }
+  const { token, exp } = mintGetToken(deps.secret, route.tag, route.resourceTag, row.version, row.incarnation)
+  res.writeHead(200, { 'content-type': 'application/json' })
+  res.end(JSON.stringify({
+    ...objectMetaWire(row),
+    urlPath: `/api/objstore/${route.tag}/${route.resourceTag}`,
+    token,
+    expiresAt: exp,
+  }))
+}
+
+// op:'put' — mirrors the WS `handlePutBegin`: verify the signed put fields,
+// freshness/replay-guard, the new-workspace operator gate, then `beginPut`
+// (advisory prev-check + staging row) and mint the put-token. Returns
+// `{ stagingId, urlPath, token, expiresAt }` for the UNCHANGED bearer-token
+// PUT; `409 conflict` (with current version/incarnation to rebase on) or
+// `403 workspace-full` map `beginPut`'s refusals.
+//
+// New-workspace gate: REST can't read a socket's operator-auth state, so
+// when a password is configured AND the workspace is never-before-seen this
+// returns 401 — the client falls back to the in-band WS put-begin (which
+// runs the operator auth flow). No-op for password-less deployments or
+// existing workspaces (the common case).
+async function handleRestPutBegin(
+  deps: ObjstoreRestDeps, res: ServerResponse, route: RouteMatch, body: object,
+): Promise<void> {
+  const auth = parseMintAuth(body)
+  if (!auth) { deny(res, 400, 'bad-request'); return }
+  // Field gates mirror `handlePutBegin`'s up-front rejects.
+  const expectedLength = (body as { expectedLength?: unknown }).expectedLength
+  if (!Number.isSafeInteger(expectedLength) || (expectedLength as number) < 0 || (expectedLength as number) > MAX_CONTENT_LENGTH) {
+    deny(res, 400, 'bad-request'); return
+  }
+  const prevVersionRaw = (body as { prevVersion?: unknown }).prevVersion
+  if (prevVersionRaw != null && (typeof prevVersionRaw !== 'number' || !Number.isSafeInteger(prevVersionRaw))) {
+    deny(res, 400, 'bad-request'); return
+  }
+  const prevVersion = typeof prevVersionRaw === 'number' ? prevVersionRaw : null
+  const prevIncarnationRaw = (body as { prevIncarnation?: unknown }).prevIncarnation
+  const prevIncarnation = typeof prevIncarnationRaw === 'string' ? prevIncarnationRaw : null
+  // prevVersion/prevIncarnation travel as a null-iff-null pair (matches the
+  // WS `validPrevPair`): a half-pair is malformed.
+  if ((prevVersion === null) !== (prevIncarnation === null)) { deny(res, 400, 'bad-request'); return }
+  // A non-null incarnation must be the wire shape (matches the WS path's
+  // `isValidIncarnation` gate). Signature-covered + CAS-checked regardless,
+  // but reject the obviously-malformed up front.
+  if (prevIncarnation !== null && !isValidIncarnation(prevIncarnation)) { deny(res, 400, 'bad-request'); return }
+  const contentHash = (body as { contentHash?: unknown }).contentHash
+  if (!isValidContentHash(contentHash)) { deny(res, 400, 'bad-request'); return }
+
+  const fields = {
+    workspaceTag: route.tag, resourceTag: route.resourceTag,
+    prevVersion, prevIncarnation, contentHash, expectedLength: expectedLength as number,
+  }
+  if (!await verifyObjstorePutBeginRestSig(fields, auth.ts, auth.signature)) { deny(res, 401, 'unauthorized'); return }
+  if (mintGuard.admit(auth.signature, auth.ts) !== 'ok') { deny(res, 401, 'unauthorized'); return }
+  // New-workspace operator gate — see the function header. Refusing here
+  // routes the client to its in-band WS put-begin fallback.
+  if (await deps.restPutGate(route.tag)) { deny(res, 401, 'unauthorized'); return }
+  const result = await beginPut(deps.handle, {
+    workspaceTag: route.tag, resourceTag: route.resourceTag,
+    prevVersion, prevIncarnation, expectedLength: expectedLength as number,
+    contentHash, signature: auth.signature,
+  })
+  if (!result.ok) {
+    if (result.reason === 'workspace-full') { deny(res, 403, 'workspace-full'); return }
+    denyConflict(res, result.conflict?.version ?? null, result.conflict?.incarnation ?? null); return
+  }
+  const { token, exp } = mintPutToken(deps.secret, route.tag, route.resourceTag, result.stagingId, expectedLength as number)
+  res.writeHead(200, { 'content-type': 'application/json' })
+  res.end(JSON.stringify({
+    stagingId: result.stagingId,
+    urlPath: `/api/objstore/${route.tag}/${route.resourceTag}`,
+    token,
+    expiresAt: exp,
+  }))
+}
+
+// op:'delete' — mirrors the WS `handleDelete`: verify the signed delete
+// fields, freshness/replay-guard, then `deleteObject` (a precondition-checked
+// version-CAS drop). Unlike put, there is NO operator gate — delete is
+// signature-gated, idempotent, and creates nothing (the WS handleDelete has
+// no authGate either). Returns 200 `{ deletedVersion }` (0 = idempotent
+// no-op on a missing row with a null precondition); 409 conflict (stale
+// prevVersion/incarnation); 404 not-found (a non-null precondition against a
+// missing row). On a real drop it broadcasts `objstore-deleted` to
+// subscribers (+ cross-instance), exactly like the WS path.
+async function handleRestDelete(
+  deps: ObjstoreRestDeps, res: ServerResponse, route: RouteMatch, body: object,
+): Promise<void> {
+  const auth = parseMintAuth(body)
+  if (!auth) { deny(res, 400, 'bad-request'); return }
+  const prevVersionRaw = (body as { prevVersion?: unknown }).prevVersion
+  if (prevVersionRaw != null && (typeof prevVersionRaw !== 'number' || !Number.isSafeInteger(prevVersionRaw))) {
+    deny(res, 400, 'bad-request'); return
+  }
+  const prevVersion = typeof prevVersionRaw === 'number' ? prevVersionRaw : null
+  const prevIncarnationRaw = (body as { prevIncarnation?: unknown }).prevIncarnation
+  const prevIncarnation = typeof prevIncarnationRaw === 'string' ? prevIncarnationRaw : null
+  // null-iff-null pair (matches the WS `validPrevPair`); a non-null
+  // incarnation must be the wire shape.
+  if ((prevVersion === null) !== (prevIncarnation === null)) { deny(res, 400, 'bad-request'); return }
+  if (prevIncarnation !== null && !isValidIncarnation(prevIncarnation)) { deny(res, 400, 'bad-request'); return }
+
+  const fields = { workspaceTag: route.tag, resourceTag: route.resourceTag, prevVersion, prevIncarnation }
+  if (!await verifyObjstoreDeleteRestSig(fields, auth.ts, auth.signature)) { deny(res, 401, 'unauthorized'); return }
+  if (mintGuard.admit(auth.signature, auth.ts) !== 'ok') { deny(res, 401, 'unauthorized'); return }
+  const result = await deleteObject(deps.handle, route.tag, route.resourceTag, prevVersion, prevIncarnation)
+  if (!result.ok) {
+    if (result.reason === 'conflict') { denyConflict(res, result.conflict?.version ?? null, result.conflict?.incarnation ?? null); return }
+    deny(res, 404, 'not-found'); return
+  }
+  res.writeHead(200, { 'content-type': 'application/json' })
+  res.end(JSON.stringify({ deletedVersion: result.deletedVersion }))
+  // deletedVersion 0 = nothing was live → nothing to broadcast. A real drop
+  // fans out to subscribers (including the originator, matching the WS path's
+  // `except: null`) so peers' `onDeleted` fire.
+  if (result.deletedVersion === 0) return
+  deps.broadcast(route.tag, { type: 'objstore-deleted', workspaceTag: route.tag, resourceTag: route.resourceTag, version: result.deletedVersion, incarnation: prevIncarnation! }, null)
+  deps.publishObjDeleted(route.tag, route.resourceTag, result.deletedVersion, prevIncarnation!)
+}

package/{server → server-e2e}/objstore/rest.ts

@@ -71,8 +71,8 @@ export type ObjstoreRestDeps = {
   // Cross-instance pub/sub for objstore-deleted — fired alongside the local
   // `broadcast` after a successful REST delete mint, so peers on OTHER
   // instances drop the resource in real time. Carries (tag, resourceTag,
-  // version) inline (the row is gone post-delete). SQLite mode passes a no-op.
-  publishObjDeleted: (tag: string, resourceTag: string, version: number) => void
+  // version, incarnation) inline (the row is gone post-delete). SQLite mode passes a no-op.
+  publishObjDeleted: (tag: string, resourceTag: string, version: number, incarnation: string) => void
   // New-workspace operator gate for the REST put-begin mint — the
   // connection-independent analog of the WS path's `authGate`. Returns
   // `true` to DENY (a password is configured AND the workspace is
@@ -105,7 +105,7 @@ export type ObjstoreRestDeps = {
 // concurrent same-token PUT is rejected (409) before it can open a
 // second writer. The slot is held only across body + commit, released
 // in a `finally`, and bounded by the REST idle-body timeout in
-// server/http.ts so a slow-loris can't pin it. NOT a commit lock —
+// server-e2e/http.ts so a slow-loris can't pin it. NOT a commit lock —
 // commits stay lock-free on the version-CAS above. Per-process: the
 // Vercel multi-replica path leans on `allowOverwrite: true` + the
 // commit CAS + content-addressing for cross-replica correctness (a

package/{server → server-e2e}/objstore/store-neon.ts

@@ -4,7 +4,7 @@
 // dialect on the wire.
 //
 // `@neondatabase/serverless` is an OPTIONAL peer dep — selected by
-// the `DATABASE_URL` branch in `server/index.ts`. The peer dep
+// the configured database URL branch in `server-e2e/index.ts`. The peer dep
 // itself is loaded lazily inside `openNeonObjstore` below. A single
 // Neon project / database holds both `workspace_revision` and the
 // two objstore tables; each plane opens its own stateless
@@ -13,14 +13,10 @@
 // release.
 //
 // Byte plane: passed in as a `BlobBackend` (./blob.ts). The
-// supported pairings selected by server/index.ts are:
-//   - Neon + Vercel Blob Private Storage (multi-replica, the only
-//     pairing that survives a replica restart without local-disk
-//     coordination). Activated by setting BLOB_READ_WRITE_TOKEN
-//     alongside DATABASE_URL.
-//   - Neon + local FS (development / operator escape hatch — bytes
-//     still need to live on a shared filesystem if you run more
-//     than one replica).
+// supported Neon pairing selected by server-e2e/index.ts is Vercel
+// Blob Private Storage. DATABASE_URL or E2E_DATABASE_URL plus BLOB_READ_WRITE_TOKEN
+// is required; the launcher never selects Neon + local FS. Tests
+// can inject a local-FS backend directly through this adapter.
 
 import type { BlobBackend } from './blob.ts'
 import type { AllStmt, GetStmt, RunStmt } from '../db-stmt.ts'
@@ -309,7 +305,7 @@ export async function openNeonObjstore(connectionString: string, blob: BlobBacke
   // local `../neon-driver.ts` re-export wrapper so the peer dep stays
   // optional AND tests can mock the driver (a local path is always
   // resolvable; the bare specifier isn't when the dep is absent). See
-  // `server/neon-driver.ts`. Cast through `unknown` because the
+  // `server-e2e/neon-driver.ts`. Cast through `unknown` because the
   // wrapper's `export *` re-exports a `@ts-ignore`'d module.
   const mod = (await import('../neon-driver.ts')) as unknown as { neon: (url: string) => NeonSql }
   const sql: NeonSql = mod.neon(connectionString)

package/{server → server-e2e}/objstore/store.ts

@@ -1,5 +1,5 @@
 // SQLite + filesystem-backed object store for the v1.objstore
-// protocol extension. Sibling of `server/db.ts`; shares the
+// protocol extension. Sibling of `server-e2e/db.ts`; shares the
 // underlying DatabaseSync handle but keeps its own tables.
 //
 //   workspace_object         — one row per LIVE resource (no
@@ -150,7 +150,7 @@ export type DeleteResult =
   | { ok: true; deletedVersion: number }
   | { ok: false; reason: 'not-found' | 'conflict'; conflict?: ObjectRow }
 
-// Async statement shapes are shared with server/db.ts via
+// Async statement shapes are shared with server-e2e/db.ts via
 // ../db-stmt.ts — same `.get/.all/.run` → Promise contract across
 // both planes.
 
@@ -186,7 +186,7 @@ export type Handle = {
   db?: DatabaseSync
   // Byte-plane backend (local FS or Vercel Blob). All bytes-side
   // operations go through this — there is no direct fs.* call in
-  // store / rest / reaper. Selected at boot in server/index.ts.
+  // store / rest / reaper. Selected at boot in server-e2e/index.ts.
   blob: BlobBackend
   // Storage root for the FS backend — set only when `blob` was
   // constructed from `openFsBlobBackend(dir)`. Production code
@@ -247,7 +247,7 @@ export type Handle = {
   // returned. The reaper passes `Date.now() - stagingTtlMs` so the
   // index `workspace_object_staging_begun_at_idx` is used and the
   // sweep is O(stale-rows) instead of O(in-flight-uploads-cluster-
-  // wide). DB-layout audit `server/objstore/store.ts:312`.
+  // wide). DB-layout audit `server-e2e/objstore/store.ts:312`.
   listAllStaging: AllStmt<[number], { workspace_tag: string; resource_tag: string; staging_id: string; begun_at: number }>
   listLiveTags: AllStmt<[], { workspace_tag: string }>
   countLive: GetStmt<[string], { c: number }>
@@ -255,7 +255,7 @@ export type Handle = {
 
 // Narrowing alias for the SQLite-backed Handle: `db` is guaranteed
 // to be set. `openObjstore` returns this so call sites (production
-// shutdown plumbing in `server/index.ts` + the entire SQLite-only
+// shutdown plumbing in `server-e2e/index.ts` + the entire SQLite-only
 // test suite in `tests/server-objstore.test.js`) can reach
 // `handle.db.prepare(...)` without an optional-chain or non-null
 // assertion. A Neon-backed Handle (`openNeonObjstore`) keeps the
@@ -352,7 +352,7 @@ export function openObjstore(db: DatabaseSync, dir: string): SqliteHandle {
   mkdirSync(dir, { recursive: true })
   db.exec(SCHEMA)
   // Fail-loud on a pre-existing non-STRICT table — same rationale as
-  // server/db.ts: `CREATE TABLE IF NOT EXISTS … STRICT` doesn't
+  // server-e2e/db.ts: `CREATE TABLE IF NOT EXISTS … STRICT` doesn't
   // upgrade an existing non-STRICT table, and dropping strict type
   // affinity opens an operator-attack path. PR #4 review F3.
   for (const name of ['workspace_object', 'workspace_object_staging']) {
@@ -362,7 +362,7 @@ export function openObjstore(db: DatabaseSync, dir: string): SqliteHandle {
   const blob = openFsBlobBackend(dir)
   // No `close` method on the returned Handle: the underlying
   // `DatabaseSync` is owned by the caller (in production, the
-  // workspace_revision handle in `server/db.ts`, closed from
+  // workspace_revision handle in `server-e2e/db.ts`, closed from
   // `shutdown()`; tests close their own DB). A `close()` here would
   // mislead — it could only no-op or leak, never close the caller-
   // owned connection.

package/{server → server-e2e}/objstore/tokens.ts

@@ -24,7 +24,7 @@
 import { createHmac, randomBytes, timingSafeEqual } from 'node:crypto'
 import { Buffer } from 'node:buffer'
 
-export const DEFAULT_TOKEN_TTL_MS = 5 * 60 * 1000
+const DEFAULT_TOKEN_TTL_MS = 5 * 60 * 1000
 
 export type PutTokenPayload = {
   op: 'put'
@@ -119,7 +119,7 @@ function isValidPayload(v: unknown): v is TokenPayload {
   // different actual value, or — for `exp` — claim a
   // `Number.MAX_SAFE_INTEGER + 1` expiry that compares ambiguously near
   // the IEEE-754 boundary. Matches the codebase's other safe-int gates
-  // (server/objstore/sign.ts, rest.ts, handlers.ts).
+  // (server-e2e/objstore/sign.ts, rest.ts, handlers.ts).
   if (!Number.isSafeInteger(o['exp']) || (o['exp'] as number) < 0) return false
   if (o['op'] === 'put') {
     return typeof o['sid'] === 'string'

package/{server → server-e2e}/pubsub.ts

@@ -1,5 +1,5 @@
 // Cross-instance pub/sub for real-time WS broadcasts. The triage-sync
-// fan-out is an in-memory subscriber map (server/hub.ts) by design — it
+// fan-out is an in-memory subscriber map (server-e2e/hub.ts) by design — it
 // routes a commit only to peers on the SAME instance. A multi-instance
 // deployment behind a load balancer needs commit-landed-on-A to reach
 // peers-on-B with the same latency the hub gives same-instance peers.
@@ -57,15 +57,15 @@ function newSenderId(): string {
 //   - 'objput': objstore-put broadcast. `res` is the resource tag; the
 //     receiver SELECTs the live row by (tag, res) for the rest of the
 //     metadata fields.
-//   - 'objdel': objstore-deleted broadcast. `ver` is the deleted version
-//     — inline because the row is gone from workspace_object after the
-//     delete commit.
+//   - 'objdel': objstore-deleted broadcast. `ver` and `incarnation` identify
+//     the deleted row, which is gone from workspace_object after commit.
+//     Incarnation is optional only for messages from older relay instances.
 export type BusMessage =
   | { kind: 'rev'; tag: string; id: string }
   | { kind: 'objput'; tag: string; res: string }
-  | { kind: 'objdel'; tag: string; res: string; ver: number }
+  | { kind: 'objdel'; tag: string; res: string; ver: number; incarnation?: string }
 
-// Receiver wired up by the hub layer (see server/index.ts). Each handler
+// Receiver wired up by the hub layer (see server-e2e/index.ts). Each handler
 // runs once per remote message; failures are logged but don't crash the
 // LISTEN loop — a missed broadcast surfaces to clients on reconnect
 // (chain re-pull). Async because the workspace-revision handler does a
@@ -164,7 +164,7 @@ type NeonState = {
   // In-flight bus-message handler promises. `dispatchNotification`
   // fires handlers fire-and-forget, but `stop()` awaits this set before
   // returning so the lifecycle's `handle.close()` (which runs after
-  // `pubsub.stop()` — see closeDb in server/index.ts) can't race a
+  // `pubsub.stop()` — see closeDb in server-e2e/index.ts) can't race a
   // handler mid-`handle.revisionById.get` / `getLive`.
   pendingHandlers: Set<Promise<void>>
 }
@@ -333,7 +333,7 @@ export function createNeonPubSub(deps: NeonPubSubDeps): PubSub {
       if (state.connectAttempt) { try { await state.connectAttempt } catch {} }
       // Drain in-flight bus-message handlers BEFORE returning. The
       // lifecycle teardown runs `pubsub.stop()` and THEN
-      // `handle.close()` (see closeDb in server/index.ts); a handler
+      // `handle.close()` (see closeDb in server-e2e/index.ts); a handler
       // still in `handle.revisionById.get` / `getLive` would otherwise
       // throw against a closed DB. `allSettled` so one handler's
       // rejection doesn't abort the drain.
@@ -388,7 +388,10 @@ function parseBusMessage(raw: Record<string, unknown>): BusMessage | null {
     const ver = raw['ver']
     if (typeof res !== 'string') return null
     if (typeof ver !== 'number' || !Number.isSafeInteger(ver)) return null
-    return { kind: 'objdel', tag, res, ver }
+    const incarnation = raw['incarnation']
+    if (incarnation === undefined) return { kind: 'objdel', tag, res, ver }
+    if (typeof incarnation !== 'string' || incarnation.length === 0) return null
+    return { kind: 'objdel', tag, res, ver, incarnation }
   }
   return null
 }

package/{server → server-e2e}/sign.ts

@@ -92,7 +92,7 @@ export function canonicalSave(
   // confusing diff in the canonical bytes rather than a clean drop.
   // Mirrors the `isSafeNonNegativeInt` rigor that the objstore
   // canonical builders apply. Input-validation audit
-  // `server/sign.ts:88`.
+  // `server-e2e/sign.ts:88`.
   if (typeof workspaceTag !== 'string') throw new TypeError('canonicalSave: workspaceTag must be string')
   if (typeof nonce !== 'string') throw new TypeError('canonicalSave: nonce must be string')
   if (typeof ciphertext !== 'string') throw new TypeError('canonicalSave: ciphertext must be string')
@@ -122,7 +122,7 @@ function canonicalSubscribe(
   return encodeUtf8([SUBSCRIBE_DOMAIN, workspaceTag as string, fromStr, connectionNonce].join('\n'))
 }
 
-// Exported because the v1.objstore signing module (server/objstore/sign.ts)
+// Exported because the v1.objstore signing module (server-e2e/objstore/sign.ts)
 // reuses it for its four verifiers — same workspaceTag-as-pubkey contract,
 // same domain-separated canonical bytes. Keeping the WebCrypto plumbing
 // in one place avoids drift between the triage-sync and objstore verify
@@ -157,7 +157,7 @@ export async function verifyEd25519(
 // `{ ok: false, canonical: null }` on bad shape / bad sig;
 // `{ ok: true, canonical: <bytes> }` on success.
 //
-// NOT used by `server/index.ts handleSave`. Production composes
+// NOT used by `server-e2e/sync-handlers.ts handleSave`. Production composes
 // the smaller helpers (`canonicalSave`, `computeRevisionIdFromCanonical`,
 // `verifyEd25519`) separately so the dup-precheck via
 // `revisionExists` can fire BEFORE Ed25519-verify and skip the
@@ -204,7 +204,7 @@ export async function computeRevisionIdFromCanonical(canonical: Uint8Array<Array
 
 // `connectionNonce` is the per-socket challenge the server issued
 // to the originating connection (see `Peer.challenge` in
-// server/peer.ts). The client signs a canonical that includes the
+// server-e2e/peer.ts). The client signs a canonical that includes the
 // nonce; verifying against the SAME nonce here is what blocks
 // cross-connection replay of a captured subscribe frame. Audit
 // round-9 H2.