@preventive/triage 1.0.0-alpha.2 → 1.0.0-alpha.20

package/{server → server-e2e}/objstore/store.ts

@@ -1,5 +1,5 @@
 // SQLite + filesystem-backed object store for the v1.objstore
-// protocol extension. Sibling of `server/db.ts`; shares the
+// protocol extension. Sibling of `server-e2e/db.ts`; shares the
 // underlying DatabaseSync handle but keeps its own tables.
 //
 //   workspace_object         — one row per LIVE resource (no
@@ -79,9 +79,8 @@ const SCHEMA = `
 
 // One LIVE row, exactly the shape the `workspace-subscribed` ack's
 // `resources` array carries on the wire, minus `keyframe`-style
-// server-only flags. `put_at` is a
-// debug aid the wire format doesn't include — operators can inspect
-// it via the DB but the server never volunteers it.
+// server-only flags. `put_at` is a debug aid the wire format never
+// includes — inspectable via the DB but the server never volunteers it.
 export type ObjectRow = {
   resourceTag: string
   version: number
@@ -118,13 +117,11 @@ export type BeginPutInput = {
 // PUT will reference. `workspace-full` is the per-workspace resource-
 // count cap rejection — see `MAX_RESOURCES_PER_WORKSPACE`.
 //
-// `filePath` is set only for the FS-backed handle (where it's the
-// absolute on-disk staging path); the Vercel-Blob handle omits it
-// since "path" isn't a meaningful concept against a remote object
-// store. Production code (rest.ts) never reads this field — the
-// REST layer goes through `handle.blob.openStagingWriter(tag, sid)`.
-// Tests for the FS path use it as a convenience to write fixture
-// bytes directly to the staging slot.
+// `filePath` is the absolute on-disk staging path, set only for the
+// FS handle; the Vercel handle omits it ("path" is meaningless against
+// a remote store). Production (rest.ts) never reads it — it goes
+// through `handle.blob.openStagingWriter(tag, sid)`. FS-path tests use
+// it as a convenience to write fixture bytes to the staging slot.
 export type BeginPutResult =
   | { ok: true; stagingId: string; filePath?: string }
   | { ok: false; reason: 'conflict'; conflict: ObjectRow | null }
@@ -134,17 +131,14 @@ export type CommitPutInput = {
   workspaceTag: string
   resourceTag: string
   stagingId: string
-  // Optional: the storage-side byte count the caller already
-  // verified after the upload landed. When provided, commitPut
-  // skips the otherwise-redundant `statStaging` round-trip — for
-  // the Vercel backend that's one fewer HTTP HEAD per PUT.
-  // Caller must ONLY pass a value it observed for THIS stagingId
-  // after its upload finished (i.e. the REST PUT path's post-upload
-  // `statStaging`). Safe without a lock because staging ids are
-  // random — no other request writes this blob — and the sole writer
-  // (this PUT) has already completed. Tests that drive commitPut
-  // directly without the REST layer should omit this and let
-  // commitPut stat for itself.
+  // Optional storage-side byte count the caller already verified
+  // post-upload. When provided, commitPut skips the redundant
+  // `statStaging` round-trip — one fewer Vercel HEAD per PUT. Caller
+  // must ONLY pass a value it observed for THIS stagingId after its
+  // upload finished (the REST PUT path's post-upload `statStaging`).
+  // Safe without a lock: staging ids are random (no other request
+  // writes this blob) and the sole writer (this PUT) has finished.
+  // Tests driving commitPut directly omit this and let it stat.
   observedSize?: number
 }
 
@@ -156,7 +150,7 @@ export type DeleteResult =
   | { ok: true; deletedVersion: number }
   | { ok: false; reason: 'not-found' | 'conflict'; conflict?: ObjectRow }
 
-// Async statement shapes are shared with server/db.ts via
+// Async statement shapes are shared with server-e2e/db.ts via
 // ../db-stmt.ts — same `.get/.all/.run` → Promise contract across
 // both planes.
 
@@ -181,9 +175,9 @@ type DbRow = {
 //   - the reaper's age grace window + an atomic conditional staging
 //     delete (`deleteStagingIfStale`) so the stale-staging sweep
 //     can't race an upload that just finished.
-// These hold both within a single process AND across replicas — the
-// old per-(tag, resourceTag) in-process mutex added nothing the CAS +
-// content-addressing didn't already give, so it was removed.
+// These hold both within a single process AND across replicas, so no
+// per-(tag, resourceTag) in-process mutex is needed — the CAS +
+// content-addressing already cover everything one would.
 export type Handle = {
   // SQLite-only: the underlying `DatabaseSync`. Unset on the Neon
   // backend (see ./store-neon.ts). Test-only fixture SQL routes
@@ -192,7 +186,7 @@ export type Handle = {
   db?: DatabaseSync
   // Byte-plane backend (local FS or Vercel Blob). All bytes-side
   // operations go through this — there is no direct fs.* call in
-  // store / rest / reaper. Selected at boot in server/index.ts.
+  // store / rest / reaper. Selected at boot in server-e2e/index.ts.
   blob: BlobBackend
   // Storage root for the FS backend — set only when `blob` was
   // constructed from `openFsBlobBackend(dir)`. Production code
@@ -212,15 +206,14 @@ export type Handle = {
   selectStagingByWsSid: GetStmt<[string, string], unknown>
   refreshStagingBegunAt: RunStmt<[number, string, string, string]>
   deleteStaging: RunStmt<[string, string, string]>
-  // Atomic conditional staging delete used by the reaper's stale-row
+  // Atomic conditional staging delete for the reaper's stale-row
   // sweep. `deleteStagingIfStale(tag, res, sid, staleBefore)` deletes
   // the row IFF its `begun_at < staleBefore`, returning `{ ok: 1 }`
-  // when a row was actually removed and `undefined` otherwise. This is
-  // the lock-free replacement for the old in-lock begun_at re-read
-  // (PR #4 "F1"): a slow PUT that finishes and calls
-  // `refreshStagingBegunAt` bumps `begun_at` fresh, so a concurrent
-  // reaper's conditional delete simply doesn't match (its predicate
-  // fails atomically) and the row survives for the commit. Mirrors the
+  // when a row was actually removed and `undefined` otherwise. Lock-
+  // free race guard (PR #4 "F1"): a slow PUT that finishes calls
+  // `refreshStagingBegunAt` to bump `begun_at` fresh, so a concurrent
+  // reaper's conditional delete doesn't match (predicate fails
+  // atomically) and the row survives for the commit. Mirrors the
   // `insertLiveIfAbsent` RETURNING pattern.
   deleteStagingIfStale: GetStmt<[string, string, string, number], { ok: number }>
   selectLive: AllStmt<[string], DbRow>
@@ -254,7 +247,7 @@ export type Handle = {
   // returned. The reaper passes `Date.now() - stagingTtlMs` so the
   // index `workspace_object_staging_begun_at_idx` is used and the
   // sweep is O(stale-rows) instead of O(in-flight-uploads-cluster-
-  // wide). DB-layout audit `server/objstore/store.ts:312`.
+  // wide). DB-layout audit `server-e2e/objstore/store.ts:312`.
   listAllStaging: AllStmt<[number], { workspace_tag: string; resource_tag: string; staging_id: string; begun_at: number }>
   listLiveTags: AllStmt<[], { workspace_tag: string }>
   countLive: GetStmt<[string], { c: number }>
@@ -262,7 +255,7 @@ export type Handle = {
 
 // Narrowing alias for the SQLite-backed Handle: `db` is guaranteed
 // to be set. `openObjstore` returns this so call sites (production
-// shutdown plumbing in `server/index.ts` + the entire SQLite-only
+// shutdown plumbing in `server-e2e/index.ts` + the entire SQLite-only
 // test suite in `tests/server-objstore.test.js`) can reach
 // `handle.db.prepare(...)` without an optional-chain or non-null
 // assertion. A Neon-backed Handle (`openNeonObjstore`) keeps the
@@ -279,9 +272,6 @@ export type SqliteHandle = Handle & { db: DatabaseSync }
 // transient over-shoot under high concurrency across DIFFERENT
 // resources is bounded by `(parallel new-resource begins - 1)` and is
 // accepted (the cap is a soft policy bound, not a security invariant).
-// This was already the contract under the old per-resource lock —
-// that lock keyed on (tag, resourceTag), so concurrent NEW-resource
-// begins held DIFFERENT locks and raced the count anyway.
 export const MAX_RESOURCES_PER_WORKSPACE = 100
 
 // Per-upload byte cap, shared by the WS plane (rejects oversize
@@ -362,7 +352,7 @@ export function openObjstore(db: DatabaseSync, dir: string): SqliteHandle {
   mkdirSync(dir, { recursive: true })
   db.exec(SCHEMA)
   // Fail-loud on a pre-existing non-STRICT table — same rationale as
-  // server/db.ts: `CREATE TABLE IF NOT EXISTS … STRICT` doesn't
+  // server-e2e/db.ts: `CREATE TABLE IF NOT EXISTS … STRICT` doesn't
   // upgrade an existing non-STRICT table, and dropping strict type
   // affinity opens an operator-attack path. PR #4 review F3.
   for (const name of ['workspace_object', 'workspace_object_staging']) {
@@ -372,12 +362,10 @@ export function openObjstore(db: DatabaseSync, dir: string): SqliteHandle {
   const blob = openFsBlobBackend(dir)
   // No `close` method on the returned Handle: the underlying
   // `DatabaseSync` is owned by the caller (in production, the
-  // workspace_revision handle in `server/db.ts`, which closes it
-  // from `shutdown()`). Exposing `close()` here was misleading —
-  // a callsite reading `await objstoreHandle.close()` would
-  // reasonably assume it closes something, when in practice it
-  // either no-op'd (production) or left the connection open
-  // (tests construct their own DB and `db.close()` separately).
+  // workspace_revision handle in `server-e2e/db.ts`, closed from
+  // `shutdown()`; tests close their own DB). A `close()` here would
+  // mislead — it could only no-op or leak, never close the caller-
+  // owned connection.
   return {
     db,
     blob,
@@ -522,11 +510,10 @@ export async function beginPut(handle: Handle, input: BeginPutInput): Promise<Be
   if (liveVersion !== input.prevVersion || liveIncarnation !== input.prevIncarnation) {
     return { ok: false, reason: 'conflict', conflict: live }
   }
-  // Per-workspace resource cap. Only enforced for NEW resources —
-  // re-uploads of an existing resourceTag (live != null) don't
-  // change the count, so they're always allowed. Not atomic with the
-  // insert below (see MAX_RESOURCES_PER_WORKSPACE) — a soft policy
-  // bound, accepted to over-shoot under concurrent NEW-resource begins.
+  // Per-workspace resource cap, NEW resources only — re-uploads
+  // (live != null) don't change the count, so they're always allowed.
+  // Not atomic with the insert below; soft bound, see
+  // MAX_RESOURCES_PER_WORKSPACE.
   if (!live) {
     const count = await handle.countLive.get(input.workspaceTag) as { c: number } | undefined
     if ((count?.c ?? 0) >= MAX_RESOURCES_PER_WORKSPACE) {
@@ -557,51 +544,48 @@ export async function beginPut(handle: Handle, input: BeginPutInput): Promise<Be
 // IF ABSENT; a re-upload bumps the version IFF it still matches the
 // precondition we read. Exactly one of N racing commits wins the CAS;
 // the losers get `conflict` (with the current live row) and rebase.
-// Because each PUT is content-addressed at its OWN hash (distinct PUTs
-// get distinct hashes — random nonce per encrypt), N racing commits
-// promote to N DIFFERENT immutable paths: no promote clobbers another's
-// bytes, and a loser's blob is just left unreferenced for the GC. There
-// is no metadata-vs-bytes desync to guard against. The
-// CAS provides the commit's atomicity both within a process and across
-// replicas, so NO in-process lock is taken. A crash between the
-// promote and the CAS leaves the staging blob/row intact alongside (at
-// most) a stranded, unreferenced live blob — the reaper's stale-staging
-// sweep cleans the row, and the GC reaps the unreferenced live blob
-// once it's past the grace window, matching the "stranded state,
-// reaper-cleaned, never row-pointing-at-nothing" crash-safety contract.
+// Each PUT is content-addressed at its OWN hash (distinct PUTs get
+// distinct hashes — random nonce per encrypt), so N racers promote to
+// N DIFFERENT immutable paths: no promote clobbers another's bytes, a
+// loser's blob is just left unreferenced for the GC, and there's no
+// metadata-vs-bytes desync to guard. The CAS gives atomicity both
+// within a process and across replicas, so NO in-process lock is taken.
+// A crash between the promote and the CAS leaves the staging blob/row
+// intact alongside (at most) a stranded, unreferenced live blob — the
+// stale-staging sweep cleans the row, the GC reaps the blob once past
+// the grace window: the "stranded state, reaper-cleaned, never row-
+// pointing-at-nothing" crash-safety contract.
 //
-// ACCEPTED TRADEOFF (lock removal): with no lock, the reaper no longer
-// WAITS for an in-flight upload on this key. An upload taking >1h FROM
-// BEGIN (i.e. exceeding the staging TTL during the body) can have its
-// staging row reaped mid-flight by `deleteStagingIfStale`; this commit
-// then sees no staging row and returns `no-staging` → REST 410. The
-// previous lock made the reaper block on any in-flight upload
-// (unbounded). Sub-1h uploads are unaffected: `begun_at` (set at begin)
-// stays within the TTL through the body, so the conditional delete
-// can't match, and the after-body `refreshStagingBegunAt` re-extends
-// the TTL to cover this commit step. This matches the staging TTL's
-// documented intent ("1h, comfortably over a 50 MiB upload on a slow
-// line").
+// ACCEPTED TRADEOFF (lock removal): without a lock the reaper no longer
+// waits for an in-flight upload on this key (the old lock blocked it on
+// any in-flight upload, unbounded). An upload taking >1h FROM BEGIN
+// (exceeding the staging TTL during the body) can have its staging row
+// reaped mid-flight by `deleteStagingIfStale` → this commit sees no
+// staging row → `no-staging` → REST 410. Sub-1h uploads are unaffected:
+// `begun_at` (set at begin) stays within the TTL through the body, so
+// the conditional delete can't match, and the after-body
+// `refreshStagingBegunAt` re-extends the TTL to cover this commit step —
+// matching the staging TTL's intent ("1h, comfortably over a 50 MiB
+// upload on a slow line").
 export async function commitPut(handle: Handle, input: CommitPutInput): Promise<CommitPutResult> {
   const staging = await handle.selectStaging.get(input.workspaceTag, input.resourceTag, input.stagingId)
   if (!staging) return { ok: false, reason: 'no-staging' }
   let stagedSize: number | null
   // statStaging failure here is a server-side issue (staging file
-  // was unlinked by a racing abort / reaper, EACCES, EIO, backend
+  // unlinked by a racing abort / reaper, EACCES, EIO, backend
   // unreachable, …) — not a client length-mismatch. Route through
   // `io-error` so the REST layer returns 5xx, not 400. PR #4 review.
   //
   // The REST PUT layer already statted the staging blob post-upload
   // and threads the result in via `observedSize` — skipping the
-  // round-trip saves one Vercel HEAD per PUT. The staging blob can't
-  // have been resized between that stat and here: staging ids are
-  // 16-byte random, so no other request targets this blob, and the
-  // sole writer (this PUT's upload pipeline) has already finished
-  // before the REST stat ran. The only other actor that touches a
-  // staging blob is the reaper, which UNLINKS (it never resizes); a
-  // racing reaper unlink surfaces below as statStaging→io-error or a
-  // promote failure, not a wrong size. WS / test paths that omit
-  // `observedSize` fall through to the explicit stat.
+  // round-trip saves one Vercel HEAD per PUT. The blob can't have been
+  // resized between that stat and here: staging ids are 16-byte random
+  // (no other request targets this blob) and the sole writer (this
+  // PUT's upload pipeline) has finished. The only other actor on a
+  // staging blob is the reaper, which UNLINKS (never resizes); a racing
+  // reaper unlink surfaces below as statStaging→io-error or a promote
+  // failure, not a wrong size. WS / test paths that omit `observedSize`
+  // fall through to the explicit stat.
   if (input.observedSize === undefined) {
     try { stagedSize = await handle.blob.statStaging(input.workspaceTag, input.stagingId) }
     catch { return { ok: false, reason: 'io-error' } }
@@ -637,15 +621,14 @@ export async function commitPut(handle: Handle, input: CommitPutInput): Promise<
   }
   // Promote to the CONTENT-ADDRESSED live path `${tag}/${hash}.bin`.
   // `promoteStagingToLive` returns false on any backend error (FS:
-  // EACCES / ENOSPC / EIO / a racing abort that already unlinked
-  // the staging file; Vercel: copy failure). 'io-error' is mapped
-  // to HTTP 500 by the REST layer — it's a server-side fault, not
-  // a client-fixable one. Because the destination path IS the content
-  // hash, any write to it is byte-identical BY CONSTRUCTION, so a
-  // retried or racing promote to the same path is an idempotent
-  // rewrite, never a clobber. (Distinct PUTs get distinct hashes — a
-  // fresh random nonce per encrypt makes each ciphertext unique — so
-  // concurrent commits to the same resource write to DIFFERENT paths.)
+  // EACCES / ENOSPC / EIO / a racing abort that already unlinked the
+  // staging file; Vercel: copy failure) → 'io-error' → REST HTTP 500
+  // (server-side fault, not client-fixable). The destination path IS
+  // the content hash, so any write to it is byte-identical BY
+  // CONSTRUCTION: a retried or racing promote to the same path is an
+  // idempotent rewrite, never a clobber. (Distinct PUTs get distinct
+  // hashes — fresh random nonce per encrypt — so concurrent commits to
+  // the same resource write to DIFFERENT paths.)
   if (!await handle.blob.promoteStagingToLive(input.workspaceTag, input.stagingId, staging.content_hash)) {
     return { ok: false, reason: 'io-error' }
   }
@@ -658,13 +641,12 @@ export async function commitPut(handle: Handle, input: CommitPutInput): Promise<
   const committedIncarnation = staging.prev_version == null ? freshIncarnation : staging.prev_incarnation!
   const putAt = Date.now()
   // Version-CAS in try/catch so a Neon transient (5xx, network
-  // hiccup, connection-pool exhaustion) doesn't bypass the abortPut
-  // ladder by throwing out of commitPut. Without this guard, a thrown
-  // rejection skips the REST layer's `if (!r.ok) abortPut` branch and
-  // bubbles to handleRest's outer catch — the live blob is already
-  // promoted, the staging blob + row stay, and the client sees a 500.
-  // The reaper reconciles (stale-staging sweep + unreferenced-blob
-  // GC) but the surface is a 500 the caller can retry.
+  // hiccup, pool exhaustion) doesn't throw out of commitPut and bypass
+  // the abortPut ladder. A thrown rejection would skip the REST layer's
+  // `if (!r.ok) abortPut` branch and bubble to handleRest's outer catch
+  // — live blob already promoted, staging blob + row left behind. The
+  // reaper reconciles (stale-staging sweep + unreferenced-blob GC); the
+  // surface is a 500 the caller can retry.
   let won: { ok: number } | undefined
   try {
     if (staging.prev_version == null) {
@@ -753,30 +735,28 @@ export async function abortPut(handle: Handle, tag: string, resourceTag: string,
 // `deletedVersion = 0` sentinel tells the broadcast path to skip.
 //
 // No lock: the drop is a version-CAS (`deleteLiveCAS` — DELETE WHERE
-// version = prev), so every race resolves to exactly one winner, the
-// same as the old lock did, with no lost update:
+// version = prev), so every race resolves to exactly one winner with
+// no lost update:
 //   - delete vs. a concurrent COMMIT on the same resource: whichever
 //     CAS lands first wins. A stale delete can NOT remove a row the
-//     commit just bumped — the `WHERE version = prev` no longer matches,
-//     so the delete gets `conflict` (re-read sees the bumped version).
-//     If the delete wins, the commit's `updateLiveCAS` matches no row →
-//     `conflict`. (The earlier draft used an UNCONDITIONAL delete here,
-//     which — without the lock — let `getLive` read v1, a commit bump
-//     to v2 slip in, and the delete then destroy v2: a lost update. The
-//     version-CAS closes that.)
+//     commit just bumped — `WHERE version = prev` no longer matches, so
+//     the delete gets `conflict` (re-read sees the bumped version). If
+//     the delete wins, the commit's `updateLiveCAS` matches no row →
+//     `conflict`. (An UNCONDITIONAL delete here would let `getLive`
+//     read v1, a commit bump to v2 slip in, then the delete destroy v2:
+//     a lost update. The version-CAS closes that.)
 //   - two concurrent deletes with the same prevVersion: one CAS removes
 //     the row; the other matches no row → re-read → `not-found`.
 //
 // On success we ONLY drop the live row — we do NOT unlink the live
-// blob. Blob reclamation is deferred to the reaper's grace-window GC
+// blob. Reclamation is deferred to the reaper's grace-window GC
 // (unlinks once no live row references the hash AND it's older than the
-// grace window) rather than done inline, so the drop stays lock-free
-// and can't race two things: a concurrent commit's promote→CAS window
-// (a just-promoted blob isn't referenced yet — the age grace protects
-// it) and an in-flight GET still streaming the bytes. NOT because the
-// hash might be shared — distinct PUTs get distinct hashes (random
-// nonce per encrypt → unique ciphertext), so the hash↔row mapping is
-// effectively 1:1; this delete simply orphans the blob for the GC.
+// grace window) so the drop stays lock-free and can't race a concurrent
+// commit's promote→CAS window (a just-promoted blob isn't referenced
+// yet — the age grace protects it) or an in-flight GET still streaming
+// the bytes. NOT because the hash might be shared — distinct PUTs get
+// distinct hashes (random nonce → unique ciphertext), so the hash↔row
+// mapping is effectively 1:1; this delete simply orphans the blob.
 export async function deleteObject(
   handle: Handle, tag: string, resourceTag: string, prevVersion: number | null, prevIncarnation: string | null,
 ): Promise<DeleteResult> {

package/{server → server-e2e}/objstore/tokens.ts

@@ -112,16 +112,14 @@ function isValidPayload(v: unknown): v is TokenPayload {
   const o = v as Record<string, unknown>
   if (typeof o['tag'] !== 'string' || typeof o['res'] !== 'string') return false
   // `Number.isSafeInteger` over `Number.isInteger`: `exp` is compared
-  // to `Date.now()` (which is itself a safe int), `len` is compared
-  // against the Content-Length the REST layer parsed (which already
-  // gates on `isSafeInteger`), and `ver` is compared against SQLite's
-  // INTEGER (capacity 2^63). An unsafe-but-integer value in the token
-  // round-trips through IEEE-754 and could spoof equality with a
-  // different actual value, or — for `exp` — let an adversarial
-  // token claim a `Number.MAX_SAFE_INTEGER + 1` expiry that compares
-  // ambiguously near the IEEE-754 boundary. Matches the rest of the
-  // codebase's safe-int gates (server/objstore/sign.ts,
-  // server/objstore/rest.ts, server/objstore/handlers.ts).
+  // to `Date.now()` (a safe int), `len` against the Content-Length the
+  // REST layer parsed (itself `isSafeInteger`-gated), and `ver` against
+  // SQLite's INTEGER (capacity 2^63). An unsafe-but-integer value in the
+  // token round-trips through IEEE-754 and could spoof equality with a
+  // different actual value, or — for `exp` — claim a
+  // `Number.MAX_SAFE_INTEGER + 1` expiry that compares ambiguously near
+  // the IEEE-754 boundary. Matches the codebase's other safe-int gates
+  // (server-e2e/objstore/sign.ts, rest.ts, handlers.ts).
   if (!Number.isSafeInteger(o['exp']) || (o['exp'] as number) < 0) return false
   if (o['op'] === 'put') {
     return typeof o['sid'] === 'string'
@@ -138,8 +136,7 @@ function isValidPayload(v: unknown): v is TokenPayload {
   return false
 }
 
-// Convenience constructors so the WS handler doesn't have to assemble
-// the payload shape inline.
+// Constructors so the WS handler doesn't assemble the payload inline.
 export function mintPutToken(
   secret: TokenSecret,
   tag: string, res: string, sid: string, len: number,

package/{server → server-e2e}/peer.ts

@@ -1,9 +1,8 @@
-// Per-connection server state. One `Peer` per accepted WebSocket,
-// replacing the former parallel per-socket WeakMaps (challenge /
-// authorized / alive / inflight / tags). The connection handler
-// constructs one on accept and holds it in a closure, so the hot
-// paths (message dispatch, pong, close) touch fields directly; the
-// few cross-function call sites look it up via `peers.get(socket)`.
+// Per-connection server state. One `Peer` per accepted WebSocket. The
+// connection handler constructs one on accept and holds it in a
+// closure, so the hot paths (message dispatch, pong, close) touch
+// fields directly; the few cross-function call sites look it up via
+// `peers.get(socket)`.
 //
 // Held in a `WeakMap<WebSocket, Peer>` so a closed socket's state GCs
 // with the socket; the close handler also `delete`s it explicitly
@@ -17,9 +16,8 @@ export class Peer {
   // `workspace-subscribe` signature — blocks cross-connection replay
   // of a captured subscribe frame.
   readonly challenge: string
-  // Password-gate authorization flag. Once the `authenticate`
-  // handshake succeeds, every subsequent first-action on this socket
-  // bypasses the new-workspace gate. `false` until then.
+  // Password-gate flag. Once the `authenticate` handshake succeeds,
+  // first-actions on this socket bypass the new-workspace gate.
   authorized = false
   // Heartbeat liveness. The sweep flips it `false` after each `ping()`;
   // the `pong` listener flips it back. A socket still `false` on the

package/{server → server-e2e}/pubsub.ts

@@ -1,5 +1,5 @@
 // Cross-instance pub/sub for real-time WS broadcasts. The triage-sync
-// fan-out is an in-memory subscriber map (server/hub.ts) by design — it
+// fan-out is an in-memory subscriber map (server-e2e/hub.ts) by design — it
 // routes a commit only to peers on the SAME instance. A multi-instance
 // deployment behind a load balancer needs commit-landed-on-A to reach
 // peers-on-B with the same latency the hub gives same-instance peers.
@@ -57,15 +57,15 @@ function newSenderId(): string {
 //   - 'objput': objstore-put broadcast. `res` is the resource tag; the
 //     receiver SELECTs the live row by (tag, res) for the rest of the
 //     metadata fields.
-//   - 'objdel': objstore-deleted broadcast. `ver` is the deleted version
-//     — inline because the row is gone from workspace_object after the
-//     delete commit.
+//   - 'objdel': objstore-deleted broadcast. `ver` and `incarnation` identify
+//     the deleted row, which is gone from workspace_object after commit.
+//     Incarnation is optional only for messages from older relay instances.
 export type BusMessage =
   | { kind: 'rev'; tag: string; id: string }
   | { kind: 'objput'; tag: string; res: string }
-  | { kind: 'objdel'; tag: string; res: string; ver: number }
+  | { kind: 'objdel'; tag: string; res: string; ver: number; incarnation?: string }
 
-// Receiver wired up by the hub layer (see server/index.ts). Each handler
+// Receiver wired up by the hub layer (see server-e2e/index.ts). Each handler
 // runs once per remote message; failures are logged but don't crash the
 // LISTEN loop — a missed broadcast surfaces to clients on reconnect
 // (chain re-pull). Async because the workspace-revision handler does a
@@ -164,14 +164,13 @@ type NeonState = {
   // In-flight bus-message handler promises. `dispatchNotification`
   // fires handlers fire-and-forget, but `stop()` awaits this set before
   // returning so the lifecycle's `handle.close()` (which runs after
-  // `pubsub.stop()` — see closeDb in server/index.ts) can't race a
+  // `pubsub.stop()` — see closeDb in server-e2e/index.ts) can't race a
   // handler mid-`handle.revisionById.get` / `getLive`.
   pendingHandlers: Set<Promise<void>>
 }
 
-// Notification dispatch — closes over `state.senderId` and
-// `state.handler`. Filters foreign channels (defensive) and our own
-// publish round-trip (Postgres NOTIFY delivers to publishers too).
+// Notification dispatch. Filters foreign channels (defensive) and our
+// own publish round-trip (Postgres NOTIFY delivers to publishers too).
 function dispatchNotification(state: NeonState, n: { channel: string; payload?: string }): void {
   if (n.channel !== CHANNEL) return
   if (typeof n.payload !== 'string') return
@@ -187,24 +186,19 @@ function dispatchNotification(state: NeonState, n: { channel: string; payload?:
   // notification dispatch (which would queue further notifications
   // behind it). Errors are logged but don't kill the loop — a missed
   // broadcast surfaces to clients on reconnect via the chain re-pull.
-  //
-  // The promise is also tracked in `state.pendingHandlers` so `stop()`
-  // can drain in-flight handlers BEFORE the lifecycle teardown closes
-  // the DB handle. Without the tracking, `onBusMessage`'s DB queries
-  // (`handle.revisionById.get` / `getLive`) could race
-  // `handle.close()` and throw inside a half-settled handler.
+  // Tracked in `state.pendingHandlers` so `stop()` drains in-flight
+  // handlers before the lifecycle closes the DB handle (see that
+  // field's doc).
   const promise: Promise<void> = fn(msg).catch((err) => {
     console.warn('pubsub: handler error:', errStack(err))
   }).finally(() => { state.pendingHandlers.delete(promise) })
   state.pendingHandlers.add(promise)
 }
 
-// Single connect attempt. Resolves once LISTEN is registered, or
-// rejects on transport / LISTEN failure. Assigns `state.client = c`
-// EAGERLY (before `c.connect()` is awaited) for the two invariants
-// documented on `NeonState.client`: the error handler's equality gate
-// must work mid-handshake, and `stop()` must be able to abort a hung
-// connect by reading `state.client?.end()`.
+// Single connect attempt. Resolves once LISTEN is registered, rejects
+// on transport / LISTEN failure. Assigns `state.client = c` EAGERLY
+// (before awaiting `c.connect()`) for the invariants on
+// `NeonState.client`.
 async function tryConnect(state: NeonState): Promise<void> {
   const c = state.newClient()
   state.client = c
@@ -221,13 +215,11 @@ async function tryConnect(state: NeonState): Promise<void> {
     })
     await c.query(`LISTEN ${CHANNEL}`)
   } catch (err) {
-    // Clear `state.client` only if it still points at OUR client (a
+    // Clear `state.client` only if it still points at OUR client — a
     // racing `stop()` may have already null'd it and ended the
-    // half-connected socket — don't clobber that). The `c.end()`
-    // below may be a redundant second call in that race (stop already
-    // ended it); pg-style Client.end() is idempotent so the second
-    // call is a harmless no-op. The try/catch additionally absorbs
-    // any rejection from end() itself.
+    // half-connected socket; don't clobber that. The `c.end()` below
+    // may then be a redundant second call, but pg-style Client.end()
+    // is idempotent (and the try/catch absorbs any rejection).
     if (state.client === c) state.client = null
     try { await c.end() } catch {}
     throw err
@@ -236,8 +228,7 @@ async function tryConnect(state: NeonState): Promise<void> {
 
 // Outer (re)connect loop. Exits silently on `state.stopped`; otherwise
 // retries with exponential backoff after a connect / LISTEN failure.
-// `tryConnect` assigns `state.client` itself (see the eager-assign
-// rationale on `NeonState.client`), so this loop only counts attempts
+// `tryConnect` owns `state.client`, so this loop only counts attempts
 // and runs the backoff sleep.
 async function connectAndListen(state: NeonState): Promise<void> {
   // oxlint-disable-next-line no-unmodified-loop-condition
@@ -262,10 +253,9 @@ async function connectAndListen(state: NeonState): Promise<void> {
 // default) waiting on the timer. We stash the cancel callback on the
 // shared state object so `stop()` can fire it; the loop's
 // `if (state.stopped) return` then exits on the next turn. The
-// `settled` flag guards against a race between the timer firing and
-// `stop()` racing the same wake-up (Promise resolves are idempotent
-// at the runtime layer, but oxlint's `no-multiple-resolved` rule is
-// stricter and the guard documents the mutual exclusion explicitly).
+// `settled` flag guards the timer-vs-`stop()` wake-up race: resolves
+// are idempotent at runtime, but it also satisfies oxlint's stricter
+// `no-multiple-resolved` and documents the mutual exclusion.
 function cancellableSleep(state: NeonState, ms: number): Promise<void> {
   return new Promise((resolve) => {
     let settled = false
@@ -343,7 +333,7 @@ export function createNeonPubSub(deps: NeonPubSubDeps): PubSub {
       if (state.connectAttempt) { try { await state.connectAttempt } catch {} }
       // Drain in-flight bus-message handlers BEFORE returning. The
       // lifecycle teardown runs `pubsub.stop()` and THEN
-      // `handle.close()` (see closeDb in server/index.ts); a handler
+      // `handle.close()` (see closeDb in server-e2e/index.ts); a handler
       // still in `handle.revisionById.get` / `getLive` would otherwise
       // throw against a closed DB. `allSettled` so one handler's
       // rejection doesn't abort the drain.
@@ -398,7 +388,10 @@ function parseBusMessage(raw: Record<string, unknown>): BusMessage | null {
     const ver = raw['ver']
     if (typeof res !== 'string') return null
     if (typeof ver !== 'number' || !Number.isSafeInteger(ver)) return null
-    return { kind: 'objdel', tag, res, ver }
+    const incarnation = raw['incarnation']
+    if (incarnation === undefined) return { kind: 'objdel', tag, res, ver }
+    if (typeof incarnation !== 'string' || incarnation.length === 0) return null
+    return { kind: 'objdel', tag, res, ver, incarnation }
   }
   return null
 }