@preventive/triage 1.0.0-alpha.2 → 1.0.0-alpha.4

package/server/objstore/rest.ts

@@ -41,20 +41,18 @@ import {
 } from './store.ts'
 import type { LiveReader } from './blob.ts'
 import { type TokenSecret, extractBearer, verifyToken } from './tokens.ts'
-import { errStack } from '../util.ts'
+import { debugId, debugTag, errMsg, errStack } from '../util.ts'
 
 // Server-side fault codes that should surface as 500 `io-error`
 // rather than 400 `aborted`. `pipeline(req, ws)` rejects with the
-// first stream error; the client-side codes (socket close, the
-// manual `overrun`) are everything else. ENOENT is included because
-// `createWriteStream` will reject with it if the `${tag}/.staging`
-// dir was removed out from under us (operator action / external
-// fs activity) — that's a server-side state, not a client-fixable
-// abort. PR #4 review. The Vercel-blob backend surfaces failures
-// through other paths (rejected put promise → caught by the
-// pipeline catch as a plain Error without a `code`); those land in
-// the default `aborted` branch and the operator-facing log line
-// carries the SDK's error message.
+// first stream error; client-side codes (socket close, the manual
+// `overrun`) are everything else. ENOENT is included because
+// `createWriteStream` rejects with it if the `${tag}/.staging` dir
+// was removed out from under us (operator / external fs activity) —
+// a server-side state, not a client-fixable abort. Vercel-blob
+// failures arrive without a `code` (rejected put promise → plain
+// Error); those land in the default `aborted` branch, with the SDK
+// error in the operator log line.
 const IO_FAULT_CODES = new Set(['ENOSPC', 'EACCES', 'EROFS', 'EIO', 'EMFILE', 'ENFILE', 'EDQUOT', 'EPERM', 'ENOENT'])
 
 export type ObjstoreRestDeps = {
@@ -71,35 +69,34 @@ export type ObjstoreRestDeps = {
   debug: boolean
 }
 
-// Concurrent commits need no lock at all: the live blob is content-
-// addressed (`${tag}/${contentHash}.bin`) so two racing commits write
-// to DIFFERENT immutable addresses, and the commit itself is an atomic
-// version compare-and-set on the live row (see commitPut in store.ts).
-// Exactly one racer wins the CAS; the loser gets a 409 `conflict` and
-// rebases. This holds within a single process and across replicas —
-// there is no in-process mutex serialising commits. (The PUT *body*
-// does take a per-process single-writer reservation — `inFlightSids`
-// below — but that only rejects a duplicate upload of one staging
-// slot; it never makes distinct commits wait.)
+// Concurrent commits need no lock: the live blob is content-addressed
+// (`${tag}/${contentHash}.bin`) so two racing commits write to
+// DIFFERENT immutable addresses, and the commit is an atomic version
+// compare-and-set on the live row (see commitPut in store.ts). Exactly
+// one racer wins the CAS; the loser gets a 409 `conflict` and rebases.
+// Holds within a process and across replicas — no in-process mutex
+// serialises commits. (The PUT *body* takes a per-process single-
+// writer reservation — `inFlightSids` below — but that only rejects a
+// duplicate upload of one staging slot; distinct commits never wait.)
 
 // Per-process single-writer guard for the REST PUT body. A put-token is
 // a REUSABLE bearer capability (tokens.ts) valid for its whole TTL, so a
-// client replaying it on overlapping PUTs — a retry that doesn't cancel
-// the in-flight request, a proxy re-issuing the PUT, a double-submit —
+// client replaying it on overlapping PUTs (a retry that doesn't cancel
+// the in-flight request, a proxy re-issuing the PUT, a double-submit)
 // would otherwise have two requests stream into the SAME staging file
-// (same sid). On the FS backend `createWriteStream(…, { flags: 'w' })`
-// truncates, so the two writers clobber each other's bytes/size and BOTH
-// can fail (size-mismatch 400 + promote-race 500) with neither
-// committing. `inFlightSids` admits exactly ONE in-flight upload per
-// staging id; a concurrent same-token PUT is rejected (409) before it
-// can open a second writer. The slot is held only across the body +
-// commit and released in a `finally` (and bounded by the REST idle-body
-// timeout in server/http.ts so a slow-loris can't pin it). It is NOT a
-// commit lock — commits stay lock-free on the version-CAS above. It's
-// per-process: the Vercel multi-replica path leans on `allowOverwrite:
-// true` + the commit CAS + content-addressing for cross-replica
-// correctness (a duplicate that lands on another replica can't clobber a
-// shared local file and still loses the CAS), not on this set.
+// (same sid). On FS, `createWriteStream(…, { flags: 'w' })` truncates,
+// so the two writers clobber each other's bytes/size and BOTH can fail
+// (size-mismatch 400 + promote-race 500) with neither committing.
+// `inFlightSids` admits exactly ONE in-flight upload per staging id; a
+// concurrent same-token PUT is rejected (409) before it can open a
+// second writer. The slot is held only across body + commit, released
+// in a `finally`, and bounded by the REST idle-body timeout in
+// server/http.ts so a slow-loris can't pin it. NOT a commit lock —
+// commits stay lock-free on the version-CAS above. Per-process: the
+// Vercel multi-replica path leans on `allowOverwrite: true` + the
+// commit CAS + content-addressing for cross-replica correctness (a
+// duplicate landing on another replica can't clobber a shared local
+// file and still loses the CAS), not on this set.
 const inFlightSids = new Set<string>()
 
 // `/api/objstore/${workspaceTag}/${resourceTag}` — base64url
@@ -121,14 +118,13 @@ export function matchRoute(url: string | undefined): RouteMatch | null {
 
 function deny(res: ServerResponse, status: number, body: string): void {
   // Uniform `{ error: <reason> }` JSON envelope for every failure so
-  // clients have one shape to parse. Status + reason are NOT
-  // intentionally indistinguishable across causes — 401, 404, 405,
-  // 410, 411, 500 each map to a documented reason in server/README.md
-  // and the client decides recovery from the code. Defense against
-  // probe-driven distinguishing isn't a property the relay aims for;
-  // every error reason is reachable only after the route + bearer-
-  // token check passes (or as 401/404 from the public surface), so
-  // there's no signal here a probe couldn't otherwise enumerate.
+  // clients parse one shape. Status + reason are NOT intentionally
+  // indistinguishable across causes — 401/404/405/410/411/500 each map
+  // to a documented reason in server/README.md and the client decides
+  // recovery from the code. Probe-distinguishing defense isn't a goal:
+  // every reason is reachable only after the route + bearer-token check
+  // passes (or as 401/404 from the public surface), so a probe gains no
+  // signal it couldn't otherwise enumerate.
   res.writeHead(status, { 'content-type': 'application/json' })
   res.end(JSON.stringify({ error: body }))
 }
@@ -209,21 +205,20 @@ async function handleRestPut(
   // No lock: the commit's version-CAS arbitrates concurrent commits
   // (the live blob is content-addressed, so racers can't desync
   // metadata vs bytes — the loser surfaces a 409 `conflict`). The
-  // staging row is protected from the reaper not by a lock but by its
-  // `begun_at`: an upload under the staging TTL stays fresh through the
-  // body, and the after-body `refreshStagingBegunAt` re-extends the
-  // TTL across the commit. (An upload exceeding the TTL during the
-  // body can be reaped mid-flight → commit 410s; documented accepted
-  // tradeoff — see commitPut in store.ts.)
+  // staging row is protected from the reaper by its `begun_at`, not a
+  // lock: an upload under the staging TTL stays fresh through the body,
+  // and the after-body `refreshStagingBegunAt` re-extends the TTL across
+  // the commit. (An upload exceeding the TTL during the body can be
+  // reaped mid-flight → commit 410s; documented accepted tradeoff —
+  // see commitPut in store.ts.)
   await handleRestPutBody(deps, req, res, route, payload, declared)
 }
 
 // Map a `commitPut` failure to its wire response. Extracted from
-// `handleRestPutBody` to keep it under the per-function line cap
-// and to make the wire-mapping ladder its own audit surface — the
-// exhaustiveness `never` guard at the bottom catches a forward-
-// compat hazard where a new `CommitPutResult` reason lands without
-// updating this dispatch.
+// `handleRestPutBody` to keep it under the per-function line cap and
+// to make the wire-mapping ladder its own audit surface — the
+// exhaustiveness `never` guard at the bottom catches a new
+// `CommitPutResult` reason landing without updating this dispatch.
 function denyCommitFailure(res: ServerResponse, result: Exclude<CommitPutResult, { ok: true }>): void {
   if (result.reason === 'conflict') {
     denyConflict(res, result.conflict?.version ?? null, result.conflict?.incarnation ?? null)
@@ -273,10 +268,9 @@ async function handleRestPutBody(
   }
   inFlightSids.add(payload.sid)
   try {
-    // Stream the upload + commit, no lock. The commit's version-CAS
-    // arbitrates concurrent commits; the staging row is kept fresh for
-    // the reaper by its `begun_at` (+ the after-body refresh), not by a
-    // mutex. See the rationale in handleRestPut above.
+    // Stream the upload + commit, no lock — version-CAS arbitrates
+    // commits, the staging row stays fresh for the reaper via its
+    // `begun_at` (+ after-body refresh). See handleRestPut above.
     const result = await runUploadAndCommit(deps, route, payload, declared, req, res)
     if (result.handled) return
     if (!result.commit.ok) { denyCommitFailure(res, result.commit); return }
@@ -294,15 +288,13 @@ async function handleRestPutBody(
       ...objectMetaWire(row),
     }, null)
     // Cross-instance fan-out (Neon mode). The bus payload carries only
-    // (tag, resourceTag); peers on other instances re-fetch the live
-    // row from workspace_object to compose their local broadcast. The
-    // committed row is durable by here (commitPut's version-CAS already
-    // landed), so the receiver typically sees either THIS version or a
-    // STRICTLY newer one (also a valid broadcast — clients are
-    // idempotent on (resourceTag, version)). The receiver is allowed
-    // to find no live row at all if a subsequent delete races the
-    // notification; bus-receiver.ts drops that case silently. SQLite
-    // mode publishes to a no-op.
+    // (tag, resourceTag); peers on other instances re-fetch the live row
+    // from workspace_object to compose their broadcast. The committed
+    // row is durable by here (commitPut's version-CAS landed), so the
+    // receiver sees THIS version or a STRICTLY newer one (also valid —
+    // clients are idempotent on (resourceTag, version)), or no live row
+    // at all if a subsequent delete races the notification (bus-
+    // receiver.ts drops that silently). SQLite mode publishes to a no-op.
     deps.publishObjPut(route.tag, route.resourceTag)
     if (deps.debug) console.log(`objstore put → ${route.tag.slice(0, 12)}…/${route.resourceTag.slice(0, 8)}… v${row.version}`)
   } finally {
@@ -439,7 +431,10 @@ async function runUploadAndCommit(
 type GetOpened =
   | { reason: 'ok'; reader: LiveReader }
   | { reason: 'not-found' }
-  | { reason: 'unavailable' }
+  // `detail` is a short non-sensitive cause tag (backend sub-reason +
+  // content-hash prefix) the GET handler logs so a 503 is diagnosable —
+  // every byte-side failure collapses to the same wire 503 otherwise.
+  | { reason: 'unavailable'; detail: string }
 
 async function openLiveSnapshot(
   deps: ObjstoreRestDeps, route: RouteMatch, payload: { ver: number; inc: string },
@@ -451,22 +446,32 @@ async function openLiveSnapshot(
   // bytes behind `live.content_hash` can never change underneath us —
   // the worst a race can do is have the reaper GC an already-superseded
   // hash just before we open it, which surfaces as openLiveReader
-  // not-found → `unavailable` → 503, and the client refetches. We can
+  // returning `unavailable` → 503, and the client refetches. We can
   // never serve torn or wrong bytes. (For the FS backend the open also
   // returns a pinned fd; for the Vercel backend a fetch-backed stream.)
   const live = await deps.handle.selectLiveOne.get(route.tag, route.resourceTag)
   if (!live || live.version !== payload.ver || live.incarnation !== payload.inc) return { reason: 'not-found' }
+  // Tag the content hash into every `unavailable` detail so an operator
+  // can go check the byte store directly for THIS blob (gone → reaper /
+  // deletion; present → transient read fault).
+  const hashTag = `hash=${debugId(live.content_hash)}`
   let opened
   try { opened = await deps.handle.blob.openLiveReader(route.tag, live.content_hash) }
-  catch { return { reason: 'unavailable' } }
-  if (!opened.ok) return { reason: opened.reason }
+  // Length-cap the thrown error text: today a non-BlobNotFound SDK throw
+  // (BlobServiceNotAvailable / store-not-found) carries no credential
+  // (the RW token rides the Authorization header, never `.message`), but
+  // a future SDK could embed a signed URL / token fragment — bound the
+  // log line so it can't dump one verbatim. Mirrors the `.slice(0, 200)`
+  // cap used on SDK error text in blob-vercel.ts.
+  catch (err) { return { reason: 'unavailable', detail: `open-threw ${hashTag} ${String(errMsg(err)).slice(0, 200)}` } }
+  if (!opened.ok) return { reason: 'unavailable', detail: `${opened.detail ?? 'backend'} ${hashTag}` }
   // Size mismatch between the live row and the on-storage bytes
   // is a transient inconsistency — reaper will reconcile. Close
   // the reader before returning so we don't leak the fd / fetch
   // reader. PR #4 review H8.
   if (opened.reader.size !== live.content_length) {
     await opened.reader.close().catch(() => {})
-    return { reason: 'unavailable' }
+    return { reason: 'unavailable', detail: `size-mismatch row=${live.content_length} blob=${opened.reader.size} ${hashTag}` }
   }
   return { reason: 'ok', reader: opened.reader }
 }
@@ -486,8 +491,19 @@ async function handleRestGet(
   // If the live row is there but the bytes are missing / wrong size,
   // it's a transient inconsistency the reaper will sort out — 503
   // (vs 404) tells the client this is a server-side state, not a
-  // "the resource truly isn't there" answer.
-  if (opened.reason === 'unavailable') { deny(res, 503, 'unavailable'); return }
+  // "the resource truly isn't there" answer. Log the cause
+  // UNCONDITIONALLY (not behind `debug`): a 503 means a live row whose
+  // bytes can't be served, and the wire response can't distinguish a
+  // permanent loss (reaper GC'd referenced bytes) from a transient read
+  // fault. `detail` carries the backend sub-reason + content-hash prefix
+  // so an operator can tell which — the only server-side breadcrumb for
+  // the "all data turned into 503" failure. (Volume is bounded: a 503 is
+  // an error path; a workspace-wide outage is exactly when these are
+  // wanted.)
+  if (opened.reason === 'unavailable') {
+    console.warn(`objstore-get: 503 unavailable ${debugTag(route.tag)}/${route.resourceTag.slice(0, 8)}… v${payload.ver} ${opened.detail}`)
+    deny(res, 503, 'unavailable'); return
+  }
   res.writeHead(200, {
     'content-type': 'application/octet-stream',
     'content-length': String(opened.reader.size),

package/server/objstore/store-neon.ts

@@ -46,10 +46,10 @@ const DDL_LOCK_KEY_OBJSTORE_SUB = 0x6f62_6a73 // 'objs'
 // defend the commitPut conflict arithmetic — a manual `UPDATE
 // workspace_object SET version = -1` would otherwise round-trip
 // through `num()` (which only rejects non-safe-integers) and corrupt
-// the version monotonicity invariant. The SQLite schema carries the
-// identical CHECKs (see `store.ts`) — STRICT there enforces the column
-// TYPE but NOT this value domain (`version = -1` is a valid integer
-// STRICT accepts), so both backends need the explicit CHECKs.
+// version monotonicity. SQLite carries the identical CHECKs (see
+// `store.ts`): STRICT there enforces column TYPE but NOT this value
+// domain (`version = -1` is a valid integer STRICT accepts), so both
+// backends need the explicit CHECKs.
 const SCHEMA_PG = [
   `CREATE TABLE IF NOT EXISTS workspace_object (
      workspace_tag  TEXT    NOT NULL,

package/server/objstore/store.ts

@@ -79,9 +79,8 @@ const SCHEMA = `
 
 // One LIVE row, exactly the shape the `workspace-subscribed` ack's
 // `resources` array carries on the wire, minus `keyframe`-style
-// server-only flags. `put_at` is a
-// debug aid the wire format doesn't include — operators can inspect
-// it via the DB but the server never volunteers it.
+// server-only flags. `put_at` is a debug aid the wire format never
+// includes — inspectable via the DB but the server never volunteers it.
 export type ObjectRow = {
   resourceTag: string
   version: number
@@ -118,13 +117,11 @@ export type BeginPutInput = {
 // PUT will reference. `workspace-full` is the per-workspace resource-
 // count cap rejection — see `MAX_RESOURCES_PER_WORKSPACE`.
 //
-// `filePath` is set only for the FS-backed handle (where it's the
-// absolute on-disk staging path); the Vercel-Blob handle omits it
-// since "path" isn't a meaningful concept against a remote object
-// store. Production code (rest.ts) never reads this field — the
-// REST layer goes through `handle.blob.openStagingWriter(tag, sid)`.
-// Tests for the FS path use it as a convenience to write fixture
-// bytes directly to the staging slot.
+// `filePath` is the absolute on-disk staging path, set only for the
+// FS handle; the Vercel handle omits it ("path" is meaningless against
+// a remote store). Production (rest.ts) never reads it — it goes
+// through `handle.blob.openStagingWriter(tag, sid)`. FS-path tests use
+// it as a convenience to write fixture bytes to the staging slot.
 export type BeginPutResult =
   | { ok: true; stagingId: string; filePath?: string }
   | { ok: false; reason: 'conflict'; conflict: ObjectRow | null }
@@ -134,17 +131,14 @@ export type CommitPutInput = {
   workspaceTag: string
   resourceTag: string
   stagingId: string
-  // Optional: the storage-side byte count the caller already
-  // verified after the upload landed. When provided, commitPut
-  // skips the otherwise-redundant `statStaging` round-trip — for
-  // the Vercel backend that's one fewer HTTP HEAD per PUT.
-  // Caller must ONLY pass a value it observed for THIS stagingId
-  // after its upload finished (i.e. the REST PUT path's post-upload
-  // `statStaging`). Safe without a lock because staging ids are
-  // random — no other request writes this blob — and the sole writer
-  // (this PUT) has already completed. Tests that drive commitPut
-  // directly without the REST layer should omit this and let
-  // commitPut stat for itself.
+  // Optional storage-side byte count the caller already verified
+  // post-upload. When provided, commitPut skips the redundant
+  // `statStaging` round-trip — one fewer Vercel HEAD per PUT. Caller
+  // must ONLY pass a value it observed for THIS stagingId after its
+  // upload finished (the REST PUT path's post-upload `statStaging`).
+  // Safe without a lock: staging ids are random (no other request
+  // writes this blob) and the sole writer (this PUT) has finished.
+  // Tests driving commitPut directly omit this and let it stat.
   observedSize?: number
 }
 
@@ -181,9 +175,9 @@ type DbRow = {
 //   - the reaper's age grace window + an atomic conditional staging
 //     delete (`deleteStagingIfStale`) so the stale-staging sweep
 //     can't race an upload that just finished.
-// These hold both within a single process AND across replicas — the
-// old per-(tag, resourceTag) in-process mutex added nothing the CAS +
-// content-addressing didn't already give, so it was removed.
+// These hold both within a single process AND across replicas, so no
+// per-(tag, resourceTag) in-process mutex is needed — the CAS +
+// content-addressing already cover everything one would.
 export type Handle = {
   // SQLite-only: the underlying `DatabaseSync`. Unset on the Neon
   // backend (see ./store-neon.ts). Test-only fixture SQL routes
@@ -212,15 +206,14 @@ export type Handle = {
   selectStagingByWsSid: GetStmt<[string, string], unknown>
   refreshStagingBegunAt: RunStmt<[number, string, string, string]>
   deleteStaging: RunStmt<[string, string, string]>
-  // Atomic conditional staging delete used by the reaper's stale-row
+  // Atomic conditional staging delete for the reaper's stale-row
   // sweep. `deleteStagingIfStale(tag, res, sid, staleBefore)` deletes
   // the row IFF its `begun_at < staleBefore`, returning `{ ok: 1 }`
-  // when a row was actually removed and `undefined` otherwise. This is
-  // the lock-free replacement for the old in-lock begun_at re-read
-  // (PR #4 "F1"): a slow PUT that finishes and calls
-  // `refreshStagingBegunAt` bumps `begun_at` fresh, so a concurrent
-  // reaper's conditional delete simply doesn't match (its predicate
-  // fails atomically) and the row survives for the commit. Mirrors the
+  // when a row was actually removed and `undefined` otherwise. Lock-
+  // free race guard (PR #4 "F1"): a slow PUT that finishes calls
+  // `refreshStagingBegunAt` to bump `begun_at` fresh, so a concurrent
+  // reaper's conditional delete doesn't match (predicate fails
+  // atomically) and the row survives for the commit. Mirrors the
   // `insertLiveIfAbsent` RETURNING pattern.
   deleteStagingIfStale: GetStmt<[string, string, string, number], { ok: number }>
   selectLive: AllStmt<[string], DbRow>
@@ -279,9 +272,6 @@ export type SqliteHandle = Handle & { db: DatabaseSync }
 // transient over-shoot under high concurrency across DIFFERENT
 // resources is bounded by `(parallel new-resource begins - 1)` and is
 // accepted (the cap is a soft policy bound, not a security invariant).
-// This was already the contract under the old per-resource lock —
-// that lock keyed on (tag, resourceTag), so concurrent NEW-resource
-// begins held DIFFERENT locks and raced the count anyway.
 export const MAX_RESOURCES_PER_WORKSPACE = 100
 
 // Per-upload byte cap, shared by the WS plane (rejects oversize
@@ -372,12 +362,10 @@ export function openObjstore(db: DatabaseSync, dir: string): SqliteHandle {
   const blob = openFsBlobBackend(dir)
   // No `close` method on the returned Handle: the underlying
   // `DatabaseSync` is owned by the caller (in production, the
-  // workspace_revision handle in `server/db.ts`, which closes it
-  // from `shutdown()`). Exposing `close()` here was misleading —
-  // a callsite reading `await objstoreHandle.close()` would
-  // reasonably assume it closes something, when in practice it
-  // either no-op'd (production) or left the connection open
-  // (tests construct their own DB and `db.close()` separately).
+  // workspace_revision handle in `server/db.ts`, closed from
+  // `shutdown()`; tests close their own DB). A `close()` here would
+  // mislead — it could only no-op or leak, never close the caller-
+  // owned connection.
   return {
     db,
     blob,
@@ -522,11 +510,10 @@ export async function beginPut(handle: Handle, input: BeginPutInput): Promise<Be
   if (liveVersion !== input.prevVersion || liveIncarnation !== input.prevIncarnation) {
     return { ok: false, reason: 'conflict', conflict: live }
   }
-  // Per-workspace resource cap. Only enforced for NEW resources —
-  // re-uploads of an existing resourceTag (live != null) don't
-  // change the count, so they're always allowed. Not atomic with the
-  // insert below (see MAX_RESOURCES_PER_WORKSPACE) — a soft policy
-  // bound, accepted to over-shoot under concurrent NEW-resource begins.
+  // Per-workspace resource cap, NEW resources only — re-uploads
+  // (live != null) don't change the count, so they're always allowed.
+  // Not atomic with the insert below; soft bound, see
+  // MAX_RESOURCES_PER_WORKSPACE.
   if (!live) {
     const count = await handle.countLive.get(input.workspaceTag) as { c: number } | undefined
     if ((count?.c ?? 0) >= MAX_RESOURCES_PER_WORKSPACE) {
@@ -557,51 +544,48 @@ export async function beginPut(handle: Handle, input: BeginPutInput): Promise<Be
 // IF ABSENT; a re-upload bumps the version IFF it still matches the
 // precondition we read. Exactly one of N racing commits wins the CAS;
 // the losers get `conflict` (with the current live row) and rebase.
-// Because each PUT is content-addressed at its OWN hash (distinct PUTs
-// get distinct hashes — random nonce per encrypt), N racing commits
-// promote to N DIFFERENT immutable paths: no promote clobbers another's
-// bytes, and a loser's blob is just left unreferenced for the GC. There
-// is no metadata-vs-bytes desync to guard against. The
-// CAS provides the commit's atomicity both within a process and across
-// replicas, so NO in-process lock is taken. A crash between the
-// promote and the CAS leaves the staging blob/row intact alongside (at
-// most) a stranded, unreferenced live blob — the reaper's stale-staging
-// sweep cleans the row, and the GC reaps the unreferenced live blob
-// once it's past the grace window, matching the "stranded state,
-// reaper-cleaned, never row-pointing-at-nothing" crash-safety contract.
+// Each PUT is content-addressed at its OWN hash (distinct PUTs get
+// distinct hashes — random nonce per encrypt), so N racers promote to
+// N DIFFERENT immutable paths: no promote clobbers another's bytes, a
+// loser's blob is just left unreferenced for the GC, and there's no
+// metadata-vs-bytes desync to guard. The CAS gives atomicity both
+// within a process and across replicas, so NO in-process lock is taken.
+// A crash between the promote and the CAS leaves the staging blob/row
+// intact alongside (at most) a stranded, unreferenced live blob — the
+// stale-staging sweep cleans the row, the GC reaps the blob once past
+// the grace window: the "stranded state, reaper-cleaned, never row-
+// pointing-at-nothing" crash-safety contract.
 //
-// ACCEPTED TRADEOFF (lock removal): with no lock, the reaper no longer
-// WAITS for an in-flight upload on this key. An upload taking >1h FROM
-// BEGIN (i.e. exceeding the staging TTL during the body) can have its
-// staging row reaped mid-flight by `deleteStagingIfStale`; this commit
-// then sees no staging row and returns `no-staging` → REST 410. The
-// previous lock made the reaper block on any in-flight upload
-// (unbounded). Sub-1h uploads are unaffected: `begun_at` (set at begin)
-// stays within the TTL through the body, so the conditional delete
-// can't match, and the after-body `refreshStagingBegunAt` re-extends
-// the TTL to cover this commit step. This matches the staging TTL's
-// documented intent ("1h, comfortably over a 50 MiB upload on a slow
-// line").
+// ACCEPTED TRADEOFF (lock removal): without a lock the reaper no longer
+// waits for an in-flight upload on this key (the old lock blocked it on
+// any in-flight upload, unbounded). An upload taking >1h FROM BEGIN
+// (exceeding the staging TTL during the body) can have its staging row
+// reaped mid-flight by `deleteStagingIfStale` → this commit sees no
+// staging row → `no-staging` → REST 410. Sub-1h uploads are unaffected:
+// `begun_at` (set at begin) stays within the TTL through the body, so
+// the conditional delete can't match, and the after-body
+// `refreshStagingBegunAt` re-extends the TTL to cover this commit step —
+// matching the staging TTL's intent ("1h, comfortably over a 50 MiB
+// upload on a slow line").
 export async function commitPut(handle: Handle, input: CommitPutInput): Promise<CommitPutResult> {
   const staging = await handle.selectStaging.get(input.workspaceTag, input.resourceTag, input.stagingId)
   if (!staging) return { ok: false, reason: 'no-staging' }
   let stagedSize: number | null
   // statStaging failure here is a server-side issue (staging file
-  // was unlinked by a racing abort / reaper, EACCES, EIO, backend
+  // unlinked by a racing abort / reaper, EACCES, EIO, backend
   // unreachable, …) — not a client length-mismatch. Route through
   // `io-error` so the REST layer returns 5xx, not 400. PR #4 review.
   //
   // The REST PUT layer already statted the staging blob post-upload
   // and threads the result in via `observedSize` — skipping the
-  // round-trip saves one Vercel HEAD per PUT. The staging blob can't
-  // have been resized between that stat and here: staging ids are
-  // 16-byte random, so no other request targets this blob, and the
-  // sole writer (this PUT's upload pipeline) has already finished
-  // before the REST stat ran. The only other actor that touches a
-  // staging blob is the reaper, which UNLINKS (it never resizes); a
-  // racing reaper unlink surfaces below as statStaging→io-error or a
-  // promote failure, not a wrong size. WS / test paths that omit
-  // `observedSize` fall through to the explicit stat.
+  // round-trip saves one Vercel HEAD per PUT. The blob can't have been
+  // resized between that stat and here: staging ids are 16-byte random
+  // (no other request targets this blob) and the sole writer (this
+  // PUT's upload pipeline) has finished. The only other actor on a
+  // staging blob is the reaper, which UNLINKS (never resizes); a racing
+  // reaper unlink surfaces below as statStaging→io-error or a promote
+  // failure, not a wrong size. WS / test paths that omit `observedSize`
+  // fall through to the explicit stat.
   if (input.observedSize === undefined) {
     try { stagedSize = await handle.blob.statStaging(input.workspaceTag, input.stagingId) }
     catch { return { ok: false, reason: 'io-error' } }
@@ -637,15 +621,14 @@ export async function commitPut(handle: Handle, input: CommitPutInput): Promise<
   }
   // Promote to the CONTENT-ADDRESSED live path `${tag}/${hash}.bin`.
   // `promoteStagingToLive` returns false on any backend error (FS:
-  // EACCES / ENOSPC / EIO / a racing abort that already unlinked
-  // the staging file; Vercel: copy failure). 'io-error' is mapped
-  // to HTTP 500 by the REST layer — it's a server-side fault, not
-  // a client-fixable one. Because the destination path IS the content
-  // hash, any write to it is byte-identical BY CONSTRUCTION, so a
-  // retried or racing promote to the same path is an idempotent
-  // rewrite, never a clobber. (Distinct PUTs get distinct hashes — a
-  // fresh random nonce per encrypt makes each ciphertext unique — so
-  // concurrent commits to the same resource write to DIFFERENT paths.)
+  // EACCES / ENOSPC / EIO / a racing abort that already unlinked the
+  // staging file; Vercel: copy failure) → 'io-error' → REST HTTP 500
+  // (server-side fault, not client-fixable). The destination path IS
+  // the content hash, so any write to it is byte-identical BY
+  // CONSTRUCTION: a retried or racing promote to the same path is an
+  // idempotent rewrite, never a clobber. (Distinct PUTs get distinct
+  // hashes — fresh random nonce per encrypt — so concurrent commits to
+  // the same resource write to DIFFERENT paths.)
   if (!await handle.blob.promoteStagingToLive(input.workspaceTag, input.stagingId, staging.content_hash)) {
     return { ok: false, reason: 'io-error' }
   }
@@ -658,13 +641,12 @@ export async function commitPut(handle: Handle, input: CommitPutInput): Promise<
   const committedIncarnation = staging.prev_version == null ? freshIncarnation : staging.prev_incarnation!
   const putAt = Date.now()
   // Version-CAS in try/catch so a Neon transient (5xx, network
-  // hiccup, connection-pool exhaustion) doesn't bypass the abortPut
-  // ladder by throwing out of commitPut. Without this guard, a thrown
-  // rejection skips the REST layer's `if (!r.ok) abortPut` branch and
-  // bubbles to handleRest's outer catch — the live blob is already
-  // promoted, the staging blob + row stay, and the client sees a 500.
-  // The reaper reconciles (stale-staging sweep + unreferenced-blob
-  // GC) but the surface is a 500 the caller can retry.
+  // hiccup, pool exhaustion) doesn't throw out of commitPut and bypass
+  // the abortPut ladder. A thrown rejection would skip the REST layer's
+  // `if (!r.ok) abortPut` branch and bubble to handleRest's outer catch
+  // — live blob already promoted, staging blob + row left behind. The
+  // reaper reconciles (stale-staging sweep + unreferenced-blob GC); the
+  // surface is a 500 the caller can retry.
   let won: { ok: number } | undefined
   try {
     if (staging.prev_version == null) {
@@ -753,30 +735,28 @@ export async function abortPut(handle: Handle, tag: string, resourceTag: string,
 // `deletedVersion = 0` sentinel tells the broadcast path to skip.
 //
 // No lock: the drop is a version-CAS (`deleteLiveCAS` — DELETE WHERE
-// version = prev), so every race resolves to exactly one winner, the
-// same as the old lock did, with no lost update:
+// version = prev), so every race resolves to exactly one winner with
+// no lost update:
 //   - delete vs. a concurrent COMMIT on the same resource: whichever
 //     CAS lands first wins. A stale delete can NOT remove a row the
-//     commit just bumped — the `WHERE version = prev` no longer matches,
-//     so the delete gets `conflict` (re-read sees the bumped version).
-//     If the delete wins, the commit's `updateLiveCAS` matches no row →
-//     `conflict`. (The earlier draft used an UNCONDITIONAL delete here,
-//     which — without the lock — let `getLive` read v1, a commit bump
-//     to v2 slip in, and the delete then destroy v2: a lost update. The
-//     version-CAS closes that.)
+//     commit just bumped — `WHERE version = prev` no longer matches, so
+//     the delete gets `conflict` (re-read sees the bumped version). If
+//     the delete wins, the commit's `updateLiveCAS` matches no row →
+//     `conflict`. (An UNCONDITIONAL delete here would let `getLive`
+//     read v1, a commit bump to v2 slip in, then the delete destroy v2:
+//     a lost update. The version-CAS closes that.)
 //   - two concurrent deletes with the same prevVersion: one CAS removes
 //     the row; the other matches no row → re-read → `not-found`.
 //
 // On success we ONLY drop the live row — we do NOT unlink the live
-// blob. Blob reclamation is deferred to the reaper's grace-window GC
+// blob. Reclamation is deferred to the reaper's grace-window GC
 // (unlinks once no live row references the hash AND it's older than the
-// grace window) rather than done inline, so the drop stays lock-free
-// and can't race two things: a concurrent commit's promote→CAS window
-// (a just-promoted blob isn't referenced yet — the age grace protects
-// it) and an in-flight GET still streaming the bytes. NOT because the
-// hash might be shared — distinct PUTs get distinct hashes (random
-// nonce per encrypt → unique ciphertext), so the hash↔row mapping is
-// effectively 1:1; this delete simply orphans the blob for the GC.
+// grace window) so the drop stays lock-free and can't race a concurrent
+// commit's promote→CAS window (a just-promoted blob isn't referenced
+// yet — the age grace protects it) or an in-flight GET still streaming
+// the bytes. NOT because the hash might be shared — distinct PUTs get
+// distinct hashes (random nonce → unique ciphertext), so the hash↔row
+// mapping is effectively 1:1; this delete simply orphans the blob.
 export async function deleteObject(
   handle: Handle, tag: string, resourceTag: string, prevVersion: number | null, prevIncarnation: string | null,
 ): Promise<DeleteResult> {