@preventive/triage 1.0.0-alpha.2 → 1.0.0-alpha.21

package/{server → server-e2e}/objstore/init.ts

@@ -15,16 +15,22 @@ import { errStack } from '../util.ts'
 
 export type ObjstoreInitDeps = {
   // Pre-opened storage handle, sharing the SQLite connection with
-  // the workspace_revision handle in server/db.ts.
+  // the workspace_revision handle in server-e2e/db.ts.
   handle: Handle
   reapIntervalMs: number
+  // Automatic-sweep switch (OBJSTORE_REAP_DISABLED). When true, NEITHER the boot
+  // sweep nor the periodic timer runs: `startupReap` resolves immediately
+  // and no timer is installed. Orphaned/superseded blobs and stale
+  // staging rows then accumulate unbounded — only safe if an external job
+  // handles GC. Omitted/false → reaper runs (the default). See config.ts.
+  reapDisabled?: boolean
   send: (socket: WebSocket, msg: object) => void
   broadcast: (tag: string, msg: object, except: WebSocket | null) => void
   // Cross-instance pub/sub publishers. SQLite mode passes no-ops; Neon
   // mode passes Postgres LISTEN/NOTIFY-backed implementations. See
-  // server/pubsub.ts for the bus design.
+  // server-e2e/pubsub.ts for the bus design.
   publishObjPut: (tag: string, resourceTag: string) => void
-  publishObjDeleted: (tag: string, resourceTag: string, version: number) => void
+  publishObjDeleted: (tag: string, resourceTag: string, version: number, incarnation: string) => void
   getNonce: (socket: WebSocket) => string | undefined
   debug: boolean
   // Auth gate for the FIRST objstore-put-begin against a workspace
@@ -39,14 +45,21 @@ export type ObjstoreInitDeps = {
   // process behaviour). Multi-replica deployments MUST supply a
   // shared secret here so a token minted on one replica's WS plane
   // validates on another replica's REST plane (the WS-to-REST hop
-  // is not load-balancer-pinned). See server/index.ts boot logic
+  // is not load-balancer-pinned). See server-e2e/index.ts boot logic
   // for the env var (`OBJSTORE_TOKEN_SECRET`).
   tokenSecret?: TokenSecret
+  // New-workspace operator gate for the REST put-begin mint — returns
+  // `true` to DENY (password configured AND workspace new). The
+  // connection-independent analog of `authGate`; the client falls back to
+  // the in-band WS put-begin on a deny. Omitted → open (never deny),
+  // matching `authGate`'s no-config default.
+  restPutGate?: (workspaceTag: string) => Promise<boolean>
 }
 
 export type ObjstoreInit = {
   handlers: ObjstoreHandlers
   restDeps: ObjstoreRestDeps
+  reap: () => Promise<void>
   startupReap: Promise<void>
   // Cancels the periodic timer AND resolves only after any
   // currently-in-flight sweep has finished, so shutdown can safely
@@ -56,6 +69,16 @@ export type ObjstoreInit = {
 }
 
 export function initObjstore(deps: ObjstoreInitDeps): ObjstoreInit {
+  // Fail loud on a lopsided auth config. The put-begin gate in
+  // handlers.ts only fires when BOTH authGate and sendUnauthorized are
+  // present (it needs the reporter to emit the `unauthorized` frame),
+  // so wiring authGate WITHOUT sendUnauthorized silently fails OPEN —
+  // unauthenticated first-writes to unknown workspaces would be accepted
+  // despite the operator's intent to gate them. Reject at boot rather
+  // than regress access control silently.
+  if (deps.authGate && !deps.sendUnauthorized) {
+    throw new Error('initObjstore: authGate requires sendUnauthorized (the put-begin gate needs it to emit the unauthorized frame)')
+  }
   const handle = deps.handle
   const secret = deps.tokenSecret ?? newTokenSecret()
   const handlers = createObjstoreHandlers({
@@ -67,7 +90,10 @@ export function initObjstore(deps: ObjstoreInitDeps): ObjstoreInit {
     ...(deps.sendUnauthorized ? { sendUnauthorized: deps.sendUnauthorized } : {}),
   })
   const restDeps: ObjstoreRestDeps = {
-    handle, secret, broadcast: deps.broadcast, publishObjPut: deps.publishObjPut, debug: deps.debug,
+    handle, secret, broadcast: deps.broadcast,
+    publishObjPut: deps.publishObjPut, publishObjDeleted: deps.publishObjDeleted,
+    restPutGate: deps.restPutGate ?? (() => Promise.resolve(false)),
+    debug: deps.debug,
   }
   // Re-entrancy guard for periodic + startup sweeps. Kicking the
   // startup sweep through the same `enqueueSweep` path means the
@@ -77,48 +103,48 @@ export function initObjstore(deps: ObjstoreInitDeps): ObjstoreInit {
   let inFlight: Promise<void> | null = null
   function enqueueSweep(): Promise<void> {
     if (inFlight) return inFlight
-    // Log reaper failures unconditionally (not gated on `debug`) — a
-    // failed sweep means stranded files / staging rows that never get
-    // cleaned, and operators need to see it. Inner catch makes the
-    // promise resolve so callers (startupReap awaiter + setInterval)
-    // don't have to handle rejection.
-    const p = reapOrphans(handle).catch((err) => {
-      console.warn('objstore reaper error:', errStack(err))
-    }).finally(() => { if (inFlight === p) inFlight = null })
+    // Keep failures visible to explicit callers; automatic sweeps log them.
+    const p = reapOrphans(handle).finally(() => { if (inFlight === p) inFlight = null })
     inFlight = p
     return p
   }
+  const drain = async () => { if (inFlight) await inFlight.catch(() => {}) }
+  const automaticSweep = () => enqueueSweep().catch(err => console.warn('objstore reaper error:', errStack(err)))
+  // Explicit cleanup remains available when automatic sweeps are disabled.
+  if (deps.reapDisabled) {
+    console.warn('objstore: automatic reaper DISABLED (OBJSTORE_REAP_DISABLED) — use /api/reap or an external cleanup job')
+    return { handlers, restDeps, reap: enqueueSweep, startupReap: Promise.resolve(), stopReaper: drain }
+  }
   // Caller awaits this before accepting traffic so a fresh boot
   // can't hand out list / fetch / put-begin against a tag whose
   // on-disk state still has stranded files from a prior crash.
-  const startupReap = enqueueSweep()
+  const startupReap = automaticSweep()
   // Jittered start of the periodic timer. Multi-replica deploys
-  // (Neon + Vercel Blob) commonly boot N replicas in tight lock-
-  // step (deploy rollout, cluster restart) and would otherwise
-  // sync every replica's reaper at the same wall-clock tick,
-  // hammering the DB + blob store with N×readdir+lock-acquire
-  // bursts. A random first-interval delay deconcurrencies the
-  // cluster without changing the long-term sweep cadence.
-  // Jitter range is 0…1× reapIntervalMs (i.e., the next sweep
-  // happens at [interval, 2×interval] after boot); subsequent
-  // sweeps stay at exactly `reapIntervalMs` apart.
+  // (Neon + Vercel Blob) commonly boot N replicas in tight lock-step
+  // (deploy rollout, cluster restart), which would otherwise sync every
+  // replica's reaper to the same wall-clock tick and hammer the DB +
+  // blob store with N×readdir+lock-acquire bursts. A random
+  // first-interval delay spreads the cluster out without changing the
+  // long-term cadence. Jitter is 0…1× reapIntervalMs (first sweep at
+  // [interval, 2×interval] after boot); subsequent sweeps stay exactly
+  // `reapIntervalMs` apart.
   let reapTimer: ReturnType<typeof setInterval> | null = null
   const jitterMs = Math.floor(Math.random() * deps.reapIntervalMs)
   const firstTimer = setTimeout(() => {
-    enqueueSweep()
-    reapTimer = setInterval(enqueueSweep, deps.reapIntervalMs)
+    void automaticSweep()
+    reapTimer = setInterval(automaticSweep, deps.reapIntervalMs)
     reapTimer.unref?.()
   }, deps.reapIntervalMs + jitterMs)
   firstTimer.unref?.()
   return {
-    handlers, restDeps, startupReap,
+    handlers, restDeps, reap: enqueueSweep, startupReap,
     stopReaper: async () => {
       clearTimeout(firstTimer)
       if (reapTimer) clearInterval(reapTimer)
       // Drain whichever sweep is currently running (startup or
       // periodic) — either could be mid-readdir/unlink at SIGTERM
       // time and would otherwise outlive `handle.close()`.
-      if (inFlight) await inFlight.catch(() => {})
+      await drain()
     },
   }
 }

package/{server → server-e2e}/objstore/reaper.ts

@@ -27,6 +27,7 @@
 // predicate can't match a row a concurrent upload just refreshed.
 
 import { type Handle, STAGING_TTL_MS_DEFAULT, isValidContentHash, isValidStagingId, isValidTag } from './store.ts'
+import { debugId, debugTag } from '../util.ts'
 
 type StagingRow = {
   workspace_tag: string
@@ -67,12 +68,19 @@ type StagingRow = {
 // live-set re-read, not via mutual exclusion.
 async function gcBlobIfUnreferenced(
   handle: Handle, tag: string, hash: string, modifiedMs: number, now: number, grace: number,
-): Promise<void> {
-  if (!isValidContentHash(hash)) return
-  if (now - modifiedMs < grace) return
+): Promise<boolean> {
+  if (!isValidContentHash(hash)) return false
+  if (now - modifiedMs < grace) return false
   const refs = await liveHashSet(handle, tag)
-  if (refs.has(hash)) return
+  if (refs.has(hash)) return false
   await handle.blob.unlinkLive(tag, hash)
+  // Log EVERY live-blob deletion unconditionally (not behind `debug`):
+  // this is the only record that the GC removed bytes. A handful per
+  // sweep is normal (superseded versions aging out); a burst across many
+  // workspaces is the smoking gun for the "all uploaded data went
+  // missing" failure — pair it with the sweep summary in `reapOrphans`.
+  console.warn(`objstore-reaper: GC live blob ${debugTag(tag)}/${debugId(hash)} (unreferenced, age ${Math.round((now - modifiedMs) / 1000)}s ≥ grace ${Math.round(grace / 1000)}s)`)
+  return true
 }
 
 // The set of content hashes referenced by the workspace's live rows.
@@ -86,18 +94,22 @@ async function liveHashSet(handle: Handle, tag: string): Promise<Set<string>> {
 // snapshot we read up front can race a concurrent commit; the
 // reference re-read inside `gcBlobIfUnreferenced` (plus the grace
 // window) ensures we never unlink a blob a live row names.
-async function reapUnreferencedForTag(handle: Handle, tag: string, now: number, grace: number): Promise<void> {
-  if (!isValidTag(tag)) return
+// Returns the number of live blobs GC'd for this tag, so `reapOrphans`
+// can surface a sweep-wide total (the headline signal for mass loss).
+async function reapUnreferencedForTag(handle: Handle, tag: string, now: number, grace: number): Promise<number> {
+  if (!isValidTag(tag)) return 0
   const blobs = await handle.blob.listLiveBlobs(tag)
-  if (blobs.length === 0) return
+  if (blobs.length === 0) return 0
   const referenced = await liveHashSet(handle, tag)
+  let gc = 0
   for (const { hash, modifiedMs } of blobs) {
     if (!isValidContentHash(hash)) continue
     // Referenced in our snapshot → skip the grace + re-read path
     // entirely; only unreferenced blobs need it.
     if (referenced.has(hash)) continue
-    await gcBlobIfUnreferenced(handle, tag, hash, modifiedMs, now, grace)
+    if (await gcBlobIfUnreferenced(handle, tag, hash, modifiedMs, now, grace)) gc++
   }
+  return gc
 }
 
 // Drop staging rows older than the TTL and unlink their on-storage
@@ -109,7 +121,7 @@ async function reapStaleStagingRows(handle: Handle, now: number, stagingTtlMs: n
   // Push the staleness filter into SQL so the
   // `workspace_object_staging_begun_at_idx` index handles the scan.
   // The snapshot is O(stale-rows) cluster-wide. DB-layout audit
-  // `server/objstore/store.ts`.
+  // `server-e2e/objstore/store.ts`.
   const staleBefore = now - stagingTtlMs
   const staging = await handle.listAllStaging.all(staleBefore) as StagingRow[]
   for (const s of staging) {
@@ -172,9 +184,10 @@ export async function reapOrphans(handle: Handle, stagingTtlMs: number = STAGING
   const grace = stagingTtlMs
   // Pass 1: tags the live table knows about — GC unreferenced live
   // blobs (past the grace window) against the referenced-hash set.
+  let liveGc = 0
   const liveTagsRows = await handle.listLiveTags.all()
   const liveTags = liveTagsRows.map((r) => r.workspace_tag)
-  for (const tag of liveTags) await reapUnreferencedForTag(handle, tag, now, grace)
+  for (const tag of liveTags) liveGc += await reapUnreferencedForTag(handle, tag, now, grace)
   // Whole-workspace deletes leave residue (dirs / blob-prefixes) the
   // live table no longer lists. Walk the backend's top-level workspace
   // listing to find them; for each straggler tag, GC its unreferenced
@@ -186,7 +199,14 @@ export async function reapOrphans(handle: Handle, stagingTtlMs: number = STAGING
   const liveSet = new Set(liveTags)
   for (const tag of topLevel) {
     if (liveSet.has(tag) || !isValidTag(tag)) continue
-    await reapUnreferencedForTag(handle, tag, now, grace)
+    liveGc += await reapUnreferencedForTag(handle, tag, now, grace)
+  }
+  // Sweep-wide total. A nonzero count means the GC deleted live bytes
+  // this pass — logged unconditionally so "all data went missing"
+  // leaves an obvious server-side trail (a large count over a short
+  // window is the signature). Per-blob lines above carry which/why.
+  if (liveGc > 0) {
+    console.warn(`objstore-reaper: swept ${liveTags.length} live + ${topLevel.length} store tag(s); GC'd ${liveGc} live blob(s)`)
   }
   // Pass 2: stale staging rows + orphan staging blobs. The orphan
   // sweep does per-blob row lookups (no caller-side snapshot), so a

package/server-e2e/objstore/rest-deny.ts

@@ -0,0 +1,28 @@
+// Shared failure-response helpers for the objstore REST plane
+// (./rest.ts + ./rest-mint.ts). A uniform `{ error: <reason> }` JSON
+// envelope so clients parse one shape; the 409 variant additionally
+// carries the live version/incarnation so a PUT / put-begin can rebase.
+// Reasons (400/401/403/404/405/409/410/411/500/503) are documented in
+// server-e2e/README.md and the client decides recovery from the code —
+// probe-distinguishing defense isn't a goal: the richer PUT/GET reasons are
+// reachable only after the bearer-token check, while the public surfaces (an
+// unauthed GET/PUT, or the signature-authed POST mint) expose only
+// 400/401/404 — none of which gives a probe signal it couldn't already
+// enumerate.
+
+import type { ServerResponse } from 'node:http'
+
+export function deny(res: ServerResponse, status: number, body: string): void {
+  res.writeHead(status, { 'content-type': 'application/json' })
+  res.end(JSON.stringify({ error: body }))
+}
+
+// Variant of `deny` that augments the envelope with the live row's
+// `currentVersion` + `currentIncarnation` so a 409'd PUT / put-begin can
+// rebase onto the right precondition token rather than retry blindly
+// against an occupied slot. Symmetric with the WS plane's
+// `objstore-conflict` frame.
+export function denyConflict(res: ServerResponse, currentVersion: number | null, currentIncarnation: string | null): void {
+  res.writeHead(409, { 'content-type': 'application/json' })
+  res.end(JSON.stringify({ error: 'conflict', currentVersion, currentIncarnation }))
+}

package/server-e2e/objstore/rest-mint.ts

@@ -0,0 +1,224 @@
+// REST endpoints for the objstore plane — the single-round-trip,
+// SSE-session-independent alternative to the WS `objstore-fetch` /
+// `objstore-put-begin` / `objstore-delete` handshakes (so these ops can't
+// be interrupted by an SSE replica hop). One route, `POST
+// /api/objstore/{tag}/{res}`, with a JSON body `{ op, ts, signature, ... }`:
+// the workspace signs the request (the workspaceTag IS the Ed25519 pubkey,
+// so verification is self-contained — no socket, no stored key), the server
+// verifies it + freshness/replay-guards (the connection-nonce stand-in).
+// fetch/put return the SAME token shape the WS path sends, for the UNCHANGED
+// bearer-token GET / PUT byte transfers; delete mutates in place and returns
+// `{ deletedVersion }` (no token — there are no bytes to move). See
+// server-e2e/README.md "REST endpoints & tokens".
+
+import type { IncomingMessage, ServerResponse } from 'node:http'
+import { Buffer } from 'node:buffer'
+
+import type { ObjstoreRestDeps, RouteMatch } from './rest.ts'
+import { deny, denyConflict } from './rest-deny.ts'
+import { MAX_CONTENT_LENGTH, beginPut, deleteObject, getLive, isValidContentHash, isValidIncarnation, isValidSignature, objectMetaWire } from './store.ts'
+import { mintGetToken, mintPutToken } from './tokens.ts'
+import { verifyObjstoreDeleteRestSig, verifyObjstoreFetchRestSig, verifyObjstorePutBeginRestSig } from './sign.ts'
+import { createFetchMintGuard } from './fetch-mint-guard.ts'
+
+// Per-process freshness + replay guard for the REST POSTs (fetch, put-begin,
+// delete). They have no connection nonce to bind, so they bind a client
+// timestamp — see ./fetch-mint-guard.ts. The three ops' signatures are
+// globally unique (distinct canonical domains), so one guard dedups all three.
+const mintGuard = createFetchMintGuard()
+
+// Hard cap on a mint JSON body. The put-begin body (op, ts, signature,
+// prevVersion, prevIncarnation, expectedLength, contentHash) is the larger
+// of the two and still only a few hundred bytes; 4 KiB is comfortable
+// headroom. Bounds the read so a hostile client can't stream an unbounded
+// body into memory before the parse.
+const MINT_BODY_MAX = 4096
+
+// Read a small JSON request body up to `maxBytes`, returning the parsed
+// value or null on overflow / parse failure / read error. Used only by
+// the mint POST; the PUT body is the raw blob and streams to disk via a
+// different path.
+async function readJsonBody(req: IncomingMessage, maxBytes: number): Promise<unknown> {
+  const chunks: Buffer[] = []
+  let total = 0
+  try {
+    for await (const chunk of req) {
+      const buf = chunk as Buffer
+      total += buf.length
+      if (total > maxBytes) return null
+      chunks.push(buf)
+    }
+  } catch { return null }
+  try { return JSON.parse(Buffer.concat(chunks).toString('utf8')) }
+  catch { return null }
+}
+
+// Extract + range-check the auth fields common to both mint ops. Gates the
+// signature to the wire shape (`isValidSignature`, matching the WS plane) so
+// an obviously-malformed sig is rejected up front rather than burning an
+// Ed25519 verify — and, since this runs before `mintGuard.admit`, a bad sig
+// never reaches the replay cache regardless.
+function parseMintAuth(body: object): { ts: number; signature: string } | null {
+  const ts = (body as { ts?: unknown }).ts
+  const signature = (body as { signature?: unknown }).signature
+  if (!Number.isSafeInteger(ts) || (ts as number) < 0) return null
+  if (!isValidSignature(signature)) return null
+  return { ts: ts as number, signature }
+}
+
+// Dispatch the mint by body `op`. Auth-proxy friendly: the signature rides
+// the JSON body (not a header or cookie, so it never collides with a
+// cookie-based proxy), and the response is JSON — no redirect, no token in
+// any URL. Client note: a replayed signature is rejected, so a retry MUST
+// re-sign with a fresh `ts` rather than resend the same body.
+export async function handleRestMint(
+  deps: ObjstoreRestDeps, req: IncomingMessage, res: ServerResponse, route: RouteMatch,
+): Promise<void> {
+  const body = await readJsonBody(req, MINT_BODY_MAX)
+  if (!body || typeof body !== 'object') { deny(res, 400, 'bad-request'); return }
+  const op = (body as { op?: unknown }).op
+  if (op === 'fetch') { await handleRestFetchMint(deps, res, route, body); return }
+  if (op === 'put') { await handleRestPutBegin(deps, res, route, body); return }
+  if (op === 'delete') { await handleRestDelete(deps, res, route, body); return }
+  deny(res, 400, 'bad-request')
+}
+
+// op:'fetch' — mirrors the WS `objstore-fetch` → `objstore-fetch-token`
+// handshake; returns `{ ...meta, urlPath, token, expiresAt }` for the
+// UNCHANGED bearer-token GET.
+async function handleRestFetchMint(
+  deps: ObjstoreRestDeps, res: ServerResponse, route: RouteMatch, body: object,
+): Promise<void> {
+  const auth = parseMintAuth(body)
+  if (!auth) { deny(res, 400, 'bad-request'); return }
+  // Verify the signature BEFORE touching the replay guard so a bad-sig
+  // request can't consume cache space. The signature commits to THIS `ts`.
+  if (!await verifyObjstoreFetchRestSig(route.tag, route.resourceTag, auth.ts, auth.signature)) {
+    deny(res, 401, 'unauthorized'); return
+  }
+  // Freshness window + single-use dedup (the connection-nonce stand-in).
+  // 'stale'/'replay' are both opaque 401s — the client re-signs with a
+  // fresh `ts` and retries.
+  if (mintGuard.admit(auth.signature, auth.ts) !== 'ok') { deny(res, 401, 'unauthorized'); return }
+  const row = await getLive(deps.handle, route.tag, route.resourceTag)
+  if (!row) { deny(res, 404, 'not-found'); return }
+  const { token, exp } = mintGetToken(deps.secret, route.tag, route.resourceTag, row.version, row.incarnation)
+  res.writeHead(200, { 'content-type': 'application/json' })
+  res.end(JSON.stringify({
+    ...objectMetaWire(row),
+    urlPath: `/api/objstore/${route.tag}/${route.resourceTag}`,
+    token,
+    expiresAt: exp,
+  }))
+}
+
+// op:'put' — mirrors the WS `handlePutBegin`: verify the signed put fields,
+// freshness/replay-guard, the new-workspace operator gate, then `beginPut`
+// (advisory prev-check + staging row) and mint the put-token. Returns
+// `{ stagingId, urlPath, token, expiresAt }` for the UNCHANGED bearer-token
+// PUT; `409 conflict` (with current version/incarnation to rebase on) or
+// `403 workspace-full` map `beginPut`'s refusals.
+//
+// New-workspace gate: REST can't read a socket's operator-auth state, so
+// when a password is configured AND the workspace is never-before-seen this
+// returns 401 — the client falls back to the in-band WS put-begin (which
+// runs the operator auth flow). No-op for password-less deployments or
+// existing workspaces (the common case).
+async function handleRestPutBegin(
+  deps: ObjstoreRestDeps, res: ServerResponse, route: RouteMatch, body: object,
+): Promise<void> {
+  const auth = parseMintAuth(body)
+  if (!auth) { deny(res, 400, 'bad-request'); return }
+  // Field gates mirror `handlePutBegin`'s up-front rejects.
+  const expectedLength = (body as { expectedLength?: unknown }).expectedLength
+  if (!Number.isSafeInteger(expectedLength) || (expectedLength as number) < 0 || (expectedLength as number) > MAX_CONTENT_LENGTH) {
+    deny(res, 400, 'bad-request'); return
+  }
+  const prevVersionRaw = (body as { prevVersion?: unknown }).prevVersion
+  if (prevVersionRaw != null && (typeof prevVersionRaw !== 'number' || !Number.isSafeInteger(prevVersionRaw))) {
+    deny(res, 400, 'bad-request'); return
+  }
+  const prevVersion = typeof prevVersionRaw === 'number' ? prevVersionRaw : null
+  const prevIncarnationRaw = (body as { prevIncarnation?: unknown }).prevIncarnation
+  const prevIncarnation = typeof prevIncarnationRaw === 'string' ? prevIncarnationRaw : null
+  // prevVersion/prevIncarnation travel as a null-iff-null pair (matches the
+  // WS `validPrevPair`): a half-pair is malformed.
+  if ((prevVersion === null) !== (prevIncarnation === null)) { deny(res, 400, 'bad-request'); return }
+  // A non-null incarnation must be the wire shape (matches the WS path's
+  // `isValidIncarnation` gate). Signature-covered + CAS-checked regardless,
+  // but reject the obviously-malformed up front.
+  if (prevIncarnation !== null && !isValidIncarnation(prevIncarnation)) { deny(res, 400, 'bad-request'); return }
+  const contentHash = (body as { contentHash?: unknown }).contentHash
+  if (!isValidContentHash(contentHash)) { deny(res, 400, 'bad-request'); return }
+
+  const fields = {
+    workspaceTag: route.tag, resourceTag: route.resourceTag,
+    prevVersion, prevIncarnation, contentHash, expectedLength: expectedLength as number,
+  }
+  if (!await verifyObjstorePutBeginRestSig(fields, auth.ts, auth.signature)) { deny(res, 401, 'unauthorized'); return }
+  if (mintGuard.admit(auth.signature, auth.ts) !== 'ok') { deny(res, 401, 'unauthorized'); return }
+  // New-workspace operator gate — see the function header. Refusing here
+  // routes the client to its in-band WS put-begin fallback.
+  if (await deps.restPutGate(route.tag)) { deny(res, 401, 'unauthorized'); return }
+  const result = await beginPut(deps.handle, {
+    workspaceTag: route.tag, resourceTag: route.resourceTag,
+    prevVersion, prevIncarnation, expectedLength: expectedLength as number,
+    contentHash, signature: auth.signature,
+  })
+  if (!result.ok) {
+    if (result.reason === 'workspace-full') { deny(res, 403, 'workspace-full'); return }
+    denyConflict(res, result.conflict?.version ?? null, result.conflict?.incarnation ?? null); return
+  }
+  const { token, exp } = mintPutToken(deps.secret, route.tag, route.resourceTag, result.stagingId, expectedLength as number)
+  res.writeHead(200, { 'content-type': 'application/json' })
+  res.end(JSON.stringify({
+    stagingId: result.stagingId,
+    urlPath: `/api/objstore/${route.tag}/${route.resourceTag}`,
+    token,
+    expiresAt: exp,
+  }))
+}
+
+// op:'delete' — mirrors the WS `handleDelete`: verify the signed delete
+// fields, freshness/replay-guard, then `deleteObject` (a precondition-checked
+// version-CAS drop). Unlike put, there is NO operator gate — delete is
+// signature-gated, idempotent, and creates nothing (the WS handleDelete has
+// no authGate either). Returns 200 `{ deletedVersion }` (0 = idempotent
+// no-op on a missing row with a null precondition); 409 conflict (stale
+// prevVersion/incarnation); 404 not-found (a non-null precondition against a
+// missing row). On a real drop it broadcasts `objstore-deleted` to
+// subscribers (+ cross-instance), exactly like the WS path.
+async function handleRestDelete(
+  deps: ObjstoreRestDeps, res: ServerResponse, route: RouteMatch, body: object,
+): Promise<void> {
+  const auth = parseMintAuth(body)
+  if (!auth) { deny(res, 400, 'bad-request'); return }
+  const prevVersionRaw = (body as { prevVersion?: unknown }).prevVersion
+  if (prevVersionRaw != null && (typeof prevVersionRaw !== 'number' || !Number.isSafeInteger(prevVersionRaw))) {
+    deny(res, 400, 'bad-request'); return
+  }
+  const prevVersion = typeof prevVersionRaw === 'number' ? prevVersionRaw : null
+  const prevIncarnationRaw = (body as { prevIncarnation?: unknown }).prevIncarnation
+  const prevIncarnation = typeof prevIncarnationRaw === 'string' ? prevIncarnationRaw : null
+  // null-iff-null pair (matches the WS `validPrevPair`); a non-null
+  // incarnation must be the wire shape.
+  if ((prevVersion === null) !== (prevIncarnation === null)) { deny(res, 400, 'bad-request'); return }
+  if (prevIncarnation !== null && !isValidIncarnation(prevIncarnation)) { deny(res, 400, 'bad-request'); return }
+
+  const fields = { workspaceTag: route.tag, resourceTag: route.resourceTag, prevVersion, prevIncarnation }
+  if (!await verifyObjstoreDeleteRestSig(fields, auth.ts, auth.signature)) { deny(res, 401, 'unauthorized'); return }
+  if (mintGuard.admit(auth.signature, auth.ts) !== 'ok') { deny(res, 401, 'unauthorized'); return }
+  const result = await deleteObject(deps.handle, route.tag, route.resourceTag, prevVersion, prevIncarnation)
+  if (!result.ok) {
+    if (result.reason === 'conflict') { denyConflict(res, result.conflict?.version ?? null, result.conflict?.incarnation ?? null); return }
+    deny(res, 404, 'not-found'); return
+  }
+  res.writeHead(200, { 'content-type': 'application/json' })
+  res.end(JSON.stringify({ deletedVersion: result.deletedVersion }))
+  // deletedVersion 0 = nothing was live → nothing to broadcast. A real drop
+  // fans out to subscribers (including the originator, matching the WS path's
+  // `except: null`) so peers' `onDeleted` fire.
+  if (result.deletedVersion === 0) return
+  deps.broadcast(route.tag, { type: 'objstore-deleted', workspaceTag: route.tag, resourceTag: route.resourceTag, version: result.deletedVersion, incarnation: prevIncarnation! }, null)
+  deps.publishObjDeleted(route.tag, route.resourceTag, result.deletedVersion, prevIncarnation!)
+}