@preventive/triage 1.0.0-alpha.3 → 1.0.0-alpha.5

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@preventive/triage",
-  "version": "1.0.0-alpha.3",
+  "version": "1.0.0-alpha.5",
   "description": "Client & relay server for triaging of automated reports",
   "license": "MIT",
   "author": {

package/server/auth.ts

@@ -31,6 +31,10 @@ export type AuthenticateMsg = { password?: unknown }
 export type Auth = {
   // True iff the gate is configured AND this socket hasn't authenticated.
   requiresAuth(socket: WebSocket): boolean
+  // Static "is a server password configured" flag — the socket-less
+  // predicate the REST put-begin gate needs (a REST request can never be
+  // operator-authorized, so `requiresAuth` collapses to this there).
+  passwordConfigured: boolean
   handleAuthenticate(socket: WebSocket, msg: AuthenticateMsg): void
   sendUnauthorized(socket: WebSocket, ctx: UnauthorizedContext): void
 }
@@ -106,5 +110,5 @@ export function createAuth(deps: {
     send(socket, { type: 'authenticated' })
   }
 
-  return { requiresAuth, handleAuthenticate, sendUnauthorized }
+  return { requiresAuth, passwordConfigured: CONFIGURED_PASSWORD_HMAC != null, handleAuthenticate, sendUnauthorized }
 }

package/server/config.ts

@@ -18,6 +18,7 @@ export type Config = {
   dbPath: string
   objstoreDir: string
   reapIntervalMs: number
+  reapDisabled: boolean
   maxInflightPerSocket: number
   debug: boolean
   neonUrl: string | null
@@ -78,6 +79,13 @@ Environment:
                              DATABASE_URL + BLOB_READ_WRITE_TOKEN
                              are set (bytes live in Vercel Blob).
   OBJSTORE_REAP_INTERVAL_MS  orphan reaper period (default 600000)
+  OBJSTORE_REAP_DISABLED     set '1' / 'true' to disable the orphan
+                             reaper ENTIRELY — no boot sweep, no
+                             periodic GC. Orphaned/superseded blobs and
+                             stale staging rows then accumulate
+                             unbounded; only set this if an external job
+                             handles GC (e.g. a cron calling reapOrphans).
+                             Default OFF (reaper runs).
   TRUST_PROXY                set '1' / 'true' to honour X-Forwarded-
                              Host / X-Forwarded-Proto when computing
                              the same-origin gate's expected origin.
@@ -163,6 +171,11 @@ export function loadConfig(): Config {
   const objstoreDir = env['OBJSTORE_DIR'] ?? join(dirname(dbPath), 'objstore')
   // No practical upper bound beyond the safe-integer range.
   const reapIntervalMs = intEnv('OBJSTORE_REAP_INTERVAL_MS', 10 * 60 * 1000, 1, Number.MAX_SAFE_INTEGER)
+  // Hard off-switch for the orphan reaper (both the boot sweep AND the
+  // periodic timer). '1' / 'true' (case-insensitive) → disabled; anything
+  // else, including unset, leaves it ON. Same boolean shape as TRUST_PROXY.
+  const reapDisabledEnv = env['OBJSTORE_REAP_DISABLED']
+  const reapDisabled = reapDisabledEnv === '1' || reapDisabledEnv?.toLowerCase() === 'true'
   const debug = env['DEBUG'] === '1'
 
   const configPath = env['CONFIG_PATH'] ?? fileURLToPath(new URL('./config.json', import.meta.url))
@@ -189,7 +202,7 @@ export function loadConfig(): Config {
   const tokenSecret = tokenSecretB64 ? decodeTokenSecret(tokenSecretB64) : null
 
   return {
-    port, host, dbPath, objstoreDir, reapIntervalMs, maxInflightPerSocket,
+    port, host, dbPath, objstoreDir, reapIntervalMs, reapDisabled, maxInflightPerSocket,
     debug, neonUrl, blobToken, tokenSecret, password,
     trustProxyEnv: env['TRUST_PROXY'],
   }

package/server/http.ts

@@ -20,6 +20,12 @@ import { errStack } from './util.ts'
 // similar) can route `/api/*` → this process and `/*` → the static UI
 // bundle with a single location block.
 export const WS_UPGRADE_PATH = '/api/sync'
+// Session-independent triage-sync save plane: `POST /api/sync/save`. The
+// SSE-mode alternative to the in-band `workspace-save` frame — a save POSTed
+// here commits + broadcasts WITHOUT taking over the client's SSE event-stream
+// (each in-band POST forces a stream takeover; see sse-server.ts). Sibling of
+// the WS upgrade path so one nginx `/api/*` block routes both.
+export const SAVE_REST_PATH = '/api/sync/save'
 function isUpgradePath(url: string | undefined): boolean {
   if (typeof url !== 'string') return false
   // Strip `?…` so clients can carry build / debug tags. Exact match
@@ -42,12 +48,68 @@ export type HttpServerDeps = {
   isOriginAllowed: (req: HasHeaders) => boolean
   isShuttingDown: () => boolean
   track: (promise: Promise<unknown>) => void
+  // Handler for `POST /api/sync/save` (see SAVE_REST_PATH). Owns body parse +
+  // the save pipeline + JSON response; this module owns the gates (method,
+  // same-origin, shutdown, idle-timeout) and the graceful-drain tracking.
+  handleSaveRest: (req: HttpRequest, res: ServerResponse) => Promise<void>
   restPutIdleTimeoutMs: number
   debug: boolean
 }
 
+// `POST /api/sync/save` dispatch. Returns the in-flight handler promise when
+// the request matched the route (the caller `track`s it for graceful drain),
+// or null when it's for a different route. The gate ladder — method →
+// shutdown → same-origin → idle-timeout — mirrors the objstore REST branch; a
+// gate rejection writes its own response and returns an already-settled
+// promise. Kept out of `createHttpServer` so that dispatcher stays compact.
+function dispatchSaveRest(
+  req: HttpRequest, res: ServerResponse,
+  deps: {
+    handleSaveRest: (req: HttpRequest, res: ServerResponse) => Promise<void>
+    isOriginAllowed: (req: HasHeaders) => boolean
+    isShuttingDown: () => boolean
+    restPutIdleTimeoutMs: number
+    debug: boolean
+  },
+): Promise<void> | null {
+  if (typeof req.url !== 'string' || req.url.split('?', 1)[0] !== SAVE_REST_PATH) return null
+  if (req.method !== 'POST') {
+    res.writeHead(405, { 'content-type': 'application/json', 'allow': 'POST', 'connection': 'close' })
+    res.end(JSON.stringify({ error: 'method-not-allowed' }))
+    return Promise.resolve()
+  }
+  // Shutdown gate — parity with the objstore REST branch (a POST on an
+  // existing keep-alive socket after SIGTERM but before close() drains).
+  if (deps.isShuttingDown()) {
+    res.writeHead(503, { 'content-type': 'application/json', 'connection': 'close' })
+    res.end(JSON.stringify({ error: 'shutting-down' }))
+    return Promise.resolve()
+  }
+  // Same-origin gate — a hostile cross-origin page would carry an Origin
+  // header (browser-set on fetch); same-origin XHR may omit it (allowed).
+  if (!deps.isOriginAllowed(req)) {
+    res.writeHead(403, { 'content-type': 'application/json' })
+    res.end(JSON.stringify({ error: 'origin-denied' }))
+    return Promise.resolve()
+  }
+  // Idle-body timeout — a slow-loris trickling the JSON body would otherwise
+  // hold the connection indefinitely.
+  req.setTimeout(deps.restPutIdleTimeoutMs, () => {
+    if (deps.debug) console.warn(`sync-save REST idle ${deps.restPutIdleTimeoutMs}ms → abort`)
+    try { req.destroy(new Error('idle-timeout')) } catch {}
+  })
+  // Outer catch is the unhandled-rejection guard for a throw OUTSIDE the
+  // handler's own try/catch — logs and terminates the response so the TCP
+  // socket doesn't dangle (same policy as the objstore handleRest wrapper).
+  return deps.handleSaveRest(req, res).catch((err) => {
+    console.warn('sync-save REST handler error:', errStack(err))
+    if (res.headersSent) { try { res.destroy() } catch {} }
+    else { try { res.writeHead(500, { 'content-type': 'application/json' }); res.end(JSON.stringify({ error: 'internal' })) } catch {} }
+  })
+}
+
 export function createHttpServer(deps: HttpServerDeps): Server {
-  const { wss, restDeps, sseServer, isOriginAllowed, isShuttingDown, track, restPutIdleTimeoutMs, debug } = deps
+  const { wss, restDeps, sseServer, isOriginAllowed, isShuttingDown, track, handleSaveRest, restPutIdleTimeoutMs, debug } = deps
   // Static-file plane (see ./static.ts). The directory is the
   // `build.js build` output sibling to this file; the loader handles
   // enumeration, pre-compression, and ETag derivation. Plugged in after
@@ -76,6 +138,12 @@ export function createHttpServer(deps: HttpServerDeps): Server {
       // the lifecycle's sseSessions() close loop.
       if (sseServer.handle(req, res)) return
     }
+    // Triage-sync save REST plane (see SAVE_REST_PATH) — session-independent
+    // `POST /api/sync/save` so an SSE-mode save doesn't take over the
+    // event-stream. The dispatch helper owns the gate ladder; we `track` the
+    // returned in-flight promise so SIGTERM drains it (mirrors npm-advisories).
+    const saveP = dispatchSaveRest(req, res, { handleSaveRest, isOriginAllowed, isShuttingDown, restPutIdleTimeoutMs, debug })
+    if (saveP) { track(saveP); return }
     // npm advisories proxy — same-origin + shutdown gates live in the
     // helper so this dispatcher stays compact. `dispatchNpmAdvisories`
     // returns the in-flight promise (or null when the route didn't
@@ -109,14 +177,16 @@ export function createHttpServer(deps: HttpServerDeps): Server {
         res.end(JSON.stringify({ error: 'origin-denied' }))
         return
       }
-      // PUT idle-body timeout — a slow-loris client trickling bytes
-      // within the declared Content-Length holds the staging fd + an
-      // inFlightSids slot indefinitely. `req.setTimeout` fires on
-      // inactivity; we destroy the request, aborting the body pipeline.
+      // Idle-body timeout for the body-bearing REST methods — a slow-loris
+      // client trickling bytes holds the connection (and, for PUT, the
+      // staging fd + inFlightSids slot) indefinitely. PUT carries the raw
+      // blob within its declared Content-Length; POST carries the small
+      // fetch-mint JSON body. `req.setTimeout` fires on inactivity; we
+      // destroy the request, aborting the body pipeline.
       // Transport audit `server/objstore/rest.ts:218`.
-      if (req.method === 'PUT') {
+      if (req.method === 'PUT' || req.method === 'POST') {
         req.setTimeout(restPutIdleTimeoutMs, () => {
-          if (debug) console.warn(`REST PUT idle ${restPutIdleTimeoutMs}ms → abort`)
+          if (debug) console.warn(`REST ${req.method} idle ${restPutIdleTimeoutMs}ms → abort`)
           try { req.destroy(new Error('idle-timeout')) } catch {}
         })
       }

package/server/index.ts

@@ -102,7 +102,7 @@ import { createBusReceiver } from './bus-receiver.ts'
 const config = loadConfig()
 const {
   port: PORT, host: HOST, dbPath: DB_PATH, objstoreDir: OBJSTORE_DIR,
-  reapIntervalMs: OBJSTORE_REAP_INTERVAL_MS,
+  reapIntervalMs: OBJSTORE_REAP_INTERVAL_MS, reapDisabled: OBJSTORE_REAP_DISABLED,
   maxInflightPerSocket: MAX_INFLIGHT_PER_SOCKET, debug: DEBUG,
   neonUrl: NEON_URL, blobToken: BLOB_TOKEN, tokenSecret: TOKEN_SECRET,
   password: CONFIG_PASSWORD, trustProxyEnv: TRUST_PROXY_ENV,
@@ -275,7 +275,7 @@ if (NEON_URL) {
 // Password gate (see ./auth.ts) — HMAC derivation + the `authenticate`
 // handshake.
 const auth = createAuth({ peers, password: CONFIG_PASSWORD, send, debug: DEBUG })
-const { requiresAuth, handleAuthenticate, sendUnauthorized } = auth
+const { requiresAuth, passwordConfigured, handleAuthenticate, sendUnauthorized } = auth
 
 // Triage-sync protocol handlers (see ./sync-handlers.ts). `getNonce`
 // resolves a socket's challenge nonce and is shared with the objstore
@@ -292,9 +292,9 @@ const publishObjDeleted = (tag: string, resourceTag: string, version: number): v
   pubsub.publish({ kind: 'objdel', tag, res: resourceTag, ver: version })
 }
 
-const { handleSave, handleSubscribe, sendSaveError } = createSyncHandlers({
+const { handleSave, handleSaveRest, handleSubscribe, sendSaveError } = createSyncHandlers({
   handle, send, broadcast, publishRevision, subscribe, getNonce,
-  requiresAuth, sendUnauthorized, workspaceExists,
+  requiresAuth, passwordConfigured, sendUnauthorized, workspaceExists,
   // Folds the objstore inventory into the `workspace-subscribed` ack.
   // The objstore store keeps its own richer `Handle`, so we wire the
   // query here where both handles exist rather than coupling
@@ -305,6 +305,7 @@ const { handleSave, handleSubscribe, sendSaveError } = createSyncHandlers({
 
 const { handlers: objstore, restDeps: objstoreRestDeps, startupReap, stopReaper } = initObjstore({
   handle: objstoreHandle, reapIntervalMs: OBJSTORE_REAP_INTERVAL_MS,
+  reapDisabled: OBJSTORE_REAP_DISABLED,
   send, broadcast, publishObjPut, publishObjDeleted,
   getNonce, debug: DEBUG,
   // Auth gate for the FIRST objstore-put-begin against a workspace
@@ -315,6 +316,11 @@ const { handlers: objstore, restDeps: objstoreRestDeps, startupReap, stopReaper
   // `unauthorized` frame and bails on `true`.
   authGate: async (socket, tag) => requiresAuth(socket) && !await workspaceExists(tag),
   sendUnauthorized,
+  // Socket-less analog of `authGate` for the REST put-begin mint: a REST
+  // request can never be operator-authorized, so the gate collapses to
+  // "password configured AND workspace new". A deny routes the client to
+  // its in-band WS put-begin fallback.
+  restPutGate: async (tag) => passwordConfigured && !await workspaceExists(tag),
   // `tokenSecret` is set only when OBJSTORE_TOKEN_SECRET was
   // provided in env (see TOKEN_SECRET resolution above). Omitted
   // → initObjstore mints a fresh per-process secret (fine for
@@ -361,10 +367,6 @@ const sseServer = installSseServer({
   // Mirror the WS `maxPayload` so the SSE plane can't accept frames
   // the WS plane would reject.
   maxBodyBytes: 4 * 1024 * 1024,
-  // 90s idle ceiling. The client's JSON ping/pong is 15s; this is ~6
-  // missed pings before we close — well past any transient network
-  // hiccup, much shorter than the kernel's hours-long TCP keepalive.
-  sessionIdleMs: 90_000,
   debug: DEBUG,
 })
 
@@ -375,7 +377,7 @@ const sseServer = installSseServer({
 // below.
 const httpServer = createHttpServer({
   wss, restDeps: objstoreRestDeps, sseServer, isOriginAllowed,
-  isShuttingDown, track,
+  isShuttingDown, track, handleSaveRest,
   restPutIdleTimeoutMs: REST_PUT_IDLE_TIMEOUT_MS, debug: DEBUG,
 })
 
@@ -444,6 +446,7 @@ const closeDb = async (): Promise<void> => {
 installLifecycle({
   httpServer, wss, heartbeatTimer, stopReaper,
   sseSessions: sseServer.sessions,
+  sseKeepaliveTimer: sseServer.keepaliveTimer,
   closeDb,
 })
 

package/server/lifecycle.ts

@@ -14,6 +14,10 @@ export type ShutdownDeps = {
   httpServer: Server
   wss: WebSocketServer
   heartbeatTimer: ReturnType<typeof setInterval>
+  // The SSE keepalive-sweep timer (server/sse-server.ts). Cleared on
+  // shutdown alongside `heartbeatTimer` so a tick can't write a keepalive
+  // comment to a session the close loop below is already tearing down.
+  sseKeepaliveTimer: ReturnType<typeof setInterval>
   // Stops the periodic reaper AND awaits any in-flight sweep.
   stopReaper: () => Promise<void>
   // Live SSE+POST session iterator (mirrors `wss.clients` for the SSE
@@ -62,7 +66,7 @@ export function createLifecycle(): Lifecycle {
   let pendingExitCode = 0
 
   function install(deps: ShutdownDeps): void {
-    const { httpServer, wss, heartbeatTimer, stopReaper, sseSessions, closeDb } = deps
+    const { httpServer, wss, heartbeatTimer, sseKeepaliveTimer, stopReaper, sseSessions, closeDb } = deps
 
     async function shutdown(exitCode: number = 0): Promise<void> {
       // Re-entry: don't restart the teardown, but escalate the pending
@@ -77,9 +81,9 @@ export function createLifecycle(): Lifecycle {
       shuttingDown = true
       pendingExitCode = exitCode
       console.log('Shutting down…')
-      // Stop the heartbeat so a tick can't fire mid-shutdown and ping a
-      // socket the close-loop below already started tearing down.
-      clearInterval(heartbeatTimer)
+      // Stop both periodic timers (WS heartbeat + SSE keepalive) so neither
+      // fires mid-shutdown against a peer the close-loop is tearing down.
+      for (const timer of [heartbeatTimer, sseKeepaliveTimer]) clearInterval(timer)
       // Send a 1001 (going away) close frame to every open socket BEFORE
       // shutting the listener. Lets clients distinguish a server-initiated
       // graceful shutdown from a network drop, so they can skip their

package/server/objstore/blob-fs.ts

@@ -48,13 +48,13 @@ async function fsOpenLiveReader(dir: string, tag: string, contentHash: string):
   const path = liveFilePath(dir, tag, contentHash)
   let fh
   try { fh = await open(path, 'r') } catch (err: unknown) {
-    if ((err as NodeJS.ErrnoException)?.code === 'ENOENT') return { ok: false, reason: 'unavailable' }
+    if ((err as NodeJS.ErrnoException)?.code === 'ENOENT') return { ok: false, reason: 'unavailable', detail: 'fs-enoent' }
     throw err
   }
   let size: number
   try { size = (await fh.stat()).size } catch {
     await fh.close().catch(() => {})
-    return { ok: false, reason: 'unavailable' }
+    return { ok: false, reason: 'unavailable', detail: 'fs-stat-failed' }
   }
   const stream = fh.createReadStream()
   let closed = false

package/server/objstore/blob-vercel.ts

@@ -364,19 +364,19 @@ function buildOpenLiveReader(sdk: VercelBlobSdk, token: string): BlobBackend['op
       // same condition (blob-fs.ts maps ENOENT → `unavailable`), telling
       // the client the resource is gone for good when it should refetch.
       // See server/README.md's GET status table.
-      if (isNotFound(err)) return { ok: false, reason: 'unavailable' }
+      if (isNotFound(err)) return { ok: false, reason: 'unavailable', detail: 'vercel-get-not-found' }
       throw err
     }
     // SDK returned null (no blob) — same "bytes missing for a live row"
     // transient as the BlobNotFoundError branch above → `unavailable`, not
     // `not-found`.
-    if (res == null) return { ok: false, reason: 'unavailable' }
+    if (res == null) return { ok: false, reason: 'unavailable', detail: 'vercel-get-null' }
     // statusCode 304 doesn't reach here in practice — the REST
     // GET layer doesn't pass If-None-Match — but a future call
     // site could. Treat as unavailable rather than streaming a
     // null body.
     if (res.statusCode !== 200 || res.stream == null) {
-      return { ok: false, reason: 'unavailable' }
+      return { ok: false, reason: 'unavailable', detail: `vercel-get-status-${res.statusCode}` }
     }
     // `@vercel/blob@2.x`'s streaming `get()` for private blobs returns
     // the body but does NOT populate `blob.size` nor pass a
@@ -395,11 +395,11 @@ function buildOpenLiveReader(sdk: VercelBlobSdk, token: string): BlobBackend['op
         // Blob vanished between get() and the head() size fallback (a
         // racing reaper GC) — still the "live row present, bytes gone"
         // transient, so `unavailable` (503), matching the get() path above.
-        if (isNotFound(headErr)) return { ok: false, reason: 'unavailable' }
+        if (isNotFound(headErr)) return { ok: false, reason: 'unavailable', detail: 'vercel-head-not-found' }
         throw headErr
       }
     }
-    if (size == null) return { ok: false, reason: 'unavailable' }
+    if (size == null) return { ok: false, reason: 'unavailable', detail: 'vercel-no-size' }
     // SDK returns a web ReadableStream<Uint8Array>; the REST layer
     // expects a Node Readable for pipeline(). Convert via
     // Readable.fromWeb — built-in and zero-copy where possible.

package/server/objstore/blob.ts

@@ -94,9 +94,16 @@ export type LiveReader = {
 // `unavailable`/503 contract, which the client retries. Both backends MUST
 // map a missing blob to `unavailable` (FS: ENOENT; Vercel: BlobNotFoundError
 // / null get()). No 404-mapping variant exists here so that bug can't recur.
+//
+// `detail` is a short, NON-SENSITIVE machine tag for the specific cause
+// (e.g. 'vercel-get-not-found', 'fs-enoent', 'vercel-no-size'). Every
+// byte-side failure collapses to the same 503 on the wire, so a permanent
+// loss (reaper GC'd the bytes) and a transient read fault are otherwise
+// indistinguishable — the REST layer logs `detail` so an operator can tell
+// them apart. Purely diagnostic; the REST status is unchanged.
 export type OpenLiveResult =
   | { ok: true; reader: LiveReader }
-  | { ok: false; reason: 'unavailable' }
+  | { ok: false; reason: 'unavailable'; detail?: string }
 
 export type BlobBackend = {
   // Per-workspace setup. FS creates the on-disk staging directory;

package/server/objstore/init.ts

@@ -18,6 +18,12 @@ export type ObjstoreInitDeps = {
   // the workspace_revision handle in server/db.ts.
   handle: Handle
   reapIntervalMs: number
+  // Hard off-switch (OBJSTORE_REAP_DISABLED). When true, NEITHER the boot
+  // sweep nor the periodic timer runs: `startupReap` resolves immediately
+  // and `stopReaper` is a no-op. Orphaned/superseded blobs and stale
+  // staging rows then accumulate unbounded — only safe if an external job
+  // handles GC. Omitted/false → reaper runs (the default). See config.ts.
+  reapDisabled?: boolean
   send: (socket: WebSocket, msg: object) => void
   broadcast: (tag: string, msg: object, except: WebSocket | null) => void
   // Cross-instance pub/sub publishers. SQLite mode passes no-ops; Neon
@@ -42,6 +48,12 @@ export type ObjstoreInitDeps = {
   // is not load-balancer-pinned). See server/index.ts boot logic
   // for the env var (`OBJSTORE_TOKEN_SECRET`).
   tokenSecret?: TokenSecret
+  // New-workspace operator gate for the REST put-begin mint — returns
+  // `true` to DENY (password configured AND workspace new). The
+  // connection-independent analog of `authGate`; the client falls back to
+  // the in-band WS put-begin on a deny. Omitted → open (never deny),
+  // matching `authGate`'s no-config default.
+  restPutGate?: (workspaceTag: string) => Promise<boolean>
 }
 
 export type ObjstoreInit = {
@@ -77,7 +89,10 @@ export function initObjstore(deps: ObjstoreInitDeps): ObjstoreInit {
     ...(deps.sendUnauthorized ? { sendUnauthorized: deps.sendUnauthorized } : {}),
   })
   const restDeps: ObjstoreRestDeps = {
-    handle, secret, broadcast: deps.broadcast, publishObjPut: deps.publishObjPut, debug: deps.debug,
+    handle, secret, broadcast: deps.broadcast,
+    publishObjPut: deps.publishObjPut, publishObjDeleted: deps.publishObjDeleted,
+    restPutGate: deps.restPutGate ?? (() => Promise.resolve(false)),
+    debug: deps.debug,
   }
   // Re-entrancy guard for periodic + startup sweeps. Kicking the
   // startup sweep through the same `enqueueSweep` path means the
@@ -98,6 +113,16 @@ export function initObjstore(deps: ObjstoreInitDeps): ObjstoreInit {
     inFlight = p
     return p
   }
+  // Hard off-switch (OBJSTORE_REAP_DISABLED). Skip the boot sweep AND the
+  // periodic timer entirely: `startupReap` resolves immediately so the
+  // index.ts `await startupReap` gate is a no-op, and `stopReaper` has
+  // nothing to clear or drain. Loud, unconditional warning — with GC off,
+  // orphaned/superseded blobs and stale staging rows are never reclaimed
+  // (they accumulate until an external job, if any, collects them).
+  if (deps.reapDisabled) {
+    console.warn('objstore: reaper DISABLED (OBJSTORE_REAP_DISABLED) — orphaned/superseded blobs and stale staging will NOT be reclaimed')
+    return { handlers, restDeps, startupReap: Promise.resolve(), stopReaper: async () => {} }
+  }
   // Caller awaits this before accepting traffic so a fresh boot
   // can't hand out list / fetch / put-begin against a tag whose
   // on-disk state still has stranded files from a prior crash.

package/server/objstore/reaper.ts

@@ -27,6 +27,7 @@
 // predicate can't match a row a concurrent upload just refreshed.
 
 import { type Handle, STAGING_TTL_MS_DEFAULT, isValidContentHash, isValidStagingId, isValidTag } from './store.ts'
+import { debugId, debugTag } from '../util.ts'
 
 type StagingRow = {
   workspace_tag: string
@@ -67,12 +68,19 @@ type StagingRow = {
 // live-set re-read, not via mutual exclusion.
 async function gcBlobIfUnreferenced(
   handle: Handle, tag: string, hash: string, modifiedMs: number, now: number, grace: number,
-): Promise<void> {
-  if (!isValidContentHash(hash)) return
-  if (now - modifiedMs < grace) return
+): Promise<boolean> {
+  if (!isValidContentHash(hash)) return false
+  if (now - modifiedMs < grace) return false
   const refs = await liveHashSet(handle, tag)
-  if (refs.has(hash)) return
+  if (refs.has(hash)) return false
   await handle.blob.unlinkLive(tag, hash)
+  // Log EVERY live-blob deletion unconditionally (not behind `debug`):
+  // this is the only record that the GC removed bytes. A handful per
+  // sweep is normal (superseded versions aging out); a burst across many
+  // workspaces is the smoking gun for the "all uploaded data went
+  // missing" failure — pair it with the sweep summary in `reapOrphans`.
+  console.warn(`objstore-reaper: GC live blob ${debugTag(tag)}/${debugId(hash)} (unreferenced, age ${Math.round((now - modifiedMs) / 1000)}s ≥ grace ${Math.round(grace / 1000)}s)`)
+  return true
 }
 
 // The set of content hashes referenced by the workspace's live rows.
@@ -86,18 +94,22 @@ async function liveHashSet(handle: Handle, tag: string): Promise<Set<string>> {
 // snapshot we read up front can race a concurrent commit; the
 // reference re-read inside `gcBlobIfUnreferenced` (plus the grace
 // window) ensures we never unlink a blob a live row names.
-async function reapUnreferencedForTag(handle: Handle, tag: string, now: number, grace: number): Promise<void> {
-  if (!isValidTag(tag)) return
+// Returns the number of live blobs GC'd for this tag, so `reapOrphans`
+// can surface a sweep-wide total (the headline signal for mass loss).
+async function reapUnreferencedForTag(handle: Handle, tag: string, now: number, grace: number): Promise<number> {
+  if (!isValidTag(tag)) return 0
   const blobs = await handle.blob.listLiveBlobs(tag)
-  if (blobs.length === 0) return
+  if (blobs.length === 0) return 0
   const referenced = await liveHashSet(handle, tag)
+  let gc = 0
   for (const { hash, modifiedMs } of blobs) {
     if (!isValidContentHash(hash)) continue
     // Referenced in our snapshot → skip the grace + re-read path
     // entirely; only unreferenced blobs need it.
     if (referenced.has(hash)) continue
-    await gcBlobIfUnreferenced(handle, tag, hash, modifiedMs, now, grace)
+    if (await gcBlobIfUnreferenced(handle, tag, hash, modifiedMs, now, grace)) gc++
   }
+  return gc
 }
 
 // Drop staging rows older than the TTL and unlink their on-storage
@@ -172,9 +184,10 @@ export async function reapOrphans(handle: Handle, stagingTtlMs: number = STAGING
   const grace = stagingTtlMs
   // Pass 1: tags the live table knows about — GC unreferenced live
   // blobs (past the grace window) against the referenced-hash set.
+  let liveGc = 0
   const liveTagsRows = await handle.listLiveTags.all()
   const liveTags = liveTagsRows.map((r) => r.workspace_tag)
-  for (const tag of liveTags) await reapUnreferencedForTag(handle, tag, now, grace)
+  for (const tag of liveTags) liveGc += await reapUnreferencedForTag(handle, tag, now, grace)
   // Whole-workspace deletes leave residue (dirs / blob-prefixes) the
   // live table no longer lists. Walk the backend's top-level workspace
   // listing to find them; for each straggler tag, GC its unreferenced
@@ -186,7 +199,14 @@ export async function reapOrphans(handle: Handle, stagingTtlMs: number = STAGING
   const liveSet = new Set(liveTags)
   for (const tag of topLevel) {
     if (liveSet.has(tag) || !isValidTag(tag)) continue
-    await reapUnreferencedForTag(handle, tag, now, grace)
+    liveGc += await reapUnreferencedForTag(handle, tag, now, grace)
+  }
+  // Sweep-wide total. A nonzero count means the GC deleted live bytes
+  // this pass — logged unconditionally so "all data went missing"
+  // leaves an obvious server-side trail (a large count over a short
+  // window is the signature). Per-blob lines above carry which/why.
+  if (liveGc > 0) {
+    console.warn(`objstore-reaper: swept ${liveTags.length} live + ${topLevel.length} store tag(s); GC'd ${liveGc} live blob(s)`)
   }
   // Pass 2: stale staging rows + orphan staging blobs. The orphan
   // sweep does per-blob row lookups (no caller-side snapshot), so a