@preventive/triage 1.0.0-alpha.2 → 1.0.0-alpha.4

package/server/objstore/blob-vercel.ts

@@ -119,21 +119,19 @@ type VercelBlobSdk = {
 
 // Recognise "blob is gone" errors uniformly across read/write/delete
 // paths so callers can treat them as success (delete) or
-// not-found (read). The SDK exposes BlobNotFoundError as a class
-// with `.name === 'BlobNotFoundError'`; checking the name string
-// avoids importing the class at the top level (which would force
-// the optional peer dep to resolve).
+// not-found (read). The SDK exposes BlobNotFoundError as a class with
+// `.name === 'BlobNotFoundError'`; checking the name string avoids
+// importing the class at the top level (which would force the optional
+// peer dep to resolve).
 //
-// Class-name check ONLY. The SDK's internal mapper translates every
-// API `not_found` code into BlobNotFoundError-by-name; a bare-404
-// transport leak doesn't reach here. A prior version of this
-// function had a `/does not exist|\b404\b/` fallback that
-// DANGEROUSLY matched BlobStoreNotFoundError's message "This store
-// does not exist." — a config fault (revoked token, deleted store)
-// would silently surface as every-blob-missing across reads and
-// unlinks, masking the fatal misconfiguration. The tight name check
-// lets BlobStoreNotFoundError / other classes propagate as real
-// exceptions.
+// Class-name check ONLY — the SDK's internal mapper turns every API
+// `not_found` into BlobNotFoundError-by-name, so a bare-404 transport
+// leak doesn't reach here. A broader `/does not exist|\b404\b/` match
+// is DANGEROUS: it also matches BlobStoreNotFoundError's "This store
+// does not exist.", so a config fault (revoked token, deleted store)
+// would silently surface as every-blob-missing across reads/unlinks,
+// masking the fatal misconfiguration. The tight name check lets
+// BlobStoreNotFoundError / other classes propagate as real exceptions.
 function isNotFound(err: unknown): boolean {
   if (err == null || typeof err !== 'object') return false
   const name = (err as { name?: unknown }).name
@@ -242,20 +240,17 @@ function buildOpenStagingWriter(sdk: VercelBlobSdk, token: string): BlobBackend[
       abortSignal: ac.signal,
     })
     // Defuse a possible unhandled-rejection if abort() is called
-    // BEFORE finalize() (the REST layer's error path). Attach a
-    // detached `.catch` on the original promise so an early
-    // rejection has a handler; finalize() awaits `putPromise`
-    // directly, which still re-throws the original rejection
-    // (the .catch returns a separate chain that doesn't replace
-    // putPromise's state).
+    // BEFORE finalize() (the REST error path). The detached `.catch`
+    // gives an early rejection a handler; finalize() still awaits
+    // `putPromise` directly and re-throws the original rejection (the
+    // .catch is a separate chain, not a replacement of putPromise).
     putPromise.catch(() => {})
     return {
       writable: pt,
       // Await the upload's completion. After pipeline(req, counter,
-      // pt) resolves, pt has emitted 'end' on the read side and
-      // `put` is finalising the last multipart part. Awaiting here
-      // gives us the same "bytes durable" guarantee that
-      // pipeline-to-WriteStream gives the FS backend.
+      // pt) resolves, pt has emitted 'end' and `put` is finalising the
+      // last multipart part. Awaiting here gives the same "bytes
+      // durable" guarantee pipeline-to-WriteStream gives the FS backend.
       finalize: async () => { await putPromise },
       // Await the SDK's put-promise settlement (rejected via the
       // AbortController). Without this await, a slow upload that
@@ -315,11 +310,11 @@ function buildPromoteStagingToLive(sdk: VercelBlobSdk, token: string): BlobBacke
       //
       // `allowOverwrite: true` because the content-addressed live
       // pathname `${tag}/${contentHash}.bin` can be (re)written by a
-      // retried or racing promote of the same blob. The destination
-      // bytes are identical by construction (the path IS the hash), so
-      // the overwrite is idempotent — never a clobber of DIFFERENT
-      // bytes. Without the flag, the SDK sends `x-allow-overwrite: 0`
-      // and Vercel rejects any such re-promote with BlobAccessError.
+      // retried or racing promote of the same blob. Destination bytes
+      // are identical by construction (the path IS the hash), so the
+      // overwrite is idempotent — never a clobber of DIFFERENT bytes.
+      // Without the flag the SDK sends `x-allow-overwrite: 0` and
+      // Vercel rejects the re-promote with BlobAccessError.
       await sdk.copy(from, to, {
         access: 'private',
         allowOverwrite: true,
@@ -356,38 +351,55 @@ function buildOpenLiveReader(sdk: VercelBlobSdk, token: string): BlobBackend['op
     // origin truth. Origin fetch is the right default for a store
     // where freshness > latency.
     try { res = await sdk.get(path, { access: 'private', useCache: false, token }) } catch (err) {
-      if (isNotFound(err)) return { ok: false, reason: 'not-found' }
+      // A missing blob HERE is never "the resource doesn't exist" — the
+      // REST layer (rest.ts openLiveSnapshot) already confirmed a live row
+      // whose (version, incarnation) matches the GET token before calling
+      // us. So BlobNotFoundError means the bytes for a still-live row are
+      // momentarily gone: the reaper GC'd a hash a racing version-bump just
+      // unreferenced, or Vercel's read-after-write / edge propagation hasn't
+      // caught up to a freshly-promoted private blob. That is the documented
+      // `unavailable` (HTTP 503) transient — reconciled by reaper /
+      // propagation, retried by the client — NOT a 404. Returning
+      // `not-found` would emit a 404 the FS backend never emits for the
+      // same condition (blob-fs.ts maps ENOENT → `unavailable`), telling
+      // the client the resource is gone for good when it should refetch.
+      // See server/README.md's GET status table.
+      if (isNotFound(err)) return { ok: false, reason: 'unavailable', detail: 'vercel-get-not-found' }
       throw err
     }
-    if (res == null) return { ok: false, reason: 'not-found' }
+    // SDK returned null (no blob) — same "bytes missing for a live row"
+    // transient as the BlobNotFoundError branch above → `unavailable`, not
+    // `not-found`.
+    if (res == null) return { ok: false, reason: 'unavailable', detail: 'vercel-get-null' }
     // statusCode 304 doesn't reach here in practice — the REST
     // GET layer doesn't pass If-None-Match — but a future call
     // site could. Treat as unavailable rather than streaming a
     // null body.
     if (res.statusCode !== 200 || res.stream == null) {
-      return { ok: false, reason: 'unavailable' }
+      return { ok: false, reason: 'unavailable', detail: `vercel-get-status-${res.statusCode}` }
     }
-    // `@vercel/blob@2.x`'s streaming `get()` for private blobs
-    // returns the body but does NOT populate `blob.size` and does
-    // NOT pass a `content-length` header through (verified
-    // empirically: get.size=0, content-length-hdr=null while
-    // head.size reports the true byte count). The REST layer
-    // depends on a size to set `content-length` on its response
-    // and for the integrity check against the DB row, so when
-    // get() leaves it 0/null we fall back to a head() lookup.
-    // Two round-trips per private read on Vercel until the SDK is
-    // fixed — small price vs. the alternative of 503ing every read.
+    // `@vercel/blob@2.x`'s streaming `get()` for private blobs returns
+    // the body but does NOT populate `blob.size` nor pass a
+    // `content-length` header (verified empirically: get.size=0,
+    // content-length-hdr=null, while head.size reports the true count).
+    // The REST layer needs a size to set `content-length` and for the
+    // integrity check against the DB row, so on 0/null we fall back to
+    // head(). Two round-trips per private read until the SDK is fixed —
+    // small price vs. 503ing every read.
     let size: number | null | undefined = res.blob?.size
     if (size == null || size === 0) {
       try {
         const h = await sdk.head(path, { token })
         size = (h as { size?: number })?.size
       } catch (headErr) {
-        if (isNotFound(headErr)) return { ok: false, reason: 'not-found' }
+        // Blob vanished between get() and the head() size fallback (a
+        // racing reaper GC) — still the "live row present, bytes gone"
+        // transient, so `unavailable` (503), matching the get() path above.
+        if (isNotFound(headErr)) return { ok: false, reason: 'unavailable', detail: 'vercel-head-not-found' }
         throw headErr
       }
     }
-    if (size == null) return { ok: false, reason: 'unavailable' }
+    if (size == null) return { ok: false, reason: 'unavailable', detail: 'vercel-no-size' }
     // SDK returns a web ReadableStream<Uint8Array>; the REST layer
     // expects a Node Readable for pipeline(). Convert via
     // Readable.fromWeb — built-in and zero-copy where possible.

package/server/objstore/blob.ts

@@ -83,13 +83,27 @@ export type LiveReader = {
   close(): Promise<void>
 }
 
-// `not-found` maps to HTTP 404 (the live blob is gone or never
-// existed); `unavailable` maps to HTTP 503 (transient backend issue
-// the reaper will eventually sort out). The REST layer uses this
-// discrimination to set the right status code.
+// A missing blob is always `unavailable` (→ HTTP 503), never a 404. The
+// byte plane has NO view of the metadata row, so it can't decide whether
+// a resource "doesn't exist" — only whether specific bytes are present
+// right now. The authoritative "this resource/version is gone" 404 is the
+// REST layer's call, made from the live row BEFORE it opens a reader
+// (rest.ts openLiveSnapshot). By the time `openLiveReader` runs the row is
+// already confirmed, so an absent blob means a transient bytes/metadata
+// desync the reaper (or store propagation) reconciles — exactly the
+// `unavailable`/503 contract, which the client retries. Both backends MUST
+// map a missing blob to `unavailable` (FS: ENOENT; Vercel: BlobNotFoundError
+// / null get()). No 404-mapping variant exists here so that bug can't recur.
+//
+// `detail` is a short, NON-SENSITIVE machine tag for the specific cause
+// (e.g. 'vercel-get-not-found', 'fs-enoent', 'vercel-no-size'). Every
+// byte-side failure collapses to the same 503 on the wire, so a permanent
+// loss (reaper GC'd the bytes) and a transient read fault are otherwise
+// indistinguishable — the REST layer logs `detail` so an operator can tell
+// them apart. Purely diagnostic; the REST status is unchanged.
 export type OpenLiveResult =
   | { ok: true; reader: LiveReader }
-  | { ok: false; reason: 'not-found' | 'unavailable' }
+  | { ok: false; reason: 'unavailable'; detail?: string }
 
 export type BlobBackend = {
   // Per-workspace setup. FS creates the on-disk staging directory;
@@ -128,9 +142,10 @@ export type BlobBackend = {
   promoteStagingToLive(tag: string, stagingId: string, contentHash: string): Promise<boolean>
 
   // Open a streaming reader for the content-addressed live blob.
-  // `not-found` lets the REST layer return 404; `unavailable` returns
-  // 503 for a transient state (file/blob missing while the row still
-  // exists — reaper will reconcile on the next sweep).
+  // Called only after the REST layer has confirmed the live row, so a
+  // missing blob is the transient "row present, bytes gone" state →
+  // `unavailable` (HTTP 503), which the reaper reconciles and the client
+  // retries. Never a 404 from here — see OpenLiveResult above.
   openLiveReader(tag: string, contentHash: string): Promise<OpenLiveResult>
 
   // Idempotent deletes. Backends MUST tolerate "already gone" as

package/server/objstore/handlers.ts

@@ -68,11 +68,11 @@ function urlPathFor(tag: string, resourceTag: string): string {
 // Shared gate every objstore handler runs after its message-specific
 // field checks: fetch the socket's challenge nonce, verify the signed
 // message against it, then re-confirm the socket is still OPEN — the
-// close handler may have fired during the verify await, and attaching
-// to / replying on a closed socket is the half-handshake leak case
-// (PR #4 review F4). Returns true when the caller may proceed.
-// Centralising the post-await readyState recheck keeps that
-// easy-to-forget invariant in one auditable place.
+// close handler may have fired during the verify await, and replying
+// on a closed socket is the half-handshake leak case (PR #4 review
+// F4). Centralising the post-await readyState recheck keeps that
+// invariant in one auditable place. Returns true when the caller may
+// proceed.
 async function verified<M extends { workspaceTag?: unknown }>(
   deps: ObjstoreDeps, socket: WebSocket, msg: M, label: string,
   verify: (m: M, nonce: string) => Promise<boolean>,
@@ -96,12 +96,10 @@ async function handlePutBegin(deps: ObjstoreDeps, socket: WebSocket, msg: Objsto
   // signature to fail. Cheaper to reject up-front, and consistent
   // with `verifyObjstorePutSig`'s `isSafeNonNegativeInt` gate.
   if (!Number.isSafeInteger(msg.expectedLength) || (msg.expectedLength as number) < 0 || (msg.expectedLength as number) > MAX_CONTENT_LENGTH) return
-  // Symmetric with `handleDelete`'s prevVersion gate (line 116) and
-  // `verifyObjstorePutSig`'s `isSafeIntOrNull` (sign.ts:119). Without
-  // this, a non-safe-integer `prevVersion` (NaN, 2^53+1, ...) would
-  // pass the typeof check below and reach sig verify, burning a
-  // hash + Ed25519 round-trip on a guaranteed-fail input. Input-
-  // validation audit `server/objstore/handlers.ts:76`.
+  // Same up-front reject for `prevVersion` (symmetric with handleDelete
+  // and `verifyObjstorePutSig`'s `isSafeIntOrNull`): a non-safe-integer
+  // (NaN, 2^53+1, ...) would pass the typeof check below and reach sig
+  // verify, burning a hash + Ed25519 round-trip on a guaranteed fail.
   if (msg.prevVersion != null && (typeof msg.prevVersion !== 'number' || !Number.isSafeInteger(msg.prevVersion))) return
   if (!await verified(deps, socket, msg, 'put-begin', verifyObjstorePutSig)) return
   const tag = msg.workspaceTag
@@ -110,8 +108,8 @@ async function handlePutBegin(deps: ObjstoreDeps, socket: WebSocket, msg: Objsto
   // workspace tag (no rows in workspace_revision AND none in
   // workspace_object). Mirrors handleSave in server/index.ts; runs
   // AFTER sig verify so `unauthorized` only reaches a legitimate
-  // signer. The gate is config-driven (server/config.json
-  // `password`) and is a no-op when no password is configured.
+  // signer. Config-driven (server/config.json `password`), no-op when
+  // unconfigured.
   if (deps.authGate && deps.sendUnauthorized && await deps.authGate(socket, tag)) {
     if (socket.readyState !== socket.OPEN) return
     if (deps.debug) console.warn(`reject objstore-put-begin: unauthorized (new workspace ${debugTag(tag)})`)

package/server/objstore/init.ts

@@ -56,6 +56,16 @@ export type ObjstoreInit = {
 }
 
 export function initObjstore(deps: ObjstoreInitDeps): ObjstoreInit {
+  // Fail loud on a lopsided auth config. The put-begin gate in
+  // handlers.ts only fires when BOTH authGate and sendUnauthorized are
+  // present (it needs the reporter to emit the `unauthorized` frame),
+  // so wiring authGate WITHOUT sendUnauthorized silently fails OPEN —
+  // unauthenticated first-writes to unknown workspaces would be accepted
+  // despite the operator's intent to gate them. Reject at boot rather
+  // than regress access control silently.
+  if (deps.authGate && !deps.sendUnauthorized) {
+    throw new Error('initObjstore: authGate requires sendUnauthorized (the put-begin gate needs it to emit the unauthorized frame)')
+  }
   const handle = deps.handle
   const secret = deps.tokenSecret ?? newTokenSecret()
   const handlers = createObjstoreHandlers({
@@ -93,15 +103,14 @@ export function initObjstore(deps: ObjstoreInitDeps): ObjstoreInit {
   // on-disk state still has stranded files from a prior crash.
   const startupReap = enqueueSweep()
   // Jittered start of the periodic timer. Multi-replica deploys
-  // (Neon + Vercel Blob) commonly boot N replicas in tight lock-
-  // step (deploy rollout, cluster restart) and would otherwise
-  // sync every replica's reaper at the same wall-clock tick,
-  // hammering the DB + blob store with N×readdir+lock-acquire
-  // bursts. A random first-interval delay deconcurrencies the
-  // cluster without changing the long-term sweep cadence.
-  // Jitter range is 0…1× reapIntervalMs (i.e., the next sweep
-  // happens at [interval, 2×interval] after boot); subsequent
-  // sweeps stay at exactly `reapIntervalMs` apart.
+  // (Neon + Vercel Blob) commonly boot N replicas in tight lock-step
+  // (deploy rollout, cluster restart), which would otherwise sync every
+  // replica's reaper to the same wall-clock tick and hammer the DB +
+  // blob store with N×readdir+lock-acquire bursts. A random
+  // first-interval delay spreads the cluster out without changing the
+  // long-term cadence. Jitter is 0…1× reapIntervalMs (first sweep at
+  // [interval, 2×interval] after boot); subsequent sweeps stay exactly
+  // `reapIntervalMs` apart.
   let reapTimer: ReturnType<typeof setInterval> | null = null
   const jitterMs = Math.floor(Math.random() * deps.reapIntervalMs)
   const firstTimer = setTimeout(() => {

package/server/objstore/reaper.ts

@@ -27,6 +27,7 @@
 // predicate can't match a row a concurrent upload just refreshed.
 
 import { type Handle, STAGING_TTL_MS_DEFAULT, isValidContentHash, isValidStagingId, isValidTag } from './store.ts'
+import { debugId, debugTag } from '../util.ts'
 
 type StagingRow = {
   workspace_tag: string
@@ -67,12 +68,19 @@ type StagingRow = {
 // live-set re-read, not via mutual exclusion.
 async function gcBlobIfUnreferenced(
   handle: Handle, tag: string, hash: string, modifiedMs: number, now: number, grace: number,
-): Promise<void> {
-  if (!isValidContentHash(hash)) return
-  if (now - modifiedMs < grace) return
+): Promise<boolean> {
+  if (!isValidContentHash(hash)) return false
+  if (now - modifiedMs < grace) return false
   const refs = await liveHashSet(handle, tag)
-  if (refs.has(hash)) return
+  if (refs.has(hash)) return false
   await handle.blob.unlinkLive(tag, hash)
+  // Log EVERY live-blob deletion unconditionally (not behind `debug`):
+  // this is the only record that the GC removed bytes. A handful per
+  // sweep is normal (superseded versions aging out); a burst across many
+  // workspaces is the smoking gun for the "all uploaded data went
+  // missing" failure — pair it with the sweep summary in `reapOrphans`.
+  console.warn(`objstore-reaper: GC live blob ${debugTag(tag)}/${debugId(hash)} (unreferenced, age ${Math.round((now - modifiedMs) / 1000)}s ≥ grace ${Math.round(grace / 1000)}s)`)
+  return true
 }
 
 // The set of content hashes referenced by the workspace's live rows.
@@ -86,18 +94,22 @@ async function liveHashSet(handle: Handle, tag: string): Promise<Set<string>> {
 // snapshot we read up front can race a concurrent commit; the
 // reference re-read inside `gcBlobIfUnreferenced` (plus the grace
 // window) ensures we never unlink a blob a live row names.
-async function reapUnreferencedForTag(handle: Handle, tag: string, now: number, grace: number): Promise<void> {
-  if (!isValidTag(tag)) return
+// Returns the number of live blobs GC'd for this tag, so `reapOrphans`
+// can surface a sweep-wide total (the headline signal for mass loss).
+async function reapUnreferencedForTag(handle: Handle, tag: string, now: number, grace: number): Promise<number> {
+  if (!isValidTag(tag)) return 0
   const blobs = await handle.blob.listLiveBlobs(tag)
-  if (blobs.length === 0) return
+  if (blobs.length === 0) return 0
   const referenced = await liveHashSet(handle, tag)
+  let gc = 0
   for (const { hash, modifiedMs } of blobs) {
     if (!isValidContentHash(hash)) continue
     // Referenced in our snapshot → skip the grace + re-read path
     // entirely; only unreferenced blobs need it.
     if (referenced.has(hash)) continue
-    await gcBlobIfUnreferenced(handle, tag, hash, modifiedMs, now, grace)
+    if (await gcBlobIfUnreferenced(handle, tag, hash, modifiedMs, now, grace)) gc++
   }
+  return gc
 }
 
 // Drop staging rows older than the TTL and unlink their on-storage
@@ -172,9 +184,10 @@ export async function reapOrphans(handle: Handle, stagingTtlMs: number = STAGING
   const grace = stagingTtlMs
   // Pass 1: tags the live table knows about — GC unreferenced live
   // blobs (past the grace window) against the referenced-hash set.
+  let liveGc = 0
   const liveTagsRows = await handle.listLiveTags.all()
   const liveTags = liveTagsRows.map((r) => r.workspace_tag)
-  for (const tag of liveTags) await reapUnreferencedForTag(handle, tag, now, grace)
+  for (const tag of liveTags) liveGc += await reapUnreferencedForTag(handle, tag, now, grace)
   // Whole-workspace deletes leave residue (dirs / blob-prefixes) the
   // live table no longer lists. Walk the backend's top-level workspace
   // listing to find them; for each straggler tag, GC its unreferenced
@@ -186,7 +199,14 @@ export async function reapOrphans(handle: Handle, stagingTtlMs: number = STAGING
   const liveSet = new Set(liveTags)
   for (const tag of topLevel) {
     if (liveSet.has(tag) || !isValidTag(tag)) continue
-    await reapUnreferencedForTag(handle, tag, now, grace)
+    liveGc += await reapUnreferencedForTag(handle, tag, now, grace)
+  }
+  // Sweep-wide total. A nonzero count means the GC deleted live bytes
+  // this pass — logged unconditionally so "all data went missing"
+  // leaves an obvious server-side trail (a large count over a short
+  // window is the signature). Per-blob lines above carry which/why.
+  if (liveGc > 0) {
+    console.warn(`objstore-reaper: swept ${liveTags.length} live + ${topLevel.length} store tag(s); GC'd ${liveGc} live blob(s)`)
   }
   // Pass 2: stale staging rows + orphan staging blobs. The orphan
   // sweep does per-blob row lookups (no caller-side snapshot), so a