@preventive/triage 1.0.0-alpha.3 → 1.0.0-alpha.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@preventive/triage",
-  "version": "1.0.0-alpha.3",
+  "version": "1.0.0-alpha.4",
   "description": "Client & relay server for triaging of automated reports",
   "license": "MIT",
   "author": {

package/server/objstore/blob-fs.ts

@@ -48,13 +48,13 @@ async function fsOpenLiveReader(dir: string, tag: string, contentHash: string):
   const path = liveFilePath(dir, tag, contentHash)
   let fh
   try { fh = await open(path, 'r') } catch (err: unknown) {
-    if ((err as NodeJS.ErrnoException)?.code === 'ENOENT') return { ok: false, reason: 'unavailable' }
+    if ((err as NodeJS.ErrnoException)?.code === 'ENOENT') return { ok: false, reason: 'unavailable', detail: 'fs-enoent' }
     throw err
   }
   let size: number
   try { size = (await fh.stat()).size } catch {
     await fh.close().catch(() => {})
-    return { ok: false, reason: 'unavailable' }
+    return { ok: false, reason: 'unavailable', detail: 'fs-stat-failed' }
   }
   const stream = fh.createReadStream()
   let closed = false

package/server/objstore/blob-vercel.ts

@@ -364,19 +364,19 @@ function buildOpenLiveReader(sdk: VercelBlobSdk, token: string): BlobBackend['op
       // same condition (blob-fs.ts maps ENOENT → `unavailable`), telling
       // the client the resource is gone for good when it should refetch.
       // See server/README.md's GET status table.
-      if (isNotFound(err)) return { ok: false, reason: 'unavailable' }
+      if (isNotFound(err)) return { ok: false, reason: 'unavailable', detail: 'vercel-get-not-found' }
       throw err
     }
     // SDK returned null (no blob) — same "bytes missing for a live row"
     // transient as the BlobNotFoundError branch above → `unavailable`, not
     // `not-found`.
-    if (res == null) return { ok: false, reason: 'unavailable' }
+    if (res == null) return { ok: false, reason: 'unavailable', detail: 'vercel-get-null' }
     // statusCode 304 doesn't reach here in practice — the REST
     // GET layer doesn't pass If-None-Match — but a future call
     // site could. Treat as unavailable rather than streaming a
     // null body.
     if (res.statusCode !== 200 || res.stream == null) {
-      return { ok: false, reason: 'unavailable' }
+      return { ok: false, reason: 'unavailable', detail: `vercel-get-status-${res.statusCode}` }
     }
     // `@vercel/blob@2.x`'s streaming `get()` for private blobs returns
     // the body but does NOT populate `blob.size` nor pass a
@@ -395,11 +395,11 @@ function buildOpenLiveReader(sdk: VercelBlobSdk, token: string): BlobBackend['op
         // Blob vanished between get() and the head() size fallback (a
         // racing reaper GC) — still the "live row present, bytes gone"
         // transient, so `unavailable` (503), matching the get() path above.
-        if (isNotFound(headErr)) return { ok: false, reason: 'unavailable' }
+        if (isNotFound(headErr)) return { ok: false, reason: 'unavailable', detail: 'vercel-head-not-found' }
         throw headErr
       }
     }
-    if (size == null) return { ok: false, reason: 'unavailable' }
+    if (size == null) return { ok: false, reason: 'unavailable', detail: 'vercel-no-size' }
     // SDK returns a web ReadableStream<Uint8Array>; the REST layer
     // expects a Node Readable for pipeline(). Convert via
     // Readable.fromWeb — built-in and zero-copy where possible.

package/server/objstore/blob.ts

@@ -94,9 +94,16 @@ export type LiveReader = {
 // `unavailable`/503 contract, which the client retries. Both backends MUST
 // map a missing blob to `unavailable` (FS: ENOENT; Vercel: BlobNotFoundError
 // / null get()). No 404-mapping variant exists here so that bug can't recur.
+//
+// `detail` is a short, NON-SENSITIVE machine tag for the specific cause
+// (e.g. 'vercel-get-not-found', 'fs-enoent', 'vercel-no-size'). Every
+// byte-side failure collapses to the same 503 on the wire, so a permanent
+// loss (reaper GC'd the bytes) and a transient read fault are otherwise
+// indistinguishable — the REST layer logs `detail` so an operator can tell
+// them apart. Purely diagnostic; the REST status is unchanged.
 export type OpenLiveResult =
   | { ok: true; reader: LiveReader }
-  | { ok: false; reason: 'unavailable' }
+  | { ok: false; reason: 'unavailable'; detail?: string }
 
 export type BlobBackend = {
   // Per-workspace setup. FS creates the on-disk staging directory;

package/server/objstore/reaper.ts

@@ -27,6 +27,7 @@
 // predicate can't match a row a concurrent upload just refreshed.
 
 import { type Handle, STAGING_TTL_MS_DEFAULT, isValidContentHash, isValidStagingId, isValidTag } from './store.ts'
+import { debugId, debugTag } from '../util.ts'
 
 type StagingRow = {
   workspace_tag: string
@@ -67,12 +68,19 @@ type StagingRow = {
 // live-set re-read, not via mutual exclusion.
 async function gcBlobIfUnreferenced(
   handle: Handle, tag: string, hash: string, modifiedMs: number, now: number, grace: number,
-): Promise<void> {
-  if (!isValidContentHash(hash)) return
-  if (now - modifiedMs < grace) return
+): Promise<boolean> {
+  if (!isValidContentHash(hash)) return false
+  if (now - modifiedMs < grace) return false
   const refs = await liveHashSet(handle, tag)
-  if (refs.has(hash)) return
+  if (refs.has(hash)) return false
   await handle.blob.unlinkLive(tag, hash)
+  // Log EVERY live-blob deletion unconditionally (not behind `debug`):
+  // this is the only record that the GC removed bytes. A handful per
+  // sweep is normal (superseded versions aging out); a burst across many
+  // workspaces is the smoking gun for the "all uploaded data went
+  // missing" failure — pair it with the sweep summary in `reapOrphans`.
+  console.warn(`objstore-reaper: GC live blob ${debugTag(tag)}/${debugId(hash)} (unreferenced, age ${Math.round((now - modifiedMs) / 1000)}s ≥ grace ${Math.round(grace / 1000)}s)`)
+  return true
 }
 
 // The set of content hashes referenced by the workspace's live rows.
@@ -86,18 +94,22 @@ async function liveHashSet(handle: Handle, tag: string): Promise<Set<string>> {
 // snapshot we read up front can race a concurrent commit; the
 // reference re-read inside `gcBlobIfUnreferenced` (plus the grace
 // window) ensures we never unlink a blob a live row names.
-async function reapUnreferencedForTag(handle: Handle, tag: string, now: number, grace: number): Promise<void> {
-  if (!isValidTag(tag)) return
+// Returns the number of live blobs GC'd for this tag, so `reapOrphans`
+// can surface a sweep-wide total (the headline signal for mass loss).
+async function reapUnreferencedForTag(handle: Handle, tag: string, now: number, grace: number): Promise<number> {
+  if (!isValidTag(tag)) return 0
   const blobs = await handle.blob.listLiveBlobs(tag)
-  if (blobs.length === 0) return
+  if (blobs.length === 0) return 0
   const referenced = await liveHashSet(handle, tag)
+  let gc = 0
   for (const { hash, modifiedMs } of blobs) {
     if (!isValidContentHash(hash)) continue
     // Referenced in our snapshot → skip the grace + re-read path
     // entirely; only unreferenced blobs need it.
     if (referenced.has(hash)) continue
-    await gcBlobIfUnreferenced(handle, tag, hash, modifiedMs, now, grace)
+    if (await gcBlobIfUnreferenced(handle, tag, hash, modifiedMs, now, grace)) gc++
   }
+  return gc
 }
 
 // Drop staging rows older than the TTL and unlink their on-storage
@@ -172,9 +184,10 @@ export async function reapOrphans(handle: Handle, stagingTtlMs: number = STAGING
   const grace = stagingTtlMs
   // Pass 1: tags the live table knows about — GC unreferenced live
   // blobs (past the grace window) against the referenced-hash set.
+  let liveGc = 0
   const liveTagsRows = await handle.listLiveTags.all()
   const liveTags = liveTagsRows.map((r) => r.workspace_tag)
-  for (const tag of liveTags) await reapUnreferencedForTag(handle, tag, now, grace)
+  for (const tag of liveTags) liveGc += await reapUnreferencedForTag(handle, tag, now, grace)
   // Whole-workspace deletes leave residue (dirs / blob-prefixes) the
   // live table no longer lists. Walk the backend's top-level workspace
   // listing to find them; for each straggler tag, GC its unreferenced
@@ -186,7 +199,14 @@ export async function reapOrphans(handle: Handle, stagingTtlMs: number = STAGING
   const liveSet = new Set(liveTags)
   for (const tag of topLevel) {
     if (liveSet.has(tag) || !isValidTag(tag)) continue
-    await reapUnreferencedForTag(handle, tag, now, grace)
+    liveGc += await reapUnreferencedForTag(handle, tag, now, grace)
+  }
+  // Sweep-wide total. A nonzero count means the GC deleted live bytes
+  // this pass — logged unconditionally so "all data went missing"
+  // leaves an obvious server-side trail (a large count over a short
+  // window is the signature). Per-blob lines above carry which/why.
+  if (liveGc > 0) {
+    console.warn(`objstore-reaper: swept ${liveTags.length} live + ${topLevel.length} store tag(s); GC'd ${liveGc} live blob(s)`)
   }
   // Pass 2: stale staging rows + orphan staging blobs. The orphan
   // sweep does per-blob row lookups (no caller-side snapshot), so a

package/server/objstore/rest.ts

@@ -41,7 +41,7 @@ import {
 } from './store.ts'
 import type { LiveReader } from './blob.ts'
 import { type TokenSecret, extractBearer, verifyToken } from './tokens.ts'
-import { errStack } from '../util.ts'
+import { debugId, debugTag, errMsg, errStack } from '../util.ts'
 
 // Server-side fault codes that should surface as 500 `io-error`
 // rather than 400 `aborted`. `pipeline(req, ws)` rejects with the
@@ -431,7 +431,10 @@ async function runUploadAndCommit(
 type GetOpened =
   | { reason: 'ok'; reader: LiveReader }
   | { reason: 'not-found' }
-  | { reason: 'unavailable' }
+  // `detail` is a short non-sensitive cause tag (backend sub-reason +
+  // content-hash prefix) the GET handler logs so a 503 is diagnosable —
+  // every byte-side failure collapses to the same wire 503 otherwise.
+  | { reason: 'unavailable'; detail: string }
 
 async function openLiveSnapshot(
   deps: ObjstoreRestDeps, route: RouteMatch, payload: { ver: number; inc: string },
@@ -448,17 +451,27 @@ async function openLiveSnapshot(
   // returns a pinned fd; for the Vercel backend a fetch-backed stream.)
   const live = await deps.handle.selectLiveOne.get(route.tag, route.resourceTag)
   if (!live || live.version !== payload.ver || live.incarnation !== payload.inc) return { reason: 'not-found' }
+  // Tag the content hash into every `unavailable` detail so an operator
+  // can go check the byte store directly for THIS blob (gone → reaper /
+  // deletion; present → transient read fault).
+  const hashTag = `hash=${debugId(live.content_hash)}`
   let opened
   try { opened = await deps.handle.blob.openLiveReader(route.tag, live.content_hash) }
-  catch { return { reason: 'unavailable' } }
-  if (!opened.ok) return { reason: opened.reason }
+  // Length-cap the thrown error text: today a non-BlobNotFound SDK throw
+  // (BlobServiceNotAvailable / store-not-found) carries no credential
+  // (the RW token rides the Authorization header, never `.message`), but
+  // a future SDK could embed a signed URL / token fragment — bound the
+  // log line so it can't dump one verbatim. Mirrors the `.slice(0, 200)`
+  // cap used on SDK error text in blob-vercel.ts.
+  catch (err) { return { reason: 'unavailable', detail: `open-threw ${hashTag} ${String(errMsg(err)).slice(0, 200)}` } }
+  if (!opened.ok) return { reason: 'unavailable', detail: `${opened.detail ?? 'backend'} ${hashTag}` }
   // Size mismatch between the live row and the on-storage bytes
   // is a transient inconsistency — reaper will reconcile. Close
   // the reader before returning so we don't leak the fd / fetch
   // reader. PR #4 review H8.
   if (opened.reader.size !== live.content_length) {
     await opened.reader.close().catch(() => {})
-    return { reason: 'unavailable' }
+    return { reason: 'unavailable', detail: `size-mismatch row=${live.content_length} blob=${opened.reader.size} ${hashTag}` }
   }
   return { reason: 'ok', reader: opened.reader }
 }
@@ -478,8 +491,19 @@ async function handleRestGet(
   // If the live row is there but the bytes are missing / wrong size,
   // it's a transient inconsistency the reaper will sort out — 503
   // (vs 404) tells the client this is a server-side state, not a
-  // "the resource truly isn't there" answer.
-  if (opened.reason === 'unavailable') { deny(res, 503, 'unavailable'); return }
+  // "the resource truly isn't there" answer. Log the cause
+  // UNCONDITIONALLY (not behind `debug`): a 503 means a live row whose
+  // bytes can't be served, and the wire response can't distinguish a
+  // permanent loss (reaper GC'd referenced bytes) from a transient read
+  // fault. `detail` carries the backend sub-reason + content-hash prefix
+  // so an operator can tell which — the only server-side breadcrumb for
+  // the "all data turned into 503" failure. (Volume is bounded: a 503 is
+  // an error path; a workspace-wide outage is exactly when these are
+  // wanted.)
+  if (opened.reason === 'unavailable') {
+    console.warn(`objstore-get: 503 unavailable ${debugTag(route.tag)}/${route.resourceTag.slice(0, 8)}… v${payload.ver} ${opened.detail}`)
+    deny(res, 503, 'unavailable'); return
+  }
   res.writeHead(200, {
     'content-type': 'application/octet-stream',
     'content-length': String(opened.reader.size),

package/server/util.ts

@@ -8,6 +8,15 @@ import { randomBytes } from 'node:crypto'
 // is an Ed25519 public key; operator logs shouldn't carry it verbatim.
 export function debugTag(s: string): string { return `${s.slice(0, 12)}…` }
 
+// Truncated view of a content hash / staging id for logs (objstore GC +
+// 503 diagnostics). Same 12-char prefix convention as `debugTag`; named
+// separately so call sites read as "this is a blob id, not a workspace
+// tag". Tolerates a non-string (logs a placeholder) so a diagnostic
+// path can't itself throw on bad input.
+export function debugId(s: unknown): string {
+  return typeof s === 'string' ? `${s.slice(0, 12)}…` : '<no-id>'
+}
+
 // 16 random bytes → 22 base64url chars (no padding). The shared shape
 // for per-socket challenge nonces and staging ids — collision is
 // 1/2^128. `isValidStagingId` in objstore/store.ts validates exactly