@preventive/triage 1.0.0-alpha.0

package/server/lifecycle.ts

@@ -0,0 +1,177 @@
+// Process lifecycle: the in-flight request tracker, the `shuttingDown`
+// gate, the graceful-shutdown choreography, and the signal / error /
+// uncaught-exception handlers that drive it. Two-phase so the bits the
+// HTTP + WS planes need (`track`, `isShuttingDown`) exist BEFORE those
+// servers are built, while the teardown is `install()`ed afterward —
+// once the server objects, heartbeat timer, and reaper exist.
+
+import type { Server } from 'node:http'
+import type { WebSocketServer } from 'ws'
+import { errStack } from './util.ts'
+
+export type ShutdownDeps = {
+  httpServer: Server
+  wss: WebSocketServer
+  heartbeatTimer: ReturnType<typeof setInterval>
+  // Stops the periodic reaper AND awaits any in-flight sweep.
+  stopReaper: () => Promise<void>
+  // App-specific teardown, run AFTER the in-flight drain: close the DB.
+  // Swallows its own errors (a failed close shouldn't abort the exit).
+  closeDb: () => Promise<void>
+}
+
+export type Lifecycle = {
+  // Collects in-flight async handlers so `shutdown` can drain them
+  // before closing the DB. Passed to the HTTP + WS planes.
+  track: (promise: Promise<unknown>) => void
+  // Read by the REST shutdown gate + the WS message loop to drop new
+  // work once teardown began.
+  isShuttingDown: () => boolean
+  // Wire the graceful shutdown + the error / signal / process-catchall
+  // handlers. Call once, after the server objects exist.
+  install: (deps: ShutdownDeps) => void
+}
+
+export function createLifecycle(): Lifecycle {
+  // In-flight async message handlers. `shutdown` awaits this set before
+  // closing the DB so a SIGINT mid-save can't resume against a closed
+  // handle (which would throw inside the commit's gated INSERT after
+  // the client believed its save was committed).
+  const inFlight = new Set<Promise<unknown>>()
+  function track(promise: Promise<unknown>): void {
+    inFlight.add(promise)
+    promise.finally(() => inFlight.delete(promise))
+  }
+  let shuttingDown = false
+  // Live exit code the in-progress shutdown will pass to `process.exit`.
+  // Re-entry can ESCALATE it from 0 → 1 (e.g. a `wss.error` firing
+  // during a SIGTERM-driven graceful shutdown shouldn't leave the
+  // launcher seeing a clean exit code) but can never DE-escalate.
+  // Audit round-13.
+  let pendingExitCode = 0
+
+  function install(deps: ShutdownDeps): void {
+    const { httpServer, wss, heartbeatTimer, stopReaper, closeDb } = deps
+
+    async function shutdown(exitCode: number = 0): Promise<void> {
+      // Re-entry: don't restart the teardown, but escalate the pending
+      // exit code if the new caller is non-zero (e.g. a wss.error during
+      // a SIGTERM-driven graceful shutdown). Without this, an error
+      // arriving mid-shutdown would silently exit 0 and the launcher
+      // would record a clean stop.
+      if (shuttingDown) {
+        if (exitCode !== 0 && pendingExitCode === 0) pendingExitCode = exitCode
+        return
+      }
+      shuttingDown = true
+      pendingExitCode = exitCode
+      console.log('Shutting down…')
+      // Stop the heartbeat so a tick can't fire mid-shutdown and ping a
+      // socket the close-loop below already started tearing down.
+      clearInterval(heartbeatTimer)
+      // Send a 1001 (going away) close frame to every open socket BEFORE
+      // shutting the listener. Lets clients distinguish a server-initiated
+      // graceful shutdown from a network drop, so they can skip their
+      // reconnect backoff. Fire-and-forget — `process.exit` below would
+      // force-kill any in-progress flush anyway. `try/catch` shrugs at
+      // sockets already in CLOSING / CLOSED.
+      for (const socket of wss.clients) {
+        try { socket.close(1001, 'Server shutting down') } catch {}
+      }
+      // Force-terminate any client that doesn't ack the close frame
+      // within a short grace window. `wss.close()` waits for every client
+      // to emit `'close'`, and `ws` only TCP-RSTs unresponsive peers
+      // after its own ~30 s `closeTimeout`. A single dead/blackholed peer
+      // would otherwise stretch SIGTERM/SIGINT response by that full
+      // timeout. Audit round-11.
+      const TERMINATE_GRACE_MS = 1_000
+      const terminateTimer = setTimeout(() => {
+        for (const socket of wss.clients) {
+          const rs = socket.readyState
+          if (rs === socket.OPEN || rs === socket.CLOSING) {
+            try { socket.terminate() } catch {}
+          }
+        }
+        // Same grace for HTTP keep-alive sockets that didn't respect the
+        // `Connection: close` hint, which would otherwise hold
+        // `httpServer.close()` until their TCP timeout.
+        try { httpServer.closeAllConnections() } catch {}
+      }, TERMINATE_GRACE_MS)
+      // Don't keep the event loop alive solely for the grace timer.
+      terminateTimer.unref?.()
+      // Stop the periodic reaper AND wait for any in-flight sweep (incl.
+      // the startup sweep) before the DB close — otherwise a
+      // readdir / unlink would race a closed DB.
+      await stopReaper()
+      // Free idle HTTP keep-alive sockets up front so close() below
+      // doesn't wait on them. Active in-flight requests still finish.
+      try { httpServer.closeIdleConnections() } catch {}
+      // Close http.Server first to stop accepting new upgrades + HTTP
+      // requests. Guard with `.listening` because `close()` throws
+      // ERR_SERVER_NOT_RUNNING when bind never succeeded (the http error
+      // handler is the path that invoked shutdown in that case).
+      if (httpServer.listening) {
+        await new Promise<void>((resolve) => { httpServer.close(() => resolve()) })
+      }
+      await new Promise<void>((resolve) => { wss.close(() => resolve()) })
+      clearTimeout(terminateTimer)
+      // Drain in-flight handlers so a save that's mid-pipeline finishes
+      // its commit before the DB closes. `handleSave` splits its
+      // canonical/id/dup-precheck/verify/commit work across awaits, so
+      // the window spans several yield points. `Promise.allSettled` so a
+      // single handler rejection doesn't abort the drain.
+      if (inFlight.size > 0) await Promise.allSettled([...inFlight])
+      // App teardown AFTER the drain: close the DB.
+      await closeDb()
+      // Read `pendingExitCode` (not the parameter) so a re-entrant
+      // `shutdown(1)` that landed during the drain wins over the original
+      // `shutdown(0)`. See round-13 escalation note.
+      process.exit(pendingExitCode)
+    }
+
+    // `.catch` defends against an unguarded `await` slipping into
+    // `shutdown`: an unhandled rejection there would skip the non-zero
+    // exit the launcher relies on.
+    function fireShutdown(code: number): void {
+      shutdown(code).catch((err) => {
+        console.warn('shutdown error:', errStack(err))
+        process.exit(code === 0 ? 1 : code)
+      })
+    }
+
+    // Route bind / post-listen failures through `shutdown` so the
+    // in-flight drain + DB close still run before exit. Without this,
+    // `ws` re-emits the error as uncaughtException and the launcher sees
+    // a confusing crash rather than the bind failure. Audit round-9 M2.
+    httpServer.on('error', (err: Error) => {
+      console.error('Server error:', errStack(err))
+      fireShutdown(1)
+    })
+    // Symmetric with the http.Server error handler — route through
+    // `fireShutdown(1)` so the launcher sees a non-zero exit, and the
+    // re-entry escalation bumps `pendingExitCode` 0 → 1 for a `wss.error`
+    // arriving mid-graceful-SIGTERM.
+    wss.on('error', (err: Error) => {
+      console.error('WS server error:', errStack(err))
+      fireShutdown(1)
+    })
+    // Wrap signal handlers so the signal name (the listener's first arg)
+    // doesn't bleed into shutdown's `exitCode`.
+    process.on('SIGINT', () => fireShutdown(0))
+    process.on('SIGTERM', () => fireShutdown(0))
+    // Process-level catchalls so a stray rejection / uncaught exception
+    // doesn't bypass `shutdown()` — Node 20+ exits on unhandled
+    // rejection, which would skip the drain + DB close. Log forensically
+    // and route through `fireShutdown(1)`. Audit round-11 observability.
+    process.on('unhandledRejection', (reason) => {
+      console.error('Unhandled rejection:', errStack(reason))
+      fireShutdown(1)
+    })
+    process.on('uncaughtException', (err) => {
+      console.error('Uncaught exception:', errStack(err))
+      fireShutdown(1)
+    })
+  }
+
+  return { track, isShuttingDown: () => shuttingDown, install }
+}

package/server/neon-driver.ts

@@ -0,0 +1,26 @@
+// Single import indirection for the optional `@neondatabase/serverless`
+// peer dep. Both Neon planes — `db-neon.ts` (workspace_revision) and
+// `objstore/store-neon.ts` (the objstore tables) — dynamically import
+// THIS module instead of the bare specifier, and tests mock THIS
+// module to swap the real driver for an in-process Postgres (PGlite).
+//
+// Why the indirection: node:test's `mock.module` can only intercept a
+// specifier it can RESOLVE. `@neondatabase/serverless` is an optional
+// peer dep that a SQLite-only checkout never installs, so mocking the
+// bare specifier fails with ERR_MODULE_NOT_FOUND at resolve time. A
+// local wrapper path always resolves; and because the mock loader
+// serves synthetic exports, it never evaluates this file's `export *`,
+// so the absent peer dep is never loaded under test.
+//
+// In production the wrapper is only ever imported on the Neon path
+// (`DATABASE_URL` set), where the operator has installed the peer dep
+// (`pnpm add @neondatabase/serverless`) so the re-export resolves. A
+// SQLite-only deploy never imports this module — the dynamic `import()`
+// sites are gated behind the `DATABASE_URL` branch in `index.ts`.
+//
+// `@ts-ignore` rather than `@ts-expect-error`: when the peer dep IS
+// installed the specifier resolves and tsc sees a real type, which
+// would flip `@ts-expect-error` into a TS2578 "unused directive" error
+// and break the operator's `tsc --noEmit`.
+// @ts-ignore optional peer dep: '@neondatabase/serverless'
+export * from '@neondatabase/serverless'

package/server/objstore/blob-fs.ts

@@ -0,0 +1,164 @@
+// Local-filesystem BlobBackend. Wraps the low-level fs primitives
+// in ./fs.ts behind the backend-agnostic interface so the v1.objstore
+// consumers (./store.ts, ./rest.ts, ./reaper.ts) can call uniformly
+// against either FS or Vercel Blob (./blob-vercel.ts).
+//
+// Layout under `dir` (passed to `openFsBlobBackend`):
+//   ${dir}/${workspaceTag}/${contentHash}.bin            — live
+//   ${dir}/${workspaceTag}/.staging/${stagingId}.bin     — staging
+//
+// Live blobs are content-addressed (the filename is the content hash,
+// not the resourceTag) so a hash names exactly one immutable byte-
+// string; tests that import `liveFilePath` / `stagingFilePath` from
+// ./fs.ts compute the right paths.
+
+import { createWriteStream } from 'node:fs'
+import { open, readdir, stat } from 'node:fs/promises'
+import { join } from 'node:path'
+import type { BlobBackend, OpenLiveResult, StagingWriter } from './blob.ts'
+import {
+  durableRenameStagedToLive,
+  ensureStagingDir,
+  liveFilePath,
+  stagingFilePath,
+  unlinkIfExists,
+} from './fs.ts'
+
+// Internal: tolerant readdir for the reaper / list helpers — a
+// missing dir (workspace never had any state, or whole-workspace
+// delete already swept it) returns []. Anything else (EACCES, EIO,
+// …) bubbles up so the reaper's wrapping catch logs it.
+async function safeReaddir(dir: string): Promise<string[]> {
+  try { return await readdir(dir) } catch (err: unknown) {
+    const code = (err as NodeJS.ErrnoException)?.code
+    if (code === 'ENOENT') return []
+    throw err
+  }
+}
+
+// Open the content-addressed live blob for reading. ENOENT → the row
+// references a hash whose blob is gone right now (racing delete/GC, or a
+// stranded row) → `unavailable` (503). `fh.createReadStream()` (not the
+// raw-fd form) binds the stream lifecycle to the FileHandle so the fd
+// closes exactly once; the inode is pinned, so even if the path is
+// unlinked/overwritten after open the stream reads the captured
+// snapshot. The stat is wrapped so a throw between open and close
+// doesn't leak the fd (PR #4 review H8).
+async function fsOpenLiveReader(dir: string, tag: string, contentHash: string): Promise<OpenLiveResult> {
+  const path = liveFilePath(dir, tag, contentHash)
+  let fh
+  try { fh = await open(path, 'r') } catch (err: unknown) {
+    if ((err as NodeJS.ErrnoException)?.code === 'ENOENT') return { ok: false, reason: 'unavailable' }
+    throw err
+  }
+  let size: number
+  try { size = (await fh.stat()).size } catch {
+    await fh.close().catch(() => {})
+    return { ok: false, reason: 'unavailable' }
+  }
+  const stream = fh.createReadStream()
+  let closed = false
+  return {
+    ok: true,
+    reader: {
+      stream,
+      size,
+      // Guard double-close (caller close + stream-end auto-close).
+      // eslint-disable-next-line require-await
+      close: async () => { if (closed) return; closed = true; stream.destroy() },
+    },
+  }
+}
+
+export function openFsBlobBackend(dir: string): BlobBackend {
+  return {
+    ensureWorkspace: (tag) => ensureStagingDir(dir, tag),
+
+    // eslint-disable-next-line require-await
+    openStagingWriter: async (tag, stagingId): Promise<StagingWriter> => {
+      const path = stagingFilePath(dir, tag, stagingId)
+      // `flags: 'w'` truncates an existing file. Two distinct begins
+      // mint distinct (16-byte random) staging ids, so they never
+      // target the same path — a same-stagingId collision is ~1/2^128.
+      // If a fixture or future refactor reused a stagingId, the second
+      // write would clobber the first.
+      const writable = createWriteStream(path, { flags: 'w' })
+      return {
+        writable,
+        // No-op: callers pipeline(req, counter, writable) which
+        // already awaits 'finish' on the WriteStream. The fd is
+        // closed by Node's stream machinery on 'finish'.
+        finalize: async () => {},
+        // `destroy(err)` synchronously starts tearing the stream
+        // down; the WriteStream emits 'close' on the next tick.
+        // For the FS backend there's no remote upload to wait for,
+        // so we resolve immediately — the REST layer awaits but
+        // doesn't block on anything real here. eslint-disable for
+        // the no-await-in-async — the function signature is
+        // dictated by the BlobBackend contract.
+        // eslint-disable-next-line require-await
+        abort: async (err) => { writable.destroy(err as Error) },
+      }
+    },
+
+    statStaging: async (tag, stagingId): Promise<number | null> => {
+      try {
+        const s = await stat(stagingFilePath(dir, tag, stagingId))
+        return s.size
+      } catch (err: unknown) {
+        const code = (err as NodeJS.ErrnoException)?.code
+        if (code === 'ENOENT') return null
+        throw err
+      }
+    },
+
+    promoteStagingToLive: (tag, stagingId, contentHash) =>
+      durableRenameStagedToLive(
+        stagingFilePath(dir, tag, stagingId),
+        liveFilePath(dir, tag, contentHash),
+      ),
+
+    openLiveReader: (tag, contentHash) => fsOpenLiveReader(dir, tag, contentHash),
+
+    unlinkStaging: (tag, stagingId) => unlinkIfExists(stagingFilePath(dir, tag, stagingId)),
+    unlinkLive: (tag, contentHash) => unlinkIfExists(liveFilePath(dir, tag, contentHash)),
+
+    listWorkspaceTags: () => safeReaddir(dir),
+
+    // Live blobs are top-level `.bin` files under `${dir}/${tag}/`,
+    // named by their content hash. The `.staging` subdirectory is
+    // excluded — it's the staging-id namespace, returned by
+    // listStagingIds. Filtering by `.bin` suffix tolerates operator-
+    // seeded foreign files (PR #4 review H2: reaper refuses to unlink
+    // anything we didn't write). Each entry carries its `mtimeMs` so
+    // the reaper's GC grace window can skip blobs younger than the
+    // grace (a just-promoted blob whose live row hasn't been read
+    // into the GC's reference set yet). A blob that vanishes between
+    // readdir and stat (a racing reaper/abort) is dropped from the
+    // result — it's already gone, nothing to GC.
+    listLiveBlobs: async (tag) => {
+      const entries = await safeReaddir(join(dir, tag))
+      const out: Array<{ hash: string; modifiedMs: number }> = []
+      for (const name of entries) {
+        if (name === '.staging' || !name.endsWith('.bin')) continue
+        let modifiedMs: number
+        try { modifiedMs = (await stat(join(dir, tag, name))).mtimeMs } catch (err: unknown) {
+          if ((err as NodeJS.ErrnoException)?.code === 'ENOENT') continue
+          throw err
+        }
+        out.push({ hash: name.slice(0, -4), modifiedMs })
+      }
+      return out
+    },
+
+    listStagingIds: async (tag) => {
+      const entries = await safeReaddir(join(dir, tag, '.staging'))
+      const out: string[] = []
+      for (const name of entries) {
+        if (!name.endsWith('.bin')) continue
+        out.push(name.slice(0, -4))
+      }
+      return out
+    },
+  }
+}