@preventive/triage 1.0.0-alpha.1 → 1.0.0-alpha.11

package/{server → server-e2e}/objstore/rest.ts

@@ -41,20 +41,20 @@ import {
 } from './store.ts'
 import type { LiveReader } from './blob.ts'
 import { type TokenSecret, extractBearer, verifyToken } from './tokens.ts'
-import { errStack } from '../util.ts'
+import { deny, denyConflict } from './rest-deny.ts'
+import { handleRestMint } from './rest-mint.ts'
+import { debugId, debugTag, errMsg, errStack } from '../util.ts'
 
 // Server-side fault codes that should surface as 500 `io-error`
 // rather than 400 `aborted`. `pipeline(req, ws)` rejects with the
-// first stream error; the client-side codes (socket close, the
-// manual `overrun`) are everything else. ENOENT is included because
-// `createWriteStream` will reject with it if the `${tag}/.staging`
-// dir was removed out from under us (operator action / external
-// fs activity) — that's a server-side state, not a client-fixable
-// abort. PR #4 review. The Vercel-blob backend surfaces failures
-// through other paths (rejected put promise → caught by the
-// pipeline catch as a plain Error without a `code`); those land in
-// the default `aborted` branch and the operator-facing log line
-// carries the SDK's error message.
+// first stream error; client-side codes (socket close, the manual
+// `overrun`) are everything else. ENOENT is included because
+// `createWriteStream` rejects with it if the `${tag}/.staging` dir
+// was removed out from under us (operator / external fs activity) —
+// a server-side state, not a client-fixable abort. Vercel-blob
+// failures arrive without a `code` (rejected put promise → plain
+// Error); those land in the default `aborted` branch, with the SDK
+// error in the operator log line.
 const IO_FAULT_CODES = new Set(['ENOSPC', 'EACCES', 'EROFS', 'EIO', 'EMFILE', 'ENFILE', 'EDQUOT', 'EPERM', 'ENOENT'])
 
 export type ObjstoreRestDeps = {
@@ -68,38 +68,49 @@ export type ObjstoreRestDeps = {
   // workspace_object for the full metadata (version, hash, length,
   // signature). SQLite mode passes a no-op.
   publishObjPut: (tag: string, resourceTag: string) => void
+  // Cross-instance pub/sub for objstore-deleted — fired alongside the local
+  // `broadcast` after a successful REST delete mint, so peers on OTHER
+  // instances drop the resource in real time. Carries (tag, resourceTag,
+  // version) inline (the row is gone post-delete). SQLite mode passes a no-op.
+  publishObjDeleted: (tag: string, resourceTag: string, version: number) => void
+  // New-workspace operator gate for the REST put-begin mint — the
+  // connection-independent analog of the WS path's `authGate`. Returns
+  // `true` to DENY (a password is configured AND the workspace is
+  // never-before-seen), which routes the client to its in-band WS
+  // put-begin fallback (REST has no socket auth state to consult).
+  // No-config default is open (never deny), matching the WS authGate.
+  restPutGate: (workspaceTag: string) => Promise<boolean>
   debug: boolean
 }
 
-// Concurrent commits need no lock at all: the live blob is content-
-// addressed (`${tag}/${contentHash}.bin`) so two racing commits write
-// to DIFFERENT immutable addresses, and the commit itself is an atomic
-// version compare-and-set on the live row (see commitPut in store.ts).
-// Exactly one racer wins the CAS; the loser gets a 409 `conflict` and
-// rebases. This holds within a single process and across replicas —
-// there is no in-process mutex serialising commits. (The PUT *body*
-// does take a per-process single-writer reservation — `inFlightSids`
-// below — but that only rejects a duplicate upload of one staging
-// slot; it never makes distinct commits wait.)
+// Concurrent commits need no lock: the live blob is content-addressed
+// (`${tag}/${contentHash}.bin`) so two racing commits write to
+// DIFFERENT immutable addresses, and the commit is an atomic version
+// compare-and-set on the live row (see commitPut in store.ts). Exactly
+// one racer wins the CAS; the loser gets a 409 `conflict` and rebases.
+// Holds within a process and across replicas — no in-process mutex
+// serialises commits. (The PUT *body* takes a per-process single-
+// writer reservation — `inFlightSids` below — but that only rejects a
+// duplicate upload of one staging slot; distinct commits never wait.)
 
 // Per-process single-writer guard for the REST PUT body. A put-token is
 // a REUSABLE bearer capability (tokens.ts) valid for its whole TTL, so a
-// client replaying it on overlapping PUTs — a retry that doesn't cancel
-// the in-flight request, a proxy re-issuing the PUT, a double-submit —
+// client replaying it on overlapping PUTs (a retry that doesn't cancel
+// the in-flight request, a proxy re-issuing the PUT, a double-submit)
 // would otherwise have two requests stream into the SAME staging file
-// (same sid). On the FS backend `createWriteStream(…, { flags: 'w' })`
-// truncates, so the two writers clobber each other's bytes/size and BOTH
-// can fail (size-mismatch 400 + promote-race 500) with neither
-// committing. `inFlightSids` admits exactly ONE in-flight upload per
-// staging id; a concurrent same-token PUT is rejected (409) before it
-// can open a second writer. The slot is held only across the body +
-// commit and released in a `finally` (and bounded by the REST idle-body
-// timeout in server/http.ts so a slow-loris can't pin it). It is NOT a
-// commit lock — commits stay lock-free on the version-CAS above. It's
-// per-process: the Vercel multi-replica path leans on `allowOverwrite:
-// true` + the commit CAS + content-addressing for cross-replica
-// correctness (a duplicate that lands on another replica can't clobber a
-// shared local file and still loses the CAS), not on this set.
+// (same sid). On FS, `createWriteStream(…, { flags: 'w' })` truncates,
+// so the two writers clobber each other's bytes/size and BOTH can fail
+// (size-mismatch 400 + promote-race 500) with neither committing.
+// `inFlightSids` admits exactly ONE in-flight upload per staging id; a
+// concurrent same-token PUT is rejected (409) before it can open a
+// second writer. The slot is held only across body + commit, released
+// in a `finally`, and bounded by the REST idle-body timeout in
+// server-e2e/http.ts so a slow-loris can't pin it. NOT a commit lock —
+// commits stay lock-free on the version-CAS above. Per-process: the
+// Vercel multi-replica path leans on `allowOverwrite: true` + the
+// commit CAS + content-addressing for cross-replica correctness (a
+// duplicate landing on another replica can't clobber a shared local
+// file and still loses the CAS), not on this set.
 const inFlightSids = new Set<string>()
 
 // `/api/objstore/${workspaceTag}/${resourceTag}` — base64url
@@ -119,35 +130,21 @@ export function matchRoute(url: string | undefined): RouteMatch | null {
   return { tag: tag!, resourceTag: resourceTag! }
 }
 
-function deny(res: ServerResponse, status: number, body: string): void {
-  // Uniform `{ error: <reason> }` JSON envelope for every failure so
-  // clients have one shape to parse. Status + reason are NOT
-  // intentionally indistinguishable across causes — 401, 404, 405,
-  // 410, 411, 500 each map to a documented reason in server/README.md
-  // and the client decides recovery from the code. Defense against
-  // probe-driven distinguishing isn't a property the relay aims for;
-  // every error reason is reachable only after the route + bearer-
-  // token check passes (or as 401/404 from the public surface), so
-  // there's no signal here a probe couldn't otherwise enumerate.
-  res.writeHead(status, { 'content-type': 'application/json' })
-  res.end(JSON.stringify({ error: body }))
-}
-
-// Variant of `deny` that augments the JSON envelope with the live
-// row's `currentVersion` + `currentIncarnation` so a REST PUT 409 lets
-// the caller rebase onto the right precondition token. Without this the
-// client only learns the slot is occupied — not at what (version,
-// incarnation) — and retries blindly against a non-empty slot, looping
-// indefinitely against a live row. Symmetric with the WS plane's
-// `objstore-conflict` envelope.
-function denyConflict(res: ServerResponse, currentVersion: number | null, currentIncarnation: string | null): void {
-  res.writeHead(409, { 'content-type': 'application/json' })
-  res.end(JSON.stringify({ error: 'conflict', currentVersion, currentIncarnation }))
-}
-
 export async function handleRest(deps: ObjstoreRestDeps, req: IncomingMessage, res: ServerResponse): Promise<void> {
   const route = matchRoute(req.url)
   if (!route) { deny(res, 404, 'not-found'); return }
+  // POST = REST mint (fetch or put-begin, by body `op`; signature-authed
+  // via the JSON body, no bearer token). Dispatched before the bearer-token
+  // gate below, which guards the token-authed GET/PUT byte transfers.
+  if (req.method === 'POST') {
+    try { await handleRestMint(deps, req, res, route) }
+    catch (err: unknown) {
+      if (deps.debug) console.warn('objstore POST error:', errStack(err))
+      if (res.headersSent) res.destroy()
+      else deny(res, 500, 'internal')
+    }
+    return
+  }
   const token = extractBearer(req.headers['authorization'])
   if (!token) { deny(res, 401, 'unauthorized'); return }
   const payload = verifyToken(deps.secret, token)
@@ -209,21 +206,20 @@ async function handleRestPut(
   // No lock: the commit's version-CAS arbitrates concurrent commits
   // (the live blob is content-addressed, so racers can't desync
   // metadata vs bytes — the loser surfaces a 409 `conflict`). The
-  // staging row is protected from the reaper not by a lock but by its
-  // `begun_at`: an upload under the staging TTL stays fresh through the
-  // body, and the after-body `refreshStagingBegunAt` re-extends the
-  // TTL across the commit. (An upload exceeding the TTL during the
-  // body can be reaped mid-flight → commit 410s; documented accepted
-  // tradeoff — see commitPut in store.ts.)
+  // staging row is protected from the reaper by its `begun_at`, not a
+  // lock: an upload under the staging TTL stays fresh through the body,
+  // and the after-body `refreshStagingBegunAt` re-extends the TTL across
+  // the commit. (An upload exceeding the TTL during the body can be
+  // reaped mid-flight → commit 410s; documented accepted tradeoff —
+  // see commitPut in store.ts.)
   await handleRestPutBody(deps, req, res, route, payload, declared)
 }
 
 // Map a `commitPut` failure to its wire response. Extracted from
-// `handleRestPutBody` to keep it under the per-function line cap
-// and to make the wire-mapping ladder its own audit surface — the
-// exhaustiveness `never` guard at the bottom catches a forward-
-// compat hazard where a new `CommitPutResult` reason lands without
-// updating this dispatch.
+// `handleRestPutBody` to keep it under the per-function line cap and
+// to make the wire-mapping ladder its own audit surface — the
+// exhaustiveness `never` guard at the bottom catches a new
+// `CommitPutResult` reason landing without updating this dispatch.
 function denyCommitFailure(res: ServerResponse, result: Exclude<CommitPutResult, { ok: true }>): void {
   if (result.reason === 'conflict') {
     denyConflict(res, result.conflict?.version ?? null, result.conflict?.incarnation ?? null)
@@ -273,10 +269,9 @@ async function handleRestPutBody(
   }
   inFlightSids.add(payload.sid)
   try {
-    // Stream the upload + commit, no lock. The commit's version-CAS
-    // arbitrates concurrent commits; the staging row is kept fresh for
-    // the reaper by its `begun_at` (+ the after-body refresh), not by a
-    // mutex. See the rationale in handleRestPut above.
+    // Stream the upload + commit, no lock — version-CAS arbitrates
+    // commits, the staging row stays fresh for the reaper via its
+    // `begun_at` (+ after-body refresh). See handleRestPut above.
     const result = await runUploadAndCommit(deps, route, payload, declared, req, res)
     if (result.handled) return
     if (!result.commit.ok) { denyCommitFailure(res, result.commit); return }
@@ -294,15 +289,13 @@ async function handleRestPutBody(
       ...objectMetaWire(row),
     }, null)
     // Cross-instance fan-out (Neon mode). The bus payload carries only
-    // (tag, resourceTag); peers on other instances re-fetch the live
-    // row from workspace_object to compose their local broadcast. The
-    // committed row is durable by here (commitPut's version-CAS already
-    // landed), so the receiver typically sees either THIS version or a
-    // STRICTLY newer one (also a valid broadcast — clients are
-    // idempotent on (resourceTag, version)). The receiver is allowed
-    // to find no live row at all if a subsequent delete races the
-    // notification; bus-receiver.ts drops that case silently. SQLite
-    // mode publishes to a no-op.
+    // (tag, resourceTag); peers on other instances re-fetch the live row
+    // from workspace_object to compose their broadcast. The committed
+    // row is durable by here (commitPut's version-CAS landed), so the
+    // receiver sees THIS version or a STRICTLY newer one (also valid —
+    // clients are idempotent on (resourceTag, version)), or no live row
+    // at all if a subsequent delete races the notification (bus-
+    // receiver.ts drops that silently). SQLite mode publishes to a no-op.
     deps.publishObjPut(route.tag, route.resourceTag)
     if (deps.debug) console.log(`objstore put → ${route.tag.slice(0, 12)}…/${route.resourceTag.slice(0, 8)}… v${row.version}`)
   } finally {
@@ -439,7 +432,10 @@ async function runUploadAndCommit(
 type GetOpened =
   | { reason: 'ok'; reader: LiveReader }
   | { reason: 'not-found' }
-  | { reason: 'unavailable' }
+  // `detail` is a short non-sensitive cause tag (backend sub-reason +
+  // content-hash prefix) the GET handler logs so a 503 is diagnosable —
+  // every byte-side failure collapses to the same wire 503 otherwise.
+  | { reason: 'unavailable'; detail: string }
 
 async function openLiveSnapshot(
   deps: ObjstoreRestDeps, route: RouteMatch, payload: { ver: number; inc: string },
@@ -451,22 +447,32 @@ async function openLiveSnapshot(
   // bytes behind `live.content_hash` can never change underneath us —
   // the worst a race can do is have the reaper GC an already-superseded
   // hash just before we open it, which surfaces as openLiveReader
-  // not-found → `unavailable` → 503, and the client refetches. We can
+  // returning `unavailable` → 503, and the client refetches. We can
   // never serve torn or wrong bytes. (For the FS backend the open also
   // returns a pinned fd; for the Vercel backend a fetch-backed stream.)
   const live = await deps.handle.selectLiveOne.get(route.tag, route.resourceTag)
   if (!live || live.version !== payload.ver || live.incarnation !== payload.inc) return { reason: 'not-found' }
+  // Tag the content hash into every `unavailable` detail so an operator
+  // can go check the byte store directly for THIS blob (gone → reaper /
+  // deletion; present → transient read fault).
+  const hashTag = `hash=${debugId(live.content_hash)}`
   let opened
   try { opened = await deps.handle.blob.openLiveReader(route.tag, live.content_hash) }
-  catch { return { reason: 'unavailable' } }
-  if (!opened.ok) return { reason: opened.reason }
+  // Length-cap the thrown error text: today a non-BlobNotFound SDK throw
+  // (BlobServiceNotAvailable / store-not-found) carries no credential
+  // (the RW token rides the Authorization header, never `.message`), but
+  // a future SDK could embed a signed URL / token fragment — bound the
+  // log line so it can't dump one verbatim. Mirrors the `.slice(0, 200)`
+  // cap used on SDK error text in blob-vercel.ts.
+  catch (err) { return { reason: 'unavailable', detail: `open-threw ${hashTag} ${String(errMsg(err)).slice(0, 200)}` } }
+  if (!opened.ok) return { reason: 'unavailable', detail: `${opened.detail ?? 'backend'} ${hashTag}` }
   // Size mismatch between the live row and the on-storage bytes
   // is a transient inconsistency — reaper will reconcile. Close
   // the reader before returning so we don't leak the fd / fetch
   // reader. PR #4 review H8.
   if (opened.reader.size !== live.content_length) {
     await opened.reader.close().catch(() => {})
-    return { reason: 'unavailable' }
+    return { reason: 'unavailable', detail: `size-mismatch row=${live.content_length} blob=${opened.reader.size} ${hashTag}` }
   }
   return { reason: 'ok', reader: opened.reader }
 }
@@ -486,8 +492,19 @@ async function handleRestGet(
   // If the live row is there but the bytes are missing / wrong size,
   // it's a transient inconsistency the reaper will sort out — 503
   // (vs 404) tells the client this is a server-side state, not a
-  // "the resource truly isn't there" answer.
-  if (opened.reason === 'unavailable') { deny(res, 503, 'unavailable'); return }
+  // "the resource truly isn't there" answer. Log the cause
+  // UNCONDITIONALLY (not behind `debug`): a 503 means a live row whose
+  // bytes can't be served, and the wire response can't distinguish a
+  // permanent loss (reaper GC'd referenced bytes) from a transient read
+  // fault. `detail` carries the backend sub-reason + content-hash prefix
+  // so an operator can tell which — the only server-side breadcrumb for
+  // the "all data turned into 503" failure. (Volume is bounded: a 503 is
+  // an error path; a workspace-wide outage is exactly when these are
+  // wanted.)
+  if (opened.reason === 'unavailable') {
+    console.warn(`objstore-get: 503 unavailable ${debugTag(route.tag)}/${route.resourceTag.slice(0, 8)}… v${payload.ver} ${opened.detail}`)
+    deny(res, 503, 'unavailable'); return
+  }
   res.writeHead(200, {
     'content-type': 'application/octet-stream',
     'content-length': String(opened.reader.size),

package/{server → server-e2e}/objstore/sign.ts

@@ -10,6 +10,18 @@ import { isValidIncarnation } from './store.ts'
 const OBJSTORE_PUT_DOMAIN = 'deepview-objstore.v1.put'
 const OBJSTORE_DELETE_DOMAIN = 'deepview-objstore.v1.delete'
 const OBJSTORE_FETCH_DOMAIN = 'deepview-objstore.v1.fetch'
+// REST fetch-mint domain — MUST match the client's FETCH_REST_DOMAIN
+// (client/sync/objstore-crypto.ts). Distinct from OBJSTORE_FETCH_DOMAIN so
+// a WS-fetch signature can't be replayed against the REST mint endpoint.
+const OBJSTORE_FETCH_REST_DOMAIN = 'deepview-objstore.v1.fetch-rest'
+// REST put-begin domain — MUST match the client's PUT_REST_DOMAIN. Distinct
+// from OBJSTORE_PUT_DOMAIN so a WS put-begin signature can't be replayed
+// against the REST mint endpoint.
+const OBJSTORE_PUT_REST_DOMAIN = 'deepview-objstore.v1.put-rest'
+// REST delete domain — MUST match the client's DELETE_REST_DOMAIN. Distinct
+// from OBJSTORE_DELETE_DOMAIN so a WS delete signature can't be replayed
+// against the REST mint endpoint.
+const OBJSTORE_DELETE_REST_DOMAIN = 'deepview-objstore.v1.delete-rest'
 
 // Wire shapes the verifiers accept. Fields land here post-
 // `JSON.parse`, so every value starts life as `unknown` — strict
@@ -95,6 +107,38 @@ function canonicalObjstoreFetch(msg: ObjstoreFetchMsg, connectionNonce: string):
   ].join('\n'))
 }
 
+// REST fetch-mint canonical. Binds a client epoch-ms timestamp (string-
+// encoded to match the client) in place of the connection nonce; the REST
+// handler enforces the freshness window + replay dedup.
+function canonicalObjstoreFetchRest(workspaceTag: string, resourceTag: string, ts: number): Uint8Array<ArrayBuffer> {
+  return encodeUtf8([
+    OBJSTORE_FETCH_REST_DOMAIN,
+    workspaceTag,
+    resourceTag,
+    String(ts),
+  ].join('\n'))
+}
+
+// REST put-begin canonical — the WS `canonicalObjstorePut` fields in the
+// same order/coercion (intOrEmpty / strOrEmpty), under the put-rest domain
+// and binding the client `ts` in place of the connection nonce.
+function canonicalObjstorePutRest(
+  workspaceTag: string, resourceTag: string,
+  prevVersion: number | null, prevIncarnation: string | null,
+  contentHash: string, expectedLength: number, ts: number,
+): Uint8Array<ArrayBuffer> {
+  return encodeUtf8([
+    OBJSTORE_PUT_REST_DOMAIN,
+    workspaceTag,
+    resourceTag,
+    intOrEmpty(prevVersion),
+    strOrEmpty(prevIncarnation),
+    contentHash,
+    String(expectedLength),
+    String(ts),
+  ].join('\n'))
+}
+
 // `Number.isSafeInteger` rather than `Number.isInteger`: JSON numbers
 // are IEEE-754 and integers above 2^53-1 aren't precisely
 // representable. Accepting non-safe integers would let a signed
@@ -162,3 +206,64 @@ export function verifyObjstoreFetchSig(msg: ObjstoreFetchMsg, connectionNonce: u
   if (typeof msg.resourceTag !== 'string') return Promise.resolve(false)
   return verifyObjstoreSig(msg, connectionNonce, (nonce) => canonicalObjstoreFetch(msg, nonce))
 }
+
+// Verify a REST fetch-mint signature. Unlike the WS verifiers this takes
+// the already-validated fields directly (the REST handler parsed +
+// range-checked `ts` and `signature`) rather than a wire `msg` + socket
+// nonce. The workspaceTag IS the Ed25519 public key, so verification is
+// fully self-contained — no session or stored key needed. A throw from
+// the canonical build (lone surrogate, etc.) is treated as a verify
+// failure, never an escaping exception.
+export function verifyObjstoreFetchRestSig(
+  workspaceTag: string, resourceTag: string, ts: number, signature: string,
+): Promise<boolean> {
+  let payload: Uint8Array<ArrayBuffer>
+  try { payload = canonicalObjstoreFetchRest(workspaceTag, resourceTag, ts) }
+  catch { return Promise.resolve(false) }
+  return verifyEd25519(workspaceTag, payload, signature)
+}
+
+// Verify a REST put-begin signature. Same self-contained shape as
+// `verifyObjstoreFetchRestSig` — the caller (rest.ts) has already
+// range-checked the put fields + `ts`. The workspaceTag IS the pubkey.
+export function verifyObjstorePutBeginRestSig(
+  fields: { workspaceTag: string; resourceTag: string; prevVersion: number | null; prevIncarnation: string | null; contentHash: string; expectedLength: number },
+  ts: number, signature: string,
+): Promise<boolean> {
+  let payload: Uint8Array<ArrayBuffer>
+  try {
+    payload = canonicalObjstorePutRest(
+      fields.workspaceTag, fields.resourceTag, fields.prevVersion, fields.prevIncarnation,
+      fields.contentHash, fields.expectedLength, ts,
+    )
+  } catch { return Promise.resolve(false) }
+  return verifyEd25519(fields.workspaceTag, payload, signature)
+}
+
+// REST delete-mint canonical. WS `canonicalObjstoreDelete` fields, under the
+// delete-rest domain, binding the client `ts` in place of the nonce.
+function canonicalObjstoreDeleteRest(
+  workspaceTag: string, resourceTag: string,
+  prevVersion: number | null, prevIncarnation: string | null, ts: number,
+): Uint8Array<ArrayBuffer> {
+  return encodeUtf8([
+    OBJSTORE_DELETE_REST_DOMAIN,
+    workspaceTag,
+    resourceTag,
+    intOrEmpty(prevVersion),
+    strOrEmpty(prevIncarnation),
+    String(ts),
+  ].join('\n'))
+}
+
+// Verify a REST delete-mint signature. Self-contained (workspaceTag IS the
+// pubkey); the caller (rest.ts) has range-checked `ts` + the prev pair.
+export function verifyObjstoreDeleteRestSig(
+  fields: { workspaceTag: string; resourceTag: string; prevVersion: number | null; prevIncarnation: string | null },
+  ts: number, signature: string,
+): Promise<boolean> {
+  let payload: Uint8Array<ArrayBuffer>
+  try { payload = canonicalObjstoreDeleteRest(fields.workspaceTag, fields.resourceTag, fields.prevVersion, fields.prevIncarnation, ts) }
+  catch { return Promise.resolve(false) }
+  return verifyEd25519(fields.workspaceTag, payload, signature)
+}

package/{server → server-e2e}/objstore/store-neon.ts

@@ -4,7 +4,7 @@
 // dialect on the wire.
 //
 // `@neondatabase/serverless` is an OPTIONAL peer dep — selected by
-// the `DATABASE_URL` branch in `server/index.ts`. The peer dep
+// the `DATABASE_URL` branch in `server-e2e/index.ts`. The peer dep
 // itself is loaded lazily inside `openNeonObjstore` below. A single
 // Neon project / database holds both `workspace_revision` and the
 // two objstore tables; each plane opens its own stateless
@@ -13,7 +13,7 @@
 // release.
 //
 // Byte plane: passed in as a `BlobBackend` (./blob.ts). The
-// supported pairings selected by server/index.ts are:
+// supported pairings selected by server-e2e/index.ts are:
 //   - Neon + Vercel Blob Private Storage (multi-replica, the only
 //     pairing that survives a replica restart without local-disk
 //     coordination). Activated by setting BLOB_READ_WRITE_TOKEN
@@ -46,10 +46,10 @@ const DDL_LOCK_KEY_OBJSTORE_SUB = 0x6f62_6a73 // 'objs'
 // defend the commitPut conflict arithmetic — a manual `UPDATE
 // workspace_object SET version = -1` would otherwise round-trip
 // through `num()` (which only rejects non-safe-integers) and corrupt
-// the version monotonicity invariant. The SQLite schema carries the
-// identical CHECKs (see `store.ts`) — STRICT there enforces the column
-// TYPE but NOT this value domain (`version = -1` is a valid integer
-// STRICT accepts), so both backends need the explicit CHECKs.
+// version monotonicity. SQLite carries the identical CHECKs (see
+// `store.ts`): STRICT there enforces column TYPE but NOT this value
+// domain (`version = -1` is a valid integer STRICT accepts), so both
+// backends need the explicit CHECKs.
 const SCHEMA_PG = [
   `CREATE TABLE IF NOT EXISTS workspace_object (
      workspace_tag  TEXT    NOT NULL,
@@ -121,7 +121,7 @@ type StagingRow = {
 
 function buildSelectStaging(sql: NeonSql): GetStmt<[string, string, string], StagingRow> {
   return { get: async (tag, resourceTag, stagingId) => {
-    const rows = await sql(
+    const rows = await sql.query(
       `SELECT prev_version, prev_incarnation, expected_length, content_hash, signature, begun_at
        FROM workspace_object_staging
        WHERE workspace_tag = $1 AND resource_tag = $2 AND staging_id = $3`,
@@ -162,7 +162,7 @@ function buildDeleteStaging(sql: NeonSql): RunStmt<[string, string, string]> {
 // Bind order: (tag, res, sid, staleBefore).
 function buildDeleteStagingIfStale(sql: NeonSql): GetStmt<[string, string, string, number], { ok: number }> {
   return { get: async (tag, resourceTag, stagingId, staleBefore) => {
-    const rows = await sql(
+    const rows = await sql.query(
       `DELETE FROM workspace_object_staging
        WHERE workspace_tag = $1 AND resource_tag = $2 AND staging_id = $3 AND begun_at < $4
        RETURNING 1 AS ok`,
@@ -175,7 +175,7 @@ function buildDeleteStagingIfStale(sql: NeonSql): GetStmt<[string, string, strin
 
 function buildSelectLive(sql: NeonSql): AllStmt<[string], LiveDbRow> {
   return { all: async (tag) => {
-    const rows = await sql(
+    const rows = await sql.query(
       `SELECT resource_tag, version, incarnation, content_hash, content_length,
               signature, put_at
        FROM workspace_object
@@ -189,7 +189,7 @@ function buildSelectLive(sql: NeonSql): AllStmt<[string], LiveDbRow> {
 
 function buildSelectLiveOne(sql: NeonSql): GetStmt<[string, string], LiveDbRow> {
   return { get: async (tag, resourceTag) => {
-    const rows = await sql(
+    const rows = await sql.query(
       `SELECT resource_tag, version, incarnation, content_hash, content_length,
               signature, put_at
        FROM workspace_object
@@ -208,7 +208,7 @@ function buildSelectLiveOne(sql: NeonSql): GetStmt<[string, string], LiveDbRow>
 // (tag, res, hash, len, sig, put_at); version is the literal 1.
 function buildInsertLiveIfAbsent(sql: NeonSql): GetStmt<[string, string, string, string, number, string, number], { ok: number }> {
   return { get: async (tag, resourceTag, incarnation, contentHash, contentLength, signature, putAt) => {
-    const rows = await sql(
+    const rows = await sql.query(
       `INSERT INTO workspace_object
          (workspace_tag, resource_tag, version, incarnation, content_hash, content_length,
           signature, put_at)
@@ -230,7 +230,7 @@ function buildInsertLiveIfAbsent(sql: NeonSql): GetStmt<[string, string, string,
 // (tag, res, nextVersion, hash, len, sig, put_at, expectedVersion).
 function buildUpdateLiveCAS(sql: NeonSql): GetStmt<[string, string, number, string, number, string, number, number, string], { ok: number }> {
   return { get: async (tag, resourceTag, nextVersion, contentHash, contentLength, signature, putAt, expectedVersion, expectedIncarnation) => {
-    const rows = await sql(
+    const rows = await sql.query(
       `UPDATE workspace_object
        SET version        = $3,
            content_hash   = $4,
@@ -252,7 +252,7 @@ function buildUpdateLiveCAS(sql: NeonSql): GetStmt<[string, string, number, stri
 // not-found, never a lost update.
 function buildDeleteLiveCAS(sql: NeonSql): GetStmt<[string, string, number, string], { ok: number }> {
   return { get: async (tag, resourceTag, expectedVersion, expectedIncarnation) => {
-    const rows = await sql(
+    const rows = await sql.query(
       `DELETE FROM workspace_object
        WHERE workspace_tag = $1 AND resource_tag = $2 AND version = $3 AND incarnation = $4
        RETURNING 1 AS ok`,
@@ -268,7 +268,7 @@ function buildListAllStaging(sql: NeonSql): AllStmt<[number], { workspace_tag: s
     // `WHERE begun_at < $1` uses workspace_object_staging_begun_at_idx
     // so the reaper sweep is O(stale-rows) cluster-wide. DB-layout
     // audit follow-up.
-    const rows = await sql(
+    const rows = await sql.query(
       `SELECT workspace_tag, resource_tag, staging_id, begun_at
        FROM workspace_object_staging
        WHERE begun_at < $1`,
@@ -285,7 +285,7 @@ function buildListAllStaging(sql: NeonSql): AllStmt<[number], { workspace_tag: s
 
 function buildListLiveTags(sql: NeonSql): AllStmt<[], { workspace_tag: string }> {
   return { all: async () => {
-    const rows = await sql(
+    const rows = await sql.query(
       `SELECT DISTINCT workspace_tag FROM workspace_object`,
       [],
     ) as Array<Record<string, unknown>>
@@ -295,7 +295,7 @@ function buildListLiveTags(sql: NeonSql): AllStmt<[], { workspace_tag: string }>
 
 function buildCountLive(sql: NeonSql): GetStmt<[string], { c: number }> {
   return { get: async (tag) => {
-    const rows = await sql(
+    const rows = await sql.query(
       `SELECT COUNT(*) AS c FROM workspace_object WHERE workspace_tag = $1`,
       [tag],
     ) as Array<{ c: number | string | bigint }>
@@ -309,7 +309,7 @@ export async function openNeonObjstore(connectionString: string, blob: BlobBacke
   // local `../neon-driver.ts` re-export wrapper so the peer dep stays
   // optional AND tests can mock the driver (a local path is always
   // resolvable; the bare specifier isn't when the dep is absent). See
-  // `server/neon-driver.ts`. Cast through `unknown` because the
+  // `server-e2e/neon-driver.ts`. Cast through `unknown` because the
   // wrapper's `export *` re-exports a `@ts-ignore`'d module.
   const mod = (await import('../neon-driver.ts')) as unknown as { neon: (url: string) => NeonSql }
   const sql: NeonSql = mod.neon(connectionString)
@@ -324,8 +324,8 @@ export async function openNeonObjstore(connectionString: string, blob: BlobBacke
   // advisory lock so two replicas booting concurrently serialize
   // their DDL (the advisory lock releases at COMMIT).
   await sql.transaction([
-    sql(`SELECT pg_advisory_xact_lock($1, $2)`, [DDL_LOCK_KEY_OBJSTORE, DDL_LOCK_KEY_OBJSTORE_SUB]),
-    ...SCHEMA_PG.map((stmt) => sql(stmt, [])),
+    sql.query(`SELECT pg_advisory_xact_lock($1, $2)`, [DDL_LOCK_KEY_OBJSTORE, DDL_LOCK_KEY_OBJSTORE_SUB]),
+    ...SCHEMA_PG.map((stmt) => sql.query(stmt, [])),
   ])
 
   return {