@opengeni/api-router 3.0.1 → 3.1.2

package/src/http/rejection-telemetry.ts

@@ -0,0 +1,120 @@
+import { createHash } from "node:crypto";
+import { ErrorCode, Permission } from "@opengeni/contracts";
+
+/**
+ * Bounded, content-free classification of a rejected (4xx/5xx) API response,
+ * so a refused request can be diagnosed from logs and counted per route without
+ * copying the response message, identifiers, or user content into telemetry.
+ *
+ * - `code` is the public `ErrorEnvelope.code` (a closed enum).
+ * - `reason` is `permission:<name>` when the refusal names a missing permission
+ *   from the closed permission catalog, else the envelope's typed
+ *   `details.code` when it is a short identifier, else `control:<code>` for a
+ *   Connected Machine control failure, else `unclassified`.
+ * - `fingerprint` is a one-way hash of the message with identifiers and
+ *   numbers normalized away. It groups identical refusals in logs; resolve it
+ *   back to source text with `bun run scripts/resolve-rejection-fingerprint.ts`.
+ *   It never becomes a metric label.
+ */
+export type HttpRejectionFacts = {
+  code: string;
+  reason: string;
+  fingerprint?: string;
+};
+
+const ERROR_CODES: ReadonlySet<string> = new Set(ErrorCode.options);
+const PERMISSIONS: ReadonlySet<string> = new Set(Permission.options);
+const DETAIL_CODE = /^[A-Za-z][A-Za-z0-9_.-]{0,63}$/;
+const MISSING_PERMISSION = /^missing (?:literal )?permission: ([a-z][a-z0-9-]*:[a-z][a-z0-9-]*)/;
+const CONTROL_CODE = /^[a-z][a-z0-9_]{0,31}$/;
+const MAX_ENVELOPE_BYTES = 32 * 1024;
+
+export const UNCLASSIFIED_REJECTION_REASON = "unclassified";
+
+export function rejectionFingerprint(message: string): string {
+  return `m_${createHash("sha256").update(normalizeRejectionMessage(message)).digest("hex").slice(0, 10)}`;
+}
+
+/** Identifiers, quoted values, and numbers never distinguish two fingerprints. */
+export function normalizeRejectionMessage(message: string): string {
+  return message
+    .replace(/[\u0000-\u001f\u007f]+/g, " ")
+    .replace(/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/gi, "<id>")
+    .replace(/"[^"]*"|'[^']*'|`[^`]*`/g, "<q>")
+    .replace(/[0-9]+/g, "0")
+    .replace(/\s+/g, " ")
+    .trim();
+}
+
+export function rejectionReasonFromMessage(message: string | undefined): string | null {
+  if (!message) return null;
+  const permission = MISSING_PERMISSION.exec(message)?.[1];
+  return permission && PERMISSIONS.has(permission) ? `permission:${permission}` : null;
+}
+
+/** Classify a parsed `ErrorEnvelope`-shaped body. Unknown shapes still get a code. */
+export function httpRejectionFactsFromEnvelope(
+  envelope: unknown,
+  fallbackCode: string,
+): HttpRejectionFacts {
+  const error =
+    envelope && typeof envelope === "object" && "error" in envelope
+      ? (envelope as { error?: unknown }).error
+      : undefined;
+  const record = error && typeof error === "object" ? (error as Record<string, unknown>) : {};
+  const code =
+    typeof record.code === "string" && ERROR_CODES.has(record.code) ? record.code : fallbackCode;
+  const message = typeof record.message === "string" ? record.message : undefined;
+  const details =
+    record.details && typeof record.details === "object"
+      ? (record.details as Record<string, unknown>)
+      : undefined;
+  const detailCode =
+    typeof details?.code === "string" && DETAIL_CODE.test(details.code) ? details.code : null;
+  const controlCode =
+    typeof details?.controlFailureCode === "string" && CONTROL_CODE.test(details.controlFailureCode)
+      ? `control:${details.controlFailureCode}`
+      : null;
+  return {
+    code,
+    reason:
+      rejectionReasonFromMessage(message) ??
+      detailCode ??
+      controlCode ??
+      UNCLASSIFIED_REJECTION_REASON,
+    ...(message ? { fingerprint: rejectionFingerprint(message) } : {}),
+  };
+}
+
+/** Classify a thrown error that escaped the central error handler. */
+export function httpRejectionFactsFromError(
+  error: unknown,
+  fallbackCode: string,
+): HttpRejectionFacts {
+  const message = error instanceof Error ? error.message : undefined;
+  return {
+    code: fallbackCode,
+    reason: rejectionReasonFromMessage(message) ?? UNCLASSIFIED_REJECTION_REASON,
+    ...(message ? { fingerprint: rejectionFingerprint(message) } : {}),
+  };
+}
+
+/**
+ * Read a small JSON error body without consuming the response that is sent to
+ * the client. Streams, non-JSON bodies, and large bodies are never read.
+ */
+export async function readRejectionEnvelope(response: Response): Promise<unknown> {
+  if (!response.body) return undefined;
+  if (!/^application\/json(?:;|$)/i.test(response.headers.get("content-type") ?? "")) {
+    return undefined;
+  }
+  const length = Number(response.headers.get("content-length") ?? "0");
+  if (Number.isFinite(length) && length > MAX_ENVELOPE_BYTES) return undefined;
+  try {
+    const text = await response.clone().text();
+    if (text.length > MAX_ENVELOPE_BYTES) return undefined;
+    return JSON.parse(text) as unknown;
+  } catch {
+    return undefined;
+  }
+}

package/src/http/request-body.ts

@@ -0,0 +1,127 @@
+import type { Context } from "hono";
+import type { z } from "zod";
+import { ZodError } from "zod";
+
+import { ApiHttpError } from "./api-error";
+
+/**
+ * Validation failures produced by parsing a client-supplied request body.
+ *
+ * The same `ZodError` instance is rethrown so route-local handlers that
+ * already map `instanceof ZodError` keep their exact behavior; only the
+ * central `app.onError` consults this set to answer a client error instead of
+ * a 500. A ZodError that is NOT registered here (for example, a server-side
+ * response projection that no longer matches its contract) remains a genuine
+ * server fault.
+ */
+const requestBodyValidationErrors = new WeakSet<ZodError>();
+/** Malformed-JSON failures raised by reading a client request body. */
+const requestBodyJsonErrors = new WeakSet<SyntaxError>();
+
+const MAX_REPORTED_ISSUES = 10;
+
+/** Parse an already-read request body against its public request schema. */
+export function parseRequestBody<Schema extends z.ZodType>(
+  schema: Schema,
+  body: unknown,
+): z.output<Schema> {
+  const parsed = schema.safeParse(body);
+  if (parsed.success) return parsed.data;
+  requestBodyValidationErrors.add(parsed.error);
+  throw parsed.error;
+}
+
+/**
+ * Read the request body as JSON. Malformed JSON is a client error (400), never
+ * an unhandled `SyntaxError` 500. Other failures (body limit, aborted stream)
+ * keep their own mapping.
+ */
+export async function readRequestJson(c: Context): Promise<unknown> {
+  try {
+    return await c.req.json();
+  } catch (error) {
+    if (error instanceof SyntaxError) {
+      throw new ApiHttpError(400, {
+        code: "validation_failed",
+        message: "Request body must be valid JSON.",
+        retryable: false,
+        details: { code: "invalid_json" },
+      });
+    }
+    throw error;
+  }
+}
+
+/** Read and validate a JSON request body in one step. */
+export async function parseRequestJson<Schema extends z.ZodType>(
+  c: Context,
+  schema: Schema,
+): Promise<z.output<Schema>> {
+  return parseRequestBody(schema, await readRequestJson(c));
+}
+
+/**
+ * Tag malformed-JSON failures of `c.req.json()` for this request. The same
+ * `SyntaxError` instance is rethrown so routes that already catch it keep
+ * their own mapping; an uncaught one becomes a 400 in `app.onError` instead of
+ * a 500. A `SyntaxError` from parsing stored or upstream data is never tagged.
+ */
+export function tagRequestJsonParseErrors(c: Context): void {
+  const original = c.req.json.bind(c.req);
+  const tagged = async () => {
+    try {
+      return await original();
+    } catch (error) {
+      if (error instanceof SyntaxError) requestBodyJsonErrors.add(error);
+      throw error;
+    }
+  };
+  (c.req as { json: typeof tagged }).json = tagged;
+}
+
+export function isRequestBodyValidationError(error: unknown): error is ZodError | SyntaxError {
+  return requestBodyInputError(error) !== null;
+}
+
+/** Walk a bounded cause chain for a tagged client-input failure. */
+function requestBodyInputError(error: unknown): ZodError | SyntaxError | null {
+  let current: unknown = error;
+  for (let depth = 0; depth < 5 && current instanceof Error; depth += 1) {
+    if (current instanceof ZodError && requestBodyValidationErrors.has(current)) return current;
+    if (current instanceof SyntaxError && requestBodyJsonErrors.has(current)) return current;
+    current = current.cause;
+  }
+  return null;
+}
+
+/**
+ * Public 400 for a registered request-body validation failure, naming every
+ * offending field (bounded) so API callers can correct the request.
+ */
+export function requestBodyValidationHttpError(error: unknown): ApiHttpError | null {
+  const inputError = requestBodyInputError(error);
+  if (!inputError) return null;
+  if (inputError instanceof SyntaxError) {
+    return new ApiHttpError(400, {
+      code: "validation_failed",
+      message: "Request body must be valid JSON.",
+      retryable: false,
+      details: { code: "invalid_json" },
+    });
+  }
+  const zodError = inputError;
+  const issues = zodError.issues.slice(0, MAX_REPORTED_ISSUES).map((issue) => ({
+    path: issue.path.map(String).join("."),
+    message: issue.message,
+  }));
+  const summary = issues.map((issue) => `${issue.path || "request"}: ${issue.message}`).join("; ");
+  const omittedIssueCount = Math.max(0, zodError.issues.length - issues.length);
+  return new ApiHttpError(400, {
+    code: "validation_failed",
+    message: `Invalid request body: ${summary}${
+      omittedIssueCount > 0 ? `; and ${omittedIssueCount} more` : ""
+    }`,
+    retryable: false,
+    details: { code: "invalid_request_body", issues, omittedIssueCount },
+  });
+}

package/src/http/request-source.ts

@@ -1,11 +1,28 @@
+import { BlockList, isIP } from "node:net";
+import { trustedProxyCidrEntries, type Settings } from "@opengeni/config";
 import type { Context } from "hono";
 
 const TRANSPORT_PEER_ADDRESS_BINDING = "opengeniTransportPeerAddress";
 
+/** Returned when the runtime supplied no transport peer (for example an
+ * in-process test request). Callers treat it as one shared bucket. */
+export const UNKNOWN_REQUEST_SOURCE_ADDRESS = "unknown";
+
+/**
+ * App-owned request header carrying the server-resolved client address into
+ * Better Auth, which reads client addresses only from request headers. It is
+ * removed from every inbound request and re-stamped on managed-auth routes, so
+ * a caller-supplied value never reaches the auth rate limiter or session IP.
+ */
+export const TRUSTED_CLIENT_ADDRESS_HEADER = "x-opengeni-trusted-client-address";
+
 type ApiRequestBindings = {
   [TRANSPORT_PEER_ADDRESS_BINDING]?: string | null;
 };
 
+/** The deployment's declared proxy chain (`OPENGENI_API_TRUSTED_PROXY_*`). */
+export type RequestSourceTrust = Pick<Settings, "apiTrustedProxyHops" | "apiTrustedProxyCidrs">;
+
 export function apiRequestBindingsForTransportPeer(
   address: string | null | undefined,
 ): ApiRequestBindings {
@@ -15,23 +32,147 @@ export function apiRequestBindingsForTransportPeer(
 /**
  * Resolve a quota/audit source from the server-owned transport peer. Forwarded
  * values are considered only when the operator declares an exact trusted proxy
- * hop count; the chain is then walked from the server side so caller-prepended
- * values cannot replace the address inserted by the trusted edge.
+ * hop count (`OPENGENI_API_TRUSTED_PROXY_HOPS`) and, when
+ * `OPENGENI_API_TRUSTED_PROXY_CIDRS` is set, only for a transport peer inside
+ * those ranges. Each trusted proxy must append the address of the peer that
+ * connected to it, or overwrite the header with the original client address;
+ * the chain is then walked from the server side so caller-prepended values
+ * cannot replace the address observed by the trusted edge. A missing, short,
+ * or malformed chain falls back to the transport peer.
  */
-export function trustedRequestSourceAddress(c: Context, trustedProxyHops: number): string {
+export function trustedRequestSourceAddress(c: Context, trust: RequestSourceTrust): string {
   const bindings = c.env as ApiRequestBindings | undefined;
-  const peer = normalizedAddress(bindings?.[TRANSPORT_PEER_ADDRESS_BINDING]) ?? "unknown";
-  if (trustedProxyHops <= 0 || peer === "unknown") return peer;
-
-  const forwarded = (c.req.header("x-forwarded-for") ?? "")
-    .split(",")
-    .map(normalizedAddress)
-    .filter((value): value is string => value !== null);
+  const peer =
+    normalizedAddress(bindings?.[TRANSPORT_PEER_ADDRESS_BINDING]) ?? UNKNOWN_REQUEST_SOURCE_ADDRESS;
+  const trustedProxyHops = trust.apiTrustedProxyHops;
+  if (
+    !Number.isInteger(trustedProxyHops) ||
+    trustedProxyHops <= 0 ||
+    peer === UNKNOWN_REQUEST_SOURCE_ADDRESS ||
+    !peerIsTrustedProxy(peer, trust.apiTrustedProxyCidrs)
+  ) {
+    return peer;
+  }
+  const header = c.req.header("x-forwarded-for");
+  if (!header) return peer;
+  // Entries are never filtered: dropping a malformed value would shift the
+  // server-side position and could promote a caller-prepended value.
+  const forwarded = header.split(",");
   const sourceIndex = forwarded.length - trustedProxyHops;
-  return sourceIndex >= 0 ? forwarded[sourceIndex]! : peer;
+  if (sourceIndex < 0) return peer;
+  return normalizedAddress(forwarded[sourceIndex]) ?? peer;
+}
+
+/**
+ * Rate-limit key for a trusted source address. An IPv6 address keys on its
+ * /64, the block one subscriber normally holds and the granularity Better
+ * Auth applies to the same stamped address, so rotating addresses inside one
+ * /64 cannot mint fresh buckets. An IPv4-mapped IPv6 address (how a
+ * dual-stack listener reports an IPv4 peer) keys on the embedded IPv4 address
+ * instead of collapsing every IPv4 client into one /64. Anything else,
+ * including {@link UNKNOWN_REQUEST_SOURCE_ADDRESS}, is returned unchanged.
+ */
+export function requestSourceRateLimitKey(address: string): string {
+  if (isIP(address) !== 6) return address;
+  const groups = ipv6Groups(address);
+  if (!groups) return address;
+  if (groups.slice(0, 5).every((group) => group === 0) && groups[5] === 0xffff) {
+    const high = groups[6]!;
+    const low = groups[7]!;
+    return `${high >> 8}.${high & 0xff}.${low >> 8}.${low & 0xff}`;
+  }
+  return `${groups
+    .slice(0, 4)
+    .map((group) => group.toString(16))
+    .join(":")}::/64`;
+}
+
+/** {@link trustedRequestSourceAddress} as an application rate-limit key. */
+export function trustedRequestSourceRateLimitKey(c: Context, trust: RequestSourceTrust): string {
+  return requestSourceRateLimitKey(trustedRequestSourceAddress(c, trust));
 }
 
+/**
+ * Replace any caller-supplied {@link TRUSTED_CLIENT_ADDRESS_HEADER}. When
+ * `stamp` is true the header is set to the trusted source address, so every
+ * Better Auth request derived from this request (direct handler calls,
+ * isolated provider requests, and `auth.api.*` calls built from its headers)
+ * keys rate limits and session records on the real client.
+ */
+export function replaceTrustedClientAddressHeader(
+  c: Context,
+  trust: RequestSourceTrust,
+  stamp: boolean,
+): void {
+  const headers = c.req.raw.headers;
+  headers.delete(TRUSTED_CLIENT_ADDRESS_HEADER);
+  if (!stamp) return;
+  const address = trustedRequestSourceAddress(c, trust);
+  if (address !== UNKNOWN_REQUEST_SOURCE_ADDRESS) {
+    headers.set(TRUSTED_CLIENT_ADDRESS_HEADER, address);
+  }
+}
+
+const trustedProxyRanges = new Map<string, BlockList | null>();
+
+function peerIsTrustedProxy(peer: string, cidrs: string): boolean {
+  let ranges = trustedProxyRanges.get(cidrs);
+  if (ranges === undefined) {
+    // Settings validation already rejected malformed entries at boot.
+    const entries = trustedProxyCidrEntries(cidrs);
+    ranges = entries.length === 0 ? null : new BlockList();
+    for (const entry of entries) ranges?.addSubnet(entry.address, entry.prefix, entry.family);
+    trustedProxyRanges.set(cidrs, ranges);
+  }
+  if (!ranges) return true;
+  return ranges.check(peer, isIP(peer) === 6 ? "ipv6" : "ipv4");
+}
+
+/**
+ * Canonical IP text, or null. Accepts the `ipv4:port` and `[ipv6]:port` forms
+ * some load balancers write into X-Forwarded-For, and drops IPv6 zone ids.
+ */
 function normalizedAddress(value: string | null | undefined): string | null {
-  const normalized = value?.trim();
-  return normalized ? normalized.slice(0, 128) : null;
+  let candidate = value?.trim();
+  if (!candidate || candidate.length > 64) return null;
+  const bracketed = /^\[([^\]]+)\](?::\d{1,5})?$/u.exec(candidate);
+  if (bracketed) {
+    candidate = bracketed[1]!;
+  } else {
+    const ipv4WithPort = /^(\d{1,3}(?:\.\d{1,3}){3}):\d{1,5}$/u.exec(candidate);
+    if (ipv4WithPort) candidate = ipv4WithPort[1]!;
+  }
+  const zone = candidate.indexOf("%");
+  if (zone >= 0) candidate = candidate.slice(0, zone);
+  const family = isIP(candidate);
+  if (family === 4) return candidate;
+  if (family === 6) return candidate.toLowerCase();
+  return null;
+}
+
+/** The eight 16-bit groups of a valid IPv6 address, or null. */
+function ipv6Groups(address: string): number[] | null {
+  let text = address;
+  const tail: number[] = [];
+  const lastColon = text.lastIndexOf(":");
+  const last = text.slice(lastColon + 1);
+  if (last.includes(".")) {
+    const octets = last.split(".").map(Number);
+    if (octets.length !== 4 || octets.some((octet) => !Number.isInteger(octet))) return null;
+    tail.push((octets[0]! << 8) | octets[1]!, (octets[2]! << 8) | octets[3]!);
+    text = text.slice(0, lastColon + 1);
+    if (!text.endsWith("::")) text = text.slice(0, -1);
+  }
+  const halves = text.split("::");
+  if (halves.length > 2) return null;
+  const parse = (part: string): number[] =>
+    part ? part.split(":").map((group) => Number.parseInt(group, 16)) : [];
+  const head = parse(halves[0]!);
+  const rest = halves.length === 2 ? parse(halves[1]!) : [];
+  const fill = halves.length === 2 ? 8 - head.length - rest.length - tail.length : 0;
+  if (fill < 0) return null;
+  const groups = [...head, ...Array.from({ length: fill }, () => 0), ...rest, ...tail];
+  if (groups.length !== 8) return null;
+  if (groups.some((group) => !Number.isInteger(group) || group < 0 || group > 0xffff)) return null;
+  return groups;
 }

package/src/http/sse.ts

@@ -5,8 +5,9 @@ import {
 } from "@opengeni/contracts";
 import {
   getWorkspaceInteractionRevisionState,
-  listSessionEvents,
+  listSessionEventPage,
   listWorkspaceControlEvents,
+  readSessionEventStorageGapEnd,
   type Database,
 } from "@opengeni/db";
 import {
@@ -21,6 +22,7 @@ import type { Observability } from "@opengeni/observability";
 import { MANAGED_AUTH_ACTOR_EPOCH_HEADER } from "@opengeni/core/managed-auth-session-sets";
 
 const SESSION_REPLAY_PAGE_SIZE = 100;
+const SESSION_REPLAY_PAGE_MAX_BYTES = 8 * 1024 * 1024;
 const WORKSPACE_CONTROL_REPLAY_PAGE_SIZE = 100;
 export const SSE_QUEUED_FRAME_MAX_COUNT = 1;
 export const SSE_WRITE_STALL_TIMEOUT_MS = 30_000;
@@ -149,12 +151,12 @@ export function createByteBoundedSseStream(
   return {
     stream,
     write: async (frame) => {
-      const chunk = encoder.encode(frame);
       for (;;) {
         if (stopped) return false;
         const desired = controller.desiredSize;
         if (desired === null) return false;
         if (desired >= 1 && queuedFrames === 0) {
+          const chunk = encoder.encode(frame);
           controller.enqueue(chunk);
           queuedFrames = 1;
           queuedBytes = chunk.byteLength;
@@ -267,6 +269,32 @@ export function createLatestWinsDelivery<T extends { sequence: number }>(
   };
 }
 
+/**
+ * Durable coverage for each projected frame. Storage may omit sequences on
+ * purpose (an archived session keeps only its readable timeline), and clients
+ * treat an uncovered jump as a gap to backfill. Within a page the next stored
+ * row bounds the gap; after the last row, `gapEnd` (when given) reads the
+ * race-free bound from storage.
+ */
+export async function coverStorageGaps(
+  projection: { events: SessionEvent[]; coveredThroughBySequence: ReadonlyMap<number, number> },
+  gapEnd: ((through: number) => Promise<number>) | null,
+): Promise<Map<number, number>> {
+  const coverage = new Map<number, number>();
+  const events = projection.events;
+  for (const [index, event] of events.entries()) {
+    const own = projection.coveredThroughBySequence.get(event.sequence) ?? event.sequence;
+    const next = events[index + 1];
+    coverage.set(event.sequence, next ? Math.max(own, next.sequence - 1) : own);
+  }
+  const last = events.at(-1);
+  if (last && gapEnd) {
+    const own = coverage.get(last.sequence)!;
+    coverage.set(last.sequence, Math.max(own, await gapEnd(own)));
+  }
+  return coverage;
+}
+
 export async function sseSessionStream(
   db: Database,
   bus: EventBus,
@@ -277,18 +305,19 @@ export async function sseSessionStream(
   options: SessionSseDeliveryOptions = {},
 ): Promise<Response> {
   if (isHttp1BrowserBatch(options)) {
-    const events = await listSessionEvents(db, workspaceId, sessionId, {
+    const { events } = await listSessionEventPage(db, workspaceId, sessionId, {
       after,
       limit: SESSION_REPLAY_PAGE_SIZE,
+      maxBytes: options.finiteResponseMaxBytes ?? HTTP1_BROWSER_SSE_BATCH_MAX_BYTES,
     });
     await options.reauthorize?.();
     const compactProjection = coalesceSessionEventDeltasWithCoverage(events);
+    const coverage = await coverStorageGaps(compactProjection, async (through) =>
+      readSessionEventStorageGapEnd(db, workspaceId, sessionId, through),
+    );
     return finiteSseBatchResponse(
       compactProjection.events.map((event) =>
-        formatSessionEventSse(
-          event,
-          compactProjection.coveredThroughBySequence.get(event.sequence) ?? event.sequence,
-        ),
+        formatSessionEventSse(event, coverage.get(event.sequence) ?? event.sequence),
       ),
       options,
     );
@@ -353,10 +382,12 @@ export async function sseSessionStream(
         targetSequence === undefined
           ? SESSION_REPLAY_PAGE_SIZE
           : Math.min(SESSION_REPLAY_PAGE_SIZE, targetSequence - lastSent);
-      const page = await listSessionEvents(db, workspaceId, sessionId, {
+      const replayPage = await listSessionEventPage(db, workspaceId, sessionId, {
         after: lastSent,
         limit,
+        maxBytes: SESSION_REPLAY_PAGE_MAX_BYTES,
       });
+      const page = replayPage.events;
       const eligible =
         targetSequence === undefined
           ? page
@@ -372,9 +403,16 @@ export async function sseSessionStream(
       // a long answer cannot create thousands of React renders and starve
       // command acknowledgements behind its own token stream.
       const compactProjection = coalesceSessionEventDeltasWithCoverage(eligible);
+      // Live delivery stops exactly at a published sequence; replay pages also
+      // cover what storage intentionally omits after their last event.
+      const coverage = await coverStorageGaps(
+        compactProjection,
+        targetSequence === undefined
+          ? async (through) => readSessionEventStorageGapEnd(db, workspaceId, sessionId, through)
+          : null,
+      );
       for (const projected of compactProjection.events) {
-        const coveredThrough =
-          compactProjection.coveredThroughBySequence.get(projected.sequence) ?? projected.sequence;
+        const coveredThrough = coverage.get(projected.sequence) ?? projected.sequence;
         await writeFrame(formatSessionEventSse(projected, coveredThrough));
         lastSent = coveredThrough;
       }
@@ -382,8 +420,10 @@ export async function sseSessionStream(
         throw new Error(`Session event replay made no progress after sequence ${lastSent}`);
       }
       if (targetSequence !== undefined && lastSent >= targetSequence) return;
+      if (targetSequence === undefined && !replayPage.hasMore) return;
       // A byte-selected page may contain fewer rows than requested, especially
-      // when one large message travels alone. Only an empty read proves EOF.
+      // when one large message travels alone. Use explicit continuation rather
+      // than page length; live publication covers appends after the snapshot.
     }
   };
   let durableDeliveryTail = Promise.resolve();
@@ -448,17 +488,28 @@ export async function sseSessionStream(
   stopReconnectObservation = durableFanout.subscribeRecovery(scheduleReconnectReconciliation);
 
   void (async () => {
-    const release = await bus.subscribe(workspaceId, sessionId, (events) => {
-      if (bootstrapping) {
-        for (const event of events) {
-          if (!newestBuffered || event.sequence > newestBuffered.sequence) {
-            newestBuffered = event;
+    const release = await bus.subscribe(
+      workspaceId,
+      sessionId,
+      (events) => {
+        if (bootstrapping) {
+          for (const event of events) {
+            if (!newestBuffered || event.sequence > newestBuffered.sequence) {
+              newestBuffered = event;
+            }
           }
+        } else {
+          delivery?.publish(events);
         }
-      } else {
-        delivery?.publish(events);
-      }
-    });
+      },
+      {
+        // Live fanout stopped while this stream still held it. Heartbeats alone
+        // would keep a silently stale timeline open; fail retryably so the
+        // client reconnects and replays from the durable Postgres cursor.
+        onTerminated: (error) =>
+          fail(retryableSseFailure("session live fanout subscription ended", error)),
+      },
+    );
     if (channel.stopped()) {
       release();
       return;
@@ -632,13 +683,22 @@ export async function sseWorkspaceControlStream(
   delivery = createLatestWinsDelivery(send, fail);
 
   void (async () => {
-    const release = await bus.subscribeWorkspaceControl(workspaceId, (event) => {
-      if (bootstrapping) {
-        if (!newestBuffered || event.sequence > newestBuffered.sequence) newestBuffered = event;
-      } else {
-        delivery?.publish([event]);
-      }
-    });
+    const release = await bus.subscribeWorkspaceControl(
+      workspaceId,
+      (event) => {
+        if (bootstrapping) {
+          if (!newestBuffered || event.sequence > newestBuffered.sequence) newestBuffered = event;
+        } else {
+          delivery?.publish([event]);
+        }
+      },
+      {
+        // Same contract as session SSE: a dead live subscription must reconnect
+        // and replay the durable control cursor, not idle on heartbeats.
+        onTerminated: (error) =>
+          fail(retryableSseFailure("workspace control live subscription ended", error)),
+      },
+    );
     if (channel.stopped()) {
       release();
       return;
@@ -678,7 +738,7 @@ export type WorkspaceInteractionSseOptions = SseDeliveryOptions & {
 
 /**
  * One HTTP connection for the two workspace-wide invalidation domains used by
- * every visible OpenGeni surface. Keeping these as separate HTTP/1 streams
+ * every visible Opengeni surface. Keeping these as separate HTTP/1 streams
  * consumes all six per-origin browser connections with only two windows and
  * starves ordinary mutations/terminal grants. The durable cursors remain
  * independent; this function only multiplexes their already-bounded SSE frames.