@harperfast/harper 5.2.13 → 5.3.0-beta.1

package/server/DESIGN.md

@@ -47,7 +47,8 @@ A request entering `http.ts` does **not** go through Fastify. The two `handleApp
 | ------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
 | `serverHelpers/Request.ts`                 | Wraps `IncomingMessage` with Harper-specific fields (user, response, headers).                                                                                                                                                                                                                                                                                                                                                                         |
 | `serverHelpers/Headers.ts`                 | Header mutation/merge utilities.                                                                                                                                                                                                                                                                                                                                                                                                                       |
-| `serverHelpers/contentTypes.ts`            | (de)serialization registry; `serialize`, `serializeMessage`, `getDeserializer`.                                                                                                                                                                                                                                                                                                                                                                        |
+| `serverHelpers/contentTypes.ts`            | (de)serialization registry; `serialize`, `serializeMessage`, `getMessageSerializer`, `getDeserializer`.                                                                                                                                                                                                                                                                                                                                                |
+| `serverHelpers/sharedMessageEncoding.ts`   | Per-message encoding shared across the subscribers of a topic — one serialization per (message, content type) and, for MQTT QoS 0, one PUBLISH packet per (payload, topic, protocol version). Keyed on the message object, which `transactionBroadcast` dispatches by identity to every subscription of a key. **Identity contract:** see below.                                                                                                       |
 | `serverHelpers/serverUtilities.ts`         | `OperationDefinition` and shared helpers.                                                                                                                                                                                                                                                                                                                                                                                                              |
 | `serverHelpers/OperationFunctionObject.ts` | Wraps an operation handler with metadata.                                                                                                                                                                                                                                                                                                                                                                                                              |
 | `serverHelpers/JSONStream.ts`              | Streaming JSON output for large responses.                                                                                                                                                                                                                                                                                                                                                                                                             |
@@ -58,6 +59,20 @@ A request entering `http.ts` does **not** go through Fastify. The two `handleApp
 | `serverRegistry.ts`                        | Trivial registry export.                                                                                                                                                                                                                                                                                                                                                                                                                               |
 | `status/`                                  | Server status reporting (cluster status, per-port info).                                                                                                                                                                                                                                                                                                                                                                                               |
 
+> **Subscription message identity contract.** `sharedMessageEncoding.ts` encodes a message once and
+> reuses the bytes for every subscriber of the topic, keyed on the message object's identity. That is
+> sound for every internal producer because each yields a fresh object per version — `transactionBroadcast`
+> hands one `auditRecord` to every subscription, `auditStore`'s `getValue` memoizes the decode in its
+> closure, and `primaryStore.getEntry` is version-guarded — so identity implies equal content. Pinned by
+> `unitTests/resources/subscriptionValueIdentity.test.js`.
+>
+> Rather than leave that as a rule custom Resources have to know, the event's record `version` is part
+> of the cache key: `DurableSubscriptionsSession` forwards it to the delivery listener, and an entry
+> whose version does not match is re-encoded. A Resource that reuses one envelope but advances the
+> version is therefore correct, not merely disallowed; one that supplies no version is not shared at
+> all. Only mutating an object _without_ changing its version can serve stale bytes, and that is
+> indistinguishable from re-sending the same message.
+
 ### Threads
 
 | File                       | Purpose                                                  |
@@ -74,6 +89,17 @@ calls fail with `ERR_HARPER_PROCESS_SHUTTING_DOWN`; scoped worker-type restarts
 `shutdownWorkersNow()` remains an immediate teardown: its worker shutdown messages are best-effort,
 and it force-terminates the remaining worker set rather than waiting for application drain hooks.
 
+A rolling restart serves the _old_ code until each worker is replaced, and where the OS grants
+`SO_REUSEPORT` the not-yet-replaced workers keep accepting connections for the whole restart — so a
+component deploy is not live pool-wide until `restartWorkers()` resolves. For the overlapping types
+(HTTP) it waits for each replacement to report `CHILD_STARTED` — including the ones that can only be
+started after their predecessor releases its exclusive ports (Windows/macOS/Bun) — and reports how
+many workers it left on the old code because a replacement never came up, versus how many
+replacements never started after their predecessor was already gone. Other thread types start their
+replacement without being awaited. Each wait is bounded by a per-worker startup backstop, so
+resolution means "the restart finished", not "every worker is new". A caller that treats its own success as
+"the component is live" must await it (see `deployComponent` in `components/operations.js`).
+
 > Workers receive `workerData.noServerStart = true` — never start the server inside a worker.
 >
 > `threadServer.listenOnDomainSocket()` skips a listener only when its path exceeds the platform's
@@ -105,7 +131,8 @@ Every entry is a top-level function or named const. Jump via go-to-symbol or `gr
 | `getPorts()`                                                                                                                             | Resolves listener options → list of `{port, secure}`.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               |
 | `httpServer()`                                                                                                                           | Main listener registration entry point.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             |
 | `getHTTPServer(port, secure, options)`                                                                                                   | **The largest function in the file.** Creates/retrieves the underlying Node HTTP/HTTPS server. Wires `request`, `upgrade`, error handlers, TLS context, and the per-port middleware chain.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          |
-| `makeCallbackChain()`                                                                                                                    | Builds the per-port handler chain via `middlewareChain.topoSort`.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   |
+| `makeCallbackChain()`                                                                                                                    | Builds the per-port handler chain via `middlewareChain.topoSort`, and records its resolved order for `get_status`.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  |
+| `buildChains()`                                                                                                                          | Stores a built chain, and rebuilds every other already-built chain of that kind when the registration is on the `'all'` pseudo-port. See "Middleware ordering" below.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               |
 | `unhandled()`                                                                                                                            | Terminal 404 handler.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               |
 | `onRequest()`                                                                                                                            | Thin alias of `httpServer({requestOnly: true})`.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    |
 | `onUpgrade()` / `upgradeListeners` (const)                                                                                               | Register HTTP upgrade listener; underlying list.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    |
@@ -125,6 +152,19 @@ Components register listeners with optional `before: 'name'` / `after: 'name'` o
 
 The default WebSocket upgrade handler is registered automatically inside `onWebSocket()` the first time it runs for a given port.
 
+**`port: 'all'` is a pseudo-port, and its chain is not the one that serves traffic.** Chain building
+folds every `'all'` entry into each concrete port's chain, so `chains.all` exists but nothing
+dispatches through it — `httpChain[port]`/`upgradeChains[port]`/`websocketChains[port]` are looked up
+by the bound port on every request (Node, Bun, and uWS alike). A registration therefore has to
+rebuild the chains it affects, not just its own key: `http.ts → buildChains()` rebuilds every
+already-built chain of that kind whenever the registration is on `'all'`. Without that, an entry
+registered on `'all'` after the concrete port's chain was built — the shape an application catch-all
+mounted `after: 'rest'` takes, since `rest` registers first — updates only `chains.all` and never
+reaches a request (#2418). Rebuilding is a pure function of the listener list and the port, so the
+extra passes can only reproduce a port's order or extend it. `buildChains()` also writes the
+`get_status` chain description in the same pass, which is what keeps that report from ever
+describing an order a port isn't running (#1573).
+
 ### Application mounts (`host` / `urlPath` in the root config)
 
 An operator mounts an application by putting `host`/`urlPath` on its entry in the **root** config; `components/scopeMount.ts` models it and the loader threads it into every `Scope` for that application (both load paths — the root-config `package` recursion and the components-root directory scan).
@@ -154,16 +194,152 @@ must run on a worker, `registeredOperations.ts` carries that state in the same-p
 separately from the structured-cloned body. Never attach trusted dispatch state to an operation
 payload.
 
+`server.registerOperation()` runs per-worker, so anything the **main** thread must later know about a
+registered op has to ride the OPERATION_REGISTERED announcement — a module-local registry populated
+during registration exists only in the worker that registered. The bridge carries two such facts
+today: name→thread routing (for execution forwarding) and `grantable` (so `validateOperations` on
+main will accept the name in a role's `operations` allowlist, for add_role/alter_role, impersonation,
+and OIDC trust policies). Adding a third main-thread consumer of a worker-registered fact means
+extending that message, not reading a registry that main never populated. Grantability is safe to
+mirror because it only widens what an allowlist may _name_; enforcement stays on the worker's
+`chooseOperation`.
+
 ## Resource ↔ HTTP boundary
 
 `REST.ts → http(request, nextHandler)` is the chief integration point: it takes a `Request`, asks the `Resources` registry for a match, builds a `RequestTarget`, and dispatches into the Resource class's static method. Cache headers are translated to `request.expiresAt` / `onlyIfCached` / `noCache` flags within the same function.
 
+### Streaming startup errors
+
+SSE and NDJSON serializers eagerly take and hold their first iterator step for GET requests. `REST.ts` waits through the next event-loop turn for that step: an immediate rejection remains an HTTP error rendered as Problem Details, while a first item or the cutoff commits the stream. Mutating requests do not use the startup-status gate because their transaction has already committed by this point; all of their stream failures use the in-band form. Later GET failures are terminal, format-valid records (`event: harper-error` for SSE and a reserved control record for NDJSON). Keep the decision in the serializer/REST boundary so Node, uWS, Bun, compression, and injection share one contract; transports must not independently prefetch the iterator.
+
+SSE terminal event data uses `{ error: <code-or-class>, message: <message>, status?: <status> }`; NDJSON wraps the same object as `{ "$harperStreamError": { ... } }` so it cannot be mistaken for an ordinary row with an `error` field.
+Sources with `mapError` remain unmapped for these two formats so their failures reach this single contract instead of becoming legacy data records.
+generic JSON-array streaming retains its older `{ error: "<name>: <message>" }` element shape.
+The `error` value is the stable programmatic discriminator; `message` is diagnostic and follows the same thrown-message exposure policy as pre-commit Problem Details.
+
+Clean stream completion does not prove completeness; clients must inspect SSE for the `harper-error` event and NDJSON for the `$harperStreamError` control record.
+
+### Deferred credential rejection (#2418)
+
+`authentication` runs before route matching, so when it meets an `Authorization` header it cannot
+resolve it does not yet know whether Harper or an application owns the URL. Rejecting there forces
+an application to choose between two things it needs: Harper owning its own routes (status, REST
+resources) and its own routes receiving their own credential scheme untouched.
+
+So a rejection is recorded rather than answered:
+
+- **Valid Harper credentials** authenticate normally and populate `request.user`. Unchanged.
+- **No credentials** continue anonymously. Unchanged.
+- **A syntactically valid credential Harper does not recognize** leaves `request.user` unset, leaves
+  the inbound `Authorization` header byte-for-byte intact, and records request-local state through
+  `security/deferredAuthentication.ts`. The state lives behind a module-private `Symbol`. Its
+  descriptor and value are immutable and non-enumerable, so downstream middleware cannot clear it
+  and it does not leak through request copies or serialization.
+- **An internal authentication fault** — unreadable or malformed JWT key material, a storage failure,
+  an unexpected error type — is never deferred, and fails closed with the in-line 401.
+- **The operations API** never defers: `request.isOperationsServer` short-circuits to the in-line
+  401, because every operations route is Harper-owned and there is nothing to defer to.
+
+**Rejection provenance is asserted, never inferred.** `security/credentialRejection.ts` holds a
+module-private `Symbol` tag; only the code that actually concludes "this credential is unacceptable"
+sets it, and `isCredentialRejection()` reads nothing else. Status ranges cannot carry that meaning:
+`findAndValidateUser()` lazily loads the user cache, whose system-table searches raise a
+default-status-400 `ClientError` when `system.hdb_role`/`system.hdb_user` is unavailable, so a 4xx
+test would classify a storage outage as an unknown credential and hand it to application
+authorization. The tag is set at exactly these points:
+
+| Tagged rejection                                                   | Where                                      |
+| ------------------------------------------------------------------ | ------------------------------------------ |
+| unknown user, inactive user, bad password                          | `security/user.ts → findAndValidateUser()` |
+| JWT syntax, signature, expiry, not-before, subject/claim rejection | `tokenAuthentication.ts → validateToken()` |
+| refresh-token hash mismatch, malformed scoped-token claims         | `tokenAuthentication.ts`                   |
+| an `Authorization` scheme Harper does not implement                | `security/auth.ts`                         |
+
+`validateToken()` separates the two in the same catch: `jsonwebtoken` reports unusable key material
+through the very same `JsonWebTokenError` type it uses for a forged token, so the public key is
+validated as key material before `jwt.verify()` runs and a residual key-material message is treated
+as a fault. An untagged error propagates unmasked.
+
+The Bearer path's refresh-token probe follows the same rule. When operation-token validation says
+`invalid token`, authentication retries the credential as a refresh token; a fault raised by that
+retry propagates, and only an ordinary _tagged_ refresh rejection restores the original
+operation-token rejection for deferral. An in-line fail-closed response logs the original fault
+server-side and returns the same generic authentication failure a rejected credential gets, so
+internal detail never reaches an unauthenticated client.
+
+Any layer that establishes Harper owns the route then settles the deferred state before doing work:
+
+| Layer                                   | Where                                                              |
+| --------------------------------------- | ------------------------------------------------------------------ |
+| `REST.ts → http()`                      | after `resources.getMatch` succeeds (and for the OpenAPI document) |
+| `REST.ts` WebSocket handler             | after `resources.getMatch(url, 'ws')` succeeds                     |
+| `graphqlQuerying.ts`                    | after the `/graphql` prefix match, ahead of its error mapping      |
+| `static.ts`                             | after a static file entry matches                                  |
+| `mqtt.ts` WebSocket handler             | once the pending HTTP chain settles, before the first packet       |
+| `components/mcp/adapters/harperHttp.ts` | after the WebSocket hand-off, before the body is read              |
+
+**Every Harper-owned handler registered `after: 'authentication'` owes this settlement**, because
+declining to call `nextHandler` is precisely the moment ownership is settled. A handler that skips
+it serves an unrecognized credential as anonymous — the MCP application mount did exactly that, and
+returned 200 for an `initialize` the base revision answered with 401.
+
+Settlement goes through `settleDeferredCredentialRejection()`, which returns the response descriptor
+`security/auth.ts` used to return in line: status 401 and `serializeMessage({error: message}, request)`
+in the request's negotiated content type. That matters because an owner's own error mapping is not
+that contract — REST renders a thrown error as an RFC 9457 Problem Details document and GraphQL as
+`{errors:[{message}]}` — and a rejected credential never reached either before deferral existed.
+`assertNoDeferredCredentialRejection()` is the throwing form, for a WebSocket upgrade that has no
+descriptor to return. A WebSocket owner cannot read the state synchronously: `server/http.ts` starts
+`httpChain[port](request)` and invokes the WebSocket chain with the still-pending completion, so
+authentication has not yet classified the credential. `mqtt.ts` therefore settles on that promise —
+the same one the session's principal resolves from — and closes the socket from its rejection, while
+its frame handlers still attach synchronously. The deferred status is pinned to 401 regardless of the underlying error's own
+status, so a 403 `token expired` reads exactly as it did before. A Harper-owned route therefore behaves identically to the pre-deferral
+build, protected or public: an unknown credential can never buy access an anonymous caller would
+have received, and can never reach an application catch-all. Only a URL that reached
+`nextHandler` — one no Harper route owns — carries the original header onward.
+
+The contract is route-ownership-based, not path-based. There is no exemption list, no carrier
+header, no credential rename, and no pre-auth stripping shim.
+
+**Authentication does not re-decorate a 401 it did not raise.** `security/auth.ts` post-processes any
+401 coming back up the chain — overwriting `WWW-Authenticate` with `Basic`, or rewriting the status to
+a 302 at `resources.loginPath` for a browser. The in-line rejection deferral replaced returned before
+that code, so a settled rejection must skip it to stay wire-identical; and a 401 an application
+catch-all raised for its own scheme (a WooCommerce or Bearer challenge) is that application's to make.
+Both cases are keyed on the deferred state, so a request that deferred nothing keeps the existing
+behavior exactly. The #1565 identity floor still applies either way — it is stamped in
+`applyResponseHeaders`, not in the challenge rewriting.
+
+**The identity cache floor survives the legacy Fastify fallbacks.** A response produced under a
+deferred credential is credential-dependent (#1565), so `authentication` stamps
+`Cache-Control: private, no-cache` and `Vary: Authorization, Cookie` on it. Before deferral an
+unrecognized credential could not reach a fallback at all, so this was unreachable. All three adapters
+now reconcile through one policy, `Headers.ts → mergeChainHeadersIntoFallback()`: Fastify wins every
+single-valued header it set, `Vary` is unioned, and the private scope is re-applied unless the final
+response explicitly opts into shared caching (`public`/`s-maxage`).
+
+`Set-Cookie` is the deliberate exception to Fastify-wins. It is a list-valued field, so the chain's
+cookies are appended beside Fastify's and de-duplicated by exact value, never by cookie name. A cookie
+is identified by name _plus_ `Domain` _plus_ `Path` (RFC 6265 §5.3), so collapsing by name drops
+legitimately distinct cookies — a same-name pair scoped to `/` and `/wp-admin`, or a `Max-Age=0`
+deletion paired with a set. Exact value is also what makes Node's `writeHead` re-merge idempotent,
+because that path sees the chain's own cookie already on the response. Two cookies that do share a
+full identity both reach the client, chain last, and the user agent resolves them last-wins.
+
+Bun and uWS rebuild their headers from Fastify's reply and merge once. Node hands Fastify the same
+`ServerResponse` the chain's headers were copied onto, so copying is not enough — a route calling
+`reply.header('Cache-Control', …)` replaces the floor outright and can make a credential-dependent
+response shared-cacheable. `bridgeChainHeadersToNodeResponse()` therefore runs the same merge from a
+`writeHead` interception, the last point the header set is still mutable and the one Node also routes
+implicit headers through.
+
 ### Response Cache-Control / Vary policy (#1518, #1565)
 
 Three tiers, applied in two places:
 
 1. **App/resource explicit** — a `Cache-Control` set by the resource (or `@table(cacheControl: "...")` for anonymous reads, emitted in `REST.ts → http()`) always wins. The declaration is required: anonymous readability alone never emits shared-cache headers, because a request-attribute-gated `allowRead` (IP, headers) would make inferred `public` unsound.
-2. **Identity floor** — `security/auth.ts → applyResponseHeaders` stamps `Cache-Control: private, no-cache` + `Vary: Authorization` (+ `Cookie` when sessions are on) on any response where a principal was resolved or credentials were rejected (401), _unless_ the app opted into shared caching with `public`/`s-maxage` (the RFC 9111 opt-in).
+2. **Identity floor** — `security/auth.ts → applyResponseHeaders` stamps `Cache-Control: private, no-cache` + `Vary: Authorization` (+ `Cookie` when sessions are on) on any response where a principal was resolved, credentials were rejected (401), or a credential rejection was deferred (#2418 — the application answered using the header Harper passed through, so the response is credential-dependent at a plain 200), _unless_ the app opted into shared caching with `public`/`s-maxage` (the RFC 9111 opt-in).
 3. **CORS partitioning** — when CORS is enabled, every response gets `Vary: Origin` (the ACAO header is reflected per-origin, and its absence on no-Origin requests is origin-dependent too).
 
 The `@table(cacheControl:)` value is persisted on the primary-key attribute (like `expiration`), so all threads and future boots see it; `resources/databases.ts → table()` treats `null` as "schema explicitly has none" (clears on reload) and `undefined` as "caller is not schema-defining" (no clobber from `add_attribute`/cluster schema events).
@@ -172,19 +348,21 @@ The `@table(cacheControl:)` value is persisted on the primary-key attribute (lik
 
 ## "Where is X" cheat sheet
 
-| Question                                                                  | Where                                                                                                                                                                     |
-| ------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| Where do I register a new HTTP handler?                                   | `http.ts → httpServer()` (or `onRequest()` for the request-only form)                                                                                                     |
-| Where do I register a WebSocket handler?                                  | `http.ts → onWebSocket()`                                                                                                                                                 |
-| How does `before`/`after` middleware ordering work?                       | `middlewareChain.ts → topoSort`                                                                                                                                           |
-| Where does PROXY protocol get parsed?                                     | `serverHelpers/proxyProtocol.ts` (applied by `http.ts → enableProxyProtocol` / `createH2CProxyFront`)                                                                     |
-| How does an app read forwarded TLS facts (JA3/JA4, ALPN, SNI, mTLS cert)? | `request.connectionInfo` (`ConnectionInfo` from `serverHelpers/proxyProtocol.ts`); only set from a trusted PROXY v2 header on the UDS mirror, never from a request header |
-| Where is the REST request → Resource dispatch?                            | `REST.ts → http()`                                                                                                                                                        |
-| Where is the operations API request handled?                              | `operationsServer.ts → handler`                                                                                                                                           |
-| How are content types (de)serialized?                                     | `serverHelpers/contentTypes.ts`                                                                                                                                           |
-| Where do durable subscriptions live?                                      | `DurableSubscriptionsSession.ts`                                                                                                                                          |
-| How are sockets dispatched to worker threads?                             | `threads/socketRouter.ts`                                                                                                                                                 |
-| Where is the Operations API wired into Fastify?                           | `operationsServer.ts → buildServer`                                                                                                                                       |
+| Question                                                                   | Where                                                                                                                                                                                                                                                                                                                                                                                                                  |
+| -------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| Where do I register a new HTTP handler?                                    | `http.ts → httpServer()` (or `onRequest()` for the request-only form)                                                                                                                                                                                                                                                                                                                                                  |
+| Where do I register a WebSocket handler?                                   | `http.ts → onWebSocket()`                                                                                                                                                                                                                                                                                                                                                                                              |
+| How does `before`/`after` middleware ordering work?                        | `middlewareChain.ts → topoSort`                                                                                                                                                                                                                                                                                                                                                                                        |
+| Where does PROXY protocol get parsed?                                      | `serverHelpers/proxyProtocol.ts` (applied by `http.ts → enableProxyProtocol` / `createH2CProxyFront`)                                                                                                                                                                                                                                                                                                                  |
+| How does an app read forwarded TLS facts (JA3/JA4, ALPN, SNI, mTLS cert)?  | `request.connectionInfo` (`ConnectionInfo` from `serverHelpers/proxyProtocol.ts`); only set from a trusted PROXY v2 header on the UDS mirror, never from a request header                                                                                                                                                                                                                                              |
+| Why does mTLS revocation checking look up the client cert's issuer itself? | `security/certificateVerification/trustedIssuers.ts` — Node only completes a peer chain from the _listener_ context's store, but Harper's client CAs live on SNI contexts (`security/keys.ts → createTLSSelector`), so a resumed TLS session (and Node 26.8.0/26.8.1, nodejs/node#65579) exposes the leaf alone; `verifyCertificate` recovers the issuer from the published CA set and otherwise applies `failureMode` |
+| Where is the REST request → Resource dispatch?                             | `REST.ts → http()`                                                                                                                                                                                                                                                                                                                                                                                                     |
+| Where is the operations API request handled?                               | `operationsServer.ts → handler`                                                                                                                                                                                                                                                                                                                                                                                        |
+| How are content types (de)serialized?                                      | `serverHelpers/contentTypes.ts`                                                                                                                                                                                                                                                                                                                                                                                        |
+| Why doesn't every MQTT subscriber re-serialize the message it receives?    | `serverHelpers/sharedMessageEncoding.ts` (memoized on the message object the fan-out shares); consumed by the outbound listener in `mqtt.ts`                                                                                                                                                                                                                                                                           |
+| Where do durable subscriptions live?                                       | `DurableSubscriptionsSession.ts`                                                                                                                                                                                                                                                                                                                                                                                       |
+| How are sockets dispatched to worker threads?                              | `threads/socketRouter.ts`                                                                                                                                                                                                                                                                                                                                                                                              |
+| Where is the Operations API wired into Fastify?                            | `operationsServer.ts → buildServer`                                                                                                                                                                                                                                                                                                                                                                                    |
 
 ---
 

package/server/DurableSubscriptionsSession.ts

@@ -32,6 +32,24 @@ function getDurableSession() {
 	return _DurableSession;
 }
 let _LastWill: any;
+/**
+ * A scoped token's only revocation is expiry, so its will must not publish past it. Keyed off the
+ * persisted will principal (not any live session user) so both will paths agree on one source.
+ * Fails closed: a scoped will with no recorded expiry is treated as expired rather than published.
+ */
+function isWillFromExpiredScopedToken(will: any): boolean {
+	const user = will?.user;
+	if (!user?._scopedToken) return false;
+	return !user.authExpiresAt || user.authExpiresAt * 1000 <= Date.now();
+}
+
+/** Drops the runtime-only pre-expanded operations Set so the permission set is storage-safe. */
+function stripRuntimePermissionState(permission: any): any {
+	if (!permission || typeof permission !== 'object') return permission;
+	const { _expandedOperations, ...durable } = permission;
+	return durable;
+}
+
 function getLastWill() {
 	if (!_LastWill) {
 		_LastWill = table({
@@ -56,13 +74,23 @@ if (getWorkerIndex() === 0) {
 		for await (const will of getLastWill().search({})) {
 			const data = will.data;
 			const message = { ...will };
-			if (message.user?.username) message.user = await (server as any).getUser(message.user.username);
 			try {
-				await publishMessage(message, data, message);
+				if (message.user?._scopedToken) {
+					// A scoped token's username is attribution only; never rehydrate it by name (that could
+					// substitute a real principal). The will carries the token's own downgraded role.
+					if (!isWillFromExpiredScopedToken(message)) {
+						await publishMessage(message, data, message);
+					} else warn('Dropping will from an expired scoped token', data);
+				} else {
+					if (message.user?.username) message.user = await (server as any).getUser(message.user.username);
+					await publishMessage(message, data, message);
+				}
+				await getLastWill().delete(will.id);
 			} catch {
+				// One will's publish/delete failure must not abort replay of the rest; the row stays and
+				// is retried on the next restart.
 				warn('Failed to publish will', data);
 			}
-			getLastWill().delete(will.id);
 		}
 	})();
 }
@@ -125,7 +153,22 @@ export async function getSession({
 	}
 	if (will) {
 		will.id = sessionId;
-		will.user = { username: user?.username };
+		// A scoped-token bearer's will must carry the token's own role and expiry: its username is
+		// attribution only and cannot be rehydrated from hdb_user at replay time. Persist only the
+		// durable permission fields — not the runtime-only _expandedOperations Set, which is rebuilt
+		// on read and would not round-trip through storage.
+		will.user = user?._scopedToken
+			? {
+					username: user.username,
+					_scopedToken: true,
+					authExpiresAt: user.authExpiresAt,
+					role: user.role && {
+						role: user.role.role,
+						id: user.role.id,
+						permission: stripRuntimePermissionState(user.role.permission),
+					},
+				}
+			: { username: user?.username };
 		// Must be durably persisted before CONNACK is sent (getSession() resolving is what lets
 		// mqtt.ts send CONNACK). Otherwise a client that connects and then disconnects abruptly
 		// (no DISCONNECT packet) can race ahead of this write: SubscriptionsSession.disconnect()
@@ -155,7 +198,7 @@ type Acknowledgement = {
 };
 
 class SubscriptionsSession {
-	listener: (message, subscription, timestamp, qos) => any;
+	listener: (topic, message, messageId, subscription, version?) => any;
 	sessionId: any;
 	user: any;
 	request: any;
@@ -303,7 +346,16 @@ class SubscriptionsSession {
 						let path = update.id;
 						if (Array.isArray(path)) path = keyArrayToString(path);
 						if (path == null) path = '';
-						const result = await this.listener(resourcePath + '/' + path, update.value, messageId, subscriptionRequest);
+						// the version is forwarded so the delivery side can tell a store-sourced event (whose
+						// value is a fresh object per version) from an app-yielded one that may be a reused
+						// mutable envelope — only the former is safe to encode once and share
+						const result = await this.listener(
+							resourcePath + '/' + path,
+							update.value,
+							messageId,
+							subscriptionRequest,
+							update.version
+						);
 						if (result === false) break;
 						if (this.awaitingAcks?.size > AWAITING_ACKS_HIGH_WATER_MARK) {
 							// slow it down if we are getting too far ahead in acks
@@ -372,7 +424,7 @@ class SubscriptionsSession {
 		}
 		return context;
 	}
-	setListener(listener: (message) => any) {
+	setListener(listener: (topic, message, messageId, subscription, version?) => any) {
 		this.listener = listener;
 	}
 	disconnect(clientTerminated) {
@@ -382,8 +434,11 @@ class SubscriptionsSession {
 			try {
 				if (!clientTerminated) {
 					const will = await getLastWill().get(this.sessionId);
-					if (will) {
-						await publishMessage(will, will.data, context);
+					if (will && !isWillFromExpiredScopedToken(will)) {
+						// A scoped will authorizes under its own embedded role, never the disconnecting
+						// session's user (which may be a later same-clientId reconnect).
+						const willContext = will.user?._scopedToken ? { ...context, user: will.user } : context;
+						await publishMessage(will, will.data, willContext);
 					}
 				}
 			} finally {
@@ -413,10 +468,15 @@ async function publishMessage(message: any, data: any, context: any) {
 	message = { ...message, data, async: true };
 	context.authorize = true;
 	const entry = resources.getMatch(topic, 'mqtt');
-	if (!entry)
-		throw new Error(
+	if (!entry) {
+		// Typed like addSubscription's identical miss, so a protocol layer can map it to a specific
+		// code rather than a generic failure.
+		const notFoundError: any = new Error(
 			`Can not publish to topic ${topic} as it does not exist, no resource has been defined to handle this topic`
 		);
+		notFoundError.statusCode = 404;
+		throw notFoundError;
+	}
 	message.url = entry.relativeURL;
 	const target = new RequestTarget(entry.relativeURL);
 	if (entry.params) Object.assign(target, entry.params); // bind parameterised path segments (e.g. :id, *rest)

package/server/REST.ts

@@ -1,6 +1,12 @@
 import { stat } from 'node:fs/promises';
 import { join } from 'node:path';
-import { serialize, serializeMessage, getDeserializer } from '../server/serverHelpers/contentTypes.ts';
+import {
+	serialize,
+	serializeMessage,
+	getDeserializer,
+	waitForStreamStartup,
+	discardSerializedStream,
+} from '../server/serverHelpers/contentTypes.ts';
 import { addAnalyticsListener, recordAction, recordActionBinary } from '../resources/analytics/write.ts';
 import * as harperLogger from '../utility/logging/harper_logger.ts';
 import { ServerError, ClientError } from '../utility/errors/hdbError.ts';
@@ -8,13 +14,17 @@ import { Resources } from '../resources/Resources.ts';
 import { Resource, missingMethod, allowedMethods } from '../resources/Resource.ts';
 import { IterableEventQueue } from '../resources/IterableEventQueue.ts';
 import { transaction } from '../resources/transaction.ts';
-import { Headers, mergeHeaders } from '../server/serverHelpers/Headers.ts';
+import { Headers, mergeHeaders, addVaryHeader } from '../server/serverHelpers/Headers.ts';
 import { generateJsonApi } from '../resources/openApi.ts';
 import { getConfigPath } from '../config/configUtils.ts';
 import { CONFIG_PARAMS } from '../utility/hdbTerms.ts';
 import { ASIDE_STAGING_DIR } from '../components/Application.ts';
 import { COMPONENT_PREPARATION_LOCK_DIR } from '../components/componentPreparationLock.ts';
 import { restartNeeded } from '../components/requestRestart.ts';
+import {
+	assertNoDeferredCredentialRejection,
+	settleDeferredCredentialRejection,
+} from '../security/deferredAuthentication.ts';
 
 import { Request } from '../server/serverHelpers/Request.ts';
 import { RequestTarget } from '../resources/RequestTarget';
@@ -117,6 +127,39 @@ async function findInactiveComponent(url: string): Promise<string | undefined> {
 	}
 }
 
+/**
+ * Emit RFC 7233-style pagination headers for a `Prefer: count=` request. Table.search returns the page
+ * with a `recordCount` (total matching records, or null when an exact scan hit its guardrail or an
+ * estimate was suppressed by an opaque filter). `Content-Range: items <start>-<end>/<total>` lets a
+ * client paginate — `<total>` is `*` when unavailable. `Preference-Applied` echoes the count mode the
+ * server actually applied (`exact` or `estimated`, after any per-mount downgrade), so a `.../*` total
+ * reads as "that mode was applied but the total is unavailable" rather than "no count was requested".
+ * All three headers are added to `Access-Control-Expose-Headers` so a browser can read them cross-origin
+ * (they aren't safelisted).
+ */
+function setCountHeaders(headers: Headers, offset: number, mode: string, page: any) {
+	const total = page.recordCount;
+	const len = Array.isArray(page) ? page.length : 0;
+	const range = len > 0 ? `${offset}-${offset + len - 1}` : '*';
+	const totalStr = typeof total === 'number' ? String(total) : '*';
+	headers.set('Range-Unit', 'items');
+	headers.set('Content-Range', `items ${range}/${totalStr}`);
+	headers.set('Preference-Applied', `count=${mode}`);
+	// Append (don't overwrite) so a resource that already exposed its own headers keeps them. Compare
+	// case-insensitive comma tokens, not substrings, so an unrelated existing token (e.g.
+	// `X-Content-Range-Metadata`) doesn't suppress the real `Content-Range` token.
+	const exposed = headers.get('Access-Control-Expose-Headers');
+	const existing = new Set(
+		(Array.isArray(exposed) ? exposed.join(',') : exposed || '')
+			.split(',')
+			.map((token) => token.trim().toLowerCase())
+			.filter(Boolean)
+	);
+	for (const name of ['Content-Range', 'Range-Unit', 'Preference-Applied']) {
+		if (!existing.has(name.toLowerCase())) headers.append('Access-Control-Expose-Headers', name, true);
+	}
+}
+
 async function http(request: Request, nextHandler, resources: Resources, httpOptions: any) {
 	const headersObject = request.headers.asObject;
 	const isSse = headersObject.accept === 'text/event-stream';
@@ -165,7 +208,32 @@ async function http(request: Request, nextHandler, resources: Resources, httpOpt
 
 			(target as any).async = true;
 			resource = entry.Resource;
+			// Pagination total-count opt-in (no default): `Prefer: count=exact|estimated`. Table.search
+			// reads target.count to compute the total emitted as Content-Range below. Only honored on
+			// GET/HEAD reads: setting it for other methods would hand their `search()` a materialized array
+			// instead of the AsyncIterable they iterate (e.g. a collection DELETE at Table.ts).
+			const prefer = headersObject['prefer'];
+			if (prefer && (method === 'GET' || method === 'HEAD')) {
+				// Exact counting scans the full matched set, so it is opt-in per mount via
+				// `rest: { exactCount: true }` (default off); count=exact is otherwise served as a cheap
+				// estimate. Estimated is always available. Accept a string `"true"` too, since not every
+				// config source coerces to a boolean.
+				const exactEnabled = (httpOptions as any)?.exactCount === true || (httpOptions as any)?.exactCount === 'true';
+				for (const pref of parseHeaderValue(prefer as any)) {
+					// The header parser can hand back a non-string value (e.g. a bare `Prefer: count` with no
+					// `=`), so coerce before lower-casing rather than calling `.toLowerCase()` on a boolean.
+					const mode = String(pref?.value ?? '').toLowerCase();
+					if (pref?.name === 'count' && (mode === 'exact' || mode === 'estimated')) {
+						// A count=exact request on a mount that hasn't opted in is downgraded to estimated,
+						// signaled back to the client via Preference-Applied.
+						(target as any).count = mode === 'exact' && !exactEnabled ? 'estimated' : mode;
+						break;
+					}
+				}
+			}
 		}
+		const settledCredentialRejection = settleDeferredCredentialRejection(request);
+		if (settledCredentialRejection) return settledCredentialRejection;
 		if ((resource as any)?.isCaching) {
 			const cacheControl = headersObject['cache-control'];
 			if (cacheControl) {
@@ -286,7 +354,15 @@ async function http(request: Request, nextHandler, resources: Resources, httpOpt
 				headers.setIfNone('Last-Modified', new Date(lastModification).toUTCString());
 		} else if (responseData.headers) {
 			// if response is a Response object (or response-like envelope with headers), use it as the response
-			return finalizeResponse(responseData, headers, status, request);
+			const response = finalizeResponse(responseData, headers, status, request);
+			if (request.method === 'HEAD') {
+				discardSerializedStream(response.body);
+				if (!(response instanceof Response)) response.body = undefined;
+			} else if (request.method === 'GET') {
+				const startup = waitForStreamStartup(response.body);
+				if (startup) await startup;
+			}
+			return response;
 		} else if (isFinite(lastModification)) {
 			etagFloat[0] = lastModification;
 			// base64 encoding of the 64-bit float encoding of the date in ms (with quotes)
@@ -348,9 +424,30 @@ async function http(request: Request, nextHandler, resources: Resources, httpOpt
 		}
 		// TODO: Handle 201 Created
 		if (responseData !== undefined) {
+			if (
+				(target as any)?.count &&
+				(method === 'GET' || method === 'HEAD') &&
+				Array.isArray(responseData) &&
+				(responseData as any).recordCount !== undefined
+			) {
+				// Array.isArray guards the single-record path: a record that happens to carry a
+				// `recordCount` attribute must not be mistaken for a count page (Table.search only ever
+				// returns the count as an array).
+				setCountHeaders(headers, (target as any).offset || 0, (target as any).count, responseData);
+			}
 			responseObject.body = serialize(responseData, request, responseObject);
-			if (method === 'HEAD') responseObject.body = undefined; // we want everything else to be the same as GET, but then omit the body
+			if (request.method === 'HEAD') {
+				discardSerializedStream(responseObject.body);
+				responseObject.body = undefined; // we want everything else to be the same as GET, but then omit the body
+			} else if (request.method === 'GET') {
+				const startup = waitForStreamStartup(responseObject.body);
+				if (startup) await startup;
+			}
 		}
+		// A collection read's count headers vary by the request's `Prefer` value; serialize() just reset
+		// `Vary`, so declare it here (after serialization) — otherwise a shared cache could serve count
+		// headers to a request that didn't ask, or a cached non-count response to one that did.
+		if ((method === 'GET' || method === 'HEAD') && (target as any)?.isCollection) addVaryHeader(headers, 'Prefer');
 		return responseObject;
 	} catch (error) {
 		error ??= new Error('Unknown error occurred');
@@ -389,6 +486,19 @@ async function http(request: Request, nextHandler, resources: Resources, httpOpt
 			headers,
 			body: undefined,
 		};
+		// The error body is a different representation, so success representation and caching headers
+		// accumulated before the startup failure must not survive its serialization.
+		for (const header of [
+			'Content-Encoding',
+			'Cache-Control',
+			'ETag',
+			'Last-Modified',
+			'Content-Range',
+			'Range-Unit',
+			'Preference-Applied',
+			'Age',
+		])
+			headers.delete(header);
 		responseObject.body = serialize(problemDetail, request, responseObject);
 		return responseObject;
 	}
@@ -482,6 +592,7 @@ export function handleApplication(scope: import('../components/Scope.ts').Scope)
 					// TODO: Ideally we would like to have a 404 response before upgrading to WebSocket protocol, probably
 					return ws.close(1011, `No resource was found to handle ${request.pathname}`);
 				} else {
+					assertNoDeferredCredentialRejection(request);
 					request.handlerPath = entry.path;
 					recordAction(
 						(action) => ({