@harperfast/harper 5.3.0-beta.1 → 5.3.0-beta.2

package/resources/recordLockCoordinator.ts

@@ -239,11 +239,11 @@ export interface DelegationReply {
 	dependencies?: LockDependencySet | null;
 	/**
 	 * Denied only. `contended` is the one reason that means another node holds the key, and so the only
-	 * one a timeout may report as 423. `generation` (the two sides hold different home maps),
-	 * `unknown-node` (this node is not named in the map), `quarantine` (the home is inside its §4.3
-	 * restart interval) and `timeout` (the home never answered) all describe something other than
-	 * contention, so each ends as a retryable 503 rather than telling the caller a key nobody holds is
-	 * held.
+	 * one an exhausted wait may report as 423. `generation` (the two sides hold different home maps),
+	 * `unknown-node` (this node is not named in the map) and `quarantine` (the home is inside its §4.3
+	 * restart interval) all describe something other than contention, so each ends as a retryable 503
+	 * rather than telling the caller a key nobody holds is held. `timeout` is not a home answer at all
+	 * — it is this node's own deadline ending its probe — and never classifies a wait (DESIGN.md).
 	 */
 	reason?: 'contended' | 'generation' | 'unknown-node' | 'capacity' | 'not-home' | 'quarantine' | 'timeout';
 	/** Denied with `generation`, so a stale requester can re-derive the ring without another round trip. */
@@ -251,6 +251,13 @@ export interface DelegationReply {
 	retryAfterMs?: number;
 }
 
+/** A completed home reply, bound to the route that produced it so a ring change retires it. */
+interface LastCompletedReply {
+	reply: DelegationReply;
+	home: string;
+	generation: number;
+}
+
 export interface DelegationRequest {
 	key: any;
 	/** The asking node. Established by the transport, never read from an untrusted payload. */
@@ -1157,13 +1164,17 @@ export class LockCoordinator {
 	 * The amortization is the first branch: a live, un-recalled delegation with enough time left costs
 	 * zero cluster messages.
 	 */
-	async acquire(key: any, leaseMs: number, waitMs: number): Promise<LockRound> {
+	acquire(key: any, leaseMs: number, waitMs: number): Promise<LockRound> {
+		return this.#acquire(key, leaseMs, waitMs);
+	}
+
+	async #acquire(key: any, leaseMs: number, waitMs: number, observed?: LastCompletedReply): Promise<LockRound> {
 		// `Table.lock()` captures a coordinator and only reaches here after the native key lock, which
 		// can wait the caller's whole timeout — long enough for a transport swap to close what it
 		// captured. Authority moved to the successor rather than away, so run there instead of
 		// rejecting a caller that is already holding the key.
 		const authority = this.#authority();
-		if (authority !== this) return authority.acquire(key, leaseMs, waitMs);
+		if (authority !== this) return authority.#acquire(key, leaseMs, waitMs, observed);
 		if (this.#closed) throw new LockUnavailableError('Cluster record lock coordination was closed for this table');
 		if (!this.transport.ownsCoordination()) {
 			// Off the coordinating thread. A transport that can relay obtains the admission from the
@@ -1180,6 +1191,7 @@ export class LockCoordinator {
 		}
 		const keyId = this.#keyIdOf(key);
 		const deadlineMono = this.#monotonic() + waitMs;
+		let lastCompleted = observed;
 
 		acquisition: for (;;) {
 			const homeMap = this.transport.homeMap(this.database);
@@ -1360,25 +1372,39 @@ export class LockCoordinator {
 				// which the terminal answer below is what handles.
 				warnOnce('record lock home disagreed about the ring', { database: this.database, table: this.table });
 
+			if (reply.reason !== 'timeout') lastCompleted = { reply, home, generation: homeMap.generation };
 			const remaining = deadlineMono - this.#monotonic();
-			if (remaining <= 0) {
-				// 423 says "someone else holds this key", so only a denial that actually means that may end
-				// as one. Every other reason ran out the clock without the key ever being held, and reporting
-				// contention for it sends the caller to retry a condition no timeout can outlast.
-				if (reply.reason === 'contended') throw new ClientError('Record is locked and was not released in time', 423);
-				throw new LockUnavailableError(
-					`Could not establish a cluster record lock on ${this.database}.${this.table} within the wait: the key's home answered ${reply.reason ?? (reply.granted ? 'grants that arrived too late to use' : 'nothing usable')}`
-				);
-			}
-			await delay(Math.min(reply.retryAfterMs ?? 25, remaining)).promise;
+			const retryAfterMs = reply.retryAfterMs ?? 25;
+			// A home this node IS costs nothing to ask again — `#grantLocally` is synchronous, so a release
+			// landing in the backoff is still grantable at the deadline. A remote home is not: its request
+			// would go out with the leftover budget and could only return this node's own `timeout`.
+			const exhausted = home === this.nodeId ? remaining <= 0 : remaining <= retryAfterMs;
+			if (!exhausted) await delay(Math.min(retryAfterMs, remaining)).promise;
 			if (this.#closed) {
-				// Same swap, landing in the backoff instead. Carry the remaining wait so the deadline the
-				// caller asked for is preserved across the hop.
+				// Same swap, landing in the backoff instead. The successor inherits both the remaining wait
+				// and what this one saw, since a successor exhausted on arrival has nothing of its own.
 				const successor = this.#authority();
 				if (successor === this)
 					throw new LockUnavailableError('Cluster record lock coordination was closed for this table');
-				return successor.acquire(key, leaseMs, Math.max(0, deadlineMono - this.#monotonic()));
+				return successor.#acquire(key, leaseMs, Math.max(0, deadlineMono - this.#monotonic()), lastCompleted);
 			}
+			if (!exhausted) continue acquisition;
+			// Only an observation made under the generation that is current NOW still describes the key —
+			// this pass's reply included, since a generation can be activated while the probe that ended
+			// the wait is still in flight. Hence a fresh read rather than the copy this pass started from.
+			const currentGeneration = this.transport.homeMap(this.database)?.generation;
+			const carried =
+				lastCompleted?.home === home && lastCompleted.generation === currentGeneration
+					? lastCompleted.reply
+					: undefined;
+			const terminal = carried ?? (homeMap.generation === currentGeneration ? reply : undefined);
+			// Only `contended` may end as 423: it is the one answer that says another node holds the key,
+			// and reporting contention for anything else sends the caller to retry a condition no wait
+			// outlasts. A wait can see `contended` and still end 503 — the rule is what the home said LAST.
+			if (terminal?.reason === 'contended') throw new ClientError('Record is locked and was not released in time', 423);
+			throw new LockUnavailableError(
+				`Could not establish a cluster record lock on ${this.database}.${this.table} within the wait: ${describeExhaustedWait(terminal, currentGeneration !== undefined)}`
+			);
 		}
 	}
 
@@ -2400,8 +2426,6 @@ export class LockCoordinator {
 				requested,
 				timeout.promise.then(() => {
 					raced = true;
-					// `timeout`, not `contended`: the home never answered, so nobody was shown to hold the
-					// key. Retried the same way, but a wait that ends here is 503 rather than 423.
 					return { granted: false, reason: 'timeout', retryAfterMs: 0 } as DelegationReply;
 				}),
 			]);
@@ -2694,6 +2718,21 @@ export class LockCoordinator {
 	}
 }
 
+/**
+ * What ended an exhausted `acquire()`, for the 503 it throws. A `timeout` is not phrased as a home
+ * answer: it is this node's own deadline, and an operator told "the home answered timeout" looks for
+ * a fault on a node that was simply not waited for.
+ */
+function describeExhaustedWait(terminal: DelegationReply | undefined, hasCurrentMap: boolean): string {
+	if (terminal)
+		return terminal.reason === 'timeout'
+			? "no reply from the key's home within the wait"
+			: `the key's home answered ${terminal.reason ?? (terminal.granted ? 'grants that arrived too late to use' : 'nothing usable')}`;
+	return hasCurrentMap
+		? 'the record lock home map changed generation before the wait ended'
+		: 'this node has no current record lock home map';
+}
+
 /** A sleep whose timer the winner of a race can drop, rather than let it run out its full delay. */
 function delay(ms: number): { promise: Promise<void>; cancel: () => void } {
 	let timer: ReturnType<typeof setTimeout>;

package/resources/scheduler/DESIGN.md

@@ -0,0 +1,40 @@
+# resources/scheduler/ — Design notes
+
+The built-in scheduler plugin.
+
+**Read this when:** touching job leadership, leases or heartbeats.
+
+Index of every design note: [DESIGN.md](../../DESIGN.md).
+
+---
+
+## Scheduler: cluster-once execution without a consensus primitive (`resources/scheduler/`)
+
+The built-in `scheduler` plugin (#951) runs config-declared jobs "exactly once per cluster." The
+non-obvious part is what Harper's substrate does and does not offer for that:
+
+- **There is no election, consensus, or cross-node CAS anywhere in harper/harper-pro.** Replication
+  converges concurrent writes by record version (LWW). A lease acquired with a plain `put` therefore
+  cannot be race-free by construction — two nodes writing the lease both succeed locally and
+  converge later. The engine's design accepts this: sticky leadership (a starter defers to a fresh
+  heartbeat), a heartbeat takeover check (a leader seeing a fresher foreign heartbeat steps down),
+  and documented handler idempotency are the mechanisms that ride out transient dual-leadership.
+  Do not "fix" the race with a conditional write — the primitive does not exist.
+- The lease + per-job run state live in the replicating system table `hdb_scheduler_state`
+  (`audit: true` because system-table replication requires auditing; name chosen because `hdb_job`
+  is taken by the legacy jobs subsystem). On constrained/directional replication topologies the
+  lease may not reach every node; that limitation is inherent.
+- The alphabetical node-name tie-break deliberately mirrors replication's deterministic failover
+  convention (sorted node names in `subscriptionManager.ts`); the escalation ladder
+  (`promotionWaitMs`) exists because a dead alphabetically-first node must not deadlock a
+  leaderless cluster (each successive node waits one more `2 × watcher interval` rung).
+- Thread-once vs cluster-once are separate layers: `getWorkerIndex() === 0` gates to one worker per
+  node (correct in every threading mode incl. `threads: 0`); the lease gates across nodes.
+  `handleApplication` holds a cross-thread load lock with a 30s timeout, so the plugin only
+  registers there — election, scheduling, and catch-up run async after.
+- Catch-up fires at most ONE missed occurrence per cron job, and a new job's `firstSeenAt` baseline
+  prevents firing immediately on first deploy. Interval jobs are excluded from the sweep — they
+  self-correct by anchoring to their persisted last run in `scheduleNextRun`.
+- Timer firing is a deliberately thin `setTimeout` layer so the durable timer service (#754) can
+  replace it without touching the config surface, election, or catch-up. Timing thresholds are
+  env-overridable (`HARPER_SCHEDULER_*`) so multi-node tests can exercise failover in seconds.

package/resources/transactionBroadcast.ts

@@ -192,7 +192,7 @@ function notifyFromTransactionData(subscriptions, auditLogIterable?, allowYield
 					let ancestorLevel = 0;
 					do {
 						// we iterate through the key hierarchy, notifying all subscribers for each key,
-						// so for an id like resource/foo/bar, we notify subscribers for resource/foo/bar, resource/foo/, resource/foo, resource/, and resource
+						// so for an id like resource/foo/bar, we notify subscribers for resource/foo/bar, resource/foo/, resource/, and the root (null)
 						// this allows for efficient subscriptions to children ids/topics
 						const keySubscriptions = tableSubscriptions.get(matchingKey);
 						if (keySubscriptions) {
@@ -230,9 +230,9 @@ function notifyFromTransactionData(subscriptions, auditLogIterable?, allowYield
 						if (lastSlash !== matchingKey.length - 1) {
 							ancestorLevel++; // don't increase the ancestor level for this going from resource/ to resource
 						}
-						if (lastSlash > -1) {
-							matchingKey = matchingKey.slice(0, lastSlash + 1);
-						} else matchingKey = null;
+						// lastIndexOf clamps a negative fromIndex to 0, so '/' would otherwise yield itself forever
+						const parentKey = lastSlash > -1 ? matchingKey.slice(0, lastSlash + 1) : null;
+						matchingKey = parentKey === matchingKey ? null : parentKey;
 					} while (true);
 				}
 			} else if (auditRecord.type === 'reload') {

package/security/DESIGN.md

@@ -0,0 +1,175 @@
+# security/ — Design notes
+
+Authentication tokens, OIDC exchange and TLS material.
+
+**Read this when:** touching `tokenAuthentication.ts`, `impersonation.ts`, `authn/oidc/` or `keys.ts`.
+
+Index of every design note: [DESIGN.md](../DESIGN.md).
+
+---
+
+## OIDC trusted publishing (`security/authn/oidc/`)
+
+`exchange_oidc_token` lets a workload authenticate with no stored Harper credential (#2171): it presents an identity token minted by its runtime, and gets back a one-hour operation token for the user a stored trust policy names. It is in `NO_AUTH_OPERATIONS` because it _is_ the authentication, the same way `create_authentication_tokens` is against a password — the same three wiring points apply (`serverHandlers.js` `NO_AUTH_OPERATIONS`, the `verifyPerms` bypass in `serverUtilities.ts`, and a `permission(false, [])` registration).
+
+**The core is issuer-agnostic; everything issuer-specific lives in `providers/`.** That split is the point of the layout, not an accident of it — a new workload-identity issuer should be a profile, not a change to verification, matching, or storage.
+
+- `claims.ts` — matching and constraint _shape_ validation. Knows nothing about any issuer.
+- `jwks.ts` — issuer keys. The rate-limit clock for unknown-`kid` refetches lives _outside_ the cache entry: a successful fetch replaces the entry, and a rate limit that resets whenever it fires is not a rate limit. Keeping it separate also means a genuine key rotation is picked up on first use rather than after the window.
+- `identityToken.ts` — signature, issuer, audience, `exp`, and a bounded lifetime. Owns `rejectToken`, shared with the exchange so both halves refuse identically.
+- `tokenExchange.ts` — policy selection, replay, minting, audit. Verification is memoized per audience, so N policies sharing one cost one signature check.
+- `providers/` — `assertPolicyIsSpecific` / `assertAudienceIsSpecific` / `normalizeClaims` / `describePrincipal` / optional `vetoClaims`, resolved by normalized issuer.
+
+**An unregistered issuer gets `providers/generic.ts`, which is strict rather than permissive:** the policy must pin `sub`. That is what makes Kubernetes service accounts, GCP service accounts, and SPIFFE SVIDs work with zero provider code — each has a stable canonical subject. GitHub needs its own profile precisely because its `sub` is the one claim you should _not_ pin: it varies by trigger, and its format changed for repositories created after 2026-07-15.
+
+Four constraints that look like choices but are not:
+
+1. **Every rejection returns the same message.** The endpoint is unauthenticated; a caller told which check failed can enumerate a policy one claim at a time. Reasons go to the `oidc-trust` logger.
+2. **A GitHub policy must gate the ref.** `githubActionsProfile.assertPolicyIsSpecific` rejects a policy pinning only repository + workflow, because anyone who can push a branch could then add that workflow to it and mint a token. Stricter than npm's trusted-publishing model, which mitigates the same hole with environment protection instead — and profile-scoped, so it never constrains another issuer.
+3. **`createOperationToken`, not `createTokens`.** `createTokens` overwrites `hdb_user.refresh_token` as a side effect, so minting for CI would silently revoke whatever credential that user already held (#2018) — the exact problem this feature removes.
+4. **The role is the boundary; the per-policy `operations` allowlist only narrows it.** Least privilege is primarily the role of the user the policy names. A policy may _optionally_ carry an `operations` scope, which can only subtract from that role — never add to it. It is deliberately not merged into `permission.operations`: gate 2 in `operation_authorization.ts` treats an explicit listing of an SU-only operation as a deliberate grant, so reusing that field would _widen_ where this must only narrow. The scope is carried as a separate `tokenOperations` claim and intersected ahead of every early return, including the super_user bypass.
+
+   Its enforcement surface is the operations API and SQL (`verifyPerms` / `verifyPermsAST`) — **not** the application REST/GraphQL resource path, which authorizes through table-level `checkPermission` and does not consult the scope. A scoped token therefore still carries its role's full CRUD there, which is why the role has to be least-privilege on its own; the scope is defense in depth, not a substitute. Closing that gap is a follow-up on the same surface as CORE-3061. Because a second authorization mechanism beside roles is one more place for the two to disagree, whether to keep this at all is an open design question on #2173 rather than a settled constraint.
+
+   Naming `sql` in a scope grants the SQL interface, not unrestricted DML through it: a write statement additionally requires its matching data operation (`insert`/`update`/`delete`) in scope. That is what keeps `read_only` — which expands to include `sql` — from admitting a DELETE, given that `verifyPermsAST` returns early for a super_user before any table check runs.
+
+`hdb_oidc_token_use` (created lazily via `table()`, not the system schema) records spent tokens keyed on a SHA-256 of the token itself, with `expiresAt` past the token's own expiry. Hashed rather than stored, so the table never holds a credential; keyed on the token's **signed input** (`header.payload`) rather than `jti` because not every issuer emits one (Azure uses `uti`). Not on the whole token string: the signature segment is covered by nothing, and base64url decoding ignores the surplus low bits of its final character, so 16 distinct spellings of an RS256 signature decode to the same bytes, all verify, and all hash differently — one leaked token would buy 16 exchanges. ES\* malleability (`s → n−s`) is a second such vector. The signed input is exactly what the issuer asserted, so every variant collapses to one fingerprint. The get-then-put is not atomic and does not claim to be: a concurrent replay is not a privilege escalation, since whoever holds the token could obtain one operation token anyway.
+
+## Scoped tokens and synthetic-role identity (`security/tokenAuthentication.ts`, `security/impersonation.ts`)
+
+`create_authentication_tokens` with an inline `role` **object** mints a `sub: 'scoped-operation'`
+JWT that embeds its whole (downgraded, deep-validated) permission set; the bearer needs no
+`hdb_user`/`hdb_role` row and the `username` is attribution only. Minting is super_user-gated
+(or trusted internal dispatch via `isOperationAuthorizationBypassed()`); a string `role` keeps its
+legacy meaning (component-defined token, rejected by `validateOperationToken`). Scoped tokens get
+no refresh token, touch no user record, and are therefore **irrevocable until expiry** — expiry is
+the only control, which is why `auth.ts` evicts cached Bearer identities at exact `authExpiresAt`
+rather than waiting for the auth-cache TTL.
+
+The attribution `username` must NOT name an existing `hdb_user` (rejected at mint; the default is
+`scoped:<minter>`): code paths that rehydrate a user by name would otherwise substitute the real
+principal's permissions for the token's — or fail-closed on the non-existent name. The three known
+by-name sites are handled, all by the same `_scopedToken` short-circuit: the MQTT last-will replay
+(`DurableSubscriptionsSession.ts` persists the scoped role/marker/expiry on the will and skips
+rehydration — and both the restart-replay and the live abnormal-disconnect paths refuse to publish
+a scoped will past `authExpiresAt`), the live-subscription stale-auth recheck (`Resource.ts`
+`registerLiveSubscriptionForContext` keeps the embedded role as the identity), and the MCP
+`list_changed` session refresh (`components/mcp/listChanged.ts` `refreshSessionUser`). The scoped
+principal also cannot self-mint standing tokens: the passwordless path of `createTokens` rejects an
+`hdb_user._scopedToken` requester. **Any future by-name rehydration must check `_scopedToken`.** A
+user _created after minting_ with a colliding name is therefore inert at every current site; the
+residual is only some _new_ unguarded by-name site — another reason to prefer short expiries.
+
+Scope of the `operations` allowlist: it gates the **operations API** (including the `sql` path,
+which never reaches `verifyPerms` and calls `verifyOperationsAllowlist` directly from
+`chooseOperation`) — it does NOT gate the application/REST/GraphQL/MQTT surfaces, which authorize
+on translated table CRUD permissions only. A scoped token intended to be read-only on app
+endpoints must carry restrictive table permissions; `operations: ['read_only']` alone does not
+constrain REST writes if table perms allow them.
+
+The invariant to preserve when touching any synthetic (inline/impersonated/scoped) role:
+`permissionsTranslator.getRolePermissions` memoizes translated permissions **by role name** (keyed
+further by `__updatedtime__` + schema). A synthetic role must therefore never carry a constant
+name or a per-request timestamp — two different permission sets would alias one cache slot (a
+same-millisecond `Date.now()` was enough), leaking one principal's translated permissions to
+another. `syntheticRoleName()` derives the name from a hash of the post-downgrade permission
+content with `__updatedtime__: 0`, so identical sets share a slot and distinct sets can't collide;
+`applyImpersonation` re-keys all three impersonation modes the same way (Mode B/C previously wrote
+downgraded copies under the _persisted_ role's name). Synthetic translations live in a separate
+256-entry LRU (`syntheticRolePermsMap`), not the permanent `rolePermsMap` — so >256 concurrently
+live distinct permission sets degrade to per-request translation (a deliberate cliff; raise the
+constant if a legitimate workload hits it). The `_` name prefix is the discriminator; a persisted
+role named with a leading underscore lands in the LRU too (correct, just evictable). Relatedly,
+the role `operations` allowlist gate in `verifyPerms` must stay **ahead of** the ambient privilege
+early-returns (super_user, structure_user, system-table allowances): persisted roles can't combine
+`super_user` with other permission keys, but inline roles can combine `structure_user` with an
+allowlist, and the gate ordering is what keeps unlisted schema ops unreachable.
+
+## TLS hot-reload: cert vs. private key follow two different propagation paths (`security/keys.ts`)
+
+A renewed **certificate** and a renewed **private key** reach a worker's live TLS secure context
+by completely separate routes, and the two must reconverge or HTTPS breaks on that worker.
+Certificates propagate through data: only the main thread watches the cert file (`isMainThread`
+guard in `loadCertificates`) and writes the new PEM into the `system.hdb_certificate` table; every
+worker is subscribed to that table and rebuilds its secure contexts (`updateTLS` inside
+`createTLSSelector`) on the notification. Private keys never touch the table — each worker watches
+its own key file (the private-key `loadAndWatch` has no `isMainThread` guard) and loads the PEM
+straight into its in-thread `privateKeys` map. `getPrivateKeyByName` reads that map first, so an
+already-built secure context has the key bytes baked in (`setCert`/`setKey` at build time); a later
+map update does not touch contexts already built.
+
+The hazard when a rotation changes **both**: the cert can win the race to a worker (table write +
+subscription) and trigger `updateTLS` before that worker has reloaded the matching key, producing a
+context that pairs the new cert with the old key — every handshake on it then fails, and nothing
+rebuilds it until the _next_ cert-table change. The fix: a private-key reload (`handlePrivateKeyReload`,
+the single sink for both the chokidar watcher and PR #1394's periodic poll) triggers a debounced
+rebuild of every live selector via the module-level `liveTLSRebuilders` set, so the worker reconverges
+on its own. Subtleties to preserve: the rotation guard (`previous !== undefined && previous !== key`)
+must skip both the initial load and identical-content reloads or watchers thrash; transient one-shot
+selectors (`getReplicationCert`) pass `liveReload=false` so they don't accumulate in the never-pruned
+set; and the cert subscription shares the same debounced `scheduleRebuild` (same 1500ms cadence), so
+its coalescing must stay a superset-safe no-op for the single-swap #586 case. Regression coverage:
+`integrationTests/security/cert-key-reload.test.ts` deterministically pins the cert-before-key ordering
+(it fails by design without the rebuild trigger); `cert-reload.test.ts` guards the cert-only #586 path.
+
+**Publication is transactional (#2382).** `updateTLS` builds the entire replacement state —
+hostname→context map, CA map, and default candidate — into pass-local candidates and reconciles the
+live maps in place only after the pass completes (their identity is load-bearing:
+`server.secureContexts` and each context's `availableCAs` alias them). A record that is still in the
+table but fails to build (`ERR_OSSL_X509_KEY_VALUES_MISMATCH` when the table's cert outruns the
+on-disk key, a missing key on this thread) keeps every live entry it owns _and_ its default
+candidacy — a record can be serving as the default with no hostname entries at all — so a transient
+mismatch never downgrades serving below last-good (the pre-fix behavior served the self-signed
+default for days). Retention is trust-aware: a context froze its `ca:` list at build time, so when
+the CA set has changed since, the retained pair is rebuilt against the current trust material —
+new handshakes never see revoked client-CA trust; established sessions and outstanding session
+tickets are unaffected, exactly as on a fresh build (ticket keys are process-wide and never rotate
+on trust changes) — and if that rebuild fails the record's entries
+drop for that pass, except when nothing else is servable: the zero-certificate guard then retains
+the old state (availability outranks the drop in that corner) while the failure keeps retrying. Deleting the record remains the way to drop its contexts; a corrupt authority
+row is a pass failure like any other (reported through the signature throttle, armed for retry) and
+its trust drops until it heals. A failed pass arms a
+self-retry on the shared debounce with a per-signature backoff (1.5s doubling to 5min) and
+signature-throttled logging; external triggers (table subscription, key reload) stay at the plain
+debounce. `loadAndWatch` latches its mtime before the callback for chokidar/poll dedupe, but rolls
+the latch back on a synchronous throw or a rejected callback promise (equality-guarded so a stale
+rejection cannot unlatch a newer reload) — the latch means "last successfully applied", so the
+periodic poll can heal a lost `hdb_certificate` write instead of deduplicating it forever.
+
+## A component-facing export needs BOTH `index.ts` and `getHarperExports` (`security/jsLoader.ts`, `index.ts`)
+
+Adding `export { x } from './…'` to `index.ts` publishes `x` on the `harper` **package** but does not
+make `import { x } from 'harper'` work inside an application. A component loaded into a VM compartment
+resolves `harper` to a synthetic module built from `getHarperExports()` — a hand-maintained object
+literal, not a re-export of `index.ts` — so a value added to only one list fails at component load with
+`The requested module 'harper' does not provide an export named 'x'`. No unit test sees that, because
+unit tests `require('#src/…')` directly; only a fixture that imports from `'harper'` does
+(`integrationTests/security/fixtures/deferred-credential-rejection/resources.js`, harper#2703).
+
+Export the value from the module that _defines_ it rather than re-exporting it through an intermediate,
+so a component-created value shares that module's private symbols. `markCredentialRejection` depends on
+this: its tag is a module-private `Symbol` that `isCredentialRejection` checks by identity.
+
+## Authentication converts every principal-resolution failure into a decision (`security/auth.ts`)
+
+`authentication()` resolves a principal from three sources — an mTLS certificate CN, the `Authorization`
+header, and the `hdb_session` cookie — each through the overridable `server.getUser`. A failure from any
+of them must become a **decision**, never a throw: an internal fault (and anything on the operations API,
+where Harper owns every route) is answered in place, and a tagged credential rejection is deferred so the
+layer owning the route settles it. A throw that escapes instead unwinds the whole middleware chain to
+`server/http.ts`'s terminal handler and renders as a plain-text body, which is what harper#2703 reported.
+`settleAuthFailure()` performs the conversion; `rejectAuthenticationInPlace()` is the only way to answer
+in place, and a new principal-resolving path must route through them.
+
+Two consequences that are easy to miss:
+
+- **A returned 401 is invisible to a WebSocket or MQTT upgrade.** `server/REST.ts` and `server/mqtt.ts`
+  only `await chainCompletion` and discard its resolved value, so an in-place decision has to be recorded
+  on the request (`markAuthenticationRejectedInPlace`) for `assertNoDeferredCredentialRejection` — which
+  both already call — to fail the upgrade closed. Converting a throw into a returned descriptor without
+  that record turns a fail-closed upgrade into one that proceeds with no principal. The certificate
+  revocation exit is the case that was missed first.
+- **A deferred rejection outranks a later success.** Route owners call
+  `settleDeferredCredentialRejection` _before_ they read `request.user`, so once a credential is deferred,
+  resolving a principal from a different credential is a contradiction. Hence a rejected certificate
+  identity stops resolution outright instead of falling through to Basic, the session, or the local bypass.

package/security/auth.ts

@@ -18,6 +18,7 @@ import {
 	deferCredentialRejection,
 	getDeferredCredentialRejection,
 	isCredentialRejection,
+	markAuthenticationRejectedInPlace,
 } from './deferredAuthentication.ts';
 import { serializeMessage } from '../server/serverHelpers/contentTypes.ts';
 import { hdbErrors } from '../utility/errors/hdbError.ts';
@@ -69,6 +70,27 @@ export function bypassAuth() {
 	bypassUser = { username: 'bypass', role: { role: 'super_user', permission: { super_user: true } } };
 }
 
+/**
+ * Turns a failed principal resolution into a decision, so it never escapes the chain as a throw.
+ * Returns a response descriptor for the caller to pass through `applyResponseHeaders`, or
+ * `undefined` once the rejection has been deferred for the route owner to settle (#2703).
+ */
+function settleAuthFailure(request, error, strategy: string): { status: number; body: any } | undefined {
+	const internalFault = !isCredentialRejection(error);
+	if (request.isOperationsServer || internalFault) {
+		if (internalFault) authLogger.error('Authentication failed internally', errorForLog(error));
+		const message = internalFault ? AUTHENTICATION_ERROR_MSGS.GENERIC_AUTH_FAIL : error.message;
+		return rejectAuthenticationInPlace(request, message);
+	}
+	deferCredentialRejection(request, error, strategy);
+	return undefined;
+}
+
+function rejectAuthenticationInPlace(request, message: string): { status: number; body: any } {
+	markAuthenticationRejectedInPlace(request, 401, message);
+	return { status: 401, body: serializeMessage({ error: message }, request) };
+}
+
 // TODO: Make this not return a promise if it can be fulfilled synchronously (from cache)
 export async function authentication(request, nextHandler) {
 	const headers = request.headers.asObject; // we cheat and use the node headers object since it is a little faster
@@ -156,6 +178,7 @@ export async function authentication(request, nextHandler) {
 		)
 			authEventLog.error?.('Authorization error:', request._nodeRequest.socket.authorizationError);
 
+		let mtlsCredentialDeferred = false;
 		if (request.mtlsConfig && request.authorized && request.peerCertificate.subject) {
 			const verificationResult = await verifyCertificate(request.peerCertificate, request.mtlsConfig);
 			if (!verificationResult.valid) {
@@ -165,10 +188,7 @@ export async function authentication(request, nextHandler) {
 					'for',
 					request.peerCertificate.subject.CN
 				);
-				return applyResponseHeaders({
-					status: 401,
-					body: serializeMessage({ error: 'Certificate revoked or verification failed' }, request),
-				});
+				return applyResponseHeaders(rejectAuthenticationInPlace(request, 'Certificate revoked or verification failed'));
 			}
 
 			// Alternative behavior: Instead of returning 401 above, we could just not set the user
@@ -183,8 +203,19 @@ export async function authentication(request, nextHandler) {
 				// null means no user is defined from certificate, need regular authentication as well
 				if (username === undefined || username === 'Common Name' || username === 'CN')
 					username = request.peerCertificate.subject.CN;
-				request.user = await server.getUser(username, null, request);
-				authAuditLog(username, AUTH_AUDIT_STATUS.SUCCESS, 'mTLS');
+				let mtlsUser;
+				try {
+					mtlsUser = await server.getUser(username, null, request);
+				} catch (error) {
+					if (LOG_AUTH_FAILED) authAuditLog(username, AUTH_AUDIT_STATUS.FAILURE, 'mTLS');
+					const failureResponse = settleAuthFailure(request, error, 'mTLS');
+					if (failureResponse) return applyResponseHeaders(failureResponse);
+					mtlsCredentialDeferred = true;
+				}
+				if (!mtlsCredentialDeferred) {
+					request.user = mtlsUser;
+					authAuditLog(username, AUTH_AUDIT_STATUS.SUCCESS, 'mTLS');
+				}
 			} else {
 				debug('HTTPS/WSS mTLS authorized connection (mTLS did not authorize a user)', 'from', request.ip);
 			}
@@ -193,6 +224,9 @@ export async function authentication(request, nextHandler) {
 		let newUser;
 		if (request.user) {
 			// already authenticated
+		} else if (mtlsCredentialDeferred) {
+			// a principal resolved from another credential here would be contradicted by the deferred
+			// certificate rejection downstream, so the certificate's decision stands alone
 		} else if (authorization) {
 			let cachedUser = authorizationCache.get(authorization);
 			// A cached Bearer identity must not outlive its token: expiry is the only revocation
@@ -214,7 +248,7 @@ export async function authentication(request, nextHandler) {
 				const strategy = authorization.slice(0, spaceIndex);
 				const credentials = authorization.slice(spaceIndex + 1);
 				let username, password;
-				let credentialRejection;
+				let credentialDeferred = false;
 				try {
 					switch (strategy) {
 						case 'Basic':
@@ -269,23 +303,12 @@ export async function authentication(request, nextHandler) {
 						}
 					}
 
-					const internalFault = !isCredentialRejection(err);
-					if (request.isOperationsServer || internalFault) {
-						if (internalFault) authLogger.error('Authentication failed internally', errorForLog(err));
-						return applyResponseHeaders({
-							status: 401,
-							body: serializeMessage(
-								{ error: internalFault ? AUTHENTICATION_ERROR_MSGS.GENERIC_AUTH_FAIL : err.message },
-								request
-							),
-						});
-					}
-					credentialRejection = err;
+					const failureResponse = settleAuthFailure(request, err, strategy);
+					if (failureResponse) return applyResponseHeaders(failureResponse);
+					credentialDeferred = true;
 				}
 
-				if (credentialRejection) {
-					deferCredentialRejection(request, credentialRejection, strategy);
-				} else {
+				if (!credentialDeferred) {
 					authorizationCache.set(authorization, newUser);
 					if (LOG_AUTH_SUCCESSFUL && newUser != null)
 						authAuditLog(newUser.username, AUTH_AUDIT_STATUS.SUCCESS, strategy);
@@ -299,8 +322,14 @@ export async function authentication(request, nextHandler) {
 
 			request.user = newUser;
 		} else if (session?.user) {
-			// or should this be cached in the session?
-			request.user = await server.getUser(session.user, null, request);
+			try {
+				// or should this be cached in the session?
+				request.user = await server.getUser(session.user, null, request);
+			} catch (error) {
+				if (LOG_AUTH_FAILED) authAuditLog(session.user, AUTH_AUDIT_STATUS.FAILURE, 'Session');
+				const failureResponse = settleAuthFailure(request, error, 'Session');
+				if (failureResponse) return applyResponseHeaders(failureResponse);
+			}
 		} else if (
 			(AUTHORIZE_LOCAL && bypassUser) || // explicit bypass (test mode); also covers ::ffff:127.x addresses
 			(AUTHORIZE_LOCAL && (request.ip?.includes('127.0.0.') || request.ip == '::1')) ||

package/security/deferredAuthentication.ts

@@ -65,7 +65,29 @@ export function settleDeferredCredentialRejection(
 	};
 }
 
+const REJECTED_IN_PLACE = Symbol('harper.authenticationRejectedInPlace');
+
+/**
+ * Records an authentication decision the middleware answered in place, with the client-safe message
+ * the HTTP response carries. A WebSocket or MQTT upgrade only awaits the chain and never reads its
+ * resolved value, so a returned 401 is invisible to it; without this record such an upgrade would
+ * continue with no principal on a fault that failed the equivalent HTTP request closed.
+ */
+export function markAuthenticationRejectedInPlace(request: any, status: number, message: string): void {
+	if (request?.[REJECTED_IN_PLACE]) return;
+	Object.defineProperty(request, REJECTED_IN_PLACE, {
+		value: Object.freeze({ status, message }),
+		enumerable: false,
+		configurable: false,
+		writable: false,
+	});
+}
+
+export function getAuthenticationRejectedInPlace(request: any): { status: number; message: string } | undefined {
+	return request?.[REJECTED_IN_PLACE];
+}
+
 export function assertNoDeferredCredentialRejection(request: any): void {
-	const deferred = getDeferredCredentialRejection(request);
-	if (deferred) throw new ClientError(deferred.message, deferred.status);
+	const rejected = getDeferredCredentialRejection(request) ?? getAuthenticationRejectedInPlace(request);
+	if (rejected) throw new ClientError(rejected.message, rejected.status);
 }

package/security/jsLoader.ts

@@ -18,6 +18,7 @@ import * as child_process from 'node:child_process';
 import { CONFIG_PARAMS, DEFAULT_DATABASE_NAME } from '../utility/hdbTerms.ts';
 
 import { contentTypes } from '../server/serverHelpers/contentTypes.ts';
+import { markCredentialRejection, credentialRejectionError } from './credentialRejection.ts';
 import type {} from 'ses';
 import {
 	existsSync,
@@ -946,6 +947,8 @@ function getHarperExports(scope: ApplicationScope) {
 		authenticateUser: server.authenticateUser,
 		operation: server.operation,
 		contentTypes,
+		markCredentialRejection,
+		credentialRejectionError,
 		Attribute: undefined,
 		Config: undefined,
 		ConfigValue: undefined,