@harperfast/harper 5.3.0-beta.4 → 5.3.0

package/resources/transactionBroadcast.ts

@@ -84,6 +84,8 @@ export function addSubscription(table, key, listener?: (key) => any, startTime?:
 		tableSubscriptions.set(key, (subscriptions = [subscription]));
 		subscriptions.tables = tableSubscriptions;
 		subscriptions.key = key;
+		subscriptions.traversals = 0;
+		subscriptions.hasEnded = false;
 	}
 	subscription.subscriptions = subscriptions;
 	databaseSubscriptions.activeCount = (databaseSubscriptions.activeCount || 0) + 1;
@@ -107,23 +109,18 @@ class Subscription extends IterableEventQueue {
 		this.on('close', () => this.end());
 	}
 	end() {
-		// cleanup
-		if (this.subscriptions) {
-			const tableSubscriptions = this.subscriptions.tables;
-			const envSubscriptions = tableSubscriptions?.envs;
-			this.subscriptions.splice(this.subscriptions.indexOf(this), 1);
-			if (this.subscriptions.length === 0) {
-				if (tableSubscriptions) {
-					// TODO: Handle cleanup of wildcard
-					const key = this.subscriptions.key;
-					tableSubscriptions.delete(key);
-					if (tableSubscriptions.size === 0) {
-						delete envSubscriptions[tableSubscriptions.tableId];
-					}
-				}
-			}
-			if (envSubscriptions?.activeCount > 0) envSubscriptions.activeCount--;
+		const subscriptions = this.subscriptions;
+		if (subscriptions) {
 			this.subscriptions = null;
+			const envSubscriptions = subscriptions.tables?.envs;
+			if (envSubscriptions?.activeCount > 0) envSubscriptions.activeCount--;
+			// splicing would shift the next subscriber past a loop walking this array, which compacts it when it finishes
+			if (subscriptions.traversals > 0) subscriptions.hasEnded = true;
+			else {
+				const index = subscriptions.indexOf(this);
+				if (index > -1) subscriptions.splice(index, 1);
+				if (subscriptions.length === 0) detachKeySubscriptions(subscriptions);
+			}
 		}
 		this.close();
 	}
@@ -131,6 +128,26 @@ class Subscription extends IterableEventQueue {
 		return { name: 'subscription' };
 	}
 }
+function endTraversal(keySubscriptions) {
+	if (--keySubscriptions.traversals === 0 && keySubscriptions.hasEnded) {
+		keySubscriptions.hasEnded = false;
+		let kept = 0;
+		for (let i = 0; i < keySubscriptions.length; i++) {
+			const subscription = keySubscriptions[i];
+			if (subscription.subscriptions) keySubscriptions[kept++] = subscription;
+		}
+		keySubscriptions.length = kept;
+		if (kept === 0) detachKeySubscriptions(keySubscriptions);
+	}
+}
+function detachKeySubscriptions(keySubscriptions) {
+	const tableSubscriptions = keySubscriptions.tables;
+	if (tableSubscriptions) {
+		// TODO: Handle cleanup of wildcard
+		tableSubscriptions.delete(keySubscriptions.key);
+		if (tableSubscriptions.size === 0) delete tableSubscriptions.envs[tableSubscriptions.tableId];
+	}
+}
 const ACTIONS_OF_INTEREST = ['put', 'patch', 'delete', 'message', 'invalidate'];
 // Maximum audit records processed per synchronous turn before yielding back to the event loop.
 // Sized to keep per-batch wall time within a few ms on commodity hardware while keeping the
@@ -196,33 +213,41 @@ function notifyFromTransactionData(subscriptions, auditLogIterable?, allowYield
 						// this allows for efficient subscriptions to children ids/topics
 						const keySubscriptions = tableSubscriptions.get(matchingKey);
 						if (keySubscriptions) {
-							for (const subscription of keySubscriptions) {
-								if (
-									ancestorLevel > 0 && // only ancestors if the subscription is for ancestors (and apply onlyChildren filtering as necessary)
-									!(subscription.includeDescendants && !(subscription.onlyChildren && ancestorLevel > 1))
-								)
-									continue;
-								if (subscription.startTime >= timestamp) {
-									continue;
-								}
-								try {
-									let beginTxn;
-									if (subscription.supportsTransactions && subscription.txnInProgress !== txnKey) {
-										// if the subscriber supports transactions, we mark this as the beginning of a new transaction
-										// tracking the subscription so that we can delimit the transaction on next transaction
-										// (with a beginTxn flag, which may be on an endTxn event)
-										beginTxn = true;
-										if (!subscription.txnInProgress) {
-											// if first txn for subscriber of this cycle, add to the transactional subscribers that we are tracking
-											if (!subscribersWithTxns) subscribersWithTxns = [subscription];
-											else subscribersWithTxns.push(subscription);
+							keySubscriptions.traversals++;
+							try {
+								// a subscriber added during this walk starts with the next record
+								for (let i = 0, length = keySubscriptions.length; i < length; i++) {
+									const subscription = keySubscriptions[i];
+									if (!subscription.subscriptions) continue;
+									if (
+										ancestorLevel > 0 && // only ancestors if the subscription is for ancestors (and apply onlyChildren filtering as necessary)
+										!(subscription.includeDescendants && !(subscription.onlyChildren && ancestorLevel > 1))
+									)
+										continue;
+									if (subscription.startTime >= timestamp) {
+										continue;
+									}
+									try {
+										let beginTxn;
+										if (subscription.supportsTransactions && subscription.txnInProgress !== txnKey) {
+											// if the subscriber supports transactions, we mark this as the beginning of a new transaction
+											// tracking the subscription so that we can delimit the transaction on next transaction
+											// (with a beginTxn flag, which may be on an endTxn event)
+											beginTxn = true;
+											if (!subscription.txnInProgress) {
+												// if first txn for subscriber of this cycle, add to the transactional subscribers that we are tracking
+												if (!subscribersWithTxns) subscribersWithTxns = [subscription];
+												else subscribersWithTxns.push(subscription);
+											}
+											subscription.txnInProgress = txnKey;
 										}
-										subscription.txnInProgress = txnKey;
+										subscription.listener(recordId, auditRecord, timestamp, beginTxn);
+									} catch (error) {
+										warn(error);
 									}
-									subscription.listener(recordId, auditRecord, timestamp, beginTxn);
-								} catch (error) {
-									warn(error);
 								}
+							} finally {
+								endTraversal(keySubscriptions);
 							}
 						}
 						if (matchingKey == null) break;
@@ -242,14 +267,23 @@ function notifyFromTransactionData(subscriptions, auditLogIterable?, allowYield
 				// the bulk-reloaded table. hdb_nodes peer discovery and hdb_certificate CA install rely on this.
 				const tableSubscriptions = subscriptions[auditRecord.tableId];
 				if (tableSubscriptions) {
+					// keys added during the walk are appended, so this cap keeps listeners from extending it
+					let remainingKeys = tableSubscriptions.size;
 					for (const keySubscriptions of tableSubscriptions.values()) {
-						for (const subscription of keySubscriptions) {
-							if (subscription.startTime >= timestamp) continue;
-							try {
-								subscription.listener(null, auditRecord, timestamp, false);
-							} catch (error) {
-								warn(error);
+						if (remainingKeys-- === 0) break;
+						keySubscriptions.traversals++;
+						try {
+							for (let i = 0, length = keySubscriptions.length; i < length; i++) {
+								const subscription = keySubscriptions[i];
+								if (!subscription.subscriptions || subscription.startTime >= timestamp) continue;
+								try {
+									subscription.listener(null, auditRecord, timestamp, false);
+								} catch (error) {
+									warn(error);
+								}
 							}
+						} finally {
+							endTraversal(keySubscriptions);
 						}
 					}
 				}
@@ -272,7 +306,12 @@ function notifyFromTransactionData(subscriptions, auditLogIterable?, allowYield
 			// any subscribers with open transactions need to have an event to indicate that their transaction has been ended
 			for (const subscription of subscribersWithTxns) {
 				subscription.txnInProgress = null; // clean up
-				subscription.listener(null, { type: 'end_txn' }, subscriptions.lastTxnTime, true);
+				if (!subscription.subscriptions) continue;
+				try {
+					subscription.listener(null, { type: 'end_txn' }, subscriptions.lastTxnTime, true);
+				} catch (error) {
+					warn(error);
+				}
 			}
 		}
 	} finally {

package/resources/typeParity.ts

@@ -0,0 +1,20 @@
+// Exact identity rather than mutual assignability, which lets `any`, dropped overloads and
+// bivariant method parameters pass unnoticed.
+export type ExactlyEqual<A, B> = (<T>() => T extends A ? 1 : 2) extends <T>() => T extends B ? 1 : 2 ? true : false;
+
+export type MemberDrift<Actual, Declared> =
+	| Exclude<keyof Actual, keyof Declared>
+	| Exclude<keyof Declared, keyof Actual>
+	| {
+			[Key in keyof Actual & keyof Declared]: ExactlyEqual<Pick<Actual, Key>, Pick<Declared, Key>> extends true
+				? never
+				: Key;
+	  }[keyof Actual & keyof Declared];
+
+export type AssertNoDrift<Drift extends never> = Drift;
+
+export type AssertTrue<Condition extends true> = Condition;
+
+export interface ParitySentinel {
+	readonly paritySentinel: true;
+}

package/schema.graphql

@@ -222,7 +222,14 @@ A source attribute included in a full-text index.
 input FullTextSource {
 	name: String!
 	weight: Float! = 1.0
+	"""
+	Enable highlighting for this source. At least one source must opt in when @fullText.highlighting is set.
+	"""
 	highlight: Boolean
+	"""
+	Required for Blob sources. Only "text/plain" is supported.
+	"""
+	mediaType: String
 }
 
 """
@@ -234,7 +241,8 @@ input FullTextSynonymRule {
 }
 
 """
-Opt-in highlighting limits. Omitting highlighting from @fullText disables it.
+Opt-in highlighting limits. Omitting highlighting from @fullText disables it. At least one source
+must set highlight: true when these limits are present.
 """
 input FullTextHighlighting {
 	maxFragments: Int! = 3
@@ -242,18 +250,23 @@ input FullTextHighlighting {
 }
 
 """
-Create a native full-text derived index over the listed source attributes.
+A query-only field naming a full-text index. It has no stored or selectable record value.
+"""
+scalar FullText
+
+"""
+Create a native full-text derived index over the listed source attributes. Declare it on a
+FullText field; the field name identifies the index.
 """
 directive @fullText(
-	name: String!
 	fields: [FullTextSource!]!
-	analyzer: String! = "english@1"
+	analyzer: String! = "english@2"
 	stopWords: Boolean! = true
 	positions: Boolean! = true
 	surfaceTerms: Boolean! = true
 	synonyms: [FullTextSynonymRule!]! = []
 	highlighting: FullTextHighlighting
-) repeatable on OBJECT
+) on FIELD_DEFINITION
 
 """
 Define a derived field whose value is computed from other fields.

package/server/DESIGN.md

@@ -575,6 +575,8 @@ port so the gap is visible instead of silent.
 
 `broadcastWithAcknowledgement` already times out (30 s) on a worker whose port stays open but never acks, and that shape is almost always a blocked event loop — a native lock, a runaway synchronous call — which nothing inside the worker can report (harper-pro#788: a restarted node's single http worker went byte-silent while main kept serving `cluster_status`, and the app log only said "not acknowledged by worker thread(s) 2"). So each worker posts its Linux thread id (`readlink /proc/thread-self`) to main once at startup, before anything else runs on it, and the timeout branch reads that thread's kernel state from `/proc/self/task/<tid>`: state, `wchan`, the syscall number (the first token only — the rest of that file is argument registers and stack/instruction pointers), CPU ticks, and context-switch counts, plus two cross-platform signals main already has, `worker.performance.eventLoopUtilization()` and the age of the last 1 s resource report. It samples again a second later and logs the deltas: no CPU ticks, no context switches and `event loop active +1000ms` is "parked on a lock"; ticks climbing with state `R` is "spinning". It is deliberately main-thread-only and best-effort: `workers` and the tid live on the main thread's `Worker` objects, every `/proc` field is reported individually (a hardened container may deny `wchan`/`syscall` while `stat` stays readable), a follow-up sample whose `starttime` differs from the first is discarded (the tid may have been recycled), one diagnostic runs per worker with a 30 s cooldown so concurrent timeouts on the same worker don't multiply reads, and nothing here runs when acks arrive on time. It does not name the lock owner; that still needs a native stack from the next occurrence.
 
+That diagnostic is reactive — it only fires once something else (an ITC ack) has already timed out. `sampleWorkerELU`, run from the same 1 s monitoring tick that already computes `recentELU`, is the proactive counterpart: it warns as soon as a worker's `eventLoopUtilization()` stays `>= 0.99` for `PINNED_ELU_SUSTAINED_TICKS` (30) consecutive ticks, and warns once more on recovery. State (`pinnedELUTicks`, `pinnedELUWarned`) lives on the `Worker` object, so a replaced worker starts a clean streak, and a single tick below threshold resets it — no partial credit across a dip. The very first sample after a worker (re)starts is skipped: `worker.performance.eventLoopUtilization()` called with no prior baseline returns the worker's whole-lifetime total, not a 1 s delta, so counting it could misattribute pre-tick startup work as a pinned tick. That baseline check requires a nonzero `idle` or `active` on the prior sample, not just a prior sample existing: before a worker's loop is online, `eventLoopUtilization()` returns a truthy-but-empty `{ idle: 0, active: 0 }`, and treating that as a real baseline would let the following tick's delta span the same whole-lifetime window. Bun has no `eventLoopUtilization()` and reports a placeholder `{ utilization: 0 }` instead (see `sampleWorkerELU`'s Bun branch), so this warning is Node-only. A worker tearing down can briefly report `idle < 0`, putting `utilization` outside `[0, 1]`; that tick is treated as unmeasured — neither counted nor a streak reset — rather than logging an absurd percentage or ending an episode early. Both log lines name the worker by thread id plus `name`/`application` (e.g. a job or isolated-application worker), since a legitimately saturated worker looks identical to a wedged one from ELU alone; this warns on saturation regardless of cause; distinguishing "busy" from "stuck" is left to the operator.
+
 ## A worker's `parentPort` close is not the main thread's exit (`server/threads/manageThreads.js`)
 
 `REMOVE_PORT {threadId}` and the `onThreadExit` listeners (database-drop owner release, deploy lifecycle, registered operations, log rotation) act on a thread's death. A sibling port's `close` is taken as the far thread's exit. A worker's `parentPort` carries `threadId = 0` so lookups can route to main, but its `close` means this worker is leaving: main's exit ends the process. `addPort`'s close listener therefore passes no dead thread id for `parentPort`. Announcing one would make every sibling fail its pending acks to main, drop its own `parentPort`, and fire exit listeners for main. Regression cover: `unitTests/server/threads/stuckWorkerDiagnostics.test.js`, "does not report the main thread as exited to a sibling of a worker that closes its parentPort".
@@ -662,3 +664,19 @@ gate is per instance on purpose: a shared one let a cache-fill shed silence the
 Reproduce with a large concurrent non-GET burst on one CPU-starved worker
 (`integrationTests/resources/sourcedfrom-eav-cache-coherence.test.ts` P1 under Bun pinned to a contended
 core).
+
+## Response compression dispatches on the serialized output, with one brotli policy (`server/serverHelpers/contentTypes.ts`)
+
+With `http.compressionThreshold` non-zero (the shipped default is 0, off) and `Accept-Encoding: br`, `serialize()` compresses. It chooses between `serialize` and `serializeStream` by the _input_ (iterable or not), but it chooses how to compress by what the serializer _returned_: a `serializeStream` handler may return a complete body (the built-in msgpack handler returns `pack(data)` for a plain array; third-party `server.contentTypes` handlers may return strings or generators). A string or byte body takes the single-buffer path, where the threshold applies; a Readable is compressed as-is; any other iterable is adapted with `Readable.from`, as `http.ts` does for uncompressed iterable bodies. Piping a non-stream failed the request (#2421).
+
+The compressor is attached with `stream.pipeline`, not `.pipe()`: `.pipe()` does not forward a source error, so a source failing mid-stream left the compressor open and the response hanging. With `pipeline` the failure destroys the compressor, which the transport reports and closes abruptly (next section).
+
+Both compressor call sites take their parameters from `brotliOptions(contentType)`. Node's brotli default is quality 11, which compresses at about 1–2 MB/s — seconds of libuv-pool CPU for a multi-megabyte response — while quality 2 runs at 400–600 MB/s for a third to a half more bytes than 11 on JSON. A second call site with its own parameters is how the single-buffer path came to run at quality 11 while the stream path ran at 2; keep one definition. The operations API's Fastify compression (`operationsServer.ts`) is configured separately.
+
+## A streamed response is completed only when its source ends cleanly (`server/http.ts`, `server/serverHelpers/uwsServer.ts`)
+
+A response stream that fails after its first byte has no status left to report the failure with. The only signal HTTP/1.1 has is an incomplete message: the connection closes without the chunked terminator. So each transport sink ends the response for a source `'end'` and aborts the connection for anything else — a source `'error'`, or a `'close'` with no `'end'`. Ending cleanly instead hands the client a truncated body framed as a complete 200; formats that truncate at a record boundary (CSV, NDJSON, a msgpack sequence) then parse as a valid shorter result.
+
+Node gets this from `stream.pipeline` in `pipeBodyToResponse`. uWS has no Writable to pipe into, so `streamResponse` wires the events itself and calls `res.close()`, which also runs `onAborted` and aborts the request signal. It also checks `source.destroyed`: the uWS handler resolves asynchronously, so a source can be destroyed, and its `'close'` already emitted, before the sink listens for it — the response would stay open forever. The `'error'` listener goes on before that check, because `destroy(error)` emits on a later tick and an `'error'` with no listener is thrown. Under Bun the main port is a `node:http` server and takes the `pipeBodyToResponse` path; the `Bun.serve` fetch handler buffers a stream body before it responds, so a failed source there is an ordinary error response.
+
+This is the transport's fallback, not the error contract. SSE and NDJSON GETs hold their first step so an immediate failure is still an HTTP status, and they report later failures in-band ([Streaming startup errors](#streaming-startup-errors)). Every other format commits when the handler returns the stream, so a failure before its first chunk aborts the connection too. Pinned by `unitTests/server/serverHelpers/uwsServer.test.js` ("a response stream that does not end cleanly") and, end to end on every runtime leg, by `integrationTests/server/response-compression.test.ts`.

package/server/REST.ts

@@ -620,7 +620,7 @@ export function handleApplication(scope: import('../components/Scope.ts').Scope)
 					let result;
 					while (!(result = await iterator.next()).done) {
 						const messageBinary = await serializeMessage(result.value, request);
-						ws.send(messageBinary);
+						ws.send(messageBinary as string | Buffer<ArrayBuffer>);
 						recordAction(messageBinary.length, 'bytes-sent', request.handlerPath, 'message', 'ws');
 						if ((ws as any)._socket.writableNeedDrain) {
 							await new Promise((resolve) => (ws as any)._socket.once('drain', resolve));

package/server/itc/serverHandlers.js

@@ -6,6 +6,10 @@ const hdbTerms = require('../../utility/hdbTerms.ts');
 const cleanLmdbMap =
 	require('../../utility/lmdb/cleanLMDBMap.ts').default || require('../../utility/lmdb/cleanLMDBMap.ts');
 const { validateEvent } = require('../threads/itc.js');
+const {
+	FULL_TEXT_QUERY_PAUSE_OPERATION,
+	FULL_TEXT_QUERY_RESUME_OPERATION,
+} = require('../../resources/indexes/fullTextQueryProtocol.ts');
 const { isMainThread, threadId, workerData } = require('node:worker_threads');
 const {
 	databases,
@@ -16,6 +20,7 @@ const {
 	reloadBranchAt,
 	markDropInProgress,
 } = require('../../resources/databases.ts');
+const { blockBlobSavesForRestore, resumeBlobSavesAfterRestore } = require('../../resources/blob.ts');
 
 /**
  * This object/functions are passed to the ITC client instance and dynamically added as event handlers.
@@ -46,6 +51,33 @@ async function schemaHandler(event) {
 		hdbLogger.error(validate);
 		return;
 	}
+	if (
+		event.message?.operation === FULL_TEXT_QUERY_PAUSE_OPERATION ||
+		event.message?.operation === FULL_TEXT_QUERY_RESUME_OPERATION
+	) {
+		if (typeof event.message.path !== 'string' || event.message.path.length === 0)
+			throw new Error('Full-text query reader coordination requires an index path');
+		if (typeof event.message.readinessId !== 'string' || event.message.readinessId.length === 0)
+			throw new Error('Full-text query reader coordination requires a readiness id');
+		if (typeof event.message.ownerEpoch !== 'string' || !/^(?:0|[1-9]\d*)$/.test(event.message.ownerEpoch))
+			throw new Error('Full-text query reader coordination requires an owner epoch');
+		if (
+			event.message.allowUnregisteredReadiness !== undefined &&
+			typeof event.message.allowUnregisteredReadiness !== 'boolean'
+		)
+			throw new Error('Full-text query reader coordination requires a boolean unregistered-readiness flag');
+		const fullTextQueries = require('../../resources/indexes/fullTextQueryIndex.ts');
+		const ownerEpoch = BigInt(event.message.ownerEpoch);
+		if (event.message.operation === FULL_TEXT_QUERY_PAUSE_OPERATION)
+			await fullTextQueries.pauseNativeFullTextQueryReaders(
+				event.message.path,
+				event.message.readinessId,
+				ownerEpoch,
+				event.message.allowUnregisteredReadiness === true
+			);
+		else fullTextQueries.resumeNativeFullTextQueryReaders(event.message.path, event.message.readinessId, ownerEpoch);
+		return;
+	}
 
 	hdbLogger.trace(`ITC schemaHandler received schema event:`, event);
 	if (event.message?.operation === hdbTerms.OPERATIONS_ENUM.DROP_SCHEMA && event.message.schema) {
@@ -71,17 +103,38 @@ async function schemaHandler(event) {
 	// restore_backup: this thread must release its store handles so the restore can purge and
 	// rewrite the database directory. The rescan below (resetDatabases) skips reloading it while
 	// the restoring marker is present, and reloads it on the completion signal (marker gone).
+	let resumeBlobSavesFor;
 	if (event.message?.operation === hdbTerms.OPERATIONS_ENUM.RESTORE_BACKUP && event.message.schema) {
-		try {
-			await closeDatabase(event.message.schema);
-		} catch (error) {
-			// Let the originator's process-wide closure check fail the restore immediately instead of
-			// withholding this worker's acknowledgement until the broadcast timeout.
-			hdbLogger.error(`Could not release database '${event.message.schema}' for restore`, error);
+		if (event.message.restorePhase === 'reload') {
+			resumeBlobSavesFor = {
+				database: event.message.schema,
+				token: event.message.restoreToken,
+				generationReplaced: event.message.generationReplaced !== false,
+			};
+		} else {
+			// Stop and drain blob writes before releasing the handles: closing a database is not a
+			// write barrier on its own, because a save's file pipeline outlives the handle it started from.
+			await blockBlobSavesForRestore(event.message.schema, event.message.restoreToken);
+			try {
+				await closeDatabase(event.message.schema);
+			} catch (error) {
+				// Let the originator's process-wide closure check fail the restore immediately instead of
+				// withholding this worker's acknowledgement until the broadcast timeout.
+				hdbLogger.error(`Could not release database '${event.message.schema}' for restore`, error);
+			}
 		}
 	}
-	await cleanLmdbMap(event.message);
-	await syncSchemaMetadata(event.message);
+	try {
+		await cleanLmdbMap(event.message);
+		await syncSchemaMetadata(event.message);
+	} finally {
+		if (resumeBlobSavesFor)
+			resumeBlobSavesAfterRestore(
+				resumeBlobSavesFor.database,
+				resumeBlobSavesFor.token,
+				resumeBlobSavesFor.generationReplaced
+			);
+	}
 	for (let listener of schemaListeners) {
 		try {
 			listener(event?.message);

package/server/serverHelpers/contentTypes.ts

@@ -414,8 +414,15 @@ export function findBestSerializer(incomingMessage) {
 	return { serializer: bestSerializer, type: bestType, parameters: bestParameters };
 }
 
-// about an average TCP packet size (if headers included)
 const COMPRESSION_THRESHOLD = envMgr.get(CONFIG_PARAMS.HTTP_COMPRESSIONTHRESHOLD);
+const brotliParams = (mode: number) => ({
+	params: { [constants.BROTLI_PARAM_MODE]: mode, [constants.BROTLI_PARAM_QUALITY]: 2 },
+});
+const BROTLI_TEXT_OPTIONS = brotliParams(constants.BROTLI_MODE_TEXT);
+const BROTLI_GENERIC_OPTIONS = brotliParams(constants.BROTLI_MODE_GENERIC);
+export function brotliOptions(contentType: string) {
+	return contentType.includes('json') || contentType.includes('text') ? BROTLI_TEXT_OPTIONS : BROTLI_GENERIC_OPTIONS;
+}
 /**
  * Serialize a response
  * @param responseData
@@ -428,24 +435,28 @@ export function serialize(responseData, request, responseObject) {
 	//  client itself actually (just) supports gzip/deflate
 	let canCompress = COMPRESSION_THRESHOLD && request.headers.asObject?.['accept-encoding']?.includes('br');
 	let responseBody;
+	let contentType: string;
 	if (responseData?.contentType != null && responseData.data != null) {
 		// we use this as a special marker for blobs of data that are explicitly one content type
-		responseObject.headers.set('Content-Type', responseData.contentType);
+		contentType = responseData.contentType;
+		responseObject.headers.set('Content-Type', contentType);
 		responseObject.headers.set('Vary', 'Accept-Encoding');
 		responseBody = responseData.data;
 	} else if (responseData instanceof Uint8Array || responseData instanceof Blob) {
 		// If a user function or property returns a direct Buffer of binary data, this is the most appropriate content
 		// type for it.
-		responseObject.headers.set('Content-Type', 'application/octet-stream');
+		contentType = 'application/octet-stream';
+		responseObject.headers.set('Content-Type', contentType);
 		responseObject.headers.set('Vary', 'Accept-Encoding');
 		responseBody = responseData;
 	} else {
 		const serializer = findBestSerializer(request);
 		if (serializer.serializer.compressible === false) canCompress = false;
+		contentType = serializer.type;
 		// TODO: If a different content type is preferred, look through resources to see if there is one
 		// specifically for that content type (most useful for html).
 		responseObject.headers.set('Vary', 'Accept, Accept-Encoding');
-		responseObject.headers.set('Content-Type', serializer.type);
+		responseObject.headers.set('Content-Type', contentType);
 		if (
 			typeof responseData === 'object' &&
 			responseData &&
@@ -454,9 +465,9 @@ export function serialize(responseData, request, responseObject) {
 		) {
 			if (
 				responseData.mapError &&
-				serializer.type !== 'application/x-ndjson' &&
-				serializer.type !== 'application/ndjson' &&
-				serializer.type !== 'text/event-stream'
+				contentType !== 'application/x-ndjson' &&
+				contentType !== 'application/ndjson' &&
+				contentType !== 'text/event-stream'
 			) {
 				// indicate that we want iterator errors to be returned so we can serialize them in a meaningful way, if possible
 				const getColumns = responseData.getColumns;
@@ -468,33 +479,27 @@ export function serialize(responseData, request, responseObject) {
 				});
 				responseData.getColumns = getColumns;
 			}
-			let stream = serializer.serializer.serializeStream(responseData, responseObject, request);
-			if (canCompress) {
+			const serialized = serializer.serializer.serializeStream(responseData, responseObject, request);
+			if (!canCompress) return serialized;
+			// a handler may return a complete body (msgpack does for arrays)
+			if (typeof serialized === 'string' || serialized instanceof Uint8Array) responseBody = serialized;
+			else {
 				responseObject.headers.set('Content-Encoding', 'br');
-				const uncompressedStream = stream;
-				stream = stream.pipe(
-					createBrotliCompress({
-						params: {
-							[constants.BROTLI_PARAM_MODE]:
-								serializer.type.includes('json') || serializer.type.includes('text')
-									? constants.BROTLI_MODE_TEXT
-									: constants.BROTLI_MODE_GENERIC,
-							[constants.BROTLI_PARAM_QUALITY]: 2, // go fast
-						},
-					})
-				);
-				if (uncompressedStream[streamStartup]) stream[streamStartup] = uncompressedStream[streamStartup];
+				const source = typeof serialized.pipe === 'function' ? serialized : Readable.from(serialized);
+				// pipeline, not pipe: a source error must destroy the compressor (which the HTTP layer reports) rather
+				// than be emitted with no listener
+				const compressed = stream.pipeline(source, createBrotliCompress(brotliOptions(contentType)), () => {});
+				if (source[streamStartup]) compressed[streamStartup] = source[streamStartup];
+				return compressed;
 			}
-			return stream;
-		}
-		responseBody = serializer.serializer.serialize(responseData, responseObject);
+		} else responseBody = serializer.serializer.serialize(responseData, responseObject);
 	}
 	if (canCompress && responseBody?.length > COMPRESSION_THRESHOLD) {
 		// TODO: Only do this if the size is large and we can cache the result (otherwise use logic above)
 		responseObject.headers.set('Content-Encoding', 'br');
 		// if we have a single buffer (or string) we compress in a single async call
 		return new Promise((resolve, reject) =>
-			brotliCompress(responseBody, (err, data) => {
+			brotliCompress(responseBody, brotliOptions(contentType), (err, data) => {
 				if (err) reject(err);
 				else resolve(data);
 			})

package/server/serverHelpers/uwsServer.ts

@@ -24,7 +24,7 @@ import { UwsRequest, UwsRequestBody } from './Request.ts';
 import { Headers } from './Headers.ts';
 import { when } from '../../utility/when.ts';
 import { ClientError } from '../../utility/errors/hdbError.ts';
-import { errorToString } from '../../utility/logging/harper_logger.ts';
+import { errorToString, errorForLog, warn } from '../../utility/logging/harper_logger.ts';
 
 // uWS has no npm package; it's installed from a GitHub tag and is platform/ABI-specific.
 // Imported lazily so harper builds/loads on platforms without a uWS binary.
@@ -323,29 +323,57 @@ function writeResponse(
  * Stream a Node Readable to the uWS response with backpressure. uWS only flushes the status/headers
  * on the first body write, so for text/event-stream (SSE) — where the client must see the stream
  * open before any event — we emit a spec-valid comment line to force the flush. Client disconnect
- * (via `signal`) or a source error destroys the source and stops writing (writing to an aborted uWS
- * response is invalid).
+ * (via `signal`) destroys the source and stops writing (writing to an aborted uWS response is
+ * invalid).
+ *
+ * The response is completed only when the source ends cleanly. A source that errors, or closes
+ * without ending, aborts the connection instead: `res.end()` would write the chunked terminator and
+ * hand the client a truncated body as a complete response.
  */
 function streamResponse(
 	res: UwsResponse,
 	status: number,
 	headers: Headers,
-	source: { on: Function; once: Function; pause: Function; resume: Function; destroy?: Function },
+	source: {
+		on: Function;
+		once: Function;
+		pause: Function;
+		resume: Function;
+		destroy?: Function;
+		destroyed?: boolean;
+		errored?: Error | null;
+	},
 	signal?: AbortSignal
 ): void {
 	let finished = false;
-	const finish = (endResponse: boolean) => {
+	const finish = (sourceEnded: boolean) => {
 		if (finished) return;
 		finished = true;
 		signal?.removeEventListener('abort', onAbort);
-		if (endResponse) res.cork(() => res.end());
+		if (sourceEnded) res.cork(() => res.end());
+		else {
+			source.destroy?.();
+			// runs the onAborted callback, so `finished` has to be set by now
+			res.close();
+		}
+	};
+	const fail = (error?: Error | null) => {
+		if (finished) return;
+		if (error) warn(errorForLog(error));
+		finish(false);
 	};
 	function onAbort() {
 		if (finished) return;
 		finished = true;
 		source.destroy?.();
 	}
+	// Attached before anything can return: destroy(error) emits on a later tick, and an 'error' with
+	// no listener is thrown. Not once(), because a source may emit more than one.
+	source.on('error', fail);
 	if (signal?.aborted) return onAbort();
+	// the handler resolves asynchronously, so a source can be destroyed, and its 'close' already
+	// emitted, before this gets to listen for it
+	if (source.destroyed) return fail(source.errored);
 	signal?.addEventListener('abort', onAbort, { once: true });
 
 	const isSse = String(headers.get('content-type') ?? '').includes('text/event-stream');
@@ -371,7 +399,7 @@ function streamResponse(
 		}
 	});
 	source.once('end', () => finish(true));
-	source.once('error', () => finish(true)); // headers already sent; just terminate the response
+	source.once('close', () => fail());
 }
 
 // Normalize uWS's remote-address text (raw IPv6 form, IPv4-mapped for v4 peers) to a readable IP.

package/server/threads/itc.js

@@ -17,6 +17,7 @@ module.exports = {
 	SchemaEventMsg,
 };
 let serverItcHandlers;
+const RESTORE_CLOSE_ACK_TIMEOUT_MS = 30000;
 onMessageFromWorkers(async (event, sender) => {
 	serverItcHandlers = serverItcHandlers || require('../itc/serverHandlers.js');
 	let error;
@@ -54,6 +55,26 @@ function sendItcEvent(event, includeJobWorkers = false) {
 	// The main thread's threadId is 0 (worker_threads convention); parentPort.threadId
 	// is set to 0 in workers, so sendToThread(0, ...) routes back to main.
 	if (event.message) event.message.originator = threadId;
+	if (
+		event.type === hdbTerms.ITC_EVENT_TYPES.SCHEMA &&
+		event.message?.operation === hdbTerms.OPERATIONS_ENUM.RESTORE_BACKUP &&
+		event.message.restorePhase
+	) {
+		// Both restore phases reach job workers, because they write blobs too. Ordinary gossip excludes
+		// them to avoid re-entrant waits, but their message handlers stay live while a job's own async
+		// work is suspended, so that objection does not apply here -- and fencing them at close while
+		// sending the release only to ordinary workers would leave every job thread fenced for the life
+		// of the process.
+		if (event.message.restorePhase === 'close') {
+			// Strict and bounded: the restore must not proceed past a worker that never acknowledged the
+			// close, nor wait on a wedged one forever -- unbounded would strand the restore holding its
+			// marker, before verifyDatabaseClosed's own deadline could report a clean 409.
+			return broadcastWithStrictAcknowledgement(event, RESTORE_CLOSE_ACK_TIMEOUT_MS, true);
+		}
+		// The release is deliberately best-effort: a restore that has already done its work must not be
+		// reported as failed because one worker was slow to take its fence back off.
+		return broadcastWithAcknowledgement(event, undefined, false, true);
+	}
 	return broadcastWithAcknowledgement(event, undefined, false, includeJobWorkers);
 }