@harperfast/harper 5.3.0-beta.3 → 5.3.0-beta.4

package/server/DESIGN.md

@@ -54,7 +54,7 @@ A request entering `http.ts` does **not** go through Fastify. The two `handleApp
 | `serverHelpers/JSONStream.ts`              | Streaming JSON output for large responses.                                                                                                                                                                                                                                                                                                                                                                                                             |
 | `nodeName.ts`                              | Resolves this node's name (config → hostname).                                                                                                                                                                                                                                                                                                                                                                                                         |
 | `static.ts`                                | Static file serving for component-bundled assets.                                                                                                                                                                                                                                                                                                                                                                                                      |
-| `throttle.ts`                              | Per-IP / per-user request throttling.                                                                                                                                                                                                                                                                                                                                                                                                                  |
+| `throttle.ts`                              | Event-loop backpressure: runs a caller's calls one per `setImmediate` cycle and hands a call to the caller's limit handler (HTTP request queues, cache-source resolution) once queue depth × average cycle time passes the limit.                                                                                                                                                                                                                      |
 | `storageReclamation.ts`                    | Disk-pressure signals to downstream consumers; `getStorageSpaceStats()` is the shared quota-aware (falls back to `statfs`) source of available/free/size storage numbers — used by `Table.getStorageStats()` (#1976). NOT used for blob storage path weighting (`resources/blob.ts`): quota-status.json is a single instance-wide figure, so it can't distinguish between multiple `STORAGE_BLOBPATHS` disks — that still needs raw per-path `statfs`. |
 | `serverRegistry.ts`                        | Trivial registry export.                                                                                                                                                                                                                                                                                                                                                                                                                               |
 | `status/`                                  | Server status reporting (cluster status, per-port info).                                                                                                                                                                                                                                                                                                                                                                                               |
@@ -219,6 +219,8 @@ The `error` value is the stable programmatic discriminator; `message` is diagnos
 
 Clean stream completion does not prove completeness; clients must inspect SSE for the `harper-error` event and NDJSON for the `$harperStreamError` control record.
 
+The operations API keeps the same line for a request refused before its progress stream starts (authentication, the role allowlist, validation): the Fastify `preSerialization` hook in `contentTypes.ts` answers an error-status reply that would negotiate `text/event-stream` as JSON, unless the route itself set `text/event-stream`. A stream payload never reaches that hook, which is how the progress stream (`progressEmitter.ts`) keeps its in-band `event: error` at status 200. The hook is shared with the legacy custom-functions server (`fastifyRoutes.ts`), whose untyped errors follow the same rule. `harper deploy` unwraps the unnamed frame older servers still send in that case.
+
 ### Deferred credential rejection (#2418)
 
 `authentication` runs before route matching, so when it meets an `Authorization` header it cannot
@@ -573,6 +575,10 @@ port so the gap is visible instead of silent.
 
 `broadcastWithAcknowledgement` already times out (30 s) on a worker whose port stays open but never acks, and that shape is almost always a blocked event loop — a native lock, a runaway synchronous call — which nothing inside the worker can report (harper-pro#788: a restarted node's single http worker went byte-silent while main kept serving `cluster_status`, and the app log only said "not acknowledged by worker thread(s) 2"). So each worker posts its Linux thread id (`readlink /proc/thread-self`) to main once at startup, before anything else runs on it, and the timeout branch reads that thread's kernel state from `/proc/self/task/<tid>`: state, `wchan`, the syscall number (the first token only — the rest of that file is argument registers and stack/instruction pointers), CPU ticks, and context-switch counts, plus two cross-platform signals main already has, `worker.performance.eventLoopUtilization()` and the age of the last 1 s resource report. It samples again a second later and logs the deltas: no CPU ticks, no context switches and `event loop active +1000ms` is "parked on a lock"; ticks climbing with state `R` is "spinning". It is deliberately main-thread-only and best-effort: `workers` and the tid live on the main thread's `Worker` objects, every `/proc` field is reported individually (a hardened container may deny `wchan`/`syscall` while `stat` stays readable), a follow-up sample whose `starttime` differs from the first is discarded (the tid may have been recycled), one diagnostic runs per worker with a 30 s cooldown so concurrent timeouts on the same worker don't multiply reads, and nothing here runs when acks arrive on time. It does not name the lock owner; that still needs a native stack from the next occurrence.
 
+## A worker's `parentPort` close is not the main thread's exit (`server/threads/manageThreads.js`)
+
+`REMOVE_PORT {threadId}` and the `onThreadExit` listeners (database-drop owner release, deploy lifecycle, registered operations, log rotation) act on a thread's death. A sibling port's `close` is taken as the far thread's exit. A worker's `parentPort` carries `threadId = 0` so lookups can route to main, but its `close` means this worker is leaving: main's exit ends the process. `addPort`'s close listener therefore passes no dead thread id for `parentPort`. Announcing one would make every sibling fail its pending acks to main, drop its own `parentPort`, and fire exit listeners for main. Regression cover: `unitTests/server/threads/stuckWorkerDiagnostics.test.js`, "does not report the main thread as exited to a sibling of a worker that closes its parentPort".
+
 ## `chooseOperation` authorizes the invoked operation against the authenticated principal (`server/serverHelpers/serverUtilities.ts`)
 
 `verifyPerms` takes a request-shaped object and reads _both_ halves of the permission question off it: the principal from `hdb_user`, and the tables from `schema`/`database`/`table`/`records`. `chooseOperation` used to hand it `json.search_operation` — a caller-supplied field — which made both halves body-controlled. Fixing one half and not the other is not a fix: with an empty `search_operation` the table map is empty, and `hasPermissions` iterating nothing authorizes everything. Regression cover: `integrationTests/security/choose-operation-authz.test.ts`.
@@ -636,3 +642,23 @@ the bare message until harper#2703, so the same error carried an error code on t
 
 REST settles a credential rejection _before_ its route lookup, so a rejected client gets the unauthorized
 close rather than `1011 No resource was found` — which would otherwise disclose whether the resource exists.
+
+## `serverErrorHandler` skips what `handlePostRequest` already logged (`server/serverHelpers/serverHandlers.js`)
+
+`handlePostRequest` logs every error it throws, at error level or the error's own higher `fatal`/`notify`, and records it against its request; `serverErrorHandler`, which Fastify calls next with the same request, skips that error. Keep the log in `handlePostRequest` rather than moving it to the handler: `serverErrorHandler` logs an error with no `logLevel` of its own at info, below the default `warn`, so most operation errors would disappear from a default log; and a custom-function route that catches `hdbCore.request()`'s rejection never reaches the handler at all. The record is keyed by request, not by error, so an error object that surfaces again in another request is still logged there.
+
+So a failure raised before the operation runs — a refusal from `chooseOperation`, a rejected request body — gets one Error line (a refusal also writes `chooseOperation`'s own `403 from operation …` and `User '…' is not permitted to …` lines; the Error line is the one that says why). An error the operation throws while it runs does not: `callOperationFunctionAsAwait` (`utility/OperationFunctionCaller.ts`) logs it, after an `Error calling operation: <handler>` line, and rethrows, and `handlePostRequest` logs it again. Its log is the only one for the callers that do not log — `server.operation()` from a component, the MCP operations tool, the SSE branch of `handlePostRequest` — so giving execution errors a single owner means deciding it for each of those, not dropping one log.
+
+## A request-queue shed is a 503 the server never logged (`server/throttle.ts`)
+
+`throttle()` sheds a queued call once `queuedCalls.length × averageEventCycleTime` exceeds the limit (20s
+by default). `server/http.ts` answers it directly with `Service unavailable, exceeded request queue limit`
+and records only an analytics action; nothing is thrown, so the HTTP error path that logs every thrown
+status (`info` for a 503, `warn` for a 500) never runs, and the `DatabaseTransaction` write-queue and
+conflict 503s log at `error` (once per stuck commit, rate-limited) as well as `info` when thrown. A 503
+with that exact body is therefore the throttle, which now writes a rate-limited `warn` per throttle instance naming
+the queue; `resources/Table.ts`'s cache-resolution throttle throws its own 503 under the same warn. The
+gate is per instance on purpose: a shared one let a cache-fill shed silence the HTTP shed that followed.
+Reproduce with a large concurrent non-GET burst on one CPU-starved worker
+(`integrationTests/resources/sourcedfrom-eav-cache-coherence.test.ts` P1 under Bun pinned to a contended
+core).

package/server/Server.ts

@@ -48,7 +48,8 @@ export interface Server {
 				replicated: boolean;
 				[key: string]: any;
 			},
-			options?: { onPeerResult?: (result: any) => void }
+			// `timeoutMs`: how long to wait for each peer's answer, in milliseconds; omitted waits for as long as the peer takes.
+			options?: { onPeerResult?: (result: any) => void; timeoutMs?: number }
 		): Promise<{ message: string; replicated?: unknown[] }>;
 		monitorNodeCAs(listener: () => void): void;
 		sendOperationToNode(node: string, operation: any, options: any): Promise<any>;

package/server/http.ts

@@ -856,7 +856,8 @@ function getHTTPServer(port: number, secure: boolean, options: ServerOptions) {
 				nodeResponse.end('Service unavailable, exceeded request queue limit');
 				recordAction(true, 'service-unavailable', port);
 			},
-			env.get(serverPrefix + '_requestQueueLimit')
+			env.get(serverPrefix + '_requestQueueLimit'),
+			`HTTP request queue on port ${port}`
 		);
 		const server = (httpServers[port] = (
 			secure ? (http2 ? createSecureServer : createSecureServerHttp1) : createServer
@@ -1127,7 +1128,8 @@ export function makeUwsHandler(port: number | string, isOperationsServer: boolea
 				body: 'Service unavailable, exceeded request queue limit',
 			};
 		},
-		requestQueueLimit
+		requestQueueLimit,
+		`HTTP request queue on port ${port}`
 	);
 	return (request: any) => {
 		const method = request.method;

package/server/itc/serverHandlers.js

@@ -11,6 +11,8 @@ const {
 	databases,
 	resetDatabases,
 	closeDatabase,
+	prepareDatabaseDrop,
+	completeDatabaseDropPreparation,
 	reloadBranchAt,
 	markDropInProgress,
 } = require('../../resources/databases.ts');
@@ -46,11 +48,37 @@ async function schemaHandler(event) {
 	}
 
 	hdbLogger.trace(`ITC schemaHandler received schema event:`, event);
+	if (event.message?.operation === hdbTerms.OPERATIONS_ENUM.DROP_SCHEMA && event.message.schema) {
+		if (event.message.prepareDrop) {
+			if (!event.message.dropPreparationId) throw new Error('Drop-schema preparation is missing its id');
+			if (!Number.isInteger(event.message.dropPreparationOwnerThreadId))
+				throw new Error('Drop-schema preparation is missing its owner thread id');
+			await prepareDatabaseDrop(
+				event.message.schema,
+				event.message.dropPreparationId,
+				event.message.dropPreparationOwnerThreadId,
+				event.message.dropPreparationRootPaths
+			);
+			return;
+		}
+		if (event.message.dropPreparationId)
+			await completeDatabaseDropPreparation(
+				event.message.schema,
+				event.message.dropPreparationId,
+				event.message.dropPreparationRootPaths
+			);
+	}
 	// restore_backup: this thread must release its store handles so the restore can purge and
 	// rewrite the database directory. The rescan below (resetDatabases) skips reloading it while
 	// the restoring marker is present, and reloads it on the completion signal (marker gone).
 	if (event.message?.operation === hdbTerms.OPERATIONS_ENUM.RESTORE_BACKUP && event.message.schema) {
-		closeDatabase(event.message.schema);
+		try {
+			await closeDatabase(event.message.schema);
+		} catch (error) {
+			// Let the originator's process-wide closure check fail the restore immediately instead of
+			// withholding this worker's acknowledgement until the broadcast timeout.
+			hdbLogger.error(`Could not release database '${event.message.schema}' for restore`, error);
+		}
 	}
 	await cleanLmdbMap(event.message);
 	await syncSchemaMetadata(event.message);

package/server/jobs/jobOwnership.ts

@@ -0,0 +1,89 @@
+'use strict';
+
+import { getDatabases } from '../../resources/databases.ts';
+import * as hdbTerms from '../../utility/hdbTerms.ts';
+import log from '../../utility/logging/harper_logger.ts';
+import * as manageThreads from '../threads/manageThreads.js';
+import { updateJob } from './jobs.ts';
+
+/**
+ * The thread fabric's process incarnation, not a pid: pids are reused, and a reused pid would make a
+ * dead job look alive. Every thread of one Harper process carries the same value, and any restart
+ * produces a new one. `undefined` on a thread started without one.
+ */
+export const JOB_OWNER_INSTANCE_ID: string | undefined = manageThreads.processIncarnation;
+
+/** Stripped from `get_job` responses. */
+export const JOB_OWNER_ATTRIBUTES = ['owner_instance', 'owner_pid'] as const;
+
+const UNFINISHED_JOB_STATUSES = [hdbTerms.JOB_STATUS_ENUM.CREATED, hdbTerms.JOB_STATUS_ENUM.IN_PROGRESS];
+
+export function stampJobOwner(job: any): void {
+	if (job == null || typeof job !== 'object') return;
+	job.owner_instance = JOB_OWNER_INSTANCE_ID;
+	job.owner_pid = process.pid;
+}
+
+let reconciliation: Promise<number> | undefined;
+
+/**
+ * Reconcile at most once per process. `loadRootComponents` re-runs on every root component reload, and
+ * no process can add a row that this one owns, so every pass after a *complete* one could only find
+ * nothing. A pass that left any row unsettled clears the memo instead, so the next reload retries the
+ * stragglers — the recovery this exists for must not be lost to one transient write failure.
+ */
+export function reconcileInterruptedJobsOnce(): Promise<number> {
+	return (reconciliation ??= reconcileInterruptedJobs().catch((error) => {
+		reconciliation = undefined;
+		throw error;
+	}));
+}
+
+/**
+ * Settle every job row left unfinished by a process that is no longer running, and report how many were
+ * settled. Callers need not await it: the rows are chosen before the first write, so a job created while
+ * it runs is this process's and out of scope by construction.
+ *
+ * Settled rows are reported as ERROR rather than a new status, because every consumer of `get_job`
+ * already handles ERROR.
+ */
+export async function reconcileInterruptedJobs(): Promise<number> {
+	// A thread with no incarnation cannot tell a dead owner from a live one, so it settles nothing rather
+	// than declaring a running job dead. The main thread always has one.
+	if (JOB_OWNER_INSTANCE_ID == null) return 0;
+
+	const jobTable = (getDatabases() as any).system?.[hdbTerms.SYSTEM_TABLE_NAMES.JOB_TABLE_NAME];
+	if (!jobTable) return 0;
+
+	// Collect before writing: the search walks the `status` attribute these updates change.
+	const interrupted: Array<{ id: any; owner_pid: any }> = [];
+	for (const status of UNFINISHED_JOB_STATUSES) {
+		for await (const job of jobTable.search([{ attribute: 'status', value: status }])) {
+			if (job.owner_instance !== JOB_OWNER_INSTANCE_ID) interrupted.push({ id: job.id, owner_pid: job.owner_pid });
+		}
+	}
+
+	let settled = 0;
+	let unsettled = 0;
+	for (const { id, owner_pid } of interrupted) {
+		const owner = owner_pid == null ? 'an earlier Harper process' : `Harper process ${owner_pid}`;
+		try {
+			// Deliberately not "rerun it": the job may have applied some of its effects before it died, and
+			// nothing here can tell how far it got.
+			await updateJob({
+				id,
+				status: hdbTerms.JOB_STATUS_ENUM.ERROR,
+				message: `Job was interrupted: ${owner} exited before it finished. Its outcome is unknown — check for partial effects before running the operation again.`,
+			});
+			settled++;
+		} catch (error) {
+			// Keep going: one unwritable row must not strand the rest.
+			unsettled++;
+			log.error(`Could not settle interrupted job ${id}`, error);
+		}
+	}
+	if (settled > 0) log.warn(`Settled ${settled} job(s) left unfinished by a Harper process that is no longer running`);
+	// Reported as a failure so the pass is not memoized as complete while rows are still stuck.
+	if (unsettled > 0) throw new Error(`Could not settle ${unsettled} of ${interrupted.length} interrupted job(s)`);
+	return settled;
+}

package/server/jobs/jobProcess.ts

@@ -19,6 +19,7 @@ import { cloneDeep } from 'lodash';
 import { pathToFileURL } from 'node:url';
 import { join } from 'node:path';
 import { parentPort } from 'node:worker_threads';
+import { notifyJobCleanupComplete } from '../threads/manageThreads.js';
 import { getEnvBuiltInComponents } from './../../components/Application.ts';
 import { PACKAGE_ROOT } from '../../utility/packageUtils.js';
 const JOB_NAME = process.env[(hdbTerms as any).PROCESS_NAME_ENV_PROP] as string;
@@ -80,7 +81,9 @@ const JOB_ID = JOB_NAME.substring(4);
 		exitCode = 1;
 		harperLogger.error(err);
 		jobObj.status = hdbTerms.JOB_STATUS_ENUM.ERROR;
-		jobObj.message = err.message ? err.message : err;
+		// get_job answers a refused bulk load with its structured permission report as the message.
+		const report = err?.http_resp_msg;
+		jobObj.message = report !== null && typeof report === 'object' ? report : err?.message ? err.message : err;
 		jobObj.end_datetime = moment().valueOf();
 	} finally {
 		// A rejected updateJob must not skip handle cleanup and exit scheduling below (that would
@@ -95,12 +98,15 @@ const JOB_ID = JOB_NAME.substring(4);
 		// that exits without closing leaks its handles process-wide, which (among other costs)
 		// blocks an online restore_backup from confirming the target database is closed. Best
 		// effort — never let cleanup mask the job result.
+		let databaseHandlesReleased = false;
 		try {
 			const { closeLoadedDatabases } = await import('../../resources/databases.ts');
-			closeLoadedDatabases();
+			await closeLoadedDatabases({ requireClosed: true });
+			databaseHandlesReleased = true;
 		} catch (closeErr) {
 			harperLogger.warn('Error releasing database handles on job worker exit:', closeErr);
 		}
+		if (databaseHandlesReleased) notifyJobCleanupComplete();
 		// On Bun 1.3.13, calling process.exit() in a worker thread with lmdb-js loaded
 		// while sibling workers are running causes a NAPI fatal error crash. Unref
 		// parentPort (which broadcastWithAcknowledgement may have ref'd during schema

package/server/jobs/jobs.ts

@@ -26,6 +26,7 @@ import bulkDeleteValidator from '../../validation/bulkDeleteValidator.ts';
 import { deleteTransactionLogsBeforeValidator } from '../../validation/transactionLogValidator.ts';
 import { handleHDBError, ClientError } from '../../utility/errors/hdbError.ts';
 import { HTTP_STATUS_CODES } from '../../utility/errors/commonErrors.ts';
+import { JOB_OWNER_ATTRIBUTES, stampJobOwner } from './jobOwnership.ts';
 
 //Promisified functions
 const pSearchByValue = search.searchByValue;
@@ -38,10 +39,16 @@ export async function handleGetJob(jsonBody: any) {
 	if (jsonBody.id === undefined) throw new ClientError("'id' is required");
 	let result = await getJobById(jsonBody.id);
 	if (!hdbUtil.isEmptyOrZeroLength(result)) {
-		result[0] = { ...result[0] };
-		if (result[0].request !== undefined) delete result[0].request;
-		delete result[0]['__createdtime__'];
-		delete result[0]['__updatedtime__'];
+		// Every row, not just the first: a lookup by id should yield at most one, but the owner
+		// attributes are internal bookkeeping and a leak here exposes process identity.
+		const rows = result as any[];
+		for (let index = 0; index < rows.length; index++) {
+			const job = (rows[index] = { ...rows[index] });
+			if (job.request !== undefined) delete job.request;
+			delete job['__createdtime__'];
+			delete job['__updatedtime__'];
+			for (const attribute of JOB_OWNER_ATTRIBUTES) delete job[attribute];
+		}
 	}
 
 	return result;
@@ -63,6 +70,7 @@ export async function handleGetJobsByStartDate(jsonBody: any) {
 				if (currRes.request !== undefined) delete currRes.request;
 				delete currRes['__createdtime__'];
 				delete currRes['__updatedtime__'];
+				for (const attribute of JOB_OWNER_ATTRIBUTES) delete currRes[attribute];
 			}
 		}
 		return result;
@@ -198,6 +206,9 @@ export async function addJob(jsonBody: any) {
 	// We save the request so that the job process can get it and run the operation.
 	// Sending the request via IPC to the job process was causing some messages to be lost under load.
 	newJob.request = jsonBody;
+	// Stamped at creation, not at IN_PROGRESS: a row that never reaches its worker is stuck in exactly
+	// the same way as one whose worker died, and only an owner makes either recoverable at boot.
+	stampJobOwner(newJob);
 
 	let insertObject = new (Insert_Object as any)(
 		hdbTerms.SYSTEM_SCHEMA_NAME,

package/server/loadRootComponents.js

@@ -75,6 +75,17 @@ async function loadRootComponents(isWorkerThread = false) {
 	getTables();
 	resources.isWorker = isWorkerThread;
 
+	// Settle jobs whose owning process is gone. Main thread only, and deliberately neither awaited nor
+	// fatal: the sweep picks its rows before its first write, so nothing started below can fall into it,
+	// and an install carrying a long backlog of interrupted rows must not hold up boot while they settle.
+	// An unreconciled job row is misleading, but it is no reason to refuse to start. This function re-runs
+	// on every root component reload; the sweep itself holds the once-per-process guard.
+	if (isMainThread) {
+		require('./jobs/jobOwnership.ts')
+			.reconcileInterruptedJobsOnce()
+			.catch((error) => console.error(errorForLog(error)));
+	}
+
 	await loadCertificates();
 	// the Harper root component
 	await loadComponent(dirname(configUtils.getConfigFilePath()), resources, 'hdb', {

package/server/serverHelpers/contentTypes.ts

@@ -314,7 +314,19 @@ const registerFastifySerializers = fp(
 		fastify.addHook('preSerialization', async (request, reply) => {
 			const contentType = reply.raw.getHeader('content-type');
 			if (contentType) return;
-			const { serializer, type } = findBestSerializer(request.raw);
+			let { serializer, type } = findBestSerializer(request.raw);
+			// An error that no route chose to stream is an ordinary response: written as an unnamed event,
+			// an SSE client reads it as a stream that ended without a result.
+			if (
+				type === 'text/event-stream' &&
+				reply.statusCode >= 400 &&
+				!String(reply.getHeader('content-type') ?? '')
+					.toLowerCase()
+					.startsWith('text/event-stream')
+			) {
+				serializer = mediaTypes.get('application/json');
+				type = 'application/json';
+			}
 			reply.type(type);
 			reply.serializer(function (data: any) {
 				let serialize: (data: any, context: any) => any;

package/server/serverHelpers/serverHandlers.js

@@ -70,8 +70,14 @@ function handleServerUncaughtException(err) {
 	realExit(1);
 }
 
+// The errors handlePostRequest logged, per request; see server/DESIGN.md.
+const errorsLoggedByRequest = new WeakMap();
+const LEVELS_ABOVE_ERROR = new Set([terms.LOG_LEVELS.FATAL, terms.LOG_LEVELS.NOTIFY]);
+
 function serverErrorHandler(error, req, resp) {
-	harperLogger[error.logLevel || 'info'](error);
+	// Fastify passes a handler's reason through as-is, so a rejection with none arrives here as nullish.
+	error ??= handleHDBError(new Error('The request failed without an error'), undefined, 500);
+	if (!errorsLoggedByRequest.get(req)?.has(error)) harperLogger[error.logLevel || 'info'](error);
 	if (error.statusCode) {
 		if (typeof error.http_resp_msg !== 'object') {
 			const body = { error: error.http_resp_msg || error.message };
@@ -220,7 +226,10 @@ async function handlePostRequest(req, res, _bypassAuth = false) {
 		}
 		return result;
 	} catch (error) {
-		harperLogger.error(error);
+		harperLogger[LEVELS_ABOVE_ERROR.has(error?.logLevel) ? error.logLevel : 'error'](error);
+		let logged = errorsLoggedByRequest.get(req);
+		if (!logged) errorsLoggedByRequest.set(req, (logged = new Set()));
+		logged.add(error);
 		throw error;
 	}
 }

package/server/threads/itc.js

@@ -4,37 +4,63 @@ const hdbUtils = require('../../utility/common_utils.ts');
 const hdbTerms = require('../../utility/hdbTerms.ts');
 const { ITC_ERRORS } = require('../../utility/errors/commonErrors.ts');
 const { threadId } = require('worker_threads');
-const { onMessageFromWorkers, broadcastWithAcknowledgement } = require('./manageThreads.js');
+const {
+	onMessageFromWorkers,
+	broadcastWithAcknowledgement,
+	broadcastWithStrictAcknowledgement,
+} = require('./manageThreads.js');
 
 module.exports = {
 	sendItcEvent,
+	sendItcEventStrict,
 	validateEvent,
 	SchemaEventMsg,
 };
 let serverItcHandlers;
 onMessageFromWorkers(async (event, sender) => {
 	serverItcHandlers = serverItcHandlers || require('../itc/serverHandlers.js');
-	validateEvent(event);
-	if (serverItcHandlers[event.type]) {
-		await serverItcHandlers[event.type](event);
+	let error;
+	try {
+		validateEvent(event);
+		if (serverItcHandlers[event.type]) {
+			await serverItcHandlers[event.type](event);
+		}
+	} catch (caught) {
+		const hdbLogger = require('../../utility/logging/harper_logger.ts');
+		hdbLogger.error('ITC event handler failed', caught);
+		error = {
+			name: caught?.name,
+			message: caught?.message ?? String(caught),
+			code: caught?.code,
+			statusCode: caught?.statusCode,
+			retryable: caught?.retryable,
+		};
 	}
 	if (event.requestId && sender)
 		sender.postMessage({
 			type: 'ack',
 			id: event.requestId,
+			error,
 		});
 });
 
 /**
  * Emits an ITC event to the ITC server.
  * @param event
+ * @param {boolean|'active'} includeJobWorkers
  */
-function sendItcEvent(event) {
+function sendItcEvent(event, includeJobWorkers = false) {
 	// Always stamp originator so handlers can send direct responses back.
 	// The main thread's threadId is 0 (worker_threads convention); parentPort.threadId
 	// is set to 0 in workers, so sendToThread(0, ...) routes back to main.
 	if (event.message) event.message.originator = threadId;
-	return broadcastWithAcknowledgement(event);
+	return broadcastWithAcknowledgement(event, undefined, false, includeJobWorkers);
+}
+
+/** @param {boolean|'active'} includeJobWorkers */
+function sendItcEventStrict(event, timeout, includeJobWorkers = false) {
+	if (event.message) event.message.originator = threadId;
+	return broadcastWithStrictAcknowledgement(event, timeout, includeJobWorkers);
 }
 
 /**