@harperfast/harper 5.2.13 → 5.3.0-beta.1

package/server/threads/socketRouter.ts

@@ -1,13 +1,29 @@
-import { startWorker, setMonitorListener, setMainIsWorker, threadsHaveStarted } from './manageThreads.js';
+import {
+	startWorker,
+	setMonitorListener,
+	setMainIsWorker,
+	threadsHaveStarted,
+	setIsolatedWorkerReconciler,
+	setRunningIsolatedApplicationsGetter,
+	workersForApplication,
+	stopWorker,
+} from './manageThreads.js';
+import {
+	presentIsolatedApplicationNames,
+	isolatedApplicationRefusal,
+	isolatedApplicationCapacityRefusal,
+} from './isolatedApplications.ts';
+import { lifecycle as componentLifecycle } from '../../components/status/index.ts';
 import * as hdbTerms from '../../utility/hdbTerms.ts';
 import * as harperLogger from '../../utility/logging/harper_logger.ts';
 import { recordHostname } from '../../resources/analytics/write.ts';
 import { startTransactionLogCooling } from '../transactionLogCooling.ts';
-import { isMainThread } from 'worker_threads';
-import { join } from 'path';
+import { startLongLivedTransactionReporting } from '../../resources/longLivedTransactions.ts';
+import { isMainThread } from 'node:worker_threads';
+import { join } from 'node:path';
 
 const workers = [];
-const workersReady = [];
+const HTTP_WORKER_STARTUP_DIAGNOSTIC_MS = 60000;
 // startHTTPThreads() can be called more than once in-process (e.g. a test harness adding more
 // worker threads to an already-running server via addThreads()). The crash-path sweep below must
 // run only on the very first call — a later call happens after real mirrors are already bound, and
@@ -18,6 +34,9 @@ let sweptSocketsDirectory = false;
 if (isMainThread) {
 	process.on('uncaughtException', (error) => {
 		// TODO: Maybe we should try to log the first of each type of error
+		// Same `isHandled` contract as threadServer.js: an error another handler has already
+		// classified (a lost native file watch, for instance) must not be logged again here.
+		if ((error as any).isHandled) return;
 		if ((error as any).code === 'ECONNRESET') return; // that's what network connections do
 		if ((error as any).code === 'EIO') {
 			// that means the terminal is closed
@@ -29,6 +48,7 @@ if (isMainThread) {
 }
 
 export async function startHTTPThreads(threadCount = 2, dynamicThreads?: boolean) {
+	const workerSlots = [];
 	// Crash-path defense: a hard crash can skip a worker's exit-time UDS cleanup and leave stale
 	// mirror files behind. This runs before any worker below can start (and thus before any mirror
 	// can bind), so it can only ever clear files nothing is using yet — never a live mirror. The
@@ -48,13 +68,16 @@ export async function startHTTPThreads(threadCount = 2, dynamicThreads?: boolean
 	// process-global singleton; see startTransactionLogCooling). Runs for all
 	// thread modes below, including the single-threaded (threadCount === 0) path.
 	startTransactionLogCooling();
+	startLongLivedTransactionReporting();
 	try {
 		if (dynamicThreads) {
 			// No caller currently passes dynamicThreads. If one ever does, note that the main thread
 			// does not bind ports in this mode, so on platforms without SO_REUSEPORT (macOS/Windows)
 			// worker 0's exclusive HTTP bind would silently swallow an external EADDRINUSE — the
 			// external-conflict detection in listenOnPorts() assumes the main thread binds first.
-			startHTTPWorker(0, 1);
+			const slot = startHTTPWorker(0, 1);
+			workerSlots.push(slot);
+			poolSlots.push(slot);
 		} else {
 			const { loadRootComponents } = require('../loadRootComponents.js');
 			if (threadCount === 0) {
@@ -72,44 +95,282 @@ export async function startHTTPThreads(threadCount = 2, dynamicThreads?: boolean
 			// Windows does not support SO_REUSEPORT, so only a single HTTP worker is supported.
 			if (process.platform === 'win32') threadCount = 1;
 		}
+		poolSize = threadCount;
+		nextIsolatedIndex = Math.max(nextIsolatedIndex, threadCount);
+		const isolated = admittedIsolatedApplications([...isolatedSlots.keys()]);
+		const heapShareCount = threadCount + isolated.length;
 		for (let i = 0; i < threadCount; i++) {
-			startHTTPWorker(i, threadCount);
+			const slot = startHTTPWorker(i, threadCount, undefined, heapShareCount);
+			workerSlots.push(slot);
+			poolSlots.push(slot);
 		}
-		await Promise.all(workersReady);
+		// One dedicated worker per isolated application, numbered past the pool so no pool-only duty
+		// (worker 0's startup log, the last worker's cleanup) ever lands on it.
+		for (const application of isolated) {
+			if (isolatedSlots.has(application)) continue;
+			const slot = startHTTPWorker(nextIsolatedIndex++, threadCount, application, heapShareCount);
+			isolatedSlots.set(application, slot);
+			workerSlots.push(slot);
+			void watchDedicatedStart(application, slot);
+		}
+		await Promise.all(workerSlots.filter((slot) => !slot.application).map((slot) => slot.ready));
 	} finally {
+		for (const slot of workerSlots) if (!slot.application) slot.finishStartup();
 		threadsHaveStarted(undefined as any);
 	}
 }
 
-function startHTTPWorker(index, threadCount = 1) {
-	startWorker(join(__dirname, './threadServer.js'), {
+let poolSize = 0;
+const refusedIsolated = new Set<string>(); // reported once, not on every reconcile
+const ISOLATED_WORKER_READY_TIMEOUT_MS = 60_000; // the pool replacement path's backstop
+let nextIsolatedIndex = 0;
+type IsolatedSlot = {
+	ready: Promise<void>;
+	finishStartup: () => void;
+	shutdown: () => Promise<void>;
+	setHeapShareCount: (count: number) => void;
+	application?: string;
+};
+const isolatedSlots = new Map<string, IsolatedSlot>();
+const poolSlots: IsolatedSlot[] = [];
+
+/**
+ * The isolated applications that get a dedicated worker. Ones already running keep their place; new
+ * ones are admitted up to `threads.maxIsolated`, and only while a dedicated worker would be reachable
+ * at all. Everything refused is refused loudly and loaded nowhere -- never downgraded to the pool.
+ */
+function admittedIsolatedApplications(running: string[]): string[] {
+	const names = presentIsolatedApplicationNames();
+	const admitted = [];
+	for (const name of running) {
+		if (!names.includes(name)) continue;
+		const refusal = isolatedApplicationRefusal(name);
+		if (refusal) {
+			reportIsolatedRefusal(name, refusal);
+			continue;
+		}
+		admitted.push(name);
+	}
+	for (const name of names) {
+		if (admitted.includes(name)) continue;
+		const refusal = isolatedApplicationRefusal(name) ?? isolatedApplicationCapacityRefusal(name, admitted);
+		if (refusal) {
+			reportIsolatedRefusal(name, refusal);
+			continue;
+		}
+		refusedIsolated.delete(name);
+		admitted.push(name);
+	}
+	return admitted;
+}
+
+function reportIsolatedRefusal(name: string, refusal: string) {
+	if (refusedIsolated.has(name)) return;
+	refusedIsolated.add(name);
+	const error = new Error(
+		`Application '${name}' is isolated but gets no dedicated worker: ${refusal}; it is not loaded anywhere`
+	);
+	harperLogger.error(error.message);
+	componentLifecycle.failed(name, error, `Component '${name}' failed to load`);
+}
+
+/**
+ * Wait for a dedicated worker to become ready, with the pool replacement path's backstop. A worker that
+ * fails, or stays alive without ever reporting ready, is recorded as a failed component, stopped, and
+ * only then has its slot freed, so the next reconcile retries against a worker that has exited.
+ * Resolves whether it became ready.
+ */
+function watchDedicatedStart(application: string, slot: IsolatedSlot): Promise<boolean> {
+	return Promise.race([
+		slot.ready,
+		new Promise<never>((_, reject) =>
+			setTimeout(
+				() => reject(new Error(`Dedicated worker for '${application}' did not become ready in time`)),
+				ISOLATED_WORKER_READY_TIMEOUT_MS
+			).unref()
+		),
+	])
+		.then(() => true)
+		.catch(async (error) => {
+			harperLogger.error(`Dedicated worker for isolated application '${application}' failed to start`, error);
+			componentLifecycle.failed(application, error, `Component '${application}' failed to load`);
+			// The slot IS the lease: holding it until the failed worker has actually exited is what stops a
+			// reconcile from starting a replacement over its still-bound UDS mirror and still-open stores,
+			// and what makes a concurrent drop's `await slot.shutdown()` cover this worker too.
+			await slot.shutdown();
+			if (isolatedSlots.get(application) !== slot) return false; // a drop already withdrew this lease
+			isolatedSlots.delete(application);
+			const { cleanupApplicationSockets } = await import('../http.ts');
+			// A reconcile may have installed a replacement while that import resolved. Never unlink its mirror.
+			if (!isolatedSlots.has(application)) cleanupApplicationSockets(application);
+			return false;
+		})
+		.finally(() => slot.finishStartup());
+}
+
+/**
+ * Bring the dedicated workers in line with the root config: start one for each isolated application
+ * that has none, stop the one of an application that is no longer isolated or no longer configured.
+ * Runs on the main thread after every root-component reload (deploy, drop, restart).
+ */
+let reconciling: Promise<string[]> = Promise.resolve([]);
+export function reconcileIsolatedWorkers(): Promise<string[]> {
+	// serialized: two reloads in flight must not compute `wanted` against each other's half-done work
+	reconciling = reconciling.then(reconcileIsolatedWorkersNow, reconcileIsolatedWorkersNow);
+	return reconciling;
+}
+
+async function reconcileIsolatedWorkersNow(): Promise<string[]> {
+	if (!isMainThread || poolSize === 0) return [];
+	const wanted = new Set(admittedIsolatedApplications([...isolatedSlots.keys()]));
+	const stopping = [];
+	for (const [application, slot] of isolatedSlots) {
+		if (wanted.has(application)) continue;
+		isolatedSlots.delete(application);
+		// awaited: a caller that goes on to remove the application's storage must see its worker gone
+		stopping.push(slot.shutdown().then(() => application));
+	}
+	const stoppedApplications = await Promise.all(stopping);
+	if (stoppedApplications.length > 0) {
+		const { cleanupApplicationSockets } = await import('../http.ts');
+		for (const application of stoppedApplications) cleanupApplicationSockets(application);
+	}
+	const started: string[] = [];
+	const heapShareCount = poolSize + wanted.size;
+	for (const slot of poolSlots) slot.setHeapShareCount(heapShareCount);
+	for (const slot of isolatedSlots.values()) slot.setHeapShareCount(heapShareCount);
+	for (const application of wanted) {
+		if (isolatedSlots.has(application)) continue;
+		const slot = startHTTPWorker(nextIsolatedIndex++, poolSize, application, heapShareCount);
+		isolatedSlots.set(application, slot);
+		started.push(application);
+		void watchDedicatedStart(application, slot);
+	}
+	return started;
+}
+if (isMainThread) {
+	setIsolatedWorkerReconciler(reconcileIsolatedWorkers);
+	setRunningIsolatedApplicationsGetter(() => [...isolatedSlots.keys()]);
+}
+
+function startHTTPWorker(index, threadCount = 1, application?: string, heapShareCount?: number) {
+	const { promise: ready, resolve: resolveReady, reject: rejectReady } = Promise.withResolvers<void>();
+	let waitingForInitialReady = true;
+	let finishCurrentStartup = () => {};
+	let startupAttempts = 0;
+	// A Worker's threadId reads back as -1 once it has exited, which is exactly when the diagnostics
+	// below run, so each attempt records its id while the worker is still alive.
+	let lastThreadId = -1;
+	let isolatedSlot: IsolatedSlot | undefined;
+	const finishStartup = () => {
+		waitingForInitialReady = false;
+		finishCurrentStartup();
+	};
+	const failStartup = (error) => {
+		if (!waitingForInitialReady) return;
+		waitingForInitialReady = false;
+		finishCurrentStartup();
+		rejectReady(error);
+	};
+	const workerOptions = {
 		name: hdbTerms.THREAD_TYPES.HTTP,
 		workerIndex: index,
 		threadCount,
-		async onStarted(worker) {
-			// note that this can be called multiple times, once when started, and again when threads are restarted
-			const ready = new Promise((resolve, reject) => {
-				function onMessage(message) {
-					if (message.type === 'child_started') {
-						worker.removeListener('message', onMessage);
-						resolve(worker);
-					}
+		application,
+		heapShareCount,
+		shouldAutoRestart: application
+			? () => isolatedSlot !== undefined && isolatedSlots.get(application) === isolatedSlot
+			: undefined,
+		onStarted(worker) {
+			const attempt = ++startupAttempts;
+			const threadId = (lastThreadId = worker.threadId);
+			let startupPhase = 'starting';
+			let workerReady = false;
+			let startupDiagnostic;
+			const removeWorker = () => {
+				const workerPosition = workers.indexOf(worker);
+				if (workerPosition > -1) workers.splice(workerPosition, 1);
+			};
+			const cleanupStartup = () => {
+				clearTimeout(startupDiagnostic);
+				worker.off('message', onMessage);
+			};
+			const describeStartup = (event) =>
+				`HTTP worker slot ${index}${application ? ` (isolated application '${application}')` : ''} ${event} before ready (thread ${threadId}, attempt ${attempt}, phase ${startupPhase})`;
+			const onMessage = (message) => {
+				if (message.type === hdbTerms.ITC_EVENT_TYPES.CHILD_STARTUP_PHASE) {
+					startupPhase = message.phase;
+					return;
 				}
-
-				worker.on('message', onMessage);
-				worker.on('error', reject);
-			});
-			workersReady.push(ready);
-			await ready;
-			workers.push(worker);
-			worker.on('exit', removeWorker);
-			worker.on('shutdown', removeWorker);
-			function removeWorker() {
-				const index = workers.indexOf(worker);
-				if (index > -1) workers.splice(index, 1);
+				if (message.type !== hdbTerms.ITC_EVENT_TYPES.CHILD_STARTED) return;
+				workerReady = true;
+				cleanupStartup();
+				if (waitingForInitialReady) {
+					waitingForInitialReady = false;
+					resolveReady();
+				}
+				if (!workers.includes(worker)) workers.push(worker);
+				worker.on('shutdown', removeWorker);
+			};
+			const onExit = () => {
+				cleanupStartup();
+				if (workerReady) removeWorker();
+				// A rolling restart marks both the outgoing worker and its still-booting replacement
+				// wasShutdown, so a shutdown landing mid-boot is routine, not a startup failure.
+				else if (!worker.wasShutdown) harperLogger.error(describeStartup('exited'));
+			};
+			worker.on('message', onMessage);
+			worker.on('exit', onExit);
+			if (waitingForInitialReady) {
+				startupDiagnostic = setTimeout(() => {
+					if (workerReady || !waitingForInitialReady) return;
+					const message = describeStartup(`has not become ready after ${HTTP_WORKER_STARTUP_DIAGNOSTIC_MS}ms`);
+					harperLogger.error(message);
+				}, HTTP_WORKER_STARTUP_DIAGNOSTIC_MS).unref();
+				finishCurrentStartup = cleanupStartup;
 			}
 		},
-	});
+		onRestartExhausted() {
+			const error = new Error(`HTTP worker slot ${index} exhausted restarts (thread ${lastThreadId})`);
+			if (waitingForInitialReady) failStartup(error);
+			else if (application && isolatedSlot && isolatedSlots.get(application) === isolatedSlot) {
+				isolatedSlots.delete(application);
+				componentLifecycle.failed(application, error, `Component '${application}' worker failed`);
+				void import('../http.ts').then(
+					({ cleanupApplicationSockets }) => {
+						// A config reconciliation may already have installed a replacement after the
+						// exhausted slot released its lease. Never unlink that replacement's mirror.
+						if (isolatedSlots.has(application)) return;
+						cleanupApplicationSockets(application);
+					},
+					(cleanupError) => {
+						harperLogger.error(
+							`Could not clean sockets for failed isolated application '${application}'`,
+							cleanupError
+						);
+					}
+				);
+			}
+		},
+	};
+	startWorker(join(__dirname, './threadServer.js'), workerOptions);
+	// Stop of a dedicated worker whose application is gone: every worker carrying the application,
+	// a crashed one's replacement still booting included, so none is left running the removed app.
+	let shutdownPromise: Promise<void> | undefined;
+	const shutdown = (): Promise<void> => {
+		if (!application) return Promise.resolve();
+		if (shutdownPromise) return shutdownPromise;
+		shutdownPromise = Promise.all(workersForApplication(application).map((worker) => stopWorker(worker))).then(
+			() => undefined
+		);
+		failStartup(new Error(`Dedicated worker for '${application}' was stopped before it became ready`));
+		return shutdownPromise;
+	};
+	const setHeapShareCount = (count: number) => {
+		workerOptions.heapShareCount = count;
+	};
+	isolatedSlot = { ready, finishStartup, shutdown, setHeapShareCount, application };
+	return isolatedSlot;
 }
 
 // basically, the amount of additional idleness to expect based on previous idleness (some work will continue, some

package/server/threads/threadHeapMemory.ts

@@ -0,0 +1,26 @@
+import harperLogger from '../../utility/logging/harper_logger.ts';
+import { MIN_THREAD_HEAP_MEMORY_MB } from '../../utility/hdbTerms.ts';
+
+let recoveryWarned = false;
+
+export function isStartableThreadHeapMemory(configuredMb: unknown): boolean {
+	return Number.isFinite(configuredMb) && (configuredMb as number) >= MIN_THREAD_HEAP_MEMORY_MB;
+}
+
+// Below the minimum, Node aborts the whole process from v8::Isolate::Initialize before any
+// JavaScript runs, so this has to be caught ahead of startWorker — no error handler downstream can
+// contain it. The value on disk is an untrusted boot input (replicated, env var, hand edit, older
+// peer), so an unstartable one falls back to the caller's computed default rather than stopping the
+// node.
+export function resolveThreadHeapMemoryMb(configured: unknown): number | undefined {
+	if (configured == null) return undefined;
+	const configuredMb = Number(configured);
+	if (isStartableThreadHeapMemory(configuredMb)) return configuredMb;
+	if (!recoveryWarned) {
+		recoveryWarned = true;
+		harperLogger.error(
+			`Ignoring threads.maxHeapMemory: ${JSON.stringify(configured)} is below the ${MIN_THREAD_HEAP_MEMORY_MB}MB a worker thread can start on. Using the default instead; replace the configured value before downgrading.`
+		);
+	}
+	return undefined;
+}

package/server/threads/threadServer.js

@@ -26,6 +26,11 @@ const {
 	getShutdownDrainCeilingMs,
 } = require('../../components/shutdownDrain.ts');
 const { realExit } = require('./workerProcessGuard.ts');
+const {
+	thisThreadsIsolatedApplication,
+	applicationSocketName,
+	shouldStartUwsListenerHere,
+} = require('./isolatedApplications.ts');
 const { isBun } = require('../serverHelpers/Request.ts');
 const { getDomainSocketPathMaxBytes, isDomainSocketPathTooLong } = require('../../utility/domainSocket.ts');
 const { createTLSSelector, getEffectiveTlsCiphers } = require('../../security/keys.ts');
@@ -169,6 +174,22 @@ function closeServers() {
 }
 
 function startServers() {
+	// A worker that has not yet posted child_started owns no ref'd handle: addPort()
+	// (manageThreads) unrefs parentPort, component watchers are persistent:false, and the
+	// reporting timers are unref'd. An await inside loadRootComponents whose completion
+	// arrives through a non-ref-holding source — e.g. rocksdb-js delivers cross-thread
+	// lock-release and parked-commit-retry wakes through unref'd threadsafe functions —
+	// can then drain the event loop, exiting the worker cleanly (code 0) before the ready
+	// handshake and aborting the whole node's startup. Hold a ref for the entire pre-ready
+	// window; the SHUTDOWN path unrefs it for graceful exit as before.
+	parentPort?.ref();
+	let startupPhase = 'loading components';
+	const reportStartupPhase = (phase) => {
+		startupPhase = phase;
+		try {
+			parentPort?.postMessage({ type: terms.ITC_EVENT_TYPES.CHILD_STARTUP_PHASE, phase });
+		} catch {}
+	};
 	const rootPath = env.get(terms.CONFIG_PARAMS.ROOTPATH);
 	if (rootPath) {
 		try {
@@ -177,7 +198,9 @@ function startServers() {
 			// ignore any errors with this; just a best effort for now
 		}
 	}
-	let loaded = require('../loadRootComponents.js')
+	reportStartupPhase(startupPhase);
+	let listening;
+	const loaded = require('../loadRootComponents.js')
 		.loadRootComponents(true)
 		.then(() => {
 			parentPort
@@ -207,6 +230,7 @@ function startServers() {
 							})
 							.then(() => closeServers())
 							.then(() => whenScopesClosed())
+							.then(() => require('../../resources/databases.ts').closeBranchDatabases())
 							.then(() => {
 								realExit(0);
 							});
@@ -222,21 +246,30 @@ function startServers() {
 					}
 				})
 				.ref(); // use this to keep the thread running until we are ready to shutdown and clean up handles
-			const listening = listenOnPorts();
-
-			// notify that we are now ready to start receiving requests
-			Promise.resolve(listening).then(() => {
-				if (getWorkerIndex() === 0) {
-					try {
-						startupLog(portServer);
-					} catch (err) {
-						console.error('Error displaying start-up log', err);
-					}
-				}
-				parentPort?.postMessage({ type: terms.ITC_EVENT_TYPES.CHILD_STARTED });
-			});
+			reportStartupPhase('binding listeners');
+			listening = listenOnPorts();
 		});
 	componentsLoadedResolve(loaded);
+	const started = loaded
+		.then(() => listening)
+		.then(() => {
+			reportStartupPhase('ready');
+			if (getWorkerIndex() === 0) {
+				try {
+					startupLog(portServer);
+				} catch (err) {
+					console.error('Error displaying start-up log', err);
+				}
+			}
+			parentPort?.postMessage({ type: terms.ITC_EVENT_TYPES.CHILD_STARTED });
+		})
+		.catch((error) => {
+			const failedPhase = startupPhase;
+			reportStartupPhase(`failed during ${failedPhase}`);
+			harperLogger.fatal(`HTTP worker startup failed during ${failedPhase}`, error);
+			if (isMainThread) throw error;
+			realExit(1);
+		});
 	// Clean up UDS files and force-close Bun server connections on unexpected exit.
 	// Without the stop(true) call, clients holding keep-alive connections to a dead Bun
 	// worker never receive a FIN/RST and hang indefinitely waiting for a response.
@@ -253,7 +286,7 @@ function startServers() {
 		}
 		httpComponent.cleanupUdsFiles();
 	});
-	return loaded;
+	return started;
 }
 /**
  * An overlong path is the only condition that can skip a domain-socket listener. Check before
@@ -302,10 +335,15 @@ function listenOnPorts() {
 		const server = SERVERS[port];
 
 		// If server is unix domain socket
+		// A worker dedicated to an isolated application binds only its own per-thread mirrors: not the shared
+		// ports (the kernel would hand it every application's connections) and not a global domain socket
+		// such as the operations API's, which it would take from the main thread.
 		if (port.includes?.('/')) {
-			listening.push(listenOnDomainSocket(port, server));
+			if (!thisThreadsIsolatedApplication() || server.isPerThreadSocket)
+				listening.push(listenOnDomainSocket(port, server));
 			continue;
 		}
+		if (thisThreadsIsolatedApplication()) continue;
 		let listen_on;
 		let ownerWorkerIndex = 0; // lowest eligible worker index for this port
 		const threadRange = env.get(terms.CONFIG_PARAMS.HTTP_THREADRANGE);
@@ -378,6 +416,7 @@ function listenOnPorts() {
 	if (uwsServeConfigs) {
 		for (const key in uwsServeConfigs) {
 			const cfg = uwsServeConfigs[key];
+			if (!shouldStartUwsListenerHere(cfg)) continue; // dedicated worker: its own mirrors only
 			if (cfg.socketPath && existsSync(cfg.socketPath)) unlinkSync(cfg.socketPath);
 			const { createUwsServer } = require('../serverHelpers/uwsServer.ts');
 			listening.push(
@@ -427,7 +466,7 @@ async function listenOnPortsBun() {
 	for (let port in bunServeConfigs) {
 		const config = bunServeConfigs[port];
 		const threadRange = env.get(terms.CONFIG_PARAMS.HTTP_THREADRANGE);
-		if (threadRange) {
+		if (threadRange && !thisThreadsIsolatedApplication()) {
 			let threadRangeArray = typeof threadRange === 'string' ? threadRange.split('-') : threadRange;
 			let threadIndex = getWorkerIndex();
 			if (threadIndex < threadRangeArray[0] || threadIndex > threadRangeArray[1]) {
@@ -504,15 +543,22 @@ async function listenOnPortsBun() {
 				delete serveOptions.port;
 			}
 			if (isNaN(serveOptions.port)) continue;
-			const bunServer = Bun.serve(serveOptions);
-			SERVERS[port] = bunServer;
-			harperLogger.trace('Bun listening on port ' + port, threadId);
+			// Keep ownership independent of address syntax: adding a Bun UDS route later must not widen
+			// a dedicated worker's ingress beyond its application-addressed mirror below.
+			if (!thisThreadsIsolatedApplication()) {
+				const bunServer = Bun.serve(serveOptions);
+				SERVERS[port] = bunServer;
+				harperLogger.trace('Bun listening on port ' + port, threadId);
+			}
 
 			// Create a corresponding Unix Domain Socket mirror for secure ports
 			if (config.isSecure && env.get(terms.CONFIG_PARAMS.TLS_UNIXDOMAINSOCKETS)) {
 				const socketsDir = join(env.getHdbBasePath(), 'sockets');
 				mkdirSync(socketsDir, { recursive: true });
-				const socketName = `${getWorkerIndex()}-${port}`;
+				const isolatedApplication = thisThreadsIsolatedApplication();
+				const socketName = isolatedApplication
+					? applicationSocketName(isolatedApplication, port)
+					: `${getWorkerIndex()}-${port}`;
 				const udsPath = join(socketsDir, `${socketName}.sock`);
 				const yamlPath = join(socketsDir, `${socketName}.yaml`);
 				if (existsSync(udsPath)) unlinkSync(udsPath);
@@ -536,6 +582,7 @@ async function listenOnPortsBun() {
 			}
 		} catch (error) {
 			harperLogger.error(`Unable to start Bun server on port ${port}`, error);
+			if (thisThreadsIsolatedApplication()) throw error;
 		}
 	}
 	// Also start any non-HTTP servers (raw socket servers) that were registered in SERVERS
@@ -546,6 +593,7 @@ async function listenOnPortsBun() {
 		if (server?.stop || bunServeConfigs[port]) continue;
 		if (server?.listen) {
 			if (port.includes?.('/')) {
+				if (thisThreadsIsolatedApplication() && !server.isPerThreadSocket) continue; // as in listenOnPorts
 				listening.push(listenOnDomainSocket(port, server));
 			} else {
 				const lastColon = String(port).lastIndexOf(':');
@@ -645,7 +693,10 @@ function onSocket(listener, options) {
 		if (env.get(terms.CONFIG_PARAMS.TLS_UNIXDOMAINSOCKETS)) {
 			const socketsDir = join(env.getHdbBasePath(), 'sockets');
 			mkdirSync(socketsDir, { recursive: true });
-			const socketName = `${getWorkerIndex()}-${options.securePort}`;
+			const isolatedApplication = thisThreadsIsolatedApplication();
+			const socketName = isolatedApplication
+				? applicationSocketName(isolatedApplication, options.securePort)
+				: `${getWorkerIndex()}-${options.securePort}`;
 			const udsPath = join(socketsDir, `${socketName}.sock`);
 			const yamlPath = join(socketsDir, `${socketName}.yaml`);
 

package/sqlTranslator/index.ts

@@ -17,7 +17,7 @@ import * as terms from '../utility/hdbTerms.ts';
 import { handleHDBError } from '../utility/errors/hdbError.ts';
 import { HTTP_STATUS_CODES } from '../utility/errors/commonErrors.ts';
 import * as sqlEngineRouter from '../sqlEngine/router.ts';
-import { isOperationAuthorizationBypassed } from '../server/serverHelpers/operationAuthorizationState.ts';
+import { getOperationAuthorizationState } from '../server/serverHelpers/operationAuthorizationState.ts';
 
 //here we call to define and import custom functions to alasql
 alasqlFunctionImporter(alasql);
@@ -73,21 +73,37 @@ export function evaluateSQL(jsonMessage: any, callback: any) {
  * @param parsedSqlObject - The Parsed SQL statement specified in the inbound json message, of type ParsedSQLObject.
  * @returns {Array} - False if permissions check denys the statement.
  */
-export function checkASTPermissions(jsonMessage: any, parsedSqlObject: any) {
+export function checkASTPermissions(jsonMessage: any, parsedSqlObject: any, apiOperation?: string) {
 	let verifyResult = undefined;
 	try {
 		verifyResult = opAuth.verifyPermsAST(
 			parsedSqlObject.ast.statements[0],
 			jsonMessage.hdb_user,
-			parsedSqlObject.variant
+			parsedSqlObject.variant,
+			// The top-level API operation for the token-scope check: `sql` for a direct SQL call, but
+			// `export_local`/`export_to_s3` when the SQL rides inside a job's search_operation.
+			//
+			// From the explicit argument or the dispatched operation — deliberately NEVER a field read
+			// off `jsonMessage`. On the direct-SQL path that object IS the client's request body, and
+			// this check is the ONLY gate there, since the `sql` branch of chooseOperation is mutually
+			// exclusive with its verifyPerms call. Any body field consulted here is therefore a way for
+			// a caller to name whichever operation their token scope happens to allow and run arbitrary
+			// SQL under it.
+			//
+			// A job's re-parse would otherwise land here with the nested search_operation, whose
+			// `operation` is the inner `sql`; the job worker supplies the real one out of async context
+			// instead (operationAuthorizationState), which a request cannot set.
+			apiOperation ?? jsonMessage.operation
 		);
-		parsedSqlObject.permissions_checked = true;
 	} catch (e) {
 		throw e;
 	}
 	if (verifyResult) {
 		return verifyResult;
 	}
+	// Only after a pass: this flag is what processAST trusts to skip the gate, so a denied AST
+	// carrying it would read as already authorized.
+	parsedSqlObject.permissions_checked = true;
 	return null;
 }
 
@@ -134,10 +150,17 @@ export function processAST(jsonMessage: any, parsedSqlObject: any, callback: any
 		// runWithOperationAuthorizationBypass), never body state — jsonMessage.bypass_auth is
 		// caller-controlled and is stripped before operation code sees it (see
 		// server/serverHelpers/serverHandlers.js and components/mcp/tools/operations.ts).
-		if (!isOperationAuthorizationBypassed() && !parsedSqlObject.permissions_checked) {
-			let permissionsCheck = checkASTPermissions(jsonMessage, parsedSqlObject);
-			if (permissionsCheck && permissionsCheck.length > 0) {
-				return callback(UNAUTHORIZED_RESPONSE, permissionsCheck);
+		if (!parsedSqlObject.permissions_checked) {
+			const authorizationState = getOperationAuthorizationState();
+			if (authorizationState?.bypassAuth !== true) {
+				let permissionsCheck = checkASTPermissions(jsonMessage, parsedSqlObject, authorizationState?.apiOperation);
+				// A denial is a PermissionResponseObject, which has no `length`.
+				if (permissionsCheck) {
+					// Logged here because this is the last place the reason exists: evaluateSQL drops the
+					// second callback argument, so a job worker records only the bare status.
+					logger.warn('SQL statement refused by AST permission check:', permissionsCheck);
+					return callback(UNAUTHORIZED_RESPONSE, permissionsCheck);
+				}
 			}
 		}