@harperfast/harper 5.2.9 → 5.3.0-alpha.1

package/server/threads/socketRouter.ts

@@ -3,11 +3,12 @@ import * as hdbTerms from '../../utility/hdbTerms.ts';
 import * as harperLogger from '../../utility/logging/harper_logger.ts';
 import { recordHostname } from '../../resources/analytics/write.ts';
 import { startTransactionLogCooling } from '../transactionLogCooling.ts';
+import { startLongLivedTransactionReporting } from '../../resources/longLivedTransactions.ts';
 import { isMainThread } from 'worker_threads';
 import { join } from 'path';
 
 const workers = [];
-const workersReady = [];
+const HTTP_WORKER_STARTUP_DIAGNOSTIC_MS = 60000;
 // startHTTPThreads() can be called more than once in-process (e.g. a test harness adding more
 // worker threads to an already-running server via addThreads()). The crash-path sweep below must
 // run only on the very first call — a later call happens after real mirrors are already bound, and
@@ -18,6 +19,9 @@ let sweptSocketsDirectory = false;
 if (isMainThread) {
 	process.on('uncaughtException', (error) => {
 		// TODO: Maybe we should try to log the first of each type of error
+		// Same `isHandled` contract as threadServer.js: an error another handler has already
+		// classified (a lost native file watch, for instance) must not be logged again here.
+		if ((error as any).isHandled) return;
 		if ((error as any).code === 'ECONNRESET') return; // that's what network connections do
 		if ((error as any).code === 'EIO') {
 			// that means the terminal is closed
@@ -29,6 +33,7 @@ if (isMainThread) {
 }
 
 export async function startHTTPThreads(threadCount = 2, dynamicThreads?: boolean) {
+	const workerSlots = [];
 	// Crash-path defense: a hard crash can skip a worker's exit-time UDS cleanup and leave stale
 	// mirror files behind. This runs before any worker below can start (and thus before any mirror
 	// can bind), so it can only ever clear files nothing is using yet — never a live mirror. The
@@ -48,13 +53,14 @@ export async function startHTTPThreads(threadCount = 2, dynamicThreads?: boolean
 	// process-global singleton; see startTransactionLogCooling). Runs for all
 	// thread modes below, including the single-threaded (threadCount === 0) path.
 	startTransactionLogCooling();
+	startLongLivedTransactionReporting();
 	try {
 		if (dynamicThreads) {
 			// No caller currently passes dynamicThreads. If one ever does, note that the main thread
 			// does not bind ports in this mode, so on platforms without SO_REUSEPORT (macOS/Windows)
 			// worker 0's exclusive HTTP bind would silently swallow an external EADDRINUSE — the
 			// external-conflict detection in listenOnPorts() assumes the main thread binds first.
-			startHTTPWorker(0, 1);
+			workerSlots.push(startHTTPWorker(0, 1));
 		} else {
 			const { loadRootComponents } = require('../loadRootComponents.js');
 			if (threadCount === 0) {
@@ -73,43 +79,92 @@ export async function startHTTPThreads(threadCount = 2, dynamicThreads?: boolean
 			if (process.platform === 'win32') threadCount = 1;
 		}
 		for (let i = 0; i < threadCount; i++) {
-			startHTTPWorker(i, threadCount);
+			workerSlots.push(startHTTPWorker(i, threadCount));
 		}
-		await Promise.all(workersReady);
+		await Promise.all(workerSlots.map((slot) => slot.ready));
 	} finally {
+		for (const slot of workerSlots) slot.finishStartup();
 		threadsHaveStarted(undefined as any);
 	}
 }
 
 function startHTTPWorker(index, threadCount = 1) {
-	startWorker(join(__dirname, './threadServer.js'), {
+	const { promise: ready, resolve: resolveReady, reject: rejectReady } = Promise.withResolvers<void>();
+	let waitingForInitialReady = true;
+	let finishCurrentStartup = () => {};
+	let startupAttempts = 0;
+	// A Worker's threadId reads back as -1 once it has exited, which is exactly when the diagnostics
+	// below run, so each attempt records its id while the worker is still alive.
+	let lastThreadId = -1;
+	const finishStartup = () => {
+		waitingForInitialReady = false;
+		finishCurrentStartup();
+	};
+	const failStartup = (error) => {
+		if (!waitingForInitialReady) return;
+		waitingForInitialReady = false;
+		finishCurrentStartup();
+		rejectReady(error);
+	};
+	const workerOptions = {
 		name: hdbTerms.THREAD_TYPES.HTTP,
 		workerIndex: index,
 		threadCount,
-		async onStarted(worker) {
-			// note that this can be called multiple times, once when started, and again when threads are restarted
-			const ready = new Promise((resolve, reject) => {
-				function onMessage(message) {
-					if (message.type === 'child_started') {
-						worker.removeListener('message', onMessage);
-						resolve(worker);
-					}
+		onStarted(worker) {
+			const attempt = ++startupAttempts;
+			const threadId = (lastThreadId = worker.threadId);
+			let startupPhase = 'starting';
+			let workerReady = false;
+			let startupDiagnostic;
+			const removeWorker = () => {
+				const workerPosition = workers.indexOf(worker);
+				if (workerPosition > -1) workers.splice(workerPosition, 1);
+			};
+			const cleanupStartup = () => {
+				clearTimeout(startupDiagnostic);
+				worker.off('message', onMessage);
+			};
+			const describeStartup = (event) =>
+				`HTTP worker slot ${index} ${event} before ready (thread ${threadId}, attempt ${attempt}, phase ${startupPhase})`;
+			const onMessage = (message) => {
+				if (message.type === hdbTerms.ITC_EVENT_TYPES.CHILD_STARTUP_PHASE) {
+					startupPhase = message.phase;
+					return;
 				}
-
-				worker.on('message', onMessage);
-				worker.on('error', reject);
-			});
-			workersReady.push(ready);
-			await ready;
-			workers.push(worker);
-			worker.on('exit', removeWorker);
-			worker.on('shutdown', removeWorker);
-			function removeWorker() {
-				const index = workers.indexOf(worker);
-				if (index > -1) workers.splice(index, 1);
+				if (message.type !== hdbTerms.ITC_EVENT_TYPES.CHILD_STARTED) return;
+				workerReady = true;
+				cleanupStartup();
+				if (waitingForInitialReady) {
+					waitingForInitialReady = false;
+					resolveReady();
+				}
+				if (!workers.includes(worker)) workers.push(worker);
+				worker.on('shutdown', removeWorker);
+			};
+			const onExit = () => {
+				cleanupStartup();
+				if (workerReady) removeWorker();
+				// A rolling restart marks both the outgoing worker and its still-booting replacement
+				// wasShutdown, so a shutdown landing mid-boot is routine, not a startup failure.
+				else if (!worker.wasShutdown) harperLogger.error(describeStartup('exited'));
+			};
+			worker.on('message', onMessage);
+			worker.on('exit', onExit);
+			if (waitingForInitialReady) {
+				startupDiagnostic = setTimeout(() => {
+					if (workerReady || !waitingForInitialReady) return;
+					const message = describeStartup(`has not become ready after ${HTTP_WORKER_STARTUP_DIAGNOSTIC_MS}ms`);
+					harperLogger.error(message);
+				}, HTTP_WORKER_STARTUP_DIAGNOSTIC_MS).unref();
+				finishCurrentStartup = cleanupStartup;
 			}
 		},
-	});
+		onRestartExhausted() {
+			failStartup(new Error(`HTTP worker slot ${index} exhausted restarts before ready (thread ${lastThreadId})`));
+		},
+	};
+	startWorker(join(__dirname, './threadServer.js'), workerOptions);
+	return { ready, finishStartup };
 }
 
 // basically, the amount of additional idleness to expect based on previous idleness (some work will continue, some

package/server/threads/threadHeapMemory.ts

@@ -0,0 +1,26 @@
+import harperLogger from '../../utility/logging/harper_logger.ts';
+import { MIN_THREAD_HEAP_MEMORY_MB } from '../../utility/hdbTerms.ts';
+
+let recoveryWarned = false;
+
+export function isStartableThreadHeapMemory(configuredMb: unknown): boolean {
+	return Number.isFinite(configuredMb) && (configuredMb as number) >= MIN_THREAD_HEAP_MEMORY_MB;
+}
+
+// Below the minimum, Node aborts the whole process from v8::Isolate::Initialize before any
+// JavaScript runs, so this has to be caught ahead of startWorker — no error handler downstream can
+// contain it. The value on disk is an untrusted boot input (replicated, env var, hand edit, older
+// peer), so an unstartable one falls back to the caller's computed default rather than stopping the
+// node.
+export function resolveThreadHeapMemoryMb(configured: unknown): number | undefined {
+	if (configured == null) return undefined;
+	const configuredMb = Number(configured);
+	if (isStartableThreadHeapMemory(configuredMb)) return configuredMb;
+	if (!recoveryWarned) {
+		recoveryWarned = true;
+		harperLogger.error(
+			`Ignoring threads.maxHeapMemory: ${JSON.stringify(configured)} is below the ${MIN_THREAD_HEAP_MEMORY_MB}MB a worker thread can start on. Using the default instead; replace the configured value before downgrading.`
+		);
+	}
+	return undefined;
+}

package/server/threads/threadServer.js

@@ -169,6 +169,22 @@ function closeServers() {
 }
 
 function startServers() {
+	// A worker that has not yet posted child_started owns no ref'd handle: addPort()
+	// (manageThreads) unrefs parentPort, component watchers are persistent:false, and the
+	// reporting timers are unref'd. An await inside loadRootComponents whose completion
+	// arrives through a non-ref-holding source — e.g. rocksdb-js delivers cross-thread
+	// lock-release and parked-commit-retry wakes through unref'd threadsafe functions —
+	// can then drain the event loop, exiting the worker cleanly (code 0) before the ready
+	// handshake and aborting the whole node's startup. Hold a ref for the entire pre-ready
+	// window; the SHUTDOWN path unrefs it for graceful exit as before.
+	parentPort?.ref();
+	let startupPhase = 'loading components';
+	const reportStartupPhase = (phase) => {
+		startupPhase = phase;
+		try {
+			parentPort?.postMessage({ type: terms.ITC_EVENT_TYPES.CHILD_STARTUP_PHASE, phase });
+		} catch {}
+	};
 	const rootPath = env.get(terms.CONFIG_PARAMS.ROOTPATH);
 	if (rootPath) {
 		try {
@@ -177,7 +193,9 @@ function startServers() {
 			// ignore any errors with this; just a best effort for now
 		}
 	}
-	let loaded = require('../loadRootComponents.js')
+	reportStartupPhase(startupPhase);
+	let listening;
+	const loaded = require('../loadRootComponents.js')
 		.loadRootComponents(true)
 		.then(() => {
 			parentPort
@@ -207,6 +225,7 @@ function startServers() {
 							})
 							.then(() => closeServers())
 							.then(() => whenScopesClosed())
+							.then(() => require('../../resources/databases.ts').closeBranchDatabases())
 							.then(() => {
 								realExit(0);
 							});
@@ -222,21 +241,30 @@ function startServers() {
 					}
 				})
 				.ref(); // use this to keep the thread running until we are ready to shutdown and clean up handles
-			const listening = listenOnPorts();
-
-			// notify that we are now ready to start receiving requests
-			Promise.resolve(listening).then(() => {
-				if (getWorkerIndex() === 0) {
-					try {
-						startupLog(portServer);
-					} catch (err) {
-						console.error('Error displaying start-up log', err);
-					}
-				}
-				parentPort?.postMessage({ type: terms.ITC_EVENT_TYPES.CHILD_STARTED });
-			});
+			reportStartupPhase('binding listeners');
+			listening = listenOnPorts();
 		});
 	componentsLoadedResolve(loaded);
+	const started = loaded
+		.then(() => listening)
+		.then(() => {
+			reportStartupPhase('ready');
+			if (getWorkerIndex() === 0) {
+				try {
+					startupLog(portServer);
+				} catch (err) {
+					console.error('Error displaying start-up log', err);
+				}
+			}
+			parentPort?.postMessage({ type: terms.ITC_EVENT_TYPES.CHILD_STARTED });
+		})
+		.catch((error) => {
+			const failedPhase = startupPhase;
+			reportStartupPhase(`failed during ${failedPhase}`);
+			harperLogger.fatal(`HTTP worker startup failed during ${failedPhase}`, error);
+			if (isMainThread) throw error;
+			realExit(1);
+		});
 	// Clean up UDS files and force-close Bun server connections on unexpected exit.
 	// Without the stop(true) call, clients holding keep-alive connections to a dead Bun
 	// worker never receive a FIN/RST and hang indefinitely waiting for a response.
@@ -253,7 +281,7 @@ function startServers() {
 		}
 		httpComponent.cleanupUdsFiles();
 	});
-	return loaded;
+	return started;
 }
 /**
  * An overlong path is the only condition that can skip a domain-socket listener. Check before

package/sqlTranslator/index.ts

@@ -17,7 +17,7 @@ import * as terms from '../utility/hdbTerms.ts';
 import { handleHDBError } from '../utility/errors/hdbError.ts';
 import { HTTP_STATUS_CODES } from '../utility/errors/commonErrors.ts';
 import * as sqlEngineRouter from '../sqlEngine/router.ts';
-import { isOperationAuthorizationBypassed } from '../server/serverHelpers/operationAuthorizationState.ts';
+import { getOperationAuthorizationState } from '../server/serverHelpers/operationAuthorizationState.ts';
 
 //here we call to define and import custom functions to alasql
 alasqlFunctionImporter(alasql);
@@ -73,21 +73,37 @@ export function evaluateSQL(jsonMessage: any, callback: any) {
  * @param parsedSqlObject - The Parsed SQL statement specified in the inbound json message, of type ParsedSQLObject.
  * @returns {Array} - False if permissions check denys the statement.
  */
-export function checkASTPermissions(jsonMessage: any, parsedSqlObject: any) {
+export function checkASTPermissions(jsonMessage: any, parsedSqlObject: any, apiOperation?: string) {
 	let verifyResult = undefined;
 	try {
 		verifyResult = opAuth.verifyPermsAST(
 			parsedSqlObject.ast.statements[0],
 			jsonMessage.hdb_user,
-			parsedSqlObject.variant
+			parsedSqlObject.variant,
+			// The top-level API operation for the token-scope check: `sql` for a direct SQL call, but
+			// `export_local`/`export_to_s3` when the SQL rides inside a job's search_operation.
+			//
+			// From the explicit argument or the dispatched operation — deliberately NEVER a field read
+			// off `jsonMessage`. On the direct-SQL path that object IS the client's request body, and
+			// this check is the ONLY gate there, since the `sql` branch of chooseOperation is mutually
+			// exclusive with its verifyPerms call. Any body field consulted here is therefore a way for
+			// a caller to name whichever operation their token scope happens to allow and run arbitrary
+			// SQL under it.
+			//
+			// A job's re-parse would otherwise land here with the nested search_operation, whose
+			// `operation` is the inner `sql`; the job worker supplies the real one out of async context
+			// instead (operationAuthorizationState), which a request cannot set.
+			apiOperation ?? jsonMessage.operation
 		);
-		parsedSqlObject.permissions_checked = true;
 	} catch (e) {
 		throw e;
 	}
 	if (verifyResult) {
 		return verifyResult;
 	}
+	// Only after a pass: this flag is what processAST trusts to skip the gate, so a denied AST
+	// carrying it would read as already authorized.
+	parsedSqlObject.permissions_checked = true;
 	return null;
 }
 
@@ -134,10 +150,17 @@ export function processAST(jsonMessage: any, parsedSqlObject: any, callback: any
 		// runWithOperationAuthorizationBypass), never body state — jsonMessage.bypass_auth is
 		// caller-controlled and is stripped before operation code sees it (see
 		// server/serverHelpers/serverHandlers.js and components/mcp/tools/operations.ts).
-		if (!isOperationAuthorizationBypassed() && !parsedSqlObject.permissions_checked) {
-			let permissionsCheck = checkASTPermissions(jsonMessage, parsedSqlObject);
-			if (permissionsCheck && permissionsCheck.length > 0) {
-				return callback(UNAUTHORIZED_RESPONSE, permissionsCheck);
+		if (!parsedSqlObject.permissions_checked) {
+			const authorizationState = getOperationAuthorizationState();
+			if (authorizationState?.bypassAuth !== true) {
+				let permissionsCheck = checkASTPermissions(jsonMessage, parsedSqlObject, authorizationState?.apiOperation);
+				// A denial is a PermissionResponseObject, which has no `length`.
+				if (permissionsCheck) {
+					// Logged here because this is the last place the reason exists: evaluateSQL drops the
+					// second callback argument, so a job worker records only the bare status.
+					logger.warn('SQL statement refused by AST permission check:', permissionsCheck);
+					return callback(UNAUTHORIZED_RESPONSE, permissionsCheck);
+				}
 			}
 		}