@harperfast/harper 5.2.0 → 5.2.2

package/server/storageReclamation.ts

@@ -1,6 +1,6 @@
-import { readFile } from 'node:fs/promises';
+import { readFile, realpath } from 'node:fs/promises';
 import { statfs } from 'node:fs/promises';
-import { join } from 'node:path';
+import { basename, dirname, isAbsolute, join, relative, resolve, sep } from 'node:path';
 import { getWorkerIndex, getWorkerCount } from '../server/threads/manageThreads.js';
 import { logger } from '../utility/logging/logger.ts';
 import { CONFIG_PARAMS } from '../utility/hdbTerms.ts';
@@ -27,6 +27,56 @@ export type QuotaStatusData = {
 	updatedAt: number;
 };
 
+/**
+ * Resolves `target` to its real (symlink-free) path. `target` itself may not exist yet (e.g. a
+ * table's storage directory that hasn't been created), so walk up to the nearest existing
+ * ancestor, resolve that, and rejoin the not-yet-existing remainder lexically — this still
+ * catches a symlinked *parent* directory, which is the case that actually matters for quota
+ * scoping (leaf directories are created by Harper itself, not symlinked).
+ */
+async function resolveRealPath(target: string): Promise<string> {
+	try {
+		return await realpath(target);
+	} catch {
+		const parent = dirname(target);
+		if (parent === target) return resolve(target);
+		return join(await resolveRealPath(parent), basename(target));
+	}
+}
+
+/**
+ * Whether `target` resolves to a path inside `root`. quota-status.json is scoped to a single
+ * ROOTPATH; a database or table configured with its own STORAGE_PATH (or a STORAGE_BLOBPATHS
+ * entry) can live on an entirely different volume, and must not be reported against the root's
+ * quota. Compares real (symlink-resolved) paths, not just lexically-normalized ones — a `..`-free
+ * path can still be a symlink into another volume, and `resolve()` alone wouldn't catch that.
+ */
+async function isPathWithinRoot(root: string, target: string): Promise<boolean> {
+	const [resolvedRoot, resolvedTarget] = await Promise.all([resolveRealPath(root), resolveRealPath(target)]);
+	const relativePath = relative(resolvedRoot, resolvedTarget);
+	if (relativePath === '') return true;
+	if (isAbsolute(relativePath)) return false;
+	// Check the first segment specifically (`..`), not a `..`-prefix match — a child directory
+	// literally named e.g. `..data` (the Kubernetes ConfigMap/Secret atomic-update symlink
+	// convention) is not a parent-traversal and must not be rejected as escaping the root.
+	const firstSegment = relativePath.split(sep)[0];
+	return firstSegment !== '..';
+}
+
+/**
+ * Guards against a syntactically valid but semantically broken quota-status.json (missing or
+ * negative usage, non-positive quota, non-finite timestamp) being read as "100% free" or worse.
+ */
+function isValidQuotaStatus(status: QuotaStatusData): boolean {
+	return (
+		Number.isFinite(status.updatedAt) &&
+		Number.isFinite(status.quotaBytes) &&
+		status.quotaBytes > 0 &&
+		Number.isFinite(status.usedBytes) &&
+		status.usedBytes >= 0
+	);
+}
+
 /**
  * Reads the quota-status.json file written by host-manager.
  * Returns undefined if the file is absent or malformed; does not apply age filtering.
@@ -70,15 +120,61 @@ export function onStorageReclamation(
 }
 let reclamationTimer: NodeJS.Timeout;
 
-// If a fresh quota-status.json exists (written by host-manager every ~90s), use quota-based ratio.
-// Otherwise fall back to statfs for the registered path.
-const defaultGetAvailableSpaceRatio = async (path: string): Promise<number> => {
+export type StorageSpaceStats = {
+	available: number;
+	free: number;
+	size: number;
+	// Whether these numbers reflect a quota (accurate for what Harper can actually write) or
+	// raw filesystem-level statfs (can be wildly wrong under a quota-limited data directory).
+	basis: 'quota' | 'filesystem';
+};
+
+/**
+ * Get available/free/size storage stats for a path, preferring a fresh quota-status.json (see
+ * getQuotaStatus) over statfs. statfs describes the filesystem, which is misleading whenever
+ * the data directory is quota-limited on a larger shared volume (#1976).
+ */
+export async function getStorageSpaceStats(path: string): Promise<StorageSpaceStats> {
+	const rootPath = envMgr.get(CONFIG_PARAMS.ROOTPATH);
 	const status = await getQuotaStatus();
-	if (status?.quotaBytes && Date.now() - status.updatedAt < QUOTA_STATUS_MAX_AGE_MS) {
-		return Math.max(0, status.quotaBytes - status.usedBytes) / status.quotaBytes;
+	const statusAge = status ? Date.now() - status.updatedAt : undefined;
+	if (
+		status &&
+		isValidQuotaStatus(status) &&
+		rootPath &&
+		// A negative age (clock skew, or a corrupt/future updatedAt) is not "fresh" — treat it the
+		// same as stale rather than trusting quota data indefinitely.
+		statusAge >= 0 &&
+		statusAge < QUOTA_STATUS_MAX_AGE_MS &&
+		// Checked last: it's the only async/IO-bound condition, so short-circuit it behind the
+		// cheap synchronous checks above.
+		(await isPathWithinRoot(rootPath, path))
+	) {
+		const available = Math.max(0, status.quotaBytes - status.usedBytes);
+		return { available, free: available, size: status.quotaBytes, basis: 'quota' };
 	}
 	const fsStats = await statfs(path);
-	return fsStats.bavail / fsStats.blocks;
+	return {
+		available: fsStats.bavail * fsStats.bsize,
+		free: fsStats.bfree * fsStats.bsize,
+		size: fsStats.blocks * fsStats.bsize,
+		basis: 'filesystem',
+	};
+}
+
+/**
+ * Ratio of available to total size, guarding the 0/0 = NaN case (a zero-size statfs/quota result,
+ * e.g. an empty or misconfigured mount) so a full disk still reports a ratio of 0 rather than NaN.
+ */
+export function computeAvailableRatio(available: number, size: number): number {
+	return size > 0 ? available / size : 0;
+}
+
+// If a fresh quota-status.json exists (written by host-manager every ~90s), use quota-based ratio.
+// Otherwise fall back to statfs for the registered path.
+const defaultGetAvailableSpaceRatio = async (path: string): Promise<number> => {
+	const { available, size } = await getStorageSpaceStats(path);
+	return computeAvailableRatio(available, size);
 };
 let getAvailableSpaceRatio: (path: string) => Promise<number> = defaultGetAvailableSpaceRatio;
 

package/server/threads/manageThreads.js

@@ -232,6 +232,14 @@ function registerWorkerDataProvider(name, provider) {
 		if (workerDataProviders.get(name) === provider) workerDataProviders.delete(name);
 	};
 }
+// Propagate this thread's in-process config overrides (env.setProperty — installer bootstrap or
+// the unit-test harness's per-run isolation) to every worker spawned from here, so a worker's
+// effective config is never silently whatever happens to be installed on disk. Worker-side replay
+// is environmentManager.ts's applyInheritedConfigOverrides(), invoked from initSync(). Registered
+// on every thread (not just main) so a nested worker-of-a-worker also inherits the full chain.
+// setProperty() clones each value as it records it, so this provider cannot hit the log-and-skip
+// path below — which for this one would mean spawning the worker on the on-disk config.
+registerWorkerDataProvider('configOverrides', () => envMgr.getConfigOverrides());
 function collectProvidedWorkerData(options) {
 	if (workerDataProviders.size === 0) return undefined;
 	let provided;

package/server/threads/socketRouter.ts

@@ -8,6 +8,12 @@ import { join } from 'path';
 
 const workers = [];
 const workersReady = [];
+// startHTTPThreads() can be called more than once in-process (e.g. a test harness adding more
+// worker threads to an already-running server via addThreads()). The crash-path sweep below must
+// run only on the very first call — a later call happens after real mirrors are already bound, and
+// sweeping the sockets directory then would delete their live files, reintroducing the exact outage
+// this guards against.
+let sweptSocketsDirectory = false;
 
 if (isMainThread) {
 	process.on('uncaughtException', (error) => {
@@ -23,6 +29,20 @@ if (isMainThread) {
 }
 
 export async function startHTTPThreads(threadCount = 2, dynamicThreads?: boolean) {
+	// Crash-path defense: a hard crash can skip a worker's exit-time UDS cleanup and leave stale
+	// mirror files behind. This runs before any worker below can start (and thus before any mirror
+	// can bind), so it can only ever clear files nothing is using yet — never a live mirror. The
+	// inode ownership guard in cleanupUdsFiles()/markUdsBindFailed() (see http.ts) is the matching
+	// defense for the in-process rolling-restart case, which this sweep does not cover.
+	// Lazy dynamic import (not a top-level one), matching server/status/index.ts's own lazy import of
+	// http.ts: http.ts's module graph reaches security/auth.ts, whose module-scope table() call needs
+	// config already initialized — pulling that graph in at this file's own top level (which
+	// bin/run.ts imports before parsing argv / initializing config) breaks startup with "Unable to
+	// determine database storage path" before main() ever runs.
+	if (isMainThread && !sweptSocketsDirectory) {
+		sweptSocketsDirectory = true;
+		(await import('../http.ts')).cleanupSocketsDirectory();
+	}
 	recordHostname().catch((err) => harperLogger.error?.('Error recording hostname for analytics:', err));
 	// Drive transaction-log cooling from the main thread (the registry is a
 	// process-global singleton; see startTransactionLogCooling). Runs for all

package/server/threads/threadServer.js

@@ -275,6 +275,11 @@ function listenOnDomainSocket(port, server) {
 		}
 		function onListening() {
 			server.removeListener('error', onError);
+			// Record ownership of the inode we just bound, so cleanupUdsFiles()/markUdsBindFailed()
+			// (see http.ts) can tell this worker's own file apart from a replacement's later rebind
+			// at the same path (see registerUdsCleanupPaths). A no-op for domain sockets that aren't
+			// UDS mirrors (e.g. the operations API's primary socket), which were never registered.
+			httpComponent.recordUdsBindSuccess(port);
 			harperLogger.info('Domain socket listening on ' + port);
 			resolve({ port, name: server.name, protocol_name: server.protocol_name });
 		}
@@ -387,6 +392,9 @@ function listenOnPorts() {
 							callback?.();
 						},
 					};
+					// createUwsServer() only resolves once uWS's own listen_unix() confirms the bind, so
+					// ownership is confirmed now (see http.ts's ownership-aware cleanupUdsFiles()).
+					if (cfg.socketPath) httpComponent.recordUdsBindSuccess(cfg.socketPath);
 					harperLogger.info('uWS listening on ' + (cfg.socketPath ?? cfg.port));
 					return { port: key };
 				})
@@ -517,6 +525,9 @@ async function listenOnPortsBun() {
 				});
 				SERVERS[udsPath] = udsServer;
 				httpComponent.registerUdsCleanupPaths(udsPath, yamlPath);
+				// Bun.serve() above already succeeded (a bind failure throws synchronously into the
+				// surrounding catch), so this worker's ownership of udsPath is confirmed now.
+				httpComponent.recordUdsBindSuccess(udsPath);
 
 				const writeMetadata = () => httpComponent.writeUdsMetadata(yamlPath, port, config.pseudoServer);
 				config.tlsSelector.ready.then(writeMetadata);

package/sqlTranslator/index.ts

@@ -17,6 +17,7 @@ import * as terms from '../utility/hdbTerms.ts';
 import { handleHDBError } from '../utility/errors/hdbError.ts';
 import { HTTP_STATUS_CODES } from '../utility/errors/commonErrors.ts';
 import * as sqlEngineRouter from '../sqlEngine/router.ts';
+import { isOperationAuthorizationBypassed } from '../server/serverHelpers/operationAuthorizationState.ts';
 
 //here we call to define and import custom functions to alasql
 alasqlFunctionImporter(alasql);
@@ -129,7 +130,11 @@ export function processAST(jsonMessage: any, parsedSqlObject: any, callback: any
 	try {
 		let sqlFunction = nullFunction;
 
-		if (!jsonMessage.bypass_auth && !parsedSqlObject.permissions_checked) {
+		// Trusted bypass is dispatch/async-context state (set by chooseOperation's caller via
+		// runWithOperationAuthorizationBypass), never body state — jsonMessage.bypass_auth is
+		// caller-controlled and is stripped before operation code sees it (see
+		// server/serverHelpers/serverHandlers.js and components/mcp/tools/operations.ts).
+		if (!isOperationAuthorizationBypassed() && !parsedSqlObject.permissions_checked) {
 			let permissionsCheck = checkASTPermissions(jsonMessage, parsedSqlObject);
 			if (permissionsCheck && permissionsCheck.length > 0) {
 				return callback(UNAUTHORIZED_RESPONSE, permissionsCheck);

package/static/defaultConfig.yaml

@@ -72,6 +72,8 @@ storage:
   writeAsync: false
   caching: true
   compression: true
+  rocks:
+    compression: null
   noReadAhead: false
   path: null
   backupPath: null