@harperfast/harper 5.2.13 → 5.3.0-beta.1

package/utility/logging/logRotator.ts

@@ -1,10 +1,6 @@
 'use strict';
 
-import { promises as fsProm, createReadStream, createWriteStream, mkdirSync } from 'fs';
-import { createGzip } from 'zlib';
-import { promisify } from 'util';
-import { pipeline } from 'stream';
-const pipe = promisify(pipeline);
+import { existsSync, mkdirSync, statSync, promises as fsProm } from 'fs';
 import * as path from 'path';
 import * as envMgr from '../environment/environmentManager.ts';
 envMgr.initSync();
@@ -12,9 +8,21 @@ import hdbLogger from './harper_logger.ts';
 import { CONFIG_PARAMS } from '../hdbTerms.ts';
 import { convertToMS } from '../common_utils.ts';
 import { onStorageReclamation } from '../../server/storageReclamation.ts';
+import { requestStaleDescriptorRelease } from './logGenerationCoordinator.ts';
+import {
+	compressPendingArchives,
+	INVALID_MAX_SIZE_MSG,
+	isArchivePendingQuiescence,
+	parseMaxSize,
+	publishArchivedGeneration,
+	resolveRotatedLogDir,
+	retryPendingGenerations,
+	rotateLogFileSync,
+} from './logRotation.ts';
 
 // Interval in ms to check log file and decide if it should be rotated.
 const LOG_AUDIT_INTERVAL = 60000;
+const RELEASE_PROOF_REPORT_INTERVAL = 5 * 60 * 1000;
 const INT_SIZE_UNDEFINED_MSG =
 	"'interval' and 'maxSize' are both undefined, to enable logging rotation at least one of these values must be defined in harperdb-config.yaml";
 
@@ -26,7 +34,16 @@ export { logRotator };
  * If log file is within the values set in config, log file will be renamed/moved and a new empty hdb.log created.
  * @returns LogRotator
  */
-function logRotator({ logger, maxSize, interval, retention, enabled, path: rotatedLogDir, auditInterval }: any) {
+function logRotator({
+	logger,
+	maxSize,
+	interval,
+	retention,
+	enabled,
+	compress,
+	path: rotatedLogDir,
+	auditInterval,
+}: any) {
 	if (enabled === false) return;
 	let reclamationPriority = 0;
 	onStorageReclamation(
@@ -41,21 +58,18 @@ function logRotator({ logger, maxSize, interval, retention, enabled, path: rotat
 		throw new Error(INT_SIZE_UNDEFINED_MSG);
 	}
 
-	if (!rotatedLogDir) {
-		rotatedLogDir = path.join(path.dirname(logger.path), 'rotated');
-	}
+	rotatedLogDir = resolveRotatedLogDir(logger.path, rotatedLogDir);
 	mkdirSync(rotatedLogDir, { recursive: true });
 
-	// Convert maxSize param to bytes.
-	let maxBytes;
-	if (maxSize) {
-		const unit = maxSize.slice(-1);
-		const size = maxSize.slice(0, -1);
-		if (unit === 'G') maxBytes = size * 1000000000;
-		else if (unit === 'M') maxBytes = size * 1000000;
-		else maxBytes = size * 1000;
+	const maxBytes = parseMaxSize(maxSize);
+	if (maxSize && !maxBytes) {
+		hdbLogger.error(`Ignoring logging.rotation.maxSize '${maxSize}': ${INVALID_MAX_SIZE_MSG}`);
 	}
 
+	// The rotation block first, because that is all the write-path guard can read — environmentManager
+	// imports harper_logger — and two sources would mean two destruction policies for one log.
+	const compressArchives = compress ?? envMgr.get(CONFIG_PARAMS.LOGGING_ROTATION_COMPRESS);
+
 	// Convert interval param to ms.
 	let maxInterval;
 	if (interval) {
@@ -63,101 +77,219 @@ function logRotator({ logger, maxSize, interval, retention, enabled, path: rotat
 	}
 
 	let lastRotatedLogPath;
-	// convert date.now to minutes
 	let lastRotationTime = Date.now();
+	let observedGeneration;
+	try {
+		const active = statSync(logger.path);
+		observedGeneration = generationIdentity(active);
+		if (active.birthtimeMs > 0) lastRotationTime = Math.min(lastRotationTime, active.birthtimeMs);
+	} catch {}
 	hdbLogger.trace('Log rotate enabled, maxSize:', maxSize, 'interval:', interval);
+	let tickInFlight = false;
+	let ended = false;
+	let releaseProofStalled = false;
+	let nextReleaseProofReport = 0;
+	let retentionCursor = 0;
+	const auditIntervalMs = auditInterval ?? LOG_AUDIT_INTERVAL;
+	const compressionBudgetMs = Math.max(1, Math.floor(auditIntervalMs / 4));
+	/**
+	 * rename() reports ENOENT for a missing destination directory as well as a missing source, and
+	 * only the second is the benign lost race. The directory is created once at rotator start, so a
+	 * rotation target removed under a running instance otherwise stops rotation permanently and
+	 * silently — and with `interval` and no `maxSize` there is no write-path guard to recreate it.
+	 */
+	function recoverRotationTarget(err: any) {
+		if (err.code !== 'ENOENT' || existsSync(rotatedLogDir)) return;
+		try {
+			mkdirSync(rotatedLogDir, { recursive: true });
+			hdbLogger.warn(`The log rotation directory ${rotatedLogDir} was missing and has been recreated`);
+		} catch (mkdirErr) {
+			hdbLogger.error('Could not recreate the log rotation directory', rotatedLogDir, mkdirErr);
+		}
+	}
 	const setIntervalId = setInterval(async () => {
-		// A missing/relocated active log file only invalidates the rotation checks below — retention cleanup
-		// must still run. So skip the individual check on ENOENT rather than returning from the whole tick.
-		if (maxBytes) {
-			let fileStats;
-			try {
-				fileStats = await fsProm.stat(logger.path);
-			} catch (err) {
-				// If the log file doesn't exist, skip the size-based rotation check
-				if (err.code !== 'ENOENT') throw err;
-			}
-
-			if (fileStats && fileStats.size >= maxBytes) {
+		// setInterval does not await the callback, and one pass can now wait on peers; overlapping
+		// passes would work the same archive twice and double-compress it.
+		if (tickInFlight || ended) return;
+		tickInFlight = true;
+		const tickDeadline = Date.now() + auditIntervalMs;
+		// The tick is async but setInterval doesn't await it, so any error that escapes this callback
+		// becomes an unhandled rejection rather than surfacing anywhere useful — and since it isn't
+		// caught, it also skips the retention cleanup below. Contain everything here and report via
+		// the logger instead, so one bad tick (e.g. an unexpected fs error) never kills rotation for
+		// the rest of the process's life.
+		try {
+			if (maxBytes) {
 				try {
-					lastRotatedLogPath = await moveLogFile(logger.path, rotatedLogDir, logger);
+					// statSync, and the rename in the same turn: an await here lets a writing thread rotate
+					// the generation this tick measured and start a fresh one, which the tick would then
+					// archive near-empty.
+					const active = statSync(logger.path);
+					if (active.size >= maxBytes) {
+						lastRotatedLogPath = await moveLogFile(logger.path, rotatedLogDir, logger, compressArchives, active);
+						// The interval clock counts from the last rotation of any kind. Without this an
+						// instance whose uptime has passed `interval` archives a freshly-created log every
+						// interval on top of the size rotations already doing the work.
+						lastRotationTime = Date.now();
+					}
 				} catch (err) {
-					// If the log file doesn't exist, skip rotation
+					// A missing or already-rotated active log only invalidates this check; retention below
+					// must still run, so skip the check rather than leaving the whole tick.
+					recoverRotationTarget(err);
 					if (err.code !== 'ENOENT') throw err;
 				}
 			}
-		}
 
-		if (maxInterval) {
-			const minSinceLastRotate = Date.now() - lastRotationTime;
-			if (minSinceLastRotate >= maxInterval) {
+			if (maxInterval) {
 				try {
-					lastRotatedLogPath = await moveLogFile(logger.path, rotatedLogDir, logger);
-					lastRotationTime = Date.now();
+					const activeGeneration = generationIdentity(statSync(logger.path));
+					if (activeGeneration && activeGeneration !== observedGeneration) {
+						observedGeneration = activeGeneration;
+						lastRotationTime = Date.now();
+					}
 				} catch (err) {
-					// If the log file doesn't exist, skip rotation
 					if (err.code !== 'ENOENT') throw err;
 				}
+				if (Date.now() - lastRotationTime >= maxInterval) {
+					try {
+						lastRotatedLogPath = await moveLogFile(logger.path, rotatedLogDir, logger, compressArchives);
+						lastRotationTime = Date.now();
+					} catch (err) {
+						// If the log file doesn't exist, skip rotation
+						recoverRotationTarget(err);
+						if (err.code !== 'ENOENT') throw err;
+					}
+				}
 			}
-		}
-		if (retention || reclamationPriority) {
-			// remove old logs after retention time
-			// adjust retention time if there is a reclamation priority in place
-			const retentionMs = convertToMS(retention ?? '1M') / (1 + reclamationPriority);
-			reclamationPriority = 0; // reset it after use
-			let files;
-			try {
-				files = await fsProm.readdir(rotatedLogDir);
-			} catch (err) {
-				// The rotated log dir may not exist yet (nothing rotated so far); nothing to clean up
-				if (err.code !== 'ENOENT') hdbLogger.error('Error reading rotated log directory', rotatedLogDir, err);
-				files = [];
-			}
-			for (const file of files) {
+			// The compression sweep and retention both destroy archives this thread did not rotate, so
+			// both need the same proof: every peer releases any descriptor that is not on the live
+			// generation, and reports the log paths it is writing. One round trip per tick serves both,
+			// and it is not gated on retention being configured — retention is unset by default, and a
+			// worker's uncompressed archive still needs finishing.
+			if (compressArchives || retention || reclamationPriority) {
+				// Enumerated BEFORE the proof, and only these are destroyed: an archive created after the
+				// proof was never covered by it, and a peer blocked mid-rotation may still be writing to it.
+				let candidates;
 				try {
-					const fileStats = await fsProm.stat(path.join(rotatedLogDir, file));
-					if (Date.now() - fileStats.mtimeMs > retentionMs) {
-						await fsProm.unlink(path.join(rotatedLogDir, file));
-					}
+					candidates = await fsProm.readdir(rotatedLogDir);
 				} catch (err) {
-					hdbLogger.error('Error trying to remove log', file, err);
+					if (err.code !== 'ENOENT') hdbLogger.error('Error reading rotated log directory', rotatedLogDir, err);
+					candidates = [];
+				}
+				const { released, liveLogPaths } = await requestStaleDescriptorRelease(tickDeadline);
+				// A peer whose event loop is blocked never answers, and the whole pass then destroys
+				// nothing — correct, but it is also the only thing bounding the rotated directory, so an
+				// operator who configured retention has to be able to see that it has stopped. Repeat at
+				// a low rate so a permanent stall does not disappear after one warning.
+				if (!released && Date.now() >= nextReleaseProofReport) {
+					nextReleaseProofReport = Date.now() + RELEASE_PROOF_REPORT_INTERVAL;
+					releaseProofStalled = true;
+					hdbLogger.warn(
+						`Log rotation could not prove every thread released its archived log descriptors; compression and retention are paused for ${rotatedLogDir}`
+					);
+				} else if (released && releaseProofStalled) {
+					releaseProofStalled = false;
+					nextReleaseProofReport = 0;
+					hdbLogger.notify(`Log rotation descriptor release recovered; retention resumed for ${rotatedLogDir}`);
+				}
+				const liveLogs = new Set([...liveLogPaths, logger.path].map((p) => path.resolve(p)));
+
+				if (released && (retention || reclamationPriority)) {
+					// remove old logs after retention time
+					// adjust retention time if there is a reclamation priority in place
+					const retentionMs = convertToMS(retention ?? '1M') / (1 + reclamationPriority);
+					let retentionCompleted = true;
+					let examined = 0;
+					for (; examined < candidates.length; examined++) {
+						if (ended || Date.now() >= tickDeadline) {
+							retentionCompleted = false;
+							break;
+						}
+						const file = candidates[(retentionCursor + examined) % candidates.length];
+						try {
+							const archivePath = path.join(rotatedLogDir, file);
+							// An explicitly configured rotation path may contain live component logs too.
+							if (liveLogs.has(path.resolve(archivePath))) continue;
+							// Unlinking an inode a stalled writer still holds loses whatever it writes next
+							// just as surely as compressing over it would.
+							if (isArchivePendingQuiescence(archivePath)) continue;
+							const fileStats = await fsProm.stat(archivePath);
+							if (Date.now() - fileStats.mtimeMs > retentionMs) {
+								await fsProm.unlink(archivePath);
+							}
+						} catch (err) {
+							if (err.code !== 'ENOENT') hdbLogger.error('Error trying to remove log', file, err);
+						}
+					}
+					if (retentionCompleted) {
+						retentionCursor = 0;
+						reclamationPriority = 0;
+					} else if (candidates.length) {
+						retentionCursor = (retentionCursor + examined) % candidates.length;
+					}
+				}
+
+				if (released && compressArchives && !ended) {
+					const compressionFailure = await compressPendingArchives(rotatedLogDir, candidates, liveLogs, {
+						deadline: Math.min(tickDeadline, Date.now() + compressionBudgetMs),
+						shouldStop: () => ended,
+					});
+					if (compressionFailure)
+						hdbLogger.error('Error compressing rotated log', compressionFailure.file, compressionFailure.error);
 				}
 			}
+		} catch (err) {
+			hdbLogger.error('Error during log rotation audit tick for', logger.path, err);
+		} finally {
+			try {
+				await retryPendingGenerations({ deadline: tickDeadline, shouldStop: () => ended });
+			} catch (err) {
+				hdbLogger.error('Error retrying pending log generations for', logger.path, err);
+			}
+			tickInFlight = false;
 		}
-	}, auditInterval ?? LOG_AUDIT_INTERVAL).unref();
+	}, auditIntervalMs).unref();
 	return {
 		end() {
+			ended = true;
 			clearInterval(setIntervalId);
 		},
 		getLastRotatedLogPath() {
 			return lastRotatedLogPath;
 		},
 	};
+
+	function generationIdentity(stats: any) {
+		return stats.ino ? `${stats.dev}:${stats.ino}` : undefined;
+	}
 }
 
-async function moveLogFile(logPath: string, rotatedLogPath: string, logger?: any) {
-	const compress = envMgr.get(CONFIG_PARAMS.LOGGING_ROTATION_COMPRESS);
-	let fullRotateLogPath = path.join(
+async function moveLogFile(
+	logPath: string,
+	rotatedLogPath: string,
+	logger?: any,
+	compress?: boolean,
+	activeStats?: any
+) {
+	// The rename and the descriptor close must not be separated by an await: the descriptor would
+	// otherwise keep feeding the archived inode while the event loop runs. Closing the rotating
+	// logger's own descriptor (not the module-global one) is what makes the next write reopen a
+	// fresh log file rather than append to the moved — and, when compressing, unlinked — inode.
+	// `activeStats` is the caller's own stat of the live generation, when it has one: the size check
+	// must rename the generation it measured, and a second stat here could pick up a newer one.
+	const generation = rotateLogFileSync(
+		logPath,
 		rotatedLogPath,
-		`HDB-${new Date(Date.now()).toISOString().replaceAll(':', '-')}.log`
+		logger?.closeLogFile ?? hdbLogger.closeLogFile,
+		activeStats
+	);
+	const publishedPath = await publishArchivedGeneration(
+		generation,
+		compress ?? envMgr.get(CONFIG_PARAMS.LOGGING_ROTATION_COMPRESS),
+		(error) => hdbLogger.error('Error compressing rotated log', generation.archivePath, error)
 	);
-	// Move log file to rotated log path first (if we crash
-	// during compression, we don't want to restart the compression with a new file)
-	await fsProm.rename(logPath, fullRotateLogPath);
-	// Close the rotating logger's own file descriptor now that the file has moved. This must be the
-	// logger's own closeLogFile (which resets its internal logFD), not the module-global one — otherwise
-	// the descriptor stays open on the moved (and, when compressing, subsequently unlinked) inode until the
-	// logger's safety timeout fires, pinning disk space and sending any writes in that window into the
-	// rotated/deleted file. Closing it here makes the next write reopen a fresh log file immediately.
-	(logger?.closeLogFile ?? hdbLogger.closeLogFile)();
-	if (compress) {
-		logPath = fullRotateLogPath;
-		fullRotateLogPath += '.gz';
-		await pipe(createReadStream(logPath), createGzip(), createWriteStream(fullRotateLogPath));
-		await fsProm.unlink(logPath);
-	}
 
 	// This notify log will create a new log file after the previous one has been rotated. It's important to keep this log as notify
-	hdbLogger.notify(`hdb.log rotated, old log moved to ${fullRotateLogPath}`);
-	return fullRotateLogPath;
+	hdbLogger.notify(`hdb.log rotated, old log moved to ${publishedPath}`);
+	return publishedPath;
 }

package/utility/nodeIdentity.ts

@@ -0,0 +1,45 @@
+import { isIP } from 'node:net';
+
+/**
+ * A node's identity — node.hostname, and the replication.hostname / certificate it falls back to —
+ * must be a bare hostname or IP literal, not a URL: it becomes the certificate common name and the
+ * host replication advertises and dials, so a scheme/port/path/non-string corrupts both (harper#2218,
+ * where "ws://http://host:9926:9925" resolves to host "http"). Returns a reason clause when `value`
+ * is not a bare host, else undefined. An empty string is "unset" (a caller's cue to fall back), not a
+ * violation.
+ */
+export function bareHostViolation(value: unknown): string | undefined {
+	if (typeof value !== 'string') return `must be a string, but is a ${value === null ? 'null' : typeof value}`;
+	if (value === '') return undefined;
+	// The identity is stored unbracketed (bracketing belongs to URL construction). Reject a bracketed
+	// form so it is never cached — otherwise net.isIP fails on it and the certificate SAN is mistyped
+	// as DNS rather than IP, and the same node's identity string would differ by source.
+	if (value.includes('[') || value.includes(']'))
+		return 'must be an unbracketed IPv6 literal (e.g. "::1", not "[::1]")';
+	// A zone id ("fe80::1%eth0") passes net.isIP but has no valid URL authority, so exclude it here.
+	if (isIP(value) && !value.includes('%')) return undefined; // a bare IPv4/IPv6 literal is a valid identity
+	if (value.includes('://')) return 'must not include a URL scheme';
+	let url;
+	try {
+		// Parse under a special scheme so the host grammar matches the ws:// replication URLs that
+		// consume this value — a host this accepts is one the consumer accepts (a non-special scheme is
+		// more lenient and would pass e.g. "node%20" or "0x7f.1"). A port or path makes url.hostname
+		// differ from the input and is caught by the round-trip check below.
+		url = new URL(`https://${value}`);
+	} catch {
+		return 'is not a valid hostname';
+	}
+	// Report the port before the round-trip check: the parser strips a default port for the scheme it
+	// was given (":443" under https), which would otherwise surface as a vaguer "is not a bare hostname".
+	if (url.port || (url.hostname && value.slice(url.hostname.length).startsWith(':'))) {
+		return 'must not include a port';
+	}
+	if (url.username || url.password) return 'must not include credentials';
+	if (url.search) return 'must not include a query string';
+	if (url.hash) return 'must not include a fragment';
+	if (url.pathname && url.pathname !== '/') return 'must not include a path';
+	// The host must survive parsing unchanged (compared case-insensitively, since URL lowercases it);
+	// anything that does not round-trip carried extra syntax and is not a bare host.
+	if (url.hostname.toLowerCase() !== value.toLowerCase()) return 'is not a bare hostname';
+	return undefined;
+}

package/utility/npmUtilities.ts

@@ -12,7 +12,7 @@ import harperLogger from './logging/harper_logger.ts';
 
 import { CONFIG_PARAMS } from './hdbTerms.ts';
 import { getConfigPath } from '../config/configUtils.ts';
-import { nonInteractiveSpawn } from '../components/Application.ts';
+import { nonInteractiveSpawn, packageManagerInstallArguments } from '../components/Application.ts';
 import { withComponentPreparationLock } from '../components/componentPreparationLock.ts';
 import { isThreadRunning } from '../server/threads/manageThreads.js';
 
@@ -26,18 +26,18 @@ export async function installModules(req: any) {
 		'install_node_modules is deprecated. Dependencies are automatically installed on' +
 		' deploy, and install_node_modules can lead to inconsistent behavior';
 	harperLogger.warn(deprecationWarning, req.projects);
-	const validation = modulesValidator(req);
+	const { error: validation, value: validatedRequest } = modulesValidator(req);
 	if (validation) {
 		throw handleHDBError(validation, validation.message, HTTP_STATUS_CODES.BAD_REQUEST);
 	}
 
-	let { projects, dryRun } = req;
+	const { projects, dry_run: dryRun, allowInstallScripts } = validatedRequest;
 
 	const componentsRootDirPath = getConfigPath(CONFIG_PARAMS.COMPONENTSROOT);
 
 	const responseObject: any = {};
 
-	const args = ['install', '--force', '--omit=dev', '--json'];
+	const args = [...packageManagerInstallArguments('npm', allowInstallScripts, true), '--json'];
 	if (dryRun) args.push('--dry-run');
 
 	for (const project of projects) {
@@ -91,15 +91,19 @@ function parseNPMStdErr(stderr: string) {
 }
 
 /**
- * Validator for both installModules & auditModules
+ * Validator for installModules
  * @param {Object} req
- * @returns {*}
  */
 function modulesValidator(req: any) {
+	// `dryRun` was the only spelling that worked before dry_run was honored; keep it accepted, but
+	// reject a request that carries both rather than picking a winner
 	const funcSchema = Joi.object({
 		projects: Joi.array().min(1).items(Joi.string()).required(),
 		dry_run: Joi.boolean().default(false),
-	});
+		allowInstallScripts: Joi.boolean().default(true),
+	})
+		.rename('dryRun', 'dry_run', { ignoreUndefined: true })
+		.rename('install_allow_scripts', 'allowInstallScripts', { ignoreUndefined: true });
 
-	return validator.validateBySchema(req, funcSchema);
+	return validator.validateAndConvertBySchema(req, funcSchema);
 }

package/utility/operationPermissions.ts

@@ -69,6 +69,7 @@ export const OPERATION_PERMISSION_GROUPS = {
 		OPERATIONS_ENUM.INSERT,
 		OPERATIONS_ENUM.UPDATE,
 		OPERATIONS_ENUM.UPSERT,
+		OPERATIONS_ENUM.PUT,
 		OPERATIONS_ENUM.DELETE,
 		// Bulk load
 		OPERATIONS_ENUM.CSV_DATA_LOAD,
@@ -97,6 +98,11 @@ const validGroups: Set<string> = new Set(Object.keys(OPERATION_PERMISSION_GROUPS
 // permission) whose names may fall outside OPERATIONS_ENUM. Tracked so they're grantable in a
 // role's `operations` allowlist (add_role/alter_role/impersonation validate against this set too).
 const dynamicallyRegisteredOps: Set<string> = new Set();
+// The same, for ops registered on a WORKER and mirrored here by the OPERATION_REGISTERED bridge.
+// Deliberately a separate set: the two threads can register the same name independently (a hot
+// deploy can add a `startOnMainThread` component while the worker offering that name is retiring),
+// so worker-lifecycle cleanup must not be able to revoke a mark this thread made itself.
+const workerRegisteredOps: Set<string> = new Set();
 
 /**
  * Mark a dynamically-registered operation name as a valid target for role `operations` grants.
@@ -115,13 +121,24 @@ export function unregisterGrantableOperation(name: string): void {
 	dynamicallyRegisteredOps.delete(name);
 }
 
+export function registerWorkerGrantableOperation(name: string): void {
+	workerRegisteredOps.add(name);
+}
+
+export function unregisterWorkerGrantableOperation(name: string): void {
+	workerRegisteredOps.delete(name);
+}
+
 /**
  * Validates that every entry in an operations array is a known operation name or group name.
  * Returns the first invalid entry, or null if all entries are valid.
  */
 export function validateOperations(operations: readonly unknown[]): string | null {
 	for (const op of operations) {
-		if (typeof op !== 'string' || (!validOps.has(op) && !validGroups.has(op) && !dynamicallyRegisteredOps.has(op))) {
+		if (
+			typeof op !== 'string' ||
+			(!validOps.has(op) && !validGroups.has(op) && !dynamicallyRegisteredOps.has(op) && !workerRegisteredOps.has(op))
+		) {
 			return String(op);
 		}
 	}