@harperfast/harper 5.2.13 → 5.3.0-beta.1

package/config/harperConfigEnvVars.ts

@@ -25,13 +25,26 @@
 import type { Logger } from '../utility/logging/logger.ts';
 import * as fs from 'fs-extra';
 import * as path from 'node:path';
+import { isMainThread } from 'node:worker_threads';
 import * as crypto from 'node:crypto';
 import { cloneDeep } from 'lodash';
 import { getBackupDirPath } from './configHelpers.ts';
-import { atomicWriteFile } from './configUtils.ts';
+import { atomicWriteFile, renameWithRetry } from './configUtils.ts';
 import * as hdbTerms from '../utility/hdbTerms.ts';
 
 const STATE_FILE_NAME = '.harper-config-state.json';
+// Staged beside the confirmed state while a config-file write is in flight, then renamed over it.
+// Per-process, because every CLI invocation and worker runs this and one shared name would let one
+// clear another's in-flight commit. See DESIGN.md, boot-path config persistence.
+const PENDING_STATE_PREFIX = '.harper-config-state.pending.';
+const PENDING_STATE_SUFFIX = '.json';
+const pendingStateFileName = () => `${PENDING_STATE_PREFIX}${process.pid}${PENDING_STATE_SUFFIX}`;
+// Recovery from a recycled pid only has to be eventual, and deleting a slow-but-live writer's
+// sidecar is the worse error - it strands that writer's config file against an unpromoted state. So
+// the age-out is far longer than a commit (three synchronous steps) could ever legitimately take:
+// long enough that a stalled writer or clock skew between containers sharing a volume cannot reach
+// it, short enough that leaked wreckage does not suspend drift detection indefinitely.
+const PENDING_STATE_STALE_MS = 60 * 60 * 1000;
 
 /**
  * Get logger instance with tag - lazy loaded to avoid circular dependencies
@@ -546,15 +559,7 @@ function parseConfigEnvVar(envVarValue: string | undefined, envVarName: string):
 function loadConfigState(rootPath: string): ConfigState {
 	const statePath = path.join(getBackupDirPath(rootPath), STATE_FILE_NAME);
 
-	if (!fs.existsSync(statePath)) {
-		return {
-			version: '1.0',
-			sources: {},
-			originalValues: {},
-			emptyScopeOriginals: {},
-			snapshots: {},
-		};
-	}
+	if (!fs.existsSync(statePath)) return freshConfigState();
 
 	try {
 		const state = fs.readJsonSync(statePath) as ConfigState;
@@ -573,20 +578,85 @@ function loadConfigState(rootPath: string): ConfigState {
 		// If state file is corrupted, start fresh
 		const logger = getLogger();
 		logger.warn(`Failed to load config state file, starting fresh: ${(error as Error).message}`);
-		return {
-			version: '1.0',
-			sources: {},
-			originalValues: {},
-			emptyScopeOriginals: {},
-			snapshots: {},
-		};
+		return freshConfigState();
 	}
 }
 
+/**
+ * Clear a sidecar left by an interrupted commit, reporting whether one was there. The caller skips
+ * drift detection for that boot: it cannot tell a manual user edit from the write that was in
+ * flight, and calling it an edit hands those paths to 'user' for good.
+ */
+function takeInterruptedCommit(rootPath: string): boolean {
+	const backupDir = getBackupDirPath(rootPath);
+	let entries: string[];
+	try {
+		entries = fs.readdirSync(backupDir);
+	} catch {
+		return false;
+	}
+	let interrupted = false;
+	for (const entry of entries) {
+		if (!entry.startsWith(PENDING_STATE_PREFIX) || !entry.endsWith(PENDING_STATE_SUFFIX)) continue;
+		const pid = Number(entry.slice(PENDING_STATE_PREFIX.length, -PENDING_STATE_SUFFIX.length));
+		const entryPath = path.join(backupDir, entry);
+		if (pid !== process.pid && isProcessAlive(pid) && !isStale(entryPath)) {
+			// A live owner is mid-commit, so its sidecar is not wreckage to clear - but the pair it is
+			// halfway through is no more comparable than an interrupted one, so drift detection is off
+			// for this boot either way.
+			interrupted = true;
+			continue;
+		}
+		try {
+			fs.removeSync(entryPath);
+		} catch (error) {
+			// Keep drift detection on rather than disabling it every boot over a sidecar we cannot clear
+			getLogger().warn(`Could not remove an interrupted env config commit (${entry}): ${(error as Error).message}`);
+			continue;
+		}
+		interrupted = true;
+	}
+	if (interrupted) getLogger().warn('An env config commit was interrupted; skipping drift detection for this boot');
+	return interrupted;
+}
+
+function isStale(entryPath: string): boolean {
+	try {
+		return Date.now() - fs.statSync(entryPath).mtimeMs > PENDING_STATE_STALE_MS;
+	} catch {
+		return false;
+	}
+}
+
+function isProcessAlive(pid: number): boolean {
+	if (!Number.isInteger(pid) || pid <= 0) return false;
+	try {
+		process.kill(pid, 0);
+		return true;
+	} catch (error) {
+		return (error as NodeJS.ErrnoException).code === 'EPERM';
+	}
+}
+
+function freshConfigState(): ConfigState {
+	return {
+		version: '1.0',
+		sources: {},
+		originalValues: {},
+		emptyScopeOriginals: {},
+		snapshots: {},
+	};
+}
+
 /**
  * Save configuration state to file
  */
-function saveConfigState(rootPath: string, state: ConfigState): void {
+function serializeConfigState(state: ConfigState): string {
+	return JSON.stringify(state, null, 2) + '\n';
+}
+
+// Returns true when the file was rewritten, false when it already held this state.
+function saveConfigState(rootPath: string, state: ConfigState): boolean {
 	const backupDir = getBackupDirPath(rootPath);
 	const statePath = path.join(backupDir, STATE_FILE_NAME);
 
@@ -595,7 +665,49 @@ function saveConfigState(rootPath: string, state: ConfigState): void {
 
 	// Atomic write: a torn state file resets to fresh on the next load, losing every
 	// restoration record — the blast radius is user config-file content
-	atomicWriteFile(statePath, JSON.stringify(state, null, 2) + '\n');
+	return atomicWriteFile(statePath, serializeConfigState(state), { skipIfUnchanged: true });
+}
+
+function configStateMatchesDisk(rootPath: string, state: ConfigState): boolean {
+	try {
+		const statePath = path.join(getBackupDirPath(rootPath), STATE_FILE_NAME);
+		return fs.readFileSync(statePath, 'utf8') === serializeConfigState(state);
+	} catch {
+		return false;
+	}
+}
+
+function stageConfigState(rootPath: string, state: ConfigState): boolean {
+	const backupDir = getBackupDirPath(rootPath);
+	fs.ensureDirSync(backupDir);
+	return atomicWriteFile(path.join(backupDir, pendingStateFileName()), serializeConfigState(state));
+}
+
+// A rename, so an exhausted volume cannot refuse it and leave the config file described by nothing.
+
+export function commitStagedConfigState(rootPath: string): boolean {
+	const backupDir = getBackupDirPath(rootPath);
+	const pendingPath = path.join(backupDir, pendingStateFileName());
+	try {
+		renameWithRetry(pendingPath, path.join(backupDir, STATE_FILE_NAME));
+		return true;
+	} catch (error) {
+		// Loud: the config file is already on disk, so the confirmed state now describes values it no
+		// longer has, and the next boot reads that difference as a manual user edit.
+		getLogger().error(`Could not promote the staged env config state at ${pendingPath}: ${(error as Error).message}`);
+		return false;
+	}
+}
+
+// Drops the staged state for a config-file write that did not happen; the confirmed record stays.
+
+export function discardConfigState(rootPath: string): void {
+	const pendingPath = path.join(getBackupDirPath(rootPath), pendingStateFileName());
+	try {
+		fs.removeSync(pendingPath);
+	} catch (error) {
+		getLogger().warn(`Could not remove the staged env config state at ${pendingPath}: ${(error as Error).message}`);
+	}
 }
 
 /**
@@ -957,20 +1069,49 @@ export function applyRuntimeEnvConfig(
 	rootPath: string,
 	options: { isInstall?: boolean } = {}
 ): ConfigObject {
+	const { config, commitState } = prepareRuntimeEnvConfig(fileConfig, rootPath, options);
+	commitState();
+	return config;
+}
+
+/**
+ * Apply the env layers and hand the state writes back to the caller, so a caller that also persists
+ * the merged config file can commit the pair as a unit (#847): saveState() stages the new state
+ * beside the confirmed one, confirmConfigWritten() renames it over the confirmed one after the
+ * config file lands, and discardConfigState() drops the staged copy if that write never happens.
+ */
+export function prepareRuntimeEnvConfig(
+	fileConfig: ConfigObject,
+	rootPath: string,
+	options: { isInstall?: boolean } = {}
+): {
+	config: ConfigObject;
+	saveState: () => boolean;
+	confirmConfigWritten: () => boolean;
+	commitState: () => boolean;
+} {
 	const defaultEnvValue = process.env.HARPER_DEFAULT_CONFIG;
 	const configEnvValue = process.env.HARPER_CONFIG;
 	const setEnvValue = process.env.HARPER_SET_CONFIG;
 
-	// Load existing state
+	// Load existing state. Only the main thread persists, so only the main thread has wreckage to
+	// clear - and a worker shares its pid, so letting one scan would delete the main thread's
+	// in-flight sidecar as if it were last boot's.
+	const interruptedCommit = isMainThread && takeInterruptedCommit(rootPath);
 	const state = loadConfigState(rootPath);
 
 	// No env vars set and no previous state, nothing to do
 	if (!defaultEnvValue && !configEnvValue && !setEnvValue && Object.keys(state.snapshots).length === 0) {
-		return fileConfig;
+		return { config: fileConfig, saveState: () => false, confirmConfigWritten: () => false, commitState: () => false };
 	}
 
-	// Detect drift (user manual edits) - only at runtime, not install
-	if (!options.isInstall) {
+	// Detect drift (user manual edits) - only at runtime, not install, not on a boot that found an
+	// interrupted commit (where a difference could equally be the write that was in flight), and only
+	// on the main thread: a worker never owns the state, and one re-deriving inside the main thread's
+	// commit window would call its half-written config file a user edit and drop the env-supplied
+	// value for itself alone - serving different config than its siblings, with nothing on disk to
+	// show why.
+	if (!options.isInstall && !interruptedCommit && isMainThread) {
 		const driftedPaths = detectConfigDrift(fileConfig, state);
 		for (const path of driftedPaths) {
 			state.sources[path] = 'user';
@@ -998,8 +1139,10 @@ export function applyRuntimeEnvConfig(
 	processEnvVar(fileConfig, state, 'HARPER_CONFIG', 'HARPER_CONFIG', options);
 	processEnvVar(fileConfig, state, 'HARPER_SET_CONFIG', 'HARPER_SET_CONFIG', options);
 
-	// Save updated state
-	saveConfigState(rootPath, state);
-
-	return fileConfig;
+	return {
+		config: fileConfig,
+		saveState: () => (configStateMatchesDisk(rootPath, state) ? false : stageConfigState(rootPath, state)),
+		confirmConfigWritten: () => commitStagedConfigState(rootPath),
+		commitState: () => saveConfigState(rootPath, state),
+	};
 }

package/config/parseConfigFile.ts

@@ -0,0 +1,34 @@
+import { parse } from 'yaml';
+
+// yaml's prettyErrors frames the offending source lines into `message`, and a config file holds
+// credentials. Neither the cause nor the original stack is carried for the same reason.
+export class ConfigParseError extends Error {
+	constructor(filePath: string, error: unknown) {
+		const { name, code, linePos } = (error ?? {}) as {
+			name?: string;
+			code?: string;
+			linePos?: { line: number; col: number }[];
+		};
+		const at = linePos?.[0] ? ` at line ${linePos[0].line}, column ${linePos[0].col}` : '';
+		super(`Unable to parse the Harper configuration file at ${filePath}: ${code ?? name ?? 'parse failure'}${at}`);
+		this.name = 'ConfigParseError';
+	}
+}
+
+export function parseConfigFile(contents: string, filePath: string): any {
+	try {
+		// yaml routes warnings through `process.emitWarning` rather than a throw, so a framed
+		// warning would reach stderr with the config's own source lines in it, around this scrub.
+		return parse(contents, { logLevel: 'error' });
+	} catch (error) {
+		// Only yaml's own parse errors frame the source into `message`. Anything else is a fault in
+		// the parser, where the message is the whole of the debugging context.
+		if (!isYamlParseError(error)) throw error;
+		throw new ConfigParseError(filePath, error);
+	}
+}
+
+function isYamlParseError(error: unknown): boolean {
+	const { name, linePos } = (error ?? {}) as { name?: string; linePos?: unknown };
+	return linePos !== undefined || name === 'YAMLParseError';
+}

package/config/readConfigFileSync.ts

@@ -0,0 +1,44 @@
+import { readFileSync } from 'node:fs';
+
+// Why root config reads must not be async, and why this budget is shared rather than per call:
+// see "Root config watchers must read synchronously" in DESIGN.md (harper#2191).
+const READ_RETRY_BUDGET_MS = 500;
+const READ_RETRY_INITIAL_DELAY_MS = 10;
+const READ_RETRY_MAX_DELAY_MS = 100;
+const readRetrySleepBuffer = new Int32Array(new SharedArrayBuffer(4));
+const retryDeadlines = new Map<string, number>();
+
+// A caller that already owns a retry ladder passes `waitForLock` false, so one lock costs one
+// blocking window in total rather than one per rung.
+export function readConfigFileSync(filePath: string, waitForLock: boolean = true): string {
+	let delayMs = READ_RETRY_INITIAL_DELAY_MS;
+	while (true) {
+		try {
+			const contents = readFileSync(filePath, 'utf-8');
+			retryDeadlines.delete(filePath);
+			return contents;
+		} catch (error) {
+			const remainingBudgetMs = waitForLock && isSharingViolation(error) ? remainingRetryBudgetMs(filePath) : 0;
+			if (remainingBudgetMs <= 0) throw error;
+			Atomics.wait(readRetrySleepBuffer, 0, 0, Math.min(delayMs, remainingBudgetMs));
+			delayMs = Math.min(delayMs * 2, READ_RETRY_MAX_DELAY_MS);
+		}
+	}
+}
+
+function isSharingViolation(error: unknown): boolean {
+	if (process.platform !== 'win32') return false;
+	const code = (error as { code?: string } | null)?.code;
+	return code === 'EPERM' || code === 'EACCES' || code === 'EBUSY';
+}
+
+function remainingRetryBudgetMs(filePath: string): number {
+	const now = performance.now();
+	let deadline = retryDeadlines.get(filePath);
+	// A deadline more than one budget past its expiry belongs to an earlier burst, not this one.
+	if (deadline === undefined || now - deadline > READ_RETRY_BUDGET_MS) {
+		deadline = now + READ_RETRY_BUDGET_MS;
+		retryDeadlines.set(filePath, deadline);
+	}
+	return deadline - now;
+}

package/config/watcherArming.ts

@@ -0,0 +1,59 @@
+// See DESIGN.md, "`ready` means the watcher is armed". Platform is a parameter so a test on any
+// host can pin the darwin value, rather than only observing its own.
+export function armGraceMs(platform: string = process.platform): number {
+	return platform === 'darwin' ? 20 : 0;
+}
+
+const ARM_GRACE_MS = armGraceMs();
+
+/**
+ * The gate a config watcher opens when its chokidar watcher is really watching: the initial scan
+ * has finished *and* the platform's kernel-side warm-up has had its grace. Shared by
+ * `RootConfigWatcher` and `OptionsWatcher` because both read the config synchronously, which is
+ * what exposes the unarmed window in the first place.
+ */
+export class ArmGate {
+	#armed: boolean = false;
+	#timer: NodeJS.Timeout | undefined;
+	#graceMs: number;
+
+	// Taken as an argument, not read from the module constant, so the timer branch below is
+	// reachable from a host whose own platform has no grace — unit tests are ubuntu-only.
+	constructor(graceMs: number = ARM_GRACE_MS) {
+		this.#graceMs = graceMs;
+	}
+
+	get armed(): boolean {
+		return this.#armed;
+	}
+
+	// Synchronous where the platform needs no grace, so a watcher with no warm-up to wait out
+	// keeps reading inside chokidar's own dispatch.
+	arm(onArmed: () => void): void {
+		if (this.#armed || this.#timer) return;
+		if (!this.#graceMs) {
+			this.#armed = true;
+			onArmed();
+			return;
+		}
+		this.#timer = setTimeout(() => {
+			this.#timer = undefined;
+			this.#armed = true;
+			onArmed();
+		}, this.#graceMs);
+	}
+
+	// Drops a grace still counting down, for a watcher that is closing.
+	cancel(): void {
+		clearTimeout(this.#timer);
+		this.#timer = undefined;
+	}
+
+	// A replacement watcher has its own scan and its own unarmed window, so the gate arms again
+	// with it: for a file that is *absent* when the replacement scans, chokidar reports `ready` and
+	// nothing else, and the arming re-read is the only thing that would notice.
+	reset(): void {
+		this.cancel();
+		this.#armed = false;
+	}
+}

package/config-root.schema.json

@@ -172,6 +172,10 @@
 					]
 				},
 				"maxHeapMemory": { "type": "number", "description": "Heap memory limit per thread (MB)." },
+				"maxIsolated": {
+					"type": "integer",
+					"description": "Most applications marked isolated that get a dedicated worker thread; the rest fail to load. Default 8."
+				},
 				"heapSnapshotNearLimit": { "type": "boolean", "description": "Take a heap snapshot when near the heap limit." },
 				"preload": {
 					"oneOf": [{ "type": "string" }, { "type": "array", "items": { "type": "string" } }],
@@ -519,6 +523,10 @@
 					"type": "string",
 					"description": "Max write queue time before rejecting (e.g. '45s')."
 				},
+				"longTransactionReportThreshold": {
+					"type": ["string", "number"],
+					"description": "How old a RocksDB transaction handle must be before it is reported in the log (e.g. '5m'). A handle this old can hold write intents that park other writers' commits and pin a read snapshot. Set to 0 to disable the reporting. Default: '5m'"
+				},
 				"noReadAhead": { "type": "boolean", "description": "Advise OS to not read ahead. Default: false" },
 				"randomAccessFields": {
 					"type": "boolean",
@@ -534,6 +542,31 @@
 					"oneOf": [{ "type": "string" }, { "type": "array", "items": { "type": "string" } }],
 					"description": "Path or array of paths for blob storage. Default: <rootPath>/blobs"
 				},
+				"blobs": {
+					"type": "object",
+					"description": "File-backed blob storage settings.",
+					"properties": {
+						"compression": {
+							"type": "object",
+							"description": "Opt-in compression for newly written file-backed blobs, keyed by content type: an exact type ('application/json'), a wildcard ('text/*'), or 'default'. Absent means no compression. Each entry is false (never compress the type) or an object { codec, threshold }: codec currently only 'deflate' (the default); threshold is the minimum content size in bytes to compress (default 8192). Matching precedence is exact type, then 'type/*', then 'default'. Types that are already compressed (application/gzip, application/zip, application/zstd, image/*, video/*, audio/*) ship as false and stay uncompressed unless explicitly overridden. Blobs streamed without a known size are not compressed, and a threshold below 8192 only affects streamed writes — buffers under 8KB are stored inline in the record (covered by record-level compression) and never reach blob files. Applies to new writes only; existing blob files are never rewritten.",
+							"propertyNames": { "pattern": "^([\\w.+-]+/(\\*|[\\w.+-]+)|default)$" },
+							"additionalProperties": {
+								"oneOf": [
+									{ "const": false },
+									{
+										"type": "object",
+										"properties": {
+											"codec": { "const": "deflate" },
+											"threshold": { "type": "number", "minimum": 0 }
+										},
+										"additionalProperties": false
+									}
+								]
+							}
+						}
+					},
+					"additionalProperties": false
+				},
 				"blobReadTimeout": {
 					"type": "number",
 					"description": "Max time (ms) a blob read will wait for an in-progress write to finish before failing with a 503, rather than holding the connection. Default: 20000"