@harperfast/harper 5.2.13 → 5.3.0-beta.1

package/bin/deploySetup.ts

@@ -36,6 +36,7 @@ import {
 	normalizeGitHost,
 	projectNameFromPackage,
 	GIT_HOST_PATTERN,
+	isReservedComponentName,
 	PROJECT_NAME_PATTERN,
 } from '../utility/componentNames.ts';
 
@@ -74,6 +75,20 @@ export function resolveComponentName(req: any): string | undefined {
 	return undefined;
 }
 
+/** A credential sealed and granted to a name no deploy can run as is a secret nobody can ever use. */
+export function assertUsableComponentName(component: string): void {
+	if (!PROJECT_NAME_PATTERN.test(component)) {
+		throw cliError(
+			`"${component}" is not a usable component name — a deploy accepts letters, numbers, dashes and underscores.`
+		);
+	}
+	if (isReservedComponentName(component)) {
+		throw cliError(
+			`"${component}" is reserved for Harper's "${component}" configuration section — deploying a component under that name is refused.`
+		);
+	}
+}
+
 /**
  * The canonical bare host for a git credential — `https://github.com/owner/repo` and
  * `git@github.com` both identify `github.com`. Resolved once and then used for everything the host
@@ -185,11 +200,7 @@ export async function deploySetup(req: any): Promise<void> {
 				})
 			).project ?? ''
 		);
-	if (!PROJECT_NAME_PATTERN.test(component)) {
-		throw cliError(
-			`"${component}" is not a usable component name — a deploy accepts letters, numbers, dashes and underscores.`
-		);
-	}
+	assertUsableComponentName(component);
 
 	let credentialKey: string; // host (github) or registry (npm) — the credentials-entry discriminator
 	let credentialEntry: Record<string, string>;

package/bin/harper.ts

@@ -119,7 +119,7 @@ async function harper() {
 		case SERVICE_ACTIONS_ENUM.COPYDB: {
 			let sourceDb = process.argv[3];
 			let targetDbPath = process.argv[4];
-			return require('./copyDb').copyDb(sourceDb, targetDbPath);
+			return require('./copyDb').copyDb(sourceDb, targetDbPath, { blobs: 'copy' });
 		}
 		case OPERATIONS_ENUM.CREATE_BACKUP:
 		case OPERATIONS_ENUM.LIST_BACKUPS:

package/bin/help.ts

@@ -86,7 +86,10 @@ const SECTIONS: Section[] = [
 					['upgrade', 'Upgrade harperdb'],
 					['register', 'Register harperdb'],
 					['renew-certs', 'Generate a new set of self-signed certificates'],
-					['copy-db <source> <target>', 'Copies a database from source path to target path'],
+					[
+						'copy-db <source> <target>',
+						'Copies the database named <source> to the <target> environment path. File-backed blobs are copied to <target>-blobs; see its README before restoring.',
+					],
 					['version', 'Print the version'],
 					['help', 'Display this output'],
 				],

package/bin/lite.ts

@@ -1,2 +1,5 @@
 import { startHTTPThreads } from '../server/threads/socketRouter.ts';
-startHTTPThreads(1);
+startHTTPThreads(1).catch((error) => {
+	console.error(error);
+	process.exit(1);
+});

package/bin/restart.ts

@@ -9,6 +9,9 @@ import { compactOnStart } from './copyDb.ts';
 import {
 	beginProcessShutdown,
 	restartWorkers,
+	isThreadRunning,
+	decodeRestartScope,
+	getRunningIsolatedApplications,
 	onMessageByType,
 	shutdownWorkersNow,
 } from '../server/threads/manageThreads.js';
@@ -16,6 +19,8 @@ import { handleHDBError, hdbErrors } from '../utility/errors/hdbError.ts';
 const { HTTP_STATUS_CODES } = hdbErrors;
 import * as envMgr from '../utility/environment/environmentManager.ts';
 import * as path from 'node:path';
+import { getConfigObj, getConfigPath } from '../config/configUtils.ts';
+import { withComponentPreparationLock } from '../components/componentPreparationLock.ts';
 import { rmSync } from 'node:fs';
 import { getThisNodeName } from '../server/nodeName.ts';
 import { armRestartExitWatchdog } from './restartExitWatchdog.ts';
@@ -23,6 +28,7 @@ envMgr.initSync();
 
 const RESTART_RESPONSE = `Restarting Harper. This may take up to ${hdbTerms.RESTART_TIMEOUT_MS / 1000} seconds.`;
 const INVALID_SERVICE_ERR = 'Invalid service';
+const ISOLATED_TOPOLOGY_REQUEST_TIMEOUT_MS = 5000;
 
 let calledFromCli;
 
@@ -31,12 +37,72 @@ export { restart, restartService };
 // Add ITC event listener to main thread which will be called from child that receives restart request.
 if (isMainThread) {
 	onMessageByType(hdbTerms.ITC_EVENT_TYPES.RESTART, async (message, port) => {
-		if (message.workerType) await restartService({ service: message.workerType });
-		else restart({ operation: 'restart' });
-		port.postMessage({ type: 'restart-complete' });
+		try {
+			// `scope` stays in its wire form ('' pool, a name, absent = all) until restartService decodes it once
+			if (message.removeBranchesFor)
+				await restartThenRemoveBranches(message.workerType, message.removeBranchesFor, message.scope);
+			else if (message.workerType)
+				await restartService({
+					service: message.workerType,
+					scope: message.scope,
+					scopeFallback: message.scopeFallback,
+				});
+			else restart({ operation: 'restart' });
+		} finally {
+			port.postMessage({ type: 'restart-complete' });
+		}
 	});
 }
 
+/**
+ * Restart, then remove the branches of an application dropped on a worker (which cannot outlive the
+ * restart it asked for). The restart happens even if the component lock cannot be taken.
+ */
+async function restartThenRemoveBranches(service: string, project: string, scope: string | undefined): Promise<void> {
+	let restarted = false;
+	const restartHttpWorkers = async () => {
+		restarted = true;
+		processMan.expectedRestartOfChildren();
+		hdbLogger.notify('Restarting http_workers');
+		return restartWorkers('http', undefined, true, null, decodeRestartScope({ scope }));
+	};
+	try {
+		const componentPath = path.join(getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT) as string, project);
+		await withComponentPreparationLock(
+			componentPath,
+			async () => {
+				const outcome: any = await restartHttpWorkers();
+				if (!outcome || outcome.declined || outcome.workersKeptOnOldCode) {
+					hdbLogger.warn(
+						`Branched database storage of ${project} was left in place: the restart did not replace every worker`
+					);
+					return;
+				}
+				// The worker's drop lock was released before this one was taken; a same-name deploy that landed
+				// in between owns these branches now.
+				if (getConfigObj()?.[project]) {
+					hdbLogger.warn(`${project} was deployed again since it was dropped; leaving its branched databases in place`);
+					return;
+				}
+				const { removeBranchesForApplication } = await import('../resources/branchDatabase.ts');
+				await removeBranchesForApplication(project);
+			},
+			{
+				timeoutMs: 5 * 60 * 1000,
+				onWait: (owner) =>
+					hdbLogger.debug?.(
+						`Waiting to restart after dropping ${project}` +
+							(owner ? ` behind process ${owner.pid}, thread ${owner.threadId}` : '')
+					),
+				isOwnerAlive: (owner) => owner.pid !== process.pid || isThreadRunning(owner.threadId),
+			}
+		);
+	} catch (error) {
+		hdbLogger.error(`Could not remove the branched database storage of ${project}`, error);
+		if (!restarted) await restartService({ service, scope });
+	}
+}
+
 /**
  * Restart Harper.
  * It will restart all the child threads and the hub and leaf server processes.
@@ -127,6 +193,49 @@ async function restartService(req: any) {
 	if (hdbTerms.HDB_PROCESS_SERVICES[service] === undefined) {
 		throw handleHDBError(new Error(), INVALID_SERVICE_ERR, HTTP_STATUS_CODES.BAD_REQUEST, undefined, undefined, true);
 	}
+	const requestedScope = decodeRestartScope(req);
+	if (requestedScope !== undefined && typeof requestedScope !== 'string') {
+		throw handleHDBError(
+			new Error(),
+			'Invalid HTTP worker restart scope: expected a string',
+			HTTP_STATUS_CODES.BAD_REQUEST,
+			undefined,
+			undefined,
+			true
+		);
+	}
+	let fallbackScope;
+	if (req.scopeFallback !== undefined) {
+		fallbackScope = decodeRestartScope({ scope: req.scopeFallback });
+		if (fallbackScope !== undefined) {
+			throw handleHDBError(
+				new Error(),
+				'Invalid HTTP worker restart scope fallback: expected the pool scope',
+				HTTP_STATUS_CODES.BAD_REQUEST,
+				undefined,
+				undefined,
+				true
+			);
+		}
+	}
+	if (typeof requestedScope === 'string' && requestedScope !== '*' && req.scopeFallback === undefined) {
+		envMgr.initSync(true);
+		const { isIsolatedApplication } = await import('../server/threads/isolatedApplications.ts');
+		const configured = isIsolatedApplication(requestedScope);
+		const runningApplications = configured
+			? []
+			: await getRunningIsolatedApplications(ISOLATED_TOPOLOGY_REQUEST_TIMEOUT_MS);
+		if (!configured && !runningApplications.includes(requestedScope)) {
+			throw handleHDBError(
+				new Error(),
+				`Unknown isolated application restart scope: ${requestedScope}`,
+				HTTP_STATUS_CODES.BAD_REQUEST,
+				undefined,
+				undefined,
+				true
+			);
+		}
+	}
 	processMan.expectedRestartOfChildren();
 	if (!isMainThread) {
 		if (req.replicated) {
@@ -135,6 +244,8 @@ async function restartService(req: any) {
 		parentPort.postMessage({
 			type: hdbTerms.ITC_EVENT_TYPES.RESTART,
 			workerType: service,
+			scope: req.scope, // wire form, forwarded as received
+			scopeFallback: req.scopeFallback,
 		});
 		parentPort.ref(); // don't let the parent thread exit until we're done
 		await new Promise<void>((resolve) => {
@@ -209,7 +320,12 @@ async function restartService(req: any) {
 			if (calledFromCli) {
 				await processMan.restart(hdbTerms.PROCESS_DESCRIPTORS.HDB);
 			} else {
-				await restartWorkers('http');
+				let scope = requestedScope;
+				if (req.scopeFallback !== undefined && typeof scope === 'string' && scope !== '*') {
+					const runningApplications = await getRunningIsolatedApplications(ISOLATED_TOPOLOGY_REQUEST_TIMEOUT_MS);
+					if (!runningApplications.includes(scope)) scope = fallbackScope;
+				}
+				await restartWorkers('http', undefined, true, null, scope);
 			}
 			break;
 		default:

package/bin/run.ts

@@ -26,7 +26,7 @@ import { startHTTPThreads } from '../server/threads/socketRouter.ts';
 import { beginProcessShutdown } from '../server/threads/manageThreads.js';
 import * as hdbInfoController from '../dataLayer/hdbInfoController.ts';
 import { isReadOnlyMode } from '../resources/databases.ts';
-import { getThisNodeName } from '../server/nodeName.ts';
+import { getThisNodeName, getThisNodeHostname } from '../server/nodeName.ts';
 import * as hdbTerms from '../utility/hdbTerms.ts';
 import { getHdbPid, isProcessRunning } from '../utility/processManagement/processManagement.js';
 import { PACKAGE_ROOT } from '../utility/packageUtils.js';
@@ -287,6 +287,8 @@ function startupLog(portResolutions: any) {
 		logMsg += `${pad('Mode:')}${chalk.yellow('READ-ONLY')}\n`;
 	}
 
+	logMsg += `${pad('Version:')}${packageJson.version}\n`;
+
 	logMsg += `${pad('Hostname:')}${getThisNodeName()}\n`;
 
 	logMsg += `${pad('Worker Threads:')}${env.get(CONFIG_PARAMS.THREADS_COUNT)}\n`;
@@ -327,17 +329,10 @@ function startupLog(portResolutions: any) {
 	}`;
 	logMsg += `, unix socket: ${configUtils.getConfigPath(CONFIG_PARAMS.OPERATIONSAPI_NETWORK_DOMAINSOCKET)}\n`;
 	if (env.get(CONFIG_PARAMS.OPERATIONSAPI_NETWORK_PORT)) {
-		logMsg += pad('') + 'http://' + getThisNodeName() + ':' + env.get(CONFIG_PARAMS.OPERATIONSAPI_NETWORK_PORT) + '/\n';
+		logMsg += `${pad('')}http://${getThisNodeHostname()}:${env.get(CONFIG_PARAMS.OPERATIONSAPI_NETWORK_PORT)}/\n`;
 	}
 	if (env.get(CONFIG_PARAMS.OPERATIONSAPI_NETWORK_SECUREPORT)) {
-		logMsg +=
-			'\n' +
-			pad('') +
-			'https://' +
-			getThisNodeName() +
-			':' +
-			env.get(CONFIG_PARAMS.OPERATIONSAPI_NETWORK_SECUREPORT) +
-			'/\n';
+		logMsg += `\n${pad('')}https://${getThisNodeHostname()}:${env.get(CONFIG_PARAMS.OPERATIONSAPI_NETWORK_SECUREPORT)}/\n`;
 	}
 
 	// MQTT Log
@@ -389,7 +384,7 @@ function startupLog(portResolutions: any) {
 			if (!restLog.includes(pair) && name === 'rest') {
 				restLog += pair;
 				if (value.protocol_name === 'HTTP' || value.protocol_name === 'HTTPS') {
-					restHostnames.push(`${value.protocol_name.toLowerCase()}://${getThisNodeName()}:${key}/`);
+					restHostnames.push(`${value.protocol_name.toLowerCase()}://${getThisNodeHostname()}:${key}/`);
 				}
 			}
 

package/bin/upgrade.js

@@ -110,11 +110,15 @@ async function runUpgrade(upgradeObj) {
 		throw err;
 	}
 
+	// The stamp is what stops these directives running again on data they have already migrated.
 	try {
-		await hdbInfoController.insertHdbUpgradeInfo(upgradeObj[UPGRADE_VERSION]);
+		await hdbInfoController.insertHdbUpgradeInfo(upgradeObj[UPGRADE_VERSION] ?? packageJson.version);
 	} catch (err) {
-		hdbLogger.error("Error updating the 'hdb_info' system table.");
-		hdbLogger.error(err);
+		printToLogAndConsole(
+			`The data upgrade completed, but the new data version could not be recorded in the '${hdbTerms.SYSTEM_TABLE_NAMES.INFO_TABLE_NAME}' table, so Harper will not start. Every restart re-runs the upgrade directives against already-upgraded data, so stop any process supervisor that restarts Harper automatically, then check the logs and contact ${hdbTerms.HDB_SUPPORT_ADDRESS}.`,
+			hdbTerms.LOG_LEVELS.ERROR
+		);
+		throw err;
 	}
 }
 

package/bin/workloadIdentity.ts

@@ -0,0 +1,119 @@
+/**
+ * Client half of OIDC trusted publishing (#2171): ask the runtime for a workload identity token
+ * addressed to this instance, trade it for a short-lived operation token via `exchange_oidc_token`.
+ *
+ * Structured as a provider list because the runtimes differ only in how the token is obtained.
+ * GitHub Actions is entry one; a Kubernetes entry is `available()` testing for a projected
+ * service-account token path and `requestToken()` reading that file. A runtime none of them
+ * recognizes falls through to the CLI's other credential sources.
+ */
+
+import { httpRequest } from '../utility/common_utils.ts';
+
+interface WorkloadIdentityProvider {
+	name: string;
+	/** True when this process can obtain a token from this runtime. */
+	available(): boolean;
+	/**
+	 * Obtains an identity token bound to `audience`. Binding it is what makes the token unusable
+	 * anywhere else — see SHARED_DEFAULT_AUDIENCE in security/authn/oidc/providers/githubActions.ts
+	 * for what an unbound one costs.
+	 */
+	requestToken(audience: string): Promise<string>;
+}
+
+/** GitHub sets both of these on a job that declares `permissions: id-token: write`. */
+const GITHUB_TOKEN_REQUEST_URL = 'ACTIONS_ID_TOKEN_REQUEST_URL';
+const GITHUB_TOKEN_REQUEST_TOKEN = 'ACTIONS_ID_TOKEN_REQUEST_TOKEN';
+
+const IDENTITY_REQUEST_TIMEOUT_MS = 10_000;
+
+const githubActions: WorkloadIdentityProvider = {
+	name: 'GitHub Actions',
+
+	/**
+	 * Both variables are required: GitHub sets them together, so their absence means the workflow did
+	 * not grant `id-token: write` — a configuration answer, not a failure to report here.
+	 */
+	available(): boolean {
+		return Boolean(process.env[GITHUB_TOKEN_REQUEST_URL] && process.env[GITHUB_TOKEN_REQUEST_TOKEN]);
+	},
+
+	async requestToken(audience: string): Promise<string> {
+		const requestUrl = new URL(process.env[GITHUB_TOKEN_REQUEST_URL] as string);
+		requestUrl.searchParams.set('audience', audience);
+
+		const response = await fetch(requestUrl, {
+			headers: {
+				authorization: `Bearer ${process.env[GITHUB_TOKEN_REQUEST_TOKEN]}`,
+				accept: 'application/json',
+			},
+			signal: AbortSignal.timeout(IDENTITY_REQUEST_TIMEOUT_MS),
+		});
+		if (!response.ok) throw new Error(`GitHub returned ${response.status} for an identity token`);
+
+		const body: any = await response.json();
+		if (typeof body?.value !== 'string' || body.value === '') {
+			throw new Error('GitHub returned no identity token value');
+		}
+		return body.value;
+	},
+};
+
+const PROVIDERS: WorkloadIdentityProvider[] = [githubActions];
+
+function activeProvider(): WorkloadIdentityProvider | undefined {
+	return PROVIDERS.find((provider) => provider.available());
+}
+
+/** True when this runtime can prove its own identity to the cluster. */
+export function workloadIdentityAvailable(): boolean {
+	return activeProvider() !== undefined;
+}
+
+/**
+ * Trades a workload identity token for a Harper operation token, or undefined when this runtime has
+ * no identity to offer. Failures are reported and swallowed: this is the last credential source
+ * before the request goes out unauthenticated, and the resulting 401 says nothing useful, so the
+ * reason is worth printing even though it is not by itself fatal.
+ */
+export async function exchangeWorkloadIdentityForToken(options: any, audience: string): Promise<string | undefined> {
+	const provider = activeProvider();
+	if (!provider) return undefined;
+
+	console.error(`Requesting a ${provider.name} identity token for ${audience}...`);
+	let identityToken: string;
+	try {
+		identityToken = await provider.requestToken(audience);
+	} catch (error) {
+		console.error(`Could not obtain a ${provider.name} identity token: ${(error as Error).message}`);
+		return undefined;
+	}
+
+	try {
+		const response = await httpRequest(options, { operation: 'exchange_oidc_token', token: identityToken });
+		if (response.statusCode === 200) {
+			const data = JSON.parse(response.body);
+			if (data.operation_token) {
+				console.error(`Authenticated as '${data.username}' via OIDC trust policy '${data.policy}'.`);
+				return data.operation_token;
+			}
+			console.error('The OIDC exchange returned no operation token.');
+			return undefined;
+		}
+		if (response.statusCode === 401) {
+			// The server deliberately does not say which check failed, so point at the two things the
+			// operator can actually inspect rather than inventing a cause.
+			console.error(
+				'Harper rejected the identity token. Check that a trust policy matches this workload ' +
+					'(list_oidc_trust) and that its audience is this instance; the server log records the reason.'
+			);
+			return undefined;
+		}
+		console.error(`OIDC exchange failed: ${response.statusCode}`);
+		return undefined;
+	} catch (error) {
+		console.error(`Error exchanging the identity token: ${(error as Error).message}`);
+		return undefined;
+	}
+}