harper 5.1.20 → 5.2.0-alpha.5

package/components/operations.js

@@ -11,6 +11,14 @@ const hdbTerms = require('../utility/hdbTerms.ts');
 const env = require('../utility/environment/environmentManager.ts');
 const configUtils = require('../config/configUtils.ts');
 const hdbUtils = require('../utility/common_utils.ts');
+const {
+	isEnvFile,
+	isProtectedEnvFile,
+	parseEnvKeys,
+	renderMaskedEnv,
+	upsertEnvValues,
+	removeEnvKeys,
+} = require('../utility/envFile.ts');
 const { handleHDBError, ServerError, hdbErrors } = require('../utility/errors/hdbError.ts');
 const { HDB_ERROR_MSGS, HTTP_STATUS_CODES } = hdbErrors;
 const manageThreads = require('../server/threads/manageThreads.js');
@@ -18,7 +26,7 @@ const { packageDirectory } = require('../components/packageComponent.ts');
 const { Resources } = require('../resources/Resources.ts');
 const { Application, prepareApplication, ASIDE_STAGING_DIR } = require('./Application.ts');
 const { server } = require('../server/Server.ts');
-const { DeploymentRecorder, awaitDeploymentRow } = require('./deploymentRecorder.ts');
+const { DeploymentRecorder, awaitDeploymentRow, DEFAULT_AWAIT_ROW_TIMEOUT_MS } = require('./deploymentRecorder.ts');
 const { ProgressEmitter } = require('../server/serverHelpers/progressEmitter.ts');
 
 /**
@@ -362,6 +370,16 @@ async function deployComponent(req) {
 		throw handleHDBError(validation, validation.message, HTTP_STATUS_CODES.BAD_REQUEST);
 	}
 
+	// Ingest any provided registry token into the secrets store so the credential lives as
+	// replicated ciphertext (reference, not embed); already-reference entries pass through, and with
+	// no custody a literal token stays as a transient, this-node-only fallback (#1158). Peers
+	// re-running a replicated deploy already carry references and never re-ingest.
+	const { ingestRegistryAuth, resolveRegistryAuth } = require('./secretOperations.ts');
+	req.registryAuth = await ingestRegistryAuth(req, req.registryAuth, req.project);
+	// References are safe to persist (config + deployment row) and replicate; a no-custody literal
+	// token is not — it is used only for this node's install below, then stripped before replication.
+	const registryAuthReferences = (req.registryAuth ?? []).filter((entry) => entry && entry.secret !== undefined);
+
 	// Write to root config if the request contains a package identifier
 	if (req.package) {
 		// Check if trying to overwrite a core component (requires force)
@@ -385,6 +403,9 @@ async function deployComponent(req) {
 			};
 		}
 		if (req.urlPath !== undefined) applicationConfig.urlPath = req.urlPath;
+		// Persist registry-auth references (never tokens) so every cold install of this component —
+		// reboot, new peer, rollback — re-resolves the credential from the store.
+		if (registryAuthReferences.length) applicationConfig.registryAuth = registryAuthReferences;
 		await configUtils.addConfig(req.project, applicationConfig);
 	}
 
@@ -409,6 +430,8 @@ async function deployComponent(req) {
 				package_identifier: req.package ?? null,
 				user: req.hdb_user?.username,
 				restart_mode: req.restart === 'rolling' ? 'rolling' : req.restart ? 'immediate' : null,
+				// Reference form only — the rollback source for re-resolving registry auth.
+				registry_auth: registryAuthReferences.length ? registryAuthReferences : null,
 				emitter,
 			});
 	if (recorder) req._deploymentId = recorder.deploymentId;
@@ -446,6 +469,19 @@ async function deployComponent(req) {
 			extractionPayload = row.payload_blob.stream();
 		}
 
+		// Resolve registryAuth references into concrete tokens for this node's npm pack/install
+		// (a no-custody literal-token fallback passes through unchanged). On a peer running a
+		// replicated deploy, the referenced hdb_secret row may arrive just behind the deploy op, so
+		// allow a bounded grace period (same budget as the payload-row wait) for it to replicate in.
+		let registryAuthWaitMs = 0;
+		if (isReplicatedExecution) {
+			const requested = Number(req.deployment_timeout);
+			registryAuthWaitMs = Number.isFinite(requested) && requested >= 0 ? requested : DEFAULT_AWAIT_ROW_TIMEOUT_MS;
+		}
+		const resolvedRegistryAuth = await resolveRegistryAuth(req.registryAuth, req.project, {
+			waitMs: registryAuthWaitMs,
+		});
+
 		const application = new Application({
 			name: req.project,
 			payload: extractionPayload,
@@ -462,7 +498,16 @@ async function deployComponent(req) {
 				installCapture.push(manager, stream, line);
 				if (emitter) emit('install', { manager, stream, line });
 			},
+			// Private-registry auth (already resolved above), used here for this node's npm pack/install.
+			registryAuth: resolvedRegistryAuth,
 		});
+		// Reduce req.registryAuth to references only (never a token) before it can reach an error/log
+		// path or replication: references are what peers resolve from their own replicated hdb_secret
+		// copy; a no-custody literal token is dropped entirely (peers fall back to their fabric-injected
+		// NPM_CONFIG_USERCONFIG, as before). This also fixes the prior success-only strip that leaked a
+		// literal token on a prepare/load failure.
+		if (registryAuthReferences.length) req.registryAuth = registryAuthReferences;
+		else delete req.registryAuth;
 
 		emit('phase', { phase: 'prepare', status: 'start' });
 		await prepareApplication(application);
@@ -475,18 +520,32 @@ async function deployComponent(req) {
 			pseudoResources.isWorker = true;
 
 			const componentLoader = require('./componentLoader.ts').default || require('./componentLoader.ts');
+			const { trackScopeClose } = require('./scopeShutdown.ts');
 			let lastError;
 			componentLoader.setErrorReporter((error) => (lastError = error));
 			emit('phase', { phase: 'load', status: 'start' });
-			await componentLoader.loadComponent(
-				application.dirPath,
-				pseudoResources,
-				undefined,
-				false,
-				undefined,
-				false,
-				req.project
-			);
+			// This load exists only to surface load-time errors early; the Scopes it creates are
+			// throwaway. They are collected (instead of registered for worker-shutdown auto-close) so we
+			// can close them here once validation completes — otherwise each deploy leaks the Scope's
+			// deploy-lifecycle listeners on this worker, eventually tripping MaxListenersExceededWarning
+			// (#1462).
+			const validationScopes = new Set();
+			const validation = (async () => {
+				try {
+					await componentLoader.loadComponent(application.dirPath, pseudoResources, undefined, {
+						collectScopes: validationScopes,
+					});
+				} finally {
+					const closeResults = await Promise.allSettled(Array.from(validationScopes, (scope) => scope.close()));
+					for (const result of closeResults) {
+						if (result.status === 'rejected') log.warn('Failed to close a deploy-validation Scope', result.reason);
+					}
+				}
+			})();
+			// Track the load+close so a concurrent worker shutdown waits for these scopes to finish
+			// disposing — a plugin may start a native runtime in handleApplication — before realExit.
+			trackScopeClose(validation);
+			await validation;
 			emit('phase', { phase: 'load', status: 'done' });
 
 			if (lastError) throw lastError;
@@ -497,6 +556,9 @@ async function deployComponent(req) {
 		// ProgressEmitter holds function listeners that can't survive the replication
 		// channel's serialization; strip it unconditionally.
 		delete req.progress;
+		// req.registryAuth was already deleted immediately after the Application ctor (above) so the
+		// token never reaches the replication channel or a peer's operation log; peers authenticate
+		// against the private registry via their own fabric-injected NPM_CONFIG_USERCONFIG on reinstall.
 		if (systemReplicated && recorder) {
 			// The hdb_deployment row + payload_blob will reach peers via table replication,
 			// so peers can look up the payload by deployment_id. Drop req.payload to keep
@@ -748,6 +810,10 @@ async function getComponents() {
 						mtime: stats.mtime,
 						size: stats.size,
 					};
+					// Flag protected .env files so editors know their contents are masked by
+					// get_component_file and only editable via set_env_value. Template files
+					// (.env.example / .sample / .template) are not secret, so they are left alone.
+					if (isProtectedEnvFile(itemName)) res.protected = true;
 					result.entries.push(res);
 				}
 			}
@@ -762,7 +828,12 @@ async function getComponents() {
 		name: configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT).split(path.sep).slice(-1).pop(),
 		entries: [],
 	});
+	const { getUnsatisfiedEnv } = require('./componentSecrets.ts');
 	for (let entry of results.entries) {
+		// Declared-but-unsatisfied `env:` expectations (#1550) — metadata only (name, description,
+		// required, reason, tier), never values — so Studio/deploy output can render configure-me.
+		const unsatisfiedEnv = getUnsatisfiedEnv(entry.name);
+		if (unsatisfiedEnv.length > 0) entry.unsatisfiedEnv = unsatisfiedEnv;
 		const componentConfig = rootConfig?.[entry.name];
 		if (!componentConfig || typeof componentConfig !== 'object') continue;
 		if (componentConfig.package) entry.package = componentConfig.package;
@@ -846,6 +917,19 @@ async function getComponentFile(req) {
 				HTTP_STATUS_CODES.CONTENT_TOO_LARGE
 			);
 		}
+		// Protected .env files expose the key names (and a value-free masked rendering) but never
+		// the secret values. Template files (.env.example etc.) fall through and are read verbatim.
+		if (isProtectedEnvFile(req.file)) {
+			const keys = parseEnvKeys(await fs.readFile(filePath, 'utf8'));
+			return {
+				protected: true,
+				keys,
+				message: renderMaskedEnv(keys),
+				size: stats.size,
+				birthtime: stats.birthtime,
+				mtime: stats.mtime,
+			};
+		}
 		return {
 			message: await fs.readFile(filePath, options),
 			size: stats.size,
@@ -884,6 +968,113 @@ async function setComponentFile(req) {
 	return response;
 }
 
+/**
+ * Resolve the absolute path of a project's env file (defaulting to `.env`) and confirm it is one.
+ * @param req
+ * @returns {{file:string, filePath:string}}
+ */
+function resolveEnvFilePath(req) {
+	const file = req.file || '.env';
+	if (!isEnvFile(file)) {
+		const msg = `'${file}' is not a .env file`;
+		throw handleHDBError(new Error(msg), msg, HTTP_STATUS_CODES.BAD_REQUEST);
+	}
+	const compRoot = configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT);
+	return { file, filePath: path.join(compRoot, req.project, file) };
+}
+
+/**
+ * List the key names of a project's .env file. Never returns the secret values.
+ * @param req
+ * @returns {Promise<{file:string, keys:string[], size:number, mtime:Date}>}
+ */
+async function getEnvKeys(req) {
+	const validation = validator.getEnvKeysValidator(req);
+	if (validation) {
+		throw handleHDBError(validation, validation.message, HTTP_STATUS_CODES.BAD_REQUEST);
+	}
+
+	const { file, filePath } = resolveEnvFilePath(req);
+	try {
+		const [contents, stats] = await Promise.all([fs.readFile(filePath, 'utf8'), fs.stat(filePath)]);
+		return { file, keys: parseEnvKeys(contents), size: stats.size, mtime: stats.mtime };
+	} catch (err) {
+		if (err.code === hdbTerms.NODE_ERROR_CODES.ENOENT) {
+			throw new Error(`Component file not found '${path.join(req.project, file)}'`);
+		}
+		throw err;
+	}
+}
+
+/**
+ * Set one (`key` + `value`) or many (`values`) entries in a project's .env file, preserving all
+ * other keys, comments and formatting. Creates the file if it does not exist. Never echoes values.
+ * @param req
+ * @returns {Promise<{message:string, keys:string[]}>}
+ */
+async function setEnvValue(req) {
+	const validation = validator.setEnvValueValidator(req);
+	if (validation) {
+		throw handleHDBError(validation, validation.message, HTTP_STATUS_CODES.BAD_REQUEST);
+	}
+
+	const { file, filePath } = resolveEnvFilePath(req);
+	const updates = req.values ?? { [req.key]: req.value };
+
+	let existing = '';
+	try {
+		existing = await fs.readFile(filePath, 'utf8');
+	} catch (err) {
+		if (err.code !== hdbTerms.NODE_ERROR_CODES.ENOENT) throw err;
+	}
+
+	let updated;
+	try {
+		updated = upsertEnvValues(existing, updates);
+	} catch (err) {
+		throw handleHDBError(err, err.message, HTTP_STATUS_CODES.BAD_REQUEST);
+	}
+	await fs.outputFile(filePath, updated, 'utf8');
+
+	const response = await server.replication.replicateOperation(req);
+	response.message = `Successfully set env value(s) in ${file}`;
+	response.keys = parseEnvKeys(updated);
+	return response;
+}
+
+/**
+ * Remove one (`key`) or many (`keys`) entries from a project's .env file, leaving the rest intact.
+ * @param req
+ * @returns {Promise<{message:string, keys:string[]}>}
+ */
+async function deleteEnvValue(req) {
+	const validation = validator.deleteEnvValueValidator(req);
+	if (validation) {
+		throw handleHDBError(validation, validation.message, HTTP_STATUS_CODES.BAD_REQUEST);
+	}
+
+	const { file, filePath } = resolveEnvFilePath(req);
+	const keysToRemove = req.keys ?? [req.key];
+
+	let existing;
+	try {
+		existing = await fs.readFile(filePath, 'utf8');
+	} catch (err) {
+		if (err.code === hdbTerms.NODE_ERROR_CODES.ENOENT) {
+			throw new Error(`Component file not found '${path.join(req.project, file)}'`);
+		}
+		throw err;
+	}
+
+	const updated = removeEnvKeys(existing, keysToRemove);
+	await fs.outputFile(filePath, updated, 'utf8');
+
+	const response = await server.replication.replicateOperation(req);
+	response.message = `Successfully deleted env value(s) from ${file}`;
+	response.keys = parseEnvKeys(updated);
+	return response;
+}
+
 /**
  * Deletes a component dir/file
  * @param req
@@ -939,4 +1130,7 @@ exports.deployComponent = deployComponent;
 exports.getComponents = getComponents;
 exports.getComponentFile = getComponentFile;
 exports.setComponentFile = setComponentFile;
+exports.getEnvKeys = getEnvKeys;
+exports.setEnvValue = setEnvValue;
+exports.deleteEnvValue = deleteEnvValue;
 exports.dropComponent = dropComponent;

package/components/operationsValidation.js

@@ -9,10 +9,15 @@ const hdbLogger = require('../utility/logging/harper_logger.ts');
 const configUtils = require('../config/configUtils.ts');
 const { hdbErrors } = require('../utility/errors/hdbError.ts');
 const { HDB_ERROR_MSGS } = hdbErrors;
+const { ENV_ENCRYPTED_PREFIX } = require('../utility/envFile.ts');
 
 // File name can only be alphanumeric, dash and underscores
 const PROJECT_FILE_NAME_REGEX = /^[a-zA-Z0-9-_]+$/;
 
+// dotenv's accepted key character set. Restricting keys to this prevents a crafted key (e.g. one
+// containing `=` or a newline) from injecting extra assignments into a .env file.
+const ENV_KEY_REGEX = /^[\w.-]+$/;
+
 module.exports = {
 	getDropCustomFunctionValidator,
 	setCustomFunctionValidator,
@@ -23,6 +28,12 @@ module.exports = {
 	setComponentFileValidator,
 	getComponentFileValidator,
 	dropComponentFileValidator,
+	getEnvKeysValidator,
+	setEnvValueValidator,
+	deleteEnvValueValidator,
+	setSecretValidator,
+	grantSecretValidator,
+	deleteSecretValidator,
 };
 
 /**
@@ -169,6 +180,141 @@ function getComponentFileValidator(req) {
 	return validator.validateBySchema(req, getCompSchema);
 }
 
+/**
+ * Validate get_env_keys requests. `file` is optional and defaults to `.env` in the handler.
+ * @param req
+ * @returns {*}
+ */
+function getEnvKeysValidator(req) {
+	const schema = Joi.object({
+		// Patterned like the env writers (not the looser getComponentFile) so a `project` containing
+		// `..` can't traverse out of the components root when joined in resolveEnvFilePath.
+		project: Joi.string()
+			.pattern(PROJECT_FILE_NAME_REGEX)
+			.required()
+			.messages({ 'string.pattern.base': HDB_ERROR_MSGS.BAD_PROJECT_NAME }),
+		file: Joi.string().custom(checkFilePath).optional(),
+	});
+
+	return validator.validateBySchema(req, schema);
+}
+
+/**
+ * Validate set_env_value requests: exactly one of (`key` + `value`) or `values` (a key→value map).
+ * @param req
+ * @returns {*}
+ */
+function setEnvValueValidator(req) {
+	const schema = Joi.object({
+		project: Joi.string()
+			.pattern(PROJECT_FILE_NAME_REGEX)
+			.required()
+			.messages({ 'string.pattern.base': HDB_ERROR_MSGS.BAD_PROJECT_NAME }),
+		file: Joi.string().custom(checkFilePath).optional(),
+		key: Joi.string().pattern(ENV_KEY_REGEX),
+		value: Joi.string().allow(''),
+		// `.unknown(false)` rejects keys that don't match ENV_KEY_REGEX. Without it the schema-wide
+		// `allowUnknown: true` (see validateBySchema) would let an invalid key (e.g. with a space or
+		// newline) pass through unvalidated and corrupt the file.
+		values: Joi.object().pattern(ENV_KEY_REGEX, Joi.string().allow('')).unknown(false),
+	})
+		.with('key', 'value')
+		.with('value', 'key')
+		.xor('key', 'values');
+
+	return validator.validateBySchema(req, schema);
+}
+
+/**
+ * Validate delete_env_value requests: exactly one of `key` or `keys` (an array of key names).
+ * @param req
+ * @returns {*}
+ */
+function deleteEnvValueValidator(req) {
+	const schema = Joi.object({
+		project: Joi.string()
+			.pattern(PROJECT_FILE_NAME_REGEX)
+			.required()
+			.messages({ 'string.pattern.base': HDB_ERROR_MSGS.BAD_PROJECT_NAME }),
+		file: Joi.string().custom(checkFilePath).optional(),
+		key: Joi.string().pattern(ENV_KEY_REGEX),
+		keys: Joi.array().items(Joi.string().pattern(ENV_KEY_REGEX)).min(1),
+	}).xor('key', 'keys');
+
+	return validator.validateBySchema(req, schema);
+}
+
+// A secret name doubles as an env key when materialized, so it is held to the same character set.
+const SECRET_NAME = Joi.string()
+	.pattern(ENV_KEY_REGEX)
+	.required()
+	.messages({ 'string.pattern.base': `'name' must only contain word characters, dots and dashes` });
+
+// The encrypted-value marker followed by a base64url envelope body (structural validation of the
+// decoded JSON happens in the handler via parseEnvelopeFields). Derived from the shared prefix
+// constant so validator and handler can't drift; the prefix contains no regex metacharacters.
+// Trailing `=` padding is tolerated — some browser encoders emit padded base64url, and Node's
+// base64url decoder accepts either form.
+const SECRET_ENVELOPE_REGEX = new RegExp(`^${ENV_ENCRYPTED_PREFIX}[A-Za-z0-9_-]+={0,2}$`);
+
+// Size cap for secret values and envelopes: rows live forever in a replicated, audited system
+// table, so unbounded payloads are a storage/replication hazard, not a feature.
+const SECRET_MAX_LENGTH = 256 * 1024;
+
+/**
+ * Validate set_secret requests: `name` plus exactly one of `value` (plaintext) or `envelope`
+ * (`enc:v1:` ciphertext), with optional `metadata`, and a tier of either `processEnv` or `grants`
+ * (the handler rejects the two together — a processEnv secret is global, so scoping it is meaningless).
+ * @param req
+ * @returns {*}
+ */
+function setSecretValidator(req) {
+	const schema = Joi.object({
+		name: SECRET_NAME,
+		value: Joi.string().allow('').max(SECRET_MAX_LENGTH),
+		envelope: Joi.string()
+			.max(SECRET_MAX_LENGTH)
+			.pattern(SECRET_ENVELOPE_REGEX)
+			.messages({ 'string.pattern.base': `'envelope' must be an '${ENV_ENCRYPTED_PREFIX}' base64url envelope` }),
+		// Modest structural caps: metadata is a small free-form label object, not a payload store,
+		// and grants is a set (explicit duplicates rejected here; write paths also dedupe dirty state).
+		metadata: Joi.object().max(100),
+		grants: Joi.array().items(Joi.string().min(1)).max(100).unique(),
+		// process.env delivery tier; mutually exclusive with grants (enforced in the handler so the
+		// check also covers a grants add against an already-processEnv stored row).
+		processEnv: Joi.boolean(),
+	}).xor('value', 'envelope');
+
+	return validator.validateBySchema(req, schema);
+}
+
+/**
+ * Validate grant_secret / revoke_secret requests (same shape: `name` + `component`).
+ * @param req
+ * @returns {*}
+ */
+function grantSecretValidator(req) {
+	const schema = Joi.object({
+		name: SECRET_NAME,
+		component: Joi.string().min(1).required(),
+	});
+
+	return validator.validateBySchema(req, schema);
+}
+
+/**
+ * Validate delete_secret requests.
+ * @param req
+ * @returns {*}
+ */
+function deleteSecretValidator(req) {
+	const schema = Joi.object({
+		name: SECRET_NAME,
+	});
+
+	return validator.validateBySchema(req, schema);
+}
+
 /**
  * Validate addCustomFunctionProject requests.
  * @param req
@@ -252,6 +398,33 @@ function deployComponentValidator(req) {
 			})
 			.optional()
 			.messages({ 'any.invalid': 'urlPath must not contain ".."' }),
+		// Private-registry auth. Each entry supplies its credential exactly one of two ways:
+		//   - `token`: a literal token, used only for this node's npm pack/install and never
+		//     persisted or replicated (stripped from req before replicateOperation).
+		//   - `secret`: the name of an hdb_secret row (#1550); the token is resolved by decrypting
+		//     that row on this node at deploy time, so the credential lives in the secrets store
+		//     (reference, not embed) instead of travelling in the operation body.
+		registryAuth: Joi.array()
+			.items(
+				Joi.object({
+					// registry and token are written verbatim into the transient .npmrc, which is
+					// line-based; forbid CR/LF so a super_user can't inject extra npm config lines.
+					// (registry also accepts bare hosts and //host/ forms, so a strict URI validator
+					// would reject supported inputs — the newline guard is the right scope here.)
+					registry: Joi.string()
+						.pattern(/^[^\r\n]+$/)
+						.required(),
+					token: Joi.string().pattern(/^[^\r\n]+$/),
+					// A reference into the hdb_secret store; same name grammar as set_secret's `name`.
+					secret: Joi.string()
+						.pattern(ENV_KEY_REGEX)
+						.messages({ 'string.pattern.base': `'secret' must only contain word characters, dots and dashes` }),
+					scope: Joi.string()
+						.pattern(/^@[a-z0-9-_.]+$/)
+						.optional(),
+				}).xor('token', 'secret')
+			)
+			.optional(),
 	}).with('urlPath', 'package');
 
 	return validator.validateBySchema(req, deployProjSchema);