@harperfast/harper 5.0.30 → 5.1.0-beta.2

package/bin/{status.js → status.ts}

@@ -1,14 +1,14 @@
 'use strict';
 
-const fs = require('fs-extra');
-const path = require('path');
-const YAML = require('yaml');
+import * as fs from 'fs-extra';
+import * as path from 'path';
+import * as YAML from 'yaml';
 
-const hdbTerms = require('../utility/hdbTerms.ts');
-const hdbLog = require('../utility/logging/harper_logger.js');
-const systemInformation = require('../utility/environment/systemInformation.ts');
-const envMgr = require('../utility/environment/environmentManager.js');
-const installation = require('../utility/installation.ts');
+import * as hdbTerms from '../utility/hdbTerms.ts';
+import hdbLog from '../utility/logging/harper_logger.ts';
+import * as systemInformation from '../utility/environment/systemInformation.ts';
+import * as envMgr from '../utility/environment/environmentManager.ts';
+import * as installation from '../utility/installation.ts';
 envMgr.initSync();
 
 const STATUSES = {
@@ -20,10 +20,10 @@ const STATUSES = {
 
 let hdbRoot;
 
-module.exports = status;
+export default status;
 
 async function status() {
-	let status = {
+	let status: any = {
 		harperdb: {
 			status: STATUSES.STOPPED,
 		},

package/bin/{stop.js → stop.ts}

@@ -1,14 +1,14 @@
 'use strict';
 
-const hdbLogger = require('../utility/logging/harper_logger.js');
-const util = require('util');
-const childProcess = require('child_process');
+import hdbLogger from '../utility/logging/harper_logger.ts';
+import * as util from 'util';
+import * as childProcess from 'child_process';
 const exec = util.promisify(childProcess.exec);
-const systemInformation = require('../utility/environment/systemInformation.ts');
+import * as systemInformation from '../utility/environment/systemInformation.ts';
 
 const STOP_MSG = 'Stopping Harper.';
 
-module.exports = stop;
+export default stop;
 
 async function stop() {
 	console.log(STOP_MSG);

package/bin/upgrade.js

@@ -6,17 +6,16 @@
  * config file, a data model change requires a re-indexing script is run, etc.
  */
 
-const env = require('../utility/environment/environmentManager.js');
+const env = require('../utility/environment/environmentManager.ts');
 env.initSync();
 
 const chalk = require('chalk');
-const hdbLogger = require('../utility/logging/harper_logger.js');
+const hdbLogger = require('../utility/logging/harper_logger.ts');
 const hdbTerms = require('../utility/hdbTerms.ts');
-const directivesManager = require('../upgrade/directivesManager.js');
+const directivesManager = require('../upgrade/directivesManager.ts');
 const installation = require('../utility/installation.ts');
-const hdbInfoController = require('../dataLayer/hdbInfoController.js');
-const upgradePrompt = require('../upgrade/upgradePrompt.js');
-const globalSchema = require('../utility/globalSchema.js');
+const hdbInfoController = require('../dataLayer/hdbInfoController.ts');
+const globalSchema = require('../utility/globalSchema.ts');
 const { packageJson } = require('../utility/packageUtils.js');
 const promisify = require('util').promisify;
 const pSchemaToGlobal = promisify(globalSchema.setSchemaDataToGlobal);
@@ -71,24 +70,18 @@ async function upgrade(upgradeObj) {
 		process.exit(1);
 	}
 
-	let startUpgrade;
-
-	let exitCode = 0;
-	try {
-		startUpgrade = await upgradePrompt.forceUpdatePrompt(hdbUpgradeInfo);
-	} catch (err) {
-		hdbLogger.error('There was an error when prompting user about upgrade.');
-		hdbLogger.error(err);
-		startUpgrade = false;
-		exitCode = 1;
-	}
-
-	if (!startUpgrade) {
-		console.log('Cancelled upgrade, closing Harper');
-		process.exit(exitCode);
-	}
-
-	hdbLogger.info(`Starting upgrade to version ${currentHdbVersion}`);
+	// Upgrade directives run automatically; they do NOT require interactive confirmation.
+	// `upgrade()` only runs when an upgrade directive applies (getVersionUpdateInfo returns an
+	// object only when hasUpgradesRequired is true), and it runs on the normal `harper run`
+	// startup path (bin/run.ts). Blocking on a confirmation prompt there would hang on stdin —
+	// or, with no TTY, default to "no" and refuse to start — breaking unattended/scripted
+	// starts (systemd, containers, CI). We surface the upgrade as a non-blocking notice instead.
+	// (Downgrades still confirm via forceDowngradePrompt in hdbInfoController — running older
+	// software on upgraded data is the genuinely risky, lossy direction.)
+	printToLogAndConsole(
+		`Harper is completing an update to version ${currentHdbVersion}. You can read more about the changes in this upgrade at https://harperdb.io/developers/release-notes/`,
+		hdbTerms.LOG_LEVELS.INFO
+	);
 
 	await runUpgrade(hdbUpgradeInfo);
 

package/components/Application.ts

@@ -1,7 +1,8 @@
 import { type Logger } from '../utility/logging/logger.ts';
 import { getConfigObj, getConfigValue, getConfigPath } from '../config/configUtils.js';
-import { CONFIG_PARAMS } from '../utility/hdbTerms.js';
-import logger from '../utility/logging/harper_logger.js';
+import { CONFIG_PARAMS } from '../utility/hdbTerms.ts';
+import logger from '../utility/logging/harper_logger.ts';
+import { broadcastDeployStart, broadcastDeployEnd } from './deployLifecycle.ts';
 
 import { dirname, extname, join } from 'node:path';
 import {
@@ -21,6 +22,7 @@ import { spawn } from 'node:child_process';
 import { createReadStream, existsSync, readdirSync } from 'node:fs';
 import { Readable } from 'node:stream';
 import { pipeline } from 'node:stream/promises';
+import { StringDecoder } from 'node:string_decoder';
 
 import { extract } from 'tar-fs';
 import gunzip from 'gunzip-maybe';
@@ -108,6 +110,21 @@ export function assertApplicationConfig(
 	}
 }
 
+/**
+ * Returns true when npm/git stderr indicates an SSH authentication failure —
+ * git exits 128 with the standard "could not read" message, or the SSH layer
+ * reports a missing uid (no SSH daemon user), explicit publickey denial, or
+ * an unverified host key.
+ */
+export function isSSHAuthFailure(stderr: string): boolean {
+	return (
+		stderr.includes('Could not read from remote repository') ||
+		stderr.includes('Permission denied (publickey)') ||
+		stderr.includes('No user exists for uid') ||
+		stderr.includes('Host key verification failed')
+	);
+}
+
 /**
  * Extract an application given payload (content of the application) or package (npm-compatible identifier to the application).
  *
@@ -131,11 +148,20 @@ export async function extractApplication(application: Application) {
 	// Resolve the tarball from the input
 	let tarballPath: string;
 	let tarball: Readable;
+	let shouldDeleteTarball = false;
+
 	if (application.payload) {
-		// Given a payload, create a Readable from the Buffer or string
-		tarball = Readable.from(
-			application.payload instanceof Buffer ? application.payload : Buffer.from(application.payload, 'base64')
-		);
+		const payload = application.payload;
+		if (payload instanceof Readable) {
+			// Stream payloads (e.g. multipart file part from the operations API) are piped
+			// straight into extraction so multi-GB components don't have to materialize as a Buffer.
+			tarball = payload;
+		} else if (typeof payload === 'string') {
+			// base64 string payload
+			tarball = Readable.from(Buffer.from(payload, 'base64'));
+		} else {
+			tarball = Readable.from(payload as Buffer);
+		}
 	} else {
 		// Given a package, there are a a couple options
 		const parentDirPath = dirname(application.dirPath);
@@ -180,6 +206,12 @@ export async function extractApplication(application: Application) {
 				parentDirPath
 			);
 			if (code !== 0) {
+				if (isSSHAuthFailure(stderr)) {
+					throw new Error(
+						`Failed to deploy private repository ${application.packageIdentifier}: SSH access failed. Verify the repository URL, configure an SSH key on this Harper instance, ensure the key has access to the target repository, and confirm the host is present in the ssh/known_hosts file.`,
+						{ cause: new Error(stderr) }
+					);
+				}
 				throw new Error(`Failed to download package ${application.packageIdentifier}: ${stderr}`);
 			}
 
@@ -196,6 +228,7 @@ export async function extractApplication(application: Application) {
 			}
 
 			tarballPath = join(parentDirPath, packResult[0].filename);
+			shouldDeleteTarball = true;
 			tarball = createReadStream(tarballPath);
 		}
 	}
@@ -237,7 +270,7 @@ export async function extractApplication(application: Application) {
 	}
 
 	// Clean up the original tarball
-	if (tarballPath) {
+	if (shouldDeleteTarball && tarballPath) {
 		await rm(tarballPath, { force: true });
 	}
 }
@@ -275,12 +308,16 @@ export async function installApplication(application: Application) {
 	// If custom install command is specified, run it
 	if (application.install?.command) {
 		const [command, ...args] = application.install.command.split(' ');
+		const customOnLine = application.onInstallLine
+			? (stream: 'stdout' | 'stderr', line: string) => application.onInstallLine!(command, stream, line)
+			: undefined;
 		const { stdout, stderr, code } = await nonInteractiveSpawn(
 			application.name,
 			command,
 			args,
 			application.dirPath,
-			application.install?.timeout
+			application.install?.timeout,
+			customOnLine
 		);
 		// if it succeeds, return
 		if (code === 0) {
@@ -331,12 +368,16 @@ export async function installApplication(application: Application) {
 		// Would result in `pnpm@7` being used as the executable.
 		// Important note: an `npm` version should not be specifiable; the only valid npm version is the one installed alongside Node.js
 
+		const pmOnLine = application.onInstallLine
+			? (stream: 'stdout' | 'stderr', line: string) => application.onInstallLine!(packageManager.name, stream, line)
+			: undefined;
 		const { stdout, stderr, code } = await nonInteractiveSpawn(
 			application.name,
 			(application.packageManagerPrefix ? application.packageManagerPrefix + ' ' : '') + packageManager.name,
 			application.install?.allowInstallScripts ? ['install'] : ['install', '--ignore-scripts'], // All of `npm`, `yarn`, and `pnpm` support the `install` command. If we need to configure options here we may have to use some other defaults though
 			application.dirPath,
-			application.install?.timeout
+			application.install?.timeout,
+			pmOnLine
 		);
 
 		// if it succeeds, return
@@ -383,12 +424,16 @@ export async function installApplication(application: Application) {
 	const npmInstallArgs = application.install?.allowInstallScripts
 		? ['install', '--force']
 		: ['install', '--force', '--ignore-scripts'];
+	const npmOnLine = application.onInstallLine
+		? (stream: 'stdout' | 'stderr', line: string) => application.onInstallLine!('npm', stream, line)
+		: undefined;
 	const { stdout, stderr, code } = await nonInteractiveSpawn(
 		application.name,
 		(application.packageManagerPrefix ? application.packageManagerPrefix + ' ' : '') + 'npm',
 		npmInstallArgs,
 		application.dirPath,
-		application.install?.timeout
+		application.install?.timeout,
+		npmOnLine
 	);
 
 	// if it succeeds, return
@@ -409,27 +454,39 @@ export async function installApplication(application: Application) {
 	throw new Error(`Failed to install dependencies for ${application.name} using npm default. Exit code: ${code}`);
 }
 
+/**
+ * Callback invoked once per complete line of install stdout/stderr from
+ * `nonInteractiveSpawn`. Threaded through `installApplication` to the underlying spawn
+ * so a deploy can stream `npm install` output back to the caller as an SSE `install`
+ * event in real time, rather than waiting for the process to exit. Line-buffered so a
+ * chunk that splits mid-line never fires a partial line.
+ */
+export type OnInstallLine = (manager: string, stream: 'stdout' | 'stderr', line: string) => void;
+
 interface ApplicationOptions {
 	name: string;
-	payload?: Buffer | string;
+	payload?: Buffer | string | Readable;
 	packageIdentifier?: string;
 	install?: { command?: string; timeout?: number; allowInstallScripts?: boolean };
+	onInstallLine?: OnInstallLine;
 }
 
 export class Application {
 	name: string;
-	payload?: Buffer | string;
+	payload?: Buffer | string | Readable;
 	packageIdentifier?: string;
 	install?: { command?: string; timeout?: number; allowInstallScripts?: boolean };
+	onInstallLine?: OnInstallLine;
 	dirPath: string;
 	logger: Logger;
 	packageManagerPrefix: string; // can be used to configure a package manager prefix, specifically "sfw".
 
-	constructor({ name, payload, packageIdentifier, install }: ApplicationOptions) {
+	constructor({ name, payload, packageIdentifier, install, onInstallLine }: ApplicationOptions) {
 		this.name = name;
 		this.payload = payload;
 		this.packageIdentifier = packageIdentifier && derivePackageIdentifier(packageIdentifier);
 		this.install = install;
+		this.onInstallLine = onInstallLine;
 		const componentsRoot = getConfigPath(CONFIG_PARAMS.COMPONENTSROOT);
 		if (!componentsRoot) throw new Error('componentsRoot is not configured');
 		this.dirPath = join(componentsRoot, name);
@@ -462,11 +519,24 @@ export function derivePackageIdentifier(packageIdentifier: string) {
  *
  * This method should only be called from the main thread
  *
+ * Bracketed with `deploy:start`/`deploy:end` lifecycle broadcasts so every
+ * Harper thread's file watchers can suppress restart-on-change events while
+ * the component directory is being rewritten — see harper#488 and
+ * `components/deployLifecycle.ts`. The broadcast is best-effort: if it fails
+ * (e.g. workers haven't started yet during initial install), the deploy still
+ * proceeds.
+ *
  * @param application The application to prepare.
  * @returns A promise that resolves when all preparation steps complete.
  */
-export function prepareApplication(application: Application) {
-	return extractApplication(application).then(() => installApplication(application));
+export async function prepareApplication(application: Application) {
+	await broadcastDeployStart(application.name);
+	try {
+		await extractApplication(application);
+		await installApplication(application);
+	} finally {
+		broadcastDeployEnd(application.name);
+	}
 }
 
 /**
@@ -581,12 +651,53 @@ function getGitSSHCommand() {
  * @param timeoutMs The timeout for the command in milliseconds. Defaults to 5 minutes.
  * @returns A promise that resolves when the command completes.
  */
+/**
+ * Line-buffered split that emits complete `\n`-terminated lines as they
+ * arrive, holding any partial trailing fragment until the next chunk or `flush()`.
+ * Required because `child_process` stdout/stderr `'data'` events fire per OS-level
+ * chunk, with no guarantee a chunk ends on a newline — without buffering, a long
+ * `npm install` line could be reported to the caller as two halves.
+ *
+ * Uses StringDecoder so a multi-byte UTF-8 character (e.g. the ✔ emoji npm prints
+ * for resolved packages) split across two chunks is reassembled into a single code
+ * point rather than each half being decoded as replacement characters.
+ */
+function createLineSplitter(onLine: (line: string) => void): {
+	push: (chunk: Buffer | string) => void;
+	flush: () => void;
+} {
+	const decoder = new StringDecoder('utf8');
+	let pending = '';
+	return {
+		push(chunk) {
+			pending += typeof chunk === 'string' ? chunk : decoder.write(chunk);
+			let nl: number;
+			while ((nl = pending.indexOf('\n')) !== -1) {
+				const line = pending.slice(0, nl).replace(/\r$/, '');
+				pending = pending.slice(nl + 1);
+				onLine(line);
+			}
+		},
+		flush() {
+			// Drain any bytes the decoder is still holding (e.g. a multi-byte char that
+			// straddled the final chunk boundary).
+			const remaining = decoder.end();
+			if (remaining) pending += remaining;
+			if (pending.length > 0) {
+				onLine(pending);
+				pending = '';
+			}
+		},
+	};
+}
+
 export function nonInteractiveSpawn(
 	applicationName: string,
 	command: string,
 	args: string[],
 	cwd: string,
-	timeoutMs: number = 60 * 60 * 1000
+	timeoutMs: number = 60 * 60 * 1000,
+	onLine?: (stream: 'stdout' | 'stderr', line: string) => void
 ): Promise<{ stdout: string; stderr: string; code: number }> {
 	return new Promise((resolve, reject) => {
 		logger
@@ -600,6 +711,10 @@ export function nonInteractiveSpawn(
 			env.GIT_SSH_COMMAND = gitSSHCommand;
 		}
 
+		if (process.platform === 'win32' && command === 'npm') {
+			command = 'npm.cmd';
+		}
+
 		const childProcess = spawn(command, args, {
 			shell: true,
 			cwd,
@@ -612,24 +727,31 @@ export function nonInteractiveSpawn(
 			reject(new Error(`Command\`${command} ${args.join(' ')}\` timed out after ${timeoutMs}ms`));
 		}, timeoutMs);
 
+		// If a caller passed onLine, line-buffer stdout/stderr alongside the existing
+		// string accumulation so we never report a half-line.
+		const stdoutSplitter = onLine ? createLineSplitter((line) => onLine('stdout', line)) : null;
+		const stderrSplitter = onLine ? createLineSplitter((line) => onLine('stderr', line)) : null;
+
 		let stdout = '';
 		childProcess.stdout.on('data', (chunk) => {
 			// buffer stdout for later resolve
 			stdout += chunk.toString();
 			// log stdout lines immediately
-			// TODO: Technically nothing guarantees that a chunk will be a complete line so need to implement
-			// something here to buffer until a newline character, then log the complete line
 			logger.loggerWithTag(`${applicationName}:spawn:${command}:stdout`).debug?.(chunk.toString());
+			stdoutSplitter?.push(chunk);
 		});
 
 		// buffer stderr
 		let stderr = '';
 		childProcess.stderr.on('data', (chunk) => {
 			stderr += chunk.toString();
+			stderrSplitter?.push(chunk);
 		});
 
 		childProcess.on('error', (error) => {
 			clearTimeout(timeout);
+			stdoutSplitter?.flush();
+			stderrSplitter?.flush();
 			// Print out stderr before rejecting
 			if (stderr) {
 				printStd(applicationName, command, stderr, 'stderr');
@@ -639,6 +761,10 @@ export function nonInteractiveSpawn(
 
 		childProcess.on('close', (code) => {
 			clearTimeout(timeout);
+			// Flush any trailing partial lines so the caller sees process output that didn't
+			// end on a newline (some package managers do this on their final progress line).
+			stdoutSplitter?.flush();
+			stderrSplitter?.flush();
 			if (stderr) {
 				printStd(applicationName, command, stderr, 'stderr');
 			}

package/components/ApplicationScope.ts

@@ -1,8 +1,8 @@
 import type { Resources } from '../resources/Resources.ts';
 import { type Server } from '../server/Server.ts';
-import { forComponent } from '../utility/logging/harper_logger.js';
+import { forComponent } from '../utility/logging/harper_logger.ts';
 import { scopedImport } from '../security/jsLoader.ts';
-import * as env from '../utility/environment/environmentManager.js';
+import * as env from '../utility/environment/environmentManager.ts';
 import { CONFIG_PARAMS } from '../utility/hdbTerms.ts';
 
 export class MissingDefaultFilesOptionError extends Error {

package/components/ComponentV1.ts

@@ -1,7 +1,7 @@
 import { isMainThread } from 'node:worker_threads';
 import fg from 'fast-glob';
 import { Resources } from '../resources/Resources.ts';
-import harperLogger from '../utility/logging/harper_logger.js';
+import harperLogger from '../utility/logging/harper_logger.ts';
 import { resolveBaseURLPath } from './resolveBaseURLPath.ts';
 import { deriveGlobOptions, FastGlobOptions, FilesOption } from './deriveGlobOptions.ts';
 import { basename, join } from 'node:path';
@@ -33,7 +33,7 @@ interface ComponentV1Details {
 }
 
 export class ComponentV1 {
-	readonly config: Readonly<ComponentV1Config>;
+	readonly config: ComponentV1Config;
 	readonly name: string;
 	readonly directory: string;
 	readonly module: Readonly<ComponentV1Module>;