@harperfast/harper-pro 5.0.25 → 5.1.0-beta.1

package/core/bin/sseConsumer.ts

@@ -0,0 +1,126 @@
+import { StringDecoder } from 'node:string_decoder';
+import type { Readable } from 'node:stream';
+
+export interface SSEMessage {
+	event: string;
+	data: string;
+	id?: string;
+	retry?: number;
+}
+
+/**
+ * Parse a Readable carrying Server-Sent Events into structured messages.
+ *
+ * Yields one `SSEMessage` per blank-line-terminated record. Handles split data: lines,
+ * CRLF or LF line endings, and arbitrary chunk boundaries — the underlying Node http
+ * Readable does not guarantee chunks align with SSE record boundaries.
+ */
+export async function* parseSSE(stream: Readable): AsyncGenerator<SSEMessage> {
+	const decoder = new StringDecoder('utf8');
+	let buffer = '';
+	for await (const chunk of stream) {
+		buffer += typeof chunk === 'string' ? chunk : decoder.write(chunk);
+		while (true) {
+			const recordEnd = buffer.indexOf('\n\n');
+			const crlfEnd = buffer.indexOf('\r\n\r\n');
+			let endIdx = -1;
+			let delimLen = 0;
+			if (recordEnd !== -1 && (crlfEnd === -1 || recordEnd < crlfEnd)) {
+				endIdx = recordEnd;
+				delimLen = 2;
+			} else if (crlfEnd !== -1) {
+				endIdx = crlfEnd;
+				delimLen = 4;
+			}
+			if (endIdx === -1) break;
+			const record = buffer.slice(0, endIdx);
+			buffer = buffer.slice(endIdx + delimLen);
+			const msg = parseRecord(record);
+			if (msg) yield msg;
+		}
+	}
+	buffer += decoder.end();
+	// Any trailing record without a terminating blank line is treated as a final message,
+	// matching the looser behavior browsers exhibit on connection close.
+	if (buffer.trim()) {
+		const msg = parseRecord(buffer);
+		if (msg) yield msg;
+	}
+}
+
+function parseRecord(record: string): SSEMessage | null {
+	const lines = record.split(/\r?\n/);
+	let event = 'message';
+	let id: string | undefined;
+	let retry: number | undefined;
+	const dataLines: string[] = [];
+	for (const line of lines) {
+		if (line === '' || line.startsWith(':')) continue;
+		const colon = line.indexOf(':');
+		const field = colon === -1 ? line : line.slice(0, colon);
+		// Per spec, a leading space after the colon is stripped.
+		let value = colon === -1 ? '' : line.slice(colon + 1);
+		if (value.startsWith(' ')) value = value.slice(1);
+		switch (field) {
+			case 'event':
+				event = value;
+				break;
+			case 'data':
+				dataLines.push(value);
+				break;
+			case 'id':
+				id = value;
+				break;
+			case 'retry': {
+				const n = Number(value);
+				if (Number.isFinite(n)) retry = n;
+				break;
+			}
+		}
+	}
+	if (dataLines.length === 0 && event === 'message') return null;
+	return { event, data: dataLines.join('\n'), id, retry };
+}
+
+interface RenderState {
+	currentPhase?: string;
+}
+
+/**
+ * Render SSE deploy events as terse, line-oriented progress to stderr (so stdout stays
+ * reserved for the final JSON/YAML response document). Phase transitions print once.
+ */
+export function renderDeployProgress(message: SSEMessage, state: RenderState, output: NodeJS.WritableStream): void {
+	let parsed: unknown;
+	try {
+		parsed = JSON.parse(message.data);
+	} catch {
+		parsed = message.data;
+	}
+	switch (message.event) {
+		case 'phase': {
+			const p = parsed as { phase?: string; status?: string; rolling?: boolean };
+			const label = p.phase ?? '?';
+			if (p.status === 'start') {
+				if (state.currentPhase !== label) {
+					output.write(`${label}…\n`);
+					state.currentPhase = label;
+				}
+			} else if (p.status === 'done') {
+				output.write(`${label} done\n`);
+			} else if (p.status === 'error') {
+				const msg = (parsed as { message?: string }).message ?? 'failed';
+				output.write(`${label} ERROR: ${msg}\n`);
+			}
+			break;
+		}
+		case 'error': {
+			const e = parsed as { message?: string; code?: string | number };
+			output.write(`error: ${e.message ?? message.data}${e.code ? ` (${e.code})` : ''}\n`);
+			break;
+		}
+		case 'done':
+			// Caller picks up the final result via the SSE iterator; nothing to render here.
+			break;
+	}
+}

package/core/bin/{status.js → status.ts}

@@ -1,14 +1,14 @@
 'use strict';
 
-const fs = require('fs-extra');
-const path = require('path');
-const YAML = require('yaml');
+import * as fs from 'fs-extra';
+import * as path from 'path';
+import * as YAML from 'yaml';
 
-const hdbTerms = require('../utility/hdbTerms.ts');
-const hdbLog = require('../utility/logging/harper_logger.js');
-const systemInformation = require('../utility/environment/systemInformation.ts');
-const envMgr = require('../utility/environment/environmentManager.js');
-const installation = require('../utility/installation.ts');
+import * as hdbTerms from '../utility/hdbTerms.ts';
+import hdbLog from '../utility/logging/harper_logger.ts';
+import * as systemInformation from '../utility/environment/systemInformation.ts';
+import * as envMgr from '../utility/environment/environmentManager.ts';
+import * as installation from '../utility/installation.ts';
 envMgr.initSync();
 
 const STATUSES = {
@@ -20,10 +20,10 @@ const STATUSES = {
 
 let hdbRoot;
 
-module.exports = status;
+export default status;
 
 async function status() {
-	let status = {
+	let status: any = {
 		harperdb: {
 			status: STATUSES.STOPPED,
 		},

package/core/bin/stop.ts

@@ -0,0 +1,21 @@
+'use strict';
+
+import hdbLogger from '../utility/logging/harper_logger.ts';
+import * as util from 'util';
+import * as childProcess from 'child_process';
+const exec = util.promisify(childProcess.exec);
+import * as systemInformation from '../utility/environment/systemInformation.ts';
+
+const STOP_MSG = 'Stopping Harper.';
+
+export default stop;
+
+async function stop() {
+	console.log(STOP_MSG);
+	hdbLogger.notify(STOP_MSG);
+
+	const processes = await systemInformation.getHDBProcessInfo();
+	for (const { pid } of processes.core) {
+		exec(`kill ${pid}`);
+	}
+}

package/core/bin/upgrade.js

@@ -6,17 +6,17 @@
  * config file, a data model change requires a re-indexing script is run, etc.
  */
 
-const env = require('../utility/environment/environmentManager.js');
+const env = require('../utility/environment/environmentManager.ts');
 env.initSync();
 
 const chalk = require('chalk');
-const hdbLogger = require('../utility/logging/harper_logger.js');
+const hdbLogger = require('../utility/logging/harper_logger.ts');
 const hdbTerms = require('../utility/hdbTerms.ts');
-const directivesManager = require('../upgrade/directivesManager.js');
+const directivesManager = require('../upgrade/directivesManager.ts');
 const installation = require('../utility/installation.ts');
-const hdbInfoController = require('../dataLayer/hdbInfoController.js');
-const upgradePrompt = require('../upgrade/upgradePrompt.js');
-const globalSchema = require('../utility/globalSchema.js');
+const hdbInfoController = require('../dataLayer/hdbInfoController.ts');
+const upgradePrompt = require('../upgrade/upgradePrompt.ts');
+const globalSchema = require('../utility/globalSchema.ts');
 const { packageJson } = require('../utility/packageUtils.js');
 const promisify = require('util').promisify;
 const pSchemaToGlobal = promisify(globalSchema.setSchemaDataToGlobal);

package/core/components/Application.ts

@@ -1,7 +1,8 @@
 import { type Logger } from '../utility/logging/logger.ts';
 import { getConfigObj, getConfigValue, getConfigPath } from '../config/configUtils.js';
-import { CONFIG_PARAMS } from '../utility/hdbTerms.js';
-import logger from '../utility/logging/harper_logger.js';
+import { CONFIG_PARAMS } from '../utility/hdbTerms.ts';
+import logger from '../utility/logging/harper_logger.ts';
+import { broadcastDeployStart, broadcastDeployEnd } from './deployLifecycle.ts';
 
 import { dirname, extname, join } from 'node:path';
 import {
@@ -21,6 +22,7 @@ import { spawn } from 'node:child_process';
 import { createReadStream, existsSync, readdirSync } from 'node:fs';
 import { Readable } from 'node:stream';
 import { pipeline } from 'node:stream/promises';
+import { StringDecoder } from 'node:string_decoder';
 
 import { extract } from 'tar-fs';
 import gunzip from 'gunzip-maybe';
@@ -108,6 +110,21 @@ export function assertApplicationConfig(
 	}
 }
 
+/**
+ * Returns true when npm/git stderr indicates an SSH authentication failure —
+ * git exits 128 with the standard "could not read" message, or the SSH layer
+ * reports a missing uid (no SSH daemon user), explicit publickey denial, or
+ * an unverified host key.
+ */
+export function isSSHAuthFailure(stderr: string): boolean {
+	return (
+		stderr.includes('Could not read from remote repository') ||
+		stderr.includes('Permission denied (publickey)') ||
+		stderr.includes('No user exists for uid') ||
+		stderr.includes('Host key verification failed')
+	);
+}
+
 /**
  * Extract an application given payload (content of the application) or package (npm-compatible identifier to the application).
  *
@@ -131,11 +148,20 @@ export async function extractApplication(application: Application) {
 	// Resolve the tarball from the input
 	let tarballPath: string;
 	let tarball: Readable;
+	let shouldDeleteTarball = false;
+
 	if (application.payload) {
-		// Given a payload, create a Readable from the Buffer or string
-		tarball = Readable.from(
-			application.payload instanceof Buffer ? application.payload : Buffer.from(application.payload, 'base64')
-		);
+		const payload = application.payload;
+		if (payload instanceof Readable) {
+			// Stream payloads (e.g. multipart file part from the operations API) are piped
+			// straight into extraction so multi-GB components don't have to materialize as a Buffer.
+			tarball = payload;
+		} else if (typeof payload === 'string') {
+			// base64 string payload
+			tarball = Readable.from(Buffer.from(payload, 'base64'));
+		} else {
+			tarball = Readable.from(payload as Buffer);
+		}
 	} else {
 		// Given a package, there are a a couple options
 		const parentDirPath = dirname(application.dirPath);
@@ -180,6 +206,12 @@ export async function extractApplication(application: Application) {
 				parentDirPath
 			);
 			if (code !== 0) {
+				if (isSSHAuthFailure(stderr)) {
+					throw new Error(
+						`Failed to deploy private repository ${application.packageIdentifier}: SSH access failed. Verify the repository URL, configure an SSH key on this Harper instance, ensure the key has access to the target repository, and confirm the host is present in the ssh/known_hosts file.`,
+						{ cause: new Error(stderr) }
+					);
+				}
 				throw new Error(`Failed to download package ${application.packageIdentifier}: ${stderr}`);
 			}
 
@@ -196,6 +228,7 @@ export async function extractApplication(application: Application) {
 			}
 
 			tarballPath = join(parentDirPath, packResult[0].filename);
+			shouldDeleteTarball = true;
 			tarball = createReadStream(tarballPath);
 		}
 	}
@@ -237,7 +270,7 @@ export async function extractApplication(application: Application) {
 	}
 
 	// Clean up the original tarball
-	if (tarballPath) {
+	if (shouldDeleteTarball && tarballPath) {
 		await rm(tarballPath, { force: true });
 	}
 }
@@ -275,12 +308,16 @@ export async function installApplication(application: Application) {
 	// If custom install command is specified, run it
 	if (application.install?.command) {
 		const [command, ...args] = application.install.command.split(' ');
+		const customOnLine = application.onInstallLine
+			? (stream: 'stdout' | 'stderr', line: string) => application.onInstallLine!(command, stream, line)
+			: undefined;
 		const { stdout, stderr, code } = await nonInteractiveSpawn(
 			application.name,
 			command,
 			args,
 			application.dirPath,
-			application.install?.timeout
+			application.install?.timeout,
+			customOnLine
 		);
 		// if it succeeds, return
 		if (code === 0) {
@@ -331,12 +368,16 @@ export async function installApplication(application: Application) {
 		// Would result in `pnpm@7` being used as the executable.
 		// Important note: an `npm` version should not be specifiable; the only valid npm version is the one installed alongside Node.js
 
+		const pmOnLine = application.onInstallLine
+			? (stream: 'stdout' | 'stderr', line: string) => application.onInstallLine!(packageManager.name, stream, line)
+			: undefined;
 		const { stdout, stderr, code } = await nonInteractiveSpawn(
 			application.name,
 			(application.packageManagerPrefix ? application.packageManagerPrefix + ' ' : '') + packageManager.name,
 			application.install?.allowInstallScripts ? ['install'] : ['install', '--ignore-scripts'], // All of `npm`, `yarn`, and `pnpm` support the `install` command. If we need to configure options here we may have to use some other defaults though
 			application.dirPath,
-			application.install?.timeout
+			application.install?.timeout,
+			pmOnLine
 		);
 
 		// if it succeeds, return
@@ -383,12 +424,16 @@ export async function installApplication(application: Application) {
 	const npmInstallArgs = application.install?.allowInstallScripts
 		? ['install', '--force']
 		: ['install', '--force', '--ignore-scripts'];
+	const npmOnLine = application.onInstallLine
+		? (stream: 'stdout' | 'stderr', line: string) => application.onInstallLine!('npm', stream, line)
+		: undefined;
 	const { stdout, stderr, code } = await nonInteractiveSpawn(
 		application.name,
 		(application.packageManagerPrefix ? application.packageManagerPrefix + ' ' : '') + 'npm',
 		npmInstallArgs,
 		application.dirPath,
-		application.install?.timeout
+		application.install?.timeout,
+		npmOnLine
 	);
 
 	// if it succeeds, return
@@ -409,27 +454,39 @@ export async function installApplication(application: Application) {
 	throw new Error(`Failed to install dependencies for ${application.name} using npm default. Exit code: ${code}`);
 }
 
+/**
+ * Callback invoked once per complete line of install stdout/stderr from
+ * `nonInteractiveSpawn`. Threaded through `installApplication` to the underlying spawn
+ * so a deploy can stream `npm install` output back to the caller as an SSE `install`
+ * event in real time, rather than waiting for the process to exit. Line-buffered so a
+ * chunk that splits mid-line never fires a partial line.
+ */
+export type OnInstallLine = (manager: string, stream: 'stdout' | 'stderr', line: string) => void;
+
 interface ApplicationOptions {
 	name: string;
-	payload?: Buffer | string;
+	payload?: Buffer | string | Readable;
 	packageIdentifier?: string;
 	install?: { command?: string; timeout?: number; allowInstallScripts?: boolean };
+	onInstallLine?: OnInstallLine;
 }
 
 export class Application {
 	name: string;
-	payload?: Buffer | string;
+	payload?: Buffer | string | Readable;
 	packageIdentifier?: string;
 	install?: { command?: string; timeout?: number; allowInstallScripts?: boolean };
+	onInstallLine?: OnInstallLine;
 	dirPath: string;
 	logger: Logger;
 	packageManagerPrefix: string; // can be used to configure a package manager prefix, specifically "sfw".
 
-	constructor({ name, payload, packageIdentifier, install }: ApplicationOptions) {
+	constructor({ name, payload, packageIdentifier, install, onInstallLine }: ApplicationOptions) {
 		this.name = name;
 		this.payload = payload;
 		this.packageIdentifier = packageIdentifier && derivePackageIdentifier(packageIdentifier);
 		this.install = install;
+		this.onInstallLine = onInstallLine;
 		const componentsRoot = getConfigPath(CONFIG_PARAMS.COMPONENTSROOT);
 		if (!componentsRoot) throw new Error('componentsRoot is not configured');
 		this.dirPath = join(componentsRoot, name);
@@ -462,11 +519,24 @@ export function derivePackageIdentifier(packageIdentifier: string) {
  *
  * This method should only be called from the main thread
  *
+ * Bracketed with `deploy:start`/`deploy:end` lifecycle broadcasts so every
+ * Harper thread's file watchers can suppress restart-on-change events while
+ * the component directory is being rewritten — see harper#488 and
+ * `components/deployLifecycle.ts`. The broadcast is best-effort: if it fails
+ * (e.g. workers haven't started yet during initial install), the deploy still
+ * proceeds.
+ *
  * @param application The application to prepare.
  * @returns A promise that resolves when all preparation steps complete.
  */
-export function prepareApplication(application: Application) {
-	return extractApplication(application).then(() => installApplication(application));
+export async function prepareApplication(application: Application) {
+	await broadcastDeployStart(application.name);
+	try {
+		await extractApplication(application);
+		await installApplication(application);
+	} finally {
+		broadcastDeployEnd(application.name);
+	}
 }
 
 /**
@@ -581,12 +651,53 @@ function getGitSSHCommand() {
  * @param timeoutMs The timeout for the command in milliseconds. Defaults to 5 minutes.
  * @returns A promise that resolves when the command completes.
  */
+/**
+ * Line-buffered split that emits complete `\n`-terminated lines as they
+ * arrive, holding any partial trailing fragment until the next chunk or `flush()`.
+ * Required because `child_process` stdout/stderr `'data'` events fire per OS-level
+ * chunk, with no guarantee a chunk ends on a newline — without buffering, a long
+ * `npm install` line could be reported to the caller as two halves.
+ *
+ * Uses StringDecoder so a multi-byte UTF-8 character (e.g. the ✔ emoji npm prints
+ * for resolved packages) split across two chunks is reassembled into a single code
+ * point rather than each half being decoded as replacement characters.
+ */
+function createLineSplitter(onLine: (line: string) => void): {
+	push: (chunk: Buffer | string) => void;
+	flush: () => void;
+} {
+	const decoder = new StringDecoder('utf8');
+	let pending = '';
+	return {
+		push(chunk) {
+			pending += typeof chunk === 'string' ? chunk : decoder.write(chunk);
+			let nl: number;
+			while ((nl = pending.indexOf('\n')) !== -1) {
+				const line = pending.slice(0, nl).replace(/\r$/, '');
+				pending = pending.slice(nl + 1);
+				onLine(line);
+			}
+		},
+		flush() {
+			// Drain any bytes the decoder is still holding (e.g. a multi-byte char that
+			// straddled the final chunk boundary).
+			const remaining = decoder.end();
+			if (remaining) pending += remaining;
+			if (pending.length > 0) {
+				onLine(pending);
+				pending = '';
+			}
+		},
+	};
+}
+
 export function nonInteractiveSpawn(
 	applicationName: string,
 	command: string,
 	args: string[],
 	cwd: string,
-	timeoutMs: number = 60 * 60 * 1000
+	timeoutMs: number = 60 * 60 * 1000,
+	onLine?: (stream: 'stdout' | 'stderr', line: string) => void
 ): Promise<{ stdout: string; stderr: string; code: number }> {
 	return new Promise((resolve, reject) => {
 		logger
@@ -600,6 +711,10 @@ export function nonInteractiveSpawn(
 			env.GIT_SSH_COMMAND = gitSSHCommand;
 		}
 
+		if (process.platform === 'win32' && command === 'npm') {
+			command = 'npm.cmd';
+		}
+
 		const childProcess = spawn(command, args, {
 			shell: true,
 			cwd,
@@ -612,24 +727,31 @@ export function nonInteractiveSpawn(
 			reject(new Error(`Command\`${command} ${args.join(' ')}\` timed out after ${timeoutMs}ms`));
 		}, timeoutMs);
 
+		// If a caller passed onLine, line-buffer stdout/stderr alongside the existing
+		// string accumulation so we never report a half-line.
+		const stdoutSplitter = onLine ? createLineSplitter((line) => onLine('stdout', line)) : null;
+		const stderrSplitter = onLine ? createLineSplitter((line) => onLine('stderr', line)) : null;
+
 		let stdout = '';
 		childProcess.stdout.on('data', (chunk) => {
 			// buffer stdout for later resolve
 			stdout += chunk.toString();
 			// log stdout lines immediately
-			// TODO: Technically nothing guarantees that a chunk will be a complete line so need to implement
-			// something here to buffer until a newline character, then log the complete line
 			logger.loggerWithTag(`${applicationName}:spawn:${command}:stdout`).debug?.(chunk.toString());
+			stdoutSplitter?.push(chunk);
 		});
 
 		// buffer stderr
 		let stderr = '';
 		childProcess.stderr.on('data', (chunk) => {
 			stderr += chunk.toString();
+			stderrSplitter?.push(chunk);
 		});
 
 		childProcess.on('error', (error) => {
 			clearTimeout(timeout);
+			stdoutSplitter?.flush();
+			stderrSplitter?.flush();
 			// Print out stderr before rejecting
 			if (stderr) {
 				printStd(applicationName, command, stderr, 'stderr');
@@ -639,6 +761,10 @@ export function nonInteractiveSpawn(
 
 		childProcess.on('close', (code) => {
 			clearTimeout(timeout);
+			// Flush any trailing partial lines so the caller sees process output that didn't
+			// end on a newline (some package managers do this on their final progress line).
+			stdoutSplitter?.flush();
+			stderrSplitter?.flush();
 			if (stderr) {
 				printStd(applicationName, command, stderr, 'stderr');
 			}

package/core/components/ApplicationScope.ts

@@ -1,8 +1,8 @@
 import type { Resources } from '../resources/Resources.ts';
 import { type Server } from '../server/Server.ts';
-import { forComponent } from '../utility/logging/harper_logger.js';
+import { forComponent } from '../utility/logging/harper_logger.ts';
 import { scopedImport } from '../security/jsLoader.ts';
-import * as env from '../utility/environment/environmentManager.js';
+import * as env from '../utility/environment/environmentManager.ts';
 import { CONFIG_PARAMS } from '../utility/hdbTerms.ts';
 
 export class MissingDefaultFilesOptionError extends Error {

package/core/components/ComponentV1.ts

@@ -1,7 +1,7 @@
 import { isMainThread } from 'node:worker_threads';
 import fg from 'fast-glob';
 import { Resources } from '../resources/Resources.ts';
-import harperLogger from '../utility/logging/harper_logger.js';
+import harperLogger from '../utility/logging/harper_logger.ts';
 import { resolveBaseURLPath } from './resolveBaseURLPath.ts';
 import { deriveGlobOptions, FastGlobOptions, FilesOption } from './deriveGlobOptions.ts';
 import { basename, join } from 'node:path';
@@ -33,7 +33,7 @@ interface ComponentV1Details {
 }
 
 export class ComponentV1 {
-	readonly config: Readonly<ComponentV1Config>;
+	readonly config: ComponentV1Config;
 	readonly name: string;
 	readonly directory: string;
 	readonly module: Readonly<ComponentV1Module>;