@harperfast/harper-pro 5.0.0-alpha.5 → 5.0.0-alpha.7

package/core/.github/workflows/integration-tests.yml

@@ -160,5 +160,16 @@ jobs:
           name: harper-build-artifacts-node-${{ matrix.node-version }}
 
       - name: Run Integration Test Shard ${{ matrix.shard }}
+        env:
+          HARPER_INTEGRATION_TEST_LOG_DIR: /tmp/harper-integration-test-logs
         run: |
           npm run test:integration -- --shard=${{ matrix.shard }}/4
+
+      - name: Upload Harper server logs
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: harper-server-logs-node-${{ matrix.node-version }}-shard-${{ matrix.shard }}
+          path: /tmp/harper-integration-test-logs/
+          retention-days: 3
+          if-no-files-found: ignore

package/core/.github/workflows/notify-release-published.yaml

@@ -20,13 +20,13 @@ jobs:
           payload: |
             {
               "channel": "#development-ci",
-              "text": "Harper v${{ github.event.release.tag_name }} GitHub release published",
+              "text": "Harper ${{ github.event.release.tag_name }} GitHub release published",
               "blocks": [
                 {
                   "type": "header",
                   "text": {
                     "type": "plain_text",
-                    "text": "Harper v${{ github.event.release.tag_name }} release"
+                    "text": "Harper ${{ github.event.release.tag_name }} release"
                   }
                 },
                 {

package/core/.github/workflows/unit-test.yml

@@ -50,6 +50,8 @@ jobs:
           HDB_ADMIN_PASSWORD: 'password'
           ROOTPATH: '/tmp/hdb'
           NODE_HOSTNAME: 'localhost'
+          LOGGING_LEVEL: 'info'
+
         run: node --enable-source-maps ./dist/bin/harper.js install
 
       - name: Run tests

package/core/bin/cliOperations.js

@@ -12,60 +12,6 @@ const { encode } = require('cbor-x');
 const { getHdbPid } = require('../utility/processManagement/processManagement.js');
 const { initConfig } = require('../config/configUtils.js');
 
-const SUPPORTED_OPS = [
-	'describe_table',
-	'describe_all',
-	'describe_database',
-	'list_users',
-	'list_roles',
-	'drop_role',
-	'add_user',
-	'alter_user',
-	'drop_user',
-	'restart_service',
-	'restart',
-	'create_database',
-	'drop_database',
-	'create_table',
-	'drop_table',
-	'create_attribute',
-	'drop_attribute',
-	'search_by_id',
-	'insert',
-	'update',
-	'upsert',
-	'delete',
-	'search_by_value',
-	'csv_file_load',
-	'csv_url_load',
-	'add_component',
-	'deploy_component',
-	'package_component',
-	'drop_component',
-	'get_components',
-	'get_component_file',
-	'set_component_file',
-	'get_job',
-	'search_jobs_by_start_date',
-	'read_log',
-	'read_transaction_log',
-	'read_audit_log',
-	'delete_transaction_logs_before',
-	'purge_stream',
-	'delete_records_before',
-	'install_node_modules',
-	'set_configuration',
-	'get_configuration',
-	'create_authentication_tokens',
-	'refresh_operation_token',
-	'system_information',
-	'sql',
-	'get_status',
-	'set_status',
-	'clear_status',
-	'get_usage_licenses',
-];
-
 const OP_ALIASES = { deploy: 'deploy_component', package: 'package_component' };
 
 module.exports = { cliOperations, buildRequest };
@@ -88,9 +34,7 @@ const PREPARE_OPERATION = {
 function buildRequest() {
 	const req = {};
 	for (const arg of process.argv.slice(2)) {
-		if (SUPPORTED_OPS.includes(arg)) {
-			req.operation = arg;
-		} else if (OP_ALIASES.hasOwnProperty(arg)) {
+		if (OP_ALIASES.hasOwnProperty(arg)) {
 			req.operation = OP_ALIASES[arg];
 		} else if (arg.includes('=')) {
 			let [first, ...rest] = arg.split('=');
@@ -103,6 +47,9 @@ function buildRequest() {
 			}
 
 			req[first] = rest;
+		} else {
+			// operation should only be in the first arg
+			req.operation ??= arg;
 		}
 	}
 
@@ -143,12 +90,12 @@ async function cliOperations(req) {
 		initConfig();
 		if (!getHdbPid()) {
 			console.error('Harper Pro must be running to perform this operation');
-			process.exit();
+			process.exit(1);
 		}
 
 		if (!fs.existsSync(envMgr.get(terms.CONFIG_PARAMS.OPERATIONSAPI_NETWORK_DOMAINSOCKET))) {
 			console.error('No domain socket found, unable to perform this operation');
-			process.exit();
+			process.exit(1);
 		}
 	}
 	await PREPARE_OPERATION[req.operation]?.(req);
@@ -178,22 +125,33 @@ async function cliOperations(req) {
 			};
 		}
 
+		let responseLog;
 		if (req.json) {
-			console.log(JSON.stringify(responseData, null, 2));
+			responseLog = JSON.stringify(responseData, null, 2);
 		} else {
-			console.log(YAML.stringify(responseData).trim());
+			responseLog = YAML.stringify(responseData).trim();
 		}
 
+		const { statusCode } = response;
+		if (statusCode < 200 || (statusCode >= 300 && statusCode !== 304)) {
+			const errorPrefix = responseLog.startsWith('error:') ? '' : 'error: ';
+			console.error(`${errorPrefix}${responseLog}`);
+			process.exit(1);
+		}
+
+		console.log(responseLog);
+
 		return responseData;
 	} catch (err) {
-		let errMsg = 'Error: ';
-		if (err?.response?.data?.error) {
-			errMsg += err.response.data.error;
-		} else if (err?.response?.data) {
-			errMsg += err?.response?.data;
+		if (err.code === 'ENOENT' || err.code === 'ECONNREFUSED') {
+			console.error(`error: Failed to connect to Harper Pro (${err.code}): ${err.message}`);
+		} else if (err.code === 'EACCES') {
+			console.error(`error: Permission denied accessing the domain socket: ${err.message}`);
+		} else if (err.code === 'ENOTFOUND') {
+			console.error(`error: Host not found: "${err.hostname}" ${err.message}`);
 		} else {
-			return console.error(err);
+			console.error(`error: ${err.message ?? err}`);
 		}
-		console.error(errMsg);
+		process.exit(1);
 	}
 }

package/core/bin/harper.js

@@ -56,14 +56,9 @@ async function harper() {
 		service = process.argv[2].toLowerCase();
 	}
 
-	const cliApiOp = cliOperations.buildRequest();
-	if (cliApiOp.operation) service = SERVICE_ACTIONS_ENUM.OPERATION;
-
 	switch (service) {
-		case SERVICE_ACTIONS_ENUM.OPERATION:
-			logger.trace('calling cli operations with:', cliApiOp);
-			await cliOperations.cliOperations(cliApiOp);
-			return;
+		case SERVICE_ACTIONS_ENUM.HELP:
+			return HELP;
 		case SERVICE_ACTIONS_ENUM.START:
 			return require('./run.js').launch();
 		case SERVICE_ACTIONS_ENUM.INSTALL:
@@ -121,22 +116,22 @@ async function harper() {
 				console.warn(
 					`It appears you are running Harper Pro in an application directory, but did not specify the path. I'll go ahead and run the application for you since that's probably what you meant. But to avoid this warning in the future, run applications in the current directory like this: "harper ${service} ."`
 				);
-				process.env.RUN_HDB_APP = '.';
+				process.env.RUN_HDB_APP = process.cwd();
 			} else if (fs.existsSync(hdbTerms.HARPER_CONFIG_FILE) || fs.existsSync(hdbTerms.HDB_CONFIG_FILE)) {
 				console.warn(
 					`It appears you are running Harper Pro in a root data directory, but did not specify the path. I'll go ahead and run Harper Pro with its root path set to "." for you since that's probably what you meant. But to avoid this warning in the future, run it like this: "harper ${service} ."`
 				);
-				process.env.ROOTPATH = '.';
+				process.env.ROOTPATH = process.cwd();
 			}
 		}
 		// fall through
 		case undefined: // run harperdb in the foreground in standard mode
 			return require('./run.js').main();
 		default:
-			console.warn(`The "${service}" command is not understood.`);
-		// fall through
-		case SERVICE_ACTIONS_ENUM.HELP:
-			return HELP;
+			const cliApiOp = cliOperations.buildRequest();
+			logger.trace('calling cli operations with:', cliApiOp);
+			await cliOperations.cliOperations(cliApiOp);
+			return;
 	}
 }
 exports.harper = harper;

package/core/config/configUtils.js

@@ -835,6 +835,17 @@ function applyRuntimeEnvVarConfig(configDoc, configFilePath, options = {}) {
 		// Apply env vars with source tracking and drift detection
 		applyRuntimeEnvConfig(configObj, rootPath, options);
 
+		// If securePort was set to the same value as port, auto-null port to avoid clashing
+		if (configObj.http?.port && configObj.http?.port === configObj.http?.securePort) {
+			configObj.http.port = null;
+		}
+		if (
+			configObj.operationsApi?.network?.port &&
+			configObj.operationsApi?.network?.port === configObj.operationsApi?.network?.securePort
+		) {
+			configObj.operationsApi.network.port = null;
+		}
+
 		// Update the YAML document's contents
 		// We update only the 'contents' property to preserve the Document instance and its methods
 		const mergedDoc = YAML.parseDocument(YAML.stringify(configObj), { simpleKeys: true });

package/core/integrationTests/deploy/deploy-from-github.test.ts

@@ -8,7 +8,6 @@ import { deepStrictEqual, ok, strictEqual } from 'node:assert/strict';
 import { setupHarper, teardownHarper, type ContextWithHarper } from '../utils/harperLifecycle.ts';
 import { join } from 'node:path';
 import { existsSync } from 'node:fs';
-import { setTimeout as sleep } from 'node:timers/promises';
 import { readFile } from 'node:fs/promises';
 import { parse } from 'yaml';
 
@@ -44,7 +43,18 @@ suite('GitHub application deployment', (ctx: ContextWithHarper) => {
 		strictEqual(response.status, 200);
 		const body = await response.json();
 		deepStrictEqual(body, { message: 'Successfully deployed: test-application, restarting Harper' });
-		await sleep(5000);
+		// Poll until the application API is ready (restart is async, fixed sleep is flaky)
+		const deadline = Date.now() + 30_000;
+		while (true) {
+			try {
+				const check = await fetch(`${ctx.harper.httpURL}/Greeting`);
+				if (check.status === 200) break;
+			} catch {
+				// server not yet accepting connections
+			}
+			if (Date.now() > deadline) throw new Error('Timed out waiting for application to be ready after restart');
+			await new Promise((resolve) => setTimeout(resolve, 250));
+		}
 		ok(existsSync(join(ctx.harper.installDir, 'components', project)));
 
 		// const harperAppLock = await readFile(join(ctx.harper.installDir, 'harper-application-lock.json'), 'utf-8');

package/core/integrationTests/utils/harperLifecycle.ts

@@ -1,9 +1,9 @@
 import assert from 'node:assert/strict';
 import { spawn, ChildProcess } from 'node:child_process';
+import { createWriteStream, existsSync, rmSync, type WriteStream } from 'node:fs';
 import { join } from 'node:path';
-import { existsSync } from 'node:fs';
 import { tmpdir } from 'node:os';
-import { mkdtemp, rm } from 'node:fs/promises';
+import { mkdtemp, mkdir, rm } from 'node:fs/promises';
 import { type SuiteContext, type TestContext } from 'node:test';
 import { getNextAvailableLoopbackAddress, releaseLoopbackAddress } from './loopbackAddressPool.ts';
 
@@ -13,6 +13,7 @@ export const OPERATIONS_API_PORT = 9925;
 export const DEFAULT_ADMIN_USERNAME = 'admin';
 export const DEFAULT_ADMIN_PASSWORD = 'Abc1234!';
 const DEFAULT_STARTUP_TIMEOUT_MS = parseInt(process.env.HARPER_INTEGRATION_TEST_STARTUP_TIMEOUT_MS, 10) || 30000;
+const LOG_DIR = process.env.HARPER_INTEGRATION_TEST_LOG_DIR;
 
 /**
  * Options for setting up a Harper instance.
@@ -51,6 +52,8 @@ export interface HarperContext {
 	hostname: string;
 	/** Child process for the Harper instance */
 	process: ChildProcess;
+	/** Absolute path to the log directory for this suite (only set when HARPER_INTEGRATION_TEST_LOG_DIR is configured) */
+	logDir?: string;
 }
 
 /**
@@ -81,27 +84,63 @@ function getHarperScript(): string {
 	return harperScript;
 }
 
+/**
+ * Sanitizes a string for use as a filesystem directory name.
+ */
+function sanitizeForFilesystem(name: string): string {
+	return name
+		.replace(/[^a-zA-Z0-9_-]/g, '_')
+		.replace(/_+/g, '_')
+		.substring(0, 100);
+}
+
+interface RunHarperCommandOptions {
+	args: string[];
+	env: any;
+	completionMessage?: string;
+	/** When set, stdout and stderr are written to files in this directory */
+	logDir?: string;
+}
+
 /**
  * Runs a Harper CLI command and captures output.
  *
- * @param args - Additional arguments to pass to the command
+ * When `logDir` is provided, stdout and stderr are also written to files
+ * (`stdout.log` and `stderr.log`) in that directory.
+ *
  * @throws {AssertionError} If the command exits with a non-zero status code
  */
-function runHarperCommand(args: string[], env: any, completionMessage?: string): Promise<ChildProcess> {
+function runHarperCommand({ args, env, completionMessage, logDir }: RunHarperCommandOptions): Promise<ChildProcess> {
 	const harperScript = getHarperScript();
 	const proc = spawn('node', ['--trace-warnings', harperScript, ...args], {
 		env: { ...process.env, ...env },
 	});
+
+	let stdoutStream: WriteStream | undefined;
+	let stderrStream: WriteStream | undefined;
+	if (logDir) {
+		stdoutStream = createWriteStream(join(logDir, 'stdout.log'));
+		stderrStream = createWriteStream(join(logDir, 'stderr.log'));
+	}
+
 	return new Promise((resolve, reject) => {
 		let stdout = '';
 		let stderr = '';
 		let timer = setTimeout(() => {
-			reject(`Harper process timed out after ${DEFAULT_STARTUP_TIMEOUT_MS}ms`);
+			let errorMessage = `Harper process timed out after ${DEFAULT_STARTUP_TIMEOUT_MS}ms`;
+			if (stdout) {
+				errorMessage += `\n\nstdout:\n${stdout}`;
+			}
+			if (stderr) {
+				errorMessage += `\n\nstderr:\n${stderr}`;
+			}
+			reject(errorMessage);
 			proc.kill();
 		}, DEFAULT_STARTUP_TIMEOUT_MS);
 
 		proc.stdout?.on('data', (data: Buffer) => {
 			const dataString = data.toString();
+			stdoutStream?.write(data);
 			if (completionMessage && dataString.includes(completionMessage)) {
 				clearTimeout(timer);
 				resolve(proc);
@@ -110,8 +149,15 @@ function runHarperCommand(args: string[], env: any, completionMessage?: string):
 		});
 
 		proc.stderr?.on('data', (data: Buffer) => {
+			stderrStream?.write(data);
 			stderr += data.toString();
 		});
+
+		proc.on('exit', () => {
+			stdoutStream?.end();
+			stderrStream?.end();
+		});
+
 		proc.on('error', (error) => {
 			reject(error);
 		});
@@ -179,9 +225,33 @@ export async function startHarper(ctx: ContextWithHarper, options?: SetupHarperO
 	);
 	const installDir = ctx.harper?.installDir ?? (await mkdtemp(installDirPrefix));
 
-	const loopbackAddress = ctx.hostname ?? (await getNextAvailableLoopbackAddress());
-	const harperProcess = await runHarperCommand(
-		[
+	const loopbackAddress = ctx.harper?.hostname ?? (await getNextAvailableLoopbackAddress());
+
+	// Set up per-suite log directory when HARPER_INTEGRATION_TEST_LOG_DIR is configured
+	let logDir: string | undefined;
+	if (LOG_DIR) {
+		const suiteName = sanitizeForFilesystem(ctx.name || 'unknown');
+		logDir = join(LOG_DIR, `${suiteName}-${sanitizeForFilesystem(loopbackAddress)}`);
+		await mkdir(logDir, { recursive: true });
+	}
+
+	// Point Harper's log directory to the suite log dir so hdb.log is preserved for upload
+	const config = { ...(options?.config || {}) };
+	if (logDir) {
+		config.logging = { ...config.logging, root: logDir };
+
+		// Clean up log directory on successful exit — only keep logs when tests fail
+		process.on('exit', (code) => {
+			if (code === 0) {
+				try {
+					rmSync(logDir, { recursive: true, force: true });
+				} catch {}
+			}
+		});
+	}
+
+	const harperProcess = await runHarperCommand({
+		args: [
 			`--ROOTPATH=${installDir}`,
 			'--DEFAULTS_MODE=dev',
 			`--HDB_ADMIN_USERNAME=${DEFAULT_ADMIN_USERNAME}`,
@@ -192,11 +262,12 @@ export async function startHarper(ctx: ContextWithHarper, options?: SetupHarperO
 			`--HTTP_PORT=${loopbackAddress}:${HTTP_PORT}`,
 			`--OPERATIONSAPI_NETWORK_PORT=${loopbackAddress}:${OPERATIONS_API_PORT}`,
 			'--LOGGING_LEVEL=debug',
-			'--HARPER_SET_CONFIG=' + JSON.stringify(options?.config || {}),
+			'--HARPER_SET_CONFIG=' + JSON.stringify(config),
 		],
-		options?.env || {},
-		'successfully started'
-	);
+		env: options?.env || {},
+		completionMessage: 'successfully started',
+		logDir,
+	});
 
 	ctx.harper = {
 		installDir,
@@ -208,6 +279,7 @@ export async function startHarper(ctx: ContextWithHarper, options?: SetupHarperO
 		operationsAPIURL: `http://${loopbackAddress}:${OPERATIONS_API_PORT}`,
 		hostname: loopbackAddress,
 		process: harperProcess,
+		logDir,
 	};
 
 	return ctx;
@@ -218,7 +290,6 @@ export async function startHarper(ctx: ContextWithHarper, options?: SetupHarperO
  *
  * This function stops the Harper instance, releases the loopback address,
  * and removes the installation directory.
- *
  * @param ctx - The test context with Harper instance details
  *
  * @example
@@ -235,7 +306,7 @@ export async function startHarper(ctx: ContextWithHarper, options?: SetupHarperO
  * ```
  */
 export async function teardownHarper(ctx: ContextWithHarper): Promise<void> {
-	await new Promise((resolve) => {
+	await new Promise<void>((resolve) => {
 		let timer: NodeJS.Timeout;
 		ctx.harper.process.on('exit', () => {
 			resolve();
@@ -246,7 +317,7 @@ export async function teardownHarper(ctx: ContextWithHarper): Promise<void> {
 			try {
 				ctx.harper.process.kill('SIGKILL');
 			} catch {
-				// possible that the process terminated but the exit event hasn't reached us yet
+				// possible that the process terminated but the exit event hasn't fired yet
 			}
 			resolve();
 		}, 200);

package/core/package.json

@@ -95,9 +95,7 @@
 		}
 	},
 	"exports": {
-		".": "./index.js",
-		"./v1": "./v1.js",
-		"./v2": "./v2.js"
+		".": "./dist/index.js"
 	},
 	"devDependencies": {
 		"@harperdb/code-guidelines": "^0.0.6",
@@ -140,7 +138,7 @@
 		"@fastify/cors": "~9.0.1",
 		"@fastify/static": "~7.0.4",
 		"@harperfast/extended-iterable": "^1.0.1",
-		"@harperfast/rocksdb-js": "^0.1.10",
+		"@harperfast/rocksdb-js": "^0.1.11",
 		"@turf/area": "6.5.0",
 		"@turf/boolean-contains": "6.5.0",
 		"@turf/boolean-disjoint": "6.5.0",
@@ -153,7 +151,7 @@
 		"alasql": "4.6.6",
 		"argon2": "0.44.0",
 		"asn1js": "3.0.7",
-		"cbor-x": "1.6.0",
+		"cbor-x": "1.6.4",
 		"chalk": "4.1.2",
 		"chokidar": "^4.0.3",
 		"cli-progress": "3.12.0",
@@ -182,14 +180,14 @@
 		"minimist": "1.2.8",
 		"moment": "2.30.1",
 		"mqtt-packet": "~9.0.1",
-		"msgpackr": "1.11.8",
+		"msgpackr": "1.11.9",
 		"needle": "3.3.1",
 		"node-forge": "^1.3.1",
 		"node-stream-zip": "1.15.0",
 		"node-unix-socket": "0.2.7",
 		"normalize-path": "^3.0.0",
 		"ora": "8.2.0",
-		"ordered-binary": "1.5.3",
+		"ordered-binary": "1.6.1",
 		"papaparse": "5.5.3",
 		"passport": "0.7.0",
 		"passport-http": "0.3.0",

package/core/resources/DatabaseTransaction.ts

@@ -351,7 +351,12 @@ function startMonitoringTxns() {
 				);
 				// reset the transaction
 				try {
-					txn.commit();
+					const result = txn.commit();
+					if (result?.then) {
+						result.catch((error) => {
+							harperLogger.debug?.(`Error committing timed out transaction: ${error.message}`);
+						});
+					}
 				} catch (error) {
 					harperLogger.debug?.(`Error committing timed out transaction: ${error.message}`);
 				}