@harperfast/harper 5.2.0 → 5.2.2

package/bin/copyDb.ts

@@ -1,4 +1,10 @@
-import { getDatabases, getDefaultCompression, resetDatabases } from '../resources/databases.ts';
+import {
+	getDatabases,
+	getDefaultCompression,
+	resetDatabases,
+	getRocksCompression,
+	toRocksCompression,
+} from '../resources/databases.ts';
 import { open, asBinary } from 'lmdb';
 import { join } from 'path';
 import { move, remove } from 'fs-extra';
@@ -329,6 +335,10 @@ export function shapeForStructure(value: any): any {
 
 function openRocksDb(path: string, options: RocksDatabaseOptions & { dupSort?: boolean } = {}) {
 	options.disableWAL ??= false;
+	// Migration creates a complete replacement database, so use the deployment codec for the files
+	// it writes; runtime opens additionally reconcile pre-existing sibling column families.
+	const legacyOptions = options as { compression?: unknown };
+	legacyOptions.compression = getRocksCompression() ?? toRocksCompression(legacyOptions.compression);
 	if (!existsSync(path)) {
 		mkdirSync(path, { recursive: true });
 	}
@@ -444,15 +454,22 @@ function countRecords(rawDbi): { records: number; unversioned: number } {
  * the RocksDB lock — run in-process (inspector) on a live instance, or offline.
  */
 export function verifyMigratedDatabase(databasePath: string): Record<string, { records: number; unversioned: number }> {
-	const rootStore = RocksDatabase.open(databasePath, {});
-	const dbisDb = RocksDatabase.open(databasePath, {
-		name: INTERNAL_DBIS_NAME,
-		sharedStructuresKey: Symbol.for('structures'),
-	});
+	// Every open handle, so a failure at any point (e.g. the second open throwing on lock
+	// contention) cannot leak an earlier handle that would hold the RocksDB lock on the very
+	// diagnostic path operators use after a broken migration.
+	const handles: RocksDatabase[] = [];
 	const report: Record<string, { records: number; unversioned: number }> = {};
 	try {
+		handles.push(RocksDatabase.open(databasePath, {}));
+		const dbisDb = RocksDatabase.open(databasePath, {
+			name: INTERNAL_DBIS_NAME,
+			sharedStructuresKey: Symbol.for('structures'),
+		});
+		handles.push(dbisDb);
 		for (const { key, value: attribute } of dbisDb.getRange({})) {
 			if (typeof key === 'symbol' || !attribute?.isPrimaryKey) continue;
+			// per-table handles close per-iteration so a many-table sweep does not hold every CF
+			// handle open at once; only the two pre-loop opens need the leak-safety array
 			const rawDbi = RocksDatabase.open(databasePath, { name: key, encoding: false });
 			try {
 				report[key] = countRecords(rawDbi);
@@ -461,8 +478,13 @@ export function verifyMigratedDatabase(databasePath: string): Record<string, { r
 			}
 		}
 	} finally {
-		dbisDb.close();
-		rootStore.close();
+		for (const handle of handles.reverse()) {
+			try {
+				handle.close();
+			} catch (error) {
+				console.error('Error closing verification store', error);
+			}
+		}
 	}
 	return report;
 }
@@ -510,7 +532,14 @@ export async function migrateDatabaseToRocks(sourceRootStore, databaseName: stri
 export async function copyDbToRocks(sourceRootStore, sourceDatabase: string, targetPath: string) {
 	console.log(`Migrating database ${sourceDatabase} to RocksDB at ${targetPath}`);
 	const sourceDbisDb = sourceRootStore.dbisDb;
-
+	// Runtime Harper stores disable RocksDB's native WAL for data/index column families and recover
+	// them from rocksdb-js transaction logs. This copier does not write those transaction logs, so
+	// only use the same fast write path for migrateOnStart's disposable staging directory: after an
+	// interruption it is deleted and recopied from LMDB. Direct copyDbToRocks callers retain WAL.
+	const disableDataWAL = targetPath.endsWith(MIGRATING_DIR_SUFFIX);
+
+	// Keep native WAL for the root/log-owner and __dbis__ handles, matching Harper's runtime policy.
+	// Their migration writes are metadata-sized; the 600 GB bulk is in the data/index handles below.
 	const targetRootStore = openRocksDb(targetPath, { disableWAL: false });
 	// Every handle opened on targetPath. All must be closed before returning so the caller can
 	// atomically rename a staging directory into place — rocksdb-js registers descriptors by
@@ -587,10 +616,10 @@ export async function copyDbToRocks(sourceRootStore, sourceDatabase: string, tar
 			let observerEncoder: any;
 			let canonicalStructures: any;
 			if (!isPrimary) {
-				targetDbi = openRocksDb(targetPath, { dupSort: true, name: key });
+				targetDbi = openRocksDb(targetPath, { disableWAL: disableDataWAL, dupSort: true, name: key });
 				targetHandles.push(targetDbi);
 			} else {
-				targetDbi = openRocksDb(targetPath, { name: key });
+				targetDbi = openRocksDb(targetPath, { disableWAL: disableDataWAL, name: key });
 				targetHandles.push(targetDbi);
 				// Patch the existing encoder (encoder is a getter-only property on RocksDatabase, cannot be replaced)
 				// to install RecordEncoder's encode method so metadata headers (timestamps, HAS_BLOBS flag) are written

package/bin/harper.ts

@@ -5,6 +5,7 @@ import * as fs from 'node:fs';
 import * as path from 'node:path';
 import logger from '../utility/logging/harper_logger.ts';
 import * as cliOperations from './cliOperations.ts';
+import { help } from './help.ts';
 import { packageJson } from '../utility/packageUtils.js';
 import checkNode from '../launchServiceScripts/utility/checkNodeVersion.js';
 import * as hdbTerms from '../utility/hdbTerms.ts';
@@ -13,55 +14,6 @@ if (typeof process.setSourceMapsEnabled === 'function') {
 	process.setSourceMapsEnabled(true); // this is necessary for source maps to work, at least on the main thread.
 }
 
-const HELP = `
-Usage: harperdb [command]
-
-With no command, harper will simply run Harper (in the foreground)
-
-Documentation: https://docs.harperdb.io/
-
-By default, the CLI also supports certain Operation APIs. Specify the operation name and any required parameters, and omit the 'operation' command.
-
-Commands:
-agent [message]                 - Chat with the built-in agent (interactive, or one-shot with a message; alias: chat)
-copy-db <source> <target>       - Copies a database from source path to target path
-dev <path>                      - Run the application in dev mode with debugging, foreground logging, no auth
-install                         - Install harperdb
-<api-operation> <param>=<value> - Run an API operation and return result to the CLI, not all operations are supported
-                                   To authenticate as a different user than the one being operated on
-                                   (e.g. add_user/alter_user), set HARPER_CLI_USERNAME/HARPER_CLI_PASSWORD
-                                   or run 'harper login'. The equivalent auth_username=<value>
-                                   auth_password=<value> args also work, but a password passed as an
-                                   argument is exposed in shell history, process listings and CI logs.
-                                   A saved login token always outranks username=/password=, so a
-                                   stale token that fails to refresh will 401 rather than falling
-                                   back to them — run 'harper logout' or pass auth_username=/
-                                   auth_password= to override it.
-login [target] [username]       - Login to a remote or local Harper instance
-                                   --for-ci prints the CI/CD credentials (target + long-lived
-                                   refresh token) to stdout in dotenv format, and everything else
-                                   to stderr, so it pipes without the token hitting your screen:
-                                     harper login --for-ci | gh secret set --env-file -
-                                   Log in as a user dedicated to that one CI consumer: Harper
-                                   stores a single refresh token per user, so this revokes any
-                                   refresh token that user already holds — another runner, another
-                                   machine, or an earlier 'harper login' will 401 on its next
-                                   refresh. Two consumers cannot share a user.
-logout [target]                 - Logout from Harper and clear saved JWT
-mcp [subcommand]                - MCP stdio bridge / print-config / doctor (see 'harper mcp help')
-register                        - Register harperdb
-renew-certs                     - Generate a new set of self-signed certificates
-restart                         - Restart the harperdb background process
-run <path>                      - Run the application in the specified path
-start                           - Starts a separate background process for harperdb and CLI will exit
-status                          - Print the status of Harper
-stop                            - Stop the harperdb background process
-help                            - Display this output
-upgrade                         - Upgrade harperdb
-version                         - Print the version
-deploy                          - Deploy the application locally or remotely with target=<remote url>
-`;
-
 /**
  * Format a CLI error for the terminal. Expected, user-facing errors (a `ClientError` from an
  * operation — bad args, not found, a locked backup repo — which carry a numeric `statusCode`) get
@@ -74,6 +26,19 @@ export function formatCliError(error: any): string {
 	return message;
 }
 
+/**
+ * Whether a `-h`/`--help` anywhere in `argv` should print the top-level help. Returns false for the
+ * subcommands that own their own `--help` — mcp and agent/chat parse `process.argv.slice(3)`
+ * themselves downstream — so `harper mcp --help` reaches the mcp handler instead of this help.
+ */
+export function wantsTopLevelHelp(argv: readonly string[], service: string | undefined): boolean {
+	const delegatesHelp =
+		service === SERVICE_ACTIONS_ENUM.MCP ||
+		service === SERVICE_ACTIONS_ENUM.AGENT ||
+		service === SERVICE_ACTIONS_ENUM.CHAT;
+	return !delegatesHelp && (argv.includes('-h') || argv.includes('--help'));
+}
+
 async function harper() {
 	let nodeResults = checkNode();
 
@@ -90,13 +55,17 @@ async function harper() {
 
 	let service;
 
-	if (process.argv && process.argv[2] && !process.argv[2].startsWith('-')) {
+	if (process.argv?.[2] && !process.argv[2].startsWith('-')) {
 		service = process.argv[2].toLowerCase();
 	}
 
+	if (wantsTopLevelHelp(process.argv, service)) {
+		return help();
+	}
+
 	switch (service) {
 		case SERVICE_ACTIONS_ENUM.HELP:
-			return HELP;
+			return help();
 		case SERVICE_ACTIONS_ENUM.START:
 			return require('./run').launch();
 		case SERVICE_ACTIONS_ENUM.INSTALL:

package/bin/help.ts

@@ -0,0 +1,216 @@
+/**
+ * The `harper help` / `harper -h` output, kept as structured data so {@link help} can render it to
+ * the current terminal width (capped at 120 columns) at call time. Content lives separately from
+ * layout: editing the text can't break the wrapping, command names stay column-aligned across
+ * sections, and the CI/CD sample stays a verbatim, copy-pasteable line at any width.
+ */
+
+const MAX_WIDTH = 120;
+const INDENT = 2; // leading spaces for command names and detail blocks
+const DASH = ' - '; // separator between a command name and its description
+// Minimum description width for the two-column command layout. On terminals too narrow for it
+// (width < descCol + this), command rows stack the name above an indented description instead of
+// overflowing. Set to the widest word in any command description so two-column never overflows.
+const MIN_DESC_WIDTH = 20;
+
+/** A paragraph of prose, reflowed to the target width at `indent` spaces. */
+interface TextBlock {
+	text: string;
+	indent?: number;
+}
+
+/** A verbatim line (a code sample); never joined or wrapped so it stays copy-pasteable. */
+interface PreBlock {
+	pre: string;
+	indent?: number;
+}
+
+/** A two-column list of `[name, description]` rows; descriptions wrap with a hanging indent. */
+interface CommandsBlock {
+	commands: [name: string, description: string][];
+}
+
+type Block = TextBlock | PreBlock | CommandsBlock;
+
+interface Section {
+	heading?: string;
+	blocks: Block[];
+}
+
+const SECTIONS: Section[] = [
+	{ blocks: [{ text: 'Usage: harperdb [command]' }] },
+	{ blocks: [{ text: 'With no command, harper will simply run Harper (in the foreground)' }] },
+	{ blocks: [{ text: 'Documentation: https://docs.harperdb.io/' }] },
+	{
+		blocks: [
+			{
+				text: [
+					'By default, the CLI also supports certain Operation APIs. Specify the operation name and any required',
+					"parameters, and omit the 'operation' command.",
+				].join(' '),
+			},
+		],
+	},
+	{ blocks: [{ text: 'Commands:' }] },
+	{
+		heading: 'Server',
+		blocks: [
+			{
+				commands: [
+					['start', 'Starts a separate background process for harperdb and CLI will exit'],
+					['stop', 'Stop the harperdb background process'],
+					['restart', 'Restart the harperdb background process'],
+					['status', 'Print the status of Harper'],
+				],
+			},
+		],
+	},
+	{
+		heading: 'Applications',
+		blocks: [
+			{
+				commands: [
+					['run <path>', 'Run the application in the specified path'],
+					['dev <path>', 'Run the application in dev mode with debugging, foreground logging, no auth'],
+					['deploy', 'Deploy the application locally or remotely with target=<remote url>'],
+				],
+			},
+		],
+	},
+	{
+		heading: 'Install & maintenance',
+		blocks: [
+			{
+				commands: [
+					['install', 'Install harperdb'],
+					['upgrade', 'Upgrade harperdb'],
+					['register', 'Register harperdb'],
+					['renew-certs', 'Generate a new set of self-signed certificates'],
+					['copy-db <source> <target>', 'Copies a database from source path to target path'],
+					['version', 'Print the version'],
+					['help', 'Display this output'],
+				],
+			},
+		],
+	},
+	{
+		heading: 'Accounts',
+		blocks: [
+			{ pre: 'login [target] [username]', indent: INDENT },
+			{
+				text: [
+					'Login to a remote or local Harper instance. --for-ci prints the CI/CD credentials (target +',
+					'long-lived refresh token) to stdout in dotenv format, and everything else to stderr, so it pipes',
+					'without the token hitting your screen:',
+				].join(' '),
+				indent: 4,
+			},
+			{ pre: 'harper login --for-ci | gh secret set --env-file -', indent: 6 },
+			{
+				text: [
+					'Log in as a user dedicated to that one CI consumer: Harper stores a single refresh token per user, so',
+					'this revokes any refresh token that user already holds — another runner, another machine, or an earlier',
+					"'harper login' will 401 on its next refresh. Two consumers cannot share a user.",
+				].join(' '),
+				indent: 4,
+			},
+			{ commands: [['logout [target]', 'Logout from Harper and clear saved JWT']] },
+		],
+	},
+	{
+		heading: 'Assistants',
+		blocks: [
+			{
+				commands: [
+					['agent [message]', 'Chat with the built-in agent (interactive, or one-shot with a message; alias: chat)'],
+					['mcp [subcommand]', "MCP stdio bridge / print-config / doctor (see 'harper mcp help')"],
+				],
+			},
+		],
+	},
+	{
+		heading: 'Operations API',
+		blocks: [
+			{ pre: '<api-operation> <param>=<value>', indent: INDENT },
+			{
+				text: [
+					'Run an API operation and return the result to the CLI (not all operations are supported). See the full',
+					'list of operations at: https://docs.harperdb.io/reference/v5/operations-api/operations',
+				].join(' '),
+				indent: 4,
+			},
+			{
+				text: [
+					'To authenticate as a different user than the one being operated on (e.g. add_user/alter_user), set',
+					"HARPER_CLI_USERNAME/HARPER_CLI_PASSWORD or run 'harper login'. The equivalent auth_username=<value>",
+					'auth_password=<value> args also work, but a password passed as an argument is exposed in shell',
+					'history, process listings and CI logs. A saved login token always outranks username=/password=, so a',
+					"stale token that fails to refresh will 401 rather than falling back to them — run 'harper logout' or",
+					'pass auth_username=/auth_password= to override it.',
+				].join(' '),
+				indent: 4,
+			},
+		],
+	},
+];
+
+const pad = (count: number): string => ' '.repeat(count);
+
+/** Greedy word wrap. Never splits a token, so URLs and flags survive intact (they overflow). */
+function wrap(text: string, width: number): string[] {
+	const words = text.split(/\s+/).filter(Boolean);
+	if (words.length === 0) return [];
+	const lines: string[] = [];
+	let line = words[0];
+	for (let i = 1; i < words.length; i++) {
+		if (line.length + 1 + words[i].length > width) {
+			lines.push(line);
+			line = words[i];
+		} else {
+			line += ' ' + words[i];
+		}
+	}
+	lines.push(line);
+	return lines;
+}
+
+function renderBlock(block: Block, width: number, nameWidth: number, descCol: number): string[] {
+	if ('commands' in block) {
+		// Two columns when the terminal leaves a readable description column; otherwise stack the name
+		// above an indented description so narrow terminals don't overflow the command rows.
+		const twoColumn = width - descCol >= MIN_DESC_WIDTH;
+		return block.commands.flatMap(([name, description]) => {
+			if (twoColumn) {
+				const wrapped = wrap(description, width - descCol);
+				const head = pad(INDENT) + name.padEnd(nameWidth) + DASH;
+				return [head + (wrapped[0] ?? ''), ...wrapped.slice(1).map((line) => pad(descCol) + line)];
+			}
+			const detail = INDENT * 2;
+			return [pad(INDENT) + name, ...wrap(description, Math.max(1, width - detail)).map((line) => pad(detail) + line)];
+		});
+	}
+	if ('pre' in block) return [pad(block.indent ?? 0) + block.pre];
+	const indent = block.indent ?? 0;
+	return wrap(block.text, Math.max(1, width - indent)).map((line) => pad(indent) + line);
+}
+
+/** Render the CLI help, wrapped to the terminal width (capped at 120, and 120 when not a TTY). */
+export function help(): string {
+	const width = Math.min(process.stdout.columns || MAX_WIDTH, MAX_WIDTH);
+
+	// One shared description column across every command list so names line up between sections,
+	// driven by the longest command name.
+	const names = SECTIONS.flatMap((section) =>
+		section.blocks.flatMap((block) => ('commands' in block ? block.commands.map(([name]) => name) : []))
+	);
+	const nameWidth = Math.max(...names.map((name) => name.length));
+	const descCol = INDENT + nameWidth + DASH.length;
+
+	const lines: string[] = [];
+	for (const section of SECTIONS) {
+		if (lines.length) lines.push(''); // blank line between sections
+		if (section.heading) lines.push(section.heading);
+		for (const block of section.blocks) lines.push(...renderBlock(block, width, nameWidth, descCol));
+	}
+	return '\n' + lines.join('\n') + '\n';
+}