@harperfast/harper 5.0.25 → 5.1.0-beta.1

package/agent/types.ts

@@ -0,0 +1,85 @@
+/**
+ * Shared types for the built-in Harper Agent component (#626).
+ *
+ * `AgentMessage` mirrors `resources/models/types.ts:Message` rather than
+ * re-exporting it: the session row is durable storage and shouldn't trail
+ * shape changes in the model-access surface.
+ */
+
+import type { ToolCall, ToolDef } from '../resources/models/types.ts';
+
+export type AgentRunStatus = 'idle' | 'running' | 'awaiting_approval' | 'completed' | 'aborted' | 'error';
+
+export interface AgentMessage {
+	role: 'system' | 'user' | 'assistant' | 'tool';
+	content: string;
+	toolCalls?: ToolCall[];
+	toolCallId?: string;
+	/** Wall-clock timestamp of when this item entered the transcript. */
+	createdAt: number;
+}
+
+export interface ApprovalRequest {
+	id: string;
+	toolName: string;
+	arguments: object;
+	/** Tool-call id from the assistant message that produced this request. Used to resume execution. */
+	toolCallId: string;
+	/** Human-readable rationale for why approval is required (e.g. 'destructive', 'first_use'). */
+	reason: string;
+	createdAt: number;
+	resolved?: boolean;
+	approved?: boolean;
+	resolvedAt?: number;
+	/** Set to true once the loop has consumed the resolved approval (executed or refused). */
+	consumed?: boolean;
+}
+
+export interface AgentSessionRow {
+	session_id: string;
+	user: string;
+	status: AgentRunStatus;
+	messages: AgentMessage[];
+	pendingApprovals: ApprovalRequest[];
+	model?: string;
+	provider?: string;
+	createdAt: number;
+	updatedAt: number;
+	lastError?: string;
+}
+
+export interface AgentToolHandler {
+	(args: object, ctx: AgentToolContext): Promise<unknown>;
+}
+
+export interface AgentTool {
+	def: ToolDef;
+	handler: AgentToolHandler;
+	/** When true, invocations of this tool require an approval gate unless `autoApprove` is set. */
+	destructive?: boolean;
+}
+
+export interface AgentToolContext {
+	sessionId: string;
+	signal?: AbortSignal;
+	/** Filesystem scope roots accessible to FS tools. Read-only after composition. */
+	scopes: AgentScopes;
+}
+
+export interface AgentScopes {
+	componentsRoot: string;
+	logDir: string;
+	configDir: string;
+}
+
+export interface AgentConfig {
+	enabled: boolean;
+	provider?: string;
+	model?: string;
+	maxTurns: number;
+	maxCostUsd: number;
+	autoApprove: boolean;
+	allowDestructive: boolean;
+	user: string;
+	componentsScope?: string;
+}

package/bin/{BinObjects.js → BinObjects.ts}

@@ -4,14 +4,13 @@
  * This is meant as a central place to defined POJOs used by functions in the /bin/ directory.
  */
 
-class HdbInfoInsertObject {
+export class HdbInfoInsertObject {
+	info_id: any;
+	data_version_num: any;
+	hdb_version_num: any;
 	constructor(id, dataVersionNum, hdbVersionNum) {
 		this.info_id = id;
 		this.data_version_num = dataVersionNum;
 		this.hdb_version_num = hdbVersionNum;
 	}
 }
-
-module.exports = {
-	HdbInfoInsertObject,
-};

package/bin/cliCredentials.ts

@@ -0,0 +1,133 @@
+import { getHomeDir } from '../utility/common_utils.ts';
+import fs from 'node:fs';
+import path from 'node:path';
+
+const ownerRWDenyAllOthers = 0o600;
+
+interface TargetedCredentials {
+	last_target: string | null;
+	targets: {
+		[target: string]: Tokens;
+	};
+}
+
+interface Tokens {
+	operation_token: string;
+	refresh_token: string;
+}
+
+/**
+ * Normalizes a target operations API URL to a canonical form (with trailing slash).
+ */
+export function normalizeTarget(target: string): string {
+	if (!target) return target;
+	let normalized = target;
+	if (!normalized.startsWith('http://') && !normalized.startsWith('https://')) {
+		normalized = 'https://' + normalized;
+	}
+	try {
+		const url = new URL(normalized);
+		if (!url.port && !normalized.includes(':', normalized.indexOf('://') + 3)) {
+			url.port = '9925';
+		}
+		normalized = url.toString();
+	} catch {
+		// If it's not a valid URL yet, we'll let it be handled later or it will fail
+	}
+	if (!normalized.endsWith('/')) {
+		normalized += '/';
+	}
+	return normalized;
+}
+
+/**
+ * Loads the JWT credentials from the ~/.harperdb/credentials.json file.
+ */
+export function loadCredentials(): TargetedCredentials {
+	const credentialsFile = getCredentialsFile();
+	try {
+		return JSON.parse(fs.readFileSync(credentialsFile, 'utf8'));
+	} catch (err) {
+		if (err.code !== 'ENOENT') {
+			throw new Error(`Error reading credentials file: ${err.message}`);
+		}
+	}
+	return {
+		last_target: null,
+		targets: {},
+	};
+}
+
+/**
+ * Saves the JWT credentials to the ~/.harperdb/credentials.json file.
+ */
+export function saveCredentials(target: string, tokens: Tokens): void {
+	if (!target) {
+		throw new Error('Target is required to save credentials.');
+	}
+
+	target = normalizeTarget(target);
+	const allCredentials = loadCredentials();
+
+	allCredentials.targets ||= {};
+	allCredentials.targets[target] = tokens;
+	allCredentials.last_target = target;
+
+	try {
+		fs.mkdirSync(getCredentialsDir(), { recursive: true });
+		fs.writeFileSync(getCredentialsFile(), JSON.stringify(allCredentials, null, 2), { mode: ownerRWDenyAllOthers });
+	} catch (err) {
+		throw new Error(`Error saving credentials file: ${err.message}`);
+	}
+}
+
+/**
+ * Deletes the credentials for a specific target or all if no target provided.
+ */
+export function clearCredentials(target: string): void {
+	const credentialsFile = getCredentialsFile();
+	if (target) {
+		target = normalizeTarget(target);
+		const allCredentials = loadCredentials();
+		if (allCredentials && allCredentials.targets) {
+			if (allCredentials.targets[target]) {
+				delete allCredentials.targets[target];
+			} else {
+				console.error(`No credentials found for ${target}`);
+				process.exit(1);
+			}
+
+			if (allCredentials.last_target === target) {
+				allCredentials.last_target = null;
+			}
+			try {
+				fs.writeFileSync(credentialsFile, JSON.stringify(allCredentials, null, 2), {
+					mode: ownerRWDenyAllOthers,
+				});
+				console.log(`Logged out from ${target}`);
+			} catch (err) {
+				throw new Error(`Error clearing credentials file: ${err.message}`);
+			}
+		} else {
+			console.error(`No credentials found for ${target}`);
+			process.exit(1);
+		}
+	} else if (fs.existsSync(credentialsFile)) {
+		try {
+			fs.unlinkSync(credentialsFile);
+			console.log('Logged out from all targets');
+		} catch (err) {
+			throw new Error(`Error clearing credentials file: ${err.message}`);
+		}
+	} else {
+		console.log('No credentials found to clear.');
+	}
+}
+
+function getCredentialsFile(): string {
+	return path.join(getHomeDir(), '.harperdb', 'credentials.json');
+}
+
+function getCredentialsDir(): string {
+	return path.join(getHomeDir(), '.harperdb');
+}

package/bin/cliOperations.ts

@@ -0,0 +1,339 @@
+'use strict';
+
+import { loadCredentials, saveCredentials, normalizeTarget } from './cliCredentials.ts';
+import { isJWTExpired } from '../security/tokenAuthentication.ts';
+import * as envMgr from '../utility/environment/environmentManager.ts';
+envMgr.initSync();
+import * as terms from '../utility/hdbTerms.ts';
+import { httpRequest } from '../utility/common_utils.ts';
+import * as path from 'path';
+import * as fs from 'fs-extra';
+import * as YAML from 'yaml';
+import { streamPackagedDirectory, getPackagedDirectorySize } from '../components/packageComponent.ts';
+import { buildMultipartBody } from './multipartBuilder.ts';
+import { parseSSE } from './sseConsumer.ts';
+import { DeployRenderer } from './deployRenderer.ts';
+import { getHdbPid } from '../utility/processManagement/processManagement.js';
+import { initConfig, getConfigPath } from '../config/configUtils.js';
+
+const OP_ALIASES = { deploy: 'deploy_component', package: 'package_component' };
+
+// Operations whose responses should be consumed as text/event-stream so live phase events
+// (prepare, load, replicate, restart) render as they happen instead of after the whole
+// deploy completes. Add an operation here only after wiring its server-side
+// SSE_PROGRESS_OPERATIONS entry — otherwise the server returns the buffered JSON path and
+// the SSE parser sees no events.
+const SSE_OPERATIONS = new Set(['deploy_component']);
+
+// Properties on `req` that the CLI itself uses for transport/UX, not the operations API.
+// They never get serialized into the request body.
+const TRANSPORT_ONLY_FIELDS = new Set([
+	'target',
+	'username',
+	'password',
+	'rejectUnauthorized',
+	'json',
+	'skip_node_modules',
+	'skip_symlinks',
+]);
+
+export { cliOperations, buildRequest };
+const PREPARE_OPERATION: any = {
+	deploy_component: async (req) => {
+		if (req.package) {
+			return;
+		}
+
+		const projectPath = process.cwd();
+		if (!req.project) req.project = path.basename(projectPath);
+		const packageOptions = {
+			skip_node_modules: req.skip_node_modules !== false,
+			skip_symlinks: req.skip_symlinks === true,
+		};
+		// Store path + options for deferred stream creation after the renderer is set up,
+		// so the pre-gzip onBytes callback can be wired directly to renderer.countUploadBytes.
+		req._projectPath = projectPath;
+		req._packageOptions = packageOptions;
+		// Pre-walk the directory for an uncompressed-size estimate. Both the progress counter
+		// and this total are in uncompressed units so the bar tracks to 100% naturally.
+		req._uploadSizeEstimate = await getPackagedDirectorySize(projectPath, packageOptions);
+		req._multipart = true;
+	},
+};
+
+/**
+ * Builds an Op-API request object from CLI args
+ */
+function buildRequest(): any {
+	const req: any = {};
+	for (const arg of process.argv.slice(2)) {
+		if (OP_ALIASES.hasOwnProperty(arg)) {
+			req.operation = OP_ALIASES[arg];
+		} else if (arg.includes('=')) {
+			let [first, ...rest] = arg.split('=');
+			let restStr: any = rest.join('=');
+
+			try {
+				restStr = JSON.parse(restStr);
+			} catch {
+				/* noop */
+			}
+
+			req[first] = restStr;
+		} else {
+			// operation should only be in the first arg
+			req.operation ??= arg;
+		}
+	}
+
+	return req;
+}
+
+/**
+ * Resolves the target URL from various sources.
+ * @param {Object} req The request object.
+ * @param {Object} allCredentials Stored credentials.
+ * @returns {string|null} The resolved target URL.
+ */
+function resolveTarget(req, allCredentials) {
+	return (
+		req.target ||
+		process.env.HARPER_CLI_TARGET ||
+		process.env.CLI_TARGET ||
+		(allCredentials && allCredentials.last_target)
+	);
+}
+
+/**
+ * Using a unix domain socket will send a request to hdb operations API server
+ * @param req
+ * @param skipResponseLog By default, the response is logged to the console. Set this to true to skip logging it, which can be useful for sensitive responses like login calls!
+ * @returns {Promise<void>}
+ */
+async function cliOperations(req: any, skipResponseLog = false) {
+	require('dotenv').config();
+
+	const allCredentials = loadCredentials();
+	req.target = normalizeTarget(resolveTarget(req, allCredentials));
+	let target;
+	if (req.target) {
+		try {
+			target = new URL(req.target);
+		} catch (error) {
+			try {
+				target = new URL(`https://${req.target}:9925`);
+			} catch {
+				throw error;
+			}
+		}
+		const resolvedTarget = req.target;
+		target = {
+			protocol: target.protocol,
+			hostname: target.hostname,
+			port: target.port,
+			username: req.username || target.username || process.env.HARPER_CLI_USERNAME || process.env.CLI_TARGET_USERNAME,
+			password: req.password || target.password || process.env.HARPER_CLI_PASSWORD || process.env.CLI_TARGET_PASSWORD,
+			rejectUnauthorized: req.rejectUnauthorized,
+			resolvedTarget,
+		};
+		console.error(`Connecting to ${resolvedTarget}`);
+	} else {
+		// if we aren't doing a targeted operation (like deploy), we initialize the config and verify that local harper
+		// is running and that we can communicate with it.
+		console.error('Connecting to local Harper instance');
+		initConfig();
+		if (!getHdbPid()) {
+			console.error('Harper must be running to perform this operation');
+			process.exit(1);
+		}
+
+		if (!fs.existsSync(getConfigPath(terms.CONFIG_PARAMS.OPERATIONSAPI_NETWORK_DOMAINSOCKET))) {
+			console.error('No domain socket found, unable to perform this operation');
+			process.exit(1);
+		}
+	}
+	await PREPARE_OPERATION[req.operation]?.(req);
+	try {
+		let options = target ?? {
+			protocol: 'http:',
+			socketPath: getConfigPath(terms.CONFIG_PARAMS.OPERATIONSAPI_NETWORK_DOMAINSOCKET),
+		};
+		options.method = 'POST';
+		options.headers = { 'Content-Type': 'application/json' };
+		if (target?.username) {
+			options.headers.Authorization = `Basic ${Buffer.from(`${target.username}:${target.password}`).toString('base64')}`;
+		} else if (allCredentials) {
+			let tokens = null;
+			let lookupKey = null;
+			if (target && allCredentials.targets) {
+				lookupKey = target.resolvedTarget;
+				tokens = allCredentials.targets[lookupKey] ?? null;
+			}
+
+			if (tokens?.operation_token) {
+				if (tokens.refresh_token && isJWTExpired(tokens.operation_token)) {
+					console.error('Operation token expired, attempting to refresh...');
+					try {
+						const refreshOptions = { ...options };
+						refreshOptions.headers = { ...options.headers, Authorization: `Bearer ${tokens.refresh_token}` };
+						const refreshResponse = await httpRequest(refreshOptions, {
+							operation: 'refresh_operation_token',
+						});
+						if (refreshResponse.statusCode === 200) {
+							const refreshData = JSON.parse(refreshResponse.body);
+							if (refreshData.operation_token) {
+								tokens.operation_token = refreshData.operation_token;
+								saveCredentials(lookupKey || target?.resolvedTarget, {
+									operation_token: tokens.operation_token,
+									refresh_token: tokens.refresh_token,
+								});
+								console.error('Operation token refreshed successfully.');
+								// Update the original request's authorization header with the new token
+								options.headers.Authorization = `Bearer ${tokens.operation_token}`;
+							}
+						} else if (refreshResponse.statusCode === 401) {
+							console.error('Refresh token expired or invalid. Please run harper login again.');
+							process.exit(1);
+						} else {
+							console.error(`Failed to refresh operation token: ${refreshResponse.statusCode}`);
+						}
+					} catch (refreshErr) {
+						console.error(`Error refreshing operation token: ${refreshErr.message}`);
+					}
+				}
+				options.headers.Authorization = `Bearer ${tokens.operation_token}`;
+			}
+		}
+		const useSse = SSE_OPERATIONS.has(req.operation);
+		if (useSse) {
+			options.headers.Accept = 'text/event-stream';
+			options.streamResponse = true;
+		}
+		// One renderer owns the (future) upload bar and the SSE event rendering for a
+		// multipart deploy. Created here so the upload-stream tap and the SSE consumer
+		// below share the same instance.
+		const renderer = req._multipart ? new DeployRenderer({ uploadTotal: req._uploadSizeEstimate ?? 0 }) : null;
+		let body;
+		if (req._multipart) {
+			// Create the package stream here — after the renderer exists — so we can pass
+			// renderer.countUploadBytes as the onBytes callback. Both progress and total are
+			// uncompressed bytes, so the bar tracks accurately to 100% without premature snapping.
+			const packageStream = streamPackagedDirectory(
+				req._projectPath,
+				req._packageOptions,
+				renderer ? (n) => renderer.countUploadBytes(n) : undefined
+			);
+			const fields = {};
+			for (const [key, value] of Object.entries(req)) {
+				if (key.startsWith('_') || TRANSPORT_ONLY_FIELDS.has(key)) continue;
+				fields[key] = value;
+			}
+			const multipart = buildMultipartBody(fields, {
+				name: 'payload',
+				filename: 'package.tar.gz',
+				contentType: 'application/gzip',
+				stream: packageStream,
+			});
+			options.headers['Content-Type'] = multipart.contentType;
+			// Use chunked transfer-encoding: we don't know the total size up front because the
+			// payload is streamed from `tar.pack` and never fully buffered.
+			options.headers['Transfer-Encoding'] = 'chunked';
+			// Tap the body so bytes flowing into the HTTP request advance the upload bar.
+			// The renderer's Transform is identity — chunks pass through unmodified.
+			body = renderer ? renderer.tapUploadStream(multipart.stream) : multipart.stream;
+		} else {
+			body = req;
+		}
+		let response: any = await httpRequest(options, body);
+
+		// endUpload() is called from the counter Transform's flush callback in tapUploadStream
+		// once all multipart bytes have flowed through. For SSE deploys, httpRequest resolves
+		// when response headers arrive (streamResponse: true), which happens before the full
+		// upload completes — calling endUpload() here would snap the bar prematurely.
+
+		let responseData;
+		if (useSse && response.headers['content-type']?.startsWith('text/event-stream')) {
+			// Consume SSE: render phase events live, capture the final result from the `done`
+			// event (or the error message from the `error` event). The HTTP status stays 200
+			// until end-of-stream; failures are signaled in-band.
+			let finalResult;
+			let sseError;
+			for await (const message of parseSSE(response)) {
+				renderer?.renderEvent(message);
+				if (message.event === 'done') {
+					try {
+						finalResult = JSON.parse(message.data)?.result;
+					} catch {
+						finalResult = message.data;
+					}
+				} else if (message.event === 'error') {
+					try {
+						sseError = JSON.parse(message.data);
+					} catch {
+						sseError = { message: message.data };
+					}
+				}
+			}
+			if (sseError) {
+				const errMsg = sseError.message ?? (typeof sseError === 'object' ? JSON.stringify(sseError) : sseError);
+				console.error(`error: ${errMsg}`);
+				process.exit(1);
+			}
+			responseData = finalResult ?? { message: 'Deploy completed (no result payload).' };
+		} else {
+			// When useSse is true, httpRequest returns a raw IncomingMessage (streamResponse mode),
+			// so .body is undefined. Drain the stream to get the text (e.g. a 401 error body).
+			let bodyText: string;
+			if (useSse) {
+				const chunks: Buffer[] = [];
+				for await (const chunk of response as AsyncIterable<Buffer>) chunks.push(Buffer.from(chunk));
+				bodyText = Buffer.concat(chunks).toString('utf8');
+			} else {
+				bodyText = response.body;
+			}
+			try {
+				responseData = JSON.parse(bodyText);
+			} catch {
+				responseData = {
+					status: response.statusCode + ' ' + (response.statusMessage || 'Unknown'),
+					body: bodyText,
+				};
+			}
+		}
+
+		let responseLog;
+		if (req.json) {
+			responseLog = JSON.stringify(responseData, null, 2);
+		} else {
+			responseLog = YAML.stringify(responseData).trim();
+		}
+
+		const { statusCode } = response;
+		if (statusCode < 200 || (statusCode >= 300 && statusCode !== 304)) {
+			const errorPrefix = responseLog.startsWith('error:') ? '' : 'error: ';
+			console.error(`${errorPrefix}${responseLog}`);
+			process.exit(1);
+		}
+
+		if (!skipResponseLog) {
+			console.log(responseLog);
+		}
+
+		if (target) {
+			responseData.resolvedTarget = target.resolvedTarget;
+		}
+
+		return responseData;
+	} catch (err) {
+		if (err.code === 'ENOENT' || err.code === 'ECONNREFUSED') {
+			console.error(`error: Failed to connect to Harper (${err.code}): ${err.message}`);
+		} else if (err.code === 'EACCES') {
+			console.error(`error: Permission denied accessing the domain socket: ${err.message}`);
+		} else if (err.code === 'ENOTFOUND') {
+			console.error(`error: Host not found: "${err.hostname}" ${err.message}`);
+		} else {
+			console.error(`error: ${err.message ?? err}`);
+		}
+		process.exit(1);
+	}
+}

package/bin/copyDb.ts

@@ -3,15 +3,15 @@ import { open, asBinary } from 'lmdb';
 import { join } from 'path';
 import { move, remove } from 'fs-extra';
 import { existsSync, mkdirSync } from 'node:fs';
-import { get } from '../utility/environment/environmentManager.js';
-import OpenEnvironmentObject from '../utility/lmdb/OpenEnvironmentObject.js';
-import { OpenDBIObject } from '../utility/lmdb/OpenDBIObject.js';
-import { INTERNAL_DBIS_NAME, AUDIT_STORE_NAME } from '../utility/lmdb/terms.js';
+import { get } from '../utility/environment/environmentManager.ts';
+import OpenEnvironmentObject from '../utility/lmdb/OpenEnvironmentObject.ts';
+import { OpenDBIObject } from '../utility/lmdb/OpenDBIObject.ts';
+import { INTERNAL_DBIS_NAME, AUDIT_STORE_NAME } from '../utility/lmdb/terms.ts';
 import { CONFIG_PARAMS, DATABASES_DIR_NAME } from '../utility/hdbTerms.ts';
 import { AUDIT_STORE_OPTIONS } from '../resources/auditStore.ts';
-import { describeSchema } from '../dataLayer/schemaDescribe.js';
+import { describeSchema } from '../dataLayer/schemaDescribe.ts';
 import { updateConfigValue } from '../config/configUtils.js';
-import * as hdbLogger from '../utility/logging/harper_logger.js';
+import * as hdbLogger from '../utility/logging/harper_logger.ts';
 import { RocksDatabase, type RocksDatabaseOptions } from '@harperfast/rocksdb-js';
 import { RocksIndexStore } from '../resources/RocksIndexStore.ts';
 import { encodeBlobsWithFilePath } from '../resources/blob.ts';
@@ -163,7 +163,7 @@ export async function copyDb(sourceDatabase: string, targetDatabasePath: string)
 	const sourceDbisDb = rootStore.dbisDb;
 	const sourceAuditStore = rootStore.auditStore;
 	const targetEnv = open(new OpenEnvironmentObject(targetDatabasePath));
-	const targetDbisDb = targetEnv.openDB(INTERNAL_DBIS_NAME);
+	const targetDbisDb = targetEnv.openDB({ name: INTERNAL_DBIS_NAME });
 	let written;
 	let outstandingWrites = 0;
 	// we use a single transaction to get a snapshot, also we can't use snapshot: false on dupsort dbs
@@ -196,8 +196,8 @@ export async function copyDb(sourceDatabase: string, targetDatabasePath: string)
 			sourceDbi.decoderCopies = false;
 			sourceDbi.encoding = 'binary';
 			dbiInit.compression = newCompression;
-			const targetDbi = targetEnv.openDB(key, dbiInit);
-			targetDbi.encoder = null;
+			const targetDbi = (targetEnv as any).openDB(key, dbiInit);
+			(targetDbi as any).encoder = null;
 			console.log('copying', key, 'from', sourceDatabase, 'to', targetDatabasePath);
 			await copyDbi(sourceDbi, targetDbi, isPrimary, transaction);
 		}
@@ -291,7 +291,7 @@ function openRocksDb(path: string, options: RocksDatabaseOptions & { dupSort?: b
 	}
 	let db;
 	if (options.dupSort) {
-		db = new RocksIndexStore(path, options).open();
+		db = new (RocksIndexStore as any)(path, options).open();
 	} else {
 		db = RocksDatabase.open(path, options);
 		db.encoder.name = options.name;