@harperfast/harper 5.0.30 → 5.1.0-beta.2

package/config/RootConfigWatcher.ts

@@ -3,25 +3,78 @@ import { readFile } from 'node:fs/promises';
 import { getConfigFilePath } from './configUtils.js';
 import { EventEmitter, once } from 'node:events';
 import { parse } from 'yaml';
+import { POLLING_FALLBACK_OPTIONS, isWatcherExhaustionError, warnWatcherFallback } from '../utility/watcherFallback.ts';
 
 export class RootConfigWatcher extends EventEmitter {
 	#configFilePath: string;
-	#watcher: FSWatcher;
+	#watcher!: FSWatcher;
 	#config: any;
+	#usingPolling: boolean;
+	#closed: boolean;
+	#openCount: number = 0;
 	ready: Promise<any[]>;
 
 	constructor() {
 		super();
 		this.#configFilePath = getConfigFilePath();
+		this.#usingPolling = false;
+		this.#closed = false;
 		this.ready = once(this, 'ready');
+		this.#openWatcher();
+	}
+
+	#openWatcher() {
+		this.#openCount++;
 		this.#watcher = chokidar
-			.watch(this.#configFilePath, { persistent: false })
+			.watch(this.#configFilePath, {
+				persistent: false,
+				...(this.#usingPolling ? POLLING_FALLBACK_OPTIONS : {}),
+			})
 			.on('add', this.handleChange.bind(this))
 			.on('change', this.handleChange.bind(this))
 			.on('error', this.handleError.bind(this));
 	}
 
+	// Test-only: simulate the underlying chokidar watcher emitting an error.
+	// Exposed so the polling-fallback path can be exercised without triggering a
+	// real ENOSPC/EMFILE on the host.
+	_simulateWatcherErrorForTests(error: unknown): void {
+		this.handleError(error);
+	}
+
+	// Test-only: whether the watcher has fallen back to polling.
+	get _usingPollingForTests(): boolean {
+		return this.#usingPolling;
+	}
+
+	// Test-only: number of times the underlying watcher has been (re)opened.
+	get _openCountForTests(): number {
+		return this.#openCount;
+	}
+
 	handleError(error: unknown) {
+		if (isWatcherExhaustionError(error)) {
+			// Swallow every exhaustion error — chokidar can emit several before the
+			// failed native watcher closes, and we don't want a flurry of ENOSPC to
+			// surface to consumers in the middle of recovery.
+			if (!this.#usingPolling) {
+				warnWatcherFallback(this.#configFilePath);
+				this.#usingPolling = true;
+				// Guard against reopen-after-close: the caller may have invoked
+				// close() while this teardown was in flight. The .catch is required
+				// because `finally` would re-raise a teardown rejection as an
+				// unhandled one.
+				this.#watcher
+					.close()
+					.catch(() => {
+						// Teardown errors on an already-failed watcher are not actionable.
+					})
+					.finally(() => {
+						if (!this.#closed) this.#openWatcher();
+					});
+			}
+			return;
+		}
 		this.emit('error', error);
 	}
 
@@ -46,6 +99,7 @@ export class RootConfigWatcher extends EventEmitter {
 	}
 
 	close() {
+		this.#closed = true;
 		this.#watcher.close();
 		this.#config = undefined;
 		this.emit('close');

package/config/configUtils.js

@@ -1,9 +1,9 @@
 'use strict';
 
 const hdbTerms = require('../utility/hdbTerms.ts');
-const hdbUtils = require('../utility/common_utils.js');
-const logger = require('../utility/logging/harper_logger.js');
-const { configValidator } = require('../validation/configValidator.js');
+const hdbUtils = require('../utility/common_utils.ts');
+const logger = require('../utility/logging/harper_logger.ts');
+const { configValidator } = require('../validation/configValidator.ts');
 const fs = require('fs-extra');
 const YAML = require('yaml');
 const path = require('path');
@@ -12,8 +12,8 @@ const { randomBytes } = require('node:crypto');
 const isNumber = require('is-number');
 const PropertiesReader = require('properties-reader');
 const _ = require('lodash');
-const { handleHDBError } = require('../utility/errors/hdbError.js');
-const { HTTP_STATUS_CODES, HDB_ERROR_MSGS } = require('../utility/errors/commonErrors.js');
+const { handleHDBError } = require('../utility/errors/hdbError.ts');
+const { HTTP_STATUS_CODES, HDB_ERROR_MSGS } = require('../utility/errors/commonErrors.ts');
 const { server } = require('../server/Server.ts');
 const { getBackupDirPath } = require('./configHelpers.ts');
 const { PACKAGE_ROOT } = require('../utility/packageUtils');
@@ -65,7 +65,7 @@ function resolvePath(relativePath) {
 	if (relativePath?.startsWith('~/')) {
 		return path.join(hdbUtils.getHomeDir(), relativePath.slice(1));
 	}
-	const env = require('../utility/environment/environmentManager.js');
+	const env = require('../utility/environment/environmentManager.ts');
 	try {
 		return path.resolve(env.getHdbBasePath(), relativePath);
 	} catch (error) {
@@ -79,7 +79,7 @@ function resolvePath(relativePath) {
  * @param param
  */
 function getConfigPath(param) {
-	const env = require('../utility/environment/environmentManager.js');
+	const env = require('../utility/environment/environmentManager.ts');
 	const value = env.get(param);
 	if (!value || typeof value !== 'string') return value;
 	if (value.startsWith('~/')) {
@@ -97,7 +97,28 @@ function getConfigPath(param) {
 function atomicWriteFile(filePath, content) {
 	const tempPath = `${filePath}.${process.pid}.${threadId}.${randomBytes(4).toString('hex')}.tmp`;
 	fs.writeFileSync(tempPath, content);
-	fs.renameSync(tempPath, filePath);
+	let retries = 5;
+	while (true) {
+		try {
+			fs.renameSync(tempPath, filePath);
+			break;
+		} catch (err) {
+			if (retries > 0 && (err.code === 'EPERM' || err.code === 'EACCES')) {
+				retries--;
+				// sleep synchronously to allow the reader to close the file
+				const start = Date.now();
+				while (Date.now() - start < 10) {}
+				continue;
+			}
+			// if it fails we should clean up the tmp file
+			try {
+				fs.unlinkSync(tempPath);
+			} catch {
+				// ignore cleanup errors
+			}
+			throw err;
+		}
+	}
 }
 
 /**

package/config/harperConfigEnvVars.ts

@@ -25,7 +25,7 @@ const STATE_FILE_NAME = '.harper-config-state.json';
  * and ensure logger is initialized before use
  */
 function getLogger(): Logger {
-	const { loggerWithTag } = require('../utility/logging/harper_logger.js');
+	const { loggerWithTag } = require('../utility/logging/harper_logger');
 	return loggerWithTag('env-config');
 }
 

package/config-root.schema.json

@@ -284,6 +284,16 @@
 					}
 				},
 				"blobTimeout": { "type": "number", "description": "Max time (ms) to wait for blob transfer. Default: 120000" },
+				"pingInterval": {
+					"type": "number",
+					"minimum": 1,
+					"description": "Interval (ms) between replication keep-alive pings. Default: 30000"
+				},
+				"pingTimeout": {
+					"type": "number",
+					"minimum": 1,
+					"description": "Time (ms) without socket activity before a replication connection is considered dead and terminated. Default: 2x pingInterval"
+				},
 				"failOver": {
 					"type": "boolean",
 					"description": "Attempt fail-over to different node when unreachable. Default: true"
@@ -395,6 +405,17 @@
 						"timeout": { "type": "integer" }
 					}
 				},
+				"componentFile": {
+					"type": "object",
+					"additionalProperties": false,
+					"properties": {
+						"maxSize": {
+							"type": "integer",
+							"minimum": 1,
+							"description": "Maximum size in bytes for files returned by get_component_file. Default: 5242880 (5 MB)"
+						}
+					}
+				},
 				"tls": {
 					"$ref": "#/definitions/tlsConfig",
 					"description": "TLS configuration overriding root TLS for Operations API"
@@ -412,10 +433,6 @@
 					"description": "Disable fsync for faster writes (risk of data loss on crash). Default: false"
 				},
 				"caching": { "type": "boolean", "description": "Enable in-memory caching of records. Default: true" },
-				"randomAccessFields": {
-					"type": "boolean",
-					"description": "Encode records as typed random-access structures, optimizing for fast field access and smaller records. Best for tables with stable, homogeneous field types; wide or variably-typed schemas can set this to false to fall back to classic shared-structure encoding. Applies to each primary table on open: existing records still decode either way, only new writes change. Default: true"
-				},
 				"compression": {
 					"oneOf": [
 						{ "type": "boolean" },
@@ -443,6 +460,10 @@
 					"description": "Max write queue time before rejecting (e.g. '45s')."
 				},
 				"noReadAhead": { "type": "boolean", "description": "Advise OS to not read ahead. Default: false" },
+				"randomAccessFields": {
+					"type": "boolean",
+					"description": "Encode records as typed random-access structures, optimizing for fast field access and smaller records. Best for tables with stable, homogeneous field types; wide or variably-typed schemas should leave this off. Applies to each primary table on open, so changing it switches encoding for tables that have not pinned the choice via the @table(randomAccessFields:) directive (safe at runtime: existing records still decode, only new writes change). Default: false"
+				},
 				"prefetchWrites": { "type": "boolean", "description": "Load data prior to write transactions. Default: true" },
 				"path": { "type": "string", "description": "Directory for all database files. Default: <rootPath>/database" },
 				"blobPaths": {
@@ -554,6 +575,49 @@
 			}
 		},
 
+		"agent": {
+			"type": "object",
+			"description": "Built-in Harper Agent component. Disabled by default to avoid surprise LLM costs.",
+			"additionalProperties": false,
+			"properties": {
+				"enabled": { "type": "boolean", "description": "Enable the built-in agent component. Default: false" },
+				"provider": {
+					"type": "string",
+					"description": "Optional model provider override. Falls back to scope.models default when omitted."
+				},
+				"model": {
+					"type": "string",
+					"description": "Optional model id override. Falls back to scope.models default when omitted."
+				},
+				"maxTurns": {
+					"type": "integer",
+					"minimum": 1,
+					"description": "Maximum tool-call iterations per agent run. Default: 50"
+				},
+				"maxCostUsd": {
+					"type": "number",
+					"minimum": 0,
+					"description": "Per-session cost budget. NOT YET ENFORCED — surfacing operator intent until #612 ships per-call cost telemetry. Default: 5.00"
+				},
+				"autoApprove": {
+					"type": "boolean",
+					"description": "Run without per-action approval gates. Destructive ops still require allowDestructive and approval. Default: false"
+				},
+				"allowDestructive": {
+					"type": "boolean",
+					"description": "Allow destructive operator tools (drop_component, restart, set_configuration, ...). Default: false"
+				},
+				"user": {
+					"type": "string",
+					"description": "Harper user the agent acts as. Default: hdb_agent (created at startup if missing)."
+				},
+				"componentsScope": {
+					"type": "string",
+					"description": "Filesystem scope for component edits, relative to rootPath. Default: ./components"
+				}
+			}
+		},
+
 		"analytics": {
 			"type": "object",
 			"additionalProperties": false,
@@ -570,6 +634,16 @@
 					"type": "number",
 					"description": "Number of aggregation cycles between node storage measurements. 0 to disable. Default: 10"
 				},
+				"rawRetentionMs": {
+					"type": "number",
+					"minimum": 0,
+					"description": "Milliseconds to retain raw (pre-aggregation) analytics. Clamped to at least aggregatePeriod to prevent data loss. Default: 3600000 (1 hour)"
+				},
+				"aggregateRetentionMs": {
+					"type": "number",
+					"minimum": 0,
+					"description": "Milliseconds to retain aggregated analytics. 0 disables cleanup (keep forever). Default: 31536000000 (1 year)"
+				},
 				"logging": { "$ref": "#/definitions/loggerConfig" }
 			}
 		}

package/dataLayer/{CreateAttributeObject.js → CreateAttributeObject.ts}

@@ -1,11 +1,12 @@
 'use strict';
 
-const uuid = require('uuid');
+import { v4 as uuidv4 } from 'uuid';
 
 /**
  * Constructor class for inserting an attirbute in HDB
  */
 class CreateAttributeObject {
+	[key: string]: any;
 	/**
 	 *
 	 * @param schema
@@ -17,9 +18,9 @@ class CreateAttributeObject {
 		this.schema = schema;
 		this.table = table;
 		this.attribute = attribute;
-		this.id = id ? id : uuid.v4();
+		this.id = id ? id : uuidv4();
 		this.schema_table = `${this.schema}.${this.table}`;
 	}
 }
 
-module.exports = CreateAttributeObject;
+export default CreateAttributeObject;

package/dataLayer/{CreateTableObject.js → CreateTableObject.ts}

@@ -1,6 +1,7 @@
 'use strict';
 
 class CreateTableObject {
+	[key: string]: any;
 	constructor(schema, table, primary_key) {
 		this.schema = schema;
 		this.table = table;
@@ -8,4 +9,4 @@ class CreateTableObject {
 	}
 }
 
-module.exports = CreateTableObject;
+export default CreateTableObject;

package/dataLayer/{DataLayerObjects.js → DataLayerObjects.ts}

@@ -1,6 +1,11 @@
 'use strict';
 
-class InsertObject {
+export class InsertObject {
+	operation: string;
+	schema: string;
+	table: string;
+	hash_attribute: string;
+	records: any[];
 	constructor(operationString, schemaString, tableString, hashAttributeString, recordsArray) {
 		this.operation = operationString;
 		this.schema = schemaString;
@@ -10,7 +15,13 @@ class InsertObject {
 	}
 }
 
-class NoSQLSeachObject {
+export class NoSQLSeachObject {
+	schema: string;
+	table: string;
+	attribute: string;
+	hash_attribute: string;
+	get_attributes: string[];
+	value: any;
 	constructor(
 		schemaString,
 		tableString,
@@ -28,16 +39,13 @@ class NoSQLSeachObject {
 	}
 }
 
-class DeleteResponseObject {
+export class DeleteResponseObject {
+	message: string | undefined;
+	deleted_hashes: any[];
+	skipped_hashes: any;
 	constructor() {
 		this.message = undefined;
 		this.deleted_hashes = [];
 		this.skipped_hashes = [];
 	}
 }
-
-module.exports = {
-	InsertObject,
-	NoSQLSeachObject,
-	DeleteResponseObject,
-};

package/dataLayer/{DeleteBeforeObject.js → DeleteBeforeObject.ts}

@@ -7,6 +7,7 @@
  * @param {Date|Number|String} timestamp
  */
 class DeleteBeforeObject {
+	[key: string]: any;
 	/**
 	 * @param {string} schema
 	 * @param {string} table
@@ -19,4 +20,4 @@ class DeleteBeforeObject {
 	}
 }
 
-module.exports = DeleteBeforeObject;
+export default DeleteBeforeObject;

package/dataLayer/{DeleteObject.js → DeleteObject.ts}

@@ -1,11 +1,12 @@
 'use strict';
 
-const OPERATIONS_ENUM = require('../utility/hdbTerms.ts').OPERATIONS_ENUM;
+import { OPERATIONS_ENUM } from '../utility/hdbTerms.ts';
 
 /**
  * This class represents the data that is passed into the delete functions.
  */
 class DeleteObject {
+	[key: string]: any;
 	/**
 	 *
 	 * @param {string} schema
@@ -22,4 +23,4 @@ class DeleteObject {
 	}
 }
 
-module.exports = DeleteObject;
+export default DeleteObject;

package/dataLayer/{DropAttributeObject.js → DropAttributeObject.ts}

@@ -1,6 +1,7 @@
 'use strict';
 
 class DropAttributeObject {
+	[key: string]: any;
 	constructor(schema, table, attribute) {
 		this.schema = schema;
 		this.table = table;
@@ -8,4 +9,4 @@ class DropAttributeObject {
 	}
 }
 
-module.exports = DropAttributeObject;
+export default DropAttributeObject;

package/dataLayer/{GetBackupObject.js → GetBackupObject.ts}

@@ -1,11 +1,12 @@
 'use strict';
 
-const { OPERATIONS_ENUM } = require('../utility/hdbTerms.ts');
+import { OPERATIONS_ENUM } from '../utility/hdbTerms.ts';
 
 /**
  * class that represents the readAuditLog operation
  */
 class GetBackupObject {
+	[key: string]: any;
 	/**
 	 * @param {string} schema
 	 * @param {string} table
@@ -19,4 +20,4 @@ class GetBackupObject {
 	}
 }
 
-module.exports = GetBackupObject;
+export default GetBackupObject;

package/dataLayer/{InsertObject.js → InsertObject.ts}

@@ -1,9 +1,10 @@
 'use strict';
-const OPERATIONS_ENUM = require('../utility/hdbTerms.ts').OPERATIONS_ENUM;
+import { OPERATIONS_ENUM } from '../utility/hdbTerms.ts';
 /**
  * This class represents the data that is passed into the Insert functions.
  */
 class InsertObject {
+	[key: string]: any;
 	/**
 	 * @param {String} schema
 	 * @param {String} table
@@ -21,4 +22,4 @@ class InsertObject {
 	}
 }
 
-module.exports = InsertObject;
+export default InsertObject;

package/dataLayer/{ReadAuditLogObject.js → ReadAuditLogObject.ts}

@@ -1,11 +1,12 @@
 'use strict';
 
-const { OPERATIONS_ENUM } = require('../utility/hdbTerms.ts');
+import { OPERATIONS_ENUM } from '../utility/hdbTerms.ts';
 
 /**
  * class that represents the readAuditLog operation
  */
 class ReadAuditLogObject {
+	[key: string]: any;
 	/**
 	 * @param {string} schema
 	 * @param {string} table
@@ -21,4 +22,4 @@ class ReadAuditLogObject {
 	}
 }
 
-module.exports = ReadAuditLogObject;
+export default ReadAuditLogObject;