@harperfast/harper-pro 5.0.0-alpha.9 → 5.0.0-beta.2

package/core/integrationTests/server/operations-server.test.ts

@@ -180,7 +180,7 @@ suite('Operations Server', (ctx: ContextWithHarper) => {
 				operation: 'create_table',
 				schema: 'csv_test',
 				table: 'items',
-				hash_attribute: 'id',
+				primary_key: 'id',
 			}),
 		});
 

package/core/integrationTests/server/storage-reclamation.test.ts

@@ -78,7 +78,7 @@ suite('Storage reclamation', (ctx: ContextWithHarper) => {
 				operation: 'create_table',
 				schema: TEST_DATABASE,
 				table: TEST_TABLE,
-				hash_attribute: 'id',
+				primary_key: 'id',
 				expiration: 2, // 2 second expiration (in seconds)
 				eviction: 1, // 1 second eviction (in seconds)
 				audit: true, // Enable audit logging

package/core/integrationTests/utils/README.md

@@ -68,33 +68,19 @@ suite('My test suite', (ctx: ContextWithHarper) => {
 
 Configuration options for `setupHarper()`.
 
-**Interface Definition:**
-
 ```typescript
 export interface SetupHarperOptions {
-	/**
-	 * Timeout in milliseconds to wait for Harper to start.
-	 * @default 30000
-	 */
 	startupTimeoutMs?: number;
-	/**
-	 * Additional configuration options to pass to the Harper CLI.
-	 */
 	config: any;
-	/**
-	 * Environment variables to set when running Harper.
-	 */
 	env: any;
 }
 ```
 
 **Properties:**
 
-30000 (5 seconds), or the value of the `HARPER_INTEGRATION_TEST_STARTUP_TIMEOUT_MS` environment variable if set.
-
 - **`config`** - `object` (optional) - Additional configuration options to pass to the Harper CLI.
 - **`env`** - `object` (optional) - Additional environment variables to set when starting Harper.
-- **`startupTimeoutMs`** - `number` (optional) - Timeout in milliseconds to wait for Harper to start. Defaults to
+- **`startupTimeoutMs`** - `number` (optional) - Timeout in milliseconds to wait for Harper to start. Defaults to 30000, or the value of the `HARPER_INTEGRATION_TEST_STARTUP_TIMEOUT_MS` environment variable if set.
 
 **Environment Variables:**
 

package/core/integrationTests/utils/harperLifecycle.ts

@@ -112,9 +112,13 @@ interface RunHarperCommandOptions {
  */
 function runHarperCommand({ args, env, completionMessage, logDir }: RunHarperCommandOptions): Promise<ChildProcess> {
 	const harperScript = getHarperScript();
-	const proc = spawn('node', ['--trace-warnings', harperScript, ...args], {
-		env: { ...process.env, ...env },
-	});
+	const proc = spawn(
+		'node',
+		['--trace-warnings', '--force-node-api-uncaught-exceptions-policy=true', harperScript, ...args],
+		{
+			env: { ...process.env, ...env },
+		}
+	);
 
 	let stdoutStream: WriteStream | undefined;
 	let stderrStream: WriteStream | undefined;
@@ -156,12 +160,12 @@ function runHarperCommand({ args, env, completionMessage, logDir }: RunHarperCom
 		proc.on('error', (error) => {
 			reject(error);
 		});
-		proc.on('exit', (statusCode) => {
+		proc.on('exit', (statusCode, signal) => {
 			clearTimeout(timer);
 			if (statusCode === 0) {
 				resolve(proc);
 			} else {
-				let errorMessage = `Harper process failed with exit code ${statusCode}`;
+				let errorMessage = `Harper process failed with exit code/signal ${statusCode ?? signal}`;
 				stderrStream?.write(errorMessage);
 				if (stderr) {
 					errorMessage += `\n\nstderr:\n${stderr}`;
@@ -284,6 +288,29 @@ export async function startHarper(ctx: ContextWithHarper, options?: SetupHarperO
 	return ctx;
 }
 
+/**
+ * Kill harper process (can be used for teardown, or killing it before a restart)
+ * @param ctx
+ */
+export async function killHarper(ctx: ContextWithHarper): Promise<void> {
+	await new Promise<void>((resolve) => {
+		let timer: NodeJS.Timeout;
+		ctx.harper.process.on('exit', () => {
+			resolve();
+			clearTimeout(timer);
+		});
+		ctx.harper.process.kill();
+		timer = setTimeout(() => {
+			try {
+				ctx.harper.process.kill('SIGKILL');
+			} catch {
+				// possible that the process terminated but the exit event hasn't fired yet
+			}
+			resolve();
+		}, 200);
+	});
+}
+
 /**
  * Tears down a Harper instance and cleans up all resources.
  *
@@ -305,22 +332,7 @@ export async function startHarper(ctx: ContextWithHarper, options?: SetupHarperO
  * ```
  */
 export async function teardownHarper(ctx: ContextWithHarper): Promise<void> {
-	await new Promise<void>((resolve) => {
-		let timer: NodeJS.Timeout;
-		ctx.harper.process.on('exit', () => {
-			resolve();
-			clearTimeout(timer);
-		});
-		ctx.harper.process.kill();
-		timer = setTimeout(() => {
-			try {
-				ctx.harper.process.kill('SIGKILL');
-			} catch {
-				// possible that the process terminated but the exit event hasn't fired yet
-			}
-			resolve();
-		}, 200);
-	});
+	await killHarper(ctx);
 
 	await releaseLoopbackAddress(ctx.harper.hostname);
 

package/core/package.json

@@ -1,8 +1,7 @@
 {
 	"name": "harper",
 	"description": "Harper is an open-source Node.js performance platform that unifies database, cache, application, and messaging layers into one in-memory process.",
-	"version": "5.0.0-alpha.8",
-	"private": true,
+	"version": "5.0.0-beta.1",
 	"license": "Apache-2.0",
 	"homepage": "https://harper.fast",
 	"bugs": {
@@ -24,7 +23,25 @@
 	},
 	"files": [
 		"dist",
+		"bin",
+		"components",
+		"config",
+		"dataLayer",
+		"json",
+		"launchServiceScripts",
+		"resources",
+		"security",
+		"server",
+		"sqlTranslator",
+		"upgrade",
+		"utility",
+		"validation",
+		"studio",
 		"static",
+		"index.*",
+		"v1.*",
+		"v2.*",
+		"README.md",
 		"SECURITY.md",
 		"SUPPORT.md",
 		"CODE_OF_CONDUCT.md"
@@ -32,7 +49,7 @@
 	"scripts": {
 		"build": "tsc --project tsconfig.build.json",
 		"build:watch": "npm run build -- --watch --incremental",
-		"package": "npm run build; npm shrinkwrap && npm pack",
+		"package": "./build-tools/build.sh",
 		"lint": "oxlint --deny-warnings .",
 		"lint:required": "oxlint --quiet .",
 		"lint:fix": "npm run lint -- --fix",
@@ -140,7 +157,7 @@
 		"@fastify/cors": "~9.0.1",
 		"@fastify/static": "~7.0.4",
 		"@harperfast/extended-iterable": "^1.0.1",
-		"@harperfast/rocksdb-js": "^0.1.11",
+		"@harperfast/rocksdb-js": "^0.1.12",
 		"@turf/area": "6.5.0",
 		"@turf/boolean-contains": "6.5.0",
 		"@turf/boolean-disjoint": "6.5.0",
@@ -151,6 +168,7 @@
 		"@turf/helpers": "6.5.0",
 		"@turf/length": "6.5.0",
 		"alasql": "4.6.6",
+		"amaro": "^1.1.8",
 		"argon2": "0.44.0",
 		"asn1js": "3.0.7",
 		"cbor-x": "1.6.4",
@@ -175,7 +193,7 @@
 		"json2csv": "5.0.7",
 		"jsonata": "1.8.7",
 		"jsonwebtoken": "9.0.3",
-		"lmdb": "3.5.1",
+		"lmdb": "3.5.2",
 		"lodash": "4.17.21",
 		"mathjs": "11.12.0",
 		"micromatch": "^4.0.8",

package/core/resources/ResourceInterface.ts

@@ -30,7 +30,7 @@ export interface ResourceInterface<Record extends object = any>
 	allowUpdate(user: User, record: Promise<Record & RecordObject>, context: Context): boolean | Promise<boolean>;
 	put?(
 		record: Record & RecordObject,
-		target: RequestTargetOrId
+		target?: RequestTargetOrId
 	): void | (Record & Partial<RecordObject>) | Promise<void | (Record & Partial<RecordObject>)>;
 	patch?(
 		record: Partial<Record & RecordObject>,

package/core/resources/Table.ts

@@ -71,11 +71,17 @@ const { validateAttribute } = lmdbProcessRows;
 
 export type Attribute = {
 	name: string;
-	type: string;
+	type: 'ID' | 'Int' | 'Float' | 'Long' | 'String' | 'Boolean' | 'Date' | 'Bytes' | 'Any' | 'BigInt' | 'Blob' | string;
 	assignCreatedTime?: boolean;
 	assignUpdatedTime?: boolean;
+	nullable?: boolean;
 	expiresAt?: boolean;
 	isPrimaryKey?: boolean;
+	indexed?: unknown;
+	relationship?: unknown;
+	computed?: unknown;
+	properties?: Array<Attribute>;
+	elements?: Attribute;
 };
 
 type MaybePromise<T> = T | Promise<T>;
@@ -106,7 +112,7 @@ export interface Table {
 	databasePath: string;
 	tableName: string;
 	databaseName: string;
-	attributes: any[];
+	attributes: Attribute[];
 	primaryKey: string;
 	splitSegments?: boolean;
 	replicate?: boolean;
@@ -141,7 +147,7 @@ export function makeTable(options) {
 	} = options;
 	let { expirationMS: expirationMs, evictionMS: evictionMs, audit, trackDeletes } = options;
 	evictionMs ??= 0;
-	let { attributes } = options;
+	let { attributes }: { attributes: Attribute[] } = options;
 	if (!attributes) attributes = [];
 	const updateRecord = recordUpdater(primaryStore, tableId, auditStore);
 	let sourceLoad: any; // if a source has a load function (replicator), record it here
@@ -2233,11 +2239,12 @@ export function makeTable(options) {
 				entries = transformToEntries(entries, select, context, readTxn, null);
 				let ordered;
 				// if we are doing post-ordering, we need to get records first, then sort them
-				results.iterate = function () {
+				results.iterate = function (options: { async: boolean }) {
 					let sortedArrayIterator: IterableIterator<any>;
-					const dbIterator = entries[Symbol.asyncIterator]
-						? entries[Symbol.asyncIterator]()
-						: entries[Symbol.iterator]();
+					const dbIterator =
+						options?.async && entries[Symbol.asyncIterator]
+							? entries[Symbol.asyncIterator]()
+							: entries[Symbol.iterator]();
 					let dbDone: boolean;
 					const dbOrderedAttribute = sort.dbOrderedAttribute;
 					let enqueuedEntryForNextGroup: any;
@@ -2351,7 +2358,10 @@ export function makeTable(options) {
 				};
 				applySortingOnSelect(sort);
 			} else {
-				results.iterate = (entries[Symbol.asyncIterator] || entries[Symbol.iterator]).bind(entries);
+				results.iterate = (options: { async: boolean }) => {
+					if (options?.async && entries[Symbol.asyncIterator]) return entries[Symbol.asyncIterator]();
+					else return entries[Symbol.iterator]();
+				};
 				results = results.map(function (entry) {
 					try {
 						// because this is a part of a stream of results, we will often be continuing to iterate over the results when there are errors,
@@ -2893,7 +2903,7 @@ export function makeTable(options) {
 		}
 		validate(record: any, patch?: boolean) {
 			let validationErrors;
-			const validateValue = (value, attribute, name) => {
+			const validateValue = (value, attribute: Attribute, name) => {
 				if (attribute.type && value != null) {
 					if (patch && value.__op__) value = value.value;
 					if (attribute.properties) {
@@ -3063,7 +3073,7 @@ export function makeTable(options) {
 		getUpdatedTime() {
 			return this.#version;
 		}
-		static async addAttributes(attributesToAdd) {
+		static async addAttributes(attributesToAdd: Attribute[]) {
 			const new_attributes = attributes.slice(0);
 			for (const attribute of attributesToAdd) {
 				if (!attribute.name) throw new ClientError('Attribute name is required');
@@ -3927,7 +3937,12 @@ export function makeTable(options) {
 			// it should be resolved now and we can use the value it saved.
 			clearTimeout(timer);
 			const entry = primaryStore.getEntry(id);
-			if (!entry || !entry.value || entry.metadataFlags & (INVALIDATED | EVICTED))
+			if (
+				!entry ||
+				!entry.value ||
+				entry.metadataFlags & (INVALIDATED | EVICTED) ||
+				(entry.expiresAt != undefined && entry.expiresAt < Date.now())
+			)
 				// try again
 				whenResolved(getFromSource(source, id, primaryStore.getEntry(id), context));
 			else whenResolved(entry);

package/core/resources/analytics/read.ts

@@ -1,7 +1,7 @@
 import type { Metric } from './write.ts';
 import harperLogger from '../../utility/logging/harper_logger.js';
 const { forComponent } = harperLogger;
-import { getAnalyticsHostnameTable } from './hostnames.ts';
+import { getAnalyticsHostnameTable, stableNodeId } from './hostnames.ts';
 import type { Condition, Conditions } from '../ResourceInterface.ts';
 import { METRIC, type BuiltInMetricName } from './metadata.ts';
 import { CONFIG_PARAMS } from '../../utility/hdbTerms.ts';
@@ -12,9 +12,11 @@ const defaultCustomMetricWindow = 1000 * 60 * 60 * 24 * 7;
 
 const log = forComponent('analytics').conditional;
 
-async function lookupHostname(nodeId: number): Promise<string> {
+async function lookupHostname(nodeId: number): Promise<string | undefined> {
 	const result = await getAnalyticsHostnameTable().get(nodeId);
-	return result.hostname;
+	if (result?.hostname) return result.hostname;
+	if (nodeId === stableNodeId(server.hostname)) return server.hostname;
+	return undefined;
 }
 
 function isSelected(querySelect: string[], attr: string) {
@@ -57,25 +59,21 @@ function conformCondition(condition: Condition): Condition {
 	};
 }
 
-async function coalesceResults(results: Metric[], window: number): Promise<Metric[]> {
-	const coalescedResults: Metric[] = [];
-	let coalesceId;
-	let lastCoalescedId = new Map<string, number>();
+async function* coalesceResults(results: Metric[], window: number): AsyncGenerator<Metric> {
+	let coalesceId: any;
 	for await (const result of results) {
 		const id = result.id;
 		if (!coalesceId) {
 			coalesceId = id;
 		}
 		const delta = Math.abs(id - coalesceId);
-		if (delta < window && lastCoalescedId.get(result.node) !== id) {
-			coalescedResults.push({ ...result, id: coalesceId });
-			lastCoalescedId[result.node] = id;
+		if (delta < window) {
+			yield { ...result, id: coalesceId };
 		} else {
-			coalescedResults.push(result);
+			yield result;
 			coalesceId = id;
 		}
 	}
-	return coalescedResults;
 }
 
 interface GetAnalyticsOpts {
@@ -99,19 +97,27 @@ export async function get(metric: string, opts?: GetAnalyticsOpts): Promise<Metr
 		select.push('id');
 	}
 
-	if (startTime) {
+	if (startTime && endTime) {
 		conditions.push({
 			attribute: 'id',
-			comparator: 'greater_than_equal',
-			value: startTime,
-		});
-	}
-	if (endTime) {
-		conditions.push({
-			attribute: 'id',
-			comparator: 'less_than',
-			value: endTime,
+			comparator: 'between',
+			value: [startTime, endTime],
 		});
+	} else {
+		if (startTime) {
+			conditions.push({
+				attribute: 'id',
+				comparator: 'greater_than_equal',
+				value: startTime,
+			});
+		}
+		if (endTime) {
+			conditions.push({
+				attribute: 'id',
+				comparator: 'less_than',
+				value: endTime,
+			});
+		}
 	}
 
 	const request = { conditions, allowConditionsOnDynamicAttributes: true };
@@ -128,16 +134,17 @@ export async function get(metric: string, opts?: GetAnalyticsOpts): Promise<Metr
 		result['id'] = result['id'][0];
 		if (isSelected(select, 'node')) {
 			log.trace?.(`get_analytics lookup hostname for nodeId: ${nodeId}`);
-			result['node'] = await lookupHostname(nodeId);
+			const hostname = await lookupHostname(nodeId);
+			result['node'] = hostname ?? nodeId;
 		}
 		log.trace?.(`get_analytics result:`, JSON.stringify(result));
 		return result;
 	});
 
 	if (opts?.coalesceTime) {
-		// coalescing window is the aggregate period plus 10% & converted to milliseconds
-		const window = envGet(CONFIG_PARAMS.ANALYTICS_AGGREGATEPERIOD) * 1.1 * 1000;
-		results = await coalesceResults(results, window);
+		// coalescing window is the aggregate period converted to milliseconds
+		const window = envGet(CONFIG_PARAMS.ANALYTICS_AGGREGATEPERIOD) * 1000;
+		results = coalesceResults(results, window);
 	}
 
 	return results;

package/core/resources/analytics/write.ts

@@ -1,5 +1,5 @@
 import { parentPort, threadId } from 'worker_threads';
-import { setChildListenerByType } from '../../server/threads/manageThreads.js';
+import { onMessageByType } from '../../server/threads/manageThreads.js';
 import { getDatabases, table } from '../databases.ts';
 import type { Databases, Table, Tables } from '../databases.ts';
 import harperLogger from '../../utility/logging/harper_logger.js';
@@ -14,10 +14,6 @@ import { server } from '../../server/Server.ts';
 import * as fs from 'node:fs';
 import { getAnalyticsHostnameTable, nodeIds, stableNodeId } from './hostnames.ts';
 import { METRIC } from './metadata.ts';
-setTimeout(() => {
-	// let everything load before we actually load and start the profiler
-	import('./profile.ts');
-}, 1000);
 import { RocksDatabase } from '@harperfast/rocksdb-js';
 
 const log = forComponent('analytics').conditional;
@@ -230,7 +226,7 @@ export async function recordHostname() {
 			hostname,
 		};
 		log.trace?.(`recordHostname storing hostname: ${JSON.stringify(hostnameRecord)}`);
-		hostnamesTable.put(hostnameRecord.id, hostnameRecord);
+		await hostnamesTable.put(hostnameRecord.id, hostnameRecord);
 	}
 }
 
@@ -654,7 +650,7 @@ function getAnalyticsTable() {
 	);
 }
 
-setChildListenerByType(ANALYTICS_REPORT_TYPE, recordAnalytics);
+if (!parentPort) onMessageByType(ANALYTICS_REPORT_TYPE, recordAnalytics);
 let scheduledTasksRunning;
 function startScheduledTasks() {
 	scheduledTasksRunning = true;

package/core/resources/databases.ts

@@ -1,3 +1,4 @@
+import { EventEmitter } from 'node:events';
 import { initSync, getHdbBasePath, get as envGet } from '../utility/environment/environmentManager.js';
 import { INTERNAL_DBIS_NAME } from '../utility/lmdb/terms.js';
 import { open, compareKeys, type Database, type RootDatabase } from 'lmdb';
@@ -97,6 +98,14 @@ interface RocksRootDatabase extends RocksDatabaseEx {
 
 export type RootDatabaseKind = LMDBRootDatabase | RocksRootDatabase;
 
+export type DatabaseWatcherEventMap = {
+	updateTable: [table: Table, originIsNotCluster?: boolean];
+	dropTable: [tableName: string, databaseName: string];
+	dropDatabase: [databaseName: string];
+};
+
+export const databaseEventsEmitter = new EventEmitter<DatabaseWatcherEventMap>();
+
 export const tables: Tables = Object.create(null);
 export const databases: Databases = Object.create(null);
 
@@ -127,8 +136,6 @@ _assignPackageExport('databases', databases);
 _assignPackageExport('tables', tables);
 
 const NEXT_TABLE_ID = Symbol.for('next-table-id');
-const tableListeners = [];
-const dbRemovalListeners = [];
 let loadedDatabases; // indicates if we have loaded databases from the file system yet
 
 // This is used to track all the databases that are found when iterating through the file system so that anything that is missing
@@ -577,9 +584,7 @@ function initStores(
 				})
 			);
 			table.schemaVersion = 1;
-			for (const listener of tableListeners) {
-				listener(table);
-			}
+			databaseEventsEmitter.emit('updateTable', table);
 		}
 	}
 	return rootStore;
@@ -600,7 +605,7 @@ export function resetDatabases() {
 				const table = db[tableName];
 				if (table.primaryStore.path === path) {
 					delete databases[store.databaseName];
-					dbRemovalListeners.forEach((listener) => listener(store.databaseName));
+					databaseEventsEmitter.emit('dropDatabase', store.databaseName);
 					break;
 				}
 			}
@@ -745,6 +750,7 @@ export async function dropDatabase(databaseName) {
 				await unlink(rootStore.path);
 			}
 		}
+		databaseEventsEmitter.emit('dropTable', tableName, databaseName);
 	}
 	if (!rootStore) {
 		rootStore = database({ database: databaseName, table: null });
@@ -762,7 +768,7 @@ export async function dropDatabase(databaseName) {
 		delete tables[DEFINED_TABLES];
 	}
 	delete databases[databaseName];
-	dbRemovalListeners.forEach((listener) => listener(databaseName));
+	databaseEventsEmitter.emit('dropDatabase', databaseName);
 	await deleteRootBlobPathsForDB(rootStore);
 }
 // opens an index, consulting with custom indexes that may use alternate store configuration
@@ -1090,9 +1096,7 @@ export function table<TableResourceType>(tableDefinition: TableDefinition): Tabl
 
 	Table.origin = origin;
 	if (hasChanges) {
-		for (const listener of tableListeners) {
-			listener(Table, origin !== 'cluster');
-		}
+		databaseEventsEmitter.emit('updateTable', Table, origin !== 'cluster');
 	}
 	if (expiration || eviction || scanInterval)
 		Table.setTTLExpiration({
@@ -1244,24 +1248,31 @@ export function dropTableMeta({ table: tableName, database: databaseName }) {
 	for (const key of dbisDb.getKeys({ start: tableName + '/', end: tableName + '0' })) {
 		removals.push(dbisDb.remove(key));
 	}
+	databaseEventsEmitter.emit('dropTable', tableName, databaseName);
 	return Promise.all(removals);
 }
 
-export function onUpdatedTable(listener) {
-	tableListeners.push(listener);
+export function onUpdatedTable(listener: (table: Table) => void) {
+	databaseEventsEmitter.on('updateTable', listener);
+	return {
+		remove() {
+			databaseEventsEmitter.off('updateTable', listener);
+		},
+	};
+}
+export function onRemovedTable(listener: (tableName: string, databaseName: string) => void) {
+	databaseEventsEmitter.on('dropTable', listener);
 	return {
 		remove() {
-			const index = tableListeners.indexOf(listener);
-			if (index > -1) tableListeners.splice(index, 1);
+			databaseEventsEmitter.off('dropTable', listener);
 		},
 	};
 }
-export function onRemovedDB(listener) {
-	dbRemovalListeners.push(listener);
+export function onRemovedDB(listener: (databaseName: string) => void) {
+	databaseEventsEmitter.on('dropDatabase', listener);
 	return {
 		remove() {
-			const index = dbRemovalListeners.indexOf(listener);
-			if (index > -1) dbRemovalListeners.splice(index, 1);
+			databaseEventsEmitter.off('dropDatabase', listener);
 		},
 	};
 }

package/core/resources/search.ts

@@ -337,11 +337,16 @@ export function searchByIndex(
 							})
 						);
 					}
-				: (entry) => {
-						if (entry.value == null && !(entry.metadataFlags & (INVALIDATED | EVICTED))) return SKIP;
-						Object.freeze(entry.value);
-						recordRead(entry);
-						return entry;
+				: function (entry) {
+						let result: any;
+						if (entry.value == null && !(entry.metadataFlags & (INVALIDATED | EVICTED))) result = SKIP;
+						else {
+							Object.freeze(entry.value);
+							recordRead(entry);
+							result = entry;
+						}
+						if (this.isSync) return result;
+						return new Promise((resolve) => setImmediate(() => resolve(result)));
 					}
 		);
 		results.hasEntries = true;

package/core/security/auth.ts

@@ -102,7 +102,7 @@ export async function authentication(request, nextHandler) {
 					break;
 				}
 			}
-			request.session = session || (session = {});
+			request.session = session ? { ...session } : (session = {});
 		}
 
 		const authAuditLog = (username, status, strategy) => {