@harperfast/harper 5.0.0-alpha.10 → 5.0.0-beta.1

package/resources/jsResource.ts

@@ -0,0 +1,86 @@
+import { Scope } from '../components/Scope.ts';
+import { dirname } from 'path';
+
+function isResource(value: any) {
+	return (
+		value &&
+		(typeof value.get === 'function' ||
+			typeof value.put === 'function' ||
+			typeof value.post === 'function' ||
+			typeof value.delete === 'function')
+	);
+}
+
+/**
+ * Error thrown when a JavaScript resource module fails to load
+ */
+export class ResourceLoadError extends Error {
+	public readonly filePath: string;
+	public readonly cause?: Error;
+
+	constructor(filePath: string, cause?: Error) {
+		super(`Failed to load resource module ${filePath}${cause ? `: ${cause.message}` : ''}`);
+		this.name = 'ResourceLoadError';
+		this.filePath = filePath;
+		this.cause = cause;
+	}
+}
+
+/**
+ * This plugin loads JavaScript files and registers their exports as resources.
+ *
+ * The export can be the default export and will be assigned to the root URL path.
+ *
+ * Otherwise, the name of the export will be used.
+ *
+ * After loading the JavaScript code using the secure import, it adds it to the global `resources` map.
+ *
+ * Once a file has been loaded it cannot be unloaded without a restart.
+ *
+ * Thus, this plugin only handle files as they are added (`add` event). All other events result in a restart request.
+ *
+ */
+export async function handleApplication(scope: Scope) {
+	scope.handleEntry(async function handleResourceEntry(entryEvent) {
+		if (entryEvent.entryType !== 'file') {
+			scope.logger.warn(
+				`jsResource plugin cannot handle entry type ${entryEvent.entryType}. Modify the 'files' option in ${scope.configFilePath} to only include files.`
+			);
+			return;
+		}
+
+		if (entryEvent.eventType !== 'add') {
+			scope.requestRestart();
+			return;
+		}
+
+		try {
+			const resourceModule: any = await scope.import(entryEvent.absolutePath);
+			const root = dirname(entryEvent.urlPath).replace(/\\/g, '/').replace(/^\/$/, '');
+			if (isResource(resourceModule.default)) {
+				// register the resource
+				scope.resources.set(root, resourceModule.default);
+				scope.logger.debug?.(`Registered root resource: ${root}`);
+			}
+			recurseForResources(scope, resourceModule, root);
+		} catch (error) {
+			// Rethrow with more context
+			throw new ResourceLoadError(entryEvent.absolutePath, error);
+		}
+	});
+}
+
+function recurseForResources(scope: Scope, resourceModule: any, prefix: string) {
+	for (const name in resourceModule) {
+		// check each of the module exports to see if it implements a Resource handler
+		const exported = resourceModule[name];
+		const resourcePath = `${prefix}/${name}`;
+		if (isResource(exported)) {
+			// expose as an endpoint
+			scope.resources.set(resourcePath, exported);
+			scope.logger.debug?.(`Registered resource: ${resourcePath}`);
+		} else if (typeof exported === 'object') {
+			recurseForResources(scope, exported, resourcePath);
+		}
+	}
+}

package/resources/loadEnv.ts

@@ -0,0 +1,22 @@
+import { parse } from 'dotenv';
+import logger from '../utility/logging/harper_logger.js';
+
+export function start({ override }: { override: boolean }) {
+	return {
+		handleFile: (contents, _, filePath) => {
+			logger.debug(`Loading env file: ${filePath}`);
+			for (const [key, value] of Object.entries(parse(contents))) {
+				if (process.env[key] !== undefined) {
+					logger.warn(`Environment variable conflict: ${key} from ${filePath} is already set on process.env`);
+					if (override) {
+						logger.debug(`override option enabled. overriding environment variable: ${key}`);
+					} else {
+						continue;
+					}
+				}
+
+				process.env[key] = value;
+			}
+		},
+	};
+}

package/resources/login.ts

@@ -0,0 +1,18 @@
+import { Resource } from './Resource.ts';
+export function start({ resources }) {
+	resources.set('login', Login);
+	resources.loginPath = (request) => {
+		return '/login?redirect=' + encodeURIComponent(request.url);
+	};
+}
+class Login extends Resource {
+	static async get(_id, _body, _request) {
+		// TODO: Return a login page
+	}
+	static async post(_id, body, request) {
+		const { username, password } = body;
+		return {
+			data: await request.login(username, password),
+		};
+	}
+}

package/resources/openApi.ts

@@ -0,0 +1,409 @@
+import { packageJson } from '../utility/packageUtils.js';
+import { Resources } from './Resources.ts';
+import { Resource } from './Resource.ts';
+
+const OPENAPI_VERSION = '3.0.3';
+// Maps graphql primitive types to open api types
+const DATA_TYPES = {
+	Int: 'integer',
+	Float: 'number',
+	Long: 'integer',
+	ID: 'string',
+	String: 'string',
+	Boolean: 'boolean',
+	Date: 'string',
+	Bytes: 'string',
+	BigInt: 'integer',
+};
+
+const SCHEMA_COMP_REF = '#/components/schemas/';
+const DESCRIPTION_200 = 'successful operation';
+
+export function generateJsonApi(resources: Resources, serverHttpURL: string) {
+	const api = {
+		openapi: OPENAPI_VERSION,
+		info: {
+			title: 'Harper HTTP REST interface',
+			version: packageJson.version,
+		},
+		servers: [
+			{
+				description: 'REST API',
+				url: serverHttpURL,
+			},
+		],
+		paths: {},
+		components: {
+			schemas: {},
+			securitySchemes: {
+				basicAuth: {
+					type: 'http',
+					scheme: 'basic',
+				},
+				bearerAuth: {
+					type: 'http',
+					scheme: 'bearer',
+					bearerFormat: 'JWT',
+				},
+			},
+		},
+	};
+
+	const security = [
+		{
+			basicAuth: [],
+			bearerAuth: [],
+		},
+	];
+
+	const includeDefinitionInSchema = (def) => {
+		if (def.type && !api.components.schemas[def.type]) {
+			// Immediately define the type so we don't get caught in infinite recursions, until...
+			api.components.schemas[def.type] = {};
+			const defProps: Record<string, unknown> = {};
+			const defRequired: string[] = [];
+			for (const prop of def.properties) {
+				if (DATA_TYPES[prop.type]) {
+					defProps[prop.name] = new Type(DATA_TYPES[prop.type], prop.type);
+				} else if (prop.properties) {
+					defProps[prop.name] = new Ref(prop.type);
+					includeDefinitionInSchema(prop);
+				} else if (prop.elements?.properties) {
+					defProps[prop.name] = new ArrayRef(prop.elements.type);
+					includeDefinitionInSchema(prop.elements);
+				}
+				if (prop.nullable === false) {
+					defRequired.push(prop.name);
+				}
+			}
+			// ... down here we actually define the value for the new type.
+			api.components.schemas[def.type] = new ResourceSchema(defProps, !def.sealed, defRequired);
+		}
+	};
+
+	for (const [, resource] of resources) {
+		// skip invalid and error resources
+		if (!resource.path || resource.Resource.isError) continue;
+
+		const { path } = resource;
+		const strippedPath = path.split('/').pop(); // strip any namespace from path
+		let { attributes, sealed } = resource.Resource;
+		const { prototype, primaryKey = 'id' } = resource.Resource;
+		if (!attributes && resources.allTypes.has(resource.path)) {
+			const possibleType = resources.allTypes.get(resource.path);
+			sealed = possibleType.sealed;
+			attributes = possibleType.properties;
+		}
+		if (!primaryKey) continue;
+		const props = {};
+		const queryParamsArray = [];
+		const resourceRequired: string[] = [];
+
+		if (attributes) {
+			for (const { type, name, elements, relationship, definition, nullable } of attributes) {
+				const def = definition ?? elements?.definition;
+				if (def) {
+					includeDefinitionInSchema(def);
+				}
+
+				if (nullable === false) {
+					resourceRequired.push(name);
+				}
+				if (relationship) {
+					if (type === 'array') {
+						props[name] = { type: 'array', items: { $ref: SCHEMA_COMP_REF + elements.type } };
+					} else {
+						props[name] = { $ref: SCHEMA_COMP_REF + type };
+					}
+				} else {
+					if (def) {
+						if (type === 'array') {
+							props[name] = { type: 'array', items: { $ref: SCHEMA_COMP_REF + def.type } };
+						} else {
+							props[name] = { $ref: SCHEMA_COMP_REF + def.type };
+						}
+					} else if (type === 'array') {
+						if (elements.type === 'Any') {
+							props[name] = { type: 'array', items: { format: elements.type } };
+						} else {
+							props[name] = { type: 'array', items: new Type(DATA_TYPES[elements.type], elements.type) };
+						}
+					} else if (type === 'Any') {
+						props[name] = { format: type };
+					} else {
+						props[name] = new Type(DATA_TYPES[type], type);
+					}
+				}
+				queryParamsArray.push(new Parameter(name, 'query', props[name]));
+			}
+		}
+
+		const propsArray = Object.keys(props);
+		const primaryKeyParam = new Parameter(primaryKey, 'path', { type: 'string', format: 'ID' });
+		primaryKeyParam.required = true;
+		primaryKeyParam.description = 'primary key of record';
+		const propertyParamPath = new Parameter('property', 'path', { enum: propsArray });
+		propertyParamPath.required = true;
+		api.components.schemas[strippedPath] = new ResourceSchema(props, !sealed, resourceRequired);
+
+		const hasPost = prototype.post !== Resource.prototype.post || prototype.update;
+		const hasPut = typeof prototype.put === 'function';
+		const hasGet = typeof prototype.get === 'function';
+		const hasDelete = typeof prototype.delete === 'function';
+		const hasPatch = typeof prototype.patch === 'function';
+
+		const url = `/${path}/`;
+		if (!api.paths[url]) {
+			api.paths[url] = {};
+		}
+
+		// API for path structure /my-resource/
+		if (hasPost) {
+			api.paths[url].post = new Post(
+				strippedPath,
+				security,
+				{
+					'200': new Response200(
+						{ $ref: SCHEMA_COMP_REF + strippedPath },
+						{
+							Location: {
+								description: 'primary key of new record',
+								schema: {
+									type: 'string',
+									format: 'ID',
+								},
+							},
+						}
+					),
+				},
+				'create a new record auto-assigning a primary key'
+			);
+		}
+
+		api.paths[url].options = new Options(
+			queryParamsArray,
+			security,
+			{ '200': new ResponseOptions200() },
+			'retrieve information about the communication options available for a target resource or the server as a whole, without performing any resource action'
+		);
+
+		if (hasGet) {
+			api.paths[url].get = new Get(
+				queryParamsArray,
+				security,
+				{ '200': new Response200({ type: 'array', items: { $ref: SCHEMA_COMP_REF + strippedPath } }) },
+				'search for records by the specified property name and value pairs'
+			);
+		}
+
+		if (hasDelete) {
+			api.paths[url].delete = new Delete(
+				queryParamsArray,
+				security,
+				'delete all the records that match the provided query',
+				{
+					'204': new Response204(),
+				}
+			);
+		}
+
+		// API for path structure /my-resource/<record-id>
+		const urlById = '/' + path + '/{' + primaryKey + '}';
+		if (!api.paths[urlById]) {
+			api.paths[urlById] = {};
+		}
+
+		api.paths[urlById].options = new Options(
+			queryParamsArray,
+			security,
+			{ '200': new ResponseOptions200() },
+			'retrieve information about the communication options available for a target resource or the server as a whole, without performing any resource action'
+		);
+
+		if (hasGet) {
+			api.paths[urlById].get = new Get(
+				[primaryKeyParam],
+				security,
+				{ '200': new Response200({ $ref: SCHEMA_COMP_REF + strippedPath }) },
+				'retrieve a record by its primary key'
+			);
+		}
+
+		if (hasPut) {
+			api.paths[urlById].put = new Put(
+				[primaryKeyParam],
+				security,
+				strippedPath,
+				{ '200': new Response200({ $ref: SCHEMA_COMP_REF + strippedPath }) },
+				"create or update the record with the URL path that maps to the record's primary key"
+			);
+		}
+
+		if (hasPatch) {
+			api.paths[urlById].patch = new Patch(
+				[primaryKeyParam],
+				security,
+				strippedPath,
+				{ '200': new Response200({ $ref: SCHEMA_COMP_REF + strippedPath }) },
+				"patch the record with the URL path that maps to the record's primary key"
+			);
+		}
+
+		if (hasDelete) {
+			api.paths[urlById].delete = new Delete(
+				[primaryKeyParam],
+				security,
+				'delete a record with the given primary key',
+				{
+					'204': new Response204(),
+				}
+			);
+		}
+
+		// API for path structure /my-resource/<record-id>.property
+		if (hasGet && propertyParamPath.schema.enum.length > 0) {
+			const urlByProperty = `/${path}/{${primaryKey}}.{property}`;
+			if (!api.paths[urlByProperty]) {
+				api.paths[urlByProperty] = {};
+			}
+			api.paths[urlByProperty].get = new Get(
+				[primaryKeyParam, propertyParamPath],
+				security,
+				{
+					'200': new Response200({ enum: propsArray }),
+				},
+
+				'used to retrieve the specified property of the specified record'
+			);
+		}
+	}
+
+	for (const [, value] of resources.allTypes) {
+		includeDefinitionInSchema(value);
+		if (value.sealed && api.components.schemas[value.type].additionalProperties) {
+			api.components.schemas[value.type].additionalProperties = false;
+		}
+	}
+
+	return api;
+}
+
+function Post(path, security, responses, description) {
+	this.description = description;
+	this.requestBody = {
+		content: {
+			'application/json': {
+				schema: {
+					$ref: SCHEMA_COMP_REF + path,
+				},
+			},
+		},
+	};
+
+	this.security = security;
+	this.responses = responses;
+}
+
+function Get(parameters, security, responses, description) {
+	this.description = description;
+	this.parameters = parameters;
+	this.security = security;
+	this.responses = responses;
+}
+
+function Options(parameters, security, responses, description) {
+	this.description = description;
+	this.parameters = parameters;
+	this.security = security;
+	this.responses = responses;
+}
+
+function ResponseOptions200() {
+	this.description = DESCRIPTION_200;
+	this.headers = {};
+	this.content = {};
+}
+
+function Response200(schema, headers?: any) {
+	this.description = DESCRIPTION_200;
+	this.content = {
+		'application/json': {
+			schema,
+		},
+	};
+	this.headers = headers;
+}
+
+function Response204() {
+	this.description = 'successfully processed request, no content returned to client';
+}
+
+function Put(parameters, security, path, responses, description) {
+	this.description = description;
+	this.parameters = parameters;
+	this.security = security;
+	this.requestBody = {
+		content: {
+			'application/json': {
+				schema: {
+					$ref: SCHEMA_COMP_REF + path,
+				},
+			},
+		},
+	};
+	this.responses = responses;
+}
+
+function Patch(parameters, security, path, responses, description) {
+	this.description = description;
+	this.parameters = parameters;
+	this.security = security;
+	this.requestBody = {
+		content: {
+			'application/json': {
+				schema: {
+					$ref: SCHEMA_COMP_REF + path,
+				},
+			},
+		},
+	};
+	this.responses = responses;
+}
+
+function Delete(parameters, security, description, responses) {
+	this.description = description;
+	this.parameters = parameters;
+	this.security = security;
+	this.responses = responses;
+}
+
+function ResourceSchema(properties, additionalProperties?: boolean, required?: string[]) {
+	this.type = 'object';
+	this.properties = properties;
+	this.additionalProperties = additionalProperties;
+	this.required = required;
+}
+
+function Type(type, format) {
+	this.type = type;
+	if (type === 'string' || type === 'number' || type === 'integer') {
+		if (format !== 'String') {
+			this.format = format;
+		}
+	}
+}
+
+function Ref(ref: string) {
+	this.$ref = `#/components/schemas/${ref}`;
+}
+
+function ArrayRef(ref: string) {
+	this.type = 'array';
+	this.items = new Ref(ref);
+}
+
+function Parameter(name, i, type) {
+	this.name = name;
+	this.in = i;
+	this.schema = type;
+}

package/resources/registrationDeprecated.ts

@@ -0,0 +1,8 @@
+import { packageJson } from '../utility/packageUtils.js';
+
+export function getRegistrationInfo() {
+	return {
+		version: packageJson.version,
+		deprecated: true,
+	};
+}

package/resources/replayLogs.ts

@@ -0,0 +1,136 @@
+import { RocksDatabase, Transaction as RocksTransaction } from '@harperfast/rocksdb-js';
+import { Resource } from './Resource.ts';
+import type { Context } from './ResourceInterface.ts';
+import * as logger from '../utility/logging/harper_logger.js';
+import { DatabaseTransaction } from './DatabaseTransaction.ts';
+import { RocksTransactionLogStore } from './RocksTransactionLogStore.ts';
+import { isMainThread } from 'node:worker_threads';
+import { RequestTarget } from './RequestTarget.ts';
+
+let warnedReplayHappening = false;
+export function replayLogs(rootStore: RocksDatabase, tables: any): Promise<void> {
+	if (!isMainThread) return; // ideally we don't do it like this, but for now this is predictable
+	return new Promise((resolve) => {
+		const acquired = rootStore.tryLock('replayLogs', async () => {
+			resolve();
+		});
+		if (!acquired) return;
+		const tableById = new Map<number, typeof Resource>();
+		for (const tableName in tables) {
+			const table = tables[tableName];
+			tableById.set(table.tableId, table);
+		}
+		// replay all the logs
+		let transaction: DatabaseTransaction;
+		let lastTimestamp = 0;
+		let writes = 0;
+		const txnLog: RocksTransactionLogStore = rootStore.auditStore;
+		for (const auditRecord of txnLog.getRange({ startFromLastFlushed: true, readUncommitted: true })) {
+			const { type, tableId, nodeId, recordId, version, residencyId, expiresAt, originatingOperation, username } =
+				auditRecord;
+			try {
+				const Table = tableById.get(tableId);
+				if (!Table) continue;
+				const context: Context = { nodeId, alreadyLogged: true, version, expiresAt, user: { name: username } };
+				const { primaryStore } = Table;
+				const target = new RequestTarget();
+				target.id = null;
+				const tableInstance = Table.getResource(target, context, {});
+				// TODO: If this throws an error due to being unable to access structures, we need to iterate through
+				// other transaction logs to get the latest structure. Ultimately we may have to skip records
+				if (!warnedReplayHappening) {
+					warnedReplayHappening = true;
+					console.warn('Harper was not properly shutdown, replaying transaction logs to synchronize database');
+				}
+				const record = auditRecord.getValue(primaryStore);
+				if (lastTimestamp !== version) {
+					lastTimestamp = version;
+					try {
+						// commit the last transaction since we are starting a new one
+						transaction?.directCommitSync();
+					} catch (error) {
+						logger.error('Error committing replay transaction', error);
+					}
+					transaction = new DatabaseTransaction();
+					transaction.db = primaryStore;
+					transaction.timestamp = version;
+					// we treat this as a retry, because it is (and we want to skip validation and writing to the transaction log)
+					transaction.retries = 1;
+				}
+				context.transaction = transaction;
+				const options = { context, residencyId, nodeId, originatingOperation };
+				writes++;
+				switch (type) {
+					case 'put':
+						tableInstance._writeUpdate(recordId, record, true, options);
+						tableInstance.save(); // requires an explicit save
+						break;
+					case 'patch':
+						tableInstance._writeUpdate(recordId, record, false, options);
+						tableInstance.save(); // requires an explicit save
+						break;
+					case 'message':
+						tableInstance._writePublish(recordId, record, options);
+						break;
+					case 'relocate':
+						tableInstance._writeRelocate(recordId, options);
+						break;
+					case 'delete':
+						tableInstance._writeDelete(recordId, options);
+						break;
+					case 'invalidate':
+						tableInstance._writeInvalidate(recordId, record, options);
+						break;
+					case 'structures': {
+						const rocksTransaction = new RocksTransaction(primaryStore.store);
+						const structuresAsBinary = auditRecord.getBinaryValue(primaryStore);
+						const updatedStructures = structuresAsBinary ? primaryStore.decoder.decode(structuresAsBinary) : undefined;
+						const existingStructures = primaryStore.getSync(Symbol.for('structures'), {
+							transaction: rocksTransaction,
+						});
+						if (existingStructures) {
+							if (existingStructures instanceof Array) {
+								if (updatedStructures.length < existingStructures.length) {
+									logger.warn(
+										`Found ${existingStructures.length} structures in audit store, but ${updatedStructures.length} in replay log. Using ${updatedStructures.length} structures.`
+									);
+								}
+							} else {
+								if (existingStructures.get('named').length > updatedStructures.get('named').length) {
+									logger.warn(
+										`Found named ${existingStructures.length} structures in audit store, but ${updatedStructures.length} in replay log. Using named ${updatedStructures.length} structures.`
+									);
+								}
+								if (existingStructures.get('typed').length > updatedStructures.get('typed').length) {
+									logger.warn(
+										`Found named ${existingStructures.length} structures in audit store, but ${updatedStructures.length} in replay log. Using named ${updatedStructures.length} structures.`
+									);
+								}
+							}
+						}
+						primaryStore.putSync(Symbol.for('structures'), asBinary(structuresAsBinary), {
+							transaction: rocksTransaction,
+						});
+						rocksTransaction.commitSync();
+						primaryStore.decoder.structure = updatedStructures;
+					}
+				}
+			} catch (err) {
+				logger.error(`Error writing from replay of log`, err, {
+					version,
+				});
+			}
+		}
+		try {
+			transaction?.directCommitSync();
+		} catch (error) {
+			logger.error('Error committing replay transaction', error);
+		}
+		if (writes > 0) logger.warn(`Replayed ${writes} records in ${rootStore.databaseName} database`);
+		// we never actually release the lock because we only want to ever run one time
+		// rootStore.unlock('replayLogs');
+	});
+}
+function asBinary(buffer) {
+	return { ['\x10binary-data\x02']: buffer };
+}