@harperfast/harper 5.0.0-alpha.10 → 5.0.0-beta.1

package/server/REST.ts

@@ -0,0 +1,407 @@
+import { serialize, serializeMessage, getDeserializer } from '../server/serverHelpers/contentTypes.ts';
+import { addAnalyticsListener, recordAction, recordActionBinary } from '../resources/analytics/write.ts';
+import * as harperLogger from '../utility/logging/harper_logger.js';
+import { ServerOptions } from 'http';
+import { ServerError, ClientError } from '../utility/errors/hdbError.js';
+import { Resources } from '../resources/Resources.ts';
+import { Resource } from '../resources/Resource.ts';
+import { IterableEventQueue } from '../resources/IterableEventQueue.ts';
+import { transaction } from '../resources/transaction.ts';
+import { Headers, mergeHeaders } from '../server/serverHelpers/Headers.ts';
+import { generateJsonApi } from '../resources/openApi.ts';
+import type { Context } from '../resources/ResourceInterface.ts';
+import { Request } from '../server/serverHelpers/Request.ts';
+import { RequestTarget } from '../resources/RequestTarget';
+
+const { errorToString } = harperLogger;
+const etagBytes = new Uint8Array(8);
+const etagFloat = new Float64Array(etagBytes.buffer, 0, 1);
+let httpOptions = {};
+
+const OPENAPI_DOMAIN = 'openapi';
+
+async function http(request: Context & Request, nextHandler) {
+	const headersObject = request.headers.asObject;
+	const isSse = headersObject.accept === 'text/event-stream';
+	const method = isSse ? 'CONNECT' : request.method;
+	const headers = new Headers();
+	try {
+		request.responseHeaders = headers;
+		request.response = {
+			status: undefined,
+			headers,
+		};
+		const url = request.url.slice(1);
+
+		let target: RequestTarget;
+		let resource: typeof Resource;
+		if (url !== OPENAPI_DOMAIN) {
+			const entry = resources.getMatch(url, isSse ? 'sse' : 'rest');
+			if (!entry) return nextHandler(request); // no resource handler found
+			request.handlerPath = entry.path;
+			target = new RequestTarget(entry.relativeURL); // TODO: We don't want to have to remove the forward slash and then re-add it
+
+			target.async = true;
+			resource = entry.Resource;
+		}
+		if (resource?.isCaching) {
+			const cacheControl = headersObject['cache-control'];
+			if (cacheControl) {
+				const cacheControlParts = parseHeaderValue(cacheControl);
+				for (const part of cacheControlParts) {
+					switch (part.name) {
+						case 'max-age':
+							request.expiresAt = part.value * 1000 + Date.now();
+							break;
+						case 'only-if-cached':
+							request.onlyIfCached = true;
+							break;
+						case 'no-cache':
+							request.noCache = true;
+							break;
+						case 'no-store':
+							request.noCacheStore = true;
+							break;
+						case 'stale-if-error':
+							request.staleIfError = true;
+							break;
+						case 'must-revalidate':
+							request.mustRevalidate = true;
+							break;
+					}
+				}
+			}
+		}
+		const replicateTo = headersObject['x-replicate-to'];
+		if (replicateTo) {
+			const parsed = parseHeaderValue(replicateTo).map((node: { name: string }) => {
+				// we can use a component argument to indicate that number that should be confirmed
+				// for example, to replicate to three nodes and wait for confirmation from two: X-Replicate-To: 3;confirm=2
+				// or to specify nodes with confirm: X-Replicate-To: node-1, node-2, node-3;confirm=2
+				if (node.next?.name === 'confirm' && node.next.value >= 0) {
+					request.replicatedConfirmation = +node.next.value;
+				}
+				return node.name;
+			});
+			request.replicateTo =
+				parsed.length === 1 && +parsed[0] >= 0 ? +parsed[0] : parsed[0] === '*' ? undefined : parsed;
+		}
+		const replicateFrom = headersObject['x-replicate-from'];
+		if (replicateFrom === 'none') {
+			request.replicateFrom = false;
+		}
+		let responseData = await transaction(request, () => {
+			if (headersObject['content-length'] || headersObject['transfer-encoding']) {
+				// TODO: Support cancellation (if the request otherwise fails or takes too many bytes)
+				try {
+					request.data = getDeserializer(headersObject['content-type'], true)(request.body, request.headers);
+				} catch (error) {
+					throw new ClientError(error, 400);
+				}
+			}
+			request.authorize = true;
+
+			if (url === OPENAPI_DOMAIN && method === 'GET') {
+				target = {};
+				if (request?.user?.role?.permission?.super_user) {
+					return generateJsonApi(resources, `${request.protocol}://${request.hostname}`);
+				} else {
+					throw new ServerError(`Forbidden`, 403);
+				}
+			}
+			target.checkPermission = request.user?.role?.permission ?? {};
+
+			switch (method) {
+				case 'GET':
+				case 'HEAD':
+					return resource.get(target, request);
+				case 'POST':
+					return resource.post(target, request.data, request);
+				case 'PUT':
+					return resource.put(target, request.data, request);
+				case 'DELETE':
+					return resource.delete(target, request);
+				case 'PATCH':
+					return resource.patch(target, request.data, request);
+				case 'OPTIONS': // used primarily for CORS
+					headers.setIfNone('Allow', 'GET, HEAD, POST, PUT, DELETE, PATCH, OPTIONS, TRACE, QUERY, COPY, MOVE');
+					return;
+				case 'CONNECT':
+					// websockets? and event-stream
+					return resource.connect(target, null, request);
+				case 'TRACE':
+					return 'Harper is the terminating server';
+				case 'QUERY':
+					return resource.query(target, request.data, request);
+				case 'COPY': // methods suggested from webdav RFC 4918
+					return resource.copy(target, headersObject.destination, request);
+				case 'MOVE':
+					return resource.move(target, headersObject.destination, request);
+				case 'BREW': // RFC 2324
+					throw new ClientError("Harper is short and stout and can't brew coffee", 418);
+				default:
+					throw new ServerError(`Method ${method} is not recognized`, 501);
+			}
+		});
+		let status = request.response.status;
+		let lastModification = request.lastModified;
+		if (responseData == undefined) {
+			status ??= method === 'GET' || method === 'HEAD' ? 404 : 204;
+			// deleted entries can have a timestamp of when they were deleted
+			if (httpOptions.lastModified && isFinite(lastModification))
+				headers.setIfNone('Last-Modified', new Date(lastModification).toUTCString());
+		} else if (responseData.headers) {
+			// if response is a Response object, use it as the response
+			if (Object.isFrozen(responseData)) {
+				// make a copy if it is a frozen record
+				responseData = Object.assign({}, responseData);
+			}
+			// merge headers from response
+			const responseHeaders = mergeHeaders(responseData.headers, headers);
+			if (responseData.headers !== responseHeaders)
+				// if we rebuilt the headers, reassign it, but we don't want to assign to a Response object (which should already
+				// have a valid Headers object) or it will throw an error
+				responseData.headers = responseHeaders;
+			// if no body, look for provided data to serialize
+			if (!responseData.body) {
+				if ('data' in responseData) responseData.body = serialize(responseData.data, request, responseData);
+				else responseData.body = serialize(responseData, request, responseData);
+			}
+			responseData.status ??= status ?? 200;
+			return responseData;
+		} else if (isFinite(lastModification)) {
+			etagFloat[0] = lastModification;
+			// base64 encoding of the 64-bit float encoding of the date in ms (with quotes)
+			// very fast and efficient
+			const etag = String.fromCharCode(
+				34,
+				(etagBytes[0] & 0x3f) + 62,
+				(etagBytes[0] >> 6) + ((etagBytes[1] << 2) & 0x3f) + 62,
+				(etagBytes[1] >> 4) + ((etagBytes[2] << 4) & 0x3f) + 62,
+				(etagBytes[2] >> 2) + 62,
+				(etagBytes[3] & 0x3f) + 62,
+				(etagBytes[3] >> 6) + ((etagBytes[4] << 2) & 0x3f) + 62,
+				(etagBytes[4] >> 4) + ((etagBytes[5] << 4) & 0x3f) + 62,
+				(etagBytes[5] >> 2) + 62,
+				(etagBytes[6] & 0x3f) + 62,
+				(etagBytes[6] >> 6) + ((etagBytes[7] << 2) & 0x3f) + 62,
+				34
+			);
+			const lastEtag = headersObject['if-none-match'];
+			if (lastEtag && etag == lastEtag) {
+				if (responseData?.onDone) responseData.onDone();
+				status = 304;
+				responseData = undefined;
+			} else {
+				headers.setIfNone('ETag', etag);
+			}
+			if (httpOptions.lastModified) headers.setIfNone('Last-Modified', new Date(lastModification).toUTCString());
+		}
+		if (request.createdResource) status = 201;
+		if (request.newLocation) headers.setIfNone('Location', request.newLocation);
+
+		const responseObject = {
+			status: status ?? 200,
+			headers,
+			body: undefined,
+		};
+		const loadedFromSource = target.loadedFromSource;
+		if (loadedFromSource !== undefined) {
+			// this appears to be a caching table with a source
+			responseObject.wasCacheMiss = loadedFromSource; // indicate if it was a missed cache
+			if (!loadedFromSource && isFinite(lastModification)) {
+				headers.setIfNone('Age', Math.round((Date.now() - (request.lastRefreshed || lastModification)) / 1000));
+			}
+		}
+		// TODO: Handle 201 Created
+		if (responseData !== undefined) {
+			responseObject.body = serialize(responseData, request, responseObject);
+			if (method === 'HEAD') responseObject.body = undefined; // we want everything else to be the same as GET, but then omit the body
+		}
+		return responseObject;
+	} catch (error) {
+		error ??= new Error('Unknown error occurred');
+		let statusCode = error.statusCode ?? request.response.status;
+		if (statusCode) {
+			if (statusCode === 500) harperLogger.warn(error);
+			else harperLogger.info(error);
+			if (statusCode === 405) {
+				if (error.method) error.message += ` to handle HTTP method ${error.method.toUpperCase() || ''}`;
+				if (error.allow) {
+					error.allow.push('trace', 'head', 'options');
+					headers.setIfNone('Allow', error.allow.map((method) => method.toUpperCase()).join(', '));
+				}
+			}
+		} else harperLogger.error(error);
+
+		// RFC 9457 Problem Details
+		const status = statusCode || 500;
+		// we prefer to use error classes for error codes (constructor.name), but if there is a code, it is probably a node.js
+		// error that denotes error codes with a separate property
+		const code = error.code ?? error.constructor.name;
+		const problemDetail = {
+			type: `error:${code}`, // eventually we want this to be a resolvable URI to our docs
+			code,
+			title: error.message ?? error.toString(),
+			status,
+			detail: error.detail,
+			instance: request.url,
+		};
+
+		const responseObject = {
+			status,
+			headers,
+			body: undefined,
+		};
+		responseObject.body = serialize(problemDetail, request, responseObject);
+		return responseObject;
+	}
+}
+
+let started;
+let resources: Resources;
+let addedMetrics;
+let connectionCount = 0;
+
+export function start(options: ServerOptions & { path: string; port: number; server: any; resources: Resources }) {
+	httpOptions = options;
+	if (options.includeExpensiveRecordCountEstimates) {
+		// If they really want to enable expensive record count estimates
+		Request.prototype.includeExpensiveRecordCountEstimates = true;
+	}
+	if (started) return;
+	started = true;
+	resources = options.resources;
+	options.server.http(async (request: Request, nextHandler) => {
+		if (request.isWebSocket) return;
+		return http(request, nextHandler);
+	}, options);
+	if (options.webSocket === false) return;
+	options.server.ws(async (ws, request, chainCompletion) => {
+		connectionCount++;
+		const incomingMessages = new IterableEventQueue();
+		if (!addedMetrics) {
+			addedMetrics = true;
+			addAnalyticsListener((metrics) => {
+				if (connectionCount > 0)
+					metrics.push({
+						metric: 'ws-connections',
+						connections: connectionCount,
+						byThread: true,
+					});
+			});
+		}
+		// TODO: We should set a lower keep-alive ws.socket.setKeepAlive(600000);
+		let hasError;
+		ws.on('error', (error) => {
+			hasError = true;
+			harperLogger.warn(error);
+		});
+		let deserializer;
+		ws.on('message', function message(body) {
+			if (!deserializer)
+				deserializer = getDeserializer(request.requestedContentType ?? request.headers.asObject['content-type'], false);
+			const data = deserializer(body);
+			recordAction(body.length, 'bytes-received', request.handlerPath, 'message', 'ws');
+			incomingMessages.push(data);
+		});
+		let iterator;
+		ws.on('close', () => {
+			connectionCount--;
+			recordActionBinary(!hasError, 'connection', 'ws', 'disconnect');
+			incomingMessages.emit('close');
+			if (iterator) iterator.return();
+		});
+		try {
+			await chainCompletion;
+			const url = request.url.slice(1);
+			const entry = resources.getMatch(url, 'ws');
+			recordActionBinary(Boolean(entry), 'connection', 'ws', 'connect');
+			if (!entry) {
+				// TODO: Ideally we would like to have a 404 response before upgrading to WebSocket protocol, probably
+				return ws.close(1011, `No resource was found to handle ${request.pathname}`);
+			} else {
+				request.handlerPath = entry.path;
+				recordAction(
+					(action) => ({
+						count: action.count,
+						total: connectionCount,
+					}),
+					'connections',
+					request.handlerPath,
+					'connect',
+					'ws'
+				);
+				request.authorize = true;
+				const resourceRequest = new RequestTarget(entry.relativeURL); // TODO: We don't want to have to remove the forward slash and then re-add it
+				resourceRequest.checkPermission = request.user?.role?.permission ?? {};
+				const resource = entry.Resource;
+				const responseStream = await transaction(request, () => {
+					return resource.connect(resourceRequest, incomingMessages, request);
+				});
+				iterator = responseStream[Symbol.asyncIterator]();
+
+				let result;
+				while (!(result = await iterator.next()).done) {
+					const messageBinary = await serializeMessage(result.value, request);
+					ws.send(messageBinary);
+					recordAction(messageBinary.length, 'bytes-sent', request.handlerPath, 'message', 'ws');
+					if (ws._socket.writableNeedDrain) {
+						await new Promise((resolve) => ws._socket.once('drain', resolve));
+					}
+				}
+			}
+		} catch (error) {
+			if (error.statusCode) {
+				if (error.statusCode === 500) harperLogger.warn(error);
+				else harperLogger.info(error);
+			} else harperLogger.error(error);
+			ws.close(
+				HTTP_TO_WEBSOCKET_CLOSE_CODES[error.statusCode] || // try to return a helpful code
+					1011, // otherwise generic internal error
+				errorToString(error)
+			);
+		}
+		ws.close();
+	}, options);
+}
+const HTTP_TO_WEBSOCKET_CLOSE_CODES = {
+	401: 3000,
+	403: 3003,
+};
+
+/**
+ * This parser is used to parse header values.
+ *
+ * It is used within this file for parsing the `Cache-Control` and `X-Replicate-To` headers.
+ *
+ * @param value
+ */
+export function parseHeaderValue(value: string) {
+	return value
+		.trim()
+		.split(',')
+		.map((part) => {
+			let parsed;
+			const components = part.trim().split(';');
+			let component;
+			while ((component = components.pop())) {
+				if (component.includes('=')) {
+					let [name, value] = component.trim().split('=');
+					name = name.trim();
+					if (value) value = value.trim();
+					parsed = {
+						name: name.toLowerCase(),
+						value,
+						next: parsed,
+					};
+				} else {
+					parsed = {
+						name: component.toLowerCase(),
+						next: parsed,
+					};
+				}
+			}
+			return parsed;
+		});
+}

package/server/Server.ts

@@ -0,0 +1,89 @@
+import { Socket } from 'net';
+import { _assignPackageExport } from '../globals.js';
+import type { Value } from '../resources/analytics/write.ts';
+import type { Resources } from '../resources/Resources.ts';
+import { OperationDefinition } from './serverHelpers/serverUtilities.ts';
+
+/**
+ * This is the central interface by which we define entry points for different server protocol plugins to listen for
+ * incoming connections and requests.
+ */
+export interface Server {
+	socket?(listener: (socket: Socket) => void, options: ServerOptions): void;
+	http?(listener: (request: Request, nextLayer: (request: Request) => Response) => void, options?: HttpOptions): void;
+	request?(
+		listener: (request: Request, nextLayer: (request: Request) => Response) => void,
+		options?: HttpOptions
+	): void;
+	ws?(
+		listener: (ws: WebSocket, request: Request, requestCompletion: Promise<any>) => any,
+		options?: WebSocketOptions
+	): void;
+	contentTypes: Map<string, ContentTypeHandler>;
+	getUser(username: string, password: string | null, request: Request): any;
+	authenticateUser(username: string, password: string, request: Request): any;
+	operation(operation: any, context: any, authorize?: boolean): Promise<any>;
+	registerOperation(operationDefinition: OperationDefinition): void;
+	recordAnalytics(value: Value, metric: string, path?: string, method?: string, type?: string): void;
+	nodes: Node[];
+	shards: Map<number, string[]>;
+	hostname: string;
+	resources: Resources;
+	replication: {
+		getThisNodeId(auditStore: any): number;
+		exportIdMapping(auditStore: any): any;
+		getIdOfRemoteNode(remoteNodeName: string, auditStore: any): number;
+		replicateOperation(operation: {
+			replicated: boolean;
+			[key: string]: any;
+		}): Promise<{ message: string; replicated?: unknown[] }>;
+		monitorNodeCAs(listener: () => void): void;
+		sendOperationToNode(node: string, operation: any, options: any): Promise<any>;
+	};
+}
+interface Node {
+	name: string;
+	shard: number;
+	url: string;
+}
+export interface ServerOptions {
+	port?: number;
+	securePort?: number;
+	mtls?: boolean;
+	usageType?: string;
+}
+interface WebSocketOptions extends ServerOptions {
+	subProtocol: string;
+}
+export interface HttpOptions extends ServerOptions {
+	runFirst?: boolean;
+}
+export interface ContentTypeHandler {
+	serialize(data: any): Buffer | string;
+	serializeStream(data: any): Buffer | string;
+	deserialize(data: any): Buffer | string;
+	q: number;
+}
+
+export const server: Server = {
+	replication: {
+		getThisNodeId() {
+			return 0;
+		},
+		exportIdMapping() {
+			return undefined;
+		},
+		replicateOperation(operation) {
+			return operation.replicated
+				? Promise.reject(new Error('Replication not implemented.'))
+				: Promise.resolve({ message: '' });
+		},
+		monitorNodeCAs(_listener: () => void) {
+			throw new Error('Replication not implemented.');
+		},
+		sendOperationToNode() {
+			return Promise.reject(new Error('Replication not implemented.'));
+		},
+	},
+};
+_assignPackageExport('server', server);

package/server/fastifyRoutes/helpers/getCORSOptions.js

@@ -0,0 +1,36 @@
+'use strict';
+
+const env = require('../../../utility/environment/environmentManager.js');
+env.initSync();
+const { CONFIG_PARAMS } = require('../../../utility/hdbTerms.ts');
+
+/**
+ * Builds CORS options object to pass to cors plugin when/if it needs to be registered with Fastify
+ *
+ * @returns {{credentials: boolean, origin: boolean, allowedHeaders: [string, string]}}
+ */
+function getCORSOptions() {
+	let propsCorsAccesslist = env.get(CONFIG_PARAMS.HTTP_CORSACCESSLIST);
+	let propsCors = env.get(CONFIG_PARAMS.HTTP_CORS);
+	let corsOptions;
+	if (propsCors) {
+		corsOptions = {
+			origin: true,
+			allowedHeaders: ['Content-Type', 'Authorization', 'Accept'],
+			credentials: false,
+		};
+		if (
+			propsCorsAccesslist &&
+			propsCorsAccesslist.length > 0 &&
+			propsCorsAccesslist[0] !== null &&
+			propsCorsAccesslist[0] !== '*'
+		) {
+			corsOptions.origin = (origin, callback) => {
+				return callback(null, propsCorsAccesslist.indexOf(origin) !== -1);
+			};
+		}
+	}
+	return corsOptions;
+}
+
+module.exports = getCORSOptions;

package/server/fastifyRoutes/helpers/getHeaderTimeoutConfig.js

@@ -0,0 +1,15 @@
+'use strict';
+
+const env = require('../../../utility/environment/environmentManager.js');
+env.initSync();
+const terms = require('../../../utility/hdbTerms.ts');
+
+/**
+ * Returns header timeout value from config file
+ * @returns {*}
+ */
+function getHeaderTimeoutConfig() {
+	return env.get(terms.CONFIG_PARAMS.HTTP_HEADERSTIMEOUT) ?? 60000;
+}
+
+module.exports = getHeaderTimeoutConfig;

package/server/fastifyRoutes/helpers/getServerOptions.js

@@ -0,0 +1,33 @@
+'use strict';
+
+const env = require('../../../utility/environment/environmentManager.js');
+env.initSync();
+const { CONFIG_PARAMS } = require('../../../utility/hdbTerms.ts');
+
+// eslint-disable-next-line no-magic-numbers
+const REQ_MAX_BODY_SIZE = 1024 * 1024 * 1024; //this is 1GB in bytes
+
+/**
+ * Builds server options object to pass to Fastify when using server factory.
+ * @param isHttps
+ * @returns {{keepAliveTimeout: (*), bodyLimit: number, ignoreTrailingSlash: boolean, connectionTimeout: (*)}}
+ */
+function getServerOptions(isHttps) {
+	const server_timeout = env.get(CONFIG_PARAMS.HTTP_TIMEOUT);
+	const keep_alive_timeout = env.get(CONFIG_PARAMS.HTTP_KEEPALIVETIMEOUT);
+	return {
+		bodyLimit: REQ_MAX_BODY_SIZE,
+		connectionTimeout: server_timeout,
+		keepAliveTimeout: keep_alive_timeout,
+		return503OnClosing: false,
+		forceCloseConnections: true,
+		ignoreTrailingSlash: true,
+		maxParamLength: env.get(CONFIG_PARAMS.HTTP_MAXPARAMLENGTH) ?? 1000,
+		// http2: isHttps, // for now we are not enabling HTTP/2 since it seems to show slower performance
+		https: isHttps /* && {
+			allowHTTP1: true,
+		},*/,
+	};
+}
+
+module.exports = getServerOptions;

package/server/fastifyRoutes/plugins/hdbCore.js

@@ -0,0 +1,39 @@
+'use strict';
+
+const fp = require('fastify-plugin');
+
+const {
+	handlePostRequest,
+	authHandler,
+	reqBodyValidationHandler,
+} = require('../../../server/serverHelpers/serverHandlers.js');
+
+/**
+ * Generates a fastify plugin containing three core methods
+ *
+ * @param server - a fastify server instance.
+ */
+async function hdbCore(server) {
+	server.decorate('hdbCore', {
+		preValidation: [reqBodyValidationHandler, authHandler],
+		request: (request, reply) => convertAsyncIterators(handlePostRequest(request, reply)),
+		requestWithoutAuthentication: (request, response) =>
+			convertAsyncIterators(handlePostRequest(request, response, true)),
+	});
+}
+// We convert responses that can only be asynchronously iterated to (promises of) arrays for
+// backwards compatibility, we do not assume custom functions can handle these.
+async function convertAsyncIterators(response) {
+	response = await response;
+	if (response?.[Symbol.asyncIterator] && !response[Symbol.iterator]) {
+		// requires async iteration to access elements
+		let array = [];
+		for await (let element of response) {
+			array.push(element);
+		}
+		return array;
+	}
+	return response;
+}
+
+module.exports = fp(hdbCore);