@harperfast/harper 5.0.0-alpha.10 → 5.0.0-beta.1

package/resources/ResourceInterface.ts

@@ -0,0 +1,221 @@
+import type { User } from '../security/user.ts';
+import type { OperationFunctionName } from '../server/serverHelpers/serverUtilities.ts';
+import { DatabaseTransaction } from './DatabaseTransaction.ts';
+import { IterableEventQueue } from './IterableEventQueue.ts';
+import type { Entry, RecordObject } from './RecordEncoder.ts';
+import { RequestTarget } from './RequestTarget.ts';
+
+export interface ResourceInterface<Record extends object = any>
+	extends Partial<RecordObject>, Pick<UpdatableRecord<Record>, 'addTo' | 'subtractFrom'> {
+	allowRead(user: User, target: RequestTarget, context: Context): boolean | Promise<boolean>;
+	get?(
+		target?: RequestTargetOrId
+	):
+		| (Record & Partial<RecordObject>)
+		| Promise<Record & Partial<RecordObject>>
+		| AsyncIterable<Record & Partial<RecordObject>>
+		| Promise<AsyncIterable<Record & Partial<RecordObject>>>;
+	search?(target: RequestTarget): AsyncIterable<Record & Partial<RecordObject>>;
+
+	allowCreate(user: User, record: Promise<Record & RecordObject>, context: Context): boolean | Promise<boolean>;
+	create?(
+		newRecord: Partial<Record & RecordObject>,
+		target: RequestTargetOrId
+	): void | (Record & Partial<RecordObject>) | Promise<Record & Partial<RecordObject>>;
+	post?(
+		target: RequestTargetOrId,
+		newRecord: Partial<Record & RecordObject>
+	): void | (Record & Partial<RecordObject>) | Promise<Record & Partial<RecordObject>>;
+
+	allowUpdate(user: User, record: Promise<Record & RecordObject>, context: Context): boolean | Promise<boolean>;
+	put?(
+		record: Record & RecordObject,
+		target: RequestTargetOrId
+	): void | (Record & Partial<RecordObject>) | Promise<void | (Record & Partial<RecordObject>)>;
+	patch?(
+		record: Partial<Record & RecordObject>,
+		target: RequestTargetOrId
+	): void | (Record & Partial<RecordObject>) | Promise<void | (Record & Partial<RecordObject>)>;
+	update?(updates: Record & RecordObject, fullUpdate: true): ResourceInterface<Record & Partial<RecordObject>>;
+	update?(
+		updates: Partial<Record & RecordObject>,
+		fullUpdate?: boolean
+	):
+		| ResourceInterface<Record & Partial<RecordObject>>
+		| Promise<ResourceInterface<Record & Partial<RecordObject>> | UpdatableRecord<Record & Partial<RecordObject>>>;
+
+	allowDelete(user: User, target: RequestTarget, context: Context): boolean | Promise<boolean>;
+	delete?(target: RequestTargetOrId): boolean | Promise<boolean>;
+
+	invalidate(target: RequestTargetOrId): void | Promise<void>;
+
+	publish?(target: RequestTargetOrId, record: Record, options?: any): void;
+	subscribe?(request: SubscriptionRequest): AsyncIterable<Record> | Promise<AsyncIterable<Record>>;
+
+	doesExist(): boolean;
+	wasLoadedFromSource(): boolean | void;
+
+	getCurrentUser(): User | undefined;
+}
+
+export interface Session {
+	id?: any;
+	user?: User;
+	update: (updatedSession: any) => unknown;
+	delete: (id: any) => Promise<void>;
+}
+
+export interface Context {
+	/**	 The user making the request */
+	user?: User;
+	/** Check the username and password against the core user table to verify user identity */
+	login: (username: string, password: string) => Promise<string>;
+	/** Describes the current cookie-based session if it is present and grants the capacity to delete it. authentication.enableSessions must be turned on in the harperdb-config.yaml  */
+	session?: Session;
+	/**	 The database transaction object */
+	transaction?: DatabaseTransaction;
+	/**	 If the operation that will be performed with this context should check user authorization	 */
+	authorize?: boolean;
+	/**	 The last modification time of any data that has been accessed with this context	 */
+	lastModified?: number;
+	/**	 The time	at which a saved record should expire */
+	expiresAt?: number;
+	/**	 Indicates that caching should not be applied */
+	noCache?: boolean;
+	/**	 Indicates that values from the source data should be stored as a cached value */
+	noCacheStore?: boolean;
+	/**	 Only return values from the table, and don't use data from the source */
+	onlyIfCached?: boolean;
+	/**	 Allows data from a caching table to be used if there is an error retrieving data from the source */
+	staleIfError?: boolean;
+	/**	 Indicates any cached data must be revalidated */
+	mustRevalidate?: boolean;
+	/**	 An array of nodes to replicate to */
+	replicateTo?: string[];
+	replicateFrom?: boolean;
+	replicatedConfirmation?: number;
+	originatingOperation?: OperationFunctionName;
+	previousResidency?: string[];
+	loadedFromSource?: boolean;
+	nodeName?: string;
+	resourceCache?: Map<Id, any>;
+	_freezeRecords?: boolean; // until v5, we conditionally freeze records for back-compat
+	timestamp?: number;
+}
+
+export interface SourceContext<TRequestContext = Context, Record extends object = any> {
+	/** The original request context passed from the caching layer */
+	requestContext: TRequestContext;
+	/** The existing record, from the existing entry (if any) */
+	replacingRecord?: Record;
+	/** The existing database entry (if any) */
+	replacingEntry?: Entry;
+	/** The version/timestamp of the existing record */
+	replacingVersion?: number;
+	/** Indicates that values from the source data should NOT be stored as a cached value */
+	noCacheStore?: boolean;
+	/** Reference to the source Resource instance */
+	source?: ResourceInterface<Record>;
+	/** Shared resource cache from parent context for visibility of modifications */
+	resourceCache?: Map<Id, any>;
+	/** Database transaction for the context */
+	transaction?: DatabaseTransaction;
+	/** The time at which the cached entry should expire (ms since epoch) */
+	expiresAt?: number;
+	/** The last modification time of any data accessed with this context */
+	lastModified?: number;
+}
+
+export type Operator = 'and' | 'or';
+
+export type Comparator =
+	| 'between'
+	| 'contains'
+	| 'ends_with'
+	| 'eq'
+	| 'equals'
+	| 'greater_than'
+	| 'greater_than_equal'
+	| 'less_than'
+	| 'less_than_equal'
+	| 'ne'
+	| 'not_equal'
+	| 'starts_with';
+
+export type DirectCondition<Record extends object = any> = TypedDirectCondition<Record, keyof Record>;
+
+interface TypedDirectCondition<Record extends object, Property extends keyof Record> {
+	attribute?: keyof Record | Array<keyof Record> | string | string[];
+	search_attribute?: keyof Record | Array<keyof Record> | string | string[];
+	comparator?: Comparator;
+	search_type?: Comparator;
+	value?: Record[Property];
+	search_value?: Record[Property];
+}
+
+interface ConditionGroup<Record extends object = any> {
+	conditions?: Conditions<Record>;
+	operator?: Operator;
+}
+export type Condition<Record extends object = any> = DirectCondition<Record> & ConditionGroup<Record>;
+export type Conditions<Record extends object = any> = Condition<Record>[];
+
+export interface Sort<Record extends object = any> {
+	attribute: keyof Record;
+	descending?: boolean;
+	next?: Sort<Record>;
+}
+export interface SubSelect {
+	name: string;
+	select: (string | SubSelect)[];
+}
+export type Select = (string | SubSelect)[];
+
+export interface SubscriptionRequest extends RequestTarget {
+	/** The starting time of events to return (defaults to now) */
+	startTime?: number;
+	/** The count of previously recorded events to return */
+	previousCount?: number;
+	/** If the current record state should be omitted as the first event */
+	omitCurrent?: boolean;
+	onlyChildren?: boolean;
+	includeDescendants?: boolean;
+	supportsTransactions?: boolean;
+	rawEvents?: boolean;
+	listener?: Listener;
+}
+
+export type Query = RequestTarget; // for back-compat
+export type RequestTargetOrId = RequestTarget | Id;
+
+export type Id = number | string | (number | string | null)[] | null;
+
+export type UpdatableRecord<Record extends object = any> = TypedUpdatableRecord<Record, keyof Record>;
+interface TypedUpdatableRecord<Record extends object, Property extends keyof Record> extends RecordObject {
+	set(property: Property, value: Record[Property]): void;
+	getProperty(property: Property): Record[Property];
+	addTo(property: Property, value: Record[Property]): void;
+	subtractFrom(property: Property, value: Record[Property]): void;
+}
+
+export interface Subscription<Event extends object = any> extends IterableEventQueue<Event> {
+	new (listener: Listener<Event>);
+
+	listener: Listener<Event>;
+	subscriptions: Listener<Event>[];
+	startTime?: number;
+
+	end(): void;
+	toJSON(): { name: 'subscription' };
+}
+
+type Listener<Payload extends object = any> = (payload: ListenerPayload<Payload>) => void;
+
+interface ListenerPayload<Payload extends object = any> {
+	id: Id;
+	localTime: number;
+	value: Payload;
+	version: number;
+	type: string;
+	beginTxn: boolean;
+}

package/resources/ResourceInterfaceV2.ts

@@ -0,0 +1,53 @@
+import type { User } from '../security/user.js';
+import type { RecordObject } from './RecordEncoder.ts';
+import { RequestTarget } from './RequestTarget.ts';
+import { RequestTargetOrId, ResourceInterface, SubscriptionRequest, UpdatableRecord } from './ResourceInterface.ts';
+
+// @ts-expect-error We changed the interface in v2 with breaking changes (by flipping target and newRecord/record)
+export interface ResourceInterfaceV2<Record extends object = any> extends ResourceInterface<Record> {
+	get?(
+		target?: RequestTargetOrId
+	):
+		| (Record & Partial<RecordObject>)
+		| Promise<Record & Partial<RecordObject>>
+		| AsyncIterable<Record & Partial<RecordObject>>
+		| Promise<AsyncIterable<Record & Partial<RecordObject>>>;
+	search?(target: RequestTarget): AsyncIterable<Record & Partial<RecordObject>>;
+
+	create?(
+		target: RequestTargetOrId,
+		newRecord: Partial<Record & RecordObject>
+	): void | (Record & Partial<RecordObject>) | Promise<Record & Partial<RecordObject>>;
+	post?(
+		target: RequestTargetOrId,
+		newRecord: Partial<Record & RecordObject>
+	): void | (Record & Partial<RecordObject>) | Promise<Record & Partial<RecordObject>>;
+
+	put?(
+		target: RequestTargetOrId,
+		record: Record & RecordObject
+	): void | (Record & Partial<RecordObject>) | Promise<void | (Record & Partial<RecordObject>)>;
+	patch?(
+		target: RequestTargetOrId,
+		record: Partial<Record & RecordObject>
+	): void | (Record & Partial<RecordObject>) | Promise<void | (Record & Partial<RecordObject>)>;
+	update?(updates: Record & RecordObject, fullUpdate: true): ResourceInterface<Record & Partial<RecordObject>>;
+	update?(
+		updates: Partial<Record & RecordObject>,
+		fullUpdate?: boolean
+	):
+		| ResourceInterface<Record & Partial<RecordObject>>
+		| Promise<ResourceInterface<Record & Partial<RecordObject>> | UpdatableRecord<Record & Partial<RecordObject>>>;
+
+	delete?(target: RequestTargetOrId): boolean | Promise<boolean>;
+
+	invalidate(target: RequestTargetOrId): void | Promise<void>;
+
+	publish?(target: RequestTargetOrId, record: Record): void;
+	subscribe?(request: SubscriptionRequest): AsyncIterable<Record> | Promise<AsyncIterable<Record>>;
+
+	doesExist(): boolean;
+	wasLoadedFromSource(): boolean | void;
+
+	getCurrentUser(): User | undefined;
+}

package/resources/ResourceV2.ts

@@ -0,0 +1,67 @@
+import { _assignPackageExport } from '../globals.js';
+import { IterableEventQueue } from './IterableEventQueue.js';
+import type { RecordObject } from './RecordEncoder.js';
+import { RequestTarget } from './RequestTarget.js';
+import { Resource } from './Resource.ts';
+import { RequestTargetOrId } from './ResourceInterface.ts';
+
+/**
+ * This is the main class that can be extended for any resource in HarperDB and provides the essential reusable
+ * uniform interface for interacting with data, defining the API for providing data (data sources) and for consuming
+ * data. This interface is used pervasively in HarperDB and is implemented by database tables and can be used to define
+ * sources for caching, real-data sources for messaging protocols, and RESTful endpoints, as well as any other types of
+ * data aggregation, processing, or monitoring.
+ *
+ * This base Resource class provides a set of static methods that are main entry points for querying and updating data
+ * in resources/tables. The static methods provide the default handling of arguments, context, and ensuring that
+ * internal actions are wrapped in a transaction. The base Resource class intended to be extended, and the instance
+ * methods can be overridden to provide specific implementations of actions like get, put, post, delete, and subscribe.
+ */
+export class ResourceV2<Record extends object = any> extends Resource<Record> {
+	static loadAsInstance: boolean = false;
+
+	get?(
+		target?: RequestTargetOrId
+	):
+		| (Record & Partial<RecordObject>)
+		| Promise<Record & Partial<RecordObject>>
+		| AsyncIterable<Record & Partial<RecordObject>>
+		| Promise<AsyncIterable<Record & Partial<RecordObject>>>;
+
+	search?(target: RequestTargetOrId): AsyncIterable<Record & Partial<RecordObject>>;
+
+	// @ts-expect-error We swapped the order of target and newRecord.
+	create?(
+		target: RequestTargetOrId,
+		newRecord: Partial<Record & RecordObject>
+	): void | (Record & Partial<RecordObject>) | Promise<Record & Partial<RecordObject>>;
+
+	// @ts-expect-error In v2, we're adjusting the types.
+	post(
+		target: RequestTargetOrId,
+		newRecord: Partial<Record & RecordObject>
+	): void | (Record & Partial<RecordObject>) | Promise<Record & Partial<RecordObject>> {
+		return super.post(target, newRecord);
+	}
+
+	// @ts-expect-error We swapped the order of target and record.
+	put?(
+		target: RequestTargetOrId,
+		record: Record & RecordObject
+	): void | (Record & Partial<RecordObject>) | Promise<void | (Record & Partial<RecordObject>)>;
+
+	// @ts-expect-error We swapped the order of target and record.
+	patch?(
+		target: RequestTargetOrId,
+		record: Partial<Record & RecordObject>
+	): void | (Record & Partial<RecordObject>) | Promise<void | (Record & Partial<RecordObject>)>;
+
+	delete?(target: RequestTargetOrId): boolean | Promise<boolean>;
+	invalidate?(target: RequestTargetOrId): void | Promise<void>;
+
+	// @ts-expect-error We swapped the order of target and record.
+	connect?(incomingMessages: IterableEventQueue<Record>, target: RequestTarget): AsyncIterable<Record>;
+	publish?(target: RequestTargetOrId, record: Record): void;
+}
+
+_assignPackageExport('ResourceV2', ResourceV2);

package/resources/Resources.ts

@@ -0,0 +1,162 @@
+import { Resource } from './Resource.ts';
+import { transaction } from './transaction.ts';
+import { ErrorResource } from './ErrorResource.ts';
+import logger from '../utility/logging/harper_logger.js';
+import { ServerError } from '../utility/errors/hdbError.js';
+import { server } from '../server/Server.ts';
+
+interface ResourceEntry {
+	Resource: typeof Resource;
+	path: string;
+	exportTypes: any;
+	hasSubPaths: boolean;
+	relativeURL: string;
+}
+
+/**
+ * This is the global set of all resources that have been registered on this server.
+ */
+export class Resources extends Map<string, ResourceEntry> {
+	isWorker = true;
+	loginPath?: (request) => string;
+
+	allTypes: Map<any, any> = new Map();
+
+	set(path, resource, exportTypes?: { [key: string]: boolean }, force?: boolean): void {
+		if (!resource) throw new Error('Must provide a resource');
+		if (path.startsWith('/')) path = path.replace(/^\/+/, '');
+		const entry = {
+			Resource: resource,
+			path,
+			exportTypes,
+			hasSubPaths: false,
+			relativeURL: '', // reset after each match
+		};
+		const existingEntry = super.get(path);
+		if (
+			existingEntry &&
+			(existingEntry.Resource.databaseName !== resource.databaseName ||
+				existingEntry.Resource.tableName !== resource.tableName) &&
+			!force
+		) {
+			// there was a conflict in endpoint paths. We don't want this to be ignored, so we log it
+			// and create an error resource to make sure it is reported in any attempt to access this path.
+			// it was be a 500 error; clearly a server error (not client error), unfortunate that the 5xx errors
+			// don't provide anything more descriptive.
+			const error = new ServerError(`Conflicting paths for ${path}`);
+			logger.error(error);
+			entry.Resource = new ErrorResource(error);
+		}
+		super.set(path, entry);
+		// now mark any entries that have sub paths so we can efficiently route forward
+		for (const [path] of this) {
+			let slashIndex = 2;
+			while ((slashIndex = path.indexOf('/', slashIndex)) > -1) {
+				const parentEntry = this.get(path.slice(0, slashIndex));
+				if (parentEntry) parentEntry.hasSubPaths = true;
+				slashIndex += 2;
+			}
+		}
+	}
+
+	/**
+	 * Find the best (longest) match resource path that matches the (beginning of the) provided path, in order to find
+	 * the correct Resource to handle this URL path.
+	 * @param path The URL Path
+	 * @param exportType Optional request content or protocol type, allows control of which protocols can access a resource
+	 * and future layering of resources (for defining HTML handlers
+	 * that can further transform data from the main structured object resources).
+	 * @return The matched Resource class. Note that the remaining path is "returned" by setting the relativeURL property
+	 */
+	getMatch(url: string, exportType?: string): ResourceEntry | undefined {
+		let slashIndex = 2;
+		let prevSlashIndex = 0;
+		let foundEntry: ResourceEntry;
+
+		const urlLength = url.length;
+
+		while (slashIndex < urlLength) {
+			prevSlashIndex = slashIndex;
+			slashIndex = url.indexOf('/', slashIndex);
+
+			if (slashIndex === -1) {
+				slashIndex = urlLength;
+			}
+
+			const resourcePath = slashIndex === urlLength ? url : url.slice(0, slashIndex);
+			let entry = this.get(resourcePath);
+			let queryIndex = -1;
+			if (!entry && slashIndex === urlLength) {
+				// try to match the first part of the path if there's a query
+				queryIndex = resourcePath.indexOf('?', prevSlashIndex);
+				if (queryIndex !== -1) {
+					const pathPart = resourcePath.slice(0, queryIndex);
+					entry = this.get(pathPart);
+				}
+			}
+			if (entry && (!exportType || entry.exportTypes?.[exportType] !== false)) {
+				entry.relativeURL = url.slice(queryIndex !== -1 ? queryIndex : slashIndex);
+				if (!entry.hasSubPaths) {
+					return entry;
+				}
+				foundEntry = entry;
+			}
+
+			slashIndex += 2;
+		}
+
+		if (foundEntry) return foundEntry;
+
+		// try the exact path
+		const searchIndex = url.indexOf('?');
+		const path = searchIndex > -1 ? url.slice(0, searchIndex) : url;
+		foundEntry = this.get(path);
+		if (!foundEntry && path.indexOf('.') > -1) {
+			foundEntry = this.get(path.split('.')[0]);
+		}
+		if (foundEntry && (!exportType || foundEntry.exportTypes?.[exportType] !== false)) {
+			foundEntry.relativeURL = searchIndex > -1 ? url.slice(searchIndex) : '';
+		} else if (!foundEntry) {
+			// still not found, see if there is an explicit root path
+			foundEntry = this.get('');
+			if (foundEntry && (!exportType || foundEntry.exportTypes?.[exportType] !== false)) {
+				if (url.charAt(0) !== '/') url = '/' + url;
+				foundEntry.relativeURL = url;
+			}
+		}
+		return foundEntry;
+	}
+
+	getResource(path: string, resourceInfo) {
+		const entry = this.getMatch(path);
+		if (entry) {
+			path = entry.relativeURL;
+			return entry.Resource.getResource(this.pathToId(path, entry.Resource), resourceInfo);
+		}
+	}
+	call(path: string, request, callback: Function) {
+		return transaction(request, async () => {
+			const entry = this.getMatch(path);
+			if (entry) {
+				path = entry.relativeURL;
+				return callback(entry.Resource, entry.path, path);
+			}
+		});
+	}
+	// eslint-disable-next-line no-unused-vars
+	setRepresentation(path, type, representation) {}
+}
+export let resources: Resources;
+export function resetResources() {
+	resources = new Resources();
+	server.resources = resources;
+	return resources;
+}
+
+export function keyArrayToString(key) {
+	if (Array.isArray(key)) {
+		if (key[key.length - 1] === null) return key.slice(0, -1).join('/') + '/';
+		else return key.join('/');
+	}
+	return key;
+}

package/resources/RocksIndexStore.ts

@@ -0,0 +1,70 @@
+import {
+	DBI,
+	Store,
+	type StoreContext,
+	type StoreIteratorOptions,
+	type StorePutOptions,
+	type StoreRemoveOptions,
+} from '@harperfast/rocksdb-js';
+import { Id } from './ResourceInterface.ts';
+import { MAXIMUM_KEY } from 'ordered-binary';
+
+declare module '@harperfast/rocksdb-js' {
+	// eslint-disable-next-line no-unused-vars
+	interface DBI<T> {
+		getValuesCount(indexedValue: any): number;
+	}
+}
+
+export class RocksIndexStore extends Store {
+	/**
+	 * Get all entries matching the range
+	 * @param options
+	 */
+	getRange(context: StoreContext, options: StoreIteratorOptions): Iterable<any> {
+		let { start, end, exclusiveStart, inclusiveEnd, reverse } = options;
+		if ((reverse ? !exclusiveStart : exclusiveStart) && start !== undefined) {
+			start = [start, MAXIMUM_KEY];
+		}
+		if ((reverse ? !inclusiveEnd : inclusiveEnd) && end !== undefined) {
+			end = [end, MAXIMUM_KEY];
+		}
+		const translatedOptions = { ...options, start, end };
+		return super.getRange(context, translatedOptions).map(({ key }) => {
+			return { key: key[0], value: key.length > 2 ? key.slice(1) : key[1] };
+		});
+	}
+
+	/**
+	 * Translate a put with indexed value and primary key to an underlying put
+	 * @param indexedValue - ignored, only used by LMDB
+	 * @param primaryKey
+	 * @param txnId
+	 */
+	put(context: StoreContext, indexedValue: any, primaryKey: Id, options: StorePutOptions) {
+		return super.putSync(context, [indexedValue, primaryKey], null, options);
+	}
+
+	putSync(context: StoreContext, indexedValue: any, primaryKey: Id, options: StorePutOptions) {
+		return super.putSync(context, [indexedValue, primaryKey], null, options);
+	}
+
+	remove(context: StoreContext, indexedValue: any, primaryKey: Id, options?: StoreRemoveOptions) {
+		return super.removeSync(context, [indexedValue, primaryKey], options);
+	}
+
+	removeSync(context: StoreContext, indexedValue: any, primaryKey: Id, options?: StoreRemoveOptions) {
+		super.removeSync(context, [indexedValue, primaryKey], options);
+	}
+}
+
+/**
+ * Add `getValuesCount` to the DBI prototype which is used by the `RocksDatabase` and `Transaction`
+ * classes.
+ */
+DBI.prototype.getValuesCount = function getValuesCount(indexedValue: any) {
+	if (this.store instanceof RocksIndexStore) {
+		return this.store.getCount(this._context, { start: indexedValue, end: [indexedValue, MAXIMUM_KEY] });
+	}
+	throw new Error('getValuesCount is only supported if dupSort=true');
+};