@verdant-web/common 1.11.0

package/src/batching.ts

@@ -0,0 +1,135 @@
+export class Batcher<T, UserData = any> {
+	private batches: Map<string, Batch<T, UserData>> = new Map();
+
+	constructor(
+		private flusher: (items: T[], batchKey: string, userData: UserData) => any,
+	) {}
+
+	add({
+		key,
+		userData,
+		items,
+		max,
+		timeout,
+	}: {
+		key: string;
+		userData?: UserData;
+		items: T[];
+		max?: number | null;
+		timeout?: number | null;
+	}) {
+		let batch = this.batches.get(key);
+		if (!batch) {
+			batch = new Batch({
+				max: max || null,
+				startedAt: Date.now(),
+				userData,
+				timeout: timeout || null,
+				flusher: this.flusher,
+				key,
+			});
+			this.batches.set(key, batch);
+		}
+		batch.update({
+			items,
+			max,
+			timeout,
+			userData,
+		});
+
+		return batch;
+	}
+
+	flush = (key: string) => {
+		const batch = this.batches.get(key);
+		if (!batch) return;
+
+		return batch.flush();
+	};
+
+	discard = (key: string) => {
+		const batch = this.batches.get(key);
+		if (!batch) return;
+
+		batch.discard();
+		this.batches.delete(key);
+	};
+}
+
+export class Batch<T, UserData = any> {
+	items: Array<T> = [];
+	max: number | null;
+	startedAt: number;
+	timeout: number | null;
+	flushTimeout?: NodeJS.Timeout;
+	userData?: any;
+	flusher: (items: T[], batchKey: string, userData: UserData) => any;
+	key: string;
+
+	constructor({
+		max,
+		startedAt,
+		timeout,
+		userData,
+		flusher,
+		key,
+	}: {
+		key: string;
+		max: number | null;
+		startedAt: number;
+		timeout: number | null;
+		userData?: UserData;
+		flusher: (items: T[], batchKey: string, userData: UserData) => any;
+	}) {
+		this.max = max;
+		this.startedAt = startedAt;
+		this.timeout = timeout;
+		this.userData = userData;
+		this.flusher = flusher;
+		this.key = key;
+	}
+
+	update = ({
+		items,
+		max,
+		timeout,
+		userData,
+	}: {
+		items: Array<T>;
+		max?: number | null;
+		timeout?: number | null;
+		userData?: UserData;
+	}) => {
+		this.items.push(...items);
+		if (max !== undefined) this.max = max;
+		if (timeout !== undefined) this.timeout = timeout;
+		if (userData) this.userData = userData;
+
+		// if the batch has items and a timeout but has not
+		// scheduled yet, schedule it
+		const needsSchedule =
+			this.items.length !== 0 && this.timeout !== null && !this.flushTimeout;
+
+		// if the batch has already reached its max, skip scheduling
+		// and flush immediately
+		if (this.max !== null && this.items.length >= this.max) {
+			this.flush();
+		} else if (needsSchedule && this.timeout !== null) {
+			this.flushTimeout = setTimeout(this.flush, this.timeout);
+		}
+	};
+
+	flush = () => {
+		this.flushTimeout && clearTimeout(this.flushTimeout);
+		this.flushTimeout = undefined;
+		const items = this.items;
+		this.items = [];
+		return this.flusher(items, this.key, this.userData);
+	};
+
+	discard = () => {
+		this.flushTimeout && clearTimeout(this.flushTimeout);
+		this.flushTimeout = undefined;
+		this.items = [];
+	};
+}

package/src/files.ts

@@ -0,0 +1,34 @@
+export type FileRef = {
+	'@@type': 'file';
+	id: string;
+};
+
+export function isFileRef(value: any): value is FileRef {
+	return value && value['@@type'] === 'file';
+}
+export function createFileRef(id: string): FileRef {
+	return {
+		'@@type': 'file',
+		id,
+	};
+}
+
+export type FileData = {
+	id: string;
+	/**
+	 * For locally created files, this starts false, until it's uploaded
+	 * Remote files this is always true
+	 */
+	remote: boolean;
+	name: string;
+	type: string;
+	file?: Blob;
+	url?: string;
+};
+
+export function getAllFileFields(snapshot: any): [string, FileRef][] {
+	return Object.entries(snapshot).filter((entry) => isFileRef(entry[1])) as [
+		string,
+		FileRef,
+	][];
+}

package/src/index.ts

@@ -0,0 +1,26 @@
+export * from './protocol.js';
+export * from './timestamp.js';
+export * from './operation.js';
+export * from './baseline.js';
+export * from './replica.js';
+export * from './schema/index.js';
+export * from './utils.js';
+export * from './indexes.js';
+export {
+	migrate,
+	migrationRange,
+	createDefaultMigration,
+} from './migration.js';
+export type {
+	Migration,
+	MigrationIndexDescription,
+	MigrationEngine,
+} from './migration.js';
+export type { UserInfo } from './presence.js';
+export * from './patch.js';
+export * from './oids.js';
+export * from './EventSubscriber.js';
+export * from './undo.js';
+export * from './batching.js';
+export * from './files.js';
+export type { Ref } from './refs.js';

package/src/indexes.test.ts

@@ -0,0 +1,158 @@
+import { describe, expect, it } from 'vitest';
+import {
+	COMPOUND_INDEX_LOWER_BOUND_SEPARATOR,
+	COMPOUND_INDEX_SEPARATOR,
+	COMPOUND_INDEX_UPPER_BOUND_SEPARATOR,
+	createCompoundIndexValue,
+	createLowerBoundIndexValue,
+	createUpperBoundIndexValue,
+} from './indexes.js';
+
+describe('compound indexes', () => {
+	it('can be created from two string values', () => {
+		expect(createCompoundIndexValue('a', 'b')).toEqual(
+			`a${COMPOUND_INDEX_SEPARATOR}b`,
+		);
+	});
+	it('can be created from two number values', () => {
+		expect(createCompoundIndexValue(1, 2)).toEqual(
+			`1${COMPOUND_INDEX_SEPARATOR}2`,
+		);
+	});
+	it('can be created from an array and string value', () => {
+		expect(createCompoundIndexValue(['a', 'b'], 'c')).toEqual([
+			`a${COMPOUND_INDEX_SEPARATOR}c`,
+			`b${COMPOUND_INDEX_SEPARATOR}c`,
+		]);
+	});
+	it('can be created from a string and array of numbers', () => {
+		expect(createCompoundIndexValue('a', [1, 2])).toEqual([
+			`a${COMPOUND_INDEX_SEPARATOR}1`,
+			`a${COMPOUND_INDEX_SEPARATOR}2`,
+		]);
+	});
+	it('can be created from an array of strings and an array of booleans', () => {
+		expect(createCompoundIndexValue(['a', 'b'], [true, false])).toEqual([
+			`a${COMPOUND_INDEX_SEPARATOR}true`,
+			`a${COMPOUND_INDEX_SEPARATOR}false`,
+			`b${COMPOUND_INDEX_SEPARATOR}true`,
+			`b${COMPOUND_INDEX_SEPARATOR}false`,
+		]);
+	});
+	it('remove duplicate array compound index values', () => {
+		expect(
+			createCompoundIndexValue(['a', 'b', 'b'], [true, false, true]),
+		).toEqual([
+			`a${COMPOUND_INDEX_SEPARATOR}true`,
+			`a${COMPOUND_INDEX_SEPARATOR}false`,
+			`b${COMPOUND_INDEX_SEPARATOR}true`,
+			`b${COMPOUND_INDEX_SEPARATOR}false`,
+		]);
+	});
+
+	it('creates lower bound value for multi-value indexes', () => {
+		expect(createLowerBoundIndexValue('a', 'b')).toEqual(
+			`a${COMPOUND_INDEX_SEPARATOR}b${COMPOUND_INDEX_SEPARATOR}${COMPOUND_INDEX_LOWER_BOUND_SEPARATOR}`,
+		);
+	});
+
+	it('creates upper bound value for multi-value indexes', () => {
+		expect(createUpperBoundIndexValue('a', 'b')).toEqual(
+			`a${COMPOUND_INDEX_SEPARATOR}b${COMPOUND_INDEX_UPPER_BOUND_SEPARATOR}`,
+		);
+	}),
+		it('will always be grouped by the first item', () => {
+			expect(
+				[
+					createCompoundIndexValue('aaaaaa', 'b'),
+					createCompoundIndexValue('aaaaaa', 'c'),
+					createCompoundIndexValue('aaaaaa', 'd'),
+					createCompoundIndexValue('aaaaaa', 'e'),
+					// extra nefarious!
+					createCompoundIndexValue('aaaaaaa', 'no!'),
+					createCompoundIndexValue('aaaaaa ', 'no!'),
+					createCompoundIndexValue(`aaaaaa😎`, 'no!'),
+					createCompoundIndexValue(`aaaaaa😎😎`, 'no!'),
+					createCompoundIndexValue('aaa', 'no!'),
+					createCompoundIndexValue(`aaaaaaa😎no!`, 'no!'),
+				].sort(),
+			).toEqual([
+				`aaaaaa ${COMPOUND_INDEX_SEPARATOR}no!`,
+				`aaaaaaa😎no!${COMPOUND_INDEX_SEPARATOR}no!`,
+				`aaaaaaa${COMPOUND_INDEX_SEPARATOR}no!`,
+				`aaaaaa😎😎${COMPOUND_INDEX_SEPARATOR}no!`,
+				`aaaaaa😎${COMPOUND_INDEX_SEPARATOR}no!`,
+				// this is the contiguous block of relevant values
+				`aaaaaa${COMPOUND_INDEX_SEPARATOR}b`,
+				`aaaaaa${COMPOUND_INDEX_SEPARATOR}c`,
+				`aaaaaa${COMPOUND_INDEX_SEPARATOR}d`,
+				`aaaaaa${COMPOUND_INDEX_SEPARATOR}e`,
+				// here ends the block
+				`aaa${COMPOUND_INDEX_SEPARATOR}no!`,
+			]);
+		});
+
+	it('will be bounded by upper and lower bound values', () => {
+		const match = 'aaaaaa';
+		const sorted = [
+			createCompoundIndexValue(match, 'b'),
+			createCompoundIndexValue(match, 'c'),
+			createCompoundIndexValue(match, 'd'),
+			createCompoundIndexValue(match, 'e'),
+			// extra nefarious!
+			createCompoundIndexValue(`${match}a`, 'no!'),
+			createCompoundIndexValue(`${match} `, 'no!'),
+			createCompoundIndexValue(`${match}😎`, 'no!'),
+			createCompoundIndexValue(`${match}😎😎`, 'no!'),
+			createCompoundIndexValue('aaa', 'no!'),
+			createCompoundIndexValue(`${match}a😎no!`, 'no!'),
+		].sort();
+
+		const boundedBlock = sorted.filter(
+			(value) =>
+				value > createLowerBoundIndexValue(match) &&
+				value < createUpperBoundIndexValue(match),
+		);
+		expect(boundedBlock).toEqual([
+			`aaaaaa${COMPOUND_INDEX_SEPARATOR}b`,
+			`aaaaaa${COMPOUND_INDEX_SEPARATOR}c`,
+			`aaaaaa${COMPOUND_INDEX_SEPARATOR}d`,
+			`aaaaaa${COMPOUND_INDEX_SEPARATOR}e`,
+		]);
+	});
+
+	it('will be bounded by upper and lower bounds when matching multiple sections', () => {
+		const match1 = 'aaaaaa';
+		const match2 = 'bbbbbb';
+		const sorted = [
+			createCompoundIndexValue(match1, match2, 'b'),
+			createCompoundIndexValue(match1, match2, 'c'),
+			createCompoundIndexValue(match1, match2, 'd'),
+			createCompoundIndexValue(match1, match2, 'e'),
+			// extra nefarious!
+			createCompoundIndexValue(`${match1}a`, match2, 'no!'),
+			createCompoundIndexValue(`${match1} `, match2, 'no!'),
+			createCompoundIndexValue(`${match1}😎`, match2, 'no!'),
+			createCompoundIndexValue(`${match1}😎😎`, match2, 'no!'),
+			createCompoundIndexValue('aaa', match2, 'no!'),
+			createCompoundIndexValue(`${match1}a😎no!`, match2, 'no!'),
+			createCompoundIndexValue(match1, `${match2}a😎no!`, 'no!'),
+			createCompoundIndexValue(match1, `${match2} `, 'no!'),
+			createCompoundIndexValue(match1, ' ', 'no!'),
+			createCompoundIndexValue(match1, '\u00001', 'no!'),
+			createCompoundIndexValue(match1, '\uE007f', 'no!'),
+		].sort();
+		expect(
+			sorted.filter(
+				(value) =>
+					value > createLowerBoundIndexValue(match1, match2) &&
+					value < createUpperBoundIndexValue(match1, match2),
+			),
+		).toEqual([
+			`aaaaaa${COMPOUND_INDEX_SEPARATOR}bbbbbb${COMPOUND_INDEX_SEPARATOR}b`,
+			`aaaaaa${COMPOUND_INDEX_SEPARATOR}bbbbbb${COMPOUND_INDEX_SEPARATOR}c`,
+			`aaaaaa${COMPOUND_INDEX_SEPARATOR}bbbbbb${COMPOUND_INDEX_SEPARATOR}d`,
+			`aaaaaa${COMPOUND_INDEX_SEPARATOR}bbbbbb${COMPOUND_INDEX_SEPARATOR}e`,
+		]);
+	});
+});

package/src/indexes.ts

@@ -0,0 +1,151 @@
+import {
+	StorageCollectionSchema,
+	StorageSyntheticIndexSchema,
+	CollectionCompoundIndex,
+} from './index.js';
+
+// unlikely to be used unicode character
+export const COMPOUND_INDEX_SEPARATOR = '\uFFFFFE';
+// 1 lower than separator
+export const COMPOUND_INDEX_LOWER_BOUND_SEPARATOR = '\u0000';
+// 1 higher than separator
+export const COMPOUND_INDEX_UPPER_BOUND_SEPARATOR = '\uFFFFFF';
+
+type IndexableFieldValue = string | number | boolean | any[];
+export type CompoundIndexValue = string | string[];
+
+export function createCompoundIndexValue(
+	...fields: IndexableFieldValue[]
+): CompoundIndexValue {
+	const value = expandArrayIndex(fields);
+	if (value.length === 1) {
+		return value[0];
+	}
+	return value;
+}
+
+export function createUpperBoundIndexValue(
+	...fields: IndexableFieldValue[]
+): string {
+	return (
+		fields.join(COMPOUND_INDEX_SEPARATOR) +
+		`${COMPOUND_INDEX_UPPER_BOUND_SEPARATOR}`
+	);
+}
+
+export function createLowerBoundIndexValue(
+	...fields: IndexableFieldValue[]
+): string {
+	return (
+		fields.join(COMPOUND_INDEX_SEPARATOR) +
+		`${COMPOUND_INDEX_SEPARATOR}${COMPOUND_INDEX_LOWER_BOUND_SEPARATOR}`
+	);
+}
+
+/**
+ * Whenever an array value is included in a compound index, we have to expand
+ * the value to include all permutations of values in the array.
+ * For example if we had an index id + tags on a document
+ *
+ * {
+ *   id: '1',
+ *   tags: ['a', 'b']
+ * }
+ *
+ * we want to create an index:
+ *
+ * id_tags: ['1#a', '1#b']
+ *
+ * If multiple arrays are indexed we have to exponentially expand...
+ *
+ * id_tags_tags2: ['1#a#a', '1#a#b', '1#b#a', '1#b#b']
+ *
+ * To generalize this we construct a 2-level array of strings.
+ * Iterating over indexed values, we expand each item into N items if
+ * the current value is an array.
+ *
+ * Then we combine all nested arrays into compound index values.
+ * This will produce an array of 1 element if none of the indexed values
+ * were arrays. The caller should unwrap that.
+ *
+ * This function also deduplicates values.
+ */
+function expandArrayIndex(fields: IndexableFieldValue[]): string[] {
+	let value: string[][] = [[]];
+	for (const field of fields) {
+		if (Array.isArray(field)) {
+			const newValue: string[][] = [];
+			for (const previousValue of value) {
+				for (const fieldValue of field) {
+					newValue.push(previousValue.concat(fieldValue));
+				}
+			}
+			value = newValue;
+		} else {
+			for (const item of value) {
+				item.push(`${field}`);
+			}
+		}
+	}
+	return Array.from(
+		new Set(
+			value.map((item) => {
+				return item.join(COMPOUND_INDEX_SEPARATOR);
+			}),
+		),
+	);
+}
+
+export function computeSynthetics(schema: StorageCollectionSchema, obj: any) {
+	const result: Record<string, any> = {};
+	for (const [name, property] of Object.entries(schema.synthetics || {})) {
+		result[name] = sanitizeIndexValue(
+			(property as StorageSyntheticIndexSchema<any>).compute(obj),
+		);
+	}
+	return result;
+}
+
+export function computeCompoundIndices(
+	schema: StorageCollectionSchema<any, any, any>,
+	doc: any,
+): any {
+	return Object.entries(schema.compounds || {}).reduce<
+		Record<string, CompoundIndexValue>
+	>((acc, [indexKey, index]) => {
+		acc[indexKey] = createCompoundIndexValue(
+			...(index as CollectionCompoundIndex<any, any>).of.map(
+				(key) => doc[key] as string | number,
+			),
+		);
+		return acc;
+	}, {} as Record<string, CompoundIndexValue>);
+}
+
+export function assignIndexValues(
+	schema: StorageCollectionSchema<any, any, any>,
+	doc: any,
+) {
+	Object.assign(doc, computeSynthetics(schema, doc));
+	Object.assign(doc, computeCompoundIndices(schema, doc));
+	return doc;
+}
+
+export function sanitizeIndexValue(
+	value: unknown,
+): string | number | (string | number)[] {
+	if (typeof value === 'string' || typeof value === 'number') {
+		return value;
+	}
+	if (typeof value === 'boolean' || value === null) {
+		return `${value}`;
+	}
+	if (value === undefined) {
+		// this shouldn't happen ,but for resiliency...
+		return 'undefined';
+	}
+	if (Array.isArray(value)) {
+		return value.map(sanitizeIndexValue) as any;
+	}
+	throw new Error(`Unsupported index value: ${value}`);
+}