@verdant-web/common 1.11.0

package/src/schema/types/collection.ts

@@ -0,0 +1,133 @@
+import { CollectionCompoundIndices } from './compounds.js';
+import {
+	NestedStorageFieldSchema,
+	StorageFieldSchema,
+	StorageFieldsSchema,
+	StorageIndexableFields,
+} from './fields.js';
+import { StorageDocument } from './shapes.js';
+import {
+	StorageSyntheticIndexSchema,
+	StorageSyntheticIndices,
+} from './synthetics.js';
+
+/**
+ * The main collection schema
+ */
+export type StorageCollectionSchema<
+	Fields extends StorageFieldsSchema = StorageFieldsSchema,
+	Synthetics extends StorageSyntheticIndices<Fields> = StorageSyntheticIndices<Fields>,
+	Compounds extends CollectionCompoundIndices<
+		Fields,
+		Synthetics
+	> = CollectionCompoundIndices<Fields, Synthetics>,
+> = {
+	name: string;
+	/**
+	 * Optionally provide a plural name. Useful for some code generation purposes.
+	 */
+	pluralName?: string;
+	fields: Fields;
+	synthetics?: Synthetics;
+	compounds?: Compounds;
+	primaryKey: StorageIndexablePropertyName<Fields, Synthetics>;
+};
+
+export type NamedSchema<
+	Schemas extends StorageCollectionSchema<any, any, any>,
+	Name extends string,
+> = Schemas extends { name: Name } ? Schemas : never;
+
+export type IndexedSchemaProperties<
+	Schema extends StorageCollectionSchema<any, any, any>,
+> = {
+	[K in keyof CollectionSchemaFields<Schema> as CollectionSchemaFields<Schema>[K] extends {
+		indexed: true;
+	}
+		? K
+		: never]: CollectionSchemaFields<Schema>[K];
+} & CollectionSchemaComputedIndexes<Schema> &
+	CollectionSchemaCompoundIndexes<Schema>;
+
+export type CollectionIndexName<
+	Collection extends StorageCollectionSchema<any, any, any>,
+> = Extract<keyof IndexedSchemaProperties<Collection>, string>;
+
+export type CollectionProperties<
+	Collection extends StorageCollectionSchema<any, any, any>,
+> = Collection['fields'] & Collection['synthetics'];
+
+export type CollectionPropertyName<
+	Collection extends StorageCollectionSchema<any, any, any>,
+> = Exclude<keyof CollectionProperties<Collection>, number | symbol>;
+
+export type CollectionEvents<
+	Collection extends StorageCollectionSchema<any, any, any>,
+> = {
+	put: (value: StorageDocument<Collection>) => void;
+	delete: (id: string) => void;
+	[key: `put:${string}`]: (value: StorageDocument<Collection>) => void;
+	[key: `delete:${string}`]: () => void;
+};
+
+export type SchemaForCollection<
+	Collection extends StorageCollectionSchema<any, any, any>,
+> = Collection;
+
+export type CollectionSchemaFields<
+	Schema extends StorageCollectionSchema<any, any, any>,
+> = Schema extends StorageCollectionSchema<infer F, any, any> ? F : never;
+export type CollectionSchemaComputedIndexes<
+	Schema extends StorageCollectionSchema<any, any, any>,
+> = Schema extends StorageCollectionSchema<any, infer S, any> ? S : never;
+export type CollectionSchemaCompoundIndexes<
+	Schema extends StorageCollectionSchema<any, any, any>,
+> = Schema extends StorageCollectionSchema<any, any, infer C> ? C : never;
+
+export type StorageIndexableProperties<
+	Schema extends StorageCollectionSchema<any, any, any>,
+> = {
+	[K in keyof CollectionSchemaFields<Schema>]: CollectionSchemaFields<Schema>[K] extends {
+		indexed: boolean;
+	}
+		? CollectionSchemaFields<Schema>[K]
+		: never;
+} & CollectionSchemaComputedIndexes<Schema>;
+
+export type StorageSchemaProperty<
+	Schema extends StorageCollectionSchema<any, any, any>,
+> = StorageSchemaProperties<Schema>[keyof StorageSchemaProperties<Schema>];
+
+export type StorageSchemaPropertyName<
+	Schema extends StorageCollectionSchema<any, any, any>,
+> = Extract<keyof StorageSchemaProperties<Schema>, string>;
+
+export type GetSchemaProperty<
+	Schema extends StorageCollectionSchema<any, any, any>,
+	Key extends StorageSchemaPropertyName<Schema>,
+> = StorageSchemaProperties<Schema>[Key];
+
+export type StoragePropertySchema<BaseFields extends StorageFieldsSchema> =
+	| StorageFieldSchema
+	| NestedStorageFieldSchema
+	| StorageSyntheticIndexSchema<BaseFields>;
+
+export type StoragePropertyName<
+	Fields extends StorageFieldsSchema,
+	Synthetics extends StorageSyntheticIndices<Fields>,
+> =
+	| Exclude<keyof Fields, number | symbol>
+	| Exclude<keyof Synthetics, number | symbol>;
+
+export type StorageIndexablePropertyName<
+	Fields extends StorageFieldsSchema,
+	Synthetics extends StorageSyntheticIndices<Fields>,
+> =
+	| Extract<keyof StorageIndexableFields<Fields>, string>
+	| Extract<keyof Synthetics, string>;
+
+export type StorageSchemaProperties<
+	Schema extends StorageCollectionSchema<any, any, any>,
+> = Schema extends StorageCollectionSchema<infer F, infer S, infer C>
+	? F & S & C
+	: never;

package/src/schema/types/compounds.ts

@@ -0,0 +1,33 @@
+import {
+	CollectionSchemaCompoundIndexes,
+	StorageCollectionSchema,
+	StoragePropertyName,
+} from './collection.js';
+import { StorageFieldsSchema, StorageObjectFieldSchema } from './fields.js';
+import { StorageSyntheticIndices } from './synthetics.js';
+
+type OmitObjectFields<T extends StorageFieldsSchema> = {
+	[K in keyof T]: T[K] extends StorageObjectFieldSchema ? never : T[K];
+};
+
+export type CollectionCompoundIndex<
+	Fields extends StorageFieldsSchema,
+	Synthetics extends StorageSyntheticIndices<Fields>,
+> = {
+	// object fields cannot be compound index inputs
+	of: StoragePropertyName<OmitObjectFields<Fields>, Synthetics>[];
+};
+type CompoundFieldNameTuple<Name extends string> =
+	| [Name, Name]
+	| [Name, Name, Name]
+	| [Name, Name, Name, Name]
+	| [Name, Name, Name, Name, Name];
+
+export type CollectionCompoundIndices<
+	Fields extends StorageFieldsSchema,
+	Synthetics extends StorageSyntheticIndices<Fields>,
+> = Record<string, CollectionCompoundIndex<Fields, Synthetics>>;
+
+export type CollectionCompoundIndexName<
+	Collection extends StorageCollectionSchema<any, any, any>,
+> = Exclude<keyof CollectionSchemaCompoundIndexes<Collection>, number | symbol>;

package/src/schema/types/fields.ts

@@ -0,0 +1,85 @@
+export type StorageStringFieldSchema = {
+	type: 'string';
+	indexed?: boolean;
+	nullable?: boolean;
+	default?: string | (() => string);
+};
+export type StorageNumberFieldSchema = {
+	type: 'number';
+	indexed?: boolean;
+	nullable?: boolean;
+	default?: number | (() => number);
+};
+export type StorageBooleanFieldSchema = {
+	type: 'boolean';
+	nullable?: boolean;
+	default?: boolean | (() => boolean);
+	indexed?: boolean;
+};
+export type StorageArrayFieldSchema = {
+	type: 'array';
+	items: NestedStorageFieldSchema;
+	nullable?: boolean;
+};
+export type StorageObjectFieldSchema = {
+	type: 'object';
+	properties: NestedStorageFieldsSchema;
+	nullable?: boolean;
+};
+export type StorageAnyFieldSchema = {
+	type: 'any';
+	default?: any;
+};
+export type StorageMapFieldSchema<V extends NestedStorageFieldSchema> = {
+	type: 'map';
+	values: V;
+};
+export type StorageFileFieldSchema = {
+	type: 'file';
+	nullable?: boolean;
+	// downloadRemote?: boolean; TODO: is this a good feature? maybe later.
+};
+
+export type StorageFieldSchema =
+	| StorageStringFieldSchema
+	| StorageNumberFieldSchema
+	| StorageBooleanFieldSchema
+	| StorageArrayFieldSchema
+	| StorageObjectFieldSchema
+	| StorageAnyFieldSchema
+	| StorageMapFieldSchema<any>
+	| StorageFileFieldSchema;
+
+// nested versions don't have index info
+export type NestedStorageStringFieldSchema = {
+	type: 'string';
+	nullable?: boolean;
+	default?: string | (() => string);
+};
+export type NestedStorageNumberFieldSchema = {
+	type: 'number';
+	nullable?: boolean;
+	default?: number | (() => number);
+};
+
+export type NestedStorageFieldSchema =
+	| NestedStorageStringFieldSchema
+	| NestedStorageNumberFieldSchema
+	| StorageBooleanFieldSchema
+	| StorageArrayFieldSchema
+	| StorageObjectFieldSchema
+	| StorageAnyFieldSchema
+	| StorageMapFieldSchema<any>
+	| StorageFileFieldSchema;
+
+export type StorageFieldsSchema = Record<string, StorageFieldSchema>;
+
+export type NestedStorageFieldsSchema = Record<
+	string,
+	NestedStorageFieldSchema
+>;
+
+// filters to only fields which can be indexed
+export type StorageIndexableFields<Fields extends StorageFieldsSchema> = {
+	[K in keyof Fields]: Fields[K] extends { indexed: boolean } ? K : never;
+};

package/src/schema/types/filters.ts

@@ -0,0 +1,39 @@
+export type MatchCollectionIndexFilter = {
+	where: string;
+	equals: any;
+	order?: 'asc' | 'desc';
+};
+
+export type RangeCollectionIndexFilter = {
+	where: string;
+	gte?: any;
+	lte?: any;
+	gt?: any;
+	lt?: any;
+	order?: 'asc' | 'desc';
+};
+
+export type CollectionCompoundIndexFilter = {
+	where: string;
+	match: Record<string, any>;
+	order: 'asc' | 'desc';
+};
+
+export type SortIndexFilter = {
+	where: string;
+	order: 'asc' | 'desc';
+};
+
+export type StartsWithIndexFilter = {
+	where: string;
+	startsWith: string;
+	order?: 'asc' | 'desc';
+};
+
+export type CollectionIndexFilter =
+	| MatchCollectionIndexFilter
+	| RangeCollectionIndexFilter
+	| CollectionCompoundIndexFilter
+	| StartsWithIndexFilter
+	| SortIndexFilter;
+export type CollectionFilter = CollectionIndexFilter;

package/src/schema/types/shapes.ts

@@ -0,0 +1,83 @@
+import {
+	StorageCollectionSchema,
+	StoragePropertySchema,
+} from './collection.js';
+import {
+	NestedStorageFieldsSchema,
+	StorageArrayFieldSchema,
+	StorageFieldsSchema,
+	StorageMapFieldSchema,
+	StorageObjectFieldSchema,
+} from './fields.js';
+
+type StoragePropertyIsNullable<T extends StoragePropertySchema<any>> =
+	T extends { nullable?: boolean }
+		? T['nullable'] extends boolean
+			? true
+			: false
+		: T['type'] extends 'any'
+		? true
+		: false;
+
+export type BaseShapeFromProperty<T extends StoragePropertySchema<any>> =
+	T['type'] extends 'string'
+		? string
+		: T['type'] extends 'number'
+		? number
+		: T['type'] extends 'boolean'
+		? boolean
+		: T extends StorageArrayFieldSchema
+		? ShapeFromProperty<T['items']>[]
+		: T extends StorageObjectFieldSchema
+		? ShapeFromFields<T['properties']>
+		: T extends StorageMapFieldSchema<any>
+		? Record<string, ShapeFromProperty<T['values']>>
+		: T['type'] extends 'any'
+		? any
+		: T['type'] extends 'file'
+		? File
+		: never;
+
+export type ShapeFromProperty<T extends StoragePropertySchema<any>> =
+	StoragePropertyIsNullable<T> extends true
+		? BaseShapeFromProperty<T> | null
+		: BaseShapeFromProperty<T>;
+
+export type ShapeFromFields<
+	T extends StorageFieldsSchema | NestedStorageFieldsSchema,
+> = {
+	[K in keyof T]: ShapeFromProperty<T[K]>;
+};
+
+export type StorageDocument<
+	Collection extends StorageCollectionSchema<any, any, any>,
+> = ShapeFromFields<Collection['fields']>;
+
+type StoragePropertyIsOptional<T extends StoragePropertySchema<any>> =
+	StoragePropertyIsNullable<T> extends true
+		? true
+		: T extends { default?: any }
+		? true
+		: T['type'] extends 'any'
+		? true
+		: T['type'] extends 'array'
+		? true
+		: T['type'] extends 'map'
+		? true
+		: false;
+
+type ShapeFromFieldsWithDefaults<
+	T extends StorageFieldsSchema | NestedStorageFieldsSchema,
+> = {
+	[K in keyof T as StoragePropertyIsOptional<T[K]> extends true
+		? K
+		: never]?: ShapeFromProperty<T[K]>;
+} & {
+	[K in keyof T as StoragePropertyIsOptional<T[K]> extends true
+		? never
+		: K]: ShapeFromProperty<T[K]>;
+};
+
+export type StorageDocumentInit<
+	Collection extends StorageCollectionSchema<any, any, any>,
+> = ShapeFromFieldsWithDefaults<Collection['fields']>;

package/src/schema/types/synthetics.ts

@@ -0,0 +1,45 @@
+import { StorageFieldsSchema } from './fields.js';
+import { ShapeFromFields } from './shapes.js';
+
+export type StorageStringSyntheticSchema<Fields extends StorageFieldsSchema> = {
+	type: 'string';
+	compute: (value: ShapeFromFields<Fields>) => string | null;
+};
+export type StorageNumberSyntheticSchema<Fields extends StorageFieldsSchema> = {
+	type: 'number';
+	compute: (value: ShapeFromFields<Fields>) => number | null;
+};
+export type StorageBooleanSyntheticSchema<Fields extends StorageFieldsSchema> =
+	{
+		type: 'boolean';
+		compute: (value: ShapeFromFields<Fields>) => boolean | null;
+	};
+export type StorageStringArraySyntheticSchema<
+	Fields extends StorageFieldsSchema,
+> = {
+	type: 'string[]';
+	compute: (value: ShapeFromFields<Fields>) => string[];
+};
+export type StorageNumberArraySyntheticSchema<
+	Fields extends StorageFieldsSchema,
+> = {
+	type: 'number[]';
+	compute: (value: ShapeFromFields<Fields>) => number[];
+};
+export type StorageBooleanArraySyntheticSchema<
+	Fields extends StorageFieldsSchema,
+> = {
+	type: 'boolean[]';
+	compute: (value: ShapeFromFields<Fields>) => boolean[];
+};
+
+export type StorageSyntheticIndices<Fields extends StorageFieldsSchema> =
+	Record<string, StorageSyntheticIndexSchema<Fields>>;
+
+export type StorageSyntheticIndexSchema<Fields extends StorageFieldsSchema> =
+	| StorageStringSyntheticSchema<Fields>
+	| StorageNumberSyntheticSchema<Fields>
+	| StorageBooleanSyntheticSchema<Fields>
+	| StorageStringArraySyntheticSchema<Fields>
+	| StorageNumberArraySyntheticSchema<Fields>
+	| StorageBooleanArraySyntheticSchema<Fields>;

package/src/schema/types.ts

@@ -0,0 +1,28 @@
+import { StorageCollectionSchema } from './types/collection.js';
+
+export * from './types/collection.js';
+export * from './types/compounds.js';
+export * from './types/fields.js';
+export * from './types/filters.js';
+export * from './types/shapes.js';
+export * from './types/synthetics.js';
+
+export interface StorageInit<Schemas extends StorageCollectionSchema> {
+	collections: Record<string, Schemas>;
+}
+
+export type StorageSchema<
+	Collections extends {
+		[k: string]: StorageCollectionSchema;
+	} = any,
+> = { version: number; collections: Collections };
+
+export type SchemaCollectionName<Schema extends StorageSchema<any>> =
+	Schema extends StorageSchema<infer Cs>
+		? Exclude<keyof Cs, number | symbol>
+		: never;
+
+export type SchemaCollection<
+	Schema extends StorageSchema<any>,
+	Name extends SchemaCollectionName<Schema>,
+> = Schema extends StorageSchema<infer Cs> ? Cs[Name] : never;

package/src/timestamp.test.ts

@@ -0,0 +1,113 @@
+import { describe, expect, it, vitest } from 'vitest';
+import {
+	convertOldHlcTimestamp,
+	deserializeHlcTimestamp,
+	HybridLogicalClockTimestampProvider,
+	OLD_encodeVersion,
+	OLD_serializeHlcTimestamp,
+	serializeHlcTimestamp,
+} from './timestamp.js';
+
+describe('the hybrid logical clock', () => {
+	it('always produces incrementing timestamps', async () => {
+		const clock = new HybridLogicalClockTimestampProvider();
+		let prev = clock.now(1);
+
+		clock.update(prev);
+		let now;
+		for (let i = 0; i < 1000; i++) {
+			now = clock.now(1);
+			expect(now > prev, `${now} not later than ${prev}`).toBe(true);
+			clock.update(now);
+			prev = now;
+		}
+
+		let time = new Date().getTime();
+		for (let i = 0; i < 1000; i++) {
+			vitest.setSystemTime(time++);
+			now = clock.now(1);
+			expect(now > prev, `${now} not later than ${prev}`).toBe(true);
+			clock.update(now);
+			prev = now;
+		}
+	});
+
+	it('groups blocks of time by version', () => {
+		const clock = new HybridLogicalClockTimestampProvider();
+		let prev = clock.now(1);
+		clock.update(prev);
+		let now: string;
+		for (let i = 0; i < 10; i++) {
+			now = clock.now(1);
+			expect(now > prev, `${now} not later than ${prev}`).toBe(true);
+			clock.update(now);
+			prev = now;
+		}
+		for (let i = 0; i < 10; i++) {
+			now = clock.now(2);
+			expect(now > prev, `${now} not later than ${prev}`).toBe(true);
+			clock.update(now);
+			prev = now;
+		}
+		const startOfVersion2 = now!;
+		// jump back to 1
+		prev = clock.now(1);
+		clock.update(prev);
+
+		for (let i = 0; i < 10; i++) {
+			now = clock.now(1);
+			expect(now > prev, `${now} not later than ${prev}`).toBe(true);
+			expect(
+				now < startOfVersion2,
+				`${now} not earlier than ${startOfVersion2}`,
+			).toBe(true);
+			clock.update(now);
+			prev = now;
+		}
+	});
+
+	it('is shorter than the previous format', () => {
+		const clock = new HybridLogicalClockTimestampProvider();
+
+		expect(clock.now(1).length).toBeLessThan(clock.OLD_now(1).length);
+	});
+
+	it('correctly converts old to new format', () => {
+		const clock = new HybridLogicalClockTimestampProvider();
+		const oldFormat = clock.OLD_now(1);
+		const value = clock.timerState();
+		const converted = convertOldHlcTimestamp(oldFormat);
+		const newFormat = clock.get(1, value);
+		expect(converted).toEqual(newFormat);
+	});
+
+	// this is mostly to verify as I move to the new format.
+	// instead of attempting a distributed migration of timestamps, it
+	// will be easier that the new format just acts as always in the
+	// future of the old one. this will only mess with offline clients,
+	// but nobody is using verdant but me yet at time of this change (2022-12-19)
+	// which is kind of why I'm doing it now anyway.
+	it('old format always comes before new format', () => {
+		const clock = new HybridLogicalClockTimestampProvider();
+		const newFormat = clock.now(1);
+		const oldFormat = clock.OLD_now(1);
+		expect(oldFormat < newFormat).toBe(true);
+
+		// even far-future old timestamps are before new timestamps
+		const oldFormat2 = OLD_serializeHlcTimestamp({
+			counter: 999,
+			node: '9876543210',
+			time: new Date(3000, 1, 1).getTime(),
+		});
+		expect(OLD_encodeVersion(1) + oldFormat2 < newFormat).toBe(true);
+	});
+
+	it('decodes timestamps', () => {
+		const time = new Date();
+		vitest.setSystemTime(time);
+
+		const clock = new HybridLogicalClockTimestampProvider();
+		const now = clock.now(1);
+		expect(clock.getWallClockTime(now)).toEqual(time.getTime());
+	});
+});