@verdant-web/common 1.11.0

package/src/patch.ts

@@ -0,0 +1,277 @@
+/**
+ * High-level patch creation for use with complex nested objects.
+ */
+
+import { createRef, createSubOid, KeyPath, ObjectIdentifier } from './oids.js';
+import {
+	diffToPatches,
+	initialToPatches,
+	ObjectRef,
+	Operation,
+	PropertyName,
+} from './operation.js';
+import { isRef } from './refs.js';
+import { isObject } from './utils.js';
+
+export class PatchCreator {
+	constructor(
+		private getNow: () => string,
+		private createSubId?: () => string,
+	) {}
+
+	createDiff = (
+		from: any,
+		to: any,
+		keyPath: KeyPath,
+		options: { mergeUnknownObjects?: boolean; defaultUndefined?: boolean } = {},
+	) => {
+		return diffToPatches(
+			from,
+			to,
+			this.getNow,
+			keyPath,
+			this.createSubId,
+			[],
+			options,
+		);
+	};
+
+	createInitialize = (obj: any, oid: ObjectIdentifier) => {
+		return initialToPatches(obj, oid, this.getNow, this.createSubId);
+	};
+
+	createSet = (
+		oid: ObjectIdentifier,
+		key: PropertyName,
+		value: any,
+		keyPath: KeyPath,
+	): Operation[] => {
+		// incoming value must be normalized. if it's not a primitive, it and all sub-objects
+		// must be created
+		if (!isObject(value) || isRef(value)) {
+			return [
+				{
+					oid,
+					timestamp: this.getNow(),
+					data: {
+						op: 'set',
+						name: key,
+						value,
+					},
+				},
+			];
+		} else {
+			const itemOid = createSubOid(oid, key, this.createSubId);
+			return [
+				// since we're setting a complex nested object, we can initialize it wholesale.
+				// no diffing to do.
+				...initialToPatches(value, itemOid, this.getNow),
+				// then set the reference to the object
+				{
+					oid,
+					timestamp: this.getNow(),
+					data: {
+						op: 'set',
+						value: createRef(itemOid),
+						name: key,
+					},
+				},
+			];
+		}
+	};
+
+	createRemove = (oid: ObjectIdentifier, key: PropertyName): Operation[] => {
+		return [
+			{
+				oid,
+				timestamp: this.getNow(),
+				data: {
+					op: 'remove',
+					name: key,
+				},
+			},
+		];
+	};
+
+	createListPush = (oid: ObjectIdentifier, value: any): Operation[] => {
+		if (!isObject(value)) {
+			return [
+				{
+					oid,
+					timestamp: this.getNow(),
+					data: {
+						op: 'list-push',
+						value,
+					},
+				},
+			];
+		} else {
+			const itemOid = createSubOid(oid, 0, this.createSubId);
+			return [
+				...initialToPatches(value, itemOid, this.getNow),
+				{
+					oid,
+					timestamp: this.getNow(),
+					data: {
+						op: 'list-push',
+						value: createRef(itemOid),
+					},
+				},
+			];
+		}
+	};
+
+	createListAdd = (oid: ObjectIdentifier, value: any): Operation[] => {
+		if (isObject(value)) {
+			return [
+				{
+					oid,
+					timestamp: this.getNow(),
+					data: {
+						op: 'list-add',
+						value: createRef(value),
+					},
+				},
+			];
+		} else {
+			return [
+				{
+					oid,
+					timestamp: this.getNow(),
+					data: {
+						op: 'list-add',
+						value,
+					},
+				},
+			];
+		}
+	};
+
+	createListInsert = (
+		oid: ObjectIdentifier,
+		index: number,
+		value: any,
+	): Operation[] => {
+		if (!isObject(value)) {
+			return [
+				{
+					oid,
+					timestamp: this.getNow(),
+					data: {
+						op: 'list-insert',
+						value,
+						index,
+					},
+				},
+			];
+		} else {
+			const itemOid = createSubOid(oid, index, this.createSubId);
+			return [
+				...initialToPatches(value, itemOid, this.getNow),
+				{
+					oid,
+					timestamp: this.getNow(),
+					data: {
+						op: 'list-insert',
+						value: createRef(itemOid),
+						index,
+					},
+				},
+			];
+		}
+	};
+
+	createListRemove = (
+		oid: ObjectIdentifier,
+		value: any,
+		only?: 'first' | 'last',
+	): Operation[] => {
+		return [
+			{
+				oid,
+				timestamp: this.getNow(),
+				data: {
+					op: 'list-remove',
+					value,
+					only,
+				},
+			},
+		];
+	};
+
+	createListDelete = (
+		oid: ObjectIdentifier,
+		index: number,
+		count: number = 1,
+	): Operation[] => {
+		return [
+			{
+				oid,
+				timestamp: this.getNow(),
+				data: {
+					op: 'list-delete',
+					index,
+					count,
+				},
+			},
+		];
+	};
+
+	createListMoveByRef = (
+		oid: ObjectIdentifier,
+		value: ObjectRef,
+		index: number,
+	): Operation[] => {
+		return [
+			{
+				oid,
+				timestamp: this.getNow(),
+				data: {
+					op: 'list-move-by-ref',
+					value,
+					index,
+				},
+			},
+		];
+	};
+
+	createListMoveByIndex = (
+		oid: ObjectIdentifier,
+		fromIndex: number,
+		toIndex: number,
+	): Operation[] => {
+		return [
+			{
+				oid,
+				timestamp: this.getNow(),
+				data: {
+					op: 'list-move-by-index',
+					from: fromIndex,
+					to: toIndex,
+				},
+			},
+		];
+	};
+
+	createDelete = (oid: ObjectIdentifier): Operation[] => {
+		return [
+			{
+				oid,
+				timestamp: this.getNow(),
+				data: {
+					op: 'delete',
+				},
+			},
+		];
+	};
+
+	createDeleteAll = (oids: ObjectIdentifier[]): Operation[] => {
+		return oids.map((oid) => ({
+			oid,
+			timestamp: this.getNow(),
+			data: {
+				op: 'delete',
+			},
+		}));
+	};
+}

package/src/presence.ts

@@ -0,0 +1,29 @@
+export interface UserInfo<Profile, Presence> {
+	/**
+	 * This is the ID representing the user who is utilizing a
+	 * replica client to connect to the storage network.
+	 * One user may have multiple replica clients active at once,
+	 * but their presence will only reflect the most recent
+	 * replica used.
+	 */
+	id: string;
+	/**
+	 * This is the ID representing the replica client that is
+	 * connected to the storage network. This is used to
+	 * identify the client when sending messages to it.
+	 */
+	replicaId: string;
+
+	/**
+	 * This is the user's server profile. This data is associated
+	 * with the logged in user of the app and cannot be modified
+	 * by the local replica directly.
+	 */
+	profile: Profile;
+
+	/**
+	 * Presence info which can update frequently as a replica
+	 * makes changes. The shape of presence is up to you.
+	 */
+	presence: Presence;
+}

package/src/protocol.ts

@@ -0,0 +1,184 @@
+// client ID, and therefore library ID, is inferred
+
+import { DocumentBaseline } from './baseline.js';
+import { Operation, OperationPatch } from './operation.js';
+import { UserInfo } from './presence.js';
+
+export type HeartbeatMessage = {
+	type: 'heartbeat';
+	timestamp: string;
+	replicaId: string;
+};
+
+export type HeartbeatResponseMessage = {
+	type: 'heartbeat-response';
+};
+
+/**
+ * Used by clients to indicate they have
+ * successfully applied all operations from the
+ * server up to this one. Uses the nonce from the
+ * operation rebroadcast message as the ack.
+ */
+export type AckMessage = {
+	type: 'ack';
+	replicaId: string;
+	timestamp?: string;
+	nonce?: string;
+};
+
+export type OperationMessage = {
+	type: 'op';
+	replicaId: string;
+	operations: Operation[];
+	timestamp: string;
+};
+
+export type OperationRebroadcastMessage = {
+	type: 'op-re';
+	operations: Operation[];
+	baselines: DocumentBaseline[];
+	replicaId: string;
+	globalAckTimestamp: string | undefined;
+	ackThisNonce?: string;
+};
+
+export type SyncMessage = {
+	type: 'sync';
+	/** This client's replica ID */
+	replicaId: string;
+	/** the logical time this message was sent */
+	timestamp: string;
+	/** Any new operations created since the requested time */
+	operations: Operation[];
+	/** Any new baselines created since the requested time */
+	baselines: DocumentBaseline[];
+	/** the schema version known by this client */
+	schemaVersion: number;
+	/**
+	 * the client may have lost its local data, in which
+	 * case it may set this flag to be treated as a new client
+	 * and receive a full baseline
+	 */
+	resyncAll?: boolean;
+	/**
+	 * timestamp of when the replica changes began. null means
+	 * full changeset from start of time
+	 */
+	since: string | null;
+};
+
+export type SyncResponseMessage = {
+	type: 'sync-resp';
+	// operations this client should apply
+	operations: Operation[];
+	// baselines this client should apply
+	baselines: DocumentBaseline[];
+	/**
+	 * If this flag is set, the client should discard local data
+	 * and reset to incoming data only. Used when a client requested
+	 * resyncAll in its sync message, or for clients which have been
+	 * offline for too long. When specified true, provideChangesSince
+	 * should be ignored.
+	 */
+	overwriteLocalData: boolean;
+	/**
+	 * Update client on the global ack
+	 */
+	globalAckTimestamp: string | undefined;
+
+	/**
+	 * A map of connected clients' presences values
+	 */
+	peerPresence: Record<string, UserInfo<any, any>>;
+
+	/**
+	 * The timestamp sent in the original sync message -
+	 * this confirms the server has received the client's
+	 * state up to this point. Subsequent syncs should not
+	 * include operations or baselines older than this timestamp.
+	 */
+	ackedTimestamp: string;
+
+	/**
+	 * The client should respond with a sync-ack containing
+	 * this nonce to confirm it has received this message if
+	 * it is not undefined
+	 */
+	ackThisNonce?: string;
+};
+
+/** @deprecated use ack */
+export type SyncAckMessage = {
+	type: 'sync-ack';
+	/** The client's replica ID */
+	replicaId: string;
+	/** the logical time this message was sent */
+	timestamp: string;
+	/** the nonce sent in the sync-resp message */
+	nonce: string;
+};
+
+export type PresenceUpdateMessage = {
+	type: 'presence-update';
+	/** The client's replica ID */
+	replicaId: string;
+	/** new presence value */
+	presence: any;
+};
+
+export type PresenceChangedMessage = {
+	type: 'presence-changed';
+	/** The client's replica ID */
+	replicaId: string;
+
+	userInfo: UserInfo<any, any>;
+};
+
+/**
+ * This is only emitted when all of a user's replicas
+ * go offline.
+ */
+export type PresenceOfflineMessage = {
+	type: 'presence-offline';
+	userId: string;
+	/** The last replicaID seen by the server before the user was offline */
+	replicaId: string;
+};
+
+export type GlobalAckMessage = {
+	type: 'global-ack';
+	timestamp: string;
+};
+
+export type ForbiddenMessage = {
+	type: 'forbidden';
+};
+
+export type ServerAckMessage = {
+	type: 'server-ack';
+	timestamp: string;
+};
+
+export type ServerNeedSinceMessage = {
+	type: 'need-since';
+	since: string | null;
+};
+
+export type ClientMessage =
+	| HeartbeatMessage
+	| SyncMessage
+	| OperationMessage
+	| AckMessage
+	| SyncAckMessage
+	| PresenceUpdateMessage;
+export type ServerMessage =
+	| HeartbeatResponseMessage
+	| SyncResponseMessage
+	| OperationRebroadcastMessage
+	| PresenceChangedMessage
+	| PresenceOfflineMessage
+	| GlobalAckMessage
+	| ForbiddenMessage
+	| ServerAckMessage
+	| ServerNeedSinceMessage;

package/src/refs.ts

@@ -0,0 +1,16 @@
+import { FileRef, isFileRef } from './files.js';
+import { isObjectRef, ObjectRef } from './operation.js';
+
+export function isRef(obj: any) {
+	return isObjectRef(obj) || isFileRef(obj);
+}
+
+export function compareRefs(a: any, b: any) {
+	if (a === b) return true;
+	if (!isRef(a) || !isRef(b)) return false;
+	if (a['@@type'] !== b['@@type']) return false;
+	if (a.id !== b.id) return false;
+	return true;
+}
+
+export type Ref = ObjectRef | FileRef;

package/src/replica.ts

@@ -0,0 +1,29 @@
+export interface ReplicaInfo {
+	id: string;
+	ackedLogicalTime: string | null;
+}
+
+/**
+ * Different token types allow different replica client behaviors.
+ * - Realtime: allows the client to subscribe to realtime events.
+ * - Push: allows the client to push and pull data with HTTP, but not use realtime.
+ * - PassivePush: allows the client to push and pull data with HTTP, but offline changes
+ *     will be discarded on reconnect.
+ * - PassiveRealtime: allows the client to subscribe to realtime events, but offline changes
+ * 		 will be discarded on reconnect.
+ * - ReadOnlyPull: the client may only pull changes using HTTP. It may not subscribe
+ *     to realtime events or push changes.
+ * - ReadOnlyRealtime: the client may only subscribe to realtime events or pull from HTTP.
+ *     It may not push changes.
+ *
+ * Choosing the right token type can optimize client storage metrics significantly when
+ * many replicas are connecting to a library.
+ */
+export enum ReplicaType {
+	Realtime,
+	Push,
+	PassiveRealtime,
+	PassivePush,
+	ReadOnlyPull,
+	ReadOnlyRealtime,
+}

package/src/schema/defaults.test.ts

@@ -0,0 +1,19 @@
+import { describe, expect, it } from 'vitest';
+import { getFieldDefault } from './defaults.js';
+import { assignOid, createOid, getOid, maybeGetOid } from '../oids.js';
+
+describe('getting field defaults', () => {
+	it('does not apply the same oid from the previous application', () => {
+		const field = {
+			type: 'any',
+			default: {
+				id: '123',
+				name: 'test',
+			},
+		} as const;
+		const first = getFieldDefault(field);
+		assignOid(first, createOid('test', '1', []));
+		const second = getFieldDefault(field);
+		expect(maybeGetOid(second)).toBe(undefined);
+	});
+});

package/src/schema/defaults.ts

@@ -0,0 +1,72 @@
+import { LEGACY_OID_KEY, OID_KEY } from '../oids.js';
+import type { StorageFieldSchema, StorageCollectionSchema } from './types.js';
+
+export function addFieldDefaults(
+	collection: StorageCollectionSchema,
+	value: any,
+) {
+	for (const [key, field] of Object.entries(collection.fields)) {
+		const defaultValue = getFieldDefault(field);
+		if (defaultValue !== undefined && value[key] === undefined) {
+			value[key] = defaultValue;
+		}
+		if (value[key]) {
+			traverseCollectionFieldsAndApplyDefaults(value[key], field);
+		}
+	}
+	return value;
+}
+
+export function traverseCollectionFieldsAndApplyDefaults(
+	value: any,
+	field: StorageFieldSchema,
+) {
+	if (value === undefined || value === null) return value;
+	if (field.type === 'object') {
+		for (const [key, subField] of Object.entries(field.properties)) {
+			if (value[key] === undefined) {
+				const defaultValue = getFieldDefault(subField);
+				if (defaultValue !== undefined) {
+					value[key] = defaultValue;
+				}
+			}
+			traverseCollectionFieldsAndApplyDefaults(value[key], subField);
+		}
+	} else if (field.type === 'array') {
+		for (const item of value) {
+			traverseCollectionFieldsAndApplyDefaults(item, field.items);
+		}
+	} else if (field.type === 'map') {
+		for (const [key, item] of Object.entries(value)) {
+			// santiy check to weed out any id field
+			if (key === OID_KEY || key === LEGACY_OID_KEY) continue;
+			traverseCollectionFieldsAndApplyDefaults(item, field.values);
+		}
+	}
+}
+
+export function getFieldDefault(field: StorageFieldSchema) {
+	if (
+		field.type === 'string' ||
+		field.type === 'number' ||
+		field.type === 'boolean' ||
+		field.type === 'any'
+	) {
+		if (field.default && typeof field.default === 'function') {
+			return field.default();
+		} else if (field.default !== undefined) {
+			// TODO: structuredClone?
+			return JSON.parse(JSON.stringify(field.default));
+		}
+	}
+	if (field.type === 'array') {
+		return [];
+	}
+	if (field.type === 'map') {
+		return {};
+	}
+	if (field.type !== 'any' && field.nullable) {
+		return null;
+	}
+	return undefined;
+}

package/src/schema/index.ts

@@ -0,0 +1,28 @@
+import {
+	CollectionCompoundIndices,
+	StorageCollectionSchema,
+	StorageFieldsSchema,
+	StorageSchema,
+	StorageSyntheticIndices,
+} from './types.js';
+
+export function collection<
+	Fields extends StorageFieldsSchema,
+	Synthetics extends StorageSyntheticIndices<Fields>,
+	Compounds extends CollectionCompoundIndices<Fields, Synthetics>,
+>(input: StorageCollectionSchema<Fields, Synthetics, Compounds>) {
+	return input;
+}
+
+export function schema<
+	Schema extends StorageSchema<{
+		[key: string]: StorageCollectionSchema<any, any, any>;
+	}>,
+>(input: Schema) {
+	return input;
+}
+
+export * from './types.js';
+
+export * from './indexFilters.js';
+export * from './defaults.js';

package/src/schema/indexFilters.ts

@@ -0,0 +1,49 @@
+import {
+	CollectionCompoundIndexFilter,
+	CollectionFilter,
+	MatchCollectionIndexFilter,
+	RangeCollectionIndexFilter,
+	SortIndexFilter,
+	StartsWithIndexFilter,
+} from './types.js';
+
+export function isMatchIndexFilter(
+	filter: CollectionFilter,
+): filter is MatchCollectionIndexFilter {
+	return (filter as any).equals !== undefined;
+}
+
+export function isRangeIndexFilter(
+	filter: CollectionFilter,
+): filter is RangeCollectionIndexFilter {
+	return (
+		(filter as any).gte !== undefined ||
+		(filter as any).lte !== undefined ||
+		(filter as any).gt !== undefined ||
+		(filter as any).lt !== undefined
+	);
+}
+
+export function isCompoundIndexFilter(
+	filter: CollectionFilter,
+): filter is CollectionCompoundIndexFilter {
+	return !!(filter as any).match;
+}
+
+export function isStartsWithIndexFilter(
+	filter: CollectionFilter,
+): filter is StartsWithIndexFilter {
+	return (filter as any).startsWith !== undefined;
+}
+
+export function isSortIndexFilter(
+	filter: CollectionFilter,
+): filter is SortIndexFilter {
+	return (
+		!isRangeIndexFilter(filter) &&
+		!isMatchIndexFilter(filter) &&
+		!isCompoundIndexFilter(filter) &&
+		!isStartsWithIndexFilter(filter) &&
+		(filter as any).order
+	);
+}