@mengine/storage 0.1.0

package/dist/auto-reconnect-0HwaV1v8.js

@@ -0,0 +1,250 @@
+import { EventBus, MANUALLY_STOP, Task } from "@mengine/utils";
+import { LoroDoc, VersionVector } from "loro-crdt";
+//#region src/lock.ts
+var SingletonLocker = class {
+	lockedResource = /* @__PURE__ */ new Map();
+	constructor() {}
+	async lock(domain, resource) {
+		const key = `${domain}:${resource}`;
+		let lock = this.lockedResource.get(key);
+		if (!lock) {
+			lock = new Lock();
+			this.lockedResource.set(key, lock);
+		}
+		await lock.acquire();
+		return lock;
+	}
+};
+var Lock = class {
+	inner = Promise.resolve();
+	release = () => {};
+	async acquire() {
+		let release = null;
+		const nextLock = new Promise((resolve) => {
+			release = resolve;
+		});
+		await this.inner;
+		this.inner = nextLock;
+		this.release = release;
+	}
+	[Symbol.asyncDispose]() {
+		this.release();
+		return Promise.resolve();
+	}
+};
+//#endregion
+//#region src/utils.ts
+function mergeLoroUpdates(updates) {
+	const doc = new LoroDoc();
+	for (const update of updates) doc.import(update);
+	return doc.export({ mode: "snapshot" });
+}
+function diffLoroUpdate(snapshot, knownVersion) {
+	const doc = new LoroDoc();
+	doc.import(snapshot);
+	return {
+		missing: doc.export({
+			mode: "update",
+			from: knownVersion ? VersionVector.decode(knownVersion) : void 0
+		}),
+		version: doc.version().encode()
+	};
+}
+//#endregion
+//#region src/base.ts
+var BaseDocStorage = class {
+	options;
+	get isReadonly() {
+		return this.options.readonlyMode ?? false;
+	}
+	storageType = "doc";
+	locker = new SingletonLocker();
+	event = new EventBus();
+	constructor(options) {
+		this.options = options;
+	}
+	async getDoc(docId) {
+		const lock = this.isReadonly ? void 0 : await this.lockDocForUpdate(docId);
+		try {
+			const snapshot = await this.getDocSnapshot(docId);
+			const updates = await this.getDocUpdates(docId);
+			if (updates.length) {
+				const newSnapshot = this.mergeUpdates(snapshot, updates);
+				if (!this.isReadonly) {
+					await this.setDocSnapshot(newSnapshot, snapshot);
+					await this.markUpdatesMerged(docId, updates);
+				}
+				return newSnapshot;
+			}
+			return snapshot;
+		} finally {
+			await lock?.[Symbol.asyncDispose]();
+		}
+	}
+	async getDocDiff(docId, knownVersion) {
+		const snapshot = await this.getDoc(docId);
+		if (!snapshot) return null;
+		const { missing, version } = this.diffUpdates(snapshot.data, knownVersion);
+		return {
+			docId,
+			missing,
+			version
+		};
+	}
+	subscribeDocUpdate(callback) {
+		return this.event.on("update", ({ update, origin }) => {
+			callback(update, origin);
+		});
+	}
+	/**
+	* Apply doc updates to a snapshot.
+	*/
+	mergeUpdates(snapshot, updates) {
+		return {
+			docId: snapshot?.docId ?? updates[0].docId,
+			data: (this.options.mergeUpdates ?? mergeLoroUpdates)([...snapshot ? [snapshot.data] : [], ...updates.map((update) => update.data)]),
+			createdAt: snapshot?.createdAt ?? /* @__PURE__ */ new Date(),
+			updatedAt: updates.at(-1)?.createdAt ?? /* @__PURE__ */ new Date()
+		};
+	}
+	diffUpdates(snapshot, knownVersion) {
+		return (this.options.diffUpdates ?? diffLoroUpdate)(snapshot, knownVersion);
+	}
+	async lockDocForUpdate(docId) {
+		return this.locker.lock(`Snapshotting`, docId);
+	}
+};
+//#endregion
+//#region src/connection/auto-reconnect.ts
+var AutoReconnectConnection = class {
+	event = new EventBus();
+	_inner = void 0;
+	_status = "idle";
+	_error = void 0;
+	retryDelay = 3e3;
+	connectingTimeout = 15e3;
+	refCount = 0;
+	connectingAbort;
+	reconnectingAbort;
+	constructor() {}
+	get shareId() {}
+	get maybeConnection() {
+		return this._inner;
+	}
+	get inner() {
+		if (this._inner === void 0) throw new Error(`Connection ${this.constructor.name} has not been established.`);
+		return this._inner;
+	}
+	set inner(inner) {
+		this._inner = inner;
+	}
+	get status() {
+		return this._status;
+	}
+	get error() {
+		return this._error;
+	}
+	set error(error) {
+		this.handleError(error);
+	}
+	setStatus(status, error) {
+		const shouldEmit = status !== this._status || error !== this._error;
+		this._status = status;
+		if (error || status === "connected") this._error = error;
+		if (shouldEmit) this.emitStatusChanged(status, this._error);
+	}
+	innerConnect() {
+		if (this.status !== "connecting") {
+			this.setStatus("connecting");
+			const connectingAbort = new AbortController();
+			this.connectingAbort = connectingAbort;
+			const signal = connectingAbort.signal;
+			const timeout = setTimeout(() => {
+				if (!signal.aborted) this.handleError(/* @__PURE__ */ new Error("connecting timeout"));
+			}, this.connectingTimeout);
+			this.doConnect(signal).then((value) => {
+				clearTimeout(timeout);
+				if (!signal.aborted) {
+					this._inner = value;
+					this.setStatus("connected");
+				} else try {
+					this.doDisconnect(value);
+				} catch (error) {
+					console.error("failed to disconnect", error);
+				}
+			}).catch((error) => {
+				if (!signal.aborted) {
+					clearTimeout(timeout);
+					console.error("failed to connect", error);
+					this.handleError(error);
+				}
+			});
+		}
+	}
+	innerDisconnect() {
+		this.connectingAbort?.abort(MANUALLY_STOP);
+		this.reconnectingAbort?.abort(MANUALLY_STOP);
+		try {
+			if (this._inner) this.doDisconnect(this._inner);
+		} catch (error) {
+			console.error("failed to disconnect", error);
+		}
+		this.reconnectingAbort = void 0;
+		this.connectingAbort = void 0;
+		this._inner = void 0;
+	}
+	handleError(reason) {
+		console.error("connection error, will reconnect", reason);
+		this.innerDisconnect();
+		if (this.status === "closed") return;
+		this.setStatus("error", reason);
+		this.reconnectingAbort = new AbortController();
+		const signal = this.reconnectingAbort.signal;
+		const timeout = setTimeout(() => {
+			if (!signal.aborted) this.innerConnect();
+		}, this.retryDelay);
+		signal.addEventListener("abort", () => {
+			clearTimeout(timeout);
+		});
+	}
+	connect() {
+		this.refCount++;
+		if (this.refCount === 1) this.innerConnect();
+	}
+	disconnect(force) {
+		if (force) this.refCount = 0;
+		else this.refCount = Math.max(this.refCount - 1, 0);
+		if (this.refCount === 0) {
+			this.innerDisconnect();
+			this.setStatus("closed");
+		}
+	}
+	waitForConnected() {
+		return new Task((resolve, _reject, scope) => {
+			if (this.status === "connected") {
+				resolve();
+				return;
+			}
+			const off = this.onStatusChanged((status) => {
+				if (status === "connected") {
+					resolve();
+					off();
+				}
+			});
+			scope.disposer.add(off);
+		});
+	}
+	onStatusChanged(cb) {
+		return this.event.on("statusChanged", (data) => {
+			cb(data.status, data.error);
+		});
+	}
+	emitStatusChanged = (status, error) => {
+		this.event.emit("statusChanged", {
+			status,
+			error
+		});
+	};
+};
+//#endregion
+export { BaseDocStorage as n, AutoReconnectConnection as t };

package/dist/base-B0LSXWTK.d.ts

@@ -0,0 +1,191 @@
+import { EventBus, Task } from "@mengine/utils";
+
+//#region src/connection/api.d.ts
+type ConnectionStatus = 'idle' | 'connecting' | 'connected' | 'error' | 'closed';
+interface Connection<T = any> {
+  readonly shareId?: string;
+  readonly status: ConnectionStatus;
+  readonly error?: Error;
+  readonly inner: T;
+  connect(): void;
+  disconnect(): void;
+  waitForConnected(): Task<void>;
+  onStatusChanged(cb: (status: ConnectionStatus, error?: Error) => void): () => void;
+}
+declare class DummyConnection implements Connection<undefined> {
+  readonly status: ConnectionStatus;
+  readonly inner: undefined;
+  connect(): void;
+  disconnect(): void;
+  waitForConnected(): Task<void>;
+  onStatusChanged(_cb: (status: ConnectionStatus, error?: Error) => void): () => void;
+}
+//#endregion
+//#region src/connection/auto-reconnect.d.ts
+declare abstract class AutoReconnectConnection<T = any> implements Connection<T> {
+  private readonly event;
+  private _inner;
+  private _status;
+  private _error;
+  retryDelay: number;
+  connectingTimeout: number;
+  private refCount;
+  private connectingAbort?;
+  private reconnectingAbort?;
+  constructor();
+  get shareId(): string | undefined;
+  get maybeConnection(): T | undefined;
+  get inner(): T;
+  private set inner(value);
+  get status(): ConnectionStatus;
+  get error(): Error | undefined;
+  protected set error(error: Error | undefined);
+  private setStatus;
+  protected abstract doConnect(signal?: AbortSignal): Promise<T>;
+  protected abstract doDisconnect(conn: T): void;
+  private innerConnect;
+  private innerDisconnect;
+  private handleError;
+  connect(): void;
+  disconnect(force?: boolean): void;
+  waitForConnected(): Task<void>;
+  onStatusChanged(cb: (status: ConnectionStatus, error?: Error) => void): () => void;
+  private readonly emitStatusChanged;
+}
+//#endregion
+//#region src/connection/shared-connection.d.ts
+declare function share<T extends Connection>(conn: T): T;
+//#endregion
+//#region src/api.d.ts
+interface DocSnapshotRecord {
+  docId: string;
+  data: Uint8Array;
+  createdAt: Date;
+  updatedAt: Date;
+}
+interface DocDiff {
+  docId: string;
+  version: Uint8Array;
+  missing: Uint8Array;
+}
+interface DocUpdate {
+  docId: string;
+  data: Uint8Array;
+}
+interface DocUpdateRecord extends DocUpdate {
+  seq: number;
+  createdAt: Date;
+}
+interface DocStorageOptions {
+  /**
+   * open as readonly mode.
+   */
+  readonlyMode?: boolean;
+  /**
+   * Merge updates into a single update(snapshot).
+   */
+  mergeUpdates?: (updates: Uint8Array[]) => Uint8Array;
+  /**
+   * Diff the snapshot with the given version.
+   */
+  diffUpdates?: (snapshot: Uint8Array, knownVersion?: Uint8Array) => {
+    missing: Uint8Array;
+    version: Uint8Array;
+  };
+}
+interface DocStorage {
+  readonly isReadonly: boolean;
+  readonly connection: Connection;
+  /**
+   * Get a doc record with latest binary.
+   */
+  getDoc(docId: string): Promise<DocSnapshotRecord | null>;
+  /**
+   * Get a diff with the given state vector.
+   */
+  getDocDiff(docId: string, knownVersion?: Uint8Array): Promise<DocDiff | null>;
+  /**
+   * Push updates into storage
+   */
+  pushDocUpdate(update: DocUpdate, origin: unknown): Promise<void>;
+  /**
+   * Delete a specific doc data with all snapshots and updates
+   */
+  deleteDoc(docId: string): Promise<void>;
+  /**
+   * Subscribe to doc updates
+   */
+  subscribeDocUpdate(callback: (update: DocUpdate, origin: unknown) => void): () => void;
+}
+//#endregion
+//#region src/lock.d.ts
+interface Locker {
+  lock(domain: string, resource: string): Promise<AsyncDisposable>;
+}
+//#endregion
+//#region src/base.d.ts
+type DocStorageEvents = {
+  update: [{
+    update: DocUpdate;
+    origin: unknown;
+  }];
+};
+declare abstract class BaseDocStorage<Opts = {}> implements DocStorage {
+  protected readonly options: Opts & DocStorageOptions;
+  get isReadonly(): boolean;
+  abstract readonly connection: Connection;
+  readonly storageType = "doc";
+  protected locker: Locker;
+  protected readonly event: EventBus<DocStorageEvents>;
+  constructor(options: Opts & DocStorageOptions);
+  getDoc(docId: string): Promise<DocSnapshotRecord | null>;
+  getDocDiff(docId: string, knownVersion?: Uint8Array): Promise<DocDiff | null>;
+  abstract pushDocUpdate(update: DocUpdate, origin: unknown): Promise<void>;
+  abstract deleteDoc(docId: string): Promise<void>;
+  subscribeDocUpdate(callback: (update: DocUpdate, origin: unknown) => void): () => void;
+  /**
+   * Get a doc snapshot from storage
+   */
+  protected abstract getDocSnapshot(docId: string): Promise<DocSnapshotRecord | null>;
+  /**
+   * Set the doc snapshot into storage
+   *
+   * @safety
+   * be careful when implementing this method.
+   *
+   * It might be called with outdated snapshot when running in multi-thread environment.
+   *
+   * A common solution is update the snapshot record is DB only when the coming one's timestamp is newer.
+   *
+   * @example
+   * ```ts
+   * await using _lock = await this.lockDocForUpdate(docId);
+   * // set snapshot
+   *
+   * ```
+   */
+  protected abstract setDocSnapshot(snapshot: DocSnapshotRecord, prevSnapshot: DocSnapshotRecord | null): Promise<boolean>;
+  /**
+   * Get all updates of a doc that haven't been merged into snapshot.
+   *
+   * Updates queue design exists for a performance concern:
+   * A huge amount of write time will be saved if we don't merge updates into snapshot immediately.
+   * Updates will be merged into snapshot when the latest doc is requested.
+   */
+  protected abstract getDocUpdates(docId: string): Promise<DocUpdateRecord[]>;
+  /**
+   * Mark updates as merged into snapshot.
+   */
+  protected abstract markUpdatesMerged(docId: string, updates: DocUpdateRecord[]): Promise<number>;
+  /**
+   * Apply doc updates to a snapshot.
+   */
+  protected mergeUpdates(snapshot: DocSnapshotRecord | null, updates: DocUpdateRecord[]): DocSnapshotRecord;
+  protected diffUpdates(snapshot: Uint8Array, knownVersion?: Uint8Array): {
+    missing: Uint8Array;
+    version: Uint8Array;
+  };
+  protected lockDocForUpdate(docId: string): Promise<AsyncDisposable>;
+}
+//#endregion
+export { DocStorage as a, DocUpdateRecord as c, Connection as d, ConnectionStatus as f, DocSnapshotRecord as i, share as l, Locker as n, DocStorageOptions as o, DummyConnection as p, DocDiff as r, DocUpdate as s, BaseDocStorage as t, AutoReconnectConnection as u };

package/dist/idb.d.ts

@@ -0,0 +1,119 @@
+import { c as DocUpdateRecord, i as DocSnapshotRecord, n as Locker, s as DocUpdate, t as BaseDocStorage, u as AutoReconnectConnection } from "./base-B0LSXWTK.js";
+import { DBSchema, IDBPDatabase } from "idb";
+
+//#region src/impls/idb/schema.d.ts
+/**
+IndexedDB
+  > DB(medeo-engine)
+     > Table(Snapshots)
+     > Table(Updates)
+
+Table(Snapshots)
+| docId | data | seq | createdAt | updatedAt |
+|-------|------|-----|-----------|-----------|
+|  str  | bin  | num |   Date    |   Date    |
+
+Table(Updates)
+| id | docId | data | createdAt |
+|----|-------|------|-----------|
+|auto|  str  | bin  |   Date    |
+
+Table(Locks)
+| key | lock |
+|-----|------|
+| str | Date |
+ */
+interface DocStorageSchema extends DBSchema {
+  snapshots: {
+    key: string;
+    value: {
+      docId: string;
+      data: Uint8Array;
+      createdAt: Date;
+      updatedAt: Date;
+    };
+  };
+  updates: {
+    key: [string, number];
+    value: {
+      docId: string;
+      seq: number;
+      data: Uint8Array;
+      createdAt: Date;
+    };
+    indexes: {
+      docId: string;
+    };
+  };
+  sequences: {
+    key: string;
+    value: {
+      docId: string;
+      seq: number;
+    };
+  };
+  locks: {
+    key: string;
+    value: {
+      key: string;
+      lockedAt: Date;
+    };
+  };
+}
+//#endregion
+//#region src/impls/idb/connection.d.ts
+declare class IDBConnection extends AutoReconnectConnection<{
+  db: IDBPDatabase<DocStorageSchema>;
+  channel: BroadcastChannel;
+}> {
+  readonly dbName: string;
+  constructor(dbName: string);
+  get shareId(): string;
+  doConnect(): Promise<{
+    db: IDBPDatabase<DocStorageSchema>;
+    channel: BroadcastChannel;
+  }>;
+  doDisconnect(db: {
+    db: IDBPDatabase<DocStorageSchema>;
+    channel: BroadcastChannel;
+  }): void;
+  handleVersionChange: (e: IDBVersionChangeEvent) => void;
+}
+//#endregion
+//#region src/impls/idb/lock.d.ts
+declare class IndexedDBLocker implements Locker {
+  private readonly dbConnection;
+  get db(): import("idb").IDBPDatabase<DocStorageSchema>;
+  get channel(): BroadcastChannel;
+  constructor(dbConnection: IDBConnection);
+  lock(domain: string, resource: string): Promise<{
+    [Symbol.asyncDispose]: () => Promise<void>;
+  }>;
+}
+//#endregion
+//#region src/impls/idb/index.d.ts
+interface ChannelMessage {
+  type: 'update';
+  update: DocUpdate;
+  origin?: unknown;
+}
+declare class IndexedDBDocStorage extends BaseDocStorage<{
+  dbName: string;
+}> {
+  static readonly identifier = "IndexedDBDocStorage";
+  readonly connection: IDBConnection;
+  protected readonly locker: IndexedDBLocker;
+  get db(): import("idb").IDBPDatabase<DocStorageSchema>;
+  get channel(): BroadcastChannel;
+  pushDocUpdate(update: DocUpdate, origin: unknown): Promise<void>;
+  protected getDocSnapshot(docId: string): Promise<DocSnapshotRecord | null>;
+  deleteDoc(docId: string): Promise<void>;
+  protected setDocSnapshot(snapshot: DocSnapshotRecord): Promise<boolean>;
+  protected getDocUpdates(docId: string): Promise<DocUpdateRecord[]>;
+  protected markUpdatesMerged(docId: string, updates: DocUpdateRecord[]): Promise<number>;
+  private docUpdateListener;
+  subscribeDocUpdate(callback: (update: DocUpdate, origin: unknown) => void): () => void;
+  handleChannelMessage: (event: MessageEvent<ChannelMessage>) => void;
+}
+//#endregion
+export { IndexedDBDocStorage };