@peerbit/any-store 2.0.2 → 2.1.0-55cebfe

package/src/opfs-worker.ts

@@ -1,347 +0,0 @@
-import { deserialize, serialize } from "@dao-xyz/borsh";
-import { AnyStore } from "./interface.js";
-import * as memory from "./opfs-worker-messages.js";
-import { fromBase64URL, toBase64URL, ready } from "@peerbit/crypto";
-import { BinaryReader, BinaryWriter } from "@dao-xyz/borsh";
-import { waitForResolved } from "@peerbit/time";
-
-const directory = location.hash.split("#")?.[1];
-
-const encodeName = (name: string): string => {
-	// since "/" and perhaps other characters might not be allowed we do encode
-	const writer = new BinaryWriter();
-	writer.string(name);
-	return toBase64URL(writer.finalize());
-};
-
-const decodeName = (name: string): string => {
-	// since "/" and perhaps other characters might not be allowed we do encode
-	const writer = new BinaryReader(fromBase64URL(name));
-	return writer.string();
-};
-
-const waitForSyncAcccess = async (
-	fileHandle: FileSystemFileHandle
-): Promise<FileSystemSyncAccessHandle> => {
-	try {
-		const handle = await fileHandle.createSyncAccessHandle();
-		return handle;
-	} catch (error) {
-		const handle = await waitForResolved(() =>
-			fileHandle.createSyncAccessHandle()
-		);
-		if (!handle) {
-			throw error;
-		}
-		return handle;
-	}
-};
-
-const createWriteHandle = async (fileHandle: FileSystemFileHandle) => {
-	//  In Chrome on GET DOMException: Failed to execute 'createWritable' on 'FileSystemFileHandle': Failed to create swap file
-	// hence below is not used for now
-	/*
-	if (fileHandle.createWritable != null) {
-		return fileHandle.createWritable({ keepExistingData: false });
-	} */
-	return waitForSyncAcccess(fileHandle);
-};
-
-export class OPFSStoreWorker {
-	private _levels: Map<string, AnyStore>;
-	private _rootStore: AnyStore;
-
-	private _memoryIterator: Map<
-		string,
-		AsyncIterator<[string, ArrayBuffer], void, void>
-	>;
-
-	constructor() {
-		const postMessageFn = postMessage;
-		this._memoryIterator = new Map();
-		this._levels = new Map();
-		let isOpen = false;
-
-		const createMemory = (
-			root?: FileSystemDirectoryHandle,
-			levels: string[] = []
-		): AnyStore => {
-			let m: FileSystemDirectoryHandle = root!;
-			let sizeCache: number = 0;
-			const sizeMap: Map<string, number> = new Map(); // files size per key
-
-			const calculateSize = async () => {
-				sizeCache = 0;
-				for await (const value of m.values()) {
-					if (value.kind === "file") {
-						try {
-							const handle = await waitForSyncAcccess(value);
-							const handleSize = handle.getSize();
-							sizeMap.set(value.name, handleSize);
-							sizeCache += handleSize;
-							handle.close();
-						} catch (error) {
-							// TODO better error handling.
-							// most commonly error can be thrown when we are invoking getSize and files does not exist (anymore)
-						}
-					}
-				}
-			};
-			// 'open' | 'closed' is just a virtual thing since OPFS is always open as soone as we get the FileSystemDirectoryHandle
-			// TODO remove status? or assume not storage adapters can be closed?
-			const open = async () => {
-				await ready;
-
-				if (!m) {
-					m = await navigator.storage.getDirectory();
-					if (directory) {
-						m = await m.getDirectoryHandle(encodeName(directory), {
-							create: true
-						});
-					}
-					await calculateSize();
-				}
-			};
-			return {
-				clear: async () => {
-					for await (const key of m.keys()) {
-						m.removeEntry(key, { recursive: true });
-					}
-					sizeCache = 0;
-					sizeMap.clear();
-				},
-
-				del: async (key: string) => {
-					const encodedKey = encodeName(key);
-					try {
-						await m.removeEntry(encodedKey, { recursive: true });
-					} catch (error) {
-						if (error instanceof DOMException) {
-							return;
-						} else {
-							throw error;
-						}
-					}
-					const prevSize = sizeMap.get(encodedKey);
-					if (prevSize != null) {
-						sizeCache -= prevSize;
-						sizeMap.delete(encodedKey);
-					}
-				},
-
-				get: async (key: string) => {
-					try {
-						const fileHandle = await m.getFileHandle(encodeName(key));
-						const file = await fileHandle.getFile();
-						const buffer = await file.arrayBuffer();
-						return new Uint8Array(buffer);
-					} catch (error) {
-						if (
-							error instanceof DOMException &&
-							error.name === "NotFoundError"
-						) {
-							return undefined;
-						} else if (error) {
-							return undefined;
-						} else {
-							throw error;
-						}
-					}
-				},
-				put: async (key: string, value: Uint8Array) => {
-					const encodedKey = encodeName(key);
-
-					const fileHandle = await m.getFileHandle(encodedKey, {
-						create: true
-					});
-					const writeFileHandle = await createWriteHandle(fileHandle);
-					writeFileHandle.write(value);
-					writeFileHandle.flush();
-					writeFileHandle.close();
-
-					const prevSize = sizeMap.get(encodedKey);
-					if (prevSize) {
-						sizeCache -= prevSize;
-					}
-					sizeCache += value.byteLength;
-					sizeMap.set(encodedKey, value.byteLength);
-				},
-
-				size: async () => {
-					return sizeCache;
-				},
-				status: () => (isOpen ? "open" : "closed"),
-
-				sublevel: async (name) => {
-					const encodedName = encodeName(name);
-					const fileHandle = await m.getDirectoryHandle(encodedName, {
-						create: true
-					});
-					const sublevel = [...levels, name];
-					const subMemory = createMemory(fileHandle, sublevel);
-					this._levels.set(memory.levelKey(sublevel), subMemory);
-					await subMemory.open();
-					return subMemory;
-				},
-
-				async *iterator(): AsyncGenerator<[string, Uint8Array], void, void> {
-					for await (const v of m.values()) {
-						if (v.kind == "file") {
-							yield [
-								decodeName(v.name),
-								new Uint8Array(await (await v.getFile()).arrayBuffer())
-							];
-						}
-					}
-				},
-				close: async () => {
-					sizeCache = undefined as any; // TODO types
-					isOpen = false;
-					this._memoryIterator.clear();
-				},
-				open
-			};
-		};
-
-		this._rootStore = createMemory();
-
-		self.addEventListener("message", async (ev) => {
-			const message = deserialize(ev["data"], memory.MemoryRequest);
-			try {
-				if (message instanceof memory.MemoryMessage) {
-					if (message instanceof memory.REQ_Open) {
-						if (!isOpen) {
-							await this._rootStore.open();
-							isOpen = true;
-						}
-						await this.respond(
-							message,
-							new memory.RESP_Open({ level: message.level }),
-							postMessageFn
-						);
-						return;
-					}
-
-					const m =
-						message.level.length === 0
-							? this._rootStore
-							: this._levels.get(memory.levelKey(message.level));
-					if (!m) {
-						throw new Error("Recieved memory message for an undefined level");
-					} else if (message instanceof memory.REQ_Clear) {
-						await m.clear();
-						await this.respond(
-							message,
-							new memory.RESP_Clear({ level: message.level }),
-							postMessageFn
-						);
-					} else if (message instanceof memory.REQ_Close) {
-						await m.close();
-						await this.respond(
-							message,
-							new memory.RESP_Close({ level: message.level }),
-							postMessageFn
-						);
-					} else if (message instanceof memory.REQ_Del) {
-						await m.del(message.key);
-						await this.respond(
-							message,
-							new memory.RESP_Del({ level: message.level }),
-							postMessageFn
-						);
-					} else if (message instanceof memory.REQ_Iterator_Next) {
-						let iterator = this._memoryIterator.get(message.id);
-						if (!iterator) {
-							iterator = m.iterator()[Symbol.asyncIterator]();
-							this._memoryIterator.set(message.id, iterator);
-						}
-						const next = await iterator.next();
-						await this.respond(
-							message,
-							new memory.RESP_Iterator_Next({
-								keys: next.done ? [] : [next.value[0]],
-								values: next.done ? [] : [new Uint8Array(next.value[1])],
-								level: message.level
-							}),
-							postMessageFn
-						);
-						if (next.done) {
-							this._memoryIterator.delete(message.id);
-						}
-					} else if (message instanceof memory.REQ_Iterator_Stop) {
-						this._memoryIterator.delete(message.id);
-						await this.respond(
-							message,
-							new memory.RESP_Iterator_Stop({ level: message.level }),
-							postMessageFn
-						);
-					} else if (message instanceof memory.REQ_Get) {
-						const value = await m.get(message.key);
-						await this.respond(
-							message,
-							new memory.RESP_Get({
-								bytes: value ? new Uint8Array(value) : undefined,
-								level: message.level
-							}),
-							postMessageFn
-						);
-					} else if (message instanceof memory.REQ_Put) {
-						await m.put(message.key, message.bytes);
-						await this.respond(
-							message,
-							new memory.RESP_Put({ level: message.level }),
-							postMessageFn
-						);
-					} else if (message instanceof memory.REQ_Size) {
-						await this.respond(
-							message,
-							new memory.RESP_Size({
-								size: await m.size(),
-								level: message.level
-							}),
-							postMessageFn
-						);
-					} else if (message instanceof memory.REQ_Status) {
-						await this.respond(
-							message,
-							new memory.RESP_Status({
-								status: await m.status(),
-								level: message.level
-							}),
-							postMessageFn
-						);
-					} else if (message instanceof memory.REQ_Sublevel) {
-						await m.sublevel(message.name);
-
-						await this.respond(
-							message,
-							new memory.RESP_Sublevel({ level: message.level }),
-							postMessageFn
-						);
-					}
-				}
-			} catch (error) {
-				await this.respond(
-					message,
-					new memory.RESP_Error({
-						error: error?.["message"] || error?.constructor.name,
-						level: message["level"] || []
-					}),
-					postMessageFn
-				);
-			}
-		});
-	}
-
-	async respond(
-		request: memory.MemoryRequest,
-		response: memory.MemoryRequest,
-		postMessageFn = postMessage
-	) {
-		response.messageId = request.messageId;
-		const bytes = serialize(response);
-		postMessageFn(bytes, { transfer: [bytes.buffer] });
-	}
-}
-
-new OPFSStoreWorker();

package/src/opfs.ts

@@ -1,205 +0,0 @@
-import { AnyStore, MaybePromise } from "./interface.js";
-import * as memory from "./opfs-worker-messages.js";
-import { v4 as uuid } from "uuid";
-import { serialize, deserialize } from "@dao-xyz/borsh";
-import workerUrl from "./opfs-worker?worker&url";
-
-function memoryIterator(
-	client: {
-		request<T extends memory.MemoryRequest>(
-			request: memory.MemoryRequest
-		): Promise<T>;
-	},
-	level: string[]
-): {
-	[Symbol.asyncIterator]: () => AsyncIterator<[string, Uint8Array], void, void>;
-} {
-	return {
-		[Symbol.asyncIterator]() {
-			const iteratorId = uuid();
-			return {
-				next: async () => {
-					const resp = await client.request<memory.RESP_Iterator_Next>(
-						new memory.REQ_Iterator_Next({ id: iteratorId, level })
-					);
-					if (resp.keys.length > 1) {
-						throw new Error("Unsupported iteration response");
-					}
-					// Will only have 0 or 1 element for now
-					for (let i = 0; i < resp.keys.length; i++) {
-						return {
-							done: false,
-							value: [resp.keys[i], resp.values[i]] as [string, Uint8Array]
-						} as { done: false; value: [string, Uint8Array] };
-					}
-					return { done: true, value: undefined } as {
-						done: true;
-						value: undefined;
-					};
-				},
-				async return() {
-					await client.request<memory.RESP_Iterator_Next>(
-						new memory.REQ_Iterator_Stop({ id: iteratorId, level })
-					);
-					return { done: true, value: undefined } as {
-						done: true;
-						value: undefined;
-					};
-				}
-			};
-		}
-	};
-}
-
-const createWorker = (directory) => {
-	const workerURL = workerUrl + "#" + directory;
-	return new Worker(workerURL, { type: "module" });
-};
-
-export class OPFSStore implements AnyStore {
-	worker: Worker;
-	levelMap: Map<string, AnyStore>;
-	root: AnyStore;
-
-	private _responseCallbacks: Map<
-		string,
-		{ fn: (message: memory.MemoryRequest) => any; once: boolean }
-	> = new Map();
-
-	private _createStorage: (level: string[]) => AnyStore;
-	constructor(readonly directory?: string) {
-		this.levelMap = new Map();
-		this._createStorage = (level: string[] = []): AnyStore => {
-			return {
-				clear: async () => {
-					await this.request<memory.RESP_Clear>(
-						new memory.REQ_Clear({ level })
-					);
-				},
-				del: async (key) => {
-					await this.request<memory.RESP_Del>(
-						new memory.REQ_Del({ level, key })
-					);
-				},
-				get: async (key) => {
-					return (
-						await this.request<memory.RESP_Get>(
-							new memory.REQ_Get({ level, key })
-						)
-					).bytes;
-				},
-				put: async (key, value) => {
-					await this.request<memory.RESP_Put>(
-						new memory.REQ_Put({ level, key, bytes: value })
-					);
-				},
-				status: async () =>
-					(
-						await this.request<memory.RESP_Status>(
-							new memory.REQ_Status({ level })
-						)
-					).status,
-				sublevel: async (name) => {
-					await this.request(new memory.REQ_Sublevel({ level, name }));
-					const newLevels = [...level, name];
-					const sublevel = this._createStorage(newLevels);
-					this.levelMap.set(memory.levelKey(newLevels), sublevel);
-					return sublevel;
-				},
-
-				iterator: () => memoryIterator(this, level),
-				close: async () => {
-					await this.request<memory.RESP_Close>(
-						new memory.REQ_Close({ level })
-					);
-					/*     this.levelMap.delete(memory.levelKey(level)); */
-				},
-				open: async () => {
-					await this.request<memory.RESP_Open>(new memory.REQ_Open({ level }));
-				},
-
-				size: async () => {
-					const size = await this.request<memory.RESP_Size>(
-						new memory.REQ_Size({ level })
-					);
-					return size.size;
-				}
-			};
-		};
-	}
-	status() {
-		return this.worker ? this.root.status() : "closed";
-	}
-	async close(): Promise<void> {
-		this.worker.terminate();
-		this.worker = undefined!;
-		this._responseCallbacks.clear();
-		this.levelMap.clear();
-	}
-	async open(): Promise<void> {
-		if (!this.worker) {
-			if (
-				!globalThis["__playwright_test__"] &&
-				(await navigator.storage.persist()) === false
-			) {
-				throw new Error("OPFS not allowed to persist data");
-			}
-			this.worker = createWorker(this.directory);
-			this.root = this._createStorage([]);
-			this.worker.addEventListener("message", async (ev) => {
-				const message = deserialize(ev.data, memory.MemoryMessage);
-				this._responseCallbacks.get(message.messageId)!.fn(message);
-			});
-			await this.root.open();
-		}
-	}
-	get(key: string): MaybePromise<Uint8Array | undefined> {
-		return this.root.get(key);
-	}
-	put(key: string, value: Uint8Array) {
-		return this.root.put(key, value);
-	}
-	del(key: any): MaybePromise<void> {
-		return this.root.del(key);
-	}
-	sublevel(name: string): AnyStore | MaybePromise<AnyStore> {
-		return this.root.sublevel(name);
-	}
-	iterator(): {
-		[Symbol.asyncIterator]: () => AsyncIterator<
-			[string, Uint8Array],
-			void,
-			void
-		>;
-	} {
-		return this.root.iterator();
-	}
-	clear(): MaybePromise<void> {
-		return this.root.clear();
-	}
-
-	size(): MaybePromise<number> {
-		return this.root.size();
-	}
-
-	async request<T extends memory.MemoryRequest>(
-		request: memory.MemoryRequest
-	): Promise<T> {
-		return new Promise<T>((resolve, reject) => {
-			const onResponse = (message: memory.MemoryRequest) => {
-				this._responseCallbacks.delete(request.messageId);
-				if (message instanceof memory.RESP_Error) {
-					reject(new Error(message.error));
-				} else {
-					resolve(message as T);
-				}
-			};
-			this._responseCallbacks.set(request.messageId, {
-				fn: onResponse,
-				once: true
-			});
-			const bytes = serialize(request);
-			this.worker.postMessage(bytes, [bytes.buffer]);
-		});
-	}
-}