@peerbit/any-store-opfs 2.0.2-0691c73

package/src/opfs-store.ts

@@ -0,0 +1,199 @@
+import { type AnyStore, type MaybePromise } from "@peerbit/any-store-interface";
+import * as memory from "@peerbit/any-store-interface/messages";
+import { createWorker } from "./create.js"
+import { v4 as uuid } from "uuid";
+import { serialize, deserialize } from "@dao-xyz/borsh";
+function memoryIterator(
+	client: {
+		request<T extends memory.MemoryRequest>(
+			request: memory.MemoryRequest
+		): Promise<T>;
+	},
+	level: string[]
+): {
+	[Symbol.asyncIterator]: () => AsyncIterator<[string, Uint8Array], void, void>;
+} {
+	return {
+		[Symbol.asyncIterator]() {
+			const iteratorId = uuid();
+			return {
+				next: async () => {
+					const resp = await client.request<memory.RESP_Iterator_Next>(
+						new memory.REQ_Iterator_Next({ id: iteratorId, level })
+					);
+					if (resp.keys.length > 1) {
+						throw new Error("Unsupported iteration response");
+					}
+					// Will only have 0 or 1 element for now
+					for (let i = 0; i < resp.keys.length; i++) {
+						return {
+							done: false,
+							value: [resp.keys[i], resp.values[i]] as [string, Uint8Array]
+						} as { done: false; value: [string, Uint8Array] };
+					}
+					return { done: true, value: undefined } as {
+						done: true;
+						value: undefined;
+					};
+				},
+				async return() {
+					await client.request<memory.RESP_Iterator_Next>(
+						new memory.REQ_Iterator_Stop({ id: iteratorId, level })
+					);
+					return { done: true, value: undefined } as {
+						done: true;
+						value: undefined;
+					};
+				}
+			};
+		}
+	};
+}
+
+export class OPFSStore implements AnyStore {
+	worker: Worker;
+	levelMap: Map<string, AnyStore>;
+	root: AnyStore;
+
+	private _responseCallbacks: Map<
+		string,
+		{ fn: (message: memory.MemoryRequest) => any; once: boolean }
+	> = new Map();
+
+	private _createStorage: (level: string[]) => AnyStore;
+	constructor(readonly directory?: string) {
+		this.levelMap = new Map();
+		this._createStorage = (level: string[] = []): AnyStore => {
+			return {
+				clear: async () => {
+					await this.request<memory.RESP_Clear>(
+						new memory.REQ_Clear({ level })
+					);
+				},
+				del: async (key) => {
+					await this.request<memory.RESP_Del>(
+						new memory.REQ_Del({ level, key })
+					);
+				},
+				get: async (key) => {
+					return (
+						await this.request<memory.RESP_Get>(
+							new memory.REQ_Get({ level, key })
+						)
+					).bytes;
+				},
+				put: async (key, value) => {
+					await this.request<memory.RESP_Put>(
+						new memory.REQ_Put({ level, key, bytes: value })
+					);
+				},
+				status: async () =>
+					(
+						await this.request<memory.RESP_Status>(
+							new memory.REQ_Status({ level })
+						)
+					).status,
+				sublevel: async (name) => {
+					await this.request(new memory.REQ_Sublevel({ level, name }));
+					const newLevels = [...level, name];
+					const sublevel = this._createStorage(newLevels);
+					this.levelMap.set(memory.levelKey(newLevels), sublevel);
+					return sublevel;
+				},
+
+				iterator: () => memoryIterator(this, level),
+				close: async () => {
+					await this.request<memory.RESP_Close>(
+						new memory.REQ_Close({ level })
+					);
+					/*     this.levelMap.delete(memory.levelKey(level)); */
+				},
+				open: async () => {
+					await this.request<memory.RESP_Open>(new memory.REQ_Open({ level }));
+				},
+
+				size: async () => {
+					const size = await this.request<memory.RESP_Size>(
+						new memory.REQ_Size({ level })
+					);
+					return size.size;
+				}
+			};
+		};
+	}
+	status() {
+		return this.worker ? this.root.status() : "closed";
+	}
+	async close(): Promise<void> {
+		this.worker.terminate();
+		this.worker = undefined!;
+		this._responseCallbacks.clear();
+		this.levelMap.clear();
+	}
+	async open(): Promise<void> {
+		if (!this.worker) {
+			/* if (
+				!(globalThis as any)["__playwright_test__"] &&
+				(await navigator.storage.persist()) === false
+			) {
+				throw new Error("OPFS not allowed to persist data");
+			} */
+			this.worker = createWorker(this.directory);
+			this.root = this._createStorage([]);
+			this.worker.addEventListener("message", async (ev) => {
+				const message = deserialize(ev.data, memory.MemoryMessage);
+				this._responseCallbacks.get(message.messageId)!.fn(message);
+			});
+			await this.root.open();
+		}
+	}
+	get(key: string): MaybePromise<Uint8Array | undefined> {
+		return this.root.get(key);
+	}
+	put(key: string, value: Uint8Array) {
+		return this.root.put(key, value);
+	}
+	del(key: any): MaybePromise<void> {
+		return this.root.del(key);
+	}
+	sublevel(name: string): AnyStore | MaybePromise<AnyStore> {
+		return this.root.sublevel(name);
+	}
+	iterator(): {
+		[Symbol.asyncIterator]: () => AsyncIterator<
+			[string, Uint8Array],
+			void,
+			void
+		>;
+	} {
+		return this.root.iterator();
+	}
+	clear(): MaybePromise<void> {
+		return this.root.clear();
+	}
+
+	size(): MaybePromise<number> {
+		return this.root.size();
+	}
+
+	async request<T extends memory.MemoryRequest>(
+		request: memory.MemoryRequest
+	): Promise<T> {
+		return new Promise<T>((resolve, reject) => {
+			const onResponse = (message: memory.MemoryRequest) => {
+				this._responseCallbacks.delete(request.messageId);
+				if (message instanceof memory.RESP_Error) {
+					reject(new Error(message.error));
+				} else {
+					resolve(message as T);
+				}
+			};
+			this._responseCallbacks.set(request.messageId, {
+				fn: onResponse,
+				once: true
+			});
+			const bytes = serialize(request);
+			this.worker.postMessage(bytes, [bytes.buffer]);
+		});
+	}
+}

package/src/worker.ts

@@ -0,0 +1,350 @@
+// TODO make bundle by removing unecessary dependencies
+import { deserialize, serialize } from "@dao-xyz/borsh";
+import { type AnyStore } from "@peerbit/any-store-interface"
+import * as memory from "@peerbit/any-store-interface/messages";
+import { fromBase64URL, toBase64URL, ready } from "@peerbit/crypto";
+import { BinaryReader, BinaryWriter } from "@dao-xyz/borsh";
+import { waitForResolved } from "@peerbit/time";
+
+const directory = location.hash.split("#")?.[1];
+
+const encodeName = (name: string): string => {
+	// since "/" and perhaps other characters might not be allowed we do encode
+	const writer = new BinaryWriter();
+	writer.string(name);
+	return toBase64URL(writer.finalize());
+};
+
+const decodeName = (name: string): string => {
+	// since "/" and perhaps other characters might not be allowed we do encode
+	const writer = new BinaryReader(fromBase64URL(name));
+	return writer.string();
+};
+
+const waitForSyncAcccess = async (
+	fileHandle: FileSystemFileHandle
+): Promise<FileSystemSyncAccessHandle> => {
+	try {
+		const handle = await fileHandle.createSyncAccessHandle();
+		return handle;
+	} catch (error) {
+		const handle = await waitForResolved(() =>
+			fileHandle.createSyncAccessHandle()
+		);
+		if (!handle) {
+			throw error;
+		}
+		return handle;
+	}
+};
+
+const createWriteHandle = async (fileHandle: FileSystemFileHandle) => {
+	//  In Chrome on GET DOMException: Failed to execute 'createWritable' on 'FileSystemFileHandle': Failed to create swap file
+	// hence below is not used for now
+	/*
+	if (fileHandle.createWritable != null) {
+		return fileHandle.createWritable({ keepExistingData: false });
+	} */
+	return waitForSyncAcccess(fileHandle);
+};
+
+export class OPFSStoreWorker {
+	private _levels: Map<string, AnyStore>;
+	private _rootStore: AnyStore;
+
+	private _memoryIterator: Map<
+		string,
+		AsyncIterator<[string, ArrayBuffer], void, void>
+	>;
+
+	constructor() {
+		const postMessageFn = postMessage;
+		this._memoryIterator = new Map();
+		this._levels = new Map();
+
+		const createMemory = (
+			root?: FileSystemDirectoryHandle,
+			levels: string[] = []
+		): AnyStore => {
+			let isOpen = false;
+
+			let m: FileSystemDirectoryHandle = root!;
+			let sizeCache: number = 0;
+			const sizeMap: Map<string, number> = new Map(); // files size per key
+
+			const calculateSize = async () => {
+				sizeCache = 0;
+				for await (const value of m.values()) {
+					if (value.kind === "file") {
+						try {
+							const handle = await waitForSyncAcccess(value);
+							const handleSize = handle.getSize();
+							sizeMap.set(value.name, handleSize);
+							sizeCache += handleSize;
+							handle.close();
+						} catch (error) {
+							// TODO better error handling.
+							// most commonly error can be thrown when we are invoking getSize and files does not exist (anymore)
+						}
+					}
+				}
+			};
+			// 'open' | 'closed' is just a virtual thing since OPFS is always open as soone as we get the FileSystemDirectoryHandle
+			// TODO remove status? or assume not storage adapters can be closed?
+			const open = async () => {
+				await ready;
+
+				if (!m) {
+					m = await navigator.storage.getDirectory();
+					if (directory) {
+						m = await m.getDirectoryHandle(encodeName(directory), {
+							create: true
+						});
+					}
+					await calculateSize();
+				};
+				isOpen = true;
+			};
+			return {
+				clear: async () => {
+					for await (const key of m.keys()) {
+						m.removeEntry(key, { recursive: true });
+					}
+					sizeCache = 0;
+					sizeMap.clear();
+				},
+
+				del: async (key: string) => {
+					const encodedKey = encodeName(key);
+					try {
+						await m.removeEntry(encodedKey, { recursive: true });
+					} catch (error) {
+						if (error instanceof DOMException) {
+							return;
+						} else {
+							throw error;
+						}
+					}
+					const prevSize = sizeMap.get(encodedKey);
+					if (prevSize != null) {
+						sizeCache -= prevSize;
+						sizeMap.delete(encodedKey);
+					}
+				},
+
+				get: async (key: string) => {
+					try {
+						const fileHandle = await m.getFileHandle(encodeName(key));
+						const file = await fileHandle.getFile();
+						const buffer = await file.arrayBuffer();
+						return new Uint8Array(buffer);
+					} catch (error) {
+						if (
+							error instanceof DOMException &&
+							error.name === "NotFoundError"
+						) {
+							return undefined;
+						} else if (error) {
+							return undefined;
+						} else {
+							throw error;
+						}
+					}
+				},
+				put: async (key: string, value: Uint8Array) => {
+					const encodedKey = encodeName(key);
+
+					const fileHandle = await m.getFileHandle(encodedKey, {
+						create: true
+					});
+					const writeFileHandle = await createWriteHandle(fileHandle);
+					writeFileHandle.write(value);
+					writeFileHandle.flush();
+					writeFileHandle.close();
+
+					const prevSize = sizeMap.get(encodedKey);
+					if (prevSize) {
+						sizeCache -= prevSize;
+					}
+					sizeCache += value.byteLength;
+					sizeMap.set(encodedKey, value.byteLength);
+				},
+
+				size: async () => {
+					return sizeCache;
+				},
+				status: () => (isOpen ? "open" : "closed"),
+
+				sublevel: async (name) => {
+					const encodedName = encodeName(name);
+					const fileHandle = await m.getDirectoryHandle(encodedName, {
+						create: true
+					});
+					const sublevel = [...levels, name];
+					const subMemory = createMemory(fileHandle, sublevel);
+					this._levels.set(memory.levelKey(sublevel), subMemory);
+					await subMemory.open();
+					return subMemory;
+				},
+
+				async *iterator(): AsyncGenerator<[string, Uint8Array], void, void> {
+					for await (const v of m.values()) {
+						if (v.kind == "file") {
+							yield [
+								decodeName(v.name),
+								new Uint8Array(await (await v.getFile()).arrayBuffer())
+							];
+						}
+					}
+				},
+				close: async () => {
+					sizeCache = undefined as any; // TODO types
+					isOpen = false;
+					this._memoryIterator.clear();
+				},
+				open
+			};
+		};
+
+		this._rootStore = createMemory();
+
+		self.addEventListener("message", async (ev) => {
+			const message = deserialize(ev["data"], memory.MemoryRequest);
+			try {
+				if (message instanceof memory.MemoryMessage) {
+					if (message instanceof memory.REQ_Open) {
+						if (await this._rootStore.status() === "closed") {
+							await this._rootStore.open();
+						}
+						await this.respond(
+							message,
+							new memory.RESP_Open({ level: message.level }),
+							postMessageFn
+						);
+						return;
+					}
+
+					const m =
+						message.level.length === 0
+							? this._rootStore
+							: this._levels.get(memory.levelKey(message.level));
+					if (!m) {
+						throw new Error("Recieved memory message for an undefined level");
+					} else if (message instanceof memory.REQ_Clear) {
+						await m.clear();
+						await this.respond(
+							message,
+							new memory.RESP_Clear({ level: message.level }),
+							postMessageFn
+						);
+					} else if (message instanceof memory.REQ_Close) {
+						console.log("CLOSE > ", message.level)
+						await m.close();
+						await this.respond(
+							message,
+							new memory.RESP_Close({ level: message.level }),
+							postMessageFn
+						);
+					} else if (message instanceof memory.REQ_Del) {
+						await m.del(message.key);
+						await this.respond(
+							message,
+							new memory.RESP_Del({ level: message.level }),
+							postMessageFn
+						);
+					} else if (message instanceof memory.REQ_Iterator_Next) {
+						let iterator = this._memoryIterator.get(message.id);
+						if (!iterator) {
+							iterator = m.iterator()[Symbol.asyncIterator]();
+							this._memoryIterator.set(message.id, iterator);
+						}
+						const next = await iterator.next();
+						await this.respond(
+							message,
+							new memory.RESP_Iterator_Next({
+								keys: next.done ? [] : [(next.value as any)[0]],
+								values: next.done ? [] : [new Uint8Array((next.value as any)[1])],
+								level: message.level
+							}),
+							postMessageFn
+						);
+						if (next.done) {
+							this._memoryIterator.delete(message.id);
+						}
+					} else if (message instanceof memory.REQ_Iterator_Stop) {
+						this._memoryIterator.delete(message.id);
+						await this.respond(
+							message,
+							new memory.RESP_Iterator_Stop({ level: message.level }),
+							postMessageFn
+						);
+					} else if (message instanceof memory.REQ_Get) {
+						const value = await m.get(message.key);
+						await this.respond(
+							message,
+							new memory.RESP_Get({
+								bytes: value ? new Uint8Array(value) : undefined,
+								level: message.level
+							}),
+							postMessageFn
+						);
+					} else if (message instanceof memory.REQ_Put) {
+						await m.put(message.key, message.bytes);
+						await this.respond(
+							message,
+							new memory.RESP_Put({ level: message.level }),
+							postMessageFn
+						);
+					} else if (message instanceof memory.REQ_Size) {
+						await this.respond(
+							message,
+							new memory.RESP_Size({
+								size: await m.size(),
+								level: message.level
+							}),
+							postMessageFn
+						);
+					} else if (message instanceof memory.REQ_Status) {
+						await this.respond(
+							message,
+							new memory.RESP_Status({
+								status: await m.status(),
+								level: message.level
+							}),
+							postMessageFn
+						);
+					} else if (message instanceof memory.REQ_Sublevel) {
+						await m.sublevel(message.name);
+
+						await this.respond(
+							message,
+							new memory.RESP_Sublevel({ level: message.level }),
+							postMessageFn
+						);
+					}
+				}
+			} catch (error) {
+				await this.respond(
+					message,
+					new memory.RESP_Error({
+						error: (error as any)?.["message"] || error?.constructor.name,
+						level: (message as any)["level"] || []
+					}),
+					postMessageFn
+				);
+			}
+		});
+	}
+
+	async respond(
+		request: memory.MemoryRequest,
+		response: memory.MemoryRequest,
+		postMessageFn = postMessage
+	) {
+		response.messageId = request.messageId;
+		const bytes = serialize(response);
+		postMessageFn(bytes, { transfer: [bytes.buffer] });
+	}
+}
+
+new OPFSStoreWorker();