@peerbit/any-store 1.0.1

package/src/opfs-worker.ts

@@ -0,0 +1,244 @@
+import { deserialize, serialize } from "@dao-xyz/borsh";
+import { AnyStore } from "./interface.js";
+import * as memory from "./opfs-worker-messages.js";
+
+export class OPFSStoreWorker {
+	level: string[];
+	private _levels: Map<string, AnyStore>;
+	private _rootStore: AnyStore;
+
+	private _memoryIterator: Map<
+		string,
+		AsyncIterator<[string, ArrayBuffer], void, void>
+	>;
+
+	constructor() {
+		const postMessageFn = postMessage;
+		this._memoryIterator = new Map();
+		this._levels = new Map();
+		const createMemory = (
+			root?: FileSystemDirectoryHandle,
+			level: string[] = []
+		): AnyStore => {
+			let isOpen = false;
+
+			let m: FileSystemDirectoryHandle = root!;
+
+			// 'open' | 'closed' is just a virtual thing since OPFS is always open as soone as we get the FileSystemDirectoryHandle
+			// TODO remove status? or assume not storage adapters can be closed?
+			const open = async () => {
+				isOpen = true;
+				m = m || (await navigator.storage.getDirectory());
+			};
+
+			return {
+				clear: async () => {
+					for await (const key of m.keys()) {
+						m.removeEntry(key, { recursive: true });
+					}
+				},
+
+				del: async (key) => {
+					try {
+						await m.removeEntry(key, { recursive: true });
+					} catch (error) {
+						if (
+							error instanceof DOMException &&
+							error.name === "NotFoundError"
+						) {
+							return;
+						} else {
+							throw error;
+						}
+					}
+				},
+
+				get: async (key) => {
+					try {
+						const fileHandle = await m.getFileHandle(key);
+						const buffer = await (await fileHandle.getFile()).arrayBuffer();
+						return new Uint8Array(buffer);
+					} catch (error) {
+						if (
+							error instanceof DOMException &&
+							error.name === "NotFoundError"
+						) {
+							return;
+						} else {
+							throw error;
+						}
+					}
+				},
+				put: async (key, value) => {
+					const fileHandle = await m.getFileHandle(key, { create: true });
+					const writeFileHandle = await fileHandle.createSyncAccessHandle();
+
+					writeFileHandle.write(value);
+					writeFileHandle.close();
+				},
+
+				size: async () => {
+					let size = 0;
+					for await (const value of m.values()) {
+						if (value.kind === "file") {
+							const handle = await value.createSyncAccessHandle();
+							size += handle.getSize();
+							handle.close();
+						}
+					}
+					return size;
+				},
+				status: () => (isOpen ? "open" : "closed"),
+
+				sublevel: async (name) => {
+					const fileHandle = await m.getDirectoryHandle(name, { create: true });
+					const sublevel = [...level, name];
+					const subMemory = createMemory(fileHandle, sublevel);
+					this._levels.set(memory.levelKey(sublevel), subMemory);
+					await subMemory.open();
+					return subMemory;
+				},
+
+				async *iterator(): AsyncGenerator<[string, Uint8Array], void, void> {
+					for await (const v of m.values()) {
+						if (v.kind == "file") {
+							yield [
+								v.name,
+								new Uint8Array(await (await v.getFile()).arrayBuffer())
+							];
+						}
+					}
+				},
+				close: async () => {
+					isOpen = false;
+					this._memoryIterator.clear();
+				},
+				open
+			};
+		};
+
+		this._rootStore = createMemory();
+
+		self.addEventListener("message", async (ev) => {
+			const message = deserialize(ev["data"], memory.MemoryRequest);
+			if (message instanceof memory.MemoryMessage) {
+				const m =
+					message.level.length === 0
+						? this._rootStore
+						: this._levels.get(memory.levelKey(message.level));
+				if (!m) {
+					throw new Error("Recieved memory message for an undefined level");
+				} else if (message instanceof memory.REQ_Clear) {
+					await m.clear();
+					await this.respond(
+						message,
+						new memory.RESP_Clear({ level: message.level }),
+						postMessageFn
+					);
+				} else if (message instanceof memory.REQ_Close) {
+					await m.close();
+					await this.respond(
+						message,
+						new memory.RESP_Close({ level: message.level }),
+						postMessageFn
+					);
+				} else if (message instanceof memory.REQ_Del) {
+					await m.del(message.key);
+					await this.respond(
+						message,
+						new memory.RESP_Del({ level: message.level }),
+						postMessageFn
+					);
+				} else if (message instanceof memory.REQ_Iterator_Next) {
+					let iterator = this._memoryIterator.get(message.id);
+					if (!iterator) {
+						iterator = m.iterator()[Symbol.asyncIterator]();
+						this._memoryIterator.set(message.id, iterator);
+					}
+					const next = await iterator.next();
+					await this.respond(
+						message,
+						new memory.RESP_Iterator_Next({
+							keys: next.done ? [] : [next.value[0]],
+							values: next.done ? [] : [new Uint8Array(next.value[1])],
+							level: message.level
+						}),
+						postMessageFn
+					);
+					if (next.done) {
+						this._memoryIterator.delete(message.id);
+					}
+				} else if (message instanceof memory.REQ_Iterator_Stop) {
+					this._memoryIterator.delete(message.id);
+					await this.respond(
+						message,
+						new memory.RESP_Iterator_Stop({ level: message.level }),
+						postMessageFn
+					);
+				} else if (message instanceof memory.REQ_Get) {
+					const value = await m.get(message.key);
+					await this.respond(
+						message,
+						new memory.RESP_Get({
+							bytes: value ? new Uint8Array(value) : undefined,
+							level: message.level
+						}),
+						postMessageFn
+					);
+				} else if (message instanceof memory.REQ_Open) {
+					await m.open();
+					await this.respond(
+						message,
+						new memory.RESP_Open({ level: message.level }),
+						postMessageFn
+					);
+				} else if (message instanceof memory.REQ_Put) {
+					await m.put(message.key, message.bytes);
+					await this.respond(
+						message,
+						new memory.RESP_Put({ level: message.level }),
+						postMessageFn
+					);
+				} else if (message instanceof memory.REQ_Size) {
+					await this.respond(
+						message,
+						new memory.RESP_Size({
+							size: await m.size(),
+							level: message.level
+						}),
+						postMessageFn
+					);
+				} else if (message instanceof memory.REQ_Status) {
+					await this.respond(
+						message,
+						new memory.RESP_Status({
+							status: await m.status(),
+							level: message.level
+						}),
+						postMessageFn
+					);
+				} else if (message instanceof memory.REQ_Sublevel) {
+					await m.sublevel(message.name);
+
+					await this.respond(
+						message,
+						new memory.RESP_Sublevel({ level: message.level }),
+						postMessageFn
+					);
+				}
+			}
+		});
+	}
+
+	async respond(
+		request: memory.MemoryRequest,
+		response: memory.MemoryRequest,
+		postMessageFn = postMessage
+	) {
+		response.messageId = request.messageId;
+		const bytes = serialize(response);
+		postMessageFn(bytes, { transfer: [bytes.buffer] });
+	}
+}
+
+new OPFSStoreWorker();

package/src/opfs.ts

@@ -0,0 +1,201 @@
+import { AnyStore, MaybePromise } from "./interface.js";
+import * as memory from "./opfs-worker-messages.js";
+import { v4 as uuid } from "uuid";
+import { serialize, deserialize } from "@dao-xyz/borsh";
+
+function memoryIterator(
+	client: {
+		request<T extends memory.MemoryRequest>(
+			request: memory.MemoryRequest
+		): Promise<T>;
+	},
+	level: string[]
+): {
+	[Symbol.asyncIterator]: () => AsyncIterator<[string, Uint8Array], void, void>;
+} {
+	return {
+		[Symbol.asyncIterator]() {
+			const iteratorId = uuid();
+			return {
+				next: async () => {
+					const resp = await client.request<memory.RESP_Iterator_Next>(
+						new memory.REQ_Iterator_Next({ id: iteratorId, level })
+					);
+					if (resp.keys.length > 1) {
+						throw new Error("Unsupported iteration response");
+					}
+					// Will only have 0 or 1 element for now
+					for (let i = 0; i < resp.keys.length; i++) {
+						return {
+							done: false,
+							value: [resp.keys[i], resp.values[i]] as [string, Uint8Array]
+						} as { done: false; value: [string, Uint8Array] };
+					}
+					return { done: true, value: undefined } as {
+						done: true;
+						value: undefined;
+					};
+				},
+				async return() {
+					await client.request<memory.RESP_Iterator_Next>(
+						new memory.REQ_Iterator_Stop({ id: iteratorId, level })
+					);
+					return { done: true, value: undefined } as {
+						done: true;
+						value: undefined;
+					};
+				}
+			};
+		}
+	};
+}
+
+const workerURL = new URL("./opfs-worker.js", import.meta.url);
+
+/* new Worker(workerURL, { type: 'module' }) */
+const createWorker = () => new Worker(workerURL, { type: "module" });
+export class OPFSStore implements AnyStore {
+	worker: Worker;
+	level: string[];
+	levelMap: Map<string, AnyStore>;
+	root: AnyStore;
+
+	private _responseCallbacks: Map<
+		string,
+		{ fn: (message: memory.MemoryRequest) => any; once: boolean }
+	> = new Map();
+
+	private _createMemory: (level: string[]) => AnyStore;
+	constructor(level: string[] = []) {
+		this.level = level;
+		this.levelMap = new Map();
+		this._createMemory = (level: string[] = []): AnyStore => {
+			return {
+				clear: async () => {
+					await this.request<memory.RESP_Clear>(
+						new memory.REQ_Clear({ level })
+					);
+				},
+				del: async (key) => {
+					await this.request<memory.RESP_Del>(
+						new memory.REQ_Del({ level, key })
+					);
+				},
+				get: async (key) => {
+					return (
+						await this.request<memory.RESP_Get>(
+							new memory.REQ_Get({ level, key })
+						)
+					).bytes;
+				},
+				put: async (key, value) => {
+					await this.request<memory.RESP_Put>(
+						new memory.REQ_Put({ level, key, bytes: value })
+					);
+				},
+				status: async () =>
+					(
+						await this.request<memory.RESP_Status>(
+							new memory.REQ_Status({ level })
+						)
+					).status,
+				sublevel: async (name) => {
+					await this.request<memory.RESP_Sublevel>(
+						new memory.REQ_Sublevel({ level, name })
+					);
+					const newLevels = [...level, name];
+					const sublevel = this._createMemory(newLevels);
+					this.levelMap.set(memory.levelKey(newLevels), sublevel);
+					return sublevel;
+				},
+
+				iterator: () => memoryIterator(this, level),
+				close: async () => {
+					await this.request<memory.RESP_Close>(
+						new memory.REQ_Close({ level })
+					);
+					/*     this.levelMap.delete(memory.levelKey(level)); */
+				},
+				open: async () => {
+					await this.request<memory.RESP_Open>(new memory.REQ_Open({ level }));
+				},
+
+				size: async () => {
+					const size = await this.request<memory.RESP_Size>(
+						new memory.REQ_Size({ level })
+					);
+					return size.size;
+				}
+			};
+		};
+	}
+	status() {
+		return this.worker ? this.root.status() : "closed";
+	}
+	async close(): Promise<void> {
+		this.worker.terminate();
+		this.worker = undefined!;
+		this._responseCallbacks.clear();
+		this.levelMap.clear();
+	}
+	async open(): Promise<void> {
+		if (!this.worker) {
+			this.root = this._createMemory([]);
+			this.worker = createWorker();
+			this.worker.addEventListener("message", async (ev) => {
+				const message = deserialize(ev.data, memory.MemoryMessage);
+				this._responseCallbacks.get(message.messageId)!.fn(message);
+			});
+			await this.root.open();
+		}
+	}
+	async get(key: string): Promise<Uint8Array | undefined> {
+		return this.root.get(key);
+	}
+	async put(key: string, value: Uint8Array) {
+		return this.root.put(key, value);
+	}
+	del(key: any): MaybePromise<void> {
+		return this.root.del(key);
+	}
+	sublevel(name: string): AnyStore | Promise<AnyStore> {
+		return this.root.sublevel(name);
+	}
+	iterator(): {
+		[Symbol.asyncIterator]: () => AsyncIterator<
+			[string, Uint8Array],
+			void,
+			void
+		>;
+	} {
+		return this.root.iterator();
+	}
+	clear(): MaybePromise<void> {
+		return this.root.clear();
+	}
+
+	size(): MaybePromise<number> {
+		return this.root.size();
+	}
+
+	async request<T extends memory.MemoryRequest>(
+		request: memory.MemoryRequest
+	): Promise<T> {
+		return new Promise<T>((resolve, reject) => {
+			const onResponse = (message: memory.MemoryRequest) => {
+				this._responseCallbacks.delete(request.messageId);
+				if (message instanceof memory.RESP_Error) {
+					reject(message.error);
+				} else {
+					resolve(message as T);
+				}
+			};
+			this._responseCallbacks.set(request.messageId, {
+				fn: onResponse,
+				once: true
+			});
+			const bytes = serialize(request);
+			this.worker.postMessage(bytes, [bytes.buffer]);
+		});
+	}
+}

package/src/store.browser.ts

@@ -0,0 +1,10 @@
+import { MemoryStore } from "./memory.js";
+import { OPFSStore } from "./opfs.js";
+
+export const createStore = (directory?: string) => {
+	return directory ? new OPFSStore([directory]) : new MemoryStore();
+};
+
+/* export const estimate = (directory: string): Promise<{ quota?: number, usage?: number }> => {
+	return navigator.storage.estimate().then(x => { return { quota: x.quota, usage: x.usage } })
+} */

package/src/store.ts

@@ -0,0 +1,15 @@
+import { LevelStore } from "./level.js";
+import { Level } from "level";
+import { MemoryStore } from "./memory.js";
+/* import os from 'os'
+import { check } from 'diskusage' */
+
+export const createStore = (directory?: string) => {
+	return directory
+		? new LevelStore(new Level(directory, { valueEncoding: "view" }))
+		: new MemoryStore();
+};
+
+/* export const estimate = (directory: string): Promise<{ quota?: number, usage?: number }> => {
+	return check(directory).then(x => { return { quota: x.total, usage: x.total - x.free } })
+} */