@wovin/storage-fs 0.1.36 → 0.2.0

package/dist/{index.min.js → index.js}

@@ -1,2 +1,2 @@
 import{open as S}from"lmdb";import{Logger as c}from"besonders-logger";var{DEBUG:s,VERBOSE:f,ERROR:h}=c.setup(c.INFO),l=class d{root;applogs;threadIndex;blocks;static async init(t){s(`[StorageFs.init] ENTER - path=${t}`),s("[StorageFs.init] Opening LMDB root database");let o=S({path:t,compression:!0});s("[StorageFs.init] LMDB root database opened"),s("[StorageFs.init] Opening applogs DB");let r=o.openDB({name:"applogs"});s("[StorageFs.init] Applogs DB opened"),s("[StorageFs.init] Opening threadIndex DB");let e=o.openDB({name:"thread_index",dupSort:!0});s("[StorageFs.init] ThreadIndex DB opened"),s("[StorageFs.init] Opening blocks DB");let n=o.openDB({name:"blocks"});s("[StorageFs.init] Blocks DB opened"),s("[StorageFs.init] Creating StorageFs instance");let p=new d(o,r,e,n);return s("[StorageFs.init] EXIT - StorageFs instance created successfully"),p}constructor(t,o,r,e){this.root=t,this.applogs=o,this.threadIndex=r,this.blocks=e}async storeApplogs(t,o){let r=Date.now();s("[StorageFs.storeApplogs] ===== ENTER ====="),s(`[StorageFs.storeApplogs] threadId: ${t}`),s(`[StorageFs.storeApplogs] applogs.length: ${o.length}`);let e=0;s("[StorageFs.storeApplogs] About to start transaction");let n=Date.now();try{this.root.transactionSync(()=>{for(let g of o){let i=g.cid;if(this.applogs.doesExist(i)){this.threadIndex.doesExist(t,i)||this.threadIndex.put(t,i);continue}this.applogs.put(i,JSON.stringify(g)),this.threadIndex.put(t,i),e++}});let a=Date.now()-n;s(`[StorageFs.storeApplogs] Transaction completed in ${a}ms`)}catch(a){let g=Date.now()-n;throw h(`[StorageFs.storeApplogs] Transaction error after ${g}ms:`,a),a}let p=Date.now()-r;return s(`[StorageFs.storeApplogs] ===== SUCCESS in ${p}ms, stored ${e} applogs`),e}getApplogs(t){let o=[],r=this.threadIndex.getValues(t);for(let e of r){let n=this.applogs.get(e);n&&o.push(JSON.parse(n))}return o}getApplogByCid(t){let o=typeof t=="string"?t:t.toString(),r=this.applogs.get(o);return r?JSON.parse(r):null}hasApplog(t){let o=typeof t=="string"?t:t.toString();return this.applogs.doesExist(o)}async get(t){let o=this.blocks.get(t.toString());if(!o)throw new Error(`Block not found: ${t}`);return o}async put(t,o){await this.blocks.put(t.toString(),o)}async has(t){return this.blocks.doesExist(t.toString())}listThreads(){let t=new Set;for(let{key:o}of this.threadIndex.getRange({}))t.add(o);return Array.from(t)}async deleteThread(t){this.root.transactionSync(()=>{let o=Array.from(this.threadIndex.getValues(t));for(let r of o)this.threadIndex.remove(t,r)}),s(`[StorageFs] Deleted thread "${t}"`)}getThreadApplogCount(t){return this.threadIndex.getValuesCount(t)}isApplogInThread(t,o){let r=typeof o=="string"?o:o.toString();return this.threadIndex.doesExist(t,r)}close(){this.root.close()}};export{l as LmdbConnector,l as StorageFs};
-//# sourceMappingURL=index.min.js.map
+//# sourceMappingURL=index.js.map

package/package.json

@@ -1,19 +1,20 @@
 {
   "name": "@wovin/storage-fs",
-  "version": "0.1.36",
+  "version": "0.2.0",
   "type": "module",
-  "main": "./dist/index.min.js",
-  "module": "./dist/index.min.js",
-  "browser": "./dist/index.min.js",
+  "main": "./dist/index.js",
+  "module": "./dist/index.js",
+  "browser": "./dist/index.js",
   "types": "./dist/index.d.ts",
   "exports": {
     ".": {
-      "import": "./dist/index.min.js",
+      "import": "./dist/index.js",
       "types": "./dist/index.d.ts"
     }
   },
   "files": [
-    "./dist/"
+    "./dist/",
+    "./src/"
   ],
   "publishConfig": {
     "access": "public"
@@ -26,7 +27,7 @@
     "besonders-logger": "1.0.1",
     "lmdb": "^3.0.0",
     "multiformats": "^13.0.1",
-    "@wovin/core": "^0.1.36"
+    "@wovin/core": "^0.2.0"
   },
   "devDependencies": {
     "concurrently": "^8.2.2",

package/src/index.ts

@@ -0,0 +1,210 @@
+import { open, RootDatabase, Database } from 'lmdb'
+import type { Applog, CidString } from '@wovin/core'
+import type { BlockStore } from '@wovin/core/blockstore'
+import { Logger } from 'besonders-logger'
+import { CID } from 'multiformats/cid'
+
+const { DEBUG, VERBOSE, ERROR } = Logger.setup(Logger.INFO)
+
+/**
+ * File-system based storage using LMDB.
+ * Implements BlockStore for use with wovin block caching and snapshot chain walking.
+ */
+export class StorageFs implements BlockStore {
+	private root: RootDatabase
+	private applogs: Database<string, string>      // cidString → Applog JSON
+	private threadIndex: Database<string, string>  // threadId → cidString (dupSort)
+	private blocks: Database<Uint8Array, string>   // cidString → bytes
+
+	static async init(path: string): Promise<StorageFs> {
+		DEBUG(`[StorageFs.init] ENTER - path=${path}`)
+
+		DEBUG(`[StorageFs.init] Opening LMDB root database`)
+		const root = open({
+			path,
+			compression: true,
+		})
+		DEBUG(`[StorageFs.init] LMDB root database opened`)
+
+		DEBUG(`[StorageFs.init] Opening applogs DB`)
+		const applogs = root.openDB<string, string>({
+			name: 'applogs',
+		})
+		DEBUG(`[StorageFs.init] Applogs DB opened`)
+
+		DEBUG(`[StorageFs.init] Opening threadIndex DB`)
+		const threadIndex = root.openDB<string, string>({
+			name: 'thread_index',
+			dupSort: true,
+		})
+		DEBUG(`[StorageFs.init] ThreadIndex DB opened`)
+
+		DEBUG(`[StorageFs.init] Opening blocks DB`)
+		const blocks = root.openDB<Uint8Array, string>({
+			name: 'blocks',
+		})
+		DEBUG(`[StorageFs.init] Blocks DB opened`)
+
+		DEBUG(`[StorageFs.init] Creating StorageFs instance`)
+		const instance = new StorageFs(root, applogs, threadIndex, blocks)
+		DEBUG(`[StorageFs.init] EXIT - StorageFs instance created successfully`)
+		return instance
+	}
+
+	private constructor(
+		root: RootDatabase,
+		applogs: Database<string, string>,
+		threadIndex: Database<string, string>,
+		blocks: Database<Uint8Array, string>,
+	) {
+		this.root = root
+		this.applogs = applogs
+		this.threadIndex = threadIndex
+		this.blocks = blocks
+	}
+
+	// === Applogs ===
+
+	async storeApplogs(threadId: string, applogs: Applog[]): Promise<number> {
+		const fnStart = Date.now()
+		DEBUG(`[StorageFs.storeApplogs] ===== ENTER =====`)
+		DEBUG(`[StorageFs.storeApplogs] threadId: ${threadId}`)
+		DEBUG(`[StorageFs.storeApplogs] applogs.length: ${applogs.length}`)
+		let stored = 0
+
+		DEBUG(`[StorageFs.storeApplogs] About to start transaction`)
+		const txStart = Date.now()
+
+		try {
+			// CRITICAL FIX: Use transactionSync instead of transaction
+			// Async transactions (transaction method) are NOT supported in Deno's NAPI
+			// Reference: https://github.com/kriszyp/lmdb-js#usage
+			// "Deno and Bun's support for NAPI is not very stable yet, and currently
+			//  asynchronous transactions (transaction method) are not supported"
+
+			// NOTE: No logging inside transaction - LMDB transactions must be fast and synchronous
+			this.root.transactionSync(() => {
+				for (const applog of applogs) {
+					const cidStr = applog.cid
+
+					// Check if already stored (dedup by CID)
+					if (this.applogs.doesExist(cidStr)) {
+						// Applog exists, but still add to thread index if not already there
+						if (!this.threadIndex.doesExist(threadId, cidStr)) {
+							this.threadIndex.put(threadId, cidStr)
+						}
+						continue
+					}
+
+					// Store applog
+					this.applogs.put(cidStr, JSON.stringify(applog))
+
+					// Add to thread index
+					this.threadIndex.put(threadId, cidStr)
+
+					stored++
+				}
+			})
+
+			const txDuration = Date.now() - txStart
+			DEBUG(`[StorageFs.storeApplogs] Transaction completed in ${txDuration}ms`)
+		} catch (err) {
+			const txDuration = Date.now() - txStart
+			ERROR(`[StorageFs.storeApplogs] Transaction error after ${txDuration}ms:`, err)
+			throw err
+		}
+
+		const fnDuration = Date.now() - fnStart
+		DEBUG(`[StorageFs.storeApplogs] ===== SUCCESS in ${fnDuration}ms, stored ${stored} applogs`)
+		return stored
+	}
+
+	getApplogs(threadId: string): Applog[] {
+		const applogs: Applog[] = []
+		const cidStrings = this.threadIndex.getValues(threadId)
+
+		for (const cidStr of cidStrings) {
+			const json = this.applogs.get(cidStr)
+			if (json) {
+				applogs.push(JSON.parse(json))
+			}
+		}
+
+		return applogs
+	}
+
+	getApplogByCid(cid: CID | CidString): Applog | null {
+		const cidStr = typeof cid === 'string' ? cid : cid.toString()
+		const json = this.applogs.get(cidStr)
+		if (!json) return null
+		return JSON.parse(json)
+	}
+
+	hasApplog(cid: CID | CidString): boolean {
+		const cidStr = typeof cid === 'string' ? cid : cid.toString()
+		return this.applogs.doesExist(cidStr)
+	}
+
+	// === Blocks (BlockStore interface) ===
+
+	async get(cid: CID): Promise<Uint8Array> {
+		const bytes = this.blocks.get(cid.toString())
+		if (!bytes) throw new Error(`Block not found: ${cid}`)
+		return bytes
+	}
+
+	async put(cid: CID, bytes: Uint8Array): Promise<void> {
+		await this.blocks.put(cid.toString(), bytes)
+	}
+
+	async has(cid: CID): Promise<boolean> {
+		return this.blocks.doesExist(cid.toString())
+	}
+
+	// === Threads ===
+
+	listThreads(): string[] {
+		const threads = new Set<string>()
+		// Iterate over all keys in thread_index
+		for (const { key } of this.threadIndex.getRange({})) {
+			threads.add(key)
+		}
+		return Array.from(threads)
+	}
+
+	async deleteThread(threadId: string): Promise<void> {
+		// Use transactionSync for Deno compatibility (async transactions not supported in Deno NAPI)
+		this.root.transactionSync(() => {
+			// Get all CIDs in this thread
+			const cidStrings = Array.from(this.threadIndex.getValues(threadId))
+
+			// Remove all entries for this thread from index
+			for (const cidStr of cidStrings) {
+				this.threadIndex.remove(threadId, cidStr)
+			}
+
+			// Note: We don't delete the applogs themselves - they may be in other threads
+			// Orphan cleanup could be a separate maintenance operation
+		})
+
+		DEBUG(`[StorageFs] Deleted thread "${threadId}"`)
+	}
+
+	// === Utilities ===
+
+	getThreadApplogCount(threadId: string): number {
+		return this.threadIndex.getValuesCount(threadId)
+	}
+
+	isApplogInThread(threadId: string, cid: CID | CidString): boolean {
+		const cidStr = typeof cid === 'string' ? cid : cid.toString()
+		return this.threadIndex.doesExist(threadId, cidStr)
+	}
+
+	close(): void {
+		this.root.close()
+	}
+}
+
+// Re-export for backwards compatibility during transition
+export { StorageFs as LmdbConnector }