@wovin/storage-fs 0.0.25 → 0.1.0

package/README.md

@@ -6,11 +6,11 @@ Persists wovin threads (applogs/blocks) to the local filesystem.
 
 ## Features
 
-- **StorageConnector** interface - Store CAR files in LMDB
-- **RetrievalConnector** interface - Retrieve CAR files by CID
-- **Single file storage** - Stores all blocks in one LMDB database file
+- **Applog storage** - Store and retrieve applogs by CID
+- **Thread management** - Track which applogs belong to which threads (many-to-many)
+- **Block storage** - Store raw bytes (for binary content referenced by applogs)
+- **Deduplication** - Applogs stored once, referenced by multiple threads
 - **ACID transactions** - Atomic writes via LMDB transactions
-- **Built on LMDB** - Leverages Lightning Memory-Mapped Database for reliability
 
 ## Installation
 
@@ -20,150 +20,150 @@ pnpm add @wovin/storage-fs
 
 ## Usage
 
-### Basic Storage
+### Basic Example
 
 ```typescript
-import { LmdbConnector } from '@wovin/storage-fs'
-import { makeCarBlob, encodeBlockOriginal } from '@wovin/core/ipfs'
+import { StorageFs } from '@wovin/storage-fs'
+import type { Applog } from '@wovin/core/applog'
 
-// Initialize connector
-const connector = await LmdbConnector.init('./data/wovin-storage')
+// Initialize storage
+const storage = await StorageFs.init('./data/wovin-storage')
 
-// Create and store a block
-const testData = { hello: 'world', ts: Date.now() }
-const block = await encodeBlockOriginal(testData)
-const carBlob = await makeCarBlob(block.cid, [block])
+// Create applogs (CID must be pre-computed)
+const applogs: Applog[] = [
+  { cid: 'bafyrei...', en: 'entity-1', at: 'text', vl: 'Hello', ts: '2024-01-01T00:00:00Z', ag: 'alice', pv: null },
+  { cid: 'bafyrei...', en: 'entity-1', at: 'text', vl: 'World', ts: '2024-01-01T00:00:01Z', ag: 'alice', pv: null },
+]
 
-// Store the CAR file
-const rootCid = await connector.storeCar(carBlob)
-console.log('Stored with CID:', rootCid.toString())
+// Store applogs in a thread
+const stored = await storage.storeApplogs('my-thread', applogs)
+console.log(`Stored ${stored} new applogs`)
 
-// Verify block exists
-console.log('Block exists:', connector.has(rootCid))
+// Retrieve applogs from thread
+const retrieved = storage.getApplogs('my-thread')
 
-// Retrieve the block
-const retrievedBytes = connector.get(rootCid)
+// Get specific applog by CID
+const applog = storage.getApplogByCid('bafyrei...')
 
-// Retrieve as CAR
-const car = await connector.retrieveCar(rootCid)
+// Check membership
+storage.isApplogInThread('my-thread', 'bafyrei...')  // true
 
 // Clean up
-connector.close()
+storage.close()
 ```
 
-### Thread Snapshots
+### Multi-Thread Example
+
+Applogs are deduplicated - the same applog can exist in multiple threads:
 
 ```typescript
-import { LmdbConnector } from '@wovin/storage-fs'
-import { makeCarBlob, encodeBlock } from '@wovin/core/ipfs'
-import { ThreadInMemory } from '@wovin/core/thread'
+// Store in first thread
+await storage.storeApplogs('thread-a', [applog1, applog2])
 
-const connector = await LmdbConnector.init('./data/wovin-storage')
+// Store same applogs in second thread (no duplication)
+await storage.storeApplogs('thread-b', [applog1, applog2])
 
-// Create applogs
-const applogs = [
-  { ag: 'alice', en: 'thread-123', at: 'text', vl: 'Hello' },
-  { ag: 'alice', en: 'thread-123', at: 'text', vl: 'World' },
-]
+// Applog exists in both threads
+storage.isApplogInThread('thread-a', applog1.cid)  // true
+storage.isApplogInThread('thread-b', applog1.cid)  // true
 
-// Encode as blocks
-const blocks = applogs.map(applog => encodeBlock(applog))
+// Delete thread-b (applogs remain, still in thread-a)
+await storage.deleteThread('thread-b')
+storage.hasApplog(applog1.cid)  // still true
+```
+
+### Block Storage
 
-// Create thread snapshot CAR
-const snapshotRoot = blocks[0]
-const snapshotCar = await makeCarBlob(snapshotRoot.cid, blocks)
+For binary content referenced by applogs:
 
-// Store thread snapshot
-const snapshotCid = await connector.storeCar(snapshotCar)
+```typescript
+// Store a block
+await storage.storeBlock('bafyrei...', new Uint8Array([1, 2, 3, 4]))
 
-// Later: retrieve and restore thread
-const retrievedCar = await connector.retrieveCar(snapshotCid)
-// Use CarReader to decode applogs...
+// Retrieve block
+const bytes = storage.getBlock('bafyrei...')
 
-connector.close()
+// Check existence
+storage.hasBlock('bafyrei...')
 ```
 
 ## API
 
-### `LmdbConnector.init(path: string): Promise<LmdbConnector>`
+### Initialization
 
-Initialize the connector with an LMDB database at the specified path.
+#### `StorageFs.init(path: string): Promise<StorageFs>`
 
-**Parameters:**
-- `path` - Filesystem path where LMDB data will be stored
+Initialize storage at the specified path.
 
-**Returns:** Promise resolving to LmdbConnector instance
+### Applog Methods
 
-### `storeCar(car: Blob): Promise<CID>`
+#### `storeApplogs(threadId: string, applogs: Applog[]): Promise<number>`
 
-Store a CAR (Content Addressable aRchive) file. Extracts all blocks and stores them individually in LMDB.
+Store applogs in a thread. Returns count of newly stored applogs (excludes duplicates).
 
-**Parameters:**
-- `car` - CAR file as Blob
+#### `getApplogs(threadId: string): Applog[]`
 
-**Returns:** Promise resolving to the root CID
+Get all applogs in a thread.
 
-### `retrieveCar(cid: CID): Promise<CarReader>`
+#### `getApplogByCid(cid: CID | string): Applog | null`
 
-Retrieve a CAR file by CID. Reconstructs the CAR from stored blocks.
+Get a specific applog by CID.
 
-**Parameters:**
-- `cid` - The CID to retrieve
+#### `hasApplog(cid: CID | string): boolean`
 
-**Returns:** Promise resolving to CarReader
+Check if an applog exists.
 
-### `get(cid: CID): Uint8Array | undefined`
+### Block Methods
 
-Get a single block's bytes by CID.
+#### `storeBlock(cid: CID | string, bytes: Uint8Array): Promise<void>`
 
-**Parameters:**
-- `cid` - The CID to retrieve
+Store raw bytes by CID.
 
-**Returns:** Block bytes or undefined if not found
+#### `getBlock(cid: CID | string): Uint8Array | null`
 
-### `has(cid: CID): boolean`
+Get block bytes by CID.
 
-Check if a block exists in the store.
+#### `hasBlock(cid: CID | string): boolean`
 
-**Parameters:**
-- `cid` - The CID to check
+Check if a block exists.
 
-**Returns:** boolean
+### Thread Methods
 
-### `close(): void`
+#### `listThreads(): string[]`
 
-Close the LMDB database and clean up resources.
+List all thread IDs.
 
-## Storage Backend: LMDB
+#### `deleteThread(threadId: string): Promise<void>`
 
-This package uses [lmdb-js](https://github.com/kriszyp/lmdb-js) as the storage backend:
+Delete a thread (removes membership, keeps applogs).
 
-- **Memory-mapped** - OS-level memory mapping for efficient data access
-- **ACID** - Atomic, consistent, isolated, durable transactions
-- **Single file** - Stores entire database in one file
-- **Cross-process** - Shared memory allows multiple processes to access simultaneously
-- **Compression** - Optional compression for small values
+#### `getThreadApplogCount(threadId: string): number`
 
-## When to Use
+Get count of applogs in a thread.
 
-- ✅ Local development environments
-- ✅ Desktop/Electron apps
-- ✅ Node.js servers with persistent storage
-- ✅ Testing and integration tests
-- ✅ Offline-first applications
+#### `isApplogInThread(threadId: string, cid: CID | string): boolean`
 
-## When NOT to Use
+Check if an applog is in a thread.
 
-- ❌ Browser/WASM environments (use `@wovin/connect-web3storage` instead)
-- ❌ Cloud-only deployments (use `@wovin/connect-nftstorage` instead)
-- ❌ Multi-machine clusters (use `@wovin/connect-s3` instead)
+### Utilities
+
+#### `close(): void`
+
+Close the storage and release resources.
+
+## LMDB Structure
+
+```
+storage-fs/data.mdb
+├── applogs        # cidString → Applog JSON
+├── thread_index   # threadId → [cidString...] (dupSort)
+└── blocks         # cidString → Uint8Array
+```
 
 ## Related Packages
 
 - [`@wovin/core`](../core) - Core wovin types and utilities
-- [`@wovin/connect-nftstorage`](../connect-nftstorage) - Cloud storage via NFT.storage
-- [`@wovin/connect-s3`](../connect-s3) - Cloud storage via AWS S3
-- [`@wovin/connect-web3storage`](../connect-web3storage) - Cloud storage via Web3.storage
+- [`@wovin/daemon-utils`](../daemon-utils) - Daemon utilities
 
 ## License
 

package/dist/index.d.ts

@@ -1,14 +1,24 @@
-import { CarReader } from '@ipld/car';
-import type { StorageConnector, RetrievalConnector } from '@wovin/core/pubsub';
+import type { Applog, CidString } from '@wovin/core/applog';
 import { CID } from 'multiformats/cid';
-export declare class LmdbConnector implements StorageConnector, RetrievalConnector {
-    private db;
-    static init(path: string): Promise<LmdbConnector>;
+export declare class StorageFs {
+    private root;
+    private applogs;
+    private threadIndex;
+    private blocks;
+    static init(path: string): Promise<StorageFs>;
     private constructor();
-    storeCar(car: Blob): Promise<CID>;
-    retrieveCar(cid: CID): Promise<CarReader>;
-    get(cid: CID): Uint8Array | undefined;
-    has(cid: CID): boolean;
+    storeApplogs(threadId: string, applogs: Applog[]): Promise<number>;
+    getApplogs(threadId: string): Applog[];
+    getApplogByCid(cid: CID | CidString): Applog | null;
+    hasApplog(cid: CID | CidString): boolean;
+    storeBlock(cid: CID | CidString, bytes: Uint8Array): Promise<void>;
+    getBlock(cid: CID | CidString): Uint8Array | null;
+    hasBlock(cid: CID | CidString): boolean;
+    listThreads(): string[];
+    deleteThread(threadId: string): Promise<void>;
+    getThreadApplogCount(threadId: string): number;
+    isApplogInThread(threadId: string, cid: CID | CidString): boolean;
     close(): void;
 }
+export { StorageFs as LmdbConnector };
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAa,MAAM,WAAW,CAAA;AAChD,OAAO,KAAK,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAA;AAE9E,OAAO,EAAE,GAAG,EAAE,MAAM,kBAAkB,CAAA;AAItC,qBAAa,aAAc,YAAW,gBAAgB,EAAE,kBAAkB;IACzE,OAAO,CAAC,EAAE,CAA8B;WAE3B,IAAI,CAAC,IAAI,EAAE,MAAM;IAQ9B,OAAO;IAID,QAAQ,CAAC,GAAG,EAAE,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC;IAwBjC,WAAW,CAAC,GAAG,EAAE,GAAG,GAAG,OAAO,CAAC,SAAS,CAAC;IAyB/C,GAAG,CAAC,GAAG,EAAE,GAAG,GAAG,UAAU,GAAG,SAAS;IAIrC,GAAG,CAAC,GAAG,EAAE,GAAG,GAAG,OAAO;IAItB,KAAK;CAGL"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAA;AAE3D,OAAO,EAAE,GAAG,EAAE,MAAM,kBAAkB,CAAA;AAItC,qBAAa,SAAS;IACrB,OAAO,CAAC,IAAI,CAAc;IAC1B,OAAO,CAAC,OAAO,CAA0B;IACzC,OAAO,CAAC,WAAW,CAA0B;IAC7C,OAAO,CAAC,MAAM,CAA8B;WAE/B,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,CAAC;IAsBnD,OAAO;IAcD,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC;IA8BxE,UAAU,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,EAAE;IActC,cAAc,CAAC,GAAG,EAAE,GAAG,GAAG,SAAS,GAAG,MAAM,GAAG,IAAI;IAOnD,SAAS,CAAC,GAAG,EAAE,GAAG,GAAG,SAAS,GAAG,OAAO;IAOlC,UAAU,CAAC,GAAG,EAAE,GAAG,GAAG,SAAS,EAAE,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAKxE,QAAQ,CAAC,GAAG,EAAE,GAAG,GAAG,SAAS,GAAG,UAAU,GAAG,IAAI;IAMjD,QAAQ,CAAC,GAAG,EAAE,GAAG,GAAG,SAAS,GAAG,OAAO;IAOvC,WAAW,IAAI,MAAM,EAAE;IASjB,YAAY,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAmBnD,oBAAoB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM;IAI9C,gBAAgB,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,EAAE,GAAG,GAAG,SAAS,GAAG,OAAO;IAKjE,KAAK,IAAI,IAAI;CAGb;AAGD,OAAO,EAAE,SAAS,IAAI,aAAa,EAAE,CAAA"}

package/dist/index.min.js

@@ -1,2 +1,2 @@
-import{open as g}from"lmdb";import{CarReader as b,CarWriter as l}from"@ipld/car";import{Logger as y}from"besonders-logger";var{DEBUG:p,VERBOSE:A,ERROR:h}=y.setup(y.INFO),f=class u{db;static async init(t){let o=g({path:t,compression:!0});return new u(o)}constructor(t){this.db=t}async storeCar(t){let o=new Uint8Array(await t.arrayBuffer()),i=await b.fromBytes(o),a=(await i.getRoots())[0],e=[];for await(let{cid:s,bytes:n}of i.blocks()){let r=s.toString();e.push({cidStr:r,bytes:n})}return await this.db.transaction(()=>{for(let{cidStr:s,bytes:n}of e)this.db.put(s,n)}),p("[LmdbConnector] Stored CAR with root:",a.toString()),a}async retrieveCar(t){let o=[],i=this.db.get(t.toString());if(!i)throw h(`Block not found: ${t.toString()}`);o.push({cid:t,bytes:i});let{writer:c,out:a}=l.create([t]);for(let r of o)c.put(r);c.close();let e=[];for await(let r of a)e.push(r);let s=new Uint8Array(e.reduce((r,d)=>r+d.length,0)),n=0;for(let r of e)s.set(r,n),n+=r.length;return b.fromBytes(s)}get(t){return this.db.get(t.toString())}has(t){return this.db.doesExist(t.toString())}close(){this.db.close()}};export{f as LmdbConnector};
+import{open as p}from"lmdb";import{Logger as e}from"besonders-logger";var{DEBUG:i,VERBOSE:u,ERROR:S}=e.setup(e.INFO),a=class g{root;applogs;threadIndex;blocks;static async init(t){let s=p({path:t,compression:!0}),r=s.openDB({name:"applogs"}),n=s.openDB({name:"thread_index",dupSort:!0}),o=s.openDB({name:"blocks"});return new g(s,r,n,o)}constructor(t,s,r,n){this.root=t,this.applogs=s,this.threadIndex=r,this.blocks=n}async storeApplogs(t,s){let r=0;return await this.root.transaction(()=>{for(let n of s){let o=n.cid;if(this.applogs.doesExist(o)){this.threadIndex.doesExist(t,o)||this.threadIndex.put(t,o);continue}this.applogs.put(o,JSON.stringify(n)),this.threadIndex.put(t,o),r++}}),i(`[StorageFs] Stored ${r} applogs in thread "${t}"`),r}getApplogs(t){let s=[],r=this.threadIndex.getValues(t);for(let n of r){let o=this.applogs.get(n);o&&s.push(JSON.parse(o))}return s}getApplogByCid(t){let s=typeof t=="string"?t:t.toString(),r=this.applogs.get(s);return r?JSON.parse(r):null}hasApplog(t){let s=typeof t=="string"?t:t.toString();return this.applogs.doesExist(s)}async storeBlock(t,s){let r=typeof t=="string"?t:t.toString();await this.blocks.put(r,s)}getBlock(t){let s=typeof t=="string"?t:t.toString();return this.blocks.get(s)??null}hasBlock(t){let s=typeof t=="string"?t:t.toString();return this.blocks.doesExist(s)}listThreads(){let t=new Set;for(let{key:s}of this.threadIndex.getRange({}))t.add(s);return Array.from(t)}async deleteThread(t){await this.root.transaction(()=>{let s=Array.from(this.threadIndex.getValues(t));for(let r of s)this.threadIndex.remove(t,r)}),i(`[StorageFs] Deleted thread "${t}"`)}getThreadApplogCount(t){return this.threadIndex.getValuesCount(t)}isApplogInThread(t,s){let r=typeof s=="string"?s:s.toString();return this.threadIndex.doesExist(t,r)}close(){this.root.close()}};export{a as LmdbConnector,a as StorageFs};
 //# sourceMappingURL=index.min.js.map

package/dist/index.min.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../src/index.ts"],"sourcesContent":["import { open, Database } from 'lmdb'\nimport { CarReader, CarWriter } from '@ipld/car'\nimport type { StorageConnector, RetrievalConnector } from '@wovin/core/pubsub'\nimport { Logger } from 'besonders-logger'\nimport { CID } from 'multiformats/cid'\n\nconst { DEBUG, VERBOSE, ERROR } = Logger.setup(Logger.INFO)\n\nexport class LmdbConnector implements StorageConnector, RetrievalConnector {\n\tprivate db: Database<Uint8Array, string>\n\n\tstatic async init(path: string) {\n\t\tconst db = open<Uint8Array, string>({\n\t\t\tpath,\n\t\t\tcompression: true,\n\t\t})\n\t\treturn new LmdbConnector(db)\n\t}\n\n\tprivate constructor(db: Database<Uint8Array, string>) {\n\t\tthis.db = db\n\t}\n\n\tasync storeCar(car: Blob): Promise<CID> {\n\t\tconst bytes = new Uint8Array(await car.arrayBuffer())\n\t\tconst reader = await CarReader.fromBytes(bytes)\n\t\tconst roots = await reader.getRoots()\n\t\tconst rootCid = roots[0]\n\n\t\t// Collect blocks first (async operation)\n\t\tconst blocks: Array<{ cidStr: string; bytes: Uint8Array }> = []\n\t\tfor await (const { cid, bytes: blockBytes } of reader.blocks()) {\n\t\t\tconst cidStr = cid.toString()\n\t\t\tblocks.push({ cidStr, bytes: blockBytes })\n\t\t}\n\n\t\t// Store each block individually in a transaction\n\t\tawait this.db.transaction(() => {\n\t\t\tfor (const { cidStr, bytes: blockBytes } of blocks) {\n\t\t\t\tthis.db.put(cidStr, blockBytes)\n\t\t\t}\n\t\t})\n\n\t\tDEBUG(`[LmdbConnector] Stored CAR with root:`, rootCid.toString())\n\t\treturn rootCid\n\t}\n\n\tasync retrieveCar(cid: CID): Promise<CarReader> {\n\t\t// Collect all blocks starting from root\n\t\tconst blocks: Array<{ cid: CID; bytes: Uint8Array }> = []\n\t\tconst bytes = this.db.get(cid.toString())\n\t\tif (!bytes) throw ERROR(`Block not found: ${cid.toString()}`)\n\t\tblocks.push({ cid, bytes })\n\n\t\t// Reconstruct CAR\n\t\tconst { writer, out } = CarWriter.create([cid])\n\t\tfor (const block of blocks) {\n\t\t\twriter.put(block)\n\t\t}\n\t\twriter.close()\n\n\t\tconst chunks: Uint8Array[] = []\n\t\tfor await (const chunk of out) chunks.push(chunk)\n\t\tconst combined = new Uint8Array(chunks.reduce((a, c) => a + c.length, 0))\n\t\tlet offset = 0\n\t\tfor (const chunk of chunks) {\n\t\t\tcombined.set(chunk, offset)\n\t\t\toffset += chunk.length\n\t\t}\n\t\treturn CarReader.fromBytes(combined)\n\t}\n\n\tget(cid: CID): Uint8Array | undefined {\n\t\treturn this.db.get(cid.toString())\n\t}\n\n\thas(cid: CID): boolean {\n\t\treturn this.db.doesExist(cid.toString())\n\t}\n\n\tclose() {\n\t\tthis.db.close()\n\t}\n}\n"],"mappings":"AAAA,OAAS,QAAAA,MAAsB,OAC/B,OAAS,aAAAC,EAAW,aAAAC,MAAiB,YAErC,OAAS,UAAAC,MAAc,mBAGvB,GAAM,CAAE,MAAAC,EAAO,QAAAC,EAAS,MAAAC,CAAM,EAAIH,EAAO,MAAMA,EAAO,IAAI,EAE7CI,EAAN,MAAMC,CAA8D,CAClE,GAER,aAAa,KAAKC,EAAc,CAC/B,IAAMC,EAAKV,EAAyB,CACnC,KAAAS,EACA,YAAa,EACd,CAAC,EACD,OAAO,IAAID,EAAcE,CAAE,CAC5B,CAEQ,YAAYA,EAAkC,CACrD,KAAK,GAAKA,CACX,CAEA,MAAM,SAASC,EAAyB,CACvC,IAAMC,EAAQ,IAAI,WAAW,MAAMD,EAAI,YAAY,CAAC,EAC9CE,EAAS,MAAMZ,EAAU,UAAUW,CAAK,EAExCE,GADQ,MAAMD,EAAO,SAAS,GACd,CAAC,EAGjBE,EAAuD,CAAC,EAC9D,aAAiB,CAAE,IAAAC,EAAK,MAAOC,CAAW,IAAKJ,EAAO,OAAO,EAAG,CAC/D,IAAMK,EAASF,EAAI,SAAS,EAC5BD,EAAO,KAAK,CAAE,OAAAG,EAAQ,MAAOD,CAAW,CAAC,CAC1C,CAGA,aAAM,KAAK,GAAG,YAAY,IAAM,CAC/B,OAAW,CAAE,OAAAC,EAAQ,MAAOD,CAAW,IAAKF,EAC3C,KAAK,GAAG,IAAIG,EAAQD,CAAU,CAEhC,CAAC,EAEDb,EAAM,wCAAyCU,EAAQ,SAAS,CAAC,EAC1DA,CACR,CAEA,MAAM,YAAYE,EAA8B,CAE/C,IAAMD,EAAiD,CAAC,EAClDH,EAAQ,KAAK,GAAG,IAAII,EAAI,SAAS,CAAC,EACxC,GAAI,CAACJ,EAAO,MAAMN,EAAM,oBAAoBU,EAAI,SAAS,CAAC,EAAE,EAC5DD,EAAO,KAAK,CAAE,IAAAC,EAAK,MAAAJ,CAAM,CAAC,EAG1B,GAAM,CAAE,OAAAO,EAAQ,IAAAC,CAAI,EAAIlB,EAAU,OAAO,CAACc,CAAG,CAAC,EAC9C,QAAWK,KAASN,EACnBI,EAAO,IAAIE,CAAK,EAEjBF,EAAO,MAAM,EAEb,IAAMG,EAAuB,CAAC,EAC9B,cAAiBC,KAASH,EAAKE,EAAO,KAAKC,CAAK,EAChD,IAAMC,EAAW,IAAI,WAAWF,EAAO,OAAO,CAACG,EAAGC,IAAMD,EAAIC,EAAE,OAAQ,CAAC,CAAC,EACpEC,EAAS,EACb,QAAWJ,KAASD,EACnBE,EAAS,IAAID,EAAOI,CAAM,EAC1BA,GAAUJ,EAAM,OAEjB,OAAOtB,EAAU,UAAUuB,CAAQ,CACpC,CAEA,IAAIR,EAAkC,CACrC,OAAO,KAAK,GAAG,IAAIA,EAAI,SAAS,CAAC,CAClC,CAEA,IAAIA,EAAmB,CACtB,OAAO,KAAK,GAAG,UAAUA,EAAI,SAAS,CAAC,CACxC,CAEA,OAAQ,CACP,KAAK,GAAG,MAAM,CACf,CACD","names":["open","CarReader","CarWriter","Logger","DEBUG","VERBOSE","ERROR","LmdbConnector","_LmdbConnector","path","db","car","bytes","reader","rootCid","blocks","cid","blockBytes","cidStr","writer","out","block","chunks","chunk","combined","a","c","offset"]}
+{"version":3,"sources":["../src/index.ts"],"sourcesContent":["import { open, RootDatabase, Database } from 'lmdb'\nimport type { Applog, CidString } from '@wovin/core/applog'\nimport { Logger } from 'besonders-logger'\nimport { CID } from 'multiformats/cid'\n\nconst { DEBUG, VERBOSE, ERROR } = Logger.setup(Logger.INFO)\n\nexport class StorageFs {\n\tprivate root: RootDatabase\n\tprivate applogs: Database<string, string>      // cidString → Applog JSON\n\tprivate threadIndex: Database<string, string>  // threadId → cidString (dupSort)\n\tprivate blocks: Database<Uint8Array, string>   // cidString → bytes\n\n\tstatic async init(path: string): Promise<StorageFs> {\n\t\tconst root = open({\n\t\t\tpath,\n\t\t\tcompression: true,\n\t\t})\n\n\t\tconst applogs = root.openDB<string, string>({\n\t\t\tname: 'applogs',\n\t\t})\n\n\t\tconst threadIndex = root.openDB<string, string>({\n\t\t\tname: 'thread_index',\n\t\t\tdupSort: true,\n\t\t})\n\n\t\tconst blocks = root.openDB<Uint8Array, string>({\n\t\t\tname: 'blocks',\n\t\t})\n\n\t\treturn new StorageFs(root, applogs, threadIndex, blocks)\n\t}\n\n\tprivate constructor(\n\t\troot: RootDatabase,\n\t\tapplogs: Database<string, string>,\n\t\tthreadIndex: Database<string, string>,\n\t\tblocks: Database<Uint8Array, string>,\n\t) {\n\t\tthis.root = root\n\t\tthis.applogs = applogs\n\t\tthis.threadIndex = threadIndex\n\t\tthis.blocks = blocks\n\t}\n\n\t// === Applogs ===\n\n\tasync storeApplogs(threadId: string, applogs: Applog[]): Promise<number> {\n\t\tlet stored = 0\n\n\t\tawait this.root.transaction(() => {\n\t\t\tfor (const applog of applogs) {\n\t\t\t\tconst cidStr = applog.cid\n\n\t\t\t\t// Check if already stored (dedup by CID)\n\t\t\t\tif (this.applogs.doesExist(cidStr)) {\n\t\t\t\t\t// Applog exists, but still add to thread index if not already there\n\t\t\t\t\tif (!this.threadIndex.doesExist(threadId, cidStr)) {\n\t\t\t\t\t\tthis.threadIndex.put(threadId, cidStr)\n\t\t\t\t\t}\n\t\t\t\t\tcontinue\n\t\t\t\t}\n\n\t\t\t\t// Store applog\n\t\t\t\tthis.applogs.put(cidStr, JSON.stringify(applog))\n\n\t\t\t\t// Add to thread index\n\t\t\t\tthis.threadIndex.put(threadId, cidStr)\n\n\t\t\t\tstored++\n\t\t\t}\n\t\t})\n\n\t\tDEBUG(`[StorageFs] Stored ${stored} applogs in thread \"${threadId}\"`)\n\t\treturn stored\n\t}\n\n\tgetApplogs(threadId: string): Applog[] {\n\t\tconst applogs: Applog[] = []\n\t\tconst cidStrings = this.threadIndex.getValues(threadId)\n\n\t\tfor (const cidStr of cidStrings) {\n\t\t\tconst json = this.applogs.get(cidStr)\n\t\t\tif (json) {\n\t\t\t\tapplogs.push(JSON.parse(json))\n\t\t\t}\n\t\t}\n\n\t\treturn applogs\n\t}\n\n\tgetApplogByCid(cid: CID | CidString): Applog | null {\n\t\tconst cidStr = typeof cid === 'string' ? cid : cid.toString()\n\t\tconst json = this.applogs.get(cidStr)\n\t\tif (!json) return null\n\t\treturn JSON.parse(json)\n\t}\n\n\thasApplog(cid: CID | CidString): boolean {\n\t\tconst cidStr = typeof cid === 'string' ? cid : cid.toString()\n\t\treturn this.applogs.doesExist(cidStr)\n\t}\n\n\t// === Blocks ===\n\n\tasync storeBlock(cid: CID | CidString, bytes: Uint8Array): Promise<void> {\n\t\tconst cidStr = typeof cid === 'string' ? cid : cid.toString()\n\t\tawait this.blocks.put(cidStr, bytes)\n\t}\n\n\tgetBlock(cid: CID | CidString): Uint8Array | null {\n\t\tconst cidStr = typeof cid === 'string' ? cid : cid.toString()\n\t\tconst bytes = this.blocks.get(cidStr)\n\t\treturn bytes ?? null\n\t}\n\n\thasBlock(cid: CID | CidString): boolean {\n\t\tconst cidStr = typeof cid === 'string' ? cid : cid.toString()\n\t\treturn this.blocks.doesExist(cidStr)\n\t}\n\n\t// === Threads ===\n\n\tlistThreads(): string[] {\n\t\tconst threads = new Set<string>()\n\t\t// Iterate over all keys in thread_index\n\t\tfor (const { key } of this.threadIndex.getRange({})) {\n\t\t\tthreads.add(key)\n\t\t}\n\t\treturn Array.from(threads)\n\t}\n\n\tasync deleteThread(threadId: string): Promise<void> {\n\t\tawait this.root.transaction(() => {\n\t\t\t// Get all CIDs in this thread\n\t\t\tconst cidStrings = Array.from(this.threadIndex.getValues(threadId))\n\n\t\t\t// Remove all entries for this thread from index\n\t\t\tfor (const cidStr of cidStrings) {\n\t\t\t\tthis.threadIndex.remove(threadId, cidStr)\n\t\t\t}\n\n\t\t\t// Note: We don't delete the applogs themselves - they may be in other threads\n\t\t\t// Orphan cleanup could be a separate maintenance operation\n\t\t})\n\n\t\tDEBUG(`[StorageFs] Deleted thread \"${threadId}\"`)\n\t}\n\n\t// === Utilities ===\n\n\tgetThreadApplogCount(threadId: string): number {\n\t\treturn this.threadIndex.getValuesCount(threadId)\n\t}\n\n\tisApplogInThread(threadId: string, cid: CID | CidString): boolean {\n\t\tconst cidStr = typeof cid === 'string' ? cid : cid.toString()\n\t\treturn this.threadIndex.doesExist(threadId, cidStr)\n\t}\n\n\tclose(): void {\n\t\tthis.root.close()\n\t}\n}\n\n// Re-export for backwards compatibility during transition\nexport { StorageFs as LmdbConnector }\n"],"mappings":"AAAA,OAAS,QAAAA,MAAoC,OAE7C,OAAS,UAAAC,MAAc,mBAGvB,GAAM,CAAE,MAAAC,EAAO,QAAAC,EAAS,MAAAC,CAAM,EAAIH,EAAO,MAAMA,EAAO,IAAI,EAE7CI,EAAN,MAAMC,CAAU,CACd,KACA,QACA,YACA,OAER,aAAa,KAAKC,EAAkC,CACnD,IAAMC,EAAOR,EAAK,CACjB,KAAAO,EACA,YAAa,EACd,CAAC,EAEKE,EAAUD,EAAK,OAAuB,CAC3C,KAAM,SACP,CAAC,EAEKE,EAAcF,EAAK,OAAuB,CAC/C,KAAM,eACN,QAAS,EACV,CAAC,EAEKG,EAASH,EAAK,OAA2B,CAC9C,KAAM,QACP,CAAC,EAED,OAAO,IAAIF,EAAUE,EAAMC,EAASC,EAAaC,CAAM,CACxD,CAEQ,YACPH,EACAC,EACAC,EACAC,EACC,CACD,KAAK,KAAOH,EACZ,KAAK,QAAUC,EACf,KAAK,YAAcC,EACnB,KAAK,OAASC,CACf,CAIA,MAAM,aAAaC,EAAkBH,EAAoC,CACxE,IAAII,EAAS,EAEb,aAAM,KAAK,KAAK,YAAY,IAAM,CACjC,QAAWC,KAAUL,EAAS,CAC7B,IAAMM,EAASD,EAAO,IAGtB,GAAI,KAAK,QAAQ,UAAUC,CAAM,EAAG,CAE9B,KAAK,YAAY,UAAUH,EAAUG,CAAM,GAC/C,KAAK,YAAY,IAAIH,EAAUG,CAAM,EAEtC,QACD,CAGA,KAAK,QAAQ,IAAIA,EAAQ,KAAK,UAAUD,CAAM,CAAC,EAG/C,KAAK,YAAY,IAAIF,EAAUG,CAAM,EAErCF,GACD,CACD,CAAC,EAEDX,EAAM,sBAAsBW,CAAM,uBAAuBD,CAAQ,GAAG,EAC7DC,CACR,CAEA,WAAWD,EAA4B,CACtC,IAAMH,EAAoB,CAAC,EACrBO,EAAa,KAAK,YAAY,UAAUJ,CAAQ,EAEtD,QAAWG,KAAUC,EAAY,CAChC,IAAMC,EAAO,KAAK,QAAQ,IAAIF,CAAM,EAChCE,GACHR,EAAQ,KAAK,KAAK,MAAMQ,CAAI,CAAC,CAE/B,CAEA,OAAOR,CACR,CAEA,eAAeS,EAAqC,CACnD,IAAMH,EAAS,OAAOG,GAAQ,SAAWA,EAAMA,EAAI,SAAS,EACtDD,EAAO,KAAK,QAAQ,IAAIF,CAAM,EACpC,OAAKE,EACE,KAAK,MAAMA,CAAI,EADJ,IAEnB,CAEA,UAAUC,EAA+B,CACxC,IAAMH,EAAS,OAAOG,GAAQ,SAAWA,EAAMA,EAAI,SAAS,EAC5D,OAAO,KAAK,QAAQ,UAAUH,CAAM,CACrC,CAIA,MAAM,WAAWG,EAAsBC,EAAkC,CACxE,IAAMJ,EAAS,OAAOG,GAAQ,SAAWA,EAAMA,EAAI,SAAS,EAC5D,MAAM,KAAK,OAAO,IAAIH,EAAQI,CAAK,CACpC,CAEA,SAASD,EAAyC,CACjD,IAAMH,EAAS,OAAOG,GAAQ,SAAWA,EAAMA,EAAI,SAAS,EAE5D,OADc,KAAK,OAAO,IAAIH,CAAM,GACpB,IACjB,CAEA,SAASG,EAA+B,CACvC,IAAMH,EAAS,OAAOG,GAAQ,SAAWA,EAAMA,EAAI,SAAS,EAC5D,OAAO,KAAK,OAAO,UAAUH,CAAM,CACpC,CAIA,aAAwB,CACvB,IAAMK,EAAU,IAAI,IAEpB,OAAW,CAAE,IAAAC,CAAI,IAAK,KAAK,YAAY,SAAS,CAAC,CAAC,EACjDD,EAAQ,IAAIC,CAAG,EAEhB,OAAO,MAAM,KAAKD,CAAO,CAC1B,CAEA,MAAM,aAAaR,EAAiC,CACnD,MAAM,KAAK,KAAK,YAAY,IAAM,CAEjC,IAAMI,EAAa,MAAM,KAAK,KAAK,YAAY,UAAUJ,CAAQ,CAAC,EAGlE,QAAWG,KAAUC,EACpB,KAAK,YAAY,OAAOJ,EAAUG,CAAM,CAK1C,CAAC,EAEDb,EAAM,+BAA+BU,CAAQ,GAAG,CACjD,CAIA,qBAAqBA,EAA0B,CAC9C,OAAO,KAAK,YAAY,eAAeA,CAAQ,CAChD,CAEA,iBAAiBA,EAAkBM,EAA+B,CACjE,IAAMH,EAAS,OAAOG,GAAQ,SAAWA,EAAMA,EAAI,SAAS,EAC5D,OAAO,KAAK,YAAY,UAAUN,EAAUG,CAAM,CACnD,CAEA,OAAc,CACb,KAAK,KAAK,MAAM,CACjB,CACD","names":["open","Logger","DEBUG","VERBOSE","ERROR","StorageFs","_StorageFs","path","root","applogs","threadIndex","blocks","threadId","stored","applog","cidStr","cidStrings","json","cid","bytes","threads","key"]}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wovin/storage-fs",
-  "version": "0.0.25",
+  "version": "0.1.0",
   "type": "module",
   "main": "./dist/index.min.js",
   "module": "./dist/index.min.js",
@@ -23,7 +23,7 @@
   },
   "dependencies": {
     "@ipld/car": "^5.2.6",
-    "@wovin/core": "0.0.25",
+    "@wovin/core": "0.1.0",
     "besonders-logger": "1.0.1",
     "lmdb": "^3.0.0",
     "multiformats": "^13.0.1"
@@ -32,9 +32,15 @@
     "concurrently": "^8.2.2",
     "esbuild-plugin-polyfill-node": "^0.3.0",
     "tsup": "^8.0.2",
+    "tsx": "^4.19.2",
     "typescript": "^5.8.3",
     "tsupconfig": "^0.0.0"
   },
+  "repository": {
+    "type": "git",
+    "url": "git+https://gitlab.com/wovin/wovin.git",
+    "directory": "packages/@wovin/storage-fs"
+  },
   "scripts": {
     "build": "rm -rf dist/ && concurrently \"pnpm build:code\" \"pnpm build:types\"",
     "build:code": "tsup --minify --external esbuild",
@@ -42,6 +48,7 @@
     "dev": "concurrently \"pnpm dev:code\" \"pnpm dev:types\"",
     "dev:code": "tsup --watch",
     "dev:types": "tsc --emitDeclarationOnly --declaration --watch",
+    "test": "tsx test.ts",
     "lint": "eslint .",
     "clean": "rm -rf .turbo && rm -rf node_modules && rm -rf dist",
     "pub": "npm publish --tag latest --access=public"